Cache Implementation

- Agency
- Officer
- Unit
- Location

Author: DMalone87

backend/README.md

@@ -49,3 +49,52 @@ To authenticate your API requests, include the JWT token in the `Authorization`
 ```
 Authorization: Bearer <your_token>
 ```
+
+## Data Retrieval Benchmarking
+
+### Unit Cache Benchmarking
+
+There is a small benchmarking script at [backend/tasks/benchmark_unit_cache.py](/Users/darrellmalone/Sync/Clients/NPDC/code/police-data-trust/backend/tasks/benchmark_unit_cache.py:1) for measuring unit-heavy endpoints.
+
+Example using an existing token:
+```bash
+python3 backend/tasks/benchmark_unit_cache.py \
+  --base-url http://localhost:5000 \
+  --token "$ACCESS_TOKEN" \
+  --unit-id <unit-uid-1> \
+  --unit-id <unit-uid-2> \
+  --search-term "precinct 1" \
+  --search-term "investigations" \
+  --output-json /tmp/unit-cache-before.json
+```
+
+Example logging in directly:
+```bash
+python3 backend/tasks/benchmark_unit_cache.py \
+  --base-url http://localhost:5000 \
+  --email you@example.com \
+  --password "your-password" \
+  --unit-id <unit-uid> \
+  --search-term "precinct 1"
+```
+
+To compare an after-run against a saved baseline:
+```bash
+python3 backend/tasks/benchmark_unit_cache.py \
+  --base-url http://localhost:5000 \
+  --token "$ACCESS_TOKEN" \
+  --unit-id <unit-uid> \
+  --search-term "precinct 1" \
+  --compare-json /tmp/unit-cache-before.json \
+  --output-json /tmp/unit-cache-after.json
+```
+
+For Neo4j query-plan profiling, there is also [backend/tasks/profile_unit_cache_queries.py](/Users/darrellmalone/Sync/Clients/NPDC/code/police-data-trust/backend/tasks/profile_unit_cache_queries.py:1). It runs `PROFILE` for the current cached query shape and a legacy traversal version of the same unit-related workload, then prints DB-hit and timing deltas.
+
+Example:
+```bash
+python3 backend/tasks/profile_unit_cache_queries.py \
+  --unit-id <unit-uid-1> \
+  --unit-id <unit-uid-2> \
+  --output-json /tmp/unit-query-profile.json
+```

backend/api.py

@@ -190,6 +190,36 @@ def refresh_location_cache():
         result = LocationCacheService().refresh_location_richness_cache()
         click.echo(result)
 
+    @app.cli.command("refresh-agency-cache")
+    def refresh_agency_cache():
+        """Refresh cached agency metric fields."""
+        from backend.services.agency_cache_service import (
+            AgencyCacheService,
+        )
+
+        result = AgencyCacheService().refresh_agency_metrics_cache()
+        click.echo(result)
+
+    @app.cli.command("refresh-unit-cache")
+    def refresh_unit_cache():
+        """Refresh cached unit metric fields."""
+        from backend.services.unit_cache_service import (
+            UnitCacheService,
+        )
+
+        result = UnitCacheService().refresh_unit_metrics_cache()
+        click.echo(result)
+
+    @app.cli.command("refresh-officer-cache")
+    def refresh_officer_cache():
+        """Refresh cached officer metric fields."""
+        from backend.services.officer_cache_service import (
+            OfficerCacheService,
+        )
+
+        result = OfficerCacheService().refresh_officer_metrics_cache()
+        click.echo(result)
+
 
 def register_routes(app: Flask):
     app.register_blueprint(sources_bp)

backend/database/models/agency.py

@@ -7,6 +7,8 @@
 from neomodel import (
     db,
     StructuredNode,
+    DateTimeNeo4jFormatProperty,
+    IntegerProperty,
     StringProperty,
     RelationshipTo,
     UniqueIdProperty,
@@ -44,7 +46,11 @@ class Unit(StructuredNode, HasCitations, JsonSerializable, SearchableMixin):
         "hq_address", "hq_city", "hq_state", "hq_zip",
         "agency", "date_established"
     ]
-    __hidden_properties__ = ["citations", "city_node"]
+    __hidden_properties__ = [
+        "citations", "city_node",
+        "officer_count_cached", "complaint_count_cached",
+        "allegation_count_cached", "metrics_updated_at",
+    ]
 
     uid = UniqueIdProperty()
     name = StringProperty()
@@ -57,6 +63,10 @@ class Unit(StructuredNode, HasCitations, JsonSerializable, SearchableMixin):
     website_url = StringProperty()
     description = StringProperty()
     date_established = DateNeo4jFormatProperty()
+    officer_count_cached = IntegerProperty(default=0, index=True)
+    complaint_count_cached = IntegerProperty(default=0, index=True)
+    allegation_count_cached = IntegerProperty(default=0, index=True)
+    metrics_updated_at = DateTimeNeo4jFormatProperty()
 
     # Relationships
     agency = RelationshipTo("Agency", "ESTABLISHED_BY", cardinality=One)
@@ -86,14 +96,7 @@ def total_officers(self):
         Returns:
             int: The total number of officers.
         """
-        cy = """
-        MATCH (u:Unit {uid: $uid})-[]-(:Employment)-[]-(o:Officer)
-        RETURN COUNT(o) AS total_officers
-        """
-        result, meta = db.cypher_query(cy, {'uid': self.uid})
-        if result:
-            return result[0][0]
-        return 0
+        return self.officer_count_cached or 0
 
     @property
     def current_commander(self):
@@ -156,8 +159,12 @@ class Agency(StructuredNode, HasCitations, JsonSerializable, SearchableMixin):
         "hq_address", "hq_city", "hq_state", "hq_zip",
         "jurisdiction", "date_established"
     ]
-    __hidden_properties__ = ["citations", "state_node",
-                             "county_node", "city_node"]
+    __hidden_properties__ = [
+        "citations", "state_node", "county_node", "city_node",
+        "unit_count_cached", "officer_count_cached",
+        "complaint_count_cached", "allegation_count_cached",
+        "metrics_updated_at",
+    ]
     __virtual_relationships__ = ["units"]
 
     uid = UniqueIdProperty()
@@ -172,6 +179,11 @@ class Agency(StructuredNode, HasCitations, JsonSerializable, SearchableMixin):
     description = StringProperty()
     date_established = DateNeo4jFormatProperty()
     jurisdiction = StringProperty(choices=Jurisdiction.choices())
+    unit_count_cached = IntegerProperty(default=0, index=True)
+    officer_count_cached = IntegerProperty(default=0, index=True)
+    complaint_count_cached = IntegerProperty(default=0, index=True)
+    allegation_count_cached = IntegerProperty(default=0, index=True)
+    metrics_updated_at = DateTimeNeo4jFormatProperty()
 
     # Relationships
     city_node = RelationshipTo(

backend/database/models/infra/locations.py

@@ -2,6 +2,7 @@
     StructuredNode,
     StringProperty,
     IntegerProperty,
+    DateTimeNeo4jFormatProperty,
     FloatProperty,
     UniqueIdProperty,
     Relationship,
@@ -10,7 +11,6 @@
     One
 )
 from neomodel.contrib.spatial_properties import PointProperty
-from backend.database.properties.datetime import DateNeo4jFormatProperty
 
 
 STATE_INFO = {
@@ -102,7 +102,7 @@ class CountyNode(Place):
     officer_count_cached = IntegerProperty(default=0, index=True)
     complaint_count_cached = IntegerProperty(default=0, index=True)
     richness_score_cached = FloatProperty(default=0.0, index=True)
-    richness_updated_at = DateNeo4jFormatProperty()
+    richness_updated_at = DateTimeNeo4jFormatProperty()
 
     # Relationships
     state = RelationshipTo("StateNode", "WITHIN_STATE", cardinality=One)
@@ -118,7 +118,7 @@ class CityNode(Place):
     officer_count_cached = IntegerProperty(default=0, index=True)
     complaint_count_cached = IntegerProperty(default=0, index=True)
     richness_score_cached = FloatProperty(default=0.0, index=True)
-    richness_updated_at = DateNeo4jFormatProperty()
+    richness_updated_at = DateTimeNeo4jFormatProperty()
 
     # Relationships
     county = RelationshipTo("CountyNode", "WITHIN_COUNTY", cardinality=One)

backend/database/models/officer.py

@@ -9,6 +9,7 @@
 from neomodel import (
     db, StructuredNode, Relationship,
     StringProperty, IntegerProperty,
+    DateTimeNeo4jFormatProperty,
     UniqueIdProperty, One
 )
 
@@ -40,7 +41,13 @@ class Officer(StructuredNode, HasCitations, JsonSerializable):
         "last_name", "suffix", "ethnicity",
         "gender", "year_of_birth"
     ]
-    __hidden_properties__ = ["citations"]
+    __hidden_properties__ = [
+        "citations",
+        "complaint_count_cached",
+        "allegation_count_cached",
+        "substantiated_count_cached",
+        "metrics_updated_at",
+    ]
     __virtual_relationships__ = ["state_ids"]
 
     uid = UniqueIdProperty()
@@ -51,6 +58,10 @@ class Officer(StructuredNode, HasCitations, JsonSerializable):
     ethnicity = StringProperty(choices=Ethnicity.choices())
     gender = StringProperty(choices=Gender.choices())
     year_of_birth = IntegerProperty()
+    complaint_count_cached = IntegerProperty(default=0, index=True)
+    allegation_count_cached = IntegerProperty(default=0, index=True)
+    substantiated_count_cached = IntegerProperty(default=0, index=True)
+    metrics_updated_at = DateTimeNeo4jFormatProperty()
 
     def __repr__(self):
         return f"<Officer {self.uid}>"

backend/queries/agency_cache.py

@@ -0,0 +1,58 @@
+from neomodel import db
+
+
+AGENCY_METRICS_INPUT_QUERY = """
+MATCH (a:Agency)
+CALL (a) {
+    OPTIONAL MATCH (a)-[:ESTABLISHED_BY]-(u:Unit)
+    RETURN count(DISTINCT u) AS unit_count
+}
+CALL (a) {
+    OPTIONAL MATCH (a)-[:ESTABLISHED_BY]-(:Unit)<-[:IN_UNIT]
+        -(:Employment)-[:HELD_BY]-(o:Officer)
+    RETURN count(DISTINCT o) AS officer_count
+}
+CALL (a) {
+    OPTIONAL MATCH (a)-[:ESTABLISHED_BY]-(:Unit)<-[:IN_UNIT]
+        -(:Employment)-[:HELD_BY]-(:Officer)
+        -[:ACCUSED_OF]->(al:Allegation)-[:ALLEGED]-(c:Complaint)
+    RETURN
+        count(DISTINCT c) AS complaint_count,
+        count(DISTINCT al) AS allegation_count
+}
+RETURN
+    a.uid AS agency_uid,
+    unit_count,
+    officer_count,
+    complaint_count,
+    allegation_count
+ORDER BY agency_uid ASC
+"""
+
+UPDATE_AGENCY_METRICS_CACHE_QUERY = """
+UNWIND $updates AS row
+MATCH (a:Agency {uid: row.agency_uid})
+SET
+    a.unit_count_cached = row.unit_count_cached,
+    a.officer_count_cached = row.officer_count_cached,
+    a.complaint_count_cached = row.complaint_count_cached,
+    a.allegation_count_cached = row.allegation_count_cached,
+    a.metrics_updated_at = datetime(row.metrics_updated_at)
+RETURN count(a) AS updated
+"""
+
+
+class AgencyCacheQueries:
+    def fetch_agency_metrics_inputs(self):
+        rows, _ = db.cypher_query(AGENCY_METRICS_INPUT_QUERY)
+        return rows
+
+    def update_agency_metrics_cache(self, updates: list[dict]) -> int:
+        if not updates:
+            return 0
+
+        rows, _ = db.cypher_query(
+            UPDATE_AGENCY_METRICS_CACHE_QUERY,
+            {"updates": updates},
+        )
+        return rows[0][0] if rows else 0

backend/queries/location_cache.py

@@ -67,7 +67,7 @@
     city.officer_count_cached = row.officer_count_cached,
     city.complaint_count_cached = row.complaint_count_cached,
     city.richness_score_cached = row.richness_score_cached,
-    city.richness_updated_at = date(row.richness_updated_at)
+    city.richness_updated_at = datetime(row.richness_updated_at)
 RETURN count(city) AS updated
 """
 
@@ -79,7 +79,7 @@
     county.officer_count_cached = row.officer_count_cached,
     county.complaint_count_cached = row.complaint_count_cached,
     county.richness_score_cached = row.richness_score_cached,
-    county.richness_updated_at = date(row.richness_updated_at)
+    county.richness_updated_at = datetime(row.richness_updated_at)
 RETURN count(county) AS updated
 """
 

backend/queries/officer_cache.py

@@ -0,0 +1,51 @@
+from neomodel import db
+
+
+OFFICER_METRICS_INPUT_QUERY = """
+MATCH (o:Officer)
+CALL (o) {
+    OPTIONAL MATCH (o)-[:ACCUSED_OF]->(al:Allegation)-[:ALLEGED]->(c:Complaint)
+    RETURN
+        count(DISTINCT c) AS complaint_count,
+        count(DISTINCT al) AS allegation_count,
+        sum(
+            CASE
+                WHEN toLower(trim(coalesce(al.finding, ""))) = "substantiated"
+                THEN 1 ELSE 0
+            END
+        ) AS substantiated_count
+}
+RETURN
+    o.uid AS officer_uid,
+    complaint_count,
+    allegation_count,
+    substantiated_count
+ORDER BY officer_uid ASC
+"""
+
+UPDATE_OFFICER_METRICS_CACHE_QUERY = """
+UNWIND $updates AS row
+MATCH (o:Officer {uid: row.officer_uid})
+SET
+    o.complaint_count_cached = row.complaint_count_cached,
+    o.allegation_count_cached = row.allegation_count_cached,
+    o.substantiated_count_cached = row.substantiated_count_cached,
+    o.metrics_updated_at = datetime(row.metrics_updated_at)
+RETURN count(o) AS updated
+"""
+
+
+class OfficerCacheQueries:
+    def fetch_officer_metrics_inputs(self):
+        rows, _ = db.cypher_query(OFFICER_METRICS_INPUT_QUERY)
+        return rows
+
+    def update_officer_metrics_cache(self, updates: list[dict]) -> int:
+        if not updates:
+            return 0
+
+        rows, _ = db.cypher_query(
+            UPDATE_OFFICER_METRICS_CACHE_QUERY,
+            {"updates": updates},
+        )
+        return rows[0][0] if rows else 0