perf: live-data deltas, payload caps, and map render polish

Cut fast-tier payload cost with zoom-aware sampling, row deltas, CCTV bbox columns, and MapLibre/motion polish; force viewport snapshot refetches so regional pans refill aircraft immediately.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/services/fetchers/_store.py

@@ -146,6 +146,123 @@ source_timestamps = {}
 source_freshness: dict[str, dict] = {}
 
 
+# Layers that support row-level live-data deltas (P2).
+_DELTA_LAYER_KEYS: frozenset[str] = frozenset(
+    {
+        "ships",
+        "commercial_flights",
+        "military_flights",
+        "tracked_flights",
+        "private_flights",
+        "private_jets",
+    }
+)
+# Ring of (layer_version, id→item) snapshots for delta computation.
+_LAYER_ID_RING: dict[str, list[tuple[int, dict[str, Any]]]] = {}
+_LAYER_ID_RING_MAX = 4
+
+
+def entity_id_for_layer(layer: str, item: dict) -> str:
+    """Stable entity id for delta upsert/delete keys."""
+    if not isinstance(item, dict):
+        return ""
+    if layer == "ships":
+        return str(item.get("mmsi") or item.get("id") or "").strip()
+    return str(
+        item.get("icao24") or item.get("icao") or item.get("id") or item.get("hex") or ""
+    ).strip().lower()
+
+
+def _track_fingerprint(item: dict) -> tuple:
+    lng = item.get("lng")
+    if lng is None:
+        lng = item.get("lon")
+    return (
+        item.get("lat"),
+        lng,
+        item.get("heading") or item.get("hdg") or item.get("cog") or item.get("true_track"),
+        item.get("speed_knots") or item.get("sog") or item.get("spd"),
+        item.get("alt") or item.get("altitude") or item.get("alt_km"),
+    )
+
+
+def _capture_layer_id_map(layer: str, items: list) -> dict[str, Any]:
+    out: dict[str, Any] = {}
+    for item in items or []:
+        if not isinstance(item, dict):
+            continue
+        eid = entity_id_for_layer(layer, item)
+        if eid:
+            out[eid] = item
+    return out
+
+
+def _record_delta_layer_snapshot_locked(layer: str) -> None:
+    """Caller must hold ``_data_lock``."""
+    if layer not in _DELTA_LAYER_KEYS:
+        return
+    ver = _layer_versions.get(layer, 0)
+    val = latest_data.get(layer)
+    items = val if isinstance(val, list) else []
+    id_map = _capture_layer_id_map(layer, items)
+    ring = _LAYER_ID_RING.setdefault(layer, [])
+    ring.append((ver, id_map))
+    while len(ring) > _LAYER_ID_RING_MAX:
+        ring.pop(0)
+
+
+def compute_layer_row_delta(layer: str, since_version: int) -> dict[str, Any] | None:
+    """Build upsert/delete delta vs a prior layer version.
+
+    Returns ``None`` when the base version is no longer in the ring (client
+    must take a full snapshot). Returns an empty upsert/delete when unchanged.
+    """
+    if layer not in _DELTA_LAYER_KEYS:
+        return None
+    try:
+        since = int(since_version)
+    except (TypeError, ValueError):
+        return None
+
+    with _data_lock:
+        current_ver = int(_layer_versions.get(layer, 0) or 0)
+        if since == current_ver or since > current_ver:
+            return {
+                "upsert": [],
+                "delete": [],
+                "version": current_ver,
+                "unchanged": True,
+            }
+        ring = list(_LAYER_ID_RING.get(layer) or [])
+        base_map = None
+        for ver, id_map in ring:
+            if ver == since:
+                base_map = id_map
+                break
+        if base_map is None:
+            # Base version aged out of the ring — full snapshot required.
+            return None
+        if ring and ring[-1][0] == current_ver:
+            cur_map = ring[-1][1]
+        else:
+            val = latest_data.get(layer)
+            items = val if isinstance(val, list) else []
+            cur_map = _capture_layer_id_map(layer, items)
+
+    upsert: list[Any] = []
+    for eid, item in cur_map.items():
+        prev = base_map.get(eid)
+        if prev is None or _track_fingerprint(prev) != _track_fingerprint(item):
+            upsert.append(item)
+    delete = [eid for eid in base_map if eid not in cur_map]
+    return {
+        "upsert": upsert,
+        "delete": delete,
+        "version": current_ver,
+        "unchanged": not upsert and not delete,
+    }
+
+
 def _mark_fresh(*keys):
     """Record the current UTC time for one or more data source keys."""
     now = datetime.utcnow().isoformat()
@@ -159,6 +276,7 @@ def _mark_fresh(*keys):
             val = latest_data.get(k)
             count = len(val) if isinstance(val, list) else (1 if val is not None else 0)
             changed.append((k, _layer_versions[k], count))
+            _record_delta_layer_snapshot_locked(k)
         # Publish partial fetch progress immediately so the frontend can
         # observe newly available data without waiting for the entire tier.
         _data_version += 1
@@ -273,7 +391,10 @@ def get_latest_data_subset(*keys: str) -> DashboardData:
 
 
 def get_latest_data_deepcopy_snapshot() -> DashboardData:
-    """Deep-copy the full dashboard for /api/health and legacy /api/live-data.
+    """Deep-copy the full dashboard for callers that need an isolated mutate-safe snap.
+
+    Prefer ``get_latest_data_refs_snapshot`` / ``get_latest_data_subset_refs`` on
+    read-only hot paths (legacy ``/api/live-data`` uses refs + orjson).
 
     The per-value deepcopy runs OUTSIDE ``_data_lock`` so a large clone cannot
     block fetcher writers (#375). The store contract is replace-don't-mutate,
@@ -310,6 +431,17 @@ def get_latest_data_subset_refs(*keys: str) -> DashboardData:
         return snap
 
 
+def get_latest_data_refs_snapshot() -> DashboardData:
+    """Return a shallow dict of all top-level store refs (read-only callers).
+
+    Copies the mapping under the lock without deep-copying values. Safe for
+    orjson serialization and other read-only consumers; callers MUST NOT
+    mutate nested objects.
+    """
+    with _data_lock:
+        return {key: value for key, value in latest_data.items()}
+
+
 def get_source_timestamps_snapshot() -> dict[str, str]:
     """Return a stable copy of per-source freshness timestamps."""
     with _data_lock: