perf: live-data deltas, payload caps, and map render polish

Cut fast-tier payload cost with zoom-aware sampling, row deltas, CCTV bbox columns, and MapLibre/motion polish; force viewport snapshot refetches so regional pans refill aircraft immediately.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/tests/test_issue_375_production_hardening.py

@@ -76,6 +76,7 @@ class TestLiveDataFullEndpoint:
             _store.latest_data["sigint"] = [
                 {"source": "aprs", "observed": datetime(2026, 1, 1, tzinfo=timezone.utc)},
             ]
+        _store.bump_data_version()
         try:
             r = client.get("/api/live-data/fast")
             assert r.status_code == 200
@@ -83,6 +84,7 @@ class TestLiveDataFullEndpoint:
         finally:
             with _store._data_lock:
                 _store.latest_data["sigint"] = prior
+            _store.bump_data_version()
 
     def test_live_data_serializes_non_json_native_values(self, client):
         from datetime import datetime, timezone
@@ -94,6 +96,7 @@ class TestLiveDataFullEndpoint:
             _store.latest_data["gdelt"] = [
                 {"observed": datetime(2026, 1, 1, tzinfo=timezone.utc)},
             ]
+        _store.bump_data_version()
         try:
             r = client.get("/api/live-data")
             assert r.status_code == 200
@@ -101,6 +104,7 @@ class TestLiveDataFullEndpoint:
         finally:
             with _store._data_lock:
                 _store.latest_data["gdelt"] = prior
+            _store.bump_data_version()
 
 
 class TestSlowTaskConcurrency:

backend/tests/test_layer_enable_refresh.py

@@ -38,7 +38,8 @@ def test_refresh_skips_when_layer_stays_off():
     fetch_cctv.assert_not_called()
 
 
-def test_refresh_cctv_runs_inline():
+def test_refresh_cctv_runs_on_slow_executor():
+    """CCTV SELECT can be large — never block the API worker on enable."""
     before = {**snapshot_active_layers(), "cctv": False}
     active_layers["cctv"] = True
 
@@ -49,8 +50,9 @@ def test_refresh_cctv_runs_inline():
     ):
         refresh_newly_enabled_layers(before)
 
-    fetch_cctv.assert_called_once()
-    bump.assert_called_once()
-    slow_exec.submit.assert_not_called()
+    fetch_cctv.assert_not_called()
+    bump.assert_not_called()
+    slow_exec.submit.assert_called_once()
+    assert slow_exec.submit.call_args[0][1] == ("cctv",)
 
     active_layers["cctv"] = before.get("cctv", False)

backend/tests/test_live_data_etag_cache.py

@@ -0,0 +1,44 @@
+"""Unit tests for ETag-keyed live-data orjson byte cache (P4)."""
+from __future__ import annotations
+
+from routers import data as data_router
+
+
+def test_cached_live_data_bytes_invokes_factory_once_per_etag():
+    calls = {"n": 0}
+
+    def _build():
+        calls["n"] += 1
+        return {"ok": True, "n": calls["n"]}
+
+    etag = "test|etag|cache-hit"
+    # Isolate from other tests / endpoint warm-up.
+    with data_router._LIVE_DATA_BYTES_CACHE_LOCK:
+        data_router._LIVE_DATA_BYTES_CACHE.pop(etag, None)
+
+    first = data_router._cached_live_data_bytes(etag, _build)
+    second = data_router._cached_live_data_bytes(etag, _build)
+
+    assert first == second
+    assert calls["n"] == 1
+    assert b'"ok"' in first
+
+
+def test_cached_live_data_bytes_misses_on_etag_change():
+    calls = {"n": 0}
+
+    def _build():
+        calls["n"] += 1
+        return {"n": calls["n"]}
+
+    etag_a = "test|etag|a"
+    etag_b = "test|etag|b"
+    with data_router._LIVE_DATA_BYTES_CACHE_LOCK:
+        data_router._LIVE_DATA_BYTES_CACHE.pop(etag_a, None)
+        data_router._LIVE_DATA_BYTES_CACHE.pop(etag_b, None)
+
+    a = data_router._cached_live_data_bytes(etag_a, _build)
+    b = data_router._cached_live_data_bytes(etag_b, _build)
+
+    assert a != b
+    assert calls["n"] == 2

backend/tests/test_live_data_viewport_bbox.py

@@ -24,6 +24,12 @@ class TestFastBboxFiltering:
     def _seed_fast(self, monkeypatch):
         """Plant deterministic heavy + light fixtures across the globe."""
         from services.fetchers import _store
+        from routers import data as data_router
+
+        # Avoid cross-test ETag byte-cache pollution.
+        with data_router._LIVE_DATA_BYTES_CACHE_LOCK:
+            data_router._LIVE_DATA_BYTES_CACHE.clear()
+        _store.bump_data_version()
 
         # Heavy collections: dense across the world.
         commercial = [
@@ -35,7 +41,11 @@ class TestFastBboxFiltering:
             {"lat": -60.0, "lng": -120.0, "id": "s-sw"},
             {"lat": 35.0, "lng": -75.0, "id": "s-ne"},
         ]
-        cctv = [{"lat": 35.0, "lng": -75.0, "id": "c-1"}]
+        # Real CCTV rows use ``lon`` (not ``lng``) — bbox must honor that alias.
+        cctv = [
+            {"lat": 35.0, "lon": -75.0, "id": "c-1"},
+            {"lat": 35.0, "lon": 100.0, "id": "c-asia"},
+        ]
 
         # Sigint heavy collection.
         sigint = [
@@ -85,7 +95,7 @@ class TestFastBboxFiltering:
         # Heavy layers: only the eastern-US fixture survives.
         assert {f["id"] for f in data["commercial_flights"]} == {"f-ne"}
         assert {s["id"] for s in data["ships"]} == {"s-ne"}
-        assert {c["id"] for c in data["cctv"]} == {"c-1"}
+        assert {c["id"] for c in data["cctv"]} == {"c-1"}  # lon alias filtered
         assert {s["id"] for s in data["sigint"]} == {"sig-east"}
 
     def test_bbox_does_not_filter_light_layers(self, client, monkeypatch):

backend/tests/test_perf_guardrail_caps.py

@@ -0,0 +1,126 @@
+"""P5 / P11 guardrail tests — zoom-aware caps + CCTV lon bbox + enable path."""
+from __future__ import annotations
+
+
+def test_sample_items_even_stride():
+    from routers.data import _sample_items
+
+    items = [{"id": i} for i in range(100)]
+    sampled = _sample_items(items, 10)
+    assert len(sampled) == 10
+    assert sampled[0]["id"] == 0
+    assert sampled[-1]["id"] == 90
+
+
+def test_world_zoom_caps_dense_layers_and_keeps_totals():
+    from routers.data import _cap_fast_dashboard_payload
+
+    flights = [{"lat": 0.0, "lng": float(i), "id": f"f-{i}"} for i in range(2000)]
+    cctv = [{"lat": 0.0, "lon": float(i % 180), "id": f"c-{i}"} for i in range(900)]
+    payload = {
+        "commercial_flights": flights,
+        "ships": [{"lat": 1.0, "lng": 1.0, "id": "s1"}],
+        "cctv": cctv,
+        "cctv_total": 900,
+        "satellites": [{"lat": -10.0, "lng": 10.0, "id": "sat"}],  # never capped
+    }
+    out = _cap_fast_dashboard_payload(payload)
+    assert out["payload_scale"] == "world"
+    assert out["payload_sampled"] is True
+    assert len(out["commercial_flights"]) == 1200
+    assert out["layer_totals"]["commercial_flights"] == 2000
+    assert len(out["cctv"]) == 600
+    assert out["cctv_total"] == 900
+    assert len(out["satellites"]) == 1
+
+
+def test_regional_zoom_does_not_cap():
+    from routers.data import _cap_fast_dashboard_payload
+
+    flights = [{"lat": 35.0, "lng": -75.0, "id": f"f-{i}"} for i in range(1500)]
+    out = _cap_fast_dashboard_payload(
+        {"commercial_flights": flights},
+        s=30,
+        w=-80,
+        n=40,
+        e=-70,
+    )
+    assert out["payload_scale"] == "regional"
+    assert "payload_sampled" not in out
+    assert len(out["commercial_flights"]) == 1500
+
+
+def test_bbox_filter_respects_lon_alias():
+    from routers.data import _bbox_filter
+
+    items = [
+        {"id": "in", "lat": 35.0, "lon": -75.0},
+        {"id": "out", "lat": 35.0, "lon": 100.0},
+    ]
+    filtered = _bbox_filter(items, 30, -80, 40, -70)
+    assert {c["id"] for c in filtered} == {"in"}
+
+
+def test_get_all_cameras_column_subset(tmp_path, monkeypatch):
+    import sqlite3
+
+    from services import cctv_pipeline
+
+    db = tmp_path / "cctv.db"
+    monkeypatch.setattr(cctv_pipeline, "DB_PATH", db)
+    conn = sqlite3.connect(str(db))
+    conn.execute(
+        """
+        CREATE TABLE cameras (
+            id TEXT PRIMARY KEY,
+            source_agency TEXT,
+            lat REAL,
+            lon REAL,
+            direction_facing TEXT,
+            media_url TEXT,
+            media_type TEXT,
+            refresh_rate_seconds INTEGER,
+            last_updated TIMESTAMP,
+            secret_extra TEXT
+        )
+        """
+    )
+    conn.execute(
+        "INSERT INTO cameras VALUES (?,?,?,?,?,?,?,?,?,?)",
+        ("cam-1", "dot", 35.0, -75.0, "N", "https://example.com/a.jpg", "image", 60, None, "nope"),
+    )
+    conn.execute(
+        "INSERT INTO cameras VALUES (?,?,?,?,?,?,?,?,?,?)",
+        ("cam-2", "dot", 10.0, 10.0, "S", "https://example.com/b.jpg", "image", 60, None, "nope"),
+    )
+    conn.commit()
+    conn.close()
+
+    all_cams = cctv_pipeline.get_all_cameras()
+    assert len(all_cams) == 2
+    assert "secret_extra" not in all_cams[0]
+    assert set(all_cams[0].keys()) <= set(cctv_pipeline._CAMERA_SELECT_COLS)
+
+    boxed = cctv_pipeline.get_all_cameras(south=30, west=-80, north=40, east=-70)
+    assert [c["id"] for c in boxed] == ["cam-1"]
+    assert cctv_pipeline.get_camera_count() == 2
+
+
+def test_live_data_fast_world_samples_under_cap(client, monkeypatch):
+    from services.fetchers import _store
+    from routers import data as data_router
+
+    with data_router._LIVE_DATA_BYTES_CACHE_LOCK:
+        data_router._LIVE_DATA_BYTES_CACHE.clear()
+    _store.bump_data_version()
+
+    flights = [{"lat": 0.0, "lng": float(i % 170), "id": f"f-{i}"} for i in range(1500)]
+    monkeypatch.setitem(_store.latest_data, "commercial_flights", flights)
+    monkeypatch.setitem(_store.active_layers, "flights", True)
+
+    r = client.get("/api/live-data/fast")
+    assert r.status_code == 200
+    data = r.json()
+    assert data.get("payload_scale") == "world"
+    assert len(data["commercial_flights"]) == 1200
+    assert data["layer_totals"]["commercial_flights"] == 1500

backend/tests/test_perf_p2_p8_p12.py

@@ -0,0 +1,148 @@
+"""P2 live-data delta + P8 hashchain memory + P12 mesh-only import guards."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from unittest.mock import patch
+
+
+def test_compute_layer_row_delta_upsert_and_delete(monkeypatch):
+    from services.fetchers import _store
+
+    monkeypatch.setitem(_store.latest_data, "ships", [
+        {"mmsi": "1", "lat": 1.0, "lng": 1.0},
+        {"mmsi": "2", "lat": 2.0, "lng": 2.0},
+    ])
+    _store._LAYER_ID_RING.clear()
+    with _store._data_lock:
+        _store._layer_versions["ships"] = 1
+        _store._record_delta_layer_snapshot_locked("ships")
+
+    monkeypatch.setitem(_store.latest_data, "ships", [
+        {"mmsi": "1", "lat": 1.5, "lng": 1.0},  # moved
+        {"mmsi": "3", "lat": 3.0, "lng": 3.0},  # new
+    ])
+    with _store._data_lock:
+        _store._layer_versions["ships"] = 2
+        _store._record_delta_layer_snapshot_locked("ships")
+
+    delta = _store.compute_layer_row_delta("ships", 1)
+    assert delta is not None
+    assert {u["mmsi"] for u in delta["upsert"]} == {"1", "3"}
+    assert delta["delete"] == ["2"]
+    assert delta["version"] == 2
+
+
+def test_compute_layer_row_delta_returns_none_when_base_missing():
+    from services.fetchers import _store
+
+    _store._LAYER_ID_RING.clear()
+    with _store._data_lock:
+        _store._layer_versions["ships"] = 5
+    # Client holds an older version that is no longer in the ring.
+    assert _store.compute_layer_row_delta("ships", 1) is None
+
+
+def test_live_data_fast_delta_mode(client, monkeypatch):
+    from services.fetchers import _store
+
+    ships_v1 = [
+        {"mmsi": "10", "lat": 10.0, "lng": 10.0},
+        {"mmsi": "20", "lat": 20.0, "lng": 20.0},
+    ]
+    monkeypatch.setitem(_store.latest_data, "ships", ships_v1)
+    monkeypatch.setitem(_store.active_layers, "ships_military", True)
+    _store._LAYER_ID_RING.clear()
+    with _store._data_lock:
+        _store._layer_versions["ships"] = 1
+        for key in (
+            "commercial_flights",
+            "military_flights",
+            "tracked_flights",
+            "private_flights",
+            "private_jets",
+            "cctv",
+            "uavs",
+            "liveuamap",
+            "gps_jamming",
+            "satellites",
+            "sigint",
+            "trains",
+        ):
+            _store._layer_versions.setdefault(key, 1)
+        _store._record_delta_layer_snapshot_locked("ships")
+        for key in (
+            "commercial_flights",
+            "military_flights",
+            "tracked_flights",
+            "private_flights",
+            "private_jets",
+        ):
+            monkeypatch.setitem(_store.latest_data, key, [])
+            _store._record_delta_layer_snapshot_locked(key)
+
+    # Advance ships
+    monkeypatch.setitem(
+        _store.latest_data,
+        "ships",
+        [
+            {"mmsi": "10", "lat": 10.5, "lng": 10.0},
+            {"mmsi": "30", "lat": 30.0, "lng": 30.0},
+        ],
+    )
+    with _store._data_lock:
+        _store._layer_versions["ships"] = 2
+        _store._record_delta_layer_snapshot_locked("ships")
+
+    lv = (
+        "ships:1,commercial_flights:1,military_flights:1,tracked_flights:1,"
+        "private_flights:1,private_jets:1,cctv:1,uavs:1,liveuamap:1,"
+        "gps_jamming:1,satellites:1,sigint:1,trains:1"
+    )
+    r = client.get(f"/api/live-data/fast?lv={lv}")
+    assert r.status_code == 200
+    data = r.json()
+    assert data["mode"] == "delta"
+    assert "ships" in data["deltas"]
+    assert {u["mmsi"] for u in data["deltas"]["ships"]["upsert"]} == {"10", "30"}
+    assert data["deltas"]["ships"]["delete"] == ["20"]
+
+
+def test_hashchain_flush_uses_orjson_compact(tmp_path, monkeypatch):
+    from services.mesh import mesh_hashchain as hc
+
+    monkeypatch.setattr(hc, "DATA_DIR", tmp_path)
+    monkeypatch.setattr(hc, "CHAIN_FILE", tmp_path / "infonet.json")
+    monkeypatch.setattr(hc, "WAL_FILE", tmp_path / "infonet.wal")
+    monkeypatch.setattr(hc, "CHAIN_COLD_DIR", tmp_path / "infonet_cold")
+    monkeypatch.setattr(hc, "MAX_CHAIN_MEMORY", 3)
+
+    net = hc.Infonet()
+    net.events = [{"event_id": f"e{i}", "event_type": "message", "payload": {}} for i in range(5)]
+    net.head_hash = "e4"
+    net._dirty = True
+    net._enforce_memory_cap()
+    net._flush()
+
+    raw = (tmp_path / "infonet.json").read_bytes()
+    assert b"\n  " not in raw  # compact, not indent=2
+    data = json.loads(raw)
+    assert len(data["events"]) == 3
+    assert data["cold_segments"]
+    cold_file = tmp_path / "infonet_cold" / data["cold_segments"][0]["filename"]
+    assert cold_file.exists()
+    cold = json.loads(cold_file.read_bytes())
+    assert len(cold) == 2
+
+
+def test_mesh_only_skips_osint_router_modules():
+    """Source-level guard: MESH_ONLY branch must not load routers.data at import."""
+    import inspect
+    from pathlib import Path
+
+    src = Path("backend/main.py").read_text(encoding="utf-8")
+    assert "if _MESH_ONLY:" in src
+    assert 'data_router = APIRouter()' in src
+    assert "from services.data_fetcher import" in src
+    # data_fetcher import is gated
+    assert "if not _MESH_ONLY:" in src

backend/tests/test_perf_safe_optimizations.py

@@ -33,6 +33,25 @@ def test_health_uses_subset_refs_not_full_deepcopy():
     assert "deepcopy" not in snap_source
 
 
+def test_legacy_live_data_uses_refs_snapshot_not_deepcopy():
+    from routers import data as data_router
+
+    source = inspect.getsource(data_router.live_data)
+    assert "get_latest_data_refs_snapshot" in source
+    assert "get_latest_data_deepcopy_snapshot" not in source
+    assert "deepcopy" not in source
+
+
+def test_openclaw_watchdog_uses_telemetry_refs():
+    from services import openclaw_watchdog
+
+    source = inspect.getsource(openclaw_watchdog._evaluate_watches)
+    assert "get_cached_telemetry_refs" in source
+    assert "get_cached_slow_telemetry_refs" in source
+    assert "get_cached_telemetry()" not in source
+    assert "get_cached_slow_telemetry()" not in source
+
+
 def test_active_layers_defaults_match_dashboard_first_paint():
     """Backend must not prefetch layers the dashboard starts with disabled."""
     from services.fetchers import _store