fix: stop truncating live telemetry payloads

Remove sampling, viewport bbox truncation, and buffer ceilings that were cherry-picking ships, flights, and related layers.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/tests/test_live_data_viewport_bbox.py

@@ -1,59 +1,34 @@
-"""Tests for issue #288: viewport bbox filtering on /api/live-data/{fast,slow}.
-
-Behaviour contract:
- * Without s/w/n/e params, the response is byte-for-byte identical to the
-   pre-#288 implementation. (No filtering, no extra fields, no ETag change.)
- * With s/w/n/e supplied, heavy/dense layers are filtered to that viewport
-   with a 20% padding box.
- * Light reference layers (datacenters, military_bases, power_plants,
-   satellites, news, weather, …) are NEVER filtered, even when bounds are
-   supplied — panning must never reveal an "empty world" of infrastructure.
- * World-scale bounds (lng_span >= 300 OR lat_span >= 120) short-circuit
-   filtering and share the global ETag.
- * The ETag includes a 1°-quantized bbox so two viewports never poison each
-   other's 304 cache.
-"""
+"""Live-data viewport params are accepted but no longer truncate telemetry."""
 
 import pytest
 
 
-# ───────────────────────── /api/live-data/fast ─────────────────────────────
-
-
 class TestFastBboxFiltering:
     def _seed_fast(self, monkeypatch):
-        """Plant deterministic heavy + light fixtures across the globe."""
         from services.fetchers import _store
         from routers import data as data_router
 
-        # Avoid cross-test ETag byte-cache pollution.
         with data_router._LIVE_DATA_BYTES_CACHE_LOCK:
             data_router._LIVE_DATA_BYTES_CACHE.clear()
         _store.bump_data_version()
 
-        # Heavy collections: dense across the world.
         commercial = [
-            {"lat": -60.0, "lng": -120.0, "id": "f-sw"},   # south Pacific
-            {"lat": 35.0, "lng": -75.0, "id": "f-ne"},     # eastern US
-            {"lat": 35.0, "lng": 100.0, "id": "f-asia"},   # Asia
+            {"lat": -60.0, "lng": -120.0, "id": "f-sw"},
+            {"lat": 35.0, "lng": -75.0, "id": "f-ne"},
+            {"lat": 35.0, "lng": 100.0, "id": "f-asia"},
         ]
         ships = [
             {"lat": -60.0, "lng": -120.0, "id": "s-sw"},
             {"lat": 35.0, "lng": -75.0, "id": "s-ne"},
         ]
-        # Real CCTV rows use ``lon`` (not ``lng``) — bbox must honor that alias.
         cctv = [
             {"lat": 35.0, "lon": -75.0, "id": "c-1"},
             {"lat": 35.0, "lon": 100.0, "id": "c-asia"},
         ]
-
-        # Sigint heavy collection.
         sigint = [
             {"source": "meshtastic", "lat": 35.0, "lng": -75.0, "id": "sig-east"},
             {"source": "meshtastic", "lat": 35.0, "lng": 100.0, "id": "sig-asia"},
         ]
-
-        # Light/reference layer — must NEVER be filtered.
         satellites = [
             {"lat": -60.0, "lng": -120.0, "id": "sat-sw"},
             {"lat": 35.0, "lng": -75.0, "id": "sat-ne"},
@@ -65,7 +40,6 @@ class TestFastBboxFiltering:
         monkeypatch.setitem(_store.latest_data, "cctv", cctv)
         monkeypatch.setitem(_store.latest_data, "sigint", sigint)
         monkeypatch.setitem(_store.latest_data, "satellites", satellites)
-        # Ensure all layers are on so the response includes them.
         for layer in (
             "flights", "ships_military", "ships_cargo", "ships_civilian",
             "ships_passenger", "ships_tracked_yachts", "cctv",
@@ -78,37 +52,24 @@ class TestFastBboxFiltering:
         r = client.get("/api/live-data/fast")
         assert r.status_code == 200
         data = r.json()
-        # All heavy fixtures pass through unchanged.
         assert len(data["commercial_flights"]) == 3
         assert len(data["ships"]) == 2
         assert len(data["sigint"]) == 2
-        # Light layer also full.
         assert len(data["satellites"]) == 3
 
-    def test_bbox_filters_heavy_layers(self, client, monkeypatch):
-        self._seed_fast(monkeypatch)
-        # Box tightly around the eastern-US fixture (lat 35, lng -75).
-        # ±5° → after 20% padding inside _bbox_filter, ~±6° window.
-        r = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70")
-        assert r.status_code == 200
-        data = r.json()
-        # Heavy layers: only the eastern-US fixture survives.
-        assert {f["id"] for f in data["commercial_flights"]} == {"f-ne"}
-        assert {s["id"] for s in data["ships"]} == {"s-ne"}
-        assert {c["id"] for c in data["cctv"]} == {"c-1"}  # lon alias filtered
-        assert {s["id"] for s in data["sigint"]} == {"sig-east"}
-
-    def test_bbox_does_not_filter_light_layers(self, client, monkeypatch):
+    def test_bbox_does_not_truncate_heavy_layers(self, client, monkeypatch):
         self._seed_fast(monkeypatch)
         r = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70")
         assert r.status_code == 200
         data = r.json()
-        # Satellites are a reference layer — must NOT be bbox-filtered.
+        assert {f["id"] for f in data["commercial_flights"]} == {"f-sw", "f-ne", "f-asia"}
+        assert {s["id"] for s in data["ships"]} == {"s-sw", "s-ne"}
+        assert {c["id"] for c in data["cctv"]} == {"c-1", "c-asia"}
+        assert {s["id"] for s in data["sigint"]} == {"sig-east", "sig-asia"}
         assert len(data["satellites"]) == 3
 
-    def test_world_scale_bbox_skips_filtering(self, client, monkeypatch):
+    def test_world_scale_bbox_returns_full_set(self, client, monkeypatch):
         self._seed_fast(monkeypatch)
-        # lng_span = 360 → treated as world-scale; same as no bbox.
         r = client.get("/api/live-data/fast?s=-90&w=-180&n=90&e=180")
         assert r.status_code == 200
         data = r.json()
@@ -117,83 +78,52 @@ class TestFastBboxFiltering:
 
     def test_partial_bbox_is_treated_as_no_bbox(self, client, monkeypatch):
         self._seed_fast(monkeypatch)
-        # Only three of four bounds → filtering must NOT engage.
         r = client.get("/api/live-data/fast?s=30&w=-80&n=40")
         assert r.status_code == 200
         data = r.json()
         assert len(data["commercial_flights"]) == 3
 
-    def test_etag_changes_with_bbox(self, client, monkeypatch):
+    def test_etag_shared_across_bbox_and_world(self, client, monkeypatch):
         self._seed_fast(monkeypatch)
         r_world = client.get("/api/live-data/fast")
         r_local = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70")
         assert r_world.status_code == 200
         assert r_local.status_code == 200
-        etag_world = r_world.headers.get("etag")
-        etag_local = r_local.headers.get("etag")
-        assert etag_world and etag_local
-        assert etag_world != etag_local, (
-            "ETag must differ between world and regional bbox to prevent "
-            "304 cache poisoning across viewports"
-        )
+        assert r_world.headers.get("ETag") == r_local.headers.get("ETag")
 
-    def test_etag_stable_for_subdegree_pan(self, client, monkeypatch):
+    def test_if_none_match_returns_304(self, client, monkeypatch):
         self._seed_fast(monkeypatch)
-        # Sub-degree pan should land in the same 1°-quantized bucket.
-        r_a = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70")
-        r_b = client.get("/api/live-data/fast?s=30.3&w=-79.8&n=39.7&e=-70.4")
-        assert r_a.headers.get("etag") == r_b.headers.get("etag")
-
-    def test_if_none_match_returns_304_for_same_bbox(self, client, monkeypatch):
-        self._seed_fast(monkeypatch)
-        r1 = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70")
-        etag = r1.headers.get("etag")
-        r2 = client.get(
-            "/api/live-data/fast?s=30&w=-80&n=40&e=-70",
-            headers={"If-None-Match": etag},
-        )
+        r1 = client.get("/api/live-data/fast")
+        etag = r1.headers.get("ETag")
+        r2 = client.get("/api/live-data/fast", headers={"If-None-Match": etag})
         assert r2.status_code == 304
 
 
-# ───────────────────────── /api/live-data/slow ─────────────────────────────
-
-
 class TestSlowBboxFiltering:
     def _seed_slow(self, monkeypatch):
         from services.fetchers import _store
+        from routers import data as data_router
+
+        with data_router._LIVE_DATA_BYTES_CACHE_LOCK:
+            data_router._LIVE_DATA_BYTES_CACHE.clear()
+        _store.bump_data_version()
 
-        # Heavy collections.
         gdelt = [
-            {"lat": 35.0, "lng": -75.0, "id": "g-east"},
+            {"lat": 35.0, "lng": -75.0, "id": "g-ne"},
             {"lat": 35.0, "lng": 100.0, "id": "g-asia"},
         ]
-        firms_fires = [
-            {"lat": 35.0, "lng": -75.0, "id": "fire-east"},
-            {"lat": -10.0, "lng": 120.0, "id": "fire-ido"},
+        firms = [
+            {"lat": 35.0, "lng": -75.0, "id": "fire-ne"},
+            {"lat": -20.0, "lng": 140.0, "id": "fire-au"},
         ]
-        # Light/reference layers — must always ship in full.
         datacenters = [
-            {"lat": 35.0, "lng": -75.0, "id": "dc-east"},
-            {"lat": 35.0, "lng": 100.0, "id": "dc-asia"},
-            {"lat": -10.0, "lng": 120.0, "id": "dc-ido"},
+            {"lat": 35.0, "lng": -75.0, "id": "dc-ne"},
+            {"lat": 51.0, "lng": 0.0, "id": "dc-uk"},
         ]
-        military_bases = [
-            {"lat": 35.0, "lng": -75.0, "id": "mb-east"},
-            {"lat": -10.0, "lng": 120.0, "id": "mb-ido"},
-        ]
-        power_plants = [
-            {"lat": 35.0, "lng": -75.0, "id": "pp-east"},
-            {"lat": 35.0, "lng": 100.0, "id": "pp-asia"},
-        ]
-
         monkeypatch.setitem(_store.latest_data, "gdelt", gdelt)
-        monkeypatch.setitem(_store.latest_data, "firms_fires", firms_fires)
+        monkeypatch.setitem(_store.latest_data, "firms_fires", firms)
         monkeypatch.setitem(_store.latest_data, "datacenters", datacenters)
-        monkeypatch.setitem(_store.latest_data, "military_bases", military_bases)
-        monkeypatch.setitem(_store.latest_data, "power_plants", power_plants)
-        for layer in (
-            "global_incidents", "firms", "datacenters", "military_bases", "power_plants",
-        ):
+        for layer in ("global_incidents", "firms", "datacenters"):
             monkeypatch.setitem(_store.active_layers, layer, True)
 
     def test_no_bbox_returns_world_data(self, client, monkeypatch):
@@ -203,81 +133,40 @@ class TestSlowBboxFiltering:
         data = r.json()
         assert len(data["gdelt"]) == 2
         assert len(data["firms_fires"]) == 2
-        assert len(data["datacenters"]) == 3
+        assert len(data["datacenters"]) == 2
 
-    def test_bbox_filters_heavy_layers(self, client, monkeypatch):
+    def test_bbox_does_not_truncate_slow_layers(self, client, monkeypatch):
         self._seed_slow(monkeypatch)
         r = client.get("/api/live-data/slow?s=30&w=-80&n=40&e=-70")
         assert r.status_code == 200
         data = r.json()
-        assert {g["id"] for g in data["gdelt"]} == {"g-east"}
-        assert {f["id"] for f in data["firms_fires"]} == {"fire-east"}
-
-    def test_bbox_leaves_reference_layers_untouched(self, client, monkeypatch):
-        """Datacenters, bases, and power plants are infrastructure overlays —
-        they must remain world-scale so panning never hides them."""
-        self._seed_slow(monkeypatch)
-        r = client.get("/api/live-data/slow?s=30&w=-80&n=40&e=-70")
-        assert r.status_code == 200
-        data = r.json()
-        assert len(data["datacenters"]) == 3
-        assert len(data["military_bases"]) == 2
-        assert len(data["power_plants"]) == 2
-
-    def test_antimeridian_bbox(self, client, monkeypatch):
-        from services.fetchers import _store
-        # Box that straddles the antimeridian (Pacific): w=170, e=-170.
-        gdelt = [
-            {"lat": 0.0, "lng": 175.0, "id": "in-west"},
-            {"lat": 0.0, "lng": -175.0, "id": "in-east"},
-            {"lat": 0.0, "lng": 0.0, "id": "out-mid"},
-        ]
-        monkeypatch.setitem(_store.latest_data, "gdelt", gdelt)
-        monkeypatch.setitem(_store.active_layers, "global_incidents", True)
-        r = client.get("/api/live-data/slow?s=-10&w=170&n=10&e=-170")
-        assert r.status_code == 200
-        data = r.json()
-        ids = {g["id"] for g in data["gdelt"]}
-        assert "in-west" in ids
-        assert "in-east" in ids
-        assert "out-mid" not in ids
+        assert {g["id"] for g in data["gdelt"]} == {"g-ne", "g-asia"}
+        assert {f["id"] for f in data["firms_fires"]} == {"fire-ne", "fire-au"}
+        assert len(data["datacenters"]) == 2
 
 
-# ─────────────────── Direct helper coverage (defensive) ─────────────────────
-
-
-class TestHelpers:
+class TestBboxHelpers:
     def test_has_full_bbox(self):
         from routers.data import _has_full_bbox
         assert _has_full_bbox(1, 2, 3, 4)
         assert not _has_full_bbox(None, 2, 3, 4)
-        assert not _has_full_bbox(1, None, 3, 4)
-        assert not _has_full_bbox(1, 2, None, 4)
-        assert not _has_full_bbox(1, 2, 3, None)
 
-    def test_bbox_etag_suffix_quantizes(self):
+    def test_bbox_etag_suffix_always_empty(self):
         from routers.data import _bbox_etag_suffix
-        a = _bbox_etag_suffix(30.1, -79.6, 39.9, -70.1)
-        b = _bbox_etag_suffix(30.4, -79.2, 39.4, -70.8)
-        assert a == b, "Sub-degree pan must collapse to the same ETag suffix"
-        assert a.startswith("|bbox=")
-
-    def test_bbox_etag_suffix_world_collapses(self):
-        from routers.data import _bbox_etag_suffix
-        # World-scale → empty suffix (shares the global ETag).
+        assert _bbox_etag_suffix(30.1, -79.6, 39.9, -70.1) == ""
         assert _bbox_etag_suffix(-90, -180, 90, 180) == ""
-
-    def test_bbox_etag_suffix_partial_is_empty(self):
-        from routers.data import _bbox_etag_suffix
         assert _bbox_etag_suffix(None, -180, 90, 180) == ""
 
-    def test_apply_bbox_preserves_non_list_values(self):
+    def test_apply_bbox_is_noop(self):
         from routers.data import _apply_bbox_to_payload, _FAST_BBOX_HEAVY_KEYS
+
         payload = {
-            "commercial_flights": [{"lat": 35, "lng": -75, "id": "x"}],
-            "satellite_source": "tle",  # not a list, must pass through
-            "sigint_totals": {"total": 1},  # dict — must pass through
+            "commercial_flights": [
+                {"lat": 35.0, "lng": -75.0, "id": "in"},
+                {"lat": 35.0, "lng": 100.0, "id": "out"},
+            ],
+            "note": "keep",
         }
         out = _apply_bbox_to_payload(dict(payload), _FAST_BBOX_HEAVY_KEYS, 30, -80, 40, -70)
-        assert out["satellite_source"] == "tle"
-        assert out["sigint_totals"] == {"total": 1}
+        assert len(out["commercial_flights"]) == 2
+        assert out["note"] == "keep"

backend/tests/test_perf_guardrail_caps.py

@@ -1,4 +1,4 @@
-"""P5 / P11 guardrail tests — zoom-aware caps + CCTV lon bbox + enable path."""
+"""Live-data payload helpers — no telemetry sampling; bbox densify only."""
 from __future__ import annotations
 
 
@@ -12,7 +12,7 @@ def test_sample_items_even_stride():
     assert sampled[-1]["id"] == 90
 
 
-def test_world_zoom_caps_dense_layers_and_keeps_totals():
+def test_world_zoom_does_not_sample_telemetry():
     from routers.data import _cap_fast_dashboard_payload
 
     flights = [{"lat": 0.0, "lng": float(i), "id": f"f-{i}"} for i in range(2000)]
@@ -22,19 +22,18 @@ def test_world_zoom_caps_dense_layers_and_keeps_totals():
         "ships": [{"lat": 1.0, "lng": 1.0, "id": "s1"}],
         "cctv": cctv,
         "cctv_total": 900,
-        "satellites": [{"lat": -10.0, "lng": 10.0, "id": "sat"}],  # never capped
+        "satellites": [{"lat": -10.0, "lng": 10.0, "id": "sat"}],
     }
     out = _cap_fast_dashboard_payload(payload)
     assert out["payload_scale"] == "world"
-    assert out["payload_sampled"] is True
-    assert len(out["commercial_flights"]) == 1200
-    assert out["layer_totals"]["commercial_flights"] == 2000
-    assert len(out["cctv"]) == 600
+    assert "payload_sampled" not in out
+    assert len(out["commercial_flights"]) == 2000
+    assert len(out["cctv"]) == 900
     assert out["cctv_total"] == 900
     assert len(out["satellites"]) == 1
 
 
-def test_regional_zoom_does_not_cap():
+def test_regional_zoom_annotates_scale_without_sampling():
     from routers.data import _cap_fast_dashboard_payload
 
     flights = [{"lat": 35.0, "lng": -75.0, "id": f"f-{i}"} for i in range(1500)]
@@ -101,12 +100,13 @@ def test_get_all_cameras_column_subset(tmp_path, monkeypatch):
     assert "secret_extra" not in all_cams[0]
     assert set(all_cams[0].keys()) <= set(cctv_pipeline._CAMERA_SELECT_COLS)
 
+    # Viewport args are ignored — full catalog always.
     boxed = cctv_pipeline.get_all_cameras(south=30, west=-80, north=40, east=-70)
-    assert [c["id"] for c in boxed] == ["cam-1"]
+    assert {c["id"] for c in boxed} == {"cam-1", "cam-2"}
     assert cctv_pipeline.get_camera_count() == 2
 
 
-def test_live_data_fast_world_samples_under_cap(client, monkeypatch):
+def test_live_data_fast_world_keeps_full_telemetry(client, monkeypatch):
     from services.fetchers import _store
     from routers import data as data_router
 
@@ -122,5 +122,5 @@ def test_live_data_fast_world_samples_under_cap(client, monkeypatch):
     assert r.status_code == 200
     data = r.json()
     assert data.get("payload_scale") == "world"
-    assert len(data["commercial_flights"]) == 1200
-    assert data["layer_totals"]["commercial_flights"] == 1500
+    assert "payload_sampled" not in data
+    assert len(data["commercial_flights"]) == 1500