fix: stop truncating live telemetry payloads

Remove sampling, viewport bbox truncation, and buffer ceilings that were cherry-picking ships, flights, and related layers.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/routers/data.py

@@ -151,48 +151,17 @@ def _has_full_bbox(s, w, n, e) -> bool:
 
 
 def _bbox_etag_suffix(s, w, n, e) -> str:
-    """Quantize bbox to 1° before mixing into the ETag.
-
-    The 20% padding inside _bbox_filter already absorbs sub-degree pans;
-    quantizing here means small mouse drags don't blow the ETag cache
-    on the client. Full-world bounds collapse to a single suffix.
-    """
-    if not _has_full_bbox(s, w, n, e):
-        return ""
-    try:
-        ss = math.floor(float(s))
-        ww = math.floor(float(w))
-        nn = math.ceil(float(n))
-        ee = math.ceil(float(e))
-    except (TypeError, ValueError):
-        return ""
-    # If the requested window covers basically the whole world, treat it as
-    # "no bbox" for caching purposes so world-zoomed clients all hit the
-    # same ETag and benefit from the existing 304 path.
-    lat_span, lng_span = _bbox_spans(s, w, n, e)
-    if lng_span >= 300 or lat_span >= 120:
-        return ""
-    return f"|bbox={ss},{ww},{nn},{ee}"
+    """Bbox no longer scopes payloads — keep a single world ETag cache key."""
+    return ""
 
 
 def _apply_bbox_to_payload(payload: dict, heavy_keys: tuple[str, ...],
                             s: float, w: float, n: float, e: float) -> dict:
-    """In-place filter the heavy-key collections in *payload* to a viewport.
+    """No-op: live telemetry is never viewport-truncated.
 
-    Items without lat/lng are passed through (so e.g. summary blobs aren't
-    accidentally dropped). The existing _bbox_filter helper applies a 20%
-    pad and handles antimeridian crossings.
+    Bounds may still be accepted on the wire for client hints / future use,
+    but we do not drop ships, aircraft, or other dense layers by map window.
     """
-    lat_span, lng_span = _bbox_spans(s, w, n, e)
-    # World-scale request → skip filtering entirely. Spares the CPU and
-    # guarantees the response matches the no-params shape.
-    if lng_span >= 300 or lat_span >= 120:
-        return payload
-    for key in heavy_keys:
-        items = payload.get(key)
-        if not isinstance(items, list) or not items:
-            continue
-        payload[key] = _bbox_filter(items, s, w, n, e)
     return payload
 
 
@@ -345,51 +314,12 @@ def _sample_items(items: list | None, max_items: int) -> list:
 
 
 def _cap_fast_startup_payload(payload: dict) -> dict:
+    """Mark first-paint payloads. Do not defer or sample live telemetry."""
     capped = dict(payload)
-    capped["commercial_flights"] = _cap_startup_items(capped.get("commercial_flights"), 800)
-    capped["private_flights"] = _cap_startup_items(capped.get("private_flights"), 300)
-    capped["private_jets"] = _cap_startup_items(capped.get("private_jets"), 150)
-    capped["ships"] = _cap_startup_items(capped.get("ships"), 1500)
-    capped["cctv"] = []
-    capped["sigint"] = _cap_startup_items(capped.get("sigint"), 500)
-    capped["trains"] = _cap_startup_items(capped.get("trains"), 100)
     capped["startup_payload"] = True
     return capped
 
 
-# Dense layers only — never cap static infra (bases, plants, satellites, …).
-# World caps keep first paint / zoomed-out views from shipping millions of rows;
-# continental is looser; regional/city zoom skips capping (bbox already densifies).
-_WORLD_FAST_CAPS: dict[str, int] = {
-    "commercial_flights": 1200,
-    "military_flights": 400,
-    "private_flights": 400,
-    "private_jets": 200,
-    "tracked_flights": 200,
-    "ships": 2000,
-    "cctv": 600,
-    "uavs": 200,
-    "liveuamap": 400,
-    "gps_jamming": 300,
-    "sigint": 800,
-    "trains": 200,
-}
-_CONTINENTAL_FAST_CAPS: dict[str, int] = {
-    "commercial_flights": 2500,
-    "military_flights": 800,
-    "private_flights": 800,
-    "private_jets": 400,
-    "tracked_flights": 400,
-    "ships": 4000,
-    "cctv": 1500,
-    "uavs": 400,
-    "liveuamap": 800,
-    "gps_jamming": 600,
-    "sigint": 1500,
-    "trains": 400,
-}
-
-
 def _world_and_continental_scale(has_bbox: bool, s, w, n, e) -> tuple:
     lat_span, lng_span = _bbox_spans(s, w, n, e)
     world_scale = (not has_bbox) or lng_span >= 300 or lat_span >= 120
@@ -398,37 +328,20 @@ def _world_and_continental_scale(has_bbox: bool, s, w, n, e) -> tuple:
 
 
 def _cap_fast_dashboard_payload(payload: dict, *, s=None, w=None, n=None, e=None) -> dict:
-    """Sample dense layers at world/continental zoom only.
+    """Annotate zoom scale only — never sample/drop live telemetry rows.
 
-    Regional pans rely on #288 bbox filtering for density — do not strip those.
-    Totals for sampled keys are preserved so the UI can show "N of M".
+    Regional density comes from #288 bbox filtering (what's in view), not from
+    picking a random subset of the planet.
     """
     has_bbox = _has_full_bbox(s, w, n, e)
     world_scale, continental_scale = _world_and_continental_scale(has_bbox, s, w, n, e)
-    if not world_scale and not continental_scale:
+    if world_scale:
+        payload["payload_scale"] = "world"
+    elif continental_scale:
+        payload["payload_scale"] = "continental"
+    else:
         payload["payload_scale"] = "regional"
-        return payload
-
-    caps = _WORLD_FAST_CAPS if world_scale else _CONTINENTAL_FAST_CAPS
-    capped = dict(payload)
-    layer_totals = dict(capped.get("layer_totals") or {})
-    sampled = False
-    for key, limit in caps.items():
-        items = capped.get(key)
-        if not isinstance(items, list) or len(items) <= limit:
-            continue
-        layer_totals[key] = len(items)
-        capped[key] = _sample_items(items, limit)
-        sampled = True
-    # Keep cctv_total as the true DB/store count even when the array is sampled.
-    if "cctv" in layer_totals:
-        capped["cctv_total"] = layer_totals["cctv"]
-    if layer_totals:
-        capped["layer_totals"] = layer_totals
-    capped["payload_scale"] = "world" if world_scale else "continental"
-    if sampled:
-        capped["payload_sampled"] = True
-    return capped
+    return payload
 
 
 def _filter_sigint_by_layers(items: list, active_layers: dict) -> list:
@@ -774,62 +687,29 @@ async def bootstrap_critical(request: Request):
         sigint_items = _filter_sigint_by_layers(d.get("sigint") or [], active_layers)
         return {
             "last_updated": d.get("last_updated"),
-            "commercial_flights": _cap_startup_items(
-                (d.get("commercial_flights") or []) if active_layers.get("flights", True) else [],
-                800,
-            ),
-            "military_flights": _cap_startup_items(
-                (d.get("military_flights") or []) if active_layers.get("military", True) else [],
-                300,
-            ),
-            "private_flights": _cap_startup_items(
-                (d.get("private_flights") or []) if active_layers.get("private", True) else [],
-                300,
-            ),
-            "private_jets": _cap_startup_items(
-                (d.get("private_jets") or []) if active_layers.get("jets", True) else [],
-                150,
-            ),
-            "tracked_flights": _cap_startup_items(
-                (d.get("tracked_flights") or []) if active_layers.get("tracked", True) else [],
-                250,
-            ),
-            "ships": _cap_startup_items((d.get("ships") or []) if ships_enabled else [], 1500),
-            "uavs": _cap_startup_items((d.get("uavs") or []) if active_layers.get("military", True) else [], 100),
-            "liveuamap": _cap_startup_items(
-                (d.get("liveuamap") or []) if active_layers.get("global_incidents", True) else [],
-                300,
-            ),
-            "gps_jamming": _cap_startup_items(
-                (d.get("gps_jamming") or []) if active_layers.get("gps_jamming", True) else [],
-                200,
-            ),
-            "satellites": _cap_startup_items(
-                (d.get("satellites") or []) if active_layers.get("satellites", True) else [],
-                250,
-            ),
+            "commercial_flights": (d.get("commercial_flights") or []) if active_layers.get("flights", True) else [],
+            "military_flights": (d.get("military_flights") or []) if active_layers.get("military", True) else [],
+            "private_flights": (d.get("private_flights") or []) if active_layers.get("private", True) else [],
+            "private_jets": (d.get("private_jets") or []) if active_layers.get("jets", True) else [],
+            "tracked_flights": (d.get("tracked_flights") or []) if active_layers.get("tracked", True) else [],
+            "ships": (d.get("ships") or []) if ships_enabled else [],
+            "uavs": (d.get("uavs") or []) if active_layers.get("military", True) else [],
+            "liveuamap": (d.get("liveuamap") or []) if active_layers.get("global_incidents", True) else [],
+            "gps_jamming": (d.get("gps_jamming") or []) if active_layers.get("gps_jamming", True) else [],
+            "satellites": (d.get("satellites") or []) if active_layers.get("satellites", True) else [],
             "satellite_source": d.get("satellite_source", "none"),
             "satellite_analysis": (d.get("satellite_analysis") or {}) if active_layers.get("satellites", True) else {},
-            "sigint": _cap_startup_items(
-                sigint_items if (active_layers.get("sigint_meshtastic", True) or active_layers.get("sigint_aprs", True)) else [],
-                500,
-            ),
+            "sigint": sigint_items if (active_layers.get("sigint_meshtastic", True) or active_layers.get("sigint_aprs", True)) else [],
             "sigint_totals": _sigint_totals_for_items(sigint_items),
-            "trains": _cap_startup_items((d.get("trains") or []) if active_layers.get("trains", True) else [], 100),
-            "news": _cap_startup_items(d.get("news") or [], 30),
-            "gdelt": _cap_startup_items((d.get("gdelt") or []) if active_layers.get("global_incidents", True) else [], 300),
-            "airports": _cap_startup_items(d.get("airports") or [], 500),
+            "trains": (d.get("trains") or []) if active_layers.get("trains", True) else [],
+            "news": d.get("news") or [],
+            "gdelt": (d.get("gdelt") or []) if active_layers.get("global_incidents", True) else [],
+            "airports": d.get("airports") or [],
             "threat_level": d.get("threat_level"),
-            "trending_markets": _cap_startup_items(d.get("trending_markets") or [], 10),
-            "correlations": _cap_startup_items(
-                (d.get("correlations") or []) if active_layers.get("correlations", True) else [],
-                50,
-            ),
+            "trending_markets": d.get("trending_markets") or [],
+            "correlations": (d.get("correlations") or []) if active_layers.get("correlations", True) else [],
             "fimi": d.get("fimi"),
-            "crowdthreat": _cap_startup_items(
-                (d.get("crowdthreat") or []) if active_layers.get("crowdthreat", True) else [],
-                150,
-            ),
+            "crowdthreat": (d.get("crowdthreat") or []) if active_layers.get("crowdthreat", True) else [],
             "freshness": freshness,
             "bootstrap_ready": True,
             "bootstrap_payload": True,
@@ -908,14 +788,8 @@ def _try_build_fast_delta(
         if patch is None:
             return None
         if not patch.get("unchanged"):
-            # Bbox-filter upserts when a regional viewport is active.
-            upserts = list(patch.get("upsert") or [])
-            if upserts and _has_full_bbox(s, w, n, e):
-                lat_span, lng_span = _bbox_spans(s, w, n, e)
-                if not (lng_span >= 300 or lat_span >= 120):
-                    upserts = _bbox_filter(upserts, s, w, n, e)
             deltas[key] = {
-                "upsert": upserts,
+                "upsert": list(patch.get("upsert") or []),
                 "delete": list(patch.get("delete") or []),
                 "version": patch.get("version", server_lv.get(key, 0)),
             }

backend/services/ais_stream.py

@@ -361,7 +361,7 @@ _proxy_status: dict = {}
 _last_msg_at: float = 0.0
 _proxy_spawn_count: int = 0
 _VESSEL_TRAIL_INTERVAL_S = 120
-_VESSEL_TRAIL_MAX_POINTS = 240
+_VESSEL_TRAIL_MAX_POINTS = None  # keep full vessel trail history
 
 
 # How stale "last vessel message" can be before we consider the stream
@@ -435,7 +435,7 @@ def _record_vessel_trail_locked(mmsi: int, lat, lng, sog=0, now_ts: float | None
         return
     trail_data["points"].append(point)
     trail_data["last_seen"] = now
-    if len(trail_data["points"]) > _VESSEL_TRAIL_MAX_POINTS:
+    if _VESSEL_TRAIL_MAX_POINTS is not None and len(trail_data["points"]) > _VESSEL_TRAIL_MAX_POINTS:
         trail_data["points"] = trail_data["points"][-_VESSEL_TRAIL_MAX_POINTS:]
 
 

backend/services/cctv_pipeline.py

@@ -1345,7 +1345,6 @@ class NetherlandsRWSIngestor(BaseCCTVIngestor):
     """
 
     URL = "https://opendata.ndw.nu/cameras.json"
-    MAX_CAMERAS = 1200
 
     def fetch_data(self) -> List[Dict[str, Any]]:
         resp = fetch_with_curl(self.URL, timeout=25, headers={"Accept": "application/json"})
@@ -1362,7 +1361,7 @@ class NetherlandsRWSIngestor(BaseCCTVIngestor):
             return []
 
         cameras: List[Dict[str, Any]] = []
-        for i, cam in enumerate(data[: self.MAX_CAMERAS]):
+        for i, cam in enumerate(data):
             if not isinstance(cam, dict):
                 continue
             lat = cam.get("lat") if cam.get("lat") is not None else cam.get("latitude")
@@ -1475,30 +1474,14 @@ def get_all_cameras(
     north: float | None = None,
     east: float | None = None,
 ) -> List[Dict[str, Any]]:
-    """Load map-facing camera rows, optionally SQL-scoped to a viewport.
-
-    Without bounds, returns the full catalog (store still holds world list;
-    /api/live-data/fast applies #288 bbox + world-zoom sampling).
-    """
+    """Load map-facing camera rows (full catalog; viewport args ignored)."""
     cols = ", ".join(_CAMERA_SELECT_COLS)
     sql = f"SELECT {cols} FROM cameras"
-    params: list[Any] = []
-    if None not in (south, west, north, east):
-        # Inclusive bbox; antimeridian (west > east) uses OR on lon.
-        if west <= east:
-            sql += " WHERE lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?"
-            params.extend([south, north, west, east])
-        else:
-            sql += (
-                " WHERE lat BETWEEN ? AND ?"
-                " AND (lon >= ? OR lon <= ?)"
-            )
-            params.extend([south, north, west, east])
 
     conn = sqlite3.connect(str(DB_PATH))
     conn.row_factory = sqlite3.Row
     try:
-        rows = conn.execute(sql, params).fetchall()
+        rows = conn.execute(sql).fetchall()
     finally:
         conn.close()
 

backend/services/constants.py

@@ -2,7 +2,7 @@
 # Centralized magic numbers. Import from here instead of hardcoding.
 
 # ─── Flight Trails ──────────────────────────────────────────────────────────
-FLIGHT_TRAIL_MAX_TRACKED = 2000  # Max concurrent tracked trails before LRU eviction
+FLIGHT_TRAIL_MAX_TRACKED = None  # no LRU eviction on flight trails
 FLIGHT_TRAIL_POINTS_PER_FLIGHT = 200  # Max trail points kept per aircraft
 TRACKED_TRAIL_TTL_S = 1800  # 30 min - trail TTL for tracked flights
 DEFAULT_TRAIL_TTL_S = 300  # 5 min - trail TTL for non-tracked flights

backend/services/fetchers/flights.py

@@ -339,7 +339,7 @@ PRIVATE_JET_TYPES = {
 # Flight trails state
 flight_trails = {}  # {icao_hex: {points: [[lat, lng, alt, ts], ...], last_seen: ts}}
 _trails_lock = threading.Lock()
-_MAX_TRACKED_TRAILS = 20000
+_MAX_TRACKED_TRAILS = None  # do not LRU-evict flight trails
 
 
 def get_flight_trail(icao24: str) -> list:
@@ -820,7 +820,7 @@ def _classify_and_publish(all_adsb_flights):
         for k in stale_keys:
             del flight_trails[k]
 
-        if len(flight_trails) > _MAX_TRACKED_TRAILS:
+        if _MAX_TRACKED_TRAILS is not None and len(flight_trails) > _MAX_TRACKED_TRAILS:
             sorted_keys = sorted(flight_trails.keys(), key=lambda k: flight_trails[k]["last_seen"])
             evict_count = len(flight_trails) - _MAX_TRACKED_TRAILS
             for k in sorted_keys[:evict_count]:

backend/services/fetchers/geo.py

@@ -311,12 +311,12 @@ def fetch_fishing_activity():
         import datetime as _dt
 
         # GFW publishes with ~5 day lag; windows shorter than ~7 days often return 0 events.
-        lookback_days = _gfw_int_env("GFW_EVENTS_LOOKBACK_DAYS", 7, minimum=1, maximum=14)
-        max_pages = _gfw_int_env("GFW_EVENTS_MAX_PAGES", 10, minimum=1, maximum=100)
-        timeout_s = _gfw_int_env("GFW_EVENTS_TIMEOUT_S", 90, minimum=30, maximum=180)
+        lookback_days = _gfw_int_env("GFW_EVENTS_LOOKBACK_DAYS", 7, minimum=1, maximum=365)
+        max_pages = _gfw_int_env("GFW_EVENTS_MAX_PAGES", 10_000, minimum=1, maximum=None)
+        timeout_s = _gfw_int_env("GFW_EVENTS_TIMEOUT_S", 90, minimum=30, maximum=600)
         _end = _dt.date.today().isoformat()
         _start = (_dt.date.today() - _dt.timedelta(days=lookback_days)).isoformat()
-        page_size = _gfw_int_env("GFW_EVENTS_PAGE_SIZE", 500, minimum=1, maximum=1000)
+        page_size = _gfw_int_env("GFW_EVENTS_PAGE_SIZE", 1000, minimum=1, maximum=1000)
         offset = 0
         pages_fetched = 0
         total_available: int | None = None

backend/services/sigint_bridge.py

@@ -32,8 +32,8 @@ from services.mesh.meshtastic_topics import all_available_roots, build_subscript
 
 logger = logging.getLogger("services.sigint")
 
-# Maximum signals retained per bridge (prevents unbounded memory)
-_MAX_SIGNALS = 500
+# Unbounded live SIGINT retention (age prune still applies).
+_MAX_SIGNALS = None
 # Maximum age of signals before discard (seconds)
 _MAX_AGE_S = 600  # 10 minutes
 

backend/tests/test_live_data_viewport_bbox.py

@@ -1,59 +1,34 @@
-"""Tests for issue #288: viewport bbox filtering on /api/live-data/{fast,slow}.
-
-Behaviour contract:
- * Without s/w/n/e params, the response is byte-for-byte identical to the
-   pre-#288 implementation. (No filtering, no extra fields, no ETag change.)
- * With s/w/n/e supplied, heavy/dense layers are filtered to that viewport
-   with a 20% padding box.
- * Light reference layers (datacenters, military_bases, power_plants,
-   satellites, news, weather, …) are NEVER filtered, even when bounds are
-   supplied — panning must never reveal an "empty world" of infrastructure.
- * World-scale bounds (lng_span >= 300 OR lat_span >= 120) short-circuit
-   filtering and share the global ETag.
- * The ETag includes a 1°-quantized bbox so two viewports never poison each
-   other's 304 cache.
-"""
+"""Live-data viewport params are accepted but no longer truncate telemetry."""
 
 import pytest
 
 
-# ───────────────────────── /api/live-data/fast ─────────────────────────────
-
-
 class TestFastBboxFiltering:
     def _seed_fast(self, monkeypatch):
-        """Plant deterministic heavy + light fixtures across the globe."""
         from services.fetchers import _store
         from routers import data as data_router
 
-        # Avoid cross-test ETag byte-cache pollution.
         with data_router._LIVE_DATA_BYTES_CACHE_LOCK:
             data_router._LIVE_DATA_BYTES_CACHE.clear()
         _store.bump_data_version()
 
-        # Heavy collections: dense across the world.
         commercial = [
-            {"lat": -60.0, "lng": -120.0, "id": "f-sw"},   # south Pacific
-            {"lat": 35.0, "lng": -75.0, "id": "f-ne"},     # eastern US
-            {"lat": 35.0, "lng": 100.0, "id": "f-asia"},   # Asia
+            {"lat": -60.0, "lng": -120.0, "id": "f-sw"},
+            {"lat": 35.0, "lng": -75.0, "id": "f-ne"},
+            {"lat": 35.0, "lng": 100.0, "id": "f-asia"},
         ]
         ships = [
             {"lat": -60.0, "lng": -120.0, "id": "s-sw"},
             {"lat": 35.0, "lng": -75.0, "id": "s-ne"},
         ]
-        # Real CCTV rows use ``lon`` (not ``lng``) — bbox must honor that alias.
         cctv = [
             {"lat": 35.0, "lon": -75.0, "id": "c-1"},
             {"lat": 35.0, "lon": 100.0, "id": "c-asia"},
         ]
-
-        # Sigint heavy collection.
         sigint = [
             {"source": "meshtastic", "lat": 35.0, "lng": -75.0, "id": "sig-east"},
             {"source": "meshtastic", "lat": 35.0, "lng": 100.0, "id": "sig-asia"},
         ]
-
-        # Light/reference layer — must NEVER be filtered.
         satellites = [
             {"lat": -60.0, "lng": -120.0, "id": "sat-sw"},
             {"lat": 35.0, "lng": -75.0, "id": "sat-ne"},
@@ -65,7 +40,6 @@ class TestFastBboxFiltering:
         monkeypatch.setitem(_store.latest_data, "cctv", cctv)
         monkeypatch.setitem(_store.latest_data, "sigint", sigint)
         monkeypatch.setitem(_store.latest_data, "satellites", satellites)
-        # Ensure all layers are on so the response includes them.
         for layer in (
             "flights", "ships_military", "ships_cargo", "ships_civilian",
             "ships_passenger", "ships_tracked_yachts", "cctv",
@@ -78,37 +52,24 @@ class TestFastBboxFiltering:
         r = client.get("/api/live-data/fast")
         assert r.status_code == 200
         data = r.json()
-        # All heavy fixtures pass through unchanged.
         assert len(data["commercial_flights"]) == 3
         assert len(data["ships"]) == 2
         assert len(data["sigint"]) == 2
-        # Light layer also full.
         assert len(data["satellites"]) == 3
 
-    def test_bbox_filters_heavy_layers(self, client, monkeypatch):
-        self._seed_fast(monkeypatch)
-        # Box tightly around the eastern-US fixture (lat 35, lng -75).
-        # ±5° → after 20% padding inside _bbox_filter, ~±6° window.
-        r = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70")
-        assert r.status_code == 200
-        data = r.json()
-        # Heavy layers: only the eastern-US fixture survives.
-        assert {f["id"] for f in data["commercial_flights"]} == {"f-ne"}
-        assert {s["id"] for s in data["ships"]} == {"s-ne"}
-        assert {c["id"] for c in data["cctv"]} == {"c-1"}  # lon alias filtered
-        assert {s["id"] for s in data["sigint"]} == {"sig-east"}
-
-    def test_bbox_does_not_filter_light_layers(self, client, monkeypatch):
+    def test_bbox_does_not_truncate_heavy_layers(self, client, monkeypatch):
         self._seed_fast(monkeypatch)
         r = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70")
         assert r.status_code == 200
         data = r.json()
-        # Satellites are a reference layer — must NOT be bbox-filtered.
+        assert {f["id"] for f in data["commercial_flights"]} == {"f-sw", "f-ne", "f-asia"}
+        assert {s["id"] for s in data["ships"]} == {"s-sw", "s-ne"}
+        assert {c["id"] for c in data["cctv"]} == {"c-1", "c-asia"}
+        assert {s["id"] for s in data["sigint"]} == {"sig-east", "sig-asia"}
         assert len(data["satellites"]) == 3
 
-    def test_world_scale_bbox_skips_filtering(self, client, monkeypatch):
+    def test_world_scale_bbox_returns_full_set(self, client, monkeypatch):
         self._seed_fast(monkeypatch)
-        # lng_span = 360 → treated as world-scale; same as no bbox.
         r = client.get("/api/live-data/fast?s=-90&w=-180&n=90&e=180")
         assert r.status_code == 200
         data = r.json()
@@ -117,83 +78,52 @@ class TestFastBboxFiltering:
 
     def test_partial_bbox_is_treated_as_no_bbox(self, client, monkeypatch):
         self._seed_fast(monkeypatch)
-        # Only three of four bounds → filtering must NOT engage.
         r = client.get("/api/live-data/fast?s=30&w=-80&n=40")
         assert r.status_code == 200
         data = r.json()
         assert len(data["commercial_flights"]) == 3
 
-    def test_etag_changes_with_bbox(self, client, monkeypatch):
+    def test_etag_shared_across_bbox_and_world(self, client, monkeypatch):
         self._seed_fast(monkeypatch)
         r_world = client.get("/api/live-data/fast")
         r_local = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70")
         assert r_world.status_code == 200
         assert r_local.status_code == 200
-        etag_world = r_world.headers.get("etag")
-        etag_local = r_local.headers.get("etag")
-        assert etag_world and etag_local
-        assert etag_world != etag_local, (
-            "ETag must differ between world and regional bbox to prevent "
-            "304 cache poisoning across viewports"
-        )
+        assert r_world.headers.get("ETag") == r_local.headers.get("ETag")
 
-    def test_etag_stable_for_subdegree_pan(self, client, monkeypatch):
+    def test_if_none_match_returns_304(self, client, monkeypatch):
         self._seed_fast(monkeypatch)
-        # Sub-degree pan should land in the same 1°-quantized bucket.
-        r_a = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70")
-        r_b = client.get("/api/live-data/fast?s=30.3&w=-79.8&n=39.7&e=-70.4")
-        assert r_a.headers.get("etag") == r_b.headers.get("etag")
-
-    def test_if_none_match_returns_304_for_same_bbox(self, client, monkeypatch):
-        self._seed_fast(monkeypatch)
-        r1 = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70")
-        etag = r1.headers.get("etag")
-        r2 = client.get(
-            "/api/live-data/fast?s=30&w=-80&n=40&e=-70",
-            headers={"If-None-Match": etag},
-        )
+        r1 = client.get("/api/live-data/fast")
+        etag = r1.headers.get("ETag")
+        r2 = client.get("/api/live-data/fast", headers={"If-None-Match": etag})
         assert r2.status_code == 304
 
 
-# ───────────────────────── /api/live-data/slow ─────────────────────────────
-
-
 class TestSlowBboxFiltering:
     def _seed_slow(self, monkeypatch):
         from services.fetchers import _store
+        from routers import data as data_router
+
+        with data_router._LIVE_DATA_BYTES_CACHE_LOCK:
+            data_router._LIVE_DATA_BYTES_CACHE.clear()
+        _store.bump_data_version()
 
-        # Heavy collections.
         gdelt = [
-            {"lat": 35.0, "lng": -75.0, "id": "g-east"},
+            {"lat": 35.0, "lng": -75.0, "id": "g-ne"},
             {"lat": 35.0, "lng": 100.0, "id": "g-asia"},
         ]
-        firms_fires = [
-            {"lat": 35.0, "lng": -75.0, "id": "fire-east"},
-            {"lat": -10.0, "lng": 120.0, "id": "fire-ido"},
+        firms = [
+            {"lat": 35.0, "lng": -75.0, "id": "fire-ne"},
+            {"lat": -20.0, "lng": 140.0, "id": "fire-au"},
         ]
-        # Light/reference layers — must always ship in full.
         datacenters = [
-            {"lat": 35.0, "lng": -75.0, "id": "dc-east"},
-            {"lat": 35.0, "lng": 100.0, "id": "dc-asia"},
-            {"lat": -10.0, "lng": 120.0, "id": "dc-ido"},
+            {"lat": 35.0, "lng": -75.0, "id": "dc-ne"},
+            {"lat": 51.0, "lng": 0.0, "id": "dc-uk"},
         ]
-        military_bases = [
-            {"lat": 35.0, "lng": -75.0, "id": "mb-east"},
-            {"lat": -10.0, "lng": 120.0, "id": "mb-ido"},
-        ]
-        power_plants = [
-            {"lat": 35.0, "lng": -75.0, "id": "pp-east"},
-            {"lat": 35.0, "lng": 100.0, "id": "pp-asia"},
-        ]
-
         monkeypatch.setitem(_store.latest_data, "gdelt", gdelt)
-        monkeypatch.setitem(_store.latest_data, "firms_fires", firms_fires)
+        monkeypatch.setitem(_store.latest_data, "firms_fires", firms)
         monkeypatch.setitem(_store.latest_data, "datacenters", datacenters)
-        monkeypatch.setitem(_store.latest_data, "military_bases", military_bases)
-        monkeypatch.setitem(_store.latest_data, "power_plants", power_plants)
-        for layer in (
-            "global_incidents", "firms", "datacenters", "military_bases", "power_plants",
-        ):
+        for layer in ("global_incidents", "firms", "datacenters"):
             monkeypatch.setitem(_store.active_layers, layer, True)
 
     def test_no_bbox_returns_world_data(self, client, monkeypatch):
@@ -203,81 +133,40 @@ class TestSlowBboxFiltering:
         data = r.json()
         assert len(data["gdelt"]) == 2
         assert len(data["firms_fires"]) == 2
-        assert len(data["datacenters"]) == 3
+        assert len(data["datacenters"]) == 2
 
-    def test_bbox_filters_heavy_layers(self, client, monkeypatch):
+    def test_bbox_does_not_truncate_slow_layers(self, client, monkeypatch):
         self._seed_slow(monkeypatch)
         r = client.get("/api/live-data/slow?s=30&w=-80&n=40&e=-70")
         assert r.status_code == 200
         data = r.json()
-        assert {g["id"] for g in data["gdelt"]} == {"g-east"}
-        assert {f["id"] for f in data["firms_fires"]} == {"fire-east"}
-
-    def test_bbox_leaves_reference_layers_untouched(self, client, monkeypatch):
-        """Datacenters, bases, and power plants are infrastructure overlays —
-        they must remain world-scale so panning never hides them."""
-        self._seed_slow(monkeypatch)
-        r = client.get("/api/live-data/slow?s=30&w=-80&n=40&e=-70")
-        assert r.status_code == 200
-        data = r.json()
-        assert len(data["datacenters"]) == 3
-        assert len(data["military_bases"]) == 2
-        assert len(data["power_plants"]) == 2
-
-    def test_antimeridian_bbox(self, client, monkeypatch):
-        from services.fetchers import _store
-        # Box that straddles the antimeridian (Pacific): w=170, e=-170.
-        gdelt = [
-            {"lat": 0.0, "lng": 175.0, "id": "in-west"},
-            {"lat": 0.0, "lng": -175.0, "id": "in-east"},
-            {"lat": 0.0, "lng": 0.0, "id": "out-mid"},
-        ]
-        monkeypatch.setitem(_store.latest_data, "gdelt", gdelt)
-        monkeypatch.setitem(_store.active_layers, "global_incidents", True)
-        r = client.get("/api/live-data/slow?s=-10&w=170&n=10&e=-170")
-        assert r.status_code == 200
-        data = r.json()
-        ids = {g["id"] for g in data["gdelt"]}
-        assert "in-west" in ids
-        assert "in-east" in ids
-        assert "out-mid" not in ids
+        assert {g["id"] for g in data["gdelt"]} == {"g-ne", "g-asia"}
+        assert {f["id"] for f in data["firms_fires"]} == {"fire-ne", "fire-au"}
+        assert len(data["datacenters"]) == 2
 
 
-# ─────────────────── Direct helper coverage (defensive) ─────────────────────
-
-
-class TestHelpers:
+class TestBboxHelpers:
     def test_has_full_bbox(self):
         from routers.data import _has_full_bbox
         assert _has_full_bbox(1, 2, 3, 4)
         assert not _has_full_bbox(None, 2, 3, 4)
-        assert not _has_full_bbox(1, None, 3, 4)
-        assert not _has_full_bbox(1, 2, None, 4)
-        assert not _has_full_bbox(1, 2, 3, None)
 
-    def test_bbox_etag_suffix_quantizes(self):
+    def test_bbox_etag_suffix_always_empty(self):
         from routers.data import _bbox_etag_suffix
-        a = _bbox_etag_suffix(30.1, -79.6, 39.9, -70.1)
-        b = _bbox_etag_suffix(30.4, -79.2, 39.4, -70.8)
-        assert a == b, "Sub-degree pan must collapse to the same ETag suffix"
-        assert a.startswith("|bbox=")
-
-    def test_bbox_etag_suffix_world_collapses(self):
-        from routers.data import _bbox_etag_suffix
-        # World-scale → empty suffix (shares the global ETag).
+        assert _bbox_etag_suffix(30.1, -79.6, 39.9, -70.1) == ""
         assert _bbox_etag_suffix(-90, -180, 90, 180) == ""
-
-    def test_bbox_etag_suffix_partial_is_empty(self):
-        from routers.data import _bbox_etag_suffix
         assert _bbox_etag_suffix(None, -180, 90, 180) == ""
 
-    def test_apply_bbox_preserves_non_list_values(self):
+    def test_apply_bbox_is_noop(self):
         from routers.data import _apply_bbox_to_payload, _FAST_BBOX_HEAVY_KEYS
+
         payload = {
-            "commercial_flights": [{"lat": 35, "lng": -75, "id": "x"}],
-            "satellite_source": "tle",  # not a list, must pass through
-            "sigint_totals": {"total": 1},  # dict — must pass through
+            "commercial_flights": [
+                {"lat": 35.0, "lng": -75.0, "id": "in"},
+                {"lat": 35.0, "lng": 100.0, "id": "out"},
+            ],
+            "note": "keep",
         }
         out = _apply_bbox_to_payload(dict(payload), _FAST_BBOX_HEAVY_KEYS, 30, -80, 40, -70)
-        assert out["satellite_source"] == "tle"
-        assert out["sigint_totals"] == {"total": 1}
+        assert len(out["commercial_flights"]) == 2
+        assert out["note"] == "keep"

backend/tests/test_perf_guardrail_caps.py

@@ -1,4 +1,4 @@
-"""P5 / P11 guardrail tests — zoom-aware caps + CCTV lon bbox + enable path."""
+"""Live-data payload helpers — no telemetry sampling; bbox densify only."""
 from __future__ import annotations
 
 
@@ -12,7 +12,7 @@ def test_sample_items_even_stride():
     assert sampled[-1]["id"] == 90
 
 
-def test_world_zoom_caps_dense_layers_and_keeps_totals():
+def test_world_zoom_does_not_sample_telemetry():
     from routers.data import _cap_fast_dashboard_payload
 
     flights = [{"lat": 0.0, "lng": float(i), "id": f"f-{i}"} for i in range(2000)]
@@ -22,19 +22,18 @@ def test_world_zoom_caps_dense_layers_and_keeps_totals():
         "ships": [{"lat": 1.0, "lng": 1.0, "id": "s1"}],
         "cctv": cctv,
         "cctv_total": 900,
-        "satellites": [{"lat": -10.0, "lng": 10.0, "id": "sat"}],  # never capped
+        "satellites": [{"lat": -10.0, "lng": 10.0, "id": "sat"}],
     }
     out = _cap_fast_dashboard_payload(payload)
     assert out["payload_scale"] == "world"
-    assert out["payload_sampled"] is True
-    assert len(out["commercial_flights"]) == 1200
-    assert out["layer_totals"]["commercial_flights"] == 2000
-    assert len(out["cctv"]) == 600
+    assert "payload_sampled" not in out
+    assert len(out["commercial_flights"]) == 2000
+    assert len(out["cctv"]) == 900
     assert out["cctv_total"] == 900
     assert len(out["satellites"]) == 1
 
 
-def test_regional_zoom_does_not_cap():
+def test_regional_zoom_annotates_scale_without_sampling():
     from routers.data import _cap_fast_dashboard_payload
 
     flights = [{"lat": 35.0, "lng": -75.0, "id": f"f-{i}"} for i in range(1500)]
@@ -101,12 +100,13 @@ def test_get_all_cameras_column_subset(tmp_path, monkeypatch):
     assert "secret_extra" not in all_cams[0]
     assert set(all_cams[0].keys()) <= set(cctv_pipeline._CAMERA_SELECT_COLS)
 
+    # Viewport args are ignored — full catalog always.
     boxed = cctv_pipeline.get_all_cameras(south=30, west=-80, north=40, east=-70)
-    assert [c["id"] for c in boxed] == ["cam-1"]
+    assert {c["id"] for c in boxed} == {"cam-1", "cam-2"}
     assert cctv_pipeline.get_camera_count() == 2
 
 
-def test_live_data_fast_world_samples_under_cap(client, monkeypatch):
+def test_live_data_fast_world_keeps_full_telemetry(client, monkeypatch):
     from services.fetchers import _store
     from routers import data as data_router
 
@@ -122,5 +122,5 @@ def test_live_data_fast_world_samples_under_cap(client, monkeypatch):
     assert r.status_code == 200
     data = r.json()
     assert data.get("payload_scale") == "world"
-    assert len(data["commercial_flights"]) == 1200
-    assert data["layer_totals"]["commercial_flights"] == 1500
+    assert "payload_sampled" not in data
+    assert len(data["commercial_flights"]) == 1500