diff --git a/backend/routers/data.py b/backend/routers/data.py index 5a98bda..9ba0a16 100644 --- a/backend/routers/data.py +++ b/backend/routers/data.py @@ -151,48 +151,17 @@ def _has_full_bbox(s, w, n, e) -> bool: def _bbox_etag_suffix(s, w, n, e) -> str: - """Quantize bbox to 1° before mixing into the ETag. - - The 20% padding inside _bbox_filter already absorbs sub-degree pans; - quantizing here means small mouse drags don't blow the ETag cache - on the client. Full-world bounds collapse to a single suffix. - """ - if not _has_full_bbox(s, w, n, e): - return "" - try: - ss = math.floor(float(s)) - ww = math.floor(float(w)) - nn = math.ceil(float(n)) - ee = math.ceil(float(e)) - except (TypeError, ValueError): - return "" - # If the requested window covers basically the whole world, treat it as - # "no bbox" for caching purposes so world-zoomed clients all hit the - # same ETag and benefit from the existing 304 path. - lat_span, lng_span = _bbox_spans(s, w, n, e) - if lng_span >= 300 or lat_span >= 120: - return "" - return f"|bbox={ss},{ww},{nn},{ee}" + """Bbox no longer scopes payloads — keep a single world ETag cache key.""" + return "" def _apply_bbox_to_payload(payload: dict, heavy_keys: tuple[str, ...], s: float, w: float, n: float, e: float) -> dict: - """In-place filter the heavy-key collections in *payload* to a viewport. + """No-op: live telemetry is never viewport-truncated. - Items without lat/lng are passed through (so e.g. summary blobs aren't - accidentally dropped). The existing _bbox_filter helper applies a 20% - pad and handles antimeridian crossings. + Bounds may still be accepted on the wire for client hints / future use, + but we do not drop ships, aircraft, or other dense layers by map window. """ - lat_span, lng_span = _bbox_spans(s, w, n, e) - # World-scale request → skip filtering entirely. Spares the CPU and - # guarantees the response matches the no-params shape. - if lng_span >= 300 or lat_span >= 120: - return payload - for key in heavy_keys: - items = payload.get(key) - if not isinstance(items, list) or not items: - continue - payload[key] = _bbox_filter(items, s, w, n, e) return payload @@ -345,51 +314,12 @@ def _sample_items(items: list | None, max_items: int) -> list: def _cap_fast_startup_payload(payload: dict) -> dict: + """Mark first-paint payloads. Do not defer or sample live telemetry.""" capped = dict(payload) - capped["commercial_flights"] = _cap_startup_items(capped.get("commercial_flights"), 800) - capped["private_flights"] = _cap_startup_items(capped.get("private_flights"), 300) - capped["private_jets"] = _cap_startup_items(capped.get("private_jets"), 150) - capped["ships"] = _cap_startup_items(capped.get("ships"), 1500) - capped["cctv"] = [] - capped["sigint"] = _cap_startup_items(capped.get("sigint"), 500) - capped["trains"] = _cap_startup_items(capped.get("trains"), 100) capped["startup_payload"] = True return capped -# Dense layers only — never cap static infra (bases, plants, satellites, …). -# World caps keep first paint / zoomed-out views from shipping millions of rows; -# continental is looser; regional/city zoom skips capping (bbox already densifies). -_WORLD_FAST_CAPS: dict[str, int] = { - "commercial_flights": 1200, - "military_flights": 400, - "private_flights": 400, - "private_jets": 200, - "tracked_flights": 200, - "ships": 2000, - "cctv": 600, - "uavs": 200, - "liveuamap": 400, - "gps_jamming": 300, - "sigint": 800, - "trains": 200, -} -_CONTINENTAL_FAST_CAPS: dict[str, int] = { - "commercial_flights": 2500, - "military_flights": 800, - "private_flights": 800, - "private_jets": 400, - "tracked_flights": 400, - "ships": 4000, - "cctv": 1500, - "uavs": 400, - "liveuamap": 800, - "gps_jamming": 600, - "sigint": 1500, - "trains": 400, -} - - def _world_and_continental_scale(has_bbox: bool, s, w, n, e) -> tuple: lat_span, lng_span = _bbox_spans(s, w, n, e) world_scale = (not has_bbox) or lng_span >= 300 or lat_span >= 120 @@ -398,37 +328,20 @@ def _world_and_continental_scale(has_bbox: bool, s, w, n, e) -> tuple: def _cap_fast_dashboard_payload(payload: dict, *, s=None, w=None, n=None, e=None) -> dict: - """Sample dense layers at world/continental zoom only. + """Annotate zoom scale only — never sample/drop live telemetry rows. - Regional pans rely on #288 bbox filtering for density — do not strip those. - Totals for sampled keys are preserved so the UI can show "N of M". + Regional density comes from #288 bbox filtering (what's in view), not from + picking a random subset of the planet. """ has_bbox = _has_full_bbox(s, w, n, e) world_scale, continental_scale = _world_and_continental_scale(has_bbox, s, w, n, e) - if not world_scale and not continental_scale: + if world_scale: + payload["payload_scale"] = "world" + elif continental_scale: + payload["payload_scale"] = "continental" + else: payload["payload_scale"] = "regional" - return payload - - caps = _WORLD_FAST_CAPS if world_scale else _CONTINENTAL_FAST_CAPS - capped = dict(payload) - layer_totals = dict(capped.get("layer_totals") or {}) - sampled = False - for key, limit in caps.items(): - items = capped.get(key) - if not isinstance(items, list) or len(items) <= limit: - continue - layer_totals[key] = len(items) - capped[key] = _sample_items(items, limit) - sampled = True - # Keep cctv_total as the true DB/store count even when the array is sampled. - if "cctv" in layer_totals: - capped["cctv_total"] = layer_totals["cctv"] - if layer_totals: - capped["layer_totals"] = layer_totals - capped["payload_scale"] = "world" if world_scale else "continental" - if sampled: - capped["payload_sampled"] = True - return capped + return payload def _filter_sigint_by_layers(items: list, active_layers: dict) -> list: @@ -774,62 +687,29 @@ async def bootstrap_critical(request: Request): sigint_items = _filter_sigint_by_layers(d.get("sigint") or [], active_layers) return { "last_updated": d.get("last_updated"), - "commercial_flights": _cap_startup_items( - (d.get("commercial_flights") or []) if active_layers.get("flights", True) else [], - 800, - ), - "military_flights": _cap_startup_items( - (d.get("military_flights") or []) if active_layers.get("military", True) else [], - 300, - ), - "private_flights": _cap_startup_items( - (d.get("private_flights") or []) if active_layers.get("private", True) else [], - 300, - ), - "private_jets": _cap_startup_items( - (d.get("private_jets") or []) if active_layers.get("jets", True) else [], - 150, - ), - "tracked_flights": _cap_startup_items( - (d.get("tracked_flights") or []) if active_layers.get("tracked", True) else [], - 250, - ), - "ships": _cap_startup_items((d.get("ships") or []) if ships_enabled else [], 1500), - "uavs": _cap_startup_items((d.get("uavs") or []) if active_layers.get("military", True) else [], 100), - "liveuamap": _cap_startup_items( - (d.get("liveuamap") or []) if active_layers.get("global_incidents", True) else [], - 300, - ), - "gps_jamming": _cap_startup_items( - (d.get("gps_jamming") or []) if active_layers.get("gps_jamming", True) else [], - 200, - ), - "satellites": _cap_startup_items( - (d.get("satellites") or []) if active_layers.get("satellites", True) else [], - 250, - ), + "commercial_flights": (d.get("commercial_flights") or []) if active_layers.get("flights", True) else [], + "military_flights": (d.get("military_flights") or []) if active_layers.get("military", True) else [], + "private_flights": (d.get("private_flights") or []) if active_layers.get("private", True) else [], + "private_jets": (d.get("private_jets") or []) if active_layers.get("jets", True) else [], + "tracked_flights": (d.get("tracked_flights") or []) if active_layers.get("tracked", True) else [], + "ships": (d.get("ships") or []) if ships_enabled else [], + "uavs": (d.get("uavs") or []) if active_layers.get("military", True) else [], + "liveuamap": (d.get("liveuamap") or []) if active_layers.get("global_incidents", True) else [], + "gps_jamming": (d.get("gps_jamming") or []) if active_layers.get("gps_jamming", True) else [], + "satellites": (d.get("satellites") or []) if active_layers.get("satellites", True) else [], "satellite_source": d.get("satellite_source", "none"), "satellite_analysis": (d.get("satellite_analysis") or {}) if active_layers.get("satellites", True) else {}, - "sigint": _cap_startup_items( - sigint_items if (active_layers.get("sigint_meshtastic", True) or active_layers.get("sigint_aprs", True)) else [], - 500, - ), + "sigint": sigint_items if (active_layers.get("sigint_meshtastic", True) or active_layers.get("sigint_aprs", True)) else [], "sigint_totals": _sigint_totals_for_items(sigint_items), - "trains": _cap_startup_items((d.get("trains") or []) if active_layers.get("trains", True) else [], 100), - "news": _cap_startup_items(d.get("news") or [], 30), - "gdelt": _cap_startup_items((d.get("gdelt") or []) if active_layers.get("global_incidents", True) else [], 300), - "airports": _cap_startup_items(d.get("airports") or [], 500), + "trains": (d.get("trains") or []) if active_layers.get("trains", True) else [], + "news": d.get("news") or [], + "gdelt": (d.get("gdelt") or []) if active_layers.get("global_incidents", True) else [], + "airports": d.get("airports") or [], "threat_level": d.get("threat_level"), - "trending_markets": _cap_startup_items(d.get("trending_markets") or [], 10), - "correlations": _cap_startup_items( - (d.get("correlations") or []) if active_layers.get("correlations", True) else [], - 50, - ), + "trending_markets": d.get("trending_markets") or [], + "correlations": (d.get("correlations") or []) if active_layers.get("correlations", True) else [], "fimi": d.get("fimi"), - "crowdthreat": _cap_startup_items( - (d.get("crowdthreat") or []) if active_layers.get("crowdthreat", True) else [], - 150, - ), + "crowdthreat": (d.get("crowdthreat") or []) if active_layers.get("crowdthreat", True) else [], "freshness": freshness, "bootstrap_ready": True, "bootstrap_payload": True, @@ -908,14 +788,8 @@ def _try_build_fast_delta( if patch is None: return None if not patch.get("unchanged"): - # Bbox-filter upserts when a regional viewport is active. - upserts = list(patch.get("upsert") or []) - if upserts and _has_full_bbox(s, w, n, e): - lat_span, lng_span = _bbox_spans(s, w, n, e) - if not (lng_span >= 300 or lat_span >= 120): - upserts = _bbox_filter(upserts, s, w, n, e) deltas[key] = { - "upsert": upserts, + "upsert": list(patch.get("upsert") or []), "delete": list(patch.get("delete") or []), "version": patch.get("version", server_lv.get(key, 0)), } diff --git a/backend/services/ais_stream.py b/backend/services/ais_stream.py index f741a73..3973adc 100644 --- a/backend/services/ais_stream.py +++ b/backend/services/ais_stream.py @@ -361,7 +361,7 @@ _proxy_status: dict = {} _last_msg_at: float = 0.0 _proxy_spawn_count: int = 0 _VESSEL_TRAIL_INTERVAL_S = 120 -_VESSEL_TRAIL_MAX_POINTS = 240 +_VESSEL_TRAIL_MAX_POINTS = None # keep full vessel trail history # How stale "last vessel message" can be before we consider the stream @@ -435,7 +435,7 @@ def _record_vessel_trail_locked(mmsi: int, lat, lng, sog=0, now_ts: float | None return trail_data["points"].append(point) trail_data["last_seen"] = now - if len(trail_data["points"]) > _VESSEL_TRAIL_MAX_POINTS: + if _VESSEL_TRAIL_MAX_POINTS is not None and len(trail_data["points"]) > _VESSEL_TRAIL_MAX_POINTS: trail_data["points"] = trail_data["points"][-_VESSEL_TRAIL_MAX_POINTS:] diff --git a/backend/services/cctv_pipeline.py b/backend/services/cctv_pipeline.py index 174d276..a67e6b2 100644 --- a/backend/services/cctv_pipeline.py +++ b/backend/services/cctv_pipeline.py @@ -1345,7 +1345,6 @@ class NetherlandsRWSIngestor(BaseCCTVIngestor): """ URL = "https://opendata.ndw.nu/cameras.json" - MAX_CAMERAS = 1200 def fetch_data(self) -> List[Dict[str, Any]]: resp = fetch_with_curl(self.URL, timeout=25, headers={"Accept": "application/json"}) @@ -1362,7 +1361,7 @@ class NetherlandsRWSIngestor(BaseCCTVIngestor): return [] cameras: List[Dict[str, Any]] = [] - for i, cam in enumerate(data[: self.MAX_CAMERAS]): + for i, cam in enumerate(data): if not isinstance(cam, dict): continue lat = cam.get("lat") if cam.get("lat") is not None else cam.get("latitude") @@ -1475,30 +1474,14 @@ def get_all_cameras( north: float | None = None, east: float | None = None, ) -> List[Dict[str, Any]]: - """Load map-facing camera rows, optionally SQL-scoped to a viewport. - - Without bounds, returns the full catalog (store still holds world list; - /api/live-data/fast applies #288 bbox + world-zoom sampling). - """ + """Load map-facing camera rows (full catalog; viewport args ignored).""" cols = ", ".join(_CAMERA_SELECT_COLS) sql = f"SELECT {cols} FROM cameras" - params: list[Any] = [] - if None not in (south, west, north, east): - # Inclusive bbox; antimeridian (west > east) uses OR on lon. - if west <= east: - sql += " WHERE lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?" - params.extend([south, north, west, east]) - else: - sql += ( - " WHERE lat BETWEEN ? AND ?" - " AND (lon >= ? OR lon <= ?)" - ) - params.extend([south, north, west, east]) conn = sqlite3.connect(str(DB_PATH)) conn.row_factory = sqlite3.Row try: - rows = conn.execute(sql, params).fetchall() + rows = conn.execute(sql).fetchall() finally: conn.close() diff --git a/backend/services/constants.py b/backend/services/constants.py index 8790c6f..b8565fc 100644 --- a/backend/services/constants.py +++ b/backend/services/constants.py @@ -2,7 +2,7 @@ # Centralized magic numbers. Import from here instead of hardcoding. # ─── Flight Trails ────────────────────────────────────────────────────────── -FLIGHT_TRAIL_MAX_TRACKED = 2000 # Max concurrent tracked trails before LRU eviction +FLIGHT_TRAIL_MAX_TRACKED = None # no LRU eviction on flight trails FLIGHT_TRAIL_POINTS_PER_FLIGHT = 200 # Max trail points kept per aircraft TRACKED_TRAIL_TTL_S = 1800 # 30 min - trail TTL for tracked flights DEFAULT_TRAIL_TTL_S = 300 # 5 min - trail TTL for non-tracked flights diff --git a/backend/services/fetchers/flights.py b/backend/services/fetchers/flights.py index 0412bf0..5b44ae4 100644 --- a/backend/services/fetchers/flights.py +++ b/backend/services/fetchers/flights.py @@ -339,7 +339,7 @@ PRIVATE_JET_TYPES = { # Flight trails state flight_trails = {} # {icao_hex: {points: [[lat, lng, alt, ts], ...], last_seen: ts}} _trails_lock = threading.Lock() -_MAX_TRACKED_TRAILS = 20000 +_MAX_TRACKED_TRAILS = None # do not LRU-evict flight trails def get_flight_trail(icao24: str) -> list: @@ -820,7 +820,7 @@ def _classify_and_publish(all_adsb_flights): for k in stale_keys: del flight_trails[k] - if len(flight_trails) > _MAX_TRACKED_TRAILS: + if _MAX_TRACKED_TRAILS is not None and len(flight_trails) > _MAX_TRACKED_TRAILS: sorted_keys = sorted(flight_trails.keys(), key=lambda k: flight_trails[k]["last_seen"]) evict_count = len(flight_trails) - _MAX_TRACKED_TRAILS for k in sorted_keys[:evict_count]: diff --git a/backend/services/fetchers/geo.py b/backend/services/fetchers/geo.py index eae79f9..6127786 100644 --- a/backend/services/fetchers/geo.py +++ b/backend/services/fetchers/geo.py @@ -311,12 +311,12 @@ def fetch_fishing_activity(): import datetime as _dt # GFW publishes with ~5 day lag; windows shorter than ~7 days often return 0 events. - lookback_days = _gfw_int_env("GFW_EVENTS_LOOKBACK_DAYS", 7, minimum=1, maximum=14) - max_pages = _gfw_int_env("GFW_EVENTS_MAX_PAGES", 10, minimum=1, maximum=100) - timeout_s = _gfw_int_env("GFW_EVENTS_TIMEOUT_S", 90, minimum=30, maximum=180) + lookback_days = _gfw_int_env("GFW_EVENTS_LOOKBACK_DAYS", 7, minimum=1, maximum=365) + max_pages = _gfw_int_env("GFW_EVENTS_MAX_PAGES", 10_000, minimum=1, maximum=None) + timeout_s = _gfw_int_env("GFW_EVENTS_TIMEOUT_S", 90, minimum=30, maximum=600) _end = _dt.date.today().isoformat() _start = (_dt.date.today() - _dt.timedelta(days=lookback_days)).isoformat() - page_size = _gfw_int_env("GFW_EVENTS_PAGE_SIZE", 500, minimum=1, maximum=1000) + page_size = _gfw_int_env("GFW_EVENTS_PAGE_SIZE", 1000, minimum=1, maximum=1000) offset = 0 pages_fetched = 0 total_available: int | None = None diff --git a/backend/services/sigint_bridge.py b/backend/services/sigint_bridge.py index cfd0f4a..7d5a92e 100644 --- a/backend/services/sigint_bridge.py +++ b/backend/services/sigint_bridge.py @@ -32,8 +32,8 @@ from services.mesh.meshtastic_topics import all_available_roots, build_subscript logger = logging.getLogger("services.sigint") -# Maximum signals retained per bridge (prevents unbounded memory) -_MAX_SIGNALS = 500 +# Unbounded live SIGINT retention (age prune still applies). +_MAX_SIGNALS = None # Maximum age of signals before discard (seconds) _MAX_AGE_S = 600 # 10 minutes diff --git a/backend/tests/test_live_data_viewport_bbox.py b/backend/tests/test_live_data_viewport_bbox.py index 316b9c1..2b97068 100644 --- a/backend/tests/test_live_data_viewport_bbox.py +++ b/backend/tests/test_live_data_viewport_bbox.py @@ -1,59 +1,34 @@ -"""Tests for issue #288: viewport bbox filtering on /api/live-data/{fast,slow}. - -Behaviour contract: - * Without s/w/n/e params, the response is byte-for-byte identical to the - pre-#288 implementation. (No filtering, no extra fields, no ETag change.) - * With s/w/n/e supplied, heavy/dense layers are filtered to that viewport - with a 20% padding box. - * Light reference layers (datacenters, military_bases, power_plants, - satellites, news, weather, …) are NEVER filtered, even when bounds are - supplied — panning must never reveal an "empty world" of infrastructure. - * World-scale bounds (lng_span >= 300 OR lat_span >= 120) short-circuit - filtering and share the global ETag. - * The ETag includes a 1°-quantized bbox so two viewports never poison each - other's 304 cache. -""" +"""Live-data viewport params are accepted but no longer truncate telemetry.""" import pytest -# ───────────────────────── /api/live-data/fast ───────────────────────────── - - class TestFastBboxFiltering: def _seed_fast(self, monkeypatch): - """Plant deterministic heavy + light fixtures across the globe.""" from services.fetchers import _store from routers import data as data_router - # Avoid cross-test ETag byte-cache pollution. with data_router._LIVE_DATA_BYTES_CACHE_LOCK: data_router._LIVE_DATA_BYTES_CACHE.clear() _store.bump_data_version() - # Heavy collections: dense across the world. commercial = [ - {"lat": -60.0, "lng": -120.0, "id": "f-sw"}, # south Pacific - {"lat": 35.0, "lng": -75.0, "id": "f-ne"}, # eastern US - {"lat": 35.0, "lng": 100.0, "id": "f-asia"}, # Asia + {"lat": -60.0, "lng": -120.0, "id": "f-sw"}, + {"lat": 35.0, "lng": -75.0, "id": "f-ne"}, + {"lat": 35.0, "lng": 100.0, "id": "f-asia"}, ] ships = [ {"lat": -60.0, "lng": -120.0, "id": "s-sw"}, {"lat": 35.0, "lng": -75.0, "id": "s-ne"}, ] - # Real CCTV rows use ``lon`` (not ``lng``) — bbox must honor that alias. cctv = [ {"lat": 35.0, "lon": -75.0, "id": "c-1"}, {"lat": 35.0, "lon": 100.0, "id": "c-asia"}, ] - - # Sigint heavy collection. sigint = [ {"source": "meshtastic", "lat": 35.0, "lng": -75.0, "id": "sig-east"}, {"source": "meshtastic", "lat": 35.0, "lng": 100.0, "id": "sig-asia"}, ] - - # Light/reference layer — must NEVER be filtered. satellites = [ {"lat": -60.0, "lng": -120.0, "id": "sat-sw"}, {"lat": 35.0, "lng": -75.0, "id": "sat-ne"}, @@ -65,7 +40,6 @@ class TestFastBboxFiltering: monkeypatch.setitem(_store.latest_data, "cctv", cctv) monkeypatch.setitem(_store.latest_data, "sigint", sigint) monkeypatch.setitem(_store.latest_data, "satellites", satellites) - # Ensure all layers are on so the response includes them. for layer in ( "flights", "ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts", "cctv", @@ -78,37 +52,24 @@ class TestFastBboxFiltering: r = client.get("/api/live-data/fast") assert r.status_code == 200 data = r.json() - # All heavy fixtures pass through unchanged. assert len(data["commercial_flights"]) == 3 assert len(data["ships"]) == 2 assert len(data["sigint"]) == 2 - # Light layer also full. assert len(data["satellites"]) == 3 - def test_bbox_filters_heavy_layers(self, client, monkeypatch): - self._seed_fast(monkeypatch) - # Box tightly around the eastern-US fixture (lat 35, lng -75). - # ±5° → after 20% padding inside _bbox_filter, ~±6° window. - r = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70") - assert r.status_code == 200 - data = r.json() - # Heavy layers: only the eastern-US fixture survives. - assert {f["id"] for f in data["commercial_flights"]} == {"f-ne"} - assert {s["id"] for s in data["ships"]} == {"s-ne"} - assert {c["id"] for c in data["cctv"]} == {"c-1"} # lon alias filtered - assert {s["id"] for s in data["sigint"]} == {"sig-east"} - - def test_bbox_does_not_filter_light_layers(self, client, monkeypatch): + def test_bbox_does_not_truncate_heavy_layers(self, client, monkeypatch): self._seed_fast(monkeypatch) r = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70") assert r.status_code == 200 data = r.json() - # Satellites are a reference layer — must NOT be bbox-filtered. + assert {f["id"] for f in data["commercial_flights"]} == {"f-sw", "f-ne", "f-asia"} + assert {s["id"] for s in data["ships"]} == {"s-sw", "s-ne"} + assert {c["id"] for c in data["cctv"]} == {"c-1", "c-asia"} + assert {s["id"] for s in data["sigint"]} == {"sig-east", "sig-asia"} assert len(data["satellites"]) == 3 - def test_world_scale_bbox_skips_filtering(self, client, monkeypatch): + def test_world_scale_bbox_returns_full_set(self, client, monkeypatch): self._seed_fast(monkeypatch) - # lng_span = 360 → treated as world-scale; same as no bbox. r = client.get("/api/live-data/fast?s=-90&w=-180&n=90&e=180") assert r.status_code == 200 data = r.json() @@ -117,83 +78,52 @@ class TestFastBboxFiltering: def test_partial_bbox_is_treated_as_no_bbox(self, client, monkeypatch): self._seed_fast(monkeypatch) - # Only three of four bounds → filtering must NOT engage. r = client.get("/api/live-data/fast?s=30&w=-80&n=40") assert r.status_code == 200 data = r.json() assert len(data["commercial_flights"]) == 3 - def test_etag_changes_with_bbox(self, client, monkeypatch): + def test_etag_shared_across_bbox_and_world(self, client, monkeypatch): self._seed_fast(monkeypatch) r_world = client.get("/api/live-data/fast") r_local = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70") assert r_world.status_code == 200 assert r_local.status_code == 200 - etag_world = r_world.headers.get("etag") - etag_local = r_local.headers.get("etag") - assert etag_world and etag_local - assert etag_world != etag_local, ( - "ETag must differ between world and regional bbox to prevent " - "304 cache poisoning across viewports" - ) + assert r_world.headers.get("ETag") == r_local.headers.get("ETag") - def test_etag_stable_for_subdegree_pan(self, client, monkeypatch): + def test_if_none_match_returns_304(self, client, monkeypatch): self._seed_fast(monkeypatch) - # Sub-degree pan should land in the same 1°-quantized bucket. - r_a = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70") - r_b = client.get("/api/live-data/fast?s=30.3&w=-79.8&n=39.7&e=-70.4") - assert r_a.headers.get("etag") == r_b.headers.get("etag") - - def test_if_none_match_returns_304_for_same_bbox(self, client, monkeypatch): - self._seed_fast(monkeypatch) - r1 = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70") - etag = r1.headers.get("etag") - r2 = client.get( - "/api/live-data/fast?s=30&w=-80&n=40&e=-70", - headers={"If-None-Match": etag}, - ) + r1 = client.get("/api/live-data/fast") + etag = r1.headers.get("ETag") + r2 = client.get("/api/live-data/fast", headers={"If-None-Match": etag}) assert r2.status_code == 304 -# ───────────────────────── /api/live-data/slow ───────────────────────────── - - class TestSlowBboxFiltering: def _seed_slow(self, monkeypatch): from services.fetchers import _store + from routers import data as data_router + + with data_router._LIVE_DATA_BYTES_CACHE_LOCK: + data_router._LIVE_DATA_BYTES_CACHE.clear() + _store.bump_data_version() - # Heavy collections. gdelt = [ - {"lat": 35.0, "lng": -75.0, "id": "g-east"}, + {"lat": 35.0, "lng": -75.0, "id": "g-ne"}, {"lat": 35.0, "lng": 100.0, "id": "g-asia"}, ] - firms_fires = [ - {"lat": 35.0, "lng": -75.0, "id": "fire-east"}, - {"lat": -10.0, "lng": 120.0, "id": "fire-ido"}, + firms = [ + {"lat": 35.0, "lng": -75.0, "id": "fire-ne"}, + {"lat": -20.0, "lng": 140.0, "id": "fire-au"}, ] - # Light/reference layers — must always ship in full. datacenters = [ - {"lat": 35.0, "lng": -75.0, "id": "dc-east"}, - {"lat": 35.0, "lng": 100.0, "id": "dc-asia"}, - {"lat": -10.0, "lng": 120.0, "id": "dc-ido"}, + {"lat": 35.0, "lng": -75.0, "id": "dc-ne"}, + {"lat": 51.0, "lng": 0.0, "id": "dc-uk"}, ] - military_bases = [ - {"lat": 35.0, "lng": -75.0, "id": "mb-east"}, - {"lat": -10.0, "lng": 120.0, "id": "mb-ido"}, - ] - power_plants = [ - {"lat": 35.0, "lng": -75.0, "id": "pp-east"}, - {"lat": 35.0, "lng": 100.0, "id": "pp-asia"}, - ] - monkeypatch.setitem(_store.latest_data, "gdelt", gdelt) - monkeypatch.setitem(_store.latest_data, "firms_fires", firms_fires) + monkeypatch.setitem(_store.latest_data, "firms_fires", firms) monkeypatch.setitem(_store.latest_data, "datacenters", datacenters) - monkeypatch.setitem(_store.latest_data, "military_bases", military_bases) - monkeypatch.setitem(_store.latest_data, "power_plants", power_plants) - for layer in ( - "global_incidents", "firms", "datacenters", "military_bases", "power_plants", - ): + for layer in ("global_incidents", "firms", "datacenters"): monkeypatch.setitem(_store.active_layers, layer, True) def test_no_bbox_returns_world_data(self, client, monkeypatch): @@ -203,81 +133,40 @@ class TestSlowBboxFiltering: data = r.json() assert len(data["gdelt"]) == 2 assert len(data["firms_fires"]) == 2 - assert len(data["datacenters"]) == 3 + assert len(data["datacenters"]) == 2 - def test_bbox_filters_heavy_layers(self, client, monkeypatch): + def test_bbox_does_not_truncate_slow_layers(self, client, monkeypatch): self._seed_slow(monkeypatch) r = client.get("/api/live-data/slow?s=30&w=-80&n=40&e=-70") assert r.status_code == 200 data = r.json() - assert {g["id"] for g in data["gdelt"]} == {"g-east"} - assert {f["id"] for f in data["firms_fires"]} == {"fire-east"} - - def test_bbox_leaves_reference_layers_untouched(self, client, monkeypatch): - """Datacenters, bases, and power plants are infrastructure overlays — - they must remain world-scale so panning never hides them.""" - self._seed_slow(monkeypatch) - r = client.get("/api/live-data/slow?s=30&w=-80&n=40&e=-70") - assert r.status_code == 200 - data = r.json() - assert len(data["datacenters"]) == 3 - assert len(data["military_bases"]) == 2 - assert len(data["power_plants"]) == 2 - - def test_antimeridian_bbox(self, client, monkeypatch): - from services.fetchers import _store - # Box that straddles the antimeridian (Pacific): w=170, e=-170. - gdelt = [ - {"lat": 0.0, "lng": 175.0, "id": "in-west"}, - {"lat": 0.0, "lng": -175.0, "id": "in-east"}, - {"lat": 0.0, "lng": 0.0, "id": "out-mid"}, - ] - monkeypatch.setitem(_store.latest_data, "gdelt", gdelt) - monkeypatch.setitem(_store.active_layers, "global_incidents", True) - r = client.get("/api/live-data/slow?s=-10&w=170&n=10&e=-170") - assert r.status_code == 200 - data = r.json() - ids = {g["id"] for g in data["gdelt"]} - assert "in-west" in ids - assert "in-east" in ids - assert "out-mid" not in ids + assert {g["id"] for g in data["gdelt"]} == {"g-ne", "g-asia"} + assert {f["id"] for f in data["firms_fires"]} == {"fire-ne", "fire-au"} + assert len(data["datacenters"]) == 2 -# ─────────────────── Direct helper coverage (defensive) ───────────────────── - - -class TestHelpers: +class TestBboxHelpers: def test_has_full_bbox(self): from routers.data import _has_full_bbox assert _has_full_bbox(1, 2, 3, 4) assert not _has_full_bbox(None, 2, 3, 4) - assert not _has_full_bbox(1, None, 3, 4) - assert not _has_full_bbox(1, 2, None, 4) - assert not _has_full_bbox(1, 2, 3, None) - def test_bbox_etag_suffix_quantizes(self): + def test_bbox_etag_suffix_always_empty(self): from routers.data import _bbox_etag_suffix - a = _bbox_etag_suffix(30.1, -79.6, 39.9, -70.1) - b = _bbox_etag_suffix(30.4, -79.2, 39.4, -70.8) - assert a == b, "Sub-degree pan must collapse to the same ETag suffix" - assert a.startswith("|bbox=") - - def test_bbox_etag_suffix_world_collapses(self): - from routers.data import _bbox_etag_suffix - # World-scale → empty suffix (shares the global ETag). + assert _bbox_etag_suffix(30.1, -79.6, 39.9, -70.1) == "" assert _bbox_etag_suffix(-90, -180, 90, 180) == "" - - def test_bbox_etag_suffix_partial_is_empty(self): - from routers.data import _bbox_etag_suffix assert _bbox_etag_suffix(None, -180, 90, 180) == "" - def test_apply_bbox_preserves_non_list_values(self): + def test_apply_bbox_is_noop(self): from routers.data import _apply_bbox_to_payload, _FAST_BBOX_HEAVY_KEYS + payload = { - "commercial_flights": [{"lat": 35, "lng": -75, "id": "x"}], - "satellite_source": "tle", # not a list, must pass through - "sigint_totals": {"total": 1}, # dict — must pass through + "commercial_flights": [ + {"lat": 35.0, "lng": -75.0, "id": "in"}, + {"lat": 35.0, "lng": 100.0, "id": "out"}, + ], + "note": "keep", } out = _apply_bbox_to_payload(dict(payload), _FAST_BBOX_HEAVY_KEYS, 30, -80, 40, -70) - assert out["satellite_source"] == "tle" - assert out["sigint_totals"] == {"total": 1} + assert len(out["commercial_flights"]) == 2 + assert out["note"] == "keep" diff --git a/backend/tests/test_perf_guardrail_caps.py b/backend/tests/test_perf_guardrail_caps.py index 8d2b182..81bc50c 100644 --- a/backend/tests/test_perf_guardrail_caps.py +++ b/backend/tests/test_perf_guardrail_caps.py @@ -1,4 +1,4 @@ -"""P5 / P11 guardrail tests — zoom-aware caps + CCTV lon bbox + enable path.""" +"""Live-data payload helpers — no telemetry sampling; bbox densify only.""" from __future__ import annotations @@ -12,7 +12,7 @@ def test_sample_items_even_stride(): assert sampled[-1]["id"] == 90 -def test_world_zoom_caps_dense_layers_and_keeps_totals(): +def test_world_zoom_does_not_sample_telemetry(): from routers.data import _cap_fast_dashboard_payload flights = [{"lat": 0.0, "lng": float(i), "id": f"f-{i}"} for i in range(2000)] @@ -22,19 +22,18 @@ def test_world_zoom_caps_dense_layers_and_keeps_totals(): "ships": [{"lat": 1.0, "lng": 1.0, "id": "s1"}], "cctv": cctv, "cctv_total": 900, - "satellites": [{"lat": -10.0, "lng": 10.0, "id": "sat"}], # never capped + "satellites": [{"lat": -10.0, "lng": 10.0, "id": "sat"}], } out = _cap_fast_dashboard_payload(payload) assert out["payload_scale"] == "world" - assert out["payload_sampled"] is True - assert len(out["commercial_flights"]) == 1200 - assert out["layer_totals"]["commercial_flights"] == 2000 - assert len(out["cctv"]) == 600 + assert "payload_sampled" not in out + assert len(out["commercial_flights"]) == 2000 + assert len(out["cctv"]) == 900 assert out["cctv_total"] == 900 assert len(out["satellites"]) == 1 -def test_regional_zoom_does_not_cap(): +def test_regional_zoom_annotates_scale_without_sampling(): from routers.data import _cap_fast_dashboard_payload flights = [{"lat": 35.0, "lng": -75.0, "id": f"f-{i}"} for i in range(1500)] @@ -101,12 +100,13 @@ def test_get_all_cameras_column_subset(tmp_path, monkeypatch): assert "secret_extra" not in all_cams[0] assert set(all_cams[0].keys()) <= set(cctv_pipeline._CAMERA_SELECT_COLS) + # Viewport args are ignored — full catalog always. boxed = cctv_pipeline.get_all_cameras(south=30, west=-80, north=40, east=-70) - assert [c["id"] for c in boxed] == ["cam-1"] + assert {c["id"] for c in boxed} == {"cam-1", "cam-2"} assert cctv_pipeline.get_camera_count() == 2 -def test_live_data_fast_world_samples_under_cap(client, monkeypatch): +def test_live_data_fast_world_keeps_full_telemetry(client, monkeypatch): from services.fetchers import _store from routers import data as data_router @@ -122,5 +122,5 @@ def test_live_data_fast_world_samples_under_cap(client, monkeypatch): assert r.status_code == 200 data = r.json() assert data.get("payload_scale") == "world" - assert len(data["commercial_flights"]) == 1200 - assert data["layer_totals"]["commercial_flights"] == 1500 + assert "payload_sampled" not in data + assert len(data["commercial_flights"]) == 1500