fix: stop truncating live telemetry payloads

Remove sampling, viewport bbox truncation, and buffer ceilings that were cherry-picking ships, flights, and related layers.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/services/ais_stream.py

@@ -361,7 +361,7 @@ _proxy_status: dict = {}
 _last_msg_at: float = 0.0
 _proxy_spawn_count: int = 0
 _VESSEL_TRAIL_INTERVAL_S = 120
-_VESSEL_TRAIL_MAX_POINTS = 240
+_VESSEL_TRAIL_MAX_POINTS = None  # keep full vessel trail history
 
 
 # How stale "last vessel message" can be before we consider the stream
@@ -435,7 +435,7 @@ def _record_vessel_trail_locked(mmsi: int, lat, lng, sog=0, now_ts: float | None
         return
     trail_data["points"].append(point)
     trail_data["last_seen"] = now
-    if len(trail_data["points"]) > _VESSEL_TRAIL_MAX_POINTS:
+    if _VESSEL_TRAIL_MAX_POINTS is not None and len(trail_data["points"]) > _VESSEL_TRAIL_MAX_POINTS:
         trail_data["points"] = trail_data["points"][-_VESSEL_TRAIL_MAX_POINTS:]
 
 

backend/services/cctv_pipeline.py

@@ -1345,7 +1345,6 @@ class NetherlandsRWSIngestor(BaseCCTVIngestor):
     """
 
     URL = "https://opendata.ndw.nu/cameras.json"
-    MAX_CAMERAS = 1200
 
     def fetch_data(self) -> List[Dict[str, Any]]:
         resp = fetch_with_curl(self.URL, timeout=25, headers={"Accept": "application/json"})
@@ -1362,7 +1361,7 @@ class NetherlandsRWSIngestor(BaseCCTVIngestor):
             return []
 
         cameras: List[Dict[str, Any]] = []
-        for i, cam in enumerate(data[: self.MAX_CAMERAS]):
+        for i, cam in enumerate(data):
             if not isinstance(cam, dict):
                 continue
             lat = cam.get("lat") if cam.get("lat") is not None else cam.get("latitude")
@@ -1475,30 +1474,14 @@ def get_all_cameras(
     north: float | None = None,
     east: float | None = None,
 ) -> List[Dict[str, Any]]:
-    """Load map-facing camera rows, optionally SQL-scoped to a viewport.
-
-    Without bounds, returns the full catalog (store still holds world list;
-    /api/live-data/fast applies #288 bbox + world-zoom sampling).
-    """
+    """Load map-facing camera rows (full catalog; viewport args ignored)."""
     cols = ", ".join(_CAMERA_SELECT_COLS)
     sql = f"SELECT {cols} FROM cameras"
-    params: list[Any] = []
-    if None not in (south, west, north, east):
-        # Inclusive bbox; antimeridian (west > east) uses OR on lon.
-        if west <= east:
-            sql += " WHERE lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?"
-            params.extend([south, north, west, east])
-        else:
-            sql += (
-                " WHERE lat BETWEEN ? AND ?"
-                " AND (lon >= ? OR lon <= ?)"
-            )
-            params.extend([south, north, west, east])
 
     conn = sqlite3.connect(str(DB_PATH))
     conn.row_factory = sqlite3.Row
     try:
-        rows = conn.execute(sql, params).fetchall()
+        rows = conn.execute(sql).fetchall()
     finally:
         conn.close()
 

backend/services/constants.py

@@ -2,7 +2,7 @@
 # Centralized magic numbers. Import from here instead of hardcoding.
 
 # ─── Flight Trails ──────────────────────────────────────────────────────────
-FLIGHT_TRAIL_MAX_TRACKED = 2000  # Max concurrent tracked trails before LRU eviction
+FLIGHT_TRAIL_MAX_TRACKED = None  # no LRU eviction on flight trails
 FLIGHT_TRAIL_POINTS_PER_FLIGHT = 200  # Max trail points kept per aircraft
 TRACKED_TRAIL_TTL_S = 1800  # 30 min - trail TTL for tracked flights
 DEFAULT_TRAIL_TTL_S = 300  # 5 min - trail TTL for non-tracked flights

backend/services/fetchers/flights.py

@@ -339,7 +339,7 @@ PRIVATE_JET_TYPES = {
 # Flight trails state
 flight_trails = {}  # {icao_hex: {points: [[lat, lng, alt, ts], ...], last_seen: ts}}
 _trails_lock = threading.Lock()
-_MAX_TRACKED_TRAILS = 20000
+_MAX_TRACKED_TRAILS = None  # do not LRU-evict flight trails
 
 
 def get_flight_trail(icao24: str) -> list:
@@ -820,7 +820,7 @@ def _classify_and_publish(all_adsb_flights):
         for k in stale_keys:
             del flight_trails[k]
 
-        if len(flight_trails) > _MAX_TRACKED_TRAILS:
+        if _MAX_TRACKED_TRAILS is not None and len(flight_trails) > _MAX_TRACKED_TRAILS:
             sorted_keys = sorted(flight_trails.keys(), key=lambda k: flight_trails[k]["last_seen"])
             evict_count = len(flight_trails) - _MAX_TRACKED_TRAILS
             for k in sorted_keys[:evict_count]:

backend/services/fetchers/geo.py

@@ -311,12 +311,12 @@ def fetch_fishing_activity():
         import datetime as _dt
 
         # GFW publishes with ~5 day lag; windows shorter than ~7 days often return 0 events.
-        lookback_days = _gfw_int_env("GFW_EVENTS_LOOKBACK_DAYS", 7, minimum=1, maximum=14)
-        max_pages = _gfw_int_env("GFW_EVENTS_MAX_PAGES", 10, minimum=1, maximum=100)
-        timeout_s = _gfw_int_env("GFW_EVENTS_TIMEOUT_S", 90, minimum=30, maximum=180)
+        lookback_days = _gfw_int_env("GFW_EVENTS_LOOKBACK_DAYS", 7, minimum=1, maximum=365)
+        max_pages = _gfw_int_env("GFW_EVENTS_MAX_PAGES", 10_000, minimum=1, maximum=None)
+        timeout_s = _gfw_int_env("GFW_EVENTS_TIMEOUT_S", 90, minimum=30, maximum=600)
         _end = _dt.date.today().isoformat()
         _start = (_dt.date.today() - _dt.timedelta(days=lookback_days)).isoformat()
-        page_size = _gfw_int_env("GFW_EVENTS_PAGE_SIZE", 500, minimum=1, maximum=1000)
+        page_size = _gfw_int_env("GFW_EVENTS_PAGE_SIZE", 1000, minimum=1, maximum=1000)
         offset = 0
         pages_fetched = 0
         total_available: int | None = None

backend/services/sigint_bridge.py

@@ -32,8 +32,8 @@ from services.mesh.meshtastic_topics import all_available_roots, build_subscript
 
 logger = logging.getLogger("services.sigint")
 
-# Maximum signals retained per bridge (prevents unbounded memory)
-_MAX_SIGNALS = 500
+# Unbounded live SIGINT retention (age prune still applies).
+_MAX_SIGNALS = None
 # Maximum age of signals before discard (seconds)
 _MAX_AGE_S = 600  # 10 minutes