feat: integrate AI codebase optimizations (memory safety, spatial hashing, centralized API base)

backend/main.py

@@ -112,7 +112,25 @@ async def debug_latest_data():
 
 @app.get("/api/health")
 async def health_check():
-    return {"status": "ok"}
+    import time
+    d = get_latest_data()
+    last = d.get("last_updated")
+    return {
+        "status": "ok",
+        "last_updated": last,
+        "sources": {
+            "flights": len(d.get("commercial_flights", [])),
+            "military": len(d.get("military_flights", [])),
+            "ships": len(d.get("ships", [])),
+            "satellites": len(d.get("satellites", [])),
+            "earthquakes": len(d.get("earthquakes", [])),
+            "cctv": len(d.get("cctv", [])),
+            "news": len(d.get("news", [])),
+        },
+        "uptime_seconds": round(time.time() - _start_time),
+    }
+
+_start_time = __import__("time").time()
 
 from services.radio_intercept import get_top_broadcastify_feeds, get_openmhz_systems, get_recent_openmhz_calls, find_nearest_openmhz_system
 

backend/services/ais_stream.py

@@ -211,9 +211,10 @@ def _ais_stream_loop():
     """Main loop: spawn node proxy and process messages from stdout."""
     import subprocess
     import os
-    
+
     proxy_script = os.path.join(os.path.dirname(os.path.dirname(__file__)), "ais_proxy.js")
-    
+    backoff = 1  # Exponential backoff starting at 1 second
+
     while _ws_running:
         try:
             logger.info("Starting Node.js AIS Stream Proxy...")
@@ -323,8 +324,12 @@ def _ais_stream_loop():
         except Exception as e:
             logger.error(f"AIS proxy connection error: {e}")
             if _ws_running:
-                logger.info("Restarting AIS proxy in 5 seconds...")
-                time.sleep(5)
+                logger.info(f"Restarting AIS proxy in {backoff}s (exponential backoff)...")
+                time.sleep(backoff)
+                backoff = min(backoff * 2, 60)  # Double up to 60s max
+            continue
+        # Reset backoff on successful connection (got at least some messages)
+        backoff = 1
 
 
 def _run_ais_loop():

backend/services/data_fetcher.py

@@ -72,8 +72,8 @@ class OpenSkyClient:
 
 # User provided credentials
 opensky_client = OpenSkyClient(
-    client_id=os.environ.get("OPENSKY_CLIENT_ID", "vancecook-api-client"),
-    client_secret=os.environ.get("OPENSKY_CLIENT_SECRET", "YOUR_OPENSKY_SECRET")
+    client_id=os.environ.get("OPENSKY_CLIENT_ID", ""),
+    client_secret=os.environ.get("OPENSKY_CLIENT_SECRET", "")
 )
 
 # Throttling and caching for OpenSky to observe the 400 req/day limit
@@ -885,9 +885,10 @@ def fetch_flights():
                     by_icao[id(f)] = f  # no icao — keep as unique
             return list(by_icao.values())
 
-        latest_data['commercial_flights'] = _merge_category(commercial, latest_data.get('commercial_flights', []))
-        latest_data['private_jets'] = _merge_category(private_jets, latest_data.get('private_jets', []))
-        latest_data['private_flights'] = _merge_category(private_ga, latest_data.get('private_flights', []))
+        with _data_lock:
+            latest_data['commercial_flights'] = _merge_category(commercial, latest_data.get('commercial_flights', []))
+            latest_data['private_jets'] = _merge_category(private_jets, latest_data.get('private_jets', []))
+            latest_data['private_flights'] = _merge_category(private_ga, latest_data.get('private_flights', []))
 
     # Always write raw flights for GPS jamming analysis (nac_p field)
     if flights:
@@ -964,27 +965,39 @@ def fetch_flights():
     all_lists = [commercial, private_jets, private_ga, existing_tracked]
     seen_hexes = set()
     trail_count = 0
-    for flist in all_lists:
-        for f in flist:
-            count, hex_id = _accumulate_trail(f, now_ts, check_route=True)
+    with _trails_lock:
+        for flist in all_lists:
+            for f in flist:
+                count, hex_id = _accumulate_trail(f, now_ts, check_route=True)
+                trail_count += count
+                if hex_id:
+                    seen_hexes.add(hex_id)
+
+        # Also process military flights (separate list)
+        for mf in latest_data.get('military_flights', []):
+            count, hex_id = _accumulate_trail(mf, now_ts, check_route=False)
             trail_count += count
             if hex_id:
                 seen_hexes.add(hex_id)
-    
-    # Also process military flights (separate list)
-    for mf in latest_data.get('military_flights', []):
-        count, hex_id = _accumulate_trail(mf, now_ts, check_route=False)
-        trail_count += count
-        if hex_id:
-            seen_hexes.add(hex_id)
-    
-    # Prune trails for aircraft not seen in 30 minutes
-    stale_cutoff = now_ts - 1800
-    stale_keys = [k for k, v in flight_trails.items() if v['last_seen'] < stale_cutoff]
-    for k in stale_keys:
-        del flight_trails[k]
-    
-    logger.info(f"Trail accumulation: {trail_count} active trails, {len(stale_keys)} pruned")
+
+        # Prune stale trails (10 min for non-tracked, 30 min for tracked)
+        tracked_hexes = {t.get('icao24', '').lower() for t in latest_data.get('tracked_flights', [])}
+        stale_keys = []
+        for k, v in flight_trails.items():
+            cutoff = now_ts - 1800 if k in tracked_hexes else now_ts - 600
+            if v['last_seen'] < cutoff:
+                stale_keys.append(k)
+        for k in stale_keys:
+            del flight_trails[k]
+
+        # Enforce global cap — evict oldest trails first
+        if len(flight_trails) > _MAX_TRACKED_TRAILS:
+            sorted_keys = sorted(flight_trails.keys(), key=lambda k: flight_trails[k]['last_seen'])
+            evict_count = len(flight_trails) - _MAX_TRACKED_TRAILS
+            for k in sorted_keys[:evict_count]:
+                del flight_trails[k]
+
+    logger.info(f"Trail accumulation: {trail_count} active trails, {len(stale_keys)} pruned, {len(flight_trails)} total")
 
     # -----------------------------------------------------------------------
     # GPS / GNSS Jamming Detection — aggregate NACp from ADS-B transponders
@@ -1567,6 +1580,8 @@ def fetch_uavs():
 
 cached_airports = []
 flight_trails = {}  # {icao_hex: {points: [[lat, lng, alt, ts], ...], last_seen: ts}}
+_trails_lock = threading.Lock()
+_MAX_TRACKED_TRAILS = 2000  # Global cap on number of aircraft trails in memory
 
 # (math imported at module top)
 
@@ -1751,5 +1766,6 @@ def stop_scheduler():
     scheduler.shutdown()
 
 def get_latest_data():
-    return latest_data
+    with _data_lock:
+        return dict(latest_data)
 

backend/services/network_utils.py

@@ -3,10 +3,19 @@ import json
 import subprocess
 import shutil
 import time
+import requests
 from urllib.parse import urlparse
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 
 logger = logging.getLogger(__name__)
 
+# Reusable session with connection pooling and retry logic
+_session = requests.Session()
+_retry = Retry(total=2, backoff_factor=0.5, status_forcelist=[502, 503, 504])
+_session.mount("https://", HTTPAdapter(max_retries=_retry, pool_maxsize=20))
+_session.mount("http://", HTTPAdapter(max_retries=_retry, pool_maxsize=10))
+
 # Find bash for curl fallback — Git bash's curl has the TLS features
 # needed to pass CDN fingerprint checks (brotli, zstd, libpsl)
 _BASH_PATH = shutil.which("bash") or "bash"
@@ -50,11 +59,10 @@ def fetch_with_curl(url, method="GET", json_data=None, timeout=15, headers=None)
         pass  # Fall through to curl below
     else:
         try:
-            import requests
             if method == "POST":
-                res = requests.post(url, json=json_data, timeout=timeout, headers=default_headers)
+                res = _session.post(url, json=json_data, timeout=timeout, headers=default_headers)
             else:
-                res = requests.get(url, timeout=timeout, headers=default_headers)
+                res = _session.get(url, timeout=timeout, headers=default_headers)
             res.raise_for_status()
             # Clear failure cache on success
             _domain_fail_cache.pop(domain, None)