v0.9.6: InfoNet hashchain, Wormhole gate encryption, mesh reputation, 16 community contributors

Gate messages now propagate via the Infonet hashchain as encrypted blobs — every node syncs them
through normal chain sync while only Gate members with MLS keys can decrypt. Added mesh reputation
system, peer push workers, voluntary Wormhole opt-in for node participation, fork recovery,
killwormhole scripts, obfuscated terminology, and hardened the self-updater to protect encryption
keys and chain state during updates.

New features: Shodan search, train tracking, Sentinel Hub imagery, 8 new intelligence layers,
CCTV expansion to 11,000+ cameras across 6 countries, Mesh Terminal CLI, prediction markets,
desktop-shell scaffold, and comprehensive mesh test suite (215 frontend + backend tests passing).

Community contributors: @wa1id, @AlborzNazari, @adust09, @Xpirix, @imqdcr, @csysp, @suranyami,
@chr0n1x, @johan-martensson, @singularfailure, @smithbh, @OrfeoTerkuci, @deuza, @tm-const,
@Elhard1, @ttulttul

backend/services/fetchers/_store.py

@@ -3,14 +3,68 @@
 Central location for latest_data, source_timestamps, and the data lock.
 Every fetcher imports from here instead of maintaining its own copy.
 """
+
 import threading
 import logging
 from datetime import datetime
+from typing import Any, Dict, List, Optional, TypedDict
 
 logger = logging.getLogger("services.data_fetcher")
 
+
+class DashboardData(TypedDict, total=False):
+    """Schema for the in-memory data store. Catches key typos at dev time."""
+
+    last_updated: Optional[str]
+    news: List[Dict[str, Any]]
+    stocks: Dict[str, Any]
+    oil: Dict[str, Any]
+    commercial_flights: List[Dict[str, Any]]
+    private_flights: List[Dict[str, Any]]
+    private_jets: List[Dict[str, Any]]
+    flights: List[Dict[str, Any]]
+    ships: List[Dict[str, Any]]
+    military_flights: List[Dict[str, Any]]
+    tracked_flights: List[Dict[str, Any]]
+    cctv: List[Dict[str, Any]]
+    weather: Optional[Dict[str, Any]]
+    earthquakes: List[Dict[str, Any]]
+    uavs: List[Dict[str, Any]]
+    frontlines: Optional[Any]
+    gdelt: List[Dict[str, Any]]
+    liveuamap: List[Dict[str, Any]]
+    kiwisdr: List[Dict[str, Any]]
+    space_weather: Optional[Dict[str, Any]]
+    internet_outages: List[Dict[str, Any]]
+    firms_fires: List[Dict[str, Any]]
+    datacenters: List[Dict[str, Any]]
+    airports: List[Dict[str, Any]]
+    gps_jamming: List[Dict[str, Any]]
+    satellites: List[Dict[str, Any]]
+    satellite_source: str
+    prediction_markets: List[Dict[str, Any]]
+    sigint: List[Dict[str, Any]]
+    sigint_totals: Dict[str, Any]
+    mesh_channel_stats: Dict[str, Any]
+    meshtastic_map_nodes: List[Dict[str, Any]]
+    meshtastic_map_fetched_at: Optional[float]
+    weather_alerts: List[Dict[str, Any]]
+    air_quality: List[Dict[str, Any]]
+    volcanoes: List[Dict[str, Any]]
+    fishing_activity: List[Dict[str, Any]]
+    satnogs_stations: List[Dict[str, Any]]
+    satnogs_observations: List[Dict[str, Any]]
+    tinygs_satellites: List[Dict[str, Any]]
+    ukraine_alerts: List[Dict[str, Any]]
+    power_plants: List[Dict[str, Any]]
+    viirs_change_nodes: List[Dict[str, Any]]
+    fimi: Dict[str, Any]
+    psk_reporter: List[Dict[str, Any]]
+    correlations: List[Dict[str, Any]]
+
+
 # In-memory store
-latest_data = {
+latest_data: DashboardData = {
     "last_updated": None,
     "news": [],
     "stocks": {},
@@ -32,17 +86,158 @@ latest_data = {
     "firms_fires": [],
     "datacenters": [],
     "military_bases": [],
-    "power_plants": []
+    "prediction_markets": [],
+    "sigint": [],
+    "sigint_totals": {},
+    "mesh_channel_stats": {},
+    "meshtastic_map_nodes": [],
+    "meshtastic_map_fetched_at": None,
+    "weather_alerts": [],
+    "air_quality": [],
+    "volcanoes": [],
+    "fishing_activity": [],
+    "satnogs_stations": [],
+    "satnogs_observations": [],
+    "tinygs_satellites": [],
+    "ukraine_alerts": [],
+    "power_plants": [],
+    "viirs_change_nodes": [],
+    "fimi": {},
+    "psk_reporter": [],
+    "correlations": [],
 }
 
 # Per-source freshness timestamps
 source_timestamps = {}
 
+# Per-source health/freshness metadata (last ok/error)
+source_freshness: dict[str, dict] = {}
+
+
 def _mark_fresh(*keys):
     """Record the current UTC time for one or more data source keys."""
     now = datetime.utcnow().isoformat()
-    for k in keys:
-        source_timestamps[k] = now
+    with _data_lock:
+        for k in keys:
+            source_timestamps[k] = now
+
 
 # Thread lock for safe reads/writes to latest_data
 _data_lock = threading.Lock()
+
+# Monotonic version counter — incremented on each data update cycle.
+# Used for cheap ETag generation instead of MD5-hashing the full response.
+_data_version: int = 0
+
+
+def bump_data_version() -> None:
+    """Increment the data version counter after a fetch cycle completes."""
+    global _data_version
+    _data_version += 1
+
+
+def get_data_version() -> int:
+    """Return the current data version (for ETag generation)."""
+    return _data_version
+
+
+_active_layers_version: int = 0
+
+
+def bump_active_layers_version() -> None:
+    """Increment the active-layer version when frontend toggles change response shape."""
+    global _active_layers_version
+    _active_layers_version += 1
+
+
+def get_active_layers_version() -> int:
+    """Return the current active-layer version (for ETag generation)."""
+    return _active_layers_version
+
+
+def get_latest_data_subset(*keys: str) -> DashboardData:
+    """Return a shallow snapshot of only the requested top-level keys.
+
+    This avoids cloning the entire dashboard store for endpoints that only need
+    a small tier-specific subset.
+    """
+    with _data_lock:
+        snap: DashboardData = {}
+        for key in keys:
+            value = latest_data.get(key)
+            if isinstance(value, list):
+                snap[key] = list(value)
+            elif isinstance(value, dict):
+                snap[key] = dict(value)
+            else:
+                snap[key] = value
+        return snap
+
+
+def get_latest_data_subset_refs(*keys: str) -> DashboardData:
+    """Return direct top-level references for read-only hot paths.
+
+    Writers replace top-level values under the lock instead of mutating them
+    in place, so readers can safely use these references after releasing the
+    lock as long as they do not modify them.
+    """
+    with _data_lock:
+        snap: DashboardData = {}
+        for key in keys:
+            snap[key] = latest_data.get(key)
+        return snap
+
+
+def get_source_timestamps_snapshot() -> dict[str, str]:
+    """Return a stable copy of per-source freshness timestamps."""
+    with _data_lock:
+        return dict(source_timestamps)
+
+
+# ---------------------------------------------------------------------------
+# Active layers — frontend POSTs toggles, fetchers check before running.
+# Keep these aligned with the dashboard's default layer state so startup does
+# not fetch heavyweight feeds the UI starts with disabled.
+# ---------------------------------------------------------------------------
+active_layers: dict[str, bool] = {
+    "flights": True,
+    "private": True,
+    "jets": True,
+    "military": True,
+    "tracked": True,
+    "satellites": True,
+    "ships_military": True,
+    "ships_cargo": True,
+    "ships_civilian": True,
+    "ships_passenger": True,
+    "ships_tracked_yachts": True,
+    "earthquakes": True,
+    "cctv": True,
+    "ukraine_frontline": True,
+    "global_incidents": True,
+    "gps_jamming": True,
+    "kiwisdr": True,
+    "scanners": True,
+    "firms": True,
+    "internet_outages": True,
+    "datacenters": True,
+    "military_bases": True,
+    "sigint_meshtastic": True,
+    "sigint_aprs": True,
+    "weather_alerts": True,
+    "air_quality": True,
+    "volcanoes": True,
+    "fishing_activity": True,
+    "satnogs": True,
+    "tinygs": True,
+    "ukraine_alerts": True,
+    "power_plants": False,
+    "viirs_nightlights": False,
+    "psk_reporter": True,
+    "correlations": True,
+}
+
+
+def is_any_active(*layer_names: str) -> bool:
+    """Return True if any of the given layer names is currently active."""
+    return any(active_layers.get(name, True) for name in layer_names)

backend/services/fetchers/earth_observation.py

@@ -1,8 +1,15 @@
-"""Earth-observation fetchers — earthquakes, FIRMS fires, space weather, weather radar."""
+"""Earth-observation fetchers — earthquakes, FIRMS fires, space weather, weather radar,
+severe weather alerts, air quality, volcanoes."""
+
 import csv
 import io
+import json
 import logging
+import os
+import time
 import heapq
+from datetime import datetime
+from pathlib import Path
 from services.network_utils import fetch_with_curl
 from services.fetchers._store import latest_data, _data_lock, _mark_fresh
 from services.fetchers.retry import with_retry
@@ -15,6 +22,10 @@ logger = logging.getLogger(__name__)
 # ---------------------------------------------------------------------------
 @with_retry(max_retries=1, base_delay=1)
 def fetch_earthquakes():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("earthquakes"):
+        return
     quakes = []
     try:
         url = "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/2.5_day.geojson"
@@ -24,12 +35,16 @@ def fetch_earthquakes():
             for f in features[:50]:
                 mag = f["properties"]["mag"]
                 lng, lat, depth = f["geometry"]["coordinates"]
-                quakes.append({
-                    "id": f["id"], "mag": mag,
-                    "lat": lat, "lng": lng,
-                    "place": f["properties"]["place"]
-                })
-    except Exception as e:
+                quakes.append(
+                    {
+                        "id": f["id"],
+                        "mag": mag,
+                        "lat": lat,
+                        "lng": lng,
+                        "place": f["properties"]["place"],
+                    }
+                )
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
         logger.error(f"Error fetching earthquakes: {e}")
     with _data_lock:
         latest_data["earthquakes"] = quakes
@@ -43,6 +58,10 @@ def fetch_earthquakes():
 @with_retry(max_retries=1, base_delay=2)
 def fetch_firms_fires():
     """Fetch global fire/thermal anomalies from NASA FIRMS (NOAA-20 VIIRS, 24h, no key needed)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("firms"):
+        return
     fires = []
     try:
         url = "https://firms.modaps.eosdis.nasa.gov/data/active_fire/noaa-20-viirs-c2/csv/J1_VIIRS_C2_Global_24h.csv"
@@ -58,18 +77,23 @@ def fetch_firms_fires():
                     conf = row.get("confidence", "nominal")
                     daynight = row.get("daynight", "")
                     bright = float(row.get("bright_ti4", 0))
-                    all_rows.append({
-                        "lat": lat, "lng": lng, "frp": frp,
-                        "brightness": bright, "confidence": conf,
-                        "daynight": daynight,
-                        "acq_date": row.get("acq_date", ""),
-                        "acq_time": row.get("acq_time", ""),
-                    })
+                    all_rows.append(
+                        {
+                            "lat": lat,
+                            "lng": lng,
+                            "frp": frp,
+                            "brightness": bright,
+                            "confidence": conf,
+                            "daynight": daynight,
+                            "acq_date": row.get("acq_date", ""),
+                            "acq_time": row.get("acq_time", ""),
+                        }
+                    )
                 except (ValueError, TypeError):
                     continue
             fires = heapq.nlargest(5000, all_rows, key=lambda x: x["frp"])
         logger.info(f"FIRMS fires: {len(fires)} hotspots (from {response.status_code})")
-    except Exception as e:
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
         logger.error(f"Error fetching FIRMS fires: {e}")
     with _data_lock:
         latest_data["firms_fires"] = fires
@@ -77,6 +101,90 @@ def fetch_firms_fires():
         _mark_fresh("firms_fires")
 
 
+# ---------------------------------------------------------------------------
+# NASA FIRMS Country-Scoped Fires (enriches global CSV with conflict zones)
+# ---------------------------------------------------------------------------
+# Conflict-zone countries of interest for higher-detail fire/thermal data
+_FIRMS_COUNTRIES = ["ISR", "IRN", "IRQ", "LBN", "SYR", "YEM", "SAU", "UKR", "RUS", "TUR"]
+
+
+@with_retry(max_retries=1, base_delay=2)
+def fetch_firms_country_fires():
+    """Fetch country-scoped fire hotspots from NASA FIRMS MAP_KEY API.
+
+    Supplements the global CSV feed with more granular data for conflict zones.
+    Merges results into the existing firms_fires data store (no new frontend key).
+    Requires FIRMS_MAP_KEY env var (free from NASA Earthdata). Skips if not set.
+    """
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("firms"):
+        return
+
+    map_key = os.environ.get("FIRMS_MAP_KEY", "")
+    if not map_key:
+        logger.debug("FIRMS_MAP_KEY not set, skipping country-scoped FIRMS fetch")
+        return
+
+    # Build a set of existing (lat, lng) rounded to 0.01° for dedup
+    with _data_lock:
+        existing = set()
+        for f in latest_data.get("firms_fires", []):
+            existing.add((round(f["lat"], 2), round(f["lng"], 2)))
+
+    new_fires = []
+    for country in _FIRMS_COUNTRIES:
+        try:
+            url = (
+                f"https://firms.modaps.eosdis.nasa.gov/api/country/csv/"
+                f"{map_key}/VIIRS_NOAA20_NRT/{country}/1"
+            )
+            response = fetch_with_curl(url, timeout=15)
+            if response.status_code != 200:
+                logger.debug(f"FIRMS country {country}: HTTP {response.status_code}")
+                continue
+
+            reader = csv.DictReader(io.StringIO(response.text))
+            for row in reader:
+                try:
+                    lat = float(row.get("latitude", 0))
+                    lng = float(row.get("longitude", 0))
+                    key = (round(lat, 2), round(lng, 2))
+                    if key in existing:
+                        continue  # Already in global data
+                    existing.add(key)
+
+                    frp = float(row.get("frp", 0))
+                    new_fires.append({
+                        "lat": lat,
+                        "lng": lng,
+                        "frp": frp,
+                        "brightness": float(row.get("bright_ti4", 0)),
+                        "confidence": row.get("confidence", "nominal"),
+                        "daynight": row.get("daynight", ""),
+                        "acq_date": row.get("acq_date", ""),
+                        "acq_time": row.get("acq_time", ""),
+                    })
+                except (ValueError, TypeError):
+                    continue
+
+        except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+            logger.debug(f"FIRMS country {country} failed: {e}")
+
+    if new_fires:
+        with _data_lock:
+            current = latest_data.get("firms_fires", [])
+            merged = current + new_fires
+            # Keep top 6000 by FRP (slightly more than global-only cap of 5000)
+            if len(merged) > 6000:
+                merged = heapq.nlargest(6000, merged, key=lambda x: x["frp"])
+            latest_data["firms_fires"] = merged
+        logger.info(f"FIRMS country enrichment: +{len(new_fires)} fires from {len(_FIRMS_COUNTRIES)} countries")
+        _mark_fresh("firms_fires")
+    else:
+        logger.debug("FIRMS country enrichment: no new fires found")
+
+
 # ---------------------------------------------------------------------------
 # Space Weather (NOAA SWPC)
 # ---------------------------------------------------------------------------
@@ -84,7 +192,9 @@ def fetch_firms_fires():
 def fetch_space_weather():
     """Fetch NOAA SWPC Kp index and recent solar events."""
     try:
-        kp_resp = fetch_with_curl("https://services.swpc.noaa.gov/json/planetary_k_index_1m.json", timeout=10)
+        kp_resp = fetch_with_curl(
+            "https://services.swpc.noaa.gov/json/planetary_k_index_1m.json", timeout=10
+        )
         kp_value = None
         kp_text = "QUIET"
         if kp_resp.status_code == 200:
@@ -102,16 +212,20 @@ def fetch_space_weather():
                     kp_text = "UNSETTLED"
 
         events = []
-        ev_resp = fetch_with_curl("https://services.swpc.noaa.gov/json/edited_events.json", timeout=10)
+        ev_resp = fetch_with_curl(
+            "https://services.swpc.noaa.gov/json/edited_events.json", timeout=10
+        )
         if ev_resp.status_code == 200:
             all_events = ev_resp.json()
             for ev in all_events[-10:]:
-                events.append({
-                    "type": ev.get("type", ""),
-                    "begin": ev.get("begin", ""),
-                    "end": ev.get("end", ""),
-                    "classtype": ev.get("classtype", ""),
-                })
+                events.append(
+                    {
+                        "type": ev.get("type", ""),
+                        "begin": ev.get("begin", ""),
+                        "end": ev.get("end", ""),
+                        "classtype": ev.get("classtype", ""),
+                    }
+                )
 
         with _data_lock:
             latest_data["space_weather"] = {
@@ -121,7 +235,7 @@ def fetch_space_weather():
             }
         _mark_fresh("space_weather")
         logger.info(f"Space weather: Kp={kp_value} ({kp_text}), {len(events)} events")
-    except Exception as e:
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
         logger.error(f"Error fetching space weather: {e}")
 
 
@@ -138,7 +252,347 @@ def fetch_weather():
             if "radar" in data and "past" in data["radar"]:
                 latest_time = data["radar"]["past"][-1]["time"]
                 with _data_lock:
-                    latest_data["weather"] = {"time": latest_time, "host": data.get("host", "https://tilecache.rainviewer.com")}
+                    latest_data["weather"] = {
+                        "time": latest_time,
+                        "host": data.get("host", "https://tilecache.rainviewer.com"),
+                    }
                 _mark_fresh("weather")
-    except Exception as e:
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
         logger.error(f"Error fetching weather: {e}")
+
+
+# ---------------------------------------------------------------------------
+# NOAA/NWS Severe Weather Alerts
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=1, base_delay=2)
+def fetch_weather_alerts():
+    """Fetch active severe weather alerts from NOAA/NWS (US coverage, GeoJSON polygons)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("weather_alerts"):
+        return
+    alerts = []
+    try:
+        url = "https://api.weather.gov/alerts/active?status=actual"
+        headers = {
+            "User-Agent": "(ShadowBroker OSINT Dashboard, github.com/BigBodyCobain/Shadowbroker)",
+            "Accept": "application/geo+json",
+        }
+        response = fetch_with_curl(url, timeout=15, headers=headers)
+        if response.status_code == 200:
+            features = response.json().get("features", [])
+            for f in features:
+                props = f.get("properties", {})
+                geom = f.get("geometry")
+                if not geom:
+                    continue  # skip zone-only alerts with no polygon
+                alerts.append(
+                    {
+                        "id": props.get("id", ""),
+                        "event": props.get("event", ""),
+                        "severity": props.get("severity", "Unknown"),
+                        "certainty": props.get("certainty", ""),
+                        "urgency": props.get("urgency", ""),
+                        "headline": props.get("headline", ""),
+                        "description": (props.get("description", "") or "")[:300],
+                        "expires": props.get("expires", ""),
+                        "geometry": geom,
+                    }
+                )
+        logger.info(f"Weather alerts: {len(alerts)} active (with polygons)")
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching weather alerts: {e}")
+    with _data_lock:
+        latest_data["weather_alerts"] = alerts
+    if alerts:
+        _mark_fresh("weather_alerts")
+
+
+# ---------------------------------------------------------------------------
+# Air Quality (OpenAQ v3)
+# ---------------------------------------------------------------------------
+def _pm25_to_aqi(pm25: float) -> int:
+    """Convert PM2.5 concentration (µg/m³) to US EPA AQI."""
+    breakpoints = [
+        (0, 12.0, 0, 50),
+        (12.1, 35.4, 51, 100),
+        (35.5, 55.4, 101, 150),
+        (55.5, 150.4, 151, 200),
+        (150.5, 250.4, 201, 300),
+        (250.5, 500.4, 301, 500),
+    ]
+    for c_lo, c_hi, i_lo, i_hi in breakpoints:
+        if pm25 <= c_hi:
+            return round(((i_hi - i_lo) / (c_hi - c_lo)) * (pm25 - c_lo) + i_lo)
+    return 500
+
+
+@with_retry(max_retries=1, base_delay=2)
+def fetch_air_quality():
+    """Fetch global air quality stations with PM2.5 data from OpenAQ."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("air_quality"):
+        return
+    stations = []
+    api_key = os.environ.get("OPENAQ_API_KEY", "")
+    if not api_key:
+        logger.debug("OPENAQ_API_KEY not set, skipping air quality fetch")
+        return
+    try:
+        url = "https://api.openaq.org/v3/locations?limit=5000&parameter_id=2&order_by=datetime&sort_order=desc"
+        headers = {"X-API-Key": api_key}
+        response = fetch_with_curl(url, timeout=30, headers=headers)
+        if response.status_code == 200:
+            results = response.json().get("results", [])
+            for loc in results:
+                coords = loc.get("coordinates", {})
+                lat = coords.get("latitude")
+                lng = coords.get("longitude")
+                if lat is None or lng is None:
+                    continue
+                pm25 = None
+                for p in loc.get("parameters", []):
+                    if p.get("id") == 2:
+                        pm25 = p.get("lastValue")
+                        break
+                if pm25 is None:
+                    continue
+                pm25_val = float(pm25)
+                if pm25_val < 0:
+                    continue
+                stations.append(
+                    {
+                        "id": loc.get("id"),
+                        "name": loc.get("name", "Unknown"),
+                        "lat": lat,
+                        "lng": lng,
+                        "pm25": round(pm25_val, 1),
+                        "aqi": _pm25_to_aqi(pm25_val),
+                        "country": loc.get("country", {}).get("code", ""),
+                    }
+                )
+        logger.info(f"Air quality: {len(stations)} stations")
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching air quality: {e}")
+    with _data_lock:
+        latest_data["air_quality"] = stations
+    if stations:
+        _mark_fresh("air_quality")
+
+
+# ---------------------------------------------------------------------------
+# Volcanoes (Smithsonian Global Volcanism Program)
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=2, base_delay=5)
+def fetch_volcanoes():
+    """Fetch Holocene volcanoes from Smithsonian GVP WFS (static reference data)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("volcanoes"):
+        return
+    volcanoes = []
+    try:
+        url = (
+            "https://webservices.volcano.si.edu/geoserver/GVP-VOTW/wfs"
+            "?service=WFS&version=2.0.0&request=GetFeature"
+            "&typeName=GVP-VOTW:E3WebApp_HoloceneVolcanoes"
+            "&outputFormat=application/json"
+        )
+        response = fetch_with_curl(url, timeout=30)
+        if response.status_code == 200:
+            features = response.json().get("features", [])
+            for f in features:
+                props = f.get("properties", {})
+                geom = f.get("geometry", {})
+                coords = geom.get("coordinates", [None, None])
+                if coords[0] is None:
+                    continue
+                last_eruption = props.get("LastEruption")
+                last_eruption_year = None
+                if last_eruption is not None:
+                    try:
+                        last_eruption_year = int(last_eruption)
+                    except (ValueError, TypeError):
+                        pass
+                volcanoes.append(
+                    {
+                        "name": props.get("VolcanoName", "Unknown"),
+                        "type": props.get("VolcanoType", ""),
+                        "country": props.get("Country", ""),
+                        "region": props.get("TectonicSetting", ""),
+                        "elevation": props.get("Elevation", 0),
+                        "last_eruption_year": last_eruption_year,
+                        "lat": coords[1],
+                        "lng": coords[0],
+                    }
+                )
+        logger.info(f"Volcanoes: {len(volcanoes)} Holocene volcanoes loaded")
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching volcanoes: {e}")
+    with _data_lock:
+        latest_data["volcanoes"] = volcanoes
+    if volcanoes:
+        _mark_fresh("volcanoes")
+
+
+# ---------------------------------------------------------------------------
+# VIIRS Night Lights Change Detection (Google Earth Engine — optional)
+# ---------------------------------------------------------------------------
+_VIIRS_CACHE_PATH = Path(__file__).parent.parent.parent / "data" / "viirs_change_nodes.json"
+_VIIRS_CACHE_MAX_AGE_S = 86400  # 24 hours
+
+# Conflict-zone AOIs: (name, south, west, north, east)
+_VIIRS_AOIS = [
+    ("Gaza Strip", 31.2, 34.2, 31.6, 34.6),
+    ("Kharkiv Oblast", 48.5, 35.0, 50.5, 38.5),
+    ("Donetsk Oblast", 47.0, 36.5, 49.0, 39.5),
+    ("Zaporizhzhia Oblast", 46.5, 34.5, 48.5, 37.0),
+    ("Aleppo", 35.8, 36.5, 36.5, 37.5),
+    ("Khartoum", 15.2, 32.2, 15.9, 32.9),
+    ("Sana'a", 14.9, 43.8, 15.6, 44.5),
+    ("Mosul", 36.0, 42.8, 36.7, 43.5),
+    ("Mariupol", 46.9, 37.2, 47.3, 37.8),
+    ("Southern Lebanon", 33.0, 35.0, 33.5, 36.0),
+]
+
+_VIIRS_SEVERITY_THRESHOLDS = [
+    (-100, -70, "severe"),
+    (-70, -50, "high"),
+    (-50, -30, "moderate"),
+    (30, 100, "growth"),
+    (100, 500, "rapid_growth"),
+]
+
+
+def _classify_viirs_severity(pct_change: float):
+    for lo, hi, label in _VIIRS_SEVERITY_THRESHOLDS:
+        if lo <= pct_change <= hi:
+            return label
+    return None
+
+
+def _load_viirs_stale_cache():
+    """Load stale cache if available (when GEE is not configured)."""
+    if _VIIRS_CACHE_PATH.exists():
+        try:
+            cached = json.loads(_VIIRS_CACHE_PATH.read_text(encoding="utf-8"))
+            with _data_lock:
+                latest_data["viirs_change_nodes"] = cached
+            _mark_fresh("viirs_change_nodes")
+            logger.info(f"VIIRS change nodes: loaded {len(cached)} from stale cache")
+        except Exception:
+            pass
+
+
+@with_retry(max_retries=1, base_delay=5)
+def fetch_viirs_change_nodes():
+    """Compute VIIRS nighttime radiance change nodes via GEE (optional)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("viirs_nightlights"):
+        return
+
+    # Check cache freshness first
+    if _VIIRS_CACHE_PATH.exists():
+        age = time.time() - _VIIRS_CACHE_PATH.stat().st_mtime
+        if age < _VIIRS_CACHE_MAX_AGE_S:
+            try:
+                cached = json.loads(_VIIRS_CACHE_PATH.read_text(encoding="utf-8"))
+                with _data_lock:
+                    latest_data["viirs_change_nodes"] = cached
+                _mark_fresh("viirs_change_nodes")
+                logger.info(f"VIIRS change nodes: loaded {len(cached)} from cache (age {age:.0f}s)")
+                return
+            except Exception as e:
+                logger.warning(f"VIIRS cache read failed: {e}")
+
+    # Try importing earthengine-api (optional dependency)
+    try:
+        import ee
+    except ImportError:
+        logger.debug("earthengine-api not installed, skipping VIIRS change detection")
+        _load_viirs_stale_cache()
+        return
+
+    # Authenticate with service account
+    sa_key_path = os.environ.get("GEE_SERVICE_ACCOUNT_KEY", "")
+    if not sa_key_path:
+        logger.debug("GEE_SERVICE_ACCOUNT_KEY not set, skipping VIIRS change detection")
+        _load_viirs_stale_cache()
+        return
+
+    try:
+        credentials = ee.ServiceAccountCredentials(None, key_file=sa_key_path)
+        ee.Initialize(credentials)
+    except Exception as e:
+        logger.error(f"GEE authentication failed: {e}")
+        _load_viirs_stale_cache()
+        return
+
+    # Compute change nodes for each AOI
+    nodes = []
+    viirs = ee.ImageCollection("NOAA/VIIRS/DNB/MONTHLY_V1/VCMCFG").select("avg_rad")
+
+    for aoi_name, s_lat, w_lng, n_lat, e_lng in _VIIRS_AOIS:
+        try:
+            aoi = ee.Geometry.Rectangle([w_lng, s_lat, e_lng, n_lat])
+
+            # Most recent available date
+            now = ee.Date(datetime.utcnow().isoformat()[:10])
+
+            # Current: 12-month rolling mean ending now
+            current = viirs.filterDate(now.advance(-12, "month"), now).mean().clip(aoi)
+
+            # Baseline: 12-month mean ending 12 months ago
+            baseline = viirs.filterDate(
+                now.advance(-24, "month"), now.advance(-12, "month")
+            ).mean().clip(aoi)
+
+            # Floor baseline at 0.5 nW/cm²/sr to avoid div-by-zero in dark areas
+            baseline_safe = baseline.max(0.5)
+
+            # Percentage change
+            change = current.subtract(baseline).divide(baseline_safe).multiply(100)
+
+            # Only keep pixels with >30% absolute change
+            sig_mask = change.abs().gt(30)
+            change_masked = change.updateMask(sig_mask)
+
+            # Sample up to 200 points per AOI
+            samples = change_masked.sample(
+                region=aoi, scale=500, numPixels=200, geometries=True
+            )
+            sample_list = samples.getInfo()
+
+            for feat in sample_list.get("features", []):
+                coords = feat["geometry"]["coordinates"]
+                pct = feat["properties"].get("avg_rad", 0)
+                severity = _classify_viirs_severity(pct)
+                if severity is None:
+                    continue
+                nodes.append({
+                    "lat": round(coords[1], 4),
+                    "lng": round(coords[0], 4),
+                    "mean_change_pct": round(pct, 1),
+                    "severity": severity,
+                    "aoi_name": aoi_name,
+                })
+        except Exception as e:
+            logger.warning(f"VIIRS change detection failed for {aoi_name}: {e}")
+            continue
+
+    # Save to cache
+    try:
+        _VIIRS_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
+        _VIIRS_CACHE_PATH.write_text(
+            json.dumps(nodes, separators=(",", ":")), encoding="utf-8"
+        )
+    except Exception as e:
+        logger.warning(f"Failed to write VIIRS cache: {e}")
+
+    with _data_lock:
+        latest_data["viirs_change_nodes"] = nodes
+    if nodes:
+        _mark_fresh("viirs_change_nodes")
+    logger.info(f"VIIRS change nodes: {len(nodes)} nodes from {len(_VIIRS_AOIS)} AOIs")

backend/services/fetchers/emissions.py

@@ -0,0 +1,131 @@
+"""
+Fuel burn & CO2 emissions estimator for private jets.
+Based on manufacturer-published cruise fuel burn rates (GPH at long-range cruise).
+1 US gallon of Jet-A produces ~21.1 lbs (9.57 kg) of CO2.
+"""
+
+JET_A_CO2_KG_PER_GALLON = 9.57
+
+# ICAO type code -> gallons per hour at long-range cruise
+FUEL_BURN_GPH: dict[str, int] = {
+    # Gulfstream
+    "GLF6": 430,   # G650/G650ER
+    "G700": 480,   # G700
+    "GLF5": 390,   # G550
+    "GVSP": 400,   # GV-SP
+    "GLF4": 330,   # G-IV
+    # Bombardier
+    "GL7T": 490,   # Global 7500
+    "GLEX": 430,   # Global Express/6000/6500
+    "GL5T": 420,   # Global 5000/5500
+    "CL35": 220,   # Challenger 350
+    "CL60": 310,   # Challenger 604/605
+    "CL30": 200,   # Challenger 300
+    "CL65": 320,   # Challenger 650
+    # Dassault
+    "F7X":  350,   # Falcon 7X
+    "F8X":  370,   # Falcon 8X
+    "F900": 285,   # Falcon 900/900EX/900LX
+    "F2TH": 230,   # Falcon 2000
+    "FA50": 240,   # Falcon 50
+    # Cessna
+    "CITX": 280,   # Citation X
+    "C68A": 195,   # Citation Latitude
+    "C700": 230,   # Citation Longitude
+    "C680": 220,   # Citation Sovereign
+    "C560": 190,   # Citation Excel/XLS
+    "C510": 75,    # Citation Mustang
+    "CJ3":  120,   # CJ3
+    "CJ4":  135,   # CJ4
+    # Boeing
+    "B737": 850,   # BBJ (737)
+    "B738": 920,   # BBJ2 (737-800)
+    "B752": 1100,  # 757-200
+    "B762": 1400,  # 767-200
+    "B788": 1200,  # 787-8
+    # Airbus
+    "A318": 780,   # ACJ318
+    "A319": 850,   # ACJ319
+    "A320": 900,   # ACJ320
+    "A343": 1800,  # A340-300
+    "A346": 2100,  # A340-600
+    # Pilatus
+    "PC24": 115,   # PC-24
+    "PC12": 60,    # PC-12
+    # Embraer
+    "E55P": 185,   # Legacy 500
+    "E135": 300,   # Legacy 600/650
+    "E50P": 135,   # Phenom 300
+    "E500": 80,    # Phenom 100
+    # Learjet
+    "LJ60": 195,   # Learjet 60
+    "LJ75": 185,   # Learjet 75
+    "LJ45": 175,   # Learjet 45
+    # Hawker
+    "H25B": 210,   # Hawker 800/800XP
+    "H25C": 215,   # Hawker 900XP
+    # Beechcraft
+    "B350": 100,   # King Air 350
+    "B200": 80,    # King Air 200/250
+}
+
+# Common string names -> ICAO type code
+_ALIASES: dict[str, str] = {
+    "Gulfstream G650": "GLF6", "Gulfstream G650ER": "GLF6", "G650": "GLF6", "G650ER": "GLF6",
+    "Gulfstream G700": "G700",
+    "Gulfstream G550": "GLF5", "G550": "GLF5", "G500": "GLF5",
+    "Gulfstream GV": "GVSP", "Gulfstream G-V": "GVSP", "GV": "GVSP",
+    "Gulfstream G-IV": "GLF4", "Gulfstream GIV": "GLF4", "G450": "GLF4",
+    "Global 7500": "GL7T", "Bombardier Global 7500": "GL7T",
+    "Global 6000": "GLEX", "Global Express": "GLEX", "Bombardier Global 6000": "GLEX",
+    "Global 5000": "GL5T",
+    "Challenger 350": "CL35", "Challenger 300": "CL30",
+    "Challenger 604": "CL60", "Challenger 605": "CL60", "Challenger 650": "CL65",
+    "Falcon 7X": "F7X", "Dassault Falcon 7X": "F7X",
+    "Falcon 8X": "F8X", "Dassault Falcon 8X": "F8X",
+    "Falcon 900": "F900", "Falcon 900LX": "F900", "Falcon 900EX": "F900",
+    "Falcon 2000": "F2TH",
+    "Citation X": "CITX", "Citation Latitude": "C68A", "Citation Longitude": "C700",
+    "Boeing 757-200": "B752", "757-200": "B752", "Boeing 757": "B752",
+    "Boeing 767-200": "B762", "767-200": "B762", "Boeing 767": "B762",
+    "Boeing 787-8": "B788", "Boeing 787": "B788",
+    "Boeing 737": "B737", "737 BBJ": "B737", "BBJ": "B737",
+    "Airbus A340-300": "A343", "A340-300": "A343", "A340": "A343",
+    "Airbus A318": "A318",
+    "Pilatus PC-24": "PC24", "PC-24": "PC24",
+    "Legacy 500": "E55P", "Legacy 600": "E135", "Phenom 300": "E50P",
+    "Learjet 60": "LJ60", "Learjet 75": "LJ75",
+    "Hawker 800": "H25B", "Hawker 900XP": "H25C",
+    "King Air 350": "B350", "King Air 200": "B200",
+}
+
+
+def get_emissions_info(model: str) -> dict | None:
+    """
+    Given an aircraft model string (ICAO type code or common name),
+    return emissions info dict or None if unknown.
+    """
+    if not model:
+        return None
+    model_clean = model.strip()
+    # Try direct ICAO code match first
+    gph = FUEL_BURN_GPH.get(model_clean.upper())
+    if gph is None:
+        # Try alias lookup
+        code = _ALIASES.get(model_clean)
+        if code:
+            gph = FUEL_BURN_GPH.get(code)
+    if gph is None:
+        # Fuzzy: check if any alias is a substring
+        model_lower = model_clean.lower()
+        for alias, code in _ALIASES.items():
+            if alias.lower() in model_lower or model_lower in alias.lower():
+                gph = FUEL_BURN_GPH.get(code)
+                if gph:
+                    break
+    if gph is None:
+        return None
+    return {
+        "fuel_gph": gph,
+        "co2_kg_per_hour": round(gph * JET_A_CO2_KG_PER_GALLON, 1),
+    }

backend/services/fetchers/fimi.py

@@ -0,0 +1,274 @@
+"""EUvsDisinfo FIMI (Foreign Information Manipulation & Interference) fetcher.
+
+Parses the EUvsDisinfo RSS feed to extract disinformation narratives,
+debunked claims, threat actor mentions, and target country references.
+Refreshes every 12 hours (FIMI data updates weekly).
+"""
+
+import re
+import logging
+from datetime import datetime, timezone
+
+import feedparser
+from services.network_utils import fetch_with_curl
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger("services.data_fetcher")
+
+_FIMI_FEED_URL = "https://euvsdisinfo.eu/feed/"
+
+# ── Threat actor keywords ──────────────────────────────────────────────────
+# Map of keyword → canonical actor name.  Checked case-insensitively.
+_THREAT_ACTORS: dict[str, str] = {
+    "russia":       "Russia",
+    "russian":      "Russia",
+    "kremlin":      "Russia",
+    "pro-kremlin":  "Russia",
+    "moscow":       "Russia",
+    "china":        "China",
+    "chinese":      "China",
+    "beijing":      "China",
+    "iran":         "Iran",
+    "iranian":      "Iran",
+    "tehran":       "Iran",
+    "north korea":  "North Korea",
+    "pyongyang":    "North Korea",
+    "dprk":         "North Korea",
+    "belarus":      "Belarus",
+    "belarusian":   "Belarus",
+    "minsk":        "Belarus",
+}
+
+# ── Target country/region keywords ─────────────────────────────────────────
+_TARGET_KEYWORDS: dict[str, str] = {
+    "ukraine":      "Ukraine",
+    "kyiv":         "Ukraine",
+    "moldova":      "Moldova",
+    "georgia":      "Georgia",
+    "tbilisi":      "Georgia",
+    "eu":           "EU",
+    "european union": "EU",
+    "europe":       "Europe",
+    "nato":         "NATO",
+    "united states": "United States",
+    "usa":          "United States",
+    "germany":      "Germany",
+    "france":       "France",
+    "poland":       "Poland",
+    "baltic":       "Baltics",
+    "lithuania":    "Baltics",
+    "latvia":       "Baltics",
+    "estonia":      "Baltics",
+    "romania":      "Romania",
+    "czech":        "Czech Republic",
+    "slovakia":     "Slovakia",
+    "armenia":      "Armenia",
+    "africa":       "Africa",
+    "middle east":  "Middle East",
+    "syria":        "Syria",
+    "israel":       "Israel",
+    "serbia":       "Serbia",
+    "india":        "India",
+    "brazil":       "Brazil",
+}
+
+# ── Disinformation topic keywords (for cross-referencing news) ─────────────
+_DISINFO_TOPICS = [
+    "sanctions",
+    "energy crisis",
+    "gas supply",
+    "nuclear threat",
+    "nato expansion",
+    "biolab",
+    "biological weapon",
+    "provocation",
+    "false flag",
+    "staged",
+    "nazi",
+    "genocide",
+    "referendum",
+    "regime change",
+    "coup",
+    "puppet government",
+    "election interference",
+    "election meddling",
+    "voter fraud",
+    "migrant invasion",
+    "refugee crisis",
+    "civil war",
+    "food crisis",
+    "grain deal",
+]
+
+# Regex for extracting debunked report URLs from feed HTML
+_REPORT_URL_RE = re.compile(
+    r'https?://euvsdisinfo\.eu/report/[a-z0-9\-]+/?',
+    re.IGNORECASE,
+)
+
+# Regex for extracting the claim title from a report URL slug
+_SLUG_RE = re.compile(r'/report/([a-z0-9\-]+)/?$', re.IGNORECASE)
+
+
+def _slug_to_title(url: str) -> str:
+    """Convert a report URL slug to a human-readable title."""
+    m = _SLUG_RE.search(url)
+    if not m:
+        return url
+    return m.group(1).replace("-", " ").title()
+
+
+def _count_mentions(text: str, keywords: dict[str, str]) -> dict[str, int]:
+    """Count keyword mentions, mapping to canonical names."""
+    counts: dict[str, int] = {}
+    text_lower = text.lower()
+    for kw, canonical in keywords.items():
+        # Word-boundary match, case-insensitive
+        pattern = r'\b' + re.escape(kw) + r'\b'
+        matches = re.findall(pattern, text_lower)
+        if matches:
+            counts[canonical] = counts.get(canonical, 0) + len(matches)
+    return counts
+
+
+def _extract_disinfo_keywords(text: str) -> list[str]:
+    """Return which disinformation topic keywords appear in the text."""
+    text_lower = text.lower()
+    found = []
+    for topic in _DISINFO_TOPICS:
+        if topic in text_lower:
+            found.append(topic)
+    return found
+
+
+def _is_major_wave(narratives: list[dict], targets: dict[str, int]) -> bool:
+    """Heuristic: detect a 'major disinformation wave'.
+
+    Triggers when:
+    - 3+ narratives in the feed mention the same target, OR
+    - A single target has 10+ total mentions across all narratives, OR
+    - 5+ distinct debunked claims extracted in one fetch
+    """
+    if not narratives:
+        return False
+
+    # Check per-target narrative count
+    target_narrative_counts: dict[str, int] = {}
+    total_claims = 0
+    for n in narratives:
+        for t in n.get("targets", []):
+            target_narrative_counts[t] = target_narrative_counts.get(t, 0) + 1
+        total_claims += len(n.get("claims", []))
+
+    if any(c >= 3 for c in target_narrative_counts.values()):
+        return True
+    if any(c >= 10 for c in targets.values()):
+        return True
+    if total_claims >= 5:
+        return True
+    return False
+
+
+@with_retry(max_retries=1, base_delay=5)
+def fetch_fimi():
+    """Fetch and parse the EUvsDisinfo RSS feed."""
+    try:
+        resp = fetch_with_curl(_FIMI_FEED_URL, timeout=15)
+        feed = feedparser.parse(resp.text)
+    except Exception as e:
+        logger.warning(f"FIMI feed fetch failed: {e}")
+        return
+
+    if not feed.entries:
+        logger.warning("FIMI feed: no entries found")
+        return
+
+    narratives = []
+    all_claims: list[dict] = []
+    agg_actors: dict[str, int] = {}
+    agg_targets: dict[str, int] = {}
+    all_disinfo_kw: set[str] = set()
+
+    for entry in feed.entries[:15]:  # Cap at 15 entries
+        title = entry.get("title", "")
+        link = entry.get("link", "")
+        published = entry.get("published", "")
+        summary_html = entry.get("summary", "") or entry.get("description", "")
+
+        # Strip HTML tags for text analysis
+        summary_text = re.sub(r"<[^>]+>", " ", summary_html)
+        summary_text = re.sub(r"\s+", " ", summary_text).strip()
+        full_text = f"{title} {summary_text}"
+
+        # Extract debunked report URLs
+        report_urls = list(set(_REPORT_URL_RE.findall(summary_html)))
+        claims = [{"url": url, "title": _slug_to_title(url)} for url in report_urls]
+        all_claims.extend(claims)
+
+        # Count threat actors
+        actors = _count_mentions(full_text, _THREAT_ACTORS)
+        for actor, count in actors.items():
+            agg_actors[actor] = agg_actors.get(actor, 0) + count
+
+        # Count target countries
+        targets = _count_mentions(full_text, _TARGET_KEYWORDS)
+        for target, count in targets.items():
+            agg_targets[target] = agg_targets.get(target, 0) + count
+
+        # Extract disinfo topic keywords
+        disinfo_kw = _extract_disinfo_keywords(full_text)
+        all_disinfo_kw.update(disinfo_kw)
+
+        # Truncate summary for storage
+        snippet = summary_text[:300] + ("..." if len(summary_text) > 300 else "")
+
+        narratives.append({
+            "title": title,
+            "link": link,
+            "published": published,
+            "snippet": snippet,
+            "claims": claims,
+            "actors": list(actors.keys()),
+            "targets": list(targets.keys()),
+            "disinfo_keywords": disinfo_kw,
+        })
+
+    # Sort actors and targets by count (descending)
+    sorted_actors = dict(sorted(agg_actors.items(), key=lambda x: x[1], reverse=True))
+    sorted_targets = dict(sorted(agg_targets.items(), key=lambda x: x[1], reverse=True))
+
+    # Deduplicate claims
+    seen_urls: set[str] = set()
+    unique_claims = []
+    for c in all_claims:
+        if c["url"] not in seen_urls:
+            seen_urls.add(c["url"])
+            unique_claims.append(c)
+
+    major_wave = _is_major_wave(narratives, sorted_targets)
+
+    fimi_data = {
+        "narratives": narratives,
+        "claims": unique_claims,
+        "threat_actors": sorted_actors,
+        "targets": sorted_targets,
+        "disinfo_keywords": sorted(all_disinfo_kw),
+        "major_wave": major_wave,
+        "major_wave_target": (
+            max(sorted_targets, key=sorted_targets.get) if major_wave and sorted_targets else None
+        ),
+        "last_fetched": datetime.now(timezone.utc).isoformat(),
+        "source": "EUvsDisinfo",
+        "source_url": "https://euvsdisinfo.eu",
+    }
+
+    with _data_lock:
+        latest_data["fimi"] = fimi_data
+    _mark_fresh("fimi")
+    logger.info(
+        f"FIMI fetch complete: {len(narratives)} narratives, "
+        f"{len(unique_claims)} claims, "
+        f"{len(sorted_actors)} actors, "
+        f"major_wave={major_wave}"
+    )

backend/services/fetchers/financial.py

@@ -1,97 +1,161 @@
-"""Financial data fetchers — defense stocks and oil prices.
-
-Uses yfinance batch download to minimise Yahoo Finance requests and avoid rate limiting.
-"""
 import logging
-import yfinance as yf
+import math
+import random
+import time
+import os
+import urllib.request
+import json
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timezone
 from services.fetchers._store import latest_data, _data_lock, _mark_fresh
 from services.fetchers.retry import with_retry
 
 logger = logging.getLogger(__name__)
 
+_YFINANCE_REQUEST_DELAY_SECONDS = 0.5
+_YFINANCE_REQUEST_JITTER_SECONDS = 0.2
 
-def _batch_fetch(symbols: list[str], period: str = "5d") -> dict:
-    """Fetch multiple tickers in a single yfinance request. Returns {symbol: {price, change_percent, up}}."""
+TICKERS_DEFENSE = ["RTX", "LMT", "NOC", "GD", "BA", "PLTR"]
+TICKERS_TECH = ["NVDA", "AMD", "TSM", "INTC", "GOOGL", "AMZN", "MSFT", "AAPL", "TSLA", "META", "NFLX", "SMCI", "ARM", "ASML"]
+TICKERS_CRYPTO = [
+    ("BTC", "BINANCE:BTCUSDT", "BTC-USD"),
+    ("ETH", "BINANCE:ETHUSDT", "ETH-USD"),
+    ("SOL", "BINANCE:SOLUSDT", "SOL-USD"),
+    ("XRP", "BINANCE:XRPUSDT", "XRP-USD"),
+    ("ADA", "BINANCE:ADAUSDT", "ADA-USD"),
+]
+
+# Ticker priority for high-frequency updates (we update these every tick)
+PRIORITY_SYMBOLS = ["BTC", "ETH", "NVDA", "PLTR"]
+
+# Persistence for state between short-lived scheduler ticks
+_last_fetch_results = {}
+_last_fetch_time = 0.0
+_rotating_index = 0
+_executor = ThreadPoolExecutor(max_workers=10)
+
+
+def _fetch_finnhub_quote(symbol: str, api_key: str):
+    """Fetch from Finnhub. Returns (symbol, data) or (symbol, None)."""
+    url = f"https://finnhub.io/api/v1/quote?symbol={symbol}&token={api_key}"
     try:
-        hist = yf.download(symbols, period=period, auto_adjust=True, progress=False)
-        if hist.empty:
-            return {}
-        close = hist["Close"]
-        result = {}
-        for sym in symbols:
-            try:
-                col = close[sym] if len(symbols) > 1 else close
-                col = col.dropna()
-                if len(col) < 1:
-                    continue
-                current = float(col.iloc[-1])
-                prev = float(col.iloc[0]) if len(col) > 1 else current
-                change = ((current - prev) / prev * 100) if prev else 0
-                result[sym] = {
-                    "price": round(current, 2),
-                    "change_percent": round(change, 2),
-                    "up": bool(change >= 0),
-                }
-            except Exception as e:
-                logger.warning(f"Could not parse {sym}: {e}")
-        return result
+        req = urllib.request.Request(url)
+        with urllib.request.urlopen(req, timeout=5) as response:
+            data = json.loads(response.read().decode())
+            if "c" not in data or data["c"] == 0:
+                return symbol, None
+            current = float(data["c"])
+            change_p = float(data.get("dp", 0.0) or 0.0)
+            return symbol, {
+                "price": round(current, 2),
+                "change_percent": round(change_p, 2),
+                "up": bool(change_p >= 0),
+            }
     except Exception as e:
-        logger.warning(f"Batch fetch failed: {e}")
-        return {}
+        logger.debug(f"Finnhub error for {symbol}: {e}")
+        return symbol, None
 
 
-_STOCK_TICKERS = ["RTX", "LMT", "NOC", "GD", "BA", "PLTR"]
-_OIL_MAP = {"WTI Crude": "CL=F", "Brent Crude": "BZ=F"}
-_ALL_TICKERS = _STOCK_TICKERS + list(_OIL_MAP.values())
-
-_MARKET_COOLDOWN_SECONDS = 1800  # fetch at most once every 30 minutes
-_last_market_fetch: float = 0.0
+def _fetch_yfinance_single(symbol: str, period: str = "2d"):
+    """Fetch from yfinance. Returns (symbol, data) or (symbol, None)."""
+    try:
+        import yfinance as yf
+        ticker = yf.Ticker(symbol)
+        hist = ticker.history(period=period)
+        if len(hist) >= 1:
+            current_price = hist["Close"].iloc[-1]
+            prev_close = hist["Close"].iloc[0] if len(hist) > 1 else current_price
+            change_percent = ((current_price - prev_close) / prev_close) * 100 if prev_close else 0
+            current_price_f = float(current_price)
+            change_percent_f = float(change_percent)
+            if not math.isfinite(current_price_f) or not math.isfinite(change_percent_f):
+                return symbol, None
+            return symbol, {
+                "price": round(current_price_f, 2),
+                "change_percent": round(change_percent_f, 2),
+                "up": bool(change_percent_f >= 0),
+            }
+    except Exception as e:
+        logger.debug(f"Yfinance error for {symbol}: {e}")
+    return symbol, None
 
 
-def _fetch_all_market_data():
-    """Single yfinance download for all market tickers to avoid rate limiting."""
-    raw = _batch_fetch(_ALL_TICKERS, period="5d")
-    stocks = {sym: raw[sym] for sym in _STOCK_TICKERS if sym in raw}
-    oil = {name: raw[sym] for name, sym in _OIL_MAP.items() if sym in raw}
-    return stocks, oil
+@with_retry(max_retries=1, base_delay=1)
+def fetch_financial_markets():
+    """Fetches full market list with smart throttling (3s for Finnhub, 60s for yfinance)."""
+    global _last_fetch_time, _last_fetch_results, _rotating_index
+    
+    finnhub_key = os.getenv("FINNHUB_API_KEY", "").strip()
+    use_finnhub = bool(finnhub_key)
+    
+    now = time.time()
+    # Throttle logic: 3s for Finnhub, 60s for yfinance fallback
+    throttle_s = 3.0 if use_finnhub else 60.0
+    
+    if now - _last_fetch_time < throttle_s and _last_fetch_results:
+        return # Skip if too frequent
 
+    _last_fetch_time = now
+    
+    # Prepare symbol lists
+    all_crypto = {label: (f_sym, y_sym) for label, f_sym, y_sym in TICKERS_CRYPTO}
+    all_stocks = TICKERS_TECH + TICKERS_DEFENSE
+    
+    subset_to_fetch = []
+    
+    if use_finnhub:
+        # Finnhub Free Limit: 60/min. 
+        # Ticking every 3s = 20 ticks/min. 
+        # To stay safe, we fetch only ~3 items per tick.
+        # Priority items (BTC, ETH) + 1 rotating item.
+        subset_to_fetch = ["BINANCE:BTCUSDT", "BINANCE:ETHUSDT"]
+        
+        # Determine rotating ticker
+        all_other_symbols = []
+        for sym in all_stocks:
+            all_other_symbols.append(sym)
+        for label, (f_sym, y_sym) in all_crypto.items():
+            if label not in ["BTC", "ETH"]:
+                all_other_symbols.append(f_sym)
+        
+        if all_other_symbols:
+            rotated = all_other_symbols[_rotating_index % len(all_other_symbols)]
+            subset_to_fetch.append(rotated)
+            _rotating_index += 1
+            
+        # Concurrently fetch
+        futures = [_executor.submit(_fetch_finnhub_quote, s, finnhub_key) for s in subset_to_fetch]
+        for f in futures:
+            sym, data = f.result()
+            if data:
+                # Map back to readable label if it was crypto
+                label = sym
+                for l, (fs, ys) in all_crypto.items():
+                    if fs == sym:
+                        label = l
+                        break
+                _last_fetch_results[label] = data
+    else:
+        # Yahoo Finance Fallback - fetch all (once per minute)
+        logger.info("Finnhub key missing, using Yahoo Finance 60s update cycle.")
+        to_fetch = all_stocks + [y_sym for l, (fs, y_sym) in all_crypto.items()]
+        futures = [_executor.submit(_fetch_yfinance_single, s) for s in to_fetch]
+        for f in futures:
+            sym, data = f.result()
+            if data:
+                # Map back to readable label if it was crypto
+                label = sym
+                for l, (fs, ys) in all_crypto.items():
+                    if ys == sym:
+                        label = l
+                        break
+                _last_fetch_results[label] = data
 
-@with_retry(max_retries=2, base_delay=10)
-def fetch_defense_stocks():
-    global _last_market_fetch
-    import time
-    if time.time() - _last_market_fetch < _MARKET_COOLDOWN_SECONDS:
+    if not _last_fetch_results:
         return
-    try:
-        stocks, oil = _fetch_all_market_data()
-        if stocks:
-            _last_market_fetch = time.time()
-            with _data_lock:
-                latest_data['stocks'] = stocks
-                if oil:
-                    latest_data['oil'] = oil
-            _mark_fresh("stocks")
-            if oil:
-                _mark_fresh("oil")
-            logger.info(f"Markets: {len(stocks)} stocks, {len(oil)} oil tickers")
-        else:
-            logger.warning("Markets: empty result from yfinance (rate limited?)")
-    except Exception as e:
-        logger.error(f"Error fetching market data: {e}")
-
-
-@with_retry(max_retries=1, base_delay=10)
-def fetch_oil_prices():
-    # Oil is now fetched together with stocks in fetch_defense_stocks to use a single request.
-    # This function is kept for scheduler compatibility but is a no-op if stocks already ran.
+        
     with _data_lock:
-        if latest_data.get('oil'):
-            return  # Already populated by fetch_defense_stocks
-    try:
-        _, oil = _fetch_all_market_data()
-        if oil:
-            with _data_lock:
-                latest_data['oil'] = oil
-            _mark_fresh("oil")
-    except Exception as e:
-        logger.error(f"Error fetching oil: {e}")
+        latest_data["stocks"] = dict(_last_fetch_results)
+        latest_data["financial_source"] = "finnhub" if use_finnhub else "yfinance"
+    _mark_fresh("stocks")

backend/services/fetchers/flights.py

@@ -1,5 +1,7 @@
 """Commercial flight fetching — ADS-B, OpenSky, supplemental sources, routes,
 trail accumulation, GPS jamming detection, and holding pattern detection."""
+
+import copy
 import re
 import os
 import time
@@ -8,19 +10,23 @@ import json
 import logging
 import threading
 import concurrent.futures
+import random
 import requests
 from datetime import datetime
 from cachetools import TTLCache
 from services.network_utils import fetch_with_curl
 from services.fetchers._store import latest_data, _data_lock, _mark_fresh
 from services.fetchers.plane_alert import enrich_with_plane_alert, enrich_with_tracked_names
+from services.fetchers.emissions import get_emissions_info
 from services.fetchers.retry import with_retry
+from services.constants import GPS_JAMMING_NACP_THRESHOLD, GPS_JAMMING_MIN_RATIO, GPS_JAMMING_MIN_AIRCRAFT
 
 logger = logging.getLogger("services.data_fetcher")
 
 # Pre-compiled regex patterns for airline code extraction (used in hot loop)
-_RE_AIRLINE_CODE_1 = re.compile(r'^([A-Z]{3})\d')
-_RE_AIRLINE_CODE_2 = re.compile(r'^([A-Z]{3})[A-Z\d]')
+_RE_AIRLINE_CODE_1 = re.compile(r"^([A-Z]{3})\d")
+_RE_AIRLINE_CODE_2 = re.compile(r"^([A-Z]{3})[A-Z\d]")
+
 
 # ---------------------------------------------------------------------------
 # OpenSky Network API Client (OAuth2)
@@ -39,7 +45,7 @@ class OpenSkyClient:
         data = {
             "grant_type": "client_credentials",
             "client_id": self.client_id,
-            "client_secret": self.client_secret
+            "client_secret": self.client_secret,
         }
         try:
             r = requests.post(url, data=data, timeout=10)
@@ -51,13 +57,20 @@ class OpenSkyClient:
                 return self.token
             else:
                 logger.error(f"OpenSky Auth Failed: {r.status_code} {r.text}")
-        except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
+        except (
+            requests.RequestException,
+            ConnectionError,
+            TimeoutError,
+            ValueError,
+            KeyError,
+        ) as e:
             logger.error(f"OpenSky Auth Exception: {e}")
         return None
 
+
 opensky_client = OpenSkyClient(
     client_id=os.environ.get("OPENSKY_CLIENT_ID", ""),
-    client_secret=os.environ.get("OPENSKY_CLIENT_SECRET", "")
+    client_secret=os.environ.get("OPENSKY_CLIENT_SECRET", ""),
 )
 
 # Throttling and caching for OpenSky (400 req/day limit)
@@ -68,46 +81,173 @@ cached_opensky_flights = []
 # Supplemental ADS-B sources for blind-spot gap-filling
 # ---------------------------------------------------------------------------
 _BLIND_SPOT_REGIONS = [
-    {"name": "Yekaterinburg",  "lat": 56.8, "lon": 60.6,  "radius_nm": 250},
-    {"name": "Novosibirsk",   "lat": 55.0, "lon": 82.9,  "radius_nm": 250},
-    {"name": "Krasnoyarsk",   "lat": 56.0, "lon": 92.9,  "radius_nm": 250},
-    {"name": "Vladivostok",   "lat": 43.1, "lon": 131.9, "radius_nm": 250},
-    {"name": "Urumqi",        "lat": 43.8, "lon": 87.6,  "radius_nm": 250},
-    {"name": "Chengdu",       "lat": 30.6, "lon": 104.1, "radius_nm": 250},
-    {"name": "Lagos-Accra",   "lat": 6.5,  "lon": 3.4,   "radius_nm": 250},
-    {"name": "Addis Ababa",   "lat": 9.0,  "lon": 38.7,  "radius_nm": 250},
+    {"name": "Yekaterinburg", "lat": 56.8, "lon": 60.6, "radius_nm": 250},
+    {"name": "Novosibirsk", "lat": 55.0, "lon": 82.9, "radius_nm": 250},
+    {"name": "Krasnoyarsk", "lat": 56.0, "lon": 92.9, "radius_nm": 250},
+    {"name": "Vladivostok", "lat": 43.1, "lon": 131.9, "radius_nm": 250},
+    {"name": "Urumqi", "lat": 43.8, "lon": 87.6, "radius_nm": 250},
+    {"name": "Chengdu", "lat": 30.6, "lon": 104.1, "radius_nm": 250},
+    {"name": "Lagos-Accra", "lat": 6.5, "lon": 3.4, "radius_nm": 250},
+    {"name": "Addis Ababa", "lat": 9.0, "lon": 38.7, "radius_nm": 250},
 ]
-_SUPPLEMENTAL_FETCH_INTERVAL = 120
+# The blind-spot supplement previously burst several airplanes.live point
+# queries in parallel and triggered repeated 429s in real startup logs, so we
+# keep it on a long cache interval and pace each regional point query serially.
+_SUPPLEMENTAL_FETCH_INTERVAL = 1800
+_AIRPLANES_LIVE_DELAY_SECONDS = 1.2
+_AIRPLANES_LIVE_DELAY_JITTER_SECONDS = 0.4
 last_supplemental_fetch = 0
 cached_supplemental_flights = []
 
 # Helicopter type codes (backend classification)
 _HELI_TYPES_BACKEND = {
-    "R22", "R44", "R66", "B06", "B06T", "B204", "B205", "B206", "B212", "B222", "B230",
-    "B407", "B412", "B427", "B429", "B430", "B505", "B525",
-    "AS32", "AS35", "AS50", "AS55", "AS65",
-    "EC20", "EC25", "EC30", "EC35", "EC45", "EC55", "EC75",
-    "H125", "H130", "H135", "H145", "H155", "H160", "H175", "H215", "H225",
-    "S55", "S58", "S61", "S64", "S70", "S76", "S92",
-    "A109", "A119", "A139", "A169", "A189", "AW09",
-    "MD52", "MD60", "MDHI", "MD90", "NOTR",
-    "B47G", "HUEY", "GAMA", "CABR", "EXE",
+    "R22",
+    "R44",
+    "R66",
+    "B06",
+    "B06T",
+    "B204",
+    "B205",
+    "B206",
+    "B212",
+    "B222",
+    "B230",
+    "B407",
+    "B412",
+    "B427",
+    "B429",
+    "B430",
+    "B505",
+    "B525",
+    "AS32",
+    "AS35",
+    "AS50",
+    "AS55",
+    "AS65",
+    "EC20",
+    "EC25",
+    "EC30",
+    "EC35",
+    "EC45",
+    "EC55",
+    "EC75",
+    "H125",
+    "H130",
+    "H135",
+    "H145",
+    "H155",
+    "H160",
+    "H175",
+    "H215",
+    "H225",
+    "S55",
+    "S58",
+    "S61",
+    "S64",
+    "S70",
+    "S76",
+    "S92",
+    "A109",
+    "A119",
+    "A139",
+    "A169",
+    "A189",
+    "AW09",
+    "MD52",
+    "MD60",
+    "MDHI",
+    "MD90",
+    "NOTR",
+    "B47G",
+    "HUEY",
+    "GAMA",
+    "CABR",
+    "EXE",
 }
 
 # Private jet ICAO type designator codes
 PRIVATE_JET_TYPES = {
-    "G150", "G200", "G280", "GLEX", "G500", "G550", "G600", "G650", "G700",
-    "GLF2", "GLF3", "GLF4", "GLF5", "GLF6", "GL5T", "GL7T", "GV", "GIV",
-    "CL30", "CL35", "CL60", "BD70", "BD10", "GL5T", "GL7T",
-    "CRJ1", "CRJ2",
-    "C25A", "C25B", "C25C", "C500", "C501", "C510", "C525", "C526",
-    "C550", "C560", "C56X", "C680", "C68A", "C700", "C750",
-    "FA10", "FA20", "FA50", "FA7X", "FA8X", "F900", "F2TH", "ASTR",
-    "E35L", "E545", "E550", "E55P", "LEGA", "PH10", "PH30",
-    "LJ23", "LJ24", "LJ25", "LJ28", "LJ31", "LJ35", "LJ36",
-    "LJ40", "LJ45", "LJ55", "LJ60", "LJ70", "LJ75",
-    "H25A", "H25B", "H25C", "HA4T", "BE40", "PRM1",
-    "HDJT", "PC24", "EA50", "SF50", "GALX",
+    "G150",
+    "G200",
+    "G280",
+    "GLEX",
+    "G500",
+    "G550",
+    "G600",
+    "G650",
+    "G700",
+    "GLF2",
+    "GLF3",
+    "GLF4",
+    "GLF5",
+    "GLF6",
+    "GL5T",
+    "GL7T",
+    "GV",
+    "GIV",
+    "CL30",
+    "CL35",
+    "CL60",
+    "BD70",
+    "BD10",
+    "GL5T",
+    "GL7T",
+    "CRJ1",
+    "CRJ2",
+    "C25A",
+    "C25B",
+    "C25C",
+    "C500",
+    "C501",
+    "C510",
+    "C525",
+    "C526",
+    "C550",
+    "C560",
+    "C56X",
+    "C680",
+    "C68A",
+    "C700",
+    "C750",
+    "FA10",
+    "FA20",
+    "FA50",
+    "FA7X",
+    "FA8X",
+    "F900",
+    "F2TH",
+    "ASTR",
+    "E35L",
+    "E545",
+    "E550",
+    "E55P",
+    "LEGA",
+    "PH10",
+    "PH30",
+    "LJ23",
+    "LJ24",
+    "LJ25",
+    "LJ28",
+    "LJ31",
+    "LJ35",
+    "LJ36",
+    "LJ40",
+    "LJ45",
+    "LJ55",
+    "LJ60",
+    "LJ70",
+    "LJ75",
+    "H25A",
+    "H25B",
+    "H25C",
+    "HA4T",
+    "BE40",
+    "PRM1",
+    "HDJT",
+    "PC24",
+    "EA50",
+    "SF50",
+    "GALX",
 }
 
 # Flight trails state
@@ -127,35 +267,59 @@ def _fetch_supplemental_sources(seen_hex: set) -> list:
 
     now = time.time()
     if now - last_supplemental_fetch < _SUPPLEMENTAL_FETCH_INTERVAL:
-        return [f for f in cached_supplemental_flights
-                if f.get("hex", "").lower().strip() not in seen_hex]
+        return [
+            f
+            for f in cached_supplemental_flights
+            if f.get("hex", "").lower().strip() not in seen_hex
+        ]
 
     new_supplemental = []
     supplemental_hex = set()
 
     def _fetch_airplaneslive(region):
         try:
-            url = (f"https://api.airplanes.live/v2/point/"
-                   f"{region['lat']}/{region['lon']}/{region['radius_nm']}")
+            url = (
+                f"https://api.airplanes.live/v2/point/"
+                f"{region['lat']}/{region['lon']}/{region['radius_nm']}"
+            )
             res = fetch_with_curl(url, timeout=10)
             if res.status_code == 200:
                 data = res.json()
                 return data.get("ac", [])
-        except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, json.JSONDecodeError, OSError) as e:
+        except (
+            requests.RequestException,
+            ConnectionError,
+            TimeoutError,
+            ValueError,
+            KeyError,
+            json.JSONDecodeError,
+            OSError,
+        ) as e:
             logger.debug(f"airplanes.live {region['name']} failed: {e}")
         return []
 
     try:
-        with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
-            results = list(pool.map(_fetch_airplaneslive, _BLIND_SPOT_REGIONS))
-        for region_flights in results:
+        for idx, region in enumerate(_BLIND_SPOT_REGIONS):
+            region_flights = _fetch_airplaneslive(region)
             for f in region_flights:
                 h = f.get("hex", "").lower().strip()
                 if h and h not in seen_hex and h not in supplemental_hex:
                     f["supplemental_source"] = "airplanes.live"
                     new_supplemental.append(f)
                     supplemental_hex.add(h)
-    except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
+            if idx < len(_BLIND_SPOT_REGIONS) - 1:
+                time.sleep(
+                    _AIRPLANES_LIVE_DELAY_SECONDS
+                    + random.uniform(0.0, _AIRPLANES_LIVE_DELAY_JITTER_SECONDS)
+                )
+    except (
+        requests.RequestException,
+        ConnectionError,
+        TimeoutError,
+        ValueError,
+        KeyError,
+        OSError,
+    ) as e:
         logger.warning(f"airplanes.live supplemental fetch failed: {e}")
 
     ap_count = len(new_supplemental)
@@ -163,8 +327,10 @@ def _fetch_supplemental_sources(seen_hex: set) -> list:
     try:
         for region in _BLIND_SPOT_REGIONS:
             try:
-                url = (f"https://opendata.adsb.fi/api/v3/lat/"
-                       f"{region['lat']}/lon/{region['lon']}/dist/{region['radius_nm']}")
+                url = (
+                    f"https://opendata.adsb.fi/api/v3/lat/"
+                    f"{region['lat']}/lon/{region['lon']}/dist/{region['radius_nm']}"
+                )
                 res = fetch_with_curl(url, timeout=10)
                 if res.status_code == 200:
                     data = res.json()
@@ -174,10 +340,25 @@ def _fetch_supplemental_sources(seen_hex: set) -> list:
                             f["supplemental_source"] = "adsb.fi"
                             new_supplemental.append(f)
                             supplemental_hex.add(h)
-            except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, json.JSONDecodeError, OSError) as e:
+            except (
+                requests.RequestException,
+                ConnectionError,
+                TimeoutError,
+                ValueError,
+                KeyError,
+                json.JSONDecodeError,
+                OSError,
+            ) as e:
                 logger.debug(f"adsb.fi {region['name']} failed: {e}")
             time.sleep(1.1)
-    except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
+    except (
+        requests.RequestException,
+        ConnectionError,
+        TimeoutError,
+        ValueError,
+        KeyError,
+        OSError,
+    ) as e:
         logger.warning(f"adsb.fi supplemental fetch failed: {e}")
 
     fi_count = len(new_supplemental) - ap_count
@@ -187,8 +368,10 @@ def _fetch_supplemental_sources(seen_hex: set) -> list:
     if new_supplemental:
         _mark_fresh("supplemental_flights")
 
-    logger.info(f"Supplemental: +{len(new_supplemental)} new aircraft from blind-spot "
-                f"hotspots (airplanes.live: {ap_count}, adsb.fi: {fi_count})")
+    logger.info(
+        f"Supplemental: +{len(new_supplemental)} new aircraft from blind-spot "
+        f"hotspots (airplanes.live: {ap_count}, adsb.fi: {fi_count})"
+    )
     return new_supplemental
 
 
@@ -204,18 +387,24 @@ def fetch_routes_background(sampled):
         for f in sampled:
             c_sign = str(f.get("flight", "")).strip()
             if c_sign and c_sign != "UNKNOWN":
-                callsigns_to_query.append({
-                    "callsign": c_sign,
-                    "lat": f.get("lat", 0),
-                    "lng": f.get("lon", 0)
-                })
+                callsigns_to_query.append(
+                    {"callsign": c_sign, "lat": f.get("lat", 0), "lng": f.get("lon", 0)}
+                )
 
         batch_size = 100
-        batches = [callsigns_to_query[i:i+batch_size] for i in range(0, len(callsigns_to_query), batch_size)]
+        batches = [
+            callsigns_to_query[i : i + batch_size]
+            for i in range(0, len(callsigns_to_query), batch_size)
+        ]
 
         for batch in batches:
             try:
-                r = fetch_with_curl("https://api.adsb.lol/api/0/routeset", method="POST", json_data={"planes": batch}, timeout=15)
+                r = fetch_with_curl(
+                    "https://api.adsb.lol/api/0/routeset",
+                    method="POST",
+                    json_data={"planes": batch},
+                    timeout=15,
+                )
                 if r.status_code == 200:
                     route_data = r.json()
                     route_list = []
@@ -238,7 +427,15 @@ def fetch_routes_background(sampled):
                                     "dest_loc": [dest_apt.get("lon", 0), dest_apt.get("lat", 0)],
                                 }
                 time.sleep(0.25)
-            except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, json.JSONDecodeError, OSError) as e:
+            except (
+                requests.RequestException,
+                ConnectionError,
+                TimeoutError,
+                ValueError,
+                KeyError,
+                json.JSONDecodeError,
+                OSError,
+            ) as e:
                 logger.debug(f"Route batch request failed: {e}")
     finally:
         with _routes_lock:
@@ -259,7 +456,9 @@ def _classify_and_publish(all_adsb_flights):
     with _routes_lock:
         already_running = routes_fetch_in_progress
     if not already_running:
-        threading.Thread(target=fetch_routes_background, args=(all_adsb_flights,), daemon=True).start()
+        threading.Thread(
+            target=fetch_routes_background, args=(all_adsb_flights,), daemon=True
+        ).start()
 
     for f in all_adsb_flights:
         try:
@@ -308,27 +507,29 @@ def _classify_and_publish(all_adsb_flights):
 
             ac_category = "heli" if model_upper in _HELI_TYPES_BACKEND else "plane"
 
-            flights.append({
-                "callsign": flight_str,
-                "country": f.get("r", "N/A"),
-                "lng": float(lng),
-                "lat": float(lat),
-                "alt": alt_value,
-                "heading": heading,
-                "type": "flight",
-                "origin_loc": origin_loc,
-                "dest_loc": dest_loc,
-                "origin_name": origin_name,
-                "dest_name": dest_name,
-                "registration": f.get("r", "N/A"),
-                "model": f.get("t", "Unknown"),
-                "icao24": f.get("hex", ""),
-                "speed_knots": speed_knots,
-                "squawk": f.get("squawk", ""),
-                "airline_code": airline_code,
-                "aircraft_category": ac_category,
-                "nac_p": f.get("nac_p")
-            })
+            flights.append(
+                {
+                    "callsign": flight_str,
+                    "country": f.get("r", "N/A"),
+                    "lng": float(lng),
+                    "lat": float(lat),
+                    "alt": alt_value,
+                    "heading": heading,
+                    "type": "flight",
+                    "origin_loc": origin_loc,
+                    "dest_loc": dest_loc,
+                    "origin_name": origin_name,
+                    "dest_name": dest_name,
+                    "registration": f.get("r", "N/A"),
+                    "model": f.get("t", "Unknown"),
+                    "icao24": f.get("hex", ""),
+                    "speed_knots": speed_knots,
+                    "squawk": f.get("squawk", ""),
+                    "airline_code": airline_code,
+                    "aircraft_category": ac_category,
+                    "nac_p": f.get("nac_p"),
+                }
+            )
         except (ValueError, TypeError, KeyError, AttributeError) as loop_e:
             logger.error(f"Flight interpolation error: {loop_e}")
             continue
@@ -342,80 +543,97 @@ def _classify_and_publish(all_adsb_flights):
     for f in flights:
         enrich_with_plane_alert(f)
         enrich_with_tracked_names(f)
+        # Attach fuel-burn / CO2 emissions estimate when model is known
+        model = f.get("model")
+        if model:
+            emi = get_emissions_info(model)
+            if emi:
+                f["emissions"] = emi
 
-        callsign = f.get('callsign', '').strip().upper()
-        is_commercial_format = bool(re.match(r'^[A-Z]{3}\d{1,4}[A-Z]{0,2}$', callsign))
+        callsign = f.get("callsign", "").strip().upper()
+        is_commercial_format = bool(re.match(r"^[A-Z]{3}\d{1,4}[A-Z]{0,2}$", callsign))
 
-        if f.get('alert_category'):
-            f['type'] = 'tracked_flight'
+        if f.get("alert_category"):
+            f["type"] = "tracked_flight"
             tracked.append(f)
-        elif f.get('airline_code') or is_commercial_format:
-            f['type'] = 'commercial_flight'
+        elif f.get("airline_code") or is_commercial_format:
+            f["type"] = "commercial_flight"
             commercial.append(f)
-        elif f.get('model', '').upper() in PRIVATE_JET_TYPES:
-            f['type'] = 'private_jet'
+        elif f.get("model", "").upper() in PRIVATE_JET_TYPES:
+            f["type"] = "private_jet"
             private_jets.append(f)
         else:
-            f['type'] = 'private_ga'
+            f["type"] = "private_ga"
             private_ga.append(f)
 
     # --- Smart merge: protect against partial API failures ---
-    prev_commercial_count = len(latest_data.get('commercial_flights', []))
-    prev_total = prev_commercial_count + len(latest_data.get('private_jets', [])) + len(latest_data.get('private_flights', []))
+    with _data_lock:
+        prev_commercial_count = len(latest_data.get("commercial_flights", []))
+        prev_private_jets_count = len(latest_data.get("private_jets", []))
+        prev_private_flights_count = len(latest_data.get("private_flights", []))
+    prev_total = prev_commercial_count + prev_private_jets_count + prev_private_flights_count
     new_total = len(commercial) + len(private_jets) + len(private_ga)
 
     if new_total == 0:
         logger.warning("No civilian flights found! Skipping overwrite to prevent clearing the map.")
     elif prev_total > 100 and new_total < prev_total * 0.5:
-        logger.warning(f"Flight count dropped from {prev_total} to {new_total} (>50% loss). Keeping previous data to prevent flicker.")
+        logger.warning(
+            f"Flight count dropped from {prev_total} to {new_total} (>50% loss). Keeping previous data to prevent flicker."
+        )
     else:
         _now = time.time()
 
         def _merge_category(new_list, old_list, max_stale_s=120):
             by_icao = {}
             for f in old_list:
-                icao = f.get('icao24', '')
+                icao = f.get("icao24", "")
                 if icao:
-                    f.setdefault('_seen_at', _now)
-                    if (_now - f.get('_seen_at', _now)) < max_stale_s:
+                    f.setdefault("_seen_at", _now)
+                    if (_now - f.get("_seen_at", _now)) < max_stale_s:
                         by_icao[icao] = f
             for f in new_list:
-                icao = f.get('icao24', '')
+                icao = f.get("icao24", "")
                 if icao:
-                    f['_seen_at'] = _now
+                    f["_seen_at"] = _now
                     by_icao[icao] = f
                 else:
                     continue
             return list(by_icao.values())
 
         with _data_lock:
-            latest_data['commercial_flights'] = _merge_category(commercial, latest_data.get('commercial_flights', []))
-            latest_data['private_jets'] = _merge_category(private_jets, latest_data.get('private_jets', []))
-            latest_data['private_flights'] = _merge_category(private_ga, latest_data.get('private_flights', []))
+            latest_data["commercial_flights"] = _merge_category(
+                commercial, latest_data.get("commercial_flights", [])
+            )
+            latest_data["private_jets"] = _merge_category(
+                private_jets, latest_data.get("private_jets", [])
+            )
+            latest_data["private_flights"] = _merge_category(
+                private_ga, latest_data.get("private_flights", [])
+            )
 
     _mark_fresh("commercial_flights", "private_jets", "private_flights")
 
     with _data_lock:
         if flights:
-            latest_data['flights'] = flights
+            latest_data["flights"] = flights
 
     # Merge tracked civilian flights with tracked military flights
     with _data_lock:
-        existing_tracked = list(latest_data.get('tracked_flights', []))
+        existing_tracked = copy.deepcopy(latest_data.get("tracked_flights", []))
 
     fresh_tracked_map = {}
     for t in tracked:
-        icao = t.get('icao24', '').upper()
+        icao = t.get("icao24", "").upper()
         if icao:
             fresh_tracked_map[icao] = t
 
     merged_tracked = []
     seen_icaos = set()
     for old_t in existing_tracked:
-        icao = old_t.get('icao24', '').upper()
+        icao = old_t.get("icao24", "").upper()
         if icao in fresh_tracked_map:
             fresh = fresh_tracked_map[icao]
-            for key in ('alert_category', 'alert_operator', 'alert_special', 'alert_flag'):
+            for key in ("alert_category", "alert_operator", "alert_special", "alert_flag"):
                 if key in old_t and key not in fresh:
                     fresh[key] = old_t[key]
             merged_tracked.append(fresh)
@@ -429,36 +647,47 @@ def _classify_and_publish(all_adsb_flights):
             merged_tracked.append(t)
 
     with _data_lock:
-        latest_data['tracked_flights'] = merged_tracked
-    logger.info(f"Tracked flights: {len(merged_tracked)} total ({len(fresh_tracked_map)} fresh from civilian)")
+        latest_data["tracked_flights"] = merged_tracked
+    logger.info(
+        f"Tracked flights: {len(merged_tracked)} total ({len(fresh_tracked_map)} fresh from civilian)"
+    )
 
     # --- Trail Accumulation ---
     def _accumulate_trail(f, now_ts, check_route=True):
-        hex_id = f.get('icao24', '').lower()
+        hex_id = f.get("icao24", "").lower()
         if not hex_id:
             return 0, None
-        if check_route and f.get('origin_name', 'UNKNOWN') != 'UNKNOWN':
-            f['trail'] = []
+        if check_route and f.get("origin_name", "UNKNOWN") != "UNKNOWN":
+            f["trail"] = []
             return 0, hex_id
-        lat, lng, alt = f.get('lat'), f.get('lng'), f.get('alt', 0)
+        lat, lng, alt = f.get("lat"), f.get("lng"), f.get("alt", 0)
         if lat is None or lng is None:
-            f['trail'] = flight_trails.get(hex_id, {}).get('points', [])
+            f["trail"] = flight_trails.get(hex_id, {}).get("points", [])
             return 0, hex_id
         point = [round(lat, 5), round(lng, 5), round(alt, 1), round(now_ts)]
         if hex_id not in flight_trails:
-            flight_trails[hex_id] = {'points': [], 'last_seen': now_ts}
+            flight_trails[hex_id] = {"points": [], "last_seen": now_ts}
         trail_data = flight_trails[hex_id]
-        if trail_data['points'] and trail_data['points'][-1][0] == point[0] and trail_data['points'][-1][1] == point[1]:
-            trail_data['last_seen'] = now_ts
+        if (
+            trail_data["points"]
+            and trail_data["points"][-1][0] == point[0]
+            and trail_data["points"][-1][1] == point[1]
+        ):
+            trail_data["last_seen"] = now_ts
         else:
-            trail_data['points'].append(point)
-            trail_data['last_seen'] = now_ts
-        if len(trail_data['points']) > 200:
-            trail_data['points'] = trail_data['points'][-200:]
-        f['trail'] = trail_data['points']
+            trail_data["points"].append(point)
+            trail_data["last_seen"] = now_ts
+        if len(trail_data["points"]) > 200:
+            trail_data["points"] = trail_data["points"][-200:]
+        f["trail"] = trail_data["points"]
         return 1, hex_id
 
     now_ts = datetime.utcnow().timestamp()
+    with _data_lock:
+        military_snapshot = copy.deepcopy(latest_data.get("military_flights", []))
+        tracked_snapshot = copy.deepcopy(latest_data.get("tracked_flights", []))
+        raw_flights_snapshot = list(latest_data.get("flights", []))
+
     all_lists = [commercial, private_jets, private_ga, existing_tracked]
     seen_hexes = set()
     trail_count = 0
@@ -470,97 +699,121 @@ def _classify_and_publish(all_adsb_flights):
                 if hex_id:
                     seen_hexes.add(hex_id)
 
-        for mf in latest_data.get('military_flights', []):
+        for mf in military_snapshot:
             count, hex_id = _accumulate_trail(mf, now_ts, check_route=False)
             trail_count += count
             if hex_id:
                 seen_hexes.add(hex_id)
 
-        tracked_hexes = {t.get('icao24', '').lower() for t in latest_data.get('tracked_flights', [])}
+        tracked_hexes = {t.get("icao24", "").lower() for t in tracked_snapshot}
         stale_keys = []
         for k, v in flight_trails.items():
             cutoff = now_ts - 1800 if k in tracked_hexes else now_ts - 300
-            if v['last_seen'] < cutoff:
+            if v["last_seen"] < cutoff:
                 stale_keys.append(k)
         for k in stale_keys:
             del flight_trails[k]
 
         if len(flight_trails) > _MAX_TRACKED_TRAILS:
-            sorted_keys = sorted(flight_trails.keys(), key=lambda k: flight_trails[k]['last_seen'])
+            sorted_keys = sorted(flight_trails.keys(), key=lambda k: flight_trails[k]["last_seen"])
             evict_count = len(flight_trails) - _MAX_TRACKED_TRAILS
             for k in sorted_keys[:evict_count]:
                 del flight_trails[k]
 
-    logger.info(f"Trail accumulation: {trail_count} active trails, {len(stale_keys)} pruned, {len(flight_trails)} total")
+    logger.info(
+        f"Trail accumulation: {trail_count} active trails, {len(stale_keys)} pruned, {len(flight_trails)} total"
+    )
 
     # --- GPS Jamming Detection ---
+    # Uses NACp (Navigation Accuracy Category – Position) from ADS-B to infer
+    # GPS interference zones, similar to GPSJam.org / Flightradar24.
+    # NACp < 8 = position accuracy worse than the FAA-mandated 0.05 NM.
+    #
+    # Denoising (to suppress false positives from old GA transponders):
+    # 1. Skip nac_p == 0 ("unknown accuracy") — old transponders that never
+    #    computed accuracy, NOT evidence of jamming.  Real jamming shows 1-7.
+    # 2. Require minimum aircraft per grid cell for statistical validity.
+    # 3. Subtract 1 from degraded count per cell (GPSJam's technique) so a
+    #    single quirky transponder can't flag an entire zone.
+    # 4. Require the adjusted ratio to exceed the threshold.
     try:
         jamming_grid = {}
-        raw_flights = latest_data.get('flights', [])
+        raw_flights = raw_flights_snapshot
         for rf in raw_flights:
-            rlat = rf.get('lat')
-            rlng = rf.get('lng') or rf.get('lon')
+            rlat = rf.get("lat")
+            rlng = rf.get("lng") or rf.get("lon")
             if rlat is None or rlng is None:
                 continue
-            nacp = rf.get('nac_p')
-            if nacp is None:
+            nacp = rf.get("nac_p")
+            if nacp is None or nacp == 0:
                 continue
             grid_key = f"{int(rlat)},{int(rlng)}"
             if grid_key not in jamming_grid:
                 jamming_grid[grid_key] = {"degraded": 0, "total": 0}
             jamming_grid[grid_key]["total"] += 1
-            if nacp < 8:
+            if nacp < GPS_JAMMING_NACP_THRESHOLD:
                 jamming_grid[grid_key]["degraded"] += 1
 
         jamming_zones = []
         for gk, counts in jamming_grid.items():
-            if counts["total"] < 3:
+            if counts["total"] < GPS_JAMMING_MIN_AIRCRAFT:
                 continue
-            ratio = counts["degraded"] / counts["total"]
-            if ratio > 0.25:
+            adjusted_degraded = max(counts["degraded"] - 1, 0)
+            if adjusted_degraded == 0:
+                continue
+            ratio = adjusted_degraded / counts["total"]
+            if ratio > GPS_JAMMING_MIN_RATIO:
                 lat_i, lng_i = gk.split(",")
                 severity = "low" if ratio < 0.5 else "medium" if ratio < 0.75 else "high"
-                jamming_zones.append({
-                    "lat": int(lat_i) + 0.5,
-                    "lng": int(lng_i) + 0.5,
-                    "severity": severity,
-                    "ratio": round(ratio, 2),
-                    "degraded": counts["degraded"],
-                    "total": counts["total"]
-                })
+                jamming_zones.append(
+                    {
+                        "lat": int(lat_i) + 0.5,
+                        "lng": int(lng_i) + 0.5,
+                        "severity": severity,
+                        "ratio": round(ratio, 2),
+                        "degraded": counts["degraded"],
+                        "total": counts["total"],
+                    }
+                )
         with _data_lock:
-            latest_data['gps_jamming'] = jamming_zones
+            latest_data["gps_jamming"] = jamming_zones
         if jamming_zones:
             logger.info(f"GPS Jamming: {len(jamming_zones)} interference zones detected")
     except (ValueError, TypeError, KeyError, ZeroDivisionError) as e:
         logger.error(f"GPS Jamming detection error: {e}")
         with _data_lock:
-            latest_data['gps_jamming'] = []
+            latest_data["gps_jamming"] = []
 
     # --- Holding Pattern Detection ---
     try:
         holding_count = 0
-        all_flight_lists = [commercial, private_jets, private_ga,
-                            latest_data.get('tracked_flights', []),
-                            latest_data.get('military_flights', [])]
+        all_flight_lists = [
+            commercial,
+            private_jets,
+            private_ga,
+            tracked_snapshot,
+            military_snapshot,
+        ]
         with _trails_lock:
-            trails_snapshot = {k: v.get('points', [])[:] for k, v in flight_trails.items()}
+            trails_snapshot = {k: v.get("points", [])[:] for k, v in flight_trails.items()}
         for flist in all_flight_lists:
             for f in flist:
-                hex_id = f.get('icao24', '').lower()
+                hex_id = f.get("icao24", "").lower()
                 trail = trails_snapshot.get(hex_id, [])
                 if len(trail) < 6:
-                    f['holding'] = False
+                    f["holding"] = False
                     continue
                 pts = trail[-8:]
                 total_turn = 0.0
                 prev_bearing = 0.0
                 for i in range(1, len(pts)):
-                    lat1, lng1 = math.radians(pts[i-1][0]), math.radians(pts[i-1][1])
+                    lat1, lng1 = math.radians(pts[i - 1][0]), math.radians(pts[i - 1][1])
                     lat2, lng2 = math.radians(pts[i][0]), math.radians(pts[i][1])
                     dlng = lng2 - lng1
                     x = math.sin(dlng) * math.cos(lat2)
-                    y = math.cos(lat1) * math.sin(lat2) - math.sin(lat1) * math.cos(lat2) * math.cos(dlng)
+                    y = math.cos(lat1) * math.sin(lat2) - math.sin(lat1) * math.cos(
+                        lat2
+                    ) * math.cos(dlng)
                     bearing = math.degrees(math.atan2(x, y)) % 360
                     if i > 1:
                         delta = abs(bearing - prev_bearing)
@@ -568,8 +821,8 @@ def _classify_and_publish(all_adsb_flights):
                             delta = 360 - delta
                         total_turn += delta
                     prev_bearing = bearing
-                f['holding'] = total_turn > 300
-                if f['holding']:
+                f["holding"] = total_turn > 300
+                if f["holding"]:
                     holding_count += 1
         if holding_count:
             logger.info(f"Holding patterns: {holding_count} aircraft circling")
@@ -577,7 +830,7 @@ def _classify_and_publish(all_adsb_flights):
         logger.error(f"Holding pattern detection error: {e}")
 
     with _data_lock:
-        latest_data['last_updated'] = datetime.utcnow().isoformat()
+        latest_data["last_updated"] = datetime.utcnow().isoformat()
 
 
 def _fetch_adsb_lol_regions():
@@ -588,7 +841,7 @@ def _fetch_adsb_lol_regions():
         {"lat": 35.0, "lon": 105.0, "dist": 2000},
         {"lat": -25.0, "lon": 133.0, "dist": 2000},
         {"lat": 0.0, "lon": 20.0, "dist": 2500},
-        {"lat": -15.0, "lon": -60.0, "dist": 2000}
+        {"lat": -15.0, "lon": -60.0, "dist": 2000},
     ]
 
     def _fetch_region(r):
@@ -598,7 +851,15 @@ def _fetch_adsb_lol_regions():
             if res.status_code == 200:
                 data = res.json()
                 return data.get("ac", [])
-        except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, json.JSONDecodeError, OSError) as e:
+        except (
+            requests.RequestException,
+            ConnectionError,
+            TimeoutError,
+            ValueError,
+            KeyError,
+            json.JSONDecodeError,
+            OSError,
+        ) as e:
             logger.warning(f"Region fetch failed for lat={r['lat']}: {e}")
         return []
 
@@ -632,9 +893,18 @@ def _enrich_with_opensky_and_supplemental(adsb_flights):
             token = opensky_client.get_token()
             if token:
                 opensky_regions = [
-                    {"name": "Africa", "bbox": {"lamin": -35.0, "lomin": -20.0, "lamax": 38.0, "lomax": 55.0}},
-                    {"name": "Asia", "bbox": {"lamin": 0.0, "lomin": 30.0, "lamax": 75.0, "lomax": 150.0}},
-                    {"name": "South America", "bbox": {"lamin": -60.0, "lomin": -95.0, "lamax": 15.0, "lomax": -30.0}}
+                    {
+                        "name": "Africa",
+                        "bbox": {"lamin": -35.0, "lomin": -20.0, "lamax": 38.0, "lomax": 55.0},
+                    },
+                    {
+                        "name": "Asia",
+                        "bbox": {"lamin": 0.0, "lomin": 30.0, "lamax": 75.0, "lomax": 150.0},
+                    },
+                    {
+                        "name": "South America",
+                        "bbox": {"lamin": -60.0, "lomin": -95.0, "lamax": 15.0, "lomax": -30.0},
+                    },
                 ]
 
                 new_opensky_flights = []
@@ -648,24 +918,38 @@ def _enrich_with_opensky_and_supplemental(adsb_flights):
                         if os_res.status_code == 200:
                             os_data = os_res.json()
                             states = os_data.get("states") or []
-                            logger.info(f"OpenSky: Fetched {len(states)} states for {os_reg['name']}")
+                            logger.info(
+                                f"OpenSky: Fetched {len(states)} states for {os_reg['name']}"
+                            )
 
                             for s in states:
-                                new_opensky_flights.append({
-                                    "hex": s[0],
-                                    "flight": s[1].strip() if s[1] else "UNKNOWN",
-                                    "r": s[2],
-                                    "lon": s[5],
-                                    "lat": s[6],
-                                    "alt_baro": (s[7] * 3.28084) if s[7] else 0,
-                                    "track": s[10] or 0,
-                                    "gs": (s[9] * 1.94384) if s[9] else 0,
-                                    "t": "Unknown",
-                                    "is_opensky": True
-                                })
+                                new_opensky_flights.append(
+                                    {
+                                        "hex": s[0],
+                                        "flight": s[1].strip() if s[1] else "UNKNOWN",
+                                        "r": s[2],
+                                        "lon": s[5],
+                                        "lat": s[6],
+                                        "alt_baro": (s[7] * 3.28084) if s[7] else 0,
+                                        "track": s[10] or 0,
+                                        "gs": (s[9] * 1.94384) if s[9] else 0,
+                                        "t": "Unknown",
+                                        "is_opensky": True,
+                                    }
+                                )
                         else:
-                            logger.warning(f"OpenSky API {os_reg['name']} failed: {os_res.status_code}")
-                    except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, json.JSONDecodeError, OSError) as ex:
+                            logger.warning(
+                                f"OpenSky API {os_reg['name']} failed: {os_res.status_code}"
+                            )
+                    except (
+                        requests.RequestException,
+                        ConnectionError,
+                        TimeoutError,
+                        ValueError,
+                        KeyError,
+                        json.JSONDecodeError,
+                        OSError,
+                    ) as ex:
                         logger.error(f"OpenSky fetching error for {os_reg['name']}: {ex}")
 
                 cached_opensky_flights = new_opensky_flights
@@ -688,12 +972,21 @@ def _enrich_with_opensky_and_supplemental(adsb_flights):
                     seen_hex.add(h)
             if gap_fill:
                 logger.info(f"Gap-fill: added {len(gap_fill)} aircraft to pipeline")
-        except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
+        except (
+            requests.RequestException,
+            ConnectionError,
+            TimeoutError,
+            ValueError,
+            KeyError,
+            OSError,
+        ) as e:
             logger.warning(f"Supplemental source fetch failed (non-fatal): {e}")
 
         # Re-publish with enriched data
         if len(all_flights) > len(adsb_flights):
-            logger.info(f"Enrichment: {len(all_flights) - len(adsb_flights)} additional aircraft from OpenSky + supplemental")
+            logger.info(
+                f"Enrichment: {len(all_flights) - len(adsb_flights)} additional aircraft from OpenSky + supplemental"
+            )
             _classify_and_publish(all_flights)
     except Exception as e:
         logger.error(f"OpenSky/supplemental enrichment error: {e}")
@@ -705,6 +998,10 @@ def fetch_flights():
     Phase 1 (fast): Fetch adsb.lol → classify → publish immediately (~3-5s)
     Phase 2 (background): Merge OpenSky + supplemental → re-publish (~15-30s)
     """
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("flights", "private", "jets", "tracked", "gps_jamming"):
+        return
     try:
         # Phase 1: adsb.lol — fast, parallel, publish immediately
         adsb_flights = _fetch_adsb_lol_regions()

backend/services/fetchers/geo.py

@@ -1,7 +1,9 @@
-"""Ship and geopolitics fetchers — AIS vessels, carriers, frontlines, GDELT, LiveUAmap."""
+"""Ship and geopolitics fetchers — AIS vessels, carriers, frontlines, GDELT, LiveUAmap, fishing."""
+
 import csv
 import io
 import math
+import os
 import logging
 from services.network_utils import fetch_with_curl
 from services.fetchers._store import latest_data, _data_lock, _mark_fresh
@@ -16,6 +18,12 @@ logger = logging.getLogger(__name__)
 @with_retry(max_retries=1, base_delay=1)
 def fetch_ships():
     """Fetch real-time AIS vessel data and combine with OSINT carrier positions."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active(
+        "ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts"
+    ):
+        return
     from services.ais_stream import get_ais_vessels
     from services.carrier_tracker import get_carrier_positions
 
@@ -23,19 +31,20 @@ def fetch_ships():
     try:
         carriers = get_carrier_positions()
         ships.extend(carriers)
-    except Exception as e:
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
         logger.error(f"Carrier tracker error (non-fatal): {e}")
         carriers = []
 
     try:
         ais_vessels = get_ais_vessels()
         ships.extend(ais_vessels)
-    except Exception as e:
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
         logger.error(f"AIS stream error (non-fatal): {e}")
         ais_vessels = []
 
     # Enrich ships with yacht alert data (tracked superyachts)
     from services.fetchers.yacht_alert import enrich_with_yacht_alert
+
     for ship in ships:
         enrich_with_yacht_alert(ship)
 
@@ -46,7 +55,7 @@ def fetch_ships():
 
     logger.info(f"Ships: {len(carriers)} carriers + {len(ais_vessels)} AIS vessels")
     with _data_lock:
-        latest_data['ships'] = ships
+        latest_data["ships"] = ships
     _mark_fresh("ships")
 
 
@@ -62,16 +71,19 @@ def find_nearest_airport(lat, lng, max_distance_nm=200):
         return None
 
     best = None
-    best_dist = float('inf')
+    best_dist = float("inf")
     lat_r = math.radians(lat)
     lng_r = math.radians(lng)
 
     for apt in cached_airports:
-        apt_lat_r = math.radians(apt['lat'])
-        apt_lng_r = math.radians(apt['lng'])
+        apt_lat_r = math.radians(apt["lat"])
+        apt_lng_r = math.radians(apt["lng"])
         dlat = apt_lat_r - lat_r
         dlng = apt_lng_r - lng_r
-        a = math.sin(dlat / 2) ** 2 + math.cos(lat_r) * math.cos(apt_lat_r) * math.sin(dlng / 2) ** 2
+        a = (
+            math.sin(dlat / 2) ** 2
+            + math.cos(lat_r) * math.cos(apt_lat_r) * math.sin(dlng / 2) ** 2
+        )
         c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
         dist_nm = 3440.065 * c
 
@@ -81,9 +93,11 @@ def find_nearest_airport(lat, lng, max_distance_nm=200):
 
     if best and best_dist <= max_distance_nm:
         return {
-            "iata": best['iata'], "name": best['name'],
-            "lat": best['lat'], "lng": best['lng'],
-            "distance_nm": round(best_dist, 1)
+            "iata": best["iata"],
+            "name": best["name"],
+            "lat": best["lat"],
+            "lng": best["lng"],
+            "distance_nm": round(best_dist, 1),
         }
     return None
 
@@ -99,21 +113,23 @@ def fetch_airports():
                 f = io.StringIO(response.text)
                 reader = csv.DictReader(f)
                 for row in reader:
-                    if row['type'] == 'large_airport' and row['iata_code']:
-                        cached_airports.append({
-                            "id": row['ident'],
-                            "name": row['name'],
-                            "iata": row['iata_code'],
-                            "lat": float(row['latitude_deg']),
-                            "lng": float(row['longitude_deg']),
-                            "type": "airport"
-                        })
+                    if row["type"] == "large_airport" and row["iata_code"]:
+                        cached_airports.append(
+                            {
+                                "id": row["ident"],
+                                "name": row["name"],
+                                "iata": row["iata_code"],
+                                "lat": float(row["latitude_deg"]),
+                                "lng": float(row["longitude_deg"]),
+                                "type": "airport",
+                            }
+                        )
                 logger.info(f"Loaded {len(cached_airports)} large airports into cache.")
-        except Exception as e:
+        except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
             logger.error(f"Error fetching airports: {e}")
 
     with _data_lock:
-        latest_data['airports'] = cached_airports
+        latest_data["airports"] = cached_airports
 
 
 # ---------------------------------------------------------------------------
@@ -122,28 +138,38 @@ def fetch_airports():
 @with_retry(max_retries=1, base_delay=2)
 def fetch_frontlines():
     """Fetch Ukraine frontline data (fast — single GitHub API call)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("ukraine_frontline"):
+        return
     try:
         from services.geopolitics import fetch_ukraine_frontlines
+
         frontlines = fetch_ukraine_frontlines()
         if frontlines:
             with _data_lock:
-                latest_data['frontlines'] = frontlines
+                latest_data["frontlines"] = frontlines
             _mark_fresh("frontlines")
-    except Exception as e:
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
         logger.error(f"Error fetching frontlines: {e}")
 
 
 @with_retry(max_retries=1, base_delay=3)
 def fetch_gdelt():
     """Fetch GDELT global military incidents (slow — downloads 32 ZIP files)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("global_incidents"):
+        return
     try:
         from services.geopolitics import fetch_global_military_incidents
+
         gdelt = fetch_global_military_incidents()
         if gdelt is not None:
             with _data_lock:
-                latest_data['gdelt'] = gdelt
+                latest_data["gdelt"] = gdelt
             _mark_fresh("gdelt")
-    except Exception as e:
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
         logger.error(f"Error fetching GDELT: {e}")
 
 
@@ -154,13 +180,72 @@ def fetch_geopolitics():
 
 
 def update_liveuamap():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("global_incidents"):
+        return
     logger.info("Running scheduled Liveuamap scraper...")
     try:
         from services.liveuamap_scraper import fetch_liveuamap
+
         res = fetch_liveuamap()
         if res:
             with _data_lock:
-                latest_data['liveuamap'] = res
+                latest_data["liveuamap"] = res
             _mark_fresh("liveuamap")
-    except Exception as e:
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
         logger.error(f"Liveuamap scraper error: {e}")
+
+
+# ---------------------------------------------------------------------------
+# Fishing Activity (Global Fishing Watch)
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=1, base_delay=5)
+def fetch_fishing_activity():
+    """Fetch recent fishing events from Global Fishing Watch (~5 day lag)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("fishing_activity"):
+        return
+    token = os.environ.get("GFW_API_TOKEN", "")
+    if not token:
+        logger.debug("GFW_API_TOKEN not set, skipping fishing activity fetch")
+        return
+    events = []
+    try:
+        url = (
+            "https://gateway.api.globalfishingwatch.org/v3/events"
+            "?datasets[0]=public-global-fishing-events:latest"
+            "&limit=500&sort=start&sort-direction=DESC"
+        )
+        headers = {"Authorization": f"Bearer {token}"}
+        response = fetch_with_curl(url, timeout=30, headers=headers)
+        if response.status_code == 200:
+            entries = response.json().get("entries", [])
+            for e in entries:
+                pos = e.get("position", {})
+                lat = pos.get("lat")
+                lng = pos.get("lon")
+                if lat is None or lng is None:
+                    continue
+                dur = e.get("event", {}).get("duration", 0) or 0
+                events.append(
+                    {
+                        "id": e.get("id", ""),
+                        "type": e.get("type", "fishing"),
+                        "lat": lat,
+                        "lng": lng,
+                        "start": e.get("start", ""),
+                        "end": e.get("end", ""),
+                        "vessel_name": (e.get("vessel") or {}).get("name", "Unknown"),
+                        "vessel_flag": (e.get("vessel") or {}).get("flag", ""),
+                        "duration_hrs": round(dur / 3600, 1),
+                    }
+                )
+        logger.info(f"Fishing activity: {len(events)} events")
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching fishing activity: {e}")
+    with _data_lock:
+        latest_data["fishing_activity"] = events
+    if events:
+        _mark_fresh("fishing_activity")

backend/services/fetchers/infrastructure.py

@@ -1,4 +1,5 @@
 """Infrastructure fetchers — internet outages (IODA), data centers, CCTV, KiwiSDR."""
+
 import json
 import time
 import heapq
@@ -25,6 +26,7 @@ def _geocode_region(region_name: str, country_name: str) -> tuple:
         return _region_geocode_cache[cache_key]
     try:
         import urllib.parse
+
         query = urllib.parse.quote(f"{region_name}, {country_name}")
         url = f"https://nominatim.openstreetmap.org/search?q={query}&format=json&limit=1"
         response = fetch_with_curl(url, timeout=8, headers={"User-Agent": "ShadowBroker-OSINT/1.0"})
@@ -35,7 +37,7 @@ def _geocode_region(region_name: str, country_name: str) -> tuple:
                 lon = float(results[0]["lon"])
                 _region_geocode_cache[cache_key] = (lat, lon)
                 return (lat, lon)
-    except Exception:
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError):
         pass
     _region_geocode_cache[cache_key] = None
     return None
@@ -44,6 +46,10 @@ def _geocode_region(region_name: str, country_name: str) -> tuple:
 @with_retry(max_retries=1, base_delay=1)
 def fetch_internet_outages():
     """Fetch regional internet outage alerts from IODA (Georgia Tech)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("internet_outages"):
+        return
     RELIABLE_DATASOURCES = {"bgp", "ping-slash24"}
     outages = []
     try:
@@ -96,7 +102,15 @@ def fetch_internet_outages():
                     geocoded.append(r)
             outages = heapq.nlargest(100, geocoded, key=lambda x: x["severity"])
         logger.info(f"Internet outages: {len(outages)} regions affected")
-    except Exception as e:
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
         logger.error(f"Error fetching internet outages: {e}")
     with _data_lock:
         latest_data["internet_outages"] = outages
@@ -104,6 +118,116 @@ def fetch_internet_outages():
         _mark_fresh("internet_outages")
 
 
+# ---------------------------------------------------------------------------
+# RIPE Atlas — complement IODA with probe-level disconnection data
+# ---------------------------------------------------------------------------
+
+@with_retry(max_retries=1, base_delay=3)
+def fetch_ripe_atlas_probes():
+    """Fetch disconnected RIPE Atlas probes and merge into internet_outages (complementing IODA)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("internet_outages"):
+        return
+    try:
+        # 1. Fetch disconnected probes (status=2) — ~2,000 probes, no auth needed
+        url_disc = "https://atlas.ripe.net/api/v2/probes/?status=2&page_size=500&format=json"
+        resp_disc = fetch_with_curl(url_disc, timeout=20)
+        if resp_disc.status_code != 200:
+            logger.warning(f"RIPE Atlas probes API returned {resp_disc.status_code}")
+            return
+        disc_data = resp_disc.json()
+        disconnected = disc_data.get("results", [])
+
+        # 2. Fetch connected probe count (page_size=1 — we only need the count)
+        url_conn = "https://atlas.ripe.net/api/v2/probes/?status=1&page_size=1&format=json"
+        resp_conn = fetch_with_curl(url_conn, timeout=10)
+        total_connected = 0
+        if resp_conn.status_code == 200:
+            total_connected = resp_conn.json().get("count", 0)
+
+        # 3. Group disconnected probes by country
+        country_disc: dict = {}
+        for p in disconnected:
+            cc = p.get("country_code", "")
+            if not cc:
+                continue
+            if cc not in country_disc:
+                country_disc[cc] = []
+            country_disc[cc].append(p)
+
+        # 4. Get IODA-covered countries to avoid double-reporting
+        with _data_lock:
+            ioda_outages = list(latest_data.get("internet_outages", []))
+        ioda_countries = {
+            o.get("country_code", "").upper()
+            for o in ioda_outages
+            if o.get("datasource") != "ripe-atlas"
+        }
+
+        # 5. Build RIPE-only alerts for countries NOT already in IODA
+        ripe_alerts = []
+        for cc, probes in country_disc.items():
+            if cc.upper() in ioda_countries:
+                continue  # IODA already covers this country
+            if len(probes) < 3:
+                continue  # Too few probes to be meaningful
+
+            # Use centroid of disconnected probes as marker location
+            lats = [
+                p["geometry"]["coordinates"][1]
+                for p in probes
+                if p.get("geometry") and p["geometry"].get("coordinates")
+            ]
+            lngs = [
+                p["geometry"]["coordinates"][0]
+                for p in probes
+                if p.get("geometry") and p["geometry"].get("coordinates")
+            ]
+            if not lats:
+                continue
+
+            disc_count = len(probes)
+            # Severity: scale 10-80 based on disconnected probe count
+            severity = min(80, 10 + disc_count * 2)
+
+            ripe_alerts.append({
+                "region_code": f"RIPE-{cc}",
+                "region_name": f"{cc} (Atlas probes)",
+                "country_code": cc,
+                "country_name": cc,
+                "level": "critical" if disc_count >= 10 else "warning",
+                "datasource": "ripe-atlas",
+                "severity": severity,
+                "lat": sum(lats) / len(lats),
+                "lng": sum(lngs) / len(lngs),
+                "probe_count": disc_count,
+            })
+
+        # 6. Merge into internet_outages — keep IODA entries, replace old RIPE entries
+        with _data_lock:
+            current = latest_data.get("internet_outages", [])
+            ioda_only = [o for o in current if o.get("datasource") != "ripe-atlas"]
+            latest_data["internet_outages"] = ioda_only + ripe_alerts
+
+        if ripe_alerts:
+            _mark_fresh("internet_outages")
+        logger.info(
+            f"RIPE Atlas: {len(ripe_alerts)} countries with probe disconnections "
+            f"(from {len(disconnected)} disconnected / ~{total_connected} connected probes)"
+        )
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching RIPE Atlas probes: {e}")
+
+
 # ---------------------------------------------------------------------------
 # Data Centers (local geocoded JSON)
 # ---------------------------------------------------------------------------
@@ -112,6 +236,10 @@ _DC_GEOCODED_PATH = Path(__file__).parent.parent.parent / "data" / "datacenters_
 
 def fetch_datacenters():
     """Load geocoded data centers (5K+ street-level precise locations)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("datacenters"):
+        return
     dcs = []
     try:
         if not _DC_GEOCODED_PATH.exists():
@@ -125,17 +253,28 @@ def fetch_datacenters():
                 continue
             if not (-90 <= lat <= 90 and -180 <= lng <= 180):
                 continue
-            dcs.append({
-                "name": entry.get("name", "Unknown"),
-                "company": entry.get("company", ""),
-                "street": entry.get("street", ""),
-                "city": entry.get("city", ""),
-                "country": entry.get("country", ""),
-                "zip": entry.get("zip", ""),
-                "lat": lat, "lng": lng,
-            })
+            dcs.append(
+                {
+                    "name": entry.get("name", "Unknown"),
+                    "company": entry.get("company", ""),
+                    "street": entry.get("street", ""),
+                    "city": entry.get("city", ""),
+                    "country": entry.get("country", ""),
+                    "zip": entry.get("zip", ""),
+                    "lat": lat,
+                    "lng": lng,
+                }
+            )
         logger.info(f"Data centers: {len(dcs)} geocoded locations loaded")
-    except Exception as e:
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
         logger.error(f"Error loading data centers: {e}")
     with _data_lock:
         latest_data["datacenters"] = dcs
@@ -222,16 +361,34 @@ def fetch_power_plants():
 # CCTV Cameras
 # ---------------------------------------------------------------------------
 def fetch_cctv():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("cctv"):
+        return
     try:
         from services.cctv_pipeline import get_all_cameras
+
         cameras = get_all_cameras()
+        if len(cameras) < 500:
+            # Serve the current DB snapshot immediately and let the scheduled
+            # ingest cycle populate/refresh cameras asynchronously.
+            logger.info(
+                "CCTV DB currently has %d cameras — serving cached snapshot and waiting for scheduled ingest",
+                len(cameras),
+            )
         with _data_lock:
             latest_data["cctv"] = cameras
         _mark_fresh("cctv")
-    except Exception as e:
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
         logger.error(f"Error fetching cctv from DB: {e}")
-        with _data_lock:
-            latest_data["cctv"] = []
 
 
 # ---------------------------------------------------------------------------
@@ -239,13 +396,332 @@ def fetch_cctv():
 # ---------------------------------------------------------------------------
 @with_retry(max_retries=2, base_delay=2)
 def fetch_kiwisdr():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("kiwisdr"):
+        return
     try:
         from services.kiwisdr_fetcher import fetch_kiwisdr_nodes
+
         nodes = fetch_kiwisdr_nodes()
         with _data_lock:
             latest_data["kiwisdr"] = nodes
         _mark_fresh("kiwisdr")
-    except Exception as e:
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
         logger.error(f"Error fetching KiwiSDR nodes: {e}")
         with _data_lock:
             latest_data["kiwisdr"] = []
+
+
+# ---------------------------------------------------------------------------
+# SatNOGS Ground Stations + Observations
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=2, base_delay=2)
+def fetch_satnogs():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("satnogs"):
+        return
+    try:
+        from services.satnogs_fetcher import fetch_satnogs_stations, fetch_satnogs_observations
+
+        stations = fetch_satnogs_stations()
+        obs = fetch_satnogs_observations()
+        with _data_lock:
+            latest_data["satnogs_stations"] = stations
+            latest_data["satnogs_observations"] = obs
+        _mark_fresh("satnogs_stations", "satnogs_observations")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching SatNOGS: {e}")
+
+
+# ---------------------------------------------------------------------------
+# PSK Reporter — HF Digital Mode Spots
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=2, base_delay=2)
+def fetch_psk_reporter():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("psk_reporter"):
+        return
+    try:
+        from services.psk_reporter_fetcher import fetch_psk_reporter_spots
+
+        spots = fetch_psk_reporter_spots()
+        with _data_lock:
+            latest_data["psk_reporter"] = spots
+        _mark_fresh("psk_reporter")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching PSK Reporter: {e}")
+        with _data_lock:
+            latest_data["psk_reporter"] = []
+
+
+# ---------------------------------------------------------------------------
+# TinyGS LoRa Satellites
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=2, base_delay=2)
+def fetch_tinygs():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("tinygs"):
+        return
+    try:
+        from services.tinygs_fetcher import fetch_tinygs_satellites
+
+        sats = fetch_tinygs_satellites()
+        with _data_lock:
+            latest_data["tinygs_satellites"] = sats
+        _mark_fresh("tinygs_satellites")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching TinyGS: {e}")
+
+
+# ---------------------------------------------------------------------------
+# Police Scanners (OpenMHZ) — geocode city+state via local GeoNames DB
+# ---------------------------------------------------------------------------
+_scanner_geo_cache: dict = {}  # city|state -> (lat, lng) — populated once from GeoNames
+
+
+def _build_scanner_geo_lookup():
+    """Build a US city/county→coords lookup from reverse_geocoder's bundled GeoNames CSV."""
+    if _scanner_geo_cache:
+        return
+    try:
+        import csv, os, reverse_geocoder as rg
+
+        geo_file = os.path.join(os.path.dirname(rg.__file__), "rg_cities1000.csv")
+        # US state abbreviation → admin1 name mapping
+        _abbr = {
+            "AL": "Alabama",
+            "AK": "Alaska",
+            "AZ": "Arizona",
+            "AR": "Arkansas",
+            "CA": "California",
+            "CO": "Colorado",
+            "CT": "Connecticut",
+            "DE": "Delaware",
+            "FL": "Florida",
+            "GA": "Georgia",
+            "HI": "Hawaii",
+            "ID": "Idaho",
+            "IL": "Illinois",
+            "IN": "Indiana",
+            "IA": "Iowa",
+            "KS": "Kansas",
+            "KY": "Kentucky",
+            "LA": "Louisiana",
+            "ME": "Maine",
+            "MD": "Maryland",
+            "MA": "Massachusetts",
+            "MI": "Michigan",
+            "MN": "Minnesota",
+            "MS": "Mississippi",
+            "MO": "Missouri",
+            "MT": "Montana",
+            "NE": "Nebraska",
+            "NV": "Nevada",
+            "NH": "New Hampshire",
+            "NJ": "New Jersey",
+            "NM": "New Mexico",
+            "NY": "New York",
+            "NC": "North Carolina",
+            "ND": "North Dakota",
+            "OH": "Ohio",
+            "OK": "Oklahoma",
+            "OR": "Oregon",
+            "PA": "Pennsylvania",
+            "RI": "Rhode Island",
+            "SC": "South Carolina",
+            "SD": "South Dakota",
+            "TN": "Tennessee",
+            "TX": "Texas",
+            "UT": "Utah",
+            "VT": "Vermont",
+            "VA": "Virginia",
+            "WA": "Washington",
+            "WV": "West Virginia",
+            "WI": "Wisconsin",
+            "WY": "Wyoming",
+            "DC": "Washington, D.C.",
+        }
+        state_full = {v.lower(): k for k, v in _abbr.items()}
+        state_full["washington, d.c."] = "DC"
+
+        county_coords = {}  # admin2(county)|state -> (lat, lon) — first city per county
+        with open(geo_file, "r", encoding="utf-8") as f:
+            reader = csv.reader(f)
+            next(reader, None)  # skip header
+            for row in reader:
+                if len(row) < 6 or row[5] != "US":
+                    continue
+                lat_s, lon_s, name, admin1, admin2 = row[0], row[1], row[2], row[3], row[4]
+                st = state_full.get(admin1.lower(), "")
+                if not st:
+                    continue
+                coords = (float(lat_s), float(lon_s))
+                # City name → coords
+                _scanner_geo_cache[f"{name.lower()}|{st}"] = coords
+                # County name → coords (keep first match per county, usually the largest city)
+                if admin2:
+                    county_key = f"{admin2.lower()}|{st}"
+                    if county_key not in county_coords:
+                        county_coords[county_key] = coords
+                    # Also strip " County" suffix for matching
+                    stripped = admin2.lower().replace(" county", "").strip()
+                    stripped_key = f"{stripped}|{st}"
+                    if stripped_key not in county_coords:
+                        county_coords[stripped_key] = coords
+
+        # Merge county lookups (don't override city entries)
+        for k, v in county_coords.items():
+            if k not in _scanner_geo_cache:
+                _scanner_geo_cache[k] = v
+        # Special case: DC
+        _scanner_geo_cache["washington|DC"] = (38.89511, -77.03637)
+        logger.info(f"Scanner geo lookup: {len(_scanner_geo_cache)} US entries loaded")
+    except Exception as e:
+        logger.warning(f"Failed to build scanner geo lookup: {e}")
+
+
+def _geocode_scanner(city: str, state: str):
+    """Look up city+state coordinates from local GeoNames cache."""
+    _build_scanner_geo_lookup()
+    if not city or not state:
+        return None
+    st = state.upper()
+    # Strip trailing state from city (e.g. "Lehigh, PA")
+    c = city.strip()
+    if ", " in c:
+        parts = c.rsplit(", ", 1)
+        if len(parts[1]) <= 2:
+            c = parts[0]
+    name = c.lower()
+    # Try exact city match
+    result = _scanner_geo_cache.get(f"{name}|{st}")
+    if result:
+        return result
+    # Strip "County" / "Co" suffix
+    stripped = name.replace(" county", "").replace(" co", "").strip()
+    result = _scanner_geo_cache.get(f"{stripped}|{st}")
+    if result:
+        return result
+    # Normalize "St." / "St" → "Saint"
+    import re
+
+    normed = re.sub(r"\bst\.?\s", "saint ", name)
+    if normed != name:
+        result = _scanner_geo_cache.get(f"{normed}|{st}")
+        if result:
+            return result
+        # Also try with "s" suffix: "St. Marys" → "Saint Marys" and "Saint Mary's"
+        for variant in [normed.rstrip("s"), normed.replace("ys", "y's")]:
+            result = _scanner_geo_cache.get(f"{variant}|{st}")
+            if result:
+                return result
+    # "Prince Georges" → "Prince George's" (apostrophe variants)
+    if "georges" in name:
+        key = name.replace("georges", "george's") + "|" + st
+        result = _scanner_geo_cache.get(key)
+        if result:
+            return result
+    # Multi-location: "Scott and Carver" → try first part
+    if " and " in name:
+        first = name.split(" and ")[0].strip()
+        result = _scanner_geo_cache.get(f"{first}|{st}")
+        if result:
+            return result
+    # Comma-separated list: "Adams, Jackson, Juneau" → try first
+    if ", " in name:
+        first = name.split(", ")[0].strip()
+        result = _scanner_geo_cache.get(f"{first}|{st}")
+        if result:
+            return result
+    # Drop directional prefix: "North Fulton" → "Fulton"
+    for prefix in ("north ", "south ", "east ", "west "):
+        if name.startswith(prefix):
+            result = _scanner_geo_cache.get(f"{name[len(prefix):]}|{st}")
+            if result:
+                return result
+    return None
+
+
+@with_retry(max_retries=2, base_delay=2)
+def fetch_scanners():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("scanners"):
+        return
+    try:
+        from services.radio_intercept import get_openmhz_systems
+
+        systems = get_openmhz_systems()
+        scanners = []
+        for s in systems:
+            city = s.get("city", "") or s.get("county", "") or ""
+            state = s.get("state", "")
+            coords = _geocode_scanner(city, state)
+            if not coords:
+                continue
+            lat, lng = coords
+            scanners.append(
+                {
+                    "shortName": s.get("shortName", ""),
+                    "name": s.get("name", "Unknown Scanner"),
+                    "lat": round(lat, 5),
+                    "lng": round(lng, 5),
+                    "city": city,
+                    "state": state,
+                    "clientCount": s.get("clientCount", 0),
+                    "description": s.get("description", ""),
+                }
+            )
+        with _data_lock:
+            latest_data["scanners"] = scanners
+        if scanners:
+            _mark_fresh("scanners")
+        logger.info(f"Scanners: {len(scanners)}/{len(systems)} geocoded")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching scanners: {e}")
+        with _data_lock:
+            latest_data["scanners"] = []

backend/services/fetchers/meshtastic_map.py

@@ -0,0 +1,222 @@
+"""Meshtastic Map fetcher — pulls global node positions from meshtastic.liamcottle.net.
+
+Bootstrap + top-up strategy:
+  - On startup: fetch all nodes with positions to seed the map
+  - Every 4 hours: refresh from the API
+  - Persists to JSON cache so data survives restarts
+  - MQTT bridge provides real-time updates between API fetches
+
+API source: https://meshtastic.liamcottle.net/api/v1/nodes (community project by Liam Cottle)
+Polling interval deliberately kept low (4h) to be respectful to the service.
+"""
+
+import json
+import logging
+import time
+from datetime import datetime, timezone, timedelta
+from pathlib import Path
+
+import requests
+
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+
+logger = logging.getLogger("services.data_fetcher")
+
+_API_URL = "https://meshtastic.liamcottle.net/api/v1/nodes"
+_CACHE_FILE = Path(__file__).resolve().parent.parent.parent / "data" / "meshtastic_nodes_cache.json"
+_FETCH_TIMEOUT = 90  # seconds — response is ~37MB, needs time on slow connections
+_MAX_AGE_HOURS = 4  # discard nodes not seen within this window (matches refresh interval)
+
+# Track when we last fetched so the frontend can show staleness
+_last_fetch_ts: float = 0.0
+
+
+def _parse_node(node: dict) -> dict | None:
+    """Convert an API node into a slim signal-like dict."""
+    lat_i = node.get("latitude")
+    lng_i = node.get("longitude")
+    if lat_i is None or lng_i is None:
+        return None
+
+    lat = lat_i / 1e7
+    lng = lng_i / 1e7
+
+    # Basic validity
+    if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+        return None
+    if abs(lat) < 0.1 and abs(lng) < 0.1:
+        return None
+
+    callsign = node.get("node_id_hex", "")
+    if not callsign:
+        nid = node.get("node_id")
+        callsign = f"!{int(nid):08x}" if nid else ""
+    if not callsign:
+        return None
+
+    # Position age from API — reject nodes older than _MAX_AGE_HOURS
+    pos_updated = node.get("position_updated_at") or node.get("updated_at", "")
+    if pos_updated:
+        try:
+            ts = datetime.fromisoformat(pos_updated.replace("Z", "+00:00"))
+            if datetime.now(timezone.utc) - ts > timedelta(hours=_MAX_AGE_HOURS):
+                return None
+        except (ValueError, TypeError):
+            pass
+    else:
+        return None  # no timestamp at all — skip
+
+    return {
+        "callsign": callsign[:20],
+        "lat": round(lat, 5),
+        "lng": round(lng, 5),
+        "source": "meshtastic",
+        "confidence": 0.5,
+        "timestamp": pos_updated,
+        "position_updated_at": pos_updated,
+        "from_api": True,
+        "long_name": (node.get("long_name") or "")[:40],
+        "short_name": (node.get("short_name") or "")[:4],
+        "hardware": node.get("hardware_model_name", ""),
+        "role": node.get("role_name", ""),
+        "battery_level": node.get("battery_level"),
+        "voltage": node.get("voltage"),
+        "altitude": node.get("altitude"),
+    }
+
+
+def _is_fresh(node: dict) -> bool:
+    """Check if a cached node is still within the _MAX_AGE_HOURS window."""
+    ts_str = node.get("position_updated_at") or node.get("timestamp", "")
+    if not ts_str:
+        return False
+    try:
+        ts = datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
+        return datetime.now(timezone.utc) - ts <= timedelta(hours=_MAX_AGE_HOURS)
+    except (ValueError, TypeError):
+        return False
+
+
+def _load_cache() -> list[dict]:
+    """Load cached nodes from disk, filtering out stale entries."""
+    if _CACHE_FILE.exists():
+        try:
+            data = json.loads(_CACHE_FILE.read_text(encoding="utf-8"))
+            nodes = data.get("nodes", [])
+            fresh = [n for n in nodes if _is_fresh(n)]
+            logger.info(f"Meshtastic map cache loaded: {len(fresh)} fresh / {len(nodes)} total")
+            return fresh
+        except Exception as e:
+            logger.warning(f"Failed to load meshtastic cache: {e}")
+    return []
+
+
+def _save_cache(nodes: list[dict], fetch_ts: float):
+    """Persist processed nodes to disk."""
+    try:
+        _CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
+        _CACHE_FILE.write_text(
+            json.dumps(
+                {
+                    "fetched_at": fetch_ts,
+                    "count": len(nodes),
+                    "nodes": nodes,
+                }
+            ),
+            encoding="utf-8",
+        )
+    except Exception as e:
+        logger.warning(f"Failed to save meshtastic cache: {e}")
+
+
+def fetch_meshtastic_nodes():
+    """Fetch global Meshtastic node positions from Liam Cottle's map API.
+
+    Stores processed nodes in latest_data["meshtastic_map_nodes"].
+    Persists to JSON cache for restart resilience.
+    """
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("sigint_meshtastic"):
+        return
+    global _last_fetch_ts
+
+    try:
+        logger.info("Fetching Meshtastic map nodes from API...")
+        resp = requests.get(
+            _API_URL,
+            timeout=_FETCH_TIMEOUT,
+            headers={
+                "User-Agent": "ShadowBroker/1.0 (OSINT dashboard, 4h polling)",
+                "Accept": "application/json",
+            },
+        )
+        resp.raise_for_status()
+
+        raw = resp.json()
+        raw_nodes = raw.get("nodes", []) if isinstance(raw, dict) else raw
+
+        # Parse and filter to only nodes with valid positions
+        parsed = []
+        for node in raw_nodes:
+            sig = _parse_node(node)
+            if sig:
+                parsed.append(sig)
+
+        _last_fetch_ts = time.time()
+        _save_cache(parsed, _last_fetch_ts)
+
+        with _data_lock:
+            latest_data["meshtastic_map_nodes"] = parsed
+            latest_data["meshtastic_map_fetched_at"] = _last_fetch_ts
+        try:
+            from services.fetchers.sigint import refresh_sigint_snapshot
+
+            refresh_sigint_snapshot()
+        except Exception as exc:
+            logger.debug(f"Meshtastic map: SIGINT snapshot refresh skipped: {exc}")
+
+        logger.info(
+            f"Meshtastic map: {len(parsed)} nodes with positions " f"(from {len(raw_nodes)} total)"
+        )
+
+    except Exception as e:
+        logger.error(f"Meshtastic map fetch failed: {e}")
+        # Fall back to cache if available and we have nothing in memory
+        with _data_lock:
+            if not latest_data.get("meshtastic_map_nodes"):
+                cached = _load_cache()
+                if cached:
+                    latest_data["meshtastic_map_nodes"] = cached
+                    latest_data["meshtastic_map_fetched_at"] = (
+                        _CACHE_FILE.stat().st_mtime if _CACHE_FILE.exists() else 0
+                    )
+                    logger.info(
+                        f"Meshtastic map: using {len(cached)} cached nodes (API unavailable)"
+                    )
+                    try:
+                        from services.fetchers.sigint import refresh_sigint_snapshot
+
+                        refresh_sigint_snapshot()
+                    except Exception as exc:
+                        logger.debug(f"Meshtastic map cache: SIGINT snapshot refresh skipped: {exc}")
+
+    _mark_fresh("meshtastic_map")
+
+
+def load_meshtastic_cache_if_available():
+    """On startup, load cached nodes immediately (before first API fetch)."""
+    global _last_fetch_ts
+    cached = _load_cache()
+    if cached:
+        with _data_lock:
+            latest_data["meshtastic_map_nodes"] = cached
+            _last_fetch_ts = _CACHE_FILE.stat().st_mtime if _CACHE_FILE.exists() else 0
+            latest_data["meshtastic_map_fetched_at"] = _last_fetch_ts
+        try:
+            from services.fetchers.sigint import refresh_sigint_snapshot
+
+            refresh_sigint_snapshot()
+        except Exception as exc:
+            logger.debug(f"Meshtastic preload: SIGINT snapshot refresh skipped: {exc}")
+        logger.info(f"Meshtastic map: preloaded {len(cached)} nodes from cache")

backend/services/fetchers/military.py

@@ -1,4 +1,5 @@
 """Military flight tracking and UAV detection from ADS-B data."""
+
 import json
 import logging
 import requests
@@ -13,7 +14,21 @@ logger = logging.getLogger("services.data_fetcher")
 # ---------------------------------------------------------------------------
 _UAV_TYPE_CODES = {"Q9", "R4", "TB2", "MALE", "HALE", "HERM", "HRON"}
 _UAV_CALLSIGN_PREFIXES = ("FORTE", "GHAWK", "REAP", "BAMS", "UAV", "UAS")
-_UAV_MODEL_KEYWORDS = ("RQ-", "MQ-", "RQ4", "MQ9", "MQ4", "MQ1", "REAPER", "GLOBALHAWK", "TRITON", "PREDATOR", "HERMES", "HERON", "BAYRAKTAR")
+_UAV_MODEL_KEYWORDS = (
+    "RQ-",
+    "MQ-",
+    "RQ4",
+    "MQ9",
+    "MQ4",
+    "MQ1",
+    "REAPER",
+    "GLOBALHAWK",
+    "TRITON",
+    "PREDATOR",
+    "HERMES",
+    "HERON",
+    "BAYRAKTAR",
+)
 _UAV_WIKI = {
     "RQ4": "https://en.wikipedia.org/wiki/Northrop_Grumman_RQ-4_Global_Hawk",
     "RQ-4": "https://en.wikipedia.org/wiki/Northrop_Grumman_RQ-4_Global_Hawk",
@@ -137,13 +152,41 @@ def _classify_uav(model: str, callsign: str):
 
 
 def fetch_military_flights():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("military"):
+        return
     military_flights = []
     detected_uavs = []
+    # Fetch from primary + supplemental military endpoints
+    all_mil_ac = []
+    seen_hex = set()
     try:
         url = "https://api.adsb.lol/v2/mil"
         response = fetch_with_curl(url, timeout=10)
         if response.status_code == 200:
-            ac = response.json().get('ac', [])
+            for a in response.json().get("ac", []):
+                h = a.get("hex", "").lower()
+                if h and h not in seen_hex:
+                    seen_hex.add(h)
+                    all_mil_ac.append(a)
+    except Exception as e:
+        logger.warning(f"adsb.lol mil fetch failed: {e}")
+    # Supplemental: airplanes.live military endpoint
+    try:
+        resp2 = fetch_with_curl("https://api.airplanes.live/v2/mil", timeout=10)
+        if resp2.status_code == 200:
+            for a in resp2.json().get("ac", []):
+                h = a.get("hex", "").lower()
+                if h and h not in seen_hex:
+                    seen_hex.add(h)
+                    all_mil_ac.append(a)
+            logger.info(f"airplanes.live mil: +{len(resp2.json().get('ac', []))} raw, {len(all_mil_ac)} total unique")
+    except Exception as e:
+        logger.debug(f"airplanes.live mil supplemental failed: {e}")
+    try:
+        if all_mil_ac:
+            ac = all_mil_ac
             for f in ac:
                 try:
                     lat = f.get("lat")
@@ -218,18 +261,25 @@ def fetch_military_flights():
                 except Exception as loop_e:
                     logger.error(f"Mil flight interpolation error: {loop_e}")
                     continue
-    except Exception as e:
+    except (
+        requests.RequestException,
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+    ) as e:
         logger.error(f"Error fetching military flights: {e}")
 
     if not military_flights and not detected_uavs:
         logger.warning("No military flights retrieved — keeping previous data if available")
         with _data_lock:
-            if latest_data.get('military_flights'):
+            if latest_data.get("military_flights"):
                 return
 
     with _data_lock:
-        latest_data['military_flights'] = military_flights
-        latest_data['uavs'] = detected_uavs
+        latest_data["military_flights"] = military_flights
+        latest_data["uavs"] = detected_uavs
     _mark_fresh("military_flights", "uavs")
     logger.info(f"UAVs: {len(detected_uavs)} real drones detected via ADS-B")
 
@@ -238,30 +288,30 @@ def fetch_military_flights():
     remaining_mil = []
     for mf in military_flights:
         enrich_with_plane_alert(mf)
-        if mf.get('alert_category'):
-            mf['type'] = 'tracked_flight'
+        if mf.get("alert_category"):
+            mf["type"] = "tracked_flight"
             tracked_mil.append(mf)
         else:
             remaining_mil.append(mf)
     with _data_lock:
-        latest_data['military_flights'] = remaining_mil
+        latest_data["military_flights"] = remaining_mil
 
     # Store tracked military flights — update positions for existing entries
     with _data_lock:
-        existing_tracked = list(latest_data.get('tracked_flights', []))
+        existing_tracked = list(latest_data.get("tracked_flights", []))
     fresh_mil_map = {}
     for t in tracked_mil:
-        icao = t.get('icao24', '').upper()
+        icao = t.get("icao24", "").upper()
         if icao:
             fresh_mil_map[icao] = t
 
     updated_tracked = []
     seen_icaos = set()
     for old_t in existing_tracked:
-        icao = old_t.get('icao24', '').upper()
+        icao = old_t.get("icao24", "").upper()
         if icao in fresh_mil_map:
             fresh = fresh_mil_map[icao]
-            for key in ('alert_category', 'alert_operator', 'alert_special', 'alert_flag'):
+            for key in ("alert_category", "alert_operator", "alert_special", "alert_flag"):
                 if key in old_t and key not in fresh:
                     fresh[key] = old_t[key]
             updated_tracked.append(fresh)
@@ -273,5 +323,5 @@ def fetch_military_flights():
         if icao not in seen_icaos:
             updated_tracked.append(t)
     with _data_lock:
-        latest_data['tracked_flights'] = updated_tracked
+        latest_data["tracked_flights"] = updated_tracked
     logger.info(f"Tracked flights: {len(updated_tracked)} total ({len(tracked_mil)} from military)")

backend/services/fetchers/news.py

@@ -7,6 +7,7 @@ import feedparser
 from services.network_utils import fetch_with_curl
 from services.fetchers._store import latest_data, _data_lock, _mark_fresh
 from services.fetchers.retry import with_retry
+from services.oracle_service import enrich_news_items, compute_global_threat_level, detect_breaking_events
 
 logger = logging.getLogger("services.data_fetcher")
 
@@ -170,7 +171,7 @@ def fetch_news():
             logger.warning(f"Feed {source_name} failed: {e}")
             return source_name, None
 
-    with concurrent.futures.ThreadPoolExecutor(max_workers=len(feeds)) as pool:
+    with concurrent.futures.ThreadPoolExecutor(max_workers=min(len(feeds), 6)) as pool:
         feed_results = list(pool.map(_fetch_feed, feeds.items()))
 
     for source_name, feed in feed_results:
@@ -191,7 +192,14 @@ def fetch_news():
                 elif alert_level == "Orange": risk_score = 7
                 else: risk_score = 4
             else:
-                risk_keywords = ['war', 'missile', 'strike', 'attack', 'crisis', 'tension', 'military', 'conflict', 'defense', 'clash', 'nuclear']
+                risk_keywords = [
+                    'war', 'missile', 'strike', 'attack', 'crisis', 'tension',
+                    'military', 'conflict', 'defense', 'clash', 'nuclear',
+                    'sanctions', 'ceasefire', 'invasion', 'drone', 'artillery',
+                    'blockade', 'escalation', 'casualties', 'airspace',
+                    'mobilization', 'proxy', 'insurgent', 'coup',
+                    'assassination', 'bioweapon', 'chemical',
+                ]
                 text = (title + " " + summary).lower()
 
                 risk_score = 1
@@ -268,6 +276,36 @@ def fetch_news():
         })
 
     news_items.sort(key=lambda x: x['risk_score'], reverse=True)
+
+    # Oracle enrichment: sentiment, oracle scores, prediction market odds
+    try:
+        with _data_lock:
+            markets = list(latest_data.get("prediction_markets", []))
+        enrich_news_items(news_items, source_weights, markets)
+        detect_breaking_events(news_items)
+    except Exception as e:
+        logger.warning(f"Oracle enrichment failed (news still usable): {e}")
+
+    # Global threat level computation (fuses news + markets + military + jamming)
+    try:
+        with _data_lock:
+            markets = list(latest_data.get("prediction_markets", []))
+            mil_flights = list(latest_data.get("military_flights", []))
+            jam_zones = list(latest_data.get("gps_jamming", []))
+            ships = list(latest_data.get("ships", []))
+            corr_alerts = list(latest_data.get("correlations", []))
+        threat_level = compute_global_threat_level(
+            news_items, markets,
+            military_flights=mil_flights,
+            gps_jamming=jam_zones,
+            ships=ships,
+            correlations=corr_alerts,
+        )
+    except Exception as e:
+        logger.warning(f"Threat level computation failed: {e}")
+        threat_level = {"score": 0, "level": "GREEN", "color": "#22c55e", "drivers": []}
+
     with _data_lock:
         latest_data['news'] = news_items
+        latest_data['threat_level'] = threat_level
     _mark_fresh("news")

backend/services/fetchers/plane_alert.py

@@ -1,4 +1,5 @@
 """Plane-Alert DB — load and enrich aircraft with tracked metadata."""
+
 import os
 import json
 import logging
@@ -71,38 +72,126 @@ _CATEGORY_COLOR: dict[str, str] = {
     "Radiohead": "purple",
 }
 
+
 def _category_to_color(cat: str) -> str:
     """O(1) exact lookup. Unknown categories default to purple."""
     return _CATEGORY_COLOR.get(cat, "purple")
 
+
 _PLANE_ALERT_DB: dict = {}
 
 # ---------------------------------------------------------------------------
 # POTUS Fleet — override colors and operator names for presidential aircraft.
 # ---------------------------------------------------------------------------
 _POTUS_FLEET: dict[str, dict] = {
-    "ADFDF8": {"color": "#ff1493", "operator": "Air Force One (82-8000)", "category": "Head of State", "wiki": "Air_Force_One", "fleet": "AF1"},
-    "ADFDF9": {"color": "#ff1493", "operator": "Air Force One (92-9000)", "category": "Head of State", "wiki": "Air_Force_One", "fleet": "AF1"},
-    "ADFEB7": {"color": "blue", "operator": "Air Force Two (98-0001)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"},
-    "ADFEB8": {"color": "blue", "operator": "Air Force Two (98-0002)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"},
-    "ADFEB9": {"color": "blue", "operator": "Air Force Two (99-0003)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"},
-    "ADFEBA": {"color": "blue", "operator": "Air Force Two (99-0004)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"},
-    "AE4AE6": {"color": "blue", "operator": "Air Force Two (09-0015)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"},
-    "AE4AE8": {"color": "blue", "operator": "Air Force Two (09-0016)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"},
-    "AE4AEA": {"color": "blue", "operator": "Air Force Two (09-0017)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"},
-    "AE4AEC": {"color": "blue", "operator": "Air Force Two (19-0018)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"},
-    "AE0865": {"color": "#ff1493", "operator": "Marine One (VH-3D)", "category": "Head of State", "wiki": "Marine_One", "fleet": "M1"},
-    "AE5E76": {"color": "#ff1493", "operator": "Marine One (VH-92A)", "category": "Head of State", "wiki": "Marine_One", "fleet": "M1"},
-    "AE5E77": {"color": "#ff1493", "operator": "Marine One (VH-92A)", "category": "Head of State", "wiki": "Marine_One", "fleet": "M1"},
-    "AE5E79": {"color": "#ff1493", "operator": "Marine One (VH-92A)", "category": "Head of State", "wiki": "Marine_One", "fleet": "M1"},
+    "ADFDF8": {
+        "color": "#ff1493",
+        "operator": "Air Force One (82-8000)",
+        "category": "Head of State",
+        "wiki": "Air_Force_One",
+        "fleet": "AF1",
+    },
+    "ADFDF9": {
+        "color": "#ff1493",
+        "operator": "Air Force One (92-9000)",
+        "category": "Head of State",
+        "wiki": "Air_Force_One",
+        "fleet": "AF1",
+    },
+    "ADFEB7": {
+        "color": "blue",
+        "operator": "Air Force Two (98-0001)",
+        "category": "Governments",
+        "wiki": "Air_Force_Two",
+        "fleet": "AF2",
+    },
+    "ADFEB8": {
+        "color": "blue",
+        "operator": "Air Force Two (98-0002)",
+        "category": "Governments",
+        "wiki": "Air_Force_Two",
+        "fleet": "AF2",
+    },
+    "ADFEB9": {
+        "color": "blue",
+        "operator": "Air Force Two (99-0003)",
+        "category": "Governments",
+        "wiki": "Air_Force_Two",
+        "fleet": "AF2",
+    },
+    "ADFEBA": {
+        "color": "blue",
+        "operator": "Air Force Two (99-0004)",
+        "category": "Governments",
+        "wiki": "Air_Force_Two",
+        "fleet": "AF2",
+    },
+    "AE4AE6": {
+        "color": "blue",
+        "operator": "Air Force Two (09-0015)",
+        "category": "Governments",
+        "wiki": "Air_Force_Two",
+        "fleet": "AF2",
+    },
+    "AE4AE8": {
+        "color": "blue",
+        "operator": "Air Force Two (09-0016)",
+        "category": "Governments",
+        "wiki": "Air_Force_Two",
+        "fleet": "AF2",
+    },
+    "AE4AEA": {
+        "color": "blue",
+        "operator": "Air Force Two (09-0017)",
+        "category": "Governments",
+        "wiki": "Air_Force_Two",
+        "fleet": "AF2",
+    },
+    "AE4AEC": {
+        "color": "blue",
+        "operator": "Air Force Two (19-0018)",
+        "category": "Governments",
+        "wiki": "Air_Force_Two",
+        "fleet": "AF2",
+    },
+    "AE0865": {
+        "color": "#ff1493",
+        "operator": "Marine One (VH-3D)",
+        "category": "Head of State",
+        "wiki": "Marine_One",
+        "fleet": "M1",
+    },
+    "AE5E76": {
+        "color": "#ff1493",
+        "operator": "Marine One (VH-92A)",
+        "category": "Head of State",
+        "wiki": "Marine_One",
+        "fleet": "M1",
+    },
+    "AE5E77": {
+        "color": "#ff1493",
+        "operator": "Marine One (VH-92A)",
+        "category": "Head of State",
+        "wiki": "Marine_One",
+        "fleet": "M1",
+    },
+    "AE5E79": {
+        "color": "#ff1493",
+        "operator": "Marine One (VH-92A)",
+        "category": "Head of State",
+        "wiki": "Marine_One",
+        "fleet": "M1",
+    },
 }
 
+
 def _load_plane_alert_db():
     """Load plane_alert_db.json (exported from SQLite) into memory."""
     global _PLANE_ALERT_DB
     json_path = os.path.join(
         os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
-        "data", "plane_alert_db.json"
+        "data",
+        "plane_alert_db.json",
     )
     if not os.path.exists(json_path):
         logger.warning(f"Plane-Alert DB not found at {json_path}")
@@ -124,8 +213,10 @@ def _load_plane_alert_db():
     except (IOError, OSError, json.JSONDecodeError, ValueError, KeyError) as e:
         logger.error(f"Failed to load Plane-Alert DB: {e}")
 
+
 _load_plane_alert_db()
 
+
 def enrich_with_plane_alert(flight: dict) -> dict:
     """If flight's icao24 is in the Plane-Alert DB, add alert metadata."""
     icao = flight.get("icao24", "").strip().upper()
@@ -145,13 +236,16 @@ def enrich_with_plane_alert(flight: dict) -> dict:
             flight["registration"] = info["registration"]
     return flight
 
+
 _TRACKED_NAMES_DB: dict = {}
 
+
 def _load_tracked_names():
     global _TRACKED_NAMES_DB
     json_path = os.path.join(
         os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
-        "data", "tracked_names.json"
+        "data",
+        "tracked_names.json",
     )
     if not os.path.exists(json_path):
         return
@@ -160,16 +254,22 @@ def _load_tracked_names():
             data = json.load(f)
             for name, info in data.get("details", {}).items():
                 cat = info.get("category", "Other")
+                socials = info.get("socials")
                 for reg in info.get("registrations", []):
                     reg_clean = reg.strip().upper()
                     if reg_clean:
-                        _TRACKED_NAMES_DB[reg_clean] = {"name": name, "category": cat}
+                        entry = {"name": name, "category": cat}
+                        if socials:
+                            entry["socials"] = socials
+                        _TRACKED_NAMES_DB[reg_clean] = entry
         logger.info(f"Tracked Names DB loaded: {len(_TRACKED_NAMES_DB)} registrations")
     except (IOError, OSError, json.JSONDecodeError, ValueError, KeyError) as e:
         logger.error(f"Failed to load Tracked Names DB: {e}")
 
+
 _load_tracked_names()
 
+
 def enrich_with_tracked_names(flight: dict) -> dict:
     """If flight's registration matches our Excel extraction, tag it as tracked."""
     icao = flight.get("icao24", "").strip().upper()
@@ -189,11 +289,50 @@ def enrich_with_tracked_names(flight: dict) -> dict:
         name = match["name"]
         flight["alert_operator"] = name
         flight["alert_category"] = match["category"]
+        if match.get("socials"):
+            flight["alert_socials"] = match["socials"]
 
         name_lower = name.lower()
-        is_gov = any(w in name_lower for w in ['state of ', 'government', 'republic', 'ministry', 'department', 'federal', 'cia'])
-        is_law = any(w in name_lower for w in ['police', 'marshal', 'sheriff', 'douane', 'customs', 'patrol', 'gendarmerie', 'guardia', 'law enforcement'])
-        is_med = any(w in name_lower for w in ['fire', 'bomberos', 'ambulance', 'paramedic', 'medevac', 'rescue', 'hospital', 'medical', 'lifeflight'])
+        is_gov = any(
+            w in name_lower
+            for w in [
+                "state of ",
+                "government",
+                "republic",
+                "ministry",
+                "department",
+                "federal",
+                "cia",
+            ]
+        )
+        is_law = any(
+            w in name_lower
+            for w in [
+                "police",
+                "marshal",
+                "sheriff",
+                "douane",
+                "customs",
+                "patrol",
+                "gendarmerie",
+                "guardia",
+                "law enforcement",
+            ]
+        )
+        is_med = any(
+            w in name_lower
+            for w in [
+                "fire",
+                "bomberos",
+                "ambulance",
+                "paramedic",
+                "medevac",
+                "rescue",
+                "hospital",
+                "medical",
+                "lifeflight",
+            ]
+        )
 
         if is_gov or is_law:
             flight["alert_color"] = "blue"

backend/services/fetchers/prediction_markets.py

@@ -0,0 +1,647 @@
+"""Prediction market fetcher — Polymarket (Gamma API) + Kalshi.
+
+Fetches active prediction market events from both platforms, merges them by
+topic similarity, classifies into categories, and stores merged odds with
+full metadata (volume, end dates, descriptions, source badges).
+"""
+
+import json
+import logging
+import math
+from cachetools import TTLCache, cached
+
+logger = logging.getLogger("services.data_fetcher")
+
+_market_cache = TTLCache(maxsize=1, ttl=60)  # 60-second TTL — markets change fast
+
+# Delta tracking: {market_title: previous_consensus_pct}
+_prev_probabilities: dict[str, float] = {}
+
+
+def _finite_or_none(value):
+    try:
+        n = float(value)
+    except (TypeError, ValueError):
+        return None
+    return n if math.isfinite(n) else None
+
+# ---------------------------------------------------------------------------
+# Category classification
+# ---------------------------------------------------------------------------
+CATEGORIES = ["POLITICS", "CONFLICT", "NEWS", "FINANCE", "CRYPTO"]
+
+_KALSHI_CATEGORY_MAP = {
+    "Politics": "POLITICS",
+    "World": "NEWS",
+    "Economics": "FINANCE",
+    "Financials": "FINANCE",
+    "Tech": "FINANCE",
+    "Science": "NEWS",
+    "Climate and Weather": "NEWS",
+    "Sports": "NEWS",
+    "Culture": "NEWS",
+}
+
+_TAG_CATEGORY_MAP = {
+    "Politics": "POLITICS",
+    "Elections": "POLITICS",
+    "US Politics": "POLITICS",
+    "Trump": "POLITICS",
+    "Congress": "POLITICS",
+    "Supreme Court": "POLITICS",
+    "Geopolitics": "CONFLICT",
+    "War": "CONFLICT",
+    "Military": "CONFLICT",
+    "Finance": "FINANCE",
+    "Stocks": "FINANCE",
+    "Economy": "FINANCE",
+    "Business": "FINANCE",
+    "IPOs": "FINANCE",
+    "Crypto": "CRYPTO",
+    "Bitcoin": "CRYPTO",
+    "Ethereum": "CRYPTO",
+    "AI": "NEWS",
+    "Science": "NEWS",
+    "Sports": "NEWS",
+    "Culture": "NEWS",
+    "Entertainment": "NEWS",
+    "Tech": "FINANCE",
+}
+
+_KEYWORD_CATEGORIES = {
+    "CONFLICT": [
+        "war",
+        "military",
+        "attack",
+        "missile",
+        "invasion",
+        "ukraine",
+        "russia",
+        "gaza",
+        "israel",
+        "nato",
+        "troops",
+        "bombing",
+        "nuclear",
+        "sanctions",
+        "ceasefire",
+        "houthi",
+        "iran",
+        "china taiwan",
+        "clash",
+        "conflict",
+        "strike",
+        "weapon",
+    ],
+    "POLITICS": [
+        "trump",
+        "biden",
+        "election",
+        "congress",
+        "senate",
+        "governor",
+        "president",
+        "democrat",
+        "republican",
+        "vote",
+        "party",
+        "cabinet",
+        "impeach",
+        "legislation",
+        "scotus",
+        "poll",
+        "vance",
+        "speaker",
+        "parliament",
+        "prime minister",
+        "macron",
+        "starmer",
+    ],
+    "CRYPTO": [
+        "bitcoin",
+        "btc",
+        "ethereum",
+        "eth",
+        "crypto",
+        "blockchain",
+        "solana",
+        "defi",
+        "nft",
+        "binance",
+        "coinbase",
+        "token",
+        "microstrategy",
+        "stablecoin",
+    ],
+    "FINANCE": [
+        "stock",
+        "fed",
+        "interest rate",
+        "inflation",
+        "gdp",
+        "recession",
+        "s&p",
+        "nasdaq",
+        "dow",
+        "oil",
+        "gold",
+        "treasury",
+        "tariff",
+        "ipo",
+        "earnings",
+        "market cap",
+        "revenue",
+    ],
+}
+
+
+def _classify_category(title: str, poly_tags: list[str], kalshi_category: str) -> str:
+    """Classify a market into one of the 5 categories."""
+    # 1. Kalshi native category
+    if kalshi_category:
+        mapped = _KALSHI_CATEGORY_MAP.get(kalshi_category)
+        if mapped:
+            return mapped
+    # 2. Polymarket tag labels
+    for tag in poly_tags:
+        mapped = _TAG_CATEGORY_MAP.get(tag)
+        if mapped:
+            return mapped
+    # 3. Keyword matching
+    title_lower = title.lower()
+    for cat, keywords in _KEYWORD_CATEGORIES.items():
+        for kw in keywords:
+            if kw in title_lower:
+                return cat
+    # 4. Default
+    return "NEWS"
+
+
+# ---------------------------------------------------------------------------
+# Polymarket
+# ---------------------------------------------------------------------------
+def _fetch_polymarket_events() -> list[dict]:
+    """Fetch active events from Polymarket Gamma API (no auth required).
+
+    Fetches up to 500 events (multiple pages) for better search coverage.
+    """
+    from services.network_utils import fetch_with_curl
+
+    all_events = []
+    for offset in range(0, 500, 100):
+        try:
+            resp = fetch_with_curl(
+                f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=100&offset={offset}",
+                timeout=15,
+            )
+            if not resp or resp.status_code != 200:
+                break
+            page = resp.json()
+            if not isinstance(page, list) or not page:
+                break
+            all_events.extend(page)
+        except Exception as e:
+            logger.warning(f"Polymarket page offset={offset} error: {e}")
+            break
+
+    if not all_events:
+        return []
+
+    try:
+        results = []
+        for ev in all_events:
+            title = ev.get("title", "")
+            if not title:
+                continue
+            # Extract best probability + outcomes from markets
+            markets = ev.get("markets", [])
+            best_pct = None
+            total_volume = 0
+            outcomes = []
+            for m in markets:
+                # Use outcomePrices[0] (Yes price) when available — lastTradePrice
+                # can be for either Yes or No side, causing "99%" for unlikely events
+                raw_op = m.get("outcomePrices")
+                price = None
+                try:
+                    op = json.loads(raw_op) if isinstance(raw_op, str) else raw_op
+                    if isinstance(op, list) and len(op) >= 1:
+                        price = _finite_or_none(op[0])
+                except (json.JSONDecodeError, ValueError, TypeError):
+                    pass
+                if price is None:
+                    price = _finite_or_none(m.get("lastTradePrice") or m.get("bestBid"))
+                pct = None
+                if price is not None:
+                    try:
+                        pct = round(price * 100, 1)
+                        if best_pct is None or pct > best_pct:
+                            best_pct = pct
+                    except (ValueError, TypeError):
+                        pass
+                try:
+                    volume = _finite_or_none(m.get("volume", 0) or 0)
+                    if volume is not None:
+                        total_volume += volume
+                except (ValueError, TypeError):
+                    pass
+                # Collect named outcomes for multi-outcome events
+                oname = m.get("groupItemTitle") or ""
+                if oname and pct is not None:
+                    outcomes.append({"name": oname, "pct": pct})
+            # Only keep outcomes for multi-outcome markets (3+ named outcomes)
+            if len(outcomes) > 2:
+                outcomes.sort(key=lambda x: x["pct"], reverse=True)
+            else:
+                outcomes = []
+
+            # Extract tag labels
+            tag_labels = [t.get("label", "") for t in ev.get("tags", []) if t.get("label")]
+
+            results.append(
+                {
+                    "title": title,
+                    "source": "polymarket",
+                    "pct": best_pct,
+                    "slug": ev.get("slug", ""),
+                    "description": ev.get("description") or "",
+                    "end_date": ev.get("endDate"),
+                    "volume": round(total_volume, 2),
+                    "volume_24h": round(_finite_or_none(ev.get("volume24hr", 0) or 0) or 0, 2),
+                    "tags": tag_labels,
+                    "outcomes": outcomes,
+                }
+            )
+        logger.info(f"Polymarket: fetched {len(results)} active events")
+        return results
+    except Exception as e:
+        logger.error(f"Polymarket fetch error: {e}")
+        return []
+
+
+# ---------------------------------------------------------------------------
+# Kalshi
+# ---------------------------------------------------------------------------
+def _fetch_kalshi_events() -> list[dict]:
+    """Fetch active events from Kalshi public API (no auth required)."""
+    from services.network_utils import fetch_with_curl
+
+    try:
+        resp = fetch_with_curl(
+            "https://api.elections.kalshi.com/v1/events?status=open&limit=100",
+            timeout=15,
+        )
+        if not resp or resp.status_code != 200:
+            logger.warning(f"Kalshi API returned {getattr(resp, 'status_code', 'N/A')}")
+            return []
+        data = resp.json()
+        events = data.get("events", []) if isinstance(data, dict) else []
+
+        results = []
+        for ev in events:
+            title = ev.get("title", "")
+            if not title:
+                continue
+            markets = ev.get("markets", [])
+            best_pct = None
+            total_volume = 0
+            close_dates = []
+            outcomes = []
+            for m in markets:
+                price = m.get("yes_price") or m.get("last_price")
+                pct = None
+                if price is not None:
+                    try:
+                        price = _finite_or_none(price)
+                        if price is None:
+                            raise ValueError("non-finite")
+                        pct = round(price, 1)
+                        if pct <= 1:
+                            pct = round(pct * 100, 1)
+                        if best_pct is None or pct > best_pct:
+                            best_pct = pct
+                    except (ValueError, TypeError):
+                        pass
+                try:
+                        volume = _finite_or_none(
+                            m.get("dollar_volume", 0) or m.get("volume", 0) or 0
+                        )
+                        if volume is not None:
+                            total_volume += int(volume)
+                except (ValueError, TypeError):
+                    pass
+                cd = m.get("close_date")
+                if cd:
+                    close_dates.append(cd)
+                # Collect named outcomes for multi-outcome events
+                oname = m.get("title") or m.get("subtitle", "")
+                if oname and pct is not None:
+                    outcomes.append({"name": oname, "pct": pct})
+            # Only keep outcomes for multi-outcome markets (3+ named outcomes)
+            if len(outcomes) > 2:
+                outcomes.sort(key=lambda x: x["pct"], reverse=True)
+            else:
+                outcomes = []
+
+            # Description: settle_details or underlying
+            desc = (ev.get("settle_details") or ev.get("underlying") or "").strip()
+            sub = ev.get("sub_title", "")
+
+            results.append(
+                {
+                    "title": title,
+                    "source": "kalshi",
+                    "pct": best_pct,
+                    "ticker": ev.get("ticker", ""),
+                    "description": desc,
+                    "sub_title": sub,
+                    "end_date": max(close_dates) if close_dates else None,
+                    "volume": total_volume,
+                    "category": ev.get("category", ""),
+                    "outcomes": outcomes,
+                }
+            )
+        logger.info(f"Kalshi: fetched {len(results)} active events")
+        return results
+    except Exception as e:
+        logger.error(f"Kalshi fetch error: {e}")
+        return []
+
+
+# ---------------------------------------------------------------------------
+# Merge + classify
+# ---------------------------------------------------------------------------
+def _jaccard(a: str, b: str) -> float:
+    """Word-level Jaccard similarity between two strings."""
+    wa = set(a.lower().split())
+    wb = set(b.lower().split())
+    if not wa or not wb:
+        return 0.0
+    return len(wa & wb) / len(wa | wb)
+
+
+def _merge_markets(poly_events: list[dict], kalshi_events: list[dict]) -> list[dict]:
+    """Merge Polymarket and Kalshi events by title similarity.
+
+    Returns a unified list with full metadata, categorized.
+    """
+    merged = []
+    used_kalshi = set()
+
+    for pe in poly_events:
+        best_match = None
+        best_score = 0.0
+        for i, ke in enumerate(kalshi_events):
+            if i in used_kalshi:
+                continue
+            score = _jaccard(pe["title"], ke["title"])
+            if score > best_score and score >= 0.25:
+                best_score = score
+                best_match = (i, ke)
+
+        poly_pct = _finite_or_none(pe.get("pct"))
+        kalshi_pct = None
+        kalshi_vol = 0
+        kalshi_cat = ""
+        kalshi_end = None
+        kalshi_desc = ""
+        kalshi_ticker = ""
+
+        if best_match:
+            used_kalshi.add(best_match[0])
+            ke = best_match[1]
+            kalshi_pct = _finite_or_none(ke.get("pct"))
+            kalshi_vol = _finite_or_none(ke.get("volume", 0)) or 0
+            kalshi_cat = ke.get("category", "")
+            kalshi_end = ke.get("end_date")
+            kalshi_desc = ke.get("description", "")
+            kalshi_ticker = ke.get("ticker", "")
+
+        pcts = [p for p in [poly_pct, kalshi_pct] if p is not None]
+        consensus = round(sum(pcts) / len(pcts), 1) if pcts else None
+
+        # Build sources list
+        sources = []
+        if poly_pct is not None:
+            sources.append({"name": "POLY", "pct": poly_pct})
+        if kalshi_pct is not None:
+            sources.append({"name": "KALSHI", "pct": kalshi_pct})
+
+        category = _classify_category(pe["title"], pe.get("tags", []), kalshi_cat)
+
+        # Use best available description
+        desc = pe.get("description", "") or kalshi_desc
+        end_date = pe.get("end_date") or kalshi_end
+
+        # Use whichever source has more outcomes
+        poly_outcomes = pe.get("outcomes", [])
+        kalshi_outcomes = best_match[1].get("outcomes", []) if best_match else []
+        outcomes = poly_outcomes if len(poly_outcomes) >= len(kalshi_outcomes) else kalshi_outcomes
+
+        merged.append(
+            {
+                "title": pe["title"],
+                "polymarket_pct": poly_pct,
+                "kalshi_pct": kalshi_pct,
+                "consensus_pct": consensus,
+                "description": desc,
+                "end_date": end_date,
+                "volume": _finite_or_none(pe.get("volume", 0)) or 0,
+                "volume_24h": _finite_or_none(pe.get("volume_24h", 0)) or 0,
+                "kalshi_volume": kalshi_vol,
+                "category": category,
+                "sources": sources,
+                "slug": pe.get("slug", ""),
+                "kalshi_ticker": kalshi_ticker,
+                "outcomes": outcomes,
+            }
+        )
+
+    # Unmatched Kalshi events
+    for i, ke in enumerate(kalshi_events):
+        if i in used_kalshi:
+            continue
+        pct = _finite_or_none(ke.get("pct"))
+        sources = []
+        if pct is not None:
+            sources.append({"name": "KALSHI", "pct": pct})
+        category = _classify_category(ke["title"], [], ke.get("category", ""))
+        merged.append(
+            {
+                "title": ke["title"],
+                "polymarket_pct": None,
+                "kalshi_pct": pct,
+                "consensus_pct": pct,
+                "description": ke.get("description", ""),
+                "end_date": ke.get("end_date"),
+                "volume": 0,
+                "volume_24h": 0,
+                "kalshi_volume": _finite_or_none(ke.get("volume", 0)) or 0,
+                "category": category,
+                "sources": sources,
+                "slug": "",
+                "kalshi_ticker": ke.get("ticker", ""),
+                "outcomes": ke.get("outcomes", []),
+            }
+        )
+
+    return merged
+
+
+@cached(_market_cache)
+def fetch_prediction_markets_raw() -> list[dict]:
+    """Fetch and merge prediction markets from both sources. Cached 5 min."""
+    poly = _fetch_polymarket_events()
+    kalshi = _fetch_kalshi_events()
+    merged = _merge_markets(poly, kalshi)
+    logger.info(
+        f"Prediction markets: {len(merged)} merged events "
+        f"({len(poly)} Polymarket, {len(kalshi)} Kalshi)"
+    )
+    return merged
+
+
+def fetch_prediction_markets():
+    """Fetcher entry point — writes merged markets to latest_data."""
+    from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+    global _prev_probabilities
+
+    markets = fetch_prediction_markets_raw()
+
+    # Compute probability deltas vs previous fetch
+    new_probs: dict[str, float] = {}
+    for m in markets:
+        title = m.get("title", "")
+        pct = m.get("consensus_pct")
+        if title and pct is not None:
+            prev = _prev_probabilities.get(title)
+            if prev is not None:
+                m["delta_pct"] = round(pct - prev, 1)
+            else:
+                m["delta_pct"] = None
+            new_probs[title] = pct
+        else:
+            m["delta_pct"] = None
+    _prev_probabilities = new_probs
+
+    # Build trending list (top 10 by absolute delta)
+    trending = sorted(
+        [m for m in markets if m.get("delta_pct") is not None and m["delta_pct"] != 0],
+        key=lambda x: abs(x["delta_pct"]),
+        reverse=True,
+    )[:10]
+
+    with _data_lock:
+        latest_data["prediction_markets"] = markets
+        latest_data["trending_markets"] = trending
+    _mark_fresh("prediction_markets")
+
+
+# ---------------------------------------------------------------------------
+# Direct API search (not limited to cached data)
+# ---------------------------------------------------------------------------
+def search_polymarket_direct(query: str, limit: int = 20) -> list[dict]:
+    """Search Polymarket by scanning API pages for title matches.
+
+    The Gamma API has no text search parameter, so we scan cached events
+    plus additional pages until we find enough matches or exhaust the scan.
+    """
+    from services.network_utils import fetch_with_curl
+
+    q_lower = query.lower()
+    q_words = set(q_lower.split())
+    results = []
+
+    # Scan up to 2000 events (10 pages of 200) looking for title matches
+    for offset in range(0, 2000, 200):
+        try:
+            resp = fetch_with_curl(
+                f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=200&offset={offset}",
+                timeout=15,
+            )
+            if not resp or resp.status_code != 200:
+                break
+            events = resp.json()
+            if not isinstance(events, list) or not events:
+                break
+
+            for ev in events:
+                title = ev.get("title", "")
+                if not title:
+                    continue
+                title_lower = title.lower()
+                # Check if query appears in title or word overlap
+                if q_lower not in title_lower and not any(w in title_lower for w in q_words):
+                    continue
+
+                # Extract same fields as regular fetch
+                markets = ev.get("markets", [])
+                best_pct = None
+                total_volume = 0
+                outcomes = []
+                for m in markets:
+                    # Use outcomePrices[0] (Yes price) when available
+                    raw_op = m.get("outcomePrices")
+                    price = None
+                    try:
+                        op = json.loads(raw_op) if isinstance(raw_op, str) else raw_op
+                        if isinstance(op, list) and len(op) >= 1:
+                            price = _finite_or_none(op[0])
+                    except (json.JSONDecodeError, ValueError, TypeError):
+                        pass
+                    if price is None:
+                        price = _finite_or_none(m.get("lastTradePrice") or m.get("bestBid"))
+                    pct = None
+                    if price is not None:
+                        try:
+                            pct = round(price * 100, 1)
+                            if best_pct is None or pct > best_pct:
+                                best_pct = pct
+                        except (ValueError, TypeError):
+                            pass
+                    try:
+                        volume = _finite_or_none(m.get("volume", 0) or 0)
+                        if volume is not None:
+                            total_volume += volume
+                    except (ValueError, TypeError):
+                        pass
+                    oname = m.get("groupItemTitle") or ""
+                    if oname and pct is not None:
+                        outcomes.append({"name": oname, "pct": pct})
+                if len(outcomes) > 2:
+                    outcomes.sort(key=lambda x: x["pct"], reverse=True)
+                else:
+                    outcomes = []
+
+                tag_labels = [t.get("label", "") for t in ev.get("tags", []) if t.get("label")]
+                category = _classify_category(title, tag_labels, "")
+                sources = []
+                if best_pct is not None:
+                    sources.append({"name": "POLY", "pct": best_pct})
+
+                results.append(
+                    {
+                        "title": title,
+                        "polymarket_pct": best_pct,
+                        "kalshi_pct": None,
+                        "consensus_pct": best_pct,
+                        "description": ev.get("description") or "",
+                        "end_date": ev.get("endDate"),
+                        "volume": round(total_volume, 2),
+                        "volume_24h": round(_finite_or_none(ev.get("volume24hr", 0) or 0) or 0, 2),
+                        "kalshi_volume": 0,
+                        "category": category,
+                        "sources": sources,
+                        "slug": ev.get("slug", ""),
+                        "outcomes": outcomes,
+                    }
+                )
+            # Stop scanning if we have enough results
+            if len(results) >= limit:
+                break
+        except Exception as e:
+            logger.warning(f"Polymarket search scan offset={offset} error: {e}")
+            break
+
+    logger.info(f"Polymarket search '{query}': {len(results)} results (scanned API)")
+    return results[:limit]

backend/services/fetchers/retry.py

@@ -5,22 +5,37 @@ Usage:
     def fetch_something():
         ...
 """
+
 import time
 import random
 import logging
 import functools
+import requests
 
 logger = logging.getLogger(__name__)
 
+# Only retry on transient network/OS errors — not on parse errors, key errors, etc.
+TRANSIENT_ERRORS = (
+    TimeoutError,
+    ConnectionError,
+    OSError,
+    requests.RequestException,
+)
+
 
 def with_retry(max_retries: int = 3, base_delay: float = 2.0, max_delay: float = 30.0):
-    """Decorator: retries the wrapped function on any exception with exponential backoff + jitter.
+    """Decorator: retries the wrapped function on transient errors with exponential backoff + jitter.
+
+    Only retries on network/OS errors (TimeoutError, ConnectionError, OSError,
+    requests.RequestException). Non-transient errors (ValueError, KeyError, etc.)
+    propagate immediately.
 
     Args:
         max_retries:  Number of retry attempts after the initial failure.
         base_delay:   Base delay (seconds) for exponential backoff (2 → 4 → 8 …).
         max_delay:    Cap on the delay between retries.
     """
+
     def decorator(func):
         @functools.wraps(func)
         def wrapper(*args, **kwargs):
@@ -28,22 +43,30 @@ def with_retry(max_retries: int = 3, base_delay: float = 2.0, max_delay: float =
             for attempt in range(1 + max_retries):
                 try:
                     return func(*args, **kwargs)
-                except Exception as exc:
+                except TRANSIENT_ERRORS as exc:
                     last_exc = exc
                     if attempt < max_retries:
-                        delay = min(base_delay * (2 ** attempt), max_delay)
+                        delay = min(base_delay * (2**attempt), max_delay)
                         jitter = random.uniform(0, delay * 0.25)
                         total = delay + jitter
                         logger.warning(
                             "%s failed (attempt %d/%d): %s — retrying in %.1fs",
-                            func.__name__, attempt + 1, max_retries + 1, exc, total,
+                            func.__name__,
+                            attempt + 1,
+                            max_retries + 1,
+                            exc,
+                            total,
                         )
                         time.sleep(total)
                     else:
                         logger.error(
                             "%s failed after %d attempts: %s",
-                            func.__name__, max_retries + 1, exc,
+                            func.__name__,
+                            max_retries + 1,
+                            exc,
                         )
             raise last_exc  # type: ignore[misc]
+
         return wrapper
+
     return decorator

backend/services/fetchers/satellites.py

@@ -6,6 +6,7 @@ CelesTrak Fair Use Policy (https://celestrak.org/NORAD/elements/):
   - No parallel/concurrent connections — one request at a time
   - Set a descriptive User-Agent
 """
+
 import math
 import time
 import json
@@ -24,7 +25,9 @@ logger = logging.getLogger("services.data_fetcher")
 def _gmst(jd_ut1):
     """Greenwich Mean Sidereal Time in radians from Julian Date."""
     t = (jd_ut1 - 2451545.0) / 36525.0
-    gmst_sec = 67310.54841 + (876600.0 * 3600 + 8640184.812866) * t + 0.093104 * t * t - 6.2e-6 * t * t * t
+    gmst_sec = (
+        67310.54841 + (876600.0 * 3600 + 8640184.812866) * t + 0.093104 * t * t - 6.2e-6 * t * t * t
+    )
     gmst_rad = (gmst_sec % 86400) / 86400.0 * 2 * math.pi
     return gmst_rad
 
@@ -38,17 +41,21 @@ _sat_classified_cache = {"data": None, "gp_fetch_ts": 0}
 _SAT_CACHE_PATH = Path(__file__).parent.parent.parent / "data" / "sat_gp_cache.json"
 _SAT_CACHE_META_PATH = Path(__file__).parent.parent.parent / "data" / "sat_gp_cache_meta.json"
 
+
 def _load_sat_cache():
     """Load satellite GP data from local disk cache."""
     try:
         if _SAT_CACHE_PATH.exists():
             import os
+
             age_hours = (time.time() - os.path.getmtime(str(_SAT_CACHE_PATH))) / 3600
             if age_hours < 48:
                 with open(_SAT_CACHE_PATH, "r") as f:
                     data = json.load(f)
                 if isinstance(data, list) and len(data) > 10:
-                    logger.info(f"Satellites: Loaded {len(data)} records from disk cache ({age_hours:.1f}h old)")
+                    logger.info(
+                        f"Satellites: Loaded {len(data)} records from disk cache ({age_hours:.1f}h old)"
+                    )
                     # Restore last_modified from metadata
                     _load_cache_meta()
                     return data
@@ -58,6 +65,7 @@ def _load_sat_cache():
         logger.warning(f"Satellites: Failed to load disk cache: {e}")
     return None
 
+
 def _save_sat_cache(data):
     """Save satellite GP data to local disk cache."""
     try:
@@ -69,6 +77,7 @@ def _save_sat_cache(data):
     except (IOError, OSError) as e:
         logger.warning(f"Satellites: Failed to save disk cache: {e}")
 
+
 def _load_cache_meta():
     """Load cache metadata (Last-Modified timestamp) from disk."""
     try:
@@ -79,6 +88,7 @@ def _load_cache_meta():
     except (IOError, OSError, json.JSONDecodeError, ValueError, KeyError):
         pass
 
+
 def _save_cache_meta():
     """Save cache metadata to disk."""
     try:
@@ -90,54 +100,357 @@ def _save_cache_meta():
 
 # Satellite intelligence classification database
 _SAT_INTEL_DB = [
-    ("USA 224", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
-    ("USA 245", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
-    ("USA 290", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
-    ("USA 314", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
-    ("USA 338", {"country": "USA", "mission": "military_recon", "sat_type": "Keyhole Successor", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
-    ("TOPAZ", {"country": "Russia", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)"}),
-    ("PERSONA", {"country": "Russia", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)"}),
-    ("KONDOR", {"country": "Russia", "mission": "military_sar", "sat_type": "SAR Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Kondor_(satellite)"}),
-    ("BARS-M", {"country": "Russia", "mission": "military_recon", "sat_type": "Mapping Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Bars-M"}),
-    ("YAOGAN", {"country": "China", "mission": "military_recon", "sat_type": "Remote Sensing / ELINT", "wiki": "https://en.wikipedia.org/wiki/Yaogan"}),
-    ("GAOFEN", {"country": "China", "mission": "military_recon", "sat_type": "High-Res Imaging", "wiki": "https://en.wikipedia.org/wiki/Gaofen"}),
-    ("JILIN", {"country": "China", "mission": "commercial_imaging", "sat_type": "Video / Imaging", "wiki": "https://en.wikipedia.org/wiki/Jilin-1"}),
-    ("OFEK", {"country": "Israel", "mission": "military_recon", "sat_type": "Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Ofeq"}),
-    ("CSO", {"country": "France", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/CSO_(satellite)"}),
-    ("IGS", {"country": "Japan", "mission": "military_recon", "sat_type": "Intelligence Gathering", "wiki": "https://en.wikipedia.org/wiki/Information_Gathering_Satellite"}),
-    ("CAPELLA", {"country": "USA", "mission": "sar", "sat_type": "SAR Imaging", "wiki": "https://en.wikipedia.org/wiki/Capella_Space"}),
-    ("ICEYE", {"country": "Finland", "mission": "sar", "sat_type": "SAR Microsatellite", "wiki": "https://en.wikipedia.org/wiki/ICEYE"}),
-    ("COSMO", {"country": "Italy", "mission": "sar", "sat_type": "SAR Constellation", "wiki": "https://en.wikipedia.org/wiki/COSMO-SkyMed"}),
-    ("TANDEM", {"country": "Germany", "mission": "sar", "sat_type": "SAR Interferometry", "wiki": "https://en.wikipedia.org/wiki/TanDEM-X"}),
-    ("PAZ", {"country": "Spain", "mission": "sar", "sat_type": "SAR Imaging", "wiki": "https://en.wikipedia.org/wiki/PAZ_(satellite)"}),
-    ("WORLDVIEW", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Maxar High-Res", "wiki": "https://en.wikipedia.org/wiki/WorldView-3"}),
-    ("GEOEYE", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Maxar Imaging", "wiki": "https://en.wikipedia.org/wiki/GeoEye-1"}),
-    ("PLEIADES", {"country": "France", "mission": "commercial_imaging", "sat_type": "Airbus Imaging", "wiki": "https://en.wikipedia.org/wiki/Pl%C3%A9iades_(satellite)"}),
-    ("SPOT", {"country": "France", "mission": "commercial_imaging", "sat_type": "Airbus Medium-Res", "wiki": "https://en.wikipedia.org/wiki/SPOT_(satellite)"}),
-    ("PLANET", {"country": "USA", "mission": "commercial_imaging", "sat_type": "PlanetScope", "wiki": "https://en.wikipedia.org/wiki/Planet_Labs"}),
-    ("SKYSAT", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Planet Video", "wiki": "https://en.wikipedia.org/wiki/SkySat"}),
-    ("BLACKSKY", {"country": "USA", "mission": "commercial_imaging", "sat_type": "BlackSky Imaging", "wiki": "https://en.wikipedia.org/wiki/BlackSky"}),
-    ("NROL", {"country": "USA", "mission": "sigint", "sat_type": "Classified NRO", "wiki": "https://en.wikipedia.org/wiki/National_Reconnaissance_Office"}),
-    ("MENTOR", {"country": "USA", "mission": "sigint", "sat_type": "SIGINT / ELINT", "wiki": "https://en.wikipedia.org/wiki/Mentor_(satellite)"}),
-    ("LUCH", {"country": "Russia", "mission": "sigint", "sat_type": "Relay / SIGINT", "wiki": "https://en.wikipedia.org/wiki/Luch_(satellite)"}),
-    ("SHIJIAN", {"country": "China", "mission": "sigint", "sat_type": "ELINT / Tech Demo", "wiki": "https://en.wikipedia.org/wiki/Shijian"}),
-    ("NAVSTAR", {"country": "USA", "mission": "navigation", "sat_type": "GPS", "wiki": "https://en.wikipedia.org/wiki/GPS_satellite_blocks"}),
-    ("GLONASS", {"country": "Russia", "mission": "navigation", "sat_type": "GLONASS", "wiki": "https://en.wikipedia.org/wiki/GLONASS"}),
-    ("BEIDOU", {"country": "China", "mission": "navigation", "sat_type": "BeiDou", "wiki": "https://en.wikipedia.org/wiki/BeiDou"}),
-    ("GALILEO", {"country": "EU", "mission": "navigation", "sat_type": "Galileo", "wiki": "https://en.wikipedia.org/wiki/Galileo_(satellite_navigation)"}),
-    ("SBIRS", {"country": "USA", "mission": "early_warning", "sat_type": "Missile Warning", "wiki": "https://en.wikipedia.org/wiki/Space-Based_Infrared_System"}),
-    ("TUNDRA", {"country": "Russia", "mission": "early_warning", "sat_type": "Missile Warning", "wiki": "https://en.wikipedia.org/wiki/Tundra_(satellite)"}),
-    ("ISS", {"country": "Intl", "mission": "space_station", "sat_type": "Space Station", "wiki": "https://en.wikipedia.org/wiki/International_Space_Station"}),
-    ("TIANGONG", {"country": "China", "mission": "space_station", "sat_type": "Space Station", "wiki": "https://en.wikipedia.org/wiki/Tiangong_space_station"}),
-    ("CSS", {"country": "China", "mission": "space_station", "sat_type": "Chinese Space Station", "wiki": "https://en.wikipedia.org/wiki/Tiangong_space_station"}),
-    # Russian military — COSMOS covers the bulk of active Russian military/SIGINT satellites
-    ("COSMOS", {"country": "Russia", "mission": "military_recon", "sat_type": "Russian Military / COSMOS", "wiki": "https://en.wikipedia.org/wiki/Kosmos_(satellite)"}),
-    # US military communications
-    ("WGS", {"country": "USA", "mission": "sigint", "sat_type": "Wideband Global SATCOM", "wiki": "https://en.wikipedia.org/wiki/Wideband_Global_SATCOM"}),
-    ("AEHF", {"country": "USA", "mission": "sigint", "sat_type": "Advanced EHF MILSATCOM", "wiki": "https://en.wikipedia.org/wiki/Advanced_Extremely_High_Frequency"}),
-    ("MUOS", {"country": "USA", "mission": "sigint", "sat_type": "Mobile User Objective System", "wiki": "https://en.wikipedia.org/wiki/Mobile_User_Objective_System"}),
-    # EU Earth observation
-    ("SENTINEL", {"country": "EU", "mission": "commercial_imaging", "sat_type": "ESA Copernicus", "wiki": "https://en.wikipedia.org/wiki/Sentinel_(satellite)"}),
+    (
+        "USA 224",
+        {
+            "country": "USA",
+            "mission": "military_recon",
+            "sat_type": "KH-11 Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
+        },
+    ),
+    (
+        "USA 245",
+        {
+            "country": "USA",
+            "mission": "military_recon",
+            "sat_type": "KH-11 Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
+        },
+    ),
+    (
+        "USA 290",
+        {
+            "country": "USA",
+            "mission": "military_recon",
+            "sat_type": "KH-11 Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
+        },
+    ),
+    (
+        "USA 314",
+        {
+            "country": "USA",
+            "mission": "military_recon",
+            "sat_type": "KH-11 Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
+        },
+    ),
+    (
+        "USA 338",
+        {
+            "country": "USA",
+            "mission": "military_recon",
+            "sat_type": "Keyhole Successor",
+            "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
+        },
+    ),
+    (
+        "TOPAZ",
+        {
+            "country": "Russia",
+            "mission": "military_recon",
+            "sat_type": "Optical Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)",
+        },
+    ),
+    (
+        "PERSONA",
+        {
+            "country": "Russia",
+            "mission": "military_recon",
+            "sat_type": "Optical Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)",
+        },
+    ),
+    (
+        "KONDOR",
+        {
+            "country": "Russia",
+            "mission": "military_sar",
+            "sat_type": "SAR Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/Kondor_(satellite)",
+        },
+    ),
+    (
+        "BARS-M",
+        {
+            "country": "Russia",
+            "mission": "military_recon",
+            "sat_type": "Mapping Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/Bars-M",
+        },
+    ),
+    (
+        "YAOGAN",
+        {
+            "country": "China",
+            "mission": "military_recon",
+            "sat_type": "Remote Sensing / ELINT",
+            "wiki": "https://en.wikipedia.org/wiki/Yaogan",
+        },
+    ),
+    (
+        "GAOFEN",
+        {
+            "country": "China",
+            "mission": "military_recon",
+            "sat_type": "High-Res Imaging",
+            "wiki": "https://en.wikipedia.org/wiki/Gaofen",
+        },
+    ),
+    (
+        "JILIN",
+        {
+            "country": "China",
+            "mission": "commercial_imaging",
+            "sat_type": "Video / Imaging",
+            "wiki": "https://en.wikipedia.org/wiki/Jilin-1",
+        },
+    ),
+    (
+        "OFEK",
+        {
+            "country": "Israel",
+            "mission": "military_recon",
+            "sat_type": "Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/Ofeq",
+        },
+    ),
+    (
+        "CSO",
+        {
+            "country": "France",
+            "mission": "military_recon",
+            "sat_type": "Optical Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/CSO_(satellite)",
+        },
+    ),
+    (
+        "IGS",
+        {
+            "country": "Japan",
+            "mission": "military_recon",
+            "sat_type": "Intelligence Gathering",
+            "wiki": "https://en.wikipedia.org/wiki/Information_Gathering_Satellite",
+        },
+    ),
+    (
+        "CAPELLA",
+        {
+            "country": "USA",
+            "mission": "sar",
+            "sat_type": "SAR Imaging",
+            "wiki": "https://en.wikipedia.org/wiki/Capella_Space",
+        },
+    ),
+    (
+        "ICEYE",
+        {
+            "country": "Finland",
+            "mission": "sar",
+            "sat_type": "SAR Microsatellite",
+            "wiki": "https://en.wikipedia.org/wiki/ICEYE",
+        },
+    ),
+    (
+        "COSMO-SKYMED",
+        {
+            "country": "Italy",
+            "mission": "sar",
+            "sat_type": "SAR Constellation",
+            "wiki": "https://en.wikipedia.org/wiki/COSMO-SkyMed",
+        },
+    ),
+    (
+        "TANDEM",
+        {
+            "country": "Germany",
+            "mission": "sar",
+            "sat_type": "SAR Interferometry",
+            "wiki": "https://en.wikipedia.org/wiki/TanDEM-X",
+        },
+    ),
+    (
+        "PAZ",
+        {
+            "country": "Spain",
+            "mission": "sar",
+            "sat_type": "SAR Imaging",
+            "wiki": "https://en.wikipedia.org/wiki/PAZ_(satellite)",
+        },
+    ),
+    (
+        "WORLDVIEW",
+        {
+            "country": "USA",
+            "mission": "commercial_imaging",
+            "sat_type": "Maxar High-Res",
+            "wiki": "https://en.wikipedia.org/wiki/WorldView-3",
+        },
+    ),
+    (
+        "GEOEYE",
+        {
+            "country": "USA",
+            "mission": "commercial_imaging",
+            "sat_type": "Maxar Imaging",
+            "wiki": "https://en.wikipedia.org/wiki/GeoEye-1",
+        },
+    ),
+    (
+        "PLEIADES",
+        {
+            "country": "France",
+            "mission": "commercial_imaging",
+            "sat_type": "Airbus Imaging",
+            "wiki": "https://en.wikipedia.org/wiki/Pl%C3%A9iades_(satellite)",
+        },
+    ),
+    (
+        "SPOT",
+        {
+            "country": "France",
+            "mission": "commercial_imaging",
+            "sat_type": "Airbus Medium-Res",
+            "wiki": "https://en.wikipedia.org/wiki/SPOT_(satellite)",
+        },
+    ),
+    (
+        "PLANET",
+        {
+            "country": "USA",
+            "mission": "commercial_imaging",
+            "sat_type": "PlanetScope",
+            "wiki": "https://en.wikipedia.org/wiki/Planet_Labs",
+        },
+    ),
+    (
+        "SKYSAT",
+        {
+            "country": "USA",
+            "mission": "commercial_imaging",
+            "sat_type": "Planet Video",
+            "wiki": "https://en.wikipedia.org/wiki/SkySat",
+        },
+    ),
+    (
+        "BLACKSKY",
+        {
+            "country": "USA",
+            "mission": "commercial_imaging",
+            "sat_type": "BlackSky Imaging",
+            "wiki": "https://en.wikipedia.org/wiki/BlackSky",
+        },
+    ),
+    (
+        "NROL",
+        {
+            "country": "USA",
+            "mission": "sigint",
+            "sat_type": "Classified NRO",
+            "wiki": "https://en.wikipedia.org/wiki/National_Reconnaissance_Office",
+        },
+    ),
+    (
+        "MENTOR",
+        {
+            "country": "USA",
+            "mission": "sigint",
+            "sat_type": "SIGINT / ELINT",
+            "wiki": "https://en.wikipedia.org/wiki/Mentor_(satellite)",
+        },
+    ),
+    (
+        "LUCH",
+        {
+            "country": "Russia",
+            "mission": "sigint",
+            "sat_type": "Relay / SIGINT",
+            "wiki": "https://en.wikipedia.org/wiki/Luch_(satellite)",
+        },
+    ),
+    (
+        "SHIJIAN",
+        {
+            "country": "China",
+            "mission": "sigint",
+            "sat_type": "ELINT / Tech Demo",
+            "wiki": "https://en.wikipedia.org/wiki/Shijian",
+        },
+    ),
+    (
+        "NAVSTAR",
+        {
+            "country": "USA",
+            "mission": "navigation",
+            "sat_type": "GPS",
+            "wiki": "https://en.wikipedia.org/wiki/GPS_satellite_blocks",
+        },
+    ),
+    (
+        "GLONASS",
+        {
+            "country": "Russia",
+            "mission": "navigation",
+            "sat_type": "GLONASS",
+            "wiki": "https://en.wikipedia.org/wiki/GLONASS",
+        },
+    ),
+    (
+        "BEIDOU",
+        {
+            "country": "China",
+            "mission": "navigation",
+            "sat_type": "BeiDou",
+            "wiki": "https://en.wikipedia.org/wiki/BeiDou",
+        },
+    ),
+    (
+        "GALILEO",
+        {
+            "country": "EU",
+            "mission": "navigation",
+            "sat_type": "Galileo",
+            "wiki": "https://en.wikipedia.org/wiki/Galileo_(satellite_navigation)",
+        },
+    ),
+    (
+        "SBIRS",
+        {
+            "country": "USA",
+            "mission": "early_warning",
+            "sat_type": "Missile Warning",
+            "wiki": "https://en.wikipedia.org/wiki/Space-Based_Infrared_System",
+        },
+    ),
+    (
+        "TUNDRA",
+        {
+            "country": "Russia",
+            "mission": "early_warning",
+            "sat_type": "Missile Warning",
+            "wiki": "https://en.wikipedia.org/wiki/Tundra_(satellite)",
+        },
+    ),
+    (
+        "ISS",
+        {
+            "country": "Intl",
+            "mission": "space_station",
+            "sat_type": "Space Station",
+            "wiki": "https://en.wikipedia.org/wiki/International_Space_Station",
+        },
+    ),
+    (
+        "TIANGONG",
+        {
+            "country": "China",
+            "mission": "space_station",
+            "sat_type": "Space Station",
+            "wiki": "https://en.wikipedia.org/wiki/Tiangong_space_station",
+        },
+    ),
 ]
 
 
@@ -154,7 +467,7 @@ def _parse_tle_to_gp(name, norad_id, line1, line2):
         if bstar_str:
             mantissa = float(bstar_str[:-2]) / 1e5
             exponent = int(bstar_str[-2:])
-            bstar = mantissa * (10 ** exponent)
+            bstar = mantissa * (10**exponent)
         else:
             bstar = 0.0
         epoch_yr = int(line1[18:20])
@@ -206,17 +519,50 @@ def _fetch_satellites_from_tle_api():
                         seen_ids.add(sat_id)
                         all_results.append(gp)
             time.sleep(1)  # Polite delay between requests
-        except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, json.JSONDecodeError, OSError) as e:
+        except (
+            requests.RequestException,
+            ConnectionError,
+            TimeoutError,
+            ValueError,
+            KeyError,
+            json.JSONDecodeError,
+            OSError,
+        ) as e:
             logger.debug(f"TLE fallback search '{term}' failed: {e}")
 
     return all_results
 
 
 def fetch_satellites():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("satellites"):
+        return
     sats = []
     try:
         now_ts = time.time()
-        if _sat_gp_cache["data"] is None or (now_ts - _sat_gp_cache["last_fetch"]) > _CELESTRAK_FETCH_INTERVAL:
+
+        # On first call, try disk cache before hitting CelesTrak
+        if _sat_gp_cache["data"] is None:
+            disk_data = _load_sat_cache()
+            if disk_data:
+                import os
+
+                cache_mtime = (
+                    os.path.getmtime(str(_SAT_CACHE_PATH)) if _SAT_CACHE_PATH.exists() else 0
+                )
+                _sat_gp_cache["data"] = disk_data
+                _sat_gp_cache["last_fetch"] = cache_mtime  # real fetch time so 24h check works
+                _sat_gp_cache["source"] = "disk_cache"
+                logger.info(
+                    f"Satellites: Bootstrapped from disk cache ({len(disk_data)} records, "
+                    f"{(now_ts - cache_mtime) / 3600:.1f}h old)"
+                )
+
+        if (
+            _sat_gp_cache["data"] is None
+            or (now_ts - _sat_gp_cache["last_fetch"]) > _CELESTRAK_FETCH_INTERVAL
+        ):
             gp_urls = [
                 "https://celestrak.org/NORAD/elements/gp.php?GROUP=active&FORMAT=json",
                 "https://celestrak.com/NORAD/elements/gp.php?GROUP=active&FORMAT=json",
@@ -232,7 +578,9 @@ def fetch_satellites():
                     if response.status_code == 304:
                         # Data unchanged — reset timer without re-downloading
                         _sat_gp_cache["last_fetch"] = now_ts
-                        logger.info(f"Satellites: CelesTrak returned 304 Not Modified (data unchanged)")
+                        logger.info(
+                            f"Satellites: CelesTrak returned 304 Not Modified (data unchanged)"
+                        )
                         break
                     if response.status_code == 200:
                         gp_data = response.json()
@@ -241,14 +589,24 @@ def fetch_satellites():
                             _sat_gp_cache["last_fetch"] = now_ts
                             _sat_gp_cache["source"] = "celestrak"
                             # Store Last-Modified header for future conditional requests
-                            if hasattr(response, 'headers'):
+                            if hasattr(response, "headers"):
                                 lm = response.headers.get("Last-Modified")
                                 if lm:
                                     _sat_gp_cache["last_modified"] = lm
                             _save_sat_cache(gp_data)
-                            logger.info(f"Satellites: Downloaded {len(gp_data)} GP records from CelesTrak")
+                            logger.info(
+                                f"Satellites: Downloaded {len(gp_data)} GP records from CelesTrak"
+                            )
                             break
-                except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, json.JSONDecodeError, OSError) as e:
+                except (
+                    requests.RequestException,
+                    ConnectionError,
+                    TimeoutError,
+                    ValueError,
+                    KeyError,
+                    json.JSONDecodeError,
+                    OSError,
+                ) as e:
                     logger.warning(f"Satellites: Failed to fetch from {url}: {e}")
                     continue
 
@@ -261,8 +619,17 @@ def fetch_satellites():
                         _sat_gp_cache["last_fetch"] = now_ts
                         _sat_gp_cache["source"] = "tle_api"
                         _save_sat_cache(fallback_data)
-                        logger.info(f"Satellites: Got {len(fallback_data)} records from TLE fallback API")
-                except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
+                        logger.info(
+                            f"Satellites: Got {len(fallback_data)} records from TLE fallback API"
+                        )
+                except (
+                    requests.RequestException,
+                    ConnectionError,
+                    TimeoutError,
+                    ValueError,
+                    KeyError,
+                    OSError,
+                ) as e:
                     logger.error(f"Satellites: TLE fallback also failed: {e}")
 
             if _sat_gp_cache["data"] is None:
@@ -279,9 +646,14 @@ def fetch_satellites():
                 latest_data["satellites"] = sats
             return
 
-        if _sat_classified_cache["gp_fetch_ts"] == _sat_gp_cache["last_fetch"] and _sat_classified_cache["data"]:
+        if (
+            _sat_classified_cache["gp_fetch_ts"] == _sat_gp_cache["last_fetch"]
+            and _sat_classified_cache["data"]
+        ):
             classified = _sat_classified_cache["data"]
-            logger.info(f"Satellites: Using cached classification ({len(classified)} sats, TLEs unchanged)")
+            logger.info(
+                f"Satellites: Using cached classification ({len(classified)} sats, TLEs unchanged)"
+            )
         else:
             classified = []
             for sat in data:
@@ -309,41 +681,57 @@ def fetch_satellites():
                 classified.append(entry)
             _sat_classified_cache["data"] = classified
             _sat_classified_cache["gp_fetch_ts"] = _sat_gp_cache["last_fetch"]
-            logger.info(f"Satellites: {len(classified)} intel-classified out of {len(data)} total in catalog")
+            logger.info(
+                f"Satellites: {len(classified)} intel-classified out of {len(data)} total in catalog"
+            )
 
         all_sats = classified
 
         now = datetime.utcnow()
-        jd, fr = jday(now.year, now.month, now.day, now.hour, now.minute, now.second + now.microsecond / 1e6)
+        jd, fr = jday(
+            now.year, now.month, now.day, now.hour, now.minute, now.second + now.microsecond / 1e6
+        )
 
         for s in all_sats:
             try:
-                mean_motion = s.get('MEAN_MOTION')
-                ecc = s.get('ECCENTRICITY')
-                incl = s.get('INCLINATION')
-                raan = s.get('RA_OF_ASC_NODE')
-                argp = s.get('ARG_OF_PERICENTER')
-                ma = s.get('MEAN_ANOMALY')
-                bstar = s.get('BSTAR', 0)
-                epoch_str = s.get('EPOCH')
-                norad_id = s.get('id', 0)
+                mean_motion = s.get("MEAN_MOTION")
+                ecc = s.get("ECCENTRICITY")
+                incl = s.get("INCLINATION")
+                raan = s.get("RA_OF_ASC_NODE")
+                argp = s.get("ARG_OF_PERICENTER")
+                ma = s.get("MEAN_ANOMALY")
+                bstar = s.get("BSTAR", 0)
+                epoch_str = s.get("EPOCH")
+                norad_id = s.get("id", 0)
 
                 if mean_motion is None or ecc is None or incl is None:
                     continue
 
-                epoch_dt = datetime.strptime(epoch_str[:19], '%Y-%m-%dT%H:%M:%S')
-                epoch_jd, epoch_fr = jday(epoch_dt.year, epoch_dt.month, epoch_dt.day,
-                                          epoch_dt.hour, epoch_dt.minute, epoch_dt.second)
+                epoch_dt = datetime.strptime(epoch_str[:19], "%Y-%m-%dT%H:%M:%S")
+                epoch_jd, epoch_fr = jday(
+                    epoch_dt.year,
+                    epoch_dt.month,
+                    epoch_dt.day,
+                    epoch_dt.hour,
+                    epoch_dt.minute,
+                    epoch_dt.second,
+                )
 
                 sat_obj = Satrec()
                 sat_obj.sgp4init(
-                    WGS72, 'i', norad_id,
+                    WGS72,
+                    "i",
+                    norad_id,
                     (epoch_jd + epoch_fr) - 2433281.5,
-                    bstar, 0.0, 0.0, ecc,
-                    math.radians(argp), math.radians(incl),
+                    bstar,
+                    0.0,
+                    0.0,
+                    ecc,
+                    math.radians(argp),
+                    math.radians(incl),
                     math.radians(ma),
                     mean_motion * 2 * math.pi / 1440.0,
-                    math.radians(raan)
+                    math.radians(raan),
                 )
 
                 e, r, v = sat_obj.sgp4(jd, fr)
@@ -353,13 +741,13 @@ def fetch_satellites():
                 x, y, z = r
                 gmst = _gmst(jd + fr)
                 lng_rad = math.atan2(y, x) - gmst
-                lat_rad = math.atan2(z, math.sqrt(x*x + y*y))
-                alt_km = math.sqrt(x*x + y*y + z*z) - 6371.0
+                lat_rad = math.atan2(z, math.sqrt(x * x + y * y))
+                alt_km = math.sqrt(x * x + y * y + z * z) - 6371.0
 
-                s['lat'] = round(math.degrees(lat_rad), 4)
+                s["lat"] = round(math.degrees(lat_rad), 4)
                 lng_deg = math.degrees(lng_rad) % 360
-                s['lng'] = round(lng_deg - 360 if lng_deg > 180 else lng_deg, 4)
-                s['alt_km'] = round(alt_km, 1)
+                s["lng"] = round(lng_deg - 360 if lng_deg > 180 else lng_deg, 4)
+                s["alt_km"] = round(alt_km, 1)
 
                 vx, vy, vz = v
                 omega_e = 7.2921159e-5
@@ -373,23 +761,40 @@ def fetch_satellites():
                 v_east = -sin_lng * vx_g + cos_lng * vy_g
                 v_north = -sin_lat * cos_lng * vx_g - sin_lat * sin_lng * vy_g + cos_lat * vz_g
                 ground_speed_kms = math.sqrt(v_east**2 + v_north**2)
-                s['speed_knots'] = round(ground_speed_kms * 1943.84, 1)
+                s["speed_knots"] = round(ground_speed_kms * 1943.84, 1)
                 heading_rad = math.atan2(v_east, v_north)
-                s['heading'] = round(math.degrees(heading_rad) % 360, 1)
-                sat_name = s.get('name', '')
-                usa_match = re.search(r'USA[\s\-]*(\d+)', sat_name)
+                s["heading"] = round(math.degrees(heading_rad) % 360, 1)
+                sat_name = s.get("name", "")
+                usa_match = re.search(r"USA[\s\-]*(\d+)", sat_name)
                 if usa_match:
-                    s['wiki'] = f"https://en.wikipedia.org/wiki/USA-{usa_match.group(1)}"
-                for k in ('MEAN_MOTION', 'ECCENTRICITY', 'INCLINATION',
-                          'RA_OF_ASC_NODE', 'ARG_OF_PERICENTER', 'MEAN_ANOMALY',
-                          'BSTAR', 'EPOCH', 'tle1', 'tle2'):
+                    s["wiki"] = f"https://en.wikipedia.org/wiki/USA-{usa_match.group(1)}"
+                for k in (
+                    "MEAN_MOTION",
+                    "ECCENTRICITY",
+                    "INCLINATION",
+                    "RA_OF_ASC_NODE",
+                    "ARG_OF_PERICENTER",
+                    "MEAN_ANOMALY",
+                    "BSTAR",
+                    "EPOCH",
+                    "tle1",
+                    "tle2",
+                ):
                     s.pop(k, None)
                 sats.append(s)
             except (ValueError, TypeError, KeyError, AttributeError, ZeroDivisionError):
                 continue
 
         logger.info(f"Satellites: {len(classified)} classified, {len(sats)} positioned")
-    except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, json.JSONDecodeError, OSError) as e:
+    except (
+        requests.RequestException,
+        ConnectionError,
+        TimeoutError,
+        ValueError,
+        KeyError,
+        json.JSONDecodeError,
+        OSError,
+    ) as e:
         logger.error(f"Error fetching satellites: {e}")
     if sats:
         with _data_lock:

backend/services/fetchers/sigint.py

@@ -0,0 +1,102 @@
+"""SIGINT fetcher — pulls latest signals from the SIGINT Grid into latest_data.
+
+Merges live MQTT signals with cached Meshtastic map API nodes.
+Live MQTT signals always take priority (fresher) — API nodes fill in the gaps
+for the thousands of nodes our MQTT listener hasn't heard yet.
+"""
+
+import logging
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+
+logger = logging.getLogger("services.data_fetcher")
+
+
+def _merge_sigint_snapshot(
+    live_signals: list[dict],
+    api_nodes: list[dict],
+) -> list[dict]:
+    """Merge live bridge signals with cached Meshtastic map nodes.
+
+    Live Meshtastic observations always win over map/API nodes for the same callsign
+    because they include fresher region/channel metadata.
+    """
+
+    merged = list(live_signals)
+    live_callsigns = {s["callsign"] for s in merged if s.get("source") == "meshtastic"}
+    for node in api_nodes:
+        if node.get("callsign") in live_callsigns:
+            continue
+        merged.append(node)
+    merged.sort(key=lambda item: str(item.get("timestamp", "") or ""), reverse=True)
+    return merged
+
+
+def _sigint_totals(signals: list[dict]) -> dict[str, int]:
+    totals = {
+        "total": len(signals),
+        "meshtastic": 0,
+        "meshtastic_live": 0,
+        "meshtastic_map": 0,
+        "aprs": 0,
+        "js8call": 0,
+    }
+    for sig in signals:
+        source = str(sig.get("source", "") or "").lower()
+        if source == "meshtastic":
+            totals["meshtastic"] += 1
+            if bool(sig.get("from_api")):
+                totals["meshtastic_map"] += 1
+            else:
+                totals["meshtastic_live"] += 1
+        elif source == "aprs":
+            totals["aprs"] += 1
+        elif source == "js8call":
+            totals["js8call"] += 1
+    return totals
+
+
+def build_sigint_snapshot() -> tuple[list[dict], dict[str, object], dict[str, int]]:
+    """Build the current merged SIGINT snapshot without hitting the network."""
+
+    from services.sigint_bridge import sigint_grid
+
+    live_signals = sigint_grid.get_all_signals()
+    with _data_lock:
+        api_nodes = list(latest_data.get("meshtastic_map_nodes", []))
+    merged = _merge_sigint_snapshot(live_signals, api_nodes)
+    channel_stats = sigint_grid.get_mesh_channel_stats(api_nodes or None)
+    totals = _sigint_totals(merged)
+    return merged, channel_stats, totals
+
+
+def refresh_sigint_snapshot() -> tuple[list[dict], dict[str, object], dict[str, int]]:
+    """Refresh latest_data SIGINT state from current bridge + cache state."""
+
+    signals, channel_stats, totals = build_sigint_snapshot()
+    with _data_lock:
+        latest_data["sigint"] = signals
+        latest_data["mesh_channel_stats"] = channel_stats
+        latest_data["sigint_totals"] = totals
+    _mark_fresh("sigint")
+    return signals, channel_stats, totals
+
+
+def fetch_sigint():
+    """Fetch all signals from the SIGINT Grid, merge with Meshtastic map nodes."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("sigint_meshtastic", "sigint_aprs"):
+        return
+    from services.sigint_bridge import sigint_grid
+
+    # Start bridges on first call (idempotent)
+    sigint_grid.start()
+
+    signals, channel_stats, totals = refresh_sigint_snapshot()
+
+    status = sigint_grid.status
+    logger.info(
+        f"SIGINT: {len(signals)} signals "
+        f"(APRS:{status['aprs']} MESH:{status['meshtastic']} "
+        f"JS8:{status['js8call']} MAP:{totals['meshtastic_map']})"
+    )

backend/services/fetchers/trains.py

@@ -0,0 +1,457 @@
+"""Train tracking fetchers with normalized metadata and non-redundant merging."""
+
+from __future__ import annotations
+
+import logging
+import math
+from collections.abc import Callable
+from datetime import datetime, timezone
+
+from services.fetchers._store import _data_lock, _mark_fresh, latest_data
+from services.network_utils import fetch_with_curl
+
+logger = logging.getLogger(__name__)
+
+_EARTH_RADIUS_KM = 6371.0
+_MERGE_DISTANCE_KM = 5.0
+_MAX_INFERRED_SPEED_KMH = 350.0
+_TRACK_CACHE_TTL_S = 6 * 60 * 60
+
+_SOURCE_METADATA: dict[str, dict[str, object]] = {
+    "amtrak": {
+        "source_label": "Amtraker",
+        "operator": "Amtrak",
+        "country": "US",
+        "telemetry_quality": "aggregated",
+        "priority": 70,
+    },
+    "digitraffic": {
+        "source_label": "Digitraffic Finland",
+        "operator": "Finnish Rail",
+        "country": "FI",
+        "telemetry_quality": "official",
+        "priority": 100,
+    },
+    # Future slots so better official feeds can be merged without changing the
+    # rest of the train pipeline or duplicating map entities.
+    "networkrail": {
+        "source_label": "Network Rail Open Data",
+        "operator": "Network Rail",
+        "country": "GB",
+        "telemetry_quality": "official",
+        "priority": 98,
+    },
+    "dbcargo": {
+        "source_label": "DB Cargo link2rail",
+        "operator": "DB Cargo",
+        "country": "DE",
+        "telemetry_quality": "commercial",
+        "priority": 96,
+    },
+    "railinc": {
+        "source_label": "Railinc RailSight",
+        "operator": "Railinc",
+        "country": "US",
+        "telemetry_quality": "commercial",
+        "priority": 97,
+    },
+    "sncf": {
+        "source_label": "SNCF Open Data",
+        "operator": "SNCF",
+        "country": "FR",
+        "telemetry_quality": "official",
+        "priority": 94,
+    },
+}
+
+_TRAIN_TRACK_CACHE: dict[str, dict[str, float]] = {}
+
+
+def _safe_float(value) -> float | None:
+    try:
+        if value is None or value == "":
+            return None
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _parse_observed_at(value) -> float | None:
+    if value is None or value == "":
+        return None
+    if isinstance(value, (int, float)):
+        raw = float(value)
+        return raw / 1000.0 if raw > 1_000_000_000_000 else raw
+    if not isinstance(value, str):
+        return None
+    text = value.strip()
+    if not text:
+        return None
+    if text.endswith("Z"):
+        text = f"{text[:-1]}+00:00"
+    try:
+        return datetime.fromisoformat(text).timestamp()
+    except ValueError:
+        return None
+
+
+def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
+    lat1_rad, lon1_rad = math.radians(lat1), math.radians(lon1)
+    lat2_rad, lon2_rad = math.radians(lat2), math.radians(lon2)
+    dlat = lat2_rad - lat1_rad
+    dlon = lon2_rad - lon1_rad
+    a = (
+        math.sin(dlat / 2.0) ** 2
+        + math.cos(lat1_rad) * math.cos(lat2_rad) * math.sin(dlon / 2.0) ** 2
+    )
+    return 2.0 * _EARTH_RADIUS_KM * math.asin(math.sqrt(a))
+
+
+def _bearing_degrees(lat1: float, lon1: float, lat2: float, lon2: float) -> float | None:
+    if lat1 == lat2 and lon1 == lon2:
+        return None
+    lat1_rad, lat2_rad = math.radians(lat1), math.radians(lat2)
+    dlon_rad = math.radians(lon2 - lon1)
+    y = math.sin(dlon_rad) * math.cos(lat2_rad)
+    x = (
+        math.cos(lat1_rad) * math.sin(lat2_rad)
+        - math.sin(lat1_rad) * math.cos(lat2_rad) * math.cos(dlon_rad)
+    )
+    return (math.degrees(math.atan2(y, x)) + 360.0) % 360.0
+
+
+def _source_meta(source: str) -> dict[str, object]:
+    return dict(_SOURCE_METADATA.get(source, {}))
+
+
+def _normalize_train(
+    *,
+    source: str,
+    raw_id: str,
+    number: str,
+    lat,
+    lng,
+    name: str = "",
+    status: str = "Active",
+    route: str = "",
+    speed_kmh=None,
+    heading=None,
+    operator: str | None = None,
+    country: str | None = None,
+    source_label: str | None = None,
+    telemetry_quality: str | None = None,
+    observed_at=None,
+) -> dict | None:
+    lat_f = _safe_float(lat)
+    lng_f = _safe_float(lng)
+    if lat_f is None or lng_f is None:
+        return None
+    if not (-90.0 <= lat_f <= 90.0 and -180.0 <= lng_f <= 180.0):
+        return None
+
+    number_text = str(number or "").strip()
+    meta = _source_meta(source)
+    observed_ts = _parse_observed_at(observed_at) or datetime.now(timezone.utc).timestamp()
+    speed_f = _safe_float(speed_kmh)
+    heading_f = _safe_float(heading)
+    normalized = {
+        "id": str(raw_id or f"{source}-{number_text or 'unknown'}"),
+        "name": str(name or f"Train {number_text or '?'}").strip(),
+        "number": number_text,
+        "source": source,
+        "source_label": str(source_label or meta.get("source_label") or source.upper()),
+        "operator": str(operator or meta.get("operator") or "").strip(),
+        "country": str(country or meta.get("country") or "").strip(),
+        "telemetry_quality": str(
+            telemetry_quality or meta.get("telemetry_quality") or "unknown"
+        ).strip(),
+        "lat": lat_f,
+        "lng": lng_f,
+        "speed_kmh": speed_f,
+        "heading": heading_f,
+        "status": str(status or "Active").strip(),
+        "route": str(route or "").strip(),
+        "_source_priority": int(meta.get("priority") or 0),
+        "_observed_ts": observed_ts,
+    }
+    _apply_motion_estimates(normalized)
+    return normalized
+
+
+def _prune_track_cache(now_ts: float) -> None:
+    stale_before = now_ts - _TRACK_CACHE_TTL_S
+    stale_ids = [train_id for train_id, entry in _TRAIN_TRACK_CACHE.items() if entry["ts"] < stale_before]
+    for train_id in stale_ids:
+        _TRAIN_TRACK_CACHE.pop(train_id, None)
+
+
+def _apply_motion_estimates(train: dict) -> None:
+    train_id = str(train.get("id") or "")
+    if not train_id:
+        return
+    now_ts = float(train.get("_observed_ts") or datetime.now(timezone.utc).timestamp())
+    _prune_track_cache(now_ts)
+    previous = _TRAIN_TRACK_CACHE.get(train_id)
+    if previous:
+        dt_s = now_ts - previous["ts"]
+        if 5.0 <= dt_s <= 15.0 * 60.0:
+            distance_km = _haversine_km(
+                float(previous["lat"]),
+                float(previous["lng"]),
+                float(train["lat"]),
+                float(train["lng"]),
+            )
+            if 0.02 <= distance_km <= (_MAX_INFERRED_SPEED_KMH * (dt_s / 3600.0)):
+                if train.get("speed_kmh") is None:
+                    inferred_speed = distance_km / (dt_s / 3600.0)
+                    train["speed_kmh"] = round(min(inferred_speed, _MAX_INFERRED_SPEED_KMH), 1)
+                if train.get("heading") is None:
+                    inferred_heading = _bearing_degrees(
+                        float(previous["lat"]),
+                        float(previous["lng"]),
+                        float(train["lat"]),
+                        float(train["lng"]),
+                    )
+                    if inferred_heading is not None:
+                        train["heading"] = round(inferred_heading, 1)
+
+    _TRAIN_TRACK_CACHE[train_id] = {
+        "lat": float(train["lat"]),
+        "lng": float(train["lng"]),
+        "ts": now_ts,
+    }
+
+
+def _train_merge_key(train: dict) -> str:
+    operator = str(train.get("operator") or "").strip().lower()
+    country = str(train.get("country") or "").strip().lower()
+    number = str(train.get("number") or "").strip().lower()
+    if operator and number:
+        return f"{country}|{operator}|{number}"
+    return f"{str(train.get('source') or '').lower()}|{str(train.get('id') or '').lower()}"
+
+
+def _train_completeness(train: dict) -> tuple[int, int, int]:
+    return (
+        1 if train.get("speed_kmh") is not None else 0,
+        1 if train.get("heading") is not None else 0,
+        1 if train.get("route") else 0,
+    )
+
+
+def _should_merge(existing: dict, candidate: dict) -> bool:
+    if _train_merge_key(existing) != _train_merge_key(candidate):
+        return False
+    return _haversine_km(
+        float(existing["lat"]),
+        float(existing["lng"]),
+        float(candidate["lat"]),
+        float(candidate["lng"]),
+    ) <= _MERGE_DISTANCE_KM
+
+
+def _merge_train_pair(existing: dict, candidate: dict) -> dict:
+    existing_priority = int(existing.get("_source_priority") or 0)
+    candidate_priority = int(candidate.get("_source_priority") or 0)
+    existing_score = (existing_priority, _train_completeness(existing))
+    candidate_score = (candidate_priority, _train_completeness(candidate))
+    primary = candidate if candidate_score > existing_score else existing
+    secondary = existing if primary is candidate else candidate
+    merged = dict(primary)
+
+    for field in (
+        "speed_kmh",
+        "heading",
+        "route",
+        "status",
+        "operator",
+        "country",
+        "source_label",
+        "telemetry_quality",
+    ):
+        if merged.get(field) in (None, "", "Active"):
+            replacement = secondary.get(field)
+            if replacement not in (None, ""):
+                merged[field] = replacement
+
+    if primary is not candidate and float(candidate.get("_observed_ts") or 0) > float(
+        primary.get("_observed_ts") or 0
+    ):
+        merged["lat"] = candidate["lat"]
+        merged["lng"] = candidate["lng"]
+        merged["_observed_ts"] = candidate["_observed_ts"]
+    return merged
+
+
+def _merge_nonredundant_trains(*sources: list[dict]) -> list[dict]:
+    merged: list[dict] = []
+    for source_trains in sources:
+        for train in source_trains:
+            exact_match = next(
+                (
+                    idx
+                    for idx, existing in enumerate(merged)
+                    if existing.get("source") == train.get("source")
+                    and existing.get("id") == train.get("id")
+                ),
+                None,
+            )
+            if exact_match is not None:
+                merged[exact_match] = _merge_train_pair(merged[exact_match], train)
+                continue
+
+            merged_idx = next(
+                (idx for idx, existing in enumerate(merged) if _should_merge(existing, train)),
+                None,
+            )
+            if merged_idx is not None:
+                merged[merged_idx] = _merge_train_pair(merged[merged_idx], train)
+                continue
+            merged.append(train)
+
+    merged.sort(
+        key=lambda train: (
+            str(train.get("country") or ""),
+            str(train.get("operator") or ""),
+            str(train.get("number") or ""),
+            str(train.get("id") or ""),
+        )
+    )
+    for train in merged:
+        train.pop("_source_priority", None)
+        train.pop("_observed_ts", None)
+    return merged
+
+
+def _fetch_amtraker() -> list[dict]:
+    """Fetch all active Amtrak trains from the Amtraker API."""
+    try:
+        resp = fetch_with_curl(
+            "https://api.amtraker.com/v3/trains",
+            timeout=20,
+            headers={
+                "User-Agent": (
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                    "AppleWebKit/537.36 (KHTML, like Gecko) "
+                    "Chrome/136.0.0.0 Safari/537.36"
+                ),
+                "Accept": "application/json,text/plain,*/*",
+                "Referer": "https://www.amtraker.com/",
+            },
+        )
+        if resp.status_code != 200:
+            logger.warning("Amtraker returned %s", resp.status_code)
+            return []
+        raw = resp.json()
+        trains: list[dict] = []
+        for train_num, variants in raw.items():
+            if not isinstance(variants, list):
+                continue
+            for item in variants:
+                normalized = _normalize_train(
+                    source="amtrak",
+                    raw_id=f"AMTK-{item.get('trainID', train_num)}",
+                    name=item.get("routeName", f"Train {train_num}"),
+                    number=str(item.get("trainNum", train_num) or train_num),
+                    lat=item.get("lat"),
+                    lng=item.get("lon"),
+                    speed_kmh=item.get("velocity") or item.get("speed"),
+                    heading=item.get("heading") or item.get("bearing"),
+                    status=item.get("trainTimely") or "On Time",
+                    route=item.get("routeName", ""),
+                    observed_at=item.get("updatedAt")
+                    or item.get("lastValTS")
+                    or item.get("eventDT"),
+                )
+                if normalized:
+                    trains.append(normalized)
+        return trains
+    except Exception as exc:
+        logger.warning("Amtraker fetch error: %s", exc)
+        return []
+
+
+def _fetch_digitraffic() -> list[dict]:
+    """Fetch live train positions from Finnish DigiTraffic API."""
+    try:
+        resp = fetch_with_curl(
+            "https://rata.digitraffic.fi/api/v1/train-locations/latest",
+            timeout=15,
+            headers={
+                "Accept-Encoding": "gzip",
+                "User-Agent": "ShadowBroker-OSINT/1.0",
+            },
+        )
+        if resp.status_code != 200:
+            logger.warning("DigiTraffic returned %s", resp.status_code)
+            return []
+        raw = resp.json()
+        trains: list[dict] = []
+        for item in raw:
+            location = item.get("location", {})
+            coords = location.get("coordinates")
+            if not coords or len(coords) < 2:
+                continue
+            lon, lat = coords[0], coords[1]
+            train_number = str(item.get("trainNumber", "") or "").strip()
+            route_bits = [
+                str(item.get("departureStationShortCode") or "").strip(),
+                str(item.get("stationShortCode") or "").strip(),
+            ]
+            route = " -> ".join([bit for bit in route_bits if bit])
+            train_type = str(item.get("trainType") or "").strip()
+            normalized = _normalize_train(
+                source="digitraffic",
+                raw_id=f"FIN-{train_number or len(trains)}",
+                name=f"{train_type} {train_number}".strip() or f"Train {train_number or '?'}",
+                number=train_number,
+                lat=lat,
+                lng=lon,
+                speed_kmh=item.get("speed"),
+                heading=item.get("heading"),
+                status="Active",
+                route=route,
+                observed_at=item.get("timestamp"),
+            )
+            if normalized:
+                trains.append(normalized)
+        return trains
+    except Exception as exc:
+        logger.warning("DigiTraffic fetch error: %s", exc)
+        return []
+
+
+_TRAIN_FETCHERS: tuple[tuple[str, Callable[[], list[dict]]], ...] = (
+    ("amtrak", _fetch_amtraker),
+    ("digitraffic", _fetch_digitraffic),
+)
+
+
+def fetch_trains():
+    """Fetch trains from all configured sources and merge without duplicates."""
+    with _data_lock:
+        existing_trains = list(latest_data.get("trains") or [])
+    source_batches: list[list[dict]] = []
+    source_counts: list[str] = []
+    for source_name, fetcher in _TRAIN_FETCHERS:
+        batch = fetcher()
+        source_batches.append(batch)
+        if batch:
+            source_counts.append(f"{source_name}:{len(batch)}")
+
+    trains = _merge_nonredundant_trains(*source_batches)
+    if not trains and existing_trains:
+        logger.warning(
+            "Train refresh returned 0 records — preserving %s cached trains until the next successful poll",
+            len(existing_trains),
+        )
+        trains = existing_trains
+
+    with _data_lock:
+        latest_data["trains"] = trains
+    _mark_fresh("trains")
+    logger.info(
+        "Trains: %s total%s",
+        len(trains),
+        f" ({', '.join(source_counts)})" if source_counts else "",
+    )

backend/services/fetchers/ukraine_alerts.py

@@ -0,0 +1,139 @@
+"""Ukraine air raid alerts via alerts.in.ua API.
+
+Polls active alerts every 2 minutes, matches to oblast boundary polygons,
+and produces GeoJSON-style records for map rendering.
+
+Requires ALERTS_IN_UA_TOKEN env var (free registration at alerts.in.ua).
+Gracefully skips if token is not set.
+"""
+
+import json
+import logging
+import os
+from pathlib import Path
+
+from services.network_utils import fetch_with_curl
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger(__name__)
+
+# ─── Alert type → color mapping ──────────────────────────────────────────────
+ALERT_COLORS = {
+    "air_raid": "#ef4444",            # red
+    "artillery_shelling": "#f97316",  # orange
+    "urban_fights": "#eab308",        # yellow
+    "chemical": "#a855f7",            # purple
+    "nuclear": "#dc2626",             # dark red
+}
+
+# ─── Load oblast boundary polygons (once) ────────────────────────────────────
+_oblast_geojson = None
+
+
+def _load_oblasts():
+    global _oblast_geojson
+    if _oblast_geojson is not None:
+        return _oblast_geojson
+
+    data_path = Path(__file__).resolve().parent.parent.parent / "data" / "ukraine_oblasts.geojson"
+    if not data_path.exists():
+        logger.error(f"Ukraine oblasts GeoJSON not found at {data_path}")
+        _oblast_geojson = {}
+        return _oblast_geojson
+
+    with open(data_path, "r", encoding="utf-8") as f:
+        _oblast_geojson = json.load(f)
+
+    logger.info(f"Loaded {len(_oblast_geojson.get('features', []))} Ukraine oblast boundaries")
+    return _oblast_geojson
+
+
+def _find_oblast_geometry(location_title: str):
+    """Find the polygon geometry for an oblast by matching Ukrainian name."""
+    oblasts = _load_oblasts()
+    features = oblasts.get("features", [])
+    for feat in features:
+        props = feat.get("properties", {})
+        name = props.get("name", "")
+        # Exact match on Ukrainian name (e.g. "Луганська область")
+        if name == location_title:
+            return feat.get("geometry"), props.get("name_en", "")
+    # Fuzzy: try partial match (alert may say "Київська область" but GeoJSON says "Київ")
+    for feat in features:
+        props = feat.get("properties", {})
+        name = props.get("name", "")
+        if location_title in name or name in location_title:
+            return feat.get("geometry"), props.get("name_en", "")
+    return None, ""
+
+
+# ─── Fetcher ─────────────────────────────────────────────────────────────────
+
+@with_retry(max_retries=1, base_delay=2)
+def fetch_ukraine_air_raid_alerts():
+    """Fetch active Ukraine air raid alerts from alerts.in.ua."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("ukraine_alerts"):
+        return
+
+    token = os.environ.get("ALERTS_IN_UA_TOKEN", "")
+    if not token:
+        logger.debug("ALERTS_IN_UA_TOKEN not set, skipping Ukraine air raid alerts")
+        return
+
+    alerts_out = []
+    try:
+        url = f"https://api.alerts.in.ua/v1/alerts/active.json?token={token}"
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/json",
+        }
+        response = fetch_with_curl(url, timeout=10, headers=headers)
+
+        if response.status_code == 200:
+            data = response.json()
+            raw_alerts = data.get("alerts", [])
+
+            for alert in raw_alerts:
+                loc_type = alert.get("location_type", "")
+                # Only render oblast-level alerts (not raion/city/hromada)
+                if loc_type != "oblast":
+                    continue
+
+                location_title = alert.get("location_title", "")
+                alert_type = alert.get("alert_type", "air_raid")
+                geometry, name_en = _find_oblast_geometry(location_title)
+
+                if not geometry:
+                    logger.debug(f"No geometry for oblast: {location_title}")
+                    continue
+
+                alerts_out.append({
+                    "id": alert.get("id", 0),
+                    "alert_type": alert_type,
+                    "location_title": location_title,
+                    "location_uid": alert.get("location_uid", ""),
+                    "name_en": name_en,
+                    "started_at": alert.get("started_at", ""),
+                    "color": ALERT_COLORS.get(alert_type, "#ef4444"),
+                    "geometry": geometry,
+                })
+
+            logger.info(f"Ukraine alerts: {len(alerts_out)} active oblast-level alerts "
+                        f"(from {len(raw_alerts)} total)")
+        elif response.status_code == 401:
+            logger.warning("alerts.in.ua returned 401 — check ALERTS_IN_UA_TOKEN")
+        elif response.status_code == 429:
+            logger.warning("alerts.in.ua rate-limited (429)")
+        else:
+            logger.warning(f"alerts.in.ua returned HTTP {response.status_code}")
+
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching Ukraine alerts: {e}")
+
+    with _data_lock:
+        latest_data["ukraine_alerts"] = alerts_out
+    if alerts_out:
+        _mark_fresh("ukraine_alerts")

backend/services/fetchers/unusual_whales.py

@@ -0,0 +1,76 @@
+"""Finnhub scheduled fetcher — congress trades, insider transactions, defense quotes.
+
+Runs on a 15-minute schedule and stores results in latest_data["unusual_whales"].
+Also updates latest_data["stocks"] with Finnhub quotes (replaces yfinance for defense tickers).
+Falls back gracefully if no API key is configured.
+"""
+
+import logging
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger(__name__)
+
+
+@with_retry(max_retries=1, base_delay=2)
+def fetch_unusual_whales():
+    """Fetch congress trades, insider txns, and defense quotes from Finnhub."""
+    import os
+
+    if not os.environ.get("FINNHUB_API_KEY", "").strip():
+        logger.debug("FINNHUB_API_KEY not set — skipping scheduled fetch.")
+        return
+
+    from services.unusual_whales_connector import (
+        fetch_congress_trades,
+        fetch_insider_transactions,
+        fetch_defense_quotes,
+        FinnhubConnectorError,
+    )
+
+    result: dict = {}
+
+    # Defense stock quotes (also populates latest_data["stocks"])
+    try:
+        quotes = fetch_defense_quotes()
+        if quotes:
+            result["quotes"] = quotes
+            # Mirror into stocks for backward compat with existing MarketsPanel fallback
+            with _data_lock:
+                latest_data["stocks"] = quotes
+            _mark_fresh("stocks")
+    except FinnhubConnectorError as e:
+        logger.warning(f"Finnhub quotes fetch failed: {e.detail}")
+    except Exception as e:
+        logger.warning(f"Finnhub quotes fetch error: {e}")
+
+    # Congress trades
+    try:
+        congress = fetch_congress_trades()
+        result["congress_trades"] = congress.get("trades", [])
+    except FinnhubConnectorError as e:
+        logger.warning(f"Finnhub congress trades fetch failed: {e.detail}")
+    except Exception as e:
+        logger.warning(f"Finnhub congress trades fetch error: {e}")
+
+    # Insider transactions
+    try:
+        insiders = fetch_insider_transactions()
+        result["insider_transactions"] = insiders.get("transactions", [])
+    except FinnhubConnectorError as e:
+        logger.warning(f"Finnhub insider fetch failed: {e.detail}")
+    except Exception as e:
+        logger.warning(f"Finnhub insider fetch error: {e}")
+
+    if not result:
+        logger.warning("Finnhub update produced no data; keeping previous cache.")
+        return
+
+    with _data_lock:
+        latest_data["unusual_whales"] = result
+    _mark_fresh("unusual_whales")
+    logger.info(
+        f"Finnhub updated: {len(result.get('congress_trades', []))} congress, "
+        f"{len(result.get('insider_transactions', []))} insider, "
+        f"{len(result.get('quotes', {}))} quotes"
+    )

backend/services/fetchers/yacht_alert.py

@@ -1,4 +1,5 @@
 """Yacht-Alert DB — load and enrich AIS vessels with tracked yacht metadata."""
+
 import os
 import json
 import logging
@@ -26,7 +27,8 @@ def _load_yacht_alert_db():
     global _YACHT_ALERT_DB
     json_path = os.path.join(
         os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
-        "data", "yacht_alert_db.json"
+        "data",
+        "yacht_alert_db.json",
     )
     if not os.path.exists(json_path):
         logger.warning(f"Yacht-Alert DB not found at {json_path}")