perf: UX-safe fetch trimming and instant layer-enable refresh.

Drop duplicate slow-tier weather/ukraine jobs, gate correlations when off, slim health probes, keyed layer-panel subscriptions, align backend layer defaults with the dashboard, and fetch CCTV/FIRMS/PSK/etc. synchronously on enable so toggles stay responsive without background prefetch waste.

backend/routers/data.py

@@ -498,12 +498,13 @@ def _run_prediction_markets_disable() -> None:
 async def update_layers(update: LayerUpdate, request: Request):
     """Receive frontend layer toggle state. Starts/stops streams accordingly."""
     from services.fetchers._store import active_layers, bump_active_layers_version, is_any_active
+    from services.layer_enable_refresh import refresh_newly_enabled_layers, snapshot_active_layers
+
+    layers_before = snapshot_active_layers()
     old_ships = is_any_active("ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts")
     old_mesh = is_any_active("sigint_meshtastic")
     old_aprs = is_any_active("sigint_aprs")
     old_viirs = is_any_active("viirs_nightlights")
-    old_datacenters = is_any_active("datacenters")
-    old_fishing = is_any_active("fishing_activity")
     changed = False
     for key, value in update.layers.items():
         if key in active_layers:
@@ -516,8 +517,6 @@ async def update_layers(update: LayerUpdate, request: Request):
     new_mesh = is_any_active("sigint_meshtastic")
     new_aprs = is_any_active("sigint_aprs")
     new_viirs = is_any_active("viirs_nightlights")
-    new_datacenters = is_any_active("datacenters")
-    new_fishing = is_any_active("fishing_activity")
     if old_ships and not new_ships:
         from services.ais_stream import stop_ais_stream
         stop_ais_stream()
@@ -561,16 +560,7 @@ async def update_layers(update: LayerUpdate, request: Request):
     if not old_viirs and new_viirs:
         _queue_viirs_change_refresh()
         logger.info("VIIRS change refresh queued (layer enabled)")
-    if not old_datacenters and new_datacenters:
-        from services.fetchers.infrastructure import fetch_datacenters
-
-        fetch_datacenters()
-        logger.info("Datacenters loaded (layer enabled)")
-    if not old_fishing and new_fishing:
-        from services.fetchers.geo import fetch_fishing_activity
-
-        fetch_fishing_activity()
-        logger.info("Fishing activity refresh queued (layer enabled)")
+    refresh_newly_enabled_layers(layers_before)
     return {"status": "ok"}
 
 

backend/routers/health.py

@@ -4,10 +4,50 @@ from fastapi.responses import JSONResponse
 from pydantic import BaseModel
 from limiter import limiter
 from auth import require_admin
-from services.data_fetcher import get_latest_data
 from services.schemas import HealthResponse
 import os
 
+# Health/SLO probes only need counts + freshness — not a full dashboard deepcopy.
+_HEALTH_DATA_KEYS: tuple[str, ...] = (
+    "last_updated",
+    "commercial_flights",
+    "military_flights",
+    "private_jets",
+    "ships",
+    "satellites",
+    "earthquakes",
+    "cctv",
+    "news",
+    "uavs",
+    "firms_fires",
+    "liveuamap",
+    "gdelt",
+    "uap_sightings",
+    "wastewater",
+    "fimi",
+    "space_weather",
+    "weather_alerts",
+    "volcanoes",
+    "prediction_markets",
+)
+
+
+def _health_data_snapshot() -> dict:
+    from services.fetchers._store import get_latest_data_subset_refs
+    from services.slo import SLO_REGISTRY
+
+    keys = tuple(dict.fromkeys((*_HEALTH_DATA_KEYS, *SLO_REGISTRY.keys())))
+    return get_latest_data_subset_refs(*keys)
+
+
+def _health_row_count(value) -> int:
+    if value is None:
+        return 0
+    try:
+        return len(value)
+    except TypeError:
+        return 0
+
 APP_VERSION = os.environ.get("_HEALTH_APP_VERSION", "0.9.82")
 
 router = APIRouter()
@@ -41,7 +81,7 @@ async def health_check(request: Request):
     from services.fetchers._store import get_source_timestamps_snapshot
     from services.slo import compute_all_statuses, summarise_statuses
 
-    d = get_latest_data()
+    d = _health_data_snapshot()
     last = d.get("last_updated")
     timestamps = get_source_timestamps_snapshot()
     slo_statuses = compute_all_statuses(d, timestamps)
@@ -102,18 +142,18 @@ async def health_check(request: Request):
         "version": _get_app_version(),
         "last_updated": last,
         "sources": {
-            "flights": len(d.get("commercial_flights", [])),
-            "military": len(d.get("military_flights", [])),
-            "ships": len(d.get("ships", [])),
-            "satellites": len(d.get("satellites", [])),
-            "earthquakes": len(d.get("earthquakes", [])),
-            "cctv": len(d.get("cctv", [])),
-            "news": len(d.get("news", [])),
-            "uavs": len(d.get("uavs", [])),
-            "firms_fires": len(d.get("firms_fires", [])),
-            "liveuamap": len(d.get("liveuamap", [])),
-            "gdelt": len(d.get("gdelt", [])),
-            "uap_sightings": len(d.get("uap_sightings", [])),
+            "flights": _health_row_count(d.get("commercial_flights")),
+            "military": _health_row_count(d.get("military_flights")),
+            "ships": _health_row_count(d.get("ships")),
+            "satellites": _health_row_count(d.get("satellites")),
+            "earthquakes": _health_row_count(d.get("earthquakes")),
+            "cctv": _health_row_count(d.get("cctv")),
+            "news": _health_row_count(d.get("news")),
+            "uavs": _health_row_count(d.get("uavs")),
+            "firms_fires": _health_row_count(d.get("firms_fires")),
+            "liveuamap": _health_row_count(d.get("liveuamap")),
+            "gdelt": _health_row_count(d.get("gdelt")),
+            "uap_sightings": _health_row_count(d.get("uap_sightings")),
         },
         "freshness": timestamps,
         "uptime_seconds": round(_time_mod.time() - _get_start_time()),
@@ -127,4 +167,7 @@ async def health_check(request: Request):
 @router.get("/api/debug-latest", dependencies=[Depends(require_admin)])
 @limiter.limit("30/minute")
 async def debug_latest_data(request: Request):
-    return list(get_latest_data().keys())
+    from services.fetchers._store import latest_data, _data_lock
+
+    with _data_lock:
+        return list(latest_data.keys())