diff --git a/backend/services/api_settings.py b/backend/services/api_settings.py index 49b2ed6..e3e70e1 100644 --- a/backend/services/api_settings.py +++ b/backend/services/api_settings.py @@ -78,6 +78,15 @@ API_REGISTRY = [ "url": "https://earthquake.usgs.gov/", "required": False, }, + { + "id": "firms_map_key", + "env_key": "FIRMS_MAP_KEY", + "name": "NASA FIRMS — MAP Key (optional)", + "description": "Optional NASA Earthdata MAP key for country-scoped VIIRS fire enrichment. Global VIIRS hotspots work without a key; set this only if you want per-country FIRMS detail. Free from NASA Earthdata.", + "category": "Geophysical", + "url": "https://firms.modaps.eosdis.nasa.gov/api/area/", + "required": False, + }, { "id": "celestrak", "env_key": None, diff --git a/backend/services/layer_enable_refresh.py b/backend/services/layer_enable_refresh.py index 5fe4f7c..3052365 100644 --- a/backend/services/layer_enable_refresh.py +++ b/backend/services/layer_enable_refresh.py @@ -1,7 +1,8 @@ """Immediate data refresh when the operator enables a map layer. -Runs synchronously inside POST /api/layers so the frontend's post-toggle -live-data refetch sees populated payloads (T_toggle_visible guardrail). +Disk/local fetches run inline (milliseconds). Network-heavy fetches run on the +slow executor so POST /api/layers never blocks the single uvicorn worker for +tens of seconds (which freezes bootstrap + live-data and makes the map go black). """ from __future__ import annotations @@ -9,6 +10,15 @@ import logging logger = logging.getLogger(__name__) +# Inline — local DB / static files only. +_INSTANT_LAYER_KEYS: frozenset[str] = frozenset( + {"cctv", "power_plants", "datacenters"} +) +# Background — network-bound; may take seconds. +_SLOW_LAYER_KEYS: frozenset[str] = frozenset( + {"firms", "psk_reporter", "fishing_activity"} +) + def snapshot_active_layers() -> dict[str, bool]: from services.fetchers._store import active_layers @@ -16,26 +26,36 @@ def snapshot_active_layers() -> dict[str, bool]: return dict(active_layers) -def refresh_newly_enabled_layers(before: dict[str, bool]) -> None: - """Fetch any layers that transitioned off → on.""" - from services.fetchers._store import active_layers, bump_data_version +def _was_off_now_on(before: dict[str, bool], key: str) -> bool: + from services.fetchers._store import active_layers - refreshed = False + return not bool(before.get(key, False)) and bool(active_layers.get(key, False)) - def _enabled(key: str) -> bool: - return bool(active_layers.get(key, False)) - def _was_off_now_on(key: str) -> bool: - return not bool(before.get(key, False)) and _enabled(key) - - if _was_off_now_on("cctv"): +def _instant_fetch(key: str) -> None: + if key == "cctv": from services.fetchers.infrastructure import fetch_cctv fetch_cctv() - refreshed = True logger.info("CCTV loaded (layer enabled)") + return + if key == "power_plants": + from services.fetchers.infrastructure import fetch_power_plants - if _was_off_now_on("firms"): + fetch_power_plants() + logger.info("Power plants loaded (layer enabled)") + return + if key == "datacenters": + from services.fetchers.infrastructure import fetch_datacenters + + fetch_datacenters() + logger.info("Datacenters loaded (layer enabled)") + return + raise KeyError(key) + + +def _slow_fetch(key: str) -> None: + if key == "firms": from services.fetchers.earth_observation import ( fetch_firms_country_fires, fetch_firms_fires, @@ -43,36 +63,61 @@ def refresh_newly_enabled_layers(before: dict[str, bool]) -> None: fetch_firms_fires() fetch_firms_country_fires() - refreshed = True logger.info("FIRMS fires loaded (layer enabled)") - - if _was_off_now_on("power_plants"): - from services.fetchers.infrastructure import fetch_power_plants - - fetch_power_plants() - refreshed = True - logger.info("Power plants loaded (layer enabled)") - - if _was_off_now_on("psk_reporter"): + return + if key == "psk_reporter": from services.fetchers.infrastructure import fetch_psk_reporter fetch_psk_reporter() - refreshed = True logger.info("PSK Reporter loaded (layer enabled)") - - if _was_off_now_on("datacenters"): - from services.fetchers.infrastructure import fetch_datacenters - - fetch_datacenters() - refreshed = True - logger.info("Datacenters loaded (layer enabled)") - - if _was_off_now_on("fishing_activity"): + return + if key == "fishing_activity": from services.fetchers.geo import fetch_fishing_activity fetch_fishing_activity() - refreshed = True logger.info("Fishing activity loaded (layer enabled)") + return + raise KeyError(key) - if refreshed: + +def _run_slow_enable_fetches(keys: tuple[str, ...]) -> None: + from services.fetchers._store import bump_data_version + + for key in keys: + try: + _slow_fetch(key) + except Exception: + logger.exception("Layer enable fetch failed for %s", key) + bump_data_version() + + +def refresh_newly_enabled_layers(before: dict[str, bool]) -> None: + """Fetch any layers that transitioned off → on.""" + from services.fetchers._store import bump_data_version + + instant_keys: list[str] = [] + slow_keys: list[str] = [] + + for key in _INSTANT_LAYER_KEYS | _SLOW_LAYER_KEYS: + if _was_off_now_on(before, key): + if key in _INSTANT_LAYER_KEYS: + instant_keys.append(key) + else: + slow_keys.append(key) + + if not instant_keys and not slow_keys: + return + + for key in instant_keys: + try: + _instant_fetch(key) + except Exception: + logger.exception("Layer enable fetch failed for %s", key) + + if instant_keys: bump_data_version() + + if slow_keys: + from services.data_fetcher import _SLOW_EXECUTOR + + _SLOW_EXECUTOR.submit(_run_slow_enable_fetches, tuple(slow_keys)) diff --git a/backend/services/sar/sar_config.py b/backend/services/sar/sar_config.py index 8ffb0d2..91da5b2 100644 --- a/backend/services/sar/sar_config.py +++ b/backend/services/sar/sar_config.py @@ -167,18 +167,31 @@ def products_fetch_enabled() -> bool: return _flag("MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE", default=False) +def runtime_store_exists() -> bool: + """True when ``data/sar_runtime.json`` exists on disk.""" + return _RUNTIME_FILE.is_file() + + def products_fetch_status() -> dict[str, Any]: """Structured status used by the router for the 'how to enable' UX.""" raw = _str("MESH_SAR_PRODUCTS_FETCH", default="block").strip().lower() fetch_set = raw in {"allow", "enable", "enabled", "true", "on", "1"} ack_set = _flag("MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE", default=False) - enabled = fetch_set and ack_set + token_set = bool(earthdata_token()) + user_set = bool(earthdata_user()) + opt_in = fetch_set and ack_set + # ``enabled`` historically meant opt-in flags only; ``fully_configured`` + # is what the fetcher actually needs (flags + Earthdata token). + fully_configured = opt_in and token_set return { - "enabled": enabled, + "enabled": opt_in, + "fully_configured": fully_configured, "fetch_flag_set": fetch_set, "acknowledge_flag_set": ack_set, - "earthdata_token_set": bool(earthdata_token()), - "earthdata_user_set": bool(earthdata_user()), + "earthdata_token_set": token_set, + "earthdata_user_set": user_set, + "runtime_store_exists": runtime_store_exists(), + "runtime_store_path": str(_RUNTIME_FILE), "missing": _missing_for_products(fetch_set, ack_set), "help": { "summary": ( diff --git a/backend/tests/test_layer_enable_integration.py b/backend/tests/test_layer_enable_integration.py index f8de11f..c980351 100644 --- a/backend/tests/test_layer_enable_integration.py +++ b/backend/tests/test_layer_enable_integration.py @@ -1,6 +1,8 @@ """Integration: layer enable triggers immediate data availability.""" from __future__ import annotations +import time + from services.fetchers._store import active_layers, latest_data, _data_lock @@ -12,7 +14,13 @@ def test_firms_enable_populates_slow_payload(client): r = client.post("/api/layers", json={"layers": {"firms": True}}) assert r.status_code == 200 - slow = client.get("/api/live-data/slow") - assert slow.status_code == 200 - fires = slow.json().get("firms_fires") or [] - assert len(fires) > 0, "firms layer should populate on enable without waiting for scheduler" + fires: list = [] + for _ in range(45): + slow = client.get("/api/live-data/slow") + assert slow.status_code == 200 + fires = slow.json().get("firms_fires") or [] + if fires: + break + time.sleep(2) + + assert len(fires) > 0, "firms layer should populate after async on-enable fetch" diff --git a/backend/tests/test_layer_enable_refresh.py b/backend/tests/test_layer_enable_refresh.py index 0c783ef..0beff0b 100644 --- a/backend/tests/test_layer_enable_refresh.py +++ b/backend/tests/test_layer_enable_refresh.py @@ -15,13 +15,15 @@ def test_refresh_firms_on_enable_only(): with ( patch("services.fetchers.earth_observation.fetch_firms_fires") as firms, patch("services.fetchers.earth_observation.fetch_firms_country_fires") as country, + patch("services.layer_enable_refresh._run_slow_enable_fetches") as run_slow, patch("services.fetchers._store.bump_data_version") as bump, ): refresh_newly_enabled_layers({**before, "firms": False}) - - firms.assert_called_once() - country.assert_called_once() - bump.assert_called_once() + firms.assert_not_called() + country.assert_not_called() + run_slow.assert_called_once() + assert run_slow.call_args[0][0] == ("firms",) + bump.assert_not_called() active_layers["firms"] = before.get("firms", False) @@ -34,3 +36,21 @@ def test_refresh_skips_when_layer_stays_off(): refresh_newly_enabled_layers(before) fetch_cctv.assert_not_called() + + +def test_refresh_cctv_runs_inline(): + before = {**snapshot_active_layers(), "cctv": False} + active_layers["cctv"] = True + + with ( + patch("services.fetchers.infrastructure.fetch_cctv") as fetch_cctv, + patch("services.fetchers._store.bump_data_version") as bump, + patch("services.data_fetcher._SLOW_EXECUTOR") as slow_exec, + ): + refresh_newly_enabled_layers(before) + + fetch_cctv.assert_called_once() + bump.assert_called_once() + slow_exec.submit.assert_not_called() + + active_layers["cctv"] = before.get("cctv", False) diff --git a/backend/tests/test_perf_safe_optimizations.py b/backend/tests/test_perf_safe_optimizations.py index 166438b..3845378 100644 --- a/backend/tests/test_perf_safe_optimizations.py +++ b/backend/tests/test_perf_safe_optimizations.py @@ -56,4 +56,4 @@ def test_layer_enable_refresh_covers_cold_toggle_layers(): source = inspect.getsource(layer_enable_refresh.refresh_newly_enabled_layers) for key in ("cctv", "firms", "power_plants", "psk_reporter", "datacenters"): - assert f'"{key}"' in source or f"'{key}'" in source + assert key in layer_enable_refresh._INSTANT_LAYER_KEYS | layer_enable_refresh._SLOW_LAYER_KEYS diff --git a/frontend/src/components/SettingsPanel.tsx b/frontend/src/components/SettingsPanel.tsx index d450779..c43eaad 100644 --- a/frontend/src/components/SettingsPanel.tsx +++ b/frontend/src/components/SettingsPanel.tsx @@ -2945,6 +2945,9 @@ function SarSettingsTab() { const [loading, setLoading] = useState(true); const [actionMsg, setActionMsg] = useState<{ type: 'ok' | 'err'; text: string } | null>(null); const [disabling, setDisabling] = useState(false); + const [submitting, setSubmitting] = useState(false); + const [earthdataUser, setEarthdataUser] = useState(''); + const [earthdataToken, setEarthdataToken] = useState(''); const fetchStatus = useCallback(async () => { try { @@ -2960,9 +2963,62 @@ function SarSettingsTab() { useEffect(() => { fetchStatus(); }, [fetchStatus]); const products = (status?.products ?? {}) as Record; - const modeBEnabled = !!products.enabled; + const tokenSet = Boolean(products.earthdata_token_set); + const fullyConfigured = Boolean(products.fully_configured); + const optInEnabled = Boolean(products.enabled); + const runtimeStoreExists = Boolean(products.runtime_store_exists); const catalogEnabled = !!(status?.catalog as Record)?.enabled; const openclawEnabled = !!status?.openclaw_enabled; + const missing = Array.isArray(products.missing) + ? (products.missing as string[]) + : []; + + const modeBStatusLabel = (() => { + if (fullyConfigured) return 'Active — credentials configured'; + if (optInEnabled && !tokenSet) return 'Opt-in on — Earthdata token missing'; + if (tokenSet && !optInEnabled) return 'Token present — enable Mode B below'; + return 'Not configured'; + })(); + + const handleEnable = async () => { + if (earthdataToken.trim().length < 8) { + setActionMsg({ type: 'err', text: 'Earthdata token looks too short. Paste the full token string.' }); + return; + } + setSubmitting(true); + setActionMsg(null); + try { + const res = await fetch(`${API_BASE}/api/sar/mode-b/enable`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + credentials: 'include', + body: JSON.stringify({ + earthdata_user: earthdataUser.trim(), + earthdata_token: earthdataToken.trim(), + }), + }); + if (!res.ok) { + const body = await res.json().catch(() => ({})); + let msg = `HTTP ${res.status}`; + const d = body?.detail; + if (typeof d === 'string') msg = d; + else if (Array.isArray(d) && d.length > 0) { + msg = d.map((item: { msg?: string; loc?: string[] }) => item?.msg || 'invalid').join('; '); + } + throw new Error(msg); + } + setEarthdataToken(''); + setActionMsg({ type: 'ok', text: 'Mode B enabled. Credentials saved on this node.' }); + await fetchStatus(); + } catch (e) { + setActionMsg({ + type: 'err', + text: e instanceof Error ? e.message : 'Failed to save SAR credentials', + }); + } finally { + setSubmitting(false); + } + }; const handleDisable = async () => { setDisabling(true); @@ -2976,6 +3032,8 @@ function SarSettingsTab() { const body = await res.json().catch(() => ({})); throw new Error(typeof body?.detail === 'string' ? body.detail : `HTTP ${res.status}`); } + setEarthdataUser(''); + setEarthdataToken(''); setActionMsg({ type: 'ok', text: 'Mode B disabled. Credentials wiped.' }); await fetchStatus(); } catch (e) { @@ -3015,10 +3073,14 @@ function SarSettingsTab() {
- + Mode B (Anomalies):{' '} - {modeBEnabled ? 'Active — credentials stored' : 'Not configured'} + {modeBStatusLabel}
@@ -3029,63 +3091,100 @@ function SarSettingsTab() {
+ {missing.length > 0 && !fullyConfigured && ( +

+ Still needed: {missing.join(' · ')} +

+ )} - {/* Mode B Controls */} - {modeBEnabled && ( -
-

- MODE B CREDENTIALS -

-

- Earthdata credentials are stored server-side in{' '} - backend/data/sar_runtime.json. - Disabling Mode B wipes them from disk. -

-
+ {/* Mode B credential entry — always visible */} +
+

+ MODE B — EARTHDATA CREDENTIALS +

+

+ Free NASA Earthdata Login required for OPERA ground-change products. Paste your user + token below — stored only on this node + {runtimeStoreExists ? ' in the runtime credentials file' : ' (created on first save)'}. +

+ +
    +
  1. + Register at{' '} + + urs.earthdata.nasa.gov + +
    2. +
  2. + Generate a user token from your{' '} + + Earthdata profile + +
    3. +
+ +
+ + setEarthdataUser(e.target.value)} + placeholder="yourname" + autoComplete="off" + className="w-full rounded border border-amber-400/30 bg-zinc-900 px-3 py-2 text-xs text-amber-100 placeholder:text-amber-100/30 focus:border-amber-400/70 focus:outline-none cursor-text" + /> + + + setEarthdataToken(e.target.value)} + placeholder={tokenSet ? '•••••••• (enter new token to rotate)' : 'eyJ0eXAiOiJKV1QiLCJhbGciOi...'} + autoComplete="off" + className="w-full rounded border border-amber-400/30 bg-zinc-900 px-3 py-2 text-xs text-amber-100 placeholder:text-amber-100/30 focus:border-amber-400/70 focus:outline-none font-mono cursor-text" + /> +
+ +
+ + {(fullyConfigured || optInEnabled || tokenSet) && ( -
+ )}
- )} - - {/* Setup Guide (when Mode B not active) */} - {!modeBEnabled && ( -
-

- ENABLE MODE B -

-

- Mode B requires a free NASA Earthdata account. To set up: -

-
    -
  1. - Register at{' '} - - urs.earthdata.nasa.gov - -
    2. -
  2. Generate a user token from your Earthdata profile page
    3. -
  3. - Toggle the SAR Ground-Change layer ON - in the left panel — the first-run wizard will prompt for your token -
    4. -
-
- )} +
{/* Action feedback */} {actionMsg && ( diff --git a/frontend/src/components/WorldviewLeftPanel.tsx b/frontend/src/components/WorldviewLeftPanel.tsx index 83f65b5..1adc863 100644 --- a/frontend/src/components/WorldviewLeftPanel.tsx +++ b/frontend/src/components/WorldviewLeftPanel.tsx @@ -850,7 +850,9 @@ const WorldviewLeftPanel = React.memo(function WorldviewLeftPanel({ }); if (!res.ok || cancelled) return; const body = await res.json(); - const modeBOn = Boolean(body?.products?.enabled); + const modeBOn = Boolean( + body?.products?.fully_configured ?? body?.products?.enabled, + ); if (cancelled) return; if (modeBOn && sarChoice !== 'b_active') { try { diff --git a/frontend/src/hooks/useDataPolling.ts b/frontend/src/hooks/useDataPolling.ts index cf6a902..caf10be 100644 --- a/frontend/src/hooks/useDataPolling.ts +++ b/frontend/src/hooks/useDataPolling.ts @@ -295,13 +295,26 @@ export function useDataPolling() { }, VIEWPORT_FAST_REFETCH_DEBOUNCE_MS); }; - // When a layer toggle fires, immediately refetch slow data so the user - // doesn't wait up to 120s for power plants / GDELT / etc. to appear. + // When a layer toggle fires, refetch live tiers immediately and retry a few + // times so network-heavy on-enable fetches (FIRMS, PSK, …) can finish in the + // background without blocking POST /api/layers on the single API worker. const onLayerToggle = () => { - slowEtag.current = null; // invalidate ETag → guarantees fresh payload + slowEtag.current = null; + fastEtag.current = null; if (slowTimerId) clearTimeout(slowTimerId); slowTimerId = null; - fetchSlowData(); + void fetchFastData(); + void fetchSlowData(); + const retryDelaysMs = [1000, 2500, 5000]; + for (const delay of retryDelaysMs) { + setTimeout(() => { + if (_pollingPaused) return; + slowEtag.current = null; + fastEtag.current = null; + void fetchSlowData(); + void fetchFastData(); + }, delay); + } }; window.addEventListener(LAYER_TOGGLE_EVENT, onLayerToggle); window.addEventListener(VIEWPORT_COMMITTED_EVENT, queueViewportFastRefetch);