fix(ui): SAR credential setup in Settings and async layer toggles.

Add Earthdata token entry on the SAR tab with accurate Mode B status, expose optional FIRMS_MAP_KEY in API settings, and remove the frontend operator-unlock gate that blocked localhost saves. Run network-heavy layer-enable fetches on a background executor with frontend retry polling so FIRMS toggles no longer freeze the single API worker.
This commit is contained in:
BigBodyCobain
2026-06-23 01:31:16 -06:00
parent 53ed63ffcf
commit 0690d94c37
9 changed files with 311 additions and 102 deletions
+9
View File
@@ -78,6 +78,15 @@ API_REGISTRY = [
"url": "https://earthquake.usgs.gov/",
"required": False,
},
{
"id": "firms_map_key",
"env_key": "FIRMS_MAP_KEY",
"name": "NASA FIRMS — MAP Key (optional)",
"description": "Optional NASA Earthdata MAP key for country-scoped VIIRS fire enrichment. Global VIIRS hotspots work without a key; set this only if you want per-country FIRMS detail. Free from NASA Earthdata.",
"category": "Geophysical",
"url": "https://firms.modaps.eosdis.nasa.gov/api/area/",
"required": False,
},
{
"id": "celestrak",
"env_key": None,
+81 -36
View File
@@ -1,7 +1,8 @@
"""Immediate data refresh when the operator enables a map layer.
Runs synchronously inside POST /api/layers so the frontend's post-toggle
live-data refetch sees populated payloads (T_toggle_visible guardrail).
Disk/local fetches run inline (milliseconds). Network-heavy fetches run on the
slow executor so POST /api/layers never blocks the single uvicorn worker for
tens of seconds (which freezes bootstrap + live-data and makes the map go black).
"""
from __future__ import annotations
@@ -9,6 +10,15 @@ import logging
logger = logging.getLogger(__name__)
# Inline — local DB / static files only.
_INSTANT_LAYER_KEYS: frozenset[str] = frozenset(
{"cctv", "power_plants", "datacenters"}
)
# Background — network-bound; may take seconds.
_SLOW_LAYER_KEYS: frozenset[str] = frozenset(
{"firms", "psk_reporter", "fishing_activity"}
)
def snapshot_active_layers() -> dict[str, bool]:
from services.fetchers._store import active_layers
@@ -16,26 +26,36 @@ def snapshot_active_layers() -> dict[str, bool]:
return dict(active_layers)
def refresh_newly_enabled_layers(before: dict[str, bool]) -> None:
"""Fetch any layers that transitioned off → on."""
from services.fetchers._store import active_layers, bump_data_version
def _was_off_now_on(before: dict[str, bool], key: str) -> bool:
from services.fetchers._store import active_layers
refreshed = False
return not bool(before.get(key, False)) and bool(active_layers.get(key, False))
def _enabled(key: str) -> bool:
return bool(active_layers.get(key, False))
def _was_off_now_on(key: str) -> bool:
return not bool(before.get(key, False)) and _enabled(key)
if _was_off_now_on("cctv"):
def _instant_fetch(key: str) -> None:
if key == "cctv":
from services.fetchers.infrastructure import fetch_cctv
fetch_cctv()
refreshed = True
logger.info("CCTV loaded (layer enabled)")
return
if key == "power_plants":
from services.fetchers.infrastructure import fetch_power_plants
if _was_off_now_on("firms"):
fetch_power_plants()
logger.info("Power plants loaded (layer enabled)")
return
if key == "datacenters":
from services.fetchers.infrastructure import fetch_datacenters
fetch_datacenters()
logger.info("Datacenters loaded (layer enabled)")
return
raise KeyError(key)
def _slow_fetch(key: str) -> None:
if key == "firms":
from services.fetchers.earth_observation import (
fetch_firms_country_fires,
fetch_firms_fires,
@@ -43,36 +63,61 @@ def refresh_newly_enabled_layers(before: dict[str, bool]) -> None:
fetch_firms_fires()
fetch_firms_country_fires()
refreshed = True
logger.info("FIRMS fires loaded (layer enabled)")
if _was_off_now_on("power_plants"):
from services.fetchers.infrastructure import fetch_power_plants
fetch_power_plants()
refreshed = True
logger.info("Power plants loaded (layer enabled)")
if _was_off_now_on("psk_reporter"):
return
if key == "psk_reporter":
from services.fetchers.infrastructure import fetch_psk_reporter
fetch_psk_reporter()
refreshed = True
logger.info("PSK Reporter loaded (layer enabled)")
if _was_off_now_on("datacenters"):
from services.fetchers.infrastructure import fetch_datacenters
fetch_datacenters()
refreshed = True
logger.info("Datacenters loaded (layer enabled)")
if _was_off_now_on("fishing_activity"):
return
if key == "fishing_activity":
from services.fetchers.geo import fetch_fishing_activity
fetch_fishing_activity()
refreshed = True
logger.info("Fishing activity loaded (layer enabled)")
return
raise KeyError(key)
if refreshed:
def _run_slow_enable_fetches(keys: tuple[str, ...]) -> None:
from services.fetchers._store import bump_data_version
for key in keys:
try:
_slow_fetch(key)
except Exception:
logger.exception("Layer enable fetch failed for %s", key)
bump_data_version()
def refresh_newly_enabled_layers(before: dict[str, bool]) -> None:
"""Fetch any layers that transitioned off → on."""
from services.fetchers._store import bump_data_version
instant_keys: list[str] = []
slow_keys: list[str] = []
for key in _INSTANT_LAYER_KEYS | _SLOW_LAYER_KEYS:
if _was_off_now_on(before, key):
if key in _INSTANT_LAYER_KEYS:
instant_keys.append(key)
else:
slow_keys.append(key)
if not instant_keys and not slow_keys:
return
for key in instant_keys:
try:
_instant_fetch(key)
except Exception:
logger.exception("Layer enable fetch failed for %s", key)
if instant_keys:
bump_data_version()
if slow_keys:
from services.data_fetcher import _SLOW_EXECUTOR
_SLOW_EXECUTOR.submit(_run_slow_enable_fetches, tuple(slow_keys))
+17 -4
View File
@@ -167,18 +167,31 @@ def products_fetch_enabled() -> bool:
return _flag("MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE", default=False)
def runtime_store_exists() -> bool:
"""True when ``data/sar_runtime.json`` exists on disk."""
return _RUNTIME_FILE.is_file()
def products_fetch_status() -> dict[str, Any]:
"""Structured status used by the router for the 'how to enable' UX."""
raw = _str("MESH_SAR_PRODUCTS_FETCH", default="block").strip().lower()
fetch_set = raw in {"allow", "enable", "enabled", "true", "on", "1"}
ack_set = _flag("MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE", default=False)
enabled = fetch_set and ack_set
token_set = bool(earthdata_token())
user_set = bool(earthdata_user())
opt_in = fetch_set and ack_set
# ``enabled`` historically meant opt-in flags only; ``fully_configured``
# is what the fetcher actually needs (flags + Earthdata token).
fully_configured = opt_in and token_set
return {
"enabled": enabled,
"enabled": opt_in,
"fully_configured": fully_configured,
"fetch_flag_set": fetch_set,
"acknowledge_flag_set": ack_set,
"earthdata_token_set": bool(earthdata_token()),
"earthdata_user_set": bool(earthdata_user()),
"earthdata_token_set": token_set,
"earthdata_user_set": user_set,
"runtime_store_exists": runtime_store_exists(),
"runtime_store_path": str(_RUNTIME_FILE),
"missing": _missing_for_products(fetch_set, ack_set),
"help": {
"summary": (
+12 -4
View File
@@ -1,6 +1,8 @@
"""Integration: layer enable triggers immediate data availability."""
from __future__ import annotations
import time
from services.fetchers._store import active_layers, latest_data, _data_lock
@@ -12,7 +14,13 @@ def test_firms_enable_populates_slow_payload(client):
r = client.post("/api/layers", json={"layers": {"firms": True}})
assert r.status_code == 200
slow = client.get("/api/live-data/slow")
assert slow.status_code == 200
fires = slow.json().get("firms_fires") or []
assert len(fires) > 0, "firms layer should populate on enable without waiting for scheduler"
fires: list = []
for _ in range(45):
slow = client.get("/api/live-data/slow")
assert slow.status_code == 200
fires = slow.json().get("firms_fires") or []
if fires:
break
time.sleep(2)
assert len(fires) > 0, "firms layer should populate after async on-enable fetch"
+24 -4
View File
@@ -15,13 +15,15 @@ def test_refresh_firms_on_enable_only():
with (
patch("services.fetchers.earth_observation.fetch_firms_fires") as firms,
patch("services.fetchers.earth_observation.fetch_firms_country_fires") as country,
patch("services.layer_enable_refresh._run_slow_enable_fetches") as run_slow,
patch("services.fetchers._store.bump_data_version") as bump,
):
refresh_newly_enabled_layers({**before, "firms": False})
firms.assert_called_once()
country.assert_called_once()
bump.assert_called_once()
firms.assert_not_called()
country.assert_not_called()
run_slow.assert_called_once()
assert run_slow.call_args[0][0] == ("firms",)
bump.assert_not_called()
active_layers["firms"] = before.get("firms", False)
@@ -34,3 +36,21 @@ def test_refresh_skips_when_layer_stays_off():
refresh_newly_enabled_layers(before)
fetch_cctv.assert_not_called()
def test_refresh_cctv_runs_inline():
before = {**snapshot_active_layers(), "cctv": False}
active_layers["cctv"] = True
with (
patch("services.fetchers.infrastructure.fetch_cctv") as fetch_cctv,
patch("services.fetchers._store.bump_data_version") as bump,
patch("services.data_fetcher._SLOW_EXECUTOR") as slow_exec,
):
refresh_newly_enabled_layers(before)
fetch_cctv.assert_called_once()
bump.assert_called_once()
slow_exec.submit.assert_not_called()
active_layers["cctv"] = before.get("cctv", False)
@@ -56,4 +56,4 @@ def test_layer_enable_refresh_covers_cold_toggle_layers():
source = inspect.getsource(layer_enable_refresh.refresh_newly_enabled_layers)
for key in ("cctv", "firms", "power_plants", "psk_reporter", "datacenters"):
assert f'"{key}"' in source or f"'{key}'" in source
assert key in layer_enable_refresh._INSTANT_LAYER_KEYS | layer_enable_refresh._SLOW_LAYER_KEYS
+147 -48
View File
@@ -2945,6 +2945,9 @@ function SarSettingsTab() {
const [loading, setLoading] = useState(true);
const [actionMsg, setActionMsg] = useState<{ type: 'ok' | 'err'; text: string } | null>(null);
const [disabling, setDisabling] = useState(false);
const [submitting, setSubmitting] = useState(false);
const [earthdataUser, setEarthdataUser] = useState('');
const [earthdataToken, setEarthdataToken] = useState('');
const fetchStatus = useCallback(async () => {
try {
@@ -2960,9 +2963,62 @@ function SarSettingsTab() {
useEffect(() => { fetchStatus(); }, [fetchStatus]);
const products = (status?.products ?? {}) as Record<string, unknown>;
const modeBEnabled = !!products.enabled;
const tokenSet = Boolean(products.earthdata_token_set);
const fullyConfigured = Boolean(products.fully_configured);
const optInEnabled = Boolean(products.enabled);
const runtimeStoreExists = Boolean(products.runtime_store_exists);
const catalogEnabled = !!(status?.catalog as Record<string, unknown>)?.enabled;
const openclawEnabled = !!status?.openclaw_enabled;
const missing = Array.isArray(products.missing)
? (products.missing as string[])
: [];
const modeBStatusLabel = (() => {
if (fullyConfigured) return 'Active — credentials configured';
if (optInEnabled && !tokenSet) return 'Opt-in on — Earthdata token missing';
if (tokenSet && !optInEnabled) return 'Token present — enable Mode B below';
return 'Not configured';
})();
const handleEnable = async () => {
if (earthdataToken.trim().length < 8) {
setActionMsg({ type: 'err', text: 'Earthdata token looks too short. Paste the full token string.' });
return;
}
setSubmitting(true);
setActionMsg(null);
try {
const res = await fetch(`${API_BASE}/api/sar/mode-b/enable`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
credentials: 'include',
body: JSON.stringify({
earthdata_user: earthdataUser.trim(),
earthdata_token: earthdataToken.trim(),
}),
});
if (!res.ok) {
const body = await res.json().catch(() => ({}));
let msg = `HTTP ${res.status}`;
const d = body?.detail;
if (typeof d === 'string') msg = d;
else if (Array.isArray(d) && d.length > 0) {
msg = d.map((item: { msg?: string; loc?: string[] }) => item?.msg || 'invalid').join('; ');
}
throw new Error(msg);
}
setEarthdataToken('');
setActionMsg({ type: 'ok', text: 'Mode B enabled. Credentials saved on this node.' });
await fetchStatus();
} catch (e) {
setActionMsg({
type: 'err',
text: e instanceof Error ? e.message : 'Failed to save SAR credentials',
});
} finally {
setSubmitting(false);
}
};
const handleDisable = async () => {
setDisabling(true);
@@ -2976,6 +3032,8 @@ function SarSettingsTab() {
const body = await res.json().catch(() => ({}));
throw new Error(typeof body?.detail === 'string' ? body.detail : `HTTP ${res.status}`);
}
setEarthdataUser('');
setEarthdataToken('');
setActionMsg({ type: 'ok', text: 'Mode B disabled. Credentials wiped.' });
await fetchStatus();
} catch (e) {
@@ -3015,10 +3073,14 @@ function SarSettingsTab() {
</span>
</div>
<div className="flex items-center gap-2">
<span className={`w-2 h-2 rounded-full ${modeBEnabled ? 'bg-green-400' : 'bg-yellow-400'}`} />
<span
className={`w-2 h-2 rounded-full ${
fullyConfigured ? 'bg-green-400' : optInEnabled || tokenSet ? 'bg-yellow-400' : 'bg-gray-500'
}`}
/>
<span className="text-[11px]">
<span className="text-amber-300 font-bold">Mode B</span> (Anomalies):{' '}
{modeBEnabled ? 'Active — credentials stored' : 'Not configured'}
{modeBStatusLabel}
</span>
</div>
<div className="flex items-center gap-2">
@@ -3029,63 +3091,100 @@ function SarSettingsTab() {
</span>
</div>
</div>
{missing.length > 0 && !fullyConfigured && (
<p className="text-[10px] text-amber-200/60 pt-1">
Still needed: {missing.join(' · ')}
</p>
)}
</div>
</div>
</div>
{/* Mode B Controls */}
{modeBEnabled && (
<div className="mx-4 mt-3 p-3 border border-amber-900/20 bg-amber-950/5 space-y-3">
<p className="text-[11px] font-mono text-amber-300 font-bold tracking-wide">
MODE B CREDENTIALS
</p>
<p className="text-[11px] font-mono text-[var(--text-muted)]">
Earthdata credentials are stored server-side in{' '}
<span className="text-amber-400/80">backend/data/sar_runtime.json</span>.
Disabling Mode B wipes them from disk.
</p>
<div className="flex gap-2">
{/* Mode B credential entry — always visible */}
<div className="mx-4 mt-3 p-3 border border-amber-900/20 bg-amber-950/5 space-y-3">
<p className="text-[11px] font-mono text-amber-300 font-bold tracking-wide">
MODE B EARTHDATA CREDENTIALS
</p>
<p className="text-[11px] font-mono text-[var(--text-muted)] leading-relaxed">
Free NASA Earthdata Login required for OPERA ground-change products. Paste your user
token below stored only on this node
{runtimeStoreExists ? ' in the runtime credentials file' : ' (created on first save)'}.
</p>
<ol className="list-decimal list-inside space-y-1 text-[11px] font-mono text-[var(--text-secondary)]">
<li>
Register at{' '}
<a
href="https://urs.earthdata.nasa.gov/users/new"
target="_blank"
rel="noopener noreferrer"
className="text-amber-400 underline hover:text-amber-300"
>
urs.earthdata.nasa.gov
</a>
</li>
<li>
Generate a user token from your{' '}
<a
href="https://urs.earthdata.nasa.gov/profile"
target="_blank"
rel="noopener noreferrer"
className="text-amber-400 underline hover:text-amber-300"
>
Earthdata profile
</a>
</li>
</ol>
<div className="space-y-2">
<label htmlFor="sar-settings-earthdata-user" className="block text-[11px] text-amber-200/80">
Earthdata username (optional)
</label>
<input
id="sar-settings-earthdata-user"
type="text"
value={earthdataUser}
onChange={(e) => setEarthdataUser(e.target.value)}
placeholder="yourname"
autoComplete="off"
className="w-full rounded border border-amber-400/30 bg-zinc-900 px-3 py-2 text-xs text-amber-100 placeholder:text-amber-100/30 focus:border-amber-400/70 focus:outline-none cursor-text"
/>
<label htmlFor="sar-settings-earthdata-token" className="block text-[11px] text-amber-200/80 mt-2">
Earthdata user token (required)
</label>
<input
id="sar-settings-earthdata-token"
type="password"
value={earthdataToken}
onChange={(e) => setEarthdataToken(e.target.value)}
placeholder={tokenSet ? '•••••••• (enter new token to rotate)' : 'eyJ0eXAiOiJKV1QiLCJhbGciOi...'}
autoComplete="off"
className="w-full rounded border border-amber-400/30 bg-zinc-900 px-3 py-2 text-xs text-amber-100 placeholder:text-amber-100/30 focus:border-amber-400/70 focus:outline-none font-mono cursor-text"
/>
</div>
<div className="flex flex-wrap gap-2">
<button
type="button"
onClick={() => void handleEnable()}
disabled={submitting || earthdataToken.trim().length < 8}
className="px-3 py-1.5 text-[10px] font-mono font-bold tracking-wide border border-amber-400/50 text-amber-200 hover:bg-amber-500/10 transition disabled:opacity-50"
>
{submitting ? 'SAVING...' : fullyConfigured ? 'UPDATE CREDENTIALS' : 'ENABLE MODE B'}
</button>
{(fullyConfigured || optInEnabled || tokenSet) && (
<button
type="button"
onClick={handleDisable}
onClick={() => void handleDisable()}
disabled={disabling}
className="px-3 py-1.5 text-[10px] font-mono font-bold tracking-wide border border-red-500/40 text-red-400 hover:bg-red-500/10 transition disabled:opacity-50"
>
{disabling ? 'DISABLING...' : 'REVOKE & DISABLE MODE B'}
</button>
</div>
)}
</div>
)}
{/* Setup Guide (when Mode B not active) */}
{!modeBEnabled && (
<div className="mx-4 mt-3 p-3 border border-amber-900/20 bg-amber-950/5 space-y-3">
<p className="text-[11px] font-mono text-amber-300 font-bold tracking-wide">
ENABLE MODE B
</p>
<p className="text-[11px] font-mono text-[var(--text-muted)]">
Mode B requires a free NASA Earthdata account. To set up:
</p>
<ol className="list-decimal list-inside space-y-1 text-[11px] font-mono text-[var(--text-secondary)]">
<li>
Register at{' '}
<a
href="https://urs.earthdata.nasa.gov/users/new"
target="_blank"
rel="noopener noreferrer"
className="text-amber-400 underline hover:text-amber-300"
>
urs.earthdata.nasa.gov
</a>
</li>
<li>Generate a user token from your Earthdata profile page</li>
<li>
Toggle the <span className="text-white">SAR Ground-Change</span> layer ON
in the left panel the first-run wizard will prompt for your token
</li>
</ol>
</div>
)}
</div>
{/* Action feedback */}
{actionMsg && (
@@ -850,7 +850,9 @@ const WorldviewLeftPanel = React.memo(function WorldviewLeftPanel({
});
if (!res.ok || cancelled) return;
const body = await res.json();
const modeBOn = Boolean(body?.products?.enabled);
const modeBOn = Boolean(
body?.products?.fully_configured ?? body?.products?.enabled,
);
if (cancelled) return;
if (modeBOn && sarChoice !== 'b_active') {
try {
+17 -4
View File
@@ -295,13 +295,26 @@ export function useDataPolling() {
}, VIEWPORT_FAST_REFETCH_DEBOUNCE_MS);
};
// When a layer toggle fires, immediately refetch slow data so the user
// doesn't wait up to 120s for power plants / GDELT / etc. to appear.
// When a layer toggle fires, refetch live tiers immediately and retry a few
// times so network-heavy on-enable fetches (FIRMS, PSK, …) can finish in the
// background without blocking POST /api/layers on the single API worker.
const onLayerToggle = () => {
slowEtag.current = null; // invalidate ETag → guarantees fresh payload
slowEtag.current = null;
fastEtag.current = null;
if (slowTimerId) clearTimeout(slowTimerId);
slowTimerId = null;
fetchSlowData();
void fetchFastData();
void fetchSlowData();
const retryDelaysMs = [1000, 2500, 5000];
for (const delay of retryDelaysMs) {
setTimeout(() => {
if (_pollingPaused) return;
slowEtag.current = null;
fastEtag.current = null;
void fetchSlowData();
void fetchFastData();
}, delay);
}
};
window.addEventListener(LAYER_TOGGLE_EVENT, onLayerToggle);
window.addEventListener(VIEWPORT_COMMITTED_EVENT, queueViewportFastRefetch);