mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-08-09 20:26:03 +02:00
fix(ui): SAR credential setup in Settings and async layer toggles.
Add Earthdata token entry on the SAR tab with accurate Mode B status, expose optional FIRMS_MAP_KEY in API settings, and remove the frontend operator-unlock gate that blocked localhost saves. Run network-heavy layer-enable fetches on a background executor with frontend retry polling so FIRMS toggles no longer freeze the single API worker.
This commit is contained in:
@@ -78,6 +78,15 @@ API_REGISTRY = [
|
||||
"url": "https://earthquake.usgs.gov/",
|
||||
"required": False,
|
||||
},
|
||||
{
|
||||
"id": "firms_map_key",
|
||||
"env_key": "FIRMS_MAP_KEY",
|
||||
"name": "NASA FIRMS — MAP Key (optional)",
|
||||
"description": "Optional NASA Earthdata MAP key for country-scoped VIIRS fire enrichment. Global VIIRS hotspots work without a key; set this only if you want per-country FIRMS detail. Free from NASA Earthdata.",
|
||||
"category": "Geophysical",
|
||||
"url": "https://firms.modaps.eosdis.nasa.gov/api/area/",
|
||||
"required": False,
|
||||
},
|
||||
{
|
||||
"id": "celestrak",
|
||||
"env_key": None,
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
"""Immediate data refresh when the operator enables a map layer.
|
||||
|
||||
Runs synchronously inside POST /api/layers so the frontend's post-toggle
|
||||
live-data refetch sees populated payloads (T_toggle_visible guardrail).
|
||||
Disk/local fetches run inline (milliseconds). Network-heavy fetches run on the
|
||||
slow executor so POST /api/layers never blocks the single uvicorn worker for
|
||||
tens of seconds (which freezes bootstrap + live-data and makes the map go black).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -9,6 +10,15 @@ import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Inline — local DB / static files only.
|
||||
_INSTANT_LAYER_KEYS: frozenset[str] = frozenset(
|
||||
{"cctv", "power_plants", "datacenters"}
|
||||
)
|
||||
# Background — network-bound; may take seconds.
|
||||
_SLOW_LAYER_KEYS: frozenset[str] = frozenset(
|
||||
{"firms", "psk_reporter", "fishing_activity"}
|
||||
)
|
||||
|
||||
|
||||
def snapshot_active_layers() -> dict[str, bool]:
|
||||
from services.fetchers._store import active_layers
|
||||
@@ -16,26 +26,36 @@ def snapshot_active_layers() -> dict[str, bool]:
|
||||
return dict(active_layers)
|
||||
|
||||
|
||||
def refresh_newly_enabled_layers(before: dict[str, bool]) -> None:
|
||||
"""Fetch any layers that transitioned off → on."""
|
||||
from services.fetchers._store import active_layers, bump_data_version
|
||||
def _was_off_now_on(before: dict[str, bool], key: str) -> bool:
|
||||
from services.fetchers._store import active_layers
|
||||
|
||||
refreshed = False
|
||||
return not bool(before.get(key, False)) and bool(active_layers.get(key, False))
|
||||
|
||||
def _enabled(key: str) -> bool:
|
||||
return bool(active_layers.get(key, False))
|
||||
|
||||
def _was_off_now_on(key: str) -> bool:
|
||||
return not bool(before.get(key, False)) and _enabled(key)
|
||||
|
||||
if _was_off_now_on("cctv"):
|
||||
def _instant_fetch(key: str) -> None:
|
||||
if key == "cctv":
|
||||
from services.fetchers.infrastructure import fetch_cctv
|
||||
|
||||
fetch_cctv()
|
||||
refreshed = True
|
||||
logger.info("CCTV loaded (layer enabled)")
|
||||
return
|
||||
if key == "power_plants":
|
||||
from services.fetchers.infrastructure import fetch_power_plants
|
||||
|
||||
if _was_off_now_on("firms"):
|
||||
fetch_power_plants()
|
||||
logger.info("Power plants loaded (layer enabled)")
|
||||
return
|
||||
if key == "datacenters":
|
||||
from services.fetchers.infrastructure import fetch_datacenters
|
||||
|
||||
fetch_datacenters()
|
||||
logger.info("Datacenters loaded (layer enabled)")
|
||||
return
|
||||
raise KeyError(key)
|
||||
|
||||
|
||||
def _slow_fetch(key: str) -> None:
|
||||
if key == "firms":
|
||||
from services.fetchers.earth_observation import (
|
||||
fetch_firms_country_fires,
|
||||
fetch_firms_fires,
|
||||
@@ -43,36 +63,61 @@ def refresh_newly_enabled_layers(before: dict[str, bool]) -> None:
|
||||
|
||||
fetch_firms_fires()
|
||||
fetch_firms_country_fires()
|
||||
refreshed = True
|
||||
logger.info("FIRMS fires loaded (layer enabled)")
|
||||
|
||||
if _was_off_now_on("power_plants"):
|
||||
from services.fetchers.infrastructure import fetch_power_plants
|
||||
|
||||
fetch_power_plants()
|
||||
refreshed = True
|
||||
logger.info("Power plants loaded (layer enabled)")
|
||||
|
||||
if _was_off_now_on("psk_reporter"):
|
||||
return
|
||||
if key == "psk_reporter":
|
||||
from services.fetchers.infrastructure import fetch_psk_reporter
|
||||
|
||||
fetch_psk_reporter()
|
||||
refreshed = True
|
||||
logger.info("PSK Reporter loaded (layer enabled)")
|
||||
|
||||
if _was_off_now_on("datacenters"):
|
||||
from services.fetchers.infrastructure import fetch_datacenters
|
||||
|
||||
fetch_datacenters()
|
||||
refreshed = True
|
||||
logger.info("Datacenters loaded (layer enabled)")
|
||||
|
||||
if _was_off_now_on("fishing_activity"):
|
||||
return
|
||||
if key == "fishing_activity":
|
||||
from services.fetchers.geo import fetch_fishing_activity
|
||||
|
||||
fetch_fishing_activity()
|
||||
refreshed = True
|
||||
logger.info("Fishing activity loaded (layer enabled)")
|
||||
return
|
||||
raise KeyError(key)
|
||||
|
||||
if refreshed:
|
||||
|
||||
def _run_slow_enable_fetches(keys: tuple[str, ...]) -> None:
|
||||
from services.fetchers._store import bump_data_version
|
||||
|
||||
for key in keys:
|
||||
try:
|
||||
_slow_fetch(key)
|
||||
except Exception:
|
||||
logger.exception("Layer enable fetch failed for %s", key)
|
||||
bump_data_version()
|
||||
|
||||
|
||||
def refresh_newly_enabled_layers(before: dict[str, bool]) -> None:
|
||||
"""Fetch any layers that transitioned off → on."""
|
||||
from services.fetchers._store import bump_data_version
|
||||
|
||||
instant_keys: list[str] = []
|
||||
slow_keys: list[str] = []
|
||||
|
||||
for key in _INSTANT_LAYER_KEYS | _SLOW_LAYER_KEYS:
|
||||
if _was_off_now_on(before, key):
|
||||
if key in _INSTANT_LAYER_KEYS:
|
||||
instant_keys.append(key)
|
||||
else:
|
||||
slow_keys.append(key)
|
||||
|
||||
if not instant_keys and not slow_keys:
|
||||
return
|
||||
|
||||
for key in instant_keys:
|
||||
try:
|
||||
_instant_fetch(key)
|
||||
except Exception:
|
||||
logger.exception("Layer enable fetch failed for %s", key)
|
||||
|
||||
if instant_keys:
|
||||
bump_data_version()
|
||||
|
||||
if slow_keys:
|
||||
from services.data_fetcher import _SLOW_EXECUTOR
|
||||
|
||||
_SLOW_EXECUTOR.submit(_run_slow_enable_fetches, tuple(slow_keys))
|
||||
|
||||
@@ -167,18 +167,31 @@ def products_fetch_enabled() -> bool:
|
||||
return _flag("MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE", default=False)
|
||||
|
||||
|
||||
def runtime_store_exists() -> bool:
|
||||
"""True when ``data/sar_runtime.json`` exists on disk."""
|
||||
return _RUNTIME_FILE.is_file()
|
||||
|
||||
|
||||
def products_fetch_status() -> dict[str, Any]:
|
||||
"""Structured status used by the router for the 'how to enable' UX."""
|
||||
raw = _str("MESH_SAR_PRODUCTS_FETCH", default="block").strip().lower()
|
||||
fetch_set = raw in {"allow", "enable", "enabled", "true", "on", "1"}
|
||||
ack_set = _flag("MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE", default=False)
|
||||
enabled = fetch_set and ack_set
|
||||
token_set = bool(earthdata_token())
|
||||
user_set = bool(earthdata_user())
|
||||
opt_in = fetch_set and ack_set
|
||||
# ``enabled`` historically meant opt-in flags only; ``fully_configured``
|
||||
# is what the fetcher actually needs (flags + Earthdata token).
|
||||
fully_configured = opt_in and token_set
|
||||
return {
|
||||
"enabled": enabled,
|
||||
"enabled": opt_in,
|
||||
"fully_configured": fully_configured,
|
||||
"fetch_flag_set": fetch_set,
|
||||
"acknowledge_flag_set": ack_set,
|
||||
"earthdata_token_set": bool(earthdata_token()),
|
||||
"earthdata_user_set": bool(earthdata_user()),
|
||||
"earthdata_token_set": token_set,
|
||||
"earthdata_user_set": user_set,
|
||||
"runtime_store_exists": runtime_store_exists(),
|
||||
"runtime_store_path": str(_RUNTIME_FILE),
|
||||
"missing": _missing_for_products(fetch_set, ack_set),
|
||||
"help": {
|
||||
"summary": (
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
"""Integration: layer enable triggers immediate data availability."""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
from services.fetchers._store import active_layers, latest_data, _data_lock
|
||||
|
||||
|
||||
@@ -12,7 +14,13 @@ def test_firms_enable_populates_slow_payload(client):
|
||||
r = client.post("/api/layers", json={"layers": {"firms": True}})
|
||||
assert r.status_code == 200
|
||||
|
||||
slow = client.get("/api/live-data/slow")
|
||||
assert slow.status_code == 200
|
||||
fires = slow.json().get("firms_fires") or []
|
||||
assert len(fires) > 0, "firms layer should populate on enable without waiting for scheduler"
|
||||
fires: list = []
|
||||
for _ in range(45):
|
||||
slow = client.get("/api/live-data/slow")
|
||||
assert slow.status_code == 200
|
||||
fires = slow.json().get("firms_fires") or []
|
||||
if fires:
|
||||
break
|
||||
time.sleep(2)
|
||||
|
||||
assert len(fires) > 0, "firms layer should populate after async on-enable fetch"
|
||||
|
||||
@@ -15,13 +15,15 @@ def test_refresh_firms_on_enable_only():
|
||||
with (
|
||||
patch("services.fetchers.earth_observation.fetch_firms_fires") as firms,
|
||||
patch("services.fetchers.earth_observation.fetch_firms_country_fires") as country,
|
||||
patch("services.layer_enable_refresh._run_slow_enable_fetches") as run_slow,
|
||||
patch("services.fetchers._store.bump_data_version") as bump,
|
||||
):
|
||||
refresh_newly_enabled_layers({**before, "firms": False})
|
||||
|
||||
firms.assert_called_once()
|
||||
country.assert_called_once()
|
||||
bump.assert_called_once()
|
||||
firms.assert_not_called()
|
||||
country.assert_not_called()
|
||||
run_slow.assert_called_once()
|
||||
assert run_slow.call_args[0][0] == ("firms",)
|
||||
bump.assert_not_called()
|
||||
|
||||
active_layers["firms"] = before.get("firms", False)
|
||||
|
||||
@@ -34,3 +36,21 @@ def test_refresh_skips_when_layer_stays_off():
|
||||
refresh_newly_enabled_layers(before)
|
||||
|
||||
fetch_cctv.assert_not_called()
|
||||
|
||||
|
||||
def test_refresh_cctv_runs_inline():
|
||||
before = {**snapshot_active_layers(), "cctv": False}
|
||||
active_layers["cctv"] = True
|
||||
|
||||
with (
|
||||
patch("services.fetchers.infrastructure.fetch_cctv") as fetch_cctv,
|
||||
patch("services.fetchers._store.bump_data_version") as bump,
|
||||
patch("services.data_fetcher._SLOW_EXECUTOR") as slow_exec,
|
||||
):
|
||||
refresh_newly_enabled_layers(before)
|
||||
|
||||
fetch_cctv.assert_called_once()
|
||||
bump.assert_called_once()
|
||||
slow_exec.submit.assert_not_called()
|
||||
|
||||
active_layers["cctv"] = before.get("cctv", False)
|
||||
|
||||
@@ -56,4 +56,4 @@ def test_layer_enable_refresh_covers_cold_toggle_layers():
|
||||
|
||||
source = inspect.getsource(layer_enable_refresh.refresh_newly_enabled_layers)
|
||||
for key in ("cctv", "firms", "power_plants", "psk_reporter", "datacenters"):
|
||||
assert f'"{key}"' in source or f"'{key}'" in source
|
||||
assert key in layer_enable_refresh._INSTANT_LAYER_KEYS | layer_enable_refresh._SLOW_LAYER_KEYS
|
||||
|
||||
Reference in New Issue
Block a user