diff --git a/backend/routers/ai_intel.py b/backend/routers/ai_intel.py index 60ea84d..0220249 100644 --- a/backend/routers/ai_intel.py +++ b/backend/routers/ai_intel.py @@ -381,6 +381,50 @@ async def api_refresh_layer_feed(request: Request, layer_id: str): return {"ok": True, "layer": updated} +# --------------------------------------------------------------------------- +# Native map layer overrides — additive, transient, agent-driven. +# +# These are the DATA LAYERS toggles, not the pin layers above. An override +# switches an overlay on without touching the operator's own toggle state, and +# lapses on its own after the TTL. Agents holding an overlay open should re-PUT +# on their refresh tick rather than sending a long TTL. +# --------------------------------------------------------------------------- + +class LayerOverrideUpdate(BaseModel): + layers: dict[str, bool] + ttl_seconds: float = 300.0 + + +@router.put("/api/ai/layer-overrides", dependencies=[Depends(require_openclaw_or_local)]) +@limiter.limit("30/minute") +async def put_layer_overrides(request: Request, body: LayerOverrideUpdate): + """Replace the override map. Returns which keys were accepted and ignored.""" + from services.fetchers._store import set_layer_overrides + + accepted = set_layer_overrides(body.layers, body.ttl_seconds) + ignored = sorted(set(body.layers) - set(accepted)) + return {"ok": True, "overrides": accepted, "ignored": ignored} + + +@router.get("/api/ai/layer-overrides", dependencies=[Depends(require_openclaw_or_local)]) +@limiter.limit("60/minute") +async def read_layer_overrides(request: Request): + """Return the overrides that are currently live.""" + from services.fetchers._store import get_layer_overrides + + return {"ok": True, "overrides": get_layer_overrides()} + + +@router.delete("/api/ai/layer-overrides", dependencies=[Depends(require_openclaw_or_local)]) +@limiter.limit("30/minute") +async def delete_layer_overrides(request: Request): + """Drop all overrides, restoring the operator's own layer state.""" + from services.fetchers._store import clear_layer_overrides + + clear_layer_overrides() + return {"ok": True} + + # --------------------------------------------------------------------------- # Agent Actions endpoint — frontend polls this for UI commands from the agent # --------------------------------------------------------------------------- diff --git a/backend/routers/data.py b/backend/routers/data.py index 9ba0a16..9a5e131 100644 --- a/backend/routers/data.py +++ b/backend/routers/data.py @@ -566,6 +566,18 @@ def _run_prediction_markets_disable() -> None: logger.warning("Prediction markets disable cleanup failed: %s", e) +@router.get("/api/layers", dependencies=[Depends(require_local_operator)]) +async def get_layers(request: Request): + """Report operator layer state plus any live overrides. + + The UI polls this so the DATA LAYERS toggles can show an overlay that an + agent switched on. ``layers`` is the operator's own state and is what the + browser persists; ``overrides`` is transient and must not be saved. + """ + from services.fetchers._store import active_layers, get_layer_overrides + return {"layers": dict(active_layers), "overrides": get_layer_overrides()} + + @router.post("/api/layers", dependencies=[Depends(require_local_operator)]) @limiter.limit("30/minute") async def update_layers(update: LayerUpdate, request: Request): @@ -668,11 +680,15 @@ async def bootstrap_critical(request: Request): if request.headers.get("if-none-match") == etag: return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"}) from services.fetchers._store import ( - active_layers, + effective_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot, ) + # Rebind the local name to the override-merged map. Every layer filter + # below reads this name, so one merge here covers all of them. + active_layers = effective_layers() + def _build() -> dict: d = get_latest_data_subset_refs( "last_updated", "commercial_flights", "military_flights", "private_flights", @@ -769,14 +785,17 @@ def _try_build_fast_delta( ) -> dict | None: """Return a delta payload, or None to fall back to a full snapshot.""" from services.fetchers._store import ( - active_layers, compute_layer_row_delta, + effective_layers, get_data_version, get_layer_versions, get_latest_data_subset_refs, get_source_timestamps_snapshot, ) + # Rebind the local name to the override-merged map (see bootstrap_critical). + active_layers = effective_layers() + server_lv = get_layer_versions() deltas: dict[str, Any] = {} for key in _DELTA_FAST_KEYS: @@ -894,7 +913,10 @@ async def live_data_fast( etag = _current_etag(prefix=("fast|initial|" if initial else "fast|full|") + bbox_suffix.lstrip("|") + ("|" if bbox_suffix else "")) if request.headers.get("if-none-match") == etag: return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"}) - from services.fetchers._store import (active_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot) + from services.fetchers._store import (effective_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot) + + # Rebind the local name to the override-merged map (see bootstrap_critical). + active_layers = effective_layers() def _build() -> dict: d = get_latest_data_subset_refs( @@ -959,7 +981,10 @@ async def live_data_slow( etag = _current_etag(prefix="slow|full|" + bbox_suffix.lstrip("|") + ("|" if bbox_suffix else "")) if request.headers.get("if-none-match") == etag: return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"}) - from services.fetchers._store import (active_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot) + from services.fetchers._store import (effective_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot) + + # Rebind the local name to the override-merged map (see bootstrap_critical). + active_layers = effective_layers() def _build() -> dict: d = get_latest_data_subset_refs( diff --git a/backend/services/fetchers/_store.py b/backend/services/fetchers/_store.py index 135dd8f..20a9c43 100644 --- a/backend/services/fetchers/_store.py +++ b/backend/services/fetchers/_store.py @@ -7,6 +7,7 @@ Every fetcher imports from here instead of maintaining its own copy. import copy import threading import logging +import time from datetime import datetime from typing import Any, Dict, List, Optional, TypedDict @@ -505,6 +506,69 @@ active_layers: dict[str, bool] = { } +# --------------------------------------------------------------------------- +# Layer overrides — additive, agent-driven, never persisted. +# An automation (e.g. a hotspot daemon) can switch an overlay on for a while +# without touching the operator's own toggles. Overrides merge ON TOP of +# active_layers for reads; active_layers itself is only ever written by the +# operator via POST /api/layers. The whole map shares one expiry, evaluated +# lazily on read so no background task is needed. When it lapses the operator's +# own view returns with no save/restore bookkeeping. +# --------------------------------------------------------------------------- +_MAX_OVERRIDE_TTL_S = 3600.0 + +layer_overrides: dict[str, bool] = {} +_layer_overrides_expires_at: float = 0.0 + + +def get_layer_overrides() -> dict[str, bool]: + """Return the live overrides, or {} once the TTL has lapsed.""" + global _layer_overrides_expires_at + if not layer_overrides: + return {} + if time.monotonic() >= _layer_overrides_expires_at: + layer_overrides.clear() + _layer_overrides_expires_at = 0.0 + bump_active_layers_version() + return {} + return dict(layer_overrides) + + +def set_layer_overrides(overrides: dict[str, bool], ttl_seconds: float) -> dict[str, bool]: + """Replace the override map, returning the entries that were accepted. + + Keys that are not real layers are dropped so a typo cannot silently do + nothing — the caller compares the return value against what it sent. + """ + global _layer_overrides_expires_at + ttl = max(0.0, min(float(ttl_seconds), _MAX_OVERRIDE_TTL_S)) + accepted = {k: bool(v) for k, v in overrides.items() if k in active_layers} + layer_overrides.clear() + layer_overrides.update(accepted) + _layer_overrides_expires_at = time.monotonic() + ttl if accepted else 0.0 + bump_active_layers_version() + return accepted + + +def clear_layer_overrides() -> None: + """Drop all overrides, restoring the operator's own layer state.""" + global _layer_overrides_expires_at + if layer_overrides: + layer_overrides.clear() + _layer_overrides_expires_at = 0.0 + bump_active_layers_version() + + +def effective_layers() -> dict[str, bool]: + """Operator layer state merged with any live overrides.""" + overrides = get_layer_overrides() + return {**active_layers, **overrides} if overrides else dict(active_layers) + + def is_any_active(*layer_names: str) -> bool: """Return True if any of the given layer names is currently active.""" - return any(active_layers.get(name, True) for name in layer_names) + overrides = get_layer_overrides() + return any( + overrides.get(name, active_layers.get(name, True)) + for name in layer_names + ) diff --git a/backend/tests/test_layer_overrides.py b/backend/tests/test_layer_overrides.py new file mode 100644 index 0000000..18a69d4 --- /dev/null +++ b/backend/tests/test_layer_overrides.py @@ -0,0 +1,235 @@ +"""Tests for daemon-driven native layer overrides. + +Overrides are an additive map that merges on top of the operator's +``active_layers`` state. They gate fetchers via ``is_any_active`` and are +merged into the four ``data.py`` serving paths, but they never enter +``active_layers`` itself and are never persisted by the frontend. +""" +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from services.fetchers._store import ( + active_layers, + clear_layer_overrides, + effective_layers, + get_layer_overrides, + is_any_active, + set_layer_overrides, +) + + +@pytest.fixture(autouse=True) +def _restore_layer_state(): + """Overrides and operator state must not leak between tests.""" + before = dict(active_layers) + clear_layer_overrides() + yield + clear_layer_overrides() + active_layers.clear() + active_layers.update(before) + + +class TestStoreAccessors: + def test_override_activates_layer_the_operator_turned_off(self): + active_layers["military"] = False + assert is_any_active("military") is False + + set_layer_overrides({"military": True}, 120) + + assert is_any_active("military") is True + assert get_layer_overrides() == {"military": True} + + def test_override_does_not_mutate_operator_state(self): + active_layers["military"] = False + set_layer_overrides({"military": True}, 120) + + assert active_layers["military"] is False + assert effective_layers()["military"] is True + + def test_override_expires_after_ttl(self): + import time as _time + + active_layers["military"] = False + set_layer_overrides({"military": True}, 120) + assert is_any_active("military") is True + + later = _time.monotonic() + 121 + with patch("services.fetchers._store.time.monotonic", return_value=later): + assert get_layer_overrides() == {} + assert is_any_active("military") is False + + # Expiry is sticky — the map was cleared, not merely hidden. + assert get_layer_overrides() == {} + + def test_clear_restores_operator_view(self): + active_layers["military"] = False + set_layer_overrides({"military": True}, 120) + assert is_any_active("military") is True + + clear_layer_overrides() + + assert get_layer_overrides() == {} + assert is_any_active("military") is False + + def test_unknown_keys_are_rejected(self): + accepted = set_layer_overrides({"military": True, "not_a_layer": True}, 120) + + assert accepted == {"military": True} + assert "not_a_layer" not in get_layer_overrides() + assert "not_a_layer" not in effective_layers() + + def test_ttl_is_capped(self): + active_layers["military"] = False + set_layer_overrides({"military": True}, 10_000_000) + + import time as _time + + later = _time.monotonic() + 3601 + with patch("services.fetchers._store.time.monotonic", return_value=later): + assert get_layer_overrides() == {} + + def test_empty_override_map_is_a_no_op(self): + assert get_layer_overrides() == {} + assert effective_layers() == dict(active_layers) + + def test_setting_bumps_active_layers_version(self): + from services.fetchers._store import get_active_layers_version + + before = get_active_layers_version() + set_layer_overrides({"military": True}, 120) + assert get_active_layers_version() > before + + mid = get_active_layers_version() + clear_layer_overrides() + assert get_active_layers_version() > mid + + +class TestServingPaths: + """The audit finding: ~70 payload filters read the local ``active_layers``. + + Gating the fetcher is not enough — the response builder must see the + merged map too, or the payload is stripped on the way out. + """ + + def test_bootstrap_critical_includes_overridden_layer(self, client): + active_layers["military"] = False + with patch( + "services.fetchers._store.get_latest_data_subset_refs", + side_effect=lambda *keys: {k: ([{"id": "x"}] if k == "military_flights" else None) for k in keys}, + ): + off = client.get("/api/bootstrap/critical").json() + assert off["military_flights"] == [] + + set_layer_overrides({"military": True}, 120) + on = client.get("/api/bootstrap/critical").json() + + assert on["military_flights"] == [{"id": "x"}] + + def test_fast_path_includes_overridden_layer(self, client): + active_layers["military"] = False + with patch( + "services.fetchers._store.get_latest_data_subset_refs", + side_effect=lambda *keys: {k: ([{"id": "x"}] if k == "military_flights" else None) for k in keys}, + ): + off = client.get("/api/live-data/fast").json() + assert off["military_flights"] == [] + + set_layer_overrides({"military": True}, 120) + on = client.get("/api/live-data/fast").json() + + assert on["military_flights"] == [{"id": "x"}] + + def test_slow_path_includes_overridden_layer(self, client): + active_layers["volcanoes"] = False + with patch( + "services.fetchers._store.get_latest_data_subset_refs", + side_effect=lambda *keys: {k: ([{"id": "v"}] if k == "volcanoes" else None) for k in keys}, + ): + off = client.get("/api/live-data/slow").json() + assert off["volcanoes"] == [] + + set_layer_overrides({"volcanoes": True}, 120) + on = client.get("/api/live-data/slow").json() + + assert on["volcanoes"] == [{"id": "v"}] + + def test_no_overrides_leaves_payload_unchanged(self, client): + active_layers["military"] = True + with patch( + "services.fetchers._store.get_latest_data_subset_refs", + side_effect=lambda *keys: {k: ([{"id": "x"}] if k == "military_flights" else None) for k in keys}, + ): + baseline = client.get("/api/bootstrap/critical").json() + clear_layer_overrides() + again = client.get("/api/bootstrap/critical").json() + + assert baseline == again + + def test_write_path_never_sees_overrides(self, client): + """POST /api/layers writes operator state; overrides must not bleed in.""" + active_layers["military"] = False + set_layer_overrides({"military": True}, 120) + + r = client.post("/api/layers", json={"layers": {"cctv": True}}) + assert r.status_code == 200 + + assert active_layers["military"] is False + assert active_layers["cctv"] is True + + +class TestRoutes: + def test_get_layers_reports_state_and_overrides(self, client): + r = client.get("/api/layers") + assert r.status_code == 200 + body = r.json() + assert body["overrides"] == {} + assert body["layers"]["military"] == active_layers["military"] + + set_layer_overrides({"military": True}, 120) + assert client.get("/api/layers").json()["overrides"] == {"military": True} + + def test_put_override_route(self, client): + r = client.put( + "/api/ai/layer-overrides", + json={"layers": {"military": True}, "ttl_seconds": 120}, + ) + assert r.status_code == 200 + body = r.json() + assert body["ok"] is True + assert body["overrides"] == {"military": True} + assert body["ignored"] == [] + assert get_layer_overrides() == {"military": True} + + def test_put_reports_ignored_keys(self, client): + r = client.put( + "/api/ai/layer-overrides", + json={"layers": {"military": True, "not_a_layer": True}, "ttl_seconds": 120}, + ) + assert r.status_code == 200 + body = r.json() + assert body["ignored"] == ["not_a_layer"] + assert body["overrides"] == {"military": True} + + def test_get_override_route(self, client): + set_layer_overrides({"military": True}, 120) + r = client.get("/api/ai/layer-overrides") + assert r.status_code == 200 + assert r.json() == {"ok": True, "overrides": {"military": True}} + + def test_delete_override_route(self, client): + set_layer_overrides({"military": True}, 120) + r = client.delete("/api/ai/layer-overrides") + assert r.status_code == 200 + assert r.json()["ok"] is True + assert get_layer_overrides() == {} + + def test_unsigned_remote_put_is_rejected(self, remote_client): + r = remote_client.put( + "/api/ai/layer-overrides", + json={"layers": {"military": True}, "ttl_seconds": 120}, + ) + assert r.status_code == 403 + assert get_layer_overrides() == {} diff --git a/frontend/src/app/page.tsx b/frontend/src/app/page.tsx index cbaf31e..95ed68a 100644 --- a/frontend/src/app/page.tsx +++ b/frontend/src/app/page.tsx @@ -191,6 +191,9 @@ export default function Dashboard() { }, []); const [activeLayers, setActiveLayers] = useState(getDefaultActiveLayers); + // Backend-driven layer overrides. Additive on top of activeLayers, never + // persisted and never pushed back — the operator's own toggles stay theirs. + const [layerOverrides, setLayerOverrides] = useState>({}); const [activeStyle, setActiveStyle] = useState(getDefaultMapStyle); const [activeFilters, setActiveFilters] = useState>({}); const [layerPrefsHydrated, setLayerPrefsHydrated] = useState(false); @@ -323,6 +326,27 @@ export default function Dashboard() { }; }, [activeLayers, secondaryBootReady, layerPrefsHydrated]); + // Poll for backend layer overrides so an agent can switch an overlay on + // without the operator reloading. Overrides carry a TTL server-side, so a + // missed poll self-corrects and a dropped backend just lets them lapse. + useEffect(() => { + if (!secondaryBootReady) return; + let cancelled = false; + const pollOverrides = () => + fetch(`${API_BASE}/api/layers`) + .then((r) => r.json()) + .then((d) => { + if (!cancelled) setLayerOverrides(d?.overrides ?? {}); + }) + .catch(() => {}); + void pollOverrides(); + const id = setInterval(pollOverrides, 5000); + return () => { + cancelled = true; + clearInterval(id); + }; + }, [secondaryBootReady]); + // Left panel accordion state const [leftDataMinimized, setLeftDataMinimized] = useState(false); const [leftMeshExpanded, setLeftMeshExpanded] = useState(true); @@ -457,8 +481,10 @@ export default function Dashboard() { }); }; + // Overrides merge last so they win over both the operator's toggles and the + // first-paint suppressions below. const firstPaintActiveLayers = useMemo(() => { - if (secondaryBootReady) return activeLayers; + if (secondaryBootReady) return { ...activeLayers, ...layerOverrides }; return { ...activeLayers, cctv: false, @@ -472,8 +498,9 @@ export default function Dashboard() { tinygs: false, datacenters: false, power_plants: false, + ...layerOverrides, }; - }, [activeLayers, secondaryBootReady]); + }, [activeLayers, layerOverrides, secondaryBootReady]); // Agent fly_to handler (sar_focus_aoi etc.) — wired here now that // setFlyToLocation is in scope. show_image is routed through // useAgentActions at the top of Dashboard. @@ -587,7 +614,7 @@ export default function Dashboard() { {secondaryBootReady ? (