diff --git a/.env.example b/.env.example index ad72720..e925239 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,10 @@ OPENSKY_CLIENT_ID= OPENSKY_CLIENT_SECRET= AIS_API_KEY= +# Optional AISHub REST backup when AISStream is silent/offline (same ships layer, ~20 min cadence). +# Free registration at https://www.aishub.net/api — paste the account username (not a password). +# AISHUB_USERNAME= + # Global Fishing Watch — fishing vessel activity events (Fishing Activity map layer). # Free API token from https://globalfishingwatch.org/our-apis/tokens # Without this the fishing_activity layer stays empty. diff --git a/backend/main.py b/backend/main.py index 320e934..de567ff 100644 --- a/backend/main.py +++ b/backend/main.py @@ -168,6 +168,7 @@ def _gate_privileged_access_status_snapshot_local() -> dict[str, Any]: # --------------------------------------------------------------------------- _SECRET_VARS = [ "AIS_API_KEY", + "AISHUB_USERNAME", "OPENSKY_CLIENT_ID", "OPENSKY_CLIENT_SECRET", "LTA_ACCOUNT_KEY", diff --git a/backend/routers/health.py b/backend/routers/health.py index f610270..ac5076a 100644 --- a/backend/routers/health.py +++ b/backend/routers/health.py @@ -111,6 +111,12 @@ async def health_check(request: Request): ais_status = ais_proxy_status() or {} except Exception: ais_status = {} + try: + from services.fetchers.aishub_fallback import aishub_fallback_enabled + + ais_status["aishub_configured"] = bool(aishub_fallback_enabled()) + except Exception: + ais_status["aishub_configured"] = bool(str(os.environ.get("AISHUB_USERNAME", "") or "").strip()) if ais_status.get("degraded_tls") and top_status == "ok": # Don't override a worse top-level status if SLOs already failed, # but escalate ok -> degraded so the field surfaces in dashboards. diff --git a/backend/services/ai_intel_store.py b/backend/services/ai_intel_store.py index 48c19f5..c7fda01 100644 --- a/backend/services/ai_intel_store.py +++ b/backend/services/ai_intel_store.py @@ -137,15 +137,16 @@ def inject_layer_data( tagged.append(entry) with _data_lock: - existing = latest_data.get(layer) - if not isinstance(existing, list): - existing = [] + current = latest_data.get(layer) + existing = list(current) if isinstance(current, list) else [] if mode == "replace": existing = [e for e in existing if not e.get("_injected")] - existing.extend(tagged) - latest_data[layer] = existing + # Readers can hold references to published layer lists after releasing + # _data_lock. Build a fresh list and swap it atomically rather than + # mutating the published object in place with list.extend(). + latest_data[layer] = [*existing, *tagged] bump_data_version() diff --git a/backend/services/api_settings.py b/backend/services/api_settings.py index 30cfe8f..2a29e87 100644 --- a/backend/services/api_settings.py +++ b/backend/services/api_settings.py @@ -56,6 +56,15 @@ API_REGISTRY = [ "url": "https://aisstream.io/", "required": True, }, + { + "id": "aishub_username", + "env_key": "AISHUB_USERNAME", + "name": "AISHub Username (backup)", + "description": "Free AISHub account username used as a slow REST backup when AISStream is silent or offline. Does not replace live AIS — polls about every 20 minutes into the same ships layer. Register at aishub.net/api.", + "category": "Maritime", + "url": "https://www.aishub.net/api", + "required": False, + }, { "id": "gfw_api_token", "env_key": "GFW_API_TOKEN", diff --git a/backend/services/config.py b/backend/services/config.py index 3b09ff7..d22a910 100644 --- a/backend/services/config.py +++ b/backend/services/config.py @@ -16,6 +16,7 @@ class Settings(BaseSettings): # Data sources AIS_API_KEY: str = "" + AISHUB_USERNAME: str = "" # Optional AISHub REST backup when AISStream is silent OPENSKY_CLIENT_ID: str = "" OPENSKY_CLIENT_SECRET: str = "" LTA_ACCOUNT_KEY: str = "" diff --git a/backend/services/env_check.py b/backend/services/env_check.py index d7e4fbb..5eaad70 100644 --- a/backend/services/env_check.py +++ b/backend/services/env_check.py @@ -46,6 +46,7 @@ _CRITICAL_WARN = { _OPTIONAL = { "AIS_API_KEY": "AIS vessel streaming (ships layer will be empty without it)", + "AISHUB_USERNAME": "AISHub REST backup when AISStream is silent (optional; free at aishub.net/api)", "GFW_API_TOKEN": "Global Fishing Watch fishing-vessel activity (fishing_activity layer)", "LTA_ACCOUNT_KEY": "Singapore LTA traffic cameras (CCTV layer)", "PUBLIC_API_KEY": "Optional client auth for public endpoints (recommended for exposed deployments)", diff --git a/backend/tests/test_ai_intel_store_copy_on_write.py b/backend/tests/test_ai_intel_store_copy_on_write.py new file mode 100644 index 0000000..8c828e6 --- /dev/null +++ b/backend/tests/test_ai_intel_store_copy_on_write.py @@ -0,0 +1,59 @@ +"""Regression coverage for copy-on-write OpenClaw layer injection.""" + +from services import ai_intel_store +from services.fetchers import _store + + +def _publish_test_layer(monkeypatch, items): + published = list(items) + monkeypatch.setitem(_store.latest_data, "air_quality", published) + monkeypatch.setattr(_store, "bump_data_version", lambda: None) + return published + + +def test_append_does_not_mutate_previously_published_list(monkeypatch): + before = _publish_test_layer(monkeypatch, [{"id": "existing"}]) + + result = ai_intel_store.inject_layer_data( + "air_quality", + [{"id": "injected"}], + mode="append", + ) + + after = _store.latest_data["air_quality"] + assert result == { + "ok": True, + "layer": "air_quality", + "injected": 1, + "mode": "append", + } + assert after is not before + assert before == [{"id": "existing"}] + assert [item["id"] for item in after] == ["existing", "injected"] + assert after[-1]["_injected"] is True + assert after[-1]["_source"] == "user:openclaw" + + +def test_replace_does_not_mutate_previously_published_list(monkeypatch): + before = _publish_test_layer( + monkeypatch, + [ + {"id": "native"}, + {"id": "old-injected", "_injected": True}, + ], + ) + + result = ai_intel_store.inject_layer_data( + "air_quality", + [{"id": "new-injected"}], + mode="replace", + ) + + after = _store.latest_data["air_quality"] + assert result["ok"] is True + assert after is not before + assert before == [ + {"id": "native"}, + {"id": "old-injected", "_injected": True}, + ] + assert [item["id"] for item in after] == ["native", "new-injected"] diff --git a/backend/tests/test_ais_upstream_health.py b/backend/tests/test_ais_upstream_health.py index de7ee65..9001726 100644 --- a/backend/tests/test_ais_upstream_health.py +++ b/backend/tests/test_ais_upstream_health.py @@ -130,6 +130,7 @@ class TestHealthEndpointEscalation: body = res.json() assert body["ais_proxy"]["connected"] is False assert body["ais_proxy"]["proxy_spawn_count"] == 5 + assert "aishub_configured" in body["ais_proxy"] # Without API_KEY this would stay "ok"; with it set + connected=false, # we expect at least "degraded" (could be "error" if an SLO is also # red, but never "ok"). @@ -138,6 +139,18 @@ class TestHealthEndpointEscalation: f"got {body['status']!r}" ) + def test_health_reports_aishub_configured_flag(self, client, monkeypatch): + _reset_ais_module() + monkeypatch.setenv("AISHUB_USERNAME", "shadowbroker-test") + res = client.get("/api/health") + assert res.status_code == 200 + assert res.json()["ais_proxy"]["aishub_configured"] is True + + monkeypatch.delenv("AISHUB_USERNAME", raising=False) + res = client.get("/api/health") + assert res.status_code == 200 + assert res.json()["ais_proxy"]["aishub_configured"] is False + def test_no_api_key_does_not_escalate(self, client, monkeypatch): """When AIS_API_KEY isn't set, the operator hasn't opted in. Don't flag the system as degraded just because AIS isn't running — that's diff --git a/backend/tests/test_aishub_api_settings.py b/backend/tests/test_aishub_api_settings.py new file mode 100644 index 0000000..cc651c0 --- /dev/null +++ b/backend/tests/test_aishub_api_settings.py @@ -0,0 +1,9 @@ +from services.api_settings import ALLOWED_ENV_KEYS, API_REGISTRY + + +def test_aishub_username_is_in_api_registry(): + entry = next((item for item in API_REGISTRY if item.get("env_key") == "AISHUB_USERNAME"), None) + assert entry is not None + assert entry["category"] == "Maritime" + assert entry["required"] is False + assert "AISHUB_USERNAME" in ALLOWED_ENV_KEYS diff --git a/backend/tests/test_regen_duplicate_routes_baseline.py b/backend/tests/test_regen_duplicate_routes_baseline.py new file mode 100644 index 0000000..3494140 --- /dev/null +++ b/backend/tests/test_regen_duplicate_routes_baseline.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import json +from types import SimpleNamespace + +from scripts.regen_duplicate_routes_baseline import ( + build_baseline_payload, + collect_duplicate_routes, + write_baseline, +) + + +def _route(path: str, methods: set[str], module: str): + def endpoint(): + return None + + endpoint.__module__ = module + return SimpleNamespace(path=path, methods=methods, endpoint=endpoint) + + +def test_collect_duplicate_routes_matches_ci_guard_shape(): + routes = [ + _route("/api/layers", {"POST"}, "routers.data"), + _route("/api/layers", {"POST"}, "main"), + _route("/api/health", {"GET", "HEAD"}, "routers.health"), + _route("/api/health", {"GET", "HEAD"}, "main"), + _route("/api/only-once", {"GET"}, "routers.example"), + SimpleNamespace(path=None, methods={"GET"}, endpoint=lambda: None), + ] + + assert collect_duplicate_routes(routes) == { + "GET /api/health": ["main", "routers.health"], + "POST /api/layers": ["main", "routers.data"], + } + + +def test_build_baseline_payload_is_deterministic(): + payload = build_baseline_payload( + { + "POST /z": ["routers.z", "main"], + "GET /a": ["routers.a", "main"], + } + ) + + assert list(payload["duplicates"]) == ["GET /a", "POST /z"] + assert payload["duplicates"]["GET /a"] == ["main", "routers.a"] + assert payload["_meta"]["issue"] == "#239" + assert payload["_meta"]["generated_with"] == ( + "python -m scripts.regen_duplicate_routes_baseline" + ) + + +def test_write_baseline_emits_stable_json(tmp_path): + output = tmp_path / "duplicate_routes_baseline.json" + duplicates = {"POST /api/layers": ["routers.data", "main"]} + + payload = write_baseline(output, duplicates=duplicates) + + assert json.loads(output.read_text(encoding="utf-8")) == payload + assert output.read_text(encoding="utf-8").endswith("\n") + assert payload["duplicates"] == { + "POST /api/layers": ["main", "routers.data"] + } diff --git a/frontend/src/app/page.tsx b/frontend/src/app/page.tsx index 3d0506d..dca49ae 100644 --- a/frontend/src/app/page.tsx +++ b/frontend/src/app/page.tsx @@ -994,7 +994,7 @@ export default function Dashboard() { {/* AIS UPSTREAM OUTAGE BANNER — renders only when AIS is configured but the WebSocket upstream is unreachable. Tells users the empty ocean isn't their fault. */} - + setSettingsOpen(true)} /> {/* ONBOARDING MODAL */} {showOnboarding && ( diff --git a/frontend/src/components/AisUpstreamBanner.tsx b/frontend/src/components/AisUpstreamBanner.tsx index 883c045..30415ad 100644 --- a/frontend/src/components/AisUpstreamBanner.tsx +++ b/frontend/src/components/AisUpstreamBanner.tsx @@ -1,15 +1,18 @@ /** - * AisUpstreamBanner — visible notice that AIS ship data is unavailable - * because the upstream provider (AISStream) is offline. + * AisUpstreamBanner — visible notice that AISStream ship data is unavailable. * * Renders nothing when AIS is healthy or when AIS isn't configured at all. - * Mounted at the app shell level so users see it before they wonder why - * the ocean looks empty. + * When AISStream is silent, nudge operators toward the existing AISHub REST + * backup (Settings → API Keys) or confirm that backup is already active. */ import { useState } from 'react'; import { useAisUpstreamHealth } from '@/hooks/useAisUpstreamHealth'; -export function AisUpstreamBanner() { +type AisUpstreamBannerProps = { + onOpenApiKeys?: () => void; +}; + +export function AisUpstreamBanner({ onOpenApiKeys }: AisUpstreamBannerProps = {}) { const health = useAisUpstreamHealth(); const [dismissed, setDismissed] = useState(false); @@ -29,6 +32,10 @@ export function AisUpstreamBanner() { } } + const detail = health.aishubConfigured + ? `AISStream is silent (${stalenessLabel}). AISHub backup polling stays active on a slower cadence (~20 min) — live WebSocket traffic will resume when AISStream recovers.` + : `AISStream is silent (${stalenessLabel}). Add a free AISHub username under Settings → API Keys → Maritime for slow backup ship coverage while AISStream is down.`; + return (
⚠
-
Ship data temporarily unavailable
-
- AISStream upstream is offline ({stalenessLabel}). The map will - refill once their service comes back online — nothing is wrong - with your install. +
+ {health.aishubConfigured + ? 'Live AIS offline — AISHub backup active' + : 'Ship data temporarily unavailable'}
+
{detail}
+ {!health.aishubConfigured && onOpenApiKeys ? ( + + ) : null}
@@ -599,6 +616,7 @@ const OnboardingModal = React.memo(function OnboardingModal({ ['OPENSKY_CLIENT_ID', 'OpenSky Client ID'], ['OPENSKY_CLIENT_SECRET', 'OpenSky Client Secret'], ['AIS_API_KEY', 'AIS Stream API Key'], + ['AISHUB_USERNAME', 'AISHub Username (optional backup)'], ['GFW_API_TOKEN', 'Global Fishing Watch API Token (optional)'], ].map(([key, label]) => ( 0, + aishubConfigured: Boolean(proxy.aishub_configured), }); } catch { // Backend unreachable — separate problem. Banner not relevant. diff --git a/scripts/regen_duplicate_routes_baseline.py b/scripts/regen_duplicate_routes_baseline.py new file mode 100644 index 0000000..c90963b --- /dev/null +++ b/scripts/regen_duplicate_routes_baseline.py @@ -0,0 +1,106 @@ +"""Regenerate the tolerated duplicate-route baseline used by issue #239 tests. + +Run from the repository root with:: + + python -m scripts.regen_duplicate_routes_baseline + +The command imports the backend application, inspects FastAPI's registered +routes, and rewrites ``backend/tests/data/duplicate_routes_baseline.json`` in a +deterministic order. +""" +from __future__ import annotations + +import json +import sys +from collections import defaultdict +from collections.abc import Iterable +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[1] +BACKEND_DIR = REPO_ROOT / "backend" +DEFAULT_BASELINE_PATH = BACKEND_DIR / "tests" / "data" / "duplicate_routes_baseline.json" + +_BASELINE_NOTE = ( + "Snapshot of currently-tolerated duplicate route registrations. The test in " + "test_no_new_duplicate_routes.py fails if any NEW (method, path) duplicate " + "appears outside this list. Removing entries (by actually deduping) is fine " + "and the test stays green. New entries here require explicit, reviewed updates." +) + + +def collect_duplicate_routes(routes: Iterable[Any]) -> dict[str, list[str]]: + """Return duplicate ``METHOD /path`` registrations and their modules.""" + by_key: dict[str, list[str]] = defaultdict(list) + + for route in routes: + path = getattr(route, "path", None) + methods = getattr(route, "methods", None) + endpoint = getattr(route, "endpoint", None) + if not path or not methods or endpoint is None: + continue + + module = str(getattr(endpoint, "__module__", "") or "") + for method in sorted(methods): + if method in {"HEAD", "OPTIONS"}: + continue + by_key[f"{method} {path}"].append(module) + + return { + key: sorted(modules) + for key, modules in sorted(by_key.items()) + if len(modules) > 1 + } + + +def current_duplicates() -> dict[str, list[str]]: + """Import the backend application and inspect its live route table.""" + backend = str(BACKEND_DIR) + if backend not in sys.path: + sys.path.insert(0, backend) + + import main + + return collect_duplicate_routes(main.app.routes) + + +def build_baseline_payload(duplicates: dict[str, list[str]]) -> dict[str, Any]: + """Build the canonical JSON payload written by the regeneration command.""" + return { + "_meta": { + "issue": "#239", + "note": _BASELINE_NOTE, + "generated_with": "python -m scripts.regen_duplicate_routes_baseline", + }, + "duplicates": { + key: sorted(modules) + for key, modules in sorted(duplicates.items()) + }, + } + + +def write_baseline( + path: Path = DEFAULT_BASELINE_PATH, + *, + duplicates: dict[str, list[str]] | None = None, +) -> dict[str, Any]: + """Write the baseline and return the payload for callers and tests.""" + payload = build_baseline_payload( + current_duplicates() if duplicates is None else duplicates + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + return payload + + +def main() -> int: + payload = write_baseline() + print( + f"Wrote {len(payload['duplicates'])} duplicate route entries to " + f"{DEFAULT_BASELINE_PATH.relative_to(REPO_ROOT)}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())