release: prepare v0.9.7

backend/routers/admin.py

@@ -0,0 +1,277 @@
+import json as json_mod
+import logging
+import os
+import threading
+from pathlib import Path
+from typing import Any
+from fastapi import APIRouter, Request, Depends, Response
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+from node_state import (
+    _current_node_mode,
+    _participant_node_enabled,
+    _refresh_node_peer_store,
+    _set_participant_node_enabled,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+class NodeSettingsUpdate(BaseModel):
+    enabled: bool
+
+
+class TimeMachineToggle(BaseModel):
+    enabled: bool
+
+
+@router.get("/api/settings/api-keys", dependencies=[Depends(require_admin)])
+@limiter.limit("30/minute")
+async def api_get_keys(request: Request):
+    from services.api_settings import get_api_keys
+    return get_api_keys()
+
+
+@router.get("/api/settings/api-keys/meta")
+@limiter.limit("30/minute")
+async def api_get_keys_meta(request: Request):
+    """Return absolute paths for the backend .env and .env.example template.
+
+    Not gated behind admin auth: the paths are not sensitive, and the frontend
+    needs them to render the API Keys panel banner before the user has had a
+    chance to enter an admin key. Helps users find the file when in-app editing
+    is blocked or when the backend is read-only.
+    """
+    from services.api_settings import get_env_path_info
+    return get_env_path_info()
+
+
+@router.get("/api/settings/news-feeds")
+@limiter.limit("30/minute")
+async def api_get_news_feeds(request: Request):
+    from services.news_feed_config import get_feeds
+    return get_feeds()
+
+
+@router.put("/api/settings/news-feeds", dependencies=[Depends(require_admin)])
+@limiter.limit("10/minute")
+async def api_save_news_feeds(request: Request):
+    from services.news_feed_config import save_feeds
+    body = await request.json()
+    ok = save_feeds(body)
+    if ok:
+        return {"status": "updated", "count": len(body)}
+    return Response(
+        content=json_mod.dumps({"status": "error",
+            "message": "Validation failed (max 20 feeds, each needs name/url/weight 1-5)"}),
+        status_code=400,
+        media_type="application/json",
+    )
+
+
+@router.post("/api/settings/news-feeds/reset", dependencies=[Depends(require_admin)])
+@limiter.limit("10/minute")
+async def api_reset_news_feeds(request: Request):
+    from services.news_feed_config import get_feeds, reset_feeds
+    ok = reset_feeds()
+    if ok:
+        return {"status": "reset", "feeds": get_feeds()}
+    return {"status": "error", "message": "Failed to reset feeds"}
+
+
+@router.get("/api/settings/node")
+@limiter.limit("30/minute")
+async def api_get_node_settings(request: Request):
+    import asyncio
+    from services.node_settings import read_node_settings
+    data = await asyncio.to_thread(read_node_settings)
+    return {
+        **data,
+        "node_mode": _current_node_mode(),
+        "node_enabled": _participant_node_enabled(),
+    }
+
+
+@router.put("/api/settings/node", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_set_node_settings(request: Request, body: NodeSettingsUpdate):
+    _refresh_node_peer_store()
+    return _set_participant_node_enabled(bool(body.enabled))
+
+
+@router.get("/api/settings/timemachine")
+@limiter.limit("30/minute")
+async def api_get_timemachine_settings(request: Request):
+    import asyncio
+    from services.node_settings import read_node_settings
+    data = await asyncio.to_thread(read_node_settings)
+    return {
+        "enabled": data.get("timemachine_enabled", False),
+        "storage_warning": "Time Machine auto-snapshots use ~68 MB/day compressed (~2 GB/month). "
+                           "Snapshots capture entity positions (flights, ships, satellites) for historical playback.",
+    }
+
+
+@router.put("/api/settings/timemachine", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_set_timemachine_settings(request: Request, body: TimeMachineToggle):
+    import asyncio
+    from services.node_settings import write_node_settings
+    result = await asyncio.to_thread(write_node_settings, timemachine_enabled=body.enabled)
+    return {
+        "ok": True,
+        "enabled": result.get("timemachine_enabled", False),
+    }
+
+
+@router.post("/api/system/update", dependencies=[Depends(require_admin)])
+@limiter.limit("1/minute")
+async def system_update(request: Request):
+    """Download latest release, backup current files, extract update, and restart."""
+    from services.updater import perform_update, schedule_restart
+    candidate = Path(__file__).resolve().parent.parent.parent
+    if (candidate / "frontend").is_dir() or (candidate / "backend").is_dir():
+        project_root = str(candidate)
+    else:
+        project_root = os.getcwd()
+    result = perform_update(project_root)
+    if result.get("status") == "error":
+        return Response(content=json_mod.dumps(result), status_code=500, media_type="application/json")
+    if result.get("status") == "docker":
+        return result
+    threading.Timer(2.0, schedule_restart, args=[project_root]).start()
+    return result
+
+
+# ── Tor Hidden Service ──────────────────────────────────────────────
+
+
+@router.get("/api/settings/tor", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_tor_status(request: Request):
+    """Return Tor hidden service status and .onion address if available."""
+    import asyncio
+    from services.tor_hidden_service import tor_service
+
+    return await asyncio.to_thread(tor_service.status)
+
+
+@router.post("/api/settings/tor/start", dependencies=[Depends(require_local_operator)])
+@limiter.limit("5/minute")
+async def api_tor_start(request: Request):
+    """Start Tor and provision a hidden service for this ShadowBroker instance.
+
+    Also enables MESH_ARTI so the mesh/wormhole system can route traffic
+    through the Tor SOCKS proxy (port 9050) automatically.
+    """
+    import asyncio
+    from services.tor_hidden_service import tor_service
+
+    result = await asyncio.to_thread(tor_service.start)
+
+    # If Tor started successfully, enable Arti (Tor SOCKS proxy for mesh)
+    if result.get("ok"):
+        try:
+            from routers.ai_intel import _write_env_value
+            from services.config import get_settings
+            _write_env_value("MESH_ARTI_ENABLED", "true")
+            get_settings.cache_clear()
+        except Exception:
+            pass  # Non-fatal — hidden service still works without mesh Arti
+
+    return result
+
+
+@router.post("/api/settings/tor/reset-identity", dependencies=[Depends(require_local_operator)])
+@limiter.limit("2/minute")
+async def api_tor_reset_identity(request: Request):
+    """Destroy current .onion identity and generate a fresh one on next start.
+
+    This is irreversible — the old .onion address is permanently lost.
+    """
+    import asyncio, shutil
+    from services.tor_hidden_service import tor_service, TOR_DIR
+
+    # Stop Tor if running
+    await asyncio.to_thread(tor_service.stop)
+
+    # Delete the hidden service directory (contains the private key)
+    hs_dir = TOR_DIR / "hidden_service"
+    if hs_dir.exists():
+        shutil.rmtree(str(hs_dir), ignore_errors=True)
+
+    # Clear cached address
+    tor_service._onion_address = ""
+
+    return {"ok": True, "detail": "Tor identity destroyed. A new .onion will be generated on next start."}
+
+
+@router.post("/api/settings/agent/reset-all", dependencies=[Depends(require_local_operator)])
+@limiter.limit("2/minute")
+async def api_reset_all_agent_credentials(request: Request):
+    """Nuclear reset: regenerate HMAC key, destroy .onion, revoke agent identity.
+
+    After this, the agent is fully disconnected and needs new credentials.
+    """
+    import asyncio, secrets, shutil
+    from services.tor_hidden_service import tor_service, TOR_DIR
+    from services.config import get_settings
+
+    results = {}
+
+    # 1. Regenerate HMAC key
+    new_secret = secrets.token_hex(24)
+    from routers.ai_intel import _write_env_value
+    _write_env_value("OPENCLAW_HMAC_SECRET", new_secret)
+    results["hmac"] = "regenerated"
+
+    # 2. Revoke agent identity (Ed25519 keypair)
+    try:
+        from services.openclaw_bridge import revoke_agent_identity
+        revoke_agent_identity()
+        results["identity"] = "revoked"
+    except Exception as e:
+        results["identity"] = f"error: {e}"
+
+    # 3. Destroy .onion and restart Tor with new identity
+    await asyncio.to_thread(tor_service.stop)
+    hs_dir = TOR_DIR / "hidden_service"
+    if hs_dir.exists():
+        shutil.rmtree(str(hs_dir), ignore_errors=True)
+    tor_service._onion_address = ""
+    results["tor"] = "identity destroyed"
+
+    # 4. Bootstrap fresh identity + start Tor with new .onion
+    try:
+        from services.openclaw_bridge import generate_agent_keypair
+        keypair = generate_agent_keypair(force=True)
+        results["new_node_id"] = keypair.get("node_id", "")
+    except Exception as e:
+        results["new_node_id"] = f"error: {e}"
+
+    tor_result = await asyncio.to_thread(tor_service.start)
+    results["new_onion"] = tor_result.get("onion_address", "")
+    results["tor_ok"] = tor_result.get("ok", False)
+
+    # Clear settings cache
+    get_settings.cache_clear()
+
+    return {
+        "ok": True,
+        "new_hmac_secret": new_secret,
+        "detail": "All agent credentials have been reset. Reconfigure your agent with the new credentials.",
+        **results,
+    }
+
+
+@router.post("/api/settings/tor/stop", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_tor_stop(request: Request):
+    """Stop the Tor hidden service."""
+    import asyncio
+    from services.tor_hidden_service import tor_service
+
+    return await asyncio.to_thread(tor_service.stop)

backend/routers/cctv.py

@@ -0,0 +1,259 @@
+import logging
+from dataclasses import dataclass, field
+from fastapi import APIRouter, Request, Query, HTTPException
+from fastapi.responses import StreamingResponse
+from starlette.background import BackgroundTask
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+_CCTV_PROXY_ALLOWED_HOSTS = {
+    "s3-eu-west-1.amazonaws.com",
+    "jamcams.tfl.gov.uk",
+    "images.data.gov.sg",
+    "cctv.austinmobility.io",
+    "webcams.nyctmc.org",
+    "cwwp2.dot.ca.gov",
+    "wzmedia.dot.ca.gov",
+    "images.wsdot.wa.gov",
+    "olypen.com",
+    "flyykm.com",
+    "cam.pangbornairport.com",
+    "navigator-c2c.dot.ga.gov",
+    "navigator-c2c.ga.gov",
+    "navigator-csc.dot.ga.gov",
+    "vss1live.dot.ga.gov",
+    "vss2live.dot.ga.gov",
+    "vss3live.dot.ga.gov",
+    "vss4live.dot.ga.gov",
+    "vss5live.dot.ga.gov",
+    "511ga.org",
+    "gettingaroundillinois.com",
+    "cctv.travelmidwest.com",
+    "mdotjboss.state.mi.us",
+    "micamerasimages.net",
+    "publicstreamer1.cotrip.org",
+    "publicstreamer2.cotrip.org",
+    "publicstreamer3.cotrip.org",
+    "publicstreamer4.cotrip.org",
+    "cocam.carsprogram.org",
+    "tripcheck.com",
+    "www.tripcheck.com",
+    "infocar.dgt.es",
+    "informo.madrid.es",
+    "www.windy.com",
+}
+
+
+@dataclass(frozen=True)
+class _CCTVProxyProfile:
+    name: str
+    timeout: tuple = (5.0, 10.0)
+    cache_seconds: int = 30
+    headers: dict = field(default_factory=dict)
+
+
+def _cctv_host_allowed(hostname) -> bool:
+    host = str(hostname or "").strip().lower()
+    if not host:
+        return False
+    for allowed in _CCTV_PROXY_ALLOWED_HOSTS:
+        normalized = str(allowed or "").strip().lower()
+        if host == normalized or host.endswith(f".{normalized}"):
+            return True
+    return False
+
+
+def _proxied_cctv_url(target_url: str) -> str:
+    from urllib.parse import quote
+    return f"/api/cctv/media?url={quote(target_url, safe='')}"
+
+
+def _cctv_proxy_profile_for_url(target_url: str) -> _CCTVProxyProfile:
+    from urllib.parse import urlparse
+    parsed = urlparse(target_url)
+    host = str(parsed.hostname or "").strip().lower()
+    path = str(parsed.path or "").strip().lower()
+
+    if host in {"jamcams.tfl.gov.uk", "s3-eu-west-1.amazonaws.com"}:
+        return _CCTVProxyProfile(name="tfl-jamcam", timeout=(5.0, 20.0), cache_seconds=15,
+            headers={"Accept": "video/mp4,image/avif,image/webp,image/apng,image/*,*/*;q=0.8", "Referer": "https://tfl.gov.uk/"})
+    if host == "images.data.gov.sg":
+        return _CCTVProxyProfile(name="lta-singapore", timeout=(5.0, 10.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host == "cctv.austinmobility.io":
+        return _CCTVProxyProfile(name="austin-mobility", timeout=(5.0, 8.0), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://data.mobility.austin.gov/", "Origin": "https://data.mobility.austin.gov"})
+    if host == "webcams.nyctmc.org":
+        return _CCTVProxyProfile(name="nyc-dot", timeout=(5.0, 10.0), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host in {"cwwp2.dot.ca.gov", "wzmedia.dot.ca.gov"}:
+        return _CCTVProxyProfile(name="caltrans", timeout=(5.0, 15.0), cache_seconds=15,
+            headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,image/*,*/*;q=0.8",
+                     "Referer": "https://cwwp2.dot.ca.gov/"})
+    if host in {"images.wsdot.wa.gov", "olypen.com", "flyykm.com", "cam.pangbornairport.com"}:
+        return _CCTVProxyProfile(name="wsdot", timeout=(5.0, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host in {"navigator-c2c.dot.ga.gov", "navigator-c2c.ga.gov", "navigator-csc.dot.ga.gov"}:
+        read_timeout = 18.0 if "/snapshots/" in path else 12.0
+        return _CCTVProxyProfile(name="gdot-snapshot", timeout=(5.0, read_timeout), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "http://navigator-c2c.dot.ga.gov/"})
+    if host == "511ga.org":
+        return _CCTVProxyProfile(name="gdot-511ga-image", timeout=(5.0, 12.0), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://511ga.org/cctv"})
+    if host.startswith("vss") and host.endswith("dot.ga.gov"):
+        return _CCTVProxyProfile(name="gdot-hls", timeout=(5.0, 20.0), cache_seconds=10,
+            headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,*/*;q=0.8",
+                     "Referer": "http://navigator-c2c.dot.ga.gov/"})
+    if host in {"gettingaroundillinois.com", "cctv.travelmidwest.com"}:
+        return _CCTVProxyProfile(name="illinois-dot", timeout=(5.0, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host in {"mdotjboss.state.mi.us", "micamerasimages.net"}:
+        return _CCTVProxyProfile(name="michigan-dot", timeout=(5.0, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://mdotjboss.state.mi.us/"})
+    if host in {"publicstreamer1.cotrip.org", "publicstreamer2.cotrip.org",
+                "publicstreamer3.cotrip.org", "publicstreamer4.cotrip.org"}:
+        return _CCTVProxyProfile(name="cotrip-hls", timeout=(5.0, 20.0), cache_seconds=10,
+            headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,*/*;q=0.8",
+                     "Referer": "https://www.cotrip.org/"})
+    if host == "cocam.carsprogram.org":
+        return _CCTVProxyProfile(name="cotrip-preview", timeout=(5.0, 12.0), cache_seconds=20,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.cotrip.org/"})
+    if host in {"tripcheck.com", "www.tripcheck.com"}:
+        return _CCTVProxyProfile(name="odot-tripcheck", timeout=(5.0, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host == "infocar.dgt.es":
+        return _CCTVProxyProfile(name="dgt-spain", timeout=(5.0, 8.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://infocar.dgt.es/"})
+    if host == "informo.madrid.es":
+        return _CCTVProxyProfile(name="madrid-city", timeout=(5.0, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://informo.madrid.es/"})
+    if host == "www.windy.com":
+        return _CCTVProxyProfile(name="windy-webcams", timeout=(5.0, 12.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    return _CCTVProxyProfile(name="generic-cctv", timeout=(5.0, 10.0), cache_seconds=30,
+        headers={"Accept": "*/*"})
+
+
+def _cctv_upstream_headers(request: Request, profile: _CCTVProxyProfile) -> dict:
+    headers = {"User-Agent": "Mozilla/5.0 (compatible; ShadowBroker CCTV proxy)", **profile.headers}
+    range_header = request.headers.get("range")
+    if range_header:
+        headers["Range"] = range_header
+    if_none_match = request.headers.get("if-none-match")
+    if if_none_match:
+        headers["If-None-Match"] = if_none_match
+    if_modified_since = request.headers.get("if-modified-since")
+    if if_modified_since:
+        headers["If-Modified-Since"] = if_modified_since
+    return headers
+
+
+def _cctv_response_headers(resp, cache_seconds: int, include_length: bool = True) -> dict:
+    headers = {"Cache-Control": f"public, max-age={cache_seconds}", "Access-Control-Allow-Origin": "*"}
+    for key in ("Accept-Ranges", "Content-Range", "ETag", "Last-Modified"):
+        value = resp.headers.get(key)
+        if value:
+            headers[key] = value
+    if include_length:
+        content_length = resp.headers.get("Content-Length")
+        if content_length:
+            headers["Content-Length"] = content_length
+    return headers
+
+
+def _fetch_cctv_upstream_response(request: Request, target_url: str, profile: _CCTVProxyProfile):
+    import requests as _req
+    headers = _cctv_upstream_headers(request, profile)
+    try:
+        resp = _req.get(target_url, timeout=profile.timeout, stream=True, allow_redirects=True, headers=headers)
+    except _req.exceptions.Timeout as exc:
+        logger.warning("CCTV upstream timeout [%s] %s", profile.name, target_url)
+        raise HTTPException(status_code=504, detail="Upstream timeout") from exc
+    except _req.exceptions.RequestException as exc:
+        logger.warning("CCTV upstream request failure [%s] %s: %s", profile.name, target_url, exc)
+        raise HTTPException(status_code=502, detail="Upstream fetch failed") from exc
+    if resp.status_code >= 400:
+        logger.info("CCTV upstream HTTP %s [%s] %s", resp.status_code, profile.name, target_url)
+        resp.close()
+        raise HTTPException(status_code=int(resp.status_code), detail=f"Upstream returned {resp.status_code}")
+    return resp
+
+
+def _rewrite_cctv_hls_playlist(base_url: str, body: str) -> str:
+    import re
+    from urllib.parse import urljoin, urlparse
+
+    def _rewrite_target(target: str) -> str:
+        candidate = str(target or "").strip()
+        if not candidate or candidate.startswith("data:"):
+            return candidate
+        absolute = urljoin(base_url, candidate)
+        parsed_target = urlparse(absolute)
+        if parsed_target.scheme not in ("http", "https"):
+            return candidate
+        if not _cctv_host_allowed(parsed_target.hostname):
+            return candidate
+        return _proxied_cctv_url(absolute)
+
+    rewritten_lines: list = []
+    for raw_line in body.splitlines():
+        stripped = raw_line.strip()
+        if not stripped:
+            rewritten_lines.append(raw_line)
+            continue
+        if stripped.startswith("#"):
+            rewritten_lines.append(re.sub(r'URI="([^"]+)"',
+                lambda match: f'URI="{_rewrite_target(match.group(1))}"', raw_line))
+            continue
+        rewritten_lines.append(_rewrite_target(stripped))
+    return "\n".join(rewritten_lines) + ("\n" if body.endswith("\n") else "")
+
+
+def _proxy_cctv_media_response(request: Request, target_url: str):
+    from urllib.parse import urlparse
+    from fastapi.responses import Response
+    parsed = urlparse(target_url)
+    profile = _cctv_proxy_profile_for_url(target_url)
+    resp = _fetch_cctv_upstream_response(request, target_url, profile)
+    content_type = resp.headers.get("Content-Type", "application/octet-stream")
+    is_hls_playlist = (
+        ".m3u8" in str(parsed.path or "").lower()
+        or "mpegurl" in content_type.lower()
+        or "vnd.apple.mpegurl" in content_type.lower()
+    )
+    if is_hls_playlist:
+        body = resp.text
+        if "#EXTM3U" in body:
+            body = _rewrite_cctv_hls_playlist(target_url, body)
+        resp.close()
+        return Response(content=body, media_type=content_type,
+            headers=_cctv_response_headers(resp, cache_seconds=profile.cache_seconds, include_length=False))
+    return StreamingResponse(resp.iter_content(chunk_size=65536), status_code=resp.status_code,
+        media_type=content_type,
+        headers=_cctv_response_headers(resp, cache_seconds=profile.cache_seconds),
+        background=BackgroundTask(resp.close))
+
+
+@router.get("/api/cctv/media")
+@limiter.limit("120/minute")
+async def cctv_media_proxy(request: Request, url: str = Query(...)):
+    """Proxy CCTV media through the backend to bypass browser CORS restrictions."""
+    from urllib.parse import urlparse
+    parsed = urlparse(url)
+    if not _cctv_host_allowed(parsed.hostname):
+        raise HTTPException(status_code=403, detail="Host not allowed")
+    if parsed.scheme not in ("http", "https"):
+        raise HTTPException(status_code=400, detail="Invalid scheme")
+    return _proxy_cctv_media_response(request, url)

backend/routers/data.py

@@ -0,0 +1,469 @@
+import asyncio
+import logging
+import math
+import threading
+from typing import Any
+from fastapi import APIRouter, Request, Response, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+from services.data_fetcher import get_latest_data, update_all_data
+import orjson
+import json as json_mod
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+_refresh_lock = threading.Lock()
+
+
+class ViewportUpdate(BaseModel):
+    s: float
+    w: float
+    n: float
+    e: float
+
+
+class LayerUpdate(BaseModel):
+    layers: dict[str, bool]
+
+
+_LAST_VIEWPORT_UPDATE: tuple | None = None
+_LAST_VIEWPORT_UPDATE_TS = 0.0
+_VIEWPORT_UPDATE_LOCK = threading.Lock()
+_VIEWPORT_DEDUPE_EPSILON = 1.0
+_VIEWPORT_MIN_UPDATE_S = 10.0
+
+
+def _normalize_longitude(value: float) -> float:
+    normalized = ((value + 180.0) % 360.0 + 360.0) % 360.0 - 180.0
+    if normalized == -180.0 and value > 0:
+        return 180.0
+    return normalized
+
+
+def _normalize_viewport_bounds(s: float, w: float, n: float, e: float) -> tuple:
+    south = max(-90.0, min(90.0, s))
+    north = max(-90.0, min(90.0, n))
+    raw_width = abs(e - w)
+    if not math.isfinite(raw_width) or raw_width >= 360.0:
+        return south, -180.0, north, 180.0
+    west = _normalize_longitude(w)
+    east = _normalize_longitude(e)
+    if east < west:
+        return south, -180.0, north, 180.0
+    return south, west, north, east
+
+
+def _viewport_changed_enough(bounds: tuple) -> bool:
+    global _LAST_VIEWPORT_UPDATE, _LAST_VIEWPORT_UPDATE_TS
+    import time
+    now = time.monotonic()
+    with _VIEWPORT_UPDATE_LOCK:
+        if _LAST_VIEWPORT_UPDATE is None:
+            _LAST_VIEWPORT_UPDATE = bounds
+            _LAST_VIEWPORT_UPDATE_TS = now
+            return True
+        changed = any(
+            abs(current - previous) > _VIEWPORT_DEDUPE_EPSILON
+            for current, previous in zip(bounds, _LAST_VIEWPORT_UPDATE)
+        )
+        if not changed and (now - _LAST_VIEWPORT_UPDATE_TS) < _VIEWPORT_MIN_UPDATE_S:
+            return False
+        if (now - _LAST_VIEWPORT_UPDATE_TS) < _VIEWPORT_MIN_UPDATE_S:
+            return False
+        _LAST_VIEWPORT_UPDATE = bounds
+        _LAST_VIEWPORT_UPDATE_TS = now
+        return True
+
+
+def _queue_viirs_change_refresh() -> None:
+    from services.fetchers.earth_observation import fetch_viirs_change_nodes
+    threading.Thread(target=fetch_viirs_change_nodes, daemon=True).start()
+
+
+def _etag_response(request: Request, payload: dict, prefix: str = "", default=None):
+    etag = _current_etag(prefix)
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    content = json_mod.dumps(_json_safe(payload), default=default, allow_nan=False)
+    return Response(content=content, media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"})
+
+
+def _current_etag(prefix: str = "") -> str:
+    from services.fetchers._store import get_active_layers_version, get_data_version
+    return f"{prefix}v{get_data_version()}-l{get_active_layers_version()}"
+
+
+def _json_safe(value):
+    if isinstance(value, float):
+        return value if math.isfinite(value) else None
+    if isinstance(value, dict):
+        return {k: _json_safe(v) for k, v in list(value.items())}
+    if isinstance(value, list):
+        return [_json_safe(v) for v in list(value)]
+    if isinstance(value, tuple):
+        return [_json_safe(v) for v in list(value)]
+    return value
+
+
+def _sanitize_payload(value):
+    if isinstance(value, float):
+        return value if math.isfinite(value) else None
+    if isinstance(value, dict):
+        return {k: _sanitize_payload(v) for k, v in list(value.items())}
+    if isinstance(value, (list, tuple)):
+        return list(value)
+    return value
+
+
+def _bbox_filter(items: list, s: float, w: float, n: float, e: float,
+                 lat_key: str = "lat", lng_key: str = "lng") -> list:
+    pad_lat = (n - s) * 0.2
+    pad_lng = (e - w) * 0.2 if e > w else ((e + 360 - w) * 0.2)
+    s2, n2 = s - pad_lat, n + pad_lat
+    w2, e2 = w - pad_lng, e + pad_lng
+    crosses_antimeridian = w2 > e2
+    out = []
+    for item in items:
+        lat = item.get(lat_key)
+        lng = item.get(lng_key)
+        if lat is None or lng is None:
+            out.append(item)
+            continue
+        if not (s2 <= lat <= n2):
+            continue
+        if crosses_antimeridian:
+            if lng >= w2 or lng <= e2:
+                out.append(item)
+        else:
+            if w2 <= lng <= e2:
+                out.append(item)
+    return out
+
+
+def _bbox_filter_geojson_points(items: list, s: float, w: float, n: float, e: float) -> list:
+    pad_lat = (n - s) * 0.2
+    pad_lng = (e - w) * 0.2 if e > w else ((e + 360 - w) * 0.2)
+    s2, n2 = s - pad_lat, n + pad_lat
+    w2, e2 = w - pad_lng, e + pad_lng
+    crosses_antimeridian = w2 > e2
+    out = []
+    for item in items:
+        geometry = item.get("geometry") if isinstance(item, dict) else None
+        coords = geometry.get("coordinates") if isinstance(geometry, dict) else None
+        if not isinstance(coords, (list, tuple)) or len(coords) < 2:
+            out.append(item)
+            continue
+        lng, lat = coords[0], coords[1]
+        if lat is None or lng is None:
+            out.append(item)
+            continue
+        if not (s2 <= lat <= n2):
+            continue
+        if crosses_antimeridian:
+            if lng >= w2 or lng <= e2:
+                out.append(item)
+        else:
+            if w2 <= lng <= e2:
+                out.append(item)
+    return out
+
+
+def _bbox_spans(s, w, n, e) -> tuple:
+    if None in (s, w, n, e):
+        return 180.0, 360.0
+    lat_span = max(0.0, float(n) - float(s))
+    lng_span = float(e) - float(w)
+    if lng_span < 0:
+        lng_span += 360.0
+    if lng_span == 0 and w == -180 and e == 180:
+        lng_span = 360.0
+    return lat_span, max(0.0, lng_span)
+
+
+def _downsample_points(items: list, max_items: int) -> list:
+    if max_items <= 0 or len(items) <= max_items:
+        return items
+    step = len(items) / float(max_items)
+    return [items[min(len(items) - 1, int(i * step))] for i in range(max_items)]
+
+
+def _world_and_continental_scale(has_bbox: bool, s, w, n, e) -> tuple:
+    lat_span, lng_span = _bbox_spans(s, w, n, e)
+    world_scale = (not has_bbox) or lng_span >= 300 or lat_span >= 120
+    continental_scale = has_bbox and not world_scale and (lng_span >= 120 or lat_span >= 55)
+    return world_scale, continental_scale
+
+
+def _filter_sigint_by_layers(items: list, active_layers: dict) -> list:
+    allow_aprs = bool(active_layers.get("sigint_aprs", True))
+    allow_mesh = bool(active_layers.get("sigint_meshtastic", True))
+    if allow_aprs and allow_mesh:
+        return items
+    allowed_sources: set = {"js8call"}
+    if allow_aprs:
+        allowed_sources.add("aprs")
+    if allow_mesh:
+        allowed_sources.update({"meshtastic", "meshtastic-map"})
+    return [item for item in items if str(item.get("source") or "").lower() in allowed_sources]
+
+
+def _sigint_totals_for_items(items: list) -> dict:
+    totals = {"total": len(items), "meshtastic": 0, "meshtastic_live": 0, "meshtastic_map": 0,
+              "aprs": 0, "js8call": 0}
+    for item in items:
+        source = str(item.get("source") or "").lower()
+        if source == "meshtastic":
+            totals["meshtastic"] += 1
+            if bool(item.get("from_api")):
+                totals["meshtastic_map"] += 1
+            else:
+                totals["meshtastic_live"] += 1
+        elif source == "aprs":
+            totals["aprs"] += 1
+        elif source == "js8call":
+            totals["js8call"] += 1
+    return totals
+
+
+@router.get("/api/refresh", dependencies=[Depends(require_admin)])
+@limiter.limit("2/minute")
+async def force_refresh(request: Request):
+    from services.schemas import RefreshResponse
+    if not _refresh_lock.acquire(blocking=False):
+        return {"status": "refresh already in progress"}
+
+    def _do_refresh():
+        try:
+            update_all_data()
+        finally:
+            _refresh_lock.release()
+
+    t = threading.Thread(target=_do_refresh)
+    t.start()
+    return {"status": "refreshing in background"}
+
+
+@router.post("/api/ais/feed")
+@limiter.limit("60/minute")
+async def ais_feed(request: Request):
+    """Accept AIS-catcher HTTP JSON feed (POST decoded AIS messages)."""
+    from services.ais_stream import ingest_ais_catcher
+    try:
+        body = await request.json()
+    except Exception:
+        return JSONResponse(status_code=422, content={"ok": False, "detail": "invalid JSON body"})
+    msgs = body.get("msgs", [])
+    if not msgs:
+        return {"status": "ok", "ingested": 0}
+    count = ingest_ais_catcher(msgs)
+    return {"status": "ok", "ingested": count}
+
+
+@router.post("/api/viewport")
+@limiter.limit("60/minute")
+async def update_viewport(vp: ViewportUpdate, request: Request):  # noqa: ARG001
+    """Receive frontend map bounds. AIS stream stays global so open-ocean
+    vessels are never dropped — the frontend worker handles viewport culling."""
+    return {"status": "ok"}
+
+
+@router.post("/api/layers")
+@limiter.limit("30/minute")
+async def update_layers(update: LayerUpdate, request: Request):
+    """Receive frontend layer toggle state. Starts/stops streams accordingly."""
+    from services.fetchers._store import active_layers, bump_active_layers_version, is_any_active
+    old_ships = is_any_active("ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts")
+    old_mesh = is_any_active("sigint_meshtastic")
+    old_aprs = is_any_active("sigint_aprs")
+    old_viirs = is_any_active("viirs_nightlights")
+    changed = False
+    for key, value in update.layers.items():
+        if key in active_layers:
+            if active_layers[key] != value:
+                changed = True
+            active_layers[key] = value
+    if changed:
+        bump_active_layers_version()
+    new_ships = is_any_active("ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts")
+    new_mesh = is_any_active("sigint_meshtastic")
+    new_aprs = is_any_active("sigint_aprs")
+    new_viirs = is_any_active("viirs_nightlights")
+    if old_ships and not new_ships:
+        from services.ais_stream import stop_ais_stream
+        stop_ais_stream()
+        logger.info("AIS stream stopped (all ship layers disabled)")
+    elif not old_ships and new_ships:
+        from services.ais_stream import start_ais_stream
+        start_ais_stream()
+        logger.info("AIS stream started (ship layer enabled)")
+    from services.sigint_bridge import sigint_grid
+    if old_mesh and not new_mesh:
+        sigint_grid.mesh.stop()
+        logger.info("Meshtastic MQTT bridge stopped (layer disabled)")
+    elif not old_mesh and new_mesh:
+        sigint_grid.mesh.start()
+        logger.info("Meshtastic MQTT bridge started (layer enabled)")
+    if old_aprs and not new_aprs:
+        sigint_grid.aprs.stop()
+        logger.info("APRS bridge stopped (layer disabled)")
+    elif not old_aprs and new_aprs:
+        sigint_grid.aprs.start()
+        logger.info("APRS bridge started (layer enabled)")
+    if not old_viirs and new_viirs:
+        _queue_viirs_change_refresh()
+        logger.info("VIIRS change refresh queued (layer enabled)")
+    return {"status": "ok"}
+
+
+@router.get("/api/live-data")
+@limiter.limit("120/minute")
+async def live_data(request: Request):
+    return get_latest_data()
+
+
+@router.get("/api/live-data/fast")
+@limiter.limit("120/minute")
+async def live_data_fast(
+    request: Request,
+    s: float = Query(None, description="South bound (ignored)", ge=-90, le=90),
+    w: float = Query(None, description="West bound (ignored)", ge=-180, le=180),
+    n: float = Query(None, description="North bound (ignored)", ge=-90, le=90),
+    e: float = Query(None, description="East bound (ignored)", ge=-180, le=180),
+):
+    etag = _current_etag(prefix="fast|full|")
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    from services.fetchers._store import (active_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot)
+    d = get_latest_data_subset_refs(
+        "last_updated", "commercial_flights", "military_flights", "private_flights",
+        "private_jets", "tracked_flights", "ships", "cctv", "uavs", "liveuamap",
+        "gps_jamming", "satellites", "satellite_source", "satellite_analysis",
+        "sigint", "sigint_totals", "trains",
+    )
+    freshness = get_source_timestamps_snapshot()
+    ships_enabled = any(active_layers.get(key, True) for key in (
+        "ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts"))
+    cctv_total = len(d.get("cctv") or [])
+    sigint_items = _filter_sigint_by_layers(d.get("sigint") or [], active_layers)
+    sigint_totals = _sigint_totals_for_items(sigint_items)
+    payload = {
+        "commercial_flights": (d.get("commercial_flights") or []) if active_layers.get("flights", True) else [],
+        "military_flights": (d.get("military_flights") or []) if active_layers.get("military", True) else [],
+        "private_flights": (d.get("private_flights") or []) if active_layers.get("private", True) else [],
+        "private_jets": (d.get("private_jets") or []) if active_layers.get("jets", True) else [],
+        "tracked_flights": (d.get("tracked_flights") or []) if active_layers.get("tracked", True) else [],
+        "ships": (d.get("ships") or []) if ships_enabled else [],
+        "cctv": (d.get("cctv") or []) if active_layers.get("cctv", True) else [],
+        "uavs": (d.get("uavs") or []) if active_layers.get("military", True) else [],
+        "liveuamap": (d.get("liveuamap") or []) if active_layers.get("global_incidents", True) else [],
+        "gps_jamming": (d.get("gps_jamming") or []) if active_layers.get("gps_jamming", True) else [],
+        "satellites": (d.get("satellites") or []) if active_layers.get("satellites", True) else [],
+        "satellite_source": d.get("satellite_source", "none"),
+        "satellite_analysis": (d.get("satellite_analysis") or {}) if active_layers.get("satellites", True) else {},
+        "sigint": sigint_items if (active_layers.get("sigint_meshtastic", True) or active_layers.get("sigint_aprs", True)) else [],
+        "sigint_totals": sigint_totals,
+        "cctv_total": cctv_total,
+        "trains": (d.get("trains") or []) if active_layers.get("trains", True) else [],
+        "freshness": freshness,
+    }
+    return Response(content=orjson.dumps(_sanitize_payload(payload)), media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"})
+
+
+@router.get("/api/live-data/slow")
+@limiter.limit("60/minute")
+async def live_data_slow(
+    request: Request,
+    s: float = Query(None, description="South bound (ignored)", ge=-90, le=90),
+    w: float = Query(None, description="West bound (ignored)", ge=-180, le=180),
+    n: float = Query(None, description="North bound (ignored)", ge=-90, le=90),
+    e: float = Query(None, description="East bound (ignored)", ge=-180, le=180),
+):
+    etag = _current_etag(prefix="slow|full|")
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    from services.fetchers._store import (active_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot)
+    d = get_latest_data_subset_refs(
+        "last_updated", "news", "stocks", "financial_source", "oil", "weather", "traffic",
+        "earthquakes", "frontlines", "gdelt", "airports", "kiwisdr", "satnogs_stations",
+        "satnogs_observations", "tinygs_satellites", "space_weather", "internet_outages",
+        "firms_fires", "datacenters", "military_bases", "power_plants", "viirs_change_nodes",
+        "scanners", "weather_alerts", "ukraine_alerts", "air_quality", "volcanoes",
+        "fishing_activity", "psk_reporter", "correlations", "uap_sightings", "wastewater",
+        "crowdthreat", "threat_level", "trending_markets",
+    )
+    freshness = get_source_timestamps_snapshot()
+    payload = {
+        "last_updated": d.get("last_updated"),
+        "threat_level": d.get("threat_level"),
+        "trending_markets": d.get("trending_markets", []),
+        "news": d.get("news", []),
+        "stocks": d.get("stocks", {}),
+        "financial_source": d.get("financial_source", ""),
+        "oil": d.get("oil", {}),
+        "weather": d.get("weather"),
+        "traffic": d.get("traffic", []),
+        "earthquakes": (d.get("earthquakes") or []) if active_layers.get("earthquakes", True) else [],
+        "frontlines": d.get("frontlines") if active_layers.get("ukraine_frontline", True) else None,
+        "gdelt": (d.get("gdelt") or []) if active_layers.get("global_incidents", True) else [],
+        "airports": d.get("airports") or [],
+        "kiwisdr": (d.get("kiwisdr") or []) if active_layers.get("kiwisdr", True) else [],
+        "satnogs_stations": (d.get("satnogs_stations") or []) if active_layers.get("satnogs", True) else [],
+        "satnogs_total": len(d.get("satnogs_stations") or []),
+        "satnogs_observations": (d.get("satnogs_observations") or []) if active_layers.get("satnogs", True) else [],
+        "tinygs_satellites": (d.get("tinygs_satellites") or []) if active_layers.get("tinygs", True) else [],
+        "tinygs_total": len(d.get("tinygs_satellites") or []),
+        "psk_reporter": (d.get("psk_reporter") or []) if active_layers.get("psk_reporter", True) else [],
+        "space_weather": d.get("space_weather"),
+        "internet_outages": (d.get("internet_outages") or []) if active_layers.get("internet_outages", True) else [],
+        "firms_fires": (d.get("firms_fires") or []) if active_layers.get("firms", True) else [],
+        "datacenters": (d.get("datacenters") or []) if active_layers.get("datacenters", True) else [],
+        "military_bases": (d.get("military_bases") or []) if active_layers.get("military_bases", True) else [],
+        "power_plants": (d.get("power_plants") or []) if active_layers.get("power_plants", True) else [],
+        "viirs_change_nodes": (d.get("viirs_change_nodes") or []) if active_layers.get("viirs_nightlights", True) else [],
+        "scanners": (d.get("scanners") or []) if active_layers.get("scanners", True) else [],
+        "weather_alerts": d.get("weather_alerts", []) if active_layers.get("weather_alerts", True) else [],
+        "ukraine_alerts": d.get("ukraine_alerts", []) if active_layers.get("ukraine_alerts", True) else [],
+        "air_quality": (d.get("air_quality") or []) if active_layers.get("air_quality", True) else [],
+        "volcanoes": (d.get("volcanoes") or []) if active_layers.get("volcanoes", True) else [],
+        "fishing_activity": (d.get("fishing_activity") or []) if active_layers.get("fishing_activity", True) else [],
+        "correlations": (d.get("correlations") or []) if active_layers.get("correlations", True) else [],
+        "uap_sightings": (d.get("uap_sightings") or []) if active_layers.get("uap_sightings", True) else [],
+        "wastewater": (d.get("wastewater") or []) if active_layers.get("wastewater", True) else [],
+        "crowdthreat": (d.get("crowdthreat") or []) if active_layers.get("crowdthreat", True) else [],
+        "freshness": freshness,
+    }
+    return Response(
+        content=orjson.dumps(_sanitize_payload(payload), default=str, option=orjson.OPT_NON_STR_KEYS),
+        media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"},
+    )
+
+
+# ── Satellite Overflight Counting ───────────────────────────────────────────
+# Counts unique satellites whose ground track entered a bounding box over 24h.
+# Uses cached TLEs + SGP4 propagation — no extra network requests.
+
+class OverflightRequest(BaseModel):
+    s: float
+    w: float
+    n: float
+    e: float
+    hours: int = 24
+
+
+@router.post("/api/satellites/overflights")
+@limiter.limit("10/minute")
+async def satellite_overflights(request: Request, body: OverflightRequest):
+    from services.fetchers.satellites import compute_overflights, _sat_gp_cache
+    gp_data = _sat_gp_cache.get("data")
+    if not gp_data:
+        return JSONResponse({"total": 0, "by_mission": {}, "satellites": [], "error": "No GP data cached yet"})
+    bbox = {"s": body.s, "w": body.w, "n": body.n, "e": body.e}
+    result = compute_overflights(gp_data, bbox, hours=body.hours)
+    return JSONResponse(result)

backend/routers/health.py

@@ -0,0 +1,85 @@
+import time as _time_mod
+from fastapi import APIRouter, Request, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin
+from services.data_fetcher import get_latest_data
+from services.schemas import HealthResponse
+import os
+
+APP_VERSION = os.environ.get("_HEALTH_APP_VERSION", "0.9.7")
+
+router = APIRouter()
+
+
+def _get_app_version() -> str:
+    # Import lazily to avoid circular import; main sets APP_VERSION before including routers
+    try:
+        import main as _main
+        return _main.APP_VERSION
+    except Exception:
+        return APP_VERSION
+
+
+_start_time_ref: dict = {"value": None}
+
+
+def _get_start_time() -> float:
+    if _start_time_ref["value"] is None:
+        try:
+            import main as _main
+            _start_time_ref["value"] = _main._start_time
+        except Exception:
+            _start_time_ref["value"] = _time_mod.time()
+    return _start_time_ref["value"]
+
+
+@router.get("/api/health", response_model=HealthResponse)
+@limiter.limit("30/minute")
+async def health_check(request: Request):
+    from services.fetchers._store import get_source_timestamps_snapshot
+    from services.slo import compute_all_statuses, summarise_statuses
+
+    d = get_latest_data()
+    last = d.get("last_updated")
+    timestamps = get_source_timestamps_snapshot()
+    slo_statuses = compute_all_statuses(d, timestamps)
+    slo_summary = summarise_statuses(slo_statuses)
+    # Top-level status reflects worst SLO result — "degraded" if any
+    # yellow, "error" if any red, "ok" otherwise. This is the single
+    # field an external probe / pager can watch.
+    top_status = "ok"
+    if slo_summary.get("red", 0) > 0:
+        top_status = "error"
+    elif slo_summary.get("yellow", 0) > 0:
+        top_status = "degraded"
+    return {
+        "status": top_status,
+        "version": _get_app_version(),
+        "last_updated": last,
+        "sources": {
+            "flights": len(d.get("commercial_flights", [])),
+            "military": len(d.get("military_flights", [])),
+            "ships": len(d.get("ships", [])),
+            "satellites": len(d.get("satellites", [])),
+            "earthquakes": len(d.get("earthquakes", [])),
+            "cctv": len(d.get("cctv", [])),
+            "news": len(d.get("news", [])),
+            "uavs": len(d.get("uavs", [])),
+            "firms_fires": len(d.get("firms_fires", [])),
+            "liveuamap": len(d.get("liveuamap", [])),
+            "gdelt": len(d.get("gdelt", [])),
+            "uap_sightings": len(d.get("uap_sightings", [])),
+        },
+        "freshness": timestamps,
+        "uptime_seconds": round(_time_mod.time() - _get_start_time()),
+        "slo": slo_statuses,
+        "slo_summary": slo_summary,
+    }
+
+
+@router.get("/api/debug-latest", dependencies=[Depends(require_admin)])
+@limiter.limit("30/minute")
+async def debug_latest_data(request: Request):
+    return list(get_latest_data().keys())

backend/routers/infonet.py

@@ -0,0 +1,598 @@
+"""Infonet economy / governance / gates / bootstrap HTTP surface.
+
+Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §2.1.
+
+Read endpoints return chain-derived state (computed by the
+``services.infonet`` adapters / pure functions). Write endpoints take
+a payload, validate it through the cutover-registered validators, and
+return a structured "would-emit" preview. Production wiring (signing
++ ``Infonet.append`` persistence) is a thin follow-on; the validation
+contract is locked here.
+
+Cross-cutting design rule: errors are diagnostic, not punitive. Each
+write endpoint returns ``{"ok": False, "reason": "..."}`` on
+validation failure with the exact field that failed. Frontend
+surfaces the reason in the UI.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any
+
+from fastapi import APIRouter, Body, Path
+
+# Triggers the chain cutover at module-load time so registered
+# validators are live for any subsequent route invocation.
+from services.infonet import _chain_cutover  # noqa: F401
+from services.infonet.adapters.gate_adapter import InfonetGateAdapter
+from services.infonet.adapters.oracle_adapter import InfonetOracleAdapter
+from services.infonet.adapters.reputation_adapter import InfonetReputationAdapter
+from services.infonet.bootstrap import compute_active_features
+from services.infonet.config import (
+    CONFIG,
+    IMMUTABLE_PRINCIPLES,
+)
+from services.infonet.governance import (
+    apply_petition_payload,
+    compute_petition_state,
+    compute_upgrade_state,
+)
+from services.infonet.governance.dsl_executor import InvalidPetition
+from services.infonet.partition import (
+    classify_event_type,
+    is_chain_stale,
+    should_mark_provisional,
+)
+from services.infonet.privacy import (
+    DEXScaffolding,
+    RingCTScaffolding,
+    ShieldedBalanceScaffolding,
+    StealthAddressScaffolding,
+)
+from services.infonet.schema import (
+    INFONET_ECONOMY_EVENT_TYPES,
+    validate_infonet_event_payload,
+)
+from services.infonet.time_validity import chain_majority_time
+
+logger = logging.getLogger("routers.infonet")
+
+router = APIRouter(prefix="/api/infonet", tags=["infonet"])
+
+
+# ─── Chain access helper ─────────────────────────────────────────────────
+# Every adapter takes a ``chain_provider`` callable. We pull the live
+# Infonet chain from mesh_hashchain. Tests can monkeypatch this.
+
+def _live_chain() -> list[dict[str, Any]]:
+    try:
+        from services.mesh.mesh_hashchain import infonet
+        events = getattr(infonet, "events", None)
+        if isinstance(events, list):
+            return list(events)
+        # Some implementations use a deque; convert to list.
+        if events is not None:
+            return list(events)
+    except Exception as exc:
+        logger.debug("infonet chain unavailable: %s", exc)
+    return []
+
+
+def _now() -> float:
+    cmt = chain_majority_time(_live_chain())
+    return cmt if cmt > 0 else float(time.time())
+
+
+# ─── Status ──────────────────────────────────────────────────────────────
+
+@router.get("/status")
+def infonet_status() -> dict[str, Any]:
+    """Top-level health snapshot for the InfonetTerminal HUD.
+
+    Returns ramp activation flags, partition staleness, privacy
+    primitive statuses, immutable principles, and counts of
+    chain-derived state (markets / petitions / gates / etc).
+    """
+    chain = _live_chain()
+    now = _now()
+    features = compute_active_features(chain)
+
+    # Privacy primitive statuses (truthful — most are NOT_IMPLEMENTED).
+    privacy = {
+        "ringct": RingCTScaffolding().status().value,
+        "stealth_address": StealthAddressScaffolding().status().value,
+        "shielded_balance": ShieldedBalanceScaffolding().status().value,
+        "dex": DEXScaffolding().status().value,
+    }
+
+    return {
+        "ok": True,
+        "now": now,
+        "chain_majority_time": chain_majority_time(chain),
+        "chain_event_count": len(chain),
+        "chain_stale": is_chain_stale(chain, now=now),
+        "ramp": {
+            "node_count": features.node_count,
+            "bootstrap_resolution_active": features.bootstrap_resolution_active,
+            "staked_resolution_active": features.staked_resolution_active,
+            "governance_petitions_active": features.governance_petitions_active,
+            "upgrade_governance_active": features.upgrade_governance_active,
+            "commoncoin_active": features.commoncoin_active,
+        },
+        "privacy_primitive_status": privacy,
+        "immutable_principles": dict(IMMUTABLE_PRINCIPLES),
+        "config_keys_count": len(CONFIG),
+        "infonet_economy_event_types_count": len(INFONET_ECONOMY_EVENT_TYPES),
+    }
+
+
+# ─── Petitions / governance ──────────────────────────────────────────────
+
+@router.get("/petitions")
+def list_petitions() -> dict[str, Any]:
+    """List petition_file events on the chain with their current state."""
+    chain = _live_chain()
+    now = _now()
+    out: list[dict[str, Any]] = []
+    for ev in chain:
+        if ev.get("event_type") != "petition_file":
+            continue
+        pid = (ev.get("payload") or {}).get("petition_id")
+        if not isinstance(pid, str):
+            continue
+        try:
+            state = compute_petition_state(pid, chain, now=now)
+            out.append({
+                "petition_id": state.petition_id,
+                "status": state.status,
+                "filer_id": state.filer_id,
+                "filed_at": state.filed_at,
+                "petition_payload": state.petition_payload,
+                "signature_governance_weight": state.signature_governance_weight,
+                "signature_threshold_at_filing": state.signature_threshold_at_filing,
+                "votes_for_weight": state.votes_for_weight,
+                "votes_against_weight": state.votes_against_weight,
+                "voting_deadline": state.voting_deadline,
+                "challenge_window_until": state.challenge_window_until,
+            })
+        except Exception as exc:
+            logger.warning("petition state error for %s: %s", pid, exc)
+    return {"ok": True, "petitions": out, "now": now}
+
+
+@router.get("/petitions/{petition_id}")
+def get_petition(petition_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    state = compute_petition_state(petition_id, chain, now=now)
+    return {"ok": True, "petition": state.__dict__, "now": now}
+
+
+@router.post("/petitions/preview")
+def preview_petition_payload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
+    """Validate a petition payload through the DSL executor without
+    emitting it. Returns the candidate config diff so the UI can show
+    "this petition would change vote_decay_days from 90 to 30".
+    """
+    try:
+        result = apply_petition_payload(payload)
+        return {
+            "ok": True,
+            "changed_keys": list(result.changed_keys),
+            "new_values": {k: result.new_config[k] for k in result.changed_keys},
+        }
+    except InvalidPetition as exc:
+        return {"ok": False, "reason": str(exc)}
+
+
+@router.post("/events/validate")
+def validate_event(body: dict[str, Any] = Body(...)) -> dict[str, Any]:
+    """Validate an arbitrary Infonet economy event payload.
+
+    Frontend uses this for client-side preflight before signing /
+    submitting an event. Returns ``{ok: True}`` on success or
+    ``{ok: False, reason: ...}`` with the exact validation failure.
+    """
+    event_type = body.get("event_type")
+    payload = body.get("payload", {})
+    if not isinstance(event_type, str) or not event_type:
+        return {"ok": False, "reason": "event_type required"}
+    if not isinstance(payload, dict):
+        return {"ok": False, "reason": "payload must be an object"}
+    ok, reason = validate_infonet_event_payload(event_type, payload)
+    return {
+        "ok": ok,
+        "reason": reason if not ok else None,
+        "tier": classify_event_type(event_type),
+        "would_be_provisional": should_mark_provisional(event_type, _live_chain(), now=_now()),
+    }
+
+
+# ─── Upgrade-hash governance ────────────────────────────────────────────
+
+@router.get("/upgrades")
+def list_upgrades() -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    out: list[dict[str, Any]] = []
+    for ev in chain:
+        if ev.get("event_type") != "upgrade_propose":
+            continue
+        pid = (ev.get("payload") or {}).get("proposal_id")
+        if not isinstance(pid, str):
+            continue
+        try:
+            # Heavy node set is a runtime concept (transport tier ==
+            # private_strong per plan §3.5). Empty here for the
+            # snapshot endpoint; production will pass the live set.
+            state = compute_upgrade_state(pid, chain, now=now, heavy_node_ids=set())
+            out.append({
+                "proposal_id": state.proposal_id,
+                "status": state.status,
+                "proposer_id": state.proposer_id,
+                "filed_at": state.filed_at,
+                "release_hash": state.release_hash,
+                "target_protocol_version": state.target_protocol_version,
+                "votes_for_weight": state.votes_for_weight,
+                "votes_against_weight": state.votes_against_weight,
+                "readiness_fraction": state.readiness.fraction,
+                "readiness_threshold_met": state.readiness.threshold_met,
+            })
+        except Exception as exc:
+            logger.warning("upgrade state error for %s: %s", pid, exc)
+    return {"ok": True, "upgrades": out, "now": now}
+
+
+@router.get("/upgrades/{proposal_id}")
+def get_upgrade(proposal_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    state = compute_upgrade_state(proposal_id, chain, now=now, heavy_node_ids=set())
+    return {
+        "ok": True,
+        "upgrade": {
+            "proposal_id": state.proposal_id,
+            "status": state.status,
+            "proposer_id": state.proposer_id,
+            "filed_at": state.filed_at,
+            "release_hash": state.release_hash,
+            "target_protocol_version": state.target_protocol_version,
+            "signature_governance_weight": state.signature_governance_weight,
+            "votes_for_weight": state.votes_for_weight,
+            "votes_against_weight": state.votes_against_weight,
+            "voting_deadline": state.voting_deadline,
+            "challenge_window_until": state.challenge_window_until,
+            "activation_deadline": state.activation_deadline,
+            "readiness": {
+                "total_heavy_nodes": state.readiness.total_heavy_nodes,
+                "ready_count": state.readiness.ready_count,
+                "fraction": state.readiness.fraction,
+                "threshold_met": state.readiness.threshold_met,
+            },
+        },
+        "now": now,
+    }
+
+
+# ─── Markets / resolution / disputes ────────────────────────────────────
+
+@router.get("/markets/{market_id}")
+def get_market_state(market_id: str = Path(...)) -> dict[str, Any]:
+    """Full market view: lifecycle, snapshot, evidence, stakes,
+    excluded predictors, dispute state."""
+    chain = _live_chain()
+    now = _now()
+    oracle = InfonetOracleAdapter(lambda: chain)
+
+    status = oracle.market_status(market_id, now=now)
+    snap = oracle.find_snapshot(market_id)
+    bundles = oracle.collect_evidence(market_id)
+    excluded = sorted(oracle.excluded_predictor_ids(market_id))
+    disputes = oracle.collect_disputes(market_id)
+    reversed_flag = oracle.market_was_reversed(market_id)
+
+    return {
+        "ok": True,
+        "market_id": market_id,
+        "status": status.value,
+        "snapshot": snap,
+        "evidence_bundles": [
+            {
+                "node_id": b.node_id,
+                "claimed_outcome": b.claimed_outcome,
+                "evidence_hashes": list(b.evidence_hashes),
+                "source_description": b.source_description,
+                "bond": b.bond,
+                "timestamp": b.timestamp,
+                "is_first_for_side": b.is_first_for_side,
+                "submission_hash": b.submission_hash,
+            }
+            for b in bundles
+        ],
+        "excluded_predictor_ids": excluded,
+        "disputes": [
+            {
+                "dispute_id": d.dispute_id,
+                "challenger_id": d.challenger_id,
+                "challenger_stake": d.challenger_stake,
+                "opened_at": d.opened_at,
+                "is_resolved": d.is_resolved,
+                "resolved_outcome": d.resolved_outcome,
+                "confirm_stakes": d.confirm_stakes,
+                "reverse_stakes": d.reverse_stakes,
+            }
+            for d in disputes
+        ],
+        "was_reversed": reversed_flag,
+        "now": now,
+    }
+
+
+@router.get("/markets/{market_id}/preview-resolution")
+def preview_resolution(market_id: str = Path(...)) -> dict[str, Any]:
+    """Run the resolution decision procedure without emitting a
+    finalize event. UI uses this to show "if resolution closed now,
+    the market would resolve as <outcome> for <reason>"."""
+    chain = _live_chain()
+    oracle = InfonetOracleAdapter(lambda: chain)
+    result = oracle.resolve_market(market_id)
+    return {
+        "ok": True,
+        "preview": {
+            "outcome": result.outcome,
+            "reason": result.reason,
+            "is_provisional": result.is_provisional,
+            "burned_amount": result.burned_amount,
+            "stake_returns": [
+                {"node_id": k[0], "rep_type": k[1], "amount": v}
+                for k, v in result.stake_returns.items()
+            ],
+            "stake_winnings": [
+                {"node_id": k[0], "rep_type": k[1], "amount": v}
+                for k, v in result.stake_winnings.items()
+            ],
+            "bond_returns": [
+                {"node_id": k, "amount": v} for k, v in result.bond_returns.items()
+            ],
+            "bond_forfeits": [
+                {"node_id": k, "amount": v} for k, v in result.bond_forfeits.items()
+            ],
+            "first_submitter_bonuses": [
+                {"node_id": k, "amount": v}
+                for k, v in result.first_submitter_bonuses.items()
+            ],
+        },
+    }
+
+
+# ─── Gate shutdown lifecycle ────────────────────────────────────────────
+
+@router.get("/gates/{gate_id}")
+def get_gate_state(gate_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    gates = InfonetGateAdapter(lambda: chain)
+    meta = gates.gate_meta(gate_id)
+    if meta is None:
+        return {"ok": False, "reason": "gate_not_found"}
+    suspension = gates.suspension_state(gate_id, now=now)
+    shutdown = gates.shutdown_state(gate_id, now=now)
+    locked = gates.locked_state(gate_id)
+    members = sorted(gates.member_set(gate_id))
+    return {
+        "ok": True,
+        "gate_id": gate_id,
+        "meta": {
+            "creator_node_id": meta.creator_node_id,
+            "display_name": meta.display_name,
+            "entry_sacrifice": meta.entry_sacrifice,
+            "min_overall_rep": meta.min_overall_rep,
+            "min_gate_rep": dict(meta.min_gate_rep),
+            "created_at": meta.created_at,
+        },
+        "members": members,
+        "ratified": gates.is_ratified(gate_id),
+        "cumulative_member_oracle_rep": gates.cumulative_member_oracle_rep(gate_id),
+        "locked": {
+            "is_locked": locked.locked,
+            "locked_at": locked.locked_at,
+            "locked_by": list(locked.locked_by),
+        },
+        "suspension": {
+            "status": suspension.status,
+            "suspended_at": suspension.suspended_at,
+            "suspended_until": suspension.suspended_until,
+            "last_shutdown_petition_at": suspension.last_shutdown_petition_at,
+        },
+        "shutdown": {
+            "has_pending": shutdown.has_pending,
+            "pending_petition_id": shutdown.pending_petition_id,
+            "pending_status": shutdown.pending_status,
+            "execution_at": shutdown.execution_at,
+            "executed": shutdown.executed,
+        },
+        "now": now,
+    }
+
+
+# ─── Reputation views ───────────────────────────────────────────────────
+
+@router.get("/nodes/{node_id}/reputation")
+def get_node_reputation(node_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    rep = InfonetReputationAdapter(lambda: chain)
+    breakdown = rep.oracle_rep_breakdown(node_id)
+    return {
+        "ok": True,
+        "node_id": node_id,
+        "oracle_rep": rep.oracle_rep(node_id),
+        "oracle_rep_active": rep.oracle_rep_active(node_id),
+        "oracle_rep_lifetime": rep.oracle_rep_lifetime(node_id),
+        "common_rep": rep.common_rep(node_id),
+        "decay_factor": rep.decay_factor(node_id),
+        "last_successful_prediction_ts": rep.last_successful_prediction_ts(node_id),
+        "breakdown": {
+            "free_prediction_mints": breakdown.free_prediction_mints,
+            "staked_prediction_returns": breakdown.staked_prediction_returns,
+            "staked_prediction_losses": breakdown.staked_prediction_losses,
+            "total": breakdown.total,
+        },
+    }
+
+
+# ─── Bootstrap ──────────────────────────────────────────────────────────
+
+@router.get("/bootstrap/markets/{market_id}")
+def get_bootstrap_market_state(market_id: str = Path(...)) -> dict[str, Any]:
+    """Bootstrap-mode-specific market view: who has voted, who is
+    eligible, current tally."""
+    from services.infonet.bootstrap import (
+        deduplicate_votes,
+        validate_bootstrap_eligibility,
+    )
+
+    chain = _live_chain()
+    canonical = deduplicate_votes(market_id, chain)
+    votes_summary: list[dict[str, Any]] = []
+    yes = 0
+    no = 0
+    for v in canonical:
+        node_id = v.get("node_id") or ""
+        side = (v.get("payload") or {}).get("side")
+        decision = validate_bootstrap_eligibility(node_id, market_id, chain)
+        votes_summary.append({
+            "node_id": node_id,
+            "side": side,
+            "eligible": decision.eligible,
+            "ineligible_reason": decision.reason if not decision.eligible else None,
+        })
+        if decision.eligible:
+            if side == "yes":
+                yes += 1
+            elif side == "no":
+                no += 1
+    total = yes + no
+    return {
+        "ok": True,
+        "market_id": market_id,
+        "votes": votes_summary,
+        "tally": {
+            "yes": yes,
+            "no": no,
+            "total_eligible": total,
+            "min_market_participants": int(CONFIG["min_market_participants"]),
+            "supermajority_threshold": float(CONFIG["bootstrap_resolution_supermajority"]),
+        },
+    }
+
+
+# ─── Signed write: append an Infonet economy event ──────────────────────
+
+@router.post("/append")
+def append_event(body: dict[str, Any] = Body(...)) -> dict[str, Any]:
+    """Append a signed Infonet economy event to the chain.
+
+    Body shape (all required for production):
+
+        {
+            "event_type": str,       # one of INFONET_ECONOMY_EVENT_TYPES
+            "node_id":    str,       # signer
+            "payload":    dict,      # event-specific fields
+            "signature":  str,       # hex
+            "sequence":   int,       # node-monotonic
+            "public_key": str,       # base64
+            "public_key_algo": str,  # "ed25519" or "ecdsa"
+            "protocol_version": str  # optional, defaults to current
+        }
+
+    The cutover-registered validators run automatically via
+    ``mesh_hashchain.Infonet.append`` — payload validation, signature
+    verification, replay protection, sequence ordering, public-key
+    binding, revocation status. No additional security wrapper is
+    needed because ``Infonet.append`` IS the secure entry point.
+
+    Returns the appended event dict on success, or
+    ``{"ok": False, "reason": "..."}`` on validation / signing failure.
+    """
+    if not isinstance(body, dict):
+        return {"ok": False, "reason": "body_must_be_object"}
+
+    event_type = body.get("event_type")
+    if not isinstance(event_type, str) or event_type not in INFONET_ECONOMY_EVENT_TYPES:
+        return {
+            "ok": False,
+            "reason": f"event_type must be one of INFONET_ECONOMY_EVENT_TYPES "
+                      f"(got {event_type!r})",
+        }
+
+    node_id = body.get("node_id")
+    if not isinstance(node_id, str) or not node_id:
+        return {"ok": False, "reason": "node_id required"}
+
+    payload = body.get("payload", {})
+    if not isinstance(payload, dict):
+        return {"ok": False, "reason": "payload must be an object"}
+
+    sequence = body.get("sequence", 0)
+    try:
+        sequence = int(sequence)
+    except (TypeError, ValueError):
+        return {"ok": False, "reason": "sequence must be an integer"}
+    if sequence <= 0:
+        return {"ok": False, "reason": "sequence must be > 0"}
+
+    signature = str(body.get("signature") or "")
+    public_key = str(body.get("public_key") or "")
+    public_key_algo = str(body.get("public_key_algo") or "")
+    protocol_version = str(body.get("protocol_version") or "")
+
+    if not signature or not public_key or not public_key_algo:
+        return {
+            "ok": False,
+            "reason": "signature, public_key, and public_key_algo are required",
+        }
+
+    try:
+        from services.mesh.mesh_hashchain import infonet
+        event = infonet.append(
+            event_type=event_type,
+            node_id=node_id,
+            payload=payload,
+            signature=signature,
+            sequence=sequence,
+            public_key=public_key,
+            public_key_algo=public_key_algo,
+            protocol_version=protocol_version,
+        )
+    except ValueError as exc:
+        # Infonet.append raises ValueError for any validation failure
+        # — payload / signature / replay / sequence / binding. The
+        # message is user-facing per the non-hostile UX rule.
+        return {"ok": False, "reason": str(exc)}
+    except Exception as exc:
+        logger.exception("infonet append failed")
+        return {"ok": False, "reason": f"server_error: {type(exc).__name__}"}
+
+    return {"ok": True, "event": event}
+
+
+# ─── Function Keys (citizen + operator views) ───────────────────────────
+
+@router.get("/function-keys/operator/{operator_id}/batch-summary")
+def operator_batch_summary(operator_id: str = Path(...)) -> dict[str, Any]:
+    """Sprint 11+ scaffolding: returns the operator's local batch
+    counter for the current period. Production wires this through the
+    operator's local-store implementation (Sprint 11+ scaffolding
+    doesn't persist; counts reset per process)."""
+    return {
+        "ok": True,
+        "operator_id": operator_id,
+        "scaffolding_only": True,
+        "note": "Production operators maintain a persistent BatchedSettlementBatch. "
+                "This endpoint reports the in-memory state of the local batch.",
+    }
+
+
+__all__ = ["router"]

backend/routers/mesh_dm.py

@@ -0,0 +1,565 @@
+import asyncio
+import hashlib
+import hmac
+import logging
+import secrets
+import time
+from typing import Any
+
+from fastapi import APIRouter, Depends, Request
+from fastapi.responses import JSONResponse
+
+from auth import (
+    _is_debug_test_request,
+    _scoped_view_authenticated,
+    _verify_peer_push_hmac,
+    require_admin,
+)
+from limiter import limiter
+from services.config import get_settings
+from services.mesh.mesh_compatibility import (
+    LEGACY_AGENT_ID_LOOKUP_TARGET,
+    legacy_agent_id_lookup_blocked,
+    record_legacy_agent_id_lookup,
+    sunset_target_label,
+)
+from services.mesh.mesh_signed_events import (
+    MeshWriteExemption,
+    SignedWriteKind,
+    get_prepared_signed_write,
+    mesh_write_exempt,
+    requires_signed_write,
+)
+
+logger = logging.getLogger(__name__)
+_WARNED_LEGACY_DM_PUBKEY_LOOKUPS: set[str] = set()
+
+router = APIRouter()
+
+
+# ---------------------------------------------------------------------------
+# Local helpers
+# ---------------------------------------------------------------------------
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _warn_legacy_dm_pubkey_lookup(agent_id: str) -> None:
+    peer_id = str(agent_id or "").strip().lower()
+    if not peer_id or peer_id in _WARNED_LEGACY_DM_PUBKEY_LOOKUPS:
+        return
+    _WARNED_LEGACY_DM_PUBKEY_LOOKUPS.add(peer_id)
+    logger.warning(
+        "mesh legacy DH pubkey lookup used for %s via direct agent_id; prefer invite-scoped lookup handles before removal in %s",
+        peer_id,
+        sunset_target_label(LEGACY_AGENT_ID_LOOKUP_TARGET),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Transition delegates: forward to main.py so test monkeypatches still work.
+# These will move to a shared module once main.py routes are removed.
+# ---------------------------------------------------------------------------
+def _main_delegate(name):
+    def _wrapper(*a, **kw):
+        import main as _m
+        return getattr(_m, name)(*a, **kw)
+    _wrapper.__name__ = name
+    return _wrapper
+
+
+_verify_signed_write = _main_delegate("_verify_signed_write")
+_secure_dm_enabled = _main_delegate("_secure_dm_enabled")
+_legacy_dm_get_allowed = _main_delegate("_legacy_dm_get_allowed")
+_rns_private_dm_ready = _main_delegate("_rns_private_dm_ready")
+_anonymous_dm_hidden_transport_enforced = _main_delegate("_anonymous_dm_hidden_transport_enforced")
+_high_privacy_profile_enabled = _main_delegate("_high_privacy_profile_enabled")
+_dm_send_from_signed_request = _main_delegate("_dm_send_from_signed_request")
+_dm_poll_secure_from_signed_request = _main_delegate("_dm_poll_secure_from_signed_request")
+_dm_count_secure_from_signed_request = _main_delegate("_dm_count_secure_from_signed_request")
+_validate_private_signed_sequence = _main_delegate("_validate_private_signed_sequence")
+
+
+def _signed_body(request: Request) -> dict[str, Any]:
+    prepared = get_prepared_signed_write(request)
+    if prepared is None:
+        return {}
+    return dict(prepared.body)
+
+
+async def _maybe_apply_dm_relay_jitter() -> None:
+    if not _high_privacy_profile_enabled():
+        return
+    await asyncio.sleep((50 + secrets.randbelow(451)) / 1000.0)
+
+
+_REQUEST_V2_REDUCED_VERSION = "request-v2-reduced-v3"
+_REQUEST_V2_RECOVERY_STATES = {"pending", "verified", "failed"}
+
+
+def _is_canonical_reduced_request_message(message: dict[str, Any]) -> bool:
+    item = dict(message or {})
+    return (
+        str(item.get("delivery_class", "") or "").strip().lower() == "request"
+        and str(item.get("request_contract_version", "") or "").strip()
+        == _REQUEST_V2_REDUCED_VERSION
+        and item.get("sender_recovery_required") is True
+    )
+
+
+def _annotate_request_recovery_message(message: dict[str, Any]) -> dict[str, Any]:
+    item = dict(message or {})
+    delivery_class = str(item.get("delivery_class", "") or "").strip().lower()
+    sender_id = str(item.get("sender_id", "") or "").strip()
+    sender_seal = str(item.get("sender_seal", "") or "").strip()
+    sender_is_blinded = sender_id.startswith("sealed:") or sender_id.startswith("sender_token:")
+    if delivery_class != "request" or not sender_is_blinded or not sender_seal.startswith("v3:"):
+        return item
+    if not str(item.get("request_contract_version", "") or "").strip():
+        item["request_contract_version"] = _REQUEST_V2_REDUCED_VERSION
+    item["sender_recovery_required"] = True
+    state = str(item.get("sender_recovery_state", "") or "").strip().lower()
+    if state not in _REQUEST_V2_RECOVERY_STATES:
+        state = "pending"
+    item["sender_recovery_state"] = state
+    return item
+
+
+def _annotate_request_recovery_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    return [_annotate_request_recovery_message(message) for message in (messages or [])]
+
+
+def _request_duplicate_authority_rank(message: dict[str, Any]) -> int:
+    item = dict(message or {})
+    if str(item.get("delivery_class", "") or "").strip().lower() != "request":
+        return 0
+    if _is_canonical_reduced_request_message(item):
+        return 3
+    sender_id = str(item.get("sender_id", "") or "").strip()
+    if sender_id.startswith("sealed:") or sender_id.startswith("sender_token:"):
+        return 1
+    if sender_id:
+        return 2
+    return 0
+
+
+def _request_duplicate_recovery_rank(message: dict[str, Any]) -> int:
+    if not _is_canonical_reduced_request_message(message):
+        return 0
+    state = str(dict(message or {}).get("sender_recovery_state", "") or "").strip().lower()
+    if state == "verified":
+        return 2
+    if state == "pending":
+        return 1
+    return 0
+
+
+def _poll_duplicate_source_rank(source: str) -> int:
+    normalized = str(source or "").strip().lower()
+    if normalized == "relay":
+        return 2
+    if normalized == "reticulum":
+        return 1
+    return 0
+
+
+def _should_replace_dm_poll_duplicate(
+    existing: dict[str, Any],
+    existing_source: str,
+    candidate: dict[str, Any],
+    candidate_source: str,
+) -> bool:
+    candidate_authority = _request_duplicate_authority_rank(candidate)
+    existing_authority = _request_duplicate_authority_rank(existing)
+    if candidate_authority != existing_authority:
+        return candidate_authority > existing_authority
+
+    candidate_recovery = _request_duplicate_recovery_rank(candidate)
+    existing_recovery = _request_duplicate_recovery_rank(existing)
+    if candidate_recovery != existing_recovery:
+        return candidate_recovery > existing_recovery
+
+    candidate_source_rank = _poll_duplicate_source_rank(candidate_source)
+    existing_source_rank = _poll_duplicate_source_rank(existing_source)
+    if candidate_source_rank != existing_source_rank:
+        return candidate_source_rank > existing_source_rank
+
+    try:
+        candidate_ts = float(candidate.get("timestamp", 0) or 0)
+    except Exception:
+        candidate_ts = 0.0
+    try:
+        existing_ts = float(existing.get("timestamp", 0) or 0)
+    except Exception:
+        existing_ts = 0.0
+    return candidate_ts > existing_ts
+
+
+def _merge_dm_poll_messages(
+    relay_messages: list[dict[str, Any]],
+    direct_messages: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    merged: list[dict[str, Any]] = []
+    index_by_msg_id: dict[str, tuple[int, str]] = {}
+
+    def add_messages(items: list[dict[str, Any]], source: str) -> None:
+        for original in items or []:
+            item = dict(original or {})
+            msg_id = str(item.get("msg_id", "") or "").strip()
+            if not msg_id:
+                merged.append(item)
+                continue
+            existing = index_by_msg_id.get(msg_id)
+            if existing is None:
+                index_by_msg_id[msg_id] = (len(merged), source)
+                merged.append(item)
+                continue
+            index, existing_source = existing
+            if _should_replace_dm_poll_duplicate(merged[index], existing_source, item, source):
+                merged[index] = item
+                index_by_msg_id[msg_id] = (index, source)
+
+    add_messages(relay_messages, "relay")
+    add_messages(direct_messages, "reticulum")
+    return sorted(merged, key=lambda item: float(item.get("timestamp", 0) or 0))
+
+
+# ---------------------------------------------------------------------------
+# Route handlers
+# ---------------------------------------------------------------------------
+
+@router.post("/api/mesh/dm/register")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_REGISTER)
+async def dm_register_key(request: Request):
+    """Register a DH public key for encrypted DM key exchange."""
+    body = _signed_body(request)
+    agent_id = body.get("agent_id", "").strip()
+    dh_pub_key = body.get("dh_pub_key", "").strip()
+    dh_algo = body.get("dh_algo", "").strip()
+    timestamp = _safe_int(body.get("timestamp", 0) or 0)
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not agent_id or not dh_pub_key or not dh_algo or not timestamp:
+        return {"ok": False, "detail": "Missing agent_id, dh_pub_key, dh_algo, or timestamp"}
+    if dh_algo.upper() not in ("X25519", "ECDH_P256", "ECDH"):
+        return {"ok": False, "detail": "Unsupported dh_algo"}
+    now_ts = int(time.time())
+    if abs(timestamp - now_ts) > 7 * 86400:
+        return {"ok": False, "detail": "DH key timestamp is too far from current time"}
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+
+        reputation_ledger.register_node(agent_id, public_key, public_key_algo)
+    except Exception:
+        pass
+
+    accepted, detail, metadata = dm_relay.register_dh_key(
+        agent_id,
+        dh_pub_key,
+        dh_algo,
+        timestamp,
+        signature,
+        public_key,
+        public_key_algo,
+        protocol_version,
+        sequence,
+    )
+    if not accepted:
+        return {"ok": False, "detail": detail}
+
+    return {"ok": True, **(metadata or {})}
+
+
+@router.get("/api/mesh/dm/pubkey")
+@limiter.limit("30/minute")
+async def dm_get_pubkey(request: Request, agent_id: str = "", lookup_token: str = ""):
+    import main as _m
+
+    return await _m.dm_get_pubkey(request, agent_id=agent_id, lookup_token=lookup_token)
+
+
+@router.get("/api/mesh/dm/prekey-bundle")
+@limiter.limit("30/minute")
+async def dm_get_prekey_bundle(request: Request, agent_id: str = "", lookup_token: str = ""):
+    import main as _m
+
+    return await _m.dm_get_prekey_bundle(request, agent_id=agent_id, lookup_token=lookup_token)
+
+
+@router.post("/api/mesh/dm/prekey-peer-lookup")
+@limiter.limit("60/minute")
+@mesh_write_exempt(MeshWriteExemption.PEER_GOSSIP)
+async def dm_prekey_peer_lookup(request: Request):
+    """Peer-authenticated invite lookup handle resolution.
+
+    This endpoint exists for private/bootstrap peers to import signed invites
+    without exposing a stable agent_id on the ordinary lookup surface. It only
+    accepts HMAC-authenticated peer calls and only resolves lookup_token.
+    """
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 4096:
+                return JSONResponse(
+                    status_code=413,
+                    content={"ok": False, "detail": "Request body too large"},
+                )
+        except (TypeError, ValueError):
+            pass
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return JSONResponse(
+            status_code=403,
+            content={"ok": False, "detail": "Invalid or missing peer HMAC"},
+        )
+    try:
+        import json
+
+        body = json.loads(body_bytes or b"{}")
+    except Exception:
+        return {"ok": False, "detail": "invalid json"}
+    lookup_token = str(dict(body or {}).get("lookup_token", "") or "").strip()
+    if not lookup_token:
+        return {"ok": False, "detail": "lookup_token required"}
+    from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle
+
+    result = fetch_dm_prekey_bundle(
+        agent_id="",
+        lookup_token=lookup_token,
+        allow_peer_lookup=False,
+    )
+    if not result.get("ok"):
+        return {"ok": False, "detail": str(result.get("detail", "") or "Prekey bundle not found")}
+    safe = dict(result)
+    safe.pop("resolved_agent_id", None)
+    safe["lookup_mode"] = "invite_lookup_handle"
+    return safe
+
+
+@router.post("/api/mesh/dm/send")
+@limiter.limit("20/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_SEND)
+async def dm_send(request: Request):
+    return await _dm_send_from_signed_request(request)
+
+
+@router.post("/api/mesh/dm/poll")
+@limiter.limit("30/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_POLL)
+async def dm_poll_secure(request: Request):
+    return await _dm_poll_secure_from_signed_request(request)
+
+
+@router.get("/api/mesh/dm/poll")
+@limiter.limit("30/minute")
+async def dm_poll(
+    request: Request,
+    agent_id: str = "",
+    agent_token: str = "",
+    agent_token_prev: str = "",
+    agent_tokens: str = "",
+):
+    import main as _m
+
+    return await _m.dm_poll(
+        request,
+        agent_id=agent_id,
+        agent_token=agent_token,
+        agent_token_prev=agent_token_prev,
+        agent_tokens=agent_tokens,
+    )
+
+
+@router.post("/api/mesh/dm/count")
+@limiter.limit("60/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_COUNT)
+async def dm_count_secure(request: Request):
+    return await _dm_count_secure_from_signed_request(request)
+
+
+@router.get("/api/mesh/dm/count")
+@limiter.limit("60/minute")
+async def dm_count(
+    request: Request,
+    agent_id: str = "",
+    agent_token: str = "",
+    agent_token_prev: str = "",
+    agent_tokens: str = "",
+):
+    import main as _m
+
+    return await _m.dm_count(
+        request,
+        agent_id=agent_id,
+        agent_token=agent_token,
+        agent_token_prev=agent_token_prev,
+        agent_tokens=agent_tokens,
+    )
+
+
+@router.post("/api/mesh/dm/block")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_BLOCK)
+async def dm_block(request: Request):
+    """Block or unblock a sender from DMing you."""
+    body = _signed_body(request)
+    agent_id = body.get("agent_id", "").strip()
+    blocked_id = body.get("blocked_id", "").strip()
+    action = body.get("action", "block").strip().lower()
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not agent_id or not blocked_id:
+        return {"ok": False, "detail": "Missing agent_id or blocked_id"}
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    try:
+        from services.mesh.mesh_hashchain import infonet
+
+        ok_seq, seq_reason = _validate_private_signed_sequence(
+            infonet,
+            agent_id,
+            sequence,
+            domain=f"dm_block:{action}",
+        )
+        if not ok_seq:
+            return {"ok": False, "detail": seq_reason}
+    except Exception:
+        pass
+
+    if action == "unblock":
+        dm_relay.unblock(agent_id, blocked_id)
+    else:
+        dm_relay.block(agent_id, blocked_id)
+    return {"ok": True, "action": action, "blocked_id": blocked_id}
+
+
+@router.post("/api/mesh/dm/witness")
+@limiter.limit("20/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_WITNESS)
+async def dm_key_witness(request: Request):
+    """Record a lightweight witness for a DM key (dual-path spot-check)."""
+    body = _signed_body(request)
+    witness_id = body.get("witness_id", "").strip()
+    target_id = body.get("target_id", "").strip()
+    dh_pub_key = body.get("dh_pub_key", "").strip()
+    timestamp = _safe_int(body.get("timestamp", 0) or 0)
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not witness_id or not target_id or not dh_pub_key or not timestamp:
+        return {"ok": False, "detail": "Missing witness_id, target_id, dh_pub_key, or timestamp"}
+    now_ts = int(time.time())
+    if abs(timestamp - now_ts) > 7 * 86400:
+        return {"ok": False, "detail": "Witness timestamp is too far from current time"}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+
+        reputation_ledger.register_node(witness_id, public_key, public_key_algo)
+    except Exception:
+        pass
+    try:
+        from services.mesh.mesh_hashchain import infonet
+
+        ok_seq, seq_reason = _validate_private_signed_sequence(
+            infonet,
+            witness_id,
+            sequence,
+            domain="dm_witness",
+        )
+        if not ok_seq:
+            return {"ok": False, "detail": seq_reason}
+    except Exception:
+        pass
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    ok, reason = dm_relay.record_witness(witness_id, target_id, dh_pub_key, timestamp)
+    return {"ok": ok, "detail": reason}
+
+
+@router.get("/api/mesh/dm/witness")
+@limiter.limit("60/minute")
+async def dm_key_witness_get(request: Request, target_id: str = "", dh_pub_key: str = ""):
+    """Get witness counts for a target's DH key."""
+    if not target_id:
+        return {"ok": False, "detail": "Missing target_id"}
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    witnesses = dm_relay.get_witnesses(target_id, dh_pub_key if dh_pub_key else None, limit=5)
+    response = {
+        "ok": True,
+        "count": len(witnesses),
+    }
+    if _scoped_view_authenticated(request, "mesh.audit"):
+        response["target_id"] = target_id
+        response["dh_pub_key"] = dh_pub_key or ""
+        response["witnesses"] = witnesses
+    return response
+
+
+@router.post("/api/mesh/trust/vouch")
+@limiter.limit("20/minute")
+@requires_signed_write(kind=SignedWriteKind.TRUST_VOUCH)
+async def trust_vouch(request: Request):
+    """Record a trust vouch for a node (web-of-trust signal)."""
+    body = _signed_body(request)
+    voucher_id = body.get("voucher_id", "").strip()
+    target_id = body.get("target_id", "").strip()
+    note = body.get("note", "").strip()
+    timestamp = _safe_int(body.get("timestamp", 0) or 0)
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not voucher_id or not target_id or not timestamp:
+        return {"ok": False, "detail": "Missing voucher_id, target_id, or timestamp"}
+    now_ts = int(time.time())
+    if abs(timestamp - now_ts) > 7 * 86400:
+        return {"ok": False, "detail": "Vouch timestamp is too far from current time"}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+        from services.mesh.mesh_hashchain import infonet
+
+        reputation_ledger.register_node(voucher_id, public_key, public_key_algo)
+        ok_seq, seq_reason = _validate_private_signed_sequence(
+            infonet,
+            voucher_id,
+            sequence,
+            domain="trust_vouch",
+        )
+        if not ok_seq:
+            return {"ok": False, "detail": seq_reason}
+        ok, reason = reputation_ledger.add_vouch(voucher_id, target_id, note, timestamp)
+        return {"ok": ok, "detail": reason}
+    except Exception:
+        return {"ok": False, "detail": "Failed to record vouch"}
+
+
+@router.get("/api/mesh/trust/vouches", dependencies=[Depends(require_admin)])
+@limiter.limit("60/minute")
+async def trust_vouches(request: Request, node_id: str = "", limit: int = 20):
+    """Fetch latest vouches for a node."""
+    if not node_id:
+        return {"ok": False, "detail": "Missing node_id"}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+
+        vouches = reputation_ledger.get_vouches(node_id, limit=limit)
+        return {"ok": True, "node_id": node_id, "vouches": vouches, "count": len(vouches)}
+    except Exception:
+        return {"ok": False, "detail": "Failed to fetch vouches"}

backend/routers/mesh_operator.py

@@ -0,0 +1,145 @@
+import time
+import logging
+from fastapi import APIRouter, Request, Response, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def list_peers(request: Request, bucket: str = Query(None)):
+    """List all peers (or filter by bucket: sync, push, bootstrap)."""
+    from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerStore
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception as exc:
+        return {"ok": False, "detail": f"Failed to load peer store: {exc}"}
+    if bucket:
+        records = store.records_for_bucket(bucket)
+    else:
+        records = store.records()
+    return {"ok": True, "count": len(records), "peers": [r.to_dict() for r in records]}
+
+
+@router.post("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def add_peer(request: Request):
+    """Add a peer to the store. Body: {peer_url, transport?, label?, role?, buckets?[]}."""
+    from services.mesh.mesh_crypto import normalize_peer_url
+    from services.mesh.mesh_peer_store import (
+        DEFAULT_PEER_STORE_PATH, PeerStore, PeerStoreError,
+        make_push_peer_record, make_sync_peer_record,
+    )
+    from services.mesh.mesh_router import peer_transport_kind
+    body = await request.json()
+    peer_url_raw = str(body.get("peer_url", "") or "").strip()
+    if not peer_url_raw:
+        return {"ok": False, "detail": "peer_url is required"}
+    peer_url = normalize_peer_url(peer_url_raw)
+    if not peer_url:
+        return {"ok": False, "detail": "Invalid peer_url"}
+    transport = str(body.get("transport", "") or "").strip().lower()
+    if not transport:
+        transport = peer_transport_kind(peer_url)
+    if not transport:
+        return {"ok": False, "detail": "Cannot determine transport for peer_url — provide transport explicitly"}
+    label = str(body.get("label", "") or "").strip()
+    role = str(body.get("role", "") or "").strip().lower() or "relay"
+    buckets = body.get("buckets", ["sync", "push"])
+    if isinstance(buckets, str):
+        buckets = [buckets]
+    if not isinstance(buckets, list):
+        buckets = ["sync", "push"]
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    added: list = []
+    try:
+        for b in buckets:
+            b = str(b).strip().lower()
+            if b == "sync":
+                store.upsert(make_sync_peer_record(peer_url=peer_url, transport=transport, role=role, label=label))
+                added.append("sync")
+            elif b == "push":
+                store.upsert(make_push_peer_record(peer_url=peer_url, transport=transport, role=role, label=label))
+                added.append("push")
+        store.save()
+    except PeerStoreError as exc:
+        return {"ok": False, "detail": str(exc)}
+    return {"ok": True, "peer_url": peer_url, "buckets": added}
+
+
+@router.delete("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def remove_peer(request: Request):
+    """Remove a peer. Body: {peer_url, bucket?}. If bucket omitted, removes from all buckets."""
+    from services.mesh.mesh_crypto import normalize_peer_url
+    from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerStore
+    body = await request.json()
+    peer_url_raw = str(body.get("peer_url", "") or "").strip()
+    if not peer_url_raw:
+        return {"ok": False, "detail": "peer_url is required"}
+    peer_url = normalize_peer_url(peer_url_raw)
+    if not peer_url:
+        return {"ok": False, "detail": "Invalid peer_url"}
+    bucket_filter = str(body.get("bucket", "") or "").strip().lower()
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        return {"ok": False, "detail": "Failed to load peer store"}
+    removed: list = []
+    for b in ["bootstrap", "sync", "push"]:
+        if bucket_filter and b != bucket_filter:
+            continue
+        key = f"{b}:{peer_url}"
+        if key in store._records:
+            del store._records[key]
+            removed.append(b)
+    if not removed:
+        return {"ok": False, "detail": "Peer not found in any bucket"}
+    store.save()
+    return {"ok": True, "peer_url": peer_url, "removed_from": removed}
+
+
+@router.patch("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def toggle_peer(request: Request):
+    """Enable or disable a peer. Body: {peer_url, bucket, enabled: bool}."""
+    from services.mesh.mesh_crypto import normalize_peer_url
+    from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerRecord, PeerStore
+    body = await request.json()
+    peer_url_raw = str(body.get("peer_url", "") or "").strip()
+    bucket = str(body.get("bucket", "") or "").strip().lower()
+    enabled = body.get("enabled")
+    if not peer_url_raw:
+        return {"ok": False, "detail": "peer_url is required"}
+    if not bucket:
+        return {"ok": False, "detail": "bucket is required"}
+    if enabled is None:
+        return {"ok": False, "detail": "enabled (true/false) is required"}
+    peer_url = normalize_peer_url(peer_url_raw)
+    if not peer_url:
+        return {"ok": False, "detail": "Invalid peer_url"}
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        return {"ok": False, "detail": "Failed to load peer store"}
+    key = f"{bucket}:{peer_url}"
+    record = store._records.get(key)
+    if not record:
+        return {"ok": False, "detail": f"Peer not found in {bucket} bucket"}
+    updated = PeerRecord(**{**record.to_dict(), "enabled": bool(enabled), "updated_at": int(time.time())})
+    store._records[key] = updated
+    store.save()
+    return {"ok": True, "peer_url": peer_url, "bucket": bucket, "enabled": bool(enabled)}

backend/routers/mesh_oracle.py

@@ -0,0 +1,337 @@
+import math
+from typing import Any
+from fastapi import APIRouter, Request, Response, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator, _scoped_view_authenticated
+from services.data_fetcher import get_latest_data
+from services.mesh.mesh_protocol import normalize_payload
+from services.mesh.mesh_signed_events import (
+    MeshWriteExemption,
+    SignedWriteKind,
+    get_prepared_signed_write,
+    mesh_write_exempt,
+    requires_signed_write,
+)
+
+router = APIRouter()
+
+
+def _signed_body(request: Request) -> dict[str, Any]:
+    prepared = get_prepared_signed_write(request)
+    if prepared is None:
+        return {}
+    return dict(prepared.body)
+
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _safe_float(val, default=0.0):
+    try:
+        parsed = float(val)
+        if not math.isfinite(parsed):
+            return default
+        return parsed
+    except (TypeError, ValueError):
+        return default
+
+
+def _redact_public_oracle_profile(payload: dict, authenticated: bool) -> dict:
+    redacted = dict(payload)
+    if authenticated:
+        return redacted
+    redacted["active_stakes"] = []
+    redacted["prediction_history"] = []
+    return redacted
+
+
+def _redact_public_oracle_predictions(predictions: list, authenticated: bool) -> dict:
+    if authenticated:
+        return {"predictions": list(predictions)}
+    return {"predictions": [], "count": len(predictions)}
+
+
+def _redact_public_oracle_stakes(payload: dict, authenticated: bool) -> dict:
+    redacted = dict(payload)
+    if authenticated:
+        return redacted
+    redacted["truth_stakers"] = []
+    redacted["false_stakers"] = []
+    return redacted
+
+
+@router.post("/api/mesh/oracle/predict")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.ORACLE_PREDICT)
+async def oracle_predict(request: Request):
+    """Place a prediction on a market outcome."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    body = _signed_body(request)
+    node_id = body.get("node_id", "")
+    market_title = body.get("market_title", "")
+    side = body.get("side", "")
+    stake_amount = _safe_float(body.get("stake_amount", 0))
+    public_key = body.get("public_key", "")
+    public_key_algo = body.get("public_key_algo", "")
+    signature = body.get("signature", "")
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "")
+    if not node_id or not market_title or not side:
+        return {"ok": False, "detail": "Missing node_id, market_title, or side"}
+    prediction_payload = {"market_title": market_title, "side": side, "stake_amount": stake_amount}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+        reputation_ledger.register_node(node_id, public_key, public_key_algo)
+    except Exception:
+        pass
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    matched = None
+    for m in markets:
+        if m.get("title", "").lower() == market_title.lower():
+            matched = m
+            break
+    if not matched:
+        for m in markets:
+            if market_title.lower() in m.get("title", "").lower():
+                matched = m
+                break
+    if not matched:
+        return {"ok": False, "detail": f"Market '{market_title}' not found in active markets."}
+    probability = 50.0
+    side_lower = side.lower()
+    outcomes = matched.get("outcomes", [])
+    if outcomes:
+        for o in outcomes:
+            if o.get("name", "").lower() == side_lower:
+                probability = float(o.get("pct", 50))
+                break
+    else:
+        consensus = matched.get("consensus_pct")
+        if consensus is None:
+            consensus = matched.get("polymarket_pct") or matched.get("kalshi_pct") or 50
+        probability = float(consensus)
+        if side_lower == "no":
+            probability = 100.0 - probability
+    if stake_amount > 0:
+        ok, detail = oracle_ledger.place_market_stake(node_id, matched["title"], side, stake_amount, probability)
+        mode = "staked"
+    else:
+        ok, detail = oracle_ledger.place_prediction(node_id, matched["title"], side, probability)
+        mode = "free"
+    if ok:
+        try:
+            from services.mesh.mesh_hashchain import infonet
+            normalized_payload = normalize_payload("prediction", prediction_payload)
+            infonet.append(event_type="prediction", node_id=node_id, payload=normalized_payload,
+                signature=signature, sequence=sequence, public_key=public_key,
+                public_key_algo=public_key_algo, protocol_version=protocol_version)
+        except Exception:
+            pass
+    return {"ok": ok, "detail": detail, "probability": probability, "mode": mode}
+
+
+@router.get("/api/mesh/oracle/markets")
+@limiter.limit("30/minute")
+async def oracle_markets(request: Request):
+    """List active prediction markets."""
+    from collections import defaultdict
+    from services.mesh.mesh_oracle import oracle_ledger
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    all_consensus = oracle_ledger.get_all_market_consensus()
+    by_category = defaultdict(list)
+    for m in markets:
+        by_category[m.get("category", "NEWS")].append(m)
+    _fields = ("title", "consensus_pct", "polymarket_pct", "kalshi_pct", "volume", "volume_24h",
+               "end_date", "description", "category", "sources", "slug", "kalshi_ticker", "outcomes")
+    categories = {}
+    cat_totals = {}
+    for cat in ["POLITICS", "CONFLICT", "NEWS", "FINANCE", "CRYPTO"]:
+        all_cat = sorted(by_category.get(cat, []), key=lambda x: x.get("volume", 0) or 0, reverse=True)
+        cat_totals[cat] = len(all_cat)
+        cat_list = []
+        for m in all_cat[:10]:
+            entry = {k: m.get(k) for k in _fields}
+            entry["consensus"] = all_consensus.get(m.get("title", ""), {})
+            cat_list.append(entry)
+        categories[cat] = cat_list
+    return {"categories": categories, "total_count": len(markets), "cat_totals": cat_totals}
+
+
+@router.get("/api/mesh/oracle/search")
+@limiter.limit("20/minute")
+async def oracle_search(request: Request, q: str = "", limit: int = 50):
+    """Search prediction markets across Polymarket + Kalshi APIs."""
+    if not q or len(q) < 2:
+        return {"results": [], "query": q, "count": 0}
+    from services.fetchers.prediction_markets import search_polymarket_direct, search_kalshi_direct
+    import concurrent.futures
+    # Search both APIs in parallel for speed
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
+        poly_fut = pool.submit(search_polymarket_direct, q, limit)
+        kalshi_fut = pool.submit(search_kalshi_direct, q, limit)
+        poly_results = poly_fut.result(timeout=20)
+        kalshi_results = kalshi_fut.result(timeout=20)
+    # Also check cached/merged markets
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    q_lower = q.lower()
+    cached_matches = [m for m in markets if q_lower in m.get("title", "").lower()]
+    seen_titles = set()
+    combined = []
+    # Cached first (already merged Poly+Kalshi with consensus)
+    for m in cached_matches:
+        seen_titles.add(m["title"].lower())
+        combined.append(m)
+    # Then Polymarket direct hits
+    for m in poly_results:
+        if m["title"].lower() not in seen_titles:
+            seen_titles.add(m["title"].lower())
+            combined.append(m)
+    # Then Kalshi direct hits
+    for m in kalshi_results:
+        if m["title"].lower() not in seen_titles:
+            seen_titles.add(m["title"].lower())
+            combined.append(m)
+    combined.sort(key=lambda x: x.get("volume", 0) or 0, reverse=True)
+    _fields = ("title", "consensus_pct", "polymarket_pct", "kalshi_pct", "volume", "volume_24h",
+               "end_date", "description", "category", "sources", "slug", "kalshi_ticker", "outcomes")
+    results = [{k: m.get(k) for k in _fields} for m in combined[:limit]]
+    return {"results": results, "query": q, "count": len(results)}
+
+
+@router.get("/api/mesh/oracle/markets/more")
+@limiter.limit("30/minute")
+async def oracle_markets_more(request: Request, category: str = "NEWS", offset: int = 0, limit: int = 10):
+    """Load more markets for a specific category (paginated)."""
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    cat_markets = sorted([m for m in markets if m.get("category") == category],
+        key=lambda x: x.get("volume", 0) or 0, reverse=True)
+    page = cat_markets[offset : offset + limit]
+    _fields = ("title", "consensus_pct", "polymarket_pct", "kalshi_pct", "volume", "volume_24h",
+               "end_date", "description", "category", "sources", "slug", "kalshi_ticker", "outcomes")
+    results = [{k: m.get(k) for k in _fields} for m in page]
+    return {"markets": results, "category": category, "offset": offset,
+            "has_more": offset + limit < len(cat_markets), "total": len(cat_markets)}
+
+
+@router.post("/api/mesh/oracle/resolve")
+@limiter.limit("5/minute")
+@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
+async def oracle_resolve(request: Request):
+    """Resolve a prediction market."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    body = await request.json()
+    market_title = body.get("market_title", "")
+    outcome = body.get("outcome", "")
+    if not market_title or not outcome:
+        return {"ok": False, "detail": "Need market_title and outcome"}
+    winners, losers = oracle_ledger.resolve_market(market_title, outcome)
+    stake_result = oracle_ledger.resolve_market_stakes(market_title, outcome)
+    return {"ok": True,
+            "detail": f"Resolved: {winners} free winners, {losers} free losers, "
+                      f"{stake_result.get('winners', 0)} stake winners, {stake_result.get('losers', 0)} stake losers",
+            "free": {"winners": winners, "losers": losers}, "stakes": stake_result}
+
+
+@router.get("/api/mesh/oracle/consensus")
+@limiter.limit("30/minute")
+async def oracle_consensus(request: Request, market_title: str = ""):
+    """Get network consensus for a market."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    if not market_title:
+        return {"error": "market_title required"}
+    return oracle_ledger.get_market_consensus(market_title)
+
+
+@router.post("/api/mesh/oracle/stake")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.ORACLE_STAKE)
+async def oracle_stake(request: Request):
+    """Stake oracle rep on a post's truthfulness."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    body = _signed_body(request)
+    staker_id = body.get("staker_id", "")
+    message_id = body.get("message_id", "")
+    poster_id = body.get("poster_id", "")
+    side = body.get("side", "").lower()
+    amount = _safe_float(body.get("amount", 0))
+    duration_days = _safe_int(body.get("duration_days", 1), 1)
+    public_key = body.get("public_key", "")
+    public_key_algo = body.get("public_key_algo", "")
+    signature = body.get("signature", "")
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "")
+    if not staker_id or not message_id or not side:
+        return {"ok": False, "detail": "Missing staker_id, message_id, or side"}
+    stake_payload = {"message_id": message_id, "poster_id": poster_id, "side": side,
+                     "amount": amount, "duration_days": duration_days}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+        reputation_ledger.register_node(staker_id, public_key, public_key_algo)
+    except Exception:
+        pass
+    ok, detail = oracle_ledger.place_stake(staker_id, message_id, poster_id, side, amount, duration_days)
+    if ok:
+        try:
+            from services.mesh.mesh_hashchain import infonet
+            normalized_payload = normalize_payload("stake", stake_payload)
+            infonet.append(event_type="stake", node_id=staker_id, payload=normalized_payload,
+                signature=signature, sequence=sequence, public_key=public_key,
+                public_key_algo=public_key_algo, protocol_version=protocol_version)
+        except Exception:
+            pass
+    return {"ok": ok, "detail": detail}
+
+
+@router.get("/api/mesh/oracle/stakes/{message_id}")
+@limiter.limit("30/minute")
+async def oracle_stakes_for_message(request: Request, message_id: str):
+    """Get all oracle stakes on a message."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    return _redact_public_oracle_stakes(
+        oracle_ledger.get_stakes_for_message(message_id),
+        authenticated=_scoped_view_authenticated(request, "mesh.audit"),
+    )
+
+
+@router.get("/api/mesh/oracle/profile")
+@limiter.limit("30/minute")
+async def oracle_profile(request: Request, node_id: str = ""):
+    """Get full oracle profile."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    if not node_id:
+        return {"ok": False, "detail": "Provide ?node_id=xxx"}
+    profile = oracle_ledger.get_oracle_profile(node_id)
+    return _redact_public_oracle_profile(
+        profile, authenticated=_scoped_view_authenticated(request, "mesh.audit"))
+
+
+@router.get("/api/mesh/oracle/predictions")
+@limiter.limit("30/minute")
+async def oracle_predictions(request: Request, node_id: str = ""):
+    """Get a node's active (unresolved) predictions."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    if not node_id:
+        return {"ok": False, "detail": "Provide ?node_id=xxx"}
+    active_predictions = oracle_ledger.get_active_predictions(node_id)
+    return _redact_public_oracle_predictions(
+        active_predictions, authenticated=_scoped_view_authenticated(request, "mesh.audit"))
+
+
+@router.post("/api/mesh/oracle/resolve-stakes")
+@limiter.limit("5/minute")
+@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
+async def oracle_resolve_stakes(request: Request):
+    """Resolve all expired stake contests."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    resolutions = oracle_ledger.resolve_expired_stakes()
+    return {"ok": True, "resolutions": resolutions, "count": len(resolutions)}

backend/routers/mesh_peer_sync.py

@@ -0,0 +1,235 @@
+import json as json_mod
+import logging
+from typing import Any
+from fastapi import APIRouter, Request, Response
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator, _verify_peer_push_hmac
+from services.config import get_settings
+from services.mesh.mesh_crypto import normalize_peer_url
+from services.mesh.mesh_router import peer_transport_kind
+from auth import _peer_hmac_url_from_request
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+_PEER_PUSH_BATCH_SIZE = 50
+
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _hydrate_gate_store_from_chain(events: list) -> int:
+    """Copy any gate_message chain events into the local gate_store for read/decrypt.
+
+    Only events that are resident in the local infonet (accepted or already
+    present) are hydrated.  The canonical infonet-resident event is used —
+    never the raw batch event — so a forged batch entry carrying a valid
+    event_id but attacker-chosen payload cannot pollute gate_store.
+    """
+    import copy
+    from services.mesh.mesh_hashchain import gate_store, infonet
+    count = 0
+    for evt in events:
+        if evt.get("event_type") != "gate_message":
+            continue
+        event_id = str(evt.get("event_id", "") or "").strip()
+        if not event_id or event_id not in infonet.event_index:
+            continue
+        canonical = infonet.events[infonet.event_index[event_id]]
+        payload = canonical.get("payload") or {}
+        gate_id = str(payload.get("gate", "") or "").strip()
+        if not gate_id:
+            continue
+        try:
+            gate_store.append(gate_id, copy.deepcopy(canonical))
+            count += 1
+        except Exception:
+            pass
+    return count
+
+
+@router.post("/api/mesh/infonet/peer-push")
+@limiter.limit("30/minute")
+async def infonet_peer_push(request: Request):
+    """Accept pushed Infonet events from relay peers (HMAC-authenticated)."""
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 524_288:
+                return Response(content='{"ok":false,"detail":"Request body too large (max 512KB)"}',
+                    status_code=413, media_type="application/json")
+        except (ValueError, TypeError):
+            pass
+    from services.mesh.mesh_hashchain import infonet
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json")
+    body = json_mod.loads(body_bytes or b"{}")
+    events = body.get("events", [])
+    if not isinstance(events, list):
+        return {"ok": False, "detail": "events must be a list"}
+    if len(events) > 50:
+        return {"ok": False, "detail": "Too many events in one push (max 50)"}
+    if not events:
+        return {"ok": True, "accepted": 0, "duplicates": 0, "rejected": []}
+    result = infonet.ingest_events(events)
+    _hydrate_gate_store_from_chain(events)
+    return {"ok": True, **result}
+
+
+@router.post("/api/mesh/gate/peer-push")
+@limiter.limit("30/minute")
+async def gate_peer_push(request: Request):
+    """Accept pushed gate events from relay peers (private plane)."""
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 524_288:
+                return Response(content='{"ok":false,"detail":"Request body too large"}',
+                    status_code=413, media_type="application/json")
+        except (ValueError, TypeError):
+            pass
+    from services.mesh.mesh_hashchain import gate_store
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json")
+    body = json_mod.loads(body_bytes or b"{}")
+    events = body.get("events", [])
+    if not isinstance(events, list):
+        return {"ok": False, "detail": "events must be a list"}
+    if len(events) > 50:
+        return {"ok": False, "detail": "Too many events (max 50)"}
+    if not events:
+        return {"ok": True, "accepted": 0, "duplicates": 0}
+    from services.mesh.mesh_hashchain import resolve_gate_wire_ref
+    # Sprint 3 / Rec #4: the gate_ref is HMACed with a key bound to the
+    # receiver's peer URL (the URL the push was delivered to). This is
+    # the same URL _verify_peer_push_hmac validated the X-Peer-HMAC
+    # header against, so we can trust it for ref resolution.
+    hop_peer_url = _peer_hmac_url_from_request(request)
+    grouped_events: dict[str, list] = {}
+    for evt in events:
+        evt_dict = evt if isinstance(evt, dict) else {}
+        payload = evt_dict.get("payload")
+        if not isinstance(payload, dict):
+            payload = {}
+        clean_event = {
+            "event_id": str(evt_dict.get("event_id", "") or ""),
+            "event_type": "gate_message",
+            "timestamp": evt_dict.get("timestamp", 0),
+            "node_id": str(evt_dict.get("node_id", "") or evt_dict.get("sender_id", "") or ""),
+            "sequence": evt_dict.get("sequence", 0),
+            "signature": str(evt_dict.get("signature", "") or ""),
+            "public_key": str(evt_dict.get("public_key", "") or ""),
+            "public_key_algo": str(evt_dict.get("public_key_algo", "") or ""),
+            "protocol_version": str(evt_dict.get("protocol_version", "") or ""),
+            "payload": {
+                "ciphertext": str(payload.get("ciphertext", "") or ""),
+                "format": str(payload.get("format", "") or ""),
+                "nonce": str(payload.get("nonce", "") or ""),
+                "sender_ref": str(payload.get("sender_ref", "") or ""),
+            },
+        }
+        epoch = _safe_int(payload.get("epoch", 0) or 0)
+        if epoch > 0:
+            clean_event["payload"]["epoch"] = epoch
+        envelope_hash_val = str(payload.get("envelope_hash", "") or "").strip()
+        gate_envelope_val = str(payload.get("gate_envelope", "") or "").strip()
+        reply_to_val = str(payload.get("reply_to", "") or "").strip()
+        if envelope_hash_val:
+            clean_event["payload"]["envelope_hash"] = envelope_hash_val
+        if gate_envelope_val:
+            clean_event["payload"]["gate_envelope"] = gate_envelope_val
+        if reply_to_val:
+            clean_event["payload"]["reply_to"] = reply_to_val
+        event_gate_id = str(payload.get("gate", "") or evt_dict.get("gate", "") or "").strip().lower()
+        if not event_gate_id:
+            event_gate_id = resolve_gate_wire_ref(
+                str(payload.get("gate_ref", "") or evt_dict.get("gate_ref", "") or ""),
+                clean_event,
+                peer_url=hop_peer_url,
+            )
+        if not event_gate_id:
+            return {"ok": False, "detail": "gate resolution failed"}
+        final_payload: dict[str, Any] = {
+            "gate": event_gate_id,
+            "ciphertext": clean_event["payload"]["ciphertext"],
+            "format": clean_event["payload"]["format"],
+            "nonce": clean_event["payload"]["nonce"],
+            "sender_ref": clean_event["payload"]["sender_ref"],
+        }
+        if epoch > 0:
+            final_payload["epoch"] = epoch
+        if clean_event["payload"].get("envelope_hash"):
+            final_payload["envelope_hash"] = clean_event["payload"]["envelope_hash"]
+        if clean_event["payload"].get("gate_envelope"):
+            final_payload["gate_envelope"] = clean_event["payload"]["gate_envelope"]
+        if clean_event["payload"].get("reply_to"):
+            final_payload["reply_to"] = clean_event["payload"]["reply_to"]
+        grouped_events.setdefault(event_gate_id, []).append({
+            "event_id": clean_event["event_id"],
+            "event_type": "gate_message",
+            "timestamp": clean_event["timestamp"],
+            "node_id": clean_event["node_id"],
+            "sequence": clean_event["sequence"],
+            "signature": clean_event["signature"],
+            "public_key": clean_event["public_key"],
+            "public_key_algo": clean_event["public_key_algo"],
+            "protocol_version": clean_event["protocol_version"],
+            "payload": final_payload,
+        })
+    accepted = 0
+    duplicates = 0
+    rejected = 0
+    for event_gate_id, items in grouped_events.items():
+        result = gate_store.ingest_peer_events(event_gate_id, items)
+        a = int(result.get("accepted", 0) or 0)
+        accepted += a
+        duplicates += int(result.get("duplicates", 0) or 0)
+        rejected += int(result.get("rejected", 0) or 0)
+    return {"ok": True, "accepted": accepted, "duplicates": duplicates, "rejected": rejected}
+
+
+@router.post("/api/mesh/gate/peer-pull")
+@limiter.limit("30/minute")
+async def gate_peer_pull(request: Request):
+    """Return gate events a peer is missing (HMAC-authenticated pull sync)."""
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 65_536:
+                return Response(content='{"ok":false,"detail":"Request body too large"}',
+                    status_code=413, media_type="application/json")
+        except (ValueError, TypeError):
+            pass
+    from services.mesh.mesh_hashchain import gate_store
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json")
+    body = json_mod.loads(body_bytes or b"{}")
+    gate_id = str(body.get("gate_id", "") or "").strip().lower()
+    after_count = _safe_int(body.get("after_count", 0) or 0)
+    if not gate_id:
+        gate_ids = gate_store.known_gate_ids()
+        gate_counts: dict[str, int] = {}
+        for gid in gate_ids:
+            with gate_store._lock:
+                gate_counts[gid] = len(gate_store._gates.get(gid, []))
+        return {"ok": True, "gates": gate_counts}
+    with gate_store._lock:
+        all_events = list(gate_store._gates.get(gate_id, []))
+    total = len(all_events)
+    if after_count >= total:
+        return {"ok": True, "events": [], "total": total, "gate_id": gate_id}
+    batch = all_events[after_count : after_count + _PEER_PUSH_BATCH_SIZE]
+    return {"ok": True, "events": batch, "total": total, "gate_id": gate_id}

backend/routers/radio.py

@@ -0,0 +1,91 @@
+from fastapi import APIRouter, Request, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+
+router = APIRouter()
+
+
+@router.get("/api/radio/top")
+@limiter.limit("30/minute")
+async def get_top_radios(request: Request):
+    from services.radio_intercept import get_top_broadcastify_feeds
+    return get_top_broadcastify_feeds()
+
+
+@router.get("/api/radio/openmhz/systems")
+@limiter.limit("30/minute")
+async def api_get_openmhz_systems(request: Request):
+    from services.radio_intercept import get_openmhz_systems
+    return get_openmhz_systems()
+
+
+@router.get("/api/radio/openmhz/calls/{sys_name}")
+@limiter.limit("60/minute")
+async def api_get_openmhz_calls(request: Request, sys_name: str):
+    from services.radio_intercept import get_recent_openmhz_calls
+    return get_recent_openmhz_calls(sys_name)
+
+
+@router.get("/api/radio/openmhz/audio")
+@limiter.limit("120/minute")
+async def api_get_openmhz_audio(request: Request, url: str = Query(..., min_length=10)):
+    from services.radio_intercept import openmhz_audio_response
+    return openmhz_audio_response(url)
+
+
+@router.get("/api/radio/nearest")
+@limiter.limit("60/minute")
+async def api_get_nearest_radio(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    from services.radio_intercept import find_nearest_openmhz_system
+    return find_nearest_openmhz_system(lat, lng)
+
+
+@router.get("/api/radio/nearest-list")
+@limiter.limit("60/minute")
+async def api_get_nearest_radios_list(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+    limit: int = Query(5, ge=1, le=20),
+):
+    from services.radio_intercept import find_nearest_openmhz_systems_list
+    return find_nearest_openmhz_systems_list(lat, lng, limit=limit)
+
+
+@router.get("/api/route/{callsign}")
+@limiter.limit("60/minute")
+async def get_flight_route(request: Request, callsign: str, lat: float = 0.0, lng: float = 0.0):
+    from services.network_utils import fetch_with_curl
+    r = fetch_with_curl(
+        "https://api.adsb.lol/api/0/routeset",
+        method="POST",
+        json_data={"planes": [{"callsign": callsign, "lat": lat, "lng": lng}]},
+        timeout=10,
+    )
+    if r and r.status_code == 200:
+        data = r.json()
+        route_list = []
+        if isinstance(data, dict):
+            route_list = data.get("value", [])
+        elif isinstance(data, list):
+            route_list = data
+
+        if route_list and len(route_list) > 0:
+            route = route_list[0]
+            airports = route.get("_airports", [])
+            if len(airports) >= 2:
+                orig = airports[0]
+                dest = airports[-1]
+                return {
+                    "orig_loc": [orig.get("lon", 0), orig.get("lat", 0)],
+                    "dest_loc": [dest.get("lon", 0), dest.get("lat", 0)],
+                    "origin_name": f"{orig.get('iata', '') or orig.get('icao', '')}: {orig.get('name', 'Unknown')}",
+                    "dest_name": f"{dest.get('iata', '') or dest.get('icao', '')}: {dest.get('name', 'Unknown')}",
+                }
+    return {}

backend/routers/sar.py

@@ -0,0 +1,260 @@
+"""SAR (Synthetic Aperture Radar) layer endpoints.
+
+Exposes:
+  - GET  /api/sar/status            — feature gates + signup links for the UI
+  - GET  /api/sar/anomalies         — Mode B pre-processed anomalies
+  - GET  /api/sar/scenes            — Mode A scene catalog
+  - GET  /api/sar/coverage          — per-AOI coverage and next-pass hints
+  - GET  /api/sar/aois              — operator-defined AOIs
+  - POST /api/sar/aois              — create or replace an AOI
+  - DELETE /api/sar/aois/{aoi_id}   — remove an AOI
+  - GET  /api/sar/near              — anomalies within radius_km of (lat, lon)
+
+The /status endpoint is the load-bearing UX: when Mode B is disabled it
+returns the structured help payload from sar_config.products_fetch_status()
+so the frontend can render in-app links to the free signup pages instead of
+making the user hunt around.
+"""
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+from pydantic import BaseModel, Field
+
+from auth import require_local_operator
+from limiter import limiter
+from services.fetchers._store import get_latest_data_subset_refs
+from services.sar.sar_aoi import (
+    SarAoi,
+    add_aoi,
+    haversine_km,
+    load_aois,
+    remove_aoi,
+)
+from services.sar.sar_config import (
+    catalog_enabled,
+    clear_runtime_credentials,
+    openclaw_enabled,
+    products_fetch_enabled,
+    products_fetch_status,
+    require_private_tier_for_publish,
+    set_runtime_credentials,
+)
+
+router = APIRouter()
+
+
+# ---------------------------------------------------------------------------
+# Status — the in-app onboarding hook
+# ---------------------------------------------------------------------------
+@router.get("/api/sar/status")
+@limiter.limit("60/minute")
+async def sar_status(request: Request) -> dict:
+    """Layer status + signup links.
+
+    The frontend calls this whenever the SAR panel is opened.  When Mode B
+    is off, the response includes a step-by-step ``help`` block with the
+    free signup URLs so the user can enable everything without leaving the
+    app.
+    """
+    products_status = products_fetch_status()
+    return {
+        "ok": True,
+        "catalog": {
+            "mode": "A",
+            "enabled": catalog_enabled(),
+            "needs_account": False,
+            "description": "Free Sentinel-1 scene catalog from ASF Search.",
+        },
+        "products": {
+            "mode": "B",
+            **products_status,
+        },
+        "openclaw_enabled": openclaw_enabled(),
+        "require_private_tier": require_private_tier_for_publish(),
+    }
+
+
+# ---------------------------------------------------------------------------
+# Data feeds
+# ---------------------------------------------------------------------------
+@router.get("/api/sar/anomalies")
+@limiter.limit("60/minute")
+async def sar_anomalies(
+    request: Request,
+    kind: str = Query("", description="Optional anomaly kind filter"),
+    aoi_id: str = Query("", description="Optional AOI id filter"),
+    limit: int = Query(200, ge=1, le=1000),
+) -> dict:
+    """Return the latest cached SAR anomalies (Mode B)."""
+    snap = get_latest_data_subset_refs("sar_anomalies")
+    items = list(snap.get("sar_anomalies") or [])
+    if kind:
+        items = [a for a in items if a.get("kind") == kind]
+    if aoi_id:
+        aoi_id = aoi_id.strip().lower()
+        items = [a for a in items if (a.get("stack_id") or "").lower() == aoi_id]
+    items = items[:limit]
+    return {
+        "ok": True,
+        "count": len(items),
+        "anomalies": items,
+        "products_enabled": products_fetch_enabled(),
+    }
+
+
+@router.get("/api/sar/scenes")
+@limiter.limit("60/minute")
+async def sar_scenes(
+    request: Request,
+    aoi_id: str = Query(""),
+    limit: int = Query(200, ge=1, le=1000),
+) -> dict:
+    """Return the latest cached scene catalog (Mode A)."""
+    snap = get_latest_data_subset_refs("sar_scenes")
+    items = list(snap.get("sar_scenes") or [])
+    if aoi_id:
+        aoi_id = aoi_id.strip().lower()
+        items = [s for s in items if (s.get("aoi_id") or "").lower() == aoi_id]
+    items = items[:limit]
+    return {
+        "ok": True,
+        "count": len(items),
+        "scenes": items,
+        "catalog_enabled": catalog_enabled(),
+    }
+
+
+@router.get("/api/sar/coverage")
+@limiter.limit("60/minute")
+async def sar_coverage(request: Request) -> dict:
+    """Per-AOI coverage and rough next-pass estimate."""
+    snap = get_latest_data_subset_refs("sar_aoi_coverage")
+    return {
+        "ok": True,
+        "coverage": list(snap.get("sar_aoi_coverage") or []),
+    }
+
+
+@router.get("/api/sar/near")
+@limiter.limit("60/minute")
+async def sar_near(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lon: float = Query(..., ge=-180, le=180),
+    radius_km: float = Query(50, ge=1, le=2000),
+    kind: str = Query(""),
+    limit: int = Query(50, ge=1, le=500),
+) -> dict:
+    """Return anomalies whose center sits within ``radius_km`` of (lat, lon)."""
+    snap = get_latest_data_subset_refs("sar_anomalies")
+    items = list(snap.get("sar_anomalies") or [])
+    matches = []
+    for a in items:
+        try:
+            a_lat = float(a.get("lat", 0.0))
+            a_lon = float(a.get("lon", 0.0))
+        except (TypeError, ValueError):
+            continue
+        d = haversine_km(lat, lon, a_lat, a_lon)
+        if d > radius_km:
+            continue
+        if kind and a.get("kind") != kind:
+            continue
+        a = dict(a)
+        a["distance_km"] = round(d, 2)
+        matches.append(a)
+    matches.sort(key=lambda x: x.get("distance_km", 0))
+    return {
+        "ok": True,
+        "count": len(matches[:limit]),
+        "anomalies": matches[:limit],
+    }
+
+
+# ---------------------------------------------------------------------------
+# AOI CRUD
+# ---------------------------------------------------------------------------
+@router.get("/api/sar/aois")
+@limiter.limit("60/minute")
+async def sar_aoi_list(request: Request) -> dict:
+    return {
+        "ok": True,
+        "aois": [a.to_dict() for a in load_aois(force=True)],
+    }
+
+
+class AoiPayload(BaseModel):
+    id: str = Field(..., min_length=1, max_length=64)
+    name: str = Field(..., min_length=1, max_length=120)
+    description: str = Field("", max_length=400)
+    center_lat: float = Field(..., ge=-90, le=90)
+    center_lon: float = Field(..., ge=-180, le=180)
+    radius_km: float = Field(25.0, ge=1.0, le=500.0)
+    category: str = Field("watchlist", max_length=40)
+    polygon: list[list[float]] | None = None
+
+
+@router.post("/api/sar/aois", dependencies=[Depends(require_local_operator)])
+@limiter.limit("20/minute")
+async def sar_aoi_upsert(request: Request, payload: AoiPayload) -> dict:
+    aoi = SarAoi(
+        id=payload.id.strip().lower(),
+        name=payload.name.strip(),
+        description=payload.description.strip(),
+        center_lat=payload.center_lat,
+        center_lon=payload.center_lon,
+        radius_km=payload.radius_km,
+        polygon=payload.polygon,
+        category=(payload.category or "watchlist").strip().lower(),
+    )
+    add_aoi(aoi)
+    return {"ok": True, "aoi": aoi.to_dict()}
+
+
+@router.delete("/api/sar/aois/{aoi_id}", dependencies=[Depends(require_local_operator)])
+@limiter.limit("20/minute")
+async def sar_aoi_delete(request: Request, aoi_id: str) -> dict:
+    removed = remove_aoi(aoi_id)
+    if not removed:
+        raise HTTPException(status_code=404, detail="AOI not found")
+    return {"ok": True, "removed": aoi_id}
+
+
+# ---------------------------------------------------------------------------
+# Mode B enable / disable — one-click setup from the frontend
+# ---------------------------------------------------------------------------
+class ModeBEnablePayload(BaseModel):
+    earthdata_user: str = Field("", max_length=120)
+    earthdata_token: str = Field(..., min_length=8, max_length=2048)
+    copernicus_user: str = Field("", max_length=120)
+    copernicus_token: str = Field("", max_length=2048)
+
+
+@router.post("/api/sar/mode-b/enable", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def sar_mode_b_enable(request: Request, payload: ModeBEnablePayload) -> dict:
+    """Store Earthdata (and optional Copernicus) credentials and flip both
+    two-step opt-in flags.  Returns the fresh status payload so the UI can
+    immediately reflect the change.
+    """
+    set_runtime_credentials(
+        earthdata_user=payload.earthdata_user,
+        earthdata_token=payload.earthdata_token,
+        copernicus_user=payload.copernicus_user,
+        copernicus_token=payload.copernicus_token,
+        mode_b_opt_in=True,
+    )
+    return {
+        "ok": True,
+        "products": products_fetch_status(),
+    }
+
+
+@router.post("/api/sar/mode-b/disable", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def sar_mode_b_disable(request: Request) -> dict:
+    """Wipe runtime credentials and revert to Mode A only."""
+    clear_runtime_credentials()
+    return {
+        "ok": True,
+        "products": products_fetch_status(),
+    }

backend/routers/sigint.py

@@ -0,0 +1,67 @@
+from fastapi import APIRouter, Request, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+from services.data_fetcher import get_latest_data
+
+router = APIRouter()
+
+
+@router.get("/api/oracle/region-intel")
+@limiter.limit("30/minute")
+async def oracle_region_intel(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Get oracle intelligence summary for a geographic region."""
+    from services.oracle_service import get_region_oracle_intel
+    news_items = get_latest_data().get("news", [])
+    return get_region_oracle_intel(lat, lng, news_items)
+
+
+@router.get("/api/thermal/verify")
+@limiter.limit("10/minute")
+async def thermal_verify(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+    radius_km: float = Query(10, ge=1, le=100),
+):
+    """On-demand thermal anomaly verification using Sentinel-2 SWIR bands."""
+    from services.thermal_sentinel import search_thermal_anomaly
+    result = search_thermal_anomaly(lat, lng, radius_km)
+    return result
+
+
+@router.post("/api/sigint/transmit")
+@limiter.limit("5/minute")
+async def sigint_transmit(request: Request):
+    """Send an APRS-IS message to a specific callsign. Requires ham radio credentials."""
+    from services.wormhole_supervisor import get_transport_tier
+    tier = get_transport_tier()
+    if str(tier or "").startswith("private_"):
+        return {"ok": False, "detail": "APRS transmit blocked in private transport mode"}
+    body = await request.json()
+    callsign = body.get("callsign", "")
+    passcode = body.get("passcode", "")
+    target = body.get("target", "")
+    message = body.get("message", "")
+    if not all([callsign, passcode, target, message]):
+        return {"ok": False, "detail": "Missing required fields: callsign, passcode, target, message"}
+    from services.sigint_bridge import send_aprs_message
+    return send_aprs_message(callsign, passcode, target, message)
+
+
+@router.get("/api/sigint/nearest-sdr")
+@limiter.limit("30/minute")
+async def nearest_sdr(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Find the nearest KiwiSDR receivers to a given coordinate."""
+    from services.sigint_bridge import find_nearest_kiwisdr
+    kiwisdr_data = get_latest_data().get("kiwisdr", [])
+    return find_nearest_kiwisdr(lat, lng, kiwisdr_data)

backend/routers/tools.py

@@ -0,0 +1,303 @@
+import asyncio
+import logging
+import math
+from typing import Any
+from fastapi import APIRouter, Request, Query, Depends, HTTPException, Response
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _safe_float(val, default=0.0):
+    try:
+        parsed = float(val)
+        if not math.isfinite(parsed):
+            return default
+        return parsed
+    except (TypeError, ValueError):
+        return default
+
+
+class ShodanSearchRequest(BaseModel):
+    query: str
+    page: int = 1
+    facets: list[str] = []
+
+
+class ShodanCountRequest(BaseModel):
+    query: str
+    facets: list[str] = []
+
+
+class ShodanHostRequest(BaseModel):
+    ip: str
+    history: bool = False
+
+
+@router.get("/api/region-dossier")
+@limiter.limit("30/minute")
+def api_region_dossier(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Sync def so FastAPI runs it in a threadpool — prevents blocking the event loop."""
+    from services.region_dossier import get_region_dossier
+    return get_region_dossier(lat, lng)
+
+
+@router.get("/api/geocode/search")
+@limiter.limit("30/minute")
+async def api_geocode_search(
+    request: Request,
+    q: str = "",
+    limit: int = 5,
+    local_only: bool = False,
+):
+    from services.geocode import search_geocode
+    if not q or len(q.strip()) < 2:
+        return {"results": [], "query": q, "count": 0}
+    results = await asyncio.to_thread(search_geocode, q, limit, local_only)
+    return {"results": results, "query": q, "count": len(results)}
+
+
+@router.get("/api/geocode/reverse")
+@limiter.limit("60/minute")
+async def api_geocode_reverse(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+    local_only: bool = False,
+):
+    from services.geocode import reverse_geocode
+    return await asyncio.to_thread(reverse_geocode, lat, lng, local_only)
+
+
+@router.get("/api/sentinel2/search")
+@limiter.limit("30/minute")
+def api_sentinel2_search(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Search for latest Sentinel-2 imagery at a point. Sync for threadpool execution."""
+    from services.sentinel_search import search_sentinel2_scene
+    return search_sentinel2_scene(lat, lng)
+
+
+@router.post("/api/sentinel/token")
+@limiter.limit("60/minute")
+async def api_sentinel_token(request: Request):
+    """Proxy Copernicus CDSE OAuth2 token request (avoids browser CORS block)."""
+    import requests as req
+    body = await request.body()
+    from urllib.parse import parse_qs
+    params = parse_qs(body.decode("utf-8"))
+    client_id = params.get("client_id", [""])[0]
+    client_secret = params.get("client_secret", [""])[0]
+    if not client_id or not client_secret:
+        raise HTTPException(400, "client_id and client_secret required")
+    token_url = "https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token"
+    try:
+        resp = await asyncio.to_thread(req.post, token_url,
+            data={"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret},
+            timeout=15)
+        return Response(content=resp.content, status_code=resp.status_code, media_type="application/json")
+    except Exception:
+        logger.exception("Token request failed")
+        raise HTTPException(502, "Token request failed")
+
+
+_sh_token_cache: dict = {"token": None, "expiry": 0, "client_id": ""}
+
+
+@router.post("/api/sentinel/tile")
+@limiter.limit("300/minute")
+async def api_sentinel_tile(request: Request):
+    """Proxy Sentinel Hub Process API tile request (avoids CORS block)."""
+    import requests as req
+    import time as _time
+    try:
+        body = await request.json()
+    except Exception:
+        return JSONResponse(status_code=422, content={"ok": False, "detail": "invalid JSON body"})
+
+    client_id = body.get("client_id", "")
+    client_secret = body.get("client_secret", "")
+    preset = body.get("preset", "TRUE-COLOR")
+    date_str = body.get("date", "")
+    z = body.get("z", 0)
+    x = body.get("x", 0)
+    y = body.get("y", 0)
+
+    if not client_id or not client_secret or not date_str:
+        raise HTTPException(400, "client_id, client_secret, and date required")
+
+    now = _time.time()
+    if (_sh_token_cache["token"] and _sh_token_cache["client_id"] == client_id
+            and now < _sh_token_cache["expiry"] - 30):
+        token = _sh_token_cache["token"]
+    else:
+        token_url = "https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token"
+        try:
+            tresp = await asyncio.to_thread(req.post, token_url,
+                data={"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret},
+                timeout=15)
+            if tresp.status_code != 200:
+                raise HTTPException(401, f"Token auth failed: {tresp.text[:200]}")
+            tdata = tresp.json()
+            token = tdata["access_token"]
+            _sh_token_cache["token"] = token
+            _sh_token_cache["expiry"] = now + tdata.get("expires_in", 300)
+            _sh_token_cache["client_id"] = client_id
+        except HTTPException:
+            raise
+        except Exception:
+            logger.exception("Token request failed")
+            raise HTTPException(502, "Token request failed")
+
+    half = 20037508.342789244
+    tile_size = (2 * half) / math.pow(2, z)
+    min_x = -half + x * tile_size
+    max_x = min_x + tile_size
+    max_y = half - y * tile_size
+    min_y = max_y - tile_size
+    bbox = [min_x, min_y, max_x, max_y]
+
+    evalscripts = {
+        "TRUE-COLOR": '//VERSION=3\nfunction setup(){return{input:["B04","B03","B02"],output:{bands:3}};}\nfunction evaluatePixel(s){return[2.5*s.B04,2.5*s.B03,2.5*s.B02];}',
+        "FALSE-COLOR": '//VERSION=3\nfunction setup(){return{input:["B08","B04","B03"],output:{bands:3}};}\nfunction evaluatePixel(s){return[2.5*s.B08,2.5*s.B04,2.5*s.B03];}',
+        "NDVI": '//VERSION=3\nfunction setup(){return{input:["B04","B08"],output:{bands:3}};}\nfunction evaluatePixel(s){var n=(s.B08-s.B04)/(s.B08+s.B04);if(n<-0.2)return[0.05,0.05,0.05];if(n<0)return[0.75,0.75,0.75];if(n<0.1)return[0.86,0.86,0.86];if(n<0.2)return[0.92,0.84,0.68];if(n<0.3)return[0.77,0.88,0.55];if(n<0.4)return[0.56,0.80,0.32];if(n<0.5)return[0.35,0.72,0.18];if(n<0.6)return[0.20,0.60,0.08];if(n<0.7)return[0.10,0.48,0.04];return[0.0,0.36,0.0];}',
+        "MOISTURE-INDEX": '//VERSION=3\nfunction setup(){return{input:["B8A","B11"],output:{bands:3}};}\nfunction evaluatePixel(s){var m=(s.B8A-s.B11)/(s.B8A+s.B11);var r=Math.max(0,Math.min(1,1.5-3*m));var g=Math.max(0,Math.min(1,m<0?1.5+3*m:1.5-3*m));var b=Math.max(0,Math.min(1,1.5+3*(m-0.5)));return[r,g,b];}',
+    }
+    evalscript = evalscripts.get(preset, evalscripts["TRUE-COLOR"])
+
+    from datetime import datetime as _dt, timedelta as _td
+    try:
+        end_date = _dt.strptime(date_str, "%Y-%m-%d")
+    except ValueError:
+        end_date = _dt.utcnow()
+
+    if z <= 6:
+        lookback_days = 30
+    elif z <= 9:
+        lookback_days = 14
+    elif z <= 11:
+        lookback_days = 7
+    else:
+        lookback_days = 5
+
+    start_date = end_date - _td(days=lookback_days)
+
+    process_body = {
+        "input": {
+            "bounds": {"bbox": bbox, "properties": {"crs": "http://www.opengis.net/def/crs/EPSG/0/3857"}},
+            "data": [{"type": "sentinel-2-l2a", "dataFilter": {
+                "timeRange": {
+                    "from": start_date.strftime("%Y-%m-%dT00:00:00Z"),
+                    "to": end_date.strftime("%Y-%m-%dT23:59:59Z"),
+                },
+                "maxCloudCoverage": 30, "mosaickingOrder": "leastCC",
+            }}],
+        },
+        "output": {"width": 256, "height": 256,
+            "responses": [{"identifier": "default", "format": {"type": "image/png"}}]},
+        "evalscript": evalscript,
+    }
+    try:
+        resp = await asyncio.to_thread(req.post,
+            "https://sh.dataspace.copernicus.eu/api/v1/process",
+            json=process_body,
+            headers={"Authorization": f"Bearer {token}", "Accept": "image/png"},
+            timeout=30)
+        return Response(content=resp.content, status_code=resp.status_code,
+            media_type=resp.headers.get("content-type", "image/png"))
+    except Exception:
+        logger.exception("Process API failed")
+        raise HTTPException(502, "Process API failed")
+
+
+@router.get("/api/tools/shodan/status", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_shodan_status(request: Request):
+    from services.shodan_connector import get_shodan_connector_status
+    return get_shodan_connector_status()
+
+
+@router.post("/api/tools/shodan/search", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_shodan_search(request: Request, body: ShodanSearchRequest):
+    from services.shodan_connector import ShodanConnectorError, search_shodan
+    try:
+        return search_shodan(body.query, page=body.page, facets=body.facets)
+    except ShodanConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/shodan/count", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_shodan_count(request: Request, body: ShodanCountRequest):
+    from services.shodan_connector import ShodanConnectorError, count_shodan
+    try:
+        return count_shodan(body.query, facets=body.facets)
+    except ShodanConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/shodan/host", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_shodan_host(request: Request, body: ShodanHostRequest):
+    from services.shodan_connector import ShodanConnectorError, lookup_shodan_host
+    try:
+        return lookup_shodan_host(body.ip, history=body.history)
+    except ShodanConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.get("/api/tools/uw/status", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_uw_status(request: Request):
+    from services.unusual_whales_connector import get_uw_status
+    return get_uw_status()
+
+
+@router.post("/api/tools/uw/congress", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_uw_congress(request: Request):
+    from services.unusual_whales_connector import FinnhubConnectorError, fetch_congress_trades
+    try:
+        return fetch_congress_trades()
+    except FinnhubConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/uw/darkpool", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_uw_darkpool(request: Request):
+    from services.unusual_whales_connector import FinnhubConnectorError, fetch_insider_transactions
+    try:
+        return fetch_insider_transactions()
+    except FinnhubConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/uw/flow", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_uw_flow(request: Request):
+    from services.unusual_whales_connector import FinnhubConnectorError, fetch_defense_quotes
+    try:
+        return fetch_defense_quotes()
+    except FinnhubConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc