Harden Infonet DM address flow and seed sync

Allow local-operator DM invite import without requiring a full admin session.

Prioritize bundled/bootstrap seed peers and shorten stale seed cooldowns for faster Infonet recovery.

Replace raw DM invite dumps with copyable signed-address controls, contact request handling, and safer sealed-send behavior while the private delivery route connects.

.dockerignore

@@ -0,0 +1,56 @@
+# Exclude build artifacts, caches, and large directories from Docker context
+.git/
+.git_backup/
+node_modules/
+.next/
+__pycache__/
+*.pyc
+venv/
+.venv/
+.ruff_cache/
+local-artifacts/
+release-secrets/
+
+# Never send local configuration or credentials into Docker builds
+.env
+.env.*
+**/.env
+**/.env.*
+*.pem
+*.key
+*.p12
+*.pfx
+
+# privacy-core build caches (source is needed, artifacts are not)
+privacy-core/target/
+privacy-core/target-test/
+privacy-core/.codex-tmp/
+
+# Large data/cache files
+*.db
+*.sqlite
+*.xlsx
+*.log
+extra/
+prototype/
+
+# Runtime state generated by local backend runs
+backend/.pytest_cache/
+backend/.ruff_cache/
+backend/backend.egg-info/
+backend/build/
+backend/node_modules/
+backend/timemachine/
+backend/venv/
+backend/data/*cache*.json
+backend/data/**/*cache*.json
+backend/data/wormhole*.json
+backend/data/**/wormhole*.json
+backend/data/dm_*.json
+backend/data/**/dm_*.json
+backend/data/**/peer_store.json
+backend/data/**/node.json
+backend/data/*.log
+backend/data/**/*.log
+backend/data/*.key
+backend/data/**/*.key

.env.example

@@ -3,14 +3,135 @@
 #   cp .env.example .env
 
 # ── Required for backend container ─────────────────────────────
+# OpenSky Network OAuth2 — REQUIRED for airplane telemetry.
+# Free registration at https://opensky-network.org/index.php?option=com_users&view=registration
+# Without these the flights layer falls back to ADS-B-only with major gaps in Africa, Asia, and LatAm.
 OPENSKY_CLIENT_ID=
 OPENSKY_CLIENT_SECRET=
 AIS_API_KEY=
 
+# Admin key to protect sensitive endpoints (settings, updates).
+# If blank, loopback/localhost requests still work for local single-host dev.
+# Remote/non-loopback admin access requires ADMIN_KEY, or ALLOW_INSECURE_ADMIN=true in debug-only setups.
+ADMIN_KEY=
+
+# Allow insecure admin access without ADMIN_KEY (local dev only, beyond loopback).
+# Requires MESH_DEBUG_MODE=true on the backend; do not enable this for normal use.
+# ALLOW_INSECURE_ADMIN=false
+
+# User-Agent for Nominatim geocoding requests (per OSM usage policy).
+# NOMINATIM_USER_AGENT=ShadowBroker/1.0 (https://github.com/BigBodyCobain/Shadowbroker)
+
 # ── Optional ───────────────────────────────────────────────────
 
 # LTA (Singapore traffic cameras) — leave blank to skip
 # LTA_ACCOUNT_KEY=
 
+# NASA FIRMS country-scoped fire data — enriches global CSV with conflict-zone hotspots.
+# Free MAP_KEY from https://firms.modaps.eosdis.nasa.gov/
+# FIRMS_MAP_KEY=
+
+# Ukraine air raid alerts — free token from https://alerts.in.ua/
+# ALERTS_IN_UA_TOKEN=
+
+# Optional NUFORC UAP sighting map enrichment via Mapbox Tilequery.
+# Leave blank to skip this optional enrichment.
+# NUFORC_MAPBOX_TOKEN=
+
+# Optional startup-risk controls.
+# On Windows, external curl fallback and the Playwright LiveUAMap scraper are
+# disabled by default so blocked upstream feeds cannot interrupt start.bat.
+# SHADOWBROKER_ENABLE_WINDOWS_CURL_FALLBACK=false
+# SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=false
+# AIS starts by default when AIS_API_KEY is set. Set to 0/false to force-disable.
+# SHADOWBROKER_ENABLE_AIS_STREAM_PROXY=true
+# Minimum visible satellite catalog before forcing a CelesTrak refresh.
+# SHADOWBROKER_MIN_VISIBLE_SATELLITES=350
+# Upper bound for TLE fallback satellite search when CelesTrak is unreachable.
+# SHADOWBROKER_MAX_VISIBLE_SATELLITES=450
+# NUFORC fallback uses the Hugging Face mirror when live NUFORC is unavailable.
+# NUFORC_HF_FALLBACK_LIMIT=250
+# NUFORC_HF_GEOCODE_LIMIT=150
+
+# First-paint cache age budgets. These let the map and Global Threat Intercept
+# paint from the last local snapshot while live feeds refresh in the background.
+# FAST_STARTUP_CACHE_MAX_AGE_S=21600
+# INTEL_STARTUP_CACHE_MAX_AGE_S=21600
+
+# Docker resource tuning. The backend synthesizes large geospatial feeds; keep
+# this at 4G or higher on hosts that run AIS, OpenSky, CCTV, satellites, and
+# threat feeds together. Lower caps can cause Docker OOM restarts and empty
+# slow layers such as news, UAP sightings, military bases, and wastewater.
+# BACKEND_MEMORY_LIMIT=4G
+# SHADOWBROKER_FETCH_WORKERS=8
+# SHADOWBROKER_SLOW_FETCH_CONCURRENCY=4
+# SHADOWBROKER_STARTUP_HEAVY_CONCURRENCY=2
+
+# Infonet bootstrap/sync responsiveness. Defaults favor fast seed failure
+# detection so stale onion peers do not make the terminal look hung.
+# MESH_SYNC_TIMEOUT_S=5
+# MESH_SYNC_MAX_PEERS_PER_CYCLE=3
+# MESH_BOOTSTRAP_SEED_FAILURE_COOLDOWN_S=15
+
+# Google Earth Engine for VIIRS night lights change detection (optional).
+# pip install earthengine-api
+# GEE_SERVICE_ACCOUNT_KEY=
+
 # Override the backend URL the frontend uses (leave blank for auto-detect)
 # NEXT_PUBLIC_API_URL=http://192.168.1.50:8000
+
+# ── Mesh / Reticulum (RNS) ─────────────────────────────────────
+# MESH_RNS_ENABLED=false
+# MESH_RNS_APP_NAME=shadowbroker
+# MESH_RNS_ASPECT=infonet
+# MESH_RNS_IDENTITY_PATH=
+# MESH_RNS_PEERS=
+# MESH_RNS_DANDELION_HOPS=2
+# MESH_RNS_DANDELION_DELAY_MS=400
+# MESH_RNS_CHURN_INTERVAL_S=300
+# MESH_RNS_MAX_PEERS=32
+# MESH_RNS_MAX_PAYLOAD=8192
+# MESH_RNS_PEER_BUCKET_PREFIX=4
+# MESH_RNS_MAX_PEERS_PER_BUCKET=4
+# MESH_RNS_PEER_FAIL_THRESHOLD=3
+# MESH_RNS_PEER_COOLDOWN_S=300
+# MESH_RNS_SHARD_ENABLED=false
+# MESH_RNS_SHARD_DATA_SHARDS=3
+# MESH_RNS_SHARD_PARITY_SHARDS=1
+# MESH_RNS_SHARD_TTL_S=30
+# MESH_RNS_FEC_CODEC=xor
+# MESH_RNS_BATCH_MS=200
+# MESH_RNS_COVER_INTERVAL_S=0
+# MESH_RNS_COVER_SIZE=64
+# MESH_RNS_IBF_WINDOW=256
+# MESH_RNS_IBF_TABLE_SIZE=64
+# MESH_RNS_IBF_MINHASH_SIZE=16
+# MESH_RNS_IBF_MINHASH_THRESHOLD=0.25
+# MESH_RNS_IBF_WINDOW_JITTER=32
+# MESH_RNS_IBF_INTERVAL_S=120
+# MESH_RNS_IBF_SYNC_PEERS=3
+# MESH_RNS_IBF_QUORUM_TIMEOUT_S=6
+# MESH_RNS_IBF_MAX_REQUEST_IDS=64
+# MESH_RNS_IBF_MAX_EVENTS=64
+# MESH_RNS_SESSION_ROTATE_S=0
+# MESH_RNS_IBF_FAIL_THRESHOLD=3
+# MESH_RNS_IBF_COOLDOWN_S=120
+# MESH_VERIFY_INTERVAL_S=600
+# MESH_VERIFY_SIGNATURES=false
+
+# ── Mesh DM Relay ──────────────────────────────────────────────
+# MESH_DM_TOKEN_PEPPER=change-me
+# Optional local-dev DM root external assurance bridge.
+# These stay commented because they are machine-local file paths, not safe global defaults.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH=backend/../ops/root_witness_receipt_import.json
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH=backend/../ops/root_transparency_ledger.json
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI=backend/../ops/root_transparency_ledger.json
+
+# ── Self Update ────────────────────────────────────────────────
+# MESH_UPDATE_SHA256=
+
+# ── Wormhole (Local Agent) ─────────────────────────────────────
+# WORMHOLE_URL=http://127.0.0.1:8787
+# WORMHOLE_TRANSPORT=direct
+# WORMHOLE_SOCKS_PROXY=127.0.0.1:9050
+# WORMHOLE_SOCKS_DNS=true

.gitattributes

@@ -0,0 +1,2 @@
+# Force LF line endings for shell scripts
+*.sh text eol=lf

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "npm"
+    directory: "/frontend"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "pip"
+    directory: "/backend"
+    schedule:
+      interval: "weekly"

.github/workflows/ci.yml

@@ -0,0 +1,60 @@
+name: CI - Lint & Test
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_call:
+
+jobs:
+  frontend:
+    name: Frontend Tests & Build
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: frontend
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+          cache-dependency-path: frontend/package-lock.json
+      - run: npm ci
+      - run: npm run lint
+      - run: npm run format:check
+      - run: npx vitest run --reporter=verbose
+      - run: npm run build
+      - run: npm run bundle:report
+
+  backend:
+    name: Backend Lint & Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run secret scan
+        run: bash backend/scripts/scan-secrets.sh --all
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: cd backend && uv sync --frozen --group dev
+      - run: cd backend && uv run ruff check .
+      - run: cd backend && uv run black --check .
+      - run: cd backend && uv run python -c "from services.fetchers.retry import with_retry; from services.env_check import validate_env; print('Module imports OK')"
+      - name: Run release smoke tests
+        run: |
+          cd backend
+          uv run pytest \
+            tests/mesh/test_mesh_node_bootstrap_runtime.py \
+            tests/mesh/test_mesh_infonet_sync_support.py \
+            tests/mesh/test_mesh_canonical.py \
+            tests/mesh/test_mesh_merkle.py \
+            tests/test_release_helper.py \
+            -v --tb=short

.github/workflows/docker-publish.yml

@@ -9,17 +9,20 @@ on:
 
 env:
   REGISTRY: ghcr.io
-  # github.repository as <account>/<repo>
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
+  ci-gate:
+    name: CI Gate
+    uses: ./.github/workflows/ci.yml
+
   build-frontend:
+    needs: ci-gate
     runs-on: ${{ matrix.runner }}
     permissions:
       contents: read
       packages: write
       id-token: write
-
     strategy:
       fail-fast: false
       matrix:
@@ -28,33 +31,23 @@ jobs:
             runner: ubuntu-latest
           - platform: linux/arm64
             runner: ubuntu-24.04-arm
-
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
+      - uses: actions/checkout@v4
       - name: Lowercase image name
         run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.0.0
-
-      - name: Log into registry ${{ env.REGISTRY }}
+      - uses: docker/setup-buildx-action@v3.0.0
+      - name: Log into registry
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v3.0.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract Docker metadata
-        id: meta
+      - id: meta
         uses: docker/metadata-action@v5.0.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend
-
-      - name: Build and push Docker image by digest
-        id: build
+      - id: build
         uses: docker/build-push-action@v5.0.0
         with:
           context: ./frontend
@@ -64,17 +57,14 @@ jobs:
           cache-from: type=gha,scope=frontend-${{ matrix.platform }}
           cache-to: type=gha,mode=max,scope=frontend-${{ matrix.platform }}
           outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend,push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }}
-
       - name: Export digest
         if: github.event_name != 'pull_request'
         run: |
           mkdir -p /tmp/digests/frontend
           digest="${{ steps.build.outputs.digest }}"
           touch "/tmp/digests/frontend/${digest#sha256:}"
-
-      - name: Upload digest
+      - uses: actions/upload-artifact@v4
         if: github.event_name != 'pull_request'
-        uses: actions/upload-artifact@v4
         with:
           name: digests-frontend-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
           path: /tmp/digests/frontend/*
@@ -82,36 +72,27 @@ jobs:
           retention-days: 1
 
   merge-frontend:
-    runs-on: ubuntu-latest
     if: github.event_name != 'pull_request'
     needs: build-frontend
+    runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
-
     steps:
       - name: Lowercase image name
         run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
-
-      - name: Download digests
-        uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v4
         with:
           path: /tmp/digests/frontend
           pattern: digests-frontend-*
           merge-multiple: true
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.0.0
-
-      - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v3.0.0
+      - uses: docker/setup-buildx-action@v3.0.0
+      - uses: docker/login-action@v3.0.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract Docker metadata
-        id: meta
+      - id: meta
         uses: docker/metadata-action@v5.0.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend
@@ -119,7 +100,6 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=raw,value=latest,enable={{is_default_branch}}
-
       - name: Create and push manifest
         working-directory: /tmp/digests/frontend
         run: |
@@ -128,12 +108,12 @@ jobs:
             $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend@sha256:%s ' *)
 
   build-backend:
+    needs: ci-gate
     runs-on: ${{ matrix.runner }}
     permissions:
       contents: read
       packages: write
       id-token: write
-
     strategy:
       fail-fast: false
       matrix:
@@ -142,53 +122,41 @@ jobs:
             runner: ubuntu-latest
           - platform: linux/arm64
             runner: ubuntu-24.04-arm
-
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
+      - uses: actions/checkout@v4
       - name: Lowercase image name
         run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.0.0
-
-      - name: Log into registry ${{ env.REGISTRY }}
+      - uses: docker/setup-buildx-action@v3.0.0
+      - name: Log into registry
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v3.0.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract Docker metadata
-        id: meta
+      - id: meta
         uses: docker/metadata-action@v5.0.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend
-
-      - name: Build and push Docker image by digest
-        id: build
+      - id: build
         uses: docker/build-push-action@v5.0.0
         with:
-          context: ./backend
+          context: .
+          file: ./backend/Dockerfile
           platforms: ${{ matrix.platform }}
           push: ${{ github.event_name != 'pull_request' }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha,scope=backend-${{ matrix.platform }}
           cache-to: type=gha,mode=max,scope=backend-${{ matrix.platform }}
           outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend,push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }}
-
       - name: Export digest
         if: github.event_name != 'pull_request'
         run: |
           mkdir -p /tmp/digests/backend
           digest="${{ steps.build.outputs.digest }}"
           touch "/tmp/digests/backend/${digest#sha256:}"
-
-      - name: Upload digest
+      - uses: actions/upload-artifact@v4
         if: github.event_name != 'pull_request'
-        uses: actions/upload-artifact@v4
         with:
           name: digests-backend-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
           path: /tmp/digests/backend/*
@@ -196,36 +164,27 @@ jobs:
           retention-days: 1
 
   merge-backend:
-    runs-on: ubuntu-latest
     if: github.event_name != 'pull_request'
     needs: build-backend
+    runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
-
     steps:
       - name: Lowercase image name
         run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
-
-      - name: Download digests
-        uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v4
         with:
           path: /tmp/digests/backend
           pattern: digests-backend-*
           merge-multiple: true
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.0.0
-
-      - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v3.0.0
+      - uses: docker/setup-buildx-action@v3.0.0
+      - uses: docker/login-action@v3.0.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract Docker metadata
-        id: meta
+      - id: meta
         uses: docker/metadata-action@v5.0.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend
@@ -233,7 +192,6 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=raw,value=latest,enable={{is_default_branch}}
-
       - name: Create and push manifest
         working-directory: /tmp/digests/backend
         run: |

.gitignore

@@ -6,13 +6,32 @@ node_modules/
 venv/
 env/
 .venv/
+backend/.venv-dir
+backend/venv-repair*/
+backend/.venv-repair*/
 
 # Environment Variables & Secrets
 .env
+.envrc
 .env.local
 .env.development.local
 .env.test.local
 .env.production.local
+.npmrc
+.pypirc
+.netrc
+*.pem
+*.key
+*.crt
+*.csr
+*.p12
+*.pfx
+id_rsa
+id_rsa.*
+id_ed25519
+id_ed25519.*
+known_hosts
+authorized_keys
 
 # Python caches & compiled files
 __pycache__/
@@ -20,18 +39,59 @@ __pycache__/
 *$py.class
 *.so
 .Python
+.ruff_cache/
+.pytest_cache/
+.mypy_cache/
+.hypothesis/
+.tox/
 
 # Next.js build output
 .next/
 out/
 build/
+*.tsbuildinfo
 
-# Application Specific Caches & DBs
+# Deprecated standalone Infonet Terminal skeleton (migrated into frontend/src/components/InfonetTerminal/)
+frontend/infonet-terminal/
+
+# Rust build artifacts (privacy-core)
+target/
+target-test/
+
+# ========================
+# LOCAL-ONLY: extra/ folder
+# ========================
+# All internal docs, planning files, raw data, backups, and dev scratch
+# live here. NEVER commit this folder.
+extra/
+
+# ========================
+# Application caches & runtime DBs (regenerate on startup)
+# ========================
 backend/ais_cache.json
 backend/carrier_cache.json
 backend/cctv.db
+cctv.db
+*.db
+*.sqlite
 *.sqlite3
 
+# ========================
+# backend/data/ — blanket ignore, whitelist static reference files
+# ========================
+# Everything in data/ is runtime-generated state (encrypted keys,
+# MLS bindings, relay spools, caches) and MUST NOT be committed.
+# Only static reference datasets that ship with the repo are whitelisted.
+backend/data/*
+!backend/data/datacenters.json
+!backend/data/datacenters_geocoded.json
+!backend/data/military_bases.json
+!backend/data/plan_ccg_vessels.json
+!backend/data/plane_alert_db.json
+!backend/data/power_plants.json
+!backend/data/tracked_names.json
+!backend/data/yacht_alert_db.json
+
 # OS generated files
 .DS_Store
 .DS_Store?
@@ -53,17 +113,24 @@ Thumbs.db
 # Vercel / Deployment
 .vercel
 
-# Temp files
+# ========================
+# Temp / scratch / debug files
+# ========================
 tmp/
 *.log
 *.tmp
 *.bak
+*.swp
+*.swo
 out.txt
 out_sys.txt
 rss_output.txt
 merged.txt
 tmp_fast.json
-TheAirTraffic Database.xlsx
+diff.txt
+local_diff.txt
+map_diff.txt
+TERMINAL
 
 # Debug dumps & release artifacts
 backend/dump.json
@@ -72,22 +139,61 @@ backend/nyc_sample.json
 backend/nyc_full.json
 backend/liveua_test.html
 backend/out_liveua.json
+backend/out.json
+backend/temp.json
+backend/seattle_sample.json
+backend/sgp_sample.json
+backend/wsdot_sample.json
+backend/xlsx_analysis.txt
 frontend/server_logs*.txt
 frontend/cctv.db
+frontend/eslint-report.json
 *.zip
-.git_backup/
+*.tar.gz
+*.xlsx
 
-# Test files (may contain hardcoded keys)
+# Old backups & repo clones
+.git_backup/
+local-artifacts/
+release-secrets/
+shadowbroker_repo/
+frontend/src/components.bak/
+frontend/src/components/map/icons/backups/
+
+# Coverage
+coverage/
+.coverage
+.coverage.*
+dist/
+
+# Test scratch files (not in tests/ folder)
 backend/test_*.py
 backend/services/test_*.py
 
 # Local analysis & dev tools
 backend/analyze_xlsx.py
-backend/xlsx_analysis.txt
 backend/services/ais_cache.json
+graphify/
+graphify-out/
 
-# Internal update tracking (not for repo)
+# ========================
+# Internal docs & brainstorming (never commit)
+# ========================
+docs/*
+!docs/mesh/
+docs/mesh/*
+!docs/mesh/threat-model.md
+!docs/mesh/claims-reconciliation.md
+!docs/mesh/mesh-canonical-fixtures.json
+!docs/mesh/mesh-merkle-fixtures.json
+!docs/mesh/wormhole-dm-root-operations-runbook.md
+.local-docs/
+infonet-economy/
 updatestuff.md
+ROADMAP.md
+UPDATEPROTOCOL.md
+CLAUDE.md
+DOCKER_SECRETS.md
 
 # Misc dev artifacts
 clean_zip.py
@@ -95,5 +201,45 @@ zip_repo.py
 refactor_cesium.py
 jobs.json
 
+# Claude / AI
 .claude
 .mise.local.toml
+.codex-tmp/
+prototype/
+.runtime/
+
+# ========================
+# Runtime state & operator-local data (never commit)
+# ========================
+# TimeMachine snapshot cache — regenerated at runtime, can be 100 MB+
+backend/timemachine/
+# Operator witness keys, identity material, transparency ledgers (machine-local)
+ops/
+# Runtime DM relay state
+dm_relay.json
+# Dev scratch notes
+improvements.txt
+
+# ========================
+# Custody verification temp dirs (runtime test artifacts with private keys!)
+# ========================
+backend/sb-custody-verify-*/
+
+# Python egg-info (build artifact, regenerated by pip install -e)
+*.egg-info/
+
+# Privacy-core debug build (Windows DLL, 3.6 MB, not shipped)
+privacy-core/debug/
+
+# Desktop-shell export stash dirs (empty temp dirs from Tauri build)
+frontend/.desktop-export-stash-*/
+
+# Wormhole logs (can be 30 MB+ each, runtime-generated)
+backend/data/wormhole_stderr.log
+backend/data/wormhole_stdout.log
+
+# Runtime caches that already slip through the backend/data/* blanket
+# (these are caught by the wildcard but listing for clarity)
+
+# Compressed snapshot archives (can be 100 MB+)
+*.json.gz

.pre-commit-config.yaml

@@ -0,0 +1,32 @@
+repos:
+  - repo: local
+    hooks:
+      - id: shadowbroker-secret-scan
+        name: ShadowBroker secret scan
+        entry: bash backend/scripts/scan-secrets.sh --staged
+        language: system
+        pass_filenames: false
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-yaml
+      - id: check-json
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.9
+    hooks:
+      - id: ruff
+        args: ["--fix"]
+
+  - repo: https://github.com/psf/black
+    rev: 25.1.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.3.3
+    hooks:
+      - id: prettier

.python-version

@@ -0,0 +1 @@
+3.10

DATA-ATTRIBUTION.md

@@ -0,0 +1,71 @@
+# Data Attribution & Licensing
+
+ShadowBroker aggregates publicly available data from many third-party sources.
+This file documents each source and its license so operators and users can
+comply with the terms under which we access that data.
+
+ShadowBroker itself is licensed under AGPL-3.0 (see `LICENSE`). **This file
+concerns the *data* rendered by the dashboard, not the source code.**
+
+---
+
+## ODbL-licensed sources (Open Database License v1.0)
+
+Data from these sources is licensed under the
+[Open Database License v1.0](https://opendatacommons.org/licenses/odbl/1-0/).
+If you redistribute a derivative database built from these sources, the
+derivative must also be offered under ODbL and must preserve attribution.
+
+| Source | URL | What we use it for |
+|---|---|---|
+| adsb.lol | https://adsb.lol | Military aircraft positions, regional commercial gap-fill, route enrichment |
+| OpenStreetMap contributors | https://www.openstreetmap.org/copyright | Nominatim geocoding (LOCATE bar), CARTO basemap tiles (OSM-derived) |
+
+**Attribution requirement:** the ShadowBroker map UI displays
+"© OpenStreetMap contributors" and "adsb.lol (ODbL)" in the map attribution
+control. Do not remove this attribution if you fork or redistribute the app.
+
+---
+
+## Other third-party data sources
+
+These sources have their own terms; consult each link before redistributing.
+
+| Source | URL | License / Terms | Notes |
+|---|---|---|---|
+| OpenSky Network | https://opensky-network.org | OpenSky API terms | Commercial and private aircraft tracking |
+| CelesTrak | https://celestrak.org | Public domain / no restrictions | Satellite TLE data |
+| USGS Earthquake Hazards | https://earthquake.usgs.gov | Public domain (US Federal) | Seismic events |
+| NASA FIRMS | https://firms.modaps.eosdis.nasa.gov | NASA Open Data | Fire/thermal anomalies (VIIRS) |
+| NASA GIBS | https://gibs.earthdata.nasa.gov | NASA Open Data | MODIS imagery tiles |
+| NOAA SWPC | https://services.swpc.noaa.gov | Public domain (US Federal) | Space weather, Kp index |
+| GDELT Project | https://www.gdeltproject.org | CC BY (non-commercial friendly) | Global conflict events |
+| DeepState Map | https://deepstatemap.live | Per-site terms | Ukraine frontline GeoJSON |
+| aisstream.io | https://aisstream.io | Free-tier API terms (attribution required) | AIS vessel positions |
+| Global Fishing Watch | https://globalfishingwatch.org | CC BY 4.0 (for public data) | Fishing activity events |
+| Microsoft Planetary Computer | https://planetarycomputer.microsoft.com | Sentinel-2 / ESA Copernicus terms | Sentinel-2 imagery |
+| Copernicus CDSE (Sentinel Hub) | https://dataspace.copernicus.eu | ESA Copernicus open data terms | SAR + optical imagery |
+| Shodan | https://www.shodan.io | Operator-supplied API key, Shodan ToS | Internet device search |
+| Smithsonian GVP | https://volcano.si.edu | Attribution required | Volcanoes |
+| OpenAQ | https://openaq.org | CC BY 4.0 | Air quality stations |
+| NOAA NWS | https://www.weather.gov | Public domain (US Federal) | Severe weather alerts |
+| WRI Global Power Plant DB | https://datasets.wri.org | CC BY 4.0 | Power plants |
+| Wikidata | https://www.wikidata.org | CC0 | Head-of-state lookup |
+| Wikipedia | https://en.wikipedia.org | CC BY-SA 4.0 | Region summaries |
+| KiwiSDR (via dyatlov mirror) | http://rx.linkfanel.net | Per-site terms (community mirror by Pierre Ynard) | SDR receiver list — pulled from rx.linkfanel.net to keep load off jks-prv's bandwidth at kiwisdr.com |
+| OpenMHZ | https://openmhz.com | Per-site terms | Police/fire scanner feeds |
+| Meshtastic | https://meshtastic.org | Open Source | Mesh radio nodes (protocol) |
+| Meshtastic Map (Liam Cottle) | https://meshtastic.liamcottle.net | Community project (per-site terms) | Global Meshtastic node positions — polled once per day with on-disk cache trust to minimize load on this volunteer-run HTTP API |
+| APRS-IS | https://www.aprs-is.net | Open / attribution-based | Amateur radio positions |
+| CARTO basemaps | https://carto.com | CARTO attribution required | Dark map tiles (OSM-derived) |
+| Esri World Imagery | https://www.arcgis.com | Esri terms | High-res satellite basemap |
+| IODA (Georgia Tech) | https://ioda.inetintel.cc.gatech.edu | Research/academic terms | Internet outage data |
+
+---
+
+## Contact
+
+If you represent a data provider and have concerns about how ShadowBroker
+uses your data, please open an issue or contact the maintainer at
+`bigbodycobain@gmail.com`. We will respond promptly and, if needed, adjust
+usage or remove the source.

LICENSE

@@ -0,0 +1,661 @@
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU Affero General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<https://www.gnu.org/licenses/>.

Makefile

@@ -0,0 +1,55 @@
+.PHONY: up-local up-lan down restart-local restart-lan logs status help
+
+COMPOSE = docker compose
+
+# Detect LAN IP (tries Wi-Fi first, falls back to Ethernet)
+LAN_IP := $(shell ipconfig getifaddr en0 2>/dev/null || ipconfig getifaddr en1 2>/dev/null)
+
+## Default target — print help
+help:
+	@echo ""
+	@echo "Shadowbroker taskrunner"
+	@echo ""
+	@echo "Usage: make <target>"
+	@echo ""
+	@echo "  up-local        Start with loopback binding (local access only)"
+	@echo "  up-lan          Start with 0.0.0.0 binding (LAN accessible)"
+	@echo "  down            Stop all containers"
+	@echo "  restart-local   Bounce and restart in local mode"
+	@echo "  restart-lan     Bounce and restart in LAN mode"
+	@echo "  logs            Tail logs for all services"
+	@echo "  status          Show container status"
+	@echo ""
+
+## Start in local-only mode (loopback only)
+up-local:
+	BIND=127.0.0.1 $(COMPOSE) up -d
+
+## Start in LAN mode (accessible to other hosts on the network)
+up-lan:
+	@if [ -z "$(LAN_IP)" ]; then \
+		echo "ERROR: Could not detect LAN IP. Check your network connection."; \
+		exit 1; \
+	fi
+	@echo "Detected LAN IP: $(LAN_IP)"
+	BIND=0.0.0.0 CORS_ORIGINS=http://$(LAN_IP):3000 $(COMPOSE) up -d
+	@echo ""
+	@echo "Shadowbroker is now running and can be accessed by LAN devices at http://$(LAN_IP):3000"
+
+## Stop all containers
+down:
+	$(COMPOSE) down
+
+## Restart in local-only mode
+restart-local: down up-local
+
+## Restart in LAN mode
+restart-lan: down up-lan
+
+## Tail logs for all services
+logs:
+	$(COMPOSE) logs -f
+
+## Show running container status
+status:
+	$(COMPOSE) ps

Mesh.md

@@ -0,0 +1,89 @@
+# ShadowBroker — Meshtastic MQTT Remediation
+
+**Version:** 0.9.6  
+**Date:** 2026-04-12  
+**Re:** [meshtastic/firmware#6131](https://github.com/meshtastic/firmware/issues/6131) — Excessive MQTT traffic from ShadowBroker clients
+
+---
+
+## What happened
+
+ShadowBroker is an open-source OSINT situational awareness platform that includes a Meshtastic MQTT listener for displaying mesh network activity on a global map. In prior versions, the MQTT bridge:
+
+- Subscribed to **28 wildcard topics** (`msh/{region}/#`) covering every known official and community root on startup
+- Used an aggressive reconnect policy (min 1s / max 30s backoff)
+- Set keepalive to 30 seconds
+- Had no client-side rate limiting on inbound messages
+- Auto-started on every launch with no opt-out
+
+This produced 1-2 orders of magnitude more traffic than typical Meshtastic clients on the public broker at `mqtt.meshtastic.org`.
+
+---
+
+## What we fixed
+
+### 1. Bridge disabled by default
+
+The MQTT bridge no longer starts automatically. Operators must explicitly opt in:
+
+```env
+MESH_MQTT_ENABLED=true
+```
+
+### 2. US-only default subscription
+
+When enabled, the bridge subscribes to **1 topic** (`msh/US/#`) instead of 28. Additional regions are opt-in:
+
+```env
+MESH_MQTT_EXTRA_ROOTS=EU_868,ANZ
+```
+
+The UI still displays all regions in its dropdown — only the MQTT subscription scope changed.
+
+### 3. Client-side rate limiter
+
+Inbound messages are capped at **100 messages per minute** using a sliding window. Excess messages are silently dropped. A warning is logged periodically when the limiter activates so operators are aware.
+
+### 4. Conservative connection parameters
+
+| Parameter | Before | After |
+|-----------|--------|-------|
+| Keepalive | 30s | 120s |
+| Reconnect min delay | 1s | 15s |
+| Reconnect max delay | 30s | 300s |
+| QoS | 0 | 0 (unchanged) |
+
+### 5. Versioned client ID
+
+Client IDs changed from `sbmesh-{uuid}` to `sb096-{uuid}` so the Meshtastic team can identify ShadowBroker clients and track adoption of the fix by version.
+
+---
+
+## Configuration reference
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `MESH_MQTT_ENABLED` | `false` | Master switch for the MQTT bridge |
+| `MESH_MQTT_EXTRA_ROOTS` | _(empty)_ | Comma-separated additional region roots (e.g. `EU_868,ANZ,JP`) |
+| `MESH_MQTT_INCLUDE_DEFAULT_ROOTS` | `true` | Include US in subscriptions |
+| `MESH_MQTT_BROKER` | `mqtt.meshtastic.org` | Broker hostname |
+| `MESH_MQTT_PORT` | `1883` | Broker port |
+| `MESH_MQTT_USER` | `meshdev` | Broker username |
+| `MESH_MQTT_PASS` | `large4cats` | Broker password |
+| `MESH_MQTT_PSK` | _(empty)_ | Hex-encoded PSK (empty = default LongFast key) |
+
+---
+
+## Files changed
+
+- `backend/services/config.py` — Added `MESH_MQTT_ENABLED` flag
+- `backend/services/mesh/meshtastic_topics.py` — Reduced default roots to US-only
+- `backend/services/sigint_bridge.py` — Rate limiter, keepalive/backoff tuning, versioned client ID, opt-in gate
+- `backend/.env.example` — Documented all MQTT options
+
+---
+
+## Contact
+
+Repository: [github.com/BigBodyCobain/Shadowbroker](https://github.com/BigBodyCobain/Shadowbroker)  
+Maintainer: BigBodyCobain

backend/.dockerignore

@@ -4,13 +4,29 @@ __pycache__/
 .env
 .pytest_cache/
 .coverage
+.git/
+node_modules/
 cctv.db
+*.sqlite
+*.db
+
+# Debug/log files
 *.txt
 !requirements.txt
-# Exclude debug/cache JSON but keep package.json and tracked_names
+!requirements-dev.txt
+*.html
+*.xlsx
+
+# Debug/cache JSON (keep package*.json and data files)
 ais_cache.json
+carrier_cache.json
 carrier_positions.json
 dump.json
 debug_fast.json
 nyc_full.json
 nyc_sample.json
+tmp_fast.json
+
+# Test files (not needed in production image)
+test_*.py
+tests/

backend/.env.example

@@ -13,3 +13,293 @@ AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
 
 # Override allowed CORS origins (comma-separated). Defaults to localhost + LAN auto-detect.
 # CORS_ORIGINS=http://192.168.1.50:3000,https://my-domain.com
+
+# Admin key — protects sensitive endpoints (API key management, system update).
+# If unset, loopback/localhost requests still work for local single-host dev.
+# Remote/non-loopback admin access requires ADMIN_KEY, or ALLOW_INSECURE_ADMIN=true in debug-only setups.
+# Set this in production and enter the same key in Settings → Admin Key.
+# ADMIN_KEY=your-secret-admin-key-here
+
+# Allow insecure admin access without ADMIN_KEY (local dev only, beyond loopback).
+# Requires MESH_DEBUG_MODE=true; do not enable this for ordinary use.
+# ALLOW_INSECURE_ADMIN=false
+
+# User-Agent for Nominatim geocoding requests (per OSM usage policy).
+# NOMINATIM_USER_AGENT=ShadowBroker/1.0 (https://github.com/BigBodyCobain/Shadowbroker)
+
+# LTA Singapore traffic cameras — leave blank to skip this data source.
+# LTA_ACCOUNT_KEY=
+
+# NASA FIRMS country-scoped fire data — enriches global CSV with conflict-zone hotspots.
+# Free MAP_KEY from https://firms.modaps.eosdis.nasa.gov/map/#d:24hrs;@0.0,0.0,3.0z
+# FIRMS_MAP_KEY=
+
+# Ukraine air raid alerts from alerts.in.ua — free token from https://alerts.in.ua/
+# ALERTS_IN_UA_TOKEN=
+
+# Optional NUFORC UAP sighting map enrichment via Mapbox Tilequery.
+# Leave blank to skip this optional enrichment.
+# NUFORC_MAPBOX_TOKEN=
+
+# Google Earth Engine service account for VIIRS change detection (optional).
+# Download JSON key from https://console.cloud.google.com/iam-admin/serviceaccounts
+# pip install earthengine-api
+# GEE_SERVICE_ACCOUNT_KEY=
+
+# ── Meshtastic MQTT Bridge ─────────────────────────────────────
+# Disabled by default to respect the public Meshtastic broker.
+# When enabled, subscribes to US region only. Add more regions via MESH_MQTT_EXTRA_ROOTS.
+# MESH_MQTT_ENABLED=false
+# MESH_MQTT_EXTRA_ROOTS=EU_868,ANZ    # comma-separated additional region roots
+# MESH_MQTT_INCLUDE_DEFAULT_ROOTS=true
+# MESH_MQTT_BROKER=mqtt.meshtastic.org
+# MESH_MQTT_PORT=1883
+# Leave user/pass blank for the public Meshtastic broker default.
+# MESH_MQTT_USER=
+# MESH_MQTT_PASS=
+
+# Optional Meshtastic node ID (e.g. "!abcd1234"). When set, included in the
+# User-Agent sent to meshtastic.liamcottle.net so the upstream service operator
+# can identify per-install traffic instead of aggregated "ShadowBroker" hits.
+# Leave blank to send a generic UA with the project contact email only.
+# MESHTASTIC_OPERATOR_CALLSIGN=
+# MESH_MQTT_PSK=                       # hex-encoded, empty = default LongFast key
+
+# ── Mesh / Reticulum (RNS) ─────────────────────────────────────
+# Full-node / participant-node posture for public Infonet sync.
+# MESH_NODE_MODE=participant      # participant | relay | perimeter
+# Legacy compatibility sunset toggles. Default posture is to block these.
+# Legacy 16-hex node-id binding no longer has a boolean escape hatch; use a
+# dated migration override only when you intentionally need older peers during
+# migration before the hard removal target in v0.10.0 / 2026-06-01.
+# MESH_BLOCK_LEGACY_NODE_ID_COMPAT=true
+# MESH_ALLOW_LEGACY_NODE_ID_COMPAT_UNTIL=2026-05-15
+# MESH_BLOCK_LEGACY_AGENT_ID_LOOKUP=true
+# Temporary DM invite migration escape hatch. Default posture blocks importing
+# legacy/compat v1/v2 DM invites; use a dated override only while retiring
+# older exports and ask senders to re-export a current signed invite.
+# MESH_ALLOW_COMPAT_DM_INVITE_IMPORT_UNTIL=2026-05-15
+# Temporary legacy GET DM poll/count escape hatch. Default posture requires the
+# signed mailbox-claim POST APIs; only use this dated override while retiring
+# older clients that still call GET poll/count directly.
+# MESH_ALLOW_LEGACY_DM_GET_UNTIL=2026-05-15
+# Temporary raw dm1 compose/decrypt escape hatch. Default posture expects MLS
+# DM bootstrap on supported peers; only use this dated override while retiring
+# older clients that still need the raw dm1 helper path.
+# MESH_ALLOW_LEGACY_DM1_UNTIL=2026-05-15
+# Temporary legacy dm_message signature escape hatch. Default posture requires
+# the full modern signed payload; only enable this with a dated migration
+# override while older senders are being retired.
+# MESH_ALLOW_LEGACY_DM_SIGNATURE_COMPAT_UNTIL=2026-05-15
+# Rotate voter-blinding salts so new reputation events stop reusing one
+# forever-stable blinded ID. Keep grace >= rotation cadence so older votes
+# remain matchable while they age out of the ledger.
+# MESH_VOTER_BLIND_SALT_ROTATE_DAYS=30
+# MESH_VOTER_BLIND_SALT_GRACE_DAYS=30
+# Deprecated legacy env vars kept only for backward config compatibility.
+# Ordinary shipped gate flows keep MLS decrypt local; service-side decrypt is
+# reserved for explicit recovery reads.
+# MESH_GATE_BACKEND_DECRYPT_COMPAT=false
+# MESH_GATE_BACKEND_DECRYPT_COMPAT_ACKNOWLEDGE=false
+# Deprecated legacy env vars kept only for backward config compatibility.
+# Ordinary shipped gate flows keep plaintext compose/post local and only submit
+# encrypted envelopes to the backend for sign/post.
+# MESH_GATE_BACKEND_PLAINTEXT_COMPAT=false
+# MESH_GATE_BACKEND_PLAINTEXT_COMPAT_ACKNOWLEDGE=false
+# Legacy runtime switches for recovery envelopes. Per-gate envelope_policy is
+# the source of truth; leave these at the default unless testing old behavior.
+# MESH_GATE_RECOVERY_ENVELOPE_ENABLE=true
+# MESH_GATE_RECOVERY_ENVELOPE_ENABLE_ACKNOWLEDGE=true
+# Optional operator-only recovery tradeoff. Leave off for the default posture:
+# ordinary gate reads keep plaintext local/in-memory unless you explicitly use
+# the recovery-envelope path.
+# MESH_GATE_PLAINTEXT_PERSIST=false
+# MESH_GATE_PLAINTEXT_PERSIST_ACKNOWLEDGE=false
+# Legacy Phase-1 gate envelope fallback is now explicit and time-bounded per
+# gate. This only controls the default expiry window when you deliberately
+# re-enable that migration path for older stored envelopes.
+# MESH_GATE_LEGACY_ENVELOPE_FALLBACK_MAX_DAYS=30
+# Feature-flagged multiplexed gate session stream. Stream-first room ownership
+# is implemented; keep off until you want that rollout enabled in your env.
+# MESH_GATE_SESSION_STREAM_ENABLED=false
+# MESH_GATE_SESSION_STREAM_HEARTBEAT_S=20
+# MESH_GATE_SESSION_STREAM_BATCH_MS=1500
+# MESH_GATE_SESSION_STREAM_MAX_GATES=16
+# MESH_BOOTSTRAP_DISABLED=false
+# MESH_BOOTSTRAP_MANIFEST_PATH=data/bootstrap_peers.json
+# MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY=
+# Infonet/Wormhole fails closed to onion/RNS by default. Only enable clearnet
+# sync for local relay development or an explicitly public testnet.
+# MESH_INFONET_ALLOW_CLEARNET_SYNC=false
+# MESH_BOOTSTRAP_SEED_PEERS=http://gqpbunqbgtkcqilvclm3xrkt3zowjyl3s62kkktvojgvxzizamvbrqid.onion:8000
+# Add comma-separated http://*.onion peers as more private seed/relay nodes come online.
+# MESH_DEFAULT_SYNC_PEERS=         # legacy alias; prefer MESH_BOOTSTRAP_SEED_PEERS
+# MESH_RELAY_PEERS=               # comma-separated operator-trusted sync/push peers (empty by default)
+# MESH_PEER_PUSH_SECRET=          # REQUIRED when relay/RNS peers are configured (min 16 chars, generate with: python -c "import secrets; print(secrets.token_urlsafe(32))")
+# MESH_SYNC_INTERVAL_S=300
+# MESH_SYNC_FAILURE_BACKOFF_S=60
+#
+# Enable Reticulum bridge for Infonet event gossip.
+# MESH_RNS_ENABLED=false
+# MESH_RNS_APP_NAME=shadowbroker
+# MESH_RNS_ASPECT=infonet
+# MESH_RNS_IDENTITY_PATH=
+# MESH_RNS_PEERS=  # comma-separated destination hashes
+# MESH_RNS_DANDELION_HOPS=2
+# MESH_RNS_DANDELION_DELAY_MS=400
+# MESH_RNS_CHURN_INTERVAL_S=300
+# MESH_RNS_MAX_PEERS=32
+# MESH_RNS_MAX_PAYLOAD=8192
+# MESH_RNS_PEER_BUCKET_PREFIX=4
+# MESH_RNS_MAX_PEERS_PER_BUCKET=4
+# MESH_RNS_PEER_FAIL_THRESHOLD=3
+# MESH_RNS_PEER_COOLDOWN_S=300
+# MESH_RNS_SHARD_ENABLED=false
+# MESH_RNS_SHARD_DATA_SHARDS=3
+# MESH_RNS_SHARD_PARITY_SHARDS=1
+# MESH_RNS_SHARD_TTL_S=30
+# MESH_RNS_FEC_CODEC=xor
+# MESH_RNS_BATCH_MS=200
+# MESH_RNS_COVER_INTERVAL_S=0
+# MESH_RNS_COVER_SIZE=64
+# MESH_RNS_IBF_WINDOW=256
+# MESH_RNS_IBF_TABLE_SIZE=64
+# MESH_RNS_IBF_MINHASH_SIZE=16
+# MESH_RNS_IBF_MINHASH_THRESHOLD=0.25
+# MESH_RNS_IBF_WINDOW_JITTER=32
+# MESH_RNS_IBF_INTERVAL_S=120
+# MESH_RNS_IBF_SYNC_PEERS=3
+# MESH_RNS_IBF_QUORUM_TIMEOUT_S=6
+# MESH_RNS_IBF_MAX_REQUEST_IDS=64
+# MESH_RNS_IBF_MAX_EVENTS=64
+# MESH_RNS_SESSION_ROTATE_S=0
+# MESH_RNS_IBF_FAIL_THRESHOLD=3
+# MESH_RNS_IBF_COOLDOWN_S=120
+# MESH_VERIFY_INTERVAL_S=600
+# MESH_VERIFY_SIGNATURES=false
+
+# ── Secure Storage (non-Windows) ───────────────────────────────
+# Required on Linux/Docker to protect Wormhole key material at rest.
+# Generate with: python -c "import secrets; print(secrets.token_urlsafe(32))"
+# Also supports Docker secrets via MESH_SECURE_STORAGE_SECRET_FILE.
+# MESH_SECURE_STORAGE_SECRET=
+#
+# To rotate the storage secret, stop the backend and run:
+#   1. Dry-run first (validates without writing):
+#      MESH_OLD_STORAGE_SECRET=<current> MESH_NEW_STORAGE_SECRET=<new> \
+#        python -m scripts.rotate_secure_storage_secret --dry-run
+#   2. Rotate (creates .bak backups, then rewraps envelopes):
+#      MESH_OLD_STORAGE_SECRET=<current> MESH_NEW_STORAGE_SECRET=<new> \
+#        python -m scripts.rotate_secure_storage_secret
+#   3. Update MESH_SECURE_STORAGE_SECRET to the new value and restart.
+#
+# If rotation is interrupted, .bak files preserve the old envelopes.
+# To repair corrupted secure-json payloads (not key envelopes), use:
+#   python -m scripts.repair_wormhole_secure_storage
+
+# ── Mesh DM Relay ──────────────────────────────────────────────
+# MESH_DM_TOKEN_PEPPER=change-me
+# Keep DM relay metadata retention explicit and bounded.
+# MESH_DM_KEY_TTL_DAYS=30
+# MESH_DM_PREKEY_LOOKUP_ALIAS_TTL_DAYS=14
+# MESH_DM_WITNESS_TTL_DAYS=14
+# MESH_DM_BINDING_TTL_DAYS=3
+# Optional operational bridge for externally sourced root witnesses / transparency.
+# Relative paths resolve from the backend directory.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH=data/root_witness_import.json
+# Local single-host dev example after bootstrapping an external witness locally:
+# MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH=../ops/root_witness_receipt_import.json
+# Optional URI bridge for externally retrieved root witness packages.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_URI=file:///absolute/path/root_witness_import.json
+# Maximum acceptable age for external witness packages before strong DM trust fails closed.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_MAX_AGE_S=3600
+# Warning threshold for external witness packages before fail-closed max age.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_WARN_AGE_S=2700
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH=data/root_transparency_ledger.json
+# Local single-host dev example after publishing the transparency ledger locally:
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH=../ops/root_transparency_ledger.json
+# Optional URI used to read back and verify a published transparency ledger.
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI=file:///absolute/path/root_transparency_ledger.json
+# Local single-host dev readback example:
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI=../ops/root_transparency_ledger.json
+# Maximum acceptable age for external transparency ledgers before strong DM trust fails closed.
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_MAX_AGE_S=3600
+# Warning threshold for external transparency ledgers before fail-closed max age.
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_WARN_AGE_S=2700
+
+# ── Self Update ────────────────────────────────────────────────
+# MESH_UPDATE_SHA256=
+
+# ── Wormhole (Local Agent) ─────────────────────────────────────
+# WORMHOLE_HOST=127.0.0.1
+# WORMHOLE_PORT=8787
+# WORMHOLE_RELOAD=false
+# WORMHOLE_TRANSPORT=direct
+# WORMHOLE_SOCKS_PROXY=127.0.0.1:9050
+# WORMHOLE_SOCKS_DNS=true
+# Optional override for the loaded Rust privacy-core shared library. Leave
+# unset for the default repo search order. When you override this, verify the
+# authenticated wormhole status surfaces show the expected version, absolute
+# library path, and SHA-256 for the loaded artifact before making stronger
+# privacy claims about the deployment.
+# PRIVACY_CORE_LIB=
+# Minimum privacy-core version accepted when hidden/private carriers are
+# enabled. Private-lane startup fails closed if the loaded artifact is
+# missing, reports no parseable version, or falls below this minimum.
+# PRIVACY_CORE_MIN_VERSION=0.1.0
+# Comma-separated SHA-256 allowlist for the exact privacy-core artifact(s)
+# your deployment is allowed to load. Required for Arti/RNS private-lane
+# startup. Generate with:
+#   PowerShell: Get-FileHash .\privacy-core\target\release\privacy_core.dll -Algorithm SHA256
+#   macOS/Linux: sha256sum ./privacy-core/target/release/libprivacy_core.so
+# PRIVACY_CORE_ALLOWED_SHA256=
+# Optional structured release attestation artifact for the Sprint 8 release gate.
+# Relative paths resolve from the backend directory. When set explicitly, a
+# missing or unreadable file fails the DM relay security-suite criterion closed.
+# CI/release tooling can generate this automatically via:
+#   uv run python scripts/release_helper.py write-attestation ...
+# MESH_RELEASE_ATTESTATION_PATH=data/release_attestation.json
+# Operator-only Sprint 8 release attestation. Set this only when the DM relay
+# security suite has been run and passed for the current release candidate.
+# File-based release attestation takes precedence when present.
+# MESH_RELEASE_DM_RELAY_SECURITY_SUITE_GREEN=false
+
+# ── OpenClaw Agent ─────────────────────────────────────────────
+# HMAC shared secret for remote OpenClaw agent authentication.
+# Auto-generated via the Connect OpenClaw modal — do not set manually.
+# OPENCLAW_HMAC_SECRET=
+# Access tier: "restricted" (read-only) or "full" (read+write+inject)
+# OPENCLAW_ACCESS_TIER=restricted
+
+# ── SAR (Synthetic Aperture Radar) Layer ───────────────────────
+# Mode A — Free catalog metadata from Alaska Satellite Facility (ASF Search).
+# No account, no downloads.  Default-on.  Set to false to disable entirely.
+# MESH_SAR_CATALOG_ENABLED=true
+#
+# Mode B — Free pre-processed ground-change anomalies (deformation, flood,
+# damage assessments) from NASA OPERA, Copernicus EGMS, GFM, EMS, UNOSAT.
+# Two-step opt-in: BOTH of the following must be set together.
+#   1. MESH_SAR_PRODUCTS_FETCH=allow
+#   2. MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE=true
+# Either flag alone keeps Mode B disabled.  You can also enable this from
+# the Settings → SAR panel inside the app.
+# MESH_SAR_PRODUCTS_FETCH=block
+# MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE=false
+#
+# NASA Earthdata Login (free, ~1 minute signup) — required for OPERA products.
+#   Sign up: https://urs.earthdata.nasa.gov/users/new
+#   Generate token: https://urs.earthdata.nasa.gov/profile  →  "Generate Token"
+# MESH_SAR_EARTHDATA_USER=
+# MESH_SAR_EARTHDATA_TOKEN=
+#
+# Copernicus Data Space (free, ~1 minute signup) — required for EGMS / EMS.
+#   Sign up: https://dataspace.copernicus.eu/
+# MESH_SAR_COPERNICUS_USER=
+# MESH_SAR_COPERNICUS_TOKEN=
+#
+# Allow OpenClaw agents to read and act on the SAR layer (default true).
+# MESH_SAR_OPENCLAW_ENABLED=true
+#
+# Require private-tier transport (Tor / RNS) before signing and broadcasting
+# SAR anomalies to the mesh.  Default true — disable only for testnet/local use.
+# MESH_SAR_REQUIRE_PRIVATE_TIER=true

backend/Dockerfile

@@ -1,29 +1,81 @@
-FROM python:3.10-slim
+# ---- Stage 1: Compile privacy-core Rust library ----
+FROM --platform=$BUILDPLATFORM rust:1.88-slim-bookworm AS rust-builder
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    git \
+    pkg-config \
+    libssl-dev \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
+ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
+
+COPY privacy-core /build/privacy-core
+WORKDIR /build/privacy-core
+RUN cargo build --release --lib \
+    && ls -la target/release/libprivacy_core.so
+
+# ---- Stage 2: Python backend ----
+FROM python:3.11-slim-bookworm
 
 WORKDIR /app
 
-# Install Node.js (for AIS WebSocket proxy) and curl (for network fallback)
+# Install Node.js (for AIS WebSocket proxy), curl (for network fallback), and
+# Tor (for Wormhole/remote-agent .onion transport).
 RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
     curl \
+    tor \
     && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
     && apt-get install -y --no-install-recommends nodejs \
     && rm -rf /var/lib/apt/lists/*
 
-# Install Python dependencies
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
+# Install UV for fast, reproducible Python dependency management
+ADD https://astral.sh/uv/install.sh /uv-installer.sh
+RUN sh /uv-installer.sh && rm /uv-installer.sh
+ENV PATH="/root/.local/bin:$PATH"
+# Install into system Python (no venv needed inside container)
+ENV UV_PROJECT_ENVIRONMENT=/usr/local
+
+# Copy workspace root files for UV resolution (build context is repo root)
+COPY pyproject.toml /workspace/pyproject.toml
+COPY uv.lock /workspace/uv.lock
+COPY backend/pyproject.toml /workspace/backend/pyproject.toml
+
+# Install Python dependencies using the lockfile
+RUN cd /workspace/backend && uv sync --frozen --no-dev \
+    && playwright install --with-deps chromium
+
+# Copy backend source code
+COPY backend/ .
+
+# Preserve safe static data outside /app/data. The compose named volume mounted
+# at /app/data hides image-baked files on first run, so the entrypoint seeds
+# missing static JSON into fresh volumes before the backend starts.
+RUN mkdir -p /app/image-data \
+    && if [ -d /app/data ]; then cp -a /app/data/. /app/image-data/; fi \
+    && chmod +x /app/docker-entrypoint.sh
 
 # Install Node.js dependencies (ws module for AIS WebSocket proxy)
-# Copy manifests first so this layer is cached unless deps change
-COPY package*.json ./
-RUN npm install --omit=dev
+COPY backend/package*.json ./
+RUN npm ci --omit=dev
 
-# Copy source code
-COPY . .
+# Clean up workspace scaffold
+RUN rm -rf /workspace
+
+# Copy compiled privacy-core library from Rust builder stage
+COPY --from=rust-builder /build/privacy-core/target/release/libprivacy_core.so /app/libprivacy_core.so
+ENV PRIVACY_CORE_LIB=/app/libprivacy_core.so
 
 # Create a non-root user for security
+# Grant write access to /app so the auto-updater can extract files
+# Pre-create /app/data so mounted volumes inherit correct ownership
 RUN adduser --system --uid 1001 backenduser \
-    && chown -R backenduser /app
+    && mkdir -p /app/data \
+    && chown -R backenduser /app \
+    && chmod -R u+w /app
 
 # Switch to the non-root user
 USER backenduser
@@ -32,4 +84,5 @@ USER backenduser
 EXPOSE 8000
 
 # Start FastAPI server
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
+ENTRYPOINT ["/app/docker-entrypoint.sh"]
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--timeout-keep-alive", "120"]

backend/ais_proxy.js

@@ -1,4 +1,5 @@
 const WebSocket = require('ws');
+const readline = require('readline');
 
 const args = process.argv.slice(2);
 const API_KEY = args[0] || process.env.AIS_API_KEY;
@@ -8,22 +9,15 @@ if (!API_KEY) {
     process.exit(1);
 }
 
-const FILTER = [
-    // US Aircraft Carriers and major naval groups
-    { "MMSI": 338000000 }, { "MMSI": 338100000 }, // US Navy general prefixes
-    // Plus let's grab some global shipping for density
-    { "BoundingBoxes": [[[-90, -180], [90, 180]]] }
-];
+// Start with global coverage, until frontend updates it
+let currentBboxes = [[[-90, -180], [90, 180]]];
+let activeWs = null;
 
-function connect() {
-    const ws = new WebSocket('wss://stream.aisstream.io/v0/stream');
-
-    ws.on('open', () => {
+function sendSub(ws) {
+    if (ws && ws.readyState === WebSocket.OPEN) {
         const subMsg = {
             APIKey: API_KEY,
-            BoundingBoxes: [
-                [[-90, -180], [90, 180]]
-            ],
+            BoundingBoxes: currentBboxes,
             FilterMessageTypes: [
                 "PositionReport",
                 "ShipStaticData",
@@ -31,17 +25,39 @@ function connect() {
             ]
         };
         ws.send(JSON.stringify(subMsg));
+    }
+}
+
+// Listen for dynamic bounding box updates via stdin from Python orchestrator
+const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+    terminal: false
+});
+
+rl.on('line', (line) => {
+    try {
+        const cmd = JSON.parse(line);
+        if (cmd.type === "update_bbox" && cmd.bboxes) {
+            currentBboxes = cmd.bboxes;
+            if (activeWs) sendSub(activeWs); // Resend subscription (swap and replace)
+        }
+    } catch (e) {}
+});
+
+function connect() {
+    const ws = new WebSocket('wss://stream.aisstream.io/v0/stream');
+    activeWs = ws;
+
+    ws.on('open', () => {
+        sendSub(ws);
     });
 
     ws.on('message', (data) => {
-        // Output raw AIS message JSON to stdout so Python can consume it
-        // We ensure exactly one JSON object per line.
         try {
             const parsed = JSON.parse(data);
             console.log(JSON.stringify(parsed));
-        } catch (e) {
-            // ignore non-json
-        }
+        } catch (e) {}
     });
 
     ws.on('error', (err) => {
@@ -49,6 +65,7 @@ function connect() {
     });
 
     ws.on('close', () => {
+        activeWs = null;
         console.error("WebSocket Proxy Closed. Reconnecting in 5s...");
         setTimeout(connect, 5000);
     });

backend/check_regions.py

@@ -1,17 +0,0 @@
-import requests
-
-regions = [
-    {"lat": 39.8, "lon": -98.5, "dist": 2000},  # USA
-    {"lat": 50.0, "lon": 15.0, "dist": 2000},   # Europe
-    {"lat": 35.0, "lon": 105.0, "dist": 2000}   # Asia / China
-]
-
-for r in regions:
-    url = f"https://api.adsb.lol/v2/lat/{r['lat']}/lon/{r['lon']}/dist/{r['dist']}"
-    res = requests.get(url, timeout=10)
-    if res.status_code == 200:
-        data = res.json()
-        acs = data.get("ac", [])
-        print(f"Region lat:{r['lat']} lon:{r['lon']} dist:{r['dist']} -> Flights: {len(acs)}")
-    else:
-        print(f"Error for Region lat:{r['lat']} lon:{r['lon']}: HTTP {res.status_code}")

backend/clean_osm_cctvs.py

@@ -1,10 +0,0 @@
-import sqlite3
-import os
-
-db_path = os.path.join(os.path.dirname(__file__), 'cctv.db')
-conn = sqlite3.connect(db_path)
-cur = conn.cursor()
-cur.execute("DELETE FROM cameras WHERE id LIKE 'OSM-%'")
-print(f"Deleted {cur.rowcount} OSM cameras from DB.")
-conn.commit()
-conn.close()

backend/config/news_feeds.json

@@ -26,19 +26,89 @@
       "weight": 5
     },
     {
-      "name": "NHK",
-      "url": "https://www3.nhk.or.jp/nhkworld/rss/world.xml",
+      "name": "The War Zone",
+      "url": "https://www.twz.com/feed",
+      "weight": 4
+    },
+    {
+      "name": "Bellingcat",
+      "url": "https://www.bellingcat.com/feed/",
+      "weight": 4
+    },
+    {
+      "name": "Guardian",
+      "url": "https://www.theguardian.com/world/rss",
       "weight": 3
     },
+    {
+      "name": "TASS",
+      "url": "https://tass.com/rss/v2.xml",
+      "weight": 2
+    },
+    {
+      "name": "Xinhua",
+      "url": "http://www.news.cn/english/rss/worldrss.xml",
+      "weight": 2
+    },
     {
       "name": "CNA",
-      "url": "https://www.channelnewsasia.com/rssfeed/8395986",
+      "url": "https://www.channelnewsasia.com/api/v1/rss-outbound-feed?_format=xml",
       "weight": 3
     },
     {
       "name": "Mercopress",
       "url": "https://en.mercopress.com/rss/",
       "weight": 3
+    },
+    {
+      "name": "SCMP",
+      "url": "https://www.scmp.com/rss/91/feed",
+      "weight": 4
+    },
+    {
+      "name": "The Diplomat",
+      "url": "https://thediplomat.com/feed/",
+      "weight": 4
+    },
+    {
+      "name": "Yonhap",
+      "url": "https://en.yna.co.kr/RSS/news.xml",
+      "weight": 4
+    },
+    {
+      "name": "Asia Times",
+      "url": "https://asiatimes.com/feed/",
+      "weight": 3
+    },
+    {
+      "name": "Defense News",
+      "url": "https://www.defensenews.com/arc/outboundfeeds/rss/",
+      "weight": 3
+    },
+    {
+      "name": "Japan Times",
+      "url": "https://www.japantimes.co.jp/feed/",
+      "weight": 3
+    },
+    {
+      "name": "CSM",
+      "url": "https://www.csmonitor.com/rss/world",
+      "weight": 4
+    },
+    {
+      "name": "PBS NewsHour",
+      "url": "https://www.pbs.org/newshour/feeds/rss/world",
+      "weight": 4
+    },
+    {
+      "name": "France 24",
+      "url": "https://www.france24.com/en/rss",
+      "weight": 4
+    },
+    {
+      "name": "DW",
+      "url": "https://rss.dw.com/xml/rss-en-world",
+      "weight": 4
     }
   ]
-}
+}

backend/data/datacenters.json.REMOVED.git-id

@@ -1 +0,0 @@
-430ac93c4f7c4fb5a3e596ec38e3b7794c731cc1

backend/data/datacenters_geocoded.json.REMOVED.git-id

@@ -1 +0,0 @@
-476b691be156eb4fe6a6ad80f882c1dbaded8c33

backend/data/plan_ccg_vessels.json

@@ -0,0 +1,646 @@
+{
+  "412000001": {
+    "hull_number": "101",
+    "name": "Nanchang",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000002": {
+    "hull_number": "102",
+    "name": "Lhasa",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000003": {
+    "hull_number": "103",
+    "name": "Anshan",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000004": {
+    "hull_number": "104",
+    "name": "Wuxi",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000005": {
+    "hull_number": "105",
+    "name": "Dalian",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000006": {
+    "hull_number": "106",
+    "name": "Yan'an",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000007": {
+    "hull_number": "107",
+    "name": "Zunyi",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000008": {
+    "hull_number": "108",
+    "name": "Xianyang",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000101": {
+    "hull_number": "117",
+    "name": "Xining",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000102": {
+    "hull_number": "118",
+    "name": "Urumqi",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000103": {
+    "hull_number": "119",
+    "name": "Guiyang",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000104": {
+    "hull_number": "120",
+    "name": "Chengdu",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000105": {
+    "hull_number": "131",
+    "name": "Taiyuan",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000106": {
+    "hull_number": "132",
+    "name": "Suzhou",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000107": {
+    "hull_number": "133",
+    "name": "Nantong",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000108": {
+    "hull_number": "134",
+    "name": "Suqian",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000109": {
+    "hull_number": "135",
+    "name": "Lianyungang",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000110": {
+    "hull_number": "136",
+    "name": "Xuchang",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000111": {
+    "hull_number": "155",
+    "name": "Nanjing",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000112": {
+    "hull_number": "156",
+    "name": "Zibo",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000113": {
+    "hull_number": "157",
+    "name": "Lishui",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000114": {
+    "hull_number": "161",
+    "name": "Hohhot",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000115": {
+    "hull_number": "162",
+    "name": "Yancheng",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000116": {
+    "hull_number": "163",
+    "name": "Kaifeng",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000117": {
+    "hull_number": "164",
+    "name": "Taizhou",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000201": {
+    "hull_number": "538",
+    "name": "Yantai",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000202": {
+    "hull_number": "539",
+    "name": "Wuhu",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000203": {
+    "hull_number": "540",
+    "name": "Huainan",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000204": {
+    "hull_number": "541",
+    "name": "Huaihua",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000205": {
+    "hull_number": "542",
+    "name": "Zaozhuang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000206": {
+    "hull_number": "529",
+    "name": "Zhoushan",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000207": {
+    "hull_number": "530",
+    "name": "Xuzhou",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000208": {
+    "hull_number": "531",
+    "name": "Xiangtan",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000209": {
+    "hull_number": "532",
+    "name": "Jingzhou",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000210": {
+    "hull_number": "536",
+    "name": "Xuchang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000211": {
+    "hull_number": "546",
+    "name": "Yancheng",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000212": {
+    "hull_number": "547",
+    "name": "Linyi",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000213": {
+    "hull_number": "548",
+    "name": "Yiyang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000214": {
+    "hull_number": "549",
+    "name": "Changzhou",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000215": {
+    "hull_number": "550",
+    "name": "Weifang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000301": {
+    "hull_number": "31",
+    "name": "Hainan",
+    "class": "Type 075",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_075_landing_helicopter_dock"
+  },
+  "412000302": {
+    "hull_number": "32",
+    "name": "Guangxi",
+    "class": "Type 075",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_075_landing_helicopter_dock"
+  },
+  "412000303": {
+    "hull_number": "33",
+    "name": "Anhui",
+    "class": "Type 075",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_075_landing_helicopter_dock"
+  },
+  "412000401": {
+    "hull_number": "16",
+    "name": "Liaoning",
+    "class": "Type 001",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Chinese_aircraft_carrier_Liaoning"
+  },
+  "412000402": {
+    "hull_number": "17",
+    "name": "Shandong",
+    "class": "Type 002",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Chinese_aircraft_carrier_Shandong"
+  },
+  "412000403": {
+    "hull_number": "18",
+    "name": "Fujian",
+    "class": "Type 003",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Chinese_aircraft_carrier_Fujian"
+  },
+  "412000501": {
+    "hull_number": "980",
+    "name": "Hulunhu",
+    "class": "Type 901",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_901_replenishment_ship"
+  },
+  "412000502": {
+    "hull_number": "981",
+    "name": "Chaganhu",
+    "class": "Type 901",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_901_replenishment_ship"
+  },
+  "412000601": {
+    "hull_number": "998",
+    "name": "Kunlun Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000602": {
+    "hull_number": "999",
+    "name": "Jinggang Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000603": {
+    "hull_number": "989",
+    "name": "Changbai Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000604": {
+    "hull_number": "988",
+    "name": "Yimeng Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000605": {
+    "hull_number": "987",
+    "name": "Wuzhi Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000606": {
+    "hull_number": "986",
+    "name": "Longhu Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000607": {
+    "hull_number": "985",
+    "name": "Dabie Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000608": {
+    "hull_number": "984",
+    "name": "Wuyi Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000701": {
+    "hull_number": "815A-1",
+    "name": "Dongdiao",
+    "class": "Type 815A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_815_electronic_reconnaissance_ship"
+  },
+  "412000702": {
+    "hull_number": "815A-2",
+    "name": "Haiwangxing",
+    "class": "Type 815A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_815_electronic_reconnaissance_ship"
+  },
+  "412000703": {
+    "hull_number": "815A-3",
+    "name": "Tianwangxing",
+    "class": "Type 815A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_815_electronic_reconnaissance_ship"
+  },
+  "412009001": {
+    "hull_number": "2901",
+    "name": "CCG 2901",
+    "class": "12000-ton Cutter",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009002": {
+    "hull_number": "3901",
+    "name": "CCG 3901",
+    "class": "12000-ton Cutter",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009003": {
+    "hull_number": "1305",
+    "name": "CCG 1305",
+    "class": "Type 818",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009004": {
+    "hull_number": "1306",
+    "name": "CCG 1306",
+    "class": "Type 818",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009005": {
+    "hull_number": "2502",
+    "name": "CCG 2502",
+    "class": "5000-ton Cutter",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009006": {
+    "hull_number": "2302",
+    "name": "CCG 2302",
+    "class": "3000-ton Cutter",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009007": {
+    "hull_number": "2303",
+    "name": "CCG 2303",
+    "class": "3000-ton Cutter",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009008": {
+    "hull_number": "1103",
+    "name": "CCG 1103",
+    "class": "Type 718B",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009009": {
+    "hull_number": "1105",
+    "name": "CCG 1105",
+    "class": "Type 718B",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009010": {
+    "hull_number": "1302",
+    "name": "CCG 1302",
+    "class": "Type 818",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412000801": {
+    "hull_number": "171",
+    "name": "Haikou",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000802": {
+    "hull_number": "170",
+    "name": "Lanzhou",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000803": {
+    "hull_number": "150",
+    "name": "Changchun",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000804": {
+    "hull_number": "151",
+    "name": "Zhengzhou",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000805": {
+    "hull_number": "152",
+    "name": "Jinan",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000806": {
+    "hull_number": "153",
+    "name": "Xi'an",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000901": {
+    "hull_number": "572",
+    "name": "Hengshui",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000902": {
+    "hull_number": "573",
+    "name": "Liuzhou",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000903": {
+    "hull_number": "574",
+    "name": "Sanya",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000904": {
+    "hull_number": "575",
+    "name": "Yueyang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000905": {
+    "hull_number": "576",
+    "name": "Daqing",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000906": {
+    "hull_number": "577",
+    "name": "Huanggang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412001001": {
+    "hull_number": "500",
+    "name": "Xianfeng",
+    "class": "Type 056A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_056_corvette"
+  },
+  "412001002": {
+    "hull_number": "501",
+    "name": "Xinyang",
+    "class": "Type 056A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_056_corvette"
+  },
+  "412001003": {
+    "hull_number": "502",
+    "name": "Huangshi",
+    "class": "Type 056",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_056_corvette"
+  },
+  "412001004": {
+    "hull_number": "509",
+    "name": "Huaian",
+    "class": "Type 056A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_056_corvette"
+  },
+  "412001005": {
+    "hull_number": "510",
+    "name": "Ningde",
+    "class": "Type 056A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_056_corvette"
+  },
+  "412001101": {
+    "hull_number": "795",
+    "name": "Nanchong",
+    "class": "Type 039A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_039A_submarine"
+  },
+  "412001201": {
+    "hull_number": "892",
+    "name": "Hualuoshan",
+    "class": "Type 903A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_903_replenishment_ship"
+  },
+  "412001202": {
+    "hull_number": "889",
+    "name": "Taihu",
+    "class": "Type 903",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_903_replenishment_ship"
+  },
+  "412001301": {
+    "hull_number": "636",
+    "name": "Nanning",
+    "class": "Type 052DL",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412001302": {
+    "hull_number": "165",
+    "name": "Zhanjiang",
+    "class": "Type 052DL",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412001303": {
+    "hull_number": "166",
+    "name": "Huainan",
+    "class": "Type 052DL",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  }
+}

backend/data/plane_alert_db.json.REMOVED.git-id

@@ -1 +0,0 @@
-38a18cbbf1acbec5eb9266b809c28d31e2941c53

backend/data/yacht_alert_db.json

@@ -0,0 +1,122 @@
+{
+  "319225400": {
+    "name": "KORU",
+    "owner": "Jeff Bezos",
+    "builder": "Oceanco",
+    "length_m": 127,
+    "year": 2023,
+    "category": "Tech Billionaire",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Koru_(yacht)"
+  },
+  "538072122": {
+    "name": "LAUNCHPAD",
+    "owner": "Mark Zuckerberg",
+    "builder": "Feadship",
+    "length_m": 118,
+    "year": 2024,
+    "category": "Tech Billionaire",
+    "flag": "Marshall Islands",
+    "link": "https://www.superyachtfan.com/yacht/launchpad/"
+  },
+  "319032600": {
+    "name": "MUSASHI",
+    "owner": "Larry Ellison",
+    "builder": "Feadship",
+    "length_m": 88,
+    "year": 2011,
+    "category": "Tech Billionaire",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Musashi_(yacht)"
+  },
+  "319011000": {
+    "name": "RISING SUN",
+    "owner": "David Geffen",
+    "builder": "Lurssen",
+    "length_m": 138,
+    "year": 2004,
+    "category": "Celebrity / Mogul",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Rising_Sun_(yacht)"
+  },
+  "310593000": {
+    "name": "ECLIPSE",
+    "owner": "Roman Abramovich",
+    "builder": "Blohm+Voss",
+    "length_m": 162,
+    "year": 2010,
+    "category": "Oligarch Watch",
+    "flag": "Bermuda",
+    "link": "https://en.wikipedia.org/wiki/Eclipse_(yacht)"
+  },
+  "310792000": {
+    "name": "SOLARIS",
+    "owner": "Roman Abramovich",
+    "builder": "Lloyd Werft",
+    "length_m": 140,
+    "year": 2021,
+    "category": "Oligarch Watch",
+    "flag": "Bermuda",
+    "link": "https://en.wikipedia.org/wiki/Solaris_(yacht)"
+  },
+  "319094900": {
+    "name": "DILBAR",
+    "owner": "Alisher Usmanov (seized)",
+    "builder": "Lurssen",
+    "length_m": 156,
+    "year": 2016,
+    "category": "Oligarch Watch",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Dilbar_(yacht)"
+  },
+  "273610820": {
+    "name": "NORD",
+    "owner": "Alexei Mordashov",
+    "builder": "Lurssen",
+    "length_m": 142,
+    "year": 2021,
+    "category": "Oligarch Watch",
+    "flag": "Russia",
+    "link": "https://en.wikipedia.org/wiki/Nord_(yacht)"
+  },
+  "319179200": {
+    "name": "SCHEHERAZADE",
+    "owner": "Eduard Khudainatov (alleged Putin)",
+    "builder": "Lurssen",
+    "length_m": 140,
+    "year": 2020,
+    "category": "Oligarch Watch",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Scheherazade_(yacht)"
+  },
+  "319112900": {
+    "name": "AMADEA",
+    "owner": "Suleiman Kerimov (seized by US DOJ)",
+    "builder": "Lurssen",
+    "length_m": 106,
+    "year": 2017,
+    "category": "Oligarch Watch",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Amadea_(yacht)"
+  },
+  "319156800": {
+    "name": "BRAVO EUGENIA",
+    "owner": "Jerry Jones",
+    "builder": "Oceanco",
+    "length_m": 109,
+    "year": 2018,
+    "category": "Celebrity / Mogul",
+    "flag": "Cayman Islands",
+    "link": "https://www.superyachtfan.com/yacht/bravo-eugenia/"
+  },
+  "319137200": {
+    "name": "LADY S",
+    "owner": "Dan Snyder",
+    "builder": "Feadship",
+    "length_m": 93,
+    "year": 2019,
+    "category": "Celebrity / Mogul",
+    "flag": "Cayman Islands",
+    "link": "https://www.superyachtfan.com/yacht/lady-s/"
+  }
+}

backend/docker-entrypoint.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+set -eu
+
+# Docker named volumes hide files that were baked into /app/data at image build
+# time. Seed safe, static data into a fresh volume so first-run Docker installs
+# behave like source installs without bundling local runtime secrets.
+if [ -d /app/image-data ]; then
+  mkdir -p /app/data
+  find /app/image-data -mindepth 1 -maxdepth 1 -type f | while IFS= read -r src; do
+    dest="/app/data/$(basename "$src")"
+    if [ ! -e "$dest" ]; then
+      cp "$src" "$dest" || true
+    fi
+  done
+fi
+
+if [ -z "${PRIVACY_CORE_ALLOWED_SHA256:-}" ] && [ -f /app/libprivacy_core.so ]; then
+  PRIVACY_CORE_ALLOWED_SHA256="$(sha256sum /app/libprivacy_core.so | awk '{print $1}')"
+  export PRIVACY_CORE_ALLOWED_SHA256
+fi
+
+exec "$@"

backend/extract_ovens.py

@@ -1,25 +0,0 @@
-import re
-import json
-
-try:
-    with open('liveua_test.html', 'r', encoding='utf-8') as f:
-        html = f.read()
-        
-    m = re.search(r"var\s+ovens\s*=\s*(.*?);(?!function)", html, re.DOTALL)
-    if m:
-        json_str = m.group(1)
-        # Handle if it is a string containing base64
-        if json_str.startswith("'") or json_str.startswith('"'):
-            json_str = json_str.strip('"\'')
-            import base64
-            import urllib.parse
-            json_str = base64.b64decode(urllib.parse.unquote(json_str)).decode('utf-8')
-            
-        data = json.loads(json_str)
-        with open('out_liveua.json', 'w', encoding='utf-8') as f:
-            json.dump(data, f, indent=2)
-        print(f"Successfully extracted {len(data)} ovens items.")
-    else:
-        print("var ovens not found.")
-except Exception as e:
-    print("Error:", e)

backend/gate_sse.py

@@ -0,0 +1,11 @@
+"""gate_sse.py — DEPRECATED. Gate SSE broadcast removed in S3A.
+
+Gate activity is no longer broadcast via SSE. The frontend uses the
+authenticated poll loop for gate message refresh.
+
+Stubs are kept so any late imports do not crash at startup.
+"""
+
+
+def _broadcast_gate_events(gate_id: str, events: list[dict]) -> None:  # noqa: ARG001
+    """No-op — gate SSE broadcast removed."""

backend/geocode_datacenters.py

@@ -1,166 +0,0 @@
-"""
-Geocode data center street addresses via Nominatim (OpenStreetMap).
-Rate limit: 1 request/second (Nominatim policy).
-Resumable: caches results in geocode_cache.json so interrupted runs can continue.
-"""
-import json
-import time
-import urllib.request
-import urllib.parse
-import os
-import sys
-
-# Fix Windows console encoding + force unbuffered output
-if sys.platform == "win32":
-    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
-    sys.stderr.reconfigure(encoding="utf-8", errors="replace")
-
-# Force line-buffered stdout for detached processes
-class Unbuffered:
-    def __init__(self, stream):
-        self.stream = stream
-    def write(self, data):
-        self.stream.write(data)
-        self.stream.flush()
-    def writelines(self, datas):
-        self.stream.writelines(datas)
-        self.stream.flush()
-    def __getattr__(self, attr):
-        return getattr(self.stream, attr)
-
-sys.stdout = Unbuffered(sys.stdout)
-
-DATA_FILE = os.path.join(os.path.dirname(__file__), "data", "datacenters.json")
-CACHE_FILE = os.path.join(os.path.dirname(__file__), "data", "geocode_cache.json")
-OUTPUT_FILE = os.path.join(os.path.dirname(__file__), "data", "datacenters_geocoded.json")
-
-NOMINATIM_URL = "https://nominatim.openstreetmap.org/search"
-USER_AGENT = "ShadowBroker-DataCenterGeocoder/1.0"
-
-
-def geocode_address(address: str, retries: int = 3) -> tuple[float, float] | None:
-    """Geocode a single address via Nominatim. Returns (lat, lng) or None."""
-    params = urllib.parse.urlencode({"q": address, "format": "json", "limit": 1})
-    url = f"{NOMINATIM_URL}?{params}"
-    for attempt in range(retries):
-        req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
-        try:
-            resp = urllib.request.urlopen(req, timeout=15)
-            data = json.loads(resp.read())
-            if data:
-                return float(data[0]["lat"]), float(data[0]["lon"])
-            return None  # Valid response but no results
-        except Exception as e:
-            if attempt < retries - 1:
-                wait = 2 ** (attempt + 1)
-                print(f"  RETRY ({attempt+1}/{retries}): {e} — waiting {wait}s")
-                time.sleep(wait)
-            else:
-                print(f"  ERROR (gave up after {retries} attempts): {e}")
-    return None
-
-
-def main():
-    with open(DATA_FILE, "r", encoding="utf-8") as f:
-        dcs = json.load(f)
-
-    # Load cache
-    cache = {}
-    if os.path.exists(CACHE_FILE):
-        with open(CACHE_FILE, "r", encoding="utf-8") as f:
-            cache = json.load(f)
-        print(f"Loaded {len(cache)} cached geocode results")
-
-    # Filter to DCs with real street addresses
-    to_geocode = []
-    skipped = 0
-    for i, dc in enumerate(dcs):
-        street = (dc.get("street") or "").strip()
-        if not street or len(street) <= 3 or street.lower() in ("tbc", "n/a", "na", "-"):
-            skipped += 1
-            continue
-        to_geocode.append((i, dc))
-
-    print(f"Total DCs: {len(dcs)}")
-    print(f"Skipped (no real address): {skipped}")
-    print(f"To geocode: {len(to_geocode)}")
-
-    # Count how many already cached
-    already_cached = sum(1 for _, dc in to_geocode if dc.get("address", "") in cache)
-    need_api = len(to_geocode) - already_cached
-    print(f"Already cached: {already_cached}")
-    print(f"Need API calls: {need_api}")
-    if need_api > 0:
-        print(f"Estimated time: {need_api // 60}m {need_api % 60}s")
-    print()
-
-    geocoded = 0
-    failed = 0
-    api_calls = 0
-    save_interval = 50  # Save cache every 50 API calls
-
-    for idx, (i, dc) in enumerate(to_geocode):
-        address = dc.get("address", "").strip()
-        if not address:
-            # Build address from parts
-            parts = [dc.get("street", ""), dc.get("zip", ""), dc.get("city", ""), dc.get("country", "")]
-            address = " ".join(p.strip() for p in parts if p and p.strip())
-
-        if not address:
-            failed += 1
-            continue
-
-        # Check cache first
-        if address in cache:
-            result = cache[address]
-            if result:
-                dcs[i]["lat"] = result[0]
-                dcs[i]["lng"] = result[1]
-                dcs[i]["geocode_source"] = "nominatim"
-                geocoded += 1
-            else:
-                failed += 1
-            continue
-
-        # API call — Nominatim requires 1 req/s, use 1.5s to avoid 429s after heavy use
-        time.sleep(1.5)
-        coords = geocode_address(address)
-        api_calls += 1
-
-        if coords:
-            cache[address] = coords
-            dcs[i]["lat"] = coords[0]
-            dcs[i]["lng"] = coords[1]
-            dcs[i]["geocode_source"] = "nominatim"
-            geocoded += 1
-            print(f"[{api_calls}/{need_api}] OK: {dc.get('name', '?')} -> ({coords[0]:.4f}, {coords[1]:.4f})")
-        else:
-            cache[address] = None
-            failed += 1
-            print(f"[{api_calls}/{need_api}] FAIL: {dc.get('name', '?')} | {address}")
-
-        # Periodic cache save
-        if api_calls % save_interval == 0:
-            with open(CACHE_FILE, "w", encoding="utf-8") as f:
-                json.dump(cache, f)
-            print(f"  -- Cache saved ({len(cache)} entries) --")
-
-    # Final save
-    with open(CACHE_FILE, "w", encoding="utf-8") as f:
-        json.dump(cache, f)
-
-    # Write output - only DCs with real coordinates
-    output = [dc for dc in dcs if dc.get("lat") is not None and dc.get("lng") is not None]
-
-    with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
-        json.dump(output, f, indent=2)
-
-    print(f"\nDone!")
-    print(f"Geocoded: {geocoded}")
-    print(f"Failed: {failed}")
-    print(f"API calls made: {api_calls}")
-    print(f"Output: {len(output)} DCs with coordinates -> {OUTPUT_FILE}")
-
-
-if __name__ == "__main__":
-    main()

backend/limiter.py

@@ -0,0 +1,4 @@
+from slowapi import Limiter
+from slowapi.util import get_remote_address
+
+limiter = Limiter(key_func=get_remote_address)

backend/node_state.py

@@ -0,0 +1,313 @@
+"""node_state.py — Shared mutable node runtime state and node helper functions.
+
+Extracted from main.py so that background worker functions and route handlers
+can reference the same state objects without importing the full application.
+
+_NODE_SYNC_STATE is a reassignable value (SyncWorkerState is replaced whole,
+not mutated), so callers must use get_sync_state() / set_sync_state() instead
+of binding to the name at import time.
+
+All other _NODE_* objects are mutable containers (Lock, Event, dict) whose
+identity never changes; importing them directly by name is safe.
+"""
+
+import threading
+import time
+from typing import Any
+
+from services.mesh.mesh_infonet_sync_support import SyncWorkerState
+
+# ---------------------------------------------------------------------------
+# Runtime state objects
+# ---------------------------------------------------------------------------
+
+_NODE_RUNTIME_LOCK = threading.RLock()
+_NODE_SYNC_STOP = threading.Event()
+_NODE_SYNC_STATE = SyncWorkerState()
+_NODE_BOOTSTRAP_STATE: dict[str, Any] = {
+    "node_mode": "participant",
+    "manifest_loaded": False,
+    "manifest_signer_id": "",
+    "manifest_valid_until": 0,
+    "bootstrap_peer_count": 0,
+    "sync_peer_count": 0,
+    "push_peer_count": 0,
+    "operator_peer_count": 0,
+    "last_bootstrap_error": "",
+}
+_NODE_PUSH_STATE: dict[str, Any] = {
+    "last_event_id": "",
+    "last_push_ok_at": 0,
+    "last_push_error": "",
+    "last_results": [],
+}
+
+# ---------------------------------------------------------------------------
+# Getter / setter for _NODE_SYNC_STATE
+#
+# Use these instead of globals()["_NODE_SYNC_STATE"] = ... in any module that
+# imports this package.  The setter modifies *this* module's namespace so
+# subsequent get_sync_state() calls see the new value regardless of which
+# module calls set_sync_state().
+# ---------------------------------------------------------------------------
+
+def get_sync_state() -> SyncWorkerState:
+    return _NODE_SYNC_STATE
+
+
+def set_sync_state(state: SyncWorkerState) -> None:
+    global _NODE_SYNC_STATE
+    _NODE_SYNC_STATE = state
+
+
+# ---------------------------------------------------------------------------
+# Node helper functions
+#
+# These were in main.py but are needed by both route handlers and background
+# workers, so they live here to avoid circular imports.
+# ---------------------------------------------------------------------------
+
+def _current_node_mode() -> str:
+    from services.config import get_settings
+    mode = str(get_settings().MESH_NODE_MODE or "participant").strip().lower()
+    if mode not in {"participant", "relay", "perimeter"}:
+        return "participant"
+    return mode
+
+
+def _node_runtime_supported() -> bool:
+    return _current_node_mode() in {"participant", "relay"}
+
+
+def _node_activation_enabled() -> bool:
+    from services.node_settings import read_node_settings
+
+    try:
+        settings = read_node_settings()
+    except Exception:
+        return False
+    return bool(settings.get("enabled", False))
+
+
+def _participant_node_enabled() -> bool:
+    return _node_runtime_supported() and _node_activation_enabled()
+
+
+def _node_runtime_snapshot() -> dict[str, Any]:
+    with _NODE_RUNTIME_LOCK:
+        return {
+            "node_mode": _current_node_mode(),
+            "node_enabled": _participant_node_enabled(),
+            "private_transport_required": _infonet_private_transport_required(),
+            "bootstrap": {**dict(_NODE_BOOTSTRAP_STATE), "node_mode": _current_node_mode()},
+            "sync_runtime": get_sync_state().to_dict(),
+            "push_runtime": dict(_NODE_PUSH_STATE),
+        }
+
+
+def _set_node_sync_disabled_state(*, current_head: str = "") -> SyncWorkerState:
+    return SyncWorkerState(
+        current_head=str(current_head or ""),
+        last_outcome="disabled",
+    )
+
+
+def _set_participant_node_enabled(enabled: bool) -> dict[str, Any]:
+    from services.mesh.mesh_hashchain import infonet
+    from services.node_settings import write_node_settings
+
+    settings = write_node_settings(enabled=bool(enabled))
+    current_head = str(infonet.head_hash or "")
+    with _NODE_RUNTIME_LOCK:
+        _NODE_BOOTSTRAP_STATE["node_mode"] = _current_node_mode()
+        set_sync_state(
+            SyncWorkerState(current_head=current_head)
+            if bool(enabled) and _node_runtime_supported()
+            else _set_node_sync_disabled_state(current_head=current_head)
+        )
+    return {
+        **settings,
+        "node_mode": _current_node_mode(),
+        "node_enabled": _participant_node_enabled(),
+    }
+
+
+def _infonet_private_transport_required() -> bool:
+    from services.config import get_settings
+
+    return not bool(getattr(get_settings(), "MESH_INFONET_ALLOW_CLEARNET_SYNC", False))
+
+
+def _infonet_private_transport_error() -> str:
+    return "private Infonet requires onion/RNS transport; no clearnet sync fallback"
+
+
+def _is_private_infonet_transport(transport: str) -> bool:
+    return str(transport or "").strip().lower() in {"onion", "rns"}
+
+
+def _configured_bootstrap_seed_peer_urls() -> list[str]:
+    from services.config import get_settings
+    from services.mesh.mesh_router import parse_configured_relay_peers
+
+    settings = get_settings()
+    primary = str(getattr(settings, "MESH_BOOTSTRAP_SEED_PEERS", "") or "").strip()
+    legacy = str(getattr(settings, "MESH_DEFAULT_SYNC_PEERS", "") or "").strip()
+    return parse_configured_relay_peers(primary or legacy)
+
+
+def _refresh_node_peer_store(*, now: float | None = None) -> dict[str, Any]:
+    from services.config import get_settings
+    from services.mesh.mesh_bootstrap_manifest import load_bootstrap_manifest_from_settings
+    from services.mesh.mesh_peer_store import (
+        DEFAULT_PEER_STORE_PATH,
+        PeerStore,
+        make_bootstrap_peer_record,
+        make_push_peer_record,
+        make_sync_peer_record,
+    )
+    from services.mesh.mesh_router import (
+        configured_relay_peer_urls,
+        parse_configured_relay_peers,
+        peer_transport_kind,
+    )
+
+    timestamp = int(now if now is not None else time.time())
+    mode = _current_node_mode()
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        store = PeerStore(DEFAULT_PEER_STORE_PATH)
+
+    private_transport_required = _infonet_private_transport_required()
+    operator_peers = configured_relay_peer_urls()
+    bootstrap_seed_peers = _configured_bootstrap_seed_peer_urls()
+    skipped_clearnet_peers = 0
+    for peer_url in operator_peers:
+        transport = peer_transport_kind(peer_url)
+        if not transport:
+            continue
+        if private_transport_required and not _is_private_infonet_transport(transport):
+            skipped_clearnet_peers += 1
+            continue
+        store.upsert(
+            make_sync_peer_record(
+                peer_url=peer_url,
+                transport=transport,
+                role="relay",
+                source="operator",
+                now=timestamp,
+            )
+        )
+        store.upsert(
+            make_push_peer_record(
+                peer_url=peer_url,
+                transport=transport,
+                role="relay",
+                source="operator",
+                now=timestamp,
+            )
+        )
+
+    operator_peer_set = set(operator_peers)
+    for peer_url in bootstrap_seed_peers:
+        if peer_url in operator_peer_set:
+            continue
+        transport = peer_transport_kind(peer_url)
+        if not transport:
+            continue
+        if private_transport_required and not _is_private_infonet_transport(transport):
+            skipped_clearnet_peers += 1
+            continue
+        store.upsert(
+            make_bootstrap_peer_record(
+                peer_url=peer_url,
+                transport=transport,
+                role="seed",
+                label="ShadowBroker bootstrap seed",
+                signer_id="shadowbroker-bootstrap",
+                now=timestamp,
+            )
+        )
+        store.upsert(
+            make_sync_peer_record(
+                peer_url=peer_url,
+                transport=transport,
+                role="seed",
+                source="bundle",
+                label="ShadowBroker bootstrap seed",
+                signer_id="shadowbroker-bootstrap",
+                now=timestamp,
+            )
+        )
+
+    manifest = None
+    bootstrap_error = ""
+    try:
+        manifest = load_bootstrap_manifest_from_settings(now=timestamp)
+    except Exception as exc:
+        bootstrap_error = str(exc or "").strip()
+
+    if manifest is not None:
+        for peer in manifest.peers:
+            if private_transport_required and not _is_private_infonet_transport(peer.transport):
+                skipped_clearnet_peers += 1
+                continue
+            store.upsert(
+                make_bootstrap_peer_record(
+                    peer_url=peer.peer_url,
+                    transport=peer.transport,
+                    role=peer.role,
+                    label=peer.label,
+                    signer_id=manifest.signer_id,
+                    now=timestamp,
+                )
+            )
+            store.upsert(
+                make_sync_peer_record(
+                    peer_url=peer.peer_url,
+                    transport=peer.transport,
+                    role=peer.role,
+                    source="bootstrap_promoted",
+                    label=peer.label,
+                    signer_id=manifest.signer_id,
+                    now=timestamp,
+                )
+            )
+
+    if private_transport_required and skipped_clearnet_peers and not bootstrap_error:
+        bootstrap_error = _infonet_private_transport_error()
+
+    store.save()
+    bootstrap_records = store.records_for_bucket("bootstrap")
+    sync_records = store.records_for_bucket("sync")
+    push_records = store.records_for_bucket("push")
+    if private_transport_required:
+        bootstrap_records = [record for record in bootstrap_records if _is_private_infonet_transport(record.transport)]
+        sync_records = [record for record in sync_records if _is_private_infonet_transport(record.transport)]
+        push_records = [record for record in push_records if _is_private_infonet_transport(record.transport)]
+    snapshot = {
+        "node_mode": mode,
+        "private_transport_required": private_transport_required,
+        "skipped_clearnet_peer_count": skipped_clearnet_peers,
+        "manifest_loaded": manifest is not None,
+        "manifest_signer_id": manifest.signer_id if manifest is not None else "",
+        "manifest_valid_until": int(manifest.valid_until or 0) if manifest is not None else 0,
+        "bootstrap_peer_count": len(bootstrap_records),
+        "sync_peer_count": len(sync_records),
+        "push_peer_count": len(push_records),
+        "operator_peer_count": len(operator_peers),
+        "bootstrap_seed_peer_count": len(bootstrap_seed_peers),
+        "default_sync_peer_count": len(bootstrap_seed_peers),
+        "last_bootstrap_error": bootstrap_error,
+    }
+    with _NODE_RUNTIME_LOCK:
+        _NODE_BOOTSTRAP_STATE.update(snapshot)
+    return snapshot
+
+
+def _materialize_local_infonet_state() -> None:
+    from services.mesh.mesh_hashchain import infonet
+
+    infonet.ensure_materialized()

backend/out.json

@@ -1 +0,0 @@
-{"callsign": "JWZ7", "country": "N625GN", "lng": -111.914754, "lat": 33.620235, "alt": 0, "heading": 0, "type": "tracked_flight", "origin_loc": null, "dest_loc": null, "origin_name": "UNKNOWN", "dest_name": "UNKNOWN", "registration": "N625GN", "model": "GLF5", "icao24": "a82973", "speed_knots": 6.8, "squawk": "1200", "airline_code": "", "aircraft_category": "plane", "alert_operator": "Tilman Fertitta", "alert_category": "People", "alert_color": "pink", "trail": [[33.62024, -111.91475, 0, 1772302052]]}

backend/pyproject.toml

@@ -0,0 +1,52 @@
+[build-system]
+requires = ["setuptools>=68.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+py-modules = []
+
+[project]
+name = "backend"
+version = "0.9.79"
+requires-python = ">=3.10"
+dependencies = [
+    "apscheduler==3.10.3",
+    "beautifulsoup4>=4.9.0",
+    "cachetools==5.5.2",
+    "cloudscraper==1.2.71",
+    "cryptography>=41.0.0",
+    "fastapi==0.115.12",
+    "feedparser==6.0.10",
+    "httpx==0.28.1",
+    "playwright==1.59.0",
+    "playwright-stealth==1.0.6",
+    "pydantic==2.13.3",
+    "pydantic-settings==2.8.1",
+    "pystac-client==0.8.6",
+    "python-dotenv==1.2.2",
+    "requests==2.31.0",
+    "PySocks==1.7.1",
+    "reverse-geocoder==1.5.1",
+    "sgp4==2.25",
+    "meshtastic>=2.5.0",
+    "orjson>=3.10.0",
+    "paho-mqtt>=1.6.0,<2.0.0",
+    "PyNaCl>=1.5.0",
+    "slowapi==0.1.9",
+    "vaderSentiment>=3.3.0",
+    "uvicorn==0.34.0",
+    "yfinance==1.3.0",
+]
+
+[dependency-groups]
+dev = ["pytest>=8.3.4", "pytest-asyncio==0.25.0", "ruff>=0.9.0", "black>=24.0.0"]
+
+[tool.ruff.lint]
+# The current backend carries historical style debt in large legacy modules.
+# Keep CI focused on actionable correctness checks for the v0.9.79 release.
+ignore = ["E401", "E402", "E701", "E731", "E741", "F401", "F402", "F541", "F811", "F841"]
+
+[tool.black]
+# Avoid a release-time whole-backend formatting rewrite. Re-enable by narrowing
+# this once the legacy tree is formatted in a dedicated cleanup PR.
+force-exclude = ".*"

backend/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+testpaths = tests
+python_files = test_*.py
+python_functions = test_*
+asyncio_default_fixture_loop_scope = function

backend/requirements.txt

@@ -1,20 +0,0 @@
-fastapi>=0.103.1
-uvicorn>=0.23.2
-yfinance>=0.2.40
-feedparser==6.0.10
-legacy-cgi>=2.6
-requests==2.31.0
-apscheduler==3.10.3
-pydantic>=2.3.0
-pydantic-settings>=2.0.3
-playwright>=1.58.0
-beautifulsoup4>=4.12.0
-cachetools>=5.3
-cloudscraper>=1.2.71
-python-dotenv>=1.0
-lxml>=5.0
-reverse_geocoder>=1.5
-sgp4>=2.23
-geopy>=2.4.0
-pytz>=2023.3
-pystac-client>=0.7.0

backend/routers/admin.py

@@ -0,0 +1,385 @@
+import json as json_mod
+import logging
+import os
+import threading
+from pathlib import Path
+from typing import Any
+from fastapi import APIRouter, Request, Depends, Response
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+from node_state import (
+    _current_node_mode,
+    _participant_node_enabled,
+    _refresh_node_peer_store,
+    _set_participant_node_enabled,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+class NodeSettingsUpdate(BaseModel):
+    enabled: bool
+
+
+class TimeMachineToggle(BaseModel):
+    enabled: bool
+
+
+class MeshtasticMqttUpdate(BaseModel):
+    enabled: bool | None = None
+    broker: str | None = None
+    port: int | None = None
+    username: str | None = None
+    password: str | None = None
+    psk: str | None = None
+    include_default_roots: bool | None = None
+    extra_roots: str | None = None
+    extra_topics: str | None = None
+
+
+@router.get("/api/settings/api-keys", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_get_keys(request: Request):
+    from services.api_settings import get_api_keys
+    return get_api_keys()
+
+
+@router.put("/api/settings/api-keys", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_save_keys(request: Request):
+    from services.api_settings import save_api_keys
+    body = await request.json()
+    if not isinstance(body, dict):
+        return Response(
+            content=json_mod.dumps({"ok": False, "detail": "Expected a JSON object."}),
+            status_code=400,
+            media_type="application/json",
+        )
+    result = save_api_keys({str(k): str(v) for k, v in body.items()})
+    if result.get("ok"):
+        return result
+    return Response(
+        content=json_mod.dumps(result),
+        status_code=400,
+        media_type="application/json",
+    )
+
+
+@router.get("/api/settings/api-keys/meta")
+@limiter.limit("30/minute")
+async def api_get_keys_meta(request: Request):
+    """Return absolute paths for the backend .env and .env.example template.
+
+    Not gated behind admin auth: the paths are not sensitive, and the frontend
+    needs them to render the API Keys panel banner before the user has had a
+    chance to enter an admin key. Helps users find the file when in-app editing
+    is blocked or when the backend is read-only.
+    """
+    from services.api_settings import get_env_path_info
+    return get_env_path_info()
+
+
+@router.get("/api/settings/news-feeds")
+@limiter.limit("30/minute")
+async def api_get_news_feeds(request: Request):
+    from services.news_feed_config import get_feeds
+    return get_feeds()
+
+
+@router.put("/api/settings/news-feeds", dependencies=[Depends(require_admin)])
+@limiter.limit("10/minute")
+async def api_save_news_feeds(request: Request):
+    from services.news_feed_config import save_feeds
+    body = await request.json()
+    ok = save_feeds(body)
+    if ok:
+        return {"status": "updated", "count": len(body)}
+    return Response(
+        content=json_mod.dumps({"status": "error",
+            "message": "Validation failed (max 20 feeds, each needs name/url/weight 1-5)"}),
+        status_code=400,
+        media_type="application/json",
+    )
+
+
+@router.post("/api/settings/news-feeds/reset", dependencies=[Depends(require_admin)])
+@limiter.limit("10/minute")
+async def api_reset_news_feeds(request: Request):
+    from services.news_feed_config import get_feeds, reset_feeds
+    ok = reset_feeds()
+    if ok:
+        return {"status": "reset", "feeds": get_feeds()}
+    return {"status": "error", "message": "Failed to reset feeds"}
+
+
+@router.get("/api/settings/node")
+@limiter.limit("30/minute")
+async def api_get_node_settings(request: Request):
+    import asyncio
+    from services.node_settings import read_node_settings
+    data = await asyncio.to_thread(read_node_settings)
+    return {
+        **data,
+        "node_mode": _current_node_mode(),
+        "node_enabled": _participant_node_enabled(),
+    }
+
+
+@router.put("/api/settings/node", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_set_node_settings(request: Request, body: NodeSettingsUpdate):
+    _refresh_node_peer_store()
+    if bool(body.enabled):
+        try:
+            from services.transport_lane_isolation import disable_public_mesh_lane
+
+            disable_public_mesh_lane(reason="private_node_enabled")
+        except Exception as exc:
+            logger.warning("Failed to disable public Mesh while enabling private node: %s", exc)
+    result = _set_participant_node_enabled(bool(body.enabled))
+    if bool(body.enabled):
+        try:
+            import main as _main
+
+            _main._kick_public_sync_background("operator_enable")
+        except Exception:
+            logger.debug("Unable to kick Infonet sync after node enable", exc_info=True)
+    return result
+
+
+def _meshtastic_runtime_snapshot() -> dict[str, Any]:
+    from services.meshtastic_mqtt_settings import redacted_meshtastic_mqtt_settings
+    from services.sigint_bridge import sigint_grid
+
+    return {
+        **redacted_meshtastic_mqtt_settings(),
+        "runtime": sigint_grid.mesh.status(),
+    }
+
+
+@router.get("/api/settings/meshtastic-mqtt", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_get_meshtastic_mqtt_settings(request: Request):
+    return _meshtastic_runtime_snapshot()
+
+
+@router.put("/api/settings/meshtastic-mqtt", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_set_meshtastic_mqtt_settings(request: Request, body: MeshtasticMqttUpdate):
+    from services.meshtastic_mqtt_settings import write_meshtastic_mqtt_settings
+    from services.sigint_bridge import sigint_grid
+
+    updates = body.model_dump(exclude_unset=True)
+    # Empty secret fields mean "keep existing"; explicit non-empty values replace.
+    if updates.get("password") == "":
+        updates.pop("password", None)
+    if updates.get("psk") == "":
+        updates.pop("psk", None)
+
+    enabled_requested = updates.get("enabled")
+    settings = write_meshtastic_mqtt_settings(**updates)
+    if isinstance(enabled_requested, bool):
+        logger.info("Meshtastic MQTT settings update: enabled=%s", enabled_requested)
+
+    if enabled_requested is True:
+        # Public MQTT and Wormhole are intentionally mutually exclusive lanes.
+        try:
+            from services.node_settings import write_node_settings
+            from services.wormhole_settings import write_wormhole_settings
+            from services.wormhole_supervisor import disconnect_wormhole
+
+            write_wormhole_settings(enabled=False)
+            disconnect_wormhole(reason="public_mesh_enabled")
+            write_node_settings(enabled=False)
+            _set_participant_node_enabled(False)
+        except Exception as exc:
+            logger.warning("Failed to disable private mesh lane while enabling public mesh: %s", exc)
+
+    if bool(settings.get("enabled")):
+        if sigint_grid.mesh.is_running():
+            sigint_grid.mesh.stop()
+            threading.Timer(1.0, sigint_grid.mesh.start).start()
+        else:
+            sigint_grid.mesh.start()
+    else:
+        sigint_grid.mesh.stop()
+
+    return _meshtastic_runtime_snapshot()
+
+
+@router.get("/api/settings/timemachine")
+@limiter.limit("30/minute")
+async def api_get_timemachine_settings(request: Request):
+    import asyncio
+    from services.node_settings import read_node_settings
+    data = await asyncio.to_thread(read_node_settings)
+    return {
+        "enabled": data.get("timemachine_enabled", False),
+        "storage_warning": "Time Machine auto-snapshots use ~68 MB/day compressed (~2 GB/month). "
+                           "Snapshots capture entity positions (flights, ships, satellites) for historical playback.",
+    }
+
+
+@router.put("/api/settings/timemachine", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_set_timemachine_settings(request: Request, body: TimeMachineToggle):
+    import asyncio
+    from services.node_settings import write_node_settings
+    result = await asyncio.to_thread(write_node_settings, timemachine_enabled=body.enabled)
+    return {
+        "ok": True,
+        "enabled": result.get("timemachine_enabled", False),
+    }
+
+
+@router.post("/api/system/update", dependencies=[Depends(require_admin)])
+@limiter.limit("1/minute")
+async def system_update(request: Request):
+    """Download latest release, backup current files, extract update, and restart."""
+    from services.updater import perform_update, schedule_restart
+    candidate = Path(__file__).resolve().parent.parent.parent
+    if (candidate / "frontend").is_dir() or (candidate / "backend").is_dir():
+        project_root = str(candidate)
+    else:
+        project_root = os.getcwd()
+    result = perform_update(project_root)
+    if result.get("status") == "error":
+        return Response(content=json_mod.dumps(result), status_code=500, media_type="application/json")
+    if result.get("status") == "docker":
+        return result
+    threading.Timer(2.0, schedule_restart, args=[project_root]).start()
+    return result
+
+
+# ── Tor Hidden Service ──────────────────────────────────────────────
+
+
+@router.get("/api/settings/tor", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_tor_status(request: Request):
+    """Return Tor hidden service status and .onion address if available."""
+    import asyncio
+    from services.tor_hidden_service import tor_service
+
+    return await asyncio.to_thread(tor_service.status)
+
+
+@router.post("/api/settings/tor/start", dependencies=[Depends(require_local_operator)])
+@limiter.limit("5/minute")
+async def api_tor_start(request: Request):
+    """Start Tor and provision a hidden service for this ShadowBroker instance.
+
+    Also enables MESH_ARTI so the mesh/wormhole system can route traffic
+    through the Tor SOCKS proxy (port 9050) automatically.
+    """
+    import asyncio
+    from services.tor_hidden_service import tor_service
+
+    result = await asyncio.to_thread(tor_service.start)
+
+    # If Tor started successfully, enable Arti (Tor SOCKS proxy for mesh)
+    if result.get("ok"):
+        try:
+            from routers.ai_intel import _write_env_value
+            from services.config import get_settings
+            _write_env_value("MESH_ARTI_ENABLED", "true")
+            get_settings.cache_clear()
+        except Exception:
+            pass  # Non-fatal — hidden service still works without mesh Arti
+
+    return result
+
+
+@router.post("/api/settings/tor/reset-identity", dependencies=[Depends(require_local_operator)])
+@limiter.limit("2/minute")
+async def api_tor_reset_identity(request: Request):
+    """Destroy current .onion identity and generate a fresh one on next start.
+
+    This is irreversible — the old .onion address is permanently lost.
+    """
+    import asyncio, shutil
+    from services.tor_hidden_service import tor_service, TOR_DIR
+
+    # Stop Tor if running
+    await asyncio.to_thread(tor_service.stop)
+
+    # Delete the hidden service directory (contains the private key)
+    hs_dir = TOR_DIR / "hidden_service"
+    if hs_dir.exists():
+        shutil.rmtree(str(hs_dir), ignore_errors=True)
+
+    # Clear cached address
+    tor_service._onion_address = ""
+
+    return {"ok": True, "detail": "Tor identity destroyed. A new .onion will be generated on next start."}
+
+
+@router.post("/api/settings/agent/reset-all", dependencies=[Depends(require_local_operator)])
+@limiter.limit("2/minute")
+async def api_reset_all_agent_credentials(request: Request):
+    """Nuclear reset: regenerate HMAC key, destroy .onion, revoke agent identity.
+
+    After this, the agent is fully disconnected and needs new credentials.
+    """
+    import asyncio, secrets, shutil
+    from services.tor_hidden_service import tor_service, TOR_DIR
+    from services.config import get_settings
+
+    results = {}
+
+    # 1. Regenerate HMAC key
+    new_secret = secrets.token_hex(24)
+    from routers.ai_intel import _write_env_value
+    _write_env_value("OPENCLAW_HMAC_SECRET", new_secret)
+    results["hmac"] = "regenerated"
+
+    # 2. Revoke agent identity (Ed25519 keypair)
+    try:
+        from services.openclaw_bridge import revoke_agent_identity
+        revoke_agent_identity()
+        results["identity"] = "revoked"
+    except Exception as e:
+        results["identity"] = f"error: {e}"
+
+    # 3. Destroy .onion and restart Tor with new identity
+    await asyncio.to_thread(tor_service.stop)
+    hs_dir = TOR_DIR / "hidden_service"
+    if hs_dir.exists():
+        shutil.rmtree(str(hs_dir), ignore_errors=True)
+    tor_service._onion_address = ""
+    results["tor"] = "identity destroyed"
+
+    # 4. Bootstrap fresh identity + start Tor with new .onion
+    try:
+        from services.openclaw_bridge import generate_agent_keypair
+        keypair = generate_agent_keypair(force=True)
+        results["new_node_id"] = keypair.get("node_id", "")
+    except Exception as e:
+        results["new_node_id"] = f"error: {e}"
+
+    tor_result = await asyncio.to_thread(tor_service.start)
+    results["new_onion"] = tor_result.get("onion_address", "")
+    results["tor_ok"] = tor_result.get("ok", False)
+
+    # Clear settings cache
+    get_settings.cache_clear()
+
+    return {
+        "ok": True,
+        "hmac_regenerated": True,
+        "detail": "All agent credentials have been reset. Use the agent connection screen to generate or reveal replacement credentials.",
+        **results,
+    }
+
+
+@router.post("/api/settings/tor/stop", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_tor_stop(request: Request):
+    """Stop the Tor hidden service."""
+    import asyncio
+    from services.tor_hidden_service import tor_service
+
+    return await asyncio.to_thread(tor_service.stop)

backend/routers/cctv.py

@@ -0,0 +1,304 @@
+import logging
+from dataclasses import dataclass, field
+from fastapi import APIRouter, Request, Query, HTTPException
+from fastapi.responses import StreamingResponse
+from starlette.background import BackgroundTask
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+_CCTV_PROXY_CONNECT_TIMEOUT_S = 2.0
+
+_CCTV_PROXY_ALLOWED_HOSTS = {
+    "s3-eu-west-1.amazonaws.com",
+    "jamcams.tfl.gov.uk",
+    "images.data.gov.sg",
+    "cctv.austinmobility.io",
+    "webcams.nyctmc.org",
+    "cwwp2.dot.ca.gov",
+    "wzmedia.dot.ca.gov",
+    "images.wsdot.wa.gov",
+    "olypen.com",
+    "flyykm.com",
+    "cam.pangbornairport.com",
+    "navigator-c2c.dot.ga.gov",
+    "navigator-c2c.ga.gov",
+    "navigator-csc.dot.ga.gov",
+    "vss1live.dot.ga.gov",
+    "vss2live.dot.ga.gov",
+    "vss3live.dot.ga.gov",
+    "vss4live.dot.ga.gov",
+    "vss5live.dot.ga.gov",
+    "511ga.org",
+    "gettingaroundillinois.com",
+    "cctv.travelmidwest.com",
+    "mdotjboss.state.mi.us",
+    "micamerasimages.net",
+    "publicstreamer1.cotrip.org",
+    "publicstreamer2.cotrip.org",
+    "publicstreamer3.cotrip.org",
+    "publicstreamer4.cotrip.org",
+    "cocam.carsprogram.org",
+    "tripcheck.com",
+    "www.tripcheck.com",
+    "infocar.dgt.es",
+    "informo.madrid.es",
+    "www.windy.com",
+    "imgproxy.windy.com",
+    "www.lakecountypassage.com",
+    "webcam.forkswa.com",
+    "webcam.sunmountainlodge.com",
+    "www.nps.gov",
+    "home.lewiscounty.com",
+    "www.seattle.gov",
+}
+
+
+@dataclass(frozen=True)
+class _CCTVProxyProfile:
+    name: str
+    timeout: tuple = (_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0)
+    cache_seconds: int = 30
+    headers: dict = field(default_factory=dict)
+
+
+def _cctv_host_allowed(hostname) -> bool:
+    host = str(hostname or "").strip().lower()
+    if not host:
+        return False
+    for allowed in _CCTV_PROXY_ALLOWED_HOSTS:
+        normalized = str(allowed or "").strip().lower()
+        if host == normalized or host.endswith(f".{normalized}"):
+            return True
+    return False
+
+
+def _proxied_cctv_url(target_url: str) -> str:
+    from urllib.parse import quote
+    return f"/api/cctv/media?url={quote(target_url, safe='')}"
+
+
+def _cctv_proxy_profile_for_url(target_url: str) -> _CCTVProxyProfile:
+    from urllib.parse import urlparse
+    parsed = urlparse(target_url)
+    host = str(parsed.hostname or "").strip().lower()
+    path = str(parsed.path or "").strip().lower()
+
+    if host in {"jamcams.tfl.gov.uk", "s3-eu-west-1.amazonaws.com"}:
+        return _CCTVProxyProfile(name="tfl-jamcam", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 20.0), cache_seconds=15,
+            headers={"Accept": "video/mp4,image/avif,image/webp,image/apng,image/*,*/*;q=0.8", "Referer": "https://tfl.gov.uk/"})
+    if host == "images.data.gov.sg":
+        return _CCTVProxyProfile(name="lta-singapore", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host == "cctv.austinmobility.io":
+        return _CCTVProxyProfile(name="austin-mobility", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://data.mobility.austin.gov/", "Origin": "https://data.mobility.austin.gov"})
+    if host == "webcams.nyctmc.org":
+        return _CCTVProxyProfile(name="nyc-dot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host in {"cwwp2.dot.ca.gov", "wzmedia.dot.ca.gov"}:
+        return _CCTVProxyProfile(name="caltrans", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=15,
+            headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,image/*,*/*;q=0.8",
+                     "Referer": "https://cwwp2.dot.ca.gov/"})
+    if host in {"images.wsdot.wa.gov", "olypen.com", "flyykm.com", "cam.pangbornairport.com"}:
+        return _CCTVProxyProfile(name="wsdot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host in {"www.lakecountypassage.com", "webcam.forkswa.com", "webcam.sunmountainlodge.com", "home.lewiscounty.com", "www.seattle.gov"}:
+        return _CCTVProxyProfile(name="regional-cctv-image", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=45,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": f"https://{host}/"})
+    if host == "www.nps.gov":
+        return _CCTVProxyProfile(name="nps-webcam", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.nps.gov/"})
+    if host in {"navigator-c2c.dot.ga.gov", "navigator-c2c.ga.gov", "navigator-csc.dot.ga.gov"}:
+        read_timeout = 18.0 if "/snapshots/" in path else 12.0
+        return _CCTVProxyProfile(name="gdot-snapshot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, read_timeout), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "http://navigator-c2c.dot.ga.gov/"})
+    if host == "511ga.org":
+        return _CCTVProxyProfile(name="gdot-511ga-image", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://511ga.org/cctv"})
+    if host.startswith("vss") and host.endswith("dot.ga.gov"):
+        return _CCTVProxyProfile(name="gdot-hls", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 20.0), cache_seconds=10,
+            headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,*/*;q=0.8",
+                     "Referer": "http://navigator-c2c.dot.ga.gov/"})
+    if host in {"gettingaroundillinois.com", "cctv.travelmidwest.com"}:
+        return _CCTVProxyProfile(name="illinois-dot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host in {"mdotjboss.state.mi.us", "micamerasimages.net"}:
+        return _CCTVProxyProfile(name="michigan-dot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://mdotjboss.state.mi.us/"})
+    if host in {"publicstreamer1.cotrip.org", "publicstreamer2.cotrip.org",
+                "publicstreamer3.cotrip.org", "publicstreamer4.cotrip.org"}:
+        return _CCTVProxyProfile(name="cotrip-hls", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 20.0), cache_seconds=10,
+            headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,*/*;q=0.8",
+                     "Referer": "https://www.cotrip.org/"})
+    if host == "cocam.carsprogram.org":
+        return _CCTVProxyProfile(name="cotrip-preview", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=20,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.cotrip.org/"})
+    if host in {"tripcheck.com", "www.tripcheck.com"}:
+        return _CCTVProxyProfile(name="odot-tripcheck", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host == "infocar.dgt.es":
+        return _CCTVProxyProfile(name="dgt-spain", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://infocar.dgt.es/"})
+    if host == "informo.madrid.es":
+        return _CCTVProxyProfile(name="madrid-city", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://informo.madrid.es/"})
+    if host in {"www.windy.com", "imgproxy.windy.com"}:
+        return _CCTVProxyProfile(name="windy-webcams", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.windy.com/"})
+    return _CCTVProxyProfile(name="generic-cctv", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0), cache_seconds=30,
+        headers={"Accept": "*/*"})
+
+
+def _cctv_upstream_headers(request: Request, profile: _CCTVProxyProfile) -> dict:
+    headers = {"User-Agent": "Mozilla/5.0 (compatible; ShadowBroker CCTV proxy)", **profile.headers}
+    range_header = request.headers.get("range")
+    if range_header:
+        headers["Range"] = range_header
+    if_none_match = request.headers.get("if-none-match")
+    if if_none_match:
+        headers["If-None-Match"] = if_none_match
+    if_modified_since = request.headers.get("if-modified-since")
+    if if_modified_since:
+        headers["If-Modified-Since"] = if_modified_since
+    return headers
+
+
+def _cctv_response_headers(resp, cache_seconds: int, include_length: bool = True) -> dict:
+    headers = {"Cache-Control": f"public, max-age={cache_seconds}", "Access-Control-Allow-Origin": "*"}
+    for key in ("Accept-Ranges", "Content-Range", "ETag", "Last-Modified"):
+        value = resp.headers.get(key)
+        if value:
+            headers[key] = value
+    if include_length:
+        content_length = resp.headers.get("Content-Length")
+        if content_length:
+            headers["Content-Length"] = content_length
+    return headers
+
+
+def _fetch_cctv_upstream_response(request: Request, target_url: str, profile: _CCTVProxyProfile):
+    import requests as _req
+    headers = _cctv_upstream_headers(request, profile)
+    try:
+        resp = _req.get(target_url, timeout=profile.timeout, stream=True, allow_redirects=True, headers=headers)
+    except _req.exceptions.Timeout as exc:
+        logger.warning("CCTV upstream timeout [%s] %s", profile.name, target_url)
+        raise HTTPException(status_code=504, detail="Upstream timeout") from exc
+    except _req.exceptions.RequestException as exc:
+        logger.warning("CCTV upstream request failure [%s] %s: %s", profile.name, target_url, exc)
+        raise HTTPException(status_code=502, detail="Upstream fetch failed") from exc
+    if resp.status_code >= 400:
+        logger.info("CCTV upstream HTTP %s [%s] %s", resp.status_code, profile.name, target_url)
+        resp.close()
+        raise HTTPException(status_code=int(resp.status_code), detail=f"Upstream returned {resp.status_code}")
+    return resp
+
+
+def _rewrite_cctv_hls_playlist(base_url: str, body: str) -> str:
+    import re
+    from urllib.parse import urljoin, urlparse
+
+    def _rewrite_target(target: str) -> str:
+        candidate = str(target or "").strip()
+        if not candidate or candidate.startswith("data:"):
+            return candidate
+        absolute = urljoin(base_url, candidate)
+        parsed_target = urlparse(absolute)
+        if parsed_target.scheme not in ("http", "https"):
+            return candidate
+        if not _cctv_host_allowed(parsed_target.hostname):
+            return candidate
+        return _proxied_cctv_url(absolute)
+
+    rewritten_lines: list = []
+    for raw_line in body.splitlines():
+        stripped = raw_line.strip()
+        if not stripped:
+            rewritten_lines.append(raw_line)
+            continue
+        if stripped.startswith("#"):
+            rewritten_lines.append(re.sub(r'URI="([^"]+)"',
+                lambda match: f'URI="{_rewrite_target(match.group(1))}"', raw_line))
+            continue
+        rewritten_lines.append(_rewrite_target(stripped))
+    return "\n".join(rewritten_lines) + ("\n" if body.endswith("\n") else "")
+
+
+def _infer_cctv_media_type_from_url(target_url: str, content_type: str) -> str:
+    from urllib.parse import urlparse
+
+    clean_type = str(content_type or "").split(";", 1)[0].strip().lower()
+    if clean_type and clean_type not in {"application/octet-stream", "binary/octet-stream"}:
+        return content_type
+    path = str(urlparse(target_url).path or "").lower()
+    if path.endswith((".jpg", ".jpeg")):
+        return "image/jpeg"
+    if path.endswith(".png"):
+        return "image/png"
+    if path.endswith(".webp"):
+        return "image/webp"
+    if path.endswith(".gif"):
+        return "image/gif"
+    if path.endswith(".mp4"):
+        return "video/mp4"
+    if path.endswith((".m3u8", ".m3u")):
+        return "application/vnd.apple.mpegurl"
+    if path.endswith((".mjpg", ".mjpeg")):
+        return "multipart/x-mixed-replace"
+    return content_type or "application/octet-stream"
+
+
+def _proxy_cctv_media_response(request: Request, target_url: str):
+    from urllib.parse import urlparse
+    from fastapi.responses import Response
+    parsed = urlparse(target_url)
+    profile = _cctv_proxy_profile_for_url(target_url)
+    resp = _fetch_cctv_upstream_response(request, target_url, profile)
+    content_type = _infer_cctv_media_type_from_url(
+        target_url,
+        resp.headers.get("Content-Type", "application/octet-stream"),
+    )
+    is_hls_playlist = (
+        ".m3u8" in str(parsed.path or "").lower()
+        or "mpegurl" in content_type.lower()
+        or "vnd.apple.mpegurl" in content_type.lower()
+    )
+    if is_hls_playlist:
+        body = resp.text
+        if "#EXTM3U" in body:
+            body = _rewrite_cctv_hls_playlist(target_url, body)
+        resp.close()
+        return Response(content=body, media_type=content_type,
+            headers=_cctv_response_headers(resp, cache_seconds=profile.cache_seconds, include_length=False))
+    return StreamingResponse(resp.iter_content(chunk_size=65536), status_code=resp.status_code,
+        media_type=content_type,
+        headers=_cctv_response_headers(resp, cache_seconds=profile.cache_seconds),
+        background=BackgroundTask(resp.close))
+
+
+@router.get("/api/cctv/media")
+@limiter.limit("120/minute")
+async def cctv_media_proxy(request: Request, url: str = Query(...)):
+    """Proxy CCTV media through the backend to bypass browser CORS restrictions."""
+    from urllib.parse import urlparse
+    parsed = urlparse(url)
+    if not _cctv_host_allowed(parsed.hostname):
+        raise HTTPException(status_code=403, detail="Host not allowed")
+    if parsed.scheme not in ("http", "https"):
+        raise HTTPException(status_code=400, detail="Invalid scheme")
+    return _proxy_cctv_media_response(request, url)

backend/routers/data.py

@@ -0,0 +1,623 @@
+import asyncio
+import logging
+import math
+import threading
+from typing import Any
+from fastapi import APIRouter, Request, Response, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+from services.data_fetcher import get_latest_data, update_all_data
+import orjson
+import json as json_mod
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+_refresh_lock = threading.Lock()
+
+
+class ViewportUpdate(BaseModel):
+    s: float
+    w: float
+    n: float
+    e: float
+
+
+class LayerUpdate(BaseModel):
+    layers: dict[str, bool]
+
+
+_LAST_VIEWPORT_UPDATE: tuple | None = None
+_LAST_VIEWPORT_UPDATE_TS = 0.0
+_VIEWPORT_UPDATE_LOCK = threading.Lock()
+_VIEWPORT_DEDUPE_EPSILON = 1.0
+_VIEWPORT_MIN_UPDATE_S = 10.0
+
+
+def _normalize_longitude(value: float) -> float:
+    normalized = ((value + 180.0) % 360.0 + 360.0) % 360.0 - 180.0
+    if normalized == -180.0 and value > 0:
+        return 180.0
+    return normalized
+
+
+def _normalize_viewport_bounds(s: float, w: float, n: float, e: float) -> tuple:
+    south = max(-90.0, min(90.0, s))
+    north = max(-90.0, min(90.0, n))
+    raw_width = abs(e - w)
+    if not math.isfinite(raw_width) or raw_width >= 360.0:
+        return south, -180.0, north, 180.0
+    west = _normalize_longitude(w)
+    east = _normalize_longitude(e)
+    if east < west:
+        return south, -180.0, north, 180.0
+    return south, west, north, east
+
+
+def _viewport_changed_enough(bounds: tuple) -> bool:
+    global _LAST_VIEWPORT_UPDATE, _LAST_VIEWPORT_UPDATE_TS
+    import time
+    now = time.monotonic()
+    with _VIEWPORT_UPDATE_LOCK:
+        if _LAST_VIEWPORT_UPDATE is None:
+            _LAST_VIEWPORT_UPDATE = bounds
+            _LAST_VIEWPORT_UPDATE_TS = now
+            return True
+        changed = any(
+            abs(current - previous) > _VIEWPORT_DEDUPE_EPSILON
+            for current, previous in zip(bounds, _LAST_VIEWPORT_UPDATE)
+        )
+        if not changed and (now - _LAST_VIEWPORT_UPDATE_TS) < _VIEWPORT_MIN_UPDATE_S:
+            return False
+        if (now - _LAST_VIEWPORT_UPDATE_TS) < _VIEWPORT_MIN_UPDATE_S:
+            return False
+        _LAST_VIEWPORT_UPDATE = bounds
+        _LAST_VIEWPORT_UPDATE_TS = now
+        return True
+
+
+def _queue_viirs_change_refresh() -> None:
+    from services.fetchers.earth_observation import fetch_viirs_change_nodes
+    threading.Thread(target=fetch_viirs_change_nodes, daemon=True).start()
+
+
+def _etag_response(request: Request, payload: dict, prefix: str = "", default=None):
+    etag = _current_etag(prefix)
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    content = json_mod.dumps(_json_safe(payload), default=default, allow_nan=False)
+    return Response(content=content, media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"})
+
+
+def _current_etag(prefix: str = "") -> str:
+    from services.fetchers._store import get_active_layers_version, get_data_version
+    return f"{prefix}v{get_data_version()}-l{get_active_layers_version()}"
+
+
+def _json_safe(value):
+    if isinstance(value, float):
+        return value if math.isfinite(value) else None
+    if isinstance(value, dict):
+        return {k: _json_safe(v) for k, v in list(value.items())}
+    if isinstance(value, list):
+        return [_json_safe(v) for v in list(value)]
+    if isinstance(value, tuple):
+        return [_json_safe(v) for v in list(value)]
+    return value
+
+
+def _sanitize_payload(value):
+    if isinstance(value, float):
+        return value if math.isfinite(value) else None
+    if isinstance(value, dict):
+        return {k: _sanitize_payload(v) for k, v in list(value.items())}
+    if isinstance(value, (list, tuple)):
+        return list(value)
+    return value
+
+
+def _bbox_filter(items: list, s: float, w: float, n: float, e: float,
+                 lat_key: str = "lat", lng_key: str = "lng") -> list:
+    pad_lat = (n - s) * 0.2
+    pad_lng = (e - w) * 0.2 if e > w else ((e + 360 - w) * 0.2)
+    s2, n2 = s - pad_lat, n + pad_lat
+    w2, e2 = w - pad_lng, e + pad_lng
+    crosses_antimeridian = w2 > e2
+    out = []
+    for item in items:
+        lat = item.get(lat_key)
+        lng = item.get(lng_key)
+        if lat is None or lng is None:
+            out.append(item)
+            continue
+        if not (s2 <= lat <= n2):
+            continue
+        if crosses_antimeridian:
+            if lng >= w2 or lng <= e2:
+                out.append(item)
+        else:
+            if w2 <= lng <= e2:
+                out.append(item)
+    return out
+
+
+def _bbox_filter_geojson_points(items: list, s: float, w: float, n: float, e: float) -> list:
+    pad_lat = (n - s) * 0.2
+    pad_lng = (e - w) * 0.2 if e > w else ((e + 360 - w) * 0.2)
+    s2, n2 = s - pad_lat, n + pad_lat
+    w2, e2 = w - pad_lng, e + pad_lng
+    crosses_antimeridian = w2 > e2
+    out = []
+    for item in items:
+        geometry = item.get("geometry") if isinstance(item, dict) else None
+        coords = geometry.get("coordinates") if isinstance(geometry, dict) else None
+        if not isinstance(coords, (list, tuple)) or len(coords) < 2:
+            out.append(item)
+            continue
+        lng, lat = coords[0], coords[1]
+        if lat is None or lng is None:
+            out.append(item)
+            continue
+        if not (s2 <= lat <= n2):
+            continue
+        if crosses_antimeridian:
+            if lng >= w2 or lng <= e2:
+                out.append(item)
+        else:
+            if w2 <= lng <= e2:
+                out.append(item)
+    return out
+
+
+def _bbox_spans(s, w, n, e) -> tuple:
+    if None in (s, w, n, e):
+        return 180.0, 360.0
+    lat_span = max(0.0, float(n) - float(s))
+    lng_span = float(e) - float(w)
+    if lng_span < 0:
+        lng_span += 360.0
+    if lng_span == 0 and w == -180 and e == 180:
+        lng_span = 360.0
+    return lat_span, max(0.0, lng_span)
+
+
+def _cap_startup_items(items: list | None, max_items: int) -> list:
+    if not items:
+        return []
+    if len(items) <= max_items:
+        return items
+    return items[:max_items]
+
+
+def _cap_fast_startup_payload(payload: dict) -> dict:
+    capped = dict(payload)
+    capped["commercial_flights"] = _cap_startup_items(capped.get("commercial_flights"), 800)
+    capped["private_flights"] = _cap_startup_items(capped.get("private_flights"), 300)
+    capped["private_jets"] = _cap_startup_items(capped.get("private_jets"), 150)
+    capped["ships"] = _cap_startup_items(capped.get("ships"), 1500)
+    capped["cctv"] = []
+    capped["sigint"] = _cap_startup_items(capped.get("sigint"), 500)
+    capped["trains"] = _cap_startup_items(capped.get("trains"), 100)
+    capped["startup_payload"] = True
+    return capped
+
+
+def _cap_fast_dashboard_payload(payload: dict) -> dict:
+    return payload
+
+
+def _world_and_continental_scale(has_bbox: bool, s, w, n, e) -> tuple:
+    lat_span, lng_span = _bbox_spans(s, w, n, e)
+    world_scale = (not has_bbox) or lng_span >= 300 or lat_span >= 120
+    continental_scale = has_bbox and not world_scale and (lng_span >= 120 or lat_span >= 55)
+    return world_scale, continental_scale
+
+
+def _filter_sigint_by_layers(items: list, active_layers: dict) -> list:
+    allow_aprs = bool(active_layers.get("sigint_aprs", True))
+    allow_mesh = bool(active_layers.get("sigint_meshtastic", True))
+    if allow_aprs and allow_mesh:
+        return items
+    allowed_sources: set = {"js8call"}
+    if allow_aprs:
+        allowed_sources.add("aprs")
+    if allow_mesh:
+        allowed_sources.update({"meshtastic", "meshtastic-map"})
+    return [item for item in items if str(item.get("source") or "").lower() in allowed_sources]
+
+
+def _sigint_totals_for_items(items: list) -> dict:
+    totals = {"total": len(items), "meshtastic": 0, "meshtastic_live": 0, "meshtastic_map": 0,
+              "aprs": 0, "js8call": 0}
+    for item in items:
+        source = str(item.get("source") or "").lower()
+        if source == "meshtastic":
+            totals["meshtastic"] += 1
+            if bool(item.get("from_api")):
+                totals["meshtastic_map"] += 1
+            else:
+                totals["meshtastic_live"] += 1
+        elif source == "aprs":
+            totals["aprs"] += 1
+        elif source == "js8call":
+            totals["js8call"] += 1
+    return totals
+
+
+@router.get("/api/refresh", dependencies=[Depends(require_admin)])
+@limiter.limit("2/minute")
+async def force_refresh(request: Request):
+    from services.schemas import RefreshResponse
+    if not _refresh_lock.acquire(blocking=False):
+        return {"status": "refresh already in progress"}
+
+    def _do_refresh():
+        try:
+            update_all_data()
+        finally:
+            _refresh_lock.release()
+
+    t = threading.Thread(target=_do_refresh)
+    t.start()
+    return {"status": "refreshing in background"}
+
+
+@router.post("/api/ais/feed")
+@limiter.limit("60/minute")
+async def ais_feed(request: Request):
+    """Accept AIS-catcher HTTP JSON feed (POST decoded AIS messages)."""
+    from services.ais_stream import ingest_ais_catcher
+    try:
+        body = await request.json()
+    except Exception:
+        return JSONResponse(status_code=422, content={"ok": False, "detail": "invalid JSON body"})
+    msgs = body.get("msgs", [])
+    if not msgs:
+        return {"status": "ok", "ingested": 0}
+    count = ingest_ais_catcher(msgs)
+    return {"status": "ok", "ingested": count}
+
+
+@router.get("/api/trail/flight/{icao24}")
+@limiter.limit("120/minute")
+async def get_selected_flight_trail(icao24: str, request: Request):  # noqa: ARG001
+    from services.fetchers.flights import get_flight_trail
+    return {"id": icao24, "trail": get_flight_trail(icao24)}
+
+
+@router.get("/api/trail/ship/{mmsi}")
+@limiter.limit("120/minute")
+async def get_selected_ship_trail(mmsi: int, request: Request):  # noqa: ARG001
+    from services.ais_stream import get_vessel_trail
+    return {"id": mmsi, "trail": get_vessel_trail(mmsi)}
+
+
+@router.post("/api/viewport")
+@limiter.limit("60/minute")
+async def update_viewport(vp: ViewportUpdate, request: Request):  # noqa: ARG001
+    """Receive frontend map bounds. AIS stream stays global so open-ocean
+    vessels are never dropped — the frontend worker handles viewport culling."""
+    return {"status": "ok"}
+
+
+@router.post("/api/layers")
+@limiter.limit("30/minute")
+async def update_layers(update: LayerUpdate, request: Request):
+    """Receive frontend layer toggle state. Starts/stops streams accordingly."""
+    from services.fetchers._store import active_layers, bump_active_layers_version, is_any_active
+    old_ships = is_any_active("ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts")
+    old_mesh = is_any_active("sigint_meshtastic")
+    old_aprs = is_any_active("sigint_aprs")
+    old_viirs = is_any_active("viirs_nightlights")
+    changed = False
+    for key, value in update.layers.items():
+        if key in active_layers:
+            if active_layers[key] != value:
+                changed = True
+            active_layers[key] = value
+    if changed:
+        bump_active_layers_version()
+    new_ships = is_any_active("ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts")
+    new_mesh = is_any_active("sigint_meshtastic")
+    new_aprs = is_any_active("sigint_aprs")
+    new_viirs = is_any_active("viirs_nightlights")
+    if old_ships and not new_ships:
+        from services.ais_stream import stop_ais_stream
+        stop_ais_stream()
+        logger.info("AIS stream stopped (all ship layers disabled)")
+    elif not old_ships and new_ships:
+        from services.ais_stream import start_ais_stream
+        start_ais_stream()
+        logger.info("AIS stream started (ship layer enabled)")
+    from services.sigint_bridge import sigint_grid
+    if old_mesh and not new_mesh:
+        try:
+            from services.meshtastic_mqtt_settings import mqtt_bridge_enabled
+            keep_chat_running = mqtt_bridge_enabled()
+        except Exception:
+            keep_chat_running = False
+        if keep_chat_running:
+            logger.info("Meshtastic map layer disabled; MQTT bridge kept running for MeshChat")
+        else:
+            sigint_grid.mesh.stop()
+            logger.info("Meshtastic MQTT bridge stopped (layer disabled)")
+    elif not old_mesh and new_mesh:
+        try:
+            from services.meshtastic_mqtt_settings import mqtt_bridge_enabled
+            mqtt_enabled = mqtt_bridge_enabled()
+        except Exception:
+            mqtt_enabled = False
+        if mqtt_enabled:
+            sigint_grid.mesh.start()
+            logger.info("Meshtastic MQTT bridge started (layer enabled)")
+        else:
+            logger.info(
+                "Meshtastic layer enabled; MQTT bridge remains disabled "
+                "(set MESH_MQTT_ENABLED=true to participate in the public broker)"
+            )
+    if old_aprs and not new_aprs:
+        sigint_grid.aprs.stop()
+        logger.info("APRS bridge stopped (layer disabled)")
+    elif not old_aprs and new_aprs:
+        sigint_grid.aprs.start()
+        logger.info("APRS bridge started (layer enabled)")
+    if not old_viirs and new_viirs:
+        _queue_viirs_change_refresh()
+        logger.info("VIIRS change refresh queued (layer enabled)")
+    return {"status": "ok"}
+
+
+@router.get("/api/live-data")
+@limiter.limit("120/minute")
+async def live_data(request: Request):
+    return get_latest_data()
+
+
+@router.get("/api/bootstrap/critical")
+@limiter.limit("180/minute")
+async def bootstrap_critical(request: Request):
+    """Cached first-paint payload for the dashboard.
+
+    This endpoint is intentionally memory-only: no upstream calls, no refresh,
+    and a bounded response. It exists so the map and threat feed can paint
+    before slower panels and background enrichers finish warming up.
+    """
+    etag = _current_etag(prefix="bootstrap|critical|")
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    from services.fetchers._store import (
+        active_layers,
+        get_latest_data_subset_refs,
+        get_source_timestamps_snapshot,
+    )
+
+    d = get_latest_data_subset_refs(
+        "last_updated", "commercial_flights", "military_flights", "private_flights",
+        "private_jets", "tracked_flights", "ships", "uavs", "liveuamap", "gps_jamming",
+        "satellites", "satellite_source", "satellite_analysis", "sigint", "sigint_totals",
+        "trains", "news", "gdelt", "airports", "threat_level", "trending_markets",
+        "correlations", "fimi", "crowdthreat",
+    )
+    freshness = get_source_timestamps_snapshot()
+    ships_enabled = any(active_layers.get(key, True) for key in (
+        "ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts"))
+    sigint_items = _filter_sigint_by_layers(d.get("sigint") or [], active_layers)
+    payload = {
+        "last_updated": d.get("last_updated"),
+        "commercial_flights": _cap_startup_items(
+            (d.get("commercial_flights") or []) if active_layers.get("flights", True) else [],
+            800,
+        ),
+        "military_flights": _cap_startup_items(
+            (d.get("military_flights") or []) if active_layers.get("military", True) else [],
+            300,
+        ),
+        "private_flights": _cap_startup_items(
+            (d.get("private_flights") or []) if active_layers.get("private", True) else [],
+            300,
+        ),
+        "private_jets": _cap_startup_items(
+            (d.get("private_jets") or []) if active_layers.get("jets", True) else [],
+            150,
+        ),
+        "tracked_flights": _cap_startup_items(
+            (d.get("tracked_flights") or []) if active_layers.get("tracked", True) else [],
+            250,
+        ),
+        "ships": _cap_startup_items((d.get("ships") or []) if ships_enabled else [], 1500),
+        "uavs": _cap_startup_items((d.get("uavs") or []) if active_layers.get("military", True) else [], 100),
+        "liveuamap": _cap_startup_items(
+            (d.get("liveuamap") or []) if active_layers.get("global_incidents", True) else [],
+            300,
+        ),
+        "gps_jamming": _cap_startup_items(
+            (d.get("gps_jamming") or []) if active_layers.get("gps_jamming", True) else [],
+            200,
+        ),
+        "satellites": _cap_startup_items(
+            (d.get("satellites") or []) if active_layers.get("satellites", True) else [],
+            250,
+        ),
+        "satellite_source": d.get("satellite_source", "none"),
+        "satellite_analysis": (d.get("satellite_analysis") or {}) if active_layers.get("satellites", True) else {},
+        "sigint": _cap_startup_items(
+            sigint_items if (active_layers.get("sigint_meshtastic", True) or active_layers.get("sigint_aprs", True)) else [],
+            500,
+        ),
+        "sigint_totals": _sigint_totals_for_items(sigint_items),
+        "trains": _cap_startup_items((d.get("trains") or []) if active_layers.get("trains", True) else [], 100),
+        "news": _cap_startup_items(d.get("news") or [], 30),
+        "gdelt": _cap_startup_items((d.get("gdelt") or []) if active_layers.get("global_incidents", True) else [], 300),
+        "airports": _cap_startup_items(d.get("airports") or [], 500),
+        "threat_level": d.get("threat_level"),
+        "trending_markets": _cap_startup_items(d.get("trending_markets") or [], 10),
+        "correlations": _cap_startup_items(
+            (d.get("correlations") or []) if active_layers.get("correlations", True) else [],
+            50,
+        ),
+        "fimi": d.get("fimi"),
+        "crowdthreat": _cap_startup_items(
+            (d.get("crowdthreat") or []) if active_layers.get("crowdthreat", True) else [],
+            150,
+        ),
+        "freshness": freshness,
+        "bootstrap_ready": True,
+        "bootstrap_payload": True,
+    }
+    return Response(
+        content=orjson.dumps(_sanitize_payload(payload), default=str, option=orjson.OPT_NON_STR_KEYS),
+        media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"},
+    )
+
+
+@router.get("/api/live-data/fast")
+@limiter.limit("120/minute")
+async def live_data_fast(
+    request: Request,
+    s: float = Query(None, description="South bound (ignored)", ge=-90, le=90),
+    w: float = Query(None, description="West bound (ignored)", ge=-180, le=180),
+    n: float = Query(None, description="North bound (ignored)", ge=-90, le=90),
+    e: float = Query(None, description="East bound (ignored)", ge=-180, le=180),
+    initial: bool = Query(False, description="Return a capped startup payload for first paint"),
+):
+    etag = _current_etag(prefix="fast|initial|" if initial else "fast|full|")
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    from services.fetchers._store import (active_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot)
+    d = get_latest_data_subset_refs(
+        "last_updated", "commercial_flights", "military_flights", "private_flights",
+        "private_jets", "tracked_flights", "ships", "cctv", "uavs", "liveuamap",
+        "gps_jamming", "satellites", "satellite_source", "satellite_analysis",
+        "sigint", "sigint_totals", "trains",
+    )
+    freshness = get_source_timestamps_snapshot()
+    ships_enabled = any(active_layers.get(key, True) for key in (
+        "ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts"))
+    cctv_total = len(d.get("cctv") or [])
+    sigint_items = _filter_sigint_by_layers(d.get("sigint") or [], active_layers)
+    sigint_totals = _sigint_totals_for_items(sigint_items)
+    payload = {
+        "commercial_flights": (d.get("commercial_flights") or []) if active_layers.get("flights", True) else [],
+        "military_flights": (d.get("military_flights") or []) if active_layers.get("military", True) else [],
+        "private_flights": (d.get("private_flights") or []) if active_layers.get("private", True) else [],
+        "private_jets": (d.get("private_jets") or []) if active_layers.get("jets", True) else [],
+        "tracked_flights": (d.get("tracked_flights") or []) if active_layers.get("tracked", True) else [],
+        "ships": (d.get("ships") or []) if ships_enabled else [],
+        "cctv": (d.get("cctv") or []) if active_layers.get("cctv", True) else [],
+        "uavs": (d.get("uavs") or []) if active_layers.get("military", True) else [],
+        "liveuamap": (d.get("liveuamap") or []) if active_layers.get("global_incidents", True) else [],
+        "gps_jamming": (d.get("gps_jamming") or []) if active_layers.get("gps_jamming", True) else [],
+        "satellites": (d.get("satellites") or []) if active_layers.get("satellites", True) else [],
+        "satellite_source": d.get("satellite_source", "none"),
+        "satellite_analysis": (d.get("satellite_analysis") or {}) if active_layers.get("satellites", True) else {},
+        "sigint": sigint_items if (active_layers.get("sigint_meshtastic", True) or active_layers.get("sigint_aprs", True)) else [],
+        "sigint_totals": sigint_totals,
+        "cctv_total": cctv_total,
+        "trains": (d.get("trains") or []) if active_layers.get("trains", True) else [],
+        "freshness": freshness,
+    }
+    if initial:
+        payload = _cap_fast_startup_payload(payload)
+    else:
+        payload = _cap_fast_dashboard_payload(payload)
+    return Response(content=orjson.dumps(_sanitize_payload(payload)), media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"})
+
+
+@router.get("/api/live-data/slow")
+@limiter.limit("60/minute")
+async def live_data_slow(
+    request: Request,
+    s: float = Query(None, description="South bound (ignored)", ge=-90, le=90),
+    w: float = Query(None, description="West bound (ignored)", ge=-180, le=180),
+    n: float = Query(None, description="North bound (ignored)", ge=-90, le=90),
+    e: float = Query(None, description="East bound (ignored)", ge=-180, le=180),
+):
+    etag = _current_etag(prefix="slow|full|")
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    from services.fetchers._store import (active_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot)
+    d = get_latest_data_subset_refs(
+        "last_updated", "news", "stocks", "financial_source", "oil", "weather", "traffic",
+        "earthquakes", "frontlines", "gdelt", "airports", "kiwisdr", "satnogs_stations",
+        "satnogs_observations", "tinygs_satellites", "space_weather", "internet_outages",
+        "firms_fires", "datacenters", "military_bases", "power_plants", "viirs_change_nodes",
+        "scanners", "weather_alerts", "ukraine_alerts", "air_quality", "volcanoes",
+        "fishing_activity", "psk_reporter", "correlations", "uap_sightings", "wastewater",
+        "crowdthreat", "threat_level", "trending_markets",
+    )
+    freshness = get_source_timestamps_snapshot()
+    payload = {
+        "last_updated": d.get("last_updated"),
+        "threat_level": d.get("threat_level"),
+        "trending_markets": d.get("trending_markets", []),
+        "news": d.get("news", []),
+        "stocks": d.get("stocks", {}),
+        "financial_source": d.get("financial_source", ""),
+        "oil": d.get("oil", {}),
+        "weather": d.get("weather"),
+        "traffic": d.get("traffic", []),
+        "earthquakes": (d.get("earthquakes") or []) if active_layers.get("earthquakes", True) else [],
+        "frontlines": d.get("frontlines") if active_layers.get("ukraine_frontline", True) else None,
+        "gdelt": (d.get("gdelt") or []) if active_layers.get("global_incidents", True) else [],
+        "airports": d.get("airports") or [],
+        "kiwisdr": (d.get("kiwisdr") or []) if active_layers.get("kiwisdr", True) else [],
+        "satnogs_stations": (d.get("satnogs_stations") or []) if active_layers.get("satnogs", True) else [],
+        "satnogs_total": len(d.get("satnogs_stations") or []),
+        "satnogs_observations": (d.get("satnogs_observations") or []) if active_layers.get("satnogs", True) else [],
+        "tinygs_satellites": (d.get("tinygs_satellites") or []) if active_layers.get("tinygs", True) else [],
+        "tinygs_total": len(d.get("tinygs_satellites") or []),
+        "psk_reporter": (d.get("psk_reporter") or []) if active_layers.get("psk_reporter", True) else [],
+        "space_weather": d.get("space_weather"),
+        "internet_outages": (d.get("internet_outages") or []) if active_layers.get("internet_outages", True) else [],
+        "firms_fires": (d.get("firms_fires") or []) if active_layers.get("firms", True) else [],
+        "datacenters": (d.get("datacenters") or []) if active_layers.get("datacenters", True) else [],
+        "military_bases": (d.get("military_bases") or []) if active_layers.get("military_bases", True) else [],
+        "power_plants": (d.get("power_plants") or []) if active_layers.get("power_plants", True) else [],
+        "viirs_change_nodes": (d.get("viirs_change_nodes") or []) if active_layers.get("viirs_nightlights", True) else [],
+        "scanners": (d.get("scanners") or []) if active_layers.get("scanners", True) else [],
+        "weather_alerts": d.get("weather_alerts", []) if active_layers.get("weather_alerts", True) else [],
+        "ukraine_alerts": d.get("ukraine_alerts", []) if active_layers.get("ukraine_alerts", True) else [],
+        "air_quality": (d.get("air_quality") or []) if active_layers.get("air_quality", True) else [],
+        "volcanoes": (d.get("volcanoes") or []) if active_layers.get("volcanoes", True) else [],
+        "fishing_activity": (d.get("fishing_activity") or []) if active_layers.get("fishing_activity", True) else [],
+        "correlations": (d.get("correlations") or []) if active_layers.get("correlations", True) else [],
+        "uap_sightings": (d.get("uap_sightings") or []) if active_layers.get("uap_sightings", True) else [],
+        "wastewater": (d.get("wastewater") or []) if active_layers.get("wastewater", True) else [],
+        "crowdthreat": (d.get("crowdthreat") or []) if active_layers.get("crowdthreat", True) else [],
+        "freshness": freshness,
+    }
+    return Response(
+        content=orjson.dumps(_sanitize_payload(payload), default=str, option=orjson.OPT_NON_STR_KEYS),
+        media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"},
+    )
+
+
+# ── Satellite Overflight Counting ───────────────────────────────────────────
+# Counts unique satellites whose ground track entered a bounding box over 24h.
+# Uses cached TLEs + SGP4 propagation — no extra network requests.
+
+class OverflightRequest(BaseModel):
+    s: float
+    w: float
+    n: float
+    e: float
+    hours: int = 24
+
+
+@router.post("/api/satellites/overflights")
+@limiter.limit("10/minute")
+async def satellite_overflights(request: Request, body: OverflightRequest):
+    from services.fetchers.satellites import compute_overflights, _sat_gp_cache
+    gp_data = _sat_gp_cache.get("data")
+    if not gp_data:
+        return JSONResponse({"total": 0, "by_mission": {}, "satellites": [], "error": "No GP data cached yet"})
+    bbox = {"s": body.s, "w": body.w, "n": body.n, "e": body.e}
+    result = compute_overflights(gp_data, bbox, hours=body.hours)
+    return JSONResponse(result)

backend/routers/health.py

@@ -0,0 +1,85 @@
+import time as _time_mod
+from fastapi import APIRouter, Request, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin
+from services.data_fetcher import get_latest_data
+from services.schemas import HealthResponse
+import os
+
+APP_VERSION = os.environ.get("_HEALTH_APP_VERSION", "0.9.79")
+
+router = APIRouter()
+
+
+def _get_app_version() -> str:
+    # Import lazily to avoid circular import; main sets APP_VERSION before including routers
+    try:
+        import main as _main
+        return _main.APP_VERSION
+    except Exception:
+        return APP_VERSION
+
+
+_start_time_ref: dict = {"value": None}
+
+
+def _get_start_time() -> float:
+    if _start_time_ref["value"] is None:
+        try:
+            import main as _main
+            _start_time_ref["value"] = _main._start_time
+        except Exception:
+            _start_time_ref["value"] = _time_mod.time()
+    return _start_time_ref["value"]
+
+
+@router.get("/api/health", response_model=HealthResponse)
+@limiter.limit("30/minute")
+async def health_check(request: Request):
+    from services.fetchers._store import get_source_timestamps_snapshot
+    from services.slo import compute_all_statuses, summarise_statuses
+
+    d = get_latest_data()
+    last = d.get("last_updated")
+    timestamps = get_source_timestamps_snapshot()
+    slo_statuses = compute_all_statuses(d, timestamps)
+    slo_summary = summarise_statuses(slo_statuses)
+    # Top-level status reflects worst SLO result — "degraded" if any
+    # yellow, "error" if any red, "ok" otherwise. This is the single
+    # field an external probe / pager can watch.
+    top_status = "ok"
+    if slo_summary.get("red", 0) > 0:
+        top_status = "error"
+    elif slo_summary.get("yellow", 0) > 0:
+        top_status = "degraded"
+    return {
+        "status": top_status,
+        "version": _get_app_version(),
+        "last_updated": last,
+        "sources": {
+            "flights": len(d.get("commercial_flights", [])),
+            "military": len(d.get("military_flights", [])),
+            "ships": len(d.get("ships", [])),
+            "satellites": len(d.get("satellites", [])),
+            "earthquakes": len(d.get("earthquakes", [])),
+            "cctv": len(d.get("cctv", [])),
+            "news": len(d.get("news", [])),
+            "uavs": len(d.get("uavs", [])),
+            "firms_fires": len(d.get("firms_fires", [])),
+            "liveuamap": len(d.get("liveuamap", [])),
+            "gdelt": len(d.get("gdelt", [])),
+            "uap_sightings": len(d.get("uap_sightings", [])),
+        },
+        "freshness": timestamps,
+        "uptime_seconds": round(_time_mod.time() - _get_start_time()),
+        "slo": slo_statuses,
+        "slo_summary": slo_summary,
+    }
+
+
+@router.get("/api/debug-latest", dependencies=[Depends(require_admin)])
+@limiter.limit("30/minute")
+async def debug_latest_data(request: Request):
+    return list(get_latest_data().keys())

backend/routers/infonet.py

@@ -0,0 +1,598 @@
+"""Infonet economy / governance / gates / bootstrap HTTP surface.
+
+Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §2.1.
+
+Read endpoints return chain-derived state (computed by the
+``services.infonet`` adapters / pure functions). Write endpoints take
+a payload, validate it through the cutover-registered validators, and
+return a structured "would-emit" preview. Production wiring (signing
++ ``Infonet.append`` persistence) is a thin follow-on; the validation
+contract is locked here.
+
+Cross-cutting design rule: errors are diagnostic, not punitive. Each
+write endpoint returns ``{"ok": False, "reason": "..."}`` on
+validation failure with the exact field that failed. Frontend
+surfaces the reason in the UI.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any
+
+from fastapi import APIRouter, Body, Path
+
+# Triggers the chain cutover at module-load time so registered
+# validators are live for any subsequent route invocation.
+from services.infonet import _chain_cutover  # noqa: F401
+from services.infonet.adapters.gate_adapter import InfonetGateAdapter
+from services.infonet.adapters.oracle_adapter import InfonetOracleAdapter
+from services.infonet.adapters.reputation_adapter import InfonetReputationAdapter
+from services.infonet.bootstrap import compute_active_features
+from services.infonet.config import (
+    CONFIG,
+    IMMUTABLE_PRINCIPLES,
+)
+from services.infonet.governance import (
+    apply_petition_payload,
+    compute_petition_state,
+    compute_upgrade_state,
+)
+from services.infonet.governance.dsl_executor import InvalidPetition
+from services.infonet.partition import (
+    classify_event_type,
+    is_chain_stale,
+    should_mark_provisional,
+)
+from services.infonet.privacy import (
+    DEXScaffolding,
+    RingCTScaffolding,
+    ShieldedBalanceScaffolding,
+    StealthAddressScaffolding,
+)
+from services.infonet.schema import (
+    INFONET_ECONOMY_EVENT_TYPES,
+    validate_infonet_event_payload,
+)
+from services.infonet.time_validity import chain_majority_time
+
+logger = logging.getLogger("routers.infonet")
+
+router = APIRouter(prefix="/api/infonet", tags=["infonet"])
+
+
+# ─── Chain access helper ─────────────────────────────────────────────────
+# Every adapter takes a ``chain_provider`` callable. We pull the live
+# Infonet chain from mesh_hashchain. Tests can monkeypatch this.
+
+def _live_chain() -> list[dict[str, Any]]:
+    try:
+        from services.mesh.mesh_hashchain import infonet
+        events = getattr(infonet, "events", None)
+        if isinstance(events, list):
+            return list(events)
+        # Some implementations use a deque; convert to list.
+        if events is not None:
+            return list(events)
+    except Exception as exc:
+        logger.debug("infonet chain unavailable: %s", exc)
+    return []
+
+
+def _now() -> float:
+    cmt = chain_majority_time(_live_chain())
+    return cmt if cmt > 0 else float(time.time())
+
+
+# ─── Status ──────────────────────────────────────────────────────────────
+
+@router.get("/status")
+def infonet_status() -> dict[str, Any]:
+    """Top-level health snapshot for the InfonetTerminal HUD.
+
+    Returns ramp activation flags, partition staleness, privacy
+    primitive statuses, immutable principles, and counts of
+    chain-derived state (markets / petitions / gates / etc).
+    """
+    chain = _live_chain()
+    now = _now()
+    features = compute_active_features(chain)
+
+    # Privacy primitive statuses (truthful — most are NOT_IMPLEMENTED).
+    privacy = {
+        "ringct": RingCTScaffolding().status().value,
+        "stealth_address": StealthAddressScaffolding().status().value,
+        "shielded_balance": ShieldedBalanceScaffolding().status().value,
+        "dex": DEXScaffolding().status().value,
+    }
+
+    return {
+        "ok": True,
+        "now": now,
+        "chain_majority_time": chain_majority_time(chain),
+        "chain_event_count": len(chain),
+        "chain_stale": is_chain_stale(chain, now=now),
+        "ramp": {
+            "node_count": features.node_count,
+            "bootstrap_resolution_active": features.bootstrap_resolution_active,
+            "staked_resolution_active": features.staked_resolution_active,
+            "governance_petitions_active": features.governance_petitions_active,
+            "upgrade_governance_active": features.upgrade_governance_active,
+            "commoncoin_active": features.commoncoin_active,
+        },
+        "privacy_primitive_status": privacy,
+        "immutable_principles": dict(IMMUTABLE_PRINCIPLES),
+        "config_keys_count": len(CONFIG),
+        "infonet_economy_event_types_count": len(INFONET_ECONOMY_EVENT_TYPES),
+    }
+
+
+# ─── Petitions / governance ──────────────────────────────────────────────
+
+@router.get("/petitions")
+def list_petitions() -> dict[str, Any]:
+    """List petition_file events on the chain with their current state."""
+    chain = _live_chain()
+    now = _now()
+    out: list[dict[str, Any]] = []
+    for ev in chain:
+        if ev.get("event_type") != "petition_file":
+            continue
+        pid = (ev.get("payload") or {}).get("petition_id")
+        if not isinstance(pid, str):
+            continue
+        try:
+            state = compute_petition_state(pid, chain, now=now)
+            out.append({
+                "petition_id": state.petition_id,
+                "status": state.status,
+                "filer_id": state.filer_id,
+                "filed_at": state.filed_at,
+                "petition_payload": state.petition_payload,
+                "signature_governance_weight": state.signature_governance_weight,
+                "signature_threshold_at_filing": state.signature_threshold_at_filing,
+                "votes_for_weight": state.votes_for_weight,
+                "votes_against_weight": state.votes_against_weight,
+                "voting_deadline": state.voting_deadline,
+                "challenge_window_until": state.challenge_window_until,
+            })
+        except Exception as exc:
+            logger.warning("petition state error for %s: %s", pid, exc)
+    return {"ok": True, "petitions": out, "now": now}
+
+
+@router.get("/petitions/{petition_id}")
+def get_petition(petition_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    state = compute_petition_state(petition_id, chain, now=now)
+    return {"ok": True, "petition": state.__dict__, "now": now}
+
+
+@router.post("/petitions/preview")
+def preview_petition_payload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
+    """Validate a petition payload through the DSL executor without
+    emitting it. Returns the candidate config diff so the UI can show
+    "this petition would change vote_decay_days from 90 to 30".
+    """
+    try:
+        result = apply_petition_payload(payload)
+        return {
+            "ok": True,
+            "changed_keys": list(result.changed_keys),
+            "new_values": {k: result.new_config[k] for k in result.changed_keys},
+        }
+    except InvalidPetition as exc:
+        return {"ok": False, "reason": str(exc)}
+
+
+@router.post("/events/validate")
+def validate_event(body: dict[str, Any] = Body(...)) -> dict[str, Any]:
+    """Validate an arbitrary Infonet economy event payload.
+
+    Frontend uses this for client-side preflight before signing /
+    submitting an event. Returns ``{ok: True}`` on success or
+    ``{ok: False, reason: ...}`` with the exact validation failure.
+    """
+    event_type = body.get("event_type")
+    payload = body.get("payload", {})
+    if not isinstance(event_type, str) or not event_type:
+        return {"ok": False, "reason": "event_type required"}
+    if not isinstance(payload, dict):
+        return {"ok": False, "reason": "payload must be an object"}
+    ok, reason = validate_infonet_event_payload(event_type, payload)
+    return {
+        "ok": ok,
+        "reason": reason if not ok else None,
+        "tier": classify_event_type(event_type),
+        "would_be_provisional": should_mark_provisional(event_type, _live_chain(), now=_now()),
+    }
+
+
+# ─── Upgrade-hash governance ────────────────────────────────────────────
+
+@router.get("/upgrades")
+def list_upgrades() -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    out: list[dict[str, Any]] = []
+    for ev in chain:
+        if ev.get("event_type") != "upgrade_propose":
+            continue
+        pid = (ev.get("payload") or {}).get("proposal_id")
+        if not isinstance(pid, str):
+            continue
+        try:
+            # Heavy node set is a runtime concept (transport tier ==
+            # private_strong per plan §3.5). Empty here for the
+            # snapshot endpoint; production will pass the live set.
+            state = compute_upgrade_state(pid, chain, now=now, heavy_node_ids=set())
+            out.append({
+                "proposal_id": state.proposal_id,
+                "status": state.status,
+                "proposer_id": state.proposer_id,
+                "filed_at": state.filed_at,
+                "release_hash": state.release_hash,
+                "target_protocol_version": state.target_protocol_version,
+                "votes_for_weight": state.votes_for_weight,
+                "votes_against_weight": state.votes_against_weight,
+                "readiness_fraction": state.readiness.fraction,
+                "readiness_threshold_met": state.readiness.threshold_met,
+            })
+        except Exception as exc:
+            logger.warning("upgrade state error for %s: %s", pid, exc)
+    return {"ok": True, "upgrades": out, "now": now}
+
+
+@router.get("/upgrades/{proposal_id}")
+def get_upgrade(proposal_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    state = compute_upgrade_state(proposal_id, chain, now=now, heavy_node_ids=set())
+    return {
+        "ok": True,
+        "upgrade": {
+            "proposal_id": state.proposal_id,
+            "status": state.status,
+            "proposer_id": state.proposer_id,
+            "filed_at": state.filed_at,
+            "release_hash": state.release_hash,
+            "target_protocol_version": state.target_protocol_version,
+            "signature_governance_weight": state.signature_governance_weight,
+            "votes_for_weight": state.votes_for_weight,
+            "votes_against_weight": state.votes_against_weight,
+            "voting_deadline": state.voting_deadline,
+            "challenge_window_until": state.challenge_window_until,
+            "activation_deadline": state.activation_deadline,
+            "readiness": {
+                "total_heavy_nodes": state.readiness.total_heavy_nodes,
+                "ready_count": state.readiness.ready_count,
+                "fraction": state.readiness.fraction,
+                "threshold_met": state.readiness.threshold_met,
+            },
+        },
+        "now": now,
+    }
+
+
+# ─── Markets / resolution / disputes ────────────────────────────────────
+
+@router.get("/markets/{market_id}")
+def get_market_state(market_id: str = Path(...)) -> dict[str, Any]:
+    """Full market view: lifecycle, snapshot, evidence, stakes,
+    excluded predictors, dispute state."""
+    chain = _live_chain()
+    now = _now()
+    oracle = InfonetOracleAdapter(lambda: chain)
+
+    status = oracle.market_status(market_id, now=now)
+    snap = oracle.find_snapshot(market_id)
+    bundles = oracle.collect_evidence(market_id)
+    excluded = sorted(oracle.excluded_predictor_ids(market_id))
+    disputes = oracle.collect_disputes(market_id)
+    reversed_flag = oracle.market_was_reversed(market_id)
+
+    return {
+        "ok": True,
+        "market_id": market_id,
+        "status": status.value,
+        "snapshot": snap,
+        "evidence_bundles": [
+            {
+                "node_id": b.node_id,
+                "claimed_outcome": b.claimed_outcome,
+                "evidence_hashes": list(b.evidence_hashes),
+                "source_description": b.source_description,
+                "bond": b.bond,
+                "timestamp": b.timestamp,
+                "is_first_for_side": b.is_first_for_side,
+                "submission_hash": b.submission_hash,
+            }
+            for b in bundles
+        ],
+        "excluded_predictor_ids": excluded,
+        "disputes": [
+            {
+                "dispute_id": d.dispute_id,
+                "challenger_id": d.challenger_id,
+                "challenger_stake": d.challenger_stake,
+                "opened_at": d.opened_at,
+                "is_resolved": d.is_resolved,
+                "resolved_outcome": d.resolved_outcome,
+                "confirm_stakes": d.confirm_stakes,
+                "reverse_stakes": d.reverse_stakes,
+            }
+            for d in disputes
+        ],
+        "was_reversed": reversed_flag,
+        "now": now,
+    }
+
+
+@router.get("/markets/{market_id}/preview-resolution")
+def preview_resolution(market_id: str = Path(...)) -> dict[str, Any]:
+    """Run the resolution decision procedure without emitting a
+    finalize event. UI uses this to show "if resolution closed now,
+    the market would resolve as <outcome> for <reason>"."""
+    chain = _live_chain()
+    oracle = InfonetOracleAdapter(lambda: chain)
+    result = oracle.resolve_market(market_id)
+    return {
+        "ok": True,
+        "preview": {
+            "outcome": result.outcome,
+            "reason": result.reason,
+            "is_provisional": result.is_provisional,
+            "burned_amount": result.burned_amount,
+            "stake_returns": [
+                {"node_id": k[0], "rep_type": k[1], "amount": v}
+                for k, v in result.stake_returns.items()
+            ],
+            "stake_winnings": [
+                {"node_id": k[0], "rep_type": k[1], "amount": v}
+                for k, v in result.stake_winnings.items()
+            ],
+            "bond_returns": [
+                {"node_id": k, "amount": v} for k, v in result.bond_returns.items()
+            ],
+            "bond_forfeits": [
+                {"node_id": k, "amount": v} for k, v in result.bond_forfeits.items()
+            ],
+            "first_submitter_bonuses": [
+                {"node_id": k, "amount": v}
+                for k, v in result.first_submitter_bonuses.items()
+            ],
+        },
+    }
+
+
+# ─── Gate shutdown lifecycle ────────────────────────────────────────────
+
+@router.get("/gates/{gate_id}")
+def get_gate_state(gate_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    gates = InfonetGateAdapter(lambda: chain)
+    meta = gates.gate_meta(gate_id)
+    if meta is None:
+        return {"ok": False, "reason": "gate_not_found"}
+    suspension = gates.suspension_state(gate_id, now=now)
+    shutdown = gates.shutdown_state(gate_id, now=now)
+    locked = gates.locked_state(gate_id)
+    members = sorted(gates.member_set(gate_id))
+    return {
+        "ok": True,
+        "gate_id": gate_id,
+        "meta": {
+            "creator_node_id": meta.creator_node_id,
+            "display_name": meta.display_name,
+            "entry_sacrifice": meta.entry_sacrifice,
+            "min_overall_rep": meta.min_overall_rep,
+            "min_gate_rep": dict(meta.min_gate_rep),
+            "created_at": meta.created_at,
+        },
+        "members": members,
+        "ratified": gates.is_ratified(gate_id),
+        "cumulative_member_oracle_rep": gates.cumulative_member_oracle_rep(gate_id),
+        "locked": {
+            "is_locked": locked.locked,
+            "locked_at": locked.locked_at,
+            "locked_by": list(locked.locked_by),
+        },
+        "suspension": {
+            "status": suspension.status,
+            "suspended_at": suspension.suspended_at,
+            "suspended_until": suspension.suspended_until,
+            "last_shutdown_petition_at": suspension.last_shutdown_petition_at,
+        },
+        "shutdown": {
+            "has_pending": shutdown.has_pending,
+            "pending_petition_id": shutdown.pending_petition_id,
+            "pending_status": shutdown.pending_status,
+            "execution_at": shutdown.execution_at,
+            "executed": shutdown.executed,
+        },
+        "now": now,
+    }
+
+
+# ─── Reputation views ───────────────────────────────────────────────────
+
+@router.get("/nodes/{node_id}/reputation")
+def get_node_reputation(node_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    rep = InfonetReputationAdapter(lambda: chain)
+    breakdown = rep.oracle_rep_breakdown(node_id)
+    return {
+        "ok": True,
+        "node_id": node_id,
+        "oracle_rep": rep.oracle_rep(node_id),
+        "oracle_rep_active": rep.oracle_rep_active(node_id),
+        "oracle_rep_lifetime": rep.oracle_rep_lifetime(node_id),
+        "common_rep": rep.common_rep(node_id),
+        "decay_factor": rep.decay_factor(node_id),
+        "last_successful_prediction_ts": rep.last_successful_prediction_ts(node_id),
+        "breakdown": {
+            "free_prediction_mints": breakdown.free_prediction_mints,
+            "staked_prediction_returns": breakdown.staked_prediction_returns,
+            "staked_prediction_losses": breakdown.staked_prediction_losses,
+            "total": breakdown.total,
+        },
+    }
+
+
+# ─── Bootstrap ──────────────────────────────────────────────────────────
+
+@router.get("/bootstrap/markets/{market_id}")
+def get_bootstrap_market_state(market_id: str = Path(...)) -> dict[str, Any]:
+    """Bootstrap-mode-specific market view: who has voted, who is
+    eligible, current tally."""
+    from services.infonet.bootstrap import (
+        deduplicate_votes,
+        validate_bootstrap_eligibility,
+    )
+
+    chain = _live_chain()
+    canonical = deduplicate_votes(market_id, chain)
+    votes_summary: list[dict[str, Any]] = []
+    yes = 0
+    no = 0
+    for v in canonical:
+        node_id = v.get("node_id") or ""
+        side = (v.get("payload") or {}).get("side")
+        decision = validate_bootstrap_eligibility(node_id, market_id, chain)
+        votes_summary.append({
+            "node_id": node_id,
+            "side": side,
+            "eligible": decision.eligible,
+            "ineligible_reason": decision.reason if not decision.eligible else None,
+        })
+        if decision.eligible:
+            if side == "yes":
+                yes += 1
+            elif side == "no":
+                no += 1
+    total = yes + no
+    return {
+        "ok": True,
+        "market_id": market_id,
+        "votes": votes_summary,
+        "tally": {
+            "yes": yes,
+            "no": no,
+            "total_eligible": total,
+            "min_market_participants": int(CONFIG["min_market_participants"]),
+            "supermajority_threshold": float(CONFIG["bootstrap_resolution_supermajority"]),
+        },
+    }
+
+
+# ─── Signed write: append an Infonet economy event ──────────────────────
+
+@router.post("/append")
+def append_event(body: dict[str, Any] = Body(...)) -> dict[str, Any]:
+    """Append a signed Infonet economy event to the chain.
+
+    Body shape (all required for production):
+
+        {
+            "event_type": str,       # one of INFONET_ECONOMY_EVENT_TYPES
+            "node_id":    str,       # signer
+            "payload":    dict,      # event-specific fields
+            "signature":  str,       # hex
+            "sequence":   int,       # node-monotonic
+            "public_key": str,       # base64
+            "public_key_algo": str,  # "ed25519" or "ecdsa"
+            "protocol_version": str  # optional, defaults to current
+        }
+
+    The cutover-registered validators run automatically via
+    ``mesh_hashchain.Infonet.append`` — payload validation, signature
+    verification, replay protection, sequence ordering, public-key
+    binding, revocation status. No additional security wrapper is
+    needed because ``Infonet.append`` IS the secure entry point.
+
+    Returns the appended event dict on success, or
+    ``{"ok": False, "reason": "..."}`` on validation / signing failure.
+    """
+    if not isinstance(body, dict):
+        return {"ok": False, "reason": "body_must_be_object"}
+
+    event_type = body.get("event_type")
+    if not isinstance(event_type, str) or event_type not in INFONET_ECONOMY_EVENT_TYPES:
+        return {
+            "ok": False,
+            "reason": f"event_type must be one of INFONET_ECONOMY_EVENT_TYPES "
+                      f"(got {event_type!r})",
+        }
+
+    node_id = body.get("node_id")
+    if not isinstance(node_id, str) or not node_id:
+        return {"ok": False, "reason": "node_id required"}
+
+    payload = body.get("payload", {})
+    if not isinstance(payload, dict):
+        return {"ok": False, "reason": "payload must be an object"}
+
+    sequence = body.get("sequence", 0)
+    try:
+        sequence = int(sequence)
+    except (TypeError, ValueError):
+        return {"ok": False, "reason": "sequence must be an integer"}
+    if sequence <= 0:
+        return {"ok": False, "reason": "sequence must be > 0"}
+
+    signature = str(body.get("signature") or "")
+    public_key = str(body.get("public_key") or "")
+    public_key_algo = str(body.get("public_key_algo") or "")
+    protocol_version = str(body.get("protocol_version") or "")
+
+    if not signature or not public_key or not public_key_algo:
+        return {
+            "ok": False,
+            "reason": "signature, public_key, and public_key_algo are required",
+        }
+
+    try:
+        from services.mesh.mesh_hashchain import infonet
+        event = infonet.append(
+            event_type=event_type,
+            node_id=node_id,
+            payload=payload,
+            signature=signature,
+            sequence=sequence,
+            public_key=public_key,
+            public_key_algo=public_key_algo,
+            protocol_version=protocol_version,
+        )
+    except ValueError as exc:
+        # Infonet.append raises ValueError for any validation failure
+        # — payload / signature / replay / sequence / binding. The
+        # message is user-facing per the non-hostile UX rule.
+        return {"ok": False, "reason": str(exc)}
+    except Exception as exc:
+        logger.exception("infonet append failed")
+        return {"ok": False, "reason": f"server_error: {type(exc).__name__}"}
+
+    return {"ok": True, "event": event}
+
+
+# ─── Function Keys (citizen + operator views) ───────────────────────────
+
+@router.get("/function-keys/operator/{operator_id}/batch-summary")
+def operator_batch_summary(operator_id: str = Path(...)) -> dict[str, Any]:
+    """Sprint 11+ scaffolding: returns the operator's local batch
+    counter for the current period. Production wires this through the
+    operator's local-store implementation (Sprint 11+ scaffolding
+    doesn't persist; counts reset per process)."""
+    return {
+        "ok": True,
+        "operator_id": operator_id,
+        "scaffolding_only": True,
+        "note": "Production operators maintain a persistent BatchedSettlementBatch. "
+                "This endpoint reports the in-memory state of the local batch.",
+    }
+
+
+__all__ = ["router"]

backend/routers/mesh_dm.py

@@ -0,0 +1,565 @@
+import asyncio
+import hashlib
+import hmac
+import logging
+import secrets
+import time
+from typing import Any
+
+from fastapi import APIRouter, Depends, Request
+from fastapi.responses import JSONResponse
+
+from auth import (
+    _is_debug_test_request,
+    _scoped_view_authenticated,
+    _verify_peer_push_hmac,
+    require_admin,
+)
+from limiter import limiter
+from services.config import get_settings
+from services.mesh.mesh_compatibility import (
+    LEGACY_AGENT_ID_LOOKUP_TARGET,
+    legacy_agent_id_lookup_blocked,
+    record_legacy_agent_id_lookup,
+    sunset_target_label,
+)
+from services.mesh.mesh_signed_events import (
+    MeshWriteExemption,
+    SignedWriteKind,
+    get_prepared_signed_write,
+    mesh_write_exempt,
+    requires_signed_write,
+)
+
+logger = logging.getLogger(__name__)
+_WARNED_LEGACY_DM_PUBKEY_LOOKUPS: set[str] = set()
+
+router = APIRouter()
+
+
+# ---------------------------------------------------------------------------
+# Local helpers
+# ---------------------------------------------------------------------------
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _warn_legacy_dm_pubkey_lookup(agent_id: str) -> None:
+    peer_id = str(agent_id or "").strip().lower()
+    if not peer_id or peer_id in _WARNED_LEGACY_DM_PUBKEY_LOOKUPS:
+        return
+    _WARNED_LEGACY_DM_PUBKEY_LOOKUPS.add(peer_id)
+    logger.warning(
+        "mesh legacy DH pubkey lookup used for %s via direct agent_id; prefer invite-scoped lookup handles before removal in %s",
+        peer_id,
+        sunset_target_label(LEGACY_AGENT_ID_LOOKUP_TARGET),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Transition delegates: forward to main.py so test monkeypatches still work.
+# These will move to a shared module once main.py routes are removed.
+# ---------------------------------------------------------------------------
+def _main_delegate(name):
+    def _wrapper(*a, **kw):
+        import main as _m
+        return getattr(_m, name)(*a, **kw)
+    _wrapper.__name__ = name
+    return _wrapper
+
+
+_verify_signed_write = _main_delegate("_verify_signed_write")
+_secure_dm_enabled = _main_delegate("_secure_dm_enabled")
+_legacy_dm_get_allowed = _main_delegate("_legacy_dm_get_allowed")
+_rns_private_dm_ready = _main_delegate("_rns_private_dm_ready")
+_anonymous_dm_hidden_transport_enforced = _main_delegate("_anonymous_dm_hidden_transport_enforced")
+_high_privacy_profile_enabled = _main_delegate("_high_privacy_profile_enabled")
+_dm_send_from_signed_request = _main_delegate("_dm_send_from_signed_request")
+_dm_poll_secure_from_signed_request = _main_delegate("_dm_poll_secure_from_signed_request")
+_dm_count_secure_from_signed_request = _main_delegate("_dm_count_secure_from_signed_request")
+_validate_private_signed_sequence = _main_delegate("_validate_private_signed_sequence")
+
+
+def _signed_body(request: Request) -> dict[str, Any]:
+    prepared = get_prepared_signed_write(request)
+    if prepared is None:
+        return {}
+    return dict(prepared.body)
+
+
+async def _maybe_apply_dm_relay_jitter() -> None:
+    if not _high_privacy_profile_enabled():
+        return
+    await asyncio.sleep((50 + secrets.randbelow(451)) / 1000.0)
+
+
+_REQUEST_V2_REDUCED_VERSION = "request-v2-reduced-v3"
+_REQUEST_V2_RECOVERY_STATES = {"pending", "verified", "failed"}
+
+
+def _is_canonical_reduced_request_message(message: dict[str, Any]) -> bool:
+    item = dict(message or {})
+    return (
+        str(item.get("delivery_class", "") or "").strip().lower() == "request"
+        and str(item.get("request_contract_version", "") or "").strip()
+        == _REQUEST_V2_REDUCED_VERSION
+        and item.get("sender_recovery_required") is True
+    )
+
+
+def _annotate_request_recovery_message(message: dict[str, Any]) -> dict[str, Any]:
+    item = dict(message or {})
+    delivery_class = str(item.get("delivery_class", "") or "").strip().lower()
+    sender_id = str(item.get("sender_id", "") or "").strip()
+    sender_seal = str(item.get("sender_seal", "") or "").strip()
+    sender_is_blinded = sender_id.startswith("sealed:") or sender_id.startswith("sender_token:")
+    if delivery_class != "request" or not sender_is_blinded or not sender_seal.startswith("v3:"):
+        return item
+    if not str(item.get("request_contract_version", "") or "").strip():
+        item["request_contract_version"] = _REQUEST_V2_REDUCED_VERSION
+    item["sender_recovery_required"] = True
+    state = str(item.get("sender_recovery_state", "") or "").strip().lower()
+    if state not in _REQUEST_V2_RECOVERY_STATES:
+        state = "pending"
+    item["sender_recovery_state"] = state
+    return item
+
+
+def _annotate_request_recovery_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    return [_annotate_request_recovery_message(message) for message in (messages or [])]
+
+
+def _request_duplicate_authority_rank(message: dict[str, Any]) -> int:
+    item = dict(message or {})
+    if str(item.get("delivery_class", "") or "").strip().lower() != "request":
+        return 0
+    if _is_canonical_reduced_request_message(item):
+        return 3
+    sender_id = str(item.get("sender_id", "") or "").strip()
+    if sender_id.startswith("sealed:") or sender_id.startswith("sender_token:"):
+        return 1
+    if sender_id:
+        return 2
+    return 0
+
+
+def _request_duplicate_recovery_rank(message: dict[str, Any]) -> int:
+    if not _is_canonical_reduced_request_message(message):
+        return 0
+    state = str(dict(message or {}).get("sender_recovery_state", "") or "").strip().lower()
+    if state == "verified":
+        return 2
+    if state == "pending":
+        return 1
+    return 0
+
+
+def _poll_duplicate_source_rank(source: str) -> int:
+    normalized = str(source or "").strip().lower()
+    if normalized == "relay":
+        return 2
+    if normalized == "reticulum":
+        return 1
+    return 0
+
+
+def _should_replace_dm_poll_duplicate(
+    existing: dict[str, Any],
+    existing_source: str,
+    candidate: dict[str, Any],
+    candidate_source: str,
+) -> bool:
+    candidate_authority = _request_duplicate_authority_rank(candidate)
+    existing_authority = _request_duplicate_authority_rank(existing)
+    if candidate_authority != existing_authority:
+        return candidate_authority > existing_authority
+
+    candidate_recovery = _request_duplicate_recovery_rank(candidate)
+    existing_recovery = _request_duplicate_recovery_rank(existing)
+    if candidate_recovery != existing_recovery:
+        return candidate_recovery > existing_recovery
+
+    candidate_source_rank = _poll_duplicate_source_rank(candidate_source)
+    existing_source_rank = _poll_duplicate_source_rank(existing_source)
+    if candidate_source_rank != existing_source_rank:
+        return candidate_source_rank > existing_source_rank
+
+    try:
+        candidate_ts = float(candidate.get("timestamp", 0) or 0)
+    except Exception:
+        candidate_ts = 0.0
+    try:
+        existing_ts = float(existing.get("timestamp", 0) or 0)
+    except Exception:
+        existing_ts = 0.0
+    return candidate_ts > existing_ts
+
+
+def _merge_dm_poll_messages(
+    relay_messages: list[dict[str, Any]],
+    direct_messages: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    merged: list[dict[str, Any]] = []
+    index_by_msg_id: dict[str, tuple[int, str]] = {}
+
+    def add_messages(items: list[dict[str, Any]], source: str) -> None:
+        for original in items or []:
+            item = dict(original or {})
+            msg_id = str(item.get("msg_id", "") or "").strip()
+            if not msg_id:
+                merged.append(item)
+                continue
+            existing = index_by_msg_id.get(msg_id)
+            if existing is None:
+                index_by_msg_id[msg_id] = (len(merged), source)
+                merged.append(item)
+                continue
+            index, existing_source = existing
+            if _should_replace_dm_poll_duplicate(merged[index], existing_source, item, source):
+                merged[index] = item
+                index_by_msg_id[msg_id] = (index, source)
+
+    add_messages(relay_messages, "relay")
+    add_messages(direct_messages, "reticulum")
+    return sorted(merged, key=lambda item: float(item.get("timestamp", 0) or 0))
+
+
+# ---------------------------------------------------------------------------
+# Route handlers
+# ---------------------------------------------------------------------------
+
+@router.post("/api/mesh/dm/register")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_REGISTER)
+async def dm_register_key(request: Request):
+    """Register a DH public key for encrypted DM key exchange."""
+    body = _signed_body(request)
+    agent_id = body.get("agent_id", "").strip()
+    dh_pub_key = body.get("dh_pub_key", "").strip()
+    dh_algo = body.get("dh_algo", "").strip()
+    timestamp = _safe_int(body.get("timestamp", 0) or 0)
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not agent_id or not dh_pub_key or not dh_algo or not timestamp:
+        return {"ok": False, "detail": "Missing agent_id, dh_pub_key, dh_algo, or timestamp"}
+    if dh_algo.upper() not in ("X25519", "ECDH_P256", "ECDH"):
+        return {"ok": False, "detail": "Unsupported dh_algo"}
+    now_ts = int(time.time())
+    if abs(timestamp - now_ts) > 7 * 86400:
+        return {"ok": False, "detail": "DH key timestamp is too far from current time"}
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+
+        reputation_ledger.register_node(agent_id, public_key, public_key_algo)
+    except Exception:
+        pass
+
+    accepted, detail, metadata = dm_relay.register_dh_key(
+        agent_id,
+        dh_pub_key,
+        dh_algo,
+        timestamp,
+        signature,
+        public_key,
+        public_key_algo,
+        protocol_version,
+        sequence,
+    )
+    if not accepted:
+        return {"ok": False, "detail": detail}
+
+    return {"ok": True, **(metadata or {})}
+
+
+@router.get("/api/mesh/dm/pubkey")
+@limiter.limit("30/minute")
+async def dm_get_pubkey(request: Request, agent_id: str = "", lookup_token: str = ""):
+    import main as _m
+
+    return await _m.dm_get_pubkey(request, agent_id=agent_id, lookup_token=lookup_token)
+
+
+@router.get("/api/mesh/dm/prekey-bundle")
+@limiter.limit("30/minute")
+async def dm_get_prekey_bundle(request: Request, agent_id: str = "", lookup_token: str = ""):
+    import main as _m
+
+    return await _m.dm_get_prekey_bundle(request, agent_id=agent_id, lookup_token=lookup_token)
+
+
+@router.post("/api/mesh/dm/prekey-peer-lookup")
+@limiter.limit("60/minute")
+@mesh_write_exempt(MeshWriteExemption.PEER_GOSSIP)
+async def dm_prekey_peer_lookup(request: Request):
+    """Peer-authenticated invite lookup handle resolution.
+
+    This endpoint exists for private/bootstrap peers to import signed invites
+    without exposing a stable agent_id on the ordinary lookup surface. It only
+    accepts HMAC-authenticated peer calls and only resolves lookup_token.
+    """
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 4096:
+                return JSONResponse(
+                    status_code=413,
+                    content={"ok": False, "detail": "Request body too large"},
+                )
+        except (TypeError, ValueError):
+            pass
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return JSONResponse(
+            status_code=403,
+            content={"ok": False, "detail": "Invalid or missing peer HMAC"},
+        )
+    try:
+        import json
+
+        body = json.loads(body_bytes or b"{}")
+    except Exception:
+        return {"ok": False, "detail": "invalid json"}
+    lookup_token = str(dict(body or {}).get("lookup_token", "") or "").strip()
+    if not lookup_token:
+        return {"ok": False, "detail": "lookup_token required"}
+    from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle
+
+    result = fetch_dm_prekey_bundle(
+        agent_id="",
+        lookup_token=lookup_token,
+        allow_peer_lookup=False,
+    )
+    if not result.get("ok"):
+        return {"ok": False, "detail": str(result.get("detail", "") or "Prekey bundle not found")}
+    safe = dict(result)
+    safe.pop("resolved_agent_id", None)
+    safe["lookup_mode"] = "invite_lookup_handle"
+    return safe
+
+
+@router.post("/api/mesh/dm/send")
+@limiter.limit("20/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_SEND)
+async def dm_send(request: Request):
+    return await _dm_send_from_signed_request(request)
+
+
+@router.post("/api/mesh/dm/poll")
+@limiter.limit("30/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_POLL)
+async def dm_poll_secure(request: Request):
+    return await _dm_poll_secure_from_signed_request(request)
+
+
+@router.get("/api/mesh/dm/poll")
+@limiter.limit("30/minute")
+async def dm_poll(
+    request: Request,
+    agent_id: str = "",
+    agent_token: str = "",
+    agent_token_prev: str = "",
+    agent_tokens: str = "",
+):
+    import main as _m
+
+    return await _m.dm_poll(
+        request,
+        agent_id=agent_id,
+        agent_token=agent_token,
+        agent_token_prev=agent_token_prev,
+        agent_tokens=agent_tokens,
+    )
+
+
+@router.post("/api/mesh/dm/count")
+@limiter.limit("60/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_COUNT)
+async def dm_count_secure(request: Request):
+    return await _dm_count_secure_from_signed_request(request)
+
+
+@router.get("/api/mesh/dm/count")
+@limiter.limit("60/minute")
+async def dm_count(
+    request: Request,
+    agent_id: str = "",
+    agent_token: str = "",
+    agent_token_prev: str = "",
+    agent_tokens: str = "",
+):
+    import main as _m
+
+    return await _m.dm_count(
+        request,
+        agent_id=agent_id,
+        agent_token=agent_token,
+        agent_token_prev=agent_token_prev,
+        agent_tokens=agent_tokens,
+    )
+
+
+@router.post("/api/mesh/dm/block")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_BLOCK)
+async def dm_block(request: Request):
+    """Block or unblock a sender from DMing you."""
+    body = _signed_body(request)
+    agent_id = body.get("agent_id", "").strip()
+    blocked_id = body.get("blocked_id", "").strip()
+    action = body.get("action", "block").strip().lower()
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not agent_id or not blocked_id:
+        return {"ok": False, "detail": "Missing agent_id or blocked_id"}
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    try:
+        from services.mesh.mesh_hashchain import infonet
+
+        ok_seq, seq_reason = _validate_private_signed_sequence(
+            infonet,
+            agent_id,
+            sequence,
+            domain=f"dm_block:{action}",
+        )
+        if not ok_seq:
+            return {"ok": False, "detail": seq_reason}
+    except Exception:
+        pass
+
+    if action == "unblock":
+        dm_relay.unblock(agent_id, blocked_id)
+    else:
+        dm_relay.block(agent_id, blocked_id)
+    return {"ok": True, "action": action, "blocked_id": blocked_id}
+
+
+@router.post("/api/mesh/dm/witness")
+@limiter.limit("20/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_WITNESS)
+async def dm_key_witness(request: Request):
+    """Record a lightweight witness for a DM key (dual-path spot-check)."""
+    body = _signed_body(request)
+    witness_id = body.get("witness_id", "").strip()
+    target_id = body.get("target_id", "").strip()
+    dh_pub_key = body.get("dh_pub_key", "").strip()
+    timestamp = _safe_int(body.get("timestamp", 0) or 0)
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not witness_id or not target_id or not dh_pub_key or not timestamp:
+        return {"ok": False, "detail": "Missing witness_id, target_id, dh_pub_key, or timestamp"}
+    now_ts = int(time.time())
+    if abs(timestamp - now_ts) > 7 * 86400:
+        return {"ok": False, "detail": "Witness timestamp is too far from current time"}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+
+        reputation_ledger.register_node(witness_id, public_key, public_key_algo)
+    except Exception:
+        pass
+    try:
+        from services.mesh.mesh_hashchain import infonet
+
+        ok_seq, seq_reason = _validate_private_signed_sequence(
+            infonet,
+            witness_id,
+            sequence,
+            domain="dm_witness",
+        )
+        if not ok_seq:
+            return {"ok": False, "detail": seq_reason}
+    except Exception:
+        pass
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    ok, reason = dm_relay.record_witness(witness_id, target_id, dh_pub_key, timestamp)
+    return {"ok": ok, "detail": reason}
+
+
+@router.get("/api/mesh/dm/witness")
+@limiter.limit("60/minute")
+async def dm_key_witness_get(request: Request, target_id: str = "", dh_pub_key: str = ""):
+    """Get witness counts for a target's DH key."""
+    if not target_id:
+        return {"ok": False, "detail": "Missing target_id"}
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    witnesses = dm_relay.get_witnesses(target_id, dh_pub_key if dh_pub_key else None, limit=5)
+    response = {
+        "ok": True,
+        "count": len(witnesses),
+    }
+    if _scoped_view_authenticated(request, "mesh.audit"):
+        response["target_id"] = target_id
+        response["dh_pub_key"] = dh_pub_key or ""
+        response["witnesses"] = witnesses
+    return response
+
+
+@router.post("/api/mesh/trust/vouch")
+@limiter.limit("20/minute")
+@requires_signed_write(kind=SignedWriteKind.TRUST_VOUCH)
+async def trust_vouch(request: Request):
+    """Record a trust vouch for a node (web-of-trust signal)."""
+    body = _signed_body(request)
+    voucher_id = body.get("voucher_id", "").strip()
+    target_id = body.get("target_id", "").strip()
+    note = body.get("note", "").strip()
+    timestamp = _safe_int(body.get("timestamp", 0) or 0)
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not voucher_id or not target_id or not timestamp:
+        return {"ok": False, "detail": "Missing voucher_id, target_id, or timestamp"}
+    now_ts = int(time.time())
+    if abs(timestamp - now_ts) > 7 * 86400:
+        return {"ok": False, "detail": "Vouch timestamp is too far from current time"}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+        from services.mesh.mesh_hashchain import infonet
+
+        reputation_ledger.register_node(voucher_id, public_key, public_key_algo)
+        ok_seq, seq_reason = _validate_private_signed_sequence(
+            infonet,
+            voucher_id,
+            sequence,
+            domain="trust_vouch",
+        )
+        if not ok_seq:
+            return {"ok": False, "detail": seq_reason}
+        ok, reason = reputation_ledger.add_vouch(voucher_id, target_id, note, timestamp)
+        return {"ok": ok, "detail": reason}
+    except Exception:
+        return {"ok": False, "detail": "Failed to record vouch"}
+
+
+@router.get("/api/mesh/trust/vouches", dependencies=[Depends(require_admin)])
+@limiter.limit("60/minute")
+async def trust_vouches(request: Request, node_id: str = "", limit: int = 20):
+    """Fetch latest vouches for a node."""
+    if not node_id:
+        return {"ok": False, "detail": "Missing node_id"}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+
+        vouches = reputation_ledger.get_vouches(node_id, limit=limit)
+        return {"ok": True, "node_id": node_id, "vouches": vouches, "count": len(vouches)}
+    except Exception:
+        return {"ok": False, "detail": "Failed to fetch vouches"}

backend/routers/mesh_operator.py

@@ -0,0 +1,145 @@
+import time
+import logging
+from fastapi import APIRouter, Request, Response, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def list_peers(request: Request, bucket: str = Query(None)):
+    """List all peers (or filter by bucket: sync, push, bootstrap)."""
+    from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerStore
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception as exc:
+        return {"ok": False, "detail": f"Failed to load peer store: {exc}"}
+    if bucket:
+        records = store.records_for_bucket(bucket)
+    else:
+        records = store.records()
+    return {"ok": True, "count": len(records), "peers": [r.to_dict() for r in records]}
+
+
+@router.post("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def add_peer(request: Request):
+    """Add a peer to the store. Body: {peer_url, transport?, label?, role?, buckets?[]}."""
+    from services.mesh.mesh_crypto import normalize_peer_url
+    from services.mesh.mesh_peer_store import (
+        DEFAULT_PEER_STORE_PATH, PeerStore, PeerStoreError,
+        make_push_peer_record, make_sync_peer_record,
+    )
+    from services.mesh.mesh_router import peer_transport_kind
+    body = await request.json()
+    peer_url_raw = str(body.get("peer_url", "") or "").strip()
+    if not peer_url_raw:
+        return {"ok": False, "detail": "peer_url is required"}
+    peer_url = normalize_peer_url(peer_url_raw)
+    if not peer_url:
+        return {"ok": False, "detail": "Invalid peer_url"}
+    transport = str(body.get("transport", "") or "").strip().lower()
+    if not transport:
+        transport = peer_transport_kind(peer_url)
+    if not transport:
+        return {"ok": False, "detail": "Cannot determine transport for peer_url — provide transport explicitly"}
+    label = str(body.get("label", "") or "").strip()
+    role = str(body.get("role", "") or "").strip().lower() or "relay"
+    buckets = body.get("buckets", ["sync", "push"])
+    if isinstance(buckets, str):
+        buckets = [buckets]
+    if not isinstance(buckets, list):
+        buckets = ["sync", "push"]
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    added: list = []
+    try:
+        for b in buckets:
+            b = str(b).strip().lower()
+            if b == "sync":
+                store.upsert(make_sync_peer_record(peer_url=peer_url, transport=transport, role=role, label=label))
+                added.append("sync")
+            elif b == "push":
+                store.upsert(make_push_peer_record(peer_url=peer_url, transport=transport, role=role, label=label))
+                added.append("push")
+        store.save()
+    except PeerStoreError as exc:
+        return {"ok": False, "detail": str(exc)}
+    return {"ok": True, "peer_url": peer_url, "buckets": added}
+
+
+@router.delete("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def remove_peer(request: Request):
+    """Remove a peer. Body: {peer_url, bucket?}. If bucket omitted, removes from all buckets."""
+    from services.mesh.mesh_crypto import normalize_peer_url
+    from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerStore
+    body = await request.json()
+    peer_url_raw = str(body.get("peer_url", "") or "").strip()
+    if not peer_url_raw:
+        return {"ok": False, "detail": "peer_url is required"}
+    peer_url = normalize_peer_url(peer_url_raw)
+    if not peer_url:
+        return {"ok": False, "detail": "Invalid peer_url"}
+    bucket_filter = str(body.get("bucket", "") or "").strip().lower()
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        return {"ok": False, "detail": "Failed to load peer store"}
+    removed: list = []
+    for b in ["bootstrap", "sync", "push"]:
+        if bucket_filter and b != bucket_filter:
+            continue
+        key = f"{b}:{peer_url}"
+        if key in store._records:
+            del store._records[key]
+            removed.append(b)
+    if not removed:
+        return {"ok": False, "detail": "Peer not found in any bucket"}
+    store.save()
+    return {"ok": True, "peer_url": peer_url, "removed_from": removed}
+
+
+@router.patch("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def toggle_peer(request: Request):
+    """Enable or disable a peer. Body: {peer_url, bucket, enabled: bool}."""
+    from services.mesh.mesh_crypto import normalize_peer_url
+    from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerRecord, PeerStore
+    body = await request.json()
+    peer_url_raw = str(body.get("peer_url", "") or "").strip()
+    bucket = str(body.get("bucket", "") or "").strip().lower()
+    enabled = body.get("enabled")
+    if not peer_url_raw:
+        return {"ok": False, "detail": "peer_url is required"}
+    if not bucket:
+        return {"ok": False, "detail": "bucket is required"}
+    if enabled is None:
+        return {"ok": False, "detail": "enabled (true/false) is required"}
+    peer_url = normalize_peer_url(peer_url_raw)
+    if not peer_url:
+        return {"ok": False, "detail": "Invalid peer_url"}
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        return {"ok": False, "detail": "Failed to load peer store"}
+    key = f"{bucket}:{peer_url}"
+    record = store._records.get(key)
+    if not record:
+        return {"ok": False, "detail": f"Peer not found in {bucket} bucket"}
+    updated = PeerRecord(**{**record.to_dict(), "enabled": bool(enabled), "updated_at": int(time.time())})
+    store._records[key] = updated
+    store.save()
+    return {"ok": True, "peer_url": peer_url, "bucket": bucket, "enabled": bool(enabled)}

backend/routers/mesh_oracle.py

@@ -0,0 +1,337 @@
+import math
+from typing import Any
+from fastapi import APIRouter, Request, Response, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator, _scoped_view_authenticated
+from services.data_fetcher import get_latest_data
+from services.mesh.mesh_protocol import normalize_payload
+from services.mesh.mesh_signed_events import (
+    MeshWriteExemption,
+    SignedWriteKind,
+    get_prepared_signed_write,
+    mesh_write_exempt,
+    requires_signed_write,
+)
+
+router = APIRouter()
+
+
+def _signed_body(request: Request) -> dict[str, Any]:
+    prepared = get_prepared_signed_write(request)
+    if prepared is None:
+        return {}
+    return dict(prepared.body)
+
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _safe_float(val, default=0.0):
+    try:
+        parsed = float(val)
+        if not math.isfinite(parsed):
+            return default
+        return parsed
+    except (TypeError, ValueError):
+        return default
+
+
+def _redact_public_oracle_profile(payload: dict, authenticated: bool) -> dict:
+    redacted = dict(payload)
+    if authenticated:
+        return redacted
+    redacted["active_stakes"] = []
+    redacted["prediction_history"] = []
+    return redacted
+
+
+def _redact_public_oracle_predictions(predictions: list, authenticated: bool) -> dict:
+    if authenticated:
+        return {"predictions": list(predictions)}
+    return {"predictions": [], "count": len(predictions)}
+
+
+def _redact_public_oracle_stakes(payload: dict, authenticated: bool) -> dict:
+    redacted = dict(payload)
+    if authenticated:
+        return redacted
+    redacted["truth_stakers"] = []
+    redacted["false_stakers"] = []
+    return redacted
+
+
+@router.post("/api/mesh/oracle/predict")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.ORACLE_PREDICT)
+async def oracle_predict(request: Request):
+    """Place a prediction on a market outcome."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    body = _signed_body(request)
+    node_id = body.get("node_id", "")
+    market_title = body.get("market_title", "")
+    side = body.get("side", "")
+    stake_amount = _safe_float(body.get("stake_amount", 0))
+    public_key = body.get("public_key", "")
+    public_key_algo = body.get("public_key_algo", "")
+    signature = body.get("signature", "")
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "")
+    if not node_id or not market_title or not side:
+        return {"ok": False, "detail": "Missing node_id, market_title, or side"}
+    prediction_payload = {"market_title": market_title, "side": side, "stake_amount": stake_amount}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+        reputation_ledger.register_node(node_id, public_key, public_key_algo)
+    except Exception:
+        pass
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    matched = None
+    for m in markets:
+        if m.get("title", "").lower() == market_title.lower():
+            matched = m
+            break
+    if not matched:
+        for m in markets:
+            if market_title.lower() in m.get("title", "").lower():
+                matched = m
+                break
+    if not matched:
+        return {"ok": False, "detail": f"Market '{market_title}' not found in active markets."}
+    probability = 50.0
+    side_lower = side.lower()
+    outcomes = matched.get("outcomes", [])
+    if outcomes:
+        for o in outcomes:
+            if o.get("name", "").lower() == side_lower:
+                probability = float(o.get("pct", 50))
+                break
+    else:
+        consensus = matched.get("consensus_pct")
+        if consensus is None:
+            consensus = matched.get("polymarket_pct") or matched.get("kalshi_pct") or 50
+        probability = float(consensus)
+        if side_lower == "no":
+            probability = 100.0 - probability
+    if stake_amount > 0:
+        ok, detail = oracle_ledger.place_market_stake(node_id, matched["title"], side, stake_amount, probability)
+        mode = "staked"
+    else:
+        ok, detail = oracle_ledger.place_prediction(node_id, matched["title"], side, probability)
+        mode = "free"
+    if ok:
+        try:
+            from services.mesh.mesh_hashchain import infonet
+            normalized_payload = normalize_payload("prediction", prediction_payload)
+            infonet.append(event_type="prediction", node_id=node_id, payload=normalized_payload,
+                signature=signature, sequence=sequence, public_key=public_key,
+                public_key_algo=public_key_algo, protocol_version=protocol_version)
+        except Exception:
+            pass
+    return {"ok": ok, "detail": detail, "probability": probability, "mode": mode}
+
+
+@router.get("/api/mesh/oracle/markets")
+@limiter.limit("30/minute")
+async def oracle_markets(request: Request):
+    """List active prediction markets."""
+    from collections import defaultdict
+    from services.mesh.mesh_oracle import oracle_ledger
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    all_consensus = oracle_ledger.get_all_market_consensus()
+    by_category = defaultdict(list)
+    for m in markets:
+        by_category[m.get("category", "NEWS")].append(m)
+    _fields = ("title", "consensus_pct", "polymarket_pct", "kalshi_pct", "volume", "volume_24h",
+               "end_date", "description", "category", "sources", "slug", "kalshi_ticker", "outcomes")
+    categories = {}
+    cat_totals = {}
+    for cat in ["POLITICS", "CONFLICT", "NEWS", "FINANCE", "CRYPTO"]:
+        all_cat = sorted(by_category.get(cat, []), key=lambda x: x.get("volume", 0) or 0, reverse=True)
+        cat_totals[cat] = len(all_cat)
+        cat_list = []
+        for m in all_cat[:10]:
+            entry = {k: m.get(k) for k in _fields}
+            entry["consensus"] = all_consensus.get(m.get("title", ""), {})
+            cat_list.append(entry)
+        categories[cat] = cat_list
+    return {"categories": categories, "total_count": len(markets), "cat_totals": cat_totals}
+
+
+@router.get("/api/mesh/oracle/search")
+@limiter.limit("20/minute")
+async def oracle_search(request: Request, q: str = "", limit: int = 50):
+    """Search prediction markets across Polymarket + Kalshi APIs."""
+    if not q or len(q) < 2:
+        return {"results": [], "query": q, "count": 0}
+    from services.fetchers.prediction_markets import search_polymarket_direct, search_kalshi_direct
+    import concurrent.futures
+    # Search both APIs in parallel for speed
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
+        poly_fut = pool.submit(search_polymarket_direct, q, limit)
+        kalshi_fut = pool.submit(search_kalshi_direct, q, limit)
+        poly_results = poly_fut.result(timeout=20)
+        kalshi_results = kalshi_fut.result(timeout=20)
+    # Also check cached/merged markets
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    q_lower = q.lower()
+    cached_matches = [m for m in markets if q_lower in m.get("title", "").lower()]
+    seen_titles = set()
+    combined = []
+    # Cached first (already merged Poly+Kalshi with consensus)
+    for m in cached_matches:
+        seen_titles.add(m["title"].lower())
+        combined.append(m)
+    # Then Polymarket direct hits
+    for m in poly_results:
+        if m["title"].lower() not in seen_titles:
+            seen_titles.add(m["title"].lower())
+            combined.append(m)
+    # Then Kalshi direct hits
+    for m in kalshi_results:
+        if m["title"].lower() not in seen_titles:
+            seen_titles.add(m["title"].lower())
+            combined.append(m)
+    combined.sort(key=lambda x: x.get("volume", 0) or 0, reverse=True)
+    _fields = ("title", "consensus_pct", "polymarket_pct", "kalshi_pct", "volume", "volume_24h",
+               "end_date", "description", "category", "sources", "slug", "kalshi_ticker", "outcomes")
+    results = [{k: m.get(k) for k in _fields} for m in combined[:limit]]
+    return {"results": results, "query": q, "count": len(results)}
+
+
+@router.get("/api/mesh/oracle/markets/more")
+@limiter.limit("30/minute")
+async def oracle_markets_more(request: Request, category: str = "NEWS", offset: int = 0, limit: int = 10):
+    """Load more markets for a specific category (paginated)."""
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    cat_markets = sorted([m for m in markets if m.get("category") == category],
+        key=lambda x: x.get("volume", 0) or 0, reverse=True)
+    page = cat_markets[offset : offset + limit]
+    _fields = ("title", "consensus_pct", "polymarket_pct", "kalshi_pct", "volume", "volume_24h",
+               "end_date", "description", "category", "sources", "slug", "kalshi_ticker", "outcomes")
+    results = [{k: m.get(k) for k in _fields} for m in page]
+    return {"markets": results, "category": category, "offset": offset,
+            "has_more": offset + limit < len(cat_markets), "total": len(cat_markets)}
+
+
+@router.post("/api/mesh/oracle/resolve")
+@limiter.limit("5/minute")
+@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
+async def oracle_resolve(request: Request):
+    """Resolve a prediction market."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    body = await request.json()
+    market_title = body.get("market_title", "")
+    outcome = body.get("outcome", "")
+    if not market_title or not outcome:
+        return {"ok": False, "detail": "Need market_title and outcome"}
+    winners, losers = oracle_ledger.resolve_market(market_title, outcome)
+    stake_result = oracle_ledger.resolve_market_stakes(market_title, outcome)
+    return {"ok": True,
+            "detail": f"Resolved: {winners} free winners, {losers} free losers, "
+                      f"{stake_result.get('winners', 0)} stake winners, {stake_result.get('losers', 0)} stake losers",
+            "free": {"winners": winners, "losers": losers}, "stakes": stake_result}
+
+
+@router.get("/api/mesh/oracle/consensus")
+@limiter.limit("30/minute")
+async def oracle_consensus(request: Request, market_title: str = ""):
+    """Get network consensus for a market."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    if not market_title:
+        return {"error": "market_title required"}
+    return oracle_ledger.get_market_consensus(market_title)
+
+
+@router.post("/api/mesh/oracle/stake")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.ORACLE_STAKE)
+async def oracle_stake(request: Request):
+    """Stake oracle rep on a post's truthfulness."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    body = _signed_body(request)
+    staker_id = body.get("staker_id", "")
+    message_id = body.get("message_id", "")
+    poster_id = body.get("poster_id", "")
+    side = body.get("side", "").lower()
+    amount = _safe_float(body.get("amount", 0))
+    duration_days = _safe_int(body.get("duration_days", 1), 1)
+    public_key = body.get("public_key", "")
+    public_key_algo = body.get("public_key_algo", "")
+    signature = body.get("signature", "")
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "")
+    if not staker_id or not message_id or not side:
+        return {"ok": False, "detail": "Missing staker_id, message_id, or side"}
+    stake_payload = {"message_id": message_id, "poster_id": poster_id, "side": side,
+                     "amount": amount, "duration_days": duration_days}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+        reputation_ledger.register_node(staker_id, public_key, public_key_algo)
+    except Exception:
+        pass
+    ok, detail = oracle_ledger.place_stake(staker_id, message_id, poster_id, side, amount, duration_days)
+    if ok:
+        try:
+            from services.mesh.mesh_hashchain import infonet
+            normalized_payload = normalize_payload("stake", stake_payload)
+            infonet.append(event_type="stake", node_id=staker_id, payload=normalized_payload,
+                signature=signature, sequence=sequence, public_key=public_key,
+                public_key_algo=public_key_algo, protocol_version=protocol_version)
+        except Exception:
+            pass
+    return {"ok": ok, "detail": detail}
+
+
+@router.get("/api/mesh/oracle/stakes/{message_id}")
+@limiter.limit("30/minute")
+async def oracle_stakes_for_message(request: Request, message_id: str):
+    """Get all oracle stakes on a message."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    return _redact_public_oracle_stakes(
+        oracle_ledger.get_stakes_for_message(message_id),
+        authenticated=_scoped_view_authenticated(request, "mesh.audit"),
+    )
+
+
+@router.get("/api/mesh/oracle/profile")
+@limiter.limit("30/minute")
+async def oracle_profile(request: Request, node_id: str = ""):
+    """Get full oracle profile."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    if not node_id:
+        return {"ok": False, "detail": "Provide ?node_id=xxx"}
+    profile = oracle_ledger.get_oracle_profile(node_id)
+    return _redact_public_oracle_profile(
+        profile, authenticated=_scoped_view_authenticated(request, "mesh.audit"))
+
+
+@router.get("/api/mesh/oracle/predictions")
+@limiter.limit("30/minute")
+async def oracle_predictions(request: Request, node_id: str = ""):
+    """Get a node's active (unresolved) predictions."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    if not node_id:
+        return {"ok": False, "detail": "Provide ?node_id=xxx"}
+    active_predictions = oracle_ledger.get_active_predictions(node_id)
+    return _redact_public_oracle_predictions(
+        active_predictions, authenticated=_scoped_view_authenticated(request, "mesh.audit"))
+
+
+@router.post("/api/mesh/oracle/resolve-stakes")
+@limiter.limit("5/minute")
+@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
+async def oracle_resolve_stakes(request: Request):
+    """Resolve all expired stake contests."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    resolutions = oracle_ledger.resolve_expired_stakes()
+    return {"ok": True, "resolutions": resolutions, "count": len(resolutions)}

backend/routers/mesh_peer_sync.py

@@ -0,0 +1,235 @@
+import json as json_mod
+import logging
+from typing import Any
+from fastapi import APIRouter, Request, Response
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator, _verify_peer_push_hmac
+from services.config import get_settings
+from services.mesh.mesh_crypto import normalize_peer_url
+from services.mesh.mesh_router import peer_transport_kind
+from auth import _peer_hmac_url_from_request
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+_PEER_PUSH_BATCH_SIZE = 50
+
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _hydrate_gate_store_from_chain(events: list) -> int:
+    """Copy any gate_message chain events into the local gate_store for read/decrypt.
+
+    Only events that are resident in the local infonet (accepted or already
+    present) are hydrated.  The canonical infonet-resident event is used —
+    never the raw batch event — so a forged batch entry carrying a valid
+    event_id but attacker-chosen payload cannot pollute gate_store.
+    """
+    import copy
+    from services.mesh.mesh_hashchain import gate_store, infonet
+    count = 0
+    for evt in events:
+        if evt.get("event_type") != "gate_message":
+            continue
+        event_id = str(evt.get("event_id", "") or "").strip()
+        if not event_id or event_id not in infonet.event_index:
+            continue
+        canonical = infonet.events[infonet.event_index[event_id]]
+        payload = canonical.get("payload") or {}
+        gate_id = str(payload.get("gate", "") or "").strip()
+        if not gate_id:
+            continue
+        try:
+            gate_store.append(gate_id, copy.deepcopy(canonical))
+            count += 1
+        except Exception:
+            pass
+    return count
+
+
+@router.post("/api/mesh/infonet/peer-push")
+@limiter.limit("30/minute")
+async def infonet_peer_push(request: Request):
+    """Accept pushed Infonet events from relay peers (HMAC-authenticated)."""
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 524_288:
+                return Response(content='{"ok":false,"detail":"Request body too large (max 512KB)"}',
+                    status_code=413, media_type="application/json")
+        except (ValueError, TypeError):
+            pass
+    from services.mesh.mesh_hashchain import infonet
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json")
+    body = json_mod.loads(body_bytes or b"{}")
+    events = body.get("events", [])
+    if not isinstance(events, list):
+        return {"ok": False, "detail": "events must be a list"}
+    if len(events) > 50:
+        return {"ok": False, "detail": "Too many events in one push (max 50)"}
+    if not events:
+        return {"ok": True, "accepted": 0, "duplicates": 0, "rejected": []}
+    result = infonet.ingest_events(events)
+    _hydrate_gate_store_from_chain(events)
+    return {"ok": True, **result}
+
+
+@router.post("/api/mesh/gate/peer-push")
+@limiter.limit("30/minute")
+async def gate_peer_push(request: Request):
+    """Accept pushed gate events from relay peers (private plane)."""
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 524_288:
+                return Response(content='{"ok":false,"detail":"Request body too large"}',
+                    status_code=413, media_type="application/json")
+        except (ValueError, TypeError):
+            pass
+    from services.mesh.mesh_hashchain import gate_store
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json")
+    body = json_mod.loads(body_bytes or b"{}")
+    events = body.get("events", [])
+    if not isinstance(events, list):
+        return {"ok": False, "detail": "events must be a list"}
+    if len(events) > 50:
+        return {"ok": False, "detail": "Too many events (max 50)"}
+    if not events:
+        return {"ok": True, "accepted": 0, "duplicates": 0}
+    from services.mesh.mesh_hashchain import resolve_gate_wire_ref
+    # Sprint 3 / Rec #4: the gate_ref is HMACed with a key bound to the
+    # receiver's peer URL (the URL the push was delivered to). This is
+    # the same URL _verify_peer_push_hmac validated the X-Peer-HMAC
+    # header against, so we can trust it for ref resolution.
+    hop_peer_url = _peer_hmac_url_from_request(request)
+    grouped_events: dict[str, list] = {}
+    for evt in events:
+        evt_dict = evt if isinstance(evt, dict) else {}
+        payload = evt_dict.get("payload")
+        if not isinstance(payload, dict):
+            payload = {}
+        clean_event = {
+            "event_id": str(evt_dict.get("event_id", "") or ""),
+            "event_type": "gate_message",
+            "timestamp": evt_dict.get("timestamp", 0),
+            "node_id": str(evt_dict.get("node_id", "") or evt_dict.get("sender_id", "") or ""),
+            "sequence": evt_dict.get("sequence", 0),
+            "signature": str(evt_dict.get("signature", "") or ""),
+            "public_key": str(evt_dict.get("public_key", "") or ""),
+            "public_key_algo": str(evt_dict.get("public_key_algo", "") or ""),
+            "protocol_version": str(evt_dict.get("protocol_version", "") or ""),
+            "payload": {
+                "ciphertext": str(payload.get("ciphertext", "") or ""),
+                "format": str(payload.get("format", "") or ""),
+                "nonce": str(payload.get("nonce", "") or ""),
+                "sender_ref": str(payload.get("sender_ref", "") or ""),
+            },
+        }
+        epoch = _safe_int(payload.get("epoch", 0) or 0)
+        if epoch > 0:
+            clean_event["payload"]["epoch"] = epoch
+        envelope_hash_val = str(payload.get("envelope_hash", "") or "").strip()
+        gate_envelope_val = str(payload.get("gate_envelope", "") or "").strip()
+        reply_to_val = str(payload.get("reply_to", "") or "").strip()
+        if envelope_hash_val:
+            clean_event["payload"]["envelope_hash"] = envelope_hash_val
+        if gate_envelope_val:
+            clean_event["payload"]["gate_envelope"] = gate_envelope_val
+        if reply_to_val:
+            clean_event["payload"]["reply_to"] = reply_to_val
+        event_gate_id = str(payload.get("gate", "") or evt_dict.get("gate", "") or "").strip().lower()
+        if not event_gate_id:
+            event_gate_id = resolve_gate_wire_ref(
+                str(payload.get("gate_ref", "") or evt_dict.get("gate_ref", "") or ""),
+                clean_event,
+                peer_url=hop_peer_url,
+            )
+        if not event_gate_id:
+            return {"ok": False, "detail": "gate resolution failed"}
+        final_payload: dict[str, Any] = {
+            "gate": event_gate_id,
+            "ciphertext": clean_event["payload"]["ciphertext"],
+            "format": clean_event["payload"]["format"],
+            "nonce": clean_event["payload"]["nonce"],
+            "sender_ref": clean_event["payload"]["sender_ref"],
+        }
+        if epoch > 0:
+            final_payload["epoch"] = epoch
+        if clean_event["payload"].get("envelope_hash"):
+            final_payload["envelope_hash"] = clean_event["payload"]["envelope_hash"]
+        if clean_event["payload"].get("gate_envelope"):
+            final_payload["gate_envelope"] = clean_event["payload"]["gate_envelope"]
+        if clean_event["payload"].get("reply_to"):
+            final_payload["reply_to"] = clean_event["payload"]["reply_to"]
+        grouped_events.setdefault(event_gate_id, []).append({
+            "event_id": clean_event["event_id"],
+            "event_type": "gate_message",
+            "timestamp": clean_event["timestamp"],
+            "node_id": clean_event["node_id"],
+            "sequence": clean_event["sequence"],
+            "signature": clean_event["signature"],
+            "public_key": clean_event["public_key"],
+            "public_key_algo": clean_event["public_key_algo"],
+            "protocol_version": clean_event["protocol_version"],
+            "payload": final_payload,
+        })
+    accepted = 0
+    duplicates = 0
+    rejected = 0
+    for event_gate_id, items in grouped_events.items():
+        result = gate_store.ingest_peer_events(event_gate_id, items)
+        a = int(result.get("accepted", 0) or 0)
+        accepted += a
+        duplicates += int(result.get("duplicates", 0) or 0)
+        rejected += int(result.get("rejected", 0) or 0)
+    return {"ok": True, "accepted": accepted, "duplicates": duplicates, "rejected": rejected}
+
+
+@router.post("/api/mesh/gate/peer-pull")
+@limiter.limit("30/minute")
+async def gate_peer_pull(request: Request):
+    """Return gate events a peer is missing (HMAC-authenticated pull sync)."""
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 65_536:
+                return Response(content='{"ok":false,"detail":"Request body too large"}',
+                    status_code=413, media_type="application/json")
+        except (ValueError, TypeError):
+            pass
+    from services.mesh.mesh_hashchain import gate_store
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json")
+    body = json_mod.loads(body_bytes or b"{}")
+    gate_id = str(body.get("gate_id", "") or "").strip().lower()
+    after_count = _safe_int(body.get("after_count", 0) or 0)
+    if not gate_id:
+        gate_ids = gate_store.known_gate_ids()
+        gate_counts: dict[str, int] = {}
+        for gid in gate_ids:
+            with gate_store._lock:
+                gate_counts[gid] = len(gate_store._gates.get(gid, []))
+        return {"ok": True, "gates": gate_counts}
+    with gate_store._lock:
+        all_events = list(gate_store._gates.get(gate_id, []))
+    total = len(all_events)
+    if after_count >= total:
+        return {"ok": True, "events": [], "total": total, "gate_id": gate_id}
+    batch = all_events[after_count : after_count + _PEER_PUSH_BATCH_SIZE]
+    return {"ok": True, "events": batch, "total": total, "gate_id": gate_id}

backend/routers/radio.py

@@ -0,0 +1,91 @@
+from fastapi import APIRouter, Request, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+
+router = APIRouter()
+
+
+@router.get("/api/radio/top")
+@limiter.limit("30/minute")
+async def get_top_radios(request: Request):
+    from services.radio_intercept import get_top_broadcastify_feeds
+    return get_top_broadcastify_feeds()
+
+
+@router.get("/api/radio/openmhz/systems")
+@limiter.limit("30/minute")
+async def api_get_openmhz_systems(request: Request):
+    from services.radio_intercept import get_openmhz_systems
+    return get_openmhz_systems()
+
+
+@router.get("/api/radio/openmhz/calls/{sys_name}")
+@limiter.limit("60/minute")
+async def api_get_openmhz_calls(request: Request, sys_name: str):
+    from services.radio_intercept import get_recent_openmhz_calls
+    return get_recent_openmhz_calls(sys_name)
+
+
+@router.get("/api/radio/openmhz/audio")
+@limiter.limit("120/minute")
+async def api_get_openmhz_audio(request: Request, url: str = Query(..., min_length=10)):
+    from services.radio_intercept import openmhz_audio_response
+    return openmhz_audio_response(url)
+
+
+@router.get("/api/radio/nearest")
+@limiter.limit("60/minute")
+async def api_get_nearest_radio(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    from services.radio_intercept import find_nearest_openmhz_system
+    return find_nearest_openmhz_system(lat, lng)
+
+
+@router.get("/api/radio/nearest-list")
+@limiter.limit("60/minute")
+async def api_get_nearest_radios_list(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+    limit: int = Query(5, ge=1, le=20),
+):
+    from services.radio_intercept import find_nearest_openmhz_systems_list
+    return find_nearest_openmhz_systems_list(lat, lng, limit=limit)
+
+
+@router.get("/api/route/{callsign}")
+@limiter.limit("60/minute")
+async def get_flight_route(request: Request, callsign: str, lat: float = 0.0, lng: float = 0.0):
+    from services.network_utils import fetch_with_curl
+    r = fetch_with_curl(
+        "https://api.adsb.lol/api/0/routeset",
+        method="POST",
+        json_data={"planes": [{"callsign": callsign, "lat": lat, "lng": lng}]},
+        timeout=10,
+    )
+    if r and r.status_code == 200:
+        data = r.json()
+        route_list = []
+        if isinstance(data, dict):
+            route_list = data.get("value", [])
+        elif isinstance(data, list):
+            route_list = data
+
+        if route_list and len(route_list) > 0:
+            route = route_list[0]
+            airports = route.get("_airports", [])
+            if len(airports) >= 2:
+                orig = airports[0]
+                dest = airports[-1]
+                return {
+                    "orig_loc": [orig.get("lon", 0), orig.get("lat", 0)],
+                    "dest_loc": [dest.get("lon", 0), dest.get("lat", 0)],
+                    "origin_name": f"{orig.get('iata', '') or orig.get('icao', '')}: {orig.get('name', 'Unknown')}",
+                    "dest_name": f"{dest.get('iata', '') or dest.get('icao', '')}: {dest.get('name', 'Unknown')}",
+                }
+    return {}

backend/routers/sar.py

@@ -0,0 +1,260 @@
+"""SAR (Synthetic Aperture Radar) layer endpoints.
+
+Exposes:
+  - GET  /api/sar/status            — feature gates + signup links for the UI
+  - GET  /api/sar/anomalies         — Mode B pre-processed anomalies
+  - GET  /api/sar/scenes            — Mode A scene catalog
+  - GET  /api/sar/coverage          — per-AOI coverage and next-pass hints
+  - GET  /api/sar/aois              — operator-defined AOIs
+  - POST /api/sar/aois              — create or replace an AOI
+  - DELETE /api/sar/aois/{aoi_id}   — remove an AOI
+  - GET  /api/sar/near              — anomalies within radius_km of (lat, lon)
+
+The /status endpoint is the load-bearing UX: when Mode B is disabled it
+returns the structured help payload from sar_config.products_fetch_status()
+so the frontend can render in-app links to the free signup pages instead of
+making the user hunt around.
+"""
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+from pydantic import BaseModel, Field
+
+from auth import require_local_operator
+from limiter import limiter
+from services.fetchers._store import get_latest_data_subset_refs
+from services.sar.sar_aoi import (
+    SarAoi,
+    add_aoi,
+    haversine_km,
+    load_aois,
+    remove_aoi,
+)
+from services.sar.sar_config import (
+    catalog_enabled,
+    clear_runtime_credentials,
+    openclaw_enabled,
+    products_fetch_enabled,
+    products_fetch_status,
+    require_private_tier_for_publish,
+    set_runtime_credentials,
+)
+
+router = APIRouter()
+
+
+# ---------------------------------------------------------------------------
+# Status — the in-app onboarding hook
+# ---------------------------------------------------------------------------
+@router.get("/api/sar/status")
+@limiter.limit("60/minute")
+async def sar_status(request: Request) -> dict:
+    """Layer status + signup links.
+
+    The frontend calls this whenever the SAR panel is opened.  When Mode B
+    is off, the response includes a step-by-step ``help`` block with the
+    free signup URLs so the user can enable everything without leaving the
+    app.
+    """
+    products_status = products_fetch_status()
+    return {
+        "ok": True,
+        "catalog": {
+            "mode": "A",
+            "enabled": catalog_enabled(),
+            "needs_account": False,
+            "description": "Free Sentinel-1 scene catalog from ASF Search.",
+        },
+        "products": {
+            "mode": "B",
+            **products_status,
+        },
+        "openclaw_enabled": openclaw_enabled(),
+        "require_private_tier": require_private_tier_for_publish(),
+    }
+
+
+# ---------------------------------------------------------------------------
+# Data feeds
+# ---------------------------------------------------------------------------
+@router.get("/api/sar/anomalies")
+@limiter.limit("60/minute")
+async def sar_anomalies(
+    request: Request,
+    kind: str = Query("", description="Optional anomaly kind filter"),
+    aoi_id: str = Query("", description="Optional AOI id filter"),
+    limit: int = Query(200, ge=1, le=1000),
+) -> dict:
+    """Return the latest cached SAR anomalies (Mode B)."""
+    snap = get_latest_data_subset_refs("sar_anomalies")
+    items = list(snap.get("sar_anomalies") or [])
+    if kind:
+        items = [a for a in items if a.get("kind") == kind]
+    if aoi_id:
+        aoi_id = aoi_id.strip().lower()
+        items = [a for a in items if (a.get("stack_id") or "").lower() == aoi_id]
+    items = items[:limit]
+    return {
+        "ok": True,
+        "count": len(items),
+        "anomalies": items,
+        "products_enabled": products_fetch_enabled(),
+    }
+
+
+@router.get("/api/sar/scenes")
+@limiter.limit("60/minute")
+async def sar_scenes(
+    request: Request,
+    aoi_id: str = Query(""),
+    limit: int = Query(200, ge=1, le=1000),
+) -> dict:
+    """Return the latest cached scene catalog (Mode A)."""
+    snap = get_latest_data_subset_refs("sar_scenes")
+    items = list(snap.get("sar_scenes") or [])
+    if aoi_id:
+        aoi_id = aoi_id.strip().lower()
+        items = [s for s in items if (s.get("aoi_id") or "").lower() == aoi_id]
+    items = items[:limit]
+    return {
+        "ok": True,
+        "count": len(items),
+        "scenes": items,
+        "catalog_enabled": catalog_enabled(),
+    }
+
+
+@router.get("/api/sar/coverage")
+@limiter.limit("60/minute")
+async def sar_coverage(request: Request) -> dict:
+    """Per-AOI coverage and rough next-pass estimate."""
+    snap = get_latest_data_subset_refs("sar_aoi_coverage")
+    return {
+        "ok": True,
+        "coverage": list(snap.get("sar_aoi_coverage") or []),
+    }
+
+
+@router.get("/api/sar/near")
+@limiter.limit("60/minute")
+async def sar_near(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lon: float = Query(..., ge=-180, le=180),
+    radius_km: float = Query(50, ge=1, le=2000),
+    kind: str = Query(""),
+    limit: int = Query(50, ge=1, le=500),
+) -> dict:
+    """Return anomalies whose center sits within ``radius_km`` of (lat, lon)."""
+    snap = get_latest_data_subset_refs("sar_anomalies")
+    items = list(snap.get("sar_anomalies") or [])
+    matches = []
+    for a in items:
+        try:
+            a_lat = float(a.get("lat", 0.0))
+            a_lon = float(a.get("lon", 0.0))
+        except (TypeError, ValueError):
+            continue
+        d = haversine_km(lat, lon, a_lat, a_lon)
+        if d > radius_km:
+            continue
+        if kind and a.get("kind") != kind:
+            continue
+        a = dict(a)
+        a["distance_km"] = round(d, 2)
+        matches.append(a)
+    matches.sort(key=lambda x: x.get("distance_km", 0))
+    return {
+        "ok": True,
+        "count": len(matches[:limit]),
+        "anomalies": matches[:limit],
+    }
+
+
+# ---------------------------------------------------------------------------
+# AOI CRUD
+# ---------------------------------------------------------------------------
+@router.get("/api/sar/aois")
+@limiter.limit("60/minute")
+async def sar_aoi_list(request: Request) -> dict:
+    return {
+        "ok": True,
+        "aois": [a.to_dict() for a in load_aois(force=True)],
+    }
+
+
+class AoiPayload(BaseModel):
+    id: str = Field(..., min_length=1, max_length=64)
+    name: str = Field(..., min_length=1, max_length=120)
+    description: str = Field("", max_length=400)
+    center_lat: float = Field(..., ge=-90, le=90)
+    center_lon: float = Field(..., ge=-180, le=180)
+    radius_km: float = Field(25.0, ge=1.0, le=500.0)
+    category: str = Field("watchlist", max_length=40)
+    polygon: list[list[float]] | None = None
+
+
+@router.post("/api/sar/aois", dependencies=[Depends(require_local_operator)])
+@limiter.limit("20/minute")
+async def sar_aoi_upsert(request: Request, payload: AoiPayload) -> dict:
+    aoi = SarAoi(
+        id=payload.id.strip().lower(),
+        name=payload.name.strip(),
+        description=payload.description.strip(),
+        center_lat=payload.center_lat,
+        center_lon=payload.center_lon,
+        radius_km=payload.radius_km,
+        polygon=payload.polygon,
+        category=(payload.category or "watchlist").strip().lower(),
+    )
+    add_aoi(aoi)
+    return {"ok": True, "aoi": aoi.to_dict()}
+
+
+@router.delete("/api/sar/aois/{aoi_id}", dependencies=[Depends(require_local_operator)])
+@limiter.limit("20/minute")
+async def sar_aoi_delete(request: Request, aoi_id: str) -> dict:
+    removed = remove_aoi(aoi_id)
+    if not removed:
+        raise HTTPException(status_code=404, detail="AOI not found")
+    return {"ok": True, "removed": aoi_id}
+
+
+# ---------------------------------------------------------------------------
+# Mode B enable / disable — one-click setup from the frontend
+# ---------------------------------------------------------------------------
+class ModeBEnablePayload(BaseModel):
+    earthdata_user: str = Field("", max_length=120)
+    earthdata_token: str = Field(..., min_length=8, max_length=2048)
+    copernicus_user: str = Field("", max_length=120)
+    copernicus_token: str = Field("", max_length=2048)
+
+
+@router.post("/api/sar/mode-b/enable", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def sar_mode_b_enable(request: Request, payload: ModeBEnablePayload) -> dict:
+    """Store Earthdata (and optional Copernicus) credentials and flip both
+    two-step opt-in flags.  Returns the fresh status payload so the UI can
+    immediately reflect the change.
+    """
+    set_runtime_credentials(
+        earthdata_user=payload.earthdata_user,
+        earthdata_token=payload.earthdata_token,
+        copernicus_user=payload.copernicus_user,
+        copernicus_token=payload.copernicus_token,
+        mode_b_opt_in=True,
+    )
+    return {
+        "ok": True,
+        "products": products_fetch_status(),
+    }
+
+
+@router.post("/api/sar/mode-b/disable", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def sar_mode_b_disable(request: Request) -> dict:
+    """Wipe runtime credentials and revert to Mode A only."""
+    clear_runtime_credentials()
+    return {
+        "ok": True,
+        "products": products_fetch_status(),
+    }

backend/routers/sigint.py

@@ -0,0 +1,67 @@
+from fastapi import APIRouter, Request, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+from services.data_fetcher import get_latest_data
+
+router = APIRouter()
+
+
+@router.get("/api/oracle/region-intel")
+@limiter.limit("30/minute")
+async def oracle_region_intel(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Get oracle intelligence summary for a geographic region."""
+    from services.oracle_service import get_region_oracle_intel
+    news_items = get_latest_data().get("news", [])
+    return get_region_oracle_intel(lat, lng, news_items)
+
+
+@router.get("/api/thermal/verify")
+@limiter.limit("10/minute")
+async def thermal_verify(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+    radius_km: float = Query(10, ge=1, le=100),
+):
+    """On-demand thermal anomaly verification using Sentinel-2 SWIR bands."""
+    from services.thermal_sentinel import search_thermal_anomaly
+    result = search_thermal_anomaly(lat, lng, radius_km)
+    return result
+
+
+@router.post("/api/sigint/transmit")
+@limiter.limit("5/minute")
+async def sigint_transmit(request: Request):
+    """Send an APRS-IS message to a specific callsign. Requires ham radio credentials."""
+    from services.wormhole_supervisor import get_transport_tier
+    tier = get_transport_tier()
+    if str(tier or "").startswith("private_"):
+        return {"ok": False, "detail": "APRS transmit blocked in private transport mode"}
+    body = await request.json()
+    callsign = body.get("callsign", "")
+    passcode = body.get("passcode", "")
+    target = body.get("target", "")
+    message = body.get("message", "")
+    if not all([callsign, passcode, target, message]):
+        return {"ok": False, "detail": "Missing required fields: callsign, passcode, target, message"}
+    from services.sigint_bridge import send_aprs_message
+    return send_aprs_message(callsign, passcode, target, message)
+
+
+@router.get("/api/sigint/nearest-sdr")
+@limiter.limit("30/minute")
+async def nearest_sdr(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Find the nearest KiwiSDR receivers to a given coordinate."""
+    from services.sigint_bridge import find_nearest_kiwisdr
+    kiwisdr_data = get_latest_data().get("kiwisdr", [])
+    return find_nearest_kiwisdr(lat, lng, kiwisdr_data)

backend/routers/tools.py

@@ -0,0 +1,303 @@
+import asyncio
+import logging
+import math
+from typing import Any
+from fastapi import APIRouter, Request, Query, Depends, HTTPException, Response
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _safe_float(val, default=0.0):
+    try:
+        parsed = float(val)
+        if not math.isfinite(parsed):
+            return default
+        return parsed
+    except (TypeError, ValueError):
+        return default
+
+
+class ShodanSearchRequest(BaseModel):
+    query: str
+    page: int = 1
+    facets: list[str] = []
+
+
+class ShodanCountRequest(BaseModel):
+    query: str
+    facets: list[str] = []
+
+
+class ShodanHostRequest(BaseModel):
+    ip: str
+    history: bool = False
+
+
+@router.get("/api/region-dossier")
+@limiter.limit("30/minute")
+def api_region_dossier(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Sync def so FastAPI runs it in a threadpool — prevents blocking the event loop."""
+    from services.region_dossier import get_region_dossier
+    return get_region_dossier(lat, lng)
+
+
+@router.get("/api/geocode/search")
+@limiter.limit("30/minute")
+async def api_geocode_search(
+    request: Request,
+    q: str = "",
+    limit: int = 5,
+    local_only: bool = False,
+):
+    from services.geocode import search_geocode
+    if not q or len(q.strip()) < 2:
+        return {"results": [], "query": q, "count": 0}
+    results = await asyncio.to_thread(search_geocode, q, limit, local_only)
+    return {"results": results, "query": q, "count": len(results)}
+
+
+@router.get("/api/geocode/reverse")
+@limiter.limit("60/minute")
+async def api_geocode_reverse(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+    local_only: bool = False,
+):
+    from services.geocode import reverse_geocode
+    return await asyncio.to_thread(reverse_geocode, lat, lng, local_only)
+
+
+@router.get("/api/sentinel2/search")
+@limiter.limit("30/minute")
+def api_sentinel2_search(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Search for latest Sentinel-2 imagery at a point. Sync for threadpool execution."""
+    from services.sentinel_search import search_sentinel2_scene
+    return search_sentinel2_scene(lat, lng)
+
+
+@router.post("/api/sentinel/token")
+@limiter.limit("60/minute")
+async def api_sentinel_token(request: Request):
+    """Proxy Copernicus CDSE OAuth2 token request (avoids browser CORS block)."""
+    import requests as req
+    body = await request.body()
+    from urllib.parse import parse_qs
+    params = parse_qs(body.decode("utf-8"))
+    client_id = params.get("client_id", [""])[0]
+    client_secret = params.get("client_secret", [""])[0]
+    if not client_id or not client_secret:
+        raise HTTPException(400, "client_id and client_secret required")
+    token_url = "https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token"
+    try:
+        resp = await asyncio.to_thread(req.post, token_url,
+            data={"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret},
+            timeout=15)
+        return Response(content=resp.content, status_code=resp.status_code, media_type="application/json")
+    except Exception:
+        logger.exception("Token request failed")
+        raise HTTPException(502, "Token request failed")
+
+
+_sh_token_cache: dict = {"token": None, "expiry": 0, "client_id": ""}
+
+
+@router.post("/api/sentinel/tile")
+@limiter.limit("300/minute")
+async def api_sentinel_tile(request: Request):
+    """Proxy Sentinel Hub Process API tile request (avoids CORS block)."""
+    import requests as req
+    import time as _time
+    try:
+        body = await request.json()
+    except Exception:
+        return JSONResponse(status_code=422, content={"ok": False, "detail": "invalid JSON body"})
+
+    client_id = body.get("client_id", "")
+    client_secret = body.get("client_secret", "")
+    preset = body.get("preset", "TRUE-COLOR")
+    date_str = body.get("date", "")
+    z = body.get("z", 0)
+    x = body.get("x", 0)
+    y = body.get("y", 0)
+
+    if not client_id or not client_secret or not date_str:
+        raise HTTPException(400, "client_id, client_secret, and date required")
+
+    now = _time.time()
+    if (_sh_token_cache["token"] and _sh_token_cache["client_id"] == client_id
+            and now < _sh_token_cache["expiry"] - 30):
+        token = _sh_token_cache["token"]
+    else:
+        token_url = "https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token"
+        try:
+            tresp = await asyncio.to_thread(req.post, token_url,
+                data={"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret},
+                timeout=15)
+            if tresp.status_code != 200:
+                raise HTTPException(401, f"Token auth failed: {tresp.text[:200]}")
+            tdata = tresp.json()
+            token = tdata["access_token"]
+            _sh_token_cache["token"] = token
+            _sh_token_cache["expiry"] = now + tdata.get("expires_in", 300)
+            _sh_token_cache["client_id"] = client_id
+        except HTTPException:
+            raise
+        except Exception:
+            logger.exception("Token request failed")
+            raise HTTPException(502, "Token request failed")
+
+    half = 20037508.342789244
+    tile_size = (2 * half) / math.pow(2, z)
+    min_x = -half + x * tile_size
+    max_x = min_x + tile_size
+    max_y = half - y * tile_size
+    min_y = max_y - tile_size
+    bbox = [min_x, min_y, max_x, max_y]
+
+    evalscripts = {
+        "TRUE-COLOR": '//VERSION=3\nfunction setup(){return{input:["B04","B03","B02"],output:{bands:3}};}\nfunction evaluatePixel(s){return[2.5*s.B04,2.5*s.B03,2.5*s.B02];}',
+        "FALSE-COLOR": '//VERSION=3\nfunction setup(){return{input:["B08","B04","B03"],output:{bands:3}};}\nfunction evaluatePixel(s){return[2.5*s.B08,2.5*s.B04,2.5*s.B03];}',
+        "NDVI": '//VERSION=3\nfunction setup(){return{input:["B04","B08"],output:{bands:3}};}\nfunction evaluatePixel(s){var n=(s.B08-s.B04)/(s.B08+s.B04);if(n<-0.2)return[0.05,0.05,0.05];if(n<0)return[0.75,0.75,0.75];if(n<0.1)return[0.86,0.86,0.86];if(n<0.2)return[0.92,0.84,0.68];if(n<0.3)return[0.77,0.88,0.55];if(n<0.4)return[0.56,0.80,0.32];if(n<0.5)return[0.35,0.72,0.18];if(n<0.6)return[0.20,0.60,0.08];if(n<0.7)return[0.10,0.48,0.04];return[0.0,0.36,0.0];}',
+        "MOISTURE-INDEX": '//VERSION=3\nfunction setup(){return{input:["B8A","B11"],output:{bands:3}};}\nfunction evaluatePixel(s){var m=(s.B8A-s.B11)/(s.B8A+s.B11);var r=Math.max(0,Math.min(1,1.5-3*m));var g=Math.max(0,Math.min(1,m<0?1.5+3*m:1.5-3*m));var b=Math.max(0,Math.min(1,1.5+3*(m-0.5)));return[r,g,b];}',
+    }
+    evalscript = evalscripts.get(preset, evalscripts["TRUE-COLOR"])
+
+    from datetime import datetime as _dt, timedelta as _td
+    try:
+        end_date = _dt.strptime(date_str, "%Y-%m-%d")
+    except ValueError:
+        end_date = _dt.utcnow()
+
+    if z <= 6:
+        lookback_days = 30
+    elif z <= 9:
+        lookback_days = 14
+    elif z <= 11:
+        lookback_days = 7
+    else:
+        lookback_days = 5
+
+    start_date = end_date - _td(days=lookback_days)
+
+    process_body = {
+        "input": {
+            "bounds": {"bbox": bbox, "properties": {"crs": "http://www.opengis.net/def/crs/EPSG/0/3857"}},
+            "data": [{"type": "sentinel-2-l2a", "dataFilter": {
+                "timeRange": {
+                    "from": start_date.strftime("%Y-%m-%dT00:00:00Z"),
+                    "to": end_date.strftime("%Y-%m-%dT23:59:59Z"),
+                },
+                "maxCloudCoverage": 30, "mosaickingOrder": "leastCC",
+            }}],
+        },
+        "output": {"width": 256, "height": 256,
+            "responses": [{"identifier": "default", "format": {"type": "image/png"}}]},
+        "evalscript": evalscript,
+    }
+    try:
+        resp = await asyncio.to_thread(req.post,
+            "https://sh.dataspace.copernicus.eu/api/v1/process",
+            json=process_body,
+            headers={"Authorization": f"Bearer {token}", "Accept": "image/png"},
+            timeout=30)
+        return Response(content=resp.content, status_code=resp.status_code,
+            media_type=resp.headers.get("content-type", "image/png"))
+    except Exception:
+        logger.exception("Process API failed")
+        raise HTTPException(502, "Process API failed")
+
+
+@router.get("/api/tools/shodan/status", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_shodan_status(request: Request):
+    from services.shodan_connector import get_shodan_connector_status
+    return get_shodan_connector_status()
+
+
+@router.post("/api/tools/shodan/search", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_shodan_search(request: Request, body: ShodanSearchRequest):
+    from services.shodan_connector import ShodanConnectorError, search_shodan
+    try:
+        return search_shodan(body.query, page=body.page, facets=body.facets)
+    except ShodanConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/shodan/count", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_shodan_count(request: Request, body: ShodanCountRequest):
+    from services.shodan_connector import ShodanConnectorError, count_shodan
+    try:
+        return count_shodan(body.query, facets=body.facets)
+    except ShodanConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/shodan/host", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_shodan_host(request: Request, body: ShodanHostRequest):
+    from services.shodan_connector import ShodanConnectorError, lookup_shodan_host
+    try:
+        return lookup_shodan_host(body.ip, history=body.history)
+    except ShodanConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.get("/api/tools/uw/status", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_uw_status(request: Request):
+    from services.unusual_whales_connector import get_uw_status
+    return get_uw_status()
+
+
+@router.post("/api/tools/uw/congress", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_uw_congress(request: Request):
+    from services.unusual_whales_connector import FinnhubConnectorError, fetch_congress_trades
+    try:
+        return fetch_congress_trades()
+    except FinnhubConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/uw/darkpool", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_uw_darkpool(request: Request):
+    from services.unusual_whales_connector import FinnhubConnectorError, fetch_insider_transactions
+    try:
+        return fetch_insider_transactions()
+    except FinnhubConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/uw/flow", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_uw_flow(request: Request):
+    from services.unusual_whales_connector import FinnhubConnectorError, fetch_defense_quotes
+    try:
+        return fetch_defense_quotes()
+    except FinnhubConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc

backend/scripts/bootstrap_manifest_helper.py

@@ -0,0 +1,115 @@
+import argparse
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+BACKEND_DIR = ROOT / "backend"
+
+if str(BACKEND_DIR) not in sys.path:
+    sys.path.insert(0, str(BACKEND_DIR))
+
+from services.mesh.mesh_bootstrap_manifest import (  # noqa: E402
+    bootstrap_signer_public_key_b64,
+    generate_bootstrap_signer,
+    write_signed_bootstrap_manifest,
+)
+
+
+def _load_peers(args: argparse.Namespace) -> list[dict]:
+    peers: list[dict] = []
+    if args.peers_file:
+        raw = json.loads(Path(args.peers_file).read_text(encoding="utf-8"))
+        if not isinstance(raw, list):
+            raise ValueError("peers file must be a JSON array")
+        for entry in raw:
+            if not isinstance(entry, dict):
+                raise ValueError("peers file entries must be objects")
+            peers.append(dict(entry))
+    for peer_arg in args.peer or []:
+        parts = [part.strip() for part in str(peer_arg).split(",", 3)]
+        if len(parts) < 3:
+            raise ValueError("peer entries must look like url,transport,role[,label]")
+        peer_url, transport, role = parts[:3]
+        label = parts[3] if len(parts) > 3 else ""
+        peers.append(
+            {
+                "peer_url": peer_url,
+                "transport": transport,
+                "role": role,
+                "label": label,
+            }
+        )
+    if not peers:
+        raise ValueError("at least one peer is required")
+    return peers
+
+
+def cmd_generate_keypair(_args: argparse.Namespace) -> int:
+    signer = generate_bootstrap_signer()
+    print(json.dumps(signer, indent=2))
+    return 0
+
+
+def cmd_sign(args: argparse.Namespace) -> int:
+    peers = _load_peers(args)
+    manifest = write_signed_bootstrap_manifest(
+        args.output,
+        signer_id=args.signer_id,
+        signer_private_key_b64=args.private_key_b64,
+        peers=peers,
+        valid_for_hours=int(args.valid_hours),
+    )
+    print(f"Wrote signed bootstrap manifest to {Path(args.output).resolve()}")
+    print(f"signer_id={manifest.signer_id}")
+    print(f"valid_until={manifest.valid_until}")
+    print(f"peer_count={len(manifest.peers)}")
+    print(f"MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY={bootstrap_signer_public_key_b64(args.private_key_b64)}")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Generate and sign Infonet bootstrap manifests for participant nodes."
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    keygen = subparsers.add_parser("generate-keypair", help="Generate an Ed25519 bootstrap signer keypair")
+    keygen.set_defaults(func=cmd_generate_keypair)
+
+    sign = subparsers.add_parser("sign", help="Sign a bootstrap manifest from peer entries")
+    sign.add_argument("--output", required=True, help="Output path for bootstrap_peers.json")
+    sign.add_argument("--signer-id", required=True, help="Manifest signer identifier")
+    sign.add_argument(
+        "--private-key-b64",
+        required=True,
+        help="Raw Ed25519 private key in base64 returned by generate-keypair",
+    )
+    sign.add_argument(
+        "--peers-file",
+        help="JSON file containing an array of peer objects with peer_url, transport, role, and optional label",
+    )
+    sign.add_argument(
+        "--peer",
+        action="append",
+        help="Inline peer in the form url,transport,role[,label]. May be repeated.",
+    )
+    sign.add_argument(
+        "--valid-hours",
+        type=int,
+        default=168,
+        help="Manifest validity window in hours (default: 168)",
+    )
+    sign.set_defaults(func=cmd_sign)
+
+    return parser
+
+
+def main() -> int:
+    parser = build_parser()
+    args = parser.parse_args()
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/scripts/check-env.ps1

@@ -0,0 +1,5 @@
+param(
+  [string]$Python = "python"
+)
+
+& $Python -c "from services.env_check import validate_env; validate_env(strict=False)"

backend/scripts/check-env.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+PYTHON="${PYTHON:-python3}"
+"$PYTHON" -c "from services.env_check import validate_env; validate_env(strict=False)"

backend/scripts/convert_power_plants.py

@@ -0,0 +1,58 @@
+"""Download WRI Global Power Plant Database CSV and convert to compact JSON.
+
+Usage:
+    python backend/scripts/convert_power_plants.py
+
+Output:
+    backend/data/power_plants.json
+"""
+import csv
+import json
+import io
+import zipfile
+import urllib.request
+from pathlib import Path
+
+# WRI Global Power Plant Database v1.3.0 (GitHub release)
+CSV_URL = "https://raw.githubusercontent.com/wri/global-power-plant-database/master/output_database/global_power_plant_database.csv"
+OUT_PATH = Path(__file__).parent.parent / "data" / "power_plants.json"
+
+
+def main() -> None:
+    print(f"Downloading WRI Global Power Plant Database from GitHub...")
+    req = urllib.request.Request(CSV_URL, headers={"User-Agent": "ShadowBroker-OSINT/1.0"})
+    with urllib.request.urlopen(req, timeout=60) as resp:
+        raw = resp.read().decode("utf-8")
+
+    reader = csv.DictReader(io.StringIO(raw))
+    plants: list[dict] = []
+    skipped = 0
+    for row in reader:
+        try:
+            lat = float(row["latitude"])
+            lng = float(row["longitude"])
+        except (ValueError, KeyError):
+            skipped += 1
+            continue
+        if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+            skipped += 1
+            continue
+        capacity_raw = row.get("capacity_mw", "")
+        capacity_mw = float(capacity_raw) if capacity_raw else None
+        plants.append({
+            "name": row.get("name", "Unknown"),
+            "country": row.get("country_long", ""),
+            "fuel_type": row.get("primary_fuel", "Unknown"),
+            "capacity_mw": capacity_mw,
+            "owner": row.get("owner", ""),
+            "lat": round(lat, 5),
+            "lng": round(lng, 5),
+        })
+
+    OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
+    OUT_PATH.write_text(json.dumps(plants, ensure_ascii=False, separators=(",", ":")), encoding="utf-8")
+    print(f"Wrote {len(plants)} power plants to {OUT_PATH} (skipped {skipped})")
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/diagnostics.py

@@ -0,0 +1,45 @@
+from datetime import datetime
+from services.data_fetcher import get_latest_data
+from services.fetchers._store import source_timestamps, active_layers, source_freshness
+from services.fetch_health import get_health_snapshot
+
+
+def _fmt_ts(ts: str | None) -> str:
+    if not ts:
+        return "-"
+    try:
+        return datetime.fromisoformat(ts).strftime("%Y-%m-%d %H:%M:%S")
+    except Exception:
+        return ts
+
+
+def main():
+    data = get_latest_data()
+    print("=== Diagnostics ===")
+    print(f"Last updated: {_fmt_ts(data.get('last_updated'))}")
+    print(
+        f"Active layers: {sum(1 for v in active_layers.values() if v)} enabled / {len(active_layers)} total"
+    )
+
+    print("\n--- Source Timestamps ---")
+    for k, v in sorted(source_timestamps.items()):
+        print(f"{k:20} {_fmt_ts(v)}")
+
+    print("\n--- Source Freshness ---")
+    for k, v in sorted(source_freshness.items()):
+        last_ok = _fmt_ts(v.get("last_ok"))
+        last_err = _fmt_ts(v.get("last_error"))
+        print(f"{k:20} ok={last_ok} err={last_err}")
+
+    print("\n--- Fetch Health ---")
+    health = get_health_snapshot()
+    for k, v in sorted(health.items()):
+        print(
+            f"{k:20} ok={v.get('ok_count', 0)} err={v.get('error_count', 0)} "
+            f"last_ok={_fmt_ts(v.get('last_ok'))} last_err={_fmt_ts(v.get('last_error'))} "
+            f"avg_ms={v.get('avg_duration_ms')}"
+        )
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/release_helper.py

@@ -0,0 +1,303 @@
+import argparse
+import hashlib
+import json
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[2]
+PACKAGE_JSON = ROOT / "frontend" / "package.json"
+
+
+def _normalize_version(raw: str) -> str:
+    version = str(raw or "").strip()
+    if version.startswith("v"):
+        version = version[1:]
+    parts = version.split(".")
+    if len(parts) != 3 or not all(part.isdigit() for part in parts):
+        raise ValueError("Version must look like X.Y.Z")
+    return version
+
+
+def _read_package_json() -> dict:
+    return json.loads(PACKAGE_JSON.read_text(encoding="utf-8"))
+
+
+def _write_package_json(data: dict) -> None:
+    PACKAGE_JSON.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
+
+
+def current_version() -> str:
+    return str(_read_package_json().get("version") or "").strip()
+
+
+def set_version(version: str) -> str:
+    normalized = _normalize_version(version)
+    data = _read_package_json()
+    data["version"] = normalized
+    _write_package_json(data)
+    return normalized
+
+
+def expected_tag(version: str) -> str:
+    return f"v{_normalize_version(version)}"
+
+
+def expected_asset(version: str) -> str:
+    normalized = _normalize_version(version)
+    return f"ShadowBroker_v{normalized}.zip"
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 128), b""):
+            digest.update(chunk)
+    return digest.hexdigest().lower()
+
+
+def _default_generated_at() -> str:
+    return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def build_release_attestation(
+    *,
+    suite_green: bool,
+    suite_name: str = "dm_relay_security",
+    detail: str = "",
+    report: str = "",
+    command: str = "",
+    commit: str = "",
+    generated_at: str = "",
+    threat_model_reference: str = "docs/mesh/threat-model.md",
+    workflow: str = "",
+    run_id: str = "",
+    run_attempt: str = "",
+    ref: str = "",
+) -> dict:
+    normalized_generated_at = str(generated_at or "").strip() or _default_generated_at()
+    normalized_commit = str(commit or "").strip() or os.environ.get("GITHUB_SHA", "").strip()
+    normalized_workflow = str(workflow or "").strip() or os.environ.get("GITHUB_WORKFLOW", "").strip()
+    normalized_run_id = str(run_id or "").strip() or os.environ.get("GITHUB_RUN_ID", "").strip()
+    normalized_run_attempt = str(run_attempt or "").strip() or os.environ.get("GITHUB_RUN_ATTEMPT", "").strip()
+    normalized_ref = str(ref or "").strip() or os.environ.get("GITHUB_REF", "").strip()
+    normalized_suite_name = str(suite_name or "").strip() or "dm_relay_security"
+    normalized_report = str(report or "").strip()
+    normalized_command = str(command or "").strip()
+    normalized_detail = str(detail or "").strip() or (
+        "CI attestation confirms the DM relay security suite is green."
+        if suite_green
+        else "CI attestation recorded a failing DM relay security suite run."
+    )
+    payload = {
+        "generated_at": normalized_generated_at,
+        "commit": normalized_commit,
+        "threat_model_reference": str(threat_model_reference or "").strip()
+        or "docs/mesh/threat-model.md",
+        "dm_relay_security_suite": {
+            "name": normalized_suite_name,
+            "green": bool(suite_green),
+            "detail": normalized_detail,
+            "report": normalized_report,
+        },
+    }
+    if normalized_command:
+        payload["dm_relay_security_suite"]["command"] = normalized_command
+    ci = {
+        "workflow": normalized_workflow,
+        "run_id": normalized_run_id,
+        "run_attempt": normalized_run_attempt,
+        "ref": normalized_ref,
+    }
+    if any(ci.values()):
+        payload["ci"] = ci
+    return payload
+
+
+def write_release_attestation(output_path: Path | str, **kwargs) -> dict:
+    path = Path(output_path).resolve()
+    payload = build_release_attestation(**kwargs)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+    return payload
+
+
+def cmd_show(_args: argparse.Namespace) -> int:
+    version = current_version()
+    if not version:
+        print("package.json has no version", file=sys.stderr)
+        return 1
+    print(f"package.json version : {version}")
+    print(f"expected git tag     : {expected_tag(version)}")
+    print(f"expected zip asset   : {expected_asset(version)}")
+    return 0
+
+
+def cmd_set_version(args: argparse.Namespace) -> int:
+    version = set_version(args.version)
+    print(f"Set frontend/package.json version to {version}")
+    print(f"Next release tag  : {expected_tag(version)}")
+    print(f"Next zip asset    : {expected_asset(version)}")
+    return 0
+
+
+def cmd_hash(args: argparse.Namespace) -> int:
+    version = _normalize_version(args.version) if args.version else current_version()
+    if not version:
+        print("No version available; pass --version or set frontend/package.json", file=sys.stderr)
+        return 1
+
+    zip_path = Path(args.zip_path).resolve()
+    if not zip_path.is_file():
+        print(f"ZIP not found: {zip_path}", file=sys.stderr)
+        return 1
+
+    digest = sha256_file(zip_path)
+    expected_name = expected_asset(version)
+    asset_matches = zip_path.name == expected_name
+
+    print(f"release version     : {version}")
+    print(f"expected git tag    : {expected_tag(version)}")
+    print(f"zip path            : {zip_path}")
+    print(f"zip name matches    : {'yes' if asset_matches else 'no'}")
+    print(f"expected zip asset  : {expected_name}")
+    print(f"SHA-256             : {digest}")
+    print("")
+    print("Updater pin:")
+    print(f"MESH_UPDATE_SHA256={digest}")
+    return 0 if asset_matches else 2
+
+
+def cmd_write_attestation(args: argparse.Namespace) -> int:
+    suite_green = bool(args.suite_green)
+    payload = write_release_attestation(
+        args.output_path,
+        suite_green=suite_green,
+        suite_name=args.suite_name,
+        detail=args.detail,
+        report=args.report,
+        command=args.command,
+        commit=args.commit,
+        generated_at=args.generated_at,
+        threat_model_reference=args.threat_model_reference,
+        workflow=args.workflow,
+        run_id=args.run_id,
+        run_attempt=args.run_attempt,
+        ref=args.ref,
+    )
+    output_path = Path(args.output_path).resolve()
+    print(f"Wrote release attestation: {output_path}")
+    print(f"DM relay security suite : {'green' if suite_green else 'red'}")
+    print(f"Commit                  : {payload.get('commit', '')}")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Helper for ShadowBroker release version/tag/asset consistency."
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    show_parser = subparsers.add_parser("show", help="Show current version, expected tag, and asset")
+    show_parser.set_defaults(func=cmd_show)
+
+    set_version_parser = subparsers.add_parser("set-version", help="Update frontend/package.json version")
+    set_version_parser.add_argument("version", help="Version like 0.9.7")
+    set_version_parser.set_defaults(func=cmd_set_version)
+
+    hash_parser = subparsers.add_parser(
+        "hash", help="Compute SHA-256 for a release ZIP and print the updater pin"
+    )
+    hash_parser.add_argument("zip_path", help="Path to the release ZIP")
+    hash_parser.add_argument(
+        "--version",
+        help="Release version like 0.9.7. Defaults to frontend/package.json version.",
+    )
+    hash_parser.set_defaults(func=cmd_hash)
+
+    attestation_parser = subparsers.add_parser(
+        "write-attestation",
+        help="Write a structured Sprint 8 release attestation JSON file",
+    )
+    attestation_parser.add_argument("output_path", help="Where to write the attestation JSON")
+    suite_group = attestation_parser.add_mutually_exclusive_group(required=True)
+    suite_group.add_argument(
+        "--suite-green",
+        action="store_true",
+        help="Mark the DM relay security suite as green",
+    )
+    suite_group.add_argument(
+        "--suite-red",
+        action="store_true",
+        help="Mark the DM relay security suite as failing",
+    )
+    attestation_parser.add_argument(
+        "--suite-name",
+        default="dm_relay_security",
+        help="Suite name to record in the attestation",
+    )
+    attestation_parser.add_argument(
+        "--detail",
+        default="",
+        help="Human-readable suite detail. Defaults to a CI-generated message.",
+    )
+    attestation_parser.add_argument(
+        "--report",
+        default="",
+        help="Path to the suite report or artifact reference to embed in the attestation.",
+    )
+    attestation_parser.add_argument(
+        "--command",
+        default="",
+        help="Exact suite command used to generate the attestation.",
+    )
+    attestation_parser.add_argument(
+        "--commit",
+        default="",
+        help="Commit SHA. Defaults to GITHUB_SHA when available.",
+    )
+    attestation_parser.add_argument(
+        "--generated-at",
+        default="",
+        help="UTC timestamp for the attestation. Defaults to current UTC time.",
+    )
+    attestation_parser.add_argument(
+        "--threat-model-reference",
+        default="docs/mesh/threat-model.md",
+        help="Threat model reference to embed in the attestation.",
+    )
+    attestation_parser.add_argument(
+        "--workflow",
+        default="",
+        help="Workflow name. Defaults to GITHUB_WORKFLOW when available.",
+    )
+    attestation_parser.add_argument(
+        "--run-id",
+        default="",
+        help="Workflow run ID. Defaults to GITHUB_RUN_ID when available.",
+    )
+    attestation_parser.add_argument(
+        "--run-attempt",
+        default="",
+        help="Workflow run attempt. Defaults to GITHUB_RUN_ATTEMPT when available.",
+    )
+    attestation_parser.add_argument(
+        "--ref",
+        default="",
+        help="Git ref. Defaults to GITHUB_REF when available.",
+    )
+    attestation_parser.set_defaults(func=cmd_write_attestation)
+
+    return parser
+
+
+def main() -> int:
+    parser = build_parser()
+    args = parser.parse_args()
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/scripts/repair_wormhole_secure_storage.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from services.mesh import mesh_secure_storage
+from services.mesh.mesh_wormhole_contacts import CONTACTS_FILE
+from services.mesh.mesh_wormhole_identity import IDENTITY_FILE, _default_identity
+from services.mesh.mesh_wormhole_persona import PERSONA_FILE, _default_state as _default_persona_state
+from services.mesh.mesh_wormhole_ratchet import STATE_FILE as RATCHET_FILE
+
+
+def _load_payloads() -> dict[Path, object]:
+    return {
+        IDENTITY_FILE: mesh_secure_storage.read_secure_json(IDENTITY_FILE, _default_identity),
+        PERSONA_FILE: mesh_secure_storage.read_secure_json(PERSONA_FILE, _default_persona_state),
+        RATCHET_FILE: mesh_secure_storage.read_secure_json(RATCHET_FILE, lambda: {}),
+        CONTACTS_FILE: mesh_secure_storage.read_secure_json(CONTACTS_FILE, lambda: {}),
+    }
+
+
+def main() -> None:
+    payloads = _load_payloads()
+
+    master_key_file = mesh_secure_storage.MASTER_KEY_FILE
+    backup_key_file = master_key_file.with_suffix(master_key_file.suffix + ".bak")
+    if master_key_file.exists():
+        if backup_key_file.exists():
+            backup_key_file.unlink()
+        master_key_file.replace(backup_key_file)
+
+    for path, payload in payloads.items():
+        mesh_secure_storage.write_secure_json(path, payload)
+
+    print(
+        json.dumps(
+            {
+                "ok": True,
+                "rewrapped": [str(path.name) for path in payloads.keys()],
+                "master_key": str(master_key_file),
+                "backup_master_key": str(backup_key_file) if backup_key_file.exists() else "",
+            }
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/rotate_secure_storage_secret.py

@@ -0,0 +1,75 @@
+"""Rotate the MESH_SECURE_STORAGE_SECRET used to protect key envelopes at rest.
+
+Usage — stop the backend first, then run:
+
+    MESH_OLD_STORAGE_SECRET=<current>  \\
+    MESH_NEW_STORAGE_SECRET=<new>      \\
+    python -m scripts.rotate_secure_storage_secret
+
+Dry-run mode (validates old secret without writing anything):
+
+    MESH_OLD_STORAGE_SECRET=<current>  \\
+    MESH_NEW_STORAGE_SECRET=<new>      \\
+    python -m scripts.rotate_secure_storage_secret --dry-run
+
+Or, for Docker deployments:
+
+    docker exec -e MESH_OLD_STORAGE_SECRET=<current> \\
+                -e MESH_NEW_STORAGE_SECRET=<new>      \\
+                <container> python -m scripts.rotate_secure_storage_secret
+
+After successful rotation, update your .env (or Docker secret file) to set
+MESH_SECURE_STORAGE_SECRET to the new value, then restart the backend.
+
+The script fails closed: if the old secret cannot unwrap any existing envelope,
+nothing is written. Non-passphrase envelopes (DPAPI, raw) are skipped with a
+warning.
+
+Before rewriting, .bak copies of every envelope are created so a mid-rotation
+crash leaves recoverable backups on disk.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+
+def main() -> None:
+    dry_run = "--dry-run" in sys.argv
+
+    old_secret = os.environ.get("MESH_OLD_STORAGE_SECRET", "").strip()
+    new_secret = os.environ.get("MESH_NEW_STORAGE_SECRET", "").strip()
+
+    if not old_secret:
+        print("ERROR: MESH_OLD_STORAGE_SECRET environment variable is required.", file=sys.stderr)
+        sys.exit(1)
+    if not new_secret:
+        print("ERROR: MESH_NEW_STORAGE_SECRET environment variable is required.", file=sys.stderr)
+        sys.exit(1)
+
+    from services.mesh.mesh_secure_storage import SecureStorageError, rotate_storage_secret
+
+    try:
+        result = rotate_storage_secret(old_secret, new_secret, dry_run=dry_run)
+    except SecureStorageError as exc:
+        print(f"ROTATION FAILED: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+    print(json.dumps(result, indent=2))
+    if dry_run:
+        print(
+            "\nDry run complete. No files were modified. Run again without --dry-run to perform the rotation.",
+            file=sys.stderr,
+        )
+    else:
+        print(
+            "\nRotation complete. Update MESH_SECURE_STORAGE_SECRET to the new value and restart the backend."
+            "\nBackup files (.bak) were created alongside each rotated envelope.",
+            file=sys.stderr,
+        )
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/scan-secrets.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+# scan-secrets.sh — Catch keys, secrets, and credentials before they hit git.
+#
+# Usage:
+#   ./backend/scripts/scan-secrets.sh          # Scan staged files (pre-commit)
+#   ./backend/scripts/scan-secrets.sh --all     # Scan entire working tree
+#   ./backend/scripts/scan-secrets.sh --staged  # Scan staged files only (default)
+#
+# Exit code: 0 = clean, 1 = secrets found
+
+set -euo pipefail
+
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+GREEN='\033[0;32m'
+NC='\033[0m'
+
+MODE="${1:---staged}"
+FOUND=0
+
+# ── Get file list based on mode ─────────────────────────────────────────
+if [[ "$MODE" == "--all" ]]; then
+    FILELIST=$(mktemp)
+    { git ls-files 2>/dev/null; git ls-files --others --exclude-standard 2>/dev/null; } > "$FILELIST"
+    echo -e "${YELLOW}Scanning entire working tree...${NC}"
+else
+    FILELIST=$(mktemp)
+    git diff --cached --name-only --diff-filter=ACMR 2>/dev/null > "$FILELIST" || true
+    if [[ ! -s "$FILELIST" ]]; then
+        echo -e "${GREEN}No staged files to scan.${NC}"
+        rm -f "$FILELIST"
+        exit 0
+    fi
+    echo -e "${YELLOW}Scanning $(wc -l < "$FILELIST" | tr -d ' ') staged files...${NC}"
+fi
+
+# ── Check 1: Dangerous file extensions ──────────────────────────────────
+KEY_EXT='\.key$|\.pem$|\.p12$|\.pfx$|\.jks$|\.keystore$|\.p8$|\.der$'
+SECRET_EXT='\.secret$|\.secrets$|\.credential$|\.credentials$'
+
+HITS=$(grep -iE "$KEY_EXT|$SECRET_EXT" "$FILELIST" 2>/dev/null || true)
+if [[ -n "$HITS" ]]; then
+    echo -e "\n${RED}BLOCKED: Key/secret files detected:${NC}"
+    echo "$HITS" | while read -r f; do echo -e "  ${RED}$f${NC}"; done
+    FOUND=1
+fi
+
+# ── Check 2: Dangerous filenames ────────────────────────────────────────
+RISKY='id_rsa|id_ed25519|id_ecdsa|private_key|private\.key|secret_key|master\.key'
+RISKY+='|serviceaccount|gcloud.*\.json|firebase.*\.json|\.htpasswd'
+
+HITS=$(grep -iE "$RISKY" "$FILELIST" 2>/dev/null || true)
+if [[ -n "$HITS" ]]; then
+    echo -e "\n${RED}BLOCKED: Risky filenames detected:${NC}"
+    echo "$HITS" | while read -r f; do echo -e "  ${RED}$f${NC}"; done
+    FOUND=1
+fi
+
+# ── Check 3: .env files (not .env.example) ──────────────────────────────
+HITS=$(grep -E '(^|/)\.env(\.[^e].*)?$' "$FILELIST" 2>/dev/null | grep -v '\.example' || true)
+if [[ -n "$HITS" ]]; then
+    echo -e "\n${RED}BLOCKED: Environment files detected:${NC}"
+    echo "$HITS" | while read -r f; do echo -e "  ${RED}$f${NC}"; done
+    FOUND=1
+fi
+
+# ── Check 4: _domain_keys directory (project-specific) ──────────────────
+HITS=$(grep '_domain_keys/' "$FILELIST" 2>/dev/null || true)
+if [[ -n "$HITS" ]]; then
+    echo -e "\n${RED}BLOCKED: Domain keys directory detected:${NC}"
+    echo "$HITS" | while read -r f; do echo -e "  ${RED}$f${NC}"; done
+    FOUND=1
+fi
+
+# ── Check 5: Content scan for embedded secrets (single grep pass) ───────
+# Build one mega-pattern and run grep once across all files (fast!)
+SECRET_REGEX='PRIVATE KEY-----|'
+SECRET_REGEX+='ssh-rsa AAAA[0-9A-Za-z+/]|'
+SECRET_REGEX+='ssh-ed25519 AAAA[0-9A-Za-z+/]|'
+SECRET_REGEX+='ghp_[0-9a-zA-Z]{36}|'                          # GitHub PAT
+SECRET_REGEX+='github_pat_[0-9a-zA-Z]{22}_[0-9a-zA-Z]{59}|'   # GitHub fine-grained
+SECRET_REGEX+='gho_[0-9a-zA-Z]{36}|'                          # GitHub OAuth
+SECRET_REGEX+='sk-[0-9a-zA-Z]{48}|'                           # OpenAI key
+SECRET_REGEX+='sk-ant-[0-9a-zA-Z-]{90,}|'                     # Anthropic key
+SECRET_REGEX+='AKIA[0-9A-Z]{16}|'                             # AWS access key
+SECRET_REGEX+='AIzaSy[0-9A-Za-z_-]{33}|'                      # Google API key
+SECRET_REGEX+='xox[bpoas]-[0-9a-zA-Z-]+|'                     # Slack token
+SECRET_REGEX+='npm_[0-9a-zA-Z]{36}|'                          # npm token
+SECRET_REGEX+='pypi-[0-9a-zA-Z-]{50,}'                        # PyPI token
+
+# Filter to text-like files only (skip binaries by extension + skip this script)
+TEXT_FILES=$(grep -ivE '\.(png|jpg|jpeg|gif|ico|svg|woff2?|ttf|eot|pbf|zip|tar|gz|db|sqlite|xlsx|pdf|mp[34]|wav|ogg|webm|webp|avif)$' "$FILELIST" | grep -v 'scan-secrets\.sh$' || true)
+
+if [[ -n "$TEXT_FILES" ]]; then
+    # Use grep with file list, skip missing/binary, limit output
+    CONTENT_HITS=$(echo "$TEXT_FILES" | xargs grep -lE "$SECRET_REGEX" 2>/dev/null || true)
+    if [[ -n "$CONTENT_HITS" ]]; then
+        echo -e "\n${RED}BLOCKED: Embedded secrets/tokens found in:${NC}"
+        echo "$CONTENT_HITS" | while read -r f; do
+            echo -e "  ${RED}$f${NC}"
+            # Show first matching line for context
+            grep -nE "$SECRET_REGEX" "$f" 2>/dev/null | head -2 | while read -r line; do
+                echo -e "    ${YELLOW}$line${NC}"
+            done
+        done
+        FOUND=1
+    fi
+fi
+
+rm -f "$FILELIST"
+
+# ── Result ──────────────────────────────────────────────────────────────
+echo ""
+if [[ $FOUND -eq 1 ]]; then
+    echo -e "${RED}Secret scan FAILED. Add these to .gitignore or remove them before committing.${NC}"
+    echo -e "${YELLOW}If intentional (e.g. test fixtures): git commit --no-verify${NC}"
+    exit 1
+else
+    echo -e "${GREEN}Secret scan passed. No keys or secrets detected.${NC}"
+    exit 0
+fi

backend/scripts/setup-venv.ps1

@@ -0,0 +1,16 @@
+param(
+  [string]$Python = "py"
+)
+
+$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
+$venvPath = Join-Path $repoRoot "venv"
+$venvMarker = Join-Path $repoRoot ".venv-dir"
+& $Python -3.11 -m venv $venvPath
+
+$pip = Join-Path $venvPath "Scripts\pip.exe"
+& $pip install --upgrade pip
+Push-Location $repoRoot
+& (Join-Path $venvPath "Scripts\python.exe") -m pip install -e .
+& $pip install pytest pytest-asyncio ruff black
+"venv" | Set-Content -LiteralPath $venvMarker -NoNewline
+Pop-Location

backend/scripts/setup-venv.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+PYTHON="${PYTHON:-python3.11}"
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+VENV_DIR="$REPO_ROOT/venv"
+VENV_MARKER="$REPO_ROOT/.venv-dir"
+
+"$PYTHON" -m venv "$VENV_DIR"
+"$VENV_DIR/bin/pip" install --upgrade pip
+cd "$REPO_ROOT"
+"$VENV_DIR/bin/python" -m pip install -e .
+"$VENV_DIR/bin/pip" install pytest pytest-asyncio ruff black
+printf 'venv\n' > "$VENV_MARKER"

backend/seattle_sample.json

@@ -1,8 +0,0 @@
-{
-  "code" : "dataset.missing",
-  "error" : true,
-  "message" : "Not found",
-  "data" : {
-    "id" : "xqwu-hwdm"
-  }
-}

backend/services/ai_intel_store.py

@@ -0,0 +1,178 @@
+"""ai_intel_store — compatibility wrapper around ai_pin_store + layer injection.
+
+openclaw_channel.py and routers/ai_intel.py import from this module name.
+All pin/layer logic lives in ai_pin_store.py; this module re-exports with the
+expected function signatures and adds the layer injection helper.
+"""
+
+import logging
+import time
+from typing import Any
+
+from services.ai_pin_store import (
+    create_pin,
+    create_pins_batch,
+    get_pins,
+    delete_pin,
+    clear_pins,
+    pin_count,
+    pins_as_geojson,
+    purge_expired,
+    # Layer CRUD
+    create_layer,
+    get_layers,
+    update_layer,
+    delete_layer,
+    # Feed layers
+    get_feed_layers,
+    replace_layer_pins,
+)
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Re-exports expected by openclaw_channel._dispatch_command
+# ---------------------------------------------------------------------------
+
+
+def get_all_intel_pins() -> list[dict[str, Any]]:
+    """Return all active pins (no filter, generous limit)."""
+    return get_pins(limit=2000)
+
+
+def add_intel_pin(args: dict[str, Any]) -> dict[str, Any]:
+    """Create a single pin from a command-channel args dict."""
+    ea = args.get("entity_attachment")
+    return create_pin(
+        lat=float(args.get("lat", 0)),
+        lng=float(args.get("lng", 0)),
+        label=str(args.get("label", ""))[:200],
+        category=str(args.get("category", "custom")),
+        layer_id=str(args.get("layer_id", "")),
+        color=str(args.get("color", "")),
+        description=str(args.get("description", "")),
+        source=str(args.get("source", "openclaw")),
+        source_url=str(args.get("source_url", "")),
+        confidence=float(args.get("confidence", 1.0)),
+        ttl_hours=float(args.get("ttl_hours", 0)),
+        metadata=args.get("metadata") or {},
+        entity_attachment=ea if isinstance(ea, dict) else None,
+    )
+
+
+def delete_intel_pin(pin_id: str) -> bool:
+    """Delete a pin by ID."""
+    return delete_pin(pin_id)
+
+
+# Layer helpers for OpenClaw
+def create_intel_layer(args: dict[str, Any]) -> dict[str, Any]:
+    """Create a layer from a command-channel args dict."""
+    return create_layer(
+        name=str(args.get("name", "Untitled"))[:100],
+        description=str(args.get("description", ""))[:500],
+        source=str(args.get("source", "openclaw"))[:50],
+        color=str(args.get("color", "")),
+        feed_url=str(args.get("feed_url", "")),
+        feed_interval=int(args.get("feed_interval", 300)),
+    )
+
+
+def get_intel_layers() -> list[dict[str, Any]]:
+    """Return all layers with pin counts."""
+    return get_layers()
+
+
+def update_intel_layer(layer_id: str, args: dict[str, Any]) -> dict[str, Any] | None:
+    """Update a layer from a command-channel args dict."""
+    return update_layer(layer_id, **{
+        k: v for k, v in args.items()
+        if k in ("name", "description", "visible", "color", "feed_url", "feed_interval")
+    })
+
+
+def delete_intel_layer(layer_id: str) -> int:
+    """Delete a layer and its pins. Returns pin count removed."""
+    return delete_layer(layer_id)
+
+
+# ---------------------------------------------------------------------------
+# Layer injection — inserts agent data into native telemetry layers
+# ---------------------------------------------------------------------------
+
+# Layers that agents are allowed to inject into.
+_INJECTABLE_LAYERS = frozenset({
+    "cctv", "ships", "sigint", "kiwisdr", "military_bases",
+    "datacenters", "power_plants", "satnogs_stations",
+    "volcanoes", "earthquakes", "news", "viirs_change_nodes",
+    "air_quality",
+})
+
+
+def inject_layer_data(
+    layer: str,
+    items: list[dict[str, Any]],
+    mode: str = "append",
+) -> dict[str, Any]:
+    """Inject agent data into a native telemetry layer."""
+    from services.fetchers._store import latest_data, _data_lock, bump_data_version
+
+    layer = str(layer or "").strip()
+    if layer not in _INJECTABLE_LAYERS:
+        return {"ok": False, "detail": f"layer '{layer}' not injectable"}
+
+    items = list(items or [])[:200]
+    if not items:
+        return {"ok": False, "detail": "no items provided"}
+
+    now = time.time()
+    tagged = []
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        entry = dict(item)
+        entry["_injected"] = True
+        entry["_source"] = "user:openclaw"
+        entry["_injected_at"] = now
+        tagged.append(entry)
+
+    with _data_lock:
+        existing = latest_data.get(layer)
+        if not isinstance(existing, list):
+            existing = []
+
+        if mode == "replace":
+            existing = [e for e in existing if not e.get("_injected")]
+
+        existing.extend(tagged)
+        latest_data[layer] = existing
+
+    bump_data_version()
+
+    return {
+        "ok": True,
+        "layer": layer,
+        "injected": len(tagged),
+        "mode": mode,
+    }
+
+
+def clear_injected_data(layer: str = "") -> dict[str, Any]:
+    """Remove all injected items from a layer (or all layers)."""
+    from services.fetchers._store import latest_data, _data_lock, bump_data_version
+
+    removed = 0
+    with _data_lock:
+        targets = [layer] if layer else list(_INJECTABLE_LAYERS)
+        for lyr in targets:
+            existing = latest_data.get(lyr)
+            if not isinstance(existing, list):
+                continue
+            before = len(existing)
+            latest_data[lyr] = [e for e in existing if not e.get("_injected")]
+            removed += before - len(latest_data[lyr])
+
+    if removed:
+        bump_data_version()
+
+    return {"ok": True, "removed": removed}

backend/services/ai_pin_store.py

@@ -0,0 +1,633 @@
+"""AI Intel pin storage — layered pin system with JSON file persistence.
+
+Supports:
+  - Named pin layers (created by user or AI)
+  - Pins with optional entity attachment (track moving objects)
+  - Pin source tracking (user vs openclaw)
+  - Layer visibility toggles
+  - External feed URL per layer (for Phase 5)
+  - GeoJSON export per layer or all layers
+"""
+
+import json
+import logging
+import os
+import threading
+import time
+import uuid
+from datetime import datetime
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Pin schema
+# ---------------------------------------------------------------------------
+
+PIN_CATEGORIES = {
+    "threat", "news", "geolocation", "custom", "anomaly",
+    "military", "maritime", "flight", "infrastructure", "weather",
+    "sigint", "prediction", "research",
+}
+
+PIN_COLORS = {
+    "threat": "#ef4444",       # red
+    "news": "#f59e0b",         # amber
+    "geolocation": "#8b5cf6",  # violet
+    "custom": "#3b82f6",       # blue
+    "anomaly": "#f97316",      # orange
+    "military": "#dc2626",     # dark red
+    "maritime": "#0ea5e9",     # sky
+    "flight": "#6366f1",       # indigo
+    "infrastructure": "#64748b",  # slate
+    "weather": "#22d3ee",      # cyan
+    "sigint": "#a855f7",       # purple
+    "prediction": "#eab308",   # yellow
+    "research": "#10b981",     # emerald
+}
+
+LAYER_COLORS = [
+    "#3b82f6", "#ef4444", "#22d3ee", "#f59e0b", "#8b5cf6",
+    "#10b981", "#f97316", "#6366f1", "#ec4899", "#14b8a6",
+]
+
+# ---------------------------------------------------------------------------
+# In-memory store
+# ---------------------------------------------------------------------------
+
+_layers: list[dict[str, Any]] = []
+_pins: list[dict[str, Any]] = []
+_lock = threading.Lock()
+
+# Persistence file path
+_PERSIST_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data")
+_PERSIST_FILE = os.path.join(_PERSIST_DIR, "pin_layers.json")
+_OLD_PERSIST_FILE = os.path.join(_PERSIST_DIR, "ai_pins.json")
+
+
+def _ensure_persist_dir():
+    try:
+        os.makedirs(_PERSIST_DIR, exist_ok=True)
+    except OSError:
+        pass
+
+
+def _save_to_disk():
+    """Persist layers and pins to JSON file. Called under lock."""
+    try:
+        _ensure_persist_dir()
+        with open(_PERSIST_FILE, "w", encoding="utf-8") as f:
+            json.dump({"layers": _layers, "pins": _pins}, f, indent=2, default=str)
+    except (OSError, IOError) as e:
+        logger.warning(f"Failed to persist pin layers: {e}")
+
+
+def _load_from_disk():
+    """Load layers and pins from disk on startup."""
+    global _layers, _pins
+    try:
+        if os.path.exists(_PERSIST_FILE):
+            with open(_PERSIST_FILE, "r", encoding="utf-8") as f:
+                data = json.load(f)
+            if isinstance(data, dict):
+                _layers = data.get("layers", [])
+                _pins = data.get("pins", [])
+                logger.info(f"Loaded {len(_layers)} layers, {len(_pins)} pins from disk")
+                return
+
+        # Migrate from old flat pin file
+        if os.path.exists(_OLD_PERSIST_FILE):
+            with open(_OLD_PERSIST_FILE, "r", encoding="utf-8") as f:
+                old_pins = json.load(f)
+            if isinstance(old_pins, list) and old_pins:
+                legacy_layer = _make_layer("Legacy", "Migrated pins", source="system")
+                _layers.append(legacy_layer)
+                for p in old_pins:
+                    if isinstance(p, dict):
+                        p["layer_id"] = legacy_layer["id"]
+                        _pins.append(p)
+                logger.info(f"Migrated {len(_pins)} pins from ai_pins.json into Legacy layer")
+                _save_to_disk()
+    except (OSError, IOError, json.JSONDecodeError) as e:
+        logger.warning(f"Failed to load pin layers from disk: {e}")
+
+
+def _make_layer(
+    name: str,
+    description: str = "",
+    source: str = "user",
+    color: str = "",
+    feed_url: str = "",
+    feed_interval: int = 300,
+) -> dict[str, Any]:
+    """Create a layer dict."""
+    layer_id = str(uuid.uuid4())[:12]
+    now = time.time()
+    return {
+        "id": layer_id,
+        "name": name[:100],
+        "description": description[:500],
+        "source": source[:50],
+        "visible": True,
+        "color": color or LAYER_COLORS[len(_layers) % len(LAYER_COLORS)],
+        "created_at": now,
+        "created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
+        "feed_url": feed_url[:1000] if feed_url else "",
+        "feed_interval": max(60, min(86400, feed_interval)),
+        "pin_count": 0,
+    }
+
+
+# Load on import
+_load_from_disk()
+
+# One-time cleanup: remove correlation_engine auto-pins (no longer generated)
+_corr_before = len(_pins)
+_pins[:] = [p for p in _pins if p.get("source") != "correlation_engine"]
+if len(_pins) < _corr_before:
+    logger.info("Cleaned up %d legacy correlation_engine pins", _corr_before - len(_pins))
+    _save_to_disk()
+
+
+# ---------------------------------------------------------------------------
+# Layer CRUD
+# ---------------------------------------------------------------------------
+
+def create_layer(
+    name: str,
+    description: str = "",
+    source: str = "user",
+    color: str = "",
+    feed_url: str = "",
+    feed_interval: int = 300,
+) -> dict[str, Any]:
+    """Create a new pin layer."""
+    with _lock:
+        layer = _make_layer(name, description, source, color, feed_url, feed_interval)
+        _layers.append(layer)
+        _save_to_disk()
+    return layer
+
+
+def get_layers() -> list[dict[str, Any]]:
+    """Return all layers with current pin counts."""
+    now = time.time()
+    with _lock:
+        result = []
+        for layer in _layers:
+            count = sum(
+                1 for p in _pins
+                if p.get("layer_id") == layer["id"]
+                and not (p.get("expires_at") and p["expires_at"] < now)
+            )
+            result.append({**layer, "pin_count": count})
+        return result
+
+
+def update_layer(layer_id: str, **updates) -> Optional[dict[str, Any]]:
+    """Update layer fields. Returns updated layer or None if not found."""
+    allowed = {"name", "description", "visible", "color", "feed_url", "feed_interval", "feed_last_fetched"}
+    with _lock:
+        for layer in _layers:
+            if layer["id"] == layer_id:
+                for k, v in updates.items():
+                    if k in allowed and v is not None:
+                        if k == "name":
+                            layer[k] = str(v)[:100]
+                        elif k == "description":
+                            layer[k] = str(v)[:500]
+                        elif k == "visible":
+                            layer[k] = bool(v)
+                        elif k == "color":
+                            layer[k] = str(v)[:20]
+                        elif k == "feed_url":
+                            layer[k] = str(v)[:1000]
+                        elif k == "feed_interval":
+                            layer[k] = max(60, min(86400, int(v)))
+                        elif k == "feed_last_fetched":
+                            layer[k] = float(v)
+                _save_to_disk()
+                return dict(layer)
+    return None
+
+
+def delete_layer(layer_id: str) -> int:
+    """Delete a layer and all its pins. Returns count of pins removed."""
+    with _lock:
+        before_layers = len(_layers)
+        _layers[:] = [l for l in _layers if l["id"] != layer_id]
+        if len(_layers) == before_layers:
+            return 0  # not found
+        before_pins = len(_pins)
+        _pins[:] = [p for p in _pins if p.get("layer_id") != layer_id]
+        removed = before_pins - len(_pins)
+        _save_to_disk()
+        return removed
+
+
+# ---------------------------------------------------------------------------
+# Pin CRUD
+# ---------------------------------------------------------------------------
+
+def create_pin(
+    lat: float,
+    lng: float,
+    label: str,
+    category: str = "custom",
+    *,
+    layer_id: str = "",
+    color: str = "",
+    description: str = "",
+    source: str = "openclaw",
+    source_url: str = "",
+    confidence: float = 1.0,
+    ttl_hours: float = 0,
+    metadata: Optional[dict] = None,
+    entity_attachment: Optional[dict] = None,
+) -> dict[str, Any]:
+    """Create a single pin and return it."""
+    pin_id = str(uuid.uuid4())[:12]
+    now = time.time()
+
+    cat = category if category in PIN_CATEGORIES else "custom"
+    pin_color = color or PIN_COLORS.get(cat, "#3b82f6")
+
+    # Validate entity_attachment if provided
+    attachment = None
+    if entity_attachment and isinstance(entity_attachment, dict):
+        etype = str(entity_attachment.get("entity_type", "")).strip()
+        eid = str(entity_attachment.get("entity_id", "")).strip()
+        if etype and eid:
+            attachment = {
+                "entity_type": etype[:50],
+                "entity_id": eid[:100],
+                "entity_label": str(entity_attachment.get("entity_label", ""))[:200],
+            }
+
+    pin = {
+        "id": pin_id,
+        "layer_id": layer_id or "",
+        "lat": lat,
+        "lng": lng,
+        "label": label[:200],
+        "category": cat,
+        "color": pin_color,
+        "description": description[:2000],
+        "source": source[:100],
+        "source_url": source_url[:500],
+        "confidence": max(0.0, min(1.0, confidence)),
+        "created_at": now,
+        "created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
+        "expires_at": now + (ttl_hours * 3600) if ttl_hours > 0 else None,
+        "metadata": metadata or {},
+        "entity_attachment": attachment,
+        "comments": [],
+    }
+
+    with _lock:
+        _pins.append(pin)
+        _save_to_disk()
+
+    return pin
+
+
+def create_pins_batch(items: list[dict], default_layer_id: str = "") -> list[dict[str, Any]]:
+    """Create multiple pins at once."""
+    created = []
+    now = time.time()
+
+    with _lock:
+        for item in items[:200]:  # max 200 per batch
+            pin_id = str(uuid.uuid4())[:12]
+            cat = item.get("category", "custom")
+            if cat not in PIN_CATEGORIES:
+                cat = "custom"
+            pin_color = item.get("color", "") or PIN_COLORS.get(cat, "#3b82f6")
+            ttl = float(item.get("ttl_hours", 0) or 0)
+
+            attachment = None
+            ea = item.get("entity_attachment")
+            if ea and isinstance(ea, dict):
+                etype = str(ea.get("entity_type", "")).strip()
+                eid = str(ea.get("entity_id", "")).strip()
+                if etype and eid:
+                    attachment = {
+                        "entity_type": etype[:50],
+                        "entity_id": eid[:100],
+                        "entity_label": str(ea.get("entity_label", ""))[:200],
+                    }
+
+            pin = {
+                "id": pin_id,
+                "layer_id": item.get("layer_id", default_layer_id) or "",
+                "lat": float(item.get("lat", 0)),
+                "lng": float(item.get("lng", 0)),
+                "label": str(item.get("label", ""))[:200],
+                "category": cat,
+                "color": pin_color,
+                "description": str(item.get("description", ""))[:2000],
+                "source": str(item.get("source", "openclaw"))[:100],
+                "source_url": str(item.get("source_url", ""))[:500],
+                "confidence": max(0.0, min(1.0, float(item.get("confidence", 1.0)))),
+                "created_at": now,
+                "created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
+                "expires_at": now + (ttl * 3600) if ttl > 0 else None,
+                "metadata": item.get("metadata", {}),
+                "entity_attachment": attachment,
+                "comments": [],
+            }
+            _pins.append(pin)
+            created.append(pin)
+
+        _save_to_disk()
+    return created
+
+
+def get_pins(
+    category: str = "",
+    source: str = "",
+    layer_id: str = "",
+    limit: int = 500,
+    include_expired: bool = False,
+) -> list[dict[str, Any]]:
+    """Get pins with optional filters."""
+    now = time.time()
+    with _lock:
+        results = []
+        for pin in _pins:
+            if not include_expired and pin.get("expires_at") and pin["expires_at"] < now:
+                continue
+            if category and pin.get("category") != category:
+                continue
+            if source and pin.get("source") != source:
+                continue
+            if layer_id and pin.get("layer_id") != layer_id:
+                continue
+            results.append(pin)
+            if len(results) >= limit:
+                break
+        return results
+
+
+def get_pin(pin_id: str) -> Optional[dict[str, Any]]:
+    """Return a single pin by ID (including comments), or None."""
+    with _lock:
+        for pin in _pins:
+            if pin.get("id") == pin_id:
+                # Ensure comments key exists for legacy pins
+                if "comments" not in pin:
+                    pin["comments"] = []
+                return dict(pin)
+    return None
+
+
+def update_pin(pin_id: str, **updates) -> Optional[dict[str, Any]]:
+    """Update a pin's editable fields (label, description, category, color)."""
+    allowed = {"label", "description", "category", "color"}
+    with _lock:
+        for pin in _pins:
+            if pin.get("id") != pin_id:
+                continue
+            for k, v in updates.items():
+                if k not in allowed or v is None:
+                    continue
+                if k == "label":
+                    pin[k] = str(v)[:200]
+                elif k == "description":
+                    pin[k] = str(v)[:2000]
+                elif k == "category":
+                    cat = str(v)
+                    if cat in PIN_CATEGORIES:
+                        pin[k] = cat
+                        # Refresh color if it was the category default
+                        if not updates.get("color"):
+                            pin["color"] = PIN_COLORS.get(cat, pin.get("color", "#3b82f6"))
+                elif k == "color":
+                    pin[k] = str(v)[:20]
+            pin["updated_at"] = time.time()
+            _save_to_disk()
+            return dict(pin)
+    return None
+
+
+def add_pin_comment(
+    pin_id: str,
+    text: str,
+    author: str = "user",
+    author_label: str = "",
+    reply_to: str = "",
+) -> Optional[dict[str, Any]]:
+    """Append a comment to a pin. Returns the updated pin (with all comments)."""
+    text = (text or "").strip()
+    if not text:
+        return None
+    with _lock:
+        for pin in _pins:
+            if pin.get("id") != pin_id:
+                continue
+            if "comments" not in pin or not isinstance(pin["comments"], list):
+                pin["comments"] = []
+            comment = {
+                "id": str(uuid.uuid4())[:12],
+                "text": text[:4000],
+                "author": (author or "user")[:50],
+                "author_label": (author_label or "")[:100],
+                "reply_to": (reply_to or "")[:12],
+                "created_at": time.time(),
+                "created_at_iso": datetime.utcnow().isoformat() + "Z",
+            }
+            pin["comments"].append(comment)
+            _save_to_disk()
+            return dict(pin)
+    return None
+
+
+def delete_pin_comment(pin_id: str, comment_id: str) -> bool:
+    """Remove a single comment from a pin."""
+    with _lock:
+        for pin in _pins:
+            if pin.get("id") != pin_id:
+                continue
+            comments = pin.get("comments") or []
+            before = len(comments)
+            pin["comments"] = [c for c in comments if c.get("id") != comment_id]
+            if len(pin["comments"]) < before:
+                _save_to_disk()
+                return True
+            return False
+    return False
+
+
+def delete_pin(pin_id: str) -> bool:
+    """Delete a single pin by ID."""
+    with _lock:
+        before = len(_pins)
+        _pins[:] = [p for p in _pins if p.get("id") != pin_id]
+        if len(_pins) < before:
+            _save_to_disk()
+            return True
+        return False
+
+
+def clear_pins(category: str = "", source: str = "", layer_id: str = "") -> int:
+    """Clear pins, optionally filtered. Returns count removed."""
+    with _lock:
+        before = len(_pins)
+
+        def keep(p):
+            if layer_id and p.get("layer_id") != layer_id:
+                return True  # different layer, keep
+            if category and source:
+                return not (p.get("category") == category and p.get("source") == source)
+            if category:
+                return p.get("category") != category
+            if source:
+                return p.get("source") != source
+            if layer_id:
+                return p.get("layer_id") != layer_id
+            return False
+
+        if not category and not source and not layer_id:
+            _pins.clear()
+        else:
+            _pins[:] = [p for p in _pins if keep(p)]
+
+        removed = before - len(_pins)
+        if removed:
+            _save_to_disk()
+        return removed
+
+
+def get_feed_layers() -> list[dict[str, Any]]:
+    """Return layers that have a non-empty feed_url."""
+    with _lock:
+        return [dict(l) for l in _layers if l.get("feed_url")]
+
+
+def replace_layer_pins(layer_id: str, new_pins: list[dict[str, Any]]) -> int:
+    """Atomically replace all pins in a layer with new_pins. Returns count added."""
+    now = time.time()
+    with _lock:
+        # Remove old pins for this layer
+        _pins[:] = [p for p in _pins if p.get("layer_id") != layer_id]
+        # Add new pins
+        added = 0
+        for item in new_pins[:500]:  # cap at 500 per feed
+            pin_id = str(uuid.uuid4())[:12]
+            cat = item.get("category", "custom")
+            if cat not in PIN_CATEGORIES:
+                cat = "custom"
+            pin_color = item.get("color", "") or PIN_COLORS.get(cat, "#3b82f6")
+
+            attachment = None
+            ea = item.get("entity_attachment")
+            if ea and isinstance(ea, dict):
+                etype = str(ea.get("entity_type", "")).strip()
+                eid = str(ea.get("entity_id", "")).strip()
+                if etype and eid:
+                    attachment = {
+                        "entity_type": etype[:50],
+                        "entity_id": eid[:100],
+                        "entity_label": str(ea.get("entity_label", ""))[:200],
+                    }
+
+            pin = {
+                "id": pin_id,
+                "layer_id": layer_id,
+                "lat": float(item.get("lat", 0)),
+                "lng": float(item.get("lng", 0)),
+                "label": str(item.get("label", item.get("name", "")))[:200],
+                "category": cat,
+                "color": pin_color,
+                "description": str(item.get("description", ""))[:2000],
+                "source": str(item.get("source", "feed"))[:100],
+                "source_url": str(item.get("source_url", ""))[:500],
+                "confidence": max(0.0, min(1.0, float(item.get("confidence", 1.0)))),
+                "created_at": now,
+                "created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
+                "expires_at": None,
+                "metadata": item.get("metadata", {}),
+                "entity_attachment": attachment,
+                "comments": [],
+            }
+            _pins.append(pin)
+            added += 1
+        _save_to_disk()
+    return added
+
+
+def purge_expired() -> int:
+    """Remove expired pins. Called periodically."""
+    now = time.time()
+    with _lock:
+        before = len(_pins)
+        _pins[:] = [p for p in _pins if not (p.get("expires_at") and p["expires_at"] < now)]
+        removed = before - len(_pins)
+        if removed:
+            _save_to_disk()
+        return removed
+
+
+def pin_count() -> dict[str, int]:
+    """Return counts by category."""
+    now = time.time()
+    counts: dict[str, int] = {}
+    with _lock:
+        for pin in _pins:
+            if pin.get("expires_at") and pin["expires_at"] < now:
+                continue
+            cat = pin.get("category", "custom")
+            counts[cat] = counts.get(cat, 0) + 1
+    return counts
+
+
+def pins_as_geojson(layer_id: str = "") -> dict[str, Any]:
+    """Convert active pins to GeoJSON FeatureCollection for the map layer."""
+    now = time.time()
+    features = []
+    with _lock:
+        # Build set of visible layer IDs
+        visible_layers = {l["id"] for l in _layers if l.get("visible", True)}
+
+        for pin in _pins:
+            if pin.get("expires_at") and pin["expires_at"] < now:
+                continue
+            # Layer filter
+            pid_layer = pin.get("layer_id", "")
+            if layer_id and pid_layer != layer_id:
+                continue
+            # Skip pins in hidden layers
+            if pid_layer and pid_layer not in visible_layers:
+                continue
+
+            props = {
+                "id": pin["id"],
+                "layer_id": pid_layer,
+                "label": pin["label"],
+                "category": pin["category"],
+                "color": pin["color"],
+                "description": pin.get("description", ""),
+                "source": pin["source"],
+                "source_url": pin.get("source_url", ""),
+                "confidence": pin.get("confidence", 1.0),
+                "created_at": pin.get("created_at_iso", ""),
+                "comment_count": len(pin.get("comments") or []),
+            }
+
+            # Entity attachment info (frontend resolves position)
+            ea = pin.get("entity_attachment")
+            if ea:
+                props["entity_attachment"] = ea
+
+            features.append({
+                "type": "Feature",
+                "geometry": {
+                    "type": "Point",
+                    "coordinates": [pin["lng"], pin["lat"]],
+                },
+                "properties": props,
+            })
+    return {
+        "type": "FeatureCollection",
+        "features": features,
+    }

backend/services/ais_stream.py

@@ -16,18 +16,31 @@ logger = logging.getLogger(__name__)
 AIS_WS_URL = "wss://stream.aisstream.io/v0/stream"
 API_KEY = os.environ.get("AIS_API_KEY", "")
 
+
+def _env_truthy(name: str) -> bool:
+    return str(os.getenv(name, "")).strip().lower() in {"1", "true", "yes", "on"}
+
+
+def ais_stream_proxy_enabled() -> bool:
+    """Return whether the external Node AIS proxy may be started."""
+    setting = str(os.getenv("SHADOWBROKER_ENABLE_AIS_STREAM_PROXY", "")).strip().lower()
+    if setting:
+        return _env_truthy("SHADOWBROKER_ENABLE_AIS_STREAM_PROXY")
+    return True
+
+
 # AIS vessel type code classification
 # See: https://coast.noaa.gov/data/marinecadastre/ais/VesselTypeCodes2018.pdf
 def classify_vessel(ais_type: int, mmsi: int) -> str:
     """Classify a vessel by its AIS type code into a rendering category."""
     if 80 <= ais_type <= 89:
-        return "tanker"        # Oil/Chemical/Gas tankers → RED
+        return "tanker"  # Oil/Chemical/Gas tankers → RED
     if 70 <= ais_type <= 79:
-        return "cargo"         # Cargo ships, container vessels → RED
+        return "cargo"  # Cargo ships, container vessels → RED
     if 60 <= ais_type <= 69:
-        return "passenger"     # Cruise ships, ferries → GRAY
+        return "passenger"  # Cruise ships, ferries → GRAY
     if ais_type in (36, 37):
-        return "yacht"         # Sailing/Pleasure craft → DARK BLUE
+        return "yacht"  # Sailing/Pleasure craft → DARK BLUE
     if ais_type == 35:
         return "military_vessel"  # Military → YELLOW
     # MMSI-based military detection: military MMSIs often start with certain prefixes
@@ -35,87 +48,286 @@ def classify_vessel(ais_type: int, mmsi: int) -> str:
     if mmsi_str.startswith("3380") or mmsi_str.startswith("3381"):
         return "military_vessel"  # US Navy
     if ais_type in (30, 31, 32, 33, 34):
-        return "other"         # Fishing, towing, dredging, diving, etc.
+        return "other"  # Fishing, towing, dredging, diving, etc.
     if ais_type in (50, 51, 52, 53, 54, 55, 56, 57, 58, 59):
-        return "other"         # Pilot, SAR, tug, port tender, etc.
-    return "unknown"            # Not yet classified — will update when ShipStaticData arrives
+        return "other"  # Pilot, SAR, tug, port tender, etc.
+    return "unknown"  # Not yet classified — will update when ShipStaticData arrives
 
 
 # MMSI Maritime Identification Digit (MID) → Country mapping
 # First 3 digits of MMSI (for 9-digit MMSIs) encode the flag state
 MID_COUNTRY = {
-    201: "Albania", 202: "Andorra", 203: "Austria", 204: "Portugal", 205: "Belgium",
-    206: "Belarus", 207: "Bulgaria", 208: "Vatican", 209: "Cyprus", 210: "Cyprus",
-    211: "Germany", 212: "Cyprus", 213: "Georgia", 214: "Moldova", 215: "Malta",
-    216: "Armenia", 218: "Germany", 219: "Denmark", 220: "Denmark", 224: "Spain",
-    225: "Spain", 226: "France", 227: "France", 228: "France", 229: "Malta",
-    230: "Finland", 231: "Faroe Islands", 232: "United Kingdom", 233: "United Kingdom",
-    234: "United Kingdom", 235: "United Kingdom", 236: "Gibraltar", 237: "Greece",
-    238: "Croatia", 239: "Greece", 240: "Greece", 241: "Greece", 242: "Morocco",
-    243: "Hungary", 244: "Netherlands", 245: "Netherlands", 246: "Netherlands",
-    247: "Italy", 248: "Malta", 249: "Malta", 250: "Ireland", 251: "Iceland",
-    252: "Liechtenstein", 253: "Luxembourg", 254: "Monaco", 255: "Portugal",
-    256: "Malta", 257: "Norway", 258: "Norway", 259: "Norway", 261: "Poland",
-    263: "Portugal", 264: "Romania", 265: "Sweden", 266: "Sweden", 267: "Slovakia",
-    268: "San Marino", 269: "Switzerland", 270: "Czech Republic", 271: "Turkey",
-    272: "Ukraine", 273: "Russia", 274: "North Macedonia", 275: "Latvia",
-    276: "Estonia", 277: "Lithuania", 278: "Slovenia",
-    301: "Anguilla", 303: "Alaska", 304: "Antigua", 305: "Antigua",
-    306: "Netherlands Antilles", 307: "Aruba", 308: "Bahamas", 309: "Bahamas",
-    310: "Bermuda", 311: "Bahamas", 312: "Belize", 314: "Barbados", 316: "Canada",
-    319: "Cayman Islands", 321: "Costa Rica", 323: "Cuba", 325: "Dominica",
-    327: "Dominican Republic", 329: "Guadeloupe", 330: "Grenada", 331: "Greenland",
-    332: "Guatemala", 334: "Honduras", 336: "Haiti", 338: "United States",
-    339: "Jamaica", 341: "Saint Kitts", 343: "Saint Lucia", 345: "Mexico",
-    347: "Martinique", 348: "Montserrat", 350: "Nicaragua", 351: "Panama",
-    352: "Panama", 353: "Panama", 354: "Panama", 355: "Panama",
-    356: "Panama", 357: "Panama", 358: "Puerto Rico", 359: "El Salvador",
-    361: "Saint Pierre", 362: "Trinidad", 364: "Turks and Caicos",
-    366: "United States", 367: "United States", 368: "United States", 369: "United States",
-    370: "Panama", 371: "Panama", 372: "Panama", 373: "Panama",
-    374: "Panama", 375: "Saint Vincent", 376: "Saint Vincent", 377: "Saint Vincent",
-    378: "British Virgin Islands", 379: "US Virgin Islands",
-    401: "Afghanistan", 403: "Saudi Arabia", 405: "Bangladesh", 408: "Bahrain",
-    410: "Bhutan", 412: "China", 413: "China", 414: "China",
-    416: "Taiwan", 417: "Sri Lanka", 419: "India", 422: "Iran",
-    423: "Azerbaijan", 425: "Iraq", 428: "Israel", 431: "Japan",
-    432: "Japan", 434: "Turkmenistan", 436: "Kazakhstan", 437: "Uzbekistan",
-    438: "Jordan", 440: "South Korea", 441: "South Korea", 443: "Palestine",
-    445: "North Korea", 447: "Kuwait", 450: "Lebanon", 451: "Kyrgyzstan",
-    453: "Macao", 455: "Maldives", 457: "Mongolia", 459: "Nepal",
-    461: "Oman", 463: "Pakistan", 466: "Qatar", 468: "Syria",
-    470: "UAE", 472: "Tajikistan", 473: "Yemen", 475: "Tonga",
-    477: "Hong Kong", 478: "Bosnia",
-    501: "Antarctica", 503: "Australia", 506: "Myanmar",
-    508: "Brunei", 510: "Micronesia", 511: "Palau", 512: "New Zealand",
-    514: "Cambodia", 515: "Cambodia", 516: "Christmas Island",
-    518: "Cook Islands", 520: "Fiji", 523: "Cocos Islands",
-    525: "Indonesia", 529: "Kiribati", 531: "Laos", 533: "Malaysia",
-    536: "Northern Mariana Islands", 538: "Marshall Islands",
-    540: "New Caledonia", 542: "Niue", 544: "Nauru", 546: "French Polynesia",
-    548: "Philippines", 553: "Papua New Guinea", 555: "Pitcairn",
-    557: "Solomon Islands", 559: "American Samoa", 561: "Samoa",
-    563: "Singapore", 564: "Singapore", 565: "Singapore", 566: "Singapore",
-    567: "Thailand", 570: "Tonga", 572: "Tuvalu", 574: "Vietnam",
-    576: "Vanuatu", 577: "Vanuatu", 578: "Wallis and Futuna",
-    601: "South Africa", 603: "Angola", 605: "Algeria", 607: "Benin",
-    609: "Botswana", 610: "Burundi", 611: "Cameroon", 612: "Cape Verde",
-    613: "Central African Republic", 615: "Congo", 616: "Comoros",
-    617: "DR Congo", 618: "Ivory Coast", 619: "Djibouti",
-    620: "Egypt", 621: "Equatorial Guinea", 622: "Ethiopia",
-    624: "Eritrea", 625: "Gabon", 626: "Gambia", 627: "Ghana",
-    629: "Guinea", 630: "Guinea-Bissau", 631: "Kenya", 632: "Lesotho",
-    633: "Liberia", 634: "Liberia", 635: "Liberia", 636: "Liberia",
-    637: "Libya", 642: "Madagascar", 644: "Malawi", 645: "Mali",
-    647: "Mauritania", 649: "Mauritius", 650: "Mozambique",
-    654: "Namibia", 655: "Niger", 656: "Nigeria", 657: "Guinea",
-    659: "Rwanda", 660: "Senegal", 661: "Sierra Leone",
-    662: "Somalia", 663: "South Africa", 664: "Sudan",
-    667: "Tanzania", 668: "Togo", 669: "Tunisia", 670: "Uganda",
-    671: "Egypt", 672: "Tanzania", 674: "Zambia", 675: "Zimbabwe",
-    676: "Comoros", 677: "Tanzania",
+    201: "Albania",
+    202: "Andorra",
+    203: "Austria",
+    204: "Portugal",
+    205: "Belgium",
+    206: "Belarus",
+    207: "Bulgaria",
+    208: "Vatican",
+    209: "Cyprus",
+    210: "Cyprus",
+    211: "Germany",
+    212: "Cyprus",
+    213: "Georgia",
+    214: "Moldova",
+    215: "Malta",
+    216: "Armenia",
+    218: "Germany",
+    219: "Denmark",
+    220: "Denmark",
+    224: "Spain",
+    225: "Spain",
+    226: "France",
+    227: "France",
+    228: "France",
+    229: "Malta",
+    230: "Finland",
+    231: "Faroe Islands",
+    232: "United Kingdom",
+    233: "United Kingdom",
+    234: "United Kingdom",
+    235: "United Kingdom",
+    236: "Gibraltar",
+    237: "Greece",
+    238: "Croatia",
+    239: "Greece",
+    240: "Greece",
+    241: "Greece",
+    242: "Morocco",
+    243: "Hungary",
+    244: "Netherlands",
+    245: "Netherlands",
+    246: "Netherlands",
+    247: "Italy",
+    248: "Malta",
+    249: "Malta",
+    250: "Ireland",
+    251: "Iceland",
+    252: "Liechtenstein",
+    253: "Luxembourg",
+    254: "Monaco",
+    255: "Portugal",
+    256: "Malta",
+    257: "Norway",
+    258: "Norway",
+    259: "Norway",
+    261: "Poland",
+    263: "Portugal",
+    264: "Romania",
+    265: "Sweden",
+    266: "Sweden",
+    267: "Slovakia",
+    268: "San Marino",
+    269: "Switzerland",
+    270: "Czech Republic",
+    271: "Turkey",
+    272: "Ukraine",
+    273: "Russia",
+    274: "North Macedonia",
+    275: "Latvia",
+    276: "Estonia",
+    277: "Lithuania",
+    278: "Slovenia",
+    301: "Anguilla",
+    303: "Alaska",
+    304: "Antigua",
+    305: "Antigua",
+    306: "Netherlands Antilles",
+    307: "Aruba",
+    308: "Bahamas",
+    309: "Bahamas",
+    310: "Bermuda",
+    311: "Bahamas",
+    312: "Belize",
+    314: "Barbados",
+    316: "Canada",
+    319: "Cayman Islands",
+    321: "Costa Rica",
+    323: "Cuba",
+    325: "Dominica",
+    327: "Dominican Republic",
+    329: "Guadeloupe",
+    330: "Grenada",
+    331: "Greenland",
+    332: "Guatemala",
+    334: "Honduras",
+    336: "Haiti",
+    338: "United States",
+    339: "Jamaica",
+    341: "Saint Kitts",
+    343: "Saint Lucia",
+    345: "Mexico",
+    347: "Martinique",
+    348: "Montserrat",
+    350: "Nicaragua",
+    351: "Panama",
+    352: "Panama",
+    353: "Panama",
+    354: "Panama",
+    355: "Panama",
+    356: "Panama",
+    357: "Panama",
+    358: "Puerto Rico",
+    359: "El Salvador",
+    361: "Saint Pierre",
+    362: "Trinidad",
+    364: "Turks and Caicos",
+    366: "United States",
+    367: "United States",
+    368: "United States",
+    369: "United States",
+    370: "Panama",
+    371: "Panama",
+    372: "Panama",
+    373: "Panama",
+    374: "Panama",
+    375: "Saint Vincent",
+    376: "Saint Vincent",
+    377: "Saint Vincent",
+    378: "British Virgin Islands",
+    379: "US Virgin Islands",
+    401: "Afghanistan",
+    403: "Saudi Arabia",
+    405: "Bangladesh",
+    408: "Bahrain",
+    410: "Bhutan",
+    412: "China",
+    413: "China",
+    414: "China",
+    416: "Taiwan",
+    417: "Sri Lanka",
+    419: "India",
+    422: "Iran",
+    423: "Azerbaijan",
+    425: "Iraq",
+    428: "Israel",
+    431: "Japan",
+    432: "Japan",
+    434: "Turkmenistan",
+    436: "Kazakhstan",
+    437: "Uzbekistan",
+    438: "Jordan",
+    440: "South Korea",
+    441: "South Korea",
+    443: "Palestine",
+    445: "North Korea",
+    447: "Kuwait",
+    450: "Lebanon",
+    451: "Kyrgyzstan",
+    453: "Macao",
+    455: "Maldives",
+    457: "Mongolia",
+    459: "Nepal",
+    461: "Oman",
+    463: "Pakistan",
+    466: "Qatar",
+    468: "Syria",
+    470: "UAE",
+    472: "Tajikistan",
+    473: "Yemen",
+    475: "Tonga",
+    477: "Hong Kong",
+    478: "Bosnia",
+    501: "Antarctica",
+    503: "Australia",
+    506: "Myanmar",
+    508: "Brunei",
+    510: "Micronesia",
+    511: "Palau",
+    512: "New Zealand",
+    514: "Cambodia",
+    515: "Cambodia",
+    516: "Christmas Island",
+    518: "Cook Islands",
+    520: "Fiji",
+    523: "Cocos Islands",
+    525: "Indonesia",
+    529: "Kiribati",
+    531: "Laos",
+    533: "Malaysia",
+    536: "Northern Mariana Islands",
+    538: "Marshall Islands",
+    540: "New Caledonia",
+    542: "Niue",
+    544: "Nauru",
+    546: "French Polynesia",
+    548: "Philippines",
+    553: "Papua New Guinea",
+    555: "Pitcairn",
+    557: "Solomon Islands",
+    559: "American Samoa",
+    561: "Samoa",
+    563: "Singapore",
+    564: "Singapore",
+    565: "Singapore",
+    566: "Singapore",
+    567: "Thailand",
+    570: "Tonga",
+    572: "Tuvalu",
+    574: "Vietnam",
+    576: "Vanuatu",
+    577: "Vanuatu",
+    578: "Wallis and Futuna",
+    601: "South Africa",
+    603: "Angola",
+    605: "Algeria",
+    607: "Benin",
+    609: "Botswana",
+    610: "Burundi",
+    611: "Cameroon",
+    612: "Cape Verde",
+    613: "Central African Republic",
+    615: "Congo",
+    616: "Comoros",
+    617: "DR Congo",
+    618: "Ivory Coast",
+    619: "Djibouti",
+    620: "Egypt",
+    621: "Equatorial Guinea",
+    622: "Ethiopia",
+    624: "Eritrea",
+    625: "Gabon",
+    626: "Gambia",
+    627: "Ghana",
+    629: "Guinea",
+    630: "Guinea-Bissau",
+    631: "Kenya",
+    632: "Lesotho",
+    633: "Liberia",
+    634: "Liberia",
+    635: "Liberia",
+    636: "Liberia",
+    637: "Libya",
+    642: "Madagascar",
+    644: "Malawi",
+    645: "Mali",
+    647: "Mauritania",
+    649: "Mauritius",
+    650: "Mozambique",
+    654: "Namibia",
+    655: "Niger",
+    656: "Nigeria",
+    657: "Guinea",
+    659: "Rwanda",
+    660: "Senegal",
+    661: "Sierra Leone",
+    662: "Somalia",
+    663: "South Africa",
+    664: "Sudan",
+    667: "Tanzania",
+    668: "Togo",
+    669: "Tunisia",
+    670: "Uganda",
+    671: "Egypt",
+    672: "Tanzania",
+    674: "Zambia",
+    675: "Zimbabwe",
+    676: "Comoros",
+    677: "Tanzania",
 }
 
+
 def get_country_from_mmsi(mmsi: int) -> str:
     """Look up flag state from MMSI Maritime Identification Digit."""
     mmsi_str = str(mmsi)
@@ -127,24 +339,71 @@ def get_country_from_mmsi(mmsi: int) -> str:
 
 # Global vessel store: MMSI → vessel dict
 _vessels: dict[int, dict] = {}
+_vessel_trails: dict[int, dict] = {}
 _vessels_lock = threading.Lock()
 _ws_thread: threading.Thread | None = None
 _ws_running = False
+_proxy_process = None
+_VESSEL_TRAIL_INTERVAL_S = 120
+_VESSEL_TRAIL_MAX_POINTS = 240
 
 import os
+
 CACHE_FILE = os.path.join(os.path.dirname(__file__), "ais_cache.json")
 
 
+def _record_vessel_trail_locked(mmsi: int, lat, lng, sog=0, now_ts: float | None = None) -> None:
+    """Append a sampled AIS trail point. Caller must hold _vessels_lock."""
+    if lat is None or lng is None:
+        return
+    try:
+        lat_f = float(lat)
+        lng_f = float(lng)
+    except (TypeError, ValueError):
+        return
+    if abs(lat_f) > 90 or abs(lng_f) > 180 or (lat_f == 0 and lng_f == 0):
+        return
+    now = now_ts or time.time()
+    trail_data = _vessel_trails.setdefault(int(mmsi), {"points": [], "last_seen": now})
+    point = [round(lat_f, 5), round(lng_f, 5), round(float(sog or 0), 1), round(now)]
+    last_point_ts = trail_data["points"][-1][3] if trail_data["points"] else 0
+    if now - last_point_ts < _VESSEL_TRAIL_INTERVAL_S:
+        trail_data["last_seen"] = now
+        return
+    if (
+        trail_data["points"]
+        and trail_data["points"][-1][0] == point[0]
+        and trail_data["points"][-1][1] == point[1]
+    ):
+        trail_data["last_seen"] = now
+        return
+    trail_data["points"].append(point)
+    trail_data["last_seen"] = now
+    if len(trail_data["points"]) > _VESSEL_TRAIL_MAX_POINTS:
+        trail_data["points"] = trail_data["points"][-_VESSEL_TRAIL_MAX_POINTS:]
+
+
+def get_vessel_trail(mmsi: int) -> list:
+    """Return the accumulated trail for a single vessel without expanding live payloads."""
+    try:
+        key = int(mmsi)
+    except (TypeError, ValueError):
+        return []
+    with _vessels_lock:
+        points = _vessel_trails.get(key, {}).get("points", [])
+        return [list(point) for point in points]
+
+
 def _save_cache():
     """Save vessel data to disk for persistence across restarts."""
     try:
         with _vessels_lock:
             # Convert int keys to strings for JSON
             data = {str(k): v for k, v in _vessels.items()}
-        with open(CACHE_FILE, 'w') as f:
+        with open(CACHE_FILE, "w") as f:
             json.dump(data, f)
         logger.info(f"AIS cache saved: {len(data)} vessels")
-    except Exception as e:
+    except (IOError, OSError) as e:
         logger.error(f"Failed to save AIS cache: {e}")
 
 
@@ -154,7 +413,7 @@ def _load_cache():
     if not os.path.exists(CACHE_FILE):
         return
     try:
-        with open(CACHE_FILE, 'r') as f:
+        with open(CACHE_FILE, "r") as f:
             data = json.load(f)
         now = time.time()
         stale_cutoff = now - 3600  # Accept vessels up to 1 hour old on restart
@@ -165,82 +424,173 @@ def _load_cache():
                     _vessels[int(k)] = v
                     loaded += 1
         logger.info(f"AIS cache loaded: {loaded} vessels from disk")
-    except Exception as e:
+    except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
         logger.error(f"Failed to load AIS cache: {e}")
 
 
-def get_ais_vessels() -> list[dict]:
-    """Return a snapshot of tracked AIS vessels, excluding 'other' type, pruning stale."""
+def prune_stale_vessels():
+    """Remove vessels not updated in the last 15 minutes. Safe to call from a scheduler."""
     now = time.time()
-    stale_cutoff = now - 900  # 15 minutes
-    
+    stale_cutoff = now - 900
     with _vessels_lock:
-        # Prune stale vessels
         stale_keys = [k for k, v in _vessels.items() if v.get("_updated", 0) < stale_cutoff]
         for k in stale_keys:
             del _vessels[k]
-        
+            _vessel_trails.pop(k, None)
+    if stale_keys:
+        logger.info(f"AIS pruned {len(stale_keys)} stale vessels")
+
+
+def get_ais_vessels() -> list[dict]:
+    """Return a snapshot of tracked AIS vessels, pruning stale."""
+    prune_stale_vessels()
+
+    with _vessels_lock:
         result = []
         for mmsi, v in _vessels.items():
             v_type = v.get("type", "unknown")
-            # Skip 'other' vessels (fishing, tug, pilot, etc.) to reduce load
-            if v_type == "other":
-                continue
             # Skip vessels without valid position
             if not v.get("lat") or not v.get("lng"):
                 continue
-            
-            result.append({
-                "mmsi": mmsi,
-                "name": v.get("name", "UNKNOWN"),
-                "type": v_type,
-                "lat": round(v.get("lat", 0), 5),
-                "lng": round(v.get("lng", 0), 5),
-                "heading": v.get("heading", 0),
-                "sog": round(v.get("sog", 0), 1),
-                "cog": round(v.get("cog", 0), 1),
-                "callsign": v.get("callsign", ""),
-                "destination": v.get("destination", "") or "UNKNOWN",
-                "imo": v.get("imo", 0),
-                "country": get_country_from_mmsi(mmsi),
-            })
+
+            # Sanitize speed: AIS 102.3 kn = "speed not available"
+            sog = v.get("sog", 0)
+            if sog >= 102.2:
+                sog = 0
+
+            result.append(
+                {
+                    "mmsi": mmsi,
+                    "name": v.get("name", "UNKNOWN"),
+                    "type": v_type,
+                    "lat": round(v.get("lat", 0), 5),
+                    "lng": round(v.get("lng", 0), 5),
+                    "heading": v.get("heading", 0),
+                    "sog": round(sog, 1),
+                    "cog": round(v.get("cog", 0), 1),
+                    "callsign": v.get("callsign", ""),
+                    "destination": v.get("destination", "") or "UNKNOWN",
+                    "imo": v.get("imo", 0),
+                    "country": get_country_from_mmsi(mmsi),
+                }
+            )
         return result
 
 
+def ingest_ais_catcher(msgs: list[dict]) -> int:
+    """Ingest decoded AIS messages from AIS-catcher HTTP feed.
+    Returns number of vessels updated."""
+    count = 0
+    now = time.time()
+    with _vessels_lock:
+        for msg in msgs:
+            mmsi = msg.get("mmsi")
+            if not mmsi or not isinstance(mmsi, int):
+                continue
+
+            vessel = _vessels.setdefault(mmsi, {"mmsi": mmsi})
+            msg_type = msg.get("type", 0)
+
+            # Position reports (types 1, 2, 3 = Class A; 18, 19 = Class B)
+            if msg_type in (1, 2, 3, 18, 19):
+                lat = msg.get("lat")
+                lon = msg.get("lon")
+                if lat is not None and lon is not None and lat != 91.0 and lon != 181.0:
+                    vessel["lat"] = lat
+                    vessel["lng"] = lon
+                    # AIS raw value 1023 (102.3 kn) = "speed not available"
+                    raw_speed = msg.get("speed", 0)
+                    vessel["sog"] = 0 if raw_speed >= 102.2 else raw_speed
+                    vessel["cog"] = msg.get("course", 0)
+                    heading = msg.get("heading", 511)
+                    vessel["heading"] = heading if heading != 511 else vessel.get("cog", 0)
+                    vessel["_updated"] = now
+                    _record_vessel_trail_locked(mmsi, lat, lon, vessel["sog"], now)
+                    if msg.get("shipname"):
+                        vessel["name"] = msg["shipname"].strip()
+                    count += 1
+
+            # Static data (type 5 = Class A static; 24 = Class B static)
+            elif msg_type in (5, 24):
+                if msg.get("shipname"):
+                    vessel["name"] = msg["shipname"].strip()
+                if msg.get("callsign"):
+                    vessel["callsign"] = msg["callsign"].strip()
+                if msg.get("imo"):
+                    vessel["imo"] = msg["imo"]
+                if msg.get("destination"):
+                    vessel["destination"] = msg["destination"].strip().replace("@", "")
+                ship_type = msg.get("shiptype", 0)
+                if ship_type:
+                    vessel["ais_type_code"] = ship_type
+                    vessel["type"] = classify_vessel(ship_type, mmsi)
+                vessel["_updated"] = now
+
+            # Ensure country is set from MMSI MID
+            if "country" not in vessel:
+                vessel["country"] = get_country_from_mmsi(mmsi)
+
+            # Ensure name exists
+            if "name" not in vessel:
+                vessel["name"] = msg.get("shipname", "UNKNOWN") or "UNKNOWN"
+
+    return count
+
+
 def _ais_stream_loop():
     """Main loop: spawn node proxy and process messages from stdout."""
+    global _proxy_process
     import subprocess
     import os
 
     proxy_script = os.path.join(os.path.dirname(os.path.dirname(__file__)), "ais_proxy.js")
     backoff = 1  # Exponential backoff starting at 1 second
 
+    if not API_KEY:
+        logger.info("AIS_API_KEY not set — ship tracking disabled. Set AIS_API_KEY to enable.")
+        return
+
     while _ws_running:
         try:
             logger.info("Starting Node.js AIS Stream Proxy...")
+            proxy_env = os.environ.copy()
+            proxy_env["AIS_API_KEY"] = API_KEY
+            popen_kwargs = {}
+            if os.name == "nt":
+                popen_kwargs["creationflags"] = (
+                    getattr(subprocess, "CREATE_NO_WINDOW", 0)
+                    | getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
+                )
             process = subprocess.Popen(
-                ['node', proxy_script, API_KEY],
+                ["node", proxy_script],
+                stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
                 text=True,
-                bufsize=1
+                bufsize=1,
+                env=proxy_env,
+                **popen_kwargs,
             )
-            
+            with _vessels_lock:
+                _proxy_process = process
+
             # Drain stderr in a background thread to prevent deadlock
             import threading
+
             def _drain_stderr():
-                for errline in iter(process.stderr.readline, ''):
+                for errline in iter(process.stderr.readline, ""):
                     errline = errline.strip()
                     if errline:
                         logger.warning(f"AIS proxy stderr: {errline}")
+
             threading.Thread(target=_drain_stderr, daemon=True).start()
-            
+
             logger.info("AIS Stream proxy started — receiving vessel data")
-            
+
             msg_count = 0
             ok_streak = 0  # Track consecutive successful messages for backoff reset
             last_log_time = time.time()
-            for raw_msg in iter(process.stdout.readline, ''):
+            for raw_msg in iter(process.stdout.readline, ""):
                 if not _ws_running:
                     process.terminate()
                     break
@@ -286,14 +636,20 @@ def _ais_stream_loop():
                     with _vessels_lock:
                         vessel["lat"] = lat
                         vessel["lng"] = lng
-                        vessel["sog"] = report.get("Sog", 0)
+                        # AIS raw value 1023 (102.3 kn) = "speed not available"
+                        raw_sog = report.get("Sog", 0)
+                        vessel["sog"] = 0 if raw_sog >= 102.2 else raw_sog
                         vessel["cog"] = report.get("Cog", 0)
                         heading = report.get("TrueHeading", 511)
                         vessel["heading"] = heading if heading != 511 else report.get("Cog", 0)
-                        vessel["_updated"] = time.time()
+                        now_ts = time.time()
+                        vessel["_updated"] = now_ts
+                        _record_vessel_trail_locked(mmsi, lat, lng, vessel["sog"], now_ts)
                         # Use metadata name if we don't have one yet
                         if not vessel.get("name") or vessel["name"] == "UNKNOWN":
-                            vessel["name"] = metadata.get("ShipName", "UNKNOWN").strip() or "UNKNOWN"
+                            vessel["name"] = (
+                                metadata.get("ShipName", "UNKNOWN").strip() or "UNKNOWN"
+                            )
 
                 # Update static data from ShipStaticData
                 elif msg_type == "ShipStaticData":
@@ -301,10 +657,14 @@ def _ais_stream_loop():
                     ais_type = static.get("Type", 0)
 
                     with _vessels_lock:
-                        vessel["name"] = (static.get("Name", "") or metadata.get("ShipName", "UNKNOWN")).strip() or "UNKNOWN"
+                        vessel["name"] = (
+                            static.get("Name", "") or metadata.get("ShipName", "UNKNOWN")
+                        ).strip() or "UNKNOWN"
                         vessel["callsign"] = (static.get("CallSign", "") or "").strip()
                         vessel["imo"] = static.get("ImoNumber", 0)
-                        vessel["destination"] = (static.get("Destination", "") or "").strip().replace("@", "")
+                        vessel["destination"] = (
+                            (static.get("Destination", "") or "").strip().replace("@", "")
+                        )
                         vessel["ais_type_code"] = ais_type
                         vessel["type"] = classify_vessel(ais_type, mmsi)
                         vessel["_updated"] = time.time()
@@ -322,11 +682,13 @@ def _ais_stream_loop():
                 if now - last_log_time >= 60:
                     with _vessels_lock:
                         count = len(_vessels)
-                    logger.info(f"AIS Stream: processed {msg_count} messages, tracking {count} vessels")
+                    logger.info(
+                        f"AIS Stream: processed {msg_count} messages, tracking {count} vessels"
+                    )
                     _save_cache()
                     last_log_time = now
 
-        except Exception as e:
+        except (ConnectionError, TimeoutError, OSError, ValueError, KeyError) as e:
             logger.error(f"AIS proxy connection error: {e}")
             if _ws_running:
                 logger.info(f"Restarting AIS proxy in {backoff}s (exponential backoff)...")
@@ -337,23 +699,47 @@ def _ais_stream_loop():
 
 def _run_ais_loop():
     """Thread target: run the AIS loop."""
+    global _ws_running, _ws_thread, _proxy_process
     try:
         _ais_stream_loop()
     except Exception as e:
         logger.error(f"AIS Stream thread crashed: {e}")
+    finally:
+        with _vessels_lock:
+            _ws_running = False
+            _ws_thread = None
+            _proxy_process = None
 
 
 def start_ais_stream():
     """Start the AIS WebSocket stream in a background thread."""
     global _ws_thread, _ws_running
-    if _ws_thread and _ws_thread.is_alive():
+
+    # Always load cached vessel data first so the ships layer can paint even
+    # when live streaming is disabled or the upstream is unavailable.
+    _load_cache()
+
+    if not API_KEY:
+        logger.info("AIS_API_KEY not set — ship tracking disabled. Set AIS_API_KEY to enable.")
+        return
+
+    if not ais_stream_proxy_enabled():
+        logger.info(
+            "AIS live stream proxy disabled for this runtime; using cached AIS vessels. "
+            "Set SHADOWBROKER_ENABLE_AIS_STREAM_PROXY=1 to opt in."
+        )
+        return
+
+    with _vessels_lock:
+        if _ws_running:
+            logger.info("AIS Stream already running")
+            return
+        _ws_running = True
+        existing_thread = _ws_thread
+    if existing_thread and existing_thread.is_alive():
         logger.info("AIS Stream already running")
         return
-    
-    # Load cached vessel data from disk
-    _load_cache()
-    
-    _ws_running = True
+
     _ws_thread = threading.Thread(target=_run_ais_loop, daemon=True, name="ais-stream")
     _ws_thread.start()
     logger.info("AIS Stream background thread started")
@@ -361,7 +747,36 @@ def start_ais_stream():
 
 def stop_ais_stream():
     """Stop the AIS WebSocket stream and save cache."""
-    global _ws_running
-    _ws_running = False
+    global _ws_running, _ws_thread, _proxy_process
+    with _vessels_lock:
+        _ws_running = False
+        _ws_thread = None
+        proc = _proxy_process
+        _proxy_process = None
+
+    if proc and proc.stdin:
+        try:
+            proc.stdin.close()
+        except Exception:
+            pass
+
     _save_cache()  # Save on shutdown
     logger.info("AIS Stream stopping...")
+
+
+def update_ais_bbox(south: float, west: float, north: float, east: float):
+    """Dynamically update the AIS stream bounding box via proxy stdin."""
+    with _vessels_lock:
+        proc = _proxy_process
+    if not proc or not proc.stdin:
+        return
+
+    try:
+        cmd = json.dumps({"type": "update_bbox", "bboxes": [[[south, west], [north, east]]]})
+        proc.stdin.write(cmd + "\n")
+        proc.stdin.flush()
+        logger.info(
+            f"Updated AIS bounding box to: S:{south:.2f} W:{west:.2f} N:{north:.2f} E:{east:.2f}"
+        )
+    except Exception as e:
+        logger.error(f"Failed to update AIS bbox: {e}")

backend/services/analysis_zone_store.py

@@ -0,0 +1,189 @@
+"""Analysis Zone store — OpenClaw-placed map overlays with analyst notes.
+
+These render as the dashed-border squares on the correlations layer.
+Unlike automated correlations (which are recomputed every cycle), analysis
+zones persist until the agent or user deletes them, or their TTL expires.
+
+Shape matches the correlation alert schema so the frontend renders them
+identically — the ``source`` field marks them as agent-placed and enables
+the delete button in the popup.
+"""
+
+import json
+import logging
+import os
+import threading
+import time
+import uuid
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_zones: list[dict[str, Any]] = []
+_lock = threading.Lock()
+
+_PERSIST_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data")
+_PERSIST_FILE = os.path.join(_PERSIST_DIR, "analysis_zones.json")
+
+ZONE_CATEGORIES = {
+    "contradiction",   # narrative vs telemetry mismatch
+    "analysis",        # general analyst note / assessment
+    "warning",         # potential threat or risk area
+    "observation",     # neutral observation worth marking
+    "hypothesis",      # unverified theory to investigate
+}
+
+# Map categories to correlation type colors on the frontend
+CATEGORY_COLORS = {
+    "contradiction": "amber",
+    "analysis": "cyan",
+    "warning": "red",
+    "observation": "blue",
+    "hypothesis": "purple",
+}
+
+
+def _ensure_dir():
+    try:
+        os.makedirs(_PERSIST_DIR, exist_ok=True)
+    except OSError:
+        pass
+
+
+def _save():
+    """Persist to disk. Called under lock."""
+    try:
+        _ensure_dir()
+        with open(_PERSIST_FILE, "w", encoding="utf-8") as f:
+            json.dump(_zones, f, indent=2, default=str)
+    except Exception as e:
+        logger.warning("Failed to save analysis zones: %s", e)
+
+
+def _load():
+    """Load from disk on startup."""
+    global _zones
+    try:
+        if os.path.exists(_PERSIST_FILE):
+            with open(_PERSIST_FILE, "r", encoding="utf-8") as f:
+                data = json.load(f)
+            if isinstance(data, list):
+                _zones = data
+                logger.info("Loaded %d analysis zones from disk", len(_zones))
+    except Exception as e:
+        logger.warning("Failed to load analysis zones: %s", e)
+
+
+# Load on import
+_load()
+
+
+def _expire():
+    """Remove zones past their TTL. Called under lock."""
+    now = time.time()
+    before = len(_zones)
+    _zones[:] = [
+        z for z in _zones
+        if z.get("ttl_hours", 0) <= 0
+        or (now - z.get("created_at", now)) < z["ttl_hours"] * 3600
+    ]
+    removed = before - len(_zones)
+    if removed:
+        logger.info("Expired %d analysis zones", removed)
+
+
+def create_zone(
+    *,
+    lat: float,
+    lng: float,
+    title: str,
+    body: str,
+    category: str = "analysis",
+    severity: str = "medium",
+    cell_size_deg: float = 1.0,
+    ttl_hours: float = 0,
+    source: str = "openclaw",
+    drivers: list[str] | None = None,
+) -> dict[str, Any]:
+    """Create an analysis zone. Returns the created zone dict."""
+    category = category if category in ZONE_CATEGORIES else "analysis"
+    if severity not in ("high", "medium", "low"):
+        severity = "medium"
+    cell_size_deg = max(0.1, min(cell_size_deg, 10.0))
+
+    zone: dict[str, Any] = {
+        "id": str(uuid.uuid4())[:12],
+        "lat": lat,
+        "lng": lng,
+        "type": "analysis_zone",
+        "category": category,
+        "severity": severity,
+        "score": {"high": 90, "medium": 60, "low": 30}.get(severity, 60),
+        "title": title[:200],
+        "body": body[:2000],
+        "drivers": (drivers or [title])[:5],
+        "cell_size": cell_size_deg,
+        "source": source,
+        "created_at": time.time(),
+        "ttl_hours": ttl_hours,
+    }
+
+    with _lock:
+        _expire()
+        _zones.append(zone)
+        _save()
+
+    logger.info("Analysis zone created: %s at (%.2f, %.2f)", title[:40], lat, lng)
+    return zone
+
+
+def list_zones() -> list[dict[str, Any]]:
+    """Return all live (non-expired) zones."""
+    with _lock:
+        _expire()
+        return list(_zones)
+
+
+def get_zone(zone_id: str) -> dict[str, Any] | None:
+    """Get a single zone by ID."""
+    with _lock:
+        for z in _zones:
+            if z["id"] == zone_id:
+                return dict(z)
+    return None
+
+
+def delete_zone(zone_id: str) -> bool:
+    """Delete a zone by ID. Returns True if found and removed."""
+    with _lock:
+        before = len(_zones)
+        _zones[:] = [z for z in _zones if z["id"] != zone_id]
+        if len(_zones) < before:
+            _save()
+            return True
+    return False
+
+
+def clear_zones(*, source: str | None = None) -> int:
+    """Clear all zones, optionally filtered by source. Returns count removed."""
+    with _lock:
+        before = len(_zones)
+        if source:
+            _zones[:] = [z for z in _zones if z.get("source") != source]
+        else:
+            _zones.clear()
+        removed = before - len(_zones)
+        if removed:
+            _save()
+        return removed
+
+
+def get_live_zones() -> list[dict[str, Any]]:
+    """Return zones formatted for the correlation engine merge.
+
+    This is called by compute_correlations() to inject agent-placed zones
+    into the correlations list that the frontend renders as map squares.
+    """
+    with _lock:
+        _expire()
+        return [dict(z) for z in _zones]

backend/services/api_settings.py

@@ -2,12 +2,23 @@
 API Settings management — serves the API key registry and allows updates.
 Keys are stored in the backend .env file and loaded via python-dotenv.
 """
+
 import os
 import re
+import tempfile
 from pathlib import Path
 
 # Path to the backend .env file
 ENV_PATH = Path(__file__).parent.parent / ".env"
+# Path to the example template that ships with the repo
+ENV_EXAMPLE_PATH = Path(__file__).parent.parent.parent / ".env.example"
+DATA_DIR = Path(os.environ.get("SB_DATA_DIR", str(Path(__file__).parent.parent / "data")))
+if not DATA_DIR.is_absolute():
+    DATA_DIR = Path(__file__).parent.parent / DATA_DIR
+OPERATOR_KEYS_ENV_PATH = Path(
+    os.environ.get("SHADOWBROKER_OPERATOR_KEYS_ENV", str(DATA_DIR / "operator_api_keys.env"))
+)
+_ENV_KEY_RE = re.compile(r"^[A-Z][A-Z0-9_]*$")
 
 # ---------------------------------------------------------------------------
 # API Registry — every external service the dashboard depends on
@@ -121,18 +132,138 @@ API_REGISTRY = [
         "url": "https://openmhz.com/",
         "required": False,
     },
+    {
+        "id": "shodan_api_key",
+        "env_key": "SHODAN_API_KEY",
+        "name": "Shodan — Operator API Key",
+        "description": "Paid Shodan API key for local operator-driven searches and temporary map overlays. Results are attributed to Shodan and are not merged into ShadowBroker core feeds.",
+        "category": "Reconnaissance",
+        "url": "https://account.shodan.io/billing",
+        "required": False,
+    },
+    {
+        "id": "finnhub_api_key",
+        "env_key": "FINNHUB_API_KEY",
+        "name": "Finnhub — API Key",
+        "description": "Free market data API. Defense stock quotes, congressional trading disclosures, and insider transactions. 60 calls/min free tier.",
+        "category": "Financial",
+        "url": "https://finnhub.io/register",
+        "required": False,
+    },
 ]
 
+ALLOWED_ENV_KEYS = {
+    str(api["env_key"])
+    for api in API_REGISTRY
+    if api.get("env_key")
+}
 
-def _obfuscate(value: str) -> str:
-    """Show first 4 chars, mask the rest with bullets."""
-    if not value or len(value) <= 4:
-        return "••••••••"
-    return value[:4] + "•" * (len(value) - 4)
+
+def _parse_env_file(path: Path) -> dict[str, str]:
+    values: dict[str, str] = {}
+    if not path.exists():
+        return values
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError:
+        return values
+    for raw_line in text.splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        key = key.strip()
+        if not _ENV_KEY_RE.match(key):
+            continue
+        value = value.strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
+            value = value[1:-1]
+        values[key] = value
+    return values
+
+
+def _quote_env_value(value: str) -> str:
+    escaped = value.replace("\\", "\\\\").replace('"', '\\"')
+    return f'"{escaped}"'
+
+
+def _write_env_values(path: Path, updates: dict[str, str]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    lines = path.read_text(encoding="utf-8").splitlines() if path.exists() else []
+    seen: set[str] = set()
+    next_lines: list[str] = []
+    for raw_line in lines:
+        stripped = raw_line.strip()
+        if "=" not in stripped or stripped.startswith("#"):
+            next_lines.append(raw_line)
+            continue
+        key = stripped.split("=", 1)[0].strip()
+        if key in updates:
+            next_lines.append(f"{key}={_quote_env_value(updates[key])}")
+            seen.add(key)
+        else:
+            next_lines.append(raw_line)
+    for key, value in updates.items():
+        if key not in seen:
+            next_lines.append(f"{key}={_quote_env_value(value)}")
+
+    fd, tmp_name = tempfile.mkstemp(dir=str(path.parent), prefix=f"{path.name}.tmp.", text=True)
+    tmp_path = Path(tmp_name)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
+            handle.write("\n".join(next_lines).rstrip() + "\n")
+        if os.name != "nt":
+            os.chmod(tmp_path, 0o600)
+        os.replace(tmp_path, path)
+        if os.name != "nt":
+            os.chmod(path, 0o600)
+    finally:
+        try:
+            if tmp_path.exists():
+                tmp_path.unlink()
+        except OSError:
+            pass
+
+
+def load_persisted_api_keys_into_environ() -> None:
+    """Load persisted operator API keys if no process env value exists."""
+    for key, value in _parse_env_file(OPERATOR_KEYS_ENV_PATH).items():
+        if key in ALLOWED_ENV_KEYS and value and not os.environ.get(key):
+            os.environ[key] = value
+
+
+def get_env_path_info() -> dict:
+    """Return absolute paths for the backend .env and .env.example template.
+
+    Surfaced to the frontend so the API Keys settings panel can tell users
+    exactly where to put their keys when in-app editing fails (admin-not-set,
+    file permissions, read-only filesystem, etc.).
+    """
+    env_path = ENV_PATH.resolve()
+    example_path = ENV_EXAMPLE_PATH.resolve()
+    return {
+        "env_path": str(env_path),
+        "env_path_exists": env_path.exists(),
+        "env_path_writable": os.access(env_path.parent, os.W_OK)
+            and (not env_path.exists() or os.access(env_path, os.W_OK)),
+        "env_example_path": str(example_path),
+        "env_example_path_exists": example_path.exists(),
+        "operator_keys_env_path": str(OPERATOR_KEYS_ENV_PATH.resolve()),
+        "operator_keys_env_path_exists": OPERATOR_KEYS_ENV_PATH.exists(),
+        "operator_keys_env_path_writable": os.access(OPERATOR_KEYS_ENV_PATH.parent, os.W_OK)
+            and (not OPERATOR_KEYS_ENV_PATH.exists() or os.access(OPERATOR_KEYS_ENV_PATH, os.W_OK)),
+    }
 
 
 def get_api_keys():
-    """Return the full API registry with obfuscated key values."""
+    """Return the API registry with a binary set/unset flag per key.
+
+    Key values themselves are NEVER returned to the client — not even an
+    obfuscated prefix. Users edit the .env file directly; the panel uses
+    `is_set` to render a CONFIGURED / NOT CONFIGURED badge and the path
+    info from `get_env_path_info()` to tell them where to put each key.
+    """
+    load_persisted_api_keys_into_environ()
     result = []
     for api in API_REGISTRY:
         entry = {
@@ -144,41 +275,64 @@ def get_api_keys():
             "required": api["required"],
             "has_key": api["env_key"] is not None,
             "env_key": api["env_key"],
-            "value_obfuscated": None,
             "is_set": False,
         }
         if api["env_key"]:
             raw = os.environ.get(api["env_key"], "")
-            entry["value_obfuscated"] = _obfuscate(raw)
             entry["is_set"] = bool(raw)
         result.append(entry)
     return result
 
 
-def update_api_key(env_key: str, new_value: str) -> bool:
-    """Update a single key in the .env file and in the current process env."""
-    valid_keys = {api["env_key"] for api in API_REGISTRY if api.get("env_key")}
-    if env_key not in valid_keys:
-        return False
-        
-    if not isinstance(new_value, str):
-        return False
-    if "\n" in new_value or "\r" in new_value:
-        return False
+def save_api_keys(updates: dict[str, str]) -> dict:
+    """Persist allowed API keys from a local operator request.
 
-    if not ENV_PATH.exists():
-        ENV_PATH.write_text("", encoding="utf-8")
+    Values are accepted write-only: the response includes only configured flags.
+    """
+    clean: dict[str, str] = {}
+    for key, value in updates.items():
+        env_key = str(key or "").strip().upper()
+        if env_key not in ALLOWED_ENV_KEYS:
+            continue
+        clean_value = str(value or "").strip()
+        if clean_value:
+            clean[env_key] = clean_value
+    if not clean:
+        return {"ok": False, "detail": "No supported API keys were provided."}
 
-    # Update os.environ immediately
-    os.environ[env_key] = new_value
+    _write_env_values(OPERATOR_KEYS_ENV_PATH, clean)
+    try:
+        _write_env_values(ENV_PATH, clean)
+    except OSError:
+        # The persistent operator key file is the source of truth for Docker.
+        pass
+    for key, value in clean.items():
+        os.environ[key] = value
+    if "AIS_API_KEY" in clean:
+        try:
+            from services import ais_stream
+            ais_stream.API_KEY = clean["AIS_API_KEY"]
+        except Exception:
+            pass
+    if "OPENSKY_CLIENT_ID" in clean or "OPENSKY_CLIENT_SECRET" in clean:
+        try:
+            from services.fetchers import flights
+            flights.opensky_client.client_id = os.environ.get("OPENSKY_CLIENT_ID", "")
+            flights.opensky_client.client_secret = os.environ.get("OPENSKY_CLIENT_SECRET", "")
+            flights.opensky_client.token = None
+            flights.opensky_client.expires_at = 0
+        except Exception:
+            pass
 
-    # Update the .env file on disk
-    content = ENV_PATH.read_text(encoding="utf-8")
-    pattern = re.compile(rf"^{re.escape(env_key)}=.*$", re.MULTILINE)
-    if pattern.search(content):
-        content = pattern.sub(f"{env_key}={new_value}", content)
-    else:
-        content = content.rstrip("\n") + f"\n{env_key}={new_value}\n"
+    try:
+        from services.config import get_settings
+        get_settings.cache_clear()
+    except Exception:
+        pass
 
-    ENV_PATH.write_text(content, encoding="utf-8")
-    return True
+    return {
+        "ok": True,
+        "updated": sorted(clean.keys()),
+        "keys": get_api_keys(),
+        "env": get_env_path_info(),
+    }

backend/services/carrier_tracker.py

@@ -15,6 +15,7 @@ import json
 import time
 import logging
 import threading
+import random
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Dict, List, Optional
@@ -34,108 +35,127 @@ CARRIER_REGISTRY: Dict[str, dict] = {
         "name": "USS Nimitz (CVN-68)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Nimitz",
         "homeport": "Bremerton, WA",
-        "homeport_lat": 47.5535, "homeport_lng": -122.6400,
-        "fallback_lat": 47.5535, "fallback_lng": -122.6400,
+        "homeport_lat": 47.5535,
+        "homeport_lng": -122.6400,
+        "fallback_lat": 47.5535,
+        "fallback_lng": -122.6400,
         "fallback_heading": 90,
-        "fallback_desc": "Bremerton, WA (Maintenance)"
+        "fallback_desc": "Bremerton, WA (Maintenance)",
     },
     "CVN-76": {
         "name": "USS Ronald Reagan (CVN-76)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Ronald_Reagan",
         "homeport": "Bremerton, WA",
-        "homeport_lat": 47.5580, "homeport_lng": -122.6360,
-        "fallback_lat": 47.5580, "fallback_lng": -122.6360,
+        "homeport_lat": 47.5580,
+        "homeport_lng": -122.6360,
+        "fallback_lat": 47.5580,
+        "fallback_lng": -122.6360,
         "fallback_heading": 90,
-        "fallback_desc": "Bremerton, WA (Decommissioning)"
+        "fallback_desc": "Bremerton, WA (Decommissioning)",
     },
-
     # --- Norfolk, VA (Naval Station Norfolk) ---
     # Piers run N-S along Willoughby Bay; each carrier gets a distinct berth
     "CVN-69": {
         "name": "USS Dwight D. Eisenhower (CVN-69)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Dwight_D._Eisenhower",
         "homeport": "Norfolk, VA",
-        "homeport_lat": 36.9465, "homeport_lng": -76.3265,
-        "fallback_lat": 36.9465, "fallback_lng": -76.3265,
+        "homeport_lat": 36.9465,
+        "homeport_lng": -76.3265,
+        "fallback_lat": 36.9465,
+        "fallback_lng": -76.3265,
         "fallback_heading": 0,
-        "fallback_desc": "Norfolk, VA (Post-deployment maintenance)"
+        "fallback_desc": "Norfolk, VA (Post-deployment maintenance)",
     },
     "CVN-78": {
         "name": "USS Gerald R. Ford (CVN-78)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Gerald_R._Ford",
         "homeport": "Norfolk, VA",
-        "homeport_lat": 36.9505, "homeport_lng": -76.3250,
-        "fallback_lat": 18.0, "fallback_lng": 39.5,
+        "homeport_lat": 36.9505,
+        "homeport_lng": -76.3250,
+        "fallback_lat": 18.0,
+        "fallback_lng": 39.5,
         "fallback_heading": 0,
-        "fallback_desc": "Red Sea — Operation Epic Fury (USNI Mar 9)"
+        "fallback_desc": "Red Sea — Operation Epic Fury (USNI Mar 9)",
     },
     "CVN-74": {
         "name": "USS John C. Stennis (CVN-74)",
         "wiki": "https://en.wikipedia.org/wiki/USS_John_C._Stennis",
         "homeport": "Norfolk, VA",
-        "homeport_lat": 36.9540, "homeport_lng": -76.3235,
-        "fallback_lat": 36.98, "fallback_lng": -76.43,
+        "homeport_lat": 36.9540,
+        "homeport_lng": -76.3235,
+        "fallback_lat": 36.98,
+        "fallback_lng": -76.43,
         "fallback_heading": 0,
-        "fallback_desc": "Newport News, VA (RCOH refueling overhaul)"
+        "fallback_desc": "Newport News, VA (RCOH refueling overhaul)",
     },
     "CVN-75": {
         "name": "USS Harry S. Truman (CVN-75)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Harry_S._Truman",
         "homeport": "Norfolk, VA",
-        "homeport_lat": 36.9580, "homeport_lng": -76.3220,
-        "fallback_lat": 36.0, "fallback_lng": 15.0,
+        "homeport_lat": 36.9580,
+        "homeport_lng": -76.3220,
+        "fallback_lat": 36.0,
+        "fallback_lng": 15.0,
         "fallback_heading": 0,
-        "fallback_desc": "Mediterranean Sea deployment (USNI Mar 9)"
+        "fallback_desc": "Mediterranean Sea deployment (USNI Mar 9)",
     },
     "CVN-77": {
         "name": "USS George H.W. Bush (CVN-77)",
         "wiki": "https://en.wikipedia.org/wiki/USS_George_H.W._Bush",
         "homeport": "Norfolk, VA",
-        "homeport_lat": 36.9620, "homeport_lng": -76.3210,
-        "fallback_lat": 36.5, "fallback_lng": -74.0,
+        "homeport_lat": 36.9620,
+        "homeport_lng": -76.3210,
+        "fallback_lat": 36.5,
+        "fallback_lng": -74.0,
         "fallback_heading": 0,
-        "fallback_desc": "Atlantic — Pre-deployment workups (USNI Mar 9)"
+        "fallback_desc": "Atlantic — Pre-deployment workups (USNI Mar 9)",
     },
-
     # --- San Diego, CA (Naval Base San Diego) ---
     # Carrier piers along the east shore of San Diego Bay, spread N-S
     "CVN-70": {
         "name": "USS Carl Vinson (CVN-70)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Carl_Vinson",
         "homeport": "San Diego, CA",
-        "homeport_lat": 32.6840, "homeport_lng": -117.1290,
-        "fallback_lat": 32.6840, "fallback_lng": -117.1290,
+        "homeport_lat": 32.6840,
+        "homeport_lng": -117.1290,
+        "fallback_lat": 32.6840,
+        "fallback_lng": -117.1290,
         "fallback_heading": 180,
-        "fallback_desc": "San Diego, CA (Homeport)"
+        "fallback_desc": "San Diego, CA (Homeport)",
     },
     "CVN-71": {
         "name": "USS Theodore Roosevelt (CVN-71)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Theodore_Roosevelt_(CVN-71)",
         "homeport": "San Diego, CA",
-        "homeport_lat": 32.6885, "homeport_lng": -117.1280,
-        "fallback_lat": 32.6885, "fallback_lng": -117.1280,
+        "homeport_lat": 32.6885,
+        "homeport_lng": -117.1280,
+        "fallback_lat": 32.6885,
+        "fallback_lng": -117.1280,
         "fallback_heading": 180,
-        "fallback_desc": "San Diego, CA (Maintenance)"
+        "fallback_desc": "San Diego, CA (Maintenance)",
     },
     "CVN-72": {
         "name": "USS Abraham Lincoln (CVN-72)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Abraham_Lincoln_(CVN-72)",
         "homeport": "San Diego, CA",
-        "homeport_lat": 32.6925, "homeport_lng": -117.1275,
-        "fallback_lat": 20.0, "fallback_lng": 64.0,
+        "homeport_lat": 32.6925,
+        "homeport_lng": -117.1275,
+        "fallback_lat": 20.0,
+        "fallback_lng": 64.0,
         "fallback_heading": 0,
-        "fallback_desc": "Arabian Sea — Operation Epic Fury (USNI Mar 9)"
+        "fallback_desc": "Arabian Sea — Operation Epic Fury (USNI Mar 9)",
     },
-
     # --- Yokosuka, Japan (CFAY) ---
     "CVN-73": {
         "name": "USS George Washington (CVN-73)",
         "wiki": "https://en.wikipedia.org/wiki/USS_George_Washington_(CVN-73)",
         "homeport": "Yokosuka, Japan",
-        "homeport_lat": 35.2830, "homeport_lng": 139.6700,
-        "fallback_lat": 35.2830, "fallback_lng": 139.6700,
+        "homeport_lat": 35.2830,
+        "homeport_lng": 139.6700,
+        "fallback_lat": 35.2830,
+        "fallback_lng": 139.6700,
         "fallback_heading": 180,
-        "fallback_desc": "Yokosuka, Japan (Forward deployed)"
+        "fallback_desc": "Yokosuka, Japan (Forward deployed)",
     },
 }
 
@@ -175,7 +195,6 @@ REGION_COORDS: Dict[str, tuple] = {
     "coral sea": (-18.0, 155.0),
     "gulf of mexico": (25.0, -90.0),
     "caribbean": (15.0, -75.0),
-
     # Specific bases / ports
     "norfolk": (36.95, -76.33),
     "san diego": (32.68, -117.15),
@@ -188,7 +207,6 @@ REGION_COORDS: Dict[str, tuple] = {
     "bremerton": (47.56, -122.63),
     "puget sound": (47.56, -122.63),
     "newport news": (36.98, -76.43),
-
     # Areas of operation
     "centcom": (25.0, 55.0),
     "indopacom": (20.0, 130.0),
@@ -209,6 +227,11 @@ CACHE_FILE = Path(__file__).parent.parent / "carrier_cache.json"
 _carrier_positions: Dict[str, dict] = {}
 _positions_lock = threading.Lock()
 _last_update: Optional[datetime] = None
+_last_gdelt_fetch_at = 0.0
+_cached_gdelt_articles: List[dict] = []
+_GDELT_FETCH_INTERVAL_SECONDS = 1800
+_GDELT_REQUEST_DELAY_SECONDS = 1.25
+_GDELT_REQUEST_JITTER_SECONDS = 0.35
 
 
 def _load_cache() -> Dict[str, dict]:
@@ -218,7 +241,7 @@ def _load_cache() -> Dict[str, dict]:
             data = json.loads(CACHE_FILE.read_text())
             logger.info(f"Carrier cache loaded: {len(data)} carriers from {CACHE_FILE}")
             return data
-    except Exception as e:
+    except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
         logger.warning(f"Failed to load carrier cache: {e}")
     return {}
 
@@ -228,7 +251,7 @@ def _save_cache(positions: Dict[str, dict]):
     try:
         CACHE_FILE.write_text(json.dumps(positions, indent=2))
         logger.info(f"Carrier cache saved: {len(positions)} carriers")
-    except Exception as e:
+    except (IOError, OSError) as e:
         logger.warning(f"Failed to save carrier cache: {e}")
 
 
@@ -260,33 +283,59 @@ def _match_carrier(text: str) -> Optional[str]:
 
 def _fetch_gdelt_carrier_news() -> List[dict]:
     """Search GDELT for recent carrier movement news."""
+    global _last_gdelt_fetch_at, _cached_gdelt_articles
+
+    now = time.time()
+    if _cached_gdelt_articles and (now - _last_gdelt_fetch_at) < _GDELT_FETCH_INTERVAL_SECONDS:
+        logger.info("Carrier OSINT: using cached GDELT article set to avoid startup bursts")
+        return list(_cached_gdelt_articles)
+
     results = []
     search_terms = [
         "aircraft+carrier+deployed",
         "carrier+strike+group+navy",
-        "USS+Nimitz+carrier", "USS+Ford+carrier", "USS+Eisenhower+carrier",
-        "USS+Vinson+carrier", "USS+Roosevelt+carrier+navy",
-        "USS+Lincoln+carrier", "USS+Truman+carrier",
-        "USS+Reagan+carrier", "USS+Washington+carrier+navy",
-        "USS+Bush+carrier", "USS+Stennis+carrier",
+        "USS+Nimitz+carrier",
+        "USS+Ford+carrier",
+        "USS+Eisenhower+carrier",
+        "USS+Vinson+carrier",
+        "USS+Roosevelt+carrier+navy",
+        "USS+Lincoln+carrier",
+        "USS+Truman+carrier",
+        "USS+Reagan+carrier",
+        "USS+Washington+carrier+navy",
+        "USS+Bush+carrier",
+        "USS+Stennis+carrier",
     ]
 
-    for term in search_terms:
+    for idx, term in enumerate(search_terms):
         try:
             url = f"https://api.gdeltproject.org/api/v2/doc/doc?query={term}&mode=artlist&maxrecords=5&format=json&timespan=14d"
             raw = fetch_with_curl(url, timeout=8)
-            if not raw:
+            if getattr(raw, "status_code", 500) == 429:
+                logger.warning(
+                    "GDELT returned 429 for '%s'; preserving cached carrier OSINT results",
+                    term,
+                )
                 continue
-            data = json.loads(raw)
+            if not raw or not hasattr(raw, "text"):
+                continue
+            data = raw.json()
             articles = data.get("articles", [])
             for art in articles:
                 title = art.get("title", "")
                 url = art.get("url", "")
                 results.append({"title": title, "url": url})
-        except Exception as e:
+        except (ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
             logger.debug(f"GDELT search failed for '{term}': {e}")
             continue
+        if idx < len(search_terms) - 1:
+            time.sleep(
+                _GDELT_REQUEST_DELAY_SECONDS
+                + random.uniform(0.0, _GDELT_REQUEST_JITTER_SECONDS)
+            )
 
+    _cached_gdelt_articles = list(results)
+    _last_gdelt_fetch_at = time.time()
     logger.info(f"Carrier OSINT: found {len(results)} GDELT articles")
     return results
 
@@ -316,20 +365,17 @@ def _parse_carrier_positions_from_news(articles: List[dict]) -> Dict[str, dict]:
                 "desc": title[:100],
                 "source": "GDELT News API",
                 "source_url": article.get("url", "https://api.gdeltproject.org"),
-                "updated": datetime.now(timezone.utc).isoformat()
+                "updated": datetime.now(timezone.utc).isoformat(),
             }
-            logger.info(f"Carrier update: {CARRIER_REGISTRY[hull]['name']} → {coords} (from: {title[:80]})")
+            logger.info(
+                f"Carrier update: {CARRIER_REGISTRY[hull]['name']} → {coords} (from: {title[:80]})"
+            )
 
     return updates
 
 
-def update_carrier_positions():
-    """Main update function — called on startup and every 12h."""
-    global _last_update
-
-    logger.info("Carrier tracker: updating positions from OSINT sources...")
-
-    # Start with fallback positions (sourced from USNI News Fleet Tracker)
+def _load_carrier_fallbacks() -> Dict[str, dict]:
+    """Build carrier positions from static fallbacks + disk cache (instant, no network)."""
     positions: Dict[str, dict] = {}
     for hull, info in CARRIER_REGISTRY.items():
         positions[hull] = {
@@ -341,24 +387,50 @@ def update_carrier_positions():
             "wiki": info["wiki"],
             "source": "USNI News Fleet & Marine Tracker",
             "source_url": "https://news.usni.org/category/fleet-tracker",
-            "updated": datetime.now(timezone.utc).isoformat()
+            "updated": datetime.now(timezone.utc).isoformat(),
         }
 
-    # Load cached positions (may have better data from previous runs)
+    # Overlay cached positions from previous runs (may have GDELT data)
     cached = _load_cache()
     for hull, cached_pos in cached.items():
         if hull in positions:
-            # Only use cache if it has a real OSINT source (not just static)
-            if cached_pos.get("source", "").startswith("GDELT") or cached_pos.get("source", "").startswith("News"):
-                positions[hull].update({
-                    "lat": cached_pos["lat"],
-                    "lng": cached_pos["lng"],
-                    "desc": cached_pos.get("desc", positions[hull]["desc"]),
-                    "source": cached_pos.get("source", "Cached OSINT"),
-                    "updated": cached_pos.get("updated", "")
-                })
+            if cached_pos.get("source", "").startswith("GDELT") or cached_pos.get(
+                "source", ""
+            ).startswith("News"):
+                positions[hull].update(
+                    {
+                        "lat": cached_pos["lat"],
+                        "lng": cached_pos["lng"],
+                        "desc": cached_pos.get("desc", positions[hull]["desc"]),
+                        "source": cached_pos.get("source", "Cached OSINT"),
+                        "updated": cached_pos.get("updated", ""),
+                    }
+                )
+    return positions
 
-    # Try GDELT news for fresh positions
+
+def update_carrier_positions():
+    """Main update function — called on startup and every 12h.
+
+    Phase 1 (instant): publish fallback + cached positions so the map has carriers immediately.
+    Phase 2 (slow):    query GDELT for fresh OSINT positions and update in-place.
+    """
+    global _last_update
+
+    # --- Phase 1: instant fallback + cache ---
+    positions = _load_carrier_fallbacks()
+
+    with _positions_lock:
+        # Only overwrite if positions are currently empty (first startup).
+        # If we already have data from a previous cycle, keep it while GDELT runs.
+        if not _carrier_positions:
+            _carrier_positions.update(positions)
+            _last_update = datetime.now(timezone.utc)
+    logger.info(
+        f"Carrier tracker: {len(positions)} carriers loaded from fallback/cache (GDELT enrichment starting...)"
+    )
+
+    # --- Phase 2: slow GDELT enrichment ---
     try:
         articles = _fetch_gdelt_carrier_news()
         news_positions = _parse_carrier_positions_from_news(articles)
@@ -366,10 +438,10 @@ def update_carrier_positions():
             if hull in positions:
                 positions[hull].update(pos)
                 logger.info(f"Carrier OSINT: updated {CARRIER_REGISTRY[hull]['name']} from news")
-    except Exception as e:
+    except (ValueError, KeyError, json.JSONDecodeError, OSError) as e:
         logger.warning(f"GDELT carrier fetch failed: {e}")
 
-    # Save and update the global state
+    # Save and update the global state with enriched positions
     with _positions_lock:
         _carrier_positions.clear()
         _carrier_positions.update(positions)
@@ -393,6 +465,7 @@ def _deconflict_positions(result: List[dict]) -> List[dict]:
     """
     # Group by rounded lat/lng (within ~0.01° ≈ 1km = same spot)
     from collections import defaultdict
+
     groups: dict[str, list[int]] = defaultdict(list)
     for i, c in enumerate(result):
         key = f"{round(c['lat'], 2)},{round(c['lng'], 2)}"
@@ -439,22 +512,26 @@ def get_carrier_positions() -> List[dict]:
         result = []
         for hull, pos in _carrier_positions.items():
             info = CARRIER_REGISTRY.get(hull, {})
-            result.append({
-                "name": pos.get("name", info.get("name", hull)),
-                "type": "carrier",
-                "lat": pos["lat"],
-                "lng": pos["lng"],
-                "heading": None,  # Heading unknown for carriers — OSINT cannot determine true heading
-                "sog": 0,
-                "cog": 0,
-                "country": "United States",
-                "desc": pos.get("desc", ""),
-                "wiki": pos.get("wiki", info.get("wiki", "")),
-                "estimated": True,
-                "source": pos.get("source", "OSINT estimated position"),
-                "source_url": pos.get("source_url", "https://news.usni.org/category/fleet-tracker"),
-                "last_osint_update": pos.get("updated", "")
-            })
+            result.append(
+                {
+                    "name": pos.get("name", info.get("name", hull)),
+                    "type": "carrier",
+                    "lat": pos["lat"],
+                    "lng": pos["lng"],
+                    "heading": None,  # Heading unknown for carriers — OSINT cannot determine true heading
+                    "sog": 0,
+                    "cog": 0,
+                    "country": "United States",
+                    "desc": pos.get("desc", ""),
+                    "wiki": pos.get("wiki", info.get("wiki", "")),
+                    "estimated": True,
+                    "source": pos.get("source", "OSINT estimated position"),
+                    "source_url": pos.get(
+                        "source_url", "https://news.usni.org/category/fleet-tracker"
+                    ),
+                    "last_osint_update": pos.get("updated", ""),
+                }
+            )
         return _deconflict_positions(result)
 
 
@@ -485,10 +562,13 @@ def _scheduler_loop():
         next_run = now.replace(hour=next_hour % 24, minute=0, second=0, microsecond=0)
         if next_hour == 24:
             from datetime import timedelta
+
             next_run = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
 
         wait_seconds = (next_run - now).total_seconds()
-        logger.info(f"Carrier tracker: next update at {next_run.isoformat()} ({wait_seconds/3600:.1f}h)")
+        logger.info(
+            f"Carrier tracker: next update at {next_run.isoformat()} ({wait_seconds/3600:.1f}h)"
+        )
 
         # Wait until next scheduled time, or until stop event
         if _scheduler_stop.wait(timeout=wait_seconds):
@@ -506,7 +586,9 @@ def start_carrier_tracker():
     if _scheduler_thread and _scheduler_thread.is_alive():
         return
     _scheduler_stop.clear()
-    _scheduler_thread = threading.Thread(target=_scheduler_loop, daemon=True, name="carrier-tracker")
+    _scheduler_thread = threading.Thread(
+        target=_scheduler_loop, daemon=True, name="carrier-tracker"
+    )
     _scheduler_thread.start()
     logger.info("Carrier tracker started")
 

backend/services/config.py

@@ -0,0 +1,391 @@
+"""Typed configuration via pydantic-settings."""
+
+from functools import lru_cache
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    # Admin/security
+    ADMIN_KEY: str = ""
+    ALLOW_INSECURE_ADMIN: bool = False
+    PUBLIC_API_KEY: str = ""
+
+    # OpenClaw agent connectivity
+    OPENCLAW_HMAC_SECRET: str = ""        # HMAC shared secret for direct mode (auto-generated if empty)
+    OPENCLAW_ACCESS_TIER: str = "restricted"  # "full" or "restricted"
+
+    # Data sources
+    AIS_API_KEY: str = ""
+    OPENSKY_CLIENT_ID: str = ""
+    OPENSKY_CLIENT_SECRET: str = ""
+    LTA_ACCOUNT_KEY: str = ""
+
+    # Runtime
+    CORS_ORIGINS: str = ""
+    FETCH_SLOW_THRESHOLD_S: float = 5.0
+    MESH_STRICT_SIGNATURES: bool = True
+    MESH_DEBUG_MODE: bool = False
+    MESH_MQTT_EXTRA_ROOTS: str = ""
+    MESH_MQTT_EXTRA_TOPICS: str = ""
+    MESH_MQTT_INCLUDE_DEFAULT_ROOTS: bool = True
+    MESH_RNS_ENABLED: bool = False
+    MESH_ARTI_ENABLED: bool = False
+    MESH_ARTI_SOCKS_PORT: int = 9050
+    MESH_RELAY_PEERS: str = ""
+    # Bootstrap seeds are discovery hints, not authoritative network roots.
+    # Nodes promote healthy discovered peers from the store/manifest over time.
+    MESH_BOOTSTRAP_SEED_PEERS: str = "http://gqpbunqbgtkcqilvclm3xrkt3zowjyl3s62kkktvojgvxzizamvbrqid.onion:8000"
+    # Legacy name kept for older compose/.env files.
+    MESH_DEFAULT_SYNC_PEERS: str = ""
+    # Infonet/Wormhole must fail closed to private transports by default.
+    # Set true only for local relay development or explicitly public testnets.
+    MESH_INFONET_ALLOW_CLEARNET_SYNC: bool = False
+    MESH_BOOTSTRAP_DISABLED: bool = False
+    MESH_BOOTSTRAP_MANIFEST_PATH: str = "data/bootstrap_peers.json"
+    MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY: str = ""
+    MESH_NODE_MODE: str = "participant"
+    MESH_SYNC_INTERVAL_S: int = 300
+    MESH_SYNC_FAILURE_BACKOFF_S: int = 60
+    MESH_SYNC_TIMEOUT_S: int = 5
+    MESH_SYNC_MAX_PEERS_PER_CYCLE: int = 3
+    MESH_RELAY_PUSH_TIMEOUT_S: int = 10
+    MESH_RELAY_MAX_FAILURES: int = 3
+    MESH_RELAY_FAILURE_COOLDOWN_S: int = 120
+    MESH_BOOTSTRAP_SEED_FAILURE_COOLDOWN_S: int = 15
+    MESH_PEER_PUSH_SECRET: str = ""
+    MESH_RNS_APP_NAME: str = "shadowbroker"
+    MESH_RNS_ASPECT: str = "infonet"
+    MESH_RNS_IDENTITY_PATH: str = ""
+    MESH_RNS_PEERS: str = ""
+    MESH_RNS_DANDELION_HOPS: int = 2
+    MESH_RNS_DANDELION_DELAY_MS: int = 400
+    MESH_RNS_CHURN_INTERVAL_S: int = 300
+    MESH_RNS_MAX_PEERS: int = 32
+    MESH_RNS_MAX_PAYLOAD: int = 8192
+    MESH_RNS_PEER_BUCKET_PREFIX: int = 4
+    MESH_RNS_MAX_PEERS_PER_BUCKET: int = 4
+    MESH_RNS_PEER_FAIL_THRESHOLD: int = 3
+    MESH_RNS_PEER_COOLDOWN_S: int = 300
+    MESH_RNS_SHARD_ENABLED: bool = False
+    MESH_RNS_SHARD_DATA_SHARDS: int = 3
+    MESH_RNS_SHARD_PARITY_SHARDS: int = 1
+    MESH_RNS_SHARD_TTL_S: int = 30
+    MESH_RNS_FEC_CODEC: str = "xor"  # xor | rs
+    MESH_RNS_BATCH_MS: int = 200
+    # Keep a low background cadence on private RNS links so quiet nodes are less
+    # trivially fingerprintable by silence alone. Set to 0 to disable explicitly.
+    MESH_RNS_COVER_INTERVAL_S: int = 30
+    MESH_RNS_COVER_SIZE: int = 512
+    MESH_DM_MAILBOX_TTL_S: int = 900
+    MESH_RNS_IBF_WINDOW: int = 256
+    MESH_RNS_IBF_TABLE_SIZE: int = 64
+    MESH_RNS_IBF_MINHASH_SIZE: int = 16
+    MESH_RNS_IBF_MINHASH_THRESHOLD: float = 0.25
+    MESH_RNS_IBF_WINDOW_JITTER: int = 32
+    MESH_RNS_IBF_INTERVAL_S: int = 120
+    MESH_RNS_IBF_SYNC_PEERS: int = 3
+    MESH_RNS_IBF_QUORUM_TIMEOUT_S: int = 6
+    MESH_RNS_IBF_MAX_REQUEST_IDS: int = 64
+    MESH_RNS_IBF_MAX_EVENTS: int = 64
+    MESH_RNS_SESSION_ROTATE_S: int = 1800
+    MESH_RNS_IBF_FAIL_THRESHOLD: int = 3
+    MESH_RNS_IBF_COOLDOWN_S: int = 120
+    MESH_VERIFY_INTERVAL_S: int = 600
+    # MESH_VERIFY_SIGNATURES is intentionally removed — the audit loop in main.py
+    # always calls validate_chain_incremental(verify_signatures=True). Any value
+    # set in the environment is ignored.
+    MESH_DM_SECURE_MODE: bool = True
+    MESH_DM_TOKEN_PEPPER: str = ""
+    MESH_ALLOW_LEGACY_DM1_UNTIL: str = ""
+    MESH_ALLOW_LEGACY_DM_GET_UNTIL: str = ""
+    MESH_ALLOW_LEGACY_DM_SIGNATURE_COMPAT_UNTIL: str = ""
+    MESH_DM_PERSIST_SPOOL: bool = False
+    MESH_DM_RELAY_FILE_PATH: str = ""
+    MESH_DM_RELAY_AUTO_RELOAD: bool = False
+    MESH_DM_REQUIRE_SENDER_SEAL_SHARED: bool = True
+    MESH_DM_NONCE_TTL_S: int = 300
+    MESH_DM_NONCE_CACHE_MAX: int = 4096
+    MESH_DM_NONCE_PER_AGENT_MAX: int = 256
+    MESH_DM_REQUEST_MAX_AGE_S: int = 300
+    MESH_DM_REQUEST_MAILBOX_LIMIT: int = 12
+    MESH_DM_SHARED_MAILBOX_LIMIT: int = 48
+    MESH_DM_SELF_MAILBOX_LIMIT: int = 12
+    MESH_BLOCK_LEGACY_AGENT_ID_LOOKUP: bool = True
+    MESH_ALLOW_COMPAT_DM_INVITE_IMPORT: bool = False
+    MESH_ALLOW_COMPAT_DM_INVITE_IMPORT_UNTIL: str = ""
+    MESH_ALLOW_LEGACY_NODE_ID_COMPAT_UNTIL: str = ""
+    # Rotate voter-blinding salts on a rolling cadence so new reputation
+    # events do not reuse one forever-stable blinded identity.
+    MESH_VOTER_BLIND_SALT_ROTATE_DAYS: int = 30
+    # Keep historical salts long enough to cover live vote records, so
+    # duplicate-vote detection and wallet-cost accounting survive rotation.
+    MESH_VOTER_BLIND_SALT_GRACE_DAYS: int = 30
+    MESH_DM_MAX_MSG_BYTES: int = 8192
+    MESH_DM_ALLOW_SENDER_SEAL: bool = False
+    # TTL for DH key and prekey bundle registrations — stale entries are pruned.
+    MESH_DM_KEY_TTL_DAYS: int = 30
+    # TTL for invite-scoped prekey lookup aliases; shorter windows reduce
+    # long-lived relay linkage between opaque lookup handles and agent IDs.
+    MESH_DM_PREKEY_LOOKUP_ALIAS_TTL_DAYS: int = 14
+    # TTL for relay witness history; keep continuity metadata bounded instead
+    # of relying on a hidden hardcoded retention window.
+    MESH_DM_WITNESS_TTL_DAYS: int = 14
+    # TTL for mailbox binding metadata — shorter = smaller metadata footprint on disk.
+    MESH_DM_BINDING_TTL_DAYS: int = 3
+    # When False, mailbox bindings are memory-only (agents re-register on restart).
+    # Enable explicitly only if restart continuity is worth persisting DM graph metadata.
+    MESH_DM_METADATA_PERSIST: bool = False
+    # Second explicit opt-in for at-rest DM metadata persistence. This keeps a
+    # single boolean flip from silently writing mailbox graph metadata to disk.
+    MESH_DM_METADATA_PERSIST_ACKNOWLEDGE: bool = False
+    # Optional import path for externally managed root witness material packages.
+    # Relative paths resolve from the backend directory.
+    MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH: str = ""
+    # Optional URI for externally managed root witness material packages.
+    # Supports file:// and http(s):// sources; when set it overrides the local path.
+    MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_URI: str = ""
+    # Maximum acceptable age for externally sourced root witness packages.
+    # Strong DM trust fails closed when the imported package exported_at is older than this.
+    MESH_DM_ROOT_EXTERNAL_WITNESS_MAX_AGE_S: int = 3600
+    # Warning threshold for externally sourced root witness packages.
+    # When current external witness material reaches this age, operator health degrades to warning
+    # before the strong path eventually fails closed at MAX_AGE.
+    MESH_DM_ROOT_EXTERNAL_WITNESS_WARN_AGE_S: int = 2700
+    # Optional export path for the append-only stable-root transparency ledger.
+    # Relative paths resolve from the backend directory.
+    MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH: str = ""
+    # Optional URI used to read back and verify published transparency ledgers.
+    # Supports file:// and http(s):// sources.
+    MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI: str = ""
+    # Maximum acceptable age for externally read transparency ledgers.
+    # Strong DM trust fails closed when exported_at is older than this.
+    MESH_DM_ROOT_TRANSPARENCY_LEDGER_MAX_AGE_S: int = 3600
+    # Warning threshold for externally read transparency ledgers.
+    # When current external transparency readback reaches this age, operator health degrades to warning
+    # before the strong path eventually fails closed at MAX_AGE.
+    MESH_DM_ROOT_TRANSPARENCY_LEDGER_WARN_AGE_S: int = 2700
+    MESH_SCOPED_TOKENS: str = ""
+    # Deprecated legacy env vars kept for backward config compatibility only.
+    # Ordinary shipped gate flows keep MLS decrypt local; backend decrypt is
+    # reserved for explicit recovery reads.
+    MESH_GATE_BACKEND_DECRYPT_COMPAT: bool = False
+    MESH_GATE_BACKEND_DECRYPT_COMPAT_ACKNOWLEDGE: bool = False
+    MESH_BACKEND_GATE_DECRYPT_COMPAT: bool = False
+    # Deprecated legacy env vars kept for backward config compatibility only.
+    # Ordinary shipped gate flows keep compose/post local and submit encrypted
+    # payloads to the backend for sign/post only.
+    MESH_GATE_BACKEND_PLAINTEXT_COMPAT: bool = False
+    MESH_GATE_BACKEND_PLAINTEXT_COMPAT_ACKNOWLEDGE: bool = False
+    MESH_BACKEND_GATE_PLAINTEXT_COMPAT: bool = False
+    # Runtime gate for recovery envelopes. When off, per-gate
+    # envelope_recovery / envelope_always policies fail closed to
+    # envelope_disabled. Default True so the Reddit-like durable history
+    # model works out of the box: any member with the gate_secret can
+    # decrypt every envelope encrypted from the moment they had that key.
+    # Set MESH_GATE_RECOVERY_ENVELOPE_ENABLE=false to revert to MLS-only
+    # forward-secret behavior (your own history becomes unreadable after
+    # the sending ratchet advances).
+    MESH_GATE_RECOVERY_ENVELOPE_ENABLE: bool = True
+    MESH_GATE_RECOVERY_ENVELOPE_ENABLE_ACKNOWLEDGE: bool = True
+    # Durable gate plaintext retention is disabled by default. Enable only
+    # when the operator explicitly accepts the at-rest privacy tradeoff.
+    MESH_GATE_PLAINTEXT_PERSIST: bool = False
+    MESH_GATE_PLAINTEXT_PERSIST_ACKNOWLEDGE: bool = False
+    MESH_GATE_SESSION_ROTATE_MSGS: int = 50
+    MESH_GATE_SESSION_ROTATE_S: int = 3600
+    MESH_GATE_LEGACY_ENVELOPE_FALLBACK_MAX_DAYS: int = 30
+    # Add a randomized grace window before anonymous gate-session auto-rotation
+    # so threshold-triggered identity swaps are less trivially correlated.
+    MESH_GATE_SESSION_ROTATE_JITTER_S: int = 180
+    # Gate persona (named identity) rotation thresholds.  Rotating the signing
+    # key limits the linkability window.  Zero = disabled.
+    MESH_GATE_PERSONA_ROTATE_MSGS: int = 200
+    MESH_GATE_PERSONA_ROTATE_S: int = 604800  # 7 days
+    MESH_GATE_PERSONA_ROTATE_JITTER_S: int = 600
+    # Feature-flagged session stream for multiplexed gate room updates.
+    # Disabled by default so rollout stays explicit while stream-first rooms bake.
+    MESH_GATE_SESSION_STREAM_ENABLED: bool = False
+    MESH_GATE_SESSION_STREAM_HEARTBEAT_S: int = 20
+    MESH_GATE_SESSION_STREAM_BATCH_MS: int = 1500
+    MESH_GATE_SESSION_STREAM_MAX_GATES: int = 16
+    # Private gate APIs expose a backward-jittered timestamp view so observers
+    # cannot trivially align exact send times from response metadata alone.
+    MESH_GATE_TIMESTAMP_JITTER_S: int = 60
+    # Ban/kick gate-secret rotation is on by default (hardening Rec #10): the
+    # invariant has baked and a ban that does not rotate is effectively a
+    # display-only removal. Set MESH_GATE_BAN_KICK_ROTATION_ENABLE=false to
+    # revert to observe-only during incident triage.
+    MESH_GATE_BAN_KICK_ROTATION_ENABLE: bool = True
+    MESH_BLOCK_LEGACY_NODE_ID_COMPAT: bool = True
+    MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK: bool = False
+    MESH_ACK_RAW_FALLBACK_AT_OWN_RISK: bool = False
+    MESH_SECURE_STORAGE_SECRET: str = ""
+    MESH_SECURE_STORAGE_SECRET_FILE: str = ""
+    MESH_PRIVATE_LOG_TTL_S: int = 900
+    # Sprint 1 rollout: restored DM boot probes stay disabled by default until
+    # the architect reviews false positives from the observe-only path.
+    MESH_DM_RESTORED_SESSION_BOOT_PROBE_ENABLE: bool = False
+    # Queued DM release requires explicit per-item approval before any weaker
+    # relay fallback. Silent fallback is not a safe private-mode default.
+    MESH_PRIVATE_RELEASE_APPROVAL_ENABLE: bool = True
+    # Expiry for user-approved scoped private relay fallback policy. The policy
+    # is still bounded by hidden-transport checks before it can auto-release.
+    MESH_PRIVATE_RELAY_POLICY_TTL_S: int = 3600
+    # Background privacy prewarm prepares keys/aliases/transport readiness
+    # before send-time. Anonymous mode uses a cadence gate so user clicks do
+    # not directly create hidden-transport activity.
+    MESH_PRIVACY_PREWARM_ENABLE: bool = True
+    MESH_PRIVACY_PREWARM_INTERVAL_S: int = 300
+    MESH_PRIVACY_PREWARM_ANON_CADENCE_S: int = 300
+    # Sprint 4 rollout: authenticated RNS cover markers remain disabled until
+    # the observer-equivalence and receive-path DoS tests are green.
+    MESH_RNS_COVER_AUTH_MARKER_ENABLE: bool = False
+    # Signed-write revocation lookups use a short local TTL; stale entries force
+    # a local rebuild before honor. Offline/local-refresh failures remain
+    # observe-only until the later enforcement sprint.
+    MESH_SIGNED_REVOCATION_CACHE_TTL_S: int = 300
+    MESH_SIGNED_REVOCATION_CACHE_ENFORCE: bool = True
+    MESH_SIGNED_WRITE_CONTEXT_REQUIRED: bool = True
+    # Sprint 5 rollout: when enabled, root witness finality requires
+    # independent quorum for threshold>1 witnessed roots before they count as
+    # verified first-contact provenance.
+    WORMHOLE_ROOT_WITNESS_FINALITY_ENFORCE: bool = False
+    # Optional JSON artifact generated by CI/release workflow for the Sprint 8
+    # release gate. Relative paths resolve from the backend directory.
+    # dev = permissive local/dev behavior; testnet-private = strict private
+    # defaults; release-candidate = no compatibility/debug escape hatches.
+    MESH_RELEASE_PROFILE: str = "dev"
+    MESH_RELEASE_ATTESTATION_PATH: str = ""
+    # Operator release attestation for the Sprint 8 release gate. This does
+    # not change runtime behavior; it only records that the DM relay security
+    # suite was run and passed for the release candidate.
+    MESH_RELEASE_DM_RELAY_SECURITY_SUITE_GREEN: bool = False
+    PRIVACY_CORE_MIN_VERSION: str = "0.1.0"
+    PRIVACY_CORE_ALLOWED_SHA256: str = ""
+    PRIVACY_CORE_DEV_OVERRIDE: bool = False
+    # Sprint 4 rollout: fail fast when the loaded privacy-core artifact is
+    # missing required FFI symbols expected by the current Python bridge.
+    PRIVACY_CORE_EXPORT_SET_AUDIT_ENABLE: bool = True
+    # Clearnet fallback policy for private-tier messages.
+    # "block" (default) = refuse to send private messages over clearnet.
+    # "allow" = fall back to clearnet when Tor/RNS is unavailable (weaker privacy).
+    MESH_PRIVATE_CLEARNET_FALLBACK: str = "block"
+    # Second explicit opt-in for private-tier clearnet fallback. Without this
+    # acknowledgement, "allow" remains requested but not effective.
+    MESH_PRIVATE_CLEARNET_FALLBACK_ACKNOWLEDGE: bool = False
+    # Meshtastic MQTT bridge — disabled by default to avoid hammering the
+    # public broker.  Users opt in explicitly.
+    MESH_MQTT_ENABLED: bool = False
+    # Meshtastic MQTT broker credentials (defaults match public firmware).
+    MESH_MQTT_BROKER: str = "mqtt.meshtastic.org"
+    MESH_MQTT_PORT: int = 1883
+    MESH_MQTT_USER: str = "meshdev"
+    MESH_MQTT_PASS: str = "large4cats"
+    # Hex-encoded PSK — empty string means use the default LongFast key.
+    # Must decode to exactly 16 or 32 bytes when set.
+    MESH_MQTT_PSK: str = ""
+    # Optional operator-provided Meshtastic node ID (e.g. "!abcd1234") included
+    # in the User-Agent when fetching from meshtastic.liamcottle.net so the
+    # service operator can identify per-install traffic instead of a generic
+    # "ShadowBroker" aggregate.
+    MESHTASTIC_OPERATOR_CALLSIGN: str = ""
+
+    # SAR (Synthetic Aperture Radar) data layer
+    # Mode A — free catalog metadata, no account, default-on
+    MESH_SAR_CATALOG_ENABLED: bool = True
+    # Mode B — free pre-processed anomalies (OPERA / EGMS / GFM / EMS / UNOSAT)
+    # Two-step opt-in: must be "allow" AND _ACKNOWLEDGE must be true
+    MESH_SAR_PRODUCTS_FETCH: str = "block"
+    MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE: bool = False
+    # NASA Earthdata Login (free) — required for OPERA products
+    MESH_SAR_EARTHDATA_USER: str = ""
+    MESH_SAR_EARTHDATA_TOKEN: str = ""
+    # Copernicus Data Space (free) — required for EGMS / EMS products
+    MESH_SAR_COPERNICUS_USER: str = ""
+    MESH_SAR_COPERNICUS_TOKEN: str = ""
+    # Whether OpenClaw agents may read/act on the SAR layer
+    MESH_SAR_OPENCLAW_ENABLED: bool = True
+    # Require private-tier transport before signing/broadcasting SAR anomalies
+    MESH_SAR_REQUIRE_PRIVATE_TIER: bool = True
+
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore")
+
+
+@lru_cache
+def get_settings() -> Settings:
+    try:
+        from services.api_settings import load_persisted_api_keys_into_environ
+        load_persisted_api_keys_into_environ()
+    except Exception:
+        pass
+    return Settings()
+
+
+def private_clearnet_fallback_requested(settings: Settings | None = None) -> str:
+    snapshot = settings or get_settings()
+    policy = str(getattr(snapshot, "MESH_PRIVATE_CLEARNET_FALLBACK", "block") or "block").strip().lower()
+    return "allow" if policy == "allow" else "block"
+
+
+def private_clearnet_fallback_effective(settings: Settings | None = None) -> str:
+    snapshot = settings or get_settings()
+    requested = private_clearnet_fallback_requested(snapshot)
+    acknowledged = bool(getattr(snapshot, "MESH_PRIVATE_CLEARNET_FALLBACK_ACKNOWLEDGE", False))
+    if requested == "allow" and acknowledged:
+        return "allow"
+    return "block"
+
+
+def backend_gate_decrypt_compat_effective(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(
+        getattr(snapshot, "MESH_BACKEND_GATE_DECRYPT_COMPAT", False)
+        or getattr(snapshot, "MESH_GATE_BACKEND_DECRYPT_COMPAT", False)
+    )
+
+
+def backend_gate_plaintext_compat_effective(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(
+        getattr(snapshot, "MESH_BACKEND_GATE_PLAINTEXT_COMPAT", False)
+        or getattr(snapshot, "MESH_GATE_BACKEND_PLAINTEXT_COMPAT", False)
+    )
+
+
+def gate_recovery_envelope_effective(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    requested = bool(getattr(snapshot, "MESH_GATE_RECOVERY_ENVELOPE_ENABLE", False))
+    acknowledged = bool(getattr(snapshot, "MESH_GATE_RECOVERY_ENVELOPE_ENABLE_ACKNOWLEDGE", False))
+    return requested and acknowledged
+
+
+def gate_plaintext_persist_effective(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    requested = bool(getattr(snapshot, "MESH_GATE_PLAINTEXT_PERSIST", False))
+    acknowledged = bool(getattr(snapshot, "MESH_GATE_PLAINTEXT_PERSIST_ACKNOWLEDGE", False))
+    return requested and acknowledged
+
+
+def gate_ban_kick_rotation_enabled(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(getattr(snapshot, "MESH_GATE_BAN_KICK_ROTATION_ENABLE", False))
+
+
+def dm_restored_session_boot_probe_enabled(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(getattr(snapshot, "MESH_DM_RESTORED_SESSION_BOOT_PROBE_ENABLE", False))
+
+
+def signed_revocation_cache_ttl_s(settings: Settings | None = None) -> int:
+    snapshot = settings or get_settings()
+    return max(0, int(getattr(snapshot, "MESH_SIGNED_REVOCATION_CACHE_TTL_S", 300) or 0))
+
+
+def signed_revocation_cache_enforce(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(getattr(snapshot, "MESH_SIGNED_REVOCATION_CACHE_ENFORCE", False))
+
+
+def wormhole_root_witness_finality_enforce(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(getattr(snapshot, "WORMHOLE_ROOT_WITNESS_FINALITY_ENFORCE", False))

backend/services/constants.py

@@ -0,0 +1,34 @@
+# ─── ShadowBroker Backend Constants ──────────────────────────────────────────
+# Centralized magic numbers. Import from here instead of hardcoding.
+
+# ─── Flight Trails ──────────────────────────────────────────────────────────
+FLIGHT_TRAIL_MAX_TRACKED = 2000  # Max concurrent tracked trails before LRU eviction
+FLIGHT_TRAIL_POINTS_PER_FLIGHT = 200  # Max trail points kept per aircraft
+TRACKED_TRAIL_TTL_S = 1800  # 30 min - trail TTL for tracked flights
+DEFAULT_TRAIL_TTL_S = 300  # 5 min - trail TTL for non-tracked flights
+
+# ─── Detection Thresholds ──────────────────────────────────────────────────
+HOLD_PATTERN_DEGREES = 300  # Total heading change to flag holding pattern
+GPS_JAMMING_NACP_THRESHOLD = 8  # NACp below this = degraded GPS signal
+GPS_JAMMING_GRID_SIZE = 1.0  # 1 degree grid for aggregation
+GPS_JAMMING_MIN_RATIO = 0.30  # 30% degraded aircraft to flag zone
+GPS_JAMMING_MIN_AIRCRAFT = 5  # Min aircraft in grid cell for statistical significance
+
+# ─── Network & Circuit Breaker ──────────────────────────────────────────────
+CIRCUIT_BREAKER_TTL_S = 120  # Skip domain for 2 min after total failure
+DOMAIN_FAIL_TTL_S = 300  # Skip requests.get for 5 min, go straight to curl
+CONNECT_TIMEOUT_S = 3  # Short connect timeout for fast firewall-block detection
+
+# ─── Data Fetcher Intervals ────────────────────────────────────────────────
+FAST_FETCH_INTERVAL_S = 60  # Flights, ships, satellites, military
+SLOW_FETCH_INTERVAL_MIN = 30  # News, markets, space weather
+CCTV_FETCH_INTERVAL_MIN = 1  # CCTV camera pipeline
+LIVEUAMAP_FETCH_INTERVAL_HR = 12  # LiveUAMap scraper
+
+# ─── External API ──────────────────────────────────────────────────────────
+OPENSKY_RATE_LIMIT_S = 300  # Only re-fetch OpenSky every 5 minutes
+OPENSKY_REQUEST_TIMEOUT_S = 15  # Timeout for OpenSky API calls
+ROUTE_FETCH_TIMEOUT_S = 15  # Timeout for adsb.lol route lookups
+
+# ─── Internet Outage Detection ─────────────────────────────────────────────
+INTERNET_OUTAGE_MIN_SEVERITY = 0.10  # 10% drop minimum to show

backend/services/correlation_engine.py

@@ -0,0 +1,783 @@
+"""
+Emergent Intelligence — Cross-layer correlation engine.
+
+Scans co-located events across multiple data layers and emits composite
+alerts that no single source could generate alone.
+
+Correlation types:
+  - RF Anomaly:          GPS jamming + internet outage (both required)
+  - Military Buildup:    Military flights + naval vessels + GDELT conflict events
+  - Infrastructure Cascade: Internet outage + KiwiSDR offline in same zone
+  - Possible Contradiction: Official denial/statement + infrastructure disruption
+                            in same region — hypothesis generator, NOT verdict
+"""
+
+import logging
+import math
+import re
+from collections import defaultdict
+
+logger = logging.getLogger(__name__)
+
+# Grid cell size in degrees — 1° ≈ 111 km at equator.
+# Tighter than the previous 2° to reduce false co-locations.
+_CELL_SIZE = 1
+
+# Quality gates for RF anomaly correlation — only high-confidence inputs.
+# GPS jamming + internet outage overlap in a 111km cell is easily a coincidence
+# (IODA returns ~100 regional outages; GPS NACp dips are common in busy airspace).
+# Only fire when the evidence is strong enough to indicate deliberate RF interference.
+_RF_CORR_MIN_GPS_RATIO = 0.60   # Need strong jamming signal, not marginal NACp dips
+_RF_CORR_MIN_OUTAGE_PCT = 40    # Need a serious outage, not routine BGP fluctuation
+_RF_CORR_MIN_INDICATORS = 3     # Require 3+ corroborating signals (not just GPS+outage)
+
+
+def _cell_key(lat: float, lng: float) -> str:
+    """Convert lat/lng to a grid cell key."""
+    clat = int(lat // _CELL_SIZE) * _CELL_SIZE
+    clng = int(lng // _CELL_SIZE) * _CELL_SIZE
+    return f"{clat},{clng}"
+
+
+def _cell_center(key: str) -> tuple[float, float]:
+    """Get center lat/lng from a cell key."""
+    parts = key.split(",")
+    return float(parts[0]) + _CELL_SIZE / 2, float(parts[1]) + _CELL_SIZE / 2
+
+
+def _severity(indicator_count: int) -> str:
+    if indicator_count >= 3:
+        return "high"
+    if indicator_count >= 2:
+        return "medium"
+    return "low"
+
+
+def _severity_score(sev: str) -> float:
+    return {"high": 90, "medium": 60, "low": 30}.get(sev, 0)
+
+
+def _outage_pct(outage: dict) -> float:
+    """Extract outage severity percentage from an outage dict."""
+    return float(outage.get("severity", 0) or outage.get("severity_pct", 0) or 0)
+
+
+# ---------------------------------------------------------------------------
+# RF Anomaly: GPS jamming + internet outage (both must be present)
+# ---------------------------------------------------------------------------
+
+
+def _detect_rf_anomalies(data: dict) -> list[dict]:
+    gps_jamming = data.get("gps_jamming") or []
+    internet_outages = data.get("internet_outages") or []
+
+    if not gps_jamming:
+        return []  # No GPS jamming → no RF anomalies possible
+
+    # Build grid of indicators
+    cells: dict[str, dict] = defaultdict(lambda: {
+        "gps_jam": False, "gps_ratio": 0.0,
+        "outage": False, "outage_pct": 0.0,
+    })
+
+    for z in gps_jamming:
+        lat, lng = z.get("lat"), z.get("lng")
+        if lat is None or lng is None:
+            continue
+        ratio = z.get("ratio", 0)
+        if ratio < _RF_CORR_MIN_GPS_RATIO:
+            continue  # Skip marginal jamming zones
+        key = _cell_key(lat, lng)
+        cells[key]["gps_jam"] = True
+        cells[key]["gps_ratio"] = max(cells[key]["gps_ratio"], ratio)
+
+    for o in internet_outages:
+        lat = o.get("lat") or o.get("latitude")
+        lng = o.get("lng") or o.get("lon") or o.get("longitude")
+        if lat is None or lng is None:
+            continue
+        pct = _outage_pct(o)
+        if pct < _RF_CORR_MIN_OUTAGE_PCT:
+            continue  # Skip minor outages (ISP maintenance noise)
+        key = _cell_key(float(lat), float(lng))
+        cells[key]["outage"] = True
+        cells[key]["outage_pct"] = max(cells[key]["outage_pct"], pct)
+
+    # PSK Reporter: presence = healthy RF.  Only used as a bonus indicator,
+    # NOT as a standalone trigger (absence is normal in most cells).
+    psk_reporter = data.get("psk_reporter") or []
+    psk_cells: set[str] = set()
+    for s in psk_reporter:
+        lat, lng = s.get("lat"), s.get("lon")
+        if lat is not None and lng is not None:
+            psk_cells.add(_cell_key(lat, lng))
+
+    # When PSK data is unavailable, we can't get a 3rd indicator, so require
+    # an even higher GPS jamming ratio to compensate (real EW shows 75%+).
+    psk_available = len(psk_reporter) > 0
+
+    alerts: list[dict] = []
+    for key, c in cells.items():
+        # GPS jamming is the anchor — required for every RF anomaly alert
+        if not c["gps_jam"]:
+            continue
+        if not c["outage"]:
+            continue  # Both GPS jamming AND outage are always required
+
+        indicators = 2  # GPS jamming + outage
+        drivers: list[str] = [f"GPS jamming {int(c['gps_ratio'] * 100)}%"]
+        pct = c["outage_pct"]
+        drivers.append(f"Internet outage{f' {pct:.0f}%' if pct else ''}")
+
+        # PSK absence confirms RF environment is disrupted
+        if psk_available and key not in psk_cells:
+            indicators += 1
+            drivers.append("No HF digital activity (PSK Reporter)")
+
+        if indicators < _RF_CORR_MIN_INDICATORS:
+            # Without PSK data, only allow through if GPS ratio is extreme
+            # (75%+ indicates deliberate, sustained jamming — not noise)
+            if not psk_available and c["gps_ratio"] >= 0.75 and pct >= 50:
+                pass  # Allow this high-confidence 2-indicator alert through
+            else:
+                continue
+
+        lat, lng = _cell_center(key)
+        sev = _severity(indicators)
+        alerts.append({
+            "lat": lat,
+            "lng": lng,
+            "type": "rf_anomaly",
+            "severity": sev,
+            "score": _severity_score(sev),
+            "drivers": drivers[:3],
+            "cell_size": _CELL_SIZE,
+        })
+
+    return alerts
+
+
+# ---------------------------------------------------------------------------
+# Military Buildup: flights + ships + GDELT conflict
+# ---------------------------------------------------------------------------
+
+
+def _detect_military_buildups(data: dict) -> list[dict]:
+    mil_flights = data.get("military_flights") or []
+    ships = data.get("ships") or []
+    gdelt = data.get("gdelt") or []
+
+    cells: dict[str, dict] = defaultdict(lambda: {
+        "mil_flights": 0, "mil_ships": 0, "gdelt_events": 0,
+    })
+
+    for f in mil_flights:
+        lat = f.get("lat") or f.get("latitude")
+        lng = f.get("lng") or f.get("lon") or f.get("longitude")
+        if lat is None or lng is None:
+            continue
+        try:
+            key = _cell_key(float(lat), float(lng))
+            cells[key]["mil_flights"] += 1
+        except (ValueError, TypeError):
+            continue
+
+    mil_ship_types = {"military_vessel", "military", "warship", "patrol", "destroyer",
+                      "frigate", "corvette", "carrier", "submarine", "cruiser"}
+    for s in ships:
+        stype = (s.get("type") or s.get("ship_type") or "").lower()
+        if not any(mt in stype for mt in mil_ship_types):
+            continue
+        lat = s.get("lat") or s.get("latitude")
+        lng = s.get("lng") or s.get("lon") or s.get("longitude")
+        if lat is None or lng is None:
+            continue
+        try:
+            key = _cell_key(float(lat), float(lng))
+            cells[key]["mil_ships"] += 1
+        except (ValueError, TypeError):
+            continue
+
+    for g in gdelt:
+        lat = g.get("lat") or g.get("latitude") or g.get("actionGeo_Lat")
+        lng = g.get("lng") or g.get("lon") or g.get("longitude") or g.get("actionGeo_Long")
+        if lat is None or lng is None:
+            continue
+        try:
+            key = _cell_key(float(lat), float(lng))
+            cells[key]["gdelt_events"] += 1
+        except (ValueError, TypeError):
+            continue
+
+    alerts: list[dict] = []
+    for key, c in cells.items():
+        mil_total = c["mil_flights"] + c["mil_ships"]
+        has_gdelt = c["gdelt_events"] > 0
+
+        # Need meaningful military presence AND a conflict indicator
+        if mil_total < 3 or not has_gdelt:
+            continue
+
+        drivers: list[str] = []
+        if c["mil_flights"]:
+            drivers.append(f"{c['mil_flights']} military aircraft")
+        if c["mil_ships"]:
+            drivers.append(f"{c['mil_ships']} military vessels")
+        if c["gdelt_events"]:
+            drivers.append(f"{c['gdelt_events']} conflict events")
+
+        if mil_total >= 11:
+            sev = "high"
+        elif mil_total >= 6:
+            sev = "medium"
+        else:
+            sev = "low"
+
+        lat, lng = _cell_center(key)
+        alerts.append({
+            "lat": lat,
+            "lng": lng,
+            "type": "military_buildup",
+            "severity": sev,
+            "score": _severity_score(sev),
+            "drivers": drivers[:3],
+            "cell_size": _CELL_SIZE,
+        })
+
+    return alerts
+
+
+# ---------------------------------------------------------------------------
+# Infrastructure Cascade: outage + KiwiSDR co-location
+#
+# Power plants are removed from this detector — with 35K plants globally,
+# virtually every 2° cell contains one, making every outage a false hit.
+# KiwiSDR receivers (~300 worldwide) are sparse enough to be meaningful:
+# an outage in the same cell as a KiwiSDR indicates real infrastructure
+# disruption affecting radio monitoring capability.
+# ---------------------------------------------------------------------------
+
+
+def _detect_infra_cascades(data: dict) -> list[dict]:
+    internet_outages = data.get("internet_outages") or []
+    kiwisdr = data.get("kiwisdr") or []
+
+    if not kiwisdr:
+        return []
+
+    # Build set of cells with KiwiSDR receivers
+    kiwi_cells: set[str] = set()
+    for k in kiwisdr:
+        lat, lng = k.get("lat"), k.get("lon") or k.get("lng")
+        if lat is not None and lng is not None:
+            try:
+                kiwi_cells.add(_cell_key(float(lat), float(lng)))
+            except (ValueError, TypeError):
+                pass
+
+    if not kiwi_cells:
+        return []
+
+    alerts: list[dict] = []
+    for o in internet_outages:
+        lat = o.get("lat") or o.get("latitude")
+        lng = o.get("lng") or o.get("lon") or o.get("longitude")
+        if lat is None or lng is None:
+            continue
+        try:
+            key = _cell_key(float(lat), float(lng))
+        except (ValueError, TypeError):
+            continue
+
+        if key not in kiwi_cells:
+            continue
+
+        pct = _outage_pct(o)
+        drivers = [f"Internet outage{f' {pct:.0f}%' if pct else ''}",
+                    "KiwiSDR receivers in affected zone"]
+
+        lat_c, lng_c = _cell_center(key)
+        alerts.append({
+            "lat": lat_c,
+            "lng": lng_c,
+            "type": "infra_cascade",
+            "severity": "medium",
+            "score": _severity_score("medium"),
+            "drivers": drivers,
+            "cell_size": _CELL_SIZE,
+        })
+
+    return alerts
+
+
+# ---------------------------------------------------------------------------
+# Possible Contradiction: official denial/statement + infra disruption
+#
+# This is a HYPOTHESIS GENERATOR, not a verdict engine.  It says "LOOK HERE"
+# when an official statement (denial, clarification, refusal) co-locates with
+# infrastructure disruption (internet outage, sigint change).  The human or
+# higher-order reasoning decides what actually happened.
+#
+# Context ratings:
+#   STRONG   — denial + outage + prediction market movement in same region
+#   MODERATE — denial + outage (no market signal)
+#   WEAK     — denial + minor outage or distant co-location
+#   DETECTION_GAP — denial found but NO telemetry to verify (equally valuable)
+# ---------------------------------------------------------------------------
+
+# Denial / official-statement patterns in headlines and URL slugs
+_DENIAL_PATTERNS = [
+    re.compile(p, re.IGNORECASE) for p in [
+        r"\bden(?:y|ies|ied|ial)\b",
+        r"\brefut(?:e[ds]?|ing)\b",
+        r"\breject(?:s|ed|ing)?\b",
+        r"\bclarif(?:y|ies|ied|ication)\b",
+        r"\bdismiss(?:es|ed|ing)?\b",
+        r"\bno\s+attack\b",
+        r"\bdid\s+not\s+(?:attack|strike|bomb|target|order|invade|kill)\b",
+        r"\bnever\s+(?:attack|strike|bomb|target|order|invade|happen)\b",
+        r"\bfalse\s+(?:report|claim|allegation|rumor|narrative)\b",
+        r"\bmisinformation\b",
+        r"\bdisinformation\b",
+        r"\bpropaganda\b",
+        r"\b(?:army|military|government|ministry|official)\s+(?:says|clarifies|denies|refutes)\b",
+        r"\brumor[s]?\b.*\buntrue\b",
+        r"\bcategorically\b",
+        r"\bbaseless\b",
+    ]
+]
+
+# Broader cell radius for sparse telemetry regions (Africa, Central Asia, etc.)
+# These regions have fewer IODA/RIPE probes so outage data is sparser
+_SPARSE_REGIONS_LAT_RANGES = [
+    (-35, 37),   # Africa roughly
+    (25, 50),    # Central Asia band (when lng 40-90)
+]
+
+
+def _is_sparse_region(lat: float, lng: float) -> bool:
+    """Check if coordinates fall in a region with sparse telemetry coverage."""
+    # Africa
+    if -35 <= lat <= 37 and -20 <= lng <= 55:
+        return True
+    # Central Asia
+    if 25 <= lat <= 50 and 40 <= lng <= 90:
+        return True
+    # South America interior
+    if -55 <= lat <= 12 and -80 <= lng <= -35:
+        return True
+    return False
+
+
+def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
+    """Great-circle distance in km."""
+    R = 6371.0
+    dlat = math.radians(lat2 - lat1)
+    dlon = math.radians(lon2 - lon1)
+    a = (math.sin(dlat / 2) ** 2 +
+         math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) *
+         math.sin(dlon / 2) ** 2)
+    return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
+
+
+def _matches_denial(text: str) -> bool:
+    """Check if text matches any denial/official-statement pattern."""
+    return any(p.search(text) for p in _DENIAL_PATTERNS)
+
+
+def _detect_contradictions(data: dict) -> list[dict]:
+    """Detect possible contradictions between official statements and telemetry.
+
+    Scans GDELT headlines for denial language, then checks whether internet
+    outages or other infrastructure disruptions exist in the same geographic
+    region.  Scores confidence and lists alternative explanations.
+    """
+    gdelt = data.get("gdelt") or []
+    internet_outages = data.get("internet_outages") or []
+    news = data.get("news") or []
+    prediction_markets = data.get("prediction_markets") or []
+
+    # ── Step 1: Find GDELT events with denial/official-statement language ──
+    denial_events: list[dict] = []
+
+    # GDELT comes as GeoJSON features
+    gdelt_features = gdelt
+    if isinstance(gdelt, dict):
+        gdelt_features = gdelt.get("features", [])
+
+    for feature in gdelt_features:
+        # Handle both GeoJSON features and flat dicts
+        if "properties" in feature and "geometry" in feature:
+            props = feature.get("properties", {})
+            geom = feature.get("geometry", {})
+            coords = geom.get("coordinates", [])
+            if len(coords) >= 2:
+                lng, lat = float(coords[0]), float(coords[1])
+            else:
+                continue
+            headlines = props.get("_headlines_list", [])
+            urls = props.get("_urls_list", [])
+            name = props.get("name", "")
+            count = props.get("count", 1)
+        else:
+            lat = feature.get("lat") or feature.get("actionGeo_Lat")
+            lng = feature.get("lng") or feature.get("lon") or feature.get("actionGeo_Long")
+            if lat is None or lng is None:
+                continue
+            lat, lng = float(lat), float(lng)
+            headlines = [feature.get("title", "")]
+            urls = [feature.get("sourceurl", "")]
+            name = feature.get("name", "")
+            count = 1
+
+        # Check all headlines + URL slugs for denial patterns
+        all_text = " ".join(str(h) for h in headlines if h)
+        all_text += " " + " ".join(str(u) for u in urls if u)
+
+        if _matches_denial(all_text):
+            denial_events.append({
+                "lat": lat,
+                "lng": lng,
+                "headlines": [h for h in headlines if h][:5],
+                "urls": [u for u in urls if u][:3],
+                "location_name": name,
+                "event_count": count,
+            })
+
+    # Also scan news articles for denial language
+    for article in news:
+        title = str(article.get("title", "") or "")
+        desc = str(article.get("description", "") or article.get("summary", "") or "")
+        if not _matches_denial(title + " " + desc):
+            continue
+        # News articles often lack coordinates — try to match to GDELT locations
+        # For now, only include if we have coordinates
+        lat = article.get("lat") or article.get("latitude")
+        lng = article.get("lng") or article.get("lon") or article.get("longitude")
+        if lat is not None and lng is not None:
+            denial_events.append({
+                "lat": float(lat),
+                "lng": float(lng),
+                "headlines": [title],
+                "urls": [article.get("url") or article.get("link") or ""],
+                "location_name": "",
+                "event_count": 1,
+            })
+
+    if not denial_events:
+        return []
+
+    # ── Step 2: Cross-reference with internet outages ──
+    alerts: list[dict] = []
+
+    for denial in denial_events:
+        d_lat, d_lng = denial["lat"], denial["lng"]
+        sparse = _is_sparse_region(d_lat, d_lng)
+        search_radius_km = 1500.0 if sparse else 500.0
+
+        # Find nearby outages
+        nearby_outages: list[dict] = []
+        for outage in internet_outages:
+            o_lat = outage.get("lat") or outage.get("latitude")
+            o_lng = outage.get("lng") or outage.get("lon") or outage.get("longitude")
+            if o_lat is None or o_lng is None:
+                continue
+            try:
+                dist = _haversine_km(d_lat, d_lng, float(o_lat), float(o_lng))
+            except (ValueError, TypeError):
+                continue
+            if dist <= search_radius_km:
+                nearby_outages.append({
+                    "region": outage.get("region_name") or outage.get("country_name", ""),
+                    "severity": _outage_pct(outage),
+                    "distance_km": round(dist, 0),
+                    "level": outage.get("level", ""),
+                })
+
+        # ── Step 3: Check prediction markets for related movements ──
+        denial_text = " ".join(denial["headlines"]).lower()
+        related_markets: list[dict] = []
+        for market in prediction_markets:
+            m_title = str(market.get("title", "") or market.get("question", "") or "").lower()
+            # Look for keyword overlap between denial and market
+            denial_words = set(re.findall(r"[a-z]{4,}", denial_text))
+            market_words = set(re.findall(r"[a-z]{4,}", m_title))
+            overlap = denial_words & market_words - {"that", "this", "with", "from", "have", "been", "were", "will", "says", "said"}
+            if len(overlap) >= 2:
+                prob = market.get("probability") or market.get("lastTradePrice") or market.get("yes_price")
+                if prob is not None:
+                    related_markets.append({
+                        "title": market.get("title") or market.get("question"),
+                        "probability": float(prob),
+                    })
+
+        # ── Step 4: Score confidence and assign context rating ──
+        indicators = 1  # denial itself
+        drivers: list[str] = []
+
+        # Primary driver: the denial headline
+        headline_display = denial["headlines"][0] if denial["headlines"] else "Official statement"
+        if len(headline_display) > 80:
+            headline_display = headline_display[:77] + "..."
+        drivers.append(f'"{headline_display}"')
+
+        # Outage co-location
+        has_outage = False
+        if nearby_outages:
+            best_outage = max(nearby_outages, key=lambda o: o["severity"])
+            if best_outage["severity"] >= 10:
+                indicators += 1
+                has_outage = True
+                drivers.append(
+                    f"Internet outage {best_outage['severity']:.0f}% "
+                    f"({best_outage['region']}, {best_outage['distance_km']:.0f}km away)"
+                )
+            elif best_outage["severity"] > 0:
+                indicators += 0.5  # minor outage, partial indicator
+                has_outage = True
+                drivers.append(
+                    f"Minor outage ({best_outage['region']}, "
+                    f"{best_outage['distance_km']:.0f}km away)"
+                )
+
+        # Prediction market signal
+        has_market = False
+        if related_markets:
+            indicators += 1
+            has_market = True
+            top_market = related_markets[0]
+            drivers.append(
+                f"Market: \"{top_market['title'][:50]}\" "
+                f"at {top_market['probability']:.0%}"
+            )
+
+        # Multiple denial sources strengthen the signal
+        if denial["event_count"] > 1:
+            indicators += 0.5
+            drivers.append(f"{denial['event_count']} sources reporting")
+
+        # Context rating
+        if has_outage and has_market:
+            context = "STRONG"
+        elif has_outage:
+            context = "MODERATE"
+        elif has_market:
+            context = "WEAK"  # market signal without infra disruption
+        else:
+            context = "DETECTION_GAP"
+
+        # Severity mapping
+        if context == "STRONG":
+            sev = "high"
+        elif context == "MODERATE":
+            sev = "medium"
+        else:
+            sev = "low"
+
+        # Alternative explanations (always present — this is a hypothesis generator)
+        alternatives: list[str] = []
+        if has_outage:
+            alternatives.append("Routine infrastructure maintenance or cable damage")
+            alternatives.append("Weather-related outage coinciding with news cycle")
+        if not has_outage and context == "DETECTION_GAP":
+            alternatives.append("Statement may be truthful — no contradicting telemetry found")
+            alternatives.append("Telemetry coverage gap in this region")
+        alternatives.append("Denial may be responding to social media rumors, not real events")
+
+        lat_c, lng_c = _cell_center(_cell_key(d_lat, d_lng))
+        alerts.append({
+            "lat": lat_c,
+            "lng": lng_c,
+            "type": "contradiction",
+            "severity": sev,
+            "score": _severity_score(sev),
+            "drivers": drivers[:4],
+            "cell_size": _CELL_SIZE,
+            "context": context,
+            "alternatives": alternatives[:3],
+            "location_name": denial.get("location_name", ""),
+            "headlines": denial["headlines"][:3],
+            "related_markets": related_markets[:3],
+            "nearby_outages": nearby_outages[:5],
+        })
+
+    # Deduplicate: keep highest-scored alert per cell
+    seen_cells: dict[str, dict] = {}
+    for alert in alerts:
+        key = _cell_key(alert["lat"], alert["lng"])
+        if key not in seen_cells or alert["score"] > seen_cells[key]["score"]:
+            seen_cells[key] = alert
+
+    result = list(seen_cells.values())
+    if result:
+        by_context = defaultdict(int)
+        for a in result:
+            by_context[a["context"]] += 1
+        logger.info(
+            "Contradictions: %d possible (%s)",
+            len(result),
+            ", ".join(f"{v} {k}" for k, v in sorted(by_context.items())),
+        )
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+# ---------------------------------------------------------------------------
+# Correlation → Pin bridge
+# ---------------------------------------------------------------------------
+
+# Types and their pin categories
+_CORR_PIN_CATEGORIES = {
+    "rf_anomaly": "anomaly",
+    "military_buildup": "military",
+    "infra_cascade": "infrastructure",
+    "contradiction": "research",
+}
+
+# Deduplicate: don't re-pin the same cell within this window (seconds).
+_CORR_PIN_DEDUP_WINDOW = 600  # 10 minutes
+_recent_corr_pins: dict[str, float] = {}
+
+
+def _auto_pin_correlations(alerts: list[dict]) -> int:
+    """Create AI Intel pins for high-severity correlation alerts.
+
+    Only pins alerts with severity >= medium.  Uses cell-key dedup so the
+    same grid cell doesn't get re-pinned every fetch cycle.
+
+    Returns the number of pins created this cycle.
+    """
+    import time as _time
+
+    now = _time.time()
+
+    # Evict stale dedup entries
+    expired = [k for k, ts in _recent_corr_pins.items() if now - ts > _CORR_PIN_DEDUP_WINDOW]
+    for k in expired:
+        _recent_corr_pins.pop(k, None)
+
+    created = 0
+    for alert in alerts:
+        sev = alert.get("severity", "low")
+        if sev == "low":
+            continue  # Don't pin low-severity noise
+
+        lat = alert.get("lat")
+        lng = alert.get("lng")
+        if lat is None or lng is None:
+            continue
+
+        # Dedup key: type + cell
+        dedup_key = f"{alert['type']}:{_cell_key(lat, lng)}"
+        if dedup_key in _recent_corr_pins:
+            continue
+
+        category = _CORR_PIN_CATEGORIES.get(alert["type"], "anomaly")
+        drivers = alert.get("drivers", [])
+        atype = alert["type"]
+
+        if atype == "contradiction":
+            ctx = alert.get("context", "")
+            label = f"[{ctx}] Possible Contradiction"
+            parts = list(drivers)
+            if alert.get("alternatives"):
+                parts.append("Alternatives: " + "; ".join(alert["alternatives"][:2]))
+            description = " | ".join(parts) if parts else "Narrative contradiction detected"
+        else:
+            label = f"[{sev.upper()}] {atype.replace('_', ' ').title()}"
+            description = "; ".join(drivers) if drivers else "Multi-layer correlation alert"
+
+        try:
+            from services.ai_pin_store import create_pin
+
+            meta = {
+                "correlation_type": atype,
+                "severity": sev,
+                "drivers": drivers,
+                "cell_size": alert.get("cell_size", _CELL_SIZE),
+            }
+            # Add contradiction-specific metadata
+            if atype == "contradiction":
+                meta["context_rating"] = alert.get("context", "")
+                meta["alternatives"] = alert.get("alternatives", [])
+                meta["headlines"] = alert.get("headlines", [])
+                meta["location_name"] = alert.get("location_name", "")
+                if alert.get("related_markets"):
+                    meta["related_markets"] = alert["related_markets"]
+
+            create_pin(
+                lat=lat,
+                lng=lng,
+                label=label,
+                category=category,
+                description=description,
+                source="correlation_engine",
+                confidence=alert.get("score", 60) / 100.0,
+                ttl_hours=2.0,  # Auto-expire correlation pins after 2 hours
+                metadata=meta,
+            )
+            _recent_corr_pins[dedup_key] = now
+            created += 1
+        except Exception as exc:
+            logger.warning("Failed to auto-pin correlation: %s", exc)
+
+    if created:
+        logger.info("Correlation engine auto-pinned %d alerts", created)
+    return created
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def compute_correlations(data: dict) -> list[dict]:
+    """Run all correlation detectors and return merged alert list."""
+    alerts: list[dict] = []
+
+    try:
+        alerts.extend(_detect_rf_anomalies(data))
+    except Exception as e:
+        logger.error("Correlation engine RF anomaly error: %s", e)
+
+    try:
+        alerts.extend(_detect_military_buildups(data))
+    except Exception as e:
+        logger.error("Correlation engine military buildup error: %s", e)
+
+    try:
+        alerts.extend(_detect_infra_cascades(data))
+    except Exception as e:
+        logger.error("Correlation engine infra cascade error: %s", e)
+
+    # Contradiction detection removed from automated engine — too many false
+    # positives from regex headline matching.  Contradiction/analysis alerts are
+    # now placed by OpenClaw agents via place_analysis_zone, which lets an LLM
+    # reason about the evidence rather than pattern-matching keywords.
+    try:
+        from services.analysis_zone_store import get_live_zones
+        alerts.extend(get_live_zones())
+    except Exception as e:
+        logger.error("Analysis zone merge error: %s", e)
+
+    rf = sum(1 for a in alerts if a["type"] == "rf_anomaly")
+    mil = sum(1 for a in alerts if a["type"] == "military_buildup")
+    infra = sum(1 for a in alerts if a["type"] == "infra_cascade")
+    contra = sum(1 for a in alerts if a["type"] == "contradiction")
+    if alerts:
+        logger.info(
+            "Correlations: %d alerts (%d rf, %d mil, %d infra, %d contra)",
+            len(alerts), rf, mil, infra, contra,
+        )
+
+    # Correlation alerts are returned in the correlations data feed only.
+    # They are NOT auto-pinned to AI Intel — that layer is reserved for
+    # user / OpenClaw pins.  Correlations are visualised via the dedicated
+    # correlations overlay on the map.
+
+    return alerts

backend/services/feed_ingester.py

@@ -0,0 +1,238 @@
+"""Feed Ingester — background daemon that refreshes feed-backed pin layers.
+
+Layers with a non-empty `feed_url` are polled at their `feed_interval`
+(seconds, minimum 60). The feed is expected to return either:
+
+  1. GeoJSON FeatureCollection — features are converted to pins
+  2. JSON array of pin objects — used directly
+
+Each refresh atomically replaces the layer's pins with the new data.
+"""
+
+import logging
+import threading
+import time
+from typing import Any
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# State
+# ---------------------------------------------------------------------------
+
+_running = False
+_thread: threading.Thread | None = None
+_CHECK_INTERVAL = 30  # seconds between scanning for layers that need refresh
+_last_fetched: dict[str, float] = {}  # layer_id → last fetch timestamp
+_FETCH_TIMEOUT = 20  # seconds
+
+# ---------------------------------------------------------------------------
+# GeoJSON → pin conversion
+# ---------------------------------------------------------------------------
+
+
+def _geojson_features_to_pins(features: list[dict]) -> list[dict[str, Any]]:
+    """Convert GeoJSON Feature objects to pin dicts."""
+    pins: list[dict[str, Any]] = []
+    for feat in features:
+        if not isinstance(feat, dict):
+            continue
+        geom = feat.get("geometry") or {}
+        props = feat.get("properties") or {}
+
+        # Extract coordinates
+        coords = geom.get("coordinates")
+        if geom.get("type") != "Point" or not coords or len(coords) < 2:
+            continue
+
+        lng, lat = float(coords[0]), float(coords[1])
+        if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+            continue
+
+        pin: dict[str, Any] = {
+            "lat": lat,
+            "lng": lng,
+            "label": str(props.get("label", props.get("name", props.get("title", ""))))[:200],
+            "category": str(props.get("category", "custom"))[:50],
+            "color": str(props.get("color", ""))[:20],
+            "description": str(props.get("description", props.get("summary", "")))[:2000],
+            "source": "feed",
+            "source_url": str(props.get("source_url", props.get("url", props.get("link", ""))))[:500],
+            "confidence": float(props.get("confidence", 1.0)),
+        }
+
+        # Entity attachment if present
+        entity_type = props.get("entity_type", "")
+        entity_id = props.get("entity_id", "")
+        if entity_type and entity_id:
+            pin["entity_attachment"] = {
+                "entity_type": str(entity_type),
+                "entity_id": str(entity_id),
+                "entity_label": str(props.get("entity_label", "")),
+            }
+
+        pins.append(pin)
+    return pins
+
+
+def _parse_feed_response(data: Any) -> list[dict[str, Any]]:
+    """Parse a feed response into a list of pin dicts."""
+    if isinstance(data, dict):
+        # GeoJSON FeatureCollection
+        if data.get("type") == "FeatureCollection" and isinstance(data.get("features"), list):
+            return _geojson_features_to_pins(data["features"])
+        # Single Feature
+        if data.get("type") == "Feature":
+            return _geojson_features_to_pins([data])
+        # Wrapped response like {"ok": true, "data": [...]}
+        inner = data.get("data") or data.get("results") or data.get("pins") or data.get("items")
+        if isinstance(inner, list):
+            return _normalize_pin_list(inner)
+
+    if isinstance(data, list):
+        # Check if first item looks like a GeoJSON Feature
+        if data and isinstance(data[0], dict) and data[0].get("type") == "Feature":
+            return _geojson_features_to_pins(data)
+        return _normalize_pin_list(data)
+
+    return []
+
+
+def _normalize_pin_list(items: list) -> list[dict[str, Any]]:
+    """Normalize a list of raw pin objects, ensuring lat/lng are present."""
+    pins: list[dict[str, Any]] = []
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        lat = item.get("lat") or item.get("latitude")
+        lng = item.get("lng") or item.get("lon") or item.get("longitude")
+        if lat is None or lng is None:
+            continue
+        try:
+            lat, lng = float(lat), float(lng)
+        except (ValueError, TypeError):
+            continue
+        if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+            continue
+
+        pin: dict[str, Any] = {
+            "lat": lat,
+            "lng": lng,
+            "label": str(item.get("label", item.get("name", item.get("title", ""))))[:200],
+            "category": str(item.get("category", "custom"))[:50],
+            "color": str(item.get("color", ""))[:20],
+            "description": str(item.get("description", item.get("summary", "")))[:2000],
+            "source": "feed",
+            "source_url": str(item.get("source_url", item.get("url", item.get("link", ""))))[:500],
+            "confidence": float(item.get("confidence", 1.0)),
+        }
+
+        entity_type = item.get("entity_type", "")
+        entity_id = item.get("entity_id", "")
+        if entity_type and entity_id:
+            pin["entity_attachment"] = {
+                "entity_type": str(entity_type),
+                "entity_id": str(entity_id),
+                "entity_label": str(item.get("entity_label", "")),
+            }
+
+        pins.append(pin)
+    return pins
+
+
+# ---------------------------------------------------------------------------
+# Fetch a single layer
+# ---------------------------------------------------------------------------
+
+
+def _fetch_layer_feed(layer: dict[str, Any]) -> None:
+    """Fetch a feed URL and replace the layer's pins."""
+    layer_id = layer["id"]
+    feed_url = layer["feed_url"]
+    layer_name = layer.get("name", layer_id)
+
+    try:
+        resp = requests.get(
+            feed_url,
+            timeout=_FETCH_TIMEOUT,
+            headers={"User-Agent": "ShadowBroker-FeedIngester/1.0"},
+        )
+        resp.raise_for_status()
+        data = resp.json()
+    except requests.RequestException as e:
+        logger.warning("Feed fetch failed for layer '%s' (%s): %s", layer_name, feed_url, e)
+        return
+    except (ValueError, TypeError) as e:
+        logger.warning("Feed parse failed for layer '%s' (%s): %s", layer_name, feed_url, e)
+        return
+
+    pins = _parse_feed_response(data)
+
+    from services.ai_pin_store import replace_layer_pins, update_layer
+    count = replace_layer_pins(layer_id, pins)
+
+    # Update layer metadata with last_fetched timestamp
+    update_layer(layer_id, feed_last_fetched=time.time())
+
+    _last_fetched[layer_id] = time.time()
+    logger.info("Feed refresh for layer '%s': %d pins from %s", layer_name, count, feed_url)
+
+
+# ---------------------------------------------------------------------------
+# Main loop
+# ---------------------------------------------------------------------------
+
+
+def _ingest_loop() -> None:
+    """Daemon loop: scan for feed layers and refresh those that are due."""
+    while _running:
+        try:
+            from services.ai_pin_store import get_feed_layers
+
+            layers = get_feed_layers()
+            now = time.time()
+
+            for layer in layers:
+                layer_id = layer["id"]
+                interval = max(60, layer.get("feed_interval", 300))
+                last = _last_fetched.get(layer_id, 0)
+
+                if now - last >= interval:
+                    try:
+                        _fetch_layer_feed(layer)
+                    except Exception as e:
+                        logger.warning("Feed ingestion error for layer %s: %s",
+                                       layer.get("name", layer_id), e)
+
+        except Exception as e:
+            logger.error("Feed ingester loop error: %s", e)
+
+        # Sleep in short increments so we can stop cleanly
+        for _ in range(int(_CHECK_INTERVAL)):
+            if not _running:
+                break
+            time.sleep(1)
+
+
+# ---------------------------------------------------------------------------
+# Start / stop
+# ---------------------------------------------------------------------------
+
+
+def start_feed_ingester() -> None:
+    """Start the feed ingester daemon thread."""
+    global _running, _thread
+    if _thread and _thread.is_alive():
+        return
+    _running = True
+    _thread = threading.Thread(target=_ingest_loop, daemon=True, name="feed-ingester")
+    _thread.start()
+    logger.info("Feed ingester daemon started (check interval=%ds)", _CHECK_INTERVAL)
+
+
+def stop_feed_ingester() -> None:
+    """Stop the feed ingester daemon."""
+    global _running
+    _running = False

backend/services/fetch_health.py

@@ -0,0 +1,94 @@
+"""Fetch health registry — tracks per-source success/failure counts and timings."""
+
+import logging
+import threading
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+from services.fetchers._store import _data_lock, source_freshness
+
+logger = logging.getLogger(__name__)
+
+_health: Dict[str, Dict[str, Any]] = {}
+_lock = threading.Lock()
+
+
+def _now_iso() -> str:
+    return datetime.utcnow().isoformat()
+
+
+def _update_source_freshness(source: str, *, ok: bool, error_msg: Optional[str] = None):
+    """Mirror health summary into shared store for visibility."""
+    with _data_lock:
+        entry = source_freshness.get(source, {})
+        if ok:
+            entry["last_ok"] = _now_iso()
+        else:
+            entry["last_error"] = _now_iso()
+            if error_msg:
+                entry["last_error_msg"] = error_msg[:200]
+        source_freshness[source] = entry
+
+
+def record_success(source: str, duration_s: Optional[float] = None, count: Optional[int] = None):
+    """Record a successful fetch for a source."""
+    now = _now_iso()
+    with _lock:
+        entry = _health.setdefault(
+            source,
+            {
+                "ok_count": 0,
+                "error_count": 0,
+                "last_ok": None,
+                "last_error": None,
+                "last_error_msg": None,
+                "last_duration_ms": None,
+                "avg_duration_ms": None,
+                "last_count": None,
+            },
+        )
+        entry["ok_count"] += 1
+        entry["last_ok"] = now
+        if duration_s is not None:
+            dur_ms = round(duration_s * 1000, 1)
+            entry["last_duration_ms"] = dur_ms
+            prev_avg = entry["avg_duration_ms"] or 0.0
+            n = entry["ok_count"]
+            entry["avg_duration_ms"] = round(((prev_avg * (n - 1)) + dur_ms) / n, 1)
+        if count is not None:
+            entry["last_count"] = count
+
+    _update_source_freshness(source, ok=True)
+
+
+def record_failure(source: str, error: Exception, duration_s: Optional[float] = None):
+    """Record a failed fetch for a source."""
+    now = _now_iso()
+    err_msg = str(error)
+    with _lock:
+        entry = _health.setdefault(
+            source,
+            {
+                "ok_count": 0,
+                "error_count": 0,
+                "last_ok": None,
+                "last_error": None,
+                "last_error_msg": None,
+                "last_duration_ms": None,
+                "avg_duration_ms": None,
+                "last_count": None,
+            },
+        )
+        entry["error_count"] += 1
+        entry["last_error"] = now
+        entry["last_error_msg"] = err_msg[:200]
+        if duration_s is not None:
+            entry["last_duration_ms"] = round(duration_s * 1000, 1)
+
+    _update_source_freshness(source, ok=False, error_msg=err_msg)
+
+
+def get_health_snapshot() -> Dict[str, Dict[str, Any]]:
+    """Return a snapshot of current fetch health state."""
+    with _lock:
+        return {k: dict(v) for k, v in _health.items()}

backend/services/fetchers/_store.py

@@ -0,0 +1,328 @@
+"""Shared in-memory data store for all fetcher modules.
+
+Central location for latest_data, source_timestamps, and the data lock.
+Every fetcher imports from here instead of maintaining its own copy.
+"""
+
+import copy
+import threading
+import logging
+from datetime import datetime
+from typing import Any, Dict, List, Optional, TypedDict
+
+logger = logging.getLogger("services.data_fetcher")
+
+
+class DashboardData(TypedDict, total=False):
+    """Schema for the in-memory data store. Catches key typos at dev time."""
+
+    last_updated: Optional[str]
+    news: List[Dict[str, Any]]
+    stocks: Dict[str, Any]
+    oil: Dict[str, Any]
+    commercial_flights: List[Dict[str, Any]]
+    private_flights: List[Dict[str, Any]]
+    private_jets: List[Dict[str, Any]]
+    flights: List[Dict[str, Any]]
+    ships: List[Dict[str, Any]]
+    military_flights: List[Dict[str, Any]]
+    tracked_flights: List[Dict[str, Any]]
+    cctv: List[Dict[str, Any]]
+    weather: Optional[Dict[str, Any]]
+    earthquakes: List[Dict[str, Any]]
+    uavs: List[Dict[str, Any]]
+    frontlines: Optional[Any]
+    gdelt: List[Dict[str, Any]]
+    liveuamap: List[Dict[str, Any]]
+    kiwisdr: List[Dict[str, Any]]
+    space_weather: Optional[Dict[str, Any]]
+    internet_outages: List[Dict[str, Any]]
+    firms_fires: List[Dict[str, Any]]
+    datacenters: List[Dict[str, Any]]
+    airports: List[Dict[str, Any]]
+    gps_jamming: List[Dict[str, Any]]
+    satellites: List[Dict[str, Any]]
+    satellite_source: str
+    satellite_analysis: Dict[str, Any]
+    prediction_markets: List[Dict[str, Any]]
+    sigint: List[Dict[str, Any]]
+    sigint_totals: Dict[str, Any]
+    mesh_channel_stats: Dict[str, Any]
+    meshtastic_map_nodes: List[Dict[str, Any]]
+    meshtastic_map_fetched_at: Optional[float]
+    weather_alerts: List[Dict[str, Any]]
+    air_quality: List[Dict[str, Any]]
+    volcanoes: List[Dict[str, Any]]
+    fishing_activity: List[Dict[str, Any]]
+    satnogs_stations: List[Dict[str, Any]]
+    satnogs_observations: List[Dict[str, Any]]
+    tinygs_satellites: List[Dict[str, Any]]
+    ukraine_alerts: List[Dict[str, Any]]
+    power_plants: List[Dict[str, Any]]
+    viirs_change_nodes: List[Dict[str, Any]]
+    fimi: Dict[str, Any]
+    psk_reporter: List[Dict[str, Any]]
+    correlations: List[Dict[str, Any]]
+    uap_sightings: List[Dict[str, Any]]
+    wastewater: List[Dict[str, Any]]
+    crowdthreat: List[Dict[str, Any]]
+    sar_scenes: List[Dict[str, Any]]
+    sar_anomalies: List[Dict[str, Any]]
+    sar_aoi_coverage: List[Dict[str, Any]]
+
+
+# In-memory store
+latest_data: DashboardData = {
+    "last_updated": None,
+    "news": [],
+    "stocks": {},
+    "oil": {},
+    "flights": [],
+    "ships": [],
+    "military_flights": [],
+    "tracked_flights": [],
+    "cctv": [],
+    "weather": None,
+    "earthquakes": [],
+    "uavs": [],
+    "frontlines": None,
+    "gdelt": [],
+    "liveuamap": [],
+    "kiwisdr": [],
+    "space_weather": None,
+    "internet_outages": [],
+    "firms_fires": [],
+    "datacenters": [],
+    "military_bases": [],
+    "prediction_markets": [],
+    "sigint": [],
+    "sigint_totals": {},
+    "mesh_channel_stats": {},
+    "meshtastic_map_nodes": [],
+    "meshtastic_map_fetched_at": None,
+    "weather_alerts": [],
+    "air_quality": [],
+    "volcanoes": [],
+    "fishing_activity": [],
+    "satnogs_stations": [],
+    "satnogs_observations": [],
+    "tinygs_satellites": [],
+    "ukraine_alerts": [],
+    "power_plants": [],
+    "viirs_change_nodes": [],
+    "fimi": {},
+    "psk_reporter": [],
+    "correlations": [],
+    "uap_sightings": [],
+    "wastewater": [],
+    "crowdthreat": [],
+    "sar_scenes": [],
+    "sar_anomalies": [],
+    "sar_aoi_coverage": [],
+}
+
+# Per-source freshness timestamps
+source_timestamps = {}
+
+# Per-source health/freshness metadata (last ok/error)
+source_freshness: dict[str, dict] = {}
+
+
+def _mark_fresh(*keys):
+    """Record the current UTC time for one or more data source keys."""
+    now = datetime.utcnow().isoformat()
+    global _data_version
+    changed: list[tuple[str, int, int]] = []  # (layer, version, count)
+    with _data_lock:
+        for k in keys:
+            source_timestamps[k] = now
+            _layer_versions[k] = _layer_versions.get(k, 0) + 1
+            # Grab entity count while we hold the lock (cheap len())
+            val = latest_data.get(k)
+            count = len(val) if isinstance(val, list) else (1 if val is not None else 0)
+            changed.append((k, _layer_versions[k], count))
+        # Publish partial fetch progress immediately so the frontend can
+        # observe newly available data without waiting for the entire tier.
+        _data_version += 1
+    # Notify SSE listeners outside the lock to avoid deadlocks
+    _notify_layer_change(changed)
+
+
+# Thread lock for safe reads/writes to latest_data
+_data_lock = threading.Lock()
+
+# Monotonic version counter — incremented on each data update cycle.
+# Used for cheap ETag generation instead of MD5-hashing the full response.
+_data_version: int = 0
+
+# Per-layer version counters — incremented only when that specific layer
+# refreshes.  Used by get_layer_slice for per-layer incremental updates
+# and by the SSE stream to push targeted layer_changed notifications.
+_layer_versions: dict[str, int] = {}
+
+
+# ---------------------------------------------------------------------------
+# Layer-change notification callbacks (thread → async SSE bridge)
+# ---------------------------------------------------------------------------
+_layer_change_callbacks: list = []
+_layer_change_callbacks_lock = threading.Lock()
+
+
+def register_layer_change_callback(callback) -> None:
+    """Register a callback invoked on every _mark_fresh().
+
+    Signature: callback(layer: str, version: int, count: int)
+    Called from fetcher threads — must be thread-safe.
+    """
+    with _layer_change_callbacks_lock:
+        _layer_change_callbacks.append(callback)
+
+
+def unregister_layer_change_callback(callback) -> None:
+    """Remove a previously registered callback."""
+    with _layer_change_callbacks_lock:
+        try:
+            _layer_change_callbacks.remove(callback)
+        except ValueError:
+            pass
+
+
+def _notify_layer_change(changed: list[tuple[str, int, int]]) -> None:
+    """Fire all registered callbacks for each changed layer."""
+    with _layer_change_callbacks_lock:
+        cbs = list(_layer_change_callbacks)
+    for cb in cbs:
+        for layer, version, count in changed:
+            try:
+                cb(layer, version, count)
+            except Exception:
+                pass
+
+
+def get_layer_versions() -> dict[str, int]:
+    """Return a snapshot of all per-layer version counters."""
+    with _data_lock:
+        return dict(_layer_versions)
+
+
+def get_layer_version(layer: str) -> int:
+    """Return the version counter for a single layer (0 if never refreshed)."""
+    with _data_lock:
+        return _layer_versions.get(layer, 0)
+
+
+def bump_data_version() -> None:
+    """Increment the data version counter after a fetch cycle completes."""
+    global _data_version
+    with _data_lock:
+        _data_version += 1
+
+
+def get_data_version() -> int:
+    """Return the current data version (for ETag generation)."""
+    with _data_lock:
+        return _data_version
+
+
+_active_layers_version: int = 0
+
+
+def bump_active_layers_version() -> None:
+    """Increment the active-layer version when frontend toggles change response shape."""
+    global _active_layers_version
+    _active_layers_version += 1
+
+
+def get_active_layers_version() -> int:
+    """Return the current active-layer version (for ETag generation)."""
+    return _active_layers_version
+
+
+def get_latest_data_subset(*keys: str) -> DashboardData:
+    """Return a deep snapshot of only the requested top-level keys.
+
+    This avoids cloning the entire dashboard store for endpoints that only need
+    a small tier-specific subset.  Deep copy ensures callers cannot mutate
+    nested structures (e.g. individual flight dicts) and affect the live store.
+    """
+    with _data_lock:
+        snap: DashboardData = {}
+        for key in keys:
+            value = latest_data.get(key)
+            snap[key] = copy.deepcopy(value)
+        return snap
+
+
+def get_latest_data_subset_refs(*keys: str) -> DashboardData:
+    """Return direct top-level references for read-only hot paths.
+
+    Writers replace top-level values under the lock instead of mutating them
+    in place, so readers can safely use these references after releasing the
+    lock as long as they do not modify them.
+    """
+    with _data_lock:
+        snap: DashboardData = {}
+        for key in keys:
+            snap[key] = latest_data.get(key)
+        return snap
+
+
+def get_source_timestamps_snapshot() -> dict[str, str]:
+    """Return a stable copy of per-source freshness timestamps."""
+    with _data_lock:
+        return dict(source_timestamps)
+
+
+# ---------------------------------------------------------------------------
+# Active layers — frontend POSTs toggles, fetchers check before running.
+# Keep these aligned with the dashboard's default layer state so startup does
+# not fetch heavyweight feeds the UI starts with disabled.
+# ---------------------------------------------------------------------------
+active_layers: dict[str, bool] = {
+    "flights": True,
+    "private": True,
+    "jets": True,
+    "military": True,
+    "tracked": True,
+    "satellites": True,
+    "ships_military": True,
+    "ships_cargo": True,
+    "ships_civilian": True,
+    "ships_passenger": True,
+    "ships_tracked_yachts": True,
+    "earthquakes": True,
+    "cctv": True,
+    "ukraine_frontline": True,
+    "global_incidents": True,
+    "gps_jamming": True,
+    "kiwisdr": True,
+    "scanners": True,
+    "firms": True,
+    "internet_outages": True,
+    "datacenters": True,
+    "military_bases": True,
+    "sigint_meshtastic": True,
+    "sigint_aprs": True,
+    "weather_alerts": True,
+    "air_quality": True,
+    "volcanoes": True,
+    "fishing_activity": True,
+    "satnogs": True,
+    "tinygs": True,
+    "ukraine_alerts": True,
+    "power_plants": True,
+    "viirs_nightlights": False,
+    "psk_reporter": True,
+    "correlations": True,
+    "contradictions": True,
+    "uap_sightings": True,
+    "wastewater": True,
+    "ai_intel": True,
+    "crowdthreat": True,
+    "sar": True,
+}
+
+
+def is_any_active(*layer_names: str) -> bool:
+    """Return True if any of the given layer names is currently active."""
+    return any(active_layers.get(name, True) for name in layer_names)

backend/services/fetchers/aircraft_database.py

@@ -0,0 +1,177 @@
+"""OpenSky aircraft metadata: ICAO24 hex -> ICAO type code + friendly model.
+
+OpenSky's /states/all does not include aircraft type, so OpenSky-sourced
+flights arrive with ``t`` field empty. This module bulk-loads the public
+OpenSky aircraft database (one snapshot CSV per month, ~108 MB uncompressed,
+~600k aircraft) once every 5 days and exposes a fast in-memory hex lookup.
+
+The data is also useful when adsb.lol's live API is degraded: even the
+adsb.lol /v2 feed sometimes returns aircraft with empty ``t`` for newly seen
+transponders, and the lookup gracefully fills those in too.
+"""
+
+from __future__ import annotations
+
+import csv
+import logging
+import threading
+import time
+import xml.etree.ElementTree as ET
+from typing import Any
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+_BUCKET_LIST_URL = (
+    "https://s3.opensky-network.org/data-samples?prefix=metadata/&list-type=2"
+)
+_BUCKET_BASE = "https://s3.opensky-network.org/data-samples/"
+_S3_NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
+_REFRESH_INTERVAL_S = 5 * 24 * 3600
+_LIST_TIMEOUT_S = 30
+_DOWNLOAD_TIMEOUT_S = 600
+_USER_AGENT = (
+    "ShadowBroker-OSINT/0.9.79 "
+    "(+https://github.com/BigBodyCobain/Shadowbroker; "
+    "contact: bigbodycobain@gmail.com)"
+)
+
+_lock = threading.RLock()
+_aircraft_by_hex: dict[str, dict[str, str]] = {}
+_last_refresh = 0.0
+_in_progress = False
+
+
+def _latest_snapshot_key() -> str:
+    """Discover the most recent aircraft-database-complete snapshot key."""
+    response = requests.get(
+        _BUCKET_LIST_URL,
+        timeout=_LIST_TIMEOUT_S,
+        headers={"User-Agent": _USER_AGENT},
+    )
+    response.raise_for_status()
+    root = ET.fromstring(response.text)
+    keys: list[str] = []
+    for content in root.iter(f"{_S3_NS}Contents"):
+        key_el = content.find(f"{_S3_NS}Key")
+        if key_el is None or not key_el.text:
+            continue
+        if "aircraft-database-complete-" in key_el.text and key_el.text.endswith(".csv"):
+            keys.append(key_el.text)
+    if not keys:
+        raise RuntimeError("no aircraft-database-complete snapshot found in bucket listing")
+    return sorted(keys)[-1]
+
+
+def _stream_csv_index(url: str) -> dict[str, dict[str, str]]:
+    """Stream-parse the OpenSky aircraft CSV into a hex-keyed index.
+
+    The CSV uses single-quote quoting, so csv.DictReader is configured with
+    ``quotechar="'"``. Rows are processed line-by-line via iter_lines() to
+    keep memory bounded even though the file is ~108 MB.
+    """
+    with requests.get(
+        url,
+        timeout=_DOWNLOAD_TIMEOUT_S,
+        stream=True,
+        headers={"User-Agent": _USER_AGENT},
+    ) as response:
+        response.raise_for_status()
+        line_iter = (
+            line.decode("utf-8", errors="replace")
+            for line in response.iter_lines(decode_unicode=False)
+            if line
+        )
+        reader = csv.DictReader(line_iter, quotechar="'")
+        index: dict[str, dict[str, str]] = {}
+        for row in reader:
+            hex_code = (row.get("icao24") or "").strip().lower()
+            if not hex_code or hex_code == "000000":
+                continue
+            typecode = (row.get("typecode") or "").strip().upper()
+            model = (row.get("model") or "").strip()
+            mfr = (row.get("manufacturerName") or "").strip()
+            registration = (row.get("registration") or "").strip().upper()
+            operator = (row.get("operator") or "").strip()
+            if not (typecode or model):
+                continue
+            entry: dict[str, str] = {}
+            if typecode:
+                entry["typecode"] = typecode
+            if model:
+                entry["model"] = model
+            if mfr:
+                entry["manufacturer"] = mfr
+            if registration:
+                entry["registration"] = registration
+            if operator:
+                entry["operator"] = operator
+            index[hex_code] = entry
+    return index
+
+
+def refresh_aircraft_database(force: bool = False) -> bool:
+    """Download the latest OpenSky aircraft snapshot and rebuild the index.
+
+    Returns True if a refresh was performed (success or attempted), False if
+    skipped because the cache is still fresh or another refresh is in flight.
+    """
+    global _last_refresh, _in_progress
+
+    now = time.time()
+    with _lock:
+        if _in_progress:
+            return False
+        if not force and (now - _last_refresh) < _REFRESH_INTERVAL_S and _aircraft_by_hex:
+            return False
+        _in_progress = True
+
+    try:
+        started = time.time()
+        key = _latest_snapshot_key()
+        index = _stream_csv_index(_BUCKET_BASE + key)
+        with _lock:
+            _aircraft_by_hex.clear()
+            _aircraft_by_hex.update(index)
+            _last_refresh = time.time()
+        logger.info(
+            "aircraft database refreshed in %.1fs from %s: %d aircraft",
+            time.time() - started,
+            key,
+            len(index),
+        )
+        return True
+    except (requests.RequestException, OSError, ValueError, ET.ParseError) as exc:
+        logger.warning("aircraft database refresh failed: %s", exc)
+        return True
+    finally:
+        with _lock:
+            _in_progress = False
+
+
+def lookup_aircraft(icao24: str) -> dict[str, str] | None:
+    """Return the metadata record for an ICAO24 hex code, or None."""
+    key = (icao24 or "").strip().lower()
+    if not key:
+        return None
+    with _lock:
+        entry = _aircraft_by_hex.get(key)
+    return dict(entry) if entry else None
+
+
+def lookup_aircraft_type(icao24: str) -> str:
+    """Return the ICAO type code (e.g. 'B738', 'GLF4') or '' if unknown."""
+    entry = lookup_aircraft(icao24)
+    if not entry:
+        return ""
+    return entry.get("typecode", "")
+
+
+def aircraft_database_status() -> dict[str, Any]:
+    with _lock:
+        return {
+            "last_refresh": _last_refresh,
+            "aircraft": len(_aircraft_by_hex),
+            "in_progress": _in_progress,
+        }

backend/services/fetchers/crowdthreat.py

@@ -0,0 +1,129 @@
+"""CrowdThreat fetcher — crowdsourced global threat intelligence.
+
+Polls verified threat reports from CrowdThreat's public API and normalises
+them into map-ready records with category-based icon IDs.
+
+No API key required — the /threats endpoint is unauthenticated.
+"""
+
+import logging
+
+from services.network_utils import fetch_with_curl
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh, is_any_active
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger("services.data_fetcher")
+
+_CT_BASE = "https://backend.crowdthreat.world"
+
+# CrowdThreat category_id → icon ID used on the MapLibre layer
+_CATEGORY_ICON = {
+    1: "ct-security",       # Security & Conflict  (red)
+    2: "ct-crime",          # Crime & Safety        (blue)
+    3: "ct-aviation",       # Aviation              (green)
+    4: "ct-maritime",       # Maritime              (teal)
+    5: "ct-infrastructure", # Industrial & Infra    (orange)
+    6: "ct-special",        # Special Threats        (purple)
+    7: "ct-social",         # Social & Political    (pink)
+    8: "ct-other",          # Other                 (gray)
+}
+
+_CATEGORY_COLOUR = {
+    1: "#ef4444",  # red
+    2: "#3b82f6",  # blue
+    3: "#22c55e",  # green
+    4: "#14b8a6",  # teal
+    5: "#f97316",  # orange
+    6: "#a855f7",  # purple
+    7: "#ec4899",  # pink
+    8: "#6b7280",  # gray
+}
+
+
+@with_retry(max_retries=2, base_delay=5)
+def fetch_crowdthreat():
+    """Fetch verified threat reports from CrowdThreat public API."""
+    if not is_any_active("crowdthreat"):
+        return
+
+    try:
+        resp = fetch_with_curl(f"{_CT_BASE}/threats", timeout=20)
+        if not resp or resp.status_code != 200:
+            logger.warning("CrowdThreat API returned %s", getattr(resp, "status_code", "None"))
+            return
+
+        payload = resp.json()
+        raw_threats = payload.get("data", {}).get("threats", [])
+        if not raw_threats:
+            logger.debug("CrowdThreat returned 0 threats")
+            return
+
+    except Exception as e:
+        logger.error("CrowdThreat fetch error: %s", e)
+        return
+
+    processed = []
+    for t in raw_threats:
+        loc = t.get("location") or {}
+        lng_lat = loc.get("lng_lat")
+        if not lng_lat or len(lng_lat) < 2:
+            continue
+        try:
+            lng = float(lng_lat[0])
+            lat = float(lng_lat[1])
+        except (TypeError, ValueError):
+            continue
+
+        cat = t.get("category") or {}
+        cat_id = cat.get("id", 8)
+        subcat = t.get("subcategory") or {}
+        threat_type = t.get("type") or {}
+        dates = t.get("dates") or {}
+        occurred = dates.get("occurred") or {}
+        reported = dates.get("reported") or {}
+
+        # Extract all available detail from the API response
+        summary = (t.get("summary") or t.get("description") or "").strip()
+        verification = (t.get("verification_status") or t.get("status") or "").strip()
+        country_obj = loc.get("country") or {}
+        country = country_obj.get("name", "") if isinstance(country_obj, dict) else str(country_obj or "")
+        media = t.get("media") or t.get("images") or t.get("attachments") or []
+        source_url = t.get("source_url") or t.get("url") or t.get("link") or ""
+        severity = t.get("severity") or t.get("severity_level") or t.get("risk_level") or ""
+        votes = t.get("votes") or t.get("upvotes") or 0
+        reporter = t.get("user") or t.get("reporter") or {}
+        reporter_name = reporter.get("name", "") if isinstance(reporter, dict) else ""
+
+        processed.append({
+            "id": t.get("id"),
+            "title": t.get("title", ""),
+            "summary": summary[:500] if summary else "",
+            "lat": lat,
+            "lng": lng,
+            "address": loc.get("name", ""),
+            "city": loc.get("city", ""),
+            "country": country,
+            "category": cat.get("name", "Other"),
+            "category_id": cat_id,
+            "category_colour": _CATEGORY_COLOUR.get(cat_id, "#6b7280"),
+            "subcategory": subcat.get("name", ""),
+            "threat_type": threat_type.get("name", ""),
+            "icon_id": _CATEGORY_ICON.get(cat_id, "ct-other"),
+            "occurred": occurred.get("raw", ""),
+            "occurred_iso": occurred.get("iso", ""),
+            "timeago": occurred.get("timeago", ""),
+            "reported": reported.get("raw", ""),
+            "verification": verification,
+            "severity": str(severity),
+            "source_url": source_url,
+            "media_urls": [m.get("url") or m for m in media[:3]] if isinstance(media, list) else [],
+            "votes": int(votes) if votes else 0,
+            "reporter": reporter_name,
+            "source": "CrowdThreat",
+        })
+
+    logger.info("CrowdThreat: fetched %d verified threats", len(processed))
+
+    with _data_lock:
+        latest_data["crowdthreat"] = processed
+    _mark_fresh("crowdthreat")

backend/services/fetchers/emissions.py

@@ -0,0 +1,302 @@
+"""
+Fuel burn & CO2 emissions estimator.
+Based on manufacturer-published cruise fuel burn rates (GPH at long-range cruise).
+1 US gallon of Jet-A produces ~21.1 lbs (9.57 kg) of CO2.
+
+Piston entries use 100LL (avgas), which is close enough to Jet-A in CO2 yield
+(~8.4 kg/gal vs 9.57 kg/gal); we keep one constant to stay simple — the result
+is a slight over-estimate for piston aircraft, which is preferable to under.
+"""
+
+JET_A_CO2_KG_PER_GALLON = 9.57
+
+# ICAO type code -> gallons per hour at long-range cruise
+FUEL_BURN_GPH: dict[str, int] = {
+    # ── Gulfstream ─────────────────────────────────────────────────────
+    "GLF6": 430,   # G650/G650ER
+    "G700": 480,   # G700
+    "GLF5": 390,   # G550
+    "GVSP": 400,   # GV-SP
+    "GLF4": 330,   # G-IV
+    # ── Bombardier business ────────────────────────────────────────────
+    "GL7T": 490,   # Global 7500
+    "GLEX": 430,   # Global Express/6000/6500
+    "GL5T": 420,   # Global 5000/5500
+    "CL35": 220,   # Challenger 350
+    "CL60": 310,   # Challenger 604/605
+    "CL30": 200,   # Challenger 300
+    "CL65": 320,   # Challenger 650
+    # ── Bombardier regional jets ──────────────────────────────────────
+    "CRJ2": 360,   # CRJ-100/200
+    "CRJ7": 380,   # CRJ-700
+    "CRJ9": 410,   # CRJ-900
+    "CRJX": 440,   # CRJ-1000
+    # ── Dassault ───────────────────────────────────────────────────────
+    "F7X":  350,   # Falcon 7X
+    "F8X":  370,   # Falcon 8X
+    "F900": 285,   # Falcon 900/900EX/900LX
+    "F2TH": 230,   # Falcon 2000
+    "FA50": 240,   # Falcon 50
+    # ── Cessna Citation ────────────────────────────────────────────────
+    "CITX": 280,   # Citation X
+    "C750": 280,   # Citation X (alt code)
+    "C68A": 195,   # Citation Latitude
+    "C700": 230,   # Citation Longitude
+    "C680": 220,   # Citation Sovereign
+    "C56X": 195,   # Citation Excel/XLS/XLS+
+    "C560": 190,   # Citation Excel/XLS (legacy)
+    "C550": 165,   # Citation II/Bravo/V
+    "C525": 80,    # Citation CJ1
+    "C25A": 100,   # CJ1+ / 525A
+    "C25B": 110,   # CJ2+ / 525B
+    "C25C": 130,   # CJ4 (some operators)
+    "C510": 75,    # Citation Mustang
+    "C650": 240,   # Citation III/VI/VII
+    "CJ3":  120,   # CJ3
+    "CJ4":  135,   # CJ4
+    # ── Cessna piston / turboprop singles & twins ─────────────────────
+    "C172": 9,     # Skyhawk
+    "C152": 6,
+    "C150": 6,
+    "C170": 8,
+    "C177": 11,
+    "C180": 12,
+    "C182": 13,    # Skylane
+    "C185": 14,
+    "C206": 15,
+    "C208": 50,    # Caravan (turboprop)
+    "C210": 18,
+    "C310": 32,
+    "C340": 38,
+    "C414": 36,
+    "C421": 40,
+    # ── Boeing mainline ────────────────────────────────────────────────
+    "B737": 850,   # 737-700 / BBJ
+    "B738": 920,   # 737-800
+    "B739": 880,   # 737-900/900ER
+    "B38M": 700,   # 737-8 MAX
+    "B39M": 740,   # 737-9 MAX
+    "B752": 1100,  # 757-200
+    "B753": 1200,  # 757-300
+    "B762": 1400,  # 767-200
+    "B763": 1450,  # 767-300/300ER
+    "B764": 1500,  # 767-400ER
+    "B772": 1850,  # 777-200
+    "B77L": 1900,  # 777-200LR / 777F
+    "B77W": 2050,  # 777-300ER
+    "B788": 1200,  # 787-8
+    "B789": 1300,  # 787-9
+    "B78X": 1350,  # 787-10
+    "B744": 3050,  # 747-400
+    "B748": 2900,  # 747-8
+    # ── Airbus mainline ────────────────────────────────────────────────
+    "A318": 780,   # A318
+    "A319": 850,   # A319
+    "A320": 900,   # A320
+    "A321": 990,   # A321
+    "A19N": 580,   # A319neo
+    "A20N": 580,   # A320neo
+    "A21N": 700,   # A321neo
+    "A332": 1500,  # A330-200
+    "A333": 1550,  # A330-300
+    "A338": 1300,  # A330-800neo
+    "A339": 1350,  # A330-900neo
+    "A343": 1800,  # A340-300
+    "A346": 2100,  # A340-600
+    "A359": 1450,  # A350-900
+    "A35K": 1600,  # A350-1000
+    "A388": 3200,  # A380-800
+    # ── Embraer regional / business ───────────────────────────────────
+    "E135": 300,   # Legacy 600/650 (regional ERJ-135 base)
+    "E145": 320,   # ERJ-145
+    "E170": 460,   # E170
+    "E75L": 490,   # E175-LR
+    "E75S": 490,   # E175 standard
+    "E175": 490,   # E175 (some)
+    "E190": 580,   # E190
+    "E195": 600,   # E195
+    "E290": 510,   # E190-E2
+    "E295": 540,   # E195-E2
+    "E50P": 135,   # Phenom 300 (also Phenom 100 var)
+    "E55P": 185,   # Praetor 500 / Legacy 500
+    "E545": 170,   # Praetor 500 (alt)
+    "E500": 80,    # Phenom 100
+    # ── ATR / Bombardier / Saab turboprops ────────────────────────────
+    "AT43": 230,   # ATR 42-300/-320
+    "AT45": 230,   # ATR 42-500
+    "AT46": 250,   # ATR 42-600
+    "AT72": 300,   # ATR 72-200/-210
+    "AT75": 280,   # ATR 72-500
+    "AT76": 280,   # ATR 72-600
+    "DH8A": 220,   # Dash 8 -100
+    "DH8B": 240,   # Dash 8 -200
+    "DH8C": 280,   # Dash 8 -300
+    "DH8D": 300,   # Dash 8 Q400
+    "SF34": 200,   # Saab 340
+    "SB20": 220,   # Saab 2000
+    # ── Pilatus / Daher single-engine turboprops ──────────────────────
+    "PC24": 115,   # PC-24
+    "PC12": 60,    # PC-12
+    "TBM7": 60,    # TBM 700/850
+    "TBM8": 65,    # TBM 850 alt
+    "TBM9": 70,    # TBM 900/930/940/960
+    "M600": 60,    # Piper M600
+    "P46T": 22,    # PA-46 Meridian (turboprop variant)
+    # ── Learjet ────────────────────────────────────────────────────────
+    "LJ60": 195,   # Learjet 60
+    "LJ75": 185,   # Learjet 75
+    "LJ45": 175,   # Learjet 45
+    "LJ31": 165,   # Learjet 31
+    "LJ40": 175,   # Learjet 40
+    "LJ55": 195,   # Learjet 55
+    # ── Hawker / Beechjet ─────────────────────────────────────────────
+    "H25B": 210,   # Hawker 800/800XP
+    "H25C": 215,   # Hawker 900XP
+    "BE40": 150,   # Beechjet 400 / Hawker 400XP
+    "PRM1": 130,   # Premier I
+    # ── Beechcraft King Air ───────────────────────────────────────────
+    "B350": 100,   # King Air 350
+    "B200": 80,    # King Air 200/250
+    "BE20": 80,    # K-Air 200 (alt)
+    "BE9L": 60,    # K-Air 90
+    "BE9T": 70,    # K-Air F90
+    "BE10": 100,   # K-Air 100
+    "BE30": 90,    # K-Air 300
+    # ── Beechcraft / Cirrus / Piper / Mooney pistons ──────────────────
+    "BE23": 9,     # Sundowner
+    "BE33": 13,    # Bonanza 33
+    "BE35": 14,    # Bonanza V-tail
+    "BE36": 16,    # A36 Bonanza
+    "BE55": 24,    # Baron 55
+    "BE58": 28,    # Baron 58
+    "BE76": 17,    # Duchess
+    "BE95": 20,    # Travel Air
+    "P28A": 10,    # PA-28 Warrior/Archer
+    "P28B": 11,    # PA-28 Cherokee
+    "P28R": 12,    # PA-28R Arrow
+    "P32R": 14,    # PA-32R Lance/Saratoga
+    "PA11": 5,     # Cub Special
+    "PA12": 6,     # Super Cruiser
+    "PA18": 6,     # Super Cub
+    "PA22": 8,     # Tri-Pacer
+    "PA23": 18,    # Apache / Aztec
+    "PA24": 12,    # Comanche
+    "PA25": 12,    # Pawnee
+    "PA28": 10,    # PA-28 generic
+    "PA30": 16,    # Twin Comanche
+    "PA31": 30,    # Navajo
+    "PA32": 14,    # Cherokee Six / Saratoga
+    "PA34": 18,    # Seneca
+    "PA38": 5,     # Tomahawk
+    "PA44": 17,    # Seminole
+    "PA46": 18,    # Malibu / Mirage / Matrix
+    "M20P": 12,    # Mooney M20 (generic)
+    "SR20": 11,    # Cirrus SR20
+    "SR22": 16,    # Cirrus SR22
+    "S22T": 19,    # SR22T (turbo)
+    "DA40": 9,     # Diamond DA40
+    "DA42": 14,    # Diamond DA42 TwinStar
+    "DA62": 17,    # Diamond DA62
+    "DV20": 6,     # Diamond Katana
+    # ── Helicopters (civilian) ────────────────────────────────────────
+    "A109": 60,    # AW109
+    "A119": 50,    # AW119
+    "A139": 130,   # AW139
+    "A169": 90,    # AW169
+    "A189": 145,   # AW189
+    "AS35": 55,    # AS350 AStar
+    "AS50": 55,    # AStar (alt)
+    "AS65": 110,   # Dauphin
+    "B06":  35,    # Bell 206 JetRanger
+    "B407": 50,    # Bell 407
+    "B412": 145,   # Bell 412
+    "B429": 80,    # Bell 429
+    "B505": 35,    # Bell 505
+    "EC30": 50,    # H125 / EC130
+    "EC35": 70,    # EC135
+    "EC45": 85,    # EC145
+    "EC75": 130,   # EC175
+    "H125": 55,
+    "H130": 50,
+    "H135": 70,
+    "H145": 85,
+    "H155": 110,
+    "H160": 95,
+    "H175": 130,
+    "R22":  9,     # Robinson R22 (piston)
+    "R44":  16,    # Robinson R44 (piston)
+    "R66":  30,    # Robinson R66 (turbine)
+    "S76":  140,   # Sikorsky S-76
+    "S92":  220,   # Sikorsky S-92
+}
+
+# Common string names -> ICAO type code
+_ALIASES: dict[str, str] = {
+    "Gulfstream G650": "GLF6", "Gulfstream G650ER": "GLF6", "G650": "GLF6", "G650ER": "GLF6",
+    "Gulfstream G700": "G700",
+    "Gulfstream G550": "GLF5", "G550": "GLF5", "G500": "GLF5",
+    "Gulfstream GV": "GVSP", "Gulfstream G-V": "GVSP", "GV": "GVSP",
+    "Gulfstream G-IV": "GLF4", "Gulfstream GIV": "GLF4", "G450": "GLF4",
+    "Global 7500": "GL7T", "Bombardier Global 7500": "GL7T",
+    "Global 6000": "GLEX", "Global Express": "GLEX", "Bombardier Global 6000": "GLEX",
+    "Global 5000": "GL5T",
+    "Challenger 350": "CL35", "Challenger 300": "CL30",
+    "Challenger 604": "CL60", "Challenger 605": "CL60", "Challenger 650": "CL65",
+    "Falcon 7X": "F7X", "Dassault Falcon 7X": "F7X",
+    "Falcon 8X": "F8X", "Dassault Falcon 8X": "F8X",
+    "Falcon 900": "F900", "Falcon 900LX": "F900", "Falcon 900EX": "F900",
+    "Falcon 2000": "F2TH",
+    "Citation X": "CITX", "Citation Latitude": "C68A", "Citation Longitude": "C700",
+    "Boeing 757-200": "B752", "757-200": "B752", "Boeing 757": "B752",
+    "Boeing 767-200": "B762", "767-200": "B762", "Boeing 767": "B762",
+    "Boeing 787-8": "B788", "Boeing 787": "B788",
+    "Boeing 737": "B737", "737 BBJ": "B737", "BBJ": "B737",
+    "Airbus A340-300": "A343", "A340-300": "A343", "A340": "A343",
+    "Airbus A318": "A318",
+    "Pilatus PC-24": "PC24", "PC-24": "PC24",
+    "Legacy 500": "E55P", "Legacy 600": "E135", "Phenom 300": "E50P",
+    "Learjet 60": "LJ60", "Learjet 75": "LJ75",
+    "Hawker 800": "H25B", "Hawker 900XP": "H25C",
+    "King Air 350": "B350", "King Air 200": "B200",
+}
+
+
+def get_emissions_info(model: str) -> dict | None:
+    """
+    Given an aircraft model string (ICAO type code or common name),
+    return emissions info dict or None if unknown.
+    """
+    if not model:
+        return None
+    model_clean = model.strip()
+    model_upper = model_clean.upper()
+    # Try direct ICAO code match first
+    gph = FUEL_BURN_GPH.get(model_upper)
+    if gph is None:
+        # Try alias lookup
+        code = _ALIASES.get(model_clean)
+        if code:
+            gph = FUEL_BURN_GPH.get(code)
+    if gph is None:
+        # Friendly names from the Plane-Alert DB often lead with the ICAO type
+        # code as the first token (e.g. "B200 Super King Air"). Probe each
+        # token against FUEL_BURN_GPH directly.
+        for token in model_upper.replace("-", " ").replace(",", " ").split():
+            candidate = FUEL_BURN_GPH.get(token)
+            if candidate is not None:
+                gph = candidate
+                break
+    if gph is None:
+        # Fuzzy: check if any alias is a substring
+        model_lower = model_clean.lower()
+        for alias, code in _ALIASES.items():
+            if alias.lower() in model_lower or model_lower in alias.lower():
+                gph = FUEL_BURN_GPH.get(code)
+                if gph:
+                    break
+    if gph is None:
+        return None
+    return {
+        "fuel_gph": gph,
+        "co2_kg_per_hour": round(gph * JET_A_CO2_KG_PER_GALLON, 1),
+    }

backend/services/fetchers/fimi.py

@@ -0,0 +1,274 @@
+"""EUvsDisinfo FIMI (Foreign Information Manipulation & Interference) fetcher.
+
+Parses the EUvsDisinfo RSS feed to extract disinformation narratives,
+debunked claims, threat actor mentions, and target country references.
+Refreshes every 12 hours (FIMI data updates weekly).
+"""
+
+import re
+import logging
+from datetime import datetime, timezone
+
+import feedparser
+from services.network_utils import fetch_with_curl
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger("services.data_fetcher")
+
+_FIMI_FEED_URL = "https://euvsdisinfo.eu/feed/"
+
+# ── Threat actor keywords ──────────────────────────────────────────────────
+# Map of keyword → canonical actor name.  Checked case-insensitively.
+_THREAT_ACTORS: dict[str, str] = {
+    "russia":       "Russia",
+    "russian":      "Russia",
+    "kremlin":      "Russia",
+    "pro-kremlin":  "Russia",
+    "moscow":       "Russia",
+    "china":        "China",
+    "chinese":      "China",
+    "beijing":      "China",
+    "iran":         "Iran",
+    "iranian":      "Iran",
+    "tehran":       "Iran",
+    "north korea":  "North Korea",
+    "pyongyang":    "North Korea",
+    "dprk":         "North Korea",
+    "belarus":      "Belarus",
+    "belarusian":   "Belarus",
+    "minsk":        "Belarus",
+}
+
+# ── Target country/region keywords ─────────────────────────────────────────
+_TARGET_KEYWORDS: dict[str, str] = {
+    "ukraine":      "Ukraine",
+    "kyiv":         "Ukraine",
+    "moldova":      "Moldova",
+    "georgia":      "Georgia",
+    "tbilisi":      "Georgia",
+    "eu":           "EU",
+    "european union": "EU",
+    "europe":       "Europe",
+    "nato":         "NATO",
+    "united states": "United States",
+    "usa":          "United States",
+    "germany":      "Germany",
+    "france":       "France",
+    "poland":       "Poland",
+    "baltic":       "Baltics",
+    "lithuania":    "Baltics",
+    "latvia":       "Baltics",
+    "estonia":      "Baltics",
+    "romania":      "Romania",
+    "czech":        "Czech Republic",
+    "slovakia":     "Slovakia",
+    "armenia":      "Armenia",
+    "africa":       "Africa",
+    "middle east":  "Middle East",
+    "syria":        "Syria",
+    "israel":       "Israel",
+    "serbia":       "Serbia",
+    "india":        "India",
+    "brazil":       "Brazil",
+}
+
+# ── Disinformation topic keywords (for cross-referencing news) ─────────────
+_DISINFO_TOPICS = [
+    "sanctions",
+    "energy crisis",
+    "gas supply",
+    "nuclear threat",
+    "nato expansion",
+    "biolab",
+    "biological weapon",
+    "provocation",
+    "false flag",
+    "staged",
+    "nazi",
+    "genocide",
+    "referendum",
+    "regime change",
+    "coup",
+    "puppet government",
+    "election interference",
+    "election meddling",
+    "voter fraud",
+    "migrant invasion",
+    "refugee crisis",
+    "civil war",
+    "food crisis",
+    "grain deal",
+]
+
+# Regex for extracting debunked report URLs from feed HTML
+_REPORT_URL_RE = re.compile(
+    r'https?://euvsdisinfo\.eu/report/[a-z0-9\-]+/?',
+    re.IGNORECASE,
+)
+
+# Regex for extracting the claim title from a report URL slug
+_SLUG_RE = re.compile(r'/report/([a-z0-9\-]+)/?$', re.IGNORECASE)
+
+
+def _slug_to_title(url: str) -> str:
+    """Convert a report URL slug to a human-readable title."""
+    m = _SLUG_RE.search(url)
+    if not m:
+        return url
+    return m.group(1).replace("-", " ").title()
+
+
+def _count_mentions(text: str, keywords: dict[str, str]) -> dict[str, int]:
+    """Count keyword mentions, mapping to canonical names."""
+    counts: dict[str, int] = {}
+    text_lower = text.lower()
+    for kw, canonical in keywords.items():
+        # Word-boundary match, case-insensitive
+        pattern = r'\b' + re.escape(kw) + r'\b'
+        matches = re.findall(pattern, text_lower)
+        if matches:
+            counts[canonical] = counts.get(canonical, 0) + len(matches)
+    return counts
+
+
+def _extract_disinfo_keywords(text: str) -> list[str]:
+    """Return which disinformation topic keywords appear in the text."""
+    text_lower = text.lower()
+    found = []
+    for topic in _DISINFO_TOPICS:
+        if topic in text_lower:
+            found.append(topic)
+    return found
+
+
+def _is_major_wave(narratives: list[dict], targets: dict[str, int]) -> bool:
+    """Heuristic: detect a 'major disinformation wave'.
+
+    Triggers when:
+    - 3+ narratives in the feed mention the same target, OR
+    - A single target has 10+ total mentions across all narratives, OR
+    - 5+ distinct debunked claims extracted in one fetch
+    """
+    if not narratives:
+        return False
+
+    # Check per-target narrative count
+    target_narrative_counts: dict[str, int] = {}
+    total_claims = 0
+    for n in narratives:
+        for t in n.get("targets", []):
+            target_narrative_counts[t] = target_narrative_counts.get(t, 0) + 1
+        total_claims += len(n.get("claims", []))
+
+    if any(c >= 3 for c in target_narrative_counts.values()):
+        return True
+    if any(c >= 10 for c in targets.values()):
+        return True
+    if total_claims >= 5:
+        return True
+    return False
+
+
+@with_retry(max_retries=1, base_delay=5)
+def fetch_fimi():
+    """Fetch and parse the EUvsDisinfo RSS feed."""
+    try:
+        resp = fetch_with_curl(_FIMI_FEED_URL, timeout=15)
+        feed = feedparser.parse(resp.text)
+    except Exception as e:
+        logger.warning(f"FIMI feed fetch failed: {e}")
+        return
+
+    if not feed.entries:
+        logger.warning("FIMI feed: no entries found")
+        return
+
+    narratives = []
+    all_claims: list[dict] = []
+    agg_actors: dict[str, int] = {}
+    agg_targets: dict[str, int] = {}
+    all_disinfo_kw: set[str] = set()
+
+    for entry in feed.entries[:15]:  # Cap at 15 entries
+        title = entry.get("title", "")
+        link = entry.get("link", "")
+        published = entry.get("published", "")
+        summary_html = entry.get("summary", "") or entry.get("description", "")
+
+        # Strip HTML tags for text analysis
+        summary_text = re.sub(r"<[^>]+>", " ", summary_html)
+        summary_text = re.sub(r"\s+", " ", summary_text).strip()
+        full_text = f"{title} {summary_text}"
+
+        # Extract debunked report URLs
+        report_urls = list(set(_REPORT_URL_RE.findall(summary_html)))
+        claims = [{"url": url, "title": _slug_to_title(url)} for url in report_urls]
+        all_claims.extend(claims)
+
+        # Count threat actors
+        actors = _count_mentions(full_text, _THREAT_ACTORS)
+        for actor, count in actors.items():
+            agg_actors[actor] = agg_actors.get(actor, 0) + count
+
+        # Count target countries
+        targets = _count_mentions(full_text, _TARGET_KEYWORDS)
+        for target, count in targets.items():
+            agg_targets[target] = agg_targets.get(target, 0) + count
+
+        # Extract disinfo topic keywords
+        disinfo_kw = _extract_disinfo_keywords(full_text)
+        all_disinfo_kw.update(disinfo_kw)
+
+        # Truncate summary for storage
+        snippet = summary_text[:300] + ("..." if len(summary_text) > 300 else "")
+
+        narratives.append({
+            "title": title,
+            "link": link,
+            "published": published,
+            "snippet": snippet,
+            "claims": claims,
+            "actors": list(actors.keys()),
+            "targets": list(targets.keys()),
+            "disinfo_keywords": disinfo_kw,
+        })
+
+    # Sort actors and targets by count (descending)
+    sorted_actors = dict(sorted(agg_actors.items(), key=lambda x: x[1], reverse=True))
+    sorted_targets = dict(sorted(agg_targets.items(), key=lambda x: x[1], reverse=True))
+
+    # Deduplicate claims
+    seen_urls: set[str] = set()
+    unique_claims = []
+    for c in all_claims:
+        if c["url"] not in seen_urls:
+            seen_urls.add(c["url"])
+            unique_claims.append(c)
+
+    major_wave = _is_major_wave(narratives, sorted_targets)
+
+    fimi_data = {
+        "narratives": narratives,
+        "claims": unique_claims,
+        "threat_actors": sorted_actors,
+        "targets": sorted_targets,
+        "disinfo_keywords": sorted(all_disinfo_kw),
+        "major_wave": major_wave,
+        "major_wave_target": (
+            max(sorted_targets, key=sorted_targets.get) if major_wave and sorted_targets else None
+        ),
+        "last_fetched": datetime.now(timezone.utc).isoformat(),
+        "source": "EUvsDisinfo",
+        "source_url": "https://euvsdisinfo.eu",
+    }
+
+    with _data_lock:
+        latest_data["fimi"] = fimi_data
+    _mark_fresh("fimi")
+    logger.info(
+        f"FIMI fetch complete: {len(narratives)} narratives, "
+        f"{len(unique_claims)} claims, "
+        f"{len(sorted_actors)} actors, "
+        f"major_wave={major_wave}"
+    )

backend/services/fetchers/financial.py

@@ -0,0 +1,161 @@
+import logging
+import math
+import random
+import time
+import os
+import urllib.request
+import json
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timezone
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger(__name__)
+
+_YFINANCE_REQUEST_DELAY_SECONDS = 0.5
+_YFINANCE_REQUEST_JITTER_SECONDS = 0.2
+
+TICKERS_DEFENSE = ["RTX", "LMT", "NOC", "GD", "BA", "PLTR"]
+TICKERS_TECH = ["NVDA", "AMD", "TSM", "INTC", "GOOGL", "AMZN", "MSFT", "AAPL", "TSLA", "META", "NFLX", "SMCI", "ARM", "ASML"]
+TICKERS_CRYPTO = [
+    ("BTC", "BINANCE:BTCUSDT", "BTC-USD"),
+    ("ETH", "BINANCE:ETHUSDT", "ETH-USD"),
+    ("SOL", "BINANCE:SOLUSDT", "SOL-USD"),
+    ("XRP", "BINANCE:XRPUSDT", "XRP-USD"),
+    ("ADA", "BINANCE:ADAUSDT", "ADA-USD"),
+]
+
+# Ticker priority for high-frequency updates (we update these every tick)
+PRIORITY_SYMBOLS = ["BTC", "ETH", "NVDA", "PLTR"]
+
+# Persistence for state between short-lived scheduler ticks
+_last_fetch_results = {}
+_last_fetch_time = 0.0
+_rotating_index = 0
+_executor = ThreadPoolExecutor(max_workers=10)
+
+
+def _fetch_finnhub_quote(symbol: str, api_key: str):
+    """Fetch from Finnhub. Returns (symbol, data) or (symbol, None)."""
+    url = f"https://finnhub.io/api/v1/quote?symbol={symbol}&token={api_key}"
+    try:
+        req = urllib.request.Request(url)
+        with urllib.request.urlopen(req, timeout=5) as response:
+            data = json.loads(response.read().decode())
+            if "c" not in data or data["c"] == 0:
+                return symbol, None
+            current = float(data["c"])
+            change_p = float(data.get("dp", 0.0) or 0.0)
+            return symbol, {
+                "price": round(current, 2),
+                "change_percent": round(change_p, 2),
+                "up": bool(change_p >= 0),
+            }
+    except Exception as e:
+        logger.debug(f"Finnhub error for {symbol}: {e}")
+        return symbol, None
+
+
+def _fetch_yfinance_single(symbol: str, period: str = "2d"):
+    """Fetch from yfinance. Returns (symbol, data) or (symbol, None)."""
+    try:
+        import yfinance as yf
+        ticker = yf.Ticker(symbol)
+        hist = ticker.history(period=period)
+        if len(hist) >= 1:
+            current_price = hist["Close"].iloc[-1]
+            prev_close = hist["Close"].iloc[0] if len(hist) > 1 else current_price
+            change_percent = ((current_price - prev_close) / prev_close) * 100 if prev_close else 0
+            current_price_f = float(current_price)
+            change_percent_f = float(change_percent)
+            if not math.isfinite(current_price_f) or not math.isfinite(change_percent_f):
+                return symbol, None
+            return symbol, {
+                "price": round(current_price_f, 2),
+                "change_percent": round(change_percent_f, 2),
+                "up": bool(change_percent_f >= 0),
+            }
+    except Exception as e:
+        logger.debug(f"Yfinance error for {symbol}: {e}")
+    return symbol, None
+
+
+@with_retry(max_retries=1, base_delay=1)
+def fetch_financial_markets():
+    """Fetches full market list with smart throttling (3s for Finnhub, 60s for yfinance)."""
+    global _last_fetch_time, _last_fetch_results, _rotating_index
+    
+    finnhub_key = os.getenv("FINNHUB_API_KEY", "").strip()
+    use_finnhub = bool(finnhub_key)
+    
+    now = time.time()
+    # Throttle logic: 3s for Finnhub, 60s for yfinance fallback
+    throttle_s = 3.0 if use_finnhub else 60.0
+    
+    if now - _last_fetch_time < throttle_s and _last_fetch_results:
+        return # Skip if too frequent
+
+    _last_fetch_time = now
+    
+    # Prepare symbol lists
+    all_crypto = {label: (f_sym, y_sym) for label, f_sym, y_sym in TICKERS_CRYPTO}
+    all_stocks = TICKERS_TECH + TICKERS_DEFENSE
+    
+    subset_to_fetch = []
+    
+    if use_finnhub:
+        # Finnhub Free Limit: 60/min. 
+        # Ticking every 3s = 20 ticks/min. 
+        # To stay safe, we fetch only ~3 items per tick.
+        # Priority items (BTC, ETH) + 1 rotating item.
+        subset_to_fetch = ["BINANCE:BTCUSDT", "BINANCE:ETHUSDT"]
+        
+        # Determine rotating ticker
+        all_other_symbols = []
+        for sym in all_stocks:
+            all_other_symbols.append(sym)
+        for label, (f_sym, y_sym) in all_crypto.items():
+            if label not in ["BTC", "ETH"]:
+                all_other_symbols.append(f_sym)
+        
+        if all_other_symbols:
+            rotated = all_other_symbols[_rotating_index % len(all_other_symbols)]
+            subset_to_fetch.append(rotated)
+            _rotating_index += 1
+            
+        # Concurrently fetch
+        futures = [_executor.submit(_fetch_finnhub_quote, s, finnhub_key) for s in subset_to_fetch]
+        for f in futures:
+            sym, data = f.result()
+            if data:
+                # Map back to readable label if it was crypto
+                label = sym
+                for l, (fs, ys) in all_crypto.items():
+                    if fs == sym:
+                        label = l
+                        break
+                _last_fetch_results[label] = data
+    else:
+        # Yahoo Finance Fallback - fetch all (once per minute)
+        logger.info("Finnhub key missing, using Yahoo Finance 60s update cycle.")
+        to_fetch = all_stocks + [y_sym for l, (fs, y_sym) in all_crypto.items()]
+        futures = [_executor.submit(_fetch_yfinance_single, s) for s in to_fetch]
+        for f in futures:
+            sym, data = f.result()
+            if data:
+                # Map back to readable label if it was crypto
+                label = sym
+                for l, (fs, ys) in all_crypto.items():
+                    if ys == sym:
+                        label = l
+                        break
+                _last_fetch_results[label] = data
+
+    if not _last_fetch_results:
+        return
+        
+    with _data_lock:
+        latest_data["stocks"] = dict(_last_fetch_results)
+        latest_data["financial_source"] = "finnhub" if use_finnhub else "yfinance"
+    _mark_fresh("stocks")

backend/services/fetchers/geo.py

@@ -0,0 +1,407 @@
+"""Ship and geopolitics fetchers — AIS vessels, carriers, frontlines, GDELT, LiveUAmap, fishing."""
+
+import csv
+import concurrent.futures
+import io
+import math
+import os
+import logging
+import time
+from urllib.parse import urlencode
+from services.network_utils import fetch_with_curl
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger(__name__)
+
+
+def _env_flag(name: str) -> str:
+    return str(os.getenv(name, "")).strip().lower()
+
+
+def liveuamap_scraper_enabled() -> bool:
+    """Return whether the Playwright-based LiveUAMap scraper should run.
+
+    It is useful enrichment, but it starts a browser/Node driver and must not be
+    allowed to destabilize Windows local startup.
+    """
+    setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
+    if setting in {"1", "true", "yes", "on"}:
+        return True
+    if setting in {"0", "false", "no", "off"}:
+        return False
+    return os.name != "nt"
+
+
+# ---------------------------------------------------------------------------
+# Ships (AIS + Carriers)
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=1, base_delay=1)
+def fetch_ships():
+    """Fetch real-time AIS vessel data and combine with OSINT carrier positions."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active(
+        "ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts"
+    ):
+        return
+    from services.ais_stream import get_ais_vessels
+    from services.carrier_tracker import get_carrier_positions
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2, thread_name_prefix="ship_fetch") as executor:
+        carrier_future = executor.submit(get_carrier_positions)
+        ais_future = executor.submit(get_ais_vessels)
+
+        try:
+            carriers = carrier_future.result()
+        except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+            logger.error(f"Carrier tracker error (non-fatal): {e}")
+            carriers = []
+
+        try:
+            ais_vessels = ais_future.result()
+        except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+            logger.error(f"AIS stream error (non-fatal): {e}")
+            ais_vessels = []
+
+    ships = list(carriers or [])
+    ships.extend(ais_vessels or [])
+
+    # Enrich ships with yacht alert data (tracked superyachts)
+    from services.fetchers.yacht_alert import enrich_with_yacht_alert
+
+    for ship in ships:
+        enrich_with_yacht_alert(ship)
+
+    # Enrich ships with PLAN/CCG vessel data
+    from services.fetchers.plan_vessel_alert import enrich_with_plan_vessel
+    for ship in ships:
+        enrich_with_plan_vessel(ship)
+
+    logger.info(f"Ships: {len(carriers)} carriers + {len(ais_vessels)} AIS vessels")
+    with _data_lock:
+        latest_data["ships"] = ships
+    _mark_fresh("ships")
+
+
+# ---------------------------------------------------------------------------
+# Airports (ourairports.com)
+# ---------------------------------------------------------------------------
+cached_airports = []
+
+
+def find_nearest_airport(lat, lng, max_distance_nm=200):
+    """Find the nearest large airport to a given lat/lng using haversine distance."""
+    if not cached_airports:
+        return None
+
+    best = None
+    best_dist = float("inf")
+    lat_r = math.radians(lat)
+    lng_r = math.radians(lng)
+
+    for apt in cached_airports:
+        apt_lat_r = math.radians(apt["lat"])
+        apt_lng_r = math.radians(apt["lng"])
+        dlat = apt_lat_r - lat_r
+        dlng = apt_lng_r - lng_r
+        a = (
+            math.sin(dlat / 2) ** 2
+            + math.cos(lat_r) * math.cos(apt_lat_r) * math.sin(dlng / 2) ** 2
+        )
+        c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
+        dist_nm = 3440.065 * c
+
+        if dist_nm < best_dist:
+            best_dist = dist_nm
+            best = apt
+
+    if best and best_dist <= max_distance_nm:
+        return {
+            "iata": best["iata"],
+            "name": best["name"],
+            "lat": best["lat"],
+            "lng": best["lng"],
+            "distance_nm": round(best_dist, 1),
+        }
+    return None
+
+
+def fetch_airports():
+    global cached_airports
+    if not cached_airports:
+        logger.info("Downloading global airports database from ourairports.com...")
+        try:
+            url = "https://ourairports.com/data/airports.csv"
+            response = fetch_with_curl(url, timeout=15)
+            if response.status_code == 200:
+                f = io.StringIO(response.text)
+                reader = csv.DictReader(f)
+                for row in reader:
+                    if row["type"] == "large_airport" and row["iata_code"]:
+                        cached_airports.append(
+                            {
+                                "id": row["ident"],
+                                "name": row["name"],
+                                "iata": row["iata_code"],
+                                "lat": float(row["latitude_deg"]),
+                                "lng": float(row["longitude_deg"]),
+                                "type": "airport",
+                            }
+                        )
+                logger.info(f"Loaded {len(cached_airports)} large airports into cache.")
+        except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+            logger.error(f"Error fetching airports: {e}")
+
+    with _data_lock:
+        latest_data["airports"] = cached_airports
+
+
+# ---------------------------------------------------------------------------
+# Geopolitics & LiveUAMap
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=1, base_delay=2)
+def fetch_frontlines():
+    """Fetch Ukraine frontline data (fast — single GitHub API call)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("ukraine_frontline"):
+        return
+    try:
+        from services.geopolitics import fetch_ukraine_frontlines
+
+        frontlines = fetch_ukraine_frontlines()
+        if frontlines:
+            with _data_lock:
+                latest_data["frontlines"] = frontlines
+            _mark_fresh("frontlines")
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching frontlines: {e}")
+
+
+@with_retry(max_retries=1, base_delay=3)
+def fetch_gdelt():
+    """Fetch GDELT global military incidents (slow — downloads 32 ZIP files)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("global_incidents"):
+        return
+    try:
+        from services.geopolitics import fetch_global_military_incidents
+
+        gdelt = fetch_global_military_incidents()
+        if gdelt is not None:
+            with _data_lock:
+                latest_data["gdelt"] = gdelt
+            _mark_fresh("gdelt")
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching GDELT: {e}")
+
+
+def fetch_geopolitics():
+    """Legacy wrapper — runs both sequentially. Used by recurring scheduler."""
+    fetch_frontlines()
+    fetch_gdelt()
+
+
+def update_liveuamap():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("global_incidents"):
+        return
+    if not liveuamap_scraper_enabled():
+        logger.info(
+            "Liveuamap scraper disabled for this runtime; set "
+            "SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=1 to opt in."
+        )
+        return
+    logger.info("Running scheduled Liveuamap scraper...")
+    try:
+        from services.liveuamap_scraper import fetch_liveuamap
+
+        res = fetch_liveuamap()
+        if res:
+            with _data_lock:
+                latest_data["liveuamap"] = res
+            _mark_fresh("liveuamap")
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Liveuamap scraper error: {e}")
+
+
+# ---------------------------------------------------------------------------
+# Fishing Activity (Global Fishing Watch)
+# ---------------------------------------------------------------------------
+def _fishing_vessel_key(event: dict) -> str:
+    vessel_ssvid = str(event.get("vessel_ssvid", "") or "").strip()
+    if vessel_ssvid:
+        return f"ssvid:{vessel_ssvid}"
+    vessel_id = str(event.get("vessel_id", "") or "").strip()
+    if vessel_id:
+        return f"vid:{vessel_id}"
+    vessel_name = str(event.get("vessel_name", "") or "").strip().upper()
+    vessel_flag = str(event.get("vessel_flag", "") or "").strip().upper()
+    if vessel_name:
+        return f"name:{vessel_name}|flag:{vessel_flag}"
+    return f"event:{event.get('id', '')}"
+
+
+def _fishing_event_rank(event: dict) -> tuple[str, str, float, str]:
+    return (
+        str(event.get("end", "") or ""),
+        str(event.get("start", "") or ""),
+        float(event.get("duration_hrs", 0) or 0),
+        str(event.get("id", "") or ""),
+    )
+
+
+def _dedupe_fishing_events(events: list[dict]) -> list[dict]:
+    latest_by_vessel: dict[str, dict] = {}
+    counts_by_vessel: dict[str, int] = {}
+
+    for event in events:
+        vessel_key = _fishing_vessel_key(event)
+        counts_by_vessel[vessel_key] = counts_by_vessel.get(vessel_key, 0) + 1
+        current = latest_by_vessel.get(vessel_key)
+        if current is None or _fishing_event_rank(event) > _fishing_event_rank(current):
+            latest_by_vessel[vessel_key] = event
+
+    deduped: list[dict] = []
+    for vessel_key, event in latest_by_vessel.items():
+        event_copy = dict(event)
+        event_copy["event_count"] = counts_by_vessel.get(vessel_key, 1)
+        deduped.append(event_copy)
+
+    deduped.sort(key=_fishing_event_rank, reverse=True)
+    return deduped
+
+
+_FISHING_FETCH_INTERVAL_S = 3600  # once per hour — GFW data has ~5 day lag
+_last_fishing_fetch_ts: float = 0.0
+
+
+@with_retry(max_retries=1, base_delay=5)
+def fetch_fishing_activity():
+    """Fetch recent fishing events from Global Fishing Watch (~5 day lag)."""
+    global _last_fishing_fetch_ts
+    from services.fetchers._store import is_any_active, latest_data
+
+    if not is_any_active("fishing_activity"):
+        return
+
+    # Skip if we already have data and fetched less than an hour ago
+    now = time.time()
+    if latest_data.get("fishing_activity") and (now - _last_fishing_fetch_ts) < _FISHING_FETCH_INTERVAL_S:
+        return
+
+    token = os.environ.get("GFW_API_TOKEN", "")
+    if not token:
+        logger.debug("GFW_API_TOKEN not set, skipping fishing activity fetch")
+        return
+    events = []
+    try:
+        import datetime as _dt
+
+        _end = _dt.date.today().isoformat()
+        _start = (_dt.date.today() - _dt.timedelta(days=7)).isoformat()
+        page_size = max(1, int(os.environ.get("GFW_EVENTS_PAGE_SIZE", "500") or "500"))
+        offset = 0
+        seen_offsets: set[int] = set()
+        seen_ids: set[str] = set()
+        headers = {"Authorization": f"Bearer {token}"}
+
+        while True:
+            if offset in seen_offsets:
+                logger.warning("Fishing activity pagination repeated offset=%s; stopping fetch", offset)
+                break
+            seen_offsets.add(offset)
+
+            query = urlencode(
+                {
+                    "datasets[0]": "public-global-fishing-events:latest",
+                    "start-date": _start,
+                    "end-date": _end,
+                    "limit": page_size,
+                    "offset": offset,
+                }
+            )
+            url = f"https://gateway.api.globalfishingwatch.org/v3/events?{query}"
+            response = fetch_with_curl(url, timeout=30, headers=headers)
+            if response.status_code != 200:
+                logger.warning(
+                    "Fishing activity fetch failed at offset=%s: HTTP %s",
+                    offset,
+                    response.status_code,
+                )
+                break
+
+            payload = response.json() or {}
+            entries = payload.get("entries", [])
+            if not entries:
+                break
+
+            added_this_page = 0
+            for e in entries:
+                pos = e.get("position", {})
+                vessel = e.get("vessel") or {}
+                lat = pos.get("lat")
+                lng = pos.get("lon")
+                if lat is None or lng is None:
+                    continue
+                event_id = str(e.get("id", "") or "")
+                if event_id and event_id in seen_ids:
+                    continue
+                if event_id:
+                    seen_ids.add(event_id)
+                dur = e.get("event", {}).get("duration", 0) or 0
+                events.append(
+                    {
+                        "id": event_id,
+                        "type": e.get("type", "fishing"),
+                        "lat": lat,
+                        "lng": lng,
+                        "start": e.get("start", ""),
+                        "end": e.get("end", ""),
+                        "vessel_id": str(vessel.get("id", "") or ""),
+                        "vessel_ssvid": str(vessel.get("ssvid", "") or ""),
+                        "vessel_name": vessel.get("name", "Unknown"),
+                        "vessel_flag": vessel.get("flag", ""),
+                        "duration_hrs": round(dur / 3600, 1),
+                    }
+                )
+                added_this_page += 1
+
+            if len(entries) < page_size:
+                break
+
+            next_offset = payload.get("nextOffset")
+            if next_offset is None:
+                next_offset = (payload.get("pagination") or {}).get("nextOffset")
+            if next_offset is None:
+                next_offset = offset + page_size
+            try:
+                next_offset = int(next_offset)
+            except (TypeError, ValueError):
+                next_offset = offset + page_size
+            if next_offset <= offset:
+                logger.warning(
+                    "Fishing activity pagination produced non-increasing next offset=%s; stopping fetch",
+                    next_offset,
+                )
+                break
+            if added_this_page == 0:
+                logger.warning(
+                    "Fishing activity page at offset=%s added no new events; stopping fetch",
+                    offset,
+                )
+                break
+            offset = next_offset
+        raw_event_count = len(events)
+        events = _dedupe_fishing_events(events)
+        logger.info("Fishing activity: %s raw events -> %s deduped vessels", raw_event_count, len(events))
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching fishing activity: {e}")
+    with _data_lock:
+        latest_data["fishing_activity"] = events
+    if events:
+        _mark_fresh("fishing_activity")
+        _last_fishing_fetch_ts = time.time()

backend/services/fetchers/infrastructure.py

@@ -0,0 +1,727 @@
+"""Infrastructure fetchers — internet outages (IODA), data centers, CCTV, KiwiSDR."""
+
+import json
+import time
+import heapq
+import logging
+from pathlib import Path
+from cachetools import TTLCache
+from services.network_utils import fetch_with_curl
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Internet Outages (IODA — Georgia Tech)
+# ---------------------------------------------------------------------------
+_region_geocode_cache: TTLCache = TTLCache(maxsize=2000, ttl=86400)
+
+
+def _geocode_region(region_name: str, country_name: str) -> tuple:
+    """Geocode a region using OpenStreetMap Nominatim (cached, respects rate limit)."""
+    cache_key = f"{region_name}|{country_name}"
+    if cache_key in _region_geocode_cache:
+        return _region_geocode_cache[cache_key]
+    try:
+        import urllib.parse
+
+        query = urllib.parse.quote(f"{region_name}, {country_name}")
+        url = f"https://nominatim.openstreetmap.org/search?q={query}&format=json&limit=1"
+        response = fetch_with_curl(url, timeout=8, headers={"User-Agent": "ShadowBroker-OSINT/1.0"})
+        if response.status_code == 200:
+            results = response.json()
+            if results:
+                lat = float(results[0]["lat"])
+                lon = float(results[0]["lon"])
+                _region_geocode_cache[cache_key] = (lat, lon)
+                return (lat, lon)
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError):
+        pass
+    _region_geocode_cache[cache_key] = None
+    return None
+
+
+@with_retry(max_retries=1, base_delay=1)
+def fetch_internet_outages():
+    """Fetch regional internet outage alerts from IODA (Georgia Tech)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("internet_outages"):
+        return
+    RELIABLE_DATASOURCES = {"bgp", "ping-slash24"}
+    outages = []
+    try:
+        now = int(time.time())
+        start = now - 86400
+        url = f"https://api.ioda.inetintel.cc.gatech.edu/v2/outages/alerts?from={start}&until={now}&limit=500"
+        response = fetch_with_curl(url, timeout=15)
+        if response.status_code == 200:
+            data = response.json()
+            alerts = data.get("data", [])
+            region_outages = {}
+            for alert in alerts:
+                entity = alert.get("entity", {})
+                etype = entity.get("type", "")
+                level = alert.get("level", "")
+                if level == "normal" or etype != "region":
+                    continue
+                datasource = alert.get("datasource", "")
+                if datasource not in RELIABLE_DATASOURCES:
+                    continue
+                code = entity.get("code", "")
+                name = entity.get("name", "")
+                attrs = entity.get("attrs", {})
+                country_code = attrs.get("country_code", "")
+                country_name = attrs.get("country_name", "")
+                value = alert.get("value", 0)
+                history_value = alert.get("historyValue", 0)
+                severity = 0
+                if history_value and history_value > 0:
+                    severity = round((1 - value / history_value) * 100)
+                severity = max(0, min(severity, 100))
+                if severity < 10:
+                    continue
+                if code not in region_outages or severity > region_outages[code]["severity"]:
+                    region_outages[code] = {
+                        "region_code": code,
+                        "region_name": name,
+                        "country_code": country_code,
+                        "country_name": country_name,
+                        "level": level,
+                        "datasource": datasource,
+                        "severity": severity,
+                    }
+            geocoded = []
+            for rcode, r in region_outages.items():
+                coords = _geocode_region(r["region_name"], r["country_name"])
+                if coords:
+                    r["lat"] = coords[0]
+                    r["lng"] = coords[1]
+                    geocoded.append(r)
+            outages = heapq.nlargest(100, geocoded, key=lambda x: x["severity"])
+        logger.info(f"Internet outages: {len(outages)} regions affected")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching internet outages: {e}")
+    with _data_lock:
+        latest_data["internet_outages"] = outages
+    if outages:
+        _mark_fresh("internet_outages")
+
+
+# ---------------------------------------------------------------------------
+# RIPE Atlas — complement IODA with probe-level disconnection data
+# ---------------------------------------------------------------------------
+
+@with_retry(max_retries=1, base_delay=3)
+def fetch_ripe_atlas_probes():
+    """Fetch disconnected RIPE Atlas probes and merge into internet_outages (complementing IODA)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("internet_outages"):
+        return
+    try:
+        # 1. Fetch disconnected probes (status=2) — ~2,000 probes, no auth needed
+        url_disc = "https://atlas.ripe.net/api/v2/probes/?status=2&page_size=500&format=json"
+        resp_disc = fetch_with_curl(url_disc, timeout=20)
+        if resp_disc.status_code != 200:
+            logger.warning(f"RIPE Atlas probes API returned {resp_disc.status_code}")
+            return
+        disc_data = resp_disc.json()
+        disconnected = disc_data.get("results", [])
+
+        # 2. Fetch connected probe count (page_size=1 — we only need the count)
+        url_conn = "https://atlas.ripe.net/api/v2/probes/?status=1&page_size=1&format=json"
+        resp_conn = fetch_with_curl(url_conn, timeout=10)
+        total_connected = 0
+        if resp_conn.status_code == 200:
+            total_connected = resp_conn.json().get("count", 0)
+
+        # 3. Group disconnected probes by country
+        country_disc: dict = {}
+        for p in disconnected:
+            cc = p.get("country_code", "")
+            if not cc:
+                continue
+            if cc not in country_disc:
+                country_disc[cc] = []
+            country_disc[cc].append(p)
+
+        # 4. Get IODA-covered countries to avoid double-reporting
+        with _data_lock:
+            ioda_outages = list(latest_data.get("internet_outages", []))
+        ioda_countries = {
+            o.get("country_code", "").upper()
+            for o in ioda_outages
+            if o.get("datasource") != "ripe-atlas"
+        }
+
+        # 5. Build RIPE-only alerts for countries NOT already in IODA
+        ripe_alerts = []
+        for cc, probes in country_disc.items():
+            if cc.upper() in ioda_countries:
+                continue  # IODA already covers this country
+            if len(probes) < 3:
+                continue  # Too few probes to be meaningful
+
+            # Use centroid of disconnected probes as marker location
+            lats = [
+                p["geometry"]["coordinates"][1]
+                for p in probes
+                if p.get("geometry") and p["geometry"].get("coordinates")
+            ]
+            lngs = [
+                p["geometry"]["coordinates"][0]
+                for p in probes
+                if p.get("geometry") and p["geometry"].get("coordinates")
+            ]
+            if not lats:
+                continue
+
+            disc_count = len(probes)
+            # Severity: scale 10-80 based on disconnected probe count
+            severity = min(80, 10 + disc_count * 2)
+
+            ripe_alerts.append({
+                "region_code": f"RIPE-{cc}",
+                "region_name": f"{cc} (Atlas probes)",
+                "country_code": cc,
+                "country_name": cc,
+                "level": "critical" if disc_count >= 10 else "warning",
+                "datasource": "ripe-atlas",
+                "severity": severity,
+                "lat": sum(lats) / len(lats),
+                "lng": sum(lngs) / len(lngs),
+                "probe_count": disc_count,
+            })
+
+        # 6. Merge into internet_outages — keep IODA entries, replace old RIPE entries
+        with _data_lock:
+            current = latest_data.get("internet_outages", [])
+            ioda_only = [o for o in current if o.get("datasource") != "ripe-atlas"]
+            latest_data["internet_outages"] = ioda_only + ripe_alerts
+
+        if ripe_alerts:
+            _mark_fresh("internet_outages")
+        logger.info(
+            f"RIPE Atlas: {len(ripe_alerts)} countries with probe disconnections "
+            f"(from {len(disconnected)} disconnected / ~{total_connected} connected probes)"
+        )
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching RIPE Atlas probes: {e}")
+
+
+# ---------------------------------------------------------------------------
+# Data Centers (local geocoded JSON)
+# ---------------------------------------------------------------------------
+_DC_GEOCODED_PATH = Path(__file__).parent.parent.parent / "data" / "datacenters_geocoded.json"
+
+
+def fetch_datacenters():
+    """Load geocoded data centers (5K+ street-level precise locations)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("datacenters"):
+        return
+    dcs = []
+    try:
+        if not _DC_GEOCODED_PATH.exists():
+            logger.warning(f"Geocoded DC file not found: {_DC_GEOCODED_PATH}")
+            return
+        raw = json.loads(_DC_GEOCODED_PATH.read_text(encoding="utf-8"))
+        for entry in raw:
+            lat = entry.get("lat")
+            lng = entry.get("lng")
+            if lat is None or lng is None:
+                continue
+            if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+                continue
+            dcs.append(
+                {
+                    "name": entry.get("name", "Unknown"),
+                    "company": entry.get("company", ""),
+                    "street": entry.get("street", ""),
+                    "city": entry.get("city", ""),
+                    "country": entry.get("country", ""),
+                    "zip": entry.get("zip", ""),
+                    "lat": lat,
+                    "lng": lng,
+                }
+            )
+        logger.info(f"Data centers: {len(dcs)} geocoded locations loaded")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error loading data centers: {e}")
+    with _data_lock:
+        latest_data["datacenters"] = dcs
+    if dcs:
+        _mark_fresh("datacenters")
+
+
+# ---------------------------------------------------------------------------
+# Military Bases (static JSON — Western Pacific)
+# ---------------------------------------------------------------------------
+_MILITARY_BASES_PATH = Path(__file__).parent.parent.parent / "data" / "military_bases.json"
+
+
+def fetch_military_bases():
+    """Load static military base locations (Western Pacific focus)."""
+    bases = []
+    try:
+        if not _MILITARY_BASES_PATH.exists():
+            logger.warning(f"Military bases file not found: {_MILITARY_BASES_PATH}")
+            return
+        raw = json.loads(_MILITARY_BASES_PATH.read_text(encoding="utf-8"))
+        for entry in raw:
+            lat = entry.get("lat")
+            lng = entry.get("lng")
+            if lat is None or lng is None:
+                continue
+            if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+                continue
+            bases.append({
+                "name": entry.get("name", "Unknown"),
+                "country": entry.get("country", ""),
+                "operator": entry.get("operator", ""),
+                "branch": entry.get("branch", ""),
+                "lat": lat, "lng": lng,
+            })
+        logger.info(f"Military bases: {len(bases)} locations loaded")
+    except Exception as e:
+        logger.error(f"Error loading military bases: {e}")
+    with _data_lock:
+        latest_data["military_bases"] = bases
+    if bases:
+        _mark_fresh("military_bases")
+
+
+# ---------------------------------------------------------------------------
+# Power Plants (WRI Global Power Plant Database)
+# ---------------------------------------------------------------------------
+_POWER_PLANTS_PATH = Path(__file__).parent.parent.parent / "data" / "power_plants.json"
+
+
+def fetch_power_plants():
+    """Load WRI Global Power Plant Database (~35K facilities)."""
+    plants = []
+    try:
+        if not _POWER_PLANTS_PATH.exists():
+            logger.warning(f"Power plants file not found: {_POWER_PLANTS_PATH}")
+            return
+        raw = json.loads(_POWER_PLANTS_PATH.read_text(encoding="utf-8"))
+        for entry in raw:
+            lat = entry.get("lat")
+            lng = entry.get("lng")
+            if lat is None or lng is None:
+                continue
+            if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+                continue
+            plants.append({
+                "name": entry.get("name", "Unknown"),
+                "country": entry.get("country", ""),
+                "fuel_type": entry.get("fuel_type", "Unknown"),
+                "capacity_mw": entry.get("capacity_mw"),
+                "owner": entry.get("owner", ""),
+                "lat": lat, "lng": lng,
+            })
+        logger.info(f"Power plants: {len(plants)} facilities loaded")
+    except Exception as e:
+        logger.error(f"Error loading power plants: {e}")
+    with _data_lock:
+        latest_data["power_plants"] = plants
+    if plants:
+        _mark_fresh("power_plants")
+
+
+# ---------------------------------------------------------------------------
+# CCTV Cameras
+# ---------------------------------------------------------------------------
+def fetch_cctv():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("cctv"):
+        return
+    try:
+        from services.cctv_pipeline import get_all_cameras
+
+        cameras = get_all_cameras()
+        if len(cameras) < 500:
+            # Serve the current DB snapshot immediately and let the scheduled
+            # ingest cycle populate/refresh cameras asynchronously.
+            logger.info(
+                "CCTV DB currently has %d cameras — serving cached snapshot and waiting for scheduled ingest",
+                len(cameras),
+            )
+        with _data_lock:
+            latest_data["cctv"] = cameras
+        _mark_fresh("cctv")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching cctv from DB: {e}")
+
+
+# ---------------------------------------------------------------------------
+# KiwiSDR Receivers
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=2, base_delay=2)
+def fetch_kiwisdr():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("kiwisdr"):
+        return
+    try:
+        from services.kiwisdr_fetcher import fetch_kiwisdr_nodes
+
+        nodes = fetch_kiwisdr_nodes()
+        with _data_lock:
+            latest_data["kiwisdr"] = nodes
+        _mark_fresh("kiwisdr")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching KiwiSDR nodes: {e}")
+        with _data_lock:
+            latest_data["kiwisdr"] = []
+
+
+# ---------------------------------------------------------------------------
+# SatNOGS Ground Stations + Observations
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=2, base_delay=2)
+def fetch_satnogs():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("satnogs"):
+        return
+    try:
+        from services.satnogs_fetcher import fetch_satnogs_stations, fetch_satnogs_observations
+
+        stations = fetch_satnogs_stations()
+        obs = fetch_satnogs_observations()
+        with _data_lock:
+            latest_data["satnogs_stations"] = stations
+            latest_data["satnogs_observations"] = obs
+        _mark_fresh("satnogs_stations", "satnogs_observations")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching SatNOGS: {e}")
+
+
+# ---------------------------------------------------------------------------
+# PSK Reporter — HF Digital Mode Spots
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=2, base_delay=2)
+def fetch_psk_reporter():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("psk_reporter"):
+        return
+    try:
+        from services.psk_reporter_fetcher import fetch_psk_reporter_spots
+
+        spots = fetch_psk_reporter_spots()
+        with _data_lock:
+            latest_data["psk_reporter"] = spots
+        _mark_fresh("psk_reporter")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching PSK Reporter: {e}")
+        with _data_lock:
+            latest_data["psk_reporter"] = []
+
+
+# ---------------------------------------------------------------------------
+# TinyGS LoRa Satellites
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=2, base_delay=2)
+def fetch_tinygs():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("tinygs"):
+        return
+    try:
+        from services.tinygs_fetcher import fetch_tinygs_satellites
+
+        sats = fetch_tinygs_satellites()
+        with _data_lock:
+            latest_data["tinygs_satellites"] = sats
+        _mark_fresh("tinygs_satellites")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching TinyGS: {e}")
+
+
+# ---------------------------------------------------------------------------
+# Police Scanners (OpenMHZ) — geocode city+state via local GeoNames DB
+# ---------------------------------------------------------------------------
+_scanner_geo_cache: dict = {}  # city|state -> (lat, lng) — populated once from GeoNames
+
+
+def _build_scanner_geo_lookup():
+    """Build a US city/county→coords lookup from reverse_geocoder's bundled GeoNames CSV."""
+    if _scanner_geo_cache:
+        return
+    try:
+        import csv, os, reverse_geocoder as rg
+
+        geo_file = os.path.join(os.path.dirname(rg.__file__), "rg_cities1000.csv")
+        # US state abbreviation → admin1 name mapping
+        _abbr = {
+            "AL": "Alabama",
+            "AK": "Alaska",
+            "AZ": "Arizona",
+            "AR": "Arkansas",
+            "CA": "California",
+            "CO": "Colorado",
+            "CT": "Connecticut",
+            "DE": "Delaware",
+            "FL": "Florida",
+            "GA": "Georgia",
+            "HI": "Hawaii",
+            "ID": "Idaho",
+            "IL": "Illinois",
+            "IN": "Indiana",
+            "IA": "Iowa",
+            "KS": "Kansas",
+            "KY": "Kentucky",
+            "LA": "Louisiana",
+            "ME": "Maine",
+            "MD": "Maryland",
+            "MA": "Massachusetts",
+            "MI": "Michigan",
+            "MN": "Minnesota",
+            "MS": "Mississippi",
+            "MO": "Missouri",
+            "MT": "Montana",
+            "NE": "Nebraska",
+            "NV": "Nevada",
+            "NH": "New Hampshire",
+            "NJ": "New Jersey",
+            "NM": "New Mexico",
+            "NY": "New York",
+            "NC": "North Carolina",
+            "ND": "North Dakota",
+            "OH": "Ohio",
+            "OK": "Oklahoma",
+            "OR": "Oregon",
+            "PA": "Pennsylvania",
+            "RI": "Rhode Island",
+            "SC": "South Carolina",
+            "SD": "South Dakota",
+            "TN": "Tennessee",
+            "TX": "Texas",
+            "UT": "Utah",
+            "VT": "Vermont",
+            "VA": "Virginia",
+            "WA": "Washington",
+            "WV": "West Virginia",
+            "WI": "Wisconsin",
+            "WY": "Wyoming",
+            "DC": "Washington, D.C.",
+        }
+        state_full = {v.lower(): k for k, v in _abbr.items()}
+        state_full["washington, d.c."] = "DC"
+
+        county_coords = {}  # admin2(county)|state -> (lat, lon) — first city per county
+        with open(geo_file, "r", encoding="utf-8") as f:
+            reader = csv.reader(f)
+            next(reader, None)  # skip header
+            for row in reader:
+                if len(row) < 6 or row[5] != "US":
+                    continue
+                lat_s, lon_s, name, admin1, admin2 = row[0], row[1], row[2], row[3], row[4]
+                st = state_full.get(admin1.lower(), "")
+                if not st:
+                    continue
+                coords = (float(lat_s), float(lon_s))
+                # City name → coords
+                _scanner_geo_cache[f"{name.lower()}|{st}"] = coords
+                # County name → coords (keep first match per county, usually the largest city)
+                if admin2:
+                    county_key = f"{admin2.lower()}|{st}"
+                    if county_key not in county_coords:
+                        county_coords[county_key] = coords
+                    # Also strip " County" suffix for matching
+                    stripped = admin2.lower().replace(" county", "").strip()
+                    stripped_key = f"{stripped}|{st}"
+                    if stripped_key not in county_coords:
+                        county_coords[stripped_key] = coords
+
+        # Merge county lookups (don't override city entries)
+        for k, v in county_coords.items():
+            if k not in _scanner_geo_cache:
+                _scanner_geo_cache[k] = v
+        # Special case: DC
+        _scanner_geo_cache["washington|DC"] = (38.89511, -77.03637)
+        logger.info(f"Scanner geo lookup: {len(_scanner_geo_cache)} US entries loaded")
+    except Exception as e:
+        logger.warning(f"Failed to build scanner geo lookup: {e}")
+
+
+def _geocode_scanner(city: str, state: str):
+    """Look up city+state coordinates from local GeoNames cache."""
+    _build_scanner_geo_lookup()
+    if not city or not state:
+        return None
+    st = state.upper()
+    # Strip trailing state from city (e.g. "Lehigh, PA")
+    c = city.strip()
+    if ", " in c:
+        parts = c.rsplit(", ", 1)
+        if len(parts[1]) <= 2:
+            c = parts[0]
+    name = c.lower()
+    # Try exact city match
+    result = _scanner_geo_cache.get(f"{name}|{st}")
+    if result:
+        return result
+    # Strip "County" / "Co" suffix
+    stripped = name.replace(" county", "").replace(" co", "").strip()
+    result = _scanner_geo_cache.get(f"{stripped}|{st}")
+    if result:
+        return result
+    # Normalize "St." / "St" → "Saint"
+    import re
+
+    normed = re.sub(r"\bst\.?\s", "saint ", name)
+    if normed != name:
+        result = _scanner_geo_cache.get(f"{normed}|{st}")
+        if result:
+            return result
+        # Also try with "s" suffix: "St. Marys" → "Saint Marys" and "Saint Mary's"
+        for variant in [normed.rstrip("s"), normed.replace("ys", "y's")]:
+            result = _scanner_geo_cache.get(f"{variant}|{st}")
+            if result:
+                return result
+    # "Prince Georges" → "Prince George's" (apostrophe variants)
+    if "georges" in name:
+        key = name.replace("georges", "george's") + "|" + st
+        result = _scanner_geo_cache.get(key)
+        if result:
+            return result
+    # Multi-location: "Scott and Carver" → try first part
+    if " and " in name:
+        first = name.split(" and ")[0].strip()
+        result = _scanner_geo_cache.get(f"{first}|{st}")
+        if result:
+            return result
+    # Comma-separated list: "Adams, Jackson, Juneau" → try first
+    if ", " in name:
+        first = name.split(", ")[0].strip()
+        result = _scanner_geo_cache.get(f"{first}|{st}")
+        if result:
+            return result
+    # Drop directional prefix: "North Fulton" → "Fulton"
+    for prefix in ("north ", "south ", "east ", "west "):
+        if name.startswith(prefix):
+            result = _scanner_geo_cache.get(f"{name[len(prefix):]}|{st}")
+            if result:
+                return result
+    return None
+
+
+@with_retry(max_retries=2, base_delay=2)
+def fetch_scanners():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("scanners"):
+        return
+    try:
+        from services.radio_intercept import get_openmhz_systems
+
+        systems = get_openmhz_systems()
+        scanners = []
+        for s in systems:
+            city = s.get("city", "") or s.get("county", "") or ""
+            state = s.get("state", "")
+            coords = _geocode_scanner(city, state)
+            if not coords:
+                continue
+            lat, lng = coords
+            scanners.append(
+                {
+                    "shortName": s.get("shortName", ""),
+                    "name": s.get("name", "Unknown Scanner"),
+                    "lat": round(lat, 5),
+                    "lng": round(lng, 5),
+                    "city": city,
+                    "state": state,
+                    "clientCount": s.get("clientCount", 0),
+                    "description": s.get("description", ""),
+                }
+            )
+        with _data_lock:
+            latest_data["scanners"] = scanners
+        if scanners:
+            _mark_fresh("scanners")
+        logger.info(f"Scanners: {len(scanners)}/{len(systems)} geocoded")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching scanners: {e}")
+        with _data_lock:
+            latest_data["scanners"] = []