Pin v0.9.82 release digests for updater integrity verification.

Carry SHA-256 hashes for the source zip, MSI, and setup EXE into release_digests.json while retaining prior release entries.

Co-authored-by: Cursor <cursoragent@cursor.com>

.dockerignore

@@ -0,0 +1,56 @@
+# Exclude build artifacts, caches, and large directories from Docker context
+.git/
+.git_backup/
+node_modules/
+.next/
+__pycache__/
+*.pyc
+venv/
+.venv/
+.ruff_cache/
+local-artifacts/
+release-secrets/
+
+# Never send local configuration or credentials into Docker builds
+.env
+.env.*
+**/.env
+**/.env.*
+*.pem
+*.key
+*.p12
+*.pfx
+
+# privacy-core build caches (source is needed, artifacts are not)
+privacy-core/target/
+privacy-core/target-test/
+privacy-core/.codex-tmp/
+
+# Large data/cache files
+*.db
+*.sqlite
+*.xlsx
+*.log
+extra/
+prototype/
+
+# Runtime state generated by local backend runs
+backend/.pytest_cache/
+backend/.ruff_cache/
+backend/backend.egg-info/
+backend/build/
+backend/node_modules/
+backend/timemachine/
+backend/venv/
+backend/data/*cache*.json
+backend/data/**/*cache*.json
+backend/data/wormhole*.json
+backend/data/**/wormhole*.json
+backend/data/dm_*.json
+backend/data/**/dm_*.json
+backend/data/**/peer_store.json
+backend/data/**/node.json
+backend/data/*.log
+backend/data/**/*.log
+backend/data/*.key
+backend/data/**/*.key

.env.example

@@ -0,0 +1,173 @@
+# ShadowBroker — Docker Compose Environment Variables
+# Copy this file to .env and fill in your keys:
+#   cp .env.example .env
+
+# ── Required for backend container ─────────────────────────────
+# OpenSky Network OAuth2 — REQUIRED for airplane telemetry.
+# Free registration at https://opensky-network.org/index.php?option=com_users&view=registration
+# Without these the flights layer falls back to ADS-B-only with major gaps in Africa, Asia, and LatAm.
+OPENSKY_CLIENT_ID=
+OPENSKY_CLIENT_SECRET=
+AIS_API_KEY=
+
+# Global Fishing Watch — fishing vessel activity events (Fishing Activity map layer).
+# Free API token from https://globalfishingwatch.org/our-apis/tokens
+# Without this the fishing_activity layer stays empty.
+# GFW_API_TOKEN=
+# Optional tuning — GFW can return 40k+ global events; defaults cap fetch for map paint.
+# GFW_EVENTS_PAGE_SIZE=500
+# GFW_EVENTS_MAX_PAGES=10
+# GFW_EVENTS_LOOKBACK_DAYS=7
+# GFW_EVENTS_TIMEOUT_S=90
+
+# Windy Webcams global CCTV layer — free key from https://api.windy.com/webcams/docs
+# WINDY_API_KEY=
+
+# Telegram OSINT map layer — scrapes public t.me/s channel previews (no bot token).
+# TELEGRAM_OSINT_ENABLED=true
+# TELEGRAM_OSINT_CHANNELS=osintdefender,insiderpaper,aljazeeraenglish,nexta_live,war_monitor
+
+# Admin key to protect sensitive endpoints (settings, updates).
+# If blank, loopback/localhost requests still work for local single-host dev.
+# Remote/non-loopback admin access requires ADMIN_KEY, or ALLOW_INSECURE_ADMIN=true in debug-only setups.
+ADMIN_KEY=
+
+# Allow insecure admin access without ADMIN_KEY (local dev only, beyond loopback).
+# Requires MESH_DEBUG_MODE=true on the backend; do not enable this for normal use.
+# ALLOW_INSECURE_ADMIN=false
+
+# User-Agent for Nominatim geocoding requests (per OSM usage policy).
+# NOMINATIM_USER_AGENT=ShadowBroker/1.0 (https://github.com/BigBodyCobain/Shadowbroker)
+
+# ── Optional ───────────────────────────────────────────────────
+
+# LTA (Singapore traffic cameras) — leave blank to skip
+# LTA_ACCOUNT_KEY=
+
+# NASA FIRMS country-scoped fire data — enriches global CSV with conflict-zone hotspots.
+# Free MAP_KEY from https://firms.modaps.eosdis.nasa.gov/
+# FIRMS_MAP_KEY=
+
+# Ukraine air raid alerts — free token from https://alerts.in.ua/
+# ALERTS_IN_UA_TOKEN=
+
+# Optional NUFORC UAP sighting map enrichment via Mapbox Tilequery.
+# Leave blank to skip this optional enrichment.
+# NUFORC_MAPBOX_TOKEN=
+
+# Optional startup-risk controls.
+# On Windows, external curl fallback is off by default. LiveUAMap uses UI consent
+# when you enable Global Incidents (or set SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=true).
+# SHADOWBROKER_ENABLE_WINDOWS_CURL_FALLBACK=false
+# SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=false
+# AIS starts by default when AIS_API_KEY is set. Set to 0/false to force-disable.
+# SHADOWBROKER_ENABLE_AIS_STREAM_PROXY=true
+# Minimum visible satellite catalog before forcing a CelesTrak refresh.
+# SHADOWBROKER_MIN_VISIBLE_SATELLITES=350
+# Upper bound for TLE fallback satellite search when CelesTrak is unreachable.
+# SHADOWBROKER_MAX_VISIBLE_SATELLITES=450
+# NUFORC fallback uses the Hugging Face mirror when live NUFORC is unavailable.
+# NUFORC_HF_FALLBACK_LIMIT=250
+# NUFORC_HF_GEOCODE_LIMIT=150
+
+# First-paint cache age budgets. These let the map and Global Threat Intercept
+# paint from the last local snapshot while live feeds refresh in the background.
+# FAST_STARTUP_CACHE_MAX_AGE_S=21600
+# INTEL_STARTUP_CACHE_MAX_AGE_S=21600
+
+# Docker resource tuning. The backend synthesizes large geospatial feeds; keep
+# this at 4G or higher on hosts that run AIS, OpenSky, CCTV, satellites, and
+# threat feeds together. Lower caps can cause Docker OOM restarts and empty
+# slow layers such as news, UAP sightings, military bases, and wastewater.
+# BACKEND_MEMORY_LIMIT=4G
+# SHADOWBROKER_FETCH_WORKERS=8
+# SHADOWBROKER_SLOW_FETCH_CONCURRENCY=4
+# SHADOWBROKER_STARTUP_HEAVY_CONCURRENCY=2
+
+# Infonet bootstrap/sync responsiveness. Defaults favor fast seed failure
+# detection so stale onion peers do not make the terminal look hung.
+# MESH_SYNC_TIMEOUT_S=5
+# MESH_SYNC_MAX_PEERS_PER_CYCLE=3
+# MESH_BOOTSTRAP_SEED_FAILURE_COOLDOWN_S=15
+
+# Google Earth Engine for VIIRS night lights change detection (optional).
+# pip install earthengine-api
+# GEE_SERVICE_ACCOUNT_KEY=
+
+# Copernicus CDSE — Sentinel-2 imagery (Settings → Imagery, or backend .env).
+# Free OAuth app at https://dataspace.copernicus.eu/
+# SENTINEL_CLIENT_ID=
+# SENTINEL_CLIENT_SECRET=
+
+# Sentinel-2 road corridor freight trends (DrishX engine port — opt-in slow layer).
+# pip install -e backend[road-corridor]  (or uv sync --extra road-corridor)
+# ROAD_CORRIDOR_SAT_ENABLED=false
+# ROAD_CORRIDOR_SCHEDULED_PRESETS=laredo_i35
+# ROAD_CORRIDOR_MONTHS=2
+# ROAD_CORRIDOR_MAX_FRAMES=6
+# ROAD_CORRIDOR_REFRESH_HOURS=24
+
+# Override the backend URL the frontend uses (leave blank for auto-detect)
+# NEXT_PUBLIC_API_URL=http://192.168.1.50:8000
+
+# ── Mesh / Reticulum (RNS) ─────────────────────────────────────
+# MESH_RNS_ENABLED=false
+# MESH_RNS_APP_NAME=shadowbroker
+# MESH_RNS_ASPECT=infonet
+# MESH_RNS_IDENTITY_PATH=
+# MESH_RNS_PEERS=
+# MESH_RNS_DANDELION_HOPS=2
+# MESH_RNS_DANDELION_DELAY_MS=400
+# MESH_RNS_CHURN_INTERVAL_S=300
+# MESH_RNS_MAX_PEERS=32
+# MESH_RNS_MAX_PAYLOAD=8192
+# MESH_RNS_PEER_BUCKET_PREFIX=4
+# MESH_RNS_MAX_PEERS_PER_BUCKET=4
+# MESH_RNS_PEER_FAIL_THRESHOLD=3
+# MESH_RNS_PEER_COOLDOWN_S=300
+# MESH_RNS_SHARD_ENABLED=false
+# MESH_RNS_SHARD_DATA_SHARDS=3
+# MESH_RNS_SHARD_PARITY_SHARDS=1
+# MESH_RNS_SHARD_TTL_S=30
+# MESH_RNS_FEC_CODEC=xor
+# MESH_RNS_BATCH_MS=200
+# MESH_RNS_COVER_INTERVAL_S=0
+# MESH_RNS_COVER_SIZE=64
+# MESH_RNS_IBF_WINDOW=256
+# MESH_RNS_IBF_TABLE_SIZE=64
+# MESH_RNS_IBF_MINHASH_SIZE=16
+# MESH_RNS_IBF_MINHASH_THRESHOLD=0.25
+# MESH_RNS_IBF_WINDOW_JITTER=32
+# MESH_RNS_IBF_INTERVAL_S=120
+# MESH_RNS_IBF_SYNC_PEERS=3
+# MESH_RNS_IBF_QUORUM_TIMEOUT_S=6
+# MESH_RNS_IBF_MAX_REQUEST_IDS=64
+# MESH_RNS_IBF_MAX_EVENTS=64
+# MESH_RNS_SESSION_ROTATE_S=0
+# MESH_RNS_IBF_FAIL_THRESHOLD=3
+# MESH_RNS_IBF_COOLDOWN_S=120
+# MESH_VERIFY_INTERVAL_S=600
+# MESH_VERIFY_SIGNATURES=false
+
+# ── Mesh DM Relay ──────────────────────────────────────────────
+# MESH_DM_TOKEN_PEPPER=change-me
+# Optional local-dev DM root external assurance bridge.
+# These stay commented because they are machine-local file paths, not safe global defaults.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH=backend/../ops/root_witness_receipt_import.json
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH=backend/../ops/root_transparency_ledger.json
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI=backend/../ops/root_transparency_ledger.json
+
+# ── Self Update ────────────────────────────────────────────────
+# Optional ZIP updater digest pin. The updater checks this first, then
+# backend/data/release_digests.json, then the release SHA256SUMS.txt asset.
+# MESH_UPDATE_SHA256=
+
+# Optional strict nonce-only frontend CSP. Leave unset unless the exact build
+# has been verified to hydrate cleanly in your deployment.
+# SHADOWBROKER_STRICT_CSP=1
+
+# ── Wormhole (Local Agent) ─────────────────────────────────────
+# WORMHOLE_URL=http://127.0.0.1:8787
+# WORMHOLE_TRANSPORT=direct
+# WORMHOLE_SOCKS_PROXY=127.0.0.1:9050
+# WORMHOLE_SOCKS_DNS=true

.gitattributes

@@ -0,0 +1,2 @@
+# Force LF line endings for shell scripts
+*.sh text eol=lf

.github/CODEOWNERS

@@ -0,0 +1,32 @@
+# CODEOWNERS — assigns required reviewers for sensitive paths.
+# Format: <path glob>  <user-or-team> [<user-or-team> ...]
+# See https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
+#
+# Owners listed here are auto-requested for review when matching files
+# change in a PR. If branch protection requires CODEOWNERS approval, the
+# PR cannot be merged until an owner approves.
+
+# ── Internationalization / translations ──
+# Translation contributions are held to a stricter neutrality standard
+# than most code changes — see CONTRIBUTING.md "Translation contributions".
+# The i18n layer itself (no network calls, no telemetry, static JSON
+# bundled at build) is the structural guarantee that makes this safe;
+# changes to it need owner review.
+/frontend/src/i18n/                @BigBodyCobain
+
+# ── Security-sensitive code paths ──
+/backend/auth.py                   @BigBodyCobain
+/backend/routers/wormhole.py       @BigBodyCobain
+/backend/services/mesh/            @BigBodyCobain
+/backend/services/fetchers/        @BigBodyCobain
+
+# ── CI / build / deploy infra ──
+/.github/workflows/                @BigBodyCobain
+/.gitlab-ci.yml                    @BigBodyCobain
+/docker-compose.yml                @BigBodyCobain
+/docker-compose.gitlab.yml         @BigBodyCobain
+/helm/                             @BigBodyCobain
+
+# ── This file and policy docs ──
+/.github/CODEOWNERS                @BigBodyCobain
+/CONTRIBUTING.md                   @BigBodyCobain

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "npm"
+    directory: "/frontend"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "pip"
+    directory: "/backend"
+    schedule:
+      interval: "weekly"

.github/pull_request_template.md

@@ -0,0 +1,13 @@
+## Summary
+
+<!-- What changed and why (1–3 bullets). -->
+
+## Test plan
+
+- [ ] <!-- How you verified the change -->
+
+## Production hardening (data path / fetchers / unattended deploys only)
+
+If this PR touches the data path, fetchers, or live-data APIs, walk through [docs/production-hardening.md](https://github.com/BigBodyCobain/Shadowbroker/blob/main/docs/production-hardening.md) and note any N/A items here.
+
+- [ ] Checklist reviewed (or N/A — explain why)

.github/workflows/ci.yml

@@ -0,0 +1,82 @@
+name: CI - Lint & Test
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_call:
+
+# CI flake mitigation:
+# ci.yml is triggered TWICE per PR on the same commit — once directly via
+# the `pull_request` trigger above ("Frontend Tests & Build" check) and once
+# via `workflow_call` from docker-publish.yml ("CI Gate / Frontend Tests &
+# Build" check). Both jobs land on the same Actions runner pool at the same
+# time and fight for CPU/RAM. Under contention, React's reconciliation in
+# `messagesViewFirstContact.test.tsx > removes an approved contact …`
+# overruns its 5s waitFor timeout — that's the single failure mode we've
+# seen flake on PRs #226, #237, #261, #262, #265, #294, #303, and the
+# fd7d6fa push. Backend tests and every other frontend test pass under
+# the same conditions, which is what made this look random.
+#
+# Pinning a concurrency group on the SHA (PR head, or the pushed commit
+# for main) serializes the two invocations so neither starves the other.
+# We use cancel-in-progress: false so the second one queues instead of
+# cancelling — cancelling could leave the PR check stuck "Expected" if
+# only one of the two ever finishes. Total CI time grows by ~2 min in
+# exchange for deterministic outcomes.
+concurrency:
+  group: ci-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: false
+
+jobs:
+  frontend:
+    name: Frontend Tests & Build
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: frontend
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+          cache-dependency-path: frontend/package-lock.json
+      - run: npm ci
+      - run: npm run lint
+      - run: npm run format:check
+      - run: npx vitest run --reporter=verbose
+      - run: npm run build
+      - run: npm run bundle:report
+
+  backend:
+    name: Backend Lint & Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run secret scan
+        run: bash backend/scripts/scan-secrets.sh --all
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: cd backend && uv sync --frozen --group dev
+      - run: cd backend && uv run ruff check .
+      - run: cd backend && uv run black --check .
+      - run: cd backend && uv run python -c "from services.fetchers.retry import with_retry; from services.env_check import validate_env; print('Module imports OK')"
+      - name: Run release smoke tests
+        run: |
+          cd backend
+          uv run pytest \
+            tests/mesh/test_mesh_node_bootstrap_runtime.py \
+            tests/mesh/test_mesh_infonet_sync_support.py \
+            tests/mesh/test_mesh_canonical.py \
+            tests/mesh/test_mesh_merkle.py \
+            tests/test_release_helper.py \
+            -v --tb=short

.github/workflows/docker-publish.yml

@@ -9,34 +9,90 @@ on:
 
 env:
   REGISTRY: ghcr.io
-  # github.repository as <account>/<repo>
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  build-and-push-frontend:
+  ci-gate:
-    runs-on: ubuntu-latest
+    name: CI Gate
+    uses: ./.github/workflows/ci.yml
+
+  build-frontend:
+    needs: ci-gate
+    runs-on: ${{ matrix.runner }}
     permissions:
       contents: read
       packages: write
       id-token: write
-
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
     steps:
-      - name: Checkout repository
+      - uses: actions/checkout@v4
-        uses: actions/checkout@v4
+      - name: Lowercase image name
-
+        run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
-      - name: Set up Docker Buildx
+      - uses: docker/setup-buildx-action@v3.0.0
-        uses: docker/setup-buildx-action@v3.0.0
+      - name: Log into registry
-
-      - name: Log into registry ${{ env.REGISTRY }}
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v3.0.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+      - id: meta
+        uses: docker/metadata-action@v5.0.0
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend
+      - id: build
+        uses: docker/build-push-action@v5.0.0
+        with:
+          context: ./frontend
+          platforms: ${{ matrix.platform }}
+          push: ${{ github.event_name != 'pull_request' }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,scope=frontend-${{ matrix.platform }}
+          cache-to: type=gha,mode=max,scope=frontend-${{ matrix.platform }}
+          outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend,push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }}
+      - name: Export digest
+        if: github.event_name != 'pull_request'
+        run: |
+          mkdir -p /tmp/digests/frontend
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/frontend/${digest#sha256:}"
+      - uses: actions/upload-artifact@v4
+        if: github.event_name != 'pull_request'
+        with:
+          name: digests-frontend-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
+          path: /tmp/digests/frontend/*
+          if-no-files-found: error
+          retention-days: 1
 
-      - name: Extract Docker metadata
+  merge-frontend:
-        id: meta
+    if: github.event_name != 'pull_request'
+    needs: build-frontend
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Lowercase image name
+        run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
+      - uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests/frontend
+          pattern: digests-frontend-*
+          merge-multiple: true
+      - uses: docker/setup-buildx-action@v3.0.0
+      - uses: docker/login-action@v3.0.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - id: meta
         uses: docker/metadata-action@v5.0.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend
@@ -44,43 +100,91 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=raw,value=latest,enable={{is_default_branch}}
+      - name: Create and push manifest
+        working-directory: /tmp/digests/frontend
+        run: |
+          docker buildx imagetools create \
+            $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend@sha256:%s ' *)
 
-      - name: Build and push Docker image
+  build-backend:
-        id: build-and-push
+    needs: ci-gate
-        uses: docker/build-push-action@v5.0.0
+    runs-on: ${{ matrix.runner }}
-        with:
-          context: ./frontend
-          platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-  build-and-push-backend:
-    runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
       id-token: write
-
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
     steps:
-      - name: Checkout repository
+      - uses: actions/checkout@v4
-        uses: actions/checkout@v4
+      - name: Lowercase image name
-
+        run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
-      - name: Set up Docker Buildx
+      - uses: docker/setup-buildx-action@v3.0.0
-        uses: docker/setup-buildx-action@v3.0.0
+      - name: Log into registry
-
-      - name: Log into registry ${{ env.REGISTRY }}
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v3.0.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+      - id: meta
+        uses: docker/metadata-action@v5.0.0
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend
+      - id: build
+        uses: docker/build-push-action@v5.0.0
+        with:
+          context: .
+          file: ./backend/Dockerfile
+          platforms: ${{ matrix.platform }}
+          push: ${{ github.event_name != 'pull_request' }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,scope=backend-${{ matrix.platform }}
+          cache-to: type=gha,mode=max,scope=backend-${{ matrix.platform }}
+          outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend,push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }}
+      - name: Export digest
+        if: github.event_name != 'pull_request'
+        run: |
+          mkdir -p /tmp/digests/backend
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/backend/${digest#sha256:}"
+      - uses: actions/upload-artifact@v4
+        if: github.event_name != 'pull_request'
+        with:
+          name: digests-backend-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
+          path: /tmp/digests/backend/*
+          if-no-files-found: error
+          retention-days: 1
 
-      - name: Extract Docker metadata
+  merge-backend:
-        id: meta
+    if: github.event_name != 'pull_request'
+    needs: build-backend
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Lowercase image name
+        run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
+      - uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests/backend
+          pattern: digests-backend-*
+          merge-multiple: true
+      - uses: docker/setup-buildx-action@v3.0.0
+      - uses: docker/login-action@v3.0.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - id: meta
         uses: docker/metadata-action@v5.0.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend
@@ -88,15 +192,9 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=raw,value=latest,enable={{is_default_branch}}
-
+      - name: Create and push manifest
-      - name: Build and push Docker image
+        working-directory: /tmp/digests/backend
-        id: build-and-push
+        run: |
-        uses: docker/build-push-action@v5.0.0
+          docker buildx imagetools create \
-        with:
+            $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-          context: ./backend
+            $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend@sha256:%s ' *)
-          platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max

.gitignore

@@ -6,13 +6,32 @@ node_modules/
 venv/
 env/
 .venv/
+backend/.venv-dir
+backend/venv-repair*/
+backend/.venv-repair*/
 
 # Environment Variables & Secrets
 .env
+.envrc
 .env.local
 .env.development.local
 .env.test.local
 .env.production.local
+.npmrc
+.pypirc
+.netrc
+*.pem
+*.key
+*.crt
+*.csr
+*.p12
+*.pfx
+id_rsa
+id_rsa.*
+id_ed25519
+id_ed25519.*
+known_hosts
+authorized_keys
 
 # Python caches & compiled files
 __pycache__/
@@ -20,18 +39,80 @@ __pycache__/
 *$py.class
 *.so
 .Python
+.ruff_cache/
+.pytest_cache/
+.mypy_cache/
+.hypothesis/
+.tox/
 
 # Next.js build output
 .next/
 out/
 build/
+*.tsbuildinfo
 
-# Application Specific Caches & DBs
+# Deprecated standalone Infonet Terminal skeleton (migrated into frontend/src/components/InfonetTerminal/)
+frontend/infonet-terminal/
+
+# Rust build artifacts (privacy-core)
+target/
+target-test/
+
+# ========================
+# LOCAL-ONLY: extra/ folder
+# ========================
+# All internal docs, planning files, raw data, backups, and dev scratch
+# live here. NEVER commit this folder.
+extra/
+
+# ========================
+# Application caches & runtime DBs (regenerate on startup)
+# ========================
 backend/ais_cache.json
 backend/carrier_cache.json
 backend/cctv.db
+cctv.db
+*.db
+*.sqlite
 *.sqlite3
 
+# ========================
+# backend/data/ — blanket ignore, whitelist static reference files
+# ========================
+# Everything in data/ is runtime-generated state (encrypted keys,
+# MLS bindings, relay spools, caches) and MUST NOT be committed.
+# Only static reference datasets that ship with the repo are whitelisted.
+backend/data/*
+!backend/data/datacenters.json
+!backend/data/datacenters_geocoded.json
+!backend/data/military_bases.json
+!backend/data/plan_ccg_vessels.json
+!backend/data/plane_alert_db.json
+!backend/data/power_plants.json
+!backend/data/tracked_names.json
+!backend/data/yacht_alert_db.json
+# Issue #206: bundled KiwiSDR receiver directory used as last-resort
+# fallback when rx.linkfanel.net (HTTP-only upstream) is unreachable
+# or returns content that fails our integrity validation.
+!backend/data/kiwisdr_directory.json
+# Issue #201: pinned SHA-256 digests for known Tor Expert Bundle URLs.
+# Used as a second verification source when upstream .sha256sum fails.
+!backend/data/tor_bundle_digests.json
+# Issue #258: SPKI pins for stream.aisstream.io so we can survive upstream
+# Let's Encrypt renewal failures without disabling TLS validation entirely.
+!backend/data/aisstream_spki_pins.json
+# Issue #231: pinned SHA-256 digests for known release archives. Used by
+# the self-updater as a second-line integrity check when the release's
+# SHA256SUMS.txt asset can't be fetched.
+!backend/data/release_digests.json
+# Issue #244/#245/#246: one-shot carrier-position seed shipped with each
+# release. Used ONLY on first-ever startup to bootstrap carrier_cache.json;
+# after that the cache reflects this install's own GDELT observations.
+!backend/data/carrier_seed.json
+# DrishX RF model weights (MIT — see backend/third_party/drishx/NOTICE.md)
+!backend/data/drishx/
+!backend/data/drishx/rf_model.pickle
+
 # OS generated files
 .DS_Store
 .DS_Store?
@@ -53,17 +134,24 @@ Thumbs.db
 # Vercel / Deployment
 .vercel
 
-# Temp files
+# ========================
+# Temp / scratch / debug files
+# ========================
 tmp/
 *.log
 *.tmp
 *.bak
+*.swp
+*.swo
 out.txt
 out_sys.txt
 rss_output.txt
 merged.txt
 tmp_fast.json
-TheAirTraffic Database.xlsx
+diff.txt
+local_diff.txt
+map_diff.txt
+TERMINAL
 
 # Debug dumps & release artifacts
 backend/dump.json
@@ -72,25 +160,138 @@ backend/nyc_sample.json
 backend/nyc_full.json
 backend/liveua_test.html
 backend/out_liveua.json
+backend/out.json
+backend/temp.json
+backend/seattle_sample.json
+backend/sgp_sample.json
+backend/wsdot_sample.json
+backend/xlsx_analysis.txt
 frontend/server_logs*.txt
 frontend/cctv.db
+frontend/eslint-report.json
 *.zip
-.git_backup/
+*.tar.gz
+*.xlsx
 
-# Test files (may contain hardcoded keys)
+# Old backups & repo clones
+.git_backup/
+local-artifacts/
+release-secrets/
+shadowbroker_repo/
+frontend/src/components.bak/
+frontend/src/components/map/icons/backups/
+
+# Coverage
+coverage/
+.coverage
+.coverage.*
+dist/
+
+# Test scratch files (not in tests/ folder)
 backend/test_*.py
 backend/services/test_*.py
 
 # Local analysis & dev tools
 backend/analyze_xlsx.py
-backend/xlsx_analysis.txt
 backend/services/ais_cache.json
+graphify/
+graphify-out/
 
-# Internal update tracking (not for repo)
+# ========================
+# Internal docs & brainstorming (never commit)
+# ========================
+docs/*
+!docs/OUTBOUND_DATA.md
+!docs/production-hardening.md
+!docs/mesh/
+docs/mesh/*
+!docs/mesh/threat-model.md
+!docs/mesh/claims-reconciliation.md
+!docs/mesh/mesh-canonical-fixtures.json
+!docs/mesh/mesh-merkle-fixtures.json
+!docs/mesh/wormhole-dm-root-operations-runbook.md
+.local-docs/
+infonet-economy/
 updatestuff.md
+ROADMAP.md
+UPDATEPROTOCOL.md
+CLAUDE.md
+DOCKER_SECRETS.md
 
 # Misc dev artifacts
 clean_zip.py
 zip_repo.py
 refactor_cesium.py
 jobs.json
+
+# Claude / AI
+.claude
+.mise.local.toml
+.codex-tmp/
+prototype/
+.runtime/
+
+# ========================
+# Runtime state & operator-local data (never commit)
+# ========================
+# TimeMachine snapshot cache — regenerated at runtime, can be 100 MB+
+backend/timemachine/
+# Operator witness keys, identity material, transparency ledgers (machine-local)
+ops/
+# Runtime DM relay state
+dm_relay.json
+# Dev scratch notes
+improvements.txt
+
+# ========================
+# Custody verification temp dirs (runtime test artifacts with private keys!)
+# ========================
+backend/sb-custody-verify-*/
+
+# Python egg-info (build artifact, regenerated by pip install -e)
+*.egg-info/
+
+# Privacy-core debug build (Windows DLL, 3.6 MB, not shipped)
+privacy-core/debug/
+
+# Desktop-shell export stash dirs (empty temp dirs from Tauri build)
+frontend/.desktop-export-stash-*/
+
+# Wormhole logs (can be 30 MB+ each, runtime-generated)
+backend/data/wormhole_stderr.log
+backend/data/wormhole_stdout.log
+
+# Runtime caches that already slip through the backend/data/* blanket
+# (these are caught by the wildcard but listing for clarity)
+
+# Compressed snapshot archives (can be 100 MB+)
+*.json.gz
+
+# ──────────────────────────────────────────────────────────────────────
+# AI assistant / coding-agent scratch
+# ──────────────────────────────────────────────────────────────────────
+# Per-tool config + scratch directories. These are private to whichever
+# coding agent the operator happens to be using and have no business in
+# the repo. If a tool's instructions need to be canonical for the project,
+# we'll put them in docs/ explicitly — not let the agent dump them at the
+# repo root.
+
+# OpenAI Codex CLI
+.codex/
+.codex-app-schema/
+.codex-app-ts/
+
+# Per-agent instruction files dropped at repo root by various tools.
+# These are operator-side preferences, not part of the project contract.
+AGENTS.md
+GEMINI.md
+CLAUDE.md
+.github/copilot-instructions.md
+
+# Stale AI-generated test file that referenced fields that don't exist in
+# the current `_parse_carrier_positions_from_news` implementation. Kept
+# ignored so it doesn't accidentally get committed if it shows up again
+# from a tool that's working off an out-of-date understanding of the
+# module. If a real test for that function is needed, write it under a
+# meaningful name in tests/test_carrier_tracker_quality.py.
+backend/tests/test_carrier_tracker_region_centers.py

.gitlab-ci.yml

@@ -0,0 +1,151 @@
+# GitLab CI/CD for Shadowbroker
+#
+# Mirror of .github/workflows/docker-publish.yml — keeps the GitLab install
+# path (image registry + source) at parity with GitHub so users who prefer
+# GitLab get the same experience.
+#
+# What this does on every push to main:
+#   1. Builds multi-arch (amd64 + arm64) Docker images for the backend and
+#      frontend, pushes them to the project's GitLab Container Registry:
+#        registry.gitlab.com/bigbodycobain/shadowbroker/backend:latest
+#        registry.gitlab.com/bigbodycobain/shadowbroker/frontend:latest
+#      Both also get a :$CI_COMMIT_SHORT_SHA tag for traceability.
+#   2. Reverse-mirrors main back to GitHub (only if commits land directly
+#      on GitLab) so the two sources stay in sync.
+#
+# Pipelines on this repo were instant-failing for free-tier accounts until
+# identity verification was added — the May 2026 bump in this comment is
+# the marker commit that confirms runner allocation after verification.
+#
+# Auth notes:
+#   - The image build/push uses $CI_JOB_TOKEN, which GitLab provides
+#     automatically. No credentials need to be configured.
+#   - The reverse mirror authenticates to GitHub via a per-repo SSH
+#     deploy key. The private half is stored as the File-type GitLab
+#     CI/CD variable GITHUB_MIRROR_SSH_KEY (Protected). The matching
+#     public key is added to github.com/BigBodyCobain/Shadowbroker/
+#     settings/keys with write access. This is a tighter-scoped
+#     replacement for a personal access token: it can ONLY push to
+#     Shadowbroker, never expires, and rotating it is a one-click
+#     delete on GitHub's deploy-keys page. If the variable isn't set,
+#     the mirror job is skipped — image builds still run.
+
+stages:
+  - build
+  - mirror
+
+variables:
+  # Use the dind service for buildx multi-arch builds.
+  DOCKER_HOST: tcp://docker:2376
+  DOCKER_TLS_CERTDIR: "/certs"
+  DOCKER_DRIVER: overlay2
+  # QEMU is what lets a single x86 runner build arm64 images. dind doesn't
+  # install it by default; we install via tonistiigi/binfmt below.
+  BUILDX_VERSION: "v0.14.1"
+  # Repository-relative paths.
+  BACKEND_IMAGE: $CI_REGISTRY_IMAGE/backend
+  FRONTEND_IMAGE: $CI_REGISTRY_IMAGE/frontend
+
+# Shared template: bootstraps buildx + QEMU on the dind service so a single
+# runner can produce both amd64 and arm64 manifests in one push.
+.buildx-setup: &buildx-setup
+  image: docker:24
+  services:
+    - name: docker:24-dind
+      command: ["--tls=true"]
+  before_script:
+    - docker info
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
+    - docker run --privileged --rm tonistiigi/binfmt --install all
+    # buildx --driver docker-container can't read TLS from the env vars
+    # the GitLab dind service exports. Wrap them in a docker context and
+    # bind buildx to it. See https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-buildx
+    - docker context create tls-env
+    - docker buildx create --use --name multiarch --driver docker-container tls-env
+
+# ── Backend image ────────────────────────────────────────────────────────
+build-backend:
+  <<: *buildx-setup
+  stage: build
+  script:
+    - >
+      docker buildx build
+      --platform linux/amd64,linux/arm64
+      --file backend/Dockerfile
+      --tag $BACKEND_IMAGE:latest
+      --tag $BACKEND_IMAGE:$CI_COMMIT_SHORT_SHA
+      --push
+      .
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+        - backend/**/*
+        - .gitlab-ci.yml
+
+# ── Frontend image ───────────────────────────────────────────────────────
+build-frontend:
+  <<: *buildx-setup
+  stage: build
+  script:
+    - cd frontend
+    - >
+      docker buildx build
+      --platform linux/amd64,linux/arm64
+      --tag $FRONTEND_IMAGE:latest
+      --tag $FRONTEND_IMAGE:$CI_COMMIT_SHORT_SHA
+      --push
+      .
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+        - frontend/**/*
+        - .gitlab-ci.yml
+
+# ── Reverse mirror to GitHub ─────────────────────────────────────────────
+# Pushes refs/heads/main to github.com/BigBodyCobain/Shadowbroker via SSH
+# using a per-repo deploy key. Fast-forward-only by default — if GitLab
+# main and GitHub main have diverged, the push fails loudly rather than
+# silently overwriting either side.
+#
+# Only runs if GITHUB_MIRROR_SSH_KEY is set as a File-type CI/CD variable.
+# See the header comment of this file for setup instructions.
+mirror-to-github:
+  stage: mirror
+  image: alpine:3.20
+  needs: []
+  before_script:
+    - apk add --no-cache git openssh-client ca-certificates
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    # Install the deploy key. File-type CI variable exposes the path; copy
+    # to ~/.ssh/id_ed25519 with restrictive perms so ssh accepts it.
+    - cp "$GITHUB_MIRROR_SSH_KEY" ~/.ssh/id_ed25519
+    - chmod 600 ~/.ssh/id_ed25519
+    # Pin github.com's current host keys so we never trust a man-in-the-
+    # middle. Sourced from https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints
+    # (rotated 2023-03-24 after the previous RSA key leak).
+    - |
+      cat > ~/.ssh/known_hosts <<'EOF'
+      github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
+      github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
+      github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
+      EOF
+    - chmod 644 ~/.ssh/known_hosts
+  script:
+    - git config --global user.email "ci-mirror@gitlab.com"
+    - git config --global user.name "GitLab CI Mirror"
+    - >
+      git clone --depth=50 --branch main
+      "https://oauth2:${CI_JOB_TOKEN}@gitlab.com/${CI_PROJECT_PATH}.git"
+      repo
+    - cd repo
+    - >
+      git push
+      "git@github.com:BigBodyCobain/Shadowbroker.git"
+      "${CI_COMMIT_SHA}:refs/heads/main"
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $GITHUB_MIRROR_SSH_KEY

.pre-commit-config.yaml

@@ -0,0 +1,32 @@
+repos:
+  - repo: local
+    hooks:
+      - id: shadowbroker-secret-scan
+        name: ShadowBroker secret scan
+        entry: bash backend/scripts/scan-secrets.sh --staged
+        language: system
+        pass_filenames: false
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-yaml
+      - id: check-json
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.9
+    hooks:
+      - id: ruff
+        args: ["--fix"]
+
+  - repo: https://github.com/psf/black
+    rev: 25.1.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.3.3
+    hooks:
+      - id: prettier

.python-version

@@ -0,0 +1 @@
+3.10

CONTRIBUTING.md

@@ -0,0 +1,75 @@
+# Contributing to Shadowbroker
+
+Thank you for taking the time to contribute. This document covers things specific to this project — for general open-source contribution etiquette, see the GitHub docs.
+
+---
+
+## Code contributions
+
+1. Fork the repo on GitHub (`bigbodycobain/Shadowbroker`) or GitLab (`bigbodycobain/Shadowbroker` mirror).
+2. Make your changes on a feature branch.
+3. Run the local test suite:
+   - Backend: `pytest backend/tests/`
+   - Frontend: `cd frontend && npx vitest run`
+4. Open a Pull Request against `main`.
+
+CI runs on every PR. If CI fails, that's blocking — please push fixes rather than asking for it to be merged anyway.
+
+---
+
+## Reporting security issues
+
+Do **not** file security issues as public GitHub issues. Email the maintainer or use a private security advisory on GitHub. Public disclosure of an exploitable vulnerability without prior coordination will be rejected from the project.
+
+---
+
+## Translation contributions
+
+Shadowbroker supports UI localization (`frontend/src/i18n/`). Translation contributions are welcome but held to a stricter standard than most code changes, because translations can subtly reshape user perception in ways that are hard to spot during review. Read this section before submitting one.
+
+### The neutrality requirement
+
+**Translations must be technically faithful to the English source.** That means:
+
+- Each `t('key')` entry should mean approximately the same thing in the target language as in English, modulo idiom.
+- Technical terms with established meanings (e.g. "GPS jamming," "military flight," "Tor," "onion routing," "encryption") should be translated using the corresponding established technical term in the target language — **not** softened, rebranded, or politically reframed.
+- The set of UI strings should be **the same** between languages. Don't omit features from one locale that are visible in another.
+
+### What will get a translation PR rejected
+
+Translation choices that align the project with the framing or terminology of state propaganda — from **any** country — will be rejected. This applies symmetrically:
+
+| Country / source | Examples of substitutions we will reject |
+|---|---|
+| **PRC / CCP** | Calling Taiwan a "province" or "renegade province"; reframing protest layers as "riots"; using softened or euphemistic terms for surveillance, internment, or jamming when the source text is direct |
+| **Russia** | Calling the Ukraine war a "special military operation"; relabeling occupied territories as Russian; softening sanctions/jamming/disinfo terminology |
+| **United States / EU** | Reframing adversaries with editorial labels not in the source (e.g. inserting "regime" where the English says "government"); applying labels like "terrorist" or "rogue state" to entities the English source describes neutrally |
+| **Israel / Palestine / any active conflict** | Substituting one side's preferred terminology when the source uses the other side's or a neutral term |
+| **Any government** | Adding political slogans, omitting features that government finds inconvenient, or inserting terminology associated with a specific political faction |
+
+The test is **"would a translator working strictly from the English source produce this rendering?"** If the answer requires assuming a political stance the source does not take, the substitution does not belong in the translation.
+
+### How translation PRs are reviewed
+
+Changes to `frontend/src/i18n/**` are owned by the maintainer (see `CODEOWNERS`) and require explicit approval. We will:
+
+1. Diff the translation against the English source key-by-key.
+2. Spot-check a sample of entries with a native speaker of the target language when possible.
+3. Look for the patterns above.
+4. Look for suspicious additions to the i18n infrastructure itself (e.g. a remote translation fetcher, telemetry on language choice) — the i18n layer is supposed to be 100% client-side static JSON.
+
+A PR that adds a new language is harder to review than one that fixes typos in an existing language. For new languages, please be patient and expect a real review window. For typo fixes, please describe each change in the PR body so the reviewer can verify intent.
+
+### What about adding a new language?
+
+We welcome new languages. The mechanical setup is documented in the header comment of `frontend/src/i18n/index.ts`. Beyond that:
+
+- We are more likely to merge a new language quickly if at least one reviewer in the maintainer's network speaks it.
+- If you are the *only* speaker of the target language reading this repo, your translation is welcome but the merge timeline will be longer while a reviewer is found.
+- Partial translations are fine — the system falls back to English for any missing key.
+
+---
+
+## Anything else
+
+If you have a question that isn't a security report, opening a GitHub Discussion or a draft PR with a question in the body is the fastest way to get a response. Direct emails are read but not always replied to promptly.

DATA-ATTRIBUTION.md

@@ -0,0 +1,72 @@
+# Data Attribution & Licensing
+
+ShadowBroker aggregates publicly available data from many third-party sources.
+This file documents each source and its license so operators and users can
+comply with the terms under which we access that data.
+
+ShadowBroker itself is licensed under AGPL-3.0 (see `LICENSE`). **This file
+concerns the *data* rendered by the dashboard, not the source code.**
+
+---
+
+## ODbL-licensed sources (Open Database License v1.0)
+
+Data from these sources is licensed under the
+[Open Database License v1.0](https://opendatacommons.org/licenses/odbl/1-0/).
+If you redistribute a derivative database built from these sources, the
+derivative must also be offered under ODbL and must preserve attribution.
+
+| Source | URL | What we use it for |
+|---|---|---|
+| adsb.lol | https://adsb.lol | Military aircraft positions, regional commercial gap-fill, route enrichment |
+| OpenStreetMap contributors | https://www.openstreetmap.org/copyright | Nominatim geocoding (LOCATE bar), CARTO basemap tiles (OSM-derived) |
+
+**Attribution requirement:** the ShadowBroker map UI displays
+"© OpenStreetMap contributors" and "adsb.lol (ODbL)" in the map attribution
+control. Do not remove this attribution if you fork or redistribute the app.
+
+---
+
+## Other third-party data sources
+
+These sources have their own terms; consult each link before redistributing.
+
+| Source | URL | License / Terms | Notes |
+|---|---|---|---|
+| OpenSky Network | https://opensky-network.org | OpenSky API terms | Commercial and private aircraft tracking |
+| CelesTrak | https://celestrak.org | Public domain / no restrictions | Satellite TLE data |
+| USGS Earthquake Hazards | https://earthquake.usgs.gov | Public domain (US Federal) | Seismic events |
+| NASA FIRMS | https://firms.modaps.eosdis.nasa.gov | NASA Open Data | Fire/thermal anomalies (VIIRS) |
+| NASA GIBS | https://gibs.earthdata.nasa.gov | NASA Open Data | MODIS imagery tiles |
+| NOAA SWPC | https://services.swpc.noaa.gov | Public domain (US Federal) | Space weather, Kp index |
+| GDELT Project | https://www.gdeltproject.org | CC BY (non-commercial friendly) | Global conflict events |
+| DeepState Map | https://deepstatemap.live | Per-site terms | Ukraine frontline GeoJSON |
+| aisstream.io | https://aisstream.io | Free-tier API terms (attribution required) | AIS vessel positions |
+| Global Fishing Watch | https://globalfishingwatch.org | CC BY 4.0 (for public data) | Fishing activity events |
+| Microsoft Planetary Computer | https://planetarycomputer.microsoft.com | Sentinel-2 / ESA Copernicus terms | Sentinel-2 imagery |
+| Copernicus CDSE (Sentinel Hub) | https://dataspace.copernicus.eu | ESA Copernicus open data terms | SAR + optical imagery, optional road-corridor truck trends |
+| DrishX / Fisser et al. 2022 | https://github.com/sparkyniner/DRISH-X-Satellite-powered-freight-intelligence- | MIT (engine); research methodology attribution | Sentinel-2 motion-smear truck detection on major roads (opt-in) |
+| Shodan | https://www.shodan.io | Operator-supplied API key, Shodan ToS | Internet device search |
+| Smithsonian GVP | https://volcano.si.edu | Attribution required | Volcanoes |
+| OpenAQ | https://openaq.org | CC BY 4.0 | Air quality stations |
+| NOAA NWS | https://www.weather.gov | Public domain (US Federal) | Severe weather alerts |
+| WRI Global Power Plant DB | https://datasets.wri.org | CC BY 4.0 | Power plants |
+| Wikidata | https://www.wikidata.org | CC0 | Head-of-state lookup |
+| Wikipedia | https://en.wikipedia.org | CC BY-SA 4.0 | Region summaries |
+| KiwiSDR (via dyatlov mirror) | http://rx.linkfanel.net | Per-site terms (community mirror by Pierre Ynard) | SDR receiver list — pulled from rx.linkfanel.net to keep load off jks-prv's bandwidth at kiwisdr.com |
+| OpenMHZ | https://openmhz.com | Per-site terms | Police/fire scanner feeds |
+| Meshtastic | https://meshtastic.org | Open Source | Mesh radio nodes (protocol) |
+| Meshtastic Map (Liam Cottle) | https://meshtastic.liamcottle.net | Community project (per-site terms) | Global Meshtastic node positions — polled once per day with on-disk cache trust to minimize load on this volunteer-run HTTP API |
+| APRS-IS | https://www.aprs-is.net | Open / attribution-based | Amateur radio positions |
+| CARTO basemaps | https://carto.com | CARTO attribution required | Dark map tiles (OSM-derived) |
+| Esri World Imagery | https://www.arcgis.com | Esri terms | High-res satellite basemap |
+| IODA (Georgia Tech) | https://ioda.inetintel.cc.gatech.edu | Research/academic terms | Internet outage data |
+
+---
+
+## Contact
+
+If you represent a data provider and have concerns about how ShadowBroker
+uses your data, please open an issue or contact the maintainer at
+`bigbodycobain@gmail.com`. We will respond promptly and, if needed, adjust
+usage or remove the source.

LICENSE

@@ -0,0 +1,661 @@
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU Affero General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<https://www.gnu.org/licenses/>.

Makefile

@@ -0,0 +1,55 @@
+.PHONY: up-local up-lan down restart-local restart-lan logs status help
+
+COMPOSE = docker compose
+
+# Detect LAN IP (tries Wi-Fi first, falls back to Ethernet)
+LAN_IP := $(shell ipconfig getifaddr en0 2>/dev/null || ipconfig getifaddr en1 2>/dev/null)
+
+## Default target — print help
+help:
+	@echo ""
+	@echo "Shadowbroker taskrunner"
+	@echo ""
+	@echo "Usage: make <target>"
+	@echo ""
+	@echo "  up-local        Start with loopback binding (local access only)"
+	@echo "  up-lan          Start with 0.0.0.0 binding (LAN accessible)"
+	@echo "  down            Stop all containers"
+	@echo "  restart-local   Bounce and restart in local mode"
+	@echo "  restart-lan     Bounce and restart in LAN mode"
+	@echo "  logs            Tail logs for all services"
+	@echo "  status          Show container status"
+	@echo ""
+
+## Start in local-only mode (loopback only)
+up-local:
+	BIND=127.0.0.1 $(COMPOSE) up -d
+
+## Start in LAN mode (accessible to other hosts on the network)
+up-lan:
+	@if [ -z "$(LAN_IP)" ]; then \
+		echo "ERROR: Could not detect LAN IP. Check your network connection."; \
+		exit 1; \
+	fi
+	@echo "Detected LAN IP: $(LAN_IP)"
+	BIND=0.0.0.0 CORS_ORIGINS=http://$(LAN_IP):3000 $(COMPOSE) up -d
+	@echo ""
+	@echo "Shadowbroker is now running and can be accessed by LAN devices at http://$(LAN_IP):3000"
+
+## Stop all containers
+down:
+	$(COMPOSE) down
+
+## Restart in local-only mode
+restart-local: down up-local
+
+## Restart in LAN mode
+restart-lan: down up-lan
+
+## Tail logs for all services
+logs:
+	$(COMPOSE) logs -f
+
+## Show running container status
+status:
+	$(COMPOSE) ps

Mesh.md

@@ -0,0 +1,89 @@
+# ShadowBroker — Meshtastic MQTT Remediation
+
+**Version:** 0.9.6  
+**Date:** 2026-04-12  
+**Re:** [meshtastic/firmware#6131](https://github.com/meshtastic/firmware/issues/6131) — Excessive MQTT traffic from ShadowBroker clients
+
+---
+
+## What happened
+
+ShadowBroker is an open-source OSINT situational awareness platform that includes a Meshtastic MQTT listener for displaying mesh network activity on a global map. In prior versions, the MQTT bridge:
+
+- Subscribed to **28 wildcard topics** (`msh/{region}/#`) covering every known official and community root on startup
+- Used an aggressive reconnect policy (min 1s / max 30s backoff)
+- Set keepalive to 30 seconds
+- Had no client-side rate limiting on inbound messages
+- Auto-started on every launch with no opt-out
+
+This produced 1-2 orders of magnitude more traffic than typical Meshtastic clients on the public broker at `mqtt.meshtastic.org`.
+
+---
+
+## What we fixed
+
+### 1. Bridge disabled by default
+
+The MQTT bridge no longer starts automatically. Operators must explicitly opt in:
+
+```env
+MESH_MQTT_ENABLED=true
+```
+
+### 2. US-only default subscription
+
+When enabled, the bridge subscribes to **1 topic** (`msh/US/#`) instead of 28. Additional regions are opt-in:
+
+```env
+MESH_MQTT_EXTRA_ROOTS=EU_868,ANZ
+```
+
+The UI still displays all regions in its dropdown — only the MQTT subscription scope changed.
+
+### 3. Client-side rate limiter
+
+Inbound messages are capped at **100 messages per minute** using a sliding window. Excess messages are silently dropped. A warning is logged periodically when the limiter activates so operators are aware.
+
+### 4. Conservative connection parameters
+
+| Parameter | Before | After |
+|-----------|--------|-------|
+| Keepalive | 30s | 120s |
+| Reconnect min delay | 1s | 15s |
+| Reconnect max delay | 30s | 300s |
+| QoS | 0 | 0 (unchanged) |
+
+### 5. Versioned client ID
+
+Client IDs changed from `sbmesh-{uuid}` to `sb096-{uuid}` so the Meshtastic team can identify ShadowBroker clients and track adoption of the fix by version.
+
+---
+
+## Configuration reference
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `MESH_MQTT_ENABLED` | `false` | Master switch for the MQTT bridge |
+| `MESH_MQTT_EXTRA_ROOTS` | _(empty)_ | Comma-separated additional region roots (e.g. `EU_868,ANZ,JP`) |
+| `MESH_MQTT_INCLUDE_DEFAULT_ROOTS` | `true` | Include US in subscriptions |
+| `MESH_MQTT_BROKER` | `mqtt.meshtastic.org` | Broker hostname |
+| `MESH_MQTT_PORT` | `1883` | Broker port |
+| `MESH_MQTT_USER` | `meshdev` | Broker username |
+| `MESH_MQTT_PASS` | `large4cats` | Broker password |
+| `MESH_MQTT_PSK` | _(empty)_ | Hex-encoded PSK (empty = default LongFast key) |
+
+---
+
+## Files changed
+
+- `backend/services/config.py` — Added `MESH_MQTT_ENABLED` flag
+- `backend/services/mesh/meshtastic_topics.py` — Reduced default roots to US-only
+- `backend/services/sigint_bridge.py` — Rate limiter, keepalive/backoff tuning, versioned client ID, opt-in gate
+- `backend/.env.example` — Documented all MQTT options
+
+---
+
+## Contact
+
+Repository: [github.com/BigBodyCobain/Shadowbroker](https://github.com/BigBodyCobain/Shadowbroker)  
+Maintainer: BigBodyCobain

backend/.dockerignore

@@ -4,13 +4,29 @@ __pycache__/
 .env
 .pytest_cache/
 .coverage
+.git/
+node_modules/
 cctv.db
+*.sqlite
+*.db
+
+# Debug/log files
 *.txt
 !requirements.txt
-# Exclude debug/cache JSON but keep package.json and tracked_names
+!requirements-dev.txt
+*.html
+*.xlsx
+
+# Debug/cache JSON (keep package*.json and data files)
 ais_cache.json
+carrier_cache.json
 carrier_positions.json
 dump.json
 debug_fast.json
 nyc_full.json
 nyc_sample.json
+tmp_fast.json
+
+# Test files (not needed in production image)
+test_*.py
+tests/

backend/.env.example

@@ -0,0 +1,405 @@
+# ShadowBroker Backend — Environment Variables
+# Copy this file to .env and fill in your keys:
+#   cp .env.example .env
+
+# ── Required Keys ──────────────────────────────────────────────
+# Without these, the corresponding data layers will be empty.
+
+OPENSKY_CLIENT_ID=        # https://opensky-network.org/ — free account, OAuth2 client ID
+OPENSKY_CLIENT_SECRET=    # OAuth2 client secret from your OpenSky dashboard
+AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
+
+# ── Optional ───────────────────────────────────────────────────
+
+# AISHub REST fallback. Used when stream.aisstream.io is unreachable
+# (e.g. their cert expires or server goes offline). Free tier requires
+# registration at https://www.aishub.net/api. Poll cadence defaults to
+# 20 min to stay courteous; tunable via AISHUB_POLL_INTERVAL_MINUTES.
+# AISHUB_USERNAME=
+# AISHUB_POLL_INTERVAL_MINUTES=20
+
+# `python main.py` (uvicorn reload) binds 127.0.0.1:8000 by default so LAN clients
+# cannot reach a dev server with empty ADMIN_KEY (#375). Set true only when you
+# intentionally need 0.0.0.0 and understand the local-trust implications.
+# SHADOWBROKER_DEV_BIND_ALL=false
+#
+# Thread pool for GDELT, LiveUAMap, CCTV ingest, and slow-tier refresh batches.
+# Keeps heavy jobs from starving fast flight/ship workers (default 2).
+# SHADOWBROKER_HEAVY_FETCH_WORKERS=2
+
+# Override allowed CORS origins (comma-separated). Defaults to localhost + LAN auto-detect.
+# CORS_ORIGINS=http://192.168.1.50:3000,https://my-domain.com
+
+# Admin key — protects sensitive endpoints (API key management, system update).
+# If unset, loopback/localhost requests still work for local single-host dev.
+# Remote/non-loopback admin access requires ADMIN_KEY, or ALLOW_INSECURE_ADMIN=true in debug-only setups.
+# Set this in production and enter the same key in Settings → Admin Key.
+# ADMIN_KEY=your-secret-admin-key-here
+
+# Allow insecure admin access without ADMIN_KEY (local dev only, beyond loopback).
+# Requires MESH_DEBUG_MODE=true; do not enable this for ordinary use.
+# ALLOW_INSECURE_ADMIN=false
+
+# Per-install operator handle. Round 7a: outbound third-party API calls send
+# this handle as the User-Agent (e.g. operator-7f3a92), not a shared app name,
+# so upstreams rate-limit one install instead of blocking every user.
+#
+# Default empty -> a stable pseudonymous handle (e.g. "operator-7f3a92") is
+# auto-generated on first run and persisted to backend/data/operator_handle.json.
+# Operators who want a meaningful handle (real name, org, GitHub login) can
+# set it here. Special characters are sanitized to dashes.
+# OPERATOR_HANDLE=
+
+# Full User-Agent override (replaces the operator handle entirely). Rare;
+# most installs should use OPERATOR_HANDLE only.
+# SHADOWBROKER_USER_AGENT=
+
+# Nominatim-specific User-Agent override (OSM usage policy). Leave unset to
+# use the per-install handle (default) — set only if you have a registered
+# Nominatim relay identity.
+# NOMINATIM_USER_AGENT=
+
+# ── Third-party fetcher opt-ins ────────────────────────────────
+# These data sources phone home to politically/commercially sensitive
+# upstreams. Disabled by default; set to "true" only if the operator
+# explicitly wants the node's IP to contact these services.
+#
+# CrowdThreat — backend.crowdthreat.world (paid threat-intel aggregator).
+# CROWDTHREAT_ENABLED=false
+#
+# EUvsDisinfo FIMI — euvsdisinfo.eu (EU disinformation tracker).
+# FIMI_ENABLED=false
+#
+# Polymarket + Kalshi — US political/election prediction markets.
+# Default off; enable from Global Threat Intercept (MKT toggle) or set true here.
+# PREDICTION_MARKETS_ENABLED=false
+# When enabled, polls use a jittered schedule (not the fixed 5-minute slow tier):
+# PREDICTION_MARKETS_INTERVAL_MINUTES=7
+# PREDICTION_MARKETS_SCHEDULER_JITTER_S=240
+# PREDICTION_MARKETS_INITIAL_DELAY_MAX_S=180
+# PREDICTION_MARKETS_PRE_FETCH_JITTER_S=90
+# PREDICTION_MARKETS_PROVIDER_GAP_JITTER_S=45
+# MESH_POLYMARKET_PAGE_DELAY_JITTER_S=0.08
+# MESH_KALSHI_PAGE_DELAY_JITTER_S=0.2
+#
+# Finnhub fallback / yfinance — financial market data.
+# Set FINNHUB_API_KEY to enable Finnhub, or set FINANCIAL_ENABLED=true to allow
+# the unauthenticated yfinance fallback to call Yahoo Finance.
+# FINANCIAL_ENABLED=false
+#
+# NUFORC UAP map layer — live scrape from nuforc.org (rolling window, default 60 days).
+# Refreshed weekly (Mon 12:00 UTC); cache reused for up to 7 days between runs.
+# NUFORC_RECENT_DAYS=60
+# NUFORC_CACHE_TTL_HOURS=168
+# On Windows, live scrape uses Python requests by default; optional:
+# SHADOWBROKER_ENABLE_WINDOWS_CURL_FALLBACK=true
+# NUFORC enrichment index (HF dataset) is separate — opt-in only:
+# NUFORC_ENABLED=false
+#
+# News RSS aggregator — defaults ON. Set to "false" to disable all
+# configured news feeds (kill switch for the news layer).
+# NEWS_ENABLED=true
+
+# Global Fishing Watch — fishing vessel activity events (Fishing Activity map layer).
+# Free API token from https://globalfishingwatch.org/our-apis/tokens
+# Without this the fishing_activity layer stays empty.
+# GFW_API_TOKEN=
+# Optional tuning — GFW can return 40k+ global events; defaults cap fetch for map paint.
+# GFW_EVENTS_PAGE_SIZE=500
+# GFW_EVENTS_MAX_PAGES=10
+# GFW_EVENTS_LOOKBACK_DAYS=7
+# GFW_EVENTS_TIMEOUT_S=90
+
+# Windy Webcams global CCTV layer — free key from https://api.windy.com/webcams/docs
+# WINDY_API_KEY=
+
+# LTA Singapore traffic cameras — leave blank to skip this data source.
+# LTA_ACCOUNT_KEY=
+
+# NASA FIRMS country-scoped fire data — enriches global CSV with conflict-zone hotspots.
+# Free MAP_KEY from https://firms.modaps.eosdis.nasa.gov/map/#d:24hrs;@0.0,0.0,3.0z
+# FIRMS_MAP_KEY=
+
+# Ukraine frontline mirror (GitHub). Default follows cyterat/deepstate-map-data@main.
+# Pin an immutable commit SHA so ingest cannot silently change if main is force-pushed (#362).
+# Example (verify on GitHub before use): main @ b479954e94696bc5622c7818fd20a64a699f4fe8
+# DEEPSTATE_MIRROR_COMMIT=b479954e94696bc5622c7818fd20a64a699f4fe8
+# DEEPSTATE_MIRROR_REPO=cyterat/deepstate-map-data
+
+# Ukraine air raid alerts from alerts.in.ua — free token from https://alerts.in.ua/
+# ALERTS_IN_UA_TOKEN=
+
+# Optional NUFORC UAP sighting map enrichment via Mapbox Tilequery.
+# Leave blank to skip this optional enrichment.
+# NUFORC_MAPBOX_TOKEN=
+
+# Google Earth Engine service account for VIIRS change detection (optional).
+# Download JSON key from https://console.cloud.google.com/iam-admin/serviceaccounts
+# pip install earthengine-api
+# GEE_SERVICE_ACCOUNT_KEY=
+
+# ── Meshtastic MQTT Bridge ─────────────────────────────────────
+# Disabled by default to respect the public Meshtastic broker.
+# When enabled, subscribes to US region only. Add more regions via MESH_MQTT_EXTRA_ROOTS.
+# MESH_MQTT_ENABLED=false
+# MESH_MQTT_EXTRA_ROOTS=EU_868,ANZ    # comma-separated additional region roots
+# MESH_MQTT_INCLUDE_DEFAULT_ROOTS=true
+# MESH_MQTT_BROKER=mqtt.meshtastic.org
+# MESH_MQTT_PORT=1883
+# Leave user/pass blank for the public Meshtastic broker default.
+# MESH_MQTT_USER=
+# MESH_MQTT_PASS=
+
+# Optional Meshtastic node ID (e.g. "!abcd1234"). When set, included in the
+# User-Agent sent to meshtastic.liamcottle.net so the upstream service operator
+# can identify per-install traffic instead of aggregated "ShadowBroker" hits.
+# Leave blank to send a generic UA. If you set MESHTASTIC_OPERATOR_CALLSIGN,
+# it is included in outbound headers to meshtastic.org by default so they
+# can rate-limit per-operator. Callsign is NOT sent upstream unless you opt in.
+# MESHTASTIC_OPERATOR_CALLSIGN=
+# MESHTASTIC_SEND_CALLSIGN_HEADER=false
+# MESH_MQTT_PSK=                       # hex-encoded, empty = default LongFast key
+
+# LiveUAMap Playwright scraper (#348). Linux/macOS: on by default when Global
+# Incidents layer is active. Windows: off until the operator enables Global
+# Incidents in the UI (consent dialog) or sets SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=true.
+# SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=false forces off on all platforms.
+
+# ── Mesh / Reticulum (RNS) ─────────────────────────────────────
+# Full-node / participant-node posture for public Infonet sync.
+# MESH_NODE_MODE=participant      # participant | relay | perimeter
+# Legacy compatibility sunset toggles. Default posture is to block these.
+# Legacy 16-hex node-id binding no longer has a boolean escape hatch; use a
+# dated migration override only when you intentionally need older peers during
+# migration before the hard removal target in v0.10.0 / 2026-06-01.
+# MESH_BLOCK_LEGACY_NODE_ID_COMPAT=true
+# MESH_ALLOW_LEGACY_NODE_ID_COMPAT_UNTIL=2026-05-15
+# MESH_BLOCK_LEGACY_AGENT_ID_LOOKUP=true
+# Temporary DM invite migration escape hatch. Default posture blocks importing
+# legacy/compat v1/v2 DM invites; use a dated override only while retiring
+# older exports and ask senders to re-export a current signed invite.
+# MESH_ALLOW_COMPAT_DM_INVITE_IMPORT_UNTIL=2026-05-15
+# Temporary legacy GET DM poll/count escape hatch. Default posture requires the
+# signed mailbox-claim POST APIs; only use this dated override while retiring
+# older clients that still call GET poll/count directly.
+# MESH_ALLOW_LEGACY_DM_GET_UNTIL=2026-05-15
+# Temporary raw dm1 compose/decrypt escape hatch. Default posture expects MLS
+# DM bootstrap on supported peers; only use this dated override while retiring
+# older clients that still need the raw dm1 helper path.
+# MESH_ALLOW_LEGACY_DM1_UNTIL=2026-05-15
+# Temporary legacy dm_message signature escape hatch. Default posture requires
+# the full modern signed payload; only enable this with a dated migration
+# override while older senders are being retired.
+# MESH_ALLOW_LEGACY_DM_SIGNATURE_COMPAT_UNTIL=2026-05-15
+# Rotate voter-blinding salts so new reputation events stop reusing one
+# forever-stable blinded ID. Keep grace >= rotation cadence so older votes
+# remain matchable while they age out of the ledger.
+# MESH_VOTER_BLIND_SALT_ROTATE_DAYS=30
+# MESH_VOTER_BLIND_SALT_GRACE_DAYS=30
+# Deprecated legacy env vars kept only for backward config compatibility.
+# Ordinary shipped gate flows keep MLS decrypt local; service-side decrypt is
+# reserved for explicit recovery reads.
+# MESH_GATE_BACKEND_DECRYPT_COMPAT=false
+# MESH_GATE_BACKEND_DECRYPT_COMPAT_ACKNOWLEDGE=false
+# Deprecated legacy env vars kept only for backward config compatibility.
+# Ordinary shipped gate flows keep plaintext compose/post local and only submit
+# encrypted envelopes to the backend for sign/post.
+# MESH_GATE_BACKEND_PLAINTEXT_COMPAT=false
+# MESH_GATE_BACKEND_PLAINTEXT_COMPAT_ACKNOWLEDGE=false
+# Legacy runtime switches for recovery envelopes. Per-gate envelope_policy is
+# the source of truth; leave these at the default unless testing old behavior.
+# MESH_GATE_RECOVERY_ENVELOPE_ENABLE=true
+# MESH_GATE_RECOVERY_ENVELOPE_ENABLE_ACKNOWLEDGE=true
+# Optional operator-only recovery tradeoff. Leave off for the default posture:
+# ordinary gate reads keep plaintext local/in-memory unless you explicitly use
+# the recovery-envelope path.
+# MESH_GATE_PLAINTEXT_PERSIST=false
+# MESH_GATE_PLAINTEXT_PERSIST_ACKNOWLEDGE=false
+# Legacy Phase-1 gate envelope fallback is now explicit and time-bounded per
+# gate. This only controls the default expiry window when you deliberately
+# re-enable that migration path for older stored envelopes.
+# MESH_GATE_LEGACY_ENVELOPE_FALLBACK_MAX_DAYS=30
+# Feature-flagged multiplexed gate session stream. Stream-first room ownership
+# is implemented; keep off until you want that rollout enabled in your env.
+# MESH_GATE_SESSION_STREAM_ENABLED=false
+# MESH_GATE_SESSION_STREAM_HEARTBEAT_S=20
+# MESH_GATE_SESSION_STREAM_BATCH_MS=1500
+# MESH_GATE_SESSION_STREAM_MAX_GATES=16
+# MESH_BOOTSTRAP_DISABLED=false
+# MESH_BOOTSTRAP_MANIFEST_PATH=data/bootstrap_peers.json
+# MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY=
+# Infonet/Wormhole fails closed to onion/RNS by default. Only enable clearnet
+# sync for local relay development or an explicitly public testnet.
+# MESH_INFONET_ALLOW_CLEARNET_SYNC=false
+# MESH_BOOTSTRAP_SEED_PEERS=http://gqpbunqbgtkcqilvclm3xrkt3zowjyl3s62kkktvojgvxzizamvbrqid.onion:8000
+# Add comma-separated http://*.onion peers as more private seed/relay nodes come online.
+# MESH_DEFAULT_SYNC_PEERS=         # legacy alias; prefer MESH_BOOTSTRAP_SEED_PEERS
+# MESH_RELAY_PEERS=               # comma-separated operator-trusted sync/push peers (empty by default)
+# MESH_PEER_PUSH_SECRET=          # REQUIRED when relay/RNS peers are configured (min 16 chars, generate with: python -c "import secrets; print(secrets.token_urlsafe(32))")
+# MESH_SYNC_INTERVAL_S=300
+# MESH_SYNC_FAILURE_BACKOFF_S=60
+#
+# Enable Reticulum bridge for Infonet event gossip.
+# MESH_RNS_ENABLED=false
+# MESH_RNS_APP_NAME=shadowbroker
+# MESH_RNS_ASPECT=infonet
+# MESH_RNS_IDENTITY_PATH=
+# MESH_RNS_PEERS=  # comma-separated destination hashes
+# MESH_RNS_DANDELION_HOPS=2
+# MESH_RNS_DANDELION_DELAY_MS=400
+# MESH_RNS_CHURN_INTERVAL_S=300
+# MESH_RNS_MAX_PEERS=32
+# MESH_RNS_MAX_PAYLOAD=8192
+# MESH_RNS_PEER_BUCKET_PREFIX=4
+# MESH_RNS_MAX_PEERS_PER_BUCKET=4
+# MESH_RNS_PEER_FAIL_THRESHOLD=3
+# MESH_RNS_PEER_COOLDOWN_S=300
+# MESH_RNS_SHARD_ENABLED=false
+# MESH_RNS_SHARD_DATA_SHARDS=3
+# MESH_RNS_SHARD_PARITY_SHARDS=1
+# MESH_RNS_SHARD_TTL_S=30
+# MESH_RNS_FEC_CODEC=xor
+# MESH_RNS_BATCH_MS=200
+# MESH_RNS_COVER_INTERVAL_S=0
+# MESH_RNS_COVER_SIZE=64
+# MESH_RNS_IBF_WINDOW=256
+# MESH_RNS_IBF_TABLE_SIZE=64
+# MESH_RNS_IBF_MINHASH_SIZE=16
+# MESH_RNS_IBF_MINHASH_THRESHOLD=0.25
+# MESH_RNS_IBF_WINDOW_JITTER=32
+# MESH_RNS_IBF_INTERVAL_S=120
+# MESH_RNS_IBF_SYNC_PEERS=3
+# MESH_RNS_IBF_QUORUM_TIMEOUT_S=6
+# MESH_RNS_IBF_MAX_REQUEST_IDS=64
+# MESH_RNS_IBF_MAX_EVENTS=64
+# MESH_RNS_SESSION_ROTATE_S=0
+# MESH_RNS_IBF_FAIL_THRESHOLD=3
+# MESH_RNS_IBF_COOLDOWN_S=120
+# MESH_VERIFY_INTERVAL_S=600
+# MESH_VERIFY_SIGNATURES=false
+
+# ── Secure Storage (non-Windows) ───────────────────────────────
+# Required on Linux/Docker to protect Wormhole key material at rest.
+# Generate with: python -c "import secrets; print(secrets.token_urlsafe(32))"
+# Also supports Docker secrets via MESH_SECURE_STORAGE_SECRET_FILE.
+# MESH_SECURE_STORAGE_SECRET=
+#
+# To rotate the storage secret, stop the backend and run:
+#   1. Dry-run first (validates without writing):
+#      MESH_OLD_STORAGE_SECRET=<current> MESH_NEW_STORAGE_SECRET=<new> \
+#        python -m scripts.rotate_secure_storage_secret --dry-run
+#   2. Rotate (creates .bak backups, then rewraps envelopes):
+#      MESH_OLD_STORAGE_SECRET=<current> MESH_NEW_STORAGE_SECRET=<new> \
+#        python -m scripts.rotate_secure_storage_secret
+#   3. Update MESH_SECURE_STORAGE_SECRET to the new value and restart.
+#
+# If rotation is interrupted, .bak files preserve the old envelopes.
+# To repair corrupted secure-json payloads (not key envelopes), use:
+#   python -m scripts.repair_wormhole_secure_storage
+
+# ── Mesh DM Relay ──────────────────────────────────────────────
+# MESH_DM_TOKEN_PEPPER=change-me
+# Keep DM relay metadata retention explicit and bounded.
+# MESH_DM_KEY_TTL_DAYS=30
+# MESH_DM_PREKEY_LOOKUP_ALIAS_TTL_DAYS=14
+# MESH_DM_WITNESS_TTL_DAYS=14
+# MESH_DM_BINDING_TTL_DAYS=3
+# Optional operational bridge for externally sourced root witnesses / transparency.
+# Relative paths resolve from the backend directory.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH=data/root_witness_import.json
+# Local single-host dev example after bootstrapping an external witness locally:
+# MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH=../ops/root_witness_receipt_import.json
+# Optional URI bridge for externally retrieved root witness packages.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_URI=file:///absolute/path/root_witness_import.json
+# Maximum acceptable age for external witness packages before strong DM trust fails closed.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_MAX_AGE_S=3600
+# Warning threshold for external witness packages before fail-closed max age.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_WARN_AGE_S=2700
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH=data/root_transparency_ledger.json
+# Local single-host dev example after publishing the transparency ledger locally:
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH=../ops/root_transparency_ledger.json
+# Optional URI used to read back and verify a published transparency ledger.
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI=file:///absolute/path/root_transparency_ledger.json
+# Local single-host dev readback example:
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI=../ops/root_transparency_ledger.json
+# Maximum acceptable age for external transparency ledgers before strong DM trust fails closed.
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_MAX_AGE_S=3600
+# Warning threshold for external transparency ledgers before fail-closed max age.
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_WARN_AGE_S=2700
+
+# ── Self Update ────────────────────────────────────────────────
+# MESH_UPDATE_SHA256=
+
+# ── Wormhole (Local Agent) ─────────────────────────────────────
+# WORMHOLE_HOST=127.0.0.1
+# WORMHOLE_PORT=8787
+# WORMHOLE_RELOAD=false
+# WORMHOLE_TRANSPORT=direct
+# WORMHOLE_SOCKS_PROXY=127.0.0.1:9050
+# WORMHOLE_SOCKS_DNS=true
+# Optional override for the loaded Rust privacy-core shared library. Leave
+# unset for the default repo search order. When you override this, verify the
+# authenticated wormhole status surfaces show the expected version, absolute
+# library path, and SHA-256 for the loaded artifact before making stronger
+# privacy claims about the deployment.
+# PRIVACY_CORE_LIB=
+# Minimum privacy-core version accepted when hidden/private carriers are
+# enabled. Private-lane startup fails closed if the loaded artifact is
+# missing, reports no parseable version, or falls below this minimum.
+# PRIVACY_CORE_MIN_VERSION=0.1.0
+# Comma-separated SHA-256 allowlist for the exact privacy-core artifact(s)
+# your deployment is allowed to load. Required for Arti/RNS private-lane
+# startup. Generate with:
+#   PowerShell: Get-FileHash .\privacy-core\target\release\privacy_core.dll -Algorithm SHA256
+#   macOS/Linux: sha256sum ./privacy-core/target/release/libprivacy_core.so
+# PRIVACY_CORE_ALLOWED_SHA256=
+# Optional structured release attestation artifact for the Sprint 8 release gate.
+# Relative paths resolve from the backend directory. When set explicitly, a
+# missing or unreadable file fails the DM relay security-suite criterion closed.
+# CI/release tooling can generate this automatically via:
+#   uv run python scripts/release_helper.py write-attestation ...
+# MESH_RELEASE_ATTESTATION_PATH=data/release_attestation.json
+# Operator-only Sprint 8 release attestation. Set this only when the DM relay
+# security suite has been run and passed for the current release candidate.
+# File-based release attestation takes precedence when present.
+# MESH_RELEASE_DM_RELAY_SECURITY_SUITE_GREEN=false
+
+# ── OpenClaw Agent ─────────────────────────────────────────────
+# HMAC shared secret for remote OpenClaw agent authentication.
+# Auto-generated via the Connect OpenClaw modal — do not set manually.
+# OPENCLAW_HMAC_SECRET=
+# Access tier: "restricted" (read-only) or "full" (read+write+inject)
+# OPENCLAW_ACCESS_TIER=restricted
+
+# ── SAR (Synthetic Aperture Radar) Layer ───────────────────────
+# Mode A — Free catalog metadata from Alaska Satellite Facility (ASF Search).
+# No account, no downloads.  Default-on.  Set to false to disable entirely.
+# MESH_SAR_CATALOG_ENABLED=true
+#
+# Mode B — Free pre-processed ground-change anomalies (deformation, flood,
+# damage assessments) from NASA OPERA, Copernicus EGMS, GFM, EMS, UNOSAT.
+# Two-step opt-in: BOTH of the following must be set together.
+#   1. MESH_SAR_PRODUCTS_FETCH=allow
+#   2. MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE=true
+# Either flag alone keeps Mode B disabled.  You can also enable this from
+# the Settings → SAR panel inside the app.
+# MESH_SAR_PRODUCTS_FETCH=block
+# MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE=false
+#
+# NASA Earthdata Login (free, ~1 minute signup) — required for OPERA products.
+#   Sign up: https://urs.earthdata.nasa.gov/users/new
+#   Generate token: https://urs.earthdata.nasa.gov/profile  →  "Generate Token"
+# MESH_SAR_EARTHDATA_USER=
+# MESH_SAR_EARTHDATA_TOKEN=
+#
+# Copernicus Data Space (free, ~1 minute signup) — required for EGMS / EMS.
+#   Sign up: https://dataspace.copernicus.eu/
+# MESH_SAR_COPERNICUS_USER=
+# MESH_SAR_COPERNICUS_TOKEN=
+#
+# Allow OpenClaw agents to read and act on the SAR layer (default true).
+# MESH_SAR_OPENCLAW_ENABLED=true
+#
+# Require private-tier transport (Tor / RNS) before signing and broadcasting
+# SAR anomalies to the mesh.  Default true — disable only for testnet/local use.
+# MESH_SAR_REQUIRE_PRIVATE_TIER=true

backend/Dockerfile

@@ -1,27 +1,81 @@
-FROM python:3.10-slim
+# ---- Stage 1: Compile privacy-core Rust library ----
+FROM --platform=$BUILDPLATFORM rust:1.88-slim-bookworm AS rust-builder
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    git \
+    pkg-config \
+    libssl-dev \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
+ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
+
+COPY privacy-core /build/privacy-core
+WORKDIR /build/privacy-core
+RUN cargo build --release --lib \
+    && ls -la target/release/libprivacy_core.so
+
+# ---- Stage 2: Python backend ----
+FROM python:3.11-slim-bookworm
 
 WORKDIR /app
 
-# Install Node.js (for AIS WebSocket proxy) and curl (for network fallback)
+# Install Node.js (for AIS WebSocket proxy), curl (for network fallback), and
+# Tor (for Wormhole/remote-agent .onion transport).
 RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
     curl \
+    tor \
     && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
     && apt-get install -y --no-install-recommends nodejs \
     && rm -rf /var/lib/apt/lists/*
 
-# Install dependencies
+# Install UV for fast, reproducible Python dependency management
-COPY requirements.txt .
+ADD https://astral.sh/uv/install.sh /uv-installer.sh
-RUN pip install --no-cache-dir -r requirements.txt
+RUN sh /uv-installer.sh && rm /uv-installer.sh
+ENV PATH="/root/.local/bin:$PATH"
+# Install into system Python (no venv needed inside container)
+ENV UV_PROJECT_ENVIRONMENT=/usr/local
 
-# Copy source code
+# Copy workspace root files for UV resolution (build context is repo root)
-COPY . .
+COPY pyproject.toml /workspace/pyproject.toml
+COPY uv.lock /workspace/uv.lock
+COPY backend/pyproject.toml /workspace/backend/pyproject.toml
+
+# Install Python dependencies using the lockfile
+RUN cd /workspace/backend && uv sync --frozen --no-dev --extra road-corridor \
+    && playwright install --with-deps chromium
+
+# Copy backend source code
+COPY backend/ .
+
+# Preserve safe static data outside /app/data. The compose named volume mounted
+# at /app/data hides image-baked files on first run, so the entrypoint seeds
+# missing static JSON into fresh volumes before the backend starts.
+RUN mkdir -p /app/image-data \
+    && if [ -d /app/data ]; then cp -a /app/data/. /app/image-data/; fi \
+    && chmod +x /app/docker-entrypoint.sh
 
 # Install Node.js dependencies (ws module for AIS WebSocket proxy)
-RUN npm install --omit=dev
+COPY backend/package*.json ./
+RUN npm ci --omit=dev
+
+# Clean up workspace scaffold
+RUN rm -rf /workspace
+
+# Copy compiled privacy-core library from Rust builder stage
+COPY --from=rust-builder /build/privacy-core/target/release/libprivacy_core.so /app/libprivacy_core.so
+ENV PRIVACY_CORE_LIB=/app/libprivacy_core.so
 
 # Create a non-root user for security
+# Grant write access to /app so the auto-updater can extract files
+# Pre-create /app/data so mounted volumes inherit correct ownership
 RUN adduser --system --uid 1001 backenduser \
-    && chown -R backenduser /app
+    && mkdir -p /app/data \
+    && chown -R backenduser /app \
+    && chmod -R u+w /app
 
 # Switch to the non-root user
 USER backenduser
@@ -30,4 +84,5 @@ USER backenduser
 EXPOSE 8000
 
 # Start FastAPI server
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
+ENTRYPOINT ["/app/docker-entrypoint.sh"]
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--timeout-keep-alive", "120"]

backend/ais_proxy.js

@@ -1,4 +1,37 @@
+// AIS Stream WebSocket proxy.
+//
+// Reads AIS_API_KEY from argv or env, opens a wss:// connection to
+// stream.aisstream.io, subscribes for vessel position reports inside the
+// active map bounding boxes, and pipes JSON messages to stdout for the
+// Python backend to ingest.
+//
+// Issue #258 — SPKI pinning fallback for upstream cert outages
+// -------------------------------------------------------------
+// AISStream uses Let's Encrypt and their renewal pipeline has been observed
+// to fail (cert expired on 2026-05-20). The naive fix the issue reporter
+// applied — passing { rejectUnauthorized: false } — turns off TLS validation
+// entirely, which lets any network attacker MITM the WebSocket and inject
+// fake ship positions onto the operator's map. Same class as the GDELT
+// plaintext-HTTP MITM issue (#199).
+//
+// Instead, when the normal TLS handshake fails with CERT_HAS_EXPIRED, we
+// do a custom TLS connection that ignores ONLY the expiry check, capture
+// the leaf certificate, and compare its public-key SPKI hash against a
+// pinned list (backend/data/aisstream_spki_pins.json). If the SPKI matches,
+// the upstream is still the genuine AISStream — just with an expired cert —
+// and we proceed in "degraded TLS" mode. If the SPKI does not match, we
+// refuse the connection and log loudly: an actual MITM is in progress.
+//
+// Let's Encrypt renewals keep the same public key by default, so the pinned
+// SPKI survives normal cert rotation. The pin list MUST be updated before
+// the operator's pinned key is rotated upstream.
+
 const WebSocket = require('ws');
+const readline = require('readline');
+const fs = require('fs');
+const path = require('path');
+const tls = require('tls');
+const crypto = require('crypto');
 
 const args = process.argv.slice(2);
 const API_KEY = args[0] || process.env.AIS_API_KEY;
@@ -8,22 +41,144 @@ if (!API_KEY) {
     process.exit(1);
 }
 
-const FILTER = [
+// ── SPKI pin support (issue #258) ─────────────────────────────────────────
-    // US Aircraft Carriers and major naval groups
-    { "MMSI": 338000000 }, { "MMSI": 338100000 }, // US Navy general prefixes
-    // Plus let's grab some global shipping for density
-    { "BoundingBoxes": [[[-90, -180], [90, 180]]] }
-];
 
-function connect() {
+const AIS_HOST = 'stream.aisstream.io';
-    const ws = new WebSocket('wss://stream.aisstream.io/v0/stream');
+const AIS_PORT = 443;
+const AIS_WS_URL = `wss://${AIS_HOST}/v0/stream`;
 
-    ws.on('open', () => {
+// Pin file is looked up in several layouts so the same JS works in:
+//   - the Docker backend image (PIN_FILE_CANDIDATES[0])
+//   - the Tauri desktop runtime (PIN_FILE_CANDIDATES[1])
+//   - a future relocated layout (operator can drop a file at
+//     SHADOWBROKER_AIS_PINS env var)
+const PIN_FILE_CANDIDATES = [
+    process.env.SHADOWBROKER_AIS_PINS || '',
+    path.join(__dirname, 'data', 'aisstream_spki_pins.json'),
+    path.join(__dirname, 'aisstream_spki_pins.json'),
+].filter(Boolean);
+
+// Embedded fallback. Used when no external pin file is reachable so the
+// SPKI fallback still works on minimal/portable installs. The external
+// file (when present) takes priority so operators can update pins without
+// needing a new build.
+const EMBEDDED_PINS = {
+    [AIS_HOST]: [
+        // Captured 2026-05-20 from AISStream's leaf cert (Let's Encrypt R12).
+        // Replace when AISStream rotates server keys.
+        'GJ10H0UPgLrO+2d3ZXROR/TXSVFXKUfRC3QEI2ibEg4=',
+    ],
+};
+
+let aisDegradedMode = false;  // surfaced via stdout status_query marker
+
+function loadSpkiPins() {
+    for (const candidate of PIN_FILE_CANDIDATES) {
+        try {
+            const raw = fs.readFileSync(candidate, 'utf-8');
+            const parsed = JSON.parse(raw);
+            const pins = Array.isArray(parsed[AIS_HOST]) ? parsed[AIS_HOST] : [];
+            const cleaned = pins
+                .filter((p) => typeof p === 'string' && p.length > 0)
+                .map((p) => p.trim());
+            if (cleaned.length > 0) {
+                return cleaned;
+            }
+        } catch (e) {
+            // Try the next candidate — file may not exist in this layout.
+            continue;
+        }
+    }
+    const embedded = (EMBEDDED_PINS[AIS_HOST] || []).slice();
+    if (embedded.length > 0) {
+        console.error(
+            '[AIS Proxy] No external SPKI pin file found; using embedded fallback. '
+            + `(Set SHADOWBROKER_AIS_PINS or drop ${PIN_FILE_CANDIDATES[1]} to override.)`
+        );
+    }
+    return embedded;
+}
+
+function spkiHashFromPeerCert(peerCert) {
+    // tls.TLSSocket.getPeerCertificate() exposes .pubkey when called with
+    // detailed=true. The pubkey buffer is the DER-encoded SubjectPublicKeyInfo,
+    // which is exactly the value we hash for SPKI pinning.
+    if (!peerCert || !peerCert.pubkey) return null;
+    return crypto.createHash('sha256').update(peerCert.pubkey).digest('base64');
+}
+
+// Probe the upstream when normal TLS failed with CERT_HAS_EXPIRED. We open
+// a raw TLS connection with rejectUnauthorized=false ONLY to inspect the
+// leaf cert; we do NOT use this socket for the actual WebSocket traffic.
+// Returns { ok: true } if the leaf SPKI matches the pin list, { ok: false }
+// with a reason otherwise.
+function verifyExpiredCertAgainstPins() {
+    return new Promise((resolve) => {
+        const pins = loadSpkiPins();
+        if (pins.length === 0) {
+            resolve({ ok: false, reason: 'no SPKI pins configured' });
+            return;
+        }
+        const sock = tls.connect(
+            {
+                host: AIS_HOST,
+                port: AIS_PORT,
+                servername: AIS_HOST,
+                // Allow the handshake to complete despite the expired cert
+                // so we can inspect the leaf. We do NOT trust this connection
+                // for any application data.
+                rejectUnauthorized: false,
+            },
+            () => {
+                const peer = sock.getPeerCertificate(true);
+                sock.end();
+                if (!peer || Object.keys(peer).length === 0) {
+                    resolve({ ok: false, reason: 'no peer certificate returned' });
+                    return;
+                }
+                if (peer.subject && peer.subject.CN !== AIS_HOST) {
+                    resolve({
+                        ok: false,
+                        reason: `cert CN mismatch (got ${peer.subject.CN}, expected ${AIS_HOST})`,
+                    });
+                    return;
+                }
+                const hash = spkiHashFromPeerCert(peer);
+                if (!hash) {
+                    resolve({ ok: false, reason: 'could not compute SPKI hash from peer cert' });
+                    return;
+                }
+                if (pins.includes(hash)) {
+                    resolve({ ok: true, hash });
+                } else {
+                    resolve({
+                        ok: false,
+                        reason: `SPKI ${hash} not in pin list (possible MITM)`,
+                    });
+                }
+            },
+        );
+        sock.setTimeout(10000, () => {
+            sock.destroy();
+            resolve({ ok: false, reason: 'TLS probe timeout' });
+        });
+        sock.on('error', (err) => {
+            resolve({ ok: false, reason: `TLS probe error: ${err.message}` });
+        });
+    });
+}
+
+// ── Subscription state ───────────────────────────────────────────────────
+
+// Start with global coverage, until frontend updates it
+let currentBboxes = [[[-90, -180], [90, 180]]];
+let activeWs = null;
+
+function sendSub(ws) {
+    if (ws && ws.readyState === WebSocket.OPEN) {
         const subMsg = {
             APIKey: API_KEY,
-            BoundingBoxes: [
+            BoundingBoxes: currentBboxes,
-                [[-90, -180], [90, 180]]
-            ],
             FilterMessageTypes: [
                 "PositionReport",
                 "ShipStaticData",
@@ -31,27 +186,119 @@ function connect() {
             ]
         };
         ws.send(JSON.stringify(subMsg));
+    }
+}
+
+// Listen for dynamic bounding box updates via stdin from Python orchestrator
+const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+    terminal: false
+});
+
+rl.on('line', (line) => {
+    try {
+        const cmd = JSON.parse(line);
+        if (cmd.type === "update_bbox" && cmd.bboxes) {
+            currentBboxes = cmd.bboxes;
+            if (activeWs) sendSub(activeWs); // Resend subscription (swap and replace)
+        }
+        if (cmd.type === "status_query") {
+            // Allow the Python side to probe degraded-mode state by sending
+            // {"type": "status_query"} on stdin. Reply on stdout as a marker.
+            process.stdout.write(JSON.stringify({
+                __ais_proxy_status: { degraded_tls: aisDegradedMode }
+            }) + '\n');
+        }
+    } catch (e) {}
+});
+
+function attachWsHandlers(ws, { degraded } = { degraded: false }) {
+    activeWs = ws;
+
+    ws.on('open', () => {
+        if (degraded) {
+            console.error(
+                '[AIS Proxy] Connected in DEGRADED TLS MODE — upstream cert is expired '
+                + 'but SPKI matches the pinned key, so identity is still verified. '
+                + 'AISStream needs to renew their cert; until then MITM protection '
+                + 'depends only on the SPKI match. Watch backend logs for resolution.'
+            );
+            aisDegradedMode = true;
+        } else {
+            if (aisDegradedMode) {
+                console.error('[AIS Proxy] Reconnected with full TLS validation — degraded mode cleared.');
+            }
+            aisDegradedMode = false;
+        }
+        sendSub(ws);
     });
 
     ws.on('message', (data) => {
-        // Output raw AIS message JSON to stdout so Python can consume it
-        // We ensure exactly one JSON object per line.
         try {
             const parsed = JSON.parse(data);
             console.log(JSON.stringify(parsed));
-        } catch (e) {
+        } catch (e) {}
-            // ignore non-json
-        }
     });
 
     ws.on('error', (err) => {
-        console.error("WebSocket Proxy Error:", err.message);
+        console.error('WebSocket Proxy Error:', err.message);
     });
 
     ws.on('close', () => {
-        console.error("WebSocket Proxy Closed. Reconnecting in 5s...");
+        activeWs = null;
+        console.error('WebSocket Proxy Closed. Reconnecting in 5s...');
         setTimeout(connect, 5000);
     });
 }
 
+function connect() {
+    // Path A: normal TLS validation (the 99.9% case). If this succeeds we
+    // never touch the SPKI fallback.
+    const ws = new WebSocket(AIS_WS_URL);
+
+    let openedOk = false;
+    ws.on('open', () => { openedOk = true; });
+
+    ws.on('error', async (err) => {
+        // Only the CERT_HAS_EXPIRED case triggers SPKI verification. Any
+        // other TLS or network error gets the standard reconnect path so we
+        // don't accidentally cover up legitimate problems.
+        if (!openedOk && err && err.code === 'CERT_HAS_EXPIRED') {
+            console.error(
+                '[AIS Proxy] Upstream certificate is expired. Verifying SPKI '
+                + 'against pinned keys before deciding whether to proceed in '
+                + 'degraded mode...'
+            );
+            const verdict = await verifyExpiredCertAgainstPins();
+            if (verdict.ok) {
+                console.error(
+                    `[AIS Proxy] SPKI ${verdict.hash} matches pinned key — `
+                    + 'identity is verified, proceeding in DEGRADED TLS mode.'
+                );
+                const insecureWs = new WebSocket(AIS_WS_URL, {
+                    rejectUnauthorized: false,
+                });
+                attachWsHandlers(insecureWs, { degraded: true });
+            } else {
+                console.error(
+                    `[AIS Proxy] SPKI verification FAILED (${verdict.reason}). `
+                    + 'Refusing to connect — this would normally indicate an active '
+                    + 'MITM attack. If AISStream rotated their server key, update '
+                    + 'backend/data/aisstream_spki_pins.json with the new SPKI hash.'
+                );
+                // Schedule a retry — operator may have updated the pin file.
+                setTimeout(connect, 60000);
+            }
+            return;
+        }
+        // Default: surface the error and let the close handler reconnect.
+        console.error('WebSocket Proxy Error:', err.message);
+    });
+
+    // Wire normal handlers — these apply unless the error handler above
+    // takes over and replaces activeWs with an insecure socket.
+    attachWsHandlers(ws, { degraded: false });
+}
+
 connect();

backend/check_regions.py

@@ -1,17 +0,0 @@
-import requests
-
-regions = [
-    {"lat": 39.8, "lon": -98.5, "dist": 2000},  # USA
-    {"lat": 50.0, "lon": 15.0, "dist": 2000},   # Europe
-    {"lat": 35.0, "lon": 105.0, "dist": 2000}   # Asia / China
-]
-
-for r in regions:
-    url = f"https://api.adsb.lol/v2/lat/{r['lat']}/lon/{r['lon']}/dist/{r['dist']}"
-    res = requests.get(url, timeout=10)
-    if res.status_code == 200:
-        data = res.json()
-        acs = data.get("ac", [])
-        print(f"Region lat:{r['lat']} lon:{r['lon']} dist:{r['dist']} -> Flights: {len(acs)}")
-    else:
-        print(f"Error for Region lat:{r['lat']} lon:{r['lon']}: HTTP {res.status_code}")

backend/clean_osm_cctvs.py

@@ -1,10 +0,0 @@
-import sqlite3
-import os
-
-db_path = os.path.join(os.path.dirname(__file__), 'cctv.db')
-conn = sqlite3.connect(db_path)
-cur = conn.cursor()
-cur.execute("DELETE FROM cameras WHERE id LIKE 'OSM-%'")
-print(f"Deleted {cur.rowcount} OSM cameras from DB.")
-conn.commit()
-conn.close()

backend/config/news_feeds.json

@@ -7,7 +7,7 @@
     },
     {
       "name": "BBC",
-      "url": "http://feeds.bbci.co.uk/news/world/rss.xml",
+      "url": "https://feeds.bbci.co.uk/news/world/rss.xml",
       "weight": 3
     },
     {
@@ -26,19 +26,89 @@
       "weight": 5
     },
     {
-      "name": "NHK",
+      "name": "The War Zone",
-      "url": "https://www3.nhk.or.jp/nhkworld/rss/world.xml",
+      "url": "https://www.twz.com/feed",
+      "weight": 4
+    },
+    {
+      "name": "Bellingcat",
+      "url": "https://www.bellingcat.com/feed/",
+      "weight": 4
+    },
+    {
+      "name": "Guardian",
+      "url": "https://www.theguardian.com/world/rss",
       "weight": 3
     },
+    {
+      "name": "TASS",
+      "url": "https://tass.com/rss/v2.xml",
+      "weight": 2
+    },
+    {
+      "name": "Xinhua",
+      "url": "https://www.news.cn/english/rss/worldrss.xml",
+      "weight": 2
+    },
     {
       "name": "CNA",
-      "url": "https://www.channelnewsasia.com/rssfeed/8395986",
+      "url": "https://www.channelnewsasia.com/api/v1/rss-outbound-feed?_format=xml",
       "weight": 3
     },
     {
       "name": "Mercopress",
       "url": "https://en.mercopress.com/rss/",
       "weight": 3
+    },
+    {
+      "name": "SCMP",
+      "url": "https://www.scmp.com/rss/91/feed",
+      "weight": 4
+    },
+    {
+      "name": "The Diplomat",
+      "url": "https://thediplomat.com/feed/",
+      "weight": 4
+    },
+    {
+      "name": "Yonhap",
+      "url": "https://en.yna.co.kr/RSS/news.xml",
+      "weight": 4
+    },
+    {
+      "name": "Asia Times",
+      "url": "https://asiatimes.com/feed/",
+      "weight": 3
+    },
+    {
+      "name": "Defense News",
+      "url": "https://www.defensenews.com/arc/outboundfeeds/rss/",
+      "weight": 3
+    },
+    {
+      "name": "Japan Times",
+      "url": "https://www.japantimes.co.jp/feed/",
+      "weight": 3
+    },
+    {
+      "name": "CSM",
+      "url": "https://www.csmonitor.com/rss/world",
+      "weight": 4
+    },
+    {
+      "name": "PBS NewsHour",
+      "url": "https://www.pbs.org/newshour/feeds/rss/world",
+      "weight": 4
+    },
+    {
+      "name": "France 24",
+      "url": "https://www.france24.com/en/rss",
+      "weight": 4
+    },
+    {
+      "name": "DW",
+      "url": "https://rss.dw.com/xml/rss-en-world",
+      "weight": 4
     }
   ]
 }

backend/data/aisstream_spki_pins.json

@@ -0,0 +1,31 @@
+{
+  "_comment": [
+    "SPKI (Subject Public Key Info) pin list for stream.aisstream.io.",
+    "",
+    "Issue #258: AISStream's Let's Encrypt cert expired on 2026-05-20 due to an",
+    "upstream renewal-pipeline failure. Disabling TLS verification entirely",
+    "would let any network attacker MITM the AIS WebSocket and inject fake",
+    "ship positions onto the operator's map (same class as #199 GDELT MITM).",
+    "Instead we pin the leaf certificate's public-key SPKI hash: if normal",
+    "TLS validation fails specifically with CERT_HAS_EXPIRED, ais_proxy.js",
+    "re-checks the leaf cert's SPKI against this list. A match means the",
+    "key is still the genuine AISStream key (Let's Encrypt renewals keep the",
+    "same key unless rekey is requested), so we proceed in 'degraded TLS'",
+    "mode. A mismatch means a real MITM attempt and we refuse the connection.",
+    "",
+    "Format: each entry is a SHA-256 hash of the DER-encoded SPKI bytes,",
+    "encoded as standard base64 (matches the format produced by:",
+    "  openssl s_client -connect host:443 | \\",
+    "    openssl x509 -pubkey -noout | openssl pkey -pubin -outform DER | \\",
+    "    openssl dgst -sha256 -binary | openssl base64",
+    ").",
+    "",
+    "When AISStream rotates their server key (rare — Let's Encrypt renewals",
+    "default to keeping the same key), capture the new SPKI and add it to",
+    "this list BEFORE removing the old one. That way operators on the old",
+    "code still validate against the previous key during the transition."
+  ],
+  "stream.aisstream.io": [
+    "GJ10H0UPgLrO+2d3ZXROR/TXSVFXKUfRC3QEI2ibEg4="
+  ]
+}

backend/data/carrier_seed.json

@@ -0,0 +1,120 @@
+{
+  "_meta": {
+    "as_of": "2026-03-09",
+    "source": "USNI News Fleet & Marine Tracker",
+    "source_url": "https://news.usni.org/2026/03/09/usni-news-fleet-and-marine-tracker-march-9-2026",
+    "note": "One-shot bootstrap for first-run carrier positions. Once carrier_cache.json exists in the runtime data volume, this seed file is never read again. All subsequent updates come from GDELT (and any future sources) and are written to carrier_cache.json. A year from now, your runtime cache reflects whatever your install has observed since first launch — not these snapshot positions."
+  },
+  "carriers": {
+    "CVN-68": {
+      "lat": 47.5535,
+      "lng": -122.6400,
+      "heading": 90,
+      "desc": "Bremerton, WA (Maintenance)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-76": {
+      "lat": 47.5580,
+      "lng": -122.6360,
+      "heading": 90,
+      "desc": "Bremerton, WA (Decommissioning)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-69": {
+      "lat": 36.9465,
+      "lng": -76.3265,
+      "heading": 0,
+      "desc": "Norfolk, VA (Post-deployment maintenance)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-78": {
+      "lat": 18.0,
+      "lng": 39.5,
+      "heading": 0,
+      "desc": "Red Sea — Operation Epic Fury (USNI Mar 9)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-74": {
+      "lat": 36.98,
+      "lng": -76.43,
+      "heading": 0,
+      "desc": "Newport News, VA (RCOH refueling overhaul)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-75": {
+      "lat": 36.0,
+      "lng": 15.0,
+      "heading": 0,
+      "desc": "Mediterranean Sea deployment (USNI Mar 9)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-77": {
+      "lat": 36.5,
+      "lng": -74.0,
+      "heading": 0,
+      "desc": "Atlantic — Pre-deployment workups (USNI Mar 9)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-70": {
+      "lat": 32.6840,
+      "lng": -117.1290,
+      "heading": 180,
+      "desc": "San Diego, CA (Homeport)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-71": {
+      "lat": 32.6885,
+      "lng": -117.1280,
+      "heading": 180,
+      "desc": "San Diego, CA (Maintenance)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-72": {
+      "lat": 20.0,
+      "lng": 64.0,
+      "heading": 0,
+      "desc": "Arabian Sea — Operation Epic Fury (USNI Mar 9)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-73": {
+      "lat": 35.2830,
+      "lng": 139.6700,
+      "heading": 180,
+      "desc": "Yokosuka, Japan (Forward deployed)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    }
+  }
+}

backend/data/drishx/rf_model.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72b69418aa860a0d92ccae398a08722bc85e64a992b5515dd7bf9ae9f79f2fd1
+size 107194128

backend/data/plan_ccg_vessels.json

@@ -0,0 +1,646 @@
+{
+  "412000001": {
+    "hull_number": "101",
+    "name": "Nanchang",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000002": {
+    "hull_number": "102",
+    "name": "Lhasa",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000003": {
+    "hull_number": "103",
+    "name": "Anshan",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000004": {
+    "hull_number": "104",
+    "name": "Wuxi",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000005": {
+    "hull_number": "105",
+    "name": "Dalian",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000006": {
+    "hull_number": "106",
+    "name": "Yan'an",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000007": {
+    "hull_number": "107",
+    "name": "Zunyi",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000008": {
+    "hull_number": "108",
+    "name": "Xianyang",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000101": {
+    "hull_number": "117",
+    "name": "Xining",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000102": {
+    "hull_number": "118",
+    "name": "Urumqi",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000103": {
+    "hull_number": "119",
+    "name": "Guiyang",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000104": {
+    "hull_number": "120",
+    "name": "Chengdu",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000105": {
+    "hull_number": "131",
+    "name": "Taiyuan",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000106": {
+    "hull_number": "132",
+    "name": "Suzhou",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000107": {
+    "hull_number": "133",
+    "name": "Nantong",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000108": {
+    "hull_number": "134",
+    "name": "Suqian",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000109": {
+    "hull_number": "135",
+    "name": "Lianyungang",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000110": {
+    "hull_number": "136",
+    "name": "Xuchang",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000111": {
+    "hull_number": "155",
+    "name": "Nanjing",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000112": {
+    "hull_number": "156",
+    "name": "Zibo",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000113": {
+    "hull_number": "157",
+    "name": "Lishui",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000114": {
+    "hull_number": "161",
+    "name": "Hohhot",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000115": {
+    "hull_number": "162",
+    "name": "Yancheng",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000116": {
+    "hull_number": "163",
+    "name": "Kaifeng",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000117": {
+    "hull_number": "164",
+    "name": "Taizhou",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000201": {
+    "hull_number": "538",
+    "name": "Yantai",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000202": {
+    "hull_number": "539",
+    "name": "Wuhu",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000203": {
+    "hull_number": "540",
+    "name": "Huainan",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000204": {
+    "hull_number": "541",
+    "name": "Huaihua",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000205": {
+    "hull_number": "542",
+    "name": "Zaozhuang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000206": {
+    "hull_number": "529",
+    "name": "Zhoushan",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000207": {
+    "hull_number": "530",
+    "name": "Xuzhou",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000208": {
+    "hull_number": "531",
+    "name": "Xiangtan",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000209": {
+    "hull_number": "532",
+    "name": "Jingzhou",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000210": {
+    "hull_number": "536",
+    "name": "Xuchang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000211": {
+    "hull_number": "546",
+    "name": "Yancheng",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000212": {
+    "hull_number": "547",
+    "name": "Linyi",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000213": {
+    "hull_number": "548",
+    "name": "Yiyang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000214": {
+    "hull_number": "549",
+    "name": "Changzhou",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000215": {
+    "hull_number": "550",
+    "name": "Weifang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000301": {
+    "hull_number": "31",
+    "name": "Hainan",
+    "class": "Type 075",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_075_landing_helicopter_dock"
+  },
+  "412000302": {
+    "hull_number": "32",
+    "name": "Guangxi",
+    "class": "Type 075",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_075_landing_helicopter_dock"
+  },
+  "412000303": {
+    "hull_number": "33",
+    "name": "Anhui",
+    "class": "Type 075",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_075_landing_helicopter_dock"
+  },
+  "412000401": {
+    "hull_number": "16",
+    "name": "Liaoning",
+    "class": "Type 001",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Chinese_aircraft_carrier_Liaoning"
+  },
+  "412000402": {
+    "hull_number": "17",
+    "name": "Shandong",
+    "class": "Type 002",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Chinese_aircraft_carrier_Shandong"
+  },
+  "412000403": {
+    "hull_number": "18",
+    "name": "Fujian",
+    "class": "Type 003",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Chinese_aircraft_carrier_Fujian"
+  },
+  "412000501": {
+    "hull_number": "980",
+    "name": "Hulunhu",
+    "class": "Type 901",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_901_replenishment_ship"
+  },
+  "412000502": {
+    "hull_number": "981",
+    "name": "Chaganhu",
+    "class": "Type 901",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_901_replenishment_ship"
+  },
+  "412000601": {
+    "hull_number": "998",
+    "name": "Kunlun Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000602": {
+    "hull_number": "999",
+    "name": "Jinggang Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000603": {
+    "hull_number": "989",
+    "name": "Changbai Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000604": {
+    "hull_number": "988",
+    "name": "Yimeng Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000605": {
+    "hull_number": "987",
+    "name": "Wuzhi Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000606": {
+    "hull_number": "986",
+    "name": "Longhu Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000607": {
+    "hull_number": "985",
+    "name": "Dabie Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000608": {
+    "hull_number": "984",
+    "name": "Wuyi Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000701": {
+    "hull_number": "815A-1",
+    "name": "Dongdiao",
+    "class": "Type 815A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_815_electronic_reconnaissance_ship"
+  },
+  "412000702": {
+    "hull_number": "815A-2",
+    "name": "Haiwangxing",
+    "class": "Type 815A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_815_electronic_reconnaissance_ship"
+  },
+  "412000703": {
+    "hull_number": "815A-3",
+    "name": "Tianwangxing",
+    "class": "Type 815A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_815_electronic_reconnaissance_ship"
+  },
+  "412009001": {
+    "hull_number": "2901",
+    "name": "CCG 2901",
+    "class": "12000-ton Cutter",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009002": {
+    "hull_number": "3901",
+    "name": "CCG 3901",
+    "class": "12000-ton Cutter",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009003": {
+    "hull_number": "1305",
+    "name": "CCG 1305",
+    "class": "Type 818",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009004": {
+    "hull_number": "1306",
+    "name": "CCG 1306",
+    "class": "Type 818",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009005": {
+    "hull_number": "2502",
+    "name": "CCG 2502",
+    "class": "5000-ton Cutter",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009006": {
+    "hull_number": "2302",
+    "name": "CCG 2302",
+    "class": "3000-ton Cutter",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009007": {
+    "hull_number": "2303",
+    "name": "CCG 2303",
+    "class": "3000-ton Cutter",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009008": {
+    "hull_number": "1103",
+    "name": "CCG 1103",
+    "class": "Type 718B",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009009": {
+    "hull_number": "1105",
+    "name": "CCG 1105",
+    "class": "Type 718B",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009010": {
+    "hull_number": "1302",
+    "name": "CCG 1302",
+    "class": "Type 818",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412000801": {
+    "hull_number": "171",
+    "name": "Haikou",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000802": {
+    "hull_number": "170",
+    "name": "Lanzhou",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000803": {
+    "hull_number": "150",
+    "name": "Changchun",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000804": {
+    "hull_number": "151",
+    "name": "Zhengzhou",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000805": {
+    "hull_number": "152",
+    "name": "Jinan",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000806": {
+    "hull_number": "153",
+    "name": "Xi'an",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000901": {
+    "hull_number": "572",
+    "name": "Hengshui",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000902": {
+    "hull_number": "573",
+    "name": "Liuzhou",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000903": {
+    "hull_number": "574",
+    "name": "Sanya",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000904": {
+    "hull_number": "575",
+    "name": "Yueyang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000905": {
+    "hull_number": "576",
+    "name": "Daqing",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000906": {
+    "hull_number": "577",
+    "name": "Huanggang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412001001": {
+    "hull_number": "500",
+    "name": "Xianfeng",
+    "class": "Type 056A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_056_corvette"
+  },
+  "412001002": {
+    "hull_number": "501",
+    "name": "Xinyang",
+    "class": "Type 056A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_056_corvette"
+  },
+  "412001003": {
+    "hull_number": "502",
+    "name": "Huangshi",
+    "class": "Type 056",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_056_corvette"
+  },
+  "412001004": {
+    "hull_number": "509",
+    "name": "Huaian",
+    "class": "Type 056A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_056_corvette"
+  },
+  "412001005": {
+    "hull_number": "510",
+    "name": "Ningde",
+    "class": "Type 056A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_056_corvette"
+  },
+  "412001101": {
+    "hull_number": "795",
+    "name": "Nanchong",
+    "class": "Type 039A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_039A_submarine"
+  },
+  "412001201": {
+    "hull_number": "892",
+    "name": "Hualuoshan",
+    "class": "Type 903A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_903_replenishment_ship"
+  },
+  "412001202": {
+    "hull_number": "889",
+    "name": "Taihu",
+    "class": "Type 903",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_903_replenishment_ship"
+  },
+  "412001301": {
+    "hull_number": "636",
+    "name": "Nanning",
+    "class": "Type 052DL",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412001302": {
+    "hull_number": "165",
+    "name": "Zhanjiang",
+    "class": "Type 052DL",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412001303": {
+    "hull_number": "166",
+    "name": "Huainan",
+    "class": "Type 052DL",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  }
+}

backend/data/release_digests.json

@@ -0,0 +1,55 @@
+{
+  "_comment": [
+    "Baked-in SHA-256 digests for known Shadowbroker release archives.",
+    "",
+    "Issue #231: the self-updater previously skipped integrity verification",
+    "entirely whenever the MESH_UPDATE_SHA256 env var was unset (which is the",
+    "default — nothing in the install docs tells operators to set it). That",
+    "made the auto-update a supply-chain RCE on any compromise of the GitHub",
+    "release pipeline.",
+    "",
+    "The fix uses a multi-source verification chain mirroring the Tor bundle",
+    "digest approach in #201:",
+    "",
+    "  1. MESH_UPDATE_SHA256 env var (operator override, preserved)",
+    "  2. SHA256SUMS.txt asset published alongside each release (primary —",
+    "     the maintainer's release process already publishes this)",
+    "  3. This baked-in digest list (second line of defense for releases",
+    "     missing a SHA256SUMS asset, or when the asset can't be fetched)",
+    "  4. HTTPS-only fallback with a loud warning (preserves auto-update",
+    "     flow during transient outages so users don't get stuck)",
+    "",
+    "Mismatch from a source that DID respond is fatal — the update is",
+    "refused and the existing install keeps running. Only the 'no source",
+    "reachable at all' case falls back to HTTPS-only.",
+    "",
+    "Format: each entry is keyed by release tag and maps asset filenames",
+    "to their canonical SHA-256 digest (hex, lowercase). The updater",
+    "compares the locally-computed digest of the downloaded asset against",
+    "the value here.",
+    "",
+    "When the maintainer ships a new release, add its digests here BEFORE",
+    "removing the old ones so operators on the old code still validate",
+    "against the previous entries during the transition."
+  ],
+  "v0.9.79": {
+    "ShadowBroker_v0.9.79.zip": "f6877c1d66614525315ea82636ce9f7b41178332c4dbf90d27431a1ea1d9cd47",
+    "ShadowBroker_0.9.79_x64-setup.exe": "f7b676ada45cac7da05868b0a353678c9ee700e3abcf456a7c0c038c36da446f",
+    "ShadowBroker_0.9.79_x64_en-US.msi": "e0713c3cdda184cfbea750bfac0d62a35678fec00847e6476f2cac8e7e42046e"
+  },
+  "v0.9.8": {
+    "ShadowBroker_v0.9.8.zip": "183bb5cd62b9b9349d95df5ef7696cb6ca810ab4b991fa9dab6f898af4c7a175",
+    "ShadowBroker_0.9.8_x64-setup.exe": "94a0309862e9c81c92cdcbfea8eec9dbb97eef19ded82b26217b397defbc810c",
+    "ShadowBroker_0.9.8_x64_en-US.msi": "fe22f9d51e4360d74c18a7250c2fbb9ed4fa4c7a884b3ac0d04a21115466386b"
+  },
+  "v0.9.81": {
+    "ShadowBroker_v0.9.81.zip": "f81f454bdc88e9a32c351df38212b8cfa624704d65764b971bb091eef62259c6",
+    "ShadowBroker_0.9.81_x64-setup.exe": "25e9a95d0d8ce959a7d08fe8e7406772ae24b596652793e81d1de5d02510a5a6",
+    "ShadowBroker_0.9.81_x64_en-US.msi": "34e655fc0c0f195ee4ac978f228a4b2b9d5565253b8771aca9ef4693409e9e70"
+  },
+  "v0.9.82": {
+    "ShadowBroker_v0.9.82.zip": "202ab043465741dcc06de57c19ec8314904332f8e818b891d7174655719d084c",
+    "ShadowBroker_0.9.82_x64-setup.exe": "0eb9f2bda02ab691b39687641abc97e6bfb507b42f48de21970ad7dfb4ea15fc",
+    "ShadowBroker_0.9.82_x64_en-US.msi": "ced08f930171c0c08009a958cc30b0171a09f982230fc217c6808c2ed7ab2e30"
+  }
+}

backend/data/tor_bundle_digests.json

@@ -0,0 +1,16 @@
+{
+  "_comment": [
+    "Pinned SHA-256 digests for the Tor Expert Bundle archives we know how to install.",
+    "Used as the LAST-RESORT verification source when the upstream .sha256sum file is",
+    "unreachable, MITM'd, or doesn't match what we downloaded. Issue #201.",
+    "",
+    "Each entry is keyed by the archive URL (so multiple platforms / versions",
+    "can share this one file) and contains the canonical SHA-256 we trust.",
+    "",
+    "When the project tests a new Tor release, add its digest here in the same",
+    "PR that bumps _TOR_EXPERT_BUNDLE_URLS. Old entries are kept indefinitely so",
+    "users on older versions keep working — we only ever ADD here, never remove."
+  ],
+  "https://dist.torproject.org/torbrowser/15.0.11/tor-expert-bundle-windows-x86_64-15.0.11.tar.gz": "PLACEHOLDER_REPLACE_BEFORE_RELEASE",
+  "https://dist.torproject.org/torbrowser/15.0.8/tor-expert-bundle-windows-x86_64-15.0.8.tar.gz": "PLACEHOLDER_REPLACE_BEFORE_RELEASE"
+}

backend/data/yacht_alert_db.json

@@ -0,0 +1,122 @@
+{
+  "319225400": {
+    "name": "KORU",
+    "owner": "Jeff Bezos",
+    "builder": "Oceanco",
+    "length_m": 127,
+    "year": 2023,
+    "category": "Tech Billionaire",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Koru_(yacht)"
+  },
+  "538072122": {
+    "name": "LAUNCHPAD",
+    "owner": "Mark Zuckerberg",
+    "builder": "Feadship",
+    "length_m": 118,
+    "year": 2024,
+    "category": "Tech Billionaire",
+    "flag": "Marshall Islands",
+    "link": "https://www.superyachtfan.com/yacht/launchpad/"
+  },
+  "319032600": {
+    "name": "MUSASHI",
+    "owner": "Larry Ellison",
+    "builder": "Feadship",
+    "length_m": 88,
+    "year": 2011,
+    "category": "Tech Billionaire",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Musashi_(yacht)"
+  },
+  "319011000": {
+    "name": "RISING SUN",
+    "owner": "David Geffen",
+    "builder": "Lurssen",
+    "length_m": 138,
+    "year": 2004,
+    "category": "Celebrity / Mogul",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Rising_Sun_(yacht)"
+  },
+  "310593000": {
+    "name": "ECLIPSE",
+    "owner": "Roman Abramovich",
+    "builder": "Blohm+Voss",
+    "length_m": 162,
+    "year": 2010,
+    "category": "Oligarch Watch",
+    "flag": "Bermuda",
+    "link": "https://en.wikipedia.org/wiki/Eclipse_(yacht)"
+  },
+  "310792000": {
+    "name": "SOLARIS",
+    "owner": "Roman Abramovich",
+    "builder": "Lloyd Werft",
+    "length_m": 140,
+    "year": 2021,
+    "category": "Oligarch Watch",
+    "flag": "Bermuda",
+    "link": "https://en.wikipedia.org/wiki/Solaris_(yacht)"
+  },
+  "319094900": {
+    "name": "DILBAR",
+    "owner": "Alisher Usmanov (seized)",
+    "builder": "Lurssen",
+    "length_m": 156,
+    "year": 2016,
+    "category": "Oligarch Watch",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Dilbar_(yacht)"
+  },
+  "273610820": {
+    "name": "NORD",
+    "owner": "Alexei Mordashov",
+    "builder": "Lurssen",
+    "length_m": 142,
+    "year": 2021,
+    "category": "Oligarch Watch",
+    "flag": "Russia",
+    "link": "https://en.wikipedia.org/wiki/Nord_(yacht)"
+  },
+  "319179200": {
+    "name": "SCHEHERAZADE",
+    "owner": "Eduard Khudainatov (alleged Putin)",
+    "builder": "Lurssen",
+    "length_m": 140,
+    "year": 2020,
+    "category": "Oligarch Watch",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Scheherazade_(yacht)"
+  },
+  "319112900": {
+    "name": "AMADEA",
+    "owner": "Suleiman Kerimov (seized by US DOJ)",
+    "builder": "Lurssen",
+    "length_m": 106,
+    "year": 2017,
+    "category": "Oligarch Watch",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Amadea_(yacht)"
+  },
+  "319156800": {
+    "name": "BRAVO EUGENIA",
+    "owner": "Jerry Jones",
+    "builder": "Oceanco",
+    "length_m": 109,
+    "year": 2018,
+    "category": "Celebrity / Mogul",
+    "flag": "Cayman Islands",
+    "link": "https://www.superyachtfan.com/yacht/bravo-eugenia/"
+  },
+  "319137200": {
+    "name": "LADY S",
+    "owner": "Dan Snyder",
+    "builder": "Feadship",
+    "length_m": 93,
+    "year": 2019,
+    "category": "Celebrity / Mogul",
+    "flag": "Cayman Islands",
+    "link": "https://www.superyachtfan.com/yacht/lady-s/"
+  }
+}

backend/docker-entrypoint.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+set -eu
+
+# Docker named volumes hide files that were baked into /app/data at image build
+# time. Seed safe, static data into a fresh volume so first-run Docker installs
+# behave like source installs without bundling local runtime secrets.
+if [ -d /app/image-data ]; then
+  mkdir -p /app/data
+  find /app/image-data -mindepth 1 -maxdepth 1 -type f | while IFS= read -r src; do
+    dest="/app/data/$(basename "$src")"
+    if [ ! -e "$dest" ]; then
+      cp "$src" "$dest" || true
+    fi
+  done
+fi
+
+if [ -z "${PRIVACY_CORE_ALLOWED_SHA256:-}" ] && [ -f /app/libprivacy_core.so ]; then
+  PRIVACY_CORE_ALLOWED_SHA256="$(sha256sum /app/libprivacy_core.so | awk '{print $1}')"
+  export PRIVACY_CORE_ALLOWED_SHA256
+fi
+
+exec "$@"

backend/extract_ovens.py

@@ -1,25 +0,0 @@
-import re
-import json
-
-try:
-    with open('liveua_test.html', 'r', encoding='utf-8') as f:
-        html = f.read()
-        
-    m = re.search(r"var\s+ovens\s*=\s*(.*?);(?!function)", html, re.DOTALL)
-    if m:
-        json_str = m.group(1)
-        # Handle if it is a string containing base64
-        if json_str.startswith("'") or json_str.startswith('"'):
-            json_str = json_str.strip('"\'')
-            import base64
-            import urllib.parse
-            json_str = base64.b64decode(urllib.parse.unquote(json_str)).decode('utf-8')
-            
-        data = json.loads(json_str)
-        with open('out_liveua.json', 'w', encoding='utf-8') as f:
-            json.dump(data, f, indent=2)
-        print(f"Successfully extracted {len(data)} ovens items.")
-    else:
-        print("var ovens not found.")
-except Exception as e:
-    print("Error:", e)

backend/gate_sse.py

@@ -0,0 +1,11 @@
+"""gate_sse.py — DEPRECATED. Gate SSE broadcast removed in S3A.
+
+Gate activity is no longer broadcast via SSE. The frontend uses the
+authenticated poll loop for gate message refresh.
+
+Stubs are kept so any late imports do not crash at startup.
+"""
+
+
+def _broadcast_gate_events(gate_id: str, events: list[dict]) -> None:  # noqa: ARG001
+    """No-op — gate SSE broadcast removed."""

backend/limiter.py

@@ -0,0 +1,108 @@
+"""Rate-limit key function for slowapi.
+
+Issue #287 (tg12): the previous implementation used
+``slowapi.util.get_remote_address`` which only ever returns
+``request.client.host``. Behind the bundled Next.js proxy (or any other
+reverse proxy), every connected operator's ``client.host`` is the
+frontend container's bridge IP. ``@limiter.limit("120/minute")`` then
+collapses into one shared bucket for everybody on the same backend —
+one heavy tab can starve every other operator on the node.
+
+This module replaces that key function with one that:
+
+  * Reads ``X-Forwarded-For`` ONLY when the immediate peer is a trusted
+    frontend container (same allowlist used by the Docker bridge
+    local-operator trust path — see ``backend/auth.py`` ``#250``).
+  * Picks the FIRST entry in the XFF chain. That's the client end of
+    the proxy chain, which is the operator we want to bucket on.
+  * Falls back to ``request.client.host`` for any peer that isn't on
+    the trusted-frontend allowlist. Direct hits, unrelated containers,
+    and unknown hosts are bucketed exactly like before — there is no
+    way for an untrusted caller to spoof XFF and steal another
+    operator's rate-limit bucket.
+
+Single-operator nodes are unaffected: the frontend resolves to one IP,
+that IP is on the trust list, the XFF header is read, and you get one
+bucket per operator (i.e. you).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from slowapi import Limiter
+from slowapi.util import get_remote_address
+
+
+def _client_host(request: Any) -> str:
+    """Return the immediate peer's IP, normalised to a lowercase string."""
+    client = getattr(request, "client", None)
+    if client is None:
+        return ""
+    host = getattr(client, "host", "") or ""
+    return host.lower()
+
+
+def _first_forwarded_for(value: str) -> str:
+    """Return the first non-empty entry from an ``X-Forwarded-For`` header.
+
+    RFC 7239 / de-facto XFF format is ``client, proxy1, proxy2, …``. The
+    client end is what we want to bucket on. Empty parts (which appear
+    in some malformed headers) are skipped so we don't end up keying on
+    an empty string.
+    """
+    for raw in value.split(","):
+        candidate = raw.strip()
+        if candidate:
+            return candidate.lower()
+    return ""
+
+
+def _is_trusted_frontend_peer(host: str) -> bool:
+    """True iff ``host`` is one of the resolved trusted-frontend IPs.
+
+    Imported lazily so this module stays usable in unit tests that
+    don't want to pull the whole auth module into scope.
+    """
+    if not host:
+        return False
+    try:
+        from auth import _resolve_trusted_bridge_ips
+    except Exception:  # pragma: no cover - defensive
+        return False
+    try:
+        trusted_ips = _resolve_trusted_bridge_ips()
+    except Exception:  # pragma: no cover - defensive
+        return False
+    return host in trusted_ips
+
+
+def shadowbroker_rate_limit_key(request: Any) -> str:
+    """slowapi key_func that is proxy-aware on trusted frontend peers only.
+
+    Behaviour matrix:
+
+    * Direct loopback / unknown peer → ``request.client.host``
+      (identical to slowapi's default ``get_remote_address``).
+    * Peer is a trusted frontend container AND ``X-Forwarded-For`` is
+      present → first XFF entry (the actual operator).
+    * Peer is a trusted frontend container but no XFF → fall back to
+      ``request.client.host`` (the bridge IP). One shared bucket for
+      everyone in that case, same as before — but you only get there
+      if the trusted frontend forgot to forward XFF, which it won't.
+    """
+    peer = _client_host(request)
+    if _is_trusted_frontend_peer(peer):
+        headers = getattr(request, "headers", None)
+        if headers is not None:
+            xff = headers.get("x-forwarded-for") or headers.get("X-Forwarded-For")
+            if xff:
+                first = _first_forwarded_for(xff)
+                if first:
+                    return first
+    # Untrusted peer (or trusted peer without XFF): match the original
+    # get_remote_address behaviour byte-for-byte.
+    return get_remote_address(request)
+
+
+limiter = Limiter(key_func=shadowbroker_rate_limit_key)

backend/node_state.py

@@ -0,0 +1,313 @@
+"""node_state.py — Shared mutable node runtime state and node helper functions.
+
+Extracted from main.py so that background worker functions and route handlers
+can reference the same state objects without importing the full application.
+
+_NODE_SYNC_STATE is a reassignable value (SyncWorkerState is replaced whole,
+not mutated), so callers must use get_sync_state() / set_sync_state() instead
+of binding to the name at import time.
+
+All other _NODE_* objects are mutable containers (Lock, Event, dict) whose
+identity never changes; importing them directly by name is safe.
+"""
+
+import threading
+import time
+from typing import Any
+
+from services.mesh.mesh_infonet_sync_support import SyncWorkerState
+
+# ---------------------------------------------------------------------------
+# Runtime state objects
+# ---------------------------------------------------------------------------
+
+_NODE_RUNTIME_LOCK = threading.RLock()
+_NODE_SYNC_STOP = threading.Event()
+_NODE_SYNC_STATE = SyncWorkerState()
+_NODE_BOOTSTRAP_STATE: dict[str, Any] = {
+    "node_mode": "participant",
+    "manifest_loaded": False,
+    "manifest_signer_id": "",
+    "manifest_valid_until": 0,
+    "bootstrap_peer_count": 0,
+    "sync_peer_count": 0,
+    "push_peer_count": 0,
+    "operator_peer_count": 0,
+    "last_bootstrap_error": "",
+}
+_NODE_PUSH_STATE: dict[str, Any] = {
+    "last_event_id": "",
+    "last_push_ok_at": 0,
+    "last_push_error": "",
+    "last_results": [],
+}
+
+# ---------------------------------------------------------------------------
+# Getter / setter for _NODE_SYNC_STATE
+#
+# Use these instead of globals()["_NODE_SYNC_STATE"] = ... in any module that
+# imports this package.  The setter modifies *this* module's namespace so
+# subsequent get_sync_state() calls see the new value regardless of which
+# module calls set_sync_state().
+# ---------------------------------------------------------------------------
+
+def get_sync_state() -> SyncWorkerState:
+    return _NODE_SYNC_STATE
+
+
+def set_sync_state(state: SyncWorkerState) -> None:
+    global _NODE_SYNC_STATE
+    _NODE_SYNC_STATE = state
+
+
+# ---------------------------------------------------------------------------
+# Node helper functions
+#
+# These were in main.py but are needed by both route handlers and background
+# workers, so they live here to avoid circular imports.
+# ---------------------------------------------------------------------------
+
+def _current_node_mode() -> str:
+    from services.config import get_settings
+    mode = str(get_settings().MESH_NODE_MODE or "participant").strip().lower()
+    if mode not in {"participant", "relay", "perimeter"}:
+        return "participant"
+    return mode
+
+
+def _node_runtime_supported() -> bool:
+    return _current_node_mode() in {"participant", "relay"}
+
+
+def _node_activation_enabled() -> bool:
+    from services.node_settings import read_node_settings
+
+    try:
+        settings = read_node_settings()
+    except Exception:
+        return False
+    return bool(settings.get("enabled", False))
+
+
+def _participant_node_enabled() -> bool:
+    return _node_runtime_supported() and _node_activation_enabled()
+
+
+def _node_runtime_snapshot() -> dict[str, Any]:
+    with _NODE_RUNTIME_LOCK:
+        return {
+            "node_mode": _current_node_mode(),
+            "node_enabled": _participant_node_enabled(),
+            "private_transport_required": _infonet_private_transport_required(),
+            "bootstrap": {**dict(_NODE_BOOTSTRAP_STATE), "node_mode": _current_node_mode()},
+            "sync_runtime": get_sync_state().to_dict(),
+            "push_runtime": dict(_NODE_PUSH_STATE),
+        }
+
+
+def _set_node_sync_disabled_state(*, current_head: str = "") -> SyncWorkerState:
+    return SyncWorkerState(
+        current_head=str(current_head or ""),
+        last_outcome="disabled",
+    )
+
+
+def _set_participant_node_enabled(enabled: bool) -> dict[str, Any]:
+    from services.mesh.mesh_hashchain import infonet
+    from services.node_settings import write_node_settings
+
+    settings = write_node_settings(enabled=bool(enabled))
+    current_head = str(infonet.head_hash or "")
+    with _NODE_RUNTIME_LOCK:
+        _NODE_BOOTSTRAP_STATE["node_mode"] = _current_node_mode()
+        set_sync_state(
+            SyncWorkerState(current_head=current_head)
+            if bool(enabled) and _node_runtime_supported()
+            else _set_node_sync_disabled_state(current_head=current_head)
+        )
+    return {
+        **settings,
+        "node_mode": _current_node_mode(),
+        "node_enabled": _participant_node_enabled(),
+    }
+
+
+def _infonet_private_transport_required() -> bool:
+    from services.config import get_settings
+
+    return not bool(getattr(get_settings(), "MESH_INFONET_ALLOW_CLEARNET_SYNC", False))
+
+
+def _infonet_private_transport_error() -> str:
+    return "private Infonet requires onion/RNS transport; no clearnet sync fallback"
+
+
+def _is_private_infonet_transport(transport: str) -> bool:
+    return str(transport or "").strip().lower() in {"onion", "rns"}
+
+
+def _configured_bootstrap_seed_peer_urls() -> list[str]:
+    from services.config import get_settings
+    from services.mesh.mesh_router import parse_configured_relay_peers
+
+    settings = get_settings()
+    primary = str(getattr(settings, "MESH_BOOTSTRAP_SEED_PEERS", "") or "").strip()
+    legacy = str(getattr(settings, "MESH_DEFAULT_SYNC_PEERS", "") or "").strip()
+    return parse_configured_relay_peers(primary or legacy)
+
+
+def _refresh_node_peer_store(*, now: float | None = None) -> dict[str, Any]:
+    from services.config import get_settings
+    from services.mesh.mesh_bootstrap_manifest import load_bootstrap_manifest_from_settings
+    from services.mesh.mesh_peer_store import (
+        DEFAULT_PEER_STORE_PATH,
+        PeerStore,
+        make_bootstrap_peer_record,
+        make_push_peer_record,
+        make_sync_peer_record,
+    )
+    from services.mesh.mesh_router import (
+        configured_relay_peer_urls,
+        parse_configured_relay_peers,
+        peer_transport_kind,
+    )
+
+    timestamp = int(now if now is not None else time.time())
+    mode = _current_node_mode()
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        store = PeerStore(DEFAULT_PEER_STORE_PATH)
+
+    private_transport_required = _infonet_private_transport_required()
+    operator_peers = configured_relay_peer_urls()
+    bootstrap_seed_peers = _configured_bootstrap_seed_peer_urls()
+    skipped_clearnet_peers = 0
+    for peer_url in operator_peers:
+        transport = peer_transport_kind(peer_url)
+        if not transport:
+            continue
+        if private_transport_required and not _is_private_infonet_transport(transport):
+            skipped_clearnet_peers += 1
+            continue
+        store.upsert(
+            make_sync_peer_record(
+                peer_url=peer_url,
+                transport=transport,
+                role="relay",
+                source="operator",
+                now=timestamp,
+            )
+        )
+        store.upsert(
+            make_push_peer_record(
+                peer_url=peer_url,
+                transport=transport,
+                role="relay",
+                source="operator",
+                now=timestamp,
+            )
+        )
+
+    operator_peer_set = set(operator_peers)
+    for peer_url in bootstrap_seed_peers:
+        if peer_url in operator_peer_set:
+            continue
+        transport = peer_transport_kind(peer_url)
+        if not transport:
+            continue
+        if private_transport_required and not _is_private_infonet_transport(transport):
+            skipped_clearnet_peers += 1
+            continue
+        store.upsert(
+            make_bootstrap_peer_record(
+                peer_url=peer_url,
+                transport=transport,
+                role="seed",
+                label="ShadowBroker bootstrap seed",
+                signer_id="shadowbroker-bootstrap",
+                now=timestamp,
+            )
+        )
+        store.upsert(
+            make_sync_peer_record(
+                peer_url=peer_url,
+                transport=transport,
+                role="seed",
+                source="bundle",
+                label="ShadowBroker bootstrap seed",
+                signer_id="shadowbroker-bootstrap",
+                now=timestamp,
+            )
+        )
+
+    manifest = None
+    bootstrap_error = ""
+    try:
+        manifest = load_bootstrap_manifest_from_settings(now=timestamp)
+    except Exception as exc:
+        bootstrap_error = str(exc or "").strip()
+
+    if manifest is not None:
+        for peer in manifest.peers:
+            if private_transport_required and not _is_private_infonet_transport(peer.transport):
+                skipped_clearnet_peers += 1
+                continue
+            store.upsert(
+                make_bootstrap_peer_record(
+                    peer_url=peer.peer_url,
+                    transport=peer.transport,
+                    role=peer.role,
+                    label=peer.label,
+                    signer_id=manifest.signer_id,
+                    now=timestamp,
+                )
+            )
+            store.upsert(
+                make_sync_peer_record(
+                    peer_url=peer.peer_url,
+                    transport=peer.transport,
+                    role=peer.role,
+                    source="bootstrap_promoted",
+                    label=peer.label,
+                    signer_id=manifest.signer_id,
+                    now=timestamp,
+                )
+            )
+
+    if private_transport_required and skipped_clearnet_peers and not bootstrap_error:
+        bootstrap_error = _infonet_private_transport_error()
+
+    store.save()
+    bootstrap_records = store.records_for_bucket("bootstrap")
+    sync_records = store.records_for_bucket("sync")
+    push_records = store.records_for_bucket("push")
+    if private_transport_required:
+        bootstrap_records = [record for record in bootstrap_records if _is_private_infonet_transport(record.transport)]
+        sync_records = [record for record in sync_records if _is_private_infonet_transport(record.transport)]
+        push_records = [record for record in push_records if _is_private_infonet_transport(record.transport)]
+    snapshot = {
+        "node_mode": mode,
+        "private_transport_required": private_transport_required,
+        "skipped_clearnet_peer_count": skipped_clearnet_peers,
+        "manifest_loaded": manifest is not None,
+        "manifest_signer_id": manifest.signer_id if manifest is not None else "",
+        "manifest_valid_until": int(manifest.valid_until or 0) if manifest is not None else 0,
+        "bootstrap_peer_count": len(bootstrap_records),
+        "sync_peer_count": len(sync_records),
+        "push_peer_count": len(push_records),
+        "operator_peer_count": len(operator_peers),
+        "bootstrap_seed_peer_count": len(bootstrap_seed_peers),
+        "default_sync_peer_count": len(bootstrap_seed_peers),
+        "last_bootstrap_error": bootstrap_error,
+    }
+    with _NODE_RUNTIME_LOCK:
+        _NODE_BOOTSTRAP_STATE.update(snapshot)
+    return snapshot
+
+
+def _materialize_local_infonet_state() -> None:
+    from services.mesh.mesh_hashchain import infonet
+
+    infonet.ensure_materialized()

backend/out.json

@@ -1 +0,0 @@
-{"callsign": "JWZ7", "country": "N625GN", "lng": -111.914754, "lat": 33.620235, "alt": 0, "heading": 0, "type": "tracked_flight", "origin_loc": null, "dest_loc": null, "origin_name": "UNKNOWN", "dest_name": "UNKNOWN", "registration": "N625GN", "model": "GLF5", "icao24": "a82973", "speed_knots": 6.8, "squawk": "1200", "airline_code": "", "aircraft_category": "plane", "alert_operator": "Tilman Fertitta", "alert_category": "People", "alert_color": "pink", "trail": [[33.62024, -111.91475, 0, 1772302052]]}

backend/pyproject.toml

@@ -0,0 +1,64 @@
+[build-system]
+requires = ["setuptools>=68.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+py-modules = []
+
+[project]
+name = "backend"
+version = "0.9.82"
+requires-python = ">=3.10"
+dependencies = [
+    "apscheduler==3.10.3",
+    "beautifulsoup4>=4.9.0",
+    "cachetools==5.5.2",
+    "cryptography>=46.0.7",
+    "defusedxml>=0.7.1",
+    "fastapi==0.136.3",
+    "feedparser==6.0.10",
+    "httpx==0.28.1",
+    "playwright==1.59.0",
+    "playwright-stealth==1.0.6",
+    "pydantic==2.13.3",
+    "pydantic-settings==2.8.1",
+    "pystac-client==0.8.6",
+    "python-dotenv==1.2.2",
+    "requests==2.33.0",
+    "PySocks==1.7.1",
+    "reverse-geocoder==1.5.1",
+    "sgp4==2.25",
+    "meshtastic>=2.5.0",
+    "orjson>=3.10.0",
+    "paho-mqtt>=1.6.0,<2.0.0",
+    "PyNaCl>=1.5.0",
+    "slowapi==0.1.9",
+    "starlette==1.0.1",
+    "vaderSentiment>=3.3.0",
+    "uvicorn==0.34.0",
+    "yfinance==1.3.0",
+]
+
+[project.optional-dependencies]
+road-corridor = [
+    "geopandas>=1.0.0",
+    "imageio>=2.34.0",
+    "osmnx>=2.0.0",
+    "rasterio>=1.4.0",
+    "scikit-learn>=1.5.0",
+    "sentinelhub>=3.10.0",
+    "shapely>=2.0.0",
+]
+
+[dependency-groups]
+dev = ["pytest>=9.0.3", "pytest-asyncio>=1.4.0", "ruff>=0.9.0", "black>=24.0.0"]
+
+[tool.ruff.lint]
+# The current backend carries historical style debt in large legacy modules.
+# Keep CI focused on actionable correctness checks for the v0.9.82 release.
+ignore = ["E401", "E402", "E701", "E731", "E741", "F401", "F402", "F541", "F811", "F841"]
+
+[tool.black]
+# Avoid a release-time whole-backend formatting rewrite. Re-enable by narrowing
+# this once the legacy tree is formatted in a dedicated cleanup PR.
+force-exclude = ".*"

backend/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+testpaths = tests
+python_files = test_*.py
+python_functions = test_*
+asyncio_default_fixture_loop_scope = function

backend/requirements.txt

@@ -1,20 +0,0 @@
-fastapi>=0.103.1
-uvicorn>=0.23.2
-yfinance>=0.2.40
-feedparser==6.0.10
-legacy-cgi>=2.6
-requests==2.31.0
-apscheduler==3.10.3
-pydantic>=2.3.0
-pydantic-settings>=2.0.3
-playwright>=1.58.0
-beautifulsoup4>=4.12.0
-cachetools>=5.3
-cloudscraper>=1.2.71
-python-dotenv>=1.0
-lxml>=5.0
-reverse_geocoder>=1.5
-sgp4>=2.23
-geopy>=2.4.0
-pytz>=2023.3
-pystac-client>=0.7.0

backend/routers/admin.py

@@ -0,0 +1,435 @@
+import json as json_mod
+import logging
+import os
+import threading
+from pathlib import Path
+from typing import Any
+from fastapi import APIRouter, Request, Depends, Response
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+from node_state import (
+    _current_node_mode,
+    _participant_node_enabled,
+    _refresh_node_peer_store,
+    _set_participant_node_enabled,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+class NodeSettingsUpdate(BaseModel):
+    enabled: bool
+
+
+class TimeMachineToggle(BaseModel):
+    enabled: bool
+
+
+class MeshtasticMqttUpdate(BaseModel):
+    enabled: bool | None = None
+    broker: str | None = None
+    port: int | None = None
+    username: str | None = None
+    password: str | None = None
+    psk: str | None = None
+    include_default_roots: bool | None = None
+    extra_roots: str | None = None
+    extra_topics: str | None = None
+
+
+@router.get("/api/settings/api-keys", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_get_keys(request: Request):
+    from services.api_settings import get_api_keys
+    return get_api_keys()
+
+
+@router.put("/api/settings/api-keys", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_save_keys(request: Request):
+    from services.api_settings import save_api_keys
+    body = await request.json()
+    if not isinstance(body, dict):
+        return Response(
+            content=json_mod.dumps({"ok": False, "detail": "Expected a JSON object."}),
+            status_code=400,
+            media_type="application/json",
+        )
+    result = save_api_keys({str(k): str(v) for k, v in body.items()})
+    if result.get("ok"):
+        return result
+    return Response(
+        content=json_mod.dumps(result),
+        status_code=400,
+        media_type="application/json",
+    )
+
+
+@router.get("/api/settings/api-keys/meta")
+@limiter.limit("30/minute")
+async def api_get_keys_meta(request: Request):
+    """Return absolute paths for the backend .env and .env.example template.
+
+    Not gated behind admin auth: the paths are not sensitive, and the frontend
+    needs them to render the API Keys panel banner before the user has had a
+    chance to enter an admin key. Helps users find the file when in-app editing
+    is blocked or when the backend is read-only.
+    """
+    from services.api_settings import get_env_path_info
+    return get_env_path_info()
+
+
+@router.get(
+    "/api/settings/operator-handle",
+    dependencies=[Depends(require_local_operator)],
+)
+@limiter.limit("60/minute")
+async def api_get_operator_handle(request: Request):
+    """Round 7a: return the per-install operator handle so the frontend
+    can include it in browser-direct third-party API calls (Wikipedia /
+    Wikidata via lib/wikimediaClient). The handle is auto-generated on
+    first use; operators can override it via the OPERATOR_HANDLE setting
+    or the env var of the same name.
+
+    Gated on local-operator: legitimate browser usage goes through the
+    Next.js proxy which auto-attaches the admin key; remote scanners get
+    403. The handle itself isn't a secret (it's sent to every third-party
+    API the operator touches), but admin-gating it matches the rest of
+    the settings endpoints and follows least-privilege.
+    """
+    from services.network_utils import get_operator_handle
+    return {"handle": get_operator_handle()}
+
+
+@router.get(
+    "/api/settings/news-feeds",
+    dependencies=[Depends(require_local_operator)],
+)
+@limiter.limit("30/minute")
+async def api_get_news_feeds(request: Request):
+    """Issue #252 (tg12): the curated feed inventory is configuration
+    state, not a public data feed. Gated on local-operator so the
+    Tauri shell, the Docker bridge frontend, and any caller with an
+    admin key all see the full list; anonymous LAN/internet callers
+    can no longer enumerate operator source URLs.
+    """
+    from services.news_feed_config import get_feeds
+    return get_feeds()
+
+
+@router.put("/api/settings/news-feeds", dependencies=[Depends(require_admin)])
+@limiter.limit("10/minute")
+async def api_save_news_feeds(request: Request):
+    from services.news_feed_config import save_feeds
+    body = await request.json()
+    ok = save_feeds(body)
+    if ok:
+        return {"status": "updated", "count": len(body)}
+    return Response(
+        content=json_mod.dumps({"status": "error",
+            "message": "Validation failed (max 20 feeds, each needs name/url/weight 1-5)"}),
+        status_code=400,
+        media_type="application/json",
+    )
+
+
+@router.post("/api/settings/news-feeds/reset", dependencies=[Depends(require_admin)])
+@limiter.limit("10/minute")
+async def api_reset_news_feeds(request: Request):
+    from services.news_feed_config import get_feeds, reset_feeds
+    ok = reset_feeds()
+    if ok:
+        return {"status": "reset", "feeds": get_feeds()}
+    return {"status": "error", "message": "Failed to reset feeds"}
+
+
+@router.get("/api/settings/node")
+@limiter.limit("30/minute")
+async def api_get_node_settings(request: Request):
+    """Issue #243 (tg12): node_mode and node_enabled are operational
+    posture. Anonymous callers receive an empty stub; authenticated
+    callers (local-operator or admin/scoped token) see the full
+    state. See the canonical handler in backend/main.py for the full
+    rationale.
+    """
+    import asyncio
+    from auth import _scoped_view_authenticated
+    from services.node_settings import read_node_settings
+    data = await asyncio.to_thread(read_node_settings)
+    if not _scoped_view_authenticated(request, "node"):
+        return {}
+    return {
+        **data,
+        "node_mode": _current_node_mode(),
+        "node_enabled": _participant_node_enabled(),
+    }
+
+
+@router.put("/api/settings/node", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_set_node_settings(request: Request, body: NodeSettingsUpdate):
+    _refresh_node_peer_store()
+    if bool(body.enabled):
+        try:
+            from services.transport_lane_isolation import disable_public_mesh_lane
+
+            disable_public_mesh_lane(reason="private_node_enabled")
+        except Exception as exc:
+            logger.warning("Failed to disable public Mesh while enabling private node: %s", exc)
+    result = _set_participant_node_enabled(bool(body.enabled))
+    if bool(body.enabled):
+        try:
+            import main as _main
+
+            _main._kick_public_sync_background("operator_enable")
+        except Exception:
+            logger.debug("Unable to kick Infonet sync after node enable", exc_info=True)
+    return result
+
+
+def _meshtastic_runtime_snapshot() -> dict[str, Any]:
+    from services.meshtastic_mqtt_settings import redacted_meshtastic_mqtt_settings
+    from services.sigint_bridge import sigint_grid
+
+    return {
+        **redacted_meshtastic_mqtt_settings(),
+        "runtime": sigint_grid.mesh.status(),
+    }
+
+
+@router.get("/api/settings/meshtastic-mqtt", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_get_meshtastic_mqtt_settings(request: Request):
+    return _meshtastic_runtime_snapshot()
+
+
+@router.put("/api/settings/meshtastic-mqtt", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_set_meshtastic_mqtt_settings(request: Request, body: MeshtasticMqttUpdate):
+    from services.meshtastic_mqtt_settings import write_meshtastic_mqtt_settings
+    from services.sigint_bridge import sigint_grid
+
+    updates = body.model_dump(exclude_unset=True)
+    # Empty secret fields mean "keep existing"; explicit non-empty values replace.
+    if updates.get("password") == "":
+        updates.pop("password", None)
+    if updates.get("psk") == "":
+        updates.pop("psk", None)
+
+    enabled_requested = updates.get("enabled")
+    settings = write_meshtastic_mqtt_settings(**updates)
+    if isinstance(enabled_requested, bool):
+        logger.info("Meshtastic MQTT settings update: enabled=%s", enabled_requested)
+
+    if enabled_requested is True:
+        # Public MQTT and Wormhole are intentionally mutually exclusive lanes.
+        try:
+            from services.node_settings import write_node_settings
+            from services.wormhole_settings import write_wormhole_settings
+            from services.wormhole_supervisor import disconnect_wormhole
+
+            write_wormhole_settings(enabled=False)
+            disconnect_wormhole(reason="public_mesh_enabled")
+            write_node_settings(enabled=False)
+            _set_participant_node_enabled(False)
+        except Exception as exc:
+            logger.warning("Failed to disable private mesh lane while enabling public mesh: %s", exc)
+
+    if bool(settings.get("enabled")):
+        if sigint_grid.mesh.is_running():
+            sigint_grid.mesh.stop()
+            threading.Timer(1.0, sigint_grid.mesh.start).start()
+        else:
+            sigint_grid.mesh.start()
+    else:
+        sigint_grid.mesh.stop()
+
+    return _meshtastic_runtime_snapshot()
+
+
+@router.get(
+    "/api/settings/timemachine",
+    dependencies=[Depends(require_local_operator)],
+)
+@limiter.limit("30/minute")
+async def api_get_timemachine_settings(request: Request):
+    """Issue #253 (tg12): archival-capture posture is operationally
+    sensitive — it tells a remote caller whether this deployment is
+    retaining replayable historical surveillance data. Gated on
+    local-operator so the Tauri shell and Docker bridge frontend
+    still see the toggle state, but anonymous LAN/internet callers
+    can no longer fingerprint Time Machine state.
+    """
+    import asyncio
+    from services.node_settings import read_node_settings
+    data = await asyncio.to_thread(read_node_settings)
+    return {
+        "enabled": data.get("timemachine_enabled", False),
+        "storage_warning": "Time Machine auto-snapshots use ~68 MB/day compressed (~2 GB/month). "
+                           "Snapshots capture entity positions (flights, ships, satellites) for historical playback.",
+    }
+
+
+@router.put("/api/settings/timemachine", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_set_timemachine_settings(request: Request, body: TimeMachineToggle):
+    import asyncio
+    from services.node_settings import write_node_settings
+    result = await asyncio.to_thread(write_node_settings, timemachine_enabled=body.enabled)
+    return {
+        "ok": True,
+        "enabled": result.get("timemachine_enabled", False),
+    }
+
+
+@router.post("/api/system/update", dependencies=[Depends(require_admin)])
+@limiter.limit("1/minute")
+async def system_update(request: Request):
+    """Download latest release, backup current files, extract update, and restart."""
+    from services.updater import perform_update, schedule_restart
+    candidate = Path(__file__).resolve().parent.parent.parent
+    if (candidate / "frontend").is_dir() or (candidate / "backend").is_dir():
+        project_root = str(candidate)
+    else:
+        project_root = os.getcwd()
+    result = perform_update(project_root)
+    if result.get("status") == "error":
+        return Response(content=json_mod.dumps(result), status_code=500, media_type="application/json")
+    if result.get("status") == "docker":
+        return result
+    threading.Timer(2.0, schedule_restart, args=[project_root]).start()
+    return result
+
+
+# ── Tor Hidden Service ──────────────────────────────────────────────
+
+
+@router.get("/api/settings/tor", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_tor_status(request: Request):
+    """Return Tor hidden service status and .onion address if available."""
+    import asyncio
+    from services.tor_hidden_service import tor_service
+
+    return await asyncio.to_thread(tor_service.status)
+
+
+@router.post("/api/settings/tor/start", dependencies=[Depends(require_local_operator)])
+@limiter.limit("5/minute")
+async def api_tor_start(request: Request):
+    """Start Tor and provision a hidden service for this ShadowBroker instance.
+
+    Also enables MESH_ARTI so the mesh/wormhole system can route traffic
+    through the Tor SOCKS proxy (port 9050) automatically.
+    """
+    import asyncio
+    from services.tor_hidden_service import tor_service
+
+    result = await asyncio.to_thread(tor_service.start)
+
+    # If Tor started successfully, enable Arti (Tor SOCKS proxy for mesh)
+    if result.get("ok"):
+        try:
+            from routers.ai_intel import _write_env_value
+            from services.config import get_settings
+            _write_env_value("MESH_ARTI_ENABLED", "true")
+            get_settings.cache_clear()
+        except Exception:
+            pass  # Non-fatal — hidden service still works without mesh Arti
+
+    return result
+
+
+@router.post("/api/settings/tor/reset-identity", dependencies=[Depends(require_local_operator)])
+@limiter.limit("2/minute")
+async def api_tor_reset_identity(request: Request):
+    """Destroy current .onion identity and generate a fresh one on next start.
+
+    This is irreversible — the old .onion address is permanently lost.
+    """
+    import asyncio, shutil
+    from services.tor_hidden_service import tor_service, TOR_DIR
+
+    # Stop Tor if running
+    await asyncio.to_thread(tor_service.stop)
+
+    # Delete the hidden service directory (contains the private key)
+    hs_dir = TOR_DIR / "hidden_service"
+    if hs_dir.exists():
+        shutil.rmtree(str(hs_dir), ignore_errors=True)
+
+    # Clear cached address
+    tor_service._onion_address = ""
+
+    return {"ok": True, "detail": "Tor identity destroyed. A new .onion will be generated on next start."}
+
+
+@router.post("/api/settings/agent/reset-all", dependencies=[Depends(require_local_operator)])
+@limiter.limit("2/minute")
+async def api_reset_all_agent_credentials(request: Request):
+    """Nuclear reset: regenerate HMAC key, destroy .onion, revoke agent identity.
+
+    After this, the agent is fully disconnected and needs new credentials.
+    """
+    import asyncio, secrets, shutil
+    from services.tor_hidden_service import tor_service, TOR_DIR
+    from services.config import get_settings
+
+    results = {}
+
+    # 1. Regenerate HMAC key
+    new_secret = secrets.token_hex(24)
+    from routers.ai_intel import _write_env_value
+    _write_env_value("OPENCLAW_HMAC_SECRET", new_secret)
+    results["hmac"] = "regenerated"
+
+    # 2. Revoke agent identity (Ed25519 keypair)
+    try:
+        from services.openclaw_bridge import revoke_agent_identity
+        revoke_agent_identity()
+        results["identity"] = "revoked"
+    except Exception as e:
+        results["identity"] = f"error: {e}"
+
+    # 3. Destroy .onion and restart Tor with new identity
+    await asyncio.to_thread(tor_service.stop)
+    hs_dir = TOR_DIR / "hidden_service"
+    if hs_dir.exists():
+        shutil.rmtree(str(hs_dir), ignore_errors=True)
+    tor_service._onion_address = ""
+    results["tor"] = "identity destroyed"
+
+    # 4. Bootstrap fresh identity + start Tor with new .onion
+    try:
+        from services.openclaw_bridge import generate_agent_keypair
+        keypair = generate_agent_keypair(force=True)
+        results["new_node_id"] = keypair.get("node_id", "")
+    except Exception as e:
+        results["new_node_id"] = f"error: {e}"
+
+    tor_result = await asyncio.to_thread(tor_service.start)
+    results["new_onion"] = tor_result.get("onion_address", "")
+    results["tor_ok"] = tor_result.get("ok", False)
+
+    # Clear settings cache
+    get_settings.cache_clear()
+
+    return {
+        "ok": True,
+        "hmac_regenerated": True,
+        "detail": "All agent credentials have been reset. Use the agent connection screen to generate or reveal replacement credentials.",
+        **results,
+    }
+
+
+@router.post("/api/settings/tor/stop", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_tor_stop(request: Request):
+    """Stop the Tor hidden service."""
+    import asyncio
+    from services.tor_hidden_service import tor_service
+
+    return await asyncio.to_thread(tor_service.stop)

backend/routers/cctv.py

@@ -0,0 +1,401 @@
+import logging
+from dataclasses import dataclass, field
+from fastapi import APIRouter, Request, Query, HTTPException
+from fastapi.responses import StreamingResponse
+from starlette.background import BackgroundTask
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+_CCTV_PROXY_CONNECT_TIMEOUT_S = 2.0
+
+_CCTV_PROXY_ALLOWED_HOSTS = {
+    "s3-eu-west-1.amazonaws.com",
+    "jamcams.tfl.gov.uk",
+    "images.data.gov.sg",
+    "cctv.austinmobility.io",
+    "webcams.nyctmc.org",
+    "cwwp2.dot.ca.gov",
+    "wzmedia.dot.ca.gov",
+    "images.wsdot.wa.gov",
+    "olypen.com",
+    "flyykm.com",
+    "cam.pangbornairport.com",
+    "navigator-c2c.dot.ga.gov",
+    "navigator-c2c.ga.gov",
+    "navigator-csc.dot.ga.gov",
+    "vss1live.dot.ga.gov",
+    "vss2live.dot.ga.gov",
+    "vss3live.dot.ga.gov",
+    "vss4live.dot.ga.gov",
+    "vss5live.dot.ga.gov",
+    "511ga.org",
+    "gettingaroundillinois.com",
+    "cctv.travelmidwest.com",
+    "mdotjboss.state.mi.us",
+    "micamerasimages.net",
+    "publicstreamer1.cotrip.org",
+    "publicstreamer2.cotrip.org",
+    "publicstreamer3.cotrip.org",
+    "publicstreamer4.cotrip.org",
+    "cocam.carsprogram.org",
+    "tripcheck.com",
+    "www.tripcheck.com",
+    "infocar.dgt.es",
+    "informo.madrid.es",
+    "webcams2.asfinag.at",
+    "odo.asfinag.at",
+    "www.windy.com",
+    "imgproxy.windy.com",
+    "www.lakecountypassage.com",
+    "webcam.forkswa.com",
+    "webcam.sunmountainlodge.com",
+    "www.nps.gov",
+    "home.lewiscounty.com",
+    "www.seattle.gov",
+    "511on.ca",
+    "511.alberta.ca",
+    "fl511.com",
+    "www.fl511.com",
+    "webcams.transport.nsw.gov.au",
+    "www.livetraffic.com",
+    "livetraffic.com",
+    "opendata.ndw.nu",
+}
+
+
+@dataclass(frozen=True)
+class _CCTVProxyProfile:
+    name: str
+    timeout: tuple = (_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0)
+    cache_seconds: int = 30
+    headers: dict = field(default_factory=dict)
+
+
+def _cctv_host_allowed(hostname) -> bool:
+    host = str(hostname or "").strip().lower()
+    if not host:
+        return False
+    for allowed in _CCTV_PROXY_ALLOWED_HOSTS:
+        normalized = str(allowed or "").strip().lower()
+        if host == normalized or host.endswith(f".{normalized}"):
+            return True
+    return False
+
+
+def _proxied_cctv_url(target_url: str) -> str:
+    from urllib.parse import quote
+    return f"/api/cctv/media?url={quote(target_url, safe='')}"
+
+
+def _cctv_proxy_profile_for_url(target_url: str) -> _CCTVProxyProfile:
+    from urllib.parse import urlparse
+    parsed = urlparse(target_url)
+    host = str(parsed.hostname or "").strip().lower()
+    path = str(parsed.path or "").strip().lower()
+
+    if host in {"jamcams.tfl.gov.uk", "s3-eu-west-1.amazonaws.com"}:
+        return _CCTVProxyProfile(name="tfl-jamcam", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 20.0), cache_seconds=15,
+            headers={"Accept": "video/mp4,image/avif,image/webp,image/apng,image/*,*/*;q=0.8", "Referer": "https://tfl.gov.uk/"})
+    if host == "images.data.gov.sg":
+        return _CCTVProxyProfile(name="lta-singapore", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host == "cctv.austinmobility.io":
+        return _CCTVProxyProfile(name="austin-mobility", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://data.mobility.austin.gov/", "Origin": "https://data.mobility.austin.gov"})
+    if host == "webcams.nyctmc.org":
+        return _CCTVProxyProfile(name="nyc-dot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host in {"cwwp2.dot.ca.gov", "wzmedia.dot.ca.gov"}:
+        return _CCTVProxyProfile(name="caltrans", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=15,
+            headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,image/*,*/*;q=0.8",
+                     "Referer": "https://cwwp2.dot.ca.gov/"})
+    if host in {"images.wsdot.wa.gov", "olypen.com", "flyykm.com", "cam.pangbornairport.com"}:
+        return _CCTVProxyProfile(name="wsdot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host in {"www.lakecountypassage.com", "webcam.forkswa.com", "webcam.sunmountainlodge.com", "home.lewiscounty.com", "www.seattle.gov"}:
+        return _CCTVProxyProfile(name="regional-cctv-image", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=45,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": f"https://{host}/"})
+    if host == "www.nps.gov":
+        return _CCTVProxyProfile(name="nps-webcam", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.nps.gov/"})
+    if host in {"navigator-c2c.dot.ga.gov", "navigator-c2c.ga.gov", "navigator-csc.dot.ga.gov"}:
+        read_timeout = 18.0 if "/snapshots/" in path else 12.0
+        return _CCTVProxyProfile(name="gdot-snapshot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, read_timeout), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://navigator-c2c.dot.ga.gov/"})
+    if host == "511ga.org":
+        return _CCTVProxyProfile(name="gdot-511ga-image", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://511ga.org/cctv"})
+    if host.startswith("vss") and host.endswith("dot.ga.gov"):
+        return _CCTVProxyProfile(name="gdot-hls", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 20.0), cache_seconds=10,
+            headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,*/*;q=0.8",
+                     "Referer": "https://navigator-c2c.dot.ga.gov/"})
+    if host in {"gettingaroundillinois.com", "cctv.travelmidwest.com"}:
+        return _CCTVProxyProfile(name="illinois-dot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host in {"mdotjboss.state.mi.us", "micamerasimages.net"}:
+        return _CCTVProxyProfile(name="michigan-dot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://mdotjboss.state.mi.us/"})
+    if host in {"publicstreamer1.cotrip.org", "publicstreamer2.cotrip.org",
+                "publicstreamer3.cotrip.org", "publicstreamer4.cotrip.org"}:
+        return _CCTVProxyProfile(name="cotrip-hls", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 20.0), cache_seconds=10,
+            headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,*/*;q=0.8",
+                     "Referer": "https://www.cotrip.org/"})
+    if host == "cocam.carsprogram.org":
+        return _CCTVProxyProfile(name="cotrip-preview", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=20,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.cotrip.org/"})
+    if host in {"tripcheck.com", "www.tripcheck.com"}:
+        return _CCTVProxyProfile(name="odot-tripcheck", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host == "infocar.dgt.es":
+        return _CCTVProxyProfile(name="dgt-spain", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://infocar.dgt.es/"})
+    if host == "informo.madrid.es":
+        return _CCTVProxyProfile(name="madrid-city", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://informo.madrid.es/"})
+    if host in {"webcams2.asfinag.at", "odo.asfinag.at"}:
+        return _CCTVProxyProfile(name="asfinag-austria", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.asfinag.at/"})
+    if host in {"www.windy.com", "imgproxy.windy.com"}:
+        return _CCTVProxyProfile(name="windy-webcams", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.windy.com/"})
+    if host == "511on.ca":
+        return _CCTVProxyProfile(name="ontario-511", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://511on.ca/"})
+    if host == "511.alberta.ca":
+        return _CCTVProxyProfile(name="alberta-511", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://511.alberta.ca/"})
+    if host in {"fl511.com", "www.fl511.com"}:
+        return _CCTVProxyProfile(name="florida-511", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://fl511.com/"})
+    if host == "webcams.transport.nsw.gov.au":
+        return _CCTVProxyProfile(name="nsw-live-traffic", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.livetraffic.com/"})
+    if host in {"opendata.ndw.nu", "www.ndw.nu"}:
+        return _CCTVProxyProfile(name="ndw-netherlands", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=120,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.ndw.nu/"})
+    return _CCTVProxyProfile(name="generic-cctv", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0), cache_seconds=30,
+        headers={"Accept": "*/*"})
+
+
+def _cctv_upstream_headers(request: Request, profile: _CCTVProxyProfile) -> dict:
+    # Round 7a: per-install operator handle. Mozilla/5.0 prefix retained
+    # because many CCTV endpoints sniff for a browser-like prefix.
+    from services.network_utils import outbound_user_agent
+    headers = {
+        "User-Agent": f"Mozilla/5.0 (compatible; {outbound_user_agent('cctv-proxy')})",
+        **profile.headers,
+    }
+    range_header = request.headers.get("range")
+    if range_header:
+        headers["Range"] = range_header
+    if_none_match = request.headers.get("if-none-match")
+    if if_none_match:
+        headers["If-None-Match"] = if_none_match
+    if_modified_since = request.headers.get("if-modified-since")
+    if if_modified_since:
+        headers["If-Modified-Since"] = if_modified_since
+    return headers
+
+
+def _cctv_response_headers(resp, cache_seconds: int, include_length: bool = True) -> dict:
+    headers = {"Cache-Control": f"public, max-age={cache_seconds}", "Access-Control-Allow-Origin": "*"}
+    for key in ("Accept-Ranges", "Content-Range", "ETag", "Last-Modified"):
+        value = resp.headers.get(key)
+        if value:
+            headers[key] = value
+    if include_length:
+        content_length = resp.headers.get("Content-Length")
+        if content_length:
+            headers["Content-Length"] = content_length
+    return headers
+
+
+# Maximum number of redirects we'll follow on the CCTV upstream. Each hop is
+# re-validated against _cctv_host_allowed() before continuing, so this caps
+# the redirect-chain SSRF blast radius.
+_CCTV_MAX_REDIRECTS = 5
+
+
+def _fetch_cctv_upstream_response(request: Request, target_url: str, profile: _CCTVProxyProfile):
+    """Fetch an upstream CCTV URL, following redirects manually with host re-validation.
+
+    Why manual redirect following:
+      The original code used ``allow_redirects=True``, which only validated
+      the initial caller-supplied URL host against the allowlist. An attacker
+      could submit an allowed host that 302-redirected to an internal address
+      (e.g. ``http://localhost:8000/api/...`` or a private RFC1918 range),
+      and the backend would dutifully follow and proxy the response — a
+      classic open-redirect-to-SSRF chain.
+
+      With this loop, we re-run ``_cctv_host_allowed()`` on every hop's
+      ``Location`` header. A redirect to a host that isn't on the allowlist
+      is rejected with 502 rather than silently followed.
+    """
+    import requests as _req
+    from urllib.parse import urlparse, urljoin
+
+    headers = _cctv_upstream_headers(request, profile)
+    current_url = target_url
+    hops = 0
+    try:
+        while True:
+            resp = _req.get(
+                current_url,
+                timeout=profile.timeout,
+                stream=True,
+                allow_redirects=False,
+                headers=headers,
+            )
+            # Redirect handling — re-validate the next-hop host before following.
+            if resp.is_redirect or resp.status_code in (301, 302, 303, 307, 308):
+                location = resp.headers.get("Location", "")
+                resp.close()
+                if hops >= _CCTV_MAX_REDIRECTS:
+                    logger.warning(
+                        "CCTV upstream redirect chain exceeded limit [%s] %s",
+                        profile.name, target_url,
+                    )
+                    raise HTTPException(status_code=502, detail="Upstream redirect chain too long")
+                if not location:
+                    raise HTTPException(status_code=502, detail="Upstream redirect missing Location")
+                next_url = urljoin(current_url, location)
+                next_parsed = urlparse(next_url)
+                if next_parsed.scheme not in ("http", "https"):
+                    raise HTTPException(status_code=502, detail="Upstream redirect to non-HTTP scheme")
+                if not _cctv_host_allowed(next_parsed.hostname):
+                    logger.warning(
+                        "CCTV upstream redirect to disallowed host [%s] %s -> %s",
+                        profile.name, current_url, next_url,
+                    )
+                    raise HTTPException(status_code=502, detail="Upstream redirect to disallowed host")
+                current_url = next_url
+                hops += 1
+                continue
+            break
+    except _req.exceptions.Timeout as exc:
+        logger.warning("CCTV upstream timeout [%s] %s", profile.name, target_url)
+        raise HTTPException(status_code=504, detail="Upstream timeout") from exc
+    except _req.exceptions.RequestException as exc:
+        logger.warning("CCTV upstream request failure [%s] %s: %s", profile.name, target_url, exc)
+        raise HTTPException(status_code=502, detail="Upstream fetch failed") from exc
+    if resp.status_code >= 400:
+        logger.info("CCTV upstream HTTP %s [%s] %s", resp.status_code, profile.name, target_url)
+        resp.close()
+        raise HTTPException(status_code=int(resp.status_code), detail=f"Upstream returned {resp.status_code}")
+    return resp
+
+
+def _rewrite_cctv_hls_playlist(base_url: str, body: str) -> str:
+    import re
+    from urllib.parse import urljoin, urlparse
+
+    def _rewrite_target(target: str) -> str:
+        candidate = str(target or "").strip()
+        if not candidate or candidate.startswith("data:"):
+            return candidate
+        absolute = urljoin(base_url, candidate)
+        parsed_target = urlparse(absolute)
+        if parsed_target.scheme not in ("http", "https"):
+            return candidate
+        if not _cctv_host_allowed(parsed_target.hostname):
+            return candidate
+        return _proxied_cctv_url(absolute)
+
+    rewritten_lines: list = []
+    for raw_line in body.splitlines():
+        stripped = raw_line.strip()
+        if not stripped:
+            rewritten_lines.append(raw_line)
+            continue
+        if stripped.startswith("#"):
+            rewritten_lines.append(re.sub(r'URI="([^"]+)"',
+                lambda match: f'URI="{_rewrite_target(match.group(1))}"', raw_line))
+            continue
+        rewritten_lines.append(_rewrite_target(stripped))
+    return "\n".join(rewritten_lines) + ("\n" if body.endswith("\n") else "")
+
+
+def _infer_cctv_media_type_from_url(target_url: str, content_type: str) -> str:
+    from urllib.parse import urlparse
+
+    clean_type = str(content_type or "").split(";", 1)[0].strip().lower()
+    if clean_type and clean_type not in {"application/octet-stream", "binary/octet-stream"}:
+        return content_type
+    path = str(urlparse(target_url).path or "").lower()
+    if path.endswith((".jpg", ".jpeg")):
+        return "image/jpeg"
+    if path.endswith(".png"):
+        return "image/png"
+    if path.endswith(".webp"):
+        return "image/webp"
+    if path.endswith(".gif"):
+        return "image/gif"
+    if path.endswith(".mp4"):
+        return "video/mp4"
+    if path.endswith((".m3u8", ".m3u")):
+        return "application/vnd.apple.mpegurl"
+    if path.endswith((".mjpg", ".mjpeg")):
+        return "multipart/x-mixed-replace"
+    return content_type or "application/octet-stream"
+
+
+def _proxy_cctv_media_response(request: Request, target_url: str):
+    from urllib.parse import urlparse
+    from fastapi.responses import Response
+    parsed = urlparse(target_url)
+    profile = _cctv_proxy_profile_for_url(target_url)
+    resp = _fetch_cctv_upstream_response(request, target_url, profile)
+    content_type = _infer_cctv_media_type_from_url(
+        target_url,
+        resp.headers.get("Content-Type", "application/octet-stream"),
+    )
+    is_hls_playlist = (
+        ".m3u8" in str(parsed.path or "").lower()
+        or "mpegurl" in content_type.lower()
+        or "vnd.apple.mpegurl" in content_type.lower()
+    )
+    if is_hls_playlist:
+        body = resp.text
+        if "#EXTM3U" in body:
+            body = _rewrite_cctv_hls_playlist(target_url, body)
+        resp.close()
+        return Response(content=body, media_type=content_type,
+            headers=_cctv_response_headers(resp, cache_seconds=profile.cache_seconds, include_length=False))
+    return StreamingResponse(resp.iter_content(chunk_size=65536), status_code=resp.status_code,
+        media_type=content_type,
+        headers=_cctv_response_headers(resp, cache_seconds=profile.cache_seconds),
+        background=BackgroundTask(resp.close))
+
+
+@router.get("/api/cctv/media")
+@limiter.limit("120/minute")
+async def cctv_media_proxy(request: Request, url: str = Query(...)):
+    """Proxy CCTV media through the backend to bypass browser CORS restrictions."""
+    from urllib.parse import urlparse
+    parsed = urlparse(url)
+    if not _cctv_host_allowed(parsed.hostname):
+        raise HTTPException(status_code=403, detail="Host not allowed")
+    if parsed.scheme not in ("http", "https"):
+        raise HTTPException(status_code=400, detail="Invalid scheme")
+    return _proxy_cctv_media_response(request, url)

backend/routers/data.py

@@ -0,0 +1,905 @@
+import asyncio
+import logging
+import math
+import os
+import threading
+from typing import Any
+from fastapi import APIRouter, Request, Response, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+from services.data_fetcher import update_all_data
+import orjson
+import json as json_mod
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+_refresh_lock = threading.Lock()
+
+
+class ViewportUpdate(BaseModel):
+    s: float
+    w: float
+    n: float
+    e: float
+
+
+class LayerUpdate(BaseModel):
+    layers: dict[str, bool]
+
+
+class LiveUamapOptInUpdate(BaseModel):
+    opted_in: bool
+
+
+class PredictionMarketsOptInUpdate(BaseModel):
+    opted_in: bool
+
+
+_LAST_VIEWPORT_UPDATE: tuple | None = None
+_LAST_VIEWPORT_UPDATE_TS = 0.0
+_VIEWPORT_UPDATE_LOCK = threading.Lock()
+_VIEWPORT_DEDUPE_EPSILON = 1.0
+_VIEWPORT_MIN_UPDATE_S = 10.0
+
+
+def _normalize_longitude(value: float) -> float:
+    normalized = ((value + 180.0) % 360.0 + 360.0) % 360.0 - 180.0
+    if normalized == -180.0 and value > 0:
+        return 180.0
+    return normalized
+
+
+def _normalize_viewport_bounds(s: float, w: float, n: float, e: float) -> tuple:
+    south = max(-90.0, min(90.0, s))
+    north = max(-90.0, min(90.0, n))
+    raw_width = abs(e - w)
+    if not math.isfinite(raw_width) or raw_width >= 360.0:
+        return south, -180.0, north, 180.0
+    west = _normalize_longitude(w)
+    east = _normalize_longitude(e)
+    if east < west:
+        return south, -180.0, north, 180.0
+    return south, west, north, east
+
+
+def _viewport_changed_enough(bounds: tuple) -> bool:
+    global _LAST_VIEWPORT_UPDATE, _LAST_VIEWPORT_UPDATE_TS
+    import time
+    now = time.monotonic()
+    with _VIEWPORT_UPDATE_LOCK:
+        if _LAST_VIEWPORT_UPDATE is None:
+            _LAST_VIEWPORT_UPDATE = bounds
+            _LAST_VIEWPORT_UPDATE_TS = now
+            return True
+        changed = any(
+            abs(current - previous) > _VIEWPORT_DEDUPE_EPSILON
+            for current, previous in zip(bounds, _LAST_VIEWPORT_UPDATE)
+        )
+        if not changed and (now - _LAST_VIEWPORT_UPDATE_TS) < _VIEWPORT_MIN_UPDATE_S:
+            return False
+        if (now - _LAST_VIEWPORT_UPDATE_TS) < _VIEWPORT_MIN_UPDATE_S:
+            return False
+        _LAST_VIEWPORT_UPDATE = bounds
+        _LAST_VIEWPORT_UPDATE_TS = now
+        return True
+
+
+def _queue_viirs_change_refresh() -> None:
+    from services.fetchers.earth_observation import fetch_viirs_change_nodes
+    threading.Thread(target=fetch_viirs_change_nodes, daemon=True).start()
+
+
+def _etag_response(request: Request, payload: dict, prefix: str = "", default=None):
+    etag = _current_etag(prefix)
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    content = json_mod.dumps(_json_safe(payload), default=default, allow_nan=False)
+    return Response(content=content, media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"})
+
+
+def _current_etag(prefix: str = "") -> str:
+    from services.fetchers._store import get_active_layers_version, get_data_version
+    return f"{prefix}v{get_data_version()}-l{get_active_layers_version()}"
+
+
+# ── Issue #288: viewport-aware payloads ─────────────────────────────────────
+# Heavy, density-driven, time-sensitive layers that benefit from bbox
+# filtering. Light reference layers (datacenters, military_bases,
+# power_plants, satellites, weather, news, etc.) are intentionally NOT
+# in these sets — they ship world-scale even when bounds are supplied so
+# panning never reveals an "empty world" of static infrastructure.
+#
+# When the caller does NOT pass s/w/n/e, none of this runs and the response
+# is byte-for-byte identical to the pre-#288 behavior.
+_FAST_BBOX_HEAVY_KEYS: tuple[str, ...] = (
+    "commercial_flights",
+    "military_flights",
+    "private_flights",
+    "private_jets",
+    "tracked_flights",
+    "ships",
+    "cctv",
+    "uavs",
+    "liveuamap",
+    "gps_jamming",
+    "sigint",
+    "trains",
+)
+_SLOW_BBOX_HEAVY_KEYS: tuple[str, ...] = (
+    "gdelt",
+    "firms_fires",
+    "kiwisdr",
+    "scanners",
+    "psk_reporter",
+)
+
+
+def _has_full_bbox(s, w, n, e) -> bool:
+    return None not in (s, w, n, e)
+
+
+def _bbox_etag_suffix(s, w, n, e) -> str:
+    """Quantize bbox to 1° before mixing into the ETag.
+
+    The 20% padding inside _bbox_filter already absorbs sub-degree pans;
+    quantizing here means small mouse drags don't blow the ETag cache
+    on the client. Full-world bounds collapse to a single suffix.
+    """
+    if not _has_full_bbox(s, w, n, e):
+        return ""
+    try:
+        ss = math.floor(float(s))
+        ww = math.floor(float(w))
+        nn = math.ceil(float(n))
+        ee = math.ceil(float(e))
+    except (TypeError, ValueError):
+        return ""
+    # If the requested window covers basically the whole world, treat it as
+    # "no bbox" for caching purposes so world-zoomed clients all hit the
+    # same ETag and benefit from the existing 304 path.
+    lat_span, lng_span = _bbox_spans(s, w, n, e)
+    if lng_span >= 300 or lat_span >= 120:
+        return ""
+    return f"|bbox={ss},{ww},{nn},{ee}"
+
+
+def _apply_bbox_to_payload(payload: dict, heavy_keys: tuple[str, ...],
+                            s: float, w: float, n: float, e: float) -> dict:
+    """In-place filter the heavy-key collections in *payload* to a viewport.
+
+    Items without lat/lng are passed through (so e.g. summary blobs aren't
+    accidentally dropped). The existing _bbox_filter helper applies a 20%
+    pad and handles antimeridian crossings.
+    """
+    lat_span, lng_span = _bbox_spans(s, w, n, e)
+    # World-scale request → skip filtering entirely. Spares the CPU and
+    # guarantees the response matches the no-params shape.
+    if lng_span >= 300 or lat_span >= 120:
+        return payload
+    for key in heavy_keys:
+        items = payload.get(key)
+        if not isinstance(items, list) or not items:
+            continue
+        payload[key] = _bbox_filter(items, s, w, n, e)
+    return payload
+
+
+def _json_safe(value):
+    if isinstance(value, float):
+        return value if math.isfinite(value) else None
+    if isinstance(value, dict):
+        return {k: _json_safe(v) for k, v in list(value.items())}
+    if isinstance(value, list):
+        return [_json_safe(v) for v in list(value)]
+    if isinstance(value, tuple):
+        return [_json_safe(v) for v in list(value)]
+    return value
+
+
+def _sanitize_payload(value):
+    if isinstance(value, float):
+        return value if math.isfinite(value) else None
+    if isinstance(value, dict):
+        return {k: _sanitize_payload(v) for k, v in list(value.items())}
+    if isinstance(value, (list, tuple)):
+        return list(value)
+    return value
+
+
+def _live_data_json_bytes(payload: dict) -> bytes:
+    """Serialize dashboard payloads with the same defensive orjson options everywhere."""
+    return orjson.dumps(
+        _sanitize_payload(payload),
+        default=str,
+        option=orjson.OPT_NON_STR_KEYS,
+    )
+
+
+def _bbox_filter(items: list, s: float, w: float, n: float, e: float,
+                 lat_key: str = "lat", lng_key: str = "lng") -> list:
+    pad_lat = (n - s) * 0.2
+    pad_lng = (e - w) * 0.2 if e > w else ((e + 360 - w) * 0.2)
+    s2, n2 = s - pad_lat, n + pad_lat
+    w2, e2 = w - pad_lng, e + pad_lng
+    crosses_antimeridian = w2 > e2
+    out = []
+    for item in items:
+        lat = item.get(lat_key)
+        lng = item.get(lng_key)
+        if lat is None or lng is None:
+            out.append(item)
+            continue
+        if not (s2 <= lat <= n2):
+            continue
+        if crosses_antimeridian:
+            if lng >= w2 or lng <= e2:
+                out.append(item)
+        else:
+            if w2 <= lng <= e2:
+                out.append(item)
+    return out
+
+
+def _bbox_filter_geojson_points(items: list, s: float, w: float, n: float, e: float) -> list:
+    pad_lat = (n - s) * 0.2
+    pad_lng = (e - w) * 0.2 if e > w else ((e + 360 - w) * 0.2)
+    s2, n2 = s - pad_lat, n + pad_lat
+    w2, e2 = w - pad_lng, e + pad_lng
+    crosses_antimeridian = w2 > e2
+    out = []
+    for item in items:
+        geometry = item.get("geometry") if isinstance(item, dict) else None
+        coords = geometry.get("coordinates") if isinstance(geometry, dict) else None
+        if not isinstance(coords, (list, tuple)) or len(coords) < 2:
+            out.append(item)
+            continue
+        lng, lat = coords[0], coords[1]
+        if lat is None or lng is None:
+            out.append(item)
+            continue
+        if not (s2 <= lat <= n2):
+            continue
+        if crosses_antimeridian:
+            if lng >= w2 or lng <= e2:
+                out.append(item)
+        else:
+            if w2 <= lng <= e2:
+                out.append(item)
+    return out
+
+
+def _bbox_spans(s, w, n, e) -> tuple:
+    if None in (s, w, n, e):
+        return 180.0, 360.0
+    lat_span = max(0.0, float(n) - float(s))
+    lng_span = float(e) - float(w)
+    if lng_span < 0:
+        lng_span += 360.0
+    if lng_span == 0 and w == -180 and e == 180:
+        lng_span = 360.0
+    return lat_span, max(0.0, lng_span)
+
+
+def _cap_startup_items(items: list | None, max_items: int) -> list:
+    if not items:
+        return []
+    if len(items) <= max_items:
+        return items
+    return items[:max_items]
+
+
+def _cap_fast_startup_payload(payload: dict) -> dict:
+    capped = dict(payload)
+    capped["commercial_flights"] = _cap_startup_items(capped.get("commercial_flights"), 800)
+    capped["private_flights"] = _cap_startup_items(capped.get("private_flights"), 300)
+    capped["private_jets"] = _cap_startup_items(capped.get("private_jets"), 150)
+    capped["ships"] = _cap_startup_items(capped.get("ships"), 1500)
+    capped["cctv"] = []
+    capped["sigint"] = _cap_startup_items(capped.get("sigint"), 500)
+    capped["trains"] = _cap_startup_items(capped.get("trains"), 100)
+    capped["startup_payload"] = True
+    return capped
+
+
+def _cap_fast_dashboard_payload(payload: dict) -> dict:
+    return payload
+
+
+def _world_and_continental_scale(has_bbox: bool, s, w, n, e) -> tuple:
+    lat_span, lng_span = _bbox_spans(s, w, n, e)
+    world_scale = (not has_bbox) or lng_span >= 300 or lat_span >= 120
+    continental_scale = has_bbox and not world_scale and (lng_span >= 120 or lat_span >= 55)
+    return world_scale, continental_scale
+
+
+def _filter_sigint_by_layers(items: list, active_layers: dict) -> list:
+    allow_aprs = bool(active_layers.get("sigint_aprs", True))
+    allow_mesh = bool(active_layers.get("sigint_meshtastic", True))
+    if allow_aprs and allow_mesh:
+        return items
+    allowed_sources: set = {"js8call"}
+    if allow_aprs:
+        allowed_sources.add("aprs")
+    if allow_mesh:
+        allowed_sources.update({"meshtastic", "meshtastic-map"})
+    return [item for item in items if str(item.get("source") or "").lower() in allowed_sources]
+
+
+def _sigint_totals_for_items(items: list) -> dict:
+    totals = {"total": len(items), "meshtastic": 0, "meshtastic_live": 0, "meshtastic_map": 0,
+              "aprs": 0, "js8call": 0}
+    for item in items:
+        source = str(item.get("source") or "").lower()
+        if source == "meshtastic":
+            totals["meshtastic"] += 1
+            if bool(item.get("from_api")):
+                totals["meshtastic_map"] += 1
+            else:
+                totals["meshtastic_live"] += 1
+        elif source == "aprs":
+            totals["aprs"] += 1
+        elif source == "js8call":
+            totals["js8call"] += 1
+    return totals
+
+
+@router.get("/api/refresh", dependencies=[Depends(require_admin)])
+@limiter.limit("2/minute")
+async def force_refresh(request: Request):
+    from services.schemas import RefreshResponse
+    if not _refresh_lock.acquire(blocking=False):
+        return {"status": "refresh already in progress"}
+
+    def _do_refresh():
+        try:
+            update_all_data()
+        finally:
+            _refresh_lock.release()
+
+    t = threading.Thread(target=_do_refresh)
+    t.start()
+    return {"status": "refreshing in background"}
+
+
+@router.post("/api/ais/feed", dependencies=[Depends(require_local_operator)])
+@limiter.limit("60/minute")
+async def ais_feed(request: Request):
+    """Accept AIS-catcher HTTP JSON feed (POST decoded AIS messages)."""
+    from services.ais_stream import ingest_ais_catcher
+    try:
+        body = await request.json()
+    except Exception:
+        return JSONResponse(status_code=422, content={"ok": False, "detail": "invalid JSON body"})
+    msgs = body.get("msgs", [])
+    if not msgs:
+        return {"status": "ok", "ingested": 0}
+    count = ingest_ais_catcher(msgs)
+    return {"status": "ok", "ingested": count}
+
+
+@router.get("/api/trail/flight/{icao24}")
+@limiter.limit("120/minute")
+async def get_selected_flight_trail(icao24: str, request: Request):  # noqa: ARG001
+    from services.fetchers.flights import get_flight_trail
+    return {"id": icao24, "trail": get_flight_trail(icao24)}
+
+
+@router.get("/api/trail/ship/{mmsi}")
+@limiter.limit("120/minute")
+async def get_selected_ship_trail(mmsi: int, request: Request):  # noqa: ARG001
+    from services.ais_stream import get_vessel_trail
+    return {"id": mmsi, "trail": get_vessel_trail(mmsi)}
+
+
+@router.post("/api/viewport")
+@limiter.limit("60/minute")
+async def update_viewport(vp: ViewportUpdate, request: Request):  # noqa: ARG001
+    """Receive frontend map bounds. AIS stream stays global so open-ocean
+    vessels are never dropped — the frontend worker handles viewport culling."""
+    return {"status": "ok"}
+
+
+@router.get("/api/liveuamap/scraper-status", dependencies=[Depends(require_local_operator)])
+async def api_liveuamap_scraper_status():
+    """Whether LiveUAMap Playwright may run (Windows needs UI opt-in unless env forces)."""
+    from services.liveuamap_settings import liveuamap_scraper_status
+
+    return liveuamap_scraper_status()
+
+
+@router.post("/api/liveuamap/scraper-opt-in", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_liveuamap_scraper_opt_in(body: LiveUamapOptInUpdate, request: Request):
+    """Persist operator consent for LiveUAMap scraper (#348)."""
+    from services.liveuamap_settings import liveuamap_scraper_status, set_liveuamap_ui_opt_in
+
+    set_liveuamap_ui_opt_in(body.opted_in)
+    if body.opted_in:
+        from services.fetchers._store import is_any_active
+
+        if is_any_active("global_incidents"):
+            threading.Thread(target=_run_liveuamap_refresh, daemon=True).start()
+    return liveuamap_scraper_status()
+
+
+def _run_liveuamap_refresh() -> None:
+    try:
+        from services.fetchers.geo import update_liveuamap
+
+        update_liveuamap()
+    except Exception as e:
+        logger.warning("LiveUAMap refresh after opt-in failed: %s", e)
+
+
+@router.get("/api/prediction-markets/status", dependencies=[Depends(require_local_operator)])
+async def api_prediction_markets_status():
+    """Whether Polymarket/Kalshi fetches and news market correlation are enabled."""
+    from services.prediction_markets_settings import prediction_markets_status
+
+    return prediction_markets_status()
+
+
+@router.post("/api/prediction-markets/opt-in", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_prediction_markets_opt_in(body: PredictionMarketsOptInUpdate, request: Request):
+    """Enable or disable prediction market fetches + intercept story correlation."""
+    from services.config import get_settings
+    from services.prediction_markets_settings import (
+        prediction_markets_status,
+        set_prediction_markets_ui_opt_in,
+    )
+    from routers.ai_intel import _write_env_value
+
+    set_prediction_markets_ui_opt_in(body.opted_in)
+    _write_env_value("PREDICTION_MARKETS_ENABLED", "true" if body.opted_in else "false")
+    os.environ["PREDICTION_MARKETS_ENABLED"] = "true" if body.opted_in else "false"
+    get_settings.cache_clear()
+
+    if body.opted_in:
+        threading.Thread(target=_run_prediction_markets_refresh, daemon=True).start()
+    else:
+        threading.Thread(target=_run_prediction_markets_disable, daemon=True).start()
+
+    return prediction_markets_status()
+
+
+def _run_prediction_markets_refresh() -> None:
+    try:
+        from services.fetchers.prediction_markets import fetch_prediction_markets
+        from services.fetchers.news import fetch_news
+
+        fetch_prediction_markets()
+        fetch_news()
+    except Exception as e:
+        logger.warning("Prediction markets refresh after opt-in failed: %s", e)
+
+
+def _run_prediction_markets_disable() -> None:
+    try:
+        from services.fetchers._store import _data_lock, _mark_fresh, latest_data
+        from services.fetchers.news import fetch_news
+
+        with _data_lock:
+            latest_data["prediction_markets"] = []
+            latest_data["trending_markets"] = []
+        _mark_fresh("prediction_markets")
+        fetch_news()
+    except Exception as e:
+        logger.warning("Prediction markets disable cleanup failed: %s", e)
+
+
+@router.post("/api/layers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def update_layers(update: LayerUpdate, request: Request):
+    """Receive frontend layer toggle state. Starts/stops streams accordingly."""
+    from services.fetchers._store import active_layers, bump_active_layers_version, is_any_active
+    old_ships = is_any_active("ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts")
+    old_mesh = is_any_active("sigint_meshtastic")
+    old_aprs = is_any_active("sigint_aprs")
+    old_viirs = is_any_active("viirs_nightlights")
+    old_datacenters = is_any_active("datacenters")
+    old_fishing = is_any_active("fishing_activity")
+    changed = False
+    for key, value in update.layers.items():
+        if key in active_layers:
+            if active_layers[key] != value:
+                changed = True
+            active_layers[key] = value
+    if changed:
+        bump_active_layers_version()
+    new_ships = is_any_active("ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts")
+    new_mesh = is_any_active("sigint_meshtastic")
+    new_aprs = is_any_active("sigint_aprs")
+    new_viirs = is_any_active("viirs_nightlights")
+    new_datacenters = is_any_active("datacenters")
+    new_fishing = is_any_active("fishing_activity")
+    if old_ships and not new_ships:
+        from services.ais_stream import stop_ais_stream
+        stop_ais_stream()
+        logger.info("AIS stream stopped (all ship layers disabled)")
+    elif not old_ships and new_ships:
+        from services.ais_stream import start_ais_stream
+        start_ais_stream()
+        logger.info("AIS stream started (ship layer enabled)")
+    from services.sigint_bridge import sigint_grid
+    if old_mesh and not new_mesh:
+        try:
+            from services.meshtastic_mqtt_settings import mqtt_bridge_enabled
+            keep_chat_running = mqtt_bridge_enabled()
+        except Exception:
+            keep_chat_running = False
+        if keep_chat_running:
+            logger.info("Meshtastic map layer disabled; MQTT bridge kept running for MeshChat")
+        else:
+            sigint_grid.mesh.stop()
+            logger.info("Meshtastic MQTT bridge stopped (layer disabled)")
+    elif not old_mesh and new_mesh:
+        try:
+            from services.meshtastic_mqtt_settings import mqtt_bridge_enabled
+            mqtt_enabled = mqtt_bridge_enabled()
+        except Exception:
+            mqtt_enabled = False
+        if mqtt_enabled:
+            sigint_grid.mesh.start()
+            logger.info("Meshtastic MQTT bridge started (layer enabled)")
+        else:
+            logger.info(
+                "Meshtastic layer enabled; MQTT bridge remains disabled "
+                "(set MESH_MQTT_ENABLED=true to participate in the public broker)"
+            )
+    if old_aprs and not new_aprs:
+        sigint_grid.aprs.stop()
+        logger.info("APRS bridge stopped (layer disabled)")
+    elif not old_aprs and new_aprs:
+        sigint_grid.aprs.start()
+        logger.info("APRS bridge started (layer enabled)")
+    if not old_viirs and new_viirs:
+        _queue_viirs_change_refresh()
+        logger.info("VIIRS change refresh queued (layer enabled)")
+    if not old_datacenters and new_datacenters:
+        from services.fetchers.infrastructure import fetch_datacenters
+
+        fetch_datacenters()
+        logger.info("Datacenters loaded (layer enabled)")
+    if not old_fishing and new_fishing:
+        from services.fetchers.geo import fetch_fishing_activity
+
+        fetch_fishing_activity()
+        logger.info("Fishing activity refresh queued (layer enabled)")
+    return {"status": "ok"}
+
+
+@router.get("/api/live-data")
+@limiter.limit("120/minute")
+async def live_data(request: Request):
+    etag = _current_etag(prefix="live|full|")
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    from services.fetchers._store import get_latest_data_deepcopy_snapshot
+
+    payload = get_latest_data_deepcopy_snapshot()
+    return Response(
+        content=_live_data_json_bytes(payload),
+        media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"},
+    )
+
+
+@router.get("/api/bootstrap/critical")
+@limiter.limit("180/minute")
+async def bootstrap_critical(request: Request):
+    """Cached first-paint payload for the dashboard.
+
+    This endpoint is intentionally memory-only: no upstream calls, no refresh,
+    and a bounded response. It exists so the map and threat feed can paint
+    before slower panels and background enrichers finish warming up.
+    """
+    etag = _current_etag(prefix="bootstrap|critical|")
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    from services.fetchers._store import (
+        active_layers,
+        get_latest_data_subset_refs,
+        get_source_timestamps_snapshot,
+    )
+
+    d = get_latest_data_subset_refs(
+        "last_updated", "commercial_flights", "military_flights", "private_flights",
+        "private_jets", "tracked_flights", "ships", "uavs", "liveuamap", "gps_jamming",
+        "satellites", "satellite_source", "satellite_analysis", "sigint", "sigint_totals",
+        "trains", "news", "gdelt", "airports", "threat_level", "trending_markets",
+        "correlations", "fimi", "crowdthreat",
+    )
+    freshness = get_source_timestamps_snapshot()
+    ships_enabled = any(active_layers.get(key, True) for key in (
+        "ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts"))
+    sigint_items = _filter_sigint_by_layers(d.get("sigint") or [], active_layers)
+    payload = {
+        "last_updated": d.get("last_updated"),
+        "commercial_flights": _cap_startup_items(
+            (d.get("commercial_flights") or []) if active_layers.get("flights", True) else [],
+            800,
+        ),
+        "military_flights": _cap_startup_items(
+            (d.get("military_flights") or []) if active_layers.get("military", True) else [],
+            300,
+        ),
+        "private_flights": _cap_startup_items(
+            (d.get("private_flights") or []) if active_layers.get("private", True) else [],
+            300,
+        ),
+        "private_jets": _cap_startup_items(
+            (d.get("private_jets") or []) if active_layers.get("jets", True) else [],
+            150,
+        ),
+        "tracked_flights": _cap_startup_items(
+            (d.get("tracked_flights") or []) if active_layers.get("tracked", True) else [],
+            250,
+        ),
+        "ships": _cap_startup_items((d.get("ships") or []) if ships_enabled else [], 1500),
+        "uavs": _cap_startup_items((d.get("uavs") or []) if active_layers.get("military", True) else [], 100),
+        "liveuamap": _cap_startup_items(
+            (d.get("liveuamap") or []) if active_layers.get("global_incidents", True) else [],
+            300,
+        ),
+        "gps_jamming": _cap_startup_items(
+            (d.get("gps_jamming") or []) if active_layers.get("gps_jamming", True) else [],
+            200,
+        ),
+        "satellites": _cap_startup_items(
+            (d.get("satellites") or []) if active_layers.get("satellites", True) else [],
+            250,
+        ),
+        "satellite_source": d.get("satellite_source", "none"),
+        "satellite_analysis": (d.get("satellite_analysis") or {}) if active_layers.get("satellites", True) else {},
+        "sigint": _cap_startup_items(
+            sigint_items if (active_layers.get("sigint_meshtastic", True) or active_layers.get("sigint_aprs", True)) else [],
+            500,
+        ),
+        "sigint_totals": _sigint_totals_for_items(sigint_items),
+        "trains": _cap_startup_items((d.get("trains") or []) if active_layers.get("trains", True) else [], 100),
+        "news": _cap_startup_items(d.get("news") or [], 30),
+        "gdelt": _cap_startup_items((d.get("gdelt") or []) if active_layers.get("global_incidents", True) else [], 300),
+        "airports": _cap_startup_items(d.get("airports") or [], 500),
+        "threat_level": d.get("threat_level"),
+        "trending_markets": _cap_startup_items(d.get("trending_markets") or [], 10),
+        "correlations": _cap_startup_items(
+            (d.get("correlations") or []) if active_layers.get("correlations", True) else [],
+            50,
+        ),
+        "fimi": d.get("fimi"),
+        "crowdthreat": _cap_startup_items(
+            (d.get("crowdthreat") or []) if active_layers.get("crowdthreat", True) else [],
+            150,
+        ),
+        "freshness": freshness,
+        "bootstrap_ready": True,
+        "bootstrap_payload": True,
+    }
+    return Response(
+        content=_live_data_json_bytes(payload),
+        media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"},
+    )
+
+
+@router.get("/api/live-data/fast")
+@limiter.limit("120/minute")
+async def live_data_fast(
+    request: Request,
+    s: float = Query(None, description="South bound — when all four bounds are supplied, heavy/dense layers (vessels, aircraft, sigint, CCTV, …) are filtered to this viewport with 20% padding. Static reference layers (satellites, etc.) always ship world-scale.", ge=-90, le=90),
+    w: float = Query(None, description="West bound (see s)", ge=-180, le=180),
+    n: float = Query(None, description="North bound (see s)", ge=-90, le=90),
+    e: float = Query(None, description="East bound (see s)", ge=-180, le=180),
+    initial: bool = Query(False, description="Return a capped startup payload for first paint"),
+):
+    bbox_suffix = _bbox_etag_suffix(s, w, n, e)
+    etag = _current_etag(prefix=("fast|initial|" if initial else "fast|full|") + bbox_suffix.lstrip("|") + ("|" if bbox_suffix else ""))
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    from services.fetchers._store import (active_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot)
+    d = get_latest_data_subset_refs(
+        "last_updated", "commercial_flights", "military_flights", "private_flights",
+        "private_jets", "tracked_flights", "ships", "cctv", "uavs", "liveuamap",
+        "gps_jamming", "satellites", "satellite_source", "satellite_analysis",
+        "sigint", "sigint_totals", "trains",
+    )
+    freshness = get_source_timestamps_snapshot()
+    ships_enabled = any(active_layers.get(key, True) for key in (
+        "ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts"))
+    cctv_total = len(d.get("cctv") or [])
+    sigint_items = _filter_sigint_by_layers(d.get("sigint") or [], active_layers)
+    sigint_totals = _sigint_totals_for_items(sigint_items)
+    payload = {
+        "commercial_flights": (d.get("commercial_flights") or []) if active_layers.get("flights", True) else [],
+        "military_flights": (d.get("military_flights") or []) if active_layers.get("military", True) else [],
+        "private_flights": (d.get("private_flights") or []) if active_layers.get("private", True) else [],
+        "private_jets": (d.get("private_jets") or []) if active_layers.get("jets", True) else [],
+        "tracked_flights": (d.get("tracked_flights") or []) if active_layers.get("tracked", True) else [],
+        "ships": (d.get("ships") or []) if ships_enabled else [],
+        "cctv": (d.get("cctv") or []) if active_layers.get("cctv", True) else [],
+        "uavs": (d.get("uavs") or []) if active_layers.get("military", True) else [],
+        "liveuamap": (d.get("liveuamap") or []) if active_layers.get("global_incidents", True) else [],
+        "gps_jamming": (d.get("gps_jamming") or []) if active_layers.get("gps_jamming", True) else [],
+        "satellites": (d.get("satellites") or []) if active_layers.get("satellites", True) else [],
+        "satellite_source": d.get("satellite_source", "none"),
+        "satellite_analysis": (d.get("satellite_analysis") or {}) if active_layers.get("satellites", True) else {},
+        "sigint": sigint_items if (active_layers.get("sigint_meshtastic", True) or active_layers.get("sigint_aprs", True)) else [],
+        "sigint_totals": sigint_totals,
+        "cctv_total": cctv_total,
+        "trains": (d.get("trains") or []) if active_layers.get("trains", True) else [],
+        "freshness": freshness,
+    }
+    if initial:
+        payload = _cap_fast_startup_payload(payload)
+    else:
+        payload = _cap_fast_dashboard_payload(payload)
+    # Issue #288: bbox filter heavy/dense layers only when all four bounds
+    # are supplied. Without bounds, behaviour is byte-for-byte identical
+    # to the pre-#288 implementation.
+    if _has_full_bbox(s, w, n, e):
+        payload = _apply_bbox_to_payload(payload, _FAST_BBOX_HEAVY_KEYS, s, w, n, e)
+    return Response(
+        content=_live_data_json_bytes(payload),
+        media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"},
+    )
+
+
+@router.get("/api/live-data/slow")
+@limiter.limit("60/minute")
+async def live_data_slow(
+    request: Request,
+    s: float = Query(None, description="South bound — when all four bounds are supplied, heavy/dense layers (gdelt, firms_fires, kiwisdr, scanners, psk_reporter) are filtered to this viewport with 20% padding. Static reference layers (datacenters, military bases, power plants, weather, news, …) always ship world-scale.", ge=-90, le=90),
+    w: float = Query(None, description="West bound (see s)", ge=-180, le=180),
+    n: float = Query(None, description="North bound (see s)", ge=-90, le=90),
+    e: float = Query(None, description="East bound (see s)", ge=-180, le=180),
+):
+    bbox_suffix = _bbox_etag_suffix(s, w, n, e)
+    etag = _current_etag(prefix="slow|full|" + bbox_suffix.lstrip("|") + ("|" if bbox_suffix else ""))
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    from services.fetchers._store import (active_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot)
+    d = get_latest_data_subset_refs(
+        "last_updated", "news", "stocks", "financial_source", "oil", "weather", "traffic",
+        "earthquakes", "frontlines", "gdelt", "airports", "kiwisdr", "satnogs_stations",
+        "satnogs_observations", "tinygs_satellites", "space_weather", "internet_outages",
+        "firms_fires", "datacenters", "military_bases", "power_plants", "viirs_change_nodes",
+        "scanners", "weather_alerts", "ukraine_alerts", "air_quality", "volcanoes",
+        "fishing_activity", "psk_reporter", "correlations", "uap_sightings", "wastewater",
+        "crowdthreat", "threat_level", "trending_markets", "road_corridor_trends",
+        "malware_threats", "cyber_threats", "scm_suppliers", "telegram_osint",
+    )
+    freshness = get_source_timestamps_snapshot()
+    payload = {
+        "last_updated": d.get("last_updated"),
+        "threat_level": d.get("threat_level"),
+        "trending_markets": d.get("trending_markets", []),
+        "news": d.get("news", []),
+        "stocks": d.get("stocks", {}),
+        "financial_source": d.get("financial_source", ""),
+        "oil": d.get("oil", {}),
+        "weather": d.get("weather"),
+        "traffic": d.get("traffic", []),
+        "earthquakes": (d.get("earthquakes") or []) if active_layers.get("earthquakes", True) else [],
+        "frontlines": d.get("frontlines") if active_layers.get("ukraine_frontline", True) else None,
+        "gdelt": (d.get("gdelt") or []) if active_layers.get("global_incidents", True) else [],
+        "airports": d.get("airports") or [],
+        "kiwisdr": (d.get("kiwisdr") or []) if active_layers.get("kiwisdr", True) else [],
+        "satnogs_stations": (d.get("satnogs_stations") or []) if active_layers.get("satnogs", True) else [],
+        "satnogs_total": len(d.get("satnogs_stations") or []),
+        "satnogs_observations": (d.get("satnogs_observations") or []) if active_layers.get("satnogs", True) else [],
+        "tinygs_satellites": (d.get("tinygs_satellites") or []) if active_layers.get("tinygs", True) else [],
+        "tinygs_total": len(d.get("tinygs_satellites") or []),
+        "psk_reporter": (d.get("psk_reporter") or []) if active_layers.get("psk_reporter", True) else [],
+        "space_weather": d.get("space_weather"),
+        "internet_outages": (d.get("internet_outages") or []) if active_layers.get("internet_outages", True) else [],
+        "firms_fires": (d.get("firms_fires") or []) if active_layers.get("firms", True) else [],
+        "datacenters": (d.get("datacenters") or []) if active_layers.get("datacenters", True) else [],
+        "military_bases": (d.get("military_bases") or []) if active_layers.get("military_bases", True) else [],
+        "power_plants": (d.get("power_plants") or []) if active_layers.get("power_plants", True) else [],
+        "viirs_change_nodes": (d.get("viirs_change_nodes") or []) if active_layers.get("viirs_nightlights", True) else [],
+        "scanners": (d.get("scanners") or []) if active_layers.get("scanners", True) else [],
+        "weather_alerts": d.get("weather_alerts", []) if active_layers.get("weather_alerts", True) else [],
+        "ukraine_alerts": d.get("ukraine_alerts", []) if active_layers.get("ukraine_alerts", True) else [],
+        "air_quality": (d.get("air_quality") or []) if active_layers.get("air_quality", True) else [],
+        "volcanoes": (d.get("volcanoes") or []) if active_layers.get("volcanoes", True) else [],
+        "fishing_activity": (d.get("fishing_activity") or []) if active_layers.get("fishing_activity", True) else [],
+        "correlations": (d.get("correlations") or []) if active_layers.get("correlations", True) else [],
+        "uap_sightings": (d.get("uap_sightings") or []) if active_layers.get("uap_sightings", True) else [],
+        "wastewater": (d.get("wastewater") or []) if active_layers.get("wastewater", True) else [],
+        "crowdthreat": (d.get("crowdthreat") or []) if active_layers.get("crowdthreat", True) else [],
+        "road_corridor_trends": (
+            d.get("road_corridor_trends") or {"updated_at": None, "corridors": []}
+        )
+        if active_layers.get("road_corridor_trends", False)
+        else {"updated_at": None, "corridors": []},
+        "malware_threats": (
+            d.get("malware_threats") or {"threats": [], "total": 0}
+        )
+        if active_layers.get("malware_c2", False)
+        else {"threats": [], "total": 0},
+        "cyber_threats": (
+            d.get("cyber_threats") or {"threats": [], "stats": {}}
+        )
+        if active_layers.get("cyber_threats", False)
+        else {"threats": [], "stats": {}},
+        "scm_suppliers": (
+            d.get("scm_suppliers") or {"suppliers": [], "total": 0, "critical_count": 0}
+        )
+        if active_layers.get("scm_suppliers", False)
+        else {"suppliers": [], "total": 0, "critical_count": 0},
+        "telegram_osint": (
+            d.get("telegram_osint") or {"posts": [], "total": 0, "geolocated": 0}
+        )
+        if active_layers.get("telegram_osint", True)
+        else {"posts": [], "total": 0, "geolocated": 0},
+        "freshness": freshness,
+    }
+    # Issue #288: bbox filter heavy/dense layers only when all four bounds
+    # are supplied. Static reference layers (datacenters, military bases,
+    # power_plants, etc.) deliberately stay world-scale so panning never
+    # hides the infrastructure overlay the operator already has on screen.
+    if _has_full_bbox(s, w, n, e):
+        payload = _apply_bbox_to_payload(payload, _SLOW_BBOX_HEAVY_KEYS, s, w, n, e)
+    return Response(
+        content=_live_data_json_bytes(payload),
+        media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"},
+    )
+
+
+# ── Satellite Overflight Counting ───────────────────────────────────────────
+# Counts unique satellites whose ground track entered a bounding box over 24h.
+# Uses cached TLEs + SGP4 propagation — no extra network requests.
+
+class OverflightRequest(BaseModel):
+    s: float
+    w: float
+    n: float
+    e: float
+    hours: int = 24
+
+
+# Issue #202: compute_overflights() is O(catalog_size × timesteps), where
+# timesteps grows linearly with `hours`. An unbounded `hours` value is a
+# trivial CPU-exhaustion vector. We clamp silently rather than raising 422 —
+# the response shape is unchanged, callers asking for too many hours just
+# get a shorter window, which is friendlier than a hostile error.
+#
+# Override via OVERFLIGHTS_MAX_HOURS env var if you legitimately need a
+# longer window (e.g. a planning use case that wants a full week).
+def _overflight_max_hours() -> int:
+    import os as _os
+    try:
+        raw = int(str(_os.environ.get("OVERFLIGHTS_MAX_HOURS", "72")).strip())
+    except (TypeError, ValueError):
+        raw = 72
+    return max(1, raw)
+
+
+@router.post("/api/satellites/overflights")
+@limiter.limit("10/minute")
+async def satellite_overflights(request: Request, body: OverflightRequest):
+    from services.fetchers.satellites import compute_overflights, _sat_gp_cache
+    gp_data = _sat_gp_cache.get("data")
+    if not gp_data:
+        return JSONResponse({"total": 0, "by_mission": {}, "satellites": [], "error": "No GP data cached yet"})
+    bbox = {"s": body.s, "w": body.w, "n": body.n, "e": body.e}
+
+    # Silent clamp — see comment on _overflight_max_hours().
+    requested_hours = max(1, int(body.hours or 0))
+    effective_hours = min(requested_hours, _overflight_max_hours())
+
+    result = compute_overflights(gp_data, bbox, hours=effective_hours)
+    # If we clamped, surface the effective window in the response so the
+    # caller can detect it if they care, without it being an error.
+    if isinstance(result, dict) and effective_hours != requested_hours:
+        result.setdefault("requested_hours", requested_hours)
+        result.setdefault("effective_hours", effective_hours)
+    return JSONResponse(result)

backend/routers/entity_graph.py

@@ -0,0 +1,30 @@
+"""Entity graph expansion (intel layer)."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+
+from auth import require_local_operator
+from limiter import limiter
+from services.osint_intel.resolve import resolve_entity
+
+router = APIRouter()
+
+
+@router.get("/api/entity/expand")
+@limiter.limit("30/minute")
+async def entity_expand(
+    request: Request,
+    _: None = Depends(require_local_operator),
+    type: str = Query(..., min_length=3, max_length=32),
+    id: str = Query(..., min_length=2, max_length=200),
+    registration: str | None = Query(default=None, max_length=32),
+    model: str | None = Query(default=None, max_length=64),
+    icao24: str | None = Query(default=None, max_length=16),
+) -> dict:
+    props = {"label": id, "registration": registration, "model": model, "icao24": icao24}
+    try:
+        return resolve_entity(type, id, props)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    except Exception as exc:
+        raise HTTPException(status_code=502, detail="Intelligence layer unavailable") from exc

backend/routers/health.py

@@ -0,0 +1,117 @@
+import time as _time_mod
+from fastapi import APIRouter, Request, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin
+from services.data_fetcher import get_latest_data
+from services.schemas import HealthResponse
+import os
+
+APP_VERSION = os.environ.get("_HEALTH_APP_VERSION", "0.9.82")
+
+router = APIRouter()
+
+
+def _get_app_version() -> str:
+    # Import lazily to avoid circular import; main sets APP_VERSION before including routers
+    try:
+        import main as _main
+        return _main.APP_VERSION
+    except Exception:
+        return APP_VERSION
+
+
+_start_time_ref: dict = {"value": None}
+
+
+def _get_start_time() -> float:
+    if _start_time_ref["value"] is None:
+        try:
+            import main as _main
+            _start_time_ref["value"] = _main._start_time
+        except Exception:
+            _start_time_ref["value"] = _time_mod.time()
+    return _start_time_ref["value"]
+
+
+@router.get("/api/health", response_model=HealthResponse)
+@limiter.limit("30/minute")
+async def health_check(request: Request):
+    from services.fetchers._store import get_source_timestamps_snapshot
+    from services.slo import compute_all_statuses, summarise_statuses
+
+    d = get_latest_data()
+    last = d.get("last_updated")
+    timestamps = get_source_timestamps_snapshot()
+    slo_statuses = compute_all_statuses(d, timestamps)
+    slo_summary = summarise_statuses(slo_statuses)
+    # Top-level status reflects worst SLO result — "degraded" if any
+    # yellow, "error" if any red, "ok" otherwise. This is the single
+    # field an external probe / pager can watch.
+    top_status = "ok"
+    if slo_summary.get("red", 0) > 0:
+        top_status = "error"
+    elif slo_summary.get("yellow", 0) > 0:
+        top_status = "degraded"
+
+    # Issue #258: surface AIS proxy degraded TLS state so operators can see
+    # when the SPKI-pinned fallback is in effect. The data plane keeps
+    # flowing (this is by design — see ais_proxy.js comments) but observers
+    # who care about MITM-protection posture deserve a visible signal.
+    #
+    # Plus connectivity health (added 2026-05-23 when stream.aisstream.io
+    # went fully offline): ``connected`` tells the frontend whether ship
+    # data is actually flowing. When false, a banner explains that ships
+    # are unavailable due to an upstream outage — better than the user
+    # silently seeing an empty ocean and assuming we broke something.
+    ais_status: dict = {}
+    try:
+        from services.ais_stream import ais_proxy_status
+        ais_status = ais_proxy_status() or {}
+    except Exception:
+        ais_status = {}
+    if ais_status.get("degraded_tls") and top_status == "ok":
+        # Don't override a worse top-level status if SLOs already failed,
+        # but escalate ok -> degraded so the field surfaces in dashboards.
+        top_status = "degraded"
+    # AIS_API_KEY not configured is "feature off", not "system broken" —
+    # so we only escalate when the operator opted into AIS (key set) AND
+    # the stream is currently offline.
+    if (
+        os.environ.get("AIS_API_KEY")
+        and ais_status.get("connected") is False
+        and top_status == "ok"
+    ):
+        top_status = "degraded"
+
+    return {
+        "status": top_status,
+        "version": _get_app_version(),
+        "last_updated": last,
+        "sources": {
+            "flights": len(d.get("commercial_flights", [])),
+            "military": len(d.get("military_flights", [])),
+            "ships": len(d.get("ships", [])),
+            "satellites": len(d.get("satellites", [])),
+            "earthquakes": len(d.get("earthquakes", [])),
+            "cctv": len(d.get("cctv", [])),
+            "news": len(d.get("news", [])),
+            "uavs": len(d.get("uavs", [])),
+            "firms_fires": len(d.get("firms_fires", [])),
+            "liveuamap": len(d.get("liveuamap", [])),
+            "gdelt": len(d.get("gdelt", [])),
+            "uap_sightings": len(d.get("uap_sightings", [])),
+        },
+        "freshness": timestamps,
+        "uptime_seconds": round(_time_mod.time() - _get_start_time()),
+        "slo": slo_statuses,
+        "slo_summary": slo_summary,
+        "ais_proxy": ais_status,
+    }
+
+
+@router.get("/api/debug-latest", dependencies=[Depends(require_admin)])
+@limiter.limit("30/minute")
+async def debug_latest_data(request: Request):
+    return list(get_latest_data().keys())

backend/routers/infonet.py

@@ -0,0 +1,598 @@
+"""Infonet economy / governance / gates / bootstrap HTTP surface.
+
+Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §2.1.
+
+Read endpoints return chain-derived state (computed by the
+``services.infonet`` adapters / pure functions). Write endpoints take
+a payload, validate it through the cutover-registered validators, and
+return a structured "would-emit" preview. Production wiring (signing
++ ``Infonet.append`` persistence) is a thin follow-on; the validation
+contract is locked here.
+
+Cross-cutting design rule: errors are diagnostic, not punitive. Each
+write endpoint returns ``{"ok": False, "reason": "..."}`` on
+validation failure with the exact field that failed. Frontend
+surfaces the reason in the UI.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any
+
+from fastapi import APIRouter, Body, Path
+
+# Triggers the chain cutover at module-load time so registered
+# validators are live for any subsequent route invocation.
+from services.infonet import _chain_cutover  # noqa: F401
+from services.infonet.adapters.gate_adapter import InfonetGateAdapter
+from services.infonet.adapters.oracle_adapter import InfonetOracleAdapter
+from services.infonet.adapters.reputation_adapter import InfonetReputationAdapter
+from services.infonet.bootstrap import compute_active_features
+from services.infonet.config import (
+    CONFIG,
+    IMMUTABLE_PRINCIPLES,
+)
+from services.infonet.governance import (
+    apply_petition_payload,
+    compute_petition_state,
+    compute_upgrade_state,
+)
+from services.infonet.governance.dsl_executor import InvalidPetition
+from services.infonet.partition import (
+    classify_event_type,
+    is_chain_stale,
+    should_mark_provisional,
+)
+from services.infonet.privacy import (
+    DEXScaffolding,
+    RingCTScaffolding,
+    ShieldedBalanceScaffolding,
+    StealthAddressScaffolding,
+)
+from services.infonet.schema import (
+    INFONET_ECONOMY_EVENT_TYPES,
+    validate_infonet_event_payload,
+)
+from services.infonet.time_validity import chain_majority_time
+
+logger = logging.getLogger("routers.infonet")
+
+router = APIRouter(prefix="/api/infonet", tags=["infonet"])
+
+
+# ─── Chain access helper ─────────────────────────────────────────────────
+# Every adapter takes a ``chain_provider`` callable. We pull the live
+# Infonet chain from mesh_hashchain. Tests can monkeypatch this.
+
+def _live_chain() -> list[dict[str, Any]]:
+    try:
+        from services.mesh.mesh_hashchain import infonet
+        events = getattr(infonet, "events", None)
+        if isinstance(events, list):
+            return list(events)
+        # Some implementations use a deque; convert to list.
+        if events is not None:
+            return list(events)
+    except Exception as exc:
+        logger.debug("infonet chain unavailable: %s", exc)
+    return []
+
+
+def _now() -> float:
+    cmt = chain_majority_time(_live_chain())
+    return cmt if cmt > 0 else float(time.time())
+
+
+# ─── Status ──────────────────────────────────────────────────────────────
+
+@router.get("/status")
+def infonet_status() -> dict[str, Any]:
+    """Top-level health snapshot for the InfonetTerminal HUD.
+
+    Returns ramp activation flags, partition staleness, privacy
+    primitive statuses, immutable principles, and counts of
+    chain-derived state (markets / petitions / gates / etc).
+    """
+    chain = _live_chain()
+    now = _now()
+    features = compute_active_features(chain)
+
+    # Privacy primitive statuses (truthful — most are NOT_IMPLEMENTED).
+    privacy = {
+        "ringct": RingCTScaffolding().status().value,
+        "stealth_address": StealthAddressScaffolding().status().value,
+        "shielded_balance": ShieldedBalanceScaffolding().status().value,
+        "dex": DEXScaffolding().status().value,
+    }
+
+    return {
+        "ok": True,
+        "now": now,
+        "chain_majority_time": chain_majority_time(chain),
+        "chain_event_count": len(chain),
+        "chain_stale": is_chain_stale(chain, now=now),
+        "ramp": {
+            "node_count": features.node_count,
+            "bootstrap_resolution_active": features.bootstrap_resolution_active,
+            "staked_resolution_active": features.staked_resolution_active,
+            "governance_petitions_active": features.governance_petitions_active,
+            "upgrade_governance_active": features.upgrade_governance_active,
+            "commoncoin_active": features.commoncoin_active,
+        },
+        "privacy_primitive_status": privacy,
+        "immutable_principles": dict(IMMUTABLE_PRINCIPLES),
+        "config_keys_count": len(CONFIG),
+        "infonet_economy_event_types_count": len(INFONET_ECONOMY_EVENT_TYPES),
+    }
+
+
+# ─── Petitions / governance ──────────────────────────────────────────────
+
+@router.get("/petitions")
+def list_petitions() -> dict[str, Any]:
+    """List petition_file events on the chain with their current state."""
+    chain = _live_chain()
+    now = _now()
+    out: list[dict[str, Any]] = []
+    for ev in chain:
+        if ev.get("event_type") != "petition_file":
+            continue
+        pid = (ev.get("payload") or {}).get("petition_id")
+        if not isinstance(pid, str):
+            continue
+        try:
+            state = compute_petition_state(pid, chain, now=now)
+            out.append({
+                "petition_id": state.petition_id,
+                "status": state.status,
+                "filer_id": state.filer_id,
+                "filed_at": state.filed_at,
+                "petition_payload": state.petition_payload,
+                "signature_governance_weight": state.signature_governance_weight,
+                "signature_threshold_at_filing": state.signature_threshold_at_filing,
+                "votes_for_weight": state.votes_for_weight,
+                "votes_against_weight": state.votes_against_weight,
+                "voting_deadline": state.voting_deadline,
+                "challenge_window_until": state.challenge_window_until,
+            })
+        except Exception as exc:
+            logger.warning("petition state error for %s: %s", pid, exc)
+    return {"ok": True, "petitions": out, "now": now}
+
+
+@router.get("/petitions/{petition_id}")
+def get_petition(petition_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    state = compute_petition_state(petition_id, chain, now=now)
+    return {"ok": True, "petition": state.__dict__, "now": now}
+
+
+@router.post("/petitions/preview")
+def preview_petition_payload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
+    """Validate a petition payload through the DSL executor without
+    emitting it. Returns the candidate config diff so the UI can show
+    "this petition would change vote_decay_days from 90 to 30".
+    """
+    try:
+        result = apply_petition_payload(payload)
+        return {
+            "ok": True,
+            "changed_keys": list(result.changed_keys),
+            "new_values": {k: result.new_config[k] for k in result.changed_keys},
+        }
+    except InvalidPetition as exc:
+        return {"ok": False, "reason": str(exc)}
+
+
+@router.post("/events/validate")
+def validate_event(body: dict[str, Any] = Body(...)) -> dict[str, Any]:
+    """Validate an arbitrary Infonet economy event payload.
+
+    Frontend uses this for client-side preflight before signing /
+    submitting an event. Returns ``{ok: True}`` on success or
+    ``{ok: False, reason: ...}`` with the exact validation failure.
+    """
+    event_type = body.get("event_type")
+    payload = body.get("payload", {})
+    if not isinstance(event_type, str) or not event_type:
+        return {"ok": False, "reason": "event_type required"}
+    if not isinstance(payload, dict):
+        return {"ok": False, "reason": "payload must be an object"}
+    ok, reason = validate_infonet_event_payload(event_type, payload)
+    return {
+        "ok": ok,
+        "reason": reason if not ok else None,
+        "tier": classify_event_type(event_type),
+        "would_be_provisional": should_mark_provisional(event_type, _live_chain(), now=_now()),
+    }
+
+
+# ─── Upgrade-hash governance ────────────────────────────────────────────
+
+@router.get("/upgrades")
+def list_upgrades() -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    out: list[dict[str, Any]] = []
+    for ev in chain:
+        if ev.get("event_type") != "upgrade_propose":
+            continue
+        pid = (ev.get("payload") or {}).get("proposal_id")
+        if not isinstance(pid, str):
+            continue
+        try:
+            # Heavy node set is a runtime concept (transport tier ==
+            # private_strong per plan §3.5). Empty here for the
+            # snapshot endpoint; production will pass the live set.
+            state = compute_upgrade_state(pid, chain, now=now, heavy_node_ids=set())
+            out.append({
+                "proposal_id": state.proposal_id,
+                "status": state.status,
+                "proposer_id": state.proposer_id,
+                "filed_at": state.filed_at,
+                "release_hash": state.release_hash,
+                "target_protocol_version": state.target_protocol_version,
+                "votes_for_weight": state.votes_for_weight,
+                "votes_against_weight": state.votes_against_weight,
+                "readiness_fraction": state.readiness.fraction,
+                "readiness_threshold_met": state.readiness.threshold_met,
+            })
+        except Exception as exc:
+            logger.warning("upgrade state error for %s: %s", pid, exc)
+    return {"ok": True, "upgrades": out, "now": now}
+
+
+@router.get("/upgrades/{proposal_id}")
+def get_upgrade(proposal_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    state = compute_upgrade_state(proposal_id, chain, now=now, heavy_node_ids=set())
+    return {
+        "ok": True,
+        "upgrade": {
+            "proposal_id": state.proposal_id,
+            "status": state.status,
+            "proposer_id": state.proposer_id,
+            "filed_at": state.filed_at,
+            "release_hash": state.release_hash,
+            "target_protocol_version": state.target_protocol_version,
+            "signature_governance_weight": state.signature_governance_weight,
+            "votes_for_weight": state.votes_for_weight,
+            "votes_against_weight": state.votes_against_weight,
+            "voting_deadline": state.voting_deadline,
+            "challenge_window_until": state.challenge_window_until,
+            "activation_deadline": state.activation_deadline,
+            "readiness": {
+                "total_heavy_nodes": state.readiness.total_heavy_nodes,
+                "ready_count": state.readiness.ready_count,
+                "fraction": state.readiness.fraction,
+                "threshold_met": state.readiness.threshold_met,
+            },
+        },
+        "now": now,
+    }
+
+
+# ─── Markets / resolution / disputes ────────────────────────────────────
+
+@router.get("/markets/{market_id}")
+def get_market_state(market_id: str = Path(...)) -> dict[str, Any]:
+    """Full market view: lifecycle, snapshot, evidence, stakes,
+    excluded predictors, dispute state."""
+    chain = _live_chain()
+    now = _now()
+    oracle = InfonetOracleAdapter(lambda: chain)
+
+    status = oracle.market_status(market_id, now=now)
+    snap = oracle.find_snapshot(market_id)
+    bundles = oracle.collect_evidence(market_id)
+    excluded = sorted(oracle.excluded_predictor_ids(market_id))
+    disputes = oracle.collect_disputes(market_id)
+    reversed_flag = oracle.market_was_reversed(market_id)
+
+    return {
+        "ok": True,
+        "market_id": market_id,
+        "status": status.value,
+        "snapshot": snap,
+        "evidence_bundles": [
+            {
+                "node_id": b.node_id,
+                "claimed_outcome": b.claimed_outcome,
+                "evidence_hashes": list(b.evidence_hashes),
+                "source_description": b.source_description,
+                "bond": b.bond,
+                "timestamp": b.timestamp,
+                "is_first_for_side": b.is_first_for_side,
+                "submission_hash": b.submission_hash,
+            }
+            for b in bundles
+        ],
+        "excluded_predictor_ids": excluded,
+        "disputes": [
+            {
+                "dispute_id": d.dispute_id,
+                "challenger_id": d.challenger_id,
+                "challenger_stake": d.challenger_stake,
+                "opened_at": d.opened_at,
+                "is_resolved": d.is_resolved,
+                "resolved_outcome": d.resolved_outcome,
+                "confirm_stakes": d.confirm_stakes,
+                "reverse_stakes": d.reverse_stakes,
+            }
+            for d in disputes
+        ],
+        "was_reversed": reversed_flag,
+        "now": now,
+    }
+
+
+@router.get("/markets/{market_id}/preview-resolution")
+def preview_resolution(market_id: str = Path(...)) -> dict[str, Any]:
+    """Run the resolution decision procedure without emitting a
+    finalize event. UI uses this to show "if resolution closed now,
+    the market would resolve as <outcome> for <reason>"."""
+    chain = _live_chain()
+    oracle = InfonetOracleAdapter(lambda: chain)
+    result = oracle.resolve_market(market_id)
+    return {
+        "ok": True,
+        "preview": {
+            "outcome": result.outcome,
+            "reason": result.reason,
+            "is_provisional": result.is_provisional,
+            "burned_amount": result.burned_amount,
+            "stake_returns": [
+                {"node_id": k[0], "rep_type": k[1], "amount": v}
+                for k, v in result.stake_returns.items()
+            ],
+            "stake_winnings": [
+                {"node_id": k[0], "rep_type": k[1], "amount": v}
+                for k, v in result.stake_winnings.items()
+            ],
+            "bond_returns": [
+                {"node_id": k, "amount": v} for k, v in result.bond_returns.items()
+            ],
+            "bond_forfeits": [
+                {"node_id": k, "amount": v} for k, v in result.bond_forfeits.items()
+            ],
+            "first_submitter_bonuses": [
+                {"node_id": k, "amount": v}
+                for k, v in result.first_submitter_bonuses.items()
+            ],
+        },
+    }
+
+
+# ─── Gate shutdown lifecycle ────────────────────────────────────────────
+
+@router.get("/gates/{gate_id}")
+def get_gate_state(gate_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    gates = InfonetGateAdapter(lambda: chain)
+    meta = gates.gate_meta(gate_id)
+    if meta is None:
+        return {"ok": False, "reason": "gate_not_found"}
+    suspension = gates.suspension_state(gate_id, now=now)
+    shutdown = gates.shutdown_state(gate_id, now=now)
+    locked = gates.locked_state(gate_id)
+    members = sorted(gates.member_set(gate_id))
+    return {
+        "ok": True,
+        "gate_id": gate_id,
+        "meta": {
+            "creator_node_id": meta.creator_node_id,
+            "display_name": meta.display_name,
+            "entry_sacrifice": meta.entry_sacrifice,
+            "min_overall_rep": meta.min_overall_rep,
+            "min_gate_rep": dict(meta.min_gate_rep),
+            "created_at": meta.created_at,
+        },
+        "members": members,
+        "ratified": gates.is_ratified(gate_id),
+        "cumulative_member_oracle_rep": gates.cumulative_member_oracle_rep(gate_id),
+        "locked": {
+            "is_locked": locked.locked,
+            "locked_at": locked.locked_at,
+            "locked_by": list(locked.locked_by),
+        },
+        "suspension": {
+            "status": suspension.status,
+            "suspended_at": suspension.suspended_at,
+            "suspended_until": suspension.suspended_until,
+            "last_shutdown_petition_at": suspension.last_shutdown_petition_at,
+        },
+        "shutdown": {
+            "has_pending": shutdown.has_pending,
+            "pending_petition_id": shutdown.pending_petition_id,
+            "pending_status": shutdown.pending_status,
+            "execution_at": shutdown.execution_at,
+            "executed": shutdown.executed,
+        },
+        "now": now,
+    }
+
+
+# ─── Reputation views ───────────────────────────────────────────────────
+
+@router.get("/nodes/{node_id}/reputation")
+def get_node_reputation(node_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    rep = InfonetReputationAdapter(lambda: chain)
+    breakdown = rep.oracle_rep_breakdown(node_id)
+    return {
+        "ok": True,
+        "node_id": node_id,
+        "oracle_rep": rep.oracle_rep(node_id),
+        "oracle_rep_active": rep.oracle_rep_active(node_id),
+        "oracle_rep_lifetime": rep.oracle_rep_lifetime(node_id),
+        "common_rep": rep.common_rep(node_id),
+        "decay_factor": rep.decay_factor(node_id),
+        "last_successful_prediction_ts": rep.last_successful_prediction_ts(node_id),
+        "breakdown": {
+            "free_prediction_mints": breakdown.free_prediction_mints,
+            "staked_prediction_returns": breakdown.staked_prediction_returns,
+            "staked_prediction_losses": breakdown.staked_prediction_losses,
+            "total": breakdown.total,
+        },
+    }
+
+
+# ─── Bootstrap ──────────────────────────────────────────────────────────
+
+@router.get("/bootstrap/markets/{market_id}")
+def get_bootstrap_market_state(market_id: str = Path(...)) -> dict[str, Any]:
+    """Bootstrap-mode-specific market view: who has voted, who is
+    eligible, current tally."""
+    from services.infonet.bootstrap import (
+        deduplicate_votes,
+        validate_bootstrap_eligibility,
+    )
+
+    chain = _live_chain()
+    canonical = deduplicate_votes(market_id, chain)
+    votes_summary: list[dict[str, Any]] = []
+    yes = 0
+    no = 0
+    for v in canonical:
+        node_id = v.get("node_id") or ""
+        side = (v.get("payload") or {}).get("side")
+        decision = validate_bootstrap_eligibility(node_id, market_id, chain)
+        votes_summary.append({
+            "node_id": node_id,
+            "side": side,
+            "eligible": decision.eligible,
+            "ineligible_reason": decision.reason if not decision.eligible else None,
+        })
+        if decision.eligible:
+            if side == "yes":
+                yes += 1
+            elif side == "no":
+                no += 1
+    total = yes + no
+    return {
+        "ok": True,
+        "market_id": market_id,
+        "votes": votes_summary,
+        "tally": {
+            "yes": yes,
+            "no": no,
+            "total_eligible": total,
+            "min_market_participants": int(CONFIG["min_market_participants"]),
+            "supermajority_threshold": float(CONFIG["bootstrap_resolution_supermajority"]),
+        },
+    }
+
+
+# ─── Signed write: append an Infonet economy event ──────────────────────
+
+@router.post("/append")
+def append_event(body: dict[str, Any] = Body(...)) -> dict[str, Any]:
+    """Append a signed Infonet economy event to the chain.
+
+    Body shape (all required for production):
+
+        {
+            "event_type": str,       # one of INFONET_ECONOMY_EVENT_TYPES
+            "node_id":    str,       # signer
+            "payload":    dict,      # event-specific fields
+            "signature":  str,       # hex
+            "sequence":   int,       # node-monotonic
+            "public_key": str,       # base64
+            "public_key_algo": str,  # "ed25519" or "ecdsa"
+            "protocol_version": str  # optional, defaults to current
+        }
+
+    The cutover-registered validators run automatically via
+    ``mesh_hashchain.Infonet.append`` — payload validation, signature
+    verification, replay protection, sequence ordering, public-key
+    binding, revocation status. No additional security wrapper is
+    needed because ``Infonet.append`` IS the secure entry point.
+
+    Returns the appended event dict on success, or
+    ``{"ok": False, "reason": "..."}`` on validation / signing failure.
+    """
+    if not isinstance(body, dict):
+        return {"ok": False, "reason": "body_must_be_object"}
+
+    event_type = body.get("event_type")
+    if not isinstance(event_type, str) or event_type not in INFONET_ECONOMY_EVENT_TYPES:
+        return {
+            "ok": False,
+            "reason": f"event_type must be one of INFONET_ECONOMY_EVENT_TYPES "
+                      f"(got {event_type!r})",
+        }
+
+    node_id = body.get("node_id")
+    if not isinstance(node_id, str) or not node_id:
+        return {"ok": False, "reason": "node_id required"}
+
+    payload = body.get("payload", {})
+    if not isinstance(payload, dict):
+        return {"ok": False, "reason": "payload must be an object"}
+
+    sequence = body.get("sequence", 0)
+    try:
+        sequence = int(sequence)
+    except (TypeError, ValueError):
+        return {"ok": False, "reason": "sequence must be an integer"}
+    if sequence <= 0:
+        return {"ok": False, "reason": "sequence must be > 0"}
+
+    signature = str(body.get("signature") or "")
+    public_key = str(body.get("public_key") or "")
+    public_key_algo = str(body.get("public_key_algo") or "")
+    protocol_version = str(body.get("protocol_version") or "")
+
+    if not signature or not public_key or not public_key_algo:
+        return {
+            "ok": False,
+            "reason": "signature, public_key, and public_key_algo are required",
+        }
+
+    try:
+        from services.mesh.mesh_hashchain import infonet
+        event = infonet.append(
+            event_type=event_type,
+            node_id=node_id,
+            payload=payload,
+            signature=signature,
+            sequence=sequence,
+            public_key=public_key,
+            public_key_algo=public_key_algo,
+            protocol_version=protocol_version,
+        )
+    except ValueError as exc:
+        # Infonet.append raises ValueError for any validation failure
+        # — payload / signature / replay / sequence / binding. The
+        # message is user-facing per the non-hostile UX rule.
+        return {"ok": False, "reason": str(exc)}
+    except Exception as exc:
+        logger.exception("infonet append failed")
+        return {"ok": False, "reason": f"server_error: {type(exc).__name__}"}
+
+    return {"ok": True, "event": event}
+
+
+# ─── Function Keys (citizen + operator views) ───────────────────────────
+
+@router.get("/function-keys/operator/{operator_id}/batch-summary")
+def operator_batch_summary(operator_id: str = Path(...)) -> dict[str, Any]:
+    """Sprint 11+ scaffolding: returns the operator's local batch
+    counter for the current period. Production wires this through the
+    operator's local-store implementation (Sprint 11+ scaffolding
+    doesn't persist; counts reset per process)."""
+    return {
+        "ok": True,
+        "operator_id": operator_id,
+        "scaffolding_only": True,
+        "note": "Production operators maintain a persistent BatchedSettlementBatch. "
+                "This endpoint reports the in-memory state of the local batch.",
+    }
+
+
+__all__ = ["router"]

backend/routers/intel_feeds.py

@@ -0,0 +1,122 @@
+"""Malware, cyber threats, and country risk feeds."""
+from __future__ import annotations
+
+import logging
+from urllib.parse import urlparse
+
+import requests
+from fastapi import APIRouter, HTTPException, Query, Request
+from fastapi.responses import StreamingResponse
+from starlette.background import BackgroundTask
+
+from limiter import limiter
+from services.fetchers._store import get_latest_data_subset_refs
+from services.fetchers.telegram_osint import telegram_media_host_allowed
+from services.intel_feeds.country_risk import build_country_risk_payload
+from services.network_utils import outbound_user_agent
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/api/malware")
+@limiter.limit("60/minute")
+async def malware_feed(request: Request) -> dict:
+    snap = get_latest_data_subset_refs("malware_threats")
+    payload = snap.get("malware_threats")
+    if isinstance(payload, dict) and payload.get("threats") is not None:
+        return payload
+    return {"threats": [], "total": 0, "timestamp": None, "source": "abuse.ch"}
+
+
+@router.get("/api/cyber-threats")
+@limiter.limit("60/minute")
+async def cyber_threats(request: Request) -> dict:
+    snap = get_latest_data_subset_refs("cyber_threats")
+    return snap.get("cyber_threats") or {"threats": [], "stats": {}}
+
+
+@router.get("/api/country-risk")
+@limiter.limit("30/minute")
+async def country_risk(request: Request) -> dict:
+    return build_country_risk_payload()
+
+
+@router.get("/api/telegram-feed")
+@limiter.limit("30/minute")
+async def telegram_feed(request: Request) -> dict:
+    snap = get_latest_data_subset_refs("telegram_osint")
+    payload = snap.get("telegram_osint")
+    if isinstance(payload, dict) and payload.get("posts") is not None:
+        return payload
+    return {"posts": [], "total": 0, "geolocated": 0, "timestamp": None}
+
+
+def _infer_telegram_media_type(target_url: str, content_type: str) -> str:
+    clean_type = str(content_type or "").split(";", 1)[0].strip().lower()
+    if clean_type and clean_type not in {"application/octet-stream", "binary/octet-stream"}:
+        return content_type
+    path = str(urlparse(target_url).path or "").lower()
+    if path.endswith((".jpg", ".jpeg")):
+        return "image/jpeg"
+    if path.endswith(".png"):
+        return "image/png"
+    if path.endswith(".webp"):
+        return "image/webp"
+    if path.endswith(".gif"):
+        return "image/gif"
+    if path.endswith(".mp4"):
+        return "video/mp4"
+    if path.endswith(".webm"):
+        return "video/webm"
+    return content_type or "application/octet-stream"
+
+
+@router.get("/api/telegram/media")
+@limiter.limit("60/minute")
+async def telegram_media_proxy(request: Request, url: str = Query(...)) -> StreamingResponse:
+    """Stream Telegram CDN media for in-app playback (host allowlist only)."""
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https"):
+        raise HTTPException(status_code=400, detail="Invalid scheme")
+    if not telegram_media_host_allowed(parsed.hostname):
+        raise HTTPException(status_code=403, detail="Host not allowed")
+
+    headers = {
+        "User-Agent": (
+            f"Mozilla/5.0 (compatible; {outbound_user_agent('telegram-media')}) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+        ),
+        "Accept": "*/*",
+    }
+    if range_header := request.headers.get("range"):
+        headers["Range"] = range_header
+
+    try:
+        resp = requests.get(url, stream=True, timeout=(3, 45), headers=headers)
+    except requests.RequestException as exc:
+        logger.warning("Telegram media upstream failure %s: %s", url, exc)
+        raise HTTPException(status_code=502, detail="Upstream fetch failed") from exc
+
+    if resp.status_code >= 400:
+        resp.close()
+        raise HTTPException(status_code=int(resp.status_code), detail=f"Upstream returned {resp.status_code}")
+
+    media_type = _infer_telegram_media_type(url, resp.headers.get("Content-Type", "application/octet-stream"))
+    response_headers = {
+        "Cache-Control": "private, max-age=300",
+        "Accept-Ranges": resp.headers.get("Accept-Ranges", "bytes"),
+    }
+    if content_length := resp.headers.get("Content-Length"):
+        response_headers["Content-Length"] = content_length
+    if content_range := resp.headers.get("Content-Range"):
+        response_headers["Content-Range"] = content_range
+
+    return StreamingResponse(
+        resp.iter_content(chunk_size=65536),
+        status_code=resp.status_code,
+        media_type=media_type,
+        headers=response_headers,
+        background=BackgroundTask(resp.close),
+    )

backend/routers/mesh_dm.py

@@ -0,0 +1,565 @@
+import asyncio
+import hashlib
+import hmac
+import logging
+import secrets
+import time
+from typing import Any
+
+from fastapi import APIRouter, Depends, Request
+from fastapi.responses import JSONResponse
+
+from auth import (
+    _is_debug_test_request,
+    _scoped_view_authenticated,
+    _verify_peer_push_hmac,
+    require_admin,
+)
+from limiter import limiter
+from services.config import get_settings
+from services.mesh.mesh_compatibility import (
+    LEGACY_AGENT_ID_LOOKUP_TARGET,
+    legacy_agent_id_lookup_blocked,
+    record_legacy_agent_id_lookup,
+    sunset_target_label,
+)
+from services.mesh.mesh_signed_events import (
+    MeshWriteExemption,
+    SignedWriteKind,
+    get_prepared_signed_write,
+    mesh_write_exempt,
+    requires_signed_write,
+)
+
+logger = logging.getLogger(__name__)
+_WARNED_LEGACY_DM_PUBKEY_LOOKUPS: set[str] = set()
+
+router = APIRouter()
+
+
+# ---------------------------------------------------------------------------
+# Local helpers
+# ---------------------------------------------------------------------------
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _warn_legacy_dm_pubkey_lookup(agent_id: str) -> None:
+    peer_id = str(agent_id or "").strip().lower()
+    if not peer_id or peer_id in _WARNED_LEGACY_DM_PUBKEY_LOOKUPS:
+        return
+    _WARNED_LEGACY_DM_PUBKEY_LOOKUPS.add(peer_id)
+    logger.warning(
+        "mesh legacy DH pubkey lookup used for %s via direct agent_id; prefer invite-scoped lookup handles before removal in %s",
+        peer_id,
+        sunset_target_label(LEGACY_AGENT_ID_LOOKUP_TARGET),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Transition delegates: forward to main.py so test monkeypatches still work.
+# These will move to a shared module once main.py routes are removed.
+# ---------------------------------------------------------------------------
+def _main_delegate(name):
+    def _wrapper(*a, **kw):
+        import main as _m
+        return getattr(_m, name)(*a, **kw)
+    _wrapper.__name__ = name
+    return _wrapper
+
+
+_verify_signed_write = _main_delegate("_verify_signed_write")
+_secure_dm_enabled = _main_delegate("_secure_dm_enabled")
+_legacy_dm_get_allowed = _main_delegate("_legacy_dm_get_allowed")
+_rns_private_dm_ready = _main_delegate("_rns_private_dm_ready")
+_anonymous_dm_hidden_transport_enforced = _main_delegate("_anonymous_dm_hidden_transport_enforced")
+_high_privacy_profile_enabled = _main_delegate("_high_privacy_profile_enabled")
+_dm_send_from_signed_request = _main_delegate("_dm_send_from_signed_request")
+_dm_poll_secure_from_signed_request = _main_delegate("_dm_poll_secure_from_signed_request")
+_dm_count_secure_from_signed_request = _main_delegate("_dm_count_secure_from_signed_request")
+_validate_private_signed_sequence = _main_delegate("_validate_private_signed_sequence")
+
+
+def _signed_body(request: Request) -> dict[str, Any]:
+    prepared = get_prepared_signed_write(request)
+    if prepared is None:
+        return {}
+    return dict(prepared.body)
+
+
+async def _maybe_apply_dm_relay_jitter() -> None:
+    if not _high_privacy_profile_enabled():
+        return
+    await asyncio.sleep((50 + secrets.randbelow(451)) / 1000.0)
+
+
+_REQUEST_V2_REDUCED_VERSION = "request-v2-reduced-v3"
+_REQUEST_V2_RECOVERY_STATES = {"pending", "verified", "failed"}
+
+
+def _is_canonical_reduced_request_message(message: dict[str, Any]) -> bool:
+    item = dict(message or {})
+    return (
+        str(item.get("delivery_class", "") or "").strip().lower() == "request"
+        and str(item.get("request_contract_version", "") or "").strip()
+        == _REQUEST_V2_REDUCED_VERSION
+        and item.get("sender_recovery_required") is True
+    )
+
+
+def _annotate_request_recovery_message(message: dict[str, Any]) -> dict[str, Any]:
+    item = dict(message or {})
+    delivery_class = str(item.get("delivery_class", "") or "").strip().lower()
+    sender_id = str(item.get("sender_id", "") or "").strip()
+    sender_seal = str(item.get("sender_seal", "") or "").strip()
+    sender_is_blinded = sender_id.startswith("sealed:") or sender_id.startswith("sender_token:")
+    if delivery_class != "request" or not sender_is_blinded or not sender_seal.startswith("v3:"):
+        return item
+    if not str(item.get("request_contract_version", "") or "").strip():
+        item["request_contract_version"] = _REQUEST_V2_REDUCED_VERSION
+    item["sender_recovery_required"] = True
+    state = str(item.get("sender_recovery_state", "") or "").strip().lower()
+    if state not in _REQUEST_V2_RECOVERY_STATES:
+        state = "pending"
+    item["sender_recovery_state"] = state
+    return item
+
+
+def _annotate_request_recovery_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    return [_annotate_request_recovery_message(message) for message in (messages or [])]
+
+
+def _request_duplicate_authority_rank(message: dict[str, Any]) -> int:
+    item = dict(message or {})
+    if str(item.get("delivery_class", "") or "").strip().lower() != "request":
+        return 0
+    if _is_canonical_reduced_request_message(item):
+        return 3
+    sender_id = str(item.get("sender_id", "") or "").strip()
+    if sender_id.startswith("sealed:") or sender_id.startswith("sender_token:"):
+        return 1
+    if sender_id:
+        return 2
+    return 0
+
+
+def _request_duplicate_recovery_rank(message: dict[str, Any]) -> int:
+    if not _is_canonical_reduced_request_message(message):
+        return 0
+    state = str(dict(message or {}).get("sender_recovery_state", "") or "").strip().lower()
+    if state == "verified":
+        return 2
+    if state == "pending":
+        return 1
+    return 0
+
+
+def _poll_duplicate_source_rank(source: str) -> int:
+    normalized = str(source or "").strip().lower()
+    if normalized == "relay":
+        return 2
+    if normalized == "reticulum":
+        return 1
+    return 0
+
+
+def _should_replace_dm_poll_duplicate(
+    existing: dict[str, Any],
+    existing_source: str,
+    candidate: dict[str, Any],
+    candidate_source: str,
+) -> bool:
+    candidate_authority = _request_duplicate_authority_rank(candidate)
+    existing_authority = _request_duplicate_authority_rank(existing)
+    if candidate_authority != existing_authority:
+        return candidate_authority > existing_authority
+
+    candidate_recovery = _request_duplicate_recovery_rank(candidate)
+    existing_recovery = _request_duplicate_recovery_rank(existing)
+    if candidate_recovery != existing_recovery:
+        return candidate_recovery > existing_recovery
+
+    candidate_source_rank = _poll_duplicate_source_rank(candidate_source)
+    existing_source_rank = _poll_duplicate_source_rank(existing_source)
+    if candidate_source_rank != existing_source_rank:
+        return candidate_source_rank > existing_source_rank
+
+    try:
+        candidate_ts = float(candidate.get("timestamp", 0) or 0)
+    except Exception:
+        candidate_ts = 0.0
+    try:
+        existing_ts = float(existing.get("timestamp", 0) or 0)
+    except Exception:
+        existing_ts = 0.0
+    return candidate_ts > existing_ts
+
+
+def _merge_dm_poll_messages(
+    relay_messages: list[dict[str, Any]],
+    direct_messages: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    merged: list[dict[str, Any]] = []
+    index_by_msg_id: dict[str, tuple[int, str]] = {}
+
+    def add_messages(items: list[dict[str, Any]], source: str) -> None:
+        for original in items or []:
+            item = dict(original or {})
+            msg_id = str(item.get("msg_id", "") or "").strip()
+            if not msg_id:
+                merged.append(item)
+                continue
+            existing = index_by_msg_id.get(msg_id)
+            if existing is None:
+                index_by_msg_id[msg_id] = (len(merged), source)
+                merged.append(item)
+                continue
+            index, existing_source = existing
+            if _should_replace_dm_poll_duplicate(merged[index], existing_source, item, source):
+                merged[index] = item
+                index_by_msg_id[msg_id] = (index, source)
+
+    add_messages(relay_messages, "relay")
+    add_messages(direct_messages, "reticulum")
+    return sorted(merged, key=lambda item: float(item.get("timestamp", 0) or 0))
+
+
+# ---------------------------------------------------------------------------
+# Route handlers
+# ---------------------------------------------------------------------------
+
+@router.post("/api/mesh/dm/register")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_REGISTER)
+async def dm_register_key(request: Request):
+    """Register a DH public key for encrypted DM key exchange."""
+    body = _signed_body(request)
+    agent_id = body.get("agent_id", "").strip()
+    dh_pub_key = body.get("dh_pub_key", "").strip()
+    dh_algo = body.get("dh_algo", "").strip()
+    timestamp = _safe_int(body.get("timestamp", 0) or 0)
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not agent_id or not dh_pub_key or not dh_algo or not timestamp:
+        return {"ok": False, "detail": "Missing agent_id, dh_pub_key, dh_algo, or timestamp"}
+    if dh_algo.upper() not in ("X25519", "ECDH_P256", "ECDH"):
+        return {"ok": False, "detail": "Unsupported dh_algo"}
+    now_ts = int(time.time())
+    if abs(timestamp - now_ts) > 7 * 86400:
+        return {"ok": False, "detail": "DH key timestamp is too far from current time"}
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+
+        reputation_ledger.register_node(agent_id, public_key, public_key_algo)
+    except Exception:
+        pass
+
+    accepted, detail, metadata = dm_relay.register_dh_key(
+        agent_id,
+        dh_pub_key,
+        dh_algo,
+        timestamp,
+        signature,
+        public_key,
+        public_key_algo,
+        protocol_version,
+        sequence,
+    )
+    if not accepted:
+        return {"ok": False, "detail": detail}
+
+    return {"ok": True, **(metadata or {})}
+
+
+@router.get("/api/mesh/dm/pubkey")
+@limiter.limit("30/minute")
+async def dm_get_pubkey(request: Request, agent_id: str = "", lookup_token: str = ""):
+    import main as _m
+
+    return await _m.dm_get_pubkey(request, agent_id=agent_id, lookup_token=lookup_token)
+
+
+@router.get("/api/mesh/dm/prekey-bundle")
+@limiter.limit("30/minute")
+async def dm_get_prekey_bundle(request: Request, agent_id: str = "", lookup_token: str = ""):
+    import main as _m
+
+    return await _m.dm_get_prekey_bundle(request, agent_id=agent_id, lookup_token=lookup_token)
+
+
+@router.post("/api/mesh/dm/prekey-peer-lookup")
+@limiter.limit("60/minute")
+@mesh_write_exempt(MeshWriteExemption.PEER_GOSSIP)
+async def dm_prekey_peer_lookup(request: Request):
+    """Peer-authenticated invite lookup handle resolution.
+
+    This endpoint exists for private/bootstrap peers to import signed invites
+    without exposing a stable agent_id on the ordinary lookup surface. It only
+    accepts HMAC-authenticated peer calls and only resolves lookup_token.
+    """
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 4096:
+                return JSONResponse(
+                    status_code=413,
+                    content={"ok": False, "detail": "Request body too large"},
+                )
+        except (TypeError, ValueError):
+            pass
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return JSONResponse(
+            status_code=403,
+            content={"ok": False, "detail": "Invalid or missing peer HMAC"},
+        )
+    try:
+        import json
+
+        body = json.loads(body_bytes or b"{}")
+    except Exception:
+        return {"ok": False, "detail": "invalid json"}
+    lookup_token = str(dict(body or {}).get("lookup_token", "") or "").strip()
+    if not lookup_token:
+        return {"ok": False, "detail": "lookup_token required"}
+    from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle
+
+    result = fetch_dm_prekey_bundle(
+        agent_id="",
+        lookup_token=lookup_token,
+        allow_peer_lookup=False,
+    )
+    if not result.get("ok"):
+        return {"ok": False, "detail": str(result.get("detail", "") or "Prekey bundle not found")}
+    safe = dict(result)
+    safe.pop("resolved_agent_id", None)
+    safe["lookup_mode"] = "invite_lookup_handle"
+    return safe
+
+
+@router.post("/api/mesh/dm/send")
+@limiter.limit("20/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_SEND)
+async def dm_send(request: Request):
+    return await _dm_send_from_signed_request(request)
+
+
+@router.post("/api/mesh/dm/poll")
+@limiter.limit("30/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_POLL)
+async def dm_poll_secure(request: Request):
+    return await _dm_poll_secure_from_signed_request(request)
+
+
+@router.get("/api/mesh/dm/poll")
+@limiter.limit("30/minute")
+async def dm_poll(
+    request: Request,
+    agent_id: str = "",
+    agent_token: str = "",
+    agent_token_prev: str = "",
+    agent_tokens: str = "",
+):
+    import main as _m
+
+    return await _m.dm_poll(
+        request,
+        agent_id=agent_id,
+        agent_token=agent_token,
+        agent_token_prev=agent_token_prev,
+        agent_tokens=agent_tokens,
+    )
+
+
+@router.post("/api/mesh/dm/count")
+@limiter.limit("60/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_COUNT)
+async def dm_count_secure(request: Request):
+    return await _dm_count_secure_from_signed_request(request)
+
+
+@router.get("/api/mesh/dm/count")
+@limiter.limit("60/minute")
+async def dm_count(
+    request: Request,
+    agent_id: str = "",
+    agent_token: str = "",
+    agent_token_prev: str = "",
+    agent_tokens: str = "",
+):
+    import main as _m
+
+    return await _m.dm_count(
+        request,
+        agent_id=agent_id,
+        agent_token=agent_token,
+        agent_token_prev=agent_token_prev,
+        agent_tokens=agent_tokens,
+    )
+
+
+@router.post("/api/mesh/dm/block")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_BLOCK)
+async def dm_block(request: Request):
+    """Block or unblock a sender from DMing you."""
+    body = _signed_body(request)
+    agent_id = body.get("agent_id", "").strip()
+    blocked_id = body.get("blocked_id", "").strip()
+    action = body.get("action", "block").strip().lower()
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not agent_id or not blocked_id:
+        return {"ok": False, "detail": "Missing agent_id or blocked_id"}
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    try:
+        from services.mesh.mesh_hashchain import infonet
+
+        ok_seq, seq_reason = _validate_private_signed_sequence(
+            infonet,
+            agent_id,
+            sequence,
+            domain=f"dm_block:{action}",
+        )
+        if not ok_seq:
+            return {"ok": False, "detail": seq_reason}
+    except Exception:
+        pass
+
+    if action == "unblock":
+        dm_relay.unblock(agent_id, blocked_id)
+    else:
+        dm_relay.block(agent_id, blocked_id)
+    return {"ok": True, "action": action, "blocked_id": blocked_id}
+
+
+@router.post("/api/mesh/dm/witness")
+@limiter.limit("20/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_WITNESS)
+async def dm_key_witness(request: Request):
+    """Record a lightweight witness for a DM key (dual-path spot-check)."""
+    body = _signed_body(request)
+    witness_id = body.get("witness_id", "").strip()
+    target_id = body.get("target_id", "").strip()
+    dh_pub_key = body.get("dh_pub_key", "").strip()
+    timestamp = _safe_int(body.get("timestamp", 0) or 0)
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not witness_id or not target_id or not dh_pub_key or not timestamp:
+        return {"ok": False, "detail": "Missing witness_id, target_id, dh_pub_key, or timestamp"}
+    now_ts = int(time.time())
+    if abs(timestamp - now_ts) > 7 * 86400:
+        return {"ok": False, "detail": "Witness timestamp is too far from current time"}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+
+        reputation_ledger.register_node(witness_id, public_key, public_key_algo)
+    except Exception:
+        pass
+    try:
+        from services.mesh.mesh_hashchain import infonet
+
+        ok_seq, seq_reason = _validate_private_signed_sequence(
+            infonet,
+            witness_id,
+            sequence,
+            domain="dm_witness",
+        )
+        if not ok_seq:
+            return {"ok": False, "detail": seq_reason}
+    except Exception:
+        pass
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    ok, reason = dm_relay.record_witness(witness_id, target_id, dh_pub_key, timestamp)
+    return {"ok": ok, "detail": reason}
+
+
+@router.get("/api/mesh/dm/witness")
+@limiter.limit("60/minute")
+async def dm_key_witness_get(request: Request, target_id: str = "", dh_pub_key: str = ""):
+    """Get witness counts for a target's DH key."""
+    if not target_id:
+        return {"ok": False, "detail": "Missing target_id"}
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    witnesses = dm_relay.get_witnesses(target_id, dh_pub_key if dh_pub_key else None, limit=5)
+    response = {
+        "ok": True,
+        "count": len(witnesses),
+    }
+    if _scoped_view_authenticated(request, "mesh.audit"):
+        response["target_id"] = target_id
+        response["dh_pub_key"] = dh_pub_key or ""
+        response["witnesses"] = witnesses
+    return response
+
+
+@router.post("/api/mesh/trust/vouch")
+@limiter.limit("20/minute")
+@requires_signed_write(kind=SignedWriteKind.TRUST_VOUCH)
+async def trust_vouch(request: Request):
+    """Record a trust vouch for a node (web-of-trust signal)."""
+    body = _signed_body(request)
+    voucher_id = body.get("voucher_id", "").strip()
+    target_id = body.get("target_id", "").strip()
+    note = body.get("note", "").strip()
+    timestamp = _safe_int(body.get("timestamp", 0) or 0)
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not voucher_id or not target_id or not timestamp:
+        return {"ok": False, "detail": "Missing voucher_id, target_id, or timestamp"}
+    now_ts = int(time.time())
+    if abs(timestamp - now_ts) > 7 * 86400:
+        return {"ok": False, "detail": "Vouch timestamp is too far from current time"}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+        from services.mesh.mesh_hashchain import infonet
+
+        reputation_ledger.register_node(voucher_id, public_key, public_key_algo)
+        ok_seq, seq_reason = _validate_private_signed_sequence(
+            infonet,
+            voucher_id,
+            sequence,
+            domain="trust_vouch",
+        )
+        if not ok_seq:
+            return {"ok": False, "detail": seq_reason}
+        ok, reason = reputation_ledger.add_vouch(voucher_id, target_id, note, timestamp)
+        return {"ok": ok, "detail": reason}
+    except Exception:
+        return {"ok": False, "detail": "Failed to record vouch"}
+
+
+@router.get("/api/mesh/trust/vouches", dependencies=[Depends(require_admin)])
+@limiter.limit("60/minute")
+async def trust_vouches(request: Request, node_id: str = "", limit: int = 20):
+    """Fetch latest vouches for a node."""
+    if not node_id:
+        return {"ok": False, "detail": "Missing node_id"}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+
+        vouches = reputation_ledger.get_vouches(node_id, limit=limit)
+        return {"ok": True, "node_id": node_id, "vouches": vouches, "count": len(vouches)}
+    except Exception:
+        return {"ok": False, "detail": "Failed to fetch vouches"}

backend/routers/mesh_operator.py

@@ -0,0 +1,145 @@
+import time
+import logging
+from fastapi import APIRouter, Request, Response, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def list_peers(request: Request, bucket: str = Query(None)):
+    """List all peers (or filter by bucket: sync, push, bootstrap)."""
+    from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerStore
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception as exc:
+        return {"ok": False, "detail": f"Failed to load peer store: {exc}"}
+    if bucket:
+        records = store.records_for_bucket(bucket)
+    else:
+        records = store.records()
+    return {"ok": True, "count": len(records), "peers": [r.to_dict() for r in records]}
+
+
+@router.post("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def add_peer(request: Request):
+    """Add a peer to the store. Body: {peer_url, transport?, label?, role?, buckets?[]}."""
+    from services.mesh.mesh_crypto import normalize_peer_url
+    from services.mesh.mesh_peer_store import (
+        DEFAULT_PEER_STORE_PATH, PeerStore, PeerStoreError,
+        make_push_peer_record, make_sync_peer_record,
+    )
+    from services.mesh.mesh_router import peer_transport_kind
+    body = await request.json()
+    peer_url_raw = str(body.get("peer_url", "") or "").strip()
+    if not peer_url_raw:
+        return {"ok": False, "detail": "peer_url is required"}
+    peer_url = normalize_peer_url(peer_url_raw)
+    if not peer_url:
+        return {"ok": False, "detail": "Invalid peer_url"}
+    transport = str(body.get("transport", "") or "").strip().lower()
+    if not transport:
+        transport = peer_transport_kind(peer_url)
+    if not transport:
+        return {"ok": False, "detail": "Cannot determine transport for peer_url — provide transport explicitly"}
+    label = str(body.get("label", "") or "").strip()
+    role = str(body.get("role", "") or "").strip().lower() or "relay"
+    buckets = body.get("buckets", ["sync", "push"])
+    if isinstance(buckets, str):
+        buckets = [buckets]
+    if not isinstance(buckets, list):
+        buckets = ["sync", "push"]
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    added: list = []
+    try:
+        for b in buckets:
+            b = str(b).strip().lower()
+            if b == "sync":
+                store.upsert(make_sync_peer_record(peer_url=peer_url, transport=transport, role=role, label=label))
+                added.append("sync")
+            elif b == "push":
+                store.upsert(make_push_peer_record(peer_url=peer_url, transport=transport, role=role, label=label))
+                added.append("push")
+        store.save()
+    except PeerStoreError as exc:
+        return {"ok": False, "detail": str(exc)}
+    return {"ok": True, "peer_url": peer_url, "buckets": added}
+
+
+@router.delete("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def remove_peer(request: Request):
+    """Remove a peer. Body: {peer_url, bucket?}. If bucket omitted, removes from all buckets."""
+    from services.mesh.mesh_crypto import normalize_peer_url
+    from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerStore
+    body = await request.json()
+    peer_url_raw = str(body.get("peer_url", "") or "").strip()
+    if not peer_url_raw:
+        return {"ok": False, "detail": "peer_url is required"}
+    peer_url = normalize_peer_url(peer_url_raw)
+    if not peer_url:
+        return {"ok": False, "detail": "Invalid peer_url"}
+    bucket_filter = str(body.get("bucket", "") or "").strip().lower()
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        return {"ok": False, "detail": "Failed to load peer store"}
+    removed: list = []
+    for b in ["bootstrap", "sync", "push"]:
+        if bucket_filter and b != bucket_filter:
+            continue
+        key = f"{b}:{peer_url}"
+        if key in store._records:
+            del store._records[key]
+            removed.append(b)
+    if not removed:
+        return {"ok": False, "detail": "Peer not found in any bucket"}
+    store.save()
+    return {"ok": True, "peer_url": peer_url, "removed_from": removed}
+
+
+@router.patch("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def toggle_peer(request: Request):
+    """Enable or disable a peer. Body: {peer_url, bucket, enabled: bool}."""
+    from services.mesh.mesh_crypto import normalize_peer_url
+    from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerRecord, PeerStore
+    body = await request.json()
+    peer_url_raw = str(body.get("peer_url", "") or "").strip()
+    bucket = str(body.get("bucket", "") or "").strip().lower()
+    enabled = body.get("enabled")
+    if not peer_url_raw:
+        return {"ok": False, "detail": "peer_url is required"}
+    if not bucket:
+        return {"ok": False, "detail": "bucket is required"}
+    if enabled is None:
+        return {"ok": False, "detail": "enabled (true/false) is required"}
+    peer_url = normalize_peer_url(peer_url_raw)
+    if not peer_url:
+        return {"ok": False, "detail": "Invalid peer_url"}
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        return {"ok": False, "detail": "Failed to load peer store"}
+    key = f"{bucket}:{peer_url}"
+    record = store._records.get(key)
+    if not record:
+        return {"ok": False, "detail": f"Peer not found in {bucket} bucket"}
+    updated = PeerRecord(**{**record.to_dict(), "enabled": bool(enabled), "updated_at": int(time.time())})
+    store._records[key] = updated
+    store.save()
+    return {"ok": True, "peer_url": peer_url, "bucket": bucket, "enabled": bool(enabled)}

backend/routers/mesh_oracle.py

@@ -0,0 +1,354 @@
+import math
+from typing import Any
+from fastapi import APIRouter, Request, Response, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator, _scoped_view_authenticated
+from services.data_fetcher import get_latest_data
+from services.mesh.mesh_protocol import normalize_payload
+from services.mesh.mesh_signed_events import (
+    MeshWriteExemption,
+    SignedWriteKind,
+    get_prepared_signed_write,
+    mesh_write_exempt,
+    requires_signed_write,
+)
+
+router = APIRouter()
+
+
+def _signed_body(request: Request) -> dict[str, Any]:
+    prepared = get_prepared_signed_write(request)
+    if prepared is None:
+        return {}
+    return dict(prepared.body)
+
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _safe_float(val, default=0.0):
+    try:
+        parsed = float(val)
+        if not math.isfinite(parsed):
+            return default
+        return parsed
+    except (TypeError, ValueError):
+        return default
+
+
+def _redact_public_oracle_profile(payload: dict, authenticated: bool) -> dict:
+    redacted = dict(payload)
+    if authenticated:
+        return redacted
+    redacted["active_stakes"] = []
+    redacted["prediction_history"] = []
+    return redacted
+
+
+def _redact_public_oracle_predictions(predictions: list, authenticated: bool) -> dict:
+    if authenticated:
+        return {"predictions": list(predictions)}
+    return {"predictions": [], "count": len(predictions)}
+
+
+def _redact_public_oracle_stakes(payload: dict, authenticated: bool) -> dict:
+    redacted = dict(payload)
+    if authenticated:
+        return redacted
+    redacted["truth_stakers"] = []
+    redacted["false_stakers"] = []
+    return redacted
+
+
+@router.post("/api/mesh/oracle/predict")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.ORACLE_PREDICT)
+async def oracle_predict(request: Request):
+    """Place a prediction on a market outcome."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    body = _signed_body(request)
+    node_id = body.get("node_id", "")
+    market_title = body.get("market_title", "")
+    side = body.get("side", "")
+    stake_amount = _safe_float(body.get("stake_amount", 0))
+    public_key = body.get("public_key", "")
+    public_key_algo = body.get("public_key_algo", "")
+    signature = body.get("signature", "")
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "")
+    if not node_id or not market_title or not side:
+        return {"ok": False, "detail": "Missing node_id, market_title, or side"}
+    prediction_payload = {"market_title": market_title, "side": side, "stake_amount": stake_amount}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+        reputation_ledger.register_node(node_id, public_key, public_key_algo)
+    except Exception:
+        pass
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    matched = None
+    for m in markets:
+        if m.get("title", "").lower() == market_title.lower():
+            matched = m
+            break
+    if not matched:
+        for m in markets:
+            if market_title.lower() in m.get("title", "").lower():
+                matched = m
+                break
+    if not matched:
+        return {"ok": False, "detail": f"Market '{market_title}' not found in active markets."}
+    probability = 50.0
+    side_lower = side.lower()
+    outcomes = matched.get("outcomes", [])
+    if outcomes:
+        for o in outcomes:
+            if o.get("name", "").lower() == side_lower:
+                probability = float(o.get("pct", 50))
+                break
+    else:
+        consensus = matched.get("consensus_pct")
+        if consensus is None:
+            consensus = matched.get("polymarket_pct") or matched.get("kalshi_pct") or 50
+        probability = float(consensus)
+        if side_lower == "no":
+            probability = 100.0 - probability
+    if stake_amount > 0:
+        ok, detail = oracle_ledger.place_market_stake(node_id, matched["title"], side, stake_amount, probability)
+        mode = "staked"
+    else:
+        ok, detail = oracle_ledger.place_prediction(node_id, matched["title"], side, probability)
+        mode = "free"
+    if ok:
+        try:
+            from services.mesh.mesh_hashchain import infonet
+            normalized_payload = normalize_payload("prediction", prediction_payload)
+            infonet.append(event_type="prediction", node_id=node_id, payload=normalized_payload,
+                signature=signature, sequence=sequence, public_key=public_key,
+                public_key_algo=public_key_algo, protocol_version=protocol_version)
+        except Exception:
+            pass
+    return {"ok": ok, "detail": detail, "probability": probability, "mode": mode}
+
+
+@router.get("/api/mesh/oracle/markets")
+@limiter.limit("30/minute")
+async def oracle_markets(request: Request):
+    """List active prediction markets."""
+    from collections import defaultdict
+    from services.mesh.mesh_oracle import oracle_ledger
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    all_consensus = oracle_ledger.get_all_market_consensus()
+    by_category = defaultdict(list)
+    for m in markets:
+        by_category[m.get("category", "NEWS")].append(m)
+    _fields = ("title", "consensus_pct", "polymarket_pct", "kalshi_pct", "volume", "volume_24h",
+               "end_date", "description", "category", "sources", "slug", "kalshi_ticker", "outcomes")
+    categories = {}
+    cat_totals = {}
+    for cat in ["POLITICS", "CONFLICT", "NEWS", "FINANCE", "CRYPTO"]:
+        all_cat = sorted(by_category.get(cat, []), key=lambda x: x.get("volume", 0) or 0, reverse=True)
+        cat_totals[cat] = len(all_cat)
+        cat_list = []
+        for m in all_cat[:10]:
+            entry = {k: m.get(k) for k in _fields}
+            entry["consensus"] = all_consensus.get(m.get("title", ""), {})
+            cat_list.append(entry)
+        categories[cat] = cat_list
+    return {"categories": categories, "total_count": len(markets), "cat_totals": cat_totals}
+
+
+@router.get("/api/mesh/oracle/search")
+@limiter.limit("20/minute")
+async def oracle_search(request: Request, q: str = "", limit: int = 50):
+    """Search prediction markets across Polymarket + Kalshi APIs."""
+    if not q or len(q) < 2:
+        return {"results": [], "query": q, "count": 0}
+    from services.fetchers.prediction_markets import search_polymarket_direct, search_kalshi_direct
+    import concurrent.futures
+    # Search both APIs in parallel for speed
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
+        poly_fut = pool.submit(search_polymarket_direct, q, limit)
+        kalshi_fut = pool.submit(search_kalshi_direct, q, limit)
+        poly_results = poly_fut.result(timeout=20)
+        kalshi_results = kalshi_fut.result(timeout=20)
+    # Also check cached/merged markets
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    q_lower = q.lower()
+    cached_matches = [m for m in markets if q_lower in m.get("title", "").lower()]
+    seen_titles = set()
+    combined = []
+    # Cached first (already merged Poly+Kalshi with consensus)
+    for m in cached_matches:
+        seen_titles.add(m["title"].lower())
+        combined.append(m)
+    # Then Polymarket direct hits
+    for m in poly_results:
+        if m["title"].lower() not in seen_titles:
+            seen_titles.add(m["title"].lower())
+            combined.append(m)
+    # Then Kalshi direct hits
+    for m in kalshi_results:
+        if m["title"].lower() not in seen_titles:
+            seen_titles.add(m["title"].lower())
+            combined.append(m)
+    combined.sort(key=lambda x: x.get("volume", 0) or 0, reverse=True)
+    _fields = ("title", "consensus_pct", "polymarket_pct", "kalshi_pct", "volume", "volume_24h",
+               "end_date", "description", "category", "sources", "slug", "kalshi_ticker", "outcomes")
+    results = [{k: m.get(k) for k in _fields} for m in combined[:limit]]
+    return {"results": results, "query": q, "count": len(results)}
+
+
+@router.get("/api/mesh/oracle/markets/more")
+@limiter.limit("30/minute")
+async def oracle_markets_more(request: Request, category: str = "NEWS", offset: int = 0, limit: int = 10):
+    """Load more markets for a specific category (paginated)."""
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    cat_markets = sorted([m for m in markets if m.get("category") == category],
+        key=lambda x: x.get("volume", 0) or 0, reverse=True)
+    page = cat_markets[offset : offset + limit]
+    _fields = ("title", "consensus_pct", "polymarket_pct", "kalshi_pct", "volume", "volume_24h",
+               "end_date", "description", "category", "sources", "slug", "kalshi_ticker", "outcomes")
+    results = [{k: m.get(k) for k in _fields} for m in page]
+    return {"markets": results, "category": category, "offset": offset,
+            "has_more": offset + limit < len(cat_markets), "total": len(cat_markets)}
+
+
+@router.post(
+    "/api/mesh/oracle/resolve",
+    dependencies=[Depends(require_admin)],
+)
+@limiter.limit("5/minute")
+@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
+async def oracle_resolve(request: Request):
+    """Resolve a prediction market.
+
+    Issue #240 (tg12): requires admin authentication. The
+    ``mesh_write_exempt`` decorator below is **metadata only** — it tags
+    the route as not requiring a mesh signed-write envelope, it does
+    NOT itself enforce caller authorization. The ``Depends(require_admin)``
+    on the route decorator is what actually gates access.
+    """
+    from services.mesh.mesh_oracle import oracle_ledger
+    body = await request.json()
+    market_title = body.get("market_title", "")
+    outcome = body.get("outcome", "")
+    if not market_title or not outcome:
+        return {"ok": False, "detail": "Need market_title and outcome"}
+    winners, losers = oracle_ledger.resolve_market(market_title, outcome)
+    stake_result = oracle_ledger.resolve_market_stakes(market_title, outcome)
+    return {"ok": True,
+            "detail": f"Resolved: {winners} free winners, {losers} free losers, "
+                      f"{stake_result.get('winners', 0)} stake winners, {stake_result.get('losers', 0)} stake losers",
+            "free": {"winners": winners, "losers": losers}, "stakes": stake_result}
+
+
+@router.get("/api/mesh/oracle/consensus")
+@limiter.limit("30/minute")
+async def oracle_consensus(request: Request, market_title: str = ""):
+    """Get network consensus for a market."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    if not market_title:
+        return {"error": "market_title required"}
+    return oracle_ledger.get_market_consensus(market_title)
+
+
+@router.post("/api/mesh/oracle/stake")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.ORACLE_STAKE)
+async def oracle_stake(request: Request):
+    """Stake oracle rep on a post's truthfulness."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    body = _signed_body(request)
+    staker_id = body.get("staker_id", "")
+    message_id = body.get("message_id", "")
+    poster_id = body.get("poster_id", "")
+    side = body.get("side", "").lower()
+    amount = _safe_float(body.get("amount", 0))
+    duration_days = _safe_int(body.get("duration_days", 1), 1)
+    public_key = body.get("public_key", "")
+    public_key_algo = body.get("public_key_algo", "")
+    signature = body.get("signature", "")
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "")
+    if not staker_id or not message_id or not side:
+        return {"ok": False, "detail": "Missing staker_id, message_id, or side"}
+    stake_payload = {"message_id": message_id, "poster_id": poster_id, "side": side,
+                     "amount": amount, "duration_days": duration_days}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+        reputation_ledger.register_node(staker_id, public_key, public_key_algo)
+    except Exception:
+        pass
+    ok, detail = oracle_ledger.place_stake(staker_id, message_id, poster_id, side, amount, duration_days)
+    if ok:
+        try:
+            from services.mesh.mesh_hashchain import infonet
+            normalized_payload = normalize_payload("stake", stake_payload)
+            infonet.append(event_type="stake", node_id=staker_id, payload=normalized_payload,
+                signature=signature, sequence=sequence, public_key=public_key,
+                public_key_algo=public_key_algo, protocol_version=protocol_version)
+        except Exception:
+            pass
+    return {"ok": ok, "detail": detail}
+
+
+@router.get("/api/mesh/oracle/stakes/{message_id}")
+@limiter.limit("30/minute")
+async def oracle_stakes_for_message(request: Request, message_id: str):
+    """Get all oracle stakes on a message."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    return _redact_public_oracle_stakes(
+        oracle_ledger.get_stakes_for_message(message_id),
+        authenticated=_scoped_view_authenticated(request, "mesh.audit"),
+    )
+
+
+@router.get("/api/mesh/oracle/profile")
+@limiter.limit("30/minute")
+async def oracle_profile(request: Request, node_id: str = ""):
+    """Get full oracle profile."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    if not node_id:
+        return {"ok": False, "detail": "Provide ?node_id=xxx"}
+    profile = oracle_ledger.get_oracle_profile(node_id)
+    return _redact_public_oracle_profile(
+        profile, authenticated=_scoped_view_authenticated(request, "mesh.audit"))
+
+
+@router.get("/api/mesh/oracle/predictions")
+@limiter.limit("30/minute")
+async def oracle_predictions(request: Request, node_id: str = ""):
+    """Get a node's active (unresolved) predictions."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    if not node_id:
+        return {"ok": False, "detail": "Provide ?node_id=xxx"}
+    active_predictions = oracle_ledger.get_active_predictions(node_id)
+    return _redact_public_oracle_predictions(
+        active_predictions, authenticated=_scoped_view_authenticated(request, "mesh.audit"))
+
+
+@router.post(
+    "/api/mesh/oracle/resolve-stakes",
+    dependencies=[Depends(require_admin)],
+)
+@limiter.limit("5/minute")
+@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
+async def oracle_resolve_stakes(request: Request):
+    """Resolve all expired stake contests.
+
+    Issue #241 (tg12): requires admin authentication. See the note on
+    ``oracle_resolve`` above — ``mesh_write_exempt`` is metadata only.
+    """
+    from services.mesh.mesh_oracle import oracle_ledger
+    resolutions = oracle_ledger.resolve_expired_stakes()
+    return {"ok": True, "resolutions": resolutions, "count": len(resolutions)}

backend/routers/mesh_peer_sync.py

@@ -0,0 +1,300 @@
+import json as json_mod
+import logging
+from typing import Any
+from fastapi import APIRouter, Request, Response
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator, _verify_peer_push_hmac
+from services.config import get_settings
+from services.mesh.mesh_crypto import normalize_peer_url
+from services.mesh.mesh_router import peer_transport_kind
+from auth import _peer_hmac_url_from_request
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+_PEER_PUSH_BATCH_SIZE = 50
+
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _hydrate_gate_store_from_chain(events: list) -> int:
+    """Copy any gate_message chain events into the local gate_store for read/decrypt.
+
+    Only events that are resident in the local infonet (accepted or already
+    present) are hydrated.  The canonical infonet-resident event is used —
+    never the raw batch event — so a forged batch entry carrying a valid
+    event_id but attacker-chosen payload cannot pollute gate_store.
+    """
+    import copy
+    from services.mesh.mesh_hashchain import gate_store, infonet
+    count = 0
+    for evt in events:
+        if evt.get("event_type") != "gate_message":
+            continue
+        event_id = str(evt.get("event_id", "") or "").strip()
+        if not event_id or event_id not in infonet.event_index:
+            continue
+        canonical = infonet.events[infonet.event_index[event_id]]
+        payload = canonical.get("payload") or {}
+        gate_id = str(payload.get("gate", "") or "").strip()
+        if not gate_id:
+            continue
+        try:
+            gate_store.append(gate_id, copy.deepcopy(canonical))
+            count += 1
+        except Exception:
+            pass
+    return count
+
+
+def _hydrate_dm_relay_from_chain(events: list) -> int:
+    import main as _m
+
+    return int(_m._hydrate_dm_relay_from_chain(events))
+
+
+@router.post("/api/mesh/infonet/peer-push")
+@limiter.limit("30/minute")
+async def infonet_peer_push(request: Request):
+    """Accept pushed Infonet events from relay peers (HMAC-authenticated)."""
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 524_288:
+                return Response(content='{"ok":false,"detail":"Request body too large (max 512KB)"}',
+                    status_code=413, media_type="application/json")
+        except (ValueError, TypeError):
+            pass
+    from services.mesh.mesh_hashchain import infonet
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json")
+    body = json_mod.loads(body_bytes or b"{}")
+    events = body.get("events", [])
+    if not isinstance(events, list):
+        return {"ok": False, "detail": "events must be a list"}
+    if len(events) > 50:
+        return {"ok": False, "detail": "Too many events in one push (max 50)"}
+    if not events:
+        return {"ok": True, "accepted": 0, "duplicates": 0, "rejected": []}
+    result = infonet.ingest_events(events)
+    _hydrate_gate_store_from_chain(events)
+    _hydrate_dm_relay_from_chain(events)
+    return {"ok": True, **result}
+
+
+@router.post("/api/mesh/dm/replicate-envelope")
+@limiter.limit("60/minute")
+async def dm_replicate_envelope(request: Request):
+    """Accept a DM envelope replicated from a peer relay (cross-node mailbox).
+
+    Companion endpoint to ``DMRelay.replicate_to_peers`` (outbound, in
+    ``mesh_dm_relay.py``). The sender's relay POSTs an encrypted DM
+    envelope here after a successful local ``deposit``; this endpoint
+    re-enforces the per-(sender, recipient) anti-spam cap and stores
+    the envelope in the local mailbox if accepted.
+
+    The cap is the network rule: a hostile sender's relay can spool
+    extras locally, but every honest peer enforces the cap on inbound
+    replication. Recipient polling from any honest peer therefore
+    never sees more than ``MESH_DM_PENDING_PER_SENDER_LIMIT`` pending
+    from any one sender, no matter how many spam attempts were tried.
+
+    Same HMAC auth pattern as ``infonet_peer_push`` and ``gate_peer_push``.
+    """
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            # DM envelopes are bounded by MESH_DM_MAX_MSG_BYTES + envelope
+            # overhead; 64 KB is a generous ceiling.
+            if int(content_length) > 65_536:
+                return Response(
+                    content='{"ok":false,"detail":"Request body too large (max 64KB)"}',
+                    status_code=413, media_type="application/json",
+                )
+        except (ValueError, TypeError):
+            pass
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(
+            content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json",
+        )
+    try:
+        body = json_mod.loads(body_bytes or b"{}")
+    except (ValueError, TypeError):
+        return Response(
+            content='{"ok":false,"detail":"Invalid JSON body"}',
+            status_code=400, media_type="application/json",
+        )
+    envelope = body.get("envelope")
+    if not isinstance(envelope, dict):
+        return {"ok": False, "detail": "envelope must be an object"}
+
+    originating_peer = _peer_hmac_url_from_request(request) or ""
+
+    from services.mesh.mesh_dm_relay import dm_relay
+    result = dm_relay.accept_replica(
+        envelope=envelope,
+        originating_peer_url=originating_peer,
+    )
+    return result
+
+
+@router.post("/api/mesh/gate/peer-push")
+@limiter.limit("30/minute")
+async def gate_peer_push(request: Request):
+    """Accept pushed gate events from relay peers (private plane)."""
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 524_288:
+                return Response(content='{"ok":false,"detail":"Request body too large"}',
+                    status_code=413, media_type="application/json")
+        except (ValueError, TypeError):
+            pass
+    from services.mesh.mesh_hashchain import gate_store
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json")
+    body = json_mod.loads(body_bytes or b"{}")
+    events = body.get("events", [])
+    if not isinstance(events, list):
+        return {"ok": False, "detail": "events must be a list"}
+    if len(events) > 50:
+        return {"ok": False, "detail": "Too many events (max 50)"}
+    if not events:
+        return {"ok": True, "accepted": 0, "duplicates": 0}
+    from services.mesh.mesh_hashchain import resolve_gate_wire_ref
+    # Sprint 3 / Rec #4: the gate_ref is HMACed with a key bound to the
+    # receiver's peer URL (the URL the push was delivered to). This is
+    # the same URL _verify_peer_push_hmac validated the X-Peer-HMAC
+    # header against, so we can trust it for ref resolution.
+    hop_peer_url = _peer_hmac_url_from_request(request)
+    grouped_events: dict[str, list] = {}
+    for evt in events:
+        evt_dict = evt if isinstance(evt, dict) else {}
+        payload = evt_dict.get("payload")
+        if not isinstance(payload, dict):
+            payload = {}
+        clean_event = {
+            "event_id": str(evt_dict.get("event_id", "") or ""),
+            "event_type": "gate_message",
+            "timestamp": evt_dict.get("timestamp", 0),
+            "node_id": str(evt_dict.get("node_id", "") or evt_dict.get("sender_id", "") or ""),
+            "sequence": evt_dict.get("sequence", 0),
+            "signature": str(evt_dict.get("signature", "") or ""),
+            "public_key": str(evt_dict.get("public_key", "") or ""),
+            "public_key_algo": str(evt_dict.get("public_key_algo", "") or ""),
+            "protocol_version": str(evt_dict.get("protocol_version", "") or ""),
+            "payload": {
+                "ciphertext": str(payload.get("ciphertext", "") or ""),
+                "format": str(payload.get("format", "") or ""),
+                "nonce": str(payload.get("nonce", "") or ""),
+                "sender_ref": str(payload.get("sender_ref", "") or ""),
+            },
+        }
+        epoch = _safe_int(payload.get("epoch", 0) or 0)
+        if epoch > 0:
+            clean_event["payload"]["epoch"] = epoch
+        envelope_hash_val = str(payload.get("envelope_hash", "") or "").strip()
+        gate_envelope_val = str(payload.get("gate_envelope", "") or "").strip()
+        reply_to_val = str(payload.get("reply_to", "") or "").strip()
+        if envelope_hash_val:
+            clean_event["payload"]["envelope_hash"] = envelope_hash_val
+        if gate_envelope_val:
+            clean_event["payload"]["gate_envelope"] = gate_envelope_val
+        if reply_to_val:
+            clean_event["payload"]["reply_to"] = reply_to_val
+        event_gate_id = str(payload.get("gate", "") or evt_dict.get("gate", "") or "").strip().lower()
+        if not event_gate_id:
+            event_gate_id = resolve_gate_wire_ref(
+                str(payload.get("gate_ref", "") or evt_dict.get("gate_ref", "") or ""),
+                clean_event,
+                peer_url=hop_peer_url,
+            )
+        if not event_gate_id:
+            return {"ok": False, "detail": "gate resolution failed"}
+        final_payload: dict[str, Any] = {
+            "gate": event_gate_id,
+            "ciphertext": clean_event["payload"]["ciphertext"],
+            "format": clean_event["payload"]["format"],
+            "nonce": clean_event["payload"]["nonce"],
+            "sender_ref": clean_event["payload"]["sender_ref"],
+        }
+        if epoch > 0:
+            final_payload["epoch"] = epoch
+        if clean_event["payload"].get("envelope_hash"):
+            final_payload["envelope_hash"] = clean_event["payload"]["envelope_hash"]
+        if clean_event["payload"].get("gate_envelope"):
+            final_payload["gate_envelope"] = clean_event["payload"]["gate_envelope"]
+        if clean_event["payload"].get("reply_to"):
+            final_payload["reply_to"] = clean_event["payload"]["reply_to"]
+        grouped_events.setdefault(event_gate_id, []).append({
+            "event_id": clean_event["event_id"],
+            "event_type": "gate_message",
+            "timestamp": clean_event["timestamp"],
+            "node_id": clean_event["node_id"],
+            "sequence": clean_event["sequence"],
+            "signature": clean_event["signature"],
+            "public_key": clean_event["public_key"],
+            "public_key_algo": clean_event["public_key_algo"],
+            "protocol_version": clean_event["protocol_version"],
+            "payload": final_payload,
+        })
+    accepted = 0
+    duplicates = 0
+    rejected = 0
+    for event_gate_id, items in grouped_events.items():
+        result = gate_store.ingest_peer_events(event_gate_id, items)
+        a = int(result.get("accepted", 0) or 0)
+        accepted += a
+        duplicates += int(result.get("duplicates", 0) or 0)
+        rejected += int(result.get("rejected", 0) or 0)
+    return {"ok": True, "accepted": accepted, "duplicates": duplicates, "rejected": rejected}
+
+
+@router.post("/api/mesh/gate/peer-pull")
+@limiter.limit("30/minute")
+async def gate_peer_pull(request: Request):
+    """Return gate events a peer is missing (HMAC-authenticated pull sync)."""
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 65_536:
+                return Response(content='{"ok":false,"detail":"Request body too large"}',
+                    status_code=413, media_type="application/json")
+        except (ValueError, TypeError):
+            pass
+    from services.mesh.mesh_hashchain import gate_store
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json")
+    body = json_mod.loads(body_bytes or b"{}")
+    gate_id = str(body.get("gate_id", "") or "").strip().lower()
+    after_count = _safe_int(body.get("after_count", 0) or 0)
+    if not gate_id:
+        gate_ids = gate_store.known_gate_ids()
+        gate_counts: dict[str, int] = {}
+        for gid in gate_ids:
+            with gate_store._lock:
+                gate_counts[gid] = len(gate_store._gates.get(gid, []))
+        return {"ok": True, "gates": gate_counts}
+    with gate_store._lock:
+        all_events = list(gate_store._gates.get(gate_id, []))
+    total = len(all_events)
+    if after_count >= total:
+        return {"ok": True, "events": [], "total": total, "gate_id": gate_id}
+    batch = all_events[after_count : after_count + _PEER_PUSH_BATCH_SIZE]
+    return {"ok": True, "events": batch, "total": total, "gate_id": gate_id}

backend/routers/osint.py

@@ -0,0 +1,151 @@
+"""Operator OSINT recon routes (server-side proxies, SSRF guarded)."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+from pydantic import BaseModel, Field
+
+from auth import require_local_operator
+from limiter import limiter
+from services.osint import lookups
+
+router = APIRouter(dependencies=[Depends(require_local_operator)])
+
+_ALLOWED_SCHEMAS = {
+    "Person",
+    "Organization",
+    "Company",
+    "Vessel",
+    "Airplane",
+    "LegalEntity",
+}
+
+
+class SweepScanRequest(BaseModel):
+    ip: str = Field(min_length=7, max_length=45)
+    cidr: int = Field(default=24, ge=24, le=32)
+
+
+def _bad_request(exc: ValueError) -> HTTPException:
+    return HTTPException(status_code=400, detail=str(exc))
+
+
+@router.get("/api/osint/ip")
+@limiter.limit("20/minute")
+async def osint_ip(request: Request, ip: str = Query(..., min_length=7, max_length=45)) -> dict:
+    try:
+        return lookups.lookup_ip(ip)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/dns")
+@limiter.limit("20/minute")
+async def osint_dns(request: Request, domain: str = Query(..., min_length=4, max_length=253)) -> dict:
+    try:
+        return lookups.lookup_dns(domain)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/whois")
+@limiter.limit("20/minute")
+async def osint_whois(request: Request, domain: str = Query(..., min_length=4, max_length=253)) -> dict:
+    try:
+        return lookups.lookup_whois(domain)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/certs")
+@limiter.limit("20/minute")
+async def osint_certs(request: Request, domain: str = Query(..., min_length=4, max_length=253)) -> dict:
+    try:
+        return lookups.lookup_certs(domain)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/threats")
+@limiter.limit("20/minute")
+async def osint_threats(request: Request, query: str | None = Query(default=None, max_length=253)) -> dict:
+    return lookups.lookup_threats(query)
+
+
+@router.get("/api/osint/bgp")
+@limiter.limit("20/minute")
+async def osint_bgp(request: Request, query: str = Query(..., min_length=2, max_length=64)) -> dict:
+    try:
+        return lookups.lookup_bgp(query)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/sanctions")
+@limiter.limit("20/minute")
+async def osint_sanctions(
+    request: Request,
+    query: str = Query(..., min_length=4, max_length=200),
+    schema: str | None = Query(default=None),
+    limit: int = Query(default=25, ge=1, le=100),
+) -> dict:
+    if schema and schema not in _ALLOWED_SCHEMAS:
+        raise HTTPException(status_code=400, detail=f"Invalid schema. Allowed: {', '.join(sorted(_ALLOWED_SCHEMAS))}")
+    return lookups.lookup_sanctions(query, schema=schema, limit=limit)
+
+
+@router.get("/api/osint/cve")
+@limiter.limit("30/minute")
+async def osint_cve(request: Request, cve: str = Query(..., min_length=10, max_length=32)) -> dict:
+    try:
+        return lookups.lookup_cve(cve)
+    except ValueError as exc:
+        raise HTTPException(status_code=404 if "not found" in str(exc).lower() else 400, detail=str(exc)) from exc
+
+
+@router.get("/api/osint/mac")
+@limiter.limit("20/minute")
+async def osint_mac(request: Request, mac: str = Query(..., min_length=5, max_length=32)) -> dict:
+    return lookups.lookup_mac(mac)
+
+
+@router.get("/api/osint/github")
+@limiter.limit("20/minute")
+async def osint_github(request: Request, username: str = Query(..., min_length=1, max_length=64)) -> dict:
+    try:
+        return lookups.lookup_github(username)
+    except ValueError as exc:
+        raise HTTPException(status_code=404, detail=str(exc)) from exc
+
+
+@router.get("/api/osint/leaks")
+@limiter.limit("10/minute")
+async def osint_leaks(request: Request, email: str = Query(..., min_length=5, max_length=254)) -> dict:
+    try:
+        return lookups.lookup_leaks(email)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/sweep")
+@limiter.limit("5/minute")
+async def osint_sweep_init(
+    request: Request,
+    ip: str = Query(..., min_length=7, max_length=45),
+    cidr: int = Query(default=24, ge=24, le=32),
+) -> dict:
+    try:
+        return lookups.sweep_init(ip, cidr)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.post("/api/osint/sweep/scan")
+@limiter.limit("3/minute")
+async def osint_sweep_scan(request: Request, payload: SweepScanRequest) -> dict:
+    try:
+        subnet = lookups.subnet_start_for(payload.ip, payload.cidr)
+        scan = lookups.sweep_scan(subnet, payload.cidr)
+        init = lookups.sweep_init(payload.ip, payload.cidr)
+        return {**init, **scan, "subnet": f"{subnet}/{payload.cidr}"}
+    except ValueError as exc:
+        raise _bad_request(exc) from exc

backend/routers/radio.py

@@ -0,0 +1,107 @@
+from fastapi import APIRouter, Request, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+
+router = APIRouter()
+
+
+@router.get("/api/radio/top")
+@limiter.limit("30/minute")
+async def get_top_radios(request: Request):
+    from services.radio_intercept import get_top_broadcastify_feeds
+    return get_top_broadcastify_feeds()
+
+
+@router.get("/api/radio/openmhz/systems")
+@limiter.limit("30/minute")
+async def api_get_openmhz_systems(request: Request):
+    from services.radio_intercept import get_openmhz_systems
+    return get_openmhz_systems()
+
+
+# Issue #213: rotating sys_name bypasses the 20s TTL cache and lets an
+# anonymous caller hammer api.openmhz.com through this proxy, risking an
+# IP-ban for the project. require_local_operator scopes this to the local
+# UI (which goes through the Next.js proxy with admin-key injection) and
+# scoped agent tokens.
+@router.get(
+    "/api/radio/openmhz/calls/{sys_name}",
+    dependencies=[Depends(require_local_operator)],
+)
+@limiter.limit("60/minute")
+async def api_get_openmhz_calls(request: Request, sys_name: str):
+    from services.radio_intercept import get_recent_openmhz_calls
+    return get_recent_openmhz_calls(sys_name)
+
+
+# Issue #214: this is a streaming bandwidth relay. An anonymous caller can
+# stream audio through the backend, saturating the operator's outbound
+# bandwidth. Scope to local operator; the legitimate browser UI still
+# works because relative /api/... paths go through the Next.js proxy
+# which injects the admin key automatically.
+@router.get(
+    "/api/radio/openmhz/audio",
+    dependencies=[Depends(require_local_operator)],
+)
+@limiter.limit("120/minute")
+async def api_get_openmhz_audio(request: Request, url: str = Query(..., min_length=10)):
+    from services.radio_intercept import openmhz_audio_response
+    return openmhz_audio_response(url)
+
+
+@router.get("/api/radio/nearest")
+@limiter.limit("60/minute")
+async def api_get_nearest_radio(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    from services.radio_intercept import find_nearest_openmhz_system
+    return find_nearest_openmhz_system(lat, lng)
+
+
+@router.get("/api/radio/nearest-list")
+@limiter.limit("60/minute")
+async def api_get_nearest_radios_list(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+    limit: int = Query(5, ge=1, le=20),
+):
+    from services.radio_intercept import find_nearest_openmhz_systems_list
+    return find_nearest_openmhz_systems_list(lat, lng, limit=limit)
+
+
+@router.get("/api/route/{callsign}")
+@limiter.limit("60/minute")
+async def get_flight_route(request: Request, callsign: str, lat: float = 0.0, lng: float = 0.0):
+    from services.network_utils import fetch_with_curl
+    r = fetch_with_curl(
+        "https://api.adsb.lol/api/0/routeset",
+        method="POST",
+        json_data={"planes": [{"callsign": callsign, "lat": lat, "lng": lng}]},
+        timeout=10,
+    )
+    if r and r.status_code == 200:
+        data = r.json()
+        route_list = []
+        if isinstance(data, dict):
+            route_list = data.get("value", [])
+        elif isinstance(data, list):
+            route_list = data
+
+        if route_list and len(route_list) > 0:
+            route = route_list[0]
+            airports = route.get("_airports", [])
+            if len(airports) >= 2:
+                orig = airports[0]
+                dest = airports[-1]
+                return {
+                    "orig_loc": [orig.get("lon", 0), orig.get("lat", 0)],
+                    "dest_loc": [dest.get("lon", 0), dest.get("lat", 0)],
+                    "origin_name": f"{orig.get('iata', '') or orig.get('icao', '')}: {orig.get('name', 'Unknown')}",
+                    "dest_name": f"{dest.get('iata', '') or dest.get('icao', '')}: {dest.get('name', 'Unknown')}",
+                }
+    return {}

backend/routers/road_corridors.py

@@ -0,0 +1,105 @@
+"""Road corridor Sentinel-2 freight trend endpoints (opt-in slow layer)."""
+
+from fastapi import APIRouter, HTTPException, Query, Request
+from pydantic import BaseModel, Field
+
+from limiter import limiter
+from services.road_corridor_sat.config import optional_deps_available, road_corridor_sat_enabled
+from services.road_corridor_sat.credentials import sentinel_credentials_configured
+from services.road_corridor_sat.jobs import enqueue_analyze, get_job, get_latest_job, job_to_dict
+from services.road_corridor_sat.presets import CORRIDOR_PRESETS, get_preset
+from services.road_corridor_sat.storage import build_trends_payload, preset_metadata
+
+router = APIRouter()
+
+
+def _status_payload() -> dict:
+    latest = get_latest_job()
+    return {
+        "enabled": road_corridor_sat_enabled(),
+        "deps_installed": optional_deps_available(),
+        "credentials_configured": sentinel_credentials_configured(),
+        "preset_count": len(CORRIDOR_PRESETS),
+        "attribution": "backend/third_party/drishx/NOTICE.md",
+        "active_job": job_to_dict(latest) if latest and latest.status in {"queued", "running"} else None,
+    }
+
+
+def _require_analyze_ready() -> None:
+    if not optional_deps_available():
+        raise HTTPException(
+            status_code=503,
+            detail="Install optional road-corridor dependencies (uv sync --extra road-corridor)",
+        )
+    if not sentinel_credentials_configured():
+        raise HTTPException(
+            status_code=503,
+            detail="Set SENTINEL_CLIENT_ID and SENTINEL_CLIENT_SECRET in Imagery settings",
+        )
+
+
+class AnalyzeRequest(BaseModel):
+    lat: float = Field(ge=-90, le=90)
+    lon: float = Field(ge=-180, le=180)
+    label: str | None = Field(default=None, max_length=120)
+
+
+@router.get("/api/road-corridors/status")
+@limiter.limit("60/minute")
+async def road_corridors_status(request: Request) -> dict:
+    return {"ok": True, **_status_payload()}
+
+
+@router.get("/api/road-corridors")
+@limiter.limit("60/minute")
+async def list_road_corridors(request: Request) -> dict:
+    return {
+        "ok": True,
+        "status": _status_payload(),
+        "presets": CORRIDOR_PRESETS,
+        "trends": build_trends_payload(),
+    }
+
+
+@router.post("/api/road-corridors/analyze")
+@limiter.limit("6/minute")
+async def analyze_road_corridor_here(request: Request, payload: AnalyzeRequest) -> dict:
+    """Start an on-demand Sentinel-2 corridor analysis at map center."""
+    _require_analyze_ready()
+    try:
+        job = enqueue_analyze(payload.lat, payload.lon, payload.label)
+    except RuntimeError as exc:
+        if str(exc) == "analysis_already_running":
+            active = get_latest_job()
+            raise HTTPException(
+                status_code=409,
+                detail="Analysis already in progress",
+                headers={"X-Job-Id": active.job_id if active else ""},
+            ) from exc
+        raise
+    return {"ok": True, **job_to_dict(job)}
+
+
+@router.get("/api/road-corridors/analyze/status")
+@limiter.limit("120/minute")
+async def analyze_road_corridor_status(
+    request: Request,
+    job_id: str | None = Query(default=None),
+) -> dict:
+    job = get_job(job_id) if job_id else get_latest_job()
+    if job is None:
+        return {"ok": True, "job": None}
+    return {"ok": True, "job": job_to_dict(job)}
+
+
+@router.get("/api/road-corridors/{preset_id}")
+@limiter.limit("60/minute")
+async def get_road_corridor(preset_id: str, request: Request) -> dict:
+    meta = preset_metadata(preset_id)
+    if meta is None:
+        raise HTTPException(status_code=404, detail="Unknown corridor preset")
+    preset = get_preset(preset_id)
+    if preset is None:
+        # Ad-hoc viewport runs are stored on disk but not in CORRIDOR_PRESETS.
+        return {"ok": True, "preset": None, "result": meta, "status": _status_payload()}
+    return {"ok": True, "preset": preset, "result": meta, "status": _status_payload()}

backend/routers/sar.py

@@ -0,0 +1,260 @@
+"""SAR (Synthetic Aperture Radar) layer endpoints.
+
+Exposes:
+  - GET  /api/sar/status            — feature gates + signup links for the UI
+  - GET  /api/sar/anomalies         — Mode B pre-processed anomalies
+  - GET  /api/sar/scenes            — Mode A scene catalog
+  - GET  /api/sar/coverage          — per-AOI coverage and next-pass hints
+  - GET  /api/sar/aois              — operator-defined AOIs
+  - POST /api/sar/aois              — create or replace an AOI
+  - DELETE /api/sar/aois/{aoi_id}   — remove an AOI
+  - GET  /api/sar/near              — anomalies within radius_km of (lat, lon)
+
+The /status endpoint is the load-bearing UX: when Mode B is disabled it
+returns the structured help payload from sar_config.products_fetch_status()
+so the frontend can render in-app links to the free signup pages instead of
+making the user hunt around.
+"""
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+from pydantic import BaseModel, Field
+
+from auth import require_local_operator
+from limiter import limiter
+from services.fetchers._store import get_latest_data_subset_refs
+from services.sar.sar_aoi import (
+    SarAoi,
+    add_aoi,
+    haversine_km,
+    load_aois,
+    remove_aoi,
+)
+from services.sar.sar_config import (
+    catalog_enabled,
+    clear_runtime_credentials,
+    openclaw_enabled,
+    products_fetch_enabled,
+    products_fetch_status,
+    require_private_tier_for_publish,
+    set_runtime_credentials,
+)
+
+router = APIRouter()
+
+
+# ---------------------------------------------------------------------------
+# Status — the in-app onboarding hook
+# ---------------------------------------------------------------------------
+@router.get("/api/sar/status")
+@limiter.limit("60/minute")
+async def sar_status(request: Request) -> dict:
+    """Layer status + signup links.
+
+    The frontend calls this whenever the SAR panel is opened.  When Mode B
+    is off, the response includes a step-by-step ``help`` block with the
+    free signup URLs so the user can enable everything without leaving the
+    app.
+    """
+    products_status = products_fetch_status()
+    return {
+        "ok": True,
+        "catalog": {
+            "mode": "A",
+            "enabled": catalog_enabled(),
+            "needs_account": False,
+            "description": "Free Sentinel-1 scene catalog from ASF Search.",
+        },
+        "products": {
+            "mode": "B",
+            **products_status,
+        },
+        "openclaw_enabled": openclaw_enabled(),
+        "require_private_tier": require_private_tier_for_publish(),
+    }
+
+
+# ---------------------------------------------------------------------------
+# Data feeds
+# ---------------------------------------------------------------------------
+@router.get("/api/sar/anomalies")
+@limiter.limit("60/minute")
+async def sar_anomalies(
+    request: Request,
+    kind: str = Query("", description="Optional anomaly kind filter"),
+    aoi_id: str = Query("", description="Optional AOI id filter"),
+    limit: int = Query(200, ge=1, le=1000),
+) -> dict:
+    """Return the latest cached SAR anomalies (Mode B)."""
+    snap = get_latest_data_subset_refs("sar_anomalies")
+    items = list(snap.get("sar_anomalies") or [])
+    if kind:
+        items = [a for a in items if a.get("kind") == kind]
+    if aoi_id:
+        aoi_id = aoi_id.strip().lower()
+        items = [a for a in items if (a.get("stack_id") or "").lower() == aoi_id]
+    items = items[:limit]
+    return {
+        "ok": True,
+        "count": len(items),
+        "anomalies": items,
+        "products_enabled": products_fetch_enabled(),
+    }
+
+
+@router.get("/api/sar/scenes")
+@limiter.limit("60/minute")
+async def sar_scenes(
+    request: Request,
+    aoi_id: str = Query(""),
+    limit: int = Query(200, ge=1, le=1000),
+) -> dict:
+    """Return the latest cached scene catalog (Mode A)."""
+    snap = get_latest_data_subset_refs("sar_scenes")
+    items = list(snap.get("sar_scenes") or [])
+    if aoi_id:
+        aoi_id = aoi_id.strip().lower()
+        items = [s for s in items if (s.get("aoi_id") or "").lower() == aoi_id]
+    items = items[:limit]
+    return {
+        "ok": True,
+        "count": len(items),
+        "scenes": items,
+        "catalog_enabled": catalog_enabled(),
+    }
+
+
+@router.get("/api/sar/coverage")
+@limiter.limit("60/minute")
+async def sar_coverage(request: Request) -> dict:
+    """Per-AOI coverage and rough next-pass estimate."""
+    snap = get_latest_data_subset_refs("sar_aoi_coverage")
+    return {
+        "ok": True,
+        "coverage": list(snap.get("sar_aoi_coverage") or []),
+    }
+
+
+@router.get("/api/sar/near")
+@limiter.limit("60/minute")
+async def sar_near(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lon: float = Query(..., ge=-180, le=180),
+    radius_km: float = Query(50, ge=1, le=2000),
+    kind: str = Query(""),
+    limit: int = Query(50, ge=1, le=500),
+) -> dict:
+    """Return anomalies whose center sits within ``radius_km`` of (lat, lon)."""
+    snap = get_latest_data_subset_refs("sar_anomalies")
+    items = list(snap.get("sar_anomalies") or [])
+    matches = []
+    for a in items:
+        try:
+            a_lat = float(a.get("lat", 0.0))
+            a_lon = float(a.get("lon", 0.0))
+        except (TypeError, ValueError):
+            continue
+        d = haversine_km(lat, lon, a_lat, a_lon)
+        if d > radius_km:
+            continue
+        if kind and a.get("kind") != kind:
+            continue
+        a = dict(a)
+        a["distance_km"] = round(d, 2)
+        matches.append(a)
+    matches.sort(key=lambda x: x.get("distance_km", 0))
+    return {
+        "ok": True,
+        "count": len(matches[:limit]),
+        "anomalies": matches[:limit],
+    }
+
+
+# ---------------------------------------------------------------------------
+# AOI CRUD
+# ---------------------------------------------------------------------------
+@router.get("/api/sar/aois")
+@limiter.limit("60/minute")
+async def sar_aoi_list(request: Request) -> dict:
+    return {
+        "ok": True,
+        "aois": [a.to_dict() for a in load_aois(force=True)],
+    }
+
+
+class AoiPayload(BaseModel):
+    id: str = Field(..., min_length=1, max_length=64)
+    name: str = Field(..., min_length=1, max_length=120)
+    description: str = Field("", max_length=400)
+    center_lat: float = Field(..., ge=-90, le=90)
+    center_lon: float = Field(..., ge=-180, le=180)
+    radius_km: float = Field(25.0, ge=1.0, le=500.0)
+    category: str = Field("watchlist", max_length=40)
+    polygon: list[list[float]] | None = None
+
+
+@router.post("/api/sar/aois", dependencies=[Depends(require_local_operator)])
+@limiter.limit("20/minute")
+async def sar_aoi_upsert(request: Request, payload: AoiPayload) -> dict:
+    aoi = SarAoi(
+        id=payload.id.strip().lower(),
+        name=payload.name.strip(),
+        description=payload.description.strip(),
+        center_lat=payload.center_lat,
+        center_lon=payload.center_lon,
+        radius_km=payload.radius_km,
+        polygon=payload.polygon,
+        category=(payload.category or "watchlist").strip().lower(),
+    )
+    add_aoi(aoi)
+    return {"ok": True, "aoi": aoi.to_dict()}
+
+
+@router.delete("/api/sar/aois/{aoi_id}", dependencies=[Depends(require_local_operator)])
+@limiter.limit("20/minute")
+async def sar_aoi_delete(request: Request, aoi_id: str) -> dict:
+    removed = remove_aoi(aoi_id)
+    if not removed:
+        raise HTTPException(status_code=404, detail="AOI not found")
+    return {"ok": True, "removed": aoi_id}
+
+
+# ---------------------------------------------------------------------------
+# Mode B enable / disable — one-click setup from the frontend
+# ---------------------------------------------------------------------------
+class ModeBEnablePayload(BaseModel):
+    earthdata_user: str = Field("", max_length=120)
+    earthdata_token: str = Field(..., min_length=8, max_length=2048)
+    copernicus_user: str = Field("", max_length=120)
+    copernicus_token: str = Field("", max_length=2048)
+
+
+@router.post("/api/sar/mode-b/enable", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def sar_mode_b_enable(request: Request, payload: ModeBEnablePayload) -> dict:
+    """Store Earthdata (and optional Copernicus) credentials and flip both
+    two-step opt-in flags.  Returns the fresh status payload so the UI can
+    immediately reflect the change.
+    """
+    set_runtime_credentials(
+        earthdata_user=payload.earthdata_user,
+        earthdata_token=payload.earthdata_token,
+        copernicus_user=payload.copernicus_user,
+        copernicus_token=payload.copernicus_token,
+        mode_b_opt_in=True,
+    )
+    return {
+        "ok": True,
+        "products": products_fetch_status(),
+    }
+
+
+@router.post("/api/sar/mode-b/disable", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def sar_mode_b_disable(request: Request) -> dict:
+    """Wipe runtime credentials and revert to Mode A only."""
+    clear_runtime_credentials()
+    return {
+        "ok": True,
+        "products": products_fetch_status(),
+    }

backend/routers/scm.py

@@ -0,0 +1,16 @@
+"""Supply-chain risk overlay."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, Request
+
+from auth import require_local_operator
+from limiter import limiter
+from services.scm.suppliers import build_scm_payload
+
+router = APIRouter()
+
+
+@router.get("/api/scm-suppliers")
+@limiter.limit("30/minute")
+async def scm_suppliers(request: Request, _: None = Depends(require_local_operator)) -> dict:
+    return build_scm_payload()

backend/routers/sigint.py

@@ -0,0 +1,67 @@
+from fastapi import APIRouter, Request, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+from services.data_fetcher import get_latest_data
+
+router = APIRouter()
+
+
+@router.get("/api/oracle/region-intel")
+@limiter.limit("30/minute")
+async def oracle_region_intel(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Get oracle intelligence summary for a geographic region."""
+    from services.oracle_service import get_region_oracle_intel
+    news_items = get_latest_data().get("news", [])
+    return get_region_oracle_intel(lat, lng, news_items)
+
+
+@router.get("/api/thermal/verify", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def thermal_verify(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+    radius_km: float = Query(10, ge=1, le=100),
+):
+    """On-demand thermal anomaly verification using Sentinel-2 SWIR bands."""
+    from services.thermal_sentinel import search_thermal_anomaly
+    result = search_thermal_anomaly(lat, lng, radius_km)
+    return result
+
+
+@router.post("/api/sigint/transmit", dependencies=[Depends(require_local_operator)])
+@limiter.limit("5/minute")
+async def sigint_transmit(request: Request):
+    """Send an APRS-IS message to a specific callsign. Requires ham radio credentials."""
+    from services.wormhole_supervisor import get_transport_tier
+    tier = get_transport_tier()
+    if str(tier or "").startswith("private_"):
+        return {"ok": False, "detail": "APRS transmit blocked in private transport mode"}
+    body = await request.json()
+    callsign = body.get("callsign", "")
+    passcode = body.get("passcode", "")
+    target = body.get("target", "")
+    message = body.get("message", "")
+    if not all([callsign, passcode, target, message]):
+        return {"ok": False, "detail": "Missing required fields: callsign, passcode, target, message"}
+    from services.sigint_bridge import send_aprs_message
+    return send_aprs_message(callsign, passcode, target, message)
+
+
+@router.get("/api/sigint/nearest-sdr")
+@limiter.limit("30/minute")
+async def nearest_sdr(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Find the nearest KiwiSDR receivers to a given coordinate."""
+    from services.sigint_bridge import find_nearest_kiwisdr
+    kiwisdr_data = get_latest_data().get("kiwisdr", [])
+    return find_nearest_kiwisdr(lat, lng, kiwisdr_data)

backend/routers/tools.py

@@ -0,0 +1,444 @@
+import asyncio
+import logging
+import math
+from typing import Any
+from fastapi import APIRouter, Request, Query, Depends, HTTPException, Response
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _safe_float(val, default=0.0):
+    try:
+        parsed = float(val)
+        if not math.isfinite(parsed):
+            return default
+        return parsed
+    except (TypeError, ValueError):
+        return default
+
+
+class ShodanSearchRequest(BaseModel):
+    query: str
+    page: int = 1
+    facets: list[str] = []
+
+
+class ShodanCountRequest(BaseModel):
+    query: str
+    facets: list[str] = []
+
+
+class ShodanHostRequest(BaseModel):
+    ip: str
+    history: bool = False
+
+
+@router.get("/api/region-dossier")
+@limiter.limit("30/minute")
+def api_region_dossier(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Sync def so FastAPI runs it in a threadpool — prevents blocking the event loop."""
+    from services.region_dossier import get_region_dossier
+    return get_region_dossier(lat, lng)
+
+
+@router.get("/api/geocode/search")
+@limiter.limit("30/minute")
+async def api_geocode_search(
+    request: Request,
+    q: str = "",
+    limit: int = 5,
+    local_only: bool = False,
+):
+    from services.geocode import search_geocode
+    if not q or len(q.strip()) < 2:
+        return {"results": [], "query": q, "count": 0}
+    results = await asyncio.to_thread(search_geocode, q, limit, local_only)
+    return {"results": results, "query": q, "count": len(results)}
+
+
+@router.get("/api/geocode/reverse")
+@limiter.limit("60/minute")
+async def api_geocode_reverse(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+    local_only: bool = False,
+):
+    from services.geocode import reverse_geocode
+    return await asyncio.to_thread(reverse_geocode, lat, lng, local_only)
+
+
+# ── Wikimedia proxy (#360) — browser calls these instead of wikipedia.org ───
+@router.get("/api/wikipedia/summary")
+@limiter.limit("60/minute")
+def api_wikipedia_summary(
+    request: Request,
+    title: str = Query(..., min_length=1, max_length=256),
+):
+    """Proxy Wikipedia REST summaries through the self-hosted backend."""
+    from services.region_dossier import fetch_wikipedia_page_summary
+
+    summary = fetch_wikipedia_page_summary(title)
+    if summary is None:
+        return JSONResponse(status_code=404, content={"detail": "not_found"})
+    return summary
+
+
+class WikidataSparqlRequest(BaseModel):
+    query: str
+
+
+@router.post("/api/wikidata/sparql")
+@limiter.limit("30/minute")
+def api_wikidata_sparql(request: Request, body: WikidataSparqlRequest):
+    """Proxy Wikidata SPARQL so the browser never contacts query.wikidata.org."""
+    from services.region_dossier import fetch_wikidata_sparql_bindings
+
+    q = (body.query or "").strip()
+    if len(q) > 12_000:
+        raise HTTPException(400, "SPARQL query too large")
+    bindings = fetch_wikidata_sparql_bindings(q)
+    return {"bindings": bindings}
+
+
+# ── Sentinel proxy routes (Issue #299/#300/#301, reported by tg12) ──────────
+# These three endpoints relay external Sentinel / Planetary Computer
+# requests through the backend to avoid browser CORS blocks. They are
+# operator-only helpers — they MUST NOT be callable by anonymous remote
+# users, because:
+#
+#   * /api/sentinel/token  — caller supplies their own Sentinel client_id +
+#     client_secret. Without operator gating, the backend becomes a free
+#     anonymous OAuth-mint relay for any Copernicus account.
+#   * /api/sentinel/tile   — same shape as the token route but for tile
+#     imagery. Without gating, the backend acts as an anonymous quota and
+#     bandwidth relay for Sentinel Hub Process API calls.
+#   * /api/sentinel2/search — hits the Planetary Computer STAC search API
+#     and falls back to Esri imagery. No caller credentials are involved,
+#     but the route is still an anonymous external-search relay. We gate
+#     it the same way for consistency with the rest of the operator-only
+#     helper surface.
+#
+# Gating is via require_local_operator (loopback / bridge / admin key),
+# matching the same allowlist already used by /api/region-dossier and
+# the other operator helpers further up this file. Single-operator nodes
+# see no behavior change — their dashboard already lives on loopback or
+# the trusted Docker bridge, so it still resolves.
+@router.get("/api/sentinel2/search", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+def api_sentinel2_search(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Search for latest Sentinel-2 imagery at a point. Sync for threadpool execution."""
+    from services.sentinel_search import search_sentinel2_scene
+    return search_sentinel2_scene(lat, lng)
+
+
+# Issue #298 (tg12): Sentinel credentials moved server-side
+# ---------------------------------------------------------------------------
+# Previously the frontend kept Copernicus CDSE client_id + client_secret in
+# browser localStorage / sessionStorage and forwarded them on every tile
+# request through this proxy. That exposed real third-party credentials to
+# any same-origin script (XSS, malicious browser extension, dev-tools HAR
+# export).
+#
+# Resolution order (first match wins):
+#   1. Request body — kept for back-compat. A small number of legacy
+#      operator setups may still post credentials; we don't break them.
+#   2. Backend .env — SENTINEL_CLIENT_ID / SENTINEL_CLIENT_SECRET, managed
+#      through the existing /api/settings/api-keys flow (admin-gated).
+#
+# The frontend in ``sentinelHub.ts`` no longer reads browser storage and no
+# longer forwards credentials — every dashboard request now lands in (2).
+# The require_local_operator gate (added in #303/PR #303) stays — both layers
+# are independent: the gate blocks anonymous callers, the env fallback lets
+# legitimate (gated) callers omit credentials from the body.
+# ---------------------------------------------------------------------------
+def _resolve_sentinel_credentials(body_id: str, body_secret: str) -> tuple[str, str]:
+    """Return (client_id, client_secret) using body values when present,
+    otherwise falling back to backend .env. Empty strings if neither is set."""
+    import os as _os
+    cid = (body_id or "").strip() or (_os.environ.get("SENTINEL_CLIENT_ID", "") or "").strip()
+    csec = (body_secret or "").strip() or (_os.environ.get("SENTINEL_CLIENT_SECRET", "") or "").strip()
+    return cid, csec
+
+
+@router.post("/api/sentinel/token", dependencies=[Depends(require_local_operator)])
+@limiter.limit("60/minute")
+async def api_sentinel_token(request: Request):
+    """Proxy Copernicus CDSE OAuth2 token request (avoids browser CORS block).
+
+    Credentials are resolved by ``_resolve_sentinel_credentials`` — body
+    fields are honored for back-compat, otherwise the backend .env values
+    populated through ``/api/settings/api-keys`` are used.
+    """
+    import requests as req
+    body = await request.body()
+    from urllib.parse import parse_qs
+    params = parse_qs(body.decode("utf-8"))
+    body_id = params.get("client_id", [""])[0]
+    body_secret = params.get("client_secret", [""])[0]
+    client_id, client_secret = _resolve_sentinel_credentials(body_id, body_secret)
+    if not client_id or not client_secret:
+        # Friendly, non-hostile error — points the operator at the place
+        # they configure other API keys instead of just saying "required".
+        raise HTTPException(
+            400,
+            "Sentinel client_id/client_secret are not configured. "
+            "Set SENTINEL_CLIENT_ID and SENTINEL_CLIENT_SECRET in the "
+            "API Keys panel (Settings → API Keys) or your backend .env.",
+        )
+    token_url = "https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token"
+    try:
+        resp = await asyncio.to_thread(req.post, token_url,
+            data={"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret},
+            timeout=15)
+        return Response(content=resp.content, status_code=resp.status_code, media_type="application/json")
+    except Exception:
+        logger.exception("Token request failed")
+        raise HTTPException(502, "Token request failed")
+
+
+# Cache key is an HMAC of (client_id, client_secret) — a caller cannot hit
+# this cache without knowing the same secret that originally populated it.
+# Without this binding, the lookup only checked client_id, so anyone who
+# knew a valid client_id could reuse another caller's cached token (and
+# burn their Copernicus quota / access tiles on their account).
+_sh_token_cache: dict = {"token": None, "expiry": 0, "credential_fp": ""}
+
+
+def _credential_fingerprint(client_id: str, client_secret: str) -> str:
+    """Return a stable, secret-binding fingerprint for the Sentinel cache key.
+
+    Uses HMAC-SHA256 so the raw secret is never stored in process memory as
+    a cache key. The HMAC key is a per-process random value, which means the
+    fingerprint cannot be precomputed across restarts (additional defense
+    against an attacker who learned a valid client_id but not the secret).
+    """
+    import hashlib
+    import hmac
+
+    return hmac.new(
+        _SH_TOKEN_CACHE_HMAC_KEY,
+        f"{client_id}\x00{client_secret}".encode("utf-8"),
+        hashlib.sha256,
+    ).hexdigest()
+
+
+# Per-process random HMAC key. Regenerated on each backend startup so cached
+# fingerprints don't survive restarts.
+import os as _os
+_SH_TOKEN_CACHE_HMAC_KEY = _os.urandom(32)
+
+
+@router.post("/api/sentinel/tile", dependencies=[Depends(require_local_operator)])
+@limiter.limit("300/minute")
+async def api_sentinel_tile(request: Request):
+    """Proxy Sentinel Hub Process API tile request (avoids CORS block)."""
+    import requests as req
+    import time as _time
+    try:
+        body = await request.json()
+    except Exception:
+        return JSONResponse(status_code=422, content={"ok": False, "detail": "invalid JSON body"})
+
+    # Issue #298: same resolution order as /api/sentinel/token — body
+    # values for back-compat, otherwise backend .env.
+    body_id = body.get("client_id", "")
+    body_secret = body.get("client_secret", "")
+    client_id, client_secret = _resolve_sentinel_credentials(body_id, body_secret)
+    preset = body.get("preset", "TRUE-COLOR")
+    date_str = body.get("date", "")
+    z = body.get("z", 0)
+    x = body.get("x", 0)
+    y = body.get("y", 0)
+
+    if not client_id or not client_secret or not date_str:
+        # Distinguish "no creds" from "no date" so the operator knows
+        # what to fix. Same friendly pointer as the /token route.
+        if not client_id or not client_secret:
+            raise HTTPException(
+                400,
+                "Sentinel client_id/client_secret are not configured. "
+                "Set SENTINEL_CLIENT_ID and SENTINEL_CLIENT_SECRET in the "
+                "API Keys panel (Settings → API Keys) or your backend .env.",
+            )
+        raise HTTPException(400, "date required")
+
+    now = _time.time()
+    credential_fp = _credential_fingerprint(client_id, client_secret)
+    if (_sh_token_cache["token"]
+            and _sh_token_cache["credential_fp"] == credential_fp
+            and now < _sh_token_cache["expiry"] - 30):
+        token = _sh_token_cache["token"]
+    else:
+        token_url = "https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token"
+        try:
+            tresp = await asyncio.to_thread(req.post, token_url,
+                data={"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret},
+                timeout=15)
+            if tresp.status_code != 200:
+                raise HTTPException(401, f"Token auth failed: {tresp.text[:200]}")
+            tdata = tresp.json()
+            token = tdata["access_token"]
+            _sh_token_cache["token"] = token
+            _sh_token_cache["expiry"] = now + tdata.get("expires_in", 300)
+            _sh_token_cache["credential_fp"] = credential_fp
+        except HTTPException:
+            raise
+        except Exception:
+            logger.exception("Token request failed")
+            raise HTTPException(502, "Token request failed")
+
+    half = 20037508.342789244
+    tile_size = (2 * half) / math.pow(2, z)
+    min_x = -half + x * tile_size
+    max_x = min_x + tile_size
+    max_y = half - y * tile_size
+    min_y = max_y - tile_size
+    bbox = [min_x, min_y, max_x, max_y]
+
+    evalscripts = {
+        "TRUE-COLOR": '//VERSION=3\nfunction setup(){return{input:["B04","B03","B02"],output:{bands:3}};}\nfunction evaluatePixel(s){return[2.5*s.B04,2.5*s.B03,2.5*s.B02];}',
+        "FALSE-COLOR": '//VERSION=3\nfunction setup(){return{input:["B08","B04","B03"],output:{bands:3}};}\nfunction evaluatePixel(s){return[2.5*s.B08,2.5*s.B04,2.5*s.B03];}',
+        "NDVI": '//VERSION=3\nfunction setup(){return{input:["B04","B08"],output:{bands:3}};}\nfunction evaluatePixel(s){var n=(s.B08-s.B04)/(s.B08+s.B04);if(n<-0.2)return[0.05,0.05,0.05];if(n<0)return[0.75,0.75,0.75];if(n<0.1)return[0.86,0.86,0.86];if(n<0.2)return[0.92,0.84,0.68];if(n<0.3)return[0.77,0.88,0.55];if(n<0.4)return[0.56,0.80,0.32];if(n<0.5)return[0.35,0.72,0.18];if(n<0.6)return[0.20,0.60,0.08];if(n<0.7)return[0.10,0.48,0.04];return[0.0,0.36,0.0];}',
+        "MOISTURE-INDEX": '//VERSION=3\nfunction setup(){return{input:["B8A","B11"],output:{bands:3}};}\nfunction evaluatePixel(s){var m=(s.B8A-s.B11)/(s.B8A+s.B11);var r=Math.max(0,Math.min(1,1.5-3*m));var g=Math.max(0,Math.min(1,m<0?1.5+3*m:1.5-3*m));var b=Math.max(0,Math.min(1,1.5+3*(m-0.5)));return[r,g,b];}',
+    }
+    evalscript = evalscripts.get(preset, evalscripts["TRUE-COLOR"])
+
+    from datetime import datetime as _dt, timedelta as _td
+    try:
+        end_date = _dt.strptime(date_str, "%Y-%m-%d")
+    except ValueError:
+        end_date = _dt.utcnow()
+
+    if z <= 6:
+        lookback_days = 30
+    elif z <= 9:
+        lookback_days = 14
+    elif z <= 11:
+        lookback_days = 7
+    else:
+        lookback_days = 5
+
+    start_date = end_date - _td(days=lookback_days)
+
+    process_body = {
+        "input": {
+            "bounds": {"bbox": bbox, "properties": {"crs": "http://www.opengis.net/def/crs/EPSG/0/3857"}},
+            "data": [{"type": "sentinel-2-l2a", "dataFilter": {
+                "timeRange": {
+                    "from": start_date.strftime("%Y-%m-%dT00:00:00Z"),
+                    "to": end_date.strftime("%Y-%m-%dT23:59:59Z"),
+                },
+                "maxCloudCoverage": 30, "mosaickingOrder": "leastCC",
+            }}],
+        },
+        "output": {"width": 256, "height": 256,
+            "responses": [{"identifier": "default", "format": {"type": "image/png"}}]},
+        "evalscript": evalscript,
+    }
+    try:
+        resp = await asyncio.to_thread(req.post,
+            "https://sh.dataspace.copernicus.eu/api/v1/process",
+            json=process_body,
+            headers={"Authorization": f"Bearer {token}", "Accept": "image/png"},
+            timeout=30)
+        return Response(content=resp.content, status_code=resp.status_code,
+            media_type=resp.headers.get("content-type", "image/png"))
+    except Exception:
+        logger.exception("Process API failed")
+        raise HTTPException(502, "Process API failed")
+
+
+@router.get("/api/tools/shodan/status", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_shodan_status(request: Request):
+    from services.shodan_connector import get_shodan_connector_status
+    return get_shodan_connector_status()
+
+
+@router.post("/api/tools/shodan/search", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_shodan_search(request: Request, body: ShodanSearchRequest):
+    from services.shodan_connector import ShodanConnectorError, search_shodan
+    try:
+        return search_shodan(body.query, page=body.page, facets=body.facets)
+    except ShodanConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/shodan/count", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_shodan_count(request: Request, body: ShodanCountRequest):
+    from services.shodan_connector import ShodanConnectorError, count_shodan
+    try:
+        return count_shodan(body.query, facets=body.facets)
+    except ShodanConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/shodan/host", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_shodan_host(request: Request, body: ShodanHostRequest):
+    from services.shodan_connector import ShodanConnectorError, lookup_shodan_host
+    try:
+        return lookup_shodan_host(body.ip, history=body.history)
+    except ShodanConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.get("/api/tools/uw/status", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_uw_status(request: Request):
+    from services.unusual_whales_connector import get_uw_status
+    return get_uw_status()
+
+
+@router.post("/api/tools/uw/congress", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_uw_congress(request: Request):
+    from services.unusual_whales_connector import FinnhubConnectorError, fetch_congress_trades
+    try:
+        return fetch_congress_trades()
+    except FinnhubConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/uw/darkpool", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_uw_darkpool(request: Request):
+    from services.unusual_whales_connector import FinnhubConnectorError, fetch_insider_transactions
+    try:
+        return fetch_insider_transactions()
+    except FinnhubConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/uw/flow", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_uw_flow(request: Request):
+    from services.unusual_whales_connector import FinnhubConnectorError, fetch_defense_quotes
+    try:
+        return fetch_defense_quotes()
+    except FinnhubConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc

backend/scripts/bootstrap_manifest_helper.py

@@ -0,0 +1,115 @@
+import argparse
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+BACKEND_DIR = ROOT / "backend"
+
+if str(BACKEND_DIR) not in sys.path:
+    sys.path.insert(0, str(BACKEND_DIR))
+
+from services.mesh.mesh_bootstrap_manifest import (  # noqa: E402
+    bootstrap_signer_public_key_b64,
+    generate_bootstrap_signer,
+    write_signed_bootstrap_manifest,
+)
+
+
+def _load_peers(args: argparse.Namespace) -> list[dict]:
+    peers: list[dict] = []
+    if args.peers_file:
+        raw = json.loads(Path(args.peers_file).read_text(encoding="utf-8"))
+        if not isinstance(raw, list):
+            raise ValueError("peers file must be a JSON array")
+        for entry in raw:
+            if not isinstance(entry, dict):
+                raise ValueError("peers file entries must be objects")
+            peers.append(dict(entry))
+    for peer_arg in args.peer or []:
+        parts = [part.strip() for part in str(peer_arg).split(",", 3)]
+        if len(parts) < 3:
+            raise ValueError("peer entries must look like url,transport,role[,label]")
+        peer_url, transport, role = parts[:3]
+        label = parts[3] if len(parts) > 3 else ""
+        peers.append(
+            {
+                "peer_url": peer_url,
+                "transport": transport,
+                "role": role,
+                "label": label,
+            }
+        )
+    if not peers:
+        raise ValueError("at least one peer is required")
+    return peers
+
+
+def cmd_generate_keypair(_args: argparse.Namespace) -> int:
+    signer = generate_bootstrap_signer()
+    print(json.dumps(signer, indent=2))
+    return 0
+
+
+def cmd_sign(args: argparse.Namespace) -> int:
+    peers = _load_peers(args)
+    manifest = write_signed_bootstrap_manifest(
+        args.output,
+        signer_id=args.signer_id,
+        signer_private_key_b64=args.private_key_b64,
+        peers=peers,
+        valid_for_hours=int(args.valid_hours),
+    )
+    print(f"Wrote signed bootstrap manifest to {Path(args.output).resolve()}")
+    print(f"signer_id={manifest.signer_id}")
+    print(f"valid_until={manifest.valid_until}")
+    print(f"peer_count={len(manifest.peers)}")
+    print(f"MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY={bootstrap_signer_public_key_b64(args.private_key_b64)}")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Generate and sign Infonet bootstrap manifests for participant nodes."
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    keygen = subparsers.add_parser("generate-keypair", help="Generate an Ed25519 bootstrap signer keypair")
+    keygen.set_defaults(func=cmd_generate_keypair)
+
+    sign = subparsers.add_parser("sign", help="Sign a bootstrap manifest from peer entries")
+    sign.add_argument("--output", required=True, help="Output path for bootstrap_peers.json")
+    sign.add_argument("--signer-id", required=True, help="Manifest signer identifier")
+    sign.add_argument(
+        "--private-key-b64",
+        required=True,
+        help="Raw Ed25519 private key in base64 returned by generate-keypair",
+    )
+    sign.add_argument(
+        "--peers-file",
+        help="JSON file containing an array of peer objects with peer_url, transport, role, and optional label",
+    )
+    sign.add_argument(
+        "--peer",
+        action="append",
+        help="Inline peer in the form url,transport,role[,label]. May be repeated.",
+    )
+    sign.add_argument(
+        "--valid-hours",
+        type=int,
+        default=168,
+        help="Manifest validity window in hours (default: 168)",
+    )
+    sign.set_defaults(func=cmd_sign)
+
+    return parser
+
+
+def main() -> int:
+    parser = build_parser()
+    args = parser.parse_args()
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/scripts/check-env.ps1

@@ -0,0 +1,5 @@
+param(
+  [string]$Python = "python"
+)
+
+& $Python -c "from services.env_check import validate_env; validate_env(strict=False)"

backend/scripts/check-env.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+PYTHON="${PYTHON:-python3}"
+"$PYTHON" -c "from services.env_check import validate_env; validate_env(strict=False)"

backend/scripts/convert_power_plants.py

@@ -0,0 +1,68 @@
+"""Download WRI Global Power Plant Database CSV and convert to compact JSON.
+
+Usage:
+    python backend/scripts/convert_power_plants.py
+
+Output:
+    backend/data/power_plants.json
+"""
+import csv
+import json
+import io
+import zipfile
+import urllib.request
+from pathlib import Path
+
+# WRI Global Power Plant Database v1.3.0 (GitHub release)
+CSV_URL = "https://raw.githubusercontent.com/wri/global-power-plant-database/master/output_database/global_power_plant_database.csv"
+OUT_PATH = Path(__file__).parent.parent / "data" / "power_plants.json"
+
+
+def main() -> None:
+    print(f"Downloading WRI Global Power Plant Database from GitHub...")
+    # Round 7a: release-time data refresher. Uses the per-operator UA if
+    # available, otherwise a release-script-specific identifier. This
+    # script is run by the maintainer at release time, NOT at runtime,
+    # so an aggregate UA is acceptable; we still use the helper so the
+    # behavior matches the rest of the project.
+    try:
+        from services.network_utils import outbound_user_agent
+        ua = outbound_user_agent("release-script-power-plants")
+    except Exception:
+        ua = "operator-release-script (purpose: power-plants)"
+    req = urllib.request.Request(CSV_URL, headers={"User-Agent": ua})
+    with urllib.request.urlopen(req, timeout=60) as resp:
+        raw = resp.read().decode("utf-8")
+
+    reader = csv.DictReader(io.StringIO(raw))
+    plants: list[dict] = []
+    skipped = 0
+    for row in reader:
+        try:
+            lat = float(row["latitude"])
+            lng = float(row["longitude"])
+        except (ValueError, KeyError):
+            skipped += 1
+            continue
+        if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+            skipped += 1
+            continue
+        capacity_raw = row.get("capacity_mw", "")
+        capacity_mw = float(capacity_raw) if capacity_raw else None
+        plants.append({
+            "name": row.get("name", "Unknown"),
+            "country": row.get("country_long", ""),
+            "fuel_type": row.get("primary_fuel", "Unknown"),
+            "capacity_mw": capacity_mw,
+            "owner": row.get("owner", ""),
+            "lat": round(lat, 5),
+            "lng": round(lng, 5),
+        })
+
+    OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
+    OUT_PATH.write_text(json.dumps(plants, ensure_ascii=False, separators=(",", ":")), encoding="utf-8")
+    print(f"Wrote {len(plants)} power plants to {OUT_PATH} (skipped {skipped})")
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/diagnostics.py

@@ -0,0 +1,45 @@
+from datetime import datetime
+from services.data_fetcher import get_latest_data
+from services.fetchers._store import source_timestamps, active_layers, source_freshness
+from services.fetch_health import get_health_snapshot
+
+
+def _fmt_ts(ts: str | None) -> str:
+    if not ts:
+        return "-"
+    try:
+        return datetime.fromisoformat(ts).strftime("%Y-%m-%d %H:%M:%S")
+    except Exception:
+        return ts
+
+
+def main():
+    data = get_latest_data()
+    print("=== Diagnostics ===")
+    print(f"Last updated: {_fmt_ts(data.get('last_updated'))}")
+    print(
+        f"Active layers: {sum(1 for v in active_layers.values() if v)} enabled / {len(active_layers)} total"
+    )
+
+    print("\n--- Source Timestamps ---")
+    for k, v in sorted(source_timestamps.items()):
+        print(f"{k:20} {_fmt_ts(v)}")
+
+    print("\n--- Source Freshness ---")
+    for k, v in sorted(source_freshness.items()):
+        last_ok = _fmt_ts(v.get("last_ok"))
+        last_err = _fmt_ts(v.get("last_error"))
+        print(f"{k:20} ok={last_ok} err={last_err}")
+
+    print("\n--- Fetch Health ---")
+    health = get_health_snapshot()
+    for k, v in sorted(health.items()):
+        print(
+            f"{k:20} ok={v.get('ok_count', 0)} err={v.get('error_count', 0)} "
+            f"last_ok={_fmt_ts(v.get('last_ok'))} last_err={_fmt_ts(v.get('last_error'))} "
+            f"avg_ms={v.get('avg_duration_ms')}"
+        )
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/release_helper.py

@@ -0,0 +1,308 @@
+import argparse
+import hashlib
+import json
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[2]
+PACKAGE_JSON = ROOT / "frontend" / "package.json"
+
+
+def _normalize_version(raw: str) -> str:
+    version = str(raw or "").strip()
+    if version.startswith("v"):
+        version = version[1:]
+    parts = version.split(".")
+    if len(parts) != 3 or not all(part.isdigit() for part in parts):
+        raise ValueError("Version must look like X.Y.Z")
+    return version
+
+
+def _read_package_json() -> dict:
+    return json.loads(PACKAGE_JSON.read_text(encoding="utf-8"))
+
+
+def _write_package_json(data: dict) -> None:
+    PACKAGE_JSON.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
+
+
+def current_version() -> str:
+    return str(_read_package_json().get("version") or "").strip()
+
+
+def set_version(version: str) -> str:
+    normalized = _normalize_version(version)
+    data = _read_package_json()
+    data["version"] = normalized
+    _write_package_json(data)
+    return normalized
+
+
+def expected_tag(version: str) -> str:
+    return f"v{_normalize_version(version)}"
+
+
+def expected_asset(version: str) -> str:
+    normalized = _normalize_version(version)
+    return f"ShadowBroker_v{normalized}.zip"
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 128), b""):
+            digest.update(chunk)
+    return digest.hexdigest().lower()
+
+
+def _default_generated_at() -> str:
+    return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def build_release_attestation(
+    *,
+    suite_green: bool,
+    suite_name: str = "dm_relay_security",
+    detail: str = "",
+    report: str = "",
+    command: str = "",
+    commit: str = "",
+    generated_at: str = "",
+    threat_model_reference: str = "docs/mesh/threat-model.md",
+    workflow: str = "",
+    run_id: str = "",
+    run_attempt: str = "",
+    ref: str = "",
+) -> dict:
+    normalized_generated_at = str(generated_at or "").strip() or _default_generated_at()
+    normalized_commit = str(commit or "").strip() or os.environ.get("GITHUB_SHA", "").strip()
+    normalized_workflow = str(workflow or "").strip() or os.environ.get("GITHUB_WORKFLOW", "").strip()
+    normalized_run_id = str(run_id or "").strip() or os.environ.get("GITHUB_RUN_ID", "").strip()
+    normalized_run_attempt = str(run_attempt or "").strip() or os.environ.get("GITHUB_RUN_ATTEMPT", "").strip()
+    normalized_ref = str(ref or "").strip() or os.environ.get("GITHUB_REF", "").strip()
+    normalized_suite_name = str(suite_name or "").strip() or "dm_relay_security"
+    normalized_report = str(report or "").strip()
+    normalized_command = str(command or "").strip()
+    normalized_detail = str(detail or "").strip() or (
+        "CI attestation confirms the DM relay security suite is green."
+        if suite_green
+        else "CI attestation recorded a failing DM relay security suite run."
+    )
+    payload = {
+        "generated_at": normalized_generated_at,
+        "commit": normalized_commit,
+        "threat_model_reference": str(threat_model_reference or "").strip()
+        or "docs/mesh/threat-model.md",
+        "dm_relay_security_suite": {
+            "name": normalized_suite_name,
+            "green": bool(suite_green),
+            "detail": normalized_detail,
+            "report": normalized_report,
+        },
+    }
+    if normalized_command:
+        payload["dm_relay_security_suite"]["command"] = normalized_command
+    ci = {
+        "workflow": normalized_workflow,
+        "run_id": normalized_run_id,
+        "run_attempt": normalized_run_attempt,
+        "ref": normalized_ref,
+    }
+    if any(ci.values()):
+        payload["ci"] = ci
+    return payload
+
+
+def write_release_attestation(output_path: Path | str, **kwargs) -> dict:
+    path = Path(output_path).resolve()
+    payload = build_release_attestation(**kwargs)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+    return payload
+
+
+def cmd_show(_args: argparse.Namespace) -> int:
+    version = current_version()
+    if not version:
+        print("package.json has no version", file=sys.stderr)
+        return 1
+    print(f"package.json version : {version}")
+    print(f"expected git tag     : {expected_tag(version)}")
+    print(f"expected zip asset   : {expected_asset(version)}")
+    return 0
+
+
+def cmd_set_version(args: argparse.Namespace) -> int:
+    version = set_version(args.version)
+    print(f"Set frontend/package.json version to {version}")
+    print(f"Next release tag  : {expected_tag(version)}")
+    print(f"Next zip asset    : {expected_asset(version)}")
+    return 0
+
+
+def cmd_hash(args: argparse.Namespace) -> int:
+    version = _normalize_version(args.version) if args.version else current_version()
+    if not version:
+        print("No version available; pass --version or set frontend/package.json", file=sys.stderr)
+        return 1
+
+    zip_path = Path(args.zip_path).resolve()
+    if not zip_path.is_file():
+        print(f"ZIP not found: {zip_path}", file=sys.stderr)
+        return 1
+
+    digest = sha256_file(zip_path)
+    expected_name = expected_asset(version)
+    asset_matches = zip_path.name == expected_name
+
+    print(f"release version     : {version}")
+    print(f"expected git tag    : {expected_tag(version)}")
+    print(f"zip path            : {zip_path}")
+    print(f"zip name matches    : {'yes' if asset_matches else 'no'}")
+    print(f"expected zip asset  : {expected_name}")
+    print(f"SHA-256             : {digest}")
+    print("")
+    print("Updater pin:")
+    print(f"MESH_UPDATE_SHA256={digest}")
+    print("")
+    print("Release checklist:")
+    print("  - add this digest to SHA256SUMS.txt for the GitHub release")
+    print("  - add/update backend/data/release_digests.json for bundled updater verification")
+    print("  - keep MESH_UPDATE_SHA256 available as the operator override path")
+    return 0 if asset_matches else 2
+
+
+def cmd_write_attestation(args: argparse.Namespace) -> int:
+    suite_green = bool(args.suite_green)
+    payload = write_release_attestation(
+        args.output_path,
+        suite_green=suite_green,
+        suite_name=args.suite_name,
+        detail=args.detail,
+        report=args.report,
+        command=args.command,
+        commit=args.commit,
+        generated_at=args.generated_at,
+        threat_model_reference=args.threat_model_reference,
+        workflow=args.workflow,
+        run_id=args.run_id,
+        run_attempt=args.run_attempt,
+        ref=args.ref,
+    )
+    output_path = Path(args.output_path).resolve()
+    print(f"Wrote release attestation: {output_path}")
+    print(f"DM relay security suite : {'green' if suite_green else 'red'}")
+    print(f"Commit                  : {payload.get('commit', '')}")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Helper for ShadowBroker release version/tag/asset consistency."
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    show_parser = subparsers.add_parser("show", help="Show current version, expected tag, and asset")
+    show_parser.set_defaults(func=cmd_show)
+
+    set_version_parser = subparsers.add_parser("set-version", help="Update frontend/package.json version")
+    set_version_parser.add_argument("version", help="Version like 0.9.7")
+    set_version_parser.set_defaults(func=cmd_set_version)
+
+    hash_parser = subparsers.add_parser(
+        "hash", help="Compute SHA-256 for a release ZIP and print the updater pin"
+    )
+    hash_parser.add_argument("zip_path", help="Path to the release ZIP")
+    hash_parser.add_argument(
+        "--version",
+        help="Release version like 0.9.7. Defaults to frontend/package.json version.",
+    )
+    hash_parser.set_defaults(func=cmd_hash)
+
+    attestation_parser = subparsers.add_parser(
+        "write-attestation",
+        help="Write a structured Sprint 8 release attestation JSON file",
+    )
+    attestation_parser.add_argument("output_path", help="Where to write the attestation JSON")
+    suite_group = attestation_parser.add_mutually_exclusive_group(required=True)
+    suite_group.add_argument(
+        "--suite-green",
+        action="store_true",
+        help="Mark the DM relay security suite as green",
+    )
+    suite_group.add_argument(
+        "--suite-red",
+        action="store_true",
+        help="Mark the DM relay security suite as failing",
+    )
+    attestation_parser.add_argument(
+        "--suite-name",
+        default="dm_relay_security",
+        help="Suite name to record in the attestation",
+    )
+    attestation_parser.add_argument(
+        "--detail",
+        default="",
+        help="Human-readable suite detail. Defaults to a CI-generated message.",
+    )
+    attestation_parser.add_argument(
+        "--report",
+        default="",
+        help="Path to the suite report or artifact reference to embed in the attestation.",
+    )
+    attestation_parser.add_argument(
+        "--command",
+        default="",
+        help="Exact suite command used to generate the attestation.",
+    )
+    attestation_parser.add_argument(
+        "--commit",
+        default="",
+        help="Commit SHA. Defaults to GITHUB_SHA when available.",
+    )
+    attestation_parser.add_argument(
+        "--generated-at",
+        default="",
+        help="UTC timestamp for the attestation. Defaults to current UTC time.",
+    )
+    attestation_parser.add_argument(
+        "--threat-model-reference",
+        default="docs/mesh/threat-model.md",
+        help="Threat model reference to embed in the attestation.",
+    )
+    attestation_parser.add_argument(
+        "--workflow",
+        default="",
+        help="Workflow name. Defaults to GITHUB_WORKFLOW when available.",
+    )
+    attestation_parser.add_argument(
+        "--run-id",
+        default="",
+        help="Workflow run ID. Defaults to GITHUB_RUN_ID when available.",
+    )
+    attestation_parser.add_argument(
+        "--run-attempt",
+        default="",
+        help="Workflow run attempt. Defaults to GITHUB_RUN_ATTEMPT when available.",
+    )
+    attestation_parser.add_argument(
+        "--ref",
+        default="",
+        help="Git ref. Defaults to GITHUB_REF when available.",
+    )
+    attestation_parser.set_defaults(func=cmd_write_attestation)
+
+    return parser
+
+
+def main() -> int:
+    parser = build_parser()
+    args = parser.parse_args()
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/scripts/repair_wormhole_secure_storage.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from services.mesh import mesh_secure_storage
+from services.mesh.mesh_wormhole_contacts import CONTACTS_FILE
+from services.mesh.mesh_wormhole_identity import IDENTITY_FILE, _default_identity
+from services.mesh.mesh_wormhole_persona import PERSONA_FILE, _default_state as _default_persona_state
+from services.mesh.mesh_wormhole_ratchet import STATE_FILE as RATCHET_FILE
+
+
+def _load_payloads() -> dict[Path, object]:
+    return {
+        IDENTITY_FILE: mesh_secure_storage.read_secure_json(IDENTITY_FILE, _default_identity),
+        PERSONA_FILE: mesh_secure_storage.read_secure_json(PERSONA_FILE, _default_persona_state),
+        RATCHET_FILE: mesh_secure_storage.read_secure_json(RATCHET_FILE, lambda: {}),
+        CONTACTS_FILE: mesh_secure_storage.read_secure_json(CONTACTS_FILE, lambda: {}),
+    }
+
+
+def main() -> None:
+    payloads = _load_payloads()
+
+    master_key_file = mesh_secure_storage.MASTER_KEY_FILE
+    backup_key_file = master_key_file.with_suffix(master_key_file.suffix + ".bak")
+    if master_key_file.exists():
+        if backup_key_file.exists():
+            backup_key_file.unlink()
+        master_key_file.replace(backup_key_file)
+
+    for path, payload in payloads.items():
+        mesh_secure_storage.write_secure_json(path, payload)
+
+    print(
+        json.dumps(
+            {
+                "ok": True,
+                "rewrapped": [str(path.name) for path in payloads.keys()],
+                "master_key": str(master_key_file),
+                "backup_master_key": str(backup_key_file) if backup_key_file.exists() else "",
+            }
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/rotate_secure_storage_secret.py

@@ -0,0 +1,75 @@
+"""Rotate the MESH_SECURE_STORAGE_SECRET used to protect key envelopes at rest.
+
+Usage — stop the backend first, then run:
+
+    MESH_OLD_STORAGE_SECRET=<current>  \\
+    MESH_NEW_STORAGE_SECRET=<new>      \\
+    python -m scripts.rotate_secure_storage_secret
+
+Dry-run mode (validates old secret without writing anything):
+
+    MESH_OLD_STORAGE_SECRET=<current>  \\
+    MESH_NEW_STORAGE_SECRET=<new>      \\
+    python -m scripts.rotate_secure_storage_secret --dry-run
+
+Or, for Docker deployments:
+
+    docker exec -e MESH_OLD_STORAGE_SECRET=<current> \\
+                -e MESH_NEW_STORAGE_SECRET=<new>      \\
+                <container> python -m scripts.rotate_secure_storage_secret
+
+After successful rotation, update your .env (or Docker secret file) to set
+MESH_SECURE_STORAGE_SECRET to the new value, then restart the backend.
+
+The script fails closed: if the old secret cannot unwrap any existing envelope,
+nothing is written. Non-passphrase envelopes (DPAPI, raw) are skipped with a
+warning.
+
+Before rewriting, .bak copies of every envelope are created so a mid-rotation
+crash leaves recoverable backups on disk.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+
+def main() -> None:
+    dry_run = "--dry-run" in sys.argv
+
+    old_secret = os.environ.get("MESH_OLD_STORAGE_SECRET", "").strip()
+    new_secret = os.environ.get("MESH_NEW_STORAGE_SECRET", "").strip()
+
+    if not old_secret:
+        print("ERROR: MESH_OLD_STORAGE_SECRET environment variable is required.", file=sys.stderr)
+        sys.exit(1)
+    if not new_secret:
+        print("ERROR: MESH_NEW_STORAGE_SECRET environment variable is required.", file=sys.stderr)
+        sys.exit(1)
+
+    from services.mesh.mesh_secure_storage import SecureStorageError, rotate_storage_secret
+
+    try:
+        result = rotate_storage_secret(old_secret, new_secret, dry_run=dry_run)
+    except SecureStorageError as exc:
+        print(f"ROTATION FAILED: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+    print(json.dumps(result, indent=2))
+    if dry_run:
+        print(
+            "\nDry run complete. No files were modified. Run again without --dry-run to perform the rotation.",
+            file=sys.stderr,
+        )
+    else:
+        print(
+            "\nRotation complete. Update MESH_SECURE_STORAGE_SECRET to the new value and restart the backend."
+            "\nBackup files (.bak) were created alongside each rotated envelope.",
+            file=sys.stderr,
+        )
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/scan-secrets.sh

@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+# scan-secrets.sh — Catch keys, secrets, and credentials before they hit git.
+#
+# Usage:
+#   ./backend/scripts/scan-secrets.sh          # Scan staged files (pre-commit)
+#   ./backend/scripts/scan-secrets.sh --all     # Scan entire working tree
+#   ./backend/scripts/scan-secrets.sh --staged  # Scan staged files only (default)
+#
+# Exit code: 0 = clean, 1 = secrets found
+
+set -euo pipefail
+
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+GREEN='\033[0;32m'
+NC='\033[0m'
+
+MODE="${1:---staged}"
+FOUND=0
+
+# ── Get file list based on mode ─────────────────────────────────────────
+if [[ "$MODE" == "--all" ]]; then
+    FILELIST=$(mktemp)
+    { git ls-files 2>/dev/null; git ls-files --others --exclude-standard 2>/dev/null; } > "$FILELIST"
+    echo -e "${YELLOW}Scanning entire working tree...${NC}"
+else
+    FILELIST=$(mktemp)
+    git diff --cached --name-only --diff-filter=ACMR 2>/dev/null > "$FILELIST" || true
+    if [[ ! -s "$FILELIST" ]]; then
+        echo -e "${GREEN}No staged files to scan.${NC}"
+        rm -f "$FILELIST"
+        exit 0
+    fi
+    echo -e "${YELLOW}Scanning $(wc -l < "$FILELIST" | tr -d ' ') staged files...${NC}"
+fi
+
+# ── Check 1: Dangerous file extensions ──────────────────────────────────
+KEY_EXT='\.key$|\.pem$|\.p12$|\.pfx$|\.jks$|\.keystore$|\.p8$|\.der$'
+SECRET_EXT='\.secret$|\.secrets$|\.credential$|\.credentials$'
+
+HITS=$(grep -iE "$KEY_EXT|$SECRET_EXT" "$FILELIST" 2>/dev/null || true)
+if [[ -n "$HITS" ]]; then
+    echo -e "\n${RED}BLOCKED: Key/secret files detected:${NC}"
+    echo "$HITS" | while read -r f; do echo -e "  ${RED}$f${NC}"; done
+    FOUND=1
+fi
+
+# ── Check 2: Dangerous filenames ────────────────────────────────────────
+RISKY='id_rsa|id_ed25519|id_ecdsa|private_key|private\.key|secret_key|master\.key'
+RISKY+='|serviceaccount|gcloud.*\.json|firebase.*\.json|\.htpasswd'
+
+HITS=$(grep -iE "$RISKY" "$FILELIST" 2>/dev/null || true)
+if [[ -n "$HITS" ]]; then
+    echo -e "\n${RED}BLOCKED: Risky filenames detected:${NC}"
+    echo "$HITS" | while read -r f; do echo -e "  ${RED}$f${NC}"; done
+    FOUND=1
+fi
+
+# ── Check 3: .env files (not .env.example) ──────────────────────────────
+HITS=$(grep -E '(^|/)\.env(\.[^e].*)?$' "$FILELIST" 2>/dev/null | grep -v '\.example' || true)
+if [[ -n "$HITS" ]]; then
+    echo -e "\n${RED}BLOCKED: Environment files detected:${NC}"
+    echo "$HITS" | while read -r f; do echo -e "  ${RED}$f${NC}"; done
+    FOUND=1
+fi
+
+# ── Check 4: _domain_keys directory (project-specific) ──────────────────
+HITS=$(grep '_domain_keys/' "$FILELIST" 2>/dev/null || true)
+if [[ -n "$HITS" ]]; then
+    echo -e "\n${RED}BLOCKED: Domain keys directory detected:${NC}"
+    echo "$HITS" | while read -r f; do echo -e "  ${RED}$f${NC}"; done
+    FOUND=1
+fi
+
+# ── Check 5: Content scan for embedded secrets (single grep pass) ───────
+# Build one mega-pattern and run grep once across all files (fast!)
+SECRET_REGEX='PRIVATE KEY-----|'
+SECRET_REGEX+='ssh-rsa AAAA[0-9A-Za-z+/]|'
+SECRET_REGEX+='ssh-ed25519 AAAA[0-9A-Za-z+/]|'
+SECRET_REGEX+='ghp_[0-9a-zA-Z]{36}|'                          # GitHub PAT
+SECRET_REGEX+='github_pat_[0-9a-zA-Z]{22}_[0-9a-zA-Z]{59}|'   # GitHub fine-grained
+SECRET_REGEX+='gho_[0-9a-zA-Z]{36}|'                          # GitHub OAuth
+SECRET_REGEX+='sk-[0-9a-zA-Z]{48}|'                           # OpenAI key
+SECRET_REGEX+='sk-ant-[0-9a-zA-Z-]{90,}|'                     # Anthropic key
+SECRET_REGEX+='AKIA[0-9A-Z]{16}|'                             # AWS access key
+SECRET_REGEX+='AIzaSy[0-9A-Za-z_-]{33}|'                      # Google API key
+SECRET_REGEX+='xox[bpoas]-[0-9a-zA-Z-]+|'                     # Slack token
+SECRET_REGEX+='npm_[0-9a-zA-Z]{36}|'                          # npm token
+SECRET_REGEX+='pypi-[0-9a-zA-Z-]{50,}'                        # PyPI token
+
+# Filter to text-like files only (skip binaries by extension + skip this script)
+TEXT_FILES=$(grep -ivE '\.(png|jpg|jpeg|gif|ico|svg|woff2?|ttf|eot|pbf|zip|tar|gz|db|sqlite|xlsx|pdf|mp[34]|wav|ogg|webm|webp|avif)$' "$FILELIST" | grep -v 'scan-secrets\.sh$' || true)
+
+if [[ -n "$TEXT_FILES" ]]; then
+    # Known-public exclusions: lines matching `<host-or-ip> ssh-<algo> <key>`
+    # are SSH known_hosts entries — the host's PUBLIC fingerprint, which is
+    # by definition safe to commit (the whole point of pinning known_hosts
+    # is to publish the fingerprint widely so MITM is detectable). Filter
+    # these out before flagging the file.
+    KNOWN_HOSTS_LINE='^[[:space:]]*[a-zA-Z0-9._:,*-]+([[:space:]]+[a-zA-Z0-9._:,*-]+)?[[:space:]]+(ssh-rsa|ssh-ed25519|ssh-dss|ecdsa-sha2-nistp256|ecdsa-sha2-nistp384|ecdsa-sha2-nistp521)[[:space:]]+AAAA'
+
+    # Use grep with file list, skip missing/binary, limit output
+    CONTENT_HITS=$(echo "$TEXT_FILES" | xargs grep -lE "$SECRET_REGEX" 2>/dev/null || true)
+    if [[ -n "$CONTENT_HITS" ]]; then
+        REAL_HITS=""
+        REAL_REPORT=""
+        while IFS= read -r f; do
+            [[ -z "$f" ]] && continue
+            # Re-grep this file, but filter out known_hosts-style lines.
+            FILE_HITS=$(grep -nE "$SECRET_REGEX" "$f" 2>/dev/null | grep -vE "$KNOWN_HOSTS_LINE" || true)
+            if [[ -n "$FILE_HITS" ]]; then
+                REAL_HITS+="$f"$'\n'
+                REAL_REPORT+="  ${RED}$f${NC}"$'\n'
+                # Show first 2 matching lines for context
+                while IFS= read -r line; do
+                    [[ -z "$line" ]] && continue
+                    REAL_REPORT+="    ${YELLOW}$line${NC}"$'\n'
+                done < <(echo "$FILE_HITS" | head -2)
+            fi
+        done <<< "$CONTENT_HITS"
+        if [[ -n "$REAL_HITS" ]]; then
+            echo -e "\n${RED}BLOCKED: Embedded secrets/tokens found in:${NC}"
+            echo -en "$REAL_REPORT"
+            FOUND=1
+        fi
+    fi
+fi
+
+rm -f "$FILELIST"
+
+# ── Result ──────────────────────────────────────────────────────────────
+echo ""
+if [[ $FOUND -eq 1 ]]; then
+    echo -e "${RED}Secret scan FAILED. Add these to .gitignore or remove them before committing.${NC}"
+    echo -e "${YELLOW}If intentional (e.g. test fixtures): git commit --no-verify${NC}"
+    exit 1
+else
+    echo -e "${GREEN}Secret scan passed. No keys or secrets detected.${NC}"
+    exit 0
+fi

backend/scripts/setup-venv.ps1

@@ -0,0 +1,16 @@
+param(
+  [string]$Python = "py"
+)
+
+$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
+$venvPath = Join-Path $repoRoot "venv"
+$venvMarker = Join-Path $repoRoot ".venv-dir"
+& $Python -3.11 -m venv $venvPath
+
+$pip = Join-Path $venvPath "Scripts\pip.exe"
+& $pip install --upgrade pip
+Push-Location $repoRoot
+& (Join-Path $venvPath "Scripts\python.exe") -m pip install -e .
+& $pip install pytest pytest-asyncio ruff black
+"venv" | Set-Content -LiteralPath $venvMarker -NoNewline
+Pop-Location

backend/scripts/setup-venv.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+PYTHON="${PYTHON:-python3.11}"
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+VENV_DIR="$REPO_ROOT/venv"
+VENV_MARKER="$REPO_ROOT/.venv-dir"
+
+"$PYTHON" -m venv "$VENV_DIR"
+"$VENV_DIR/bin/pip" install --upgrade pip
+cd "$REPO_ROOT"
+"$VENV_DIR/bin/python" -m pip install -e .
+"$VENV_DIR/bin/pip" install pytest pytest-asyncio ruff black
+printf 'venv\n' > "$VENV_MARKER"

backend/seattle_sample.json

@@ -1,8 +0,0 @@
-{
-  "code" : "dataset.missing",
-  "error" : true,
-  "message" : "Not found",
-  "data" : {
-    "id" : "xqwu-hwdm"
-  }
-}

backend/services/ai_intel_store.py

@@ -0,0 +1,178 @@
+"""ai_intel_store — compatibility wrapper around ai_pin_store + layer injection.
+
+openclaw_channel.py and routers/ai_intel.py import from this module name.
+All pin/layer logic lives in ai_pin_store.py; this module re-exports with the
+expected function signatures and adds the layer injection helper.
+"""
+
+import logging
+import time
+from typing import Any
+
+from services.ai_pin_store import (
+    create_pin,
+    create_pins_batch,
+    get_pins,
+    delete_pin,
+    clear_pins,
+    pin_count,
+    pins_as_geojson,
+    purge_expired,
+    # Layer CRUD
+    create_layer,
+    get_layers,
+    update_layer,
+    delete_layer,
+    # Feed layers
+    get_feed_layers,
+    replace_layer_pins,
+)
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Re-exports expected by openclaw_channel._dispatch_command
+# ---------------------------------------------------------------------------
+
+
+def get_all_intel_pins() -> list[dict[str, Any]]:
+    """Return all active pins (no filter, generous limit)."""
+    return get_pins(limit=2000)
+
+
+def add_intel_pin(args: dict[str, Any]) -> dict[str, Any]:
+    """Create a single pin from a command-channel args dict."""
+    ea = args.get("entity_attachment")
+    return create_pin(
+        lat=float(args.get("lat", 0)),
+        lng=float(args.get("lng", 0)),
+        label=str(args.get("label", ""))[:200],
+        category=str(args.get("category", "custom")),
+        layer_id=str(args.get("layer_id", "")),
+        color=str(args.get("color", "")),
+        description=str(args.get("description", "")),
+        source=str(args.get("source", "openclaw")),
+        source_url=str(args.get("source_url", "")),
+        confidence=float(args.get("confidence", 1.0)),
+        ttl_hours=float(args.get("ttl_hours", 0)),
+        metadata=args.get("metadata") or {},
+        entity_attachment=ea if isinstance(ea, dict) else None,
+    )
+
+
+def delete_intel_pin(pin_id: str) -> bool:
+    """Delete a pin by ID."""
+    return delete_pin(pin_id)
+
+
+# Layer helpers for OpenClaw
+def create_intel_layer(args: dict[str, Any]) -> dict[str, Any]:
+    """Create a layer from a command-channel args dict."""
+    return create_layer(
+        name=str(args.get("name", "Untitled"))[:100],
+        description=str(args.get("description", ""))[:500],
+        source=str(args.get("source", "openclaw"))[:50],
+        color=str(args.get("color", "")),
+        feed_url=str(args.get("feed_url", "")),
+        feed_interval=int(args.get("feed_interval", 300)),
+    )
+
+
+def get_intel_layers() -> list[dict[str, Any]]:
+    """Return all layers with pin counts."""
+    return get_layers()
+
+
+def update_intel_layer(layer_id: str, args: dict[str, Any]) -> dict[str, Any] | None:
+    """Update a layer from a command-channel args dict."""
+    return update_layer(layer_id, **{
+        k: v for k, v in args.items()
+        if k in ("name", "description", "visible", "color", "feed_url", "feed_interval")
+    })
+
+
+def delete_intel_layer(layer_id: str) -> int:
+    """Delete a layer and its pins. Returns pin count removed."""
+    return delete_layer(layer_id)
+
+
+# ---------------------------------------------------------------------------
+# Layer injection — inserts agent data into native telemetry layers
+# ---------------------------------------------------------------------------
+
+# Layers that agents are allowed to inject into.
+_INJECTABLE_LAYERS = frozenset({
+    "cctv", "ships", "sigint", "kiwisdr", "military_bases",
+    "datacenters", "power_plants", "satnogs_stations",
+    "volcanoes", "earthquakes", "news", "viirs_change_nodes",
+    "air_quality",
+})
+
+
+def inject_layer_data(
+    layer: str,
+    items: list[dict[str, Any]],
+    mode: str = "append",
+) -> dict[str, Any]:
+    """Inject agent data into a native telemetry layer."""
+    from services.fetchers._store import latest_data, _data_lock, bump_data_version
+
+    layer = str(layer or "").strip()
+    if layer not in _INJECTABLE_LAYERS:
+        return {"ok": False, "detail": f"layer '{layer}' not injectable"}
+
+    items = list(items or [])[:200]
+    if not items:
+        return {"ok": False, "detail": "no items provided"}
+
+    now = time.time()
+    tagged = []
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        entry = dict(item)
+        entry["_injected"] = True
+        entry["_source"] = "user:openclaw"
+        entry["_injected_at"] = now
+        tagged.append(entry)
+
+    with _data_lock:
+        existing = latest_data.get(layer)
+        if not isinstance(existing, list):
+            existing = []
+
+        if mode == "replace":
+            existing = [e for e in existing if not e.get("_injected")]
+
+        existing.extend(tagged)
+        latest_data[layer] = existing
+
+    bump_data_version()
+
+    return {
+        "ok": True,
+        "layer": layer,
+        "injected": len(tagged),
+        "mode": mode,
+    }
+
+
+def clear_injected_data(layer: str = "") -> dict[str, Any]:
+    """Remove all injected items from a layer (or all layers)."""
+    from services.fetchers._store import latest_data, _data_lock, bump_data_version
+
+    removed = 0
+    with _data_lock:
+        targets = [layer] if layer else list(_INJECTABLE_LAYERS)
+        for lyr in targets:
+            existing = latest_data.get(lyr)
+            if not isinstance(existing, list):
+                continue
+            before = len(existing)
+            latest_data[lyr] = [e for e in existing if not e.get("_injected")]
+            removed += before - len(latest_data[lyr])
+
+    if removed:
+        bump_data_version()
+
+    return {"ok": True, "removed": removed}

backend/services/ai_pin_store.py

@@ -0,0 +1,633 @@
+"""AI Intel pin storage — layered pin system with JSON file persistence.
+
+Supports:
+  - Named pin layers (created by user or AI)
+  - Pins with optional entity attachment (track moving objects)
+  - Pin source tracking (user vs openclaw)
+  - Layer visibility toggles
+  - External feed URL per layer (for Phase 5)
+  - GeoJSON export per layer or all layers
+"""
+
+import json
+import logging
+import os
+import threading
+import time
+import uuid
+from datetime import datetime
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Pin schema
+# ---------------------------------------------------------------------------
+
+PIN_CATEGORIES = {
+    "threat", "news", "geolocation", "custom", "anomaly",
+    "military", "maritime", "flight", "infrastructure", "weather",
+    "sigint", "prediction", "research",
+}
+
+PIN_COLORS = {
+    "threat": "#ef4444",       # red
+    "news": "#f59e0b",         # amber
+    "geolocation": "#8b5cf6",  # violet
+    "custom": "#3b82f6",       # blue
+    "anomaly": "#f97316",      # orange
+    "military": "#dc2626",     # dark red
+    "maritime": "#0ea5e9",     # sky
+    "flight": "#6366f1",       # indigo
+    "infrastructure": "#64748b",  # slate
+    "weather": "#22d3ee",      # cyan
+    "sigint": "#a855f7",       # purple
+    "prediction": "#eab308",   # yellow
+    "research": "#10b981",     # emerald
+}
+
+LAYER_COLORS = [
+    "#3b82f6", "#ef4444", "#22d3ee", "#f59e0b", "#8b5cf6",
+    "#10b981", "#f97316", "#6366f1", "#ec4899", "#14b8a6",
+]
+
+# ---------------------------------------------------------------------------
+# In-memory store
+# ---------------------------------------------------------------------------
+
+_layers: list[dict[str, Any]] = []
+_pins: list[dict[str, Any]] = []
+_lock = threading.Lock()
+
+# Persistence file path
+_PERSIST_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data")
+_PERSIST_FILE = os.path.join(_PERSIST_DIR, "pin_layers.json")
+_OLD_PERSIST_FILE = os.path.join(_PERSIST_DIR, "ai_pins.json")
+
+
+def _ensure_persist_dir():
+    try:
+        os.makedirs(_PERSIST_DIR, exist_ok=True)
+    except OSError:
+        pass
+
+
+def _save_to_disk():
+    """Persist layers and pins to JSON file. Called under lock."""
+    try:
+        _ensure_persist_dir()
+        with open(_PERSIST_FILE, "w", encoding="utf-8") as f:
+            json.dump({"layers": _layers, "pins": _pins}, f, indent=2, default=str)
+    except (OSError, IOError) as e:
+        logger.warning(f"Failed to persist pin layers: {e}")
+
+
+def _load_from_disk():
+    """Load layers and pins from disk on startup."""
+    global _layers, _pins
+    try:
+        if os.path.exists(_PERSIST_FILE):
+            with open(_PERSIST_FILE, "r", encoding="utf-8") as f:
+                data = json.load(f)
+            if isinstance(data, dict):
+                _layers = data.get("layers", [])
+                _pins = data.get("pins", [])
+                logger.info(f"Loaded {len(_layers)} layers, {len(_pins)} pins from disk")
+                return
+
+        # Migrate from old flat pin file
+        if os.path.exists(_OLD_PERSIST_FILE):
+            with open(_OLD_PERSIST_FILE, "r", encoding="utf-8") as f:
+                old_pins = json.load(f)
+            if isinstance(old_pins, list) and old_pins:
+                legacy_layer = _make_layer("Legacy", "Migrated pins", source="system")
+                _layers.append(legacy_layer)
+                for p in old_pins:
+                    if isinstance(p, dict):
+                        p["layer_id"] = legacy_layer["id"]
+                        _pins.append(p)
+                logger.info(f"Migrated {len(_pins)} pins from ai_pins.json into Legacy layer")
+                _save_to_disk()
+    except (OSError, IOError, json.JSONDecodeError) as e:
+        logger.warning(f"Failed to load pin layers from disk: {e}")
+
+
+def _make_layer(
+    name: str,
+    description: str = "",
+    source: str = "user",
+    color: str = "",
+    feed_url: str = "",
+    feed_interval: int = 300,
+) -> dict[str, Any]:
+    """Create a layer dict."""
+    layer_id = str(uuid.uuid4())[:12]
+    now = time.time()
+    return {
+        "id": layer_id,
+        "name": name[:100],
+        "description": description[:500],
+        "source": source[:50],
+        "visible": True,
+        "color": color or LAYER_COLORS[len(_layers) % len(LAYER_COLORS)],
+        "created_at": now,
+        "created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
+        "feed_url": feed_url[:1000] if feed_url else "",
+        "feed_interval": max(60, min(86400, feed_interval)),
+        "pin_count": 0,
+    }
+
+
+# Load on import
+_load_from_disk()
+
+# One-time cleanup: remove correlation_engine auto-pins (no longer generated)
+_corr_before = len(_pins)
+_pins[:] = [p for p in _pins if p.get("source") != "correlation_engine"]
+if len(_pins) < _corr_before:
+    logger.info("Cleaned up %d legacy correlation_engine pins", _corr_before - len(_pins))
+    _save_to_disk()
+
+
+# ---------------------------------------------------------------------------
+# Layer CRUD
+# ---------------------------------------------------------------------------
+
+def create_layer(
+    name: str,
+    description: str = "",
+    source: str = "user",
+    color: str = "",
+    feed_url: str = "",
+    feed_interval: int = 300,
+) -> dict[str, Any]:
+    """Create a new pin layer."""
+    with _lock:
+        layer = _make_layer(name, description, source, color, feed_url, feed_interval)
+        _layers.append(layer)
+        _save_to_disk()
+    return layer
+
+
+def get_layers() -> list[dict[str, Any]]:
+    """Return all layers with current pin counts."""
+    now = time.time()
+    with _lock:
+        result = []
+        for layer in _layers:
+            count = sum(
+                1 for p in _pins
+                if p.get("layer_id") == layer["id"]
+                and not (p.get("expires_at") and p["expires_at"] < now)
+            )
+            result.append({**layer, "pin_count": count})
+        return result
+
+
+def update_layer(layer_id: str, **updates) -> Optional[dict[str, Any]]:
+    """Update layer fields. Returns updated layer or None if not found."""
+    allowed = {"name", "description", "visible", "color", "feed_url", "feed_interval", "feed_last_fetched"}
+    with _lock:
+        for layer in _layers:
+            if layer["id"] == layer_id:
+                for k, v in updates.items():
+                    if k in allowed and v is not None:
+                        if k == "name":
+                            layer[k] = str(v)[:100]
+                        elif k == "description":
+                            layer[k] = str(v)[:500]
+                        elif k == "visible":
+                            layer[k] = bool(v)
+                        elif k == "color":
+                            layer[k] = str(v)[:20]
+                        elif k == "feed_url":
+                            layer[k] = str(v)[:1000]
+                        elif k == "feed_interval":
+                            layer[k] = max(60, min(86400, int(v)))
+                        elif k == "feed_last_fetched":
+                            layer[k] = float(v)
+                _save_to_disk()
+                return dict(layer)
+    return None
+
+
+def delete_layer(layer_id: str) -> int:
+    """Delete a layer and all its pins. Returns count of pins removed."""
+    with _lock:
+        before_layers = len(_layers)
+        _layers[:] = [l for l in _layers if l["id"] != layer_id]
+        if len(_layers) == before_layers:
+            return 0  # not found
+        before_pins = len(_pins)
+        _pins[:] = [p for p in _pins if p.get("layer_id") != layer_id]
+        removed = before_pins - len(_pins)
+        _save_to_disk()
+        return removed
+
+
+# ---------------------------------------------------------------------------
+# Pin CRUD
+# ---------------------------------------------------------------------------
+
+def create_pin(
+    lat: float,
+    lng: float,
+    label: str,
+    category: str = "custom",
+    *,
+    layer_id: str = "",
+    color: str = "",
+    description: str = "",
+    source: str = "openclaw",
+    source_url: str = "",
+    confidence: float = 1.0,
+    ttl_hours: float = 0,
+    metadata: Optional[dict] = None,
+    entity_attachment: Optional[dict] = None,
+) -> dict[str, Any]:
+    """Create a single pin and return it."""
+    pin_id = str(uuid.uuid4())[:12]
+    now = time.time()
+
+    cat = category if category in PIN_CATEGORIES else "custom"
+    pin_color = color or PIN_COLORS.get(cat, "#3b82f6")
+
+    # Validate entity_attachment if provided
+    attachment = None
+    if entity_attachment and isinstance(entity_attachment, dict):
+        etype = str(entity_attachment.get("entity_type", "")).strip()
+        eid = str(entity_attachment.get("entity_id", "")).strip()
+        if etype and eid:
+            attachment = {
+                "entity_type": etype[:50],
+                "entity_id": eid[:100],
+                "entity_label": str(entity_attachment.get("entity_label", ""))[:200],
+            }
+
+    pin = {
+        "id": pin_id,
+        "layer_id": layer_id or "",
+        "lat": lat,
+        "lng": lng,
+        "label": label[:200],
+        "category": cat,
+        "color": pin_color,
+        "description": description[:2000],
+        "source": source[:100],
+        "source_url": source_url[:500],
+        "confidence": max(0.0, min(1.0, confidence)),
+        "created_at": now,
+        "created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
+        "expires_at": now + (ttl_hours * 3600) if ttl_hours > 0 else None,
+        "metadata": metadata or {},
+        "entity_attachment": attachment,
+        "comments": [],
+    }
+
+    with _lock:
+        _pins.append(pin)
+        _save_to_disk()
+
+    return pin
+
+
+def create_pins_batch(items: list[dict], default_layer_id: str = "") -> list[dict[str, Any]]:
+    """Create multiple pins at once."""
+    created = []
+    now = time.time()
+
+    with _lock:
+        for item in items[:200]:  # max 200 per batch
+            pin_id = str(uuid.uuid4())[:12]
+            cat = item.get("category", "custom")
+            if cat not in PIN_CATEGORIES:
+                cat = "custom"
+            pin_color = item.get("color", "") or PIN_COLORS.get(cat, "#3b82f6")
+            ttl = float(item.get("ttl_hours", 0) or 0)
+
+            attachment = None
+            ea = item.get("entity_attachment")
+            if ea and isinstance(ea, dict):
+                etype = str(ea.get("entity_type", "")).strip()
+                eid = str(ea.get("entity_id", "")).strip()
+                if etype and eid:
+                    attachment = {
+                        "entity_type": etype[:50],
+                        "entity_id": eid[:100],
+                        "entity_label": str(ea.get("entity_label", ""))[:200],
+                    }
+
+            pin = {
+                "id": pin_id,
+                "layer_id": item.get("layer_id", default_layer_id) or "",
+                "lat": float(item.get("lat", 0)),
+                "lng": float(item.get("lng", 0)),
+                "label": str(item.get("label", ""))[:200],
+                "category": cat,
+                "color": pin_color,
+                "description": str(item.get("description", ""))[:2000],
+                "source": str(item.get("source", "openclaw"))[:100],
+                "source_url": str(item.get("source_url", ""))[:500],
+                "confidence": max(0.0, min(1.0, float(item.get("confidence", 1.0)))),
+                "created_at": now,
+                "created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
+                "expires_at": now + (ttl * 3600) if ttl > 0 else None,
+                "metadata": item.get("metadata", {}),
+                "entity_attachment": attachment,
+                "comments": [],
+            }
+            _pins.append(pin)
+            created.append(pin)
+
+        _save_to_disk()
+    return created
+
+
+def get_pins(
+    category: str = "",
+    source: str = "",
+    layer_id: str = "",
+    limit: int = 500,
+    include_expired: bool = False,
+) -> list[dict[str, Any]]:
+    """Get pins with optional filters."""
+    now = time.time()
+    with _lock:
+        results = []
+        for pin in _pins:
+            if not include_expired and pin.get("expires_at") and pin["expires_at"] < now:
+                continue
+            if category and pin.get("category") != category:
+                continue
+            if source and pin.get("source") != source:
+                continue
+            if layer_id and pin.get("layer_id") != layer_id:
+                continue
+            results.append(pin)
+            if len(results) >= limit:
+                break
+        return results
+
+
+def get_pin(pin_id: str) -> Optional[dict[str, Any]]:
+    """Return a single pin by ID (including comments), or None."""
+    with _lock:
+        for pin in _pins:
+            if pin.get("id") == pin_id:
+                # Ensure comments key exists for legacy pins
+                if "comments" not in pin:
+                    pin["comments"] = []
+                return dict(pin)
+    return None
+
+
+def update_pin(pin_id: str, **updates) -> Optional[dict[str, Any]]:
+    """Update a pin's editable fields (label, description, category, color)."""
+    allowed = {"label", "description", "category", "color"}
+    with _lock:
+        for pin in _pins:
+            if pin.get("id") != pin_id:
+                continue
+            for k, v in updates.items():
+                if k not in allowed or v is None:
+                    continue
+                if k == "label":
+                    pin[k] = str(v)[:200]
+                elif k == "description":
+                    pin[k] = str(v)[:2000]
+                elif k == "category":
+                    cat = str(v)
+                    if cat in PIN_CATEGORIES:
+                        pin[k] = cat
+                        # Refresh color if it was the category default
+                        if not updates.get("color"):
+                            pin["color"] = PIN_COLORS.get(cat, pin.get("color", "#3b82f6"))
+                elif k == "color":
+                    pin[k] = str(v)[:20]
+            pin["updated_at"] = time.time()
+            _save_to_disk()
+            return dict(pin)
+    return None
+
+
+def add_pin_comment(
+    pin_id: str,
+    text: str,
+    author: str = "user",
+    author_label: str = "",
+    reply_to: str = "",
+) -> Optional[dict[str, Any]]:
+    """Append a comment to a pin. Returns the updated pin (with all comments)."""
+    text = (text or "").strip()
+    if not text:
+        return None
+    with _lock:
+        for pin in _pins:
+            if pin.get("id") != pin_id:
+                continue
+            if "comments" not in pin or not isinstance(pin["comments"], list):
+                pin["comments"] = []
+            comment = {
+                "id": str(uuid.uuid4())[:12],
+                "text": text[:4000],
+                "author": (author or "user")[:50],
+                "author_label": (author_label or "")[:100],
+                "reply_to": (reply_to or "")[:12],
+                "created_at": time.time(),
+                "created_at_iso": datetime.utcnow().isoformat() + "Z",
+            }
+            pin["comments"].append(comment)
+            _save_to_disk()
+            return dict(pin)
+    return None
+
+
+def delete_pin_comment(pin_id: str, comment_id: str) -> bool:
+    """Remove a single comment from a pin."""
+    with _lock:
+        for pin in _pins:
+            if pin.get("id") != pin_id:
+                continue
+            comments = pin.get("comments") or []
+            before = len(comments)
+            pin["comments"] = [c for c in comments if c.get("id") != comment_id]
+            if len(pin["comments"]) < before:
+                _save_to_disk()
+                return True
+            return False
+    return False
+
+
+def delete_pin(pin_id: str) -> bool:
+    """Delete a single pin by ID."""
+    with _lock:
+        before = len(_pins)
+        _pins[:] = [p for p in _pins if p.get("id") != pin_id]
+        if len(_pins) < before:
+            _save_to_disk()
+            return True
+        return False
+
+
+def clear_pins(category: str = "", source: str = "", layer_id: str = "") -> int:
+    """Clear pins, optionally filtered. Returns count removed."""
+    with _lock:
+        before = len(_pins)
+
+        def keep(p):
+            if layer_id and p.get("layer_id") != layer_id:
+                return True  # different layer, keep
+            if category and source:
+                return not (p.get("category") == category and p.get("source") == source)
+            if category:
+                return p.get("category") != category
+            if source:
+                return p.get("source") != source
+            if layer_id:
+                return p.get("layer_id") != layer_id
+            return False
+
+        if not category and not source and not layer_id:
+            _pins.clear()
+        else:
+            _pins[:] = [p for p in _pins if keep(p)]
+
+        removed = before - len(_pins)
+        if removed:
+            _save_to_disk()
+        return removed
+
+
+def get_feed_layers() -> list[dict[str, Any]]:
+    """Return layers that have a non-empty feed_url."""
+    with _lock:
+        return [dict(l) for l in _layers if l.get("feed_url")]
+
+
+def replace_layer_pins(layer_id: str, new_pins: list[dict[str, Any]]) -> int:
+    """Atomically replace all pins in a layer with new_pins. Returns count added."""
+    now = time.time()
+    with _lock:
+        # Remove old pins for this layer
+        _pins[:] = [p for p in _pins if p.get("layer_id") != layer_id]
+        # Add new pins
+        added = 0
+        for item in new_pins[:500]:  # cap at 500 per feed
+            pin_id = str(uuid.uuid4())[:12]
+            cat = item.get("category", "custom")
+            if cat not in PIN_CATEGORIES:
+                cat = "custom"
+            pin_color = item.get("color", "") or PIN_COLORS.get(cat, "#3b82f6")
+
+            attachment = None
+            ea = item.get("entity_attachment")
+            if ea and isinstance(ea, dict):
+                etype = str(ea.get("entity_type", "")).strip()
+                eid = str(ea.get("entity_id", "")).strip()
+                if etype and eid:
+                    attachment = {
+                        "entity_type": etype[:50],
+                        "entity_id": eid[:100],
+                        "entity_label": str(ea.get("entity_label", ""))[:200],
+                    }
+
+            pin = {
+                "id": pin_id,
+                "layer_id": layer_id,
+                "lat": float(item.get("lat", 0)),
+                "lng": float(item.get("lng", 0)),
+                "label": str(item.get("label", item.get("name", "")))[:200],
+                "category": cat,
+                "color": pin_color,
+                "description": str(item.get("description", ""))[:2000],
+                "source": str(item.get("source", "feed"))[:100],
+                "source_url": str(item.get("source_url", ""))[:500],
+                "confidence": max(0.0, min(1.0, float(item.get("confidence", 1.0)))),
+                "created_at": now,
+                "created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
+                "expires_at": None,
+                "metadata": item.get("metadata", {}),
+                "entity_attachment": attachment,
+                "comments": [],
+            }
+            _pins.append(pin)
+            added += 1
+        _save_to_disk()
+    return added
+
+
+def purge_expired() -> int:
+    """Remove expired pins. Called periodically."""
+    now = time.time()
+    with _lock:
+        before = len(_pins)
+        _pins[:] = [p for p in _pins if not (p.get("expires_at") and p["expires_at"] < now)]
+        removed = before - len(_pins)
+        if removed:
+            _save_to_disk()
+        return removed
+
+
+def pin_count() -> dict[str, int]:
+    """Return counts by category."""
+    now = time.time()
+    counts: dict[str, int] = {}
+    with _lock:
+        for pin in _pins:
+            if pin.get("expires_at") and pin["expires_at"] < now:
+                continue
+            cat = pin.get("category", "custom")
+            counts[cat] = counts.get(cat, 0) + 1
+    return counts
+
+
+def pins_as_geojson(layer_id: str = "") -> dict[str, Any]:
+    """Convert active pins to GeoJSON FeatureCollection for the map layer."""
+    now = time.time()
+    features = []
+    with _lock:
+        # Build set of visible layer IDs
+        visible_layers = {l["id"] for l in _layers if l.get("visible", True)}
+
+        for pin in _pins:
+            if pin.get("expires_at") and pin["expires_at"] < now:
+                continue
+            # Layer filter
+            pid_layer = pin.get("layer_id", "")
+            if layer_id and pid_layer != layer_id:
+                continue
+            # Skip pins in hidden layers
+            if pid_layer and pid_layer not in visible_layers:
+                continue
+
+            props = {
+                "id": pin["id"],
+                "layer_id": pid_layer,
+                "label": pin["label"],
+                "category": pin["category"],
+                "color": pin["color"],
+                "description": pin.get("description", ""),
+                "source": pin["source"],
+                "source_url": pin.get("source_url", ""),
+                "confidence": pin.get("confidence", 1.0),
+                "created_at": pin.get("created_at_iso", ""),
+                "comment_count": len(pin.get("comments") or []),
+            }
+
+            # Entity attachment info (frontend resolves position)
+            ea = pin.get("entity_attachment")
+            if ea:
+                props["entity_attachment"] = ea
+
+            features.append({
+                "type": "Feature",
+                "geometry": {
+                    "type": "Point",
+                    "coordinates": [pin["lng"], pin["lat"]],
+                },
+                "properties": props,
+            })
+    return {
+        "type": "FeatureCollection",
+        "features": features,
+    }

backend/services/ais_stream.py

@@ -16,18 +16,31 @@ logger = logging.getLogger(__name__)
 AIS_WS_URL = "wss://stream.aisstream.io/v0/stream"
 API_KEY = os.environ.get("AIS_API_KEY", "")
 
+
+def _env_truthy(name: str) -> bool:
+    return str(os.getenv(name, "")).strip().lower() in {"1", "true", "yes", "on"}
+
+
+def ais_stream_proxy_enabled() -> bool:
+    """Return whether the external Node AIS proxy may be started."""
+    setting = str(os.getenv("SHADOWBROKER_ENABLE_AIS_STREAM_PROXY", "")).strip().lower()
+    if setting:
+        return _env_truthy("SHADOWBROKER_ENABLE_AIS_STREAM_PROXY")
+    return True
+
+
 # AIS vessel type code classification
 # See: https://coast.noaa.gov/data/marinecadastre/ais/VesselTypeCodes2018.pdf
 def classify_vessel(ais_type: int, mmsi: int) -> str:
     """Classify a vessel by its AIS type code into a rendering category."""
     if 80 <= ais_type <= 89:
-        return "tanker"        # Oil/Chemical/Gas tankers → RED
+        return "tanker"  # Oil/Chemical/Gas tankers → RED
     if 70 <= ais_type <= 79:
-        return "cargo"         # Cargo ships, container vessels → RED
+        return "cargo"  # Cargo ships, container vessels → RED
     if 60 <= ais_type <= 69:
-        return "passenger"     # Cruise ships, ferries → GRAY
+        return "passenger"  # Cruise ships, ferries → GRAY
     if ais_type in (36, 37):
-        return "yacht"         # Sailing/Pleasure craft → DARK BLUE
+        return "yacht"  # Sailing/Pleasure craft → DARK BLUE
     if ais_type == 35:
         return "military_vessel"  # Military → YELLOW
     # MMSI-based military detection: military MMSIs often start with certain prefixes
@@ -35,87 +48,286 @@ def classify_vessel(ais_type: int, mmsi: int) -> str:
     if mmsi_str.startswith("3380") or mmsi_str.startswith("3381"):
         return "military_vessel"  # US Navy
     if ais_type in (30, 31, 32, 33, 34):
-        return "other"         # Fishing, towing, dredging, diving, etc.
+        return "other"  # Fishing, towing, dredging, diving, etc.
     if ais_type in (50, 51, 52, 53, 54, 55, 56, 57, 58, 59):
-        return "other"         # Pilot, SAR, tug, port tender, etc.
+        return "other"  # Pilot, SAR, tug, port tender, etc.
-    return "unknown"            # Not yet classified — will update when ShipStaticData arrives
+    return "unknown"  # Not yet classified — will update when ShipStaticData arrives
 
 
 # MMSI Maritime Identification Digit (MID) → Country mapping
 # First 3 digits of MMSI (for 9-digit MMSIs) encode the flag state
 MID_COUNTRY = {
-    201: "Albania", 202: "Andorra", 203: "Austria", 204: "Portugal", 205: "Belgium",
+    201: "Albania",
-    206: "Belarus", 207: "Bulgaria", 208: "Vatican", 209: "Cyprus", 210: "Cyprus",
+    202: "Andorra",
-    211: "Germany", 212: "Cyprus", 213: "Georgia", 214: "Moldova", 215: "Malta",
+    203: "Austria",
-    216: "Armenia", 218: "Germany", 219: "Denmark", 220: "Denmark", 224: "Spain",
+    204: "Portugal",
-    225: "Spain", 226: "France", 227: "France", 228: "France", 229: "Malta",
+    205: "Belgium",
-    230: "Finland", 231: "Faroe Islands", 232: "United Kingdom", 233: "United Kingdom",
+    206: "Belarus",
-    234: "United Kingdom", 235: "United Kingdom", 236: "Gibraltar", 237: "Greece",
+    207: "Bulgaria",
-    238: "Croatia", 239: "Greece", 240: "Greece", 241: "Greece", 242: "Morocco",
+    208: "Vatican",
-    243: "Hungary", 244: "Netherlands", 245: "Netherlands", 246: "Netherlands",
+    209: "Cyprus",
-    247: "Italy", 248: "Malta", 249: "Malta", 250: "Ireland", 251: "Iceland",
+    210: "Cyprus",
-    252: "Liechtenstein", 253: "Luxembourg", 254: "Monaco", 255: "Portugal",
+    211: "Germany",
-    256: "Malta", 257: "Norway", 258: "Norway", 259: "Norway", 261: "Poland",
+    212: "Cyprus",
-    263: "Portugal", 264: "Romania", 265: "Sweden", 266: "Sweden", 267: "Slovakia",
+    213: "Georgia",
-    268: "San Marino", 269: "Switzerland", 270: "Czech Republic", 271: "Turkey",
+    214: "Moldova",
-    272: "Ukraine", 273: "Russia", 274: "North Macedonia", 275: "Latvia",
+    215: "Malta",
-    276: "Estonia", 277: "Lithuania", 278: "Slovenia",
+    216: "Armenia",
-    301: "Anguilla", 303: "Alaska", 304: "Antigua", 305: "Antigua",
+    218: "Germany",
-    306: "Netherlands Antilles", 307: "Aruba", 308: "Bahamas", 309: "Bahamas",
+    219: "Denmark",
-    310: "Bermuda", 311: "Bahamas", 312: "Belize", 314: "Barbados", 316: "Canada",
+    220: "Denmark",
-    319: "Cayman Islands", 321: "Costa Rica", 323: "Cuba", 325: "Dominica",
+    224: "Spain",
-    327: "Dominican Republic", 329: "Guadeloupe", 330: "Grenada", 331: "Greenland",
+    225: "Spain",
-    332: "Guatemala", 334: "Honduras", 336: "Haiti", 338: "United States",
+    226: "France",
-    339: "Jamaica", 341: "Saint Kitts", 343: "Saint Lucia", 345: "Mexico",
+    227: "France",
-    347: "Martinique", 348: "Montserrat", 350: "Nicaragua", 351: "Panama",
+    228: "France",
-    352: "Panama", 353: "Panama", 354: "Panama", 355: "Panama",
+    229: "Malta",
-    356: "Panama", 357: "Panama", 358: "Puerto Rico", 359: "El Salvador",
+    230: "Finland",
-    361: "Saint Pierre", 362: "Trinidad", 364: "Turks and Caicos",
+    231: "Faroe Islands",
-    366: "United States", 367: "United States", 368: "United States", 369: "United States",
+    232: "United Kingdom",
-    370: "Panama", 371: "Panama", 372: "Panama", 373: "Panama",
+    233: "United Kingdom",
-    374: "Panama", 375: "Saint Vincent", 376: "Saint Vincent", 377: "Saint Vincent",
+    234: "United Kingdom",
-    378: "British Virgin Islands", 379: "US Virgin Islands",
+    235: "United Kingdom",
-    401: "Afghanistan", 403: "Saudi Arabia", 405: "Bangladesh", 408: "Bahrain",
+    236: "Gibraltar",
-    410: "Bhutan", 412: "China", 413: "China", 414: "China",
+    237: "Greece",
-    416: "Taiwan", 417: "Sri Lanka", 419: "India", 422: "Iran",
+    238: "Croatia",
-    423: "Azerbaijan", 425: "Iraq", 428: "Israel", 431: "Japan",
+    239: "Greece",
-    432: "Japan", 434: "Turkmenistan", 436: "Kazakhstan", 437: "Uzbekistan",
+    240: "Greece",
-    438: "Jordan", 440: "South Korea", 441: "South Korea", 443: "Palestine",
+    241: "Greece",
-    445: "North Korea", 447: "Kuwait", 450: "Lebanon", 451: "Kyrgyzstan",
+    242: "Morocco",
-    453: "Macao", 455: "Maldives", 457: "Mongolia", 459: "Nepal",
+    243: "Hungary",
-    461: "Oman", 463: "Pakistan", 466: "Qatar", 468: "Syria",
+    244: "Netherlands",
-    470: "UAE", 472: "Tajikistan", 473: "Yemen", 475: "Tonga",
+    245: "Netherlands",
-    477: "Hong Kong", 478: "Bosnia",
+    246: "Netherlands",
-    501: "Antarctica", 503: "Australia", 506: "Myanmar",
+    247: "Italy",
-    508: "Brunei", 510: "Micronesia", 511: "Palau", 512: "New Zealand",
+    248: "Malta",
-    514: "Cambodia", 515: "Cambodia", 516: "Christmas Island",
+    249: "Malta",
-    518: "Cook Islands", 520: "Fiji", 523: "Cocos Islands",
+    250: "Ireland",
-    525: "Indonesia", 529: "Kiribati", 531: "Laos", 533: "Malaysia",
+    251: "Iceland",
-    536: "Northern Mariana Islands", 538: "Marshall Islands",
+    252: "Liechtenstein",
-    540: "New Caledonia", 542: "Niue", 544: "Nauru", 546: "French Polynesia",
+    253: "Luxembourg",
-    548: "Philippines", 553: "Papua New Guinea", 555: "Pitcairn",
+    254: "Monaco",
-    557: "Solomon Islands", 559: "American Samoa", 561: "Samoa",
+    255: "Portugal",
-    563: "Singapore", 564: "Singapore", 565: "Singapore", 566: "Singapore",
+    256: "Malta",
-    567: "Thailand", 570: "Tonga", 572: "Tuvalu", 574: "Vietnam",
+    257: "Norway",
-    576: "Vanuatu", 577: "Vanuatu", 578: "Wallis and Futuna",
+    258: "Norway",
-    601: "South Africa", 603: "Angola", 605: "Algeria", 607: "Benin",
+    259: "Norway",
-    609: "Botswana", 610: "Burundi", 611: "Cameroon", 612: "Cape Verde",
+    261: "Poland",
-    613: "Central African Republic", 615: "Congo", 616: "Comoros",
+    263: "Portugal",
-    617: "DR Congo", 618: "Ivory Coast", 619: "Djibouti",
+    264: "Romania",
-    620: "Egypt", 621: "Equatorial Guinea", 622: "Ethiopia",
+    265: "Sweden",
-    624: "Eritrea", 625: "Gabon", 626: "Gambia", 627: "Ghana",
+    266: "Sweden",
-    629: "Guinea", 630: "Guinea-Bissau", 631: "Kenya", 632: "Lesotho",
+    267: "Slovakia",
-    633: "Liberia", 634: "Liberia", 635: "Liberia", 636: "Liberia",
+    268: "San Marino",
-    637: "Libya", 642: "Madagascar", 644: "Malawi", 645: "Mali",
+    269: "Switzerland",
-    647: "Mauritania", 649: "Mauritius", 650: "Mozambique",
+    270: "Czech Republic",
-    654: "Namibia", 655: "Niger", 656: "Nigeria", 657: "Guinea",
+    271: "Turkey",
-    659: "Rwanda", 660: "Senegal", 661: "Sierra Leone",
+    272: "Ukraine",
-    662: "Somalia", 663: "South Africa", 664: "Sudan",
+    273: "Russia",
-    667: "Tanzania", 668: "Togo", 669: "Tunisia", 670: "Uganda",
+    274: "North Macedonia",
-    671: "Egypt", 672: "Tanzania", 674: "Zambia", 675: "Zimbabwe",
+    275: "Latvia",
-    676: "Comoros", 677: "Tanzania",
+    276: "Estonia",
+    277: "Lithuania",
+    278: "Slovenia",
+    301: "Anguilla",
+    303: "Alaska",
+    304: "Antigua",
+    305: "Antigua",
+    306: "Netherlands Antilles",
+    307: "Aruba",
+    308: "Bahamas",
+    309: "Bahamas",
+    310: "Bermuda",
+    311: "Bahamas",
+    312: "Belize",
+    314: "Barbados",
+    316: "Canada",
+    319: "Cayman Islands",
+    321: "Costa Rica",
+    323: "Cuba",
+    325: "Dominica",
+    327: "Dominican Republic",
+    329: "Guadeloupe",
+    330: "Grenada",
+    331: "Greenland",
+    332: "Guatemala",
+    334: "Honduras",
+    336: "Haiti",
+    338: "United States",
+    339: "Jamaica",
+    341: "Saint Kitts",
+    343: "Saint Lucia",
+    345: "Mexico",
+    347: "Martinique",
+    348: "Montserrat",
+    350: "Nicaragua",
+    351: "Panama",
+    352: "Panama",
+    353: "Panama",
+    354: "Panama",
+    355: "Panama",
+    356: "Panama",
+    357: "Panama",
+    358: "Puerto Rico",
+    359: "El Salvador",
+    361: "Saint Pierre",
+    362: "Trinidad",
+    364: "Turks and Caicos",
+    366: "United States",
+    367: "United States",
+    368: "United States",
+    369: "United States",
+    370: "Panama",
+    371: "Panama",
+    372: "Panama",
+    373: "Panama",
+    374: "Panama",
+    375: "Saint Vincent",
+    376: "Saint Vincent",
+    377: "Saint Vincent",
+    378: "British Virgin Islands",
+    379: "US Virgin Islands",
+    401: "Afghanistan",
+    403: "Saudi Arabia",
+    405: "Bangladesh",
+    408: "Bahrain",
+    410: "Bhutan",
+    412: "China",
+    413: "China",
+    414: "China",
+    416: "Taiwan",
+    417: "Sri Lanka",
+    419: "India",
+    422: "Iran",
+    423: "Azerbaijan",
+    425: "Iraq",
+    428: "Israel",
+    431: "Japan",
+    432: "Japan",
+    434: "Turkmenistan",
+    436: "Kazakhstan",
+    437: "Uzbekistan",
+    438: "Jordan",
+    440: "South Korea",
+    441: "South Korea",
+    443: "Palestine",
+    445: "North Korea",
+    447: "Kuwait",
+    450: "Lebanon",
+    451: "Kyrgyzstan",
+    453: "Macao",
+    455: "Maldives",
+    457: "Mongolia",
+    459: "Nepal",
+    461: "Oman",
+    463: "Pakistan",
+    466: "Qatar",
+    468: "Syria",
+    470: "UAE",
+    472: "Tajikistan",
+    473: "Yemen",
+    475: "Tonga",
+    477: "Hong Kong",
+    478: "Bosnia",
+    501: "Antarctica",
+    503: "Australia",
+    506: "Myanmar",
+    508: "Brunei",
+    510: "Micronesia",
+    511: "Palau",
+    512: "New Zealand",
+    514: "Cambodia",
+    515: "Cambodia",
+    516: "Christmas Island",
+    518: "Cook Islands",
+    520: "Fiji",
+    523: "Cocos Islands",
+    525: "Indonesia",
+    529: "Kiribati",
+    531: "Laos",
+    533: "Malaysia",
+    536: "Northern Mariana Islands",
+    538: "Marshall Islands",
+    540: "New Caledonia",
+    542: "Niue",
+    544: "Nauru",
+    546: "French Polynesia",
+    548: "Philippines",
+    553: "Papua New Guinea",
+    555: "Pitcairn",
+    557: "Solomon Islands",
+    559: "American Samoa",
+    561: "Samoa",
+    563: "Singapore",
+    564: "Singapore",
+    565: "Singapore",
+    566: "Singapore",
+    567: "Thailand",
+    570: "Tonga",
+    572: "Tuvalu",
+    574: "Vietnam",
+    576: "Vanuatu",
+    577: "Vanuatu",
+    578: "Wallis and Futuna",
+    601: "South Africa",
+    603: "Angola",
+    605: "Algeria",
+    607: "Benin",
+    609: "Botswana",
+    610: "Burundi",
+    611: "Cameroon",
+    612: "Cape Verde",
+    613: "Central African Republic",
+    615: "Congo",
+    616: "Comoros",
+    617: "DR Congo",
+    618: "Ivory Coast",
+    619: "Djibouti",
+    620: "Egypt",
+    621: "Equatorial Guinea",
+    622: "Ethiopia",
+    624: "Eritrea",
+    625: "Gabon",
+    626: "Gambia",
+    627: "Ghana",
+    629: "Guinea",
+    630: "Guinea-Bissau",
+    631: "Kenya",
+    632: "Lesotho",
+    633: "Liberia",
+    634: "Liberia",
+    635: "Liberia",
+    636: "Liberia",
+    637: "Libya",
+    642: "Madagascar",
+    644: "Malawi",
+    645: "Mali",
+    647: "Mauritania",
+    649: "Mauritius",
+    650: "Mozambique",
+    654: "Namibia",
+    655: "Niger",
+    656: "Nigeria",
+    657: "Guinea",
+    659: "Rwanda",
+    660: "Senegal",
+    661: "Sierra Leone",
+    662: "Somalia",
+    663: "South Africa",
+    664: "Sudan",
+    667: "Tanzania",
+    668: "Togo",
+    669: "Tunisia",
+    670: "Uganda",
+    671: "Egypt",
+    672: "Tanzania",
+    674: "Zambia",
+    675: "Zimbabwe",
+    676: "Comoros",
+    677: "Tanzania",
 }
 
+
 def get_country_from_mmsi(mmsi: int) -> str:
     """Look up flag state from MMSI Maritime Identification Digit."""
     mmsi_str = str(mmsi)
@@ -127,24 +339,127 @@ def get_country_from_mmsi(mmsi: int) -> str:
 
 # Global vessel store: MMSI → vessel dict
 _vessels: dict[int, dict] = {}
+_vessel_trails: dict[int, dict] = {}
 _vessels_lock = threading.Lock()
 _ws_thread: threading.Thread | None = None
 _ws_running = False
+_proxy_process = None
+# Issue #258: latest status snapshot emitted by ais_proxy.js. Populated when
+# the proxy reports e.g. {"__ais_proxy_status": {"degraded_tls": true}} on
+# stdout, which it does when it falls back to the SPKI-pinned insecure-date
+# path during an upstream cert outage. Surfaced via ais_proxy_status() for
+# /api/health.
+_proxy_status: dict = {}
+# Upstream-connectivity telemetry (added when stream.aisstream.io went fully
+# offline on 2026-05-23). ``_last_msg_at`` is the unix timestamp of the most
+# recent vessel message received from the proxy. ``_proxy_spawn_count`` is
+# how many times we've started the node proxy; combined with no recent
+# messages it tells us the proxy is respawning in a tight loop because the
+# upstream is unreachable. Surfaced via ais_proxy_status() so the operator
+# can see "AIS is dead" instead of guessing whether it's their map filter,
+# their api key, or upstream.
+_last_msg_at: float = 0.0
+_proxy_spawn_count: int = 0
+_VESSEL_TRAIL_INTERVAL_S = 120
+_VESSEL_TRAIL_MAX_POINTS = 240
+
+
+# How stale "last vessel message" can be before we consider the stream
+# disconnected. AISStream typically pushes multiple messages/sec, so a 60s
+# gap means something's wrong upstream or in transit.
+_AIS_CONNECTED_FRESHNESS_S = 60
+
+
+def ais_proxy_status() -> dict:
+    """Return a copy of the latest ais_proxy.js status + connectivity health.
+
+    Fields:
+      * ``degraded_tls`` (bool, issue #258) — true when the proxy is using
+        SPKI-pinned fallback because AISStream's cert expired.
+      * ``connected`` (bool) — true when we received a vessel message in
+        the last ``_AIS_CONNECTED_FRESHNESS_S`` seconds.
+      * ``last_msg_age_seconds`` (int | None) — seconds since the last
+        vessel message; None if we've never received one.
+      * ``proxy_spawn_count`` (int) — how many times we've spawned the
+        node proxy. Sustained increases here without ``connected`` means
+        we're respawning in a tight loop because upstream is dead.
+
+    Returns an empty dict when called before the AIS subsystem starts
+    (e.g. during tests or when no API key is set).
+    """
+    with _vessels_lock:
+        status = dict(_proxy_status)
+        last = _last_msg_at
+        spawns = _proxy_spawn_count
+
+    now = time.time()
+    if last > 0:
+        last_age = int(now - last)
+        status["last_msg_age_seconds"] = last_age
+        status["connected"] = last_age <= _AIS_CONNECTED_FRESHNESS_S
+    else:
+        status["last_msg_age_seconds"] = None
+        status["connected"] = False
+    status["proxy_spawn_count"] = spawns
+    return status
 
 import os
+
 CACHE_FILE = os.path.join(os.path.dirname(__file__), "ais_cache.json")
 
 
+def _record_vessel_trail_locked(mmsi: int, lat, lng, sog=0, now_ts: float | None = None) -> None:
+    """Append a sampled AIS trail point. Caller must hold _vessels_lock."""
+    if lat is None or lng is None:
+        return
+    try:
+        lat_f = float(lat)
+        lng_f = float(lng)
+    except (TypeError, ValueError):
+        return
+    if abs(lat_f) > 90 or abs(lng_f) > 180 or (lat_f == 0 and lng_f == 0):
+        return
+    now = now_ts or time.time()
+    trail_data = _vessel_trails.setdefault(int(mmsi), {"points": [], "last_seen": now})
+    point = [round(lat_f, 5), round(lng_f, 5), round(float(sog or 0), 1), round(now)]
+    last_point_ts = trail_data["points"][-1][3] if trail_data["points"] else 0
+    if now - last_point_ts < _VESSEL_TRAIL_INTERVAL_S:
+        trail_data["last_seen"] = now
+        return
+    if (
+        trail_data["points"]
+        and trail_data["points"][-1][0] == point[0]
+        and trail_data["points"][-1][1] == point[1]
+    ):
+        trail_data["last_seen"] = now
+        return
+    trail_data["points"].append(point)
+    trail_data["last_seen"] = now
+    if len(trail_data["points"]) > _VESSEL_TRAIL_MAX_POINTS:
+        trail_data["points"] = trail_data["points"][-_VESSEL_TRAIL_MAX_POINTS:]
+
+
+def get_vessel_trail(mmsi: int) -> list:
+    """Return the accumulated trail for a single vessel without expanding live payloads."""
+    try:
+        key = int(mmsi)
+    except (TypeError, ValueError):
+        return []
+    with _vessels_lock:
+        points = _vessel_trails.get(key, {}).get("points", [])
+        return [list(point) for point in points]
+
+
 def _save_cache():
     """Save vessel data to disk for persistence across restarts."""
     try:
         with _vessels_lock:
             # Convert int keys to strings for JSON
             data = {str(k): v for k, v in _vessels.items()}
-        with open(CACHE_FILE, 'w') as f:
+        with open(CACHE_FILE, "w") as f:
             json.dump(data, f)
         logger.info(f"AIS cache saved: {len(data)} vessels")
-    except Exception as e:
+    except (IOError, OSError) as e:
         logger.error(f"Failed to save AIS cache: {e}")
 
 
@@ -154,7 +469,7 @@ def _load_cache():
     if not os.path.exists(CACHE_FILE):
         return
     try:
-        with open(CACHE_FILE, 'r') as f:
+        with open(CACHE_FILE, "r") as f:
             data = json.load(f)
         now = time.time()
         stale_cutoff = now - 3600  # Accept vessels up to 1 hour old on restart
@@ -165,74 +480,167 @@ def _load_cache():
                     _vessels[int(k)] = v
                     loaded += 1
         logger.info(f"AIS cache loaded: {loaded} vessels from disk")
-    except Exception as e:
+    except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
         logger.error(f"Failed to load AIS cache: {e}")
 
 
-def get_ais_vessels() -> list[dict]:
+def prune_stale_vessels():
-    """Return a snapshot of tracked AIS vessels, excluding 'other' type, pruning stale."""
+    """Remove vessels not updated in the last 15 minutes. Safe to call from a scheduler."""
     now = time.time()
-    stale_cutoff = now - 900  # 15 minutes
+    stale_cutoff = now - 900
-    
     with _vessels_lock:
-        # Prune stale vessels
         stale_keys = [k for k, v in _vessels.items() if v.get("_updated", 0) < stale_cutoff]
         for k in stale_keys:
             del _vessels[k]
+            _vessel_trails.pop(k, None)
+    if stale_keys:
+        logger.info(f"AIS pruned {len(stale_keys)} stale vessels")
 
+
+def get_ais_vessels() -> list[dict]:
+    """Return a snapshot of tracked AIS vessels, pruning stale."""
+    prune_stale_vessels()
+
+    with _vessels_lock:
         result = []
         for mmsi, v in _vessels.items():
             v_type = v.get("type", "unknown")
-            # Skip 'other' vessels (fishing, tug, pilot, etc.) to reduce load
-            if v_type == "other":
-                continue
             # Skip vessels without valid position
             if not v.get("lat") or not v.get("lng"):
                 continue
 
-            result.append({
+            # Sanitize speed: AIS 102.3 kn = "speed not available"
-                "mmsi": mmsi,
+            sog = v.get("sog", 0)
-                "name": v.get("name", "UNKNOWN"),
+            if sog >= 102.2:
-                "type": v_type,
+                sog = 0
-                "lat": round(v.get("lat", 0), 5),
+
-                "lng": round(v.get("lng", 0), 5),
+            result.append(
-                "heading": v.get("heading", 0),
+                {
-                "sog": round(v.get("sog", 0), 1),
+                    "mmsi": mmsi,
-                "cog": round(v.get("cog", 0), 1),
+                    "name": v.get("name", "UNKNOWN"),
-                "callsign": v.get("callsign", ""),
+                    "type": v_type,
-                "destination": v.get("destination", "") or "UNKNOWN",
+                    "lat": round(v.get("lat", 0), 5),
-                "imo": v.get("imo", 0),
+                    "lng": round(v.get("lng", 0), 5),
-                "country": get_country_from_mmsi(mmsi),
+                    "heading": v.get("heading", 0),
-            })
+                    "sog": round(sog, 1),
+                    "cog": round(v.get("cog", 0), 1),
+                    "callsign": v.get("callsign", ""),
+                    "destination": v.get("destination", "") or "UNKNOWN",
+                    "imo": v.get("imo", 0),
+                    "country": get_country_from_mmsi(mmsi),
+                }
+            )
         return result
 
 
+def ingest_ais_catcher(msgs: list[dict]) -> int:
+    """Ingest decoded AIS messages from AIS-catcher HTTP feed.
+    Returns number of vessels updated."""
+    count = 0
+    now = time.time()
+    with _vessels_lock:
+        for msg in msgs:
+            mmsi = msg.get("mmsi")
+            if not mmsi or not isinstance(mmsi, int):
+                continue
+
+            vessel = _vessels.setdefault(mmsi, {"mmsi": mmsi})
+            msg_type = msg.get("type", 0)
+
+            # Position reports (types 1, 2, 3 = Class A; 18, 19 = Class B)
+            if msg_type in (1, 2, 3, 18, 19):
+                lat = msg.get("lat")
+                lon = msg.get("lon")
+                if lat is not None and lon is not None and lat != 91.0 and lon != 181.0:
+                    vessel["lat"] = lat
+                    vessel["lng"] = lon
+                    # AIS raw value 1023 (102.3 kn) = "speed not available"
+                    raw_speed = msg.get("speed", 0)
+                    vessel["sog"] = 0 if raw_speed >= 102.2 else raw_speed
+                    vessel["cog"] = msg.get("course", 0)
+                    heading = msg.get("heading", 511)
+                    vessel["heading"] = heading if heading != 511 else vessel.get("cog", 0)
+                    vessel["_updated"] = now
+                    _record_vessel_trail_locked(mmsi, lat, lon, vessel["sog"], now)
+                    if msg.get("shipname"):
+                        vessel["name"] = msg["shipname"].strip()
+                    count += 1
+
+            # Static data (type 5 = Class A static; 24 = Class B static)
+            elif msg_type in (5, 24):
+                if msg.get("shipname"):
+                    vessel["name"] = msg["shipname"].strip()
+                if msg.get("callsign"):
+                    vessel["callsign"] = msg["callsign"].strip()
+                if msg.get("imo"):
+                    vessel["imo"] = msg["imo"]
+                if msg.get("destination"):
+                    vessel["destination"] = msg["destination"].strip().replace("@", "")
+                ship_type = msg.get("shiptype", 0)
+                if ship_type:
+                    vessel["ais_type_code"] = ship_type
+                    vessel["type"] = classify_vessel(ship_type, mmsi)
+                vessel["_updated"] = now
+
+            # Ensure country is set from MMSI MID
+            if "country" not in vessel:
+                vessel["country"] = get_country_from_mmsi(mmsi)
+
+            # Ensure name exists
+            if "name" not in vessel:
+                vessel["name"] = msg.get("shipname", "UNKNOWN") or "UNKNOWN"
+
+    return count
+
+
 def _ais_stream_loop():
     """Main loop: spawn node proxy and process messages from stdout."""
+    global _proxy_process
     import subprocess
     import os
 
     proxy_script = os.path.join(os.path.dirname(os.path.dirname(__file__)), "ais_proxy.js")
     backoff = 1  # Exponential backoff starting at 1 second
 
+    if not API_KEY:
+        logger.info("AIS_API_KEY not set — ship tracking disabled. Set AIS_API_KEY to enable.")
+        return
+
     while _ws_running:
         try:
             logger.info("Starting Node.js AIS Stream Proxy...")
+            proxy_env = os.environ.copy()
+            proxy_env["AIS_API_KEY"] = API_KEY
+            popen_kwargs = {}
+            if os.name == "nt":
+                popen_kwargs["creationflags"] = (
+                    getattr(subprocess, "CREATE_NO_WINDOW", 0)
+                    | getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
+                )
             process = subprocess.Popen(
-                ['node', proxy_script, API_KEY],
+                ["node", proxy_script],
+                stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
                 text=True,
-                bufsize=1
+                bufsize=1,
+                env=proxy_env,
+                **popen_kwargs,
             )
+            global _proxy_spawn_count
+            with _vessels_lock:
+                _proxy_process = process
+                _proxy_spawn_count += 1
 
             # Drain stderr in a background thread to prevent deadlock
             import threading
+
             def _drain_stderr():
-                for errline in iter(process.stderr.readline, ''):
+                for errline in iter(process.stderr.readline, ""):
                     errline = errline.strip()
                     if errline:
                         logger.warning(f"AIS proxy stderr: {errline}")
+
             threading.Thread(target=_drain_stderr, daemon=True).start()
 
             logger.info("AIS Stream proxy started — receiving vessel data")
@@ -240,7 +648,7 @@ def _ais_stream_loop():
             msg_count = 0
             ok_streak = 0  # Track consecutive successful messages for backoff reset
             last_log_time = time.time()
-            for raw_msg in iter(process.stdout.readline, ''):
+            for raw_msg in iter(process.stdout.readline, ""):
                 if not _ws_running:
                     process.terminate()
                     break
@@ -258,6 +666,18 @@ def _ais_stream_loop():
                     logger.error(f"AIS Stream error: {data['error']}")
                     continue
 
+                # Issue #258: ais_proxy.js emits status markers (e.g.
+                # {"__ais_proxy_status": {"degraded_tls": true}}) when the
+                # SPKI-pinned fallback is in use. We snapshot the latest
+                # status so the backend can expose it on /api/health.
+                if isinstance(data, dict) and "__ais_proxy_status" in data:
+                    status = data.get("__ais_proxy_status") or {}
+                    if isinstance(status, dict):
+                        with _vessels_lock:
+                            _proxy_status.clear()
+                            _proxy_status.update(status)
+                    continue
+
                 msg_type = data.get("MessageType", "")
                 metadata = data.get("MetaData", {})
                 message = data.get("Message", {})
@@ -266,9 +686,15 @@ def _ais_stream_loop():
                 if not mmsi:
                     continue
 
+                # Telemetry: stamp the timestamp of the most recent real
+                # vessel message. ais_proxy_status() reads this to decide
+                # whether the stream is currently "connected" — i.e. has
+                # any data flowed in the last 60s.
+                global _last_msg_at
                 with _vessels_lock:
+                    _last_msg_at = time.time()
                     if mmsi not in _vessels:
-                        _vessels[mmsi] = {"_updated": time.time()}
+                        _vessels[mmsi] = {"_updated": _last_msg_at}
                     vessel = _vessels[mmsi]
 
                 # Update position from PositionReport or StandardClassBPositionReport
@@ -286,14 +712,20 @@ def _ais_stream_loop():
                     with _vessels_lock:
                         vessel["lat"] = lat
                         vessel["lng"] = lng
-                        vessel["sog"] = report.get("Sog", 0)
+                        # AIS raw value 1023 (102.3 kn) = "speed not available"
+                        raw_sog = report.get("Sog", 0)
+                        vessel["sog"] = 0 if raw_sog >= 102.2 else raw_sog
                         vessel["cog"] = report.get("Cog", 0)
                         heading = report.get("TrueHeading", 511)
                         vessel["heading"] = heading if heading != 511 else report.get("Cog", 0)
-                        vessel["_updated"] = time.time()
+                        now_ts = time.time()
+                        vessel["_updated"] = now_ts
+                        _record_vessel_trail_locked(mmsi, lat, lng, vessel["sog"], now_ts)
                         # Use metadata name if we don't have one yet
                         if not vessel.get("name") or vessel["name"] == "UNKNOWN":
-                            vessel["name"] = metadata.get("ShipName", "UNKNOWN").strip() or "UNKNOWN"
+                            vessel["name"] = (
+                                metadata.get("ShipName", "UNKNOWN").strip() or "UNKNOWN"
+                            )
 
                 # Update static data from ShipStaticData
                 elif msg_type == "ShipStaticData":
@@ -301,10 +733,14 @@ def _ais_stream_loop():
                     ais_type = static.get("Type", 0)
 
                     with _vessels_lock:
-                        vessel["name"] = (static.get("Name", "") or metadata.get("ShipName", "UNKNOWN")).strip() or "UNKNOWN"
+                        vessel["name"] = (
+                            static.get("Name", "") or metadata.get("ShipName", "UNKNOWN")
+                        ).strip() or "UNKNOWN"
                         vessel["callsign"] = (static.get("CallSign", "") or "").strip()
                         vessel["imo"] = static.get("ImoNumber", 0)
-                        vessel["destination"] = (static.get("Destination", "") or "").strip().replace("@", "")
+                        vessel["destination"] = (
+                            (static.get("Destination", "") or "").strip().replace("@", "")
+                        )
                         vessel["ais_type_code"] = ais_type
                         vessel["type"] = classify_vessel(ais_type, mmsi)
                         vessel["_updated"] = time.time()
@@ -322,11 +758,13 @@ def _ais_stream_loop():
                 if now - last_log_time >= 60:
                     with _vessels_lock:
                         count = len(_vessels)
-                    logger.info(f"AIS Stream: processed {msg_count} messages, tracking {count} vessels")
+                    logger.info(
+                        f"AIS Stream: processed {msg_count} messages, tracking {count} vessels"
+                    )
                     _save_cache()
                     last_log_time = now
 
-        except Exception as e:
+        except (ConnectionError, TimeoutError, OSError, ValueError, KeyError) as e:
             logger.error(f"AIS proxy connection error: {e}")
             if _ws_running:
                 logger.info(f"Restarting AIS proxy in {backoff}s (exponential backoff)...")
@@ -337,23 +775,47 @@ def _ais_stream_loop():
 
 def _run_ais_loop():
     """Thread target: run the AIS loop."""
+    global _ws_running, _ws_thread, _proxy_process
     try:
         _ais_stream_loop()
     except Exception as e:
         logger.error(f"AIS Stream thread crashed: {e}")
+    finally:
+        with _vessels_lock:
+            _ws_running = False
+            _ws_thread = None
+            _proxy_process = None
 
 
 def start_ais_stream():
     """Start the AIS WebSocket stream in a background thread."""
     global _ws_thread, _ws_running
-    if _ws_thread and _ws_thread.is_alive():
+
+    # Always load cached vessel data first so the ships layer can paint even
+    # when live streaming is disabled or the upstream is unavailable.
+    _load_cache()
+
+    if not API_KEY:
+        logger.info("AIS_API_KEY not set — ship tracking disabled. Set AIS_API_KEY to enable.")
+        return
+
+    if not ais_stream_proxy_enabled():
+        logger.info(
+            "AIS live stream proxy disabled for this runtime; using cached AIS vessels. "
+            "Set SHADOWBROKER_ENABLE_AIS_STREAM_PROXY=1 to opt in."
+        )
+        return
+
+    with _vessels_lock:
+        if _ws_running:
+            logger.info("AIS Stream already running")
+            return
+        _ws_running = True
+        existing_thread = _ws_thread
+    if existing_thread and existing_thread.is_alive():
         logger.info("AIS Stream already running")
         return
 
-    # Load cached vessel data from disk
-    _load_cache()
-    
-    _ws_running = True
     _ws_thread = threading.Thread(target=_run_ais_loop, daemon=True, name="ais-stream")
     _ws_thread.start()
     logger.info("AIS Stream background thread started")
@@ -361,7 +823,36 @@ def start_ais_stream():
 
 def stop_ais_stream():
     """Stop the AIS WebSocket stream and save cache."""
-    global _ws_running
+    global _ws_running, _ws_thread, _proxy_process
-    _ws_running = False
+    with _vessels_lock:
+        _ws_running = False
+        _ws_thread = None
+        proc = _proxy_process
+        _proxy_process = None
+
+    if proc and proc.stdin:
+        try:
+            proc.stdin.close()
+        except Exception:
+            pass
+
     _save_cache()  # Save on shutdown
     logger.info("AIS Stream stopping...")
+
+
+def update_ais_bbox(south: float, west: float, north: float, east: float):
+    """Dynamically update the AIS stream bounding box via proxy stdin."""
+    with _vessels_lock:
+        proc = _proxy_process
+    if not proc or not proc.stdin:
+        return
+
+    try:
+        cmd = json.dumps({"type": "update_bbox", "bboxes": [[[south, west], [north, east]]]})
+        proc.stdin.write(cmd + "\n")
+        proc.stdin.flush()
+        logger.info(
+            f"Updated AIS bounding box to: S:{south:.2f} W:{west:.2f} N:{north:.2f} E:{east:.2f}"
+        )
+    except Exception as e:
+        logger.error(f"Failed to update AIS bbox: {e}")

backend/services/analysis_zone_store.py

@@ -0,0 +1,189 @@
+"""Analysis Zone store — OpenClaw-placed map overlays with analyst notes.
+
+These render as the dashed-border squares on the correlations layer.
+Unlike automated correlations (which are recomputed every cycle), analysis
+zones persist until the agent or user deletes them, or their TTL expires.
+
+Shape matches the correlation alert schema so the frontend renders them
+identically — the ``source`` field marks them as agent-placed and enables
+the delete button in the popup.
+"""
+
+import json
+import logging
+import os
+import threading
+import time
+import uuid
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_zones: list[dict[str, Any]] = []
+_lock = threading.Lock()
+
+_PERSIST_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data")
+_PERSIST_FILE = os.path.join(_PERSIST_DIR, "analysis_zones.json")
+
+ZONE_CATEGORIES = {
+    "contradiction",   # narrative vs telemetry mismatch
+    "analysis",        # general analyst note / assessment
+    "warning",         # potential threat or risk area
+    "observation",     # neutral observation worth marking
+    "hypothesis",      # unverified theory to investigate
+}
+
+# Map categories to correlation type colors on the frontend
+CATEGORY_COLORS = {
+    "contradiction": "amber",
+    "analysis": "cyan",
+    "warning": "red",
+    "observation": "blue",
+    "hypothesis": "purple",
+}
+
+
+def _ensure_dir():
+    try:
+        os.makedirs(_PERSIST_DIR, exist_ok=True)
+    except OSError:
+        pass
+
+
+def _save():
+    """Persist to disk. Called under lock."""
+    try:
+        _ensure_dir()
+        with open(_PERSIST_FILE, "w", encoding="utf-8") as f:
+            json.dump(_zones, f, indent=2, default=str)
+    except Exception as e:
+        logger.warning("Failed to save analysis zones: %s", e)
+
+
+def _load():
+    """Load from disk on startup."""
+    global _zones
+    try:
+        if os.path.exists(_PERSIST_FILE):
+            with open(_PERSIST_FILE, "r", encoding="utf-8") as f:
+                data = json.load(f)
+            if isinstance(data, list):
+                _zones = data
+                logger.info("Loaded %d analysis zones from disk", len(_zones))
+    except Exception as e:
+        logger.warning("Failed to load analysis zones: %s", e)
+
+
+# Load on import
+_load()
+
+
+def _expire():
+    """Remove zones past their TTL. Called under lock."""
+    now = time.time()
+    before = len(_zones)
+    _zones[:] = [
+        z for z in _zones
+        if z.get("ttl_hours", 0) <= 0
+        or (now - z.get("created_at", now)) < z["ttl_hours"] * 3600
+    ]
+    removed = before - len(_zones)
+    if removed:
+        logger.info("Expired %d analysis zones", removed)
+
+
+def create_zone(
+    *,
+    lat: float,
+    lng: float,
+    title: str,
+    body: str,
+    category: str = "analysis",
+    severity: str = "medium",
+    cell_size_deg: float = 1.0,
+    ttl_hours: float = 0,
+    source: str = "openclaw",
+    drivers: list[str] | None = None,
+) -> dict[str, Any]:
+    """Create an analysis zone. Returns the created zone dict."""
+    category = category if category in ZONE_CATEGORIES else "analysis"
+    if severity not in ("high", "medium", "low"):
+        severity = "medium"
+    cell_size_deg = max(0.1, min(cell_size_deg, 10.0))
+
+    zone: dict[str, Any] = {
+        "id": str(uuid.uuid4())[:12],
+        "lat": lat,
+        "lng": lng,
+        "type": "analysis_zone",
+        "category": category,
+        "severity": severity,
+        "score": {"high": 90, "medium": 60, "low": 30}.get(severity, 60),
+        "title": title[:200],
+        "body": body[:2000],
+        "drivers": (drivers or [title])[:5],
+        "cell_size": cell_size_deg,
+        "source": source,
+        "created_at": time.time(),
+        "ttl_hours": ttl_hours,
+    }
+
+    with _lock:
+        _expire()
+        _zones.append(zone)
+        _save()
+
+    logger.info("Analysis zone created: %s at (%.2f, %.2f)", title[:40], lat, lng)
+    return zone
+
+
+def list_zones() -> list[dict[str, Any]]:
+    """Return all live (non-expired) zones."""
+    with _lock:
+        _expire()
+        return list(_zones)
+
+
+def get_zone(zone_id: str) -> dict[str, Any] | None:
+    """Get a single zone by ID."""
+    with _lock:
+        for z in _zones:
+            if z["id"] == zone_id:
+                return dict(z)
+    return None
+
+
+def delete_zone(zone_id: str) -> bool:
+    """Delete a zone by ID. Returns True if found and removed."""
+    with _lock:
+        before = len(_zones)
+        _zones[:] = [z for z in _zones if z["id"] != zone_id]
+        if len(_zones) < before:
+            _save()
+            return True
+    return False
+
+
+def clear_zones(*, source: str | None = None) -> int:
+    """Clear all zones, optionally filtered by source. Returns count removed."""
+    with _lock:
+        before = len(_zones)
+        if source:
+            _zones[:] = [z for z in _zones if z.get("source") != source]
+        else:
+            _zones.clear()
+        removed = before - len(_zones)
+        if removed:
+            _save()
+        return removed
+
+
+def get_live_zones() -> list[dict[str, Any]]:
+    """Return zones formatted for the correlation engine merge.
+
+    This is called by compute_correlations() to inject agent-placed zones
+    into the correlations list that the frontend renders as map squares.
+    """
+    with _lock:
+        _expire()
+        return [dict(z) for z in _zones]

backend/services/api_settings.py

@@ -2,12 +2,23 @@
 API Settings management — serves the API key registry and allows updates.
 Keys are stored in the backend .env file and loaded via python-dotenv.
 """
+
 import os
 import re
+import tempfile
 from pathlib import Path
 
 # Path to the backend .env file
 ENV_PATH = Path(__file__).parent.parent / ".env"
+# Path to the example template that ships with the repo
+ENV_EXAMPLE_PATH = Path(__file__).parent.parent.parent / ".env.example"
+DATA_DIR = Path(os.environ.get("SB_DATA_DIR", str(Path(__file__).parent.parent / "data")))
+if not DATA_DIR.is_absolute():
+    DATA_DIR = Path(__file__).parent.parent / DATA_DIR
+OPERATOR_KEYS_ENV_PATH = Path(
+    os.environ.get("SHADOWBROKER_OPERATOR_KEYS_ENV", str(DATA_DIR / "operator_api_keys.env"))
+)
+_ENV_KEY_RE = re.compile(r"^[A-Z][A-Z0-9_]*$")
 
 # ---------------------------------------------------------------------------
 # API Registry — every external service the dashboard depends on
@@ -40,6 +51,15 @@ API_REGISTRY = [
         "url": "https://aisstream.io/",
         "required": True,
     },
+    {
+        "id": "gfw_api_token",
+        "env_key": "GFW_API_TOKEN",
+        "name": "Global Fishing Watch",
+        "description": "Bearer token for Global Fishing Watch fishing-vessel activity events (Fishing Activity map layer). Free registration at globalfishingwatch.org.",
+        "category": "Maritime",
+        "url": "https://globalfishingwatch.org/our-apis/",
+        "required": False,
+    },
     {
         "id": "adsb_lol",
         "env_key": None,
@@ -121,18 +141,163 @@ API_REGISTRY = [
         "url": "https://openmhz.com/",
         "required": False,
     },
+    {
+        "id": "shodan_api_key",
+        "env_key": "SHODAN_API_KEY",
+        "name": "Shodan — Operator API Key",
+        "description": "Paid Shodan API key for local operator-driven searches and temporary map overlays. Results are attributed to Shodan and are not merged into ShadowBroker core feeds.",
+        "category": "Reconnaissance",
+        "url": "https://account.shodan.io/billing",
+        "required": False,
+    },
+    {
+        "id": "finnhub_api_key",
+        "env_key": "FINNHUB_API_KEY",
+        "name": "Finnhub — API Key",
+        "description": "Free market data API. Defense stock quotes, congressional trading disclosures, and insider transactions. 60 calls/min free tier.",
+        "category": "Financial",
+        "url": "https://finnhub.io/register",
+        "required": False,
+    },
+    # Issue #298 (tg12): Sentinel Hub / Copernicus Data Space Ecosystem
+    # credentials were previously held in browser localStorage / sessionStorage
+    # by the Settings panel. Moved server-side to the same .env-backed
+    # store every other third-party API key lives in. The Sentinel proxy
+    # routes (POST /api/sentinel/token, /tile) now fall back to these
+    # env values when the request body omits credentials — see
+    # backend/routers/tools.py for the resolution order.
+    {
+        "id": "sentinel_client_id",
+        "env_key": "SENTINEL_CLIENT_ID",
+        "name": "Sentinel Hub / Copernicus — Client ID",
+        "description": "OAuth2 client ID for Copernicus Data Space Ecosystem (CDSE). Required for the Sentinel-2 imagery overlay and the right-click Sentinel-2 Intel Card. Sign in at dataspace.copernicus.eu and create OAuth credentials.",
+        "category": "Imagery",
+        "url": "https://dataspace.copernicus.eu/",
+        "required": False,
+    },
+    {
+        "id": "sentinel_client_secret",
+        "env_key": "SENTINEL_CLIENT_SECRET",
+        "name": "Sentinel Hub / Copernicus — Client Secret",
+        "description": "OAuth2 client secret paired with the Client ID above. Used by the backend to mint short-lived access tokens against the CDSE identity provider. Stored in the backend .env; never sent to the browser.",
+        "category": "Imagery",
+        "url": "https://dataspace.copernicus.eu/",
+        "required": False,
+    },
 ]
 
+ALLOWED_ENV_KEYS = {
+    str(api["env_key"])
+    for api in API_REGISTRY
+    if api.get("env_key")
+}
 
-def _obfuscate(value: str) -> str:
+
-    """Show first 4 chars, mask the rest with bullets."""
+def _parse_env_file(path: Path) -> dict[str, str]:
-    if not value or len(value) <= 4:
+    values: dict[str, str] = {}
-        return "••••••••"
+    if not path.exists():
-    return value[:4] + "•" * (len(value) - 4)
+        return values
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError:
+        return values
+    for raw_line in text.splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        key = key.strip()
+        if not _ENV_KEY_RE.match(key):
+            continue
+        value = value.strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
+            value = value[1:-1]
+        values[key] = value
+    return values
+
+
+def _quote_env_value(value: str) -> str:
+    escaped = value.replace("\\", "\\\\").replace('"', '\\"')
+    return f'"{escaped}"'
+
+
+def _write_env_values(path: Path, updates: dict[str, str]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    lines = path.read_text(encoding="utf-8").splitlines() if path.exists() else []
+    seen: set[str] = set()
+    next_lines: list[str] = []
+    for raw_line in lines:
+        stripped = raw_line.strip()
+        if "=" not in stripped or stripped.startswith("#"):
+            next_lines.append(raw_line)
+            continue
+        key = stripped.split("=", 1)[0].strip()
+        if key in updates:
+            next_lines.append(f"{key}={_quote_env_value(updates[key])}")
+            seen.add(key)
+        else:
+            next_lines.append(raw_line)
+    for key, value in updates.items():
+        if key not in seen:
+            next_lines.append(f"{key}={_quote_env_value(value)}")
+
+    fd, tmp_name = tempfile.mkstemp(dir=str(path.parent), prefix=f"{path.name}.tmp.", text=True)
+    tmp_path = Path(tmp_name)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
+            handle.write("\n".join(next_lines).rstrip() + "\n")
+        if os.name != "nt":
+            os.chmod(tmp_path, 0o600)
+        os.replace(tmp_path, path)
+        if os.name != "nt":
+            os.chmod(path, 0o600)
+    finally:
+        try:
+            if tmp_path.exists():
+                tmp_path.unlink()
+        except OSError:
+            pass
+
+
+def load_persisted_api_keys_into_environ() -> None:
+    """Load persisted operator API keys if no process env value exists."""
+    for key, value in _parse_env_file(OPERATOR_KEYS_ENV_PATH).items():
+        if key in ALLOWED_ENV_KEYS and value and not os.environ.get(key):
+            os.environ[key] = value
+
+
+def get_env_path_info() -> dict:
+    """Return absolute paths for the backend .env and .env.example template.
+
+    Surfaced to the frontend so the API Keys settings panel can tell users
+    exactly where to put their keys when in-app editing fails (admin-not-set,
+    file permissions, read-only filesystem, etc.).
+    """
+    env_path = ENV_PATH.resolve()
+    example_path = ENV_EXAMPLE_PATH.resolve()
+    return {
+        "env_path": str(env_path),
+        "env_path_exists": env_path.exists(),
+        "env_path_writable": os.access(env_path.parent, os.W_OK)
+            and (not env_path.exists() or os.access(env_path, os.W_OK)),
+        "env_example_path": str(example_path),
+        "env_example_path_exists": example_path.exists(),
+        "operator_keys_env_path": str(OPERATOR_KEYS_ENV_PATH.resolve()),
+        "operator_keys_env_path_exists": OPERATOR_KEYS_ENV_PATH.exists(),
+        "operator_keys_env_path_writable": os.access(OPERATOR_KEYS_ENV_PATH.parent, os.W_OK)
+            and (not OPERATOR_KEYS_ENV_PATH.exists() or os.access(OPERATOR_KEYS_ENV_PATH, os.W_OK)),
+    }
 
 
 def get_api_keys():
-    """Return the full API registry with obfuscated key values."""
+    """Return the API registry with a binary set/unset flag per key.
+
+    Key values themselves are NEVER returned to the client — not even an
+    obfuscated prefix. Users edit the .env file directly; the panel uses
+    `is_set` to render a CONFIGURED / NOT CONFIGURED badge and the path
+    info from `get_env_path_info()` to tell them where to put each key.
+    """
+    load_persisted_api_keys_into_environ()
     result = []
     for api in API_REGISTRY:
         entry = {
@@ -144,41 +309,64 @@ def get_api_keys():
             "required": api["required"],
             "has_key": api["env_key"] is not None,
             "env_key": api["env_key"],
-            "value_obfuscated": None,
             "is_set": False,
         }
         if api["env_key"]:
             raw = os.environ.get(api["env_key"], "")
-            entry["value_obfuscated"] = _obfuscate(raw)
             entry["is_set"] = bool(raw)
         result.append(entry)
     return result
 
 
-def update_api_key(env_key: str, new_value: str) -> bool:
+def save_api_keys(updates: dict[str, str]) -> dict:
-    """Update a single key in the .env file and in the current process env."""
+    """Persist allowed API keys from a local operator request.
-    valid_keys = {api["env_key"] for api in API_REGISTRY if api.get("env_key")}
-    if env_key not in valid_keys:
-        return False
 
-    if not isinstance(new_value, str):
+    Values are accepted write-only: the response includes only configured flags.
-        return False
+    """
-    if "\n" in new_value or "\r" in new_value:
+    clean: dict[str, str] = {}
-        return False
+    for key, value in updates.items():
+        env_key = str(key or "").strip().upper()
+        if env_key not in ALLOWED_ENV_KEYS:
+            continue
+        clean_value = str(value or "").strip()
+        if clean_value:
+            clean[env_key] = clean_value
+    if not clean:
+        return {"ok": False, "detail": "No supported API keys were provided."}
 
-    if not ENV_PATH.exists():
+    _write_env_values(OPERATOR_KEYS_ENV_PATH, clean)
-        ENV_PATH.write_text("", encoding="utf-8")
+    try:
+        _write_env_values(ENV_PATH, clean)
+    except OSError:
+        # The persistent operator key file is the source of truth for Docker.
+        pass
+    for key, value in clean.items():
+        os.environ[key] = value
+    if "AIS_API_KEY" in clean:
+        try:
+            from services import ais_stream
+            ais_stream.API_KEY = clean["AIS_API_KEY"]
+        except Exception:
+            pass
+    if "OPENSKY_CLIENT_ID" in clean or "OPENSKY_CLIENT_SECRET" in clean:
+        try:
+            from services.fetchers import flights
+            flights.opensky_client.client_id = os.environ.get("OPENSKY_CLIENT_ID", "")
+            flights.opensky_client.client_secret = os.environ.get("OPENSKY_CLIENT_SECRET", "")
+            flights.opensky_client.token = None
+            flights.opensky_client.expires_at = 0
+        except Exception:
+            pass
 
-    # Update os.environ immediately
+    try:
-    os.environ[env_key] = new_value
+        from services.config import get_settings
+        get_settings.cache_clear()
+    except Exception:
+        pass
 
-    # Update the .env file on disk
+    return {
-    content = ENV_PATH.read_text(encoding="utf-8")
+        "ok": True,
-    pattern = re.compile(rf"^{re.escape(env_key)}=.*$", re.MULTILINE)
+        "updated": sorted(clean.keys()),
-    if pattern.search(content):
+        "keys": get_api_keys(),
-        content = pattern.sub(f"{env_key}={new_value}", content)
+        "env": get_env_path_info(),
-    else:
+    }
-        content = content.rstrip("\n") + f"\n{env_key}={new_value}\n"
-
-    ENV_PATH.write_text(content, encoding="utf-8")
-    return True

backend/services/carrier_tracker.py

@@ -1,135 +1,175 @@
 """
 Carrier Strike Group OSINT Tracker
 ===================================
-Scrapes multiple OSINT sources to maintain current estimated positions
+Maintains estimated positions for US Navy Carrier Strike Groups with
-for US Navy Carrier Strike Groups. Updates on startup + 00:00 & 12:00 UTC.
+honest provenance and freshness signals.
 
-Sources:
+Issues #244 / #245 / #246 (tg12 external audit):
-  1. GDELT News API — recent carrier movement headlines
+
-  2. WikiVoyage / public port-call databases
+The previous implementation baked a snapshot of USNI News Fleet &
-  3. Fallback — last-known or static OSINT estimates
+Marine Tracker positions (March 9, 2026) into the registry as
+``fallback_lat``/``fallback_lng`` and stamped ``updated = now()``
+every time the dossier was rendered. That presented stale editorial
+data as live state. It also persisted GDELT-derived positions to the
+on-disk cache with no freshness signal, so a single news mention from
+months ago could keep overriding the (already-stale) registry default
+indefinitely.
+
+Architecture after this PR:
+
+::
+
+    backend/data/carrier_seed.json   read-only, shipped with image,
+                                     used ONCE on first-ever startup
+                                     to bootstrap carrier_cache.json.
+
+    backend/data/carrier_cache.json  mutable, lives in the runtime data
+                                     volume, written by every GDELT
+                                     refresh + any future source.
+
+Startup flow:
+
+1.  ``carrier_cache.json`` exists?  → load it.
+2.  Otherwise, copy ``carrier_seed.json`` → ``carrier_cache.json``,
+    then load it. (This happens once, ever, per install.)
+3.  Background: GDELT fetch runs. Any carrier mentioned in fresh news
+    gets its entry replaced with the news-derived position.
+    ``position_source_at`` is set to the news article timestamp.
+
+Freshness is a *labelling* decision, not an eviction decision:
+
+- ``position_source_at`` within the configurable freshness window
+  (default 14 days) → ``position_confidence = "recent"``.
+- Older than that              → ``position_confidence = "stale"``.
+- Bootstrapped from the seed file (never updated) → ``"seed"``.
+- No cache entry at all (e.g. a carrier added to the registry after
+  first install) → carrier renders at its homeport with
+  ``"homeport_default"``.
+
+Carriers are never hidden, never teleported, never disappeared. The
+position the user sees is always the last position the system actually
+observed, with an honest "as-of" timestamp the UI can render however
+it likes. A year from now, the runtime cache reflects whatever this
+install has observed via GDELT — not the seed snapshot.
 """
 
-import re
+import os
 import json
 import time
 import logging
 import threading
-from datetime import datetime, timezone
+import random
+import shutil
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 from services.network_utils import fetch_with_curl
 
 logger = logging.getLogger(__name__)
 
 # -----------------------------------------------------------------
-# Carrier registry: hull number → metadata + fallback position
+# Carrier registry: hull number → identity only.
+#
+# Issue #244 (tg12): the previous registry carried hard-coded
+# ``fallback_lat``/``fallback_lng`` that were dated editorial
+# snapshots from a 2026-03-09 article. Those fields are DELETED. The
+# registry is now identity + homeport only; positions are sourced
+# exclusively from carrier_cache.json (and via that, from the
+# bootstrap seed or live OSINT).
 # -----------------------------------------------------------------
 CARRIER_REGISTRY: Dict[str, dict] = {
+    # --- Bremerton, WA (Naval Base Kitsap) ---
     "CVN-68": {
         "name": "USS Nimitz (CVN-68)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Nimitz",
         "homeport": "Bremerton, WA",
-        "homeport_lat": 47.56, "homeport_lng": -122.63,
+        "homeport_lat": 47.5535,
-        "fallback_lat": 21.35, "fallback_lng": -157.95,
+        "homeport_lng": -122.6400,
-        "fallback_heading": 270,
-        "fallback_desc": "Pacific Fleet / Pearl Harbor"
-    },
-    "CVN-69": {
-        "name": "USS Dwight D. Eisenhower (CVN-69)",
-        "wiki": "https://en.wikipedia.org/wiki/USS_Dwight_D._Eisenhower",
-        "homeport": "Norfolk, VA",
-        "homeport_lat": 36.95, "homeport_lng": -76.33,
-        "fallback_lat": 18.0, "fallback_lng": 39.5,
-        "fallback_heading": 120,
-        "fallback_desc": "Red Sea / CENTCOM AOR"
-    },
-    "CVN-78": {
-        "name": "USS Gerald R. Ford (CVN-78)",
-        "wiki": "https://en.wikipedia.org/wiki/USS_Gerald_R._Ford",
-        "homeport": "Norfolk, VA",
-        "homeport_lat": 36.95, "homeport_lng": -76.33,
-        "fallback_lat": 34.0, "fallback_lng": 25.0,
-        "fallback_heading": 90,
-        "fallback_desc": "Eastern Mediterranean deterrence"
-    },
-    "CVN-70": {
-        "name": "USS Carl Vinson (CVN-70)",
-        "wiki": "https://en.wikipedia.org/wiki/USS_Carl_Vinson",
-        "homeport": "San Diego, CA",
-        "homeport_lat": 32.68, "homeport_lng": -117.15,
-        "fallback_lat": 15.0, "fallback_lng": 115.0,
-        "fallback_heading": 45,
-        "fallback_desc": "South China Sea patrol"
-    },
-    "CVN-71": {
-        "name": "USS Theodore Roosevelt (CVN-71)",
-        "wiki": "https://en.wikipedia.org/wiki/USS_Theodore_Roosevelt_(CVN-71)",
-        "homeport": "San Diego, CA",
-        "homeport_lat": 32.68, "homeport_lng": -117.15,
-        "fallback_lat": 22.0, "fallback_lng": 122.0,
-        "fallback_heading": 300,
-        "fallback_desc": "Philippine Sea / Taiwan Strait"
-    },
-    "CVN-72": {
-        "name": "USS Abraham Lincoln (CVN-72)",
-        "wiki": "https://en.wikipedia.org/wiki/USS_Abraham_Lincoln_(CVN-72)",
-        "homeport": "San Diego, CA",
-        "homeport_lat": 32.68, "homeport_lng": -117.15,
-        "fallback_lat": 21.0, "fallback_lng": -158.0,
-        "fallback_heading": 270,
-        "fallback_desc": "Pacific deployment"
-    },
-    "CVN-73": {
-        "name": "USS George Washington (CVN-73)",
-        "wiki": "https://en.wikipedia.org/wiki/USS_George_Washington_(CVN-73)",
-        "homeport": "Yokosuka, Japan",
-        "homeport_lat": 35.28, "homeport_lng": 139.67,
-        "fallback_lat": 35.0, "fallback_lng": 139.0,
-        "fallback_heading": 0,
-        "fallback_desc": "Yokosuka, Japan (Forward deployed)"
-    },
-    "CVN-74": {
-        "name": "USS John C. Stennis (CVN-74)",
-        "wiki": "https://en.wikipedia.org/wiki/USS_John_C._Stennis",
-        "homeport": "Norfolk, VA",
-        "homeport_lat": 36.95, "homeport_lng": -76.33,
-        "fallback_lat": 36.95, "fallback_lng": -76.33,
-        "fallback_heading": 0,
-        "fallback_desc": "RCOH / Norfolk (maintenance)"
-    },
-    "CVN-75": {
-        "name": "USS Harry S. Truman (CVN-75)",
-        "wiki": "https://en.wikipedia.org/wiki/USS_Harry_S._Truman",
-        "homeport": "Norfolk, VA",
-        "homeport_lat": 36.95, "homeport_lng": -76.33,
-        "fallback_lat": 36.0, "fallback_lng": 15.0,
-        "fallback_heading": 90,
-        "fallback_desc": "Mediterranean deployment"
     },
     "CVN-76": {
         "name": "USS Ronald Reagan (CVN-76)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Ronald_Reagan",
         "homeport": "Bremerton, WA",
-        "homeport_lat": 47.56, "homeport_lng": -122.63,
+        "homeport_lat": 47.5580,
-        "fallback_lat": 47.56, "fallback_lng": -122.63,
+        "homeport_lng": -122.6360,
-        "fallback_heading": 0,
+    },
-        "fallback_desc": "Bremerton, WA (Homeport)"
+    # --- Norfolk, VA (Naval Station Norfolk) ---
+    "CVN-69": {
+        "name": "USS Dwight D. Eisenhower (CVN-69)",
+        "wiki": "https://en.wikipedia.org/wiki/USS_Dwight_D._Eisenhower",
+        "homeport": "Norfolk, VA",
+        "homeport_lat": 36.9465,
+        "homeport_lng": -76.3265,
+    },
+    "CVN-78": {
+        "name": "USS Gerald R. Ford (CVN-78)",
+        "wiki": "https://en.wikipedia.org/wiki/USS_Gerald_R._Ford",
+        "homeport": "Norfolk, VA",
+        "homeport_lat": 36.9505,
+        "homeport_lng": -76.3250,
+    },
+    "CVN-74": {
+        "name": "USS John C. Stennis (CVN-74)",
+        "wiki": "https://en.wikipedia.org/wiki/USS_John_C._Stennis",
+        "homeport": "Norfolk, VA",
+        "homeport_lat": 36.9540,
+        "homeport_lng": -76.3235,
+    },
+    "CVN-75": {
+        "name": "USS Harry S. Truman (CVN-75)",
+        "wiki": "https://en.wikipedia.org/wiki/USS_Harry_S._Truman",
+        "homeport": "Norfolk, VA",
+        "homeport_lat": 36.9580,
+        "homeport_lng": -76.3220,
     },
     "CVN-77": {
         "name": "USS George H.W. Bush (CVN-77)",
         "wiki": "https://en.wikipedia.org/wiki/USS_George_H.W._Bush",
         "homeport": "Norfolk, VA",
-        "homeport_lat": 36.95, "homeport_lng": -76.33,
+        "homeport_lat": 36.9620,
-        "fallback_lat": 36.95, "fallback_lng": -76.33,
+        "homeport_lng": -76.3210,
-        "fallback_heading": 0,
+    },
-        "fallback_desc": "Norfolk, VA (Homeport)"
+    # --- San Diego, CA (Naval Base San Diego) ---
+    "CVN-70": {
+        "name": "USS Carl Vinson (CVN-70)",
+        "wiki": "https://en.wikipedia.org/wiki/USS_Carl_Vinson",
+        "homeport": "San Diego, CA",
+        "homeport_lat": 32.6840,
+        "homeport_lng": -117.1290,
+    },
+    "CVN-71": {
+        "name": "USS Theodore Roosevelt (CVN-71)",
+        "wiki": "https://en.wikipedia.org/wiki/USS_Theodore_Roosevelt_(CVN-71)",
+        "homeport": "San Diego, CA",
+        "homeport_lat": 32.6885,
+        "homeport_lng": -117.1280,
+    },
+    "CVN-72": {
+        "name": "USS Abraham Lincoln (CVN-72)",
+        "wiki": "https://en.wikipedia.org/wiki/USS_Abraham_Lincoln_(CVN-72)",
+        "homeport": "San Diego, CA",
+        "homeport_lat": 32.6925,
+        "homeport_lng": -117.1275,
+    },
+    # --- Yokosuka, Japan (CFAY) ---
+    "CVN-73": {
+        "name": "USS George Washington (CVN-73)",
+        "wiki": "https://en.wikipedia.org/wiki/USS_George_Washington_(CVN-73)",
+        "homeport": "Yokosuka, Japan",
+        "homeport_lat": 35.2830,
+        "homeport_lng": 139.6700,
     },
 }
 
 # -----------------------------------------------------------------
-# Region → approximate center coordinates
+# Region → approximate center coordinates.
-# Used to map textual geographic descriptions to lat/lng
+#
+# Issue #245 (tg12): converting a region name straight into precise
+# map coordinates is false precision. We still use this table to
+# infer a coarse position from a headline mention, but the resulting
+# carrier object is now stamped ``position_confidence = "approximate"``
+# so the UI can render an uncertainty radius / dimmed icon. The
+# centroid is a best-effort midpoint of the named body of water.
 # -----------------------------------------------------------------
 REGION_COORDS: Dict[str, tuple] = {
     # Oceans & Seas
@@ -163,7 +203,6 @@ REGION_COORDS: Dict[str, tuple] = {
     "coral sea": (-18.0, 155.0),
     "gulf of mexico": (25.0, -90.0),
     "caribbean": (15.0, -75.0),
-
     # Specific bases / ports
     "norfolk": (36.95, -76.33),
     "san diego": (32.68, -117.15),
@@ -176,7 +215,6 @@ REGION_COORDS: Dict[str, tuple] = {
     "bremerton": (47.56, -122.63),
     "puget sound": (47.56, -122.63),
     "newport news": (36.98, -76.43),
-
     # Areas of operation
     "centcom": (25.0, 55.0),
     "indopacom": (20.0, 130.0),
@@ -190,34 +228,203 @@ REGION_COORDS: Dict[str, tuple] = {
 }
 
 # -----------------------------------------------------------------
-# Cache file for persisting positions between restarts
+# Files
 # -----------------------------------------------------------------
-CACHE_FILE = Path(__file__).parent.parent / "carrier_cache.json"
+#
+# The seed lives in the read-only image data dir (it ships with each
+# release). The cache lives in the same data dir but is written at
+# runtime; under Docker compose this dir is volume-mounted so the
+# cache persists across container restarts, which is the whole point
+# of the seed-then-observe model — the user's runtime observations
+# survive image upgrades.
+SEED_FILE = Path(__file__).parent.parent / "data" / "carrier_seed.json"
+CACHE_FILE = Path(__file__).parent.parent / "data" / "carrier_cache.json"
+
+# -----------------------------------------------------------------
+# Freshness window for position_confidence labeling. Issue #246 (tg12):
+# previously persisted cache entries had no freshness signal at all.
+# After this change, the position itself is preserved (we never lose
+# what was last observed) but the confidence label flips from
+# "recent" to "stale" once the underlying source is older than this
+# window. Operator-overridable via env var.
+# -----------------------------------------------------------------
+_DEFAULT_FRESHNESS_WINDOW_DAYS = 14
+
+
+def _freshness_window_days() -> int:
+    raw = str(os.environ.get("SHADOWBROKER_CARRIER_FRESHNESS_DAYS", "") or "").strip()
+    if not raw:
+        return _DEFAULT_FRESHNESS_WINDOW_DAYS
+    try:
+        n = int(raw)
+        return n if n > 0 else _DEFAULT_FRESHNESS_WINDOW_DAYS
+    except (TypeError, ValueError):
+        return _DEFAULT_FRESHNESS_WINDOW_DAYS
+
 
 _carrier_positions: Dict[str, dict] = {}
 _positions_lock = threading.Lock()
 _last_update: Optional[datetime] = None
+_last_gdelt_fetch_at = 0.0
+_cached_gdelt_articles: List[dict] = []
+_GDELT_FETCH_INTERVAL_SECONDS = 1800
+_GDELT_REQUEST_DELAY_SECONDS = 1.25
+_GDELT_REQUEST_JITTER_SECONDS = 0.35
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _parse_iso(ts: str) -> Optional[datetime]:
+    if not ts:
+        return None
+    try:
+        # Python's fromisoformat accepts +00:00 but not 'Z' until 3.11.
+        normalized = ts.replace("Z", "+00:00")
+        dt = datetime.fromisoformat(normalized)
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        return dt
+    except (TypeError, ValueError):
+        return None
+
+
+def _compute_position_confidence(entry: dict, *, now: Optional[datetime] = None) -> str:
+    """Return the public confidence label for a carrier cache entry.
+
+    Order of precedence:
+      - explicit "homeport_default" / "seed" labels are preserved.
+      - dated entries (with position_source_at) are "recent" if within
+        the configured freshness window, else "stale".
+      - missing position_source_at falls through to "stale".
+    """
+    raw_label = str(entry.get("position_confidence", "") or "").strip()
+    # Explicit "kind of provenance" labels are preserved as-is. They
+    # describe HOW we got the position, not WHEN — a fresh headline-to-
+    # centroid match (#245) is still imprecise no matter how recently
+    # it was observed, and the seed (#244) is always the seed.
+    if raw_label in {"seed", "homeport_default", "approximate"}:
+        # Approximate entries can still age into "stale_approximate" if
+        # they fall out of the freshness window — that distinction lets
+        # the UI render a different badge for old-and-imprecise vs
+        # recent-and-imprecise. seed/homeport_default never age (they
+        # were never timestamped against real observations).
+        if raw_label == "approximate":
+            source_at = _parse_iso(str(entry.get("position_source_at", "") or ""))
+            if source_at is not None:
+                reference = now or datetime.now(timezone.utc)
+                if reference - source_at > timedelta(days=_freshness_window_days()):
+                    return "stale_approximate"
+        return raw_label
+
+    source_at = _parse_iso(str(entry.get("position_source_at", "") or ""))
+    if not source_at:
+        return "stale"
+
+    reference = now or datetime.now(timezone.utc)
+    window = timedelta(days=_freshness_window_days())
+    if reference - source_at <= window:
+        return "recent"
+    return "stale"
+
+
+def _load_seed() -> Dict[str, dict]:
+    """Load the read-only seed file shipped with the image.
+
+    Returns a hull→entry dict (no _meta wrapper). Missing or malformed
+    seed files yield an empty dict — the caller falls back to homeport
+    defaults.
+    """
+    try:
+        if not SEED_FILE.exists():
+            logger.info("Carrier seed file not present at %s; first-run will fall back to homeport defaults", SEED_FILE)
+            return {}
+        raw = json.loads(SEED_FILE.read_text(encoding="utf-8"))
+        carriers = raw.get("carriers", {}) if isinstance(raw, dict) else {}
+        if not isinstance(carriers, dict):
+            return {}
+        logger.info("Carrier seed loaded: %d entries from %s", len(carriers), SEED_FILE)
+        return carriers
+    except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
+        logger.warning("Failed to load carrier seed file %s: %s", SEED_FILE, e)
+        return {}
 
 
 def _load_cache() -> Dict[str, dict]:
-    """Load cached carrier positions from disk."""
+    """Load the mutable cache (last-known positions persisted between restarts)."""
     try:
         if CACHE_FILE.exists():
-            data = json.loads(CACHE_FILE.read_text())
+            data = json.loads(CACHE_FILE.read_text(encoding="utf-8"))
-            logger.info(f"Carrier cache loaded: {len(data)} carriers from {CACHE_FILE}")
+            if isinstance(data, dict):
-            return data
+                logger.info("Carrier cache loaded: %d carriers from %s", len(data), CACHE_FILE)
-    except Exception as e:
+                return data
-        logger.warning(f"Failed to load carrier cache: {e}")
+    except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
+        logger.warning("Failed to load carrier cache: %s", e)
     return {}
 
 
-def _save_cache(positions: Dict[str, dict]):
+def _save_cache(positions: Dict[str, dict]) -> None:
-    """Persist carrier positions to disk."""
+    """Persist the mutable cache. Atomic write (temp + rename) so a crash
+    mid-write can't leave the file truncated."""
     try:
-        CACHE_FILE.write_text(json.dumps(positions, indent=2))
+        CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
-        logger.info(f"Carrier cache saved: {len(positions)} carriers")
+        tmp = CACHE_FILE.with_suffix(CACHE_FILE.suffix + ".tmp")
-    except Exception as e:
+        tmp.write_text(json.dumps(positions, indent=2), encoding="utf-8")
-        logger.warning(f"Failed to save carrier cache: {e}")
+        # On Windows os.replace is atomic and overwrites existing files.
+        os.replace(tmp, CACHE_FILE)
+        logger.info("Carrier cache saved: %d carriers", len(positions))
+    except (IOError, OSError) as e:
+        logger.warning("Failed to save carrier cache: %s", e)
+
+
+def _homeport_entry_for(hull: str) -> Optional[dict]:
+    """Return a homeport-default cache entry for a hull, or None if the
+    hull is not in the registry."""
+    info = CARRIER_REGISTRY.get(hull)
+    if not info:
+        return None
+    return {
+        "lat": info["homeport_lat"],
+        "lng": info["homeport_lng"],
+        "heading": 0,
+        "desc": f"{info['homeport']} (no observations yet)",
+        "source": f"Homeport default ({info['homeport']})",
+        "source_url": info.get("wiki", ""),
+        "position_source_at": _now_iso(),
+        "position_confidence": "homeport_default",
+    }
+
+
+def _bootstrap_cache_if_missing() -> Dict[str, dict]:
+    """One-shot: if no cache exists, materialize one from the seed file.
+
+    Returns the cache contents (hull→entry). On first-ever startup,
+    this writes ``carrier_cache.json`` so subsequent restarts skip the
+    seed entirely. Operator-deleted caches re-bootstrap the same way —
+    operators can use that to "reset" carrier positions, but it's an
+    explicit operator action.
+    """
+    if CACHE_FILE.exists():
+        return _load_cache()
+
+    seed = _load_seed()
+    if not seed:
+        # No seed file either. Build a homeport-default cache so the
+        # first save_cache call still produces something honest.
+        homeports: Dict[str, dict] = {}
+        for hull in CARRIER_REGISTRY:
+            entry = _homeport_entry_for(hull)
+            if entry is not None:
+                homeports[hull] = entry
+        if homeports:
+            _save_cache(homeports)
+        return homeports
+
+    # Persist the seed as the first cache so subsequent runs skip this branch.
+    _save_cache(seed)
+    logger.info("Carrier cache bootstrapped from seed (first-ever startup)")
+    return dict(seed)
 
 
 def _match_region(text: str) -> Optional[tuple]:
@@ -235,10 +442,8 @@ def _match_carrier(text: str) -> Optional[str]:
     for hull, info in CARRIER_REGISTRY.items():
         hull_check = hull.lower().replace("-", "")
         name_parts = info["name"].lower()
-        # Match hull number (e.g., "CVN-78", "CVN78")
         if hull.lower() in text_lower or hull_check in text_lower.replace("-", ""):
             return hull
-        # Match ship name (e.g., "Ford", "Eisenhower", "Vinson")
         ship_name = name_parts.split("(")[0].strip()
         last_name = ship_name.split()[-1] if ship_name else ""
         if last_name and len(last_name) > 3 and last_name in text_lower:
@@ -248,114 +453,233 @@ def _match_carrier(text: str) -> Optional[str]:
 
 def _fetch_gdelt_carrier_news() -> List[dict]:
     """Search GDELT for recent carrier movement news."""
+    global _last_gdelt_fetch_at, _cached_gdelt_articles
+
+    now = time.time()
+    if _cached_gdelt_articles and (now - _last_gdelt_fetch_at) < _GDELT_FETCH_INTERVAL_SECONDS:
+        logger.info("Carrier OSINT: using cached GDELT article set to avoid startup bursts")
+        return list(_cached_gdelt_articles)
+
     results = []
     search_terms = [
         "aircraft+carrier+deployed",
         "carrier+strike+group+navy",
-        "USS+Nimitz+carrier", "USS+Ford+carrier", "USS+Eisenhower+carrier",
+        "USS+Nimitz+carrier",
-        "USS+Vinson+carrier", "USS+Roosevelt+carrier+navy",
+        "USS+Ford+carrier",
-        "USS+Lincoln+carrier", "USS+Truman+carrier",
+        "USS+Eisenhower+carrier",
-        "USS+Reagan+carrier", "USS+Washington+carrier+navy",
+        "USS+Vinson+carrier",
-        "USS+Bush+carrier", "USS+Stennis+carrier",
+        "USS+Roosevelt+carrier+navy",
+        "USS+Lincoln+carrier",
+        "USS+Truman+carrier",
+        "USS+Reagan+carrier",
+        "USS+Washington+carrier+navy",
+        "USS+Bush+carrier",
+        "USS+Stennis+carrier",
     ]
 
-    for term in search_terms:
+    for idx, term in enumerate(search_terms):
         try:
             url = f"https://api.gdeltproject.org/api/v2/doc/doc?query={term}&mode=artlist&maxrecords=5&format=json&timespan=14d"
             raw = fetch_with_curl(url, timeout=8)
-            if not raw:
+            if getattr(raw, "status_code", 500) == 429:
+                logger.warning(
+                    "GDELT returned 429 for '%s'; preserving cached carrier OSINT results",
+                    term,
+                )
                 continue
-            data = json.loads(raw)
+            if not raw or not hasattr(raw, "text"):
+                continue
+            data = raw.json()
             articles = data.get("articles", [])
             for art in articles:
                 title = art.get("title", "")
-                url = art.get("url", "")
+                article_url = art.get("url", "")
-                results.append({"title": title, "url": url})
+                article_at = art.get("seendate") or art.get("date") or ""
-        except Exception as e:
+                results.append({"title": title, "url": article_url, "seendate": article_at})
+        except (ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
             logger.debug(f"GDELT search failed for '{term}': {e}")
             continue
+        if idx < len(search_terms) - 1:
+            time.sleep(
+                _GDELT_REQUEST_DELAY_SECONDS
+                + random.uniform(0.0, _GDELT_REQUEST_JITTER_SECONDS)
+            )
 
+    _cached_gdelt_articles = list(results)
+    _last_gdelt_fetch_at = time.time()
     logger.info(f"Carrier OSINT: found {len(results)} GDELT articles")
     return results
 
 
+def _gdelt_seendate_to_iso(seendate: str) -> Optional[str]:
+    """GDELT returns YYYYMMDDhhmmss (UTC). Convert to ISO8601 for
+    position_source_at. Returns None if the input is unparseable."""
+    raw = (seendate or "").strip()
+    if len(raw) < 8 or not raw.isdigit():
+        return None
+    try:
+        dt = datetime.strptime(raw[:14] if len(raw) >= 14 else raw[:8] + "000000", "%Y%m%d%H%M%S")
+        return dt.replace(tzinfo=timezone.utc).isoformat()
+    except (TypeError, ValueError):
+        return None
+
+
 def _parse_carrier_positions_from_news(articles: List[dict]) -> Dict[str, dict]:
-    """Parse carrier positions from news article titles and descriptions."""
+    """Parse carrier positions from news article titles.
+
+    Issue #245 (tg12): the position is a region centroid, which is
+    coarse — we now stamp ``position_confidence = "approximate"`` so
+    the UI can render that uncertainty. Issue #244: the
+    ``position_source_at`` field is the news article's actual seen
+    date, NOT now(), so the freshness check correctly flips entries
+    to "stale" once they age past the configured window.
+    """
     updates: Dict[str, dict] = {}
 
     for article in articles:
         title = article.get("title", "")
-
-        # Try to match a carrier from the title
         hull = _match_carrier(title)
         if not hull:
             continue
-
-        # Try to match a region from the title
         coords = _match_region(title)
         if not coords:
             continue
 
-        # Only update if we haven't seen this carrier yet (first match wins — most recent)
+        # First match wins (most recent article, GDELT returns newest first
+        # per term).
         if hull not in updates:
+            iso_at = _gdelt_seendate_to_iso(str(article.get("seendate", ""))) or _now_iso()
             updates[hull] = {
                 "lat": coords[0],
                 "lng": coords[1],
+                "heading": 0,
                 "desc": title[:100],
-                "source": "GDELT OSINT",
+                "source": "GDELT News API (headline region match — approximate)",
-                "updated": datetime.now(timezone.utc).isoformat()
+                "source_url": article.get("url", "https://api.gdeltproject.org"),
+                "position_source_at": iso_at,
+                # Headline-to-centroid match is explicitly approximate.
+                "position_confidence": "approximate",
             }
-            logger.info(f"Carrier update: {CARRIER_REGISTRY[hull]['name']} → {coords} (from: {title[:80]})")
+            logger.info(
+                "Carrier update: %s → %s (from: %s)",
+                CARRIER_REGISTRY[hull]["name"],
+                coords,
+                title[:80],
+            )
 
     return updates
 
 
-def update_carrier_positions():
+def _enrich_for_rendering(hull: str, entry: dict, *, now: Optional[datetime] = None) -> dict:
-    """Main update function — called on startup and every 12h."""
+    """Add live computed fields (confidence label, last_osint_update)
+    on top of the persisted cache entry. The persisted entry is left
+    untouched; this function builds the public-facing object.
+    """
+    info = CARRIER_REGISTRY.get(hull, {})
+    confidence = _compute_position_confidence(entry, now=now)
+    return {
+        "name": entry.get("name", info.get("name", hull)),
+        "lat": entry["lat"],
+        "lng": entry["lng"],
+        "heading": entry.get("heading", 0),
+        "desc": entry.get("desc", ""),
+        "wiki": entry.get("wiki", info.get("wiki", "")),
+        "source": entry.get("source", "OSINT estimated position"),
+        "source_url": entry.get("source_url", ""),
+        "position_source_at": entry.get("position_source_at", ""),
+        "position_confidence": confidence,
+        # Existing field preserved for backward compatibility with the
+        # current frontend ShipPopup; now reflects the SOURCE's observed
+        # time (not now()), so "last reported X days ago" is honest.
+        "last_osint_update": entry.get("position_source_at", ""),
+        # Convenience boolean for the UI: true when the position is
+        # NOT live OSINT (used to render dimmed icons / badges).
+        "is_fallback": confidence in {"seed", "stale", "stale_approximate", "homeport_default"},
+    }
+
+
+def update_carrier_positions() -> None:
+    """Refresh carrier positions.
+
+    Phase 1 (instant): publish whatever's in carrier_cache.json (or
+    bootstrap from seed on first-ever run), so the map has carriers
+    immediately.
+
+    Phase 2 (slow): query GDELT and replace position entries for any
+    carrier mentioned in fresh news. Persist back to cache.
+    """
     global _last_update
 
-    logger.info("Carrier tracker: updating positions from OSINT sources...")
+    # --- Phase 1: instant cache (bootstrap from seed on first-ever run) ---
+    positions = _bootstrap_cache_if_missing()
 
-    # Start with fallback positions
+    # Ensure every registered hull has SOMETHING in the cache. A hull
-    positions: Dict[str, dict] = {}
+    # the seed didn't cover (e.g. added after install) renders at its
-    for hull, info in CARRIER_REGISTRY.items():
+    # homeport with "homeport_default" confidence.
-        positions[hull] = {
+    for hull in CARRIER_REGISTRY:
-            "name": info["name"],
+        if hull not in positions:
-            "lat": info["fallback_lat"],
+            entry = _homeport_entry_for(hull)
-            "lng": info["fallback_lng"],
+            if entry is not None:
-            "heading": info["fallback_heading"],
+                positions[hull] = entry
-            "desc": info["fallback_desc"],
-            "wiki": info["wiki"],
-            "source": "Static OSINT estimate",
-            "updated": datetime.now(timezone.utc).isoformat()
-        }
 
-    # Load cached positions (may have better data from previous runs)
+    with _positions_lock:
-    cached = _load_cache()
+        if not _carrier_positions:
-    for hull, cached_pos in cached.items():
+            _carrier_positions.update(positions)
-        if hull in positions:
+            _last_update = datetime.now(timezone.utc)
-            # Only use cache if it has a real OSINT source (not just static)
+    logger.info(
-            if cached_pos.get("source", "").startswith("GDELT") or cached_pos.get("source", "").startswith("News"):
+        "Carrier tracker: %d carriers loaded from cache (USNI + GDELT enrichment starting...)",
-                positions[hull].update({
+        len(positions),
-                    "lat": cached_pos["lat"],
+    )
-                    "lng": cached_pos["lng"],
-                    "desc": cached_pos.get("desc", positions[hull]["desc"]),
-                    "source": cached_pos.get("source", "Cached OSINT"),
-                    "updated": cached_pos.get("updated", "")
-                })
 
-    # Try GDELT news for fresh positions
+    # --- Phase 2: USNI Fleet & Marine Tracker (PRIMARY source) ---
+    #
+    # USNI publishes a weekly editorial tracker with each carrier's
+    # actual operating area, parsed from explicit prose like
+    #   "The Gerald R. Ford Carrier Strike Group is operating in the Red Sea"
+    # These positions are tagged ``position_confidence: "recent"`` because
+    # they reflect actual reporting, not headline-keyword centroids.
+    # USNI updates are preferred over GDELT — they're authoritative on
+    # US Navy positions where GDELT is just article-title text mining.
+    try:
+        from services.fetchers.usni_fleet_tracker import (
+            fetch_latest_fleet_tracker_positions,
+        )
+        usni_positions = fetch_latest_fleet_tracker_positions()
+        for hull, pos in usni_positions.items():
+            positions[hull] = pos
+            logger.info(
+                "Carrier USNI update: %s → %s",
+                CARRIER_REGISTRY[hull]["name"],
+                pos.get("desc", ""),
+            )
+    except Exception as e:
+        logger.warning("USNI fleet-tracker fetch failed: %s", e)
+
+    # --- Phase 3: GDELT enrichment (SECONDARY — fills gaps) ---
+    #
+    # Used only to backfill carriers USNI didn't mention this week. The
+    # position is stamped ``approximate`` so the UI knows it's a
+    # headline-centroid match (Issue #245).
     try:
         articles = _fetch_gdelt_carrier_news()
         news_positions = _parse_carrier_positions_from_news(articles)
         for hull, pos in news_positions.items():
-            if hull in positions:
+            # Only overwrite if the existing entry is NOT a recent USNI
-                positions[hull].update(pos)
+            # observation. A "recent" USNI position is higher-confidence
-                logger.info(f"Carrier OSINT: updated {CARRIER_REGISTRY[hull]['name']} from news")
+            # than a GDELT headline-centroid match — don't let GDELT
-    except Exception as e:
+            # demote a real position to an approximate one.
-        logger.warning(f"GDELT carrier fetch failed: {e}")
+            existing = positions.get(hull, {})
+            existing_conf = _compute_position_confidence(existing)
+            if existing_conf == "recent":
+                continue
+            positions[hull] = pos
+            logger.info(
+                "Carrier OSINT: updated %s from GDELT news",
+                CARRIER_REGISTRY[hull]["name"],
+            )
+    except (ValueError, KeyError, json.JSONDecodeError, OSError) as e:
+        logger.warning("GDELT carrier fetch failed: %s", e)
 
-    # Save and update the global state
     with _positions_lock:
         _carrier_positions.clear()
         _carrier_positions.update(positions)
@@ -363,39 +687,93 @@ def update_carrier_positions():
 
     _save_cache(positions)
 
-    sources = {}
+    confidences: Dict[str, int] = {}
-    for p in positions.values():
+    for entry in positions.values():
-        src = p.get("source", "unknown")
+        label = _compute_position_confidence(entry)
-        sources[src] = sources.get(src, 0) + 1
+        confidences[label] = confidences.get(label, 0) + 1
-    logger.info(f"Carrier tracker: {len(positions)} carriers updated. Sources: {sources}")
+    logger.info("Carrier tracker: %d carriers updated. Confidence: %s", len(positions), confidences)
+
+
+def _deconflict_positions(result: List[dict]) -> List[dict]:
+    """Offset carriers that share identical coordinates so they don't stack."""
+    from collections import defaultdict
+
+    groups: dict[str, list[int]] = defaultdict(list)
+    for i, c in enumerate(result):
+        key = f"{round(c['lat'], 2)},{round(c['lng'], 2)}"
+        groups[key].append(i)
+
+    for indices in groups.values():
+        if len(indices) < 2:
+            continue
+        n = len(indices)
+        sample = result[indices[0]]
+        at_port = any(
+            abs(sample["lat"] - info.get("homeport_lat", 0)) < 0.05
+            and abs(sample["lng"] - info.get("homeport_lng", 0)) < 0.05
+            for info in CARRIER_REGISTRY.values()
+        )
+
+        if at_port:
+            for idx in indices:
+                carrier = result[idx]
+                hull = None
+                for h, info in CARRIER_REGISTRY.items():
+                    if info["name"] == carrier["name"]:
+                        hull = h
+                        break
+                if hull:
+                    info = CARRIER_REGISTRY[hull]
+                    carrier["lat"] = info["homeport_lat"]
+                    carrier["lng"] = info["homeport_lng"]
+        else:
+            spacing = 0.08
+            start_offset = -(n - 1) * spacing / 2
+            for j, idx in enumerate(indices):
+                result[idx]["lng"] += start_offset + j * spacing
+
+    return result
 
 
 def get_carrier_positions() -> List[dict]:
-    """Return current carrier positions for the data pipeline."""
+    """Return current carrier positions for the data pipeline.
+
+    Each entry has the full provenance + freshness fields; the UI can
+    decide how to render them. Carriers are never hidden — only
+    labeled.
+    """
+    now = datetime.now(timezone.utc)
     with _positions_lock:
-        result = []
+        result: List[dict] = []
-        for hull, pos in _carrier_positions.items():
+        for hull, entry in _carrier_positions.items():
-            info = CARRIER_REGISTRY.get(hull, {})
+            enriched = _enrich_for_rendering(hull, entry, now=now)
-            result.append({
+            result.append(
-                "name": pos.get("name", info.get("name", hull)),
+                {
-                "type": "carrier",
+                    "name": enriched["name"],
-                "lat": pos["lat"],
+                    "type": "carrier",
-                "lng": pos["lng"],
+                    "lat": enriched["lat"],
-                "heading": pos.get("heading", 0),
+                    "lng": enriched["lng"],
-                "sog": 0,
+                    "heading": None,  # OSINT cannot determine true heading.
-                "cog": 0,
+                    "sog": 0,
-                "country": "United States",
+                    "cog": 0,
-                "desc": pos.get("desc", ""),
+                    "country": "United States",
-                "wiki": pos.get("wiki", info.get("wiki", "")),
+                    "desc": enriched["desc"],
-                "estimated": True,
+                    "wiki": enriched["wiki"],
-                "source": pos.get("source", "OSINT estimated position"),
+                    "estimated": True,
-                "last_osint_update": pos.get("updated", "")
+                    "source": enriched["source"],
-            })
+                    "source_url": enriched["source_url"],
-        return result
+                    "last_osint_update": enriched["last_osint_update"],
+                    # New fields (additive — existing UI continues to work):
+                    "position_source_at": enriched["position_source_at"],
+                    "position_confidence": enriched["position_confidence"],
+                    "is_fallback": enriched["is_fallback"],
+                }
+            )
+        return _deconflict_positions(result)
 
 
 # -----------------------------------------------------------------
-# Scheduler: runs at startup, then at 00:00 and 12:00 UTC daily
+# Scheduler: runs at startup, then at 00:00 and 12:00 UTC daily.
 # -----------------------------------------------------------------
 _scheduler_thread: Optional[threading.Thread] = None
 _scheduler_stop = threading.Event()
@@ -403,7 +781,6 @@ _scheduler_stop = threading.Event()
 
 def _scheduler_loop():
     """Background thread that triggers updates at 00:00 and 12:00 UTC."""
-    # Initial update on startup
     try:
         update_carrier_positions()
     except Exception as e:
@@ -411,7 +788,6 @@ def _scheduler_loop():
 
     while not _scheduler_stop.is_set():
         now = datetime.now(timezone.utc)
-        # Next target: 00:00 or 12:00 UTC, whichever is sooner
         hour = now.hour
         if hour < 12:
             next_hour = 12
@@ -420,15 +796,17 @@ def _scheduler_loop():
 
         next_run = now.replace(hour=next_hour % 24, minute=0, second=0, microsecond=0)
         if next_hour == 24:
-            from datetime import timedelta
             next_run = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
 
         wait_seconds = (next_run - now).total_seconds()
-        logger.info(f"Carrier tracker: next update at {next_run.isoformat()} ({wait_seconds/3600:.1f}h)")
+        logger.info(
+            "Carrier tracker: next update at %s (%.1fh)",
+            next_run.isoformat(),
+            wait_seconds / 3600,
+        )
 
-        # Wait until next scheduled time, or until stop event
         if _scheduler_stop.wait(timeout=wait_seconds):
-            break  # Stop event was set
+            break
 
         try:
             update_carrier_positions()
@@ -442,7 +820,9 @@ def start_carrier_tracker():
     if _scheduler_thread and _scheduler_thread.is_alive():
         return
     _scheduler_stop.clear()
-    _scheduler_thread = threading.Thread(target=_scheduler_loop, daemon=True, name="carrier-tracker")
+    _scheduler_thread = threading.Thread(
+        target=_scheduler_loop, daemon=True, name="carrier-tracker"
+    )
     _scheduler_thread.start()
     logger.info("Carrier tracker started")
 

backend/services/config.py

@@ -0,0 +1,426 @@
+"""Typed configuration via pydantic-settings."""
+
+from functools import lru_cache
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    # Admin/security
+    ADMIN_KEY: str = ""
+    ALLOW_INSECURE_ADMIN: bool = False
+    PUBLIC_API_KEY: str = ""
+
+    # OpenClaw agent connectivity
+    OPENCLAW_HMAC_SECRET: str = ""        # HMAC shared secret for direct mode (auto-generated if empty)
+    OPENCLAW_ACCESS_TIER: str = "restricted"  # "full" or "restricted"
+
+    # Data sources
+    AIS_API_KEY: str = ""
+    OPENSKY_CLIENT_ID: str = ""
+    OPENSKY_CLIENT_SECRET: str = ""
+    LTA_ACCOUNT_KEY: str = ""
+
+    # Runtime
+    CORS_ORIGINS: str = ""
+    FETCH_SLOW_THRESHOLD_S: float = 5.0
+    MESH_STRICT_SIGNATURES: bool = True
+    MESH_DEBUG_MODE: bool = False
+    MESH_MQTT_EXTRA_ROOTS: str = ""
+    MESH_MQTT_EXTRA_TOPICS: str = ""
+    MESH_MQTT_INCLUDE_DEFAULT_ROOTS: bool = True
+    MESH_RNS_ENABLED: bool = False
+    MESH_ARTI_ENABLED: bool = False
+    MESH_ARTI_SOCKS_PORT: int = 9050
+    MESH_RELAY_PEERS: str = ""
+    MESH_PUBLIC_PEER_URL: str = ""
+    # Bootstrap seeds are discovery hints, not authoritative network roots.
+    # Nodes promote healthy discovered peers from the store/manifest over time.
+    MESH_BOOTSTRAP_SEED_PEERS: str = "http://gqpbunqbgtkcqilvclm3xrkt3zowjyl3s62kkktvojgvxzizamvbrqid.onion:8000"
+    # Legacy name kept for older compose/.env files.
+    MESH_DEFAULT_SYNC_PEERS: str = ""
+    # Infonet/Wormhole must fail closed to private transports by default.
+    # Set true only for local relay development or explicitly public testnets.
+    MESH_INFONET_ALLOW_CLEARNET_SYNC: bool = False
+    MESH_BOOTSTRAP_DISABLED: bool = False
+    MESH_BOOTSTRAP_MANIFEST_PATH: str = "data/bootstrap_peers.json"
+    MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY: str = ""
+    MESH_NODE_MODE: str = "participant"
+    MESH_SYNC_INTERVAL_S: int = 300
+    MESH_SYNC_FAILURE_BACKOFF_S: int = 60
+    MESH_SYNC_TIMEOUT_S: int = 5
+    MESH_SYNC_MAX_PEERS_PER_CYCLE: int = 3
+    MESH_RELAY_PUSH_TIMEOUT_S: int = 10
+    MESH_RELAY_MAX_FAILURES: int = 3
+    MESH_RELAY_FAILURE_COOLDOWN_S: int = 120
+    MESH_BOOTSTRAP_SEED_FAILURE_COOLDOWN_S: int = 15
+    MESH_PEER_PUSH_SECRET: str = ""
+    # Issue #256 (tg12): optional per-peer HMAC secret map. Comma-separated
+    # `url=secret` pairs. When a peer URL appears here, only that per-peer
+    # secret is accepted for it — the global MESH_PEER_PUSH_SECRET above is
+    # ignored for that specific URL. Single-peer installs and unmigrated
+    # multi-peer installs leave this empty and behavior is unchanged.
+    MESH_PEER_SECRETS: str = ""
+    MESH_RNS_APP_NAME: str = "shadowbroker"
+    MESH_RNS_ASPECT: str = "infonet"
+    MESH_RNS_IDENTITY_PATH: str = ""
+    MESH_RNS_PEERS: str = ""
+    MESH_RNS_DANDELION_HOPS: int = 2
+    MESH_RNS_DANDELION_DELAY_MS: int = 400
+    MESH_RNS_CHURN_INTERVAL_S: int = 300
+    MESH_RNS_MAX_PEERS: int = 32
+    MESH_RNS_MAX_PAYLOAD: int = 8192
+    MESH_RNS_PEER_BUCKET_PREFIX: int = 4
+    MESH_RNS_MAX_PEERS_PER_BUCKET: int = 4
+    MESH_RNS_PEER_FAIL_THRESHOLD: int = 3
+    MESH_RNS_PEER_COOLDOWN_S: int = 300
+    MESH_RNS_SHARD_ENABLED: bool = False
+    MESH_RNS_SHARD_DATA_SHARDS: int = 3
+    MESH_RNS_SHARD_PARITY_SHARDS: int = 1
+    MESH_RNS_SHARD_TTL_S: int = 30
+    MESH_RNS_FEC_CODEC: str = "xor"  # xor | rs
+    MESH_RNS_BATCH_MS: int = 200
+    # Keep a low background cadence on private RNS links so quiet nodes are less
+    # trivially fingerprintable by silence alone. Set to 0 to disable explicitly.
+    MESH_RNS_COVER_INTERVAL_S: int = 30
+    MESH_RNS_COVER_SIZE: int = 512
+    MESH_DM_MAILBOX_TTL_S: int = 900
+    MESH_RNS_IBF_WINDOW: int = 256
+    MESH_RNS_IBF_TABLE_SIZE: int = 64
+    MESH_RNS_IBF_MINHASH_SIZE: int = 16
+    MESH_RNS_IBF_MINHASH_THRESHOLD: float = 0.25
+    MESH_RNS_IBF_WINDOW_JITTER: int = 32
+    MESH_RNS_IBF_INTERVAL_S: int = 120
+    MESH_RNS_IBF_SYNC_PEERS: int = 3
+    MESH_RNS_IBF_QUORUM_TIMEOUT_S: int = 6
+    MESH_RNS_IBF_MAX_REQUEST_IDS: int = 64
+    MESH_RNS_IBF_MAX_EVENTS: int = 64
+    MESH_RNS_SESSION_ROTATE_S: int = 1800
+    MESH_RNS_IBF_FAIL_THRESHOLD: int = 3
+    MESH_RNS_IBF_COOLDOWN_S: int = 120
+    MESH_VERIFY_INTERVAL_S: int = 600
+    # MESH_VERIFY_SIGNATURES is intentionally removed — the audit loop in main.py
+    # always calls validate_chain_incremental(verify_signatures=True). Any value
+    # set in the environment is ignored.
+    MESH_DM_SECURE_MODE: bool = True
+    MESH_DM_TOKEN_PEPPER: str = ""
+    MESH_ALLOW_LEGACY_DM1_UNTIL: str = ""
+    MESH_ALLOW_LEGACY_DM_GET_UNTIL: str = ""
+    MESH_ALLOW_LEGACY_DM_SIGNATURE_COMPAT_UNTIL: str = ""
+    MESH_DM_PERSIST_SPOOL: bool = False
+    MESH_DM_RELAY_FILE_PATH: str = ""
+    MESH_DM_RELAY_AUTO_RELOAD: bool = False
+    MESH_DM_REQUIRE_SENDER_SEAL_SHARED: bool = True
+    MESH_DM_NONCE_TTL_S: int = 300
+    MESH_DM_NONCE_CACHE_MAX: int = 4096
+    MESH_DM_NONCE_PER_AGENT_MAX: int = 256
+    MESH_DM_REQUEST_MAX_AGE_S: int = 300
+    MESH_DM_REQUEST_MAILBOX_LIMIT: int = 12
+    MESH_DM_SHARED_MAILBOX_LIMIT: int = 48
+    MESH_DM_SELF_MAILBOX_LIMIT: int = 12
+    # Anti-spam: cap on distinct UNACKED messages a single sender can have
+    # parked in a single recipient's mailbox at any one time. Once the
+    # recipient pulls (acks) a message, the sender's quota for that pair
+    # frees up. Default 2 — a sender who wants to deliver more must wait
+    # for the recipient to actually read the prior messages.
+    #
+    # This cap is enforced TWICE: once on the local deposit path (the
+    # sender's own node refuses to spool the 3rd message) AND once on
+    # the replication-acceptance path (honest peer relays refuse to
+    # accept inbound replicas that would put them over the cap). The
+    # double enforcement makes the rule a NETWORK rule — patching out
+    # the local check on a hostile sender's relay doesn't let extras
+    # propagate, because every honest peer enforces the same cap on
+    # inbound replication.
+    MESH_DM_PENDING_PER_SENDER_LIMIT: int = 2
+    MESH_BLOCK_LEGACY_AGENT_ID_LOOKUP: bool = True
+    MESH_ALLOW_COMPAT_DM_INVITE_IMPORT: bool = False
+    MESH_ALLOW_COMPAT_DM_INVITE_IMPORT_UNTIL: str = ""
+    MESH_ALLOW_LEGACY_NODE_ID_COMPAT_UNTIL: str = ""
+    # Rotate voter-blinding salts on a rolling cadence so new reputation
+    # events do not reuse one forever-stable blinded identity.
+    MESH_VOTER_BLIND_SALT_ROTATE_DAYS: int = 30
+    # Keep historical salts long enough to cover live vote records, so
+    # duplicate-vote detection and wallet-cost accounting survive rotation.
+    MESH_VOTER_BLIND_SALT_GRACE_DAYS: int = 30
+    MESH_DM_MAX_MSG_BYTES: int = 8192
+    MESH_DM_ALLOW_SENDER_SEAL: bool = False
+    # TTL for DH key and prekey bundle registrations — stale entries are pruned.
+    MESH_DM_KEY_TTL_DAYS: int = 30
+    # TTL for invite-scoped prekey lookup aliases; shorter windows reduce
+    # long-lived relay linkage between opaque lookup handles and agent IDs.
+    MESH_DM_PREKEY_LOOKUP_ALIAS_TTL_DAYS: int = 14
+    # TTL for relay witness history; keep continuity metadata bounded instead
+    # of relying on a hidden hardcoded retention window.
+    MESH_DM_WITNESS_TTL_DAYS: int = 14
+    # TTL for mailbox binding metadata — shorter = smaller metadata footprint on disk.
+    MESH_DM_BINDING_TTL_DAYS: int = 3
+    # When False, mailbox bindings are memory-only (agents re-register on restart).
+    # Enable explicitly only if restart continuity is worth persisting DM graph metadata.
+    MESH_DM_METADATA_PERSIST: bool = False
+    # Second explicit opt-in for at-rest DM metadata persistence. This keeps a
+    # single boolean flip from silently writing mailbox graph metadata to disk.
+    MESH_DM_METADATA_PERSIST_ACKNOWLEDGE: bool = False
+    # Optional import path for externally managed root witness material packages.
+    # Relative paths resolve from the backend directory.
+    MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH: str = ""
+    # Optional URI for externally managed root witness material packages.
+    # Supports file:// and http(s):// sources; when set it overrides the local path.
+    MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_URI: str = ""
+    # Maximum acceptable age for externally sourced root witness packages.
+    # Strong DM trust fails closed when the imported package exported_at is older than this.
+    MESH_DM_ROOT_EXTERNAL_WITNESS_MAX_AGE_S: int = 3600
+    # Warning threshold for externally sourced root witness packages.
+    # When current external witness material reaches this age, operator health degrades to warning
+    # before the strong path eventually fails closed at MAX_AGE.
+    MESH_DM_ROOT_EXTERNAL_WITNESS_WARN_AGE_S: int = 2700
+    # Optional export path for the append-only stable-root transparency ledger.
+    # Relative paths resolve from the backend directory.
+    MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH: str = ""
+    # Optional URI used to read back and verify published transparency ledgers.
+    # Supports file:// and http(s):// sources.
+    MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI: str = ""
+    # Maximum acceptable age for externally read transparency ledgers.
+    # Strong DM trust fails closed when exported_at is older than this.
+    MESH_DM_ROOT_TRANSPARENCY_LEDGER_MAX_AGE_S: int = 3600
+    # Warning threshold for externally read transparency ledgers.
+    # When current external transparency readback reaches this age, operator health degrades to warning
+    # before the strong path eventually fails closed at MAX_AGE.
+    MESH_DM_ROOT_TRANSPARENCY_LEDGER_WARN_AGE_S: int = 2700
+    MESH_SCOPED_TOKENS: str = ""
+    # Deprecated legacy env vars kept for backward config compatibility only.
+    # Ordinary shipped gate flows keep MLS decrypt local; backend decrypt is
+    # reserved for explicit recovery reads.
+    MESH_GATE_BACKEND_DECRYPT_COMPAT: bool = False
+    MESH_GATE_BACKEND_DECRYPT_COMPAT_ACKNOWLEDGE: bool = False
+    MESH_BACKEND_GATE_DECRYPT_COMPAT: bool = False
+    # Deprecated legacy env vars kept for backward config compatibility only.
+    # Ordinary shipped gate flows keep compose/post local and submit encrypted
+    # payloads to the backend for sign/post only.
+    MESH_GATE_BACKEND_PLAINTEXT_COMPAT: bool = False
+    MESH_GATE_BACKEND_PLAINTEXT_COMPAT_ACKNOWLEDGE: bool = False
+    MESH_BACKEND_GATE_PLAINTEXT_COMPAT: bool = False
+    # Runtime gate for recovery envelopes. When off, per-gate
+    # envelope_recovery / envelope_always policies fail closed to
+    # envelope_disabled. Default True so the Reddit-like durable history
+    # model works out of the box: any member with the gate_secret can
+    # decrypt every envelope encrypted from the moment they had that key.
+    # Set MESH_GATE_RECOVERY_ENVELOPE_ENABLE=false to revert to MLS-only
+    # forward-secret behavior (your own history becomes unreadable after
+    # the sending ratchet advances).
+    MESH_GATE_RECOVERY_ENVELOPE_ENABLE: bool = True
+    MESH_GATE_RECOVERY_ENVELOPE_ENABLE_ACKNOWLEDGE: bool = True
+    # Durable gate plaintext retention is disabled by default. Enable only
+    # when the operator explicitly accepts the at-rest privacy tradeoff.
+    MESH_GATE_PLAINTEXT_PERSIST: bool = False
+    MESH_GATE_PLAINTEXT_PERSIST_ACKNOWLEDGE: bool = False
+    MESH_GATE_SESSION_ROTATE_MSGS: int = 50
+    MESH_GATE_SESSION_ROTATE_S: int = 3600
+    MESH_GATE_LEGACY_ENVELOPE_FALLBACK_MAX_DAYS: int = 30
+    # Add a randomized grace window before anonymous gate-session auto-rotation
+    # so threshold-triggered identity swaps are less trivially correlated.
+    MESH_GATE_SESSION_ROTATE_JITTER_S: int = 180
+    # Gate persona (named identity) rotation thresholds.  Rotating the signing
+    # key limits the linkability window.  Zero = disabled.
+    MESH_GATE_PERSONA_ROTATE_MSGS: int = 200
+    MESH_GATE_PERSONA_ROTATE_S: int = 604800  # 7 days
+    MESH_GATE_PERSONA_ROTATE_JITTER_S: int = 600
+    # Feature-flagged session stream for multiplexed gate room updates.
+    # Disabled by default so rollout stays explicit while stream-first rooms bake.
+    MESH_GATE_SESSION_STREAM_ENABLED: bool = False
+    MESH_GATE_SESSION_STREAM_HEARTBEAT_S: int = 20
+    MESH_GATE_SESSION_STREAM_BATCH_MS: int = 1500
+    MESH_GATE_SESSION_STREAM_MAX_GATES: int = 16
+    # Private gate APIs expose a backward-jittered timestamp view so observers
+    # cannot trivially align exact send times from response metadata alone.
+    MESH_GATE_TIMESTAMP_JITTER_S: int = 60
+    # Ban/kick gate-secret rotation is on by default (hardening Rec #10): the
+    # invariant has baked and a ban that does not rotate is effectively a
+    # display-only removal. Set MESH_GATE_BAN_KICK_ROTATION_ENABLE=false to
+    # revert to observe-only during incident triage.
+    MESH_GATE_BAN_KICK_ROTATION_ENABLE: bool = True
+    MESH_BLOCK_LEGACY_NODE_ID_COMPAT: bool = True
+    MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK: bool = False
+    MESH_ACK_RAW_FALLBACK_AT_OWN_RISK: bool = False
+    MESH_SECURE_STORAGE_SECRET: str = ""
+    MESH_SECURE_STORAGE_SECRET_FILE: str = ""
+    MESH_PRIVATE_LOG_TTL_S: int = 900
+    # Sprint 1 rollout: restored DM boot probes stay disabled by default until
+    # the architect reviews false positives from the observe-only path.
+    MESH_DM_RESTORED_SESSION_BOOT_PROBE_ENABLE: bool = False
+    # Queued DM release requires explicit per-item approval before any weaker
+    # relay fallback. Silent fallback is not a safe private-mode default.
+    MESH_PRIVATE_RELEASE_APPROVAL_ENABLE: bool = True
+    # Expiry for user-approved scoped private relay fallback policy. The policy
+    # is still bounded by hidden-transport checks before it can auto-release.
+    MESH_PRIVATE_RELAY_POLICY_TTL_S: int = 3600
+    # Background privacy prewarm prepares keys/aliases/transport readiness
+    # before send-time. Anonymous mode uses a cadence gate so user clicks do
+    # not directly create hidden-transport activity.
+    MESH_PRIVACY_PREWARM_ENABLE: bool = True
+    MESH_PRIVACY_PREWARM_INTERVAL_S: int = 300
+    MESH_PRIVACY_PREWARM_ANON_CADENCE_S: int = 300
+    # Sprint 4 rollout: authenticated RNS cover markers remain disabled until
+    # the observer-equivalence and receive-path DoS tests are green.
+    MESH_RNS_COVER_AUTH_MARKER_ENABLE: bool = False
+    # Signed-write revocation lookups use a short local TTL; stale entries force
+    # a local rebuild before honor. Offline/local-refresh failures remain
+    # observe-only until the later enforcement sprint.
+    MESH_SIGNED_REVOCATION_CACHE_TTL_S: int = 300
+    MESH_SIGNED_REVOCATION_CACHE_ENFORCE: bool = True
+    MESH_SIGNED_WRITE_CONTEXT_REQUIRED: bool = True
+    # Sprint 5 rollout: when enabled, root witness finality requires
+    # independent quorum for threshold>1 witnessed roots before they count as
+    # verified first-contact provenance.
+    WORMHOLE_ROOT_WITNESS_FINALITY_ENFORCE: bool = False
+    # Optional JSON artifact generated by CI/release workflow for the Sprint 8
+    # release gate. Relative paths resolve from the backend directory.
+    # dev = permissive local/dev behavior; testnet-private = strict private
+    # defaults; release-candidate = no compatibility/debug escape hatches.
+    MESH_RELEASE_PROFILE: str = "dev"
+    MESH_RELEASE_ATTESTATION_PATH: str = ""
+    # Operator release attestation for the Sprint 8 release gate. This does
+    # not change runtime behavior; it only records that the DM relay security
+    # suite was run and passed for the release candidate.
+    MESH_RELEASE_DM_RELAY_SECURITY_SUITE_GREEN: bool = False
+    PRIVACY_CORE_MIN_VERSION: str = "0.1.0"
+    PRIVACY_CORE_ALLOWED_SHA256: str = ""
+    PRIVACY_CORE_DEV_OVERRIDE: bool = False
+    # Sprint 4 rollout: fail fast when the loaded privacy-core artifact is
+    # missing required FFI symbols expected by the current Python bridge.
+    PRIVACY_CORE_EXPORT_SET_AUDIT_ENABLE: bool = True
+    # Clearnet fallback policy for private-tier messages.
+    # "block" (default) = refuse to send private messages over clearnet.
+    # "allow" = fall back to clearnet when Tor/RNS is unavailable (weaker privacy).
+    MESH_PRIVATE_CLEARNET_FALLBACK: str = "block"
+    # Second explicit opt-in for private-tier clearnet fallback. Without this
+    # acknowledgement, "allow" remains requested but not effective.
+    MESH_PRIVATE_CLEARNET_FALLBACK_ACKNOWLEDGE: bool = False
+    # Meshtastic MQTT bridge — disabled by default to avoid hammering the
+    # public broker.  Users opt in explicitly.
+    MESH_MQTT_ENABLED: bool = False
+    # Meshtastic MQTT broker credentials (defaults match public firmware).
+    MESH_MQTT_BROKER: str = "mqtt.meshtastic.org"
+    MESH_MQTT_PORT: int = 1883
+    MESH_MQTT_USER: str = "meshdev"
+    MESH_MQTT_PASS: str = "large4cats"
+    # Hex-encoded PSK — empty string means use the default LongFast key.
+    # Must decode to exactly 16 or 32 bytes when set.
+    MESH_MQTT_PSK: str = ""
+    # Optional operator-provided Meshtastic node ID (e.g. "!abcd1234") included
+    # in the User-Agent when fetching from meshtastic.liamcottle.net so the
+    # service operator can identify per-install traffic instead of a generic
+    # "ShadowBroker" aggregate.
+    MESHTASTIC_OPERATOR_CALLSIGN: str = ""
+    # Per-install operator handle used in the User-Agent for EVERY third-party
+    # API the backend calls (Wikipedia, Wikidata, Nominatim, GDELT, OpenMHz,
+    # Broadcastify, weather.gov, NUFORC, etc.). The default is empty, in which
+    # case backend/services/network_utils.py auto-generates a stable
+    # pseudonymous handle like "operator-7f3a92" on first use and caches it.
+    # Operators who want to identify themselves with a real handle can set
+    # this; operators who want to stay pseudonymous can leave it empty.
+    #
+    # The handle is sent ONLY to public third-party APIs. It is NEVER mixed
+    # into mesh / Wormhole / Infonet identity (those have their own crypto
+    # identity layer; conflating the two would leak public attribution into
+    # private mesh state).
+    OPERATOR_HANDLE: str = ""
+
+    # SAR (Synthetic Aperture Radar) data layer
+    # Mode A — free catalog metadata, no account, default-on
+    MESH_SAR_CATALOG_ENABLED: bool = True
+    # Mode B — free pre-processed anomalies (OPERA / EGMS / GFM / EMS / UNOSAT)
+    # Two-step opt-in: must be "allow" AND _ACKNOWLEDGE must be true
+    MESH_SAR_PRODUCTS_FETCH: str = "block"
+    MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE: bool = False
+    # NASA Earthdata Login (free) — required for OPERA products
+    MESH_SAR_EARTHDATA_USER: str = ""
+    MESH_SAR_EARTHDATA_TOKEN: str = ""
+    # Copernicus Data Space (free) — required for EGMS / EMS products
+    MESH_SAR_COPERNICUS_USER: str = ""
+    MESH_SAR_COPERNICUS_TOKEN: str = ""
+    # Whether OpenClaw agents may read/act on the SAR layer
+    MESH_SAR_OPENCLAW_ENABLED: bool = True
+    # Require private-tier transport before signing/broadcasting SAR anomalies
+    MESH_SAR_REQUIRE_PRIVATE_TIER: bool = True
+
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore")
+
+
+@lru_cache
+def get_settings() -> Settings:
+    try:
+        from services.api_settings import load_persisted_api_keys_into_environ
+        load_persisted_api_keys_into_environ()
+    except Exception:
+        pass
+    return Settings()
+
+
+def private_clearnet_fallback_requested(settings: Settings | None = None) -> str:
+    snapshot = settings or get_settings()
+    policy = str(getattr(snapshot, "MESH_PRIVATE_CLEARNET_FALLBACK", "block") or "block").strip().lower()
+    return "allow" if policy == "allow" else "block"
+
+
+def private_clearnet_fallback_effective(settings: Settings | None = None) -> str:
+    snapshot = settings or get_settings()
+    requested = private_clearnet_fallback_requested(snapshot)
+    acknowledged = bool(getattr(snapshot, "MESH_PRIVATE_CLEARNET_FALLBACK_ACKNOWLEDGE", False))
+    if requested == "allow" and acknowledged:
+        return "allow"
+    return "block"
+
+
+def backend_gate_decrypt_compat_effective(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(
+        getattr(snapshot, "MESH_BACKEND_GATE_DECRYPT_COMPAT", False)
+        or getattr(snapshot, "MESH_GATE_BACKEND_DECRYPT_COMPAT", False)
+    )
+
+
+def backend_gate_plaintext_compat_effective(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(
+        getattr(snapshot, "MESH_BACKEND_GATE_PLAINTEXT_COMPAT", False)
+        or getattr(snapshot, "MESH_GATE_BACKEND_PLAINTEXT_COMPAT", False)
+    )
+
+
+def gate_recovery_envelope_effective(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    requested = bool(getattr(snapshot, "MESH_GATE_RECOVERY_ENVELOPE_ENABLE", False))
+    acknowledged = bool(getattr(snapshot, "MESH_GATE_RECOVERY_ENVELOPE_ENABLE_ACKNOWLEDGE", False))
+    return requested and acknowledged
+
+
+def gate_plaintext_persist_effective(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    requested = bool(getattr(snapshot, "MESH_GATE_PLAINTEXT_PERSIST", False))
+    acknowledged = bool(getattr(snapshot, "MESH_GATE_PLAINTEXT_PERSIST_ACKNOWLEDGE", False))
+    return requested and acknowledged
+
+
+def gate_ban_kick_rotation_enabled(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(getattr(snapshot, "MESH_GATE_BAN_KICK_ROTATION_ENABLE", False))
+
+
+def dm_restored_session_boot_probe_enabled(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(getattr(snapshot, "MESH_DM_RESTORED_SESSION_BOOT_PROBE_ENABLE", False))
+
+
+def signed_revocation_cache_ttl_s(settings: Settings | None = None) -> int:
+    snapshot = settings or get_settings()
+    return max(0, int(getattr(snapshot, "MESH_SIGNED_REVOCATION_CACHE_TTL_S", 300) or 0))
+
+
+def signed_revocation_cache_enforce(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(getattr(snapshot, "MESH_SIGNED_REVOCATION_CACHE_ENFORCE", False))
+
+
+def wormhole_root_witness_finality_enforce(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(getattr(snapshot, "WORMHOLE_ROOT_WITNESS_FINALITY_ENFORCE", False))

backend/services/constants.py

@@ -0,0 +1,39 @@
+# ─── ShadowBroker Backend Constants ──────────────────────────────────────────
+# Centralized magic numbers. Import from here instead of hardcoding.
+
+# ─── Flight Trails ──────────────────────────────────────────────────────────
+FLIGHT_TRAIL_MAX_TRACKED = 2000  # Max concurrent tracked trails before LRU eviction
+FLIGHT_TRAIL_POINTS_PER_FLIGHT = 200  # Max trail points kept per aircraft
+TRACKED_TRAIL_TTL_S = 1800  # 30 min - trail TTL for tracked flights
+DEFAULT_TRAIL_TTL_S = 300  # 5 min - trail TTL for non-tracked flights
+
+# ─── Detection Thresholds ──────────────────────────────────────────────────
+HOLD_PATTERN_DEGREES = 300  # Total heading change to flag holding pattern
+GPS_JAMMING_NACP_THRESHOLD = 8  # NACp below this = degraded GPS signal
+GPS_JAMMING_GRID_SIZE = 1.0  # 1 degree grid for aggregation
+# Tuned 2026-05: previously 0.30 / 5 aircraft which — combined with the
+# -1 noise cushion in the detector AND the pre-fix nac_p==0 filter that
+# discarded jamming victims — meant the layer almost never lit up.
+# Lowering the bar so genuine jamming zones with sparser ADS-B coverage
+# clear (eastern Med, Russia/Ukraine border, Iran/Iraq).
+GPS_JAMMING_MIN_RATIO = 0.20  # 20% degraded aircraft to flag zone
+GPS_JAMMING_MIN_AIRCRAFT = 3  # Min aircraft in grid cell for statistical significance
+
+# ─── Network & Circuit Breaker ──────────────────────────────────────────────
+CIRCUIT_BREAKER_TTL_S = 120  # Skip domain for 2 min after total failure
+DOMAIN_FAIL_TTL_S = 300  # Skip requests.get for 5 min, go straight to curl
+CONNECT_TIMEOUT_S = 3  # Short connect timeout for fast firewall-block detection
+
+# ─── Data Fetcher Intervals ────────────────────────────────────────────────
+FAST_FETCH_INTERVAL_S = 60  # Flights, ships, satellites, military
+SLOW_FETCH_INTERVAL_MIN = 30  # News, markets, space weather
+CCTV_FETCH_INTERVAL_MIN = 1  # CCTV camera pipeline
+LIVEUAMAP_FETCH_INTERVAL_HR = 12  # LiveUAMap scraper
+
+# ─── External API ──────────────────────────────────────────────────────────
+OPENSKY_RATE_LIMIT_S = 300  # Only re-fetch OpenSky every 5 minutes
+OPENSKY_REQUEST_TIMEOUT_S = 15  # Timeout for OpenSky API calls
+ROUTE_FETCH_TIMEOUT_S = 15  # Timeout for adsb.lol route lookups
+
+# ─── Internet Outage Detection ─────────────────────────────────────────────
+INTERNET_OUTAGE_MIN_SEVERITY = 0.10  # 10% drop minimum to show

backend/services/correlation_engine.py

@@ -0,0 +1,783 @@
+"""
+Emergent Intelligence — Cross-layer correlation engine.
+
+Scans co-located events across multiple data layers and emits composite
+alerts that no single source could generate alone.
+
+Correlation types:
+  - RF Anomaly:          GPS jamming + internet outage (both required)
+  - Military Buildup:    Military flights + naval vessels + GDELT conflict events
+  - Infrastructure Cascade: Internet outage + KiwiSDR offline in same zone
+  - Possible Contradiction: Official denial/statement + infrastructure disruption
+                            in same region — hypothesis generator, NOT verdict
+"""
+
+import logging
+import math
+import re
+from collections import defaultdict
+
+logger = logging.getLogger(__name__)
+
+# Grid cell size in degrees — 1° ≈ 111 km at equator.
+# Tighter than the previous 2° to reduce false co-locations.
+_CELL_SIZE = 1
+
+# Quality gates for RF anomaly correlation — only high-confidence inputs.
+# GPS jamming + internet outage overlap in a 111km cell is easily a coincidence
+# (IODA returns ~100 regional outages; GPS NACp dips are common in busy airspace).
+# Only fire when the evidence is strong enough to indicate deliberate RF interference.
+_RF_CORR_MIN_GPS_RATIO = 0.60   # Need strong jamming signal, not marginal NACp dips
+_RF_CORR_MIN_OUTAGE_PCT = 40    # Need a serious outage, not routine BGP fluctuation
+_RF_CORR_MIN_INDICATORS = 3     # Require 3+ corroborating signals (not just GPS+outage)
+
+
+def _cell_key(lat: float, lng: float) -> str:
+    """Convert lat/lng to a grid cell key."""
+    clat = int(lat // _CELL_SIZE) * _CELL_SIZE
+    clng = int(lng // _CELL_SIZE) * _CELL_SIZE
+    return f"{clat},{clng}"
+
+
+def _cell_center(key: str) -> tuple[float, float]:
+    """Get center lat/lng from a cell key."""
+    parts = key.split(",")
+    return float(parts[0]) + _CELL_SIZE / 2, float(parts[1]) + _CELL_SIZE / 2
+
+
+def _severity(indicator_count: int) -> str:
+    if indicator_count >= 3:
+        return "high"
+    if indicator_count >= 2:
+        return "medium"
+    return "low"
+
+
+def _severity_score(sev: str) -> float:
+    return {"high": 90, "medium": 60, "low": 30}.get(sev, 0)
+
+
+def _outage_pct(outage: dict) -> float:
+    """Extract outage severity percentage from an outage dict."""
+    return float(outage.get("severity", 0) or outage.get("severity_pct", 0) or 0)
+
+
+# ---------------------------------------------------------------------------
+# RF Anomaly: GPS jamming + internet outage (both must be present)
+# ---------------------------------------------------------------------------
+
+
+def _detect_rf_anomalies(data: dict) -> list[dict]:
+    gps_jamming = data.get("gps_jamming") or []
+    internet_outages = data.get("internet_outages") or []
+
+    if not gps_jamming:
+        return []  # No GPS jamming → no RF anomalies possible
+
+    # Build grid of indicators
+    cells: dict[str, dict] = defaultdict(lambda: {
+        "gps_jam": False, "gps_ratio": 0.0,
+        "outage": False, "outage_pct": 0.0,
+    })
+
+    for z in gps_jamming:
+        lat, lng = z.get("lat"), z.get("lng")
+        if lat is None or lng is None:
+            continue
+        ratio = z.get("ratio", 0)
+        if ratio < _RF_CORR_MIN_GPS_RATIO:
+            continue  # Skip marginal jamming zones
+        key = _cell_key(lat, lng)
+        cells[key]["gps_jam"] = True
+        cells[key]["gps_ratio"] = max(cells[key]["gps_ratio"], ratio)
+
+    for o in internet_outages:
+        lat = o.get("lat") or o.get("latitude")
+        lng = o.get("lng") or o.get("lon") or o.get("longitude")
+        if lat is None or lng is None:
+            continue
+        pct = _outage_pct(o)
+        if pct < _RF_CORR_MIN_OUTAGE_PCT:
+            continue  # Skip minor outages (ISP maintenance noise)
+        key = _cell_key(float(lat), float(lng))
+        cells[key]["outage"] = True
+        cells[key]["outage_pct"] = max(cells[key]["outage_pct"], pct)
+
+    # PSK Reporter: presence = healthy RF.  Only used as a bonus indicator,
+    # NOT as a standalone trigger (absence is normal in most cells).
+    psk_reporter = data.get("psk_reporter") or []
+    psk_cells: set[str] = set()
+    for s in psk_reporter:
+        lat, lng = s.get("lat"), s.get("lon")
+        if lat is not None and lng is not None:
+            psk_cells.add(_cell_key(lat, lng))
+
+    # When PSK data is unavailable, we can't get a 3rd indicator, so require
+    # an even higher GPS jamming ratio to compensate (real EW shows 75%+).
+    psk_available = len(psk_reporter) > 0
+
+    alerts: list[dict] = []
+    for key, c in cells.items():
+        # GPS jamming is the anchor — required for every RF anomaly alert
+        if not c["gps_jam"]:
+            continue
+        if not c["outage"]:
+            continue  # Both GPS jamming AND outage are always required
+
+        indicators = 2  # GPS jamming + outage
+        drivers: list[str] = [f"GPS jamming {int(c['gps_ratio'] * 100)}%"]
+        pct = c["outage_pct"]
+        drivers.append(f"Internet outage{f' {pct:.0f}%' if pct else ''}")
+
+        # PSK absence confirms RF environment is disrupted
+        if psk_available and key not in psk_cells:
+            indicators += 1
+            drivers.append("No HF digital activity (PSK Reporter)")
+
+        if indicators < _RF_CORR_MIN_INDICATORS:
+            # Without PSK data, only allow through if GPS ratio is extreme
+            # (75%+ indicates deliberate, sustained jamming — not noise)
+            if not psk_available and c["gps_ratio"] >= 0.75 and pct >= 50:
+                pass  # Allow this high-confidence 2-indicator alert through
+            else:
+                continue
+
+        lat, lng = _cell_center(key)
+        sev = _severity(indicators)
+        alerts.append({
+            "lat": lat,
+            "lng": lng,
+            "type": "rf_anomaly",
+            "severity": sev,
+            "score": _severity_score(sev),
+            "drivers": drivers[:3],
+            "cell_size": _CELL_SIZE,
+        })
+
+    return alerts
+
+
+# ---------------------------------------------------------------------------
+# Military Buildup: flights + ships + GDELT conflict
+# ---------------------------------------------------------------------------
+
+
+def _detect_military_buildups(data: dict) -> list[dict]:
+    mil_flights = data.get("military_flights") or []
+    ships = data.get("ships") or []
+    gdelt = data.get("gdelt") or []
+
+    cells: dict[str, dict] = defaultdict(lambda: {
+        "mil_flights": 0, "mil_ships": 0, "gdelt_events": 0,
+    })
+
+    for f in mil_flights:
+        lat = f.get("lat") or f.get("latitude")
+        lng = f.get("lng") or f.get("lon") or f.get("longitude")
+        if lat is None or lng is None:
+            continue
+        try:
+            key = _cell_key(float(lat), float(lng))
+            cells[key]["mil_flights"] += 1
+        except (ValueError, TypeError):
+            continue
+
+    mil_ship_types = {"military_vessel", "military", "warship", "patrol", "destroyer",
+                      "frigate", "corvette", "carrier", "submarine", "cruiser"}
+    for s in ships:
+        stype = (s.get("type") or s.get("ship_type") or "").lower()
+        if not any(mt in stype for mt in mil_ship_types):
+            continue
+        lat = s.get("lat") or s.get("latitude")
+        lng = s.get("lng") or s.get("lon") or s.get("longitude")
+        if lat is None or lng is None:
+            continue
+        try:
+            key = _cell_key(float(lat), float(lng))
+            cells[key]["mil_ships"] += 1
+        except (ValueError, TypeError):
+            continue
+
+    for g in gdelt:
+        lat = g.get("lat") or g.get("latitude") or g.get("actionGeo_Lat")
+        lng = g.get("lng") or g.get("lon") or g.get("longitude") or g.get("actionGeo_Long")
+        if lat is None or lng is None:
+            continue
+        try:
+            key = _cell_key(float(lat), float(lng))
+            cells[key]["gdelt_events"] += 1
+        except (ValueError, TypeError):
+            continue
+
+    alerts: list[dict] = []
+    for key, c in cells.items():
+        mil_total = c["mil_flights"] + c["mil_ships"]
+        has_gdelt = c["gdelt_events"] > 0
+
+        # Need meaningful military presence AND a conflict indicator
+        if mil_total < 3 or not has_gdelt:
+            continue
+
+        drivers: list[str] = []
+        if c["mil_flights"]:
+            drivers.append(f"{c['mil_flights']} military aircraft")
+        if c["mil_ships"]:
+            drivers.append(f"{c['mil_ships']} military vessels")
+        if c["gdelt_events"]:
+            drivers.append(f"{c['gdelt_events']} conflict events")
+
+        if mil_total >= 11:
+            sev = "high"
+        elif mil_total >= 6:
+            sev = "medium"
+        else:
+            sev = "low"
+
+        lat, lng = _cell_center(key)
+        alerts.append({
+            "lat": lat,
+            "lng": lng,
+            "type": "military_buildup",
+            "severity": sev,
+            "score": _severity_score(sev),
+            "drivers": drivers[:3],
+            "cell_size": _CELL_SIZE,
+        })
+
+    return alerts
+
+
+# ---------------------------------------------------------------------------
+# Infrastructure Cascade: outage + KiwiSDR co-location
+#
+# Power plants are removed from this detector — with 35K plants globally,
+# virtually every 2° cell contains one, making every outage a false hit.
+# KiwiSDR receivers (~300 worldwide) are sparse enough to be meaningful:
+# an outage in the same cell as a KiwiSDR indicates real infrastructure
+# disruption affecting radio monitoring capability.
+# ---------------------------------------------------------------------------
+
+
+def _detect_infra_cascades(data: dict) -> list[dict]:
+    internet_outages = data.get("internet_outages") or []
+    kiwisdr = data.get("kiwisdr") or []
+
+    if not kiwisdr:
+        return []
+
+    # Build set of cells with KiwiSDR receivers
+    kiwi_cells: set[str] = set()
+    for k in kiwisdr:
+        lat, lng = k.get("lat"), k.get("lon") or k.get("lng")
+        if lat is not None and lng is not None:
+            try:
+                kiwi_cells.add(_cell_key(float(lat), float(lng)))
+            except (ValueError, TypeError):
+                pass
+
+    if not kiwi_cells:
+        return []
+
+    alerts: list[dict] = []
+    for o in internet_outages:
+        lat = o.get("lat") or o.get("latitude")
+        lng = o.get("lng") or o.get("lon") or o.get("longitude")
+        if lat is None or lng is None:
+            continue
+        try:
+            key = _cell_key(float(lat), float(lng))
+        except (ValueError, TypeError):
+            continue
+
+        if key not in kiwi_cells:
+            continue
+
+        pct = _outage_pct(o)
+        drivers = [f"Internet outage{f' {pct:.0f}%' if pct else ''}",
+                    "KiwiSDR receivers in affected zone"]
+
+        lat_c, lng_c = _cell_center(key)
+        alerts.append({
+            "lat": lat_c,
+            "lng": lng_c,
+            "type": "infra_cascade",
+            "severity": "medium",
+            "score": _severity_score("medium"),
+            "drivers": drivers,
+            "cell_size": _CELL_SIZE,
+        })
+
+    return alerts
+
+
+# ---------------------------------------------------------------------------
+# Possible Contradiction: official denial/statement + infra disruption
+#
+# This is a HYPOTHESIS GENERATOR, not a verdict engine.  It says "LOOK HERE"
+# when an official statement (denial, clarification, refusal) co-locates with
+# infrastructure disruption (internet outage, sigint change).  The human or
+# higher-order reasoning decides what actually happened.
+#
+# Context ratings:
+#   STRONG   — denial + outage + prediction market movement in same region
+#   MODERATE — denial + outage (no market signal)
+#   WEAK     — denial + minor outage or distant co-location
+#   DETECTION_GAP — denial found but NO telemetry to verify (equally valuable)
+# ---------------------------------------------------------------------------
+
+# Denial / official-statement patterns in headlines and URL slugs
+_DENIAL_PATTERNS = [
+    re.compile(p, re.IGNORECASE) for p in [
+        r"\bden(?:y|ies|ied|ial)\b",
+        r"\brefut(?:e[ds]?|ing)\b",
+        r"\breject(?:s|ed|ing)?\b",
+        r"\bclarif(?:y|ies|ied|ication)\b",
+        r"\bdismiss(?:es|ed|ing)?\b",
+        r"\bno\s+attack\b",
+        r"\bdid\s+not\s+(?:attack|strike|bomb|target|order|invade|kill)\b",
+        r"\bnever\s+(?:attack|strike|bomb|target|order|invade|happen)\b",
+        r"\bfalse\s+(?:report|claim|allegation|rumor|narrative)\b",
+        r"\bmisinformation\b",
+        r"\bdisinformation\b",
+        r"\bpropaganda\b",
+        r"\b(?:army|military|government|ministry|official)\s+(?:says|clarifies|denies|refutes)\b",
+        r"\brumor[s]?\b.*\buntrue\b",
+        r"\bcategorically\b",
+        r"\bbaseless\b",
+    ]
+]
+
+# Broader cell radius for sparse telemetry regions (Africa, Central Asia, etc.)
+# These regions have fewer IODA/RIPE probes so outage data is sparser
+_SPARSE_REGIONS_LAT_RANGES = [
+    (-35, 37),   # Africa roughly
+    (25, 50),    # Central Asia band (when lng 40-90)
+]
+
+
+def _is_sparse_region(lat: float, lng: float) -> bool:
+    """Check if coordinates fall in a region with sparse telemetry coverage."""
+    # Africa
+    if -35 <= lat <= 37 and -20 <= lng <= 55:
+        return True
+    # Central Asia
+    if 25 <= lat <= 50 and 40 <= lng <= 90:
+        return True
+    # South America interior
+    if -55 <= lat <= 12 and -80 <= lng <= -35:
+        return True
+    return False
+
+
+def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
+    """Great-circle distance in km."""
+    R = 6371.0
+    dlat = math.radians(lat2 - lat1)
+    dlon = math.radians(lon2 - lon1)
+    a = (math.sin(dlat / 2) ** 2 +
+         math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) *
+         math.sin(dlon / 2) ** 2)
+    return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
+
+
+def _matches_denial(text: str) -> bool:
+    """Check if text matches any denial/official-statement pattern."""
+    return any(p.search(text) for p in _DENIAL_PATTERNS)
+
+
+def _detect_contradictions(data: dict) -> list[dict]:
+    """Detect possible contradictions between official statements and telemetry.
+
+    Scans GDELT headlines for denial language, then checks whether internet
+    outages or other infrastructure disruptions exist in the same geographic
+    region.  Scores confidence and lists alternative explanations.
+    """
+    gdelt = data.get("gdelt") or []
+    internet_outages = data.get("internet_outages") or []
+    news = data.get("news") or []
+    prediction_markets = data.get("prediction_markets") or []
+
+    # ── Step 1: Find GDELT events with denial/official-statement language ──
+    denial_events: list[dict] = []
+
+    # GDELT comes as GeoJSON features
+    gdelt_features = gdelt
+    if isinstance(gdelt, dict):
+        gdelt_features = gdelt.get("features", [])
+
+    for feature in gdelt_features:
+        # Handle both GeoJSON features and flat dicts
+        if "properties" in feature and "geometry" in feature:
+            props = feature.get("properties", {})
+            geom = feature.get("geometry", {})
+            coords = geom.get("coordinates", [])
+            if len(coords) >= 2:
+                lng, lat = float(coords[0]), float(coords[1])
+            else:
+                continue
+            headlines = props.get("_headlines_list", [])
+            urls = props.get("_urls_list", [])
+            name = props.get("name", "")
+            count = props.get("count", 1)
+        else:
+            lat = feature.get("lat") or feature.get("actionGeo_Lat")
+            lng = feature.get("lng") or feature.get("lon") or feature.get("actionGeo_Long")
+            if lat is None or lng is None:
+                continue
+            lat, lng = float(lat), float(lng)
+            headlines = [feature.get("title", "")]
+            urls = [feature.get("sourceurl", "")]
+            name = feature.get("name", "")
+            count = 1
+
+        # Check all headlines + URL slugs for denial patterns
+        all_text = " ".join(str(h) for h in headlines if h)
+        all_text += " " + " ".join(str(u) for u in urls if u)
+
+        if _matches_denial(all_text):
+            denial_events.append({
+                "lat": lat,
+                "lng": lng,
+                "headlines": [h for h in headlines if h][:5],
+                "urls": [u for u in urls if u][:3],
+                "location_name": name,
+                "event_count": count,
+            })
+
+    # Also scan news articles for denial language
+    for article in news:
+        title = str(article.get("title", "") or "")
+        desc = str(article.get("description", "") or article.get("summary", "") or "")
+        if not _matches_denial(title + " " + desc):
+            continue
+        # News articles often lack coordinates — try to match to GDELT locations
+        # For now, only include if we have coordinates
+        lat = article.get("lat") or article.get("latitude")
+        lng = article.get("lng") or article.get("lon") or article.get("longitude")
+        if lat is not None and lng is not None:
+            denial_events.append({
+                "lat": float(lat),
+                "lng": float(lng),
+                "headlines": [title],
+                "urls": [article.get("url") or article.get("link") or ""],
+                "location_name": "",
+                "event_count": 1,
+            })
+
+    if not denial_events:
+        return []
+
+    # ── Step 2: Cross-reference with internet outages ──
+    alerts: list[dict] = []
+
+    for denial in denial_events:
+        d_lat, d_lng = denial["lat"], denial["lng"]
+        sparse = _is_sparse_region(d_lat, d_lng)
+        search_radius_km = 1500.0 if sparse else 500.0
+
+        # Find nearby outages
+        nearby_outages: list[dict] = []
+        for outage in internet_outages:
+            o_lat = outage.get("lat") or outage.get("latitude")
+            o_lng = outage.get("lng") or outage.get("lon") or outage.get("longitude")
+            if o_lat is None or o_lng is None:
+                continue
+            try:
+                dist = _haversine_km(d_lat, d_lng, float(o_lat), float(o_lng))
+            except (ValueError, TypeError):
+                continue
+            if dist <= search_radius_km:
+                nearby_outages.append({
+                    "region": outage.get("region_name") or outage.get("country_name", ""),
+                    "severity": _outage_pct(outage),
+                    "distance_km": round(dist, 0),
+                    "level": outage.get("level", ""),
+                })
+
+        # ── Step 3: Check prediction markets for related movements ──
+        denial_text = " ".join(denial["headlines"]).lower()
+        related_markets: list[dict] = []
+        for market in prediction_markets:
+            m_title = str(market.get("title", "") or market.get("question", "") or "").lower()
+            # Look for keyword overlap between denial and market
+            denial_words = set(re.findall(r"[a-z]{4,}", denial_text))
+            market_words = set(re.findall(r"[a-z]{4,}", m_title))
+            overlap = denial_words & market_words - {"that", "this", "with", "from", "have", "been", "were", "will", "says", "said"}
+            if len(overlap) >= 2:
+                prob = market.get("probability") or market.get("lastTradePrice") or market.get("yes_price")
+                if prob is not None:
+                    related_markets.append({
+                        "title": market.get("title") or market.get("question"),
+                        "probability": float(prob),
+                    })
+
+        # ── Step 4: Score confidence and assign context rating ──
+        indicators = 1  # denial itself
+        drivers: list[str] = []
+
+        # Primary driver: the denial headline
+        headline_display = denial["headlines"][0] if denial["headlines"] else "Official statement"
+        if len(headline_display) > 80:
+            headline_display = headline_display[:77] + "..."
+        drivers.append(f'"{headline_display}"')
+
+        # Outage co-location
+        has_outage = False
+        if nearby_outages:
+            best_outage = max(nearby_outages, key=lambda o: o["severity"])
+            if best_outage["severity"] >= 10:
+                indicators += 1
+                has_outage = True
+                drivers.append(
+                    f"Internet outage {best_outage['severity']:.0f}% "
+                    f"({best_outage['region']}, {best_outage['distance_km']:.0f}km away)"
+                )
+            elif best_outage["severity"] > 0:
+                indicators += 0.5  # minor outage, partial indicator
+                has_outage = True
+                drivers.append(
+                    f"Minor outage ({best_outage['region']}, "
+                    f"{best_outage['distance_km']:.0f}km away)"
+                )
+
+        # Prediction market signal
+        has_market = False
+        if related_markets:
+            indicators += 1
+            has_market = True
+            top_market = related_markets[0]
+            drivers.append(
+                f"Market: \"{top_market['title'][:50]}\" "
+                f"at {top_market['probability']:.0%}"
+            )
+
+        # Multiple denial sources strengthen the signal
+        if denial["event_count"] > 1:
+            indicators += 0.5
+            drivers.append(f"{denial['event_count']} sources reporting")
+
+        # Context rating
+        if has_outage and has_market:
+            context = "STRONG"
+        elif has_outage:
+            context = "MODERATE"
+        elif has_market:
+            context = "WEAK"  # market signal without infra disruption
+        else:
+            context = "DETECTION_GAP"
+
+        # Severity mapping
+        if context == "STRONG":
+            sev = "high"
+        elif context == "MODERATE":
+            sev = "medium"
+        else:
+            sev = "low"
+
+        # Alternative explanations (always present — this is a hypothesis generator)
+        alternatives: list[str] = []
+        if has_outage:
+            alternatives.append("Routine infrastructure maintenance or cable damage")
+            alternatives.append("Weather-related outage coinciding with news cycle")
+        if not has_outage and context == "DETECTION_GAP":
+            alternatives.append("Statement may be truthful — no contradicting telemetry found")
+            alternatives.append("Telemetry coverage gap in this region")
+        alternatives.append("Denial may be responding to social media rumors, not real events")
+
+        lat_c, lng_c = _cell_center(_cell_key(d_lat, d_lng))
+        alerts.append({
+            "lat": lat_c,
+            "lng": lng_c,
+            "type": "contradiction",
+            "severity": sev,
+            "score": _severity_score(sev),
+            "drivers": drivers[:4],
+            "cell_size": _CELL_SIZE,
+            "context": context,
+            "alternatives": alternatives[:3],
+            "location_name": denial.get("location_name", ""),
+            "headlines": denial["headlines"][:3],
+            "related_markets": related_markets[:3],
+            "nearby_outages": nearby_outages[:5],
+        })
+
+    # Deduplicate: keep highest-scored alert per cell
+    seen_cells: dict[str, dict] = {}
+    for alert in alerts:
+        key = _cell_key(alert["lat"], alert["lng"])
+        if key not in seen_cells or alert["score"] > seen_cells[key]["score"]:
+            seen_cells[key] = alert
+
+    result = list(seen_cells.values())
+    if result:
+        by_context = defaultdict(int)
+        for a in result:
+            by_context[a["context"]] += 1
+        logger.info(
+            "Contradictions: %d possible (%s)",
+            len(result),
+            ", ".join(f"{v} {k}" for k, v in sorted(by_context.items())),
+        )
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+# ---------------------------------------------------------------------------
+# Correlation → Pin bridge
+# ---------------------------------------------------------------------------
+
+# Types and their pin categories
+_CORR_PIN_CATEGORIES = {
+    "rf_anomaly": "anomaly",
+    "military_buildup": "military",
+    "infra_cascade": "infrastructure",
+    "contradiction": "research",
+}
+
+# Deduplicate: don't re-pin the same cell within this window (seconds).
+_CORR_PIN_DEDUP_WINDOW = 600  # 10 minutes
+_recent_corr_pins: dict[str, float] = {}
+
+
+def _auto_pin_correlations(alerts: list[dict]) -> int:
+    """Create AI Intel pins for high-severity correlation alerts.
+
+    Only pins alerts with severity >= medium.  Uses cell-key dedup so the
+    same grid cell doesn't get re-pinned every fetch cycle.
+
+    Returns the number of pins created this cycle.
+    """
+    import time as _time
+
+    now = _time.time()
+
+    # Evict stale dedup entries
+    expired = [k for k, ts in _recent_corr_pins.items() if now - ts > _CORR_PIN_DEDUP_WINDOW]
+    for k in expired:
+        _recent_corr_pins.pop(k, None)
+
+    created = 0
+    for alert in alerts:
+        sev = alert.get("severity", "low")
+        if sev == "low":
+            continue  # Don't pin low-severity noise
+
+        lat = alert.get("lat")
+        lng = alert.get("lng")
+        if lat is None or lng is None:
+            continue
+
+        # Dedup key: type + cell
+        dedup_key = f"{alert['type']}:{_cell_key(lat, lng)}"
+        if dedup_key in _recent_corr_pins:
+            continue
+
+        category = _CORR_PIN_CATEGORIES.get(alert["type"], "anomaly")
+        drivers = alert.get("drivers", [])
+        atype = alert["type"]
+
+        if atype == "contradiction":
+            ctx = alert.get("context", "")
+            label = f"[{ctx}] Possible Contradiction"
+            parts = list(drivers)
+            if alert.get("alternatives"):
+                parts.append("Alternatives: " + "; ".join(alert["alternatives"][:2]))
+            description = " | ".join(parts) if parts else "Narrative contradiction detected"
+        else:
+            label = f"[{sev.upper()}] {atype.replace('_', ' ').title()}"
+            description = "; ".join(drivers) if drivers else "Multi-layer correlation alert"
+
+        try:
+            from services.ai_pin_store import create_pin
+
+            meta = {
+                "correlation_type": atype,
+                "severity": sev,
+                "drivers": drivers,
+                "cell_size": alert.get("cell_size", _CELL_SIZE),
+            }
+            # Add contradiction-specific metadata
+            if atype == "contradiction":
+                meta["context_rating"] = alert.get("context", "")
+                meta["alternatives"] = alert.get("alternatives", [])
+                meta["headlines"] = alert.get("headlines", [])
+                meta["location_name"] = alert.get("location_name", "")
+                if alert.get("related_markets"):
+                    meta["related_markets"] = alert["related_markets"]
+
+            create_pin(
+                lat=lat,
+                lng=lng,
+                label=label,
+                category=category,
+                description=description,
+                source="correlation_engine",
+                confidence=alert.get("score", 60) / 100.0,
+                ttl_hours=2.0,  # Auto-expire correlation pins after 2 hours
+                metadata=meta,
+            )
+            _recent_corr_pins[dedup_key] = now
+            created += 1
+        except Exception as exc:
+            logger.warning("Failed to auto-pin correlation: %s", exc)
+
+    if created:
+        logger.info("Correlation engine auto-pinned %d alerts", created)
+    return created
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def compute_correlations(data: dict) -> list[dict]:
+    """Run all correlation detectors and return merged alert list."""
+    alerts: list[dict] = []
+
+    try:
+        alerts.extend(_detect_rf_anomalies(data))
+    except Exception as e:
+        logger.error("Correlation engine RF anomaly error: %s", e)
+
+    try:
+        alerts.extend(_detect_military_buildups(data))
+    except Exception as e:
+        logger.error("Correlation engine military buildup error: %s", e)
+
+    try:
+        alerts.extend(_detect_infra_cascades(data))
+    except Exception as e:
+        logger.error("Correlation engine infra cascade error: %s", e)
+
+    # Contradiction detection removed from automated engine — too many false
+    # positives from regex headline matching.  Contradiction/analysis alerts are
+    # now placed by OpenClaw agents via place_analysis_zone, which lets an LLM
+    # reason about the evidence rather than pattern-matching keywords.
+    try:
+        from services.analysis_zone_store import get_live_zones
+        alerts.extend(get_live_zones())
+    except Exception as e:
+        logger.error("Analysis zone merge error: %s", e)
+
+    rf = sum(1 for a in alerts if a["type"] == "rf_anomaly")
+    mil = sum(1 for a in alerts if a["type"] == "military_buildup")
+    infra = sum(1 for a in alerts if a["type"] == "infra_cascade")
+    contra = sum(1 for a in alerts if a["type"] == "contradiction")
+    if alerts:
+        logger.info(
+            "Correlations: %d alerts (%d rf, %d mil, %d infra, %d contra)",
+            len(alerts), rf, mil, infra, contra,
+        )
+
+    # Correlation alerts are returned in the correlations data feed only.
+    # They are NOT auto-pinned to AI Intel — that layer is reserved for
+    # user / OpenClaw pins.  Correlations are visualised via the dedicated
+    # correlations overlay on the map.
+
+    return alerts

backend/services/feed_ingester.py

@@ -0,0 +1,245 @@
+"""Feed Ingester — background daemon that refreshes feed-backed pin layers.
+
+Layers with a non-empty `feed_url` are polled at their `feed_interval`
+(seconds, minimum 60). The feed is expected to return either:
+
+  1. GeoJSON FeatureCollection — features are converted to pins
+  2. JSON array of pin objects — used directly
+
+Each refresh atomically replaces the layer's pins with the new data.
+"""
+
+import logging
+import threading
+import time
+from typing import Any
+
+import requests
+
+from services.network_utils import outbound_user_agent
+
+logger = logging.getLogger(__name__)
+
+
+def _feed_ingester_user_agent() -> str:
+    # Round 7a: per-install attribution for operator-curated feed URLs.
+    return outbound_user_agent("feed-ingester")
+
+# ---------------------------------------------------------------------------
+# State
+# ---------------------------------------------------------------------------
+
+_running = False
+_thread: threading.Thread | None = None
+_CHECK_INTERVAL = 30  # seconds between scanning for layers that need refresh
+_last_fetched: dict[str, float] = {}  # layer_id → last fetch timestamp
+_FETCH_TIMEOUT = 20  # seconds
+
+# ---------------------------------------------------------------------------
+# GeoJSON → pin conversion
+# ---------------------------------------------------------------------------
+
+
+def _geojson_features_to_pins(features: list[dict]) -> list[dict[str, Any]]:
+    """Convert GeoJSON Feature objects to pin dicts."""
+    pins: list[dict[str, Any]] = []
+    for feat in features:
+        if not isinstance(feat, dict):
+            continue
+        geom = feat.get("geometry") or {}
+        props = feat.get("properties") or {}
+
+        # Extract coordinates
+        coords = geom.get("coordinates")
+        if geom.get("type") != "Point" or not coords or len(coords) < 2:
+            continue
+
+        lng, lat = float(coords[0]), float(coords[1])
+        if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+            continue
+
+        pin: dict[str, Any] = {
+            "lat": lat,
+            "lng": lng,
+            "label": str(props.get("label", props.get("name", props.get("title", ""))))[:200],
+            "category": str(props.get("category", "custom"))[:50],
+            "color": str(props.get("color", ""))[:20],
+            "description": str(props.get("description", props.get("summary", "")))[:2000],
+            "source": "feed",
+            "source_url": str(props.get("source_url", props.get("url", props.get("link", ""))))[:500],
+            "confidence": float(props.get("confidence", 1.0)),
+        }
+
+        # Entity attachment if present
+        entity_type = props.get("entity_type", "")
+        entity_id = props.get("entity_id", "")
+        if entity_type and entity_id:
+            pin["entity_attachment"] = {
+                "entity_type": str(entity_type),
+                "entity_id": str(entity_id),
+                "entity_label": str(props.get("entity_label", "")),
+            }
+
+        pins.append(pin)
+    return pins
+
+
+def _parse_feed_response(data: Any) -> list[dict[str, Any]]:
+    """Parse a feed response into a list of pin dicts."""
+    if isinstance(data, dict):
+        # GeoJSON FeatureCollection
+        if data.get("type") == "FeatureCollection" and isinstance(data.get("features"), list):
+            return _geojson_features_to_pins(data["features"])
+        # Single Feature
+        if data.get("type") == "Feature":
+            return _geojson_features_to_pins([data])
+        # Wrapped response like {"ok": true, "data": [...]}
+        inner = data.get("data") or data.get("results") or data.get("pins") or data.get("items")
+        if isinstance(inner, list):
+            return _normalize_pin_list(inner)
+
+    if isinstance(data, list):
+        # Check if first item looks like a GeoJSON Feature
+        if data and isinstance(data[0], dict) and data[0].get("type") == "Feature":
+            return _geojson_features_to_pins(data)
+        return _normalize_pin_list(data)
+
+    return []
+
+
+def _normalize_pin_list(items: list) -> list[dict[str, Any]]:
+    """Normalize a list of raw pin objects, ensuring lat/lng are present."""
+    pins: list[dict[str, Any]] = []
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        lat = item.get("lat") or item.get("latitude")
+        lng = item.get("lng") or item.get("lon") or item.get("longitude")
+        if lat is None or lng is None:
+            continue
+        try:
+            lat, lng = float(lat), float(lng)
+        except (ValueError, TypeError):
+            continue
+        if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+            continue
+
+        pin: dict[str, Any] = {
+            "lat": lat,
+            "lng": lng,
+            "label": str(item.get("label", item.get("name", item.get("title", ""))))[:200],
+            "category": str(item.get("category", "custom"))[:50],
+            "color": str(item.get("color", ""))[:20],
+            "description": str(item.get("description", item.get("summary", "")))[:2000],
+            "source": "feed",
+            "source_url": str(item.get("source_url", item.get("url", item.get("link", ""))))[:500],
+            "confidence": float(item.get("confidence", 1.0)),
+        }
+
+        entity_type = item.get("entity_type", "")
+        entity_id = item.get("entity_id", "")
+        if entity_type and entity_id:
+            pin["entity_attachment"] = {
+                "entity_type": str(entity_type),
+                "entity_id": str(entity_id),
+                "entity_label": str(item.get("entity_label", "")),
+            }
+
+        pins.append(pin)
+    return pins
+
+
+# ---------------------------------------------------------------------------
+# Fetch a single layer
+# ---------------------------------------------------------------------------
+
+
+def _fetch_layer_feed(layer: dict[str, Any]) -> None:
+    """Fetch a feed URL and replace the layer's pins."""
+    layer_id = layer["id"]
+    feed_url = layer["feed_url"]
+    layer_name = layer.get("name", layer_id)
+
+    try:
+        resp = requests.get(
+            feed_url,
+            timeout=_FETCH_TIMEOUT,
+            headers={"User-Agent": _feed_ingester_user_agent()},
+        )
+        resp.raise_for_status()
+        data = resp.json()
+    except requests.RequestException as e:
+        logger.warning("Feed fetch failed for layer '%s' (%s): %s", layer_name, feed_url, e)
+        return
+    except (ValueError, TypeError) as e:
+        logger.warning("Feed parse failed for layer '%s' (%s): %s", layer_name, feed_url, e)
+        return
+
+    pins = _parse_feed_response(data)
+
+    from services.ai_pin_store import replace_layer_pins, update_layer
+    count = replace_layer_pins(layer_id, pins)
+
+    # Update layer metadata with last_fetched timestamp
+    update_layer(layer_id, feed_last_fetched=time.time())
+
+    _last_fetched[layer_id] = time.time()
+    logger.info("Feed refresh for layer '%s': %d pins from %s", layer_name, count, feed_url)
+
+
+# ---------------------------------------------------------------------------
+# Main loop
+# ---------------------------------------------------------------------------
+
+
+def _ingest_loop() -> None:
+    """Daemon loop: scan for feed layers and refresh those that are due."""
+    while _running:
+        try:
+            from services.ai_pin_store import get_feed_layers
+
+            layers = get_feed_layers()
+            now = time.time()
+
+            for layer in layers:
+                layer_id = layer["id"]
+                interval = max(60, layer.get("feed_interval", 300))
+                last = _last_fetched.get(layer_id, 0)
+
+                if now - last >= interval:
+                    try:
+                        _fetch_layer_feed(layer)
+                    except Exception as e:
+                        logger.warning("Feed ingestion error for layer %s: %s",
+                                       layer.get("name", layer_id), e)
+
+        except Exception as e:
+            logger.error("Feed ingester loop error: %s", e)
+
+        # Sleep in short increments so we can stop cleanly
+        for _ in range(int(_CHECK_INTERVAL)):
+            if not _running:
+                break
+            time.sleep(1)
+
+
+# ---------------------------------------------------------------------------
+# Start / stop
+# ---------------------------------------------------------------------------
+
+
+def start_feed_ingester() -> None:
+    """Start the feed ingester daemon thread."""
+    global _running, _thread
+    if _thread and _thread.is_alive():
+        return
+    _running = True
+    _thread = threading.Thread(target=_ingest_loop, daemon=True, name="feed-ingester")
+    _thread.start()
+    logger.info("Feed ingester daemon started (check interval=%ds)", _CHECK_INTERVAL)
+
+
+def stop_feed_ingester() -> None:
+    """Stop the feed ingester daemon."""
+    global _running
+    _running = False

backend/services/fetch_health.py

@@ -0,0 +1,94 @@
+"""Fetch health registry — tracks per-source success/failure counts and timings."""
+
+import logging
+import threading
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+from services.fetchers._store import _data_lock, source_freshness
+
+logger = logging.getLogger(__name__)
+
+_health: Dict[str, Dict[str, Any]] = {}
+_lock = threading.Lock()
+
+
+def _now_iso() -> str:
+    return datetime.utcnow().isoformat()
+
+
+def _update_source_freshness(source: str, *, ok: bool, error_msg: Optional[str] = None):
+    """Mirror health summary into shared store for visibility."""
+    with _data_lock:
+        entry = source_freshness.get(source, {})
+        if ok:
+            entry["last_ok"] = _now_iso()
+        else:
+            entry["last_error"] = _now_iso()
+            if error_msg:
+                entry["last_error_msg"] = error_msg[:200]
+        source_freshness[source] = entry
+
+
+def record_success(source: str, duration_s: Optional[float] = None, count: Optional[int] = None):
+    """Record a successful fetch for a source."""
+    now = _now_iso()
+    with _lock:
+        entry = _health.setdefault(
+            source,
+            {
+                "ok_count": 0,
+                "error_count": 0,
+                "last_ok": None,
+                "last_error": None,
+                "last_error_msg": None,
+                "last_duration_ms": None,
+                "avg_duration_ms": None,
+                "last_count": None,
+            },
+        )
+        entry["ok_count"] += 1
+        entry["last_ok"] = now
+        if duration_s is not None:
+            dur_ms = round(duration_s * 1000, 1)
+            entry["last_duration_ms"] = dur_ms
+            prev_avg = entry["avg_duration_ms"] or 0.0
+            n = entry["ok_count"]
+            entry["avg_duration_ms"] = round(((prev_avg * (n - 1)) + dur_ms) / n, 1)
+        if count is not None:
+            entry["last_count"] = count
+
+    _update_source_freshness(source, ok=True)
+
+
+def record_failure(source: str, error: Exception, duration_s: Optional[float] = None):
+    """Record a failed fetch for a source."""
+    now = _now_iso()
+    err_msg = str(error)
+    with _lock:
+        entry = _health.setdefault(
+            source,
+            {
+                "ok_count": 0,
+                "error_count": 0,
+                "last_ok": None,
+                "last_error": None,
+                "last_error_msg": None,
+                "last_duration_ms": None,
+                "avg_duration_ms": None,
+                "last_count": None,
+            },
+        )
+        entry["error_count"] += 1
+        entry["last_error"] = now
+        entry["last_error_msg"] = err_msg[:200]
+        if duration_s is not None:
+            entry["last_duration_ms"] = round(duration_s * 1000, 1)
+
+    _update_source_freshness(source, ok=False, error_msg=err_msg)
+
+
+def get_health_snapshot() -> Dict[str, Dict[str, Any]]:
+    """Return a snapshot of current fetch health state."""
+    with _lock:
+        return {k: dict(v) for k, v in _health.items()}

backend/services/fetchers/_store.py

@@ -0,0 +1,352 @@
+"""Shared in-memory data store for all fetcher modules.
+
+Central location for latest_data, source_timestamps, and the data lock.
+Every fetcher imports from here instead of maintaining its own copy.
+"""
+
+import copy
+import threading
+import logging
+from datetime import datetime
+from typing import Any, Dict, List, Optional, TypedDict
+
+logger = logging.getLogger("services.data_fetcher")
+
+
+class DashboardData(TypedDict, total=False):
+    """Schema for the in-memory data store. Catches key typos at dev time."""
+
+    last_updated: Optional[str]
+    news: List[Dict[str, Any]]
+    stocks: Dict[str, Any]
+    oil: Dict[str, Any]
+    commercial_flights: List[Dict[str, Any]]
+    private_flights: List[Dict[str, Any]]
+    private_jets: List[Dict[str, Any]]
+    flights: List[Dict[str, Any]]
+    ships: List[Dict[str, Any]]
+    military_flights: List[Dict[str, Any]]
+    tracked_flights: List[Dict[str, Any]]
+    cctv: List[Dict[str, Any]]
+    weather: Optional[Dict[str, Any]]
+    earthquakes: List[Dict[str, Any]]
+    uavs: List[Dict[str, Any]]
+    frontlines: Optional[Any]
+    gdelt: List[Dict[str, Any]]
+    liveuamap: List[Dict[str, Any]]
+    kiwisdr: List[Dict[str, Any]]
+    space_weather: Optional[Dict[str, Any]]
+    internet_outages: List[Dict[str, Any]]
+    firms_fires: List[Dict[str, Any]]
+    datacenters: List[Dict[str, Any]]
+    airports: List[Dict[str, Any]]
+    gps_jamming: List[Dict[str, Any]]
+    satellites: List[Dict[str, Any]]
+    satellite_source: str
+    satellite_analysis: Dict[str, Any]
+    prediction_markets: List[Dict[str, Any]]
+    sigint: List[Dict[str, Any]]
+    sigint_totals: Dict[str, Any]
+    mesh_channel_stats: Dict[str, Any]
+    meshtastic_map_nodes: List[Dict[str, Any]]
+    meshtastic_map_fetched_at: Optional[float]
+    weather_alerts: List[Dict[str, Any]]
+    air_quality: List[Dict[str, Any]]
+    volcanoes: List[Dict[str, Any]]
+    fishing_activity: List[Dict[str, Any]]
+    satnogs_stations: List[Dict[str, Any]]
+    satnogs_observations: List[Dict[str, Any]]
+    tinygs_satellites: List[Dict[str, Any]]
+    ukraine_alerts: List[Dict[str, Any]]
+    power_plants: List[Dict[str, Any]]
+    viirs_change_nodes: List[Dict[str, Any]]
+    fimi: Dict[str, Any]
+    psk_reporter: List[Dict[str, Any]]
+    correlations: List[Dict[str, Any]]
+    uap_sightings: List[Dict[str, Any]]
+    wastewater: List[Dict[str, Any]]
+    crowdthreat: List[Dict[str, Any]]
+    sar_scenes: List[Dict[str, Any]]
+    sar_anomalies: List[Dict[str, Any]]
+    sar_aoi_coverage: List[Dict[str, Any]]
+    road_corridor_trends: Dict[str, Any]
+    malware_threats: Dict[str, Any]
+    cyber_threats: Dict[str, Any]
+    scm_suppliers: Dict[str, Any]
+    telegram_osint: Dict[str, Any]
+
+
+# In-memory store
+latest_data: DashboardData = {
+    "last_updated": None,
+    "news": [],
+    "stocks": {},
+    "oil": {},
+    "flights": [],
+    "ships": [],
+    "military_flights": [],
+    "tracked_flights": [],
+    "cctv": [],
+    "weather": None,
+    "earthquakes": [],
+    "uavs": [],
+    "frontlines": None,
+    "gdelt": [],
+    "liveuamap": [],
+    "kiwisdr": [],
+    "space_weather": None,
+    "internet_outages": [],
+    "firms_fires": [],
+    "datacenters": [],
+    "military_bases": [],
+    "prediction_markets": [],
+    "sigint": [],
+    "sigint_totals": {},
+    "mesh_channel_stats": {},
+    "meshtastic_map_nodes": [],
+    "meshtastic_map_fetched_at": None,
+    "weather_alerts": [],
+    "air_quality": [],
+    "volcanoes": [],
+    "fishing_activity": [],
+    "satnogs_stations": [],
+    "satnogs_observations": [],
+    "tinygs_satellites": [],
+    "ukraine_alerts": [],
+    "power_plants": [],
+    "viirs_change_nodes": [],
+    "fimi": {},
+    "psk_reporter": [],
+    "correlations": [],
+    "uap_sightings": [],
+    "wastewater": [],
+    "crowdthreat": [],
+    "sar_scenes": [],
+    "sar_anomalies": [],
+    "sar_aoi_coverage": [],
+    "road_corridor_trends": {"updated_at": None, "corridors": []},
+    "malware_threats": {"threats": [], "total": 0, "timestamp": None},
+    "cyber_threats": {"threats": [], "stats": {}},
+    "scm_suppliers": {"suppliers": [], "total": 0, "critical_count": 0},
+    "telegram_osint": {"posts": [], "total": 0, "geolocated": 0, "timestamp": None},
+}
+
+# Per-source freshness timestamps
+source_timestamps = {}
+
+# Per-source health/freshness metadata (last ok/error)
+source_freshness: dict[str, dict] = {}
+
+
+def _mark_fresh(*keys):
+    """Record the current UTC time for one or more data source keys."""
+    now = datetime.utcnow().isoformat()
+    global _data_version
+    changed: list[tuple[str, int, int]] = []  # (layer, version, count)
+    with _data_lock:
+        for k in keys:
+            source_timestamps[k] = now
+            _layer_versions[k] = _layer_versions.get(k, 0) + 1
+            # Grab entity count while we hold the lock (cheap len())
+            val = latest_data.get(k)
+            count = len(val) if isinstance(val, list) else (1 if val is not None else 0)
+            changed.append((k, _layer_versions[k], count))
+        # Publish partial fetch progress immediately so the frontend can
+        # observe newly available data without waiting for the entire tier.
+        _data_version += 1
+    # Notify SSE listeners outside the lock to avoid deadlocks
+    _notify_layer_change(changed)
+
+
+# Thread lock for safe reads/writes to latest_data
+_data_lock = threading.Lock()
+
+# Monotonic version counter — incremented on each data update cycle.
+# Used for cheap ETag generation instead of MD5-hashing the full response.
+_data_version: int = 0
+
+# Per-layer version counters — incremented only when that specific layer
+# refreshes.  Used by get_layer_slice for per-layer incremental updates
+# and by the SSE stream to push targeted layer_changed notifications.
+_layer_versions: dict[str, int] = {}
+
+
+# ---------------------------------------------------------------------------
+# Layer-change notification callbacks (thread → async SSE bridge)
+# ---------------------------------------------------------------------------
+_layer_change_callbacks: list = []
+_layer_change_callbacks_lock = threading.Lock()
+
+
+def register_layer_change_callback(callback) -> None:
+    """Register a callback invoked on every _mark_fresh().
+
+    Signature: callback(layer: str, version: int, count: int)
+    Called from fetcher threads — must be thread-safe.
+    """
+    with _layer_change_callbacks_lock:
+        _layer_change_callbacks.append(callback)
+
+
+def unregister_layer_change_callback(callback) -> None:
+    """Remove a previously registered callback."""
+    with _layer_change_callbacks_lock:
+        try:
+            _layer_change_callbacks.remove(callback)
+        except ValueError:
+            pass
+
+
+def _notify_layer_change(changed: list[tuple[str, int, int]]) -> None:
+    """Fire all registered callbacks for each changed layer."""
+    with _layer_change_callbacks_lock:
+        cbs = list(_layer_change_callbacks)
+    for cb in cbs:
+        for layer, version, count in changed:
+            try:
+                cb(layer, version, count)
+            except Exception:
+                pass
+
+
+def get_layer_versions() -> dict[str, int]:
+    """Return a snapshot of all per-layer version counters."""
+    with _data_lock:
+        return dict(_layer_versions)
+
+
+def get_layer_version(layer: str) -> int:
+    """Return the version counter for a single layer (0 if never refreshed)."""
+    with _data_lock:
+        return _layer_versions.get(layer, 0)
+
+
+def bump_data_version() -> None:
+    """Increment the data version counter after a fetch cycle completes."""
+    global _data_version
+    with _data_lock:
+        _data_version += 1
+
+
+def get_data_version() -> int:
+    """Return the current data version (for ETag generation)."""
+    with _data_lock:
+        return _data_version
+
+
+_active_layers_version: int = 0
+
+
+def bump_active_layers_version() -> None:
+    """Increment the active-layer version when frontend toggles change response shape."""
+    global _active_layers_version
+    with _data_lock:
+        _active_layers_version += 1
+
+
+def get_active_layers_version() -> int:
+    """Return the current active-layer version (for ETag generation)."""
+    with _data_lock:
+        return _active_layers_version
+
+
+def get_latest_data_subset(*keys: str) -> DashboardData:
+    """Return a deep snapshot of only the requested top-level keys.
+
+    Grabs references under the lock, then deep-copies outside it so fetcher
+    writers are not blocked for the duration of a large clone (#375).
+    """
+    with _data_lock:
+        items = [(key, latest_data.get(key)) for key in keys]
+    snap: DashboardData = {}
+    for key, value in items:
+        snap[key] = copy.deepcopy(value)
+    return snap
+
+
+def get_latest_data_deepcopy_snapshot() -> DashboardData:
+    """Deep-copy the full dashboard for legacy /api/live-data consumers."""
+    with _data_lock:
+        items = list(latest_data.items())
+    return {key: copy.deepcopy(value) for key, value in items}
+
+
+def get_latest_data_subset_refs(*keys: str) -> DashboardData:
+    """Return direct top-level references for read-only hot paths.
+
+    Writers replace top-level values under the lock instead of mutating them
+    in place, so readers can safely use these references after releasing the
+    lock as long as they do not modify them.
+    """
+    with _data_lock:
+        snap: DashboardData = {}
+        for key in keys:
+            snap[key] = latest_data.get(key)
+        return snap
+
+
+def get_source_timestamps_snapshot() -> dict[str, str]:
+    """Return a stable copy of per-source freshness timestamps."""
+    with _data_lock:
+        return dict(source_timestamps)
+
+
+# ---------------------------------------------------------------------------
+# Active layers — frontend POSTs toggles, fetchers check before running.
+# Keep these aligned with the dashboard's default layer state so startup does
+# not fetch heavyweight feeds the UI starts with disabled.
+# ---------------------------------------------------------------------------
+active_layers: dict[str, bool] = {
+    "flights": True,
+    "private": True,
+    "jets": True,
+    "military": True,
+    "tracked": True,
+    "satellites": True,
+    "ships_military": True,
+    "ships_cargo": True,
+    "ships_civilian": True,
+    "ships_passenger": True,
+    "ships_tracked_yachts": True,
+    "earthquakes": True,
+    "cctv": True,
+    "ukraine_frontline": True,
+    "global_incidents": True,
+    "gps_jamming": True,
+    "kiwisdr": True,
+    "scanners": True,
+    "firms": True,
+    "internet_outages": True,
+    "datacenters": True,
+    "military_bases": True,
+    "sigint_meshtastic": True,
+    "sigint_aprs": True,
+    "weather_alerts": True,
+    "air_quality": True,
+    "volcanoes": True,
+    "fishing_activity": True,
+    "satnogs": True,
+    "tinygs": True,
+    "ukraine_alerts": True,
+    "power_plants": True,
+    "viirs_nightlights": False,
+    "psk_reporter": True,
+    "correlations": True,
+    "contradictions": True,
+    "uap_sightings": True,
+    "wastewater": True,
+    "ai_intel": True,
+    "crowdthreat": False,
+    "sar": True,
+    "road_corridor_trends": False,
+    "malware_c2": False,
+    "submarine_cables": False,
+    "scm_suppliers": False,
+    "cyber_threats": False,
+    "telegram_osint": True,
+}
+
+
+def is_any_active(*layer_names: str) -> bool:
+    """Return True if any of the given layer names is currently active."""
+    return any(active_layers.get(name, True) for name in layer_names)