chore(deps): bump slowapi from 0.1.9 to 0.1.10 in /backend

Bumps [slowapi](https://github.com/laurents/slowapi) from 0.1.9 to 0.1.10.
- [Release notes](https://github.com/laurents/slowapi/releases)
- [Changelog](https://github.com/laurentS/slowapi/blob/master/CHANGELOG.md)
- [Commits](https://github.com/laurents/slowapi/compare/v0.1.9...v0.1.10)

---
updated-dependencies:
- dependency-name: slowapi
  dependency-version: 0.1.10
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

.dockerignore

@@ -8,6 +8,18 @@ __pycache__/
 venv/
 .venv/
 .ruff_cache/
+local-artifacts/
+release-secrets/
+
+# Never send local configuration or credentials into Docker builds
+.env
+.env.*
+**/.env
+**/.env.*
+*.pem
+*.key
+*.p12
+*.pfx
 
 # privacy-core build caches (source is needed, artifacts are not)
 privacy-core/target/
@@ -21,3 +33,24 @@ privacy-core/.codex-tmp/
 *.log
 extra/
 prototype/
+
+# Runtime state generated by local backend runs
+backend/.pytest_cache/
+backend/.ruff_cache/
+backend/backend.egg-info/
+backend/build/
+backend/node_modules/
+backend/timemachine/
+backend/venv/
+backend/data/*cache*.json
+backend/data/**/*cache*.json
+backend/data/wormhole*.json
+backend/data/**/wormhole*.json
+backend/data/dm_*.json
+backend/data/**/dm_*.json
+backend/data/**/peer_store.json
+backend/data/**/node.json
+backend/data/*.log
+backend/data/**/*.log
+backend/data/*.key
+backend/data/**/*.key

.env.example

@@ -3,15 +3,51 @@
 #   cp .env.example .env
 
 # ── Required for backend container ─────────────────────────────
+# OpenSky Network OAuth2 — REQUIRED for airplane telemetry.
+# Free registration at https://opensky-network.org/index.php?option=com_users&view=registration
+# Without these the flights layer falls back to ADS-B-only with major gaps in Africa, Asia, and LatAm.
 OPENSKY_CLIENT_ID=
 OPENSKY_CLIENT_SECRET=
 AIS_API_KEY=
 
+# Global Fishing Watch — fishing vessel activity events (Fishing Activity map layer).
+# Free API token from https://globalfishingwatch.org/our-apis/tokens
+# Without this the fishing_activity layer stays empty.
+# GFW_API_TOKEN=
+# Optional tuning — GFW can return 40k+ global events; defaults cap fetch for map paint.
+# GFW_EVENTS_PAGE_SIZE=500
+# GFW_EVENTS_MAX_PAGES=10
+# GFW_EVENTS_LOOKBACK_DAYS=7
+# GFW_EVENTS_TIMEOUT_S=90
+
+# Windy Webcams global CCTV layer — free key from https://api.windy.com/webcams/docs
+# WINDY_API_KEY=
+
+# Telegram OSINT map layer — scrapes public t.me/s channel previews (no bot token).
+# TELEGRAM_OSINT_ENABLED=true
+# TELEGRAM_OSINT_CHANNELS=osintdefender,insiderpaper,aljazeeraenglish,nexta_live,war_monitor
+# TELEGRAM_OSINT_TRANSLATE=true
+# TELEGRAM_OSINT_TRANSLATE_TO=en
+
+# Strategic Risk Analytics (experimental derived OSINT — off by default)
+# GT_ANALYTICS_ENABLED=false
+# GT_ANALYTICS_PROFILE=lean
+# On 1 vCPU nodes (fleet VPS), leave disabled or set profile=lean. Scheduled ingest
+# and Louvain clustering stay off until GT_ANALYTICS_ACK_LOW_CPU=true.
+# GT_ANALYTICS_ACK_LOW_CPU=false
+# GT_ANALYTICS_BASE_PRIOR=0.15
+# GT_ANALYTICS_HIGH_RISK_THRESHOLD=0.6
+# GT_ANALYTICS_SIGNAL_WEIGHTS=payroll_loan=3.0,purge=3.5,troop_movement=3.0
+# GT_ANALYTICS_WATCHED_CHANNELS=osintdefender,war_monitor,nexta_live
+# GT_ANALYTICS_LOUVAIN_INTERVAL_MINUTES=30
+
 # Admin key to protect sensitive endpoints (settings, updates).
-# If blank, admin endpoints are only accessible from localhost unless ALLOW_INSECURE_ADMIN=true.
+# If blank, loopback/localhost requests still work for local single-host dev.
+# Remote/non-loopback admin access requires ADMIN_KEY, or ALLOW_INSECURE_ADMIN=true in debug-only setups.
 ADMIN_KEY=
 
-# Allow insecure admin access without ADMIN_KEY (local dev only).
+# Allow insecure admin access without ADMIN_KEY (local dev only, beyond loopback).
+# Requires MESH_DEBUG_MODE=true on the backend; do not enable this for normal use.
 # ALLOW_INSECURE_ADMIN=false
 
 # User-Agent for Nominatim geocoding requests (per OSM usage policy).
@@ -29,10 +65,62 @@ ADMIN_KEY=
 # Ukraine air raid alerts — free token from https://alerts.in.ua/
 # ALERTS_IN_UA_TOKEN=
 
+# Optional NUFORC UAP sighting map enrichment via Mapbox Tilequery.
+# Leave blank to skip this optional enrichment.
+# NUFORC_MAPBOX_TOKEN=
+
+# Optional startup-risk controls.
+# On Windows, external curl fallback is off by default. LiveUAMap uses UI consent
+# when you enable Global Incidents (or set SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=true).
+# SHADOWBROKER_ENABLE_WINDOWS_CURL_FALLBACK=false
+# SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=false
+# AIS starts by default when AIS_API_KEY is set. Set to 0/false to force-disable.
+# SHADOWBROKER_ENABLE_AIS_STREAM_PROXY=true
+# Minimum visible satellite catalog before forcing a CelesTrak refresh.
+# SHADOWBROKER_MIN_VISIBLE_SATELLITES=350
+# Upper bound for TLE fallback satellite search when CelesTrak is unreachable.
+# SHADOWBROKER_MAX_VISIBLE_SATELLITES=450
+# NUFORC fallback uses the Hugging Face mirror when live NUFORC is unavailable.
+# NUFORC_HF_FALLBACK_LIMIT=250
+# NUFORC_HF_GEOCODE_LIMIT=150
+
+# First-paint cache age budgets. These let the map and Global Threat Intercept
+# paint from the last local snapshot while live feeds refresh in the background.
+# FAST_STARTUP_CACHE_MAX_AGE_S=21600
+# INTEL_STARTUP_CACHE_MAX_AGE_S=21600
+
+# Docker resource tuning. The backend synthesizes large geospatial feeds; keep
+# this at 4G or higher on hosts that run AIS, OpenSky, CCTV, satellites, and
+# threat feeds together. Lower caps can cause Docker OOM restarts and empty
+# slow layers such as news, UAP sightings, military bases, and wastewater.
+# BACKEND_MEMORY_LIMIT=4G
+# SHADOWBROKER_FETCH_WORKERS=8
+# SHADOWBROKER_SLOW_FETCH_CONCURRENCY=4
+# SHADOWBROKER_STARTUP_HEAVY_CONCURRENCY=2
+
+# Infonet bootstrap/sync responsiveness. Defaults favor fast seed failure
+# detection so stale onion peers do not make the terminal look hung.
+# MESH_SYNC_TIMEOUT_S=5
+# MESH_SYNC_MAX_PEERS_PER_CYCLE=3
+# MESH_BOOTSTRAP_SEED_FAILURE_COOLDOWN_S=15
+
 # Google Earth Engine for VIIRS night lights change detection (optional).
 # pip install earthengine-api
 # GEE_SERVICE_ACCOUNT_KEY=
 
+# Copernicus CDSE — Sentinel-2 imagery (Settings → Imagery, or backend .env).
+# Free OAuth app at https://dataspace.copernicus.eu/
+# SENTINEL_CLIENT_ID=
+# SENTINEL_CLIENT_SECRET=
+
+# Sentinel-2 road corridor freight trends (DrishX engine port — opt-in slow layer).
+# pip install -e backend[road-corridor]  (or uv sync --extra road-corridor)
+# ROAD_CORRIDOR_SAT_ENABLED=false
+# ROAD_CORRIDOR_SCHEDULED_PRESETS=laredo_i35
+# ROAD_CORRIDOR_MONTHS=2
+# ROAD_CORRIDOR_MAX_FRAMES=6
+# ROAD_CORRIDOR_REFRESH_HOURS=24
+
 # Override the backend URL the frontend uses (leave blank for auto-detect)
 # NEXT_PUBLIC_API_URL=http://192.168.1.50:8000
 
@@ -77,10 +165,21 @@ ADMIN_KEY=
 
 # ── Mesh DM Relay ──────────────────────────────────────────────
 # MESH_DM_TOKEN_PEPPER=change-me
+# Optional local-dev DM root external assurance bridge.
+# These stay commented because they are machine-local file paths, not safe global defaults.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH=backend/../ops/root_witness_receipt_import.json
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH=backend/../ops/root_transparency_ledger.json
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI=backend/../ops/root_transparency_ledger.json
 
 # ── Self Update ────────────────────────────────────────────────
+# Optional ZIP updater digest pin. The updater checks this first, then
+# backend/data/release_digests.json, then the release SHA256SUMS.txt asset.
 # MESH_UPDATE_SHA256=
 
+# Optional strict nonce-only frontend CSP. Leave unset unless the exact build
+# has been verified to hydrate cleanly in your deployment.
+# SHADOWBROKER_STRICT_CSP=1
+
 # ── Wormhole (Local Agent) ─────────────────────────────────────
 # WORMHOLE_URL=http://127.0.0.1:8787
 # WORMHOLE_TRANSPORT=direct

.gitattributes

@@ -0,0 +1,2 @@
+# Force LF line endings for shell scripts
+*.sh text eol=lf

.github/CODEOWNERS

@@ -0,0 +1,32 @@
+# CODEOWNERS — assigns required reviewers for sensitive paths.
+# Format: <path glob>  <user-or-team> [<user-or-team> ...]
+# See https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
+#
+# Owners listed here are auto-requested for review when matching files
+# change in a PR. If branch protection requires CODEOWNERS approval, the
+# PR cannot be merged until an owner approves.
+
+# ── Internationalization / translations ──
+# Translation contributions are held to a stricter neutrality standard
+# than most code changes — see CONTRIBUTING.md "Translation contributions".
+# The i18n layer itself (no network calls, no telemetry, static JSON
+# bundled at build) is the structural guarantee that makes this safe;
+# changes to it need owner review.
+/frontend/src/i18n/                @BigBodyCobain
+
+# ── Security-sensitive code paths ──
+/backend/auth.py                   @BigBodyCobain
+/backend/routers/wormhole.py       @BigBodyCobain
+/backend/services/mesh/            @BigBodyCobain
+/backend/services/fetchers/        @BigBodyCobain
+
+# ── CI / build / deploy infra ──
+/.github/workflows/                @BigBodyCobain
+/.gitlab-ci.yml                    @BigBodyCobain
+/docker-compose.yml                @BigBodyCobain
+/docker-compose.gitlab.yml         @BigBodyCobain
+/helm/                             @BigBodyCobain
+
+# ── This file and policy docs ──
+/.github/CODEOWNERS                @BigBodyCobain
+/CONTRIBUTING.md                   @BigBodyCobain

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "npm"
+    directory: "/frontend"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "pip"
+    directory: "/backend"
+    schedule:
+      interval: "weekly"

.github/pull_request_template.md

@@ -0,0 +1,13 @@
+## Summary
+
+<!-- What changed and why (1–3 bullets). -->
+
+## Test plan
+
+- [ ] <!-- How you verified the change -->
+
+## Production hardening (data path / fetchers / unattended deploys only)
+
+If this PR touches the data path, fetchers, or live-data APIs, walk through [docs/production-hardening.md](https://github.com/BigBodyCobain/Shadowbroker/blob/main/docs/production-hardening.md) and note any N/A items here.
+
+- [ ] Checklist reviewed (or N/A — explain why)

.github/workflows/ci.yml

@@ -1,11 +1,33 @@
-name: CI — Lint & Test
+name: CI - Lint & Test
 
 on:
   push:
     branches: [main]
   pull_request:
     branches: [main]
-  workflow_call:  # Allow docker-publish to call this workflow as a gate
+  workflow_call:
+
+# CI flake mitigation:
+# ci.yml is triggered TWICE per PR on the same commit — once directly via
+# the `pull_request` trigger above ("Frontend Tests & Build" check) and once
+# via `workflow_call` from docker-publish.yml ("CI Gate / Frontend Tests &
+# Build" check). Both jobs land on the same Actions runner pool at the same
+# time and fight for CPU/RAM. Under contention, React's reconciliation in
+# `messagesViewFirstContact.test.tsx > removes an approved contact …`
+# overruns its 5s waitFor timeout — that's the single failure mode we've
+# seen flake on PRs #226, #237, #261, #262, #265, #294, #303, and the
+# fd7d6fa push. Backend tests and every other frontend test pass under
+# the same conditions, which is what made this look random.
+#
+# Pinning a concurrency group on the SHA (PR head, or the pushed commit
+# for main) serializes the two invocations so neither starves the other.
+# We use cancel-in-progress: false so the second one queues instead of
+# cancelling — cancelling could leave the PR check stuck "Expected" if
+# only one of the two ever finishes. Total CI time grows by ~2 min in
+# exchange for deterministic outcomes.
+concurrency:
+  group: ci-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: false
 
 jobs:
   frontend:
@@ -22,9 +44,9 @@ jobs:
           cache: npm
           cache-dependency-path: frontend/package-lock.json
       - run: npm ci
-      - run: npm run lint || echo "::warning::ESLint found issues (non-blocking)"
-      - run: npm run format:check || echo "::warning::Prettier found formatting issues (non-blocking)"
-      - run: npx vitest run --reporter=verbose || echo "::warning::Some tests failed (non-blocking)"
+      - run: npm run lint
+      - run: npm run format:check
+      - run: npx vitest run --reporter=verbose
       - run: npm run build
       - run: npm run bundle:report
 
@@ -33,6 +55,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: Run secret scan
+        run: bash backend/scripts/scan-secrets.sh --all
       - name: Install uv
         uses: astral-sh/setup-uv@v5
         with:
@@ -43,8 +67,16 @@ jobs:
           python-version: "3.11"
       - name: Install dependencies
         run: cd backend && uv sync --frozen --group dev
-      - run: cd backend && uv run ruff check . || echo "::warning::Ruff found issues (non-blocking)"
-      - run: cd backend && uv run black --check . || echo "::warning::Black found formatting issues (non-blocking)"
+      - run: cd backend && uv run ruff check .
+      - run: cd backend && uv run black --check .
       - run: cd backend && uv run python -c "from services.fetchers.retry import with_retry; from services.env_check import validate_env; print('Module imports OK')"
-      - name: Run tests
-        run: cd backend && uv run pytest tests/ -v --tb=short || echo "No pytest tests found (OK)"
+      - name: Run release smoke tests
+        run: |
+          cd backend
+          uv run pytest \
+            tests/mesh/test_mesh_node_bootstrap_runtime.py \
+            tests/mesh/test_mesh_infonet_sync_support.py \
+            tests/mesh/test_mesh_canonical.py \
+            tests/mesh/test_mesh_merkle.py \
+            tests/test_release_helper.py \
+            -v --tb=short

.github/workflows/docker-publish.yml

@@ -6,10 +6,9 @@ on:
     tags: ["v*.*.*"]
   pull_request:
     branches: ["main"]
-  
+
 env:
   REGISTRY: ghcr.io
-  # github.repository as <account>/<repo>
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
@@ -24,7 +23,6 @@ jobs:
       contents: read
       packages: write
       id-token: write
-
     strategy:
       fail-fast: false
       matrix:
@@ -33,33 +31,23 @@ jobs:
             runner: ubuntu-latest
           - platform: linux/arm64
             runner: ubuntu-24.04-arm
-
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
+      - uses: actions/checkout@v4
       - name: Lowercase image name
         run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.0.0
-
-      - name: Log into registry ${{ env.REGISTRY }}
+      - uses: docker/setup-buildx-action@v3.0.0
+      - name: Log into registry
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v3.0.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract Docker metadata
-        id: meta
+      - id: meta
         uses: docker/metadata-action@v5.0.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend
-
-      - name: Build and push Docker image by digest
-        id: build
+      - id: build
         uses: docker/build-push-action@v5.0.0
         with:
           context: ./frontend
@@ -69,17 +57,14 @@ jobs:
           cache-from: type=gha,scope=frontend-${{ matrix.platform }}
           cache-to: type=gha,mode=max,scope=frontend-${{ matrix.platform }}
           outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend,push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }}
-
       - name: Export digest
         if: github.event_name != 'pull_request'
         run: |
           mkdir -p /tmp/digests/frontend
           digest="${{ steps.build.outputs.digest }}"
           touch "/tmp/digests/frontend/${digest#sha256:}"
-
-      - name: Upload digest
+      - uses: actions/upload-artifact@v4
         if: github.event_name != 'pull_request'
-        uses: actions/upload-artifact@v4
         with:
           name: digests-frontend-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
           path: /tmp/digests/frontend/*
@@ -87,36 +72,27 @@ jobs:
           retention-days: 1
 
   merge-frontend:
-    runs-on: ubuntu-latest
     if: github.event_name != 'pull_request'
     needs: build-frontend
+    runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
-
     steps:
       - name: Lowercase image name
         run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
-
-      - name: Download digests
-        uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v4
         with:
           path: /tmp/digests/frontend
           pattern: digests-frontend-*
           merge-multiple: true
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.0.0
-
-      - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v3.0.0
+      - uses: docker/setup-buildx-action@v3.0.0
+      - uses: docker/login-action@v3.0.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract Docker metadata
-        id: meta
+      - id: meta
         uses: docker/metadata-action@v5.0.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend
@@ -124,7 +100,6 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=raw,value=latest,enable={{is_default_branch}}
-
       - name: Create and push manifest
         working-directory: /tmp/digests/frontend
         run: |
@@ -139,7 +114,6 @@ jobs:
       contents: read
       packages: write
       id-token: write
-
     strategy:
       fail-fast: false
       matrix:
@@ -148,33 +122,23 @@ jobs:
             runner: ubuntu-latest
           - platform: linux/arm64
             runner: ubuntu-24.04-arm
-
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
+      - uses: actions/checkout@v4
       - name: Lowercase image name
         run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.0.0
-
-      - name: Log into registry ${{ env.REGISTRY }}
+      - uses: docker/setup-buildx-action@v3.0.0
+      - name: Log into registry
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v3.0.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract Docker metadata
-        id: meta
+      - id: meta
         uses: docker/metadata-action@v5.0.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend
-
-      - name: Build and push Docker image by digest
-        id: build
+      - id: build
         uses: docker/build-push-action@v5.0.0
         with:
           context: .
@@ -185,17 +149,14 @@ jobs:
           cache-from: type=gha,scope=backend-${{ matrix.platform }}
           cache-to: type=gha,mode=max,scope=backend-${{ matrix.platform }}
           outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend,push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }}
-
       - name: Export digest
         if: github.event_name != 'pull_request'
         run: |
           mkdir -p /tmp/digests/backend
           digest="${{ steps.build.outputs.digest }}"
           touch "/tmp/digests/backend/${digest#sha256:}"
-
-      - name: Upload digest
+      - uses: actions/upload-artifact@v4
         if: github.event_name != 'pull_request'
-        uses: actions/upload-artifact@v4
         with:
           name: digests-backend-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
           path: /tmp/digests/backend/*
@@ -203,36 +164,27 @@ jobs:
           retention-days: 1
 
   merge-backend:
-    runs-on: ubuntu-latest
     if: github.event_name != 'pull_request'
     needs: build-backend
+    runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
-
     steps:
       - name: Lowercase image name
         run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
-
-      - name: Download digests
-        uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v4
         with:
           path: /tmp/digests/backend
           pattern: digests-backend-*
           merge-multiple: true
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.0.0
-
-      - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v3.0.0
+      - uses: docker/setup-buildx-action@v3.0.0
+      - uses: docker/login-action@v3.0.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract Docker metadata
-        id: meta
+      - id: meta
         uses: docker/metadata-action@v5.0.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend
@@ -240,7 +192,6 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=raw,value=latest,enable={{is_default_branch}}
-
       - name: Create and push manifest
         working-directory: /tmp/digests/backend
         run: |

.gitignore

@@ -6,13 +6,32 @@ node_modules/
 venv/
 env/
 .venv/
+backend/.venv-dir
+backend/venv-repair*/
+backend/.venv-repair*/
 
 # Environment Variables & Secrets
 .env
+.envrc
 .env.local
 .env.development.local
 .env.test.local
 .env.production.local
+.npmrc
+.pypirc
+.netrc
+*.pem
+*.key
+*.crt
+*.csr
+*.p12
+*.pfx
+id_rsa
+id_rsa.*
+id_ed25519
+id_ed25519.*
+known_hosts
+authorized_keys
 
 # Python caches & compiled files
 __pycache__/
@@ -22,11 +41,15 @@ __pycache__/
 .Python
 .ruff_cache/
 .pytest_cache/
+.mypy_cache/
+.hypothesis/
+.tox/
 
 # Next.js build output
 .next/
 out/
 build/
+*.tsbuildinfo
 
 # Deprecated standalone Infonet Terminal skeleton (migrated into frontend/src/components/InfonetTerminal/)
 frontend/infonet-terminal/
@@ -49,6 +72,8 @@ backend/ais_cache.json
 backend/carrier_cache.json
 backend/cctv.db
 cctv.db
+*.db
+*.sqlite
 *.sqlite3
 
 # ========================
@@ -63,8 +88,30 @@ backend/data/*
 !backend/data/military_bases.json
 !backend/data/plan_ccg_vessels.json
 !backend/data/plane_alert_db.json
+!backend/data/power_plants.json
 !backend/data/tracked_names.json
 !backend/data/yacht_alert_db.json
+# Issue #206: bundled KiwiSDR receiver directory used as last-resort
+# fallback when rx.linkfanel.net (HTTP-only upstream) is unreachable
+# or returns content that fails our integrity validation.
+!backend/data/kiwisdr_directory.json
+# Issue #201: pinned SHA-256 digests for known Tor Expert Bundle URLs.
+# Used as a second verification source when upstream .sha256sum fails.
+!backend/data/tor_bundle_digests.json
+# Issue #258: SPKI pins for stream.aisstream.io so we can survive upstream
+# Let's Encrypt renewal failures without disabling TLS validation entirely.
+!backend/data/aisstream_spki_pins.json
+# Issue #231: pinned SHA-256 digests for known release archives. Used by
+# the self-updater as a second-line integrity check when the release's
+# SHA256SUMS.txt asset can't be fetched.
+!backend/data/release_digests.json
+# Issue #244/#245/#246: one-shot carrier-position seed shipped with each
+# release. Used ONLY on first-ever startup to bootstrap carrier_cache.json;
+# after that the cache reflects this install's own GDELT observations.
+!backend/data/carrier_seed.json
+# DrishX RF model weights (MIT — see backend/third_party/drishx/NOTICE.md)
+!backend/data/drishx/
+!backend/data/drishx/rf_model.pickle
 
 # OS generated files
 .DS_Store
@@ -129,6 +176,9 @@ frontend/eslint-report.json
 # Old backups & repo clones
 .git_backup/
 local-artifacts/
+release-secrets/
+release-staging/
+.tmp-release-inspect/
 shadowbroker_repo/
 frontend/src/components.bak/
 frontend/src/components/map/icons/backups/
@@ -136,6 +186,7 @@ frontend/src/components/map/icons/backups/
 # Coverage
 coverage/
 .coverage
+.coverage.*
 dist/
 
 # Test scratch files (not in tests/ folder)
@@ -145,15 +196,22 @@ backend/services/test_*.py
 # Local analysis & dev tools
 backend/analyze_xlsx.py
 backend/services/ais_cache.json
+graphify/
+graphify-out/
 
 # ========================
 # Internal docs & brainstorming (never commit)
 # ========================
 docs/*
+!docs/OUTBOUND_DATA.md
+!docs/production-hardening.md
 !docs/mesh/
 docs/mesh/*
+!docs/mesh/threat-model.md
+!docs/mesh/claims-reconciliation.md
 !docs/mesh/mesh-canonical-fixtures.json
 !docs/mesh/mesh-merkle-fixtures.json
+!docs/mesh/wormhole-dm-root-operations-runbook.md
 .local-docs/
 infonet-economy/
 updatestuff.md
@@ -173,6 +231,74 @@ jobs.json
 .mise.local.toml
 .codex-tmp/
 prototype/
+.runtime/
 
-# Python UV lock file (regenerated from pyproject.toml)
-uv.lock
+# ========================
+# Runtime state & operator-local data (never commit)
+# ========================
+# TimeMachine snapshot cache — regenerated at runtime, can be 100 MB+
+backend/timemachine/
+# Operator witness keys, identity material, transparency ledgers (machine-local)
+ops/
+# Runtime DM relay state
+dm_relay.json
+# Dev scratch notes
+improvements.txt
+
+# ========================
+# Custody verification temp dirs (runtime test artifacts with private keys!)
+# ========================
+backend/sb-custody-verify-*/
+
+# Python egg-info (build artifact, regenerated by pip install -e)
+*.egg-info/
+
+# Privacy-core debug build (Windows DLL, 3.6 MB, not shipped)
+privacy-core/debug/
+
+# Desktop-shell export stash dirs (empty temp dirs from Tauri build)
+frontend/.desktop-export-stash-*/
+
+# Wormhole logs (can be 30 MB+ each, runtime-generated)
+backend/data/wormhole_stderr.log
+backend/data/wormhole_stdout.log
+
+# Hermes Agent (operator-local runtime install — not project source)
+.hermes/
+**/.hermes/
+hermes-agent/
+
+# Runtime caches that already slip through the backend/data/* blanket
+# (these are caught by the wildcard but listing for clarity)
+
+# Compressed snapshot archives (can be 100 MB+)
+*.json.gz
+
+# ──────────────────────────────────────────────────────────────────────
+# AI assistant / coding-agent scratch
+# ──────────────────────────────────────────────────────────────────────
+# Per-tool config + scratch directories. These are private to whichever
+# coding agent the operator happens to be using and have no business in
+# the repo. If a tool's instructions need to be canonical for the project,
+# we'll put them in docs/ explicitly — not let the agent dump them at the
+# repo root.
+
+# OpenAI Codex CLI
+.codex/
+.codex-app-schema/
+.codex-app-ts/
+
+# Per-agent instruction files dropped at repo root by various tools.
+# These are operator-side preferences, not part of the project contract.
+AGENTS.md
+GEMINI.md
+CLAUDE.md
+.github/copilot-instructions.md
+
+# Stale AI-generated test file that referenced fields that don't exist in
+# the current `_parse_carrier_positions_from_news` implementation. Kept
+# ignored so it doesn't accidentally get committed if it shows up again
+# from a tool that's working off an out-of-date understanding of the
+# module. If a real test for that function is needed, write it under a
+# meaningful name in tests/test_carrier_tracker_quality.py.
+backend/tests/test_carrier_tracker_region_centers.py

.gitlab-ci.yml

@@ -0,0 +1,151 @@
+# GitLab CI/CD for Shadowbroker
+#
+# Mirror of .github/workflows/docker-publish.yml — keeps the GitLab install
+# path (image registry + source) at parity with GitHub so users who prefer
+# GitLab get the same experience.
+#
+# What this does on every push to main:
+#   1. Builds multi-arch (amd64 + arm64) Docker images for the backend and
+#      frontend, pushes them to the project's GitLab Container Registry:
+#        registry.gitlab.com/bigbodycobain/shadowbroker/backend:latest
+#        registry.gitlab.com/bigbodycobain/shadowbroker/frontend:latest
+#      Both also get a :$CI_COMMIT_SHORT_SHA tag for traceability.
+#   2. Reverse-mirrors main back to GitHub (only if commits land directly
+#      on GitLab) so the two sources stay in sync.
+#
+# Pipelines on this repo were instant-failing for free-tier accounts until
+# identity verification was added — the May 2026 bump in this comment is
+# the marker commit that confirms runner allocation after verification.
+#
+# Auth notes:
+#   - The image build/push uses $CI_JOB_TOKEN, which GitLab provides
+#     automatically. No credentials need to be configured.
+#   - The reverse mirror authenticates to GitHub via a per-repo SSH
+#     deploy key. The private half is stored as the File-type GitLab
+#     CI/CD variable GITHUB_MIRROR_SSH_KEY (Protected). The matching
+#     public key is added to github.com/BigBodyCobain/Shadowbroker/
+#     settings/keys with write access. This is a tighter-scoped
+#     replacement for a personal access token: it can ONLY push to
+#     Shadowbroker, never expires, and rotating it is a one-click
+#     delete on GitHub's deploy-keys page. If the variable isn't set,
+#     the mirror job is skipped — image builds still run.
+
+stages:
+  - build
+  - mirror
+
+variables:
+  # Use the dind service for buildx multi-arch builds.
+  DOCKER_HOST: tcp://docker:2376
+  DOCKER_TLS_CERTDIR: "/certs"
+  DOCKER_DRIVER: overlay2
+  # QEMU is what lets a single x86 runner build arm64 images. dind doesn't
+  # install it by default; we install via tonistiigi/binfmt below.
+  BUILDX_VERSION: "v0.14.1"
+  # Repository-relative paths.
+  BACKEND_IMAGE: $CI_REGISTRY_IMAGE/backend
+  FRONTEND_IMAGE: $CI_REGISTRY_IMAGE/frontend
+
+# Shared template: bootstraps buildx + QEMU on the dind service so a single
+# runner can produce both amd64 and arm64 manifests in one push.
+.buildx-setup: &buildx-setup
+  image: docker:24
+  services:
+    - name: docker:24-dind
+      command: ["--tls=true"]
+  before_script:
+    - docker info
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
+    - docker run --privileged --rm tonistiigi/binfmt --install all
+    # buildx --driver docker-container can't read TLS from the env vars
+    # the GitLab dind service exports. Wrap them in a docker context and
+    # bind buildx to it. See https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-buildx
+    - docker context create tls-env
+    - docker buildx create --use --name multiarch --driver docker-container tls-env
+
+# ── Backend image ────────────────────────────────────────────────────────
+build-backend:
+  <<: *buildx-setup
+  stage: build
+  script:
+    - >
+      docker buildx build
+      --platform linux/amd64,linux/arm64
+      --file backend/Dockerfile
+      --tag $BACKEND_IMAGE:latest
+      --tag $BACKEND_IMAGE:$CI_COMMIT_SHORT_SHA
+      --push
+      .
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+        - backend/**/*
+        - .gitlab-ci.yml
+
+# ── Frontend image ───────────────────────────────────────────────────────
+build-frontend:
+  <<: *buildx-setup
+  stage: build
+  script:
+    - cd frontend
+    - >
+      docker buildx build
+      --platform linux/amd64,linux/arm64
+      --tag $FRONTEND_IMAGE:latest
+      --tag $FRONTEND_IMAGE:$CI_COMMIT_SHORT_SHA
+      --push
+      .
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+        - frontend/**/*
+        - .gitlab-ci.yml
+
+# ── Reverse mirror to GitHub ─────────────────────────────────────────────
+# Pushes refs/heads/main to github.com/BigBodyCobain/Shadowbroker via SSH
+# using a per-repo deploy key. Fast-forward-only by default — if GitLab
+# main and GitHub main have diverged, the push fails loudly rather than
+# silently overwriting either side.
+#
+# Only runs if GITHUB_MIRROR_SSH_KEY is set as a File-type CI/CD variable.
+# See the header comment of this file for setup instructions.
+mirror-to-github:
+  stage: mirror
+  image: alpine:3.20
+  needs: []
+  before_script:
+    - apk add --no-cache git openssh-client ca-certificates
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    # Install the deploy key. File-type CI variable exposes the path; copy
+    # to ~/.ssh/id_ed25519 with restrictive perms so ssh accepts it.
+    - cp "$GITHUB_MIRROR_SSH_KEY" ~/.ssh/id_ed25519
+    - chmod 600 ~/.ssh/id_ed25519
+    # Pin github.com's current host keys so we never trust a man-in-the-
+    # middle. Sourced from https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints
+    # (rotated 2023-03-24 after the previous RSA key leak).
+    - |
+      cat > ~/.ssh/known_hosts <<'EOF'
+      github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
+      github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
+      github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
+      EOF
+    - chmod 644 ~/.ssh/known_hosts
+  script:
+    - git config --global user.email "ci-mirror@gitlab.com"
+    - git config --global user.name "GitLab CI Mirror"
+    - >
+      git clone --depth=50 --branch main
+      "https://oauth2:${CI_JOB_TOKEN}@gitlab.com/${CI_PROJECT_PATH}.git"
+      repo
+    - cd repo
+    - >
+      git push
+      "git@github.com:BigBodyCobain/Shadowbroker.git"
+      "${CI_COMMIT_SHA}:refs/heads/main"
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $GITHUB_MIRROR_SSH_KEY

.pre-commit-config.yaml

@@ -1,4 +1,12 @@
 repos:
+  - repo: local
+    hooks:
+      - id: shadowbroker-secret-scan
+        name: ShadowBroker secret scan
+        entry: bash backend/scripts/scan-secrets.sh --staged
+        language: system
+        pass_filenames: false
+
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.6.0
     hooks:

CONTRIBUTING.md

@@ -0,0 +1,75 @@
+# Contributing to Shadowbroker
+
+Thank you for taking the time to contribute. This document covers things specific to this project — for general open-source contribution etiquette, see the GitHub docs.
+
+---
+
+## Code contributions
+
+1. Fork the repo on GitHub (`bigbodycobain/Shadowbroker`) or GitLab (`bigbodycobain/Shadowbroker` mirror).
+2. Make your changes on a feature branch.
+3. Run the local test suite:
+   - Backend: `pytest backend/tests/`
+   - Frontend: `cd frontend && npx vitest run`
+4. Open a Pull Request against `main`.
+
+CI runs on every PR. If CI fails, that's blocking — please push fixes rather than asking for it to be merged anyway.
+
+---
+
+## Reporting security issues
+
+Do **not** file security issues as public GitHub issues. Email the maintainer or use a private security advisory on GitHub. Public disclosure of an exploitable vulnerability without prior coordination will be rejected from the project.
+
+---
+
+## Translation contributions
+
+Shadowbroker supports UI localization (`frontend/src/i18n/`). Translation contributions are welcome but held to a stricter standard than most code changes, because translations can subtly reshape user perception in ways that are hard to spot during review. Read this section before submitting one.
+
+### The neutrality requirement
+
+**Translations must be technically faithful to the English source.** That means:
+
+- Each `t('key')` entry should mean approximately the same thing in the target language as in English, modulo idiom.
+- Technical terms with established meanings (e.g. "GPS jamming," "military flight," "Tor," "onion routing," "encryption") should be translated using the corresponding established technical term in the target language — **not** softened, rebranded, or politically reframed.
+- The set of UI strings should be **the same** between languages. Don't omit features from one locale that are visible in another.
+
+### What will get a translation PR rejected
+
+Translation choices that align the project with the framing or terminology of state propaganda — from **any** country — will be rejected. This applies symmetrically:
+
+| Country / source | Examples of substitutions we will reject |
+|---|---|
+| **PRC / CCP** | Calling Taiwan a "province" or "renegade province"; reframing protest layers as "riots"; using softened or euphemistic terms for surveillance, internment, or jamming when the source text is direct |
+| **Russia** | Calling the Ukraine war a "special military operation"; relabeling occupied territories as Russian; softening sanctions/jamming/disinfo terminology |
+| **United States / EU** | Reframing adversaries with editorial labels not in the source (e.g. inserting "regime" where the English says "government"); applying labels like "terrorist" or "rogue state" to entities the English source describes neutrally |
+| **Israel / Palestine / any active conflict** | Substituting one side's preferred terminology when the source uses the other side's or a neutral term |
+| **Any government** | Adding political slogans, omitting features that government finds inconvenient, or inserting terminology associated with a specific political faction |
+
+The test is **"would a translator working strictly from the English source produce this rendering?"** If the answer requires assuming a political stance the source does not take, the substitution does not belong in the translation.
+
+### How translation PRs are reviewed
+
+Changes to `frontend/src/i18n/**` are owned by the maintainer (see `CODEOWNERS`) and require explicit approval. We will:
+
+1. Diff the translation against the English source key-by-key.
+2. Spot-check a sample of entries with a native speaker of the target language when possible.
+3. Look for the patterns above.
+4. Look for suspicious additions to the i18n infrastructure itself (e.g. a remote translation fetcher, telemetry on language choice) — the i18n layer is supposed to be 100% client-side static JSON.
+
+A PR that adds a new language is harder to review than one that fixes typos in an existing language. For new languages, please be patient and expect a real review window. For typo fixes, please describe each change in the PR body so the reviewer can verify intent.
+
+### What about adding a new language?
+
+We welcome new languages. The mechanical setup is documented in the header comment of `frontend/src/i18n/index.ts`. Beyond that:
+
+- We are more likely to merge a new language quickly if at least one reviewer in the maintainer's network speaks it.
+- If you are the *only* speaker of the target language reading this repo, your translation is welcome but the merge timeline will be longer while a reviewer is found.
+- Partial translations are fine — the system falls back to English for any missing key.
+
+---
+
+## Anything else
+
+If you have a question that isn't a security report, opening a GitHub Discussion or a draft PR with a question in the body is the fastest way to get a response. Direct emails are read but not always replied to promptly.

DATA-ATTRIBUTION.md

@@ -0,0 +1,72 @@
+# Data Attribution & Licensing
+
+ShadowBroker aggregates publicly available data from many third-party sources.
+This file documents each source and its license so operators and users can
+comply with the terms under which we access that data.
+
+ShadowBroker itself is licensed under AGPL-3.0 (see `LICENSE`). **This file
+concerns the *data* rendered by the dashboard, not the source code.**
+
+---
+
+## ODbL-licensed sources (Open Database License v1.0)
+
+Data from these sources is licensed under the
+[Open Database License v1.0](https://opendatacommons.org/licenses/odbl/1-0/).
+If you redistribute a derivative database built from these sources, the
+derivative must also be offered under ODbL and must preserve attribution.
+
+| Source | URL | What we use it for |
+|---|---|---|
+| adsb.lol | https://adsb.lol | Military aircraft positions, regional commercial gap-fill, route enrichment |
+| OpenStreetMap contributors | https://www.openstreetmap.org/copyright | Nominatim geocoding (LOCATE bar), CARTO basemap tiles (OSM-derived) |
+
+**Attribution requirement:** the ShadowBroker map UI displays
+"© OpenStreetMap contributors" and "adsb.lol (ODbL)" in the map attribution
+control. Do not remove this attribution if you fork or redistribute the app.
+
+---
+
+## Other third-party data sources
+
+These sources have their own terms; consult each link before redistributing.
+
+| Source | URL | License / Terms | Notes |
+|---|---|---|---|
+| OpenSky Network | https://opensky-network.org | OpenSky API terms | Commercial and private aircraft tracking |
+| CelesTrak | https://celestrak.org | Public domain / no restrictions | Satellite TLE data |
+| USGS Earthquake Hazards | https://earthquake.usgs.gov | Public domain (US Federal) | Seismic events |
+| NASA FIRMS | https://firms.modaps.eosdis.nasa.gov | NASA Open Data | Fire/thermal anomalies (VIIRS) |
+| NASA GIBS | https://gibs.earthdata.nasa.gov | NASA Open Data | MODIS imagery tiles |
+| NOAA SWPC | https://services.swpc.noaa.gov | Public domain (US Federal) | Space weather, Kp index |
+| GDELT Project | https://www.gdeltproject.org | CC BY (non-commercial friendly) | Global conflict events |
+| DeepState Map | https://deepstatemap.live | Per-site terms | Ukraine frontline GeoJSON |
+| aisstream.io | https://aisstream.io | Free-tier API terms (attribution required) | AIS vessel positions |
+| Global Fishing Watch | https://globalfishingwatch.org | CC BY 4.0 (for public data) | Fishing activity events |
+| Microsoft Planetary Computer | https://planetarycomputer.microsoft.com | Sentinel-2 / ESA Copernicus terms | Sentinel-2 imagery |
+| Copernicus CDSE (Sentinel Hub) | https://dataspace.copernicus.eu | ESA Copernicus open data terms | SAR + optical imagery, optional road-corridor truck trends |
+| DrishX / Fisser et al. 2022 | https://github.com/sparkyniner/DRISH-X-Satellite-powered-freight-intelligence- | MIT (engine); research methodology attribution | Sentinel-2 motion-smear truck detection on major roads (opt-in) |
+| Shodan | https://www.shodan.io | Operator-supplied API key, Shodan ToS | Internet device search |
+| Smithsonian GVP | https://volcano.si.edu | Attribution required | Volcanoes |
+| OpenAQ | https://openaq.org | CC BY 4.0 | Air quality stations |
+| NOAA NWS | https://www.weather.gov | Public domain (US Federal) | Severe weather alerts |
+| WRI Global Power Plant DB | https://datasets.wri.org | CC BY 4.0 | Power plants |
+| Wikidata | https://www.wikidata.org | CC0 | Head-of-state lookup |
+| Wikipedia | https://en.wikipedia.org | CC BY-SA 4.0 | Region summaries |
+| KiwiSDR (via dyatlov mirror) | http://rx.linkfanel.net | Per-site terms (community mirror by Pierre Ynard) | SDR receiver list — pulled from rx.linkfanel.net to keep load off jks-prv's bandwidth at kiwisdr.com |
+| OpenMHZ | https://openmhz.com | Per-site terms | Police/fire scanner feeds |
+| Meshtastic | https://meshtastic.org | Open Source | Mesh radio nodes (protocol) |
+| Meshtastic Map (Liam Cottle) | https://meshtastic.liamcottle.net | Community project (per-site terms) | Global Meshtastic node positions — polled once per day with on-disk cache trust to minimize load on this volunteer-run HTTP API |
+| APRS-IS | https://www.aprs-is.net | Open / attribution-based | Amateur radio positions |
+| CARTO basemaps | https://carto.com | CARTO attribution required | Dark map tiles (OSM-derived) |
+| Esri World Imagery | https://www.arcgis.com | Esri terms | High-res satellite basemap |
+| IODA (Georgia Tech) | https://ioda.inetintel.cc.gatech.edu | Research/academic terms | Internet outage data |
+
+---
+
+## Contact
+
+If you represent a data provider and have concerns about how ShadowBroker
+uses your data, please open an issue or contact the maintainer at
+`bigbodycobain@gmail.com`. We will respond promptly and, if needed, adjust
+usage or remove the source.

Mesh.md

@@ -0,0 +1,89 @@
+# ShadowBroker — Meshtastic MQTT Remediation
+
+**Version:** 0.9.6  
+**Date:** 2026-04-12  
+**Re:** [meshtastic/firmware#6131](https://github.com/meshtastic/firmware/issues/6131) — Excessive MQTT traffic from ShadowBroker clients
+
+---
+
+## What happened
+
+ShadowBroker is an open-source OSINT situational awareness platform that includes a Meshtastic MQTT listener for displaying mesh network activity on a global map. In prior versions, the MQTT bridge:
+
+- Subscribed to **28 wildcard topics** (`msh/{region}/#`) covering every known official and community root on startup
+- Used an aggressive reconnect policy (min 1s / max 30s backoff)
+- Set keepalive to 30 seconds
+- Had no client-side rate limiting on inbound messages
+- Auto-started on every launch with no opt-out
+
+This produced 1-2 orders of magnitude more traffic than typical Meshtastic clients on the public broker at `mqtt.meshtastic.org`.
+
+---
+
+## What we fixed
+
+### 1. Bridge disabled by default
+
+The MQTT bridge no longer starts automatically. Operators must explicitly opt in:
+
+```env
+MESH_MQTT_ENABLED=true
+```
+
+### 2. US-only default subscription
+
+When enabled, the bridge subscribes to **1 topic** (`msh/US/#`) instead of 28. Additional regions are opt-in:
+
+```env
+MESH_MQTT_EXTRA_ROOTS=EU_868,ANZ
+```
+
+The UI still displays all regions in its dropdown — only the MQTT subscription scope changed.
+
+### 3. Client-side rate limiter
+
+Inbound messages are capped at **100 messages per minute** using a sliding window. Excess messages are silently dropped. A warning is logged periodically when the limiter activates so operators are aware.
+
+### 4. Conservative connection parameters
+
+| Parameter | Before | After |
+|-----------|--------|-------|
+| Keepalive | 30s | 120s |
+| Reconnect min delay | 1s | 15s |
+| Reconnect max delay | 30s | 300s |
+| QoS | 0 | 0 (unchanged) |
+
+### 5. Versioned client ID
+
+Client IDs changed from `sbmesh-{uuid}` to `sb096-{uuid}` so the Meshtastic team can identify ShadowBroker clients and track adoption of the fix by version.
+
+---
+
+## Configuration reference
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `MESH_MQTT_ENABLED` | `false` | Master switch for the MQTT bridge |
+| `MESH_MQTT_EXTRA_ROOTS` | _(empty)_ | Comma-separated additional region roots (e.g. `EU_868,ANZ,JP`) |
+| `MESH_MQTT_INCLUDE_DEFAULT_ROOTS` | `true` | Include US in subscriptions |
+| `MESH_MQTT_BROKER` | `mqtt.meshtastic.org` | Broker hostname |
+| `MESH_MQTT_PORT` | `1883` | Broker port |
+| `MESH_MQTT_USER` | `meshdev` | Broker username |
+| `MESH_MQTT_PASS` | `large4cats` | Broker password |
+| `MESH_MQTT_PSK` | _(empty)_ | Hex-encoded PSK (empty = default LongFast key) |
+
+---
+
+## Files changed
+
+- `backend/services/config.py` — Added `MESH_MQTT_ENABLED` flag
+- `backend/services/mesh/meshtastic_topics.py` — Reduced default roots to US-only
+- `backend/services/sigint_bridge.py` — Rate limiter, keepalive/backoff tuning, versioned client ID, opt-in gate
+- `backend/.env.example` — Documented all MQTT options
+
+---
+
+## Contact
+
+Repository: [github.com/BigBodyCobain/Shadowbroker](https://github.com/BigBodyCobain/Shadowbroker)  
+Maintainer: BigBodyCobain

backend/.env.example

@@ -11,19 +11,107 @@ AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
 
 # ── Optional ───────────────────────────────────────────────────
 
+# AISHub REST fallback. Used when stream.aisstream.io is unreachable
+# (e.g. their cert expires or server goes offline). Free tier requires
+# registration at https://www.aishub.net/api. Poll cadence defaults to
+# 20 min to stay courteous; tunable via AISHUB_POLL_INTERVAL_MINUTES.
+# AISHUB_USERNAME=
+# AISHUB_POLL_INTERVAL_MINUTES=20
+
+# `python main.py` (uvicorn reload) binds 127.0.0.1:8000 by default so LAN clients
+# cannot reach a dev server with empty ADMIN_KEY (#375). Set true only when you
+# intentionally need 0.0.0.0 and understand the local-trust implications.
+# SHADOWBROKER_DEV_BIND_ALL=false
+#
+# Thread pool for GDELT, LiveUAMap, CCTV ingest, and slow-tier refresh batches.
+# Keeps heavy jobs from starving fast flight/ship workers (default 2).
+# SHADOWBROKER_HEAVY_FETCH_WORKERS=2
+
 # Override allowed CORS origins (comma-separated). Defaults to localhost + LAN auto-detect.
 # CORS_ORIGINS=http://192.168.1.50:3000,https://my-domain.com
 
 # Admin key — protects sensitive endpoints (API key management, system update).
-# If unset, endpoints are only accessible from localhost unless ALLOW_INSECURE_ADMIN=true.
+# If unset, loopback/localhost requests still work for local single-host dev.
+# Remote/non-loopback admin access requires ADMIN_KEY, or ALLOW_INSECURE_ADMIN=true in debug-only setups.
 # Set this in production and enter the same key in Settings → Admin Key.
 # ADMIN_KEY=your-secret-admin-key-here
 
-# Allow insecure admin access without ADMIN_KEY (local dev only).
+# Allow insecure admin access without ADMIN_KEY (local dev only, beyond loopback).
+# Requires MESH_DEBUG_MODE=true; do not enable this for ordinary use.
 # ALLOW_INSECURE_ADMIN=false
 
-# User-Agent for Nominatim geocoding requests (per OSM usage policy).
-# NOMINATIM_USER_AGENT=ShadowBroker/1.0 (https://github.com/BigBodyCobain/Shadowbroker)
+# Per-install operator handle. Round 7a: outbound third-party API calls send
+# this handle as the User-Agent (e.g. operator-7f3a92), not a shared app name,
+# so upstreams rate-limit one install instead of blocking every user.
+#
+# Default empty -> a stable pseudonymous handle (e.g. "operator-7f3a92") is
+# auto-generated on first run and persisted to backend/data/operator_handle.json.
+# Operators who want a meaningful handle (real name, org, GitHub login) can
+# set it here. Special characters are sanitized to dashes.
+# OPERATOR_HANDLE=
+
+# Full User-Agent override (replaces the operator handle entirely). Rare;
+# most installs should use OPERATOR_HANDLE only.
+# SHADOWBROKER_USER_AGENT=
+
+# Nominatim-specific User-Agent override (OSM usage policy). Leave unset to
+# use the per-install handle (default) — set only if you have a registered
+# Nominatim relay identity.
+# NOMINATIM_USER_AGENT=
+
+# ── Third-party fetcher opt-ins ────────────────────────────────
+# These data sources phone home to politically/commercially sensitive
+# upstreams. Disabled by default; set to "true" only if the operator
+# explicitly wants the node's IP to contact these services.
+#
+# CrowdThreat — backend.crowdthreat.world (paid threat-intel aggregator).
+# CROWDTHREAT_ENABLED=false
+#
+# EUvsDisinfo FIMI — euvsdisinfo.eu (EU disinformation tracker).
+# FIMI_ENABLED=false
+#
+# Polymarket + Kalshi — US political/election prediction markets.
+# Default off; enable from Global Threat Intercept (MKT toggle) or set true here.
+# PREDICTION_MARKETS_ENABLED=false
+# When enabled, polls use a jittered schedule (not the fixed 5-minute slow tier):
+# PREDICTION_MARKETS_INTERVAL_MINUTES=7
+# PREDICTION_MARKETS_SCHEDULER_JITTER_S=240
+# PREDICTION_MARKETS_INITIAL_DELAY_MAX_S=180
+# PREDICTION_MARKETS_PRE_FETCH_JITTER_S=90
+# PREDICTION_MARKETS_PROVIDER_GAP_JITTER_S=45
+# MESH_POLYMARKET_PAGE_DELAY_JITTER_S=0.08
+# MESH_KALSHI_PAGE_DELAY_JITTER_S=0.2
+#
+# Finnhub fallback / yfinance — financial market data.
+# Set FINNHUB_API_KEY to enable Finnhub, or set FINANCIAL_ENABLED=true to allow
+# the unauthenticated yfinance fallback to call Yahoo Finance.
+# FINANCIAL_ENABLED=false
+#
+# NUFORC UAP map layer — live scrape from nuforc.org (rolling window, default 60 days).
+# Refreshed weekly (Mon 12:00 UTC); cache reused for up to 7 days between runs.
+# NUFORC_RECENT_DAYS=60
+# NUFORC_CACHE_TTL_HOURS=168
+# On Windows, live scrape uses Python requests by default; optional:
+# SHADOWBROKER_ENABLE_WINDOWS_CURL_FALLBACK=true
+# NUFORC enrichment index (HF dataset) is separate — opt-in only:
+# NUFORC_ENABLED=false
+#
+# News RSS aggregator — defaults ON. Set to "false" to disable all
+# configured news feeds (kill switch for the news layer).
+# NEWS_ENABLED=true
+
+# Global Fishing Watch — fishing vessel activity events (Fishing Activity map layer).
+# Free API token from https://globalfishingwatch.org/our-apis/tokens
+# Without this the fishing_activity layer stays empty.
+# GFW_API_TOKEN=
+# Optional tuning — GFW can return 40k+ global events; defaults cap fetch for map paint.
+# GFW_EVENTS_PAGE_SIZE=500
+# GFW_EVENTS_MAX_PAGES=10
+# GFW_EVENTS_LOOKBACK_DAYS=7
+# GFW_EVENTS_TIMEOUT_S=90
+
+# Windy Webcams global CCTV layer — free key from https://api.windy.com/webcams/docs
+# WINDY_API_KEY=
 
 # LTA Singapore traffic cameras — leave blank to skip this data source.
 # LTA_ACCOUNT_KEY=
@@ -32,22 +120,138 @@ AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
 # Free MAP_KEY from https://firms.modaps.eosdis.nasa.gov/map/#d:24hrs;@0.0,0.0,3.0z
 # FIRMS_MAP_KEY=
 
+# Ukraine frontline mirror (GitHub). Default follows cyterat/deepstate-map-data@main.
+# Pin an immutable commit SHA so ingest cannot silently change if main is force-pushed (#362).
+# Example (verify on GitHub before use): main @ b479954e94696bc5622c7818fd20a64a699f4fe8
+# DEEPSTATE_MIRROR_COMMIT=b479954e94696bc5622c7818fd20a64a699f4fe8
+# DEEPSTATE_MIRROR_REPO=cyterat/deepstate-map-data
+
 # Ukraine air raid alerts from alerts.in.ua — free token from https://alerts.in.ua/
 # ALERTS_IN_UA_TOKEN=
 
+# Optional NUFORC UAP sighting map enrichment via Mapbox Tilequery.
+# Leave blank to skip this optional enrichment.
+# NUFORC_MAPBOX_TOKEN=
+
 # Google Earth Engine service account for VIIRS change detection (optional).
 # Download JSON key from https://console.cloud.google.com/iam-admin/serviceaccounts
 # pip install earthengine-api
 # GEE_SERVICE_ACCOUNT_KEY=
 
+# ── Meshtastic MQTT Bridge ─────────────────────────────────────
+# Disabled by default to respect the public Meshtastic broker.
+# When enabled, subscribes to US region only. Add more regions via MESH_MQTT_EXTRA_ROOTS.
+# MESH_MQTT_ENABLED=false
+# MESH_MQTT_EXTRA_ROOTS=EU_868,ANZ    # comma-separated additional region roots
+# MESH_MQTT_INCLUDE_DEFAULT_ROOTS=true
+# MESH_MQTT_BROKER=mqtt.meshtastic.org
+# MESH_MQTT_PORT=1883
+# Leave user/pass blank for the public Meshtastic broker default.
+# MESH_MQTT_USER=
+# MESH_MQTT_PASS=
+
+# Optional Meshtastic node ID (e.g. "!abcd1234"). When set, included in the
+# User-Agent sent to meshtastic.liamcottle.net so the upstream service operator
+# can identify per-install traffic instead of aggregated "ShadowBroker" hits.
+# Leave blank to send a generic UA. If you set MESHTASTIC_OPERATOR_CALLSIGN,
+# it is included in outbound headers to meshtastic.org by default so they
+# can rate-limit per-operator. Callsign is NOT sent upstream unless you opt in.
+# MESHTASTIC_OPERATOR_CALLSIGN=
+# MESHTASTIC_SEND_CALLSIGN_HEADER=false
+# MESH_MQTT_PSK=                       # hex-encoded, empty = default LongFast key
+
+# LiveUAMap Playwright scraper (#348). Linux/macOS: on by default when Global
+# Incidents layer is active. Windows: off until the operator enables Global
+# Incidents in the UI (consent dialog) or sets SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=true.
+# SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=false forces off on all platforms.
+
 # ── Mesh / Reticulum (RNS) ─────────────────────────────────────
 # Full-node / participant-node posture for public Infonet sync.
 # MESH_NODE_MODE=participant      # participant | relay | perimeter
+# Legacy compatibility sunset toggles. Default posture is to block these.
+# Legacy 16-hex node-id binding no longer has a boolean escape hatch; use a
+# dated migration override only when you intentionally need older peers during
+# migration before the hard removal target in v0.10.0 / 2026-06-01.
+# MESH_BLOCK_LEGACY_NODE_ID_COMPAT=true
+# MESH_ALLOW_LEGACY_NODE_ID_COMPAT_UNTIL=2026-05-15
+# MESH_BLOCK_LEGACY_AGENT_ID_LOOKUP=true
+# Temporary DM invite migration escape hatch. Default posture blocks importing
+# legacy/compat v1/v2 DM invites; use a dated override only while retiring
+# older exports and ask senders to re-export a current signed invite.
+# MESH_ALLOW_COMPAT_DM_INVITE_IMPORT_UNTIL=2026-05-15
+# Temporary legacy GET DM poll/count escape hatch. Default posture requires the
+# signed mailbox-claim POST APIs; only use this dated override while retiring
+# older clients that still call GET poll/count directly.
+# MESH_ALLOW_LEGACY_DM_GET_UNTIL=2026-05-15
+# Temporary raw dm1 compose/decrypt escape hatch. Default posture expects MLS
+# DM bootstrap on supported peers; only use this dated override while retiring
+# older clients that still need the raw dm1 helper path.
+# MESH_ALLOW_LEGACY_DM1_UNTIL=2026-05-15
+# Temporary legacy dm_message signature escape hatch. Default posture requires
+# the full modern signed payload; only enable this with a dated migration
+# override while older senders are being retired.
+# MESH_ALLOW_LEGACY_DM_SIGNATURE_COMPAT_UNTIL=2026-05-15
+# Rotate voter-blinding salts so new reputation events stop reusing one
+# forever-stable blinded ID. Keep grace >= rotation cadence so older votes
+# remain matchable while they age out of the ledger.
+# MESH_VOTER_BLIND_SALT_ROTATE_DAYS=30
+# MESH_VOTER_BLIND_SALT_GRACE_DAYS=30
+# Deprecated legacy env vars kept only for backward config compatibility.
+# Ordinary shipped gate flows keep MLS decrypt local; service-side decrypt is
+# reserved for explicit recovery reads.
+# MESH_GATE_BACKEND_DECRYPT_COMPAT=false
+# MESH_GATE_BACKEND_DECRYPT_COMPAT_ACKNOWLEDGE=false
+# Deprecated legacy env vars kept only for backward config compatibility.
+# Ordinary shipped gate flows keep plaintext compose/post local and only submit
+# encrypted envelopes to the backend for sign/post.
+# MESH_GATE_BACKEND_PLAINTEXT_COMPAT=false
+# MESH_GATE_BACKEND_PLAINTEXT_COMPAT_ACKNOWLEDGE=false
+# Legacy runtime switches for recovery envelopes. Per-gate envelope_policy is
+# the source of truth; leave these at the default unless testing old behavior.
+# MESH_GATE_RECOVERY_ENVELOPE_ENABLE=true
+# MESH_GATE_RECOVERY_ENVELOPE_ENABLE_ACKNOWLEDGE=true
+# Optional operator-only recovery tradeoff. Leave off for the default posture:
+# ordinary gate reads keep plaintext local/in-memory unless you explicitly use
+# the recovery-envelope path.
+# MESH_GATE_PLAINTEXT_PERSIST=false
+# MESH_GATE_PLAINTEXT_PERSIST_ACKNOWLEDGE=false
+# Legacy Phase-1 gate envelope fallback is now explicit and time-bounded per
+# gate. This only controls the default expiry window when you deliberately
+# re-enable that migration path for older stored envelopes.
+# MESH_GATE_LEGACY_ENVELOPE_FALLBACK_MAX_DAYS=30
+# Feature-flagged multiplexed gate session stream. Stream-first room ownership
+# is implemented; keep off until you want that rollout enabled in your env.
+# MESH_GATE_SESSION_STREAM_ENABLED=false
+# MESH_GATE_SESSION_STREAM_HEARTBEAT_S=20
+# MESH_GATE_SESSION_STREAM_BATCH_MS=1500
+# MESH_GATE_SESSION_STREAM_MAX_GATES=16
 # MESH_BOOTSTRAP_DISABLED=false
 # MESH_BOOTSTRAP_MANIFEST_PATH=data/bootstrap_peers.json
-# MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY=
-# MESH_RELAY_PEERS=               # comma-separated operator-trusted sync/push peers
-# MESH_PEER_PUSH_SECRET=Mv63UvLfwqOEVWeRBXjA8MtFl2nEkkhUlLYVHiX1Zzo   # transport auth for mesh peer push (default works out of the box)
+# Swarm discovery (signed peer manifest). Participants need only the public key;
+# the seed operator sets MESH_BOOTSTRAP_SIGNER_PRIVATE_KEY (never commit it).
+# Generate a fleet keypair: uv run python backend/scripts/bootstrap_manifest_helper.py generate-keypair
+# Public sb-testnet fleet defaults (auto-used when MESH_INFONET_FLEET_JOIN=true).
+# MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY=ul1d0kj/ODPIp0OhHzX8eLAVXzJ3CVvzW1vn2IC6q3I=
+# MESH_INFONET_FLEET_JOIN=true
+# MESH_INFONET_FLEET_JOIN_DISABLED=false
+# MESH_BOOTSTRAP_SIGNER_PRIVATE_KEY=   # seed only
+# MESH_BOOTSTRAP_SIGNER_ID=shadowbroker-seed
+# MESH_PEER_REGISTRY_ENABLED=true      # seed only (auto-enabled when private key is set)
+# Headless relay compose sets MESH_INFONET_RELAY_AUTO_WORMHOLE=true; seed nodes with
+# MESH_BOOTSTRAP_SIGNER_PRIVATE_KEY also auto-enable Tor wormhole on startup.
+# MESH_INFONET_RELAY_AUTO_WORMHOLE=false
+# MESH_INFONET_RELAY_AUTO_WORMHOLE_DISABLED=false
+# MESH_SWARM_MANIFEST_TTL_S=14400
+# MESH_SWARM_MANIFEST_PULL_INTERVAL_S=300
+# MESH_PEER_REGISTRY_STALE_S=604800
+# Infonet/Wormhole fails closed to onion/RNS by default. Only enable clearnet
+# sync for local relay development or an explicitly public testnet.
+# MESH_INFONET_ALLOW_CLEARNET_SYNC=false
+# MESH_BOOTSTRAP_SEED_PEERS=http://gqpbunqbgtkcqilvclm3xrkt3zowjyl3s62kkktvojgvxzizamvbrqid.onion:8000
+# Add comma-separated http://*.onion peers as more private seed/relay nodes come online.
+# MESH_DEFAULT_SYNC_PEERS=         # legacy alias; prefer MESH_BOOTSTRAP_SEED_PEERS
+# MESH_RELAY_PEERS=               # comma-separated operator-trusted sync/push peers (empty by default)
+# MESH_PEER_PUSH_SECRET=          # REQUIRED when relay/RNS peers are configured (min 16 chars, generate with: python -c "import secrets; print(secrets.token_urlsafe(32))")
 # MESH_SYNC_INTERVAL_S=300
 # MESH_SYNC_FAILURE_BACKOFF_S=60
 #
@@ -90,8 +294,54 @@ AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
 # MESH_VERIFY_INTERVAL_S=600
 # MESH_VERIFY_SIGNATURES=false
 
+# ── Secure Storage (non-Windows) ───────────────────────────────
+# Required on Linux/Docker to protect Wormhole key material at rest.
+# Generate with: python -c "import secrets; print(secrets.token_urlsafe(32))"
+# Also supports Docker secrets via MESH_SECURE_STORAGE_SECRET_FILE.
+# MESH_SECURE_STORAGE_SECRET=
+#
+# To rotate the storage secret, stop the backend and run:
+#   1. Dry-run first (validates without writing):
+#      MESH_OLD_STORAGE_SECRET=<current> MESH_NEW_STORAGE_SECRET=<new> \
+#        python -m scripts.rotate_secure_storage_secret --dry-run
+#   2. Rotate (creates .bak backups, then rewraps envelopes):
+#      MESH_OLD_STORAGE_SECRET=<current> MESH_NEW_STORAGE_SECRET=<new> \
+#        python -m scripts.rotate_secure_storage_secret
+#   3. Update MESH_SECURE_STORAGE_SECRET to the new value and restart.
+#
+# If rotation is interrupted, .bak files preserve the old envelopes.
+# To repair corrupted secure-json payloads (not key envelopes), use:
+#   python -m scripts.repair_wormhole_secure_storage
+
 # ── Mesh DM Relay ──────────────────────────────────────────────
 # MESH_DM_TOKEN_PEPPER=change-me
+# Keep DM relay metadata retention explicit and bounded.
+# MESH_DM_KEY_TTL_DAYS=30
+# MESH_DM_PREKEY_LOOKUP_ALIAS_TTL_DAYS=14
+# MESH_DM_WITNESS_TTL_DAYS=14
+# MESH_DM_BINDING_TTL_DAYS=3
+# Optional operational bridge for externally sourced root witnesses / transparency.
+# Relative paths resolve from the backend directory.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH=data/root_witness_import.json
+# Local single-host dev example after bootstrapping an external witness locally:
+# MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH=../ops/root_witness_receipt_import.json
+# Optional URI bridge for externally retrieved root witness packages.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_URI=file:///absolute/path/root_witness_import.json
+# Maximum acceptable age for external witness packages before strong DM trust fails closed.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_MAX_AGE_S=3600
+# Warning threshold for external witness packages before fail-closed max age.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_WARN_AGE_S=2700
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH=data/root_transparency_ledger.json
+# Local single-host dev example after publishing the transparency ledger locally:
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH=../ops/root_transparency_ledger.json
+# Optional URI used to read back and verify a published transparency ledger.
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI=file:///absolute/path/root_transparency_ledger.json
+# Local single-host dev readback example:
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI=../ops/root_transparency_ledger.json
+# Maximum acceptable age for external transparency ledgers before strong DM trust fails closed.
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_MAX_AGE_S=3600
+# Warning threshold for external transparency ledgers before fail-closed max age.
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_WARN_AGE_S=2700
 
 # ── Self Update ────────────────────────────────────────────────
 # MESH_UPDATE_SHA256=
@@ -103,3 +353,69 @@ AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
 # WORMHOLE_TRANSPORT=direct
 # WORMHOLE_SOCKS_PROXY=127.0.0.1:9050
 # WORMHOLE_SOCKS_DNS=true
+# Optional override for the loaded Rust privacy-core shared library. Leave
+# unset for the default repo search order. When you override this, verify the
+# authenticated wormhole status surfaces show the expected version, absolute
+# library path, and SHA-256 for the loaded artifact before making stronger
+# privacy claims about the deployment.
+# PRIVACY_CORE_LIB=
+# Minimum privacy-core version accepted when hidden/private carriers are
+# enabled. Private-lane startup fails closed if the loaded artifact is
+# missing, reports no parseable version, or falls below this minimum.
+# PRIVACY_CORE_MIN_VERSION=0.1.0
+# Comma-separated SHA-256 allowlist for the exact privacy-core artifact(s)
+# your deployment is allowed to load. Required for Arti/RNS private-lane
+# startup. Generate with:
+#   PowerShell: Get-FileHash .\privacy-core\target\release\privacy_core.dll -Algorithm SHA256
+#   macOS/Linux: sha256sum ./privacy-core/target/release/libprivacy_core.so
+# PRIVACY_CORE_ALLOWED_SHA256=
+# Optional structured release attestation artifact for the Sprint 8 release gate.
+# Relative paths resolve from the backend directory. When set explicitly, a
+# missing or unreadable file fails the DM relay security-suite criterion closed.
+# CI/release tooling can generate this automatically via:
+#   uv run python scripts/release_helper.py write-attestation ...
+# MESH_RELEASE_ATTESTATION_PATH=data/release_attestation.json
+# Operator-only Sprint 8 release attestation. Set this only when the DM relay
+# security suite has been run and passed for the current release candidate.
+# File-based release attestation takes precedence when present.
+# MESH_RELEASE_DM_RELAY_SECURITY_SUITE_GREEN=false
+
+# ── OpenClaw Agent ─────────────────────────────────────────────
+# HMAC shared secret for remote OpenClaw agent authentication.
+# Auto-generated via the Connect OpenClaw modal — do not set manually.
+# OPENCLAW_HMAC_SECRET=
+# Access tier: "restricted" (read-only) or "full" (read+write+inject)
+# OPENCLAW_ACCESS_TIER=restricted
+
+# ── SAR (Synthetic Aperture Radar) Layer ───────────────────────
+# Mode A — Free catalog metadata from Alaska Satellite Facility (ASF Search).
+# No account, no downloads.  Default-on.  Set to false to disable entirely.
+# MESH_SAR_CATALOG_ENABLED=true
+#
+# Mode B — Free pre-processed ground-change anomalies (deformation, flood,
+# damage assessments) from NASA OPERA, Copernicus EGMS, GFM, EMS, UNOSAT.
+# Two-step opt-in: BOTH of the following must be set together.
+#   1. MESH_SAR_PRODUCTS_FETCH=allow
+#   2. MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE=true
+# Either flag alone keeps Mode B disabled.  You can also enable this from
+# the Settings → SAR panel inside the app.
+# MESH_SAR_PRODUCTS_FETCH=block
+# MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE=false
+#
+# NASA Earthdata Login (free, ~1 minute signup) — required for OPERA products.
+#   Sign up: https://urs.earthdata.nasa.gov/users/new
+#   Generate token: https://urs.earthdata.nasa.gov/profile  →  "Generate Token"
+# MESH_SAR_EARTHDATA_USER=
+# MESH_SAR_EARTHDATA_TOKEN=
+#
+# Copernicus Data Space (free, ~1 minute signup) — required for EGMS / EMS.
+#   Sign up: https://dataspace.copernicus.eu/
+# MESH_SAR_COPERNICUS_USER=
+# MESH_SAR_COPERNICUS_TOKEN=
+#
+# Allow OpenClaw agents to read and act on the SAR layer (default true).
+# MESH_SAR_OPENCLAW_ENABLED=true
+#
+# Require private-tier transport (Tor / RNS) before signing and broadcasting
+# SAR anomalies to the mesh.  Default true — disable only for testnet/local use.
+# MESH_SAR_REQUIRE_PRIVATE_TIER=true

backend/Dockerfile

@@ -1,10 +1,17 @@
 # ---- Stage 1: Compile privacy-core Rust library ----
-FROM rust:1.88-slim-bookworm AS rust-builder
+FROM --platform=$BUILDPLATFORM rust:1.88-slim-bookworm AS rust-builder
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    pkg-config libssl-dev \
+    ca-certificates \
+    git \
+    pkg-config \
+    libssl-dev \
+    build-essential \
     && rm -rf /var/lib/apt/lists/*
 
+ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
+ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
+
 COPY privacy-core /build/privacy-core
 WORKDIR /build/privacy-core
 RUN cargo build --release --lib \
@@ -15,9 +22,13 @@ FROM python:3.11-slim-bookworm
 
 WORKDIR /app
 
-# Install Node.js (for AIS WebSocket proxy) and curl (for network fallback)
+# Install Node.js (for AIS WebSocket proxy), curl (for network fallback), and
+# Tor (for Wormhole/remote-agent .onion transport).
 RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
     curl \
+    git \
+    tor \
     && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
     && apt-get install -y --no-install-recommends nodejs \
     && rm -rf /var/lib/apt/lists/*
@@ -35,12 +46,19 @@ COPY uv.lock /workspace/uv.lock
 COPY backend/pyproject.toml /workspace/backend/pyproject.toml
 
 # Install Python dependencies using the lockfile
-RUN cd /workspace/backend && uv sync --frozen --no-dev \
+RUN cd /workspace/backend && uv sync --frozen --no-dev --extra road-corridor \
     && playwright install --with-deps chromium
 
 # Copy backend source code
 COPY backend/ .
 
+# Preserve safe static data outside /app/data. The compose named volume mounted
+# at /app/data hides image-baked files on first run, so the entrypoint seeds
+# missing static JSON into fresh volumes before the backend starts.
+RUN mkdir -p /app/image-data \
+    && if [ -d /app/data ]; then cp -a /app/data/. /app/image-data/; fi \
+    && chmod +x /app/docker-entrypoint.sh
+
 # Install Node.js dependencies (ws module for AIS WebSocket proxy)
 COPY backend/package*.json ./
 RUN npm ci --omit=dev
@@ -55,7 +73,7 @@ ENV PRIVACY_CORE_LIB=/app/libprivacy_core.so
 # Create a non-root user for security
 # Grant write access to /app so the auto-updater can extract files
 # Pre-create /app/data so mounted volumes inherit correct ownership
-RUN adduser --system --uid 1001 backenduser \
+RUN adduser --system --uid 1001 --home /app backenduser \
     && mkdir -p /app/data \
     && chown -R backenduser /app \
     && chmod -R u+w /app
@@ -67,4 +85,5 @@ USER backenduser
 EXPOSE 8000
 
 # Start FastAPI server
+ENTRYPOINT ["/app/docker-entrypoint.sh"]
 CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--timeout-keep-alive", "120"]

backend/ais_proxy.js

@@ -1,5 +1,37 @@
+// AIS Stream WebSocket proxy.
+//
+// Reads AIS_API_KEY from argv or env, opens a wss:// connection to
+// stream.aisstream.io, subscribes for vessel position reports inside the
+// active map bounding boxes, and pipes JSON messages to stdout for the
+// Python backend to ingest.
+//
+// Issue #258 — SPKI pinning fallback for upstream cert outages
+// -------------------------------------------------------------
+// AISStream uses Let's Encrypt and their renewal pipeline has been observed
+// to fail (cert expired on 2026-05-20). The naive fix the issue reporter
+// applied — passing { rejectUnauthorized: false } — turns off TLS validation
+// entirely, which lets any network attacker MITM the WebSocket and inject
+// fake ship positions onto the operator's map. Same class as the GDELT
+// plaintext-HTTP MITM issue (#199).
+//
+// Instead, when the normal TLS handshake fails with CERT_HAS_EXPIRED, we
+// do a custom TLS connection that ignores ONLY the expiry check, capture
+// the leaf certificate, and compare its public-key SPKI hash against a
+// pinned list (backend/data/aisstream_spki_pins.json). If the SPKI matches,
+// the upstream is still the genuine AISStream — just with an expired cert —
+// and we proceed in "degraded TLS" mode. If the SPKI does not match, we
+// refuse the connection and log loudly: an actual MITM is in progress.
+//
+// Let's Encrypt renewals keep the same public key by default, so the pinned
+// SPKI survives normal cert rotation. The pin list MUST be updated before
+// the operator's pinned key is rotated upstream.
+
 const WebSocket = require('ws');
 const readline = require('readline');
+const fs = require('fs');
+const path = require('path');
+const tls = require('tls');
+const crypto = require('crypto');
 
 const args = process.argv.slice(2);
 const API_KEY = args[0] || process.env.AIS_API_KEY;
@@ -9,6 +41,135 @@ if (!API_KEY) {
     process.exit(1);
 }
 
+// ── SPKI pin support (issue #258) ─────────────────────────────────────────
+
+const AIS_HOST = 'stream.aisstream.io';
+const AIS_PORT = 443;
+const AIS_WS_URL = `wss://${AIS_HOST}/v0/stream`;
+
+// Pin file is looked up in several layouts so the same JS works in:
+//   - the Docker backend image (PIN_FILE_CANDIDATES[0])
+//   - the Tauri desktop runtime (PIN_FILE_CANDIDATES[1])
+//   - a future relocated layout (operator can drop a file at
+//     SHADOWBROKER_AIS_PINS env var)
+const PIN_FILE_CANDIDATES = [
+    process.env.SHADOWBROKER_AIS_PINS || '',
+    path.join(__dirname, 'data', 'aisstream_spki_pins.json'),
+    path.join(__dirname, 'aisstream_spki_pins.json'),
+].filter(Boolean);
+
+// Embedded fallback. Used when no external pin file is reachable so the
+// SPKI fallback still works on minimal/portable installs. The external
+// file (when present) takes priority so operators can update pins without
+// needing a new build.
+const EMBEDDED_PINS = {
+    [AIS_HOST]: [
+        // Captured 2026-05-20 from AISStream's leaf cert (Let's Encrypt R12).
+        // Replace when AISStream rotates server keys.
+        'GJ10H0UPgLrO+2d3ZXROR/TXSVFXKUfRC3QEI2ibEg4=',
+    ],
+};
+
+let aisDegradedMode = false;  // surfaced via stdout status_query marker
+
+function loadSpkiPins() {
+    for (const candidate of PIN_FILE_CANDIDATES) {
+        try {
+            const raw = fs.readFileSync(candidate, 'utf-8');
+            const parsed = JSON.parse(raw);
+            const pins = Array.isArray(parsed[AIS_HOST]) ? parsed[AIS_HOST] : [];
+            const cleaned = pins
+                .filter((p) => typeof p === 'string' && p.length > 0)
+                .map((p) => p.trim());
+            if (cleaned.length > 0) {
+                return cleaned;
+            }
+        } catch (e) {
+            // Try the next candidate — file may not exist in this layout.
+            continue;
+        }
+    }
+    const embedded = (EMBEDDED_PINS[AIS_HOST] || []).slice();
+    if (embedded.length > 0) {
+        console.error(
+            '[AIS Proxy] No external SPKI pin file found; using embedded fallback. '
+            + `(Set SHADOWBROKER_AIS_PINS or drop ${PIN_FILE_CANDIDATES[1]} to override.)`
+        );
+    }
+    return embedded;
+}
+
+function spkiHashFromPeerCert(peerCert) {
+    // tls.TLSSocket.getPeerCertificate() exposes .pubkey when called with
+    // detailed=true. The pubkey buffer is the DER-encoded SubjectPublicKeyInfo,
+    // which is exactly the value we hash for SPKI pinning.
+    if (!peerCert || !peerCert.pubkey) return null;
+    return crypto.createHash('sha256').update(peerCert.pubkey).digest('base64');
+}
+
+// Probe the upstream when normal TLS failed with CERT_HAS_EXPIRED. We open
+// a raw TLS connection with rejectUnauthorized=false ONLY to inspect the
+// leaf cert; we do NOT use this socket for the actual WebSocket traffic.
+// Returns { ok: true } if the leaf SPKI matches the pin list, { ok: false }
+// with a reason otherwise.
+function verifyExpiredCertAgainstPins() {
+    return new Promise((resolve) => {
+        const pins = loadSpkiPins();
+        if (pins.length === 0) {
+            resolve({ ok: false, reason: 'no SPKI pins configured' });
+            return;
+        }
+        const sock = tls.connect(
+            {
+                host: AIS_HOST,
+                port: AIS_PORT,
+                servername: AIS_HOST,
+                // Allow the handshake to complete despite the expired cert
+                // so we can inspect the leaf. We do NOT trust this connection
+                // for any application data.
+                rejectUnauthorized: false,
+            },
+            () => {
+                const peer = sock.getPeerCertificate(true);
+                sock.end();
+                if (!peer || Object.keys(peer).length === 0) {
+                    resolve({ ok: false, reason: 'no peer certificate returned' });
+                    return;
+                }
+                if (peer.subject && peer.subject.CN !== AIS_HOST) {
+                    resolve({
+                        ok: false,
+                        reason: `cert CN mismatch (got ${peer.subject.CN}, expected ${AIS_HOST})`,
+                    });
+                    return;
+                }
+                const hash = spkiHashFromPeerCert(peer);
+                if (!hash) {
+                    resolve({ ok: false, reason: 'could not compute SPKI hash from peer cert' });
+                    return;
+                }
+                if (pins.includes(hash)) {
+                    resolve({ ok: true, hash });
+                } else {
+                    resolve({
+                        ok: false,
+                        reason: `SPKI ${hash} not in pin list (possible MITM)`,
+                    });
+                }
+            },
+        );
+        sock.setTimeout(10000, () => {
+            sock.destroy();
+            resolve({ ok: false, reason: 'TLS probe timeout' });
+        });
+        sock.on('error', (err) => {
+            resolve({ ok: false, reason: `TLS probe error: ${err.message}` });
+        });
+    });
+}
+
+// ── Subscription state ───────────────────────────────────────────────────
+
 // Start with global coverage, until frontend updates it
 let currentBboxes = [[[-90, -180], [90, 180]]];
 let activeWs = null;
@@ -42,14 +203,34 @@ rl.on('line', (line) => {
             currentBboxes = cmd.bboxes;
             if (activeWs) sendSub(activeWs); // Resend subscription (swap and replace)
         }
+        if (cmd.type === "status_query") {
+            // Allow the Python side to probe degraded-mode state by sending
+            // {"type": "status_query"} on stdin. Reply on stdout as a marker.
+            process.stdout.write(JSON.stringify({
+                __ais_proxy_status: { degraded_tls: aisDegradedMode }
+            }) + '\n');
+        }
     } catch (e) {}
 });
 
-function connect() {
-    const ws = new WebSocket('wss://stream.aisstream.io/v0/stream');
+function attachWsHandlers(ws, { degraded } = { degraded: false }) {
     activeWs = ws;
 
     ws.on('open', () => {
+        if (degraded) {
+            console.error(
+                '[AIS Proxy] Connected in DEGRADED TLS MODE — upstream cert is expired '
+                + 'but SPKI matches the pinned key, so identity is still verified. '
+                + 'AISStream needs to renew their cert; until then MITM protection '
+                + 'depends only on the SPKI match. Watch backend logs for resolution.'
+            );
+            aisDegradedMode = true;
+        } else {
+            if (aisDegradedMode) {
+                console.error('[AIS Proxy] Reconnected with full TLS validation — degraded mode cleared.');
+            }
+            aisDegradedMode = false;
+        }
         sendSub(ws);
     });
 
@@ -61,14 +242,63 @@ function connect() {
     });
 
     ws.on('error', (err) => {
-        console.error("WebSocket Proxy Error:", err.message);
+        console.error('WebSocket Proxy Error:', err.message);
     });
 
     ws.on('close', () => {
         activeWs = null;
-        console.error("WebSocket Proxy Closed. Reconnecting in 5s...");
+        console.error('WebSocket Proxy Closed. Reconnecting in 5s...');
         setTimeout(connect, 5000);
     });
 }
 
+function connect() {
+    // Path A: normal TLS validation (the 99.9% case). If this succeeds we
+    // never touch the SPKI fallback.
+    const ws = new WebSocket(AIS_WS_URL);
+
+    let openedOk = false;
+    ws.on('open', () => { openedOk = true; });
+
+    ws.on('error', async (err) => {
+        // Only the CERT_HAS_EXPIRED case triggers SPKI verification. Any
+        // other TLS or network error gets the standard reconnect path so we
+        // don't accidentally cover up legitimate problems.
+        if (!openedOk && err && err.code === 'CERT_HAS_EXPIRED') {
+            console.error(
+                '[AIS Proxy] Upstream certificate is expired. Verifying SPKI '
+                + 'against pinned keys before deciding whether to proceed in '
+                + 'degraded mode...'
+            );
+            const verdict = await verifyExpiredCertAgainstPins();
+            if (verdict.ok) {
+                console.error(
+                    `[AIS Proxy] SPKI ${verdict.hash} matches pinned key — `
+                    + 'identity is verified, proceeding in DEGRADED TLS mode.'
+                );
+                const insecureWs = new WebSocket(AIS_WS_URL, {
+                    rejectUnauthorized: false,
+                });
+                attachWsHandlers(insecureWs, { degraded: true });
+            } else {
+                console.error(
+                    `[AIS Proxy] SPKI verification FAILED (${verdict.reason}). `
+                    + 'Refusing to connect — this would normally indicate an active '
+                    + 'MITM attack. If AISStream rotated their server key, update '
+                    + 'backend/data/aisstream_spki_pins.json with the new SPKI hash.'
+                );
+                // Schedule a retry — operator may have updated the pin file.
+                setTimeout(connect, 60000);
+            }
+            return;
+        }
+        // Default: surface the error and let the close handler reconnect.
+        console.error('WebSocket Proxy Error:', err.message);
+    });
+
+    // Wire normal handlers — these apply unless the error handler above
+    // takes over and replaces activeWs with an insecure socket.
+    attachWsHandlers(ws, { degraded: false });
+}
+
 connect();

backend/analytics/__init__.py

@@ -0,0 +1,21 @@
+"""Strategic Risk Analytics — game-theoretic early warning layer."""
+
+from analytics.backtest import (
+    DEFAULT_BACKTEST_ALERT_THRESHOLD,
+    BacktestReport,
+    run_historical_backtest,
+    tune_alert_threshold,
+)
+from analytics.gt_early_warning import GT_EarlyWarning
+from analytics.integration import get_gt_engine, process_feed_item, refresh_from_latest_data
+
+__all__ = [
+    "BacktestReport",
+    "DEFAULT_BACKTEST_ALERT_THRESHOLD",
+    "GT_EarlyWarning",
+    "get_gt_engine",
+    "process_feed_item",
+    "refresh_from_latest_data",
+    "run_historical_backtest",
+    "tune_alert_threshold",
+]

backend/analytics/backtest.py

@@ -0,0 +1,287 @@
+"""Historical backtesting for Strategic Risk Analytics.
+
+This is **benchmark validation**, not forward-weeks prediction on live feeds.
+
+The suite scores whether costly-signal patterns + Bayesian updating correctly
+classify curated pre-crisis text snippets (positive cases) vs cheap-talk
+controls (negative cases) at a tuned alert threshold. A high accuracy on this
+labeled corpus does **not** imply the engine will score 100% on messy,
+adversarial, or weeks-ahead production telemetry — opponents adapt, labels are
+easier here than in the wild, and the window is retrospective.
+
+Reports accuracy and a conservative Wilson 95% confidence lower bound on the
+benchmark only. Treat 100% here as "classifier fits the benchmark," not "ship
+it for multi-week forecasting." For live week-over-week scoring with delayed
+labels, see ``rolling_backtest.py``.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+from analytics.gt_early_warning import GT_EarlyWarning
+from analytics.historical_events import (
+    HistoricalCase,
+    default_historical_cases,
+    expanded_historical_cases,
+)
+from analytics.settings import GTAnalyticsSettings
+
+DomainName = Literal["financial", "unrest", "conflict"]
+
+# Validated on expanded suite (82 cases, Wilson lower >= 0.95 at 100% accuracy).
+DEFAULT_BACKTEST_ALERT_THRESHOLD = 0.26
+MAX_BACKTEST_ALERT_THRESHOLD = 0.39
+
+
+@dataclass(frozen=True)
+class CaseResult:
+    case_id: str
+    name: str
+    kind: str
+    region: str
+    domain: str
+    expected_alert: bool
+    alerted: bool
+    correct: bool
+    peak_domain_risk: float
+    peak_composite_risk: float
+    costly_signals: list[str]
+    tags: tuple[str, ...] = field(default_factory=tuple)
+
+
+@dataclass(frozen=True)
+class BacktestReport:
+    total_cases: int
+    correct: int
+    accuracy: float
+    confidence_rate: float
+    wilson_lower_95: float
+    wilson_upper_95: float
+    true_positives: int
+    true_negatives: int
+    false_positives: int
+    false_negatives: int
+    sensitivity: float
+    specificity: float
+    alert_threshold: float
+    target_confidence: float
+    meets_target: bool
+    case_results: tuple[CaseResult, ...]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "total_cases": self.total_cases,
+            "correct": self.correct,
+            "accuracy": round(self.accuracy, 4),
+            "confidence_rate": round(self.confidence_rate, 4),
+            "wilson_lower_95": round(self.wilson_lower_95, 4),
+            "wilson_upper_95": round(self.wilson_upper_95, 4),
+            "true_positives": self.true_positives,
+            "true_negatives": self.true_negatives,
+            "false_positives": self.false_positives,
+            "false_negatives": self.false_negatives,
+            "sensitivity": round(self.sensitivity, 4),
+            "specificity": round(self.specificity, 4),
+            "alert_threshold": self.alert_threshold,
+            "target_confidence": self.target_confidence,
+            "meets_target": self.meets_target,
+            "cases": [
+                {
+                    "case_id": row.case_id,
+                    "name": row.name,
+                    "kind": row.kind,
+                    "correct": row.correct,
+                    "alerted": row.alerted,
+                    "peak_domain_risk": round(row.peak_domain_risk, 4),
+                    "peak_composite_risk": round(row.peak_composite_risk, 4),
+                    "costly_signals": row.costly_signals,
+                }
+                for row in self.case_results
+            ],
+        }
+
+
+def wilson_interval(
+    successes: int,
+    total: int,
+    z: float = 1.96,
+) -> tuple[float, float]:
+    """Wilson score interval for a binomial proportion (95% default)."""
+    if total <= 0:
+        return 0.0, 0.0
+    phat = successes / total
+    z2 = z * z
+    denom = 1.0 + z2 / total
+    center = (phat + z2 / (2.0 * total)) / denom
+    margin = (
+        z
+        * math.sqrt((phat * (1.0 - phat) + z2 / (4.0 * total)) / total)
+        / denom
+    )
+    return max(0.0, center - margin), min(1.0, center + margin)
+
+
+def _domain_risk(engine: GT_EarlyWarning, region: str, domain: str) -> float:
+    if domain in ("financial", "unrest", "conflict"):
+        return engine.get_prior(region, domain)
+    return engine.composite_risk(region)
+
+
+def _evaluate_case(
+    case: HistoricalCase,
+    *,
+    settings: GTAnalyticsSettings,
+    alert_threshold: float,
+) -> CaseResult:
+    engine = GT_EarlyWarning(settings)
+    peak_domain = float(settings.base_prior)
+    peak_composite = float(settings.base_prior)
+    detected_signals: set[str] = set()
+
+    for item in case.to_feed_dicts():
+        result = engine.process_feed_item(item)
+        for sig in (result or {}).get("signals") or {}:
+            detected_signals.add(str(sig))
+        domain_risk = _domain_risk(engine, case.region, case.domain)
+        composite = engine.composite_risk(case.region)
+        peak_domain = max(peak_domain, domain_risk)
+        peak_composite = max(peak_composite, composite)
+
+    # Domain-specific score for labeled events; composite as secondary for conflict.
+    score = peak_domain
+    if case.domain == "conflict":
+        score = max(peak_domain, peak_composite * 0.95)
+    alerted = score >= alert_threshold
+    expected_alert = case.kind == "positive"
+
+    return CaseResult(
+        case_id=case.case_id,
+        name=case.name,
+        kind=case.kind,
+        region=case.region,
+        domain=case.domain,
+        expected_alert=expected_alert,
+        alerted=alerted,
+        correct=alerted == expected_alert,
+        peak_domain_risk=peak_domain,
+        peak_composite_risk=peak_composite,
+        costly_signals=sorted(detected_signals),
+        tags=case.tags,
+    )
+
+
+def run_historical_backtest(
+    cases: tuple[HistoricalCase, ...] | None = None,
+    *,
+    settings: GTAnalyticsSettings | None = None,
+    alert_threshold: float | None = None,
+    target_confidence: float = 0.80,
+    use_expanded_suite: bool = True,
+) -> BacktestReport:
+    """
+    Run labeled historical cases and compute accuracy + Wilson 95% CI.
+
+    ``confidence_rate`` is the conservative Wilson lower bound — the metric
+    used for pass/fail against ``target_confidence``.
+    """
+    cfg = settings or GTAnalyticsSettings(enabled=True)
+    threshold = float(
+        alert_threshold
+        if alert_threshold is not None
+        else DEFAULT_BACKTEST_ALERT_THRESHOLD
+    )
+    if cases is not None:
+        suite = cases
+    elif use_expanded_suite:
+        suite = expanded_historical_cases()
+    else:
+        suite = default_historical_cases()
+
+    results = tuple(
+        _evaluate_case(case, settings=cfg, alert_threshold=threshold) for case in suite
+    )
+
+    tp = sum(1 for r in results if r.expected_alert and r.alerted)
+    tn = sum(1 for r in results if not r.expected_alert and not r.alerted)
+    fp = sum(1 for r in results if not r.expected_alert and r.alerted)
+    fn = sum(1 for r in results if r.expected_alert and not r.alerted)
+    correct = tp + tn
+    total = len(results)
+    accuracy = correct / total if total else 0.0
+    lower, upper = wilson_interval(correct, total)
+
+    pos_total = sum(1 for r in results if r.expected_alert)
+    neg_total = total - pos_total
+    sensitivity = tp / pos_total if pos_total else 0.0
+    specificity = tn / neg_total if neg_total else 0.0
+
+    return BacktestReport(
+        total_cases=total,
+        correct=correct,
+        accuracy=accuracy,
+        confidence_rate=lower,
+        wilson_lower_95=lower,
+        wilson_upper_95=upper,
+        true_positives=tp,
+        true_negatives=tn,
+        false_positives=fp,
+        false_negatives=fn,
+        sensitivity=sensitivity,
+        specificity=specificity,
+        alert_threshold=threshold,
+        target_confidence=target_confidence,
+        meets_target=lower >= target_confidence,
+        case_results=results,
+    )
+
+
+def tune_alert_threshold(
+    cases: tuple[HistoricalCase, ...] | None = None,
+    *,
+    settings: GTAnalyticsSettings | None = None,
+    min_threshold: float = 0.20,
+    max_threshold: float = 0.65,
+    step: float = 0.01,
+    target_confidence: float = 0.95,
+) -> tuple[float, BacktestReport]:
+    """Grid-search alert threshold to maximize Wilson lower bound."""
+    if cases is not None:
+        suite = cases
+    else:
+        suite = expanded_historical_cases()
+    best_threshold = min_threshold
+    best_report = run_historical_backtest(
+        suite,
+        settings=settings,
+        alert_threshold=min_threshold,
+        target_confidence=target_confidence,
+    )
+
+    steps = int(round((max_threshold - min_threshold) / step))
+    for i in range(steps + 1):
+        threshold = min_threshold + i * step
+        report = run_historical_backtest(
+            suite,
+            settings=settings,
+            alert_threshold=threshold,
+            target_confidence=target_confidence,
+        )
+        better_confidence = report.confidence_rate > best_report.confidence_rate
+        tied_confidence = math.isclose(
+            report.confidence_rate, best_report.confidence_rate, rel_tol=0.0, abs_tol=1e-9
+        )
+        better_accuracy = report.accuracy > best_report.accuracy
+        tied_accuracy = math.isclose(
+            report.accuracy, best_report.accuracy, rel_tol=0.0, abs_tol=1e-9
+        )
+        prefer_higher_threshold = (
+            tied_confidence and tied_accuracy and threshold > best_threshold
+        )
+        if better_confidence or (tied_confidence and better_accuracy) or prefer_higher_threshold:
+            best_threshold = threshold
+            best_report = report
+
+    return best_threshold, best_report

backend/analytics/daily_store.py

@@ -0,0 +1,140 @@
+"""Daily GT risk readings for micro rolling averages."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+from dataclasses import asdict, dataclass, field
+from datetime import date, datetime, timezone
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_DAILY_DIR = Path(__file__).parent.parent / "data" / "gt_rolling" / "daily"
+_store_lock = threading.Lock()
+
+
+def daily_store_dir() -> Path:
+    override = str(os.environ.get("GT_DAILY_STORE_DIR", "")).strip()
+    if override:
+        return Path(override)
+    return _DAILY_DIR
+
+
+def utc_today() -> date:
+    return datetime.now(timezone.utc).date()
+
+
+def date_id(when: date | datetime | None = None) -> str:
+    if when is None:
+        when = utc_today()
+    if isinstance(when, datetime):
+        when = when.date()
+    return when.isoformat()
+
+
+@dataclass
+class DailyRegionReading:
+    region: str
+    composite_risk: float
+    financial: float
+    unrest: float
+    conflict: float
+    peak_score: float
+    readings: int = 1
+    last_captured_at: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> DailyRegionReading:
+        return cls(
+            region=str(raw.get("region") or "").strip().lower(),
+            composite_risk=float(raw.get("composite_risk") or 0.0),
+            financial=float(raw.get("financial") or 0.0),
+            unrest=float(raw.get("unrest") or 0.0),
+            conflict=float(raw.get("conflict") or 0.0),
+            peak_score=float(raw.get("peak_score") or 0.0),
+            readings=int(raw.get("readings") or 1),
+            last_captured_at=str(raw.get("last_captured_at") or ""),
+        )
+
+
+@dataclass
+class DailySnapshot:
+    date: str
+    regions: dict[str, DailyRegionReading] = field(default_factory=dict)
+    last_updated_at: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "date": self.date,
+            "last_updated_at": self.last_updated_at,
+            "regions": {key: row.to_dict() for key, row in self.regions.items()},
+        }
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> DailySnapshot:
+        regions: dict[str, DailyRegionReading] = {}
+        for key, row in (raw.get("regions") or {}).items():
+            if isinstance(row, dict):
+                reading = DailyRegionReading.from_dict(row)
+                regions[str(key).strip().lower()] = reading
+        return cls(
+            date=str(raw.get("date") or ""),
+            regions=regions,
+            last_updated_at=str(raw.get("last_updated_at") or ""),
+        )
+
+
+def _daily_path(day_id: str) -> Path:
+    safe = day_id.replace("/", "-").replace("..", "")
+    return daily_store_dir() / f"{safe}.json"
+
+
+def _ensure_dir() -> None:
+    daily_store_dir().mkdir(parents=True, exist_ok=True)
+
+
+def list_daily_ids(*, newest_first: bool = True, limit: int | None = None) -> list[str]:
+    _ensure_dir()
+    ids = sorted(
+        (path.stem for path in daily_store_dir().glob("*.json")),
+        reverse=newest_first,
+    )
+    if limit is not None:
+        return ids[:limit]
+    return ids
+
+
+def load_daily(day: date | str | None = None) -> DailySnapshot | None:
+    day_id = date_id(day) if day is not None else date_id()
+    path = _daily_path(day_id)
+    if not path.is_file():
+        return None
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+        if not isinstance(raw, dict):
+            return None
+        return DailySnapshot.from_dict(raw)
+    except (OSError, json.JSONDecodeError, TypeError, ValueError):
+        logger.exception("Failed to load GT daily reading %s", day_id)
+        return None
+
+
+def save_daily(snapshot: DailySnapshot) -> None:
+    _ensure_dir()
+    path = _daily_path(snapshot.date)
+    tmp = path.with_suffix(".json.tmp")
+    payload = json.dumps(snapshot.to_dict(), indent=2, sort_keys=True)
+    with _store_lock:
+        tmp.write_text(payload, encoding="utf-8")
+        tmp.replace(path)
+
+
+def utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()

backend/analytics/feed_adapter.py

@@ -0,0 +1,206 @@
+"""Normalize Shadowbroker feed records into GT analytics feed items."""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Iterable
+
+_DOMAIN_CONFLICT = "conflict"
+_DOMAIN_UNREST = "unrest"
+_DOMAIN_FINANCIAL = "financial"
+
+_CONFLICT_HINTS = re.compile(
+    r"\b(war|missile|strike|attack|military|invasion|troop|shelling|drone|bomb|nuclear)\b",
+    re.I,
+)
+_UNREST_HINTS = re.compile(
+    r"\b(protest|rally|strike|riot|unrest|mobiliz|demonstrat|curfew|purge|coup)\b",
+    re.I,
+)
+_FINANCIAL_HINTS = re.compile(
+    r"\b(payroll|loan|default|bankruptcy|liquidity|sanction|supply\s+chain|delay|shortage)\b",
+    re.I,
+)
+
+
+def _clean_region(value: Any) -> str:
+    region = str(value or "").strip().lower()
+    return region or "global"
+
+
+def _infer_domain(text: str, explicit: str | None = None) -> str:
+    if explicit in {_DOMAIN_CONFLICT, _DOMAIN_UNREST, _DOMAIN_FINANCIAL}:
+        return explicit
+    if _CONFLICT_HINTS.search(text):
+        return _DOMAIN_CONFLICT
+    if _UNREST_HINTS.search(text):
+        return _DOMAIN_UNREST
+    if _FINANCIAL_HINTS.search(text):
+        return _DOMAIN_FINANCIAL
+    return _DOMAIN_FINANCIAL
+
+
+def _text_from_record(
+    record: dict[str, Any],
+    *,
+    prefer_translation: bool = False,
+) -> str:
+    """Build ingest text; prefer English translations for Telegram OSINT when set."""
+    if prefer_translation:
+        translated_parts = [
+            record.get("title_translated"),
+            record.get("description_translated"),
+        ]
+        translated = "\n".join(
+            str(p).strip() for p in translated_parts if p and str(p).strip()
+        )
+        if translated:
+            return translated
+
+    parts = [
+        record.get("title"),
+        record.get("description"),
+        record.get("text"),
+        record.get("summary"),
+    ]
+    return "\n".join(str(p).strip() for p in parts if p and str(p).strip())
+
+
+_HASHTAG_REGION = re.compile(r"#([a-z][a-z0-9_-]{2,})", re.I)
+
+
+def _region_from_hashtags(text: str) -> str | None:
+    """Map common theater hashtags (#Ukraine) to dossier/heatmap region keys."""
+    for match in _HASHTAG_REGION.finditer(text or ""):
+        tag = match.group(1).lower()
+        if tag in {
+            "ukraine",
+            "russia",
+            "israel",
+            "iran",
+            "gaza",
+            "syria",
+            "taiwan",
+            "china",
+            "belfast",
+            "uk",
+            "usa",
+        }:
+            return tag
+    return None
+
+
+def _region_from_record(record: dict[str, Any], *, text: str = "") -> str:
+    for key in ("geotag", "region", "country", "location"):
+        if record.get(key):
+            return _clean_region(record[key])
+    hashtag_region = _region_from_hashtags(text)
+    if hashtag_region:
+        return hashtag_region
+    coords = record.get("coords")
+    if isinstance(coords, (list, tuple)) and len(coords) >= 2:
+        try:
+            lat = float(coords[0])
+            lng = float(coords[1])
+            return f"{lat:.2f},{lng:.2f}"
+        except (TypeError, ValueError):
+            pass
+    return "global"
+
+
+def _entities_from_record(record: dict[str, Any]) -> list[str]:
+    entities: list[str] = []
+    for key in ("entities", "tags", "keywords"):
+        raw = record.get(key)
+        if isinstance(raw, list):
+            entities.extend(str(v).strip() for v in raw if str(v).strip())
+        elif isinstance(raw, str) and raw.strip():
+            entities.extend(part.strip() for part in raw.split(",") if part.strip())
+    channel = str(record.get("channel") or "").strip()
+    if channel:
+        entities.append(f"channel:{channel}")
+    source = str(record.get("source") or "").strip()
+    if source:
+        entities.append(f"source:{source}")
+    return entities
+
+
+def normalize_feed_item(record: dict[str, Any], *, source_type: str = "generic") -> dict[str, Any]:
+    """Map a news/Telegram/GDELT record into the GT engine schema."""
+    prefer_translation = source_type == "telegram_osint"
+    text = _text_from_record(record, prefer_translation=prefer_translation)
+    if prefer_translation and not text.strip():
+        text = _text_from_record(record, prefer_translation=False)
+    region = _region_from_record(record, text=text)
+    domain = _infer_domain(text, record.get("domain"))
+    coords = record.get("coords")
+    lat = lng = None
+    if isinstance(coords, (list, tuple)) and len(coords) >= 2:
+        try:
+            lat = float(coords[0])
+            lng = float(coords[1])
+        except (TypeError, ValueError):
+            lat = lng = None
+
+    return {
+        "id": record.get("id") or record.get("link"),
+        "text": text,
+        "source": str(record.get("source") or source_type),
+        "source_type": source_type,
+        "region": region,
+        "domain": domain,
+        "entities": _entities_from_record(record),
+        "coords": [lat, lng] if lat is not None and lng is not None else None,
+        "published": record.get("published"),
+        "risk_score": record.get("risk_score"),
+    }
+
+
+def iter_telegram_posts(payload: dict[str, Any] | None) -> Iterable[dict[str, Any]]:
+    from services.telegram_translate import apply_post_translation, telegram_translate_enabled
+
+    posts = list((payload or {}).get("posts") or [])
+    for post in posts:
+        if not isinstance(post, dict):
+            continue
+        if not (post.get("description") or post.get("title")):
+            continue
+        enriched = (
+            apply_post_translation(post)
+            if telegram_translate_enabled()
+            else post
+        )
+        yield normalize_feed_item(enriched, source_type="telegram_osint")
+
+
+def iter_news_items(payload: list[dict[str, Any]] | None) -> Iterable[dict[str, Any]]:
+    for item in list(payload or []):
+        if not isinstance(item, dict):
+            continue
+        yield normalize_feed_item(item, source_type="news")
+        for article in list(item.get("articles") or []):
+            if isinstance(article, dict):
+                yield normalize_feed_item(article, source_type="news_cluster")
+
+
+def iter_gdelt_features(payload: list[dict[str, Any]] | None) -> Iterable[dict[str, Any]]:
+    for feature in list(payload or []):
+        if not isinstance(feature, dict):
+            continue
+        props = dict(feature.get("properties") or {})
+        geometry = dict(feature.get("geometry") or {})
+        coords = None
+        if geometry.get("type") == "Point":
+            raw = geometry.get("coordinates")
+            if isinstance(raw, (list, tuple)) and len(raw) >= 2:
+                coords = [float(raw[1]), float(raw[0])]
+        record = {
+            "title": props.get("name") or props.get("title"),
+            "description": props.get("snippet") or props.get("description"),
+            "source": props.get("source") or "gdelt",
+            "coords": coords,
+            "published": props.get("date") or props.get("published"),
+            "region": props.get("location") or props.get("country"),
+        }
+        if record["title"] or record["description"]:
+            yield normalize_feed_item(record, source_type="gdelt")

backend/analytics/gt_alerts.py

@@ -0,0 +1,128 @@
+"""Top strategic-risk alerts — ranked regions with map coordinates."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from analytics.integration import get_gt_engine
+from analytics.settings import get_gt_settings
+
+
+def _peak_score(props: dict[str, Any]) -> float:
+    composite = float(props.get("risk") or 0.0)
+    financial = float(props.get("financial") or 0.0)
+    unrest = float(props.get("unrest") or 0.0)
+    conflict = float(props.get("conflict") or 0.0)
+    return max(composite, financial, unrest, conflict)
+
+
+def _valid_coords(coords: Any) -> tuple[float, float] | None:
+    if not isinstance(coords, (list, tuple)) or len(coords) < 2:
+        return None
+    try:
+        lng = float(coords[0])
+        lat = float(coords[1])
+    except (TypeError, ValueError):
+        return None
+    if not (-90.0 <= lat <= 90.0 and -180.0 <= lng <= 180.0):
+        return None
+    if abs(lat) < 0.001 and abs(lng) < 0.001:
+        return None
+    return lat, lng
+
+
+def _region_label(region: str) -> str:
+    text = str(region or "").strip()
+    if not text:
+        return "unknown"
+    if "," in text:
+        parts = [piece.strip() for piece in text.split(",") if piece.strip()]
+        if len(parts) >= 2:
+            try:
+                lat = float(parts[0])
+                lng = float(parts[-1])
+                return f"{lat:.2f}°, {lng:.2f}°"
+            except ValueError:
+                pass
+    return text.replace("_", " ")
+
+
+def parse_heatmap_alerts(
+    heatmap: dict[str, Any] | None,
+    *,
+    limit: int = 8,
+) -> tuple[list[dict[str, Any]], int]:
+    """Return ranked alerts and count of regions plottable on the map."""
+    features = (heatmap or {}).get("features") or []
+    rows: list[dict[str, Any]] = []
+
+    for feature in features:
+        if not isinstance(feature, dict):
+            continue
+        geometry = feature.get("geometry") or {}
+        coords = _valid_coords(geometry.get("coordinates"))
+        if coords is None:
+            continue
+        lat, lng = coords
+        props = feature.get("properties") or {}
+        region = str(props.get("region") or "").strip().lower()
+        if not region:
+            continue
+        score = _peak_score(props)
+        rows.append(
+            {
+                "region": region,
+                "region_label": _region_label(region),
+                "risk": round(float(props.get("risk") or 0.0), 4),
+                "financial": round(float(props.get("financial") or 0.0), 4),
+                "unrest": round(float(props.get("unrest") or 0.0), 4),
+                "conflict": round(float(props.get("conflict") or 0.0), 4),
+                "contagion": round(float(props.get("contagion") or 0.0), 4),
+                "score": round(score, 4),
+                "lat": lat,
+                "lng": lng,
+                "ignition": bool(props.get("micro_ignition")),
+                "risk_3d_avg": props.get("risk_3d_avg"),
+                "risk_delta": props.get("risk_delta"),
+                "updates": int(props.get("updates") or 0),
+            }
+        )
+
+    rows.sort(
+        key=lambda row: (
+            bool(row.get("ignition")),
+            float(row.get("risk_delta") or 0.0),
+            float(row.get("score") or 0.0),
+        ),
+        reverse=True,
+    )
+    return rows[: max(1, limit)], len(rows)
+
+
+def top_gt_alerts(*, limit: int = 8) -> dict[str, Any]:
+    """Ranked top regions for API / OpenClaw."""
+    settings = get_gt_settings()
+    engine = get_gt_engine()
+    heatmap: dict[str, Any] = {"type": "FeatureCollection", "features": []}
+    engine_regions = 0
+
+    if engine is not None:
+        heatmap = engine.get_risk_heatmap()
+        with engine._lock:  # noqa: SLF001 — intentional meta read
+            engine_regions = len(engine._regions)
+
+    alerts, plotted = parse_heatmap_alerts(heatmap, limit=limit)
+    tracked = len(heatmap.get("features") or [])
+
+    return {
+        "alerts": alerts,
+        "tracked_regions": tracked,
+        "engine_regions": engine_regions,
+        "plotted_regions": plotted,
+        "max_regions": settings.max_heatmap_features,
+        "note": (
+            "Layer count is tracked GT regions (cap "
+            f"{settings.max_heatmap_features}), not raw feed events. "
+            "Only regions with valid coordinates appear on the map."
+        ),
+    }

backend/analytics/gt_early_warning.py

@@ -0,0 +1,593 @@
+"""Game-theoretic early warning analytics with Bayesian updating and contagion graph."""
+
+from __future__ import annotations
+
+import logging
+import re
+import threading
+from collections import defaultdict
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, DefaultDict
+
+import networkx as nx
+import numpy as np
+
+from analytics.settings import GTAnalyticsSettings, get_gt_settings
+
+logger = logging.getLogger(__name__)
+
+DomainName = str  # financial | unrest | conflict
+
+_DOMAINS: tuple[DomainName, ...] = ("financial", "unrest", "conflict")
+
+_DEFAULT_LIKELIHOODS: dict[DomainName, dict[str, float]] = {
+    "financial": {"distress": 0.75, "normal": 0.25},
+    "unrest": {"distress": 0.82, "normal": 0.22},
+    "conflict": {"distress": 0.78, "normal": 0.18},
+}
+
+_DEFAULT_SIGNAL_WEIGHTS: dict[str, float] = {
+    "payroll_loan": 3.0,
+    "supply_delay": 2.2,
+    "elite_relocation": 2.8,
+    "purge": 3.5,
+    "protest_mobilize": 2.5,
+    "gps_jamming": 2.7,
+    "troop_movement": 3.0,
+    "bank_run": 3.2,
+    "sanctions_escalation": 2.4,
+    "ceasefire_break": 2.6,
+}
+
+# Costly-signal regex patterns (cheap talk filtered by absence of match).
+_SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
+    "payroll_loan": [
+        re.compile(r"payroll\s+loan", re.I),
+        re.compile(r"merchant\s+cash\s+advance", re.I),
+        re.compile(r"working\s+capital\s+loan", re.I),
+    ],
+    "supply_delay": [
+        re.compile(r"supply\s+(chain\s+)?delay", re.I),
+        re.compile(r"shipping\s+delay", re.I),
+        re.compile(r"logistics\s+backlog", re.I),
+        re.compile(r"port\s+congestion", re.I),
+    ],
+    "elite_relocation": [
+        re.compile(r"elite\s+(asset\s+)?relocation", re.I),
+        re.compile(r"oligarch\s+jet", re.I),
+        re.compile(r"private\s+jet\s+exodus", re.I),
+        re.compile(r"capital\s+flight", re.I),
+    ],
+    "purge": [
+        re.compile(r"\bpurge\b", re.I),
+        re.compile(r"political\s+purge", re.I),
+        re.compile(r"security\s+apparatus\s+reshuffle", re.I),
+    ],
+    "protest_mobilize": [
+        re.compile(r"protest\s+mobil", re.I),
+        re.compile(r"mass\s+rally", re.I),
+        re.compile(r"general\s+strike", re.I),
+        re.compile(r"\bstrike\b", re.I),
+        re.compile(r"\brally\b", re.I),
+    ],
+    "gps_jamming": [
+        re.compile(r"gps\s+jam", re.I),
+        re.compile(r"gnss\s+interference", re.I),
+        re.compile(r"spoofing\s+spike", re.I),
+    ],
+    "troop_movement": [
+        re.compile(r"troop\s+movement", re.I),
+        re.compile(r"military\s+mobil", re.I),
+        re.compile(r"armored\s+convoy", re.I),
+        re.compile(r"troop\s+buildup", re.I),
+    ],
+    "bank_run": [
+        re.compile(r"bank\s+run", re.I),
+        re.compile(r"deposit\s+flight", re.I),
+        re.compile(r"liquidity\s+crunch", re.I),
+    ],
+    "sanctions_escalation": [
+        re.compile(r"sanctions?\s+escalat", re.I),
+        re.compile(r"new\s+sanctions?", re.I),
+        re.compile(r"export\s+controls?\s+tighten", re.I),
+    ],
+    "ceasefire_break": [
+        re.compile(r"ceasefire\s+(broken|violated|collapse)", re.I),
+        re.compile(r"truce\s+end", re.I),
+    ],
+}
+
+_SIGNAL_DOMAINS: dict[str, DomainName] = {
+    "payroll_loan": "financial",
+    "supply_delay": "financial",
+    "bank_run": "financial",
+    "sanctions_escalation": "financial",
+    "protest_mobilize": "unrest",
+    "purge": "unrest",
+    "elite_relocation": "financial",
+    "gps_jamming": "conflict",
+    "troop_movement": "conflict",
+    "ceasefire_break": "conflict",
+}
+
+
+@dataclass
+class RegionState:
+    """Per-region Bayesian beliefs and metadata."""
+
+    priors: dict[DomainName, float] = field(default_factory=lambda: defaultdict(float))
+    coords: list[float] | None = None
+    signal_volume: DefaultDict[str, float] = field(default_factory=lambda: defaultdict(float))
+    update_count: int = 0
+
+
+@dataclass
+class HistoryEntry:
+    timestamp: str
+    domain: DomainName
+    signals: dict[str, float]
+    strength: float
+    prior: float
+    posterior: float
+    source: str
+    deviation_score: float
+
+
+class GT_EarlyWarning:
+    """
+    Game-Theoretic Early Warning System with Bayesian updating.
+
+    Tracks distress probabilities per region/domain, classifies costly signals vs
+    cheap talk, and propagates risk through an entity interaction graph.
+    """
+
+    def __init__(self, settings: GTAnalyticsSettings | None = None) -> None:
+        self.settings = settings or get_gt_settings()
+        self.G: nx.Graph = nx.Graph()
+        self._regions: dict[str, RegionState] = {}
+        self._history: dict[str, list[HistoryEntry]] = defaultdict(list)
+        self._seen_item_ids: set[str] = set()
+        self._lock = threading.RLock()
+
+        self.likelihoods = dict(_DEFAULT_LIKELIHOODS)
+        self.signal_weights = dict(_DEFAULT_SIGNAL_WEIGHTS)
+        self.signal_weights.update(self.settings.signal_weight_overrides)
+
+        self._base_prior = float(self.settings.base_prior)
+
+    def _utcnow(self) -> str:
+        return datetime.now(timezone.utc).isoformat()
+
+    def _region_state(self, region: str) -> RegionState:
+        key = str(region or "global").strip().lower() or "global"
+        if key not in self._regions:
+            state = RegionState()
+            for domain in _DOMAINS:
+                state.priors[domain] = self._base_prior
+            self._regions[key] = state
+        return self._regions[key]
+
+    def get_prior(self, region: str, domain: DomainName) -> float:
+        with self._lock:
+            return float(self._region_state(region).priors.get(domain, self._base_prior))
+
+    def set_prior(self, region: str, domain: DomainName, value: float) -> None:
+        with self._lock:
+            state = self._region_state(region)
+            state.priors[domain] = float(
+                np.clip(value, self.settings.min_prob, self.settings.max_prob)
+            )
+
+    def composite_risk(self, region: str) -> float:
+        """Weighted composite across domains (conflict weighted highest)."""
+        weights = {"financial": 0.25, "unrest": 0.35, "conflict": 0.40}
+        with self._lock:
+            state = self._region_state(region)
+            total = 0.0
+            weight_sum = 0.0
+            for domain, weight in weights.items():
+                total += float(state.priors.get(domain, self._base_prior)) * weight
+                weight_sum += weight
+            return float(total / weight_sum) if weight_sum else self._base_prior
+
+    def classify_signals(self, text: str, source: str = "") -> dict[str, float]:
+        """Return weighted costly-signal strengths detected in text."""
+        text_lower = (text or "").lower()
+        signals: dict[str, float] = {}
+
+        for signal_name, patterns in _SIGNAL_PATTERNS.items():
+            weight = float(self.signal_weights.get(signal_name, 1.0))
+            if any(pattern.search(text_lower) for pattern in patterns):
+                signals[signal_name] = weight
+
+        rally_strike_count = text_lower.count("rally") + text_lower.count("strike")
+        if rally_strike_count > 3:
+            signals["protest_mobilize"] = signals.get("protest_mobilize", 0.0) + 1.5
+
+        # Source credibility nudge (Telegram OSINT channels treated as moderate-cost signals).
+        if source and "t.me/" in source.lower() and signals:
+            for key in list(signals):
+                signals[key] = round(signals[key] * 1.05, 3)
+
+        return signals
+
+    def _deviation_score(self, region: str, domain: DomainName, strength: float) -> float:
+        """Deviation from rolling regional norm — herding/coordination detector input."""
+        with self._lock:
+            state = self._region_state(region)
+            baseline = max(state.signal_volume[domain], 1.0)
+            state.signal_volume[domain] += strength
+            state.update_count += 1
+            return float(strength / baseline)
+
+    def bayesian_update(
+        self,
+        region: str,
+        domain: DomainName,
+        evidence_strength: float = 1.0,
+    ) -> float:
+        """
+        Bayesian update: P(distress|evidence) from likelihood table and prior.
+
+        evidence_strength scales how far belief moves toward the likelihood posterior.
+        """
+        domain = domain if domain in _DOMAINS else "financial"
+        lik = self.likelihoods.get(domain, self.likelihoods["financial"])
+
+        with self._lock:
+            state = self._region_state(region)
+            prior = float(state.priors.get(domain, self._base_prior))
+
+            p_e_given_d = lik["distress"]
+            p_e_given_not_d = lik["normal"]
+            p_e = (p_e_given_d * prior) + (p_e_given_not_d * (1.0 - prior))
+
+            if p_e <= 0:
+                posterior = prior
+            else:
+                posterior = (p_e_given_d * prior) / p_e
+
+            scaled = prior + (posterior - prior) * float(evidence_strength)
+            clipped = float(np.clip(scaled, self.settings.min_prob, self.settings.max_prob))
+            state.priors[domain] = clipped
+            return clipped
+
+    def _update_graph(
+        self,
+        region: str,
+        entities: list[str],
+        strength: float,
+        coords: list[float] | None,
+    ) -> None:
+        region_key = str(region or "global").strip().lower() or "global"
+        self.G.add_node(region_key, node_type="region", region=region_key)
+        if coords and len(coords) >= 2:
+            self.G.nodes[region_key]["coords"] = coords
+
+        for entity in entities:
+            entity_key = str(entity).strip()
+            if not entity_key:
+                continue
+            self.G.add_node(entity_key, node_type="entity", region=region_key)
+            self.G.add_edge(
+                region_key,
+                entity_key,
+                weight=float(strength),
+                timestamp=self._utcnow(),
+            )
+
+        for i, e1 in enumerate(entities):
+            for e2 in entities[i + 1 :]:
+                k1, k2 = str(e1).strip(), str(e2).strip()
+                if not k1 or not k2:
+                    continue
+                self.G.add_edge(
+                    k1,
+                    k2,
+                    weight=float(strength),
+                    timestamp=self._utcnow(),
+                )
+
+    def process_feed_item(self, item: dict[str, Any]) -> dict[str, Any]:
+        """Process one normalized feed item and update beliefs + contagion graph."""
+        region = str(item.get("region") or item.get("geotag") or "global").strip().lower()
+        text = str(item.get("text") or "")
+        source = str(item.get("source") or "unknown")
+        explicit_domain = str(item.get("domain") or "").strip().lower()
+        entities = list(item.get("entities") or [])
+        coords = item.get("coords")
+        item_id = str(item.get("id") or f"{source}|{hash(text)}")
+
+        if self.settings.watched_channels:
+            channel = ""
+            for entity in entities:
+                if str(entity).startswith("channel:"):
+                    channel = str(entity).split(":", 1)[-1].lower()
+                    break
+            if channel and channel not in {c.lower() for c in self.settings.watched_channels}:
+                return {
+                    "region": region,
+                    "skipped": True,
+                    "reason": "channel_not_watched",
+                    "risk_score": self.composite_risk(region),
+                    "signals": {},
+                }
+
+        with self._lock:
+            if item_id and item_id in self._seen_item_ids:
+                return {
+                    "region": region,
+                    "skipped": True,
+                    "reason": "duplicate",
+                    "risk_score": self.composite_risk(region),
+                    "signals": {},
+                }
+            if item_id:
+                self._seen_item_ids.add(item_id)
+
+        signals = self.classify_signals(text, source)
+        total_strength = float(sum(signals.values()))
+
+        if total_strength <= 0:
+            return {
+                "region": region,
+                "risk_score": self.composite_risk(region),
+                "signals": {},
+                "contagion_potential": self._get_contagion_score(region),
+            }
+
+        domains_touched: set[DomainName] = set()
+        if explicit_domain in _DOMAINS:
+            domains_touched.add(explicit_domain)
+        for signal_name in signals:
+            domains_touched.add(_SIGNAL_DOMAINS.get(signal_name, explicit_domain or "financial"))
+        if not domains_touched:
+            domains_touched.add("financial")
+
+        evidence_strength = min(
+            total_strength / max(self.settings.evidence_scale, 0.1),
+            self.settings.evidence_cap,
+        )
+
+        posteriors: dict[str, float] = {}
+        deviation = 0.0
+        for domain in domains_touched:
+            prior = self.get_prior(region, domain)
+            deviation = max(deviation, self._deviation_score(region, domain, total_strength))
+            posterior = self.bayesian_update(
+                region=region,
+                domain=domain,
+                evidence_strength=evidence_strength * (1.0 + 0.15 * deviation),
+            )
+            posteriors[domain] = posterior
+
+        if isinstance(coords, (list, tuple)) and len(coords) >= 2:
+            with self._lock:
+                state = self._region_state(region)
+                try:
+                    state.coords = [float(coords[0]), float(coords[1])]
+                except (TypeError, ValueError):
+                    pass
+
+        self._update_graph(region, entities, total_strength, coords if isinstance(coords, list) else None)
+
+        composite = self.composite_risk(region)
+        entry = HistoryEntry(
+            timestamp=self._utcnow(),
+            domain=explicit_domain if explicit_domain in _DOMAINS else next(iter(domains_touched)),
+            signals=signals,
+            strength=total_strength,
+            prior=self._base_prior,
+            posterior=composite,
+            source=source,
+            deviation_score=deviation,
+        )
+        with self._lock:
+            history = self._history[region]
+            history.append(entry)
+            max_hist = max(10, int(self.settings.max_history_per_region))
+            if len(history) > max_hist:
+                self._history[region] = history[-max_hist:]
+
+        logger.info(
+            "GT update region=%s domains=%s composite=%.3f signals=%d deviation=%.2f",
+            region,
+            ",".join(sorted(domains_touched)),
+            composite,
+            len(signals),
+            deviation,
+        )
+
+        return {
+            "region": region,
+            "domains": sorted(domains_touched),
+            "domain_posteriors": posteriors,
+            "risk_score": composite,
+            "signals": signals,
+            "deviation_score": deviation,
+            "contagion_potential": self._get_contagion_score(region),
+            "interpretation": self._interpret_risk(composite),
+        }
+
+    def _interpret_risk(self, risk: float) -> str:
+        threshold = float(self.settings.high_risk_threshold)
+        if risk >= threshold:
+            return (
+                f"Elevated strategic risk ({risk:.2f} ≥ {threshold:.2f}). "
+                "Watch for costly-signal clustering and cross-region contagion."
+            )
+        if risk >= threshold * 0.7:
+            return "Moderate risk — monitor for herding and repeated costly signals."
+        return "Baseline risk — no strong costly-signal cluster detected."
+
+    def _get_contagion_score(self, region: str) -> float:
+        """Graph-based contagion: mean composite risk of graph neighbors."""
+        region_key = str(region or "global").strip().lower() or "global"
+        with self._lock:
+            if region_key not in self.G:
+                return 0.0
+            try:
+                neighbors = list(self.G.neighbors(region_key))
+            except nx.NetworkXError:
+                return 0.0
+            if not neighbors:
+                return 0.0
+            neighbor_risks = [self.composite_risk(str(n)) for n in neighbors]
+            return float(np.mean(neighbor_risks))
+
+    def compute_herding_clusters(self) -> list[dict[str, Any]]:
+        """Louvain community detection on entity graph (coordination/herding proxy)."""
+        with self._lock:
+            if self.G.number_of_edges() == 0:
+                return []
+
+            weighted = nx.Graph()
+            for u, v, data in self.G.edges(data=True):
+                weight = float(data.get("weight") or 0.0)
+                if weight < self.settings.louvain_min_weight:
+                    continue
+                if weighted.has_edge(u, v):
+                    weighted[u][v]["weight"] = weighted[u][v].get("weight", 0.0) + weight
+                else:
+                    weighted.add_edge(u, v, weight=weight)
+
+            if weighted.number_of_edges() == 0:
+                return []
+
+            try:
+                communities = list(nx.community.louvain_communities(weighted, weight="weight", seed=42))
+            except Exception as exc:
+                logger.warning("Louvain clustering failed: %s", exc)
+                return []
+
+            clusters: list[dict[str, Any]] = []
+            for idx, community in enumerate(communities):
+                members = sorted(str(node) for node in community)
+                region_members = [m for m in members if m in self._regions]
+                risks = [self.composite_risk(r) for r in region_members]
+                clusters.append(
+                    {
+                        "cluster_id": idx,
+                        "size": len(members),
+                        "members": members[:50],
+                        "mean_risk": float(np.mean(risks)) if risks else self._base_prior,
+                        "regions": region_members,
+                    }
+                )
+            clusters.sort(key=lambda row: row["mean_risk"], reverse=True)
+            return clusters
+
+    def get_risk_heatmap(self) -> dict[str, Any]:
+        """GeoJSON FeatureCollection for frontend risk overlay."""
+        features: list[dict[str, Any]] = []
+        with self._lock:
+            items = list(self._regions.items())[: max(1, self.settings.max_heatmap_features)]
+
+        for region, state in items:
+            coords = state.coords
+            geometry: dict[str, Any]
+            if coords and len(coords) >= 2:
+                geometry = {"type": "Point", "coordinates": [float(coords[1]), float(coords[0])]}
+            else:
+                geometry = {"type": "Point", "coordinates": [0.0, 0.0]}
+
+            composite = self.composite_risk(region)
+            features.append(
+                {
+                    "type": "Feature",
+                    "properties": {
+                        "region": region,
+                        "risk": round(composite, 4),
+                        "financial": round(float(state.priors.get("financial", self._base_prior)), 4),
+                        "unrest": round(float(state.priors.get("unrest", self._base_prior)), 4),
+                        "conflict": round(float(state.priors.get("conflict", self._base_prior)), 4),
+                        "contagion": round(self._get_contagion_score(region), 4),
+                        "updates": state.update_count,
+                    },
+                    "geometry": geometry,
+                }
+            )
+
+        return {"type": "FeatureCollection", "features": features}
+
+    def get_dossier(self, region: str) -> dict[str, Any]:
+        """Explainable GT rationale and recent signal history for a region."""
+        region_key = str(region or "global").strip().lower() or "global"
+        with self._lock:
+            state = self._region_state(region_key)
+            recent = list(self._history.get(region_key, [])[-10:])
+
+        composite = self.composite_risk(region_key)
+        return {
+            "region": region_key,
+            "current_risk": round(composite, 4),
+            "domain_risks": {
+                domain: round(float(state.priors.get(domain, self._base_prior)), 4)
+                for domain in _DOMAINS
+            },
+            "recent_signals": [
+                {
+                    "timestamp": entry.timestamp,
+                    "domain": entry.domain,
+                    "signals": entry.signals,
+                    "strength": entry.strength,
+                    "posterior": round(entry.posterior, 4),
+                    "source": entry.source,
+                    "deviation_score": round(entry.deviation_score, 3),
+                }
+                for entry in recent
+            ],
+            "contagion_risk": round(self._get_contagion_score(region_key), 4),
+            "herding_clusters": self.compute_herding_clusters()[:5],
+            "interpretation": self._interpret_risk(composite),
+            "scenarios": self._build_scenarios(region_key, composite),
+        }
+
+    def _build_scenarios(self, region: str, composite: float) -> list[dict[str, str]]:
+        threshold = float(self.settings.high_risk_threshold)
+        if composite < threshold * 0.7:
+            return [
+                {
+                    "name": "Status quo",
+                    "summary": "Signals remain diffuse; no coordinated costly-signal cascade.",
+                }
+            ]
+        if composite < threshold:
+            return [
+                {
+                    "name": "Escalation watch",
+                    "summary": "Rising costly-signal density — coordination risk within 4-8 weeks.",
+                },
+                {
+                    "name": "False alarm",
+                    "summary": "Cheap-talk amplification without follow-on costly signals.",
+                },
+            ]
+        return [
+            {
+                "name": "Contagion spread",
+                "summary": "High posterior + graph coupling — adjacent regions likely to update upward.",
+            },
+            {
+                "name": "Localized shock",
+                "summary": "Region-specific distress; contagion limited if graph neighbors stay quiet.",
+            },
+        ]
+
+    def snapshot(self) -> dict[str, Any]:
+        """Serialize engine state for debugging or persistence."""
+        with self._lock:
+            return {
+                "regions": {
+                    region: {
+                        "priors": dict(state.priors),
+                        "coords": state.coords,
+                        "updates": state.update_count,
+                    }
+                    for region, state in self._regions.items()
+                },
+                "graph_nodes": self.G.number_of_nodes(),
+                "graph_edges": self.G.number_of_edges(),
+                "processed_items": len(self._seen_item_ids),
+            }

backend/analytics/historical_events.py

@@ -0,0 +1,649 @@
+"""Curated historical early-warning cases for GT backtesting.
+
+Each positive case bundles pre-crisis costly-signal snippets drawn from documented
+precursors (financial, unrest, conflict). Negative cases are cheap-talk controls.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+CaseKind = Literal["positive", "negative"]
+
+
+@dataclass(frozen=True)
+class BacktestFeed:
+    text: str
+    source: str = "backtest"
+    domain: str = "financial"
+    days_before_event: int = 30
+
+
+@dataclass(frozen=True)
+class HistoricalCase:
+    """Single labeled backtest scenario."""
+
+    case_id: str
+    name: str
+    region: str
+    domain: str
+    kind: CaseKind
+    event_date: str
+    description: str
+    feeds: tuple[BacktestFeed, ...] = field(default_factory=tuple)
+    tags: tuple[str, ...] = field(default_factory=tuple)
+
+    def to_feed_dicts(self) -> list[dict[str, Any]]:
+        items: list[dict[str, Any]] = []
+        for idx, feed in enumerate(self.feeds):
+            items.append(
+                {
+                    "id": f"{self.case_id}-{idx}",
+                    "text": feed.text,
+                    "source": feed.source,
+                    "region": self.region,
+                    "domain": feed.domain or self.domain,
+                    "published": feed.days_before_event,
+                }
+            )
+        return items
+
+
+def _variant_case(case: HistoricalCase, suffix: str, feeds: tuple[BacktestFeed, ...]) -> HistoricalCase:
+    return HistoricalCase(
+        case_id=f"{case.case_id}__{suffix}",
+        name=f"{case.name} ({suffix})",
+        region=case.region,
+        domain=case.domain,
+        kind=case.kind,
+        event_date=case.event_date,
+        description=case.description,
+        feeds=feeds,
+        tags=case.tags + (f"variant:{suffix}",),
+    )
+
+
+def expanded_historical_cases() -> tuple[HistoricalCase, ...]:
+    """Base suite plus paraphrase variants for statistical confidence."""
+    base = list(default_historical_cases())
+    extras: list[HistoricalCase] = []
+
+    variant_feeds: dict[str, tuple[tuple[BacktestFeed, ...], ...]] = {
+        "fin_2008_us": (
+            (
+                BacktestFeed(
+                    "Small businesses turn to payroll loan products as credit lines freeze.",
+                    domain="financial",
+                    days_before_event=100,
+                ),
+                BacktestFeed(
+                    "FDIC monitors liquidity crunch; interbank spreads widen sharply.",
+                    domain="financial",
+                    days_before_event=60,
+                ),
+            ),
+            (
+                BacktestFeed(
+                    "Merchant cash advance volumes spike; payroll loan demand at record highs.",
+                    domain="financial",
+                    days_before_event=80,
+                ),
+                BacktestFeed(
+                    "Money market funds see inflows as deposit flight from regional banks continues.",
+                    domain="financial",
+                    days_before_event=40,
+                ),
+            ),
+        ),
+        "fin_2020_supply": (
+            (
+                BacktestFeed(
+                    "Electronics firms report shipping delay and port congestion across Pearl River Delta.",
+                    domain="financial",
+                    days_before_event=45,
+                ),
+                BacktestFeed(
+                    "Supply chain delay widens; logistics backlog hits automotive suppliers.",
+                    domain="financial",
+                    days_before_event=20,
+                ),
+            ),
+            (
+                BacktestFeed(
+                    "Container shortage fuels shipping delay; supply chain delay indices jump.",
+                    domain="financial",
+                    days_before_event=35,
+                ),
+                BacktestFeed(
+                    "Electronics assemblers warn of logistics backlog as port congestion spreads.",
+                    domain="financial",
+                    days_before_event=20,
+                ),
+                BacktestFeed(
+                    "Automotive suppliers flag supply chain delay after factory shutdowns in Hubei.",
+                    domain="financial",
+                    days_before_event=10,
+                ),
+            ),
+        ),
+        "fin_2022_sanctions": (
+            (
+                BacktestFeed(
+                    "Treasury drafts new sanctions escalation package on energy and finance sectors.",
+                    domain="financial",
+                    days_before_event=30,
+                ),
+                BacktestFeed(
+                    "Capital flight accelerates; elite relocation flights depart Moscow airports.",
+                    domain="financial",
+                    days_before_event=14,
+                ),
+            ),
+        ),
+        "unrest_arab_spring_egypt": (
+            (
+                BacktestFeed(
+                    "Cairo activists schedule mass rally; protest mobilization leaflets distributed.",
+                    domain="unrest",
+                    days_before_event=18,
+                ),
+                BacktestFeed(
+                    "Labor federations call general strike; strike posters cover downtown.",
+                    domain="unrest",
+                    days_before_event=8,
+                ),
+            ),
+        ),
+        "conflict_2022_ukraine": (
+            (
+                BacktestFeed(
+                    "Convoy of armored vehicles confirms troop movement near Sumy Oblast.",
+                    source="t.me/war_monitor",
+                    domain="conflict",
+                    days_before_event=20,
+                ),
+                BacktestFeed(
+                    "GNSS interference warnings follow GPS jamming spike along Belarus border.",
+                    source="t.me/osintdefender",
+                    domain="conflict",
+                    days_before_event=10,
+                ),
+            ),
+            (
+                BacktestFeed(
+                    "Military mobilization notices circulate; troop buildup confirmed by satellite firms.",
+                    domain="conflict",
+                    days_before_event=12,
+                ),
+            ),
+        ),
+        "neg_weather_us": (
+            (
+                BacktestFeed("Autumn foliage peaks in Vermont; pleasant hiking weather continues."),
+                BacktestFeed("County fair announces pie contest and livestock exhibitions."),
+            ),
+            (
+                BacktestFeed("Meteorologists predict mild hurricane season remainder for Gulf Coast."),
+            ),
+        ),
+        "neg_sports_uk": (
+            (
+                BacktestFeed("Rugby Six Nations standings update after weekend fixtures."),
+                BacktestFeed("Local marathon registration opens for charity runners."),
+            ),
+        ),
+        "neg_tech_global": (
+            (
+                BacktestFeed("Chipmaker announces efficiency gains in next-generation processor."),
+                BacktestFeed("Cloud provider opens new green datacenter in Nordic region."),
+            ),
+        ),
+    }
+
+    for case in base:
+        variants = variant_feeds.get(case.case_id, ())
+        for idx, feeds in enumerate(variants):
+            extras.append(_variant_case(case, f"v{idx+1}", feeds))
+
+    # Additional cheap-talk controls to widen negative sample
+    cheap_talk_regions = (
+        ("australia", "Museum opens contemporary art exhibit to strong attendance."),
+        ("spain", "Tomato harvest festival scheduled; regional trains add weekend service."),
+        ("south_korea", "K-pop group announces world tour dates for autumn."),
+        ("mexico", "Coastal cleanup volunteers restore beach habitats before holiday season."),
+        ("sweden", "City council approves bike lane expansion along waterfront."),
+        ("norway", "Salmon exports remain stable; fishing fleets report normal catch volumes."),
+        ("italy", "Truffle festival returns; restaurants publish seasonal tasting menus."),
+        ("poland", "University researchers release open-source astronomy software."),
+        ("thailand", "Monsoon rains ease; rice planting proceeds on normal schedule."),
+        ("vietnam", "Electronics assembly plants report steady export order books."),
+        ("south_africa", "Wildlife reserve reports rising ecotourism bookings."),
+        ("argentina", "Wine harvest festival opens; export cooperatives meet volume targets."),
+        ("netherlands", "Cycling championship draws international teams to canal district."),
+        ("belgium", "Chocolate exporters report stable holiday shipment schedules."),
+        ("portugal", "Offshore wind auction attracts multiple renewable bidders."),
+        ("greece", "Island ferry operators add routes ahead of summer travel season."),
+        ("turkey", "Cotton harvest forecast unchanged; textile orders stable."),
+        ("indonesia", "Volcano monitoring reports routine activity; tourism continues."),
+        ("philippines", "Coconut processors report normal logistics to export markets."),
+        ("malaysia", "Palm oil shipments on schedule; port throughput normal."),
+        ("new_zealand", "Sheep shearing competition draws rural crowds."),
+        ("ireland", "Tech conference highlights open-source database tooling."),
+        ("finland", "Sauna culture festival celebrates heritage with local artisans."),
+        ("denmark", "Wind turbine maintenance contracts renewed on prior terms."),
+        ("austria", "Ski resorts prepare slopes after early snowfall."),
+        ("switzerland", "Watchmakers unveil mechanical movement prototypes at trade fair."),
+        ("czech_republic", "Glassmakers export decorative pieces ahead of holiday season."),
+        ("romania", "Carpathian hiking trails reopen after spring maintenance."),
+        ("hungary", "Thermal bath tourism bookings rise for winter wellness season."),
+        ("peru", "Coffee cooperatives report stable harvest and export schedules."),
+        ("colombia", "Flower exporters prepare Valentine's shipments on normal cadence."),
+        ("morocco", "Citrus harvest meets forecasts; agricultural credit unchanged."),
+        ("kenya", "Tea auction volumes steady; freight routes operate normally."),
+        ("nigeria", "Nollywood studio announces family comedy release dates."),
+        ("ethiopia", "Coffee ceremony festival highlights regional bean varieties."),
+        ("saudi_arabia", "Desert conservation project plants drought-resistant shrubs."),
+        ("uae", "Airport duty-free operators expand luxury retail concourse."),
+        ("qatar", "Stadium operators prepare hospitality packages for sporting events."),
+        ("singapore", "Port authority reports container throughput on seasonal trend."),
+        ("hong_kong", "Art auction previews draw collectors to harborfront gallery."),
+        ("chile", "Vineyard tours report strong bookings ahead of harvest festival weekend."),
+        ("uruguay", "Beef exporters maintain steady shipment schedules to European buyers."),
+        ("iceland", "Geothermal spa resorts report normal winter visitor volumes."),
+        ("luxembourg", "Fund administrators publish routine quarterly disclosure filings."),
+        ("slovakia", "Mountain lodges prepare ski season openings after early snowfall."),
+        ("croatia", "Adriatic ferry operators add summer routes on prior timetable."),
+        ("bulgaria", "Rose oil cooperatives report stable export volumes to fragrance buyers."),
+        ("serbia", "Danube barge traffic proceeds on normal freight schedules."),
+        ("latvia", "Timber mills export lumber on unchanged contract terms."),
+        ("lithuania", "Baltic wind farms complete scheduled turbine maintenance rotations."),
+        ("estonia", "Digital residency applications processed at routine monthly pace."),
+        ("panama", "Canal transit volumes remain on seasonal trend; shipping fees unchanged."),
+    )
+    for idx, (region, text) in enumerate(cheap_talk_regions):
+        extras.append(
+            HistoricalCase(
+                case_id=f"neg_extra_{idx:02d}",
+                name=f"Benign regional news ({region})",
+                region=region,
+                domain="financial",
+                kind="negative",
+                event_date="2020-01-01",
+                description="Expanded cheap-talk control.",
+                feeds=(BacktestFeed(text),),
+                tags=("control", "expanded"),
+            )
+        )
+
+    return tuple(base + extras)
+
+
+def default_historical_cases() -> tuple[HistoricalCase, ...]:
+    """Benchmark suite — expand as new validated precursors are added."""
+    return (
+        # ── Financial distress ─────────────────────────────────────────────
+        HistoricalCase(
+            case_id="fin_2008_us",
+            name="2008 US financial crisis",
+            region="united_states",
+            domain="financial",
+            kind="positive",
+            event_date="2008-09-15",
+            description="Payroll-loan distress, liquidity crunch, and deposit flight precursors.",
+            tags=("2008", "financial", "lehman"),
+            feeds=(
+                BacktestFeed(
+                    "Franchise operators increasingly rely on payroll loan facilities as working capital tightens.",
+                    domain="financial",
+                    days_before_event=120,
+                ),
+                BacktestFeed(
+                    "Regional banks report liquidity crunch; CFOs warn of merchant cash advance reliance.",
+                    domain="financial",
+                    days_before_event=90,
+                ),
+                BacktestFeed(
+                    "Deposit flight accelerates at mid-size lenders; analysts flag bank run risk.",
+                    domain="financial",
+                    days_before_event=45,
+                ),
+            ),
+        ),
+        HistoricalCase(
+            case_id="fin_2020_supply",
+            name="COVID supply-chain shock",
+            region="china",
+            domain="financial",
+            kind="positive",
+            event_date="2020-02-01",
+            description="Port congestion and logistics backlog ahead of global supply shock.",
+            tags=("covid", "supply_chain", "financial"),
+            feeds=(
+                BacktestFeed(
+                    "Major port congestion reported; shipping delay spreads to electronics suppliers.",
+                    domain="financial",
+                    days_before_event=60,
+                ),
+                BacktestFeed(
+                    "Automakers warn of supply chain delay and logistics backlog across Wuhan corridor.",
+                    domain="financial",
+                    days_before_event=30,
+                ),
+                BacktestFeed(
+                    "Factory restarts slip as supply delay and port congestion persist into Q1.",
+                    domain="financial",
+                    days_before_event=14,
+                ),
+            ),
+        ),
+        HistoricalCase(
+            case_id="fin_2022_sanctions",
+            name="Russia sanctions escalation",
+            region="russia",
+            domain="financial",
+            kind="positive",
+            event_date="2022-02-24",
+            description="Sanctions escalation and capital flight ahead of invasion.",
+            tags=("sanctions", "ukraine", "financial"),
+            feeds=(
+                BacktestFeed(
+                    "Western allies prepare new sanctions escalation on major Russian banks.",
+                    domain="financial",
+                    days_before_event=45,
+                ),
+                BacktestFeed(
+                    "Oligarch jet movements suggest elite relocation and capital flight from Moscow.",
+                    domain="financial",
+                    days_before_event=21,
+                ),
+                BacktestFeed(
+                    "Central bank intervenes as new sanctions tighten export controls on finance sector.",
+                    domain="financial",
+                    days_before_event=10,
+                ),
+            ),
+        ),
+        # ── Civil unrest ─────────────────────────────────────────────────
+        HistoricalCase(
+            case_id="unrest_arab_spring_tunisia",
+            name="Arab Spring — Tunisia",
+            region="tunisia",
+            domain="unrest",
+            kind="positive",
+            event_date="2010-12-17",
+            description="Protest mobilization and strike waves before Jasmine Revolution.",
+            tags=("arab_spring", "unrest"),
+            feeds=(
+                BacktestFeed(
+                    "Student groups announce protest mobilization after vendor self-immolation.",
+                    domain="unrest",
+                    days_before_event=14,
+                ),
+                BacktestFeed(
+                    "Mass rally planned in Tunis; general strike called by labor unions.",
+                    domain="unrest",
+                    days_before_event=7,
+                ),
+            ),
+        ),
+        HistoricalCase(
+            case_id="unrest_arab_spring_egypt",
+            name="Arab Spring — Egypt",
+            region="egypt",
+            domain="unrest",
+            kind="positive",
+            event_date="2011-01-25",
+            description="Mobilization spikes and security reshuffles before Tahrir.",
+            tags=("arab_spring", "unrest"),
+            feeds=(
+                BacktestFeed(
+                    "Opposition calls protest mobilization in Cairo; strike notices circulate online.",
+                    domain="unrest",
+                    days_before_event=21,
+                ),
+                BacktestFeed(
+                    "Reports of political purge within interior ministry security apparatus reshuffle.",
+                    domain="unrest",
+                    days_before_event=10,
+                ),
+                BacktestFeed(
+                    "Mass rally and strike coordination spreads; rally posters appear in Alexandria.",
+                    domain="unrest",
+                    days_before_event=5,
+                ),
+            ),
+        ),
+        HistoricalCase(
+            case_id="unrest_2019_chile",
+            name="Chile 2019 metro protests",
+            region="chile",
+            domain="unrest",
+            kind="positive",
+            event_date="2019-10-18",
+            description="Transit fare protests escalate to general strike.",
+            tags=("unrest", "latam"),
+            feeds=(
+                BacktestFeed(
+                    "Students organize mass rally after metro fare hike; protest mobilization trending.",
+                    domain="unrest",
+                    days_before_event=10,
+                ),
+                BacktestFeed(
+                    "Unions announce general strike; rally and strike hashtags spike nationwide.",
+                    domain="unrest",
+                    days_before_event=3,
+                ),
+            ),
+        ),
+        # ── Conflict / war ───────────────────────────────────────────────
+        HistoricalCase(
+            case_id="conflict_2022_ukraine",
+            name="2022 Ukraine invasion buildup",
+            region="ukraine",
+            domain="conflict",
+            kind="positive",
+            event_date="2022-02-24",
+            description="Troop movement and GPS jamming precursors on northern border.",
+            tags=("ukraine", "conflict"),
+            feeds=(
+                BacktestFeed(
+                    "OSINT reports troop movement and armored convoy near Belarus border.",
+                    source="t.me/war_monitor",
+                    domain="conflict",
+                    days_before_event=30,
+                ),
+                BacktestFeed(
+                    "GPS jamming spike reported along northern corridor; GNSS interference warnings issued.",
+                    source="t.me/osintdefender",
+                    domain="conflict",
+                    days_before_event=14,
+                ),
+                BacktestFeed(
+                    "Satellite imagery shows troop buildup; military mobilization near Kharkiv axis.",
+                    domain="conflict",
+                    days_before_event=7,
+                ),
+            ),
+        ),
+        HistoricalCase(
+            case_id="conflict_2023_gaza",
+            name="2023 Gaza conflict escalation",
+            region="israel",
+            domain="conflict",
+            kind="positive",
+            event_date="2023-10-07",
+            description="Ceasefire breakdown and troop movement signals.",
+            tags=("gaza", "conflict"),
+            feeds=(
+                BacktestFeed(
+                    "Border units report troop movement near Gaza envelope; ceasefire broken overnight.",
+                    domain="conflict",
+                    days_before_event=14,
+                ),
+                BacktestFeed(
+                    "Truce end announced; armored convoy repositioning reported by local observers.",
+                    domain="conflict",
+                    days_before_event=5,
+                ),
+            ),
+        ),
+        HistoricalCase(
+            case_id="conflict_2020_nagorno",
+            name="2020 Nagorno-Karabakh renewal",
+            region="armenia",
+            domain="conflict",
+            kind="positive",
+            event_date="2020-09-27",
+            description="Artillery and troop buildup precursors.",
+            tags=("caucasus", "conflict"),
+            feeds=(
+                BacktestFeed(
+                    "Drone strikes reported on line of contact; troop movement on Armenian-Azeri border.",
+                    domain="conflict",
+                    days_before_event=21,
+                ),
+                BacktestFeed(
+                    "GPS jamming spike reported in conflict zone; military mobilization notices leaked.",
+                    domain="conflict",
+                    days_before_event=7,
+                ),
+            ),
+        ),
+        # ── Recent financial / corporate distress pattern ────────────────
+        HistoricalCase(
+            case_id="fin_2023_banking",
+            name="2023 regional banking stress",
+            region="united_states",
+            domain="financial",
+            kind="positive",
+            event_date="2023-03-10",
+            description="Deposit flight and liquidity stress (SVB precursor pattern).",
+            tags=("svb", "financial", "2023"),
+            feeds=(
+                BacktestFeed(
+                    "Tech lenders face deposit flight; VC portfolio companies move payroll to money market funds.",
+                    domain="financial",
+                    days_before_event=21,
+                ),
+                BacktestFeed(
+                    "Analysts warn liquidity crunch at regional banks holding long-duration bonds.",
+                    domain="financial",
+                    days_before_event=7,
+                ),
+            ),
+        ),
+        # ── Negative controls (cheap talk / benign) ─────────────────────
+        HistoricalCase(
+            case_id="neg_weather_us",
+            name="Benign weather coverage",
+            region="united_states",
+            domain="financial",
+            kind="negative",
+            event_date="2019-06-01",
+            description="No costly signals — should remain near baseline.",
+            tags=("control",),
+            feeds=(
+                BacktestFeed("Sunny weekend expected across the Midwest with mild temperatures."),
+                BacktestFeed("Local festival draws crowds; farmers market expands summer hours."),
+            ),
+        ),
+        HistoricalCase(
+            case_id="neg_sports_uk",
+            name="Benign sports coverage",
+            region="uk",
+            domain="unrest",
+            kind="negative",
+            event_date="2018-07-01",
+            description="Sports chatter without mobilization costly signals.",
+            tags=("control",),
+            feeds=(
+                BacktestFeed("Premier league season review: top scorers and transfer rumors."),
+                BacktestFeed("Cricket test match ends early due to rain delay at Lord's."),
+            ),
+        ),
+        HistoricalCase(
+            case_id="neg_tech_global",
+            name="Benign tech product launch",
+            region="global",
+            domain="financial",
+            kind="negative",
+            event_date="2021-09-01",
+            description="Corporate product news without distress markers.",
+            tags=("control",),
+            feeds=(
+                BacktestFeed("Smartphone maker unveils new camera features at annual keynote."),
+                BacktestFeed("Quarterly earnings beat expectations; dividend unchanged."),
+            ),
+        ),
+        HistoricalCase(
+            case_id="neg_tourism_france",
+            name="Benign tourism recovery",
+            region="france",
+            domain="unrest",
+            kind="negative",
+            event_date="2022-08-01",
+            description="Travel sector recovery without unrest signals.",
+            tags=("control",),
+            feeds=(
+                BacktestFeed("Paris hotels report record summer bookings as tourism rebounds."),
+                BacktestFeed("Airline adds routes to Nice and Marseille for holiday travelers."),
+            ),
+        ),
+        HistoricalCase(
+            case_id="neg_science_japan",
+            name="Benign science news",
+            region="japan",
+            domain="conflict",
+            kind="negative",
+            event_date="2020-11-01",
+            description="Research coverage without conflict markers.",
+            tags=("control",),
+            feeds=(
+                BacktestFeed("Astronomy team publishes comet observations from Mount Fuji observatory."),
+                BacktestFeed("Robotics lab demonstrates warehouse automation prototype."),
+            ),
+        ),
+        HistoricalCase(
+            case_id="neg_agriculture_brazil",
+            name="Benign agriculture report",
+            region="brazil",
+            domain="financial",
+            kind="negative",
+            event_date="2017-03-01",
+            description="Commodity harvest update without supply distress.",
+            tags=("control",),
+            feeds=(
+                BacktestFeed("Soybean harvest forecast revised upward; export volumes steady."),
+                BacktestFeed("Coffee cooperative reports normal shipping schedules to European buyers."),
+            ),
+        ),
+        HistoricalCase(
+            case_id="neg_culture_india",
+            name="Benign culture coverage",
+            region="india",
+            domain="unrest",
+            kind="negative",
+            event_date="2016-11-01",
+            description="Festival coverage without mobilization.",
+            tags=("control",),
+            feeds=(
+                BacktestFeed("Diwali celebrations begin; cities decorate markets with lights."),
+                BacktestFeed("Film festival opens in Mumbai with premiere screenings."),
+            ),
+        ),
+        HistoricalCase(
+            case_id="neg_infrastructure_canada",
+            name="Benign infrastructure ribbon-cutting",
+            region="canada",
+            domain="financial",
+            kind="negative",
+            event_date="2015-05-01",
+            description="Municipal news without financial stress.",
+            tags=("control",),
+            feeds=(
+                BacktestFeed("New light-rail segment opens on schedule; commute times improve."),
+                BacktestFeed("Municipal bond issuance funds library renovation at prior rates."),
+            ),
+        ),
+    )

backend/analytics/integration.py

@@ -0,0 +1,198 @@
+"""Singleton GT engine and feed-batch integration hooks."""
+
+from __future__ import annotations
+
+import logging
+import threading
+from datetime import datetime, timezone
+from typing import Any
+
+from analytics.feed_adapter import iter_gdelt_features, iter_news_items, iter_telegram_posts
+from analytics.gt_early_warning import GT_EarlyWarning
+from analytics.settings import gt_analytics_enabled, get_gt_settings, gt_engine_operational, gt_louvain_enabled, gt_scheduled_ingest_enabled
+from services.fetchers._store import _data_lock, _mark_fresh, latest_data
+
+logger = logging.getLogger(__name__)
+
+_engine: GT_EarlyWarning | None = None
+_engine_lock = threading.Lock()
+
+
+def get_gt_engine() -> GT_EarlyWarning | None:
+    """Return the shared engine when analytics are enabled and runtime allows it."""
+    global _engine
+    if not gt_engine_operational():
+        return None
+    with _engine_lock:
+        if _engine is None:
+            _engine = GT_EarlyWarning(get_gt_settings())
+            logger.info("Strategic Risk Analytics engine initialized")
+        return _engine
+
+
+def reset_gt_engine() -> None:
+    """Reset singleton — intended for tests."""
+    global _engine
+    get_gt_settings.cache_clear()
+    with _engine_lock:
+        _engine = None
+
+
+def process_feed_item(item: dict[str, Any]) -> dict[str, Any] | None:
+    """Process a normalized feed item if analytics are enabled."""
+    engine = get_gt_engine()
+    if engine is None:
+        return None
+    try:
+        return engine.process_feed_item(item)
+    except Exception:
+        logger.exception("GT process_feed_item failed")
+        return None
+
+
+def _persist_gt_snapshot(
+    engine: GT_EarlyWarning,
+    *,
+    processed: int,
+    sample: list[dict[str, Any]] | None = None,
+) -> dict[str, Any]:
+    timestamp = datetime.now(timezone.utc).isoformat()
+    heatmap = engine.get_risk_heatmap()
+    micro_summary: dict[str, Any] = {}
+    try:
+        from analytics.micro_rolling import capture_daily_readings, enrich_heatmap_features
+
+        micro_summary = capture_daily_readings(engine)
+        heatmap = enrich_heatmap_features(heatmap)
+    except Exception:
+        logger.exception("GT micro rolling capture failed")
+
+    clusters = engine.compute_herding_clusters()
+    from analytics.gt_alerts import parse_heatmap_alerts
+
+    _, plotted_regions = parse_heatmap_alerts(heatmap)
+    with engine._lock:  # noqa: SLF001 — snapshot meta
+        engine_regions = len(engine._regions)
+    settings = get_gt_settings()
+    payload = {
+        "enabled": True,
+        "timestamp": timestamp,
+        "processed": processed,
+        "heatmap": heatmap,
+        "clusters": clusters,
+        "sample": list(sample or [])[:5],
+        "regions": len(heatmap.get("features") or []),
+        "micro": micro_summary,
+        "meta": {
+            "tracked_regions": len(heatmap.get("features") or []),
+            "engine_regions": engine_regions,
+            "plotted_regions": plotted_regions,
+            "max_regions": settings.max_heatmap_features,
+        },
+    }
+    with _data_lock:
+        latest_data["gt_risk"] = payload
+    _mark_fresh("gt_risk")
+    return payload
+
+
+def refresh_from_latest_data(
+    data_snapshot: dict[str, Any],
+    *,
+    persist: bool = True,
+) -> dict[str, Any]:
+    """
+    Batch-ingest recent intel layers from the shared data store.
+
+    Intended to run after telegram/news/gdelt fetch cycles (near-real-time).
+    """
+    engine = get_gt_engine()
+    if engine is None:
+        return {"enabled": False, "processed": 0}
+
+    processed = 0
+    results: list[dict[str, Any]] = []
+
+    for item in iter_telegram_posts(data_snapshot.get("telegram_osint")):
+        result = engine.process_feed_item(item)
+        if result and not result.get("skipped"):
+            processed += 1
+            results.append(result)
+
+    for item in iter_news_items(data_snapshot.get("news")):
+        result = engine.process_feed_item(item)
+        if result and not result.get("skipped"):
+            processed += 1
+            if len(results) < 5:
+                results.append(result)
+
+    for item in iter_gdelt_features(data_snapshot.get("gdelt")):
+        result = engine.process_feed_item(item)
+        if result and not result.get("skipped"):
+            processed += 1
+
+    logger.info("GT refresh processed %d items", processed)
+    summary = {
+        "enabled": True,
+        "processed": processed,
+        "sample": results[:5],
+        "heatmap_features": len(engine.get_risk_heatmap().get("features") or []),
+    }
+    if persist:
+        snapshot = _persist_gt_snapshot(engine, processed=processed, sample=results)
+        summary["timestamp"] = snapshot.get("timestamp")
+        summary["clusters"] = len(snapshot.get("clusters") or [])
+    return summary
+
+
+def recompute_gt_herding_clusters() -> dict[str, Any]:
+    """Louvain community pass — run on a schedule independent of feed ingest."""
+    if not gt_louvain_enabled():
+        return {"enabled": False, "clusters": 0, "reason": "louvain_disabled_on_lean_profile"}
+
+    engine = get_gt_engine()
+    if engine is None:
+        return {"enabled": False, "clusters": 0}
+
+    clusters = engine.compute_herding_clusters()
+    timestamp = datetime.now(timezone.utc).isoformat()
+    with _data_lock:
+        current = dict(latest_data.get("gt_risk") or {})
+        current["clusters"] = clusters
+        current["clusters_updated"] = timestamp
+        current["enabled"] = True
+        latest_data["gt_risk"] = current
+    _mark_fresh("gt_risk")
+    logger.info("GT Louvain recompute: %d clusters", len(clusters))
+    return {"enabled": True, "clusters": len(clusters), "timestamp": timestamp}
+
+
+def maybe_refresh_gt_analytics() -> None:
+    """Hook for data_fetcher — no-op when analytics are disabled or lean-gated."""
+    if not gt_scheduled_ingest_enabled():
+        return
+    try:
+        with _data_lock:
+            snapshot = dict(latest_data)
+        refresh_from_latest_data(snapshot, persist=True)
+    except Exception:
+        logger.exception("GT analytics refresh failed")
+
+
+def maybe_freeze_gt_weekly_snapshot() -> None:
+    """Hook for weekly scheduler — freeze operational backtest snapshot."""
+    if not gt_engine_operational():
+        return
+    try:
+        from analytics.rolling_backtest import freeze_weekly_snapshot
+
+        result = freeze_weekly_snapshot(frozen_by="scheduler")
+        if result.get("created"):
+            logger.info(
+                "GT rolling freeze: week=%s regions=%s alerts=%s",
+                result.get("week_id"),
+                result.get("region_count"),
+                result.get("alert_count"),
+            )
+    except Exception:
+        logger.exception("GT rolling weekly freeze failed")

backend/analytics/micro_rolling.py

@@ -0,0 +1,361 @@
+"""Micro rolling 3-day average — fast ignition signal alongside weekly macro."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from datetime import date, datetime, timedelta, timezone
+from typing import Any
+
+from analytics.daily_store import (
+    DailyRegionReading,
+    DailySnapshot,
+    date_id,
+    list_daily_ids,
+    load_daily,
+    save_daily,
+    utc_now_iso,
+    utc_today,
+)
+from analytics.gt_early_warning import GT_EarlyWarning
+from analytics.rolling_backtest import rolling_alert_threshold
+
+DEFAULT_WINDOW_DAYS = 3
+DEFAULT_IGNITION_DELTA = 0.10
+
+
+def _env_int(name: str, default: int) -> int:
+    raw = str(os.environ.get(name, "")).strip()
+    if not raw:
+        return default
+    try:
+        return max(1, int(raw))
+    except ValueError:
+        return default
+
+
+def _env_float(name: str, default: float) -> float:
+    raw = str(os.environ.get(name, "")).strip()
+    if not raw:
+        return default
+    try:
+        return float(raw)
+    except ValueError:
+        return default
+
+
+def micro_window_days() -> int:
+    return _env_int("GT_MICRO_ROLLING_DAYS", DEFAULT_WINDOW_DAYS)
+
+
+def ignition_delta() -> float:
+    return _env_float("GT_MICRO_IGNITION_DELTA", DEFAULT_IGNITION_DELTA)
+
+
+def _peak_score(
+    *,
+    composite: float,
+    financial: float,
+    unrest: float,
+    conflict: float,
+) -> float:
+    return max(composite, financial, unrest, conflict)
+
+
+def _region_reading_from_feature(
+    feature: dict[str, Any],
+    *,
+    captured_at: str,
+) -> DailyRegionReading | None:
+    props = feature.get("properties") or {}
+    region = str(props.get("region") or "").strip().lower()
+    if not region:
+        return None
+    composite = float(props.get("risk") or props.get("composite_risk") or 0.0)
+    financial = float(props.get("financial") or 0.0)
+    unrest = float(props.get("unrest") or 0.0)
+    conflict = float(props.get("conflict") or 0.0)
+    peak = _peak_score(
+        composite=composite,
+        financial=financial,
+        unrest=unrest,
+        conflict=conflict,
+    )
+    return DailyRegionReading(
+        region=region,
+        composite_risk=composite,
+        financial=financial,
+        unrest=unrest,
+        conflict=conflict,
+        peak_score=peak,
+        readings=1,
+        last_captured_at=captured_at,
+    )
+
+
+def capture_daily_readings(
+    engine: GT_EarlyWarning,
+    *,
+    when: date | None = None,
+) -> dict[str, Any]:
+    """
+    Upsert today's regional readings from the live heatmap.
+
+    Each GT refresh updates the current day's latest scores (rolling window
+    uses one value per calendar day).
+    """
+    day = when or utc_today()
+    day_key = date_id(day)
+    captured_at = utc_now_iso()
+    heatmap = engine.get_risk_heatmap()
+    existing = load_daily(day) or DailySnapshot(date=day_key, regions={})
+
+    updated = 0
+    for feature in heatmap.get("features") or []:
+        if not isinstance(feature, dict):
+            continue
+        reading = _region_reading_from_feature(feature, captured_at=captured_at)
+        if reading is None:
+            continue
+        prior = existing.regions.get(reading.region)
+        if prior is None:
+            existing.regions[reading.region] = reading
+            updated += 1
+            continue
+        prior.composite_risk = reading.composite_risk
+        prior.financial = reading.financial
+        prior.unrest = reading.unrest
+        prior.conflict = reading.conflict
+        prior.peak_score = max(prior.peak_score, reading.peak_score)
+        prior.readings += 1
+        prior.last_captured_at = captured_at
+        updated += 1
+
+    existing.last_updated_at = captured_at
+    save_daily(existing)
+    return {
+        "date": day_key,
+        "regions": len(existing.regions),
+        "updated": updated,
+        "captured_at": captured_at,
+    }
+
+
+@dataclass(frozen=True)
+class MicroRegionView:
+    region: str
+    spot_risk: float
+    risk_3d_avg: float
+    risk_delta: float
+    days_in_window: int
+    day_scores: tuple[float, ...]
+    alerted_spot: bool
+    alerted_3d: bool
+    ignition: bool
+    financial: float
+    unrest: float
+    conflict: float
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "region": self.region,
+            "spot_risk": round(self.spot_risk, 4),
+            "risk_3d_avg": round(self.risk_3d_avg, 4),
+            "risk_delta": round(self.risk_delta, 4),
+            "days_in_window": self.days_in_window,
+            "day_scores": [round(score, 4) for score in self.day_scores],
+            "alerted_spot": self.alerted_spot,
+            "alerted_3d": self.alerted_3d,
+            "ignition": self.ignition,
+            "financial": round(self.financial, 4),
+            "unrest": round(self.unrest, 4),
+            "conflict": round(self.conflict, 4),
+        }
+
+
+def _day_offsets(window_days: int) -> list[int]:
+    # Today + prior (window_days - 1) days.
+    return list(range(window_days - 1, -1, -1))
+
+
+def _historical_dates(as_of: date, window_days: int) -> list[date]:
+    return [as_of - timedelta(days=offset) for offset in _day_offsets(window_days)]
+
+
+def compute_micro_view(
+    region: str,
+    *,
+    as_of: date | None = None,
+    window_days: int | None = None,
+    alert_threshold: float | None = None,
+    spot_reading: DailyRegionReading | None = None,
+) -> MicroRegionView | None:
+    """Compute rolling N-day average and ignition vs spot for one region."""
+    region_key = str(region or "").strip().lower()
+    if not region_key:
+        return None
+
+    today = as_of or utc_today()
+    window = window_days or micro_window_days()
+    threshold = float(alert_threshold if alert_threshold is not None else rolling_alert_threshold())
+    delta_min = ignition_delta()
+
+    day_scores: list[float] = []
+    latest: DailyRegionReading | None = spot_reading
+
+    for day in _historical_dates(today, window):
+        snap = load_daily(day)
+        if snap is None:
+            continue
+        row = snap.regions.get(region_key)
+        if row is None:
+            continue
+        day_scores.append(row.peak_score)
+        if day == today:
+            latest = row
+
+    if latest is None and day_scores:
+        # Spot may come from yesterday if today not captured yet.
+        snap = load_daily(today)
+        if snap:
+            latest = snap.regions.get(region_key)
+
+    if latest is None and not day_scores:
+        return None
+
+    spot = float(latest.peak_score if latest else (day_scores[-1] if day_scores else 0.0))
+    avg = sum(day_scores) / len(day_scores) if day_scores else spot
+    risk_delta = spot - avg
+    ignition = risk_delta >= delta_min and spot >= threshold * 0.75
+
+    return MicroRegionView(
+        region=region_key,
+        spot_risk=spot,
+        risk_3d_avg=avg,
+        risk_delta=risk_delta,
+        days_in_window=len(day_scores),
+        day_scores=tuple(day_scores),
+        alerted_spot=spot >= threshold,
+        alerted_3d=avg >= threshold,
+        ignition=ignition,
+        financial=float(latest.financial if latest else 0.0),
+        unrest=float(latest.unrest if latest else 0.0),
+        conflict=float(latest.conflict if latest else 0.0),
+    )
+
+
+def compute_all_micro_views(
+    *,
+    as_of: date | None = None,
+    window_days: int | None = None,
+    alert_threshold: float | None = None,
+) -> list[MicroRegionView]:
+    """Build micro views for all regions seen in the rolling window."""
+    today = as_of or utc_today()
+    window = window_days or micro_window_days()
+    regions: set[str] = set()
+
+    for day in _historical_dates(today, window):
+        snap = load_daily(day)
+        if snap is None:
+            continue
+        regions.update(snap.regions.keys())
+
+    views: list[MicroRegionView] = []
+    for region in regions:
+        view = compute_micro_view(
+            region,
+            as_of=today,
+            window_days=window,
+            alert_threshold=alert_threshold,
+        )
+        if view is not None:
+            views.append(view)
+
+    views.sort(key=lambda row: (row.ignition, row.risk_delta, row.spot_risk), reverse=True)
+    return views
+
+
+def enrich_heatmap_features(
+    heatmap: dict[str, Any],
+    *,
+    as_of: date | None = None,
+    window_days: int | None = None,
+    alert_threshold: float | None = None,
+) -> dict[str, Any]:
+    """Attach micro rolling fields to heatmap GeoJSON features."""
+    threshold = float(alert_threshold if alert_threshold is not None else rolling_alert_threshold())
+    window = window_days or micro_window_days()
+    features = heatmap.get("features") or []
+    enriched: list[dict[str, Any]] = []
+
+    for feature in features:
+        if not isinstance(feature, dict):
+            continue
+        props = dict(feature.get("properties") or {})
+        region = str(props.get("region") or "").strip().lower()
+        view = compute_micro_view(
+            region,
+            as_of=as_of,
+            window_days=window,
+            alert_threshold=threshold,
+        ) if region else None
+
+        if view is not None:
+            props["risk_spot"] = view.spot_risk
+            props["risk_3d_avg"] = view.risk_3d_avg
+            props["risk_delta"] = view.risk_delta
+            props["micro_days"] = view.days_in_window
+            props["micro_ignition"] = view.ignition
+            props["alerted_3d"] = view.alerted_3d
+            props["day_scores"] = list(view.day_scores)
+
+        enriched.append({**feature, "properties": props})
+
+    return {
+        **heatmap,
+        "features": enriched,
+        "micro_window_days": window,
+        "micro_alert_threshold": threshold,
+    }
+
+
+def micro_rolling_report(
+    *,
+    as_of: date | None = None,
+    window_days: int | None = None,
+    limit: int = 15,
+) -> dict[str, Any]:
+    """API/OpenClaw payload for micro rolling 3-day context."""
+    today = as_of or utc_today()
+    window = window_days or micro_window_days()
+    threshold = rolling_alert_threshold()
+    views = compute_all_micro_views(
+        as_of=today,
+        window_days=window,
+        alert_threshold=threshold,
+    )
+    ignitions = [row for row in views if row.ignition]
+    alerted_3d = [row for row in views if row.alerted_3d]
+    top = views[: max(1, limit)]
+
+    stored_days = list_daily_ids(newest_first=True, limit=window)
+    return {
+        "mode": "micro_rolling",
+        "window_days": window,
+        "alert_threshold": threshold,
+        "ignition_delta": ignition_delta(),
+        "as_of": date_id(today),
+        "days_stored": len(stored_days),
+        "stored_dates": stored_days,
+        "regions_tracked": len(views),
+        "ignition_count": len(ignitions),
+        "alerted_3d_count": len(alerted_3d),
+        "ignitions": [row.to_dict() for row in ignitions[:limit]],
+        "top_regions": [row.to_dict() for row in top],
+        "note": (
+            f"Micro view: {window}-day rolling average vs spot risk. "
+            "Ignition = spot jumped above the rolling baseline (events that flare fast). "
+            "Macro week-over-week validation remains on /api/analytics/rolling."
+        ),
+    }

backend/analytics/rolling_backtest.py

@@ -0,0 +1,382 @@
+"""Rolling weekly operational validation for Strategic Risk Analytics.
+
+Freezes live GT scores each ISO week, accepts delayed outcome labels, and
+scores prior-week predictions with accuracy + Wilson 95% CI. Unlike the
+static historical benchmark, this measures forward operational usefulness.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from datetime import date, datetime, timezone
+from typing import Any, Literal
+
+from analytics.backtest import DEFAULT_BACKTEST_ALERT_THRESHOLD, wilson_interval
+from analytics.gt_early_warning import GT_EarlyWarning
+from analytics.integration import get_gt_engine
+from analytics.weekly_store import (
+    VALID_LABELS,
+    LabelName,
+    RegionSnapshot,
+    WeeklySnapshot,
+    list_week_ids,
+    load_week,
+    save_week,
+    utc_now_iso,
+)
+
+MIN_LABELED_FOR_TREND = 5
+
+
+def _env_float(name: str, default: float) -> float:
+    raw = str(os.environ.get(name, "")).strip()
+    if not raw:
+        return default
+    try:
+        return float(raw)
+    except ValueError:
+        return default
+
+
+def rolling_alert_threshold() -> float:
+    """Fixed operational alert cutoff — not retroactively tuned per week."""
+    return _env_float("GT_ROLLING_ALERT_THRESHOLD", DEFAULT_BACKTEST_ALERT_THRESHOLD)
+
+
+def iso_week_id(when: datetime | date | None = None) -> str:
+    """Return ISO week id, e.g. ``2026-W24``."""
+    if when is None:
+        when = datetime.now(timezone.utc)
+    if isinstance(when, datetime):
+        when = when.date()
+    year, week, _ = when.isocalendar()
+    return f"{year}-W{week:02d}"
+
+
+def _region_rows_from_engine(
+    engine: GT_EarlyWarning,
+    *,
+    alert_threshold: float,
+) -> list[RegionSnapshot]:
+    heatmap = engine.get_risk_heatmap()
+    rows: list[RegionSnapshot] = []
+    for feature in heatmap.get("features") or []:
+        if not isinstance(feature, dict):
+            continue
+        props = feature.get("properties") or {}
+        region = str(props.get("region") or "").strip().lower()
+        if not region:
+            continue
+        composite = float(props.get("risk") or 0.0)
+        financial = float(props.get("financial") or 0.0)
+        unrest = float(props.get("unrest") or 0.0)
+        conflict = float(props.get("conflict") or 0.0)
+        peak_score = max(composite, financial, unrest, conflict)
+        rows.append(
+            RegionSnapshot(
+                region=region,
+                composite_risk=composite,
+                financial=financial,
+                unrest=unrest,
+                conflict=conflict,
+                alerted=peak_score >= alert_threshold,
+                label="pending",
+            )
+        )
+    rows.sort(key=lambda row: row.composite_risk, reverse=True)
+    return rows
+
+
+@dataclass(frozen=True)
+class WeekScore:
+    week_id: str
+    frozen_at: str
+    alert_threshold: float
+    total_regions: int
+    labeled: int
+    pending: int
+    alerted: int
+    correct: int
+    accuracy: float
+    confidence_rate: float
+    wilson_lower_95: float
+    wilson_upper_95: float
+    true_positives: int
+    true_negatives: int
+    false_positives: int
+    false_negatives: int
+    sensitivity: float
+    specificity: float
+    scorable: bool
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "week_id": self.week_id,
+            "frozen_at": self.frozen_at,
+            "alert_threshold": round(self.alert_threshold, 4),
+            "total_regions": self.total_regions,
+            "labeled": self.labeled,
+            "pending": self.pending,
+            "alerted": self.alerted,
+            "correct": self.correct,
+            "accuracy": round(self.accuracy, 4),
+            "confidence_rate": round(self.confidence_rate, 4),
+            "wilson_lower_95": round(self.wilson_lower_95, 4),
+            "wilson_upper_95": round(self.wilson_upper_95, 4),
+            "true_positives": self.true_positives,
+            "true_negatives": self.true_negatives,
+            "false_positives": self.false_positives,
+            "false_negatives": self.false_negatives,
+            "sensitivity": round(self.sensitivity, 4),
+            "specificity": round(self.specificity, 4),
+            "scorable": self.scorable,
+        }
+
+
+def _predicted_positive(row: RegionSnapshot) -> bool:
+    return row.alerted
+
+
+def _actual_positive(label: LabelName) -> bool:
+    return label == "true_escalation"
+
+
+def _is_correct(row: RegionSnapshot) -> bool:
+    if row.label == "pending":
+        return False
+    predicted = _predicted_positive(row)
+    if row.label == "true_escalation":
+        return predicted
+    if row.label in ("false_alarm", "benign"):
+        return not predicted
+    return False
+
+
+def score_week(snapshot: WeeklySnapshot) -> WeekScore:
+    """Score a frozen week against delayed labels (pending rows excluded)."""
+    labeled_rows = [row for row in snapshot.regions if row.label != "pending"]
+    pending = len(snapshot.regions) - len(labeled_rows)
+
+    tp = sum(
+        1
+        for row in labeled_rows
+        if row.alerted and row.label == "true_escalation"
+    )
+    tn = sum(
+        1
+        for row in labeled_rows
+        if not row.alerted and row.label in ("benign", "false_alarm")
+    )
+    fp = sum(
+        1
+        for row in labeled_rows
+        if row.alerted and row.label in ("false_alarm", "benign")
+    )
+    fn = sum(
+        1
+        for row in labeled_rows
+        if not row.alerted and row.label == "true_escalation"
+    )
+
+    correct = tp + tn
+    total = len(labeled_rows)
+    accuracy = correct / total if total else 0.0
+    lower, upper = wilson_interval(correct, total)
+
+    pos_total = sum(1 for row in labeled_rows if _actual_positive(row.label))  # type: ignore[arg-type]
+    neg_total = total - pos_total
+    pred_pos = sum(1 for row in labeled_rows if row.alerted)
+    pred_neg = total - pred_pos
+
+    sensitivity = tp / pos_total if pos_total else 0.0
+    specificity = tn / pred_neg if pred_neg else (1.0 if tn == total and total else 0.0)
+
+    return WeekScore(
+        week_id=snapshot.week_id,
+        frozen_at=snapshot.frozen_at,
+        alert_threshold=snapshot.alert_threshold,
+        total_regions=len(snapshot.regions),
+        labeled=total,
+        pending=pending,
+        alerted=sum(1 for row in snapshot.regions if row.alerted),
+        correct=correct,
+        accuracy=accuracy,
+        confidence_rate=lower,
+        wilson_lower_95=lower,
+        wilson_upper_95=upper,
+        true_positives=tp,
+        true_negatives=tn,
+        false_positives=fp,
+        false_negatives=fn,
+        sensitivity=sensitivity,
+        specificity=specificity,
+        scorable=total >= MIN_LABELED_FOR_TREND,
+    )
+
+
+def freeze_weekly_snapshot(
+    *,
+    week_id: str | None = None,
+    alert_threshold: float | None = None,
+    force: bool = False,
+    frozen_by: str = "system",
+    engine: GT_EarlyWarning | None = None,
+) -> dict[str, Any]:
+    """
+    Capture current GT heatmap as an immutable weekly operational snapshot.
+
+    Idempotent per week unless ``force=True``.
+    """
+    resolved_engine = engine or get_gt_engine()
+    if resolved_engine is None:
+        return {"ok": False, "detail": "GT analytics engine unavailable"}
+
+    resolved_week = week_id or iso_week_id()
+    threshold = float(
+        alert_threshold if alert_threshold is not None else rolling_alert_threshold()
+    )
+
+    existing = load_week(resolved_week)
+    if existing and existing.regions and not force:
+        score = score_week(existing)
+        return {
+            "ok": True,
+            "created": False,
+            "week_id": resolved_week,
+            "snapshot": existing.to_dict(),
+            "score": score.to_dict(),
+        }
+
+    regions = _region_rows_from_engine(resolved_engine, alert_threshold=threshold)
+    snapshot = WeeklySnapshot(
+        week_id=resolved_week,
+        frozen_at=utc_now_iso(),
+        alert_threshold=threshold,
+        regions=regions,
+        frozen_by=frozen_by,
+    )
+    save_week(snapshot)
+    score = score_week(snapshot)
+    return {
+        "ok": True,
+        "created": True,
+        "week_id": resolved_week,
+        "snapshot": snapshot.to_dict(),
+        "score": score.to_dict(),
+        "alert_count": sum(1 for row in regions if row.alerted),
+        "region_count": len(regions),
+    }
+
+
+def label_regions(
+    week_id: str,
+    labels: list[dict[str, Any]],
+    *,
+    labeled_by: str = "operator",
+) -> dict[str, Any]:
+    """Apply delayed outcome labels to a frozen week."""
+    snapshot = load_week(week_id)
+    if snapshot is None:
+        return {"ok": False, "detail": f"Week {week_id} not found"}
+
+    by_region = {row.region: row for row in snapshot.regions}
+    updated = 0
+    skipped: list[str] = []
+    now = utc_now_iso()
+
+    for entry in labels:
+        if not isinstance(entry, dict):
+            continue
+        region = str(entry.get("region") or "").strip().lower()
+        label = str(entry.get("label") or "").strip().lower()
+        if not region or label not in VALID_LABELS or label == "pending":
+            if region:
+                skipped.append(region)
+            continue
+        row = by_region.get(region)
+        if row is None:
+            skipped.append(region)
+            continue
+        row.label = label  # type: ignore[assignment]
+        row.labeled_at = now
+        notes = entry.get("notes")
+        if notes is not None:
+            row.notes = str(notes)
+        updated += 1
+
+    save_week(snapshot)
+    score = score_week(snapshot)
+    return {
+        "ok": True,
+        "week_id": week_id,
+        "updated": updated,
+        "skipped": skipped,
+        "labeled_by": labeled_by,
+        "score": score.to_dict(),
+    }
+
+
+def label_region(
+    week_id: str,
+    region: str,
+    label: LabelName,
+    *,
+    notes: str = "",
+    labeled_by: str = "operator",
+) -> dict[str, Any]:
+    return label_regions(
+        week_id,
+        [{"region": region, "label": label, "notes": notes}],
+        labeled_by=labeled_by,
+    )
+
+
+def rolling_trend(*, weeks: int = 8) -> list[WeekScore]:
+    """Return scored weeks newest-first (only weeks with stored snapshots)."""
+    ids = list_week_ids(newest_first=True)[: max(1, weeks)]
+    scores: list[WeekScore] = []
+    for week_id in ids:
+        snapshot = load_week(week_id)
+        if snapshot is None:
+            continue
+        scores.append(score_week(snapshot))
+    return scores
+
+
+def rolling_report(*, weeks: int = 8, target_confidence: float = 0.80) -> dict[str, Any]:
+    """Aggregate operational validation trend for API / OpenClaw."""
+    threshold = rolling_alert_threshold()
+    trend = rolling_trend(weeks=weeks)
+    scorable = [row for row in trend if row.scorable]
+
+    latest = scorable[0] if scorable else (trend[0] if trend else None)
+    accuracy_series = [
+        {"week_id": row.week_id, "accuracy": round(row.accuracy, 4), "labeled": row.labeled}
+        for row in reversed(scorable)
+    ]
+
+    improving = False
+    if len(scorable) >= 2:
+        improving = scorable[0].accuracy >= scorable[1].accuracy
+
+    return {
+        "mode": "rolling_operational",
+        "alert_threshold": threshold,
+        "target_confidence": target_confidence,
+        "weeks_requested": weeks,
+        "weeks_stored": len(trend),
+        "weeks_scorable": len(scorable),
+        "min_labeled_per_week": MIN_LABELED_FOR_TREND,
+        "latest": latest.to_dict() if latest else None,
+        "trend": [row.to_dict() for row in trend],
+        "accuracy_series": accuracy_series,
+        "improving_vs_prior": improving,
+        "meets_target": bool(
+            latest and latest.scorable and latest.confidence_rate >= target_confidence
+        ),
+        "note": (
+            "Operational metric: scores frozen weekly predictions against delayed "
+            "labels. Unlike the static benchmark, this measures live forward utility."
+        ),
+    }

backend/analytics/settings.py

@@ -0,0 +1,158 @@
+"""Configuration for Strategic Risk Analytics (feature-flagged)."""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass, field
+from functools import lru_cache
+from typing import Any
+
+
+def _env_bool(name: str, default: bool = False) -> bool:
+    raw = str(os.environ.get(name, "")).strip().lower()
+    if not raw:
+        return default
+    return raw not in {"0", "false", "no", "off"}
+
+
+def _env_float(name: str, default: float) -> float:
+    raw = str(os.environ.get(name, "")).strip()
+    if not raw:
+        return default
+    try:
+        return float(raw)
+    except ValueError:
+        return default
+
+
+def _env_int(name: str, default: int) -> int:
+    raw = str(os.environ.get(name, "")).strip()
+    if not raw:
+        return default
+    try:
+        return int(raw)
+    except ValueError:
+        return default
+
+
+def _parse_signal_weights(raw: str) -> dict[str, float]:
+    if not raw.strip():
+        return {}
+    try:
+        parsed = json.loads(raw)
+        if isinstance(parsed, dict):
+            return {str(k): float(v) for k, v in parsed.items()}
+    except (json.JSONDecodeError, TypeError, ValueError):
+        pass
+    weights: dict[str, float] = {}
+    for part in raw.split(","):
+        piece = part.strip()
+        if not piece or "=" not in piece:
+            continue
+        key, value = piece.split("=", 1)
+        try:
+            weights[key.strip()] = float(value.strip())
+        except ValueError:
+            continue
+    return weights
+
+
+def resolve_gt_profile() -> str:
+    from services.runtime_profile import resolve_profile_name
+
+    return resolve_profile_name()
+
+
+def gt_analytics_ack_low_cpu() -> bool:
+    return _env_bool("GT_ANALYTICS_ACK_LOW_CPU", default=False)
+
+
+def gt_engine_operational() -> bool:
+    """Full GT engine (scheduled ingest, heatmap, Louvain) — not watchdog-only."""
+    if not get_gt_settings().enabled:
+        return False
+    if resolve_gt_profile() == "lean" and not gt_analytics_ack_low_cpu():
+        return False
+    return True
+
+
+def gt_scheduled_ingest_enabled() -> bool:
+    return gt_engine_operational()
+
+
+def gt_louvain_enabled() -> bool:
+    return gt_engine_operational()
+
+
+@dataclass(frozen=True)
+class GTAnalyticsSettings:
+    enabled: bool = False
+    profile: str = "standard"
+    base_prior: float = 0.15
+    evidence_cap: float = 3.0
+    evidence_scale: float = 5.0
+    min_prob: float = 0.01
+    max_prob: float = 0.99
+    high_risk_threshold: float = 0.6
+    max_history_per_region: int = 200
+    max_heatmap_features: int = 500
+    louvain_min_weight: float = 0.5
+    louvain_interval_minutes: int = 30
+    signal_weight_overrides: dict[str, float] = field(default_factory=dict)
+    watched_channels: tuple[str, ...] = ()
+
+
+@lru_cache(maxsize=1)
+def get_gt_settings() -> GTAnalyticsSettings:
+    channels_raw = str(os.environ.get("GT_ANALYTICS_WATCHED_CHANNELS", "")).strip()
+    channels = tuple(
+        part.strip().lstrip("@")
+        for part in channels_raw.split(",")
+        if part.strip()
+    )
+    profile = resolve_gt_profile()
+    lean = profile == "lean"
+    return GTAnalyticsSettings(
+        enabled=_env_bool("GT_ANALYTICS_ENABLED", default=False),
+        profile=profile,
+        base_prior=_env_float("GT_ANALYTICS_BASE_PRIOR", 0.15),
+        evidence_cap=_env_float("GT_ANALYTICS_EVIDENCE_CAP", 3.0),
+        evidence_scale=_env_float("GT_ANALYTICS_EVIDENCE_SCALE", 5.0),
+        min_prob=_env_float("GT_ANALYTICS_MIN_PROB", 0.01),
+        max_prob=_env_float("GT_ANALYTICS_MAX_PROB", 0.99),
+        high_risk_threshold=_env_float("GT_ANALYTICS_HIGH_RISK_THRESHOLD", 0.6),
+        max_history_per_region=_env_int("GT_ANALYTICS_MAX_HISTORY", 200),
+        max_heatmap_features=_env_int(
+            "GT_ANALYTICS_MAX_HEATMAP_FEATURES",
+            50 if lean else 500,
+        ),
+        louvain_min_weight=_env_float("GT_ANALYTICS_LOUVAIN_MIN_WEIGHT", 0.5),
+        louvain_interval_minutes=max(5, _env_int("GT_ANALYTICS_LOUVAIN_INTERVAL_MINUTES", 30)),
+        signal_weight_overrides=_parse_signal_weights(
+            str(os.environ.get("GT_ANALYTICS_SIGNAL_WEIGHTS", ""))
+        ),
+        watched_channels=channels,
+    )
+
+
+def gt_analytics_enabled() -> bool:
+    return get_gt_settings().enabled
+
+
+def gt_analytics_status() -> dict[str, Any]:
+    settings = get_gt_settings()
+    from services.runtime_profile import get_runtime_profile
+
+    runtime = get_runtime_profile()
+    operational = gt_engine_operational()
+    return {
+        "enabled": settings.enabled,
+        "operational": operational,
+        "profile": settings.profile,
+        "ack_low_cpu": gt_analytics_ack_low_cpu(),
+        "recommended": bool(runtime.get("gt_analytics", {}).get("recommended")),
+        "lean_node": bool(runtime.get("gt_analytics", {}).get("lean_node")),
+        "warning": runtime.get("gt_analytics", {}).get("warning"),
+        "experimental": True,
+    }

backend/analytics/weekly_store.py

@@ -0,0 +1,154 @@
+"""Persistent JSON store for rolling GT operational backtest weeks."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Literal
+
+logger = logging.getLogger(__name__)
+
+LabelName = Literal["pending", "true_escalation", "false_alarm", "benign"]
+VALID_LABELS: frozenset[str] = frozenset(
+    {"pending", "true_escalation", "false_alarm", "benign"}
+)
+
+_STORE_DIR = Path(__file__).parent.parent / "data" / "gt_rolling"
+_store_lock = threading.Lock()
+
+
+def rolling_store_dir() -> Path:
+    """Return the rolling-backtest data directory (override via env in tests)."""
+    override = str(os.environ.get("GT_ROLLING_STORE_DIR", "")).strip()
+    if override:
+        return Path(override)
+    return _STORE_DIR
+
+
+@dataclass
+class RegionSnapshot:
+    region: str
+    composite_risk: float
+    financial: float
+    unrest: float
+    conflict: float
+    alerted: bool
+    label: LabelName = "pending"
+    labeled_at: str | None = None
+    notes: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> RegionSnapshot:
+        label = str(raw.get("label") or "pending")
+        if label not in VALID_LABELS:
+            label = "pending"
+        return cls(
+            region=str(raw.get("region") or "").strip().lower(),
+            composite_risk=float(raw.get("composite_risk") or 0.0),
+            financial=float(raw.get("financial") or 0.0),
+            unrest=float(raw.get("unrest") or 0.0),
+            conflict=float(raw.get("conflict") or 0.0),
+            alerted=bool(raw.get("alerted")),
+            label=label,  # type: ignore[arg-type]
+            labeled_at=raw.get("labeled_at"),
+            notes=str(raw.get("notes") or ""),
+        )
+
+
+@dataclass
+class WeeklySnapshot:
+    week_id: str
+    frozen_at: str
+    alert_threshold: float
+    regions: list[RegionSnapshot] = field(default_factory=list)
+    frozen_by: str = "system"
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "week_id": self.week_id,
+            "frozen_at": self.frozen_at,
+            "alert_threshold": self.alert_threshold,
+            "frozen_by": self.frozen_by,
+            "regions": [row.to_dict() for row in self.regions],
+        }
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> WeeklySnapshot:
+        regions = [
+            RegionSnapshot.from_dict(row)
+            for row in (raw.get("regions") or [])
+            if isinstance(row, dict)
+        ]
+        return cls(
+            week_id=str(raw.get("week_id") or ""),
+            frozen_at=str(raw.get("frozen_at") or ""),
+            alert_threshold=float(raw.get("alert_threshold") or 0.0),
+            regions=regions,
+            frozen_by=str(raw.get("frozen_by") or "system"),
+        )
+
+
+def _week_path(week_id: str) -> Path:
+    safe = week_id.replace("/", "-").replace("..", "")
+    return rolling_store_dir() / f"{safe}.json"
+
+
+def _ensure_dir() -> None:
+    rolling_store_dir().mkdir(parents=True, exist_ok=True)
+
+
+def list_week_ids(*, newest_first: bool = True) -> list[str]:
+    """Return stored ISO week ids."""
+    _ensure_dir()
+    ids = [
+        path.stem
+        for path in rolling_store_dir().glob("*.json")
+        if path.stem and path.stem != "index"
+    ]
+    ids.sort(reverse=newest_first)
+    return ids
+
+
+def load_week(week_id: str) -> WeeklySnapshot | None:
+    path = _week_path(week_id)
+    if not path.is_file():
+        return None
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+        if not isinstance(raw, dict):
+            return None
+        return WeeklySnapshot.from_dict(raw)
+    except (OSError, json.JSONDecodeError, TypeError, ValueError):
+        logger.exception("Failed to load GT rolling week %s", week_id)
+        return None
+
+
+def save_week(snapshot: WeeklySnapshot) -> None:
+    _ensure_dir()
+    path = _week_path(snapshot.week_id)
+    tmp = path.with_suffix(".json.tmp")
+    payload = json.dumps(snapshot.to_dict(), indent=2, sort_keys=True)
+    with _store_lock:
+        tmp.write_text(payload, encoding="utf-8")
+        tmp.replace(path)
+
+
+def delete_week(week_id: str) -> bool:
+    path = _week_path(week_id)
+    if not path.is_file():
+        return False
+    with _store_lock:
+        path.unlink()
+    return True
+
+
+def utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()

backend/config/news_feeds.json

@@ -1,15 +1,5 @@
 {
   "feeds": [
-    {
-      "name": "Reuters",
-      "url": "https://www.reutersagency.com/feed/?best-topics=world",
-      "weight": 5
-    },
-    {
-      "name": "AP News",
-      "url": "https://rsshub.app/apnews/topics/world-news",
-      "weight": 5
-    },
     {
       "name": "NPR",
       "url": "https://feeds.npr.org/1004/rss.xml",
@@ -17,7 +7,7 @@
     },
     {
       "name": "BBC",
-      "url": "http://feeds.bbci.co.uk/news/world/rss.xml",
+      "url": "https://feeds.bbci.co.uk/news/world/rss.xml",
       "weight": 3
     },
     {
@@ -57,7 +47,7 @@
     },
     {
       "name": "Xinhua",
-      "url": "http://www.news.cn/english/rss/worldrss.xml",
+      "url": "https://www.news.cn/english/rss/worldrss.xml",
       "weight": 2
     },
     {
@@ -99,6 +89,26 @@
       "name": "Japan Times",
       "url": "https://www.japantimes.co.jp/feed/",
       "weight": 3
+    },
+    {
+      "name": "CSM",
+      "url": "https://www.csmonitor.com/rss/world",
+      "weight": 4
+    },
+    {
+      "name": "PBS NewsHour",
+      "url": "https://www.pbs.org/newshour/feeds/rss/world",
+      "weight": 4
+    },
+    {
+      "name": "France 24",
+      "url": "https://www.france24.com/en/rss",
+      "weight": 4
+    },
+    {
+      "name": "DW",
+      "url": "https://rss.dw.com/xml/rss-en-world",
+      "weight": 4
     }
   ]
 }

backend/data/aisstream_spki_pins.json

@@ -0,0 +1,31 @@
+{
+  "_comment": [
+    "SPKI (Subject Public Key Info) pin list for stream.aisstream.io.",
+    "",
+    "Issue #258: AISStream's Let's Encrypt cert expired on 2026-05-20 due to an",
+    "upstream renewal-pipeline failure. Disabling TLS verification entirely",
+    "would let any network attacker MITM the AIS WebSocket and inject fake",
+    "ship positions onto the operator's map (same class as #199 GDELT MITM).",
+    "Instead we pin the leaf certificate's public-key SPKI hash: if normal",
+    "TLS validation fails specifically with CERT_HAS_EXPIRED, ais_proxy.js",
+    "re-checks the leaf cert's SPKI against this list. A match means the",
+    "key is still the genuine AISStream key (Let's Encrypt renewals keep the",
+    "same key unless rekey is requested), so we proceed in 'degraded TLS'",
+    "mode. A mismatch means a real MITM attempt and we refuse the connection.",
+    "",
+    "Format: each entry is a SHA-256 hash of the DER-encoded SPKI bytes,",
+    "encoded as standard base64 (matches the format produced by:",
+    "  openssl s_client -connect host:443 | \\",
+    "    openssl x509 -pubkey -noout | openssl pkey -pubin -outform DER | \\",
+    "    openssl dgst -sha256 -binary | openssl base64",
+    ").",
+    "",
+    "When AISStream rotates their server key (rare — Let's Encrypt renewals",
+    "default to keeping the same key), capture the new SPKI and add it to",
+    "this list BEFORE removing the old one. That way operators on the old",
+    "code still validate against the previous key during the transition."
+  ],
+  "stream.aisstream.io": [
+    "GJ10H0UPgLrO+2d3ZXROR/TXSVFXKUfRC3QEI2ibEg4="
+  ]
+}

backend/data/carrier_seed.json

@@ -0,0 +1,120 @@
+{
+  "_meta": {
+    "as_of": "2026-03-09",
+    "source": "USNI News Fleet & Marine Tracker",
+    "source_url": "https://news.usni.org/2026/03/09/usni-news-fleet-and-marine-tracker-march-9-2026",
+    "note": "One-shot bootstrap for first-run carrier positions. Once carrier_cache.json exists in the runtime data volume, this seed file is never read again. All subsequent updates come from GDELT (and any future sources) and are written to carrier_cache.json. A year from now, your runtime cache reflects whatever your install has observed since first launch — not these snapshot positions."
+  },
+  "carriers": {
+    "CVN-68": {
+      "lat": 47.5535,
+      "lng": -122.6400,
+      "heading": 90,
+      "desc": "Bremerton, WA (Maintenance)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-76": {
+      "lat": 47.5580,
+      "lng": -122.6360,
+      "heading": 90,
+      "desc": "Bremerton, WA (Decommissioning)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-69": {
+      "lat": 36.9465,
+      "lng": -76.3265,
+      "heading": 0,
+      "desc": "Norfolk, VA (Post-deployment maintenance)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-78": {
+      "lat": 18.0,
+      "lng": 39.5,
+      "heading": 0,
+      "desc": "Red Sea — Operation Epic Fury (USNI Mar 9)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-74": {
+      "lat": 36.98,
+      "lng": -76.43,
+      "heading": 0,
+      "desc": "Newport News, VA (RCOH refueling overhaul)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-75": {
+      "lat": 36.0,
+      "lng": 15.0,
+      "heading": 0,
+      "desc": "Mediterranean Sea deployment (USNI Mar 9)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-77": {
+      "lat": 36.5,
+      "lng": -74.0,
+      "heading": 0,
+      "desc": "Atlantic — Pre-deployment workups (USNI Mar 9)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-70": {
+      "lat": 32.6840,
+      "lng": -117.1290,
+      "heading": 180,
+      "desc": "San Diego, CA (Homeport)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-71": {
+      "lat": 32.6885,
+      "lng": -117.1280,
+      "heading": 180,
+      "desc": "San Diego, CA (Maintenance)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-72": {
+      "lat": 20.0,
+      "lng": 64.0,
+      "heading": 0,
+      "desc": "Arabian Sea — Operation Epic Fury (USNI Mar 9)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-73": {
+      "lat": 35.2830,
+      "lng": 139.6700,
+      "heading": 180,
+      "desc": "Yokosuka, Japan (Forward deployed)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    }
+  }
+}

backend/data/military_bases.json

@@ -1047,14 +1047,6 @@
     "lat": 37.47,
     "lng": 69.381
   },
-  {
-    "name": "Berth rights and right to station its troops in Qatar",
-    "country": "India",
-    "operator": "India",
-    "branch": "army",
-    "lat": 25.308,
-    "lng": 51.209
-  },
   {
     "name": "Ahmad al-Jaber Air Base",
     "country": "Italy",

backend/data/plane_alert_db.json

@@ -73567,6 +73567,14 @@
         "tags": "Air Ambo, Medical Evac, Saving Lives",
         "link": "https://www.airmethods.com/"
     },
+    "ABD9B5": {
+        "registration": "N8628",
+        "operator": "Elon Musk",
+        "ac_type": "Gulfstream G800",
+        "category": "Don't you know who I am?",
+        "tags": "Elon Musk, SpaceX, DOGE, Toys4Billionaires",
+        "link": "https://en.wikipedia.org/wiki/Elon_Musk"
+    },
     "A835AF": {
         "registration": "N628TS",
         "operator": "Falcon Landing LLC",

backend/data/release_digests.json

@@ -0,0 +1,60 @@
+{
+  "_comment": [
+    "Baked-in SHA-256 digests for known Shadowbroker release archives.",
+    "",
+    "Issue #231: the self-updater previously skipped integrity verification",
+    "entirely whenever the MESH_UPDATE_SHA256 env var was unset (which is the",
+    "default — nothing in the install docs tells operators to set it). That",
+    "made the auto-update a supply-chain RCE on any compromise of the GitHub",
+    "release pipeline.",
+    "",
+    "The fix uses a multi-source verification chain mirroring the Tor bundle",
+    "digest approach in #201:",
+    "",
+    "  1. MESH_UPDATE_SHA256 env var (operator override, preserved)",
+    "  2. SHA256SUMS.txt asset published alongside each release (primary —",
+    "     the maintainer's release process already publishes this)",
+    "  3. This baked-in digest list (second line of defense for releases",
+    "     missing a SHA256SUMS asset, or when the asset can't be fetched)",
+    "  4. HTTPS-only fallback with a loud warning (preserves auto-update",
+    "     flow during transient outages so users don't get stuck)",
+    "",
+    "Mismatch from a source that DID respond is fatal — the update is",
+    "refused and the existing install keeps running. Only the 'no source",
+    "reachable at all' case falls back to HTTPS-only.",
+    "",
+    "Format: each entry is keyed by release tag and maps asset filenames",
+    "to their canonical SHA-256 digest (hex, lowercase). The updater",
+    "compares the locally-computed digest of the downloaded asset against",
+    "the value here.",
+    "",
+    "When the maintainer ships a new release, add its digests here BEFORE",
+    "removing the old ones so operators on the old code still validate",
+    "against the previous entries during the transition."
+  ],
+  "v0.9.79": {
+    "ShadowBroker_v0.9.79.zip": "f6877c1d66614525315ea82636ce9f7b41178332c4dbf90d27431a1ea1d9cd47",
+    "ShadowBroker_0.9.79_x64-setup.exe": "f7b676ada45cac7da05868b0a353678c9ee700e3abcf456a7c0c038c36da446f",
+    "ShadowBroker_0.9.79_x64_en-US.msi": "e0713c3cdda184cfbea750bfac0d62a35678fec00847e6476f2cac8e7e42046e"
+  },
+  "v0.9.8": {
+    "ShadowBroker_v0.9.8.zip": "183bb5cd62b9b9349d95df5ef7696cb6ca810ab4b991fa9dab6f898af4c7a175",
+    "ShadowBroker_0.9.8_x64-setup.exe": "94a0309862e9c81c92cdcbfea8eec9dbb97eef19ded82b26217b397defbc810c",
+    "ShadowBroker_0.9.8_x64_en-US.msi": "fe22f9d51e4360d74c18a7250c2fbb9ed4fa4c7a884b3ac0d04a21115466386b"
+  },
+  "v0.9.81": {
+    "ShadowBroker_v0.9.81.zip": "f81f454bdc88e9a32c351df38212b8cfa624704d65764b971bb091eef62259c6",
+    "ShadowBroker_0.9.81_x64-setup.exe": "25e9a95d0d8ce959a7d08fe8e7406772ae24b596652793e81d1de5d02510a5a6",
+    "ShadowBroker_0.9.81_x64_en-US.msi": "34e655fc0c0f195ee4ac978f228a4b2b9d5565253b8771aca9ef4693409e9e70"
+  },
+  "v0.9.82": {
+    "ShadowBroker_v0.9.82.zip": "202ab043465741dcc06de57c19ec8314904332f8e818b891d7174655719d084c",
+    "ShadowBroker_0.9.82_x64-setup.exe": "0eb9f2bda02ab691b39687641abc97e6bfb507b42f48de21970ad7dfb4ea15fc",
+    "ShadowBroker_0.9.82_x64_en-US.msi": "ced08f930171c0c08009a958cc30b0171a09f982230fc217c6808c2ed7ab2e30"
+  },
+  "v0.9.83": {
+    "ShadowBroker_v0.9.83.zip": "53f56631731ad3cdc7be68df09bedd6570ed91ecda6fa57c39651098e15666c7",
+    "ShadowBroker_0.9.83_x64-setup.exe": "d62170af4b9df0b190832b7bb3ad6bfe8a7ac01472f2c7b39cf2a1b61edc7492",
+    "ShadowBroker_0.9.83_x64_en-US.msi": "b664cc0003a29f7ce88b04c2b425643dbe7ed897342fc6e9a2378bc1910c6850"
+  }
+}

backend/data/tor_bundle_digests.json

@@ -0,0 +1,16 @@
+{
+  "_comment": [
+    "Pinned SHA-256 digests for the Tor Expert Bundle archives we know how to install.",
+    "Used as the LAST-RESORT verification source when the upstream .sha256sum file is",
+    "unreachable, MITM'd, or doesn't match what we downloaded. Issue #201.",
+    "",
+    "Each entry is keyed by the archive URL (so multiple platforms / versions",
+    "can share this one file) and contains the canonical SHA-256 we trust.",
+    "",
+    "When the project tests a new Tor release, add its digest here in the same",
+    "PR that bumps _TOR_EXPERT_BUNDLE_URLS. Old entries are kept indefinitely so",
+    "users on older versions keep working — we only ever ADD here, never remove."
+  ],
+  "https://dist.torproject.org/torbrowser/15.0.11/tor-expert-bundle-windows-x86_64-15.0.11.tar.gz": "PLACEHOLDER_REPLACE_BEFORE_RELEASE",
+  "https://dist.torproject.org/torbrowser/15.0.8/tor-expert-bundle-windows-x86_64-15.0.8.tar.gz": "PLACEHOLDER_REPLACE_BEFORE_RELEASE"
+}

backend/docker-entrypoint.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+set -eu
+
+# Docker named volumes hide files that were baked into /app/data at image build
+# time. Seed safe, static data into a fresh volume so first-run Docker installs
+# behave like source installs without bundling local runtime secrets.
+if [ -d /app/image-data ]; then
+  mkdir -p /app/data
+  find /app/image-data -mindepth 1 -maxdepth 1 -type f | while IFS= read -r src; do
+    dest="/app/data/$(basename "$src")"
+    if [ ! -e "$dest" ]; then
+      cp "$src" "$dest" || true
+    fi
+  done
+fi
+
+if [ -z "${PRIVACY_CORE_ALLOWED_SHA256:-}" ] && [ -f /app/libprivacy_core.so ]; then
+  PRIVACY_CORE_ALLOWED_SHA256="$(sha256sum /app/libprivacy_core.so | awk '{print $1}')"
+  export PRIVACY_CORE_ALLOWED_SHA256
+fi
+
+exec "$@"

backend/gate_sse.py

@@ -0,0 +1,11 @@
+"""gate_sse.py — DEPRECATED. Gate SSE broadcast removed in S3A.
+
+Gate activity is no longer broadcast via SSE. The frontend uses the
+authenticated poll loop for gate message refresh.
+
+Stubs are kept so any late imports do not crash at startup.
+"""
+
+
+def _broadcast_gate_events(gate_id: str, events: list[dict]) -> None:  # noqa: ARG001
+    """No-op — gate SSE broadcast removed."""

backend/limiter.py

@@ -0,0 +1,108 @@
+"""Rate-limit key function for slowapi.
+
+Issue #287 (tg12): the previous implementation used
+``slowapi.util.get_remote_address`` which only ever returns
+``request.client.host``. Behind the bundled Next.js proxy (or any other
+reverse proxy), every connected operator's ``client.host`` is the
+frontend container's bridge IP. ``@limiter.limit("120/minute")`` then
+collapses into one shared bucket for everybody on the same backend —
+one heavy tab can starve every other operator on the node.
+
+This module replaces that key function with one that:
+
+  * Reads ``X-Forwarded-For`` ONLY when the immediate peer is a trusted
+    frontend container (same allowlist used by the Docker bridge
+    local-operator trust path — see ``backend/auth.py`` ``#250``).
+  * Picks the FIRST entry in the XFF chain. That's the client end of
+    the proxy chain, which is the operator we want to bucket on.
+  * Falls back to ``request.client.host`` for any peer that isn't on
+    the trusted-frontend allowlist. Direct hits, unrelated containers,
+    and unknown hosts are bucketed exactly like before — there is no
+    way for an untrusted caller to spoof XFF and steal another
+    operator's rate-limit bucket.
+
+Single-operator nodes are unaffected: the frontend resolves to one IP,
+that IP is on the trust list, the XFF header is read, and you get one
+bucket per operator (i.e. you).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from slowapi import Limiter
+from slowapi.util import get_remote_address
+
+
+def _client_host(request: Any) -> str:
+    """Return the immediate peer's IP, normalised to a lowercase string."""
+    client = getattr(request, "client", None)
+    if client is None:
+        return ""
+    host = getattr(client, "host", "") or ""
+    return host.lower()
+
+
+def _first_forwarded_for(value: str) -> str:
+    """Return the first non-empty entry from an ``X-Forwarded-For`` header.
+
+    RFC 7239 / de-facto XFF format is ``client, proxy1, proxy2, …``. The
+    client end is what we want to bucket on. Empty parts (which appear
+    in some malformed headers) are skipped so we don't end up keying on
+    an empty string.
+    """
+    for raw in value.split(","):
+        candidate = raw.strip()
+        if candidate:
+            return candidate.lower()
+    return ""
+
+
+def _is_trusted_frontend_peer(host: str) -> bool:
+    """True iff ``host`` is one of the resolved trusted-frontend IPs.
+
+    Imported lazily so this module stays usable in unit tests that
+    don't want to pull the whole auth module into scope.
+    """
+    if not host:
+        return False
+    try:
+        from auth import _resolve_trusted_bridge_ips
+    except Exception:  # pragma: no cover - defensive
+        return False
+    try:
+        trusted_ips = _resolve_trusted_bridge_ips()
+    except Exception:  # pragma: no cover - defensive
+        return False
+    return host in trusted_ips
+
+
+def shadowbroker_rate_limit_key(request: Any) -> str:
+    """slowapi key_func that is proxy-aware on trusted frontend peers only.
+
+    Behaviour matrix:
+
+    * Direct loopback / unknown peer → ``request.client.host``
+      (identical to slowapi's default ``get_remote_address``).
+    * Peer is a trusted frontend container AND ``X-Forwarded-For`` is
+      present → first XFF entry (the actual operator).
+    * Peer is a trusted frontend container but no XFF → fall back to
+      ``request.client.host`` (the bridge IP). One shared bucket for
+      everyone in that case, same as before — but you only get there
+      if the trusted frontend forgot to forward XFF, which it won't.
+    """
+    peer = _client_host(request)
+    if _is_trusted_frontend_peer(peer):
+        headers = getattr(request, "headers", None)
+        if headers is not None:
+            xff = headers.get("x-forwarded-for") or headers.get("X-Forwarded-For")
+            if xff:
+                first = _first_forwarded_for(xff)
+                if first:
+                    return first
+    # Untrusted peer (or trusted peer without XFF): match the original
+    # get_remote_address behaviour byte-for-byte.
+    return get_remote_address(request)
+
+
+limiter = Limiter(key_func=shadowbroker_rate_limit_key)

backend/node_state.py

@@ -0,0 +1,313 @@
+"""node_state.py — Shared mutable node runtime state and node helper functions.
+
+Extracted from main.py so that background worker functions and route handlers
+can reference the same state objects without importing the full application.
+
+_NODE_SYNC_STATE is a reassignable value (SyncWorkerState is replaced whole,
+not mutated), so callers must use get_sync_state() / set_sync_state() instead
+of binding to the name at import time.
+
+All other _NODE_* objects are mutable containers (Lock, Event, dict) whose
+identity never changes; importing them directly by name is safe.
+"""
+
+import threading
+import time
+from typing import Any
+
+from services.mesh.mesh_infonet_sync_support import SyncWorkerState
+
+# ---------------------------------------------------------------------------
+# Runtime state objects
+# ---------------------------------------------------------------------------
+
+_NODE_RUNTIME_LOCK = threading.RLock()
+_NODE_SYNC_STOP = threading.Event()
+_NODE_SYNC_STATE = SyncWorkerState()
+_NODE_BOOTSTRAP_STATE: dict[str, Any] = {
+    "node_mode": "participant",
+    "manifest_loaded": False,
+    "manifest_signer_id": "",
+    "manifest_valid_until": 0,
+    "bootstrap_peer_count": 0,
+    "sync_peer_count": 0,
+    "push_peer_count": 0,
+    "operator_peer_count": 0,
+    "last_bootstrap_error": "",
+}
+_NODE_PUSH_STATE: dict[str, Any] = {
+    "last_event_id": "",
+    "last_push_ok_at": 0,
+    "last_push_error": "",
+    "last_results": [],
+}
+
+# ---------------------------------------------------------------------------
+# Getter / setter for _NODE_SYNC_STATE
+#
+# Use these instead of globals()["_NODE_SYNC_STATE"] = ... in any module that
+# imports this package.  The setter modifies *this* module's namespace so
+# subsequent get_sync_state() calls see the new value regardless of which
+# module calls set_sync_state().
+# ---------------------------------------------------------------------------
+
+def get_sync_state() -> SyncWorkerState:
+    return _NODE_SYNC_STATE
+
+
+def set_sync_state(state: SyncWorkerState) -> None:
+    global _NODE_SYNC_STATE
+    _NODE_SYNC_STATE = state
+
+
+# ---------------------------------------------------------------------------
+# Node helper functions
+#
+# These were in main.py but are needed by both route handlers and background
+# workers, so they live here to avoid circular imports.
+# ---------------------------------------------------------------------------
+
+def _current_node_mode() -> str:
+    from services.config import get_settings
+    mode = str(get_settings().MESH_NODE_MODE or "participant").strip().lower()
+    if mode not in {"participant", "relay", "perimeter"}:
+        return "participant"
+    return mode
+
+
+def _node_runtime_supported() -> bool:
+    return _current_node_mode() in {"participant", "relay"}
+
+
+def _node_activation_enabled() -> bool:
+    from services.node_settings import read_node_settings
+
+    try:
+        settings = read_node_settings()
+    except Exception:
+        return False
+    return bool(settings.get("enabled", False))
+
+
+def _participant_node_enabled() -> bool:
+    return _node_runtime_supported() and _node_activation_enabled()
+
+
+def _node_runtime_snapshot() -> dict[str, Any]:
+    with _NODE_RUNTIME_LOCK:
+        return {
+            "node_mode": _current_node_mode(),
+            "node_enabled": _participant_node_enabled(),
+            "private_transport_required": _infonet_private_transport_required(),
+            "bootstrap": {**dict(_NODE_BOOTSTRAP_STATE), "node_mode": _current_node_mode()},
+            "sync_runtime": get_sync_state().to_dict(),
+            "push_runtime": dict(_NODE_PUSH_STATE),
+        }
+
+
+def _set_node_sync_disabled_state(*, current_head: str = "") -> SyncWorkerState:
+    return SyncWorkerState(
+        current_head=str(current_head or ""),
+        last_outcome="disabled",
+    )
+
+
+def _set_participant_node_enabled(enabled: bool) -> dict[str, Any]:
+    from services.mesh.mesh_hashchain import infonet
+    from services.node_settings import write_node_settings
+
+    settings = write_node_settings(enabled=bool(enabled))
+    current_head = str(infonet.head_hash or "")
+    with _NODE_RUNTIME_LOCK:
+        _NODE_BOOTSTRAP_STATE["node_mode"] = _current_node_mode()
+        set_sync_state(
+            SyncWorkerState(current_head=current_head)
+            if bool(enabled) and _node_runtime_supported()
+            else _set_node_sync_disabled_state(current_head=current_head)
+        )
+    return {
+        **settings,
+        "node_mode": _current_node_mode(),
+        "node_enabled": _participant_node_enabled(),
+    }
+
+
+def _infonet_private_transport_required() -> bool:
+    from services.config import get_settings
+
+    return not bool(getattr(get_settings(), "MESH_INFONET_ALLOW_CLEARNET_SYNC", False))
+
+
+def _infonet_private_transport_error() -> str:
+    return "private Infonet requires onion/RNS transport; no clearnet sync fallback"
+
+
+def _is_private_infonet_transport(transport: str) -> bool:
+    return str(transport or "").strip().lower() in {"onion", "rns"}
+
+
+def _configured_bootstrap_seed_peer_urls() -> list[str]:
+    from services.config import get_settings
+    from services.mesh.mesh_router import parse_configured_relay_peers
+
+    settings = get_settings()
+    primary = str(getattr(settings, "MESH_BOOTSTRAP_SEED_PEERS", "") or "").strip()
+    legacy = str(getattr(settings, "MESH_DEFAULT_SYNC_PEERS", "") or "").strip()
+    return parse_configured_relay_peers(primary or legacy)
+
+
+def _refresh_node_peer_store(*, now: float | None = None) -> dict[str, Any]:
+    from services.config import get_settings
+    from services.mesh.mesh_bootstrap_manifest import load_bootstrap_manifest_from_settings
+    from services.mesh.mesh_peer_store import (
+        DEFAULT_PEER_STORE_PATH,
+        PeerStore,
+        make_bootstrap_peer_record,
+        make_push_peer_record,
+        make_sync_peer_record,
+    )
+    from services.mesh.mesh_router import (
+        configured_relay_peer_urls,
+        parse_configured_relay_peers,
+        peer_transport_kind,
+    )
+
+    timestamp = int(now if now is not None else time.time())
+    mode = _current_node_mode()
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        store = PeerStore(DEFAULT_PEER_STORE_PATH)
+
+    private_transport_required = _infonet_private_transport_required()
+    operator_peers = configured_relay_peer_urls()
+    bootstrap_seed_peers = _configured_bootstrap_seed_peer_urls()
+    skipped_clearnet_peers = 0
+    for peer_url in operator_peers:
+        transport = peer_transport_kind(peer_url)
+        if not transport:
+            continue
+        if private_transport_required and not _is_private_infonet_transport(transport):
+            skipped_clearnet_peers += 1
+            continue
+        store.upsert(
+            make_sync_peer_record(
+                peer_url=peer_url,
+                transport=transport,
+                role="relay",
+                source="operator",
+                now=timestamp,
+            )
+        )
+        store.upsert(
+            make_push_peer_record(
+                peer_url=peer_url,
+                transport=transport,
+                role="relay",
+                source="operator",
+                now=timestamp,
+            )
+        )
+
+    operator_peer_set = set(operator_peers)
+    for peer_url in bootstrap_seed_peers:
+        if peer_url in operator_peer_set:
+            continue
+        transport = peer_transport_kind(peer_url)
+        if not transport:
+            continue
+        if private_transport_required and not _is_private_infonet_transport(transport):
+            skipped_clearnet_peers += 1
+            continue
+        store.upsert(
+            make_bootstrap_peer_record(
+                peer_url=peer_url,
+                transport=transport,
+                role="seed",
+                label="ShadowBroker bootstrap seed",
+                signer_id="shadowbroker-bootstrap",
+                now=timestamp,
+            )
+        )
+        store.upsert(
+            make_sync_peer_record(
+                peer_url=peer_url,
+                transport=transport,
+                role="seed",
+                source="bundle",
+                label="ShadowBroker bootstrap seed",
+                signer_id="shadowbroker-bootstrap",
+                now=timestamp,
+            )
+        )
+
+    manifest = None
+    bootstrap_error = ""
+    try:
+        manifest = load_bootstrap_manifest_from_settings(now=timestamp)
+    except Exception as exc:
+        bootstrap_error = str(exc or "").strip()
+
+    if manifest is not None:
+        for peer in manifest.peers:
+            if private_transport_required and not _is_private_infonet_transport(peer.transport):
+                skipped_clearnet_peers += 1
+                continue
+            store.upsert(
+                make_bootstrap_peer_record(
+                    peer_url=peer.peer_url,
+                    transport=peer.transport,
+                    role=peer.role,
+                    label=peer.label,
+                    signer_id=manifest.signer_id,
+                    now=timestamp,
+                )
+            )
+            store.upsert(
+                make_sync_peer_record(
+                    peer_url=peer.peer_url,
+                    transport=peer.transport,
+                    role=peer.role,
+                    source="bootstrap_promoted",
+                    label=peer.label,
+                    signer_id=manifest.signer_id,
+                    now=timestamp,
+                )
+            )
+
+    if private_transport_required and skipped_clearnet_peers and not bootstrap_error:
+        bootstrap_error = _infonet_private_transport_error()
+
+    store.save()
+    bootstrap_records = store.records_for_bucket("bootstrap")
+    sync_records = store.records_for_bucket("sync")
+    push_records = store.records_for_bucket("push")
+    if private_transport_required:
+        bootstrap_records = [record for record in bootstrap_records if _is_private_infonet_transport(record.transport)]
+        sync_records = [record for record in sync_records if _is_private_infonet_transport(record.transport)]
+        push_records = [record for record in push_records if _is_private_infonet_transport(record.transport)]
+    snapshot = {
+        "node_mode": mode,
+        "private_transport_required": private_transport_required,
+        "skipped_clearnet_peer_count": skipped_clearnet_peers,
+        "manifest_loaded": manifest is not None,
+        "manifest_signer_id": manifest.signer_id if manifest is not None else "",
+        "manifest_valid_until": int(manifest.valid_until or 0) if manifest is not None else 0,
+        "bootstrap_peer_count": len(bootstrap_records),
+        "sync_peer_count": len(sync_records),
+        "push_peer_count": len(push_records),
+        "operator_peer_count": len(operator_peers),
+        "bootstrap_seed_peer_count": len(bootstrap_seed_peers),
+        "default_sync_peer_count": len(bootstrap_seed_peers),
+        "last_bootstrap_error": bootstrap_error,
+    }
+    with _NODE_RUNTIME_LOCK:
+        _NODE_BOOTSTRAP_STATE.update(snapshot)
+    return snapshot
+
+
+def _materialize_local_infonet_state() -> None:
+    from services.mesh.mesh_hashchain import infonet
+
+    infonet.ensure_materialized()

backend/pyproject.toml

@@ -1,34 +1,66 @@
+[build-system]
+requires = ["setuptools>=68.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+py-modules = []
+
 [project]
 name = "backend"
-version = "0.9.6"
+version = "0.9.83"
 requires-python = ">=3.10"
 dependencies = [
     "apscheduler==3.10.3",
     "beautifulsoup4>=4.9.0",
     "cachetools==5.5.2",
-    "cloudscraper==1.2.71",
-    "cryptography>=41.0.0",
-    "fastapi==0.115.12",
+    "cryptography>=46.0.7",
+    "defusedxml>=0.7.1",
+    "fastapi==0.136.3",
     "feedparser==6.0.10",
     "httpx==0.28.1",
-    "playwright==1.50.0",
+    "playwright==1.59.0",
     "playwright-stealth==1.0.6",
-    "pydantic==2.11.1",
+    "pydantic==2.13.3",
     "pydantic-settings==2.8.1",
     "pystac-client==0.8.6",
     "python-dotenv==1.2.2",
-    "requests==2.31.0",
+    "requests==2.33.0",
+    "PySocks==1.7.1",
     "reverse-geocoder==1.5.1",
-    "sgp4==2.23",
+    "sgp4==2.25",
     "meshtastic>=2.5.0",
+    "networkx>=3.4.0",
+    "numpy>=2.2.0",
     "orjson>=3.10.0",
     "paho-mqtt>=1.6.0,<2.0.0",
     "PyNaCl>=1.5.0",
-    "slowapi==0.1.9",
+    "slowapi==0.1.10",
+    "starlette==1.0.1",
     "vaderSentiment>=3.3.0",
     "uvicorn==0.34.0",
-    "yfinance==0.2.54",
+    "yfinance==1.3.0",
+]
+
+[project.optional-dependencies]
+road-corridor = [
+    "geopandas>=1.0.0",
+    "imageio>=2.34.0",
+    "osmnx>=2.0.0",
+    "rasterio>=1.4.0",
+    "scikit-learn>=1.5.0",
+    "sentinelhub>=3.10.0",
+    "shapely>=2.0.0",
 ]
 
 [dependency-groups]
-dev = ["pytest>=8.3.4", "pytest-asyncio==0.25.0", "ruff>=0.9.0", "black>=24.0.0"]
+dev = ["pytest>=9.0.3", "pytest-asyncio>=1.4.0", "ruff>=0.9.0", "black>=24.0.0"]
+
+[tool.ruff.lint]
+# The current backend carries historical style debt in large legacy modules.
+# Keep CI focused on actionable correctness checks for the v0.9.82 release.
+ignore = ["E401", "E402", "E701", "E731", "E741", "F401", "F402", "F541", "F811", "F841"]
+
+[tool.black]
+# Avoid a release-time whole-backend formatting rewrite. Re-enable by narrowing
+# this once the legacy tree is formatted in a dedicated cleanup PR.
+force-exclude = ".*"

backend/routers/admin.py

@@ -0,0 +1,435 @@
+import json as json_mod
+import logging
+import os
+import threading
+from pathlib import Path
+from typing import Any
+from fastapi import APIRouter, Request, Depends, Response
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+from node_state import (
+    _current_node_mode,
+    _participant_node_enabled,
+    _refresh_node_peer_store,
+    _set_participant_node_enabled,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+class NodeSettingsUpdate(BaseModel):
+    enabled: bool
+
+
+class TimeMachineToggle(BaseModel):
+    enabled: bool
+
+
+class MeshtasticMqttUpdate(BaseModel):
+    enabled: bool | None = None
+    broker: str | None = None
+    port: int | None = None
+    username: str | None = None
+    password: str | None = None
+    psk: str | None = None
+    include_default_roots: bool | None = None
+    extra_roots: str | None = None
+    extra_topics: str | None = None
+
+
+@router.get("/api/settings/api-keys", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_get_keys(request: Request):
+    from services.api_settings import get_api_keys
+    return get_api_keys()
+
+
+@router.put("/api/settings/api-keys", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_save_keys(request: Request):
+    from services.api_settings import save_api_keys
+    body = await request.json()
+    if not isinstance(body, dict):
+        return Response(
+            content=json_mod.dumps({"ok": False, "detail": "Expected a JSON object."}),
+            status_code=400,
+            media_type="application/json",
+        )
+    result = save_api_keys({str(k): str(v) for k, v in body.items()})
+    if result.get("ok"):
+        return result
+    return Response(
+        content=json_mod.dumps(result),
+        status_code=400,
+        media_type="application/json",
+    )
+
+
+@router.get("/api/settings/api-keys/meta")
+@limiter.limit("30/minute")
+async def api_get_keys_meta(request: Request):
+    """Return absolute paths for the backend .env and .env.example template.
+
+    Not gated behind admin auth: the paths are not sensitive, and the frontend
+    needs them to render the API Keys panel banner before the user has had a
+    chance to enter an admin key. Helps users find the file when in-app editing
+    is blocked or when the backend is read-only.
+    """
+    from services.api_settings import get_env_path_info
+    return get_env_path_info()
+
+
+@router.get(
+    "/api/settings/operator-handle",
+    dependencies=[Depends(require_local_operator)],
+)
+@limiter.limit("60/minute")
+async def api_get_operator_handle(request: Request):
+    """Round 7a: return the per-install operator handle so the frontend
+    can include it in browser-direct third-party API calls (Wikipedia /
+    Wikidata via lib/wikimediaClient). The handle is auto-generated on
+    first use; operators can override it via the OPERATOR_HANDLE setting
+    or the env var of the same name.
+
+    Gated on local-operator: legitimate browser usage goes through the
+    Next.js proxy which auto-attaches the admin key; remote scanners get
+    403. The handle itself isn't a secret (it's sent to every third-party
+    API the operator touches), but admin-gating it matches the rest of
+    the settings endpoints and follows least-privilege.
+    """
+    from services.network_utils import get_operator_handle
+    return {"handle": get_operator_handle()}
+
+
+@router.get(
+    "/api/settings/news-feeds",
+    dependencies=[Depends(require_local_operator)],
+)
+@limiter.limit("30/minute")
+async def api_get_news_feeds(request: Request):
+    """Issue #252 (tg12): the curated feed inventory is configuration
+    state, not a public data feed. Gated on local-operator so the
+    Tauri shell, the Docker bridge frontend, and any caller with an
+    admin key all see the full list; anonymous LAN/internet callers
+    can no longer enumerate operator source URLs.
+    """
+    from services.news_feed_config import get_feeds
+    return get_feeds()
+
+
+@router.put("/api/settings/news-feeds", dependencies=[Depends(require_admin)])
+@limiter.limit("10/minute")
+async def api_save_news_feeds(request: Request):
+    from services.news_feed_config import save_feeds
+    body = await request.json()
+    ok = save_feeds(body)
+    if ok:
+        return {"status": "updated", "count": len(body)}
+    return Response(
+        content=json_mod.dumps({"status": "error",
+            "message": "Validation failed (max 20 feeds, each needs name/url/weight 1-5)"}),
+        status_code=400,
+        media_type="application/json",
+    )
+
+
+@router.post("/api/settings/news-feeds/reset", dependencies=[Depends(require_admin)])
+@limiter.limit("10/minute")
+async def api_reset_news_feeds(request: Request):
+    from services.news_feed_config import get_feeds, reset_feeds
+    ok = reset_feeds()
+    if ok:
+        return {"status": "reset", "feeds": get_feeds()}
+    return {"status": "error", "message": "Failed to reset feeds"}
+
+
+@router.get("/api/settings/node")
+@limiter.limit("30/minute")
+async def api_get_node_settings(request: Request):
+    """Issue #243 (tg12): node_mode and node_enabled are operational
+    posture. Anonymous callers receive an empty stub; authenticated
+    callers (local-operator or admin/scoped token) see the full
+    state. See the canonical handler in backend/main.py for the full
+    rationale.
+    """
+    import asyncio
+    from auth import _scoped_view_authenticated
+    from services.node_settings import read_node_settings
+    data = await asyncio.to_thread(read_node_settings)
+    if not _scoped_view_authenticated(request, "node"):
+        return {}
+    return {
+        **data,
+        "node_mode": _current_node_mode(),
+        "node_enabled": _participant_node_enabled(),
+    }
+
+
+@router.put("/api/settings/node", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_set_node_settings(request: Request, body: NodeSettingsUpdate):
+    _refresh_node_peer_store()
+    if bool(body.enabled):
+        try:
+            from services.transport_lane_isolation import disable_public_mesh_lane
+
+            disable_public_mesh_lane(reason="private_node_enabled")
+        except Exception as exc:
+            logger.warning("Failed to disable public Mesh while enabling private node: %s", exc)
+    result = _set_participant_node_enabled(bool(body.enabled))
+    if bool(body.enabled):
+        try:
+            import main as _main
+
+            _main._kick_public_sync_background("operator_enable")
+        except Exception:
+            logger.debug("Unable to kick Infonet sync after node enable", exc_info=True)
+    return result
+
+
+def _meshtastic_runtime_snapshot() -> dict[str, Any]:
+    from services.meshtastic_mqtt_settings import redacted_meshtastic_mqtt_settings
+    from services.sigint_bridge import sigint_grid
+
+    return {
+        **redacted_meshtastic_mqtt_settings(),
+        "runtime": sigint_grid.mesh.status(),
+    }
+
+
+@router.get("/api/settings/meshtastic-mqtt", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_get_meshtastic_mqtt_settings(request: Request):
+    return _meshtastic_runtime_snapshot()
+
+
+@router.put("/api/settings/meshtastic-mqtt", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_set_meshtastic_mqtt_settings(request: Request, body: MeshtasticMqttUpdate):
+    from services.meshtastic_mqtt_settings import write_meshtastic_mqtt_settings
+    from services.sigint_bridge import sigint_grid
+
+    updates = body.model_dump(exclude_unset=True)
+    # Empty secret fields mean "keep existing"; explicit non-empty values replace.
+    if updates.get("password") == "":
+        updates.pop("password", None)
+    if updates.get("psk") == "":
+        updates.pop("psk", None)
+
+    enabled_requested = updates.get("enabled")
+    settings = write_meshtastic_mqtt_settings(**updates)
+    if isinstance(enabled_requested, bool):
+        logger.info("Meshtastic MQTT settings update: enabled=%s", enabled_requested)
+
+    if enabled_requested is True:
+        # Public MQTT and Wormhole are intentionally mutually exclusive lanes.
+        try:
+            from services.node_settings import write_node_settings
+            from services.wormhole_settings import write_wormhole_settings
+            from services.wormhole_supervisor import disconnect_wormhole
+
+            write_wormhole_settings(enabled=False)
+            disconnect_wormhole(reason="public_mesh_enabled")
+            write_node_settings(enabled=False)
+            _set_participant_node_enabled(False)
+        except Exception as exc:
+            logger.warning("Failed to disable private mesh lane while enabling public mesh: %s", exc)
+
+    if bool(settings.get("enabled")):
+        if sigint_grid.mesh.is_running():
+            sigint_grid.mesh.stop()
+            threading.Timer(1.0, sigint_grid.mesh.start).start()
+        else:
+            sigint_grid.mesh.start()
+    else:
+        sigint_grid.mesh.stop()
+
+    return _meshtastic_runtime_snapshot()
+
+
+@router.get(
+    "/api/settings/timemachine",
+    dependencies=[Depends(require_local_operator)],
+)
+@limiter.limit("30/minute")
+async def api_get_timemachine_settings(request: Request):
+    """Issue #253 (tg12): archival-capture posture is operationally
+    sensitive — it tells a remote caller whether this deployment is
+    retaining replayable historical surveillance data. Gated on
+    local-operator so the Tauri shell and Docker bridge frontend
+    still see the toggle state, but anonymous LAN/internet callers
+    can no longer fingerprint Time Machine state.
+    """
+    import asyncio
+    from services.node_settings import read_node_settings
+    data = await asyncio.to_thread(read_node_settings)
+    return {
+        "enabled": data.get("timemachine_enabled", False),
+        "storage_warning": "Time Machine auto-snapshots use ~68 MB/day compressed (~2 GB/month). "
+                           "Snapshots capture entity positions (flights, ships, satellites) for historical playback.",
+    }
+
+
+@router.put("/api/settings/timemachine", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_set_timemachine_settings(request: Request, body: TimeMachineToggle):
+    import asyncio
+    from services.node_settings import write_node_settings
+    result = await asyncio.to_thread(write_node_settings, timemachine_enabled=body.enabled)
+    return {
+        "ok": True,
+        "enabled": result.get("timemachine_enabled", False),
+    }
+
+
+@router.post("/api/system/update", dependencies=[Depends(require_admin)])
+@limiter.limit("1/minute")
+async def system_update(request: Request):
+    """Download latest release, backup current files, extract update, and restart."""
+    from services.updater import perform_update, schedule_restart
+    candidate = Path(__file__).resolve().parent.parent.parent
+    if (candidate / "frontend").is_dir() or (candidate / "backend").is_dir():
+        project_root = str(candidate)
+    else:
+        project_root = os.getcwd()
+    result = perform_update(project_root)
+    if result.get("status") == "error":
+        return Response(content=json_mod.dumps(result), status_code=500, media_type="application/json")
+    if result.get("status") == "docker":
+        return result
+    threading.Timer(2.0, schedule_restart, args=[project_root]).start()
+    return result
+
+
+# ── Tor Hidden Service ──────────────────────────────────────────────
+
+
+@router.get("/api/settings/tor", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_tor_status(request: Request):
+    """Return Tor hidden service status and .onion address if available."""
+    import asyncio
+    from services.tor_hidden_service import tor_service
+
+    return await asyncio.to_thread(tor_service.status)
+
+
+@router.post("/api/settings/tor/start", dependencies=[Depends(require_local_operator)])
+@limiter.limit("5/minute")
+async def api_tor_start(request: Request):
+    """Start Tor and provision a hidden service for this ShadowBroker instance.
+
+    Also enables MESH_ARTI so the mesh/wormhole system can route traffic
+    through the Tor SOCKS proxy (port 9050) automatically.
+    """
+    import asyncio
+    from services.tor_hidden_service import tor_service
+
+    result = await asyncio.to_thread(tor_service.start)
+
+    # If Tor started successfully, enable Arti (Tor SOCKS proxy for mesh)
+    if result.get("ok"):
+        try:
+            from routers.ai_intel import _write_env_value
+            from services.config import get_settings
+            _write_env_value("MESH_ARTI_ENABLED", "true")
+            get_settings.cache_clear()
+        except Exception:
+            pass  # Non-fatal — hidden service still works without mesh Arti
+
+    return result
+
+
+@router.post("/api/settings/tor/reset-identity", dependencies=[Depends(require_local_operator)])
+@limiter.limit("2/minute")
+async def api_tor_reset_identity(request: Request):
+    """Destroy current .onion identity and generate a fresh one on next start.
+
+    This is irreversible — the old .onion address is permanently lost.
+    """
+    import asyncio, shutil
+    from services.tor_hidden_service import tor_service, TOR_DIR
+
+    # Stop Tor if running
+    await asyncio.to_thread(tor_service.stop)
+
+    # Delete the hidden service directory (contains the private key)
+    hs_dir = TOR_DIR / "hidden_service"
+    if hs_dir.exists():
+        shutil.rmtree(str(hs_dir), ignore_errors=True)
+
+    # Clear cached address
+    tor_service._onion_address = ""
+
+    return {"ok": True, "detail": "Tor identity destroyed. A new .onion will be generated on next start."}
+
+
+@router.post("/api/settings/agent/reset-all", dependencies=[Depends(require_local_operator)])
+@limiter.limit("2/minute")
+async def api_reset_all_agent_credentials(request: Request):
+    """Nuclear reset: regenerate HMAC key, destroy .onion, revoke agent identity.
+
+    After this, the agent is fully disconnected and needs new credentials.
+    """
+    import asyncio, secrets, shutil
+    from services.tor_hidden_service import tor_service, TOR_DIR
+    from services.config import get_settings
+
+    results = {}
+
+    # 1. Regenerate HMAC key
+    new_secret = secrets.token_hex(24)
+    from routers.ai_intel import _write_env_value
+    _write_env_value("OPENCLAW_HMAC_SECRET", new_secret)
+    results["hmac"] = "regenerated"
+
+    # 2. Revoke agent identity (Ed25519 keypair)
+    try:
+        from services.openclaw_bridge import revoke_agent_identity
+        revoke_agent_identity()
+        results["identity"] = "revoked"
+    except Exception as e:
+        results["identity"] = f"error: {e}"
+
+    # 3. Destroy .onion and restart Tor with new identity
+    await asyncio.to_thread(tor_service.stop)
+    hs_dir = TOR_DIR / "hidden_service"
+    if hs_dir.exists():
+        shutil.rmtree(str(hs_dir), ignore_errors=True)
+    tor_service._onion_address = ""
+    results["tor"] = "identity destroyed"
+
+    # 4. Bootstrap fresh identity + start Tor with new .onion
+    try:
+        from services.openclaw_bridge import generate_agent_keypair
+        keypair = generate_agent_keypair(force=True)
+        results["new_node_id"] = keypair.get("node_id", "")
+    except Exception as e:
+        results["new_node_id"] = f"error: {e}"
+
+    tor_result = await asyncio.to_thread(tor_service.start)
+    results["new_onion"] = tor_result.get("onion_address", "")
+    results["tor_ok"] = tor_result.get("ok", False)
+
+    # Clear settings cache
+    get_settings.cache_clear()
+
+    return {
+        "ok": True,
+        "hmac_regenerated": True,
+        "detail": "All agent credentials have been reset. Use the agent connection screen to generate or reveal replacement credentials.",
+        **results,
+    }
+
+
+@router.post("/api/settings/tor/stop", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_tor_stop(request: Request):
+    """Stop the Tor hidden service."""
+    import asyncio
+    from services.tor_hidden_service import tor_service
+
+    return await asyncio.to_thread(tor_service.stop)

backend/routers/agent_shell.py

@@ -0,0 +1,230 @@
+"""Local-operator PTY WebSocket for the Mesh Chat agent shell."""
+
+from __future__ import annotations
+
+import asyncio
+import fcntl
+import hmac
+import json
+import logging
+import os
+import pty
+import select
+import signal
+import struct
+import sys
+import termios
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException, Query, WebSocket, WebSocketDisconnect
+from pydantic import BaseModel, Field
+
+from auth import (
+    _current_admin_key,
+    _debug_mode_enabled,
+    _is_trusted_local_runtime_host,
+    require_local_operator,
+)
+from services.agent_shell_settings import (
+    get_agent_shell_settings,
+    set_agent_shell_working_directory,
+)
+
+logger = logging.getLogger(__name__)
+router = APIRouter(tags=["agent-shell"])
+
+
+class AgentShellSettingsUpdate(BaseModel):
+    working_directory: str = Field(min_length=1)
+
+
+def _set_winsize(fd: int, rows: int, cols: int) -> None:
+    winsize = struct.pack("HHHH", rows, cols, 0, 0)
+    fcntl.ioctl(fd, termios.TIOCSWINSZ, winsize)
+
+
+def _published_local_dashboard_ws(ws: WebSocket) -> bool:
+    """Browser → published Docker port appears as a bridge IP, not loopback.
+
+    For the operator shell only, also accept when the upgrade request clearly
+    targets the local dashboard (Host/Origin on localhost).
+    """
+    host_header = str(ws.headers.get("host") or "").strip().lower()
+    host_name = host_header.split(":", 1)[0]
+    if host_name in {"127.0.0.1", "localhost", "::1"}:
+        return True
+
+    origin = str(ws.headers.get("origin") or "").strip().lower()
+    if origin.startswith("http://127.0.0.1:") or origin.startswith("http://localhost:"):
+        return True
+    if origin.startswith("https://127.0.0.1:") or origin.startswith("https://localhost:"):
+        return True
+    return False
+
+
+async def _authorize_agent_shell_ws(ws: WebSocket, admin_key_query: str = "") -> None:
+    host = (ws.client.host or "").lower() if ws.client else ""
+    if (
+        _is_trusted_local_runtime_host(host)
+        or _published_local_dashboard_ws(ws)
+        or (_debug_mode_enabled() and host == "test")
+    ):
+        return
+    admin_key = _current_admin_key()
+    presented = str(admin_key_query or ws.headers.get("x-admin-key", "") or "").strip()
+    if admin_key and presented and hmac.compare_digest(presented.encode(), admin_key.encode()):
+        return
+    await ws.close(code=4403, reason="local operator access only")
+    raise WebSocketDisconnect()
+
+
+def _resolve_shell_cwd(requested: str) -> str:
+    requested = str(requested or "").strip()
+    if requested:
+        resolved = os.path.abspath(os.path.expanduser(requested))
+        if os.path.isdir(resolved):
+            return resolved
+    return get_agent_shell_settings()["working_directory"]
+
+
+def _default_shell() -> str:
+    if sys.platform == "win32":
+        return os.environ.get("COMSPEC", "cmd.exe")
+    return os.environ.get("SHELL", "/bin/bash")
+
+
+async def _relay_pty(master_fd: int, proc: asyncio.subprocess.Process, ws: WebSocket) -> None:
+    loop = asyncio.get_running_loop()
+    while True:
+        if proc.returncode is not None:
+            break
+        try:
+            readable, _, _ = await loop.run_in_executor(
+                None, lambda: select.select([master_fd], [], [], 0.05)
+            )
+        except Exception:
+            break
+        if master_fd in readable:
+            try:
+                chunk = os.read(master_fd, 4096)
+            except OSError:
+                break
+            if not chunk:
+                break
+            await ws.send_bytes(chunk)
+        try:
+            message = await asyncio.wait_for(ws.receive(), timeout=0.05)
+        except asyncio.TimeoutError:
+            continue
+        if message.get("type") == "websocket.disconnect":
+            break
+        if message.get("type") != "websocket.receive":
+            continue
+        if message.get("bytes"):
+            os.write(master_fd, message["bytes"])
+            continue
+        text = message.get("text")
+        if not text:
+            continue
+        try:
+            payload = json.loads(text)
+        except json.JSONDecodeError:
+            os.write(master_fd, text.encode("utf-8", errors="replace"))
+            continue
+        if payload.get("type") == "resize":
+            rows = int(payload.get("rows") or 24)
+            cols = int(payload.get("cols") or 80)
+            _set_winsize(master_fd, max(rows, 2), max(cols, 2))
+
+
+@router.get("/api/agent-shell/settings", dependencies=[Depends(require_local_operator)])
+async def read_agent_shell_settings() -> dict[str, Any]:
+    return get_agent_shell_settings()
+
+
+@router.put("/api/agent-shell/settings", dependencies=[Depends(require_local_operator)])
+async def write_agent_shell_settings(body: AgentShellSettingsUpdate) -> dict[str, Any]:
+    try:
+        return set_agent_shell_working_directory(body.working_directory)
+    except ValueError as exc:
+        detail = str(exc)
+        if detail == "working_directory_not_found":
+            raise HTTPException(status_code=400, detail="Working directory does not exist") from exc
+        raise HTTPException(status_code=400, detail="Working directory is required") from exc
+
+
+@router.websocket("/api/agent-shell/ws")
+async def agent_shell_websocket(
+    ws: WebSocket,
+    cwd: str = Query(default=""),
+    cols: int = Query(default=80),
+    rows: int = Query(default=24),
+    admin_key: str = Query(default=""),
+) -> None:
+    await ws.accept()
+    try:
+        await _authorize_agent_shell_ws(ws, admin_key)
+    except WebSocketDisconnect:
+        return
+
+    if sys.platform == "win32":
+        await ws.send_text(
+            json.dumps(
+                {
+                    "type": "error",
+                    "message": "Host PTY is not available on Windows backend builds yet. Use the ShadowBroker desktop app or run the backend in Docker/Linux for an embedded shell.",
+                }
+            )
+        )
+        await ws.close(code=1011)
+        return
+
+    shell_cwd = _resolve_shell_cwd(cwd)
+    shell = _default_shell()
+    master_fd, slave_fd = pty.openpty()
+    _set_winsize(master_fd, max(rows, 2), max(cols, 2))
+
+    env = os.environ.copy()
+    env.setdefault("TERM", "xterm-256color")
+    env.setdefault("COLORTERM", "truecolor")
+    home = shell_cwd if os.path.isdir(shell_cwd) else "/app"
+    env["HOME"] = home
+    env["USER"] = env.get("USER") or "operator"
+    path_prefixes = [
+        os.path.join(home, ".local", "bin"),
+        os.path.join(home, ".hermes", "bin"),
+    ]
+    path = env.get("PATH", "")
+    for prefix in path_prefixes:
+        if os.path.isdir(prefix):
+            path = f"{prefix}:{path}" if path else prefix
+    env["PATH"] = path
+
+    proc = await asyncio.create_subprocess_exec(
+        shell,
+        stdin=slave_fd,
+        stdout=slave_fd,
+        stderr=slave_fd,
+        cwd=shell_cwd,
+        env=env,
+        preexec_fn=os.setsid,
+    )
+    os.close(slave_fd)
+
+    try:
+        await _relay_pty(master_fd, proc, ws)
+    finally:
+        try:
+            os.close(master_fd)
+        except OSError:
+            pass
+        if proc.returncode is None:
+            try:
+                os.killpg(proc.pid, signal.SIGHUP)
+            except ProcessLookupError:
+                pass
+            try:
+                await asyncio.wait_for(proc.wait(), timeout=2.0)
+            except asyncio.TimeoutError:
+                proc.kill()
+                await proc.wait()

backend/routers/analytics.py

@@ -0,0 +1,339 @@
+"""Strategic Risk Analytics API — game-theoretic early warning overlays."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from pydantic import BaseModel, Field
+
+from auth import require_local_operator
+from limiter import limiter
+from analytics.backtest import (
+    DEFAULT_BACKTEST_ALERT_THRESHOLD,
+    run_historical_backtest,
+    tune_alert_threshold,
+)
+from analytics.feed_adapter import normalize_feed_item
+from analytics.integration import get_gt_engine, refresh_from_latest_data
+from analytics.gt_alerts import top_gt_alerts
+from analytics.micro_rolling import micro_rolling_report
+from analytics.rolling_backtest import (
+    freeze_weekly_snapshot,
+    label_region,
+    label_regions,
+    rolling_alert_threshold,
+    rolling_report,
+    score_week,
+)
+from analytics.weekly_store import load_week
+from analytics.settings import gt_analytics_enabled
+from services.fetchers._store import _data_lock, get_latest_data_subset_refs, latest_data
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+class RiskHeatmapRequest(BaseModel):
+    """Optional batch ingest + refresh controls for POST /api/analytics/risk_heatmap."""
+
+    refresh: bool = True
+    items: list[dict[str, Any]] = Field(default_factory=list)
+
+
+class RollingFreezeRequest(BaseModel):
+    week_id: str | None = None
+    force: bool = False
+
+
+class RollingLabelEntry(BaseModel):
+    region: str
+    label: str
+    notes: str = ""
+
+
+class RollingLabelRequest(BaseModel):
+    week_id: str
+    labels: list[RollingLabelEntry] = Field(default_factory=list)
+
+
+def _empty_heatmap() -> dict[str, Any]:
+    return {
+        "enabled": False,
+        "type": "FeatureCollection",
+        "features": [],
+        "clusters": [],
+        "processed": 0,
+        "timestamp": None,
+    }
+
+
+def _gt_risk_payload() -> dict[str, Any]:
+    snap = get_latest_data_subset_refs("gt_risk")
+    payload = snap.get("gt_risk")
+    if not isinstance(payload, dict):
+        return _empty_heatmap()
+    heatmap = payload.get("heatmap") or {"type": "FeatureCollection", "features": []}
+    return {
+        "enabled": bool(payload.get("enabled")),
+        "type": heatmap.get("type", "FeatureCollection"),
+        "features": list(heatmap.get("features") or []),
+        "clusters": list(payload.get("clusters") or []),
+        "processed": int(payload.get("processed") or 0),
+        "timestamp": payload.get("timestamp"),
+    }
+
+
+@router.get("/api/analytics/risk_heatmap")
+@limiter.limit("60/minute")
+async def risk_heatmap_get(request: Request) -> dict[str, Any]:
+    """Return cached GeoJSON risk overlay (posterior scores per region)."""
+    if not gt_analytics_enabled():
+        return _empty_heatmap()
+    return _gt_risk_payload()
+
+
+@router.post("/api/analytics/risk_heatmap")
+@limiter.limit("12/minute")
+async def risk_heatmap_post(
+    request: Request,
+    body: RiskHeatmapRequest,
+    _: None = Depends(require_local_operator),
+) -> dict[str, Any]:
+    """
+    Ingest optional feed items and/or refresh beliefs from latest intel layers.
+
+    Requires local operator auth — intended for OpenClaw agents and admin tooling.
+    """
+    if not gt_analytics_enabled():
+        raise HTTPException(status_code=503, detail="Strategic Risk Analytics is disabled")
+
+    engine = get_gt_engine()
+    if engine is None:
+        raise HTTPException(status_code=503, detail="Strategic Risk Analytics engine unavailable")
+
+    ingested = 0
+    for raw in body.items:
+        if not isinstance(raw, dict):
+            continue
+        source_type = str(raw.get("source_type") or "manual")
+        item = normalize_feed_item(raw, source_type=source_type)
+        result = engine.process_feed_item(item)
+        if result and not result.get("skipped"):
+            ingested += 1
+
+    summary: dict[str, Any] = {"ingested": ingested}
+    if body.refresh:
+        with _data_lock:
+            snapshot = dict(latest_data)
+        summary.update(refresh_from_latest_data(snapshot, persist=True))
+
+    payload = _gt_risk_payload()
+    payload["ingested"] = ingested
+    payload["refresh"] = bool(body.refresh)
+    return payload
+
+
+@router.get("/api/analytics/dossier/{region}")
+@limiter.limit("30/minute")
+async def analytics_dossier(request: Request, region: str) -> dict[str, Any]:
+    """Game-theoretic rationale, recent costly signals, and scenario sketches."""
+    region_key = str(region or "").strip().lower()
+    if not region_key or len(region_key) > 120:
+        raise HTTPException(status_code=400, detail="Invalid region identifier")
+
+    if not gt_analytics_enabled():
+        return {
+            "enabled": False,
+            "region": region_key,
+            "current_risk": 0.0,
+            "interpretation": "Strategic Risk Analytics is disabled.",
+            "recent_signals": [],
+            "scenarios": [],
+        }
+
+    engine = get_gt_engine()
+    if engine is None:
+        raise HTTPException(status_code=503, detail="Strategic Risk Analytics engine unavailable")
+
+    dossier = engine.get_dossier(region_key)
+    dossier["enabled"] = True
+    return dossier
+
+
+@router.get("/api/analytics/backtest")
+@limiter.limit("6/minute")
+async def analytics_backtest(
+    request: Request,
+    expanded: bool = True,
+    tune: bool = False,
+    target_confidence: float = 0.95,
+) -> dict[str, Any]:
+    """
+    Run labeled historical backtest and return accuracy + Wilson 95% CI.
+
+    ``confidence_rate`` is the Wilson lower bound (conservative pass metric).
+    """
+    if not gt_analytics_enabled():
+        return {
+            "enabled": False,
+            "message": "Strategic Risk Analytics is disabled.",
+        }
+
+    if tune:
+        threshold, report = tune_alert_threshold(target_confidence=target_confidence)
+    else:
+        threshold = DEFAULT_BACKTEST_ALERT_THRESHOLD
+        report = run_historical_backtest(
+            use_expanded_suite=expanded,
+            alert_threshold=threshold,
+            target_confidence=target_confidence,
+        )
+
+    payload = report.to_dict()
+    payload["enabled"] = True
+    payload["expanded_suite"] = expanded
+    payload["tuned"] = tune
+    payload["recommended_alert_threshold"] = threshold
+    return payload
+
+
+@router.get("/api/analytics/rolling")
+@limiter.limit("12/minute")
+async def analytics_rolling(
+    request: Request,
+    weeks: int = 8,
+    target_confidence: float = 0.80,
+) -> dict[str, Any]:
+    """Rolling weekly operational validation — accuracy trend with delayed labels."""
+    if not gt_analytics_enabled():
+        return {
+            "enabled": False,
+            "message": "Strategic Risk Analytics is disabled.",
+        }
+
+    report = rolling_report(weeks=max(1, min(weeks, 52)), target_confidence=target_confidence)
+    report["enabled"] = True
+    return report
+
+
+@router.get("/api/analytics/alerts")
+@limiter.limit("30/minute")
+async def analytics_top_alerts(
+    request: Request,
+    limit: int = 8,
+) -> dict[str, Any]:
+    """Top GT risk regions ranked by score — fly-to targets for the map."""
+    if not gt_analytics_enabled():
+        return {
+            "enabled": False,
+            "message": "Strategic Risk Analytics is disabled.",
+        }
+
+    report = top_gt_alerts(limit=max(1, min(limit, 25)))
+    report["enabled"] = True
+    return report
+
+
+@router.get("/api/analytics/rolling/micro")
+@limiter.limit("30/minute")
+async def analytics_rolling_micro(
+    request: Request,
+    window_days: int = 3,
+    limit: int = 15,
+) -> dict[str, Any]:
+    """Rolling 3-day micro average — spot vs baseline, ignition detection."""
+    if not gt_analytics_enabled():
+        return {
+            "enabled": False,
+            "message": "Strategic Risk Analytics is disabled.",
+        }
+
+    report = micro_rolling_report(
+        window_days=max(2, min(window_days, 7)),
+        limit=max(1, min(limit, 50)),
+    )
+    report["enabled"] = True
+    return report
+
+
+@router.get("/api/analytics/rolling/{week_id}")
+@limiter.limit("12/minute")
+async def analytics_rolling_week(request: Request, week_id: str) -> dict[str, Any]:
+    """Return a single frozen week snapshot and its score."""
+    if not gt_analytics_enabled():
+        return {"enabled": False, "message": "Strategic Risk Analytics is disabled."}
+
+    snapshot = load_week(str(week_id).strip())
+    if snapshot is None:
+        raise HTTPException(status_code=404, detail=f"Week {week_id} not found")
+
+    score = score_week(snapshot)
+    return {
+        "enabled": True,
+        "week_id": snapshot.week_id,
+        "snapshot": snapshot.to_dict(),
+        "score": score.to_dict(),
+        "alert_threshold": rolling_alert_threshold(),
+    }
+
+
+@router.post("/api/analytics/rolling/freeze")
+@limiter.limit("6/minute")
+async def analytics_rolling_freeze(
+    request: Request,
+    body: RollingFreezeRequest,
+    _: None = Depends(require_local_operator),
+) -> dict[str, Any]:
+    """Freeze current GT scores for the ISO week (idempotent unless force=true)."""
+    if not gt_analytics_enabled():
+        raise HTTPException(status_code=503, detail="Strategic Risk Analytics is disabled")
+
+    result = freeze_weekly_snapshot(
+        week_id=body.week_id,
+        force=body.force,
+        frozen_by="api",
+    )
+    if not result.get("ok"):
+        raise HTTPException(status_code=503, detail=result.get("detail", "Freeze failed"))
+    result["enabled"] = True
+    return result
+
+
+@router.post("/api/analytics/rolling/label")
+@limiter.limit("12/minute")
+async def analytics_rolling_label(
+    request: Request,
+    body: RollingLabelRequest,
+    _: None = Depends(require_local_operator),
+) -> dict[str, Any]:
+    """Apply delayed outcome labels to a frozen week."""
+    if not gt_analytics_enabled():
+        raise HTTPException(status_code=503, detail="Strategic Risk Analytics is disabled")
+
+    week_id = str(body.week_id or "").strip()
+    if not week_id:
+        raise HTTPException(status_code=400, detail="week_id required")
+
+    if len(body.labels) == 1:
+        entry = body.labels[0]
+        result = label_region(
+            week_id,
+            entry.region,
+            entry.label,  # type: ignore[arg-type]
+            notes=entry.notes,
+            labeled_by="api",
+        )
+    else:
+        result = label_regions(
+            week_id,
+            [row.model_dump() for row in body.labels],
+            labeled_by="api",
+        )
+
+    if not result.get("ok"):
+        raise HTTPException(status_code=404, detail=result.get("detail", "Label failed"))
+    result["enabled"] = True
+    return result

backend/routers/cctv.py

@@ -0,0 +1,401 @@
+import logging
+from dataclasses import dataclass, field
+from fastapi import APIRouter, Request, Query, HTTPException
+from fastapi.responses import StreamingResponse
+from starlette.background import BackgroundTask
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+_CCTV_PROXY_CONNECT_TIMEOUT_S = 2.0
+
+_CCTV_PROXY_ALLOWED_HOSTS = {
+    "s3-eu-west-1.amazonaws.com",
+    "jamcams.tfl.gov.uk",
+    "images.data.gov.sg",
+    "cctv.austinmobility.io",
+    "webcams.nyctmc.org",
+    "cwwp2.dot.ca.gov",
+    "wzmedia.dot.ca.gov",
+    "images.wsdot.wa.gov",
+    "olypen.com",
+    "flyykm.com",
+    "cam.pangbornairport.com",
+    "navigator-c2c.dot.ga.gov",
+    "navigator-c2c.ga.gov",
+    "navigator-csc.dot.ga.gov",
+    "vss1live.dot.ga.gov",
+    "vss2live.dot.ga.gov",
+    "vss3live.dot.ga.gov",
+    "vss4live.dot.ga.gov",
+    "vss5live.dot.ga.gov",
+    "511ga.org",
+    "gettingaroundillinois.com",
+    "cctv.travelmidwest.com",
+    "mdotjboss.state.mi.us",
+    "micamerasimages.net",
+    "publicstreamer1.cotrip.org",
+    "publicstreamer2.cotrip.org",
+    "publicstreamer3.cotrip.org",
+    "publicstreamer4.cotrip.org",
+    "cocam.carsprogram.org",
+    "tripcheck.com",
+    "www.tripcheck.com",
+    "infocar.dgt.es",
+    "informo.madrid.es",
+    "webcams2.asfinag.at",
+    "odo.asfinag.at",
+    "www.windy.com",
+    "imgproxy.windy.com",
+    "www.lakecountypassage.com",
+    "webcam.forkswa.com",
+    "webcam.sunmountainlodge.com",
+    "www.nps.gov",
+    "home.lewiscounty.com",
+    "www.seattle.gov",
+    "511on.ca",
+    "511.alberta.ca",
+    "fl511.com",
+    "www.fl511.com",
+    "webcams.transport.nsw.gov.au",
+    "www.livetraffic.com",
+    "livetraffic.com",
+    "opendata.ndw.nu",
+}
+
+
+@dataclass(frozen=True)
+class _CCTVProxyProfile:
+    name: str
+    timeout: tuple = (_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0)
+    cache_seconds: int = 30
+    headers: dict = field(default_factory=dict)
+
+
+def _cctv_host_allowed(hostname) -> bool:
+    host = str(hostname or "").strip().lower()
+    if not host:
+        return False
+    for allowed in _CCTV_PROXY_ALLOWED_HOSTS:
+        normalized = str(allowed or "").strip().lower()
+        if host == normalized or host.endswith(f".{normalized}"):
+            return True
+    return False
+
+
+def _proxied_cctv_url(target_url: str) -> str:
+    from urllib.parse import quote
+    return f"/api/cctv/media?url={quote(target_url, safe='')}"
+
+
+def _cctv_proxy_profile_for_url(target_url: str) -> _CCTVProxyProfile:
+    from urllib.parse import urlparse
+    parsed = urlparse(target_url)
+    host = str(parsed.hostname or "").strip().lower()
+    path = str(parsed.path or "").strip().lower()
+
+    if host in {"jamcams.tfl.gov.uk", "s3-eu-west-1.amazonaws.com"}:
+        return _CCTVProxyProfile(name="tfl-jamcam", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 20.0), cache_seconds=15,
+            headers={"Accept": "video/mp4,image/avif,image/webp,image/apng,image/*,*/*;q=0.8", "Referer": "https://tfl.gov.uk/"})
+    if host == "images.data.gov.sg":
+        return _CCTVProxyProfile(name="lta-singapore", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host == "cctv.austinmobility.io":
+        return _CCTVProxyProfile(name="austin-mobility", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://data.mobility.austin.gov/", "Origin": "https://data.mobility.austin.gov"})
+    if host == "webcams.nyctmc.org":
+        return _CCTVProxyProfile(name="nyc-dot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host in {"cwwp2.dot.ca.gov", "wzmedia.dot.ca.gov"}:
+        return _CCTVProxyProfile(name="caltrans", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=15,
+            headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,image/*,*/*;q=0.8",
+                     "Referer": "https://cwwp2.dot.ca.gov/"})
+    if host in {"images.wsdot.wa.gov", "olypen.com", "flyykm.com", "cam.pangbornairport.com"}:
+        return _CCTVProxyProfile(name="wsdot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host in {"www.lakecountypassage.com", "webcam.forkswa.com", "webcam.sunmountainlodge.com", "home.lewiscounty.com", "www.seattle.gov"}:
+        return _CCTVProxyProfile(name="regional-cctv-image", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=45,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": f"https://{host}/"})
+    if host == "www.nps.gov":
+        return _CCTVProxyProfile(name="nps-webcam", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.nps.gov/"})
+    if host in {"navigator-c2c.dot.ga.gov", "navigator-c2c.ga.gov", "navigator-csc.dot.ga.gov"}:
+        read_timeout = 18.0 if "/snapshots/" in path else 12.0
+        return _CCTVProxyProfile(name="gdot-snapshot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, read_timeout), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://navigator-c2c.dot.ga.gov/"})
+    if host == "511ga.org":
+        return _CCTVProxyProfile(name="gdot-511ga-image", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://511ga.org/cctv"})
+    if host.startswith("vss") and host.endswith("dot.ga.gov"):
+        return _CCTVProxyProfile(name="gdot-hls", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 20.0), cache_seconds=10,
+            headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,*/*;q=0.8",
+                     "Referer": "https://navigator-c2c.dot.ga.gov/"})
+    if host in {"gettingaroundillinois.com", "cctv.travelmidwest.com"}:
+        return _CCTVProxyProfile(name="illinois-dot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host in {"mdotjboss.state.mi.us", "micamerasimages.net"}:
+        return _CCTVProxyProfile(name="michigan-dot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://mdotjboss.state.mi.us/"})
+    if host in {"publicstreamer1.cotrip.org", "publicstreamer2.cotrip.org",
+                "publicstreamer3.cotrip.org", "publicstreamer4.cotrip.org"}:
+        return _CCTVProxyProfile(name="cotrip-hls", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 20.0), cache_seconds=10,
+            headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,*/*;q=0.8",
+                     "Referer": "https://www.cotrip.org/"})
+    if host == "cocam.carsprogram.org":
+        return _CCTVProxyProfile(name="cotrip-preview", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=20,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.cotrip.org/"})
+    if host in {"tripcheck.com", "www.tripcheck.com"}:
+        return _CCTVProxyProfile(name="odot-tripcheck", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host == "infocar.dgt.es":
+        return _CCTVProxyProfile(name="dgt-spain", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://infocar.dgt.es/"})
+    if host == "informo.madrid.es":
+        return _CCTVProxyProfile(name="madrid-city", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://informo.madrid.es/"})
+    if host in {"webcams2.asfinag.at", "odo.asfinag.at"}:
+        return _CCTVProxyProfile(name="asfinag-austria", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.asfinag.at/"})
+    if host in {"www.windy.com", "imgproxy.windy.com"}:
+        return _CCTVProxyProfile(name="windy-webcams", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.windy.com/"})
+    if host == "511on.ca":
+        return _CCTVProxyProfile(name="ontario-511", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://511on.ca/"})
+    if host == "511.alberta.ca":
+        return _CCTVProxyProfile(name="alberta-511", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://511.alberta.ca/"})
+    if host in {"fl511.com", "www.fl511.com"}:
+        return _CCTVProxyProfile(name="florida-511", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://fl511.com/"})
+    if host == "webcams.transport.nsw.gov.au":
+        return _CCTVProxyProfile(name="nsw-live-traffic", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.livetraffic.com/"})
+    if host in {"opendata.ndw.nu", "www.ndw.nu"}:
+        return _CCTVProxyProfile(name="ndw-netherlands", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=120,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.ndw.nu/"})
+    return _CCTVProxyProfile(name="generic-cctv", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0), cache_seconds=30,
+        headers={"Accept": "*/*"})
+
+
+def _cctv_upstream_headers(request: Request, profile: _CCTVProxyProfile) -> dict:
+    # Round 7a: per-install operator handle. Mozilla/5.0 prefix retained
+    # because many CCTV endpoints sniff for a browser-like prefix.
+    from services.network_utils import outbound_user_agent
+    headers = {
+        "User-Agent": f"Mozilla/5.0 (compatible; {outbound_user_agent('cctv-proxy')})",
+        **profile.headers,
+    }
+    range_header = request.headers.get("range")
+    if range_header:
+        headers["Range"] = range_header
+    if_none_match = request.headers.get("if-none-match")
+    if if_none_match:
+        headers["If-None-Match"] = if_none_match
+    if_modified_since = request.headers.get("if-modified-since")
+    if if_modified_since:
+        headers["If-Modified-Since"] = if_modified_since
+    return headers
+
+
+def _cctv_response_headers(resp, cache_seconds: int, include_length: bool = True) -> dict:
+    headers = {"Cache-Control": f"public, max-age={cache_seconds}", "Access-Control-Allow-Origin": "*"}
+    for key in ("Accept-Ranges", "Content-Range", "ETag", "Last-Modified"):
+        value = resp.headers.get(key)
+        if value:
+            headers[key] = value
+    if include_length:
+        content_length = resp.headers.get("Content-Length")
+        if content_length:
+            headers["Content-Length"] = content_length
+    return headers
+
+
+# Maximum number of redirects we'll follow on the CCTV upstream. Each hop is
+# re-validated against _cctv_host_allowed() before continuing, so this caps
+# the redirect-chain SSRF blast radius.
+_CCTV_MAX_REDIRECTS = 5
+
+
+def _fetch_cctv_upstream_response(request: Request, target_url: str, profile: _CCTVProxyProfile):
+    """Fetch an upstream CCTV URL, following redirects manually with host re-validation.
+
+    Why manual redirect following:
+      The original code used ``allow_redirects=True``, which only validated
+      the initial caller-supplied URL host against the allowlist. An attacker
+      could submit an allowed host that 302-redirected to an internal address
+      (e.g. ``http://localhost:8000/api/...`` or a private RFC1918 range),
+      and the backend would dutifully follow and proxy the response — a
+      classic open-redirect-to-SSRF chain.
+
+      With this loop, we re-run ``_cctv_host_allowed()`` on every hop's
+      ``Location`` header. A redirect to a host that isn't on the allowlist
+      is rejected with 502 rather than silently followed.
+    """
+    import requests as _req
+    from urllib.parse import urlparse, urljoin
+
+    headers = _cctv_upstream_headers(request, profile)
+    current_url = target_url
+    hops = 0
+    try:
+        while True:
+            resp = _req.get(
+                current_url,
+                timeout=profile.timeout,
+                stream=True,
+                allow_redirects=False,
+                headers=headers,
+            )
+            # Redirect handling — re-validate the next-hop host before following.
+            if resp.is_redirect or resp.status_code in (301, 302, 303, 307, 308):
+                location = resp.headers.get("Location", "")
+                resp.close()
+                if hops >= _CCTV_MAX_REDIRECTS:
+                    logger.warning(
+                        "CCTV upstream redirect chain exceeded limit [%s] %s",
+                        profile.name, target_url,
+                    )
+                    raise HTTPException(status_code=502, detail="Upstream redirect chain too long")
+                if not location:
+                    raise HTTPException(status_code=502, detail="Upstream redirect missing Location")
+                next_url = urljoin(current_url, location)
+                next_parsed = urlparse(next_url)
+                if next_parsed.scheme not in ("http", "https"):
+                    raise HTTPException(status_code=502, detail="Upstream redirect to non-HTTP scheme")
+                if not _cctv_host_allowed(next_parsed.hostname):
+                    logger.warning(
+                        "CCTV upstream redirect to disallowed host [%s] %s -> %s",
+                        profile.name, current_url, next_url,
+                    )
+                    raise HTTPException(status_code=502, detail="Upstream redirect to disallowed host")
+                current_url = next_url
+                hops += 1
+                continue
+            break
+    except _req.exceptions.Timeout as exc:
+        logger.warning("CCTV upstream timeout [%s] %s", profile.name, target_url)
+        raise HTTPException(status_code=504, detail="Upstream timeout") from exc
+    except _req.exceptions.RequestException as exc:
+        logger.warning("CCTV upstream request failure [%s] %s: %s", profile.name, target_url, exc)
+        raise HTTPException(status_code=502, detail="Upstream fetch failed") from exc
+    if resp.status_code >= 400:
+        logger.info("CCTV upstream HTTP %s [%s] %s", resp.status_code, profile.name, target_url)
+        resp.close()
+        raise HTTPException(status_code=int(resp.status_code), detail=f"Upstream returned {resp.status_code}")
+    return resp
+
+
+def _rewrite_cctv_hls_playlist(base_url: str, body: str) -> str:
+    import re
+    from urllib.parse import urljoin, urlparse
+
+    def _rewrite_target(target: str) -> str:
+        candidate = str(target or "").strip()
+        if not candidate or candidate.startswith("data:"):
+            return candidate
+        absolute = urljoin(base_url, candidate)
+        parsed_target = urlparse(absolute)
+        if parsed_target.scheme not in ("http", "https"):
+            return candidate
+        if not _cctv_host_allowed(parsed_target.hostname):
+            return candidate
+        return _proxied_cctv_url(absolute)
+
+    rewritten_lines: list = []
+    for raw_line in body.splitlines():
+        stripped = raw_line.strip()
+        if not stripped:
+            rewritten_lines.append(raw_line)
+            continue
+        if stripped.startswith("#"):
+            rewritten_lines.append(re.sub(r'URI="([^"]+)"',
+                lambda match: f'URI="{_rewrite_target(match.group(1))}"', raw_line))
+            continue
+        rewritten_lines.append(_rewrite_target(stripped))
+    return "\n".join(rewritten_lines) + ("\n" if body.endswith("\n") else "")
+
+
+def _infer_cctv_media_type_from_url(target_url: str, content_type: str) -> str:
+    from urllib.parse import urlparse
+
+    clean_type = str(content_type or "").split(";", 1)[0].strip().lower()
+    if clean_type and clean_type not in {"application/octet-stream", "binary/octet-stream"}:
+        return content_type
+    path = str(urlparse(target_url).path or "").lower()
+    if path.endswith((".jpg", ".jpeg")):
+        return "image/jpeg"
+    if path.endswith(".png"):
+        return "image/png"
+    if path.endswith(".webp"):
+        return "image/webp"
+    if path.endswith(".gif"):
+        return "image/gif"
+    if path.endswith(".mp4"):
+        return "video/mp4"
+    if path.endswith((".m3u8", ".m3u")):
+        return "application/vnd.apple.mpegurl"
+    if path.endswith((".mjpg", ".mjpeg")):
+        return "multipart/x-mixed-replace"
+    return content_type or "application/octet-stream"
+
+
+def _proxy_cctv_media_response(request: Request, target_url: str):
+    from urllib.parse import urlparse
+    from fastapi.responses import Response
+    parsed = urlparse(target_url)
+    profile = _cctv_proxy_profile_for_url(target_url)
+    resp = _fetch_cctv_upstream_response(request, target_url, profile)
+    content_type = _infer_cctv_media_type_from_url(
+        target_url,
+        resp.headers.get("Content-Type", "application/octet-stream"),
+    )
+    is_hls_playlist = (
+        ".m3u8" in str(parsed.path or "").lower()
+        or "mpegurl" in content_type.lower()
+        or "vnd.apple.mpegurl" in content_type.lower()
+    )
+    if is_hls_playlist:
+        body = resp.text
+        if "#EXTM3U" in body:
+            body = _rewrite_cctv_hls_playlist(target_url, body)
+        resp.close()
+        return Response(content=body, media_type=content_type,
+            headers=_cctv_response_headers(resp, cache_seconds=profile.cache_seconds, include_length=False))
+    return StreamingResponse(resp.iter_content(chunk_size=65536), status_code=resp.status_code,
+        media_type=content_type,
+        headers=_cctv_response_headers(resp, cache_seconds=profile.cache_seconds),
+        background=BackgroundTask(resp.close))
+
+
+@router.get("/api/cctv/media")
+@limiter.limit("120/minute")
+async def cctv_media_proxy(request: Request, url: str = Query(...)):
+    """Proxy CCTV media through the backend to bypass browser CORS restrictions."""
+    from urllib.parse import urlparse
+    parsed = urlparse(url)
+    if not _cctv_host_allowed(parsed.hostname):
+        raise HTTPException(status_code=403, detail="Host not allowed")
+    if parsed.scheme not in ("http", "https"):
+        raise HTTPException(status_code=400, detail="Invalid scheme")
+    return _proxy_cctv_media_response(request, url)

backend/routers/data.py

@@ -0,0 +1,911 @@
+import asyncio
+import logging
+import math
+import os
+import threading
+from typing import Any
+from fastapi import APIRouter, Request, Response, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+from services.data_fetcher import update_all_data
+import orjson
+import json as json_mod
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+_refresh_lock = threading.Lock()
+
+
+class ViewportUpdate(BaseModel):
+    s: float
+    w: float
+    n: float
+    e: float
+
+
+class LayerUpdate(BaseModel):
+    layers: dict[str, bool]
+
+
+class LiveUamapOptInUpdate(BaseModel):
+    opted_in: bool
+
+
+class PredictionMarketsOptInUpdate(BaseModel):
+    opted_in: bool
+
+
+_LAST_VIEWPORT_UPDATE: tuple | None = None
+_LAST_VIEWPORT_UPDATE_TS = 0.0
+_VIEWPORT_UPDATE_LOCK = threading.Lock()
+_VIEWPORT_DEDUPE_EPSILON = 1.0
+_VIEWPORT_MIN_UPDATE_S = 10.0
+
+
+def _normalize_longitude(value: float) -> float:
+    normalized = ((value + 180.0) % 360.0 + 360.0) % 360.0 - 180.0
+    if normalized == -180.0 and value > 0:
+        return 180.0
+    return normalized
+
+
+def _normalize_viewport_bounds(s: float, w: float, n: float, e: float) -> tuple:
+    south = max(-90.0, min(90.0, s))
+    north = max(-90.0, min(90.0, n))
+    raw_width = abs(e - w)
+    if not math.isfinite(raw_width) or raw_width >= 360.0:
+        return south, -180.0, north, 180.0
+    west = _normalize_longitude(w)
+    east = _normalize_longitude(e)
+    if east < west:
+        return south, -180.0, north, 180.0
+    return south, west, north, east
+
+
+def _viewport_changed_enough(bounds: tuple) -> bool:
+    global _LAST_VIEWPORT_UPDATE, _LAST_VIEWPORT_UPDATE_TS
+    import time
+    now = time.monotonic()
+    with _VIEWPORT_UPDATE_LOCK:
+        if _LAST_VIEWPORT_UPDATE is None:
+            _LAST_VIEWPORT_UPDATE = bounds
+            _LAST_VIEWPORT_UPDATE_TS = now
+            return True
+        changed = any(
+            abs(current - previous) > _VIEWPORT_DEDUPE_EPSILON
+            for current, previous in zip(bounds, _LAST_VIEWPORT_UPDATE)
+        )
+        if not changed and (now - _LAST_VIEWPORT_UPDATE_TS) < _VIEWPORT_MIN_UPDATE_S:
+            return False
+        if (now - _LAST_VIEWPORT_UPDATE_TS) < _VIEWPORT_MIN_UPDATE_S:
+            return False
+        _LAST_VIEWPORT_UPDATE = bounds
+        _LAST_VIEWPORT_UPDATE_TS = now
+        return True
+
+
+def _queue_viirs_change_refresh() -> None:
+    from services.fetchers.earth_observation import fetch_viirs_change_nodes
+    threading.Thread(target=fetch_viirs_change_nodes, daemon=True).start()
+
+
+def _etag_response(request: Request, payload: dict, prefix: str = "", default=None):
+    etag = _current_etag(prefix)
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    content = json_mod.dumps(_json_safe(payload), default=default, allow_nan=False)
+    return Response(content=content, media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"})
+
+
+def _current_etag(prefix: str = "") -> str:
+    from services.fetchers._store import get_active_layers_version, get_data_version
+    return f"{prefix}v{get_data_version()}-l{get_active_layers_version()}"
+
+
+# ── Issue #288: viewport-aware payloads ─────────────────────────────────────
+# Heavy, density-driven, time-sensitive layers that benefit from bbox
+# filtering. Light reference layers (datacenters, military_bases,
+# power_plants, satellites, weather, news, etc.) are intentionally NOT
+# in these sets — they ship world-scale even when bounds are supplied so
+# panning never reveals an "empty world" of static infrastructure.
+#
+# When the caller does NOT pass s/w/n/e, none of this runs and the response
+# is byte-for-byte identical to the pre-#288 behavior.
+_FAST_BBOX_HEAVY_KEYS: tuple[str, ...] = (
+    "commercial_flights",
+    "military_flights",
+    "private_flights",
+    "private_jets",
+    "tracked_flights",
+    "ships",
+    "cctv",
+    "uavs",
+    "liveuamap",
+    "gps_jamming",
+    "sigint",
+    "trains",
+)
+_SLOW_BBOX_HEAVY_KEYS: tuple[str, ...] = (
+    "gdelt",
+    "firms_fires",
+    "kiwisdr",
+    "scanners",
+    "psk_reporter",
+)
+
+
+def _has_full_bbox(s, w, n, e) -> bool:
+    return None not in (s, w, n, e)
+
+
+def _bbox_etag_suffix(s, w, n, e) -> str:
+    """Quantize bbox to 1° before mixing into the ETag.
+
+    The 20% padding inside _bbox_filter already absorbs sub-degree pans;
+    quantizing here means small mouse drags don't blow the ETag cache
+    on the client. Full-world bounds collapse to a single suffix.
+    """
+    if not _has_full_bbox(s, w, n, e):
+        return ""
+    try:
+        ss = math.floor(float(s))
+        ww = math.floor(float(w))
+        nn = math.ceil(float(n))
+        ee = math.ceil(float(e))
+    except (TypeError, ValueError):
+        return ""
+    # If the requested window covers basically the whole world, treat it as
+    # "no bbox" for caching purposes so world-zoomed clients all hit the
+    # same ETag and benefit from the existing 304 path.
+    lat_span, lng_span = _bbox_spans(s, w, n, e)
+    if lng_span >= 300 or lat_span >= 120:
+        return ""
+    return f"|bbox={ss},{ww},{nn},{ee}"
+
+
+def _apply_bbox_to_payload(payload: dict, heavy_keys: tuple[str, ...],
+                            s: float, w: float, n: float, e: float) -> dict:
+    """In-place filter the heavy-key collections in *payload* to a viewport.
+
+    Items without lat/lng are passed through (so e.g. summary blobs aren't
+    accidentally dropped). The existing _bbox_filter helper applies a 20%
+    pad and handles antimeridian crossings.
+    """
+    lat_span, lng_span = _bbox_spans(s, w, n, e)
+    # World-scale request → skip filtering entirely. Spares the CPU and
+    # guarantees the response matches the no-params shape.
+    if lng_span >= 300 or lat_span >= 120:
+        return payload
+    for key in heavy_keys:
+        items = payload.get(key)
+        if not isinstance(items, list) or not items:
+            continue
+        payload[key] = _bbox_filter(items, s, w, n, e)
+    return payload
+
+
+def _json_safe(value):
+    if isinstance(value, float):
+        return value if math.isfinite(value) else None
+    if isinstance(value, dict):
+        return {k: _json_safe(v) for k, v in list(value.items())}
+    if isinstance(value, list):
+        return [_json_safe(v) for v in list(value)]
+    if isinstance(value, tuple):
+        return [_json_safe(v) for v in list(value)]
+    return value
+
+
+def _sanitize_payload(value):
+    if isinstance(value, float):
+        return value if math.isfinite(value) else None
+    if isinstance(value, dict):
+        return {k: _sanitize_payload(v) for k, v in list(value.items())}
+    if isinstance(value, (list, tuple)):
+        return list(value)
+    return value
+
+
+def _live_data_json_bytes(payload: dict) -> bytes:
+    """Serialize dashboard payloads with the same defensive orjson options everywhere."""
+    return orjson.dumps(
+        _sanitize_payload(payload),
+        default=str,
+        option=orjson.OPT_NON_STR_KEYS,
+    )
+
+
+def _bbox_filter(items: list, s: float, w: float, n: float, e: float,
+                 lat_key: str = "lat", lng_key: str = "lng") -> list:
+    pad_lat = (n - s) * 0.2
+    pad_lng = (e - w) * 0.2 if e > w else ((e + 360 - w) * 0.2)
+    s2, n2 = s - pad_lat, n + pad_lat
+    w2, e2 = w - pad_lng, e + pad_lng
+    crosses_antimeridian = w2 > e2
+    out = []
+    for item in items:
+        lat = item.get(lat_key)
+        lng = item.get(lng_key)
+        if lat is None or lng is None:
+            out.append(item)
+            continue
+        if not (s2 <= lat <= n2):
+            continue
+        if crosses_antimeridian:
+            if lng >= w2 or lng <= e2:
+                out.append(item)
+        else:
+            if w2 <= lng <= e2:
+                out.append(item)
+    return out
+
+
+def _bbox_filter_geojson_points(items: list, s: float, w: float, n: float, e: float) -> list:
+    pad_lat = (n - s) * 0.2
+    pad_lng = (e - w) * 0.2 if e > w else ((e + 360 - w) * 0.2)
+    s2, n2 = s - pad_lat, n + pad_lat
+    w2, e2 = w - pad_lng, e + pad_lng
+    crosses_antimeridian = w2 > e2
+    out = []
+    for item in items:
+        geometry = item.get("geometry") if isinstance(item, dict) else None
+        coords = geometry.get("coordinates") if isinstance(geometry, dict) else None
+        if not isinstance(coords, (list, tuple)) or len(coords) < 2:
+            out.append(item)
+            continue
+        lng, lat = coords[0], coords[1]
+        if lat is None or lng is None:
+            out.append(item)
+            continue
+        if not (s2 <= lat <= n2):
+            continue
+        if crosses_antimeridian:
+            if lng >= w2 or lng <= e2:
+                out.append(item)
+        else:
+            if w2 <= lng <= e2:
+                out.append(item)
+    return out
+
+
+def _bbox_spans(s, w, n, e) -> tuple:
+    if None in (s, w, n, e):
+        return 180.0, 360.0
+    lat_span = max(0.0, float(n) - float(s))
+    lng_span = float(e) - float(w)
+    if lng_span < 0:
+        lng_span += 360.0
+    if lng_span == 0 and w == -180 and e == 180:
+        lng_span = 360.0
+    return lat_span, max(0.0, lng_span)
+
+
+def _cap_startup_items(items: list | None, max_items: int) -> list:
+    if not items:
+        return []
+    if len(items) <= max_items:
+        return items
+    return items[:max_items]
+
+
+def _cap_fast_startup_payload(payload: dict) -> dict:
+    capped = dict(payload)
+    capped["commercial_flights"] = _cap_startup_items(capped.get("commercial_flights"), 800)
+    capped["private_flights"] = _cap_startup_items(capped.get("private_flights"), 300)
+    capped["private_jets"] = _cap_startup_items(capped.get("private_jets"), 150)
+    capped["ships"] = _cap_startup_items(capped.get("ships"), 1500)
+    capped["cctv"] = []
+    capped["sigint"] = _cap_startup_items(capped.get("sigint"), 500)
+    capped["trains"] = _cap_startup_items(capped.get("trains"), 100)
+    capped["startup_payload"] = True
+    return capped
+
+
+def _cap_fast_dashboard_payload(payload: dict) -> dict:
+    return payload
+
+
+def _world_and_continental_scale(has_bbox: bool, s, w, n, e) -> tuple:
+    lat_span, lng_span = _bbox_spans(s, w, n, e)
+    world_scale = (not has_bbox) or lng_span >= 300 or lat_span >= 120
+    continental_scale = has_bbox and not world_scale and (lng_span >= 120 or lat_span >= 55)
+    return world_scale, continental_scale
+
+
+def _filter_sigint_by_layers(items: list, active_layers: dict) -> list:
+    allow_aprs = bool(active_layers.get("sigint_aprs", True))
+    allow_mesh = bool(active_layers.get("sigint_meshtastic", True))
+    if allow_aprs and allow_mesh:
+        return items
+    allowed_sources: set = {"js8call"}
+    if allow_aprs:
+        allowed_sources.add("aprs")
+    if allow_mesh:
+        allowed_sources.update({"meshtastic", "meshtastic-map"})
+    return [item for item in items if str(item.get("source") or "").lower() in allowed_sources]
+
+
+def _sigint_totals_for_items(items: list) -> dict:
+    totals = {"total": len(items), "meshtastic": 0, "meshtastic_live": 0, "meshtastic_map": 0,
+              "aprs": 0, "js8call": 0}
+    for item in items:
+        source = str(item.get("source") or "").lower()
+        if source == "meshtastic":
+            totals["meshtastic"] += 1
+            if bool(item.get("from_api")):
+                totals["meshtastic_map"] += 1
+            else:
+                totals["meshtastic_live"] += 1
+        elif source == "aprs":
+            totals["aprs"] += 1
+        elif source == "js8call":
+            totals["js8call"] += 1
+    return totals
+
+
+@router.get("/api/refresh", dependencies=[Depends(require_admin)])
+@limiter.limit("2/minute")
+async def force_refresh(request: Request):
+    from services.schemas import RefreshResponse
+    if not _refresh_lock.acquire(blocking=False):
+        return {"status": "refresh already in progress"}
+
+    def _do_refresh():
+        try:
+            update_all_data()
+        finally:
+            _refresh_lock.release()
+
+    t = threading.Thread(target=_do_refresh)
+    t.start()
+    return {"status": "refreshing in background"}
+
+
+@router.post("/api/ais/feed", dependencies=[Depends(require_local_operator)])
+@limiter.limit("60/minute")
+async def ais_feed(request: Request):
+    """Accept AIS-catcher HTTP JSON feed (POST decoded AIS messages)."""
+    from services.ais_stream import ingest_ais_catcher
+    try:
+        body = await request.json()
+    except Exception:
+        return JSONResponse(status_code=422, content={"ok": False, "detail": "invalid JSON body"})
+    msgs = body.get("msgs", [])
+    if not msgs:
+        return {"status": "ok", "ingested": 0}
+    count = ingest_ais_catcher(msgs)
+    return {"status": "ok", "ingested": count}
+
+
+@router.get("/api/trail/flight/{icao24}")
+@limiter.limit("120/minute")
+async def get_selected_flight_trail(icao24: str, request: Request):  # noqa: ARG001
+    from services.fetchers.flights import get_flight_trail
+    return {"id": icao24, "trail": get_flight_trail(icao24)}
+
+
+@router.get("/api/trail/ship/{mmsi}")
+@limiter.limit("120/minute")
+async def get_selected_ship_trail(mmsi: int, request: Request):  # noqa: ARG001
+    from services.ais_stream import get_vessel_trail
+    return {"id": mmsi, "trail": get_vessel_trail(mmsi)}
+
+
+@router.post("/api/viewport")
+@limiter.limit("60/minute")
+async def update_viewport(vp: ViewportUpdate, request: Request):  # noqa: ARG001
+    """Receive frontend map bounds. AIS stream stays global so open-ocean
+    vessels are never dropped — the frontend worker handles viewport culling."""
+    return {"status": "ok"}
+
+
+@router.get("/api/liveuamap/scraper-status", dependencies=[Depends(require_local_operator)])
+async def api_liveuamap_scraper_status():
+    """Whether LiveUAMap Playwright may run (Windows needs UI opt-in unless env forces)."""
+    from services.liveuamap_settings import liveuamap_scraper_status
+
+    return liveuamap_scraper_status()
+
+
+@router.post("/api/liveuamap/scraper-opt-in", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_liveuamap_scraper_opt_in(body: LiveUamapOptInUpdate, request: Request):
+    """Persist operator consent for LiveUAMap scraper (#348)."""
+    from services.liveuamap_settings import liveuamap_scraper_status, set_liveuamap_ui_opt_in
+
+    set_liveuamap_ui_opt_in(body.opted_in)
+    if body.opted_in:
+        from services.fetchers._store import is_any_active
+
+        if is_any_active("global_incidents"):
+            threading.Thread(target=_run_liveuamap_refresh, daemon=True).start()
+    return liveuamap_scraper_status()
+
+
+def _run_liveuamap_refresh() -> None:
+    try:
+        from services.fetchers.geo import update_liveuamap
+
+        update_liveuamap()
+    except Exception as e:
+        logger.warning("LiveUAMap refresh after opt-in failed: %s", e)
+
+
+@router.get("/api/prediction-markets/status", dependencies=[Depends(require_local_operator)])
+async def api_prediction_markets_status():
+    """Whether Polymarket/Kalshi fetches and news market correlation are enabled."""
+    from services.prediction_markets_settings import prediction_markets_status
+
+    return prediction_markets_status()
+
+
+@router.post("/api/prediction-markets/opt-in", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_prediction_markets_opt_in(body: PredictionMarketsOptInUpdate, request: Request):
+    """Enable or disable prediction market fetches + intercept story correlation."""
+    from services.config import get_settings
+    from services.prediction_markets_settings import (
+        prediction_markets_status,
+        set_prediction_markets_ui_opt_in,
+    )
+    from routers.ai_intel import _write_env_value
+
+    set_prediction_markets_ui_opt_in(body.opted_in)
+    _write_env_value("PREDICTION_MARKETS_ENABLED", "true" if body.opted_in else "false")
+    os.environ["PREDICTION_MARKETS_ENABLED"] = "true" if body.opted_in else "false"
+    get_settings.cache_clear()
+
+    if body.opted_in:
+        threading.Thread(target=_run_prediction_markets_refresh, daemon=True).start()
+    else:
+        threading.Thread(target=_run_prediction_markets_disable, daemon=True).start()
+
+    return prediction_markets_status()
+
+
+def _run_prediction_markets_refresh() -> None:
+    try:
+        from services.fetchers.prediction_markets import fetch_prediction_markets
+        from services.fetchers.news import fetch_news
+
+        fetch_prediction_markets()
+        fetch_news()
+    except Exception as e:
+        logger.warning("Prediction markets refresh after opt-in failed: %s", e)
+
+
+def _run_prediction_markets_disable() -> None:
+    try:
+        from services.fetchers._store import _data_lock, _mark_fresh, latest_data
+        from services.fetchers.news import fetch_news
+
+        with _data_lock:
+            latest_data["prediction_markets"] = []
+            latest_data["trending_markets"] = []
+        _mark_fresh("prediction_markets")
+        fetch_news()
+    except Exception as e:
+        logger.warning("Prediction markets disable cleanup failed: %s", e)
+
+
+@router.post("/api/layers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def update_layers(update: LayerUpdate, request: Request):
+    """Receive frontend layer toggle state. Starts/stops streams accordingly."""
+    from services.fetchers._store import active_layers, bump_active_layers_version, is_any_active
+    old_ships = is_any_active("ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts")
+    old_mesh = is_any_active("sigint_meshtastic")
+    old_aprs = is_any_active("sigint_aprs")
+    old_viirs = is_any_active("viirs_nightlights")
+    old_datacenters = is_any_active("datacenters")
+    old_fishing = is_any_active("fishing_activity")
+    changed = False
+    for key, value in update.layers.items():
+        if key in active_layers:
+            if active_layers[key] != value:
+                changed = True
+            active_layers[key] = value
+    if changed:
+        bump_active_layers_version()
+    new_ships = is_any_active("ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts")
+    new_mesh = is_any_active("sigint_meshtastic")
+    new_aprs = is_any_active("sigint_aprs")
+    new_viirs = is_any_active("viirs_nightlights")
+    new_datacenters = is_any_active("datacenters")
+    new_fishing = is_any_active("fishing_activity")
+    if old_ships and not new_ships:
+        from services.ais_stream import stop_ais_stream
+        stop_ais_stream()
+        logger.info("AIS stream stopped (all ship layers disabled)")
+    elif not old_ships and new_ships:
+        from services.ais_stream import start_ais_stream
+        start_ais_stream()
+        logger.info("AIS stream started (ship layer enabled)")
+    from services.sigint_bridge import sigint_grid
+    if old_mesh and not new_mesh:
+        try:
+            from services.meshtastic_mqtt_settings import mqtt_bridge_enabled
+            keep_chat_running = mqtt_bridge_enabled()
+        except Exception:
+            keep_chat_running = False
+        if keep_chat_running:
+            logger.info("Meshtastic map layer disabled; MQTT bridge kept running for MeshChat")
+        else:
+            sigint_grid.mesh.stop()
+            logger.info("Meshtastic MQTT bridge stopped (layer disabled)")
+    elif not old_mesh and new_mesh:
+        try:
+            from services.meshtastic_mqtt_settings import mqtt_bridge_enabled
+            mqtt_enabled = mqtt_bridge_enabled()
+        except Exception:
+            mqtt_enabled = False
+        if mqtt_enabled:
+            sigint_grid.mesh.start()
+            logger.info("Meshtastic MQTT bridge started (layer enabled)")
+        else:
+            logger.info(
+                "Meshtastic layer enabled; MQTT bridge remains disabled "
+                "(set MESH_MQTT_ENABLED=true to participate in the public broker)"
+            )
+    if old_aprs and not new_aprs:
+        sigint_grid.aprs.stop()
+        logger.info("APRS bridge stopped (layer disabled)")
+    elif not old_aprs and new_aprs:
+        sigint_grid.aprs.start()
+        logger.info("APRS bridge started (layer enabled)")
+    if not old_viirs and new_viirs:
+        _queue_viirs_change_refresh()
+        logger.info("VIIRS change refresh queued (layer enabled)")
+    if not old_datacenters and new_datacenters:
+        from services.fetchers.infrastructure import fetch_datacenters
+
+        fetch_datacenters()
+        logger.info("Datacenters loaded (layer enabled)")
+    if not old_fishing and new_fishing:
+        from services.fetchers.geo import fetch_fishing_activity
+
+        fetch_fishing_activity()
+        logger.info("Fishing activity refresh queued (layer enabled)")
+    return {"status": "ok"}
+
+
+@router.get("/api/live-data")
+@limiter.limit("120/minute")
+async def live_data(request: Request):
+    etag = _current_etag(prefix="live|full|")
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    from services.fetchers._store import get_latest_data_deepcopy_snapshot
+
+    payload = get_latest_data_deepcopy_snapshot()
+    return Response(
+        content=_live_data_json_bytes(payload),
+        media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"},
+    )
+
+
+@router.get("/api/bootstrap/critical")
+@limiter.limit("180/minute")
+async def bootstrap_critical(request: Request):
+    """Cached first-paint payload for the dashboard.
+
+    This endpoint is intentionally memory-only: no upstream calls, no refresh,
+    and a bounded response. It exists so the map and threat feed can paint
+    before slower panels and background enrichers finish warming up.
+    """
+    etag = _current_etag(prefix="bootstrap|critical|")
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    from services.fetchers._store import (
+        active_layers,
+        get_latest_data_subset_refs,
+        get_source_timestamps_snapshot,
+    )
+
+    d = get_latest_data_subset_refs(
+        "last_updated", "commercial_flights", "military_flights", "private_flights",
+        "private_jets", "tracked_flights", "ships", "uavs", "liveuamap", "gps_jamming",
+        "satellites", "satellite_source", "satellite_analysis", "sigint", "sigint_totals",
+        "trains", "news", "gdelt", "airports", "threat_level", "trending_markets",
+        "correlations", "fimi", "crowdthreat",
+    )
+    freshness = get_source_timestamps_snapshot()
+    ships_enabled = any(active_layers.get(key, True) for key in (
+        "ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts"))
+    sigint_items = _filter_sigint_by_layers(d.get("sigint") or [], active_layers)
+    payload = {
+        "last_updated": d.get("last_updated"),
+        "commercial_flights": _cap_startup_items(
+            (d.get("commercial_flights") or []) if active_layers.get("flights", True) else [],
+            800,
+        ),
+        "military_flights": _cap_startup_items(
+            (d.get("military_flights") or []) if active_layers.get("military", True) else [],
+            300,
+        ),
+        "private_flights": _cap_startup_items(
+            (d.get("private_flights") or []) if active_layers.get("private", True) else [],
+            300,
+        ),
+        "private_jets": _cap_startup_items(
+            (d.get("private_jets") or []) if active_layers.get("jets", True) else [],
+            150,
+        ),
+        "tracked_flights": _cap_startup_items(
+            (d.get("tracked_flights") or []) if active_layers.get("tracked", True) else [],
+            250,
+        ),
+        "ships": _cap_startup_items((d.get("ships") or []) if ships_enabled else [], 1500),
+        "uavs": _cap_startup_items((d.get("uavs") or []) if active_layers.get("military", True) else [], 100),
+        "liveuamap": _cap_startup_items(
+            (d.get("liveuamap") or []) if active_layers.get("global_incidents", True) else [],
+            300,
+        ),
+        "gps_jamming": _cap_startup_items(
+            (d.get("gps_jamming") or []) if active_layers.get("gps_jamming", True) else [],
+            200,
+        ),
+        "satellites": _cap_startup_items(
+            (d.get("satellites") or []) if active_layers.get("satellites", True) else [],
+            250,
+        ),
+        "satellite_source": d.get("satellite_source", "none"),
+        "satellite_analysis": (d.get("satellite_analysis") or {}) if active_layers.get("satellites", True) else {},
+        "sigint": _cap_startup_items(
+            sigint_items if (active_layers.get("sigint_meshtastic", True) or active_layers.get("sigint_aprs", True)) else [],
+            500,
+        ),
+        "sigint_totals": _sigint_totals_for_items(sigint_items),
+        "trains": _cap_startup_items((d.get("trains") or []) if active_layers.get("trains", True) else [], 100),
+        "news": _cap_startup_items(d.get("news") or [], 30),
+        "gdelt": _cap_startup_items((d.get("gdelt") or []) if active_layers.get("global_incidents", True) else [], 300),
+        "airports": _cap_startup_items(d.get("airports") or [], 500),
+        "threat_level": d.get("threat_level"),
+        "trending_markets": _cap_startup_items(d.get("trending_markets") or [], 10),
+        "correlations": _cap_startup_items(
+            (d.get("correlations") or []) if active_layers.get("correlations", True) else [],
+            50,
+        ),
+        "fimi": d.get("fimi"),
+        "crowdthreat": _cap_startup_items(
+            (d.get("crowdthreat") or []) if active_layers.get("crowdthreat", True) else [],
+            150,
+        ),
+        "freshness": freshness,
+        "bootstrap_ready": True,
+        "bootstrap_payload": True,
+    }
+    return Response(
+        content=_live_data_json_bytes(payload),
+        media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"},
+    )
+
+
+@router.get("/api/live-data/fast")
+@limiter.limit("120/minute")
+async def live_data_fast(
+    request: Request,
+    s: float = Query(None, description="South bound — when all four bounds are supplied, heavy/dense layers (vessels, aircraft, sigint, CCTV, …) are filtered to this viewport with 20% padding. Static reference layers (satellites, etc.) always ship world-scale.", ge=-90, le=90),
+    w: float = Query(None, description="West bound (see s)", ge=-180, le=180),
+    n: float = Query(None, description="North bound (see s)", ge=-90, le=90),
+    e: float = Query(None, description="East bound (see s)", ge=-180, le=180),
+    initial: bool = Query(False, description="Return a capped startup payload for first paint"),
+):
+    bbox_suffix = _bbox_etag_suffix(s, w, n, e)
+    etag = _current_etag(prefix=("fast|initial|" if initial else "fast|full|") + bbox_suffix.lstrip("|") + ("|" if bbox_suffix else ""))
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    from services.fetchers._store import (active_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot)
+    d = get_latest_data_subset_refs(
+        "last_updated", "commercial_flights", "military_flights", "private_flights",
+        "private_jets", "tracked_flights", "ships", "cctv", "uavs", "liveuamap",
+        "gps_jamming", "satellites", "satellite_source", "satellite_analysis",
+        "sigint", "sigint_totals", "trains",
+    )
+    freshness = get_source_timestamps_snapshot()
+    ships_enabled = any(active_layers.get(key, True) for key in (
+        "ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts"))
+    cctv_total = len(d.get("cctv") or [])
+    sigint_items = _filter_sigint_by_layers(d.get("sigint") or [], active_layers)
+    sigint_totals = _sigint_totals_for_items(sigint_items)
+    payload = {
+        "commercial_flights": (d.get("commercial_flights") or []) if active_layers.get("flights", True) else [],
+        "military_flights": (d.get("military_flights") or []) if active_layers.get("military", True) else [],
+        "private_flights": (d.get("private_flights") or []) if active_layers.get("private", True) else [],
+        "private_jets": (d.get("private_jets") or []) if active_layers.get("jets", True) else [],
+        "tracked_flights": (d.get("tracked_flights") or []) if active_layers.get("tracked", True) else [],
+        "ships": (d.get("ships") or []) if ships_enabled else [],
+        "cctv": (d.get("cctv") or []) if active_layers.get("cctv", True) else [],
+        "uavs": (d.get("uavs") or []) if active_layers.get("military", True) else [],
+        "liveuamap": (d.get("liveuamap") or []) if active_layers.get("global_incidents", True) else [],
+        "gps_jamming": (d.get("gps_jamming") or []) if active_layers.get("gps_jamming", True) else [],
+        "satellites": (d.get("satellites") or []) if active_layers.get("satellites", True) else [],
+        "satellite_source": d.get("satellite_source", "none"),
+        "satellite_analysis": (d.get("satellite_analysis") or {}) if active_layers.get("satellites", True) else {},
+        "sigint": sigint_items if (active_layers.get("sigint_meshtastic", True) or active_layers.get("sigint_aprs", True)) else [],
+        "sigint_totals": sigint_totals,
+        "cctv_total": cctv_total,
+        "trains": (d.get("trains") or []) if active_layers.get("trains", True) else [],
+        "freshness": freshness,
+    }
+    if initial:
+        payload = _cap_fast_startup_payload(payload)
+    else:
+        payload = _cap_fast_dashboard_payload(payload)
+    # Issue #288: bbox filter heavy/dense layers only when all four bounds
+    # are supplied. Without bounds, behaviour is byte-for-byte identical
+    # to the pre-#288 implementation.
+    if _has_full_bbox(s, w, n, e):
+        payload = _apply_bbox_to_payload(payload, _FAST_BBOX_HEAVY_KEYS, s, w, n, e)
+    return Response(
+        content=_live_data_json_bytes(payload),
+        media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"},
+    )
+
+
+@router.get("/api/live-data/slow")
+@limiter.limit("60/minute")
+async def live_data_slow(
+    request: Request,
+    s: float = Query(None, description="South bound — when all four bounds are supplied, heavy/dense layers (gdelt, firms_fires, kiwisdr, scanners, psk_reporter) are filtered to this viewport with 20% padding. Static reference layers (datacenters, military bases, power plants, weather, news, …) always ship world-scale.", ge=-90, le=90),
+    w: float = Query(None, description="West bound (see s)", ge=-180, le=180),
+    n: float = Query(None, description="North bound (see s)", ge=-90, le=90),
+    e: float = Query(None, description="East bound (see s)", ge=-180, le=180),
+):
+    bbox_suffix = _bbox_etag_suffix(s, w, n, e)
+    etag = _current_etag(prefix="slow|full|" + bbox_suffix.lstrip("|") + ("|" if bbox_suffix else ""))
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    from services.fetchers._store import (active_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot)
+    d = get_latest_data_subset_refs(
+        "last_updated", "news", "stocks", "financial_source", "oil", "weather", "traffic",
+        "earthquakes", "frontlines", "gdelt", "airports", "kiwisdr", "satnogs_stations",
+        "satnogs_observations", "tinygs_satellites", "space_weather", "internet_outages",
+        "firms_fires", "datacenters", "military_bases", "power_plants", "viirs_change_nodes",
+        "scanners", "weather_alerts", "ukraine_alerts", "air_quality", "volcanoes",
+        "fishing_activity", "psk_reporter", "correlations", "uap_sightings", "wastewater",
+        "crowdthreat", "threat_level", "trending_markets", "road_corridor_trends",
+        "malware_threats", "cyber_threats", "scm_suppliers", "telegram_osint", "gt_risk",
+    )
+    freshness = get_source_timestamps_snapshot()
+    payload = {
+        "last_updated": d.get("last_updated"),
+        "threat_level": d.get("threat_level"),
+        "trending_markets": d.get("trending_markets", []),
+        "news": d.get("news", []),
+        "stocks": d.get("stocks", {}),
+        "financial_source": d.get("financial_source", ""),
+        "oil": d.get("oil", {}),
+        "weather": d.get("weather"),
+        "traffic": d.get("traffic", []),
+        "earthquakes": (d.get("earthquakes") or []) if active_layers.get("earthquakes", True) else [],
+        "frontlines": d.get("frontlines") if active_layers.get("ukraine_frontline", True) else None,
+        "gdelt": (d.get("gdelt") or []) if active_layers.get("global_incidents", True) else [],
+        "airports": d.get("airports") or [],
+        "kiwisdr": (d.get("kiwisdr") or []) if active_layers.get("kiwisdr", True) else [],
+        "satnogs_stations": (d.get("satnogs_stations") or []) if active_layers.get("satnogs", True) else [],
+        "satnogs_total": len(d.get("satnogs_stations") or []),
+        "satnogs_observations": (d.get("satnogs_observations") or []) if active_layers.get("satnogs", True) else [],
+        "tinygs_satellites": (d.get("tinygs_satellites") or []) if active_layers.get("tinygs", True) else [],
+        "tinygs_total": len(d.get("tinygs_satellites") or []),
+        "psk_reporter": (d.get("psk_reporter") or []) if active_layers.get("psk_reporter", True) else [],
+        "space_weather": d.get("space_weather"),
+        "internet_outages": (d.get("internet_outages") or []) if active_layers.get("internet_outages", True) else [],
+        "firms_fires": (d.get("firms_fires") or []) if active_layers.get("firms", True) else [],
+        "datacenters": (d.get("datacenters") or []) if active_layers.get("datacenters", True) else [],
+        "military_bases": (d.get("military_bases") or []) if active_layers.get("military_bases", True) else [],
+        "power_plants": (d.get("power_plants") or []) if active_layers.get("power_plants", True) else [],
+        "viirs_change_nodes": (d.get("viirs_change_nodes") or []) if active_layers.get("viirs_nightlights", True) else [],
+        "scanners": (d.get("scanners") or []) if active_layers.get("scanners", True) else [],
+        "weather_alerts": d.get("weather_alerts", []) if active_layers.get("weather_alerts", True) else [],
+        "ukraine_alerts": d.get("ukraine_alerts", []) if active_layers.get("ukraine_alerts", True) else [],
+        "air_quality": (d.get("air_quality") or []) if active_layers.get("air_quality", True) else [],
+        "volcanoes": (d.get("volcanoes") or []) if active_layers.get("volcanoes", True) else [],
+        "fishing_activity": (d.get("fishing_activity") or []) if active_layers.get("fishing_activity", True) else [],
+        "correlations": (d.get("correlations") or []) if active_layers.get("correlations", True) else [],
+        "uap_sightings": (d.get("uap_sightings") or []) if active_layers.get("uap_sightings", True) else [],
+        "wastewater": (d.get("wastewater") or []) if active_layers.get("wastewater", True) else [],
+        "crowdthreat": (d.get("crowdthreat") or []) if active_layers.get("crowdthreat", True) else [],
+        "road_corridor_trends": (
+            d.get("road_corridor_trends") or {"updated_at": None, "corridors": []}
+        )
+        if active_layers.get("road_corridor_trends", False)
+        else {"updated_at": None, "corridors": []},
+        "malware_threats": (
+            d.get("malware_threats") or {"threats": [], "total": 0}
+        )
+        if active_layers.get("malware_c2", False)
+        else {"threats": [], "total": 0},
+        "cyber_threats": (
+            d.get("cyber_threats") or {"threats": [], "stats": {}}
+        )
+        if active_layers.get("cyber_threats", False)
+        else {"threats": [], "stats": {}},
+        "scm_suppliers": (
+            d.get("scm_suppliers") or {"suppliers": [], "total": 0, "critical_count": 0}
+        )
+        if active_layers.get("scm_suppliers", False)
+        else {"suppliers": [], "total": 0, "critical_count": 0},
+        "telegram_osint": (
+            d.get("telegram_osint") or {"posts": [], "total": 0, "geolocated": 0}
+        )
+        if active_layers.get("telegram_osint", True)
+        else {"posts": [], "total": 0, "geolocated": 0},
+        "gt_risk": (
+            d.get("gt_risk")
+            or {"enabled": False, "heatmap": {"type": "FeatureCollection", "features": []}, "clusters": []}
+        )
+        if active_layers.get("gt_risk", False)
+        else {"enabled": False, "heatmap": {"type": "FeatureCollection", "features": []}, "clusters": []},
+        "freshness": freshness,
+    }
+    # Issue #288: bbox filter heavy/dense layers only when all four bounds
+    # are supplied. Static reference layers (datacenters, military bases,
+    # power_plants, etc.) deliberately stay world-scale so panning never
+    # hides the infrastructure overlay the operator already has on screen.
+    if _has_full_bbox(s, w, n, e):
+        payload = _apply_bbox_to_payload(payload, _SLOW_BBOX_HEAVY_KEYS, s, w, n, e)
+    return Response(
+        content=_live_data_json_bytes(payload),
+        media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"},
+    )
+
+
+# ── Satellite Overflight Counting ───────────────────────────────────────────
+# Counts unique satellites whose ground track entered a bounding box over 24h.
+# Uses cached TLEs + SGP4 propagation — no extra network requests.
+
+class OverflightRequest(BaseModel):
+    s: float
+    w: float
+    n: float
+    e: float
+    hours: int = 24
+
+
+# Issue #202: compute_overflights() is O(catalog_size × timesteps), where
+# timesteps grows linearly with `hours`. An unbounded `hours` value is a
+# trivial CPU-exhaustion vector. We clamp silently rather than raising 422 —
+# the response shape is unchanged, callers asking for too many hours just
+# get a shorter window, which is friendlier than a hostile error.
+#
+# Override via OVERFLIGHTS_MAX_HOURS env var if you legitimately need a
+# longer window (e.g. a planning use case that wants a full week).
+def _overflight_max_hours() -> int:
+    import os as _os
+    try:
+        raw = int(str(_os.environ.get("OVERFLIGHTS_MAX_HOURS", "72")).strip())
+    except (TypeError, ValueError):
+        raw = 72
+    return max(1, raw)
+
+
+@router.post("/api/satellites/overflights")
+@limiter.limit("10/minute")
+async def satellite_overflights(request: Request, body: OverflightRequest):
+    from services.fetchers.satellites import compute_overflights, _sat_gp_cache
+    gp_data = _sat_gp_cache.get("data")
+    if not gp_data:
+        return JSONResponse({"total": 0, "by_mission": {}, "satellites": [], "error": "No GP data cached yet"})
+    bbox = {"s": body.s, "w": body.w, "n": body.n, "e": body.e}
+
+    # Silent clamp — see comment on _overflight_max_hours().
+    requested_hours = max(1, int(body.hours or 0))
+    effective_hours = min(requested_hours, _overflight_max_hours())
+
+    result = compute_overflights(gp_data, bbox, hours=effective_hours)
+    # If we clamped, surface the effective window in the response so the
+    # caller can detect it if they care, without it being an error.
+    if isinstance(result, dict) and effective_hours != requested_hours:
+        result.setdefault("requested_hours", requested_hours)
+        result.setdefault("effective_hours", effective_hours)
+    return JSONResponse(result)

backend/routers/entity_graph.py

@@ -0,0 +1,30 @@
+"""Entity graph expansion (intel layer)."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+
+from auth import require_local_operator
+from limiter import limiter
+from services.osint_intel.resolve import resolve_entity
+
+router = APIRouter()
+
+
+@router.get("/api/entity/expand")
+@limiter.limit("30/minute")
+async def entity_expand(
+    request: Request,
+    _: None = Depends(require_local_operator),
+    type: str = Query(..., min_length=3, max_length=32),
+    id: str = Query(..., min_length=2, max_length=200),
+    registration: str | None = Query(default=None, max_length=32),
+    model: str | None = Query(default=None, max_length=64),
+    icao24: str | None = Query(default=None, max_length=16),
+) -> dict:
+    props = {"label": id, "registration": registration, "model": model, "icao24": icao24}
+    try:
+        return resolve_entity(type, id, props)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    except Exception as exc:
+        raise HTTPException(status_code=502, detail="Intelligence layer unavailable") from exc

backend/routers/health.py

@@ -0,0 +1,130 @@
+import time as _time_mod
+from fastapi import APIRouter, Request, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin
+from services.data_fetcher import get_latest_data
+from services.schemas import HealthResponse
+import os
+
+APP_VERSION = os.environ.get("_HEALTH_APP_VERSION", "0.9.82")
+
+router = APIRouter()
+
+
+def _get_app_version() -> str:
+    # Import lazily to avoid circular import; main sets APP_VERSION before including routers
+    try:
+        import main as _main
+        return _main.APP_VERSION
+    except Exception:
+        return APP_VERSION
+
+
+_start_time_ref: dict = {"value": None}
+
+
+def _get_start_time() -> float:
+    if _start_time_ref["value"] is None:
+        try:
+            import main as _main
+            _start_time_ref["value"] = _main._start_time
+        except Exception:
+            _start_time_ref["value"] = _time_mod.time()
+    return _start_time_ref["value"]
+
+
+@router.get("/api/health", response_model=HealthResponse)
+@limiter.limit("30/minute")
+async def health_check(request: Request):
+    from services.fetchers._store import get_source_timestamps_snapshot
+    from services.slo import compute_all_statuses, summarise_statuses
+
+    d = get_latest_data()
+    last = d.get("last_updated")
+    timestamps = get_source_timestamps_snapshot()
+    slo_statuses = compute_all_statuses(d, timestamps)
+    slo_summary = summarise_statuses(slo_statuses)
+    # Top-level status reflects worst SLO result — "degraded" if any
+    # yellow, "error" if any red, "ok" otherwise. This is the single
+    # field an external probe / pager can watch.
+    top_status = "ok"
+    if slo_summary.get("red", 0) > 0:
+        top_status = "error"
+    elif slo_summary.get("yellow", 0) > 0:
+        top_status = "degraded"
+
+    # Issue #258: surface AIS proxy degraded TLS state so operators can see
+    # when the SPKI-pinned fallback is in effect. The data plane keeps
+    # flowing (this is by design — see ais_proxy.js comments) but observers
+    # who care about MITM-protection posture deserve a visible signal.
+    #
+    # Plus connectivity health (added 2026-05-23 when stream.aisstream.io
+    # went fully offline): ``connected`` tells the frontend whether ship
+    # data is actually flowing. When false, a banner explains that ships
+    # are unavailable due to an upstream outage — better than the user
+    # silently seeing an empty ocean and assuming we broke something.
+    ais_status: dict = {}
+    try:
+        from services.ais_stream import ais_proxy_status
+        ais_status = ais_proxy_status() or {}
+    except Exception:
+        ais_status = {}
+    if ais_status.get("degraded_tls") and top_status == "ok":
+        # Don't override a worse top-level status if SLOs already failed,
+        # but escalate ok -> degraded so the field surfaces in dashboards.
+        top_status = "degraded"
+    # AIS_API_KEY not configured is "feature off", not "system broken" —
+    # so we only escalate when the operator opted into AIS (key set) AND
+    # the stream is currently offline.
+    if (
+        os.environ.get("AIS_API_KEY")
+        and ais_status.get("connected") is False
+        and top_status == "ok"
+    ):
+        top_status = "degraded"
+
+    runtime: dict = {}
+    try:
+        from services.runtime_profile import get_runtime_profile
+        from analytics.settings import gt_analytics_status
+
+        runtime = {
+            **get_runtime_profile(),
+            "gt_analytics": gt_analytics_status(),
+        }
+    except Exception:
+        runtime = {}
+
+    return {
+        "status": top_status,
+        "version": _get_app_version(),
+        "last_updated": last,
+        "sources": {
+            "flights": len(d.get("commercial_flights", [])),
+            "military": len(d.get("military_flights", [])),
+            "ships": len(d.get("ships", [])),
+            "satellites": len(d.get("satellites", [])),
+            "earthquakes": len(d.get("earthquakes", [])),
+            "cctv": len(d.get("cctv", [])),
+            "news": len(d.get("news", [])),
+            "uavs": len(d.get("uavs", [])),
+            "firms_fires": len(d.get("firms_fires", [])),
+            "liveuamap": len(d.get("liveuamap", [])),
+            "gdelt": len(d.get("gdelt", [])),
+            "uap_sightings": len(d.get("uap_sightings", [])),
+        },
+        "freshness": timestamps,
+        "uptime_seconds": round(_time_mod.time() - _get_start_time()),
+        "slo": slo_statuses,
+        "slo_summary": slo_summary,
+        "ais_proxy": ais_status,
+        "runtime": runtime or None,
+    }
+
+
+@router.get("/api/debug-latest", dependencies=[Depends(require_admin)])
+@limiter.limit("30/minute")
+async def debug_latest_data(request: Request):
+    return list(get_latest_data().keys())

backend/routers/infonet.py

@@ -0,0 +1,598 @@
+"""Infonet economy / governance / gates / bootstrap HTTP surface.
+
+Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §2.1.
+
+Read endpoints return chain-derived state (computed by the
+``services.infonet`` adapters / pure functions). Write endpoints take
+a payload, validate it through the cutover-registered validators, and
+return a structured "would-emit" preview. Production wiring (signing
++ ``Infonet.append`` persistence) is a thin follow-on; the validation
+contract is locked here.
+
+Cross-cutting design rule: errors are diagnostic, not punitive. Each
+write endpoint returns ``{"ok": False, "reason": "..."}`` on
+validation failure with the exact field that failed. Frontend
+surfaces the reason in the UI.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any
+
+from fastapi import APIRouter, Body, Path
+
+# Triggers the chain cutover at module-load time so registered
+# validators are live for any subsequent route invocation.
+from services.infonet import _chain_cutover  # noqa: F401
+from services.infonet.adapters.gate_adapter import InfonetGateAdapter
+from services.infonet.adapters.oracle_adapter import InfonetOracleAdapter
+from services.infonet.adapters.reputation_adapter import InfonetReputationAdapter
+from services.infonet.bootstrap import compute_active_features
+from services.infonet.config import (
+    CONFIG,
+    IMMUTABLE_PRINCIPLES,
+)
+from services.infonet.governance import (
+    apply_petition_payload,
+    compute_petition_state,
+    compute_upgrade_state,
+)
+from services.infonet.governance.dsl_executor import InvalidPetition
+from services.infonet.partition import (
+    classify_event_type,
+    is_chain_stale,
+    should_mark_provisional,
+)
+from services.infonet.privacy import (
+    DEXScaffolding,
+    RingCTScaffolding,
+    ShieldedBalanceScaffolding,
+    StealthAddressScaffolding,
+)
+from services.infonet.schema import (
+    INFONET_ECONOMY_EVENT_TYPES,
+    validate_infonet_event_payload,
+)
+from services.infonet.time_validity import chain_majority_time
+
+logger = logging.getLogger("routers.infonet")
+
+router = APIRouter(prefix="/api/infonet", tags=["infonet"])
+
+
+# ─── Chain access helper ─────────────────────────────────────────────────
+# Every adapter takes a ``chain_provider`` callable. We pull the live
+# Infonet chain from mesh_hashchain. Tests can monkeypatch this.
+
+def _live_chain() -> list[dict[str, Any]]:
+    try:
+        from services.mesh.mesh_hashchain import infonet
+        events = getattr(infonet, "events", None)
+        if isinstance(events, list):
+            return list(events)
+        # Some implementations use a deque; convert to list.
+        if events is not None:
+            return list(events)
+    except Exception as exc:
+        logger.debug("infonet chain unavailable: %s", exc)
+    return []
+
+
+def _now() -> float:
+    cmt = chain_majority_time(_live_chain())
+    return cmt if cmt > 0 else float(time.time())
+
+
+# ─── Status ──────────────────────────────────────────────────────────────
+
+@router.get("/status")
+def infonet_status() -> dict[str, Any]:
+    """Top-level health snapshot for the InfonetTerminal HUD.
+
+    Returns ramp activation flags, partition staleness, privacy
+    primitive statuses, immutable principles, and counts of
+    chain-derived state (markets / petitions / gates / etc).
+    """
+    chain = _live_chain()
+    now = _now()
+    features = compute_active_features(chain)
+
+    # Privacy primitive statuses (truthful — most are NOT_IMPLEMENTED).
+    privacy = {
+        "ringct": RingCTScaffolding().status().value,
+        "stealth_address": StealthAddressScaffolding().status().value,
+        "shielded_balance": ShieldedBalanceScaffolding().status().value,
+        "dex": DEXScaffolding().status().value,
+    }
+
+    return {
+        "ok": True,
+        "now": now,
+        "chain_majority_time": chain_majority_time(chain),
+        "chain_event_count": len(chain),
+        "chain_stale": is_chain_stale(chain, now=now),
+        "ramp": {
+            "node_count": features.node_count,
+            "bootstrap_resolution_active": features.bootstrap_resolution_active,
+            "staked_resolution_active": features.staked_resolution_active,
+            "governance_petitions_active": features.governance_petitions_active,
+            "upgrade_governance_active": features.upgrade_governance_active,
+            "commoncoin_active": features.commoncoin_active,
+        },
+        "privacy_primitive_status": privacy,
+        "immutable_principles": dict(IMMUTABLE_PRINCIPLES),
+        "config_keys_count": len(CONFIG),
+        "infonet_economy_event_types_count": len(INFONET_ECONOMY_EVENT_TYPES),
+    }
+
+
+# ─── Petitions / governance ──────────────────────────────────────────────
+
+@router.get("/petitions")
+def list_petitions() -> dict[str, Any]:
+    """List petition_file events on the chain with their current state."""
+    chain = _live_chain()
+    now = _now()
+    out: list[dict[str, Any]] = []
+    for ev in chain:
+        if ev.get("event_type") != "petition_file":
+            continue
+        pid = (ev.get("payload") or {}).get("petition_id")
+        if not isinstance(pid, str):
+            continue
+        try:
+            state = compute_petition_state(pid, chain, now=now)
+            out.append({
+                "petition_id": state.petition_id,
+                "status": state.status,
+                "filer_id": state.filer_id,
+                "filed_at": state.filed_at,
+                "petition_payload": state.petition_payload,
+                "signature_governance_weight": state.signature_governance_weight,
+                "signature_threshold_at_filing": state.signature_threshold_at_filing,
+                "votes_for_weight": state.votes_for_weight,
+                "votes_against_weight": state.votes_against_weight,
+                "voting_deadline": state.voting_deadline,
+                "challenge_window_until": state.challenge_window_until,
+            })
+        except Exception as exc:
+            logger.warning("petition state error for %s: %s", pid, exc)
+    return {"ok": True, "petitions": out, "now": now}
+
+
+@router.get("/petitions/{petition_id}")
+def get_petition(petition_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    state = compute_petition_state(petition_id, chain, now=now)
+    return {"ok": True, "petition": state.__dict__, "now": now}
+
+
+@router.post("/petitions/preview")
+def preview_petition_payload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
+    """Validate a petition payload through the DSL executor without
+    emitting it. Returns the candidate config diff so the UI can show
+    "this petition would change vote_decay_days from 90 to 30".
+    """
+    try:
+        result = apply_petition_payload(payload)
+        return {
+            "ok": True,
+            "changed_keys": list(result.changed_keys),
+            "new_values": {k: result.new_config[k] for k in result.changed_keys},
+        }
+    except InvalidPetition as exc:
+        return {"ok": False, "reason": str(exc)}
+
+
+@router.post("/events/validate")
+def validate_event(body: dict[str, Any] = Body(...)) -> dict[str, Any]:
+    """Validate an arbitrary Infonet economy event payload.
+
+    Frontend uses this for client-side preflight before signing /
+    submitting an event. Returns ``{ok: True}`` on success or
+    ``{ok: False, reason: ...}`` with the exact validation failure.
+    """
+    event_type = body.get("event_type")
+    payload = body.get("payload", {})
+    if not isinstance(event_type, str) or not event_type:
+        return {"ok": False, "reason": "event_type required"}
+    if not isinstance(payload, dict):
+        return {"ok": False, "reason": "payload must be an object"}
+    ok, reason = validate_infonet_event_payload(event_type, payload)
+    return {
+        "ok": ok,
+        "reason": reason if not ok else None,
+        "tier": classify_event_type(event_type),
+        "would_be_provisional": should_mark_provisional(event_type, _live_chain(), now=_now()),
+    }
+
+
+# ─── Upgrade-hash governance ────────────────────────────────────────────
+
+@router.get("/upgrades")
+def list_upgrades() -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    out: list[dict[str, Any]] = []
+    for ev in chain:
+        if ev.get("event_type") != "upgrade_propose":
+            continue
+        pid = (ev.get("payload") or {}).get("proposal_id")
+        if not isinstance(pid, str):
+            continue
+        try:
+            # Heavy node set is a runtime concept (transport tier ==
+            # private_strong per plan §3.5). Empty here for the
+            # snapshot endpoint; production will pass the live set.
+            state = compute_upgrade_state(pid, chain, now=now, heavy_node_ids=set())
+            out.append({
+                "proposal_id": state.proposal_id,
+                "status": state.status,
+                "proposer_id": state.proposer_id,
+                "filed_at": state.filed_at,
+                "release_hash": state.release_hash,
+                "target_protocol_version": state.target_protocol_version,
+                "votes_for_weight": state.votes_for_weight,
+                "votes_against_weight": state.votes_against_weight,
+                "readiness_fraction": state.readiness.fraction,
+                "readiness_threshold_met": state.readiness.threshold_met,
+            })
+        except Exception as exc:
+            logger.warning("upgrade state error for %s: %s", pid, exc)
+    return {"ok": True, "upgrades": out, "now": now}
+
+
+@router.get("/upgrades/{proposal_id}")
+def get_upgrade(proposal_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    state = compute_upgrade_state(proposal_id, chain, now=now, heavy_node_ids=set())
+    return {
+        "ok": True,
+        "upgrade": {
+            "proposal_id": state.proposal_id,
+            "status": state.status,
+            "proposer_id": state.proposer_id,
+            "filed_at": state.filed_at,
+            "release_hash": state.release_hash,
+            "target_protocol_version": state.target_protocol_version,
+            "signature_governance_weight": state.signature_governance_weight,
+            "votes_for_weight": state.votes_for_weight,
+            "votes_against_weight": state.votes_against_weight,
+            "voting_deadline": state.voting_deadline,
+            "challenge_window_until": state.challenge_window_until,
+            "activation_deadline": state.activation_deadline,
+            "readiness": {
+                "total_heavy_nodes": state.readiness.total_heavy_nodes,
+                "ready_count": state.readiness.ready_count,
+                "fraction": state.readiness.fraction,
+                "threshold_met": state.readiness.threshold_met,
+            },
+        },
+        "now": now,
+    }
+
+
+# ─── Markets / resolution / disputes ────────────────────────────────────
+
+@router.get("/markets/{market_id}")
+def get_market_state(market_id: str = Path(...)) -> dict[str, Any]:
+    """Full market view: lifecycle, snapshot, evidence, stakes,
+    excluded predictors, dispute state."""
+    chain = _live_chain()
+    now = _now()
+    oracle = InfonetOracleAdapter(lambda: chain)
+
+    status = oracle.market_status(market_id, now=now)
+    snap = oracle.find_snapshot(market_id)
+    bundles = oracle.collect_evidence(market_id)
+    excluded = sorted(oracle.excluded_predictor_ids(market_id))
+    disputes = oracle.collect_disputes(market_id)
+    reversed_flag = oracle.market_was_reversed(market_id)
+
+    return {
+        "ok": True,
+        "market_id": market_id,
+        "status": status.value,
+        "snapshot": snap,
+        "evidence_bundles": [
+            {
+                "node_id": b.node_id,
+                "claimed_outcome": b.claimed_outcome,
+                "evidence_hashes": list(b.evidence_hashes),
+                "source_description": b.source_description,
+                "bond": b.bond,
+                "timestamp": b.timestamp,
+                "is_first_for_side": b.is_first_for_side,
+                "submission_hash": b.submission_hash,
+            }
+            for b in bundles
+        ],
+        "excluded_predictor_ids": excluded,
+        "disputes": [
+            {
+                "dispute_id": d.dispute_id,
+                "challenger_id": d.challenger_id,
+                "challenger_stake": d.challenger_stake,
+                "opened_at": d.opened_at,
+                "is_resolved": d.is_resolved,
+                "resolved_outcome": d.resolved_outcome,
+                "confirm_stakes": d.confirm_stakes,
+                "reverse_stakes": d.reverse_stakes,
+            }
+            for d in disputes
+        ],
+        "was_reversed": reversed_flag,
+        "now": now,
+    }
+
+
+@router.get("/markets/{market_id}/preview-resolution")
+def preview_resolution(market_id: str = Path(...)) -> dict[str, Any]:
+    """Run the resolution decision procedure without emitting a
+    finalize event. UI uses this to show "if resolution closed now,
+    the market would resolve as <outcome> for <reason>"."""
+    chain = _live_chain()
+    oracle = InfonetOracleAdapter(lambda: chain)
+    result = oracle.resolve_market(market_id)
+    return {
+        "ok": True,
+        "preview": {
+            "outcome": result.outcome,
+            "reason": result.reason,
+            "is_provisional": result.is_provisional,
+            "burned_amount": result.burned_amount,
+            "stake_returns": [
+                {"node_id": k[0], "rep_type": k[1], "amount": v}
+                for k, v in result.stake_returns.items()
+            ],
+            "stake_winnings": [
+                {"node_id": k[0], "rep_type": k[1], "amount": v}
+                for k, v in result.stake_winnings.items()
+            ],
+            "bond_returns": [
+                {"node_id": k, "amount": v} for k, v in result.bond_returns.items()
+            ],
+            "bond_forfeits": [
+                {"node_id": k, "amount": v} for k, v in result.bond_forfeits.items()
+            ],
+            "first_submitter_bonuses": [
+                {"node_id": k, "amount": v}
+                for k, v in result.first_submitter_bonuses.items()
+            ],
+        },
+    }
+
+
+# ─── Gate shutdown lifecycle ────────────────────────────────────────────
+
+@router.get("/gates/{gate_id}")
+def get_gate_state(gate_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    gates = InfonetGateAdapter(lambda: chain)
+    meta = gates.gate_meta(gate_id)
+    if meta is None:
+        return {"ok": False, "reason": "gate_not_found"}
+    suspension = gates.suspension_state(gate_id, now=now)
+    shutdown = gates.shutdown_state(gate_id, now=now)
+    locked = gates.locked_state(gate_id)
+    members = sorted(gates.member_set(gate_id))
+    return {
+        "ok": True,
+        "gate_id": gate_id,
+        "meta": {
+            "creator_node_id": meta.creator_node_id,
+            "display_name": meta.display_name,
+            "entry_sacrifice": meta.entry_sacrifice,
+            "min_overall_rep": meta.min_overall_rep,
+            "min_gate_rep": dict(meta.min_gate_rep),
+            "created_at": meta.created_at,
+        },
+        "members": members,
+        "ratified": gates.is_ratified(gate_id),
+        "cumulative_member_oracle_rep": gates.cumulative_member_oracle_rep(gate_id),
+        "locked": {
+            "is_locked": locked.locked,
+            "locked_at": locked.locked_at,
+            "locked_by": list(locked.locked_by),
+        },
+        "suspension": {
+            "status": suspension.status,
+            "suspended_at": suspension.suspended_at,
+            "suspended_until": suspension.suspended_until,
+            "last_shutdown_petition_at": suspension.last_shutdown_petition_at,
+        },
+        "shutdown": {
+            "has_pending": shutdown.has_pending,
+            "pending_petition_id": shutdown.pending_petition_id,
+            "pending_status": shutdown.pending_status,
+            "execution_at": shutdown.execution_at,
+            "executed": shutdown.executed,
+        },
+        "now": now,
+    }
+
+
+# ─── Reputation views ───────────────────────────────────────────────────
+
+@router.get("/nodes/{node_id}/reputation")
+def get_node_reputation(node_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    rep = InfonetReputationAdapter(lambda: chain)
+    breakdown = rep.oracle_rep_breakdown(node_id)
+    return {
+        "ok": True,
+        "node_id": node_id,
+        "oracle_rep": rep.oracle_rep(node_id),
+        "oracle_rep_active": rep.oracle_rep_active(node_id),
+        "oracle_rep_lifetime": rep.oracle_rep_lifetime(node_id),
+        "common_rep": rep.common_rep(node_id),
+        "decay_factor": rep.decay_factor(node_id),
+        "last_successful_prediction_ts": rep.last_successful_prediction_ts(node_id),
+        "breakdown": {
+            "free_prediction_mints": breakdown.free_prediction_mints,
+            "staked_prediction_returns": breakdown.staked_prediction_returns,
+            "staked_prediction_losses": breakdown.staked_prediction_losses,
+            "total": breakdown.total,
+        },
+    }
+
+
+# ─── Bootstrap ──────────────────────────────────────────────────────────
+
+@router.get("/bootstrap/markets/{market_id}")
+def get_bootstrap_market_state(market_id: str = Path(...)) -> dict[str, Any]:
+    """Bootstrap-mode-specific market view: who has voted, who is
+    eligible, current tally."""
+    from services.infonet.bootstrap import (
+        deduplicate_votes,
+        validate_bootstrap_eligibility,
+    )
+
+    chain = _live_chain()
+    canonical = deduplicate_votes(market_id, chain)
+    votes_summary: list[dict[str, Any]] = []
+    yes = 0
+    no = 0
+    for v in canonical:
+        node_id = v.get("node_id") or ""
+        side = (v.get("payload") or {}).get("side")
+        decision = validate_bootstrap_eligibility(node_id, market_id, chain)
+        votes_summary.append({
+            "node_id": node_id,
+            "side": side,
+            "eligible": decision.eligible,
+            "ineligible_reason": decision.reason if not decision.eligible else None,
+        })
+        if decision.eligible:
+            if side == "yes":
+                yes += 1
+            elif side == "no":
+                no += 1
+    total = yes + no
+    return {
+        "ok": True,
+        "market_id": market_id,
+        "votes": votes_summary,
+        "tally": {
+            "yes": yes,
+            "no": no,
+            "total_eligible": total,
+            "min_market_participants": int(CONFIG["min_market_participants"]),
+            "supermajority_threshold": float(CONFIG["bootstrap_resolution_supermajority"]),
+        },
+    }
+
+
+# ─── Signed write: append an Infonet economy event ──────────────────────
+
+@router.post("/append")
+def append_event(body: dict[str, Any] = Body(...)) -> dict[str, Any]:
+    """Append a signed Infonet economy event to the chain.
+
+    Body shape (all required for production):
+
+        {
+            "event_type": str,       # one of INFONET_ECONOMY_EVENT_TYPES
+            "node_id":    str,       # signer
+            "payload":    dict,      # event-specific fields
+            "signature":  str,       # hex
+            "sequence":   int,       # node-monotonic
+            "public_key": str,       # base64
+            "public_key_algo": str,  # "ed25519" or "ecdsa"
+            "protocol_version": str  # optional, defaults to current
+        }
+
+    The cutover-registered validators run automatically via
+    ``mesh_hashchain.Infonet.append`` — payload validation, signature
+    verification, replay protection, sequence ordering, public-key
+    binding, revocation status. No additional security wrapper is
+    needed because ``Infonet.append`` IS the secure entry point.
+
+    Returns the appended event dict on success, or
+    ``{"ok": False, "reason": "..."}`` on validation / signing failure.
+    """
+    if not isinstance(body, dict):
+        return {"ok": False, "reason": "body_must_be_object"}
+
+    event_type = body.get("event_type")
+    if not isinstance(event_type, str) or event_type not in INFONET_ECONOMY_EVENT_TYPES:
+        return {
+            "ok": False,
+            "reason": f"event_type must be one of INFONET_ECONOMY_EVENT_TYPES "
+                      f"(got {event_type!r})",
+        }
+
+    node_id = body.get("node_id")
+    if not isinstance(node_id, str) or not node_id:
+        return {"ok": False, "reason": "node_id required"}
+
+    payload = body.get("payload", {})
+    if not isinstance(payload, dict):
+        return {"ok": False, "reason": "payload must be an object"}
+
+    sequence = body.get("sequence", 0)
+    try:
+        sequence = int(sequence)
+    except (TypeError, ValueError):
+        return {"ok": False, "reason": "sequence must be an integer"}
+    if sequence <= 0:
+        return {"ok": False, "reason": "sequence must be > 0"}
+
+    signature = str(body.get("signature") or "")
+    public_key = str(body.get("public_key") or "")
+    public_key_algo = str(body.get("public_key_algo") or "")
+    protocol_version = str(body.get("protocol_version") or "")
+
+    if not signature or not public_key or not public_key_algo:
+        return {
+            "ok": False,
+            "reason": "signature, public_key, and public_key_algo are required",
+        }
+
+    try:
+        from services.mesh.mesh_hashchain import infonet
+        event = infonet.append(
+            event_type=event_type,
+            node_id=node_id,
+            payload=payload,
+            signature=signature,
+            sequence=sequence,
+            public_key=public_key,
+            public_key_algo=public_key_algo,
+            protocol_version=protocol_version,
+        )
+    except ValueError as exc:
+        # Infonet.append raises ValueError for any validation failure
+        # — payload / signature / replay / sequence / binding. The
+        # message is user-facing per the non-hostile UX rule.
+        return {"ok": False, "reason": str(exc)}
+    except Exception as exc:
+        logger.exception("infonet append failed")
+        return {"ok": False, "reason": f"server_error: {type(exc).__name__}"}
+
+    return {"ok": True, "event": event}
+
+
+# ─── Function Keys (citizen + operator views) ───────────────────────────
+
+@router.get("/function-keys/operator/{operator_id}/batch-summary")
+def operator_batch_summary(operator_id: str = Path(...)) -> dict[str, Any]:
+    """Sprint 11+ scaffolding: returns the operator's local batch
+    counter for the current period. Production wires this through the
+    operator's local-store implementation (Sprint 11+ scaffolding
+    doesn't persist; counts reset per process)."""
+    return {
+        "ok": True,
+        "operator_id": operator_id,
+        "scaffolding_only": True,
+        "note": "Production operators maintain a persistent BatchedSettlementBatch. "
+                "This endpoint reports the in-memory state of the local batch.",
+    }
+
+
+__all__ = ["router"]

backend/routers/intel_feeds.py

@@ -0,0 +1,130 @@
+"""Malware, cyber threats, and country risk feeds."""
+from __future__ import annotations
+
+import logging
+from urllib.parse import urlparse
+
+import requests
+from fastapi import APIRouter, HTTPException, Query, Request
+from fastapi.responses import StreamingResponse
+from starlette.background import BackgroundTask
+
+from limiter import limiter
+from services.fetchers._store import get_latest_data_subset_refs
+from services.fetchers.telegram_osint import telegram_media_host_allowed
+from services.intel_feeds.country_risk import build_country_risk_payload
+from services.network_utils import outbound_user_agent
+from services.telegram_translate import apply_posts_translations, normalize_translate_target
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/api/malware")
+@limiter.limit("60/minute")
+async def malware_feed(request: Request) -> dict:
+    snap = get_latest_data_subset_refs("malware_threats")
+    payload = snap.get("malware_threats")
+    if isinstance(payload, dict) and payload.get("threats") is not None:
+        return payload
+    return {"threats": [], "total": 0, "timestamp": None, "source": "abuse.ch"}
+
+
+@router.get("/api/cyber-threats")
+@limiter.limit("60/minute")
+async def cyber_threats(request: Request) -> dict:
+    snap = get_latest_data_subset_refs("cyber_threats")
+    return snap.get("cyber_threats") or {"threats": [], "stats": {}}
+
+
+@router.get("/api/country-risk")
+@limiter.limit("30/minute")
+async def country_risk(request: Request) -> dict:
+    return build_country_risk_payload()
+
+
+@router.get("/api/telegram-feed")
+@limiter.limit("30/minute")
+async def telegram_feed(request: Request, lang: str | None = Query(default=None)) -> dict:
+    snap = get_latest_data_subset_refs("telegram_osint")
+    payload = snap.get("telegram_osint")
+    if not isinstance(payload, dict) or payload.get("posts") is None:
+        return {"posts": [], "total": 0, "geolocated": 0, "timestamp": None}
+
+    if lang:
+        target = normalize_translate_target(lang)
+        localized = dict(payload)
+        localized["posts"] = apply_posts_translations(list(payload.get("posts") or []), target)
+        localized["translate_locale"] = target
+        return localized
+    return payload
+
+
+def _infer_telegram_media_type(target_url: str, content_type: str) -> str:
+    clean_type = str(content_type or "").split(";", 1)[0].strip().lower()
+    if clean_type and clean_type not in {"application/octet-stream", "binary/octet-stream"}:
+        return content_type
+    path = str(urlparse(target_url).path or "").lower()
+    if path.endswith((".jpg", ".jpeg")):
+        return "image/jpeg"
+    if path.endswith(".png"):
+        return "image/png"
+    if path.endswith(".webp"):
+        return "image/webp"
+    if path.endswith(".gif"):
+        return "image/gif"
+    if path.endswith(".mp4"):
+        return "video/mp4"
+    if path.endswith(".webm"):
+        return "video/webm"
+    return content_type or "application/octet-stream"
+
+
+@router.get("/api/telegram/media")
+@limiter.limit("60/minute")
+async def telegram_media_proxy(request: Request, url: str = Query(...)) -> StreamingResponse:
+    """Stream Telegram CDN media for in-app playback (host allowlist only)."""
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https"):
+        raise HTTPException(status_code=400, detail="Invalid scheme")
+    if not telegram_media_host_allowed(parsed.hostname):
+        raise HTTPException(status_code=403, detail="Host not allowed")
+
+    headers = {
+        "User-Agent": (
+            f"Mozilla/5.0 (compatible; {outbound_user_agent('telegram-media')}) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+        ),
+        "Accept": "*/*",
+    }
+    if range_header := request.headers.get("range"):
+        headers["Range"] = range_header
+
+    try:
+        resp = requests.get(url, stream=True, timeout=(3, 45), headers=headers)
+    except requests.RequestException as exc:
+        logger.warning("Telegram media upstream failure %s: %s", url, exc)
+        raise HTTPException(status_code=502, detail="Upstream fetch failed") from exc
+
+    if resp.status_code >= 400:
+        resp.close()
+        raise HTTPException(status_code=int(resp.status_code), detail=f"Upstream returned {resp.status_code}")
+
+    media_type = _infer_telegram_media_type(url, resp.headers.get("Content-Type", "application/octet-stream"))
+    response_headers = {
+        "Cache-Control": "private, max-age=300",
+        "Accept-Ranges": resp.headers.get("Accept-Ranges", "bytes"),
+    }
+    if content_length := resp.headers.get("Content-Length"):
+        response_headers["Content-Length"] = content_length
+    if content_range := resp.headers.get("Content-Range"):
+        response_headers["Content-Range"] = content_range
+
+    return StreamingResponse(
+        resp.iter_content(chunk_size=65536),
+        status_code=resp.status_code,
+        media_type=media_type,
+        headers=response_headers,
+        background=BackgroundTask(resp.close),
+    )

backend/routers/mesh_dm.py

@@ -0,0 +1,565 @@
+import asyncio
+import hashlib
+import hmac
+import logging
+import secrets
+import time
+from typing import Any
+
+from fastapi import APIRouter, Depends, Request
+from fastapi.responses import JSONResponse
+
+from auth import (
+    _is_debug_test_request,
+    _scoped_view_authenticated,
+    _verify_peer_push_hmac,
+    require_admin,
+)
+from limiter import limiter
+from services.config import get_settings
+from services.mesh.mesh_compatibility import (
+    LEGACY_AGENT_ID_LOOKUP_TARGET,
+    legacy_agent_id_lookup_blocked,
+    record_legacy_agent_id_lookup,
+    sunset_target_label,
+)
+from services.mesh.mesh_signed_events import (
+    MeshWriteExemption,
+    SignedWriteKind,
+    get_prepared_signed_write,
+    mesh_write_exempt,
+    requires_signed_write,
+)
+
+logger = logging.getLogger(__name__)
+_WARNED_LEGACY_DM_PUBKEY_LOOKUPS: set[str] = set()
+
+router = APIRouter()
+
+
+# ---------------------------------------------------------------------------
+# Local helpers
+# ---------------------------------------------------------------------------
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _warn_legacy_dm_pubkey_lookup(agent_id: str) -> None:
+    peer_id = str(agent_id or "").strip().lower()
+    if not peer_id or peer_id in _WARNED_LEGACY_DM_PUBKEY_LOOKUPS:
+        return
+    _WARNED_LEGACY_DM_PUBKEY_LOOKUPS.add(peer_id)
+    logger.warning(
+        "mesh legacy DH pubkey lookup used for %s via direct agent_id; prefer invite-scoped lookup handles before removal in %s",
+        peer_id,
+        sunset_target_label(LEGACY_AGENT_ID_LOOKUP_TARGET),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Transition delegates: forward to main.py so test monkeypatches still work.
+# These will move to a shared module once main.py routes are removed.
+# ---------------------------------------------------------------------------
+def _main_delegate(name):
+    def _wrapper(*a, **kw):
+        import main as _m
+        return getattr(_m, name)(*a, **kw)
+    _wrapper.__name__ = name
+    return _wrapper
+
+
+_verify_signed_write = _main_delegate("_verify_signed_write")
+_secure_dm_enabled = _main_delegate("_secure_dm_enabled")
+_legacy_dm_get_allowed = _main_delegate("_legacy_dm_get_allowed")
+_rns_private_dm_ready = _main_delegate("_rns_private_dm_ready")
+_anonymous_dm_hidden_transport_enforced = _main_delegate("_anonymous_dm_hidden_transport_enforced")
+_high_privacy_profile_enabled = _main_delegate("_high_privacy_profile_enabled")
+_dm_send_from_signed_request = _main_delegate("_dm_send_from_signed_request")
+_dm_poll_secure_from_signed_request = _main_delegate("_dm_poll_secure_from_signed_request")
+_dm_count_secure_from_signed_request = _main_delegate("_dm_count_secure_from_signed_request")
+_validate_private_signed_sequence = _main_delegate("_validate_private_signed_sequence")
+
+
+def _signed_body(request: Request) -> dict[str, Any]:
+    prepared = get_prepared_signed_write(request)
+    if prepared is None:
+        return {}
+    return dict(prepared.body)
+
+
+async def _maybe_apply_dm_relay_jitter() -> None:
+    if not _high_privacy_profile_enabled():
+        return
+    await asyncio.sleep((50 + secrets.randbelow(451)) / 1000.0)
+
+
+_REQUEST_V2_REDUCED_VERSION = "request-v2-reduced-v3"
+_REQUEST_V2_RECOVERY_STATES = {"pending", "verified", "failed"}
+
+
+def _is_canonical_reduced_request_message(message: dict[str, Any]) -> bool:
+    item = dict(message or {})
+    return (
+        str(item.get("delivery_class", "") or "").strip().lower() == "request"
+        and str(item.get("request_contract_version", "") or "").strip()
+        == _REQUEST_V2_REDUCED_VERSION
+        and item.get("sender_recovery_required") is True
+    )
+
+
+def _annotate_request_recovery_message(message: dict[str, Any]) -> dict[str, Any]:
+    item = dict(message or {})
+    delivery_class = str(item.get("delivery_class", "") or "").strip().lower()
+    sender_id = str(item.get("sender_id", "") or "").strip()
+    sender_seal = str(item.get("sender_seal", "") or "").strip()
+    sender_is_blinded = sender_id.startswith("sealed:") or sender_id.startswith("sender_token:")
+    if delivery_class != "request" or not sender_is_blinded or not sender_seal.startswith("v3:"):
+        return item
+    if not str(item.get("request_contract_version", "") or "").strip():
+        item["request_contract_version"] = _REQUEST_V2_REDUCED_VERSION
+    item["sender_recovery_required"] = True
+    state = str(item.get("sender_recovery_state", "") or "").strip().lower()
+    if state not in _REQUEST_V2_RECOVERY_STATES:
+        state = "pending"
+    item["sender_recovery_state"] = state
+    return item
+
+
+def _annotate_request_recovery_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    return [_annotate_request_recovery_message(message) for message in (messages or [])]
+
+
+def _request_duplicate_authority_rank(message: dict[str, Any]) -> int:
+    item = dict(message or {})
+    if str(item.get("delivery_class", "") or "").strip().lower() != "request":
+        return 0
+    if _is_canonical_reduced_request_message(item):
+        return 3
+    sender_id = str(item.get("sender_id", "") or "").strip()
+    if sender_id.startswith("sealed:") or sender_id.startswith("sender_token:"):
+        return 1
+    if sender_id:
+        return 2
+    return 0
+
+
+def _request_duplicate_recovery_rank(message: dict[str, Any]) -> int:
+    if not _is_canonical_reduced_request_message(message):
+        return 0
+    state = str(dict(message or {}).get("sender_recovery_state", "") or "").strip().lower()
+    if state == "verified":
+        return 2
+    if state == "pending":
+        return 1
+    return 0
+
+
+def _poll_duplicate_source_rank(source: str) -> int:
+    normalized = str(source or "").strip().lower()
+    if normalized == "relay":
+        return 2
+    if normalized == "reticulum":
+        return 1
+    return 0
+
+
+def _should_replace_dm_poll_duplicate(
+    existing: dict[str, Any],
+    existing_source: str,
+    candidate: dict[str, Any],
+    candidate_source: str,
+) -> bool:
+    candidate_authority = _request_duplicate_authority_rank(candidate)
+    existing_authority = _request_duplicate_authority_rank(existing)
+    if candidate_authority != existing_authority:
+        return candidate_authority > existing_authority
+
+    candidate_recovery = _request_duplicate_recovery_rank(candidate)
+    existing_recovery = _request_duplicate_recovery_rank(existing)
+    if candidate_recovery != existing_recovery:
+        return candidate_recovery > existing_recovery
+
+    candidate_source_rank = _poll_duplicate_source_rank(candidate_source)
+    existing_source_rank = _poll_duplicate_source_rank(existing_source)
+    if candidate_source_rank != existing_source_rank:
+        return candidate_source_rank > existing_source_rank
+
+    try:
+        candidate_ts = float(candidate.get("timestamp", 0) or 0)
+    except Exception:
+        candidate_ts = 0.0
+    try:
+        existing_ts = float(existing.get("timestamp", 0) or 0)
+    except Exception:
+        existing_ts = 0.0
+    return candidate_ts > existing_ts
+
+
+def _merge_dm_poll_messages(
+    relay_messages: list[dict[str, Any]],
+    direct_messages: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    merged: list[dict[str, Any]] = []
+    index_by_msg_id: dict[str, tuple[int, str]] = {}
+
+    def add_messages(items: list[dict[str, Any]], source: str) -> None:
+        for original in items or []:
+            item = dict(original or {})
+            msg_id = str(item.get("msg_id", "") or "").strip()
+            if not msg_id:
+                merged.append(item)
+                continue
+            existing = index_by_msg_id.get(msg_id)
+            if existing is None:
+                index_by_msg_id[msg_id] = (len(merged), source)
+                merged.append(item)
+                continue
+            index, existing_source = existing
+            if _should_replace_dm_poll_duplicate(merged[index], existing_source, item, source):
+                merged[index] = item
+                index_by_msg_id[msg_id] = (index, source)
+
+    add_messages(relay_messages, "relay")
+    add_messages(direct_messages, "reticulum")
+    return sorted(merged, key=lambda item: float(item.get("timestamp", 0) or 0))
+
+
+# ---------------------------------------------------------------------------
+# Route handlers
+# ---------------------------------------------------------------------------
+
+@router.post("/api/mesh/dm/register")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_REGISTER)
+async def dm_register_key(request: Request):
+    """Register a DH public key for encrypted DM key exchange."""
+    body = _signed_body(request)
+    agent_id = body.get("agent_id", "").strip()
+    dh_pub_key = body.get("dh_pub_key", "").strip()
+    dh_algo = body.get("dh_algo", "").strip()
+    timestamp = _safe_int(body.get("timestamp", 0) or 0)
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not agent_id or not dh_pub_key or not dh_algo or not timestamp:
+        return {"ok": False, "detail": "Missing agent_id, dh_pub_key, dh_algo, or timestamp"}
+    if dh_algo.upper() not in ("X25519", "ECDH_P256", "ECDH"):
+        return {"ok": False, "detail": "Unsupported dh_algo"}
+    now_ts = int(time.time())
+    if abs(timestamp - now_ts) > 7 * 86400:
+        return {"ok": False, "detail": "DH key timestamp is too far from current time"}
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+
+        reputation_ledger.register_node(agent_id, public_key, public_key_algo)
+    except Exception:
+        pass
+
+    accepted, detail, metadata = dm_relay.register_dh_key(
+        agent_id,
+        dh_pub_key,
+        dh_algo,
+        timestamp,
+        signature,
+        public_key,
+        public_key_algo,
+        protocol_version,
+        sequence,
+    )
+    if not accepted:
+        return {"ok": False, "detail": detail}
+
+    return {"ok": True, **(metadata or {})}
+
+
+@router.get("/api/mesh/dm/pubkey")
+@limiter.limit("30/minute")
+async def dm_get_pubkey(request: Request, agent_id: str = "", lookup_token: str = ""):
+    import main as _m
+
+    return await _m.dm_get_pubkey(request, agent_id=agent_id, lookup_token=lookup_token)
+
+
+@router.get("/api/mesh/dm/prekey-bundle")
+@limiter.limit("30/minute")
+async def dm_get_prekey_bundle(request: Request, agent_id: str = "", lookup_token: str = ""):
+    import main as _m
+
+    return await _m.dm_get_prekey_bundle(request, agent_id=agent_id, lookup_token=lookup_token)
+
+
+@router.post("/api/mesh/dm/prekey-peer-lookup")
+@limiter.limit("60/minute")
+@mesh_write_exempt(MeshWriteExemption.PEER_GOSSIP)
+async def dm_prekey_peer_lookup(request: Request):
+    """Peer-authenticated invite lookup handle resolution.
+
+    This endpoint exists for private/bootstrap peers to import signed invites
+    without exposing a stable agent_id on the ordinary lookup surface. It only
+    accepts HMAC-authenticated peer calls and only resolves lookup_token.
+    """
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 4096:
+                return JSONResponse(
+                    status_code=413,
+                    content={"ok": False, "detail": "Request body too large"},
+                )
+        except (TypeError, ValueError):
+            pass
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return JSONResponse(
+            status_code=403,
+            content={"ok": False, "detail": "Invalid or missing peer HMAC"},
+        )
+    try:
+        import json
+
+        body = json.loads(body_bytes or b"{}")
+    except Exception:
+        return {"ok": False, "detail": "invalid json"}
+    lookup_token = str(dict(body or {}).get("lookup_token", "") or "").strip()
+    if not lookup_token:
+        return {"ok": False, "detail": "lookup_token required"}
+    from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle
+
+    result = fetch_dm_prekey_bundle(
+        agent_id="",
+        lookup_token=lookup_token,
+        allow_peer_lookup=False,
+    )
+    if not result.get("ok"):
+        return {"ok": False, "detail": str(result.get("detail", "") or "Prekey bundle not found")}
+    safe = dict(result)
+    safe.pop("resolved_agent_id", None)
+    safe["lookup_mode"] = "invite_lookup_handle"
+    return safe
+
+
+@router.post("/api/mesh/dm/send")
+@limiter.limit("20/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_SEND)
+async def dm_send(request: Request):
+    return await _dm_send_from_signed_request(request)
+
+
+@router.post("/api/mesh/dm/poll")
+@limiter.limit("30/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_POLL)
+async def dm_poll_secure(request: Request):
+    return await _dm_poll_secure_from_signed_request(request)
+
+
+@router.get("/api/mesh/dm/poll")
+@limiter.limit("30/minute")
+async def dm_poll(
+    request: Request,
+    agent_id: str = "",
+    agent_token: str = "",
+    agent_token_prev: str = "",
+    agent_tokens: str = "",
+):
+    import main as _m
+
+    return await _m.dm_poll(
+        request,
+        agent_id=agent_id,
+        agent_token=agent_token,
+        agent_token_prev=agent_token_prev,
+        agent_tokens=agent_tokens,
+    )
+
+
+@router.post("/api/mesh/dm/count")
+@limiter.limit("60/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_COUNT)
+async def dm_count_secure(request: Request):
+    return await _dm_count_secure_from_signed_request(request)
+
+
+@router.get("/api/mesh/dm/count")
+@limiter.limit("60/minute")
+async def dm_count(
+    request: Request,
+    agent_id: str = "",
+    agent_token: str = "",
+    agent_token_prev: str = "",
+    agent_tokens: str = "",
+):
+    import main as _m
+
+    return await _m.dm_count(
+        request,
+        agent_id=agent_id,
+        agent_token=agent_token,
+        agent_token_prev=agent_token_prev,
+        agent_tokens=agent_tokens,
+    )
+
+
+@router.post("/api/mesh/dm/block")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_BLOCK)
+async def dm_block(request: Request):
+    """Block or unblock a sender from DMing you."""
+    body = _signed_body(request)
+    agent_id = body.get("agent_id", "").strip()
+    blocked_id = body.get("blocked_id", "").strip()
+    action = body.get("action", "block").strip().lower()
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not agent_id or not blocked_id:
+        return {"ok": False, "detail": "Missing agent_id or blocked_id"}
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    try:
+        from services.mesh.mesh_hashchain import infonet
+
+        ok_seq, seq_reason = _validate_private_signed_sequence(
+            infonet,
+            agent_id,
+            sequence,
+            domain=f"dm_block:{action}",
+        )
+        if not ok_seq:
+            return {"ok": False, "detail": seq_reason}
+    except Exception:
+        pass
+
+    if action == "unblock":
+        dm_relay.unblock(agent_id, blocked_id)
+    else:
+        dm_relay.block(agent_id, blocked_id)
+    return {"ok": True, "action": action, "blocked_id": blocked_id}
+
+
+@router.post("/api/mesh/dm/witness")
+@limiter.limit("20/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_WITNESS)
+async def dm_key_witness(request: Request):
+    """Record a lightweight witness for a DM key (dual-path spot-check)."""
+    body = _signed_body(request)
+    witness_id = body.get("witness_id", "").strip()
+    target_id = body.get("target_id", "").strip()
+    dh_pub_key = body.get("dh_pub_key", "").strip()
+    timestamp = _safe_int(body.get("timestamp", 0) or 0)
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not witness_id or not target_id or not dh_pub_key or not timestamp:
+        return {"ok": False, "detail": "Missing witness_id, target_id, dh_pub_key, or timestamp"}
+    now_ts = int(time.time())
+    if abs(timestamp - now_ts) > 7 * 86400:
+        return {"ok": False, "detail": "Witness timestamp is too far from current time"}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+
+        reputation_ledger.register_node(witness_id, public_key, public_key_algo)
+    except Exception:
+        pass
+    try:
+        from services.mesh.mesh_hashchain import infonet
+
+        ok_seq, seq_reason = _validate_private_signed_sequence(
+            infonet,
+            witness_id,
+            sequence,
+            domain="dm_witness",
+        )
+        if not ok_seq:
+            return {"ok": False, "detail": seq_reason}
+    except Exception:
+        pass
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    ok, reason = dm_relay.record_witness(witness_id, target_id, dh_pub_key, timestamp)
+    return {"ok": ok, "detail": reason}
+
+
+@router.get("/api/mesh/dm/witness")
+@limiter.limit("60/minute")
+async def dm_key_witness_get(request: Request, target_id: str = "", dh_pub_key: str = ""):
+    """Get witness counts for a target's DH key."""
+    if not target_id:
+        return {"ok": False, "detail": "Missing target_id"}
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    witnesses = dm_relay.get_witnesses(target_id, dh_pub_key if dh_pub_key else None, limit=5)
+    response = {
+        "ok": True,
+        "count": len(witnesses),
+    }
+    if _scoped_view_authenticated(request, "mesh.audit"):
+        response["target_id"] = target_id
+        response["dh_pub_key"] = dh_pub_key or ""
+        response["witnesses"] = witnesses
+    return response
+
+
+@router.post("/api/mesh/trust/vouch")
+@limiter.limit("20/minute")
+@requires_signed_write(kind=SignedWriteKind.TRUST_VOUCH)
+async def trust_vouch(request: Request):
+    """Record a trust vouch for a node (web-of-trust signal)."""
+    body = _signed_body(request)
+    voucher_id = body.get("voucher_id", "").strip()
+    target_id = body.get("target_id", "").strip()
+    note = body.get("note", "").strip()
+    timestamp = _safe_int(body.get("timestamp", 0) or 0)
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not voucher_id or not target_id or not timestamp:
+        return {"ok": False, "detail": "Missing voucher_id, target_id, or timestamp"}
+    now_ts = int(time.time())
+    if abs(timestamp - now_ts) > 7 * 86400:
+        return {"ok": False, "detail": "Vouch timestamp is too far from current time"}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+        from services.mesh.mesh_hashchain import infonet
+
+        reputation_ledger.register_node(voucher_id, public_key, public_key_algo)
+        ok_seq, seq_reason = _validate_private_signed_sequence(
+            infonet,
+            voucher_id,
+            sequence,
+            domain="trust_vouch",
+        )
+        if not ok_seq:
+            return {"ok": False, "detail": seq_reason}
+        ok, reason = reputation_ledger.add_vouch(voucher_id, target_id, note, timestamp)
+        return {"ok": ok, "detail": reason}
+    except Exception:
+        return {"ok": False, "detail": "Failed to record vouch"}
+
+
+@router.get("/api/mesh/trust/vouches", dependencies=[Depends(require_admin)])
+@limiter.limit("60/minute")
+async def trust_vouches(request: Request, node_id: str = "", limit: int = 20):
+    """Fetch latest vouches for a node."""
+    if not node_id:
+        return {"ok": False, "detail": "Missing node_id"}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+
+        vouches = reputation_ledger.get_vouches(node_id, limit=limit)
+        return {"ok": True, "node_id": node_id, "vouches": vouches, "count": len(vouches)}
+    except Exception:
+        return {"ok": False, "detail": "Failed to fetch vouches"}

backend/routers/mesh_operator.py

@@ -0,0 +1,145 @@
+import time
+import logging
+from fastapi import APIRouter, Request, Response, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def list_peers(request: Request, bucket: str = Query(None)):
+    """List all peers (or filter by bucket: sync, push, bootstrap)."""
+    from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerStore
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception as exc:
+        return {"ok": False, "detail": f"Failed to load peer store: {exc}"}
+    if bucket:
+        records = store.records_for_bucket(bucket)
+    else:
+        records = store.records()
+    return {"ok": True, "count": len(records), "peers": [r.to_dict() for r in records]}
+
+
+@router.post("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def add_peer(request: Request):
+    """Add a peer to the store. Body: {peer_url, transport?, label?, role?, buckets?[]}."""
+    from services.mesh.mesh_crypto import normalize_peer_url
+    from services.mesh.mesh_peer_store import (
+        DEFAULT_PEER_STORE_PATH, PeerStore, PeerStoreError,
+        make_push_peer_record, make_sync_peer_record,
+    )
+    from services.mesh.mesh_router import peer_transport_kind
+    body = await request.json()
+    peer_url_raw = str(body.get("peer_url", "") or "").strip()
+    if not peer_url_raw:
+        return {"ok": False, "detail": "peer_url is required"}
+    peer_url = normalize_peer_url(peer_url_raw)
+    if not peer_url:
+        return {"ok": False, "detail": "Invalid peer_url"}
+    transport = str(body.get("transport", "") or "").strip().lower()
+    if not transport:
+        transport = peer_transport_kind(peer_url)
+    if not transport:
+        return {"ok": False, "detail": "Cannot determine transport for peer_url — provide transport explicitly"}
+    label = str(body.get("label", "") or "").strip()
+    role = str(body.get("role", "") or "").strip().lower() or "relay"
+    buckets = body.get("buckets", ["sync", "push"])
+    if isinstance(buckets, str):
+        buckets = [buckets]
+    if not isinstance(buckets, list):
+        buckets = ["sync", "push"]
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    added: list = []
+    try:
+        for b in buckets:
+            b = str(b).strip().lower()
+            if b == "sync":
+                store.upsert(make_sync_peer_record(peer_url=peer_url, transport=transport, role=role, label=label))
+                added.append("sync")
+            elif b == "push":
+                store.upsert(make_push_peer_record(peer_url=peer_url, transport=transport, role=role, label=label))
+                added.append("push")
+        store.save()
+    except PeerStoreError as exc:
+        return {"ok": False, "detail": str(exc)}
+    return {"ok": True, "peer_url": peer_url, "buckets": added}
+
+
+@router.delete("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def remove_peer(request: Request):
+    """Remove a peer. Body: {peer_url, bucket?}. If bucket omitted, removes from all buckets."""
+    from services.mesh.mesh_crypto import normalize_peer_url
+    from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerStore
+    body = await request.json()
+    peer_url_raw = str(body.get("peer_url", "") or "").strip()
+    if not peer_url_raw:
+        return {"ok": False, "detail": "peer_url is required"}
+    peer_url = normalize_peer_url(peer_url_raw)
+    if not peer_url:
+        return {"ok": False, "detail": "Invalid peer_url"}
+    bucket_filter = str(body.get("bucket", "") or "").strip().lower()
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        return {"ok": False, "detail": "Failed to load peer store"}
+    removed: list = []
+    for b in ["bootstrap", "sync", "push"]:
+        if bucket_filter and b != bucket_filter:
+            continue
+        key = f"{b}:{peer_url}"
+        if key in store._records:
+            del store._records[key]
+            removed.append(b)
+    if not removed:
+        return {"ok": False, "detail": "Peer not found in any bucket"}
+    store.save()
+    return {"ok": True, "peer_url": peer_url, "removed_from": removed}
+
+
+@router.patch("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def toggle_peer(request: Request):
+    """Enable or disable a peer. Body: {peer_url, bucket, enabled: bool}."""
+    from services.mesh.mesh_crypto import normalize_peer_url
+    from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerRecord, PeerStore
+    body = await request.json()
+    peer_url_raw = str(body.get("peer_url", "") or "").strip()
+    bucket = str(body.get("bucket", "") or "").strip().lower()
+    enabled = body.get("enabled")
+    if not peer_url_raw:
+        return {"ok": False, "detail": "peer_url is required"}
+    if not bucket:
+        return {"ok": False, "detail": "bucket is required"}
+    if enabled is None:
+        return {"ok": False, "detail": "enabled (true/false) is required"}
+    peer_url = normalize_peer_url(peer_url_raw)
+    if not peer_url:
+        return {"ok": False, "detail": "Invalid peer_url"}
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        return {"ok": False, "detail": "Failed to load peer store"}
+    key = f"{bucket}:{peer_url}"
+    record = store._records.get(key)
+    if not record:
+        return {"ok": False, "detail": f"Peer not found in {bucket} bucket"}
+    updated = PeerRecord(**{**record.to_dict(), "enabled": bool(enabled), "updated_at": int(time.time())})
+    store._records[key] = updated
+    store.save()
+    return {"ok": True, "peer_url": peer_url, "bucket": bucket, "enabled": bool(enabled)}

backend/routers/mesh_oracle.py

@@ -0,0 +1,354 @@
+import math
+from typing import Any
+from fastapi import APIRouter, Request, Response, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator, _scoped_view_authenticated
+from services.data_fetcher import get_latest_data
+from services.mesh.mesh_protocol import normalize_payload
+from services.mesh.mesh_signed_events import (
+    MeshWriteExemption,
+    SignedWriteKind,
+    get_prepared_signed_write,
+    mesh_write_exempt,
+    requires_signed_write,
+)
+
+router = APIRouter()
+
+
+def _signed_body(request: Request) -> dict[str, Any]:
+    prepared = get_prepared_signed_write(request)
+    if prepared is None:
+        return {}
+    return dict(prepared.body)
+
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _safe_float(val, default=0.0):
+    try:
+        parsed = float(val)
+        if not math.isfinite(parsed):
+            return default
+        return parsed
+    except (TypeError, ValueError):
+        return default
+
+
+def _redact_public_oracle_profile(payload: dict, authenticated: bool) -> dict:
+    redacted = dict(payload)
+    if authenticated:
+        return redacted
+    redacted["active_stakes"] = []
+    redacted["prediction_history"] = []
+    return redacted
+
+
+def _redact_public_oracle_predictions(predictions: list, authenticated: bool) -> dict:
+    if authenticated:
+        return {"predictions": list(predictions)}
+    return {"predictions": [], "count": len(predictions)}
+
+
+def _redact_public_oracle_stakes(payload: dict, authenticated: bool) -> dict:
+    redacted = dict(payload)
+    if authenticated:
+        return redacted
+    redacted["truth_stakers"] = []
+    redacted["false_stakers"] = []
+    return redacted
+
+
+@router.post("/api/mesh/oracle/predict")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.ORACLE_PREDICT)
+async def oracle_predict(request: Request):
+    """Place a prediction on a market outcome."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    body = _signed_body(request)
+    node_id = body.get("node_id", "")
+    market_title = body.get("market_title", "")
+    side = body.get("side", "")
+    stake_amount = _safe_float(body.get("stake_amount", 0))
+    public_key = body.get("public_key", "")
+    public_key_algo = body.get("public_key_algo", "")
+    signature = body.get("signature", "")
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "")
+    if not node_id or not market_title or not side:
+        return {"ok": False, "detail": "Missing node_id, market_title, or side"}
+    prediction_payload = {"market_title": market_title, "side": side, "stake_amount": stake_amount}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+        reputation_ledger.register_node(node_id, public_key, public_key_algo)
+    except Exception:
+        pass
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    matched = None
+    for m in markets:
+        if m.get("title", "").lower() == market_title.lower():
+            matched = m
+            break
+    if not matched:
+        for m in markets:
+            if market_title.lower() in m.get("title", "").lower():
+                matched = m
+                break
+    if not matched:
+        return {"ok": False, "detail": f"Market '{market_title}' not found in active markets."}
+    probability = 50.0
+    side_lower = side.lower()
+    outcomes = matched.get("outcomes", [])
+    if outcomes:
+        for o in outcomes:
+            if o.get("name", "").lower() == side_lower:
+                probability = float(o.get("pct", 50))
+                break
+    else:
+        consensus = matched.get("consensus_pct")
+        if consensus is None:
+            consensus = matched.get("polymarket_pct") or matched.get("kalshi_pct") or 50
+        probability = float(consensus)
+        if side_lower == "no":
+            probability = 100.0 - probability
+    if stake_amount > 0:
+        ok, detail = oracle_ledger.place_market_stake(node_id, matched["title"], side, stake_amount, probability)
+        mode = "staked"
+    else:
+        ok, detail = oracle_ledger.place_prediction(node_id, matched["title"], side, probability)
+        mode = "free"
+    if ok:
+        try:
+            from services.mesh.mesh_hashchain import infonet
+            normalized_payload = normalize_payload("prediction", prediction_payload)
+            infonet.append(event_type="prediction", node_id=node_id, payload=normalized_payload,
+                signature=signature, sequence=sequence, public_key=public_key,
+                public_key_algo=public_key_algo, protocol_version=protocol_version)
+        except Exception:
+            pass
+    return {"ok": ok, "detail": detail, "probability": probability, "mode": mode}
+
+
+@router.get("/api/mesh/oracle/markets")
+@limiter.limit("30/minute")
+async def oracle_markets(request: Request):
+    """List active prediction markets."""
+    from collections import defaultdict
+    from services.mesh.mesh_oracle import oracle_ledger
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    all_consensus = oracle_ledger.get_all_market_consensus()
+    by_category = defaultdict(list)
+    for m in markets:
+        by_category[m.get("category", "NEWS")].append(m)
+    _fields = ("title", "consensus_pct", "polymarket_pct", "kalshi_pct", "volume", "volume_24h",
+               "end_date", "description", "category", "sources", "slug", "kalshi_ticker", "outcomes")
+    categories = {}
+    cat_totals = {}
+    for cat in ["POLITICS", "CONFLICT", "NEWS", "FINANCE", "CRYPTO"]:
+        all_cat = sorted(by_category.get(cat, []), key=lambda x: x.get("volume", 0) or 0, reverse=True)
+        cat_totals[cat] = len(all_cat)
+        cat_list = []
+        for m in all_cat[:10]:
+            entry = {k: m.get(k) for k in _fields}
+            entry["consensus"] = all_consensus.get(m.get("title", ""), {})
+            cat_list.append(entry)
+        categories[cat] = cat_list
+    return {"categories": categories, "total_count": len(markets), "cat_totals": cat_totals}
+
+
+@router.get("/api/mesh/oracle/search")
+@limiter.limit("20/minute")
+async def oracle_search(request: Request, q: str = "", limit: int = 50):
+    """Search prediction markets across Polymarket + Kalshi APIs."""
+    if not q or len(q) < 2:
+        return {"results": [], "query": q, "count": 0}
+    from services.fetchers.prediction_markets import search_polymarket_direct, search_kalshi_direct
+    import concurrent.futures
+    # Search both APIs in parallel for speed
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
+        poly_fut = pool.submit(search_polymarket_direct, q, limit)
+        kalshi_fut = pool.submit(search_kalshi_direct, q, limit)
+        poly_results = poly_fut.result(timeout=20)
+        kalshi_results = kalshi_fut.result(timeout=20)
+    # Also check cached/merged markets
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    q_lower = q.lower()
+    cached_matches = [m for m in markets if q_lower in m.get("title", "").lower()]
+    seen_titles = set()
+    combined = []
+    # Cached first (already merged Poly+Kalshi with consensus)
+    for m in cached_matches:
+        seen_titles.add(m["title"].lower())
+        combined.append(m)
+    # Then Polymarket direct hits
+    for m in poly_results:
+        if m["title"].lower() not in seen_titles:
+            seen_titles.add(m["title"].lower())
+            combined.append(m)
+    # Then Kalshi direct hits
+    for m in kalshi_results:
+        if m["title"].lower() not in seen_titles:
+            seen_titles.add(m["title"].lower())
+            combined.append(m)
+    combined.sort(key=lambda x: x.get("volume", 0) or 0, reverse=True)
+    _fields = ("title", "consensus_pct", "polymarket_pct", "kalshi_pct", "volume", "volume_24h",
+               "end_date", "description", "category", "sources", "slug", "kalshi_ticker", "outcomes")
+    results = [{k: m.get(k) for k in _fields} for m in combined[:limit]]
+    return {"results": results, "query": q, "count": len(results)}
+
+
+@router.get("/api/mesh/oracle/markets/more")
+@limiter.limit("30/minute")
+async def oracle_markets_more(request: Request, category: str = "NEWS", offset: int = 0, limit: int = 10):
+    """Load more markets for a specific category (paginated)."""
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    cat_markets = sorted([m for m in markets if m.get("category") == category],
+        key=lambda x: x.get("volume", 0) or 0, reverse=True)
+    page = cat_markets[offset : offset + limit]
+    _fields = ("title", "consensus_pct", "polymarket_pct", "kalshi_pct", "volume", "volume_24h",
+               "end_date", "description", "category", "sources", "slug", "kalshi_ticker", "outcomes")
+    results = [{k: m.get(k) for k in _fields} for m in page]
+    return {"markets": results, "category": category, "offset": offset,
+            "has_more": offset + limit < len(cat_markets), "total": len(cat_markets)}
+
+
+@router.post(
+    "/api/mesh/oracle/resolve",
+    dependencies=[Depends(require_admin)],
+)
+@limiter.limit("5/minute")
+@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
+async def oracle_resolve(request: Request):
+    """Resolve a prediction market.
+
+    Issue #240 (tg12): requires admin authentication. The
+    ``mesh_write_exempt`` decorator below is **metadata only** — it tags
+    the route as not requiring a mesh signed-write envelope, it does
+    NOT itself enforce caller authorization. The ``Depends(require_admin)``
+    on the route decorator is what actually gates access.
+    """
+    from services.mesh.mesh_oracle import oracle_ledger
+    body = await request.json()
+    market_title = body.get("market_title", "")
+    outcome = body.get("outcome", "")
+    if not market_title or not outcome:
+        return {"ok": False, "detail": "Need market_title and outcome"}
+    winners, losers = oracle_ledger.resolve_market(market_title, outcome)
+    stake_result = oracle_ledger.resolve_market_stakes(market_title, outcome)
+    return {"ok": True,
+            "detail": f"Resolved: {winners} free winners, {losers} free losers, "
+                      f"{stake_result.get('winners', 0)} stake winners, {stake_result.get('losers', 0)} stake losers",
+            "free": {"winners": winners, "losers": losers}, "stakes": stake_result}
+
+
+@router.get("/api/mesh/oracle/consensus")
+@limiter.limit("30/minute")
+async def oracle_consensus(request: Request, market_title: str = ""):
+    """Get network consensus for a market."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    if not market_title:
+        return {"error": "market_title required"}
+    return oracle_ledger.get_market_consensus(market_title)
+
+
+@router.post("/api/mesh/oracle/stake")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.ORACLE_STAKE)
+async def oracle_stake(request: Request):
+    """Stake oracle rep on a post's truthfulness."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    body = _signed_body(request)
+    staker_id = body.get("staker_id", "")
+    message_id = body.get("message_id", "")
+    poster_id = body.get("poster_id", "")
+    side = body.get("side", "").lower()
+    amount = _safe_float(body.get("amount", 0))
+    duration_days = _safe_int(body.get("duration_days", 1), 1)
+    public_key = body.get("public_key", "")
+    public_key_algo = body.get("public_key_algo", "")
+    signature = body.get("signature", "")
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "")
+    if not staker_id or not message_id or not side:
+        return {"ok": False, "detail": "Missing staker_id, message_id, or side"}
+    stake_payload = {"message_id": message_id, "poster_id": poster_id, "side": side,
+                     "amount": amount, "duration_days": duration_days}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+        reputation_ledger.register_node(staker_id, public_key, public_key_algo)
+    except Exception:
+        pass
+    ok, detail = oracle_ledger.place_stake(staker_id, message_id, poster_id, side, amount, duration_days)
+    if ok:
+        try:
+            from services.mesh.mesh_hashchain import infonet
+            normalized_payload = normalize_payload("stake", stake_payload)
+            infonet.append(event_type="stake", node_id=staker_id, payload=normalized_payload,
+                signature=signature, sequence=sequence, public_key=public_key,
+                public_key_algo=public_key_algo, protocol_version=protocol_version)
+        except Exception:
+            pass
+    return {"ok": ok, "detail": detail}
+
+
+@router.get("/api/mesh/oracle/stakes/{message_id}")
+@limiter.limit("30/minute")
+async def oracle_stakes_for_message(request: Request, message_id: str):
+    """Get all oracle stakes on a message."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    return _redact_public_oracle_stakes(
+        oracle_ledger.get_stakes_for_message(message_id),
+        authenticated=_scoped_view_authenticated(request, "mesh.audit"),
+    )
+
+
+@router.get("/api/mesh/oracle/profile")
+@limiter.limit("30/minute")
+async def oracle_profile(request: Request, node_id: str = ""):
+    """Get full oracle profile."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    if not node_id:
+        return {"ok": False, "detail": "Provide ?node_id=xxx"}
+    profile = oracle_ledger.get_oracle_profile(node_id)
+    return _redact_public_oracle_profile(
+        profile, authenticated=_scoped_view_authenticated(request, "mesh.audit"))
+
+
+@router.get("/api/mesh/oracle/predictions")
+@limiter.limit("30/minute")
+async def oracle_predictions(request: Request, node_id: str = ""):
+    """Get a node's active (unresolved) predictions."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    if not node_id:
+        return {"ok": False, "detail": "Provide ?node_id=xxx"}
+    active_predictions = oracle_ledger.get_active_predictions(node_id)
+    return _redact_public_oracle_predictions(
+        active_predictions, authenticated=_scoped_view_authenticated(request, "mesh.audit"))
+
+
+@router.post(
+    "/api/mesh/oracle/resolve-stakes",
+    dependencies=[Depends(require_admin)],
+)
+@limiter.limit("5/minute")
+@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
+async def oracle_resolve_stakes(request: Request):
+    """Resolve all expired stake contests.
+
+    Issue #241 (tg12): requires admin authentication. See the note on
+    ``oracle_resolve`` above — ``mesh_write_exempt`` is metadata only.
+    """
+    from services.mesh.mesh_oracle import oracle_ledger
+    resolutions = oracle_ledger.resolve_expired_stakes()
+    return {"ok": True, "resolutions": resolutions, "count": len(resolutions)}

backend/routers/mesh_peer_sync.py

@@ -0,0 +1,308 @@
+import json as json_mod
+import logging
+from typing import Any
+from fastapi import APIRouter, Request, Response
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator, _verify_peer_push_hmac
+from services.config import get_settings
+from services.mesh.mesh_crypto import normalize_peer_url
+from services.mesh.mesh_router import peer_transport_kind
+from auth import _peer_hmac_url_from_request
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+_PEER_PUSH_BATCH_SIZE = 50
+
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _hydrate_gate_store_from_chain(events: list) -> int:
+    """Copy any gate_message chain events into the local gate_store for read/decrypt.
+
+    Only events that are resident in the local infonet (accepted or already
+    present) are hydrated.  The canonical infonet-resident event is used —
+    never the raw batch event — so a forged batch entry carrying a valid
+    event_id but attacker-chosen payload cannot pollute gate_store.
+    """
+    import copy
+    from services.mesh.mesh_hashchain import gate_store, infonet
+    count = 0
+    for evt in events:
+        if evt.get("event_type") != "gate_message":
+            continue
+        event_id = str(evt.get("event_id", "") or "").strip()
+        if not event_id or event_id not in infonet.event_index:
+            continue
+        canonical = infonet.events[infonet.event_index[event_id]]
+        payload = canonical.get("payload") or {}
+        gate_id = str(payload.get("gate", "") or "").strip()
+        if not gate_id:
+            continue
+        try:
+            gate_store.append(gate_id, copy.deepcopy(canonical))
+            count += 1
+        except Exception:
+            pass
+    return count
+
+
+def _hydrate_dm_relay_from_chain(events: list) -> int:
+    import main as _m
+
+    return int(_m._hydrate_dm_relay_from_chain(events))
+
+
+@router.post("/api/mesh/infonet/peer-push")
+@limiter.limit("30/minute")
+async def infonet_peer_push(request: Request):
+    """Accept pushed Infonet events from relay peers (HMAC-authenticated)."""
+    from services.mesh.mesh_fleet_defaults import infonet_fleet_join_enabled
+
+    if not infonet_fleet_join_enabled():
+        return {"ok": True, "accepted": 0, "duplicates": 0, "rejected": [], "skipped": "fleet_join_disabled"}
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 524_288:
+                return Response(content='{"ok":false,"detail":"Request body too large (max 512KB)"}',
+                    status_code=413, media_type="application/json")
+        except (ValueError, TypeError):
+            pass
+    from services.mesh.mesh_hashchain import infonet
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json")
+    body = json_mod.loads(body_bytes or b"{}")
+    events = body.get("events", [])
+    if not isinstance(events, list):
+        return {"ok": False, "detail": "events must be a list"}
+    if len(events) > 50:
+        return {"ok": False, "detail": "Too many events in one push (max 50)"}
+    if not events:
+        return {"ok": True, "accepted": 0, "duplicates": 0, "rejected": []}
+    result = infonet.ingest_events(events)
+    _hydrate_gate_store_from_chain(events)
+    _hydrate_dm_relay_from_chain(events)
+    return {"ok": True, **result}
+
+
+@router.post("/api/mesh/dm/replicate-envelope")
+@limiter.limit("60/minute")
+async def dm_replicate_envelope(request: Request):
+    """Accept a DM envelope replicated from a peer relay (cross-node mailbox).
+
+    Companion endpoint to ``DMRelay.replicate_to_peers`` (outbound, in
+    ``mesh_dm_relay.py``). The sender's relay POSTs an encrypted DM
+    envelope here after a successful local ``deposit``; this endpoint
+    re-enforces the per-(sender, recipient) anti-spam cap and stores
+    the envelope in the local mailbox if accepted.
+
+    The cap is the network rule: a hostile sender's relay can spool
+    extras locally, but every honest peer enforces the cap on inbound
+    replication. Recipient polling from any honest peer therefore
+    never sees more than ``MESH_DM_PENDING_PER_SENDER_LIMIT`` pending
+    from any one sender, no matter how many spam attempts were tried.
+
+    Same HMAC auth pattern as ``infonet_peer_push`` and ``gate_peer_push``.
+    """
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            # DM envelopes are bounded by MESH_DM_MAX_MSG_BYTES + envelope
+            # overhead; 64 KB is a generous ceiling.
+            if int(content_length) > 65_536:
+                return Response(
+                    content='{"ok":false,"detail":"Request body too large (max 64KB)"}',
+                    status_code=413, media_type="application/json",
+                )
+        except (ValueError, TypeError):
+            pass
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(
+            content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json",
+        )
+    try:
+        body = json_mod.loads(body_bytes or b"{}")
+    except (ValueError, TypeError):
+        return Response(
+            content='{"ok":false,"detail":"Invalid JSON body"}',
+            status_code=400, media_type="application/json",
+        )
+    envelope = body.get("envelope")
+    if not isinstance(envelope, dict):
+        return {"ok": False, "detail": "envelope must be an object"}
+
+    originating_peer = _peer_hmac_url_from_request(request) or ""
+
+    from services.mesh.mesh_dm_relay import dm_relay
+    result = dm_relay.accept_replica(
+        envelope=envelope,
+        originating_peer_url=originating_peer,
+    )
+    return result
+
+
+@router.post("/api/mesh/gate/peer-push")
+@limiter.limit("30/minute")
+async def gate_peer_push(request: Request):
+    """Accept pushed gate events from relay peers (private plane)."""
+    from services.mesh.mesh_fleet_defaults import infonet_fleet_join_enabled
+
+    if not infonet_fleet_join_enabled():
+        return {"ok": True, "accepted": 0, "duplicates": 0, "skipped": "fleet_join_disabled"}
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 524_288:
+                return Response(content='{"ok":false,"detail":"Request body too large"}',
+                    status_code=413, media_type="application/json")
+        except (ValueError, TypeError):
+            pass
+    from services.mesh.mesh_hashchain import gate_store
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json")
+    body = json_mod.loads(body_bytes or b"{}")
+    events = body.get("events", [])
+    if not isinstance(events, list):
+        return {"ok": False, "detail": "events must be a list"}
+    if len(events) > 50:
+        return {"ok": False, "detail": "Too many events (max 50)"}
+    if not events:
+        return {"ok": True, "accepted": 0, "duplicates": 0}
+    from services.mesh.mesh_hashchain import resolve_gate_wire_ref
+    # Sprint 3 / Rec #4: the gate_ref is HMACed with a key bound to the
+    # receiver's peer URL (the URL the push was delivered to). This is
+    # the same URL _verify_peer_push_hmac validated the X-Peer-HMAC
+    # header against, so we can trust it for ref resolution.
+    hop_peer_url = _peer_hmac_url_from_request(request)
+    grouped_events: dict[str, list] = {}
+    for evt in events:
+        evt_dict = evt if isinstance(evt, dict) else {}
+        payload = evt_dict.get("payload")
+        if not isinstance(payload, dict):
+            payload = {}
+        clean_event = {
+            "event_id": str(evt_dict.get("event_id", "") or ""),
+            "event_type": "gate_message",
+            "timestamp": evt_dict.get("timestamp", 0),
+            "node_id": str(evt_dict.get("node_id", "") or evt_dict.get("sender_id", "") or ""),
+            "sequence": evt_dict.get("sequence", 0),
+            "signature": str(evt_dict.get("signature", "") or ""),
+            "public_key": str(evt_dict.get("public_key", "") or ""),
+            "public_key_algo": str(evt_dict.get("public_key_algo", "") or ""),
+            "protocol_version": str(evt_dict.get("protocol_version", "") or ""),
+            "payload": {
+                "ciphertext": str(payload.get("ciphertext", "") or ""),
+                "format": str(payload.get("format", "") or ""),
+                "nonce": str(payload.get("nonce", "") or ""),
+                "sender_ref": str(payload.get("sender_ref", "") or ""),
+            },
+        }
+        epoch = _safe_int(payload.get("epoch", 0) or 0)
+        if epoch > 0:
+            clean_event["payload"]["epoch"] = epoch
+        envelope_hash_val = str(payload.get("envelope_hash", "") or "").strip()
+        gate_envelope_val = str(payload.get("gate_envelope", "") or "").strip()
+        reply_to_val = str(payload.get("reply_to", "") or "").strip()
+        if envelope_hash_val:
+            clean_event["payload"]["envelope_hash"] = envelope_hash_val
+        if gate_envelope_val:
+            clean_event["payload"]["gate_envelope"] = gate_envelope_val
+        if reply_to_val:
+            clean_event["payload"]["reply_to"] = reply_to_val
+        event_gate_id = str(payload.get("gate", "") or evt_dict.get("gate", "") or "").strip().lower()
+        if not event_gate_id:
+            event_gate_id = resolve_gate_wire_ref(
+                str(payload.get("gate_ref", "") or evt_dict.get("gate_ref", "") or ""),
+                clean_event,
+                peer_url=hop_peer_url,
+            )
+        if not event_gate_id:
+            return {"ok": False, "detail": "gate resolution failed"}
+        final_payload: dict[str, Any] = {
+            "gate": event_gate_id,
+            "ciphertext": clean_event["payload"]["ciphertext"],
+            "format": clean_event["payload"]["format"],
+            "nonce": clean_event["payload"]["nonce"],
+            "sender_ref": clean_event["payload"]["sender_ref"],
+        }
+        if epoch > 0:
+            final_payload["epoch"] = epoch
+        if clean_event["payload"].get("envelope_hash"):
+            final_payload["envelope_hash"] = clean_event["payload"]["envelope_hash"]
+        if clean_event["payload"].get("gate_envelope"):
+            final_payload["gate_envelope"] = clean_event["payload"]["gate_envelope"]
+        if clean_event["payload"].get("reply_to"):
+            final_payload["reply_to"] = clean_event["payload"]["reply_to"]
+        grouped_events.setdefault(event_gate_id, []).append({
+            "event_id": clean_event["event_id"],
+            "event_type": "gate_message",
+            "timestamp": clean_event["timestamp"],
+            "node_id": clean_event["node_id"],
+            "sequence": clean_event["sequence"],
+            "signature": clean_event["signature"],
+            "public_key": clean_event["public_key"],
+            "public_key_algo": clean_event["public_key_algo"],
+            "protocol_version": clean_event["protocol_version"],
+            "payload": final_payload,
+        })
+    accepted = 0
+    duplicates = 0
+    rejected = 0
+    for event_gate_id, items in grouped_events.items():
+        result = gate_store.ingest_peer_events(event_gate_id, items)
+        a = int(result.get("accepted", 0) or 0)
+        accepted += a
+        duplicates += int(result.get("duplicates", 0) or 0)
+        rejected += int(result.get("rejected", 0) or 0)
+    return {"ok": True, "accepted": accepted, "duplicates": duplicates, "rejected": rejected}
+
+
+@router.post("/api/mesh/gate/peer-pull")
+@limiter.limit("30/minute")
+async def gate_peer_pull(request: Request):
+    """Return gate events a peer is missing (HMAC-authenticated pull sync)."""
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 65_536:
+                return Response(content='{"ok":false,"detail":"Request body too large"}',
+                    status_code=413, media_type="application/json")
+        except (ValueError, TypeError):
+            pass
+    from services.mesh.mesh_hashchain import gate_store
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json")
+    body = json_mod.loads(body_bytes or b"{}")
+    gate_id = str(body.get("gate_id", "") or "").strip().lower()
+    after_count = _safe_int(body.get("after_count", 0) or 0)
+    if not gate_id:
+        gate_ids = gate_store.known_gate_ids()
+        gate_counts: dict[str, int] = {}
+        for gid in gate_ids:
+            with gate_store._lock:
+                gate_counts[gid] = len(gate_store._gates.get(gid, []))
+        return {"ok": True, "gates": gate_counts}
+    with gate_store._lock:
+        all_events = list(gate_store._gates.get(gate_id, []))
+    total = len(all_events)
+    if after_count >= total:
+        return {"ok": True, "events": [], "total": total, "gate_id": gate_id}
+    batch = all_events[after_count : after_count + _PEER_PUSH_BATCH_SIZE]
+    return {"ok": True, "events": batch, "total": total, "gate_id": gate_id}

backend/routers/osint.py

@@ -0,0 +1,151 @@
+"""Operator OSINT recon routes (server-side proxies, SSRF guarded)."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+from pydantic import BaseModel, Field
+
+from auth import require_local_operator
+from limiter import limiter
+from services.osint import lookups
+
+router = APIRouter(dependencies=[Depends(require_local_operator)])
+
+_ALLOWED_SCHEMAS = {
+    "Person",
+    "Organization",
+    "Company",
+    "Vessel",
+    "Airplane",
+    "LegalEntity",
+}
+
+
+class SweepScanRequest(BaseModel):
+    ip: str = Field(min_length=7, max_length=45)
+    cidr: int = Field(default=24, ge=24, le=32)
+
+
+def _bad_request(exc: ValueError) -> HTTPException:
+    return HTTPException(status_code=400, detail=str(exc))
+
+
+@router.get("/api/osint/ip")
+@limiter.limit("20/minute")
+async def osint_ip(request: Request, ip: str = Query(..., min_length=7, max_length=45)) -> dict:
+    try:
+        return lookups.lookup_ip(ip)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/dns")
+@limiter.limit("20/minute")
+async def osint_dns(request: Request, domain: str = Query(..., min_length=4, max_length=253)) -> dict:
+    try:
+        return lookups.lookup_dns(domain)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/whois")
+@limiter.limit("20/minute")
+async def osint_whois(request: Request, domain: str = Query(..., min_length=4, max_length=253)) -> dict:
+    try:
+        return lookups.lookup_whois(domain)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/certs")
+@limiter.limit("20/minute")
+async def osint_certs(request: Request, domain: str = Query(..., min_length=4, max_length=253)) -> dict:
+    try:
+        return lookups.lookup_certs(domain)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/threats")
+@limiter.limit("20/minute")
+async def osint_threats(request: Request, query: str | None = Query(default=None, max_length=253)) -> dict:
+    return lookups.lookup_threats(query)
+
+
+@router.get("/api/osint/bgp")
+@limiter.limit("20/minute")
+async def osint_bgp(request: Request, query: str = Query(..., min_length=2, max_length=64)) -> dict:
+    try:
+        return lookups.lookup_bgp(query)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/sanctions")
+@limiter.limit("20/minute")
+async def osint_sanctions(
+    request: Request,
+    query: str = Query(..., min_length=4, max_length=200),
+    schema: str | None = Query(default=None),
+    limit: int = Query(default=25, ge=1, le=100),
+) -> dict:
+    if schema and schema not in _ALLOWED_SCHEMAS:
+        raise HTTPException(status_code=400, detail=f"Invalid schema. Allowed: {', '.join(sorted(_ALLOWED_SCHEMAS))}")
+    return lookups.lookup_sanctions(query, schema=schema, limit=limit)
+
+
+@router.get("/api/osint/cve")
+@limiter.limit("30/minute")
+async def osint_cve(request: Request, cve: str = Query(..., min_length=10, max_length=32)) -> dict:
+    try:
+        return lookups.lookup_cve(cve)
+    except ValueError as exc:
+        raise HTTPException(status_code=404 if "not found" in str(exc).lower() else 400, detail=str(exc)) from exc
+
+
+@router.get("/api/osint/mac")
+@limiter.limit("20/minute")
+async def osint_mac(request: Request, mac: str = Query(..., min_length=5, max_length=32)) -> dict:
+    return lookups.lookup_mac(mac)
+
+
+@router.get("/api/osint/github")
+@limiter.limit("20/minute")
+async def osint_github(request: Request, username: str = Query(..., min_length=1, max_length=64)) -> dict:
+    try:
+        return lookups.lookup_github(username)
+    except ValueError as exc:
+        raise HTTPException(status_code=404, detail=str(exc)) from exc
+
+
+@router.get("/api/osint/leaks")
+@limiter.limit("10/minute")
+async def osint_leaks(request: Request, email: str = Query(..., min_length=5, max_length=254)) -> dict:
+    try:
+        return lookups.lookup_leaks(email)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/sweep")
+@limiter.limit("5/minute")
+async def osint_sweep_init(
+    request: Request,
+    ip: str = Query(..., min_length=7, max_length=45),
+    cidr: int = Query(default=24, ge=24, le=32),
+) -> dict:
+    try:
+        return lookups.sweep_init(ip, cidr)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.post("/api/osint/sweep/scan")
+@limiter.limit("3/minute")
+async def osint_sweep_scan(request: Request, payload: SweepScanRequest) -> dict:
+    try:
+        subnet = lookups.subnet_start_for(payload.ip, payload.cidr)
+        scan = lookups.sweep_scan(subnet, payload.cidr)
+        init = lookups.sweep_init(payload.ip, payload.cidr)
+        return {**init, **scan, "subnet": f"{subnet}/{payload.cidr}"}
+    except ValueError as exc:
+        raise _bad_request(exc) from exc

backend/routers/radio.py

@@ -0,0 +1,107 @@
+from fastapi import APIRouter, Request, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+
+router = APIRouter()
+
+
+@router.get("/api/radio/top")
+@limiter.limit("30/minute")
+async def get_top_radios(request: Request):
+    from services.radio_intercept import get_top_broadcastify_feeds
+    return get_top_broadcastify_feeds()
+
+
+@router.get("/api/radio/openmhz/systems")
+@limiter.limit("30/minute")
+async def api_get_openmhz_systems(request: Request):
+    from services.radio_intercept import get_openmhz_systems
+    return get_openmhz_systems()
+
+
+# Issue #213: rotating sys_name bypasses the 20s TTL cache and lets an
+# anonymous caller hammer api.openmhz.com through this proxy, risking an
+# IP-ban for the project. require_local_operator scopes this to the local
+# UI (which goes through the Next.js proxy with admin-key injection) and
+# scoped agent tokens.
+@router.get(
+    "/api/radio/openmhz/calls/{sys_name}",
+    dependencies=[Depends(require_local_operator)],
+)
+@limiter.limit("60/minute")
+async def api_get_openmhz_calls(request: Request, sys_name: str):
+    from services.radio_intercept import get_recent_openmhz_calls
+    return get_recent_openmhz_calls(sys_name)
+
+
+# Issue #214: this is a streaming bandwidth relay. An anonymous caller can
+# stream audio through the backend, saturating the operator's outbound
+# bandwidth. Scope to local operator; the legitimate browser UI still
+# works because relative /api/... paths go through the Next.js proxy
+# which injects the admin key automatically.
+@router.get(
+    "/api/radio/openmhz/audio",
+    dependencies=[Depends(require_local_operator)],
+)
+@limiter.limit("120/minute")
+async def api_get_openmhz_audio(request: Request, url: str = Query(..., min_length=10)):
+    from services.radio_intercept import openmhz_audio_response
+    return openmhz_audio_response(url)
+
+
+@router.get("/api/radio/nearest")
+@limiter.limit("60/minute")
+async def api_get_nearest_radio(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    from services.radio_intercept import find_nearest_openmhz_system
+    return find_nearest_openmhz_system(lat, lng)
+
+
+@router.get("/api/radio/nearest-list")
+@limiter.limit("60/minute")
+async def api_get_nearest_radios_list(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+    limit: int = Query(5, ge=1, le=20),
+):
+    from services.radio_intercept import find_nearest_openmhz_systems_list
+    return find_nearest_openmhz_systems_list(lat, lng, limit=limit)
+
+
+@router.get("/api/route/{callsign}")
+@limiter.limit("60/minute")
+async def get_flight_route(request: Request, callsign: str, lat: float = 0.0, lng: float = 0.0):
+    from services.network_utils import fetch_with_curl
+    r = fetch_with_curl(
+        "https://api.adsb.lol/api/0/routeset",
+        method="POST",
+        json_data={"planes": [{"callsign": callsign, "lat": lat, "lng": lng}]},
+        timeout=10,
+    )
+    if r and r.status_code == 200:
+        data = r.json()
+        route_list = []
+        if isinstance(data, dict):
+            route_list = data.get("value", [])
+        elif isinstance(data, list):
+            route_list = data
+
+        if route_list and len(route_list) > 0:
+            route = route_list[0]
+            airports = route.get("_airports", [])
+            if len(airports) >= 2:
+                orig = airports[0]
+                dest = airports[-1]
+                return {
+                    "orig_loc": [orig.get("lon", 0), orig.get("lat", 0)],
+                    "dest_loc": [dest.get("lon", 0), dest.get("lat", 0)],
+                    "origin_name": f"{orig.get('iata', '') or orig.get('icao', '')}: {orig.get('name', 'Unknown')}",
+                    "dest_name": f"{dest.get('iata', '') or dest.get('icao', '')}: {dest.get('name', 'Unknown')}",
+                }
+    return {}

backend/routers/road_corridors.py

@@ -0,0 +1,105 @@
+"""Road corridor Sentinel-2 freight trend endpoints (opt-in slow layer)."""
+
+from fastapi import APIRouter, HTTPException, Query, Request
+from pydantic import BaseModel, Field
+
+from limiter import limiter
+from services.road_corridor_sat.config import optional_deps_available, road_corridor_sat_enabled
+from services.road_corridor_sat.credentials import sentinel_credentials_configured
+from services.road_corridor_sat.jobs import enqueue_analyze, get_job, get_latest_job, job_to_dict
+from services.road_corridor_sat.presets import CORRIDOR_PRESETS, get_preset
+from services.road_corridor_sat.storage import build_trends_payload, preset_metadata
+
+router = APIRouter()
+
+
+def _status_payload() -> dict:
+    latest = get_latest_job()
+    return {
+        "enabled": road_corridor_sat_enabled(),
+        "deps_installed": optional_deps_available(),
+        "credentials_configured": sentinel_credentials_configured(),
+        "preset_count": len(CORRIDOR_PRESETS),
+        "attribution": "backend/third_party/drishx/NOTICE.md",
+        "active_job": job_to_dict(latest) if latest and latest.status in {"queued", "running"} else None,
+    }
+
+
+def _require_analyze_ready() -> None:
+    if not optional_deps_available():
+        raise HTTPException(
+            status_code=503,
+            detail="Install optional road-corridor dependencies (uv sync --extra road-corridor)",
+        )
+    if not sentinel_credentials_configured():
+        raise HTTPException(
+            status_code=503,
+            detail="Set SENTINEL_CLIENT_ID and SENTINEL_CLIENT_SECRET in Imagery settings",
+        )
+
+
+class AnalyzeRequest(BaseModel):
+    lat: float = Field(ge=-90, le=90)
+    lon: float = Field(ge=-180, le=180)
+    label: str | None = Field(default=None, max_length=120)
+
+
+@router.get("/api/road-corridors/status")
+@limiter.limit("60/minute")
+async def road_corridors_status(request: Request) -> dict:
+    return {"ok": True, **_status_payload()}
+
+
+@router.get("/api/road-corridors")
+@limiter.limit("60/minute")
+async def list_road_corridors(request: Request) -> dict:
+    return {
+        "ok": True,
+        "status": _status_payload(),
+        "presets": CORRIDOR_PRESETS,
+        "trends": build_trends_payload(),
+    }
+
+
+@router.post("/api/road-corridors/analyze")
+@limiter.limit("6/minute")
+async def analyze_road_corridor_here(request: Request, payload: AnalyzeRequest) -> dict:
+    """Start an on-demand Sentinel-2 corridor analysis at map center."""
+    _require_analyze_ready()
+    try:
+        job = enqueue_analyze(payload.lat, payload.lon, payload.label)
+    except RuntimeError as exc:
+        if str(exc) == "analysis_already_running":
+            active = get_latest_job()
+            raise HTTPException(
+                status_code=409,
+                detail="Analysis already in progress",
+                headers={"X-Job-Id": active.job_id if active else ""},
+            ) from exc
+        raise
+    return {"ok": True, **job_to_dict(job)}
+
+
+@router.get("/api/road-corridors/analyze/status")
+@limiter.limit("120/minute")
+async def analyze_road_corridor_status(
+    request: Request,
+    job_id: str | None = Query(default=None),
+) -> dict:
+    job = get_job(job_id) if job_id else get_latest_job()
+    if job is None:
+        return {"ok": True, "job": None}
+    return {"ok": True, "job": job_to_dict(job)}
+
+
+@router.get("/api/road-corridors/{preset_id}")
+@limiter.limit("60/minute")
+async def get_road_corridor(preset_id: str, request: Request) -> dict:
+    meta = preset_metadata(preset_id)
+    if meta is None:
+        raise HTTPException(status_code=404, detail="Unknown corridor preset")
+    preset = get_preset(preset_id)
+    if preset is None:
+        # Ad-hoc viewport runs are stored on disk but not in CORRIDOR_PRESETS.
+        return {"ok": True, "preset": None, "result": meta, "status": _status_payload()}
+    return {"ok": True, "preset": preset, "result": meta, "status": _status_payload()}

backend/routers/sar.py

@@ -0,0 +1,260 @@
+"""SAR (Synthetic Aperture Radar) layer endpoints.
+
+Exposes:
+  - GET  /api/sar/status            — feature gates + signup links for the UI
+  - GET  /api/sar/anomalies         — Mode B pre-processed anomalies
+  - GET  /api/sar/scenes            — Mode A scene catalog
+  - GET  /api/sar/coverage          — per-AOI coverage and next-pass hints
+  - GET  /api/sar/aois              — operator-defined AOIs
+  - POST /api/sar/aois              — create or replace an AOI
+  - DELETE /api/sar/aois/{aoi_id}   — remove an AOI
+  - GET  /api/sar/near              — anomalies within radius_km of (lat, lon)
+
+The /status endpoint is the load-bearing UX: when Mode B is disabled it
+returns the structured help payload from sar_config.products_fetch_status()
+so the frontend can render in-app links to the free signup pages instead of
+making the user hunt around.
+"""
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+from pydantic import BaseModel, Field
+
+from auth import require_local_operator
+from limiter import limiter
+from services.fetchers._store import get_latest_data_subset_refs
+from services.sar.sar_aoi import (
+    SarAoi,
+    add_aoi,
+    haversine_km,
+    load_aois,
+    remove_aoi,
+)
+from services.sar.sar_config import (
+    catalog_enabled,
+    clear_runtime_credentials,
+    openclaw_enabled,
+    products_fetch_enabled,
+    products_fetch_status,
+    require_private_tier_for_publish,
+    set_runtime_credentials,
+)
+
+router = APIRouter()
+
+
+# ---------------------------------------------------------------------------
+# Status — the in-app onboarding hook
+# ---------------------------------------------------------------------------
+@router.get("/api/sar/status")
+@limiter.limit("60/minute")
+async def sar_status(request: Request) -> dict:
+    """Layer status + signup links.
+
+    The frontend calls this whenever the SAR panel is opened.  When Mode B
+    is off, the response includes a step-by-step ``help`` block with the
+    free signup URLs so the user can enable everything without leaving the
+    app.
+    """
+    products_status = products_fetch_status()
+    return {
+        "ok": True,
+        "catalog": {
+            "mode": "A",
+            "enabled": catalog_enabled(),
+            "needs_account": False,
+            "description": "Free Sentinel-1 scene catalog from ASF Search.",
+        },
+        "products": {
+            "mode": "B",
+            **products_status,
+        },
+        "openclaw_enabled": openclaw_enabled(),
+        "require_private_tier": require_private_tier_for_publish(),
+    }
+
+
+# ---------------------------------------------------------------------------
+# Data feeds
+# ---------------------------------------------------------------------------
+@router.get("/api/sar/anomalies")
+@limiter.limit("60/minute")
+async def sar_anomalies(
+    request: Request,
+    kind: str = Query("", description="Optional anomaly kind filter"),
+    aoi_id: str = Query("", description="Optional AOI id filter"),
+    limit: int = Query(200, ge=1, le=1000),
+) -> dict:
+    """Return the latest cached SAR anomalies (Mode B)."""
+    snap = get_latest_data_subset_refs("sar_anomalies")
+    items = list(snap.get("sar_anomalies") or [])
+    if kind:
+        items = [a for a in items if a.get("kind") == kind]
+    if aoi_id:
+        aoi_id = aoi_id.strip().lower()
+        items = [a for a in items if (a.get("stack_id") or "").lower() == aoi_id]
+    items = items[:limit]
+    return {
+        "ok": True,
+        "count": len(items),
+        "anomalies": items,
+        "products_enabled": products_fetch_enabled(),
+    }
+
+
+@router.get("/api/sar/scenes")
+@limiter.limit("60/minute")
+async def sar_scenes(
+    request: Request,
+    aoi_id: str = Query(""),
+    limit: int = Query(200, ge=1, le=1000),
+) -> dict:
+    """Return the latest cached scene catalog (Mode A)."""
+    snap = get_latest_data_subset_refs("sar_scenes")
+    items = list(snap.get("sar_scenes") or [])
+    if aoi_id:
+        aoi_id = aoi_id.strip().lower()
+        items = [s for s in items if (s.get("aoi_id") or "").lower() == aoi_id]
+    items = items[:limit]
+    return {
+        "ok": True,
+        "count": len(items),
+        "scenes": items,
+        "catalog_enabled": catalog_enabled(),
+    }
+
+
+@router.get("/api/sar/coverage")
+@limiter.limit("60/minute")
+async def sar_coverage(request: Request) -> dict:
+    """Per-AOI coverage and rough next-pass estimate."""
+    snap = get_latest_data_subset_refs("sar_aoi_coverage")
+    return {
+        "ok": True,
+        "coverage": list(snap.get("sar_aoi_coverage") or []),
+    }
+
+
+@router.get("/api/sar/near")
+@limiter.limit("60/minute")
+async def sar_near(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lon: float = Query(..., ge=-180, le=180),
+    radius_km: float = Query(50, ge=1, le=2000),
+    kind: str = Query(""),
+    limit: int = Query(50, ge=1, le=500),
+) -> dict:
+    """Return anomalies whose center sits within ``radius_km`` of (lat, lon)."""
+    snap = get_latest_data_subset_refs("sar_anomalies")
+    items = list(snap.get("sar_anomalies") or [])
+    matches = []
+    for a in items:
+        try:
+            a_lat = float(a.get("lat", 0.0))
+            a_lon = float(a.get("lon", 0.0))
+        except (TypeError, ValueError):
+            continue
+        d = haversine_km(lat, lon, a_lat, a_lon)
+        if d > radius_km:
+            continue
+        if kind and a.get("kind") != kind:
+            continue
+        a = dict(a)
+        a["distance_km"] = round(d, 2)
+        matches.append(a)
+    matches.sort(key=lambda x: x.get("distance_km", 0))
+    return {
+        "ok": True,
+        "count": len(matches[:limit]),
+        "anomalies": matches[:limit],
+    }
+
+
+# ---------------------------------------------------------------------------
+# AOI CRUD
+# ---------------------------------------------------------------------------
+@router.get("/api/sar/aois")
+@limiter.limit("60/minute")
+async def sar_aoi_list(request: Request) -> dict:
+    return {
+        "ok": True,
+        "aois": [a.to_dict() for a in load_aois(force=True)],
+    }
+
+
+class AoiPayload(BaseModel):
+    id: str = Field(..., min_length=1, max_length=64)
+    name: str = Field(..., min_length=1, max_length=120)
+    description: str = Field("", max_length=400)
+    center_lat: float = Field(..., ge=-90, le=90)
+    center_lon: float = Field(..., ge=-180, le=180)
+    radius_km: float = Field(25.0, ge=1.0, le=500.0)
+    category: str = Field("watchlist", max_length=40)
+    polygon: list[list[float]] | None = None
+
+
+@router.post("/api/sar/aois", dependencies=[Depends(require_local_operator)])
+@limiter.limit("20/minute")
+async def sar_aoi_upsert(request: Request, payload: AoiPayload) -> dict:
+    aoi = SarAoi(
+        id=payload.id.strip().lower(),
+        name=payload.name.strip(),
+        description=payload.description.strip(),
+        center_lat=payload.center_lat,
+        center_lon=payload.center_lon,
+        radius_km=payload.radius_km,
+        polygon=payload.polygon,
+        category=(payload.category or "watchlist").strip().lower(),
+    )
+    add_aoi(aoi)
+    return {"ok": True, "aoi": aoi.to_dict()}
+
+
+@router.delete("/api/sar/aois/{aoi_id}", dependencies=[Depends(require_local_operator)])
+@limiter.limit("20/minute")
+async def sar_aoi_delete(request: Request, aoi_id: str) -> dict:
+    removed = remove_aoi(aoi_id)
+    if not removed:
+        raise HTTPException(status_code=404, detail="AOI not found")
+    return {"ok": True, "removed": aoi_id}
+
+
+# ---------------------------------------------------------------------------
+# Mode B enable / disable — one-click setup from the frontend
+# ---------------------------------------------------------------------------
+class ModeBEnablePayload(BaseModel):
+    earthdata_user: str = Field("", max_length=120)
+    earthdata_token: str = Field(..., min_length=8, max_length=2048)
+    copernicus_user: str = Field("", max_length=120)
+    copernicus_token: str = Field("", max_length=2048)
+
+
+@router.post("/api/sar/mode-b/enable", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def sar_mode_b_enable(request: Request, payload: ModeBEnablePayload) -> dict:
+    """Store Earthdata (and optional Copernicus) credentials and flip both
+    two-step opt-in flags.  Returns the fresh status payload so the UI can
+    immediately reflect the change.
+    """
+    set_runtime_credentials(
+        earthdata_user=payload.earthdata_user,
+        earthdata_token=payload.earthdata_token,
+        copernicus_user=payload.copernicus_user,
+        copernicus_token=payload.copernicus_token,
+        mode_b_opt_in=True,
+    )
+    return {
+        "ok": True,
+        "products": products_fetch_status(),
+    }
+
+
+@router.post("/api/sar/mode-b/disable", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def sar_mode_b_disable(request: Request) -> dict:
+    """Wipe runtime credentials and revert to Mode A only."""
+    clear_runtime_credentials()
+    return {
+        "ok": True,
+        "products": products_fetch_status(),
+    }

backend/routers/scm.py

@@ -0,0 +1,16 @@
+"""Supply-chain risk overlay."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, Request
+
+from auth import require_local_operator
+from limiter import limiter
+from services.scm.suppliers import build_scm_payload
+
+router = APIRouter()
+
+
+@router.get("/api/scm-suppliers")
+@limiter.limit("30/minute")
+async def scm_suppliers(request: Request, _: None = Depends(require_local_operator)) -> dict:
+    return build_scm_payload()

backend/routers/sigint.py

@@ -0,0 +1,67 @@
+from fastapi import APIRouter, Request, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+from services.data_fetcher import get_latest_data
+
+router = APIRouter()
+
+
+@router.get("/api/oracle/region-intel")
+@limiter.limit("30/minute")
+async def oracle_region_intel(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Get oracle intelligence summary for a geographic region."""
+    from services.oracle_service import get_region_oracle_intel
+    news_items = get_latest_data().get("news", [])
+    return get_region_oracle_intel(lat, lng, news_items)
+
+
+@router.get("/api/thermal/verify", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def thermal_verify(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+    radius_km: float = Query(10, ge=1, le=100),
+):
+    """On-demand thermal anomaly verification using Sentinel-2 SWIR bands."""
+    from services.thermal_sentinel import search_thermal_anomaly
+    result = search_thermal_anomaly(lat, lng, radius_km)
+    return result
+
+
+@router.post("/api/sigint/transmit", dependencies=[Depends(require_local_operator)])
+@limiter.limit("5/minute")
+async def sigint_transmit(request: Request):
+    """Send an APRS-IS message to a specific callsign. Requires ham radio credentials."""
+    from services.wormhole_supervisor import get_transport_tier
+    tier = get_transport_tier()
+    if str(tier or "").startswith("private_"):
+        return {"ok": False, "detail": "APRS transmit blocked in private transport mode"}
+    body = await request.json()
+    callsign = body.get("callsign", "")
+    passcode = body.get("passcode", "")
+    target = body.get("target", "")
+    message = body.get("message", "")
+    if not all([callsign, passcode, target, message]):
+        return {"ok": False, "detail": "Missing required fields: callsign, passcode, target, message"}
+    from services.sigint_bridge import send_aprs_message
+    return send_aprs_message(callsign, passcode, target, message)
+
+
+@router.get("/api/sigint/nearest-sdr")
+@limiter.limit("30/minute")
+async def nearest_sdr(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Find the nearest KiwiSDR receivers to a given coordinate."""
+    from services.sigint_bridge import find_nearest_kiwisdr
+    kiwisdr_data = get_latest_data().get("kiwisdr", [])
+    return find_nearest_kiwisdr(lat, lng, kiwisdr_data)

backend/routers/tools.py

@@ -0,0 +1,444 @@
+import asyncio
+import logging
+import math
+from typing import Any
+from fastapi import APIRouter, Request, Query, Depends, HTTPException, Response
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _safe_float(val, default=0.0):
+    try:
+        parsed = float(val)
+        if not math.isfinite(parsed):
+            return default
+        return parsed
+    except (TypeError, ValueError):
+        return default
+
+
+class ShodanSearchRequest(BaseModel):
+    query: str
+    page: int = 1
+    facets: list[str] = []
+
+
+class ShodanCountRequest(BaseModel):
+    query: str
+    facets: list[str] = []
+
+
+class ShodanHostRequest(BaseModel):
+    ip: str
+    history: bool = False
+
+
+@router.get("/api/region-dossier")
+@limiter.limit("30/minute")
+def api_region_dossier(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Sync def so FastAPI runs it in a threadpool — prevents blocking the event loop."""
+    from services.region_dossier import get_region_dossier
+    return get_region_dossier(lat, lng)
+
+
+@router.get("/api/geocode/search")
+@limiter.limit("30/minute")
+async def api_geocode_search(
+    request: Request,
+    q: str = "",
+    limit: int = 5,
+    local_only: bool = False,
+):
+    from services.geocode import search_geocode
+    if not q or len(q.strip()) < 2:
+        return {"results": [], "query": q, "count": 0}
+    results = await asyncio.to_thread(search_geocode, q, limit, local_only)
+    return {"results": results, "query": q, "count": len(results)}
+
+
+@router.get("/api/geocode/reverse")
+@limiter.limit("60/minute")
+async def api_geocode_reverse(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+    local_only: bool = False,
+):
+    from services.geocode import reverse_geocode
+    return await asyncio.to_thread(reverse_geocode, lat, lng, local_only)
+
+
+# ── Wikimedia proxy (#360) — browser calls these instead of wikipedia.org ───
+@router.get("/api/wikipedia/summary")
+@limiter.limit("60/minute")
+def api_wikipedia_summary(
+    request: Request,
+    title: str = Query(..., min_length=1, max_length=256),
+):
+    """Proxy Wikipedia REST summaries through the self-hosted backend."""
+    from services.region_dossier import fetch_wikipedia_page_summary
+
+    summary = fetch_wikipedia_page_summary(title)
+    if summary is None:
+        return JSONResponse(status_code=404, content={"detail": "not_found"})
+    return summary
+
+
+class WikidataSparqlRequest(BaseModel):
+    query: str
+
+
+@router.post("/api/wikidata/sparql")
+@limiter.limit("30/minute")
+def api_wikidata_sparql(request: Request, body: WikidataSparqlRequest):
+    """Proxy Wikidata SPARQL so the browser never contacts query.wikidata.org."""
+    from services.region_dossier import fetch_wikidata_sparql_bindings
+
+    q = (body.query or "").strip()
+    if len(q) > 12_000:
+        raise HTTPException(400, "SPARQL query too large")
+    bindings = fetch_wikidata_sparql_bindings(q)
+    return {"bindings": bindings}
+
+
+# ── Sentinel proxy routes (Issue #299/#300/#301, reported by tg12) ──────────
+# These three endpoints relay external Sentinel / Planetary Computer
+# requests through the backend to avoid browser CORS blocks. They are
+# operator-only helpers — they MUST NOT be callable by anonymous remote
+# users, because:
+#
+#   * /api/sentinel/token  — caller supplies their own Sentinel client_id +
+#     client_secret. Without operator gating, the backend becomes a free
+#     anonymous OAuth-mint relay for any Copernicus account.
+#   * /api/sentinel/tile   — same shape as the token route but for tile
+#     imagery. Without gating, the backend acts as an anonymous quota and
+#     bandwidth relay for Sentinel Hub Process API calls.
+#   * /api/sentinel2/search — hits the Planetary Computer STAC search API
+#     and falls back to Esri imagery. No caller credentials are involved,
+#     but the route is still an anonymous external-search relay. We gate
+#     it the same way for consistency with the rest of the operator-only
+#     helper surface.
+#
+# Gating is via require_local_operator (loopback / bridge / admin key),
+# matching the same allowlist already used by /api/region-dossier and
+# the other operator helpers further up this file. Single-operator nodes
+# see no behavior change — their dashboard already lives on loopback or
+# the trusted Docker bridge, so it still resolves.
+@router.get("/api/sentinel2/search", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+def api_sentinel2_search(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Search for latest Sentinel-2 imagery at a point. Sync for threadpool execution."""
+    from services.sentinel_search import search_sentinel2_scene
+    return search_sentinel2_scene(lat, lng)
+
+
+# Issue #298 (tg12): Sentinel credentials moved server-side
+# ---------------------------------------------------------------------------
+# Previously the frontend kept Copernicus CDSE client_id + client_secret in
+# browser localStorage / sessionStorage and forwarded them on every tile
+# request through this proxy. That exposed real third-party credentials to
+# any same-origin script (XSS, malicious browser extension, dev-tools HAR
+# export).
+#
+# Resolution order (first match wins):
+#   1. Request body — kept for back-compat. A small number of legacy
+#      operator setups may still post credentials; we don't break them.
+#   2. Backend .env — SENTINEL_CLIENT_ID / SENTINEL_CLIENT_SECRET, managed
+#      through the existing /api/settings/api-keys flow (admin-gated).
+#
+# The frontend in ``sentinelHub.ts`` no longer reads browser storage and no
+# longer forwards credentials — every dashboard request now lands in (2).
+# The require_local_operator gate (added in #303/PR #303) stays — both layers
+# are independent: the gate blocks anonymous callers, the env fallback lets
+# legitimate (gated) callers omit credentials from the body.
+# ---------------------------------------------------------------------------
+def _resolve_sentinel_credentials(body_id: str, body_secret: str) -> tuple[str, str]:
+    """Return (client_id, client_secret) using body values when present,
+    otherwise falling back to backend .env. Empty strings if neither is set."""
+    import os as _os
+    cid = (body_id or "").strip() or (_os.environ.get("SENTINEL_CLIENT_ID", "") or "").strip()
+    csec = (body_secret or "").strip() or (_os.environ.get("SENTINEL_CLIENT_SECRET", "") or "").strip()
+    return cid, csec
+
+
+@router.post("/api/sentinel/token", dependencies=[Depends(require_local_operator)])
+@limiter.limit("60/minute")
+async def api_sentinel_token(request: Request):
+    """Proxy Copernicus CDSE OAuth2 token request (avoids browser CORS block).
+
+    Credentials are resolved by ``_resolve_sentinel_credentials`` — body
+    fields are honored for back-compat, otherwise the backend .env values
+    populated through ``/api/settings/api-keys`` are used.
+    """
+    import requests as req
+    body = await request.body()
+    from urllib.parse import parse_qs
+    params = parse_qs(body.decode("utf-8"))
+    body_id = params.get("client_id", [""])[0]
+    body_secret = params.get("client_secret", [""])[0]
+    client_id, client_secret = _resolve_sentinel_credentials(body_id, body_secret)
+    if not client_id or not client_secret:
+        # Friendly, non-hostile error — points the operator at the place
+        # they configure other API keys instead of just saying "required".
+        raise HTTPException(
+            400,
+            "Sentinel client_id/client_secret are not configured. "
+            "Set SENTINEL_CLIENT_ID and SENTINEL_CLIENT_SECRET in the "
+            "API Keys panel (Settings → API Keys) or your backend .env.",
+        )
+    token_url = "https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token"
+    try:
+        resp = await asyncio.to_thread(req.post, token_url,
+            data={"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret},
+            timeout=15)
+        return Response(content=resp.content, status_code=resp.status_code, media_type="application/json")
+    except Exception:
+        logger.exception("Token request failed")
+        raise HTTPException(502, "Token request failed")
+
+
+# Cache key is an HMAC of (client_id, client_secret) — a caller cannot hit
+# this cache without knowing the same secret that originally populated it.
+# Without this binding, the lookup only checked client_id, so anyone who
+# knew a valid client_id could reuse another caller's cached token (and
+# burn their Copernicus quota / access tiles on their account).
+_sh_token_cache: dict = {"token": None, "expiry": 0, "credential_fp": ""}
+
+
+def _credential_fingerprint(client_id: str, client_secret: str) -> str:
+    """Return a stable, secret-binding fingerprint for the Sentinel cache key.
+
+    Uses HMAC-SHA256 so the raw secret is never stored in process memory as
+    a cache key. The HMAC key is a per-process random value, which means the
+    fingerprint cannot be precomputed across restarts (additional defense
+    against an attacker who learned a valid client_id but not the secret).
+    """
+    import hashlib
+    import hmac
+
+    return hmac.new(
+        _SH_TOKEN_CACHE_HMAC_KEY,
+        f"{client_id}\x00{client_secret}".encode("utf-8"),
+        hashlib.sha256,
+    ).hexdigest()
+
+
+# Per-process random HMAC key. Regenerated on each backend startup so cached
+# fingerprints don't survive restarts.
+import os as _os
+_SH_TOKEN_CACHE_HMAC_KEY = _os.urandom(32)
+
+
+@router.post("/api/sentinel/tile", dependencies=[Depends(require_local_operator)])
+@limiter.limit("300/minute")
+async def api_sentinel_tile(request: Request):
+    """Proxy Sentinel Hub Process API tile request (avoids CORS block)."""
+    import requests as req
+    import time as _time
+    try:
+        body = await request.json()
+    except Exception:
+        return JSONResponse(status_code=422, content={"ok": False, "detail": "invalid JSON body"})
+
+    # Issue #298: same resolution order as /api/sentinel/token — body
+    # values for back-compat, otherwise backend .env.
+    body_id = body.get("client_id", "")
+    body_secret = body.get("client_secret", "")
+    client_id, client_secret = _resolve_sentinel_credentials(body_id, body_secret)
+    preset = body.get("preset", "TRUE-COLOR")
+    date_str = body.get("date", "")
+    z = body.get("z", 0)
+    x = body.get("x", 0)
+    y = body.get("y", 0)
+
+    if not client_id or not client_secret or not date_str:
+        # Distinguish "no creds" from "no date" so the operator knows
+        # what to fix. Same friendly pointer as the /token route.
+        if not client_id or not client_secret:
+            raise HTTPException(
+                400,
+                "Sentinel client_id/client_secret are not configured. "
+                "Set SENTINEL_CLIENT_ID and SENTINEL_CLIENT_SECRET in the "
+                "API Keys panel (Settings → API Keys) or your backend .env.",
+            )
+        raise HTTPException(400, "date required")
+
+    now = _time.time()
+    credential_fp = _credential_fingerprint(client_id, client_secret)
+    if (_sh_token_cache["token"]
+            and _sh_token_cache["credential_fp"] == credential_fp
+            and now < _sh_token_cache["expiry"] - 30):
+        token = _sh_token_cache["token"]
+    else:
+        token_url = "https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token"
+        try:
+            tresp = await asyncio.to_thread(req.post, token_url,
+                data={"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret},
+                timeout=15)
+            if tresp.status_code != 200:
+                raise HTTPException(401, f"Token auth failed: {tresp.text[:200]}")
+            tdata = tresp.json()
+            token = tdata["access_token"]
+            _sh_token_cache["token"] = token
+            _sh_token_cache["expiry"] = now + tdata.get("expires_in", 300)
+            _sh_token_cache["credential_fp"] = credential_fp
+        except HTTPException:
+            raise
+        except Exception:
+            logger.exception("Token request failed")
+            raise HTTPException(502, "Token request failed")
+
+    half = 20037508.342789244
+    tile_size = (2 * half) / math.pow(2, z)
+    min_x = -half + x * tile_size
+    max_x = min_x + tile_size
+    max_y = half - y * tile_size
+    min_y = max_y - tile_size
+    bbox = [min_x, min_y, max_x, max_y]
+
+    evalscripts = {
+        "TRUE-COLOR": '//VERSION=3\nfunction setup(){return{input:["B04","B03","B02"],output:{bands:3}};}\nfunction evaluatePixel(s){return[2.5*s.B04,2.5*s.B03,2.5*s.B02];}',
+        "FALSE-COLOR": '//VERSION=3\nfunction setup(){return{input:["B08","B04","B03"],output:{bands:3}};}\nfunction evaluatePixel(s){return[2.5*s.B08,2.5*s.B04,2.5*s.B03];}',
+        "NDVI": '//VERSION=3\nfunction setup(){return{input:["B04","B08"],output:{bands:3}};}\nfunction evaluatePixel(s){var n=(s.B08-s.B04)/(s.B08+s.B04);if(n<-0.2)return[0.05,0.05,0.05];if(n<0)return[0.75,0.75,0.75];if(n<0.1)return[0.86,0.86,0.86];if(n<0.2)return[0.92,0.84,0.68];if(n<0.3)return[0.77,0.88,0.55];if(n<0.4)return[0.56,0.80,0.32];if(n<0.5)return[0.35,0.72,0.18];if(n<0.6)return[0.20,0.60,0.08];if(n<0.7)return[0.10,0.48,0.04];return[0.0,0.36,0.0];}',
+        "MOISTURE-INDEX": '//VERSION=3\nfunction setup(){return{input:["B8A","B11"],output:{bands:3}};}\nfunction evaluatePixel(s){var m=(s.B8A-s.B11)/(s.B8A+s.B11);var r=Math.max(0,Math.min(1,1.5-3*m));var g=Math.max(0,Math.min(1,m<0?1.5+3*m:1.5-3*m));var b=Math.max(0,Math.min(1,1.5+3*(m-0.5)));return[r,g,b];}',
+    }
+    evalscript = evalscripts.get(preset, evalscripts["TRUE-COLOR"])
+
+    from datetime import datetime as _dt, timedelta as _td
+    try:
+        end_date = _dt.strptime(date_str, "%Y-%m-%d")
+    except ValueError:
+        end_date = _dt.utcnow()
+
+    if z <= 6:
+        lookback_days = 30
+    elif z <= 9:
+        lookback_days = 14
+    elif z <= 11:
+        lookback_days = 7
+    else:
+        lookback_days = 5
+
+    start_date = end_date - _td(days=lookback_days)
+
+    process_body = {
+        "input": {
+            "bounds": {"bbox": bbox, "properties": {"crs": "http://www.opengis.net/def/crs/EPSG/0/3857"}},
+            "data": [{"type": "sentinel-2-l2a", "dataFilter": {
+                "timeRange": {
+                    "from": start_date.strftime("%Y-%m-%dT00:00:00Z"),
+                    "to": end_date.strftime("%Y-%m-%dT23:59:59Z"),
+                },
+                "maxCloudCoverage": 30, "mosaickingOrder": "leastCC",
+            }}],
+        },
+        "output": {"width": 256, "height": 256,
+            "responses": [{"identifier": "default", "format": {"type": "image/png"}}]},
+        "evalscript": evalscript,
+    }
+    try:
+        resp = await asyncio.to_thread(req.post,
+            "https://sh.dataspace.copernicus.eu/api/v1/process",
+            json=process_body,
+            headers={"Authorization": f"Bearer {token}", "Accept": "image/png"},
+            timeout=30)
+        return Response(content=resp.content, status_code=resp.status_code,
+            media_type=resp.headers.get("content-type", "image/png"))
+    except Exception:
+        logger.exception("Process API failed")
+        raise HTTPException(502, "Process API failed")
+
+
+@router.get("/api/tools/shodan/status", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_shodan_status(request: Request):
+    from services.shodan_connector import get_shodan_connector_status
+    return get_shodan_connector_status()
+
+
+@router.post("/api/tools/shodan/search", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_shodan_search(request: Request, body: ShodanSearchRequest):
+    from services.shodan_connector import ShodanConnectorError, search_shodan
+    try:
+        return search_shodan(body.query, page=body.page, facets=body.facets)
+    except ShodanConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/shodan/count", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_shodan_count(request: Request, body: ShodanCountRequest):
+    from services.shodan_connector import ShodanConnectorError, count_shodan
+    try:
+        return count_shodan(body.query, facets=body.facets)
+    except ShodanConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/shodan/host", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_shodan_host(request: Request, body: ShodanHostRequest):
+    from services.shodan_connector import ShodanConnectorError, lookup_shodan_host
+    try:
+        return lookup_shodan_host(body.ip, history=body.history)
+    except ShodanConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.get("/api/tools/uw/status", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_uw_status(request: Request):
+    from services.unusual_whales_connector import get_uw_status
+    return get_uw_status()
+
+
+@router.post("/api/tools/uw/congress", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_uw_congress(request: Request):
+    from services.unusual_whales_connector import FinnhubConnectorError, fetch_congress_trades
+    try:
+        return fetch_congress_trades()
+    except FinnhubConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/uw/darkpool", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_uw_darkpool(request: Request):
+    from services.unusual_whales_connector import FinnhubConnectorError, fetch_insider_transactions
+    try:
+        return fetch_insider_transactions()
+    except FinnhubConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/uw/flow", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_uw_flow(request: Request):
+    from services.unusual_whales_connector import FinnhubConnectorError, fetch_defense_quotes
+    try:
+        return fetch_defense_quotes()
+    except FinnhubConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc

backend/scripts/convert_power_plants.py

@@ -20,7 +20,17 @@ OUT_PATH = Path(__file__).parent.parent / "data" / "power_plants.json"
 
 def main() -> None:
     print(f"Downloading WRI Global Power Plant Database from GitHub...")
-    req = urllib.request.Request(CSV_URL, headers={"User-Agent": "ShadowBroker-OSINT/1.0"})
+    # Round 7a: release-time data refresher. Uses the per-operator UA if
+    # available, otherwise a release-script-specific identifier. This
+    # script is run by the maintainer at release time, NOT at runtime,
+    # so an aggregate UA is acceptable; we still use the helper so the
+    # behavior matches the rest of the project.
+    try:
+        from services.network_utils import outbound_user_agent
+        ua = outbound_user_agent("release-script-power-plants")
+    except Exception:
+        ua = "operator-release-script (purpose: power-plants)"
+    req = urllib.request.Request(CSV_URL, headers={"User-Agent": ua})
     with urllib.request.urlopen(req, timeout=60) as resp:
         raw = resp.read().decode("utf-8")
 

backend/scripts/release_helper.py

@@ -1,7 +1,9 @@
 import argparse
 import hashlib
 import json
+import os
 import sys
+from datetime import datetime, timezone
 from pathlib import Path
 
 
@@ -56,6 +58,72 @@ def sha256_file(path: Path) -> str:
     return digest.hexdigest().lower()
 
 
+def _default_generated_at() -> str:
+    return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def build_release_attestation(
+    *,
+    suite_green: bool,
+    suite_name: str = "dm_relay_security",
+    detail: str = "",
+    report: str = "",
+    command: str = "",
+    commit: str = "",
+    generated_at: str = "",
+    threat_model_reference: str = "docs/mesh/threat-model.md",
+    workflow: str = "",
+    run_id: str = "",
+    run_attempt: str = "",
+    ref: str = "",
+) -> dict:
+    normalized_generated_at = str(generated_at or "").strip() or _default_generated_at()
+    normalized_commit = str(commit or "").strip() or os.environ.get("GITHUB_SHA", "").strip()
+    normalized_workflow = str(workflow or "").strip() or os.environ.get("GITHUB_WORKFLOW", "").strip()
+    normalized_run_id = str(run_id or "").strip() or os.environ.get("GITHUB_RUN_ID", "").strip()
+    normalized_run_attempt = str(run_attempt or "").strip() or os.environ.get("GITHUB_RUN_ATTEMPT", "").strip()
+    normalized_ref = str(ref or "").strip() or os.environ.get("GITHUB_REF", "").strip()
+    normalized_suite_name = str(suite_name or "").strip() or "dm_relay_security"
+    normalized_report = str(report or "").strip()
+    normalized_command = str(command or "").strip()
+    normalized_detail = str(detail or "").strip() or (
+        "CI attestation confirms the DM relay security suite is green."
+        if suite_green
+        else "CI attestation recorded a failing DM relay security suite run."
+    )
+    payload = {
+        "generated_at": normalized_generated_at,
+        "commit": normalized_commit,
+        "threat_model_reference": str(threat_model_reference or "").strip()
+        or "docs/mesh/threat-model.md",
+        "dm_relay_security_suite": {
+            "name": normalized_suite_name,
+            "green": bool(suite_green),
+            "detail": normalized_detail,
+            "report": normalized_report,
+        },
+    }
+    if normalized_command:
+        payload["dm_relay_security_suite"]["command"] = normalized_command
+    ci = {
+        "workflow": normalized_workflow,
+        "run_id": normalized_run_id,
+        "run_attempt": normalized_run_attempt,
+        "ref": normalized_ref,
+    }
+    if any(ci.values()):
+        payload["ci"] = ci
+    return payload
+
+
+def write_release_attestation(output_path: Path | str, **kwargs) -> dict:
+    path = Path(output_path).resolve()
+    payload = build_release_attestation(**kwargs)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+    return payload
+
+
 def cmd_show(_args: argparse.Namespace) -> int:
     version = current_version()
     if not version:
@@ -99,9 +167,38 @@ def cmd_hash(args: argparse.Namespace) -> int:
     print("")
     print("Updater pin:")
     print(f"MESH_UPDATE_SHA256={digest}")
+    print("")
+    print("Release checklist:")
+    print("  - add this digest to SHA256SUMS.txt for the GitHub release")
+    print("  - add/update backend/data/release_digests.json for bundled updater verification")
+    print("  - keep MESH_UPDATE_SHA256 available as the operator override path")
     return 0 if asset_matches else 2
 
 
+def cmd_write_attestation(args: argparse.Namespace) -> int:
+    suite_green = bool(args.suite_green)
+    payload = write_release_attestation(
+        args.output_path,
+        suite_green=suite_green,
+        suite_name=args.suite_name,
+        detail=args.detail,
+        report=args.report,
+        command=args.command,
+        commit=args.commit,
+        generated_at=args.generated_at,
+        threat_model_reference=args.threat_model_reference,
+        workflow=args.workflow,
+        run_id=args.run_id,
+        run_attempt=args.run_attempt,
+        ref=args.ref,
+    )
+    output_path = Path(args.output_path).resolve()
+    print(f"Wrote release attestation: {output_path}")
+    print(f"DM relay security suite : {'green' if suite_green else 'red'}")
+    print(f"Commit                  : {payload.get('commit', '')}")
+    return 0
+
+
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         description="Helper for ShadowBroker release version/tag/asset consistency."
@@ -112,7 +209,7 @@ def build_parser() -> argparse.ArgumentParser:
     show_parser.set_defaults(func=cmd_show)
 
     set_version_parser = subparsers.add_parser("set-version", help="Update frontend/package.json version")
-    set_version_parser.add_argument("version", help="Version like 0.9.6")
+    set_version_parser.add_argument("version", help="Version like 0.9.7")
     set_version_parser.set_defaults(func=cmd_set_version)
 
     hash_parser = subparsers.add_parser(
@@ -121,10 +218,83 @@ def build_parser() -> argparse.ArgumentParser:
     hash_parser.add_argument("zip_path", help="Path to the release ZIP")
     hash_parser.add_argument(
         "--version",
-        help="Release version like 0.9.6. Defaults to frontend/package.json version.",
+        help="Release version like 0.9.7. Defaults to frontend/package.json version.",
     )
     hash_parser.set_defaults(func=cmd_hash)
 
+    attestation_parser = subparsers.add_parser(
+        "write-attestation",
+        help="Write a structured Sprint 8 release attestation JSON file",
+    )
+    attestation_parser.add_argument("output_path", help="Where to write the attestation JSON")
+    suite_group = attestation_parser.add_mutually_exclusive_group(required=True)
+    suite_group.add_argument(
+        "--suite-green",
+        action="store_true",
+        help="Mark the DM relay security suite as green",
+    )
+    suite_group.add_argument(
+        "--suite-red",
+        action="store_true",
+        help="Mark the DM relay security suite as failing",
+    )
+    attestation_parser.add_argument(
+        "--suite-name",
+        default="dm_relay_security",
+        help="Suite name to record in the attestation",
+    )
+    attestation_parser.add_argument(
+        "--detail",
+        default="",
+        help="Human-readable suite detail. Defaults to a CI-generated message.",
+    )
+    attestation_parser.add_argument(
+        "--report",
+        default="",
+        help="Path to the suite report or artifact reference to embed in the attestation.",
+    )
+    attestation_parser.add_argument(
+        "--command",
+        default="",
+        help="Exact suite command used to generate the attestation.",
+    )
+    attestation_parser.add_argument(
+        "--commit",
+        default="",
+        help="Commit SHA. Defaults to GITHUB_SHA when available.",
+    )
+    attestation_parser.add_argument(
+        "--generated-at",
+        default="",
+        help="UTC timestamp for the attestation. Defaults to current UTC time.",
+    )
+    attestation_parser.add_argument(
+        "--threat-model-reference",
+        default="docs/mesh/threat-model.md",
+        help="Threat model reference to embed in the attestation.",
+    )
+    attestation_parser.add_argument(
+        "--workflow",
+        default="",
+        help="Workflow name. Defaults to GITHUB_WORKFLOW when available.",
+    )
+    attestation_parser.add_argument(
+        "--run-id",
+        default="",
+        help="Workflow run ID. Defaults to GITHUB_RUN_ID when available.",
+    )
+    attestation_parser.add_argument(
+        "--run-attempt",
+        default="",
+        help="Workflow run attempt. Defaults to GITHUB_RUN_ATTEMPT when available.",
+    )
+    attestation_parser.add_argument(
+        "--ref",
+        default="",
+        help="Git ref. Defaults to GITHUB_REF when available.",
+    )
+    attestation_parser.set_defaults(func=cmd_write_attestation)
+
     return parser
 
 

backend/scripts/rotate_secure_storage_secret.py

@@ -0,0 +1,75 @@
+"""Rotate the MESH_SECURE_STORAGE_SECRET used to protect key envelopes at rest.
+
+Usage — stop the backend first, then run:
+
+    MESH_OLD_STORAGE_SECRET=<current>  \\
+    MESH_NEW_STORAGE_SECRET=<new>      \\
+    python -m scripts.rotate_secure_storage_secret
+
+Dry-run mode (validates old secret without writing anything):
+
+    MESH_OLD_STORAGE_SECRET=<current>  \\
+    MESH_NEW_STORAGE_SECRET=<new>      \\
+    python -m scripts.rotate_secure_storage_secret --dry-run
+
+Or, for Docker deployments:
+
+    docker exec -e MESH_OLD_STORAGE_SECRET=<current> \\
+                -e MESH_NEW_STORAGE_SECRET=<new>      \\
+                <container> python -m scripts.rotate_secure_storage_secret
+
+After successful rotation, update your .env (or Docker secret file) to set
+MESH_SECURE_STORAGE_SECRET to the new value, then restart the backend.
+
+The script fails closed: if the old secret cannot unwrap any existing envelope,
+nothing is written. Non-passphrase envelopes (DPAPI, raw) are skipped with a
+warning.
+
+Before rewriting, .bak copies of every envelope are created so a mid-rotation
+crash leaves recoverable backups on disk.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+
+def main() -> None:
+    dry_run = "--dry-run" in sys.argv
+
+    old_secret = os.environ.get("MESH_OLD_STORAGE_SECRET", "").strip()
+    new_secret = os.environ.get("MESH_NEW_STORAGE_SECRET", "").strip()
+
+    if not old_secret:
+        print("ERROR: MESH_OLD_STORAGE_SECRET environment variable is required.", file=sys.stderr)
+        sys.exit(1)
+    if not new_secret:
+        print("ERROR: MESH_NEW_STORAGE_SECRET environment variable is required.", file=sys.stderr)
+        sys.exit(1)
+
+    from services.mesh.mesh_secure_storage import SecureStorageError, rotate_storage_secret
+
+    try:
+        result = rotate_storage_secret(old_secret, new_secret, dry_run=dry_run)
+    except SecureStorageError as exc:
+        print(f"ROTATION FAILED: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+    print(json.dumps(result, indent=2))
+    if dry_run:
+        print(
+            "\nDry run complete. No files were modified. Run again without --dry-run to perform the rotation.",
+            file=sys.stderr,
+        )
+    else:
+        print(
+            "\nRotation complete. Update MESH_SECURE_STORAGE_SECRET to the new value and restart the backend."
+            "\nBackup files (.bak) were created alongside each rotated envelope.",
+            file=sys.stderr,
+        )
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/scan-secrets.sh

@@ -92,18 +92,37 @@ SECRET_REGEX+='pypi-[0-9a-zA-Z-]{50,}'                        # PyPI token
 TEXT_FILES=$(grep -ivE '\.(png|jpg|jpeg|gif|ico|svg|woff2?|ttf|eot|pbf|zip|tar|gz|db|sqlite|xlsx|pdf|mp[34]|wav|ogg|webm|webp|avif)$' "$FILELIST" | grep -v 'scan-secrets\.sh$' || true)
 
 if [[ -n "$TEXT_FILES" ]]; then
+    # Known-public exclusions: lines matching `<host-or-ip> ssh-<algo> <key>`
+    # are SSH known_hosts entries — the host's PUBLIC fingerprint, which is
+    # by definition safe to commit (the whole point of pinning known_hosts
+    # is to publish the fingerprint widely so MITM is detectable). Filter
+    # these out before flagging the file.
+    KNOWN_HOSTS_LINE='^[[:space:]]*[a-zA-Z0-9._:,*-]+([[:space:]]+[a-zA-Z0-9._:,*-]+)?[[:space:]]+(ssh-rsa|ssh-ed25519|ssh-dss|ecdsa-sha2-nistp256|ecdsa-sha2-nistp384|ecdsa-sha2-nistp521)[[:space:]]+AAAA'
+
     # Use grep with file list, skip missing/binary, limit output
     CONTENT_HITS=$(echo "$TEXT_FILES" | xargs grep -lE "$SECRET_REGEX" 2>/dev/null || true)
     if [[ -n "$CONTENT_HITS" ]]; then
-        echo -e "\n${RED}BLOCKED: Embedded secrets/tokens found in:${NC}"
-        echo "$CONTENT_HITS" | while read -r f; do
-            echo -e "  ${RED}$f${NC}"
-            # Show first matching line for context
-            grep -nE "$SECRET_REGEX" "$f" 2>/dev/null | head -2 | while read -r line; do
-                echo -e "    ${YELLOW}$line${NC}"
-            done
-        done
-        FOUND=1
+        REAL_HITS=""
+        REAL_REPORT=""
+        while IFS= read -r f; do
+            [[ -z "$f" ]] && continue
+            # Re-grep this file, but filter out known_hosts-style lines.
+            FILE_HITS=$(grep -nE "$SECRET_REGEX" "$f" 2>/dev/null | grep -vE "$KNOWN_HOSTS_LINE" || true)
+            if [[ -n "$FILE_HITS" ]]; then
+                REAL_HITS+="$f"$'\n'
+                REAL_REPORT+="  ${RED}$f${NC}"$'\n'
+                # Show first 2 matching lines for context
+                while IFS= read -r line; do
+                    [[ -z "$line" ]] && continue
+                    REAL_REPORT+="    ${YELLOW}$line${NC}"$'\n'
+                done < <(echo "$FILE_HITS" | head -2)
+            fi
+        done <<< "$CONTENT_HITS"
+        if [[ -n "$REAL_HITS" ]]; then
+            echo -e "\n${RED}BLOCKED: Embedded secrets/tokens found in:${NC}"
+            echo -en "$REAL_REPORT"
+            FOUND=1
+        fi
     fi
 fi
 

backend/scripts/setup-venv.ps1

@@ -1,10 +1,16 @@
 param(
-  [string]$Python = "python"
+  [string]$Python = "py"
 )
 
 $repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
 $venvPath = Join-Path $repoRoot "venv"
-& $Python -m venv $venvPath
+$venvMarker = Join-Path $repoRoot ".venv-dir"
+& $Python -3.11 -m venv $venvPath
 
 $pip = Join-Path $venvPath "Scripts\pip.exe"
-& $pip install -r (Join-Path $repoRoot "requirements-dev.txt")
+& $pip install --upgrade pip
+Push-Location $repoRoot
+& (Join-Path $venvPath "Scripts\python.exe") -m pip install -e .
+& $pip install pytest pytest-asyncio ruff black
+"venv" | Set-Content -LiteralPath $venvMarker -NoNewline
+Pop-Location

backend/scripts/setup-venv.sh

@@ -1,9 +1,14 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-PYTHON="${PYTHON:-python3}"
+PYTHON="${PYTHON:-python3.11}"
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
 VENV_DIR="$REPO_ROOT/venv"
+VENV_MARKER="$REPO_ROOT/.venv-dir"
 
 "$PYTHON" -m venv "$VENV_DIR"
-"$VENV_DIR/bin/pip" install -r "$REPO_ROOT/requirements-dev.txt"
+"$VENV_DIR/bin/pip" install --upgrade pip
+cd "$REPO_ROOT"
+"$VENV_DIR/bin/python" -m pip install -e .
+"$VENV_DIR/bin/pip" install pytest pytest-asyncio ruff black
+printf 'venv\n' > "$VENV_MARKER"

backend/services/agent_shell_settings.py

@@ -0,0 +1,54 @@
+"""Operator settings for the embedded agent shell (working directory)."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_SETTINGS_FILE = Path(__file__).resolve().parent.parent / "data" / "agent_shell_settings.json"
+_LOCK = threading.Lock()
+
+
+def _default_working_directory() -> str:
+    explicit = str(os.environ.get("AGENT_SHELL_DEFAULT_CWD") or "").strip()
+    if explicit and os.path.isdir(explicit):
+        return explicit
+    home = str(os.environ.get("HOME") or "").strip()
+    if home and home != "/nonexistent" and os.path.isdir(home):
+        return home
+    return "/app"
+
+
+def get_agent_shell_settings() -> dict[str, Any]:
+    with _LOCK:
+        if not _SETTINGS_FILE.exists():
+            return {"working_directory": _default_working_directory()}
+        try:
+            payload = json.loads(_SETTINGS_FILE.read_text(encoding="utf-8"))
+        except Exception:
+            logger.warning("agent_shell_settings_unreadable")
+            return {"working_directory": _default_working_directory()}
+    cwd = str(payload.get("working_directory") or "").strip() or _default_working_directory()
+    return {"working_directory": cwd}
+
+
+def set_agent_shell_working_directory(path: str) -> dict[str, Any]:
+    normalized = str(path or "").strip()
+    if not normalized:
+        raise ValueError("working_directory_required")
+    resolved = os.path.abspath(os.path.expanduser(normalized))
+    if not os.path.isdir(resolved):
+        raise ValueError("working_directory_not_found")
+    with _LOCK:
+        _SETTINGS_FILE.parent.mkdir(parents=True, exist_ok=True)
+        _SETTINGS_FILE.write_text(
+            json.dumps({"working_directory": resolved}, indent=2) + "\n",
+            encoding="utf-8",
+        )
+    return {"working_directory": resolved}

backend/services/ai_intel_store.py

@@ -0,0 +1,178 @@
+"""ai_intel_store — compatibility wrapper around ai_pin_store + layer injection.
+
+openclaw_channel.py and routers/ai_intel.py import from this module name.
+All pin/layer logic lives in ai_pin_store.py; this module re-exports with the
+expected function signatures and adds the layer injection helper.
+"""
+
+import logging
+import time
+from typing import Any
+
+from services.ai_pin_store import (
+    create_pin,
+    create_pins_batch,
+    get_pins,
+    delete_pin,
+    clear_pins,
+    pin_count,
+    pins_as_geojson,
+    purge_expired,
+    # Layer CRUD
+    create_layer,
+    get_layers,
+    update_layer,
+    delete_layer,
+    # Feed layers
+    get_feed_layers,
+    replace_layer_pins,
+)
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Re-exports expected by openclaw_channel._dispatch_command
+# ---------------------------------------------------------------------------
+
+
+def get_all_intel_pins() -> list[dict[str, Any]]:
+    """Return all active pins (no filter, generous limit)."""
+    return get_pins(limit=2000)
+
+
+def add_intel_pin(args: dict[str, Any]) -> dict[str, Any]:
+    """Create a single pin from a command-channel args dict."""
+    ea = args.get("entity_attachment")
+    return create_pin(
+        lat=float(args.get("lat", 0)),
+        lng=float(args.get("lng", 0)),
+        label=str(args.get("label", ""))[:200],
+        category=str(args.get("category", "custom")),
+        layer_id=str(args.get("layer_id", "")),
+        color=str(args.get("color", "")),
+        description=str(args.get("description", "")),
+        source=str(args.get("source", "openclaw")),
+        source_url=str(args.get("source_url", "")),
+        confidence=float(args.get("confidence", 1.0)),
+        ttl_hours=float(args.get("ttl_hours", 0)),
+        metadata=args.get("metadata") or {},
+        entity_attachment=ea if isinstance(ea, dict) else None,
+    )
+
+
+def delete_intel_pin(pin_id: str) -> bool:
+    """Delete a pin by ID."""
+    return delete_pin(pin_id)
+
+
+# Layer helpers for OpenClaw
+def create_intel_layer(args: dict[str, Any]) -> dict[str, Any]:
+    """Create a layer from a command-channel args dict."""
+    return create_layer(
+        name=str(args.get("name", "Untitled"))[:100],
+        description=str(args.get("description", ""))[:500],
+        source=str(args.get("source", "openclaw"))[:50],
+        color=str(args.get("color", "")),
+        feed_url=str(args.get("feed_url", "")),
+        feed_interval=int(args.get("feed_interval", 300)),
+    )
+
+
+def get_intel_layers() -> list[dict[str, Any]]:
+    """Return all layers with pin counts."""
+    return get_layers()
+
+
+def update_intel_layer(layer_id: str, args: dict[str, Any]) -> dict[str, Any] | None:
+    """Update a layer from a command-channel args dict."""
+    return update_layer(layer_id, **{
+        k: v for k, v in args.items()
+        if k in ("name", "description", "visible", "color", "feed_url", "feed_interval")
+    })
+
+
+def delete_intel_layer(layer_id: str) -> int:
+    """Delete a layer and its pins. Returns pin count removed."""
+    return delete_layer(layer_id)
+
+
+# ---------------------------------------------------------------------------
+# Layer injection — inserts agent data into native telemetry layers
+# ---------------------------------------------------------------------------
+
+# Layers that agents are allowed to inject into.
+_INJECTABLE_LAYERS = frozenset({
+    "cctv", "ships", "sigint", "kiwisdr", "military_bases",
+    "datacenters", "power_plants", "satnogs_stations",
+    "volcanoes", "earthquakes", "news", "viirs_change_nodes",
+    "air_quality",
+})
+
+
+def inject_layer_data(
+    layer: str,
+    items: list[dict[str, Any]],
+    mode: str = "append",
+) -> dict[str, Any]:
+    """Inject agent data into a native telemetry layer."""
+    from services.fetchers._store import latest_data, _data_lock, bump_data_version
+
+    layer = str(layer or "").strip()
+    if layer not in _INJECTABLE_LAYERS:
+        return {"ok": False, "detail": f"layer '{layer}' not injectable"}
+
+    items = list(items or [])[:200]
+    if not items:
+        return {"ok": False, "detail": "no items provided"}
+
+    now = time.time()
+    tagged = []
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        entry = dict(item)
+        entry["_injected"] = True
+        entry["_source"] = "user:openclaw"
+        entry["_injected_at"] = now
+        tagged.append(entry)
+
+    with _data_lock:
+        existing = latest_data.get(layer)
+        if not isinstance(existing, list):
+            existing = []
+
+        if mode == "replace":
+            existing = [e for e in existing if not e.get("_injected")]
+
+        existing.extend(tagged)
+        latest_data[layer] = existing
+
+    bump_data_version()
+
+    return {
+        "ok": True,
+        "layer": layer,
+        "injected": len(tagged),
+        "mode": mode,
+    }
+
+
+def clear_injected_data(layer: str = "") -> dict[str, Any]:
+    """Remove all injected items from a layer (or all layers)."""
+    from services.fetchers._store import latest_data, _data_lock, bump_data_version
+
+    removed = 0
+    with _data_lock:
+        targets = [layer] if layer else list(_INJECTABLE_LAYERS)
+        for lyr in targets:
+            existing = latest_data.get(lyr)
+            if not isinstance(existing, list):
+                continue
+            before = len(existing)
+            latest_data[lyr] = [e for e in existing if not e.get("_injected")]
+            removed += before - len(latest_data[lyr])
+
+    if removed:
+        bump_data_version()
+
+    return {"ok": True, "removed": removed}

backend/services/ai_pin_store.py

@@ -0,0 +1,633 @@
+"""AI Intel pin storage — layered pin system with JSON file persistence.
+
+Supports:
+  - Named pin layers (created by user or AI)
+  - Pins with optional entity attachment (track moving objects)
+  - Pin source tracking (user vs openclaw)
+  - Layer visibility toggles
+  - External feed URL per layer (for Phase 5)
+  - GeoJSON export per layer or all layers
+"""
+
+import json
+import logging
+import os
+import threading
+import time
+import uuid
+from datetime import datetime
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Pin schema
+# ---------------------------------------------------------------------------
+
+PIN_CATEGORIES = {
+    "threat", "news", "geolocation", "custom", "anomaly",
+    "military", "maritime", "flight", "infrastructure", "weather",
+    "sigint", "prediction", "research",
+}
+
+PIN_COLORS = {
+    "threat": "#ef4444",       # red
+    "news": "#f59e0b",         # amber
+    "geolocation": "#8b5cf6",  # violet
+    "custom": "#3b82f6",       # blue
+    "anomaly": "#f97316",      # orange
+    "military": "#dc2626",     # dark red
+    "maritime": "#0ea5e9",     # sky
+    "flight": "#6366f1",       # indigo
+    "infrastructure": "#64748b",  # slate
+    "weather": "#22d3ee",      # cyan
+    "sigint": "#a855f7",       # purple
+    "prediction": "#eab308",   # yellow
+    "research": "#10b981",     # emerald
+}
+
+LAYER_COLORS = [
+    "#3b82f6", "#ef4444", "#22d3ee", "#f59e0b", "#8b5cf6",
+    "#10b981", "#f97316", "#6366f1", "#ec4899", "#14b8a6",
+]
+
+# ---------------------------------------------------------------------------
+# In-memory store
+# ---------------------------------------------------------------------------
+
+_layers: list[dict[str, Any]] = []
+_pins: list[dict[str, Any]] = []
+_lock = threading.Lock()
+
+# Persistence file path
+_PERSIST_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data")
+_PERSIST_FILE = os.path.join(_PERSIST_DIR, "pin_layers.json")
+_OLD_PERSIST_FILE = os.path.join(_PERSIST_DIR, "ai_pins.json")
+
+
+def _ensure_persist_dir():
+    try:
+        os.makedirs(_PERSIST_DIR, exist_ok=True)
+    except OSError:
+        pass
+
+
+def _save_to_disk():
+    """Persist layers and pins to JSON file. Called under lock."""
+    try:
+        _ensure_persist_dir()
+        with open(_PERSIST_FILE, "w", encoding="utf-8") as f:
+            json.dump({"layers": _layers, "pins": _pins}, f, indent=2, default=str)
+    except (OSError, IOError) as e:
+        logger.warning(f"Failed to persist pin layers: {e}")
+
+
+def _load_from_disk():
+    """Load layers and pins from disk on startup."""
+    global _layers, _pins
+    try:
+        if os.path.exists(_PERSIST_FILE):
+            with open(_PERSIST_FILE, "r", encoding="utf-8") as f:
+                data = json.load(f)
+            if isinstance(data, dict):
+                _layers = data.get("layers", [])
+                _pins = data.get("pins", [])
+                logger.info(f"Loaded {len(_layers)} layers, {len(_pins)} pins from disk")
+                return
+
+        # Migrate from old flat pin file
+        if os.path.exists(_OLD_PERSIST_FILE):
+            with open(_OLD_PERSIST_FILE, "r", encoding="utf-8") as f:
+                old_pins = json.load(f)
+            if isinstance(old_pins, list) and old_pins:
+                legacy_layer = _make_layer("Legacy", "Migrated pins", source="system")
+                _layers.append(legacy_layer)
+                for p in old_pins:
+                    if isinstance(p, dict):
+                        p["layer_id"] = legacy_layer["id"]
+                        _pins.append(p)
+                logger.info(f"Migrated {len(_pins)} pins from ai_pins.json into Legacy layer")
+                _save_to_disk()
+    except (OSError, IOError, json.JSONDecodeError) as e:
+        logger.warning(f"Failed to load pin layers from disk: {e}")
+
+
+def _make_layer(
+    name: str,
+    description: str = "",
+    source: str = "user",
+    color: str = "",
+    feed_url: str = "",
+    feed_interval: int = 300,
+) -> dict[str, Any]:
+    """Create a layer dict."""
+    layer_id = str(uuid.uuid4())[:12]
+    now = time.time()
+    return {
+        "id": layer_id,
+        "name": name[:100],
+        "description": description[:500],
+        "source": source[:50],
+        "visible": True,
+        "color": color or LAYER_COLORS[len(_layers) % len(LAYER_COLORS)],
+        "created_at": now,
+        "created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
+        "feed_url": feed_url[:1000] if feed_url else "",
+        "feed_interval": max(60, min(86400, feed_interval)),
+        "pin_count": 0,
+    }
+
+
+# Load on import
+_load_from_disk()
+
+# One-time cleanup: remove correlation_engine auto-pins (no longer generated)
+_corr_before = len(_pins)
+_pins[:] = [p for p in _pins if p.get("source") != "correlation_engine"]
+if len(_pins) < _corr_before:
+    logger.info("Cleaned up %d legacy correlation_engine pins", _corr_before - len(_pins))
+    _save_to_disk()
+
+
+# ---------------------------------------------------------------------------
+# Layer CRUD
+# ---------------------------------------------------------------------------
+
+def create_layer(
+    name: str,
+    description: str = "",
+    source: str = "user",
+    color: str = "",
+    feed_url: str = "",
+    feed_interval: int = 300,
+) -> dict[str, Any]:
+    """Create a new pin layer."""
+    with _lock:
+        layer = _make_layer(name, description, source, color, feed_url, feed_interval)
+        _layers.append(layer)
+        _save_to_disk()
+    return layer
+
+
+def get_layers() -> list[dict[str, Any]]:
+    """Return all layers with current pin counts."""
+    now = time.time()
+    with _lock:
+        result = []
+        for layer in _layers:
+            count = sum(
+                1 for p in _pins
+                if p.get("layer_id") == layer["id"]
+                and not (p.get("expires_at") and p["expires_at"] < now)
+            )
+            result.append({**layer, "pin_count": count})
+        return result
+
+
+def update_layer(layer_id: str, **updates) -> Optional[dict[str, Any]]:
+    """Update layer fields. Returns updated layer or None if not found."""
+    allowed = {"name", "description", "visible", "color", "feed_url", "feed_interval", "feed_last_fetched"}
+    with _lock:
+        for layer in _layers:
+            if layer["id"] == layer_id:
+                for k, v in updates.items():
+                    if k in allowed and v is not None:
+                        if k == "name":
+                            layer[k] = str(v)[:100]
+                        elif k == "description":
+                            layer[k] = str(v)[:500]
+                        elif k == "visible":
+                            layer[k] = bool(v)
+                        elif k == "color":
+                            layer[k] = str(v)[:20]
+                        elif k == "feed_url":
+                            layer[k] = str(v)[:1000]
+                        elif k == "feed_interval":
+                            layer[k] = max(60, min(86400, int(v)))
+                        elif k == "feed_last_fetched":
+                            layer[k] = float(v)
+                _save_to_disk()
+                return dict(layer)
+    return None
+
+
+def delete_layer(layer_id: str) -> int:
+    """Delete a layer and all its pins. Returns count of pins removed."""
+    with _lock:
+        before_layers = len(_layers)
+        _layers[:] = [l for l in _layers if l["id"] != layer_id]
+        if len(_layers) == before_layers:
+            return 0  # not found
+        before_pins = len(_pins)
+        _pins[:] = [p for p in _pins if p.get("layer_id") != layer_id]
+        removed = before_pins - len(_pins)
+        _save_to_disk()
+        return removed
+
+
+# ---------------------------------------------------------------------------
+# Pin CRUD
+# ---------------------------------------------------------------------------
+
+def create_pin(
+    lat: float,
+    lng: float,
+    label: str,
+    category: str = "custom",
+    *,
+    layer_id: str = "",
+    color: str = "",
+    description: str = "",
+    source: str = "openclaw",
+    source_url: str = "",
+    confidence: float = 1.0,
+    ttl_hours: float = 0,
+    metadata: Optional[dict] = None,
+    entity_attachment: Optional[dict] = None,
+) -> dict[str, Any]:
+    """Create a single pin and return it."""
+    pin_id = str(uuid.uuid4())[:12]
+    now = time.time()
+
+    cat = category if category in PIN_CATEGORIES else "custom"
+    pin_color = color or PIN_COLORS.get(cat, "#3b82f6")
+
+    # Validate entity_attachment if provided
+    attachment = None
+    if entity_attachment and isinstance(entity_attachment, dict):
+        etype = str(entity_attachment.get("entity_type", "")).strip()
+        eid = str(entity_attachment.get("entity_id", "")).strip()
+        if etype and eid:
+            attachment = {
+                "entity_type": etype[:50],
+                "entity_id": eid[:100],
+                "entity_label": str(entity_attachment.get("entity_label", ""))[:200],
+            }
+
+    pin = {
+        "id": pin_id,
+        "layer_id": layer_id or "",
+        "lat": lat,
+        "lng": lng,
+        "label": label[:200],
+        "category": cat,
+        "color": pin_color,
+        "description": description[:2000],
+        "source": source[:100],
+        "source_url": source_url[:500],
+        "confidence": max(0.0, min(1.0, confidence)),
+        "created_at": now,
+        "created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
+        "expires_at": now + (ttl_hours * 3600) if ttl_hours > 0 else None,
+        "metadata": metadata or {},
+        "entity_attachment": attachment,
+        "comments": [],
+    }
+
+    with _lock:
+        _pins.append(pin)
+        _save_to_disk()
+
+    return pin
+
+
+def create_pins_batch(items: list[dict], default_layer_id: str = "") -> list[dict[str, Any]]:
+    """Create multiple pins at once."""
+    created = []
+    now = time.time()
+
+    with _lock:
+        for item in items[:200]:  # max 200 per batch
+            pin_id = str(uuid.uuid4())[:12]
+            cat = item.get("category", "custom")
+            if cat not in PIN_CATEGORIES:
+                cat = "custom"
+            pin_color = item.get("color", "") or PIN_COLORS.get(cat, "#3b82f6")
+            ttl = float(item.get("ttl_hours", 0) or 0)
+
+            attachment = None
+            ea = item.get("entity_attachment")
+            if ea and isinstance(ea, dict):
+                etype = str(ea.get("entity_type", "")).strip()
+                eid = str(ea.get("entity_id", "")).strip()
+                if etype and eid:
+                    attachment = {
+                        "entity_type": etype[:50],
+                        "entity_id": eid[:100],
+                        "entity_label": str(ea.get("entity_label", ""))[:200],
+                    }
+
+            pin = {
+                "id": pin_id,
+                "layer_id": item.get("layer_id", default_layer_id) or "",
+                "lat": float(item.get("lat", 0)),
+                "lng": float(item.get("lng", 0)),
+                "label": str(item.get("label", ""))[:200],
+                "category": cat,
+                "color": pin_color,
+                "description": str(item.get("description", ""))[:2000],
+                "source": str(item.get("source", "openclaw"))[:100],
+                "source_url": str(item.get("source_url", ""))[:500],
+                "confidence": max(0.0, min(1.0, float(item.get("confidence", 1.0)))),
+                "created_at": now,
+                "created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
+                "expires_at": now + (ttl * 3600) if ttl > 0 else None,
+                "metadata": item.get("metadata", {}),
+                "entity_attachment": attachment,
+                "comments": [],
+            }
+            _pins.append(pin)
+            created.append(pin)
+
+        _save_to_disk()
+    return created
+
+
+def get_pins(
+    category: str = "",
+    source: str = "",
+    layer_id: str = "",
+    limit: int = 500,
+    include_expired: bool = False,
+) -> list[dict[str, Any]]:
+    """Get pins with optional filters."""
+    now = time.time()
+    with _lock:
+        results = []
+        for pin in _pins:
+            if not include_expired and pin.get("expires_at") and pin["expires_at"] < now:
+                continue
+            if category and pin.get("category") != category:
+                continue
+            if source and pin.get("source") != source:
+                continue
+            if layer_id and pin.get("layer_id") != layer_id:
+                continue
+            results.append(pin)
+            if len(results) >= limit:
+                break
+        return results
+
+
+def get_pin(pin_id: str) -> Optional[dict[str, Any]]:
+    """Return a single pin by ID (including comments), or None."""
+    with _lock:
+        for pin in _pins:
+            if pin.get("id") == pin_id:
+                # Ensure comments key exists for legacy pins
+                if "comments" not in pin:
+                    pin["comments"] = []
+                return dict(pin)
+    return None
+
+
+def update_pin(pin_id: str, **updates) -> Optional[dict[str, Any]]:
+    """Update a pin's editable fields (label, description, category, color)."""
+    allowed = {"label", "description", "category", "color"}
+    with _lock:
+        for pin in _pins:
+            if pin.get("id") != pin_id:
+                continue
+            for k, v in updates.items():
+                if k not in allowed or v is None:
+                    continue
+                if k == "label":
+                    pin[k] = str(v)[:200]
+                elif k == "description":
+                    pin[k] = str(v)[:2000]
+                elif k == "category":
+                    cat = str(v)
+                    if cat in PIN_CATEGORIES:
+                        pin[k] = cat
+                        # Refresh color if it was the category default
+                        if not updates.get("color"):
+                            pin["color"] = PIN_COLORS.get(cat, pin.get("color", "#3b82f6"))
+                elif k == "color":
+                    pin[k] = str(v)[:20]
+            pin["updated_at"] = time.time()
+            _save_to_disk()
+            return dict(pin)
+    return None
+
+
+def add_pin_comment(
+    pin_id: str,
+    text: str,
+    author: str = "user",
+    author_label: str = "",
+    reply_to: str = "",
+) -> Optional[dict[str, Any]]:
+    """Append a comment to a pin. Returns the updated pin (with all comments)."""
+    text = (text or "").strip()
+    if not text:
+        return None
+    with _lock:
+        for pin in _pins:
+            if pin.get("id") != pin_id:
+                continue
+            if "comments" not in pin or not isinstance(pin["comments"], list):
+                pin["comments"] = []
+            comment = {
+                "id": str(uuid.uuid4())[:12],
+                "text": text[:4000],
+                "author": (author or "user")[:50],
+                "author_label": (author_label or "")[:100],
+                "reply_to": (reply_to or "")[:12],
+                "created_at": time.time(),
+                "created_at_iso": datetime.utcnow().isoformat() + "Z",
+            }
+            pin["comments"].append(comment)
+            _save_to_disk()
+            return dict(pin)
+    return None
+
+
+def delete_pin_comment(pin_id: str, comment_id: str) -> bool:
+    """Remove a single comment from a pin."""
+    with _lock:
+        for pin in _pins:
+            if pin.get("id") != pin_id:
+                continue
+            comments = pin.get("comments") or []
+            before = len(comments)
+            pin["comments"] = [c for c in comments if c.get("id") != comment_id]
+            if len(pin["comments"]) < before:
+                _save_to_disk()
+                return True
+            return False
+    return False
+
+
+def delete_pin(pin_id: str) -> bool:
+    """Delete a single pin by ID."""
+    with _lock:
+        before = len(_pins)
+        _pins[:] = [p for p in _pins if p.get("id") != pin_id]
+        if len(_pins) < before:
+            _save_to_disk()
+            return True
+        return False
+
+
+def clear_pins(category: str = "", source: str = "", layer_id: str = "") -> int:
+    """Clear pins, optionally filtered. Returns count removed."""
+    with _lock:
+        before = len(_pins)
+
+        def keep(p):
+            if layer_id and p.get("layer_id") != layer_id:
+                return True  # different layer, keep
+            if category and source:
+                return not (p.get("category") == category and p.get("source") == source)
+            if category:
+                return p.get("category") != category
+            if source:
+                return p.get("source") != source
+            if layer_id:
+                return p.get("layer_id") != layer_id
+            return False
+
+        if not category and not source and not layer_id:
+            _pins.clear()
+        else:
+            _pins[:] = [p for p in _pins if keep(p)]
+
+        removed = before - len(_pins)
+        if removed:
+            _save_to_disk()
+        return removed
+
+
+def get_feed_layers() -> list[dict[str, Any]]:
+    """Return layers that have a non-empty feed_url."""
+    with _lock:
+        return [dict(l) for l in _layers if l.get("feed_url")]
+
+
+def replace_layer_pins(layer_id: str, new_pins: list[dict[str, Any]]) -> int:
+    """Atomically replace all pins in a layer with new_pins. Returns count added."""
+    now = time.time()
+    with _lock:
+        # Remove old pins for this layer
+        _pins[:] = [p for p in _pins if p.get("layer_id") != layer_id]
+        # Add new pins
+        added = 0
+        for item in new_pins[:500]:  # cap at 500 per feed
+            pin_id = str(uuid.uuid4())[:12]
+            cat = item.get("category", "custom")
+            if cat not in PIN_CATEGORIES:
+                cat = "custom"
+            pin_color = item.get("color", "") or PIN_COLORS.get(cat, "#3b82f6")
+
+            attachment = None
+            ea = item.get("entity_attachment")
+            if ea and isinstance(ea, dict):
+                etype = str(ea.get("entity_type", "")).strip()
+                eid = str(ea.get("entity_id", "")).strip()
+                if etype and eid:
+                    attachment = {
+                        "entity_type": etype[:50],
+                        "entity_id": eid[:100],
+                        "entity_label": str(ea.get("entity_label", ""))[:200],
+                    }
+
+            pin = {
+                "id": pin_id,
+                "layer_id": layer_id,
+                "lat": float(item.get("lat", 0)),
+                "lng": float(item.get("lng", 0)),
+                "label": str(item.get("label", item.get("name", "")))[:200],
+                "category": cat,
+                "color": pin_color,
+                "description": str(item.get("description", ""))[:2000],
+                "source": str(item.get("source", "feed"))[:100],
+                "source_url": str(item.get("source_url", ""))[:500],
+                "confidence": max(0.0, min(1.0, float(item.get("confidence", 1.0)))),
+                "created_at": now,
+                "created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
+                "expires_at": None,
+                "metadata": item.get("metadata", {}),
+                "entity_attachment": attachment,
+                "comments": [],
+            }
+            _pins.append(pin)
+            added += 1
+        _save_to_disk()
+    return added
+
+
+def purge_expired() -> int:
+    """Remove expired pins. Called periodically."""
+    now = time.time()
+    with _lock:
+        before = len(_pins)
+        _pins[:] = [p for p in _pins if not (p.get("expires_at") and p["expires_at"] < now)]
+        removed = before - len(_pins)
+        if removed:
+            _save_to_disk()
+        return removed
+
+
+def pin_count() -> dict[str, int]:
+    """Return counts by category."""
+    now = time.time()
+    counts: dict[str, int] = {}
+    with _lock:
+        for pin in _pins:
+            if pin.get("expires_at") and pin["expires_at"] < now:
+                continue
+            cat = pin.get("category", "custom")
+            counts[cat] = counts.get(cat, 0) + 1
+    return counts
+
+
+def pins_as_geojson(layer_id: str = "") -> dict[str, Any]:
+    """Convert active pins to GeoJSON FeatureCollection for the map layer."""
+    now = time.time()
+    features = []
+    with _lock:
+        # Build set of visible layer IDs
+        visible_layers = {l["id"] for l in _layers if l.get("visible", True)}
+
+        for pin in _pins:
+            if pin.get("expires_at") and pin["expires_at"] < now:
+                continue
+            # Layer filter
+            pid_layer = pin.get("layer_id", "")
+            if layer_id and pid_layer != layer_id:
+                continue
+            # Skip pins in hidden layers
+            if pid_layer and pid_layer not in visible_layers:
+                continue
+
+            props = {
+                "id": pin["id"],
+                "layer_id": pid_layer,
+                "label": pin["label"],
+                "category": pin["category"],
+                "color": pin["color"],
+                "description": pin.get("description", ""),
+                "source": pin["source"],
+                "source_url": pin.get("source_url", ""),
+                "confidence": pin.get("confidence", 1.0),
+                "created_at": pin.get("created_at_iso", ""),
+                "comment_count": len(pin.get("comments") or []),
+            }
+
+            # Entity attachment info (frontend resolves position)
+            ea = pin.get("entity_attachment")
+            if ea:
+                props["entity_attachment"] = ea
+
+            features.append({
+                "type": "Feature",
+                "geometry": {
+                    "type": "Point",
+                    "coordinates": [pin["lng"], pin["lat"]],
+                },
+                "properties": props,
+            })
+    return {
+        "type": "FeatureCollection",
+        "features": features,
+    }

backend/services/ais_stream.py

@@ -17,6 +17,18 @@ AIS_WS_URL = "wss://stream.aisstream.io/v0/stream"
 API_KEY = os.environ.get("AIS_API_KEY", "")
 
 
+def _env_truthy(name: str) -> bool:
+    return str(os.getenv(name, "")).strip().lower() in {"1", "true", "yes", "on"}
+
+
+def ais_stream_proxy_enabled() -> bool:
+    """Return whether the external Node AIS proxy may be started."""
+    setting = str(os.getenv("SHADOWBROKER_ENABLE_AIS_STREAM_PROXY", "")).strip().lower()
+    if setting:
+        return _env_truthy("SHADOWBROKER_ENABLE_AIS_STREAM_PROXY")
+    return True
+
+
 # AIS vessel type code classification
 # See: https://coast.noaa.gov/data/marinecadastre/ais/VesselTypeCodes2018.pdf
 def classify_vessel(ais_type: int, mmsi: int) -> str:
@@ -327,16 +339,117 @@ def get_country_from_mmsi(mmsi: int) -> str:
 
 # Global vessel store: MMSI → vessel dict
 _vessels: dict[int, dict] = {}
+_vessel_trails: dict[int, dict] = {}
 _vessels_lock = threading.Lock()
 _ws_thread: threading.Thread | None = None
 _ws_running = False
 _proxy_process = None
+# Issue #258: latest status snapshot emitted by ais_proxy.js. Populated when
+# the proxy reports e.g. {"__ais_proxy_status": {"degraded_tls": true}} on
+# stdout, which it does when it falls back to the SPKI-pinned insecure-date
+# path during an upstream cert outage. Surfaced via ais_proxy_status() for
+# /api/health.
+_proxy_status: dict = {}
+# Upstream-connectivity telemetry (added when stream.aisstream.io went fully
+# offline on 2026-05-23). ``_last_msg_at`` is the unix timestamp of the most
+# recent vessel message received from the proxy. ``_proxy_spawn_count`` is
+# how many times we've started the node proxy; combined with no recent
+# messages it tells us the proxy is respawning in a tight loop because the
+# upstream is unreachable. Surfaced via ais_proxy_status() so the operator
+# can see "AIS is dead" instead of guessing whether it's their map filter,
+# their api key, or upstream.
+_last_msg_at: float = 0.0
+_proxy_spawn_count: int = 0
+_VESSEL_TRAIL_INTERVAL_S = 120
+_VESSEL_TRAIL_MAX_POINTS = 240
+
+
+# How stale "last vessel message" can be before we consider the stream
+# disconnected. AISStream typically pushes multiple messages/sec, so a 60s
+# gap means something's wrong upstream or in transit.
+_AIS_CONNECTED_FRESHNESS_S = 60
+
+
+def ais_proxy_status() -> dict:
+    """Return a copy of the latest ais_proxy.js status + connectivity health.
+
+    Fields:
+      * ``degraded_tls`` (bool, issue #258) — true when the proxy is using
+        SPKI-pinned fallback because AISStream's cert expired.
+      * ``connected`` (bool) — true when we received a vessel message in
+        the last ``_AIS_CONNECTED_FRESHNESS_S`` seconds.
+      * ``last_msg_age_seconds`` (int | None) — seconds since the last
+        vessel message; None if we've never received one.
+      * ``proxy_spawn_count`` (int) — how many times we've spawned the
+        node proxy. Sustained increases here without ``connected`` means
+        we're respawning in a tight loop because upstream is dead.
+
+    Returns an empty dict when called before the AIS subsystem starts
+    (e.g. during tests or when no API key is set).
+    """
+    with _vessels_lock:
+        status = dict(_proxy_status)
+        last = _last_msg_at
+        spawns = _proxy_spawn_count
+
+    now = time.time()
+    if last > 0:
+        last_age = int(now - last)
+        status["last_msg_age_seconds"] = last_age
+        status["connected"] = last_age <= _AIS_CONNECTED_FRESHNESS_S
+    else:
+        status["last_msg_age_seconds"] = None
+        status["connected"] = False
+    status["proxy_spawn_count"] = spawns
+    return status
 
 import os
 
 CACHE_FILE = os.path.join(os.path.dirname(__file__), "ais_cache.json")
 
 
+def _record_vessel_trail_locked(mmsi: int, lat, lng, sog=0, now_ts: float | None = None) -> None:
+    """Append a sampled AIS trail point. Caller must hold _vessels_lock."""
+    if lat is None or lng is None:
+        return
+    try:
+        lat_f = float(lat)
+        lng_f = float(lng)
+    except (TypeError, ValueError):
+        return
+    if abs(lat_f) > 90 or abs(lng_f) > 180 or (lat_f == 0 and lng_f == 0):
+        return
+    now = now_ts or time.time()
+    trail_data = _vessel_trails.setdefault(int(mmsi), {"points": [], "last_seen": now})
+    point = [round(lat_f, 5), round(lng_f, 5), round(float(sog or 0), 1), round(now)]
+    last_point_ts = trail_data["points"][-1][3] if trail_data["points"] else 0
+    if now - last_point_ts < _VESSEL_TRAIL_INTERVAL_S:
+        trail_data["last_seen"] = now
+        return
+    if (
+        trail_data["points"]
+        and trail_data["points"][-1][0] == point[0]
+        and trail_data["points"][-1][1] == point[1]
+    ):
+        trail_data["last_seen"] = now
+        return
+    trail_data["points"].append(point)
+    trail_data["last_seen"] = now
+    if len(trail_data["points"]) > _VESSEL_TRAIL_MAX_POINTS:
+        trail_data["points"] = trail_data["points"][-_VESSEL_TRAIL_MAX_POINTS:]
+
+
+def get_vessel_trail(mmsi: int) -> list:
+    """Return the accumulated trail for a single vessel without expanding live payloads."""
+    try:
+        key = int(mmsi)
+    except (TypeError, ValueError):
+        return []
+    with _vessels_lock:
+        points = _vessel_trails.get(key, {}).get("points", [])
+        return [list(point) for point in points]
+
+
 def _save_cache():
     """Save vessel data to disk for persistence across restarts."""
     try:
@@ -379,6 +492,7 @@ def prune_stale_vessels():
         stale_keys = [k for k, v in _vessels.items() if v.get("_updated", 0) < stale_cutoff]
         for k in stale_keys:
             del _vessels[k]
+            _vessel_trails.pop(k, None)
     if stale_keys:
         logger.info(f"AIS pruned {len(stale_keys)} stale vessels")
 
@@ -447,6 +561,7 @@ def ingest_ais_catcher(msgs: list[dict]) -> int:
                     heading = msg.get("heading", 511)
                     vessel["heading"] = heading if heading != 511 else vessel.get("cog", 0)
                     vessel["_updated"] = now
+                    _record_vessel_trail_locked(mmsi, lat, lon, vessel["sog"], now)
                     if msg.get("shipname"):
                         vessel["name"] = msg["shipname"].strip()
                     count += 1
@@ -496,6 +611,12 @@ def _ais_stream_loop():
             logger.info("Starting Node.js AIS Stream Proxy...")
             proxy_env = os.environ.copy()
             proxy_env["AIS_API_KEY"] = API_KEY
+            popen_kwargs = {}
+            if os.name == "nt":
+                popen_kwargs["creationflags"] = (
+                    getattr(subprocess, "CREATE_NO_WINDOW", 0)
+                    | getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
+                )
             process = subprocess.Popen(
                 ["node", proxy_script],
                 stdin=subprocess.PIPE,
@@ -504,9 +625,12 @@ def _ais_stream_loop():
                 text=True,
                 bufsize=1,
                 env=proxy_env,
+                **popen_kwargs,
             )
+            global _proxy_spawn_count
             with _vessels_lock:
                 _proxy_process = process
+                _proxy_spawn_count += 1
 
             # Drain stderr in a background thread to prevent deadlock
             import threading
@@ -542,6 +666,18 @@ def _ais_stream_loop():
                     logger.error(f"AIS Stream error: {data['error']}")
                     continue
 
+                # Issue #258: ais_proxy.js emits status markers (e.g.
+                # {"__ais_proxy_status": {"degraded_tls": true}}) when the
+                # SPKI-pinned fallback is in use. We snapshot the latest
+                # status so the backend can expose it on /api/health.
+                if isinstance(data, dict) and "__ais_proxy_status" in data:
+                    status = data.get("__ais_proxy_status") or {}
+                    if isinstance(status, dict):
+                        with _vessels_lock:
+                            _proxy_status.clear()
+                            _proxy_status.update(status)
+                    continue
+
                 msg_type = data.get("MessageType", "")
                 metadata = data.get("MetaData", {})
                 message = data.get("Message", {})
@@ -550,9 +686,15 @@ def _ais_stream_loop():
                 if not mmsi:
                     continue
 
+                # Telemetry: stamp the timestamp of the most recent real
+                # vessel message. ais_proxy_status() reads this to decide
+                # whether the stream is currently "connected" — i.e. has
+                # any data flowed in the last 60s.
+                global _last_msg_at
                 with _vessels_lock:
+                    _last_msg_at = time.time()
                     if mmsi not in _vessels:
-                        _vessels[mmsi] = {"_updated": time.time()}
+                        _vessels[mmsi] = {"_updated": _last_msg_at}
                     vessel = _vessels[mmsi]
 
                 # Update position from PositionReport or StandardClassBPositionReport
@@ -576,7 +718,9 @@ def _ais_stream_loop():
                         vessel["cog"] = report.get("Cog", 0)
                         heading = report.get("TrueHeading", 511)
                         vessel["heading"] = heading if heading != 511 else report.get("Cog", 0)
-                        vessel["_updated"] = time.time()
+                        now_ts = time.time()
+                        vessel["_updated"] = now_ts
+                        _record_vessel_trail_locked(mmsi, lat, lng, vessel["sog"], now_ts)
                         # Use metadata name if we don't have one yet
                         if not vessel.get("name") or vessel["name"] == "UNKNOWN":
                             vessel["name"] = (
@@ -646,6 +790,22 @@ def _run_ais_loop():
 def start_ais_stream():
     """Start the AIS WebSocket stream in a background thread."""
     global _ws_thread, _ws_running
+
+    # Always load cached vessel data first so the ships layer can paint even
+    # when live streaming is disabled or the upstream is unavailable.
+    _load_cache()
+
+    if not API_KEY:
+        logger.info("AIS_API_KEY not set — ship tracking disabled. Set AIS_API_KEY to enable.")
+        return
+
+    if not ais_stream_proxy_enabled():
+        logger.info(
+            "AIS live stream proxy disabled for this runtime; using cached AIS vessels. "
+            "Set SHADOWBROKER_ENABLE_AIS_STREAM_PROXY=1 to opt in."
+        )
+        return
+
     with _vessels_lock:
         if _ws_running:
             logger.info("AIS Stream already running")
@@ -656,9 +816,6 @@ def start_ais_stream():
         logger.info("AIS Stream already running")
         return
 
-    # Load cached vessel data from disk
-    _load_cache()
-
     _ws_thread = threading.Thread(target=_run_ais_loop, daemon=True, name="ais-stream")
     _ws_thread.start()
     logger.info("AIS Stream background thread started")

backend/services/analysis_zone_store.py

@@ -0,0 +1,189 @@
+"""Analysis Zone store — OpenClaw-placed map overlays with analyst notes.
+
+These render as the dashed-border squares on the correlations layer.
+Unlike automated correlations (which are recomputed every cycle), analysis
+zones persist until the agent or user deletes them, or their TTL expires.
+
+Shape matches the correlation alert schema so the frontend renders them
+identically — the ``source`` field marks them as agent-placed and enables
+the delete button in the popup.
+"""
+
+import json
+import logging
+import os
+import threading
+import time
+import uuid
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_zones: list[dict[str, Any]] = []
+_lock = threading.Lock()
+
+_PERSIST_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data")
+_PERSIST_FILE = os.path.join(_PERSIST_DIR, "analysis_zones.json")
+
+ZONE_CATEGORIES = {
+    "contradiction",   # narrative vs telemetry mismatch
+    "analysis",        # general analyst note / assessment
+    "warning",         # potential threat or risk area
+    "observation",     # neutral observation worth marking
+    "hypothesis",      # unverified theory to investigate
+}
+
+# Map categories to correlation type colors on the frontend
+CATEGORY_COLORS = {
+    "contradiction": "amber",
+    "analysis": "cyan",
+    "warning": "red",
+    "observation": "blue",
+    "hypothesis": "purple",
+}
+
+
+def _ensure_dir():
+    try:
+        os.makedirs(_PERSIST_DIR, exist_ok=True)
+    except OSError:
+        pass
+
+
+def _save():
+    """Persist to disk. Called under lock."""
+    try:
+        _ensure_dir()
+        with open(_PERSIST_FILE, "w", encoding="utf-8") as f:
+            json.dump(_zones, f, indent=2, default=str)
+    except Exception as e:
+        logger.warning("Failed to save analysis zones: %s", e)
+
+
+def _load():
+    """Load from disk on startup."""
+    global _zones
+    try:
+        if os.path.exists(_PERSIST_FILE):
+            with open(_PERSIST_FILE, "r", encoding="utf-8") as f:
+                data = json.load(f)
+            if isinstance(data, list):
+                _zones = data
+                logger.info("Loaded %d analysis zones from disk", len(_zones))
+    except Exception as e:
+        logger.warning("Failed to load analysis zones: %s", e)
+
+
+# Load on import
+_load()
+
+
+def _expire():
+    """Remove zones past their TTL. Called under lock."""
+    now = time.time()
+    before = len(_zones)
+    _zones[:] = [
+        z for z in _zones
+        if z.get("ttl_hours", 0) <= 0
+        or (now - z.get("created_at", now)) < z["ttl_hours"] * 3600
+    ]
+    removed = before - len(_zones)
+    if removed:
+        logger.info("Expired %d analysis zones", removed)
+
+
+def create_zone(
+    *,
+    lat: float,
+    lng: float,
+    title: str,
+    body: str,
+    category: str = "analysis",
+    severity: str = "medium",
+    cell_size_deg: float = 1.0,
+    ttl_hours: float = 0,
+    source: str = "openclaw",
+    drivers: list[str] | None = None,
+) -> dict[str, Any]:
+    """Create an analysis zone. Returns the created zone dict."""
+    category = category if category in ZONE_CATEGORIES else "analysis"
+    if severity not in ("high", "medium", "low"):
+        severity = "medium"
+    cell_size_deg = max(0.1, min(cell_size_deg, 10.0))
+
+    zone: dict[str, Any] = {
+        "id": str(uuid.uuid4())[:12],
+        "lat": lat,
+        "lng": lng,
+        "type": "analysis_zone",
+        "category": category,
+        "severity": severity,
+        "score": {"high": 90, "medium": 60, "low": 30}.get(severity, 60),
+        "title": title[:200],
+        "body": body[:2000],
+        "drivers": (drivers or [title])[:5],
+        "cell_size": cell_size_deg,
+        "source": source,
+        "created_at": time.time(),
+        "ttl_hours": ttl_hours,
+    }
+
+    with _lock:
+        _expire()
+        _zones.append(zone)
+        _save()
+
+    logger.info("Analysis zone created: %s at (%.2f, %.2f)", title[:40], lat, lng)
+    return zone
+
+
+def list_zones() -> list[dict[str, Any]]:
+    """Return all live (non-expired) zones."""
+    with _lock:
+        _expire()
+        return list(_zones)
+
+
+def get_zone(zone_id: str) -> dict[str, Any] | None:
+    """Get a single zone by ID."""
+    with _lock:
+        for z in _zones:
+            if z["id"] == zone_id:
+                return dict(z)
+    return None
+
+
+def delete_zone(zone_id: str) -> bool:
+    """Delete a zone by ID. Returns True if found and removed."""
+    with _lock:
+        before = len(_zones)
+        _zones[:] = [z for z in _zones if z["id"] != zone_id]
+        if len(_zones) < before:
+            _save()
+            return True
+    return False
+
+
+def clear_zones(*, source: str | None = None) -> int:
+    """Clear all zones, optionally filtered by source. Returns count removed."""
+    with _lock:
+        before = len(_zones)
+        if source:
+            _zones[:] = [z for z in _zones if z.get("source") != source]
+        else:
+            _zones.clear()
+        removed = before - len(_zones)
+        if removed:
+            _save()
+        return removed
+
+
+def get_live_zones() -> list[dict[str, Any]]:
+    """Return zones formatted for the correlation engine merge.
+
+    This is called by compute_correlations() to inject agent-placed zones
+    into the correlations list that the frontend renders as map squares.
+    """
+    with _lock:
+        _expire()
+        return [dict(z) for z in _zones]

backend/services/api_settings.py

@@ -5,10 +5,20 @@ Keys are stored in the backend .env file and loaded via python-dotenv.
 
 import os
 import re
+import tempfile
 from pathlib import Path
 
 # Path to the backend .env file
 ENV_PATH = Path(__file__).parent.parent / ".env"
+# Path to the example template that ships with the repo
+ENV_EXAMPLE_PATH = Path(__file__).parent.parent.parent / ".env.example"
+DATA_DIR = Path(os.environ.get("SB_DATA_DIR", str(Path(__file__).parent.parent / "data")))
+if not DATA_DIR.is_absolute():
+    DATA_DIR = Path(__file__).parent.parent / DATA_DIR
+OPERATOR_KEYS_ENV_PATH = Path(
+    os.environ.get("SHADOWBROKER_OPERATOR_KEYS_ENV", str(DATA_DIR / "operator_api_keys.env"))
+)
+_ENV_KEY_RE = re.compile(r"^[A-Z][A-Z0-9_]*$")
 
 # ---------------------------------------------------------------------------
 # API Registry — every external service the dashboard depends on
@@ -41,6 +51,15 @@ API_REGISTRY = [
         "url": "https://aisstream.io/",
         "required": True,
     },
+    {
+        "id": "gfw_api_token",
+        "env_key": "GFW_API_TOKEN",
+        "name": "Global Fishing Watch",
+        "description": "Bearer token for Global Fishing Watch fishing-vessel activity events (Fishing Activity map layer). Free registration at globalfishingwatch.org.",
+        "category": "Maritime",
+        "url": "https://globalfishingwatch.org/our-apis/",
+        "required": False,
+    },
     {
         "id": "adsb_lol",
         "env_key": None,
@@ -140,18 +159,145 @@ API_REGISTRY = [
         "url": "https://finnhub.io/register",
         "required": False,
     },
+    # Issue #298 (tg12): Sentinel Hub / Copernicus Data Space Ecosystem
+    # credentials were previously held in browser localStorage / sessionStorage
+    # by the Settings panel. Moved server-side to the same .env-backed
+    # store every other third-party API key lives in. The Sentinel proxy
+    # routes (POST /api/sentinel/token, /tile) now fall back to these
+    # env values when the request body omits credentials — see
+    # backend/routers/tools.py for the resolution order.
+    {
+        "id": "sentinel_client_id",
+        "env_key": "SENTINEL_CLIENT_ID",
+        "name": "Sentinel Hub / Copernicus — Client ID",
+        "description": "OAuth2 client ID for Copernicus Data Space Ecosystem (CDSE). Required for the Sentinel-2 imagery overlay and the right-click Sentinel-2 Intel Card. Sign in at dataspace.copernicus.eu and create OAuth credentials.",
+        "category": "Imagery",
+        "url": "https://dataspace.copernicus.eu/",
+        "required": False,
+    },
+    {
+        "id": "sentinel_client_secret",
+        "env_key": "SENTINEL_CLIENT_SECRET",
+        "name": "Sentinel Hub / Copernicus — Client Secret",
+        "description": "OAuth2 client secret paired with the Client ID above. Used by the backend to mint short-lived access tokens against the CDSE identity provider. Stored in the backend .env; never sent to the browser.",
+        "category": "Imagery",
+        "url": "https://dataspace.copernicus.eu/",
+        "required": False,
+    },
 ]
 
+ALLOWED_ENV_KEYS = {
+    str(api["env_key"])
+    for api in API_REGISTRY
+    if api.get("env_key")
+}
 
-def _obfuscate(value: str) -> str:
-    """Show first 4 chars, mask the rest with bullets."""
-    if not value or len(value) <= 4:
-        return "••••••••"
-    return value[:4] + "•" * (len(value) - 4)
+
+def _parse_env_file(path: Path) -> dict[str, str]:
+    values: dict[str, str] = {}
+    if not path.exists():
+        return values
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError:
+        return values
+    for raw_line in text.splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        key = key.strip()
+        if not _ENV_KEY_RE.match(key):
+            continue
+        value = value.strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
+            value = value[1:-1]
+        values[key] = value
+    return values
+
+
+def _quote_env_value(value: str) -> str:
+    escaped = value.replace("\\", "\\\\").replace('"', '\\"')
+    return f'"{escaped}"'
+
+
+def _write_env_values(path: Path, updates: dict[str, str]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    lines = path.read_text(encoding="utf-8").splitlines() if path.exists() else []
+    seen: set[str] = set()
+    next_lines: list[str] = []
+    for raw_line in lines:
+        stripped = raw_line.strip()
+        if "=" not in stripped or stripped.startswith("#"):
+            next_lines.append(raw_line)
+            continue
+        key = stripped.split("=", 1)[0].strip()
+        if key in updates:
+            next_lines.append(f"{key}={_quote_env_value(updates[key])}")
+            seen.add(key)
+        else:
+            next_lines.append(raw_line)
+    for key, value in updates.items():
+        if key not in seen:
+            next_lines.append(f"{key}={_quote_env_value(value)}")
+
+    fd, tmp_name = tempfile.mkstemp(dir=str(path.parent), prefix=f"{path.name}.tmp.", text=True)
+    tmp_path = Path(tmp_name)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
+            handle.write("\n".join(next_lines).rstrip() + "\n")
+        if os.name != "nt":
+            os.chmod(tmp_path, 0o600)
+        os.replace(tmp_path, path)
+        if os.name != "nt":
+            os.chmod(path, 0o600)
+    finally:
+        try:
+            if tmp_path.exists():
+                tmp_path.unlink()
+        except OSError:
+            pass
+
+
+def load_persisted_api_keys_into_environ() -> None:
+    """Load persisted operator API keys if no process env value exists."""
+    for key, value in _parse_env_file(OPERATOR_KEYS_ENV_PATH).items():
+        if key in ALLOWED_ENV_KEYS and value and not os.environ.get(key):
+            os.environ[key] = value
+
+
+def get_env_path_info() -> dict:
+    """Return absolute paths for the backend .env and .env.example template.
+
+    Surfaced to the frontend so the API Keys settings panel can tell users
+    exactly where to put their keys when in-app editing fails (admin-not-set,
+    file permissions, read-only filesystem, etc.).
+    """
+    env_path = ENV_PATH.resolve()
+    example_path = ENV_EXAMPLE_PATH.resolve()
+    return {
+        "env_path": str(env_path),
+        "env_path_exists": env_path.exists(),
+        "env_path_writable": os.access(env_path.parent, os.W_OK)
+            and (not env_path.exists() or os.access(env_path, os.W_OK)),
+        "env_example_path": str(example_path),
+        "env_example_path_exists": example_path.exists(),
+        "operator_keys_env_path": str(OPERATOR_KEYS_ENV_PATH.resolve()),
+        "operator_keys_env_path_exists": OPERATOR_KEYS_ENV_PATH.exists(),
+        "operator_keys_env_path_writable": os.access(OPERATOR_KEYS_ENV_PATH.parent, os.W_OK)
+            and (not OPERATOR_KEYS_ENV_PATH.exists() or os.access(OPERATOR_KEYS_ENV_PATH, os.W_OK)),
+    }
 
 
 def get_api_keys():
-    """Return the full API registry with obfuscated key values."""
+    """Return the API registry with a binary set/unset flag per key.
+
+    Key values themselves are NEVER returned to the client — not even an
+    obfuscated prefix. Users edit the .env file directly; the panel uses
+    `is_set` to render a CONFIGURED / NOT CONFIGURED badge and the path
+    info from `get_env_path_info()` to tell them where to put each key.
+    """
+    load_persisted_api_keys_into_environ()
     result = []
     for api in API_REGISTRY:
         entry = {
@@ -163,41 +309,64 @@ def get_api_keys():
             "required": api["required"],
             "has_key": api["env_key"] is not None,
             "env_key": api["env_key"],
-            "value_obfuscated": None,
             "is_set": False,
         }
         if api["env_key"]:
             raw = os.environ.get(api["env_key"], "")
-            entry["value_obfuscated"] = _obfuscate(raw)
             entry["is_set"] = bool(raw)
         result.append(entry)
     return result
 
 
-def update_api_key(env_key: str, new_value: str) -> bool:
-    """Update a single key in the .env file and in the current process env."""
-    valid_keys = {api["env_key"] for api in API_REGISTRY if api.get("env_key")}
-    if env_key not in valid_keys:
-        return False
+def save_api_keys(updates: dict[str, str]) -> dict:
+    """Persist allowed API keys from a local operator request.
 
-    if not isinstance(new_value, str):
-        return False
-    if "\n" in new_value or "\r" in new_value:
-        return False
+    Values are accepted write-only: the response includes only configured flags.
+    """
+    clean: dict[str, str] = {}
+    for key, value in updates.items():
+        env_key = str(key or "").strip().upper()
+        if env_key not in ALLOWED_ENV_KEYS:
+            continue
+        clean_value = str(value or "").strip()
+        if clean_value:
+            clean[env_key] = clean_value
+    if not clean:
+        return {"ok": False, "detail": "No supported API keys were provided."}
 
-    if not ENV_PATH.exists():
-        ENV_PATH.write_text("", encoding="utf-8")
+    _write_env_values(OPERATOR_KEYS_ENV_PATH, clean)
+    try:
+        _write_env_values(ENV_PATH, clean)
+    except OSError:
+        # The persistent operator key file is the source of truth for Docker.
+        pass
+    for key, value in clean.items():
+        os.environ[key] = value
+    if "AIS_API_KEY" in clean:
+        try:
+            from services import ais_stream
+            ais_stream.API_KEY = clean["AIS_API_KEY"]
+        except Exception:
+            pass
+    if "OPENSKY_CLIENT_ID" in clean or "OPENSKY_CLIENT_SECRET" in clean:
+        try:
+            from services.fetchers import flights
+            flights.opensky_client.client_id = os.environ.get("OPENSKY_CLIENT_ID", "")
+            flights.opensky_client.client_secret = os.environ.get("OPENSKY_CLIENT_SECRET", "")
+            flights.opensky_client.token = None
+            flights.opensky_client.expires_at = 0
+        except Exception:
+            pass
 
-    # Update os.environ immediately
-    os.environ[env_key] = new_value
+    try:
+        from services.config import get_settings
+        get_settings.cache_clear()
+    except Exception:
+        pass
 
-    # Update the .env file on disk
-    content = ENV_PATH.read_text(encoding="utf-8")
-    pattern = re.compile(rf"^{re.escape(env_key)}=.*$", re.MULTILINE)
-    if pattern.search(content):
-        content = pattern.sub(f"{env_key}={new_value}", content)
-    else:
-        content = content.rstrip("\n") + f"\n{env_key}={new_value}\n"
-
-    ENV_PATH.write_text(content, encoding="utf-8")
-    return True
+    return {
+        "ok": True,
+        "updated": sorted(clean.keys()),
+        "keys": get_api_keys(),
+        "env": get_env_path_info(),
+    }

backend/services/carrier_tracker.py

@@ -1,46 +1,90 @@
 """
 Carrier Strike Group OSINT Tracker
 ===================================
-Scrapes multiple OSINT sources to maintain current estimated positions
-for US Navy Carrier Strike Groups. Updates on startup + 00:00 & 12:00 UTC.
+Maintains estimated positions for US Navy Carrier Strike Groups with
+honest provenance and freshness signals.
 
-Sources:
-  1. GDELT News API — recent carrier movement headlines
-  2. WikiVoyage / public port-call databases
-  3. Fallback — last-known or static OSINT estimates
+Issues #244 / #245 / #246 (tg12 external audit):
+
+The previous implementation baked a snapshot of USNI News Fleet &
+Marine Tracker positions (March 9, 2026) into the registry as
+``fallback_lat``/``fallback_lng`` and stamped ``updated = now()``
+every time the dossier was rendered. That presented stale editorial
+data as live state. It also persisted GDELT-derived positions to the
+on-disk cache with no freshness signal, so a single news mention from
+months ago could keep overriding the (already-stale) registry default
+indefinitely.
+
+Architecture after this PR:
+
+::
+
+    backend/data/carrier_seed.json   read-only, shipped with image,
+                                     used ONCE on first-ever startup
+                                     to bootstrap carrier_cache.json.
+
+    backend/data/carrier_cache.json  mutable, lives in the runtime data
+                                     volume, written by every GDELT
+                                     refresh + any future source.
+
+Startup flow:
+
+1.  ``carrier_cache.json`` exists?  → load it.
+2.  Otherwise, copy ``carrier_seed.json`` → ``carrier_cache.json``,
+    then load it. (This happens once, ever, per install.)
+3.  Background: GDELT fetch runs. Any carrier mentioned in fresh news
+    gets its entry replaced with the news-derived position.
+    ``position_source_at`` is set to the news article timestamp.
+
+Freshness is a *labelling* decision, not an eviction decision:
+
+- ``position_source_at`` within the configurable freshness window
+  (default 14 days) → ``position_confidence = "recent"``.
+- Older than that              → ``position_confidence = "stale"``.
+- Bootstrapped from the seed file (never updated) → ``"seed"``.
+- No cache entry at all (e.g. a carrier added to the registry after
+  first install) → carrier renders at its homeport with
+  ``"homeport_default"``.
+
+Carriers are never hidden, never teleported, never disappeared. The
+position the user sees is always the last position the system actually
+observed, with an honest "as-of" timestamp the UI can render however
+it likes. A year from now, the runtime cache reflects whatever this
+install has observed via GDELT — not the seed snapshot.
 """
 
-import re
+import os
 import json
 import time
 import logging
 import threading
 import random
-from datetime import datetime, timezone
+import shutil
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 from services.network_utils import fetch_with_curl
 
 logger = logging.getLogger(__name__)
 
 # -----------------------------------------------------------------
-# Carrier registry: hull number → metadata + fallback position
+# Carrier registry: hull number → identity only.
+#
+# Issue #244 (tg12): the previous registry carried hard-coded
+# ``fallback_lat``/``fallback_lng`` that were dated editorial
+# snapshots from a 2026-03-09 article. Those fields are DELETED. The
+# registry is now identity + homeport only; positions are sourced
+# exclusively from carrier_cache.json (and via that, from the
+# bootstrap seed or live OSINT).
 # -----------------------------------------------------------------
 CARRIER_REGISTRY: Dict[str, dict] = {
-    # Fallback positions sourced from USNI News Fleet & Marine Tracker (Mar 9, 2026)
-    # https://news.usni.org/2026/03/09/usni-news-fleet-and-marine-tracker-march-9-2026
     # --- Bremerton, WA (Naval Base Kitsap) ---
-    # Distinct pier positions along Sinclair Inlet so carriers don't stack
     "CVN-68": {
         "name": "USS Nimitz (CVN-68)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Nimitz",
         "homeport": "Bremerton, WA",
         "homeport_lat": 47.5535,
         "homeport_lng": -122.6400,
-        "fallback_lat": 47.5535,
-        "fallback_lng": -122.6400,
-        "fallback_heading": 90,
-        "fallback_desc": "Bremerton, WA (Maintenance)",
     },
     "CVN-76": {
         "name": "USS Ronald Reagan (CVN-76)",
@@ -48,23 +92,14 @@ CARRIER_REGISTRY: Dict[str, dict] = {
         "homeport": "Bremerton, WA",
         "homeport_lat": 47.5580,
         "homeport_lng": -122.6360,
-        "fallback_lat": 47.5580,
-        "fallback_lng": -122.6360,
-        "fallback_heading": 90,
-        "fallback_desc": "Bremerton, WA (Decommissioning)",
     },
     # --- Norfolk, VA (Naval Station Norfolk) ---
-    # Piers run N-S along Willoughby Bay; each carrier gets a distinct berth
     "CVN-69": {
         "name": "USS Dwight D. Eisenhower (CVN-69)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Dwight_D._Eisenhower",
         "homeport": "Norfolk, VA",
         "homeport_lat": 36.9465,
         "homeport_lng": -76.3265,
-        "fallback_lat": 36.9465,
-        "fallback_lng": -76.3265,
-        "fallback_heading": 0,
-        "fallback_desc": "Norfolk, VA (Post-deployment maintenance)",
     },
     "CVN-78": {
         "name": "USS Gerald R. Ford (CVN-78)",
@@ -72,10 +107,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
         "homeport": "Norfolk, VA",
         "homeport_lat": 36.9505,
         "homeport_lng": -76.3250,
-        "fallback_lat": 18.0,
-        "fallback_lng": 39.5,
-        "fallback_heading": 0,
-        "fallback_desc": "Red Sea — Operation Epic Fury (USNI Mar 9)",
     },
     "CVN-74": {
         "name": "USS John C. Stennis (CVN-74)",
@@ -83,10 +114,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
         "homeport": "Norfolk, VA",
         "homeport_lat": 36.9540,
         "homeport_lng": -76.3235,
-        "fallback_lat": 36.98,
-        "fallback_lng": -76.43,
-        "fallback_heading": 0,
-        "fallback_desc": "Newport News, VA (RCOH refueling overhaul)",
     },
     "CVN-75": {
         "name": "USS Harry S. Truman (CVN-75)",
@@ -94,10 +121,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
         "homeport": "Norfolk, VA",
         "homeport_lat": 36.9580,
         "homeport_lng": -76.3220,
-        "fallback_lat": 36.0,
-        "fallback_lng": 15.0,
-        "fallback_heading": 0,
-        "fallback_desc": "Mediterranean Sea deployment (USNI Mar 9)",
     },
     "CVN-77": {
         "name": "USS George H.W. Bush (CVN-77)",
@@ -105,23 +128,14 @@ CARRIER_REGISTRY: Dict[str, dict] = {
         "homeport": "Norfolk, VA",
         "homeport_lat": 36.9620,
         "homeport_lng": -76.3210,
-        "fallback_lat": 36.5,
-        "fallback_lng": -74.0,
-        "fallback_heading": 0,
-        "fallback_desc": "Atlantic — Pre-deployment workups (USNI Mar 9)",
     },
     # --- San Diego, CA (Naval Base San Diego) ---
-    # Carrier piers along the east shore of San Diego Bay, spread N-S
     "CVN-70": {
         "name": "USS Carl Vinson (CVN-70)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Carl_Vinson",
         "homeport": "San Diego, CA",
         "homeport_lat": 32.6840,
         "homeport_lng": -117.1290,
-        "fallback_lat": 32.6840,
-        "fallback_lng": -117.1290,
-        "fallback_heading": 180,
-        "fallback_desc": "San Diego, CA (Homeport)",
     },
     "CVN-71": {
         "name": "USS Theodore Roosevelt (CVN-71)",
@@ -129,10 +143,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
         "homeport": "San Diego, CA",
         "homeport_lat": 32.6885,
         "homeport_lng": -117.1280,
-        "fallback_lat": 32.6885,
-        "fallback_lng": -117.1280,
-        "fallback_heading": 180,
-        "fallback_desc": "San Diego, CA (Maintenance)",
     },
     "CVN-72": {
         "name": "USS Abraham Lincoln (CVN-72)",
@@ -140,10 +150,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
         "homeport": "San Diego, CA",
         "homeport_lat": 32.6925,
         "homeport_lng": -117.1275,
-        "fallback_lat": 20.0,
-        "fallback_lng": 64.0,
-        "fallback_heading": 0,
-        "fallback_desc": "Arabian Sea — Operation Epic Fury (USNI Mar 9)",
     },
     # --- Yokosuka, Japan (CFAY) ---
     "CVN-73": {
@@ -152,16 +158,18 @@ CARRIER_REGISTRY: Dict[str, dict] = {
         "homeport": "Yokosuka, Japan",
         "homeport_lat": 35.2830,
         "homeport_lng": 139.6700,
-        "fallback_lat": 35.2830,
-        "fallback_lng": 139.6700,
-        "fallback_heading": 180,
-        "fallback_desc": "Yokosuka, Japan (Forward deployed)",
     },
 }
 
 # -----------------------------------------------------------------
-# Region → approximate center coordinates
-# Used to map textual geographic descriptions to lat/lng
+# Region → approximate center coordinates.
+#
+# Issue #245 (tg12): converting a region name straight into precise
+# map coordinates is false precision. We still use this table to
+# infer a coarse position from a headline mention, but the resulting
+# carrier object is now stamped ``position_confidence = "approximate"``
+# so the UI can render an uncertainty radius / dimmed icon. The
+# centroid is a best-effort midpoint of the named body of water.
 # -----------------------------------------------------------------
 REGION_COORDS: Dict[str, tuple] = {
     # Oceans & Seas
@@ -220,9 +228,39 @@ REGION_COORDS: Dict[str, tuple] = {
 }
 
 # -----------------------------------------------------------------
-# Cache file for persisting positions between restarts
+# Files
 # -----------------------------------------------------------------
-CACHE_FILE = Path(__file__).parent.parent / "carrier_cache.json"
+#
+# The seed lives in the read-only image data dir (it ships with each
+# release). The cache lives in the same data dir but is written at
+# runtime; under Docker compose this dir is volume-mounted so the
+# cache persists across container restarts, which is the whole point
+# of the seed-then-observe model — the user's runtime observations
+# survive image upgrades.
+SEED_FILE = Path(__file__).parent.parent / "data" / "carrier_seed.json"
+CACHE_FILE = Path(__file__).parent.parent / "data" / "carrier_cache.json"
+
+# -----------------------------------------------------------------
+# Freshness window for position_confidence labeling. Issue #246 (tg12):
+# previously persisted cache entries had no freshness signal at all.
+# After this change, the position itself is preserved (we never lose
+# what was last observed) but the confidence label flips from
+# "recent" to "stale" once the underlying source is older than this
+# window. Operator-overridable via env var.
+# -----------------------------------------------------------------
+_DEFAULT_FRESHNESS_WINDOW_DAYS = 14
+
+
+def _freshness_window_days() -> int:
+    raw = str(os.environ.get("SHADOWBROKER_CARRIER_FRESHNESS_DAYS", "") or "").strip()
+    if not raw:
+        return _DEFAULT_FRESHNESS_WINDOW_DAYS
+    try:
+        n = int(raw)
+        return n if n > 0 else _DEFAULT_FRESHNESS_WINDOW_DAYS
+    except (TypeError, ValueError):
+        return _DEFAULT_FRESHNESS_WINDOW_DAYS
+
 
 _carrier_positions: Dict[str, dict] = {}
 _positions_lock = threading.Lock()
@@ -234,25 +272,159 @@ _GDELT_REQUEST_DELAY_SECONDS = 1.25
 _GDELT_REQUEST_JITTER_SECONDS = 0.35
 
 
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _parse_iso(ts: str) -> Optional[datetime]:
+    if not ts:
+        return None
+    try:
+        # Python's fromisoformat accepts +00:00 but not 'Z' until 3.11.
+        normalized = ts.replace("Z", "+00:00")
+        dt = datetime.fromisoformat(normalized)
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        return dt
+    except (TypeError, ValueError):
+        return None
+
+
+def _compute_position_confidence(entry: dict, *, now: Optional[datetime] = None) -> str:
+    """Return the public confidence label for a carrier cache entry.
+
+    Order of precedence:
+      - explicit "homeport_default" / "seed" labels are preserved.
+      - dated entries (with position_source_at) are "recent" if within
+        the configured freshness window, else "stale".
+      - missing position_source_at falls through to "stale".
+    """
+    raw_label = str(entry.get("position_confidence", "") or "").strip()
+    # Explicit "kind of provenance" labels are preserved as-is. They
+    # describe HOW we got the position, not WHEN — a fresh headline-to-
+    # centroid match (#245) is still imprecise no matter how recently
+    # it was observed, and the seed (#244) is always the seed.
+    if raw_label in {"seed", "homeport_default", "approximate"}:
+        # Approximate entries can still age into "stale_approximate" if
+        # they fall out of the freshness window — that distinction lets
+        # the UI render a different badge for old-and-imprecise vs
+        # recent-and-imprecise. seed/homeport_default never age (they
+        # were never timestamped against real observations).
+        if raw_label == "approximate":
+            source_at = _parse_iso(str(entry.get("position_source_at", "") or ""))
+            if source_at is not None:
+                reference = now or datetime.now(timezone.utc)
+                if reference - source_at > timedelta(days=_freshness_window_days()):
+                    return "stale_approximate"
+        return raw_label
+
+    source_at = _parse_iso(str(entry.get("position_source_at", "") or ""))
+    if not source_at:
+        return "stale"
+
+    reference = now or datetime.now(timezone.utc)
+    window = timedelta(days=_freshness_window_days())
+    if reference - source_at <= window:
+        return "recent"
+    return "stale"
+
+
+def _load_seed() -> Dict[str, dict]:
+    """Load the read-only seed file shipped with the image.
+
+    Returns a hull→entry dict (no _meta wrapper). Missing or malformed
+    seed files yield an empty dict — the caller falls back to homeport
+    defaults.
+    """
+    try:
+        if not SEED_FILE.exists():
+            logger.info("Carrier seed file not present at %s; first-run will fall back to homeport defaults", SEED_FILE)
+            return {}
+        raw = json.loads(SEED_FILE.read_text(encoding="utf-8"))
+        carriers = raw.get("carriers", {}) if isinstance(raw, dict) else {}
+        if not isinstance(carriers, dict):
+            return {}
+        logger.info("Carrier seed loaded: %d entries from %s", len(carriers), SEED_FILE)
+        return carriers
+    except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
+        logger.warning("Failed to load carrier seed file %s: %s", SEED_FILE, e)
+        return {}
+
+
 def _load_cache() -> Dict[str, dict]:
-    """Load cached carrier positions from disk."""
+    """Load the mutable cache (last-known positions persisted between restarts)."""
     try:
         if CACHE_FILE.exists():
-            data = json.loads(CACHE_FILE.read_text())
-            logger.info(f"Carrier cache loaded: {len(data)} carriers from {CACHE_FILE}")
-            return data
+            data = json.loads(CACHE_FILE.read_text(encoding="utf-8"))
+            if isinstance(data, dict):
+                logger.info("Carrier cache loaded: %d carriers from %s", len(data), CACHE_FILE)
+                return data
     except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
-        logger.warning(f"Failed to load carrier cache: {e}")
+        logger.warning("Failed to load carrier cache: %s", e)
     return {}
 
 
-def _save_cache(positions: Dict[str, dict]):
-    """Persist carrier positions to disk."""
+def _save_cache(positions: Dict[str, dict]) -> None:
+    """Persist the mutable cache. Atomic write (temp + rename) so a crash
+    mid-write can't leave the file truncated."""
     try:
-        CACHE_FILE.write_text(json.dumps(positions, indent=2))
-        logger.info(f"Carrier cache saved: {len(positions)} carriers")
+        CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
+        tmp = CACHE_FILE.with_suffix(CACHE_FILE.suffix + ".tmp")
+        tmp.write_text(json.dumps(positions, indent=2), encoding="utf-8")
+        # On Windows os.replace is atomic and overwrites existing files.
+        os.replace(tmp, CACHE_FILE)
+        logger.info("Carrier cache saved: %d carriers", len(positions))
     except (IOError, OSError) as e:
-        logger.warning(f"Failed to save carrier cache: {e}")
+        logger.warning("Failed to save carrier cache: %s", e)
+
+
+def _homeport_entry_for(hull: str) -> Optional[dict]:
+    """Return a homeport-default cache entry for a hull, or None if the
+    hull is not in the registry."""
+    info = CARRIER_REGISTRY.get(hull)
+    if not info:
+        return None
+    return {
+        "lat": info["homeport_lat"],
+        "lng": info["homeport_lng"],
+        "heading": 0,
+        "desc": f"{info['homeport']} (no observations yet)",
+        "source": f"Homeport default ({info['homeport']})",
+        "source_url": info.get("wiki", ""),
+        "position_source_at": _now_iso(),
+        "position_confidence": "homeport_default",
+    }
+
+
+def _bootstrap_cache_if_missing() -> Dict[str, dict]:
+    """One-shot: if no cache exists, materialize one from the seed file.
+
+    Returns the cache contents (hull→entry). On first-ever startup,
+    this writes ``carrier_cache.json`` so subsequent restarts skip the
+    seed entirely. Operator-deleted caches re-bootstrap the same way —
+    operators can use that to "reset" carrier positions, but it's an
+    explicit operator action.
+    """
+    if CACHE_FILE.exists():
+        return _load_cache()
+
+    seed = _load_seed()
+    if not seed:
+        # No seed file either. Build a homeport-default cache so the
+        # first save_cache call still produces something honest.
+        homeports: Dict[str, dict] = {}
+        for hull in CARRIER_REGISTRY:
+            entry = _homeport_entry_for(hull)
+            if entry is not None:
+                homeports[hull] = entry
+        if homeports:
+            _save_cache(homeports)
+        return homeports
+
+    # Persist the seed as the first cache so subsequent runs skip this branch.
+    _save_cache(seed)
+    logger.info("Carrier cache bootstrapped from seed (first-ever startup)")
+    return dict(seed)
 
 
 def _match_region(text: str) -> Optional[tuple]:
@@ -270,10 +442,8 @@ def _match_carrier(text: str) -> Optional[str]:
     for hull, info in CARRIER_REGISTRY.items():
         hull_check = hull.lower().replace("-", "")
         name_parts = info["name"].lower()
-        # Match hull number (e.g., "CVN-78", "CVN78")
         if hull.lower() in text_lower or hull_check in text_lower.replace("-", ""):
             return hull
-        # Match ship name (e.g., "Ford", "Eisenhower", "Vinson")
         ship_name = name_parts.split("(")[0].strip()
         last_name = ship_name.split()[-1] if ship_name else ""
         if last_name and len(last_name) > 3 and last_name in text_lower:
@@ -323,8 +493,9 @@ def _fetch_gdelt_carrier_news() -> List[dict]:
             articles = data.get("articles", [])
             for art in articles:
                 title = art.get("title", "")
-                url = art.get("url", "")
-                results.append({"title": title, "url": url})
+                article_url = art.get("url", "")
+                article_at = art.get("seendate") or art.get("date") or ""
+                results.append({"title": title, "url": article_url, "seendate": article_at})
         except (ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
             logger.debug(f"GDELT search failed for '{term}': {e}")
             continue
@@ -340,108 +511,175 @@ def _fetch_gdelt_carrier_news() -> List[dict]:
     return results
 
 
+def _gdelt_seendate_to_iso(seendate: str) -> Optional[str]:
+    """GDELT returns YYYYMMDDhhmmss (UTC). Convert to ISO8601 for
+    position_source_at. Returns None if the input is unparseable."""
+    raw = (seendate or "").strip()
+    if len(raw) < 8 or not raw.isdigit():
+        return None
+    try:
+        dt = datetime.strptime(raw[:14] if len(raw) >= 14 else raw[:8] + "000000", "%Y%m%d%H%M%S")
+        return dt.replace(tzinfo=timezone.utc).isoformat()
+    except (TypeError, ValueError):
+        return None
+
+
 def _parse_carrier_positions_from_news(articles: List[dict]) -> Dict[str, dict]:
-    """Parse carrier positions from news article titles and descriptions."""
+    """Parse carrier positions from news article titles.
+
+    Issue #245 (tg12): the position is a region centroid, which is
+    coarse — we now stamp ``position_confidence = "approximate"`` so
+    the UI can render that uncertainty. Issue #244: the
+    ``position_source_at`` field is the news article's actual seen
+    date, NOT now(), so the freshness check correctly flips entries
+    to "stale" once they age past the configured window.
+    """
     updates: Dict[str, dict] = {}
 
     for article in articles:
         title = article.get("title", "")
-
-        # Try to match a carrier from the title
         hull = _match_carrier(title)
         if not hull:
             continue
-
-        # Try to match a region from the title
         coords = _match_region(title)
         if not coords:
             continue
 
-        # Only update if we haven't seen this carrier yet (first match wins — most recent)
+        # First match wins (most recent article, GDELT returns newest first
+        # per term).
         if hull not in updates:
+            iso_at = _gdelt_seendate_to_iso(str(article.get("seendate", ""))) or _now_iso()
             updates[hull] = {
                 "lat": coords[0],
                 "lng": coords[1],
+                "heading": 0,
                 "desc": title[:100],
-                "source": "GDELT News API",
+                "source": "GDELT News API (headline region match — approximate)",
                 "source_url": article.get("url", "https://api.gdeltproject.org"),
-                "updated": datetime.now(timezone.utc).isoformat(),
+                "position_source_at": iso_at,
+                # Headline-to-centroid match is explicitly approximate.
+                "position_confidence": "approximate",
             }
             logger.info(
-                f"Carrier update: {CARRIER_REGISTRY[hull]['name']} → {coords} (from: {title[:80]})"
+                "Carrier update: %s → %s (from: %s)",
+                CARRIER_REGISTRY[hull]["name"],
+                coords,
+                title[:80],
             )
 
     return updates
 
 
-def _load_carrier_fallbacks() -> Dict[str, dict]:
-    """Build carrier positions from static fallbacks + disk cache (instant, no network)."""
-    positions: Dict[str, dict] = {}
-    for hull, info in CARRIER_REGISTRY.items():
-        positions[hull] = {
-            "name": info["name"],
-            "lat": info["fallback_lat"],
-            "lng": info["fallback_lng"],
-            "heading": info["fallback_heading"],
-            "desc": info["fallback_desc"],
-            "wiki": info["wiki"],
-            "source": "USNI News Fleet & Marine Tracker",
-            "source_url": "https://news.usni.org/category/fleet-tracker",
-            "updated": datetime.now(timezone.utc).isoformat(),
-        }
-
-    # Overlay cached positions from previous runs (may have GDELT data)
-    cached = _load_cache()
-    for hull, cached_pos in cached.items():
-        if hull in positions:
-            if cached_pos.get("source", "").startswith("GDELT") or cached_pos.get(
-                "source", ""
-            ).startswith("News"):
-                positions[hull].update(
-                    {
-                        "lat": cached_pos["lat"],
-                        "lng": cached_pos["lng"],
-                        "desc": cached_pos.get("desc", positions[hull]["desc"]),
-                        "source": cached_pos.get("source", "Cached OSINT"),
-                        "updated": cached_pos.get("updated", ""),
-                    }
-                )
-    return positions
+def _enrich_for_rendering(hull: str, entry: dict, *, now: Optional[datetime] = None) -> dict:
+    """Add live computed fields (confidence label, last_osint_update)
+    on top of the persisted cache entry. The persisted entry is left
+    untouched; this function builds the public-facing object.
+    """
+    info = CARRIER_REGISTRY.get(hull, {})
+    confidence = _compute_position_confidence(entry, now=now)
+    return {
+        "name": entry.get("name", info.get("name", hull)),
+        "lat": entry["lat"],
+        "lng": entry["lng"],
+        "heading": entry.get("heading", 0),
+        "desc": entry.get("desc", ""),
+        "wiki": entry.get("wiki", info.get("wiki", "")),
+        "source": entry.get("source", "OSINT estimated position"),
+        "source_url": entry.get("source_url", ""),
+        "position_source_at": entry.get("position_source_at", ""),
+        "position_confidence": confidence,
+        # Existing field preserved for backward compatibility with the
+        # current frontend ShipPopup; now reflects the SOURCE's observed
+        # time (not now()), so "last reported X days ago" is honest.
+        "last_osint_update": entry.get("position_source_at", ""),
+        # Convenience boolean for the UI: true when the position is
+        # NOT live OSINT (used to render dimmed icons / badges).
+        "is_fallback": confidence in {"seed", "stale", "stale_approximate", "homeport_default"},
+    }
 
 
-def update_carrier_positions():
-    """Main update function — called on startup and every 12h.
+def update_carrier_positions() -> None:
+    """Refresh carrier positions.
 
-    Phase 1 (instant): publish fallback + cached positions so the map has carriers immediately.
-    Phase 2 (slow):    query GDELT for fresh OSINT positions and update in-place.
+    Phase 1 (instant): publish whatever's in carrier_cache.json (or
+    bootstrap from seed on first-ever run), so the map has carriers
+    immediately.
+
+    Phase 2 (slow): query GDELT and replace position entries for any
+    carrier mentioned in fresh news. Persist back to cache.
     """
     global _last_update
 
-    # --- Phase 1: instant fallback + cache ---
-    positions = _load_carrier_fallbacks()
+    # --- Phase 1: instant cache (bootstrap from seed on first-ever run) ---
+    positions = _bootstrap_cache_if_missing()
+
+    # Ensure every registered hull has SOMETHING in the cache. A hull
+    # the seed didn't cover (e.g. added after install) renders at its
+    # homeport with "homeport_default" confidence.
+    for hull in CARRIER_REGISTRY:
+        if hull not in positions:
+            entry = _homeport_entry_for(hull)
+            if entry is not None:
+                positions[hull] = entry
 
     with _positions_lock:
-        # Only overwrite if positions are currently empty (first startup).
-        # If we already have data from a previous cycle, keep it while GDELT runs.
         if not _carrier_positions:
             _carrier_positions.update(positions)
             _last_update = datetime.now(timezone.utc)
     logger.info(
-        f"Carrier tracker: {len(positions)} carriers loaded from fallback/cache (GDELT enrichment starting...)"
+        "Carrier tracker: %d carriers loaded from cache (USNI + GDELT enrichment starting...)",
+        len(positions),
     )
 
-    # --- Phase 2: slow GDELT enrichment ---
+    # --- Phase 2: USNI Fleet & Marine Tracker (PRIMARY source) ---
+    #
+    # USNI publishes a weekly editorial tracker with each carrier's
+    # actual operating area, parsed from explicit prose like
+    #   "The Gerald R. Ford Carrier Strike Group is operating in the Red Sea"
+    # These positions are tagged ``position_confidence: "recent"`` because
+    # they reflect actual reporting, not headline-keyword centroids.
+    # USNI updates are preferred over GDELT — they're authoritative on
+    # US Navy positions where GDELT is just article-title text mining.
+    try:
+        from services.fetchers.usni_fleet_tracker import (
+            fetch_latest_fleet_tracker_positions,
+        )
+        usni_positions = fetch_latest_fleet_tracker_positions()
+        for hull, pos in usni_positions.items():
+            positions[hull] = pos
+            logger.info(
+                "Carrier USNI update: %s → %s",
+                CARRIER_REGISTRY[hull]["name"],
+                pos.get("desc", ""),
+            )
+    except Exception as e:
+        logger.warning("USNI fleet-tracker fetch failed: %s", e)
+
+    # --- Phase 3: GDELT enrichment (SECONDARY — fills gaps) ---
+    #
+    # Used only to backfill carriers USNI didn't mention this week. The
+    # position is stamped ``approximate`` so the UI knows it's a
+    # headline-centroid match (Issue #245).
     try:
         articles = _fetch_gdelt_carrier_news()
         news_positions = _parse_carrier_positions_from_news(articles)
         for hull, pos in news_positions.items():
-            if hull in positions:
-                positions[hull].update(pos)
-                logger.info(f"Carrier OSINT: updated {CARRIER_REGISTRY[hull]['name']} from news")
+            # Only overwrite if the existing entry is NOT a recent USNI
+            # observation. A "recent" USNI position is higher-confidence
+            # than a GDELT headline-centroid match — don't let GDELT
+            # demote a real position to an approximate one.
+            existing = positions.get(hull, {})
+            existing_conf = _compute_position_confidence(existing)
+            if existing_conf == "recent":
+                continue
+            positions[hull] = pos
+            logger.info(
+                "Carrier OSINT: updated %s from GDELT news",
+                CARRIER_REGISTRY[hull]["name"],
+            )
     except (ValueError, KeyError, json.JSONDecodeError, OSError) as e:
-        logger.warning(f"GDELT carrier fetch failed: {e}")
+        logger.warning("GDELT carrier fetch failed: %s", e)
 
-    # Save and update the global state with enriched positions
     with _positions_lock:
         _carrier_positions.clear()
         _carrier_positions.update(positions)
@@ -449,21 +687,15 @@ def update_carrier_positions():
 
     _save_cache(positions)
 
-    sources = {}
-    for p in positions.values():
-        src = p.get("source", "unknown")
-        sources[src] = sources.get(src, 0) + 1
-    logger.info(f"Carrier tracker: {len(positions)} carriers updated. Sources: {sources}")
+    confidences: Dict[str, int] = {}
+    for entry in positions.values():
+        label = _compute_position_confidence(entry)
+        confidences[label] = confidences.get(label, 0) + 1
+    logger.info("Carrier tracker: %d carriers updated. Confidence: %s", len(positions), confidences)
 
 
 def _deconflict_positions(result: List[dict]) -> List[dict]:
-    """Offset carriers that share identical coordinates so they don't stack.
-
-    At port: offset along the pier axis (~500m / 0.004° apart).
-    At sea: offset perpendicular to each other (~0.08° / ~9km apart)
-    so they're visibly separate but clearly operating together.
-    """
-    # Group by rounded lat/lng (within ~0.01° ≈ 1km = same spot)
+    """Offset carriers that share identical coordinates so they don't stack."""
     from collections import defaultdict
 
     groups: dict[str, list[int]] = defaultdict(list)
@@ -475,7 +707,6 @@ def _deconflict_positions(result: List[dict]) -> List[dict]:
         if len(indices) < 2:
             continue
         n = len(indices)
-        # Determine if this is a port (near a homeport) or at sea
         sample = result[indices[0]]
         at_port = any(
             abs(sample["lat"] - info.get("homeport_lat", 0)) < 0.05
@@ -484,7 +715,6 @@ def _deconflict_positions(result: List[dict]) -> List[dict]:
         )
 
         if at_port:
-            # Use each carrier's distinct homeport pier coordinates
             for idx in indices:
                 carrier = result[idx]
                 hull = None
@@ -497,8 +727,7 @@ def _deconflict_positions(result: List[dict]) -> List[dict]:
                     carrier["lat"] = info["homeport_lat"]
                     carrier["lng"] = info["homeport_lng"]
         else:
-            # At sea: spread in a line perpendicular to travel (~0.08° apart)
-            spacing = 0.08  # ~9km — close enough to see they're together
+            spacing = 0.08
             start_offset = -(n - 1) * spacing / 2
             for j, idx in enumerate(indices):
                 result[idx]["lng"] += start_offset + j * spacing
@@ -507,36 +736,44 @@ def _deconflict_positions(result: List[dict]) -> List[dict]:
 
 
 def get_carrier_positions() -> List[dict]:
-    """Return current carrier positions for the data pipeline."""
+    """Return current carrier positions for the data pipeline.
+
+    Each entry has the full provenance + freshness fields; the UI can
+    decide how to render them. Carriers are never hidden — only
+    labeled.
+    """
+    now = datetime.now(timezone.utc)
     with _positions_lock:
-        result = []
-        for hull, pos in _carrier_positions.items():
-            info = CARRIER_REGISTRY.get(hull, {})
+        result: List[dict] = []
+        for hull, entry in _carrier_positions.items():
+            enriched = _enrich_for_rendering(hull, entry, now=now)
             result.append(
                 {
-                    "name": pos.get("name", info.get("name", hull)),
+                    "name": enriched["name"],
                     "type": "carrier",
-                    "lat": pos["lat"],
-                    "lng": pos["lng"],
-                    "heading": None,  # Heading unknown for carriers — OSINT cannot determine true heading
+                    "lat": enriched["lat"],
+                    "lng": enriched["lng"],
+                    "heading": None,  # OSINT cannot determine true heading.
                     "sog": 0,
                     "cog": 0,
                     "country": "United States",
-                    "desc": pos.get("desc", ""),
-                    "wiki": pos.get("wiki", info.get("wiki", "")),
+                    "desc": enriched["desc"],
+                    "wiki": enriched["wiki"],
                     "estimated": True,
-                    "source": pos.get("source", "OSINT estimated position"),
-                    "source_url": pos.get(
-                        "source_url", "https://news.usni.org/category/fleet-tracker"
-                    ),
-                    "last_osint_update": pos.get("updated", ""),
+                    "source": enriched["source"],
+                    "source_url": enriched["source_url"],
+                    "last_osint_update": enriched["last_osint_update"],
+                    # New fields (additive — existing UI continues to work):
+                    "position_source_at": enriched["position_source_at"],
+                    "position_confidence": enriched["position_confidence"],
+                    "is_fallback": enriched["is_fallback"],
                 }
             )
         return _deconflict_positions(result)
 
 
 # -----------------------------------------------------------------
-# Scheduler: runs at startup, then at 00:00 and 12:00 UTC daily
+# Scheduler: runs at startup, then at 00:00 and 12:00 UTC daily.
 # -----------------------------------------------------------------
 _scheduler_thread: Optional[threading.Thread] = None
 _scheduler_stop = threading.Event()
@@ -544,7 +781,6 @@ _scheduler_stop = threading.Event()
 
 def _scheduler_loop():
     """Background thread that triggers updates at 00:00 and 12:00 UTC."""
-    # Initial update on startup
     try:
         update_carrier_positions()
     except Exception as e:
@@ -552,7 +788,6 @@ def _scheduler_loop():
 
     while not _scheduler_stop.is_set():
         now = datetime.now(timezone.utc)
-        # Next target: 00:00 or 12:00 UTC, whichever is sooner
         hour = now.hour
         if hour < 12:
             next_hour = 12
@@ -561,18 +796,17 @@ def _scheduler_loop():
 
         next_run = now.replace(hour=next_hour % 24, minute=0, second=0, microsecond=0)
         if next_hour == 24:
-            from datetime import timedelta
-
             next_run = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
 
         wait_seconds = (next_run - now).total_seconds()
         logger.info(
-            f"Carrier tracker: next update at {next_run.isoformat()} ({wait_seconds/3600:.1f}h)"
+            "Carrier tracker: next update at %s (%.1fh)",
+            next_run.isoformat(),
+            wait_seconds / 3600,
         )
 
-        # Wait until next scheduled time, or until stop event
         if _scheduler_stop.wait(timeout=wait_seconds):
-            break  # Stop event was set
+            break
 
         try:
             update_carrier_positions()

backend/services/cctv_pipeline.py

@@ -17,6 +17,9 @@ _KNOWN_CCTV_MEDIA_HOST_ALIASES = {
     # Trusted upstream occasionally publishes a typo for this Georgia camera
     # host. Normalize it at ingest so the proxy and client stay consistent.
     "navigatos-c2c.dot.ga.gov": "navigator-c2c.dot.ga.gov",
+    # TravelIQ staging hosts occasionally appear in 511 catalog metadata.
+    "on.stage.traveliq.co": "511on.ca",
+    "ab.stage.traveliq.co": "511.alberta.ca",
 }
 
 _POINT_WKT_RE = re.compile(
@@ -40,6 +43,17 @@ def _normalize_cctv_media_url(raw_url: str) -> str:
     return urlunparse(parsed._replace(netloc=netloc))
 
 
+def _ensure_https_url(raw_url: str) -> str:
+    """Upgrade http:// media/catalog URLs to https:// at ingest time."""
+    candidate = _normalize_cctv_media_url(str(raw_url or "").strip())
+    if not candidate:
+        return ""
+    parsed = urlparse(candidate)
+    if parsed.scheme.lower() == "http":
+        return urlunparse(parsed._replace(scheme="https"))
+    return candidate
+
+
 def _looks_like_direct_cctv_media_url(url: str) -> bool:
     candidate = str(url or "").strip().lower()
     if not candidate.startswith(("http://", "https://")):
@@ -93,6 +107,165 @@ def _parse_wkt_point(raw_point: str) -> tuple[float | None, float | None]:
     return lat, lon
 
 
+def _fetch_traveliq_v2_cameras(
+    *,
+    api_url: str,
+    base_url: str,
+    id_prefix: str,
+    source_agency: str,
+) -> List[Dict[str, Any]]:
+    """Parse TravelIQ-style GET /api/v2/get/cameras feeds (Ontario, Alberta)."""
+    resp = fetch_with_curl(
+        api_url,
+        timeout=30,
+        headers={"Accept": "application/json"},
+    )
+    if not resp or resp.status_code != 200:
+        logger.error(
+            "%s CCTV fetch failed: HTTP %s",
+            source_agency,
+            resp.status_code if resp else "no response",
+        )
+        return []
+
+    data = resp.json()
+    if not isinstance(data, list):
+        return []
+
+    cameras: List[Dict[str, Any]] = []
+    for cam in data:
+        if not isinstance(cam, dict):
+            continue
+        try:
+            lat = float(cam.get("Latitude"))
+            lon = float(cam.get("Longitude"))
+        except (TypeError, ValueError):
+            continue
+
+        site_id = cam.get("Id")
+        location = str(cam.get("Location") or cam.get("Roadway") or "Camera")[:120]
+        views = cam.get("Views") or []
+        if not views:
+            continue
+
+        for view in views:
+            if not isinstance(view, dict):
+                continue
+            status = str(view.get("Status") or "enabled").strip().lower()
+            if status and status not in {"enabled", "active"}:
+                continue
+            media_url = _ensure_https_url(
+                urljoin(base_url, str(view.get("Url") or "").strip())
+            )
+            if not media_url:
+                continue
+            view_id = view.get("Id") or site_id
+            if site_id is None or view_id is None:
+                continue
+            label = str(view.get("Description") or location or "Camera")[:120]
+            cameras.append(
+                {
+                    "id": f"{id_prefix}-{site_id}-{view_id}",
+                    "source_agency": source_agency,
+                    "lat": lat,
+                    "lon": lon,
+                    "direction_facing": label,
+                    "media_url": media_url,
+                    "media_type": "image",
+                    "refresh_rate_seconds": 60,
+                }
+            )
+    return cameras
+
+
+def _fetch_511_datatables_cameras(
+    *,
+    list_url: str,
+    base_url: str,
+    id_prefix: str,
+    source_agency: str,
+    referer: str,
+    page_size: int = 500,
+) -> List[Dict[str, Any]]:
+    """Parse 511 DataTables POST /List/GetData/Cameras feeds (Georgia, Florida)."""
+    cameras: List[Dict[str, Any]] = []
+    start = 0
+    draw = 1
+    while True:
+        resp = fetch_with_curl(
+            list_url,
+            method="POST",
+            json_data={"draw": draw, "start": start, "length": page_size},
+            timeout=30,
+            headers={
+                "Accept": "application/json",
+                "Referer": referer,
+                "Origin": base_url.rstrip("/"),
+            },
+        )
+        if not resp or resp.status_code != 200:
+            logger.error(
+                "%s CCTV fetch failed: HTTP %s",
+                source_agency,
+                resp.status_code if resp else "no response",
+            )
+            break
+
+        data = resp.json()
+        rows = data.get("data") or []
+        if not rows:
+            break
+
+        for row in rows:
+            if not isinstance(row, dict):
+                continue
+            site_id = row.get("id") or row.get("DT_RowId")
+            location = row.get("location") or row.get("roadway") or source_agency
+            lat_lng = row.get("latLng") or {}
+            geography = lat_lng.get("geography") if isinstance(lat_lng, dict) else {}
+            lat, lon = _parse_wkt_point(
+                geography.get("wellKnownText") if isinstance(geography, dict) else ""
+            )
+            images = row.get("images") or []
+            image = next(
+                (
+                    candidate
+                    for candidate in images
+                    if str(candidate.get("imageUrl") or "").strip()
+                    and not bool(candidate.get("blocked"))
+                ),
+                None,
+            )
+            if not (site_id and image and lat is not None and lon is not None):
+                continue
+            media_url = _ensure_https_url(
+                urljoin(base_url, str(image.get("imageUrl") or "").strip())
+            )
+            if not media_url:
+                continue
+            cameras.append(
+                {
+                    "id": f"{id_prefix}-{site_id}",
+                    "source_agency": source_agency,
+                    "lat": lat,
+                    "lon": lon,
+                    "direction_facing": str(location)[:120],
+                    "media_url": media_url,
+                    "media_type": "image",
+                    "refresh_rate_seconds": 60,
+                }
+            )
+
+        start += len(rows)
+        draw += 1
+        total = int(data.get("recordsTotal") or 0)
+        if total and start >= total:
+            break
+        if not total and len(rows) < page_size:
+            break
+    return cameras
+
+
 def init_db():
     DB_PATH.parent.mkdir(parents=True, exist_ok=True)
     conn = sqlite3.connect(str(DB_PATH))
@@ -169,7 +342,7 @@ class BaseCCTVIngestor(ABC):
                         cam.get("lat"),
                         cam.get("lon"),
                         cam.get("direction_facing", "Unknown"),
-                        cam.get("media_url"),
+                        _ensure_https_url(cam.get("media_url", "")),
                         cam.get("media_type", _detect_media_type(cam.get("media_url", ""))),
                         cam.get("refresh_rate_seconds", 60),
                     ),
@@ -454,77 +627,14 @@ class WSDOTIngestor(BaseCCTVIngestor):
 class GeorgiaDOTIngestor(BaseCCTVIngestor):
     """Georgia cameras via the public 511GA list feed."""
 
-    URL = "https://511ga.org/List/GetData/Cameras"
-    BASE_URL = "https://511ga.org"
-    PAGE_SIZE = 500
-
     def fetch_data(self) -> List[Dict[str, Any]]:
-        cameras = []
-        start = 0
-        draw = 1
-        while True:
-            resp = fetch_with_curl(
-                self.URL,
-                method="POST",
-                json_data={"draw": draw, "start": start, "length": self.PAGE_SIZE},
-                timeout=30,
-                headers={
-                    "Accept": "application/json",
-                    "Referer": "https://511ga.org/cctv",
-                    "Origin": "https://511ga.org",
-                },
-            )
-            if not resp or resp.status_code != 200:
-                logger.error(
-                    "Georgia CCTV fetch failed: HTTP %s",
-                    resp.status_code if resp else "no response",
-                )
-                break
-            data = resp.json()
-            rows = data.get("data") or []
-            if not rows:
-                break
-            for row in rows:
-                site_id = row.get("id") or row.get("DT_RowId")
-                location = row.get("location") or row.get("roadway") or "GA Camera"
-                lat_lng = row.get("latLng") or {}
-                geography = lat_lng.get("geography") if isinstance(lat_lng, dict) else {}
-                lat, lon = _parse_wkt_point(geography.get("wellKnownText") if isinstance(geography, dict) else "")
-                images = row.get("images") or []
-                image = next(
-                    (
-                        candidate
-                        for candidate in images
-                        if str(candidate.get("imageUrl") or "").strip()
-                        and not bool(candidate.get("blocked"))
-                    ),
-                    None,
-                )
-                if not (site_id and image and lat is not None and lon is not None):
-                    continue
-                media_url = _normalize_cctv_media_url(
-                    urljoin(self.BASE_URL, str(image.get("imageUrl") or "").strip())
-                )
-                cameras.append(
-                    {
-                        "id": f"GDOT-{site_id}",
-                        "source_agency": "Georgia DOT",
-                        "lat": lat,
-                        "lon": lon,
-                        "direction_facing": str(location)[:120],
-                        "media_url": media_url,
-                        "media_type": "image",
-                        "refresh_rate_seconds": 60,
-                    }
-                )
-            start += len(rows)
-            draw += 1
-            total = int(data.get("recordsTotal") or 0)
-            if total and start >= total:
-                break
-            if not total and len(rows) < self.PAGE_SIZE:
-                break
-        return cameras
+        return _fetch_511_datatables_cameras(
+            list_url="https://511ga.org/List/GetData/Cameras",
+            base_url="https://511ga.org",
+            id_prefix="GDOT",
+            source_agency="Georgia DOT",
+            referer="https://511ga.org/cctv",
+        )
 
 
 class IllinoisDOTIngestor(BaseCCTVIngestor):
@@ -818,6 +928,105 @@ out body;
         return cameras
 
 
+# ---------------------------------------------------------------------------
+# ALPR / Surveillance Camera Locations (OSM Overpass)
+# ---------------------------------------------------------------------------
+# Queries OpenStreetMap for ALPR/LPR tagged surveillance cameras.
+# These cameras rarely have public media URLs — this ingestor captures
+# their LOCATIONS for situational awareness (density heatmap, blind-spot
+# analysis).  No plate-read data is fetched — only publicly-mapped positions.
+
+
+class OSMALPRCameraIngestor(BaseCCTVIngestor):
+    """ALPR / license-plate reader camera locations from OpenStreetMap.
+
+    Searches for nodes tagged with surveillance:type=ALPR or
+    man_made=surveillance + camera:type values indicating plate readers.
+    Only geolocations are ingested — no live feeds or detection data.
+    """
+
+    URL = "https://overpass-api.de/api/interpreter"
+    QUERY = """
+[out:json][timeout:45];
+(
+  node["surveillance:type"="ALPR"];
+  node["surveillance:type"="alpr"];
+  node["surveillance:type"="LPR"];
+  node["surveillance:type"="lpr"];
+  node["man_made"="surveillance"]["camera:type"="ALPR"];
+  node["man_made"="surveillance"]["camera:type"="alpr"];
+  node["man_made"="surveillance"]["camera:type"="LPR"];
+  node["man_made"="surveillance"]["camera:type"="lpr"];
+  node["man_made"="surveillance"]["description"~"[Ll]icense [Pp]late"];
+  node["man_made"="surveillance"]["description"~"ALPR"];
+  node["man_made"="surveillance"]["description"~"Flock"];
+);
+out body;
+""".strip()
+
+    def fetch_data(self) -> List[Dict[str, Any]]:
+        query = quote(self.QUERY, safe="")
+        resp = fetch_with_curl(
+            f"{self.URL}?data={query}",
+            timeout=50,
+            headers={"Accept": "application/json"},
+        )
+        if not resp or resp.status_code != 200:
+            logger.warning(
+                "OSM ALPR camera fetch failed: HTTP %s",
+                resp.status_code if resp else "no response",
+            )
+            return []
+        data = resp.json()
+        cameras = []
+        for item in data.get("elements", []) if isinstance(data, dict) else []:
+            lat = item.get("lat")
+            lon = item.get("lon")
+            if lat is None or lon is None:
+                continue
+            try:
+                lat, lon = float(lat), float(lon)
+            except (ValueError, TypeError):
+                continue
+
+            tags = item.get("tags", {}) if isinstance(item.get("tags"), dict) else {}
+
+            # Extract what we can from tags
+            operator = (
+                tags.get("operator")
+                or tags.get("brand")
+                or tags.get("network")
+                or "Unknown"
+            )
+            description = (
+                tags.get("description")
+                or tags.get("name")
+                or tags.get("surveillance:type", "ALPR")
+            )
+            direction = (
+                tags.get("camera:direction")
+                or tags.get("direction")
+                or tags.get("surveillance:direction")
+                or "Unknown"
+            )
+
+            # ALPR cameras typically have no public media URL — use a
+            # placeholder so the pin renders but no proxy attempt is made.
+            cameras.append(
+                {
+                    "id": f"ALPR-{item.get('id')}",
+                    "source_agency": str(operator)[:60],
+                    "lat": lat,
+                    "lon": lon,
+                    "direction_facing": f"ALPR: {str(description)[:100]} ({str(direction)[:30]})",
+                    "media_url": "",
+                    "media_type": "none",
+                    "refresh_rate_seconds": 0,
+                }
+            )
+        logger.info("OSM ALPR ingestor found %d cameras", len(cameras))
+        return cameras
+
 
 # ---------------------------------------------------------------------------
 # DGT Spain — National Road Cameras
@@ -888,7 +1097,7 @@ _KML_NS = {"kml": "http://www.opengis.net/kml/2.2"}
 
 def _find_kml_element(element, tag):
     """Find first descendant matching tag, ignoring XML namespace prefix."""
-    import xml.etree.ElementTree as ET
+    import defusedxml.ElementTree as ET
     el = element.find(f".//{tag}")
     if el is not None:
         return el
@@ -910,17 +1119,72 @@ def _extract_img_src(html_fragment: str):
     return None
 
 
+class AsfinagIngestor(BaseCCTVIngestor):
+    """Austria ASFINAG motorway webcams (Osiris port)."""
+
+    API_URL = "https://odo.asfinag.at/odo/rest/sec/resource/001/json/webcams?language=atDE"
+    HEADERS = {
+        "User-Agent": "Shadowbroker-CCTV/1.0",
+        "Accept": "application/json",
+        "Referer": "https://www.asfinag.at/",
+        "Authorization": "Basic bWFwX3dpZGdldDp0ZWdkaXc=",
+    }
+
+    def fetch_data(self) -> List[Dict[str, Any]]:
+        try:
+            response = fetch_with_curl(self.API_URL, timeout=15, headers=self.HEADERS)
+            response.raise_for_status()
+            payload = response.json()
+        except Exception as exc:
+            logger.error("AsfinagIngestor: fetch failed: %s", exc)
+            return []
+        if not isinstance(payload, list):
+            return []
+        cameras: List[Dict[str, Any]] = []
+        for cam in payload:
+            cam_id = cam.get("wcs_id")
+            lat = cam.get("wgs84_lat")
+            lon = cam.get("wgs84_lon")
+            image_url = cam.get("url_campic")
+            if not cam_id or lat is None or lon is None or not image_url:
+                continue
+            if str(cam_id).startswith("Utinform"):
+                continue
+            label = cam.get("position_txt") or cam.get("direction_txt") or "ASFINAG Webcam"
+            secure_url = _ensure_https_url(image_url)
+            if not secure_url:
+                continue
+            cameras.append(
+                {
+                    "id": f"ASFINAG-{cam_id}",
+                    "source_agency": "ASFINAG Austria",
+                    "lat": float(lat),
+                    "lon": float(lon),
+                    "direction_facing": label,
+                    "media_url": secure_url,
+                    "media_type": "image",
+                    "refresh_rate_seconds": 300,
+                }
+            )
+        logger.info("AsfinagIngestor: parsed %s cameras", len(cameras))
+        return cameras
+
+
 class MadridCityIngestor(BaseCCTVIngestor):
     """Madrid City Hall traffic cameras from datos.madrid.es KML feed."""
 
-    KML_URL = "http://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml"
+    KML_URL = "https://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml"
+
+    def _fetch_kml(self):
+        response = fetch_with_curl(self.KML_URL, timeout=20)
+        response.raise_for_status()
+        return response
 
     def fetch_data(self) -> List[Dict[str, Any]]:
-        import xml.etree.ElementTree as ET
+        import defusedxml.ElementTree as ET
 
         try:
-            response = fetch_with_curl(self.KML_URL, timeout=20)
-            response.raise_for_status()
+            response = self._fetch_kml()
         except Exception as e:
             logger.error(f"MadridCityIngestor: failed to fetch KML: {e}")
             return []
@@ -956,6 +1220,9 @@ class MadridCityIngestor(BaseCCTVIngestor):
                 if desc_el is not None and desc_el.text:
                     image_url = _extract_img_src(desc_el.text)
 
+                if not image_url:
+                    continue
+                image_url = _ensure_https_url(image_url)
                 if not image_url:
                     continue
 
@@ -977,6 +1244,153 @@ class MadridCityIngestor(BaseCCTVIngestor):
         return cameras
 
 
+class Ontario511Ingestor(BaseCCTVIngestor):
+    """Ontario highway cameras via 511on.ca TravelIQ API."""
+
+    def fetch_data(self) -> List[Dict[str, Any]]:
+        return _fetch_traveliq_v2_cameras(
+            api_url="https://511on.ca/api/v2/get/cameras",
+            base_url="https://511on.ca",
+            id_prefix="ON511",
+            source_agency="511 Ontario",
+        )
+
+
+class Alberta511Ingestor(BaseCCTVIngestor):
+    """Alberta highway cameras via 511 Alberta TravelIQ API."""
+
+    def fetch_data(self) -> List[Dict[str, Any]]:
+        return _fetch_traveliq_v2_cameras(
+            api_url="https://511.alberta.ca/api/v2/get/cameras",
+            base_url="https://511.alberta.ca",
+            id_prefix="AB511",
+            source_agency="511 Alberta",
+        )
+
+
+class Florida511Ingestor(BaseCCTVIngestor):
+    """Florida cameras via FL511 DataTables feed (~4,800 sites)."""
+
+    def fetch_data(self) -> List[Dict[str, Any]]:
+        return _fetch_511_datatables_cameras(
+            list_url="https://fl511.com/List/GetData/Cameras",
+            base_url="https://fl511.com",
+            id_prefix="FL511",
+            source_agency="Florida 511",
+            referer="https://fl511.com/",
+        )
+
+
+class AustraliaLiveTrafficIngestor(BaseCCTVIngestor):
+    """NSW / Australia live traffic cameras via Transport for NSW JSON feed."""
+
+    URL = "https://www.livetraffic.com/datajson/all-feeds-web.json"
+
+    def fetch_data(self) -> List[Dict[str, Any]]:
+        resp = fetch_with_curl(self.URL, timeout=35, headers={"Accept": "application/json"})
+        if not resp or resp.status_code != 200:
+            logger.error(
+                "Australia Live Traffic CCTV fetch failed: HTTP %s",
+                resp.status_code if resp else "no response",
+            )
+            return []
+
+        data = resp.json()
+        if not isinstance(data, list):
+            return []
+
+        cameras: List[Dict[str, Any]] = []
+        for item in data:
+            if not isinstance(item, dict) or item.get("eventType") != "liveCams":
+                continue
+            geometry = item.get("geometry") if isinstance(item.get("geometry"), dict) else {}
+            coords = geometry.get("coordinates") if isinstance(geometry.get("coordinates"), list) else []
+            if len(coords) < 2:
+                continue
+            try:
+                lon = float(coords[0])
+                lat = float(coords[1])
+            except (TypeError, ValueError):
+                continue
+
+            props = item.get("properties") if isinstance(item.get("properties"), dict) else {}
+            media_url = _ensure_https_url(str(props.get("href") or "").strip())
+            if not media_url:
+                continue
+
+            cam_id = str(item.get("path") or props.get("id") or len(cameras)).strip("/")
+            label = str(props.get("title") or props.get("headline") or "Australia Camera")[:120]
+            cameras.append(
+                {
+                    "id": f"AUS-{cam_id}",
+                    "source_agency": "NSW Live Traffic",
+                    "lat": lat,
+                    "lon": lon,
+                    "direction_facing": label,
+                    "media_url": media_url,
+                    "media_type": "image",
+                    "refresh_rate_seconds": 120,
+                }
+            )
+        logger.info("AustraliaLiveTrafficIngestor: parsed %s cameras", len(cameras))
+        return cameras
+
+
+class NetherlandsRWSIngestor(BaseCCTVIngestor):
+    """Netherlands Rijkswaterstaat cameras from legacy NDW open-data JSON.
+
+    The opendata.ndw.nu/cameras.json feed Osiris used is often offline; when
+  unavailable this ingestor returns an empty set and logs a warning.
+    """
+
+    URL = "https://opendata.ndw.nu/cameras.json"
+    MAX_CAMERAS = 1200
+
+    def fetch_data(self) -> List[Dict[str, Any]]:
+        resp = fetch_with_curl(self.URL, timeout=25, headers={"Accept": "application/json"})
+        if not resp or resp.status_code != 200:
+            logger.warning(
+                "Netherlands RWS cameras.json unavailable (HTTP %s) — "
+                "NDW retired this open-data endpoint; no cameras ingested",
+                resp.status_code if resp else "no response",
+            )
+            return []
+
+        data = resp.json()
+        if not isinstance(data, list):
+            return []
+
+        cameras: List[Dict[str, Any]] = []
+        for i, cam in enumerate(data[: self.MAX_CAMERAS]):
+            if not isinstance(cam, dict):
+                continue
+            lat = cam.get("lat") if cam.get("lat") is not None else cam.get("latitude")
+            lon = cam.get("lng") if cam.get("lng") is not None else cam.get("longitude")
+            media_url = _ensure_https_url(
+                str(cam.get("imageUrl") or cam.get("feed_url") or cam.get("url") or "").strip()
+            )
+            if lat is None or lon is None or not media_url:
+                continue
+            try:
+                lat_f, lon_f = float(lat), float(lon)
+            except (TypeError, ValueError):
+                continue
+            cameras.append(
+                {
+                    "id": f"NLRWS-{cam.get('id') or i}",
+                    "source_agency": "Rijkswaterstaat",
+                    "lat": lat_f,
+                    "lon": lon_f,
+                    "direction_facing": str(cam.get("name") or "Netherlands Camera")[:120],
+                    "media_url": media_url,
+                    "media_type": "image",
+                    "refresh_rate_seconds": 120,
+                }
+            )
+        logger.info("NetherlandsRWSIngestor: parsed %s cameras", len(cameras))
+        return cameras
+
+
 def _detect_media_type(url: str) -> str:
     """Detect the media type from a camera URL for proper frontend rendering."""
     if not url:
@@ -995,29 +1409,40 @@ def _detect_media_type(url: str) -> str:
     return "image"
 
 
+def scheduled_cctv_ingestors() -> List[tuple["BaseCCTVIngestor", str]]:
+    """Canonical list of CCTV ingestors for startup, scheduler, and DB seeding."""
+    return [
+        (TFLJamCamIngestor(), "cctv_tfl"),
+        (LTASingaporeIngestor(), "cctv_lta"),
+        (AustinTXIngestor(), "cctv_atx"),
+        (NYCDOTIngestor(), "cctv_nyc"),
+        (CaltransIngestor(), "cctv_caltrans"),
+        (ColoradoDOTIngestor(), "cctv_codot"),
+        (WSDOTIngestor(), "cctv_wsdot"),
+        (GeorgiaDOTIngestor(), "cctv_gdot"),
+        (IllinoisDOTIngestor(), "cctv_idot"),
+        (MichiganDOTIngestor(), "cctv_mdot"),
+        (WindyWebcamsIngestor(), "cctv_windy"),
+        (DGTNationalIngestor(), "cctv_dgt"),
+        (MadridCityIngestor(), "cctv_madrid"),
+        (OSMTrafficCameraIngestor(), "cctv_osm"),
+        (AsfinagIngestor(), "cctv_asfinag"),
+        (OSMALPRCameraIngestor(), "cctv_osm_alpr"),
+        (Ontario511Ingestor(), "cctv_on511"),
+        (Alberta511Ingestor(), "cctv_ab511"),
+        (Florida511Ingestor(), "cctv_fl511"),
+        (AustraliaLiveTrafficIngestor(), "cctv_australia"),
+        (NetherlandsRWSIngestor(), "cctv_nl_rws"),
+    ]
+
+
 def run_all_ingestors():
     """Run all CCTV ingestors synchronously. Used for first-run DB seeding."""
-    ingestors = [
-        TFLJamCamIngestor(),
-        LTASingaporeIngestor(),
-        AustinTXIngestor(),
-        NYCDOTIngestor(),
-        CaltransIngestor(),
-        ColoradoDOTIngestor(),
-        WSDOTIngestor(),
-        GeorgiaDOTIngestor(),
-        IllinoisDOTIngestor(),
-        MichiganDOTIngestor(),
-        WindyWebcamsIngestor(),
-        OSMTrafficCameraIngestor(),
-        DGTNationalIngestor(),
-        MadridCityIngestor(),
-    ]
-    for ing in ingestors:
+    for ingestor, _name in scheduled_cctv_ingestors():
         try:
-            ing.ingest()
+            ingestor.ingest()
         except Exception as e:
-            logger.warning(f"Ingestor {ing.__class__.__name__} failed during seed: {e}")
+            logger.warning(f"Ingestor {ingestor.__class__.__name__} failed during seed: {e}")
 
 
 def get_all_cameras() -> List[Dict[str, Any]]:

backend/services/config.py

@@ -10,6 +10,10 @@ class Settings(BaseSettings):
     ALLOW_INSECURE_ADMIN: bool = False
     PUBLIC_API_KEY: str = ""
 
+    # OpenClaw agent connectivity
+    OPENCLAW_HMAC_SECRET: str = ""        # HMAC shared secret for direct mode (auto-generated if empty)
+    OPENCLAW_ACCESS_TIER: str = "restricted"  # "full" or "restricted"
+
     # Data sources
     AIS_API_KEY: str = ""
     OPENSKY_CLIENT_ID: str = ""
@@ -26,18 +30,57 @@ class Settings(BaseSettings):
     MESH_MQTT_INCLUDE_DEFAULT_ROOTS: bool = True
     MESH_RNS_ENABLED: bool = False
     MESH_ARTI_ENABLED: bool = False
+    # When true, trust wormhole_status.json ready bit if the child process is
+    # alive — avoids transport-tier flapping when /api/health probes time out
+    # under Tor load (common during live DM E2E).
+    MESH_WORMHOLE_TRUST_FILE_READY: bool = False
     MESH_ARTI_SOCKS_PORT: int = 9050
-    MESH_RELAY_PEERS: str = "http://cipher0.shadowbroker.info:8000"
+    MESH_RELAY_PEERS: str = ""
+    MESH_PUBLIC_PEER_URL: str = ""
+    # Bootstrap seeds are discovery hints, not authoritative network roots.
+    # Nodes promote healthy discovered peers from the store/manifest over time.
+    MESH_BOOTSTRAP_SEED_PEERS: str = "http://gqpbunqbgtkcqilvclm3xrkt3zowjyl3s62kkktvojgvxzizamvbrqid.onion:8000"
+    # Legacy name kept for older compose/.env files.
+    MESH_DEFAULT_SYNC_PEERS: str = ""
+    # Infonet/Wormhole must fail closed to private transports by default.
+    # Set true only for local relay development or explicitly public testnets.
+    MESH_INFONET_ALLOW_CLEARNET_SYNC: bool = False
     MESH_BOOTSTRAP_DISABLED: bool = False
     MESH_BOOTSTRAP_MANIFEST_PATH: str = "data/bootstrap_peers.json"
-    MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY: str = ""
+    # Public sb-testnet-0 fleet signer (participants). Seed operator holds the private key.
+    MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY: str = (
+        "ul1d0kj/ODPIp0OhHzX8eLAVXzJ3CVvzW1vn2IC6q3I="
+    )
+    MESH_BOOTSTRAP_SIGNER_PRIVATE_KEY: str = ""
+    # When true, empty MESH_PEER_PUSH_SECRET uses the public fleet HMAC for seed join/announce.
+    MESH_INFONET_FLEET_JOIN: bool = True
+    MESH_INFONET_FLEET_JOIN_DISABLED: bool = False
+    # Headless relay/seed compose: auto-enable Tor wormhole on startup so
+    # docker compose redeploys keep the fleet onion reachable.
+    MESH_INFONET_RELAY_AUTO_WORMHOLE: bool = False
+    MESH_INFONET_RELAY_AUTO_WORMHOLE_DISABLED: bool = False
+    MESH_BOOTSTRAP_SIGNER_ID: str = ""
+    MESH_PEER_REGISTRY_ENABLED: bool = False
+    MESH_PEER_REGISTRY_DISABLED: bool = False
+    MESH_PEER_REGISTRY_STALE_S: int = 604800
+    MESH_SWARM_MANIFEST_TTL_S: int = 14400
+    MESH_SWARM_MANIFEST_PULL_INTERVAL_S: int = 300
     MESH_NODE_MODE: str = "participant"
     MESH_SYNC_INTERVAL_S: int = 300
     MESH_SYNC_FAILURE_BACKOFF_S: int = 60
+    MESH_SYNC_TIMEOUT_S: int = 5
+    MESH_SYNC_MAX_PEERS_PER_CYCLE: int = 3
     MESH_RELAY_PUSH_TIMEOUT_S: int = 10
     MESH_RELAY_MAX_FAILURES: int = 3
     MESH_RELAY_FAILURE_COOLDOWN_S: int = 120
-    MESH_PEER_PUSH_SECRET: str = "Mv63UvLfwqOEVWeRBXjA8MtFl2nEkkhUlLYVHiX1Zzo"
+    MESH_BOOTSTRAP_SEED_FAILURE_COOLDOWN_S: int = 15
+    MESH_PEER_PUSH_SECRET: str = ""
+    # Issue #256 (tg12): optional per-peer HMAC secret map. Comma-separated
+    # `url=secret` pairs. When a peer URL appears here, only that per-peer
+    # secret is accepted for it — the global MESH_PEER_PUSH_SECRET above is
+    # ignored for that specific URL. Single-peer installs and unmigrated
+    # multi-peer installs leave this empty and behavior is unchanged.
+    MESH_PEER_SECRETS: str = ""
     MESH_RNS_APP_NAME: str = "shadowbroker"
     MESH_RNS_ASPECT: str = "infonet"
     MESH_RNS_IDENTITY_PATH: str = ""
@@ -60,7 +103,8 @@ class Settings(BaseSettings):
     # Keep a low background cadence on private RNS links so quiet nodes are less
     # trivially fingerprintable by silence alone. Set to 0 to disable explicitly.
     MESH_RNS_COVER_INTERVAL_S: int = 30
-    MESH_RNS_COVER_SIZE: int = 64
+    MESH_RNS_COVER_SIZE: int = 512
+    MESH_DM_MAILBOX_TTL_S: int = 900
     MESH_RNS_IBF_WINDOW: int = 256
     MESH_RNS_IBF_TABLE_SIZE: int = 64
     MESH_RNS_IBF_MINHASH_SIZE: int = 16
@@ -75,48 +119,329 @@ class Settings(BaseSettings):
     MESH_RNS_IBF_FAIL_THRESHOLD: int = 3
     MESH_RNS_IBF_COOLDOWN_S: int = 120
     MESH_VERIFY_INTERVAL_S: int = 600
-    MESH_VERIFY_SIGNATURES: bool = True
+    # MESH_VERIFY_SIGNATURES is intentionally removed — the audit loop in main.py
+    # always calls validate_chain_incremental(verify_signatures=True). Any value
+    # set in the environment is ignored.
     MESH_DM_SECURE_MODE: bool = True
     MESH_DM_TOKEN_PEPPER: str = ""
-    MESH_DM_ALLOW_LEGACY_GET: bool = False
+    MESH_ALLOW_LEGACY_DM1_UNTIL: str = ""
+    MESH_ALLOW_LEGACY_DM_GET_UNTIL: str = ""
+    MESH_ALLOW_LEGACY_DM_SIGNATURE_COMPAT_UNTIL: str = ""
     MESH_DM_PERSIST_SPOOL: bool = False
+    MESH_DM_RELAY_FILE_PATH: str = ""
+    MESH_DM_RELAY_AUTO_RELOAD: bool = False
     MESH_DM_REQUIRE_SENDER_SEAL_SHARED: bool = True
     MESH_DM_NONCE_TTL_S: int = 300
     MESH_DM_NONCE_CACHE_MAX: int = 4096
+    MESH_DM_NONCE_PER_AGENT_MAX: int = 256
     MESH_DM_REQUEST_MAX_AGE_S: int = 300
     MESH_DM_REQUEST_MAILBOX_LIMIT: int = 12
     MESH_DM_SHARED_MAILBOX_LIMIT: int = 48
     MESH_DM_SELF_MAILBOX_LIMIT: int = 12
+    # Anti-spam: cap on distinct UNACKED messages a single sender can have
+    # parked in a single recipient's mailbox at any one time. Once the
+    # recipient pulls (acks) a message, the sender's quota for that pair
+    # frees up. Default 2 — a sender who wants to deliver more must wait
+    # for the recipient to actually read the prior messages.
+    #
+    # This cap is enforced TWICE: once on the local deposit path (the
+    # sender's own node refuses to spool the 3rd message) AND once on
+    # the replication-acceptance path (honest peer relays refuse to
+    # accept inbound replicas that would put them over the cap). The
+    # double enforcement makes the rule a NETWORK rule — patching out
+    # the local check on a hostile sender's relay doesn't let extras
+    # propagate, because every honest peer enforces the same cap on
+    # inbound replication.
+    MESH_DM_PENDING_PER_SENDER_LIMIT: int = 2
+    MESH_BLOCK_LEGACY_AGENT_ID_LOOKUP: bool = True
+    MESH_ALLOW_COMPAT_DM_INVITE_IMPORT: bool = False
+    MESH_ALLOW_COMPAT_DM_INVITE_IMPORT_UNTIL: str = ""
+    MESH_ALLOW_LEGACY_NODE_ID_COMPAT_UNTIL: str = ""
+    # Rotate voter-blinding salts on a rolling cadence so new reputation
+    # events do not reuse one forever-stable blinded identity.
+    MESH_VOTER_BLIND_SALT_ROTATE_DAYS: int = 30
+    # Keep historical salts long enough to cover live vote records, so
+    # duplicate-vote detection and wallet-cost accounting survive rotation.
+    MESH_VOTER_BLIND_SALT_GRACE_DAYS: int = 30
     MESH_DM_MAX_MSG_BYTES: int = 8192
     MESH_DM_ALLOW_SENDER_SEAL: bool = False
     # TTL for DH key and prekey bundle registrations — stale entries are pruned.
     MESH_DM_KEY_TTL_DAYS: int = 30
+    # TTL for invite-scoped prekey lookup aliases; shorter windows reduce
+    # long-lived relay linkage between opaque lookup handles and agent IDs.
+    MESH_DM_PREKEY_LOOKUP_ALIAS_TTL_DAYS: int = 14
+    # TTL for relay witness history; keep continuity metadata bounded instead
+    # of relying on a hidden hardcoded retention window.
+    MESH_DM_WITNESS_TTL_DAYS: int = 14
     # TTL for mailbox binding metadata — shorter = smaller metadata footprint on disk.
-    MESH_DM_BINDING_TTL_DAYS: int = 7
+    MESH_DM_BINDING_TTL_DAYS: int = 3
     # When False, mailbox bindings are memory-only (agents re-register on restart).
-    MESH_DM_METADATA_PERSIST: bool = True
+    # Enable explicitly only if restart continuity is worth persisting DM graph metadata.
+    MESH_DM_METADATA_PERSIST: bool = False
+    # Second explicit opt-in for at-rest DM metadata persistence. This keeps a
+    # single boolean flip from silently writing mailbox graph metadata to disk.
+    MESH_DM_METADATA_PERSIST_ACKNOWLEDGE: bool = False
+    # Optional import path for externally managed root witness material packages.
+    # Relative paths resolve from the backend directory.
+    MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH: str = ""
+    # Optional URI for externally managed root witness material packages.
+    # Supports file:// and http(s):// sources; when set it overrides the local path.
+    MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_URI: str = ""
+    # Maximum acceptable age for externally sourced root witness packages.
+    # Strong DM trust fails closed when the imported package exported_at is older than this.
+    MESH_DM_ROOT_EXTERNAL_WITNESS_MAX_AGE_S: int = 3600
+    # Warning threshold for externally sourced root witness packages.
+    # When current external witness material reaches this age, operator health degrades to warning
+    # before the strong path eventually fails closed at MAX_AGE.
+    MESH_DM_ROOT_EXTERNAL_WITNESS_WARN_AGE_S: int = 2700
+    # Optional export path for the append-only stable-root transparency ledger.
+    # Relative paths resolve from the backend directory.
+    MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH: str = ""
+    # Optional URI used to read back and verify published transparency ledgers.
+    # Supports file:// and http(s):// sources.
+    MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI: str = ""
+    # Maximum acceptable age for externally read transparency ledgers.
+    # Strong DM trust fails closed when exported_at is older than this.
+    MESH_DM_ROOT_TRANSPARENCY_LEDGER_MAX_AGE_S: int = 3600
+    # Warning threshold for externally read transparency ledgers.
+    # When current external transparency readback reaches this age, operator health degrades to warning
+    # before the strong path eventually fails closed at MAX_AGE.
+    MESH_DM_ROOT_TRANSPARENCY_LEDGER_WARN_AGE_S: int = 2700
     MESH_SCOPED_TOKENS: str = ""
+    # Deprecated legacy env vars kept for backward config compatibility only.
+    # Ordinary shipped gate flows keep MLS decrypt local; backend decrypt is
+    # reserved for explicit recovery reads.
+    MESH_GATE_BACKEND_DECRYPT_COMPAT: bool = False
+    MESH_GATE_BACKEND_DECRYPT_COMPAT_ACKNOWLEDGE: bool = False
+    MESH_BACKEND_GATE_DECRYPT_COMPAT: bool = False
+    # Deprecated legacy env vars kept for backward config compatibility only.
+    # Ordinary shipped gate flows keep compose/post local and submit encrypted
+    # payloads to the backend for sign/post only.
+    MESH_GATE_BACKEND_PLAINTEXT_COMPAT: bool = False
+    MESH_GATE_BACKEND_PLAINTEXT_COMPAT_ACKNOWLEDGE: bool = False
+    MESH_BACKEND_GATE_PLAINTEXT_COMPAT: bool = False
+    # Runtime gate for recovery envelopes. When off, per-gate
+    # envelope_recovery / envelope_always policies fail closed to
+    # envelope_disabled. Default True so the Reddit-like durable history
+    # model works out of the box: any member with the gate_secret can
+    # decrypt every envelope encrypted from the moment they had that key.
+    # Set MESH_GATE_RECOVERY_ENVELOPE_ENABLE=false to revert to MLS-only
+    # forward-secret behavior (your own history becomes unreadable after
+    # the sending ratchet advances).
+    MESH_GATE_RECOVERY_ENVELOPE_ENABLE: bool = True
+    MESH_GATE_RECOVERY_ENVELOPE_ENABLE_ACKNOWLEDGE: bool = True
+    # Durable gate plaintext retention is disabled by default. Enable only
+    # when the operator explicitly accepts the at-rest privacy tradeoff.
+    MESH_GATE_PLAINTEXT_PERSIST: bool = False
+    MESH_GATE_PLAINTEXT_PERSIST_ACKNOWLEDGE: bool = False
     MESH_GATE_SESSION_ROTATE_MSGS: int = 50
     MESH_GATE_SESSION_ROTATE_S: int = 3600
+    MESH_GATE_LEGACY_ENVELOPE_FALLBACK_MAX_DAYS: int = 30
     # Add a randomized grace window before anonymous gate-session auto-rotation
     # so threshold-triggered identity swaps are less trivially correlated.
     MESH_GATE_SESSION_ROTATE_JITTER_S: int = 180
+    # Gate persona (named identity) rotation thresholds.  Rotating the signing
+    # key limits the linkability window.  Zero = disabled.
+    MESH_GATE_PERSONA_ROTATE_MSGS: int = 200
+    MESH_GATE_PERSONA_ROTATE_S: int = 604800  # 7 days
+    MESH_GATE_PERSONA_ROTATE_JITTER_S: int = 600
+    # Feature-flagged session stream for multiplexed gate room updates.
+    # Disabled by default so rollout stays explicit while stream-first rooms bake.
+    MESH_GATE_SESSION_STREAM_ENABLED: bool = False
+    MESH_GATE_SESSION_STREAM_HEARTBEAT_S: int = 20
+    MESH_GATE_SESSION_STREAM_BATCH_MS: int = 1500
+    MESH_GATE_SESSION_STREAM_MAX_GATES: int = 16
     # Private gate APIs expose a backward-jittered timestamp view so observers
     # cannot trivially align exact send times from response metadata alone.
     MESH_GATE_TIMESTAMP_JITTER_S: int = 60
+    # Ban/kick gate-secret rotation is on by default (hardening Rec #10): the
+    # invariant has baked and a ban that does not rotate is effectively a
+    # display-only removal. Set MESH_GATE_BAN_KICK_ROTATION_ENABLE=false to
+    # revert to observe-only during incident triage.
+    MESH_GATE_BAN_KICK_ROTATION_ENABLE: bool = True
+    MESH_BLOCK_LEGACY_NODE_ID_COMPAT: bool = True
     MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK: bool = False
+    MESH_ACK_RAW_FALLBACK_AT_OWN_RISK: bool = False
+    MESH_SECURE_STORAGE_SECRET: str = ""
+    MESH_SECURE_STORAGE_SECRET_FILE: str = ""
     MESH_PRIVATE_LOG_TTL_S: int = 900
+    # Sprint 1 rollout: restored DM boot probes stay disabled by default until
+    # the architect reviews false positives from the observe-only path.
+    MESH_DM_RESTORED_SESSION_BOOT_PROBE_ENABLE: bool = False
+    # Queued DM release requires explicit per-item approval before any weaker
+    # relay fallback. Silent fallback is not a safe private-mode default.
+    MESH_PRIVATE_RELEASE_APPROVAL_ENABLE: bool = True
+    # Expiry for user-approved scoped private relay fallback policy. The policy
+    # is still bounded by hidden-transport checks before it can auto-release.
+    MESH_PRIVATE_RELAY_POLICY_TTL_S: int = 3600
+    # Background privacy prewarm prepares keys/aliases/transport readiness
+    # before send-time. Anonymous mode uses a cadence gate so user clicks do
+    # not directly create hidden-transport activity.
+    MESH_PRIVACY_PREWARM_ENABLE: bool = True
+    MESH_PRIVACY_PREWARM_INTERVAL_S: int = 300
+    MESH_PRIVACY_PREWARM_ANON_CADENCE_S: int = 300
+    # Sprint 4 rollout: authenticated RNS cover markers remain disabled until
+    # the observer-equivalence and receive-path DoS tests are green.
+    MESH_RNS_COVER_AUTH_MARKER_ENABLE: bool = False
+    # Signed-write revocation lookups use a short local TTL; stale entries force
+    # a local rebuild before honor. Offline/local-refresh failures remain
+    # observe-only until the later enforcement sprint.
+    MESH_SIGNED_REVOCATION_CACHE_TTL_S: int = 300
+    MESH_SIGNED_REVOCATION_CACHE_ENFORCE: bool = True
+    MESH_SIGNED_WRITE_CONTEXT_REQUIRED: bool = True
+    # Sprint 5 rollout: when enabled, root witness finality requires
+    # independent quorum for threshold>1 witnessed roots before they count as
+    # verified first-contact provenance.
+    WORMHOLE_ROOT_WITNESS_FINALITY_ENFORCE: bool = False
+    # Optional JSON artifact generated by CI/release workflow for the Sprint 8
+    # release gate. Relative paths resolve from the backend directory.
+    # dev = permissive local/dev behavior; testnet-private = strict private
+    # defaults; release-candidate = no compatibility/debug escape hatches.
+    MESH_RELEASE_PROFILE: str = "dev"
+    MESH_RELEASE_ATTESTATION_PATH: str = ""
+    # Operator release attestation for the Sprint 8 release gate. This does
+    # not change runtime behavior; it only records that the DM relay security
+    # suite was run and passed for the release candidate.
+    MESH_RELEASE_DM_RELAY_SECURITY_SUITE_GREEN: bool = False
+    PRIVACY_CORE_MIN_VERSION: str = "0.1.0"
+    PRIVACY_CORE_ALLOWED_SHA256: str = ""
+    PRIVACY_CORE_DEV_OVERRIDE: bool = False
+    # Sprint 4 rollout: fail fast when the loaded privacy-core artifact is
+    # missing required FFI symbols expected by the current Python bridge.
+    PRIVACY_CORE_EXPORT_SET_AUDIT_ENABLE: bool = True
     # Clearnet fallback policy for private-tier messages.
     # "block" (default) = refuse to send private messages over clearnet.
     # "allow" = fall back to clearnet when Tor/RNS is unavailable (weaker privacy).
     MESH_PRIVATE_CLEARNET_FALLBACK: str = "block"
+    # Second explicit opt-in for private-tier clearnet fallback. Without this
+    # acknowledgement, "allow" remains requested but not effective.
+    MESH_PRIVATE_CLEARNET_FALLBACK_ACKNOWLEDGE: bool = False
+    # Meshtastic MQTT bridge — disabled by default to avoid hammering the
+    # public broker.  Users opt in explicitly.
+    MESH_MQTT_ENABLED: bool = False
     # Meshtastic MQTT broker credentials (defaults match public firmware).
+    MESH_MQTT_BROKER: str = "mqtt.meshtastic.org"
+    MESH_MQTT_PORT: int = 1883
     MESH_MQTT_USER: str = "meshdev"
     MESH_MQTT_PASS: str = "large4cats"
+    # Hex-encoded PSK — empty string means use the default LongFast key.
+    # Must decode to exactly 16 or 32 bytes when set.
+    MESH_MQTT_PSK: str = ""
+    # Optional operator-provided Meshtastic node ID (e.g. "!abcd1234") included
+    # in the User-Agent when fetching from meshtastic.liamcottle.net so the
+    # service operator can identify per-install traffic instead of a generic
+    # "ShadowBroker" aggregate.
+    MESHTASTIC_OPERATOR_CALLSIGN: str = ""
+    # Per-install operator handle used in the User-Agent for EVERY third-party
+    # API the backend calls (Wikipedia, Wikidata, Nominatim, GDELT, OpenMHz,
+    # Broadcastify, weather.gov, NUFORC, etc.). The default is empty, in which
+    # case backend/services/network_utils.py auto-generates a stable
+    # pseudonymous handle like "operator-7f3a92" on first use and caches it.
+    # Operators who want to identify themselves with a real handle can set
+    # this; operators who want to stay pseudonymous can leave it empty.
+    #
+    # The handle is sent ONLY to public third-party APIs. It is NEVER mixed
+    # into mesh / Wormhole / Infonet identity (those have their own crypto
+    # identity layer; conflating the two would leak public attribution into
+    # private mesh state).
+    OPERATOR_HANDLE: str = ""
+
+    # SAR (Synthetic Aperture Radar) data layer
+    # Mode A — free catalog metadata, no account, default-on
+    MESH_SAR_CATALOG_ENABLED: bool = True
+    # Mode B — free pre-processed anomalies (OPERA / EGMS / GFM / EMS / UNOSAT)
+    # Two-step opt-in: must be "allow" AND _ACKNOWLEDGE must be true
+    MESH_SAR_PRODUCTS_FETCH: str = "block"
+    MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE: bool = False
+    # NASA Earthdata Login (free) — required for OPERA products
+    MESH_SAR_EARTHDATA_USER: str = ""
+    MESH_SAR_EARTHDATA_TOKEN: str = ""
+    # Copernicus Data Space (free) — required for EGMS / EMS products
+    MESH_SAR_COPERNICUS_USER: str = ""
+    MESH_SAR_COPERNICUS_TOKEN: str = ""
+    # Whether OpenClaw agents may read/act on the SAR layer
+    MESH_SAR_OPENCLAW_ENABLED: bool = True
+    # Require private-tier transport before signing/broadcasting SAR anomalies
+    MESH_SAR_REQUIRE_PRIVATE_TIER: bool = True
 
     model_config = SettingsConfigDict(env_file=".env", extra="ignore")
 
 
 @lru_cache
 def get_settings() -> Settings:
+    try:
+        from services.api_settings import load_persisted_api_keys_into_environ
+        load_persisted_api_keys_into_environ()
+    except Exception:
+        pass
     return Settings()
+
+
+def private_clearnet_fallback_requested(settings: Settings | None = None) -> str:
+    snapshot = settings or get_settings()
+    policy = str(getattr(snapshot, "MESH_PRIVATE_CLEARNET_FALLBACK", "block") or "block").strip().lower()
+    return "allow" if policy == "allow" else "block"
+
+
+def private_clearnet_fallback_effective(settings: Settings | None = None) -> str:
+    snapshot = settings or get_settings()
+    requested = private_clearnet_fallback_requested(snapshot)
+    acknowledged = bool(getattr(snapshot, "MESH_PRIVATE_CLEARNET_FALLBACK_ACKNOWLEDGE", False))
+    if requested == "allow" and acknowledged:
+        return "allow"
+    return "block"
+
+
+def backend_gate_decrypt_compat_effective(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(
+        getattr(snapshot, "MESH_BACKEND_GATE_DECRYPT_COMPAT", False)
+        or getattr(snapshot, "MESH_GATE_BACKEND_DECRYPT_COMPAT", False)
+    )
+
+
+def backend_gate_plaintext_compat_effective(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(
+        getattr(snapshot, "MESH_BACKEND_GATE_PLAINTEXT_COMPAT", False)
+        or getattr(snapshot, "MESH_GATE_BACKEND_PLAINTEXT_COMPAT", False)
+    )
+
+
+def gate_recovery_envelope_effective(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    requested = bool(getattr(snapshot, "MESH_GATE_RECOVERY_ENVELOPE_ENABLE", False))
+    acknowledged = bool(getattr(snapshot, "MESH_GATE_RECOVERY_ENVELOPE_ENABLE_ACKNOWLEDGE", False))
+    return requested and acknowledged
+
+
+def gate_plaintext_persist_effective(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    requested = bool(getattr(snapshot, "MESH_GATE_PLAINTEXT_PERSIST", False))
+    acknowledged = bool(getattr(snapshot, "MESH_GATE_PLAINTEXT_PERSIST_ACKNOWLEDGE", False))
+    return requested and acknowledged
+
+
+def gate_ban_kick_rotation_enabled(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(getattr(snapshot, "MESH_GATE_BAN_KICK_ROTATION_ENABLE", False))
+
+
+def dm_restored_session_boot_probe_enabled(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(getattr(snapshot, "MESH_DM_RESTORED_SESSION_BOOT_PROBE_ENABLE", False))
+
+
+def signed_revocation_cache_ttl_s(settings: Settings | None = None) -> int:
+    snapshot = settings or get_settings()
+    return max(0, int(getattr(snapshot, "MESH_SIGNED_REVOCATION_CACHE_TTL_S", 300) or 0))
+
+
+def signed_revocation_cache_enforce(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(getattr(snapshot, "MESH_SIGNED_REVOCATION_CACHE_ENFORCE", False))
+
+
+def wormhole_root_witness_finality_enforce(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(getattr(snapshot, "WORMHOLE_ROOT_WITNESS_FINALITY_ENFORCE", False))

backend/services/constants.py

@@ -11,8 +11,13 @@ DEFAULT_TRAIL_TTL_S = 300  # 5 min - trail TTL for non-tracked flights
 HOLD_PATTERN_DEGREES = 300  # Total heading change to flag holding pattern
 GPS_JAMMING_NACP_THRESHOLD = 8  # NACp below this = degraded GPS signal
 GPS_JAMMING_GRID_SIZE = 1.0  # 1 degree grid for aggregation
-GPS_JAMMING_MIN_RATIO = 0.30  # 30% degraded aircraft to flag zone
-GPS_JAMMING_MIN_AIRCRAFT = 5  # Min aircraft in grid cell for statistical significance
+# Tuned 2026-05: previously 0.30 / 5 aircraft which — combined with the
+# -1 noise cushion in the detector AND the pre-fix nac_p==0 filter that
+# discarded jamming victims — meant the layer almost never lit up.
+# Lowering the bar so genuine jamming zones with sparser ADS-B coverage
+# clear (eastern Med, Russia/Ukraine border, Iran/Iraq).
+GPS_JAMMING_MIN_RATIO = 0.20  # 20% degraded aircraft to flag zone
+GPS_JAMMING_MIN_AIRCRAFT = 3  # Min aircraft in grid cell for statistical significance
 
 # ─── Network & Circuit Breaker ──────────────────────────────────────────────
 CIRCUIT_BREAKER_TTL_S = 120  # Skip domain for 2 min after total failure

backend/services/correlation_engine.py

@@ -8,9 +8,13 @@ Correlation types:
   - RF Anomaly:          GPS jamming + internet outage (both required)
   - Military Buildup:    Military flights + naval vessels + GDELT conflict events
   - Infrastructure Cascade: Internet outage + KiwiSDR offline in same zone
+  - Possible Contradiction: Official denial/statement + infrastructure disruption
+                            in same region — hypothesis generator, NOT verdict
 """
 
 import logging
+import math
+import re
 from collections import defaultdict
 
 logger = logging.getLogger(__name__)
@@ -306,6 +310,427 @@ def _detect_infra_cascades(data: dict) -> list[dict]:
     return alerts
 
 
+# ---------------------------------------------------------------------------
+# Possible Contradiction: official denial/statement + infra disruption
+#
+# This is a HYPOTHESIS GENERATOR, not a verdict engine.  It says "LOOK HERE"
+# when an official statement (denial, clarification, refusal) co-locates with
+# infrastructure disruption (internet outage, sigint change).  The human or
+# higher-order reasoning decides what actually happened.
+#
+# Context ratings:
+#   STRONG   — denial + outage + prediction market movement in same region
+#   MODERATE — denial + outage (no market signal)
+#   WEAK     — denial + minor outage or distant co-location
+#   DETECTION_GAP — denial found but NO telemetry to verify (equally valuable)
+# ---------------------------------------------------------------------------
+
+# Denial / official-statement patterns in headlines and URL slugs
+_DENIAL_PATTERNS = [
+    re.compile(p, re.IGNORECASE) for p in [
+        r"\bden(?:y|ies|ied|ial)\b",
+        r"\brefut(?:e[ds]?|ing)\b",
+        r"\breject(?:s|ed|ing)?\b",
+        r"\bclarif(?:y|ies|ied|ication)\b",
+        r"\bdismiss(?:es|ed|ing)?\b",
+        r"\bno\s+attack\b",
+        r"\bdid\s+not\s+(?:attack|strike|bomb|target|order|invade|kill)\b",
+        r"\bnever\s+(?:attack|strike|bomb|target|order|invade|happen)\b",
+        r"\bfalse\s+(?:report|claim|allegation|rumor|narrative)\b",
+        r"\bmisinformation\b",
+        r"\bdisinformation\b",
+        r"\bpropaganda\b",
+        r"\b(?:army|military|government|ministry|official)\s+(?:says|clarifies|denies|refutes)\b",
+        r"\brumor[s]?\b.*\buntrue\b",
+        r"\bcategorically\b",
+        r"\bbaseless\b",
+    ]
+]
+
+# Broader cell radius for sparse telemetry regions (Africa, Central Asia, etc.)
+# These regions have fewer IODA/RIPE probes so outage data is sparser
+_SPARSE_REGIONS_LAT_RANGES = [
+    (-35, 37),   # Africa roughly
+    (25, 50),    # Central Asia band (when lng 40-90)
+]
+
+
+def _is_sparse_region(lat: float, lng: float) -> bool:
+    """Check if coordinates fall in a region with sparse telemetry coverage."""
+    # Africa
+    if -35 <= lat <= 37 and -20 <= lng <= 55:
+        return True
+    # Central Asia
+    if 25 <= lat <= 50 and 40 <= lng <= 90:
+        return True
+    # South America interior
+    if -55 <= lat <= 12 and -80 <= lng <= -35:
+        return True
+    return False
+
+
+def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
+    """Great-circle distance in km."""
+    R = 6371.0
+    dlat = math.radians(lat2 - lat1)
+    dlon = math.radians(lon2 - lon1)
+    a = (math.sin(dlat / 2) ** 2 +
+         math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) *
+         math.sin(dlon / 2) ** 2)
+    return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
+
+
+def _matches_denial(text: str) -> bool:
+    """Check if text matches any denial/official-statement pattern."""
+    return any(p.search(text) for p in _DENIAL_PATTERNS)
+
+
+def _detect_contradictions(data: dict) -> list[dict]:
+    """Detect possible contradictions between official statements and telemetry.
+
+    Scans GDELT headlines for denial language, then checks whether internet
+    outages or other infrastructure disruptions exist in the same geographic
+    region.  Scores confidence and lists alternative explanations.
+    """
+    gdelt = data.get("gdelt") or []
+    internet_outages = data.get("internet_outages") or []
+    news = data.get("news") or []
+    prediction_markets = data.get("prediction_markets") or []
+
+    # ── Step 1: Find GDELT events with denial/official-statement language ──
+    denial_events: list[dict] = []
+
+    # GDELT comes as GeoJSON features
+    gdelt_features = gdelt
+    if isinstance(gdelt, dict):
+        gdelt_features = gdelt.get("features", [])
+
+    for feature in gdelt_features:
+        # Handle both GeoJSON features and flat dicts
+        if "properties" in feature and "geometry" in feature:
+            props = feature.get("properties", {})
+            geom = feature.get("geometry", {})
+            coords = geom.get("coordinates", [])
+            if len(coords) >= 2:
+                lng, lat = float(coords[0]), float(coords[1])
+            else:
+                continue
+            headlines = props.get("_headlines_list", [])
+            urls = props.get("_urls_list", [])
+            name = props.get("name", "")
+            count = props.get("count", 1)
+        else:
+            lat = feature.get("lat") or feature.get("actionGeo_Lat")
+            lng = feature.get("lng") or feature.get("lon") or feature.get("actionGeo_Long")
+            if lat is None or lng is None:
+                continue
+            lat, lng = float(lat), float(lng)
+            headlines = [feature.get("title", "")]
+            urls = [feature.get("sourceurl", "")]
+            name = feature.get("name", "")
+            count = 1
+
+        # Check all headlines + URL slugs for denial patterns
+        all_text = " ".join(str(h) for h in headlines if h)
+        all_text += " " + " ".join(str(u) for u in urls if u)
+
+        if _matches_denial(all_text):
+            denial_events.append({
+                "lat": lat,
+                "lng": lng,
+                "headlines": [h for h in headlines if h][:5],
+                "urls": [u for u in urls if u][:3],
+                "location_name": name,
+                "event_count": count,
+            })
+
+    # Also scan news articles for denial language
+    for article in news:
+        title = str(article.get("title", "") or "")
+        desc = str(article.get("description", "") or article.get("summary", "") or "")
+        if not _matches_denial(title + " " + desc):
+            continue
+        # News articles often lack coordinates — try to match to GDELT locations
+        # For now, only include if we have coordinates
+        lat = article.get("lat") or article.get("latitude")
+        lng = article.get("lng") or article.get("lon") or article.get("longitude")
+        if lat is not None and lng is not None:
+            denial_events.append({
+                "lat": float(lat),
+                "lng": float(lng),
+                "headlines": [title],
+                "urls": [article.get("url") or article.get("link") or ""],
+                "location_name": "",
+                "event_count": 1,
+            })
+
+    if not denial_events:
+        return []
+
+    # ── Step 2: Cross-reference with internet outages ──
+    alerts: list[dict] = []
+
+    for denial in denial_events:
+        d_lat, d_lng = denial["lat"], denial["lng"]
+        sparse = _is_sparse_region(d_lat, d_lng)
+        search_radius_km = 1500.0 if sparse else 500.0
+
+        # Find nearby outages
+        nearby_outages: list[dict] = []
+        for outage in internet_outages:
+            o_lat = outage.get("lat") or outage.get("latitude")
+            o_lng = outage.get("lng") or outage.get("lon") or outage.get("longitude")
+            if o_lat is None or o_lng is None:
+                continue
+            try:
+                dist = _haversine_km(d_lat, d_lng, float(o_lat), float(o_lng))
+            except (ValueError, TypeError):
+                continue
+            if dist <= search_radius_km:
+                nearby_outages.append({
+                    "region": outage.get("region_name") or outage.get("country_name", ""),
+                    "severity": _outage_pct(outage),
+                    "distance_km": round(dist, 0),
+                    "level": outage.get("level", ""),
+                })
+
+        # ── Step 3: Check prediction markets for related movements ──
+        denial_text = " ".join(denial["headlines"]).lower()
+        related_markets: list[dict] = []
+        for market in prediction_markets:
+            m_title = str(market.get("title", "") or market.get("question", "") or "").lower()
+            # Look for keyword overlap between denial and market
+            denial_words = set(re.findall(r"[a-z]{4,}", denial_text))
+            market_words = set(re.findall(r"[a-z]{4,}", m_title))
+            overlap = denial_words & market_words - {"that", "this", "with", "from", "have", "been", "were", "will", "says", "said"}
+            if len(overlap) >= 2:
+                prob = market.get("probability") or market.get("lastTradePrice") or market.get("yes_price")
+                if prob is not None:
+                    related_markets.append({
+                        "title": market.get("title") or market.get("question"),
+                        "probability": float(prob),
+                    })
+
+        # ── Step 4: Score confidence and assign context rating ──
+        indicators = 1  # denial itself
+        drivers: list[str] = []
+
+        # Primary driver: the denial headline
+        headline_display = denial["headlines"][0] if denial["headlines"] else "Official statement"
+        if len(headline_display) > 80:
+            headline_display = headline_display[:77] + "..."
+        drivers.append(f'"{headline_display}"')
+
+        # Outage co-location
+        has_outage = False
+        if nearby_outages:
+            best_outage = max(nearby_outages, key=lambda o: o["severity"])
+            if best_outage["severity"] >= 10:
+                indicators += 1
+                has_outage = True
+                drivers.append(
+                    f"Internet outage {best_outage['severity']:.0f}% "
+                    f"({best_outage['region']}, {best_outage['distance_km']:.0f}km away)"
+                )
+            elif best_outage["severity"] > 0:
+                indicators += 0.5  # minor outage, partial indicator
+                has_outage = True
+                drivers.append(
+                    f"Minor outage ({best_outage['region']}, "
+                    f"{best_outage['distance_km']:.0f}km away)"
+                )
+
+        # Prediction market signal
+        has_market = False
+        if related_markets:
+            indicators += 1
+            has_market = True
+            top_market = related_markets[0]
+            drivers.append(
+                f"Market: \"{top_market['title'][:50]}\" "
+                f"at {top_market['probability']:.0%}"
+            )
+
+        # Multiple denial sources strengthen the signal
+        if denial["event_count"] > 1:
+            indicators += 0.5
+            drivers.append(f"{denial['event_count']} sources reporting")
+
+        # Context rating
+        if has_outage and has_market:
+            context = "STRONG"
+        elif has_outage:
+            context = "MODERATE"
+        elif has_market:
+            context = "WEAK"  # market signal without infra disruption
+        else:
+            context = "DETECTION_GAP"
+
+        # Severity mapping
+        if context == "STRONG":
+            sev = "high"
+        elif context == "MODERATE":
+            sev = "medium"
+        else:
+            sev = "low"
+
+        # Alternative explanations (always present — this is a hypothesis generator)
+        alternatives: list[str] = []
+        if has_outage:
+            alternatives.append("Routine infrastructure maintenance or cable damage")
+            alternatives.append("Weather-related outage coinciding with news cycle")
+        if not has_outage and context == "DETECTION_GAP":
+            alternatives.append("Statement may be truthful — no contradicting telemetry found")
+            alternatives.append("Telemetry coverage gap in this region")
+        alternatives.append("Denial may be responding to social media rumors, not real events")
+
+        lat_c, lng_c = _cell_center(_cell_key(d_lat, d_lng))
+        alerts.append({
+            "lat": lat_c,
+            "lng": lng_c,
+            "type": "contradiction",
+            "severity": sev,
+            "score": _severity_score(sev),
+            "drivers": drivers[:4],
+            "cell_size": _CELL_SIZE,
+            "context": context,
+            "alternatives": alternatives[:3],
+            "location_name": denial.get("location_name", ""),
+            "headlines": denial["headlines"][:3],
+            "related_markets": related_markets[:3],
+            "nearby_outages": nearby_outages[:5],
+        })
+
+    # Deduplicate: keep highest-scored alert per cell
+    seen_cells: dict[str, dict] = {}
+    for alert in alerts:
+        key = _cell_key(alert["lat"], alert["lng"])
+        if key not in seen_cells or alert["score"] > seen_cells[key]["score"]:
+            seen_cells[key] = alert
+
+    result = list(seen_cells.values())
+    if result:
+        by_context = defaultdict(int)
+        for a in result:
+            by_context[a["context"]] += 1
+        logger.info(
+            "Contradictions: %d possible (%s)",
+            len(result),
+            ", ".join(f"{v} {k}" for k, v in sorted(by_context.items())),
+        )
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+# ---------------------------------------------------------------------------
+# Correlation → Pin bridge
+# ---------------------------------------------------------------------------
+
+# Types and their pin categories
+_CORR_PIN_CATEGORIES = {
+    "rf_anomaly": "anomaly",
+    "military_buildup": "military",
+    "infra_cascade": "infrastructure",
+    "contradiction": "research",
+}
+
+# Deduplicate: don't re-pin the same cell within this window (seconds).
+_CORR_PIN_DEDUP_WINDOW = 600  # 10 minutes
+_recent_corr_pins: dict[str, float] = {}
+
+
+def _auto_pin_correlations(alerts: list[dict]) -> int:
+    """Create AI Intel pins for high-severity correlation alerts.
+
+    Only pins alerts with severity >= medium.  Uses cell-key dedup so the
+    same grid cell doesn't get re-pinned every fetch cycle.
+
+    Returns the number of pins created this cycle.
+    """
+    import time as _time
+
+    now = _time.time()
+
+    # Evict stale dedup entries
+    expired = [k for k, ts in _recent_corr_pins.items() if now - ts > _CORR_PIN_DEDUP_WINDOW]
+    for k in expired:
+        _recent_corr_pins.pop(k, None)
+
+    created = 0
+    for alert in alerts:
+        sev = alert.get("severity", "low")
+        if sev == "low":
+            continue  # Don't pin low-severity noise
+
+        lat = alert.get("lat")
+        lng = alert.get("lng")
+        if lat is None or lng is None:
+            continue
+
+        # Dedup key: type + cell
+        dedup_key = f"{alert['type']}:{_cell_key(lat, lng)}"
+        if dedup_key in _recent_corr_pins:
+            continue
+
+        category = _CORR_PIN_CATEGORIES.get(alert["type"], "anomaly")
+        drivers = alert.get("drivers", [])
+        atype = alert["type"]
+
+        if atype == "contradiction":
+            ctx = alert.get("context", "")
+            label = f"[{ctx}] Possible Contradiction"
+            parts = list(drivers)
+            if alert.get("alternatives"):
+                parts.append("Alternatives: " + "; ".join(alert["alternatives"][:2]))
+            description = " | ".join(parts) if parts else "Narrative contradiction detected"
+        else:
+            label = f"[{sev.upper()}] {atype.replace('_', ' ').title()}"
+            description = "; ".join(drivers) if drivers else "Multi-layer correlation alert"
+
+        try:
+            from services.ai_pin_store import create_pin
+
+            meta = {
+                "correlation_type": atype,
+                "severity": sev,
+                "drivers": drivers,
+                "cell_size": alert.get("cell_size", _CELL_SIZE),
+            }
+            # Add contradiction-specific metadata
+            if atype == "contradiction":
+                meta["context_rating"] = alert.get("context", "")
+                meta["alternatives"] = alert.get("alternatives", [])
+                meta["headlines"] = alert.get("headlines", [])
+                meta["location_name"] = alert.get("location_name", "")
+                if alert.get("related_markets"):
+                    meta["related_markets"] = alert["related_markets"]
+
+            create_pin(
+                lat=lat,
+                lng=lng,
+                label=label,
+                category=category,
+                description=description,
+                source="correlation_engine",
+                confidence=alert.get("score", 60) / 100.0,
+                ttl_hours=2.0,  # Auto-expire correlation pins after 2 hours
+                metadata=meta,
+            )
+            _recent_corr_pins[dedup_key] = now
+            created += 1
+        except Exception as exc:
+            logger.warning("Failed to auto-pin correlation: %s", exc)
+
+    if created:
+        logger.info("Correlation engine auto-pinned %d alerts", created)
+    return created
+
+
 # ---------------------------------------------------------------------------
 # Public API
 # ---------------------------------------------------------------------------
@@ -330,13 +755,29 @@ def compute_correlations(data: dict) -> list[dict]:
     except Exception as e:
         logger.error("Correlation engine infra cascade error: %s", e)
 
+    # Contradiction detection removed from automated engine — too many false
+    # positives from regex headline matching.  Contradiction/analysis alerts are
+    # now placed by OpenClaw agents via place_analysis_zone, which lets an LLM
+    # reason about the evidence rather than pattern-matching keywords.
+    try:
+        from services.analysis_zone_store import get_live_zones
+        alerts.extend(get_live_zones())
+    except Exception as e:
+        logger.error("Analysis zone merge error: %s", e)
+
     rf = sum(1 for a in alerts if a["type"] == "rf_anomaly")
     mil = sum(1 for a in alerts if a["type"] == "military_buildup")
     infra = sum(1 for a in alerts if a["type"] == "infra_cascade")
+    contra = sum(1 for a in alerts if a["type"] == "contradiction")
     if alerts:
         logger.info(
-            "Correlations: %d alerts (%d rf, %d mil, %d infra)",
-            len(alerts), rf, mil, infra,
+            "Correlations: %d alerts (%d rf, %d mil, %d infra, %d contra)",
+            len(alerts), rf, mil, infra, contra,
         )
 
+    # Correlation alerts are returned in the correlations data feed only.
+    # They are NOT auto-pinned to AI Intel — that layer is reserved for
+    # user / OpenClaw pins.  Correlations are visualised via the dedicated
+    # correlations overlay on the map.
+
     return alerts

backend/services/feed_ingester.py

@@ -0,0 +1,245 @@
+"""Feed Ingester — background daemon that refreshes feed-backed pin layers.
+
+Layers with a non-empty `feed_url` are polled at their `feed_interval`
+(seconds, minimum 60). The feed is expected to return either:
+
+  1. GeoJSON FeatureCollection — features are converted to pins
+  2. JSON array of pin objects — used directly
+
+Each refresh atomically replaces the layer's pins with the new data.
+"""
+
+import logging
+import threading
+import time
+from typing import Any
+
+import requests
+
+from services.network_utils import outbound_user_agent
+
+logger = logging.getLogger(__name__)
+
+
+def _feed_ingester_user_agent() -> str:
+    # Round 7a: per-install attribution for operator-curated feed URLs.
+    return outbound_user_agent("feed-ingester")
+
+# ---------------------------------------------------------------------------
+# State
+# ---------------------------------------------------------------------------
+
+_running = False
+_thread: threading.Thread | None = None
+_CHECK_INTERVAL = 30  # seconds between scanning for layers that need refresh
+_last_fetched: dict[str, float] = {}  # layer_id → last fetch timestamp
+_FETCH_TIMEOUT = 20  # seconds
+
+# ---------------------------------------------------------------------------
+# GeoJSON → pin conversion
+# ---------------------------------------------------------------------------
+
+
+def _geojson_features_to_pins(features: list[dict]) -> list[dict[str, Any]]:
+    """Convert GeoJSON Feature objects to pin dicts."""
+    pins: list[dict[str, Any]] = []
+    for feat in features:
+        if not isinstance(feat, dict):
+            continue
+        geom = feat.get("geometry") or {}
+        props = feat.get("properties") or {}
+
+        # Extract coordinates
+        coords = geom.get("coordinates")
+        if geom.get("type") != "Point" or not coords or len(coords) < 2:
+            continue
+
+        lng, lat = float(coords[0]), float(coords[1])
+        if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+            continue
+
+        pin: dict[str, Any] = {
+            "lat": lat,
+            "lng": lng,
+            "label": str(props.get("label", props.get("name", props.get("title", ""))))[:200],
+            "category": str(props.get("category", "custom"))[:50],
+            "color": str(props.get("color", ""))[:20],
+            "description": str(props.get("description", props.get("summary", "")))[:2000],
+            "source": "feed",
+            "source_url": str(props.get("source_url", props.get("url", props.get("link", ""))))[:500],
+            "confidence": float(props.get("confidence", 1.0)),
+        }
+
+        # Entity attachment if present
+        entity_type = props.get("entity_type", "")
+        entity_id = props.get("entity_id", "")
+        if entity_type and entity_id:
+            pin["entity_attachment"] = {
+                "entity_type": str(entity_type),
+                "entity_id": str(entity_id),
+                "entity_label": str(props.get("entity_label", "")),
+            }
+
+        pins.append(pin)
+    return pins
+
+
+def _parse_feed_response(data: Any) -> list[dict[str, Any]]:
+    """Parse a feed response into a list of pin dicts."""
+    if isinstance(data, dict):
+        # GeoJSON FeatureCollection
+        if data.get("type") == "FeatureCollection" and isinstance(data.get("features"), list):
+            return _geojson_features_to_pins(data["features"])
+        # Single Feature
+        if data.get("type") == "Feature":
+            return _geojson_features_to_pins([data])
+        # Wrapped response like {"ok": true, "data": [...]}
+        inner = data.get("data") or data.get("results") or data.get("pins") or data.get("items")
+        if isinstance(inner, list):
+            return _normalize_pin_list(inner)
+
+    if isinstance(data, list):
+        # Check if first item looks like a GeoJSON Feature
+        if data and isinstance(data[0], dict) and data[0].get("type") == "Feature":
+            return _geojson_features_to_pins(data)
+        return _normalize_pin_list(data)
+
+    return []
+
+
+def _normalize_pin_list(items: list) -> list[dict[str, Any]]:
+    """Normalize a list of raw pin objects, ensuring lat/lng are present."""
+    pins: list[dict[str, Any]] = []
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        lat = item.get("lat") or item.get("latitude")
+        lng = item.get("lng") or item.get("lon") or item.get("longitude")
+        if lat is None or lng is None:
+            continue
+        try:
+            lat, lng = float(lat), float(lng)
+        except (ValueError, TypeError):
+            continue
+        if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+            continue
+
+        pin: dict[str, Any] = {
+            "lat": lat,
+            "lng": lng,
+            "label": str(item.get("label", item.get("name", item.get("title", ""))))[:200],
+            "category": str(item.get("category", "custom"))[:50],
+            "color": str(item.get("color", ""))[:20],
+            "description": str(item.get("description", item.get("summary", "")))[:2000],
+            "source": "feed",
+            "source_url": str(item.get("source_url", item.get("url", item.get("link", ""))))[:500],
+            "confidence": float(item.get("confidence", 1.0)),
+        }
+
+        entity_type = item.get("entity_type", "")
+        entity_id = item.get("entity_id", "")
+        if entity_type and entity_id:
+            pin["entity_attachment"] = {
+                "entity_type": str(entity_type),
+                "entity_id": str(entity_id),
+                "entity_label": str(item.get("entity_label", "")),
+            }
+
+        pins.append(pin)
+    return pins
+
+
+# ---------------------------------------------------------------------------
+# Fetch a single layer
+# ---------------------------------------------------------------------------
+
+
+def _fetch_layer_feed(layer: dict[str, Any]) -> None:
+    """Fetch a feed URL and replace the layer's pins."""
+    layer_id = layer["id"]
+    feed_url = layer["feed_url"]
+    layer_name = layer.get("name", layer_id)
+
+    try:
+        resp = requests.get(
+            feed_url,
+            timeout=_FETCH_TIMEOUT,
+            headers={"User-Agent": _feed_ingester_user_agent()},
+        )
+        resp.raise_for_status()
+        data = resp.json()
+    except requests.RequestException as e:
+        logger.warning("Feed fetch failed for layer '%s' (%s): %s", layer_name, feed_url, e)
+        return
+    except (ValueError, TypeError) as e:
+        logger.warning("Feed parse failed for layer '%s' (%s): %s", layer_name, feed_url, e)
+        return
+
+    pins = _parse_feed_response(data)
+
+    from services.ai_pin_store import replace_layer_pins, update_layer
+    count = replace_layer_pins(layer_id, pins)
+
+    # Update layer metadata with last_fetched timestamp
+    update_layer(layer_id, feed_last_fetched=time.time())
+
+    _last_fetched[layer_id] = time.time()
+    logger.info("Feed refresh for layer '%s': %d pins from %s", layer_name, count, feed_url)
+
+
+# ---------------------------------------------------------------------------
+# Main loop
+# ---------------------------------------------------------------------------
+
+
+def _ingest_loop() -> None:
+    """Daemon loop: scan for feed layers and refresh those that are due."""
+    while _running:
+        try:
+            from services.ai_pin_store import get_feed_layers
+
+            layers = get_feed_layers()
+            now = time.time()
+
+            for layer in layers:
+                layer_id = layer["id"]
+                interval = max(60, layer.get("feed_interval", 300))
+                last = _last_fetched.get(layer_id, 0)
+
+                if now - last >= interval:
+                    try:
+                        _fetch_layer_feed(layer)
+                    except Exception as e:
+                        logger.warning("Feed ingestion error for layer %s: %s",
+                                       layer.get("name", layer_id), e)
+
+        except Exception as e:
+            logger.error("Feed ingester loop error: %s", e)
+
+        # Sleep in short increments so we can stop cleanly
+        for _ in range(int(_CHECK_INTERVAL)):
+            if not _running:
+                break
+            time.sleep(1)
+
+
+# ---------------------------------------------------------------------------
+# Start / stop
+# ---------------------------------------------------------------------------
+
+
+def start_feed_ingester() -> None:
+    """Start the feed ingester daemon thread."""
+    global _running, _thread
+    if _thread and _thread.is_alive():
+        return
+    _running = True
+    _thread = threading.Thread(target=_ingest_loop, daemon=True, name="feed-ingester")
+    _thread.start()
+    logger.info("Feed ingester daemon started (check interval=%ds)", _CHECK_INTERVAL)
+
+
+def stop_feed_ingester() -> None:
+    """Stop the feed ingester daemon."""
+    global _running
+    _running = False

backend/services/fetchers/_store.py

@@ -4,6 +4,7 @@ Central location for latest_data, source_timestamps, and the data lock.
 Every fetcher imports from here instead of maintaining its own copy.
 """
 
+import copy
 import threading
 import logging
 from datetime import datetime
@@ -42,6 +43,7 @@ class DashboardData(TypedDict, total=False):
     gps_jamming: List[Dict[str, Any]]
     satellites: List[Dict[str, Any]]
     satellite_source: str
+    satellite_analysis: Dict[str, Any]
     prediction_markets: List[Dict[str, Any]]
     sigint: List[Dict[str, Any]]
     sigint_totals: Dict[str, Any]
@@ -61,6 +63,18 @@ class DashboardData(TypedDict, total=False):
     fimi: Dict[str, Any]
     psk_reporter: List[Dict[str, Any]]
     correlations: List[Dict[str, Any]]
+    uap_sightings: List[Dict[str, Any]]
+    wastewater: List[Dict[str, Any]]
+    crowdthreat: List[Dict[str, Any]]
+    sar_scenes: List[Dict[str, Any]]
+    sar_anomalies: List[Dict[str, Any]]
+    sar_aoi_coverage: List[Dict[str, Any]]
+    road_corridor_trends: Dict[str, Any]
+    malware_threats: Dict[str, Any]
+    cyber_threats: Dict[str, Any]
+    scm_suppliers: Dict[str, Any]
+    telegram_osint: Dict[str, Any]
+    gt_risk: Dict[str, Any]
 
 
 # In-memory store
@@ -105,6 +119,24 @@ latest_data: DashboardData = {
     "fimi": {},
     "psk_reporter": [],
     "correlations": [],
+    "uap_sightings": [],
+    "wastewater": [],
+    "crowdthreat": [],
+    "sar_scenes": [],
+    "sar_anomalies": [],
+    "sar_aoi_coverage": [],
+    "road_corridor_trends": {"updated_at": None, "corridors": []},
+    "malware_threats": {"threats": [], "total": 0, "timestamp": None},
+    "cyber_threats": {"threats": [], "stats": {}},
+    "scm_suppliers": {"suppliers": [], "total": 0, "critical_count": 0},
+    "telegram_osint": {"posts": [], "total": 0, "geolocated": 0, "timestamp": None},
+    "gt_risk": {
+        "enabled": False,
+        "heatmap": {"type": "FeatureCollection", "features": []},
+        "clusters": [],
+        "processed": 0,
+        "timestamp": None,
+    },
 }
 
 # Per-source freshness timestamps
@@ -117,9 +149,21 @@ source_freshness: dict[str, dict] = {}
 def _mark_fresh(*keys):
     """Record the current UTC time for one or more data source keys."""
     now = datetime.utcnow().isoformat()
+    global _data_version
+    changed: list[tuple[str, int, int]] = []  # (layer, version, count)
     with _data_lock:
         for k in keys:
             source_timestamps[k] = now
+            _layer_versions[k] = _layer_versions.get(k, 0) + 1
+            # Grab entity count while we hold the lock (cheap len())
+            val = latest_data.get(k)
+            count = len(val) if isinstance(val, list) else (1 if val is not None else 0)
+            changed.append((k, _layer_versions[k], count))
+        # Publish partial fetch progress immediately so the frontend can
+        # observe newly available data without waiting for the entire tier.
+        _data_version += 1
+    # Notify SSE listeners outside the lock to avoid deadlocks
+    _notify_layer_change(changed)
 
 
 # Thread lock for safe reads/writes to latest_data
@@ -129,16 +173,73 @@ _data_lock = threading.Lock()
 # Used for cheap ETag generation instead of MD5-hashing the full response.
 _data_version: int = 0
 
+# Per-layer version counters — incremented only when that specific layer
+# refreshes.  Used by get_layer_slice for per-layer incremental updates
+# and by the SSE stream to push targeted layer_changed notifications.
+_layer_versions: dict[str, int] = {}
+
+
+# ---------------------------------------------------------------------------
+# Layer-change notification callbacks (thread → async SSE bridge)
+# ---------------------------------------------------------------------------
+_layer_change_callbacks: list = []
+_layer_change_callbacks_lock = threading.Lock()
+
+
+def register_layer_change_callback(callback) -> None:
+    """Register a callback invoked on every _mark_fresh().
+
+    Signature: callback(layer: str, version: int, count: int)
+    Called from fetcher threads — must be thread-safe.
+    """
+    with _layer_change_callbacks_lock:
+        _layer_change_callbacks.append(callback)
+
+
+def unregister_layer_change_callback(callback) -> None:
+    """Remove a previously registered callback."""
+    with _layer_change_callbacks_lock:
+        try:
+            _layer_change_callbacks.remove(callback)
+        except ValueError:
+            pass
+
+
+def _notify_layer_change(changed: list[tuple[str, int, int]]) -> None:
+    """Fire all registered callbacks for each changed layer."""
+    with _layer_change_callbacks_lock:
+        cbs = list(_layer_change_callbacks)
+    for cb in cbs:
+        for layer, version, count in changed:
+            try:
+                cb(layer, version, count)
+            except Exception:
+                pass
+
+
+def get_layer_versions() -> dict[str, int]:
+    """Return a snapshot of all per-layer version counters."""
+    with _data_lock:
+        return dict(_layer_versions)
+
+
+def get_layer_version(layer: str) -> int:
+    """Return the version counter for a single layer (0 if never refreshed)."""
+    with _data_lock:
+        return _layer_versions.get(layer, 0)
+
 
 def bump_data_version() -> None:
     """Increment the data version counter after a fetch cycle completes."""
     global _data_version
-    _data_version += 1
+    with _data_lock:
+        _data_version += 1
 
 
 def get_data_version() -> int:
     """Return the current data version (for ETag generation)."""
-    return _data_version
+    with _data_lock:
+        return _data_version
 
 
 _active_layers_version: int = 0
@@ -147,31 +248,52 @@ _active_layers_version: int = 0
 def bump_active_layers_version() -> None:
     """Increment the active-layer version when frontend toggles change response shape."""
     global _active_layers_version
-    _active_layers_version += 1
+    with _data_lock:
+        _active_layers_version += 1
 
 
 def get_active_layers_version() -> int:
     """Return the current active-layer version (for ETag generation)."""
-    return _active_layers_version
+    with _data_lock:
+        return _active_layers_version
 
 
 def get_latest_data_subset(*keys: str) -> DashboardData:
-    """Return a shallow snapshot of only the requested top-level keys.
+    """Return a deep snapshot of only the requested top-level keys.
 
-    This avoids cloning the entire dashboard store for endpoints that only need
-    a small tier-specific subset.
+    Grabs references under the lock, then deep-copies outside it so fetcher
+    writers are not blocked for the duration of a large clone (#375).
     """
     with _data_lock:
-        snap: DashboardData = {}
-        for key in keys:
-            value = latest_data.get(key)
-            if isinstance(value, list):
-                snap[key] = list(value)
-            elif isinstance(value, dict):
-                snap[key] = dict(value)
-            else:
-                snap[key] = value
-        return snap
+        items = [(key, latest_data.get(key)) for key in keys]
+    snap: DashboardData = {}
+    for key, value in items:
+        snap[key] = copy.deepcopy(value)
+    return snap
+
+
+def get_latest_data_deepcopy_snapshot() -> DashboardData:
+    """Deep-copy the full dashboard for /api/health and legacy /api/live-data.
+
+    The per-value deepcopy runs OUTSIDE ``_data_lock`` so a large clone cannot
+    block fetcher writers (#375). The store contract is replace-don't-mutate,
+    but a writer that mutates a nested object in place (e.g. a live bridge
+    updating an entry that is also published in this store) can race the
+    deepcopy and raise ``RuntimeError: dictionary changed size during
+    iteration`` — surfacing a 500 on the health/live-data path. The racing
+    mutation window is tiny, so retry a few times rather than fail; a fresh
+    attempt almost always lands on a quiescent moment. Defense-in-depth on top
+    of fixing the offending writers, not a substitute for it.
+    """
+    attempts = 4
+    for attempt in range(attempts):
+        with _data_lock:
+            items = list(latest_data.items())
+        try:
+            return {key: copy.deepcopy(value) for key, value in items}
+        except RuntimeError:
+            if attempt == attempts - 1:
+                raise
 
 
 def get_latest_data_subset_refs(*keys: str) -> DashboardData:
@@ -231,10 +353,23 @@ active_layers: dict[str, bool] = {
     "satnogs": True,
     "tinygs": True,
     "ukraine_alerts": True,
-    "power_plants": False,
+    "power_plants": True,
     "viirs_nightlights": False,
     "psk_reporter": True,
     "correlations": True,
+    "contradictions": True,
+    "uap_sightings": True,
+    "wastewater": True,
+    "ai_intel": True,
+    "crowdthreat": False,
+    "sar": True,
+    "road_corridor_trends": False,
+    "malware_c2": False,
+    "submarine_cables": False,
+    "scm_suppliers": False,
+    "cyber_threats": False,
+    "telegram_osint": True,
+    "gt_risk": False,
 }
 
 

backend/services/fetchers/aircraft_database.py

@@ -0,0 +1,178 @@
+"""OpenSky aircraft metadata: ICAO24 hex -> ICAO type code + friendly model.
+
+OpenSky's /states/all does not include aircraft type, so OpenSky-sourced
+flights arrive with ``t`` field empty. This module bulk-loads the public
+OpenSky aircraft database (one snapshot CSV per month, ~108 MB uncompressed,
+~600k aircraft) once every 5 days and exposes a fast in-memory hex lookup.
+
+The data is also useful when adsb.lol's live API is degraded: even the
+adsb.lol /v2 feed sometimes returns aircraft with empty ``t`` for newly seen
+transponders, and the lookup gracefully fills those in too.
+"""
+
+from __future__ import annotations
+
+import csv
+import logging
+import threading
+import time
+from typing import Any
+
+import defusedxml.ElementTree as ET
+import requests
+
+
+
+def _aircraft_db_user_agent() -> str:
+    """Round 7a: lazy import so the per-install operator handle is included."""
+    from services.network_utils import outbound_user_agent
+    return outbound_user_agent("aircraft-database")
+
+logger = logging.getLogger(__name__)
+
+_BUCKET_LIST_URL = (
+    "https://s3.opensky-network.org/data-samples?prefix=metadata/&list-type=2"
+)
+_BUCKET_BASE = "https://s3.opensky-network.org/data-samples/"
+_S3_NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
+_REFRESH_INTERVAL_S = 5 * 24 * 3600
+_LIST_TIMEOUT_S = 30
+_DOWNLOAD_TIMEOUT_S = 600
+_lock = threading.RLock()
+_aircraft_by_hex: dict[str, dict[str, str]] = {}
+_last_refresh = 0.0
+_in_progress = False
+
+
+def _latest_snapshot_key() -> str:
+    """Discover the most recent aircraft-database-complete snapshot key."""
+    response = requests.get(
+        _BUCKET_LIST_URL,
+        timeout=_LIST_TIMEOUT_S,
+        headers={"User-Agent": _aircraft_db_user_agent()},
+    )
+    response.raise_for_status()
+    root = ET.fromstring(response.text)
+    keys: list[str] = []
+    for content in root.iter(f"{_S3_NS}Contents"):
+        key_el = content.find(f"{_S3_NS}Key")
+        if key_el is None or not key_el.text:
+            continue
+        if "aircraft-database-complete-" in key_el.text and key_el.text.endswith(".csv"):
+            keys.append(key_el.text)
+    if not keys:
+        raise RuntimeError("no aircraft-database-complete snapshot found in bucket listing")
+    return sorted(keys)[-1]
+
+
+def _stream_csv_index(url: str) -> dict[str, dict[str, str]]:
+    """Stream-parse the OpenSky aircraft CSV into a hex-keyed index.
+
+    The CSV uses single-quote quoting, so csv.DictReader is configured with
+    ``quotechar="'"``. Rows are processed line-by-line via iter_lines() to
+    keep memory bounded even though the file is ~108 MB.
+    """
+    with requests.get(
+        url,
+        timeout=_DOWNLOAD_TIMEOUT_S,
+        stream=True,
+        headers={"User-Agent": _aircraft_db_user_agent()},
+    ) as response:
+        response.raise_for_status()
+        line_iter = (
+            line.decode("utf-8", errors="replace")
+            for line in response.iter_lines(decode_unicode=False)
+            if line
+        )
+        reader = csv.DictReader(line_iter, quotechar="'")
+        index: dict[str, dict[str, str]] = {}
+        for row in reader:
+            hex_code = (row.get("icao24") or "").strip().lower()
+            if not hex_code or hex_code == "000000":
+                continue
+            typecode = (row.get("typecode") or "").strip().upper()
+            model = (row.get("model") or "").strip()
+            mfr = (row.get("manufacturerName") or "").strip()
+            registration = (row.get("registration") or "").strip().upper()
+            operator = (row.get("operator") or "").strip()
+            if not (typecode or model):
+                continue
+            entry: dict[str, str] = {}
+            if typecode:
+                entry["typecode"] = typecode
+            if model:
+                entry["model"] = model
+            if mfr:
+                entry["manufacturer"] = mfr
+            if registration:
+                entry["registration"] = registration
+            if operator:
+                entry["operator"] = operator
+            index[hex_code] = entry
+    return index
+
+
+def refresh_aircraft_database(force: bool = False) -> bool:
+    """Download the latest OpenSky aircraft snapshot and rebuild the index.
+
+    Returns True if a refresh was performed (success or attempted), False if
+    skipped because the cache is still fresh or another refresh is in flight.
+    """
+    global _last_refresh, _in_progress
+
+    now = time.time()
+    with _lock:
+        if _in_progress:
+            return False
+        if not force and (now - _last_refresh) < _REFRESH_INTERVAL_S and _aircraft_by_hex:
+            return False
+        _in_progress = True
+
+    try:
+        started = time.time()
+        key = _latest_snapshot_key()
+        index = _stream_csv_index(_BUCKET_BASE + key)
+        with _lock:
+            _aircraft_by_hex.clear()
+            _aircraft_by_hex.update(index)
+            _last_refresh = time.time()
+        logger.info(
+            "aircraft database refreshed in %.1fs from %s: %d aircraft",
+            time.time() - started,
+            key,
+            len(index),
+        )
+        return True
+    except (requests.RequestException, OSError, ValueError, ET.ParseError) as exc:
+        logger.warning("aircraft database refresh failed: %s", exc)
+        return True
+    finally:
+        with _lock:
+            _in_progress = False
+
+
+def lookup_aircraft(icao24: str) -> dict[str, str] | None:
+    """Return the metadata record for an ICAO24 hex code, or None."""
+    key = (icao24 or "").strip().lower()
+    if not key:
+        return None
+    with _lock:
+        entry = _aircraft_by_hex.get(key)
+    return dict(entry) if entry else None
+
+
+def lookup_aircraft_type(icao24: str) -> str:
+    """Return the ICAO type code (e.g. 'B738', 'GLF4') or '' if unknown."""
+    entry = lookup_aircraft(icao24)
+    if not entry:
+        return ""
+    return entry.get("typecode", "")
+
+
+def aircraft_database_status() -> dict[str, Any]:
+    with _lock:
+        return {
+            "last_refresh": _last_refresh,
+            "aircraft": len(_aircraft_by_hex),
+            "in_progress": _in_progress,
+        }

backend/services/fetchers/aishub_fallback.py

@@ -0,0 +1,290 @@
+"""AISHub REST fallback for ship tracking when AISStream is unreachable.
+
+Background
+----------
+On 2026-05-23 ``stream.aisstream.io`` (the primary live AIS WebSocket feed)
+went fully offline. Backend's only ship signal vanished. This module polls
+``data.aishub.net``'s free REST API on a slow cadence (default 20 min) when
+the WebSocket primary is disconnected, so the ships layer doesn't go fully
+dark during upstream outages.
+
+Why 20 minutes
+--------------
+AISHub's free tier is rate-limited and explicitly asks consumers to be
+courteous. 20 minutes is well inside their limits, gives ships time to
+move enough to look "alive" on the map, and won't drain their service.
+Configurable via the ``AISHUB_POLL_INTERVAL_MINUTES`` env var (clamped to
+[1, 360]).
+
+Why slow vs primary
+-------------------
+This is degraded mode, not a replacement. A ship at 20 knots moves about
+6 nautical miles in 20 minutes — visible on the map but coarser than the
+real-time WebSocket signal. When AISStream comes back online, the
+WebSocket data will overwrite these records via the same ``_vessels``
+dict and ``source`` will flip from ``"aishub"`` back to upstream-live.
+
+Opt-in
+------
+Operator must set ``AISHUB_USERNAME`` (free registration at
+https://www.aishub.net/api). If unset, this fetcher is a no-op.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import time
+from typing import Any
+
+from services.network_utils import fetch_with_curl
+
+logger = logging.getLogger(__name__)
+
+
+AISHUB_URL = "https://data.aishub.net/ws.php"
+
+
+def aishub_username() -> str:
+    return str(os.environ.get("AISHUB_USERNAME", "")).strip()
+
+
+def aishub_fallback_enabled() -> bool:
+    """Returns True only when the operator has registered with AISHub and
+    set ``AISHUB_USERNAME``. The presence of the username is the opt-in."""
+    return bool(aishub_username())
+
+
+def aishub_poll_interval_minutes() -> int:
+    """Default 20 minutes. Clamped to [1, 360] so a hostile or
+    misconfigured env var can't either hammer the upstream or silence the
+    fallback for a day."""
+    raw = os.environ.get("AISHUB_POLL_INTERVAL_MINUTES", "20")
+    try:
+        value = int(str(raw).strip())
+    except (TypeError, ValueError):
+        value = 20
+    return max(1, min(360, value))
+
+
+def _should_run_fallback() -> bool:
+    """Only run when the primary WebSocket is disconnected. Avoids stomping
+    over fresher live data when AISStream is healthy.
+
+    Returns False if:
+      * AISHub isn't configured (no username)
+      * AISStream primary is currently connected (recent vessel messages)
+
+    Returns True only when AIS is configured-but-down. The
+    ``proxy_spawn_count > 0`` guard means "the primary has at least tried
+    to run" — if the user set AISHUB_USERNAME but not AIS_API_KEY at all,
+    AISHub will still serve as a primary on its own slow cadence.
+    """
+    if not aishub_fallback_enabled():
+        return False
+    try:
+        from services.ais_stream import ais_proxy_status
+        status = ais_proxy_status() or {}
+    except Exception:
+        return True  # ais_stream not importable? still try AISHub.
+    # If the WebSocket primary is connected, skip the fallback — fresher
+    # data is already flowing.
+    if status.get("connected") is True:
+        return False
+    return True
+
+
+def _parse_aishub_response(payload: str) -> list[dict]:
+    """Parse the AISHub JSON response into a list of vessel records.
+
+    Successful response shape::
+
+        [
+            {"ERROR": false, "USERNAME": "...", "FORMAT": "1", "RECORDS": N},
+            [{"MMSI": ..., "LATITUDE": ..., "LONGITUDE": ..., ...}, ...]
+        ]
+
+    Error response shape::
+
+        [{"ERROR": true, "ERROR_MESSAGE": "..."}]
+
+    Empty payload (e.g. silent rate-limit drop) returns ``[]``.
+    """
+    if not payload or not payload.strip():
+        return []
+    try:
+        data = json.loads(payload)
+    except json.JSONDecodeError as e:
+        logger.warning("AISHub: response is not JSON: %s", e)
+        return []
+    if not isinstance(data, list) or not data:
+        return []
+    header = data[0] if isinstance(data[0], dict) else {}
+    if header.get("ERROR") is True:
+        logger.warning(
+            "AISHub: upstream error: %s",
+            header.get("ERROR_MESSAGE", "<unspecified>"),
+        )
+        return []
+    if len(data) < 2 or not isinstance(data[1], list):
+        return []
+    return [row for row in data[1] if isinstance(row, dict)]
+
+
+def _normalize_record(row: dict) -> dict | None:
+    """Map an AISHub vessel record to our internal vessel schema.
+
+    Returns None when the record can't be used (no MMSI, bad position,
+    sentinel "not available" lat/lng).
+    """
+    try:
+        mmsi = int(row.get("MMSI") or 0)
+    except (TypeError, ValueError):
+        return None
+    if not mmsi:
+        return None
+    try:
+        lat = float(row.get("LATITUDE"))
+        lng = float(row.get("LONGITUDE"))
+    except (TypeError, ValueError):
+        return None
+    # AIS uses 91/181 as "no position available" sentinels.
+    if abs(lat) > 90 or abs(lng) > 180:
+        return None
+    if lat == 91.0 or lng == 181.0:
+        return None
+    # SOG raw 102.3 is "speed not available"; sanitize to 0.
+    try:
+        sog_raw = float(row.get("SOG") or 0)
+    except (TypeError, ValueError):
+        sog_raw = 0.0
+    sog = 0.0 if sog_raw >= 102.2 else sog_raw
+    try:
+        cog = float(row.get("COG") or 0)
+    except (TypeError, ValueError):
+        cog = 0.0
+    try:
+        heading_raw = int(row.get("HEADING") or 511)
+    except (TypeError, ValueError):
+        heading_raw = 511
+    # AIS heading sentinel 511 = "not available" — fall back to COG.
+    heading = heading_raw if heading_raw != 511 else cog
+    try:
+        ais_type = int(row.get("TYPE") or 0)
+    except (TypeError, ValueError):
+        ais_type = 0
+    return {
+        "mmsi": mmsi,
+        "lat": lat,
+        "lng": lng,
+        "sog": sog,
+        "cog": cog,
+        "heading": heading,
+        "name": str(row.get("NAME") or "").strip() or "UNKNOWN",
+        "callsign": str(row.get("CALLSIGN") or "").strip(),
+        "destination": str(row.get("DEST") or "").strip().replace("@", "") or "",
+        "imo": int(row.get("IMO") or 0),
+        "ais_type_code": ais_type,
+    }
+
+
+def fetch_aishub_vessels() -> int:
+    """Poll AISHub and merge vessels into the shared ``_vessels`` store.
+
+    Returns the number of vessels updated (0 on skip, error, or no data).
+    Designed to be called by the APScheduler tier — see
+    ``data_fetcher.py`` for the 20-minute interval job that wraps this.
+    """
+    if not _should_run_fallback():
+        logger.debug("AISHub fallback skipped: primary connected or not configured")
+        return 0
+
+    username = aishub_username()
+    url = (
+        f"{AISHUB_URL}?username={username}&format=1&output=json"
+        f"&compress=0"
+    )
+
+    try:
+        response = fetch_with_curl(url, timeout=30)
+    except Exception as e:
+        logger.warning("AISHub fetch failed: %s", e)
+        return 0
+
+    if not response or response.status_code != 200:
+        logger.warning(
+            "AISHub HTTP %s",
+            getattr(response, "status_code", "None"),
+        )
+        return 0
+
+    rows = _parse_aishub_response(getattr(response, "text", "") or "")
+    if not rows:
+        return 0
+
+    # Inline imports to avoid a circular dependency at module load time
+    # (ais_stream imports lots of things and is loaded by main.py).
+    from services.ais_stream import (
+        _vessels,
+        _vessels_lock,
+        _record_vessel_trail_locked,
+        classify_vessel,
+        get_country_from_mmsi,
+    )
+
+    now = time.time()
+    count = 0
+    with _vessels_lock:
+        for row in rows:
+            normalized = _normalize_record(row)
+            if normalized is None:
+                continue
+            mmsi = normalized["mmsi"]
+            vessel = _vessels.setdefault(mmsi, {"mmsi": mmsi})
+            # Don't overwrite fresher live data: if the WebSocket pushed an
+            # update for this MMSI more recently than now-1s (race during
+            # the brief reconnection window) keep the live one.
+            last = float(vessel.get("_updated") or 0)
+            if last > now - 1:
+                continue
+            vessel.update(
+                {
+                    "lat": normalized["lat"],
+                    "lng": normalized["lng"],
+                    "sog": normalized["sog"],
+                    "cog": normalized["cog"],
+                    "heading": normalized["heading"],
+                    "_updated": now,
+                    "source": "aishub",
+                }
+            )
+            if normalized["name"] and normalized["name"] != "UNKNOWN":
+                vessel["name"] = normalized["name"]
+            if normalized["callsign"]:
+                vessel["callsign"] = normalized["callsign"]
+            if normalized["destination"]:
+                vessel["destination"] = normalized["destination"]
+            if normalized["imo"]:
+                vessel["imo"] = normalized["imo"]
+            if normalized["ais_type_code"]:
+                vessel["ais_type_code"] = normalized["ais_type_code"]
+                vessel["type"] = classify_vessel(normalized["ais_type_code"], mmsi)
+            if not vessel.get("country"):
+                vessel["country"] = get_country_from_mmsi(mmsi)
+            _record_vessel_trail_locked(
+                mmsi,
+                normalized["lat"],
+                normalized["lng"],
+                normalized["sog"],
+                now,
+            )
+            count += 1
+
+    if count:
+        logger.info(
+            "AISHub fallback: merged %d vessels (poll interval %d min)",
+            count,
+            aishub_poll_interval_minutes(),
+        )
+    return count

backend/services/fetchers/crowdthreat.py

@@ -0,0 +1,146 @@
+"""CrowdThreat fetcher — crowdsourced global threat intelligence.
+
+Polls verified threat reports from CrowdThreat's public API and normalises
+them into map-ready records with category-based icon IDs.
+
+No API key required — the /threats endpoint is unauthenticated.
+"""
+
+import logging
+import os
+
+from services.network_utils import fetch_with_curl
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh, is_any_active
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger("services.data_fetcher")
+
+_CT_BASE = "https://backend.crowdthreat.world"
+
+
+def crowdthreat_fetch_enabled() -> bool:
+    """Return True only when the operator explicitly opts into CrowdThreat pulls."""
+    return str(os.environ.get("CROWDTHREAT_ENABLED", "")).strip().lower() in {
+        "1",
+        "true",
+        "yes",
+        "on",
+    }
+
+# CrowdThreat category_id → icon ID used on the MapLibre layer
+_CATEGORY_ICON = {
+    1: "ct-security",       # Security & Conflict  (red)
+    2: "ct-crime",          # Crime & Safety        (blue)
+    3: "ct-aviation",       # Aviation              (green)
+    4: "ct-maritime",       # Maritime              (teal)
+    5: "ct-infrastructure", # Industrial & Infra    (orange)
+    6: "ct-special",        # Special Threats        (purple)
+    7: "ct-social",         # Social & Political    (pink)
+    8: "ct-other",          # Other                 (gray)
+}
+
+_CATEGORY_COLOUR = {
+    1: "#ef4444",  # red
+    2: "#3b82f6",  # blue
+    3: "#22c55e",  # green
+    4: "#14b8a6",  # teal
+    5: "#f97316",  # orange
+    6: "#a855f7",  # purple
+    7: "#ec4899",  # pink
+    8: "#6b7280",  # gray
+}
+
+
+@with_retry(max_retries=2, base_delay=5)
+def fetch_crowdthreat():
+    """Fetch verified threat reports from CrowdThreat public API."""
+    if not crowdthreat_fetch_enabled():
+        logger.debug("CrowdThreat fetch skipped; set CROWDTHREAT_ENABLED=true to opt in")
+        with _data_lock:
+            latest_data["crowdthreat"] = []
+        _mark_fresh("crowdthreat")
+        return
+    if not is_any_active("crowdthreat"):
+        return
+
+    try:
+        resp = fetch_with_curl(f"{_CT_BASE}/threats", timeout=20)
+        if not resp or resp.status_code != 200:
+            logger.warning("CrowdThreat API returned %s", getattr(resp, "status_code", "None"))
+            return
+
+        payload = resp.json()
+        raw_threats = payload.get("data", {}).get("threats", [])
+        if not raw_threats:
+            logger.debug("CrowdThreat returned 0 threats")
+            return
+
+    except Exception as e:
+        logger.error("CrowdThreat fetch error: %s", e)
+        return
+
+    processed = []
+    for t in raw_threats:
+        loc = t.get("location") or {}
+        lng_lat = loc.get("lng_lat")
+        if not lng_lat or len(lng_lat) < 2:
+            continue
+        try:
+            lng = float(lng_lat[0])
+            lat = float(lng_lat[1])
+        except (TypeError, ValueError):
+            continue
+
+        cat = t.get("category") or {}
+        cat_id = cat.get("id", 8)
+        subcat = t.get("subcategory") or {}
+        threat_type = t.get("type") or {}
+        dates = t.get("dates") or {}
+        occurred = dates.get("occurred") or {}
+        reported = dates.get("reported") or {}
+
+        # Extract all available detail from the API response
+        summary = (t.get("summary") or t.get("description") or "").strip()
+        verification = (t.get("verification_status") or t.get("status") or "").strip()
+        country_obj = loc.get("country") or {}
+        country = country_obj.get("name", "") if isinstance(country_obj, dict) else str(country_obj or "")
+        media = t.get("media") or t.get("images") or t.get("attachments") or []
+        source_url = t.get("source_url") or t.get("url") or t.get("link") or ""
+        severity = t.get("severity") or t.get("severity_level") or t.get("risk_level") or ""
+        votes = t.get("votes") or t.get("upvotes") or 0
+        reporter = t.get("user") or t.get("reporter") or {}
+        reporter_name = reporter.get("name", "") if isinstance(reporter, dict) else ""
+
+        processed.append({
+            "id": t.get("id"),
+            "title": t.get("title", ""),
+            "summary": summary[:500] if summary else "",
+            "lat": lat,
+            "lng": lng,
+            "address": loc.get("name", ""),
+            "city": loc.get("city", ""),
+            "country": country,
+            "category": cat.get("name", "Other"),
+            "category_id": cat_id,
+            "category_colour": _CATEGORY_COLOUR.get(cat_id, "#6b7280"),
+            "subcategory": subcat.get("name", ""),
+            "threat_type": threat_type.get("name", ""),
+            "icon_id": _CATEGORY_ICON.get(cat_id, "ct-other"),
+            "occurred": occurred.get("raw", ""),
+            "occurred_iso": occurred.get("iso", ""),
+            "timeago": occurred.get("timeago", ""),
+            "reported": reported.get("raw", ""),
+            "verification": verification,
+            "severity": str(severity),
+            "source_url": source_url,
+            "media_urls": [m.get("url") or m for m in media[:3]] if isinstance(media, list) else [],
+            "votes": int(votes) if votes else 0,
+            "reporter": reporter_name,
+            "source": "CrowdThreat",
+        })
+
+    logger.info("CrowdThreat: fetched %d verified threats", len(processed))
+
+    with _data_lock:
+        latest_data["crowdthreat"] = processed
+    _mark_fresh("crowdthreat")

backend/services/fetchers/cyber_status.py

@@ -0,0 +1,62 @@
+"""CISA KEV + cyber threat stats (Osiris port)."""
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+from typing import Any
+
+from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
+from services.network_utils import fetch_with_curl
+
+logger = logging.getLogger(__name__)
+
+
+def fetch_cyber_threats() -> dict[str, Any]:
+    if not is_any_active("cyber_threats"):
+        return latest_data.get("cyber_threats") or {"threats": [], "stats": {}}
+
+    results: dict[str, Any] = {"threats": [], "stats": {}, "timestamp": datetime.now(timezone.utc).isoformat()}
+    try:
+        resp = fetch_with_curl(
+            "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json",
+            timeout=15,
+        )
+        if resp.status_code == 200:
+            data = resp.json()
+            vulns = data.get("vulnerabilities") or []
+            results["stats"]["cisa_total"] = len(vulns)
+            now = datetime.now(timezone.utc)
+            recent = []
+            for v in vulns:
+                try:
+                    added = datetime.fromisoformat(v.get("dateAdded", "").replace("Z", "+00:00"))
+                    days = (now - added).total_seconds() / 86400
+                except Exception:
+                    continue
+                if days <= 30:
+                    recent.append(v)
+            recent = recent[:10]
+            results["threats"] = [
+                {
+                    "id": v.get("cveID"),
+                    "name": v.get("vulnerabilityName"),
+                    "vendor": v.get("vendorProject"),
+                    "product": v.get("product"),
+                    "severity": "CRITICAL",
+                    "date": v.get("dateAdded"),
+                    "due": v.get("dueDate"),
+                    "source": "CISA KEV",
+                }
+                for v in recent
+            ]
+    except Exception as exc:
+        logger.warning("CISA KEV fetch failed: %s", exc)
+
+    count = len(results["threats"])
+    results["stats"]["active_cves"] = count
+    results["stats"]["threat_level"] = "CRITICAL" if count >= 8 else "HIGH" if count >= 4 else "ELEVATED"
+
+    with _data_lock:
+        latest_data["cyber_threats"] = results
+    _mark_fresh("cyber_threats")
+    return results

backend/services/fetchers/emissions.py

@@ -1,20 +1,24 @@
 """
-Fuel burn & CO2 emissions estimator for private jets.
+Fuel burn & CO2 emissions estimator.
 Based on manufacturer-published cruise fuel burn rates (GPH at long-range cruise).
 1 US gallon of Jet-A produces ~21.1 lbs (9.57 kg) of CO2.
+
+Piston entries use 100LL (avgas), which is close enough to Jet-A in CO2 yield
+(~8.4 kg/gal vs 9.57 kg/gal); we keep one constant to stay simple — the result
+is a slight over-estimate for piston aircraft, which is preferable to under.
 """
 
 JET_A_CO2_KG_PER_GALLON = 9.57
 
 # ICAO type code -> gallons per hour at long-range cruise
 FUEL_BURN_GPH: dict[str, int] = {
-    # Gulfstream
+    # ── Gulfstream ─────────────────────────────────────────────────────
     "GLF6": 430,   # G650/G650ER
     "G700": 480,   # G700
     "GLF5": 390,   # G550
     "GVSP": 400,   # GV-SP
     "GLF4": 330,   # G-IV
-    # Bombardier
+    # ── Bombardier business ────────────────────────────────────────────
     "GL7T": 490,   # Global 7500
     "GLEX": 430,   # Global Express/6000/6500
     "GL5T": 420,   # Global 5000/5500
@@ -22,51 +26,208 @@ FUEL_BURN_GPH: dict[str, int] = {
     "CL60": 310,   # Challenger 604/605
     "CL30": 200,   # Challenger 300
     "CL65": 320,   # Challenger 650
-    # Dassault
+    # ── Bombardier regional jets ──────────────────────────────────────
+    "CRJ2": 360,   # CRJ-100/200
+    "CRJ7": 380,   # CRJ-700
+    "CRJ9": 410,   # CRJ-900
+    "CRJX": 440,   # CRJ-1000
+    # ── Dassault ───────────────────────────────────────────────────────
     "F7X":  350,   # Falcon 7X
     "F8X":  370,   # Falcon 8X
     "F900": 285,   # Falcon 900/900EX/900LX
     "F2TH": 230,   # Falcon 2000
     "FA50": 240,   # Falcon 50
-    # Cessna
+    # ── Cessna Citation ────────────────────────────────────────────────
     "CITX": 280,   # Citation X
+    "C750": 280,   # Citation X (alt code)
     "C68A": 195,   # Citation Latitude
     "C700": 230,   # Citation Longitude
     "C680": 220,   # Citation Sovereign
-    "C560": 190,   # Citation Excel/XLS
+    "C56X": 195,   # Citation Excel/XLS/XLS+
+    "C560": 190,   # Citation Excel/XLS (legacy)
+    "C550": 165,   # Citation II/Bravo/V
+    "C525": 80,    # Citation CJ1
+    "C25A": 100,   # CJ1+ / 525A
+    "C25B": 110,   # CJ2+ / 525B
+    "C25C": 130,   # CJ4 (some operators)
     "C510": 75,    # Citation Mustang
+    "C650": 240,   # Citation III/VI/VII
     "CJ3":  120,   # CJ3
     "CJ4":  135,   # CJ4
-    # Boeing
-    "B737": 850,   # BBJ (737)
-    "B738": 920,   # BBJ2 (737-800)
+    # ── Cessna piston / turboprop singles & twins ─────────────────────
+    "C172": 9,     # Skyhawk
+    "C152": 6,
+    "C150": 6,
+    "C170": 8,
+    "C177": 11,
+    "C180": 12,
+    "C182": 13,    # Skylane
+    "C185": 14,
+    "C206": 15,
+    "C208": 50,    # Caravan (turboprop)
+    "C210": 18,
+    "C310": 32,
+    "C340": 38,
+    "C414": 36,
+    "C421": 40,
+    # ── Boeing mainline ────────────────────────────────────────────────
+    "B737": 850,   # 737-700 / BBJ
+    "B738": 920,   # 737-800
+    "B739": 880,   # 737-900/900ER
+    "B38M": 700,   # 737-8 MAX
+    "B39M": 740,   # 737-9 MAX
     "B752": 1100,  # 757-200
+    "B753": 1200,  # 757-300
     "B762": 1400,  # 767-200
+    "B763": 1450,  # 767-300/300ER
+    "B764": 1500,  # 767-400ER
+    "B772": 1850,  # 777-200
+    "B77L": 1900,  # 777-200LR / 777F
+    "B77W": 2050,  # 777-300ER
     "B788": 1200,  # 787-8
-    # Airbus
-    "A318": 780,   # ACJ318
-    "A319": 850,   # ACJ319
-    "A320": 900,   # ACJ320
+    "B789": 1300,  # 787-9
+    "B78X": 1350,  # 787-10
+    "B744": 3050,  # 747-400
+    "B748": 2900,  # 747-8
+    # ── Airbus mainline ────────────────────────────────────────────────
+    "A318": 780,   # A318
+    "A319": 850,   # A319
+    "A320": 900,   # A320
+    "A321": 990,   # A321
+    "A19N": 580,   # A319neo
+    "A20N": 580,   # A320neo
+    "A21N": 700,   # A321neo
+    "A332": 1500,  # A330-200
+    "A333": 1550,  # A330-300
+    "A338": 1300,  # A330-800neo
+    "A339": 1350,  # A330-900neo
     "A343": 1800,  # A340-300
     "A346": 2100,  # A340-600
-    # Pilatus
+    "A359": 1450,  # A350-900
+    "A35K": 1600,  # A350-1000
+    "A388": 3200,  # A380-800
+    # ── Embraer regional / business ───────────────────────────────────
+    "E135": 300,   # Legacy 600/650 (regional ERJ-135 base)
+    "E145": 320,   # ERJ-145
+    "E170": 460,   # E170
+    "E75L": 490,   # E175-LR
+    "E75S": 490,   # E175 standard
+    "E175": 490,   # E175 (some)
+    "E190": 580,   # E190
+    "E195": 600,   # E195
+    "E290": 510,   # E190-E2
+    "E295": 540,   # E195-E2
+    "E50P": 135,   # Phenom 300 (also Phenom 100 var)
+    "E55P": 185,   # Praetor 500 / Legacy 500
+    "E545": 170,   # Praetor 500 (alt)
+    "E500": 80,    # Phenom 100
+    # ── ATR / Bombardier / Saab turboprops ────────────────────────────
+    "AT43": 230,   # ATR 42-300/-320
+    "AT45": 230,   # ATR 42-500
+    "AT46": 250,   # ATR 42-600
+    "AT72": 300,   # ATR 72-200/-210
+    "AT75": 280,   # ATR 72-500
+    "AT76": 280,   # ATR 72-600
+    "DH8A": 220,   # Dash 8 -100
+    "DH8B": 240,   # Dash 8 -200
+    "DH8C": 280,   # Dash 8 -300
+    "DH8D": 300,   # Dash 8 Q400
+    "SF34": 200,   # Saab 340
+    "SB20": 220,   # Saab 2000
+    # ── Pilatus / Daher single-engine turboprops ──────────────────────
     "PC24": 115,   # PC-24
     "PC12": 60,    # PC-12
-    # Embraer
-    "E55P": 185,   # Legacy 500
-    "E135": 300,   # Legacy 600/650
-    "E50P": 135,   # Phenom 300
-    "E500": 80,    # Phenom 100
-    # Learjet
+    "TBM7": 60,    # TBM 700/850
+    "TBM8": 65,    # TBM 850 alt
+    "TBM9": 70,    # TBM 900/930/940/960
+    "M600": 60,    # Piper M600
+    "P46T": 22,    # PA-46 Meridian (turboprop variant)
+    # ── Learjet ────────────────────────────────────────────────────────
     "LJ60": 195,   # Learjet 60
     "LJ75": 185,   # Learjet 75
     "LJ45": 175,   # Learjet 45
-    # Hawker
+    "LJ31": 165,   # Learjet 31
+    "LJ40": 175,   # Learjet 40
+    "LJ55": 195,   # Learjet 55
+    # ── Hawker / Beechjet ─────────────────────────────────────────────
     "H25B": 210,   # Hawker 800/800XP
     "H25C": 215,   # Hawker 900XP
-    # Beechcraft
+    "BE40": 150,   # Beechjet 400 / Hawker 400XP
+    "PRM1": 130,   # Premier I
+    # ── Beechcraft King Air ───────────────────────────────────────────
     "B350": 100,   # King Air 350
     "B200": 80,    # King Air 200/250
+    "BE20": 80,    # K-Air 200 (alt)
+    "BE9L": 60,    # K-Air 90
+    "BE9T": 70,    # K-Air F90
+    "BE10": 100,   # K-Air 100
+    "BE30": 90,    # K-Air 300
+    # ── Beechcraft / Cirrus / Piper / Mooney pistons ──────────────────
+    "BE23": 9,     # Sundowner
+    "BE33": 13,    # Bonanza 33
+    "BE35": 14,    # Bonanza V-tail
+    "BE36": 16,    # A36 Bonanza
+    "BE55": 24,    # Baron 55
+    "BE58": 28,    # Baron 58
+    "BE76": 17,    # Duchess
+    "BE95": 20,    # Travel Air
+    "P28A": 10,    # PA-28 Warrior/Archer
+    "P28B": 11,    # PA-28 Cherokee
+    "P28R": 12,    # PA-28R Arrow
+    "P32R": 14,    # PA-32R Lance/Saratoga
+    "PA11": 5,     # Cub Special
+    "PA12": 6,     # Super Cruiser
+    "PA18": 6,     # Super Cub
+    "PA22": 8,     # Tri-Pacer
+    "PA23": 18,    # Apache / Aztec
+    "PA24": 12,    # Comanche
+    "PA25": 12,    # Pawnee
+    "PA28": 10,    # PA-28 generic
+    "PA30": 16,    # Twin Comanche
+    "PA31": 30,    # Navajo
+    "PA32": 14,    # Cherokee Six / Saratoga
+    "PA34": 18,    # Seneca
+    "PA38": 5,     # Tomahawk
+    "PA44": 17,    # Seminole
+    "PA46": 18,    # Malibu / Mirage / Matrix
+    "M20P": 12,    # Mooney M20 (generic)
+    "SR20": 11,    # Cirrus SR20
+    "SR22": 16,    # Cirrus SR22
+    "S22T": 19,    # SR22T (turbo)
+    "DA40": 9,     # Diamond DA40
+    "DA42": 14,    # Diamond DA42 TwinStar
+    "DA62": 17,    # Diamond DA62
+    "DV20": 6,     # Diamond Katana
+    # ── Helicopters (civilian) ────────────────────────────────────────
+    "A109": 60,    # AW109
+    "A119": 50,    # AW119
+    "A139": 130,   # AW139
+    "A169": 90,    # AW169
+    "A189": 145,   # AW189
+    "AS35": 55,    # AS350 AStar
+    "AS50": 55,    # AStar (alt)
+    "AS65": 110,   # Dauphin
+    "B06":  35,    # Bell 206 JetRanger
+    "B407": 50,    # Bell 407
+    "B412": 145,   # Bell 412
+    "B429": 80,    # Bell 429
+    "B505": 35,    # Bell 505
+    "EC30": 50,    # H125 / EC130
+    "EC35": 70,    # EC135
+    "EC45": 85,    # EC145
+    "EC75": 130,   # EC175
+    "H125": 55,
+    "H130": 50,
+    "H135": 70,
+    "H145": 85,
+    "H155": 110,
+    "H160": 95,
+    "H175": 130,
+    "R22":  9,     # Robinson R22 (piston)
+    "R44":  16,    # Robinson R44 (piston)
+    "R66":  30,    # Robinson R66 (turbine)
+    "S76":  140,   # Sikorsky S-76
+    "S92":  220,   # Sikorsky S-92
 }
 
 # Common string names -> ICAO type code
@@ -108,13 +269,23 @@ def get_emissions_info(model: str) -> dict | None:
     if not model:
         return None
     model_clean = model.strip()
+    model_upper = model_clean.upper()
     # Try direct ICAO code match first
-    gph = FUEL_BURN_GPH.get(model_clean.upper())
+    gph = FUEL_BURN_GPH.get(model_upper)
     if gph is None:
         # Try alias lookup
         code = _ALIASES.get(model_clean)
         if code:
             gph = FUEL_BURN_GPH.get(code)
+    if gph is None:
+        # Friendly names from the Plane-Alert DB often lead with the ICAO type
+        # code as the first token (e.g. "B200 Super King Air"). Probe each
+        # token against FUEL_BURN_GPH directly.
+        for token in model_upper.replace("-", " ").replace(",", " ").split():
+            candidate = FUEL_BURN_GPH.get(token)
+            if candidate is not None:
+                gph = candidate
+                break
     if gph is None:
         # Fuzzy: check if any alias is a substring
         model_lower = model_clean.lower()

backend/services/fetchers/fimi.py

@@ -5,6 +5,7 @@ debunked claims, threat actor mentions, and target country references.
 Refreshes every 12 hours (FIMI data updates weekly).
 """
 
+import os
 import re
 import logging
 from datetime import datetime, timezone
@@ -18,6 +19,16 @@ logger = logging.getLogger("services.data_fetcher")
 
 _FIMI_FEED_URL = "https://euvsdisinfo.eu/feed/"
 
+
+def fimi_fetch_enabled() -> bool:
+    """Return True only when the operator explicitly opts into FIMI pulls."""
+    return str(os.environ.get("FIMI_ENABLED", "")).strip().lower() in {
+        "1",
+        "true",
+        "yes",
+        "on",
+    }
+
 # ── Threat actor keywords ──────────────────────────────────────────────────
 # Map of keyword → canonical actor name.  Checked case-insensitively.
 _THREAT_ACTORS: dict[str, str] = {
@@ -173,6 +184,12 @@ def _is_major_wave(narratives: list[dict], targets: dict[str, int]) -> bool:
 @with_retry(max_retries=1, base_delay=5)
 def fetch_fimi():
     """Fetch and parse the EUvsDisinfo RSS feed."""
+    if not fimi_fetch_enabled():
+        logger.debug("FIMI fetch skipped; set FIMI_ENABLED=true to opt in")
+        with _data_lock:
+            latest_data["fimi"] = []
+        _mark_fresh("fimi")
+        return
     try:
         resp = fetch_with_curl(_FIMI_FEED_URL, timeout=15)
         feed = feedparser.parse(resp.text)