chore(release): update v0.9.81 SHA256 digests after rebuild

Re-cut v0.9.81 binaries from current main (which now includes the
private gate + DM hashchain spool from #326 and the gate-directory
test from #327). All three artifacts were signed with the same
minisign updater key as the original v0.9.81 release, so existing
v0.9.81 installs on Tauri auto-update accept the new bundles.

Updated hashes (verified against released assets):
- ShadowBroker_v0.9.81.zip      f81f454bdc88e9a32c351df38212b8cfa624704d65764b971bb091eef62259c6
- ShadowBroker_0.9.81_x64-setup.exe   25e9a95d0d8ce959a7d08fe8e7406772ae24b596652793e81d1de5d02510a5a6
- ShadowBroker_0.9.81_x64_en-US.msi   34e655fc0c0f195ee4ac978f228a4b2b9d5565253b8771aca9ef4693409e9e70

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

.dockerignore

@@ -0,0 +1,56 @@
+# Exclude build artifacts, caches, and large directories from Docker context
+.git/
+.git_backup/
+node_modules/
+.next/
+__pycache__/
+*.pyc
+venv/
+.venv/
+.ruff_cache/
+local-artifacts/
+release-secrets/
+
+# Never send local configuration or credentials into Docker builds
+.env
+.env.*
+**/.env
+**/.env.*
+*.pem
+*.key
+*.p12
+*.pfx
+
+# privacy-core build caches (source is needed, artifacts are not)
+privacy-core/target/
+privacy-core/target-test/
+privacy-core/.codex-tmp/
+
+# Large data/cache files
+*.db
+*.sqlite
+*.xlsx
+*.log
+extra/
+prototype/
+
+# Runtime state generated by local backend runs
+backend/.pytest_cache/
+backend/.ruff_cache/
+backend/backend.egg-info/
+backend/build/
+backend/node_modules/
+backend/timemachine/
+backend/venv/
+backend/data/*cache*.json
+backend/data/**/*cache*.json
+backend/data/wormhole*.json
+backend/data/**/wormhole*.json
+backend/data/dm_*.json
+backend/data/**/dm_*.json
+backend/data/**/peer_store.json
+backend/data/**/node.json
+backend/data/*.log
+backend/data/**/*.log
+backend/data/*.key
+backend/data/**/*.key

.env.example

@@ -3,15 +3,20 @@
 #   cp .env.example .env
 
 # ── Required for backend container ─────────────────────────────
+# OpenSky Network OAuth2 — REQUIRED for airplane telemetry.
+# Free registration at https://opensky-network.org/index.php?option=com_users&view=registration
+# Without these the flights layer falls back to ADS-B-only with major gaps in Africa, Asia, and LatAm.
 OPENSKY_CLIENT_ID=
 OPENSKY_CLIENT_SECRET=
 AIS_API_KEY=
 
 # Admin key to protect sensitive endpoints (settings, updates).
-# If blank, admin endpoints are only accessible from localhost unless ALLOW_INSECURE_ADMIN=true.
+# If blank, loopback/localhost requests still work for local single-host dev.
+# Remote/non-loopback admin access requires ADMIN_KEY, or ALLOW_INSECURE_ADMIN=true in debug-only setups.
 ADMIN_KEY=
 
-# Allow insecure admin access without ADMIN_KEY (local dev only).
+# Allow insecure admin access without ADMIN_KEY (local dev only, beyond loopback).
+# Requires MESH_DEBUG_MODE=true on the backend; do not enable this for normal use.
 # ALLOW_INSECURE_ADMIN=false
 
 # User-Agent for Nominatim geocoding requests (per OSM usage policy).
@@ -29,6 +34,45 @@ ADMIN_KEY=
 # Ukraine air raid alerts — free token from https://alerts.in.ua/
 # ALERTS_IN_UA_TOKEN=
 
+# Optional NUFORC UAP sighting map enrichment via Mapbox Tilequery.
+# Leave blank to skip this optional enrichment.
+# NUFORC_MAPBOX_TOKEN=
+
+# Optional startup-risk controls.
+# On Windows, external curl fallback and the Playwright LiveUAMap scraper are
+# disabled by default so blocked upstream feeds cannot interrupt start.bat.
+# SHADOWBROKER_ENABLE_WINDOWS_CURL_FALLBACK=false
+# SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=false
+# AIS starts by default when AIS_API_KEY is set. Set to 0/false to force-disable.
+# SHADOWBROKER_ENABLE_AIS_STREAM_PROXY=true
+# Minimum visible satellite catalog before forcing a CelesTrak refresh.
+# SHADOWBROKER_MIN_VISIBLE_SATELLITES=350
+# Upper bound for TLE fallback satellite search when CelesTrak is unreachable.
+# SHADOWBROKER_MAX_VISIBLE_SATELLITES=450
+# NUFORC fallback uses the Hugging Face mirror when live NUFORC is unavailable.
+# NUFORC_HF_FALLBACK_LIMIT=250
+# NUFORC_HF_GEOCODE_LIMIT=150
+
+# First-paint cache age budgets. These let the map and Global Threat Intercept
+# paint from the last local snapshot while live feeds refresh in the background.
+# FAST_STARTUP_CACHE_MAX_AGE_S=21600
+# INTEL_STARTUP_CACHE_MAX_AGE_S=21600
+
+# Docker resource tuning. The backend synthesizes large geospatial feeds; keep
+# this at 4G or higher on hosts that run AIS, OpenSky, CCTV, satellites, and
+# threat feeds together. Lower caps can cause Docker OOM restarts and empty
+# slow layers such as news, UAP sightings, military bases, and wastewater.
+# BACKEND_MEMORY_LIMIT=4G
+# SHADOWBROKER_FETCH_WORKERS=8
+# SHADOWBROKER_SLOW_FETCH_CONCURRENCY=4
+# SHADOWBROKER_STARTUP_HEAVY_CONCURRENCY=2
+
+# Infonet bootstrap/sync responsiveness. Defaults favor fast seed failure
+# detection so stale onion peers do not make the terminal look hung.
+# MESH_SYNC_TIMEOUT_S=5
+# MESH_SYNC_MAX_PEERS_PER_CYCLE=3
+# MESH_BOOTSTRAP_SEED_FAILURE_COOLDOWN_S=15
+
 # Google Earth Engine for VIIRS night lights change detection (optional).
 # pip install earthengine-api
 # GEE_SERVICE_ACCOUNT_KEY=
@@ -77,6 +121,11 @@ ADMIN_KEY=
 
 # ── Mesh DM Relay ──────────────────────────────────────────────
 # MESH_DM_TOKEN_PEPPER=change-me
+# Optional local-dev DM root external assurance bridge.
+# These stay commented because they are machine-local file paths, not safe global defaults.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH=backend/../ops/root_witness_receipt_import.json
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH=backend/../ops/root_transparency_ledger.json
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI=backend/../ops/root_transparency_ledger.json
 
 # ── Self Update ────────────────────────────────────────────────
 # MESH_UPDATE_SHA256=

.gitattributes

@@ -0,0 +1,2 @@
+# Force LF line endings for shell scripts
+*.sh text eol=lf

.github/CODEOWNERS

@@ -0,0 +1,32 @@
+# CODEOWNERS — assigns required reviewers for sensitive paths.
+# Format: <path glob>  <user-or-team> [<user-or-team> ...]
+# See https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
+#
+# Owners listed here are auto-requested for review when matching files
+# change in a PR. If branch protection requires CODEOWNERS approval, the
+# PR cannot be merged until an owner approves.
+
+# ── Internationalization / translations ──
+# Translation contributions are held to a stricter neutrality standard
+# than most code changes — see CONTRIBUTING.md "Translation contributions".
+# The i18n layer itself (no network calls, no telemetry, static JSON
+# bundled at build) is the structural guarantee that makes this safe;
+# changes to it need owner review.
+/frontend/src/i18n/                @BigBodyCobain
+
+# ── Security-sensitive code paths ──
+/backend/auth.py                   @BigBodyCobain
+/backend/routers/wormhole.py       @BigBodyCobain
+/backend/services/mesh/            @BigBodyCobain
+/backend/services/fetchers/        @BigBodyCobain
+
+# ── CI / build / deploy infra ──
+/.github/workflows/                @BigBodyCobain
+/.gitlab-ci.yml                    @BigBodyCobain
+/docker-compose.yml                @BigBodyCobain
+/docker-compose.gitlab.yml         @BigBodyCobain
+/helm/                             @BigBodyCobain
+
+# ── This file and policy docs ──
+/.github/CODEOWNERS                @BigBodyCobain
+/CONTRIBUTING.md                   @BigBodyCobain

.github/workflows/ci.yml

@@ -1,11 +1,33 @@
-name: CI — Lint & Test
+name: CI - Lint & Test
 
 on:
   push:
     branches: [main]
   pull_request:
     branches: [main]
-  workflow_call:  # Allow docker-publish to call this workflow as a gate
+  workflow_call:
+
+# CI flake mitigation:
+# ci.yml is triggered TWICE per PR on the same commit — once directly via
+# the `pull_request` trigger above ("Frontend Tests & Build" check) and once
+# via `workflow_call` from docker-publish.yml ("CI Gate / Frontend Tests &
+# Build" check). Both jobs land on the same Actions runner pool at the same
+# time and fight for CPU/RAM. Under contention, React's reconciliation in
+# `messagesViewFirstContact.test.tsx > removes an approved contact …`
+# overruns its 5s waitFor timeout — that's the single failure mode we've
+# seen flake on PRs #226, #237, #261, #262, #265, #294, #303, and the
+# fd7d6fa push. Backend tests and every other frontend test pass under
+# the same conditions, which is what made this look random.
+#
+# Pinning a concurrency group on the SHA (PR head, or the pushed commit
+# for main) serializes the two invocations so neither starves the other.
+# We use cancel-in-progress: false so the second one queues instead of
+# cancelling — cancelling could leave the PR check stuck "Expected" if
+# only one of the two ever finishes. Total CI time grows by ~2 min in
+# exchange for deterministic outcomes.
+concurrency:
+  group: ci-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: false
 
 jobs:
   frontend:
@@ -33,6 +55,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: Run secret scan
+        run: bash backend/scripts/scan-secrets.sh --all
       - name: Install uv
         uses: astral-sh/setup-uv@v5
         with:
@@ -46,5 +70,13 @@ jobs:
       - run: cd backend && uv run ruff check .
       - run: cd backend && uv run black --check .
       - run: cd backend && uv run python -c "from services.fetchers.retry import with_retry; from services.env_check import validate_env; print('Module imports OK')"
-      - name: Run tests
-        run: cd backend && uv run pytest tests/ -v --tb=short || echo "No pytest tests found (OK)"
+      - name: Run release smoke tests
+        run: |
+          cd backend
+          uv run pytest \
+            tests/mesh/test_mesh_node_bootstrap_runtime.py \
+            tests/mesh/test_mesh_infonet_sync_support.py \
+            tests/mesh/test_mesh_canonical.py \
+            tests/mesh/test_mesh_merkle.py \
+            tests/test_release_helper.py \
+            -v --tb=short

.github/workflows/docker-publish.yml

@@ -6,10 +6,9 @@ on:
     tags: ["v*.*.*"]
   pull_request:
     branches: ["main"]
-  
+
 env:
   REGISTRY: ghcr.io
-  # github.repository as <account>/<repo>
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
@@ -24,7 +23,6 @@ jobs:
       contents: read
       packages: write
       id-token: write
-
     strategy:
       fail-fast: false
       matrix:
@@ -33,33 +31,23 @@ jobs:
             runner: ubuntu-latest
           - platform: linux/arm64
             runner: ubuntu-24.04-arm
-
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
+      - uses: actions/checkout@v4
       - name: Lowercase image name
         run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.0.0
-
-      - name: Log into registry ${{ env.REGISTRY }}
+      - uses: docker/setup-buildx-action@v3.0.0
+      - name: Log into registry
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v3.0.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract Docker metadata
-        id: meta
+      - id: meta
         uses: docker/metadata-action@v5.0.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend
-
-      - name: Build and push Docker image by digest
-        id: build
+      - id: build
         uses: docker/build-push-action@v5.0.0
         with:
           context: ./frontend
@@ -69,17 +57,14 @@ jobs:
           cache-from: type=gha,scope=frontend-${{ matrix.platform }}
           cache-to: type=gha,mode=max,scope=frontend-${{ matrix.platform }}
           outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend,push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }}
-
       - name: Export digest
         if: github.event_name != 'pull_request'
         run: |
           mkdir -p /tmp/digests/frontend
           digest="${{ steps.build.outputs.digest }}"
           touch "/tmp/digests/frontend/${digest#sha256:}"
-
-      - name: Upload digest
+      - uses: actions/upload-artifact@v4
         if: github.event_name != 'pull_request'
-        uses: actions/upload-artifact@v4
         with:
           name: digests-frontend-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
           path: /tmp/digests/frontend/*
@@ -87,36 +72,27 @@ jobs:
           retention-days: 1
 
   merge-frontend:
-    runs-on: ubuntu-latest
     if: github.event_name != 'pull_request'
     needs: build-frontend
+    runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
-
     steps:
       - name: Lowercase image name
         run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
-
-      - name: Download digests
-        uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v4
         with:
           path: /tmp/digests/frontend
           pattern: digests-frontend-*
           merge-multiple: true
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.0.0
-
-      - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v3.0.0
+      - uses: docker/setup-buildx-action@v3.0.0
+      - uses: docker/login-action@v3.0.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract Docker metadata
-        id: meta
+      - id: meta
         uses: docker/metadata-action@v5.0.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend
@@ -124,7 +100,6 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=raw,value=latest,enable={{is_default_branch}}
-
       - name: Create and push manifest
         working-directory: /tmp/digests/frontend
         run: |
@@ -139,7 +114,6 @@ jobs:
       contents: read
       packages: write
       id-token: write
-
     strategy:
       fail-fast: false
       matrix:
@@ -148,33 +122,23 @@ jobs:
             runner: ubuntu-latest
           - platform: linux/arm64
             runner: ubuntu-24.04-arm
-
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
+      - uses: actions/checkout@v4
       - name: Lowercase image name
         run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.0.0
-
-      - name: Log into registry ${{ env.REGISTRY }}
+      - uses: docker/setup-buildx-action@v3.0.0
+      - name: Log into registry
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v3.0.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract Docker metadata
-        id: meta
+      - id: meta
         uses: docker/metadata-action@v5.0.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend
-
-      - name: Build and push Docker image by digest
-        id: build
+      - id: build
         uses: docker/build-push-action@v5.0.0
         with:
           context: .
@@ -185,17 +149,14 @@ jobs:
           cache-from: type=gha,scope=backend-${{ matrix.platform }}
           cache-to: type=gha,mode=max,scope=backend-${{ matrix.platform }}
           outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend,push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }}
-
       - name: Export digest
         if: github.event_name != 'pull_request'
         run: |
           mkdir -p /tmp/digests/backend
           digest="${{ steps.build.outputs.digest }}"
           touch "/tmp/digests/backend/${digest#sha256:}"
-
-      - name: Upload digest
+      - uses: actions/upload-artifact@v4
         if: github.event_name != 'pull_request'
-        uses: actions/upload-artifact@v4
         with:
           name: digests-backend-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
           path: /tmp/digests/backend/*
@@ -203,36 +164,27 @@ jobs:
           retention-days: 1
 
   merge-backend:
-    runs-on: ubuntu-latest
     if: github.event_name != 'pull_request'
     needs: build-backend
+    runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
-
     steps:
       - name: Lowercase image name
         run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
-
-      - name: Download digests
-        uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v4
         with:
           path: /tmp/digests/backend
           pattern: digests-backend-*
           merge-multiple: true
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.0.0
-
-      - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v3.0.0
+      - uses: docker/setup-buildx-action@v3.0.0
+      - uses: docker/login-action@v3.0.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract Docker metadata
-        id: meta
+      - id: meta
         uses: docker/metadata-action@v5.0.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend
@@ -240,7 +192,6 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=raw,value=latest,enable={{is_default_branch}}
-
       - name: Create and push manifest
         working-directory: /tmp/digests/backend
         run: |

.gitignore

@@ -6,13 +6,32 @@ node_modules/
 venv/
 env/
 .venv/
+backend/.venv-dir
+backend/venv-repair*/
+backend/.venv-repair*/
 
 # Environment Variables & Secrets
 .env
+.envrc
 .env.local
 .env.development.local
 .env.test.local
 .env.production.local
+.npmrc
+.pypirc
+.netrc
+*.pem
+*.key
+*.crt
+*.csr
+*.p12
+*.pfx
+id_rsa
+id_rsa.*
+id_ed25519
+id_ed25519.*
+known_hosts
+authorized_keys
 
 # Python caches & compiled files
 __pycache__/
@@ -22,11 +41,15 @@ __pycache__/
 .Python
 .ruff_cache/
 .pytest_cache/
+.mypy_cache/
+.hypothesis/
+.tox/
 
 # Next.js build output
 .next/
 out/
 build/
+*.tsbuildinfo
 
 # Deprecated standalone Infonet Terminal skeleton (migrated into frontend/src/components/InfonetTerminal/)
 frontend/infonet-terminal/
@@ -49,6 +72,8 @@ backend/ais_cache.json
 backend/carrier_cache.json
 backend/cctv.db
 cctv.db
+*.db
+*.sqlite
 *.sqlite3
 
 # ========================
@@ -63,8 +88,27 @@ backend/data/*
 !backend/data/military_bases.json
 !backend/data/plan_ccg_vessels.json
 !backend/data/plane_alert_db.json
+!backend/data/power_plants.json
 !backend/data/tracked_names.json
 !backend/data/yacht_alert_db.json
+# Issue #206: bundled KiwiSDR receiver directory used as last-resort
+# fallback when rx.linkfanel.net (HTTP-only upstream) is unreachable
+# or returns content that fails our integrity validation.
+!backend/data/kiwisdr_directory.json
+# Issue #201: pinned SHA-256 digests for known Tor Expert Bundle URLs.
+# Used as a second verification source when upstream .sha256sum fails.
+!backend/data/tor_bundle_digests.json
+# Issue #258: SPKI pins for stream.aisstream.io so we can survive upstream
+# Let's Encrypt renewal failures without disabling TLS validation entirely.
+!backend/data/aisstream_spki_pins.json
+# Issue #231: pinned SHA-256 digests for known release archives. Used by
+# the self-updater as a second-line integrity check when the release's
+# SHA256SUMS.txt asset can't be fetched.
+!backend/data/release_digests.json
+# Issue #244/#245/#246: one-shot carrier-position seed shipped with each
+# release. Used ONLY on first-ever startup to bootstrap carrier_cache.json;
+# after that the cache reflects this install's own GDELT observations.
+!backend/data/carrier_seed.json
 
 # OS generated files
 .DS_Store
@@ -129,6 +173,7 @@ frontend/eslint-report.json
 # Old backups & repo clones
 .git_backup/
 local-artifacts/
+release-secrets/
 shadowbroker_repo/
 frontend/src/components.bak/
 frontend/src/components/map/icons/backups/
@@ -136,6 +181,7 @@ frontend/src/components/map/icons/backups/
 # Coverage
 coverage/
 .coverage
+.coverage.*
 dist/
 
 # Test scratch files (not in tests/ folder)
@@ -145,6 +191,8 @@ backend/services/test_*.py
 # Local analysis & dev tools
 backend/analyze_xlsx.py
 backend/services/ais_cache.json
+graphify/
+graphify-out/
 
 # ========================
 # Internal docs & brainstorming (never commit)
@@ -152,8 +200,11 @@ backend/services/ais_cache.json
 docs/*
 !docs/mesh/
 docs/mesh/*
+!docs/mesh/threat-model.md
+!docs/mesh/claims-reconciliation.md
 !docs/mesh/mesh-canonical-fixtures.json
 !docs/mesh/mesh-merkle-fixtures.json
+!docs/mesh/wormhole-dm-root-operations-runbook.md
 .local-docs/
 infonet-economy/
 updatestuff.md
@@ -173,6 +224,69 @@ jobs.json
 .mise.local.toml
 .codex-tmp/
 prototype/
+.runtime/
 
-# Python UV lock file (regenerated from pyproject.toml)
-uv.lock
+# ========================
+# Runtime state & operator-local data (never commit)
+# ========================
+# TimeMachine snapshot cache — regenerated at runtime, can be 100 MB+
+backend/timemachine/
+# Operator witness keys, identity material, transparency ledgers (machine-local)
+ops/
+# Runtime DM relay state
+dm_relay.json
+# Dev scratch notes
+improvements.txt
+
+# ========================
+# Custody verification temp dirs (runtime test artifacts with private keys!)
+# ========================
+backend/sb-custody-verify-*/
+
+# Python egg-info (build artifact, regenerated by pip install -e)
+*.egg-info/
+
+# Privacy-core debug build (Windows DLL, 3.6 MB, not shipped)
+privacy-core/debug/
+
+# Desktop-shell export stash dirs (empty temp dirs from Tauri build)
+frontend/.desktop-export-stash-*/
+
+# Wormhole logs (can be 30 MB+ each, runtime-generated)
+backend/data/wormhole_stderr.log
+backend/data/wormhole_stdout.log
+
+# Runtime caches that already slip through the backend/data/* blanket
+# (these are caught by the wildcard but listing for clarity)
+
+# Compressed snapshot archives (can be 100 MB+)
+*.json.gz
+
+# ──────────────────────────────────────────────────────────────────────
+# AI assistant / coding-agent scratch
+# ──────────────────────────────────────────────────────────────────────
+# Per-tool config + scratch directories. These are private to whichever
+# coding agent the operator happens to be using and have no business in
+# the repo. If a tool's instructions need to be canonical for the project,
+# we'll put them in docs/ explicitly — not let the agent dump them at the
+# repo root.
+
+# OpenAI Codex CLI
+.codex/
+.codex-app-schema/
+.codex-app-ts/
+
+# Per-agent instruction files dropped at repo root by various tools.
+# These are operator-side preferences, not part of the project contract.
+AGENTS.md
+GEMINI.md
+CLAUDE.md
+.github/copilot-instructions.md
+
+# Stale AI-generated test file that referenced fields that don't exist in
+# the current `_parse_carrier_positions_from_news` implementation. Kept
+# ignored so it doesn't accidentally get committed if it shows up again
+# from a tool that's working off an out-of-date understanding of the
+# module. If a real test for that function is needed, write it under a
+# meaningful name in tests/test_carrier_tracker_quality.py.
+backend/tests/test_carrier_tracker_region_centers.py

.gitlab-ci.yml

@@ -0,0 +1,121 @@
+# GitLab CI/CD for Shadowbroker
+#
+# Mirror of .github/workflows/docker-publish.yml — keeps the GitLab install
+# path (image registry + source) at parity with GitHub so users who prefer
+# GitLab get the same experience.
+#
+# What this does on every push to main:
+#   1. Builds multi-arch (amd64 + arm64) Docker images for the backend and
+#      frontend, pushes them to the project's GitLab Container Registry:
+#        registry.gitlab.com/bigbodycobain/shadowbroker/backend:latest
+#        registry.gitlab.com/bigbodycobain/shadowbroker/frontend:latest
+#      Both also get a :$CI_COMMIT_SHORT_SHA tag for traceability.
+#   2. Reverse-mirrors main back to GitHub (only if commits land directly
+#      on GitLab) so the two sources stay in sync.
+#
+# Auth notes:
+#   - The image build/push uses $CI_JOB_TOKEN, which GitLab provides
+#     automatically. No credentials need to be configured.
+#   - The reverse mirror requires a GitHub personal access token stored
+#     as the GitLab CI/CD variable GITHUB_MIRROR_TOKEN (Protected + Masked).
+#     Scope: public_repo (or repo for private). If the variable isn't
+#     set the mirror job is skipped — image builds still run.
+
+stages:
+  - build
+  - mirror
+
+variables:
+  # Use the dind service for buildx multi-arch builds.
+  DOCKER_HOST: tcp://docker:2376
+  DOCKER_TLS_CERTDIR: "/certs"
+  DOCKER_DRIVER: overlay2
+  # QEMU is what lets a single x86 runner build arm64 images. dind doesn't
+  # install it by default; we install via tonistiigi/binfmt below.
+  BUILDX_VERSION: "v0.14.1"
+  # Repository-relative paths.
+  BACKEND_IMAGE: $CI_REGISTRY_IMAGE/backend
+  FRONTEND_IMAGE: $CI_REGISTRY_IMAGE/frontend
+
+# Shared template: bootstraps buildx + QEMU on the dind service so a single
+# runner can produce both amd64 and arm64 manifests in one push.
+.buildx-setup: &buildx-setup
+  image: docker:24
+  services:
+    - name: docker:24-dind
+      command: ["--tls=true"]
+  before_script:
+    - docker info
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
+    - docker run --privileged --rm tonistiigi/binfmt --install all
+    - docker buildx create --use --name multiarch --driver docker-container
+
+# ── Backend image ────────────────────────────────────────────────────────
+build-backend:
+  <<: *buildx-setup
+  stage: build
+  script:
+    - >
+      docker buildx build
+      --platform linux/amd64,linux/arm64
+      --file backend/Dockerfile
+      --tag $BACKEND_IMAGE:latest
+      --tag $BACKEND_IMAGE:$CI_COMMIT_SHORT_SHA
+      --push
+      .
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+        - backend/**/*
+        - .gitlab-ci.yml
+
+# ── Frontend image ───────────────────────────────────────────────────────
+build-frontend:
+  <<: *buildx-setup
+  stage: build
+  script:
+    - cd frontend
+    - >
+      docker buildx build
+      --platform linux/amd64,linux/arm64
+      --tag $FRONTEND_IMAGE:latest
+      --tag $FRONTEND_IMAGE:$CI_COMMIT_SHORT_SHA
+      --push
+      .
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+        - frontend/**/*
+        - .gitlab-ci.yml
+
+# ── Reverse mirror to GitHub ─────────────────────────────────────────────
+# Pushes refs/heads/main to github.com/BigBodyCobain/Shadowbroker.
+# Fast-forward-only — if GitLab main and GitHub main have diverged, this
+# fails loudly rather than silently overwriting either side.
+#
+# Only runs if GITHUB_MIRROR_TOKEN is set as a CI/CD variable. See the
+# header comment of this file for setup instructions.
+mirror-to-github:
+  stage: mirror
+  image: alpine:3.20
+  needs: []
+  before_script:
+    - apk add --no-cache git openssh-client ca-certificates
+  script:
+    - git config --global user.email "ci-mirror@gitlab.com"
+    - git config --global user.name "GitLab CI Mirror"
+    - >
+      git clone --depth=50 --branch main
+      "https://oauth2:${CI_JOB_TOKEN}@gitlab.com/${CI_PROJECT_PATH}.git"
+      repo
+    - cd repo
+    - >
+      git push
+      "https://x-access-token:${GITHUB_MIRROR_TOKEN}@github.com/BigBodyCobain/Shadowbroker.git"
+      "${CI_COMMIT_SHA}:refs/heads/main"
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $GITHUB_MIRROR_TOKEN

.pre-commit-config.yaml

@@ -1,4 +1,12 @@
 repos:
+  - repo: local
+    hooks:
+      - id: shadowbroker-secret-scan
+        name: ShadowBroker secret scan
+        entry: bash backend/scripts/scan-secrets.sh --staged
+        language: system
+        pass_filenames: false
+
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.6.0
     hooks:

CONTRIBUTING.md

@@ -0,0 +1,75 @@
+# Contributing to Shadowbroker
+
+Thank you for taking the time to contribute. This document covers things specific to this project — for general open-source contribution etiquette, see the GitHub docs.
+
+---
+
+## Code contributions
+
+1. Fork the repo on GitHub (`bigbodycobain/Shadowbroker`) or GitLab (`bigbodycobain/Shadowbroker` mirror).
+2. Make your changes on a feature branch.
+3. Run the local test suite:
+   - Backend: `pytest backend/tests/`
+   - Frontend: `cd frontend && npx vitest run`
+4. Open a Pull Request against `main`.
+
+CI runs on every PR. If CI fails, that's blocking — please push fixes rather than asking for it to be merged anyway.
+
+---
+
+## Reporting security issues
+
+Do **not** file security issues as public GitHub issues. Email the maintainer or use a private security advisory on GitHub. Public disclosure of an exploitable vulnerability without prior coordination will be rejected from the project.
+
+---
+
+## Translation contributions
+
+Shadowbroker supports UI localization (`frontend/src/i18n/`). Translation contributions are welcome but held to a stricter standard than most code changes, because translations can subtly reshape user perception in ways that are hard to spot during review. Read this section before submitting one.
+
+### The neutrality requirement
+
+**Translations must be technically faithful to the English source.** That means:
+
+- Each `t('key')` entry should mean approximately the same thing in the target language as in English, modulo idiom.
+- Technical terms with established meanings (e.g. "GPS jamming," "military flight," "Tor," "onion routing," "encryption") should be translated using the corresponding established technical term in the target language — **not** softened, rebranded, or politically reframed.
+- The set of UI strings should be **the same** between languages. Don't omit features from one locale that are visible in another.
+
+### What will get a translation PR rejected
+
+Translation choices that align the project with the framing or terminology of state propaganda — from **any** country — will be rejected. This applies symmetrically:
+
+| Country / source | Examples of substitutions we will reject |
+|---|---|
+| **PRC / CCP** | Calling Taiwan a "province" or "renegade province"; reframing protest layers as "riots"; using softened or euphemistic terms for surveillance, internment, or jamming when the source text is direct |
+| **Russia** | Calling the Ukraine war a "special military operation"; relabeling occupied territories as Russian; softening sanctions/jamming/disinfo terminology |
+| **United States / EU** | Reframing adversaries with editorial labels not in the source (e.g. inserting "regime" where the English says "government"); applying labels like "terrorist" or "rogue state" to entities the English source describes neutrally |
+| **Israel / Palestine / any active conflict** | Substituting one side's preferred terminology when the source uses the other side's or a neutral term |
+| **Any government** | Adding political slogans, omitting features that government finds inconvenient, or inserting terminology associated with a specific political faction |
+
+The test is **"would a translator working strictly from the English source produce this rendering?"** If the answer requires assuming a political stance the source does not take, the substitution does not belong in the translation.
+
+### How translation PRs are reviewed
+
+Changes to `frontend/src/i18n/**` are owned by the maintainer (see `CODEOWNERS`) and require explicit approval. We will:
+
+1. Diff the translation against the English source key-by-key.
+2. Spot-check a sample of entries with a native speaker of the target language when possible.
+3. Look for the patterns above.
+4. Look for suspicious additions to the i18n infrastructure itself (e.g. a remote translation fetcher, telemetry on language choice) — the i18n layer is supposed to be 100% client-side static JSON.
+
+A PR that adds a new language is harder to review than one that fixes typos in an existing language. For new languages, please be patient and expect a real review window. For typo fixes, please describe each change in the PR body so the reviewer can verify intent.
+
+### What about adding a new language?
+
+We welcome new languages. The mechanical setup is documented in the header comment of `frontend/src/i18n/index.ts`. Beyond that:
+
+- We are more likely to merge a new language quickly if at least one reviewer in the maintainer's network speaks it.
+- If you are the *only* speaker of the target language reading this repo, your translation is welcome but the merge timeline will be longer while a reviewer is found.
+- Partial translations are fine — the system falls back to English for any missing key.
+
+---
+
+## Anything else
+
+If you have a question that isn't a security report, opening a GitHub Discussion or a draft PR with a question in the body is the fastest way to get a response. Direct emails are read but not always replied to promptly.

DATA-ATTRIBUTION.md

@@ -0,0 +1,71 @@
+# Data Attribution & Licensing
+
+ShadowBroker aggregates publicly available data from many third-party sources.
+This file documents each source and its license so operators and users can
+comply with the terms under which we access that data.
+
+ShadowBroker itself is licensed under AGPL-3.0 (see `LICENSE`). **This file
+concerns the *data* rendered by the dashboard, not the source code.**
+
+---
+
+## ODbL-licensed sources (Open Database License v1.0)
+
+Data from these sources is licensed under the
+[Open Database License v1.0](https://opendatacommons.org/licenses/odbl/1-0/).
+If you redistribute a derivative database built from these sources, the
+derivative must also be offered under ODbL and must preserve attribution.
+
+| Source | URL | What we use it for |
+|---|---|---|
+| adsb.lol | https://adsb.lol | Military aircraft positions, regional commercial gap-fill, route enrichment |
+| OpenStreetMap contributors | https://www.openstreetmap.org/copyright | Nominatim geocoding (LOCATE bar), CARTO basemap tiles (OSM-derived) |
+
+**Attribution requirement:** the ShadowBroker map UI displays
+"© OpenStreetMap contributors" and "adsb.lol (ODbL)" in the map attribution
+control. Do not remove this attribution if you fork or redistribute the app.
+
+---
+
+## Other third-party data sources
+
+These sources have their own terms; consult each link before redistributing.
+
+| Source | URL | License / Terms | Notes |
+|---|---|---|---|
+| OpenSky Network | https://opensky-network.org | OpenSky API terms | Commercial and private aircraft tracking |
+| CelesTrak | https://celestrak.org | Public domain / no restrictions | Satellite TLE data |
+| USGS Earthquake Hazards | https://earthquake.usgs.gov | Public domain (US Federal) | Seismic events |
+| NASA FIRMS | https://firms.modaps.eosdis.nasa.gov | NASA Open Data | Fire/thermal anomalies (VIIRS) |
+| NASA GIBS | https://gibs.earthdata.nasa.gov | NASA Open Data | MODIS imagery tiles |
+| NOAA SWPC | https://services.swpc.noaa.gov | Public domain (US Federal) | Space weather, Kp index |
+| GDELT Project | https://www.gdeltproject.org | CC BY (non-commercial friendly) | Global conflict events |
+| DeepState Map | https://deepstatemap.live | Per-site terms | Ukraine frontline GeoJSON |
+| aisstream.io | https://aisstream.io | Free-tier API terms (attribution required) | AIS vessel positions |
+| Global Fishing Watch | https://globalfishingwatch.org | CC BY 4.0 (for public data) | Fishing activity events |
+| Microsoft Planetary Computer | https://planetarycomputer.microsoft.com | Sentinel-2 / ESA Copernicus terms | Sentinel-2 imagery |
+| Copernicus CDSE (Sentinel Hub) | https://dataspace.copernicus.eu | ESA Copernicus open data terms | SAR + optical imagery |
+| Shodan | https://www.shodan.io | Operator-supplied API key, Shodan ToS | Internet device search |
+| Smithsonian GVP | https://volcano.si.edu | Attribution required | Volcanoes |
+| OpenAQ | https://openaq.org | CC BY 4.0 | Air quality stations |
+| NOAA NWS | https://www.weather.gov | Public domain (US Federal) | Severe weather alerts |
+| WRI Global Power Plant DB | https://datasets.wri.org | CC BY 4.0 | Power plants |
+| Wikidata | https://www.wikidata.org | CC0 | Head-of-state lookup |
+| Wikipedia | https://en.wikipedia.org | CC BY-SA 4.0 | Region summaries |
+| KiwiSDR (via dyatlov mirror) | http://rx.linkfanel.net | Per-site terms (community mirror by Pierre Ynard) | SDR receiver list — pulled from rx.linkfanel.net to keep load off jks-prv's bandwidth at kiwisdr.com |
+| OpenMHZ | https://openmhz.com | Per-site terms | Police/fire scanner feeds |
+| Meshtastic | https://meshtastic.org | Open Source | Mesh radio nodes (protocol) |
+| Meshtastic Map (Liam Cottle) | https://meshtastic.liamcottle.net | Community project (per-site terms) | Global Meshtastic node positions — polled once per day with on-disk cache trust to minimize load on this volunteer-run HTTP API |
+| APRS-IS | https://www.aprs-is.net | Open / attribution-based | Amateur radio positions |
+| CARTO basemaps | https://carto.com | CARTO attribution required | Dark map tiles (OSM-derived) |
+| Esri World Imagery | https://www.arcgis.com | Esri terms | High-res satellite basemap |
+| IODA (Georgia Tech) | https://ioda.inetintel.cc.gatech.edu | Research/academic terms | Internet outage data |
+
+---
+
+## Contact
+
+If you represent a data provider and have concerns about how ShadowBroker
+uses your data, please open an issue or contact the maintainer at
+`bigbodycobain@gmail.com`. We will respond promptly and, if needed, adjust
+usage or remove the source.

Mesh.md

@@ -0,0 +1,89 @@
+# ShadowBroker — Meshtastic MQTT Remediation
+
+**Version:** 0.9.6  
+**Date:** 2026-04-12  
+**Re:** [meshtastic/firmware#6131](https://github.com/meshtastic/firmware/issues/6131) — Excessive MQTT traffic from ShadowBroker clients
+
+---
+
+## What happened
+
+ShadowBroker is an open-source OSINT situational awareness platform that includes a Meshtastic MQTT listener for displaying mesh network activity on a global map. In prior versions, the MQTT bridge:
+
+- Subscribed to **28 wildcard topics** (`msh/{region}/#`) covering every known official and community root on startup
+- Used an aggressive reconnect policy (min 1s / max 30s backoff)
+- Set keepalive to 30 seconds
+- Had no client-side rate limiting on inbound messages
+- Auto-started on every launch with no opt-out
+
+This produced 1-2 orders of magnitude more traffic than typical Meshtastic clients on the public broker at `mqtt.meshtastic.org`.
+
+---
+
+## What we fixed
+
+### 1. Bridge disabled by default
+
+The MQTT bridge no longer starts automatically. Operators must explicitly opt in:
+
+```env
+MESH_MQTT_ENABLED=true
+```
+
+### 2. US-only default subscription
+
+When enabled, the bridge subscribes to **1 topic** (`msh/US/#`) instead of 28. Additional regions are opt-in:
+
+```env
+MESH_MQTT_EXTRA_ROOTS=EU_868,ANZ
+```
+
+The UI still displays all regions in its dropdown — only the MQTT subscription scope changed.
+
+### 3. Client-side rate limiter
+
+Inbound messages are capped at **100 messages per minute** using a sliding window. Excess messages are silently dropped. A warning is logged periodically when the limiter activates so operators are aware.
+
+### 4. Conservative connection parameters
+
+| Parameter | Before | After |
+|-----------|--------|-------|
+| Keepalive | 30s | 120s |
+| Reconnect min delay | 1s | 15s |
+| Reconnect max delay | 30s | 300s |
+| QoS | 0 | 0 (unchanged) |
+
+### 5. Versioned client ID
+
+Client IDs changed from `sbmesh-{uuid}` to `sb096-{uuid}` so the Meshtastic team can identify ShadowBroker clients and track adoption of the fix by version.
+
+---
+
+## Configuration reference
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `MESH_MQTT_ENABLED` | `false` | Master switch for the MQTT bridge |
+| `MESH_MQTT_EXTRA_ROOTS` | _(empty)_ | Comma-separated additional region roots (e.g. `EU_868,ANZ,JP`) |
+| `MESH_MQTT_INCLUDE_DEFAULT_ROOTS` | `true` | Include US in subscriptions |
+| `MESH_MQTT_BROKER` | `mqtt.meshtastic.org` | Broker hostname |
+| `MESH_MQTT_PORT` | `1883` | Broker port |
+| `MESH_MQTT_USER` | `meshdev` | Broker username |
+| `MESH_MQTT_PASS` | `large4cats` | Broker password |
+| `MESH_MQTT_PSK` | _(empty)_ | Hex-encoded PSK (empty = default LongFast key) |
+
+---
+
+## Files changed
+
+- `backend/services/config.py` — Added `MESH_MQTT_ENABLED` flag
+- `backend/services/mesh/meshtastic_topics.py` — Reduced default roots to US-only
+- `backend/services/sigint_bridge.py` — Rate limiter, keepalive/backoff tuning, versioned client ID, opt-in gate
+- `backend/.env.example` — Documented all MQTT options
+
+---
+
+## Contact
+
+Repository: [github.com/BigBodyCobain/Shadowbroker](https://github.com/BigBodyCobain/Shadowbroker)  
+Maintainer: BigBodyCobain

README.md

@@ -11,33 +11,18 @@
 
 
 
-https://github.com/user-attachments/assets/248208ec-62f7-49d1-831d-4bd0a1fa6852
+[![ShadowBroker](/uploads/46f99d19fa141a2efba37feee9de8aab/Title.jpg)](https://github.com/user-attachments/assets/248208ec-62f7-49d1-831d-4bd0a1fa6852)
 
 
 
 
 
-**ShadowBroker** is a real-time, multi-domain OSINT dashboard that fuses 60+ live intelligence feeds into a single dark-ops map interface. Aircraft, ships, satellites, conflict zones, CCTV networks, GPS jamming, internet-connected devices, police scanners, mesh radio nodes, and breaking geopolitical events — all updating in real time on one screen.
+**ShadowBroker** is a decentralized intelligence platform that aggregates real-time, multi-domain OSINT telemetry from 60+ live intelligence feeds into a single dark-ops map interface. Aircraft, ships, satellites, conflict zones, CCTV networks, GPS jamming, internet-connected devices, police scanners, mesh radio nodes, and breaking geopolitical events — all updating in real time on one screen as well as an obfuscated communications protocol and information exchange infrastructure.
 
-Built with **Next.js**, **MapLibre GL**, **FastAPI**, and **Python**. 35+ toggleable data layers. Five visual modes (DEFAULT / SATELLITE / FLIR / NVG / CRT). Right-click any point on Earth for a country dossier, head-of-state lookup, and the latest Sentinel-2 satellite photo. No user data is collected or transmitted — the dashboard runs entirely in your browser against a self-hosted backend.
+Built with **Next.js**, **MapLibre GL**, **FastAPI**, and **Python**. 35+ toggleable data layers, including SAR ground-change detection. Multiple visual modes (DEFAULT / SATELLITE / FLIR / NVG / CRT). Right-click any point on Earth for a country dossier, head-of-state lookup, and the latest Sentinel-2 satellite photo. No user data is collected or transmitted — the dashboard runs entirely in your browser against a self-hosted backend.
 
 Designed for analysts, researchers, radio operators, and anyone who wants to see what the world looks like when every public signal is on the same map.
 
----
-
-## Experimental Testnet — No Privacy Guarantee
-
-ShadowBroker v0.9.6 introduces **InfoNet**, a decentralized intelligence mesh with obfuscated messaging. This is an **experimental testnet** — not a private messenger.
-
-| Channel | Privacy Status | Details |
-|---|---|---|
-| **Meshtastic / APRS** | **PUBLIC** | RF radio transmissions are public and interceptable by design. |
-| **InfoNet Gate Chat** | **OBFUSCATED** | Messages are obfuscated with gate personas and canonical payload signing, but NOT end-to-end encrypted. Metadata is not hidden. |
-| **Dead Drop DMs** | **STRONGEST CURRENT LANE** | Token-based epoch mailbox with SAS word verification. Strongest lane in this build, but still not Signal-tier. |
-
-**Do not transmit anything sensitive on any channel.** Treat all lanes as open and public for now. E2E encryption and deeper native/Tauri hardening are the next milestones. If you fork this project, keep these labels intact and do not make stronger privacy claims than the implementation supports.
-
----
 
 ## Why This Exists
 
@@ -53,10 +38,9 @@ ShadowBroker includes an optional Shodan connector for operator-supplied API acc
 
 ## Interesting Use Cases
 
-* **Transmit on the InfoNet testnet** — the first decentralized intelligence mesh built into an OSINT tool. Obfuscated messaging with gate personas, Dead Drop peer-to-peer exchange, and a built-in terminal CLI. No accounts, no signup. Privacy is not guaranteed yet — this is an experimental testnet — but the protocol is live and being hardened.
-* **Track Air Force One**, the private jets of billionaires and dictators, and every military tanker, ISR, and fighter broadcasting ADS-B — with automatic holding pattern detection when aircraft start circling
-* **Estimate where US aircraft carriers are** using automated GDELT news scraping — no other open tool does this
-* **Search internet-connected devices worldwide** via Shodan — cameras, SCADA systems, databases — plotted as a live overlay on the map
+* **Track Air Force One**, the private jets of billionaires and dictators, and every military tanker, ISR, and fighter broadcasting ADS-B. Air Force One and all of the accompanying Presidential/Vice Presidential planes are highlighted and monitored from the moment they leave the ground. 
+* **Connect an AI agent as a co-analyst** through ShadowBroker's HMAC-signed agentic command channel — supports OpenClaw and any other agent that speaks the protocol (Claude, GPT, LangChain, custom). The agent gets full read/write access to all 35+ data layers, pin placement, map control, SAR ground-change, mesh networking, and alert delivery. It sees everything the operator sees and can take actions on the map in real time.
+* **Communicate on the InfoNet testnet** — The first decentralized intelligence mesh built into an OSINT tool. Obfuscated messaging with gate personas, Dead Drop peer-to-peer exchange, and a built-in terminal CLI. No accounts, no signup. Privacy is not guaranteed yet — this is an experimental testnet — but the protocol is live and being hardened.
 * **Right-click anywhere on Earth** for a country dossier (head of state, population, languages), Wikipedia summary, and the latest Sentinel-2 satellite photo at 10m resolution
 * **Click a KiwiSDR node** and tune into live shortwave radio directly in the dashboard. Click a police scanner feed and eavesdrop in one click.
 * **Watch 11,000+ CCTV cameras** across 6 countries — London, NYC, California, Spain, Singapore, and more — streaming live on the map
@@ -66,96 +50,90 @@ ShadowBroker includes an optional Shodan connector for operator-supplied API acc
 * **Follow earthquakes, volcanic eruptions, active wildfires** (NASA FIRMS), severe weather alerts, and air quality readings worldwide
 * **Map military bases, 35,000+ power plants**, 2,000+ data centers, and internet outage regions — cross-referenced automatically
 * **Connect to Meshtastic mesh radio nodes** and APRS amateur radio networks — visible on the map and integrated into Mesh Chat
+* **Detect ground changes through cloud cover** with SAR (Synthetic Aperture Radar) — mm-scale ground deformation, flood extent, vegetation disturbance, and damage assessments from NASA OPERA and Copernicus EGMS. Define your own watch areas and get anomaly alerts. Free with a NASA Earthdata account.
 * **Switch visual modes** — DEFAULT, SATELLITE, FLIR (thermal), NVG (night vision), CRT (retro terminal) — via the STYLE button
 * **Track trains** across the US (Amtrak) and Europe (DigiTraffic) in real time
+* **Estimate where US aircraft carriers are** using automated GDELT news scraping — no other open tool does this
+* **Search internet-connected devices worldwide** via Shodan — cameras, SCADA systems, databases — plotted as a live overlay on the map
+
 
 ---
 
-## ⚡ Quick Start (Docker or Podman)
+## ⚡ Quick Start (Docker)
 
-Linux/Mac
+### From GitHub (default — uses GHCR images)
 
 ```bash
-git clone https://github.com/BigBodyCobain/Shadowbroker.git
+git clone https://github.com/bigbodycobain/Shadowbroker.git
 cd Shadowbroker
-./compose.sh up -d
+docker compose pull
+docker compose up -d
 ```
 
-Windows
+### From GitLab (uses GitLab Container Registry)
 
 ```bash
-git clone https://github.com/BigBodyCobain/Shadowbroker.git
+git clone https://gitlab.com/bigbodycobain/Shadowbroker.git
 cd Shadowbroker
-docker-compose up -d
+docker compose -f docker-compose.yml -f docker-compose.gitlab.yml pull
+docker compose -f docker-compose.yml -f docker-compose.gitlab.yml up -d
 ```
 
-Open `http://localhost:3000` to view the dashboard! *(Requires Docker or Podman)*
+Both paths produce identical containers — same source, same CI, same images byte-for-byte. Pick whichever ecosystem you already use.
 
-`compose.sh` auto-detects `docker compose`, `docker-compose`, `podman compose`, and `podman-compose`.
-If both runtimes are installed, you can force Podman with `./compose.sh --engine podman up -d`.
-Do not append a trailing `.` to that command; Compose treats it as a service name.
+Open `http://localhost:3000` to view the dashboard! *(Requires [Docker Desktop](https://www.docker.com/products/docker-desktop/) or Docker Engine)*
+
+> **Backend port already in use?** The browser only needs port `3000`, but the backend API is also published on host port `8000` for local diagnostics. If another app already uses `8000`, create or edit `.env` next to `docker-compose.yml` and set `BACKEND_PORT=8001`, then run `docker compose up -d`.
+
+> **Blank news/UAP/bases/wastewater after several minutes?** Check for backend OOM restarts with `docker events --since 30m --filter container=shadowbroker-backend --filter event=oom`. The default compose file gives the backend 4GB; if your host has less memory, reduce enabled feeds or set `BACKEND_MEMORY_LIMIT=3G` and expect slower/heavier layers to warm more gradually.
+
+> **Podman users:** Podman works, but `podman compose` is a wrapper and still needs a Compose provider installed. On Windows/WSL, if you see `looking up compose provider failed`, install `podman-compose` and run `podman-compose pull` followed by `podman-compose up -d` from inside the cloned `Shadowbroker` folder. On Linux/macOS/WSL shells you can also use `./compose.sh --engine podman pull` and `./compose.sh --engine podman up -d`.
 
 ---
 
 ##  🔄 **How to Update**
 
-If you are coming from v0.9.5 or older, you must pull the new code and rebuild your containers to get the InfoNet testnet, Shodan integration, train tracking, 8 new intelligence layers, and all performance fixes in v0.9.6.
+ShadowBroker uses pre-built Docker images — no local building required. Updating takes seconds:
 
-### 🐧 **Linux & 🍎 macOS** (Terminal / Zsh / Bash)
-
-Since these systems are Unix-based, you can use the helper script directly.
-
-**Pull the latest code:**
 ```bash
-git pull origin main
-```
-**Run the update script:**
-```bash
-./compose.sh down
-./compose.sh up --build -d
+docker compose pull
+docker compose up -d
 ```
 
-### 🪟 **Windows** (Command Prompt or PowerShell)
+That's it. `pull` grabs the latest images, `up -d` restarts the containers.
 
-Windows handles scripts differently. You have two ways to update:
-
-**Method A: The Direct Way (Recommended)**
-Use the docker compose commands directly. This works in any Windows terminal (CMD, PowerShell, or Windows Terminal).
-
-**Pull the latest code:**
-```DOS
-git pull origin main
-```
-
-**Rebuild the containers:**
-```DOS
-docker compose down
-docker compose up --build -d
-```
-
-**Method B: Using the Script (Git Bash)**
-
-If you prefer using the ./compose.sh script on Windows, you must use Git Bash (installed with Git for Windows).
-
-Open your project folder, Right-Click, and select "Open Git Bash here".
-
-**Run the Linux commands:**
-```bash
-./compose.sh down
-./compose.sh up --build -d
-```
-
----
+> **Coming from an older version?** Pull the latest repo code first, then pull images:
+>
+> ```bash
+> git pull origin main
+> docker compose down
+> docker compose pull
+> docker compose up -d
+> ```
+>
+> Podman users should run the equivalent provider command, for example `podman-compose pull` and `podman-compose up -d`, or use `./compose.sh --engine podman pull` and `./compose.sh --engine podman up -d` from a bash-compatible shell.
 
 ### ⚠️ **Stuck on the old version?**
 
-**If the dashboard still shows old data after updating:**
+**If `git pull` fails or `docker compose up` keeps building from source instead of pulling images**, your clone predates a March 2026 repository migration that rewrote commit history. A normal `git pull` cannot fix this. Run:
 
-**Clear Docker Cache:** docker compose build --no-cache
+```bash
+# Back up any local config you want to keep (.env, etc.)
+cd ..
+rm -rf Shadowbroker
+git clone https://github.com/bigbodycobain/Shadowbroker.git
+cd Shadowbroker
+docker compose pull
+docker compose up -d
+```
 
-**Prune Images:** docker image prune -f
+**How to tell if you're affected:** If `docker compose up` shows `RUN apt-get`, `RUN npm ci`, or `RUN pip install` — it's building from source instead of pulling pre-built images. You need a fresh clone.
 
-**Check Logs:** ./compose.sh logs -f backend (or docker compose logs -f backend)
+**Other troubleshooting:**
+
+* **Force re-pull:** `docker compose pull --no-cache`
+* **Prune old images:** `docker image prune -f`
+* **Check logs:** `docker compose logs -f backend`
 
 ---
 
@@ -171,8 +149,13 @@ helm repo update
 
 **2. Install the Chart:**
 ```bash
-# Install from the local helm/chart directory
+# Default — pulls images from GHCR
 helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbroker
+
+# GitLab registry variant
+helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbroker \
+  -f helm/chart/values.yaml \
+  -f helm/chart/values-gitlab.yaml
 ```
 
 **3. Key Features:**
@@ -184,23 +167,63 @@ helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbrok
 
 ---
 
+## Experimental Testnet — No Privacy Guarantee
+
+ShadowBroker v0.9.7 ships **InfoNet** (decentralized intelligence mesh + Sovereign Shell governance economy), an **agentic AI command channel** (supports OpenClaw and any HMAC-signing agent), **Time Machine snapshot playback**, and **SAR satellite ground-change detection**. This is an **experimental testnet** — not a private messenger and not a production governance system.
+
+| Channel | Privacy Status | Details |
+|---|---|---|
+| **Meshtastic / APRS** | **PUBLIC** | RF radio transmissions are public and interceptable by design. |
+| **InfoNet Gate Chat** | **OBFUSCATED** | Messages are obfuscated with gate personas and canonical payload signing, but NOT end-to-end encrypted. Metadata is not hidden despite being designed through Tor and Reticulum (Work in progress). |
+| **Dead Drop DMs** | **STRONGEST CURRENT LANE** | Token-based epoch mailbox with SAS word verification. Strongest lane in this build, but not yet confidently private. |
+| **Sovereign Shell governance** | **PUBLIC LEDGER** | Petitions, votes, upgrade hashes, and dispute stakes are signed events on a public hashchain. Pseudonymous via gate persona, but governance actions are intentionally observable. |
+| **Privacy primitives (RingCT / stealth / DEX)** | **NOT YET WIRED** | Locked Protocol contracts are in place, but the cryptographic scheme has not been chosen. The privacy-core Rust crate is the integration target for a future sprint. |
+
+**Do not transmit anything sensitive on any channel.** Treat all lanes as open and public for now. E2E encryption and deeper native/Tauri hardening are the next milestones. If you fork this project, keep these labels intact and do not make stronger privacy claims than the implementation supports.
+
+> **For a full picture of what the mesh actually defends against and
+> what it doesn't, read the
+> [threat model](docs/mesh/threat-model.md) and the
+> [claims reconciliation](docs/mesh/claims-reconciliation.md). Every
+> sentence above is mapped there to the code path that enforces it (or
+> doesn't).**
+
+---
+
 
 ## ✨ Features
 
-### 🧅 InfoNet — Decentralized Intelligence Mesh (NEW in v0.9.6)
+### 🧅 InfoNet — Decentralized Intelligence Mesh + Sovereign Shell (expanded in v0.9.7)
 
-The first decentralized intelligence communication layer built directly into an OSINT platform. No accounts, no signup, no identity required. Nothing like this has existed in an OSINT tool before.
+The first decentralized intelligence communication and governance layer built directly into an OSINT platform. No accounts, no signup, no identity required. v0.9.7 promotes InfoNet from a chat layer into a full governance economy with a clear path to a privacy-preserving decentralized intelligence platform.
 
-* **InfoNet Experimental Testnet** — A global, obfuscated message relay. Anyone running ShadowBroker can transmit and receive on the InfoNet. Messages pass through a Wormhole relay layer with gate personas, Ed25519 canonical payload signing, and transport obfuscation.
-* **Mesh Chat Panel** — Three-tab interface:
-  * **INFONET** — Gate chat with obfuscated transport (experimental — not yet E2E encrypted)
-  * **MESH** — Meshtastic radio integration (default tab on startup)
-  * **DEAD DROP** — Peer-to-peer message exchange with token-based epoch mailboxes (strongest current lane)
-* **Gate Persona System** — Pseudonymous identities with Ed25519 signing keys, prekey bundles, SAS word contact verification, and abuse reporting
+**Communication layer (since v0.9.6):**
+
+* **InfoNet Experimental Testnet** — A global, obfuscated message relay using Tor and Reticulum. Anyone running ShadowBroker can transmit and receive on the InfoNet. Messages pass through a Wormhole relay layer with gate personas, Ed25519 canonical payload signing, and transport obfuscation.
+* **Mesh Chat Panel** — Three-tab interface: **INFONET** (gate chat with obfuscated transport), **MESH** (Meshtastic radio integration), **DEAD DROP** (peer-to-peer message exchange with token-based epoch mailboxes — strongest current lane).
+* **Gate Persona System** — Pseudonymous identities with Ed25519 signing keys, prekey bundles, SAS word contact verification, and abuse reporting.
 * **Mesh Terminal** — Built-in CLI: `send`, `dm`, market commands, gate state inspection. Draggable panel, minimizes to the top bar. Type `help` to see all commands.
 * **Crypto Stack** — Ed25519 signing, X25519 Diffie-Hellman, AESGCM encryption with HKDF key derivation, hash chain commitment system. Double-ratchet DM scaffolding in progress.
 
-> **Experimental Testnet — No Privacy Guarantee:** InfoNet messages are obfuscated but NOT end-to-end encrypted. The Mesh network (Meshtastic/APRS) is NOT private — radio transmissions are inherently public. Do not send anything sensitive on any channel. E2E encryption is being developed but is not yet implemented. Treat all channels as open and public for now.
+**Sovereign Shell — governance economy (NEW in v0.9.7):**
+
+* **Petitions + Governance DSL** — On-chain parameter changes via signed petitions. Type-safe payload executor for `UPDATE_PARAM`, `BATCH_UPDATE_PARAMS`, `ENABLE_FEATURE`, and `DISABLE_FEATURE`. Tunable knobs change by vote — no code deploys required.
+* **Upgrade-Hash Governance** — Protocol upgrades that need new logic (not just parameter changes) vote on a SHA-256 hash of the verified release. 80% supermajority, 40% quorum, 67% Heavy-Node activation. Lifecycle: signatures → voting → challenge window → awaiting readiness → activated.
+* **Resolution & Dispute Markets** — Stake on market resolution outcomes (yes / no / data_unavailable), open disputes with bonded evidence, and stake on dispute confirm-or-reverse. Per-row submission state stays isolated so concurrent actions don't share an in-flight slot.
+* **Evidence Submission** — Bonded evidence bundles with client-side SHA-256 canonicalization that matches Python `repr()` exactly, so hashes round-trip cleanly through the chain.
+* **Gate Suspension / Shutdown / Appeals** — Filing forms for suspending or shutting down a gate, with a reusable appeal flow auto-targeting the pending petition.
+* **Bootstrap Eligible-Node-One-Vote** — The first 100 markets resolve via one-vote-per-eligible-node instead of stake-weighted resolution. Eligibility: identity age ≥ 3 days, not in predictor exclusion set, valid Argon2id PoW (Heavy-Node-only). Transitions to staked resolution at 1000 nodes.
+* **Two-Tier State + Epoch Finality** — Tier 1 events propagate CRDT-style for low latency; Tier 2 events require epoch finality before they can be acted on. Identity rotation, progressive penalties, ramp milestones, and constitutional invariants enforced via `MappingProxyType`.
+* **Adaptive Polling** — Sovereign Shell views poll every 8 seconds during active voting / challenge / activation phases, every 30–60 seconds when idle. Voting feels live without a websocket layer.
+* **Verbatim Diagnostics** — Every write button surfaces the backend's verbatim rejection reason. No opaque "denied" toasts.
+
+**Privacy primitive runway (NEW in v0.9.7):**
+
+* **Function Keys — Anonymous Citizenship Proof** — A citizen proves "I am an Infonet citizen" without revealing their Infonet identity. 5 of 6 pieces shipped: nullifiers, challenge-response, two-phase commit receipts, enumerated denial codes, batched settlement. Issuance via blind signatures waits on a primitive decision (RSA blind sigs vs BBS+ vs U-Prove vs Idemix).
+* **Locked Protocol Contracts** — Stable interfaces in `services/infonet/privacy/contracts.py` for ring signatures, stealth addresses, Pedersen commitments, range proofs, and DEX matching. The `privacy-core` Rust crate is the integration target — no caller of the privacy module needs to know which scheme is active.
+* **Sprint 11+ Path** — When the cryptographic scheme is chosen, primitives wire into the locked Protocols without API churn.
+
+> **Experimental Testnet — No Privacy Guarantee:** InfoNet messages are obfuscated but NOT end-to-end encrypted. The Mesh network (Meshtastic/APRS) is NOT private — radio transmissions are inherently public. The privacy primitive contracts are scaffolded but not yet wired. Do not send anything sensitive on any channel. Treat all channels as open and public for now.
 
 ### 🔍 Shodan Device Search (NEW in v0.9.6)
 
@@ -269,6 +292,17 @@ The first decentralized intelligence communication layer built directly into an
   * **NVG** — Night vision green phosphor
   * **CRT** — Retro terminal scanline overlay
 
+### 🛰️ SAR Ground-Change Detection (NEW)
+
+* **Synthetic Aperture Radar Layer** — Detects ground changes through cloud cover, at night, anywhere on Earth. Two modes, both free:
+  * **Mode A (Catalog)** — Free Sentinel-1 scene metadata from Alaska Satellite Facility. No account required. Shows when radar passes happened over your AOIs and when the next pass is coming.
+  * **Mode B (Full Anomalies)** — Real-time ground-change alerts from NASA OPERA (DISP, DSWx, DIST-ALERT) and Copernicus EGMS. Requires a free NASA Earthdata account — the in-app wizard walks you through setup in under a minute.
+* **Anomaly Types** — Ground deformation (mm-scale subsidence, landslides), surface water change (flood extent), vegetation disturbance (deforestation, burn scars, blast craters), damage assessments (UNOSAT/Copernicus EMS verified), and coherence change detection
+* **Map Visualization** — Color-coded anomaly pins by kind (orange for deformation, cyan for water, green for vegetation, red for damage, purple for coherence). AOI boundaries drawn as dashed polygons with category-based coloring. Click any pin for a detail popup with magnitude, confidence, solver, scene count, and provenance link.
+* **AOI Editor** — Define areas of interest directly from the map. Click the "EDIT AOIs" button when the SAR layer is active, then use the crosshair tool to click-to-drop an AOI center on the map. Set name, radius (1–500 km), and category. AOIs appear on the map immediately.
+* **OpenClaw Integration** — The AI agent can inspect SAR anomaly details (`sar_pin_click`) and fly the operator's map to any AOI center (`sar_focus_aoi`) — enabling collaborative analyst workflows.
+* **Settings Panel** — Dedicated SAR tab in Settings shows Mode A/B status, OpenClaw integration state, and lets you revoke Earthdata credentials with one click.
+
 ### 📻 Software-Defined Radio & SIGINT
 
 * **KiwiSDR Receivers** — 500+ public SDR receivers plotted worldwide with clustered amber markers
@@ -316,55 +350,169 @@ The first decentralized intelligence communication layer built directly into an
 * **Measurement Tool** — Point-to-point distance & bearing measurement on the map
 * **LOCATE Bar** — Search by coordinates (31.8, 34.8) or place name (Tehran, Strait of Hormuz) to fly directly to any location — geocoded via OpenStreetMap Nominatim
 
-![Gaza](https://github.com/user-attachments/assets/f2c953b2-3528-4360-af5a-7ea34ff28489)
+![Gaza](https://gitlab.com/bigbodycobain/Shadowbroker/uploads/c55a0c8d49e5e05c6cd094279e6e089b/gaza-screenshot.jpg)
+
+### 🤖 Agentic AI Command Channel — OpenClaw + Compatible Agents (expanded in v0.9.7)
+
+ShadowBroker exposes a **bidirectional agentic AI command channel** — a signed, tier-gated bridge that gives any compatible AI agent full read/write access to the intelligence platform. **OpenClaw is the reference agent**, but the channel is an open protocol: any LLM-driven agent that signs requests with HMAC-SHA256 (Claude Code, GPT, LangChain, custom Python/TypeScript clients, or your own integration) can connect as an analyst that sees the same data as the operator and can take actions on the map. ShadowBroker does *not* bundle an LLM, an agent runtime, or model weights — it provides the surface; you bring the agent.
+
+v0.9.7 turns ShadowBroker from a dashboard a human watches into an intelligence surface any agent can act on.
+
+**Channel transport (NEW in v0.9.7):**
+
+* **Single Command Channel** — `POST /api/ai/channel/command` accepts `{cmd, args}` and dispatches to any registered tool.
+* **Batched Concurrent Execution** — `POST /api/ai/channel/batch` accepts up to 20 commands in one request. The backend runs them concurrently and returns a fan-out result map. Cuts agent latency by an order of magnitude over sequential calls.
+* **Tier-Gated Access** — `OPENCLAW_ACCESS_TIER` controls which commands the agent can call: `restricted` exposes the read-only set, `full` adds writes and injection. Discovery endpoint returns `available_commands` so the agent can introspect its own capabilities.
+* **HMAC-SHA256 Signing** — Every command is signed `HMAC-SHA256(secret, METHOD|path|timestamp|nonce|sha256(body))` with timestamp + nonce replay protection and request integrity. Supports local mode (no config) and remote mode (agent on a different machine / VPS).
+
+**Capabilities:**
+
+* **Full Telemetry Access** — The agent queries all 35+ data layers: flights, ships, satellites, SIGINT, conflict events, earthquakes, fires, wastewater, prediction markets, and more. Fast and slow tier endpoints return enriched data with geographic coordinates, timestamps, and source attribution.
+* **AI Intel Pins** — Place color-coded investigation markers directly on the operator's map. 14 pin categories (threat, anomaly, military, maritime, aviation, SIGINT, infrastructure, etc.) with confidence scores, TTL expiry, source URLs, and batch placement up to 100 pins at once.
+* **Map Control** — Fly the operator's map view to any coordinate, trigger satellite imagery lookups, and open region dossiers. The agent can direct the operator's attention to specific locations in real time.
+* **SAR Ground-Change** — Query SAR anomaly feeds, inspect pin details, manage AOIs, and fly the map to watch areas. The agent can monitor for ground deformation, flood extent, or damage and promote anomalies to pins.
+* **Native Layer Injection** — Push custom data directly into ShadowBroker's native layers (CCTV cameras, ships, SIGINT nodes, military bases, etc.) so agent-discovered sources render alongside real feeds.
+* **Wormhole Mesh Participation** — The agent can join the decentralized InfoNet, post signed messages, join encrypted gate channels, send/receive encrypted DMs, and interact with Meshtastic radio and Dead Drops — operating as a full mesh peer.
+* **Sovereign Shell Participation (v0.9.7)** — File petitions, sign and vote on governance changes, stake on resolutions and disputes, signal Heavy-Node readiness for upgrades — all programmatically, all gated by tier and HMAC. Agents become first-class participants in the decentralized intelligence economy.
+* **Geocoding & Proximity Scans** — Resolve place names to coordinates, then scan all layers within a radius for a complete proximity digest.
+* **News & GDELT Near Location** — Pull GDELT conflict events and aggregated news articles near any coordinate for regional situational awareness.
+* **Alert Delivery** — Send branded intelligence briefs, warnings, and threat notifications to Discord webhooks and Telegram channels.
+* **Intelligence Reports** — Generate structured reports with summary stats, top military flights, correlations, earthquake activity, SIGINT counts, and pin inventories.
+* **Auditable** — Every channel call is logged; the operator can introspect what the agent has done.
+
+**Connect an agent:** Open the AI Intel panel in the left sidebar, click **Connect Agent**, and copy the HMAC secret. From there, point any compatible agent at the channel — for OpenClaw, import `ShadowBrokerClient` from the OpenClaw skill package; for any other agent, use the same HMAC contract documented above (timestamp + nonce + body digest, tier-gated). The channel is the protocol, not the agent.
+
+### ⏱️ Time Machine — Snapshot Playback (NEW in v0.9.7)
+
+A media-style transport for the entire telemetry feed. Treat the live map as a recording that can be scrubbed, paused, and replayed.
+
+* **Live ↔ Snapshot Toggle** — Switching to snapshot mode pauses the global polling loop instantly; switching back to Live invalidates ETags and force-refreshes both fast and slow tiers so the dashboard catches up without a stale-frame flicker.
+* **Hourly Index** — Every captured snapshot is indexed by its hour bucket with `count`, `latest_id`, `latest_ts`, and the full `snapshot_ids` list. Jump to any captured timestamp directly from the timeline scrubber.
+* **Frame Interpolation** — Moving entities (aircraft, ships, satellites, military flights) interpolate smoothly between recorded frames during playback so motion stays continuous even when snapshots are sparse.
+* **Variable Playback Speed** — Step, play, fast-forward, and rewind through saved telemetry at adjustable speed.
+* **Profile-Aware** — Each snapshot records the privacy profile that was active when it was captured, so playback is faithful to what an operator on that profile would have seen.
+* **Operator-Side, Not Server-Side** — Snapshots are stored locally in the backend; no third party ever sees the playback timeline.
+
+### 📦 API Keys Panel — Path-First, Read-Only (NEW in v0.9.7)
+
+Settings → API Keys is now a read-only registry. Key values never reach the browser process — not even an obfuscated prefix. The panel surfaces:
+
+* The absolute path to the backend `.env` file as resolved by `Path(__file__).resolve()` — works on every OS, every drive, every install location (Linux `/home/...`, macOS `/Users/...`, Windows on any drive, Docker containers, cloud VMs).
+* `[exists]` / `[will be created on first save]` / `[NOT WRITABLE — edit by hand]` indicators on the path itself.
+* The path to the `.env.example` template so users can copy it and fill in their keys.
+* A binary `CONFIGURED` / `NOT CONFIGURED` badge per key, plus a copy-pastable env line (e.g. `OPENSKY_CLIENT_ID=YOUR_VALUE`) the user can drop into the file by hand.
+
+OpenSky API credentials are now a **critical-warn** environment requirement: the startup environment check flags missing OpenSky OAuth2 credentials with a strong warning, and the changelog modal links directly to the free registration page. Without them, the flights layer falls back to ADS-B-only coverage with significant gaps in Africa, Asia, and Latin America.
 
 ---
 
 ## 🏗️ Architecture
 
+ShadowBroker v0.9.7 is composed of three vertically-stacked planes — the **Operator UI**, the **Backend Service Plane**, and the **Decentralized Layer (InfoNet)** — plus two cross-cutting bridges (the **Time Machine** and the **Agentic AI Channel**, which is the protocol that OpenClaw and any other compatible agent connects through) and a **Privacy Core** Rust crate that backstops both the legacy mesh and the future shielded coin / DEX work.
+
 ```
-┌──────────────────────────────────────────────────────────────┐
-│                     FRONTEND (Next.js)                        │
-│                                                              │
-│  ┌─────────────┐  ┌──────────┐  ┌───────────┐  ┌─────────┐  │
-│  │ MapLibre GL │  │ NewsFeed │  │  Control   │  │  Mesh   │  │
-│  │  2D WebGL   │  │  SIGINT  │  │  Panels    │  │  Chat   │  │
-│  │ Map Render  │  │  Intel   │  │Layers/Radio│  │Terminal │  │
-│  └──────┬──────┘  └────┬─────┘  └─────┬─────┘  └────┬────┘  │
-│         └───────────────┼──────────────┼─────────────┘       │
-│                         │ REST + WebSocket                    │
-├─────────────────────────┼────────────────────────────────────┤
-│                   BACKEND (FastAPI)                           │
-│                         │                                    │
-│  ┌──────────────────────┼─────────────────────────────────┐  │
-│  │              Data Fetcher (Scheduler)                  │  │
-│  │                                                        │  │
-│  │  ┌──────────┬──────────┬──────────┬───────────┐        │  │
-│  │  │ OpenSky  │ adsb.lol │CelesTrak │   USGS    │        │  │
-│  │  │ Flights  │ Military │   Sats   │  Quakes   │        │  │
-│  │  ├──────────┼──────────┼──────────┼───────────┤        │  │
-│  │  │  AIS WS  │ Carrier  │  GDELT   │ CCTV (13) │        │  │
-│  │  │  Ships   │ Tracker  │ Conflict │  Cameras  │        │  │
-│  │  ├──────────┼──────────┼──────────┼───────────┤        │  │
-│  │  │ DeepState│   RSS    │  Region  │    GPS    │        │  │
-│  │  │ Frontline│  Intel   │ Dossier  │  Jamming  │        │  │
-│  │  ├──────────┼──────────┼──────────┼───────────┤        │  │
-│  │  │  NASA    │  NOAA    │  IODA    │  KiwiSDR  │        │  │
-│  │  │  FIRMS   │  Space Wx│ Outages  │  Radios   │        │  │
-│  │  ├──────────┼──────────┼──────────┼───────────┤        │  │
-│  │  │  Shodan  │  Amtrak  │ SatNOGS  │ Meshtastic│        │  │
-│  │  │ Devices  │  Trains  │ TinyGS   │   APRS    │        │  │
-│  │  ├──────────┼──────────┼──────────┼───────────┤        │  │
-│  │  │ Volcanoes│ Weather  │ Fishing  │ Mil Bases │        │  │
-│  │  │ Air Qual │ Alerts   │ Activity │Power Plant│        │  │
-│  │  └──────────┴──────────┴──────────┴───────────┘        │  │
-│  └────────────────────────────────────────────────────────┘  │
-│                                                              │
-│  ┌────────────────────────────────────────────────────────┐  │
-│  │              Wormhole / InfoNet Relay                   │  │
-│  │  Gate Personas │ Canonical Signing │ Dead Drop DMs     │  │
-│  └────────────────────────────────────────────────────────┘  │
-└──────────────────────────────────────────────────────────────┘
+╔═════════════════════════════════════════════════════════════════════════════╗
+║                       OPERATOR UI  (Next.js + MapLibre)                     ║
+║                                                                             ║
+║  ┌────────────────┐  ┌──────────┐  ┌────────────────┐  ┌────────────────┐   ║
+║  │ MapLibre GL    │  │ NewsFeed │  │ Sovereign Shell│  │   Mesh Chat    │   ║
+║  │  WebGL render  │  │  SIGINT  │  │  Petitions /   │  │  + Mesh Term.  │   ║
+║  │  + clusters    │  │  GDELT   │  │  Upgrades /    │  │  (Infonet /    │   ║
+║  │                │  │  Threat  │  │  Disputes /    │  │   Mesh /       │   ║
+║  │                │  │          │  │  Gates /       │  │   Dead Drop)   │   ║
+║  │                │  │          │  │  Bootstrap /   │  │                │   ║
+║  │                │  │          │  │  Function Keys │  │                │   ║
+║  └──────┬─────────┘  └────┬─────┘  └────────┬───────┘  └────────┬───────┘   ║
+║         │                 │                 │                   │           ║
+║  ┌──────┴─────────────────┴─────────────────┴───────────────────┴───────┐   ║
+║  │  Time Machine ◀── snapshot playback ── snapshotMode toggle ──▶ Live │   ║
+║  │  hourly index │ frame interpolation │ profile-aware │ per-tier ETag  │   ║
+║  └──────────────────────────────────┬───────────────────────────────────┘   ║
+║                                     │ REST  +  /api/[...path] proxy         ║
+╠═════════════════════════════════════╪═══════════════════════════════════════╣
+║                       BACKEND SERVICE PLANE  (FastAPI)                      ║
+║                                     │                                       ║
+║  ┌──────────────────────────────────┴────────────────────────────────────┐  ║
+║  │              Data Fetcher  (APScheduler — fast / slow tiers)          │  ║
+║  │                                                                       │  ║
+║  │  ┌───────────┬───────────┬───────────┬───────────┬───────────┐        │  ║
+║  │  │  OpenSky* │ adsb.lol  │ CelesTrak │   USGS    │   AIS WS  │        │  ║
+║  │  │  Flights  │ Military  │   Sats    │  Quakes   │   Ships   │        │  ║
+║  │  ├───────────┼───────────┼───────────┼───────────┼───────────┤        │  ║
+║  │  │  Carrier  │   GDELT   │ CCTV (12) │ DeepState │   NASA    │        │  ║
+║  │  │  Tracker  │ Conflict  │  Cameras  │ Frontline │   FIRMS   │        │  ║
+║  │  ├───────────┼───────────┼───────────┼───────────┼───────────┤        │  ║
+║  │  │   GPS     │  KiwiSDR  │  Shodan   │  Amtrak   │  SatNOGS  │        │  ║
+║  │  │  Jamming  │   Radios  │  Devices  │ DigiTraf  │  TinyGS   │        │  ║
+║  │  ├───────────┼───────────┼───────────┼───────────┼───────────┤        │  ║
+║  │  │ Volcanoes │  Weather  │  Fishing  │ Mil Bases │   IODA    │        │  ║
+║  │  │  Air Qual │  Alerts   │  Activity │ PwrPlants │  Outages  │        │  ║
+║  │  ├───────────┼───────────┼───────────┼───────────┼───────────┤        │  ║
+║  │  │ Sentinel  │   MODIS   │   VIIRS   │   Data    │ Meshtastic│        │  ║
+║  │  │  Hub/STAC │   Terra   │ Nightlts  │  Centers  │   APRS    │        │  ║
+║  │  ├───────────┴───────────┴───────────┴───────────┴───────────┤        │  ║
+║  │  │  SAR (NEW v0.9.7)                                         │        │  ║
+║  │  │   Mode A: ASF Search catalog (free, no account)           │        │  ║
+║  │  │   Mode B: NASA OPERA / Copernicus EGMS / GFM / EMS /      │        │  ║
+║  │  │           UNOSAT  ground-change anomalies (opt-in)        │        │  ║
+║  │  └───────────────────────────────────────────────────────────┘        │  ║
+║  │   * OpenSky: REQUIRED for global flight coverage                      │  ║
+║  └───────────────────────────────────────────────────────────────────────┘  ║
+║                                     │                                       ║
+║  ┌──────────────────────────────────┴────────────────────────────────────┐  ║
+║  │                   Snapshot Store  (Time Machine source)               │  ║
+║  │   Hourly index  │  per-snapshot layer manifest  │  profile metadata   │  ║
+║  └───────────────────────────────────────────────────────────────────────┘  ║
+║                                                                             ║
+║  ┌───────────────────────────────────────────────────────────────────────┐  ║
+║  │   Agentic AI Channel  (HMAC-SHA256, tier-gated  —  OpenClaw + others) │  ║
+║  │                                                                       │  ║
+║  │   POST /api/ai/channel/command   →  one tool call                     │  ║
+║  │   POST /api/ai/channel/batch     →  up to 20 concurrent tool calls    │  ║
+║  │                                                                       │  ║
+║  │   Tier:   restricted (read-only)   │   full (read + write + inject)   │  ║
+║  │   Auth:   X-SB-Timestamp + X-SB-Nonce + X-SB-Signature                │  ║
+║  │   Sig  =  HMAC-SHA256(secret, METHOD|path|ts|nonce|sha256(body))      │  ║
+║  └───────────────────────────────────────────────────────────────────────┘  ║
+╠═════════════════════════════════════════════════════════════════════════════╣
+║                  DECENTRALIZED LAYER  (InfoNet Testnet — signed events)     ║
+║                                                                             ║
+║  ┌────────────────────────────┐    ┌──────────────────────────────────┐     ║
+║  │    Mesh Hashchain          │    │   Sovereign Shell Governance     │     ║
+║  │                            │    │                                  │     ║
+║  │  Ed25519 signed events     │    │  Petitions  (DSL: UPDATE_PARAM,  │     ║
+║  │  Public-key binding        │    │              ENABLE_FEATURE …)   │     ║
+║  │  Replay / sequence guard   │    │  Upgrade-Hash voting (80% / 40%  │     ║
+║  │  Two-tier finality         │    │             quorum / 67% Heavy)  │     ║
+║  │   ├ Tier 1 (CRDT, fast)    │    │  Resolution & Dispute markets    │     ║
+║  │   └ Tier 2 (epoch finality)│    │  Gate suspend / shutdown / appeal│     ║
+║  │  Identity rotation         │    │  Bootstrap eligible-node-1-vote  │     ║
+║  │  Constitutional invariants │    │   (Argon2id PoW, Heavy-Node only)│     ║
+║  │  (MappingProxyType)        │    │  Function Keys (5 of 6 pieces)   │     ║
+║  └─────────────┬──────────────┘    └─────────────┬────────────────────┘     ║
+║                │                                 │                          ║
+║                └──────────────┬──────────────────┘                          ║
+║                               │                                             ║
+║  ┌────────────────────────────┴──────────────────────────────────────┐      ║
+║  │            Wormhole / InfoNet Relay  (transport layer)            │      ║
+║  │   Gate personas │ canonical signing │ Dead Drop epoch mailboxes   │      ║
+║  └───────────────────────────────────────────────────────────────────┘      ║
+╠═════════════════════════════════════════════════════════════════════════════╣
+║              PRIVACY CORE  (Rust crate — locked Protocol contracts)         ║
+║                                                                             ║
+║   privacy-core/  ─►  Argon2id │ Ed25519/X25519 │ AESGCM │ HKDF              ║
+║                      Ring sigs* │ Stealth addrs* │ Pedersen* │ Bulletproofs*║
+║                      Blind-sig issuance* (RSA / BBS+ / U-Prove / Idemix)    ║
+║                                                                             ║
+║   * = locked Protocol contract; cryptographic primitive lands Sprint 11+    ║
+╚═════════════════════════════════════════════════════════════════════════════╝
+
+   Distribution
+   ────────────
+     GitHub (primary):  ghcr.io/bigbodycobain/shadowbroker-{backend,frontend}
+     GitLab (mirror):   registry.gitlab.com/bigbodycobain/shadowbroker/{backend,frontend}
+     Multi-arch:        linux/amd64  +  linux/arm64  (Raspberry Pi 5 supported)
+     Desktop:           Tauri shell  →  packaged backend-runtime  +  Next.js frontend
 ```
 
 ---
@@ -373,7 +521,7 @@ The first decentralized intelligence communication layer built directly into an
 
 | Source | Data | Update Frequency | API Key Required |
 |---|---|---|---|
-| [OpenSky Network](https://opensky-network.org) | Commercial & private flights | ~60s | Optional (anonymous limited) |
+| [OpenSky Network](https://opensky-network.org) | Commercial & private flights | ~60s | **Yes** |
 | [adsb.lol](https://adsb.lol) | Military aircraft | ~60s | No |
 | [aisstream.io](https://aisstream.io) | AIS vessel positions | Real-time WebSocket | **Yes** |
 | [CelesTrak](https://celestrak.org) | Satellite orbital positions (TLE + SGP4) | ~60s | No |
@@ -419,15 +567,16 @@ The first decentralized intelligence communication layer built directly into an
 
 ## 🚀 Getting Started
 
-### 🐳 Docker / Podman Setup (Recommended for Self-Hosting)
+### 🐳 Docker Setup (Recommended for Self-Hosting)
 
-The repo includes a `docker-compose.yml` that builds both images locally.
+The repo includes a `docker-compose.yml` that pulls pre-built images from GitHub Container Registry.
 
 ```bash
 git clone https://github.com/BigBodyCobain/Shadowbroker.git
 cd Shadowbroker
 # Add your API keys in a repo-root .env file (optional — see Environment Variables below)
-./compose.sh up -d
+docker compose pull
+docker compose up -d
 ```
 
 Open `http://localhost:3000` to view the dashboard.
@@ -435,42 +584,73 @@ Open `http://localhost:3000` to view the dashboard.
 > **Deploying publicly or on a LAN?** No configuration needed for most setups.
 > The frontend proxies all API calls through the Next.js server to `BACKEND_URL`,
 > which defaults to `http://backend:8000` (Docker internal networking).
-> Port 8000 does not need to be exposed externally.
+> Host port `8000` is only published for local API/debug access. If it conflicts
+> with another service, set `BACKEND_PORT=8001` in `.env`; leave `BACKEND_URL`
+> as `http://backend:8000` because that is the Docker-internal port.
+> The backend memory cap is controlled by `BACKEND_MEMORY_LIMIT` and defaults
+> to `4G`. If Docker reports OOM events, the backend will restart and slow
+> layers can look empty until they repopulate.
 >
 > If your backend runs on a **different host or port**, set `BACKEND_URL` at runtime — no rebuild required:
 >
 > ```bash
 > # Linux / macOS
-> BACKEND_URL=http://myserver.com:9096 docker-compose up -d
+> BACKEND_URL=http://myserver.com:9096 docker compose up -d
 >
 > # Podman (via compose.sh wrapper)
 > BACKEND_URL=http://192.168.1.50:9096 ./compose.sh up -d
 >
 > # Windows (PowerShell)
-> $env:BACKEND_URL="http://myserver.com:9096"; docker-compose up -d
+> $env:BACKEND_URL="http://myserver.com:9096"; docker compose up -d
 >
 > # Or add to a .env file next to docker-compose.yml:
 > # BACKEND_URL=http://myserver.com:9096
 > ```
 
-If you prefer to call the container engine directly, Podman users can run `podman compose up -d`, or force the wrapper to use Podman with `./compose.sh --engine podman up -d`.
-Depending on your local Podman configuration, `podman compose` may still delegate to an external compose provider while talking to the Podman socket.
+**Podman users:** Do not pass the GitHub URL to `podman compose pull`; clone the repo first, `cd Shadowbroker`, then run compose from that folder. `podman compose` also requires a Compose provider. If Podman reports `looking up compose provider failed`, install one:
+
+```bash
+# Linux / macOS / WSL
+python3 -m pip install --user podman-compose
+podman-compose pull
+podman-compose up -d
+```
+
+```powershell
+# Windows PowerShell
+py -m pip install --user podman-compose
+podman-compose pull
+podman-compose up -d
+```
+
+If you are in a bash-compatible shell, the included wrapper can auto-detect Docker or Podman:
+
+```bash
+./compose.sh --engine podman pull
+./compose.sh --engine podman up -d
+```
 
 ---
 
 ### 🐋 Standalone Deploy (Portainer, Uncloud, NAS, etc.)
 
-No need to clone the repo. Use the pre-built images published to the GitHub Container Registry.
+No need to clone the repo. Use the pre-built images from GitHub Container Registry. GitLab registry images may be used as a mirror if you publish them there.
 
 Create a `docker-compose.yml` with the following content and deploy it directly — paste it into Portainer's stack editor, `uncloud deploy`, or any Docker host:
 
 ```yaml
+## Image registry — uncomment ONE line per service:
+##   GitHub  (primary): ghcr.io/bigbodycobain/shadowbroker-backend:latest
+##   GitLab  (mirror):  registry.gitlab.com/bigbodycobain/shadowbroker/backend:latest
+
+
 services:
   backend:
     image: ghcr.io/bigbodycobain/shadowbroker-backend:latest
+    # image: registry.gitlab.com/bigbodycobain/shadowbroker/backend:latest
     container_name: shadowbroker-backend
     ports:
-      - "8000:8000"
+      - "${BACKEND_PORT:-8000}:8000"
     environment:
       - AIS_API_KEY=your_aisstream_key          # Required — get one free at aisstream.io
       - OPENSKY_CLIENT_ID=                       # Optional — higher flight data rate limits
@@ -486,6 +666,7 @@ services:
 
   frontend:
     image: ghcr.io/bigbodycobain/shadowbroker-frontend:latest
+    # image: registry.gitlab.com/bigbodycobain/shadowbroker/frontend:latest
     container_name: shadowbroker-frontend
     ports:
       - "3000:3000"
@@ -499,7 +680,7 @@ volumes:
   backend_data:
 ```
 
-> **How it works:** The frontend container proxies all `/api/*` requests through the Next.js server to `BACKEND_URL` using Docker's internal networking. The browser only ever talks to port 3000 — port 8000 does not need to be exposed externally.
+> **How it works:** The frontend container proxies all `/api/*` requests through the Next.js server to `BACKEND_URL` using Docker's internal networking. The browser only ever talks to port 3000. The backend's host port is for local API/debug access and can be changed with `BACKEND_PORT=8001` without changing `BACKEND_URL`.
 >
 > `BACKEND_URL` is a plain runtime environment variable (not a build-time `NEXT_PUBLIC_*`), so you can change it in Portainer, Uncloud, or any compose editor without rebuilding the image. Set it to the address where your backend is reachable from inside the Docker network (e.g. `http://backend:8000`, `http://192.168.1.50:8000`).
 
@@ -509,7 +690,7 @@ volumes:
 
 If you just want to run the dashboard without dealing with terminal commands:
 
-1. Go to the **[Releases](../../releases)** tab on the right side of this GitHub page.
+1. Go to the **[Releases](../../releases)** tab on the right side of this repo page.
 2. Download the latest `.zip` file from the release.
 3. Extract the folder to your computer.
 4. **Windows:** Double-click `start.bat`.
@@ -518,9 +699,10 @@ If you just want to run the dashboard without dealing with terminal commands:
 
 Local launcher notes:
 
-- `start.bat` / `start.sh` currently run the hardened web/local stack, not the final native desktop boundary.
-- Security-sensitive paths are hardened up to the pre-Tauri boundary, but operator-facing responsiveness still matters and is part of the acceptance bar.
-- If Wormhole identity or DM contact endpoints fail after an upgrade on Windows, see `F:\Codebase\Oracle\live-risk-dashboard\docs\mesh\pre-tauri-phase-closeout.md` for the secure-storage repair workflow.
+- `start.bat` / `start.sh` run the app without Docker — they install dependencies and start both servers directly.
+- If Wormhole identity or DM contact endpoints fail after an upgrade, check the `docs/mesh/` folder for troubleshooting.
+- For DM root witness, transparency, and operator monitoring rollout, start with `docs/mesh/wormhole-dm-root-operations-runbook.md`.
+- For sample DM root ops bridge assets, also see `scripts/mesh/poll-dm-root-health-alerts.mjs`, `scripts/mesh/export-dm-root-health-prometheus.mjs`, `scripts/mesh/publish-external-root-witness-package.mjs`, `scripts/mesh/smoke-external-root-witness-flow.mjs`, `scripts/mesh/smoke-root-transparency-publication-flow.mjs`, `scripts/mesh/smoke-dm-root-deployment-flow.mjs`, `scripts/mesh/sync-dm-root-external-assurance.mjs`, and `docs/mesh/examples/`.
 
 ---
 
@@ -539,27 +721,27 @@ If you want to modify the code or run from source:
 
 ```bash
 # Clone the repository
-git clone https://github.com/your-username/shadowbroker.git
-cd shadowbroker/live-risk-dashboard
+git clone https://github.com/BigBodyCobain/Shadowbroker.git
+cd Shadowbroker
 
 # Backend setup
 cd backend
 python -m venv venv
 venv\Scripts\activate        # Windows
 # source venv/bin/activate   # macOS/Linux
-pip install -r requirements.txt   # includes pystac-client for Sentinel-2
+pip install .
 
 # Optional helper scripts (creates venv + installs dev deps)
 # Windows PowerShell
-# .\scripts\setup-venv.ps1
+# .\backend\scripts\setup-venv.ps1
 # macOS/Linux
-# ./scripts/setup-venv.sh
+# ./backend/scripts/setup-venv.sh
 
 # Optional env check (prints warnings for missing keys)
 # Windows PowerShell
-# .\scripts\check-env.ps1
+# .\backend\scripts\check-env.ps1
 # macOS/Linux
-# ./scripts/check-env.sh
+# ./backend/scripts/check-env.sh
 
 # Create .env with your API keys
 echo "AIS_API_KEY=your_aisstream_key" >> .env
@@ -568,7 +750,7 @@ echo "OPENSKY_CLIENT_SECRET=your_opensky_secret" >> .env
 
 # Frontend setup
 cd ../frontend
-npm install
+npm ci
 ```
 
 ### Running
@@ -680,7 +862,7 @@ The platform is optimized for handling massive real-time datasets:
 ## 📁 Project Structure
 
 ```
-live-risk-dashboard/
+Shadowbroker/
 ├── backend/
 │   ├── main.py                     # FastAPI app, middleware, API routes (~4,000 lines)
 │   ├── cctv.db                     # SQLite CCTV camera database (auto-generated)
@@ -728,7 +910,6 @@ live-risk-dashboard/
 │   │       ├── mesh_reputation.py  # Node reputation scoring
 │   │       ├── mesh_oracle.py      # Oracle consensus protocol
 │   │       └── mesh_secure_storage.py # Secure credential storage
-│
 ├── frontend/
 │   ├── src/
 │   │   ├── app/
@@ -759,25 +940,52 @@ live-risk-dashboard/
 ### Backend (`backend/.env`)
 
 ```env
-# Required
-AIS_API_KEY=your_aisstream_key                # Maritime vessel tracking (aisstream.io)
+# Required for airplane telemetry (NEW in v0.9.7 — startup env check flags these as critical)
+# Free registration: https://opensky-network.org/index.php?option=com_users&view=registration
+OPENSKY_CLIENT_ID=your_opensky_client_id      # OAuth2 — global flight state vectors
+OPENSKY_CLIENT_SECRET=your_opensky_secret     # OAuth2 — paired with Client ID above
 
 # Optional (enhances data quality)
-OPENSKY_CLIENT_ID=your_opensky_client_id      # OAuth2 — higher rate limits for flight data
-OPENSKY_CLIENT_SECRET=your_opensky_secret     # OAuth2 — paired with Client ID above
+AIS_API_KEY=your_aisstream_key                # Maritime vessel tracking (aisstream.io) — ships layer empty without it
 LTA_ACCOUNT_KEY=your_lta_key                  # Singapore CCTV cameras
 SHODAN_API_KEY=your_shodan_key                # Shodan device search overlay
 SH_CLIENT_ID=your_sentinel_hub_id             # Copernicus CDSE Sentinel Hub imagery
 SH_CLIENT_SECRET=your_sentinel_hub_secret     # Paired with Sentinel Hub Client ID
+MESH_SAR_EARTHDATA_USER=                      # NASA Earthdata user (SAR Mode B — OPERA products)
+MESH_SAR_EARTHDATA_TOKEN=                     # NASA Earthdata token (paired with user above)
+MESH_SAR_COPERNICUS_USER=                     # Copernicus Data Space user (SAR Mode B — EGMS / EMS)
+MESH_SAR_COPERNICUS_TOKEN=                    # Copernicus token (paired with user above)
+OPENCLAW_ACCESS_TIER=restricted               # OpenClaw agent tier: "restricted" (read-only) or "full"
+
+# Private-lane privacy-core pinning (required when Arti or RNS is enabled)
+PRIVACY_CORE_MIN_VERSION=0.1.0
+PRIVACY_CORE_ALLOWED_SHA256=your_privacy_core_sha256
+# Optional override if you load a non-default shared library path
+PRIVACY_CORE_LIB=
 ```
 
+When `MESH_ARTI_ENABLED=true` or `MESH_RNS_ENABLED=true`, backend startup now fails closed unless the loaded `privacy-core` artifact reports a parseable version at or above `PRIVACY_CORE_MIN_VERSION` and matches one of the hashes in `PRIVACY_CORE_ALLOWED_SHA256`.
+
+Generate the hash from the artifact you intend to ship:
+
+```powershell
+Get-FileHash .\privacy-core\target\release\privacy_core.dll -Algorithm SHA256
+```
+
+```bash
+sha256sum ./privacy-core/target/release/libprivacy_core.so
+```
+
+Then confirm authenticated `GET /api/wormhole/status` or `GET /api/settings/wormhole-status` shows the same `privacy_core.version`, `privacy_core.library_path`, and `privacy_core.library_sha256`.
+
 ### Frontend
 
 | Variable | Where to set | Purpose |
 |---|---|---|
 | `BACKEND_URL` | `environment` in `docker-compose.yml`, or shell env | URL the Next.js server uses to proxy API calls to the backend. Defaults to `http://backend:8000`. **Runtime variable — no rebuild needed.** |
+| `BACKEND_PORT` | repo-root `.env` or shell env before `docker compose up` | Host port used to expose the backend API for local diagnostics. Defaults to `8000`; set `BACKEND_PORT=8001` if port 8000 is already in use. Does not change Docker-internal `BACKEND_URL`. |
 
-**How it works:** The frontend proxies all `/api/*` requests through the Next.js server to `BACKEND_URL` using Docker's internal networking. Browsers only talk to port 3000; port 8000 never needs to be exposed externally. For local dev without Docker, `BACKEND_URL` defaults to `http://localhost:8000`.
+**How it works:** The frontend proxies all `/api/*` requests through the Next.js server to `BACKEND_URL` using Docker's internal networking. Browsers only talk to port 3000; the backend host port is only for local diagnostics. For local dev without Docker, `BACKEND_URL` defaults to `http://localhost:8000`.
 
 ---
 
@@ -787,6 +995,7 @@ ShadowBroker is built in the open. These people shipped real code:
 
 | Who | What | PR |
 |-----|------|----|
+| [@Alienmajik](https://gitlab.com/Alienmajik) | Raspberry Pi 5 support — ARM64 packaging, headless deployment notes, runtime tuning for Pi-class hardware | — |
 | [@wa1id](https://github.com/wa1id) | CCTV ingestion fix — threaded SQLite, persistent DB, startup hydration, cluster clickability | #92 |
 | [@AlborzNazari](https://github.com/AlborzNazari) | Spain DGT + Madrid CCTV sources, STIX 2.1 threat intel export | #91 |
 | [@adust09](https://github.com/adust09) | Power plants layer, East Asia intel coverage (JSDF bases, ICAO enrichment, Taiwan news, military classification) | #71, #72, #76, #77, #87 |

backend/.env.example

@@ -11,19 +11,74 @@ AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
 
 # ── Optional ───────────────────────────────────────────────────
 
+# AISHub REST fallback. Used when stream.aisstream.io is unreachable
+# (e.g. their cert expires or server goes offline). Free tier requires
+# registration at https://www.aishub.net/api. Poll cadence defaults to
+# 20 min to stay courteous; tunable via AISHUB_POLL_INTERVAL_MINUTES.
+# AISHUB_USERNAME=
+# AISHUB_POLL_INTERVAL_MINUTES=20
+
 # Override allowed CORS origins (comma-separated). Defaults to localhost + LAN auto-detect.
 # CORS_ORIGINS=http://192.168.1.50:3000,https://my-domain.com
 
 # Admin key — protects sensitive endpoints (API key management, system update).
-# If unset, endpoints are only accessible from localhost unless ALLOW_INSECURE_ADMIN=true.
+# If unset, loopback/localhost requests still work for local single-host dev.
+# Remote/non-loopback admin access requires ADMIN_KEY, or ALLOW_INSECURE_ADMIN=true in debug-only setups.
 # Set this in production and enter the same key in Settings → Admin Key.
 # ADMIN_KEY=your-secret-admin-key-here
 
-# Allow insecure admin access without ADMIN_KEY (local dev only).
+# Allow insecure admin access without ADMIN_KEY (local dev only, beyond loopback).
+# Requires MESH_DEBUG_MODE=true; do not enable this for ordinary use.
 # ALLOW_INSECURE_ADMIN=false
 
-# User-Agent for Nominatim geocoding requests (per OSM usage policy).
-# NOMINATIM_USER_AGENT=ShadowBroker/1.0 (https://github.com/BigBodyCobain/Shadowbroker)
+# Per-install operator handle. Round 7a: every outbound third-party API
+# call (Wikipedia, Wikidata, Nominatim, GDELT, OpenMHz, Broadcastify,
+# weather.gov, NUFORC, etc.) includes this handle in the User-Agent so
+# upstreams can rate-limit / contact the specific install instead of
+# treating every Shadowbroker user as one entity.
+#
+# Default empty -> a stable pseudonymous handle (e.g. "operator-7f3a92") is
+# auto-generated on first run and persisted to backend/data/operator_handle.json.
+# Operators who want a meaningful handle (real name, org, GitHub login) can
+# set it here. Special characters are sanitized to dashes.
+# OPERATOR_HANDLE=
+
+# Default outbound User-Agent for all third-party HTTP fetchers. Operators
+# who run a public relay and want a completely custom UA can set this; it
+# bypasses the per-operator helper entirely. Most installs should leave it
+# unset and use OPERATOR_HANDLE instead.
+# SHADOWBROKER_USER_AGENT=
+
+# Nominatim-specific User-Agent override (OSM usage policy). Leave unset to
+# use the per-install handle (default) — set only if you have a registered
+# Nominatim relay identity.
+# NOMINATIM_USER_AGENT=
+
+# ── Third-party fetcher opt-ins ────────────────────────────────
+# These data sources phone home to politically/commercially sensitive
+# upstreams. Disabled by default; set to "true" only if the operator
+# explicitly wants the node's IP to contact these services.
+#
+# CrowdThreat — backend.crowdthreat.world (paid threat-intel aggregator).
+# CROWDTHREAT_ENABLED=false
+#
+# EUvsDisinfo FIMI — euvsdisinfo.eu (EU disinformation tracker).
+# FIMI_ENABLED=false
+#
+# Polymarket + Kalshi — US political/election prediction markets.
+# PREDICTION_MARKETS_ENABLED=false
+#
+# Finnhub fallback / yfinance — financial market data.
+# Set FINNHUB_API_KEY to enable Finnhub, or set FINANCIAL_ENABLED=true to allow
+# the unauthenticated yfinance fallback to call Yahoo Finance.
+# FINANCIAL_ENABLED=false
+#
+# NUFORC UAP sightings — huggingface.co dataset download.
+# NUFORC_ENABLED=false
+#
+# News RSS aggregator — defaults ON. Set to "false" to disable all
+# configured news feeds (kill switch for the news layer).
+# NEWS_ENABLED=true
 
 # LTA Singapore traffic cameras — leave blank to skip this data source.
 # LTA_ACCOUNT_KEY=
@@ -35,19 +90,109 @@ AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
 # Ukraine air raid alerts from alerts.in.ua — free token from https://alerts.in.ua/
 # ALERTS_IN_UA_TOKEN=
 
+# Optional NUFORC UAP sighting map enrichment via Mapbox Tilequery.
+# Leave blank to skip this optional enrichment.
+# NUFORC_MAPBOX_TOKEN=
+
 # Google Earth Engine service account for VIIRS change detection (optional).
 # Download JSON key from https://console.cloud.google.com/iam-admin/serviceaccounts
 # pip install earthengine-api
 # GEE_SERVICE_ACCOUNT_KEY=
 
+# ── Meshtastic MQTT Bridge ─────────────────────────────────────
+# Disabled by default to respect the public Meshtastic broker.
+# When enabled, subscribes to US region only. Add more regions via MESH_MQTT_EXTRA_ROOTS.
+# MESH_MQTT_ENABLED=false
+# MESH_MQTT_EXTRA_ROOTS=EU_868,ANZ    # comma-separated additional region roots
+# MESH_MQTT_INCLUDE_DEFAULT_ROOTS=true
+# MESH_MQTT_BROKER=mqtt.meshtastic.org
+# MESH_MQTT_PORT=1883
+# Leave user/pass blank for the public Meshtastic broker default.
+# MESH_MQTT_USER=
+# MESH_MQTT_PASS=
+
+# Optional Meshtastic node ID (e.g. "!abcd1234"). When set, included in the
+# User-Agent sent to meshtastic.liamcottle.net so the upstream service operator
+# can identify per-install traffic instead of aggregated "ShadowBroker" hits.
+# Leave blank to send a generic UA. If you set MESHTASTIC_OPERATOR_CALLSIGN,
+# it is included in outbound headers to meshtastic.org by default so they
+# can rate-limit per-operator. Set MESHTASTIC_SEND_CALLSIGN_HEADER=false to
+# suppress the callsign while still using it locally (e.g. for APRS).
+# MESHTASTIC_OPERATOR_CALLSIGN=
+# MESHTASTIC_SEND_CALLSIGN_HEADER=true
+# MESH_MQTT_PSK=                       # hex-encoded, empty = default LongFast key
+
 # ── Mesh / Reticulum (RNS) ─────────────────────────────────────
 # Full-node / participant-node posture for public Infonet sync.
 # MESH_NODE_MODE=participant      # participant | relay | perimeter
+# Legacy compatibility sunset toggles. Default posture is to block these.
+# Legacy 16-hex node-id binding no longer has a boolean escape hatch; use a
+# dated migration override only when you intentionally need older peers during
+# migration before the hard removal target in v0.10.0 / 2026-06-01.
+# MESH_BLOCK_LEGACY_NODE_ID_COMPAT=true
+# MESH_ALLOW_LEGACY_NODE_ID_COMPAT_UNTIL=2026-05-15
+# MESH_BLOCK_LEGACY_AGENT_ID_LOOKUP=true
+# Temporary DM invite migration escape hatch. Default posture blocks importing
+# legacy/compat v1/v2 DM invites; use a dated override only while retiring
+# older exports and ask senders to re-export a current signed invite.
+# MESH_ALLOW_COMPAT_DM_INVITE_IMPORT_UNTIL=2026-05-15
+# Temporary legacy GET DM poll/count escape hatch. Default posture requires the
+# signed mailbox-claim POST APIs; only use this dated override while retiring
+# older clients that still call GET poll/count directly.
+# MESH_ALLOW_LEGACY_DM_GET_UNTIL=2026-05-15
+# Temporary raw dm1 compose/decrypt escape hatch. Default posture expects MLS
+# DM bootstrap on supported peers; only use this dated override while retiring
+# older clients that still need the raw dm1 helper path.
+# MESH_ALLOW_LEGACY_DM1_UNTIL=2026-05-15
+# Temporary legacy dm_message signature escape hatch. Default posture requires
+# the full modern signed payload; only enable this with a dated migration
+# override while older senders are being retired.
+# MESH_ALLOW_LEGACY_DM_SIGNATURE_COMPAT_UNTIL=2026-05-15
+# Rotate voter-blinding salts so new reputation events stop reusing one
+# forever-stable blinded ID. Keep grace >= rotation cadence so older votes
+# remain matchable while they age out of the ledger.
+# MESH_VOTER_BLIND_SALT_ROTATE_DAYS=30
+# MESH_VOTER_BLIND_SALT_GRACE_DAYS=30
+# Deprecated legacy env vars kept only for backward config compatibility.
+# Ordinary shipped gate flows keep MLS decrypt local; service-side decrypt is
+# reserved for explicit recovery reads.
+# MESH_GATE_BACKEND_DECRYPT_COMPAT=false
+# MESH_GATE_BACKEND_DECRYPT_COMPAT_ACKNOWLEDGE=false
+# Deprecated legacy env vars kept only for backward config compatibility.
+# Ordinary shipped gate flows keep plaintext compose/post local and only submit
+# encrypted envelopes to the backend for sign/post.
+# MESH_GATE_BACKEND_PLAINTEXT_COMPAT=false
+# MESH_GATE_BACKEND_PLAINTEXT_COMPAT_ACKNOWLEDGE=false
+# Legacy runtime switches for recovery envelopes. Per-gate envelope_policy is
+# the source of truth; leave these at the default unless testing old behavior.
+# MESH_GATE_RECOVERY_ENVELOPE_ENABLE=true
+# MESH_GATE_RECOVERY_ENVELOPE_ENABLE_ACKNOWLEDGE=true
+# Optional operator-only recovery tradeoff. Leave off for the default posture:
+# ordinary gate reads keep plaintext local/in-memory unless you explicitly use
+# the recovery-envelope path.
+# MESH_GATE_PLAINTEXT_PERSIST=false
+# MESH_GATE_PLAINTEXT_PERSIST_ACKNOWLEDGE=false
+# Legacy Phase-1 gate envelope fallback is now explicit and time-bounded per
+# gate. This only controls the default expiry window when you deliberately
+# re-enable that migration path for older stored envelopes.
+# MESH_GATE_LEGACY_ENVELOPE_FALLBACK_MAX_DAYS=30
+# Feature-flagged multiplexed gate session stream. Stream-first room ownership
+# is implemented; keep off until you want that rollout enabled in your env.
+# MESH_GATE_SESSION_STREAM_ENABLED=false
+# MESH_GATE_SESSION_STREAM_HEARTBEAT_S=20
+# MESH_GATE_SESSION_STREAM_BATCH_MS=1500
+# MESH_GATE_SESSION_STREAM_MAX_GATES=16
 # MESH_BOOTSTRAP_DISABLED=false
 # MESH_BOOTSTRAP_MANIFEST_PATH=data/bootstrap_peers.json
 # MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY=
-# MESH_RELAY_PEERS=               # comma-separated operator-trusted sync/push peers
-# MESH_PEER_PUSH_SECRET=          # shared-secret push auth for trusted testnet peers
+# Infonet/Wormhole fails closed to onion/RNS by default. Only enable clearnet
+# sync for local relay development or an explicitly public testnet.
+# MESH_INFONET_ALLOW_CLEARNET_SYNC=false
+# MESH_BOOTSTRAP_SEED_PEERS=http://gqpbunqbgtkcqilvclm3xrkt3zowjyl3s62kkktvojgvxzizamvbrqid.onion:8000
+# Add comma-separated http://*.onion peers as more private seed/relay nodes come online.
+# MESH_DEFAULT_SYNC_PEERS=         # legacy alias; prefer MESH_BOOTSTRAP_SEED_PEERS
+# MESH_RELAY_PEERS=               # comma-separated operator-trusted sync/push peers (empty by default)
+# MESH_PEER_PUSH_SECRET=          # REQUIRED when relay/RNS peers are configured (min 16 chars, generate with: python -c "import secrets; print(secrets.token_urlsafe(32))")
 # MESH_SYNC_INTERVAL_S=300
 # MESH_SYNC_FAILURE_BACKOFF_S=60
 #
@@ -90,8 +235,54 @@ AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
 # MESH_VERIFY_INTERVAL_S=600
 # MESH_VERIFY_SIGNATURES=false
 
+# ── Secure Storage (non-Windows) ───────────────────────────────
+# Required on Linux/Docker to protect Wormhole key material at rest.
+# Generate with: python -c "import secrets; print(secrets.token_urlsafe(32))"
+# Also supports Docker secrets via MESH_SECURE_STORAGE_SECRET_FILE.
+# MESH_SECURE_STORAGE_SECRET=
+#
+# To rotate the storage secret, stop the backend and run:
+#   1. Dry-run first (validates without writing):
+#      MESH_OLD_STORAGE_SECRET=<current> MESH_NEW_STORAGE_SECRET=<new> \
+#        python -m scripts.rotate_secure_storage_secret --dry-run
+#   2. Rotate (creates .bak backups, then rewraps envelopes):
+#      MESH_OLD_STORAGE_SECRET=<current> MESH_NEW_STORAGE_SECRET=<new> \
+#        python -m scripts.rotate_secure_storage_secret
+#   3. Update MESH_SECURE_STORAGE_SECRET to the new value and restart.
+#
+# If rotation is interrupted, .bak files preserve the old envelopes.
+# To repair corrupted secure-json payloads (not key envelopes), use:
+#   python -m scripts.repair_wormhole_secure_storage
+
 # ── Mesh DM Relay ──────────────────────────────────────────────
 # MESH_DM_TOKEN_PEPPER=change-me
+# Keep DM relay metadata retention explicit and bounded.
+# MESH_DM_KEY_TTL_DAYS=30
+# MESH_DM_PREKEY_LOOKUP_ALIAS_TTL_DAYS=14
+# MESH_DM_WITNESS_TTL_DAYS=14
+# MESH_DM_BINDING_TTL_DAYS=3
+# Optional operational bridge for externally sourced root witnesses / transparency.
+# Relative paths resolve from the backend directory.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH=data/root_witness_import.json
+# Local single-host dev example after bootstrapping an external witness locally:
+# MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH=../ops/root_witness_receipt_import.json
+# Optional URI bridge for externally retrieved root witness packages.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_URI=file:///absolute/path/root_witness_import.json
+# Maximum acceptable age for external witness packages before strong DM trust fails closed.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_MAX_AGE_S=3600
+# Warning threshold for external witness packages before fail-closed max age.
+# MESH_DM_ROOT_EXTERNAL_WITNESS_WARN_AGE_S=2700
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH=data/root_transparency_ledger.json
+# Local single-host dev example after publishing the transparency ledger locally:
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH=../ops/root_transparency_ledger.json
+# Optional URI used to read back and verify a published transparency ledger.
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI=file:///absolute/path/root_transparency_ledger.json
+# Local single-host dev readback example:
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI=../ops/root_transparency_ledger.json
+# Maximum acceptable age for external transparency ledgers before strong DM trust fails closed.
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_MAX_AGE_S=3600
+# Warning threshold for external transparency ledgers before fail-closed max age.
+# MESH_DM_ROOT_TRANSPARENCY_LEDGER_WARN_AGE_S=2700
 
 # ── Self Update ────────────────────────────────────────────────
 # MESH_UPDATE_SHA256=
@@ -103,3 +294,69 @@ AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
 # WORMHOLE_TRANSPORT=direct
 # WORMHOLE_SOCKS_PROXY=127.0.0.1:9050
 # WORMHOLE_SOCKS_DNS=true
+# Optional override for the loaded Rust privacy-core shared library. Leave
+# unset for the default repo search order. When you override this, verify the
+# authenticated wormhole status surfaces show the expected version, absolute
+# library path, and SHA-256 for the loaded artifact before making stronger
+# privacy claims about the deployment.
+# PRIVACY_CORE_LIB=
+# Minimum privacy-core version accepted when hidden/private carriers are
+# enabled. Private-lane startup fails closed if the loaded artifact is
+# missing, reports no parseable version, or falls below this minimum.
+# PRIVACY_CORE_MIN_VERSION=0.1.0
+# Comma-separated SHA-256 allowlist for the exact privacy-core artifact(s)
+# your deployment is allowed to load. Required for Arti/RNS private-lane
+# startup. Generate with:
+#   PowerShell: Get-FileHash .\privacy-core\target\release\privacy_core.dll -Algorithm SHA256
+#   macOS/Linux: sha256sum ./privacy-core/target/release/libprivacy_core.so
+# PRIVACY_CORE_ALLOWED_SHA256=
+# Optional structured release attestation artifact for the Sprint 8 release gate.
+# Relative paths resolve from the backend directory. When set explicitly, a
+# missing or unreadable file fails the DM relay security-suite criterion closed.
+# CI/release tooling can generate this automatically via:
+#   uv run python scripts/release_helper.py write-attestation ...
+# MESH_RELEASE_ATTESTATION_PATH=data/release_attestation.json
+# Operator-only Sprint 8 release attestation. Set this only when the DM relay
+# security suite has been run and passed for the current release candidate.
+# File-based release attestation takes precedence when present.
+# MESH_RELEASE_DM_RELAY_SECURITY_SUITE_GREEN=false
+
+# ── OpenClaw Agent ─────────────────────────────────────────────
+# HMAC shared secret for remote OpenClaw agent authentication.
+# Auto-generated via the Connect OpenClaw modal — do not set manually.
+# OPENCLAW_HMAC_SECRET=
+# Access tier: "restricted" (read-only) or "full" (read+write+inject)
+# OPENCLAW_ACCESS_TIER=restricted
+
+# ── SAR (Synthetic Aperture Radar) Layer ───────────────────────
+# Mode A — Free catalog metadata from Alaska Satellite Facility (ASF Search).
+# No account, no downloads.  Default-on.  Set to false to disable entirely.
+# MESH_SAR_CATALOG_ENABLED=true
+#
+# Mode B — Free pre-processed ground-change anomalies (deformation, flood,
+# damage assessments) from NASA OPERA, Copernicus EGMS, GFM, EMS, UNOSAT.
+# Two-step opt-in: BOTH of the following must be set together.
+#   1. MESH_SAR_PRODUCTS_FETCH=allow
+#   2. MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE=true
+# Either flag alone keeps Mode B disabled.  You can also enable this from
+# the Settings → SAR panel inside the app.
+# MESH_SAR_PRODUCTS_FETCH=block
+# MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE=false
+#
+# NASA Earthdata Login (free, ~1 minute signup) — required for OPERA products.
+#   Sign up: https://urs.earthdata.nasa.gov/users/new
+#   Generate token: https://urs.earthdata.nasa.gov/profile  →  "Generate Token"
+# MESH_SAR_EARTHDATA_USER=
+# MESH_SAR_EARTHDATA_TOKEN=
+#
+# Copernicus Data Space (free, ~1 minute signup) — required for EGMS / EMS.
+#   Sign up: https://dataspace.copernicus.eu/
+# MESH_SAR_COPERNICUS_USER=
+# MESH_SAR_COPERNICUS_TOKEN=
+#
+# Allow OpenClaw agents to read and act on the SAR layer (default true).
+# MESH_SAR_OPENCLAW_ENABLED=true
+#
+# Require private-tier transport (Tor / RNS) before signing and broadcasting
+# SAR anomalies to the mesh.  Default true — disable only for testnet/local use.
+# MESH_SAR_REQUIRE_PRIVATE_TIER=true

backend/Dockerfile

@@ -1,10 +1,33 @@
+# ---- Stage 1: Compile privacy-core Rust library ----
+FROM --platform=$BUILDPLATFORM rust:1.88-slim-bookworm AS rust-builder
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    git \
+    pkg-config \
+    libssl-dev \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
+ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
+
+COPY privacy-core /build/privacy-core
+WORKDIR /build/privacy-core
+RUN cargo build --release --lib \
+    && ls -la target/release/libprivacy_core.so
+
+# ---- Stage 2: Python backend ----
 FROM python:3.11-slim-bookworm
 
 WORKDIR /app
 
-# Install Node.js (for AIS WebSocket proxy) and curl (for network fallback)
+# Install Node.js (for AIS WebSocket proxy), curl (for network fallback), and
+# Tor (for Wormhole/remote-agent .onion transport).
 RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
     curl \
+    tor \
     && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
     && apt-get install -y --no-install-recommends nodejs \
     && rm -rf /var/lib/apt/lists/*
@@ -28,6 +51,13 @@ RUN cd /workspace/backend && uv sync --frozen --no-dev \
 # Copy backend source code
 COPY backend/ .
 
+# Preserve safe static data outside /app/data. The compose named volume mounted
+# at /app/data hides image-baked files on first run, so the entrypoint seeds
+# missing static JSON into fresh volumes before the backend starts.
+RUN mkdir -p /app/image-data \
+    && if [ -d /app/data ]; then cp -a /app/data/. /app/image-data/; fi \
+    && chmod +x /app/docker-entrypoint.sh
+
 # Install Node.js dependencies (ws module for AIS WebSocket proxy)
 COPY backend/package*.json ./
 RUN npm ci --omit=dev
@@ -35,6 +65,9 @@ RUN npm ci --omit=dev
 # Clean up workspace scaffold
 RUN rm -rf /workspace
 
+# Copy compiled privacy-core library from Rust builder stage
+COPY --from=rust-builder /build/privacy-core/target/release/libprivacy_core.so /app/libprivacy_core.so
+ENV PRIVACY_CORE_LIB=/app/libprivacy_core.so
 
 # Create a non-root user for security
 # Grant write access to /app so the auto-updater can extract files
@@ -51,4 +84,5 @@ USER backenduser
 EXPOSE 8000
 
 # Start FastAPI server
+ENTRYPOINT ["/app/docker-entrypoint.sh"]
 CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--timeout-keep-alive", "120"]

backend/ais_proxy.js

@@ -1,5 +1,37 @@
+// AIS Stream WebSocket proxy.
+//
+// Reads AIS_API_KEY from argv or env, opens a wss:// connection to
+// stream.aisstream.io, subscribes for vessel position reports inside the
+// active map bounding boxes, and pipes JSON messages to stdout for the
+// Python backend to ingest.
+//
+// Issue #258 — SPKI pinning fallback for upstream cert outages
+// -------------------------------------------------------------
+// AISStream uses Let's Encrypt and their renewal pipeline has been observed
+// to fail (cert expired on 2026-05-20). The naive fix the issue reporter
+// applied — passing { rejectUnauthorized: false } — turns off TLS validation
+// entirely, which lets any network attacker MITM the WebSocket and inject
+// fake ship positions onto the operator's map. Same class as the GDELT
+// plaintext-HTTP MITM issue (#199).
+//
+// Instead, when the normal TLS handshake fails with CERT_HAS_EXPIRED, we
+// do a custom TLS connection that ignores ONLY the expiry check, capture
+// the leaf certificate, and compare its public-key SPKI hash against a
+// pinned list (backend/data/aisstream_spki_pins.json). If the SPKI matches,
+// the upstream is still the genuine AISStream — just with an expired cert —
+// and we proceed in "degraded TLS" mode. If the SPKI does not match, we
+// refuse the connection and log loudly: an actual MITM is in progress.
+//
+// Let's Encrypt renewals keep the same public key by default, so the pinned
+// SPKI survives normal cert rotation. The pin list MUST be updated before
+// the operator's pinned key is rotated upstream.
+
 const WebSocket = require('ws');
 const readline = require('readline');
+const fs = require('fs');
+const path = require('path');
+const tls = require('tls');
+const crypto = require('crypto');
 
 const args = process.argv.slice(2);
 const API_KEY = args[0] || process.env.AIS_API_KEY;
@@ -9,6 +41,135 @@ if (!API_KEY) {
     process.exit(1);
 }
 
+// ── SPKI pin support (issue #258) ─────────────────────────────────────────
+
+const AIS_HOST = 'stream.aisstream.io';
+const AIS_PORT = 443;
+const AIS_WS_URL = `wss://${AIS_HOST}/v0/stream`;
+
+// Pin file is looked up in several layouts so the same JS works in:
+//   - the Docker backend image (PIN_FILE_CANDIDATES[0])
+//   - the Tauri desktop runtime (PIN_FILE_CANDIDATES[1])
+//   - a future relocated layout (operator can drop a file at
+//     SHADOWBROKER_AIS_PINS env var)
+const PIN_FILE_CANDIDATES = [
+    process.env.SHADOWBROKER_AIS_PINS || '',
+    path.join(__dirname, 'data', 'aisstream_spki_pins.json'),
+    path.join(__dirname, 'aisstream_spki_pins.json'),
+].filter(Boolean);
+
+// Embedded fallback. Used when no external pin file is reachable so the
+// SPKI fallback still works on minimal/portable installs. The external
+// file (when present) takes priority so operators can update pins without
+// needing a new build.
+const EMBEDDED_PINS = {
+    [AIS_HOST]: [
+        // Captured 2026-05-20 from AISStream's leaf cert (Let's Encrypt R12).
+        // Replace when AISStream rotates server keys.
+        'GJ10H0UPgLrO+2d3ZXROR/TXSVFXKUfRC3QEI2ibEg4=',
+    ],
+};
+
+let aisDegradedMode = false;  // surfaced via stdout status_query marker
+
+function loadSpkiPins() {
+    for (const candidate of PIN_FILE_CANDIDATES) {
+        try {
+            const raw = fs.readFileSync(candidate, 'utf-8');
+            const parsed = JSON.parse(raw);
+            const pins = Array.isArray(parsed[AIS_HOST]) ? parsed[AIS_HOST] : [];
+            const cleaned = pins
+                .filter((p) => typeof p === 'string' && p.length > 0)
+                .map((p) => p.trim());
+            if (cleaned.length > 0) {
+                return cleaned;
+            }
+        } catch (e) {
+            // Try the next candidate — file may not exist in this layout.
+            continue;
+        }
+    }
+    const embedded = (EMBEDDED_PINS[AIS_HOST] || []).slice();
+    if (embedded.length > 0) {
+        console.error(
+            '[AIS Proxy] No external SPKI pin file found; using embedded fallback. '
+            + `(Set SHADOWBROKER_AIS_PINS or drop ${PIN_FILE_CANDIDATES[1]} to override.)`
+        );
+    }
+    return embedded;
+}
+
+function spkiHashFromPeerCert(peerCert) {
+    // tls.TLSSocket.getPeerCertificate() exposes .pubkey when called with
+    // detailed=true. The pubkey buffer is the DER-encoded SubjectPublicKeyInfo,
+    // which is exactly the value we hash for SPKI pinning.
+    if (!peerCert || !peerCert.pubkey) return null;
+    return crypto.createHash('sha256').update(peerCert.pubkey).digest('base64');
+}
+
+// Probe the upstream when normal TLS failed with CERT_HAS_EXPIRED. We open
+// a raw TLS connection with rejectUnauthorized=false ONLY to inspect the
+// leaf cert; we do NOT use this socket for the actual WebSocket traffic.
+// Returns { ok: true } if the leaf SPKI matches the pin list, { ok: false }
+// with a reason otherwise.
+function verifyExpiredCertAgainstPins() {
+    return new Promise((resolve) => {
+        const pins = loadSpkiPins();
+        if (pins.length === 0) {
+            resolve({ ok: false, reason: 'no SPKI pins configured' });
+            return;
+        }
+        const sock = tls.connect(
+            {
+                host: AIS_HOST,
+                port: AIS_PORT,
+                servername: AIS_HOST,
+                // Allow the handshake to complete despite the expired cert
+                // so we can inspect the leaf. We do NOT trust this connection
+                // for any application data.
+                rejectUnauthorized: false,
+            },
+            () => {
+                const peer = sock.getPeerCertificate(true);
+                sock.end();
+                if (!peer || Object.keys(peer).length === 0) {
+                    resolve({ ok: false, reason: 'no peer certificate returned' });
+                    return;
+                }
+                if (peer.subject && peer.subject.CN !== AIS_HOST) {
+                    resolve({
+                        ok: false,
+                        reason: `cert CN mismatch (got ${peer.subject.CN}, expected ${AIS_HOST})`,
+                    });
+                    return;
+                }
+                const hash = spkiHashFromPeerCert(peer);
+                if (!hash) {
+                    resolve({ ok: false, reason: 'could not compute SPKI hash from peer cert' });
+                    return;
+                }
+                if (pins.includes(hash)) {
+                    resolve({ ok: true, hash });
+                } else {
+                    resolve({
+                        ok: false,
+                        reason: `SPKI ${hash} not in pin list (possible MITM)`,
+                    });
+                }
+            },
+        );
+        sock.setTimeout(10000, () => {
+            sock.destroy();
+            resolve({ ok: false, reason: 'TLS probe timeout' });
+        });
+        sock.on('error', (err) => {
+            resolve({ ok: false, reason: `TLS probe error: ${err.message}` });
+        });
+    });
+}
+
+// ── Subscription state ───────────────────────────────────────────────────
+
 // Start with global coverage, until frontend updates it
 let currentBboxes = [[[-90, -180], [90, 180]]];
 let activeWs = null;
@@ -42,14 +203,34 @@ rl.on('line', (line) => {
             currentBboxes = cmd.bboxes;
             if (activeWs) sendSub(activeWs); // Resend subscription (swap and replace)
         }
+        if (cmd.type === "status_query") {
+            // Allow the Python side to probe degraded-mode state by sending
+            // {"type": "status_query"} on stdin. Reply on stdout as a marker.
+            process.stdout.write(JSON.stringify({
+                __ais_proxy_status: { degraded_tls: aisDegradedMode }
+            }) + '\n');
+        }
     } catch (e) {}
 });
 
-function connect() {
-    const ws = new WebSocket('wss://stream.aisstream.io/v0/stream');
+function attachWsHandlers(ws, { degraded } = { degraded: false }) {
     activeWs = ws;
 
     ws.on('open', () => {
+        if (degraded) {
+            console.error(
+                '[AIS Proxy] Connected in DEGRADED TLS MODE — upstream cert is expired '
+                + 'but SPKI matches the pinned key, so identity is still verified. '
+                + 'AISStream needs to renew their cert; until then MITM protection '
+                + 'depends only on the SPKI match. Watch backend logs for resolution.'
+            );
+            aisDegradedMode = true;
+        } else {
+            if (aisDegradedMode) {
+                console.error('[AIS Proxy] Reconnected with full TLS validation — degraded mode cleared.');
+            }
+            aisDegradedMode = false;
+        }
         sendSub(ws);
     });
 
@@ -61,14 +242,63 @@ function connect() {
     });
 
     ws.on('error', (err) => {
-        console.error("WebSocket Proxy Error:", err.message);
+        console.error('WebSocket Proxy Error:', err.message);
     });
 
     ws.on('close', () => {
         activeWs = null;
-        console.error("WebSocket Proxy Closed. Reconnecting in 5s...");
+        console.error('WebSocket Proxy Closed. Reconnecting in 5s...');
         setTimeout(connect, 5000);
     });
 }
 
+function connect() {
+    // Path A: normal TLS validation (the 99.9% case). If this succeeds we
+    // never touch the SPKI fallback.
+    const ws = new WebSocket(AIS_WS_URL);
+
+    let openedOk = false;
+    ws.on('open', () => { openedOk = true; });
+
+    ws.on('error', async (err) => {
+        // Only the CERT_HAS_EXPIRED case triggers SPKI verification. Any
+        // other TLS or network error gets the standard reconnect path so we
+        // don't accidentally cover up legitimate problems.
+        if (!openedOk && err && err.code === 'CERT_HAS_EXPIRED') {
+            console.error(
+                '[AIS Proxy] Upstream certificate is expired. Verifying SPKI '
+                + 'against pinned keys before deciding whether to proceed in '
+                + 'degraded mode...'
+            );
+            const verdict = await verifyExpiredCertAgainstPins();
+            if (verdict.ok) {
+                console.error(
+                    `[AIS Proxy] SPKI ${verdict.hash} matches pinned key — `
+                    + 'identity is verified, proceeding in DEGRADED TLS mode.'
+                );
+                const insecureWs = new WebSocket(AIS_WS_URL, {
+                    rejectUnauthorized: false,
+                });
+                attachWsHandlers(insecureWs, { degraded: true });
+            } else {
+                console.error(
+                    `[AIS Proxy] SPKI verification FAILED (${verdict.reason}). `
+                    + 'Refusing to connect — this would normally indicate an active '
+                    + 'MITM attack. If AISStream rotated their server key, update '
+                    + 'backend/data/aisstream_spki_pins.json with the new SPKI hash.'
+                );
+                // Schedule a retry — operator may have updated the pin file.
+                setTimeout(connect, 60000);
+            }
+            return;
+        }
+        // Default: surface the error and let the close handler reconnect.
+        console.error('WebSocket Proxy Error:', err.message);
+    });
+
+    // Wire normal handlers — these apply unless the error handler above
+    // takes over and replaces activeWs with an insecure socket.
+    attachWsHandlers(ws, { degraded: false });
+}
+
 connect();

backend/config/news_feeds.json

@@ -1,15 +1,5 @@
 {
   "feeds": [
-    {
-      "name": "Reuters",
-      "url": "https://www.reutersagency.com/feed/?best-topics=world",
-      "weight": 5
-    },
-    {
-      "name": "AP News",
-      "url": "https://rsshub.app/apnews/topics/world-news",
-      "weight": 5
-    },
     {
       "name": "NPR",
       "url": "https://feeds.npr.org/1004/rss.xml",
@@ -99,6 +89,26 @@
       "name": "Japan Times",
       "url": "https://www.japantimes.co.jp/feed/",
       "weight": 3
+    },
+    {
+      "name": "CSM",
+      "url": "https://www.csmonitor.com/rss/world",
+      "weight": 4
+    },
+    {
+      "name": "PBS NewsHour",
+      "url": "https://www.pbs.org/newshour/feeds/rss/world",
+      "weight": 4
+    },
+    {
+      "name": "France 24",
+      "url": "https://www.france24.com/en/rss",
+      "weight": 4
+    },
+    {
+      "name": "DW",
+      "url": "https://rss.dw.com/xml/rss-en-world",
+      "weight": 4
     }
   ]
 }

backend/data/aisstream_spki_pins.json

@@ -0,0 +1,31 @@
+{
+  "_comment": [
+    "SPKI (Subject Public Key Info) pin list for stream.aisstream.io.",
+    "",
+    "Issue #258: AISStream's Let's Encrypt cert expired on 2026-05-20 due to an",
+    "upstream renewal-pipeline failure. Disabling TLS verification entirely",
+    "would let any network attacker MITM the AIS WebSocket and inject fake",
+    "ship positions onto the operator's map (same class as #199 GDELT MITM).",
+    "Instead we pin the leaf certificate's public-key SPKI hash: if normal",
+    "TLS validation fails specifically with CERT_HAS_EXPIRED, ais_proxy.js",
+    "re-checks the leaf cert's SPKI against this list. A match means the",
+    "key is still the genuine AISStream key (Let's Encrypt renewals keep the",
+    "same key unless rekey is requested), so we proceed in 'degraded TLS'",
+    "mode. A mismatch means a real MITM attempt and we refuse the connection.",
+    "",
+    "Format: each entry is a SHA-256 hash of the DER-encoded SPKI bytes,",
+    "encoded as standard base64 (matches the format produced by:",
+    "  openssl s_client -connect host:443 | \\",
+    "    openssl x509 -pubkey -noout | openssl pkey -pubin -outform DER | \\",
+    "    openssl dgst -sha256 -binary | openssl base64",
+    ").",
+    "",
+    "When AISStream rotates their server key (rare — Let's Encrypt renewals",
+    "default to keeping the same key), capture the new SPKI and add it to",
+    "this list BEFORE removing the old one. That way operators on the old",
+    "code still validate against the previous key during the transition."
+  ],
+  "stream.aisstream.io": [
+    "GJ10H0UPgLrO+2d3ZXROR/TXSVFXKUfRC3QEI2ibEg4="
+  ]
+}

backend/data/carrier_seed.json

@@ -0,0 +1,120 @@
+{
+  "_meta": {
+    "as_of": "2026-03-09",
+    "source": "USNI News Fleet & Marine Tracker",
+    "source_url": "https://news.usni.org/2026/03/09/usni-news-fleet-and-marine-tracker-march-9-2026",
+    "note": "One-shot bootstrap for first-run carrier positions. Once carrier_cache.json exists in the runtime data volume, this seed file is never read again. All subsequent updates come from GDELT (and any future sources) and are written to carrier_cache.json. A year from now, your runtime cache reflects whatever your install has observed since first launch — not these snapshot positions."
+  },
+  "carriers": {
+    "CVN-68": {
+      "lat": 47.5535,
+      "lng": -122.6400,
+      "heading": 90,
+      "desc": "Bremerton, WA (Maintenance)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-76": {
+      "lat": 47.5580,
+      "lng": -122.6360,
+      "heading": 90,
+      "desc": "Bremerton, WA (Decommissioning)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-69": {
+      "lat": 36.9465,
+      "lng": -76.3265,
+      "heading": 0,
+      "desc": "Norfolk, VA (Post-deployment maintenance)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-78": {
+      "lat": 18.0,
+      "lng": 39.5,
+      "heading": 0,
+      "desc": "Red Sea — Operation Epic Fury (USNI Mar 9)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-74": {
+      "lat": 36.98,
+      "lng": -76.43,
+      "heading": 0,
+      "desc": "Newport News, VA (RCOH refueling overhaul)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-75": {
+      "lat": 36.0,
+      "lng": 15.0,
+      "heading": 0,
+      "desc": "Mediterranean Sea deployment (USNI Mar 9)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-77": {
+      "lat": 36.5,
+      "lng": -74.0,
+      "heading": 0,
+      "desc": "Atlantic — Pre-deployment workups (USNI Mar 9)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-70": {
+      "lat": 32.6840,
+      "lng": -117.1290,
+      "heading": 180,
+      "desc": "San Diego, CA (Homeport)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-71": {
+      "lat": 32.6885,
+      "lng": -117.1280,
+      "heading": 180,
+      "desc": "San Diego, CA (Maintenance)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-72": {
+      "lat": 20.0,
+      "lng": 64.0,
+      "heading": 0,
+      "desc": "Arabian Sea — Operation Epic Fury (USNI Mar 9)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    },
+    "CVN-73": {
+      "lat": 35.2830,
+      "lng": 139.6700,
+      "heading": 180,
+      "desc": "Yokosuka, Japan (Forward deployed)",
+      "source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
+      "source_url": "https://news.usni.org/category/fleet-tracker",
+      "position_source_at": "2026-03-09T00:00:00Z",
+      "position_confidence": "seed"
+    }
+  }
+}

backend/data/military_bases.json

@@ -1047,14 +1047,6 @@
     "lat": 37.47,
     "lng": 69.381
   },
-  {
-    "name": "Berth rights and right to station its troops in Qatar",
-    "country": "India",
-    "operator": "India",
-    "branch": "army",
-    "lat": 25.308,
-    "lng": 51.209
-  },
   {
     "name": "Ahmad al-Jaber Air Base",
     "country": "Italy",

backend/data/plane_alert_db.json

@@ -73567,6 +73567,14 @@
         "tags": "Air Ambo, Medical Evac, Saving Lives",
         "link": "https://www.airmethods.com/"
     },
+    "ABD9B5": {
+        "registration": "N8628",
+        "operator": "Elon Musk",
+        "ac_type": "Gulfstream G800",
+        "category": "Don't you know who I am?",
+        "tags": "Elon Musk, SpaceX, DOGE, Toys4Billionaires",
+        "link": "https://en.wikipedia.org/wiki/Elon_Musk"
+    },
     "A835AF": {
         "registration": "N628TS",
         "operator": "Falcon Landing LLC",

backend/data/release_digests.json

@@ -0,0 +1,50 @@
+{
+  "_comment": [
+    "Baked-in SHA-256 digests for known Shadowbroker release archives.",
+    "",
+    "Issue #231: the self-updater previously skipped integrity verification",
+    "entirely whenever the MESH_UPDATE_SHA256 env var was unset (which is the",
+    "default — nothing in the install docs tells operators to set it). That",
+    "made the auto-update a supply-chain RCE on any compromise of the GitHub",
+    "release pipeline.",
+    "",
+    "The fix uses a multi-source verification chain mirroring the Tor bundle",
+    "digest approach in #201:",
+    "",
+    "  1. MESH_UPDATE_SHA256 env var (operator override, preserved)",
+    "  2. SHA256SUMS.txt asset published alongside each release (primary —",
+    "     the maintainer's release process already publishes this)",
+    "  3. This baked-in digest list (second line of defense for releases",
+    "     missing a SHA256SUMS asset, or when the asset can't be fetched)",
+    "  4. HTTPS-only fallback with a loud warning (preserves auto-update",
+    "     flow during transient outages so users don't get stuck)",
+    "",
+    "Mismatch from a source that DID respond is fatal — the update is",
+    "refused and the existing install keeps running. Only the 'no source",
+    "reachable at all' case falls back to HTTPS-only.",
+    "",
+    "Format: each entry is keyed by release tag and maps asset filenames",
+    "to their canonical SHA-256 digest (hex, lowercase). The updater",
+    "compares the locally-computed digest of the downloaded asset against",
+    "the value here.",
+    "",
+    "When the maintainer ships a new release, add its digests here BEFORE",
+    "removing the old ones so operators on the old code still validate",
+    "against the previous entries during the transition."
+  ],
+  "v0.9.79": {
+    "ShadowBroker_v0.9.79.zip": "f6877c1d66614525315ea82636ce9f7b41178332c4dbf90d27431a1ea1d9cd47",
+    "ShadowBroker_0.9.79_x64-setup.exe": "f7b676ada45cac7da05868b0a353678c9ee700e3abcf456a7c0c038c36da446f",
+    "ShadowBroker_0.9.79_x64_en-US.msi": "e0713c3cdda184cfbea750bfac0d62a35678fec00847e6476f2cac8e7e42046e"
+  },
+  "v0.9.8": {
+    "ShadowBroker_v0.9.8.zip": "183bb5cd62b9b9349d95df5ef7696cb6ca810ab4b991fa9dab6f898af4c7a175",
+    "ShadowBroker_0.9.8_x64-setup.exe": "94a0309862e9c81c92cdcbfea8eec9dbb97eef19ded82b26217b397defbc810c",
+    "ShadowBroker_0.9.8_x64_en-US.msi": "fe22f9d51e4360d74c18a7250c2fbb9ed4fa4c7a884b3ac0d04a21115466386b"
+  },
+  "v0.9.81": {
+    "ShadowBroker_v0.9.81.zip": "f81f454bdc88e9a32c351df38212b8cfa624704d65764b971bb091eef62259c6",
+    "ShadowBroker_0.9.81_x64-setup.exe": "25e9a95d0d8ce959a7d08fe8e7406772ae24b596652793e81d1de5d02510a5a6",
+    "ShadowBroker_0.9.81_x64_en-US.msi": "34e655fc0c0f195ee4ac978f228a4b2b9d5565253b8771aca9ef4693409e9e70"
+  }
+}

backend/data/tor_bundle_digests.json

@@ -0,0 +1,16 @@
+{
+  "_comment": [
+    "Pinned SHA-256 digests for the Tor Expert Bundle archives we know how to install.",
+    "Used as the LAST-RESORT verification source when the upstream .sha256sum file is",
+    "unreachable, MITM'd, or doesn't match what we downloaded. Issue #201.",
+    "",
+    "Each entry is keyed by the archive URL (so multiple platforms / versions",
+    "can share this one file) and contains the canonical SHA-256 we trust.",
+    "",
+    "When the project tests a new Tor release, add its digest here in the same",
+    "PR that bumps _TOR_EXPERT_BUNDLE_URLS. Old entries are kept indefinitely so",
+    "users on older versions keep working — we only ever ADD here, never remove."
+  ],
+  "https://dist.torproject.org/torbrowser/15.0.11/tor-expert-bundle-windows-x86_64-15.0.11.tar.gz": "PLACEHOLDER_REPLACE_BEFORE_RELEASE",
+  "https://dist.torproject.org/torbrowser/15.0.8/tor-expert-bundle-windows-x86_64-15.0.8.tar.gz": "PLACEHOLDER_REPLACE_BEFORE_RELEASE"
+}

backend/docker-entrypoint.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+set -eu
+
+# Docker named volumes hide files that were baked into /app/data at image build
+# time. Seed safe, static data into a fresh volume so first-run Docker installs
+# behave like source installs without bundling local runtime secrets.
+if [ -d /app/image-data ]; then
+  mkdir -p /app/data
+  find /app/image-data -mindepth 1 -maxdepth 1 -type f | while IFS= read -r src; do
+    dest="/app/data/$(basename "$src")"
+    if [ ! -e "$dest" ]; then
+      cp "$src" "$dest" || true
+    fi
+  done
+fi
+
+if [ -z "${PRIVACY_CORE_ALLOWED_SHA256:-}" ] && [ -f /app/libprivacy_core.so ]; then
+  PRIVACY_CORE_ALLOWED_SHA256="$(sha256sum /app/libprivacy_core.so | awk '{print $1}')"
+  export PRIVACY_CORE_ALLOWED_SHA256
+fi
+
+exec "$@"

backend/gate_sse.py

@@ -0,0 +1,11 @@
+"""gate_sse.py — DEPRECATED. Gate SSE broadcast removed in S3A.
+
+Gate activity is no longer broadcast via SSE. The frontend uses the
+authenticated poll loop for gate message refresh.
+
+Stubs are kept so any late imports do not crash at startup.
+"""
+
+
+def _broadcast_gate_events(gate_id: str, events: list[dict]) -> None:  # noqa: ARG001
+    """No-op — gate SSE broadcast removed."""

backend/limiter.py

@@ -0,0 +1,108 @@
+"""Rate-limit key function for slowapi.
+
+Issue #287 (tg12): the previous implementation used
+``slowapi.util.get_remote_address`` which only ever returns
+``request.client.host``. Behind the bundled Next.js proxy (or any other
+reverse proxy), every connected operator's ``client.host`` is the
+frontend container's bridge IP. ``@limiter.limit("120/minute")`` then
+collapses into one shared bucket for everybody on the same backend —
+one heavy tab can starve every other operator on the node.
+
+This module replaces that key function with one that:
+
+  * Reads ``X-Forwarded-For`` ONLY when the immediate peer is a trusted
+    frontend container (same allowlist used by the Docker bridge
+    local-operator trust path — see ``backend/auth.py`` ``#250``).
+  * Picks the FIRST entry in the XFF chain. That's the client end of
+    the proxy chain, which is the operator we want to bucket on.
+  * Falls back to ``request.client.host`` for any peer that isn't on
+    the trusted-frontend allowlist. Direct hits, unrelated containers,
+    and unknown hosts are bucketed exactly like before — there is no
+    way for an untrusted caller to spoof XFF and steal another
+    operator's rate-limit bucket.
+
+Single-operator nodes are unaffected: the frontend resolves to one IP,
+that IP is on the trust list, the XFF header is read, and you get one
+bucket per operator (i.e. you).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from slowapi import Limiter
+from slowapi.util import get_remote_address
+
+
+def _client_host(request: Any) -> str:
+    """Return the immediate peer's IP, normalised to a lowercase string."""
+    client = getattr(request, "client", None)
+    if client is None:
+        return ""
+    host = getattr(client, "host", "") or ""
+    return host.lower()
+
+
+def _first_forwarded_for(value: str) -> str:
+    """Return the first non-empty entry from an ``X-Forwarded-For`` header.
+
+    RFC 7239 / de-facto XFF format is ``client, proxy1, proxy2, …``. The
+    client end is what we want to bucket on. Empty parts (which appear
+    in some malformed headers) are skipped so we don't end up keying on
+    an empty string.
+    """
+    for raw in value.split(","):
+        candidate = raw.strip()
+        if candidate:
+            return candidate.lower()
+    return ""
+
+
+def _is_trusted_frontend_peer(host: str) -> bool:
+    """True iff ``host`` is one of the resolved trusted-frontend IPs.
+
+    Imported lazily so this module stays usable in unit tests that
+    don't want to pull the whole auth module into scope.
+    """
+    if not host:
+        return False
+    try:
+        from auth import _resolve_trusted_bridge_ips
+    except Exception:  # pragma: no cover - defensive
+        return False
+    try:
+        trusted_ips = _resolve_trusted_bridge_ips()
+    except Exception:  # pragma: no cover - defensive
+        return False
+    return host in trusted_ips
+
+
+def shadowbroker_rate_limit_key(request: Any) -> str:
+    """slowapi key_func that is proxy-aware on trusted frontend peers only.
+
+    Behaviour matrix:
+
+    * Direct loopback / unknown peer → ``request.client.host``
+      (identical to slowapi's default ``get_remote_address``).
+    * Peer is a trusted frontend container AND ``X-Forwarded-For`` is
+      present → first XFF entry (the actual operator).
+    * Peer is a trusted frontend container but no XFF → fall back to
+      ``request.client.host`` (the bridge IP). One shared bucket for
+      everyone in that case, same as before — but you only get there
+      if the trusted frontend forgot to forward XFF, which it won't.
+    """
+    peer = _client_host(request)
+    if _is_trusted_frontend_peer(peer):
+        headers = getattr(request, "headers", None)
+        if headers is not None:
+            xff = headers.get("x-forwarded-for") or headers.get("X-Forwarded-For")
+            if xff:
+                first = _first_forwarded_for(xff)
+                if first:
+                    return first
+    # Untrusted peer (or trusted peer without XFF): match the original
+    # get_remote_address behaviour byte-for-byte.
+    return get_remote_address(request)
+
+
+limiter = Limiter(key_func=shadowbroker_rate_limit_key)

backend/node_state.py

@@ -0,0 +1,313 @@
+"""node_state.py — Shared mutable node runtime state and node helper functions.
+
+Extracted from main.py so that background worker functions and route handlers
+can reference the same state objects without importing the full application.
+
+_NODE_SYNC_STATE is a reassignable value (SyncWorkerState is replaced whole,
+not mutated), so callers must use get_sync_state() / set_sync_state() instead
+of binding to the name at import time.
+
+All other _NODE_* objects are mutable containers (Lock, Event, dict) whose
+identity never changes; importing them directly by name is safe.
+"""
+
+import threading
+import time
+from typing import Any
+
+from services.mesh.mesh_infonet_sync_support import SyncWorkerState
+
+# ---------------------------------------------------------------------------
+# Runtime state objects
+# ---------------------------------------------------------------------------
+
+_NODE_RUNTIME_LOCK = threading.RLock()
+_NODE_SYNC_STOP = threading.Event()
+_NODE_SYNC_STATE = SyncWorkerState()
+_NODE_BOOTSTRAP_STATE: dict[str, Any] = {
+    "node_mode": "participant",
+    "manifest_loaded": False,
+    "manifest_signer_id": "",
+    "manifest_valid_until": 0,
+    "bootstrap_peer_count": 0,
+    "sync_peer_count": 0,
+    "push_peer_count": 0,
+    "operator_peer_count": 0,
+    "last_bootstrap_error": "",
+}
+_NODE_PUSH_STATE: dict[str, Any] = {
+    "last_event_id": "",
+    "last_push_ok_at": 0,
+    "last_push_error": "",
+    "last_results": [],
+}
+
+# ---------------------------------------------------------------------------
+# Getter / setter for _NODE_SYNC_STATE
+#
+# Use these instead of globals()["_NODE_SYNC_STATE"] = ... in any module that
+# imports this package.  The setter modifies *this* module's namespace so
+# subsequent get_sync_state() calls see the new value regardless of which
+# module calls set_sync_state().
+# ---------------------------------------------------------------------------
+
+def get_sync_state() -> SyncWorkerState:
+    return _NODE_SYNC_STATE
+
+
+def set_sync_state(state: SyncWorkerState) -> None:
+    global _NODE_SYNC_STATE
+    _NODE_SYNC_STATE = state
+
+
+# ---------------------------------------------------------------------------
+# Node helper functions
+#
+# These were in main.py but are needed by both route handlers and background
+# workers, so they live here to avoid circular imports.
+# ---------------------------------------------------------------------------
+
+def _current_node_mode() -> str:
+    from services.config import get_settings
+    mode = str(get_settings().MESH_NODE_MODE or "participant").strip().lower()
+    if mode not in {"participant", "relay", "perimeter"}:
+        return "participant"
+    return mode
+
+
+def _node_runtime_supported() -> bool:
+    return _current_node_mode() in {"participant", "relay"}
+
+
+def _node_activation_enabled() -> bool:
+    from services.node_settings import read_node_settings
+
+    try:
+        settings = read_node_settings()
+    except Exception:
+        return False
+    return bool(settings.get("enabled", False))
+
+
+def _participant_node_enabled() -> bool:
+    return _node_runtime_supported() and _node_activation_enabled()
+
+
+def _node_runtime_snapshot() -> dict[str, Any]:
+    with _NODE_RUNTIME_LOCK:
+        return {
+            "node_mode": _current_node_mode(),
+            "node_enabled": _participant_node_enabled(),
+            "private_transport_required": _infonet_private_transport_required(),
+            "bootstrap": {**dict(_NODE_BOOTSTRAP_STATE), "node_mode": _current_node_mode()},
+            "sync_runtime": get_sync_state().to_dict(),
+            "push_runtime": dict(_NODE_PUSH_STATE),
+        }
+
+
+def _set_node_sync_disabled_state(*, current_head: str = "") -> SyncWorkerState:
+    return SyncWorkerState(
+        current_head=str(current_head or ""),
+        last_outcome="disabled",
+    )
+
+
+def _set_participant_node_enabled(enabled: bool) -> dict[str, Any]:
+    from services.mesh.mesh_hashchain import infonet
+    from services.node_settings import write_node_settings
+
+    settings = write_node_settings(enabled=bool(enabled))
+    current_head = str(infonet.head_hash or "")
+    with _NODE_RUNTIME_LOCK:
+        _NODE_BOOTSTRAP_STATE["node_mode"] = _current_node_mode()
+        set_sync_state(
+            SyncWorkerState(current_head=current_head)
+            if bool(enabled) and _node_runtime_supported()
+            else _set_node_sync_disabled_state(current_head=current_head)
+        )
+    return {
+        **settings,
+        "node_mode": _current_node_mode(),
+        "node_enabled": _participant_node_enabled(),
+    }
+
+
+def _infonet_private_transport_required() -> bool:
+    from services.config import get_settings
+
+    return not bool(getattr(get_settings(), "MESH_INFONET_ALLOW_CLEARNET_SYNC", False))
+
+
+def _infonet_private_transport_error() -> str:
+    return "private Infonet requires onion/RNS transport; no clearnet sync fallback"
+
+
+def _is_private_infonet_transport(transport: str) -> bool:
+    return str(transport or "").strip().lower() in {"onion", "rns"}
+
+
+def _configured_bootstrap_seed_peer_urls() -> list[str]:
+    from services.config import get_settings
+    from services.mesh.mesh_router import parse_configured_relay_peers
+
+    settings = get_settings()
+    primary = str(getattr(settings, "MESH_BOOTSTRAP_SEED_PEERS", "") or "").strip()
+    legacy = str(getattr(settings, "MESH_DEFAULT_SYNC_PEERS", "") or "").strip()
+    return parse_configured_relay_peers(primary or legacy)
+
+
+def _refresh_node_peer_store(*, now: float | None = None) -> dict[str, Any]:
+    from services.config import get_settings
+    from services.mesh.mesh_bootstrap_manifest import load_bootstrap_manifest_from_settings
+    from services.mesh.mesh_peer_store import (
+        DEFAULT_PEER_STORE_PATH,
+        PeerStore,
+        make_bootstrap_peer_record,
+        make_push_peer_record,
+        make_sync_peer_record,
+    )
+    from services.mesh.mesh_router import (
+        configured_relay_peer_urls,
+        parse_configured_relay_peers,
+        peer_transport_kind,
+    )
+
+    timestamp = int(now if now is not None else time.time())
+    mode = _current_node_mode()
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        store = PeerStore(DEFAULT_PEER_STORE_PATH)
+
+    private_transport_required = _infonet_private_transport_required()
+    operator_peers = configured_relay_peer_urls()
+    bootstrap_seed_peers = _configured_bootstrap_seed_peer_urls()
+    skipped_clearnet_peers = 0
+    for peer_url in operator_peers:
+        transport = peer_transport_kind(peer_url)
+        if not transport:
+            continue
+        if private_transport_required and not _is_private_infonet_transport(transport):
+            skipped_clearnet_peers += 1
+            continue
+        store.upsert(
+            make_sync_peer_record(
+                peer_url=peer_url,
+                transport=transport,
+                role="relay",
+                source="operator",
+                now=timestamp,
+            )
+        )
+        store.upsert(
+            make_push_peer_record(
+                peer_url=peer_url,
+                transport=transport,
+                role="relay",
+                source="operator",
+                now=timestamp,
+            )
+        )
+
+    operator_peer_set = set(operator_peers)
+    for peer_url in bootstrap_seed_peers:
+        if peer_url in operator_peer_set:
+            continue
+        transport = peer_transport_kind(peer_url)
+        if not transport:
+            continue
+        if private_transport_required and not _is_private_infonet_transport(transport):
+            skipped_clearnet_peers += 1
+            continue
+        store.upsert(
+            make_bootstrap_peer_record(
+                peer_url=peer_url,
+                transport=transport,
+                role="seed",
+                label="ShadowBroker bootstrap seed",
+                signer_id="shadowbroker-bootstrap",
+                now=timestamp,
+            )
+        )
+        store.upsert(
+            make_sync_peer_record(
+                peer_url=peer_url,
+                transport=transport,
+                role="seed",
+                source="bundle",
+                label="ShadowBroker bootstrap seed",
+                signer_id="shadowbroker-bootstrap",
+                now=timestamp,
+            )
+        )
+
+    manifest = None
+    bootstrap_error = ""
+    try:
+        manifest = load_bootstrap_manifest_from_settings(now=timestamp)
+    except Exception as exc:
+        bootstrap_error = str(exc or "").strip()
+
+    if manifest is not None:
+        for peer in manifest.peers:
+            if private_transport_required and not _is_private_infonet_transport(peer.transport):
+                skipped_clearnet_peers += 1
+                continue
+            store.upsert(
+                make_bootstrap_peer_record(
+                    peer_url=peer.peer_url,
+                    transport=peer.transport,
+                    role=peer.role,
+                    label=peer.label,
+                    signer_id=manifest.signer_id,
+                    now=timestamp,
+                )
+            )
+            store.upsert(
+                make_sync_peer_record(
+                    peer_url=peer.peer_url,
+                    transport=peer.transport,
+                    role=peer.role,
+                    source="bootstrap_promoted",
+                    label=peer.label,
+                    signer_id=manifest.signer_id,
+                    now=timestamp,
+                )
+            )
+
+    if private_transport_required and skipped_clearnet_peers and not bootstrap_error:
+        bootstrap_error = _infonet_private_transport_error()
+
+    store.save()
+    bootstrap_records = store.records_for_bucket("bootstrap")
+    sync_records = store.records_for_bucket("sync")
+    push_records = store.records_for_bucket("push")
+    if private_transport_required:
+        bootstrap_records = [record for record in bootstrap_records if _is_private_infonet_transport(record.transport)]
+        sync_records = [record for record in sync_records if _is_private_infonet_transport(record.transport)]
+        push_records = [record for record in push_records if _is_private_infonet_transport(record.transport)]
+    snapshot = {
+        "node_mode": mode,
+        "private_transport_required": private_transport_required,
+        "skipped_clearnet_peer_count": skipped_clearnet_peers,
+        "manifest_loaded": manifest is not None,
+        "manifest_signer_id": manifest.signer_id if manifest is not None else "",
+        "manifest_valid_until": int(manifest.valid_until or 0) if manifest is not None else 0,
+        "bootstrap_peer_count": len(bootstrap_records),
+        "sync_peer_count": len(sync_records),
+        "push_peer_count": len(push_records),
+        "operator_peer_count": len(operator_peers),
+        "bootstrap_seed_peer_count": len(bootstrap_seed_peers),
+        "default_sync_peer_count": len(bootstrap_seed_peers),
+        "last_bootstrap_error": bootstrap_error,
+    }
+    with _NODE_RUNTIME_LOCK:
+        _NODE_BOOTSTRAP_STATE.update(snapshot)
+    return snapshot
+
+
+def _materialize_local_infonet_state() -> None:
+    from services.mesh.mesh_hashchain import infonet
+
+    infonet.ensure_materialized()

backend/pyproject.toml

@@ -1,27 +1,52 @@
+[build-system]
+requires = ["setuptools>=68.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+py-modules = []
+
 [project]
 name = "backend"
-version = "0.9.5"
+version = "0.9.81"
 requires-python = ">=3.10"
 dependencies = [
     "apscheduler==3.10.3",
+    "beautifulsoup4>=4.9.0",
     "cachetools==5.5.2",
-    "cloudscraper==1.2.71",
+    "cryptography>=41.0.0",
+    "defusedxml>=0.7.1",
     "fastapi==0.115.12",
     "feedparser==6.0.10",
     "httpx==0.28.1",
-    "playwright==1.50.0",
+    "playwright==1.59.0",
     "playwright-stealth==1.0.6",
-    "pydantic==2.11.1",
+    "pydantic==2.13.3",
     "pydantic-settings==2.8.1",
     "pystac-client==0.8.6",
-    "python-dotenv==1.0.1",
+    "python-dotenv==1.2.2",
     "requests==2.31.0",
+    "PySocks==1.7.1",
     "reverse-geocoder==1.5.1",
-    "sgp4==2.23",
+    "sgp4==2.25",
+    "meshtastic>=2.5.0",
+    "orjson>=3.10.0",
+    "paho-mqtt>=1.6.0,<2.0.0",
+    "PyNaCl>=1.5.0",
     "slowapi==0.1.9",
+    "vaderSentiment>=3.3.0",
     "uvicorn==0.34.0",
-    "yfinance==0.2.54",
+    "yfinance==1.3.0",
 ]
 
 [dependency-groups]
-test = ["pytest>=8.3.4", "pytest-asyncio==0.25.0"]
+dev = ["pytest>=8.3.4", "pytest-asyncio==0.25.0", "ruff>=0.9.0", "black>=24.0.0"]
+
+[tool.ruff.lint]
+# The current backend carries historical style debt in large legacy modules.
+# Keep CI focused on actionable correctness checks for the v0.9.81 release.
+ignore = ["E401", "E402", "E701", "E731", "E741", "F401", "F402", "F541", "F811", "F841"]
+
+[tool.black]
+# Avoid a release-time whole-backend formatting rewrite. Re-enable by narrowing
+# this once the legacy tree is formatted in a dedicated cleanup PR.
+force-exclude = ".*"

backend/routers/admin.py

@@ -0,0 +1,435 @@
+import json as json_mod
+import logging
+import os
+import threading
+from pathlib import Path
+from typing import Any
+from fastapi import APIRouter, Request, Depends, Response
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+from node_state import (
+    _current_node_mode,
+    _participant_node_enabled,
+    _refresh_node_peer_store,
+    _set_participant_node_enabled,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+class NodeSettingsUpdate(BaseModel):
+    enabled: bool
+
+
+class TimeMachineToggle(BaseModel):
+    enabled: bool
+
+
+class MeshtasticMqttUpdate(BaseModel):
+    enabled: bool | None = None
+    broker: str | None = None
+    port: int | None = None
+    username: str | None = None
+    password: str | None = None
+    psk: str | None = None
+    include_default_roots: bool | None = None
+    extra_roots: str | None = None
+    extra_topics: str | None = None
+
+
+@router.get("/api/settings/api-keys", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_get_keys(request: Request):
+    from services.api_settings import get_api_keys
+    return get_api_keys()
+
+
+@router.put("/api/settings/api-keys", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_save_keys(request: Request):
+    from services.api_settings import save_api_keys
+    body = await request.json()
+    if not isinstance(body, dict):
+        return Response(
+            content=json_mod.dumps({"ok": False, "detail": "Expected a JSON object."}),
+            status_code=400,
+            media_type="application/json",
+        )
+    result = save_api_keys({str(k): str(v) for k, v in body.items()})
+    if result.get("ok"):
+        return result
+    return Response(
+        content=json_mod.dumps(result),
+        status_code=400,
+        media_type="application/json",
+    )
+
+
+@router.get("/api/settings/api-keys/meta")
+@limiter.limit("30/minute")
+async def api_get_keys_meta(request: Request):
+    """Return absolute paths for the backend .env and .env.example template.
+
+    Not gated behind admin auth: the paths are not sensitive, and the frontend
+    needs them to render the API Keys panel banner before the user has had a
+    chance to enter an admin key. Helps users find the file when in-app editing
+    is blocked or when the backend is read-only.
+    """
+    from services.api_settings import get_env_path_info
+    return get_env_path_info()
+
+
+@router.get(
+    "/api/settings/operator-handle",
+    dependencies=[Depends(require_local_operator)],
+)
+@limiter.limit("60/minute")
+async def api_get_operator_handle(request: Request):
+    """Round 7a: return the per-install operator handle so the frontend
+    can include it in browser-direct third-party API calls (Wikipedia /
+    Wikidata via lib/wikimediaClient). The handle is auto-generated on
+    first use; operators can override it via the OPERATOR_HANDLE setting
+    or the env var of the same name.
+
+    Gated on local-operator: legitimate browser usage goes through the
+    Next.js proxy which auto-attaches the admin key; remote scanners get
+    403. The handle itself isn't a secret (it's sent to every third-party
+    API the operator touches), but admin-gating it matches the rest of
+    the settings endpoints and follows least-privilege.
+    """
+    from services.network_utils import get_operator_handle
+    return {"handle": get_operator_handle()}
+
+
+@router.get(
+    "/api/settings/news-feeds",
+    dependencies=[Depends(require_local_operator)],
+)
+@limiter.limit("30/minute")
+async def api_get_news_feeds(request: Request):
+    """Issue #252 (tg12): the curated feed inventory is configuration
+    state, not a public data feed. Gated on local-operator so the
+    Tauri shell, the Docker bridge frontend, and any caller with an
+    admin key all see the full list; anonymous LAN/internet callers
+    can no longer enumerate operator source URLs.
+    """
+    from services.news_feed_config import get_feeds
+    return get_feeds()
+
+
+@router.put("/api/settings/news-feeds", dependencies=[Depends(require_admin)])
+@limiter.limit("10/minute")
+async def api_save_news_feeds(request: Request):
+    from services.news_feed_config import save_feeds
+    body = await request.json()
+    ok = save_feeds(body)
+    if ok:
+        return {"status": "updated", "count": len(body)}
+    return Response(
+        content=json_mod.dumps({"status": "error",
+            "message": "Validation failed (max 20 feeds, each needs name/url/weight 1-5)"}),
+        status_code=400,
+        media_type="application/json",
+    )
+
+
+@router.post("/api/settings/news-feeds/reset", dependencies=[Depends(require_admin)])
+@limiter.limit("10/minute")
+async def api_reset_news_feeds(request: Request):
+    from services.news_feed_config import get_feeds, reset_feeds
+    ok = reset_feeds()
+    if ok:
+        return {"status": "reset", "feeds": get_feeds()}
+    return {"status": "error", "message": "Failed to reset feeds"}
+
+
+@router.get("/api/settings/node")
+@limiter.limit("30/minute")
+async def api_get_node_settings(request: Request):
+    """Issue #243 (tg12): node_mode and node_enabled are operational
+    posture. Anonymous callers receive an empty stub; authenticated
+    callers (local-operator or admin/scoped token) see the full
+    state. See the canonical handler in backend/main.py for the full
+    rationale.
+    """
+    import asyncio
+    from auth import _scoped_view_authenticated
+    from services.node_settings import read_node_settings
+    data = await asyncio.to_thread(read_node_settings)
+    if not _scoped_view_authenticated(request, "node"):
+        return {}
+    return {
+        **data,
+        "node_mode": _current_node_mode(),
+        "node_enabled": _participant_node_enabled(),
+    }
+
+
+@router.put("/api/settings/node", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_set_node_settings(request: Request, body: NodeSettingsUpdate):
+    _refresh_node_peer_store()
+    if bool(body.enabled):
+        try:
+            from services.transport_lane_isolation import disable_public_mesh_lane
+
+            disable_public_mesh_lane(reason="private_node_enabled")
+        except Exception as exc:
+            logger.warning("Failed to disable public Mesh while enabling private node: %s", exc)
+    result = _set_participant_node_enabled(bool(body.enabled))
+    if bool(body.enabled):
+        try:
+            import main as _main
+
+            _main._kick_public_sync_background("operator_enable")
+        except Exception:
+            logger.debug("Unable to kick Infonet sync after node enable", exc_info=True)
+    return result
+
+
+def _meshtastic_runtime_snapshot() -> dict[str, Any]:
+    from services.meshtastic_mqtt_settings import redacted_meshtastic_mqtt_settings
+    from services.sigint_bridge import sigint_grid
+
+    return {
+        **redacted_meshtastic_mqtt_settings(),
+        "runtime": sigint_grid.mesh.status(),
+    }
+
+
+@router.get("/api/settings/meshtastic-mqtt", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_get_meshtastic_mqtt_settings(request: Request):
+    return _meshtastic_runtime_snapshot()
+
+
+@router.put("/api/settings/meshtastic-mqtt", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_set_meshtastic_mqtt_settings(request: Request, body: MeshtasticMqttUpdate):
+    from services.meshtastic_mqtt_settings import write_meshtastic_mqtt_settings
+    from services.sigint_bridge import sigint_grid
+
+    updates = body.model_dump(exclude_unset=True)
+    # Empty secret fields mean "keep existing"; explicit non-empty values replace.
+    if updates.get("password") == "":
+        updates.pop("password", None)
+    if updates.get("psk") == "":
+        updates.pop("psk", None)
+
+    enabled_requested = updates.get("enabled")
+    settings = write_meshtastic_mqtt_settings(**updates)
+    if isinstance(enabled_requested, bool):
+        logger.info("Meshtastic MQTT settings update: enabled=%s", enabled_requested)
+
+    if enabled_requested is True:
+        # Public MQTT and Wormhole are intentionally mutually exclusive lanes.
+        try:
+            from services.node_settings import write_node_settings
+            from services.wormhole_settings import write_wormhole_settings
+            from services.wormhole_supervisor import disconnect_wormhole
+
+            write_wormhole_settings(enabled=False)
+            disconnect_wormhole(reason="public_mesh_enabled")
+            write_node_settings(enabled=False)
+            _set_participant_node_enabled(False)
+        except Exception as exc:
+            logger.warning("Failed to disable private mesh lane while enabling public mesh: %s", exc)
+
+    if bool(settings.get("enabled")):
+        if sigint_grid.mesh.is_running():
+            sigint_grid.mesh.stop()
+            threading.Timer(1.0, sigint_grid.mesh.start).start()
+        else:
+            sigint_grid.mesh.start()
+    else:
+        sigint_grid.mesh.stop()
+
+    return _meshtastic_runtime_snapshot()
+
+
+@router.get(
+    "/api/settings/timemachine",
+    dependencies=[Depends(require_local_operator)],
+)
+@limiter.limit("30/minute")
+async def api_get_timemachine_settings(request: Request):
+    """Issue #253 (tg12): archival-capture posture is operationally
+    sensitive — it tells a remote caller whether this deployment is
+    retaining replayable historical surveillance data. Gated on
+    local-operator so the Tauri shell and Docker bridge frontend
+    still see the toggle state, but anonymous LAN/internet callers
+    can no longer fingerprint Time Machine state.
+    """
+    import asyncio
+    from services.node_settings import read_node_settings
+    data = await asyncio.to_thread(read_node_settings)
+    return {
+        "enabled": data.get("timemachine_enabled", False),
+        "storage_warning": "Time Machine auto-snapshots use ~68 MB/day compressed (~2 GB/month). "
+                           "Snapshots capture entity positions (flights, ships, satellites) for historical playback.",
+    }
+
+
+@router.put("/api/settings/timemachine", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_set_timemachine_settings(request: Request, body: TimeMachineToggle):
+    import asyncio
+    from services.node_settings import write_node_settings
+    result = await asyncio.to_thread(write_node_settings, timemachine_enabled=body.enabled)
+    return {
+        "ok": True,
+        "enabled": result.get("timemachine_enabled", False),
+    }
+
+
+@router.post("/api/system/update", dependencies=[Depends(require_admin)])
+@limiter.limit("1/minute")
+async def system_update(request: Request):
+    """Download latest release, backup current files, extract update, and restart."""
+    from services.updater import perform_update, schedule_restart
+    candidate = Path(__file__).resolve().parent.parent.parent
+    if (candidate / "frontend").is_dir() or (candidate / "backend").is_dir():
+        project_root = str(candidate)
+    else:
+        project_root = os.getcwd()
+    result = perform_update(project_root)
+    if result.get("status") == "error":
+        return Response(content=json_mod.dumps(result), status_code=500, media_type="application/json")
+    if result.get("status") == "docker":
+        return result
+    threading.Timer(2.0, schedule_restart, args=[project_root]).start()
+    return result
+
+
+# ── Tor Hidden Service ──────────────────────────────────────────────
+
+
+@router.get("/api/settings/tor", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_tor_status(request: Request):
+    """Return Tor hidden service status and .onion address if available."""
+    import asyncio
+    from services.tor_hidden_service import tor_service
+
+    return await asyncio.to_thread(tor_service.status)
+
+
+@router.post("/api/settings/tor/start", dependencies=[Depends(require_local_operator)])
+@limiter.limit("5/minute")
+async def api_tor_start(request: Request):
+    """Start Tor and provision a hidden service for this ShadowBroker instance.
+
+    Also enables MESH_ARTI so the mesh/wormhole system can route traffic
+    through the Tor SOCKS proxy (port 9050) automatically.
+    """
+    import asyncio
+    from services.tor_hidden_service import tor_service
+
+    result = await asyncio.to_thread(tor_service.start)
+
+    # If Tor started successfully, enable Arti (Tor SOCKS proxy for mesh)
+    if result.get("ok"):
+        try:
+            from routers.ai_intel import _write_env_value
+            from services.config import get_settings
+            _write_env_value("MESH_ARTI_ENABLED", "true")
+            get_settings.cache_clear()
+        except Exception:
+            pass  # Non-fatal — hidden service still works without mesh Arti
+
+    return result
+
+
+@router.post("/api/settings/tor/reset-identity", dependencies=[Depends(require_local_operator)])
+@limiter.limit("2/minute")
+async def api_tor_reset_identity(request: Request):
+    """Destroy current .onion identity and generate a fresh one on next start.
+
+    This is irreversible — the old .onion address is permanently lost.
+    """
+    import asyncio, shutil
+    from services.tor_hidden_service import tor_service, TOR_DIR
+
+    # Stop Tor if running
+    await asyncio.to_thread(tor_service.stop)
+
+    # Delete the hidden service directory (contains the private key)
+    hs_dir = TOR_DIR / "hidden_service"
+    if hs_dir.exists():
+        shutil.rmtree(str(hs_dir), ignore_errors=True)
+
+    # Clear cached address
+    tor_service._onion_address = ""
+
+    return {"ok": True, "detail": "Tor identity destroyed. A new .onion will be generated on next start."}
+
+
+@router.post("/api/settings/agent/reset-all", dependencies=[Depends(require_local_operator)])
+@limiter.limit("2/minute")
+async def api_reset_all_agent_credentials(request: Request):
+    """Nuclear reset: regenerate HMAC key, destroy .onion, revoke agent identity.
+
+    After this, the agent is fully disconnected and needs new credentials.
+    """
+    import asyncio, secrets, shutil
+    from services.tor_hidden_service import tor_service, TOR_DIR
+    from services.config import get_settings
+
+    results = {}
+
+    # 1. Regenerate HMAC key
+    new_secret = secrets.token_hex(24)
+    from routers.ai_intel import _write_env_value
+    _write_env_value("OPENCLAW_HMAC_SECRET", new_secret)
+    results["hmac"] = "regenerated"
+
+    # 2. Revoke agent identity (Ed25519 keypair)
+    try:
+        from services.openclaw_bridge import revoke_agent_identity
+        revoke_agent_identity()
+        results["identity"] = "revoked"
+    except Exception as e:
+        results["identity"] = f"error: {e}"
+
+    # 3. Destroy .onion and restart Tor with new identity
+    await asyncio.to_thread(tor_service.stop)
+    hs_dir = TOR_DIR / "hidden_service"
+    if hs_dir.exists():
+        shutil.rmtree(str(hs_dir), ignore_errors=True)
+    tor_service._onion_address = ""
+    results["tor"] = "identity destroyed"
+
+    # 4. Bootstrap fresh identity + start Tor with new .onion
+    try:
+        from services.openclaw_bridge import generate_agent_keypair
+        keypair = generate_agent_keypair(force=True)
+        results["new_node_id"] = keypair.get("node_id", "")
+    except Exception as e:
+        results["new_node_id"] = f"error: {e}"
+
+    tor_result = await asyncio.to_thread(tor_service.start)
+    results["new_onion"] = tor_result.get("onion_address", "")
+    results["tor_ok"] = tor_result.get("ok", False)
+
+    # Clear settings cache
+    get_settings.cache_clear()
+
+    return {
+        "ok": True,
+        "hmac_regenerated": True,
+        "detail": "All agent credentials have been reset. Use the agent connection screen to generate or reveal replacement credentials.",
+        **results,
+    }
+
+
+@router.post("/api/settings/tor/stop", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_tor_stop(request: Request):
+    """Stop the Tor hidden service."""
+    import asyncio
+    from services.tor_hidden_service import tor_service
+
+    return await asyncio.to_thread(tor_service.stop)

backend/routers/cctv.py

@@ -0,0 +1,367 @@
+import logging
+from dataclasses import dataclass, field
+from fastapi import APIRouter, Request, Query, HTTPException
+from fastapi.responses import StreamingResponse
+from starlette.background import BackgroundTask
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+_CCTV_PROXY_CONNECT_TIMEOUT_S = 2.0
+
+_CCTV_PROXY_ALLOWED_HOSTS = {
+    "s3-eu-west-1.amazonaws.com",
+    "jamcams.tfl.gov.uk",
+    "images.data.gov.sg",
+    "cctv.austinmobility.io",
+    "webcams.nyctmc.org",
+    "cwwp2.dot.ca.gov",
+    "wzmedia.dot.ca.gov",
+    "images.wsdot.wa.gov",
+    "olypen.com",
+    "flyykm.com",
+    "cam.pangbornairport.com",
+    "navigator-c2c.dot.ga.gov",
+    "navigator-c2c.ga.gov",
+    "navigator-csc.dot.ga.gov",
+    "vss1live.dot.ga.gov",
+    "vss2live.dot.ga.gov",
+    "vss3live.dot.ga.gov",
+    "vss4live.dot.ga.gov",
+    "vss5live.dot.ga.gov",
+    "511ga.org",
+    "gettingaroundillinois.com",
+    "cctv.travelmidwest.com",
+    "mdotjboss.state.mi.us",
+    "micamerasimages.net",
+    "publicstreamer1.cotrip.org",
+    "publicstreamer2.cotrip.org",
+    "publicstreamer3.cotrip.org",
+    "publicstreamer4.cotrip.org",
+    "cocam.carsprogram.org",
+    "tripcheck.com",
+    "www.tripcheck.com",
+    "infocar.dgt.es",
+    "informo.madrid.es",
+    "www.windy.com",
+    "imgproxy.windy.com",
+    "www.lakecountypassage.com",
+    "webcam.forkswa.com",
+    "webcam.sunmountainlodge.com",
+    "www.nps.gov",
+    "home.lewiscounty.com",
+    "www.seattle.gov",
+}
+
+
+@dataclass(frozen=True)
+class _CCTVProxyProfile:
+    name: str
+    timeout: tuple = (_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0)
+    cache_seconds: int = 30
+    headers: dict = field(default_factory=dict)
+
+
+def _cctv_host_allowed(hostname) -> bool:
+    host = str(hostname or "").strip().lower()
+    if not host:
+        return False
+    for allowed in _CCTV_PROXY_ALLOWED_HOSTS:
+        normalized = str(allowed or "").strip().lower()
+        if host == normalized or host.endswith(f".{normalized}"):
+            return True
+    return False
+
+
+def _proxied_cctv_url(target_url: str) -> str:
+    from urllib.parse import quote
+    return f"/api/cctv/media?url={quote(target_url, safe='')}"
+
+
+def _cctv_proxy_profile_for_url(target_url: str) -> _CCTVProxyProfile:
+    from urllib.parse import urlparse
+    parsed = urlparse(target_url)
+    host = str(parsed.hostname or "").strip().lower()
+    path = str(parsed.path or "").strip().lower()
+
+    if host in {"jamcams.tfl.gov.uk", "s3-eu-west-1.amazonaws.com"}:
+        return _CCTVProxyProfile(name="tfl-jamcam", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 20.0), cache_seconds=15,
+            headers={"Accept": "video/mp4,image/avif,image/webp,image/apng,image/*,*/*;q=0.8", "Referer": "https://tfl.gov.uk/"})
+    if host == "images.data.gov.sg":
+        return _CCTVProxyProfile(name="lta-singapore", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host == "cctv.austinmobility.io":
+        return _CCTVProxyProfile(name="austin-mobility", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://data.mobility.austin.gov/", "Origin": "https://data.mobility.austin.gov"})
+    if host == "webcams.nyctmc.org":
+        return _CCTVProxyProfile(name="nyc-dot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host in {"cwwp2.dot.ca.gov", "wzmedia.dot.ca.gov"}:
+        return _CCTVProxyProfile(name="caltrans", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=15,
+            headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,image/*,*/*;q=0.8",
+                     "Referer": "https://cwwp2.dot.ca.gov/"})
+    if host in {"images.wsdot.wa.gov", "olypen.com", "flyykm.com", "cam.pangbornairport.com"}:
+        return _CCTVProxyProfile(name="wsdot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host in {"www.lakecountypassage.com", "webcam.forkswa.com", "webcam.sunmountainlodge.com", "home.lewiscounty.com", "www.seattle.gov"}:
+        return _CCTVProxyProfile(name="regional-cctv-image", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=45,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": f"https://{host}/"})
+    if host == "www.nps.gov":
+        return _CCTVProxyProfile(name="nps-webcam", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.nps.gov/"})
+    if host in {"navigator-c2c.dot.ga.gov", "navigator-c2c.ga.gov", "navigator-csc.dot.ga.gov"}:
+        read_timeout = 18.0 if "/snapshots/" in path else 12.0
+        return _CCTVProxyProfile(name="gdot-snapshot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, read_timeout), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "http://navigator-c2c.dot.ga.gov/"})
+    if host == "511ga.org":
+        return _CCTVProxyProfile(name="gdot-511ga-image", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=15,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://511ga.org/cctv"})
+    if host.startswith("vss") and host.endswith("dot.ga.gov"):
+        return _CCTVProxyProfile(name="gdot-hls", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 20.0), cache_seconds=10,
+            headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,*/*;q=0.8",
+                     "Referer": "http://navigator-c2c.dot.ga.gov/"})
+    if host in {"gettingaroundillinois.com", "cctv.travelmidwest.com"}:
+        return _CCTVProxyProfile(name="illinois-dot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host in {"mdotjboss.state.mi.us", "micamerasimages.net"}:
+        return _CCTVProxyProfile(name="michigan-dot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://mdotjboss.state.mi.us/"})
+    if host in {"publicstreamer1.cotrip.org", "publicstreamer2.cotrip.org",
+                "publicstreamer3.cotrip.org", "publicstreamer4.cotrip.org"}:
+        return _CCTVProxyProfile(name="cotrip-hls", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 20.0), cache_seconds=10,
+            headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,*/*;q=0.8",
+                     "Referer": "https://www.cotrip.org/"})
+    if host == "cocam.carsprogram.org":
+        return _CCTVProxyProfile(name="cotrip-preview", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=20,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.cotrip.org/"})
+    if host in {"tripcheck.com", "www.tripcheck.com"}:
+        return _CCTVProxyProfile(name="odot-tripcheck", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
+    if host == "infocar.dgt.es":
+        return _CCTVProxyProfile(name="dgt-spain", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://infocar.dgt.es/"})
+    if host == "informo.madrid.es":
+        return _CCTVProxyProfile(name="madrid-city", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://informo.madrid.es/"})
+    if host in {"www.windy.com", "imgproxy.windy.com"}:
+        return _CCTVProxyProfile(name="windy-webcams", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.windy.com/"})
+    return _CCTVProxyProfile(name="generic-cctv", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0), cache_seconds=30,
+        headers={"Accept": "*/*"})
+
+
+def _cctv_upstream_headers(request: Request, profile: _CCTVProxyProfile) -> dict:
+    # Round 7a: per-install operator handle. Mozilla/5.0 prefix retained
+    # because many CCTV endpoints sniff for a browser-like prefix.
+    from services.network_utils import outbound_user_agent
+    headers = {
+        "User-Agent": f"Mozilla/5.0 (compatible; {outbound_user_agent('cctv-proxy')})",
+        **profile.headers,
+    }
+    range_header = request.headers.get("range")
+    if range_header:
+        headers["Range"] = range_header
+    if_none_match = request.headers.get("if-none-match")
+    if if_none_match:
+        headers["If-None-Match"] = if_none_match
+    if_modified_since = request.headers.get("if-modified-since")
+    if if_modified_since:
+        headers["If-Modified-Since"] = if_modified_since
+    return headers
+
+
+def _cctv_response_headers(resp, cache_seconds: int, include_length: bool = True) -> dict:
+    headers = {"Cache-Control": f"public, max-age={cache_seconds}", "Access-Control-Allow-Origin": "*"}
+    for key in ("Accept-Ranges", "Content-Range", "ETag", "Last-Modified"):
+        value = resp.headers.get(key)
+        if value:
+            headers[key] = value
+    if include_length:
+        content_length = resp.headers.get("Content-Length")
+        if content_length:
+            headers["Content-Length"] = content_length
+    return headers
+
+
+# Maximum number of redirects we'll follow on the CCTV upstream. Each hop is
+# re-validated against _cctv_host_allowed() before continuing, so this caps
+# the redirect-chain SSRF blast radius.
+_CCTV_MAX_REDIRECTS = 5
+
+
+def _fetch_cctv_upstream_response(request: Request, target_url: str, profile: _CCTVProxyProfile):
+    """Fetch an upstream CCTV URL, following redirects manually with host re-validation.
+
+    Why manual redirect following:
+      The original code used ``allow_redirects=True``, which only validated
+      the initial caller-supplied URL host against the allowlist. An attacker
+      could submit an allowed host that 302-redirected to an internal address
+      (e.g. ``http://localhost:8000/api/...`` or a private RFC1918 range),
+      and the backend would dutifully follow and proxy the response — a
+      classic open-redirect-to-SSRF chain.
+
+      With this loop, we re-run ``_cctv_host_allowed()`` on every hop's
+      ``Location`` header. A redirect to a host that isn't on the allowlist
+      is rejected with 502 rather than silently followed.
+    """
+    import requests as _req
+    from urllib.parse import urlparse, urljoin
+
+    headers = _cctv_upstream_headers(request, profile)
+    current_url = target_url
+    hops = 0
+    try:
+        while True:
+            resp = _req.get(
+                current_url,
+                timeout=profile.timeout,
+                stream=True,
+                allow_redirects=False,
+                headers=headers,
+            )
+            # Redirect handling — re-validate the next-hop host before following.
+            if resp.is_redirect or resp.status_code in (301, 302, 303, 307, 308):
+                location = resp.headers.get("Location", "")
+                resp.close()
+                if hops >= _CCTV_MAX_REDIRECTS:
+                    logger.warning(
+                        "CCTV upstream redirect chain exceeded limit [%s] %s",
+                        profile.name, target_url,
+                    )
+                    raise HTTPException(status_code=502, detail="Upstream redirect chain too long")
+                if not location:
+                    raise HTTPException(status_code=502, detail="Upstream redirect missing Location")
+                next_url = urljoin(current_url, location)
+                next_parsed = urlparse(next_url)
+                if next_parsed.scheme not in ("http", "https"):
+                    raise HTTPException(status_code=502, detail="Upstream redirect to non-HTTP scheme")
+                if not _cctv_host_allowed(next_parsed.hostname):
+                    logger.warning(
+                        "CCTV upstream redirect to disallowed host [%s] %s -> %s",
+                        profile.name, current_url, next_url,
+                    )
+                    raise HTTPException(status_code=502, detail="Upstream redirect to disallowed host")
+                current_url = next_url
+                hops += 1
+                continue
+            break
+    except _req.exceptions.Timeout as exc:
+        logger.warning("CCTV upstream timeout [%s] %s", profile.name, target_url)
+        raise HTTPException(status_code=504, detail="Upstream timeout") from exc
+    except _req.exceptions.RequestException as exc:
+        logger.warning("CCTV upstream request failure [%s] %s: %s", profile.name, target_url, exc)
+        raise HTTPException(status_code=502, detail="Upstream fetch failed") from exc
+    if resp.status_code >= 400:
+        logger.info("CCTV upstream HTTP %s [%s] %s", resp.status_code, profile.name, target_url)
+        resp.close()
+        raise HTTPException(status_code=int(resp.status_code), detail=f"Upstream returned {resp.status_code}")
+    return resp
+
+
+def _rewrite_cctv_hls_playlist(base_url: str, body: str) -> str:
+    import re
+    from urllib.parse import urljoin, urlparse
+
+    def _rewrite_target(target: str) -> str:
+        candidate = str(target or "").strip()
+        if not candidate or candidate.startswith("data:"):
+            return candidate
+        absolute = urljoin(base_url, candidate)
+        parsed_target = urlparse(absolute)
+        if parsed_target.scheme not in ("http", "https"):
+            return candidate
+        if not _cctv_host_allowed(parsed_target.hostname):
+            return candidate
+        return _proxied_cctv_url(absolute)
+
+    rewritten_lines: list = []
+    for raw_line in body.splitlines():
+        stripped = raw_line.strip()
+        if not stripped:
+            rewritten_lines.append(raw_line)
+            continue
+        if stripped.startswith("#"):
+            rewritten_lines.append(re.sub(r'URI="([^"]+)"',
+                lambda match: f'URI="{_rewrite_target(match.group(1))}"', raw_line))
+            continue
+        rewritten_lines.append(_rewrite_target(stripped))
+    return "\n".join(rewritten_lines) + ("\n" if body.endswith("\n") else "")
+
+
+def _infer_cctv_media_type_from_url(target_url: str, content_type: str) -> str:
+    from urllib.parse import urlparse
+
+    clean_type = str(content_type or "").split(";", 1)[0].strip().lower()
+    if clean_type and clean_type not in {"application/octet-stream", "binary/octet-stream"}:
+        return content_type
+    path = str(urlparse(target_url).path or "").lower()
+    if path.endswith((".jpg", ".jpeg")):
+        return "image/jpeg"
+    if path.endswith(".png"):
+        return "image/png"
+    if path.endswith(".webp"):
+        return "image/webp"
+    if path.endswith(".gif"):
+        return "image/gif"
+    if path.endswith(".mp4"):
+        return "video/mp4"
+    if path.endswith((".m3u8", ".m3u")):
+        return "application/vnd.apple.mpegurl"
+    if path.endswith((".mjpg", ".mjpeg")):
+        return "multipart/x-mixed-replace"
+    return content_type or "application/octet-stream"
+
+
+def _proxy_cctv_media_response(request: Request, target_url: str):
+    from urllib.parse import urlparse
+    from fastapi.responses import Response
+    parsed = urlparse(target_url)
+    profile = _cctv_proxy_profile_for_url(target_url)
+    resp = _fetch_cctv_upstream_response(request, target_url, profile)
+    content_type = _infer_cctv_media_type_from_url(
+        target_url,
+        resp.headers.get("Content-Type", "application/octet-stream"),
+    )
+    is_hls_playlist = (
+        ".m3u8" in str(parsed.path or "").lower()
+        or "mpegurl" in content_type.lower()
+        or "vnd.apple.mpegurl" in content_type.lower()
+    )
+    if is_hls_playlist:
+        body = resp.text
+        if "#EXTM3U" in body:
+            body = _rewrite_cctv_hls_playlist(target_url, body)
+        resp.close()
+        return Response(content=body, media_type=content_type,
+            headers=_cctv_response_headers(resp, cache_seconds=profile.cache_seconds, include_length=False))
+    return StreamingResponse(resp.iter_content(chunk_size=65536), status_code=resp.status_code,
+        media_type=content_type,
+        headers=_cctv_response_headers(resp, cache_seconds=profile.cache_seconds),
+        background=BackgroundTask(resp.close))
+
+
+@router.get("/api/cctv/media")
+@limiter.limit("120/minute")
+async def cctv_media_proxy(request: Request, url: str = Query(...)):
+    """Proxy CCTV media through the backend to bypass browser CORS restrictions."""
+    from urllib.parse import urlparse
+    parsed = urlparse(url)
+    if not _cctv_host_allowed(parsed.hostname):
+        raise HTTPException(status_code=403, detail="Host not allowed")
+    if parsed.scheme not in ("http", "https"):
+        raise HTTPException(status_code=400, detail="Invalid scheme")
+    return _proxy_cctv_media_response(request, url)

backend/routers/data.py

@@ -0,0 +1,745 @@
+import asyncio
+import logging
+import math
+import threading
+from typing import Any
+from fastapi import APIRouter, Request, Response, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+from services.data_fetcher import get_latest_data, update_all_data
+import orjson
+import json as json_mod
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+_refresh_lock = threading.Lock()
+
+
+class ViewportUpdate(BaseModel):
+    s: float
+    w: float
+    n: float
+    e: float
+
+
+class LayerUpdate(BaseModel):
+    layers: dict[str, bool]
+
+
+_LAST_VIEWPORT_UPDATE: tuple | None = None
+_LAST_VIEWPORT_UPDATE_TS = 0.0
+_VIEWPORT_UPDATE_LOCK = threading.Lock()
+_VIEWPORT_DEDUPE_EPSILON = 1.0
+_VIEWPORT_MIN_UPDATE_S = 10.0
+
+
+def _normalize_longitude(value: float) -> float:
+    normalized = ((value + 180.0) % 360.0 + 360.0) % 360.0 - 180.0
+    if normalized == -180.0 and value > 0:
+        return 180.0
+    return normalized
+
+
+def _normalize_viewport_bounds(s: float, w: float, n: float, e: float) -> tuple:
+    south = max(-90.0, min(90.0, s))
+    north = max(-90.0, min(90.0, n))
+    raw_width = abs(e - w)
+    if not math.isfinite(raw_width) or raw_width >= 360.0:
+        return south, -180.0, north, 180.0
+    west = _normalize_longitude(w)
+    east = _normalize_longitude(e)
+    if east < west:
+        return south, -180.0, north, 180.0
+    return south, west, north, east
+
+
+def _viewport_changed_enough(bounds: tuple) -> bool:
+    global _LAST_VIEWPORT_UPDATE, _LAST_VIEWPORT_UPDATE_TS
+    import time
+    now = time.monotonic()
+    with _VIEWPORT_UPDATE_LOCK:
+        if _LAST_VIEWPORT_UPDATE is None:
+            _LAST_VIEWPORT_UPDATE = bounds
+            _LAST_VIEWPORT_UPDATE_TS = now
+            return True
+        changed = any(
+            abs(current - previous) > _VIEWPORT_DEDUPE_EPSILON
+            for current, previous in zip(bounds, _LAST_VIEWPORT_UPDATE)
+        )
+        if not changed and (now - _LAST_VIEWPORT_UPDATE_TS) < _VIEWPORT_MIN_UPDATE_S:
+            return False
+        if (now - _LAST_VIEWPORT_UPDATE_TS) < _VIEWPORT_MIN_UPDATE_S:
+            return False
+        _LAST_VIEWPORT_UPDATE = bounds
+        _LAST_VIEWPORT_UPDATE_TS = now
+        return True
+
+
+def _queue_viirs_change_refresh() -> None:
+    from services.fetchers.earth_observation import fetch_viirs_change_nodes
+    threading.Thread(target=fetch_viirs_change_nodes, daemon=True).start()
+
+
+def _etag_response(request: Request, payload: dict, prefix: str = "", default=None):
+    etag = _current_etag(prefix)
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    content = json_mod.dumps(_json_safe(payload), default=default, allow_nan=False)
+    return Response(content=content, media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"})
+
+
+def _current_etag(prefix: str = "") -> str:
+    from services.fetchers._store import get_active_layers_version, get_data_version
+    return f"{prefix}v{get_data_version()}-l{get_active_layers_version()}"
+
+
+# ── Issue #288: viewport-aware payloads ─────────────────────────────────────
+# Heavy, density-driven, time-sensitive layers that benefit from bbox
+# filtering. Light reference layers (datacenters, military_bases,
+# power_plants, satellites, weather, news, etc.) are intentionally NOT
+# in these sets — they ship world-scale even when bounds are supplied so
+# panning never reveals an "empty world" of static infrastructure.
+#
+# When the caller does NOT pass s/w/n/e, none of this runs and the response
+# is byte-for-byte identical to the pre-#288 behavior.
+_FAST_BBOX_HEAVY_KEYS: tuple[str, ...] = (
+    "commercial_flights",
+    "military_flights",
+    "private_flights",
+    "private_jets",
+    "tracked_flights",
+    "ships",
+    "cctv",
+    "uavs",
+    "liveuamap",
+    "gps_jamming",
+    "sigint",
+    "trains",
+)
+_SLOW_BBOX_HEAVY_KEYS: tuple[str, ...] = (
+    "gdelt",
+    "firms_fires",
+    "kiwisdr",
+    "scanners",
+    "psk_reporter",
+)
+
+
+def _has_full_bbox(s, w, n, e) -> bool:
+    return None not in (s, w, n, e)
+
+
+def _bbox_etag_suffix(s, w, n, e) -> str:
+    """Quantize bbox to 1° before mixing into the ETag.
+
+    The 20% padding inside _bbox_filter already absorbs sub-degree pans;
+    quantizing here means small mouse drags don't blow the ETag cache
+    on the client. Full-world bounds collapse to a single suffix.
+    """
+    if not _has_full_bbox(s, w, n, e):
+        return ""
+    try:
+        ss = math.floor(float(s))
+        ww = math.floor(float(w))
+        nn = math.ceil(float(n))
+        ee = math.ceil(float(e))
+    except (TypeError, ValueError):
+        return ""
+    # If the requested window covers basically the whole world, treat it as
+    # "no bbox" for caching purposes so world-zoomed clients all hit the
+    # same ETag and benefit from the existing 304 path.
+    lat_span, lng_span = _bbox_spans(s, w, n, e)
+    if lng_span >= 300 or lat_span >= 120:
+        return ""
+    return f"|bbox={ss},{ww},{nn},{ee}"
+
+
+def _apply_bbox_to_payload(payload: dict, heavy_keys: tuple[str, ...],
+                            s: float, w: float, n: float, e: float) -> dict:
+    """In-place filter the heavy-key collections in *payload* to a viewport.
+
+    Items without lat/lng are passed through (so e.g. summary blobs aren't
+    accidentally dropped). The existing _bbox_filter helper applies a 20%
+    pad and handles antimeridian crossings.
+    """
+    lat_span, lng_span = _bbox_spans(s, w, n, e)
+    # World-scale request → skip filtering entirely. Spares the CPU and
+    # guarantees the response matches the no-params shape.
+    if lng_span >= 300 or lat_span >= 120:
+        return payload
+    for key in heavy_keys:
+        items = payload.get(key)
+        if not isinstance(items, list) or not items:
+            continue
+        payload[key] = _bbox_filter(items, s, w, n, e)
+    return payload
+
+
+def _json_safe(value):
+    if isinstance(value, float):
+        return value if math.isfinite(value) else None
+    if isinstance(value, dict):
+        return {k: _json_safe(v) for k, v in list(value.items())}
+    if isinstance(value, list):
+        return [_json_safe(v) for v in list(value)]
+    if isinstance(value, tuple):
+        return [_json_safe(v) for v in list(value)]
+    return value
+
+
+def _sanitize_payload(value):
+    if isinstance(value, float):
+        return value if math.isfinite(value) else None
+    if isinstance(value, dict):
+        return {k: _sanitize_payload(v) for k, v in list(value.items())}
+    if isinstance(value, (list, tuple)):
+        return list(value)
+    return value
+
+
+def _bbox_filter(items: list, s: float, w: float, n: float, e: float,
+                 lat_key: str = "lat", lng_key: str = "lng") -> list:
+    pad_lat = (n - s) * 0.2
+    pad_lng = (e - w) * 0.2 if e > w else ((e + 360 - w) * 0.2)
+    s2, n2 = s - pad_lat, n + pad_lat
+    w2, e2 = w - pad_lng, e + pad_lng
+    crosses_antimeridian = w2 > e2
+    out = []
+    for item in items:
+        lat = item.get(lat_key)
+        lng = item.get(lng_key)
+        if lat is None or lng is None:
+            out.append(item)
+            continue
+        if not (s2 <= lat <= n2):
+            continue
+        if crosses_antimeridian:
+            if lng >= w2 or lng <= e2:
+                out.append(item)
+        else:
+            if w2 <= lng <= e2:
+                out.append(item)
+    return out
+
+
+def _bbox_filter_geojson_points(items: list, s: float, w: float, n: float, e: float) -> list:
+    pad_lat = (n - s) * 0.2
+    pad_lng = (e - w) * 0.2 if e > w else ((e + 360 - w) * 0.2)
+    s2, n2 = s - pad_lat, n + pad_lat
+    w2, e2 = w - pad_lng, e + pad_lng
+    crosses_antimeridian = w2 > e2
+    out = []
+    for item in items:
+        geometry = item.get("geometry") if isinstance(item, dict) else None
+        coords = geometry.get("coordinates") if isinstance(geometry, dict) else None
+        if not isinstance(coords, (list, tuple)) or len(coords) < 2:
+            out.append(item)
+            continue
+        lng, lat = coords[0], coords[1]
+        if lat is None or lng is None:
+            out.append(item)
+            continue
+        if not (s2 <= lat <= n2):
+            continue
+        if crosses_antimeridian:
+            if lng >= w2 or lng <= e2:
+                out.append(item)
+        else:
+            if w2 <= lng <= e2:
+                out.append(item)
+    return out
+
+
+def _bbox_spans(s, w, n, e) -> tuple:
+    if None in (s, w, n, e):
+        return 180.0, 360.0
+    lat_span = max(0.0, float(n) - float(s))
+    lng_span = float(e) - float(w)
+    if lng_span < 0:
+        lng_span += 360.0
+    if lng_span == 0 and w == -180 and e == 180:
+        lng_span = 360.0
+    return lat_span, max(0.0, lng_span)
+
+
+def _cap_startup_items(items: list | None, max_items: int) -> list:
+    if not items:
+        return []
+    if len(items) <= max_items:
+        return items
+    return items[:max_items]
+
+
+def _cap_fast_startup_payload(payload: dict) -> dict:
+    capped = dict(payload)
+    capped["commercial_flights"] = _cap_startup_items(capped.get("commercial_flights"), 800)
+    capped["private_flights"] = _cap_startup_items(capped.get("private_flights"), 300)
+    capped["private_jets"] = _cap_startup_items(capped.get("private_jets"), 150)
+    capped["ships"] = _cap_startup_items(capped.get("ships"), 1500)
+    capped["cctv"] = []
+    capped["sigint"] = _cap_startup_items(capped.get("sigint"), 500)
+    capped["trains"] = _cap_startup_items(capped.get("trains"), 100)
+    capped["startup_payload"] = True
+    return capped
+
+
+def _cap_fast_dashboard_payload(payload: dict) -> dict:
+    return payload
+
+
+def _world_and_continental_scale(has_bbox: bool, s, w, n, e) -> tuple:
+    lat_span, lng_span = _bbox_spans(s, w, n, e)
+    world_scale = (not has_bbox) or lng_span >= 300 or lat_span >= 120
+    continental_scale = has_bbox and not world_scale and (lng_span >= 120 or lat_span >= 55)
+    return world_scale, continental_scale
+
+
+def _filter_sigint_by_layers(items: list, active_layers: dict) -> list:
+    allow_aprs = bool(active_layers.get("sigint_aprs", True))
+    allow_mesh = bool(active_layers.get("sigint_meshtastic", True))
+    if allow_aprs and allow_mesh:
+        return items
+    allowed_sources: set = {"js8call"}
+    if allow_aprs:
+        allowed_sources.add("aprs")
+    if allow_mesh:
+        allowed_sources.update({"meshtastic", "meshtastic-map"})
+    return [item for item in items if str(item.get("source") or "").lower() in allowed_sources]
+
+
+def _sigint_totals_for_items(items: list) -> dict:
+    totals = {"total": len(items), "meshtastic": 0, "meshtastic_live": 0, "meshtastic_map": 0,
+              "aprs": 0, "js8call": 0}
+    for item in items:
+        source = str(item.get("source") or "").lower()
+        if source == "meshtastic":
+            totals["meshtastic"] += 1
+            if bool(item.get("from_api")):
+                totals["meshtastic_map"] += 1
+            else:
+                totals["meshtastic_live"] += 1
+        elif source == "aprs":
+            totals["aprs"] += 1
+        elif source == "js8call":
+            totals["js8call"] += 1
+    return totals
+
+
+@router.get("/api/refresh", dependencies=[Depends(require_admin)])
+@limiter.limit("2/minute")
+async def force_refresh(request: Request):
+    from services.schemas import RefreshResponse
+    if not _refresh_lock.acquire(blocking=False):
+        return {"status": "refresh already in progress"}
+
+    def _do_refresh():
+        try:
+            update_all_data()
+        finally:
+            _refresh_lock.release()
+
+    t = threading.Thread(target=_do_refresh)
+    t.start()
+    return {"status": "refreshing in background"}
+
+
+@router.post("/api/ais/feed", dependencies=[Depends(require_local_operator)])
+@limiter.limit("60/minute")
+async def ais_feed(request: Request):
+    """Accept AIS-catcher HTTP JSON feed (POST decoded AIS messages)."""
+    from services.ais_stream import ingest_ais_catcher
+    try:
+        body = await request.json()
+    except Exception:
+        return JSONResponse(status_code=422, content={"ok": False, "detail": "invalid JSON body"})
+    msgs = body.get("msgs", [])
+    if not msgs:
+        return {"status": "ok", "ingested": 0}
+    count = ingest_ais_catcher(msgs)
+    return {"status": "ok", "ingested": count}
+
+
+@router.get("/api/trail/flight/{icao24}")
+@limiter.limit("120/minute")
+async def get_selected_flight_trail(icao24: str, request: Request):  # noqa: ARG001
+    from services.fetchers.flights import get_flight_trail
+    return {"id": icao24, "trail": get_flight_trail(icao24)}
+
+
+@router.get("/api/trail/ship/{mmsi}")
+@limiter.limit("120/minute")
+async def get_selected_ship_trail(mmsi: int, request: Request):  # noqa: ARG001
+    from services.ais_stream import get_vessel_trail
+    return {"id": mmsi, "trail": get_vessel_trail(mmsi)}
+
+
+@router.post("/api/viewport")
+@limiter.limit("60/minute")
+async def update_viewport(vp: ViewportUpdate, request: Request):  # noqa: ARG001
+    """Receive frontend map bounds. AIS stream stays global so open-ocean
+    vessels are never dropped — the frontend worker handles viewport culling."""
+    return {"status": "ok"}
+
+
+@router.post("/api/layers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def update_layers(update: LayerUpdate, request: Request):
+    """Receive frontend layer toggle state. Starts/stops streams accordingly."""
+    from services.fetchers._store import active_layers, bump_active_layers_version, is_any_active
+    old_ships = is_any_active("ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts")
+    old_mesh = is_any_active("sigint_meshtastic")
+    old_aprs = is_any_active("sigint_aprs")
+    old_viirs = is_any_active("viirs_nightlights")
+    changed = False
+    for key, value in update.layers.items():
+        if key in active_layers:
+            if active_layers[key] != value:
+                changed = True
+            active_layers[key] = value
+    if changed:
+        bump_active_layers_version()
+    new_ships = is_any_active("ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts")
+    new_mesh = is_any_active("sigint_meshtastic")
+    new_aprs = is_any_active("sigint_aprs")
+    new_viirs = is_any_active("viirs_nightlights")
+    if old_ships and not new_ships:
+        from services.ais_stream import stop_ais_stream
+        stop_ais_stream()
+        logger.info("AIS stream stopped (all ship layers disabled)")
+    elif not old_ships and new_ships:
+        from services.ais_stream import start_ais_stream
+        start_ais_stream()
+        logger.info("AIS stream started (ship layer enabled)")
+    from services.sigint_bridge import sigint_grid
+    if old_mesh and not new_mesh:
+        try:
+            from services.meshtastic_mqtt_settings import mqtt_bridge_enabled
+            keep_chat_running = mqtt_bridge_enabled()
+        except Exception:
+            keep_chat_running = False
+        if keep_chat_running:
+            logger.info("Meshtastic map layer disabled; MQTT bridge kept running for MeshChat")
+        else:
+            sigint_grid.mesh.stop()
+            logger.info("Meshtastic MQTT bridge stopped (layer disabled)")
+    elif not old_mesh and new_mesh:
+        try:
+            from services.meshtastic_mqtt_settings import mqtt_bridge_enabled
+            mqtt_enabled = mqtt_bridge_enabled()
+        except Exception:
+            mqtt_enabled = False
+        if mqtt_enabled:
+            sigint_grid.mesh.start()
+            logger.info("Meshtastic MQTT bridge started (layer enabled)")
+        else:
+            logger.info(
+                "Meshtastic layer enabled; MQTT bridge remains disabled "
+                "(set MESH_MQTT_ENABLED=true to participate in the public broker)"
+            )
+    if old_aprs and not new_aprs:
+        sigint_grid.aprs.stop()
+        logger.info("APRS bridge stopped (layer disabled)")
+    elif not old_aprs and new_aprs:
+        sigint_grid.aprs.start()
+        logger.info("APRS bridge started (layer enabled)")
+    if not old_viirs and new_viirs:
+        _queue_viirs_change_refresh()
+        logger.info("VIIRS change refresh queued (layer enabled)")
+    return {"status": "ok"}
+
+
+@router.get("/api/live-data")
+@limiter.limit("120/minute")
+async def live_data(request: Request):
+    return get_latest_data()
+
+
+@router.get("/api/bootstrap/critical")
+@limiter.limit("180/minute")
+async def bootstrap_critical(request: Request):
+    """Cached first-paint payload for the dashboard.
+
+    This endpoint is intentionally memory-only: no upstream calls, no refresh,
+    and a bounded response. It exists so the map and threat feed can paint
+    before slower panels and background enrichers finish warming up.
+    """
+    etag = _current_etag(prefix="bootstrap|critical|")
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    from services.fetchers._store import (
+        active_layers,
+        get_latest_data_subset_refs,
+        get_source_timestamps_snapshot,
+    )
+
+    d = get_latest_data_subset_refs(
+        "last_updated", "commercial_flights", "military_flights", "private_flights",
+        "private_jets", "tracked_flights", "ships", "uavs", "liveuamap", "gps_jamming",
+        "satellites", "satellite_source", "satellite_analysis", "sigint", "sigint_totals",
+        "trains", "news", "gdelt", "airports", "threat_level", "trending_markets",
+        "correlations", "fimi", "crowdthreat",
+    )
+    freshness = get_source_timestamps_snapshot()
+    ships_enabled = any(active_layers.get(key, True) for key in (
+        "ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts"))
+    sigint_items = _filter_sigint_by_layers(d.get("sigint") or [], active_layers)
+    payload = {
+        "last_updated": d.get("last_updated"),
+        "commercial_flights": _cap_startup_items(
+            (d.get("commercial_flights") or []) if active_layers.get("flights", True) else [],
+            800,
+        ),
+        "military_flights": _cap_startup_items(
+            (d.get("military_flights") or []) if active_layers.get("military", True) else [],
+            300,
+        ),
+        "private_flights": _cap_startup_items(
+            (d.get("private_flights") or []) if active_layers.get("private", True) else [],
+            300,
+        ),
+        "private_jets": _cap_startup_items(
+            (d.get("private_jets") or []) if active_layers.get("jets", True) else [],
+            150,
+        ),
+        "tracked_flights": _cap_startup_items(
+            (d.get("tracked_flights") or []) if active_layers.get("tracked", True) else [],
+            250,
+        ),
+        "ships": _cap_startup_items((d.get("ships") or []) if ships_enabled else [], 1500),
+        "uavs": _cap_startup_items((d.get("uavs") or []) if active_layers.get("military", True) else [], 100),
+        "liveuamap": _cap_startup_items(
+            (d.get("liveuamap") or []) if active_layers.get("global_incidents", True) else [],
+            300,
+        ),
+        "gps_jamming": _cap_startup_items(
+            (d.get("gps_jamming") or []) if active_layers.get("gps_jamming", True) else [],
+            200,
+        ),
+        "satellites": _cap_startup_items(
+            (d.get("satellites") or []) if active_layers.get("satellites", True) else [],
+            250,
+        ),
+        "satellite_source": d.get("satellite_source", "none"),
+        "satellite_analysis": (d.get("satellite_analysis") or {}) if active_layers.get("satellites", True) else {},
+        "sigint": _cap_startup_items(
+            sigint_items if (active_layers.get("sigint_meshtastic", True) or active_layers.get("sigint_aprs", True)) else [],
+            500,
+        ),
+        "sigint_totals": _sigint_totals_for_items(sigint_items),
+        "trains": _cap_startup_items((d.get("trains") or []) if active_layers.get("trains", True) else [], 100),
+        "news": _cap_startup_items(d.get("news") or [], 30),
+        "gdelt": _cap_startup_items((d.get("gdelt") or []) if active_layers.get("global_incidents", True) else [], 300),
+        "airports": _cap_startup_items(d.get("airports") or [], 500),
+        "threat_level": d.get("threat_level"),
+        "trending_markets": _cap_startup_items(d.get("trending_markets") or [], 10),
+        "correlations": _cap_startup_items(
+            (d.get("correlations") or []) if active_layers.get("correlations", True) else [],
+            50,
+        ),
+        "fimi": d.get("fimi"),
+        "crowdthreat": _cap_startup_items(
+            (d.get("crowdthreat") or []) if active_layers.get("crowdthreat", True) else [],
+            150,
+        ),
+        "freshness": freshness,
+        "bootstrap_ready": True,
+        "bootstrap_payload": True,
+    }
+    return Response(
+        content=orjson.dumps(_sanitize_payload(payload), default=str, option=orjson.OPT_NON_STR_KEYS),
+        media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"},
+    )
+
+
+@router.get("/api/live-data/fast")
+@limiter.limit("120/minute")
+async def live_data_fast(
+    request: Request,
+    s: float = Query(None, description="South bound — when all four bounds are supplied, heavy/dense layers (vessels, aircraft, sigint, CCTV, …) are filtered to this viewport with 20% padding. Static reference layers (satellites, etc.) always ship world-scale.", ge=-90, le=90),
+    w: float = Query(None, description="West bound (see s)", ge=-180, le=180),
+    n: float = Query(None, description="North bound (see s)", ge=-90, le=90),
+    e: float = Query(None, description="East bound (see s)", ge=-180, le=180),
+    initial: bool = Query(False, description="Return a capped startup payload for first paint"),
+):
+    bbox_suffix = _bbox_etag_suffix(s, w, n, e)
+    etag = _current_etag(prefix=("fast|initial|" if initial else "fast|full|") + bbox_suffix.lstrip("|") + ("|" if bbox_suffix else ""))
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    from services.fetchers._store import (active_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot)
+    d = get_latest_data_subset_refs(
+        "last_updated", "commercial_flights", "military_flights", "private_flights",
+        "private_jets", "tracked_flights", "ships", "cctv", "uavs", "liveuamap",
+        "gps_jamming", "satellites", "satellite_source", "satellite_analysis",
+        "sigint", "sigint_totals", "trains",
+    )
+    freshness = get_source_timestamps_snapshot()
+    ships_enabled = any(active_layers.get(key, True) for key in (
+        "ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts"))
+    cctv_total = len(d.get("cctv") or [])
+    sigint_items = _filter_sigint_by_layers(d.get("sigint") or [], active_layers)
+    sigint_totals = _sigint_totals_for_items(sigint_items)
+    payload = {
+        "commercial_flights": (d.get("commercial_flights") or []) if active_layers.get("flights", True) else [],
+        "military_flights": (d.get("military_flights") or []) if active_layers.get("military", True) else [],
+        "private_flights": (d.get("private_flights") or []) if active_layers.get("private", True) else [],
+        "private_jets": (d.get("private_jets") or []) if active_layers.get("jets", True) else [],
+        "tracked_flights": (d.get("tracked_flights") or []) if active_layers.get("tracked", True) else [],
+        "ships": (d.get("ships") or []) if ships_enabled else [],
+        "cctv": (d.get("cctv") or []) if active_layers.get("cctv", True) else [],
+        "uavs": (d.get("uavs") or []) if active_layers.get("military", True) else [],
+        "liveuamap": (d.get("liveuamap") or []) if active_layers.get("global_incidents", True) else [],
+        "gps_jamming": (d.get("gps_jamming") or []) if active_layers.get("gps_jamming", True) else [],
+        "satellites": (d.get("satellites") or []) if active_layers.get("satellites", True) else [],
+        "satellite_source": d.get("satellite_source", "none"),
+        "satellite_analysis": (d.get("satellite_analysis") or {}) if active_layers.get("satellites", True) else {},
+        "sigint": sigint_items if (active_layers.get("sigint_meshtastic", True) or active_layers.get("sigint_aprs", True)) else [],
+        "sigint_totals": sigint_totals,
+        "cctv_total": cctv_total,
+        "trains": (d.get("trains") or []) if active_layers.get("trains", True) else [],
+        "freshness": freshness,
+    }
+    if initial:
+        payload = _cap_fast_startup_payload(payload)
+    else:
+        payload = _cap_fast_dashboard_payload(payload)
+    # Issue #288: bbox filter heavy/dense layers only when all four bounds
+    # are supplied. Without bounds, behaviour is byte-for-byte identical
+    # to the pre-#288 implementation.
+    if _has_full_bbox(s, w, n, e):
+        payload = _apply_bbox_to_payload(payload, _FAST_BBOX_HEAVY_KEYS, s, w, n, e)
+    return Response(content=orjson.dumps(_sanitize_payload(payload)), media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"})
+
+
+@router.get("/api/live-data/slow")
+@limiter.limit("60/minute")
+async def live_data_slow(
+    request: Request,
+    s: float = Query(None, description="South bound — when all four bounds are supplied, heavy/dense layers (gdelt, firms_fires, kiwisdr, scanners, psk_reporter) are filtered to this viewport with 20% padding. Static reference layers (datacenters, military bases, power plants, weather, news, …) always ship world-scale.", ge=-90, le=90),
+    w: float = Query(None, description="West bound (see s)", ge=-180, le=180),
+    n: float = Query(None, description="North bound (see s)", ge=-90, le=90),
+    e: float = Query(None, description="East bound (see s)", ge=-180, le=180),
+):
+    bbox_suffix = _bbox_etag_suffix(s, w, n, e)
+    etag = _current_etag(prefix="slow|full|" + bbox_suffix.lstrip("|") + ("|" if bbox_suffix else ""))
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    from services.fetchers._store import (active_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot)
+    d = get_latest_data_subset_refs(
+        "last_updated", "news", "stocks", "financial_source", "oil", "weather", "traffic",
+        "earthquakes", "frontlines", "gdelt", "airports", "kiwisdr", "satnogs_stations",
+        "satnogs_observations", "tinygs_satellites", "space_weather", "internet_outages",
+        "firms_fires", "datacenters", "military_bases", "power_plants", "viirs_change_nodes",
+        "scanners", "weather_alerts", "ukraine_alerts", "air_quality", "volcanoes",
+        "fishing_activity", "psk_reporter", "correlations", "uap_sightings", "wastewater",
+        "crowdthreat", "threat_level", "trending_markets",
+    )
+    freshness = get_source_timestamps_snapshot()
+    payload = {
+        "last_updated": d.get("last_updated"),
+        "threat_level": d.get("threat_level"),
+        "trending_markets": d.get("trending_markets", []),
+        "news": d.get("news", []),
+        "stocks": d.get("stocks", {}),
+        "financial_source": d.get("financial_source", ""),
+        "oil": d.get("oil", {}),
+        "weather": d.get("weather"),
+        "traffic": d.get("traffic", []),
+        "earthquakes": (d.get("earthquakes") or []) if active_layers.get("earthquakes", True) else [],
+        "frontlines": d.get("frontlines") if active_layers.get("ukraine_frontline", True) else None,
+        "gdelt": (d.get("gdelt") or []) if active_layers.get("global_incidents", True) else [],
+        "airports": d.get("airports") or [],
+        "kiwisdr": (d.get("kiwisdr") or []) if active_layers.get("kiwisdr", True) else [],
+        "satnogs_stations": (d.get("satnogs_stations") or []) if active_layers.get("satnogs", True) else [],
+        "satnogs_total": len(d.get("satnogs_stations") or []),
+        "satnogs_observations": (d.get("satnogs_observations") or []) if active_layers.get("satnogs", True) else [],
+        "tinygs_satellites": (d.get("tinygs_satellites") or []) if active_layers.get("tinygs", True) else [],
+        "tinygs_total": len(d.get("tinygs_satellites") or []),
+        "psk_reporter": (d.get("psk_reporter") or []) if active_layers.get("psk_reporter", True) else [],
+        "space_weather": d.get("space_weather"),
+        "internet_outages": (d.get("internet_outages") or []) if active_layers.get("internet_outages", True) else [],
+        "firms_fires": (d.get("firms_fires") or []) if active_layers.get("firms", True) else [],
+        "datacenters": (d.get("datacenters") or []) if active_layers.get("datacenters", True) else [],
+        "military_bases": (d.get("military_bases") or []) if active_layers.get("military_bases", True) else [],
+        "power_plants": (d.get("power_plants") or []) if active_layers.get("power_plants", True) else [],
+        "viirs_change_nodes": (d.get("viirs_change_nodes") or []) if active_layers.get("viirs_nightlights", True) else [],
+        "scanners": (d.get("scanners") or []) if active_layers.get("scanners", True) else [],
+        "weather_alerts": d.get("weather_alerts", []) if active_layers.get("weather_alerts", True) else [],
+        "ukraine_alerts": d.get("ukraine_alerts", []) if active_layers.get("ukraine_alerts", True) else [],
+        "air_quality": (d.get("air_quality") or []) if active_layers.get("air_quality", True) else [],
+        "volcanoes": (d.get("volcanoes") or []) if active_layers.get("volcanoes", True) else [],
+        "fishing_activity": (d.get("fishing_activity") or []) if active_layers.get("fishing_activity", True) else [],
+        "correlations": (d.get("correlations") or []) if active_layers.get("correlations", True) else [],
+        "uap_sightings": (d.get("uap_sightings") or []) if active_layers.get("uap_sightings", True) else [],
+        "wastewater": (d.get("wastewater") or []) if active_layers.get("wastewater", True) else [],
+        "crowdthreat": (d.get("crowdthreat") or []) if active_layers.get("crowdthreat", True) else [],
+        "freshness": freshness,
+    }
+    # Issue #288: bbox filter heavy/dense layers only when all four bounds
+    # are supplied. Static reference layers (datacenters, military bases,
+    # power_plants, etc.) deliberately stay world-scale so panning never
+    # hides the infrastructure overlay the operator already has on screen.
+    if _has_full_bbox(s, w, n, e):
+        payload = _apply_bbox_to_payload(payload, _SLOW_BBOX_HEAVY_KEYS, s, w, n, e)
+    return Response(
+        content=orjson.dumps(_sanitize_payload(payload), default=str, option=orjson.OPT_NON_STR_KEYS),
+        media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"},
+    )
+
+
+# ── Satellite Overflight Counting ───────────────────────────────────────────
+# Counts unique satellites whose ground track entered a bounding box over 24h.
+# Uses cached TLEs + SGP4 propagation — no extra network requests.
+
+class OverflightRequest(BaseModel):
+    s: float
+    w: float
+    n: float
+    e: float
+    hours: int = 24
+
+
+# Issue #202: compute_overflights() is O(catalog_size × timesteps), where
+# timesteps grows linearly with `hours`. An unbounded `hours` value is a
+# trivial CPU-exhaustion vector. We clamp silently rather than raising 422 —
+# the response shape is unchanged, callers asking for too many hours just
+# get a shorter window, which is friendlier than a hostile error.
+#
+# Override via OVERFLIGHTS_MAX_HOURS env var if you legitimately need a
+# longer window (e.g. a planning use case that wants a full week).
+def _overflight_max_hours() -> int:
+    import os as _os
+    try:
+        raw = int(str(_os.environ.get("OVERFLIGHTS_MAX_HOURS", "72")).strip())
+    except (TypeError, ValueError):
+        raw = 72
+    return max(1, raw)
+
+
+@router.post("/api/satellites/overflights")
+@limiter.limit("10/minute")
+async def satellite_overflights(request: Request, body: OverflightRequest):
+    from services.fetchers.satellites import compute_overflights, _sat_gp_cache
+    gp_data = _sat_gp_cache.get("data")
+    if not gp_data:
+        return JSONResponse({"total": 0, "by_mission": {}, "satellites": [], "error": "No GP data cached yet"})
+    bbox = {"s": body.s, "w": body.w, "n": body.n, "e": body.e}
+
+    # Silent clamp — see comment on _overflight_max_hours().
+    requested_hours = max(1, int(body.hours or 0))
+    effective_hours = min(requested_hours, _overflight_max_hours())
+
+    result = compute_overflights(gp_data, bbox, hours=effective_hours)
+    # If we clamped, surface the effective window in the response so the
+    # caller can detect it if they care, without it being an error.
+    if isinstance(result, dict) and effective_hours != requested_hours:
+        result.setdefault("requested_hours", requested_hours)
+        result.setdefault("effective_hours", effective_hours)
+    return JSONResponse(result)

backend/routers/health.py

@@ -0,0 +1,117 @@
+import time as _time_mod
+from fastapi import APIRouter, Request, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin
+from services.data_fetcher import get_latest_data
+from services.schemas import HealthResponse
+import os
+
+APP_VERSION = os.environ.get("_HEALTH_APP_VERSION", "0.9.81")
+
+router = APIRouter()
+
+
+def _get_app_version() -> str:
+    # Import lazily to avoid circular import; main sets APP_VERSION before including routers
+    try:
+        import main as _main
+        return _main.APP_VERSION
+    except Exception:
+        return APP_VERSION
+
+
+_start_time_ref: dict = {"value": None}
+
+
+def _get_start_time() -> float:
+    if _start_time_ref["value"] is None:
+        try:
+            import main as _main
+            _start_time_ref["value"] = _main._start_time
+        except Exception:
+            _start_time_ref["value"] = _time_mod.time()
+    return _start_time_ref["value"]
+
+
+@router.get("/api/health", response_model=HealthResponse)
+@limiter.limit("30/minute")
+async def health_check(request: Request):
+    from services.fetchers._store import get_source_timestamps_snapshot
+    from services.slo import compute_all_statuses, summarise_statuses
+
+    d = get_latest_data()
+    last = d.get("last_updated")
+    timestamps = get_source_timestamps_snapshot()
+    slo_statuses = compute_all_statuses(d, timestamps)
+    slo_summary = summarise_statuses(slo_statuses)
+    # Top-level status reflects worst SLO result — "degraded" if any
+    # yellow, "error" if any red, "ok" otherwise. This is the single
+    # field an external probe / pager can watch.
+    top_status = "ok"
+    if slo_summary.get("red", 0) > 0:
+        top_status = "error"
+    elif slo_summary.get("yellow", 0) > 0:
+        top_status = "degraded"
+
+    # Issue #258: surface AIS proxy degraded TLS state so operators can see
+    # when the SPKI-pinned fallback is in effect. The data plane keeps
+    # flowing (this is by design — see ais_proxy.js comments) but observers
+    # who care about MITM-protection posture deserve a visible signal.
+    #
+    # Plus connectivity health (added 2026-05-23 when stream.aisstream.io
+    # went fully offline): ``connected`` tells the frontend whether ship
+    # data is actually flowing. When false, a banner explains that ships
+    # are unavailable due to an upstream outage — better than the user
+    # silently seeing an empty ocean and assuming we broke something.
+    ais_status: dict = {}
+    try:
+        from services.ais_stream import ais_proxy_status
+        ais_status = ais_proxy_status() or {}
+    except Exception:
+        ais_status = {}
+    if ais_status.get("degraded_tls") and top_status == "ok":
+        # Don't override a worse top-level status if SLOs already failed,
+        # but escalate ok -> degraded so the field surfaces in dashboards.
+        top_status = "degraded"
+    # AIS_API_KEY not configured is "feature off", not "system broken" —
+    # so we only escalate when the operator opted into AIS (key set) AND
+    # the stream is currently offline.
+    if (
+        os.environ.get("AIS_API_KEY")
+        and ais_status.get("connected") is False
+        and top_status == "ok"
+    ):
+        top_status = "degraded"
+
+    return {
+        "status": top_status,
+        "version": _get_app_version(),
+        "last_updated": last,
+        "sources": {
+            "flights": len(d.get("commercial_flights", [])),
+            "military": len(d.get("military_flights", [])),
+            "ships": len(d.get("ships", [])),
+            "satellites": len(d.get("satellites", [])),
+            "earthquakes": len(d.get("earthquakes", [])),
+            "cctv": len(d.get("cctv", [])),
+            "news": len(d.get("news", [])),
+            "uavs": len(d.get("uavs", [])),
+            "firms_fires": len(d.get("firms_fires", [])),
+            "liveuamap": len(d.get("liveuamap", [])),
+            "gdelt": len(d.get("gdelt", [])),
+            "uap_sightings": len(d.get("uap_sightings", [])),
+        },
+        "freshness": timestamps,
+        "uptime_seconds": round(_time_mod.time() - _get_start_time()),
+        "slo": slo_statuses,
+        "slo_summary": slo_summary,
+        "ais_proxy": ais_status,
+    }
+
+
+@router.get("/api/debug-latest", dependencies=[Depends(require_admin)])
+@limiter.limit("30/minute")
+async def debug_latest_data(request: Request):
+    return list(get_latest_data().keys())

backend/routers/infonet.py

@@ -0,0 +1,598 @@
+"""Infonet economy / governance / gates / bootstrap HTTP surface.
+
+Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §2.1.
+
+Read endpoints return chain-derived state (computed by the
+``services.infonet`` adapters / pure functions). Write endpoints take
+a payload, validate it through the cutover-registered validators, and
+return a structured "would-emit" preview. Production wiring (signing
++ ``Infonet.append`` persistence) is a thin follow-on; the validation
+contract is locked here.
+
+Cross-cutting design rule: errors are diagnostic, not punitive. Each
+write endpoint returns ``{"ok": False, "reason": "..."}`` on
+validation failure with the exact field that failed. Frontend
+surfaces the reason in the UI.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any
+
+from fastapi import APIRouter, Body, Path
+
+# Triggers the chain cutover at module-load time so registered
+# validators are live for any subsequent route invocation.
+from services.infonet import _chain_cutover  # noqa: F401
+from services.infonet.adapters.gate_adapter import InfonetGateAdapter
+from services.infonet.adapters.oracle_adapter import InfonetOracleAdapter
+from services.infonet.adapters.reputation_adapter import InfonetReputationAdapter
+from services.infonet.bootstrap import compute_active_features
+from services.infonet.config import (
+    CONFIG,
+    IMMUTABLE_PRINCIPLES,
+)
+from services.infonet.governance import (
+    apply_petition_payload,
+    compute_petition_state,
+    compute_upgrade_state,
+)
+from services.infonet.governance.dsl_executor import InvalidPetition
+from services.infonet.partition import (
+    classify_event_type,
+    is_chain_stale,
+    should_mark_provisional,
+)
+from services.infonet.privacy import (
+    DEXScaffolding,
+    RingCTScaffolding,
+    ShieldedBalanceScaffolding,
+    StealthAddressScaffolding,
+)
+from services.infonet.schema import (
+    INFONET_ECONOMY_EVENT_TYPES,
+    validate_infonet_event_payload,
+)
+from services.infonet.time_validity import chain_majority_time
+
+logger = logging.getLogger("routers.infonet")
+
+router = APIRouter(prefix="/api/infonet", tags=["infonet"])
+
+
+# ─── Chain access helper ─────────────────────────────────────────────────
+# Every adapter takes a ``chain_provider`` callable. We pull the live
+# Infonet chain from mesh_hashchain. Tests can monkeypatch this.
+
+def _live_chain() -> list[dict[str, Any]]:
+    try:
+        from services.mesh.mesh_hashchain import infonet
+        events = getattr(infonet, "events", None)
+        if isinstance(events, list):
+            return list(events)
+        # Some implementations use a deque; convert to list.
+        if events is not None:
+            return list(events)
+    except Exception as exc:
+        logger.debug("infonet chain unavailable: %s", exc)
+    return []
+
+
+def _now() -> float:
+    cmt = chain_majority_time(_live_chain())
+    return cmt if cmt > 0 else float(time.time())
+
+
+# ─── Status ──────────────────────────────────────────────────────────────
+
+@router.get("/status")
+def infonet_status() -> dict[str, Any]:
+    """Top-level health snapshot for the InfonetTerminal HUD.
+
+    Returns ramp activation flags, partition staleness, privacy
+    primitive statuses, immutable principles, and counts of
+    chain-derived state (markets / petitions / gates / etc).
+    """
+    chain = _live_chain()
+    now = _now()
+    features = compute_active_features(chain)
+
+    # Privacy primitive statuses (truthful — most are NOT_IMPLEMENTED).
+    privacy = {
+        "ringct": RingCTScaffolding().status().value,
+        "stealth_address": StealthAddressScaffolding().status().value,
+        "shielded_balance": ShieldedBalanceScaffolding().status().value,
+        "dex": DEXScaffolding().status().value,
+    }
+
+    return {
+        "ok": True,
+        "now": now,
+        "chain_majority_time": chain_majority_time(chain),
+        "chain_event_count": len(chain),
+        "chain_stale": is_chain_stale(chain, now=now),
+        "ramp": {
+            "node_count": features.node_count,
+            "bootstrap_resolution_active": features.bootstrap_resolution_active,
+            "staked_resolution_active": features.staked_resolution_active,
+            "governance_petitions_active": features.governance_petitions_active,
+            "upgrade_governance_active": features.upgrade_governance_active,
+            "commoncoin_active": features.commoncoin_active,
+        },
+        "privacy_primitive_status": privacy,
+        "immutable_principles": dict(IMMUTABLE_PRINCIPLES),
+        "config_keys_count": len(CONFIG),
+        "infonet_economy_event_types_count": len(INFONET_ECONOMY_EVENT_TYPES),
+    }
+
+
+# ─── Petitions / governance ──────────────────────────────────────────────
+
+@router.get("/petitions")
+def list_petitions() -> dict[str, Any]:
+    """List petition_file events on the chain with their current state."""
+    chain = _live_chain()
+    now = _now()
+    out: list[dict[str, Any]] = []
+    for ev in chain:
+        if ev.get("event_type") != "petition_file":
+            continue
+        pid = (ev.get("payload") or {}).get("petition_id")
+        if not isinstance(pid, str):
+            continue
+        try:
+            state = compute_petition_state(pid, chain, now=now)
+            out.append({
+                "petition_id": state.petition_id,
+                "status": state.status,
+                "filer_id": state.filer_id,
+                "filed_at": state.filed_at,
+                "petition_payload": state.petition_payload,
+                "signature_governance_weight": state.signature_governance_weight,
+                "signature_threshold_at_filing": state.signature_threshold_at_filing,
+                "votes_for_weight": state.votes_for_weight,
+                "votes_against_weight": state.votes_against_weight,
+                "voting_deadline": state.voting_deadline,
+                "challenge_window_until": state.challenge_window_until,
+            })
+        except Exception as exc:
+            logger.warning("petition state error for %s: %s", pid, exc)
+    return {"ok": True, "petitions": out, "now": now}
+
+
+@router.get("/petitions/{petition_id}")
+def get_petition(petition_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    state = compute_petition_state(petition_id, chain, now=now)
+    return {"ok": True, "petition": state.__dict__, "now": now}
+
+
+@router.post("/petitions/preview")
+def preview_petition_payload(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
+    """Validate a petition payload through the DSL executor without
+    emitting it. Returns the candidate config diff so the UI can show
+    "this petition would change vote_decay_days from 90 to 30".
+    """
+    try:
+        result = apply_petition_payload(payload)
+        return {
+            "ok": True,
+            "changed_keys": list(result.changed_keys),
+            "new_values": {k: result.new_config[k] for k in result.changed_keys},
+        }
+    except InvalidPetition as exc:
+        return {"ok": False, "reason": str(exc)}
+
+
+@router.post("/events/validate")
+def validate_event(body: dict[str, Any] = Body(...)) -> dict[str, Any]:
+    """Validate an arbitrary Infonet economy event payload.
+
+    Frontend uses this for client-side preflight before signing /
+    submitting an event. Returns ``{ok: True}`` on success or
+    ``{ok: False, reason: ...}`` with the exact validation failure.
+    """
+    event_type = body.get("event_type")
+    payload = body.get("payload", {})
+    if not isinstance(event_type, str) or not event_type:
+        return {"ok": False, "reason": "event_type required"}
+    if not isinstance(payload, dict):
+        return {"ok": False, "reason": "payload must be an object"}
+    ok, reason = validate_infonet_event_payload(event_type, payload)
+    return {
+        "ok": ok,
+        "reason": reason if not ok else None,
+        "tier": classify_event_type(event_type),
+        "would_be_provisional": should_mark_provisional(event_type, _live_chain(), now=_now()),
+    }
+
+
+# ─── Upgrade-hash governance ────────────────────────────────────────────
+
+@router.get("/upgrades")
+def list_upgrades() -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    out: list[dict[str, Any]] = []
+    for ev in chain:
+        if ev.get("event_type") != "upgrade_propose":
+            continue
+        pid = (ev.get("payload") or {}).get("proposal_id")
+        if not isinstance(pid, str):
+            continue
+        try:
+            # Heavy node set is a runtime concept (transport tier ==
+            # private_strong per plan §3.5). Empty here for the
+            # snapshot endpoint; production will pass the live set.
+            state = compute_upgrade_state(pid, chain, now=now, heavy_node_ids=set())
+            out.append({
+                "proposal_id": state.proposal_id,
+                "status": state.status,
+                "proposer_id": state.proposer_id,
+                "filed_at": state.filed_at,
+                "release_hash": state.release_hash,
+                "target_protocol_version": state.target_protocol_version,
+                "votes_for_weight": state.votes_for_weight,
+                "votes_against_weight": state.votes_against_weight,
+                "readiness_fraction": state.readiness.fraction,
+                "readiness_threshold_met": state.readiness.threshold_met,
+            })
+        except Exception as exc:
+            logger.warning("upgrade state error for %s: %s", pid, exc)
+    return {"ok": True, "upgrades": out, "now": now}
+
+
+@router.get("/upgrades/{proposal_id}")
+def get_upgrade(proposal_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    state = compute_upgrade_state(proposal_id, chain, now=now, heavy_node_ids=set())
+    return {
+        "ok": True,
+        "upgrade": {
+            "proposal_id": state.proposal_id,
+            "status": state.status,
+            "proposer_id": state.proposer_id,
+            "filed_at": state.filed_at,
+            "release_hash": state.release_hash,
+            "target_protocol_version": state.target_protocol_version,
+            "signature_governance_weight": state.signature_governance_weight,
+            "votes_for_weight": state.votes_for_weight,
+            "votes_against_weight": state.votes_against_weight,
+            "voting_deadline": state.voting_deadline,
+            "challenge_window_until": state.challenge_window_until,
+            "activation_deadline": state.activation_deadline,
+            "readiness": {
+                "total_heavy_nodes": state.readiness.total_heavy_nodes,
+                "ready_count": state.readiness.ready_count,
+                "fraction": state.readiness.fraction,
+                "threshold_met": state.readiness.threshold_met,
+            },
+        },
+        "now": now,
+    }
+
+
+# ─── Markets / resolution / disputes ────────────────────────────────────
+
+@router.get("/markets/{market_id}")
+def get_market_state(market_id: str = Path(...)) -> dict[str, Any]:
+    """Full market view: lifecycle, snapshot, evidence, stakes,
+    excluded predictors, dispute state."""
+    chain = _live_chain()
+    now = _now()
+    oracle = InfonetOracleAdapter(lambda: chain)
+
+    status = oracle.market_status(market_id, now=now)
+    snap = oracle.find_snapshot(market_id)
+    bundles = oracle.collect_evidence(market_id)
+    excluded = sorted(oracle.excluded_predictor_ids(market_id))
+    disputes = oracle.collect_disputes(market_id)
+    reversed_flag = oracle.market_was_reversed(market_id)
+
+    return {
+        "ok": True,
+        "market_id": market_id,
+        "status": status.value,
+        "snapshot": snap,
+        "evidence_bundles": [
+            {
+                "node_id": b.node_id,
+                "claimed_outcome": b.claimed_outcome,
+                "evidence_hashes": list(b.evidence_hashes),
+                "source_description": b.source_description,
+                "bond": b.bond,
+                "timestamp": b.timestamp,
+                "is_first_for_side": b.is_first_for_side,
+                "submission_hash": b.submission_hash,
+            }
+            for b in bundles
+        ],
+        "excluded_predictor_ids": excluded,
+        "disputes": [
+            {
+                "dispute_id": d.dispute_id,
+                "challenger_id": d.challenger_id,
+                "challenger_stake": d.challenger_stake,
+                "opened_at": d.opened_at,
+                "is_resolved": d.is_resolved,
+                "resolved_outcome": d.resolved_outcome,
+                "confirm_stakes": d.confirm_stakes,
+                "reverse_stakes": d.reverse_stakes,
+            }
+            for d in disputes
+        ],
+        "was_reversed": reversed_flag,
+        "now": now,
+    }
+
+
+@router.get("/markets/{market_id}/preview-resolution")
+def preview_resolution(market_id: str = Path(...)) -> dict[str, Any]:
+    """Run the resolution decision procedure without emitting a
+    finalize event. UI uses this to show "if resolution closed now,
+    the market would resolve as <outcome> for <reason>"."""
+    chain = _live_chain()
+    oracle = InfonetOracleAdapter(lambda: chain)
+    result = oracle.resolve_market(market_id)
+    return {
+        "ok": True,
+        "preview": {
+            "outcome": result.outcome,
+            "reason": result.reason,
+            "is_provisional": result.is_provisional,
+            "burned_amount": result.burned_amount,
+            "stake_returns": [
+                {"node_id": k[0], "rep_type": k[1], "amount": v}
+                for k, v in result.stake_returns.items()
+            ],
+            "stake_winnings": [
+                {"node_id": k[0], "rep_type": k[1], "amount": v}
+                for k, v in result.stake_winnings.items()
+            ],
+            "bond_returns": [
+                {"node_id": k, "amount": v} for k, v in result.bond_returns.items()
+            ],
+            "bond_forfeits": [
+                {"node_id": k, "amount": v} for k, v in result.bond_forfeits.items()
+            ],
+            "first_submitter_bonuses": [
+                {"node_id": k, "amount": v}
+                for k, v in result.first_submitter_bonuses.items()
+            ],
+        },
+    }
+
+
+# ─── Gate shutdown lifecycle ────────────────────────────────────────────
+
+@router.get("/gates/{gate_id}")
+def get_gate_state(gate_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    now = _now()
+    gates = InfonetGateAdapter(lambda: chain)
+    meta = gates.gate_meta(gate_id)
+    if meta is None:
+        return {"ok": False, "reason": "gate_not_found"}
+    suspension = gates.suspension_state(gate_id, now=now)
+    shutdown = gates.shutdown_state(gate_id, now=now)
+    locked = gates.locked_state(gate_id)
+    members = sorted(gates.member_set(gate_id))
+    return {
+        "ok": True,
+        "gate_id": gate_id,
+        "meta": {
+            "creator_node_id": meta.creator_node_id,
+            "display_name": meta.display_name,
+            "entry_sacrifice": meta.entry_sacrifice,
+            "min_overall_rep": meta.min_overall_rep,
+            "min_gate_rep": dict(meta.min_gate_rep),
+            "created_at": meta.created_at,
+        },
+        "members": members,
+        "ratified": gates.is_ratified(gate_id),
+        "cumulative_member_oracle_rep": gates.cumulative_member_oracle_rep(gate_id),
+        "locked": {
+            "is_locked": locked.locked,
+            "locked_at": locked.locked_at,
+            "locked_by": list(locked.locked_by),
+        },
+        "suspension": {
+            "status": suspension.status,
+            "suspended_at": suspension.suspended_at,
+            "suspended_until": suspension.suspended_until,
+            "last_shutdown_petition_at": suspension.last_shutdown_petition_at,
+        },
+        "shutdown": {
+            "has_pending": shutdown.has_pending,
+            "pending_petition_id": shutdown.pending_petition_id,
+            "pending_status": shutdown.pending_status,
+            "execution_at": shutdown.execution_at,
+            "executed": shutdown.executed,
+        },
+        "now": now,
+    }
+
+
+# ─── Reputation views ───────────────────────────────────────────────────
+
+@router.get("/nodes/{node_id}/reputation")
+def get_node_reputation(node_id: str = Path(...)) -> dict[str, Any]:
+    chain = _live_chain()
+    rep = InfonetReputationAdapter(lambda: chain)
+    breakdown = rep.oracle_rep_breakdown(node_id)
+    return {
+        "ok": True,
+        "node_id": node_id,
+        "oracle_rep": rep.oracle_rep(node_id),
+        "oracle_rep_active": rep.oracle_rep_active(node_id),
+        "oracle_rep_lifetime": rep.oracle_rep_lifetime(node_id),
+        "common_rep": rep.common_rep(node_id),
+        "decay_factor": rep.decay_factor(node_id),
+        "last_successful_prediction_ts": rep.last_successful_prediction_ts(node_id),
+        "breakdown": {
+            "free_prediction_mints": breakdown.free_prediction_mints,
+            "staked_prediction_returns": breakdown.staked_prediction_returns,
+            "staked_prediction_losses": breakdown.staked_prediction_losses,
+            "total": breakdown.total,
+        },
+    }
+
+
+# ─── Bootstrap ──────────────────────────────────────────────────────────
+
+@router.get("/bootstrap/markets/{market_id}")
+def get_bootstrap_market_state(market_id: str = Path(...)) -> dict[str, Any]:
+    """Bootstrap-mode-specific market view: who has voted, who is
+    eligible, current tally."""
+    from services.infonet.bootstrap import (
+        deduplicate_votes,
+        validate_bootstrap_eligibility,
+    )
+
+    chain = _live_chain()
+    canonical = deduplicate_votes(market_id, chain)
+    votes_summary: list[dict[str, Any]] = []
+    yes = 0
+    no = 0
+    for v in canonical:
+        node_id = v.get("node_id") or ""
+        side = (v.get("payload") or {}).get("side")
+        decision = validate_bootstrap_eligibility(node_id, market_id, chain)
+        votes_summary.append({
+            "node_id": node_id,
+            "side": side,
+            "eligible": decision.eligible,
+            "ineligible_reason": decision.reason if not decision.eligible else None,
+        })
+        if decision.eligible:
+            if side == "yes":
+                yes += 1
+            elif side == "no":
+                no += 1
+    total = yes + no
+    return {
+        "ok": True,
+        "market_id": market_id,
+        "votes": votes_summary,
+        "tally": {
+            "yes": yes,
+            "no": no,
+            "total_eligible": total,
+            "min_market_participants": int(CONFIG["min_market_participants"]),
+            "supermajority_threshold": float(CONFIG["bootstrap_resolution_supermajority"]),
+        },
+    }
+
+
+# ─── Signed write: append an Infonet economy event ──────────────────────
+
+@router.post("/append")
+def append_event(body: dict[str, Any] = Body(...)) -> dict[str, Any]:
+    """Append a signed Infonet economy event to the chain.
+
+    Body shape (all required for production):
+
+        {
+            "event_type": str,       # one of INFONET_ECONOMY_EVENT_TYPES
+            "node_id":    str,       # signer
+            "payload":    dict,      # event-specific fields
+            "signature":  str,       # hex
+            "sequence":   int,       # node-monotonic
+            "public_key": str,       # base64
+            "public_key_algo": str,  # "ed25519" or "ecdsa"
+            "protocol_version": str  # optional, defaults to current
+        }
+
+    The cutover-registered validators run automatically via
+    ``mesh_hashchain.Infonet.append`` — payload validation, signature
+    verification, replay protection, sequence ordering, public-key
+    binding, revocation status. No additional security wrapper is
+    needed because ``Infonet.append`` IS the secure entry point.
+
+    Returns the appended event dict on success, or
+    ``{"ok": False, "reason": "..."}`` on validation / signing failure.
+    """
+    if not isinstance(body, dict):
+        return {"ok": False, "reason": "body_must_be_object"}
+
+    event_type = body.get("event_type")
+    if not isinstance(event_type, str) or event_type not in INFONET_ECONOMY_EVENT_TYPES:
+        return {
+            "ok": False,
+            "reason": f"event_type must be one of INFONET_ECONOMY_EVENT_TYPES "
+                      f"(got {event_type!r})",
+        }
+
+    node_id = body.get("node_id")
+    if not isinstance(node_id, str) or not node_id:
+        return {"ok": False, "reason": "node_id required"}
+
+    payload = body.get("payload", {})
+    if not isinstance(payload, dict):
+        return {"ok": False, "reason": "payload must be an object"}
+
+    sequence = body.get("sequence", 0)
+    try:
+        sequence = int(sequence)
+    except (TypeError, ValueError):
+        return {"ok": False, "reason": "sequence must be an integer"}
+    if sequence <= 0:
+        return {"ok": False, "reason": "sequence must be > 0"}
+
+    signature = str(body.get("signature") or "")
+    public_key = str(body.get("public_key") or "")
+    public_key_algo = str(body.get("public_key_algo") or "")
+    protocol_version = str(body.get("protocol_version") or "")
+
+    if not signature or not public_key or not public_key_algo:
+        return {
+            "ok": False,
+            "reason": "signature, public_key, and public_key_algo are required",
+        }
+
+    try:
+        from services.mesh.mesh_hashchain import infonet
+        event = infonet.append(
+            event_type=event_type,
+            node_id=node_id,
+            payload=payload,
+            signature=signature,
+            sequence=sequence,
+            public_key=public_key,
+            public_key_algo=public_key_algo,
+            protocol_version=protocol_version,
+        )
+    except ValueError as exc:
+        # Infonet.append raises ValueError for any validation failure
+        # — payload / signature / replay / sequence / binding. The
+        # message is user-facing per the non-hostile UX rule.
+        return {"ok": False, "reason": str(exc)}
+    except Exception as exc:
+        logger.exception("infonet append failed")
+        return {"ok": False, "reason": f"server_error: {type(exc).__name__}"}
+
+    return {"ok": True, "event": event}
+
+
+# ─── Function Keys (citizen + operator views) ───────────────────────────
+
+@router.get("/function-keys/operator/{operator_id}/batch-summary")
+def operator_batch_summary(operator_id: str = Path(...)) -> dict[str, Any]:
+    """Sprint 11+ scaffolding: returns the operator's local batch
+    counter for the current period. Production wires this through the
+    operator's local-store implementation (Sprint 11+ scaffolding
+    doesn't persist; counts reset per process)."""
+    return {
+        "ok": True,
+        "operator_id": operator_id,
+        "scaffolding_only": True,
+        "note": "Production operators maintain a persistent BatchedSettlementBatch. "
+                "This endpoint reports the in-memory state of the local batch.",
+    }
+
+
+__all__ = ["router"]

backend/routers/mesh_dm.py

@@ -0,0 +1,565 @@
+import asyncio
+import hashlib
+import hmac
+import logging
+import secrets
+import time
+from typing import Any
+
+from fastapi import APIRouter, Depends, Request
+from fastapi.responses import JSONResponse
+
+from auth import (
+    _is_debug_test_request,
+    _scoped_view_authenticated,
+    _verify_peer_push_hmac,
+    require_admin,
+)
+from limiter import limiter
+from services.config import get_settings
+from services.mesh.mesh_compatibility import (
+    LEGACY_AGENT_ID_LOOKUP_TARGET,
+    legacy_agent_id_lookup_blocked,
+    record_legacy_agent_id_lookup,
+    sunset_target_label,
+)
+from services.mesh.mesh_signed_events import (
+    MeshWriteExemption,
+    SignedWriteKind,
+    get_prepared_signed_write,
+    mesh_write_exempt,
+    requires_signed_write,
+)
+
+logger = logging.getLogger(__name__)
+_WARNED_LEGACY_DM_PUBKEY_LOOKUPS: set[str] = set()
+
+router = APIRouter()
+
+
+# ---------------------------------------------------------------------------
+# Local helpers
+# ---------------------------------------------------------------------------
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _warn_legacy_dm_pubkey_lookup(agent_id: str) -> None:
+    peer_id = str(agent_id or "").strip().lower()
+    if not peer_id or peer_id in _WARNED_LEGACY_DM_PUBKEY_LOOKUPS:
+        return
+    _WARNED_LEGACY_DM_PUBKEY_LOOKUPS.add(peer_id)
+    logger.warning(
+        "mesh legacy DH pubkey lookup used for %s via direct agent_id; prefer invite-scoped lookup handles before removal in %s",
+        peer_id,
+        sunset_target_label(LEGACY_AGENT_ID_LOOKUP_TARGET),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Transition delegates: forward to main.py so test monkeypatches still work.
+# These will move to a shared module once main.py routes are removed.
+# ---------------------------------------------------------------------------
+def _main_delegate(name):
+    def _wrapper(*a, **kw):
+        import main as _m
+        return getattr(_m, name)(*a, **kw)
+    _wrapper.__name__ = name
+    return _wrapper
+
+
+_verify_signed_write = _main_delegate("_verify_signed_write")
+_secure_dm_enabled = _main_delegate("_secure_dm_enabled")
+_legacy_dm_get_allowed = _main_delegate("_legacy_dm_get_allowed")
+_rns_private_dm_ready = _main_delegate("_rns_private_dm_ready")
+_anonymous_dm_hidden_transport_enforced = _main_delegate("_anonymous_dm_hidden_transport_enforced")
+_high_privacy_profile_enabled = _main_delegate("_high_privacy_profile_enabled")
+_dm_send_from_signed_request = _main_delegate("_dm_send_from_signed_request")
+_dm_poll_secure_from_signed_request = _main_delegate("_dm_poll_secure_from_signed_request")
+_dm_count_secure_from_signed_request = _main_delegate("_dm_count_secure_from_signed_request")
+_validate_private_signed_sequence = _main_delegate("_validate_private_signed_sequence")
+
+
+def _signed_body(request: Request) -> dict[str, Any]:
+    prepared = get_prepared_signed_write(request)
+    if prepared is None:
+        return {}
+    return dict(prepared.body)
+
+
+async def _maybe_apply_dm_relay_jitter() -> None:
+    if not _high_privacy_profile_enabled():
+        return
+    await asyncio.sleep((50 + secrets.randbelow(451)) / 1000.0)
+
+
+_REQUEST_V2_REDUCED_VERSION = "request-v2-reduced-v3"
+_REQUEST_V2_RECOVERY_STATES = {"pending", "verified", "failed"}
+
+
+def _is_canonical_reduced_request_message(message: dict[str, Any]) -> bool:
+    item = dict(message or {})
+    return (
+        str(item.get("delivery_class", "") or "").strip().lower() == "request"
+        and str(item.get("request_contract_version", "") or "").strip()
+        == _REQUEST_V2_REDUCED_VERSION
+        and item.get("sender_recovery_required") is True
+    )
+
+
+def _annotate_request_recovery_message(message: dict[str, Any]) -> dict[str, Any]:
+    item = dict(message or {})
+    delivery_class = str(item.get("delivery_class", "") or "").strip().lower()
+    sender_id = str(item.get("sender_id", "") or "").strip()
+    sender_seal = str(item.get("sender_seal", "") or "").strip()
+    sender_is_blinded = sender_id.startswith("sealed:") or sender_id.startswith("sender_token:")
+    if delivery_class != "request" or not sender_is_blinded or not sender_seal.startswith("v3:"):
+        return item
+    if not str(item.get("request_contract_version", "") or "").strip():
+        item["request_contract_version"] = _REQUEST_V2_REDUCED_VERSION
+    item["sender_recovery_required"] = True
+    state = str(item.get("sender_recovery_state", "") or "").strip().lower()
+    if state not in _REQUEST_V2_RECOVERY_STATES:
+        state = "pending"
+    item["sender_recovery_state"] = state
+    return item
+
+
+def _annotate_request_recovery_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    return [_annotate_request_recovery_message(message) for message in (messages or [])]
+
+
+def _request_duplicate_authority_rank(message: dict[str, Any]) -> int:
+    item = dict(message or {})
+    if str(item.get("delivery_class", "") or "").strip().lower() != "request":
+        return 0
+    if _is_canonical_reduced_request_message(item):
+        return 3
+    sender_id = str(item.get("sender_id", "") or "").strip()
+    if sender_id.startswith("sealed:") or sender_id.startswith("sender_token:"):
+        return 1
+    if sender_id:
+        return 2
+    return 0
+
+
+def _request_duplicate_recovery_rank(message: dict[str, Any]) -> int:
+    if not _is_canonical_reduced_request_message(message):
+        return 0
+    state = str(dict(message or {}).get("sender_recovery_state", "") or "").strip().lower()
+    if state == "verified":
+        return 2
+    if state == "pending":
+        return 1
+    return 0
+
+
+def _poll_duplicate_source_rank(source: str) -> int:
+    normalized = str(source or "").strip().lower()
+    if normalized == "relay":
+        return 2
+    if normalized == "reticulum":
+        return 1
+    return 0
+
+
+def _should_replace_dm_poll_duplicate(
+    existing: dict[str, Any],
+    existing_source: str,
+    candidate: dict[str, Any],
+    candidate_source: str,
+) -> bool:
+    candidate_authority = _request_duplicate_authority_rank(candidate)
+    existing_authority = _request_duplicate_authority_rank(existing)
+    if candidate_authority != existing_authority:
+        return candidate_authority > existing_authority
+
+    candidate_recovery = _request_duplicate_recovery_rank(candidate)
+    existing_recovery = _request_duplicate_recovery_rank(existing)
+    if candidate_recovery != existing_recovery:
+        return candidate_recovery > existing_recovery
+
+    candidate_source_rank = _poll_duplicate_source_rank(candidate_source)
+    existing_source_rank = _poll_duplicate_source_rank(existing_source)
+    if candidate_source_rank != existing_source_rank:
+        return candidate_source_rank > existing_source_rank
+
+    try:
+        candidate_ts = float(candidate.get("timestamp", 0) or 0)
+    except Exception:
+        candidate_ts = 0.0
+    try:
+        existing_ts = float(existing.get("timestamp", 0) or 0)
+    except Exception:
+        existing_ts = 0.0
+    return candidate_ts > existing_ts
+
+
+def _merge_dm_poll_messages(
+    relay_messages: list[dict[str, Any]],
+    direct_messages: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    merged: list[dict[str, Any]] = []
+    index_by_msg_id: dict[str, tuple[int, str]] = {}
+
+    def add_messages(items: list[dict[str, Any]], source: str) -> None:
+        for original in items or []:
+            item = dict(original or {})
+            msg_id = str(item.get("msg_id", "") or "").strip()
+            if not msg_id:
+                merged.append(item)
+                continue
+            existing = index_by_msg_id.get(msg_id)
+            if existing is None:
+                index_by_msg_id[msg_id] = (len(merged), source)
+                merged.append(item)
+                continue
+            index, existing_source = existing
+            if _should_replace_dm_poll_duplicate(merged[index], existing_source, item, source):
+                merged[index] = item
+                index_by_msg_id[msg_id] = (index, source)
+
+    add_messages(relay_messages, "relay")
+    add_messages(direct_messages, "reticulum")
+    return sorted(merged, key=lambda item: float(item.get("timestamp", 0) or 0))
+
+
+# ---------------------------------------------------------------------------
+# Route handlers
+# ---------------------------------------------------------------------------
+
+@router.post("/api/mesh/dm/register")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_REGISTER)
+async def dm_register_key(request: Request):
+    """Register a DH public key for encrypted DM key exchange."""
+    body = _signed_body(request)
+    agent_id = body.get("agent_id", "").strip()
+    dh_pub_key = body.get("dh_pub_key", "").strip()
+    dh_algo = body.get("dh_algo", "").strip()
+    timestamp = _safe_int(body.get("timestamp", 0) or 0)
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not agent_id or not dh_pub_key or not dh_algo or not timestamp:
+        return {"ok": False, "detail": "Missing agent_id, dh_pub_key, dh_algo, or timestamp"}
+    if dh_algo.upper() not in ("X25519", "ECDH_P256", "ECDH"):
+        return {"ok": False, "detail": "Unsupported dh_algo"}
+    now_ts = int(time.time())
+    if abs(timestamp - now_ts) > 7 * 86400:
+        return {"ok": False, "detail": "DH key timestamp is too far from current time"}
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+
+        reputation_ledger.register_node(agent_id, public_key, public_key_algo)
+    except Exception:
+        pass
+
+    accepted, detail, metadata = dm_relay.register_dh_key(
+        agent_id,
+        dh_pub_key,
+        dh_algo,
+        timestamp,
+        signature,
+        public_key,
+        public_key_algo,
+        protocol_version,
+        sequence,
+    )
+    if not accepted:
+        return {"ok": False, "detail": detail}
+
+    return {"ok": True, **(metadata or {})}
+
+
+@router.get("/api/mesh/dm/pubkey")
+@limiter.limit("30/minute")
+async def dm_get_pubkey(request: Request, agent_id: str = "", lookup_token: str = ""):
+    import main as _m
+
+    return await _m.dm_get_pubkey(request, agent_id=agent_id, lookup_token=lookup_token)
+
+
+@router.get("/api/mesh/dm/prekey-bundle")
+@limiter.limit("30/minute")
+async def dm_get_prekey_bundle(request: Request, agent_id: str = "", lookup_token: str = ""):
+    import main as _m
+
+    return await _m.dm_get_prekey_bundle(request, agent_id=agent_id, lookup_token=lookup_token)
+
+
+@router.post("/api/mesh/dm/prekey-peer-lookup")
+@limiter.limit("60/minute")
+@mesh_write_exempt(MeshWriteExemption.PEER_GOSSIP)
+async def dm_prekey_peer_lookup(request: Request):
+    """Peer-authenticated invite lookup handle resolution.
+
+    This endpoint exists for private/bootstrap peers to import signed invites
+    without exposing a stable agent_id on the ordinary lookup surface. It only
+    accepts HMAC-authenticated peer calls and only resolves lookup_token.
+    """
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 4096:
+                return JSONResponse(
+                    status_code=413,
+                    content={"ok": False, "detail": "Request body too large"},
+                )
+        except (TypeError, ValueError):
+            pass
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return JSONResponse(
+            status_code=403,
+            content={"ok": False, "detail": "Invalid or missing peer HMAC"},
+        )
+    try:
+        import json
+
+        body = json.loads(body_bytes or b"{}")
+    except Exception:
+        return {"ok": False, "detail": "invalid json"}
+    lookup_token = str(dict(body or {}).get("lookup_token", "") or "").strip()
+    if not lookup_token:
+        return {"ok": False, "detail": "lookup_token required"}
+    from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle
+
+    result = fetch_dm_prekey_bundle(
+        agent_id="",
+        lookup_token=lookup_token,
+        allow_peer_lookup=False,
+    )
+    if not result.get("ok"):
+        return {"ok": False, "detail": str(result.get("detail", "") or "Prekey bundle not found")}
+    safe = dict(result)
+    safe.pop("resolved_agent_id", None)
+    safe["lookup_mode"] = "invite_lookup_handle"
+    return safe
+
+
+@router.post("/api/mesh/dm/send")
+@limiter.limit("20/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_SEND)
+async def dm_send(request: Request):
+    return await _dm_send_from_signed_request(request)
+
+
+@router.post("/api/mesh/dm/poll")
+@limiter.limit("30/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_POLL)
+async def dm_poll_secure(request: Request):
+    return await _dm_poll_secure_from_signed_request(request)
+
+
+@router.get("/api/mesh/dm/poll")
+@limiter.limit("30/minute")
+async def dm_poll(
+    request: Request,
+    agent_id: str = "",
+    agent_token: str = "",
+    agent_token_prev: str = "",
+    agent_tokens: str = "",
+):
+    import main as _m
+
+    return await _m.dm_poll(
+        request,
+        agent_id=agent_id,
+        agent_token=agent_token,
+        agent_token_prev=agent_token_prev,
+        agent_tokens=agent_tokens,
+    )
+
+
+@router.post("/api/mesh/dm/count")
+@limiter.limit("60/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_COUNT)
+async def dm_count_secure(request: Request):
+    return await _dm_count_secure_from_signed_request(request)
+
+
+@router.get("/api/mesh/dm/count")
+@limiter.limit("60/minute")
+async def dm_count(
+    request: Request,
+    agent_id: str = "",
+    agent_token: str = "",
+    agent_token_prev: str = "",
+    agent_tokens: str = "",
+):
+    import main as _m
+
+    return await _m.dm_count(
+        request,
+        agent_id=agent_id,
+        agent_token=agent_token,
+        agent_token_prev=agent_token_prev,
+        agent_tokens=agent_tokens,
+    )
+
+
+@router.post("/api/mesh/dm/block")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_BLOCK)
+async def dm_block(request: Request):
+    """Block or unblock a sender from DMing you."""
+    body = _signed_body(request)
+    agent_id = body.get("agent_id", "").strip()
+    blocked_id = body.get("blocked_id", "").strip()
+    action = body.get("action", "block").strip().lower()
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not agent_id or not blocked_id:
+        return {"ok": False, "detail": "Missing agent_id or blocked_id"}
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    try:
+        from services.mesh.mesh_hashchain import infonet
+
+        ok_seq, seq_reason = _validate_private_signed_sequence(
+            infonet,
+            agent_id,
+            sequence,
+            domain=f"dm_block:{action}",
+        )
+        if not ok_seq:
+            return {"ok": False, "detail": seq_reason}
+    except Exception:
+        pass
+
+    if action == "unblock":
+        dm_relay.unblock(agent_id, blocked_id)
+    else:
+        dm_relay.block(agent_id, blocked_id)
+    return {"ok": True, "action": action, "blocked_id": blocked_id}
+
+
+@router.post("/api/mesh/dm/witness")
+@limiter.limit("20/minute")
+@requires_signed_write(kind=SignedWriteKind.DM_WITNESS)
+async def dm_key_witness(request: Request):
+    """Record a lightweight witness for a DM key (dual-path spot-check)."""
+    body = _signed_body(request)
+    witness_id = body.get("witness_id", "").strip()
+    target_id = body.get("target_id", "").strip()
+    dh_pub_key = body.get("dh_pub_key", "").strip()
+    timestamp = _safe_int(body.get("timestamp", 0) or 0)
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not witness_id or not target_id or not dh_pub_key or not timestamp:
+        return {"ok": False, "detail": "Missing witness_id, target_id, dh_pub_key, or timestamp"}
+    now_ts = int(time.time())
+    if abs(timestamp - now_ts) > 7 * 86400:
+        return {"ok": False, "detail": "Witness timestamp is too far from current time"}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+
+        reputation_ledger.register_node(witness_id, public_key, public_key_algo)
+    except Exception:
+        pass
+    try:
+        from services.mesh.mesh_hashchain import infonet
+
+        ok_seq, seq_reason = _validate_private_signed_sequence(
+            infonet,
+            witness_id,
+            sequence,
+            domain="dm_witness",
+        )
+        if not ok_seq:
+            return {"ok": False, "detail": seq_reason}
+    except Exception:
+        pass
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    ok, reason = dm_relay.record_witness(witness_id, target_id, dh_pub_key, timestamp)
+    return {"ok": ok, "detail": reason}
+
+
+@router.get("/api/mesh/dm/witness")
+@limiter.limit("60/minute")
+async def dm_key_witness_get(request: Request, target_id: str = "", dh_pub_key: str = ""):
+    """Get witness counts for a target's DH key."""
+    if not target_id:
+        return {"ok": False, "detail": "Missing target_id"}
+    from services.mesh.mesh_dm_relay import dm_relay
+
+    witnesses = dm_relay.get_witnesses(target_id, dh_pub_key if dh_pub_key else None, limit=5)
+    response = {
+        "ok": True,
+        "count": len(witnesses),
+    }
+    if _scoped_view_authenticated(request, "mesh.audit"):
+        response["target_id"] = target_id
+        response["dh_pub_key"] = dh_pub_key or ""
+        response["witnesses"] = witnesses
+    return response
+
+
+@router.post("/api/mesh/trust/vouch")
+@limiter.limit("20/minute")
+@requires_signed_write(kind=SignedWriteKind.TRUST_VOUCH)
+async def trust_vouch(request: Request):
+    """Record a trust vouch for a node (web-of-trust signal)."""
+    body = _signed_body(request)
+    voucher_id = body.get("voucher_id", "").strip()
+    target_id = body.get("target_id", "").strip()
+    note = body.get("note", "").strip()
+    timestamp = _safe_int(body.get("timestamp", 0) or 0)
+    public_key = body.get("public_key", "").strip()
+    public_key_algo = body.get("public_key_algo", "").strip()
+    signature = body.get("signature", "").strip()
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "").strip()
+    if not voucher_id or not target_id or not timestamp:
+        return {"ok": False, "detail": "Missing voucher_id, target_id, or timestamp"}
+    now_ts = int(time.time())
+    if abs(timestamp - now_ts) > 7 * 86400:
+        return {"ok": False, "detail": "Vouch timestamp is too far from current time"}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+        from services.mesh.mesh_hashchain import infonet
+
+        reputation_ledger.register_node(voucher_id, public_key, public_key_algo)
+        ok_seq, seq_reason = _validate_private_signed_sequence(
+            infonet,
+            voucher_id,
+            sequence,
+            domain="trust_vouch",
+        )
+        if not ok_seq:
+            return {"ok": False, "detail": seq_reason}
+        ok, reason = reputation_ledger.add_vouch(voucher_id, target_id, note, timestamp)
+        return {"ok": ok, "detail": reason}
+    except Exception:
+        return {"ok": False, "detail": "Failed to record vouch"}
+
+
+@router.get("/api/mesh/trust/vouches", dependencies=[Depends(require_admin)])
+@limiter.limit("60/minute")
+async def trust_vouches(request: Request, node_id: str = "", limit: int = 20):
+    """Fetch latest vouches for a node."""
+    if not node_id:
+        return {"ok": False, "detail": "Missing node_id"}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+
+        vouches = reputation_ledger.get_vouches(node_id, limit=limit)
+        return {"ok": True, "node_id": node_id, "vouches": vouches, "count": len(vouches)}
+    except Exception:
+        return {"ok": False, "detail": "Failed to fetch vouches"}

backend/routers/mesh_operator.py

@@ -0,0 +1,145 @@
+import time
+import logging
+from fastapi import APIRouter, Request, Response, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def list_peers(request: Request, bucket: str = Query(None)):
+    """List all peers (or filter by bucket: sync, push, bootstrap)."""
+    from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerStore
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception as exc:
+        return {"ok": False, "detail": f"Failed to load peer store: {exc}"}
+    if bucket:
+        records = store.records_for_bucket(bucket)
+    else:
+        records = store.records()
+    return {"ok": True, "count": len(records), "peers": [r.to_dict() for r in records]}
+
+
+@router.post("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def add_peer(request: Request):
+    """Add a peer to the store. Body: {peer_url, transport?, label?, role?, buckets?[]}."""
+    from services.mesh.mesh_crypto import normalize_peer_url
+    from services.mesh.mesh_peer_store import (
+        DEFAULT_PEER_STORE_PATH, PeerStore, PeerStoreError,
+        make_push_peer_record, make_sync_peer_record,
+    )
+    from services.mesh.mesh_router import peer_transport_kind
+    body = await request.json()
+    peer_url_raw = str(body.get("peer_url", "") or "").strip()
+    if not peer_url_raw:
+        return {"ok": False, "detail": "peer_url is required"}
+    peer_url = normalize_peer_url(peer_url_raw)
+    if not peer_url:
+        return {"ok": False, "detail": "Invalid peer_url"}
+    transport = str(body.get("transport", "") or "").strip().lower()
+    if not transport:
+        transport = peer_transport_kind(peer_url)
+    if not transport:
+        return {"ok": False, "detail": "Cannot determine transport for peer_url — provide transport explicitly"}
+    label = str(body.get("label", "") or "").strip()
+    role = str(body.get("role", "") or "").strip().lower() or "relay"
+    buckets = body.get("buckets", ["sync", "push"])
+    if isinstance(buckets, str):
+        buckets = [buckets]
+    if not isinstance(buckets, list):
+        buckets = ["sync", "push"]
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    added: list = []
+    try:
+        for b in buckets:
+            b = str(b).strip().lower()
+            if b == "sync":
+                store.upsert(make_sync_peer_record(peer_url=peer_url, transport=transport, role=role, label=label))
+                added.append("sync")
+            elif b == "push":
+                store.upsert(make_push_peer_record(peer_url=peer_url, transport=transport, role=role, label=label))
+                added.append("push")
+        store.save()
+    except PeerStoreError as exc:
+        return {"ok": False, "detail": str(exc)}
+    return {"ok": True, "peer_url": peer_url, "buckets": added}
+
+
+@router.delete("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def remove_peer(request: Request):
+    """Remove a peer. Body: {peer_url, bucket?}. If bucket omitted, removes from all buckets."""
+    from services.mesh.mesh_crypto import normalize_peer_url
+    from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerStore
+    body = await request.json()
+    peer_url_raw = str(body.get("peer_url", "") or "").strip()
+    if not peer_url_raw:
+        return {"ok": False, "detail": "peer_url is required"}
+    peer_url = normalize_peer_url(peer_url_raw)
+    if not peer_url:
+        return {"ok": False, "detail": "Invalid peer_url"}
+    bucket_filter = str(body.get("bucket", "") or "").strip().lower()
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        return {"ok": False, "detail": "Failed to load peer store"}
+    removed: list = []
+    for b in ["bootstrap", "sync", "push"]:
+        if bucket_filter and b != bucket_filter:
+            continue
+        key = f"{b}:{peer_url}"
+        if key in store._records:
+            del store._records[key]
+            removed.append(b)
+    if not removed:
+        return {"ok": False, "detail": "Peer not found in any bucket"}
+    store.save()
+    return {"ok": True, "peer_url": peer_url, "removed_from": removed}
+
+
+@router.patch("/api/mesh/peers", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def toggle_peer(request: Request):
+    """Enable or disable a peer. Body: {peer_url, bucket, enabled: bool}."""
+    from services.mesh.mesh_crypto import normalize_peer_url
+    from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerRecord, PeerStore
+    body = await request.json()
+    peer_url_raw = str(body.get("peer_url", "") or "").strip()
+    bucket = str(body.get("bucket", "") or "").strip().lower()
+    enabled = body.get("enabled")
+    if not peer_url_raw:
+        return {"ok": False, "detail": "peer_url is required"}
+    if not bucket:
+        return {"ok": False, "detail": "bucket is required"}
+    if enabled is None:
+        return {"ok": False, "detail": "enabled (true/false) is required"}
+    peer_url = normalize_peer_url(peer_url_raw)
+    if not peer_url:
+        return {"ok": False, "detail": "Invalid peer_url"}
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        return {"ok": False, "detail": "Failed to load peer store"}
+    key = f"{bucket}:{peer_url}"
+    record = store._records.get(key)
+    if not record:
+        return {"ok": False, "detail": f"Peer not found in {bucket} bucket"}
+    updated = PeerRecord(**{**record.to_dict(), "enabled": bool(enabled), "updated_at": int(time.time())})
+    store._records[key] = updated
+    store.save()
+    return {"ok": True, "peer_url": peer_url, "bucket": bucket, "enabled": bool(enabled)}

backend/routers/mesh_oracle.py

@@ -0,0 +1,354 @@
+import math
+from typing import Any
+from fastapi import APIRouter, Request, Response, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator, _scoped_view_authenticated
+from services.data_fetcher import get_latest_data
+from services.mesh.mesh_protocol import normalize_payload
+from services.mesh.mesh_signed_events import (
+    MeshWriteExemption,
+    SignedWriteKind,
+    get_prepared_signed_write,
+    mesh_write_exempt,
+    requires_signed_write,
+)
+
+router = APIRouter()
+
+
+def _signed_body(request: Request) -> dict[str, Any]:
+    prepared = get_prepared_signed_write(request)
+    if prepared is None:
+        return {}
+    return dict(prepared.body)
+
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _safe_float(val, default=0.0):
+    try:
+        parsed = float(val)
+        if not math.isfinite(parsed):
+            return default
+        return parsed
+    except (TypeError, ValueError):
+        return default
+
+
+def _redact_public_oracle_profile(payload: dict, authenticated: bool) -> dict:
+    redacted = dict(payload)
+    if authenticated:
+        return redacted
+    redacted["active_stakes"] = []
+    redacted["prediction_history"] = []
+    return redacted
+
+
+def _redact_public_oracle_predictions(predictions: list, authenticated: bool) -> dict:
+    if authenticated:
+        return {"predictions": list(predictions)}
+    return {"predictions": [], "count": len(predictions)}
+
+
+def _redact_public_oracle_stakes(payload: dict, authenticated: bool) -> dict:
+    redacted = dict(payload)
+    if authenticated:
+        return redacted
+    redacted["truth_stakers"] = []
+    redacted["false_stakers"] = []
+    return redacted
+
+
+@router.post("/api/mesh/oracle/predict")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.ORACLE_PREDICT)
+async def oracle_predict(request: Request):
+    """Place a prediction on a market outcome."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    body = _signed_body(request)
+    node_id = body.get("node_id", "")
+    market_title = body.get("market_title", "")
+    side = body.get("side", "")
+    stake_amount = _safe_float(body.get("stake_amount", 0))
+    public_key = body.get("public_key", "")
+    public_key_algo = body.get("public_key_algo", "")
+    signature = body.get("signature", "")
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "")
+    if not node_id or not market_title or not side:
+        return {"ok": False, "detail": "Missing node_id, market_title, or side"}
+    prediction_payload = {"market_title": market_title, "side": side, "stake_amount": stake_amount}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+        reputation_ledger.register_node(node_id, public_key, public_key_algo)
+    except Exception:
+        pass
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    matched = None
+    for m in markets:
+        if m.get("title", "").lower() == market_title.lower():
+            matched = m
+            break
+    if not matched:
+        for m in markets:
+            if market_title.lower() in m.get("title", "").lower():
+                matched = m
+                break
+    if not matched:
+        return {"ok": False, "detail": f"Market '{market_title}' not found in active markets."}
+    probability = 50.0
+    side_lower = side.lower()
+    outcomes = matched.get("outcomes", [])
+    if outcomes:
+        for o in outcomes:
+            if o.get("name", "").lower() == side_lower:
+                probability = float(o.get("pct", 50))
+                break
+    else:
+        consensus = matched.get("consensus_pct")
+        if consensus is None:
+            consensus = matched.get("polymarket_pct") or matched.get("kalshi_pct") or 50
+        probability = float(consensus)
+        if side_lower == "no":
+            probability = 100.0 - probability
+    if stake_amount > 0:
+        ok, detail = oracle_ledger.place_market_stake(node_id, matched["title"], side, stake_amount, probability)
+        mode = "staked"
+    else:
+        ok, detail = oracle_ledger.place_prediction(node_id, matched["title"], side, probability)
+        mode = "free"
+    if ok:
+        try:
+            from services.mesh.mesh_hashchain import infonet
+            normalized_payload = normalize_payload("prediction", prediction_payload)
+            infonet.append(event_type="prediction", node_id=node_id, payload=normalized_payload,
+                signature=signature, sequence=sequence, public_key=public_key,
+                public_key_algo=public_key_algo, protocol_version=protocol_version)
+        except Exception:
+            pass
+    return {"ok": ok, "detail": detail, "probability": probability, "mode": mode}
+
+
+@router.get("/api/mesh/oracle/markets")
+@limiter.limit("30/minute")
+async def oracle_markets(request: Request):
+    """List active prediction markets."""
+    from collections import defaultdict
+    from services.mesh.mesh_oracle import oracle_ledger
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    all_consensus = oracle_ledger.get_all_market_consensus()
+    by_category = defaultdict(list)
+    for m in markets:
+        by_category[m.get("category", "NEWS")].append(m)
+    _fields = ("title", "consensus_pct", "polymarket_pct", "kalshi_pct", "volume", "volume_24h",
+               "end_date", "description", "category", "sources", "slug", "kalshi_ticker", "outcomes")
+    categories = {}
+    cat_totals = {}
+    for cat in ["POLITICS", "CONFLICT", "NEWS", "FINANCE", "CRYPTO"]:
+        all_cat = sorted(by_category.get(cat, []), key=lambda x: x.get("volume", 0) or 0, reverse=True)
+        cat_totals[cat] = len(all_cat)
+        cat_list = []
+        for m in all_cat[:10]:
+            entry = {k: m.get(k) for k in _fields}
+            entry["consensus"] = all_consensus.get(m.get("title", ""), {})
+            cat_list.append(entry)
+        categories[cat] = cat_list
+    return {"categories": categories, "total_count": len(markets), "cat_totals": cat_totals}
+
+
+@router.get("/api/mesh/oracle/search")
+@limiter.limit("20/minute")
+async def oracle_search(request: Request, q: str = "", limit: int = 50):
+    """Search prediction markets across Polymarket + Kalshi APIs."""
+    if not q or len(q) < 2:
+        return {"results": [], "query": q, "count": 0}
+    from services.fetchers.prediction_markets import search_polymarket_direct, search_kalshi_direct
+    import concurrent.futures
+    # Search both APIs in parallel for speed
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
+        poly_fut = pool.submit(search_polymarket_direct, q, limit)
+        kalshi_fut = pool.submit(search_kalshi_direct, q, limit)
+        poly_results = poly_fut.result(timeout=20)
+        kalshi_results = kalshi_fut.result(timeout=20)
+    # Also check cached/merged markets
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    q_lower = q.lower()
+    cached_matches = [m for m in markets if q_lower in m.get("title", "").lower()]
+    seen_titles = set()
+    combined = []
+    # Cached first (already merged Poly+Kalshi with consensus)
+    for m in cached_matches:
+        seen_titles.add(m["title"].lower())
+        combined.append(m)
+    # Then Polymarket direct hits
+    for m in poly_results:
+        if m["title"].lower() not in seen_titles:
+            seen_titles.add(m["title"].lower())
+            combined.append(m)
+    # Then Kalshi direct hits
+    for m in kalshi_results:
+        if m["title"].lower() not in seen_titles:
+            seen_titles.add(m["title"].lower())
+            combined.append(m)
+    combined.sort(key=lambda x: x.get("volume", 0) or 0, reverse=True)
+    _fields = ("title", "consensus_pct", "polymarket_pct", "kalshi_pct", "volume", "volume_24h",
+               "end_date", "description", "category", "sources", "slug", "kalshi_ticker", "outcomes")
+    results = [{k: m.get(k) for k in _fields} for m in combined[:limit]]
+    return {"results": results, "query": q, "count": len(results)}
+
+
+@router.get("/api/mesh/oracle/markets/more")
+@limiter.limit("30/minute")
+async def oracle_markets_more(request: Request, category: str = "NEWS", offset: int = 0, limit: int = 10):
+    """Load more markets for a specific category (paginated)."""
+    data = get_latest_data()
+    markets = data.get("prediction_markets", [])
+    cat_markets = sorted([m for m in markets if m.get("category") == category],
+        key=lambda x: x.get("volume", 0) or 0, reverse=True)
+    page = cat_markets[offset : offset + limit]
+    _fields = ("title", "consensus_pct", "polymarket_pct", "kalshi_pct", "volume", "volume_24h",
+               "end_date", "description", "category", "sources", "slug", "kalshi_ticker", "outcomes")
+    results = [{k: m.get(k) for k in _fields} for m in page]
+    return {"markets": results, "category": category, "offset": offset,
+            "has_more": offset + limit < len(cat_markets), "total": len(cat_markets)}
+
+
+@router.post(
+    "/api/mesh/oracle/resolve",
+    dependencies=[Depends(require_admin)],
+)
+@limiter.limit("5/minute")
+@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
+async def oracle_resolve(request: Request):
+    """Resolve a prediction market.
+
+    Issue #240 (tg12): requires admin authentication. The
+    ``mesh_write_exempt`` decorator below is **metadata only** — it tags
+    the route as not requiring a mesh signed-write envelope, it does
+    NOT itself enforce caller authorization. The ``Depends(require_admin)``
+    on the route decorator is what actually gates access.
+    """
+    from services.mesh.mesh_oracle import oracle_ledger
+    body = await request.json()
+    market_title = body.get("market_title", "")
+    outcome = body.get("outcome", "")
+    if not market_title or not outcome:
+        return {"ok": False, "detail": "Need market_title and outcome"}
+    winners, losers = oracle_ledger.resolve_market(market_title, outcome)
+    stake_result = oracle_ledger.resolve_market_stakes(market_title, outcome)
+    return {"ok": True,
+            "detail": f"Resolved: {winners} free winners, {losers} free losers, "
+                      f"{stake_result.get('winners', 0)} stake winners, {stake_result.get('losers', 0)} stake losers",
+            "free": {"winners": winners, "losers": losers}, "stakes": stake_result}
+
+
+@router.get("/api/mesh/oracle/consensus")
+@limiter.limit("30/minute")
+async def oracle_consensus(request: Request, market_title: str = ""):
+    """Get network consensus for a market."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    if not market_title:
+        return {"error": "market_title required"}
+    return oracle_ledger.get_market_consensus(market_title)
+
+
+@router.post("/api/mesh/oracle/stake")
+@limiter.limit("10/minute")
+@requires_signed_write(kind=SignedWriteKind.ORACLE_STAKE)
+async def oracle_stake(request: Request):
+    """Stake oracle rep on a post's truthfulness."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    body = _signed_body(request)
+    staker_id = body.get("staker_id", "")
+    message_id = body.get("message_id", "")
+    poster_id = body.get("poster_id", "")
+    side = body.get("side", "").lower()
+    amount = _safe_float(body.get("amount", 0))
+    duration_days = _safe_int(body.get("duration_days", 1), 1)
+    public_key = body.get("public_key", "")
+    public_key_algo = body.get("public_key_algo", "")
+    signature = body.get("signature", "")
+    sequence = _safe_int(body.get("sequence", 0) or 0)
+    protocol_version = body.get("protocol_version", "")
+    if not staker_id or not message_id or not side:
+        return {"ok": False, "detail": "Missing staker_id, message_id, or side"}
+    stake_payload = {"message_id": message_id, "poster_id": poster_id, "side": side,
+                     "amount": amount, "duration_days": duration_days}
+    try:
+        from services.mesh.mesh_reputation import reputation_ledger
+        reputation_ledger.register_node(staker_id, public_key, public_key_algo)
+    except Exception:
+        pass
+    ok, detail = oracle_ledger.place_stake(staker_id, message_id, poster_id, side, amount, duration_days)
+    if ok:
+        try:
+            from services.mesh.mesh_hashchain import infonet
+            normalized_payload = normalize_payload("stake", stake_payload)
+            infonet.append(event_type="stake", node_id=staker_id, payload=normalized_payload,
+                signature=signature, sequence=sequence, public_key=public_key,
+                public_key_algo=public_key_algo, protocol_version=protocol_version)
+        except Exception:
+            pass
+    return {"ok": ok, "detail": detail}
+
+
+@router.get("/api/mesh/oracle/stakes/{message_id}")
+@limiter.limit("30/minute")
+async def oracle_stakes_for_message(request: Request, message_id: str):
+    """Get all oracle stakes on a message."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    return _redact_public_oracle_stakes(
+        oracle_ledger.get_stakes_for_message(message_id),
+        authenticated=_scoped_view_authenticated(request, "mesh.audit"),
+    )
+
+
+@router.get("/api/mesh/oracle/profile")
+@limiter.limit("30/minute")
+async def oracle_profile(request: Request, node_id: str = ""):
+    """Get full oracle profile."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    if not node_id:
+        return {"ok": False, "detail": "Provide ?node_id=xxx"}
+    profile = oracle_ledger.get_oracle_profile(node_id)
+    return _redact_public_oracle_profile(
+        profile, authenticated=_scoped_view_authenticated(request, "mesh.audit"))
+
+
+@router.get("/api/mesh/oracle/predictions")
+@limiter.limit("30/minute")
+async def oracle_predictions(request: Request, node_id: str = ""):
+    """Get a node's active (unresolved) predictions."""
+    from services.mesh.mesh_oracle import oracle_ledger
+    if not node_id:
+        return {"ok": False, "detail": "Provide ?node_id=xxx"}
+    active_predictions = oracle_ledger.get_active_predictions(node_id)
+    return _redact_public_oracle_predictions(
+        active_predictions, authenticated=_scoped_view_authenticated(request, "mesh.audit"))
+
+
+@router.post(
+    "/api/mesh/oracle/resolve-stakes",
+    dependencies=[Depends(require_admin)],
+)
+@limiter.limit("5/minute")
+@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
+async def oracle_resolve_stakes(request: Request):
+    """Resolve all expired stake contests.
+
+    Issue #241 (tg12): requires admin authentication. See the note on
+    ``oracle_resolve`` above — ``mesh_write_exempt`` is metadata only.
+    """
+    from services.mesh.mesh_oracle import oracle_ledger
+    resolutions = oracle_ledger.resolve_expired_stakes()
+    return {"ok": True, "resolutions": resolutions, "count": len(resolutions)}

backend/routers/mesh_peer_sync.py

@@ -0,0 +1,300 @@
+import json as json_mod
+import logging
+from typing import Any
+from fastapi import APIRouter, Request, Response
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator, _verify_peer_push_hmac
+from services.config import get_settings
+from services.mesh.mesh_crypto import normalize_peer_url
+from services.mesh.mesh_router import peer_transport_kind
+from auth import _peer_hmac_url_from_request
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+_PEER_PUSH_BATCH_SIZE = 50
+
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _hydrate_gate_store_from_chain(events: list) -> int:
+    """Copy any gate_message chain events into the local gate_store for read/decrypt.
+
+    Only events that are resident in the local infonet (accepted or already
+    present) are hydrated.  The canonical infonet-resident event is used —
+    never the raw batch event — so a forged batch entry carrying a valid
+    event_id but attacker-chosen payload cannot pollute gate_store.
+    """
+    import copy
+    from services.mesh.mesh_hashchain import gate_store, infonet
+    count = 0
+    for evt in events:
+        if evt.get("event_type") != "gate_message":
+            continue
+        event_id = str(evt.get("event_id", "") or "").strip()
+        if not event_id or event_id not in infonet.event_index:
+            continue
+        canonical = infonet.events[infonet.event_index[event_id]]
+        payload = canonical.get("payload") or {}
+        gate_id = str(payload.get("gate", "") or "").strip()
+        if not gate_id:
+            continue
+        try:
+            gate_store.append(gate_id, copy.deepcopy(canonical))
+            count += 1
+        except Exception:
+            pass
+    return count
+
+
+def _hydrate_dm_relay_from_chain(events: list) -> int:
+    import main as _m
+
+    return int(_m._hydrate_dm_relay_from_chain(events))
+
+
+@router.post("/api/mesh/infonet/peer-push")
+@limiter.limit("30/minute")
+async def infonet_peer_push(request: Request):
+    """Accept pushed Infonet events from relay peers (HMAC-authenticated)."""
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 524_288:
+                return Response(content='{"ok":false,"detail":"Request body too large (max 512KB)"}',
+                    status_code=413, media_type="application/json")
+        except (ValueError, TypeError):
+            pass
+    from services.mesh.mesh_hashchain import infonet
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json")
+    body = json_mod.loads(body_bytes or b"{}")
+    events = body.get("events", [])
+    if not isinstance(events, list):
+        return {"ok": False, "detail": "events must be a list"}
+    if len(events) > 50:
+        return {"ok": False, "detail": "Too many events in one push (max 50)"}
+    if not events:
+        return {"ok": True, "accepted": 0, "duplicates": 0, "rejected": []}
+    result = infonet.ingest_events(events)
+    _hydrate_gate_store_from_chain(events)
+    _hydrate_dm_relay_from_chain(events)
+    return {"ok": True, **result}
+
+
+@router.post("/api/mesh/dm/replicate-envelope")
+@limiter.limit("60/minute")
+async def dm_replicate_envelope(request: Request):
+    """Accept a DM envelope replicated from a peer relay (cross-node mailbox).
+
+    Companion endpoint to ``DMRelay.replicate_to_peers`` (outbound, in
+    ``mesh_dm_relay.py``). The sender's relay POSTs an encrypted DM
+    envelope here after a successful local ``deposit``; this endpoint
+    re-enforces the per-(sender, recipient) anti-spam cap and stores
+    the envelope in the local mailbox if accepted.
+
+    The cap is the network rule: a hostile sender's relay can spool
+    extras locally, but every honest peer enforces the cap on inbound
+    replication. Recipient polling from any honest peer therefore
+    never sees more than ``MESH_DM_PENDING_PER_SENDER_LIMIT`` pending
+    from any one sender, no matter how many spam attempts were tried.
+
+    Same HMAC auth pattern as ``infonet_peer_push`` and ``gate_peer_push``.
+    """
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            # DM envelopes are bounded by MESH_DM_MAX_MSG_BYTES + envelope
+            # overhead; 64 KB is a generous ceiling.
+            if int(content_length) > 65_536:
+                return Response(
+                    content='{"ok":false,"detail":"Request body too large (max 64KB)"}',
+                    status_code=413, media_type="application/json",
+                )
+        except (ValueError, TypeError):
+            pass
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(
+            content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json",
+        )
+    try:
+        body = json_mod.loads(body_bytes or b"{}")
+    except (ValueError, TypeError):
+        return Response(
+            content='{"ok":false,"detail":"Invalid JSON body"}',
+            status_code=400, media_type="application/json",
+        )
+    envelope = body.get("envelope")
+    if not isinstance(envelope, dict):
+        return {"ok": False, "detail": "envelope must be an object"}
+
+    originating_peer = _peer_hmac_url_from_request(request) or ""
+
+    from services.mesh.mesh_dm_relay import dm_relay
+    result = dm_relay.accept_replica(
+        envelope=envelope,
+        originating_peer_url=originating_peer,
+    )
+    return result
+
+
+@router.post("/api/mesh/gate/peer-push")
+@limiter.limit("30/minute")
+async def gate_peer_push(request: Request):
+    """Accept pushed gate events from relay peers (private plane)."""
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 524_288:
+                return Response(content='{"ok":false,"detail":"Request body too large"}',
+                    status_code=413, media_type="application/json")
+        except (ValueError, TypeError):
+            pass
+    from services.mesh.mesh_hashchain import gate_store
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json")
+    body = json_mod.loads(body_bytes or b"{}")
+    events = body.get("events", [])
+    if not isinstance(events, list):
+        return {"ok": False, "detail": "events must be a list"}
+    if len(events) > 50:
+        return {"ok": False, "detail": "Too many events (max 50)"}
+    if not events:
+        return {"ok": True, "accepted": 0, "duplicates": 0}
+    from services.mesh.mesh_hashchain import resolve_gate_wire_ref
+    # Sprint 3 / Rec #4: the gate_ref is HMACed with a key bound to the
+    # receiver's peer URL (the URL the push was delivered to). This is
+    # the same URL _verify_peer_push_hmac validated the X-Peer-HMAC
+    # header against, so we can trust it for ref resolution.
+    hop_peer_url = _peer_hmac_url_from_request(request)
+    grouped_events: dict[str, list] = {}
+    for evt in events:
+        evt_dict = evt if isinstance(evt, dict) else {}
+        payload = evt_dict.get("payload")
+        if not isinstance(payload, dict):
+            payload = {}
+        clean_event = {
+            "event_id": str(evt_dict.get("event_id", "") or ""),
+            "event_type": "gate_message",
+            "timestamp": evt_dict.get("timestamp", 0),
+            "node_id": str(evt_dict.get("node_id", "") or evt_dict.get("sender_id", "") or ""),
+            "sequence": evt_dict.get("sequence", 0),
+            "signature": str(evt_dict.get("signature", "") or ""),
+            "public_key": str(evt_dict.get("public_key", "") or ""),
+            "public_key_algo": str(evt_dict.get("public_key_algo", "") or ""),
+            "protocol_version": str(evt_dict.get("protocol_version", "") or ""),
+            "payload": {
+                "ciphertext": str(payload.get("ciphertext", "") or ""),
+                "format": str(payload.get("format", "") or ""),
+                "nonce": str(payload.get("nonce", "") or ""),
+                "sender_ref": str(payload.get("sender_ref", "") or ""),
+            },
+        }
+        epoch = _safe_int(payload.get("epoch", 0) or 0)
+        if epoch > 0:
+            clean_event["payload"]["epoch"] = epoch
+        envelope_hash_val = str(payload.get("envelope_hash", "") or "").strip()
+        gate_envelope_val = str(payload.get("gate_envelope", "") or "").strip()
+        reply_to_val = str(payload.get("reply_to", "") or "").strip()
+        if envelope_hash_val:
+            clean_event["payload"]["envelope_hash"] = envelope_hash_val
+        if gate_envelope_val:
+            clean_event["payload"]["gate_envelope"] = gate_envelope_val
+        if reply_to_val:
+            clean_event["payload"]["reply_to"] = reply_to_val
+        event_gate_id = str(payload.get("gate", "") or evt_dict.get("gate", "") or "").strip().lower()
+        if not event_gate_id:
+            event_gate_id = resolve_gate_wire_ref(
+                str(payload.get("gate_ref", "") or evt_dict.get("gate_ref", "") or ""),
+                clean_event,
+                peer_url=hop_peer_url,
+            )
+        if not event_gate_id:
+            return {"ok": False, "detail": "gate resolution failed"}
+        final_payload: dict[str, Any] = {
+            "gate": event_gate_id,
+            "ciphertext": clean_event["payload"]["ciphertext"],
+            "format": clean_event["payload"]["format"],
+            "nonce": clean_event["payload"]["nonce"],
+            "sender_ref": clean_event["payload"]["sender_ref"],
+        }
+        if epoch > 0:
+            final_payload["epoch"] = epoch
+        if clean_event["payload"].get("envelope_hash"):
+            final_payload["envelope_hash"] = clean_event["payload"]["envelope_hash"]
+        if clean_event["payload"].get("gate_envelope"):
+            final_payload["gate_envelope"] = clean_event["payload"]["gate_envelope"]
+        if clean_event["payload"].get("reply_to"):
+            final_payload["reply_to"] = clean_event["payload"]["reply_to"]
+        grouped_events.setdefault(event_gate_id, []).append({
+            "event_id": clean_event["event_id"],
+            "event_type": "gate_message",
+            "timestamp": clean_event["timestamp"],
+            "node_id": clean_event["node_id"],
+            "sequence": clean_event["sequence"],
+            "signature": clean_event["signature"],
+            "public_key": clean_event["public_key"],
+            "public_key_algo": clean_event["public_key_algo"],
+            "protocol_version": clean_event["protocol_version"],
+            "payload": final_payload,
+        })
+    accepted = 0
+    duplicates = 0
+    rejected = 0
+    for event_gate_id, items in grouped_events.items():
+        result = gate_store.ingest_peer_events(event_gate_id, items)
+        a = int(result.get("accepted", 0) or 0)
+        accepted += a
+        duplicates += int(result.get("duplicates", 0) or 0)
+        rejected += int(result.get("rejected", 0) or 0)
+    return {"ok": True, "accepted": accepted, "duplicates": duplicates, "rejected": rejected}
+
+
+@router.post("/api/mesh/gate/peer-pull")
+@limiter.limit("30/minute")
+async def gate_peer_pull(request: Request):
+    """Return gate events a peer is missing (HMAC-authenticated pull sync)."""
+    content_length = request.headers.get("content-length")
+    if content_length:
+        try:
+            if int(content_length) > 65_536:
+                return Response(content='{"ok":false,"detail":"Request body too large"}',
+                    status_code=413, media_type="application/json")
+        except (ValueError, TypeError):
+            pass
+    from services.mesh.mesh_hashchain import gate_store
+    body_bytes = await request.body()
+    if not _verify_peer_push_hmac(request, body_bytes):
+        return Response(content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
+            status_code=403, media_type="application/json")
+    body = json_mod.loads(body_bytes or b"{}")
+    gate_id = str(body.get("gate_id", "") or "").strip().lower()
+    after_count = _safe_int(body.get("after_count", 0) or 0)
+    if not gate_id:
+        gate_ids = gate_store.known_gate_ids()
+        gate_counts: dict[str, int] = {}
+        for gid in gate_ids:
+            with gate_store._lock:
+                gate_counts[gid] = len(gate_store._gates.get(gid, []))
+        return {"ok": True, "gates": gate_counts}
+    with gate_store._lock:
+        all_events = list(gate_store._gates.get(gate_id, []))
+    total = len(all_events)
+    if after_count >= total:
+        return {"ok": True, "events": [], "total": total, "gate_id": gate_id}
+    batch = all_events[after_count : after_count + _PEER_PUSH_BATCH_SIZE]
+    return {"ok": True, "events": batch, "total": total, "gate_id": gate_id}

backend/routers/radio.py

@@ -0,0 +1,107 @@
+from fastapi import APIRouter, Request, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+
+router = APIRouter()
+
+
+@router.get("/api/radio/top")
+@limiter.limit("30/minute")
+async def get_top_radios(request: Request):
+    from services.radio_intercept import get_top_broadcastify_feeds
+    return get_top_broadcastify_feeds()
+
+
+@router.get("/api/radio/openmhz/systems")
+@limiter.limit("30/minute")
+async def api_get_openmhz_systems(request: Request):
+    from services.radio_intercept import get_openmhz_systems
+    return get_openmhz_systems()
+
+
+# Issue #213: rotating sys_name bypasses the 20s TTL cache and lets an
+# anonymous caller hammer api.openmhz.com through this proxy, risking an
+# IP-ban for the project. require_local_operator scopes this to the local
+# UI (which goes through the Next.js proxy with admin-key injection) and
+# scoped agent tokens.
+@router.get(
+    "/api/radio/openmhz/calls/{sys_name}",
+    dependencies=[Depends(require_local_operator)],
+)
+@limiter.limit("60/minute")
+async def api_get_openmhz_calls(request: Request, sys_name: str):
+    from services.radio_intercept import get_recent_openmhz_calls
+    return get_recent_openmhz_calls(sys_name)
+
+
+# Issue #214: this is a streaming bandwidth relay. An anonymous caller can
+# stream audio through the backend, saturating the operator's outbound
+# bandwidth. Scope to local operator; the legitimate browser UI still
+# works because relative /api/... paths go through the Next.js proxy
+# which injects the admin key automatically.
+@router.get(
+    "/api/radio/openmhz/audio",
+    dependencies=[Depends(require_local_operator)],
+)
+@limiter.limit("120/minute")
+async def api_get_openmhz_audio(request: Request, url: str = Query(..., min_length=10)):
+    from services.radio_intercept import openmhz_audio_response
+    return openmhz_audio_response(url)
+
+
+@router.get("/api/radio/nearest")
+@limiter.limit("60/minute")
+async def api_get_nearest_radio(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    from services.radio_intercept import find_nearest_openmhz_system
+    return find_nearest_openmhz_system(lat, lng)
+
+
+@router.get("/api/radio/nearest-list")
+@limiter.limit("60/minute")
+async def api_get_nearest_radios_list(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+    limit: int = Query(5, ge=1, le=20),
+):
+    from services.radio_intercept import find_nearest_openmhz_systems_list
+    return find_nearest_openmhz_systems_list(lat, lng, limit=limit)
+
+
+@router.get("/api/route/{callsign}")
+@limiter.limit("60/minute")
+async def get_flight_route(request: Request, callsign: str, lat: float = 0.0, lng: float = 0.0):
+    from services.network_utils import fetch_with_curl
+    r = fetch_with_curl(
+        "https://api.adsb.lol/api/0/routeset",
+        method="POST",
+        json_data={"planes": [{"callsign": callsign, "lat": lat, "lng": lng}]},
+        timeout=10,
+    )
+    if r and r.status_code == 200:
+        data = r.json()
+        route_list = []
+        if isinstance(data, dict):
+            route_list = data.get("value", [])
+        elif isinstance(data, list):
+            route_list = data
+
+        if route_list and len(route_list) > 0:
+            route = route_list[0]
+            airports = route.get("_airports", [])
+            if len(airports) >= 2:
+                orig = airports[0]
+                dest = airports[-1]
+                return {
+                    "orig_loc": [orig.get("lon", 0), orig.get("lat", 0)],
+                    "dest_loc": [dest.get("lon", 0), dest.get("lat", 0)],
+                    "origin_name": f"{orig.get('iata', '') or orig.get('icao', '')}: {orig.get('name', 'Unknown')}",
+                    "dest_name": f"{dest.get('iata', '') or dest.get('icao', '')}: {dest.get('name', 'Unknown')}",
+                }
+    return {}

backend/routers/sar.py

@@ -0,0 +1,260 @@
+"""SAR (Synthetic Aperture Radar) layer endpoints.
+
+Exposes:
+  - GET  /api/sar/status            — feature gates + signup links for the UI
+  - GET  /api/sar/anomalies         — Mode B pre-processed anomalies
+  - GET  /api/sar/scenes            — Mode A scene catalog
+  - GET  /api/sar/coverage          — per-AOI coverage and next-pass hints
+  - GET  /api/sar/aois              — operator-defined AOIs
+  - POST /api/sar/aois              — create or replace an AOI
+  - DELETE /api/sar/aois/{aoi_id}   — remove an AOI
+  - GET  /api/sar/near              — anomalies within radius_km of (lat, lon)
+
+The /status endpoint is the load-bearing UX: when Mode B is disabled it
+returns the structured help payload from sar_config.products_fetch_status()
+so the frontend can render in-app links to the free signup pages instead of
+making the user hunt around.
+"""
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+from pydantic import BaseModel, Field
+
+from auth import require_local_operator
+from limiter import limiter
+from services.fetchers._store import get_latest_data_subset_refs
+from services.sar.sar_aoi import (
+    SarAoi,
+    add_aoi,
+    haversine_km,
+    load_aois,
+    remove_aoi,
+)
+from services.sar.sar_config import (
+    catalog_enabled,
+    clear_runtime_credentials,
+    openclaw_enabled,
+    products_fetch_enabled,
+    products_fetch_status,
+    require_private_tier_for_publish,
+    set_runtime_credentials,
+)
+
+router = APIRouter()
+
+
+# ---------------------------------------------------------------------------
+# Status — the in-app onboarding hook
+# ---------------------------------------------------------------------------
+@router.get("/api/sar/status")
+@limiter.limit("60/minute")
+async def sar_status(request: Request) -> dict:
+    """Layer status + signup links.
+
+    The frontend calls this whenever the SAR panel is opened.  When Mode B
+    is off, the response includes a step-by-step ``help`` block with the
+    free signup URLs so the user can enable everything without leaving the
+    app.
+    """
+    products_status = products_fetch_status()
+    return {
+        "ok": True,
+        "catalog": {
+            "mode": "A",
+            "enabled": catalog_enabled(),
+            "needs_account": False,
+            "description": "Free Sentinel-1 scene catalog from ASF Search.",
+        },
+        "products": {
+            "mode": "B",
+            **products_status,
+        },
+        "openclaw_enabled": openclaw_enabled(),
+        "require_private_tier": require_private_tier_for_publish(),
+    }
+
+
+# ---------------------------------------------------------------------------
+# Data feeds
+# ---------------------------------------------------------------------------
+@router.get("/api/sar/anomalies")
+@limiter.limit("60/minute")
+async def sar_anomalies(
+    request: Request,
+    kind: str = Query("", description="Optional anomaly kind filter"),
+    aoi_id: str = Query("", description="Optional AOI id filter"),
+    limit: int = Query(200, ge=1, le=1000),
+) -> dict:
+    """Return the latest cached SAR anomalies (Mode B)."""
+    snap = get_latest_data_subset_refs("sar_anomalies")
+    items = list(snap.get("sar_anomalies") or [])
+    if kind:
+        items = [a for a in items if a.get("kind") == kind]
+    if aoi_id:
+        aoi_id = aoi_id.strip().lower()
+        items = [a for a in items if (a.get("stack_id") or "").lower() == aoi_id]
+    items = items[:limit]
+    return {
+        "ok": True,
+        "count": len(items),
+        "anomalies": items,
+        "products_enabled": products_fetch_enabled(),
+    }
+
+
+@router.get("/api/sar/scenes")
+@limiter.limit("60/minute")
+async def sar_scenes(
+    request: Request,
+    aoi_id: str = Query(""),
+    limit: int = Query(200, ge=1, le=1000),
+) -> dict:
+    """Return the latest cached scene catalog (Mode A)."""
+    snap = get_latest_data_subset_refs("sar_scenes")
+    items = list(snap.get("sar_scenes") or [])
+    if aoi_id:
+        aoi_id = aoi_id.strip().lower()
+        items = [s for s in items if (s.get("aoi_id") or "").lower() == aoi_id]
+    items = items[:limit]
+    return {
+        "ok": True,
+        "count": len(items),
+        "scenes": items,
+        "catalog_enabled": catalog_enabled(),
+    }
+
+
+@router.get("/api/sar/coverage")
+@limiter.limit("60/minute")
+async def sar_coverage(request: Request) -> dict:
+    """Per-AOI coverage and rough next-pass estimate."""
+    snap = get_latest_data_subset_refs("sar_aoi_coverage")
+    return {
+        "ok": True,
+        "coverage": list(snap.get("sar_aoi_coverage") or []),
+    }
+
+
+@router.get("/api/sar/near")
+@limiter.limit("60/minute")
+async def sar_near(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lon: float = Query(..., ge=-180, le=180),
+    radius_km: float = Query(50, ge=1, le=2000),
+    kind: str = Query(""),
+    limit: int = Query(50, ge=1, le=500),
+) -> dict:
+    """Return anomalies whose center sits within ``radius_km`` of (lat, lon)."""
+    snap = get_latest_data_subset_refs("sar_anomalies")
+    items = list(snap.get("sar_anomalies") or [])
+    matches = []
+    for a in items:
+        try:
+            a_lat = float(a.get("lat", 0.0))
+            a_lon = float(a.get("lon", 0.0))
+        except (TypeError, ValueError):
+            continue
+        d = haversine_km(lat, lon, a_lat, a_lon)
+        if d > radius_km:
+            continue
+        if kind and a.get("kind") != kind:
+            continue
+        a = dict(a)
+        a["distance_km"] = round(d, 2)
+        matches.append(a)
+    matches.sort(key=lambda x: x.get("distance_km", 0))
+    return {
+        "ok": True,
+        "count": len(matches[:limit]),
+        "anomalies": matches[:limit],
+    }
+
+
+# ---------------------------------------------------------------------------
+# AOI CRUD
+# ---------------------------------------------------------------------------
+@router.get("/api/sar/aois")
+@limiter.limit("60/minute")
+async def sar_aoi_list(request: Request) -> dict:
+    return {
+        "ok": True,
+        "aois": [a.to_dict() for a in load_aois(force=True)],
+    }
+
+
+class AoiPayload(BaseModel):
+    id: str = Field(..., min_length=1, max_length=64)
+    name: str = Field(..., min_length=1, max_length=120)
+    description: str = Field("", max_length=400)
+    center_lat: float = Field(..., ge=-90, le=90)
+    center_lon: float = Field(..., ge=-180, le=180)
+    radius_km: float = Field(25.0, ge=1.0, le=500.0)
+    category: str = Field("watchlist", max_length=40)
+    polygon: list[list[float]] | None = None
+
+
+@router.post("/api/sar/aois", dependencies=[Depends(require_local_operator)])
+@limiter.limit("20/minute")
+async def sar_aoi_upsert(request: Request, payload: AoiPayload) -> dict:
+    aoi = SarAoi(
+        id=payload.id.strip().lower(),
+        name=payload.name.strip(),
+        description=payload.description.strip(),
+        center_lat=payload.center_lat,
+        center_lon=payload.center_lon,
+        radius_km=payload.radius_km,
+        polygon=payload.polygon,
+        category=(payload.category or "watchlist").strip().lower(),
+    )
+    add_aoi(aoi)
+    return {"ok": True, "aoi": aoi.to_dict()}
+
+
+@router.delete("/api/sar/aois/{aoi_id}", dependencies=[Depends(require_local_operator)])
+@limiter.limit("20/minute")
+async def sar_aoi_delete(request: Request, aoi_id: str) -> dict:
+    removed = remove_aoi(aoi_id)
+    if not removed:
+        raise HTTPException(status_code=404, detail="AOI not found")
+    return {"ok": True, "removed": aoi_id}
+
+
+# ---------------------------------------------------------------------------
+# Mode B enable / disable — one-click setup from the frontend
+# ---------------------------------------------------------------------------
+class ModeBEnablePayload(BaseModel):
+    earthdata_user: str = Field("", max_length=120)
+    earthdata_token: str = Field(..., min_length=8, max_length=2048)
+    copernicus_user: str = Field("", max_length=120)
+    copernicus_token: str = Field("", max_length=2048)
+
+
+@router.post("/api/sar/mode-b/enable", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def sar_mode_b_enable(request: Request, payload: ModeBEnablePayload) -> dict:
+    """Store Earthdata (and optional Copernicus) credentials and flip both
+    two-step opt-in flags.  Returns the fresh status payload so the UI can
+    immediately reflect the change.
+    """
+    set_runtime_credentials(
+        earthdata_user=payload.earthdata_user,
+        earthdata_token=payload.earthdata_token,
+        copernicus_user=payload.copernicus_user,
+        copernicus_token=payload.copernicus_token,
+        mode_b_opt_in=True,
+    )
+    return {
+        "ok": True,
+        "products": products_fetch_status(),
+    }
+
+
+@router.post("/api/sar/mode-b/disable", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def sar_mode_b_disable(request: Request) -> dict:
+    """Wipe runtime credentials and revert to Mode A only."""
+    clear_runtime_credentials()
+    return {
+        "ok": True,
+        "products": products_fetch_status(),
+    }

backend/routers/sigint.py

@@ -0,0 +1,67 @@
+from fastapi import APIRouter, Request, Query, Depends
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+from services.data_fetcher import get_latest_data
+
+router = APIRouter()
+
+
+@router.get("/api/oracle/region-intel")
+@limiter.limit("30/minute")
+async def oracle_region_intel(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Get oracle intelligence summary for a geographic region."""
+    from services.oracle_service import get_region_oracle_intel
+    news_items = get_latest_data().get("news", [])
+    return get_region_oracle_intel(lat, lng, news_items)
+
+
+@router.get("/api/thermal/verify", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def thermal_verify(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+    radius_km: float = Query(10, ge=1, le=100),
+):
+    """On-demand thermal anomaly verification using Sentinel-2 SWIR bands."""
+    from services.thermal_sentinel import search_thermal_anomaly
+    result = search_thermal_anomaly(lat, lng, radius_km)
+    return result
+
+
+@router.post("/api/sigint/transmit", dependencies=[Depends(require_local_operator)])
+@limiter.limit("5/minute")
+async def sigint_transmit(request: Request):
+    """Send an APRS-IS message to a specific callsign. Requires ham radio credentials."""
+    from services.wormhole_supervisor import get_transport_tier
+    tier = get_transport_tier()
+    if str(tier or "").startswith("private_"):
+        return {"ok": False, "detail": "APRS transmit blocked in private transport mode"}
+    body = await request.json()
+    callsign = body.get("callsign", "")
+    passcode = body.get("passcode", "")
+    target = body.get("target", "")
+    message = body.get("message", "")
+    if not all([callsign, passcode, target, message]):
+        return {"ok": False, "detail": "Missing required fields: callsign, passcode, target, message"}
+    from services.sigint_bridge import send_aprs_message
+    return send_aprs_message(callsign, passcode, target, message)
+
+
+@router.get("/api/sigint/nearest-sdr")
+@limiter.limit("30/minute")
+async def nearest_sdr(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Find the nearest KiwiSDR receivers to a given coordinate."""
+    from services.sigint_bridge import find_nearest_kiwisdr
+    kiwisdr_data = get_latest_data().get("kiwisdr", [])
+    return find_nearest_kiwisdr(lat, lng, kiwisdr_data)

backend/routers/tools.py

@@ -0,0 +1,411 @@
+import asyncio
+import logging
+import math
+from typing import Any
+from fastapi import APIRouter, Request, Query, Depends, HTTPException, Response
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from limiter import limiter
+from auth import require_admin, require_local_operator
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+def _safe_int(val, default=0):
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _safe_float(val, default=0.0):
+    try:
+        parsed = float(val)
+        if not math.isfinite(parsed):
+            return default
+        return parsed
+    except (TypeError, ValueError):
+        return default
+
+
+class ShodanSearchRequest(BaseModel):
+    query: str
+    page: int = 1
+    facets: list[str] = []
+
+
+class ShodanCountRequest(BaseModel):
+    query: str
+    facets: list[str] = []
+
+
+class ShodanHostRequest(BaseModel):
+    ip: str
+    history: bool = False
+
+
+@router.get("/api/region-dossier")
+@limiter.limit("30/minute")
+def api_region_dossier(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Sync def so FastAPI runs it in a threadpool — prevents blocking the event loop."""
+    from services.region_dossier import get_region_dossier
+    return get_region_dossier(lat, lng)
+
+
+@router.get("/api/geocode/search")
+@limiter.limit("30/minute")
+async def api_geocode_search(
+    request: Request,
+    q: str = "",
+    limit: int = 5,
+    local_only: bool = False,
+):
+    from services.geocode import search_geocode
+    if not q or len(q.strip()) < 2:
+        return {"results": [], "query": q, "count": 0}
+    results = await asyncio.to_thread(search_geocode, q, limit, local_only)
+    return {"results": results, "query": q, "count": len(results)}
+
+
+@router.get("/api/geocode/reverse")
+@limiter.limit("60/minute")
+async def api_geocode_reverse(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+    local_only: bool = False,
+):
+    from services.geocode import reverse_geocode
+    return await asyncio.to_thread(reverse_geocode, lat, lng, local_only)
+
+
+# ── Sentinel proxy routes (Issue #299/#300/#301, reported by tg12) ──────────
+# These three endpoints relay external Sentinel / Planetary Computer
+# requests through the backend to avoid browser CORS blocks. They are
+# operator-only helpers — they MUST NOT be callable by anonymous remote
+# users, because:
+#
+#   * /api/sentinel/token  — caller supplies their own Sentinel client_id +
+#     client_secret. Without operator gating, the backend becomes a free
+#     anonymous OAuth-mint relay for any Copernicus account.
+#   * /api/sentinel/tile   — same shape as the token route but for tile
+#     imagery. Without gating, the backend acts as an anonymous quota and
+#     bandwidth relay for Sentinel Hub Process API calls.
+#   * /api/sentinel2/search — hits the Planetary Computer STAC search API
+#     and falls back to Esri imagery. No caller credentials are involved,
+#     but the route is still an anonymous external-search relay. We gate
+#     it the same way for consistency with the rest of the operator-only
+#     helper surface.
+#
+# Gating is via require_local_operator (loopback / bridge / admin key),
+# matching the same allowlist already used by /api/region-dossier and
+# the other operator helpers further up this file. Single-operator nodes
+# see no behavior change — their dashboard already lives on loopback or
+# the trusted Docker bridge, so it still resolves.
+@router.get("/api/sentinel2/search", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+def api_sentinel2_search(
+    request: Request,
+    lat: float = Query(..., ge=-90, le=90),
+    lng: float = Query(..., ge=-180, le=180),
+):
+    """Search for latest Sentinel-2 imagery at a point. Sync for threadpool execution."""
+    from services.sentinel_search import search_sentinel2_scene
+    return search_sentinel2_scene(lat, lng)
+
+
+# Issue #298 (tg12): Sentinel credentials moved server-side
+# ---------------------------------------------------------------------------
+# Previously the frontend kept Copernicus CDSE client_id + client_secret in
+# browser localStorage / sessionStorage and forwarded them on every tile
+# request through this proxy. That exposed real third-party credentials to
+# any same-origin script (XSS, malicious browser extension, dev-tools HAR
+# export).
+#
+# Resolution order (first match wins):
+#   1. Request body — kept for back-compat. A small number of legacy
+#      operator setups may still post credentials; we don't break them.
+#   2. Backend .env — SENTINEL_CLIENT_ID / SENTINEL_CLIENT_SECRET, managed
+#      through the existing /api/settings/api-keys flow (admin-gated).
+#
+# The frontend in ``sentinelHub.ts`` no longer reads browser storage and no
+# longer forwards credentials — every dashboard request now lands in (2).
+# The require_local_operator gate (added in #303/PR #303) stays — both layers
+# are independent: the gate blocks anonymous callers, the env fallback lets
+# legitimate (gated) callers omit credentials from the body.
+# ---------------------------------------------------------------------------
+def _resolve_sentinel_credentials(body_id: str, body_secret: str) -> tuple[str, str]:
+    """Return (client_id, client_secret) using body values when present,
+    otherwise falling back to backend .env. Empty strings if neither is set."""
+    import os as _os
+    cid = (body_id or "").strip() or (_os.environ.get("SENTINEL_CLIENT_ID", "") or "").strip()
+    csec = (body_secret or "").strip() or (_os.environ.get("SENTINEL_CLIENT_SECRET", "") or "").strip()
+    return cid, csec
+
+
+@router.post("/api/sentinel/token", dependencies=[Depends(require_local_operator)])
+@limiter.limit("60/minute")
+async def api_sentinel_token(request: Request):
+    """Proxy Copernicus CDSE OAuth2 token request (avoids browser CORS block).
+
+    Credentials are resolved by ``_resolve_sentinel_credentials`` — body
+    fields are honored for back-compat, otherwise the backend .env values
+    populated through ``/api/settings/api-keys`` are used.
+    """
+    import requests as req
+    body = await request.body()
+    from urllib.parse import parse_qs
+    params = parse_qs(body.decode("utf-8"))
+    body_id = params.get("client_id", [""])[0]
+    body_secret = params.get("client_secret", [""])[0]
+    client_id, client_secret = _resolve_sentinel_credentials(body_id, body_secret)
+    if not client_id or not client_secret:
+        # Friendly, non-hostile error — points the operator at the place
+        # they configure other API keys instead of just saying "required".
+        raise HTTPException(
+            400,
+            "Sentinel client_id/client_secret are not configured. "
+            "Set SENTINEL_CLIENT_ID and SENTINEL_CLIENT_SECRET in the "
+            "API Keys panel (Settings → API Keys) or your backend .env.",
+        )
+    token_url = "https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token"
+    try:
+        resp = await asyncio.to_thread(req.post, token_url,
+            data={"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret},
+            timeout=15)
+        return Response(content=resp.content, status_code=resp.status_code, media_type="application/json")
+    except Exception:
+        logger.exception("Token request failed")
+        raise HTTPException(502, "Token request failed")
+
+
+# Cache key is an HMAC of (client_id, client_secret) — a caller cannot hit
+# this cache without knowing the same secret that originally populated it.
+# Without this binding, the lookup only checked client_id, so anyone who
+# knew a valid client_id could reuse another caller's cached token (and
+# burn their Copernicus quota / access tiles on their account).
+_sh_token_cache: dict = {"token": None, "expiry": 0, "credential_fp": ""}
+
+
+def _credential_fingerprint(client_id: str, client_secret: str) -> str:
+    """Return a stable, secret-binding fingerprint for the Sentinel cache key.
+
+    Uses HMAC-SHA256 so the raw secret is never stored in process memory as
+    a cache key. The HMAC key is a per-process random value, which means the
+    fingerprint cannot be precomputed across restarts (additional defense
+    against an attacker who learned a valid client_id but not the secret).
+    """
+    import hashlib
+    import hmac
+
+    return hmac.new(
+        _SH_TOKEN_CACHE_HMAC_KEY,
+        f"{client_id}\x00{client_secret}".encode("utf-8"),
+        hashlib.sha256,
+    ).hexdigest()
+
+
+# Per-process random HMAC key. Regenerated on each backend startup so cached
+# fingerprints don't survive restarts.
+import os as _os
+_SH_TOKEN_CACHE_HMAC_KEY = _os.urandom(32)
+
+
+@router.post("/api/sentinel/tile", dependencies=[Depends(require_local_operator)])
+@limiter.limit("300/minute")
+async def api_sentinel_tile(request: Request):
+    """Proxy Sentinel Hub Process API tile request (avoids CORS block)."""
+    import requests as req
+    import time as _time
+    try:
+        body = await request.json()
+    except Exception:
+        return JSONResponse(status_code=422, content={"ok": False, "detail": "invalid JSON body"})
+
+    # Issue #298: same resolution order as /api/sentinel/token — body
+    # values for back-compat, otherwise backend .env.
+    body_id = body.get("client_id", "")
+    body_secret = body.get("client_secret", "")
+    client_id, client_secret = _resolve_sentinel_credentials(body_id, body_secret)
+    preset = body.get("preset", "TRUE-COLOR")
+    date_str = body.get("date", "")
+    z = body.get("z", 0)
+    x = body.get("x", 0)
+    y = body.get("y", 0)
+
+    if not client_id or not client_secret or not date_str:
+        # Distinguish "no creds" from "no date" so the operator knows
+        # what to fix. Same friendly pointer as the /token route.
+        if not client_id or not client_secret:
+            raise HTTPException(
+                400,
+                "Sentinel client_id/client_secret are not configured. "
+                "Set SENTINEL_CLIENT_ID and SENTINEL_CLIENT_SECRET in the "
+                "API Keys panel (Settings → API Keys) or your backend .env.",
+            )
+        raise HTTPException(400, "date required")
+
+    now = _time.time()
+    credential_fp = _credential_fingerprint(client_id, client_secret)
+    if (_sh_token_cache["token"]
+            and _sh_token_cache["credential_fp"] == credential_fp
+            and now < _sh_token_cache["expiry"] - 30):
+        token = _sh_token_cache["token"]
+    else:
+        token_url = "https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token"
+        try:
+            tresp = await asyncio.to_thread(req.post, token_url,
+                data={"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret},
+                timeout=15)
+            if tresp.status_code != 200:
+                raise HTTPException(401, f"Token auth failed: {tresp.text[:200]}")
+            tdata = tresp.json()
+            token = tdata["access_token"]
+            _sh_token_cache["token"] = token
+            _sh_token_cache["expiry"] = now + tdata.get("expires_in", 300)
+            _sh_token_cache["credential_fp"] = credential_fp
+        except HTTPException:
+            raise
+        except Exception:
+            logger.exception("Token request failed")
+            raise HTTPException(502, "Token request failed")
+
+    half = 20037508.342789244
+    tile_size = (2 * half) / math.pow(2, z)
+    min_x = -half + x * tile_size
+    max_x = min_x + tile_size
+    max_y = half - y * tile_size
+    min_y = max_y - tile_size
+    bbox = [min_x, min_y, max_x, max_y]
+
+    evalscripts = {
+        "TRUE-COLOR": '//VERSION=3\nfunction setup(){return{input:["B04","B03","B02"],output:{bands:3}};}\nfunction evaluatePixel(s){return[2.5*s.B04,2.5*s.B03,2.5*s.B02];}',
+        "FALSE-COLOR": '//VERSION=3\nfunction setup(){return{input:["B08","B04","B03"],output:{bands:3}};}\nfunction evaluatePixel(s){return[2.5*s.B08,2.5*s.B04,2.5*s.B03];}',
+        "NDVI": '//VERSION=3\nfunction setup(){return{input:["B04","B08"],output:{bands:3}};}\nfunction evaluatePixel(s){var n=(s.B08-s.B04)/(s.B08+s.B04);if(n<-0.2)return[0.05,0.05,0.05];if(n<0)return[0.75,0.75,0.75];if(n<0.1)return[0.86,0.86,0.86];if(n<0.2)return[0.92,0.84,0.68];if(n<0.3)return[0.77,0.88,0.55];if(n<0.4)return[0.56,0.80,0.32];if(n<0.5)return[0.35,0.72,0.18];if(n<0.6)return[0.20,0.60,0.08];if(n<0.7)return[0.10,0.48,0.04];return[0.0,0.36,0.0];}',
+        "MOISTURE-INDEX": '//VERSION=3\nfunction setup(){return{input:["B8A","B11"],output:{bands:3}};}\nfunction evaluatePixel(s){var m=(s.B8A-s.B11)/(s.B8A+s.B11);var r=Math.max(0,Math.min(1,1.5-3*m));var g=Math.max(0,Math.min(1,m<0?1.5+3*m:1.5-3*m));var b=Math.max(0,Math.min(1,1.5+3*(m-0.5)));return[r,g,b];}',
+    }
+    evalscript = evalscripts.get(preset, evalscripts["TRUE-COLOR"])
+
+    from datetime import datetime as _dt, timedelta as _td
+    try:
+        end_date = _dt.strptime(date_str, "%Y-%m-%d")
+    except ValueError:
+        end_date = _dt.utcnow()
+
+    if z <= 6:
+        lookback_days = 30
+    elif z <= 9:
+        lookback_days = 14
+    elif z <= 11:
+        lookback_days = 7
+    else:
+        lookback_days = 5
+
+    start_date = end_date - _td(days=lookback_days)
+
+    process_body = {
+        "input": {
+            "bounds": {"bbox": bbox, "properties": {"crs": "http://www.opengis.net/def/crs/EPSG/0/3857"}},
+            "data": [{"type": "sentinel-2-l2a", "dataFilter": {
+                "timeRange": {
+                    "from": start_date.strftime("%Y-%m-%dT00:00:00Z"),
+                    "to": end_date.strftime("%Y-%m-%dT23:59:59Z"),
+                },
+                "maxCloudCoverage": 30, "mosaickingOrder": "leastCC",
+            }}],
+        },
+        "output": {"width": 256, "height": 256,
+            "responses": [{"identifier": "default", "format": {"type": "image/png"}}]},
+        "evalscript": evalscript,
+    }
+    try:
+        resp = await asyncio.to_thread(req.post,
+            "https://sh.dataspace.copernicus.eu/api/v1/process",
+            json=process_body,
+            headers={"Authorization": f"Bearer {token}", "Accept": "image/png"},
+            timeout=30)
+        return Response(content=resp.content, status_code=resp.status_code,
+            media_type=resp.headers.get("content-type", "image/png"))
+    except Exception:
+        logger.exception("Process API failed")
+        raise HTTPException(502, "Process API failed")
+
+
+@router.get("/api/tools/shodan/status", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_shodan_status(request: Request):
+    from services.shodan_connector import get_shodan_connector_status
+    return get_shodan_connector_status()
+
+
+@router.post("/api/tools/shodan/search", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_shodan_search(request: Request, body: ShodanSearchRequest):
+    from services.shodan_connector import ShodanConnectorError, search_shodan
+    try:
+        return search_shodan(body.query, page=body.page, facets=body.facets)
+    except ShodanConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/shodan/count", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_shodan_count(request: Request, body: ShodanCountRequest):
+    from services.shodan_connector import ShodanConnectorError, count_shodan
+    try:
+        return count_shodan(body.query, facets=body.facets)
+    except ShodanConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/shodan/host", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_shodan_host(request: Request, body: ShodanHostRequest):
+    from services.shodan_connector import ShodanConnectorError, lookup_shodan_host
+    try:
+        return lookup_shodan_host(body.ip, history=body.history)
+    except ShodanConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.get("/api/tools/uw/status", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_uw_status(request: Request):
+    from services.unusual_whales_connector import get_uw_status
+    return get_uw_status()
+
+
+@router.post("/api/tools/uw/congress", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_uw_congress(request: Request):
+    from services.unusual_whales_connector import FinnhubConnectorError, fetch_congress_trades
+    try:
+        return fetch_congress_trades()
+    except FinnhubConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/uw/darkpool", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_uw_darkpool(request: Request):
+    from services.unusual_whales_connector import FinnhubConnectorError, fetch_insider_transactions
+    try:
+        return fetch_insider_transactions()
+    except FinnhubConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
+
+
+@router.post("/api/tools/uw/flow", dependencies=[Depends(require_local_operator)])
+@limiter.limit("12/minute")
+async def api_uw_flow(request: Request):
+    from services.unusual_whales_connector import FinnhubConnectorError, fetch_defense_quotes
+    try:
+        return fetch_defense_quotes()
+    except FinnhubConnectorError as exc:
+        raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc

backend/scripts/convert_power_plants.py

@@ -20,7 +20,17 @@ OUT_PATH = Path(__file__).parent.parent / "data" / "power_plants.json"
 
 def main() -> None:
     print(f"Downloading WRI Global Power Plant Database from GitHub...")
-    req = urllib.request.Request(CSV_URL, headers={"User-Agent": "ShadowBroker-OSINT/1.0"})
+    # Round 7a: release-time data refresher. Uses the per-operator UA if
+    # available, otherwise a release-script-specific identifier. This
+    # script is run by the maintainer at release time, NOT at runtime,
+    # so an aggregate UA is acceptable; we still use the helper so the
+    # behavior matches the rest of the project.
+    try:
+        from services.network_utils import outbound_user_agent
+        ua = outbound_user_agent("release-script-power-plants")
+    except Exception:
+        ua = "Shadowbroker/0.9 (release-script-power-plants; +https://github.com/BigBodyCobain/Shadowbroker/issues)"
+    req = urllib.request.Request(CSV_URL, headers={"User-Agent": ua})
     with urllib.request.urlopen(req, timeout=60) as resp:
         raw = resp.read().decode("utf-8")
 

backend/scripts/release_helper.py

@@ -1,7 +1,9 @@
 import argparse
 import hashlib
 import json
+import os
 import sys
+from datetime import datetime, timezone
 from pathlib import Path
 
 
@@ -56,6 +58,72 @@ def sha256_file(path: Path) -> str:
     return digest.hexdigest().lower()
 
 
+def _default_generated_at() -> str:
+    return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def build_release_attestation(
+    *,
+    suite_green: bool,
+    suite_name: str = "dm_relay_security",
+    detail: str = "",
+    report: str = "",
+    command: str = "",
+    commit: str = "",
+    generated_at: str = "",
+    threat_model_reference: str = "docs/mesh/threat-model.md",
+    workflow: str = "",
+    run_id: str = "",
+    run_attempt: str = "",
+    ref: str = "",
+) -> dict:
+    normalized_generated_at = str(generated_at or "").strip() or _default_generated_at()
+    normalized_commit = str(commit or "").strip() or os.environ.get("GITHUB_SHA", "").strip()
+    normalized_workflow = str(workflow or "").strip() or os.environ.get("GITHUB_WORKFLOW", "").strip()
+    normalized_run_id = str(run_id or "").strip() or os.environ.get("GITHUB_RUN_ID", "").strip()
+    normalized_run_attempt = str(run_attempt or "").strip() or os.environ.get("GITHUB_RUN_ATTEMPT", "").strip()
+    normalized_ref = str(ref or "").strip() or os.environ.get("GITHUB_REF", "").strip()
+    normalized_suite_name = str(suite_name or "").strip() or "dm_relay_security"
+    normalized_report = str(report or "").strip()
+    normalized_command = str(command or "").strip()
+    normalized_detail = str(detail or "").strip() or (
+        "CI attestation confirms the DM relay security suite is green."
+        if suite_green
+        else "CI attestation recorded a failing DM relay security suite run."
+    )
+    payload = {
+        "generated_at": normalized_generated_at,
+        "commit": normalized_commit,
+        "threat_model_reference": str(threat_model_reference or "").strip()
+        or "docs/mesh/threat-model.md",
+        "dm_relay_security_suite": {
+            "name": normalized_suite_name,
+            "green": bool(suite_green),
+            "detail": normalized_detail,
+            "report": normalized_report,
+        },
+    }
+    if normalized_command:
+        payload["dm_relay_security_suite"]["command"] = normalized_command
+    ci = {
+        "workflow": normalized_workflow,
+        "run_id": normalized_run_id,
+        "run_attempt": normalized_run_attempt,
+        "ref": normalized_ref,
+    }
+    if any(ci.values()):
+        payload["ci"] = ci
+    return payload
+
+
+def write_release_attestation(output_path: Path | str, **kwargs) -> dict:
+    path = Path(output_path).resolve()
+    payload = build_release_attestation(**kwargs)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+    return payload
+
+
 def cmd_show(_args: argparse.Namespace) -> int:
     version = current_version()
     if not version:
@@ -102,6 +170,30 @@ def cmd_hash(args: argparse.Namespace) -> int:
     return 0 if asset_matches else 2
 
 
+def cmd_write_attestation(args: argparse.Namespace) -> int:
+    suite_green = bool(args.suite_green)
+    payload = write_release_attestation(
+        args.output_path,
+        suite_green=suite_green,
+        suite_name=args.suite_name,
+        detail=args.detail,
+        report=args.report,
+        command=args.command,
+        commit=args.commit,
+        generated_at=args.generated_at,
+        threat_model_reference=args.threat_model_reference,
+        workflow=args.workflow,
+        run_id=args.run_id,
+        run_attempt=args.run_attempt,
+        ref=args.ref,
+    )
+    output_path = Path(args.output_path).resolve()
+    print(f"Wrote release attestation: {output_path}")
+    print(f"DM relay security suite : {'green' if suite_green else 'red'}")
+    print(f"Commit                  : {payload.get('commit', '')}")
+    return 0
+
+
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         description="Helper for ShadowBroker release version/tag/asset consistency."
@@ -112,7 +204,7 @@ def build_parser() -> argparse.ArgumentParser:
     show_parser.set_defaults(func=cmd_show)
 
     set_version_parser = subparsers.add_parser("set-version", help="Update frontend/package.json version")
-    set_version_parser.add_argument("version", help="Version like 0.9.6")
+    set_version_parser.add_argument("version", help="Version like 0.9.7")
     set_version_parser.set_defaults(func=cmd_set_version)
 
     hash_parser = subparsers.add_parser(
@@ -121,10 +213,83 @@ def build_parser() -> argparse.ArgumentParser:
     hash_parser.add_argument("zip_path", help="Path to the release ZIP")
     hash_parser.add_argument(
         "--version",
-        help="Release version like 0.9.6. Defaults to frontend/package.json version.",
+        help="Release version like 0.9.7. Defaults to frontend/package.json version.",
     )
     hash_parser.set_defaults(func=cmd_hash)
 
+    attestation_parser = subparsers.add_parser(
+        "write-attestation",
+        help="Write a structured Sprint 8 release attestation JSON file",
+    )
+    attestation_parser.add_argument("output_path", help="Where to write the attestation JSON")
+    suite_group = attestation_parser.add_mutually_exclusive_group(required=True)
+    suite_group.add_argument(
+        "--suite-green",
+        action="store_true",
+        help="Mark the DM relay security suite as green",
+    )
+    suite_group.add_argument(
+        "--suite-red",
+        action="store_true",
+        help="Mark the DM relay security suite as failing",
+    )
+    attestation_parser.add_argument(
+        "--suite-name",
+        default="dm_relay_security",
+        help="Suite name to record in the attestation",
+    )
+    attestation_parser.add_argument(
+        "--detail",
+        default="",
+        help="Human-readable suite detail. Defaults to a CI-generated message.",
+    )
+    attestation_parser.add_argument(
+        "--report",
+        default="",
+        help="Path to the suite report or artifact reference to embed in the attestation.",
+    )
+    attestation_parser.add_argument(
+        "--command",
+        default="",
+        help="Exact suite command used to generate the attestation.",
+    )
+    attestation_parser.add_argument(
+        "--commit",
+        default="",
+        help="Commit SHA. Defaults to GITHUB_SHA when available.",
+    )
+    attestation_parser.add_argument(
+        "--generated-at",
+        default="",
+        help="UTC timestamp for the attestation. Defaults to current UTC time.",
+    )
+    attestation_parser.add_argument(
+        "--threat-model-reference",
+        default="docs/mesh/threat-model.md",
+        help="Threat model reference to embed in the attestation.",
+    )
+    attestation_parser.add_argument(
+        "--workflow",
+        default="",
+        help="Workflow name. Defaults to GITHUB_WORKFLOW when available.",
+    )
+    attestation_parser.add_argument(
+        "--run-id",
+        default="",
+        help="Workflow run ID. Defaults to GITHUB_RUN_ID when available.",
+    )
+    attestation_parser.add_argument(
+        "--run-attempt",
+        default="",
+        help="Workflow run attempt. Defaults to GITHUB_RUN_ATTEMPT when available.",
+    )
+    attestation_parser.add_argument(
+        "--ref",
+        default="",
+        help="Git ref. Defaults to GITHUB_REF when available.",
+    )
+    attestation_parser.set_defaults(func=cmd_write_attestation)
+
     return parser
 
 

backend/scripts/rotate_secure_storage_secret.py

@@ -0,0 +1,75 @@
+"""Rotate the MESH_SECURE_STORAGE_SECRET used to protect key envelopes at rest.
+
+Usage — stop the backend first, then run:
+
+    MESH_OLD_STORAGE_SECRET=<current>  \\
+    MESH_NEW_STORAGE_SECRET=<new>      \\
+    python -m scripts.rotate_secure_storage_secret
+
+Dry-run mode (validates old secret without writing anything):
+
+    MESH_OLD_STORAGE_SECRET=<current>  \\
+    MESH_NEW_STORAGE_SECRET=<new>      \\
+    python -m scripts.rotate_secure_storage_secret --dry-run
+
+Or, for Docker deployments:
+
+    docker exec -e MESH_OLD_STORAGE_SECRET=<current> \\
+                -e MESH_NEW_STORAGE_SECRET=<new>      \\
+                <container> python -m scripts.rotate_secure_storage_secret
+
+After successful rotation, update your .env (or Docker secret file) to set
+MESH_SECURE_STORAGE_SECRET to the new value, then restart the backend.
+
+The script fails closed: if the old secret cannot unwrap any existing envelope,
+nothing is written. Non-passphrase envelopes (DPAPI, raw) are skipped with a
+warning.
+
+Before rewriting, .bak copies of every envelope are created so a mid-rotation
+crash leaves recoverable backups on disk.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+
+def main() -> None:
+    dry_run = "--dry-run" in sys.argv
+
+    old_secret = os.environ.get("MESH_OLD_STORAGE_SECRET", "").strip()
+    new_secret = os.environ.get("MESH_NEW_STORAGE_SECRET", "").strip()
+
+    if not old_secret:
+        print("ERROR: MESH_OLD_STORAGE_SECRET environment variable is required.", file=sys.stderr)
+        sys.exit(1)
+    if not new_secret:
+        print("ERROR: MESH_NEW_STORAGE_SECRET environment variable is required.", file=sys.stderr)
+        sys.exit(1)
+
+    from services.mesh.mesh_secure_storage import SecureStorageError, rotate_storage_secret
+
+    try:
+        result = rotate_storage_secret(old_secret, new_secret, dry_run=dry_run)
+    except SecureStorageError as exc:
+        print(f"ROTATION FAILED: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+    print(json.dumps(result, indent=2))
+    if dry_run:
+        print(
+            "\nDry run complete. No files were modified. Run again without --dry-run to perform the rotation.",
+            file=sys.stderr,
+        )
+    else:
+        print(
+            "\nRotation complete. Update MESH_SECURE_STORAGE_SECRET to the new value and restart the backend."
+            "\nBackup files (.bak) were created alongside each rotated envelope.",
+            file=sys.stderr,
+        )
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/setup-venv.ps1

@@ -1,10 +1,16 @@
 param(
-  [string]$Python = "python"
+  [string]$Python = "py"
 )
 
 $repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
 $venvPath = Join-Path $repoRoot "venv"
-& $Python -m venv $venvPath
+$venvMarker = Join-Path $repoRoot ".venv-dir"
+& $Python -3.11 -m venv $venvPath
 
 $pip = Join-Path $venvPath "Scripts\pip.exe"
-& $pip install -r (Join-Path $repoRoot "requirements-dev.txt")
+& $pip install --upgrade pip
+Push-Location $repoRoot
+& (Join-Path $venvPath "Scripts\python.exe") -m pip install -e .
+& $pip install pytest pytest-asyncio ruff black
+"venv" | Set-Content -LiteralPath $venvMarker -NoNewline
+Pop-Location

backend/scripts/setup-venv.sh

@@ -1,9 +1,14 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-PYTHON="${PYTHON:-python3}"
+PYTHON="${PYTHON:-python3.11}"
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
 VENV_DIR="$REPO_ROOT/venv"
+VENV_MARKER="$REPO_ROOT/.venv-dir"
 
 "$PYTHON" -m venv "$VENV_DIR"
-"$VENV_DIR/bin/pip" install -r "$REPO_ROOT/requirements-dev.txt"
+"$VENV_DIR/bin/pip" install --upgrade pip
+cd "$REPO_ROOT"
+"$VENV_DIR/bin/python" -m pip install -e .
+"$VENV_DIR/bin/pip" install pytest pytest-asyncio ruff black
+printf 'venv\n' > "$VENV_MARKER"

backend/services/ai_intel_store.py

@@ -0,0 +1,178 @@
+"""ai_intel_store — compatibility wrapper around ai_pin_store + layer injection.
+
+openclaw_channel.py and routers/ai_intel.py import from this module name.
+All pin/layer logic lives in ai_pin_store.py; this module re-exports with the
+expected function signatures and adds the layer injection helper.
+"""
+
+import logging
+import time
+from typing import Any
+
+from services.ai_pin_store import (
+    create_pin,
+    create_pins_batch,
+    get_pins,
+    delete_pin,
+    clear_pins,
+    pin_count,
+    pins_as_geojson,
+    purge_expired,
+    # Layer CRUD
+    create_layer,
+    get_layers,
+    update_layer,
+    delete_layer,
+    # Feed layers
+    get_feed_layers,
+    replace_layer_pins,
+)
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Re-exports expected by openclaw_channel._dispatch_command
+# ---------------------------------------------------------------------------
+
+
+def get_all_intel_pins() -> list[dict[str, Any]]:
+    """Return all active pins (no filter, generous limit)."""
+    return get_pins(limit=2000)
+
+
+def add_intel_pin(args: dict[str, Any]) -> dict[str, Any]:
+    """Create a single pin from a command-channel args dict."""
+    ea = args.get("entity_attachment")
+    return create_pin(
+        lat=float(args.get("lat", 0)),
+        lng=float(args.get("lng", 0)),
+        label=str(args.get("label", ""))[:200],
+        category=str(args.get("category", "custom")),
+        layer_id=str(args.get("layer_id", "")),
+        color=str(args.get("color", "")),
+        description=str(args.get("description", "")),
+        source=str(args.get("source", "openclaw")),
+        source_url=str(args.get("source_url", "")),
+        confidence=float(args.get("confidence", 1.0)),
+        ttl_hours=float(args.get("ttl_hours", 0)),
+        metadata=args.get("metadata") or {},
+        entity_attachment=ea if isinstance(ea, dict) else None,
+    )
+
+
+def delete_intel_pin(pin_id: str) -> bool:
+    """Delete a pin by ID."""
+    return delete_pin(pin_id)
+
+
+# Layer helpers for OpenClaw
+def create_intel_layer(args: dict[str, Any]) -> dict[str, Any]:
+    """Create a layer from a command-channel args dict."""
+    return create_layer(
+        name=str(args.get("name", "Untitled"))[:100],
+        description=str(args.get("description", ""))[:500],
+        source=str(args.get("source", "openclaw"))[:50],
+        color=str(args.get("color", "")),
+        feed_url=str(args.get("feed_url", "")),
+        feed_interval=int(args.get("feed_interval", 300)),
+    )
+
+
+def get_intel_layers() -> list[dict[str, Any]]:
+    """Return all layers with pin counts."""
+    return get_layers()
+
+
+def update_intel_layer(layer_id: str, args: dict[str, Any]) -> dict[str, Any] | None:
+    """Update a layer from a command-channel args dict."""
+    return update_layer(layer_id, **{
+        k: v for k, v in args.items()
+        if k in ("name", "description", "visible", "color", "feed_url", "feed_interval")
+    })
+
+
+def delete_intel_layer(layer_id: str) -> int:
+    """Delete a layer and its pins. Returns pin count removed."""
+    return delete_layer(layer_id)
+
+
+# ---------------------------------------------------------------------------
+# Layer injection — inserts agent data into native telemetry layers
+# ---------------------------------------------------------------------------
+
+# Layers that agents are allowed to inject into.
+_INJECTABLE_LAYERS = frozenset({
+    "cctv", "ships", "sigint", "kiwisdr", "military_bases",
+    "datacenters", "power_plants", "satnogs_stations",
+    "volcanoes", "earthquakes", "news", "viirs_change_nodes",
+    "air_quality",
+})
+
+
+def inject_layer_data(
+    layer: str,
+    items: list[dict[str, Any]],
+    mode: str = "append",
+) -> dict[str, Any]:
+    """Inject agent data into a native telemetry layer."""
+    from services.fetchers._store import latest_data, _data_lock, bump_data_version
+
+    layer = str(layer or "").strip()
+    if layer not in _INJECTABLE_LAYERS:
+        return {"ok": False, "detail": f"layer '{layer}' not injectable"}
+
+    items = list(items or [])[:200]
+    if not items:
+        return {"ok": False, "detail": "no items provided"}
+
+    now = time.time()
+    tagged = []
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        entry = dict(item)
+        entry["_injected"] = True
+        entry["_source"] = "user:openclaw"
+        entry["_injected_at"] = now
+        tagged.append(entry)
+
+    with _data_lock:
+        existing = latest_data.get(layer)
+        if not isinstance(existing, list):
+            existing = []
+
+        if mode == "replace":
+            existing = [e for e in existing if not e.get("_injected")]
+
+        existing.extend(tagged)
+        latest_data[layer] = existing
+
+    bump_data_version()
+
+    return {
+        "ok": True,
+        "layer": layer,
+        "injected": len(tagged),
+        "mode": mode,
+    }
+
+
+def clear_injected_data(layer: str = "") -> dict[str, Any]:
+    """Remove all injected items from a layer (or all layers)."""
+    from services.fetchers._store import latest_data, _data_lock, bump_data_version
+
+    removed = 0
+    with _data_lock:
+        targets = [layer] if layer else list(_INJECTABLE_LAYERS)
+        for lyr in targets:
+            existing = latest_data.get(lyr)
+            if not isinstance(existing, list):
+                continue
+            before = len(existing)
+            latest_data[lyr] = [e for e in existing if not e.get("_injected")]
+            removed += before - len(latest_data[lyr])
+
+    if removed:
+        bump_data_version()
+
+    return {"ok": True, "removed": removed}

backend/services/ai_pin_store.py

@@ -0,0 +1,633 @@
+"""AI Intel pin storage — layered pin system with JSON file persistence.
+
+Supports:
+  - Named pin layers (created by user or AI)
+  - Pins with optional entity attachment (track moving objects)
+  - Pin source tracking (user vs openclaw)
+  - Layer visibility toggles
+  - External feed URL per layer (for Phase 5)
+  - GeoJSON export per layer or all layers
+"""
+
+import json
+import logging
+import os
+import threading
+import time
+import uuid
+from datetime import datetime
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Pin schema
+# ---------------------------------------------------------------------------
+
+PIN_CATEGORIES = {
+    "threat", "news", "geolocation", "custom", "anomaly",
+    "military", "maritime", "flight", "infrastructure", "weather",
+    "sigint", "prediction", "research",
+}
+
+PIN_COLORS = {
+    "threat": "#ef4444",       # red
+    "news": "#f59e0b",         # amber
+    "geolocation": "#8b5cf6",  # violet
+    "custom": "#3b82f6",       # blue
+    "anomaly": "#f97316",      # orange
+    "military": "#dc2626",     # dark red
+    "maritime": "#0ea5e9",     # sky
+    "flight": "#6366f1",       # indigo
+    "infrastructure": "#64748b",  # slate
+    "weather": "#22d3ee",      # cyan
+    "sigint": "#a855f7",       # purple
+    "prediction": "#eab308",   # yellow
+    "research": "#10b981",     # emerald
+}
+
+LAYER_COLORS = [
+    "#3b82f6", "#ef4444", "#22d3ee", "#f59e0b", "#8b5cf6",
+    "#10b981", "#f97316", "#6366f1", "#ec4899", "#14b8a6",
+]
+
+# ---------------------------------------------------------------------------
+# In-memory store
+# ---------------------------------------------------------------------------
+
+_layers: list[dict[str, Any]] = []
+_pins: list[dict[str, Any]] = []
+_lock = threading.Lock()
+
+# Persistence file path
+_PERSIST_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data")
+_PERSIST_FILE = os.path.join(_PERSIST_DIR, "pin_layers.json")
+_OLD_PERSIST_FILE = os.path.join(_PERSIST_DIR, "ai_pins.json")
+
+
+def _ensure_persist_dir():
+    try:
+        os.makedirs(_PERSIST_DIR, exist_ok=True)
+    except OSError:
+        pass
+
+
+def _save_to_disk():
+    """Persist layers and pins to JSON file. Called under lock."""
+    try:
+        _ensure_persist_dir()
+        with open(_PERSIST_FILE, "w", encoding="utf-8") as f:
+            json.dump({"layers": _layers, "pins": _pins}, f, indent=2, default=str)
+    except (OSError, IOError) as e:
+        logger.warning(f"Failed to persist pin layers: {e}")
+
+
+def _load_from_disk():
+    """Load layers and pins from disk on startup."""
+    global _layers, _pins
+    try:
+        if os.path.exists(_PERSIST_FILE):
+            with open(_PERSIST_FILE, "r", encoding="utf-8") as f:
+                data = json.load(f)
+            if isinstance(data, dict):
+                _layers = data.get("layers", [])
+                _pins = data.get("pins", [])
+                logger.info(f"Loaded {len(_layers)} layers, {len(_pins)} pins from disk")
+                return
+
+        # Migrate from old flat pin file
+        if os.path.exists(_OLD_PERSIST_FILE):
+            with open(_OLD_PERSIST_FILE, "r", encoding="utf-8") as f:
+                old_pins = json.load(f)
+            if isinstance(old_pins, list) and old_pins:
+                legacy_layer = _make_layer("Legacy", "Migrated pins", source="system")
+                _layers.append(legacy_layer)
+                for p in old_pins:
+                    if isinstance(p, dict):
+                        p["layer_id"] = legacy_layer["id"]
+                        _pins.append(p)
+                logger.info(f"Migrated {len(_pins)} pins from ai_pins.json into Legacy layer")
+                _save_to_disk()
+    except (OSError, IOError, json.JSONDecodeError) as e:
+        logger.warning(f"Failed to load pin layers from disk: {e}")
+
+
+def _make_layer(
+    name: str,
+    description: str = "",
+    source: str = "user",
+    color: str = "",
+    feed_url: str = "",
+    feed_interval: int = 300,
+) -> dict[str, Any]:
+    """Create a layer dict."""
+    layer_id = str(uuid.uuid4())[:12]
+    now = time.time()
+    return {
+        "id": layer_id,
+        "name": name[:100],
+        "description": description[:500],
+        "source": source[:50],
+        "visible": True,
+        "color": color or LAYER_COLORS[len(_layers) % len(LAYER_COLORS)],
+        "created_at": now,
+        "created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
+        "feed_url": feed_url[:1000] if feed_url else "",
+        "feed_interval": max(60, min(86400, feed_interval)),
+        "pin_count": 0,
+    }
+
+
+# Load on import
+_load_from_disk()
+
+# One-time cleanup: remove correlation_engine auto-pins (no longer generated)
+_corr_before = len(_pins)
+_pins[:] = [p for p in _pins if p.get("source") != "correlation_engine"]
+if len(_pins) < _corr_before:
+    logger.info("Cleaned up %d legacy correlation_engine pins", _corr_before - len(_pins))
+    _save_to_disk()
+
+
+# ---------------------------------------------------------------------------
+# Layer CRUD
+# ---------------------------------------------------------------------------
+
+def create_layer(
+    name: str,
+    description: str = "",
+    source: str = "user",
+    color: str = "",
+    feed_url: str = "",
+    feed_interval: int = 300,
+) -> dict[str, Any]:
+    """Create a new pin layer."""
+    with _lock:
+        layer = _make_layer(name, description, source, color, feed_url, feed_interval)
+        _layers.append(layer)
+        _save_to_disk()
+    return layer
+
+
+def get_layers() -> list[dict[str, Any]]:
+    """Return all layers with current pin counts."""
+    now = time.time()
+    with _lock:
+        result = []
+        for layer in _layers:
+            count = sum(
+                1 for p in _pins
+                if p.get("layer_id") == layer["id"]
+                and not (p.get("expires_at") and p["expires_at"] < now)
+            )
+            result.append({**layer, "pin_count": count})
+        return result
+
+
+def update_layer(layer_id: str, **updates) -> Optional[dict[str, Any]]:
+    """Update layer fields. Returns updated layer or None if not found."""
+    allowed = {"name", "description", "visible", "color", "feed_url", "feed_interval", "feed_last_fetched"}
+    with _lock:
+        for layer in _layers:
+            if layer["id"] == layer_id:
+                for k, v in updates.items():
+                    if k in allowed and v is not None:
+                        if k == "name":
+                            layer[k] = str(v)[:100]
+                        elif k == "description":
+                            layer[k] = str(v)[:500]
+                        elif k == "visible":
+                            layer[k] = bool(v)
+                        elif k == "color":
+                            layer[k] = str(v)[:20]
+                        elif k == "feed_url":
+                            layer[k] = str(v)[:1000]
+                        elif k == "feed_interval":
+                            layer[k] = max(60, min(86400, int(v)))
+                        elif k == "feed_last_fetched":
+                            layer[k] = float(v)
+                _save_to_disk()
+                return dict(layer)
+    return None
+
+
+def delete_layer(layer_id: str) -> int:
+    """Delete a layer and all its pins. Returns count of pins removed."""
+    with _lock:
+        before_layers = len(_layers)
+        _layers[:] = [l for l in _layers if l["id"] != layer_id]
+        if len(_layers) == before_layers:
+            return 0  # not found
+        before_pins = len(_pins)
+        _pins[:] = [p for p in _pins if p.get("layer_id") != layer_id]
+        removed = before_pins - len(_pins)
+        _save_to_disk()
+        return removed
+
+
+# ---------------------------------------------------------------------------
+# Pin CRUD
+# ---------------------------------------------------------------------------
+
+def create_pin(
+    lat: float,
+    lng: float,
+    label: str,
+    category: str = "custom",
+    *,
+    layer_id: str = "",
+    color: str = "",
+    description: str = "",
+    source: str = "openclaw",
+    source_url: str = "",
+    confidence: float = 1.0,
+    ttl_hours: float = 0,
+    metadata: Optional[dict] = None,
+    entity_attachment: Optional[dict] = None,
+) -> dict[str, Any]:
+    """Create a single pin and return it."""
+    pin_id = str(uuid.uuid4())[:12]
+    now = time.time()
+
+    cat = category if category in PIN_CATEGORIES else "custom"
+    pin_color = color or PIN_COLORS.get(cat, "#3b82f6")
+
+    # Validate entity_attachment if provided
+    attachment = None
+    if entity_attachment and isinstance(entity_attachment, dict):
+        etype = str(entity_attachment.get("entity_type", "")).strip()
+        eid = str(entity_attachment.get("entity_id", "")).strip()
+        if etype and eid:
+            attachment = {
+                "entity_type": etype[:50],
+                "entity_id": eid[:100],
+                "entity_label": str(entity_attachment.get("entity_label", ""))[:200],
+            }
+
+    pin = {
+        "id": pin_id,
+        "layer_id": layer_id or "",
+        "lat": lat,
+        "lng": lng,
+        "label": label[:200],
+        "category": cat,
+        "color": pin_color,
+        "description": description[:2000],
+        "source": source[:100],
+        "source_url": source_url[:500],
+        "confidence": max(0.0, min(1.0, confidence)),
+        "created_at": now,
+        "created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
+        "expires_at": now + (ttl_hours * 3600) if ttl_hours > 0 else None,
+        "metadata": metadata or {},
+        "entity_attachment": attachment,
+        "comments": [],
+    }
+
+    with _lock:
+        _pins.append(pin)
+        _save_to_disk()
+
+    return pin
+
+
+def create_pins_batch(items: list[dict], default_layer_id: str = "") -> list[dict[str, Any]]:
+    """Create multiple pins at once."""
+    created = []
+    now = time.time()
+
+    with _lock:
+        for item in items[:200]:  # max 200 per batch
+            pin_id = str(uuid.uuid4())[:12]
+            cat = item.get("category", "custom")
+            if cat not in PIN_CATEGORIES:
+                cat = "custom"
+            pin_color = item.get("color", "") or PIN_COLORS.get(cat, "#3b82f6")
+            ttl = float(item.get("ttl_hours", 0) or 0)
+
+            attachment = None
+            ea = item.get("entity_attachment")
+            if ea and isinstance(ea, dict):
+                etype = str(ea.get("entity_type", "")).strip()
+                eid = str(ea.get("entity_id", "")).strip()
+                if etype and eid:
+                    attachment = {
+                        "entity_type": etype[:50],
+                        "entity_id": eid[:100],
+                        "entity_label": str(ea.get("entity_label", ""))[:200],
+                    }
+
+            pin = {
+                "id": pin_id,
+                "layer_id": item.get("layer_id", default_layer_id) or "",
+                "lat": float(item.get("lat", 0)),
+                "lng": float(item.get("lng", 0)),
+                "label": str(item.get("label", ""))[:200],
+                "category": cat,
+                "color": pin_color,
+                "description": str(item.get("description", ""))[:2000],
+                "source": str(item.get("source", "openclaw"))[:100],
+                "source_url": str(item.get("source_url", ""))[:500],
+                "confidence": max(0.0, min(1.0, float(item.get("confidence", 1.0)))),
+                "created_at": now,
+                "created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
+                "expires_at": now + (ttl * 3600) if ttl > 0 else None,
+                "metadata": item.get("metadata", {}),
+                "entity_attachment": attachment,
+                "comments": [],
+            }
+            _pins.append(pin)
+            created.append(pin)
+
+        _save_to_disk()
+    return created
+
+
+def get_pins(
+    category: str = "",
+    source: str = "",
+    layer_id: str = "",
+    limit: int = 500,
+    include_expired: bool = False,
+) -> list[dict[str, Any]]:
+    """Get pins with optional filters."""
+    now = time.time()
+    with _lock:
+        results = []
+        for pin in _pins:
+            if not include_expired and pin.get("expires_at") and pin["expires_at"] < now:
+                continue
+            if category and pin.get("category") != category:
+                continue
+            if source and pin.get("source") != source:
+                continue
+            if layer_id and pin.get("layer_id") != layer_id:
+                continue
+            results.append(pin)
+            if len(results) >= limit:
+                break
+        return results
+
+
+def get_pin(pin_id: str) -> Optional[dict[str, Any]]:
+    """Return a single pin by ID (including comments), or None."""
+    with _lock:
+        for pin in _pins:
+            if pin.get("id") == pin_id:
+                # Ensure comments key exists for legacy pins
+                if "comments" not in pin:
+                    pin["comments"] = []
+                return dict(pin)
+    return None
+
+
+def update_pin(pin_id: str, **updates) -> Optional[dict[str, Any]]:
+    """Update a pin's editable fields (label, description, category, color)."""
+    allowed = {"label", "description", "category", "color"}
+    with _lock:
+        for pin in _pins:
+            if pin.get("id") != pin_id:
+                continue
+            for k, v in updates.items():
+                if k not in allowed or v is None:
+                    continue
+                if k == "label":
+                    pin[k] = str(v)[:200]
+                elif k == "description":
+                    pin[k] = str(v)[:2000]
+                elif k == "category":
+                    cat = str(v)
+                    if cat in PIN_CATEGORIES:
+                        pin[k] = cat
+                        # Refresh color if it was the category default
+                        if not updates.get("color"):
+                            pin["color"] = PIN_COLORS.get(cat, pin.get("color", "#3b82f6"))
+                elif k == "color":
+                    pin[k] = str(v)[:20]
+            pin["updated_at"] = time.time()
+            _save_to_disk()
+            return dict(pin)
+    return None
+
+
+def add_pin_comment(
+    pin_id: str,
+    text: str,
+    author: str = "user",
+    author_label: str = "",
+    reply_to: str = "",
+) -> Optional[dict[str, Any]]:
+    """Append a comment to a pin. Returns the updated pin (with all comments)."""
+    text = (text or "").strip()
+    if not text:
+        return None
+    with _lock:
+        for pin in _pins:
+            if pin.get("id") != pin_id:
+                continue
+            if "comments" not in pin or not isinstance(pin["comments"], list):
+                pin["comments"] = []
+            comment = {
+                "id": str(uuid.uuid4())[:12],
+                "text": text[:4000],
+                "author": (author or "user")[:50],
+                "author_label": (author_label or "")[:100],
+                "reply_to": (reply_to or "")[:12],
+                "created_at": time.time(),
+                "created_at_iso": datetime.utcnow().isoformat() + "Z",
+            }
+            pin["comments"].append(comment)
+            _save_to_disk()
+            return dict(pin)
+    return None
+
+
+def delete_pin_comment(pin_id: str, comment_id: str) -> bool:
+    """Remove a single comment from a pin."""
+    with _lock:
+        for pin in _pins:
+            if pin.get("id") != pin_id:
+                continue
+            comments = pin.get("comments") or []
+            before = len(comments)
+            pin["comments"] = [c for c in comments if c.get("id") != comment_id]
+            if len(pin["comments"]) < before:
+                _save_to_disk()
+                return True
+            return False
+    return False
+
+
+def delete_pin(pin_id: str) -> bool:
+    """Delete a single pin by ID."""
+    with _lock:
+        before = len(_pins)
+        _pins[:] = [p for p in _pins if p.get("id") != pin_id]
+        if len(_pins) < before:
+            _save_to_disk()
+            return True
+        return False
+
+
+def clear_pins(category: str = "", source: str = "", layer_id: str = "") -> int:
+    """Clear pins, optionally filtered. Returns count removed."""
+    with _lock:
+        before = len(_pins)
+
+        def keep(p):
+            if layer_id and p.get("layer_id") != layer_id:
+                return True  # different layer, keep
+            if category and source:
+                return not (p.get("category") == category and p.get("source") == source)
+            if category:
+                return p.get("category") != category
+            if source:
+                return p.get("source") != source
+            if layer_id:
+                return p.get("layer_id") != layer_id
+            return False
+
+        if not category and not source and not layer_id:
+            _pins.clear()
+        else:
+            _pins[:] = [p for p in _pins if keep(p)]
+
+        removed = before - len(_pins)
+        if removed:
+            _save_to_disk()
+        return removed
+
+
+def get_feed_layers() -> list[dict[str, Any]]:
+    """Return layers that have a non-empty feed_url."""
+    with _lock:
+        return [dict(l) for l in _layers if l.get("feed_url")]
+
+
+def replace_layer_pins(layer_id: str, new_pins: list[dict[str, Any]]) -> int:
+    """Atomically replace all pins in a layer with new_pins. Returns count added."""
+    now = time.time()
+    with _lock:
+        # Remove old pins for this layer
+        _pins[:] = [p for p in _pins if p.get("layer_id") != layer_id]
+        # Add new pins
+        added = 0
+        for item in new_pins[:500]:  # cap at 500 per feed
+            pin_id = str(uuid.uuid4())[:12]
+            cat = item.get("category", "custom")
+            if cat not in PIN_CATEGORIES:
+                cat = "custom"
+            pin_color = item.get("color", "") or PIN_COLORS.get(cat, "#3b82f6")
+
+            attachment = None
+            ea = item.get("entity_attachment")
+            if ea and isinstance(ea, dict):
+                etype = str(ea.get("entity_type", "")).strip()
+                eid = str(ea.get("entity_id", "")).strip()
+                if etype and eid:
+                    attachment = {
+                        "entity_type": etype[:50],
+                        "entity_id": eid[:100],
+                        "entity_label": str(ea.get("entity_label", ""))[:200],
+                    }
+
+            pin = {
+                "id": pin_id,
+                "layer_id": layer_id,
+                "lat": float(item.get("lat", 0)),
+                "lng": float(item.get("lng", 0)),
+                "label": str(item.get("label", item.get("name", "")))[:200],
+                "category": cat,
+                "color": pin_color,
+                "description": str(item.get("description", ""))[:2000],
+                "source": str(item.get("source", "feed"))[:100],
+                "source_url": str(item.get("source_url", ""))[:500],
+                "confidence": max(0.0, min(1.0, float(item.get("confidence", 1.0)))),
+                "created_at": now,
+                "created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
+                "expires_at": None,
+                "metadata": item.get("metadata", {}),
+                "entity_attachment": attachment,
+                "comments": [],
+            }
+            _pins.append(pin)
+            added += 1
+        _save_to_disk()
+    return added
+
+
+def purge_expired() -> int:
+    """Remove expired pins. Called periodically."""
+    now = time.time()
+    with _lock:
+        before = len(_pins)
+        _pins[:] = [p for p in _pins if not (p.get("expires_at") and p["expires_at"] < now)]
+        removed = before - len(_pins)
+        if removed:
+            _save_to_disk()
+        return removed
+
+
+def pin_count() -> dict[str, int]:
+    """Return counts by category."""
+    now = time.time()
+    counts: dict[str, int] = {}
+    with _lock:
+        for pin in _pins:
+            if pin.get("expires_at") and pin["expires_at"] < now:
+                continue
+            cat = pin.get("category", "custom")
+            counts[cat] = counts.get(cat, 0) + 1
+    return counts
+
+
+def pins_as_geojson(layer_id: str = "") -> dict[str, Any]:
+    """Convert active pins to GeoJSON FeatureCollection for the map layer."""
+    now = time.time()
+    features = []
+    with _lock:
+        # Build set of visible layer IDs
+        visible_layers = {l["id"] for l in _layers if l.get("visible", True)}
+
+        for pin in _pins:
+            if pin.get("expires_at") and pin["expires_at"] < now:
+                continue
+            # Layer filter
+            pid_layer = pin.get("layer_id", "")
+            if layer_id and pid_layer != layer_id:
+                continue
+            # Skip pins in hidden layers
+            if pid_layer and pid_layer not in visible_layers:
+                continue
+
+            props = {
+                "id": pin["id"],
+                "layer_id": pid_layer,
+                "label": pin["label"],
+                "category": pin["category"],
+                "color": pin["color"],
+                "description": pin.get("description", ""),
+                "source": pin["source"],
+                "source_url": pin.get("source_url", ""),
+                "confidence": pin.get("confidence", 1.0),
+                "created_at": pin.get("created_at_iso", ""),
+                "comment_count": len(pin.get("comments") or []),
+            }
+
+            # Entity attachment info (frontend resolves position)
+            ea = pin.get("entity_attachment")
+            if ea:
+                props["entity_attachment"] = ea
+
+            features.append({
+                "type": "Feature",
+                "geometry": {
+                    "type": "Point",
+                    "coordinates": [pin["lng"], pin["lat"]],
+                },
+                "properties": props,
+            })
+    return {
+        "type": "FeatureCollection",
+        "features": features,
+    }

backend/services/ais_stream.py

@@ -17,6 +17,18 @@ AIS_WS_URL = "wss://stream.aisstream.io/v0/stream"
 API_KEY = os.environ.get("AIS_API_KEY", "")
 
 
+def _env_truthy(name: str) -> bool:
+    return str(os.getenv(name, "")).strip().lower() in {"1", "true", "yes", "on"}
+
+
+def ais_stream_proxy_enabled() -> bool:
+    """Return whether the external Node AIS proxy may be started."""
+    setting = str(os.getenv("SHADOWBROKER_ENABLE_AIS_STREAM_PROXY", "")).strip().lower()
+    if setting:
+        return _env_truthy("SHADOWBROKER_ENABLE_AIS_STREAM_PROXY")
+    return True
+
+
 # AIS vessel type code classification
 # See: https://coast.noaa.gov/data/marinecadastre/ais/VesselTypeCodes2018.pdf
 def classify_vessel(ais_type: int, mmsi: int) -> str:
@@ -327,16 +339,117 @@ def get_country_from_mmsi(mmsi: int) -> str:
 
 # Global vessel store: MMSI → vessel dict
 _vessels: dict[int, dict] = {}
+_vessel_trails: dict[int, dict] = {}
 _vessels_lock = threading.Lock()
 _ws_thread: threading.Thread | None = None
 _ws_running = False
 _proxy_process = None
+# Issue #258: latest status snapshot emitted by ais_proxy.js. Populated when
+# the proxy reports e.g. {"__ais_proxy_status": {"degraded_tls": true}} on
+# stdout, which it does when it falls back to the SPKI-pinned insecure-date
+# path during an upstream cert outage. Surfaced via ais_proxy_status() for
+# /api/health.
+_proxy_status: dict = {}
+# Upstream-connectivity telemetry (added when stream.aisstream.io went fully
+# offline on 2026-05-23). ``_last_msg_at`` is the unix timestamp of the most
+# recent vessel message received from the proxy. ``_proxy_spawn_count`` is
+# how many times we've started the node proxy; combined with no recent
+# messages it tells us the proxy is respawning in a tight loop because the
+# upstream is unreachable. Surfaced via ais_proxy_status() so the operator
+# can see "AIS is dead" instead of guessing whether it's their map filter,
+# their api key, or upstream.
+_last_msg_at: float = 0.0
+_proxy_spawn_count: int = 0
+_VESSEL_TRAIL_INTERVAL_S = 120
+_VESSEL_TRAIL_MAX_POINTS = 240
+
+
+# How stale "last vessel message" can be before we consider the stream
+# disconnected. AISStream typically pushes multiple messages/sec, so a 60s
+# gap means something's wrong upstream or in transit.
+_AIS_CONNECTED_FRESHNESS_S = 60
+
+
+def ais_proxy_status() -> dict:
+    """Return a copy of the latest ais_proxy.js status + connectivity health.
+
+    Fields:
+      * ``degraded_tls`` (bool, issue #258) — true when the proxy is using
+        SPKI-pinned fallback because AISStream's cert expired.
+      * ``connected`` (bool) — true when we received a vessel message in
+        the last ``_AIS_CONNECTED_FRESHNESS_S`` seconds.
+      * ``last_msg_age_seconds`` (int | None) — seconds since the last
+        vessel message; None if we've never received one.
+      * ``proxy_spawn_count`` (int) — how many times we've spawned the
+        node proxy. Sustained increases here without ``connected`` means
+        we're respawning in a tight loop because upstream is dead.
+
+    Returns an empty dict when called before the AIS subsystem starts
+    (e.g. during tests or when no API key is set).
+    """
+    with _vessels_lock:
+        status = dict(_proxy_status)
+        last = _last_msg_at
+        spawns = _proxy_spawn_count
+
+    now = time.time()
+    if last > 0:
+        last_age = int(now - last)
+        status["last_msg_age_seconds"] = last_age
+        status["connected"] = last_age <= _AIS_CONNECTED_FRESHNESS_S
+    else:
+        status["last_msg_age_seconds"] = None
+        status["connected"] = False
+    status["proxy_spawn_count"] = spawns
+    return status
 
 import os
 
 CACHE_FILE = os.path.join(os.path.dirname(__file__), "ais_cache.json")
 
 
+def _record_vessel_trail_locked(mmsi: int, lat, lng, sog=0, now_ts: float | None = None) -> None:
+    """Append a sampled AIS trail point. Caller must hold _vessels_lock."""
+    if lat is None or lng is None:
+        return
+    try:
+        lat_f = float(lat)
+        lng_f = float(lng)
+    except (TypeError, ValueError):
+        return
+    if abs(lat_f) > 90 or abs(lng_f) > 180 or (lat_f == 0 and lng_f == 0):
+        return
+    now = now_ts or time.time()
+    trail_data = _vessel_trails.setdefault(int(mmsi), {"points": [], "last_seen": now})
+    point = [round(lat_f, 5), round(lng_f, 5), round(float(sog or 0), 1), round(now)]
+    last_point_ts = trail_data["points"][-1][3] if trail_data["points"] else 0
+    if now - last_point_ts < _VESSEL_TRAIL_INTERVAL_S:
+        trail_data["last_seen"] = now
+        return
+    if (
+        trail_data["points"]
+        and trail_data["points"][-1][0] == point[0]
+        and trail_data["points"][-1][1] == point[1]
+    ):
+        trail_data["last_seen"] = now
+        return
+    trail_data["points"].append(point)
+    trail_data["last_seen"] = now
+    if len(trail_data["points"]) > _VESSEL_TRAIL_MAX_POINTS:
+        trail_data["points"] = trail_data["points"][-_VESSEL_TRAIL_MAX_POINTS:]
+
+
+def get_vessel_trail(mmsi: int) -> list:
+    """Return the accumulated trail for a single vessel without expanding live payloads."""
+    try:
+        key = int(mmsi)
+    except (TypeError, ValueError):
+        return []
+    with _vessels_lock:
+        points = _vessel_trails.get(key, {}).get("points", [])
+        return [list(point) for point in points]
+
+
 def _save_cache():
     """Save vessel data to disk for persistence across restarts."""
     try:
@@ -379,6 +492,7 @@ def prune_stale_vessels():
         stale_keys = [k for k, v in _vessels.items() if v.get("_updated", 0) < stale_cutoff]
         for k in stale_keys:
             del _vessels[k]
+            _vessel_trails.pop(k, None)
     if stale_keys:
         logger.info(f"AIS pruned {len(stale_keys)} stale vessels")
 
@@ -447,6 +561,7 @@ def ingest_ais_catcher(msgs: list[dict]) -> int:
                     heading = msg.get("heading", 511)
                     vessel["heading"] = heading if heading != 511 else vessel.get("cog", 0)
                     vessel["_updated"] = now
+                    _record_vessel_trail_locked(mmsi, lat, lon, vessel["sog"], now)
                     if msg.get("shipname"):
                         vessel["name"] = msg["shipname"].strip()
                     count += 1
@@ -487,11 +602,21 @@ def _ais_stream_loop():
     proxy_script = os.path.join(os.path.dirname(os.path.dirname(__file__)), "ais_proxy.js")
     backoff = 1  # Exponential backoff starting at 1 second
 
+    if not API_KEY:
+        logger.info("AIS_API_KEY not set — ship tracking disabled. Set AIS_API_KEY to enable.")
+        return
+
     while _ws_running:
         try:
             logger.info("Starting Node.js AIS Stream Proxy...")
             proxy_env = os.environ.copy()
             proxy_env["AIS_API_KEY"] = API_KEY
+            popen_kwargs = {}
+            if os.name == "nt":
+                popen_kwargs["creationflags"] = (
+                    getattr(subprocess, "CREATE_NO_WINDOW", 0)
+                    | getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
+                )
             process = subprocess.Popen(
                 ["node", proxy_script],
                 stdin=subprocess.PIPE,
@@ -500,9 +625,12 @@ def _ais_stream_loop():
                 text=True,
                 bufsize=1,
                 env=proxy_env,
+                **popen_kwargs,
             )
+            global _proxy_spawn_count
             with _vessels_lock:
                 _proxy_process = process
+                _proxy_spawn_count += 1
 
             # Drain stderr in a background thread to prevent deadlock
             import threading
@@ -538,6 +666,18 @@ def _ais_stream_loop():
                     logger.error(f"AIS Stream error: {data['error']}")
                     continue
 
+                # Issue #258: ais_proxy.js emits status markers (e.g.
+                # {"__ais_proxy_status": {"degraded_tls": true}}) when the
+                # SPKI-pinned fallback is in use. We snapshot the latest
+                # status so the backend can expose it on /api/health.
+                if isinstance(data, dict) and "__ais_proxy_status" in data:
+                    status = data.get("__ais_proxy_status") or {}
+                    if isinstance(status, dict):
+                        with _vessels_lock:
+                            _proxy_status.clear()
+                            _proxy_status.update(status)
+                    continue
+
                 msg_type = data.get("MessageType", "")
                 metadata = data.get("MetaData", {})
                 message = data.get("Message", {})
@@ -546,9 +686,15 @@ def _ais_stream_loop():
                 if not mmsi:
                     continue
 
+                # Telemetry: stamp the timestamp of the most recent real
+                # vessel message. ais_proxy_status() reads this to decide
+                # whether the stream is currently "connected" — i.e. has
+                # any data flowed in the last 60s.
+                global _last_msg_at
                 with _vessels_lock:
+                    _last_msg_at = time.time()
                     if mmsi not in _vessels:
-                        _vessels[mmsi] = {"_updated": time.time()}
+                        _vessels[mmsi] = {"_updated": _last_msg_at}
                     vessel = _vessels[mmsi]
 
                 # Update position from PositionReport or StandardClassBPositionReport
@@ -572,7 +718,9 @@ def _ais_stream_loop():
                         vessel["cog"] = report.get("Cog", 0)
                         heading = report.get("TrueHeading", 511)
                         vessel["heading"] = heading if heading != 511 else report.get("Cog", 0)
-                        vessel["_updated"] = time.time()
+                        now_ts = time.time()
+                        vessel["_updated"] = now_ts
+                        _record_vessel_trail_locked(mmsi, lat, lng, vessel["sog"], now_ts)
                         # Use metadata name if we don't have one yet
                         if not vessel.get("name") or vessel["name"] == "UNKNOWN":
                             vessel["name"] = (
@@ -642,6 +790,22 @@ def _run_ais_loop():
 def start_ais_stream():
     """Start the AIS WebSocket stream in a background thread."""
     global _ws_thread, _ws_running
+
+    # Always load cached vessel data first so the ships layer can paint even
+    # when live streaming is disabled or the upstream is unavailable.
+    _load_cache()
+
+    if not API_KEY:
+        logger.info("AIS_API_KEY not set — ship tracking disabled. Set AIS_API_KEY to enable.")
+        return
+
+    if not ais_stream_proxy_enabled():
+        logger.info(
+            "AIS live stream proxy disabled for this runtime; using cached AIS vessels. "
+            "Set SHADOWBROKER_ENABLE_AIS_STREAM_PROXY=1 to opt in."
+        )
+        return
+
     with _vessels_lock:
         if _ws_running:
             logger.info("AIS Stream already running")
@@ -652,9 +816,6 @@ def start_ais_stream():
         logger.info("AIS Stream already running")
         return
 
-    # Load cached vessel data from disk
-    _load_cache()
-
     _ws_thread = threading.Thread(target=_run_ais_loop, daemon=True, name="ais-stream")
     _ws_thread.start()
     logger.info("AIS Stream background thread started")

backend/services/analysis_zone_store.py

@@ -0,0 +1,189 @@
+"""Analysis Zone store — OpenClaw-placed map overlays with analyst notes.
+
+These render as the dashed-border squares on the correlations layer.
+Unlike automated correlations (which are recomputed every cycle), analysis
+zones persist until the agent or user deletes them, or their TTL expires.
+
+Shape matches the correlation alert schema so the frontend renders them
+identically — the ``source`` field marks them as agent-placed and enables
+the delete button in the popup.
+"""
+
+import json
+import logging
+import os
+import threading
+import time
+import uuid
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_zones: list[dict[str, Any]] = []
+_lock = threading.Lock()
+
+_PERSIST_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data")
+_PERSIST_FILE = os.path.join(_PERSIST_DIR, "analysis_zones.json")
+
+ZONE_CATEGORIES = {
+    "contradiction",   # narrative vs telemetry mismatch
+    "analysis",        # general analyst note / assessment
+    "warning",         # potential threat or risk area
+    "observation",     # neutral observation worth marking
+    "hypothesis",      # unverified theory to investigate
+}
+
+# Map categories to correlation type colors on the frontend
+CATEGORY_COLORS = {
+    "contradiction": "amber",
+    "analysis": "cyan",
+    "warning": "red",
+    "observation": "blue",
+    "hypothesis": "purple",
+}
+
+
+def _ensure_dir():
+    try:
+        os.makedirs(_PERSIST_DIR, exist_ok=True)
+    except OSError:
+        pass
+
+
+def _save():
+    """Persist to disk. Called under lock."""
+    try:
+        _ensure_dir()
+        with open(_PERSIST_FILE, "w", encoding="utf-8") as f:
+            json.dump(_zones, f, indent=2, default=str)
+    except Exception as e:
+        logger.warning("Failed to save analysis zones: %s", e)
+
+
+def _load():
+    """Load from disk on startup."""
+    global _zones
+    try:
+        if os.path.exists(_PERSIST_FILE):
+            with open(_PERSIST_FILE, "r", encoding="utf-8") as f:
+                data = json.load(f)
+            if isinstance(data, list):
+                _zones = data
+                logger.info("Loaded %d analysis zones from disk", len(_zones))
+    except Exception as e:
+        logger.warning("Failed to load analysis zones: %s", e)
+
+
+# Load on import
+_load()
+
+
+def _expire():
+    """Remove zones past their TTL. Called under lock."""
+    now = time.time()
+    before = len(_zones)
+    _zones[:] = [
+        z for z in _zones
+        if z.get("ttl_hours", 0) <= 0
+        or (now - z.get("created_at", now)) < z["ttl_hours"] * 3600
+    ]
+    removed = before - len(_zones)
+    if removed:
+        logger.info("Expired %d analysis zones", removed)
+
+
+def create_zone(
+    *,
+    lat: float,
+    lng: float,
+    title: str,
+    body: str,
+    category: str = "analysis",
+    severity: str = "medium",
+    cell_size_deg: float = 1.0,
+    ttl_hours: float = 0,
+    source: str = "openclaw",
+    drivers: list[str] | None = None,
+) -> dict[str, Any]:
+    """Create an analysis zone. Returns the created zone dict."""
+    category = category if category in ZONE_CATEGORIES else "analysis"
+    if severity not in ("high", "medium", "low"):
+        severity = "medium"
+    cell_size_deg = max(0.1, min(cell_size_deg, 10.0))
+
+    zone: dict[str, Any] = {
+        "id": str(uuid.uuid4())[:12],
+        "lat": lat,
+        "lng": lng,
+        "type": "analysis_zone",
+        "category": category,
+        "severity": severity,
+        "score": {"high": 90, "medium": 60, "low": 30}.get(severity, 60),
+        "title": title[:200],
+        "body": body[:2000],
+        "drivers": (drivers or [title])[:5],
+        "cell_size": cell_size_deg,
+        "source": source,
+        "created_at": time.time(),
+        "ttl_hours": ttl_hours,
+    }
+
+    with _lock:
+        _expire()
+        _zones.append(zone)
+        _save()
+
+    logger.info("Analysis zone created: %s at (%.2f, %.2f)", title[:40], lat, lng)
+    return zone
+
+
+def list_zones() -> list[dict[str, Any]]:
+    """Return all live (non-expired) zones."""
+    with _lock:
+        _expire()
+        return list(_zones)
+
+
+def get_zone(zone_id: str) -> dict[str, Any] | None:
+    """Get a single zone by ID."""
+    with _lock:
+        for z in _zones:
+            if z["id"] == zone_id:
+                return dict(z)
+    return None
+
+
+def delete_zone(zone_id: str) -> bool:
+    """Delete a zone by ID. Returns True if found and removed."""
+    with _lock:
+        before = len(_zones)
+        _zones[:] = [z for z in _zones if z["id"] != zone_id]
+        if len(_zones) < before:
+            _save()
+            return True
+    return False
+
+
+def clear_zones(*, source: str | None = None) -> int:
+    """Clear all zones, optionally filtered by source. Returns count removed."""
+    with _lock:
+        before = len(_zones)
+        if source:
+            _zones[:] = [z for z in _zones if z.get("source") != source]
+        else:
+            _zones.clear()
+        removed = before - len(_zones)
+        if removed:
+            _save()
+        return removed
+
+
+def get_live_zones() -> list[dict[str, Any]]:
+    """Return zones formatted for the correlation engine merge.
+
+    This is called by compute_correlations() to inject agent-placed zones
+    into the correlations list that the frontend renders as map squares.
+    """
+    with _lock:
+        _expire()
+        return [dict(z) for z in _zones]

backend/services/api_settings.py

@@ -5,10 +5,20 @@ Keys are stored in the backend .env file and loaded via python-dotenv.
 
 import os
 import re
+import tempfile
 from pathlib import Path
 
 # Path to the backend .env file
 ENV_PATH = Path(__file__).parent.parent / ".env"
+# Path to the example template that ships with the repo
+ENV_EXAMPLE_PATH = Path(__file__).parent.parent.parent / ".env.example"
+DATA_DIR = Path(os.environ.get("SB_DATA_DIR", str(Path(__file__).parent.parent / "data")))
+if not DATA_DIR.is_absolute():
+    DATA_DIR = Path(__file__).parent.parent / DATA_DIR
+OPERATOR_KEYS_ENV_PATH = Path(
+    os.environ.get("SHADOWBROKER_OPERATOR_KEYS_ENV", str(DATA_DIR / "operator_api_keys.env"))
+)
+_ENV_KEY_RE = re.compile(r"^[A-Z][A-Z0-9_]*$")
 
 # ---------------------------------------------------------------------------
 # API Registry — every external service the dashboard depends on
@@ -140,18 +150,145 @@ API_REGISTRY = [
         "url": "https://finnhub.io/register",
         "required": False,
     },
+    # Issue #298 (tg12): Sentinel Hub / Copernicus Data Space Ecosystem
+    # credentials were previously held in browser localStorage / sessionStorage
+    # by the Settings panel. Moved server-side to the same .env-backed
+    # store every other third-party API key lives in. The Sentinel proxy
+    # routes (POST /api/sentinel/token, /tile) now fall back to these
+    # env values when the request body omits credentials — see
+    # backend/routers/tools.py for the resolution order.
+    {
+        "id": "sentinel_client_id",
+        "env_key": "SENTINEL_CLIENT_ID",
+        "name": "Sentinel Hub / Copernicus — Client ID",
+        "description": "OAuth2 client ID for Copernicus Data Space Ecosystem (CDSE). Required for the Sentinel-2 imagery overlay and the right-click Sentinel-2 Intel Card. Sign in at dataspace.copernicus.eu and create OAuth credentials.",
+        "category": "Imagery",
+        "url": "https://dataspace.copernicus.eu/",
+        "required": False,
+    },
+    {
+        "id": "sentinel_client_secret",
+        "env_key": "SENTINEL_CLIENT_SECRET",
+        "name": "Sentinel Hub / Copernicus — Client Secret",
+        "description": "OAuth2 client secret paired with the Client ID above. Used by the backend to mint short-lived access tokens against the CDSE identity provider. Stored in the backend .env; never sent to the browser.",
+        "category": "Imagery",
+        "url": "https://dataspace.copernicus.eu/",
+        "required": False,
+    },
 ]
 
+ALLOWED_ENV_KEYS = {
+    str(api["env_key"])
+    for api in API_REGISTRY
+    if api.get("env_key")
+}
 
-def _obfuscate(value: str) -> str:
-    """Show first 4 chars, mask the rest with bullets."""
-    if not value or len(value) <= 4:
-        return "••••••••"
-    return value[:4] + "•" * (len(value) - 4)
+
+def _parse_env_file(path: Path) -> dict[str, str]:
+    values: dict[str, str] = {}
+    if not path.exists():
+        return values
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError:
+        return values
+    for raw_line in text.splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        key = key.strip()
+        if not _ENV_KEY_RE.match(key):
+            continue
+        value = value.strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
+            value = value[1:-1]
+        values[key] = value
+    return values
+
+
+def _quote_env_value(value: str) -> str:
+    escaped = value.replace("\\", "\\\\").replace('"', '\\"')
+    return f'"{escaped}"'
+
+
+def _write_env_values(path: Path, updates: dict[str, str]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    lines = path.read_text(encoding="utf-8").splitlines() if path.exists() else []
+    seen: set[str] = set()
+    next_lines: list[str] = []
+    for raw_line in lines:
+        stripped = raw_line.strip()
+        if "=" not in stripped or stripped.startswith("#"):
+            next_lines.append(raw_line)
+            continue
+        key = stripped.split("=", 1)[0].strip()
+        if key in updates:
+            next_lines.append(f"{key}={_quote_env_value(updates[key])}")
+            seen.add(key)
+        else:
+            next_lines.append(raw_line)
+    for key, value in updates.items():
+        if key not in seen:
+            next_lines.append(f"{key}={_quote_env_value(value)}")
+
+    fd, tmp_name = tempfile.mkstemp(dir=str(path.parent), prefix=f"{path.name}.tmp.", text=True)
+    tmp_path = Path(tmp_name)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
+            handle.write("\n".join(next_lines).rstrip() + "\n")
+        if os.name != "nt":
+            os.chmod(tmp_path, 0o600)
+        os.replace(tmp_path, path)
+        if os.name != "nt":
+            os.chmod(path, 0o600)
+    finally:
+        try:
+            if tmp_path.exists():
+                tmp_path.unlink()
+        except OSError:
+            pass
+
+
+def load_persisted_api_keys_into_environ() -> None:
+    """Load persisted operator API keys if no process env value exists."""
+    for key, value in _parse_env_file(OPERATOR_KEYS_ENV_PATH).items():
+        if key in ALLOWED_ENV_KEYS and value and not os.environ.get(key):
+            os.environ[key] = value
+
+
+def get_env_path_info() -> dict:
+    """Return absolute paths for the backend .env and .env.example template.
+
+    Surfaced to the frontend so the API Keys settings panel can tell users
+    exactly where to put their keys when in-app editing fails (admin-not-set,
+    file permissions, read-only filesystem, etc.).
+    """
+    env_path = ENV_PATH.resolve()
+    example_path = ENV_EXAMPLE_PATH.resolve()
+    return {
+        "env_path": str(env_path),
+        "env_path_exists": env_path.exists(),
+        "env_path_writable": os.access(env_path.parent, os.W_OK)
+            and (not env_path.exists() or os.access(env_path, os.W_OK)),
+        "env_example_path": str(example_path),
+        "env_example_path_exists": example_path.exists(),
+        "operator_keys_env_path": str(OPERATOR_KEYS_ENV_PATH.resolve()),
+        "operator_keys_env_path_exists": OPERATOR_KEYS_ENV_PATH.exists(),
+        "operator_keys_env_path_writable": os.access(OPERATOR_KEYS_ENV_PATH.parent, os.W_OK)
+            and (not OPERATOR_KEYS_ENV_PATH.exists() or os.access(OPERATOR_KEYS_ENV_PATH, os.W_OK)),
+    }
 
 
 def get_api_keys():
-    """Return the full API registry with obfuscated key values."""
+    """Return the API registry with a binary set/unset flag per key.
+
+    Key values themselves are NEVER returned to the client — not even an
+    obfuscated prefix. Users edit the .env file directly; the panel uses
+    `is_set` to render a CONFIGURED / NOT CONFIGURED badge and the path
+    info from `get_env_path_info()` to tell them where to put each key.
+    """
+    load_persisted_api_keys_into_environ()
     result = []
     for api in API_REGISTRY:
         entry = {
@@ -163,41 +300,64 @@ def get_api_keys():
             "required": api["required"],
             "has_key": api["env_key"] is not None,
             "env_key": api["env_key"],
-            "value_obfuscated": None,
             "is_set": False,
         }
         if api["env_key"]:
             raw = os.environ.get(api["env_key"], "")
-            entry["value_obfuscated"] = _obfuscate(raw)
             entry["is_set"] = bool(raw)
         result.append(entry)
     return result
 
 
-def update_api_key(env_key: str, new_value: str) -> bool:
-    """Update a single key in the .env file and in the current process env."""
-    valid_keys = {api["env_key"] for api in API_REGISTRY if api.get("env_key")}
-    if env_key not in valid_keys:
-        return False
+def save_api_keys(updates: dict[str, str]) -> dict:
+    """Persist allowed API keys from a local operator request.
 
-    if not isinstance(new_value, str):
-        return False
-    if "\n" in new_value or "\r" in new_value:
-        return False
+    Values are accepted write-only: the response includes only configured flags.
+    """
+    clean: dict[str, str] = {}
+    for key, value in updates.items():
+        env_key = str(key or "").strip().upper()
+        if env_key not in ALLOWED_ENV_KEYS:
+            continue
+        clean_value = str(value or "").strip()
+        if clean_value:
+            clean[env_key] = clean_value
+    if not clean:
+        return {"ok": False, "detail": "No supported API keys were provided."}
 
-    if not ENV_PATH.exists():
-        ENV_PATH.write_text("", encoding="utf-8")
+    _write_env_values(OPERATOR_KEYS_ENV_PATH, clean)
+    try:
+        _write_env_values(ENV_PATH, clean)
+    except OSError:
+        # The persistent operator key file is the source of truth for Docker.
+        pass
+    for key, value in clean.items():
+        os.environ[key] = value
+    if "AIS_API_KEY" in clean:
+        try:
+            from services import ais_stream
+            ais_stream.API_KEY = clean["AIS_API_KEY"]
+        except Exception:
+            pass
+    if "OPENSKY_CLIENT_ID" in clean or "OPENSKY_CLIENT_SECRET" in clean:
+        try:
+            from services.fetchers import flights
+            flights.opensky_client.client_id = os.environ.get("OPENSKY_CLIENT_ID", "")
+            flights.opensky_client.client_secret = os.environ.get("OPENSKY_CLIENT_SECRET", "")
+            flights.opensky_client.token = None
+            flights.opensky_client.expires_at = 0
+        except Exception:
+            pass
 
-    # Update os.environ immediately
-    os.environ[env_key] = new_value
+    try:
+        from services.config import get_settings
+        get_settings.cache_clear()
+    except Exception:
+        pass
 
-    # Update the .env file on disk
-    content = ENV_PATH.read_text(encoding="utf-8")
-    pattern = re.compile(rf"^{re.escape(env_key)}=.*$", re.MULTILINE)
-    if pattern.search(content):
-        content = pattern.sub(f"{env_key}={new_value}", content)
-    else:
-        content = content.rstrip("\n") + f"\n{env_key}={new_value}\n"
-
-    ENV_PATH.write_text(content, encoding="utf-8")
-    return True
+    return {
+        "ok": True,
+        "updated": sorted(clean.keys()),
+        "keys": get_api_keys(),
+        "env": get_env_path_info(),
+    }

backend/services/carrier_tracker.py

@@ -1,46 +1,90 @@
 """
 Carrier Strike Group OSINT Tracker
 ===================================
-Scrapes multiple OSINT sources to maintain current estimated positions
-for US Navy Carrier Strike Groups. Updates on startup + 00:00 & 12:00 UTC.
+Maintains estimated positions for US Navy Carrier Strike Groups with
+honest provenance and freshness signals.
 
-Sources:
-  1. GDELT News API — recent carrier movement headlines
-  2. WikiVoyage / public port-call databases
-  3. Fallback — last-known or static OSINT estimates
+Issues #244 / #245 / #246 (tg12 external audit):
+
+The previous implementation baked a snapshot of USNI News Fleet &
+Marine Tracker positions (March 9, 2026) into the registry as
+``fallback_lat``/``fallback_lng`` and stamped ``updated = now()``
+every time the dossier was rendered. That presented stale editorial
+data as live state. It also persisted GDELT-derived positions to the
+on-disk cache with no freshness signal, so a single news mention from
+months ago could keep overriding the (already-stale) registry default
+indefinitely.
+
+Architecture after this PR:
+
+::
+
+    backend/data/carrier_seed.json   read-only, shipped with image,
+                                     used ONCE on first-ever startup
+                                     to bootstrap carrier_cache.json.
+
+    backend/data/carrier_cache.json  mutable, lives in the runtime data
+                                     volume, written by every GDELT
+                                     refresh + any future source.
+
+Startup flow:
+
+1.  ``carrier_cache.json`` exists?  → load it.
+2.  Otherwise, copy ``carrier_seed.json`` → ``carrier_cache.json``,
+    then load it. (This happens once, ever, per install.)
+3.  Background: GDELT fetch runs. Any carrier mentioned in fresh news
+    gets its entry replaced with the news-derived position.
+    ``position_source_at`` is set to the news article timestamp.
+
+Freshness is a *labelling* decision, not an eviction decision:
+
+- ``position_source_at`` within the configurable freshness window
+  (default 14 days) → ``position_confidence = "recent"``.
+- Older than that              → ``position_confidence = "stale"``.
+- Bootstrapped from the seed file (never updated) → ``"seed"``.
+- No cache entry at all (e.g. a carrier added to the registry after
+  first install) → carrier renders at its homeport with
+  ``"homeport_default"``.
+
+Carriers are never hidden, never teleported, never disappeared. The
+position the user sees is always the last position the system actually
+observed, with an honest "as-of" timestamp the UI can render however
+it likes. A year from now, the runtime cache reflects whatever this
+install has observed via GDELT — not the seed snapshot.
 """
 
-import re
+import os
 import json
 import time
 import logging
 import threading
 import random
-from datetime import datetime, timezone
+import shutil
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 from services.network_utils import fetch_with_curl
 
 logger = logging.getLogger(__name__)
 
 # -----------------------------------------------------------------
-# Carrier registry: hull number → metadata + fallback position
+# Carrier registry: hull number → identity only.
+#
+# Issue #244 (tg12): the previous registry carried hard-coded
+# ``fallback_lat``/``fallback_lng`` that were dated editorial
+# snapshots from a 2026-03-09 article. Those fields are DELETED. The
+# registry is now identity + homeport only; positions are sourced
+# exclusively from carrier_cache.json (and via that, from the
+# bootstrap seed or live OSINT).
 # -----------------------------------------------------------------
 CARRIER_REGISTRY: Dict[str, dict] = {
-    # Fallback positions sourced from USNI News Fleet & Marine Tracker (Mar 9, 2026)
-    # https://news.usni.org/2026/03/09/usni-news-fleet-and-marine-tracker-march-9-2026
     # --- Bremerton, WA (Naval Base Kitsap) ---
-    # Distinct pier positions along Sinclair Inlet so carriers don't stack
     "CVN-68": {
         "name": "USS Nimitz (CVN-68)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Nimitz",
         "homeport": "Bremerton, WA",
         "homeport_lat": 47.5535,
         "homeport_lng": -122.6400,
-        "fallback_lat": 47.5535,
-        "fallback_lng": -122.6400,
-        "fallback_heading": 90,
-        "fallback_desc": "Bremerton, WA (Maintenance)",
     },
     "CVN-76": {
         "name": "USS Ronald Reagan (CVN-76)",
@@ -48,23 +92,14 @@ CARRIER_REGISTRY: Dict[str, dict] = {
         "homeport": "Bremerton, WA",
         "homeport_lat": 47.5580,
         "homeport_lng": -122.6360,
-        "fallback_lat": 47.5580,
-        "fallback_lng": -122.6360,
-        "fallback_heading": 90,
-        "fallback_desc": "Bremerton, WA (Decommissioning)",
     },
     # --- Norfolk, VA (Naval Station Norfolk) ---
-    # Piers run N-S along Willoughby Bay; each carrier gets a distinct berth
     "CVN-69": {
         "name": "USS Dwight D. Eisenhower (CVN-69)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Dwight_D._Eisenhower",
         "homeport": "Norfolk, VA",
         "homeport_lat": 36.9465,
         "homeport_lng": -76.3265,
-        "fallback_lat": 36.9465,
-        "fallback_lng": -76.3265,
-        "fallback_heading": 0,
-        "fallback_desc": "Norfolk, VA (Post-deployment maintenance)",
     },
     "CVN-78": {
         "name": "USS Gerald R. Ford (CVN-78)",
@@ -72,10 +107,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
         "homeport": "Norfolk, VA",
         "homeport_lat": 36.9505,
         "homeport_lng": -76.3250,
-        "fallback_lat": 18.0,
-        "fallback_lng": 39.5,
-        "fallback_heading": 0,
-        "fallback_desc": "Red Sea — Operation Epic Fury (USNI Mar 9)",
     },
     "CVN-74": {
         "name": "USS John C. Stennis (CVN-74)",
@@ -83,10 +114,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
         "homeport": "Norfolk, VA",
         "homeport_lat": 36.9540,
         "homeport_lng": -76.3235,
-        "fallback_lat": 36.98,
-        "fallback_lng": -76.43,
-        "fallback_heading": 0,
-        "fallback_desc": "Newport News, VA (RCOH refueling overhaul)",
     },
     "CVN-75": {
         "name": "USS Harry S. Truman (CVN-75)",
@@ -94,10 +121,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
         "homeport": "Norfolk, VA",
         "homeport_lat": 36.9580,
         "homeport_lng": -76.3220,
-        "fallback_lat": 36.0,
-        "fallback_lng": 15.0,
-        "fallback_heading": 0,
-        "fallback_desc": "Mediterranean Sea deployment (USNI Mar 9)",
     },
     "CVN-77": {
         "name": "USS George H.W. Bush (CVN-77)",
@@ -105,23 +128,14 @@ CARRIER_REGISTRY: Dict[str, dict] = {
         "homeport": "Norfolk, VA",
         "homeport_lat": 36.9620,
         "homeport_lng": -76.3210,
-        "fallback_lat": 36.5,
-        "fallback_lng": -74.0,
-        "fallback_heading": 0,
-        "fallback_desc": "Atlantic — Pre-deployment workups (USNI Mar 9)",
     },
     # --- San Diego, CA (Naval Base San Diego) ---
-    # Carrier piers along the east shore of San Diego Bay, spread N-S
     "CVN-70": {
         "name": "USS Carl Vinson (CVN-70)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Carl_Vinson",
         "homeport": "San Diego, CA",
         "homeport_lat": 32.6840,
         "homeport_lng": -117.1290,
-        "fallback_lat": 32.6840,
-        "fallback_lng": -117.1290,
-        "fallback_heading": 180,
-        "fallback_desc": "San Diego, CA (Homeport)",
     },
     "CVN-71": {
         "name": "USS Theodore Roosevelt (CVN-71)",
@@ -129,10 +143,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
         "homeport": "San Diego, CA",
         "homeport_lat": 32.6885,
         "homeport_lng": -117.1280,
-        "fallback_lat": 32.6885,
-        "fallback_lng": -117.1280,
-        "fallback_heading": 180,
-        "fallback_desc": "San Diego, CA (Maintenance)",
     },
     "CVN-72": {
         "name": "USS Abraham Lincoln (CVN-72)",
@@ -140,10 +150,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
         "homeport": "San Diego, CA",
         "homeport_lat": 32.6925,
         "homeport_lng": -117.1275,
-        "fallback_lat": 20.0,
-        "fallback_lng": 64.0,
-        "fallback_heading": 0,
-        "fallback_desc": "Arabian Sea — Operation Epic Fury (USNI Mar 9)",
     },
     # --- Yokosuka, Japan (CFAY) ---
     "CVN-73": {
@@ -152,16 +158,18 @@ CARRIER_REGISTRY: Dict[str, dict] = {
         "homeport": "Yokosuka, Japan",
         "homeport_lat": 35.2830,
         "homeport_lng": 139.6700,
-        "fallback_lat": 35.2830,
-        "fallback_lng": 139.6700,
-        "fallback_heading": 180,
-        "fallback_desc": "Yokosuka, Japan (Forward deployed)",
     },
 }
 
 # -----------------------------------------------------------------
-# Region → approximate center coordinates
-# Used to map textual geographic descriptions to lat/lng
+# Region → approximate center coordinates.
+#
+# Issue #245 (tg12): converting a region name straight into precise
+# map coordinates is false precision. We still use this table to
+# infer a coarse position from a headline mention, but the resulting
+# carrier object is now stamped ``position_confidence = "approximate"``
+# so the UI can render an uncertainty radius / dimmed icon. The
+# centroid is a best-effort midpoint of the named body of water.
 # -----------------------------------------------------------------
 REGION_COORDS: Dict[str, tuple] = {
     # Oceans & Seas
@@ -220,9 +228,39 @@ REGION_COORDS: Dict[str, tuple] = {
 }
 
 # -----------------------------------------------------------------
-# Cache file for persisting positions between restarts
+# Files
 # -----------------------------------------------------------------
-CACHE_FILE = Path(__file__).parent.parent / "carrier_cache.json"
+#
+# The seed lives in the read-only image data dir (it ships with each
+# release). The cache lives in the same data dir but is written at
+# runtime; under Docker compose this dir is volume-mounted so the
+# cache persists across container restarts, which is the whole point
+# of the seed-then-observe model — the user's runtime observations
+# survive image upgrades.
+SEED_FILE = Path(__file__).parent.parent / "data" / "carrier_seed.json"
+CACHE_FILE = Path(__file__).parent.parent / "data" / "carrier_cache.json"
+
+# -----------------------------------------------------------------
+# Freshness window for position_confidence labeling. Issue #246 (tg12):
+# previously persisted cache entries had no freshness signal at all.
+# After this change, the position itself is preserved (we never lose
+# what was last observed) but the confidence label flips from
+# "recent" to "stale" once the underlying source is older than this
+# window. Operator-overridable via env var.
+# -----------------------------------------------------------------
+_DEFAULT_FRESHNESS_WINDOW_DAYS = 14
+
+
+def _freshness_window_days() -> int:
+    raw = str(os.environ.get("SHADOWBROKER_CARRIER_FRESHNESS_DAYS", "") or "").strip()
+    if not raw:
+        return _DEFAULT_FRESHNESS_WINDOW_DAYS
+    try:
+        n = int(raw)
+        return n if n > 0 else _DEFAULT_FRESHNESS_WINDOW_DAYS
+    except (TypeError, ValueError):
+        return _DEFAULT_FRESHNESS_WINDOW_DAYS
+
 
 _carrier_positions: Dict[str, dict] = {}
 _positions_lock = threading.Lock()
@@ -234,25 +272,159 @@ _GDELT_REQUEST_DELAY_SECONDS = 1.25
 _GDELT_REQUEST_JITTER_SECONDS = 0.35
 
 
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _parse_iso(ts: str) -> Optional[datetime]:
+    if not ts:
+        return None
+    try:
+        # Python's fromisoformat accepts +00:00 but not 'Z' until 3.11.
+        normalized = ts.replace("Z", "+00:00")
+        dt = datetime.fromisoformat(normalized)
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        return dt
+    except (TypeError, ValueError):
+        return None
+
+
+def _compute_position_confidence(entry: dict, *, now: Optional[datetime] = None) -> str:
+    """Return the public confidence label for a carrier cache entry.
+
+    Order of precedence:
+      - explicit "homeport_default" / "seed" labels are preserved.
+      - dated entries (with position_source_at) are "recent" if within
+        the configured freshness window, else "stale".
+      - missing position_source_at falls through to "stale".
+    """
+    raw_label = str(entry.get("position_confidence", "") or "").strip()
+    # Explicit "kind of provenance" labels are preserved as-is. They
+    # describe HOW we got the position, not WHEN — a fresh headline-to-
+    # centroid match (#245) is still imprecise no matter how recently
+    # it was observed, and the seed (#244) is always the seed.
+    if raw_label in {"seed", "homeport_default", "approximate"}:
+        # Approximate entries can still age into "stale_approximate" if
+        # they fall out of the freshness window — that distinction lets
+        # the UI render a different badge for old-and-imprecise vs
+        # recent-and-imprecise. seed/homeport_default never age (they
+        # were never timestamped against real observations).
+        if raw_label == "approximate":
+            source_at = _parse_iso(str(entry.get("position_source_at", "") or ""))
+            if source_at is not None:
+                reference = now or datetime.now(timezone.utc)
+                if reference - source_at > timedelta(days=_freshness_window_days()):
+                    return "stale_approximate"
+        return raw_label
+
+    source_at = _parse_iso(str(entry.get("position_source_at", "") or ""))
+    if not source_at:
+        return "stale"
+
+    reference = now or datetime.now(timezone.utc)
+    window = timedelta(days=_freshness_window_days())
+    if reference - source_at <= window:
+        return "recent"
+    return "stale"
+
+
+def _load_seed() -> Dict[str, dict]:
+    """Load the read-only seed file shipped with the image.
+
+    Returns a hull→entry dict (no _meta wrapper). Missing or malformed
+    seed files yield an empty dict — the caller falls back to homeport
+    defaults.
+    """
+    try:
+        if not SEED_FILE.exists():
+            logger.info("Carrier seed file not present at %s; first-run will fall back to homeport defaults", SEED_FILE)
+            return {}
+        raw = json.loads(SEED_FILE.read_text(encoding="utf-8"))
+        carriers = raw.get("carriers", {}) if isinstance(raw, dict) else {}
+        if not isinstance(carriers, dict):
+            return {}
+        logger.info("Carrier seed loaded: %d entries from %s", len(carriers), SEED_FILE)
+        return carriers
+    except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
+        logger.warning("Failed to load carrier seed file %s: %s", SEED_FILE, e)
+        return {}
+
+
 def _load_cache() -> Dict[str, dict]:
-    """Load cached carrier positions from disk."""
+    """Load the mutable cache (last-known positions persisted between restarts)."""
     try:
         if CACHE_FILE.exists():
-            data = json.loads(CACHE_FILE.read_text())
-            logger.info(f"Carrier cache loaded: {len(data)} carriers from {CACHE_FILE}")
-            return data
+            data = json.loads(CACHE_FILE.read_text(encoding="utf-8"))
+            if isinstance(data, dict):
+                logger.info("Carrier cache loaded: %d carriers from %s", len(data), CACHE_FILE)
+                return data
     except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
-        logger.warning(f"Failed to load carrier cache: {e}")
+        logger.warning("Failed to load carrier cache: %s", e)
     return {}
 
 
-def _save_cache(positions: Dict[str, dict]):
-    """Persist carrier positions to disk."""
+def _save_cache(positions: Dict[str, dict]) -> None:
+    """Persist the mutable cache. Atomic write (temp + rename) so a crash
+    mid-write can't leave the file truncated."""
     try:
-        CACHE_FILE.write_text(json.dumps(positions, indent=2))
-        logger.info(f"Carrier cache saved: {len(positions)} carriers")
+        CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
+        tmp = CACHE_FILE.with_suffix(CACHE_FILE.suffix + ".tmp")
+        tmp.write_text(json.dumps(positions, indent=2), encoding="utf-8")
+        # On Windows os.replace is atomic and overwrites existing files.
+        os.replace(tmp, CACHE_FILE)
+        logger.info("Carrier cache saved: %d carriers", len(positions))
     except (IOError, OSError) as e:
-        logger.warning(f"Failed to save carrier cache: {e}")
+        logger.warning("Failed to save carrier cache: %s", e)
+
+
+def _homeport_entry_for(hull: str) -> Optional[dict]:
+    """Return a homeport-default cache entry for a hull, or None if the
+    hull is not in the registry."""
+    info = CARRIER_REGISTRY.get(hull)
+    if not info:
+        return None
+    return {
+        "lat": info["homeport_lat"],
+        "lng": info["homeport_lng"],
+        "heading": 0,
+        "desc": f"{info['homeport']} (no observations yet)",
+        "source": f"Homeport default ({info['homeport']})",
+        "source_url": info.get("wiki", ""),
+        "position_source_at": _now_iso(),
+        "position_confidence": "homeport_default",
+    }
+
+
+def _bootstrap_cache_if_missing() -> Dict[str, dict]:
+    """One-shot: if no cache exists, materialize one from the seed file.
+
+    Returns the cache contents (hull→entry). On first-ever startup,
+    this writes ``carrier_cache.json`` so subsequent restarts skip the
+    seed entirely. Operator-deleted caches re-bootstrap the same way —
+    operators can use that to "reset" carrier positions, but it's an
+    explicit operator action.
+    """
+    if CACHE_FILE.exists():
+        return _load_cache()
+
+    seed = _load_seed()
+    if not seed:
+        # No seed file either. Build a homeport-default cache so the
+        # first save_cache call still produces something honest.
+        homeports: Dict[str, dict] = {}
+        for hull in CARRIER_REGISTRY:
+            entry = _homeport_entry_for(hull)
+            if entry is not None:
+                homeports[hull] = entry
+        if homeports:
+            _save_cache(homeports)
+        return homeports
+
+    # Persist the seed as the first cache so subsequent runs skip this branch.
+    _save_cache(seed)
+    logger.info("Carrier cache bootstrapped from seed (first-ever startup)")
+    return dict(seed)
 
 
 def _match_region(text: str) -> Optional[tuple]:
@@ -270,10 +442,8 @@ def _match_carrier(text: str) -> Optional[str]:
     for hull, info in CARRIER_REGISTRY.items():
         hull_check = hull.lower().replace("-", "")
         name_parts = info["name"].lower()
-        # Match hull number (e.g., "CVN-78", "CVN78")
         if hull.lower() in text_lower or hull_check in text_lower.replace("-", ""):
             return hull
-        # Match ship name (e.g., "Ford", "Eisenhower", "Vinson")
         ship_name = name_parts.split("(")[0].strip()
         last_name = ship_name.split()[-1] if ship_name else ""
         if last_name and len(last_name) > 3 and last_name in text_lower:
@@ -323,8 +493,9 @@ def _fetch_gdelt_carrier_news() -> List[dict]:
             articles = data.get("articles", [])
             for art in articles:
                 title = art.get("title", "")
-                url = art.get("url", "")
-                results.append({"title": title, "url": url})
+                article_url = art.get("url", "")
+                article_at = art.get("seendate") or art.get("date") or ""
+                results.append({"title": title, "url": article_url, "seendate": article_at})
         except (ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
             logger.debug(f"GDELT search failed for '{term}': {e}")
             continue
@@ -340,108 +511,175 @@ def _fetch_gdelt_carrier_news() -> List[dict]:
     return results
 
 
+def _gdelt_seendate_to_iso(seendate: str) -> Optional[str]:
+    """GDELT returns YYYYMMDDhhmmss (UTC). Convert to ISO8601 for
+    position_source_at. Returns None if the input is unparseable."""
+    raw = (seendate or "").strip()
+    if len(raw) < 8 or not raw.isdigit():
+        return None
+    try:
+        dt = datetime.strptime(raw[:14] if len(raw) >= 14 else raw[:8] + "000000", "%Y%m%d%H%M%S")
+        return dt.replace(tzinfo=timezone.utc).isoformat()
+    except (TypeError, ValueError):
+        return None
+
+
 def _parse_carrier_positions_from_news(articles: List[dict]) -> Dict[str, dict]:
-    """Parse carrier positions from news article titles and descriptions."""
+    """Parse carrier positions from news article titles.
+
+    Issue #245 (tg12): the position is a region centroid, which is
+    coarse — we now stamp ``position_confidence = "approximate"`` so
+    the UI can render that uncertainty. Issue #244: the
+    ``position_source_at`` field is the news article's actual seen
+    date, NOT now(), so the freshness check correctly flips entries
+    to "stale" once they age past the configured window.
+    """
     updates: Dict[str, dict] = {}
 
     for article in articles:
         title = article.get("title", "")
-
-        # Try to match a carrier from the title
         hull = _match_carrier(title)
         if not hull:
             continue
-
-        # Try to match a region from the title
         coords = _match_region(title)
         if not coords:
             continue
 
-        # Only update if we haven't seen this carrier yet (first match wins — most recent)
+        # First match wins (most recent article, GDELT returns newest first
+        # per term).
         if hull not in updates:
+            iso_at = _gdelt_seendate_to_iso(str(article.get("seendate", ""))) or _now_iso()
             updates[hull] = {
                 "lat": coords[0],
                 "lng": coords[1],
+                "heading": 0,
                 "desc": title[:100],
-                "source": "GDELT News API",
+                "source": "GDELT News API (headline region match — approximate)",
                 "source_url": article.get("url", "https://api.gdeltproject.org"),
-                "updated": datetime.now(timezone.utc).isoformat(),
+                "position_source_at": iso_at,
+                # Headline-to-centroid match is explicitly approximate.
+                "position_confidence": "approximate",
             }
             logger.info(
-                f"Carrier update: {CARRIER_REGISTRY[hull]['name']} → {coords} (from: {title[:80]})"
+                "Carrier update: %s → %s (from: %s)",
+                CARRIER_REGISTRY[hull]["name"],
+                coords,
+                title[:80],
             )
 
     return updates
 
 
-def _load_carrier_fallbacks() -> Dict[str, dict]:
-    """Build carrier positions from static fallbacks + disk cache (instant, no network)."""
-    positions: Dict[str, dict] = {}
-    for hull, info in CARRIER_REGISTRY.items():
-        positions[hull] = {
-            "name": info["name"],
-            "lat": info["fallback_lat"],
-            "lng": info["fallback_lng"],
-            "heading": info["fallback_heading"],
-            "desc": info["fallback_desc"],
-            "wiki": info["wiki"],
-            "source": "USNI News Fleet & Marine Tracker",
-            "source_url": "https://news.usni.org/category/fleet-tracker",
-            "updated": datetime.now(timezone.utc).isoformat(),
-        }
-
-    # Overlay cached positions from previous runs (may have GDELT data)
-    cached = _load_cache()
-    for hull, cached_pos in cached.items():
-        if hull in positions:
-            if cached_pos.get("source", "").startswith("GDELT") or cached_pos.get(
-                "source", ""
-            ).startswith("News"):
-                positions[hull].update(
-                    {
-                        "lat": cached_pos["lat"],
-                        "lng": cached_pos["lng"],
-                        "desc": cached_pos.get("desc", positions[hull]["desc"]),
-                        "source": cached_pos.get("source", "Cached OSINT"),
-                        "updated": cached_pos.get("updated", ""),
-                    }
-                )
-    return positions
+def _enrich_for_rendering(hull: str, entry: dict, *, now: Optional[datetime] = None) -> dict:
+    """Add live computed fields (confidence label, last_osint_update)
+    on top of the persisted cache entry. The persisted entry is left
+    untouched; this function builds the public-facing object.
+    """
+    info = CARRIER_REGISTRY.get(hull, {})
+    confidence = _compute_position_confidence(entry, now=now)
+    return {
+        "name": entry.get("name", info.get("name", hull)),
+        "lat": entry["lat"],
+        "lng": entry["lng"],
+        "heading": entry.get("heading", 0),
+        "desc": entry.get("desc", ""),
+        "wiki": entry.get("wiki", info.get("wiki", "")),
+        "source": entry.get("source", "OSINT estimated position"),
+        "source_url": entry.get("source_url", ""),
+        "position_source_at": entry.get("position_source_at", ""),
+        "position_confidence": confidence,
+        # Existing field preserved for backward compatibility with the
+        # current frontend ShipPopup; now reflects the SOURCE's observed
+        # time (not now()), so "last reported X days ago" is honest.
+        "last_osint_update": entry.get("position_source_at", ""),
+        # Convenience boolean for the UI: true when the position is
+        # NOT live OSINT (used to render dimmed icons / badges).
+        "is_fallback": confidence in {"seed", "stale", "stale_approximate", "homeport_default"},
+    }
 
 
-def update_carrier_positions():
-    """Main update function — called on startup and every 12h.
+def update_carrier_positions() -> None:
+    """Refresh carrier positions.
 
-    Phase 1 (instant): publish fallback + cached positions so the map has carriers immediately.
-    Phase 2 (slow):    query GDELT for fresh OSINT positions and update in-place.
+    Phase 1 (instant): publish whatever's in carrier_cache.json (or
+    bootstrap from seed on first-ever run), so the map has carriers
+    immediately.
+
+    Phase 2 (slow): query GDELT and replace position entries for any
+    carrier mentioned in fresh news. Persist back to cache.
     """
     global _last_update
 
-    # --- Phase 1: instant fallback + cache ---
-    positions = _load_carrier_fallbacks()
+    # --- Phase 1: instant cache (bootstrap from seed on first-ever run) ---
+    positions = _bootstrap_cache_if_missing()
+
+    # Ensure every registered hull has SOMETHING in the cache. A hull
+    # the seed didn't cover (e.g. added after install) renders at its
+    # homeport with "homeport_default" confidence.
+    for hull in CARRIER_REGISTRY:
+        if hull not in positions:
+            entry = _homeport_entry_for(hull)
+            if entry is not None:
+                positions[hull] = entry
 
     with _positions_lock:
-        # Only overwrite if positions are currently empty (first startup).
-        # If we already have data from a previous cycle, keep it while GDELT runs.
         if not _carrier_positions:
             _carrier_positions.update(positions)
             _last_update = datetime.now(timezone.utc)
     logger.info(
-        f"Carrier tracker: {len(positions)} carriers loaded from fallback/cache (GDELT enrichment starting...)"
+        "Carrier tracker: %d carriers loaded from cache (USNI + GDELT enrichment starting...)",
+        len(positions),
     )
 
-    # --- Phase 2: slow GDELT enrichment ---
+    # --- Phase 2: USNI Fleet & Marine Tracker (PRIMARY source) ---
+    #
+    # USNI publishes a weekly editorial tracker with each carrier's
+    # actual operating area, parsed from explicit prose like
+    #   "The Gerald R. Ford Carrier Strike Group is operating in the Red Sea"
+    # These positions are tagged ``position_confidence: "recent"`` because
+    # they reflect actual reporting, not headline-keyword centroids.
+    # USNI updates are preferred over GDELT — they're authoritative on
+    # US Navy positions where GDELT is just article-title text mining.
+    try:
+        from services.fetchers.usni_fleet_tracker import (
+            fetch_latest_fleet_tracker_positions,
+        )
+        usni_positions = fetch_latest_fleet_tracker_positions()
+        for hull, pos in usni_positions.items():
+            positions[hull] = pos
+            logger.info(
+                "Carrier USNI update: %s → %s",
+                CARRIER_REGISTRY[hull]["name"],
+                pos.get("desc", ""),
+            )
+    except Exception as e:
+        logger.warning("USNI fleet-tracker fetch failed: %s", e)
+
+    # --- Phase 3: GDELT enrichment (SECONDARY — fills gaps) ---
+    #
+    # Used only to backfill carriers USNI didn't mention this week. The
+    # position is stamped ``approximate`` so the UI knows it's a
+    # headline-centroid match (Issue #245).
     try:
         articles = _fetch_gdelt_carrier_news()
         news_positions = _parse_carrier_positions_from_news(articles)
         for hull, pos in news_positions.items():
-            if hull in positions:
-                positions[hull].update(pos)
-                logger.info(f"Carrier OSINT: updated {CARRIER_REGISTRY[hull]['name']} from news")
+            # Only overwrite if the existing entry is NOT a recent USNI
+            # observation. A "recent" USNI position is higher-confidence
+            # than a GDELT headline-centroid match — don't let GDELT
+            # demote a real position to an approximate one.
+            existing = positions.get(hull, {})
+            existing_conf = _compute_position_confidence(existing)
+            if existing_conf == "recent":
+                continue
+            positions[hull] = pos
+            logger.info(
+                "Carrier OSINT: updated %s from GDELT news",
+                CARRIER_REGISTRY[hull]["name"],
+            )
     except (ValueError, KeyError, json.JSONDecodeError, OSError) as e:
-        logger.warning(f"GDELT carrier fetch failed: {e}")
+        logger.warning("GDELT carrier fetch failed: %s", e)
 
-    # Save and update the global state with enriched positions
     with _positions_lock:
         _carrier_positions.clear()
         _carrier_positions.update(positions)
@@ -449,21 +687,15 @@ def update_carrier_positions():
 
     _save_cache(positions)
 
-    sources = {}
-    for p in positions.values():
-        src = p.get("source", "unknown")
-        sources[src] = sources.get(src, 0) + 1
-    logger.info(f"Carrier tracker: {len(positions)} carriers updated. Sources: {sources}")
+    confidences: Dict[str, int] = {}
+    for entry in positions.values():
+        label = _compute_position_confidence(entry)
+        confidences[label] = confidences.get(label, 0) + 1
+    logger.info("Carrier tracker: %d carriers updated. Confidence: %s", len(positions), confidences)
 
 
 def _deconflict_positions(result: List[dict]) -> List[dict]:
-    """Offset carriers that share identical coordinates so they don't stack.
-
-    At port: offset along the pier axis (~500m / 0.004° apart).
-    At sea: offset perpendicular to each other (~0.08° / ~9km apart)
-    so they're visibly separate but clearly operating together.
-    """
-    # Group by rounded lat/lng (within ~0.01° ≈ 1km = same spot)
+    """Offset carriers that share identical coordinates so they don't stack."""
     from collections import defaultdict
 
     groups: dict[str, list[int]] = defaultdict(list)
@@ -475,7 +707,6 @@ def _deconflict_positions(result: List[dict]) -> List[dict]:
         if len(indices) < 2:
             continue
         n = len(indices)
-        # Determine if this is a port (near a homeport) or at sea
         sample = result[indices[0]]
         at_port = any(
             abs(sample["lat"] - info.get("homeport_lat", 0)) < 0.05
@@ -484,7 +715,6 @@ def _deconflict_positions(result: List[dict]) -> List[dict]:
         )
 
         if at_port:
-            # Use each carrier's distinct homeport pier coordinates
             for idx in indices:
                 carrier = result[idx]
                 hull = None
@@ -497,8 +727,7 @@ def _deconflict_positions(result: List[dict]) -> List[dict]:
                     carrier["lat"] = info["homeport_lat"]
                     carrier["lng"] = info["homeport_lng"]
         else:
-            # At sea: spread in a line perpendicular to travel (~0.08° apart)
-            spacing = 0.08  # ~9km — close enough to see they're together
+            spacing = 0.08
             start_offset = -(n - 1) * spacing / 2
             for j, idx in enumerate(indices):
                 result[idx]["lng"] += start_offset + j * spacing
@@ -507,36 +736,44 @@ def _deconflict_positions(result: List[dict]) -> List[dict]:
 
 
 def get_carrier_positions() -> List[dict]:
-    """Return current carrier positions for the data pipeline."""
+    """Return current carrier positions for the data pipeline.
+
+    Each entry has the full provenance + freshness fields; the UI can
+    decide how to render them. Carriers are never hidden — only
+    labeled.
+    """
+    now = datetime.now(timezone.utc)
     with _positions_lock:
-        result = []
-        for hull, pos in _carrier_positions.items():
-            info = CARRIER_REGISTRY.get(hull, {})
+        result: List[dict] = []
+        for hull, entry in _carrier_positions.items():
+            enriched = _enrich_for_rendering(hull, entry, now=now)
             result.append(
                 {
-                    "name": pos.get("name", info.get("name", hull)),
+                    "name": enriched["name"],
                     "type": "carrier",
-                    "lat": pos["lat"],
-                    "lng": pos["lng"],
-                    "heading": None,  # Heading unknown for carriers — OSINT cannot determine true heading
+                    "lat": enriched["lat"],
+                    "lng": enriched["lng"],
+                    "heading": None,  # OSINT cannot determine true heading.
                     "sog": 0,
                     "cog": 0,
                     "country": "United States",
-                    "desc": pos.get("desc", ""),
-                    "wiki": pos.get("wiki", info.get("wiki", "")),
+                    "desc": enriched["desc"],
+                    "wiki": enriched["wiki"],
                     "estimated": True,
-                    "source": pos.get("source", "OSINT estimated position"),
-                    "source_url": pos.get(
-                        "source_url", "https://news.usni.org/category/fleet-tracker"
-                    ),
-                    "last_osint_update": pos.get("updated", ""),
+                    "source": enriched["source"],
+                    "source_url": enriched["source_url"],
+                    "last_osint_update": enriched["last_osint_update"],
+                    # New fields (additive — existing UI continues to work):
+                    "position_source_at": enriched["position_source_at"],
+                    "position_confidence": enriched["position_confidence"],
+                    "is_fallback": enriched["is_fallback"],
                 }
             )
         return _deconflict_positions(result)
 
 
 # -----------------------------------------------------------------
-# Scheduler: runs at startup, then at 00:00 and 12:00 UTC daily
+# Scheduler: runs at startup, then at 00:00 and 12:00 UTC daily.
 # -----------------------------------------------------------------
 _scheduler_thread: Optional[threading.Thread] = None
 _scheduler_stop = threading.Event()
@@ -544,7 +781,6 @@ _scheduler_stop = threading.Event()
 
 def _scheduler_loop():
     """Background thread that triggers updates at 00:00 and 12:00 UTC."""
-    # Initial update on startup
     try:
         update_carrier_positions()
     except Exception as e:
@@ -552,7 +788,6 @@ def _scheduler_loop():
 
     while not _scheduler_stop.is_set():
         now = datetime.now(timezone.utc)
-        # Next target: 00:00 or 12:00 UTC, whichever is sooner
         hour = now.hour
         if hour < 12:
             next_hour = 12
@@ -561,18 +796,17 @@ def _scheduler_loop():
 
         next_run = now.replace(hour=next_hour % 24, minute=0, second=0, microsecond=0)
         if next_hour == 24:
-            from datetime import timedelta
-
             next_run = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
 
         wait_seconds = (next_run - now).total_seconds()
         logger.info(
-            f"Carrier tracker: next update at {next_run.isoformat()} ({wait_seconds/3600:.1f}h)"
+            "Carrier tracker: next update at %s (%.1fh)",
+            next_run.isoformat(),
+            wait_seconds / 3600,
         )
 
-        # Wait until next scheduled time, or until stop event
         if _scheduler_stop.wait(timeout=wait_seconds):
-            break  # Stop event was set
+            break
 
         try:
             update_carrier_positions()

backend/services/cctv_pipeline.py

@@ -818,6 +818,105 @@ out body;
         return cameras
 
 
+# ---------------------------------------------------------------------------
+# ALPR / Surveillance Camera Locations (OSM Overpass)
+# ---------------------------------------------------------------------------
+# Queries OpenStreetMap for ALPR/LPR tagged surveillance cameras.
+# These cameras rarely have public media URLs — this ingestor captures
+# their LOCATIONS for situational awareness (density heatmap, blind-spot
+# analysis).  No plate-read data is fetched — only publicly-mapped positions.
+
+
+class OSMALPRCameraIngestor(BaseCCTVIngestor):
+    """ALPR / license-plate reader camera locations from OpenStreetMap.
+
+    Searches for nodes tagged with surveillance:type=ALPR or
+    man_made=surveillance + camera:type values indicating plate readers.
+    Only geolocations are ingested — no live feeds or detection data.
+    """
+
+    URL = "https://overpass-api.de/api/interpreter"
+    QUERY = """
+[out:json][timeout:45];
+(
+  node["surveillance:type"="ALPR"];
+  node["surveillance:type"="alpr"];
+  node["surveillance:type"="LPR"];
+  node["surveillance:type"="lpr"];
+  node["man_made"="surveillance"]["camera:type"="ALPR"];
+  node["man_made"="surveillance"]["camera:type"="alpr"];
+  node["man_made"="surveillance"]["camera:type"="LPR"];
+  node["man_made"="surveillance"]["camera:type"="lpr"];
+  node["man_made"="surveillance"]["description"~"[Ll]icense [Pp]late"];
+  node["man_made"="surveillance"]["description"~"ALPR"];
+  node["man_made"="surveillance"]["description"~"Flock"];
+);
+out body;
+""".strip()
+
+    def fetch_data(self) -> List[Dict[str, Any]]:
+        query = quote(self.QUERY, safe="")
+        resp = fetch_with_curl(
+            f"{self.URL}?data={query}",
+            timeout=50,
+            headers={"Accept": "application/json"},
+        )
+        if not resp or resp.status_code != 200:
+            logger.warning(
+                "OSM ALPR camera fetch failed: HTTP %s",
+                resp.status_code if resp else "no response",
+            )
+            return []
+        data = resp.json()
+        cameras = []
+        for item in data.get("elements", []) if isinstance(data, dict) else []:
+            lat = item.get("lat")
+            lon = item.get("lon")
+            if lat is None or lon is None:
+                continue
+            try:
+                lat, lon = float(lat), float(lon)
+            except (ValueError, TypeError):
+                continue
+
+            tags = item.get("tags", {}) if isinstance(item.get("tags"), dict) else {}
+
+            # Extract what we can from tags
+            operator = (
+                tags.get("operator")
+                or tags.get("brand")
+                or tags.get("network")
+                or "Unknown"
+            )
+            description = (
+                tags.get("description")
+                or tags.get("name")
+                or tags.get("surveillance:type", "ALPR")
+            )
+            direction = (
+                tags.get("camera:direction")
+                or tags.get("direction")
+                or tags.get("surveillance:direction")
+                or "Unknown"
+            )
+
+            # ALPR cameras typically have no public media URL — use a
+            # placeholder so the pin renders but no proxy attempt is made.
+            cameras.append(
+                {
+                    "id": f"ALPR-{item.get('id')}",
+                    "source_agency": str(operator)[:60],
+                    "lat": lat,
+                    "lon": lon,
+                    "direction_facing": f"ALPR: {str(description)[:100]} ({str(direction)[:30]})",
+                    "media_url": "",
+                    "media_type": "none",
+                    "refresh_rate_seconds": 0,
+                }
+            )
+        logger.info("OSM ALPR ingestor found %d cameras", len(cameras))
+        return cameras
+
 
 # ---------------------------------------------------------------------------
 # DGT Spain — National Road Cameras
@@ -888,7 +987,7 @@ _KML_NS = {"kml": "http://www.opengis.net/kml/2.2"}
 
 def _find_kml_element(element, tag):
     """Find first descendant matching tag, ignoring XML namespace prefix."""
-    import xml.etree.ElementTree as ET
+    import defusedxml.ElementTree as ET
     el = element.find(f".//{tag}")
     if el is not None:
         return el
@@ -916,7 +1015,7 @@ class MadridCityIngestor(BaseCCTVIngestor):
     KML_URL = "http://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml"
 
     def fetch_data(self) -> List[Dict[str, Any]]:
-        import xml.etree.ElementTree as ET
+        import defusedxml.ElementTree as ET
 
         try:
             response = fetch_with_curl(self.KML_URL, timeout=20)

backend/services/config.py

@@ -10,6 +10,10 @@ class Settings(BaseSettings):
     ALLOW_INSECURE_ADMIN: bool = False
     PUBLIC_API_KEY: str = ""
 
+    # OpenClaw agent connectivity
+    OPENCLAW_HMAC_SECRET: str = ""        # HMAC shared secret for direct mode (auto-generated if empty)
+    OPENCLAW_ACCESS_TIER: str = "restricted"  # "full" or "restricted"
+
     # Data sources
     AIS_API_KEY: str = ""
     OPENSKY_CLIENT_ID: str = ""
@@ -28,16 +32,34 @@ class Settings(BaseSettings):
     MESH_ARTI_ENABLED: bool = False
     MESH_ARTI_SOCKS_PORT: int = 9050
     MESH_RELAY_PEERS: str = ""
+    MESH_PUBLIC_PEER_URL: str = ""
+    # Bootstrap seeds are discovery hints, not authoritative network roots.
+    # Nodes promote healthy discovered peers from the store/manifest over time.
+    MESH_BOOTSTRAP_SEED_PEERS: str = "http://gqpbunqbgtkcqilvclm3xrkt3zowjyl3s62kkktvojgvxzizamvbrqid.onion:8000"
+    # Legacy name kept for older compose/.env files.
+    MESH_DEFAULT_SYNC_PEERS: str = ""
+    # Infonet/Wormhole must fail closed to private transports by default.
+    # Set true only for local relay development or explicitly public testnets.
+    MESH_INFONET_ALLOW_CLEARNET_SYNC: bool = False
     MESH_BOOTSTRAP_DISABLED: bool = False
     MESH_BOOTSTRAP_MANIFEST_PATH: str = "data/bootstrap_peers.json"
     MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY: str = ""
     MESH_NODE_MODE: str = "participant"
     MESH_SYNC_INTERVAL_S: int = 300
     MESH_SYNC_FAILURE_BACKOFF_S: int = 60
+    MESH_SYNC_TIMEOUT_S: int = 5
+    MESH_SYNC_MAX_PEERS_PER_CYCLE: int = 3
     MESH_RELAY_PUSH_TIMEOUT_S: int = 10
     MESH_RELAY_MAX_FAILURES: int = 3
     MESH_RELAY_FAILURE_COOLDOWN_S: int = 120
+    MESH_BOOTSTRAP_SEED_FAILURE_COOLDOWN_S: int = 15
     MESH_PEER_PUSH_SECRET: str = ""
+    # Issue #256 (tg12): optional per-peer HMAC secret map. Comma-separated
+    # `url=secret` pairs. When a peer URL appears here, only that per-peer
+    # secret is accepted for it — the global MESH_PEER_PUSH_SECRET above is
+    # ignored for that specific URL. Single-peer installs and unmigrated
+    # multi-peer installs leave this empty and behavior is unchanged.
+    MESH_PEER_SECRETS: str = ""
     MESH_RNS_APP_NAME: str = "shadowbroker"
     MESH_RNS_ASPECT: str = "infonet"
     MESH_RNS_IDENTITY_PATH: str = ""
@@ -60,7 +82,8 @@ class Settings(BaseSettings):
     # Keep a low background cadence on private RNS links so quiet nodes are less
     # trivially fingerprintable by silence alone. Set to 0 to disable explicitly.
     MESH_RNS_COVER_INTERVAL_S: int = 30
-    MESH_RNS_COVER_SIZE: int = 64
+    MESH_RNS_COVER_SIZE: int = 512
+    MESH_DM_MAILBOX_TTL_S: int = 900
     MESH_RNS_IBF_WINDOW: int = 256
     MESH_RNS_IBF_TABLE_SIZE: int = 64
     MESH_RNS_IBF_MINHASH_SIZE: int = 16
@@ -75,48 +98,329 @@ class Settings(BaseSettings):
     MESH_RNS_IBF_FAIL_THRESHOLD: int = 3
     MESH_RNS_IBF_COOLDOWN_S: int = 120
     MESH_VERIFY_INTERVAL_S: int = 600
-    MESH_VERIFY_SIGNATURES: bool = True
+    # MESH_VERIFY_SIGNATURES is intentionally removed — the audit loop in main.py
+    # always calls validate_chain_incremental(verify_signatures=True). Any value
+    # set in the environment is ignored.
     MESH_DM_SECURE_MODE: bool = True
     MESH_DM_TOKEN_PEPPER: str = ""
-    MESH_DM_ALLOW_LEGACY_GET: bool = False
+    MESH_ALLOW_LEGACY_DM1_UNTIL: str = ""
+    MESH_ALLOW_LEGACY_DM_GET_UNTIL: str = ""
+    MESH_ALLOW_LEGACY_DM_SIGNATURE_COMPAT_UNTIL: str = ""
     MESH_DM_PERSIST_SPOOL: bool = False
+    MESH_DM_RELAY_FILE_PATH: str = ""
+    MESH_DM_RELAY_AUTO_RELOAD: bool = False
     MESH_DM_REQUIRE_SENDER_SEAL_SHARED: bool = True
     MESH_DM_NONCE_TTL_S: int = 300
     MESH_DM_NONCE_CACHE_MAX: int = 4096
+    MESH_DM_NONCE_PER_AGENT_MAX: int = 256
     MESH_DM_REQUEST_MAX_AGE_S: int = 300
     MESH_DM_REQUEST_MAILBOX_LIMIT: int = 12
     MESH_DM_SHARED_MAILBOX_LIMIT: int = 48
     MESH_DM_SELF_MAILBOX_LIMIT: int = 12
+    # Anti-spam: cap on distinct UNACKED messages a single sender can have
+    # parked in a single recipient's mailbox at any one time. Once the
+    # recipient pulls (acks) a message, the sender's quota for that pair
+    # frees up. Default 2 — a sender who wants to deliver more must wait
+    # for the recipient to actually read the prior messages.
+    #
+    # This cap is enforced TWICE: once on the local deposit path (the
+    # sender's own node refuses to spool the 3rd message) AND once on
+    # the replication-acceptance path (honest peer relays refuse to
+    # accept inbound replicas that would put them over the cap). The
+    # double enforcement makes the rule a NETWORK rule — patching out
+    # the local check on a hostile sender's relay doesn't let extras
+    # propagate, because every honest peer enforces the same cap on
+    # inbound replication.
+    MESH_DM_PENDING_PER_SENDER_LIMIT: int = 2
+    MESH_BLOCK_LEGACY_AGENT_ID_LOOKUP: bool = True
+    MESH_ALLOW_COMPAT_DM_INVITE_IMPORT: bool = False
+    MESH_ALLOW_COMPAT_DM_INVITE_IMPORT_UNTIL: str = ""
+    MESH_ALLOW_LEGACY_NODE_ID_COMPAT_UNTIL: str = ""
+    # Rotate voter-blinding salts on a rolling cadence so new reputation
+    # events do not reuse one forever-stable blinded identity.
+    MESH_VOTER_BLIND_SALT_ROTATE_DAYS: int = 30
+    # Keep historical salts long enough to cover live vote records, so
+    # duplicate-vote detection and wallet-cost accounting survive rotation.
+    MESH_VOTER_BLIND_SALT_GRACE_DAYS: int = 30
     MESH_DM_MAX_MSG_BYTES: int = 8192
     MESH_DM_ALLOW_SENDER_SEAL: bool = False
     # TTL for DH key and prekey bundle registrations — stale entries are pruned.
     MESH_DM_KEY_TTL_DAYS: int = 30
+    # TTL for invite-scoped prekey lookup aliases; shorter windows reduce
+    # long-lived relay linkage between opaque lookup handles and agent IDs.
+    MESH_DM_PREKEY_LOOKUP_ALIAS_TTL_DAYS: int = 14
+    # TTL for relay witness history; keep continuity metadata bounded instead
+    # of relying on a hidden hardcoded retention window.
+    MESH_DM_WITNESS_TTL_DAYS: int = 14
     # TTL for mailbox binding metadata — shorter = smaller metadata footprint on disk.
-    MESH_DM_BINDING_TTL_DAYS: int = 7
+    MESH_DM_BINDING_TTL_DAYS: int = 3
     # When False, mailbox bindings are memory-only (agents re-register on restart).
-    MESH_DM_METADATA_PERSIST: bool = True
+    # Enable explicitly only if restart continuity is worth persisting DM graph metadata.
+    MESH_DM_METADATA_PERSIST: bool = False
+    # Second explicit opt-in for at-rest DM metadata persistence. This keeps a
+    # single boolean flip from silently writing mailbox graph metadata to disk.
+    MESH_DM_METADATA_PERSIST_ACKNOWLEDGE: bool = False
+    # Optional import path for externally managed root witness material packages.
+    # Relative paths resolve from the backend directory.
+    MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH: str = ""
+    # Optional URI for externally managed root witness material packages.
+    # Supports file:// and http(s):// sources; when set it overrides the local path.
+    MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_URI: str = ""
+    # Maximum acceptable age for externally sourced root witness packages.
+    # Strong DM trust fails closed when the imported package exported_at is older than this.
+    MESH_DM_ROOT_EXTERNAL_WITNESS_MAX_AGE_S: int = 3600
+    # Warning threshold for externally sourced root witness packages.
+    # When current external witness material reaches this age, operator health degrades to warning
+    # before the strong path eventually fails closed at MAX_AGE.
+    MESH_DM_ROOT_EXTERNAL_WITNESS_WARN_AGE_S: int = 2700
+    # Optional export path for the append-only stable-root transparency ledger.
+    # Relative paths resolve from the backend directory.
+    MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH: str = ""
+    # Optional URI used to read back and verify published transparency ledgers.
+    # Supports file:// and http(s):// sources.
+    MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI: str = ""
+    # Maximum acceptable age for externally read transparency ledgers.
+    # Strong DM trust fails closed when exported_at is older than this.
+    MESH_DM_ROOT_TRANSPARENCY_LEDGER_MAX_AGE_S: int = 3600
+    # Warning threshold for externally read transparency ledgers.
+    # When current external transparency readback reaches this age, operator health degrades to warning
+    # before the strong path eventually fails closed at MAX_AGE.
+    MESH_DM_ROOT_TRANSPARENCY_LEDGER_WARN_AGE_S: int = 2700
     MESH_SCOPED_TOKENS: str = ""
+    # Deprecated legacy env vars kept for backward config compatibility only.
+    # Ordinary shipped gate flows keep MLS decrypt local; backend decrypt is
+    # reserved for explicit recovery reads.
+    MESH_GATE_BACKEND_DECRYPT_COMPAT: bool = False
+    MESH_GATE_BACKEND_DECRYPT_COMPAT_ACKNOWLEDGE: bool = False
+    MESH_BACKEND_GATE_DECRYPT_COMPAT: bool = False
+    # Deprecated legacy env vars kept for backward config compatibility only.
+    # Ordinary shipped gate flows keep compose/post local and submit encrypted
+    # payloads to the backend for sign/post only.
+    MESH_GATE_BACKEND_PLAINTEXT_COMPAT: bool = False
+    MESH_GATE_BACKEND_PLAINTEXT_COMPAT_ACKNOWLEDGE: bool = False
+    MESH_BACKEND_GATE_PLAINTEXT_COMPAT: bool = False
+    # Runtime gate for recovery envelopes. When off, per-gate
+    # envelope_recovery / envelope_always policies fail closed to
+    # envelope_disabled. Default True so the Reddit-like durable history
+    # model works out of the box: any member with the gate_secret can
+    # decrypt every envelope encrypted from the moment they had that key.
+    # Set MESH_GATE_RECOVERY_ENVELOPE_ENABLE=false to revert to MLS-only
+    # forward-secret behavior (your own history becomes unreadable after
+    # the sending ratchet advances).
+    MESH_GATE_RECOVERY_ENVELOPE_ENABLE: bool = True
+    MESH_GATE_RECOVERY_ENVELOPE_ENABLE_ACKNOWLEDGE: bool = True
+    # Durable gate plaintext retention is disabled by default. Enable only
+    # when the operator explicitly accepts the at-rest privacy tradeoff.
+    MESH_GATE_PLAINTEXT_PERSIST: bool = False
+    MESH_GATE_PLAINTEXT_PERSIST_ACKNOWLEDGE: bool = False
     MESH_GATE_SESSION_ROTATE_MSGS: int = 50
     MESH_GATE_SESSION_ROTATE_S: int = 3600
+    MESH_GATE_LEGACY_ENVELOPE_FALLBACK_MAX_DAYS: int = 30
     # Add a randomized grace window before anonymous gate-session auto-rotation
     # so threshold-triggered identity swaps are less trivially correlated.
     MESH_GATE_SESSION_ROTATE_JITTER_S: int = 180
+    # Gate persona (named identity) rotation thresholds.  Rotating the signing
+    # key limits the linkability window.  Zero = disabled.
+    MESH_GATE_PERSONA_ROTATE_MSGS: int = 200
+    MESH_GATE_PERSONA_ROTATE_S: int = 604800  # 7 days
+    MESH_GATE_PERSONA_ROTATE_JITTER_S: int = 600
+    # Feature-flagged session stream for multiplexed gate room updates.
+    # Disabled by default so rollout stays explicit while stream-first rooms bake.
+    MESH_GATE_SESSION_STREAM_ENABLED: bool = False
+    MESH_GATE_SESSION_STREAM_HEARTBEAT_S: int = 20
+    MESH_GATE_SESSION_STREAM_BATCH_MS: int = 1500
+    MESH_GATE_SESSION_STREAM_MAX_GATES: int = 16
     # Private gate APIs expose a backward-jittered timestamp view so observers
     # cannot trivially align exact send times from response metadata alone.
     MESH_GATE_TIMESTAMP_JITTER_S: int = 60
+    # Ban/kick gate-secret rotation is on by default (hardening Rec #10): the
+    # invariant has baked and a ban that does not rotate is effectively a
+    # display-only removal. Set MESH_GATE_BAN_KICK_ROTATION_ENABLE=false to
+    # revert to observe-only during incident triage.
+    MESH_GATE_BAN_KICK_ROTATION_ENABLE: bool = True
+    MESH_BLOCK_LEGACY_NODE_ID_COMPAT: bool = True
     MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK: bool = False
+    MESH_ACK_RAW_FALLBACK_AT_OWN_RISK: bool = False
+    MESH_SECURE_STORAGE_SECRET: str = ""
+    MESH_SECURE_STORAGE_SECRET_FILE: str = ""
     MESH_PRIVATE_LOG_TTL_S: int = 900
+    # Sprint 1 rollout: restored DM boot probes stay disabled by default until
+    # the architect reviews false positives from the observe-only path.
+    MESH_DM_RESTORED_SESSION_BOOT_PROBE_ENABLE: bool = False
+    # Queued DM release requires explicit per-item approval before any weaker
+    # relay fallback. Silent fallback is not a safe private-mode default.
+    MESH_PRIVATE_RELEASE_APPROVAL_ENABLE: bool = True
+    # Expiry for user-approved scoped private relay fallback policy. The policy
+    # is still bounded by hidden-transport checks before it can auto-release.
+    MESH_PRIVATE_RELAY_POLICY_TTL_S: int = 3600
+    # Background privacy prewarm prepares keys/aliases/transport readiness
+    # before send-time. Anonymous mode uses a cadence gate so user clicks do
+    # not directly create hidden-transport activity.
+    MESH_PRIVACY_PREWARM_ENABLE: bool = True
+    MESH_PRIVACY_PREWARM_INTERVAL_S: int = 300
+    MESH_PRIVACY_PREWARM_ANON_CADENCE_S: int = 300
+    # Sprint 4 rollout: authenticated RNS cover markers remain disabled until
+    # the observer-equivalence and receive-path DoS tests are green.
+    MESH_RNS_COVER_AUTH_MARKER_ENABLE: bool = False
+    # Signed-write revocation lookups use a short local TTL; stale entries force
+    # a local rebuild before honor. Offline/local-refresh failures remain
+    # observe-only until the later enforcement sprint.
+    MESH_SIGNED_REVOCATION_CACHE_TTL_S: int = 300
+    MESH_SIGNED_REVOCATION_CACHE_ENFORCE: bool = True
+    MESH_SIGNED_WRITE_CONTEXT_REQUIRED: bool = True
+    # Sprint 5 rollout: when enabled, root witness finality requires
+    # independent quorum for threshold>1 witnessed roots before they count as
+    # verified first-contact provenance.
+    WORMHOLE_ROOT_WITNESS_FINALITY_ENFORCE: bool = False
+    # Optional JSON artifact generated by CI/release workflow for the Sprint 8
+    # release gate. Relative paths resolve from the backend directory.
+    # dev = permissive local/dev behavior; testnet-private = strict private
+    # defaults; release-candidate = no compatibility/debug escape hatches.
+    MESH_RELEASE_PROFILE: str = "dev"
+    MESH_RELEASE_ATTESTATION_PATH: str = ""
+    # Operator release attestation for the Sprint 8 release gate. This does
+    # not change runtime behavior; it only records that the DM relay security
+    # suite was run and passed for the release candidate.
+    MESH_RELEASE_DM_RELAY_SECURITY_SUITE_GREEN: bool = False
+    PRIVACY_CORE_MIN_VERSION: str = "0.1.0"
+    PRIVACY_CORE_ALLOWED_SHA256: str = ""
+    PRIVACY_CORE_DEV_OVERRIDE: bool = False
+    # Sprint 4 rollout: fail fast when the loaded privacy-core artifact is
+    # missing required FFI symbols expected by the current Python bridge.
+    PRIVACY_CORE_EXPORT_SET_AUDIT_ENABLE: bool = True
     # Clearnet fallback policy for private-tier messages.
     # "block" (default) = refuse to send private messages over clearnet.
     # "allow" = fall back to clearnet when Tor/RNS is unavailable (weaker privacy).
     MESH_PRIVATE_CLEARNET_FALLBACK: str = "block"
+    # Second explicit opt-in for private-tier clearnet fallback. Without this
+    # acknowledgement, "allow" remains requested but not effective.
+    MESH_PRIVATE_CLEARNET_FALLBACK_ACKNOWLEDGE: bool = False
+    # Meshtastic MQTT bridge — disabled by default to avoid hammering the
+    # public broker.  Users opt in explicitly.
+    MESH_MQTT_ENABLED: bool = False
     # Meshtastic MQTT broker credentials (defaults match public firmware).
+    MESH_MQTT_BROKER: str = "mqtt.meshtastic.org"
+    MESH_MQTT_PORT: int = 1883
     MESH_MQTT_USER: str = "meshdev"
     MESH_MQTT_PASS: str = "large4cats"
+    # Hex-encoded PSK — empty string means use the default LongFast key.
+    # Must decode to exactly 16 or 32 bytes when set.
+    MESH_MQTT_PSK: str = ""
+    # Optional operator-provided Meshtastic node ID (e.g. "!abcd1234") included
+    # in the User-Agent when fetching from meshtastic.liamcottle.net so the
+    # service operator can identify per-install traffic instead of a generic
+    # "ShadowBroker" aggregate.
+    MESHTASTIC_OPERATOR_CALLSIGN: str = ""
+    # Per-install operator handle used in the User-Agent for EVERY third-party
+    # API the backend calls (Wikipedia, Wikidata, Nominatim, GDELT, OpenMHz,
+    # Broadcastify, weather.gov, NUFORC, etc.). The default is empty, in which
+    # case backend/services/network_utils.py auto-generates a stable
+    # pseudonymous handle like "operator-7f3a92" on first use and caches it.
+    # Operators who want to identify themselves with a real handle can set
+    # this; operators who want to stay pseudonymous can leave it empty.
+    #
+    # The handle is sent ONLY to public third-party APIs. It is NEVER mixed
+    # into mesh / Wormhole / Infonet identity (those have their own crypto
+    # identity layer; conflating the two would leak public attribution into
+    # private mesh state).
+    OPERATOR_HANDLE: str = ""
+
+    # SAR (Synthetic Aperture Radar) data layer
+    # Mode A — free catalog metadata, no account, default-on
+    MESH_SAR_CATALOG_ENABLED: bool = True
+    # Mode B — free pre-processed anomalies (OPERA / EGMS / GFM / EMS / UNOSAT)
+    # Two-step opt-in: must be "allow" AND _ACKNOWLEDGE must be true
+    MESH_SAR_PRODUCTS_FETCH: str = "block"
+    MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE: bool = False
+    # NASA Earthdata Login (free) — required for OPERA products
+    MESH_SAR_EARTHDATA_USER: str = ""
+    MESH_SAR_EARTHDATA_TOKEN: str = ""
+    # Copernicus Data Space (free) — required for EGMS / EMS products
+    MESH_SAR_COPERNICUS_USER: str = ""
+    MESH_SAR_COPERNICUS_TOKEN: str = ""
+    # Whether OpenClaw agents may read/act on the SAR layer
+    MESH_SAR_OPENCLAW_ENABLED: bool = True
+    # Require private-tier transport before signing/broadcasting SAR anomalies
+    MESH_SAR_REQUIRE_PRIVATE_TIER: bool = True
 
     model_config = SettingsConfigDict(env_file=".env", extra="ignore")
 
 
 @lru_cache
 def get_settings() -> Settings:
+    try:
+        from services.api_settings import load_persisted_api_keys_into_environ
+        load_persisted_api_keys_into_environ()
+    except Exception:
+        pass
     return Settings()
+
+
+def private_clearnet_fallback_requested(settings: Settings | None = None) -> str:
+    snapshot = settings or get_settings()
+    policy = str(getattr(snapshot, "MESH_PRIVATE_CLEARNET_FALLBACK", "block") or "block").strip().lower()
+    return "allow" if policy == "allow" else "block"
+
+
+def private_clearnet_fallback_effective(settings: Settings | None = None) -> str:
+    snapshot = settings or get_settings()
+    requested = private_clearnet_fallback_requested(snapshot)
+    acknowledged = bool(getattr(snapshot, "MESH_PRIVATE_CLEARNET_FALLBACK_ACKNOWLEDGE", False))
+    if requested == "allow" and acknowledged:
+        return "allow"
+    return "block"
+
+
+def backend_gate_decrypt_compat_effective(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(
+        getattr(snapshot, "MESH_BACKEND_GATE_DECRYPT_COMPAT", False)
+        or getattr(snapshot, "MESH_GATE_BACKEND_DECRYPT_COMPAT", False)
+    )
+
+
+def backend_gate_plaintext_compat_effective(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(
+        getattr(snapshot, "MESH_BACKEND_GATE_PLAINTEXT_COMPAT", False)
+        or getattr(snapshot, "MESH_GATE_BACKEND_PLAINTEXT_COMPAT", False)
+    )
+
+
+def gate_recovery_envelope_effective(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    requested = bool(getattr(snapshot, "MESH_GATE_RECOVERY_ENVELOPE_ENABLE", False))
+    acknowledged = bool(getattr(snapshot, "MESH_GATE_RECOVERY_ENVELOPE_ENABLE_ACKNOWLEDGE", False))
+    return requested and acknowledged
+
+
+def gate_plaintext_persist_effective(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    requested = bool(getattr(snapshot, "MESH_GATE_PLAINTEXT_PERSIST", False))
+    acknowledged = bool(getattr(snapshot, "MESH_GATE_PLAINTEXT_PERSIST_ACKNOWLEDGE", False))
+    return requested and acknowledged
+
+
+def gate_ban_kick_rotation_enabled(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(getattr(snapshot, "MESH_GATE_BAN_KICK_ROTATION_ENABLE", False))
+
+
+def dm_restored_session_boot_probe_enabled(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(getattr(snapshot, "MESH_DM_RESTORED_SESSION_BOOT_PROBE_ENABLE", False))
+
+
+def signed_revocation_cache_ttl_s(settings: Settings | None = None) -> int:
+    snapshot = settings or get_settings()
+    return max(0, int(getattr(snapshot, "MESH_SIGNED_REVOCATION_CACHE_TTL_S", 300) or 0))
+
+
+def signed_revocation_cache_enforce(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(getattr(snapshot, "MESH_SIGNED_REVOCATION_CACHE_ENFORCE", False))
+
+
+def wormhole_root_witness_finality_enforce(settings: Settings | None = None) -> bool:
+    snapshot = settings or get_settings()
+    return bool(getattr(snapshot, "WORMHOLE_ROOT_WITNESS_FINALITY_ENFORCE", False))

backend/services/constants.py

@@ -11,8 +11,13 @@ DEFAULT_TRAIL_TTL_S = 300  # 5 min - trail TTL for non-tracked flights
 HOLD_PATTERN_DEGREES = 300  # Total heading change to flag holding pattern
 GPS_JAMMING_NACP_THRESHOLD = 8  # NACp below this = degraded GPS signal
 GPS_JAMMING_GRID_SIZE = 1.0  # 1 degree grid for aggregation
-GPS_JAMMING_MIN_RATIO = 0.30  # 30% degraded aircraft to flag zone
-GPS_JAMMING_MIN_AIRCRAFT = 5  # Min aircraft in grid cell for statistical significance
+# Tuned 2026-05: previously 0.30 / 5 aircraft which — combined with the
+# -1 noise cushion in the detector AND the pre-fix nac_p==0 filter that
+# discarded jamming victims — meant the layer almost never lit up.
+# Lowering the bar so genuine jamming zones with sparser ADS-B coverage
+# clear (eastern Med, Russia/Ukraine border, Iran/Iraq).
+GPS_JAMMING_MIN_RATIO = 0.20  # 20% degraded aircraft to flag zone
+GPS_JAMMING_MIN_AIRCRAFT = 3  # Min aircraft in grid cell for statistical significance
 
 # ─── Network & Circuit Breaker ──────────────────────────────────────────────
 CIRCUIT_BREAKER_TTL_S = 120  # Skip domain for 2 min after total failure

backend/services/correlation_engine.py

@@ -8,9 +8,13 @@ Correlation types:
   - RF Anomaly:          GPS jamming + internet outage (both required)
   - Military Buildup:    Military flights + naval vessels + GDELT conflict events
   - Infrastructure Cascade: Internet outage + KiwiSDR offline in same zone
+  - Possible Contradiction: Official denial/statement + infrastructure disruption
+                            in same region — hypothesis generator, NOT verdict
 """
 
 import logging
+import math
+import re
 from collections import defaultdict
 
 logger = logging.getLogger(__name__)
@@ -306,6 +310,427 @@ def _detect_infra_cascades(data: dict) -> list[dict]:
     return alerts
 
 
+# ---------------------------------------------------------------------------
+# Possible Contradiction: official denial/statement + infra disruption
+#
+# This is a HYPOTHESIS GENERATOR, not a verdict engine.  It says "LOOK HERE"
+# when an official statement (denial, clarification, refusal) co-locates with
+# infrastructure disruption (internet outage, sigint change).  The human or
+# higher-order reasoning decides what actually happened.
+#
+# Context ratings:
+#   STRONG   — denial + outage + prediction market movement in same region
+#   MODERATE — denial + outage (no market signal)
+#   WEAK     — denial + minor outage or distant co-location
+#   DETECTION_GAP — denial found but NO telemetry to verify (equally valuable)
+# ---------------------------------------------------------------------------
+
+# Denial / official-statement patterns in headlines and URL slugs
+_DENIAL_PATTERNS = [
+    re.compile(p, re.IGNORECASE) for p in [
+        r"\bden(?:y|ies|ied|ial)\b",
+        r"\brefut(?:e[ds]?|ing)\b",
+        r"\breject(?:s|ed|ing)?\b",
+        r"\bclarif(?:y|ies|ied|ication)\b",
+        r"\bdismiss(?:es|ed|ing)?\b",
+        r"\bno\s+attack\b",
+        r"\bdid\s+not\s+(?:attack|strike|bomb|target|order|invade|kill)\b",
+        r"\bnever\s+(?:attack|strike|bomb|target|order|invade|happen)\b",
+        r"\bfalse\s+(?:report|claim|allegation|rumor|narrative)\b",
+        r"\bmisinformation\b",
+        r"\bdisinformation\b",
+        r"\bpropaganda\b",
+        r"\b(?:army|military|government|ministry|official)\s+(?:says|clarifies|denies|refutes)\b",
+        r"\brumor[s]?\b.*\buntrue\b",
+        r"\bcategorically\b",
+        r"\bbaseless\b",
+    ]
+]
+
+# Broader cell radius for sparse telemetry regions (Africa, Central Asia, etc.)
+# These regions have fewer IODA/RIPE probes so outage data is sparser
+_SPARSE_REGIONS_LAT_RANGES = [
+    (-35, 37),   # Africa roughly
+    (25, 50),    # Central Asia band (when lng 40-90)
+]
+
+
+def _is_sparse_region(lat: float, lng: float) -> bool:
+    """Check if coordinates fall in a region with sparse telemetry coverage."""
+    # Africa
+    if -35 <= lat <= 37 and -20 <= lng <= 55:
+        return True
+    # Central Asia
+    if 25 <= lat <= 50 and 40 <= lng <= 90:
+        return True
+    # South America interior
+    if -55 <= lat <= 12 and -80 <= lng <= -35:
+        return True
+    return False
+
+
+def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
+    """Great-circle distance in km."""
+    R = 6371.0
+    dlat = math.radians(lat2 - lat1)
+    dlon = math.radians(lon2 - lon1)
+    a = (math.sin(dlat / 2) ** 2 +
+         math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) *
+         math.sin(dlon / 2) ** 2)
+    return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
+
+
+def _matches_denial(text: str) -> bool:
+    """Check if text matches any denial/official-statement pattern."""
+    return any(p.search(text) for p in _DENIAL_PATTERNS)
+
+
+def _detect_contradictions(data: dict) -> list[dict]:
+    """Detect possible contradictions between official statements and telemetry.
+
+    Scans GDELT headlines for denial language, then checks whether internet
+    outages or other infrastructure disruptions exist in the same geographic
+    region.  Scores confidence and lists alternative explanations.
+    """
+    gdelt = data.get("gdelt") or []
+    internet_outages = data.get("internet_outages") or []
+    news = data.get("news") or []
+    prediction_markets = data.get("prediction_markets") or []
+
+    # ── Step 1: Find GDELT events with denial/official-statement language ──
+    denial_events: list[dict] = []
+
+    # GDELT comes as GeoJSON features
+    gdelt_features = gdelt
+    if isinstance(gdelt, dict):
+        gdelt_features = gdelt.get("features", [])
+
+    for feature in gdelt_features:
+        # Handle both GeoJSON features and flat dicts
+        if "properties" in feature and "geometry" in feature:
+            props = feature.get("properties", {})
+            geom = feature.get("geometry", {})
+            coords = geom.get("coordinates", [])
+            if len(coords) >= 2:
+                lng, lat = float(coords[0]), float(coords[1])
+            else:
+                continue
+            headlines = props.get("_headlines_list", [])
+            urls = props.get("_urls_list", [])
+            name = props.get("name", "")
+            count = props.get("count", 1)
+        else:
+            lat = feature.get("lat") or feature.get("actionGeo_Lat")
+            lng = feature.get("lng") or feature.get("lon") or feature.get("actionGeo_Long")
+            if lat is None or lng is None:
+                continue
+            lat, lng = float(lat), float(lng)
+            headlines = [feature.get("title", "")]
+            urls = [feature.get("sourceurl", "")]
+            name = feature.get("name", "")
+            count = 1
+
+        # Check all headlines + URL slugs for denial patterns
+        all_text = " ".join(str(h) for h in headlines if h)
+        all_text += " " + " ".join(str(u) for u in urls if u)
+
+        if _matches_denial(all_text):
+            denial_events.append({
+                "lat": lat,
+                "lng": lng,
+                "headlines": [h for h in headlines if h][:5],
+                "urls": [u for u in urls if u][:3],
+                "location_name": name,
+                "event_count": count,
+            })
+
+    # Also scan news articles for denial language
+    for article in news:
+        title = str(article.get("title", "") or "")
+        desc = str(article.get("description", "") or article.get("summary", "") or "")
+        if not _matches_denial(title + " " + desc):
+            continue
+        # News articles often lack coordinates — try to match to GDELT locations
+        # For now, only include if we have coordinates
+        lat = article.get("lat") or article.get("latitude")
+        lng = article.get("lng") or article.get("lon") or article.get("longitude")
+        if lat is not None and lng is not None:
+            denial_events.append({
+                "lat": float(lat),
+                "lng": float(lng),
+                "headlines": [title],
+                "urls": [article.get("url") or article.get("link") or ""],
+                "location_name": "",
+                "event_count": 1,
+            })
+
+    if not denial_events:
+        return []
+
+    # ── Step 2: Cross-reference with internet outages ──
+    alerts: list[dict] = []
+
+    for denial in denial_events:
+        d_lat, d_lng = denial["lat"], denial["lng"]
+        sparse = _is_sparse_region(d_lat, d_lng)
+        search_radius_km = 1500.0 if sparse else 500.0
+
+        # Find nearby outages
+        nearby_outages: list[dict] = []
+        for outage in internet_outages:
+            o_lat = outage.get("lat") or outage.get("latitude")
+            o_lng = outage.get("lng") or outage.get("lon") or outage.get("longitude")
+            if o_lat is None or o_lng is None:
+                continue
+            try:
+                dist = _haversine_km(d_lat, d_lng, float(o_lat), float(o_lng))
+            except (ValueError, TypeError):
+                continue
+            if dist <= search_radius_km:
+                nearby_outages.append({
+                    "region": outage.get("region_name") or outage.get("country_name", ""),
+                    "severity": _outage_pct(outage),
+                    "distance_km": round(dist, 0),
+                    "level": outage.get("level", ""),
+                })
+
+        # ── Step 3: Check prediction markets for related movements ──
+        denial_text = " ".join(denial["headlines"]).lower()
+        related_markets: list[dict] = []
+        for market in prediction_markets:
+            m_title = str(market.get("title", "") or market.get("question", "") or "").lower()
+            # Look for keyword overlap between denial and market
+            denial_words = set(re.findall(r"[a-z]{4,}", denial_text))
+            market_words = set(re.findall(r"[a-z]{4,}", m_title))
+            overlap = denial_words & market_words - {"that", "this", "with", "from", "have", "been", "were", "will", "says", "said"}
+            if len(overlap) >= 2:
+                prob = market.get("probability") or market.get("lastTradePrice") or market.get("yes_price")
+                if prob is not None:
+                    related_markets.append({
+                        "title": market.get("title") or market.get("question"),
+                        "probability": float(prob),
+                    })
+
+        # ── Step 4: Score confidence and assign context rating ──
+        indicators = 1  # denial itself
+        drivers: list[str] = []
+
+        # Primary driver: the denial headline
+        headline_display = denial["headlines"][0] if denial["headlines"] else "Official statement"
+        if len(headline_display) > 80:
+            headline_display = headline_display[:77] + "..."
+        drivers.append(f'"{headline_display}"')
+
+        # Outage co-location
+        has_outage = False
+        if nearby_outages:
+            best_outage = max(nearby_outages, key=lambda o: o["severity"])
+            if best_outage["severity"] >= 10:
+                indicators += 1
+                has_outage = True
+                drivers.append(
+                    f"Internet outage {best_outage['severity']:.0f}% "
+                    f"({best_outage['region']}, {best_outage['distance_km']:.0f}km away)"
+                )
+            elif best_outage["severity"] > 0:
+                indicators += 0.5  # minor outage, partial indicator
+                has_outage = True
+                drivers.append(
+                    f"Minor outage ({best_outage['region']}, "
+                    f"{best_outage['distance_km']:.0f}km away)"
+                )
+
+        # Prediction market signal
+        has_market = False
+        if related_markets:
+            indicators += 1
+            has_market = True
+            top_market = related_markets[0]
+            drivers.append(
+                f"Market: \"{top_market['title'][:50]}\" "
+                f"at {top_market['probability']:.0%}"
+            )
+
+        # Multiple denial sources strengthen the signal
+        if denial["event_count"] > 1:
+            indicators += 0.5
+            drivers.append(f"{denial['event_count']} sources reporting")
+
+        # Context rating
+        if has_outage and has_market:
+            context = "STRONG"
+        elif has_outage:
+            context = "MODERATE"
+        elif has_market:
+            context = "WEAK"  # market signal without infra disruption
+        else:
+            context = "DETECTION_GAP"
+
+        # Severity mapping
+        if context == "STRONG":
+            sev = "high"
+        elif context == "MODERATE":
+            sev = "medium"
+        else:
+            sev = "low"
+
+        # Alternative explanations (always present — this is a hypothesis generator)
+        alternatives: list[str] = []
+        if has_outage:
+            alternatives.append("Routine infrastructure maintenance or cable damage")
+            alternatives.append("Weather-related outage coinciding with news cycle")
+        if not has_outage and context == "DETECTION_GAP":
+            alternatives.append("Statement may be truthful — no contradicting telemetry found")
+            alternatives.append("Telemetry coverage gap in this region")
+        alternatives.append("Denial may be responding to social media rumors, not real events")
+
+        lat_c, lng_c = _cell_center(_cell_key(d_lat, d_lng))
+        alerts.append({
+            "lat": lat_c,
+            "lng": lng_c,
+            "type": "contradiction",
+            "severity": sev,
+            "score": _severity_score(sev),
+            "drivers": drivers[:4],
+            "cell_size": _CELL_SIZE,
+            "context": context,
+            "alternatives": alternatives[:3],
+            "location_name": denial.get("location_name", ""),
+            "headlines": denial["headlines"][:3],
+            "related_markets": related_markets[:3],
+            "nearby_outages": nearby_outages[:5],
+        })
+
+    # Deduplicate: keep highest-scored alert per cell
+    seen_cells: dict[str, dict] = {}
+    for alert in alerts:
+        key = _cell_key(alert["lat"], alert["lng"])
+        if key not in seen_cells or alert["score"] > seen_cells[key]["score"]:
+            seen_cells[key] = alert
+
+    result = list(seen_cells.values())
+    if result:
+        by_context = defaultdict(int)
+        for a in result:
+            by_context[a["context"]] += 1
+        logger.info(
+            "Contradictions: %d possible (%s)",
+            len(result),
+            ", ".join(f"{v} {k}" for k, v in sorted(by_context.items())),
+        )
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+# ---------------------------------------------------------------------------
+# Correlation → Pin bridge
+# ---------------------------------------------------------------------------
+
+# Types and their pin categories
+_CORR_PIN_CATEGORIES = {
+    "rf_anomaly": "anomaly",
+    "military_buildup": "military",
+    "infra_cascade": "infrastructure",
+    "contradiction": "research",
+}
+
+# Deduplicate: don't re-pin the same cell within this window (seconds).
+_CORR_PIN_DEDUP_WINDOW = 600  # 10 minutes
+_recent_corr_pins: dict[str, float] = {}
+
+
+def _auto_pin_correlations(alerts: list[dict]) -> int:
+    """Create AI Intel pins for high-severity correlation alerts.
+
+    Only pins alerts with severity >= medium.  Uses cell-key dedup so the
+    same grid cell doesn't get re-pinned every fetch cycle.
+
+    Returns the number of pins created this cycle.
+    """
+    import time as _time
+
+    now = _time.time()
+
+    # Evict stale dedup entries
+    expired = [k for k, ts in _recent_corr_pins.items() if now - ts > _CORR_PIN_DEDUP_WINDOW]
+    for k in expired:
+        _recent_corr_pins.pop(k, None)
+
+    created = 0
+    for alert in alerts:
+        sev = alert.get("severity", "low")
+        if sev == "low":
+            continue  # Don't pin low-severity noise
+
+        lat = alert.get("lat")
+        lng = alert.get("lng")
+        if lat is None or lng is None:
+            continue
+
+        # Dedup key: type + cell
+        dedup_key = f"{alert['type']}:{_cell_key(lat, lng)}"
+        if dedup_key in _recent_corr_pins:
+            continue
+
+        category = _CORR_PIN_CATEGORIES.get(alert["type"], "anomaly")
+        drivers = alert.get("drivers", [])
+        atype = alert["type"]
+
+        if atype == "contradiction":
+            ctx = alert.get("context", "")
+            label = f"[{ctx}] Possible Contradiction"
+            parts = list(drivers)
+            if alert.get("alternatives"):
+                parts.append("Alternatives: " + "; ".join(alert["alternatives"][:2]))
+            description = " | ".join(parts) if parts else "Narrative contradiction detected"
+        else:
+            label = f"[{sev.upper()}] {atype.replace('_', ' ').title()}"
+            description = "; ".join(drivers) if drivers else "Multi-layer correlation alert"
+
+        try:
+            from services.ai_pin_store import create_pin
+
+            meta = {
+                "correlation_type": atype,
+                "severity": sev,
+                "drivers": drivers,
+                "cell_size": alert.get("cell_size", _CELL_SIZE),
+            }
+            # Add contradiction-specific metadata
+            if atype == "contradiction":
+                meta["context_rating"] = alert.get("context", "")
+                meta["alternatives"] = alert.get("alternatives", [])
+                meta["headlines"] = alert.get("headlines", [])
+                meta["location_name"] = alert.get("location_name", "")
+                if alert.get("related_markets"):
+                    meta["related_markets"] = alert["related_markets"]
+
+            create_pin(
+                lat=lat,
+                lng=lng,
+                label=label,
+                category=category,
+                description=description,
+                source="correlation_engine",
+                confidence=alert.get("score", 60) / 100.0,
+                ttl_hours=2.0,  # Auto-expire correlation pins after 2 hours
+                metadata=meta,
+            )
+            _recent_corr_pins[dedup_key] = now
+            created += 1
+        except Exception as exc:
+            logger.warning("Failed to auto-pin correlation: %s", exc)
+
+    if created:
+        logger.info("Correlation engine auto-pinned %d alerts", created)
+    return created
+
+
 # ---------------------------------------------------------------------------
 # Public API
 # ---------------------------------------------------------------------------
@@ -330,13 +755,29 @@ def compute_correlations(data: dict) -> list[dict]:
     except Exception as e:
         logger.error("Correlation engine infra cascade error: %s", e)
 
+    # Contradiction detection removed from automated engine — too many false
+    # positives from regex headline matching.  Contradiction/analysis alerts are
+    # now placed by OpenClaw agents via place_analysis_zone, which lets an LLM
+    # reason about the evidence rather than pattern-matching keywords.
+    try:
+        from services.analysis_zone_store import get_live_zones
+        alerts.extend(get_live_zones())
+    except Exception as e:
+        logger.error("Analysis zone merge error: %s", e)
+
     rf = sum(1 for a in alerts if a["type"] == "rf_anomaly")
     mil = sum(1 for a in alerts if a["type"] == "military_buildup")
     infra = sum(1 for a in alerts if a["type"] == "infra_cascade")
+    contra = sum(1 for a in alerts if a["type"] == "contradiction")
     if alerts:
         logger.info(
-            "Correlations: %d alerts (%d rf, %d mil, %d infra)",
-            len(alerts), rf, mil, infra,
+            "Correlations: %d alerts (%d rf, %d mil, %d infra, %d contra)",
+            len(alerts), rf, mil, infra, contra,
         )
 
+    # Correlation alerts are returned in the correlations data feed only.
+    # They are NOT auto-pinned to AI Intel — that layer is reserved for
+    # user / OpenClaw pins.  Correlations are visualised via the dedicated
+    # correlations overlay on the map.
+
     return alerts

backend/services/data_fetcher.py

@@ -16,9 +16,13 @@ Heavy logic has been extracted into services/fetchers/:
 
 import logging
 import concurrent.futures
+import json
+import math
 import os
+import threading
 import time
 from datetime import datetime, timedelta
+from pathlib import Path
 from dotenv import load_dotenv
 
 load_dotenv()
@@ -56,6 +60,7 @@ from services.fetchers.earth_observation import (  # noqa: F401
     fetch_air_quality,
     fetch_volcanoes,
     fetch_viirs_change_nodes,
+    fetch_uap_sightings,
 )
 from services.fetchers.infrastructure import (  # noqa: F401
     fetch_internet_outages,
@@ -90,29 +95,257 @@ from services.fetchers.meshtastic_map import (
     load_meshtastic_cache_if_available,
 )  # noqa: F401
 from services.fetchers.fimi import fetch_fimi  # noqa: F401
+from services.fetchers.crowdthreat import fetch_crowdthreat  # noqa: F401
+from services.fetchers.wastewater import fetch_wastewater  # noqa: F401
+from services.fetchers.sar_catalog import fetch_sar_catalog  # noqa: F401
+from services.fetchers.sar_products import fetch_sar_products  # noqa: F401
 from services.ais_stream import prune_stale_vessels  # noqa: F401
 
 logger = logging.getLogger(__name__)
 _SLOW_FETCH_S = float(os.environ.get("FETCH_SLOW_THRESHOLD_S", "5"))
+# Hard wall-clock limit per individual fetch task.  A task that exceeds this
+# is treated as a failure so it cannot block an entire fetch tier indefinitely.
+_TASK_HARD_TIMEOUT_S = float(os.environ.get("FETCH_TASK_TIMEOUT_S", "120"))
+_FAST_STARTUP_CACHE_MAX_AGE_S = float(os.environ.get("FAST_STARTUP_CACHE_MAX_AGE_S", "21600"))
+_FAST_STARTUP_CACHE_PATH = Path(__file__).resolve().parents[1] / "data" / "fast_startup_cache.json"
+_FAST_STARTUP_CACHE_KEYS = (
+    "commercial_flights",
+    "military_flights",
+    "private_flights",
+    "private_jets",
+    "tracked_flights",
+    "ships",
+    "uavs",
+    "gps_jamming",
+    "satellites",
+    "satellite_source",
+    "satellite_analysis",
+    "sigint",
+    "sigint_totals",
+    "trains",
+)
+_INTEL_STARTUP_CACHE_MAX_AGE_S = float(os.environ.get("INTEL_STARTUP_CACHE_MAX_AGE_S", "21600"))
+_INTEL_STARTUP_CACHE_PATH = Path(__file__).resolve().parents[1] / "data" / "intel_startup_cache.json"
+_INTEL_STARTUP_CACHE_KEYS = (
+    "news",
+    "gdelt",
+    "liveuamap",
+    "threat_level",
+    "trending_markets",
+    "correlations",
+    "fimi",
+    "crowdthreat",
+    "uap_sightings",
+    "military_bases",
+    "wastewater",
+)
+_STARTUP_PRIORITY_TIMEOUT_S = float(os.environ.get("SHADOWBROKER_STARTUP_PRIORITY_TIMEOUT_S", "18"))
+_STARTUP_HEAVY_REFRESH_DELAY_S = float(os.environ.get("SHADOWBROKER_STARTUP_HEAVY_REFRESH_DELAY_S", "90"))
+_STARTUP_HEAVY_REFRESH_STARTED = False
+_STARTUP_HEAVY_REFRESH_LOCK = threading.Lock()
+_FETCH_WORKERS = int(os.environ.get("SHADOWBROKER_FETCH_WORKERS", "8"))
+_SLOW_FETCH_CONCURRENCY = int(os.environ.get("SHADOWBROKER_SLOW_FETCH_CONCURRENCY", "4"))
+_STARTUP_HEAVY_CONCURRENCY = int(os.environ.get("SHADOWBROKER_STARTUP_HEAVY_CONCURRENCY", "2"))
 
 # Shared thread pool — reused across all fetch cycles instead of creating/destroying per tick
 _SHARED_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
-    max_workers=20, thread_name_prefix="fetch"
+    max_workers=max(2, _FETCH_WORKERS), thread_name_prefix="fetch"
 )
 
 
+def _cache_json_safe(value):
+    if isinstance(value, float):
+        return value if math.isfinite(value) else None
+    if isinstance(value, dict):
+        return {str(k): _cache_json_safe(v) for k, v in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [_cache_json_safe(v) for v in value]
+    return value
+
+
+def _has_cache_value(value) -> bool:
+    if value is None:
+        return False
+    if isinstance(value, (list, tuple, dict, set)):
+        return bool(value)
+    return True
+
+
+def _load_fast_startup_cache_if_available() -> bool:
+    """Seed moving layers from a recent disk cache while live fetches warm up."""
+    if _FAST_STARTUP_CACHE_MAX_AGE_S <= 0 or not _FAST_STARTUP_CACHE_PATH.exists():
+        return False
+    try:
+        with _FAST_STARTUP_CACHE_PATH.open("r", encoding="utf-8") as fh:
+            payload = json.load(fh)
+        cached_at = float(payload.get("cached_at") or 0)
+        age_s = time.time() - cached_at
+        if cached_at <= 0 or age_s > _FAST_STARTUP_CACHE_MAX_AGE_S:
+            logger.info("Skipping stale fast startup cache (age %.1fs)", age_s)
+            return False
+        layers = payload.get("layers") or {}
+        freshness = payload.get("freshness") or {}
+        loaded: list[str] = []
+        with _data_lock:
+            for key in _FAST_STARTUP_CACHE_KEYS:
+                if key in layers:
+                    latest_data[key] = layers[key]
+                    loaded.append(key)
+            for key, ts in freshness.items():
+                source_timestamps[str(key)] = ts
+            if payload.get("last_updated"):
+                latest_data["last_updated"] = payload.get("last_updated")
+        if not loaded:
+            return False
+        from services.fetchers._store import bump_data_version
+
+        bump_data_version()
+        logger.info(
+            "Loaded fast startup cache for %d layers (age %.1fs) so the map can paint before remote feeds finish",
+            len(loaded),
+            age_s,
+        )
+        return True
+    except Exception as e:
+        logger.warning("Fast startup cache load failed (non-fatal): %s", e)
+        return False
+
+
+def _save_fast_startup_cache() -> None:
+    """Persist recent moving layers for the next cold start."""
+    try:
+        with _data_lock:
+            layers = {
+                key: latest_data.get(key)
+                for key in _FAST_STARTUP_CACHE_KEYS
+                if _has_cache_value(latest_data.get(key))
+            }
+            payload = {
+                "cached_at": time.time(),
+                "last_updated": latest_data.get("last_updated"),
+                "layers": layers,
+                "freshness": {
+                    key: source_timestamps.get(key)
+                    for key in _FAST_STARTUP_CACHE_KEYS
+                    if source_timestamps.get(key)
+                },
+            }
+        safe_payload = _cache_json_safe(payload)
+        _FAST_STARTUP_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
+        tmp_path = _FAST_STARTUP_CACHE_PATH.with_suffix(".tmp")
+        with tmp_path.open("w", encoding="utf-8") as fh:
+            json.dump(safe_payload, fh, separators=(",", ":"))
+        tmp_path.replace(_FAST_STARTUP_CACHE_PATH)
+    except Exception as e:
+        logger.debug("Fast startup cache save skipped: %s", e)
+
+
+def _load_intel_startup_cache_if_available() -> bool:
+    """Seed the right-side intelligence panel from disk while live feeds warm up."""
+    if _INTEL_STARTUP_CACHE_MAX_AGE_S <= 0 or not _INTEL_STARTUP_CACHE_PATH.exists():
+        return False
+    try:
+        with _INTEL_STARTUP_CACHE_PATH.open("r", encoding="utf-8") as fh:
+            payload = json.load(fh)
+        cached_at = float(payload.get("cached_at") or 0)
+        age_s = time.time() - cached_at
+        if cached_at <= 0 or age_s > _INTEL_STARTUP_CACHE_MAX_AGE_S:
+            logger.info("Skipping stale intel startup cache (age %.1fs)", age_s)
+            return False
+        layers = payload.get("layers") or {}
+        freshness = payload.get("freshness") or {}
+        loaded: list[str] = []
+        with _data_lock:
+            for key in _INTEL_STARTUP_CACHE_KEYS:
+                if key in layers:
+                    latest_data[key] = layers[key]
+                    loaded.append(key)
+            for key, ts in freshness.items():
+                source_timestamps[str(key)] = ts
+            if payload.get("last_updated"):
+                latest_data["last_updated"] = payload.get("last_updated")
+        if not loaded:
+            return False
+        from services.fetchers._store import bump_data_version
+
+        bump_data_version()
+        logger.info(
+            "Loaded intel startup cache for %d layers (age %.1fs) so Global Threat Intercept can paint early",
+            len(loaded),
+            age_s,
+        )
+        return True
+    except Exception as e:
+        logger.warning("Intel startup cache load failed (non-fatal): %s", e)
+        return False
+
+
+def _save_intel_startup_cache() -> None:
+    """Persist compact right-side intelligence data for the next cold start."""
+    try:
+        with _data_lock:
+            layers = {
+                key: latest_data.get(key)
+                for key in _INTEL_STARTUP_CACHE_KEYS
+                if _has_cache_value(latest_data.get(key))
+            }
+            payload = {
+                "cached_at": time.time(),
+                "last_updated": latest_data.get("last_updated"),
+                "layers": layers,
+                "freshness": {
+                    key: source_timestamps.get(key)
+                    for key in _INTEL_STARTUP_CACHE_KEYS
+                    if source_timestamps.get(key)
+                },
+            }
+        safe_payload = _cache_json_safe(payload)
+        _INTEL_STARTUP_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
+        tmp_path = _INTEL_STARTUP_CACHE_PATH.with_suffix(".tmp")
+        with tmp_path.open("w", encoding="utf-8") as fh:
+            json.dump(safe_payload, fh, separators=(",", ":"))
+        tmp_path.replace(_INTEL_STARTUP_CACHE_PATH)
+    except Exception as e:
+        logger.debug("Intel startup cache save skipped: %s", e)
+
+
+def seed_startup_caches() -> None:
+    """Load disk-backed first-paint caches without touching remote services."""
+    load_meshtastic_cache_if_available()
+    _load_fast_startup_cache_if_available()
+    _load_intel_startup_cache_if_available()
+
+
 # ---------------------------------------------------------------------------
 # Scheduler & Orchestration
 # ---------------------------------------------------------------------------
-def _run_tasks(label: str, funcs: list):
+def _run_tasks(label: str, funcs: list, *, max_concurrency: int | None = None):
     """Run tasks concurrently and log any exceptions (do not fail silently)."""
     if not funcs:
         return
-    futures = {_SHARED_EXECUTOR.submit(func): (func.__name__, time.perf_counter()) for func in funcs}
-    for future in concurrent.futures.as_completed(futures):
-        name, start = futures[future]
+    if max_concurrency is None:
+        if label.startswith("slow-tier"):
+            max_concurrency = _SLOW_FETCH_CONCURRENCY
+        elif label.startswith("startup-heavy"):
+            max_concurrency = _STARTUP_HEAVY_CONCURRENCY
+        else:
+            max_concurrency = len(funcs)
+    max_concurrency = max(1, min(max_concurrency, len(funcs)))
+
+    remaining_funcs = list(funcs)
+    while remaining_funcs:
+        batch, remaining_funcs = remaining_funcs[:max_concurrency], remaining_funcs[max_concurrency:]
+        futures = {_SHARED_EXECUTOR.submit(func): (func.__name__, time.perf_counter()) for func in batch}
+        _drain_task_futures(label, futures)
+
+
+def _drain_task_futures(label: str, futures: dict):
+    # Iterate directly so future.result(timeout=...) is the blocking call.
+    # as_completed() blocks inside __next__() waiting for completion — the timeout
+    # on result() would never be reached for a hanging task under that pattern.
+    for future, (name, start) in futures.items():
         try:
-            future.result()
+            future.result(timeout=_TASK_HARD_TIMEOUT_S)
             duration = time.perf_counter() - start
             from services.fetch_health import record_success
 
@@ -157,13 +390,13 @@ def update_fast_data():
         fetch_satellites,
         fetch_sigint,
         fetch_trains,
-        fetch_tinygs,
     ]
     _run_tasks("fast-tier", fast_funcs)
     with _data_lock:
         latest_data["last_updated"] = datetime.utcnow().isoformat()
     from services.fetchers._store import bump_data_version
     bump_data_version()
+    _save_fast_startup_cache()
     logger.info("Fast-tier update complete.")
 
 
@@ -183,6 +416,7 @@ def update_slow_data():
         fetch_cctv,
         fetch_kiwisdr,
         fetch_satnogs,
+        fetch_tinygs,
         fetch_frontlines,
         fetch_datacenters,
         fetch_military_bases,
@@ -207,9 +441,76 @@ def update_slow_data():
         logger.error("Correlation engine failed: %s", e)
     from services.fetchers._store import bump_data_version
     bump_data_version()
+    _save_intel_startup_cache()
     logger.info("Slow-tier update complete.")
 
 
+def _record_fetch_success(label: str, name: str, start: float) -> None:
+    duration = time.perf_counter() - start
+    from services.fetch_health import record_success
+
+    record_success(name, duration_s=duration)
+    if duration > _SLOW_FETCH_S:
+        logger.warning(f"{label} task slow: {name} took {duration:.2f}s")
+
+
+def _record_fetch_failure(label: str, name: str, start: float, error: Exception) -> None:
+    duration = time.perf_counter() - start
+    from services.fetch_health import record_failure
+
+    record_failure(name, error=error, duration_s=duration)
+    logger.exception(f"{label} task failed: {name}")
+
+
+def _load_cctv_cache_for_startup() -> None:
+    """Load cached CCTV rows without running remote ingestors during first paint."""
+    try:
+        fetch_cctv()
+    except Exception as e:
+        logger.warning("Startup CCTV cache load failed (non-fatal): %s", e)
+
+
+def _run_delayed_startup_heavy_refresh() -> None:
+    if _STARTUP_HEAVY_REFRESH_DELAY_S > 0:
+        logger.info(
+            "Startup heavy synthesis delayed %.0fs so the dashboard can finish first paint",
+            _STARTUP_HEAVY_REFRESH_DELAY_S,
+        )
+        time.sleep(_STARTUP_HEAVY_REFRESH_DELAY_S)
+    logger.info("Startup heavy synthesis beginning (slow feeds, enrichment, daily products)...")
+    _run_tasks(
+        "startup-heavy",
+        [
+            update_slow_data,
+            fetch_volcanoes,
+            fetch_viirs_change_nodes,
+            fetch_unusual_whales,
+            fetch_fimi,
+            fetch_uap_sightings,
+            fetch_wastewater,
+            fetch_sar_catalog,
+            fetch_sar_products,
+        ],
+    )
+    logger.info("Startup heavy synthesis complete.")
+
+
+def _schedule_delayed_startup_heavy_refresh() -> None:
+    global _STARTUP_HEAVY_REFRESH_STARTED
+    if _STARTUP_HEAVY_REFRESH_DELAY_S < 0:
+        logger.info("Startup heavy synthesis disabled by SHADOWBROKER_STARTUP_HEAVY_REFRESH_DELAY_S")
+        return
+    with _STARTUP_HEAVY_REFRESH_LOCK:
+        if _STARTUP_HEAVY_REFRESH_STARTED:
+            return
+        _STARTUP_HEAVY_REFRESH_STARTED = True
+    threading.Thread(
+        target=_run_delayed_startup_heavy_refresh,
+        name="startup-heavy-refresh",
+        daemon=True,
+    ).start()
+
+
 def update_all_data(*, startup_mode: bool = False):
     """Full refresh.
 
@@ -218,50 +519,120 @@ def update_all_data(*, startup_mode: bool = False):
     """
     logger.info("Full data update starting (parallel)...")
     # Preload Meshtastic map cache immediately (instant, from disk)
-    load_meshtastic_cache_if_available()
+    seed_startup_caches()
     with _data_lock:
         meshtastic_seeded = bool(latest_data.get("meshtastic_map_nodes"))
-    futures = {
-        _SHARED_EXECUTOR.submit(fetch_airports): ("fetch_airports", time.perf_counter()),
-        _SHARED_EXECUTOR.submit(update_fast_data): ("update_fast_data", time.perf_counter()),
-        _SHARED_EXECUTOR.submit(update_slow_data): ("update_slow_data", time.perf_counter()),
-        _SHARED_EXECUTOR.submit(fetch_volcanoes): ("fetch_volcanoes", time.perf_counter()),
-        _SHARED_EXECUTOR.submit(fetch_viirs_change_nodes): ("fetch_viirs_change_nodes", time.perf_counter()),
-        _SHARED_EXECUTOR.submit(fetch_unusual_whales): ("fetch_unusual_whales", time.perf_counter()),
-        _SHARED_EXECUTOR.submit(fetch_fimi): ("fetch_fimi", time.perf_counter()),
-        _SHARED_EXECUTOR.submit(fetch_gdelt): ("fetch_gdelt", time.perf_counter()),
-        _SHARED_EXECUTOR.submit(update_liveuamap): ("update_liveuamap", time.perf_counter()),
-    }
+    if startup_mode:
+        _load_cctv_cache_for_startup()
+        priority_funcs = [
+            fetch_airports,
+            update_fast_data,
+            fetch_news,
+            fetch_gdelt,
+            fetch_crowdthreat,
+            fetch_firms_fires,
+            fetch_weather_alerts,
+        ]
+        if not meshtastic_seeded:
+            priority_funcs.append(fetch_meshtastic_nodes)
+        else:
+            logger.info(
+                "Startup preload: Meshtastic cache already loaded, deferring remote map refresh to scheduled cadence"
+            )
+        logger.info("Startup priority preload starting (%d tasks)...", len(priority_funcs))
+        cycle_start = time.perf_counter()
+        futures = {
+            _SHARED_EXECUTOR.submit(func): (func.__name__, time.perf_counter())
+            for func in priority_funcs
+        }
+        for future, (name, start) in futures.items():
+            remaining = _STARTUP_PRIORITY_TIMEOUT_S - (time.perf_counter() - cycle_start)
+            if remaining <= 0:
+                logger.info("Startup priority budget reached; %s will continue in background", name)
+                continue
+            try:
+                future.result(timeout=remaining)
+                _record_fetch_success("startup-priority", name, start)
+            except concurrent.futures.TimeoutError:
+                logger.info(
+                    "Startup priority task still warming after %.1fs: %s",
+                    time.perf_counter() - start,
+                    name,
+                )
+            except Exception as e:
+                _record_fetch_failure("startup-priority", name, start, e)
+        logger.info("Startup preload: deferring Playwright Liveuamap scraper to scheduled cadence")
+        _save_intel_startup_cache()
+        _schedule_delayed_startup_heavy_refresh()
+        logger.info("Startup priority preload complete; slow synthesis is warming in background.")
+        return
+    refresh_funcs = [
+        fetch_airports,
+        update_fast_data,
+        update_slow_data,
+        fetch_volcanoes,
+        fetch_viirs_change_nodes,
+        fetch_unusual_whales,
+        fetch_fimi,
+        fetch_gdelt,
+        fetch_uap_sightings,
+        fetch_wastewater,
+        fetch_crowdthreat,
+        fetch_sar_catalog,
+        fetch_sar_products,
+    ]
     if not startup_mode or not meshtastic_seeded:
-        futures[_SHARED_EXECUTOR.submit(fetch_meshtastic_nodes)] = (
-            "fetch_meshtastic_nodes",
-            time.perf_counter(),
-        )
+        refresh_funcs.append(fetch_meshtastic_nodes)
     else:
         logger.info(
             "Startup preload: Meshtastic cache already loaded, deferring remote map refresh to scheduled cadence"
         )
-    for future in concurrent.futures.as_completed(futures):
-        name, start = futures[future]
+    if not startup_mode:
+        refresh_funcs.append(update_liveuamap)
+    else:
+        logger.info("Startup preload: deferring Playwright Liveuamap scraper to scheduled cadence")
+    _run_tasks("full-refresh", refresh_funcs, max_concurrency=_STARTUP_HEAVY_CONCURRENCY)
+    # Run CCTV ingest immediately so cameras are available on first request
+    # (the scheduled job also runs every 10 min for ongoing refresh).
+    if startup_mode:
         try:
-            future.result()
-            duration = time.perf_counter() - start
-            from services.fetch_health import record_success
-
-            record_success(name, duration_s=duration)
-            if duration > _SLOW_FETCH_S:
-                logger.warning(f"full-refresh task slow: {name} took {duration:.2f}s")
+            from services.cctv_pipeline import (
+                TFLJamCamIngestor, LTASingaporeIngestor, AustinTXIngestor,
+                NYCDOTIngestor, CaltransIngestor, ColoradoDOTIngestor,
+                WSDOTIngestor, GeorgiaDOTIngestor, IllinoisDOTIngestor,
+                MichiganDOTIngestor, WindyWebcamsIngestor, DGTNationalIngestor,
+                MadridCityIngestor, OSMTrafficCameraIngestor, get_all_cameras,
+            )
+            from services.cctv_pipeline import OSMALPRCameraIngestor
+            _startup_ingestors = [
+                TFLJamCamIngestor(), LTASingaporeIngestor(), AustinTXIngestor(),
+                NYCDOTIngestor(), CaltransIngestor(), ColoradoDOTIngestor(),
+                WSDOTIngestor(), GeorgiaDOTIngestor(), IllinoisDOTIngestor(),
+                MichiganDOTIngestor(), WindyWebcamsIngestor(), DGTNationalIngestor(),
+                MadridCityIngestor(), OSMTrafficCameraIngestor(),
+                OSMALPRCameraIngestor(),
+            ]
+            logger.info("Running CCTV ingest at startup (%d ingestors)...", len(_startup_ingestors))
+            ingest_futures = {
+                _SHARED_EXECUTOR.submit(ing.ingest): ing.__class__.__name__
+                for ing in _startup_ingestors
+            }
+            for fut in concurrent.futures.as_completed(ingest_futures, timeout=90):
+                name = ingest_futures[fut]
+                try:
+                    fut.result()
+                except Exception as e:
+                    logger.warning("CCTV startup ingest %s failed: %s", name, e)
+            fetch_cctv()
+            logger.info("CCTV startup ingest complete — %d cameras in DB", len(get_all_cameras()))
         except Exception as e:
-            duration = time.perf_counter() - start
-            from services.fetch_health import record_failure
+            logger.warning("CCTV startup ingest failed (non-fatal): %s", e)
 
-            record_failure(name, error=e, duration_s=duration)
-            logger.exception(f"full-refresh task failed: {name}")
     logger.info("Full data update complete.")
 
 
 _scheduler = None
-_STARTUP_CCTV_INGEST_DELAY_S = 30
+_STARTUP_CCTV_INGEST_DELAY_S = int(os.environ.get("SHADOWBROKER_STARTUP_CCTV_INGEST_DELAY_S", "180"))
 _FINANCIAL_REFRESH_MINUTES = 30
 
 
@@ -406,6 +777,71 @@ def start_scheduler():
         misfire_grace_time=60,
     )
 
+    # Flight observation pruning — drops icao24 → first_seen_at entries we
+    # haven't seen in an hour. Same cadence as AIS prune for symmetry; the
+    # per-tick scan is O(in-flight aircraft) so it's cheap.
+    from services.fetchers.flight_observations import prune as _prune_flight_observations
+    _scheduler.add_job(
+        lambda: _run_task_with_health(_prune_flight_observations, "prune_flight_observations"),
+        "interval",
+        minutes=5,
+        id="flight_observation_prune",
+        max_instances=1,
+        misfire_grace_time=60,
+    )
+
+    # AISHub REST fallback — slow polling when the AISStream WebSocket
+    # primary is offline. Configurable interval via
+    # AISHUB_POLL_INTERVAL_MINUTES env (default 20 min). Operator must
+    # set AISHUB_USERNAME to opt in. The fetcher is gated internally on
+    # the primary being disconnected, so this job is cheap when the
+    # WebSocket is healthy (early-returns after a status check).
+    from services.fetchers.aishub_fallback import (
+        aishub_poll_interval_minutes,
+        fetch_aishub_vessels,
+    )
+    _aishub_interval = aishub_poll_interval_minutes()
+    _scheduler.add_job(
+        lambda: _run_task_with_health(fetch_aishub_vessels, "fetch_aishub_vessels"),
+        "interval",
+        minutes=_aishub_interval,
+        id="aishub_fallback",
+        max_instances=1,
+        misfire_grace_time=120,
+    )
+
+    # Route database — bulk refresh from vrs-standing-data.adsb.lol every 5
+    # days. Replaces the legacy /api/0/routeset POST (blocked under our UA,
+    # and broken upstream). Airline schedules change on a quarterly cycle,
+    # so 5 days is well within the staleness budget; new flight numbers
+    # added within the window simply fall back to UNKNOWN until refresh.
+    from services.fetchers.route_database import refresh_route_database
+
+    _scheduler.add_job(
+        lambda: _run_task_with_health(refresh_route_database, "refresh_route_database"),
+        "interval",
+        days=5,
+        id="route_database",
+        max_instances=1,
+        misfire_grace_time=3600,
+    )
+
+    # Aircraft metadata database — bulk refresh from OpenSky's public S3
+    # bucket every 5 days. Provides hex24 -> ICAO type so OpenSky-sourced
+    # flights (which lack 't' in /states/all) get aircraft category and
+    # fuel/CO2 emissions populated. Snapshots are monthly; 5 days catches
+    # newer drops without hammering the bucket.
+    from services.fetchers.aircraft_database import refresh_aircraft_database
+
+    _scheduler.add_job(
+        lambda: _run_task_with_health(refresh_aircraft_database, "refresh_aircraft_database"),
+        "interval",
+        days=5,
+        id="aircraft_database",
+        max_instances=1,
+        misfire_grace_time=3600,
+    )
+
     # GDELT — every 30 minutes (downloads 32 ZIP files per call, avoid rate limits)
     _scheduler.add_job(
         lambda: _run_task_with_health(fetch_gdelt, "fetch_gdelt"),
@@ -510,14 +946,21 @@ def start_scheduler():
         misfire_grace_time=120,
     )
 
-    # Meshtastic map API — every 4 hours, fetch global node positions
+    # Meshtastic map API — once per day with a per-install random offset to
+    # avoid thundering the one-person hobby service at the top of the hour.
+    # The fetcher also short-circuits on a fresh on-disk cache, so the
+    # practical network cadence is closer to "once per day per install".
+    import random as _random_jitter
+
+    _meshtastic_jitter_minutes = _random_jitter.randint(0, 180)
     _scheduler.add_job(
         lambda: _run_task_with_health(fetch_meshtastic_nodes, "fetch_meshtastic_nodes"),
         "interval",
-        hours=4,
+        hours=24,
+        minutes=_meshtastic_jitter_minutes,
         id="meshtastic_map",
         max_instances=1,
-        misfire_grace_time=600,
+        misfire_grace_time=3600,
     )
 
     # Oracle resolution sweep — every hour, check if any markets with predictions have concluded
@@ -550,9 +993,139 @@ def start_scheduler():
         misfire_grace_time=600,
     )
 
+    # UAP sightings (NUFORC) — weekly on Mondays at 12:00 UTC. The layer is a
+    # rolling last-60-days digest; refreshing once a week is enough cadence
+    # for human-readable map exploration and keeps load on nuforc.org light.
+    _scheduler.add_job(
+        lambda: _run_task_with_health(
+            lambda: fetch_uap_sightings(force_refresh=True),
+            "fetch_uap_sightings",
+        ),
+        "cron",
+        day_of_week="mon",
+        hour=12,
+        minute=0,
+        id="uap_sightings_weekly",
+        max_instances=1,
+        misfire_grace_time=3600,
+    )
+
+    # WastewaterSCAN pathogen surveillance — daily at 12:00 UTC (samples update ~daily)
+    _scheduler.add_job(
+        lambda: _run_task_with_health(fetch_wastewater, "fetch_wastewater"),
+        "cron",
+        hour=12,
+        minute=0,
+        id="wastewater_daily",
+        max_instances=1,
+        misfire_grace_time=3600,
+    )
+
+    # CrowdThreat verified threat intelligence — daily at 12:00 UTC
+    _scheduler.add_job(
+        lambda: _run_task_with_health(fetch_crowdthreat, "fetch_crowdthreat"),
+        "cron",
+        hour=12,
+        minute=0,
+        id="crowdthreat_daily",
+        max_instances=1,
+        misfire_grace_time=3600,
+    )
+
+    # SAR catalog (Mode A) — every hour, free metadata from ASF Search.
+    # No account, no downloads, no DSP.  Pure scene catalog + coverage hints.
+    _scheduler.add_job(
+        lambda: _run_task_with_health(fetch_sar_catalog, "fetch_sar_catalog"),
+        "interval",
+        hours=1,
+        id="sar_catalog",
+        max_instances=1,
+        misfire_grace_time=600,
+        next_run_time=datetime.utcnow() + timedelta(minutes=3),
+    )
+
+    # SAR products (Mode B) — every 30 minutes, opt-in only.
+    # Pre-processed deformation/flood/damage anomalies from OPERA, EGMS, GFM,
+    # EMS, UNOSAT.  Disabled until both MESH_SAR_PRODUCTS_FETCH=allow and
+    # MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE=true are set.
+    _scheduler.add_job(
+        lambda: _run_task_with_health(fetch_sar_products, "fetch_sar_products"),
+        "interval",
+        minutes=30,
+        id="sar_products",
+        max_instances=1,
+        misfire_grace_time=600,
+        next_run_time=datetime.utcnow() + timedelta(minutes=5),
+    )
+
+    # ── Time Machine auto-snapshots ─────────────────────────────────────
+    # Compressed snapshots taken on two profiles (high_freq + standard).
+    # Intervals are read from _timemachine_config at each invocation so
+    # config changes via the API take effect without restarting.
+
+    def _auto_snapshot_high_freq():
+        """Auto-snapshot fast-moving layers (flights, ships, satellites)."""
+        try:
+            from services.node_settings import read_node_settings
+            if not read_node_settings().get("timemachine_enabled", False):
+                return  # Time Machine is off — skip
+            from routers.ai_intel import _timemachine_config, _take_snapshot_internal
+            cfg = _timemachine_config["profiles"]["high_freq"]
+            if cfg["interval_minutes"] <= 0:
+                return  # disabled
+            layers = cfg["layers"]
+            result = _take_snapshot_internal(layers=layers, profile="auto_high_freq", compress=True)
+            logger.info("Time Machine auto-snapshot (high_freq): %s — %s layers",
+                        result.get("snapshot_id"), len(result.get("layers", [])))
+        except Exception as e:
+            logger.warning("Time Machine auto-snapshot (high_freq) failed: %s", e)
+
+    def _auto_snapshot_standard():
+        """Auto-snapshot contextual layers (news, earthquakes, weather, etc.)."""
+        try:
+            from services.node_settings import read_node_settings
+            if not read_node_settings().get("timemachine_enabled", False):
+                return  # Time Machine is off — skip
+            from routers.ai_intel import _timemachine_config, _take_snapshot_internal
+            cfg = _timemachine_config["profiles"]["standard"]
+            if cfg["interval_minutes"] <= 0:
+                return  # disabled
+            layers = cfg["layers"]
+            result = _take_snapshot_internal(layers=layers, profile="auto_standard", compress=True)
+            logger.info("Time Machine auto-snapshot (standard): %s — %s layers",
+                        result.get("snapshot_id"), len(result.get("layers", [])))
+        except Exception as e:
+            logger.warning("Time Machine auto-snapshot (standard) failed: %s", e)
+
+    _scheduler.add_job(
+        _auto_snapshot_high_freq,
+        "interval",
+        minutes=15,
+        id="timemachine_high_freq",
+        max_instances=1,
+        misfire_grace_time=60,
+        next_run_time=datetime.utcnow() + timedelta(minutes=2),  # first snapshot 2m after startup
+    )
+    _scheduler.add_job(
+        _auto_snapshot_standard,
+        "interval",
+        minutes=120,
+        id="timemachine_standard",
+        max_instances=1,
+        misfire_grace_time=300,
+        next_run_time=datetime.utcnow() + timedelta(minutes=5),  # first snapshot 5m after startup
+    )
+
     _scheduler.start()
     logger.info("Scheduler started.")
 
+    # Start the feed ingester daemon (refreshes feed-backed pin layers)
+    try:
+        from services.feed_ingester import start_feed_ingester
+        start_feed_ingester()
+    except Exception as e:
+        logger.warning("Failed to start feed ingester: %s", e)
+
 
 def stop_scheduler():
     if _scheduler:

backend/services/feed_ingester.py

@@ -0,0 +1,245 @@
+"""Feed Ingester — background daemon that refreshes feed-backed pin layers.
+
+Layers with a non-empty `feed_url` are polled at their `feed_interval`
+(seconds, minimum 60). The feed is expected to return either:
+
+  1. GeoJSON FeatureCollection — features are converted to pins
+  2. JSON array of pin objects — used directly
+
+Each refresh atomically replaces the layer's pins with the new data.
+"""
+
+import logging
+import threading
+import time
+from typing import Any
+
+import requests
+
+from services.network_utils import outbound_user_agent
+
+logger = logging.getLogger(__name__)
+
+
+def _feed_ingester_user_agent() -> str:
+    # Round 7a: per-install attribution for operator-curated feed URLs.
+    return outbound_user_agent("feed-ingester")
+
+# ---------------------------------------------------------------------------
+# State
+# ---------------------------------------------------------------------------
+
+_running = False
+_thread: threading.Thread | None = None
+_CHECK_INTERVAL = 30  # seconds between scanning for layers that need refresh
+_last_fetched: dict[str, float] = {}  # layer_id → last fetch timestamp
+_FETCH_TIMEOUT = 20  # seconds
+
+# ---------------------------------------------------------------------------
+# GeoJSON → pin conversion
+# ---------------------------------------------------------------------------
+
+
+def _geojson_features_to_pins(features: list[dict]) -> list[dict[str, Any]]:
+    """Convert GeoJSON Feature objects to pin dicts."""
+    pins: list[dict[str, Any]] = []
+    for feat in features:
+        if not isinstance(feat, dict):
+            continue
+        geom = feat.get("geometry") or {}
+        props = feat.get("properties") or {}
+
+        # Extract coordinates
+        coords = geom.get("coordinates")
+        if geom.get("type") != "Point" or not coords or len(coords) < 2:
+            continue
+
+        lng, lat = float(coords[0]), float(coords[1])
+        if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+            continue
+
+        pin: dict[str, Any] = {
+            "lat": lat,
+            "lng": lng,
+            "label": str(props.get("label", props.get("name", props.get("title", ""))))[:200],
+            "category": str(props.get("category", "custom"))[:50],
+            "color": str(props.get("color", ""))[:20],
+            "description": str(props.get("description", props.get("summary", "")))[:2000],
+            "source": "feed",
+            "source_url": str(props.get("source_url", props.get("url", props.get("link", ""))))[:500],
+            "confidence": float(props.get("confidence", 1.0)),
+        }
+
+        # Entity attachment if present
+        entity_type = props.get("entity_type", "")
+        entity_id = props.get("entity_id", "")
+        if entity_type and entity_id:
+            pin["entity_attachment"] = {
+                "entity_type": str(entity_type),
+                "entity_id": str(entity_id),
+                "entity_label": str(props.get("entity_label", "")),
+            }
+
+        pins.append(pin)
+    return pins
+
+
+def _parse_feed_response(data: Any) -> list[dict[str, Any]]:
+    """Parse a feed response into a list of pin dicts."""
+    if isinstance(data, dict):
+        # GeoJSON FeatureCollection
+        if data.get("type") == "FeatureCollection" and isinstance(data.get("features"), list):
+            return _geojson_features_to_pins(data["features"])
+        # Single Feature
+        if data.get("type") == "Feature":
+            return _geojson_features_to_pins([data])
+        # Wrapped response like {"ok": true, "data": [...]}
+        inner = data.get("data") or data.get("results") or data.get("pins") or data.get("items")
+        if isinstance(inner, list):
+            return _normalize_pin_list(inner)
+
+    if isinstance(data, list):
+        # Check if first item looks like a GeoJSON Feature
+        if data and isinstance(data[0], dict) and data[0].get("type") == "Feature":
+            return _geojson_features_to_pins(data)
+        return _normalize_pin_list(data)
+
+    return []
+
+
+def _normalize_pin_list(items: list) -> list[dict[str, Any]]:
+    """Normalize a list of raw pin objects, ensuring lat/lng are present."""
+    pins: list[dict[str, Any]] = []
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        lat = item.get("lat") or item.get("latitude")
+        lng = item.get("lng") or item.get("lon") or item.get("longitude")
+        if lat is None or lng is None:
+            continue
+        try:
+            lat, lng = float(lat), float(lng)
+        except (ValueError, TypeError):
+            continue
+        if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+            continue
+
+        pin: dict[str, Any] = {
+            "lat": lat,
+            "lng": lng,
+            "label": str(item.get("label", item.get("name", item.get("title", ""))))[:200],
+            "category": str(item.get("category", "custom"))[:50],
+            "color": str(item.get("color", ""))[:20],
+            "description": str(item.get("description", item.get("summary", "")))[:2000],
+            "source": "feed",
+            "source_url": str(item.get("source_url", item.get("url", item.get("link", ""))))[:500],
+            "confidence": float(item.get("confidence", 1.0)),
+        }
+
+        entity_type = item.get("entity_type", "")
+        entity_id = item.get("entity_id", "")
+        if entity_type and entity_id:
+            pin["entity_attachment"] = {
+                "entity_type": str(entity_type),
+                "entity_id": str(entity_id),
+                "entity_label": str(item.get("entity_label", "")),
+            }
+
+        pins.append(pin)
+    return pins
+
+
+# ---------------------------------------------------------------------------
+# Fetch a single layer
+# ---------------------------------------------------------------------------
+
+
+def _fetch_layer_feed(layer: dict[str, Any]) -> None:
+    """Fetch a feed URL and replace the layer's pins."""
+    layer_id = layer["id"]
+    feed_url = layer["feed_url"]
+    layer_name = layer.get("name", layer_id)
+
+    try:
+        resp = requests.get(
+            feed_url,
+            timeout=_FETCH_TIMEOUT,
+            headers={"User-Agent": _feed_ingester_user_agent()},
+        )
+        resp.raise_for_status()
+        data = resp.json()
+    except requests.RequestException as e:
+        logger.warning("Feed fetch failed for layer '%s' (%s): %s", layer_name, feed_url, e)
+        return
+    except (ValueError, TypeError) as e:
+        logger.warning("Feed parse failed for layer '%s' (%s): %s", layer_name, feed_url, e)
+        return
+
+    pins = _parse_feed_response(data)
+
+    from services.ai_pin_store import replace_layer_pins, update_layer
+    count = replace_layer_pins(layer_id, pins)
+
+    # Update layer metadata with last_fetched timestamp
+    update_layer(layer_id, feed_last_fetched=time.time())
+
+    _last_fetched[layer_id] = time.time()
+    logger.info("Feed refresh for layer '%s': %d pins from %s", layer_name, count, feed_url)
+
+
+# ---------------------------------------------------------------------------
+# Main loop
+# ---------------------------------------------------------------------------
+
+
+def _ingest_loop() -> None:
+    """Daemon loop: scan for feed layers and refresh those that are due."""
+    while _running:
+        try:
+            from services.ai_pin_store import get_feed_layers
+
+            layers = get_feed_layers()
+            now = time.time()
+
+            for layer in layers:
+                layer_id = layer["id"]
+                interval = max(60, layer.get("feed_interval", 300))
+                last = _last_fetched.get(layer_id, 0)
+
+                if now - last >= interval:
+                    try:
+                        _fetch_layer_feed(layer)
+                    except Exception as e:
+                        logger.warning("Feed ingestion error for layer %s: %s",
+                                       layer.get("name", layer_id), e)
+
+        except Exception as e:
+            logger.error("Feed ingester loop error: %s", e)
+
+        # Sleep in short increments so we can stop cleanly
+        for _ in range(int(_CHECK_INTERVAL)):
+            if not _running:
+                break
+            time.sleep(1)
+
+
+# ---------------------------------------------------------------------------
+# Start / stop
+# ---------------------------------------------------------------------------
+
+
+def start_feed_ingester() -> None:
+    """Start the feed ingester daemon thread."""
+    global _running, _thread
+    if _thread and _thread.is_alive():
+        return
+    _running = True
+    _thread = threading.Thread(target=_ingest_loop, daemon=True, name="feed-ingester")
+    _thread.start()
+    logger.info("Feed ingester daemon started (check interval=%ds)", _CHECK_INTERVAL)
+
+
+def stop_feed_ingester() -> None:
+    """Stop the feed ingester daemon."""
+    global _running
+    _running = False

backend/services/fetchers/_store.py

@@ -4,6 +4,7 @@ Central location for latest_data, source_timestamps, and the data lock.
 Every fetcher imports from here instead of maintaining its own copy.
 """
 
+import copy
 import threading
 import logging
 from datetime import datetime
@@ -42,6 +43,7 @@ class DashboardData(TypedDict, total=False):
     gps_jamming: List[Dict[str, Any]]
     satellites: List[Dict[str, Any]]
     satellite_source: str
+    satellite_analysis: Dict[str, Any]
     prediction_markets: List[Dict[str, Any]]
     sigint: List[Dict[str, Any]]
     sigint_totals: Dict[str, Any]
@@ -61,6 +63,12 @@ class DashboardData(TypedDict, total=False):
     fimi: Dict[str, Any]
     psk_reporter: List[Dict[str, Any]]
     correlations: List[Dict[str, Any]]
+    uap_sightings: List[Dict[str, Any]]
+    wastewater: List[Dict[str, Any]]
+    crowdthreat: List[Dict[str, Any]]
+    sar_scenes: List[Dict[str, Any]]
+    sar_anomalies: List[Dict[str, Any]]
+    sar_aoi_coverage: List[Dict[str, Any]]
 
 
 # In-memory store
@@ -105,6 +113,12 @@ latest_data: DashboardData = {
     "fimi": {},
     "psk_reporter": [],
     "correlations": [],
+    "uap_sightings": [],
+    "wastewater": [],
+    "crowdthreat": [],
+    "sar_scenes": [],
+    "sar_anomalies": [],
+    "sar_aoi_coverage": [],
 }
 
 # Per-source freshness timestamps
@@ -117,9 +131,21 @@ source_freshness: dict[str, dict] = {}
 def _mark_fresh(*keys):
     """Record the current UTC time for one or more data source keys."""
     now = datetime.utcnow().isoformat()
+    global _data_version
+    changed: list[tuple[str, int, int]] = []  # (layer, version, count)
     with _data_lock:
         for k in keys:
             source_timestamps[k] = now
+            _layer_versions[k] = _layer_versions.get(k, 0) + 1
+            # Grab entity count while we hold the lock (cheap len())
+            val = latest_data.get(k)
+            count = len(val) if isinstance(val, list) else (1 if val is not None else 0)
+            changed.append((k, _layer_versions[k], count))
+        # Publish partial fetch progress immediately so the frontend can
+        # observe newly available data without waiting for the entire tier.
+        _data_version += 1
+    # Notify SSE listeners outside the lock to avoid deadlocks
+    _notify_layer_change(changed)
 
 
 # Thread lock for safe reads/writes to latest_data
@@ -129,16 +155,73 @@ _data_lock = threading.Lock()
 # Used for cheap ETag generation instead of MD5-hashing the full response.
 _data_version: int = 0
 
+# Per-layer version counters — incremented only when that specific layer
+# refreshes.  Used by get_layer_slice for per-layer incremental updates
+# and by the SSE stream to push targeted layer_changed notifications.
+_layer_versions: dict[str, int] = {}
+
+
+# ---------------------------------------------------------------------------
+# Layer-change notification callbacks (thread → async SSE bridge)
+# ---------------------------------------------------------------------------
+_layer_change_callbacks: list = []
+_layer_change_callbacks_lock = threading.Lock()
+
+
+def register_layer_change_callback(callback) -> None:
+    """Register a callback invoked on every _mark_fresh().
+
+    Signature: callback(layer: str, version: int, count: int)
+    Called from fetcher threads — must be thread-safe.
+    """
+    with _layer_change_callbacks_lock:
+        _layer_change_callbacks.append(callback)
+
+
+def unregister_layer_change_callback(callback) -> None:
+    """Remove a previously registered callback."""
+    with _layer_change_callbacks_lock:
+        try:
+            _layer_change_callbacks.remove(callback)
+        except ValueError:
+            pass
+
+
+def _notify_layer_change(changed: list[tuple[str, int, int]]) -> None:
+    """Fire all registered callbacks for each changed layer."""
+    with _layer_change_callbacks_lock:
+        cbs = list(_layer_change_callbacks)
+    for cb in cbs:
+        for layer, version, count in changed:
+            try:
+                cb(layer, version, count)
+            except Exception:
+                pass
+
+
+def get_layer_versions() -> dict[str, int]:
+    """Return a snapshot of all per-layer version counters."""
+    with _data_lock:
+        return dict(_layer_versions)
+
+
+def get_layer_version(layer: str) -> int:
+    """Return the version counter for a single layer (0 if never refreshed)."""
+    with _data_lock:
+        return _layer_versions.get(layer, 0)
+
 
 def bump_data_version() -> None:
     """Increment the data version counter after a fetch cycle completes."""
     global _data_version
-    _data_version += 1
+    with _data_lock:
+        _data_version += 1
 
 
 def get_data_version() -> int:
     """Return the current data version (for ETag generation)."""
-    return _data_version
+    with _data_lock:
+        return _data_version
 
 
 _active_layers_version: int = 0
@@ -156,21 +239,17 @@ def get_active_layers_version() -> int:
 
 
 def get_latest_data_subset(*keys: str) -> DashboardData:
-    """Return a shallow snapshot of only the requested top-level keys.
+    """Return a deep snapshot of only the requested top-level keys.
 
     This avoids cloning the entire dashboard store for endpoints that only need
-    a small tier-specific subset.
+    a small tier-specific subset.  Deep copy ensures callers cannot mutate
+    nested structures (e.g. individual flight dicts) and affect the live store.
     """
     with _data_lock:
         snap: DashboardData = {}
         for key in keys:
             value = latest_data.get(key)
-            if isinstance(value, list):
-                snap[key] = list(value)
-            elif isinstance(value, dict):
-                snap[key] = dict(value)
-            else:
-                snap[key] = value
+            snap[key] = copy.deepcopy(value)
         return snap
 
 
@@ -231,10 +310,16 @@ active_layers: dict[str, bool] = {
     "satnogs": True,
     "tinygs": True,
     "ukraine_alerts": True,
-    "power_plants": False,
+    "power_plants": True,
     "viirs_nightlights": False,
     "psk_reporter": True,
     "correlations": True,
+    "contradictions": True,
+    "uap_sightings": True,
+    "wastewater": True,
+    "ai_intel": True,
+    "crowdthreat": False,
+    "sar": True,
 }
 
 

backend/services/fetchers/aircraft_database.py

@@ -0,0 +1,180 @@
+"""OpenSky aircraft metadata: ICAO24 hex -> ICAO type code + friendly model.
+
+OpenSky's /states/all does not include aircraft type, so OpenSky-sourced
+flights arrive with ``t`` field empty. This module bulk-loads the public
+OpenSky aircraft database (one snapshot CSV per month, ~108 MB uncompressed,
+~600k aircraft) once every 5 days and exposes a fast in-memory hex lookup.
+
+The data is also useful when adsb.lol's live API is degraded: even the
+adsb.lol /v2 feed sometimes returns aircraft with empty ``t`` for newly seen
+transponders, and the lookup gracefully fills those in too.
+"""
+
+from __future__ import annotations
+
+import csv
+import logging
+import threading
+import time
+from typing import Any
+
+import defusedxml.ElementTree as ET
+import requests
+
+
+
+def _aircraft_db_user_agent() -> str:
+    """Round 7a: lazy import so the per-install operator handle is included."""
+    from services.network_utils import outbound_user_agent
+    return outbound_user_agent("aircraft-database")
+
+logger = logging.getLogger(__name__)
+
+_BUCKET_LIST_URL = (
+    "https://s3.opensky-network.org/data-samples?prefix=metadata/&list-type=2"
+)
+_BUCKET_BASE = "https://s3.opensky-network.org/data-samples/"
+_S3_NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
+_REFRESH_INTERVAL_S = 5 * 24 * 3600
+_LIST_TIMEOUT_S = 30
+_DOWNLOAD_TIMEOUT_S = 600
+from services.network_utils import DEFAULT_USER_AGENT as _USER_AGENT
+
+_lock = threading.RLock()
+_aircraft_by_hex: dict[str, dict[str, str]] = {}
+_last_refresh = 0.0
+_in_progress = False
+
+
+def _latest_snapshot_key() -> str:
+    """Discover the most recent aircraft-database-complete snapshot key."""
+    response = requests.get(
+        _BUCKET_LIST_URL,
+        timeout=_LIST_TIMEOUT_S,
+        headers={"User-Agent": _aircraft_db_user_agent()},
+    )
+    response.raise_for_status()
+    root = ET.fromstring(response.text)
+    keys: list[str] = []
+    for content in root.iter(f"{_S3_NS}Contents"):
+        key_el = content.find(f"{_S3_NS}Key")
+        if key_el is None or not key_el.text:
+            continue
+        if "aircraft-database-complete-" in key_el.text and key_el.text.endswith(".csv"):
+            keys.append(key_el.text)
+    if not keys:
+        raise RuntimeError("no aircraft-database-complete snapshot found in bucket listing")
+    return sorted(keys)[-1]
+
+
+def _stream_csv_index(url: str) -> dict[str, dict[str, str]]:
+    """Stream-parse the OpenSky aircraft CSV into a hex-keyed index.
+
+    The CSV uses single-quote quoting, so csv.DictReader is configured with
+    ``quotechar="'"``. Rows are processed line-by-line via iter_lines() to
+    keep memory bounded even though the file is ~108 MB.
+    """
+    with requests.get(
+        url,
+        timeout=_DOWNLOAD_TIMEOUT_S,
+        stream=True,
+        headers={"User-Agent": _aircraft_db_user_agent()},
+    ) as response:
+        response.raise_for_status()
+        line_iter = (
+            line.decode("utf-8", errors="replace")
+            for line in response.iter_lines(decode_unicode=False)
+            if line
+        )
+        reader = csv.DictReader(line_iter, quotechar="'")
+        index: dict[str, dict[str, str]] = {}
+        for row in reader:
+            hex_code = (row.get("icao24") or "").strip().lower()
+            if not hex_code or hex_code == "000000":
+                continue
+            typecode = (row.get("typecode") or "").strip().upper()
+            model = (row.get("model") or "").strip()
+            mfr = (row.get("manufacturerName") or "").strip()
+            registration = (row.get("registration") or "").strip().upper()
+            operator = (row.get("operator") or "").strip()
+            if not (typecode or model):
+                continue
+            entry: dict[str, str] = {}
+            if typecode:
+                entry["typecode"] = typecode
+            if model:
+                entry["model"] = model
+            if mfr:
+                entry["manufacturer"] = mfr
+            if registration:
+                entry["registration"] = registration
+            if operator:
+                entry["operator"] = operator
+            index[hex_code] = entry
+    return index
+
+
+def refresh_aircraft_database(force: bool = False) -> bool:
+    """Download the latest OpenSky aircraft snapshot and rebuild the index.
+
+    Returns True if a refresh was performed (success or attempted), False if
+    skipped because the cache is still fresh or another refresh is in flight.
+    """
+    global _last_refresh, _in_progress
+
+    now = time.time()
+    with _lock:
+        if _in_progress:
+            return False
+        if not force and (now - _last_refresh) < _REFRESH_INTERVAL_S and _aircraft_by_hex:
+            return False
+        _in_progress = True
+
+    try:
+        started = time.time()
+        key = _latest_snapshot_key()
+        index = _stream_csv_index(_BUCKET_BASE + key)
+        with _lock:
+            _aircraft_by_hex.clear()
+            _aircraft_by_hex.update(index)
+            _last_refresh = time.time()
+        logger.info(
+            "aircraft database refreshed in %.1fs from %s: %d aircraft",
+            time.time() - started,
+            key,
+            len(index),
+        )
+        return True
+    except (requests.RequestException, OSError, ValueError, ET.ParseError) as exc:
+        logger.warning("aircraft database refresh failed: %s", exc)
+        return True
+    finally:
+        with _lock:
+            _in_progress = False
+
+
+def lookup_aircraft(icao24: str) -> dict[str, str] | None:
+    """Return the metadata record for an ICAO24 hex code, or None."""
+    key = (icao24 or "").strip().lower()
+    if not key:
+        return None
+    with _lock:
+        entry = _aircraft_by_hex.get(key)
+    return dict(entry) if entry else None
+
+
+def lookup_aircraft_type(icao24: str) -> str:
+    """Return the ICAO type code (e.g. 'B738', 'GLF4') or '' if unknown."""
+    entry = lookup_aircraft(icao24)
+    if not entry:
+        return ""
+    return entry.get("typecode", "")
+
+
+def aircraft_database_status() -> dict[str, Any]:
+    with _lock:
+        return {
+            "last_refresh": _last_refresh,
+            "aircraft": len(_aircraft_by_hex),
+            "in_progress": _in_progress,
+        }

backend/services/fetchers/aishub_fallback.py

@@ -0,0 +1,290 @@
+"""AISHub REST fallback for ship tracking when AISStream is unreachable.
+
+Background
+----------
+On 2026-05-23 ``stream.aisstream.io`` (the primary live AIS WebSocket feed)
+went fully offline. Backend's only ship signal vanished. This module polls
+``data.aishub.net``'s free REST API on a slow cadence (default 20 min) when
+the WebSocket primary is disconnected, so the ships layer doesn't go fully
+dark during upstream outages.
+
+Why 20 minutes
+--------------
+AISHub's free tier is rate-limited and explicitly asks consumers to be
+courteous. 20 minutes is well inside their limits, gives ships time to
+move enough to look "alive" on the map, and won't drain their service.
+Configurable via the ``AISHUB_POLL_INTERVAL_MINUTES`` env var (clamped to
+[1, 360]).
+
+Why slow vs primary
+-------------------
+This is degraded mode, not a replacement. A ship at 20 knots moves about
+6 nautical miles in 20 minutes — visible on the map but coarser than the
+real-time WebSocket signal. When AISStream comes back online, the
+WebSocket data will overwrite these records via the same ``_vessels``
+dict and ``source`` will flip from ``"aishub"`` back to upstream-live.
+
+Opt-in
+------
+Operator must set ``AISHUB_USERNAME`` (free registration at
+https://www.aishub.net/api). If unset, this fetcher is a no-op.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import time
+from typing import Any
+
+from services.network_utils import fetch_with_curl
+
+logger = logging.getLogger(__name__)
+
+
+AISHUB_URL = "https://data.aishub.net/ws.php"
+
+
+def aishub_username() -> str:
+    return str(os.environ.get("AISHUB_USERNAME", "")).strip()
+
+
+def aishub_fallback_enabled() -> bool:
+    """Returns True only when the operator has registered with AISHub and
+    set ``AISHUB_USERNAME``. The presence of the username is the opt-in."""
+    return bool(aishub_username())
+
+
+def aishub_poll_interval_minutes() -> int:
+    """Default 20 minutes. Clamped to [1, 360] so a hostile or
+    misconfigured env var can't either hammer the upstream or silence the
+    fallback for a day."""
+    raw = os.environ.get("AISHUB_POLL_INTERVAL_MINUTES", "20")
+    try:
+        value = int(str(raw).strip())
+    except (TypeError, ValueError):
+        value = 20
+    return max(1, min(360, value))
+
+
+def _should_run_fallback() -> bool:
+    """Only run when the primary WebSocket is disconnected. Avoids stomping
+    over fresher live data when AISStream is healthy.
+
+    Returns False if:
+      * AISHub isn't configured (no username)
+      * AISStream primary is currently connected (recent vessel messages)
+
+    Returns True only when AIS is configured-but-down. The
+    ``proxy_spawn_count > 0`` guard means "the primary has at least tried
+    to run" — if the user set AISHUB_USERNAME but not AIS_API_KEY at all,
+    AISHub will still serve as a primary on its own slow cadence.
+    """
+    if not aishub_fallback_enabled():
+        return False
+    try:
+        from services.ais_stream import ais_proxy_status
+        status = ais_proxy_status() or {}
+    except Exception:
+        return True  # ais_stream not importable? still try AISHub.
+    # If the WebSocket primary is connected, skip the fallback — fresher
+    # data is already flowing.
+    if status.get("connected") is True:
+        return False
+    return True
+
+
+def _parse_aishub_response(payload: str) -> list[dict]:
+    """Parse the AISHub JSON response into a list of vessel records.
+
+    Successful response shape::
+
+        [
+            {"ERROR": false, "USERNAME": "...", "FORMAT": "1", "RECORDS": N},
+            [{"MMSI": ..., "LATITUDE": ..., "LONGITUDE": ..., ...}, ...]
+        ]
+
+    Error response shape::
+
+        [{"ERROR": true, "ERROR_MESSAGE": "..."}]
+
+    Empty payload (e.g. silent rate-limit drop) returns ``[]``.
+    """
+    if not payload or not payload.strip():
+        return []
+    try:
+        data = json.loads(payload)
+    except json.JSONDecodeError as e:
+        logger.warning("AISHub: response is not JSON: %s", e)
+        return []
+    if not isinstance(data, list) or not data:
+        return []
+    header = data[0] if isinstance(data[0], dict) else {}
+    if header.get("ERROR") is True:
+        logger.warning(
+            "AISHub: upstream error: %s",
+            header.get("ERROR_MESSAGE", "<unspecified>"),
+        )
+        return []
+    if len(data) < 2 or not isinstance(data[1], list):
+        return []
+    return [row for row in data[1] if isinstance(row, dict)]
+
+
+def _normalize_record(row: dict) -> dict | None:
+    """Map an AISHub vessel record to our internal vessel schema.
+
+    Returns None when the record can't be used (no MMSI, bad position,
+    sentinel "not available" lat/lng).
+    """
+    try:
+        mmsi = int(row.get("MMSI") or 0)
+    except (TypeError, ValueError):
+        return None
+    if not mmsi:
+        return None
+    try:
+        lat = float(row.get("LATITUDE"))
+        lng = float(row.get("LONGITUDE"))
+    except (TypeError, ValueError):
+        return None
+    # AIS uses 91/181 as "no position available" sentinels.
+    if abs(lat) > 90 or abs(lng) > 180:
+        return None
+    if lat == 91.0 or lng == 181.0:
+        return None
+    # SOG raw 102.3 is "speed not available"; sanitize to 0.
+    try:
+        sog_raw = float(row.get("SOG") or 0)
+    except (TypeError, ValueError):
+        sog_raw = 0.0
+    sog = 0.0 if sog_raw >= 102.2 else sog_raw
+    try:
+        cog = float(row.get("COG") or 0)
+    except (TypeError, ValueError):
+        cog = 0.0
+    try:
+        heading_raw = int(row.get("HEADING") or 511)
+    except (TypeError, ValueError):
+        heading_raw = 511
+    # AIS heading sentinel 511 = "not available" — fall back to COG.
+    heading = heading_raw if heading_raw != 511 else cog
+    try:
+        ais_type = int(row.get("TYPE") or 0)
+    except (TypeError, ValueError):
+        ais_type = 0
+    return {
+        "mmsi": mmsi,
+        "lat": lat,
+        "lng": lng,
+        "sog": sog,
+        "cog": cog,
+        "heading": heading,
+        "name": str(row.get("NAME") or "").strip() or "UNKNOWN",
+        "callsign": str(row.get("CALLSIGN") or "").strip(),
+        "destination": str(row.get("DEST") or "").strip().replace("@", "") or "",
+        "imo": int(row.get("IMO") or 0),
+        "ais_type_code": ais_type,
+    }
+
+
+def fetch_aishub_vessels() -> int:
+    """Poll AISHub and merge vessels into the shared ``_vessels`` store.
+
+    Returns the number of vessels updated (0 on skip, error, or no data).
+    Designed to be called by the APScheduler tier — see
+    ``data_fetcher.py`` for the 20-minute interval job that wraps this.
+    """
+    if not _should_run_fallback():
+        logger.debug("AISHub fallback skipped: primary connected or not configured")
+        return 0
+
+    username = aishub_username()
+    url = (
+        f"{AISHUB_URL}?username={username}&format=1&output=json"
+        f"&compress=0"
+    )
+
+    try:
+        response = fetch_with_curl(url, timeout=30)
+    except Exception as e:
+        logger.warning("AISHub fetch failed: %s", e)
+        return 0
+
+    if not response or response.status_code != 200:
+        logger.warning(
+            "AISHub HTTP %s",
+            getattr(response, "status_code", "None"),
+        )
+        return 0
+
+    rows = _parse_aishub_response(getattr(response, "text", "") or "")
+    if not rows:
+        return 0
+
+    # Inline imports to avoid a circular dependency at module load time
+    # (ais_stream imports lots of things and is loaded by main.py).
+    from services.ais_stream import (
+        _vessels,
+        _vessels_lock,
+        _record_vessel_trail_locked,
+        classify_vessel,
+        get_country_from_mmsi,
+    )
+
+    now = time.time()
+    count = 0
+    with _vessels_lock:
+        for row in rows:
+            normalized = _normalize_record(row)
+            if normalized is None:
+                continue
+            mmsi = normalized["mmsi"]
+            vessel = _vessels.setdefault(mmsi, {"mmsi": mmsi})
+            # Don't overwrite fresher live data: if the WebSocket pushed an
+            # update for this MMSI more recently than now-1s (race during
+            # the brief reconnection window) keep the live one.
+            last = float(vessel.get("_updated") or 0)
+            if last > now - 1:
+                continue
+            vessel.update(
+                {
+                    "lat": normalized["lat"],
+                    "lng": normalized["lng"],
+                    "sog": normalized["sog"],
+                    "cog": normalized["cog"],
+                    "heading": normalized["heading"],
+                    "_updated": now,
+                    "source": "aishub",
+                }
+            )
+            if normalized["name"] and normalized["name"] != "UNKNOWN":
+                vessel["name"] = normalized["name"]
+            if normalized["callsign"]:
+                vessel["callsign"] = normalized["callsign"]
+            if normalized["destination"]:
+                vessel["destination"] = normalized["destination"]
+            if normalized["imo"]:
+                vessel["imo"] = normalized["imo"]
+            if normalized["ais_type_code"]:
+                vessel["ais_type_code"] = normalized["ais_type_code"]
+                vessel["type"] = classify_vessel(normalized["ais_type_code"], mmsi)
+            if not vessel.get("country"):
+                vessel["country"] = get_country_from_mmsi(mmsi)
+            _record_vessel_trail_locked(
+                mmsi,
+                normalized["lat"],
+                normalized["lng"],
+                normalized["sog"],
+                now,
+            )
+            count += 1
+
+    if count:
+        logger.info(
+            "AISHub fallback: merged %d vessels (poll interval %d min)",
+            count,
+            aishub_poll_interval_minutes(),
+        )
+    return count

backend/services/fetchers/crowdthreat.py

@@ -0,0 +1,146 @@
+"""CrowdThreat fetcher — crowdsourced global threat intelligence.
+
+Polls verified threat reports from CrowdThreat's public API and normalises
+them into map-ready records with category-based icon IDs.
+
+No API key required — the /threats endpoint is unauthenticated.
+"""
+
+import logging
+import os
+
+from services.network_utils import fetch_with_curl
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh, is_any_active
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger("services.data_fetcher")
+
+_CT_BASE = "https://backend.crowdthreat.world"
+
+
+def crowdthreat_fetch_enabled() -> bool:
+    """Return True only when the operator explicitly opts into CrowdThreat pulls."""
+    return str(os.environ.get("CROWDTHREAT_ENABLED", "")).strip().lower() in {
+        "1",
+        "true",
+        "yes",
+        "on",
+    }
+
+# CrowdThreat category_id → icon ID used on the MapLibre layer
+_CATEGORY_ICON = {
+    1: "ct-security",       # Security & Conflict  (red)
+    2: "ct-crime",          # Crime & Safety        (blue)
+    3: "ct-aviation",       # Aviation              (green)
+    4: "ct-maritime",       # Maritime              (teal)
+    5: "ct-infrastructure", # Industrial & Infra    (orange)
+    6: "ct-special",        # Special Threats        (purple)
+    7: "ct-social",         # Social & Political    (pink)
+    8: "ct-other",          # Other                 (gray)
+}
+
+_CATEGORY_COLOUR = {
+    1: "#ef4444",  # red
+    2: "#3b82f6",  # blue
+    3: "#22c55e",  # green
+    4: "#14b8a6",  # teal
+    5: "#f97316",  # orange
+    6: "#a855f7",  # purple
+    7: "#ec4899",  # pink
+    8: "#6b7280",  # gray
+}
+
+
+@with_retry(max_retries=2, base_delay=5)
+def fetch_crowdthreat():
+    """Fetch verified threat reports from CrowdThreat public API."""
+    if not crowdthreat_fetch_enabled():
+        logger.debug("CrowdThreat fetch skipped; set CROWDTHREAT_ENABLED=true to opt in")
+        with _data_lock:
+            latest_data["crowdthreat"] = []
+        _mark_fresh("crowdthreat")
+        return
+    if not is_any_active("crowdthreat"):
+        return
+
+    try:
+        resp = fetch_with_curl(f"{_CT_BASE}/threats", timeout=20)
+        if not resp or resp.status_code != 200:
+            logger.warning("CrowdThreat API returned %s", getattr(resp, "status_code", "None"))
+            return
+
+        payload = resp.json()
+        raw_threats = payload.get("data", {}).get("threats", [])
+        if not raw_threats:
+            logger.debug("CrowdThreat returned 0 threats")
+            return
+
+    except Exception as e:
+        logger.error("CrowdThreat fetch error: %s", e)
+        return
+
+    processed = []
+    for t in raw_threats:
+        loc = t.get("location") or {}
+        lng_lat = loc.get("lng_lat")
+        if not lng_lat or len(lng_lat) < 2:
+            continue
+        try:
+            lng = float(lng_lat[0])
+            lat = float(lng_lat[1])
+        except (TypeError, ValueError):
+            continue
+
+        cat = t.get("category") or {}
+        cat_id = cat.get("id", 8)
+        subcat = t.get("subcategory") or {}
+        threat_type = t.get("type") or {}
+        dates = t.get("dates") or {}
+        occurred = dates.get("occurred") or {}
+        reported = dates.get("reported") or {}
+
+        # Extract all available detail from the API response
+        summary = (t.get("summary") or t.get("description") or "").strip()
+        verification = (t.get("verification_status") or t.get("status") or "").strip()
+        country_obj = loc.get("country") or {}
+        country = country_obj.get("name", "") if isinstance(country_obj, dict) else str(country_obj or "")
+        media = t.get("media") or t.get("images") or t.get("attachments") or []
+        source_url = t.get("source_url") or t.get("url") or t.get("link") or ""
+        severity = t.get("severity") or t.get("severity_level") or t.get("risk_level") or ""
+        votes = t.get("votes") or t.get("upvotes") or 0
+        reporter = t.get("user") or t.get("reporter") or {}
+        reporter_name = reporter.get("name", "") if isinstance(reporter, dict) else ""
+
+        processed.append({
+            "id": t.get("id"),
+            "title": t.get("title", ""),
+            "summary": summary[:500] if summary else "",
+            "lat": lat,
+            "lng": lng,
+            "address": loc.get("name", ""),
+            "city": loc.get("city", ""),
+            "country": country,
+            "category": cat.get("name", "Other"),
+            "category_id": cat_id,
+            "category_colour": _CATEGORY_COLOUR.get(cat_id, "#6b7280"),
+            "subcategory": subcat.get("name", ""),
+            "threat_type": threat_type.get("name", ""),
+            "icon_id": _CATEGORY_ICON.get(cat_id, "ct-other"),
+            "occurred": occurred.get("raw", ""),
+            "occurred_iso": occurred.get("iso", ""),
+            "timeago": occurred.get("timeago", ""),
+            "reported": reported.get("raw", ""),
+            "verification": verification,
+            "severity": str(severity),
+            "source_url": source_url,
+            "media_urls": [m.get("url") or m for m in media[:3]] if isinstance(media, list) else [],
+            "votes": int(votes) if votes else 0,
+            "reporter": reporter_name,
+            "source": "CrowdThreat",
+        })
+
+    logger.info("CrowdThreat: fetched %d verified threats", len(processed))
+
+    with _data_lock:
+        latest_data["crowdthreat"] = processed
+    _mark_fresh("crowdthreat")

backend/services/fetchers/emissions.py

@@ -1,20 +1,24 @@
 """
-Fuel burn & CO2 emissions estimator for private jets.
+Fuel burn & CO2 emissions estimator.
 Based on manufacturer-published cruise fuel burn rates (GPH at long-range cruise).
 1 US gallon of Jet-A produces ~21.1 lbs (9.57 kg) of CO2.
+
+Piston entries use 100LL (avgas), which is close enough to Jet-A in CO2 yield
+(~8.4 kg/gal vs 9.57 kg/gal); we keep one constant to stay simple — the result
+is a slight over-estimate for piston aircraft, which is preferable to under.
 """
 
 JET_A_CO2_KG_PER_GALLON = 9.57
 
 # ICAO type code -> gallons per hour at long-range cruise
 FUEL_BURN_GPH: dict[str, int] = {
-    # Gulfstream
+    # ── Gulfstream ─────────────────────────────────────────────────────
     "GLF6": 430,   # G650/G650ER
     "G700": 480,   # G700
     "GLF5": 390,   # G550
     "GVSP": 400,   # GV-SP
     "GLF4": 330,   # G-IV
-    # Bombardier
+    # ── Bombardier business ────────────────────────────────────────────
     "GL7T": 490,   # Global 7500
     "GLEX": 430,   # Global Express/6000/6500
     "GL5T": 420,   # Global 5000/5500
@@ -22,51 +26,208 @@ FUEL_BURN_GPH: dict[str, int] = {
     "CL60": 310,   # Challenger 604/605
     "CL30": 200,   # Challenger 300
     "CL65": 320,   # Challenger 650
-    # Dassault
+    # ── Bombardier regional jets ──────────────────────────────────────
+    "CRJ2": 360,   # CRJ-100/200
+    "CRJ7": 380,   # CRJ-700
+    "CRJ9": 410,   # CRJ-900
+    "CRJX": 440,   # CRJ-1000
+    # ── Dassault ───────────────────────────────────────────────────────
     "F7X":  350,   # Falcon 7X
     "F8X":  370,   # Falcon 8X
     "F900": 285,   # Falcon 900/900EX/900LX
     "F2TH": 230,   # Falcon 2000
     "FA50": 240,   # Falcon 50
-    # Cessna
+    # ── Cessna Citation ────────────────────────────────────────────────
     "CITX": 280,   # Citation X
+    "C750": 280,   # Citation X (alt code)
     "C68A": 195,   # Citation Latitude
     "C700": 230,   # Citation Longitude
     "C680": 220,   # Citation Sovereign
-    "C560": 190,   # Citation Excel/XLS
+    "C56X": 195,   # Citation Excel/XLS/XLS+
+    "C560": 190,   # Citation Excel/XLS (legacy)
+    "C550": 165,   # Citation II/Bravo/V
+    "C525": 80,    # Citation CJ1
+    "C25A": 100,   # CJ1+ / 525A
+    "C25B": 110,   # CJ2+ / 525B
+    "C25C": 130,   # CJ4 (some operators)
     "C510": 75,    # Citation Mustang
+    "C650": 240,   # Citation III/VI/VII
     "CJ3":  120,   # CJ3
     "CJ4":  135,   # CJ4
-    # Boeing
-    "B737": 850,   # BBJ (737)
-    "B738": 920,   # BBJ2 (737-800)
+    # ── Cessna piston / turboprop singles & twins ─────────────────────
+    "C172": 9,     # Skyhawk
+    "C152": 6,
+    "C150": 6,
+    "C170": 8,
+    "C177": 11,
+    "C180": 12,
+    "C182": 13,    # Skylane
+    "C185": 14,
+    "C206": 15,
+    "C208": 50,    # Caravan (turboprop)
+    "C210": 18,
+    "C310": 32,
+    "C340": 38,
+    "C414": 36,
+    "C421": 40,
+    # ── Boeing mainline ────────────────────────────────────────────────
+    "B737": 850,   # 737-700 / BBJ
+    "B738": 920,   # 737-800
+    "B739": 880,   # 737-900/900ER
+    "B38M": 700,   # 737-8 MAX
+    "B39M": 740,   # 737-9 MAX
     "B752": 1100,  # 757-200
+    "B753": 1200,  # 757-300
     "B762": 1400,  # 767-200
+    "B763": 1450,  # 767-300/300ER
+    "B764": 1500,  # 767-400ER
+    "B772": 1850,  # 777-200
+    "B77L": 1900,  # 777-200LR / 777F
+    "B77W": 2050,  # 777-300ER
     "B788": 1200,  # 787-8
-    # Airbus
-    "A318": 780,   # ACJ318
-    "A319": 850,   # ACJ319
-    "A320": 900,   # ACJ320
+    "B789": 1300,  # 787-9
+    "B78X": 1350,  # 787-10
+    "B744": 3050,  # 747-400
+    "B748": 2900,  # 747-8
+    # ── Airbus mainline ────────────────────────────────────────────────
+    "A318": 780,   # A318
+    "A319": 850,   # A319
+    "A320": 900,   # A320
+    "A321": 990,   # A321
+    "A19N": 580,   # A319neo
+    "A20N": 580,   # A320neo
+    "A21N": 700,   # A321neo
+    "A332": 1500,  # A330-200
+    "A333": 1550,  # A330-300
+    "A338": 1300,  # A330-800neo
+    "A339": 1350,  # A330-900neo
     "A343": 1800,  # A340-300
     "A346": 2100,  # A340-600
-    # Pilatus
+    "A359": 1450,  # A350-900
+    "A35K": 1600,  # A350-1000
+    "A388": 3200,  # A380-800
+    # ── Embraer regional / business ───────────────────────────────────
+    "E135": 300,   # Legacy 600/650 (regional ERJ-135 base)
+    "E145": 320,   # ERJ-145
+    "E170": 460,   # E170
+    "E75L": 490,   # E175-LR
+    "E75S": 490,   # E175 standard
+    "E175": 490,   # E175 (some)
+    "E190": 580,   # E190
+    "E195": 600,   # E195
+    "E290": 510,   # E190-E2
+    "E295": 540,   # E195-E2
+    "E50P": 135,   # Phenom 300 (also Phenom 100 var)
+    "E55P": 185,   # Praetor 500 / Legacy 500
+    "E545": 170,   # Praetor 500 (alt)
+    "E500": 80,    # Phenom 100
+    # ── ATR / Bombardier / Saab turboprops ────────────────────────────
+    "AT43": 230,   # ATR 42-300/-320
+    "AT45": 230,   # ATR 42-500
+    "AT46": 250,   # ATR 42-600
+    "AT72": 300,   # ATR 72-200/-210
+    "AT75": 280,   # ATR 72-500
+    "AT76": 280,   # ATR 72-600
+    "DH8A": 220,   # Dash 8 -100
+    "DH8B": 240,   # Dash 8 -200
+    "DH8C": 280,   # Dash 8 -300
+    "DH8D": 300,   # Dash 8 Q400
+    "SF34": 200,   # Saab 340
+    "SB20": 220,   # Saab 2000
+    # ── Pilatus / Daher single-engine turboprops ──────────────────────
     "PC24": 115,   # PC-24
     "PC12": 60,    # PC-12
-    # Embraer
-    "E55P": 185,   # Legacy 500
-    "E135": 300,   # Legacy 600/650
-    "E50P": 135,   # Phenom 300
-    "E500": 80,    # Phenom 100
-    # Learjet
+    "TBM7": 60,    # TBM 700/850
+    "TBM8": 65,    # TBM 850 alt
+    "TBM9": 70,    # TBM 900/930/940/960
+    "M600": 60,    # Piper M600
+    "P46T": 22,    # PA-46 Meridian (turboprop variant)
+    # ── Learjet ────────────────────────────────────────────────────────
     "LJ60": 195,   # Learjet 60
     "LJ75": 185,   # Learjet 75
     "LJ45": 175,   # Learjet 45
-    # Hawker
+    "LJ31": 165,   # Learjet 31
+    "LJ40": 175,   # Learjet 40
+    "LJ55": 195,   # Learjet 55
+    # ── Hawker / Beechjet ─────────────────────────────────────────────
     "H25B": 210,   # Hawker 800/800XP
     "H25C": 215,   # Hawker 900XP
-    # Beechcraft
+    "BE40": 150,   # Beechjet 400 / Hawker 400XP
+    "PRM1": 130,   # Premier I
+    # ── Beechcraft King Air ───────────────────────────────────────────
     "B350": 100,   # King Air 350
     "B200": 80,    # King Air 200/250
+    "BE20": 80,    # K-Air 200 (alt)
+    "BE9L": 60,    # K-Air 90
+    "BE9T": 70,    # K-Air F90
+    "BE10": 100,   # K-Air 100
+    "BE30": 90,    # K-Air 300
+    # ── Beechcraft / Cirrus / Piper / Mooney pistons ──────────────────
+    "BE23": 9,     # Sundowner
+    "BE33": 13,    # Bonanza 33
+    "BE35": 14,    # Bonanza V-tail
+    "BE36": 16,    # A36 Bonanza
+    "BE55": 24,    # Baron 55
+    "BE58": 28,    # Baron 58
+    "BE76": 17,    # Duchess
+    "BE95": 20,    # Travel Air
+    "P28A": 10,    # PA-28 Warrior/Archer
+    "P28B": 11,    # PA-28 Cherokee
+    "P28R": 12,    # PA-28R Arrow
+    "P32R": 14,    # PA-32R Lance/Saratoga
+    "PA11": 5,     # Cub Special
+    "PA12": 6,     # Super Cruiser
+    "PA18": 6,     # Super Cub
+    "PA22": 8,     # Tri-Pacer
+    "PA23": 18,    # Apache / Aztec
+    "PA24": 12,    # Comanche
+    "PA25": 12,    # Pawnee
+    "PA28": 10,    # PA-28 generic
+    "PA30": 16,    # Twin Comanche
+    "PA31": 30,    # Navajo
+    "PA32": 14,    # Cherokee Six / Saratoga
+    "PA34": 18,    # Seneca
+    "PA38": 5,     # Tomahawk
+    "PA44": 17,    # Seminole
+    "PA46": 18,    # Malibu / Mirage / Matrix
+    "M20P": 12,    # Mooney M20 (generic)
+    "SR20": 11,    # Cirrus SR20
+    "SR22": 16,    # Cirrus SR22
+    "S22T": 19,    # SR22T (turbo)
+    "DA40": 9,     # Diamond DA40
+    "DA42": 14,    # Diamond DA42 TwinStar
+    "DA62": 17,    # Diamond DA62
+    "DV20": 6,     # Diamond Katana
+    # ── Helicopters (civilian) ────────────────────────────────────────
+    "A109": 60,    # AW109
+    "A119": 50,    # AW119
+    "A139": 130,   # AW139
+    "A169": 90,    # AW169
+    "A189": 145,   # AW189
+    "AS35": 55,    # AS350 AStar
+    "AS50": 55,    # AStar (alt)
+    "AS65": 110,   # Dauphin
+    "B06":  35,    # Bell 206 JetRanger
+    "B407": 50,    # Bell 407
+    "B412": 145,   # Bell 412
+    "B429": 80,    # Bell 429
+    "B505": 35,    # Bell 505
+    "EC30": 50,    # H125 / EC130
+    "EC35": 70,    # EC135
+    "EC45": 85,    # EC145
+    "EC75": 130,   # EC175
+    "H125": 55,
+    "H130": 50,
+    "H135": 70,
+    "H145": 85,
+    "H155": 110,
+    "H160": 95,
+    "H175": 130,
+    "R22":  9,     # Robinson R22 (piston)
+    "R44":  16,    # Robinson R44 (piston)
+    "R66":  30,    # Robinson R66 (turbine)
+    "S76":  140,   # Sikorsky S-76
+    "S92":  220,   # Sikorsky S-92
 }
 
 # Common string names -> ICAO type code
@@ -108,13 +269,23 @@ def get_emissions_info(model: str) -> dict | None:
     if not model:
         return None
     model_clean = model.strip()
+    model_upper = model_clean.upper()
     # Try direct ICAO code match first
-    gph = FUEL_BURN_GPH.get(model_clean.upper())
+    gph = FUEL_BURN_GPH.get(model_upper)
     if gph is None:
         # Try alias lookup
         code = _ALIASES.get(model_clean)
         if code:
             gph = FUEL_BURN_GPH.get(code)
+    if gph is None:
+        # Friendly names from the Plane-Alert DB often lead with the ICAO type
+        # code as the first token (e.g. "B200 Super King Air"). Probe each
+        # token against FUEL_BURN_GPH directly.
+        for token in model_upper.replace("-", " ").replace(",", " ").split():
+            candidate = FUEL_BURN_GPH.get(token)
+            if candidate is not None:
+                gph = candidate
+                break
     if gph is None:
         # Fuzzy: check if any alias is a substring
         model_lower = model_clean.lower()

backend/services/fetchers/fimi.py

@@ -5,6 +5,7 @@ debunked claims, threat actor mentions, and target country references.
 Refreshes every 12 hours (FIMI data updates weekly).
 """
 
+import os
 import re
 import logging
 from datetime import datetime, timezone
@@ -18,6 +19,16 @@ logger = logging.getLogger("services.data_fetcher")
 
 _FIMI_FEED_URL = "https://euvsdisinfo.eu/feed/"
 
+
+def fimi_fetch_enabled() -> bool:
+    """Return True only when the operator explicitly opts into FIMI pulls."""
+    return str(os.environ.get("FIMI_ENABLED", "")).strip().lower() in {
+        "1",
+        "true",
+        "yes",
+        "on",
+    }
+
 # ── Threat actor keywords ──────────────────────────────────────────────────
 # Map of keyword → canonical actor name.  Checked case-insensitively.
 _THREAT_ACTORS: dict[str, str] = {
@@ -173,6 +184,12 @@ def _is_major_wave(narratives: list[dict], targets: dict[str, int]) -> bool:
 @with_retry(max_retries=1, base_delay=5)
 def fetch_fimi():
     """Fetch and parse the EUvsDisinfo RSS feed."""
+    if not fimi_fetch_enabled():
+        logger.debug("FIMI fetch skipped; set FIMI_ENABLED=true to opt in")
+        with _data_lock:
+            latest_data["fimi"] = []
+        _mark_fresh("fimi")
+        return
     try:
         resp = fetch_with_curl(_FIMI_FEED_URL, timeout=15)
         feed = feedparser.parse(resp.text)

backend/services/fetchers/financial.py

@@ -82,10 +82,37 @@ def _fetch_yfinance_single(symbol: str, period: str = "2d"):
 
 
 @with_retry(max_retries=1, base_delay=1)
+def financial_fetch_enabled() -> bool:
+    """Return True only when the operator explicitly opts into financial pulls.
+
+    Either ``FINANCIAL_ENABLED=true`` or the presence of ``FINNHUB_API_KEY``
+    counts as an explicit opt-in. Without either, the default yfinance path
+    is disabled to avoid silent outbound calls to finance.yahoo.com.
+    """
+    if os.getenv("FINNHUB_API_KEY", "").strip():
+        return True
+    return str(os.environ.get("FINANCIAL_ENABLED", "")).strip().lower() in {
+        "1",
+        "true",
+        "yes",
+        "on",
+    }
+
+
 def fetch_financial_markets():
     """Fetches full market list with smart throttling (3s for Finnhub, 60s for yfinance)."""
     global _last_fetch_time, _last_fetch_results, _rotating_index
-    
+
+    if not financial_fetch_enabled():
+        logger.debug(
+            "Financial fetch skipped; set FINANCIAL_ENABLED=true or supply "
+            "FINNHUB_API_KEY to opt in"
+        )
+        with _data_lock:
+            latest_data["financial"] = {}
+        _mark_fresh("financial")
+        return
+
     finnhub_key = os.getenv("FINNHUB_API_KEY", "").strip()
     use_finnhub = bool(finnhub_key)
     

backend/services/fetchers/flight_observations.py

@@ -0,0 +1,148 @@
+"""Per-aircraft observation tracking for cumulative fuel/CO2 estimates.
+
+Background
+----------
+The pre-existing emissions enrichment attached a *rate* to each flight
+(GPH and kg/hr) based on aircraft model. Users — reasonably — wanted the
+running total: how much fuel HAS this plane burned since we started
+seeing it? Multiplying the rate by elapsed observation time gets us
+there, but it requires somewhere to remember "when did this icao24
+first appear on our radar?"
+
+Why this lives outside ``flight_trails``
+----------------------------------------
+``flight_trails`` is sized and pruned aggressively for map rendering
+(5-minute TTL for untracked aircraft, 200 trail points max). That's
+wrong for cumulative burn: if a plane has been airborne 2 hours but
+its trail was pruned 30 min in, the "first trail point" timestamp is
+30 min ago, not 2h ago. Worse, when the trail expires and re-creates,
+the cumulative counter would reset mid-flight.
+
+This module tracks observation lifecycle separately:
+
+* When a hex is first observed: start a new flight session.
+* While observed regularly (gap < ``REOPEN_GAP_S``): keep accumulating.
+* When unseen for longer than ``REOPEN_GAP_S``: treat next sighting as
+  a new session (the plane landed and took off again, or it's a
+  different leg). Reset ``first_seen_at``.
+* Stale sessions are pruned every ``PRUNE_INTERVAL_S`` so memory stays
+  bounded.
+
+The user explicitly asked for this counting semantic: "as soon as a
+plane appears there should be a counter that keeps a running count of
+the fuel being burned... If there is no estimate take off time then it
+can just be from the time the server starts to keep a log of whats in
+the air."
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+
+
+# Gap between sightings that resets the session. ADS-B refreshes the
+# whole aircraft list every minute or two, so anything over a few
+# minutes means the plane left our coverage window (landed, transit
+# through dead zone, etc). 15 minutes is conservative.
+REOPEN_GAP_S = 15 * 60
+
+# Don't accumulate runaway memory: drop entries unseen for an hour.
+PRUNE_AFTER_S = 60 * 60
+
+# Cap on accumulated airtime per session so a single bug elsewhere
+# (e.g. ts clock skew) can't produce comically large numbers.
+MAX_SESSION_SECONDS = 24 * 3600  # 24h — longest realistic civilian leg
+
+
+_observations: dict[str, dict[str, float]] = {}
+_lock = threading.Lock()
+_last_prune_at = 0.0
+
+
+def record_observation(icao_hex: str, *, now: float | None = None) -> int:
+    """Record a sighting of ``icao_hex`` and return airtime so far (seconds).
+
+    Returns 0 for the first-ever sighting (no elapsed time yet) or when
+    ``icao_hex`` is falsy. The caller can multiply the returned seconds
+    by ``rate_per_hour / 3600`` to get cumulative consumption.
+    """
+    if not icao_hex:
+        return 0
+    key = str(icao_hex).strip().lower()
+    if not key:
+        return 0
+    current = float(now if now is not None else time.time())
+
+    with _lock:
+        entry = _observations.get(key)
+        if entry is None:
+            _observations[key] = {"first_seen_at": current, "last_seen_at": current}
+            return 0
+        # Use explicit ``is None`` checks instead of ``or`` short-circuit:
+        # ``0.0`` is a legitimate timestamp value (e.g. test fixtures
+        # seeding a far-past first_seen_at to exercise the clamp) but
+        # ``0.0 or fallback`` collapses to ``fallback`` because 0.0 is
+        # falsy. Bit me on my own test — leaving the safer form here.
+        last_raw = entry.get("last_seen_at")
+        last_seen = float(last_raw) if last_raw is not None else current
+        gap = current - last_seen
+        if gap > REOPEN_GAP_S:
+            # Treat as a new flight session — the plane landed/disappeared
+            # long enough that the prior cumulative count is no longer
+            # the same flight.
+            _observations[key] = {"first_seen_at": current, "last_seen_at": current}
+            return 0
+        first_raw = entry.get("first_seen_at")
+        first = float(first_raw) if first_raw is not None else current
+        # Clamp absurd values from clock skew or bad input.
+        elapsed = max(0, min(int(current - first), MAX_SESSION_SECONDS))
+        entry["last_seen_at"] = current
+        return elapsed
+
+
+def prune(*, now: float | None = None) -> int:
+    """Drop entries we haven't seen in ``PRUNE_AFTER_S`` seconds.
+
+    Returns number of entries dropped. Safe to call from a scheduler tick;
+    cheap (single dict scan) so cadence doesn't matter much.
+    """
+    current = float(now if now is not None else time.time())
+    dropped = 0
+    with _lock:
+        stale_keys = []
+        for k, v in _observations.items():
+            last_raw = v.get("last_seen_at")
+            last = float(last_raw) if last_raw is not None else 0.0
+            if current - last > PRUNE_AFTER_S:
+                stale_keys.append(k)
+        for k in stale_keys:
+            del _observations[k]
+            dropped += 1
+    return dropped
+
+
+def get_session_seconds(icao_hex: str, *, now: float | None = None) -> int:
+    """Read-only accessor: airtime for a known icao without bumping last-seen.
+
+    Used by tests and external consumers (e.g. when rendering a snapshot
+    of all in-flight aircraft, you want the current value, not to update
+    last_seen_at as a side effect).
+    """
+    if not icao_hex:
+        return 0
+    key = str(icao_hex).strip().lower()
+    with _lock:
+        entry = _observations.get(key)
+        if entry is None:
+            return 0
+        current = float(now if now is not None else time.time())
+        first_raw = entry.get("first_seen_at")
+        first = float(first_raw) if first_raw is not None else current
+        return max(0, min(int(current - first), MAX_SESSION_SECONDS))
+
+
+def _reset_for_tests() -> None:
+    """Drop all observations. Test helper only."""
+    with _lock:
+        _observations.clear()

backend/services/fetchers/flights.py

@@ -13,12 +13,14 @@ import concurrent.futures
 import random
 import requests
 from datetime import datetime
-from cachetools import TTLCache
 from services.network_utils import fetch_with_curl
 from services.fetchers._store import latest_data, _data_lock, _mark_fresh
 from services.fetchers.plane_alert import enrich_with_plane_alert, enrich_with_tracked_names
 from services.fetchers.emissions import get_emissions_info
+from services.fetchers.flight_observations import record_observation as _record_flight_observation
 from services.fetchers.retry import with_retry
+from services.fetchers.route_database import lookup_route
+from services.fetchers.aircraft_database import lookup_aircraft_type
 from services.constants import GPS_JAMMING_NACP_THRESHOLD, GPS_JAMMING_MIN_RATIO, GPS_JAMMING_MIN_AIRCRAFT
 
 logger = logging.getLogger("services.data_fetcher")
@@ -28,6 +30,88 @@ _RE_AIRLINE_CODE_1 = re.compile(r"^([A-Z]{3})\d")
 _RE_AIRLINE_CODE_2 = re.compile(r"^([A-Z]{3})[A-Z\d]")
 
 
+def detect_gps_jamming_zones(
+    raw_flights: list[dict],
+    *,
+    min_aircraft: int | None = None,
+    min_ratio: float | None = None,
+    nacp_threshold: int | None = None,
+) -> list[dict]:
+    """Detect GPS interference zones from a snapshot of raw ADS-B aircraft.
+
+    Methodology mirrors GPSJam.org / Flightradar24: bin aircraft into 1°x1°
+    grid cells, flag cells where the fraction of aircraft reporting degraded
+    NACp clears a threshold.
+
+    Inputs
+    ------
+    raw_flights:
+        Iterable of dicts. Each item is expected to carry ``lat``, ``lng``
+        (or ``lon``), and ``nac_p``. Records missing position OR missing
+        ``nac_p`` entirely (typical for OpenSky-sourced flights) are
+        skipped — absence-of-data isn't evidence of anything.
+
+    nac_p == 0 IS counted as degraded. Pre-fix code skipped it on the theory
+    that "0 = old transponder, never computed accuracy." That's only half
+    right: modern Mode-S Enhanced Surveillance transponders also fall back
+    to nac_p=0 when they lose GPS lock entirely — which is exactly the
+    jamming signature we're trying to detect. Filtering 0 out was discarding
+    the strongest evidence.
+
+    Denoising:
+        1. Require ``min_aircraft`` per grid cell for statistical validity.
+        2. Subtract 1 from degraded count per cell (GPSJam's technique) so
+           a single quirky transponder can't flag an entire zone.
+        3. Require ratio ``adjusted_degraded / total > min_ratio``.
+
+    All thresholds default to the module-level constants but can be
+    overridden for testing.
+    """
+    min_aircraft = GPS_JAMMING_MIN_AIRCRAFT if min_aircraft is None else int(min_aircraft)
+    min_ratio = GPS_JAMMING_MIN_RATIO if min_ratio is None else float(min_ratio)
+    nacp_threshold = (
+        GPS_JAMMING_NACP_THRESHOLD if nacp_threshold is None else int(nacp_threshold)
+    )
+
+    jamming_grid: dict[str, dict[str, int]] = {}
+    for rf in raw_flights or []:
+        rlat = rf.get("lat")
+        rlng = rf.get("lng") if rf.get("lng") is not None else rf.get("lon")
+        if rlat is None or rlng is None:
+            continue
+        nacp = rf.get("nac_p")
+        if nacp is None:
+            continue
+        grid_key = f"{int(rlat)},{int(rlng)}"
+        cell = jamming_grid.setdefault(grid_key, {"degraded": 0, "total": 0})
+        cell["total"] += 1
+        if nacp < nacp_threshold:
+            cell["degraded"] += 1
+
+    jamming_zones: list[dict] = []
+    for gk, counts in jamming_grid.items():
+        if counts["total"] < min_aircraft:
+            continue
+        adjusted_degraded = max(counts["degraded"] - 1, 0)
+        if adjusted_degraded == 0:
+            continue
+        ratio = adjusted_degraded / counts["total"]
+        if ratio > min_ratio:
+            lat_i, lng_i = gk.split(",")
+            severity = "low" if ratio < 0.5 else "medium" if ratio < 0.75 else "high"
+            jamming_zones.append(
+                {
+                    "lat": int(lat_i) + 0.5,
+                    "lng": int(lng_i) + 0.5,
+                    "severity": severity,
+                    "ratio": round(ratio, 2),
+                    "degraded": counts["degraded"],
+                    "total": counts["total"],
+                }
+            )
+    return jamming_zones
+
+
 # ---------------------------------------------------------------------------
 # OpenSky Network API Client (OAuth2)
 # ---------------------------------------------------------------------------
@@ -76,6 +160,7 @@ opensky_client = OpenSkyClient(
 # Throttling and caching for OpenSky (400 req/day limit)
 last_opensky_fetch = 0
 cached_opensky_flights = []
+_opensky_cache_lock = threading.Lock()
 
 # ---------------------------------------------------------------------------
 # Supplemental ADS-B sources for blind-spot gap-filling
@@ -98,6 +183,7 @@ _AIRPLANES_LIVE_DELAY_SECONDS = 1.2
 _AIRPLANES_LIVE_DELAY_JITTER_SECONDS = 0.4
 last_supplemental_fetch = 0
 cached_supplemental_flights = []
+_supplemental_cache_lock = threading.Lock()
 
 # Helicopter type codes (backend classification)
 _HELI_TYPES_BACKEND = {
@@ -253,12 +339,23 @@ PRIVATE_JET_TYPES = {
 # Flight trails state
 flight_trails = {}  # {icao_hex: {points: [[lat, lng, alt, ts], ...], last_seen: ts}}
 _trails_lock = threading.Lock()
-_MAX_TRACKED_TRAILS = 2000
+_MAX_TRACKED_TRAILS = 20000
 
-# Routes cache
-dynamic_routes_cache = TTLCache(maxsize=5000, ttl=7200)
-routes_fetch_in_progress = False
-_routes_lock = threading.Lock()
+
+def get_flight_trail(icao24: str) -> list:
+    """Return the accumulated trail for a single aircraft without expanding live payloads."""
+    hex_id = str(icao24 or "").strip().lower()
+    if not hex_id:
+        return []
+    with _trails_lock:
+        points = flight_trails.get(hex_id, {}).get("points", [])
+        return [list(point) for point in points]
+
+# Route enrichment is now served from services.fetchers.route_database, which
+# bulk-loads vrs-standing-data.adsb.lol/routes.csv.gz once per day and looks up
+# callsigns from an in-memory index. Replaces the legacy /api/0/routeset POST,
+# which was both blocked under the ShadowBroker UA (HTTP 451) and broken
+# upstream (returning 201 with empty body even for unblocked clients).
 
 
 def _fetch_supplemental_sources(seen_hex: set) -> list:
@@ -266,12 +363,13 @@ def _fetch_supplemental_sources(seen_hex: set) -> list:
     global last_supplemental_fetch, cached_supplemental_flights
 
     now = time.time()
-    if now - last_supplemental_fetch < _SUPPLEMENTAL_FETCH_INTERVAL:
-        return [
-            f
-            for f in cached_supplemental_flights
-            if f.get("hex", "").lower().strip() not in seen_hex
-        ]
+    with _supplemental_cache_lock:
+        if now - last_supplemental_fetch < _SUPPLEMENTAL_FETCH_INTERVAL:
+            return [
+                f
+                for f in cached_supplemental_flights
+                if f.get("hex", "").lower().strip() not in seen_hex
+            ]
 
     new_supplemental = []
     supplemental_hex = set()
@@ -363,8 +461,9 @@ def _fetch_supplemental_sources(seen_hex: set) -> list:
 
     fi_count = len(new_supplemental) - ap_count
 
-    cached_supplemental_flights = new_supplemental
-    last_supplemental_fetch = now
+    with _supplemental_cache_lock:
+        cached_supplemental_flights = new_supplemental
+        last_supplemental_fetch = now
     if new_supplemental:
         _mark_fresh("supplemental_flights")
 
@@ -375,73 +474,6 @@ def _fetch_supplemental_sources(seen_hex: set) -> list:
     return new_supplemental
 
 
-def fetch_routes_background(sampled):
-    global routes_fetch_in_progress
-    with _routes_lock:
-        if routes_fetch_in_progress:
-            return
-        routes_fetch_in_progress = True
-
-    try:
-        callsigns_to_query = []
-        for f in sampled:
-            c_sign = str(f.get("flight", "")).strip()
-            if c_sign and c_sign != "UNKNOWN":
-                callsigns_to_query.append(
-                    {"callsign": c_sign, "lat": f.get("lat", 0), "lng": f.get("lon", 0)}
-                )
-
-        batch_size = 100
-        batches = [
-            callsigns_to_query[i : i + batch_size]
-            for i in range(0, len(callsigns_to_query), batch_size)
-        ]
-
-        for batch in batches:
-            try:
-                r = fetch_with_curl(
-                    "https://api.adsb.lol/api/0/routeset",
-                    method="POST",
-                    json_data={"planes": batch},
-                    timeout=15,
-                )
-                if r.status_code == 200:
-                    route_data = r.json()
-                    route_list = []
-                    if isinstance(route_data, dict):
-                        route_list = route_data.get("value", [])
-                    elif isinstance(route_data, list):
-                        route_list = route_data
-
-                    for route in route_list:
-                        callsign = route.get("callsign", "")
-                        airports = route.get("_airports", [])
-                        if airports and len(airports) >= 2:
-                            orig_apt = airports[0]
-                            dest_apt = airports[-1]
-                            with _routes_lock:
-                                dynamic_routes_cache[callsign] = {
-                                    "orig_name": f"{orig_apt.get('iata', '')}: {orig_apt.get('name', 'Unknown')}",
-                                    "dest_name": f"{dest_apt.get('iata', '')}: {dest_apt.get('name', 'Unknown')}",
-                                    "orig_loc": [orig_apt.get("lon", 0), orig_apt.get("lat", 0)],
-                                    "dest_loc": [dest_apt.get("lon", 0), dest_apt.get("lat", 0)],
-                                }
-                time.sleep(0.25)
-            except (
-                requests.RequestException,
-                ConnectionError,
-                TimeoutError,
-                ValueError,
-                KeyError,
-                json.JSONDecodeError,
-                OSError,
-            ) as e:
-                logger.debug(f"Route batch request failed: {e}")
-    finally:
-        with _routes_lock:
-            routes_fetch_in_progress = False
-
-
 def _classify_and_publish(all_adsb_flights):
     """Shared pipeline: normalize raw ADS-B data → classify → merge → publish to latest_data.
 
@@ -453,13 +485,6 @@ def _classify_and_publish(all_adsb_flights):
     if not all_adsb_flights:
         return
 
-    with _routes_lock:
-        already_running = routes_fetch_in_progress
-    if not already_running:
-        threading.Thread(
-            target=fetch_routes_background, args=(all_adsb_flights,), daemon=True
-        ).start()
-
     for f in all_adsb_flights:
         try:
             lat = f.get("lat")
@@ -478,8 +503,7 @@ def _classify_and_publish(all_adsb_flights):
             origin_name = "UNKNOWN"
             dest_name = "UNKNOWN"
 
-            with _routes_lock:
-                cached_route = dynamic_routes_cache.get(flight_str)
+            cached_route = lookup_route(flight_str)
             if cached_route:
                 origin_name = cached_route["orig_name"]
                 dest_name = cached_route["dest_name"]
@@ -501,12 +525,35 @@ def _classify_and_publish(all_adsb_flights):
             gs_knots = f.get("gs")
             speed_knots = round(gs_knots, 1) if isinstance(gs_knots, (int, float)) else None
 
-            model_upper = f.get("t", "").upper()
+            # OpenSky's /states/all doesn't carry the aircraft type, so its
+            # records arrive with t="Unknown". Backfill from the OpenSky
+            # aircraft metadata DB by ICAO24 hex so heli classification and
+            # downstream emissions enrichment both see a real type code.
+            raw_type = str(f.get("t") or "").strip()
+            if not raw_type or raw_type.lower() == "unknown":
+                looked_up_type = lookup_aircraft_type(f.get("hex", ""))
+                if looked_up_type:
+                    f["t"] = looked_up_type
+                    raw_type = looked_up_type
+
+            model_upper = raw_type.upper()
             if model_upper == "TWR":
                 continue
 
             ac_category = "heli" if model_upper in _HELI_TYPES_BACKEND else "plane"
 
+            # Source attribution: prefer the explicit ``source`` tag stamped
+            # at fetch time (adsb.lol, OpenSky). If absent, fall back to the
+            # legacy ``supplemental_source`` (airplanes.live, adsb.fi) so
+            # supplementals are still attributed without changing their
+            # tagger. Final fallback "adsb.lol" preserves prior behavior for
+            # any caller that synthesizes records without going through one
+            # of our fetchers (e.g. tests).
+            source = (
+                f.get("source")
+                or f.get("supplemental_source")
+                or "adsb.lol"
+            )
             flights.append(
                 {
                     "callsign": flight_str,
@@ -528,6 +575,7 @@ def _classify_and_publish(all_adsb_flights):
                     "airline_code": airline_code,
                     "aircraft_category": ac_category,
                     "nac_p": f.get("nac_p"),
+                    "source": source,
                 }
             )
         except (ValueError, TypeError, KeyError, AttributeError) as loop_e:
@@ -543,11 +591,33 @@ def _classify_and_publish(all_adsb_flights):
     for f in flights:
         enrich_with_plane_alert(f)
         enrich_with_tracked_names(f)
-        # Attach fuel-burn / CO2 emissions estimate when model is known
+        # Attach fuel-burn / CO2 emissions estimate when model is known.
+        # OpenSky's /states/all doesn't carry aircraft type, so OpenSky-sourced
+        # flights arrive with model="Unknown". For tracked planes, the
+        # Plane-Alert DB has the friendly type name in alert_type, and the
+        # emissions aliases table already maps those names to ICAO codes.
         model = f.get("model")
+        if not model or model.strip().lower() in {"", "unknown"}:
+            model = f.get("alert_type") or ""
         if model:
             emi = get_emissions_info(model)
             if emi:
+                # Cumulative fuel/CO2: multiply the per-hour rate by how
+                # long we've been observing this airframe. Users want to
+                # see the *amount* burned, not just the rate. If we've
+                # never seen this hex before, observed_seconds is 0 and
+                # the cumulative values are 0 until the next refresh —
+                # the rate is still useful info on its own.
+                observed_seconds = _record_flight_observation(
+                    f.get("icao24") or ""
+                )
+                elapsed_h = observed_seconds / 3600.0
+                emi = {
+                    **emi,
+                    "observed_seconds": observed_seconds,
+                    "fuel_gallons_burned": round(emi["fuel_gph"] * elapsed_h, 1),
+                    "co2_kg_emitted": round(emi["co2_kg_per_hour"] * elapsed_h, 1),
+                }
                 f["emissions"] = emi
 
         callsign = f.get("callsign", "").strip().upper()
@@ -618,6 +688,10 @@ def _classify_and_publish(all_adsb_flights):
             latest_data["flights"] = flights
 
     # Merge tracked civilian flights with tracked military flights
+    # Stale tracked flights (not seen in any ADS-B source for >5 min) are dropped.
+    _TRACKED_STALE_S = 300  # 5 minutes
+    _merge_ts = time.time()
+
     with _data_lock:
         existing_tracked = copy.deepcopy(latest_data.get("tracked_flights", []))
 
@@ -625,10 +699,12 @@ def _classify_and_publish(all_adsb_flights):
     for t in tracked:
         icao = t.get("icao24", "").upper()
         if icao:
+            t["_seen_at"] = _merge_ts
             fresh_tracked_map[icao] = t
 
     merged_tracked = []
     seen_icaos = set()
+    stale_dropped = 0
     for old_t in existing_tracked:
         icao = old_t.get("icao24", "").upper()
         if icao in fresh_tracked_map:
@@ -639,8 +715,13 @@ def _classify_and_publish(all_adsb_flights):
             merged_tracked.append(fresh)
             seen_icaos.add(icao)
         else:
-            merged_tracked.append(old_t)
-            seen_icaos.add(icao)
+            # Keep stale entry only if it was seen recently
+            age = _merge_ts - old_t.get("_seen_at", 0)
+            if age < _TRACKED_STALE_S:
+                merged_tracked.append(old_t)
+                seen_icaos.add(icao)
+            else:
+                stale_dropped += 1
 
     for icao, t in fresh_tracked_map.items():
         if icao not in seen_icaos:
@@ -649,26 +730,38 @@ def _classify_and_publish(all_adsb_flights):
     with _data_lock:
         latest_data["tracked_flights"] = merged_tracked
     logger.info(
-        f"Tracked flights: {len(merged_tracked)} total ({len(fresh_tracked_map)} fresh from civilian)"
+        f"Tracked flights: {len(merged_tracked)} total ({len(fresh_tracked_map)} fresh from civilian, {stale_dropped} stale dropped)"
     )
 
     # --- Trail Accumulation ---
-    def _accumulate_trail(f, now_ts, check_route=True):
+    _TRAIL_INTERVAL_S = 60  # selected trails need enough resolution to show where unknown-route traffic came from
+
+    def _accumulate_trail(f, now_ts, attach_known_route_trail=False):
         hex_id = f.get("icao24", "").lower()
         if not hex_id:
             return 0, None
-        if check_route and f.get("origin_name", "UNKNOWN") != "UNKNOWN":
-            f["trail"] = []
-            return 0, hex_id
+
+        def _known_route_name(value):
+            normalized = str(value or "").strip().upper()
+            return bool(normalized and normalized != "UNKNOWN")
+
+        has_known_route = bool(
+            (f.get("origin_loc") and f.get("dest_loc"))
+            or (_known_route_name(f.get("origin_name")) and _known_route_name(f.get("dest_name")))
+        )
         lat, lng, alt = f.get("lat"), f.get("lng"), f.get("alt", 0)
         if lat is None or lng is None:
-            f["trail"] = flight_trails.get(hex_id, {}).get("points", [])
+            f["trail"] = [] if has_known_route and not attach_known_route_trail else flight_trails.get(hex_id, {}).get("points", [])
             return 0, hex_id
         point = [round(lat, 5), round(lng, 5), round(alt, 1), round(now_ts)]
         if hex_id not in flight_trails:
             flight_trails[hex_id] = {"points": [], "last_seen": now_ts}
         trail_data = flight_trails[hex_id]
-        if (
+        # Only append a new point if enough time has passed since the last one
+        last_point_ts = trail_data["points"][-1][3] if trail_data["points"] else 0
+        if now_ts - last_point_ts < _TRAIL_INTERVAL_S:
+            trail_data["last_seen"] = now_ts
+        elif (
             trail_data["points"]
             and trail_data["points"][-1][0] == point[0]
             and trail_data["points"][-1][1] == point[1]
@@ -679,31 +772,42 @@ def _classify_and_publish(all_adsb_flights):
             trail_data["last_seen"] = now_ts
         if len(trail_data["points"]) > 200:
             trail_data["points"] = trail_data["points"][-200:]
-        f["trail"] = trail_data["points"]
+        # Keep known-route flights visually clean in the main payload; selected
+        # detail panels can still fetch this server-side trail to compute
+        # observed fuel/CO2 burn.
+        f["trail"] = [] if has_known_route and not attach_known_route_trail else trail_data["points"]
         return 1, hex_id
 
     now_ts = datetime.utcnow().timestamp()
     with _data_lock:
+        commercial_snapshot = copy.deepcopy(latest_data.get("commercial_flights", []))
+        private_jets_snapshot = copy.deepcopy(latest_data.get("private_jets", []))
+        private_ga_snapshot = copy.deepcopy(latest_data.get("private_flights", []))
         military_snapshot = copy.deepcopy(latest_data.get("military_flights", []))
         tracked_snapshot = copy.deepcopy(latest_data.get("tracked_flights", []))
         raw_flights_snapshot = list(latest_data.get("flights", []))
 
-    all_lists = [commercial, private_jets, private_ga, existing_tracked]
+    # Accumulate trails for every aircraft so selected details can estimate
+    # observed fuel/CO2 burn. Known-route flights keep an empty payload trail so
+    # the route line, not historical breadcrumbs, remains the visible map path.
+    route_check_lists = [commercial_snapshot, private_jets_snapshot, private_ga_snapshot]
+    always_trail_lists = [tracked_snapshot, military_snapshot]
     seen_hexes = set()
     trail_count = 0
     with _trails_lock:
-        for flist in all_lists:
+        for flist in route_check_lists:
             for f in flist:
-                count, hex_id = _accumulate_trail(f, now_ts, check_route=True)
+                count, hex_id = _accumulate_trail(f, now_ts, attach_known_route_trail=False)
                 trail_count += count
                 if hex_id:
                     seen_hexes.add(hex_id)
 
-        for mf in military_snapshot:
-            count, hex_id = _accumulate_trail(mf, now_ts, check_route=False)
-            trail_count += count
-            if hex_id:
-                seen_hexes.add(hex_id)
+        for flist in always_trail_lists:
+            for f in flist:
+                count, hex_id = _accumulate_trail(f, now_ts, attach_known_route_trail=False)
+                trail_count += count
+                if hex_id:
+                    seen_hexes.add(hex_id)
 
         tracked_hexes = {t.get("icao24", "").lower() for t in tracked_snapshot}
         stale_keys = []
@@ -724,57 +828,16 @@ def _classify_and_publish(all_adsb_flights):
         f"Trail accumulation: {trail_count} active trails, {len(stale_keys)} pruned, {len(flight_trails)} total"
     )
 
-    # --- GPS Jamming Detection ---
-    # Uses NACp (Navigation Accuracy Category – Position) from ADS-B to infer
-    # GPS interference zones, similar to GPSJam.org / Flightradar24.
-    # NACp < 8 = position accuracy worse than the FAA-mandated 0.05 NM.
-    #
-    # Denoising (to suppress false positives from old GA transponders):
-    # 1. Skip nac_p == 0 ("unknown accuracy") — old transponders that never
-    #    computed accuracy, NOT evidence of jamming.  Real jamming shows 1-7.
-    # 2. Require minimum aircraft per grid cell for statistical validity.
-    # 3. Subtract 1 from degraded count per cell (GPSJam's technique) so a
-    #    single quirky transponder can't flag an entire zone.
-    # 4. Require the adjusted ratio to exceed the threshold.
-    try:
-        jamming_grid = {}
-        raw_flights = raw_flights_snapshot
-        for rf in raw_flights:
-            rlat = rf.get("lat")
-            rlng = rf.get("lng") or rf.get("lon")
-            if rlat is None or rlng is None:
-                continue
-            nacp = rf.get("nac_p")
-            if nacp is None or nacp == 0:
-                continue
-            grid_key = f"{int(rlat)},{int(rlng)}"
-            if grid_key not in jamming_grid:
-                jamming_grid[grid_key] = {"degraded": 0, "total": 0}
-            jamming_grid[grid_key]["total"] += 1
-            if nacp < GPS_JAMMING_NACP_THRESHOLD:
-                jamming_grid[grid_key]["degraded"] += 1
+    with _data_lock:
+        latest_data["commercial_flights"] = commercial_snapshot
+        latest_data["private_jets"] = private_jets_snapshot
+        latest_data["private_flights"] = private_ga_snapshot
+        latest_data["tracked_flights"] = tracked_snapshot
+        latest_data["military_flights"] = military_snapshot
 
-        jamming_zones = []
-        for gk, counts in jamming_grid.items():
-            if counts["total"] < GPS_JAMMING_MIN_AIRCRAFT:
-                continue
-            adjusted_degraded = max(counts["degraded"] - 1, 0)
-            if adjusted_degraded == 0:
-                continue
-            ratio = adjusted_degraded / counts["total"]
-            if ratio > GPS_JAMMING_MIN_RATIO:
-                lat_i, lng_i = gk.split(",")
-                severity = "low" if ratio < 0.5 else "medium" if ratio < 0.75 else "high"
-                jamming_zones.append(
-                    {
-                        "lat": int(lat_i) + 0.5,
-                        "lng": int(lng_i) + 0.5,
-                        "severity": severity,
-                        "ratio": round(ratio, 2),
-                        "degraded": counts["degraded"],
-                        "total": counts["total"],
-                    }
-                )
+    # --- GPS Jamming Detection ---
+    try:
+        jamming_zones = detect_gps_jamming_zones(raw_flights_snapshot)
         with _data_lock:
             latest_data["gps_jamming"] = jamming_zones
         if jamming_zones:
@@ -850,7 +913,15 @@ def _fetch_adsb_lol_regions():
             res = fetch_with_curl(url, timeout=10)
             if res.status_code == 200:
                 data = res.json()
-                return data.get("ac", [])
+                aircraft = data.get("ac", [])
+                # Stamp the source at the fetch site so attribution survives
+                # the OpenSky/supplemental dedupe-by-hex merge downstream.
+                # Previously adsb.lol records carried no marker while OpenSky
+                # records got ``is_opensky: True`` — which made flight tooltips
+                # look like everything came from OpenSky.
+                for a in aircraft:
+                    a["source"] = "adsb.lol"
+                return aircraft
         except (
             requests.RequestException,
             ConnectionError,
@@ -889,79 +960,101 @@ def _enrich_with_opensky_and_supplemental(adsb_flights):
         now = time.time()
         global last_opensky_fetch, cached_opensky_flights
 
-        if now - last_opensky_fetch > 300:
+        with _opensky_cache_lock:
+            _need_opensky = now - last_opensky_fetch > 300
+            if not _need_opensky:
+                opensky_snapshot = list(cached_opensky_flights)
+
+        if _need_opensky:
             token = opensky_client.get_token()
             if token:
-                opensky_regions = [
-                    {
-                        "name": "Africa",
-                        "bbox": {"lamin": -35.0, "lomin": -20.0, "lamax": 38.0, "lomax": 55.0},
-                    },
-                    {
-                        "name": "Asia",
-                        "bbox": {"lamin": 0.0, "lomin": 30.0, "lamax": 75.0, "lomax": 150.0},
-                    },
-                    {
-                        "name": "South America",
-                        "bbox": {"lamin": -60.0, "lomin": -95.0, "lamax": 15.0, "lomax": -30.0},
-                    },
-                ]
-
+                # One global /states/all query = 4 credits flat per OpenSky
+                # docs (https://openskynetwork.github.io/opensky-api/rest.html).
+                # At the current 5-minute cadence that's 4 × 288 = 1152
+                # credits/day, ~29% of the 4000-credit standard daily quota,
+                # and returns every aircraft worldwide in a single call.
+                # The previous 3-regional-bbox approach cost 12 credits/cycle
+                # AND missed North America, Europe, and Oceania entirely.
                 new_opensky_flights = []
-                for os_reg in opensky_regions:
-                    try:
-                        bb = os_reg["bbox"]
-                        os_url = f"https://opensky-network.org/api/states/all?lamin={bb['lamin']}&lomin={bb['lomin']}&lamax={bb['lamax']}&lomax={bb['lomax']}"
-                        headers = {"Authorization": f"Bearer {token}"}
-                        os_res = requests.get(os_url, headers=headers, timeout=15)
+                try:
+                    os_url = "https://opensky-network.org/api/states/all"
+                    headers = {"Authorization": f"Bearer {token}"}
+                    os_res = requests.get(os_url, headers=headers, timeout=30)
 
-                        if os_res.status_code == 200:
-                            os_data = os_res.json()
-                            states = os_data.get("states") or []
-                            logger.info(
-                                f"OpenSky: Fetched {len(states)} states for {os_reg['name']}"
+                    if os_res.status_code == 200:
+                        os_data = os_res.json()
+                        states = os_data.get("states") or []
+                        remaining = os_res.headers.get("X-Rate-Limit-Remaining", "?")
+                        logger.info(
+                            f"OpenSky: fetched {len(states)} global states "
+                            f"(credits remaining: {remaining})"
+                        )
+                        for s in states:
+                            if s[5] is None or s[6] is None:
+                                continue
+                            new_opensky_flights.append(
+                                {
+                                    "hex": s[0],
+                                    "flight": s[1].strip() if s[1] else "UNKNOWN",
+                                    "r": s[2],
+                                    "lon": s[5],
+                                    "lat": s[6],
+                                    "alt_baro": (s[7] * 3.28084) if s[7] else 0,
+                                    "track": s[10] or 0,
+                                    "gs": (s[9] * 1.94384) if s[9] else 0,
+                                    "t": "Unknown",
+                                    "is_opensky": True,
+                                    "source": "OpenSky",
+                                }
                             )
+                    elif os_res.status_code == 429:
+                        retry_after = os_res.headers.get("X-Rate-Limit-Retry-After-Seconds", "?")
+                        logger.warning(
+                            f"OpenSky daily quota exhausted (4000 credits). "
+                            f"Retry after {retry_after}s. Serving stale data until reset."
+                        )
+                    else:
+                        logger.warning(
+                            f"OpenSky /states/all failed: HTTP {os_res.status_code}"
+                        )
+                except (
+                    requests.RequestException,
+                    ConnectionError,
+                    TimeoutError,
+                    ValueError,
+                    KeyError,
+                    json.JSONDecodeError,
+                    OSError,
+                ) as ex:
+                    logger.error(f"OpenSky global fetch error: {ex}")
 
-                            for s in states:
-                                new_opensky_flights.append(
-                                    {
-                                        "hex": s[0],
-                                        "flight": s[1].strip() if s[1] else "UNKNOWN",
-                                        "r": s[2],
-                                        "lon": s[5],
-                                        "lat": s[6],
-                                        "alt_baro": (s[7] * 3.28084) if s[7] else 0,
-                                        "track": s[10] or 0,
-                                        "gs": (s[9] * 1.94384) if s[9] else 0,
-                                        "t": "Unknown",
-                                        "is_opensky": True,
-                                    }
-                                )
-                        else:
-                            logger.warning(
-                                f"OpenSky API {os_reg['name']} failed: {os_res.status_code}"
-                            )
-                    except (
-                        requests.RequestException,
-                        ConnectionError,
-                        TimeoutError,
-                        ValueError,
-                        KeyError,
-                        json.JSONDecodeError,
-                        OSError,
-                    ) as ex:
-                        logger.error(f"OpenSky fetching error for {os_reg['name']}: {ex}")
-
-                cached_opensky_flights = new_opensky_flights
-                last_opensky_fetch = now
+                with _opensky_cache_lock:
+                    if new_opensky_flights:
+                        cached_opensky_flights = new_opensky_flights
+                    last_opensky_fetch = now
+                opensky_snapshot = new_opensky_flights or list(cached_opensky_flights)
+            else:
+                # Token refresh failed — fall back to existing cached data
+                with _opensky_cache_lock:
+                    opensky_snapshot = list(cached_opensky_flights)
 
         # Merge OpenSky (dedup by hex)
-        for osf in cached_opensky_flights:
+        for osf in opensky_snapshot:
             h = osf.get("hex")
             if h and h.lower().strip() not in seen_hex:
                 all_flights.append(osf)
                 seen_hex.add(h.lower().strip())
 
+        # Publish OpenSky-merged data immediately so users see flights even if
+        # supplemental gap-fill is slow or rate-limited (airplanes.live can take
+        # 100+ seconds when its regional endpoints are throttled).
+        if len(all_flights) > len(adsb_flights):
+            logger.info(
+                f"OpenSky merge: {len(all_flights) - len(adsb_flights)} additional aircraft, "
+                "publishing before supplemental gap-fill"
+            )
+            _classify_and_publish(all_flights)
+
         # Supplemental gap-fill
         try:
             gap_fill = _fetch_supplemental_sources(seen_hex)
@@ -1008,14 +1101,18 @@ def fetch_flights():
         if adsb_flights:
             logger.info(f"adsb.lol: {len(adsb_flights)} aircraft — publishing immediately")
             _classify_and_publish(adsb_flights)
-
-            # Phase 2: kick off slow enrichment in background
-            threading.Thread(
-                target=_enrich_with_opensky_and_supplemental,
-                args=(adsb_flights,),
-                daemon=True,
-            ).start()
         else:
-            logger.warning("adsb.lol returned 0 aircraft")
+            logger.warning(
+                "adsb.lol returned 0 aircraft — relying on OpenSky/supplemental sources"
+            )
+
+        # Phase 2: always run — OpenSky is the fallback when adsb.lol blocks us
+        # (it has been known to 451 the bulk regional endpoint), and supplemental
+        # gap-fill should always run regardless of Phase 1 success.
+        threading.Thread(
+            target=_enrich_with_opensky_and_supplemental,
+            args=(adsb_flights,),
+            daemon=True,
+        ).start()
     except Exception as e:
         logger.error(f"Error fetching flights: {e}")

backend/services/fetchers/geo.py

@@ -1,10 +1,13 @@
 """Ship and geopolitics fetchers — AIS vessels, carriers, frontlines, GDELT, LiveUAmap, fishing."""
 
 import csv
+import concurrent.futures
 import io
 import math
 import os
 import logging
+import time
+from urllib.parse import urlencode
 from services.network_utils import fetch_with_curl
 from services.fetchers._store import latest_data, _data_lock, _mark_fresh
 from services.fetchers.retry import with_retry
@@ -12,6 +15,24 @@ from services.fetchers.retry import with_retry
 logger = logging.getLogger(__name__)
 
 
+def _env_flag(name: str) -> str:
+    return str(os.getenv(name, "")).strip().lower()
+
+
+def liveuamap_scraper_enabled() -> bool:
+    """Return whether the Playwright-based LiveUAMap scraper should run.
+
+    It is useful enrichment, but it starts a browser/Node driver and must not be
+    allowed to destabilize Windows local startup.
+    """
+    setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
+    if setting in {"1", "true", "yes", "on"}:
+        return True
+    if setting in {"0", "false", "no", "off"}:
+        return False
+    return os.name != "nt"
+
+
 # ---------------------------------------------------------------------------
 # Ships (AIS + Carriers)
 # ---------------------------------------------------------------------------
@@ -27,20 +48,24 @@ def fetch_ships():
     from services.ais_stream import get_ais_vessels
     from services.carrier_tracker import get_carrier_positions
 
-    ships = []
-    try:
-        carriers = get_carrier_positions()
-        ships.extend(carriers)
-    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
-        logger.error(f"Carrier tracker error (non-fatal): {e}")
-        carriers = []
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2, thread_name_prefix="ship_fetch") as executor:
+        carrier_future = executor.submit(get_carrier_positions)
+        ais_future = executor.submit(get_ais_vessels)
 
-    try:
-        ais_vessels = get_ais_vessels()
-        ships.extend(ais_vessels)
-    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
-        logger.error(f"AIS stream error (non-fatal): {e}")
-        ais_vessels = []
+        try:
+            carriers = carrier_future.result()
+        except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+            logger.error(f"Carrier tracker error (non-fatal): {e}")
+            carriers = []
+
+        try:
+            ais_vessels = ais_future.result()
+        except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+            logger.error(f"AIS stream error (non-fatal): {e}")
+            ais_vessels = []
+
+    ships = list(carriers or [])
+    ships.extend(ais_vessels or [])
 
     # Enrich ships with yacht alert data (tracked superyachts)
     from services.fetchers.yacht_alert import enrich_with_yacht_alert
@@ -184,6 +209,12 @@ def update_liveuamap():
 
     if not is_any_active("global_incidents"):
         return
+    if not liveuamap_scraper_enabled():
+        logger.info(
+            "Liveuamap scraper disabled for this runtime; set "
+            "SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=1 to opt in."
+        )
+        return
     logger.info("Running scheduled Liveuamap scraper...")
     try:
         from services.liveuamap_scraper import fetch_liveuamap
@@ -200,52 +231,177 @@ def update_liveuamap():
 # ---------------------------------------------------------------------------
 # Fishing Activity (Global Fishing Watch)
 # ---------------------------------------------------------------------------
+def _fishing_vessel_key(event: dict) -> str:
+    vessel_ssvid = str(event.get("vessel_ssvid", "") or "").strip()
+    if vessel_ssvid:
+        return f"ssvid:{vessel_ssvid}"
+    vessel_id = str(event.get("vessel_id", "") or "").strip()
+    if vessel_id:
+        return f"vid:{vessel_id}"
+    vessel_name = str(event.get("vessel_name", "") or "").strip().upper()
+    vessel_flag = str(event.get("vessel_flag", "") or "").strip().upper()
+    if vessel_name:
+        return f"name:{vessel_name}|flag:{vessel_flag}"
+    return f"event:{event.get('id', '')}"
+
+
+def _fishing_event_rank(event: dict) -> tuple[str, str, float, str]:
+    return (
+        str(event.get("end", "") or ""),
+        str(event.get("start", "") or ""),
+        float(event.get("duration_hrs", 0) or 0),
+        str(event.get("id", "") or ""),
+    )
+
+
+def _dedupe_fishing_events(events: list[dict]) -> list[dict]:
+    latest_by_vessel: dict[str, dict] = {}
+    counts_by_vessel: dict[str, int] = {}
+
+    for event in events:
+        vessel_key = _fishing_vessel_key(event)
+        counts_by_vessel[vessel_key] = counts_by_vessel.get(vessel_key, 0) + 1
+        current = latest_by_vessel.get(vessel_key)
+        if current is None or _fishing_event_rank(event) > _fishing_event_rank(current):
+            latest_by_vessel[vessel_key] = event
+
+    deduped: list[dict] = []
+    for vessel_key, event in latest_by_vessel.items():
+        event_copy = dict(event)
+        event_copy["event_count"] = counts_by_vessel.get(vessel_key, 1)
+        deduped.append(event_copy)
+
+    deduped.sort(key=_fishing_event_rank, reverse=True)
+    return deduped
+
+
+_FISHING_FETCH_INTERVAL_S = 3600  # once per hour — GFW data has ~5 day lag
+_last_fishing_fetch_ts: float = 0.0
+
+
 @with_retry(max_retries=1, base_delay=5)
 def fetch_fishing_activity():
     """Fetch recent fishing events from Global Fishing Watch (~5 day lag)."""
-    from services.fetchers._store import is_any_active
+    global _last_fishing_fetch_ts
+    from services.fetchers._store import is_any_active, latest_data
 
     if not is_any_active("fishing_activity"):
         return
+
+    # Skip if we already have data and fetched less than an hour ago
+    now = time.time()
+    if latest_data.get("fishing_activity") and (now - _last_fishing_fetch_ts) < _FISHING_FETCH_INTERVAL_S:
+        return
+
     token = os.environ.get("GFW_API_TOKEN", "")
     if not token:
         logger.debug("GFW_API_TOKEN not set, skipping fishing activity fetch")
         return
     events = []
     try:
-        url = (
-            "https://gateway.api.globalfishingwatch.org/v3/events"
-            "?datasets[0]=public-global-fishing-events:latest"
-            "&limit=500&sort=start&sort-direction=DESC"
-        )
+        import datetime as _dt
+
+        _end = _dt.date.today().isoformat()
+        _start = (_dt.date.today() - _dt.timedelta(days=7)).isoformat()
+        page_size = max(1, int(os.environ.get("GFW_EVENTS_PAGE_SIZE", "500") or "500"))
+        offset = 0
+        seen_offsets: set[int] = set()
+        seen_ids: set[str] = set()
         headers = {"Authorization": f"Bearer {token}"}
-        response = fetch_with_curl(url, timeout=30, headers=headers)
-        if response.status_code == 200:
-            entries = response.json().get("entries", [])
+
+        while True:
+            if offset in seen_offsets:
+                logger.warning("Fishing activity pagination repeated offset=%s; stopping fetch", offset)
+                break
+            seen_offsets.add(offset)
+
+            query = urlencode(
+                {
+                    "datasets[0]": "public-global-fishing-events:latest",
+                    "start-date": _start,
+                    "end-date": _end,
+                    "limit": page_size,
+                    "offset": offset,
+                }
+            )
+            url = f"https://gateway.api.globalfishingwatch.org/v3/events?{query}"
+            response = fetch_with_curl(url, timeout=30, headers=headers)
+            if response.status_code != 200:
+                logger.warning(
+                    "Fishing activity fetch failed at offset=%s: HTTP %s",
+                    offset,
+                    response.status_code,
+                )
+                break
+
+            payload = response.json() or {}
+            entries = payload.get("entries", [])
+            if not entries:
+                break
+
+            added_this_page = 0
             for e in entries:
                 pos = e.get("position", {})
+                vessel = e.get("vessel") or {}
                 lat = pos.get("lat")
                 lng = pos.get("lon")
                 if lat is None or lng is None:
                     continue
+                event_id = str(e.get("id", "") or "")
+                if event_id and event_id in seen_ids:
+                    continue
+                if event_id:
+                    seen_ids.add(event_id)
                 dur = e.get("event", {}).get("duration", 0) or 0
                 events.append(
                     {
-                        "id": e.get("id", ""),
+                        "id": event_id,
                         "type": e.get("type", "fishing"),
                         "lat": lat,
                         "lng": lng,
                         "start": e.get("start", ""),
                         "end": e.get("end", ""),
-                        "vessel_name": (e.get("vessel") or {}).get("name", "Unknown"),
-                        "vessel_flag": (e.get("vessel") or {}).get("flag", ""),
+                        "vessel_id": str(vessel.get("id", "") or ""),
+                        "vessel_ssvid": str(vessel.get("ssvid", "") or ""),
+                        "vessel_name": vessel.get("name", "Unknown"),
+                        "vessel_flag": vessel.get("flag", ""),
                         "duration_hrs": round(dur / 3600, 1),
                     }
                 )
-        logger.info(f"Fishing activity: {len(events)} events")
+                added_this_page += 1
+
+            if len(entries) < page_size:
+                break
+
+            next_offset = payload.get("nextOffset")
+            if next_offset is None:
+                next_offset = (payload.get("pagination") or {}).get("nextOffset")
+            if next_offset is None:
+                next_offset = offset + page_size
+            try:
+                next_offset = int(next_offset)
+            except (TypeError, ValueError):
+                next_offset = offset + page_size
+            if next_offset <= offset:
+                logger.warning(
+                    "Fishing activity pagination produced non-increasing next offset=%s; stopping fetch",
+                    next_offset,
+                )
+                break
+            if added_this_page == 0:
+                logger.warning(
+                    "Fishing activity page at offset=%s added no new events; stopping fetch",
+                    offset,
+                )
+                break
+            offset = next_offset
+        raw_event_count = len(events)
+        events = _dedupe_fishing_events(events)
+        logger.info("Fishing activity: %s raw events -> %s deduped vessels", raw_event_count, len(events))
     except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
         logger.error(f"Error fetching fishing activity: {e}")
     with _data_lock:
         latest_data["fishing_activity"] = events
     if events:
         _mark_fresh("fishing_activity")
+        _last_fishing_fetch_ts = time.time()

backend/services/fetchers/infrastructure.py

@@ -6,7 +6,7 @@ import heapq
 import logging
 from pathlib import Path
 from cachetools import TTLCache
-from services.network_utils import fetch_with_curl
+from services.network_utils import fetch_with_curl, outbound_user_agent
 from services.fetchers._store import latest_data, _data_lock, _mark_fresh
 from services.fetchers.retry import with_retry
 
@@ -29,7 +29,7 @@ def _geocode_region(region_name: str, country_name: str) -> tuple:
 
         query = urllib.parse.quote(f"{region_name}, {country_name}")
         url = f"https://nominatim.openstreetmap.org/search?q={query}&format=json&limit=1"
-        response = fetch_with_curl(url, timeout=8, headers={"User-Agent": "ShadowBroker-OSINT/1.0"})
+        response = fetch_with_curl(url, timeout=8, headers={"User-Agent": outbound_user_agent("infrastructure-data")})
         if response.status_code == 200:
             results = response.json()
             if results:

backend/services/fetchers/meshtastic_map.py

@@ -25,7 +25,10 @@ logger = logging.getLogger("services.data_fetcher")
 _API_URL = "https://meshtastic.liamcottle.net/api/v1/nodes"
 _CACHE_FILE = Path(__file__).resolve().parent.parent.parent / "data" / "meshtastic_nodes_cache.json"
 _FETCH_TIMEOUT = 90  # seconds — response is ~37MB, needs time on slow connections
-_MAX_AGE_HOURS = 4  # discard nodes not seen within this window (matches refresh interval)
+_MAX_AGE_HOURS = 24  # discard nodes not seen within this window
+# Skip network fetch if cached data is fresher than this — the API is a
+# one-person hobby service, so we prefer stale data over hammering it.
+_CACHE_TRUST_HOURS = 20
 
 # Track when we last fetched so the frontend can show staleness
 _last_fetch_ts: float = 0.0
@@ -141,13 +144,72 @@ def fetch_meshtastic_nodes():
         return
     global _last_fetch_ts
 
+    # Trust a recent cache on disk — avoids hammering the upstream HTTP API
+    # when every install polls on roughly the same cadence.
+    try:
+        if _CACHE_FILE.exists():
+            mtime = _CACHE_FILE.stat().st_mtime
+            if time.time() - mtime < _CACHE_TRUST_HOURS * 3600:
+                # If memory is empty (cold start), hydrate from cache and skip fetch.
+                with _data_lock:
+                    has_memory = bool(latest_data.get("meshtastic_map_nodes"))
+                if not has_memory:
+                    cached = _load_cache()
+                    if cached:
+                        with _data_lock:
+                            latest_data["meshtastic_map_nodes"] = cached
+                            latest_data["meshtastic_map_fetched_at"] = mtime
+                        _mark_fresh("meshtastic_map")
+                        logger.info(
+                            "Meshtastic map: cache fresh (<%.0fh), skipping network fetch",
+                            _CACHE_TRUST_HOURS,
+                        )
+                        return
+                else:
+                    logger.info(
+                        "Meshtastic map: cache fresh (<%.0fh), skipping network fetch",
+                        _CACHE_TRUST_HOURS,
+                    )
+                    return
+    except Exception as e:
+        logger.debug(f"Meshtastic cache freshness check failed: {e}")
+
+    # Build a polite User-Agent. Historically this included the operator
+    # callsign so meshtastic.org could rate-limit per-install; that's still
+    # the default behavior for backward compatibility. Operators who want
+    # stricter outbound privacy can suppress the callsign by setting
+    # MESHTASTIC_SEND_CALLSIGN_HEADER=false. Issue #203.
+    import os as _os
+    try:
+        from services.config import get_settings
+
+        callsign = str(getattr(get_settings(), "MESHTASTIC_OPERATOR_CALLSIGN", "") or "").strip()
+    except Exception:
+        callsign = ""
+
+    send_callsign_header = str(
+        _os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "true")
+    ).strip().lower() not in {"0", "false", "no", "off", ""}
+
+    # Round 7a: outbound_user_agent already includes the per-install handle.
+    # The optional Meshtastic callsign is appended as additional context so
+    # meshtastic.liamcottle.net's operator can identify both the install AND
+    # the registered radio operator (when MESHTASTIC_OPERATOR_CALLSIGN is set
+    # and MESHTASTIC_SEND_CALLSIGN_HEADER is true; see issue #203).
+    from services.network_utils import outbound_user_agent
+    ua_base = f"{outbound_user_agent('meshtastic-map')}; 24h polling"
+    if callsign and send_callsign_header:
+        user_agent = f"{ua_base}; node={callsign}"
+    else:
+        user_agent = ua_base
+
     try:
         logger.info("Fetching Meshtastic map nodes from API...")
         resp = requests.get(
             _API_URL,
             timeout=_FETCH_TIMEOUT,
             headers={
-                "User-Agent": "ShadowBroker/1.0 (OSINT dashboard, 4h polling)",
+                "User-Agent": user_agent,
                 "Accept": "application/json",
             },
         )

backend/services/fetchers/military.py

@@ -2,9 +2,12 @@
 
 import json
 import logging
+import time
 import requests
 from services.network_utils import fetch_with_curl
 from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.emissions import get_emissions_info
+from services.fetchers.flight_observations import record_observation as _record_flight_observation
 from services.fetchers.plane_alert import enrich_with_plane_alert
 
 logger = logging.getLogger("services.data_fetcher")
@@ -169,6 +172,7 @@ def fetch_military_flights():
                 h = a.get("hex", "").lower()
                 if h and h not in seen_hex:
                     seen_hex.add(h)
+                    a["source"] = "adsb.lol"
                     all_mil_ac.append(a)
     except Exception as e:
         logger.warning(f"adsb.lol mil fetch failed: {e}")
@@ -180,6 +184,7 @@ def fetch_military_flights():
                 h = a.get("hex", "").lower()
                 if h and h not in seen_hex:
                     seen_hex.add(h)
+                    a["source"] = "airplanes.live"
                     all_mil_ac.append(a)
             logger.info(f"airplanes.live mil: +{len(resp2.json().get('ac', []))} raw, {len(all_mil_ac)} total unique")
     except Exception as e:
@@ -232,6 +237,7 @@ def fetch_military_flights():
                             "registration": f.get("r", "N/A"),
                             "icao24": icao_hex,
                             "squawk": f.get("squawk", ""),
+                            "source": f.get("source") or "adsb.lol",
                         })
                         continue
 
@@ -256,7 +262,8 @@ def fetch_military_flights():
                         "model": f.get("t", "Unknown"),
                         "icao24": icao_hex,
                         "speed_knots": speed_knots,
-                        "squawk": f.get("squawk", "")
+                        "squawk": f.get("squawk", ""),
+                        "source": f.get("source") or "adsb.lol",
                     })
                 except Exception as loop_e:
                     logger.error(f"Mil flight interpolation error: {loop_e}")
@@ -288,6 +295,25 @@ def fetch_military_flights():
     remaining_mil = []
     for mf in military_flights:
         enrich_with_plane_alert(mf)
+        model = mf.get("model")
+        if not model or str(model).strip().lower() in {"", "unknown"}:
+            model = mf.get("alert_type") or ""
+        if model:
+            emissions = get_emissions_info(model)
+            if emissions:
+                # Cumulative fuel/CO2 since first observation — mirrors
+                # the civilian path in flights._classify_and_publish.
+                observed_seconds = _record_flight_observation(
+                    mf.get("icao24") or ""
+                )
+                elapsed_h = observed_seconds / 3600.0
+                emissions = {
+                    **emissions,
+                    "observed_seconds": observed_seconds,
+                    "fuel_gallons_burned": round(emissions["fuel_gph"] * elapsed_h, 1),
+                    "co2_kg_emitted": round(emissions["co2_kg_per_hour"] * elapsed_h, 1),
+                }
+                mf["emissions"] = emissions
         if mf.get("alert_category"):
             mf["type"] = "tracked_flight"
             tracked_mil.append(mf)
@@ -296,17 +322,23 @@ def fetch_military_flights():
     with _data_lock:
         latest_data["military_flights"] = remaining_mil
 
-    # Store tracked military flights — update positions for existing entries
+    # Store tracked military flights — update positions for existing entries.
+    # Drop stale entries not refreshed by ANY source (civilian or military) within 5 min.
+    _TRACKED_STALE_S = 300  # 5 minutes
+    _merge_ts = time.time()
+
     with _data_lock:
         existing_tracked = list(latest_data.get("tracked_flights", []))
     fresh_mil_map = {}
     for t in tracked_mil:
         icao = t.get("icao24", "").upper()
         if icao:
+            t["_seen_at"] = _merge_ts
             fresh_mil_map[icao] = t
 
     updated_tracked = []
     seen_icaos = set()
+    stale_dropped = 0
     for old_t in existing_tracked:
         icao = old_t.get("icao24", "").upper()
         if icao in fresh_mil_map:
@@ -317,11 +349,16 @@ def fetch_military_flights():
             updated_tracked.append(fresh)
             seen_icaos.add(icao)
         else:
-            updated_tracked.append(old_t)
-            seen_icaos.add(icao)
+            # Keep stale entry only if it was seen recently
+            age = _merge_ts - old_t.get("_seen_at", 0)
+            if age < _TRACKED_STALE_S:
+                updated_tracked.append(old_t)
+                seen_icaos.add(icao)
+            else:
+                stale_dropped += 1
     for icao, t in fresh_mil_map.items():
         if icao not in seen_icaos:
             updated_tracked.append(t)
     with _data_lock:
         latest_data["tracked_flights"] = updated_tracked
-    logger.info(f"Tracked flights: {len(updated_tracked)} total ({len(tracked_mil)} from military)")
+    logger.info(f"Tracked flights: {len(updated_tracked)} total ({len(tracked_mil)} from military, {stale_dropped} stale dropped)")

backend/services/fetchers/news.py

@@ -1,6 +1,9 @@
 """News fetching, geocoding, clustering, and risk assessment."""
+import os
 import re
+import time
 import logging
+import calendar
 import concurrent.futures
 import requests
 import feedparser
@@ -9,8 +12,28 @@ from services.fetchers._store import latest_data, _data_lock, _mark_fresh
 from services.fetchers.retry import with_retry
 from services.oracle_service import enrich_news_items, compute_global_threat_level, detect_breaking_events
 
+
+def news_fetch_enabled() -> bool:
+    """Return True only when the operator explicitly opts into news RSS pulls.
+
+    Defaults to **on** for backward compatibility (this is the only fetcher
+    where opting out is the new behavior, not the old one). Set
+    ``NEWS_ENABLED=false`` to disable all outbound RSS feed traffic.
+    """
+    return str(os.environ.get("NEWS_ENABLED", "true")).strip().lower() not in {
+        "0",
+        "false",
+        "no",
+        "off",
+        "",
+    }
+
 logger = logging.getLogger("services.data_fetcher")
 
+# Maximum article age in seconds.  Anything older than this is dropped
+# during each fetch cycle so the threat feed stays current.
+_MAX_ARTICLE_AGE_SECS = 48 * 3600  # 48 hours
+
 
 # Keyword -> coordinate mapping for geocoding news articles
 _KEYWORD_COORDS = {
@@ -154,6 +177,12 @@ def _resolve_coords(text: str) -> tuple[float, float] | None:
 
 @with_retry(max_retries=1, base_delay=2)
 def fetch_news():
+    if not news_fetch_enabled():
+        logger.debug("News fetch skipped; unset NEWS_ENABLED=false to re-enable")
+        with _data_lock:
+            latest_data["news"] = []
+        _mark_fresh("news")
+        return
     from services.news_feed_config import get_feeds
     feed_config = get_feeds()
     feeds = {f["name"]: f["url"] for f in feed_config}
@@ -178,6 +207,17 @@ def fetch_news():
         if not feed:
             continue
         for entry in feed.entries[:5]:
+            # Drop articles older than the max-age threshold so the
+            # threat feed doesn't show stale stories across cycles.
+            pp = entry.get("published_parsed")
+            if pp:
+                try:
+                    entry_epoch = calendar.timegm(pp)
+                    if time.time() - entry_epoch > _MAX_ARTICLE_AGE_SECS:
+                        continue
+                except (TypeError, ValueError, OverflowError):
+                    pass  # unparseable date — keep the article
+
             title = entry.get('title', '')
             summary = entry.get('summary', '')
 

backend/services/fetchers/nuforc_enrichment.py

@@ -0,0 +1,376 @@
+"""NUFORC Enrichment — downloads the Hugging Face NUFORC dataset and builds
+a compact spatial+temporal index for enriching tilequery hits with shape,
+duration, city, and summary text.
+
+The full CSV (~170 MB) is streamed once and processed into a lightweight JSON
+cache (~1-3 MB) stored at ``backend/data/nuforc_enrichment.json``.  Subsequent
+startups load from cache until it expires (30 days).
+
+Index structure::
+
+    {
+        "built": "2026-04-08T12:00:00",
+        "count": 12345,
+        "by_state": {
+            "AZ": [
+                {"d": "2024-01-15", "city": "Tucson", "shape": "triangle",
+                 "dur": "5 minutes", "summary": "Bright triangular object..."},
+                ...
+            ],
+            ...
+        }
+    }
+
+Entries within each state are sorted by date descending (newest first).
+"""
+
+import csv
+import gzip
+import io
+import json
+import logging
+import os
+import re
+import threading
+import time
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from services.network_utils import fetch_with_curl
+
+logger = logging.getLogger(__name__)
+
+_DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"
+_CACHE_FILE = _DATA_DIR / "nuforc_enrichment.json"
+_CACHE_TTL_DAYS = 1  # Rebuild daily — fresh data each cycle
+
+# HuggingFace dataset — use the structured string export, not the old flat blob.
+_HF_CSV_URL = (
+    "https://huggingface.co/datasets/kcimc/NUFORC/resolve/main/nuforc_str.csv"
+)
+
+
+def nuforc_fetch_enabled() -> bool:
+    """Return True only when the operator explicitly opts into NUFORC pulls."""
+    return str(os.environ.get("NUFORC_ENABLED", "")).strip().lower() in {
+        "1",
+        "true",
+        "yes",
+        "on",
+    }
+
+# Only keep sightings from the last N years for the enrichment index
+_KEEP_YEARS = 5
+
+# ── In-memory index ────────────────────────────────────────────────────────
+_index: dict | None = None
+_index_lock = threading.Lock()
+_building = False
+
+# US state abbreviations for parsing "City, ST" locations
+_US_STATES = {
+    "AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DE", "FL", "GA",
+    "HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD",
+    "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ",
+    "NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", "RI", "SC",
+    "SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY",
+    "DC",
+}
+
+
+def _parse_location(loc: str) -> tuple[str, str]:
+    """Parse 'City, ST' or 'City, ST (explanation)' → (city, state_abbr).
+
+    Returns ('', '') if unparseable.
+    """
+    if not loc:
+        return "", ""
+    loc = re.sub(r"\s*\(.*\)\s*$", "", loc).strip()
+    parts = [p.strip() for p in loc.split(",") if p.strip()]
+    if len(parts) < 2:
+        return "", ""
+    for idx in range(len(parts) - 1):
+        candidate = parts[idx + 1].upper().strip()
+        if candidate in _US_STATES:
+            city = ", ".join(parts[: idx + 1]).strip()
+            return city, candidate
+    candidate = parts[-1].upper().strip()
+    if candidate in _US_STATES:
+        return ", ".join(parts[:-1]).strip(), candidate
+    return parts[0], ""
+
+
+def _parse_date(date_str: str) -> str:
+    """Best-effort parse NUFORC date strings → 'YYYY-MM-DD'.
+
+    Returns '' on failure.
+    """
+    if not date_str:
+        return ""
+    cleaned = str(date_str).strip()
+    cleaned = re.sub(r"\s+local$", "", cleaned, flags=re.IGNORECASE)
+    cleaned = re.sub(r"\s+utc$", "", cleaned, flags=re.IGNORECASE)
+    cleaned = cleaned.replace("T", " ")
+    for fmt in (
+        "%m/%d/%Y %H:%M",
+        "%m/%d/%Y %I:%M:%S %p",
+        "%m/%d/%Y",
+        "%Y-%m-%d %H:%M:%S",
+        "%Y-%m-%d %H:%M",
+        "%Y-%m-%d",
+    ):
+        try:
+            return datetime.strptime(cleaned, fmt).strftime("%Y-%m-%d")
+        except (ValueError, TypeError):
+            continue
+    match = re.match(r"^(\d{4}-\d{2}-\d{2})", cleaned)
+    if match:
+        return match.group(1)
+    return ""
+
+
+def _load_cache() -> dict | None:
+    """Load the on-disk cache if it exists and is fresh enough."""
+    if not _CACHE_FILE.exists():
+        return None
+    try:
+        raw = _CACHE_FILE.read_text(encoding="utf-8")
+        data = json.loads(raw)
+        built = data.get("built", "")
+        if built:
+            built_dt = datetime.fromisoformat(built)
+            if datetime.utcnow() - built_dt < timedelta(days=_CACHE_TTL_DAYS):
+                if int(data.get("count", 0) or 0) <= 0:
+                    logger.info("NUFORC enrichment: cache is fresh but empty; rebuilding")
+                    return None
+                logger.info(
+                    "NUFORC enrichment: loaded cache (%d entries, built %s)",
+                    data.get("count", 0), built,
+                )
+                return data
+            else:
+                logger.info("NUFORC enrichment: cache expired (built %s)", built)
+    except Exception as e:
+        logger.warning("NUFORC enrichment: cache load error: %s", e)
+    return None
+
+
+def _save_cache(data: dict):
+    """Persist the enrichment index to disk."""
+    try:
+        _DATA_DIR.mkdir(parents=True, exist_ok=True)
+        _CACHE_FILE.write_text(json.dumps(data, separators=(",", ":")), encoding="utf-8")
+        logger.info("NUFORC enrichment: saved cache (%d entries)", data.get("count", 0))
+    except Exception as e:
+        logger.warning("NUFORC enrichment: cache save error: %s", e)
+
+
+def _download_and_build() -> dict | None:
+    """Stream-download the HF CSV and build the enrichment index.
+
+    Returns the index dict or None on failure.
+    """
+    if not nuforc_fetch_enabled():
+        logger.debug(
+            "NUFORC enrichment skipped; set NUFORC_ENABLED=true to opt in"
+        )
+        return None
+
+    cutoff = datetime.utcnow() - timedelta(days=_KEEP_YEARS * 365)
+    cutoff_str = cutoff.strftime("%Y-%m-%d")
+
+    logger.info("NUFORC enrichment: downloading HF dataset (this may take a minute)...")
+    try:
+        resp = fetch_with_curl(_HF_CSV_URL, timeout=180, follow_redirects=True)
+        if not resp or resp.status_code != 200:
+            logger.warning(
+                "NUFORC enrichment: download failed HTTP %s",
+                getattr(resp, "status_code", "None"),
+            )
+            return None
+    except Exception as e:
+        logger.error("NUFORC enrichment: download error: %s", e)
+        return None
+
+    # Parse CSV from response text
+    by_state: dict[str, list[dict]] = {}
+    total = 0
+    kept = 0
+
+    try:
+        reader = csv.DictReader(io.StringIO(resp.text))
+        for row in reader:
+            total += 1
+            occurred = _parse_date(
+                row.get("Occurred", "")
+                or row.get("Date / Time", "")
+                or row.get("Date", "")
+            )
+            if not occurred or occurred < cutoff_str:
+                continue
+
+            city, state = _parse_location(
+                row.get("Location", "")
+                or row.get("City", "")
+                or row.get("location", "")
+            )
+            if not state:
+                continue  # can't index without state
+
+            shape = (row.get("Shape", "") or row.get("shape", "") or "").strip()
+            duration = (row.get("Duration", "") or row.get("duration", "") or "").strip()
+            summary = (
+                row.get("Summary", "")
+                or row.get("summary", "")
+                or row.get("Text", "")
+                or row.get("text", "")
+                or ""
+            ).strip()
+            if summary and len(summary) > 200:
+                summary = summary[:197] + "..."
+
+            entry = {"d": occurred, "city": city, "shape": shape}
+            if duration:
+                entry["dur"] = duration
+            if summary:
+                entry["sum"] = summary
+
+            by_state.setdefault(state, []).append(entry)
+            kept += 1
+    except Exception as e:
+        logger.error("NUFORC enrichment: CSV parse error: %s", e)
+        return None
+
+    # Sort each state's entries by date descending (newest first)
+    for st in by_state:
+        by_state[st].sort(key=lambda e: e["d"], reverse=True)
+
+    data = {
+        "built": datetime.utcnow().isoformat(),
+        "count": kept,
+        "by_state": by_state,
+    }
+    logger.info(
+        "NUFORC enrichment: built index — %d entries from %d total rows (%d states)",
+        kept, total, len(by_state),
+    )
+    return data
+
+
+def _ensure_index():
+    """Load or build the enrichment index (thread-safe, non-blocking)."""
+    global _index, _building
+
+    with _index_lock:
+        if _index is not None:
+            return
+        if _building:
+            return  # another thread is already building
+        _building = True
+
+    # Try loading from disk first
+    cached = _load_cache()
+    if cached:
+        with _index_lock:
+            _index = cached
+            _building = False
+        return
+
+    # Download and build in background so we don't block startup
+    def _build():
+        global _index, _building
+        try:
+            result = _download_and_build()
+            if result:
+                _save_cache(result)
+                with _index_lock:
+                    _index = result
+            else:
+                logger.warning("NUFORC enrichment: build failed, enrichment unavailable")
+        finally:
+            with _index_lock:
+                _building = False
+
+    thread = threading.Thread(target=_build, name="nuforc-enrichment", daemon=True)
+    thread.start()
+
+
+def refresh_enrichment_index():
+    """Force-rebuild the enrichment index.  Called by the daily cron job.
+
+    Downloads the latest HF CSV, rebuilds the in-memory + disk cache.
+    Runs synchronously (meant to be called from a background thread).
+    """
+    global _index
+    logger.info("NUFORC enrichment: daily refresh starting...")
+    result = _download_and_build()
+    if result:
+        _save_cache(result)
+        with _index_lock:
+            _index = result
+        logger.info("NUFORC enrichment: daily refresh complete (%d entries)", result.get("count", 0))
+    else:
+        logger.warning("NUFORC enrichment: daily refresh failed, keeping stale index")
+
+
+def enrich_sighting(state: str, from_date: str, to_date: str) -> dict:
+    """Look up enrichment data for a tilequery hit.
+
+    Args:
+        state: 2-letter US state code (from reverse geocode)
+        from_date: earliest sighting date (YYYY-MM-DD)
+        to_date: latest sighting date (YYYY-MM-DD)
+
+    Returns:
+        Dict with optional keys: city, shape, duration, summary.
+        Empty dict if no match found.
+    """
+    _ensure_index()
+
+    with _index_lock:
+        idx = _index
+
+    if not idx or not state:
+        return {}
+
+    entries = idx.get("by_state", {}).get(state, [])
+    if not entries:
+        return {}
+
+    # Find the best match by date proximity
+    target = to_date or from_date
+    if not target:
+        # No date filter — just return the most recent entry for this state
+        e = entries[0]
+    else:
+        best = None
+        best_dist = 999999
+        for e in entries:
+            # Simple string distance on dates (YYYY-MM-DD sorts lexicographically)
+            try:
+                t = datetime.strptime(target, "%Y-%m-%d")
+                d = datetime.strptime(e["d"], "%Y-%m-%d")
+                dist = abs((t - d).days)
+            except (ValueError, TypeError):
+                continue
+            if dist < best_dist:
+                best_dist = dist
+                best = e
+            if dist == 0:
+                break  # exact date match
+
+        if best is None or best_dist > 90:
+            return {}  # no match within 3 months
+        e = best
+
+    result = {}
+    if e.get("city"):
+        result["city"] = e["city"]
+    if e.get("shape"):
+        result["shape"] = e["shape"]
+        result["shape_raw"] = e["shape"]
+    if e.get("dur"):
+        result["duration"] = e["dur"]
+    if e.get("sum"):
+        result["summary"] = e["sum"]
+    return result

backend/services/fetchers/prediction_markets.py

@@ -8,14 +8,43 @@ full metadata (volume, end dates, descriptions, source badges).
 import json
 import logging
 import math
+import os
+import threading
+import time
+from urllib.parse import urlencode
 from cachetools import TTLCache, cached
 
 logger = logging.getLogger("services.data_fetcher")
 
-_market_cache = TTLCache(maxsize=1, ttl=60)  # 60-second TTL — markets change fast
-
 # Delta tracking: {market_title: previous_consensus_pct}
 _prev_probabilities: dict[str, float] = {}
+_market_cache = TTLCache(maxsize=1, ttl=300)
+_POLYMARKET_PAGE_DELAY_S = float(os.environ.get("MESH_POLYMARKET_PAGE_DELAY_S", "0.02"))
+_KALSHI_PAGE_DELAY_S = float(os.environ.get("MESH_KALSHI_PAGE_DELAY_S", "0.08"))
+_provider_pace_lock = threading.Lock()
+_provider_last_request_at: dict[str, float] = {}
+
+
+def prediction_markets_fetch_enabled() -> bool:
+    """Return True only when the operator explicitly opts into Polymarket/Kalshi pulls."""
+    return str(os.environ.get("PREDICTION_MARKETS_ENABLED", "")).strip().lower() in {
+        "1",
+        "true",
+        "yes",
+        "on",
+    }
+
+
+def _pace_provider(provider: str, min_interval_s: float) -> None:
+    if min_interval_s <= 0:
+        return
+    with _provider_pace_lock:
+        now = time.monotonic()
+        wait_s = min_interval_s - (now - _provider_last_request_at.get(provider, 0.0))
+        if wait_s > 0:
+            time.sleep(wait_s)
+            now = time.monotonic()
+        _provider_last_request_at[provider] = now
 
 
 def _finite_or_none(value):
@@ -28,7 +57,7 @@ def _finite_or_none(value):
 # ---------------------------------------------------------------------------
 # Category classification
 # ---------------------------------------------------------------------------
-CATEGORIES = ["POLITICS", "CONFLICT", "NEWS", "FINANCE", "CRYPTO"]
+CATEGORIES = ["POLITICS", "CONFLICT", "NEWS", "FINANCE", "CRYPTO", "SPORTS"]
 
 _KALSHI_CATEGORY_MAP = {
     "Politics": "POLITICS",
@@ -38,7 +67,7 @@ _KALSHI_CATEGORY_MAP = {
     "Tech": "FINANCE",
     "Science": "NEWS",
     "Climate and Weather": "NEWS",
-    "Sports": "NEWS",
+    "Sports": "SPORTS",
     "Culture": "NEWS",
 }
 
@@ -62,7 +91,14 @@ _TAG_CATEGORY_MAP = {
     "Ethereum": "CRYPTO",
     "AI": "NEWS",
     "Science": "NEWS",
-    "Sports": "NEWS",
+    "Sports": "SPORTS",
+    "NBA": "SPORTS",
+    "NFL": "SPORTS",
+    "MLB": "SPORTS",
+    "NHL": "SPORTS",
+    "Soccer": "SPORTS",
+    "Tennis": "SPORTS",
+    "Golf": "SPORTS",
     "Culture": "NEWS",
     "Entertainment": "NEWS",
     "Tech": "FINANCE",
@@ -152,6 +188,26 @@ _KEYWORD_CATEGORIES = {
         "market cap",
         "revenue",
     ],
+    "SPORTS": [
+        "nba",
+        "nfl",
+        "mlb",
+        "nhl",
+        "wnba",
+        "soccer",
+        "football",
+        "basketball",
+        "baseball",
+        "hockey",
+        "ufc",
+        "mma",
+        "tennis",
+        "golf",
+        "championship",
+        "playoffs",
+        "world cup",
+        "super bowl",
+    ],
 }
 
 
@@ -177,21 +233,186 @@ def _classify_category(title: str, poly_tags: list[str], kalshi_category: str) -
     return "NEWS"
 
 
+def _polymarket_event_to_entry(ev: dict) -> dict | None:
+    title = ev.get("title", "")
+    if not title:
+        return None
+
+    markets = ev.get("markets", [])
+    best_pct = None
+    total_volume = 0
+    outcomes = []
+    for m in markets:
+        raw_op = m.get("outcomePrices")
+        price = None
+        try:
+            op = json.loads(raw_op) if isinstance(raw_op, str) else raw_op
+            if isinstance(op, list) and len(op) >= 1:
+                price = _finite_or_none(op[0])
+        except (json.JSONDecodeError, ValueError, TypeError):
+            pass
+        if price is None:
+            price = _finite_or_none(m.get("lastTradePrice") or m.get("bestBid"))
+        pct = None
+        if price is not None:
+            try:
+                pct = round(price * 100, 1)
+                if best_pct is None or pct > best_pct:
+                    best_pct = pct
+            except (ValueError, TypeError):
+                pass
+        volume = _finite_or_none(m.get("volume", 0) or 0)
+        if volume is not None:
+            total_volume += volume
+        oname = m.get("groupItemTitle") or ""
+        if oname and pct is not None:
+            outcomes.append({"name": oname, "pct": pct})
+
+    if len(outcomes) > 2:
+        outcomes.sort(key=lambda x: x["pct"], reverse=True)
+    else:
+        outcomes = []
+
+    tag_labels = [t.get("label", "") for t in ev.get("tags", []) if t.get("label")]
+    return {
+        "title": title,
+        "source": "polymarket",
+        "pct": best_pct,
+        "slug": ev.get("slug", ""),
+        "description": ev.get("description") or "",
+        "end_date": ev.get("endDate"),
+        "volume": round(total_volume, 2),
+        "volume_24h": round(_finite_or_none(ev.get("volume24hr", 0) or 0) or 0, 2),
+        "tags": tag_labels,
+        "outcomes": outcomes,
+    }
+
+
+def _kalshi_market_pct(m: dict) -> float | None:
+    bid = _finite_or_none(m.get("yes_bid_dollars"))
+    ask = _finite_or_none(m.get("yes_ask_dollars"))
+    last = _finite_or_none(m.get("last_price_dollars"))
+    if bid is not None and ask is not None and ask >= bid:
+        return round(((bid + ask) / 2) * 100, 1)
+    if last is not None:
+        return round(last * 100, 1)
+    cents = _finite_or_none(m.get("yes_price") or m.get("last_price"))
+    if cents is None:
+        return None
+    return round(cents * 100, 1) if cents <= 1 else round(cents, 1)
+
+
+def _kalshi_market_volume(m: dict) -> float:
+    for key in ("volume_24h_fp", "volume_fp", "dollar_volume", "volume"):
+        value = _finite_or_none(m.get(key))
+        if value is not None:
+            return value
+    return 0
+
+
+def _kalshi_market_category(m: dict) -> str:
+    text = " ".join(
+        str(m.get(k, "") or "")
+        for k in ("ticker", "event_ticker", "mve_collection_ticker", "title", "yes_sub_title", "no_sub_title")
+    ).lower()
+    if any(token in text for token in ("sports", "xnba", "xnfl", "xmlb", "xnhl", "soccer", "tennis", "golf")):
+        return "Sports"
+    return str(m.get("category", "") or "")
+
+
+def _kalshi_event_to_entry(ev: dict, markets: list[dict] | None = None) -> dict | None:
+    title = ev.get("title", "")
+    if not title:
+        return None
+
+    markets = markets or ev.get("markets", []) or []
+    best_pct = None
+    total_volume = 0.0
+    close_dates = []
+    outcomes = []
+    first_ticker = ""
+    descriptions = []
+    for m in markets:
+        first_ticker = first_ticker or m.get("ticker", "")
+        pct = _kalshi_market_pct(m)
+        if pct is not None:
+            if best_pct is None or pct > best_pct:
+                best_pct = pct
+            oname = m.get("yes_sub_title") or m.get("sub_title") or m.get("title") or ""
+            if oname and oname != title:
+                outcomes.append({"name": oname, "pct": pct})
+        total_volume += _kalshi_market_volume(m)
+        cd = m.get("close_time") or m.get("close_date") or m.get("expiration_time")
+        if cd:
+            close_dates.append(cd)
+        desc = (m.get("rules_primary") or m.get("rules_secondary") or "").strip()
+        if desc:
+            descriptions.append(desc)
+
+    if len(outcomes) > 2:
+        outcomes.sort(key=lambda x: x["pct"], reverse=True)
+    else:
+        outcomes = []
+
+    desc = (ev.get("settle_details") or ev.get("underlying") or "").strip()
+    if not desc and descriptions:
+        desc = descriptions[0]
+
+    return {
+        "title": title,
+        "source": "kalshi",
+        "pct": best_pct,
+        "ticker": first_ticker or ev.get("event_ticker", "") or ev.get("ticker", ""),
+        "description": desc,
+        "sub_title": ev.get("sub_title", ""),
+        "end_date": max(close_dates) if close_dates else None,
+        "volume": round(total_volume, 2),
+        "category": ev.get("category", ""),
+        "outcomes": outcomes,
+    }
+
+
+def _kalshi_market_to_entry(m: dict) -> dict | None:
+    title = m.get("title") or m.get("yes_sub_title") or ""
+    if not title:
+        return None
+    pct = _kalshi_market_pct(m)
+    volume = _kalshi_market_volume(m)
+    desc = (m.get("rules_primary") or m.get("rules_secondary") or "").strip()
+    end_date = m.get("close_time") or m.get("expiration_time") or m.get("expected_expiration_time")
+    return {
+        "title": title,
+        "source": "kalshi",
+        "pct": pct,
+        "ticker": m.get("ticker", "") or m.get("event_ticker", ""),
+        "description": desc,
+        "sub_title": m.get("subtitle", ""),
+        "end_date": end_date,
+        "volume": round(volume, 2),
+        "category": _kalshi_market_category(m),
+        "outcomes": [],
+    }
+
+
 # ---------------------------------------------------------------------------
 # Polymarket
 # ---------------------------------------------------------------------------
 def _fetch_polymarket_events() -> list[dict]:
     """Fetch active events from Polymarket Gamma API (no auth required).
 
-    Fetches up to 500 events (multiple pages) for better search coverage.
+    Fetches paginated active events, bounded by MESH_POLYMARKET_MAX_EVENTS
+    so boot-time refresh does not become unbounded.
     """
     from services.network_utils import fetch_with_curl
 
     all_events = []
-    for offset in range(0, 500, 100):
+    page_size = 250
+    max_events = int(os.environ.get("MESH_POLYMARKET_MAX_EVENTS", "5000"))
+    for offset in range(0, max_events, page_size):
         try:
+            _pace_provider("polymarket", _POLYMARKET_PAGE_DELAY_S)
             resp = fetch_with_curl(
-                f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=100&offset={offset}",
+                f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit={page_size}&offset={offset}",
                 timeout=15,
             )
             if not resp or resp.status_code != 200:
@@ -200,6 +421,8 @@ def _fetch_polymarket_events() -> list[dict]:
             if not isinstance(page, list) or not page:
                 break
             all_events.extend(page)
+            if len(page) < page_size:
+                break
         except Exception as e:
             logger.warning(f"Polymarket page offset={offset} error: {e}")
             break
@@ -286,6 +509,42 @@ def _fetch_kalshi_events() -> list[dict]:
     """Fetch active events from Kalshi public API (no auth required)."""
     from services.network_utils import fetch_with_curl
 
+    try:
+        max_events = int(os.environ.get("MESH_KALSHI_MAX_EVENTS", "2000"))
+        page_size = 200
+        markets = []
+        cursor = ""
+        while len(markets) < max_events:
+            params = {"status": "open", "limit": str(page_size)}
+            if cursor:
+                params["cursor"] = cursor
+            _pace_provider("kalshi", _KALSHI_PAGE_DELAY_S)
+            resp = fetch_with_curl(
+                f"https://api.elections.kalshi.com/trade-api/v2/markets?{urlencode(params)}",
+                timeout=15,
+            )
+            if not resp or resp.status_code != 200:
+                break
+            data = resp.json()
+            page = data.get("markets", []) if isinstance(data, dict) else []
+            if not page:
+                break
+            markets.extend(page)
+            cursor = data.get("cursor") or ""
+            if not cursor or len(page) < page_size:
+                break
+
+        results = []
+        for market in markets:
+            entry = _kalshi_market_to_entry(market)
+            if entry:
+                results.append(entry)
+        if results:
+            logger.info(f"Kalshi: fetched {len(results)} active events from v2")
+            return results
+    except Exception as e:
+        logger.warning(f"Kalshi v2 fetch error, falling back to legacy v1: {e}")
+
     try:
         resp = fetch_with_curl(
             "https://api.elections.kalshi.com/v1/events?status=open&limit=100",
@@ -506,6 +765,16 @@ def fetch_prediction_markets():
     from services.fetchers._store import latest_data, _data_lock, _mark_fresh
     global _prev_probabilities
 
+    if not prediction_markets_fetch_enabled():
+        logger.debug(
+            "Prediction markets fetch skipped; set "
+            "PREDICTION_MARKETS_ENABLED=true to opt in"
+        )
+        with _data_lock:
+            latest_data["prediction_markets"] = []
+        _mark_fresh("prediction_markets")
+        return
+
     markets = fetch_prediction_markets_raw()
 
     # Compute probability deltas vs previous fetch
@@ -540,11 +809,11 @@ def fetch_prediction_markets():
 # ---------------------------------------------------------------------------
 # Direct API search (not limited to cached data)
 # ---------------------------------------------------------------------------
-def search_polymarket_direct(query: str, limit: int = 20) -> list[dict]:
+def search_polymarket_direct(query: str, limit: int = 20, offset: int = 0) -> list[dict]:
     """Search Polymarket by scanning API pages for title matches.
 
-    The Gamma API has no text search parameter, so we scan cached events
-    plus additional pages until we find enough matches or exhaust the scan.
+    Prefer Polymarket's public search endpoint, then fall back to scanning
+    Gamma event pages if search is unavailable.
     """
     from services.network_utils import fetch_with_curl
 
@@ -552,11 +821,53 @@ def search_polymarket_direct(query: str, limit: int = 20) -> list[dict]:
     q_words = set(q_lower.split())
     results = []
 
+    try:
+        params = urlencode({"q": query, "limit": str(limit), "offset": str(max(0, offset))})
+        _pace_provider("polymarket", _POLYMARKET_PAGE_DELAY_S)
+        resp = fetch_with_curl(
+            f"https://gamma-api.polymarket.com/public-search?{params}",
+            timeout=15,
+        )
+        if resp and resp.status_code == 200:
+            data = resp.json()
+            events = data.get("events", []) if isinstance(data, dict) else []
+            for ev in events:
+                if ev.get("closed") or ev.get("active") is False:
+                    continue
+                entry = _polymarket_event_to_entry(ev)
+                if not entry:
+                    continue
+                category = _classify_category(entry["title"], entry.get("tags", []), "")
+                pct = _finite_or_none(entry.get("pct"))
+                sources = [{"name": "POLY", "pct": pct}] if pct is not None else []
+                results.append(
+                    {
+                        "title": entry["title"],
+                        "polymarket_pct": pct,
+                        "kalshi_pct": None,
+                        "consensus_pct": pct,
+                        "description": entry.get("description", ""),
+                        "end_date": entry.get("end_date"),
+                        "volume": entry.get("volume", 0),
+                        "volume_24h": entry.get("volume_24h", 0),
+                        "kalshi_volume": 0,
+                        "category": category,
+                        "sources": sources,
+                        "slug": entry.get("slug", ""),
+                        "outcomes": entry.get("outcomes", []),
+                    }
+                )
+            logger.info(f"Polymarket search '{query}': {len(results)} results via public-search")
+            return results[:limit]
+    except Exception as e:
+        logger.warning(f"Polymarket public-search '{query}' error: {e}")
+
     # Scan up to 2000 events (10 pages of 200) looking for title matches
-    for offset in range(0, 2000, 200):
+    for scan_offset in range(0, 3000, 200):
         try:
+            _pace_provider("polymarket", _POLYMARKET_PAGE_DELAY_S)
             resp = fetch_with_curl(
-                f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=200&offset={offset}",
+                f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=200&offset={scan_offset}",
                 timeout=15,
             )
             if not resp or resp.status_code != 200:
@@ -637,11 +948,168 @@ def search_polymarket_direct(query: str, limit: int = 20) -> list[dict]:
                     }
                 )
             # Stop scanning if we have enough results
-            if len(results) >= limit:
+            if len(results) >= offset + limit:
                 break
         except Exception as e:
-            logger.warning(f"Polymarket search scan offset={offset} error: {e}")
+            logger.warning(f"Polymarket search scan offset={scan_offset} error: {e}")
             break
 
     logger.info(f"Polymarket search '{query}': {len(results)} results (scanned API)")
-    return results[:limit]
+    return results[offset : offset + limit]
+
+
+def search_kalshi_direct(query: str, limit: int = 20, offset: int = 0) -> list[dict]:
+    """Search Kalshi events by scanning API pages for title matches."""
+    from services.network_utils import fetch_with_curl
+
+    q_lower = query.lower()
+    q_words = set(q_lower.split())
+    results = []
+
+    try:
+        max_scan = int(os.environ.get("MESH_KALSHI_SEARCH_SCAN_EVENTS", "1200"))
+        page_size = 200
+        cursor = ""
+        scanned = 0
+        while scanned < max_scan and len(results) < offset + limit:
+            params = {"status": "open", "limit": str(page_size)}
+            if cursor:
+                params["cursor"] = cursor
+            _pace_provider("kalshi", _KALSHI_PAGE_DELAY_S)
+            resp = fetch_with_curl(
+                f"https://api.elections.kalshi.com/trade-api/v2/markets?{urlencode(params)}",
+                timeout=15,
+            )
+            if not resp or resp.status_code != 200:
+                break
+            data = resp.json()
+            markets = data.get("markets", []) if isinstance(data, dict) else []
+            if not markets:
+                break
+            scanned += len(markets)
+            for market in markets:
+                haystack = " ".join(
+                    str(market.get(k, "") or "")
+                    for k in ("title", "yes_sub_title", "no_sub_title", "event_ticker", "ticker")
+                ).lower()
+                if q_lower not in haystack and not any(w in haystack for w in q_words):
+                    continue
+                entry = _kalshi_market_to_entry(market)
+                if not entry:
+                    continue
+                pct = _finite_or_none(entry.get("pct"))
+                sources = [{"name": "KALSHI", "pct": pct}] if pct is not None else []
+                category = _classify_category(entry["title"], [], entry.get("category", ""))
+                results.append({
+                    "title": entry["title"],
+                    "polymarket_pct": None,
+                    "kalshi_pct": pct,
+                    "consensus_pct": pct,
+                    "description": entry.get("description", ""),
+                    "end_date": entry.get("end_date"),
+                    "volume": 0,
+                    "volume_24h": 0,
+                    "kalshi_volume": entry.get("volume", 0),
+                    "category": category,
+                    "sources": sources,
+                    "slug": "",
+                    "kalshi_ticker": entry.get("ticker", ""),
+                    "outcomes": entry.get("outcomes", []),
+                })
+                if len(results) >= offset + limit:
+                    break
+            cursor = data.get("cursor") or ""
+            if not cursor or len(markets) < page_size:
+                break
+        if results:
+            logger.info(f"Kalshi search '{query}': {len(results)} results via v2 scan")
+            return results[offset : offset + limit]
+    except Exception as e:
+        logger.warning(f"Kalshi v2 search '{query}' error, falling back to legacy v1: {e}")
+
+    try:
+        resp = fetch_with_curl(
+            "https://api.elections.kalshi.com/v1/events?status=open&limit=200",
+            timeout=15,
+        )
+        if not resp or resp.status_code != 200:
+            return []
+        data = resp.json()
+        events = data.get("events", []) if isinstance(data, dict) else []
+
+        for ev in events:
+            title = ev.get("title", "")
+            if not title:
+                continue
+            title_lower = title.lower()
+            if q_lower not in title_lower and not any(w in title_lower for w in q_words):
+                continue
+
+            markets = ev.get("markets", [])
+            best_pct = None
+            total_volume = 0
+            close_dates = []
+            outcomes = []
+            for m in markets:
+                price = m.get("yes_price") or m.get("last_price")
+                pct = None
+                if price is not None:
+                    try:
+                        price = _finite_or_none(price)
+                        if price is None:
+                            raise ValueError("non-finite")
+                        pct = round(price, 1)
+                        if pct <= 1:
+                            pct = round(pct * 100, 1)
+                        if best_pct is None or pct > best_pct:
+                            best_pct = pct
+                    except (ValueError, TypeError):
+                        pass
+                try:
+                    volume = _finite_or_none(
+                        m.get("dollar_volume", 0) or m.get("volume", 0) or 0
+                    )
+                    if volume is not None:
+                        total_volume += int(volume)
+                except (ValueError, TypeError):
+                    pass
+                cd = m.get("close_date")
+                if cd:
+                    close_dates.append(cd)
+                oname = m.get("title") or m.get("subtitle", "")
+                if oname and pct is not None:
+                    outcomes.append({"name": oname, "pct": pct})
+            if len(outcomes) > 2:
+                outcomes.sort(key=lambda x: x["pct"], reverse=True)
+            else:
+                outcomes = []
+
+            desc = (ev.get("settle_details") or ev.get("underlying") or "").strip()
+            category = _classify_category(title, [], ev.get("category", ""))
+            sources = []
+            if best_pct is not None:
+                sources.append({"name": "KALSHI", "pct": best_pct})
+
+            results.append({
+                "title": title,
+                "polymarket_pct": None,
+                "kalshi_pct": best_pct,
+                "consensus_pct": best_pct,
+                "description": desc,
+                "end_date": max(close_dates) if close_dates else None,
+                "volume": total_volume,
+                "volume_24h": 0,
+                "kalshi_volume": total_volume,
+                "category": category,
+                "sources": sources,
+                "slug": "",
+                "kalshi_ticker": ev.get("ticker", ""),
+                "outcomes": outcomes,
+            })
+            if len(results) >= offset + limit:
+                break
+    except Exception as e:
+        logger.warning(f"Kalshi search '{query}' error: {e}")
+
+    logger.info(f"Kalshi search '{query}': {len(results)} results")
+    return results[offset : offset + limit]

backend/services/fetchers/route_database.py

@@ -0,0 +1,168 @@
+"""Static route + airport database loaded from vrs-standing-data.adsb.lol.
+
+Replaces the per-batch /api/0/routeset POST with a single daily bulk download.
+Routes change ~weekly when airlines update schedules, so a 24h refresh cadence
+is far more than sufficient and removes ~all live-API pressure on adsb.lol.
+"""
+
+from __future__ import annotations
+
+import csv
+import gzip
+import io
+import logging
+import threading
+import time
+from typing import Any
+
+import requests
+
+
+
+def _route_db_user_agent() -> str:
+    from services.network_utils import outbound_user_agent
+    return outbound_user_agent("route-database")
+
+logger = logging.getLogger(__name__)
+
+_ROUTES_URL = "https://vrs-standing-data.adsb.lol/routes.csv.gz"
+_AIRPORTS_URL = "https://vrs-standing-data.adsb.lol/airports.csv.gz"
+_REFRESH_INTERVAL_S = 5 * 24 * 3600
+_HTTP_TIMEOUT_S = 60
+
+from services.network_utils import DEFAULT_USER_AGENT as _USER_AGENT
+
+_lock = threading.RLock()
+_routes_by_callsign: dict[str, dict[str, Any]] = {}
+_airports_by_icao: dict[str, dict[str, Any]] = {}
+_last_refresh = 0.0
+_refresh_in_progress = False
+
+
+def _fetch_csv_gz(url: str) -> list[dict[str, str]]:
+    response = requests.get(
+        url,
+        timeout=_HTTP_TIMEOUT_S,
+        headers={"User-Agent": _route_db_user_agent(), "Accept-Encoding": "gzip"},
+    )
+    response.raise_for_status()
+    text = gzip.decompress(response.content).decode("utf-8-sig")
+    return list(csv.DictReader(io.StringIO(text)))
+
+
+def _build_route_index(rows: list[dict[str, str]]) -> dict[str, dict[str, Any]]:
+    index: dict[str, dict[str, Any]] = {}
+    for row in rows:
+        callsign = (row.get("Callsign") or "").strip().upper()
+        airport_codes = (row.get("AirportCodes") or "").strip()
+        if not callsign or not airport_codes:
+            continue
+        icaos = [c.strip() for c in airport_codes.split("-") if c.strip()]
+        if len(icaos) < 2:
+            continue
+        index[callsign] = {
+            "airline_code": (row.get("AirlineCode") or "").strip(),
+            "airport_codes": airport_codes,
+            "airport_icaos": icaos,
+        }
+    return index
+
+
+def _build_airport_index(rows: list[dict[str, str]]) -> dict[str, dict[str, Any]]:
+    index: dict[str, dict[str, Any]] = {}
+    for row in rows:
+        icao = (row.get("ICAO") or "").strip().upper()
+        if not icao:
+            continue
+        try:
+            lat = float(row.get("Latitude") or 0)
+            lon = float(row.get("Longitude") or 0)
+        except (TypeError, ValueError):
+            continue
+        index[icao] = {
+            "name": (row.get("Name") or "").strip(),
+            "iata": (row.get("IATA") or "").strip(),
+            "country": (row.get("CountryISO2") or "").strip(),
+            "lat": lat,
+            "lon": lon,
+        }
+    return index
+
+
+def refresh_route_database(force: bool = False) -> bool:
+    """Pull routes.csv.gz + airports.csv.gz and rebuild the in-memory indexes.
+
+    Returns True if a refresh was performed (success or attempted), False if
+    skipped because the cache is still fresh or another refresh is in flight.
+    """
+    global _last_refresh, _refresh_in_progress
+
+    now = time.time()
+    with _lock:
+        if _refresh_in_progress:
+            return False
+        if not force and (now - _last_refresh) < _REFRESH_INTERVAL_S and _routes_by_callsign:
+            return False
+        _refresh_in_progress = True
+
+    try:
+        started = time.time()
+        airport_rows = _fetch_csv_gz(_AIRPORTS_URL)
+        route_rows = _fetch_csv_gz(_ROUTES_URL)
+        airports = _build_airport_index(airport_rows)
+        routes = _build_route_index(route_rows)
+        with _lock:
+            _airports_by_icao.clear()
+            _airports_by_icao.update(airports)
+            _routes_by_callsign.clear()
+            _routes_by_callsign.update(routes)
+            _last_refresh = time.time()
+        logger.info(
+            "route database refreshed in %.1fs: %d routes, %d airports",
+            time.time() - started,
+            len(routes),
+            len(airports),
+        )
+        return True
+    except (requests.RequestException, OSError, ValueError) as exc:
+        logger.warning("route database refresh failed: %s", exc)
+        return True
+    finally:
+        with _lock:
+            _refresh_in_progress = False
+
+
+def lookup_route(callsign: str) -> dict[str, Any] | None:
+    """Resolve a callsign to {orig_name, dest_name, orig_loc, dest_loc} or None.
+
+    Matches the shape produced by the legacy fetch_routes_background cache so
+    the caller in flights.py can be a drop-in replacement.
+    """
+    key = (callsign or "").strip().upper()
+    if not key:
+        return None
+    with _lock:
+        route = _routes_by_callsign.get(key)
+        if not route:
+            return None
+        icaos = route["airport_icaos"]
+        orig = _airports_by_icao.get(icaos[0].upper())
+        dest = _airports_by_icao.get(icaos[-1].upper())
+    if not orig or not dest:
+        return None
+    return {
+        "orig_name": f"{orig['iata']}: {orig['name']}" if orig["iata"] else orig["name"],
+        "dest_name": f"{dest['iata']}: {dest['name']}" if dest["iata"] else dest["name"],
+        "orig_loc": [orig["lon"], orig["lat"]],
+        "dest_loc": [dest["lon"], dest["lat"]],
+    }
+
+
+def route_database_status() -> dict[str, Any]:
+    with _lock:
+        return {
+            "last_refresh": _last_refresh,
+            "routes": len(_routes_by_callsign),
+            "airports": len(_airports_by_icao),
+            "in_progress": _refresh_in_progress,
+        }

backend/services/fetchers/sar_catalog.py

@@ -0,0 +1,74 @@
+"""SAR catalog fetcher (Mode A — default-on, free, no account).
+
+Hits ASF Search every hour for Sentinel-1 scenes that touched any of
+the operator-defined AOIs in the last ~36h.  Pure metadata, no
+downloads.
+
+Result is written to ``latest_data["sar_scenes"]`` and a per-AOI
+coverage summary to ``latest_data["sar_aoi_coverage"]``.
+"""
+
+from __future__ import annotations
+
+import logging
+
+from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
+from services.fetchers.retry import with_retry
+from services.sar.sar_aoi import load_aois
+from services.sar.sar_catalog_client import estimate_next_pass, search_scenes_for_aoi
+from services.sar.sar_config import catalog_enabled
+
+logger = logging.getLogger(__name__)
+
+
+@with_retry(max_retries=1, base_delay=2)
+def fetch_sar_catalog() -> None:
+    """Refresh the SAR scene catalog for all configured AOIs."""
+    if not catalog_enabled():
+        return
+    if not is_any_active("sar"):
+        return
+    aois = load_aois()
+    if not aois:
+        logger.debug("SAR catalog: no AOIs configured")
+        return
+
+    all_scenes: list[dict] = []
+    coverage: list[dict] = []
+    for aoi in aois:
+        try:
+            scenes = search_scenes_for_aoi(aoi)
+        except (ConnectionError, TimeoutError, OSError, ValueError) as exc:
+            logger.debug("SAR catalog %s: %s", aoi.id, exc)
+            scenes = []
+        scene_dicts = [s.to_dict() for s in scenes]
+        all_scenes.extend(scene_dicts)
+        next_pass = estimate_next_pass(scenes)
+        coverage.append(
+            {
+                "aoi_id": aoi.id,
+                "aoi_name": aoi.name,
+                "category": aoi.category,
+                "center_lat": aoi.center_lat,
+                "center_lon": aoi.center_lon,
+                "radius_km": aoi.radius_km,
+                "recent_scene_count": len(scene_dicts),
+                "latest_scene_time": (
+                    max((s["time"] for s in scene_dicts), default="")
+                    if scene_dicts
+                    else ""
+                ),
+                **next_pass,
+            }
+        )
+
+    with _data_lock:
+        latest_data["sar_scenes"] = all_scenes
+        latest_data["sar_aoi_coverage"] = coverage
+    if all_scenes or coverage:
+        _mark_fresh("sar_scenes", "sar_aoi_coverage")
+    logger.info(
+        "SAR catalog: %d scenes across %d AOIs",
+        len(all_scenes),
+        len(aois),
+    )

backend/services/fetchers/sar_products.py

@@ -0,0 +1,103 @@
+"""SAR pre-processed product fetcher (Mode B — opt-in, free, account needed).
+
+Pulls already-computed deformation, flood, water, and damage products
+from NASA OPERA, Copernicus EGMS, GFM, EMS, and UNOSAT.  No local DSP.
+
+Two-step opt-in: ``MESH_SAR_PRODUCTS_FETCH=allow`` AND
+``MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE=true``.  When either flag is
+unset, this fetcher logs a single startup hint and returns.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
+from services.fetchers.retry import with_retry
+from services.sar.sar_aoi import load_aois
+from services.sar.sar_config import products_fetch_enabled, products_fetch_status
+from services.sar.sar_normalize import SarAnomaly
+from services.sar.sar_products_client import (
+    fetch_egms_for_aoi,
+    fetch_ems_for_aoi,
+    fetch_gfm_for_aoi,
+    fetch_opera_for_aoi,
+    fetch_unosat_for_aoi,
+)
+from services.sar.sar_signing import emit_signed_anomaly
+
+logger = logging.getLogger(__name__)
+_LOGGED_DISABLED_HINT = False
+
+
+def _hint_disabled_once() -> None:
+    global _LOGGED_DISABLED_HINT
+    if _LOGGED_DISABLED_HINT:
+        return
+    _LOGGED_DISABLED_HINT = True
+    status = products_fetch_status()
+    missing = ", ".join(status.get("missing", [])) or "nothing"
+    logger.info(
+        "SAR Mode B (ground-change alerts) is disabled. Missing: %s. "
+        "Enable in Settings → SAR or set the env vars listed in .env.example. "
+        "Free signup: https://urs.earthdata.nasa.gov/users/new",
+        missing,
+    )
+
+
+@with_retry(max_retries=1, base_delay=3)
+def fetch_sar_products() -> None:
+    """Refresh pre-processed SAR anomalies for all configured AOIs."""
+    if not products_fetch_enabled():
+        _hint_disabled_once()
+        return
+    if not is_any_active("sar"):
+        return
+    aois = load_aois()
+    if not aois:
+        logger.debug("SAR products: no AOIs configured")
+        return
+
+    seen_ids: set[str] = set()
+    all_anomalies: list[dict[str, Any]] = []
+    publish_summary = {"signed": 0, "skipped": 0, "reasons": {}}
+
+    for aoi in aois:
+        for fetcher in (
+            fetch_opera_for_aoi,
+            fetch_egms_for_aoi,
+            fetch_gfm_for_aoi,
+            fetch_ems_for_aoi,
+            fetch_unosat_for_aoi,
+        ):
+            try:
+                anomalies: list[SarAnomaly] = fetcher(aoi) or []
+            except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as exc:
+                logger.debug("SAR %s for %s failed: %s", fetcher.__name__, aoi.id, exc)
+                anomalies = []
+            for a in anomalies:
+                if a.anomaly_id in seen_ids:
+                    continue
+                seen_ids.add(a.anomaly_id)
+                all_anomalies.append(a.to_dict())
+                status = emit_signed_anomaly(a)
+                if status.get("signed"):
+                    publish_summary["signed"] += 1
+                else:
+                    publish_summary["skipped"] += 1
+                    reason = status.get("reason", "unknown")
+                    publish_summary["reasons"][reason] = (
+                        publish_summary["reasons"].get(reason, 0) + 1
+                    )
+
+    with _data_lock:
+        latest_data["sar_anomalies"] = all_anomalies
+    if all_anomalies:
+        _mark_fresh("sar_anomalies")
+    logger.info(
+        "SAR products: %d anomalies (%d signed, %d skipped)",
+        len(all_anomalies),
+        publish_summary["signed"],
+        publish_summary["skipped"],
+    )

backend/services/fetchers/trains.py

@@ -10,6 +10,12 @@ from datetime import datetime, timezone
 from services.fetchers._store import _data_lock, _mark_fresh, latest_data
 from services.network_utils import fetch_with_curl
 
+
+
+def _trains_user_agent() -> str:
+    from services.network_utils import outbound_user_agent
+    return outbound_user_agent("trains")
+
 logger = logging.getLogger(__name__)
 
 _EARTH_RADIUS_KM = 6371.0
@@ -379,7 +385,7 @@ def _fetch_digitraffic() -> list[dict]:
             timeout=15,
             headers={
                 "Accept-Encoding": "gzip",
-                "User-Agent": "ShadowBroker-OSINT/1.0",
+                "User-Agent": _trains_user_agent(),
             },
         )
         if resp.status_code != 200:

backend/services/fetchers/usni_fleet_tracker.py

@@ -0,0 +1,457 @@
+"""USNI News Fleet & Marine Tracker — authoritative weekly carrier
+position publication.
+
+Why this exists
+---------------
+The previous carrier_tracker pipeline relied on GDELT headline matching
+(``api.gdeltproject.org``) to derive positions from text like "USS Ford
+in the Mediterranean" → centroid of "Mediterranean Sea". That was
+- low-precision (audit issue #245 — false precision from text mentions),
+- unreliable (``api.gdeltproject.org`` is sometimes unreachable from
+  certain network paths, including Docker Desktop on some Windows hosts).
+
+USNI publishes a weekly tracker that explicitly lists where every U.S.
+carrier is operating. The article body uses extremely consistent phrasing:
+
+    "The Gerald R. Ford Carrier Strike Group is operating in the Red Sea"
+    "Aircraft carrier USS George Washington (CVN-73) is in port in
+     Yokosuka, Japan."
+    "USS Dwight D. Eisenhower (CVN-69) sails down the Elizabeth River"
+
+Those are deterministic to parse. This module:
+
+  1. Pulls the WordPress RSS feeds (both site-wide and category) — the
+     site-wide feed often has fresher posts before the category feed
+     catches up, so we union them.
+  2. Picks the most recent post by parsed ``pubDate``.
+  3. For each carrier in the registry, scans the article body for a
+     "is operating in / is in port in / departed from" pattern near
+     the carrier's name.
+  4. Maps the extracted region phrase to coordinates via the carrier
+     tracker's existing REGION_COORDS.
+
+The result is a ``{hull: position_entry}`` dict that the carrier tracker
+consumes as a high-confidence source — ``position_confidence: "recent"``
+with ``position_source_at`` set to the article's actual publication
+timestamp (not ``now()``).
+
+Politeness
+----------
+We send the per-install operator handle via ``outbound_user_agent``
+(Round 7a) so USNI can rate-limit / contact the specific install if
+needed. Article-body pages return 403 to non-browser UAs (Cloudflare),
+but WordPress RSS feeds are open and serve the full article in
+``<content:encoded>`` — that's the supported path for aggregators and
+the one we use. We do not spoof browser headers.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import xml.etree.ElementTree as ET
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+from typing import Iterable
+
+from services.network_utils import fetch_with_curl, outbound_user_agent
+
+logger = logging.getLogger(__name__)
+
+_RSS_URLS: tuple[str, ...] = (
+    # Site-wide feed often has the freshest posts before the category
+    # feed catches up. We try this first.
+    "https://news.usni.org/feed",
+    # Category feed has older fleet trackers for backfill.
+    "https://news.usni.org/category/fleet-tracker/feed",
+)
+
+_RSS_NS = {"content": "http://purl.org/rss/1.0/modules/content/"}
+
+_FLEET_TRACKER_TITLE_RE = re.compile(
+    r"fleet\s+and\s+marine\s+tracker", re.IGNORECASE
+)
+
+_TAG_STRIP_RE = re.compile(r"<[^>]+>")
+_WHITESPACE_RE = re.compile(r"\s+")
+
+
+def _strip_html(html: str) -> str:
+    text = _TAG_STRIP_RE.sub(" ", html or "")
+    return _WHITESPACE_RE.sub(" ", text).strip()
+
+
+def _request_headers() -> dict[str, str]:
+    """Headers USNI's WordPress feed accepts from a legitimate aggregator.
+
+    The ``Referer`` is the category index page — that's where a real
+    feed reader navigates from. ``Accept`` declares RSS preference but
+    falls back to HTML. No browser UA spoofing.
+    """
+    return {
+        "User-Agent": outbound_user_agent("usni-fleet-tracker"),
+        "Accept": "application/rss+xml, application/xml;q=0.9, */*;q=0.1",
+        "Accept-Language": "en-US,en;q=0.5",
+        "Referer": "https://news.usni.org/category/fleet-tracker",
+    }
+
+
+def _parse_pubdate(raw: str) -> datetime | None:
+    if not raw:
+        return None
+    try:
+        dt = parsedate_to_datetime(raw)
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        return dt
+    except (TypeError, ValueError):
+        return None
+
+
+def _iter_fleet_tracker_items(rss_urls: Iterable[str]) -> list[dict]:
+    """Pull every fleet-tracker post visible across the given RSS feeds.
+
+    De-duplicates by article link. Returns a list of dicts:
+        {"title", "link", "pub_date" (datetime), "body" (plain text)}
+    """
+    items_by_link: dict[str, dict] = {}
+    for url in rss_urls:
+        try:
+            r = fetch_with_curl(url, timeout=15, headers=_request_headers())
+        except Exception as exc:
+            logger.debug("USNI RSS %s exception: %s", url, exc)
+            continue
+        if not r or r.status_code != 200 or not r.text:
+            logger.debug(
+                "USNI RSS %s returned status=%s body=%d",
+                url,
+                getattr(r, "status_code", "?"),
+                len(getattr(r, "text", "") or ""),
+            )
+            continue
+        try:
+            root = ET.fromstring(r.text)
+        except ET.ParseError as exc:
+            logger.warning("USNI RSS parse error from %s: %s", url, exc)
+            continue
+        for item in root.findall(".//item"):
+            title = (item.findtext("title") or "").strip()
+            if not _FLEET_TRACKER_TITLE_RE.search(title):
+                continue
+            link = (item.findtext("link") or "").strip()
+            if not link or link in items_by_link:
+                continue
+            pub_dt = _parse_pubdate(item.findtext("pubDate") or "")
+            body_html = (
+                item.findtext("content:encoded", default="", namespaces=_RSS_NS)
+                or item.findtext("description", default="")
+                or ""
+            )
+            items_by_link[link] = {
+                "title": title,
+                "link": link,
+                "pub_date": pub_dt,
+                "body": _strip_html(body_html),
+            }
+    return list(items_by_link.values())
+
+
+# Map USNI region phrases to keys in carrier_tracker.REGION_COORDS.
+# The carrier_tracker table already covers most named bodies of water and
+# major ports — we just need to teach this module to RECOGNIZE the
+# specific phrases USNI's editorial style uses, which sometimes spell
+# the same body of water differently.
+_USNI_REGION_ALIASES: tuple[tuple[str, str], ...] = (
+    # USNI phrase (lowercase) -> REGION_COORDS key
+    ("eastern mediterranean", "eastern mediterranean"),
+    ("western mediterranean", "western mediterranean"),
+    ("mediterranean sea", "mediterranean"),
+    ("the mediterranean", "mediterranean"),
+    ("red sea", "red sea"),
+    ("arabian sea area of responsibility", "arabian sea"),
+    ("north arabian sea", "north arabian sea"),
+    ("arabian sea", "arabian sea"),
+    ("persian gulf", "persian gulf"),
+    ("gulf of oman", "gulf of oman"),
+    ("strait of hormuz", "strait of hormuz"),
+    ("south china sea", "south china sea"),
+    ("east china sea", "east china sea"),
+    ("philippine sea", "philippine sea"),
+    ("sea of japan", "sea of japan"),
+    ("taiwan strait", "taiwan strait"),
+    ("western pacific", "western pacific"),
+    ("pacific ocean", "pacific"),
+    ("indian ocean", "indian ocean"),
+    ("north atlantic", "north atlantic"),
+    ("western atlantic", "atlantic"),
+    ("eastern atlantic", "atlantic"),
+    ("atlantic ocean", "atlantic"),
+    ("gulf of aden", "gulf of aden"),
+    ("horn of africa", "horn of africa"),
+    ("bab el-mandeb", "bab el-mandeb"),
+    ("suez canal", "suez canal"),
+    ("baltic sea", "baltic sea"),
+    ("north sea", "north sea"),
+    ("black sea", "black sea"),
+    ("south atlantic", "south atlantic"),
+    ("coral sea", "coral sea"),
+    ("gulf of mexico", "gulf of mexico"),
+    ("caribbean sea", "caribbean"),
+    ("caribbean", "caribbean"),
+    # Specific ports
+    ("naval station norfolk", "norfolk"),
+    ("norfolk naval shipyard", "newport news"),
+    ("newport news shipbuilding", "newport news"),
+    ("newport news", "newport news"),
+    # USNI tags Norfolk mentions with state suffix; match both.
+    ("norfolk, va", "norfolk"),
+    ("norfolk", "norfolk"),
+    ("naval station everett", "puget sound"),
+    ("naval base kitsap", "bremerton"),
+    ("bremerton", "bremerton"),
+    ("puget sound", "puget sound"),
+    ("naval base san diego", "san diego"),
+    ("san diego, calif", "san diego"),
+    ("san diego", "san diego"),
+    ("yokosuka, japan", "yokosuka"),
+    ("yokosuka", "yokosuka"),
+    ("pearl harbor", "pearl harbor"),
+    ("apra harbor, guam", "guam"),
+    ("guam", "guam"),
+    ("bahrain", "bahrain"),
+    ("naval station rota", "rota"),
+    ("rota, spain", "rota"),
+    ("naples, italy", "naples"),
+    # Fleets / AORs
+    ("5th fleet", "5th fleet"),
+    ("6th fleet", "6th fleet"),
+    ("7th fleet", "7th fleet"),
+    ("3rd fleet", "3rd fleet"),
+    ("2nd fleet", "2nd fleet"),
+    ("centcom", "centcom"),
+    ("indo-pacific command", "indopacom"),
+    ("eucom", "eucom"),
+    ("southcom", "southcom"),
+)
+
+
+def _resolve_region_phrase(phrase: str) -> tuple[str, str] | None:
+    """Map a USNI region phrase to a ``(canonical_key, display)`` tuple,
+    or ``None`` if we don't recognize it.
+
+    ``canonical_key`` is what ``carrier_tracker.REGION_COORDS`` keys on.
+    ``display`` is the phrase we'll show in the dossier description.
+    """
+    p = (phrase or "").lower().strip()
+    if not p:
+        return None
+    for usni_phrase, canonical in _USNI_REGION_ALIASES:
+        if usni_phrase in p:
+            return canonical, usni_phrase
+    return None
+
+
+# Operating-verb phrases USNI uses, with a capture group for the region
+# phrase that immediately follows. Each pattern is designed to swallow
+# the optional editorial filler that often appears between verb and
+# location (e.g. "returned Friday to Norfolk" — "Friday" goes in the
+# filler; "Norfolk" is the location).
+#
+# Order matters: most-specific patterns first, so e.g. "is in port in"
+# wins over the generic "is".
+_DAY_FILLER = r"(?:[A-Z][a-z]+(?:day)?,?\s+)?"  # optional "Friday" / "Monday" / etc.
+_LOC_CAPTURE = r"([A-Za-z][A-Za-z0-9\s,\.\-']{2,80})"
+
+_OPERATING_PATTERNS: tuple[re.Pattern, ...] = (
+    # "is operating in [the] {REGION}" / "is also operating in [the] {REGION}"
+    re.compile(r"\bis\s+(?:also\s+|now\s+)?operating\s+in\s+(?:the\s+)?" + _LOC_CAPTURE, re.IGNORECASE),
+    # "is conducting <stuff> in [the] {REGION}"
+    re.compile(r"\bis\s+conducting\s+[A-Za-z0-9\-\s]{2,40}\s+in\s+(?:the\s+)?" + _LOC_CAPTURE, re.IGNORECASE),
+    # "is in port in {LOCATION}"
+    re.compile(r"\bis\s+in\s+port\s+in\s+" + _LOC_CAPTURE, re.IGNORECASE),
+    # "is in port" (no location — degenerate, use carrier's homeport via separate path)
+    # → not captured here; falls through to homeport
+    # "is underway in [the] {REGION}"
+    re.compile(r"\bis\s+underway\s+in\s+(?:the\s+)?" + _LOC_CAPTURE, re.IGNORECASE),
+    # "is deployed to [the] {REGION}" / "deployed in"
+    re.compile(r"\bis\s+deployed\s+(?:to|in)\s+(?:the\s+)?" + _LOC_CAPTURE, re.IGNORECASE),
+    # "returned [Day] to {LOCATION}" / "returned [Day] from {REGION}"
+    re.compile(r"\breturned\s+" + _DAY_FILLER + r"to\s+" + _LOC_CAPTURE, re.IGNORECASE),
+    re.compile(r"\breturned\s+" + _DAY_FILLER + r"from\s+(?:the\s+)?" + _LOC_CAPTURE, re.IGNORECASE),
+    # "arrived [Day] in/at {LOCATION}"
+    re.compile(r"\barrived\s+" + _DAY_FILLER + r"(?:in|at)\s+" + _LOC_CAPTURE, re.IGNORECASE),
+    # "departed [Day] from {LOCATION}"
+    re.compile(r"\bdeparted\s+" + _DAY_FILLER + r"(?:from\s+)?" + _LOC_CAPTURE, re.IGNORECASE),
+    # "transiting [the] {REGION}" / "sailing through [the] {REGION}"
+    re.compile(r"\btransiting\s+(?:the\s+)?" + _LOC_CAPTURE, re.IGNORECASE),
+    re.compile(r"\bsailing\s+through\s+(?:the\s+)?" + _LOC_CAPTURE, re.IGNORECASE),
+    # "is homeported at {LOCATION}"
+    re.compile(r"\bis\s+homeported\s+at\s+" + _LOC_CAPTURE, re.IGNORECASE),
+)
+
+
+def _extract_region_for_carrier(
+    body: str,
+    carrier_names: list[str],
+    hull_code: str,
+) -> str | None:
+    """Return the best-guess region phrase for one carrier from the
+    article body, or None if no confident match.
+
+    Algorithm:
+      1. Find every mention of the carrier (any name variant or the hull
+         code) in the body.
+      2. For each mention, look in the ~300-char window AFTER it for any
+         of the operating-verb patterns.
+      3. Return the first hit. If a more-confident match later turns up
+         (e.g. "is operating in the X" beats "is homeported at Y"), the
+         first one in document order still wins — USNI's structure puts
+         the position-update sentence near the top of each carrier's
+         section, and the homeport mention later.
+    """
+    # Build a master mention regex covering every name variant + the hull.
+    candidates: list[str] = []
+    for name in carrier_names:
+        if name and len(name) >= 4:
+            candidates.append(re.escape(name))
+    if hull_code:
+        candidates.append(re.escape(hull_code))
+    if not candidates:
+        return None
+    mention_re = re.compile(r"\b(?:" + "|".join(candidates) + r")\b", re.IGNORECASE)
+
+    window_chars = 320
+    seen_phrases: list[str] = []
+    for mention in mention_re.finditer(body):
+        end = mention.end()
+        window = body[end : end + window_chars]
+        # Cut window at the next sentence break for tighter context.
+        # (We use the LAST period within the window so "Norfolk, Va." isn't
+        # confused for a sentence end — USNI uses ", Va." prolifically.)
+        # Sentence break candidates: ". " followed by uppercase OR newline.
+        sent_break = re.search(r"[\.!?]\s+[A-Z]", window)
+        if sent_break:
+            window = window[: sent_break.start() + 1]
+        # Try patterns in priority order.
+        for pat in _OPERATING_PATTERNS:
+            m = pat.search(window)
+            if not m:
+                continue
+            phrase = m.group(1).strip().rstrip(",.;: ")
+            if not phrase:
+                continue
+            # Strip trailing editorial filler — USNI often writes
+            # "Norfolk, Va., according to ship spotters" or
+            # "Yokosuka, Japan, according to..."
+            phrase = re.split(
+                r",\s+(?:according|as of|for|while|where|in support|in the)",
+                phrase,
+                maxsplit=1,
+            )[0].strip()
+            seen_phrases.append(phrase)
+            return phrase
+    return seen_phrases[0] if seen_phrases else None
+
+
+def fetch_latest_fleet_tracker_positions(
+    carrier_registry: dict | None = None,
+    region_coords: dict | None = None,
+) -> dict[str, dict]:
+    """Return ``{hull: position_entry}`` for the latest USNI fleet tracker.
+
+    Entries look like::
+
+        {
+          "lat": 18.0, "lng": 39.5, "heading": 0,
+          "desc": "Red Sea (USNI May 18, 2026)",
+          "source": "USNI News Fleet & Marine Tracker (May 18, 2026)",
+          "source_url": "https://news.usni.org/2026/05/18/...",
+          "position_source_at": "2026-05-18T18:58:44+00:00",
+          "position_confidence": "recent",
+        }
+
+    Carriers whose section can't be parsed (e.g. an off-week with no
+    mention) are simply absent from the result — the caller keeps
+    whatever position they had before.
+
+    ``carrier_registry`` and ``region_coords`` default to the carrier_tracker
+    module's own tables; passed in here for testability.
+    """
+    if carrier_registry is None or region_coords is None:
+        from services.carrier_tracker import CARRIER_REGISTRY, REGION_COORDS
+        carrier_registry = carrier_registry or CARRIER_REGISTRY
+        region_coords = region_coords or REGION_COORDS
+
+    items = _iter_fleet_tracker_items(_RSS_URLS)
+    if not items:
+        logger.warning("USNI fleet-tracker: no parseable RSS items")
+        return {}
+
+    # Pick the most recent by parsed pubDate. Items without a parseable
+    # date fall to the back of the list.
+    items.sort(
+        key=lambda it: it["pub_date"] or datetime(1970, 1, 1, tzinfo=timezone.utc),
+        reverse=True,
+    )
+    latest = items[0]
+
+    pub_dt: datetime | None = latest["pub_date"]
+    pub_iso = pub_dt.isoformat() if pub_dt else ""
+    pub_human = pub_dt.strftime("%b %d, %Y") if pub_dt else "unknown date"
+
+    body = latest["body"]
+    if not body:
+        logger.warning("USNI fleet-tracker: latest item has empty body")
+        return {}
+
+    positions: dict[str, dict] = {}
+    for hull, info in carrier_registry.items():
+        # Build name variants we'll try in the body.
+        full_name = info["name"]                       # "USS Gerald R. Ford (CVN-78)"
+        without_hull = full_name.split("(")[0].strip() # "USS Gerald R. Ford"
+        last_word = without_hull.split()[-1]            # "Ford"
+        ship_only = without_hull[4:]                    # "Gerald R. Ford"
+
+        # Variants ordered most-specific first.
+        variants: list[str] = []
+        for v in (without_hull, f"USS {ship_only}", ship_only, last_word):
+            if v and v not in variants and len(v) >= 4:
+                variants.append(v)
+
+        phrase = _extract_region_for_carrier(body, variants, hull)
+        if not phrase:
+            continue
+        resolved = _resolve_region_phrase(phrase)
+        if not resolved:
+            logger.debug(
+                "USNI: %s region phrase %r did not match any known region",
+                hull, phrase,
+            )
+            continue
+        canonical_key, display_phrase = resolved
+        coords = region_coords.get(canonical_key)
+        if not coords:
+            continue
+
+        positions[hull] = {
+            "lat": coords[0],
+            "lng": coords[1],
+            "heading": 0,
+            "desc": f"{display_phrase.title()} (USNI {pub_human})",
+            "source": f"USNI News Fleet & Marine Tracker ({pub_human})",
+            "source_url": latest["link"],
+            "position_source_at": pub_iso,
+            "position_confidence": "recent",
+        }
+
+    if positions:
+        logger.info(
+            "USNI fleet-tracker: parsed %d/%d carrier positions from %s",
+            len(positions), len(carrier_registry), latest["link"],
+        )
+    else:
+        logger.warning(
+            "USNI fleet-tracker: latest article %s yielded zero parseable carriers",
+            latest["link"],
+        )
+    return positions

backend/services/fetchers/wastewater.py

@@ -0,0 +1,216 @@
+"""WastewaterSCAN fetcher — pathogen surveillance via wastewater monitoring.
+
+Data source: Stanford/Emory WastewaterSCAN project
+  - Plant locations: https://storage.googleapis.com/wastewater-dev-data/json/plants.json
+  - Time series:     https://storage.googleapis.com/wastewater-dev-data/json/{uuid}.json
+
+All data is public, no authentication required.  ~192 treatment plants across
+the US with daily sampling for COVID (N Gene), Influenza A/B, RSV, Norovirus,
+MPXV, Measles, H5N1, and others.
+"""
+
+import logging
+import time
+import concurrent.futures
+from datetime import datetime, timedelta
+from services.network_utils import fetch_with_curl
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger(__name__)
+
+_GCS_BASE = "https://storage.googleapis.com/wastewater-dev-data/json"
+
+# Cache the plants list for 24 hours (it rarely changes)
+_plants_cache: list[dict] = []
+_plants_cache_ts: float = 0
+_PLANTS_CACHE_TTL = 86400  # 24 hours
+
+# Key pathogen targets to extract — maps internal target name to display label
+_TARGET_DISPLAY: dict[str, str] = {
+    "N Gene": "COVID-19",
+    "Influenza A F1R1": "Influenza A",
+    "Influenza B": "Influenza B",
+    "RSV": "RSV",
+    "Noro_G2": "Norovirus",
+    "MPXV_G2R_WA": "Mpox",
+    "InfA_H5": "H5N1 (Bird Flu)",
+    "HMPV_4": "HMPV",
+    "Rota": "Rotavirus",
+    "HAV": "Hepatitis A",
+    "C_auris": "Candida auris",
+    "EVD68": "Enterovirus D68",
+}
+
+# Activity categories that represent elevated/alert levels
+_ALERT_CATEGORIES = {"high", "very high", "above normal"}
+
+
+def _fetch_plants() -> list[dict]:
+    """Fetch the full plants list from GCS, with 24h caching."""
+    global _plants_cache, _plants_cache_ts
+
+    if _plants_cache and (time.time() - _plants_cache_ts) < _PLANTS_CACHE_TTL:
+        return _plants_cache
+
+    url = f"{_GCS_BASE}/plants.json"
+    resp = fetch_with_curl(url, timeout=30)
+    if resp.status_code != 200:
+        logger.warning(f"WastewaterSCAN plants fetch failed: HTTP {resp.status_code}")
+        return _plants_cache  # return stale cache on failure
+
+    data = resp.json()
+    plants = data.get("plants", [])
+    _plants_cache = plants
+    _plants_cache_ts = time.time()
+    logger.info(f"WastewaterSCAN: cached {len(plants)} plant locations")
+    return plants
+
+
+def _fetch_plant_latest(plant_id: str) -> dict | None:
+    """Fetch the most recent sample for a single plant.
+
+    Returns a dict with pathogen levels or None on failure.
+    """
+    url = f"{_GCS_BASE}/{plant_id}.json"
+    try:
+        resp = fetch_with_curl(url, timeout=12)
+        if resp.status_code != 200:
+            return None
+        data = resp.json()
+        samples = data.get("samples", [])
+        if not samples:
+            return None
+
+        # Find the most recent sample (last element, sorted by date)
+        latest = samples[-1]
+        collection_date = latest.get("collection_date", "")
+
+        # Skip samples older than 30 days
+        try:
+            sample_dt = datetime.strptime(collection_date, "%Y-%m-%d")
+            if sample_dt < datetime.utcnow() - timedelta(days=30):
+                return None
+        except (ValueError, TypeError):
+            pass
+
+        # Extract key pathogen levels
+        targets = latest.get("targets", {})
+        pathogens: list[dict] = []
+        alert_count = 0
+
+        for target_key, display_name in _TARGET_DISPLAY.items():
+            target_data = targets.get(target_key)
+            if not target_data:
+                continue
+
+            concentration = target_data.get("gc_g_dry_weight", 0) or 0
+            activity = target_data.get("activity_category", "not calculated")
+            normalized = target_data.get("gc_g_dry_weight_pmmov", 0) or 0
+
+            if concentration <= 0 and normalized <= 0:
+                continue  # no detection
+
+            is_alert = activity.lower() in _ALERT_CATEGORIES
+            if is_alert:
+                alert_count += 1
+
+            pathogens.append({
+                "name": display_name,
+                "target_key": target_key,
+                "concentration": round(concentration, 1),
+                "normalized": round(normalized, 6),
+                "activity": activity,
+                "alert": is_alert,
+            })
+
+        if not pathogens:
+            return None
+
+        return {
+            "collection_date": collection_date,
+            "pathogens": pathogens,
+            "alert_count": alert_count,
+        }
+    except Exception as e:
+        logger.debug(f"WastewaterSCAN: failed to fetch plant {plant_id}: {e}")
+        return None
+
+
+@with_retry(max_retries=1, base_delay=5)
+def fetch_wastewater():
+    """Fetch WastewaterSCAN plant locations and latest pathogen levels.
+
+    1. Fetches the plant list (cached 24h) for locations.
+    2. Concurrently fetches time series for all plants, extracting only
+       the most recent sample's pathogen data.
+    3. Merges into a flat list suitable for map rendering.
+    """
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("wastewater"):
+        return
+
+    plants = _fetch_plants()
+    if not plants:
+        logger.warning("WastewaterSCAN: no plant data available")
+        return
+
+    # Build base records from plant metadata
+    plant_map: dict[str, dict] = {}
+    for p in plants:
+        point = p.get("point") or {}
+        coords = point.get("coordinates") or []
+        if len(coords) < 2:
+            continue
+
+        pid = p.get("id") or p.get("uuid", "")
+        if not pid:
+            continue
+
+        plant_map[pid] = {
+            "id": pid,
+            "name": p.get("name", ""),
+            "site_name": p.get("site_name", ""),
+            "city": p.get("city", ""),
+            "state": p.get("state", ""),
+            "country": p.get("country", "US"),
+            "population": p.get("sewershed_pop"),
+            "lat": coords[1],
+            "lng": coords[0],
+            "pathogens": [],
+            "alert_count": 0,
+            "collection_date": "",
+            "source": "WastewaterSCAN",
+        }
+
+    # Fetch latest samples concurrently (up to 12 threads)
+    with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
+        futures = {
+            pool.submit(_fetch_plant_latest, pid): pid
+            for pid in plant_map
+        }
+        for fut in concurrent.futures.as_completed(futures, timeout=120):
+            pid = futures[fut]
+            try:
+                result = fut.result()
+                if result:
+                    plant_map[pid]["pathogens"] = result["pathogens"]
+                    plant_map[pid]["alert_count"] = result["alert_count"]
+                    plant_map[pid]["collection_date"] = result["collection_date"]
+            except Exception:
+                pass
+
+    nodes = list(plant_map.values())
+    active_nodes = [n for n in nodes if n["pathogens"]]
+
+    logger.info(
+        f"WastewaterSCAN: {len(nodes)} plants, "
+        f"{len(active_nodes)} with recent pathogen data, "
+        f"{sum(n['alert_count'] for n in nodes)} total alerts"
+    )
+
+    with _data_lock:
+        latest_data["wastewater"] = nodes
+    if nodes:
+        _mark_fresh("wastewater")

backend/services/geocode.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import json
 import os
+import re
 import time
 import threading
 from typing import Any, Dict, List
@@ -20,9 +21,17 @@ _cache_lock = threading.Lock()
 _local_search_cache: List[Dict[str, Any]] | None = None
 _local_search_lock = threading.Lock()
 
-_USER_AGENT = os.environ.get(
-    "NOMINATIM_USER_AGENT", "ShadowBroker/1.0 (https://github.com/BigBodyCobain/Shadowbroker)"
-)
+# Round 7a: per-install operator handle threads through every Nominatim
+# call. NOMINATIM_USER_AGENT env override is still honored for operators
+# who run a custom relay / known good identity, but the default uses the
+# per-install handle so OpenStreetMap can rate-limit per install instead
+# of treating "Shadowbroker" as one big offender.
+def _nominatim_user_agent() -> str:
+    override = os.environ.get("NOMINATIM_USER_AGENT", "").strip()
+    if override:
+        return override
+    from services.network_utils import outbound_user_agent
+    return outbound_user_agent("nominatim")
 
 
 def _get_cache(key: str):
@@ -81,43 +90,63 @@ def _load_local_search_cache() -> List[Dict[str, Any]]:
 
 
 def _search_local_fallback(query: str, limit: int) -> List[Dict[str, Any]]:
+    """Strict local lookup used only when ``local_only=True`` is set.
+
+    Historical behaviour (substring-token-in-haystack matching) produced
+    catastrophically wrong results: any query containing a common word
+    would match the first airport with that word anywhere in its name,
+    which silently poisoned every cache downstream. Fixed to require
+    whole-word matches against airport name/IATA/id and cached-geocode
+    labels.
+    """
     q = query.strip().lower()
     if not q:
         return []
+    q_tokens = set(re.findall(r"[a-z0-9]+", q))
+    if not q_tokens:
+        return []
 
     matches: List[Dict[str, Any]] = []
     seen: set[tuple[float, float, str]] = set()
 
+    def _whole_word_tokens(text: str) -> set[str]:
+        return set(re.findall(r"[a-z0-9]+", (text or "").lower()))
+
     for item in cached_airports:
-        haystacks = [
-            str(item.get("name", "")).lower(),
-            str(item.get("iata", "")).lower(),
-            str(item.get("id", "")).lower(),
-        ]
-        if any(q in h for h in haystacks):
-            label = f'{item.get("name", "Airport")} ({item.get("iata", "")})'
-            key = (float(item["lat"]), float(item["lng"]), label)
-            if key not in seen:
-                seen.add(key)
-                matches.append(
-                    {
-                        "label": label,
-                        "lat": float(item["lat"]),
-                        "lng": float(item["lng"]),
-                    }
-                )
-                if len(matches) >= limit:
-                    return matches
+        name_tokens = _whole_word_tokens(item.get("name", ""))
+        iata = str(item.get("iata", "")).lower().strip()
+        icao = str(item.get("id", "")).lower().strip()
+        # IATA/ICAO must match exactly; name must share ALL query tokens
+        # with the airport name (not "any token in haystack").
+        exact_code = bool(iata and iata in q_tokens) or bool(icao and icao in q_tokens)
+        name_match = bool(q_tokens) and q_tokens.issubset(name_tokens)
+        if not (exact_code or name_match):
+            continue
+        label = f'{item.get("name", "Airport")} ({item.get("iata", "")})'
+        key = (float(item["lat"]), float(item["lng"]), label)
+        if key not in seen:
+            seen.add(key)
+            matches.append(
+                {
+                    "label": label,
+                    "lat": float(item["lat"]),
+                    "lng": float(item["lng"]),
+                }
+            )
+            if len(matches) >= limit:
+                return matches
 
     for item in _load_local_search_cache():
         label = str(item.get("label", ""))
-        if q in label.lower():
-            key = (float(item["lat"]), float(item["lng"]), label)
-            if key not in seen:
-                seen.add(key)
-                matches.append(item)
-                if len(matches) >= limit:
-                    break
+        label_tokens = _whole_word_tokens(label)
+        if not q_tokens.issubset(label_tokens):
+            continue
+        key = (float(item["lat"]), float(item["lng"]), label)
+        if key not in seen:
+            seen.add(key)
+            matches.append(item)
+            if len(matches) >= limit:
+                break
 
     return matches
 
@@ -157,15 +186,20 @@ def search_geocode(query: str, limit: int = 5, local_only: bool = False) -> List
         res = fetch_with_curl(
             url,
             headers={
-                "User-Agent": _USER_AGENT,
+                "User-Agent": _nominatim_user_agent(),
                 "Accept-Language": "en",
             },
             timeout=6,
         )
     except Exception:
-        results = _search_local_fallback(q, limit)
-        _set_cache(key, results)
-        return results
+        # Intentionally no silent airport-name fallback. Callers that
+        # want offline results should pass ``local_only=True``; anything
+        # else means we return an empty list so the caller can decide
+        # whether to retry or propagate the failure. The old behaviour
+        # of falling through to _search_local_fallback silently poisoned
+        # every downstream cache with airport coordinates for any query.
+        _set_cache(key, [])
+        return []
 
     results: List[Dict[str, Any]] = []
     if res and res.status_code == 200:
@@ -184,9 +218,9 @@ def search_geocode(query: str, limit: int = 5, local_only: bool = False) -> List
                     continue
         except Exception:
             results = []
-    if not results:
-        results = _search_local_fallback(q, limit)
 
+    # No silent airport-name fallback on empty results either — same
+    # reason as above. Empty means empty.
     _set_cache(key, results)
     return results
 
@@ -215,7 +249,7 @@ def reverse_geocode(lat: float, lng: float, local_only: bool = False) -> Dict[st
         res = fetch_with_curl(
             url,
             headers={
-                "User-Agent": _USER_AGENT,
+                "User-Agent": _nominatim_user_agent(),
                 "Accept-Language": "en",
             },
             timeout=6,

backend/services/geocode_validate.py

@@ -0,0 +1,246 @@
+"""Country-bbox post-filter for geocoded results.
+
+Any fetcher that turns a country-tagged row into a lat/lng should call
+``coord_in_country()`` after the geocoder returns. If the coordinate
+falls outside the country's bounding box, the result is almost
+certainly a namesake collision (e.g. "Milan, WI" landing in Milan,
+Italy) and the caller should reject or retry with a stronger query.
+
+This is a cheap sanity gate that catches geocoder mistakes no human
+operator will ever spot by eye across thousands of points.
+
+Bounding boxes are deliberately generous — they include territories,
+overseas islands, and a small buffer — so that legitimate coastal or
+border cities are never false-rejected. Goal is to catch "wrong
+continent", not "off by a few km".
+"""
+
+from __future__ import annotations
+
+from typing import Optional, Tuple
+
+# (min_lat, min_lng, max_lat, max_lng)
+_COUNTRY_BBOX: dict[str, Tuple[float, float, float, float]] = {
+    # North America
+    "USA": (18.0, -180.0, 72.0, -65.0),          # inc. Alaska + Hawaii
+    "Canada": (41.0, -142.0, 84.0, -52.0),
+    "Mexico": (14.0, -120.0, 33.0, -86.0),
+    # South & Central America
+    "Brazil": (-35.0, -74.5, 6.0, -34.0),
+    "Argentina": (-56.0, -74.0, -21.5, -53.0),
+    "Chile": (-56.0, -76.0, -17.0, -66.0),
+    "Colombia": (-5.0, -82.0, 13.5, -66.5),
+    "Peru": (-19.0, -82.0, 0.5, -68.5),
+    "Venezuela": (0.5, -73.5, 12.5, -59.5),
+    "Ecuador": (-5.5, -92.5, 2.0, -75.0),        # inc. Galápagos
+    "Bolivia": (-23.0, -69.5, -9.5, -57.5),
+    "Uruguay": (-35.0, -58.5, -30.0, -53.0),
+    "Paraguay": (-28.0, -63.0, -19.0, -54.0),
+    "Guatemala": (13.5, -92.5, 18.0, -88.0),
+    "Honduras": (12.5, -89.5, 16.5, -83.0),
+    "Nicaragua": (10.5, -88.0, 15.5, -83.0),
+    "Costa Rica": (8.0, -86.0, 11.5, -82.5),
+    "Panama": (7.0, -83.5, 9.7, -77.0),
+    "El Salvador": (13.0, -90.5, 14.5, -87.5),
+    "Cuba": (19.5, -85.0, 23.5, -74.0),
+    "Dominican Republic": (17.5, -72.5, 20.0, -68.0),
+    "Haiti": (17.5, -74.5, 20.5, -71.5),
+    "Jamaica": (17.5, -78.5, 18.7, -76.0),
+    "Puerto Rico": (17.5, -68.0, 19.0, -65.0),
+    # Europe
+    "United Kingdom": (49.0, -9.0, 61.0, 2.5),
+    "Ireland": (51.0, -11.0, 56.0, -5.0),
+    "France": (41.0, -5.5, 51.5, 9.8),
+    "Germany": (47.0, 5.5, 56.0, 15.5),
+    "Spain": (27.0, -18.5, 44.0, 4.5),           # inc. Canary Islands
+    "Portugal": (32.0, -32.0, 42.5, -6.0),       # inc. Azores + Madeira
+    "Italy": (36.0, 6.5, 47.5, 19.0),
+    "Netherlands": (50.5, 3.0, 53.8, 7.3),
+    "Belgium": (49.4, 2.5, 51.6, 6.5),
+    "Switzerland": (45.7, 5.8, 48.0, 10.6),
+    "Austria": (46.3, 9.5, 49.1, 17.2),
+    "Poland": (49.0, 14.0, 55.0, 24.2),
+    "Czech Republic": (48.5, 12.0, 51.2, 18.9),
+    "Slovakia": (47.7, 16.8, 49.7, 22.6),
+    "Hungary": (45.7, 16.1, 48.6, 22.9),
+    "Romania": (43.6, 20.2, 48.3, 29.7),
+    "Bulgaria": (41.2, 22.3, 44.3, 28.7),
+    "Greece": (34.7, 19.3, 41.8, 29.7),
+    "Turkey": (35.8, 25.6, 42.2, 44.8),
+    "Ukraine": (44.3, 22.1, 52.4, 40.3),
+    "Belarus": (51.2, 23.1, 56.2, 32.8),
+    "Russia": (41.0, 19.0, 82.0, 180.0),
+    "Sweden": (55.0, 10.5, 69.1, 24.2),
+    "Norway": (57.9, 4.5, 71.2, 31.1),
+    "Finland": (59.7, 20.5, 70.1, 31.6),
+    "Denmark": (54.5, 8.0, 57.9, 15.3),
+    "Iceland": (63.3, -24.6, 66.6, -13.4),
+    "Serbia": (42.2, 18.8, 46.2, 23.0),
+    "Croatia": (42.3, 13.4, 46.6, 19.5),
+    "Slovenia": (45.4, 13.3, 46.9, 16.7),
+    "Bosnia and Herzegovina": (42.5, 15.7, 45.3, 19.7),
+    "North Macedonia": (40.8, 20.4, 42.4, 23.1),
+    "Albania": (39.6, 19.2, 42.7, 21.1),
+    "Kosovo": (41.8, 20.0, 43.3, 21.8),
+    "Moldova": (45.4, 26.6, 48.5, 30.2),
+    "Lithuania": (53.8, 20.9, 56.5, 26.9),
+    "Latvia": (55.6, 20.9, 58.1, 28.3),
+    "Estonia": (57.5, 21.7, 59.8, 28.3),
+    "Luxembourg": (49.4, 5.7, 50.2, 6.6),
+    "Malta": (35.7, 14.1, 36.1, 14.7),
+    "Cyprus": (34.5, 32.2, 35.8, 34.7),
+    # Middle East
+    "Israel": (29.4, 34.2, 33.4, 35.9),
+    "Lebanon": (33.0, 35.1, 34.7, 36.7),
+    "Jordan": (29.1, 34.9, 33.4, 39.4),
+    "Syria": (32.3, 35.7, 37.4, 42.4),
+    "Iraq": (29.0, 38.8, 37.4, 48.8),
+    "Iran": (25.0, 44.0, 40.0, 63.4),
+    "Saudi Arabia": (16.3, 34.5, 32.2, 55.7),
+    "Yemen": (12.0, 42.5, 19.0, 54.5),
+    "United Arab Emirates": (22.6, 51.5, 26.1, 56.4),
+    "Oman": (16.6, 52.0, 26.4, 59.9),
+    "Qatar": (24.4, 50.7, 26.2, 51.7),
+    "Bahrain": (25.8, 50.4, 26.4, 50.8),
+    "Kuwait": (28.5, 46.5, 30.1, 48.4),
+    "Afghanistan": (29.4, 60.5, 38.5, 74.9),
+    # Asia
+    "India": (6.0, 68.0, 36.0, 98.0),
+    "Pakistan": (23.7, 60.9, 37.1, 77.8),
+    "Bangladesh": (20.6, 88.0, 26.6, 92.7),
+    "Sri Lanka": (5.9, 79.5, 9.9, 82.0),
+    "Nepal": (26.3, 80.0, 30.5, 88.2),
+    "China": (18.0, 73.0, 54.0, 135.5),
+    "Mongolia": (41.6, 87.7, 52.2, 119.9),
+    "Japan": (24.0, 122.0, 46.0, 146.0),
+    "South Korea": (33.1, 125.1, 38.6, 131.9),
+    "North Korea": (37.7, 124.2, 43.0, 130.7),
+    "Taiwan": (21.8, 119.3, 25.4, 122.1),
+    "Hong Kong": (22.1, 113.8, 22.6, 114.5),
+    "Vietnam": (8.2, 102.1, 23.4, 109.5),
+    "Thailand": (5.6, 97.3, 20.5, 105.7),
+    "Cambodia": (10.4, 102.3, 14.7, 107.7),
+    "Laos": (13.9, 100.0, 22.5, 107.7),
+    "Myanmar": (9.5, 92.1, 28.6, 101.2),
+    "Malaysia": (0.8, 99.5, 7.5, 119.3),
+    "Singapore": (1.1, 103.5, 1.5, 104.1),
+    "Indonesia": (-11.1, 94.8, 6.1, 141.1),
+    "Philippines": (4.5, 116.0, 21.5, 127.0),
+    "Brunei": (4.0, 114.0, 5.1, 115.4),
+    "Kazakhstan": (40.5, 46.4, 55.5, 87.4),
+    "Uzbekistan": (37.1, 55.9, 45.6, 73.2),
+    "Kyrgyzstan": (39.1, 69.2, 43.3, 80.3),
+    "Tajikistan": (36.6, 67.3, 41.1, 75.2),
+    "Turkmenistan": (35.1, 52.4, 42.8, 66.7),
+    "Azerbaijan": (38.3, 44.7, 41.9, 50.6),
+    "Armenia": (38.8, 43.4, 41.3, 46.6),
+    "Georgia": (41.0, 40.0, 43.6, 46.8),
+    # Oceania
+    "Australia": (-44.0, 112.0, -9.0, 155.0),
+    "New Zealand": (-48.0, 165.0, -33.0, 179.5),
+    "Papua New Guinea": (-11.7, 140.8, -1.0, 156.0),
+    "Fiji": (-21.0, 176.8, -12.4, -178.3),       # crosses antimeridian; see handling
+    # Africa (selected — most common NUFORC reporters)
+    "South Africa": (-35.0, 16.0, -22.0, 33.0),
+    "Egypt": (21.7, 24.7, 31.7, 36.9),
+    "Morocco": (27.6, -13.2, 35.9, -1.0),
+    "Algeria": (18.9, -8.7, 37.1, 12.0),
+    "Tunisia": (30.2, 7.5, 37.5, 11.6),
+    "Libya": (19.5, 9.3, 33.2, 25.2),
+    "Sudan": (8.6, 21.8, 22.2, 38.6),
+    "Ethiopia": (3.4, 32.9, 14.9, 48.0),
+    "Kenya": (-4.7, 33.9, 5.5, 41.9),
+    "Tanzania": (-11.8, 29.3, -0.9, 40.4),
+    "Uganda": (-1.5, 29.5, 4.2, 35.0),
+    "Nigeria": (4.2, 2.6, 13.9, 14.7),
+    "Ghana": (4.7, -3.3, 11.2, 1.2),
+    "Senegal": (12.3, -17.6, 16.7, -11.3),
+    "Ivory Coast": (4.3, -8.6, 10.7, -2.5),
+    "Cameroon": (1.6, 8.5, 13.1, 16.2),
+    "Angola": (-18.1, 11.7, -4.4, 24.1),
+    "Zimbabwe": (-22.5, 25.2, -15.6, 33.1),
+    "Zambia": (-18.1, 21.9, -8.2, 33.7),
+    "Mozambique": (-26.9, 30.2, -10.5, 40.9),
+    "Madagascar": (-25.7, 43.2, -11.9, 50.5),
+    "Democratic Republic of the Congo": (-13.5, 12.2, 5.4, 31.4),
+    "Rwanda": (-2.9, 28.8, -1.0, 30.9),
+}
+
+# Common aliases used in NUFORC / other data sources.
+_COUNTRY_ALIASES: dict[str, str] = {
+    "US": "USA",
+    "U.S.": "USA",
+    "U.S.A.": "USA",
+    "United States": "USA",
+    "United States of America": "USA",
+    "America": "USA",
+    "UK": "United Kingdom",
+    "U.K.": "United Kingdom",
+    "Britain": "United Kingdom",
+    "Great Britain": "United Kingdom",
+    "England": "United Kingdom",
+    "Scotland": "United Kingdom",
+    "Wales": "United Kingdom",
+    "Northern Ireland": "United Kingdom",
+    "Czechia": "Czech Republic",
+    "Czechoslovakia": "Czech Republic",
+    "South Korea": "South Korea",
+    "Korea": "South Korea",
+    "Republic of Korea": "South Korea",
+    "Democratic People's Republic of Korea": "North Korea",
+    "DPRK": "North Korea",
+    "Russian Federation": "Russia",
+    "Viet Nam": "Vietnam",
+    "Côte d'Ivoire": "Ivory Coast",
+    "Cote d'Ivoire": "Ivory Coast",
+    "DR Congo": "Democratic Republic of the Congo",
+    "DRC": "Democratic Republic of the Congo",
+    "Congo-Kinshasa": "Democratic Republic of the Congo",
+    "Macedonia": "North Macedonia",
+    "Burma": "Myanmar",
+    "Holland": "Netherlands",
+}
+
+
+def canonical_country(country: str) -> str:
+    """Normalise a country string to its registry key."""
+    if not country:
+        return ""
+    c = country.strip()
+    return _COUNTRY_ALIASES.get(c, c)
+
+
+def coord_in_country(lat: float, lng: float, country: str) -> Optional[bool]:
+    """Return True if (lat, lng) is inside the country bbox, False if it
+    is outside, or None if the country is unknown (cannot validate — the
+    caller should treat unknown as "pass", not "fail").
+    """
+    try:
+        lat_f = float(lat)
+        lng_f = float(lng)
+    except (TypeError, ValueError):
+        return None
+    if not (-90.0 <= lat_f <= 90.0 and -180.0 <= lng_f <= 180.0):
+        return False
+    c = canonical_country(country)
+    bbox = _COUNTRY_BBOX.get(c)
+    if bbox is None:
+        return None
+    min_lat, min_lng, max_lat, max_lng = bbox
+    return min_lat <= lat_f <= max_lat and min_lng <= lng_f <= max_lng
+
+
+def validate_geocode(
+    lat: float,
+    lng: float,
+    country: str,
+) -> bool:
+    """Higher-level gate used in fetcher geocoding loops.
+
+    Returns True if the coordinate is acceptable for the given country,
+    False if it's clearly a namesake collision that should be rejected.
+    Unknown countries are treated as "accept" so we don't throw away
+    otherwise-good data for uncovered regions.
+    """
+    result = coord_in_country(lat, lng, country)
+    return result is not False

backend/services/geopolitics.py

@@ -8,6 +8,13 @@ from datetime import datetime
 from urllib.parse import urljoin, urlparse
 from services.network_utils import fetch_with_curl
 
+
+
+def _geopolitics_user_agent() -> str:
+    """Round 7a: GDELT geopolitics fetcher attribution."""
+    from services.network_utils import outbound_user_agent
+    return outbound_user_agent("geopolitics-gdelt")
+
 logger = logging.getLogger(__name__)
 
 # Cache Frontline data for 30 minutes, it doesn't move that fast
@@ -201,10 +208,12 @@ def _is_gibberish(text):
 # Persistent cache for article titles — survives across GDELT cache refreshes
 # Bounded to 5000 entries with 24hr TTL to prevent unbounded memory growth
 _article_title_cache = TTLCache(maxsize=5000, ttl=86400)
+_article_snippet_cache: dict[str, str | None] = {}
 _article_url_safety_cache = TTLCache(maxsize=5000, ttl=3600)
 _TITLE_FETCH_MAX_REDIRECTS = 3
 _TITLE_FETCH_READ_BYTES = 32768
 _ALLOWED_ARTICLE_PORTS = {80, 443, 8080, 8443}
+_MAX_SNIPPET_LEN = 200
 
 
 def _hostname_resolves_public(hostname: str, port: int) -> bool:
@@ -269,6 +278,30 @@ def _is_safe_public_article_url(url: str) -> tuple[bool, str]:
     return result
 
 
+def _extract_snippet(url: str, chunk: str) -> None:
+    """Extract og:description or meta description from an already-fetched HTML chunk."""
+    import re
+    import html as html_mod
+
+    if url in _article_snippet_cache:
+        return
+    snippet = None
+    # Try og:description first
+    for pattern in (
+        r'<meta[^>]+property=["\']og:description["\'][^>]+content=["\']([^"\'>]+)["\']',
+        r'<meta[^>]+content=["\']([^"\'>]+)["\'][^>]+property=["\']og:description["\']',
+        r'<meta[^>]+name=["\']description["\'][^>]+content=["\']([^"\'>]+)["\']',
+        r'<meta[^>]+content=["\']([^"\'>]+)["\'][^>]+name=["\']description["\']',
+    ):
+        m = re.search(pattern, chunk, re.I)
+        if m:
+            snippet = html_mod.unescape(m.group(1)).strip()
+            break
+    if snippet and len(snippet) > _MAX_SNIPPET_LEN:
+        snippet = snippet[:_MAX_SNIPPET_LEN - 3].rsplit(" ", 1)[0] + "..."
+    _article_snippet_cache[url] = snippet if snippet and len(snippet) > 15 else None
+
+
 def _fetch_article_title(url):
     """Fetch the real headline from an article's HTML <title> or og:title tag.
     Returns the title string, or None if it can't be fetched.
@@ -290,7 +323,7 @@ def _fetch_article_title(url):
             resp = requests.get(
                 current_url,
                 timeout=4,
-                headers={"User-Agent": "Mozilla/5.0 (compatible; OSINT Dashboard/1.0)"},
+                headers={"User-Agent": _geopolitics_user_agent()},
                 stream=True,
                 allow_redirects=False,
             )
@@ -343,6 +376,8 @@ def _fetch_article_title(url):
                 title = title[:117] + "..."
             if len(title) > 10:
                 _article_title_cache[url] = title
+                # Also extract og:description / meta description for snippet
+                _extract_snippet(url, chunk)
                 return title
 
         _article_title_cache[url] = None
@@ -405,21 +440,49 @@ def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_
                     actor1 = row[6].strip() if len(row) > 6 else ""
                     actor2 = row[16].strip() if len(row) > 16 else ""
 
+                    # Extract enrichment fields from GDELT CSV
+                    event_date = row[1].strip() if len(row) > 1 else ""
+                    full_event_code = row[26].strip() if len(row) > 26 else ""
+                    quad_class = int(row[29]) if len(row) > 29 and row[29].strip().isdigit() else 0
+                    goldstein = float(row[30]) if len(row) > 30 and row[30].strip() else 0.0
+                    num_mentions = int(row[31]) if len(row) > 31 and row[31].strip().isdigit() else 0
+                    num_sources = int(row[32]) if len(row) > 32 and row[32].strip().isdigit() else 0
+                    num_articles = int(row[33]) if len(row) > 33 and row[33].strip().isdigit() else 0
+                    avg_tone = float(row[34]) if len(row) > 34 and row[34].strip() else 0.0
+
                     loc_key = f"{round(lat, 1)}_{round(lng, 1)}"
                     if loc_key in seen_locs:
-                        # Merge: increment count and add source URL if new (dedup by domain)
+                        # Merge: increment count, accumulate intensity, add source URL
                         idx = loc_index[loc_key]
                         feat = features[idx]
-                        feat["properties"]["count"] = feat["properties"].get("count", 1) + 1
-                        urls = feat["properties"].get("_urls", [])
-                        seen_domains = feat["properties"].get("_domains", set())
+                        props = feat["properties"]
+                        props["count"] = props.get("count", 1) + 1
+                        # Track worst Goldstein score (most negative = most intense)
+                        if goldstein < props.get("goldstein", 0):
+                            props["goldstein"] = round(goldstein, 1)
+                        # Accumulate mentions/sources for importance ranking
+                        props["num_mentions"] = props.get("num_mentions", 0) + num_mentions
+                        props["num_sources"] = props.get("num_sources", 0) + num_sources
+                        props["num_articles"] = props.get("num_articles", 0) + num_articles
+                        # Track latest date
+                        if event_date and event_date > props.get("event_date", ""):
+                            props["event_date"] = event_date
+                        # Collect actors
+                        actors = props.get("_actors_set", set())
+                        if actor1:
+                            actors.add(actor1)
+                        if actor2:
+                            actors.add(actor2)
+                        props["_actors_set"] = actors
+                        urls = props.get("_urls", [])
+                        seen_domains = props.get("_domains", set())
                         if source_url:
                             domain = _extract_domain(source_url)
                             if domain not in seen_domains and len(urls) < 10:
                                 urls.append(source_url)
                                 seen_domains.add(domain)
-                                feat["properties"]["_urls"] = urls
-                                feat["properties"]["_domains"] = seen_domains
+                                props["_urls"] = urls
+                                props["_domains"] = seen_domains
                         continue
                     seen_locs.add(loc_key)
 
@@ -429,6 +492,11 @@ def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_
                         or "Unknown Incident"
                     )
                     domain = _extract_domain(source_url) if source_url else ""
+                    actors_set = set()
+                    if actor1:
+                        actors_set.add(actor1)
+                    if actor2:
+                        actors_set.add(actor2)
                     loc_index[loc_key] = len(features)
                     features.append(
                         {
@@ -436,6 +504,17 @@ def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_
                             "properties": {
                                 "name": name,
                                 "count": 1,
+                                "event_date": event_date,
+                                "event_code": full_event_code,
+                                "quad_class": quad_class,
+                                "goldstein": round(goldstein, 1),
+                                "num_mentions": num_mentions,
+                                "num_sources": num_sources,
+                                "num_articles": num_articles,
+                                "avg_tone": round(avg_tone, 1),
+                                "actor1": actor1,
+                                "actor2": actor2,
+                                "_actors_set": actors_set,
                                 "_urls": [source_url] if source_url else [],
                                 "_domains": {domain} if domain else set(),
                             },
@@ -449,10 +528,29 @@ def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_
         logger.warning(f"Failed to parse GDELT export zip: {e}")
 
 
+# GDELT's data.gdeltproject.org is a CNAME to a Google Cloud Storage
+# bucket of the same name. GCS returns the wildcard ``*.storage.googleapis.com``
+# certificate, which legitimately does NOT cover the GDELT custom domain
+# — Python's TLS verification correctly refuses it. Some networks/POPs
+# happen to route through a path where this works; many do not (notably
+# Docker Desktop's outbound NAT on local installs).
+#
+# Fix: rewrite the URL to hit GCS directly with a path-style bucket
+# reference, where the standard GCS cert is genuinely valid. Same data,
+# verified TLS, no operator-side workaround needed.
+def _gcs_direct_gdelt_url(url: str) -> str:
+    """If ``url`` points at data.gdeltproject.org, return the equivalent
+    GCS-direct URL. Otherwise return the URL unchanged."""
+    prefix = "://data.gdeltproject.org/"
+    if prefix in url:
+        return url.replace(prefix, "://storage.googleapis.com/data.gdeltproject.org/", 1)
+    return url
+
+
 def _download_gdelt_export(url):
     """Download a single GDELT export file, return bytes or None."""
     try:
-        res = fetch_with_curl(url, timeout=15)
+        res = fetch_with_curl(_gcs_direct_gdelt_url(url), timeout=15)
         if res.status_code == 200:
             return res.content
     except (ConnectionError, TimeoutError, OSError):  # non-critical
@@ -468,12 +566,19 @@ def _build_feature_html(features, fetched_titles=None):
     for f in features:
         urls = f["properties"].pop("_urls", [])
         f["properties"].pop("_domains", None)
+        # Convert actors set to sorted list for JSON serialization
+        actors_set = f["properties"].pop("_actors_set", set())
+        if actors_set:
+            f["properties"]["actors"] = sorted(actors_set)[:6]
         headlines = []
+        snippets = []
         for u in urls:
             real_title = fetched_titles.get(u) if fetched_titles else None
             headlines.append(real_title if real_title else _url_to_headline(u))
+            snippets.append(_article_snippet_cache.get(u) or "")
         f["properties"]["_urls_list"] = urls
         f["properties"]["_headlines_list"] = headlines
+        f["properties"]["_snippets_list"] = snippets
         if urls:
             links = []
             for u, h in zip(urls, headlines):
@@ -498,16 +603,19 @@ def _enrich_gdelt_titles_background(features, all_article_urls):
         fetched_count = sum(1 for v in fetched_titles.values() if v)
         logger.info(f"[BG] Resolved {fetched_count}/{len(all_article_urls)} article titles")
 
-        # Update features in-place with real titles
+        # Update features in-place with real titles and snippets
         for f in features:
             urls = f["properties"].get("_urls_list", [])
             if not urls:
                 continue
             headlines = []
+            snippets = []
             for u in urls:
                 real_title = fetched_titles.get(u)
                 headlines.append(real_title if real_title else _url_to_headline(u))
+                snippets.append(_article_snippet_cache.get(u) or "")
             f["properties"]["_headlines_list"] = headlines
+            f["properties"]["_snippets_list"] = snippets
             links = []
             for u, h in zip(urls, headlines):
                 safe_url = u if u.startswith(("http://", "https://")) else "about:blank"
@@ -534,9 +642,16 @@ def fetch_global_military_incidents():
     try:
         logger.info("Fetching GDELT events via export CDN (multi-file)...")
 
-        # Get the latest export URL to determine current timestamp
+        # Get the latest export URL to determine current timestamp.
+        # HTTPS is used to prevent passive network observers from injecting
+        # poisoned export records into the global incident map via MITM.
+        # GDELT serves the same content over HTTPS as HTTP.
+        # Use the GCS-direct URL because data.gdeltproject.org's CNAME
+        # serves a wildcard *.storage.googleapis.com cert that legitimately
+        # doesn't cover the GDELT hostname. See _gcs_direct_gdelt_url above.
         index_res = fetch_with_curl(
-            "http://data.gdeltproject.org/gdeltv2/lastupdate.txt", timeout=10
+            _gcs_direct_gdelt_url("https://data.gdeltproject.org/gdeltv2/lastupdate.txt"),
+            timeout=10,
         )
         if index_res.status_code != 200:
             logger.error(f"GDELT lastupdate failed: {index_res.status_code}")
@@ -554,7 +669,9 @@ def fetch_global_military_incidents():
             logger.error("Could not find GDELT export URL")
             return []
 
-        # Extract timestamp from URL like: http://data.gdeltproject.org/gdeltv2/20260301120000.export.CSV.zip
+        # Extract timestamp from URL like: https://data.gdeltproject.org/gdeltv2/20260301120000.export.CSV.zip
+        # (GDELT's lastupdate.txt may still list URLs with http:// — we ignore
+        # the scheme there and reconstruct each download URL as https:// below.)
         import re
 
         ts_match = re.search(r"(\d{14})\.export\.CSV\.zip", latest_url)
@@ -564,13 +681,13 @@ def fetch_global_military_incidents():
 
         latest_ts = datetime.strptime(ts_match.group(1), "%Y%m%d%H%M%S")
 
-        # Generate URLs for the last 8 hours (32 files at 15-min intervals)
-        NUM_FILES = 32
+        # Generate URLs for the last 12 hours (48 files at 15-min intervals)
+        NUM_FILES = 48
         urls = []
         for i in range(NUM_FILES):
             ts = latest_ts - timedelta(minutes=15 * i)
             fname = ts.strftime("%Y%m%d%H%M%S") + ".export.CSV.zip"
-            url = f"http://data.gdeltproject.org/gdeltv2/{fname}"
+            url = f"https://data.gdeltproject.org/gdeltv2/{fname}"
             urls.append(url)
 
         logger.info(f"Downloading {len(urls)} GDELT export files...")
@@ -583,7 +700,7 @@ def fetch_global_military_incidents():
         logger.info(f"Downloaded {successful}/{len(urls)} GDELT exports")
 
         # Parse all downloaded files
-        CONFLICT_CODES = {"14", "17", "18", "19", "20"}
+        CONFLICT_CODES = {"13", "14", "15", "16", "17", "18", "19", "20"}
         features = []
         seen_locs = set()
         loc_index = {}  # loc_key -> index in features

backend/services/infonet/__init__.py

@@ -0,0 +1,129 @@
+"""Infonet economy & governance layer.
+
+Layered ON TOP OF the existing mesh primitives in ``services/mesh/``.
+The chain-write cutover (2026-04-28) registers Infonet event types
+with ``mesh_schema`` and ``mesh_hashchain`` so production writes flow
+through the legacy chain. The cutover is performed at import time by
+``services.infonet._chain_cutover``.
+
+The only legacy file modified by the cutover is ``mesh_schema.py``,
+which gained a generic extension hook (``register_extension_validator``).
+``mesh_hashchain.py`` is byte-identical to its Sprint 1 baseline; the
+cutover mutates its module-level ``ACTIVE_APPEND_EVENT_TYPES`` set
+(which is a mutable ``set``, not a frozenset, by design).
+
+See ``infonet-economy/IMPLEMENTATION_PLAN.md`` and ``infonet-economy/BUILD_LOG.md``
+in the repository root for the build order, sprint scope, and integration
+principles. ``infonet-economy/RULES_SKELETON.md`` is the source of truth
+for any formula / value / state machine implemented here.
+"""
+
+# Trigger the chain-write cutover at import time. Idempotent — see
+# ``_chain_cutover.perform_cutover``. This must happen before any
+# adapter or producer uses mesh_schema.validate_event_payload on a
+# new event type.
+from services.infonet import _chain_cutover as _chain_cutover_module
+_chain_cutover_module.perform_cutover()
+del _chain_cutover_module
+
+from services.infonet.config import (
+    CONFIG,
+    CONFIG_SCHEMA,
+    CROSS_FIELD_INVARIANTS,
+    IMMUTABLE_PRINCIPLES,
+    InvalidPetition,
+    reset_config_for_tests,
+    validate_config_schema_completeness,
+    validate_cross_field_invariants,
+    validate_petition_value,
+)
+from services.infonet.identity_rotation import (
+    RotationBlocker,
+    RotationDecision,
+    rotation_descendants,
+    validate_rotation,
+)
+from services.infonet.markets import (
+    EvidenceBundle,
+    MarketStatus,
+    ResolutionResult,
+    build_snapshot,
+    collect_evidence,
+    collect_resolution_stakes,
+    compute_market_status,
+    compute_snapshot_event_hash,
+    evidence_content_hash,
+    excluded_predictor_ids,
+    find_snapshot,
+    is_first_for_side,
+    is_predictor_excluded,
+    resolve_market,
+    should_advance_phase,
+    submission_hash,
+)
+from services.infonet.reputation import (
+    OracleRepBreakdown,
+    compute_common_rep,
+    compute_oracle_rep,
+    compute_oracle_rep_active,
+    compute_oracle_rep_lifetime,
+    decay_factor_for_age,
+    last_successful_prediction_ts,
+)
+from services.infonet.schema import (
+    INFONET_ECONOMY_EVENT_TYPES,
+    InfonetEventSchema,
+    get_infonet_schema,
+    validate_infonet_event_payload,
+)
+from services.infonet.time_validity import (
+    chain_majority_time,
+    event_meets_phase_window,
+    is_event_too_future,
+)
+
+__all__ = [
+    "CONFIG",
+    "CONFIG_SCHEMA",
+    "CROSS_FIELD_INVARIANTS",
+    "IMMUTABLE_PRINCIPLES",
+    "INFONET_ECONOMY_EVENT_TYPES",
+    "EvidenceBundle",
+    "InfonetEventSchema",
+    "InvalidPetition",
+    "MarketStatus",
+    "OracleRepBreakdown",
+    "ResolutionResult",
+    "RotationBlocker",
+    "RotationDecision",
+    "build_snapshot",
+    "chain_majority_time",
+    "collect_evidence",
+    "collect_resolution_stakes",
+    "compute_common_rep",
+    "compute_market_status",
+    "compute_oracle_rep",
+    "compute_oracle_rep_active",
+    "compute_oracle_rep_lifetime",
+    "compute_snapshot_event_hash",
+    "decay_factor_for_age",
+    "event_meets_phase_window",
+    "evidence_content_hash",
+    "excluded_predictor_ids",
+    "find_snapshot",
+    "get_infonet_schema",
+    "is_event_too_future",
+    "is_first_for_side",
+    "is_predictor_excluded",
+    "last_successful_prediction_ts",
+    "reset_config_for_tests",
+    "resolve_market",
+    "rotation_descendants",
+    "should_advance_phase",
+    "submission_hash",
+    "validate_config_schema_completeness",
+    "validate_cross_field_invariants",
+    "validate_infonet_event_payload",
+    "validate_petition_value",
+    "validate_rotation",
+]

backend/services/infonet/_chain_cutover.py

@@ -0,0 +1,108 @@
+"""Chain-write cutover — register Infonet economy event types with the
+legacy mesh_schema + mesh_hashchain at import time.
+
+Source of truth: ``infonet-economy/BUILD_LOG.md`` Sprint 4 §6.2 cutover
+decision (Option C — rename + coexist with new event-type names).
+
+Before this cutover, Sprints 1-7 produced economy events through
+``InfonetHashchainAdapter.dry_run_append`` only. None of them landed
+on the legacy chain because ``mesh_hashchain.Infonet.append`` rejected
+any event_type not in ``ACTIVE_APPEND_EVENT_TYPES``.
+
+This module performs the surgical wiring needed for production writes:
+
+1. Mutates ``mesh_hashchain.ACTIVE_APPEND_EVENT_TYPES`` (a mutable
+   set, not a frozenset) to include every type in
+   ``INFONET_ECONOMY_EVENT_TYPES``.
+2. Registers each economy event type's payload validator with
+   ``mesh_schema._EXTENSION_VALIDATORS`` via the Sprint-8-polish
+   ``register_extension_validator`` hook.
+
+The cutover is **idempotent**: importing this module twice leaves the
+state unchanged.
+
+The direction is **one-way**: infonet imports mesh_*; mesh never
+imports infonet. mesh_schema's hook is generic — it doesn't know
+about infonet specifically.
+
+What is NOT modified by this cutover:
+
+- ``mesh_schema.SCHEMA_REGISTRY`` — legacy validators stay as-is.
+  Economy types use the parallel ``_EXTENSION_VALIDATORS`` registry.
+- ``mesh_schema.ACTIVE_PUBLIC_LEDGER_EVENT_TYPES`` — legacy frozenset
+  unchanged. The runtime decision in
+  ``mesh_hashchain.Infonet.append`` consults the mutable
+  ``ACTIVE_APPEND_EVENT_TYPES`` set.
+- ``mesh_hashchain.py`` — byte-identical to its Sprint 1 baseline.
+- The legacy ``normalize_payload`` and "no ephemeral on this type"
+  checks — extension events skip them. Economy event payloads
+  already have their own normalization (the schema in
+  ``services/infonet/schema.py``).
+"""
+
+from __future__ import annotations
+
+import threading
+
+from services.infonet.schema import (
+    INFONET_ECONOMY_EVENT_TYPES,
+    validate_infonet_event_payload,
+)
+from services.mesh import mesh_hashchain, mesh_schema
+
+
+_CUTOVER_LOCK = threading.Lock()
+_CUTOVER_DONE = False
+
+
+def perform_cutover() -> None:
+    """Idempotent registration of every Infonet economy event type.
+
+    Safe to call multiple times. After the first call, repeat calls
+    are no-ops (the lock + sentinel guard re-entry).
+    """
+    global _CUTOVER_DONE
+    with _CUTOVER_LOCK:
+        if _CUTOVER_DONE:
+            return
+        # Extend the active-append set so mesh_hashchain.Infonet.append
+        # accepts these types. The set is mutable by design (legacy
+        # mesh_hashchain.py line 163 uses set(), not frozenset()).
+        mesh_hashchain.ACTIVE_APPEND_EVENT_TYPES.update(INFONET_ECONOMY_EVENT_TYPES)
+        # Register a validator for each. The lambda binds to the loop
+        # variable via default-arg trick to avoid late-binding bugs.
+        for event_type in INFONET_ECONOMY_EVENT_TYPES:
+            mesh_schema.register_extension_validator(
+                event_type,
+                lambda payload, _et=event_type: validate_infonet_event_payload(_et, payload),
+            )
+        _CUTOVER_DONE = True
+
+
+def cutover_status() -> dict[str, object]:
+    """Diagnostic — used by tests and health endpoints to confirm the
+    cutover ran and registered every type."""
+    return {
+        "done": _CUTOVER_DONE,
+        "registered_types": sorted(
+            t for t in INFONET_ECONOMY_EVENT_TYPES
+            if mesh_schema.is_extension_event_type(t)
+        ),
+        "missing_types": sorted(
+            t for t in INFONET_ECONOMY_EVENT_TYPES
+            if not mesh_schema.is_extension_event_type(t)
+        ),
+        "active_append_includes_economy": INFONET_ECONOMY_EVENT_TYPES.issubset(
+            mesh_hashchain.ACTIVE_APPEND_EVENT_TYPES
+        ),
+    }
+
+
+# Run automatically when the module is imported. The infonet package
+# __init__ imports this module, so any code that uses
+# ``services.infonet`` at all triggers the cutover. Production callers
+# don't need to do anything explicit.
+perform_cutover()
+
+
+__all__ = ["cutover_status", "perform_cutover"]

backend/services/infonet/adapters/__init__.py

@@ -0,0 +1,38 @@
+"""Adapter layer between the Infonet economy package and the legacy
+``services/mesh/`` primitives.
+
+Rule: **adapters import from mesh, mesh never imports from infonet.**
+This keeps the dependency direction one-way and lets us delete the
+infonet package without touching mesh.
+
+The legacy mesh files (``mesh_schema.py``, ``mesh_signed_events.py``,
+``mesh_hashchain.py``, ``mesh_reputation.py``, ``mesh_oracle.py``) stay
+byte-identical through Sprint 3. From Sprint 4 onward, when actual chain
+writes for new event types start happening, the hashchain adapter is
+the single integration point that decides whether to:
+
+1. Modify ``ACTIVE_APPEND_EVENT_TYPES`` in ``mesh_schema.py`` (one-shot,
+   minimal mesh change), OR
+2. Maintain a parallel append surface in ``hashchain_adapter`` that
+   shares the on-disk chain file but bypasses the legacy event-type
+   gate.
+
+The decision is recorded in ``infonet-economy/BUILD_LOG.md`` Sprint 4
+when made.
+"""
+
+from services.infonet.adapters.hashchain_adapter import (
+    InfonetHashchainAdapter,
+    extended_active_event_types,
+)
+from services.infonet.adapters.signed_write_adapter import (
+    INFONET_SIGNED_WRITE_KINDS,
+    InfonetSignedWriteKind,
+)
+
+__all__ = [
+    "INFONET_SIGNED_WRITE_KINDS",
+    "InfonetHashchainAdapter",
+    "InfonetSignedWriteKind",
+    "extended_active_event_types",
+]

backend/services/infonet/adapters/gate_adapter.py

@@ -0,0 +1,178 @@
+"""Gate adapter — Sprint 6 implementation.
+
+Bridges chain history to the gate sacrifice / locking / shutdown
+lifecycle. Same ``chain_provider`` pattern as the other adapters.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any, Callable, Iterable
+
+from services.infonet.gates import (
+    AppealValidation,
+    EntryDecision,
+    GateMeta,
+    LockedGateState,
+    ShutdownState,
+    SuspensionState,
+    can_enter,
+    compute_member_set,
+    compute_shutdown_state,
+    compute_suspension_state,
+    cumulative_member_oracle_rep,
+    get_gate_meta,
+    is_locked,
+    is_member,
+    is_ratified,
+    locked_at,
+    locked_by,
+    paused_execution_remaining_sec,
+    validate_appeal_filing,
+    validate_lock_request,
+    validate_shutdown_filing,
+    validate_suspend_filing,
+)
+from services.infonet.gates.locking import LockValidation
+from services.infonet.gates.shutdown.suspend import FilingValidation
+from services.infonet.time_validity import chain_majority_time
+
+
+_ChainProvider = Callable[[], Iterable[dict[str, Any]]]
+
+
+def _empty_chain() -> list[dict[str, Any]]:
+    return []
+
+
+class InfonetGateAdapter:
+    """Project chain state into gate views."""
+
+    def __init__(self, chain_provider: _ChainProvider | None = None) -> None:
+        self._chain_provider: _ChainProvider = chain_provider or _empty_chain
+
+    def _events(self) -> list[dict[str, Any]]:
+        return [e for e in self._chain_provider() if isinstance(e, dict)]
+
+    def _now(self, override: float | None) -> float:
+        if override is not None:
+            return float(override)
+        events = self._events()
+        chain_now = chain_majority_time(events)
+        return chain_now if chain_now > 0 else float(time.time())
+
+    # ── Metadata + membership ────────────────────────────────────────
+    def gate_meta(self, gate_id: str) -> GateMeta | None:
+        return get_gate_meta(gate_id, self._events())
+
+    def member_set(self, gate_id: str) -> set[str]:
+        return compute_member_set(gate_id, self._events())
+
+    def is_member(self, node_id: str, gate_id: str) -> bool:
+        return is_member(node_id, gate_id, self._events())
+
+    def can_enter(self, node_id: str, gate_id: str) -> EntryDecision:
+        return can_enter(node_id, gate_id, self._events())
+
+    # ── Ratification ─────────────────────────────────────────────────
+    def is_ratified(self, gate_id: str) -> bool:
+        return is_ratified(gate_id, self._events())
+
+    def cumulative_member_oracle_rep(self, gate_id: str) -> float:
+        return cumulative_member_oracle_rep(gate_id, self._events())
+
+    # ── Locking ──────────────────────────────────────────────────────
+    def is_locked(self, gate_id: str) -> bool:
+        return is_locked(gate_id, self._events())
+
+    def locked_state(self, gate_id: str) -> LockedGateState:
+        events = self._events()
+        return LockedGateState(
+            locked=is_locked(gate_id, events),
+            locked_at=locked_at(gate_id, events),
+            locked_by=locked_by(gate_id, events),
+        )
+
+    def validate_lock_request(
+        self, node_id: str, gate_id: str, *, lock_cost: int | None = None,
+    ) -> LockValidation:
+        return validate_lock_request(node_id, gate_id, self._events(), lock_cost=lock_cost)
+
+    # ── Suspension ───────────────────────────────────────────────────
+    def suspension_state(
+        self, gate_id: str, *, now: float | None = None,
+    ) -> SuspensionState:
+        return compute_suspension_state(gate_id, self._events(), now=self._now(now))
+
+    def validate_suspend_filing(
+        self,
+        gate_id: str,
+        filer_id: str,
+        *,
+        reason: str,
+        evidence_hashes: list[str],
+        now: float | None = None,
+        filer_cooldown_until: float | None = None,
+    ) -> FilingValidation:
+        return validate_suspend_filing(
+            gate_id, filer_id,
+            reason=reason, evidence_hashes=evidence_hashes,
+            chain=self._events(), now=self._now(now),
+            filer_cooldown_until=filer_cooldown_until,
+        )
+
+    # ── Shutdown ─────────────────────────────────────────────────────
+    def shutdown_state(
+        self, gate_id: str, *, now: float | None = None,
+    ) -> ShutdownState:
+        return compute_shutdown_state(gate_id, self._events(), now=self._now(now))
+
+    def validate_shutdown_filing(
+        self,
+        gate_id: str,
+        filer_id: str,
+        *,
+        reason: str,
+        evidence_hashes: list[str],
+        now: float | None = None,
+        filer_cooldown_until: float | None = None,
+    ) -> FilingValidation:
+        return validate_shutdown_filing(
+            gate_id, filer_id,
+            reason=reason, evidence_hashes=evidence_hashes,
+            chain=self._events(), now=self._now(now),
+            filer_cooldown_until=filer_cooldown_until,
+        )
+
+    # ── Appeal ───────────────────────────────────────────────────────
+    def validate_appeal_filing(
+        self,
+        gate_id: str,
+        target_petition_id: str,
+        filer_id: str,
+        *,
+        reason: str,
+        evidence_hashes: list[str],
+        now: float | None = None,
+        filer_cooldown_until: float | None = None,
+    ) -> AppealValidation:
+        return validate_appeal_filing(
+            gate_id, target_petition_id, filer_id,
+            reason=reason, evidence_hashes=evidence_hashes,
+            chain=self._events(), now=self._now(now),
+            filer_cooldown_until=filer_cooldown_until,
+        )
+
+    def paused_execution_remaining_sec(
+        self,
+        target_petition_id: str,
+        *,
+        appeal_filed_at: float,
+    ) -> float:
+        return paused_execution_remaining_sec(
+            target_petition_id, self._events(),
+            appeal_filed_at=appeal_filed_at,
+        )
+
+
+__all__ = ["InfonetGateAdapter"]

backend/services/infonet/adapters/hashchain_adapter.py

@@ -0,0 +1,125 @@
+"""Bridge between Infonet economy events and the legacy ``mesh_hashchain``.
+
+Sprint 1 ships this as a **dry-run-only** surface. We do NOT call the
+legacy ``Infonet.append`` for new event types because that method
+hard-rejects anything not in ``ACTIVE_APPEND_EVENT_TYPES`` (defined in
+``mesh_schema.py``). Modifying that set is a Sprint 4 task — it requires
+the rest of the producer code to exist, otherwise a malformed
+``prediction_create`` could land on the chain with no resolver to
+process it.
+
+What this adapter DOES today:
+
+- ``extended_active_event_types()`` — returns the union of legacy active
+  types and new economy types, for tooling that needs the full surface
+  (e.g. RPC layer, frontend type generation).
+- ``InfonetHashchainAdapter.dry_run_append`` — validates a payload
+  against the new schema and returns the event dict the legacy
+  ``Infonet.append`` would have built. Useful for tests and for the
+  future cutover plan.
+
+What this adapter will do in Sprint 4:
+
+- ``append_infonet_event`` — actually call ``Infonet.append`` once
+  ``ACTIVE_APPEND_EVENT_TYPES`` is unioned with the economy types.
+
+The Sprint 1 contract:
+
+- ``mesh_hashchain.py`` is byte-identical to the pre-Sprint-1 baseline.
+- No event reaches the legacy chain via this adapter in Sprint 1.
+- Tests cover validation behavior only.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from typing import Any
+
+from services.mesh.mesh_schema import (
+    ACTIVE_PUBLIC_LEDGER_EVENT_TYPES as _LEGACY_ACTIVE_TYPES,
+)
+
+from services.infonet.schema import (
+    INFONET_ECONOMY_EVENT_TYPES,
+    validate_infonet_event_payload,
+)
+
+
+def extended_active_event_types() -> frozenset[str]:
+    """Union of legacy active types and new economy types.
+
+    Frozen at import time. The legacy set is itself a frozenset so this
+    is safe to call from any thread.
+    """
+    return _LEGACY_ACTIVE_TYPES | INFONET_ECONOMY_EVENT_TYPES
+
+
+class InfonetHashchainAdapter:
+    """Validation-only adapter for new Infonet economy events.
+
+    Real chain integration lives in Sprint 4. Tests should use
+    ``dry_run_append`` to assert that producer code is constructing
+    correctly-shaped events before the cutover.
+    """
+
+    def dry_run_append(
+        self,
+        event_type: str,
+        node_id: str,
+        payload: dict[str, Any],
+        *,
+        sequence: int = 1,
+        timestamp: float | None = None,
+    ) -> dict[str, Any]:
+        """Validate and return a synthetic event dict.
+
+        Mirrors the shape that ``mesh_hashchain.Infonet.append`` would
+        produce for legacy types — same field set, same ordering. Does
+        NOT compute a real signature (Sprint 4 territory) and does NOT
+        write to disk.
+
+        Raises ``ValueError`` on validation failure — the same exception
+        type the legacy ``append`` raises so callers don't need to
+        special-case the cutover later.
+        """
+        if event_type not in INFONET_ECONOMY_EVENT_TYPES:
+            raise ValueError(f"event_type {event_type!r} not in INFONET_ECONOMY_EVENT_TYPES")
+        if not isinstance(node_id, str) or not node_id:
+            raise ValueError("node_id is required")
+        if not isinstance(sequence, int) or isinstance(sequence, bool) or sequence <= 0:
+            raise ValueError("sequence must be a positive integer")
+
+        ok, reason = validate_infonet_event_payload(event_type, payload)
+        if not ok:
+            raise ValueError(reason)
+
+        ts = float(timestamp) if timestamp is not None else float(time.time())
+
+        canonical = {
+            "event_type": event_type,
+            "node_id": node_id,
+            "payload": payload,
+            "timestamp": ts,
+            "sequence": sequence,
+        }
+        encoded = json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+        event_id = hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+
+        return {
+            "event_id": event_id,
+            "event_type": event_type,
+            "node_id": node_id,
+            "timestamp": ts,
+            "sequence": sequence,
+            "payload": payload,
+            # signature / public_key intentionally omitted in Sprint 1.
+            "is_provisional": True,
+        }
+
+
+__all__ = [
+    "InfonetHashchainAdapter",
+    "extended_active_event_types",
+]