feat(gt): experimental Derived OSINT analytics with lean-node safeguards

Cherry-picked from @Bobpick PR #391 (GT + OpenClaw slice): Bayesian strategic-risk engine, map overlay, OpenClaw commands, and telegram_rhetoric watchdog. Off by default (GT_ANALYTICS_ENABLED=false, gt_risk layer false). 1 vCPU nodes get cgroup detection, UI warning on layer toggle, and lean profile that skips scheduled ingest/Louvain unless GT_ANALYTICS_ACK_LOW_CPU=true. Backtest HUD removed from dashboard (OpenClaw/API regression only).

Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

.env.example

@@ -10,6 +10,37 @@ OPENSKY_CLIENT_ID=
 OPENSKY_CLIENT_SECRET=
 AIS_API_KEY=
 
+# Global Fishing Watch — fishing vessel activity events (Fishing Activity map layer).
+# Free API token from https://globalfishingwatch.org/our-apis/tokens
+# Without this the fishing_activity layer stays empty.
+# GFW_API_TOKEN=
+# Optional tuning — GFW can return 40k+ global events; defaults cap fetch for map paint.
+# GFW_EVENTS_PAGE_SIZE=500
+# GFW_EVENTS_MAX_PAGES=10
+# GFW_EVENTS_LOOKBACK_DAYS=7
+# GFW_EVENTS_TIMEOUT_S=90
+
+# Windy Webcams global CCTV layer — free key from https://api.windy.com/webcams/docs
+# WINDY_API_KEY=
+
+# Telegram OSINT map layer — scrapes public t.me/s channel previews (no bot token).
+# TELEGRAM_OSINT_ENABLED=true
+# TELEGRAM_OSINT_CHANNELS=osintdefender,insiderpaper,aljazeeraenglish,nexta_live,war_monitor
+# TELEGRAM_OSINT_TRANSLATE=true
+# TELEGRAM_OSINT_TRANSLATE_TO=en
+
+# Strategic Risk Analytics (experimental derived OSINT — off by default)
+# GT_ANALYTICS_ENABLED=false
+# GT_ANALYTICS_PROFILE=lean
+# On 1 vCPU nodes (fleet VPS), leave disabled or set profile=lean. Scheduled ingest
+# and Louvain clustering stay off until GT_ANALYTICS_ACK_LOW_CPU=true.
+# GT_ANALYTICS_ACK_LOW_CPU=false
+# GT_ANALYTICS_BASE_PRIOR=0.15
+# GT_ANALYTICS_HIGH_RISK_THRESHOLD=0.6
+# GT_ANALYTICS_SIGNAL_WEIGHTS=payroll_loan=3.0,purge=3.5,troop_movement=3.0
+# GT_ANALYTICS_WATCHED_CHANNELS=osintdefender,war_monitor,nexta_live
+# GT_ANALYTICS_LOUVAIN_INTERVAL_MINUTES=30
+
 # Admin key to protect sensitive endpoints (settings, updates).
 # If blank, loopback/localhost requests still work for local single-host dev.
 # Remote/non-loopback admin access requires ADMIN_KEY, or ALLOW_INSECURE_ADMIN=true in debug-only setups.
@@ -39,8 +70,8 @@ ADMIN_KEY=
 # NUFORC_MAPBOX_TOKEN=
 
 # Optional startup-risk controls.
-# On Windows, external curl fallback and the Playwright LiveUAMap scraper are
-# disabled by default so blocked upstream feeds cannot interrupt start.bat.
+# On Windows, external curl fallback is off by default. LiveUAMap uses UI consent
+# when you enable Global Incidents (or set SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=true).
 # SHADOWBROKER_ENABLE_WINDOWS_CURL_FALLBACK=false
 # SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=false
 # AIS starts by default when AIS_API_KEY is set. Set to 0/false to force-disable.
@@ -77,6 +108,19 @@ ADMIN_KEY=
 # pip install earthengine-api
 # GEE_SERVICE_ACCOUNT_KEY=
 
+# Copernicus CDSE — Sentinel-2 imagery (Settings → Imagery, or backend .env).
+# Free OAuth app at https://dataspace.copernicus.eu/
+# SENTINEL_CLIENT_ID=
+# SENTINEL_CLIENT_SECRET=
+
+# Sentinel-2 road corridor freight trends (DrishX engine port — opt-in slow layer).
+# pip install -e backend[road-corridor]  (or uv sync --extra road-corridor)
+# ROAD_CORRIDOR_SAT_ENABLED=false
+# ROAD_CORRIDOR_SCHEDULED_PRESETS=laredo_i35
+# ROAD_CORRIDOR_MONTHS=2
+# ROAD_CORRIDOR_MAX_FRAMES=6
+# ROAD_CORRIDOR_REFRESH_HOURS=24
+
 # Override the backend URL the frontend uses (leave blank for auto-detect)
 # NEXT_PUBLIC_API_URL=http://192.168.1.50:8000
 
@@ -128,8 +172,14 @@ ADMIN_KEY=
 # MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI=backend/../ops/root_transparency_ledger.json
 
 # ── Self Update ────────────────────────────────────────────────
+# Optional ZIP updater digest pin. The updater checks this first, then
+# backend/data/release_digests.json, then the release SHA256SUMS.txt asset.
 # MESH_UPDATE_SHA256=
 
+# Optional strict nonce-only frontend CSP. Leave unset unless the exact build
+# has been verified to hydrate cleanly in your deployment.
+# SHADOWBROKER_STRICT_CSP=1
+
 # ── Wormhole (Local Agent) ─────────────────────────────────────
 # WORMHOLE_URL=http://127.0.0.1:8787
 # WORMHOLE_TRANSPORT=direct

.github/pull_request_template.md

@@ -0,0 +1,13 @@
+## Summary
+
+<!-- What changed and why (1–3 bullets). -->
+
+## Test plan
+
+- [ ] <!-- How you verified the change -->
+
+## Production hardening (data path / fetchers / unattended deploys only)
+
+If this PR touches the data path, fetchers, or live-data APIs, walk through [docs/production-hardening.md](https://github.com/BigBodyCobain/Shadowbroker/blob/main/docs/production-hardening.md) and note any N/A items here.
+
+- [ ] Checklist reviewed (or N/A — explain why)

.gitignore

@@ -109,6 +109,9 @@ backend/data/*
 # release. Used ONLY on first-ever startup to bootstrap carrier_cache.json;
 # after that the cache reflects this install's own GDELT observations.
 !backend/data/carrier_seed.json
+# DrishX RF model weights (MIT — see backend/third_party/drishx/NOTICE.md)
+!backend/data/drishx/
+!backend/data/drishx/rf_model.pickle
 
 # OS generated files
 .DS_Store
@@ -174,6 +177,8 @@ frontend/eslint-report.json
 .git_backup/
 local-artifacts/
 release-secrets/
+release-staging/
+.tmp-release-inspect/
 shadowbroker_repo/
 frontend/src/components.bak/
 frontend/src/components/map/icons/backups/
@@ -198,6 +203,8 @@ graphify-out/
 # Internal docs & brainstorming (never commit)
 # ========================
 docs/*
+!docs/OUTBOUND_DATA.md
+!docs/production-hardening.md
 !docs/mesh/
 docs/mesh/*
 !docs/mesh/threat-model.md
@@ -256,6 +263,11 @@ frontend/.desktop-export-stash-*/
 backend/data/wormhole_stderr.log
 backend/data/wormhole_stdout.log
 
+# Hermes Agent (operator-local runtime install — not project source)
+.hermes/
+**/.hermes/
+hermes-agent/
+
 # Runtime caches that already slip through the backend/data/* blanket
 # (these are caught by the wildcard but listing for clarity)
 

.gitlab-ci.yml

@@ -13,13 +13,22 @@
 #   2. Reverse-mirrors main back to GitHub (only if commits land directly
 #      on GitLab) so the two sources stay in sync.
 #
+# Pipelines on this repo were instant-failing for free-tier accounts until
+# identity verification was added — the May 2026 bump in this comment is
+# the marker commit that confirms runner allocation after verification.
+#
 # Auth notes:
 #   - The image build/push uses $CI_JOB_TOKEN, which GitLab provides
 #     automatically. No credentials need to be configured.
-#   - The reverse mirror requires a GitHub personal access token stored
-#     as the GitLab CI/CD variable GITHUB_MIRROR_TOKEN (Protected + Masked).
-#     Scope: public_repo (or repo for private). If the variable isn't
-#     set the mirror job is skipped — image builds still run.
+#   - The reverse mirror authenticates to GitHub via a per-repo SSH
+#     deploy key. The private half is stored as the File-type GitLab
+#     CI/CD variable GITHUB_MIRROR_SSH_KEY (Protected). The matching
+#     public key is added to github.com/BigBodyCobain/Shadowbroker/
+#     settings/keys with write access. This is a tighter-scoped
+#     replacement for a personal access token: it can ONLY push to
+#     Shadowbroker, never expires, and rotating it is a one-click
+#     delete on GitHub's deploy-keys page. If the variable isn't set,
+#     the mirror job is skipped — image builds still run.
 
 stages:
   - build
@@ -48,7 +57,11 @@ variables:
     - docker info
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
     - docker run --privileged --rm tonistiigi/binfmt --install all
-    - docker buildx create --use --name multiarch --driver docker-container
+    # buildx --driver docker-container can't read TLS from the env vars
+    # the GitLab dind service exports. Wrap them in a docker context and
+    # bind buildx to it. See https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-buildx
+    - docker context create tls-env
+    - docker buildx create --use --name multiarch --driver docker-container tls-env
 
 # ── Backend image ────────────────────────────────────────────────────────
 build-backend:
@@ -93,18 +106,35 @@ build-frontend:
         - .gitlab-ci.yml
 
 # ── Reverse mirror to GitHub ─────────────────────────────────────────────
-# Pushes refs/heads/main to github.com/BigBodyCobain/Shadowbroker.
-# Fast-forward-only — if GitLab main and GitHub main have diverged, this
-# fails loudly rather than silently overwriting either side.
+# Pushes refs/heads/main to github.com/BigBodyCobain/Shadowbroker via SSH
+# using a per-repo deploy key. Fast-forward-only by default — if GitLab
+# main and GitHub main have diverged, the push fails loudly rather than
+# silently overwriting either side.
 #
-# Only runs if GITHUB_MIRROR_TOKEN is set as a CI/CD variable. See the
-# header comment of this file for setup instructions.
+# Only runs if GITHUB_MIRROR_SSH_KEY is set as a File-type CI/CD variable.
+# See the header comment of this file for setup instructions.
 mirror-to-github:
   stage: mirror
   image: alpine:3.20
   needs: []
   before_script:
     - apk add --no-cache git openssh-client ca-certificates
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    # Install the deploy key. File-type CI variable exposes the path; copy
+    # to ~/.ssh/id_ed25519 with restrictive perms so ssh accepts it.
+    - cp "$GITHUB_MIRROR_SSH_KEY" ~/.ssh/id_ed25519
+    - chmod 600 ~/.ssh/id_ed25519
+    # Pin github.com's current host keys so we never trust a man-in-the-
+    # middle. Sourced from https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints
+    # (rotated 2023-03-24 after the previous RSA key leak).
+    - |
+      cat > ~/.ssh/known_hosts <<'EOF'
+      github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
+      github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
+      github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
+      EOF
+    - chmod 644 ~/.ssh/known_hosts
   script:
     - git config --global user.email "ci-mirror@gitlab.com"
     - git config --global user.name "GitLab CI Mirror"
@@ -115,7 +145,7 @@ mirror-to-github:
     - cd repo
     - >
       git push
-      "https://x-access-token:${GITHUB_MIRROR_TOKEN}@github.com/BigBodyCobain/Shadowbroker.git"
+      "git@github.com:BigBodyCobain/Shadowbroker.git"
       "${CI_COMMIT_SHA}:refs/heads/main"
   rules:
-    - if: $CI_COMMIT_BRANCH == "main" && $GITHUB_MIRROR_TOKEN
+    - if: $CI_COMMIT_BRANCH == "main" && $GITHUB_MIRROR_SSH_KEY

DATA-ATTRIBUTION.md

@@ -44,7 +44,8 @@ These sources have their own terms; consult each link before redistributing.
 | aisstream.io | https://aisstream.io | Free-tier API terms (attribution required) | AIS vessel positions |
 | Global Fishing Watch | https://globalfishingwatch.org | CC BY 4.0 (for public data) | Fishing activity events |
 | Microsoft Planetary Computer | https://planetarycomputer.microsoft.com | Sentinel-2 / ESA Copernicus terms | Sentinel-2 imagery |
-| Copernicus CDSE (Sentinel Hub) | https://dataspace.copernicus.eu | ESA Copernicus open data terms | SAR + optical imagery |
+| Copernicus CDSE (Sentinel Hub) | https://dataspace.copernicus.eu | ESA Copernicus open data terms | SAR + optical imagery, optional road-corridor truck trends |
+| DrishX / Fisser et al. 2022 | https://github.com/sparkyniner/DRISH-X-Satellite-powered-freight-intelligence- | MIT (engine); research methodology attribution | Sentinel-2 motion-smear truck detection on major roads (opt-in) |
 | Shodan | https://www.shodan.io | Operator-supplied API key, Shodan ToS | Internet device search |
 | Smithsonian GVP | https://volcano.si.edu | Attribution required | Volcanoes |
 | OpenAQ | https://openaq.org | CC BY 4.0 | Air quality stations |

README.md

@@ -19,7 +19,7 @@
 
 **ShadowBroker** is a decentralized intelligence platform that aggregates real-time, multi-domain OSINT telemetry from 60+ live intelligence feeds into a single dark-ops map interface. Aircraft, ships, satellites, conflict zones, CCTV networks, GPS jamming, internet-connected devices, police scanners, mesh radio nodes, and breaking geopolitical events — all updating in real time on one screen as well as an obfuscated communications protocol and information exchange infrastructure.
 
-Built with **Next.js**, **MapLibre GL**, **FastAPI**, and **Python**. 35+ toggleable data layers, including SAR ground-change detection. Multiple visual modes (DEFAULT / SATELLITE / FLIR / NVG / CRT). Right-click any point on Earth for a country dossier, head-of-state lookup, and the latest Sentinel-2 satellite photo. No user data is collected or transmitted — the dashboard runs entirely in your browser against a self-hosted backend.
+Built with **Next.js**, **MapLibre GL**, **FastAPI**, and **Python**. 40+ toggleable data layers, including SAR ground-change detection, **Telegram OSINT** (public channel previews geoparsed onto the map), a **server-side recon toolkit** (DNS, WHOIS, sanctions, BGP, IP sweep, and more), supply-chain risk overlays, and malware/C2 + CISA KEV cyber threat feeds. Multiple visual modes (DEFAULT / SATELLITE / FLIR / NVG / CRT). Right-click any point on Earth for a country dossier, head-of-state lookup, entity-graph expansion, and the latest Sentinel-2 satellite photo. ShadowBroker has no accounts, product telemetry, or analytics; the dashboard talks to your self-hosted backend. Sensitive recon and Shodan queries never hit third-party APIs from the browser — they are proxied through the backend with SSRF guards and local-operator auth. The **OpenClaw / agent command channel** exposes the same recon backends plus full telemetry search — no separate API integration required.
 
 Designed for analysts, researchers, radio operators, and anyone who wants to see what the world looks like when every public signal is on the same map.
 
@@ -28,18 +28,20 @@ Designed for analysts, researchers, radio operators, and anyone who wants to see
 
 A surprising amount of global telemetry is already public — aircraft ADS-B broadcasts, maritime AIS signals, satellite orbital data, earthquake sensors, mesh radio networks, police scanner feeds, environmental monitoring stations, internet infrastructure telemetry, and more. This data is scattered across dozens of tools and APIs. ShadowBroker combines all of it into a single interface.
 
-The project does not introduce new surveillance capabilities — it aggregates and visualizes existing public datasets. It is fully open-source so anyone can audit exactly what data is accessed and how. No user data is collected or transmitted — everything runs locally against a self-hosted backend. No telemetry, no analytics, no accounts.
+The project does not introduce new surveillance capabilities — it aggregates and visualizes existing public datasets. It is fully open-source so anyone can audit exactly what data is accessed and how. ShadowBroker does not include product telemetry, analytics, or accounts. Operator-supplied keys stay in your local deployment, but live OSINT features necessarily make outbound requests to the public data providers you enable or query.
 
-### Shodan Connector
+### Shodan & Recon (security-first)
 
-ShadowBroker includes an optional Shodan connector for operator-supplied API access. Shodan results are fetched with your own `SHODAN_API_KEY`, rendered as a local investigative overlay (not merged into core feeds), and remain subject to Shodan’s terms of service.
+ShadowBroker includes an optional **Shodan connector** for operator-supplied API access (`SHODAN_API_KEY`) and a **Recon Toolkit** panel for keyless OSINT lookups. Both run **server-side only**: the browser calls your self-hosted `/api/osint/*` and `/api/tools/shodan/*` routes; outbound requests are made by the backend after SSRF validation. Recon requires **local-operator** access (same trust model as layer toggles and admin routes). Shodan results render as a separate map overlay and remain subject to Shodan’s terms of service.
+
+> **Not included:** embedded live-news YouTube grids or a built-in Gemini AI analyst panel — use the **OpenClaw / agent channel** for AI-assisted analysis instead.
 
 ---
 
 ## Interesting Use Cases
 
 * **Track Air Force One**, the private jets of billionaires and dictators, and every military tanker, ISR, and fighter broadcasting ADS-B. Air Force One and all of the accompanying Presidential/Vice Presidential planes are highlighted and monitored from the moment they leave the ground. 
-* **Connect an AI agent as a co-analyst** through ShadowBroker's HMAC-signed agentic command channel — supports OpenClaw and any other agent that speaks the protocol (Claude, GPT, LangChain, custom). The agent gets full read/write access to all 35+ data layers, pin placement, map control, SAR ground-change, mesh networking, and alert delivery. It sees everything the operator sees and can take actions on the map in real time.
+* **Connect an AI agent as a co-analyst** through ShadowBroker's HMAC-signed agentic command channel — supports OpenClaw and any other agent that speaks the protocol (Claude, GPT, LangChain, custom). The agent gets full read/write access to all 40+ data layers, compact cross-layer search (`search_telemetry`, `search_news`), the full recon toolkit (`osint_lookup` for IP/DNS/WHOIS/sanctions/CVE/etc.), entity-graph expansion, pin placement, map control, SAR ground-change, mesh networking, and alert delivery. It sees everything the operator sees and can take actions on the map in real time.
 * **Communicate on the InfoNet testnet** — The first decentralized intelligence mesh built into an OSINT tool. Obfuscated messaging with gate personas, Dead Drop peer-to-peer exchange, and a built-in terminal CLI. No accounts, no signup. Privacy is not guaranteed yet — this is an experimental testnet — but the protocol is live and being hardened.
 * **Right-click anywhere on Earth** for a country dossier (head of state, population, languages), Wikipedia summary, and the latest Sentinel-2 satellite photo at 10m resolution
 * **Click a KiwiSDR node** and tune into live shortwave radio directly in the dashboard. Click a police scanner feed and eavesdrop in one click.
@@ -55,6 +57,12 @@ ShadowBroker includes an optional Shodan connector for operator-supplied API acc
 * **Track trains** across the US (Amtrak) and Europe (DigiTraffic) in real time
 * **Estimate where US aircraft carriers are** using automated GDELT news scraping — no other open tool does this
 * **Search internet-connected devices worldwide** via Shodan — cameras, SCADA systems, databases — plotted as a live overlay on the map
+* **Run a full recon toolkit** from the left sidebar — IP geolocation, DNS, RDAP/WHOIS, certificate transparency, BGP/ASN, OFAC sanctions search, CVE lookup, Tor/OTX threat checks, and subnet sweeps (InternetDB proxied server-side)
+* **Expand an entity graph** when you select an aircraft, vessel, company, or IP — Wikidata + OFAC + live store cross-links rendered in the Entity Graph panel
+* **Monitor supply-chain risk** — Tier 1/2 semiconductor and battery fabs scored against nearby earthquakes, wildfires, and conflict events (SCM panel)
+* **Toggle malware C2 hotspots** — abuse.ch Feodo Tracker + URLhaus feeds mapped by country (opt-in layer)
+* **Monitor Telegram OSINT channels** — public `t.me/s` war/conflict feeds (OSINTdefender, NEXTA, etc.) scraped hourly, risk-scored, geoparsed to metro anchors, and plotted as clickable map pins with inline media
+* **Overlay global submarine cables** — static TeleGeography-derived cable routes (opt-in layer)
 
 
 ---
@@ -83,6 +91,8 @@ Both paths produce identical containers — same source, same CI, same images by
 
 Open `http://localhost:3000` to view the dashboard! *(Requires [Docker Desktop](https://www.docker.com/products/docker-desktop/) or Docker Engine)*
 
+> **Join the private InfoNet swarm (sb-testnet-0):** Click **NODE** in the dashboard, or run `./meshnode.sh` for a headless participant. No manual peer list — fleet defaults discover the seed and pull the signed manifest automatically. Set `MESH_INFONET_FLEET_JOIN=false` in `.env` for a private solo node.
+
 > **Backend port already in use?** The browser only needs port `3000`, but the backend API is also published on host port `8000` for local diagnostics. If another app already uses `8000`, create or edit `.env` next to `docker-compose.yml` and set `BACKEND_PORT=8001`, then run `docker compose up -d`.
 
 > **Blank news/UAP/bases/wastewater after several minutes?** Check for backend OOM restarts with `docker events --since 30m --filter container=shadowbroker-backend --filter event=oom`. The default compose file gives the backend 4GB; if your host has less memory, reduce enabled feeds or set `BACKEND_MEMORY_LIMIT=3G` and expect slower/heavier layers to warm more gradually.
@@ -113,6 +123,20 @@ That's it. `pull` grabs the latest images, `up -d` restarts the containers.
 >
 > Podman users should run the equivalent provider command, for example `podman-compose pull` and `podman-compose up -d`, or use `./compose.sh --engine podman pull` and `./compose.sh --engine podman up -d` from a bash-compatible shell.
 
+### Update Integrity
+
+Docker updates are delivered through signed container registries. The legacy ZIP self-updater verifies release archives through this chain, in order:
+
+* `MESH_UPDATE_SHA256` when an operator pins a digest explicitly.
+* `backend/data/release_digests.json` for bundled release pins.
+* The release `SHA256SUMS.txt` asset on GitHub when a bundled pin is not present.
+
+Release maintainers should run `python backend/scripts/release_helper.py hash <ShadowBroker_vX.Y.Z.zip>` before publishing, then publish `SHA256SUMS.txt` and update `backend/data/release_digests.json` when shipping a ZIP updater target. The updater keeps the operator override path intact instead of failing closed on missing bundled digests, so existing installs do not get stranded by a release-process mistake.
+
+### CSP Hardening
+
+The production frontend ships with a hydration-compatible CSP and a strict nonce-only CSP in `Content-Security-Policy-Report-Only`. Set `SHADOWBROKER_STRICT_CSP=1` only after verifying the exact build hydrates correctly in your deployment. Runtime Google Fonts are not required; the bundled Next font pipeline serves the dashboard font from the app build.
+
 ### ⚠️ **Stuck on the old version?**
 
 **If `git pull` fails or `docker compose up` keeps building from source instead of pulling images**, your clone predates a March 2026 repository migration that rewrote commit history. A normal `git pull` cannot fix this. Run:
@@ -174,7 +198,7 @@ ShadowBroker v0.9.7 ships **InfoNet** (decentralized intelligence mesh + Soverei
 | Channel | Privacy Status | Details |
 |---|---|---|
 | **Meshtastic / APRS** | **PUBLIC** | RF radio transmissions are public and interceptable by design. |
-| **InfoNet Gate Chat** | **OBFUSCATED** | Messages are obfuscated with gate personas and canonical payload signing, but NOT end-to-end encrypted. Metadata is not hidden. |
+| **InfoNet Gate Chat** | **OBFUSCATED** | Messages are obfuscated with gate personas and canonical payload signing, but NOT end-to-end encrypted. Metadata is not hidden despite being designed through Tor and Reticulum (Work in progress). |
 | **Dead Drop DMs** | **STRONGEST CURRENT LANE** | Token-based epoch mailbox with SAS word verification. Strongest lane in this build, but not yet confidently private. |
 | **Sovereign Shell governance** | **PUBLIC LEDGER** | Petitions, votes, upgrade hashes, and dispute stakes are signed events on a public hashchain. Pseudonymous via gate persona, but governance actions are intentionally observable. |
 | **Privacy primitives (RingCT / stealth / DEX)** | **NOT YET WIRED** | Locked Protocol contracts are in place, but the cryptographic scheme has not been chosen. The privacy-core Rust crate is the integration target for a future sprint. |
@@ -199,7 +223,7 @@ The first decentralized intelligence communication and governance layer built di
 
 **Communication layer (since v0.9.6):**
 
-* **InfoNet Experimental Testnet** — A global, obfuscated message relay. Anyone running ShadowBroker can transmit and receive on the InfoNet. Messages pass through a Wormhole relay layer with gate personas, Ed25519 canonical payload signing, and transport obfuscation.
+* **InfoNet Experimental Testnet** — A global, obfuscated message relay using Tor and Reticulum. Anyone running ShadowBroker can transmit and receive on the InfoNet. Messages pass through a Wormhole relay layer with gate personas, Ed25519 canonical payload signing, and transport obfuscation.
 * **Mesh Chat Panel** — Three-tab interface: **INFONET** (gate chat with obfuscated transport), **MESH** (Meshtastic radio integration), **DEAD DROP** (peer-to-peer message exchange with token-based epoch mailboxes — strongest current lane).
 * **Gate Persona System** — Pseudonymous identities with Ed25519 signing keys, prekey bundles, SAS word contact verification, and abuse reporting.
 * **Mesh Terminal** — Built-in CLI: `send`, `dm`, market commands, gate state inspection. Draggable panel, minimizes to the top bar. Type `help` to see all commands.
@@ -219,17 +243,34 @@ The first decentralized intelligence communication and governance layer built di
 
 **Privacy primitive runway (NEW in v0.9.7):**
 
-* **Function Keys — Anonymous Citizenship Proof** — A citizen proves "I am an Infonet citizen" without revealing their Infonet identity. 5 of 6 pieces shipped: nullifiers, challenge-response, two-phase commit receipts, enumerated denial codes, batched settlement. Issuance via blind signatures waits on a primitive decision (RSA blind sigs vs BBS+ vs U-Prove vs Idemix).
+* **Function Keys — Anonymous Credential Scaffolding** — The plumbing is in place for nullifiers, challenge-response, two-phase commit receipts, enumerated denial codes, and batched settlement. Today's challenge-response is an HMAC-based placeholder for integration testing, not a production anonymous or zero-knowledge citizenship proof. True unlinkable issuance still waits on a primitive decision (RSA blind sigs vs BBS+ vs U-Prove vs Idemix).
 * **Locked Protocol Contracts** — Stable interfaces in `services/infonet/privacy/contracts.py` for ring signatures, stealth addresses, Pedersen commitments, range proofs, and DEX matching. The `privacy-core` Rust crate is the integration target — no caller of the privacy module needs to know which scheme is active.
 * **Sprint 11+ Path** — When the cryptographic scheme is chosen, primitives wire into the locked Protocols without API churn.
 
 > **Experimental Testnet — No Privacy Guarantee:** InfoNet messages are obfuscated but NOT end-to-end encrypted. The Mesh network (Meshtastic/APRS) is NOT private — radio transmissions are inherently public. The privacy primitive contracts are scaffolded but not yet wired. Do not send anything sensitive on any channel. Treat all channels as open and public for now.
 
-### 🔍 Shodan Device Search (NEW in v0.9.6)
+### 🔍 Recon Toolkit & Shodan (Osiris-derived, security-first)
 
-* **Internet Device Search** — Query Shodan directly from ShadowBroker. Search by keyword, CVE, port, or service — results plotted as a live overlay on the map
+Adapted from the [OSIRIS](https://github.com/simplifaisoul/osiris) recon stack (MIT) with ShadowBroker’s proxy model. Attribution: `backend/third_party/osiris/NOTICE.md`.
+
+**Recon Toolkit** (left sidebar — local operator only):
+
+* **IP / DNS / WHOIS** — ip-api.com geolocation, Google DNS-over-HTTPS, RDAP registrant data with optional HTTP security header scoring
+* **Certificates & BGP** — crt.sh subdomain discovery, bgpview.io ASN/prefix lookups
+* **Threat intel** — AlienVault OTX pulses, Tor exit-node checks, optional per-IP/domain reputation
+* **Sanctions** — OpenSanctions `us_ofac_sdn` index (CC-BY); cross-checks on WHOIS entities and IP ISP/org strings
+* **CVE / MAC / GitHub / leaks** — MITRE CVE API, MAC vendor lookup, GitHub profile recon, public breach checks
+* **IP sweep** — `/api/osint/sweep/scan` geolocates a target /24–/32 and proxies Shodan InternetDB host discovery server-side (browser never contacts InternetDB directly)
+* **SSRF guard** — Private, loopback, link-local, and metadata hostnames are blocked before any user-supplied fetch
+
+**Entity graph** — Select any map entity to open the Entity Graph panel (`GET /api/entity/expand`). Resolves aircraft, vessels, companies, persons, IPs, and countries into a node/link graph (Wikidata SPARQL + OFAC + in-memory flight/ship store).
+
+**OpenClaw / agent access** — The same recon backends are available on the HMAC command channel (no browser local-operator gate): `osint_lookup` (passive IP/DNS/WHOIS/certs/BGP/sanctions/CVE/MAC/GitHub/leaks/threats), `entity_expand` (relationship graph), and `osint_sweep` (active subnet scan — **full** access tier only). Call `osint_tools` to list supported lookup types. Skill package: `openclaw-skills/shadowbroker/` (`SKILL.md` + `sb_query.py`).
+
+**Shodan overlay** (unchanged):
+
+* **Internet Device Search** — Query Shodan with your own API key; results plotted as a live overlay
 * **Configurable Markers** — Shape, color, and size customization for Shodan results
-* **Operator-Supplied API** — Uses your own `SHODAN_API_KEY`; results rendered as a local investigative overlay
 
 ### 🛩️ Aviation Tracking
 
@@ -317,11 +358,12 @@ The first decentralized intelligence communication and governance layer built di
 
 ### 📷 Surveillance
 
-* **CCTV Mesh** — 11,000+ live traffic cameras from 13 sources across 6 countries:
+* **CCTV Mesh** — 22,000+ live traffic cameras from 21 ingestors across 10 countries (US, UK, Canada, Australia, Austria, Spain, Singapore, Netherlands when NDW feed is up, plus OSM):
   * 🇬🇧 Transport for London JamCams
   * 🇺🇸 NYC DOT, Austin TX (TxDOT)
   * 🇺🇸 California (12 Caltrans districts), Washington State (WSDOT), Georgia DOT, Illinois DOT, Michigan DOT
   * 🇪🇸 Spain DGT National (20 cities), Madrid City (357 cameras via KML)
+  * 🇦🇹 Austria ASFINAG motorway webcams
   * 🇸🇬 Singapore LTA
   * 🌍 Windy Webcams
 * **Feed Rendering** — Automatic detection & rendering of video, MJPEG, HLS, embed, satellite tile, and image feeds
@@ -342,6 +384,12 @@ The first decentralized intelligence communication and governance layer built di
 * **Data Center Mapping** — 2,000+ global data centers plotted from a curated dataset. Clustered purple markers with server-rack icons. Click for operator, location, and automatic internet outage cross-referencing by country.
 * **Military Bases** — Global military installation and missile facility database (NEW)
 * **Power Plants** — 35,000+ global power plants from the WRI database (NEW)
+* **Submarine Cables** — Global undersea cable routes from static TeleGeography-derived GeoJSON (`frontend/public/data/submarine-cables.json`). Opt-in line overlay.
+* **Malware C2 Layer** — Botnet C2 servers (Feodo Tracker) and recent malware URLs (URLhaus) from abuse.ch, refreshed on the slow tier when the layer is enabled.
+* **SCM Supplier Risk** — Tier 1/2 fabs and battery plants (TSMC, Samsung, CATL, etc.) cross-referenced against earthquakes, FIRMS fires, and GDELT conflict proximity. Alerts in the SCM panel; optional map layer.
+* **Cyber Threats Feed** — Recent CISA Known Exploited Vulnerabilities (KEV) entries exposed via `/api/cyber-threats` and the layer toggle.
+* **Country Risk Index** — Static geopolitical risk scores with USGS earthquake enrichment via `/api/country-risk`.
+* **Telegram OSINT** — Public channel web previews (`t.me/s/*`) from configurable war/OSINT feeds. Hourly incremental merge (no redundant re-scrape), keyword risk scoring, Cyrillic/Arabic place aliases, metro-anchor geocoding (separate from news centroids), inline photo/video via `/api/telegram/media` proxy. Layer key: `telegram_osint`.
 
 ### 🌐 Additional Layers & Tools
 
@@ -367,7 +415,9 @@ v0.9.7 turns ShadowBroker from a dashboard a human watches into an intelligence
 
 **Capabilities:**
 
-* **Full Telemetry Access** — The agent queries all 35+ data layers: flights, ships, satellites, SIGINT, conflict events, earthquakes, fires, wastewater, prediction markets, and more. Fast and slow tier endpoints return enriched data with geographic coordinates, timestamps, and source attribution.
+* **Full Telemetry Access** — The agent queries all 40+ data layers: flights, ships, satellites, SIGINT, conflict events, earthquakes, fires, wastewater, **Telegram OSINT**, malware/C2, **CISA KEV cyber threats**, SCM overlays, fishing activity (GFW), prediction markets, and more. Fast and slow tier endpoints return enriched data with geographic coordinates, timestamps, and source attribution.
+* **Compact Search (preferred over full dumps)** — `get_summary` → `get_layer_slice` with per-layer `since_layer_versions` (SSE `layer_changed` push tells the agent exactly which layers updated). `search_telemetry` is the Google-style cross-layer keyword index. `search_news` covers news, GDELT, CrowdThreat, LiveUAMap, frontlines, and Telegram posts. `entities_near`, `brief_area`, `find_flights`/`find_ships`/`find_entity`, and `correlate_entity` answer targeted questions without multi-megabyte pulls.
+* **Recon Toolkit on the Channel** — `osint_lookup` runs the same SSRF-guarded backends as the Recon panel (`ip`, `dns`, `whois`, `certs`, `bgp`, `sanctions`, `cve`, `mac`, `github`, `leaks`, `threats`, `sweep_init`). `entity_expand` builds Wikidata + OFAC relationship graphs. `osint_sweep` runs Shodan InternetDB subnet discovery (**full** tier). Layer aliases: `telegram`, `malware`/`botnet`, `cyber`/`cisa`/`kev`, `scm`/`suppliers`, `gfw`/`fishing`.
 * **AI Intel Pins** — Place color-coded investigation markers directly on the operator's map. 14 pin categories (threat, anomaly, military, maritime, aviation, SIGINT, infrastructure, etc.) with confidence scores, TTL expiry, source URLs, and batch placement up to 100 pins at once.
 * **Map Control** — Fly the operator's map view to any coordinate, trigger satellite imagery lookups, and open region dossiers. The agent can direct the operator's attention to specific locations in real time.
 * **SAR Ground-Change** — Query SAR anomaly feeds, inspect pin details, manage AOIs, and fly the map to watch areas. The agent can monitor for ground deformation, flood extent, or damage and promote anomalies to pins.
@@ -380,7 +430,7 @@ v0.9.7 turns ShadowBroker from a dashboard a human watches into an intelligence
 * **Intelligence Reports** — Generate structured reports with summary stats, top military flights, correlations, earthquake activity, SIGINT counts, and pin inventories.
 * **Auditable** — Every channel call is logged; the operator can introspect what the agent has done.
 
-**Connect an agent:** Open the AI Intel panel in the left sidebar, click **Connect Agent**, and copy the HMAC secret. From there, point any compatible agent at the channel — for OpenClaw, import `ShadowBrokerClient` from the OpenClaw skill package; for any other agent, use the same HMAC contract documented above (timestamp + nonce + body digest, tier-gated). The channel is the protocol, not the agent.
+**Connect an agent:** Open the AI Intel panel in the left sidebar, click **Connect Agent**, and copy the HMAC secret. From there, point any compatible agent at the channel — for OpenClaw, import `ShadowBrokerClient` from `openclaw-skills/shadowbroker/sb_query.py` (see `SKILL.md` for examples); for any other agent, use the same HMAC contract documented above (timestamp + nonce + body digest, tier-gated). Discovery: `GET /api/ai/tools` and `GET /api/ai/capabilities`. The channel is the protocol, not the agent.
 
 ### ⏱️ Time Machine — Snapshot Playback (NEW in v0.9.7)
 
@@ -529,9 +579,20 @@ ShadowBroker v0.9.7 is composed of three vertically-stacked planes — the **Ope
 | [GDELT Project](https://www.gdeltproject.org) | Global conflict events | ~6h | No |
 | [DeepState Map](https://deepstatemap.live) | Ukraine frontline | ~30min | No |
 | [Shodan](https://www.shodan.io) | Internet-connected device search | On-demand | **Yes** |
+| [OpenSanctions](https://www.opensanctions.org) | OFAC SDN sanctions index (recon + entity graph) | 24h cache | No |
+| [abuse.ch Feodo + URLhaus](https://abuse.ch) | Malware C2 / distribution URLs | ~5min (opt-in layer) | No |
+| [CISA KEV](https://www.cisa.gov/known-exploited-vulnerabilities-catalog) | Known exploited CVEs | ~5min (opt-in layer) | No |
+| [ip-api.com](https://ip-api.com) | IP geolocation (recon, entity graph) | On-demand | No |
+| [Google Public DNS](https://dns.google) | DNS-over-HTTPS lookups (recon) | On-demand | No |
+| [RDAP.org](https://rdap.org) | Domain registration data (recon) | On-demand | No |
+| [crt.sh](https://crt.sh) | Certificate transparency (recon) | On-demand | No |
+| [bgpview.io](https://bgpview.io) | BGP/ASN routing (recon) | On-demand | No |
+| TeleGeography (static) | Submarine cable routes | Static | No |
+| [ASFINAG](https://www.asfinag.at) | Austria motorway webcams | ~10min | No |
 | [Amtrak](https://www.amtrak.com) | US train positions | ~60s | No |
 | [DigiTraffic](https://www.digitraffic.fi) | European rail positions | ~60s | No |
-| [Global Fishing Watch](https://globalfishingwatch.org) | Fishing vessel activity events | ~10min | No |
+| [Global Fishing Watch](https://globalfishingwatch.org) | Fishing vessel activity events | ~1hr | **Yes** (`GFW_API_TOKEN`) |
+| [Telegram public previews](https://t.me/s) | War/OSINT channel posts (`telegram_osint`) | ~1hr | No (optional `TELEGRAM_OSINT_CHANNELS`) |
 | Transport for London, NYC DOT, TxDOT | CCTV cameras (UK, US) | ~10min | No |
 | Caltrans, WSDOT, GDOT, IDOT, MDOT | CCTV cameras (5 US states) | ~10min | No |
 | Spain DGT, Madrid City | CCTV cameras (Spain) | ~10min | No |
@@ -563,6 +624,8 @@ ShadowBroker v0.9.7 is composed of three vertically-stacked planes — the **Ope
 | [OSM Nominatim](https://nominatim.openstreetmap.org) | Place name geocoding (LOCATE bar) | On-demand | No |
 | [CARTO Basemaps](https://carto.com) | Dark map tiles | Continuous | No |
 
+**Outbound privacy & audit (#348–#366):** Each self-hosted install uses its own backend IP and per-install User-Agent handle. See [docs/OUTBOUND_DATA.md](docs/OUTBOUND_DATA.md) for what contacts third parties, opt-in/env controls, and accepted tradeoffs (CCTV Referer, basemap CDN, LiveUAMap, etc.).
+
 ---
 
 ## 🚀 Getting Started
@@ -584,9 +647,16 @@ Open `http://localhost:3000` to view the dashboard.
 > **Deploying publicly or on a LAN?** No configuration needed for most setups.
 > The frontend proxies all API calls through the Next.js server to `BACKEND_URL`,
 > which defaults to `http://backend:8000` (Docker internal networking).
-> Host port `8000` is only published for local API/debug access. If it conflicts
-> with another service, set `BACKEND_PORT=8001` in `.env`; leave `BACKEND_URL`
-> as `http://backend:8000` because that is the Docker-internal port.
+> Host port `8000` is only published for local API/debug access (`127.0.0.1:8000`
+> in `docker-compose.yml`). If it conflicts with another service, set
+> `BACKEND_PORT=8001` in `.env`; leave `BACKEND_URL` as `http://backend:8000`
+> because that is the Docker-internal port.
+>
+> **Running the backend outside Docker** (`cd backend && python main.py`):
+> the dev server binds **loopback only** (`127.0.0.1:8000`) so other machines on
+> your LAN cannot hit admin/local-trust routes with an empty `ADMIN_KEY`. Set
+> `SHADOWBROKER_DEV_BIND_ALL=true` in `.env` only when you deliberately need
+> `0.0.0.0` and use a strong `ADMIN_KEY` for any non-local callers.
 > The backend memory cap is controlled by `BACKEND_MEMORY_LIMIT` and defaults
 > to `4G`. If Docker reports OOM events, the backend will restart and slow
 > layers can look empty until they repopulate.
@@ -798,7 +868,7 @@ AIS-catcher decodes VHF radio signals on 161.975 MHz and 162.025 MHz and POSTs d
 
 ## 🎛️ Data Layers
 
-All 37 layers are independently toggleable from the left panel:
+All 41 layers are independently toggleable from the left panel:
 
 | Layer | Default | Description |
 |---|---|---|
@@ -840,6 +910,24 @@ All 37 layers are independently toggleable from the left panel:
 | VIIRS Nightlights | ❌ OFF | Night-time light change detection |
 | Power Plants | ❌ OFF | 35,000+ global power plants |
 | Shodan Overlay | ❌ OFF | Internet device search results |
+| Road Freight Trends | ❌ OFF | Sentinel-2 truck-motion trends on major highways (Analyze Here) |
+| Submarine Cables | ❌ OFF | Global undersea cable routes (static GeoJSON) |
+| Malware C2 | ❌ OFF | abuse.ch Feodo + URLhaus threat points |
+| SCM Suppliers | ❌ OFF | Tier 1/2 supply-chain risk markers + panel alerts |
+| Cyber Threats | ❌ OFF | Recent CISA KEV entries (stats in slow-tier payload) |
+| Telegram OSINT | ✅ ON | Public war/OSINT Telegram channels — hourly scrape, geoparsed pins |
+| SAR | ✅ ON | Synthetic aperture radar catalog + anomaly alerts |
+
+**Recon & entity tools** (not map layers — left sidebar / selection):
+
+| Tool | Dashboard access | OpenClaw command | Description |
+|---|---|---|---|
+| Recon Toolkit | Local operator (`/api/osint/*`) | `osint_lookup`, `osint_sweep`† | IP, DNS, WHOIS, certs, BGP, sanctions, CVE, MAC, GitHub, leaks, threats, subnet sweep |
+| Entity Graph | Local operator (`/api/entity/expand`) | `entity_expand` | Wikidata + OFAC + live-store relationship graph |
+| SCM Risk panel | Local operator (`/api/scm-suppliers`) | `get_layer_slice(["scm_suppliers"])` | Supplier threat rollup + map markers |
+| Tool discovery | — | `osint_tools` | Lists recon lookup types and entity-expand schemas |
+
+† `osint_sweep` (active InternetDB scan) requires `OPENCLAW_ACCESS_TIER=full`.
 
 ---
 
@@ -863,6 +951,7 @@ The platform is optimized for handling massive real-time datasets:
 
 ```
 Shadowbroker/
+├── openclaw-skills/shadowbroker/   # OpenClaw skill — SKILL.md, sb_query.py client, alerts/monitor helpers
 ├── backend/
 │   ├── main.py                     # FastAPI app, middleware, API routes (~4,000 lines)
 │   ├── cctv.db                     # SQLite CCTV camera database (auto-generated)
@@ -872,7 +961,18 @@ Shadowbroker/
 │   │   ├── data_fetcher.py         # Core scheduler — orchestrates all data sources
 │   │   ├── ais_stream.py           # AIS WebSocket client (25K+ vessels)
 │   │   ├── carrier_tracker.py      # OSINT carrier position estimator (GDELT news scraping)
-│   │   ├── cctv_pipeline.py        # 13-source CCTV camera ingestion pipeline
+│   │   ├── cctv_pipeline.py        # 14-source CCTV camera ingestion pipeline
+│   │   ├── ssrf_guard.py           # SSRF validation for operator recon fetches
+│   │   ├── sanctions/ofac.py       # OpenSanctions OFAC SDN index
+│   │   ├── osint/lookups.py        # Server-side recon lookups (Osiris port)
+│   │   ├── osint/openclaw_recon.py # OpenClaw dispatch for recon + entity_expand
+│   │   ├── osint_intel/resolve.py  # Entity graph resolver (Wikidata + OFAC)
+│   │   ├── scm/suppliers.py        # Supply-chain risk overlay
+│   │   ├── intel_feeds/            # Country risk index helpers
+│   │   ├── fetchers/malware.py     # abuse.ch Feodo + URLhaus
+│   │   ├── fetchers/cyber_status.py # CISA KEV feed
+│   │   ├── fetchers/telegram_osint.py # Public Telegram channel scrape + geoparse
+│   │   ├── third_party/osiris/     # MIT attribution for Osiris-derived code
 │   │   ├── geopolitics.py          # GDELT + Ukraine frontline + air alerts
 │   │   ├── region_dossier.py       # Right-click country/city intelligence
 │   │   ├── radio_intercept.py      # Police scanner feeds + OpenMHZ
@@ -910,7 +1010,14 @@ Shadowbroker/
 │   │       ├── mesh_reputation.py  # Node reputation scoring
 │   │       ├── mesh_oracle.py      # Oracle consensus protocol
 │   │       └── mesh_secure_storage.py # Secure credential storage
+│   ├── routers/
+│   │   ├── osint.py                # /api/osint/* recon routes (local operator)
+│   │   ├── entity_graph.py         # /api/entity/expand
+│   │   ├── scm.py                  # /api/scm-suppliers
+│   │   └── intel_feeds.py          # /api/malware, /api/cyber-threats, /api/telegram-feed, /api/country-risk
 ├── frontend/
+│   ├── public/data/
+│   │   └── submarine-cables.json   # Static undersea cable GeoJSON
 │   ├── src/
 │   │   ├── app/
 │   │   │   └── page.tsx            # Main dashboard — state, polling, layout
@@ -919,7 +1026,12 @@ Shadowbroker/
 │   │       ├── MeshChat.tsx        # InfoNet / Mesh / Dead Drop chat panel
 │   │       ├── MeshTerminal.tsx    # Draggable CLI terminal
 │   │       ├── NewsFeed.tsx        # SIGINT feed + entity detail panels
-│   │       ├── WorldviewLeftPanel.tsx   # Data layer toggles (35+ layers)
+│   │       ├── WorldviewLeftPanel.tsx   # Data layer toggles (40+ layers)
+│   │       ├── ShodanPanel.tsx          # Shodan device search overlay
+│   │       ├── ReconPanel.tsx           # Server-side OSINT recon toolkit
+│   │       ├── ScmPanel.tsx             # Supply-chain risk command panel
+│   │       ├── EntityGraphPanel.tsx     # Entity graph on map selection
+│   │       ├── MaplibreViewer/popups/TelegramOsintPopup.tsx  # Threat-intercept styled Telegram pin popups
 │   │       ├── WorldviewRightPanel.tsx  # Search + filter sidebar
 │   │       ├── AdvancedFilterModal.tsx  # Airport/country/owner filtering
 │   │       ├── MapLegend.tsx       # Dynamic legend with all icons
@@ -956,6 +1068,9 @@ MESH_SAR_EARTHDATA_TOKEN=                     # NASA Earthdata token (paired wit
 MESH_SAR_COPERNICUS_USER=                     # Copernicus Data Space user (SAR Mode B — EGMS / EMS)
 MESH_SAR_COPERNICUS_TOKEN=                    # Copernicus token (paired with user above)
 OPENCLAW_ACCESS_TIER=restricted               # OpenClaw agent tier: "restricted" (read-only) or "full"
+GFW_API_TOKEN=your_gfw_token                  # Global Fishing Watch — fishing_activity layer (Settings → Maritime)
+TELEGRAM_OSINT_ENABLED=true                   # Telegram OSINT layer (default on)
+TELEGRAM_OSINT_CHANNELS=osintdefender,...     # Comma-separated public channel slugs (see .env.example)
 
 # Private-lane privacy-core pinning (required when Arti or RNS is enabled)
 PRIVACY_CORE_MIN_VERSION=0.1.0

backend/.env.example

@@ -18,6 +18,15 @@ AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
 # AISHUB_USERNAME=
 # AISHUB_POLL_INTERVAL_MINUTES=20
 
+# `python main.py` (uvicorn reload) binds 127.0.0.1:8000 by default so LAN clients
+# cannot reach a dev server with empty ADMIN_KEY (#375). Set true only when you
+# intentionally need 0.0.0.0 and understand the local-trust implications.
+# SHADOWBROKER_DEV_BIND_ALL=false
+#
+# Thread pool for GDELT, LiveUAMap, CCTV ingest, and slow-tier refresh batches.
+# Keeps heavy jobs from starving fast flight/ship workers (default 2).
+# SHADOWBROKER_HEAVY_FETCH_WORKERS=2
+
 # Override allowed CORS origins (comma-separated). Defaults to localhost + LAN auto-detect.
 # CORS_ORIGINS=http://192.168.1.50:3000,https://my-domain.com
 
@@ -31,11 +40,9 @@ AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
 # Requires MESH_DEBUG_MODE=true; do not enable this for ordinary use.
 # ALLOW_INSECURE_ADMIN=false
 
-# Per-install operator handle. Round 7a: every outbound third-party API
-# call (Wikipedia, Wikidata, Nominatim, GDELT, OpenMHz, Broadcastify,
-# weather.gov, NUFORC, etc.) includes this handle in the User-Agent so
-# upstreams can rate-limit / contact the specific install instead of
-# treating every Shadowbroker user as one entity.
+# Per-install operator handle. Round 7a: outbound third-party API calls send
+# this handle as the User-Agent (e.g. operator-7f3a92), not a shared app name,
+# so upstreams rate-limit one install instead of blocking every user.
 #
 # Default empty -> a stable pseudonymous handle (e.g. "operator-7f3a92") is
 # auto-generated on first run and persisted to backend/data/operator_handle.json.
@@ -43,10 +50,8 @@ AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
 # set it here. Special characters are sanitized to dashes.
 # OPERATOR_HANDLE=
 
-# Default outbound User-Agent for all third-party HTTP fetchers. Operators
-# who run a public relay and want a completely custom UA can set this; it
-# bypasses the per-operator helper entirely. Most installs should leave it
-# unset and use OPERATOR_HANDLE instead.
+# Full User-Agent override (replaces the operator handle entirely). Rare;
+# most installs should use OPERATOR_HANDLE only.
 # SHADOWBROKER_USER_AGENT=
 
 # Nominatim-specific User-Agent override (OSM usage policy). Leave unset to
@@ -66,20 +71,48 @@ AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
 # FIMI_ENABLED=false
 #
 # Polymarket + Kalshi — US political/election prediction markets.
+# Default off; enable from Global Threat Intercept (MKT toggle) or set true here.
 # PREDICTION_MARKETS_ENABLED=false
+# When enabled, polls use a jittered schedule (not the fixed 5-minute slow tier):
+# PREDICTION_MARKETS_INTERVAL_MINUTES=7
+# PREDICTION_MARKETS_SCHEDULER_JITTER_S=240
+# PREDICTION_MARKETS_INITIAL_DELAY_MAX_S=180
+# PREDICTION_MARKETS_PRE_FETCH_JITTER_S=90
+# PREDICTION_MARKETS_PROVIDER_GAP_JITTER_S=45
+# MESH_POLYMARKET_PAGE_DELAY_JITTER_S=0.08
+# MESH_KALSHI_PAGE_DELAY_JITTER_S=0.2
 #
 # Finnhub fallback / yfinance — financial market data.
 # Set FINNHUB_API_KEY to enable Finnhub, or set FINANCIAL_ENABLED=true to allow
 # the unauthenticated yfinance fallback to call Yahoo Finance.
 # FINANCIAL_ENABLED=false
 #
-# NUFORC UAP sightings — huggingface.co dataset download.
+# NUFORC UAP map layer — live scrape from nuforc.org (rolling window, default 60 days).
+# Refreshed weekly (Mon 12:00 UTC); cache reused for up to 7 days between runs.
+# NUFORC_RECENT_DAYS=60
+# NUFORC_CACHE_TTL_HOURS=168
+# On Windows, live scrape uses Python requests by default; optional:
+# SHADOWBROKER_ENABLE_WINDOWS_CURL_FALLBACK=true
+# NUFORC enrichment index (HF dataset) is separate — opt-in only:
 # NUFORC_ENABLED=false
 #
 # News RSS aggregator — defaults ON. Set to "false" to disable all
 # configured news feeds (kill switch for the news layer).
 # NEWS_ENABLED=true
 
+# Global Fishing Watch — fishing vessel activity events (Fishing Activity map layer).
+# Free API token from https://globalfishingwatch.org/our-apis/tokens
+# Without this the fishing_activity layer stays empty.
+# GFW_API_TOKEN=
+# Optional tuning — GFW can return 40k+ global events; defaults cap fetch for map paint.
+# GFW_EVENTS_PAGE_SIZE=500
+# GFW_EVENTS_MAX_PAGES=10
+# GFW_EVENTS_LOOKBACK_DAYS=7
+# GFW_EVENTS_TIMEOUT_S=90
+
+# Windy Webcams global CCTV layer — free key from https://api.windy.com/webcams/docs
+# WINDY_API_KEY=
+
 # LTA Singapore traffic cameras — leave blank to skip this data source.
 # LTA_ACCOUNT_KEY=
 
@@ -87,6 +120,12 @@ AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
 # Free MAP_KEY from https://firms.modaps.eosdis.nasa.gov/map/#d:24hrs;@0.0,0.0,3.0z
 # FIRMS_MAP_KEY=
 
+# Ukraine frontline mirror (GitHub). Default follows cyterat/deepstate-map-data@main.
+# Pin an immutable commit SHA so ingest cannot silently change if main is force-pushed (#362).
+# Example (verify on GitHub before use): main @ b479954e94696bc5622c7818fd20a64a699f4fe8
+# DEEPSTATE_MIRROR_COMMIT=b479954e94696bc5622c7818fd20a64a699f4fe8
+# DEEPSTATE_MIRROR_REPO=cyterat/deepstate-map-data
+
 # Ukraine air raid alerts from alerts.in.ua — free token from https://alerts.in.ua/
 # ALERTS_IN_UA_TOKEN=
 
@@ -116,12 +155,16 @@ AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
 # can identify per-install traffic instead of aggregated "ShadowBroker" hits.
 # Leave blank to send a generic UA. If you set MESHTASTIC_OPERATOR_CALLSIGN,
 # it is included in outbound headers to meshtastic.org by default so they
-# can rate-limit per-operator. Set MESHTASTIC_SEND_CALLSIGN_HEADER=false to
-# suppress the callsign while still using it locally (e.g. for APRS).
+# can rate-limit per-operator. Callsign is NOT sent upstream unless you opt in.
 # MESHTASTIC_OPERATOR_CALLSIGN=
-# MESHTASTIC_SEND_CALLSIGN_HEADER=true
+# MESHTASTIC_SEND_CALLSIGN_HEADER=false
 # MESH_MQTT_PSK=                       # hex-encoded, empty = default LongFast key
 
+# LiveUAMap Playwright scraper (#348). Linux/macOS: on by default when Global
+# Incidents layer is active. Windows: off until the operator enables Global
+# Incidents in the UI (consent dialog) or sets SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=true.
+# SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=false forces off on all platforms.
+
 # ── Mesh / Reticulum (RNS) ─────────────────────────────────────
 # Full-node / participant-node posture for public Infonet sync.
 # MESH_NODE_MODE=participant      # participant | relay | perimeter
@@ -184,7 +227,23 @@ AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
 # MESH_GATE_SESSION_STREAM_MAX_GATES=16
 # MESH_BOOTSTRAP_DISABLED=false
 # MESH_BOOTSTRAP_MANIFEST_PATH=data/bootstrap_peers.json
-# MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY=
+# Swarm discovery (signed peer manifest). Participants need only the public key;
+# the seed operator sets MESH_BOOTSTRAP_SIGNER_PRIVATE_KEY (never commit it).
+# Generate a fleet keypair: uv run python backend/scripts/bootstrap_manifest_helper.py generate-keypair
+# Public sb-testnet fleet defaults (auto-used when MESH_INFONET_FLEET_JOIN=true).
+# MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY=ul1d0kj/ODPIp0OhHzX8eLAVXzJ3CVvzW1vn2IC6q3I=
+# MESH_INFONET_FLEET_JOIN=true
+# MESH_INFONET_FLEET_JOIN_DISABLED=false
+# MESH_BOOTSTRAP_SIGNER_PRIVATE_KEY=   # seed only
+# MESH_BOOTSTRAP_SIGNER_ID=shadowbroker-seed
+# MESH_PEER_REGISTRY_ENABLED=true      # seed only (auto-enabled when private key is set)
+# Headless relay compose sets MESH_INFONET_RELAY_AUTO_WORMHOLE=true; seed nodes with
+# MESH_BOOTSTRAP_SIGNER_PRIVATE_KEY also auto-enable Tor wormhole on startup.
+# MESH_INFONET_RELAY_AUTO_WORMHOLE=false
+# MESH_INFONET_RELAY_AUTO_WORMHOLE_DISABLED=false
+# MESH_SWARM_MANIFEST_TTL_S=14400
+# MESH_SWARM_MANIFEST_PULL_INTERVAL_S=300
+# MESH_PEER_REGISTRY_STALE_S=604800
 # Infonet/Wormhole fails closed to onion/RNS by default. Only enable clearnet
 # sync for local relay development or an explicitly public testnet.
 # MESH_INFONET_ALLOW_CLEARNET_SYNC=false

backend/Dockerfile

@@ -27,6 +27,7 @@ WORKDIR /app
 RUN apt-get update && apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
+    git \
     tor \
     && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
     && apt-get install -y --no-install-recommends nodejs \
@@ -45,7 +46,7 @@ COPY uv.lock /workspace/uv.lock
 COPY backend/pyproject.toml /workspace/backend/pyproject.toml
 
 # Install Python dependencies using the lockfile
-RUN cd /workspace/backend && uv sync --frozen --no-dev \
+RUN cd /workspace/backend && uv sync --frozen --no-dev --extra road-corridor \
     && playwright install --with-deps chromium
 
 # Copy backend source code
@@ -72,7 +73,7 @@ ENV PRIVACY_CORE_LIB=/app/libprivacy_core.so
 # Create a non-root user for security
 # Grant write access to /app so the auto-updater can extract files
 # Pre-create /app/data so mounted volumes inherit correct ownership
-RUN adduser --system --uid 1001 backenduser \
+RUN adduser --system --uid 1001 --home /app backenduser \
     && mkdir -p /app/data \
     && chown -R backenduser /app \
     && chmod -R u+w /app

backend/analytics/__init__.py

@@ -0,0 +1,21 @@
+"""Strategic Risk Analytics — game-theoretic early warning layer."""
+
+from analytics.backtest import (
+    DEFAULT_BACKTEST_ALERT_THRESHOLD,
+    BacktestReport,
+    run_historical_backtest,
+    tune_alert_threshold,
+)
+from analytics.gt_early_warning import GT_EarlyWarning
+from analytics.integration import get_gt_engine, process_feed_item, refresh_from_latest_data
+
+__all__ = [
+    "BacktestReport",
+    "DEFAULT_BACKTEST_ALERT_THRESHOLD",
+    "GT_EarlyWarning",
+    "get_gt_engine",
+    "process_feed_item",
+    "refresh_from_latest_data",
+    "run_historical_backtest",
+    "tune_alert_threshold",
+]

backend/analytics/backtest.py

@@ -0,0 +1,287 @@
+"""Historical backtesting for Strategic Risk Analytics.
+
+This is **benchmark validation**, not forward-weeks prediction on live feeds.
+
+The suite scores whether costly-signal patterns + Bayesian updating correctly
+classify curated pre-crisis text snippets (positive cases) vs cheap-talk
+controls (negative cases) at a tuned alert threshold. A high accuracy on this
+labeled corpus does **not** imply the engine will score 100% on messy,
+adversarial, or weeks-ahead production telemetry — opponents adapt, labels are
+easier here than in the wild, and the window is retrospective.
+
+Reports accuracy and a conservative Wilson 95% confidence lower bound on the
+benchmark only. Treat 100% here as "classifier fits the benchmark," not "ship
+it for multi-week forecasting." For live week-over-week scoring with delayed
+labels, see ``rolling_backtest.py``.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+from analytics.gt_early_warning import GT_EarlyWarning
+from analytics.historical_events import (
+    HistoricalCase,
+    default_historical_cases,
+    expanded_historical_cases,
+)
+from analytics.settings import GTAnalyticsSettings
+
+DomainName = Literal["financial", "unrest", "conflict"]
+
+# Validated on expanded suite (82 cases, Wilson lower >= 0.95 at 100% accuracy).
+DEFAULT_BACKTEST_ALERT_THRESHOLD = 0.26
+MAX_BACKTEST_ALERT_THRESHOLD = 0.39
+
+
+@dataclass(frozen=True)
+class CaseResult:
+    case_id: str
+    name: str
+    kind: str
+    region: str
+    domain: str
+    expected_alert: bool
+    alerted: bool
+    correct: bool
+    peak_domain_risk: float
+    peak_composite_risk: float
+    costly_signals: list[str]
+    tags: tuple[str, ...] = field(default_factory=tuple)
+
+
+@dataclass(frozen=True)
+class BacktestReport:
+    total_cases: int
+    correct: int
+    accuracy: float
+    confidence_rate: float
+    wilson_lower_95: float
+    wilson_upper_95: float
+    true_positives: int
+    true_negatives: int
+    false_positives: int
+    false_negatives: int
+    sensitivity: float
+    specificity: float
+    alert_threshold: float
+    target_confidence: float
+    meets_target: bool
+    case_results: tuple[CaseResult, ...]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "total_cases": self.total_cases,
+            "correct": self.correct,
+            "accuracy": round(self.accuracy, 4),
+            "confidence_rate": round(self.confidence_rate, 4),
+            "wilson_lower_95": round(self.wilson_lower_95, 4),
+            "wilson_upper_95": round(self.wilson_upper_95, 4),
+            "true_positives": self.true_positives,
+            "true_negatives": self.true_negatives,
+            "false_positives": self.false_positives,
+            "false_negatives": self.false_negatives,
+            "sensitivity": round(self.sensitivity, 4),
+            "specificity": round(self.specificity, 4),
+            "alert_threshold": self.alert_threshold,
+            "target_confidence": self.target_confidence,
+            "meets_target": self.meets_target,
+            "cases": [
+                {
+                    "case_id": row.case_id,
+                    "name": row.name,
+                    "kind": row.kind,
+                    "correct": row.correct,
+                    "alerted": row.alerted,
+                    "peak_domain_risk": round(row.peak_domain_risk, 4),
+                    "peak_composite_risk": round(row.peak_composite_risk, 4),
+                    "costly_signals": row.costly_signals,
+                }
+                for row in self.case_results
+            ],
+        }
+
+
+def wilson_interval(
+    successes: int,
+    total: int,
+    z: float = 1.96,
+) -> tuple[float, float]:
+    """Wilson score interval for a binomial proportion (95% default)."""
+    if total <= 0:
+        return 0.0, 0.0
+    phat = successes / total
+    z2 = z * z
+    denom = 1.0 + z2 / total
+    center = (phat + z2 / (2.0 * total)) / denom
+    margin = (
+        z
+        * math.sqrt((phat * (1.0 - phat) + z2 / (4.0 * total)) / total)
+        / denom
+    )
+    return max(0.0, center - margin), min(1.0, center + margin)
+
+
+def _domain_risk(engine: GT_EarlyWarning, region: str, domain: str) -> float:
+    if domain in ("financial", "unrest", "conflict"):
+        return engine.get_prior(region, domain)
+    return engine.composite_risk(region)
+
+
+def _evaluate_case(
+    case: HistoricalCase,
+    *,
+    settings: GTAnalyticsSettings,
+    alert_threshold: float,
+) -> CaseResult:
+    engine = GT_EarlyWarning(settings)
+    peak_domain = float(settings.base_prior)
+    peak_composite = float(settings.base_prior)
+    detected_signals: set[str] = set()
+
+    for item in case.to_feed_dicts():
+        result = engine.process_feed_item(item)
+        for sig in (result or {}).get("signals") or {}:
+            detected_signals.add(str(sig))
+        domain_risk = _domain_risk(engine, case.region, case.domain)
+        composite = engine.composite_risk(case.region)
+        peak_domain = max(peak_domain, domain_risk)
+        peak_composite = max(peak_composite, composite)
+
+    # Domain-specific score for labeled events; composite as secondary for conflict.
+    score = peak_domain
+    if case.domain == "conflict":
+        score = max(peak_domain, peak_composite * 0.95)
+    alerted = score >= alert_threshold
+    expected_alert = case.kind == "positive"
+
+    return CaseResult(
+        case_id=case.case_id,
+        name=case.name,
+        kind=case.kind,
+        region=case.region,
+        domain=case.domain,
+        expected_alert=expected_alert,
+        alerted=alerted,
+        correct=alerted == expected_alert,
+        peak_domain_risk=peak_domain,
+        peak_composite_risk=peak_composite,
+        costly_signals=sorted(detected_signals),
+        tags=case.tags,
+    )
+
+
+def run_historical_backtest(
+    cases: tuple[HistoricalCase, ...] | None = None,
+    *,
+    settings: GTAnalyticsSettings | None = None,
+    alert_threshold: float | None = None,
+    target_confidence: float = 0.80,
+    use_expanded_suite: bool = True,
+) -> BacktestReport:
+    """
+    Run labeled historical cases and compute accuracy + Wilson 95% CI.
+
+    ``confidence_rate`` is the conservative Wilson lower bound — the metric
+    used for pass/fail against ``target_confidence``.
+    """
+    cfg = settings or GTAnalyticsSettings(enabled=True)
+    threshold = float(
+        alert_threshold
+        if alert_threshold is not None
+        else DEFAULT_BACKTEST_ALERT_THRESHOLD
+    )
+    if cases is not None:
+        suite = cases
+    elif use_expanded_suite:
+        suite = expanded_historical_cases()
+    else:
+        suite = default_historical_cases()
+
+    results = tuple(
+        _evaluate_case(case, settings=cfg, alert_threshold=threshold) for case in suite
+    )
+
+    tp = sum(1 for r in results if r.expected_alert and r.alerted)
+    tn = sum(1 for r in results if not r.expected_alert and not r.alerted)
+    fp = sum(1 for r in results if not r.expected_alert and r.alerted)
+    fn = sum(1 for r in results if r.expected_alert and not r.alerted)
+    correct = tp + tn
+    total = len(results)
+    accuracy = correct / total if total else 0.0
+    lower, upper = wilson_interval(correct, total)
+
+    pos_total = sum(1 for r in results if r.expected_alert)
+    neg_total = total - pos_total
+    sensitivity = tp / pos_total if pos_total else 0.0
+    specificity = tn / neg_total if neg_total else 0.0
+
+    return BacktestReport(
+        total_cases=total,
+        correct=correct,
+        accuracy=accuracy,
+        confidence_rate=lower,
+        wilson_lower_95=lower,
+        wilson_upper_95=upper,
+        true_positives=tp,
+        true_negatives=tn,
+        false_positives=fp,
+        false_negatives=fn,
+        sensitivity=sensitivity,
+        specificity=specificity,
+        alert_threshold=threshold,
+        target_confidence=target_confidence,
+        meets_target=lower >= target_confidence,
+        case_results=results,
+    )
+
+
+def tune_alert_threshold(
+    cases: tuple[HistoricalCase, ...] | None = None,
+    *,
+    settings: GTAnalyticsSettings | None = None,
+    min_threshold: float = 0.20,
+    max_threshold: float = 0.65,
+    step: float = 0.01,
+    target_confidence: float = 0.95,
+) -> tuple[float, BacktestReport]:
+    """Grid-search alert threshold to maximize Wilson lower bound."""
+    if cases is not None:
+        suite = cases
+    else:
+        suite = expanded_historical_cases()
+    best_threshold = min_threshold
+    best_report = run_historical_backtest(
+        suite,
+        settings=settings,
+        alert_threshold=min_threshold,
+        target_confidence=target_confidence,
+    )
+
+    steps = int(round((max_threshold - min_threshold) / step))
+    for i in range(steps + 1):
+        threshold = min_threshold + i * step
+        report = run_historical_backtest(
+            suite,
+            settings=settings,
+            alert_threshold=threshold,
+            target_confidence=target_confidence,
+        )
+        better_confidence = report.confidence_rate > best_report.confidence_rate
+        tied_confidence = math.isclose(
+            report.confidence_rate, best_report.confidence_rate, rel_tol=0.0, abs_tol=1e-9
+        )
+        better_accuracy = report.accuracy > best_report.accuracy
+        tied_accuracy = math.isclose(
+            report.accuracy, best_report.accuracy, rel_tol=0.0, abs_tol=1e-9
+        )
+        prefer_higher_threshold = (
+            tied_confidence and tied_accuracy and threshold > best_threshold
+        )
+        if better_confidence or (tied_confidence and better_accuracy) or prefer_higher_threshold:
+            best_threshold = threshold
+            best_report = report
+
+    return best_threshold, best_report

backend/analytics/daily_store.py

@@ -0,0 +1,140 @@
+"""Daily GT risk readings for micro rolling averages."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+from dataclasses import asdict, dataclass, field
+from datetime import date, datetime, timezone
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_DAILY_DIR = Path(__file__).parent.parent / "data" / "gt_rolling" / "daily"
+_store_lock = threading.Lock()
+
+
+def daily_store_dir() -> Path:
+    override = str(os.environ.get("GT_DAILY_STORE_DIR", "")).strip()
+    if override:
+        return Path(override)
+    return _DAILY_DIR
+
+
+def utc_today() -> date:
+    return datetime.now(timezone.utc).date()
+
+
+def date_id(when: date | datetime | None = None) -> str:
+    if when is None:
+        when = utc_today()
+    if isinstance(when, datetime):
+        when = when.date()
+    return when.isoformat()
+
+
+@dataclass
+class DailyRegionReading:
+    region: str
+    composite_risk: float
+    financial: float
+    unrest: float
+    conflict: float
+    peak_score: float
+    readings: int = 1
+    last_captured_at: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> DailyRegionReading:
+        return cls(
+            region=str(raw.get("region") or "").strip().lower(),
+            composite_risk=float(raw.get("composite_risk") or 0.0),
+            financial=float(raw.get("financial") or 0.0),
+            unrest=float(raw.get("unrest") or 0.0),
+            conflict=float(raw.get("conflict") or 0.0),
+            peak_score=float(raw.get("peak_score") or 0.0),
+            readings=int(raw.get("readings") or 1),
+            last_captured_at=str(raw.get("last_captured_at") or ""),
+        )
+
+
+@dataclass
+class DailySnapshot:
+    date: str
+    regions: dict[str, DailyRegionReading] = field(default_factory=dict)
+    last_updated_at: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "date": self.date,
+            "last_updated_at": self.last_updated_at,
+            "regions": {key: row.to_dict() for key, row in self.regions.items()},
+        }
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> DailySnapshot:
+        regions: dict[str, DailyRegionReading] = {}
+        for key, row in (raw.get("regions") or {}).items():
+            if isinstance(row, dict):
+                reading = DailyRegionReading.from_dict(row)
+                regions[str(key).strip().lower()] = reading
+        return cls(
+            date=str(raw.get("date") or ""),
+            regions=regions,
+            last_updated_at=str(raw.get("last_updated_at") or ""),
+        )
+
+
+def _daily_path(day_id: str) -> Path:
+    safe = day_id.replace("/", "-").replace("..", "")
+    return daily_store_dir() / f"{safe}.json"
+
+
+def _ensure_dir() -> None:
+    daily_store_dir().mkdir(parents=True, exist_ok=True)
+
+
+def list_daily_ids(*, newest_first: bool = True, limit: int | None = None) -> list[str]:
+    _ensure_dir()
+    ids = sorted(
+        (path.stem for path in daily_store_dir().glob("*.json")),
+        reverse=newest_first,
+    )
+    if limit is not None:
+        return ids[:limit]
+    return ids
+
+
+def load_daily(day: date | str | None = None) -> DailySnapshot | None:
+    day_id = date_id(day) if day is not None else date_id()
+    path = _daily_path(day_id)
+    if not path.is_file():
+        return None
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+        if not isinstance(raw, dict):
+            return None
+        return DailySnapshot.from_dict(raw)
+    except (OSError, json.JSONDecodeError, TypeError, ValueError):
+        logger.exception("Failed to load GT daily reading %s", day_id)
+        return None
+
+
+def save_daily(snapshot: DailySnapshot) -> None:
+    _ensure_dir()
+    path = _daily_path(snapshot.date)
+    tmp = path.with_suffix(".json.tmp")
+    payload = json.dumps(snapshot.to_dict(), indent=2, sort_keys=True)
+    with _store_lock:
+        tmp.write_text(payload, encoding="utf-8")
+        tmp.replace(path)
+
+
+def utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()

backend/analytics/feed_adapter.py

@@ -0,0 +1,206 @@
+"""Normalize Shadowbroker feed records into GT analytics feed items."""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Iterable
+
+_DOMAIN_CONFLICT = "conflict"
+_DOMAIN_UNREST = "unrest"
+_DOMAIN_FINANCIAL = "financial"
+
+_CONFLICT_HINTS = re.compile(
+    r"\b(war|missile|strike|attack|military|invasion|troop|shelling|drone|bomb|nuclear)\b",
+    re.I,
+)
+_UNREST_HINTS = re.compile(
+    r"\b(protest|rally|strike|riot|unrest|mobiliz|demonstrat|curfew|purge|coup)\b",
+    re.I,
+)
+_FINANCIAL_HINTS = re.compile(
+    r"\b(payroll|loan|default|bankruptcy|liquidity|sanction|supply\s+chain|delay|shortage)\b",
+    re.I,
+)
+
+
+def _clean_region(value: Any) -> str:
+    region = str(value or "").strip().lower()
+    return region or "global"
+
+
+def _infer_domain(text: str, explicit: str | None = None) -> str:
+    if explicit in {_DOMAIN_CONFLICT, _DOMAIN_UNREST, _DOMAIN_FINANCIAL}:
+        return explicit
+    if _CONFLICT_HINTS.search(text):
+        return _DOMAIN_CONFLICT
+    if _UNREST_HINTS.search(text):
+        return _DOMAIN_UNREST
+    if _FINANCIAL_HINTS.search(text):
+        return _DOMAIN_FINANCIAL
+    return _DOMAIN_FINANCIAL
+
+
+def _text_from_record(
+    record: dict[str, Any],
+    *,
+    prefer_translation: bool = False,
+) -> str:
+    """Build ingest text; prefer English translations for Telegram OSINT when set."""
+    if prefer_translation:
+        translated_parts = [
+            record.get("title_translated"),
+            record.get("description_translated"),
+        ]
+        translated = "\n".join(
+            str(p).strip() for p in translated_parts if p and str(p).strip()
+        )
+        if translated:
+            return translated
+
+    parts = [
+        record.get("title"),
+        record.get("description"),
+        record.get("text"),
+        record.get("summary"),
+    ]
+    return "\n".join(str(p).strip() for p in parts if p and str(p).strip())
+
+
+_HASHTAG_REGION = re.compile(r"#([a-z][a-z0-9_-]{2,})", re.I)
+
+
+def _region_from_hashtags(text: str) -> str | None:
+    """Map common theater hashtags (#Ukraine) to dossier/heatmap region keys."""
+    for match in _HASHTAG_REGION.finditer(text or ""):
+        tag = match.group(1).lower()
+        if tag in {
+            "ukraine",
+            "russia",
+            "israel",
+            "iran",
+            "gaza",
+            "syria",
+            "taiwan",
+            "china",
+            "belfast",
+            "uk",
+            "usa",
+        }:
+            return tag
+    return None
+
+
+def _region_from_record(record: dict[str, Any], *, text: str = "") -> str:
+    for key in ("geotag", "region", "country", "location"):
+        if record.get(key):
+            return _clean_region(record[key])
+    hashtag_region = _region_from_hashtags(text)
+    if hashtag_region:
+        return hashtag_region
+    coords = record.get("coords")
+    if isinstance(coords, (list, tuple)) and len(coords) >= 2:
+        try:
+            lat = float(coords[0])
+            lng = float(coords[1])
+            return f"{lat:.2f},{lng:.2f}"
+        except (TypeError, ValueError):
+            pass
+    return "global"
+
+
+def _entities_from_record(record: dict[str, Any]) -> list[str]:
+    entities: list[str] = []
+    for key in ("entities", "tags", "keywords"):
+        raw = record.get(key)
+        if isinstance(raw, list):
+            entities.extend(str(v).strip() for v in raw if str(v).strip())
+        elif isinstance(raw, str) and raw.strip():
+            entities.extend(part.strip() for part in raw.split(",") if part.strip())
+    channel = str(record.get("channel") or "").strip()
+    if channel:
+        entities.append(f"channel:{channel}")
+    source = str(record.get("source") or "").strip()
+    if source:
+        entities.append(f"source:{source}")
+    return entities
+
+
+def normalize_feed_item(record: dict[str, Any], *, source_type: str = "generic") -> dict[str, Any]:
+    """Map a news/Telegram/GDELT record into the GT engine schema."""
+    prefer_translation = source_type == "telegram_osint"
+    text = _text_from_record(record, prefer_translation=prefer_translation)
+    if prefer_translation and not text.strip():
+        text = _text_from_record(record, prefer_translation=False)
+    region = _region_from_record(record, text=text)
+    domain = _infer_domain(text, record.get("domain"))
+    coords = record.get("coords")
+    lat = lng = None
+    if isinstance(coords, (list, tuple)) and len(coords) >= 2:
+        try:
+            lat = float(coords[0])
+            lng = float(coords[1])
+        except (TypeError, ValueError):
+            lat = lng = None
+
+    return {
+        "id": record.get("id") or record.get("link"),
+        "text": text,
+        "source": str(record.get("source") or source_type),
+        "source_type": source_type,
+        "region": region,
+        "domain": domain,
+        "entities": _entities_from_record(record),
+        "coords": [lat, lng] if lat is not None and lng is not None else None,
+        "published": record.get("published"),
+        "risk_score": record.get("risk_score"),
+    }
+
+
+def iter_telegram_posts(payload: dict[str, Any] | None) -> Iterable[dict[str, Any]]:
+    from services.telegram_translate import apply_post_translation, telegram_translate_enabled
+
+    posts = list((payload or {}).get("posts") or [])
+    for post in posts:
+        if not isinstance(post, dict):
+            continue
+        if not (post.get("description") or post.get("title")):
+            continue
+        enriched = (
+            apply_post_translation(post)
+            if telegram_translate_enabled()
+            else post
+        )
+        yield normalize_feed_item(enriched, source_type="telegram_osint")
+
+
+def iter_news_items(payload: list[dict[str, Any]] | None) -> Iterable[dict[str, Any]]:
+    for item in list(payload or []):
+        if not isinstance(item, dict):
+            continue
+        yield normalize_feed_item(item, source_type="news")
+        for article in list(item.get("articles") or []):
+            if isinstance(article, dict):
+                yield normalize_feed_item(article, source_type="news_cluster")
+
+
+def iter_gdelt_features(payload: list[dict[str, Any]] | None) -> Iterable[dict[str, Any]]:
+    for feature in list(payload or []):
+        if not isinstance(feature, dict):
+            continue
+        props = dict(feature.get("properties") or {})
+        geometry = dict(feature.get("geometry") or {})
+        coords = None
+        if geometry.get("type") == "Point":
+            raw = geometry.get("coordinates")
+            if isinstance(raw, (list, tuple)) and len(raw) >= 2:
+                coords = [float(raw[1]), float(raw[0])]
+        record = {
+            "title": props.get("name") or props.get("title"),
+            "description": props.get("snippet") or props.get("description"),
+            "source": props.get("source") or "gdelt",
+            "coords": coords,
+            "published": props.get("date") or props.get("published"),
+            "region": props.get("location") or props.get("country"),
+        }
+        if record["title"] or record["description"]:
+            yield normalize_feed_item(record, source_type="gdelt")

backend/analytics/gt_alerts.py

@@ -0,0 +1,128 @@
+"""Top strategic-risk alerts — ranked regions with map coordinates."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from analytics.integration import get_gt_engine
+from analytics.settings import get_gt_settings
+
+
+def _peak_score(props: dict[str, Any]) -> float:
+    composite = float(props.get("risk") or 0.0)
+    financial = float(props.get("financial") or 0.0)
+    unrest = float(props.get("unrest") or 0.0)
+    conflict = float(props.get("conflict") or 0.0)
+    return max(composite, financial, unrest, conflict)
+
+
+def _valid_coords(coords: Any) -> tuple[float, float] | None:
+    if not isinstance(coords, (list, tuple)) or len(coords) < 2:
+        return None
+    try:
+        lng = float(coords[0])
+        lat = float(coords[1])
+    except (TypeError, ValueError):
+        return None
+    if not (-90.0 <= lat <= 90.0 and -180.0 <= lng <= 180.0):
+        return None
+    if abs(lat) < 0.001 and abs(lng) < 0.001:
+        return None
+    return lat, lng
+
+
+def _region_label(region: str) -> str:
+    text = str(region or "").strip()
+    if not text:
+        return "unknown"
+    if "," in text:
+        parts = [piece.strip() for piece in text.split(",") if piece.strip()]
+        if len(parts) >= 2:
+            try:
+                lat = float(parts[0])
+                lng = float(parts[-1])
+                return f"{lat:.2f}°, {lng:.2f}°"
+            except ValueError:
+                pass
+    return text.replace("_", " ")
+
+
+def parse_heatmap_alerts(
+    heatmap: dict[str, Any] | None,
+    *,
+    limit: int = 8,
+) -> tuple[list[dict[str, Any]], int]:
+    """Return ranked alerts and count of regions plottable on the map."""
+    features = (heatmap or {}).get("features") or []
+    rows: list[dict[str, Any]] = []
+
+    for feature in features:
+        if not isinstance(feature, dict):
+            continue
+        geometry = feature.get("geometry") or {}
+        coords = _valid_coords(geometry.get("coordinates"))
+        if coords is None:
+            continue
+        lat, lng = coords
+        props = feature.get("properties") or {}
+        region = str(props.get("region") or "").strip().lower()
+        if not region:
+            continue
+        score = _peak_score(props)
+        rows.append(
+            {
+                "region": region,
+                "region_label": _region_label(region),
+                "risk": round(float(props.get("risk") or 0.0), 4),
+                "financial": round(float(props.get("financial") or 0.0), 4),
+                "unrest": round(float(props.get("unrest") or 0.0), 4),
+                "conflict": round(float(props.get("conflict") or 0.0), 4),
+                "contagion": round(float(props.get("contagion") or 0.0), 4),
+                "score": round(score, 4),
+                "lat": lat,
+                "lng": lng,
+                "ignition": bool(props.get("micro_ignition")),
+                "risk_3d_avg": props.get("risk_3d_avg"),
+                "risk_delta": props.get("risk_delta"),
+                "updates": int(props.get("updates") or 0),
+            }
+        )
+
+    rows.sort(
+        key=lambda row: (
+            bool(row.get("ignition")),
+            float(row.get("risk_delta") or 0.0),
+            float(row.get("score") or 0.0),
+        ),
+        reverse=True,
+    )
+    return rows[: max(1, limit)], len(rows)
+
+
+def top_gt_alerts(*, limit: int = 8) -> dict[str, Any]:
+    """Ranked top regions for API / OpenClaw."""
+    settings = get_gt_settings()
+    engine = get_gt_engine()
+    heatmap: dict[str, Any] = {"type": "FeatureCollection", "features": []}
+    engine_regions = 0
+
+    if engine is not None:
+        heatmap = engine.get_risk_heatmap()
+        with engine._lock:  # noqa: SLF001 — intentional meta read
+            engine_regions = len(engine._regions)
+
+    alerts, plotted = parse_heatmap_alerts(heatmap, limit=limit)
+    tracked = len(heatmap.get("features") or [])
+
+    return {
+        "alerts": alerts,
+        "tracked_regions": tracked,
+        "engine_regions": engine_regions,
+        "plotted_regions": plotted,
+        "max_regions": settings.max_heatmap_features,
+        "note": (
+            "Layer count is tracked GT regions (cap "
+            f"{settings.max_heatmap_features}), not raw feed events. "
+            "Only regions with valid coordinates appear on the map."
+        ),
+    }

backend/analytics/gt_early_warning.py

@@ -0,0 +1,593 @@
+"""Game-theoretic early warning analytics with Bayesian updating and contagion graph."""
+
+from __future__ import annotations
+
+import logging
+import re
+import threading
+from collections import defaultdict
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, DefaultDict
+
+import networkx as nx
+import numpy as np
+
+from analytics.settings import GTAnalyticsSettings, get_gt_settings
+
+logger = logging.getLogger(__name__)
+
+DomainName = str  # financial | unrest | conflict
+
+_DOMAINS: tuple[DomainName, ...] = ("financial", "unrest", "conflict")
+
+_DEFAULT_LIKELIHOODS: dict[DomainName, dict[str, float]] = {
+    "financial": {"distress": 0.75, "normal": 0.25},
+    "unrest": {"distress": 0.82, "normal": 0.22},
+    "conflict": {"distress": 0.78, "normal": 0.18},
+}
+
+_DEFAULT_SIGNAL_WEIGHTS: dict[str, float] = {
+    "payroll_loan": 3.0,
+    "supply_delay": 2.2,
+    "elite_relocation": 2.8,
+    "purge": 3.5,
+    "protest_mobilize": 2.5,
+    "gps_jamming": 2.7,
+    "troop_movement": 3.0,
+    "bank_run": 3.2,
+    "sanctions_escalation": 2.4,
+    "ceasefire_break": 2.6,
+}
+
+# Costly-signal regex patterns (cheap talk filtered by absence of match).
+_SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
+    "payroll_loan": [
+        re.compile(r"payroll\s+loan", re.I),
+        re.compile(r"merchant\s+cash\s+advance", re.I),
+        re.compile(r"working\s+capital\s+loan", re.I),
+    ],
+    "supply_delay": [
+        re.compile(r"supply\s+(chain\s+)?delay", re.I),
+        re.compile(r"shipping\s+delay", re.I),
+        re.compile(r"logistics\s+backlog", re.I),
+        re.compile(r"port\s+congestion", re.I),
+    ],
+    "elite_relocation": [
+        re.compile(r"elite\s+(asset\s+)?relocation", re.I),
+        re.compile(r"oligarch\s+jet", re.I),
+        re.compile(r"private\s+jet\s+exodus", re.I),
+        re.compile(r"capital\s+flight", re.I),
+    ],
+    "purge": [
+        re.compile(r"\bpurge\b", re.I),
+        re.compile(r"political\s+purge", re.I),
+        re.compile(r"security\s+apparatus\s+reshuffle", re.I),
+    ],
+    "protest_mobilize": [
+        re.compile(r"protest\s+mobil", re.I),
+        re.compile(r"mass\s+rally", re.I),
+        re.compile(r"general\s+strike", re.I),
+        re.compile(r"\bstrike\b", re.I),
+        re.compile(r"\brally\b", re.I),
+    ],
+    "gps_jamming": [
+        re.compile(r"gps\s+jam", re.I),
+        re.compile(r"gnss\s+interference", re.I),
+        re.compile(r"spoofing\s+spike", re.I),
+    ],
+    "troop_movement": [
+        re.compile(r"troop\s+movement", re.I),
+        re.compile(r"military\s+mobil", re.I),
+        re.compile(r"armored\s+convoy", re.I),
+        re.compile(r"troop\s+buildup", re.I),
+    ],
+    "bank_run": [
+        re.compile(r"bank\s+run", re.I),
+        re.compile(r"deposit\s+flight", re.I),
+        re.compile(r"liquidity\s+crunch", re.I),
+    ],
+    "sanctions_escalation": [
+        re.compile(r"sanctions?\s+escalat", re.I),
+        re.compile(r"new\s+sanctions?", re.I),
+        re.compile(r"export\s+controls?\s+tighten", re.I),
+    ],
+    "ceasefire_break": [
+        re.compile(r"ceasefire\s+(broken|violated|collapse)", re.I),
+        re.compile(r"truce\s+end", re.I),
+    ],
+}
+
+_SIGNAL_DOMAINS: dict[str, DomainName] = {
+    "payroll_loan": "financial",
+    "supply_delay": "financial",
+    "bank_run": "financial",
+    "sanctions_escalation": "financial",
+    "protest_mobilize": "unrest",
+    "purge": "unrest",
+    "elite_relocation": "financial",
+    "gps_jamming": "conflict",
+    "troop_movement": "conflict",
+    "ceasefire_break": "conflict",
+}
+
+
+@dataclass
+class RegionState:
+    """Per-region Bayesian beliefs and metadata."""
+
+    priors: dict[DomainName, float] = field(default_factory=lambda: defaultdict(float))
+    coords: list[float] | None = None
+    signal_volume: DefaultDict[str, float] = field(default_factory=lambda: defaultdict(float))
+    update_count: int = 0
+
+
+@dataclass
+class HistoryEntry:
+    timestamp: str
+    domain: DomainName
+    signals: dict[str, float]
+    strength: float
+    prior: float
+    posterior: float
+    source: str
+    deviation_score: float
+
+
+class GT_EarlyWarning:
+    """
+    Game-Theoretic Early Warning System with Bayesian updating.
+
+    Tracks distress probabilities per region/domain, classifies costly signals vs
+    cheap talk, and propagates risk through an entity interaction graph.
+    """
+
+    def __init__(self, settings: GTAnalyticsSettings | None = None) -> None:
+        self.settings = settings or get_gt_settings()
+        self.G: nx.Graph = nx.Graph()
+        self._regions: dict[str, RegionState] = {}
+        self._history: dict[str, list[HistoryEntry]] = defaultdict(list)
+        self._seen_item_ids: set[str] = set()
+        self._lock = threading.RLock()
+
+        self.likelihoods = dict(_DEFAULT_LIKELIHOODS)
+        self.signal_weights = dict(_DEFAULT_SIGNAL_WEIGHTS)
+        self.signal_weights.update(self.settings.signal_weight_overrides)
+
+        self._base_prior = float(self.settings.base_prior)
+
+    def _utcnow(self) -> str:
+        return datetime.now(timezone.utc).isoformat()
+
+    def _region_state(self, region: str) -> RegionState:
+        key = str(region or "global").strip().lower() or "global"
+        if key not in self._regions:
+            state = RegionState()
+            for domain in _DOMAINS:
+                state.priors[domain] = self._base_prior
+            self._regions[key] = state
+        return self._regions[key]
+
+    def get_prior(self, region: str, domain: DomainName) -> float:
+        with self._lock:
+            return float(self._region_state(region).priors.get(domain, self._base_prior))
+
+    def set_prior(self, region: str, domain: DomainName, value: float) -> None:
+        with self._lock:
+            state = self._region_state(region)
+            state.priors[domain] = float(
+                np.clip(value, self.settings.min_prob, self.settings.max_prob)
+            )
+
+    def composite_risk(self, region: str) -> float:
+        """Weighted composite across domains (conflict weighted highest)."""
+        weights = {"financial": 0.25, "unrest": 0.35, "conflict": 0.40}
+        with self._lock:
+            state = self._region_state(region)
+            total = 0.0
+            weight_sum = 0.0
+            for domain, weight in weights.items():
+                total += float(state.priors.get(domain, self._base_prior)) * weight
+                weight_sum += weight
+            return float(total / weight_sum) if weight_sum else self._base_prior
+
+    def classify_signals(self, text: str, source: str = "") -> dict[str, float]:
+        """Return weighted costly-signal strengths detected in text."""
+        text_lower = (text or "").lower()
+        signals: dict[str, float] = {}
+
+        for signal_name, patterns in _SIGNAL_PATTERNS.items():
+            weight = float(self.signal_weights.get(signal_name, 1.0))
+            if any(pattern.search(text_lower) for pattern in patterns):
+                signals[signal_name] = weight
+
+        rally_strike_count = text_lower.count("rally") + text_lower.count("strike")
+        if rally_strike_count > 3:
+            signals["protest_mobilize"] = signals.get("protest_mobilize", 0.0) + 1.5
+
+        # Source credibility nudge (Telegram OSINT channels treated as moderate-cost signals).
+        if source and "t.me/" in source.lower() and signals:
+            for key in list(signals):
+                signals[key] = round(signals[key] * 1.05, 3)
+
+        return signals
+
+    def _deviation_score(self, region: str, domain: DomainName, strength: float) -> float:
+        """Deviation from rolling regional norm — herding/coordination detector input."""
+        with self._lock:
+            state = self._region_state(region)
+            baseline = max(state.signal_volume[domain], 1.0)
+            state.signal_volume[domain] += strength
+            state.update_count += 1
+            return float(strength / baseline)
+
+    def bayesian_update(
+        self,
+        region: str,
+        domain: DomainName,
+        evidence_strength: float = 1.0,
+    ) -> float:
+        """
+        Bayesian update: P(distress|evidence) from likelihood table and prior.
+
+        evidence_strength scales how far belief moves toward the likelihood posterior.
+        """
+        domain = domain if domain in _DOMAINS else "financial"
+        lik = self.likelihoods.get(domain, self.likelihoods["financial"])
+
+        with self._lock:
+            state = self._region_state(region)
+            prior = float(state.priors.get(domain, self._base_prior))
+
+            p_e_given_d = lik["distress"]
+            p_e_given_not_d = lik["normal"]
+            p_e = (p_e_given_d * prior) + (p_e_given_not_d * (1.0 - prior))
+
+            if p_e <= 0:
+                posterior = prior
+            else:
+                posterior = (p_e_given_d * prior) / p_e
+
+            scaled = prior + (posterior - prior) * float(evidence_strength)
+            clipped = float(np.clip(scaled, self.settings.min_prob, self.settings.max_prob))
+            state.priors[domain] = clipped
+            return clipped
+
+    def _update_graph(
+        self,
+        region: str,
+        entities: list[str],
+        strength: float,
+        coords: list[float] | None,
+    ) -> None:
+        region_key = str(region or "global").strip().lower() or "global"
+        self.G.add_node(region_key, node_type="region", region=region_key)
+        if coords and len(coords) >= 2:
+            self.G.nodes[region_key]["coords"] = coords
+
+        for entity in entities:
+            entity_key = str(entity).strip()
+            if not entity_key:
+                continue
+            self.G.add_node(entity_key, node_type="entity", region=region_key)
+            self.G.add_edge(
+                region_key,
+                entity_key,
+                weight=float(strength),
+                timestamp=self._utcnow(),
+            )
+
+        for i, e1 in enumerate(entities):
+            for e2 in entities[i + 1 :]:
+                k1, k2 = str(e1).strip(), str(e2).strip()
+                if not k1 or not k2:
+                    continue
+                self.G.add_edge(
+                    k1,
+                    k2,
+                    weight=float(strength),
+                    timestamp=self._utcnow(),
+                )
+
+    def process_feed_item(self, item: dict[str, Any]) -> dict[str, Any]:
+        """Process one normalized feed item and update beliefs + contagion graph."""
+        region = str(item.get("region") or item.get("geotag") or "global").strip().lower()
+        text = str(item.get("text") or "")
+        source = str(item.get("source") or "unknown")
+        explicit_domain = str(item.get("domain") or "").strip().lower()
+        entities = list(item.get("entities") or [])
+        coords = item.get("coords")
+        item_id = str(item.get("id") or f"{source}|{hash(text)}")
+
+        if self.settings.watched_channels:
+            channel = ""
+            for entity in entities:
+                if str(entity).startswith("channel:"):
+                    channel = str(entity).split(":", 1)[-1].lower()
+                    break
+            if channel and channel not in {c.lower() for c in self.settings.watched_channels}:
+                return {
+                    "region": region,
+                    "skipped": True,
+                    "reason": "channel_not_watched",
+                    "risk_score": self.composite_risk(region),
+                    "signals": {},
+                }
+
+        with self._lock:
+            if item_id and item_id in self._seen_item_ids:
+                return {
+                    "region": region,
+                    "skipped": True,
+                    "reason": "duplicate",
+                    "risk_score": self.composite_risk(region),
+                    "signals": {},
+                }
+            if item_id:
+                self._seen_item_ids.add(item_id)
+
+        signals = self.classify_signals(text, source)
+        total_strength = float(sum(signals.values()))
+
+        if total_strength <= 0:
+            return {
+                "region": region,
+                "risk_score": self.composite_risk(region),
+                "signals": {},
+                "contagion_potential": self._get_contagion_score(region),
+            }
+
+        domains_touched: set[DomainName] = set()
+        if explicit_domain in _DOMAINS:
+            domains_touched.add(explicit_domain)
+        for signal_name in signals:
+            domains_touched.add(_SIGNAL_DOMAINS.get(signal_name, explicit_domain or "financial"))
+        if not domains_touched:
+            domains_touched.add("financial")
+
+        evidence_strength = min(
+            total_strength / max(self.settings.evidence_scale, 0.1),
+            self.settings.evidence_cap,
+        )
+
+        posteriors: dict[str, float] = {}
+        deviation = 0.0
+        for domain in domains_touched:
+            prior = self.get_prior(region, domain)
+            deviation = max(deviation, self._deviation_score(region, domain, total_strength))
+            posterior = self.bayesian_update(
+                region=region,
+                domain=domain,
+                evidence_strength=evidence_strength * (1.0 + 0.15 * deviation),
+            )
+            posteriors[domain] = posterior
+
+        if isinstance(coords, (list, tuple)) and len(coords) >= 2:
+            with self._lock:
+                state = self._region_state(region)
+                try:
+                    state.coords = [float(coords[0]), float(coords[1])]
+                except (TypeError, ValueError):
+                    pass
+
+        self._update_graph(region, entities, total_strength, coords if isinstance(coords, list) else None)
+
+        composite = self.composite_risk(region)
+        entry = HistoryEntry(
+            timestamp=self._utcnow(),
+            domain=explicit_domain if explicit_domain in _DOMAINS else next(iter(domains_touched)),
+            signals=signals,
+            strength=total_strength,
+            prior=self._base_prior,
+            posterior=composite,
+            source=source,
+            deviation_score=deviation,
+        )
+        with self._lock:
+            history = self._history[region]
+            history.append(entry)
+            max_hist = max(10, int(self.settings.max_history_per_region))
+            if len(history) > max_hist:
+                self._history[region] = history[-max_hist:]
+
+        logger.info(
+            "GT update region=%s domains=%s composite=%.3f signals=%d deviation=%.2f",
+            region,
+            ",".join(sorted(domains_touched)),
+            composite,
+            len(signals),
+            deviation,
+        )
+
+        return {
+            "region": region,
+            "domains": sorted(domains_touched),
+            "domain_posteriors": posteriors,
+            "risk_score": composite,
+            "signals": signals,
+            "deviation_score": deviation,
+            "contagion_potential": self._get_contagion_score(region),
+            "interpretation": self._interpret_risk(composite),
+        }
+
+    def _interpret_risk(self, risk: float) -> str:
+        threshold = float(self.settings.high_risk_threshold)
+        if risk >= threshold:
+            return (
+                f"Elevated strategic risk ({risk:.2f} ≥ {threshold:.2f}). "
+                "Watch for costly-signal clustering and cross-region contagion."
+            )
+        if risk >= threshold * 0.7:
+            return "Moderate risk — monitor for herding and repeated costly signals."
+        return "Baseline risk — no strong costly-signal cluster detected."
+
+    def _get_contagion_score(self, region: str) -> float:
+        """Graph-based contagion: mean composite risk of graph neighbors."""
+        region_key = str(region or "global").strip().lower() or "global"
+        with self._lock:
+            if region_key not in self.G:
+                return 0.0
+            try:
+                neighbors = list(self.G.neighbors(region_key))
+            except nx.NetworkXError:
+                return 0.0
+            if not neighbors:
+                return 0.0
+            neighbor_risks = [self.composite_risk(str(n)) for n in neighbors]
+            return float(np.mean(neighbor_risks))
+
+    def compute_herding_clusters(self) -> list[dict[str, Any]]:
+        """Louvain community detection on entity graph (coordination/herding proxy)."""
+        with self._lock:
+            if self.G.number_of_edges() == 0:
+                return []
+
+            weighted = nx.Graph()
+            for u, v, data in self.G.edges(data=True):
+                weight = float(data.get("weight") or 0.0)
+                if weight < self.settings.louvain_min_weight:
+                    continue
+                if weighted.has_edge(u, v):
+                    weighted[u][v]["weight"] = weighted[u][v].get("weight", 0.0) + weight
+                else:
+                    weighted.add_edge(u, v, weight=weight)
+
+            if weighted.number_of_edges() == 0:
+                return []
+
+            try:
+                communities = list(nx.community.louvain_communities(weighted, weight="weight", seed=42))
+            except Exception as exc:
+                logger.warning("Louvain clustering failed: %s", exc)
+                return []
+
+            clusters: list[dict[str, Any]] = []
+            for idx, community in enumerate(communities):
+                members = sorted(str(node) for node in community)
+                region_members = [m for m in members if m in self._regions]
+                risks = [self.composite_risk(r) for r in region_members]
+                clusters.append(
+                    {
+                        "cluster_id": idx,
+                        "size": len(members),
+                        "members": members[:50],
+                        "mean_risk": float(np.mean(risks)) if risks else self._base_prior,
+                        "regions": region_members,
+                    }
+                )
+            clusters.sort(key=lambda row: row["mean_risk"], reverse=True)
+            return clusters
+
+    def get_risk_heatmap(self) -> dict[str, Any]:
+        """GeoJSON FeatureCollection for frontend risk overlay."""
+        features: list[dict[str, Any]] = []
+        with self._lock:
+            items = list(self._regions.items())[: max(1, self.settings.max_heatmap_features)]
+
+        for region, state in items:
+            coords = state.coords
+            geometry: dict[str, Any]
+            if coords and len(coords) >= 2:
+                geometry = {"type": "Point", "coordinates": [float(coords[1]), float(coords[0])]}
+            else:
+                geometry = {"type": "Point", "coordinates": [0.0, 0.0]}
+
+            composite = self.composite_risk(region)
+            features.append(
+                {
+                    "type": "Feature",
+                    "properties": {
+                        "region": region,
+                        "risk": round(composite, 4),
+                        "financial": round(float(state.priors.get("financial", self._base_prior)), 4),
+                        "unrest": round(float(state.priors.get("unrest", self._base_prior)), 4),
+                        "conflict": round(float(state.priors.get("conflict", self._base_prior)), 4),
+                        "contagion": round(self._get_contagion_score(region), 4),
+                        "updates": state.update_count,
+                    },
+                    "geometry": geometry,
+                }
+            )
+
+        return {"type": "FeatureCollection", "features": features}
+
+    def get_dossier(self, region: str) -> dict[str, Any]:
+        """Explainable GT rationale and recent signal history for a region."""
+        region_key = str(region or "global").strip().lower() or "global"
+        with self._lock:
+            state = self._region_state(region_key)
+            recent = list(self._history.get(region_key, [])[-10:])
+
+        composite = self.composite_risk(region_key)
+        return {
+            "region": region_key,
+            "current_risk": round(composite, 4),
+            "domain_risks": {
+                domain: round(float(state.priors.get(domain, self._base_prior)), 4)
+                for domain in _DOMAINS
+            },
+            "recent_signals": [
+                {
+                    "timestamp": entry.timestamp,
+                    "domain": entry.domain,
+                    "signals": entry.signals,
+                    "strength": entry.strength,
+                    "posterior": round(entry.posterior, 4),
+                    "source": entry.source,
+                    "deviation_score": round(entry.deviation_score, 3),
+                }
+                for entry in recent
+            ],
+            "contagion_risk": round(self._get_contagion_score(region_key), 4),
+            "herding_clusters": self.compute_herding_clusters()[:5],
+            "interpretation": self._interpret_risk(composite),
+            "scenarios": self._build_scenarios(region_key, composite),
+        }
+
+    def _build_scenarios(self, region: str, composite: float) -> list[dict[str, str]]:
+        threshold = float(self.settings.high_risk_threshold)
+        if composite < threshold * 0.7:
+            return [
+                {
+                    "name": "Status quo",
+                    "summary": "Signals remain diffuse; no coordinated costly-signal cascade.",
+                }
+            ]
+        if composite < threshold:
+            return [
+                {
+                    "name": "Escalation watch",
+                    "summary": "Rising costly-signal density — coordination risk within 4-8 weeks.",
+                },
+                {
+                    "name": "False alarm",
+                    "summary": "Cheap-talk amplification without follow-on costly signals.",
+                },
+            ]
+        return [
+            {
+                "name": "Contagion spread",
+                "summary": "High posterior + graph coupling — adjacent regions likely to update upward.",
+            },
+            {
+                "name": "Localized shock",
+                "summary": "Region-specific distress; contagion limited if graph neighbors stay quiet.",
+            },
+        ]
+
+    def snapshot(self) -> dict[str, Any]:
+        """Serialize engine state for debugging or persistence."""
+        with self._lock:
+            return {
+                "regions": {
+                    region: {
+                        "priors": dict(state.priors),
+                        "coords": state.coords,
+                        "updates": state.update_count,
+                    }
+                    for region, state in self._regions.items()
+                },
+                "graph_nodes": self.G.number_of_nodes(),
+                "graph_edges": self.G.number_of_edges(),
+                "processed_items": len(self._seen_item_ids),
+            }

backend/analytics/historical_events.py

@@ -0,0 +1,649 @@
+"""Curated historical early-warning cases for GT backtesting.
+
+Each positive case bundles pre-crisis costly-signal snippets drawn from documented
+precursors (financial, unrest, conflict). Negative cases are cheap-talk controls.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+CaseKind = Literal["positive", "negative"]
+
+
+@dataclass(frozen=True)
+class BacktestFeed:
+    text: str
+    source: str = "backtest"
+    domain: str = "financial"
+    days_before_event: int = 30
+
+
+@dataclass(frozen=True)
+class HistoricalCase:
+    """Single labeled backtest scenario."""
+
+    case_id: str
+    name: str
+    region: str
+    domain: str
+    kind: CaseKind
+    event_date: str
+    description: str
+    feeds: tuple[BacktestFeed, ...] = field(default_factory=tuple)
+    tags: tuple[str, ...] = field(default_factory=tuple)
+
+    def to_feed_dicts(self) -> list[dict[str, Any]]:
+        items: list[dict[str, Any]] = []
+        for idx, feed in enumerate(self.feeds):
+            items.append(
+                {
+                    "id": f"{self.case_id}-{idx}",
+                    "text": feed.text,
+                    "source": feed.source,
+                    "region": self.region,
+                    "domain": feed.domain or self.domain,
+                    "published": feed.days_before_event,
+                }
+            )
+        return items
+
+
+def _variant_case(case: HistoricalCase, suffix: str, feeds: tuple[BacktestFeed, ...]) -> HistoricalCase:
+    return HistoricalCase(
+        case_id=f"{case.case_id}__{suffix}",
+        name=f"{case.name} ({suffix})",
+        region=case.region,
+        domain=case.domain,
+        kind=case.kind,
+        event_date=case.event_date,
+        description=case.description,
+        feeds=feeds,
+        tags=case.tags + (f"variant:{suffix}",),
+    )
+
+
+def expanded_historical_cases() -> tuple[HistoricalCase, ...]:
+    """Base suite plus paraphrase variants for statistical confidence."""
+    base = list(default_historical_cases())
+    extras: list[HistoricalCase] = []
+
+    variant_feeds: dict[str, tuple[tuple[BacktestFeed, ...], ...]] = {
+        "fin_2008_us": (
+            (
+                BacktestFeed(
+                    "Small businesses turn to payroll loan products as credit lines freeze.",
+                    domain="financial",
+                    days_before_event=100,
+                ),
+                BacktestFeed(
+                    "FDIC monitors liquidity crunch; interbank spreads widen sharply.",
+                    domain="financial",
+                    days_before_event=60,
+                ),
+            ),
+            (
+                BacktestFeed(
+                    "Merchant cash advance volumes spike; payroll loan demand at record highs.",
+                    domain="financial",
+                    days_before_event=80,
+                ),
+                BacktestFeed(
+                    "Money market funds see inflows as deposit flight from regional banks continues.",
+                    domain="financial",
+                    days_before_event=40,
+                ),
+            ),
+        ),
+        "fin_2020_supply": (
+            (
+                BacktestFeed(
+                    "Electronics firms report shipping delay and port congestion across Pearl River Delta.",
+                    domain="financial",
+                    days_before_event=45,
+                ),
+                BacktestFeed(
+                    "Supply chain delay widens; logistics backlog hits automotive suppliers.",
+                    domain="financial",
+                    days_before_event=20,
+                ),
+            ),
+            (
+                BacktestFeed(
+                    "Container shortage fuels shipping delay; supply chain delay indices jump.",
+                    domain="financial",
+                    days_before_event=35,
+                ),
+                BacktestFeed(
+                    "Electronics assemblers warn of logistics backlog as port congestion spreads.",
+                    domain="financial",
+                    days_before_event=20,
+                ),
+                BacktestFeed(
+                    "Automotive suppliers flag supply chain delay after factory shutdowns in Hubei.",
+                    domain="financial",
+                    days_before_event=10,
+                ),
+            ),
+        ),
+        "fin_2022_sanctions": (
+            (
+                BacktestFeed(
+                    "Treasury drafts new sanctions escalation package on energy and finance sectors.",
+                    domain="financial",
+                    days_before_event=30,
+                ),
+                BacktestFeed(
+                    "Capital flight accelerates; elite relocation flights depart Moscow airports.",
+                    domain="financial",
+                    days_before_event=14,
+                ),
+            ),
+        ),
+        "unrest_arab_spring_egypt": (
+            (
+                BacktestFeed(
+                    "Cairo activists schedule mass rally; protest mobilization leaflets distributed.",
+                    domain="unrest",
+                    days_before_event=18,
+                ),
+                BacktestFeed(
+                    "Labor federations call general strike; strike posters cover downtown.",
+                    domain="unrest",
+                    days_before_event=8,
+                ),
+            ),
+        ),
+        "conflict_2022_ukraine": (
+            (
+                BacktestFeed(
+                    "Convoy of armored vehicles confirms troop movement near Sumy Oblast.",
+                    source="t.me/war_monitor",
+                    domain="conflict",
+                    days_before_event=20,
+                ),
+                BacktestFeed(
+                    "GNSS interference warnings follow GPS jamming spike along Belarus border.",
+                    source="t.me/osintdefender",
+                    domain="conflict",
+                    days_before_event=10,
+                ),
+            ),
+            (
+                BacktestFeed(
+                    "Military mobilization notices circulate; troop buildup confirmed by satellite firms.",
+                    domain="conflict",
+                    days_before_event=12,
+                ),
+            ),
+        ),
+        "neg_weather_us": (
+            (
+                BacktestFeed("Autumn foliage peaks in Vermont; pleasant hiking weather continues."),
+                BacktestFeed("County fair announces pie contest and livestock exhibitions."),
+            ),
+            (
+                BacktestFeed("Meteorologists predict mild hurricane season remainder for Gulf Coast."),
+            ),
+        ),
+        "neg_sports_uk": (
+            (
+                BacktestFeed("Rugby Six Nations standings update after weekend fixtures."),
+                BacktestFeed("Local marathon registration opens for charity runners."),
+            ),
+        ),
+        "neg_tech_global": (
+            (
+                BacktestFeed("Chipmaker announces efficiency gains in next-generation processor."),
+                BacktestFeed("Cloud provider opens new green datacenter in Nordic region."),
+            ),
+        ),
+    }
+
+    for case in base:
+        variants = variant_feeds.get(case.case_id, ())
+        for idx, feeds in enumerate(variants):
+            extras.append(_variant_case(case, f"v{idx+1}", feeds))
+
+    # Additional cheap-talk controls to widen negative sample
+    cheap_talk_regions = (
+        ("australia", "Museum opens contemporary art exhibit to strong attendance."),
+        ("spain", "Tomato harvest festival scheduled; regional trains add weekend service."),
+        ("south_korea", "K-pop group announces world tour dates for autumn."),
+        ("mexico", "Coastal cleanup volunteers restore beach habitats before holiday season."),
+        ("sweden", "City council approves bike lane expansion along waterfront."),
+        ("norway", "Salmon exports remain stable; fishing fleets report normal catch volumes."),
+        ("italy", "Truffle festival returns; restaurants publish seasonal tasting menus."),
+        ("poland", "University researchers release open-source astronomy software."),
+        ("thailand", "Monsoon rains ease; rice planting proceeds on normal schedule."),
+        ("vietnam", "Electronics assembly plants report steady export order books."),
+        ("south_africa", "Wildlife reserve reports rising ecotourism bookings."),
+        ("argentina", "Wine harvest festival opens; export cooperatives meet volume targets."),
+        ("netherlands", "Cycling championship draws international teams to canal district."),
+        ("belgium", "Chocolate exporters report stable holiday shipment schedules."),
+        ("portugal", "Offshore wind auction attracts multiple renewable bidders."),
+        ("greece", "Island ferry operators add routes ahead of summer travel season."),
+        ("turkey", "Cotton harvest forecast unchanged; textile orders stable."),
+        ("indonesia", "Volcano monitoring reports routine activity; tourism continues."),
+        ("philippines", "Coconut processors report normal logistics to export markets."),
+        ("malaysia", "Palm oil shipments on schedule; port throughput normal."),
+        ("new_zealand", "Sheep shearing competition draws rural crowds."),
+        ("ireland", "Tech conference highlights open-source database tooling."),
+        ("finland", "Sauna culture festival celebrates heritage with local artisans."),
+        ("denmark", "Wind turbine maintenance contracts renewed on prior terms."),
+        ("austria", "Ski resorts prepare slopes after early snowfall."),
+        ("switzerland", "Watchmakers unveil mechanical movement prototypes at trade fair."),
+        ("czech_republic", "Glassmakers export decorative pieces ahead of holiday season."),
+        ("romania", "Carpathian hiking trails reopen after spring maintenance."),
+        ("hungary", "Thermal bath tourism bookings rise for winter wellness season."),
+        ("peru", "Coffee cooperatives report stable harvest and export schedules."),
+        ("colombia", "Flower exporters prepare Valentine's shipments on normal cadence."),
+        ("morocco", "Citrus harvest meets forecasts; agricultural credit unchanged."),
+        ("kenya", "Tea auction volumes steady; freight routes operate normally."),
+        ("nigeria", "Nollywood studio announces family comedy release dates."),
+        ("ethiopia", "Coffee ceremony festival highlights regional bean varieties."),
+        ("saudi_arabia", "Desert conservation project plants drought-resistant shrubs."),
+        ("uae", "Airport duty-free operators expand luxury retail concourse."),
+        ("qatar", "Stadium operators prepare hospitality packages for sporting events."),
+        ("singapore", "Port authority reports container throughput on seasonal trend."),
+        ("hong_kong", "Art auction previews draw collectors to harborfront gallery."),
+        ("chile", "Vineyard tours report strong bookings ahead of harvest festival weekend."),
+        ("uruguay", "Beef exporters maintain steady shipment schedules to European buyers."),
+        ("iceland", "Geothermal spa resorts report normal winter visitor volumes."),
+        ("luxembourg", "Fund administrators publish routine quarterly disclosure filings."),
+        ("slovakia", "Mountain lodges prepare ski season openings after early snowfall."),
+        ("croatia", "Adriatic ferry operators add summer routes on prior timetable."),
+        ("bulgaria", "Rose oil cooperatives report stable export volumes to fragrance buyers."),
+        ("serbia", "Danube barge traffic proceeds on normal freight schedules."),
+        ("latvia", "Timber mills export lumber on unchanged contract terms."),
+        ("lithuania", "Baltic wind farms complete scheduled turbine maintenance rotations."),
+        ("estonia", "Digital residency applications processed at routine monthly pace."),
+        ("panama", "Canal transit volumes remain on seasonal trend; shipping fees unchanged."),
+    )
+    for idx, (region, text) in enumerate(cheap_talk_regions):
+        extras.append(
+            HistoricalCase(
+                case_id=f"neg_extra_{idx:02d}",
+                name=f"Benign regional news ({region})",
+                region=region,
+                domain="financial",
+                kind="negative",
+                event_date="2020-01-01",
+                description="Expanded cheap-talk control.",
+                feeds=(BacktestFeed(text),),
+                tags=("control", "expanded"),
+            )
+        )
+
+    return tuple(base + extras)
+
+
+def default_historical_cases() -> tuple[HistoricalCase, ...]:
+    """Benchmark suite — expand as new validated precursors are added."""
+    return (
+        # ── Financial distress ─────────────────────────────────────────────
+        HistoricalCase(
+            case_id="fin_2008_us",
+            name="2008 US financial crisis",
+            region="united_states",
+            domain="financial",
+            kind="positive",
+            event_date="2008-09-15",
+            description="Payroll-loan distress, liquidity crunch, and deposit flight precursors.",
+            tags=("2008", "financial", "lehman"),
+            feeds=(
+                BacktestFeed(
+                    "Franchise operators increasingly rely on payroll loan facilities as working capital tightens.",
+                    domain="financial",
+                    days_before_event=120,
+                ),
+                BacktestFeed(
+                    "Regional banks report liquidity crunch; CFOs warn of merchant cash advance reliance.",
+                    domain="financial",
+                    days_before_event=90,
+                ),
+                BacktestFeed(
+                    "Deposit flight accelerates at mid-size lenders; analysts flag bank run risk.",
+                    domain="financial",
+                    days_before_event=45,
+                ),
+            ),
+        ),
+        HistoricalCase(
+            case_id="fin_2020_supply",
+            name="COVID supply-chain shock",
+            region="china",
+            domain="financial",
+            kind="positive",
+            event_date="2020-02-01",
+            description="Port congestion and logistics backlog ahead of global supply shock.",
+            tags=("covid", "supply_chain", "financial"),
+            feeds=(
+                BacktestFeed(
+                    "Major port congestion reported; shipping delay spreads to electronics suppliers.",
+                    domain="financial",
+                    days_before_event=60,
+                ),
+                BacktestFeed(
+                    "Automakers warn of supply chain delay and logistics backlog across Wuhan corridor.",
+                    domain="financial",
+                    days_before_event=30,
+                ),
+                BacktestFeed(
+                    "Factory restarts slip as supply delay and port congestion persist into Q1.",
+                    domain="financial",
+                    days_before_event=14,
+                ),
+            ),
+        ),
+        HistoricalCase(
+            case_id="fin_2022_sanctions",
+            name="Russia sanctions escalation",
+            region="russia",
+            domain="financial",
+            kind="positive",
+            event_date="2022-02-24",
+            description="Sanctions escalation and capital flight ahead of invasion.",
+            tags=("sanctions", "ukraine", "financial"),
+            feeds=(
+                BacktestFeed(
+                    "Western allies prepare new sanctions escalation on major Russian banks.",
+                    domain="financial",
+                    days_before_event=45,
+                ),
+                BacktestFeed(
+                    "Oligarch jet movements suggest elite relocation and capital flight from Moscow.",
+                    domain="financial",
+                    days_before_event=21,
+                ),
+                BacktestFeed(
+                    "Central bank intervenes as new sanctions tighten export controls on finance sector.",
+                    domain="financial",
+                    days_before_event=10,
+                ),
+            ),
+        ),
+        # ── Civil unrest ─────────────────────────────────────────────────
+        HistoricalCase(
+            case_id="unrest_arab_spring_tunisia",
+            name="Arab Spring — Tunisia",
+            region="tunisia",
+            domain="unrest",
+            kind="positive",
+            event_date="2010-12-17",
+            description="Protest mobilization and strike waves before Jasmine Revolution.",
+            tags=("arab_spring", "unrest"),
+            feeds=(
+                BacktestFeed(
+                    "Student groups announce protest mobilization after vendor self-immolation.",
+                    domain="unrest",
+                    days_before_event=14,
+                ),
+                BacktestFeed(
+                    "Mass rally planned in Tunis; general strike called by labor unions.",
+                    domain="unrest",
+                    days_before_event=7,
+                ),
+            ),
+        ),
+        HistoricalCase(
+            case_id="unrest_arab_spring_egypt",
+            name="Arab Spring — Egypt",
+            region="egypt",
+            domain="unrest",
+            kind="positive",
+            event_date="2011-01-25",
+            description="Mobilization spikes and security reshuffles before Tahrir.",
+            tags=("arab_spring", "unrest"),
+            feeds=(
+                BacktestFeed(
+                    "Opposition calls protest mobilization in Cairo; strike notices circulate online.",
+                    domain="unrest",
+                    days_before_event=21,
+                ),
+                BacktestFeed(
+                    "Reports of political purge within interior ministry security apparatus reshuffle.",
+                    domain="unrest",
+                    days_before_event=10,
+                ),
+                BacktestFeed(
+                    "Mass rally and strike coordination spreads; rally posters appear in Alexandria.",
+                    domain="unrest",
+                    days_before_event=5,
+                ),
+            ),
+        ),
+        HistoricalCase(
+            case_id="unrest_2019_chile",
+            name="Chile 2019 metro protests",
+            region="chile",
+            domain="unrest",
+            kind="positive",
+            event_date="2019-10-18",
+            description="Transit fare protests escalate to general strike.",
+            tags=("unrest", "latam"),
+            feeds=(
+                BacktestFeed(
+                    "Students organize mass rally after metro fare hike; protest mobilization trending.",
+                    domain="unrest",
+                    days_before_event=10,
+                ),
+                BacktestFeed(
+                    "Unions announce general strike; rally and strike hashtags spike nationwide.",
+                    domain="unrest",
+                    days_before_event=3,
+                ),
+            ),
+        ),
+        # ── Conflict / war ───────────────────────────────────────────────
+        HistoricalCase(
+            case_id="conflict_2022_ukraine",
+            name="2022 Ukraine invasion buildup",
+            region="ukraine",
+            domain="conflict",
+            kind="positive",
+            event_date="2022-02-24",
+            description="Troop movement and GPS jamming precursors on northern border.",
+            tags=("ukraine", "conflict"),
+            feeds=(
+                BacktestFeed(
+                    "OSINT reports troop movement and armored convoy near Belarus border.",
+                    source="t.me/war_monitor",
+                    domain="conflict",
+                    days_before_event=30,
+                ),
+                BacktestFeed(
+                    "GPS jamming spike reported along northern corridor; GNSS interference warnings issued.",
+                    source="t.me/osintdefender",
+                    domain="conflict",
+                    days_before_event=14,
+                ),
+                BacktestFeed(
+                    "Satellite imagery shows troop buildup; military mobilization near Kharkiv axis.",
+                    domain="conflict",
+                    days_before_event=7,
+                ),
+            ),
+        ),
+        HistoricalCase(
+            case_id="conflict_2023_gaza",
+            name="2023 Gaza conflict escalation",
+            region="israel",
+            domain="conflict",
+            kind="positive",
+            event_date="2023-10-07",
+            description="Ceasefire breakdown and troop movement signals.",
+            tags=("gaza", "conflict"),
+            feeds=(
+                BacktestFeed(
+                    "Border units report troop movement near Gaza envelope; ceasefire broken overnight.",
+                    domain="conflict",
+                    days_before_event=14,
+                ),
+                BacktestFeed(
+                    "Truce end announced; armored convoy repositioning reported by local observers.",
+                    domain="conflict",
+                    days_before_event=5,
+                ),
+            ),
+        ),
+        HistoricalCase(
+            case_id="conflict_2020_nagorno",
+            name="2020 Nagorno-Karabakh renewal",
+            region="armenia",
+            domain="conflict",
+            kind="positive",
+            event_date="2020-09-27",
+            description="Artillery and troop buildup precursors.",
+            tags=("caucasus", "conflict"),
+            feeds=(
+                BacktestFeed(
+                    "Drone strikes reported on line of contact; troop movement on Armenian-Azeri border.",
+                    domain="conflict",
+                    days_before_event=21,
+                ),
+                BacktestFeed(
+                    "GPS jamming spike reported in conflict zone; military mobilization notices leaked.",
+                    domain="conflict",
+                    days_before_event=7,
+                ),
+            ),
+        ),
+        # ── Recent financial / corporate distress pattern ────────────────
+        HistoricalCase(
+            case_id="fin_2023_banking",
+            name="2023 regional banking stress",
+            region="united_states",
+            domain="financial",
+            kind="positive",
+            event_date="2023-03-10",
+            description="Deposit flight and liquidity stress (SVB precursor pattern).",
+            tags=("svb", "financial", "2023"),
+            feeds=(
+                BacktestFeed(
+                    "Tech lenders face deposit flight; VC portfolio companies move payroll to money market funds.",
+                    domain="financial",
+                    days_before_event=21,
+                ),
+                BacktestFeed(
+                    "Analysts warn liquidity crunch at regional banks holding long-duration bonds.",
+                    domain="financial",
+                    days_before_event=7,
+                ),
+            ),
+        ),
+        # ── Negative controls (cheap talk / benign) ─────────────────────
+        HistoricalCase(
+            case_id="neg_weather_us",
+            name="Benign weather coverage",
+            region="united_states",
+            domain="financial",
+            kind="negative",
+            event_date="2019-06-01",
+            description="No costly signals — should remain near baseline.",
+            tags=("control",),
+            feeds=(
+                BacktestFeed("Sunny weekend expected across the Midwest with mild temperatures."),
+                BacktestFeed("Local festival draws crowds; farmers market expands summer hours."),
+            ),
+        ),
+        HistoricalCase(
+            case_id="neg_sports_uk",
+            name="Benign sports coverage",
+            region="uk",
+            domain="unrest",
+            kind="negative",
+            event_date="2018-07-01",
+            description="Sports chatter without mobilization costly signals.",
+            tags=("control",),
+            feeds=(
+                BacktestFeed("Premier league season review: top scorers and transfer rumors."),
+                BacktestFeed("Cricket test match ends early due to rain delay at Lord's."),
+            ),
+        ),
+        HistoricalCase(
+            case_id="neg_tech_global",
+            name="Benign tech product launch",
+            region="global",
+            domain="financial",
+            kind="negative",
+            event_date="2021-09-01",
+            description="Corporate product news without distress markers.",
+            tags=("control",),
+            feeds=(
+                BacktestFeed("Smartphone maker unveils new camera features at annual keynote."),
+                BacktestFeed("Quarterly earnings beat expectations; dividend unchanged."),
+            ),
+        ),
+        HistoricalCase(
+            case_id="neg_tourism_france",
+            name="Benign tourism recovery",
+            region="france",
+            domain="unrest",
+            kind="negative",
+            event_date="2022-08-01",
+            description="Travel sector recovery without unrest signals.",
+            tags=("control",),
+            feeds=(
+                BacktestFeed("Paris hotels report record summer bookings as tourism rebounds."),
+                BacktestFeed("Airline adds routes to Nice and Marseille for holiday travelers."),
+            ),
+        ),
+        HistoricalCase(
+            case_id="neg_science_japan",
+            name="Benign science news",
+            region="japan",
+            domain="conflict",
+            kind="negative",
+            event_date="2020-11-01",
+            description="Research coverage without conflict markers.",
+            tags=("control",),
+            feeds=(
+                BacktestFeed("Astronomy team publishes comet observations from Mount Fuji observatory."),
+                BacktestFeed("Robotics lab demonstrates warehouse automation prototype."),
+            ),
+        ),
+        HistoricalCase(
+            case_id="neg_agriculture_brazil",
+            name="Benign agriculture report",
+            region="brazil",
+            domain="financial",
+            kind="negative",
+            event_date="2017-03-01",
+            description="Commodity harvest update without supply distress.",
+            tags=("control",),
+            feeds=(
+                BacktestFeed("Soybean harvest forecast revised upward; export volumes steady."),
+                BacktestFeed("Coffee cooperative reports normal shipping schedules to European buyers."),
+            ),
+        ),
+        HistoricalCase(
+            case_id="neg_culture_india",
+            name="Benign culture coverage",
+            region="india",
+            domain="unrest",
+            kind="negative",
+            event_date="2016-11-01",
+            description="Festival coverage without mobilization.",
+            tags=("control",),
+            feeds=(
+                BacktestFeed("Diwali celebrations begin; cities decorate markets with lights."),
+                BacktestFeed("Film festival opens in Mumbai with premiere screenings."),
+            ),
+        ),
+        HistoricalCase(
+            case_id="neg_infrastructure_canada",
+            name="Benign infrastructure ribbon-cutting",
+            region="canada",
+            domain="financial",
+            kind="negative",
+            event_date="2015-05-01",
+            description="Municipal news without financial stress.",
+            tags=("control",),
+            feeds=(
+                BacktestFeed("New light-rail segment opens on schedule; commute times improve."),
+                BacktestFeed("Municipal bond issuance funds library renovation at prior rates."),
+            ),
+        ),
+    )

backend/analytics/integration.py

@@ -0,0 +1,198 @@
+"""Singleton GT engine and feed-batch integration hooks."""
+
+from __future__ import annotations
+
+import logging
+import threading
+from datetime import datetime, timezone
+from typing import Any
+
+from analytics.feed_adapter import iter_gdelt_features, iter_news_items, iter_telegram_posts
+from analytics.gt_early_warning import GT_EarlyWarning
+from analytics.settings import gt_analytics_enabled, get_gt_settings, gt_engine_operational, gt_louvain_enabled, gt_scheduled_ingest_enabled
+from services.fetchers._store import _data_lock, _mark_fresh, latest_data
+
+logger = logging.getLogger(__name__)
+
+_engine: GT_EarlyWarning | None = None
+_engine_lock = threading.Lock()
+
+
+def get_gt_engine() -> GT_EarlyWarning | None:
+    """Return the shared engine when analytics are enabled and runtime allows it."""
+    global _engine
+    if not gt_engine_operational():
+        return None
+    with _engine_lock:
+        if _engine is None:
+            _engine = GT_EarlyWarning(get_gt_settings())
+            logger.info("Strategic Risk Analytics engine initialized")
+        return _engine
+
+
+def reset_gt_engine() -> None:
+    """Reset singleton — intended for tests."""
+    global _engine
+    get_gt_settings.cache_clear()
+    with _engine_lock:
+        _engine = None
+
+
+def process_feed_item(item: dict[str, Any]) -> dict[str, Any] | None:
+    """Process a normalized feed item if analytics are enabled."""
+    engine = get_gt_engine()
+    if engine is None:
+        return None
+    try:
+        return engine.process_feed_item(item)
+    except Exception:
+        logger.exception("GT process_feed_item failed")
+        return None
+
+
+def _persist_gt_snapshot(
+    engine: GT_EarlyWarning,
+    *,
+    processed: int,
+    sample: list[dict[str, Any]] | None = None,
+) -> dict[str, Any]:
+    timestamp = datetime.now(timezone.utc).isoformat()
+    heatmap = engine.get_risk_heatmap()
+    micro_summary: dict[str, Any] = {}
+    try:
+        from analytics.micro_rolling import capture_daily_readings, enrich_heatmap_features
+
+        micro_summary = capture_daily_readings(engine)
+        heatmap = enrich_heatmap_features(heatmap)
+    except Exception:
+        logger.exception("GT micro rolling capture failed")
+
+    clusters = engine.compute_herding_clusters()
+    from analytics.gt_alerts import parse_heatmap_alerts
+
+    _, plotted_regions = parse_heatmap_alerts(heatmap)
+    with engine._lock:  # noqa: SLF001 — snapshot meta
+        engine_regions = len(engine._regions)
+    settings = get_gt_settings()
+    payload = {
+        "enabled": True,
+        "timestamp": timestamp,
+        "processed": processed,
+        "heatmap": heatmap,
+        "clusters": clusters,
+        "sample": list(sample or [])[:5],
+        "regions": len(heatmap.get("features") or []),
+        "micro": micro_summary,
+        "meta": {
+            "tracked_regions": len(heatmap.get("features") or []),
+            "engine_regions": engine_regions,
+            "plotted_regions": plotted_regions,
+            "max_regions": settings.max_heatmap_features,
+        },
+    }
+    with _data_lock:
+        latest_data["gt_risk"] = payload
+    _mark_fresh("gt_risk")
+    return payload
+
+
+def refresh_from_latest_data(
+    data_snapshot: dict[str, Any],
+    *,
+    persist: bool = True,
+) -> dict[str, Any]:
+    """
+    Batch-ingest recent intel layers from the shared data store.
+
+    Intended to run after telegram/news/gdelt fetch cycles (near-real-time).
+    """
+    engine = get_gt_engine()
+    if engine is None:
+        return {"enabled": False, "processed": 0}
+
+    processed = 0
+    results: list[dict[str, Any]] = []
+
+    for item in iter_telegram_posts(data_snapshot.get("telegram_osint")):
+        result = engine.process_feed_item(item)
+        if result and not result.get("skipped"):
+            processed += 1
+            results.append(result)
+
+    for item in iter_news_items(data_snapshot.get("news")):
+        result = engine.process_feed_item(item)
+        if result and not result.get("skipped"):
+            processed += 1
+            if len(results) < 5:
+                results.append(result)
+
+    for item in iter_gdelt_features(data_snapshot.get("gdelt")):
+        result = engine.process_feed_item(item)
+        if result and not result.get("skipped"):
+            processed += 1
+
+    logger.info("GT refresh processed %d items", processed)
+    summary = {
+        "enabled": True,
+        "processed": processed,
+        "sample": results[:5],
+        "heatmap_features": len(engine.get_risk_heatmap().get("features") or []),
+    }
+    if persist:
+        snapshot = _persist_gt_snapshot(engine, processed=processed, sample=results)
+        summary["timestamp"] = snapshot.get("timestamp")
+        summary["clusters"] = len(snapshot.get("clusters") or [])
+    return summary
+
+
+def recompute_gt_herding_clusters() -> dict[str, Any]:
+    """Louvain community pass — run on a schedule independent of feed ingest."""
+    if not gt_louvain_enabled():
+        return {"enabled": False, "clusters": 0, "reason": "louvain_disabled_on_lean_profile"}
+
+    engine = get_gt_engine()
+    if engine is None:
+        return {"enabled": False, "clusters": 0}
+
+    clusters = engine.compute_herding_clusters()
+    timestamp = datetime.now(timezone.utc).isoformat()
+    with _data_lock:
+        current = dict(latest_data.get("gt_risk") or {})
+        current["clusters"] = clusters
+        current["clusters_updated"] = timestamp
+        current["enabled"] = True
+        latest_data["gt_risk"] = current
+    _mark_fresh("gt_risk")
+    logger.info("GT Louvain recompute: %d clusters", len(clusters))
+    return {"enabled": True, "clusters": len(clusters), "timestamp": timestamp}
+
+
+def maybe_refresh_gt_analytics() -> None:
+    """Hook for data_fetcher — no-op when analytics are disabled or lean-gated."""
+    if not gt_scheduled_ingest_enabled():
+        return
+    try:
+        with _data_lock:
+            snapshot = dict(latest_data)
+        refresh_from_latest_data(snapshot, persist=True)
+    except Exception:
+        logger.exception("GT analytics refresh failed")
+
+
+def maybe_freeze_gt_weekly_snapshot() -> None:
+    """Hook for weekly scheduler — freeze operational backtest snapshot."""
+    if not gt_engine_operational():
+        return
+    try:
+        from analytics.rolling_backtest import freeze_weekly_snapshot
+
+        result = freeze_weekly_snapshot(frozen_by="scheduler")
+        if result.get("created"):
+            logger.info(
+                "GT rolling freeze: week=%s regions=%s alerts=%s",
+                result.get("week_id"),
+                result.get("region_count"),
+                result.get("alert_count"),
+            )
+    except Exception:
+        logger.exception("GT rolling weekly freeze failed")

backend/analytics/micro_rolling.py

@@ -0,0 +1,361 @@
+"""Micro rolling 3-day average — fast ignition signal alongside weekly macro."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from datetime import date, datetime, timedelta, timezone
+from typing import Any
+
+from analytics.daily_store import (
+    DailyRegionReading,
+    DailySnapshot,
+    date_id,
+    list_daily_ids,
+    load_daily,
+    save_daily,
+    utc_now_iso,
+    utc_today,
+)
+from analytics.gt_early_warning import GT_EarlyWarning
+from analytics.rolling_backtest import rolling_alert_threshold
+
+DEFAULT_WINDOW_DAYS = 3
+DEFAULT_IGNITION_DELTA = 0.10
+
+
+def _env_int(name: str, default: int) -> int:
+    raw = str(os.environ.get(name, "")).strip()
+    if not raw:
+        return default
+    try:
+        return max(1, int(raw))
+    except ValueError:
+        return default
+
+
+def _env_float(name: str, default: float) -> float:
+    raw = str(os.environ.get(name, "")).strip()
+    if not raw:
+        return default
+    try:
+        return float(raw)
+    except ValueError:
+        return default
+
+
+def micro_window_days() -> int:
+    return _env_int("GT_MICRO_ROLLING_DAYS", DEFAULT_WINDOW_DAYS)
+
+
+def ignition_delta() -> float:
+    return _env_float("GT_MICRO_IGNITION_DELTA", DEFAULT_IGNITION_DELTA)
+
+
+def _peak_score(
+    *,
+    composite: float,
+    financial: float,
+    unrest: float,
+    conflict: float,
+) -> float:
+    return max(composite, financial, unrest, conflict)
+
+
+def _region_reading_from_feature(
+    feature: dict[str, Any],
+    *,
+    captured_at: str,
+) -> DailyRegionReading | None:
+    props = feature.get("properties") or {}
+    region = str(props.get("region") or "").strip().lower()
+    if not region:
+        return None
+    composite = float(props.get("risk") or props.get("composite_risk") or 0.0)
+    financial = float(props.get("financial") or 0.0)
+    unrest = float(props.get("unrest") or 0.0)
+    conflict = float(props.get("conflict") or 0.0)
+    peak = _peak_score(
+        composite=composite,
+        financial=financial,
+        unrest=unrest,
+        conflict=conflict,
+    )
+    return DailyRegionReading(
+        region=region,
+        composite_risk=composite,
+        financial=financial,
+        unrest=unrest,
+        conflict=conflict,
+        peak_score=peak,
+        readings=1,
+        last_captured_at=captured_at,
+    )
+
+
+def capture_daily_readings(
+    engine: GT_EarlyWarning,
+    *,
+    when: date | None = None,
+) -> dict[str, Any]:
+    """
+    Upsert today's regional readings from the live heatmap.
+
+    Each GT refresh updates the current day's latest scores (rolling window
+    uses one value per calendar day).
+    """
+    day = when or utc_today()
+    day_key = date_id(day)
+    captured_at = utc_now_iso()
+    heatmap = engine.get_risk_heatmap()
+    existing = load_daily(day) or DailySnapshot(date=day_key, regions={})
+
+    updated = 0
+    for feature in heatmap.get("features") or []:
+        if not isinstance(feature, dict):
+            continue
+        reading = _region_reading_from_feature(feature, captured_at=captured_at)
+        if reading is None:
+            continue
+        prior = existing.regions.get(reading.region)
+        if prior is None:
+            existing.regions[reading.region] = reading
+            updated += 1
+            continue
+        prior.composite_risk = reading.composite_risk
+        prior.financial = reading.financial
+        prior.unrest = reading.unrest
+        prior.conflict = reading.conflict
+        prior.peak_score = max(prior.peak_score, reading.peak_score)
+        prior.readings += 1
+        prior.last_captured_at = captured_at
+        updated += 1
+
+    existing.last_updated_at = captured_at
+    save_daily(existing)
+    return {
+        "date": day_key,
+        "regions": len(existing.regions),
+        "updated": updated,
+        "captured_at": captured_at,
+    }
+
+
+@dataclass(frozen=True)
+class MicroRegionView:
+    region: str
+    spot_risk: float
+    risk_3d_avg: float
+    risk_delta: float
+    days_in_window: int
+    day_scores: tuple[float, ...]
+    alerted_spot: bool
+    alerted_3d: bool
+    ignition: bool
+    financial: float
+    unrest: float
+    conflict: float
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "region": self.region,
+            "spot_risk": round(self.spot_risk, 4),
+            "risk_3d_avg": round(self.risk_3d_avg, 4),
+            "risk_delta": round(self.risk_delta, 4),
+            "days_in_window": self.days_in_window,
+            "day_scores": [round(score, 4) for score in self.day_scores],
+            "alerted_spot": self.alerted_spot,
+            "alerted_3d": self.alerted_3d,
+            "ignition": self.ignition,
+            "financial": round(self.financial, 4),
+            "unrest": round(self.unrest, 4),
+            "conflict": round(self.conflict, 4),
+        }
+
+
+def _day_offsets(window_days: int) -> list[int]:
+    # Today + prior (window_days - 1) days.
+    return list(range(window_days - 1, -1, -1))
+
+
+def _historical_dates(as_of: date, window_days: int) -> list[date]:
+    return [as_of - timedelta(days=offset) for offset in _day_offsets(window_days)]
+
+
+def compute_micro_view(
+    region: str,
+    *,
+    as_of: date | None = None,
+    window_days: int | None = None,
+    alert_threshold: float | None = None,
+    spot_reading: DailyRegionReading | None = None,
+) -> MicroRegionView | None:
+    """Compute rolling N-day average and ignition vs spot for one region."""
+    region_key = str(region or "").strip().lower()
+    if not region_key:
+        return None
+
+    today = as_of or utc_today()
+    window = window_days or micro_window_days()
+    threshold = float(alert_threshold if alert_threshold is not None else rolling_alert_threshold())
+    delta_min = ignition_delta()
+
+    day_scores: list[float] = []
+    latest: DailyRegionReading | None = spot_reading
+
+    for day in _historical_dates(today, window):
+        snap = load_daily(day)
+        if snap is None:
+            continue
+        row = snap.regions.get(region_key)
+        if row is None:
+            continue
+        day_scores.append(row.peak_score)
+        if day == today:
+            latest = row
+
+    if latest is None and day_scores:
+        # Spot may come from yesterday if today not captured yet.
+        snap = load_daily(today)
+        if snap:
+            latest = snap.regions.get(region_key)
+
+    if latest is None and not day_scores:
+        return None
+
+    spot = float(latest.peak_score if latest else (day_scores[-1] if day_scores else 0.0))
+    avg = sum(day_scores) / len(day_scores) if day_scores else spot
+    risk_delta = spot - avg
+    ignition = risk_delta >= delta_min and spot >= threshold * 0.75
+
+    return MicroRegionView(
+        region=region_key,
+        spot_risk=spot,
+        risk_3d_avg=avg,
+        risk_delta=risk_delta,
+        days_in_window=len(day_scores),
+        day_scores=tuple(day_scores),
+        alerted_spot=spot >= threshold,
+        alerted_3d=avg >= threshold,
+        ignition=ignition,
+        financial=float(latest.financial if latest else 0.0),
+        unrest=float(latest.unrest if latest else 0.0),
+        conflict=float(latest.conflict if latest else 0.0),
+    )
+
+
+def compute_all_micro_views(
+    *,
+    as_of: date | None = None,
+    window_days: int | None = None,
+    alert_threshold: float | None = None,
+) -> list[MicroRegionView]:
+    """Build micro views for all regions seen in the rolling window."""
+    today = as_of or utc_today()
+    window = window_days or micro_window_days()
+    regions: set[str] = set()
+
+    for day in _historical_dates(today, window):
+        snap = load_daily(day)
+        if snap is None:
+            continue
+        regions.update(snap.regions.keys())
+
+    views: list[MicroRegionView] = []
+    for region in regions:
+        view = compute_micro_view(
+            region,
+            as_of=today,
+            window_days=window,
+            alert_threshold=alert_threshold,
+        )
+        if view is not None:
+            views.append(view)
+
+    views.sort(key=lambda row: (row.ignition, row.risk_delta, row.spot_risk), reverse=True)
+    return views
+
+
+def enrich_heatmap_features(
+    heatmap: dict[str, Any],
+    *,
+    as_of: date | None = None,
+    window_days: int | None = None,
+    alert_threshold: float | None = None,
+) -> dict[str, Any]:
+    """Attach micro rolling fields to heatmap GeoJSON features."""
+    threshold = float(alert_threshold if alert_threshold is not None else rolling_alert_threshold())
+    window = window_days or micro_window_days()
+    features = heatmap.get("features") or []
+    enriched: list[dict[str, Any]] = []
+
+    for feature in features:
+        if not isinstance(feature, dict):
+            continue
+        props = dict(feature.get("properties") or {})
+        region = str(props.get("region") or "").strip().lower()
+        view = compute_micro_view(
+            region,
+            as_of=as_of,
+            window_days=window,
+            alert_threshold=threshold,
+        ) if region else None
+
+        if view is not None:
+            props["risk_spot"] = view.spot_risk
+            props["risk_3d_avg"] = view.risk_3d_avg
+            props["risk_delta"] = view.risk_delta
+            props["micro_days"] = view.days_in_window
+            props["micro_ignition"] = view.ignition
+            props["alerted_3d"] = view.alerted_3d
+            props["day_scores"] = list(view.day_scores)
+
+        enriched.append({**feature, "properties": props})
+
+    return {
+        **heatmap,
+        "features": enriched,
+        "micro_window_days": window,
+        "micro_alert_threshold": threshold,
+    }
+
+
+def micro_rolling_report(
+    *,
+    as_of: date | None = None,
+    window_days: int | None = None,
+    limit: int = 15,
+) -> dict[str, Any]:
+    """API/OpenClaw payload for micro rolling 3-day context."""
+    today = as_of or utc_today()
+    window = window_days or micro_window_days()
+    threshold = rolling_alert_threshold()
+    views = compute_all_micro_views(
+        as_of=today,
+        window_days=window,
+        alert_threshold=threshold,
+    )
+    ignitions = [row for row in views if row.ignition]
+    alerted_3d = [row for row in views if row.alerted_3d]
+    top = views[: max(1, limit)]
+
+    stored_days = list_daily_ids(newest_first=True, limit=window)
+    return {
+        "mode": "micro_rolling",
+        "window_days": window,
+        "alert_threshold": threshold,
+        "ignition_delta": ignition_delta(),
+        "as_of": date_id(today),
+        "days_stored": len(stored_days),
+        "stored_dates": stored_days,
+        "regions_tracked": len(views),
+        "ignition_count": len(ignitions),
+        "alerted_3d_count": len(alerted_3d),
+        "ignitions": [row.to_dict() for row in ignitions[:limit]],
+        "top_regions": [row.to_dict() for row in top],
+        "note": (
+            f"Micro view: {window}-day rolling average vs spot risk. "
+            "Ignition = spot jumped above the rolling baseline (events that flare fast). "
+            "Macro week-over-week validation remains on /api/analytics/rolling."
+        ),
+    }

backend/analytics/rolling_backtest.py

@@ -0,0 +1,382 @@
+"""Rolling weekly operational validation for Strategic Risk Analytics.
+
+Freezes live GT scores each ISO week, accepts delayed outcome labels, and
+scores prior-week predictions with accuracy + Wilson 95% CI. Unlike the
+static historical benchmark, this measures forward operational usefulness.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from datetime import date, datetime, timezone
+from typing import Any, Literal
+
+from analytics.backtest import DEFAULT_BACKTEST_ALERT_THRESHOLD, wilson_interval
+from analytics.gt_early_warning import GT_EarlyWarning
+from analytics.integration import get_gt_engine
+from analytics.weekly_store import (
+    VALID_LABELS,
+    LabelName,
+    RegionSnapshot,
+    WeeklySnapshot,
+    list_week_ids,
+    load_week,
+    save_week,
+    utc_now_iso,
+)
+
+MIN_LABELED_FOR_TREND = 5
+
+
+def _env_float(name: str, default: float) -> float:
+    raw = str(os.environ.get(name, "")).strip()
+    if not raw:
+        return default
+    try:
+        return float(raw)
+    except ValueError:
+        return default
+
+
+def rolling_alert_threshold() -> float:
+    """Fixed operational alert cutoff — not retroactively tuned per week."""
+    return _env_float("GT_ROLLING_ALERT_THRESHOLD", DEFAULT_BACKTEST_ALERT_THRESHOLD)
+
+
+def iso_week_id(when: datetime | date | None = None) -> str:
+    """Return ISO week id, e.g. ``2026-W24``."""
+    if when is None:
+        when = datetime.now(timezone.utc)
+    if isinstance(when, datetime):
+        when = when.date()
+    year, week, _ = when.isocalendar()
+    return f"{year}-W{week:02d}"
+
+
+def _region_rows_from_engine(
+    engine: GT_EarlyWarning,
+    *,
+    alert_threshold: float,
+) -> list[RegionSnapshot]:
+    heatmap = engine.get_risk_heatmap()
+    rows: list[RegionSnapshot] = []
+    for feature in heatmap.get("features") or []:
+        if not isinstance(feature, dict):
+            continue
+        props = feature.get("properties") or {}
+        region = str(props.get("region") or "").strip().lower()
+        if not region:
+            continue
+        composite = float(props.get("risk") or 0.0)
+        financial = float(props.get("financial") or 0.0)
+        unrest = float(props.get("unrest") or 0.0)
+        conflict = float(props.get("conflict") or 0.0)
+        peak_score = max(composite, financial, unrest, conflict)
+        rows.append(
+            RegionSnapshot(
+                region=region,
+                composite_risk=composite,
+                financial=financial,
+                unrest=unrest,
+                conflict=conflict,
+                alerted=peak_score >= alert_threshold,
+                label="pending",
+            )
+        )
+    rows.sort(key=lambda row: row.composite_risk, reverse=True)
+    return rows
+
+
+@dataclass(frozen=True)
+class WeekScore:
+    week_id: str
+    frozen_at: str
+    alert_threshold: float
+    total_regions: int
+    labeled: int
+    pending: int
+    alerted: int
+    correct: int
+    accuracy: float
+    confidence_rate: float
+    wilson_lower_95: float
+    wilson_upper_95: float
+    true_positives: int
+    true_negatives: int
+    false_positives: int
+    false_negatives: int
+    sensitivity: float
+    specificity: float
+    scorable: bool
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "week_id": self.week_id,
+            "frozen_at": self.frozen_at,
+            "alert_threshold": round(self.alert_threshold, 4),
+            "total_regions": self.total_regions,
+            "labeled": self.labeled,
+            "pending": self.pending,
+            "alerted": self.alerted,
+            "correct": self.correct,
+            "accuracy": round(self.accuracy, 4),
+            "confidence_rate": round(self.confidence_rate, 4),
+            "wilson_lower_95": round(self.wilson_lower_95, 4),
+            "wilson_upper_95": round(self.wilson_upper_95, 4),
+            "true_positives": self.true_positives,
+            "true_negatives": self.true_negatives,
+            "false_positives": self.false_positives,
+            "false_negatives": self.false_negatives,
+            "sensitivity": round(self.sensitivity, 4),
+            "specificity": round(self.specificity, 4),
+            "scorable": self.scorable,
+        }
+
+
+def _predicted_positive(row: RegionSnapshot) -> bool:
+    return row.alerted
+
+
+def _actual_positive(label: LabelName) -> bool:
+    return label == "true_escalation"
+
+
+def _is_correct(row: RegionSnapshot) -> bool:
+    if row.label == "pending":
+        return False
+    predicted = _predicted_positive(row)
+    if row.label == "true_escalation":
+        return predicted
+    if row.label in ("false_alarm", "benign"):
+        return not predicted
+    return False
+
+
+def score_week(snapshot: WeeklySnapshot) -> WeekScore:
+    """Score a frozen week against delayed labels (pending rows excluded)."""
+    labeled_rows = [row for row in snapshot.regions if row.label != "pending"]
+    pending = len(snapshot.regions) - len(labeled_rows)
+
+    tp = sum(
+        1
+        for row in labeled_rows
+        if row.alerted and row.label == "true_escalation"
+    )
+    tn = sum(
+        1
+        for row in labeled_rows
+        if not row.alerted and row.label in ("benign", "false_alarm")
+    )
+    fp = sum(
+        1
+        for row in labeled_rows
+        if row.alerted and row.label in ("false_alarm", "benign")
+    )
+    fn = sum(
+        1
+        for row in labeled_rows
+        if not row.alerted and row.label == "true_escalation"
+    )
+
+    correct = tp + tn
+    total = len(labeled_rows)
+    accuracy = correct / total if total else 0.0
+    lower, upper = wilson_interval(correct, total)
+
+    pos_total = sum(1 for row in labeled_rows if _actual_positive(row.label))  # type: ignore[arg-type]
+    neg_total = total - pos_total
+    pred_pos = sum(1 for row in labeled_rows if row.alerted)
+    pred_neg = total - pred_pos
+
+    sensitivity = tp / pos_total if pos_total else 0.0
+    specificity = tn / pred_neg if pred_neg else (1.0 if tn == total and total else 0.0)
+
+    return WeekScore(
+        week_id=snapshot.week_id,
+        frozen_at=snapshot.frozen_at,
+        alert_threshold=snapshot.alert_threshold,
+        total_regions=len(snapshot.regions),
+        labeled=total,
+        pending=pending,
+        alerted=sum(1 for row in snapshot.regions if row.alerted),
+        correct=correct,
+        accuracy=accuracy,
+        confidence_rate=lower,
+        wilson_lower_95=lower,
+        wilson_upper_95=upper,
+        true_positives=tp,
+        true_negatives=tn,
+        false_positives=fp,
+        false_negatives=fn,
+        sensitivity=sensitivity,
+        specificity=specificity,
+        scorable=total >= MIN_LABELED_FOR_TREND,
+    )
+
+
+def freeze_weekly_snapshot(
+    *,
+    week_id: str | None = None,
+    alert_threshold: float | None = None,
+    force: bool = False,
+    frozen_by: str = "system",
+    engine: GT_EarlyWarning | None = None,
+) -> dict[str, Any]:
+    """
+    Capture current GT heatmap as an immutable weekly operational snapshot.
+
+    Idempotent per week unless ``force=True``.
+    """
+    resolved_engine = engine or get_gt_engine()
+    if resolved_engine is None:
+        return {"ok": False, "detail": "GT analytics engine unavailable"}
+
+    resolved_week = week_id or iso_week_id()
+    threshold = float(
+        alert_threshold if alert_threshold is not None else rolling_alert_threshold()
+    )
+
+    existing = load_week(resolved_week)
+    if existing and existing.regions and not force:
+        score = score_week(existing)
+        return {
+            "ok": True,
+            "created": False,
+            "week_id": resolved_week,
+            "snapshot": existing.to_dict(),
+            "score": score.to_dict(),
+        }
+
+    regions = _region_rows_from_engine(resolved_engine, alert_threshold=threshold)
+    snapshot = WeeklySnapshot(
+        week_id=resolved_week,
+        frozen_at=utc_now_iso(),
+        alert_threshold=threshold,
+        regions=regions,
+        frozen_by=frozen_by,
+    )
+    save_week(snapshot)
+    score = score_week(snapshot)
+    return {
+        "ok": True,
+        "created": True,
+        "week_id": resolved_week,
+        "snapshot": snapshot.to_dict(),
+        "score": score.to_dict(),
+        "alert_count": sum(1 for row in regions if row.alerted),
+        "region_count": len(regions),
+    }
+
+
+def label_regions(
+    week_id: str,
+    labels: list[dict[str, Any]],
+    *,
+    labeled_by: str = "operator",
+) -> dict[str, Any]:
+    """Apply delayed outcome labels to a frozen week."""
+    snapshot = load_week(week_id)
+    if snapshot is None:
+        return {"ok": False, "detail": f"Week {week_id} not found"}
+
+    by_region = {row.region: row for row in snapshot.regions}
+    updated = 0
+    skipped: list[str] = []
+    now = utc_now_iso()
+
+    for entry in labels:
+        if not isinstance(entry, dict):
+            continue
+        region = str(entry.get("region") or "").strip().lower()
+        label = str(entry.get("label") or "").strip().lower()
+        if not region or label not in VALID_LABELS or label == "pending":
+            if region:
+                skipped.append(region)
+            continue
+        row = by_region.get(region)
+        if row is None:
+            skipped.append(region)
+            continue
+        row.label = label  # type: ignore[assignment]
+        row.labeled_at = now
+        notes = entry.get("notes")
+        if notes is not None:
+            row.notes = str(notes)
+        updated += 1
+
+    save_week(snapshot)
+    score = score_week(snapshot)
+    return {
+        "ok": True,
+        "week_id": week_id,
+        "updated": updated,
+        "skipped": skipped,
+        "labeled_by": labeled_by,
+        "score": score.to_dict(),
+    }
+
+
+def label_region(
+    week_id: str,
+    region: str,
+    label: LabelName,
+    *,
+    notes: str = "",
+    labeled_by: str = "operator",
+) -> dict[str, Any]:
+    return label_regions(
+        week_id,
+        [{"region": region, "label": label, "notes": notes}],
+        labeled_by=labeled_by,
+    )
+
+
+def rolling_trend(*, weeks: int = 8) -> list[WeekScore]:
+    """Return scored weeks newest-first (only weeks with stored snapshots)."""
+    ids = list_week_ids(newest_first=True)[: max(1, weeks)]
+    scores: list[WeekScore] = []
+    for week_id in ids:
+        snapshot = load_week(week_id)
+        if snapshot is None:
+            continue
+        scores.append(score_week(snapshot))
+    return scores
+
+
+def rolling_report(*, weeks: int = 8, target_confidence: float = 0.80) -> dict[str, Any]:
+    """Aggregate operational validation trend for API / OpenClaw."""
+    threshold = rolling_alert_threshold()
+    trend = rolling_trend(weeks=weeks)
+    scorable = [row for row in trend if row.scorable]
+
+    latest = scorable[0] if scorable else (trend[0] if trend else None)
+    accuracy_series = [
+        {"week_id": row.week_id, "accuracy": round(row.accuracy, 4), "labeled": row.labeled}
+        for row in reversed(scorable)
+    ]
+
+    improving = False
+    if len(scorable) >= 2:
+        improving = scorable[0].accuracy >= scorable[1].accuracy
+
+    return {
+        "mode": "rolling_operational",
+        "alert_threshold": threshold,
+        "target_confidence": target_confidence,
+        "weeks_requested": weeks,
+        "weeks_stored": len(trend),
+        "weeks_scorable": len(scorable),
+        "min_labeled_per_week": MIN_LABELED_FOR_TREND,
+        "latest": latest.to_dict() if latest else None,
+        "trend": [row.to_dict() for row in trend],
+        "accuracy_series": accuracy_series,
+        "improving_vs_prior": improving,
+        "meets_target": bool(
+            latest and latest.scorable and latest.confidence_rate >= target_confidence
+        ),
+        "note": (
+            "Operational metric: scores frozen weekly predictions against delayed "
+            "labels. Unlike the static benchmark, this measures live forward utility."
+        ),
+    }

backend/analytics/settings.py

@@ -0,0 +1,158 @@
+"""Configuration for Strategic Risk Analytics (feature-flagged)."""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass, field
+from functools import lru_cache
+from typing import Any
+
+
+def _env_bool(name: str, default: bool = False) -> bool:
+    raw = str(os.environ.get(name, "")).strip().lower()
+    if not raw:
+        return default
+    return raw not in {"0", "false", "no", "off"}
+
+
+def _env_float(name: str, default: float) -> float:
+    raw = str(os.environ.get(name, "")).strip()
+    if not raw:
+        return default
+    try:
+        return float(raw)
+    except ValueError:
+        return default
+
+
+def _env_int(name: str, default: int) -> int:
+    raw = str(os.environ.get(name, "")).strip()
+    if not raw:
+        return default
+    try:
+        return int(raw)
+    except ValueError:
+        return default
+
+
+def _parse_signal_weights(raw: str) -> dict[str, float]:
+    if not raw.strip():
+        return {}
+    try:
+        parsed = json.loads(raw)
+        if isinstance(parsed, dict):
+            return {str(k): float(v) for k, v in parsed.items()}
+    except (json.JSONDecodeError, TypeError, ValueError):
+        pass
+    weights: dict[str, float] = {}
+    for part in raw.split(","):
+        piece = part.strip()
+        if not piece or "=" not in piece:
+            continue
+        key, value = piece.split("=", 1)
+        try:
+            weights[key.strip()] = float(value.strip())
+        except ValueError:
+            continue
+    return weights
+
+
+def resolve_gt_profile() -> str:
+    from services.runtime_profile import resolve_profile_name
+
+    return resolve_profile_name()
+
+
+def gt_analytics_ack_low_cpu() -> bool:
+    return _env_bool("GT_ANALYTICS_ACK_LOW_CPU", default=False)
+
+
+def gt_engine_operational() -> bool:
+    """Full GT engine (scheduled ingest, heatmap, Louvain) — not watchdog-only."""
+    if not get_gt_settings().enabled:
+        return False
+    if resolve_gt_profile() == "lean" and not gt_analytics_ack_low_cpu():
+        return False
+    return True
+
+
+def gt_scheduled_ingest_enabled() -> bool:
+    return gt_engine_operational()
+
+
+def gt_louvain_enabled() -> bool:
+    return gt_engine_operational()
+
+
+@dataclass(frozen=True)
+class GTAnalyticsSettings:
+    enabled: bool = False
+    profile: str = "standard"
+    base_prior: float = 0.15
+    evidence_cap: float = 3.0
+    evidence_scale: float = 5.0
+    min_prob: float = 0.01
+    max_prob: float = 0.99
+    high_risk_threshold: float = 0.6
+    max_history_per_region: int = 200
+    max_heatmap_features: int = 500
+    louvain_min_weight: float = 0.5
+    louvain_interval_minutes: int = 30
+    signal_weight_overrides: dict[str, float] = field(default_factory=dict)
+    watched_channels: tuple[str, ...] = ()
+
+
+@lru_cache(maxsize=1)
+def get_gt_settings() -> GTAnalyticsSettings:
+    channels_raw = str(os.environ.get("GT_ANALYTICS_WATCHED_CHANNELS", "")).strip()
+    channels = tuple(
+        part.strip().lstrip("@")
+        for part in channels_raw.split(",")
+        if part.strip()
+    )
+    profile = resolve_gt_profile()
+    lean = profile == "lean"
+    return GTAnalyticsSettings(
+        enabled=_env_bool("GT_ANALYTICS_ENABLED", default=False),
+        profile=profile,
+        base_prior=_env_float("GT_ANALYTICS_BASE_PRIOR", 0.15),
+        evidence_cap=_env_float("GT_ANALYTICS_EVIDENCE_CAP", 3.0),
+        evidence_scale=_env_float("GT_ANALYTICS_EVIDENCE_SCALE", 5.0),
+        min_prob=_env_float("GT_ANALYTICS_MIN_PROB", 0.01),
+        max_prob=_env_float("GT_ANALYTICS_MAX_PROB", 0.99),
+        high_risk_threshold=_env_float("GT_ANALYTICS_HIGH_RISK_THRESHOLD", 0.6),
+        max_history_per_region=_env_int("GT_ANALYTICS_MAX_HISTORY", 200),
+        max_heatmap_features=_env_int(
+            "GT_ANALYTICS_MAX_HEATMAP_FEATURES",
+            50 if lean else 500,
+        ),
+        louvain_min_weight=_env_float("GT_ANALYTICS_LOUVAIN_MIN_WEIGHT", 0.5),
+        louvain_interval_minutes=max(5, _env_int("GT_ANALYTICS_LOUVAIN_INTERVAL_MINUTES", 30)),
+        signal_weight_overrides=_parse_signal_weights(
+            str(os.environ.get("GT_ANALYTICS_SIGNAL_WEIGHTS", ""))
+        ),
+        watched_channels=channels,
+    )
+
+
+def gt_analytics_enabled() -> bool:
+    return get_gt_settings().enabled
+
+
+def gt_analytics_status() -> dict[str, Any]:
+    settings = get_gt_settings()
+    from services.runtime_profile import get_runtime_profile
+
+    runtime = get_runtime_profile()
+    operational = gt_engine_operational()
+    return {
+        "enabled": settings.enabled,
+        "operational": operational,
+        "profile": settings.profile,
+        "ack_low_cpu": gt_analytics_ack_low_cpu(),
+        "recommended": bool(runtime.get("gt_analytics", {}).get("recommended")),
+        "lean_node": bool(runtime.get("gt_analytics", {}).get("lean_node")),
+        "warning": runtime.get("gt_analytics", {}).get("warning"),
+        "experimental": True,
+    }

backend/analytics/weekly_store.py

@@ -0,0 +1,154 @@
+"""Persistent JSON store for rolling GT operational backtest weeks."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Literal
+
+logger = logging.getLogger(__name__)
+
+LabelName = Literal["pending", "true_escalation", "false_alarm", "benign"]
+VALID_LABELS: frozenset[str] = frozenset(
+    {"pending", "true_escalation", "false_alarm", "benign"}
+)
+
+_STORE_DIR = Path(__file__).parent.parent / "data" / "gt_rolling"
+_store_lock = threading.Lock()
+
+
+def rolling_store_dir() -> Path:
+    """Return the rolling-backtest data directory (override via env in tests)."""
+    override = str(os.environ.get("GT_ROLLING_STORE_DIR", "")).strip()
+    if override:
+        return Path(override)
+    return _STORE_DIR
+
+
+@dataclass
+class RegionSnapshot:
+    region: str
+    composite_risk: float
+    financial: float
+    unrest: float
+    conflict: float
+    alerted: bool
+    label: LabelName = "pending"
+    labeled_at: str | None = None
+    notes: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> RegionSnapshot:
+        label = str(raw.get("label") or "pending")
+        if label not in VALID_LABELS:
+            label = "pending"
+        return cls(
+            region=str(raw.get("region") or "").strip().lower(),
+            composite_risk=float(raw.get("composite_risk") or 0.0),
+            financial=float(raw.get("financial") or 0.0),
+            unrest=float(raw.get("unrest") or 0.0),
+            conflict=float(raw.get("conflict") or 0.0),
+            alerted=bool(raw.get("alerted")),
+            label=label,  # type: ignore[arg-type]
+            labeled_at=raw.get("labeled_at"),
+            notes=str(raw.get("notes") or ""),
+        )
+
+
+@dataclass
+class WeeklySnapshot:
+    week_id: str
+    frozen_at: str
+    alert_threshold: float
+    regions: list[RegionSnapshot] = field(default_factory=list)
+    frozen_by: str = "system"
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "week_id": self.week_id,
+            "frozen_at": self.frozen_at,
+            "alert_threshold": self.alert_threshold,
+            "frozen_by": self.frozen_by,
+            "regions": [row.to_dict() for row in self.regions],
+        }
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any]) -> WeeklySnapshot:
+        regions = [
+            RegionSnapshot.from_dict(row)
+            for row in (raw.get("regions") or [])
+            if isinstance(row, dict)
+        ]
+        return cls(
+            week_id=str(raw.get("week_id") or ""),
+            frozen_at=str(raw.get("frozen_at") or ""),
+            alert_threshold=float(raw.get("alert_threshold") or 0.0),
+            regions=regions,
+            frozen_by=str(raw.get("frozen_by") or "system"),
+        )
+
+
+def _week_path(week_id: str) -> Path:
+    safe = week_id.replace("/", "-").replace("..", "")
+    return rolling_store_dir() / f"{safe}.json"
+
+
+def _ensure_dir() -> None:
+    rolling_store_dir().mkdir(parents=True, exist_ok=True)
+
+
+def list_week_ids(*, newest_first: bool = True) -> list[str]:
+    """Return stored ISO week ids."""
+    _ensure_dir()
+    ids = [
+        path.stem
+        for path in rolling_store_dir().glob("*.json")
+        if path.stem and path.stem != "index"
+    ]
+    ids.sort(reverse=newest_first)
+    return ids
+
+
+def load_week(week_id: str) -> WeeklySnapshot | None:
+    path = _week_path(week_id)
+    if not path.is_file():
+        return None
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+        if not isinstance(raw, dict):
+            return None
+        return WeeklySnapshot.from_dict(raw)
+    except (OSError, json.JSONDecodeError, TypeError, ValueError):
+        logger.exception("Failed to load GT rolling week %s", week_id)
+        return None
+
+
+def save_week(snapshot: WeeklySnapshot) -> None:
+    _ensure_dir()
+    path = _week_path(snapshot.week_id)
+    tmp = path.with_suffix(".json.tmp")
+    payload = json.dumps(snapshot.to_dict(), indent=2, sort_keys=True)
+    with _store_lock:
+        tmp.write_text(payload, encoding="utf-8")
+        tmp.replace(path)
+
+
+def delete_week(week_id: str) -> bool:
+    path = _week_path(week_id)
+    if not path.is_file():
+        return False
+    with _store_lock:
+        path.unlink()
+    return True
+
+
+def utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()

backend/auth.py

@@ -113,8 +113,14 @@ def _scoped_admin_tokens() -> dict[str, list[str]]:
     return normalized
 
 
+def _request_scope_path(request: Request) -> str:
+    """Return the ASGI request-line path, not the Host-derived URL path."""
+    scope = getattr(request, "scope", {}) or {}
+    return str(scope.get("path") or "")
+
+
 def _required_scope_for_request(request: Request) -> str:
-    path = str(request.url.path or "")
+    path = _request_scope_path(request)
     if path.startswith("/api/wormhole/gate/"):
         return "gate"
     if path.startswith("/api/wormhole/dm/"):
@@ -443,7 +449,7 @@ async def _verify_openclaw_hmac(request: Request) -> bool:
 
     # Compute expected signature: HMAC-SHA256(secret, METHOD|path|ts|nonce|body_digest)
     method = str(request.method or "").upper()
-    path = str(request.url.path or "")
+    path = _request_scope_path(request)
     message = f"{method}|{path}|{ts_str}|{nonce}|{body_digest}"
     expected = hmac.new(
         secret.encode("utf-8"),
@@ -515,33 +521,32 @@ _KNOWN_COMPROMISED_PEER_PUSH_SECRET_SHA256 = (
 def _validate_admin_startup() -> None:
     admin_key = _current_admin_key()
 
-    if not admin_key or len(admin_key) < 32:
-        import secrets
+    if not admin_key:
+        logger.warning(
+            "ADMIN_KEY is not set. Local-operator/admin endpoints will reject "
+            "remote callers until ADMIN_KEY is configured."
+        )
+        return
 
-        reason = "not set" if not admin_key else f"too short ({len(admin_key)} chars, minimum 32)"
-        new_key = secrets.token_hex(32)  # 64-char hex string
+    if len(admin_key) < 32:
+        reason = f"too short ({len(admin_key)} chars, minimum 32)"
         try:
-            from routers.ai_intel import _write_env_value
-
-            _write_env_value("ADMIN_KEY", new_key)
-            os.environ["ADMIN_KEY"] = new_key
-            logger.info(
-                "ADMIN_KEY was %s — auto-generated a strong 64-character key and "
-                "saved it to .env. Admin/mesh endpoints are now secured.",
-                reason,
-            )
-            # Clear settings cache so the rest of startup picks up the new key
-            try:
-                get_settings.cache_clear()
-            except Exception:
-                pass
-        except Exception as exc:
+            debug_mode = bool(getattr(get_settings(), "MESH_DEBUG_MODE", False))
+        except Exception:
+            debug_mode = False
+        if debug_mode:
             logger.warning(
-                "ADMIN_KEY is %s and could not auto-generate: %s. "
-                "Admin/mesh endpoints may be unavailable.",
+                "ADMIN_KEY is %s. Debug mode is enabled, so startup will continue, "
+                "but production deployments must use a 32+ character key.",
                 reason,
-                exc,
             )
+            return
+        logger.error(
+            "ADMIN_KEY is %s. Refusing to start because auto-generating a backend-only "
+            "replacement would desynchronize the frontend and backend containers.",
+            reason,
+        )
+        raise SystemExit(1)
 
 
 def _validate_insecure_admin_startup() -> None:
@@ -744,8 +749,7 @@ def _is_debug_test_request(request: Request) -> bool:
     if not _debug_mode_enabled():
         return False
     client_host = (request.client.host or "").lower() if request.client else ""
-    url_host = (request.url.hostname or "").lower() if request.url else ""
-    return client_host == "test" or url_host == "test"
+    return client_host == "test"
 
 
 # ---------------------------------------------------------------------------
@@ -858,7 +862,9 @@ _ROUTE_TRANSPORT_POLICY: dict[tuple[str, str], RouteTransportPolicy] = {
     ("POST", "/api/wormhole/gate/messages/decrypt"): _local_only_route_policy("private_control_only"),
     # ── Wormhole DM (strong) ──────────────────────────────────────────
     ("POST", "/api/wormhole/dm/compose"): _local_only_route_policy("private_control_only"),
+    ("POST", "/api/wormhole/dm/connect-contact"): _local_only_route_policy("private_control_only"),
     ("POST", "/api/wormhole/dm/decrypt"): _local_only_route_policy("private_control_only"),
+    ("POST", "/api/wormhole/dm/mls-key-package"): _local_only_route_policy("private_control_only"),
     ("POST", "/api/wormhole/dm/register-key"): _local_only_route_policy("private_control_only"),
     ("POST", "/api/wormhole/dm/prekey/register"): _local_only_route_policy("private_control_only"),
     ("POST", "/api/wormhole/dm/bootstrap-encrypt"): _local_only_route_policy("private_control_only"),
@@ -1397,10 +1403,28 @@ def _peer_hmac_url_from_request(request: Request) -> str:
     header_url = normalize_peer_url(str(request.headers.get("x-peer-url", "") or ""))
     if header_url:
         return header_url
-    if not request.url:
-        return ""
-    base_url = f"{request.url.scheme}://{request.url.netloc}".rstrip("/")
-    return normalize_peer_url(base_url)
+    return ""
+
+
+def _verify_peer_transport_hmac(request: Request, body_bytes: bytes) -> bool:
+    """Verify HMAC-SHA256 peer authentication without an allowlist check."""
+    provided = str(request.headers.get("x-peer-hmac", "") or "").strip()
+    if not provided:
+        return False
+
+    peer_url = _peer_hmac_url_from_request(request)
+    if not peer_url:
+        return False
+    peer_key = resolve_peer_key_for_url(peer_url)
+    if not peer_key:
+        return False
+
+    expected = _hmac_mod.new(
+        peer_key,
+        body_bytes,
+        _hashlib_mod.sha256,
+    ).hexdigest()
+    return _hmac_mod.compare_digest(provided.lower(), expected.lower())
 
 
 def _verify_peer_push_hmac(request: Request, body_bytes: bytes) -> bool:

backend/config/news_feeds.json

@@ -7,7 +7,7 @@
     },
     {
       "name": "BBC",
-      "url": "http://feeds.bbci.co.uk/news/world/rss.xml",
+      "url": "https://feeds.bbci.co.uk/news/world/rss.xml",
       "weight": 3
     },
     {
@@ -47,7 +47,7 @@
     },
     {
       "name": "Xinhua",
-      "url": "http://www.news.cn/english/rss/worldrss.xml",
+      "url": "https://www.news.cn/english/rss/worldrss.xml",
       "weight": 2
     },
     {

backend/data/drishx/rf_model.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72b69418aa860a0d92ccae398a08722bc85e64a992b5515dd7bf9ae9f79f2fd1
+size 107194128

backend/data/release_digests.json

@@ -43,8 +43,18 @@
     "ShadowBroker_0.9.8_x64_en-US.msi": "fe22f9d51e4360d74c18a7250c2fbb9ed4fa4c7a884b3ac0d04a21115466386b"
   },
   "v0.9.81": {
-    "ShadowBroker_v0.9.81.zip": "42f8a51f9a5690d1e7349d90d8ecf2d163c9061d6cf90c69ee03647a785437ff",
-    "ShadowBroker_0.9.81_x64-setup.exe": "eca884b9d37eeccd0f11c91dcc6f6ae1b3609d9dee72bd73c37c9a427babfef2",
-    "ShadowBroker_0.9.81_x64_en-US.msi": "a45b177c26c95d2b28d71592d7147e88ff4e104865f214fde11249d311ec9e25"
+    "ShadowBroker_v0.9.81.zip": "f81f454bdc88e9a32c351df38212b8cfa624704d65764b971bb091eef62259c6",
+    "ShadowBroker_0.9.81_x64-setup.exe": "25e9a95d0d8ce959a7d08fe8e7406772ae24b596652793e81d1de5d02510a5a6",
+    "ShadowBroker_0.9.81_x64_en-US.msi": "34e655fc0c0f195ee4ac978f228a4b2b9d5565253b8771aca9ef4693409e9e70"
+  },
+  "v0.9.82": {
+    "ShadowBroker_v0.9.82.zip": "202ab043465741dcc06de57c19ec8314904332f8e818b891d7174655719d084c",
+    "ShadowBroker_0.9.82_x64-setup.exe": "0eb9f2bda02ab691b39687641abc97e6bfb507b42f48de21970ad7dfb4ea15fc",
+    "ShadowBroker_0.9.82_x64_en-US.msi": "ced08f930171c0c08009a958cc30b0171a09f982230fc217c6808c2ed7ab2e30"
+  },
+  "v0.9.83": {
+    "ShadowBroker_v0.9.83.zip": "53f56631731ad3cdc7be68df09bedd6570ed91ecda6fa57c39651098e15666c7",
+    "ShadowBroker_0.9.83_x64-setup.exe": "d62170af4b9df0b190832b7bb3ad6bfe8a7ac01472f2c7b39cf2a1b61edc7492",
+    "ShadowBroker_0.9.83_x64_en-US.msi": "b664cc0003a29f7ce88b04c2b425643dbe7ed897342fc6e9a2378bc1910c6850"
   }
 }

backend/pyproject.toml

@@ -7,15 +7,15 @@ py-modules = []
 
 [project]
 name = "backend"
-version = "0.9.81"
+version = "0.9.83"
 requires-python = ">=3.10"
 dependencies = [
     "apscheduler==3.10.3",
     "beautifulsoup4>=4.9.0",
     "cachetools==5.5.2",
-    "cryptography>=41.0.0",
+    "cryptography>=46.0.7",
     "defusedxml>=0.7.1",
-    "fastapi==0.115.12",
+    "fastapi==0.136.3",
     "feedparser==6.0.10",
     "httpx==0.28.1",
     "playwright==1.59.0",
@@ -24,26 +24,40 @@ dependencies = [
     "pydantic-settings==2.8.1",
     "pystac-client==0.8.6",
     "python-dotenv==1.2.2",
-    "requests==2.31.0",
+    "requests==2.33.0",
     "PySocks==1.7.1",
     "reverse-geocoder==1.5.1",
     "sgp4==2.25",
     "meshtastic>=2.5.0",
+    "networkx>=3.4.0",
+    "numpy>=2.2.0",
     "orjson>=3.10.0",
     "paho-mqtt>=1.6.0,<2.0.0",
     "PyNaCl>=1.5.0",
     "slowapi==0.1.9",
+    "starlette==1.0.1",
     "vaderSentiment>=3.3.0",
     "uvicorn==0.34.0",
     "yfinance==1.3.0",
 ]
 
+[project.optional-dependencies]
+road-corridor = [
+    "geopandas>=1.0.0",
+    "imageio>=2.34.0",
+    "osmnx>=2.0.0",
+    "rasterio>=1.4.0",
+    "scikit-learn>=1.5.0",
+    "sentinelhub>=3.10.0",
+    "shapely>=2.0.0",
+]
+
 [dependency-groups]
-dev = ["pytest>=8.3.4", "pytest-asyncio==0.25.0", "ruff>=0.9.0", "black>=24.0.0"]
+dev = ["pytest>=9.0.3", "pytest-asyncio>=1.4.0", "ruff>=0.9.0", "black>=24.0.0"]
 
 [tool.ruff.lint]
 # The current backend carries historical style debt in large legacy modules.
-# Keep CI focused on actionable correctness checks for the v0.9.81 release.
+# Keep CI focused on actionable correctness checks for the v0.9.82 release.
 ignore = ["E401", "E402", "E701", "E731", "E741", "F401", "F402", "F541", "F811", "F841"]
 
 [tool.black]

backend/routers/agent_shell.py

@@ -0,0 +1,230 @@
+"""Local-operator PTY WebSocket for the Mesh Chat agent shell."""
+
+from __future__ import annotations
+
+import asyncio
+import fcntl
+import hmac
+import json
+import logging
+import os
+import pty
+import select
+import signal
+import struct
+import sys
+import termios
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException, Query, WebSocket, WebSocketDisconnect
+from pydantic import BaseModel, Field
+
+from auth import (
+    _current_admin_key,
+    _debug_mode_enabled,
+    _is_trusted_local_runtime_host,
+    require_local_operator,
+)
+from services.agent_shell_settings import (
+    get_agent_shell_settings,
+    set_agent_shell_working_directory,
+)
+
+logger = logging.getLogger(__name__)
+router = APIRouter(tags=["agent-shell"])
+
+
+class AgentShellSettingsUpdate(BaseModel):
+    working_directory: str = Field(min_length=1)
+
+
+def _set_winsize(fd: int, rows: int, cols: int) -> None:
+    winsize = struct.pack("HHHH", rows, cols, 0, 0)
+    fcntl.ioctl(fd, termios.TIOCSWINSZ, winsize)
+
+
+def _published_local_dashboard_ws(ws: WebSocket) -> bool:
+    """Browser → published Docker port appears as a bridge IP, not loopback.
+
+    For the operator shell only, also accept when the upgrade request clearly
+    targets the local dashboard (Host/Origin on localhost).
+    """
+    host_header = str(ws.headers.get("host") or "").strip().lower()
+    host_name = host_header.split(":", 1)[0]
+    if host_name in {"127.0.0.1", "localhost", "::1"}:
+        return True
+
+    origin = str(ws.headers.get("origin") or "").strip().lower()
+    if origin.startswith("http://127.0.0.1:") or origin.startswith("http://localhost:"):
+        return True
+    if origin.startswith("https://127.0.0.1:") or origin.startswith("https://localhost:"):
+        return True
+    return False
+
+
+async def _authorize_agent_shell_ws(ws: WebSocket, admin_key_query: str = "") -> None:
+    host = (ws.client.host or "").lower() if ws.client else ""
+    if (
+        _is_trusted_local_runtime_host(host)
+        or _published_local_dashboard_ws(ws)
+        or (_debug_mode_enabled() and host == "test")
+    ):
+        return
+    admin_key = _current_admin_key()
+    presented = str(admin_key_query or ws.headers.get("x-admin-key", "") or "").strip()
+    if admin_key and presented and hmac.compare_digest(presented.encode(), admin_key.encode()):
+        return
+    await ws.close(code=4403, reason="local operator access only")
+    raise WebSocketDisconnect()
+
+
+def _resolve_shell_cwd(requested: str) -> str:
+    requested = str(requested or "").strip()
+    if requested:
+        resolved = os.path.abspath(os.path.expanduser(requested))
+        if os.path.isdir(resolved):
+            return resolved
+    return get_agent_shell_settings()["working_directory"]
+
+
+def _default_shell() -> str:
+    if sys.platform == "win32":
+        return os.environ.get("COMSPEC", "cmd.exe")
+    return os.environ.get("SHELL", "/bin/bash")
+
+
+async def _relay_pty(master_fd: int, proc: asyncio.subprocess.Process, ws: WebSocket) -> None:
+    loop = asyncio.get_running_loop()
+    while True:
+        if proc.returncode is not None:
+            break
+        try:
+            readable, _, _ = await loop.run_in_executor(
+                None, lambda: select.select([master_fd], [], [], 0.05)
+            )
+        except Exception:
+            break
+        if master_fd in readable:
+            try:
+                chunk = os.read(master_fd, 4096)
+            except OSError:
+                break
+            if not chunk:
+                break
+            await ws.send_bytes(chunk)
+        try:
+            message = await asyncio.wait_for(ws.receive(), timeout=0.05)
+        except asyncio.TimeoutError:
+            continue
+        if message.get("type") == "websocket.disconnect":
+            break
+        if message.get("type") != "websocket.receive":
+            continue
+        if message.get("bytes"):
+            os.write(master_fd, message["bytes"])
+            continue
+        text = message.get("text")
+        if not text:
+            continue
+        try:
+            payload = json.loads(text)
+        except json.JSONDecodeError:
+            os.write(master_fd, text.encode("utf-8", errors="replace"))
+            continue
+        if payload.get("type") == "resize":
+            rows = int(payload.get("rows") or 24)
+            cols = int(payload.get("cols") or 80)
+            _set_winsize(master_fd, max(rows, 2), max(cols, 2))
+
+
+@router.get("/api/agent-shell/settings", dependencies=[Depends(require_local_operator)])
+async def read_agent_shell_settings() -> dict[str, Any]:
+    return get_agent_shell_settings()
+
+
+@router.put("/api/agent-shell/settings", dependencies=[Depends(require_local_operator)])
+async def write_agent_shell_settings(body: AgentShellSettingsUpdate) -> dict[str, Any]:
+    try:
+        return set_agent_shell_working_directory(body.working_directory)
+    except ValueError as exc:
+        detail = str(exc)
+        if detail == "working_directory_not_found":
+            raise HTTPException(status_code=400, detail="Working directory does not exist") from exc
+        raise HTTPException(status_code=400, detail="Working directory is required") from exc
+
+
+@router.websocket("/api/agent-shell/ws")
+async def agent_shell_websocket(
+    ws: WebSocket,
+    cwd: str = Query(default=""),
+    cols: int = Query(default=80),
+    rows: int = Query(default=24),
+    admin_key: str = Query(default=""),
+) -> None:
+    await ws.accept()
+    try:
+        await _authorize_agent_shell_ws(ws, admin_key)
+    except WebSocketDisconnect:
+        return
+
+    if sys.platform == "win32":
+        await ws.send_text(
+            json.dumps(
+                {
+                    "type": "error",
+                    "message": "Host PTY is not available on Windows backend builds yet. Use the ShadowBroker desktop app or run the backend in Docker/Linux for an embedded shell.",
+                }
+            )
+        )
+        await ws.close(code=1011)
+        return
+
+    shell_cwd = _resolve_shell_cwd(cwd)
+    shell = _default_shell()
+    master_fd, slave_fd = pty.openpty()
+    _set_winsize(master_fd, max(rows, 2), max(cols, 2))
+
+    env = os.environ.copy()
+    env.setdefault("TERM", "xterm-256color")
+    env.setdefault("COLORTERM", "truecolor")
+    home = shell_cwd if os.path.isdir(shell_cwd) else "/app"
+    env["HOME"] = home
+    env["USER"] = env.get("USER") or "operator"
+    path_prefixes = [
+        os.path.join(home, ".local", "bin"),
+        os.path.join(home, ".hermes", "bin"),
+    ]
+    path = env.get("PATH", "")
+    for prefix in path_prefixes:
+        if os.path.isdir(prefix):
+            path = f"{prefix}:{path}" if path else prefix
+    env["PATH"] = path
+
+    proc = await asyncio.create_subprocess_exec(
+        shell,
+        stdin=slave_fd,
+        stdout=slave_fd,
+        stderr=slave_fd,
+        cwd=shell_cwd,
+        env=env,
+        preexec_fn=os.setsid,
+    )
+    os.close(slave_fd)
+
+    try:
+        await _relay_pty(master_fd, proc, ws)
+    finally:
+        try:
+            os.close(master_fd)
+        except OSError:
+            pass
+        if proc.returncode is None:
+            try:
+                os.killpg(proc.pid, signal.SIGHUP)
+            except ProcessLookupError:
+                pass
+            try:
+                await asyncio.wait_for(proc.wait(), timeout=2.0)
+            except asyncio.TimeoutError:
+                proc.kill()
+                await proc.wait()

backend/routers/ai_intel.py

@@ -1590,7 +1590,7 @@ async def agent_tool_manifest(request: Request):
 
     return {
         "ok": True,
-        "version": "0.9.81",
+        "version": "0.9.82",
         "access_tier": access_tier,
         "available_commands": available_commands,
         "transport": {
@@ -1705,11 +1705,12 @@ async def agent_tool_manifest(request: Request):
             {
                 "name": "search_news",
                 "type": "read",
-                "description": "Search news and event layers server-side by keyword. Includes news, GDELT, CrowdThreat, and major incident/event feeds without pulling the full slow telemetry feed.",
+                "description": "Search news and event layers server-side by keyword. Includes news, GDELT, CrowdThreat, Telegram OSINT, and major incident/event feeds without pulling the full slow telemetry feed.",
                 "parameters": {
                     "query": {"type": "string", "required": True, "description": "Keyword or phrase to search for"},
                     "limit": {"type": "integer", "required": False, "description": "Max results (default 10, max 50)"},
                     "include_gdelt": {"type": "boolean", "required": False, "description": "Include GDELT matches (default true)"},
+                    "include_telegram": {"type": "boolean", "required": False, "description": "Include Telegram OSINT channel posts (default true)"},
                     "compact": {"type": "boolean", "required": False, "description": "If true, strips empty/None fields from each result and rounds lat/lng to 3 decimals. Response includes format: 'compressed_v1'."},
                 },
                 "returns": "{results: [{source_layer, title, summary, source, link, lat, lng, risk_score}], version: int, truncated: bool}",
@@ -1743,6 +1744,55 @@ async def agent_tool_manifest(request: Request):
                 },
                 "returns": "{center, radius_km, nearby, topic_news, context_layers}",
             },
+            {
+                "name": "osint_lookup",
+                "type": "read",
+                "description": "Run a passive OSINT recon lookup server-side (same backends as the Recon panel). SSRF-guarded outbound proxies for IP geolocation, DNS, WHOIS, certs, BGP/ASN, sanctions, CVE, MAC vendor, GitHub profile, breach checks, and threat feeds.",
+                "parameters": {
+                    "tool": {"type": "string", "required": True, "description": "Lookup type: ip, dns, whois, certs, threats, bgp, sanctions, cve, mac, github, leaks, sweep_init"},
+                    "ip": {"type": "string", "required": False, "description": "IPv4/IPv6 for ip or sweep_init"},
+                    "domain": {"type": "string", "required": False, "description": "Domain for dns, whois, certs"},
+                    "query": {"type": "string", "required": False, "description": "Generic query (BGP ASN, sanctions name, optional threats filter)"},
+                    "cve": {"type": "string", "required": False, "description": "CVE id for cve lookup"},
+                    "mac": {"type": "string", "required": False, "description": "MAC address for mac lookup"},
+                    "username": {"type": "string", "required": False, "description": "GitHub username"},
+                    "email": {"type": "string", "required": False, "description": "Email for breach/leak lookup"},
+                    "schema": {"type": "string", "required": False, "description": "Sanctions schema filter: Person, Organization, Company, Vessel, Airplane, LegalEntity"},
+                    "limit": {"type": "integer", "required": False, "description": "Sanctions result cap (default 25, max 100)"},
+                    "cidr": {"type": "integer", "required": False, "description": "CIDR mask for sweep_init (24-32, default 24)"},
+                },
+                "returns": "Tool-specific JSON (geo, DNS records, WHOIS, sanctions hits, CVE details, etc.)",
+            },
+            {
+                "name": "osint_tools",
+                "type": "read",
+                "description": "List available OSINT recon tools, entity-expand types, and sanctions schemas.",
+                "parameters": {},
+                "returns": "{tools: [...], entity_types: [...], sanctions_schemas: [...], notes: {...}}",
+            },
+            {
+                "name": "entity_expand",
+                "type": "read",
+                "description": "Expand an entity relationship graph around an aircraft, vessel, IP, company, person, or country. Same backend as /api/entity/expand.",
+                "parameters": {
+                    "type": {"type": "string", "required": True, "description": "Entity type: aircraft, vessel, company, person, ip, country"},
+                    "id": {"type": "string", "required": True, "description": "Entity identifier (tail number, MMSI, IP, company name, etc.)"},
+                    "registration": {"type": "string", "required": False, "description": "Aircraft registration hint"},
+                    "model": {"type": "string", "required": False, "description": "Aircraft model hint"},
+                    "icao24": {"type": "string", "required": False, "description": "ICAO24 hex for aircraft"},
+                },
+                "returns": "{nodes: [...], links: [...]}",
+            },
+            {
+                "name": "osint_sweep",
+                "type": "write",
+                "description": "Active subnet device discovery via Shodan InternetDB (ports, vulns, hostnames). Requires full OpenClaw access tier. Private/reserved IPs blocked.",
+                "parameters": {
+                    "ip": {"type": "string", "required": True, "description": "Public IPv4 anchor for the sweep"},
+                    "cidr": {"type": "integer", "required": False, "description": "Subnet size /24-/32 (default 24)"},
+                },
+                "returns": "{center, target_ip, cidr, subnet, devices, summary, sweep_time_ms}",
+            },
             {
                 "name": "what_changed",
                 "type": "read",
@@ -2001,7 +2051,7 @@ async def agent_tool_manifest(request: Request):
                 "description": "Set up a watchdog alert. When triggered, alerts push instantly via SSE stream. Debounced: same watch won't re-fire within 60 seconds.",
                 "parameters": {
                     "type": {"type": "string", "required": True, "description": "Watch type",
-                             "enum": ["track_aircraft", "track_callsign", "track_registration", "track_ship", "track_entity", "geofence", "keyword", "prediction_market"]},
+                             "enum": ["track_aircraft", "track_callsign", "track_registration", "track_ship", "track_entity", "geofence", "keyword", "telegram_rhetoric", "prediction_market"]},
                     "params": {"type": "object", "required": True, "description": "Type-specific parameters (see subtypes)"},
                 },
                 "subtypes": {
@@ -2011,7 +2061,8 @@ async def agent_tool_manifest(request: Request):
                     "track_ship": {"params": {"mmsi": "string (optional)", "imo": "string (optional)", "name": "string (optional)", "owner": "string (optional)", "callsign": "string (optional)"}, "description": "Alert when ship appears by MMSI, IMO, name, owner, or callsign"},
                     "track_entity": {"params": {"query": "string", "entity_type": "string (optional)", "layers": "list (optional)"}, "description": "Generic exact-first entity tracker when aircraft/ship fields are not known yet"},
                     "geofence": {"params": {"lat": "float", "lng": "float", "radius_km": "float (default 50)", "entity_types": "list (default ['flights','ships'])"}, "description": "Alert when any entity enters a geographic zone"},
-                    "keyword": {"params": {"keyword": "string"}, "description": "Alert when keyword appears in news/GDELT headlines"},
+                    "keyword": {"params": {"keyword": "string", "include_telegram": "boolean (default true)"}, "description": "Alert when keyword appears in news, GDELT, or Telegram OSINT (searches translated + original text)"},
+                    "telegram_rhetoric": {"params": {"min_risk_score": "int 1-10 (default 7)", "keywords": "list or comma-separated string (optional)", "channels": "list or comma-separated string (optional)"}, "description": "Alert on new high-risk Telegram OSINT posts — rhetoric/escalation monitor"},
                     "prediction_market": {"params": {"query": "string", "threshold": "float 0-1 (optional)"}, "description": "Alert on prediction market movements matching query"},
                 },
                 "example": {"cmd": "add_watch", "args": {"type": "track_registration", "params": {"registration": "N3880"}}},
@@ -2194,6 +2245,11 @@ async def agent_tool_manifest(request: Request):
             "Prefer compact lookups first: search_telemetry, find_flights, find_ships, search_news, entities_near, get_layer_slice. Use get_telemetry/get_slow_telemetry/get_report only when focused commands are insufficient.",
             "ShadowBroker does expose UAP sightings, wastewater, and tracked_flights/VIP aircraft when those layers are populated. Verify with get_summary or get_layer_slice before claiming a layer is absent.",
             "ShadowBroker also exposes fishing_activity, which is the fishing-vessel activity layer backed by Global Fishing Watch data when GFW_API_TOKEN is configured. Do not confuse it with the AIS ships layer.",
+            "telegram_osint, malware_threats, cyber_threats, and scm_suppliers are live map layers. Use get_summary or get_layer_slice(['telegram_osint']) before claiming they are absent. Aliases: telegram, malware/botnet, cyber/cisa/kev, scm/suppliers.",
+            "search_telemetry and search_news both index Telegram OSINT posts. For malware C2, botnet IPs, CISA KEV CVEs, or semiconductor suppliers, use search_telemetry or get_layer_slice on the matching layer.",
+            "The Recon toolkit is available via osint_lookup: IP geolocation, DNS, WHOIS, certs, BGP, sanctions, CVE, MAC vendor, GitHub, breach checks, threat feeds. Call osint_tools first to list supported tools.",
+            "entity_expand builds relationship graphs for aircraft, vessels, IPs, companies, people, and countries — use after resolving an entity from telemetry or osint_lookup.",
+            "osint_sweep runs active subnet discovery (Shodan InternetDB) and requires full OpenClaw access tier. Use osint_lookup tool=sweep_init for passive geolocation context only.",
             "Use search_telemetry as the Google-style entry point whenever the user gives you a person, place, company, topic, owner, nickname, or natural-language phrase and you do not already know the source layer.",
             "Example: for 'Where is Jerry Jones yacht?' search 'Jerry Jones' across all telemetry first, identify the ship match, then refine with find_ships or raw layer context only if needed.",
             "For fuzzy natural-language lookups like 'Patriots jet' or 'Jerry Jones yacht', use search_telemetry first and inspect the ranked candidate list before making a hard claim.",
@@ -2221,12 +2277,14 @@ async def agent_tool_manifest(request: Request):
 async def api_capabilities(request: Request):
     """Return full API manifest so the agent knows every available endpoint."""
     from services.openclaw_channel import READ_COMMANDS, WRITE_COMMANDS, detect_tier
+    from services.openclaw_routing import routing_manifest
     from services.config import get_settings
     tier = detect_tier()
     access_tier = str(get_settings().OPENCLAW_ACCESS_TIER or "restricted").strip().lower()
     return {
         "ok": True,
-        "version": "0.9.81",
+        "version": "0.9.82",
+        "routing": routing_manifest(),
         "auth": {
             "method": "HMAC-SHA256",
             "headers": ["X-SB-Timestamp", "X-SB-Nonce", "X-SB-Signature"],
@@ -2342,8 +2400,16 @@ async def api_capabilities(request: Request):
                     "description": "Compact server-side ship search by MMSI/IMO/name/query, including yacht-owner enrichment.",
                 },
                 "find_entity": {
-                    "args": {"query": "str (optional)", "entity_type": "aircraft|ship|person|event|infrastructure (optional)", "callsign": "str (optional)", "registration": "str (optional)", "icao24": "str (optional)", "mmsi": "str (optional)", "imo": "str (optional)", "name": "str (optional)", "owner": "str (optional)", "layers": "list[str] (optional)", "limit": "int (default 10)"},
-                    "description": "Exact-first resolver for planes, ships, operators, callsigns, registrations, MMSI/IMO, and named entities. Use before tracking to avoid fuzzy prompt matching.",
+                    "args": {"query": "str (optional)", "entity_type": "aircraft|ship|person|event|infrastructure (optional)", "callsign": "str (optional)", "registration": "str (optional)", "icao24": "str (optional)", "mmsi": "str (optional)", "imo": "str (optional)", "name": "str (optional)", "owner": "str (optional)", "layers": "list[str] (optional)", "limit": "int (default 10)", "fallback_search": "bool (default false)", "confirm_fuzzy": "bool (alias for fallback_search)"},
+                    "description": "Exact-first resolver for planes, ships, operators, callsigns, registrations, MMSI/IMO, and named entities. Skips fuzzy search unless fallback_search=true or no exact match.",
+                },
+                "route_query": {
+                    "args": {"text": "str", "lat": "float (optional)", "lng": "float (optional)", "radius_km": "float (default 50)", "compact": "bool (default true)"},
+                    "description": "Deterministic intent router — returns recommended fast command, alternates, and latency estimate. Preferred entry for natural-language reads.",
+                },
+                "run_playbook": {
+                    "args": {"name": "str", "query": "str (optional)", "lat": "float (optional)", "lng": "float (optional)"},
+                    "description": "Execute a named batch plan (hot_snapshot, morning_brief, monitor_heartbeat, track_snapshot, area_brief, entity_recon).",
                 },
                 "correlate_entity": {
                     "args": {"query": "str (optional)", "entity_type": "str (optional)", "callsign": "str (optional)", "registration": "str (optional)", "icao24": "str (optional)", "mmsi": "str (optional)", "imo": "str (optional)", "name": "str (optional)", "owner": "str (optional)", "radius_km": "float (default 100)", "limit": "int (default 10)"},
@@ -2354,13 +2420,29 @@ async def api_capabilities(request: Request):
                     "description": "Universal compact search across telemetry when the entity type or source layer is not obvious.",
                 },
                 "search_news": {
-                    "args": {"query": "str", "limit": "int (default 10)", "include_gdelt": "bool (default true)"},
-                    "description": "Search news and event layers by keyword without pulling the whole slow feed.",
+                    "args": {"query": "str", "limit": "int (default 10)", "include_gdelt": "bool (default true)", "include_telegram": "bool (default true)"},
+                    "description": "Search news and event layers by keyword without pulling the whole slow feed. Includes Telegram OSINT when include_telegram is true.",
                 },
                 "entities_near": {
                     "args": {"lat": "float", "lng": "float", "radius_km": "float (default 50)", "entity_types": "list[str] (optional)", "limit": "int (default 25)"},
                     "description": "Compact proximity search around a point across selected layers.",
                 },
+                "osint_lookup": {
+                    "args": {"tool": "str (ip|dns|whois|certs|threats|bgp|sanctions|cve|mac|github|leaks|sweep_init)", "...": "tool-specific params"},
+                    "description": "Passive OSINT recon lookup — same backends as the Recon panel.",
+                },
+                "osint_tools": {
+                    "args": {},
+                    "description": "List available recon tools and entity-expand types.",
+                },
+                "entity_expand": {
+                    "args": {"type": "str", "id": "str", "registration": "str (optional)", "icao24": "str (optional)"},
+                    "description": "Entity relationship graph expansion.",
+                },
+                "osint_sweep": {
+                    "args": {"ip": "str", "cidr": "int (default 24)"},
+                    "description": "Active subnet scan — requires full access tier.",
+                },
                 "brief_area": {
                     "args": {"lat": "float", "lng": "float", "radius_km": "float (default 50)", "entity_types": "list[str] (optional)", "query": "str (optional)", "limit": "int (default 25)", "context_limit": "int (default 10)"},
                     "description": "One compact area brief: nearby aircraft/ships/entities, optional topic news, and selected context layers.",
@@ -2483,7 +2565,8 @@ async def api_capabilities(request: Request):
                         "track_ship": {"params": {"mmsi": "str (optional)", "imo": "str (optional)", "name": "str (optional)", "owner": "str (optional)", "callsign": "str (optional)"}, "description": "Alert when ship appears by MMSI, IMO, name, owner, or callsign"},
                         "track_entity": {"params": {"query": "str", "entity_type": "str (optional)", "layers": "list[str] (optional)"}, "description": "Generic exact-first entity watch"},
                         "geofence": {"params": {"lat": "float", "lng": "float", "radius_km": "float (default 50)", "entity_types": "list (default ['flights','ships'])"}, "description": "Alert when any entity enters a geographic zone"},
-                        "keyword": {"params": {"keyword": "str"}, "description": "Alert when keyword appears in news/GDELT"},
+                        "keyword": {"params": {"keyword": "str", "include_telegram": "bool (default true)"}, "description": "Alert when keyword appears in news, GDELT, or Telegram OSINT"},
+                        "telegram_rhetoric": {"params": {"min_risk_score": "int 1-10 (default 7)", "keywords": "list[str] or comma string (optional)", "channels": "list[str] or comma string (optional)"}, "description": "Alert on new high-risk Telegram OSINT posts"},
                         "prediction_market": {"params": {"query": "str", "threshold": "float 0-1 (optional)"}, "description": "Alert on prediction market movements"},
                     },
                 },
@@ -2507,7 +2590,8 @@ async def api_capabilities(request: Request):
                           "layers are serialized, unchanged layers transfer zero bytes. The client tracks versions "
                           "automatically from SSE events and previous responses. "
                           "3) Pass compact=true on every read command for compressed_v1 responses (~60-90% smaller). "
-                          "4) Use targeted commands first (find_flights, search_telemetry, entities_near). "
+                          "4) Use route_query / find_entity / run_playbook before search_telemetry. "
+                          "Expensive commands require confirm_expensive=true. "
                           "Reserve get_telemetry/get_slow_telemetry for rare full-context pulls.",
             "pins": "Pins are server-side, NOT localStorage. Use place_pin command or POST /api/ai/pins. The agent can place and delete pins.",
             "tracking": "To track a specific aircraft without polling: use add_watch with track_callsign or track_registration. Over SSE, you'll get instant push alerts.",
@@ -2637,6 +2721,7 @@ def _connect_info_metadata(settings) -> dict:
                     "get_telemetry", "get_pins", "satellite_images",
                     "news_near", "ai_summary", "ai_report",
                     "timemachine_list", "timemachine_view",
+                    "infonet_status", "list_gates", "read_gate_messages", "poll_dms",
                 ],
             },
             "full": {
@@ -2647,6 +2732,8 @@ def _connect_info_metadata(settings) -> dict:
                     "satellite_images", "news_near", "data_injection",
                     "ai_summary", "ai_report", "timemachine_snapshot",
                     "timemachine_list", "timemachine_view", "timemachine_diff",
+                    "ensure_infonet_ready", "join_infonet_swarm",
+                    "post_gate_message", "cast_vote", "send_dm",
                 ],
             },
         },

backend/routers/analytics.py

@@ -0,0 +1,339 @@
+"""Strategic Risk Analytics API — game-theoretic early warning overlays."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from pydantic import BaseModel, Field
+
+from auth import require_local_operator
+from limiter import limiter
+from analytics.backtest import (
+    DEFAULT_BACKTEST_ALERT_THRESHOLD,
+    run_historical_backtest,
+    tune_alert_threshold,
+)
+from analytics.feed_adapter import normalize_feed_item
+from analytics.integration import get_gt_engine, refresh_from_latest_data
+from analytics.gt_alerts import top_gt_alerts
+from analytics.micro_rolling import micro_rolling_report
+from analytics.rolling_backtest import (
+    freeze_weekly_snapshot,
+    label_region,
+    label_regions,
+    rolling_alert_threshold,
+    rolling_report,
+    score_week,
+)
+from analytics.weekly_store import load_week
+from analytics.settings import gt_analytics_enabled
+from services.fetchers._store import _data_lock, get_latest_data_subset_refs, latest_data
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+class RiskHeatmapRequest(BaseModel):
+    """Optional batch ingest + refresh controls for POST /api/analytics/risk_heatmap."""
+
+    refresh: bool = True
+    items: list[dict[str, Any]] = Field(default_factory=list)
+
+
+class RollingFreezeRequest(BaseModel):
+    week_id: str | None = None
+    force: bool = False
+
+
+class RollingLabelEntry(BaseModel):
+    region: str
+    label: str
+    notes: str = ""
+
+
+class RollingLabelRequest(BaseModel):
+    week_id: str
+    labels: list[RollingLabelEntry] = Field(default_factory=list)
+
+
+def _empty_heatmap() -> dict[str, Any]:
+    return {
+        "enabled": False,
+        "type": "FeatureCollection",
+        "features": [],
+        "clusters": [],
+        "processed": 0,
+        "timestamp": None,
+    }
+
+
+def _gt_risk_payload() -> dict[str, Any]:
+    snap = get_latest_data_subset_refs("gt_risk")
+    payload = snap.get("gt_risk")
+    if not isinstance(payload, dict):
+        return _empty_heatmap()
+    heatmap = payload.get("heatmap") or {"type": "FeatureCollection", "features": []}
+    return {
+        "enabled": bool(payload.get("enabled")),
+        "type": heatmap.get("type", "FeatureCollection"),
+        "features": list(heatmap.get("features") or []),
+        "clusters": list(payload.get("clusters") or []),
+        "processed": int(payload.get("processed") or 0),
+        "timestamp": payload.get("timestamp"),
+    }
+
+
+@router.get("/api/analytics/risk_heatmap")
+@limiter.limit("60/minute")
+async def risk_heatmap_get(request: Request) -> dict[str, Any]:
+    """Return cached GeoJSON risk overlay (posterior scores per region)."""
+    if not gt_analytics_enabled():
+        return _empty_heatmap()
+    return _gt_risk_payload()
+
+
+@router.post("/api/analytics/risk_heatmap")
+@limiter.limit("12/minute")
+async def risk_heatmap_post(
+    request: Request,
+    body: RiskHeatmapRequest,
+    _: None = Depends(require_local_operator),
+) -> dict[str, Any]:
+    """
+    Ingest optional feed items and/or refresh beliefs from latest intel layers.
+
+    Requires local operator auth — intended for OpenClaw agents and admin tooling.
+    """
+    if not gt_analytics_enabled():
+        raise HTTPException(status_code=503, detail="Strategic Risk Analytics is disabled")
+
+    engine = get_gt_engine()
+    if engine is None:
+        raise HTTPException(status_code=503, detail="Strategic Risk Analytics engine unavailable")
+
+    ingested = 0
+    for raw in body.items:
+        if not isinstance(raw, dict):
+            continue
+        source_type = str(raw.get("source_type") or "manual")
+        item = normalize_feed_item(raw, source_type=source_type)
+        result = engine.process_feed_item(item)
+        if result and not result.get("skipped"):
+            ingested += 1
+
+    summary: dict[str, Any] = {"ingested": ingested}
+    if body.refresh:
+        with _data_lock:
+            snapshot = dict(latest_data)
+        summary.update(refresh_from_latest_data(snapshot, persist=True))
+
+    payload = _gt_risk_payload()
+    payload["ingested"] = ingested
+    payload["refresh"] = bool(body.refresh)
+    return payload
+
+
+@router.get("/api/analytics/dossier/{region}")
+@limiter.limit("30/minute")
+async def analytics_dossier(request: Request, region: str) -> dict[str, Any]:
+    """Game-theoretic rationale, recent costly signals, and scenario sketches."""
+    region_key = str(region or "").strip().lower()
+    if not region_key or len(region_key) > 120:
+        raise HTTPException(status_code=400, detail="Invalid region identifier")
+
+    if not gt_analytics_enabled():
+        return {
+            "enabled": False,
+            "region": region_key,
+            "current_risk": 0.0,
+            "interpretation": "Strategic Risk Analytics is disabled.",
+            "recent_signals": [],
+            "scenarios": [],
+        }
+
+    engine = get_gt_engine()
+    if engine is None:
+        raise HTTPException(status_code=503, detail="Strategic Risk Analytics engine unavailable")
+
+    dossier = engine.get_dossier(region_key)
+    dossier["enabled"] = True
+    return dossier
+
+
+@router.get("/api/analytics/backtest")
+@limiter.limit("6/minute")
+async def analytics_backtest(
+    request: Request,
+    expanded: bool = True,
+    tune: bool = False,
+    target_confidence: float = 0.95,
+) -> dict[str, Any]:
+    """
+    Run labeled historical backtest and return accuracy + Wilson 95% CI.
+
+    ``confidence_rate`` is the Wilson lower bound (conservative pass metric).
+    """
+    if not gt_analytics_enabled():
+        return {
+            "enabled": False,
+            "message": "Strategic Risk Analytics is disabled.",
+        }
+
+    if tune:
+        threshold, report = tune_alert_threshold(target_confidence=target_confidence)
+    else:
+        threshold = DEFAULT_BACKTEST_ALERT_THRESHOLD
+        report = run_historical_backtest(
+            use_expanded_suite=expanded,
+            alert_threshold=threshold,
+            target_confidence=target_confidence,
+        )
+
+    payload = report.to_dict()
+    payload["enabled"] = True
+    payload["expanded_suite"] = expanded
+    payload["tuned"] = tune
+    payload["recommended_alert_threshold"] = threshold
+    return payload
+
+
+@router.get("/api/analytics/rolling")
+@limiter.limit("12/minute")
+async def analytics_rolling(
+    request: Request,
+    weeks: int = 8,
+    target_confidence: float = 0.80,
+) -> dict[str, Any]:
+    """Rolling weekly operational validation — accuracy trend with delayed labels."""
+    if not gt_analytics_enabled():
+        return {
+            "enabled": False,
+            "message": "Strategic Risk Analytics is disabled.",
+        }
+
+    report = rolling_report(weeks=max(1, min(weeks, 52)), target_confidence=target_confidence)
+    report["enabled"] = True
+    return report
+
+
+@router.get("/api/analytics/alerts")
+@limiter.limit("30/minute")
+async def analytics_top_alerts(
+    request: Request,
+    limit: int = 8,
+) -> dict[str, Any]:
+    """Top GT risk regions ranked by score — fly-to targets for the map."""
+    if not gt_analytics_enabled():
+        return {
+            "enabled": False,
+            "message": "Strategic Risk Analytics is disabled.",
+        }
+
+    report = top_gt_alerts(limit=max(1, min(limit, 25)))
+    report["enabled"] = True
+    return report
+
+
+@router.get("/api/analytics/rolling/micro")
+@limiter.limit("30/minute")
+async def analytics_rolling_micro(
+    request: Request,
+    window_days: int = 3,
+    limit: int = 15,
+) -> dict[str, Any]:
+    """Rolling 3-day micro average — spot vs baseline, ignition detection."""
+    if not gt_analytics_enabled():
+        return {
+            "enabled": False,
+            "message": "Strategic Risk Analytics is disabled.",
+        }
+
+    report = micro_rolling_report(
+        window_days=max(2, min(window_days, 7)),
+        limit=max(1, min(limit, 50)),
+    )
+    report["enabled"] = True
+    return report
+
+
+@router.get("/api/analytics/rolling/{week_id}")
+@limiter.limit("12/minute")
+async def analytics_rolling_week(request: Request, week_id: str) -> dict[str, Any]:
+    """Return a single frozen week snapshot and its score."""
+    if not gt_analytics_enabled():
+        return {"enabled": False, "message": "Strategic Risk Analytics is disabled."}
+
+    snapshot = load_week(str(week_id).strip())
+    if snapshot is None:
+        raise HTTPException(status_code=404, detail=f"Week {week_id} not found")
+
+    score = score_week(snapshot)
+    return {
+        "enabled": True,
+        "week_id": snapshot.week_id,
+        "snapshot": snapshot.to_dict(),
+        "score": score.to_dict(),
+        "alert_threshold": rolling_alert_threshold(),
+    }
+
+
+@router.post("/api/analytics/rolling/freeze")
+@limiter.limit("6/minute")
+async def analytics_rolling_freeze(
+    request: Request,
+    body: RollingFreezeRequest,
+    _: None = Depends(require_local_operator),
+) -> dict[str, Any]:
+    """Freeze current GT scores for the ISO week (idempotent unless force=true)."""
+    if not gt_analytics_enabled():
+        raise HTTPException(status_code=503, detail="Strategic Risk Analytics is disabled")
+
+    result = freeze_weekly_snapshot(
+        week_id=body.week_id,
+        force=body.force,
+        frozen_by="api",
+    )
+    if not result.get("ok"):
+        raise HTTPException(status_code=503, detail=result.get("detail", "Freeze failed"))
+    result["enabled"] = True
+    return result
+
+
+@router.post("/api/analytics/rolling/label")
+@limiter.limit("12/minute")
+async def analytics_rolling_label(
+    request: Request,
+    body: RollingLabelRequest,
+    _: None = Depends(require_local_operator),
+) -> dict[str, Any]:
+    """Apply delayed outcome labels to a frozen week."""
+    if not gt_analytics_enabled():
+        raise HTTPException(status_code=503, detail="Strategic Risk Analytics is disabled")
+
+    week_id = str(body.week_id or "").strip()
+    if not week_id:
+        raise HTTPException(status_code=400, detail="week_id required")
+
+    if len(body.labels) == 1:
+        entry = body.labels[0]
+        result = label_region(
+            week_id,
+            entry.region,
+            entry.label,  # type: ignore[arg-type]
+            notes=entry.notes,
+            labeled_by="api",
+        )
+    else:
+        result = label_regions(
+            week_id,
+            [row.model_dump() for row in body.labels],
+            labeled_by="api",
+        )
+
+    if not result.get("ok"):
+        raise HTTPException(status_code=404, detail=result.get("detail", "Label failed"))
+    result["enabled"] = True
+    return result

backend/routers/cctv.py

@@ -47,6 +47,8 @@ _CCTV_PROXY_ALLOWED_HOSTS = {
     "www.tripcheck.com",
     "infocar.dgt.es",
     "informo.madrid.es",
+    "webcams2.asfinag.at",
+    "odo.asfinag.at",
     "www.windy.com",
     "imgproxy.windy.com",
     "www.lakecountypassage.com",
@@ -55,6 +57,14 @@ _CCTV_PROXY_ALLOWED_HOSTS = {
     "www.nps.gov",
     "home.lewiscounty.com",
     "www.seattle.gov",
+    "511on.ca",
+    "511.alberta.ca",
+    "fl511.com",
+    "www.fl511.com",
+    "webcams.transport.nsw.gov.au",
+    "www.livetraffic.com",
+    "livetraffic.com",
+    "opendata.ndw.nu",
 }
 
 
@@ -120,7 +130,7 @@ def _cctv_proxy_profile_for_url(target_url: str) -> _CCTVProxyProfile:
         read_timeout = 18.0 if "/snapshots/" in path else 12.0
         return _CCTVProxyProfile(name="gdot-snapshot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, read_timeout), cache_seconds=15,
             headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
-                     "Referer": "http://navigator-c2c.dot.ga.gov/"})
+                     "Referer": "https://navigator-c2c.dot.ga.gov/"})
     if host == "511ga.org":
         return _CCTVProxyProfile(name="gdot-511ga-image", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=15,
             headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
@@ -128,7 +138,7 @@ def _cctv_proxy_profile_for_url(target_url: str) -> _CCTVProxyProfile:
     if host.startswith("vss") and host.endswith("dot.ga.gov"):
         return _CCTVProxyProfile(name="gdot-hls", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 20.0), cache_seconds=10,
             headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,*/*;q=0.8",
-                     "Referer": "http://navigator-c2c.dot.ga.gov/"})
+                     "Referer": "https://navigator-c2c.dot.ga.gov/"})
     if host in {"gettingaroundillinois.com", "cctv.travelmidwest.com"}:
         return _CCTVProxyProfile(name="illinois-dot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
             headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
@@ -156,10 +166,34 @@ def _cctv_proxy_profile_for_url(target_url: str) -> _CCTVProxyProfile:
         return _CCTVProxyProfile(name="madrid-city", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
             headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
                      "Referer": "https://informo.madrid.es/"})
+    if host in {"webcams2.asfinag.at", "odo.asfinag.at"}:
+        return _CCTVProxyProfile(name="asfinag-austria", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.asfinag.at/"})
     if host in {"www.windy.com", "imgproxy.windy.com"}:
         return _CCTVProxyProfile(name="windy-webcams", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=60,
             headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
                      "Referer": "https://www.windy.com/"})
+    if host == "511on.ca":
+        return _CCTVProxyProfile(name="ontario-511", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://511on.ca/"})
+    if host == "511.alberta.ca":
+        return _CCTVProxyProfile(name="alberta-511", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://511.alberta.ca/"})
+    if host in {"fl511.com", "www.fl511.com"}:
+        return _CCTVProxyProfile(name="florida-511", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://fl511.com/"})
+    if host == "webcams.transport.nsw.gov.au":
+        return _CCTVProxyProfile(name="nsw-live-traffic", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.livetraffic.com/"})
+    if host in {"opendata.ndw.nu", "www.ndw.nu"}:
+        return _CCTVProxyProfile(name="ndw-netherlands", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=120,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.ndw.nu/"})
     return _CCTVProxyProfile(name="generic-cctv", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0), cache_seconds=30,
         headers={"Accept": "*/*"})
 

backend/routers/data.py

@@ -1,6 +1,7 @@
 import asyncio
 import logging
 import math
+import os
 import threading
 from typing import Any
 from fastapi import APIRouter, Request, Response, Query, Depends
@@ -8,7 +9,7 @@ from fastapi.responses import JSONResponse
 from pydantic import BaseModel
 from limiter import limiter
 from auth import require_admin, require_local_operator
-from services.data_fetcher import get_latest_data, update_all_data
+from services.data_fetcher import update_all_data
 import orjson
 import json as json_mod
 
@@ -30,6 +31,14 @@ class LayerUpdate(BaseModel):
     layers: dict[str, bool]
 
 
+class LiveUamapOptInUpdate(BaseModel):
+    opted_in: bool
+
+
+class PredictionMarketsOptInUpdate(BaseModel):
+    opted_in: bool
+
+
 _LAST_VIEWPORT_UPDATE: tuple | None = None
 _LAST_VIEWPORT_UPDATE_TS = 0.0
 _VIEWPORT_UPDATE_LOCK = threading.Lock()
@@ -202,6 +211,15 @@ def _sanitize_payload(value):
     return value
 
 
+def _live_data_json_bytes(payload: dict) -> bytes:
+    """Serialize dashboard payloads with the same defensive orjson options everywhere."""
+    return orjson.dumps(
+        _sanitize_payload(payload),
+        default=str,
+        option=orjson.OPT_NON_STR_KEYS,
+    )
+
+
 def _bbox_filter(items: list, s: float, w: float, n: float, e: float,
                  lat_key: str = "lat", lng_key: str = "lng") -> list:
     pad_lat = (n - s) * 0.2
@@ -386,6 +404,95 @@ async def update_viewport(vp: ViewportUpdate, request: Request):  # noqa: ARG001
     return {"status": "ok"}
 
 
+@router.get("/api/liveuamap/scraper-status", dependencies=[Depends(require_local_operator)])
+async def api_liveuamap_scraper_status():
+    """Whether LiveUAMap Playwright may run (Windows needs UI opt-in unless env forces)."""
+    from services.liveuamap_settings import liveuamap_scraper_status
+
+    return liveuamap_scraper_status()
+
+
+@router.post("/api/liveuamap/scraper-opt-in", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_liveuamap_scraper_opt_in(body: LiveUamapOptInUpdate, request: Request):
+    """Persist operator consent for LiveUAMap scraper (#348)."""
+    from services.liveuamap_settings import liveuamap_scraper_status, set_liveuamap_ui_opt_in
+
+    set_liveuamap_ui_opt_in(body.opted_in)
+    if body.opted_in:
+        from services.fetchers._store import is_any_active
+
+        if is_any_active("global_incidents"):
+            threading.Thread(target=_run_liveuamap_refresh, daemon=True).start()
+    return liveuamap_scraper_status()
+
+
+def _run_liveuamap_refresh() -> None:
+    try:
+        from services.fetchers.geo import update_liveuamap
+
+        update_liveuamap()
+    except Exception as e:
+        logger.warning("LiveUAMap refresh after opt-in failed: %s", e)
+
+
+@router.get("/api/prediction-markets/status", dependencies=[Depends(require_local_operator)])
+async def api_prediction_markets_status():
+    """Whether Polymarket/Kalshi fetches and news market correlation are enabled."""
+    from services.prediction_markets_settings import prediction_markets_status
+
+    return prediction_markets_status()
+
+
+@router.post("/api/prediction-markets/opt-in", dependencies=[Depends(require_local_operator)])
+@limiter.limit("10/minute")
+async def api_prediction_markets_opt_in(body: PredictionMarketsOptInUpdate, request: Request):
+    """Enable or disable prediction market fetches + intercept story correlation."""
+    from services.config import get_settings
+    from services.prediction_markets_settings import (
+        prediction_markets_status,
+        set_prediction_markets_ui_opt_in,
+    )
+    from routers.ai_intel import _write_env_value
+
+    set_prediction_markets_ui_opt_in(body.opted_in)
+    _write_env_value("PREDICTION_MARKETS_ENABLED", "true" if body.opted_in else "false")
+    os.environ["PREDICTION_MARKETS_ENABLED"] = "true" if body.opted_in else "false"
+    get_settings.cache_clear()
+
+    if body.opted_in:
+        threading.Thread(target=_run_prediction_markets_refresh, daemon=True).start()
+    else:
+        threading.Thread(target=_run_prediction_markets_disable, daemon=True).start()
+
+    return prediction_markets_status()
+
+
+def _run_prediction_markets_refresh() -> None:
+    try:
+        from services.fetchers.prediction_markets import fetch_prediction_markets
+        from services.fetchers.news import fetch_news
+
+        fetch_prediction_markets()
+        fetch_news()
+    except Exception as e:
+        logger.warning("Prediction markets refresh after opt-in failed: %s", e)
+
+
+def _run_prediction_markets_disable() -> None:
+    try:
+        from services.fetchers._store import _data_lock, _mark_fresh, latest_data
+        from services.fetchers.news import fetch_news
+
+        with _data_lock:
+            latest_data["prediction_markets"] = []
+            latest_data["trending_markets"] = []
+        _mark_fresh("prediction_markets")
+        fetch_news()
+    except Exception as e:
+        logger.warning("Prediction markets disable cleanup failed: %s", e)
+
+
 @router.post("/api/layers", dependencies=[Depends(require_local_operator)])
 @limiter.limit("30/minute")
 async def update_layers(update: LayerUpdate, request: Request):
@@ -395,6 +502,8 @@ async def update_layers(update: LayerUpdate, request: Request):
     old_mesh = is_any_active("sigint_meshtastic")
     old_aprs = is_any_active("sigint_aprs")
     old_viirs = is_any_active("viirs_nightlights")
+    old_datacenters = is_any_active("datacenters")
+    old_fishing = is_any_active("fishing_activity")
     changed = False
     for key, value in update.layers.items():
         if key in active_layers:
@@ -407,6 +516,8 @@ async def update_layers(update: LayerUpdate, request: Request):
     new_mesh = is_any_active("sigint_meshtastic")
     new_aprs = is_any_active("sigint_aprs")
     new_viirs = is_any_active("viirs_nightlights")
+    new_datacenters = is_any_active("datacenters")
+    new_fishing = is_any_active("fishing_activity")
     if old_ships and not new_ships:
         from services.ais_stream import stop_ais_stream
         stop_ais_stream()
@@ -450,13 +561,33 @@ async def update_layers(update: LayerUpdate, request: Request):
     if not old_viirs and new_viirs:
         _queue_viirs_change_refresh()
         logger.info("VIIRS change refresh queued (layer enabled)")
+    if not old_datacenters and new_datacenters:
+        from services.fetchers.infrastructure import fetch_datacenters
+
+        fetch_datacenters()
+        logger.info("Datacenters loaded (layer enabled)")
+    if not old_fishing and new_fishing:
+        from services.fetchers.geo import fetch_fishing_activity
+
+        fetch_fishing_activity()
+        logger.info("Fishing activity refresh queued (layer enabled)")
     return {"status": "ok"}
 
 
 @router.get("/api/live-data")
 @limiter.limit("120/minute")
 async def live_data(request: Request):
-    return get_latest_data()
+    etag = _current_etag(prefix="live|full|")
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
+    from services.fetchers._store import get_latest_data_deepcopy_snapshot
+
+    payload = get_latest_data_deepcopy_snapshot()
+    return Response(
+        content=_live_data_json_bytes(payload),
+        media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"},
+    )
 
 
 @router.get("/api/bootstrap/critical")
@@ -551,7 +682,7 @@ async def bootstrap_critical(request: Request):
         "bootstrap_payload": True,
     }
     return Response(
-        content=orjson.dumps(_sanitize_payload(payload), default=str, option=orjson.OPT_NON_STR_KEYS),
+        content=_live_data_json_bytes(payload),
         media_type="application/json",
         headers={"ETag": etag, "Cache-Control": "no-cache"},
     )
@@ -613,8 +744,11 @@ async def live_data_fast(
     # to the pre-#288 implementation.
     if _has_full_bbox(s, w, n, e):
         payload = _apply_bbox_to_payload(payload, _FAST_BBOX_HEAVY_KEYS, s, w, n, e)
-    return Response(content=orjson.dumps(_sanitize_payload(payload)), media_type="application/json",
-        headers={"ETag": etag, "Cache-Control": "no-cache"})
+    return Response(
+        content=_live_data_json_bytes(payload),
+        media_type="application/json",
+        headers={"ETag": etag, "Cache-Control": "no-cache"},
+    )
 
 
 @router.get("/api/live-data/slow")
@@ -638,7 +772,8 @@ async def live_data_slow(
         "firms_fires", "datacenters", "military_bases", "power_plants", "viirs_change_nodes",
         "scanners", "weather_alerts", "ukraine_alerts", "air_quality", "volcanoes",
         "fishing_activity", "psk_reporter", "correlations", "uap_sightings", "wastewater",
-        "crowdthreat", "threat_level", "trending_markets",
+        "crowdthreat", "threat_level", "trending_markets", "road_corridor_trends",
+        "malware_threats", "cyber_threats", "scm_suppliers", "telegram_osint", "gt_risk",
     )
     freshness = get_source_timestamps_snapshot()
     payload = {
@@ -679,6 +814,37 @@ async def live_data_slow(
         "uap_sightings": (d.get("uap_sightings") or []) if active_layers.get("uap_sightings", True) else [],
         "wastewater": (d.get("wastewater") or []) if active_layers.get("wastewater", True) else [],
         "crowdthreat": (d.get("crowdthreat") or []) if active_layers.get("crowdthreat", True) else [],
+        "road_corridor_trends": (
+            d.get("road_corridor_trends") or {"updated_at": None, "corridors": []}
+        )
+        if active_layers.get("road_corridor_trends", False)
+        else {"updated_at": None, "corridors": []},
+        "malware_threats": (
+            d.get("malware_threats") or {"threats": [], "total": 0}
+        )
+        if active_layers.get("malware_c2", False)
+        else {"threats": [], "total": 0},
+        "cyber_threats": (
+            d.get("cyber_threats") or {"threats": [], "stats": {}}
+        )
+        if active_layers.get("cyber_threats", False)
+        else {"threats": [], "stats": {}},
+        "scm_suppliers": (
+            d.get("scm_suppliers") or {"suppliers": [], "total": 0, "critical_count": 0}
+        )
+        if active_layers.get("scm_suppliers", False)
+        else {"suppliers": [], "total": 0, "critical_count": 0},
+        "telegram_osint": (
+            d.get("telegram_osint") or {"posts": [], "total": 0, "geolocated": 0}
+        )
+        if active_layers.get("telegram_osint", True)
+        else {"posts": [], "total": 0, "geolocated": 0},
+        "gt_risk": (
+            d.get("gt_risk")
+            or {"enabled": False, "heatmap": {"type": "FeatureCollection", "features": []}, "clusters": []}
+        )
+        if active_layers.get("gt_risk", False)
+        else {"enabled": False, "heatmap": {"type": "FeatureCollection", "features": []}, "clusters": []},
         "freshness": freshness,
     }
     # Issue #288: bbox filter heavy/dense layers only when all four bounds
@@ -688,7 +854,7 @@ async def live_data_slow(
     if _has_full_bbox(s, w, n, e):
         payload = _apply_bbox_to_payload(payload, _SLOW_BBOX_HEAVY_KEYS, s, w, n, e)
     return Response(
-        content=orjson.dumps(_sanitize_payload(payload), default=str, option=orjson.OPT_NON_STR_KEYS),
+        content=_live_data_json_bytes(payload),
         media_type="application/json",
         headers={"ETag": etag, "Cache-Control": "no-cache"},
     )

backend/routers/entity_graph.py

@@ -0,0 +1,30 @@
+"""Entity graph expansion (intel layer)."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+
+from auth import require_local_operator
+from limiter import limiter
+from services.osint_intel.resolve import resolve_entity
+
+router = APIRouter()
+
+
+@router.get("/api/entity/expand")
+@limiter.limit("30/minute")
+async def entity_expand(
+    request: Request,
+    _: None = Depends(require_local_operator),
+    type: str = Query(..., min_length=3, max_length=32),
+    id: str = Query(..., min_length=2, max_length=200),
+    registration: str | None = Query(default=None, max_length=32),
+    model: str | None = Query(default=None, max_length=64),
+    icao24: str | None = Query(default=None, max_length=16),
+) -> dict:
+    props = {"label": id, "registration": registration, "model": model, "icao24": icao24}
+    try:
+        return resolve_entity(type, id, props)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    except Exception as exc:
+        raise HTTPException(status_code=502, detail="Intelligence layer unavailable") from exc

backend/routers/health.py

@@ -8,7 +8,7 @@ from services.data_fetcher import get_latest_data
 from services.schemas import HealthResponse
 import os
 
-APP_VERSION = os.environ.get("_HEALTH_APP_VERSION", "0.9.81")
+APP_VERSION = os.environ.get("_HEALTH_APP_VERSION", "0.9.82")
 
 router = APIRouter()
 
@@ -85,6 +85,18 @@ async def health_check(request: Request):
     ):
         top_status = "degraded"
 
+    runtime: dict = {}
+    try:
+        from services.runtime_profile import get_runtime_profile
+        from analytics.settings import gt_analytics_status
+
+        runtime = {
+            **get_runtime_profile(),
+            "gt_analytics": gt_analytics_status(),
+        }
+    except Exception:
+        runtime = {}
+
     return {
         "status": top_status,
         "version": _get_app_version(),
@@ -108,6 +120,7 @@ async def health_check(request: Request):
         "slo": slo_statuses,
         "slo_summary": slo_summary,
         "ais_proxy": ais_status,
+        "runtime": runtime or None,
     }
 
 

backend/routers/intel_feeds.py

@@ -0,0 +1,130 @@
+"""Malware, cyber threats, and country risk feeds."""
+from __future__ import annotations
+
+import logging
+from urllib.parse import urlparse
+
+import requests
+from fastapi import APIRouter, HTTPException, Query, Request
+from fastapi.responses import StreamingResponse
+from starlette.background import BackgroundTask
+
+from limiter import limiter
+from services.fetchers._store import get_latest_data_subset_refs
+from services.fetchers.telegram_osint import telegram_media_host_allowed
+from services.intel_feeds.country_risk import build_country_risk_payload
+from services.network_utils import outbound_user_agent
+from services.telegram_translate import apply_posts_translations, normalize_translate_target
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/api/malware")
+@limiter.limit("60/minute")
+async def malware_feed(request: Request) -> dict:
+    snap = get_latest_data_subset_refs("malware_threats")
+    payload = snap.get("malware_threats")
+    if isinstance(payload, dict) and payload.get("threats") is not None:
+        return payload
+    return {"threats": [], "total": 0, "timestamp": None, "source": "abuse.ch"}
+
+
+@router.get("/api/cyber-threats")
+@limiter.limit("60/minute")
+async def cyber_threats(request: Request) -> dict:
+    snap = get_latest_data_subset_refs("cyber_threats")
+    return snap.get("cyber_threats") or {"threats": [], "stats": {}}
+
+
+@router.get("/api/country-risk")
+@limiter.limit("30/minute")
+async def country_risk(request: Request) -> dict:
+    return build_country_risk_payload()
+
+
+@router.get("/api/telegram-feed")
+@limiter.limit("30/minute")
+async def telegram_feed(request: Request, lang: str | None = Query(default=None)) -> dict:
+    snap = get_latest_data_subset_refs("telegram_osint")
+    payload = snap.get("telegram_osint")
+    if not isinstance(payload, dict) or payload.get("posts") is None:
+        return {"posts": [], "total": 0, "geolocated": 0, "timestamp": None}
+
+    if lang:
+        target = normalize_translate_target(lang)
+        localized = dict(payload)
+        localized["posts"] = apply_posts_translations(list(payload.get("posts") or []), target)
+        localized["translate_locale"] = target
+        return localized
+    return payload
+
+
+def _infer_telegram_media_type(target_url: str, content_type: str) -> str:
+    clean_type = str(content_type or "").split(";", 1)[0].strip().lower()
+    if clean_type and clean_type not in {"application/octet-stream", "binary/octet-stream"}:
+        return content_type
+    path = str(urlparse(target_url).path or "").lower()
+    if path.endswith((".jpg", ".jpeg")):
+        return "image/jpeg"
+    if path.endswith(".png"):
+        return "image/png"
+    if path.endswith(".webp"):
+        return "image/webp"
+    if path.endswith(".gif"):
+        return "image/gif"
+    if path.endswith(".mp4"):
+        return "video/mp4"
+    if path.endswith(".webm"):
+        return "video/webm"
+    return content_type or "application/octet-stream"
+
+
+@router.get("/api/telegram/media")
+@limiter.limit("60/minute")
+async def telegram_media_proxy(request: Request, url: str = Query(...)) -> StreamingResponse:
+    """Stream Telegram CDN media for in-app playback (host allowlist only)."""
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https"):
+        raise HTTPException(status_code=400, detail="Invalid scheme")
+    if not telegram_media_host_allowed(parsed.hostname):
+        raise HTTPException(status_code=403, detail="Host not allowed")
+
+    headers = {
+        "User-Agent": (
+            f"Mozilla/5.0 (compatible; {outbound_user_agent('telegram-media')}) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+        ),
+        "Accept": "*/*",
+    }
+    if range_header := request.headers.get("range"):
+        headers["Range"] = range_header
+
+    try:
+        resp = requests.get(url, stream=True, timeout=(3, 45), headers=headers)
+    except requests.RequestException as exc:
+        logger.warning("Telegram media upstream failure %s: %s", url, exc)
+        raise HTTPException(status_code=502, detail="Upstream fetch failed") from exc
+
+    if resp.status_code >= 400:
+        resp.close()
+        raise HTTPException(status_code=int(resp.status_code), detail=f"Upstream returned {resp.status_code}")
+
+    media_type = _infer_telegram_media_type(url, resp.headers.get("Content-Type", "application/octet-stream"))
+    response_headers = {
+        "Cache-Control": "private, max-age=300",
+        "Accept-Ranges": resp.headers.get("Accept-Ranges", "bytes"),
+    }
+    if content_length := resp.headers.get("Content-Length"):
+        response_headers["Content-Length"] = content_length
+    if content_range := resp.headers.get("Content-Range"):
+        response_headers["Content-Range"] = content_range
+
+    return StreamingResponse(
+        resp.iter_content(chunk_size=65536),
+        status_code=resp.status_code,
+        media_type=media_type,
+        headers=response_headers,
+        background=BackgroundTask(resp.close),
+    )

backend/routers/mesh_peer_sync.py

@@ -55,10 +55,20 @@ def _hydrate_gate_store_from_chain(events: list) -> int:
     return count
 
 
+def _hydrate_dm_relay_from_chain(events: list) -> int:
+    import main as _m
+
+    return int(_m._hydrate_dm_relay_from_chain(events))
+
+
 @router.post("/api/mesh/infonet/peer-push")
 @limiter.limit("30/minute")
 async def infonet_peer_push(request: Request):
     """Accept pushed Infonet events from relay peers (HMAC-authenticated)."""
+    from services.mesh.mesh_fleet_defaults import infonet_fleet_join_enabled
+
+    if not infonet_fleet_join_enabled():
+        return {"ok": True, "accepted": 0, "duplicates": 0, "rejected": [], "skipped": "fleet_join_disabled"}
     content_length = request.headers.get("content-length")
     if content_length:
         try:
@@ -82,6 +92,7 @@ async def infonet_peer_push(request: Request):
         return {"ok": True, "accepted": 0, "duplicates": 0, "rejected": []}
     result = infonet.ingest_events(events)
     _hydrate_gate_store_from_chain(events)
+    _hydrate_dm_relay_from_chain(events)
     return {"ok": True, **result}
 
 
@@ -147,6 +158,10 @@ async def dm_replicate_envelope(request: Request):
 @limiter.limit("30/minute")
 async def gate_peer_push(request: Request):
     """Accept pushed gate events from relay peers (private plane)."""
+    from services.mesh.mesh_fleet_defaults import infonet_fleet_join_enabled
+
+    if not infonet_fleet_join_enabled():
+        return {"ok": True, "accepted": 0, "duplicates": 0, "skipped": "fleet_join_disabled"}
     content_length = request.headers.get("content-length")
     if content_length:
         try:

backend/routers/mesh_public.py

@@ -65,6 +65,7 @@ from services.mesh.mesh_signed_events import (
 logger = logging.getLogger(__name__)
 
 router = APIRouter()
+_INFONET_SYNC_RATE_LIMIT = "600/minute"
 
 
 def _signed_body(request: Request) -> dict[str, Any]:
@@ -263,6 +264,19 @@ def _redact_public_event(event: dict) -> dict:
     return _redact_vote_gate(_redact_key_rotate_payload(_redact_gate_metadata(event)))
 
 
+def _infonet_private_transport_required() -> bool:
+    import main as _m
+
+    return bool(_m._infonet_private_transport_required())
+
+
+def _infonet_sync_response_events(events: list[dict], request=None) -> list[dict]:
+    """Build the sync event surface for the current transport policy."""
+    import main as _m
+
+    return _m._infonet_sync_response_events(events, request=request)
+
+
 def _trusted_gate_reply_to(event: dict) -> str:
     if not isinstance(event, dict):
         return ""
@@ -574,6 +588,12 @@ def _hydrate_gate_store_from_chain(events: list[dict]) -> int:
             pass
     return count
 
+
+def _hydrate_dm_relay_from_chain(events: list[dict]) -> int:
+    import main as _m
+
+    return int(_m._hydrate_dm_relay_from_chain(events))
+
 # --- Safe type helpers ---
 
 def _safe_int(val, default=0):
@@ -1531,7 +1551,7 @@ async def infonet_locator(request: Request, limit: int = Query(32, ge=4, le=128)
 
 
 @router.post("/api/mesh/infonet/sync")
-@limiter.limit("30/minute")
+@limiter.limit(_INFONET_SYNC_RATE_LIMIT)
 @mesh_write_exempt(MeshWriteExemption.PEER_GOSSIP)
 async def infonet_sync_post(
     request: Request,
@@ -1584,8 +1604,7 @@ async def infonet_sync_post(
     elif matched_hash == GENESIS_HASH and len(locator) > 1:
         forked = True
 
-    # Filter out legacy gate_message events — not part of the public sync surface.
-    events = [_redact_public_event(e) for e in events if e.get("event_type") != "gate_message"]
+    events = _infonet_sync_response_events(events, request=request)
 
     response = {
         "events": events,
@@ -1646,7 +1665,7 @@ async def mesh_rns_status(request: Request):
 
 
 @router.get("/api/mesh/infonet/sync")
-@limiter.limit("30/minute")
+@limiter.limit(_INFONET_SYNC_RATE_LIMIT)
 async def infonet_sync(
     request: Request,
     after_hash: str = "",
@@ -1684,8 +1703,7 @@ async def infonet_sync(
         )
     base = after_hash or GENESIS_HASH
     events = infonet.get_events_after(base, limit=limit)
-    # Filter out legacy gate_message events — not part of the public sync surface.
-    events = [_redact_public_event(e) for e in events if e.get("event_type") != "gate_message"]
+    events = _infonet_sync_response_events(events, request=request)
     return {
         "events": events,
         "after_hash": base,
@@ -1724,6 +1742,7 @@ async def infonet_ingest(request: Request):
 
     result = infonet.ingest_events(events)
     _hydrate_gate_store_from_chain(events)
+    _hydrate_dm_relay_from_chain(events)
     return {"ok": True, **result}
 
 
@@ -2279,6 +2298,12 @@ async def infonet_event(request: Request, event_id: str):
                 )
             return _strip_gate_for_access(evt, access)
         return {"ok": False, "detail": "Event not found"}
+    if evt.get("event_type") == "dm_message":
+        return await _private_plane_refusal_response(
+            request,
+            status_code=403,
+            payload=_private_plane_access_denied_payload(),
+        )
     if evt.get("event_type") == "gate_message":
         gate_id = str(evt.get("payload", {}).get("gate", "") or evt.get("gate", "") or "").strip()
         access = _verify_gate_access(request, gate_id) if gate_id else ""
@@ -2303,7 +2328,7 @@ async def infonet_node_events(
     from services.mesh.mesh_hashchain import infonet
 
     events = infonet.get_events_by_node(node_id, limit=limit)
-    events = [e for e in events if e.get("event_type") != "gate_message"]
+    events = [e for e in events if e.get("event_type") not in {"gate_message", "dm_message"}]
     events = [_redact_public_event(e) for e in infonet.decorate_events(events)]
     events = _redact_public_node_history(
         events,
@@ -2328,7 +2353,7 @@ async def infonet_events_by_type(
     else:
         events = list(reversed(infonet.events))
         events = events[offset : offset + limit]
-    events = [e for e in events if e.get("event_type") != "gate_message"]
+    events = [e for e in events if e.get("event_type") not in {"gate_message", "dm_message"}]
     events = [_redact_public_event(e) for e in infonet.decorate_events(events)]
     return {
         "events": events,

backend/routers/osint.py

@@ -0,0 +1,151 @@
+"""Operator OSINT recon routes (server-side proxies, SSRF guarded)."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+from pydantic import BaseModel, Field
+
+from auth import require_local_operator
+from limiter import limiter
+from services.osint import lookups
+
+router = APIRouter(dependencies=[Depends(require_local_operator)])
+
+_ALLOWED_SCHEMAS = {
+    "Person",
+    "Organization",
+    "Company",
+    "Vessel",
+    "Airplane",
+    "LegalEntity",
+}
+
+
+class SweepScanRequest(BaseModel):
+    ip: str = Field(min_length=7, max_length=45)
+    cidr: int = Field(default=24, ge=24, le=32)
+
+
+def _bad_request(exc: ValueError) -> HTTPException:
+    return HTTPException(status_code=400, detail=str(exc))
+
+
+@router.get("/api/osint/ip")
+@limiter.limit("20/minute")
+async def osint_ip(request: Request, ip: str = Query(..., min_length=7, max_length=45)) -> dict:
+    try:
+        return lookups.lookup_ip(ip)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/dns")
+@limiter.limit("20/minute")
+async def osint_dns(request: Request, domain: str = Query(..., min_length=4, max_length=253)) -> dict:
+    try:
+        return lookups.lookup_dns(domain)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/whois")
+@limiter.limit("20/minute")
+async def osint_whois(request: Request, domain: str = Query(..., min_length=4, max_length=253)) -> dict:
+    try:
+        return lookups.lookup_whois(domain)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/certs")
+@limiter.limit("20/minute")
+async def osint_certs(request: Request, domain: str = Query(..., min_length=4, max_length=253)) -> dict:
+    try:
+        return lookups.lookup_certs(domain)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/threats")
+@limiter.limit("20/minute")
+async def osint_threats(request: Request, query: str | None = Query(default=None, max_length=253)) -> dict:
+    return lookups.lookup_threats(query)
+
+
+@router.get("/api/osint/bgp")
+@limiter.limit("20/minute")
+async def osint_bgp(request: Request, query: str = Query(..., min_length=2, max_length=64)) -> dict:
+    try:
+        return lookups.lookup_bgp(query)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/sanctions")
+@limiter.limit("20/minute")
+async def osint_sanctions(
+    request: Request,
+    query: str = Query(..., min_length=4, max_length=200),
+    schema: str | None = Query(default=None),
+    limit: int = Query(default=25, ge=1, le=100),
+) -> dict:
+    if schema and schema not in _ALLOWED_SCHEMAS:
+        raise HTTPException(status_code=400, detail=f"Invalid schema. Allowed: {', '.join(sorted(_ALLOWED_SCHEMAS))}")
+    return lookups.lookup_sanctions(query, schema=schema, limit=limit)
+
+
+@router.get("/api/osint/cve")
+@limiter.limit("30/minute")
+async def osint_cve(request: Request, cve: str = Query(..., min_length=10, max_length=32)) -> dict:
+    try:
+        return lookups.lookup_cve(cve)
+    except ValueError as exc:
+        raise HTTPException(status_code=404 if "not found" in str(exc).lower() else 400, detail=str(exc)) from exc
+
+
+@router.get("/api/osint/mac")
+@limiter.limit("20/minute")
+async def osint_mac(request: Request, mac: str = Query(..., min_length=5, max_length=32)) -> dict:
+    return lookups.lookup_mac(mac)
+
+
+@router.get("/api/osint/github")
+@limiter.limit("20/minute")
+async def osint_github(request: Request, username: str = Query(..., min_length=1, max_length=64)) -> dict:
+    try:
+        return lookups.lookup_github(username)
+    except ValueError as exc:
+        raise HTTPException(status_code=404, detail=str(exc)) from exc
+
+
+@router.get("/api/osint/leaks")
+@limiter.limit("10/minute")
+async def osint_leaks(request: Request, email: str = Query(..., min_length=5, max_length=254)) -> dict:
+    try:
+        return lookups.lookup_leaks(email)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/sweep")
+@limiter.limit("5/minute")
+async def osint_sweep_init(
+    request: Request,
+    ip: str = Query(..., min_length=7, max_length=45),
+    cidr: int = Query(default=24, ge=24, le=32),
+) -> dict:
+    try:
+        return lookups.sweep_init(ip, cidr)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.post("/api/osint/sweep/scan")
+@limiter.limit("3/minute")
+async def osint_sweep_scan(request: Request, payload: SweepScanRequest) -> dict:
+    try:
+        subnet = lookups.subnet_start_for(payload.ip, payload.cidr)
+        scan = lookups.sweep_scan(subnet, payload.cidr)
+        init = lookups.sweep_init(payload.ip, payload.cidr)
+        return {**init, **scan, "subnet": f"{subnet}/{payload.cidr}"}
+    except ValueError as exc:
+        raise _bad_request(exc) from exc

backend/routers/road_corridors.py

@@ -0,0 +1,105 @@
+"""Road corridor Sentinel-2 freight trend endpoints (opt-in slow layer)."""
+
+from fastapi import APIRouter, HTTPException, Query, Request
+from pydantic import BaseModel, Field
+
+from limiter import limiter
+from services.road_corridor_sat.config import optional_deps_available, road_corridor_sat_enabled
+from services.road_corridor_sat.credentials import sentinel_credentials_configured
+from services.road_corridor_sat.jobs import enqueue_analyze, get_job, get_latest_job, job_to_dict
+from services.road_corridor_sat.presets import CORRIDOR_PRESETS, get_preset
+from services.road_corridor_sat.storage import build_trends_payload, preset_metadata
+
+router = APIRouter()
+
+
+def _status_payload() -> dict:
+    latest = get_latest_job()
+    return {
+        "enabled": road_corridor_sat_enabled(),
+        "deps_installed": optional_deps_available(),
+        "credentials_configured": sentinel_credentials_configured(),
+        "preset_count": len(CORRIDOR_PRESETS),
+        "attribution": "backend/third_party/drishx/NOTICE.md",
+        "active_job": job_to_dict(latest) if latest and latest.status in {"queued", "running"} else None,
+    }
+
+
+def _require_analyze_ready() -> None:
+    if not optional_deps_available():
+        raise HTTPException(
+            status_code=503,
+            detail="Install optional road-corridor dependencies (uv sync --extra road-corridor)",
+        )
+    if not sentinel_credentials_configured():
+        raise HTTPException(
+            status_code=503,
+            detail="Set SENTINEL_CLIENT_ID and SENTINEL_CLIENT_SECRET in Imagery settings",
+        )
+
+
+class AnalyzeRequest(BaseModel):
+    lat: float = Field(ge=-90, le=90)
+    lon: float = Field(ge=-180, le=180)
+    label: str | None = Field(default=None, max_length=120)
+
+
+@router.get("/api/road-corridors/status")
+@limiter.limit("60/minute")
+async def road_corridors_status(request: Request) -> dict:
+    return {"ok": True, **_status_payload()}
+
+
+@router.get("/api/road-corridors")
+@limiter.limit("60/minute")
+async def list_road_corridors(request: Request) -> dict:
+    return {
+        "ok": True,
+        "status": _status_payload(),
+        "presets": CORRIDOR_PRESETS,
+        "trends": build_trends_payload(),
+    }
+
+
+@router.post("/api/road-corridors/analyze")
+@limiter.limit("6/minute")
+async def analyze_road_corridor_here(request: Request, payload: AnalyzeRequest) -> dict:
+    """Start an on-demand Sentinel-2 corridor analysis at map center."""
+    _require_analyze_ready()
+    try:
+        job = enqueue_analyze(payload.lat, payload.lon, payload.label)
+    except RuntimeError as exc:
+        if str(exc) == "analysis_already_running":
+            active = get_latest_job()
+            raise HTTPException(
+                status_code=409,
+                detail="Analysis already in progress",
+                headers={"X-Job-Id": active.job_id if active else ""},
+            ) from exc
+        raise
+    return {"ok": True, **job_to_dict(job)}
+
+
+@router.get("/api/road-corridors/analyze/status")
+@limiter.limit("120/minute")
+async def analyze_road_corridor_status(
+    request: Request,
+    job_id: str | None = Query(default=None),
+) -> dict:
+    job = get_job(job_id) if job_id else get_latest_job()
+    if job is None:
+        return {"ok": True, "job": None}
+    return {"ok": True, "job": job_to_dict(job)}
+
+
+@router.get("/api/road-corridors/{preset_id}")
+@limiter.limit("60/minute")
+async def get_road_corridor(preset_id: str, request: Request) -> dict:
+    meta = preset_metadata(preset_id)
+    if meta is None:
+        raise HTTPException(status_code=404, detail="Unknown corridor preset")
+    preset = get_preset(preset_id)
+    if preset is None:
+        # Ad-hoc viewport runs are stored on disk but not in CORRIDOR_PRESETS.
+        return {"ok": True, "preset": None, "result": meta, "status": _status_payload()}
+    return {"ok": True, "preset": preset, "result": meta, "status": _status_payload()}

backend/routers/scm.py

@@ -0,0 +1,16 @@
+"""Supply-chain risk overlay."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, Request
+
+from auth import require_local_operator
+from limiter import limiter
+from services.scm.suppliers import build_scm_payload
+
+router = APIRouter()
+
+
+@router.get("/api/scm-suppliers")
+@limiter.limit("30/minute")
+async def scm_suppliers(request: Request, _: None = Depends(require_local_operator)) -> dict:
+    return build_scm_payload()

backend/routers/tools.py

@@ -85,6 +85,39 @@ async def api_geocode_reverse(
     return await asyncio.to_thread(reverse_geocode, lat, lng, local_only)
 
 
+# ── Wikimedia proxy (#360) — browser calls these instead of wikipedia.org ───
+@router.get("/api/wikipedia/summary")
+@limiter.limit("60/minute")
+def api_wikipedia_summary(
+    request: Request,
+    title: str = Query(..., min_length=1, max_length=256),
+):
+    """Proxy Wikipedia REST summaries through the self-hosted backend."""
+    from services.region_dossier import fetch_wikipedia_page_summary
+
+    summary = fetch_wikipedia_page_summary(title)
+    if summary is None:
+        return JSONResponse(status_code=404, content={"detail": "not_found"})
+    return summary
+
+
+class WikidataSparqlRequest(BaseModel):
+    query: str
+
+
+@router.post("/api/wikidata/sparql")
+@limiter.limit("30/minute")
+def api_wikidata_sparql(request: Request, body: WikidataSparqlRequest):
+    """Proxy Wikidata SPARQL so the browser never contacts query.wikidata.org."""
+    from services.region_dossier import fetch_wikidata_sparql_bindings
+
+    q = (body.query or "").strip()
+    if len(q) > 12_000:
+        raise HTTPException(400, "SPARQL query too large")
+    bindings = fetch_wikidata_sparql_bindings(q)
+    return {"bindings": bindings}
+
+
 # ── Sentinel proxy routes (Issue #299/#300/#301, reported by tg12) ──────────
 # These three endpoints relay external Sentinel / Planetary Computer
 # requests through the backend to avoid browser CORS blocks. They are

backend/routers/wormhole.py

@@ -308,6 +308,10 @@ class WormholeDmDecryptRequest(BaseModel):
     session_welcome: str | None = None
 
 
+class WormholeDmMlsKeyPackageRequest(BaseModel):
+    alias: str
+
+
 class WormholeDmResetRequest(BaseModel):
     peer_id: str | None = None
 
@@ -326,6 +330,14 @@ class WormholeDmBootstrapDecryptRequest(BaseModel):
     ciphertext: str
 
 
+class WormholeDmConnectContactRequest(BaseModel):
+    lookup_token: str = ""
+    peer_id: str = ""
+    note: str = ""
+    lookup_peer_url: str = ""
+    cached_prekey_bundle: dict[str, Any] | None = None
+
+
 class WormholeDmInviteImportRequest(BaseModel):
     invite: dict[str, Any]
     alias: str = ""
@@ -1085,7 +1097,21 @@ async def api_wormhole_dm_bootstrap_decrypt(request: Request, body: WormholeDmBo
     )
 
 
-@router.post("/api/wormhole/dm/sender-token", dependencies=[Depends(require_admin)])
+@router.post("/api/wormhole/dm/connect-contact", dependencies=[Depends(require_local_operator)])
+@limiter.limit("30/minute")
+async def api_wormhole_dm_connect_contact(request: Request, body: WormholeDmConnectContactRequest):
+    from services.openclaw_infonet import send_contact_request
+
+    return send_contact_request(
+        lookup_token=str(body.lookup_token or ""),
+        peer_id=str(body.peer_id or ""),
+        note=str(body.note or ""),
+        lookup_peer_url=str(body.lookup_peer_url or ""),
+        cached_prekey_bundle=dict(body.cached_prekey_bundle or {}) if body.cached_prekey_bundle else None,
+    )
+
+
+@router.post("/api/wormhole/dm/sender-token", dependencies=[Depends(require_local_operator)])
 @limiter.limit("60/minute")
 async def api_wormhole_dm_sender_token(request: Request, body: WormholeDmSenderTokenRequest):
     if _safe_int(body.count or 1, 1) > 1:
@@ -1228,6 +1254,23 @@ async def api_wormhole_dm_decrypt(request: Request, body: WormholeDmDecryptReque
     )
 
 
+@router.post("/api/wormhole/dm/mls-key-package", dependencies=[Depends(require_admin)])
+@limiter.limit("60/minute")
+async def api_wormhole_dm_mls_key_package(request: Request, body: WormholeDmMlsKeyPackageRequest):
+    from services.mesh.mesh_dm_mls import export_dm_key_package_for_alias
+
+    return export_dm_key_package_for_alias(str(body.alias or "").strip())
+
+
+@router.post("/api/wormhole/dm/mls-reset", dependencies=[Depends(require_admin)])
+@limiter.limit("30/minute")
+async def api_wormhole_dm_mls_reset(request: Request):
+    from services.mesh.mesh_dm_mls import reset_dm_mls_state
+
+    reset_dm_mls_state(clear_privacy_core=True, clear_persistence=True)
+    return {"ok": True}
+
+
 @router.post("/api/wormhole/dm/reset", dependencies=[Depends(require_admin)])
 @limiter.limit("30/minute")
 async def api_wormhole_dm_reset(request: Request, body: WormholeDmResetRequest):
@@ -1287,7 +1330,25 @@ async def api_wormhole_dm_contact_delete(request: Request, peer_id: str):
     return {"ok": True, "peer_id": peer_id, "deleted": deleted}
 
 
-_WORMHOLE_PUBLIC_FIELDS = {"installed", "configured", "running", "ready"}
+@router.post("/api/wormhole/dm/contact/{peer_id}/sever", dependencies=[Depends(require_admin)])
+@limiter.limit("60/minute")
+async def api_wormhole_dm_contact_sever(request: Request, peer_id: str):
+    from services.mesh.mesh_wormhole_contacts import sever_wormhole_dm_contact
+
+    try:
+        body = await request.json()
+    except Exception:
+        body = {}
+    if not isinstance(body, dict):
+        body = {}
+    block = bool(body.get("block", False))
+    try:
+        return sever_wormhole_dm_contact(peer_id, block=block)
+    except ValueError as exc:
+        return {"ok": False, "detail": str(exc)}
+
+
+_WORMHOLE_PUBLIC_FIELDS = {"installed", "configured", "running", "ready", "arti_ready"}
 
 
 def _redact_wormhole_status(state: dict[str, Any], authenticated: bool) -> dict[str, Any]:
@@ -1308,6 +1369,25 @@ async def api_wormhole_status(request: Request):
     return await _m.api_wormhole_status(request)
 
 
+@router.get(
+    "/api/wormhole/private-delivery/{item_id}",
+    dependencies=[Depends(require_local_operator)],
+)
+@limiter.limit("120/minute")
+async def api_wormhole_private_delivery_item(request: Request, item_id: str):
+    from services.mesh.mesh_metadata_exposure import metadata_exposure_for_request
+    from services.mesh.mesh_private_outbox import private_delivery_outbox
+
+    exposure = metadata_exposure_for_request(
+        request,
+        authenticated=True,
+    )
+    item = private_delivery_outbox.get_item(item_id, exposure=exposure)
+    if item is None:
+        raise HTTPException(status_code=404, detail="private_delivery_item_not_found")
+    return {"ok": True, "item": item}
+
+
 @router.post("/api/wormhole/private-delivery/{item_id}/action", dependencies=[Depends(require_local_operator)])
 @limiter.limit("30/minute")
 async def api_wormhole_private_delivery_action(

backend/scripts/convert_power_plants.py

@@ -29,7 +29,7 @@ def main() -> None:
         from services.network_utils import outbound_user_agent
         ua = outbound_user_agent("release-script-power-plants")
     except Exception:
-        ua = "Shadowbroker/0.9 (release-script-power-plants; +https://github.com/BigBodyCobain/Shadowbroker/issues)"
+        ua = "operator-release-script (purpose: power-plants)"
     req = urllib.request.Request(CSV_URL, headers={"User-Agent": ua})
     with urllib.request.urlopen(req, timeout=60) as resp:
         raw = resp.read().decode("utf-8")

backend/scripts/release_helper.py

@@ -167,6 +167,11 @@ def cmd_hash(args: argparse.Namespace) -> int:
     print("")
     print("Updater pin:")
     print(f"MESH_UPDATE_SHA256={digest}")
+    print("")
+    print("Release checklist:")
+    print("  - add this digest to SHA256SUMS.txt for the GitHub release")
+    print("  - add/update backend/data/release_digests.json for bundled updater verification")
+    print("  - keep MESH_UPDATE_SHA256 available as the operator override path")
     return 0 if asset_matches else 2
 
 

backend/scripts/scan-secrets.sh

@@ -92,18 +92,37 @@ SECRET_REGEX+='pypi-[0-9a-zA-Z-]{50,}'                        # PyPI token
 TEXT_FILES=$(grep -ivE '\.(png|jpg|jpeg|gif|ico|svg|woff2?|ttf|eot|pbf|zip|tar|gz|db|sqlite|xlsx|pdf|mp[34]|wav|ogg|webm|webp|avif)$' "$FILELIST" | grep -v 'scan-secrets\.sh$' || true)
 
 if [[ -n "$TEXT_FILES" ]]; then
+    # Known-public exclusions: lines matching `<host-or-ip> ssh-<algo> <key>`
+    # are SSH known_hosts entries — the host's PUBLIC fingerprint, which is
+    # by definition safe to commit (the whole point of pinning known_hosts
+    # is to publish the fingerprint widely so MITM is detectable). Filter
+    # these out before flagging the file.
+    KNOWN_HOSTS_LINE='^[[:space:]]*[a-zA-Z0-9._:,*-]+([[:space:]]+[a-zA-Z0-9._:,*-]+)?[[:space:]]+(ssh-rsa|ssh-ed25519|ssh-dss|ecdsa-sha2-nistp256|ecdsa-sha2-nistp384|ecdsa-sha2-nistp521)[[:space:]]+AAAA'
+
     # Use grep with file list, skip missing/binary, limit output
     CONTENT_HITS=$(echo "$TEXT_FILES" | xargs grep -lE "$SECRET_REGEX" 2>/dev/null || true)
     if [[ -n "$CONTENT_HITS" ]]; then
-        echo -e "\n${RED}BLOCKED: Embedded secrets/tokens found in:${NC}"
-        echo "$CONTENT_HITS" | while read -r f; do
-            echo -e "  ${RED}$f${NC}"
-            # Show first matching line for context
-            grep -nE "$SECRET_REGEX" "$f" 2>/dev/null | head -2 | while read -r line; do
-                echo -e "    ${YELLOW}$line${NC}"
-            done
-        done
-        FOUND=1
+        REAL_HITS=""
+        REAL_REPORT=""
+        while IFS= read -r f; do
+            [[ -z "$f" ]] && continue
+            # Re-grep this file, but filter out known_hosts-style lines.
+            FILE_HITS=$(grep -nE "$SECRET_REGEX" "$f" 2>/dev/null | grep -vE "$KNOWN_HOSTS_LINE" || true)
+            if [[ -n "$FILE_HITS" ]]; then
+                REAL_HITS+="$f"$'\n'
+                REAL_REPORT+="  ${RED}$f${NC}"$'\n'
+                # Show first 2 matching lines for context
+                while IFS= read -r line; do
+                    [[ -z "$line" ]] && continue
+                    REAL_REPORT+="    ${YELLOW}$line${NC}"$'\n'
+                done < <(echo "$FILE_HITS" | head -2)
+            fi
+        done <<< "$CONTENT_HITS"
+        if [[ -n "$REAL_HITS" ]]; then
+            echo -e "\n${RED}BLOCKED: Embedded secrets/tokens found in:${NC}"
+            echo -en "$REAL_REPORT"
+            FOUND=1
+        fi
     fi
 fi
 

backend/services/agent_shell_settings.py

@@ -0,0 +1,54 @@
+"""Operator settings for the embedded agent shell (working directory)."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_SETTINGS_FILE = Path(__file__).resolve().parent.parent / "data" / "agent_shell_settings.json"
+_LOCK = threading.Lock()
+
+
+def _default_working_directory() -> str:
+    explicit = str(os.environ.get("AGENT_SHELL_DEFAULT_CWD") or "").strip()
+    if explicit and os.path.isdir(explicit):
+        return explicit
+    home = str(os.environ.get("HOME") or "").strip()
+    if home and home != "/nonexistent" and os.path.isdir(home):
+        return home
+    return "/app"
+
+
+def get_agent_shell_settings() -> dict[str, Any]:
+    with _LOCK:
+        if not _SETTINGS_FILE.exists():
+            return {"working_directory": _default_working_directory()}
+        try:
+            payload = json.loads(_SETTINGS_FILE.read_text(encoding="utf-8"))
+        except Exception:
+            logger.warning("agent_shell_settings_unreadable")
+            return {"working_directory": _default_working_directory()}
+    cwd = str(payload.get("working_directory") or "").strip() or _default_working_directory()
+    return {"working_directory": cwd}
+
+
+def set_agent_shell_working_directory(path: str) -> dict[str, Any]:
+    normalized = str(path or "").strip()
+    if not normalized:
+        raise ValueError("working_directory_required")
+    resolved = os.path.abspath(os.path.expanduser(normalized))
+    if not os.path.isdir(resolved):
+        raise ValueError("working_directory_not_found")
+    with _LOCK:
+        _SETTINGS_FILE.parent.mkdir(parents=True, exist_ok=True)
+        _SETTINGS_FILE.write_text(
+            json.dumps({"working_directory": resolved}, indent=2) + "\n",
+            encoding="utf-8",
+        )
+    return {"working_directory": resolved}

backend/services/api_settings.py

@@ -51,6 +51,15 @@ API_REGISTRY = [
         "url": "https://aisstream.io/",
         "required": True,
     },
+    {
+        "id": "gfw_api_token",
+        "env_key": "GFW_API_TOKEN",
+        "name": "Global Fishing Watch",
+        "description": "Bearer token for Global Fishing Watch fishing-vessel activity events (Fishing Activity map layer). Free registration at globalfishingwatch.org.",
+        "category": "Maritime",
+        "url": "https://globalfishingwatch.org/our-apis/",
+        "required": False,
+    },
     {
         "id": "adsb_lol",
         "env_key": None,

backend/services/cctv_pipeline.py

@@ -17,6 +17,9 @@ _KNOWN_CCTV_MEDIA_HOST_ALIASES = {
     # Trusted upstream occasionally publishes a typo for this Georgia camera
     # host. Normalize it at ingest so the proxy and client stay consistent.
     "navigatos-c2c.dot.ga.gov": "navigator-c2c.dot.ga.gov",
+    # TravelIQ staging hosts occasionally appear in 511 catalog metadata.
+    "on.stage.traveliq.co": "511on.ca",
+    "ab.stage.traveliq.co": "511.alberta.ca",
 }
 
 _POINT_WKT_RE = re.compile(
@@ -40,6 +43,17 @@ def _normalize_cctv_media_url(raw_url: str) -> str:
     return urlunparse(parsed._replace(netloc=netloc))
 
 
+def _ensure_https_url(raw_url: str) -> str:
+    """Upgrade http:// media/catalog URLs to https:// at ingest time."""
+    candidate = _normalize_cctv_media_url(str(raw_url or "").strip())
+    if not candidate:
+        return ""
+    parsed = urlparse(candidate)
+    if parsed.scheme.lower() == "http":
+        return urlunparse(parsed._replace(scheme="https"))
+    return candidate
+
+
 def _looks_like_direct_cctv_media_url(url: str) -> bool:
     candidate = str(url or "").strip().lower()
     if not candidate.startswith(("http://", "https://")):
@@ -93,6 +107,165 @@ def _parse_wkt_point(raw_point: str) -> tuple[float | None, float | None]:
     return lat, lon
 
 
+def _fetch_traveliq_v2_cameras(
+    *,
+    api_url: str,
+    base_url: str,
+    id_prefix: str,
+    source_agency: str,
+) -> List[Dict[str, Any]]:
+    """Parse TravelIQ-style GET /api/v2/get/cameras feeds (Ontario, Alberta)."""
+    resp = fetch_with_curl(
+        api_url,
+        timeout=30,
+        headers={"Accept": "application/json"},
+    )
+    if not resp or resp.status_code != 200:
+        logger.error(
+            "%s CCTV fetch failed: HTTP %s",
+            source_agency,
+            resp.status_code if resp else "no response",
+        )
+        return []
+
+    data = resp.json()
+    if not isinstance(data, list):
+        return []
+
+    cameras: List[Dict[str, Any]] = []
+    for cam in data:
+        if not isinstance(cam, dict):
+            continue
+        try:
+            lat = float(cam.get("Latitude"))
+            lon = float(cam.get("Longitude"))
+        except (TypeError, ValueError):
+            continue
+
+        site_id = cam.get("Id")
+        location = str(cam.get("Location") or cam.get("Roadway") or "Camera")[:120]
+        views = cam.get("Views") or []
+        if not views:
+            continue
+
+        for view in views:
+            if not isinstance(view, dict):
+                continue
+            status = str(view.get("Status") or "enabled").strip().lower()
+            if status and status not in {"enabled", "active"}:
+                continue
+            media_url = _ensure_https_url(
+                urljoin(base_url, str(view.get("Url") or "").strip())
+            )
+            if not media_url:
+                continue
+            view_id = view.get("Id") or site_id
+            if site_id is None or view_id is None:
+                continue
+            label = str(view.get("Description") or location or "Camera")[:120]
+            cameras.append(
+                {
+                    "id": f"{id_prefix}-{site_id}-{view_id}",
+                    "source_agency": source_agency,
+                    "lat": lat,
+                    "lon": lon,
+                    "direction_facing": label,
+                    "media_url": media_url,
+                    "media_type": "image",
+                    "refresh_rate_seconds": 60,
+                }
+            )
+    return cameras
+
+
+def _fetch_511_datatables_cameras(
+    *,
+    list_url: str,
+    base_url: str,
+    id_prefix: str,
+    source_agency: str,
+    referer: str,
+    page_size: int = 500,
+) -> List[Dict[str, Any]]:
+    """Parse 511 DataTables POST /List/GetData/Cameras feeds (Georgia, Florida)."""
+    cameras: List[Dict[str, Any]] = []
+    start = 0
+    draw = 1
+    while True:
+        resp = fetch_with_curl(
+            list_url,
+            method="POST",
+            json_data={"draw": draw, "start": start, "length": page_size},
+            timeout=30,
+            headers={
+                "Accept": "application/json",
+                "Referer": referer,
+                "Origin": base_url.rstrip("/"),
+            },
+        )
+        if not resp or resp.status_code != 200:
+            logger.error(
+                "%s CCTV fetch failed: HTTP %s",
+                source_agency,
+                resp.status_code if resp else "no response",
+            )
+            break
+
+        data = resp.json()
+        rows = data.get("data") or []
+        if not rows:
+            break
+
+        for row in rows:
+            if not isinstance(row, dict):
+                continue
+            site_id = row.get("id") or row.get("DT_RowId")
+            location = row.get("location") or row.get("roadway") or source_agency
+            lat_lng = row.get("latLng") or {}
+            geography = lat_lng.get("geography") if isinstance(lat_lng, dict) else {}
+            lat, lon = _parse_wkt_point(
+                geography.get("wellKnownText") if isinstance(geography, dict) else ""
+            )
+            images = row.get("images") or []
+            image = next(
+                (
+                    candidate
+                    for candidate in images
+                    if str(candidate.get("imageUrl") or "").strip()
+                    and not bool(candidate.get("blocked"))
+                ),
+                None,
+            )
+            if not (site_id and image and lat is not None and lon is not None):
+                continue
+            media_url = _ensure_https_url(
+                urljoin(base_url, str(image.get("imageUrl") or "").strip())
+            )
+            if not media_url:
+                continue
+            cameras.append(
+                {
+                    "id": f"{id_prefix}-{site_id}",
+                    "source_agency": source_agency,
+                    "lat": lat,
+                    "lon": lon,
+                    "direction_facing": str(location)[:120],
+                    "media_url": media_url,
+                    "media_type": "image",
+                    "refresh_rate_seconds": 60,
+                }
+            )
+
+        start += len(rows)
+        draw += 1
+        total = int(data.get("recordsTotal") or 0)
+        if total and start >= total:
+            break
+        if not total and len(rows) < page_size:
+            break
+    return cameras
+
+
 def init_db():
     DB_PATH.parent.mkdir(parents=True, exist_ok=True)
     conn = sqlite3.connect(str(DB_PATH))
@@ -169,7 +342,7 @@ class BaseCCTVIngestor(ABC):
                         cam.get("lat"),
                         cam.get("lon"),
                         cam.get("direction_facing", "Unknown"),
-                        cam.get("media_url"),
+                        _ensure_https_url(cam.get("media_url", "")),
                         cam.get("media_type", _detect_media_type(cam.get("media_url", ""))),
                         cam.get("refresh_rate_seconds", 60),
                     ),
@@ -454,77 +627,14 @@ class WSDOTIngestor(BaseCCTVIngestor):
 class GeorgiaDOTIngestor(BaseCCTVIngestor):
     """Georgia cameras via the public 511GA list feed."""
 
-    URL = "https://511ga.org/List/GetData/Cameras"
-    BASE_URL = "https://511ga.org"
-    PAGE_SIZE = 500
-
     def fetch_data(self) -> List[Dict[str, Any]]:
-        cameras = []
-        start = 0
-        draw = 1
-        while True:
-            resp = fetch_with_curl(
-                self.URL,
-                method="POST",
-                json_data={"draw": draw, "start": start, "length": self.PAGE_SIZE},
-                timeout=30,
-                headers={
-                    "Accept": "application/json",
-                    "Referer": "https://511ga.org/cctv",
-                    "Origin": "https://511ga.org",
-                },
-            )
-            if not resp or resp.status_code != 200:
-                logger.error(
-                    "Georgia CCTV fetch failed: HTTP %s",
-                    resp.status_code if resp else "no response",
-                )
-                break
-            data = resp.json()
-            rows = data.get("data") or []
-            if not rows:
-                break
-            for row in rows:
-                site_id = row.get("id") or row.get("DT_RowId")
-                location = row.get("location") or row.get("roadway") or "GA Camera"
-                lat_lng = row.get("latLng") or {}
-                geography = lat_lng.get("geography") if isinstance(lat_lng, dict) else {}
-                lat, lon = _parse_wkt_point(geography.get("wellKnownText") if isinstance(geography, dict) else "")
-                images = row.get("images") or []
-                image = next(
-                    (
-                        candidate
-                        for candidate in images
-                        if str(candidate.get("imageUrl") or "").strip()
-                        and not bool(candidate.get("blocked"))
-                    ),
-                    None,
-                )
-                if not (site_id and image and lat is not None and lon is not None):
-                    continue
-                media_url = _normalize_cctv_media_url(
-                    urljoin(self.BASE_URL, str(image.get("imageUrl") or "").strip())
-                )
-                cameras.append(
-                    {
-                        "id": f"GDOT-{site_id}",
-                        "source_agency": "Georgia DOT",
-                        "lat": lat,
-                        "lon": lon,
-                        "direction_facing": str(location)[:120],
-                        "media_url": media_url,
-                        "media_type": "image",
-                        "refresh_rate_seconds": 60,
-                    }
-                )
-            start += len(rows)
-            draw += 1
-            total = int(data.get("recordsTotal") or 0)
-            if total and start >= total:
-                break
-            if not total and len(rows) < self.PAGE_SIZE:
-                break
-        return cameras
+        return _fetch_511_datatables_cameras(
+            list_url="https://511ga.org/List/GetData/Cameras",
+            base_url="https://511ga.org",
+            id_prefix="GDOT",
+            source_agency="Georgia DOT",
+            referer="https://511ga.org/cctv",
+        )
 
 
 class IllinoisDOTIngestor(BaseCCTVIngestor):
@@ -1009,17 +1119,72 @@ def _extract_img_src(html_fragment: str):
     return None
 
 
+class AsfinagIngestor(BaseCCTVIngestor):
+    """Austria ASFINAG motorway webcams (Osiris port)."""
+
+    API_URL = "https://odo.asfinag.at/odo/rest/sec/resource/001/json/webcams?language=atDE"
+    HEADERS = {
+        "User-Agent": "Shadowbroker-CCTV/1.0",
+        "Accept": "application/json",
+        "Referer": "https://www.asfinag.at/",
+        "Authorization": "Basic bWFwX3dpZGdldDp0ZWdkaXc=",
+    }
+
+    def fetch_data(self) -> List[Dict[str, Any]]:
+        try:
+            response = fetch_with_curl(self.API_URL, timeout=15, headers=self.HEADERS)
+            response.raise_for_status()
+            payload = response.json()
+        except Exception as exc:
+            logger.error("AsfinagIngestor: fetch failed: %s", exc)
+            return []
+        if not isinstance(payload, list):
+            return []
+        cameras: List[Dict[str, Any]] = []
+        for cam in payload:
+            cam_id = cam.get("wcs_id")
+            lat = cam.get("wgs84_lat")
+            lon = cam.get("wgs84_lon")
+            image_url = cam.get("url_campic")
+            if not cam_id or lat is None or lon is None or not image_url:
+                continue
+            if str(cam_id).startswith("Utinform"):
+                continue
+            label = cam.get("position_txt") or cam.get("direction_txt") or "ASFINAG Webcam"
+            secure_url = _ensure_https_url(image_url)
+            if not secure_url:
+                continue
+            cameras.append(
+                {
+                    "id": f"ASFINAG-{cam_id}",
+                    "source_agency": "ASFINAG Austria",
+                    "lat": float(lat),
+                    "lon": float(lon),
+                    "direction_facing": label,
+                    "media_url": secure_url,
+                    "media_type": "image",
+                    "refresh_rate_seconds": 300,
+                }
+            )
+        logger.info("AsfinagIngestor: parsed %s cameras", len(cameras))
+        return cameras
+
+
 class MadridCityIngestor(BaseCCTVIngestor):
     """Madrid City Hall traffic cameras from datos.madrid.es KML feed."""
 
-    KML_URL = "http://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml"
+    KML_URL = "https://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml"
+
+    def _fetch_kml(self):
+        response = fetch_with_curl(self.KML_URL, timeout=20)
+        response.raise_for_status()
+        return response
 
     def fetch_data(self) -> List[Dict[str, Any]]:
         import defusedxml.ElementTree as ET
 
         try:
-            response = fetch_with_curl(self.KML_URL, timeout=20)
-            response.raise_for_status()
+            response = self._fetch_kml()
         except Exception as e:
             logger.error(f"MadridCityIngestor: failed to fetch KML: {e}")
             return []
@@ -1055,6 +1220,9 @@ class MadridCityIngestor(BaseCCTVIngestor):
                 if desc_el is not None and desc_el.text:
                     image_url = _extract_img_src(desc_el.text)
 
+                if not image_url:
+                    continue
+                image_url = _ensure_https_url(image_url)
                 if not image_url:
                     continue
 
@@ -1076,6 +1244,153 @@ class MadridCityIngestor(BaseCCTVIngestor):
         return cameras
 
 
+class Ontario511Ingestor(BaseCCTVIngestor):
+    """Ontario highway cameras via 511on.ca TravelIQ API."""
+
+    def fetch_data(self) -> List[Dict[str, Any]]:
+        return _fetch_traveliq_v2_cameras(
+            api_url="https://511on.ca/api/v2/get/cameras",
+            base_url="https://511on.ca",
+            id_prefix="ON511",
+            source_agency="511 Ontario",
+        )
+
+
+class Alberta511Ingestor(BaseCCTVIngestor):
+    """Alberta highway cameras via 511 Alberta TravelIQ API."""
+
+    def fetch_data(self) -> List[Dict[str, Any]]:
+        return _fetch_traveliq_v2_cameras(
+            api_url="https://511.alberta.ca/api/v2/get/cameras",
+            base_url="https://511.alberta.ca",
+            id_prefix="AB511",
+            source_agency="511 Alberta",
+        )
+
+
+class Florida511Ingestor(BaseCCTVIngestor):
+    """Florida cameras via FL511 DataTables feed (~4,800 sites)."""
+
+    def fetch_data(self) -> List[Dict[str, Any]]:
+        return _fetch_511_datatables_cameras(
+            list_url="https://fl511.com/List/GetData/Cameras",
+            base_url="https://fl511.com",
+            id_prefix="FL511",
+            source_agency="Florida 511",
+            referer="https://fl511.com/",
+        )
+
+
+class AustraliaLiveTrafficIngestor(BaseCCTVIngestor):
+    """NSW / Australia live traffic cameras via Transport for NSW JSON feed."""
+
+    URL = "https://www.livetraffic.com/datajson/all-feeds-web.json"
+
+    def fetch_data(self) -> List[Dict[str, Any]]:
+        resp = fetch_with_curl(self.URL, timeout=35, headers={"Accept": "application/json"})
+        if not resp or resp.status_code != 200:
+            logger.error(
+                "Australia Live Traffic CCTV fetch failed: HTTP %s",
+                resp.status_code if resp else "no response",
+            )
+            return []
+
+        data = resp.json()
+        if not isinstance(data, list):
+            return []
+
+        cameras: List[Dict[str, Any]] = []
+        for item in data:
+            if not isinstance(item, dict) or item.get("eventType") != "liveCams":
+                continue
+            geometry = item.get("geometry") if isinstance(item.get("geometry"), dict) else {}
+            coords = geometry.get("coordinates") if isinstance(geometry.get("coordinates"), list) else []
+            if len(coords) < 2:
+                continue
+            try:
+                lon = float(coords[0])
+                lat = float(coords[1])
+            except (TypeError, ValueError):
+                continue
+
+            props = item.get("properties") if isinstance(item.get("properties"), dict) else {}
+            media_url = _ensure_https_url(str(props.get("href") or "").strip())
+            if not media_url:
+                continue
+
+            cam_id = str(item.get("path") or props.get("id") or len(cameras)).strip("/")
+            label = str(props.get("title") or props.get("headline") or "Australia Camera")[:120]
+            cameras.append(
+                {
+                    "id": f"AUS-{cam_id}",
+                    "source_agency": "NSW Live Traffic",
+                    "lat": lat,
+                    "lon": lon,
+                    "direction_facing": label,
+                    "media_url": media_url,
+                    "media_type": "image",
+                    "refresh_rate_seconds": 120,
+                }
+            )
+        logger.info("AustraliaLiveTrafficIngestor: parsed %s cameras", len(cameras))
+        return cameras
+
+
+class NetherlandsRWSIngestor(BaseCCTVIngestor):
+    """Netherlands Rijkswaterstaat cameras from legacy NDW open-data JSON.
+
+    The opendata.ndw.nu/cameras.json feed Osiris used is often offline; when
+  unavailable this ingestor returns an empty set and logs a warning.
+    """
+
+    URL = "https://opendata.ndw.nu/cameras.json"
+    MAX_CAMERAS = 1200
+
+    def fetch_data(self) -> List[Dict[str, Any]]:
+        resp = fetch_with_curl(self.URL, timeout=25, headers={"Accept": "application/json"})
+        if not resp or resp.status_code != 200:
+            logger.warning(
+                "Netherlands RWS cameras.json unavailable (HTTP %s) — "
+                "NDW retired this open-data endpoint; no cameras ingested",
+                resp.status_code if resp else "no response",
+            )
+            return []
+
+        data = resp.json()
+        if not isinstance(data, list):
+            return []
+
+        cameras: List[Dict[str, Any]] = []
+        for i, cam in enumerate(data[: self.MAX_CAMERAS]):
+            if not isinstance(cam, dict):
+                continue
+            lat = cam.get("lat") if cam.get("lat") is not None else cam.get("latitude")
+            lon = cam.get("lng") if cam.get("lng") is not None else cam.get("longitude")
+            media_url = _ensure_https_url(
+                str(cam.get("imageUrl") or cam.get("feed_url") or cam.get("url") or "").strip()
+            )
+            if lat is None or lon is None or not media_url:
+                continue
+            try:
+                lat_f, lon_f = float(lat), float(lon)
+            except (TypeError, ValueError):
+                continue
+            cameras.append(
+                {
+                    "id": f"NLRWS-{cam.get('id') or i}",
+                    "source_agency": "Rijkswaterstaat",
+                    "lat": lat_f,
+                    "lon": lon_f,
+                    "direction_facing": str(cam.get("name") or "Netherlands Camera")[:120],
+                    "media_url": media_url,
+                    "media_type": "image",
+                    "refresh_rate_seconds": 120,
+                }
+            )
+        logger.info("NetherlandsRWSIngestor: parsed %s cameras", len(cameras))
+        return cameras
+
+
 def _detect_media_type(url: str) -> str:
     """Detect the media type from a camera URL for proper frontend rendering."""
     if not url:
@@ -1094,29 +1409,40 @@ def _detect_media_type(url: str) -> str:
     return "image"
 
 
+def scheduled_cctv_ingestors() -> List[tuple["BaseCCTVIngestor", str]]:
+    """Canonical list of CCTV ingestors for startup, scheduler, and DB seeding."""
+    return [
+        (TFLJamCamIngestor(), "cctv_tfl"),
+        (LTASingaporeIngestor(), "cctv_lta"),
+        (AustinTXIngestor(), "cctv_atx"),
+        (NYCDOTIngestor(), "cctv_nyc"),
+        (CaltransIngestor(), "cctv_caltrans"),
+        (ColoradoDOTIngestor(), "cctv_codot"),
+        (WSDOTIngestor(), "cctv_wsdot"),
+        (GeorgiaDOTIngestor(), "cctv_gdot"),
+        (IllinoisDOTIngestor(), "cctv_idot"),
+        (MichiganDOTIngestor(), "cctv_mdot"),
+        (WindyWebcamsIngestor(), "cctv_windy"),
+        (DGTNationalIngestor(), "cctv_dgt"),
+        (MadridCityIngestor(), "cctv_madrid"),
+        (OSMTrafficCameraIngestor(), "cctv_osm"),
+        (AsfinagIngestor(), "cctv_asfinag"),
+        (OSMALPRCameraIngestor(), "cctv_osm_alpr"),
+        (Ontario511Ingestor(), "cctv_on511"),
+        (Alberta511Ingestor(), "cctv_ab511"),
+        (Florida511Ingestor(), "cctv_fl511"),
+        (AustraliaLiveTrafficIngestor(), "cctv_australia"),
+        (NetherlandsRWSIngestor(), "cctv_nl_rws"),
+    ]
+
+
 def run_all_ingestors():
     """Run all CCTV ingestors synchronously. Used for first-run DB seeding."""
-    ingestors = [
-        TFLJamCamIngestor(),
-        LTASingaporeIngestor(),
-        AustinTXIngestor(),
-        NYCDOTIngestor(),
-        CaltransIngestor(),
-        ColoradoDOTIngestor(),
-        WSDOTIngestor(),
-        GeorgiaDOTIngestor(),
-        IllinoisDOTIngestor(),
-        MichiganDOTIngestor(),
-        WindyWebcamsIngestor(),
-        OSMTrafficCameraIngestor(),
-        DGTNationalIngestor(),
-        MadridCityIngestor(),
-    ]
-    for ing in ingestors:
+    for ingestor, _name in scheduled_cctv_ingestors():
         try:
-            ing.ingest()
+            ingestor.ingest()
         except Exception as e:
-            logger.warning(f"Ingestor {ing.__class__.__name__} failed during seed: {e}")
+            logger.warning(f"Ingestor {ingestor.__class__.__name__} failed during seed: {e}")
 
 
 def get_all_cameras() -> List[Dict[str, Any]]:

backend/services/config.py

@@ -30,8 +30,13 @@ class Settings(BaseSettings):
     MESH_MQTT_INCLUDE_DEFAULT_ROOTS: bool = True
     MESH_RNS_ENABLED: bool = False
     MESH_ARTI_ENABLED: bool = False
+    # When true, trust wormhole_status.json ready bit if the child process is
+    # alive — avoids transport-tier flapping when /api/health probes time out
+    # under Tor load (common during live DM E2E).
+    MESH_WORMHOLE_TRUST_FILE_READY: bool = False
     MESH_ARTI_SOCKS_PORT: int = 9050
     MESH_RELAY_PEERS: str = ""
+    MESH_PUBLIC_PEER_URL: str = ""
     # Bootstrap seeds are discovery hints, not authoritative network roots.
     # Nodes promote healthy discovered peers from the store/manifest over time.
     MESH_BOOTSTRAP_SEED_PEERS: str = "http://gqpbunqbgtkcqilvclm3xrkt3zowjyl3s62kkktvojgvxzizamvbrqid.onion:8000"
@@ -42,7 +47,24 @@ class Settings(BaseSettings):
     MESH_INFONET_ALLOW_CLEARNET_SYNC: bool = False
     MESH_BOOTSTRAP_DISABLED: bool = False
     MESH_BOOTSTRAP_MANIFEST_PATH: str = "data/bootstrap_peers.json"
-    MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY: str = ""
+    # Public sb-testnet-0 fleet signer (participants). Seed operator holds the private key.
+    MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY: str = (
+        "ul1d0kj/ODPIp0OhHzX8eLAVXzJ3CVvzW1vn2IC6q3I="
+    )
+    MESH_BOOTSTRAP_SIGNER_PRIVATE_KEY: str = ""
+    # When true, empty MESH_PEER_PUSH_SECRET uses the public fleet HMAC for seed join/announce.
+    MESH_INFONET_FLEET_JOIN: bool = True
+    MESH_INFONET_FLEET_JOIN_DISABLED: bool = False
+    # Headless relay/seed compose: auto-enable Tor wormhole on startup so
+    # docker compose redeploys keep the fleet onion reachable.
+    MESH_INFONET_RELAY_AUTO_WORMHOLE: bool = False
+    MESH_INFONET_RELAY_AUTO_WORMHOLE_DISABLED: bool = False
+    MESH_BOOTSTRAP_SIGNER_ID: str = ""
+    MESH_PEER_REGISTRY_ENABLED: bool = False
+    MESH_PEER_REGISTRY_DISABLED: bool = False
+    MESH_PEER_REGISTRY_STALE_S: int = 604800
+    MESH_SWARM_MANIFEST_TTL_S: int = 14400
+    MESH_SWARM_MANIFEST_PULL_INTERVAL_S: int = 300
     MESH_NODE_MODE: str = "participant"
     MESH_SYNC_INTERVAL_S: int = 300
     MESH_SYNC_FAILURE_BACKOFF_S: int = 60

backend/services/data_fetcher.py

@@ -19,6 +19,7 @@ import concurrent.futures
 import json
 import math
 import os
+import random
 import threading
 import time
 from datetime import datetime, timedelta
@@ -75,6 +76,7 @@ from services.fetchers.infrastructure import (  # noqa: F401
     fetch_tinygs,
     fetch_psk_reporter,
 )
+from services.fetchers.road_corridor_sat import fetch_road_corridor_trends  # noqa: F401
 from services.fetchers.geo import (  # noqa: F401
     fetch_ships,
     fetch_airports,
@@ -99,6 +101,10 @@ from services.fetchers.crowdthreat import fetch_crowdthreat  # noqa: F401
 from services.fetchers.wastewater import fetch_wastewater  # noqa: F401
 from services.fetchers.sar_catalog import fetch_sar_catalog  # noqa: F401
 from services.fetchers.sar_products import fetch_sar_products  # noqa: F401
+from services.fetchers.malware import fetch_malware_threats  # noqa: F401
+from services.fetchers.telegram_osint import fetch_telegram_osint  # noqa: F401
+from services.fetchers.cyber_status import fetch_cyber_threats  # noqa: F401
+from services.scm.suppliers import fetch_scm_suppliers  # noqa: F401
 from services.ais_stream import prune_stale_vessels  # noqa: F401
 
 logger = logging.getLogger(__name__)
@@ -144,13 +150,18 @@ _STARTUP_HEAVY_REFRESH_DELAY_S = float(os.environ.get("SHADOWBROKER_STARTUP_HEAV
 _STARTUP_HEAVY_REFRESH_STARTED = False
 _STARTUP_HEAVY_REFRESH_LOCK = threading.Lock()
 _FETCH_WORKERS = int(os.environ.get("SHADOWBROKER_FETCH_WORKERS", "8"))
+_HEAVY_FETCH_WORKERS = int(os.environ.get("SHADOWBROKER_HEAVY_FETCH_WORKERS", "2"))
 _SLOW_FETCH_CONCURRENCY = int(os.environ.get("SHADOWBROKER_SLOW_FETCH_CONCURRENCY", "4"))
 _STARTUP_HEAVY_CONCURRENCY = int(os.environ.get("SHADOWBROKER_STARTUP_HEAVY_CONCURRENCY", "2"))
 
-# Shared thread pool — reused across all fetch cycles instead of creating/destroying per tick
+# Fast-tier pool (flights, ships, sigint, …). Slow / heavy work uses a separate pool
+# so Playwright, GDELT, CCTV ingest, etc. cannot starve the 60s refresh path (#375).
 _SHARED_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
     max_workers=max(2, _FETCH_WORKERS), thread_name_prefix="fetch"
 )
+_SLOW_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
+    max_workers=max(1, _HEAVY_FETCH_WORKERS), thread_name_prefix="fetch-slow"
+)
 
 
 def _cache_json_safe(value):
@@ -319,10 +330,49 @@ def seed_startup_caches() -> None:
 # ---------------------------------------------------------------------------
 # Scheduler & Orchestration
 # ---------------------------------------------------------------------------
+def _executor_for_task_label(label: str) -> concurrent.futures.ThreadPoolExecutor:
+    if label.startswith(("slow-tier", "startup-heavy")):
+        return _SLOW_EXECUTOR
+    return _SHARED_EXECUTOR
+
+
+def _run_task_with_health_on_executor(
+    executor: concurrent.futures.ThreadPoolExecutor,
+    func,
+    name: str | None = None,
+) -> None:
+    """Run a scheduled job on the given pool so it cannot starve fast-tier workers."""
+    task_name = name or getattr(func, "__name__", "task")
+    future = executor.submit(func)
+    start = time.perf_counter()
+    try:
+        future.result(timeout=_TASK_HARD_TIMEOUT_S)
+        duration = time.perf_counter() - start
+        from services.fetch_health import record_success
+
+        record_success(task_name, duration_s=duration)
+        if duration > _SLOW_FETCH_S:
+            logger.warning("task slow: %s took %.2fs", task_name, duration)
+    except concurrent.futures.TimeoutError:
+        future.cancel()
+        duration = time.perf_counter() - start
+        from services.fetch_health import record_failure
+
+        record_failure(task_name, error=TimeoutError(f"{task_name} timed out"), duration_s=duration)
+        logger.error("task timed out: %s (%.2fs)", task_name, duration)
+    except Exception as e:
+        duration = time.perf_counter() - start
+        from services.fetch_health import record_failure
+
+        record_failure(task_name, error=e, duration_s=duration)
+        logger.exception("task failed: %s", task_name)
+
+
 def _run_tasks(label: str, funcs: list, *, max_concurrency: int | None = None):
     """Run tasks concurrently and log any exceptions (do not fail silently)."""
     if not funcs:
         return
+    executor = _executor_for_task_label(label)
     if max_concurrency is None:
         if label.startswith("slow-tier"):
             max_concurrency = _SLOW_FETCH_CONCURRENCY
@@ -330,12 +380,13 @@ def _run_tasks(label: str, funcs: list, *, max_concurrency: int | None = None):
             max_concurrency = _STARTUP_HEAVY_CONCURRENCY
         else:
             max_concurrency = len(funcs)
-    max_concurrency = max(1, min(max_concurrency, len(funcs)))
+    pool_workers = getattr(executor, "_max_workers", len(funcs))
+    max_concurrency = max(1, min(max_concurrency, len(funcs), pool_workers))
 
     remaining_funcs = list(funcs)
     while remaining_funcs:
         batch, remaining_funcs = remaining_funcs[:max_concurrency], remaining_funcs[max_concurrency:]
-        futures = {_SHARED_EXECUTOR.submit(func): (func.__name__, time.perf_counter()) for func in batch}
+        futures = {executor.submit(func): (func.__name__, time.perf_counter()) for func in batch}
         _drain_task_futures(label, futures)
 
 
@@ -352,6 +403,13 @@ def _drain_task_futures(label: str, futures: dict):
             record_success(name, duration_s=duration)
             if duration > _SLOW_FETCH_S:
                 logger.warning(f"{label} task slow: {name} took {duration:.2f}s")
+        except concurrent.futures.TimeoutError:
+            future.cancel()
+            duration = time.perf_counter() - start
+            from services.fetch_health import record_failure
+
+            record_failure(name, error=TimeoutError(f"{name} timed out"), duration_s=duration)
+            logger.error("%s task timed out: %s (%.2fs)", label, name, duration)
         except Exception as e:
             duration = time.perf_counter() - start
             from services.fetch_health import record_failure
@@ -405,7 +463,6 @@ def update_slow_data():
     logger.info("Slow-tier data update starting...")
     slow_funcs = [
         fetch_news,
-        fetch_prediction_markets,
         fetch_earthquakes,
         fetch_firms_fires,
         fetch_firms_country_fires,
@@ -427,6 +484,9 @@ def update_slow_data():
         fetch_fishing_activity,
         fetch_power_plants,
         fetch_ukraine_air_raid_alerts,
+        fetch_malware_threats,
+        fetch_cyber_threats,
+        fetch_scm_suppliers,
     ]
     _run_tasks("slow-tier", slow_funcs)
     # Run correlation engine after all data is fresh
@@ -439,6 +499,12 @@ def update_slow_data():
             latest_data["correlations"] = correlations
     except Exception as e:
         logger.error("Correlation engine failed: %s", e)
+    try:
+        from analytics.integration import maybe_refresh_gt_analytics
+
+        maybe_refresh_gt_analytics()
+    except Exception as e:
+        logger.error("GT analytics refresh failed: %s", e)
     from services.fetchers._store import bump_data_version
     bump_data_version()
     _save_intel_startup_cache()
@@ -470,6 +536,15 @@ def _load_cctv_cache_for_startup() -> None:
         logger.warning("Startup CCTV cache load failed (non-fatal): %s", e)
 
 
+def _load_static_infrastructure_for_startup() -> None:
+    """Disk-backed reference layers — instant, no network."""
+    for func in (fetch_datacenters, fetch_military_bases, fetch_power_plants):
+        try:
+            func()
+        except Exception as e:
+            logger.warning("Startup static infrastructure load failed for %s: %s", func.__name__, e)
+
+
 def _run_delayed_startup_heavy_refresh() -> None:
     if _STARTUP_HEAVY_REFRESH_DELAY_S > 0:
         logger.info(
@@ -482,6 +557,7 @@ def _run_delayed_startup_heavy_refresh() -> None:
         "startup-heavy",
         [
             update_slow_data,
+            fetch_telegram_osint,
             fetch_volcanoes,
             fetch_viirs_change_nodes,
             fetch_unusual_whales,
@@ -520,6 +596,7 @@ def update_all_data(*, startup_mode: bool = False):
     logger.info("Full data update starting (parallel)...")
     # Preload Meshtastic map cache immediately (instant, from disk)
     seed_startup_caches()
+    _load_static_infrastructure_for_startup()
     with _data_lock:
         meshtastic_seeded = bool(latest_data.get("meshtastic_map_nodes"))
     if startup_mode:
@@ -596,22 +673,9 @@ def update_all_data(*, startup_mode: bool = False):
     # (the scheduled job also runs every 10 min for ongoing refresh).
     if startup_mode:
         try:
-            from services.cctv_pipeline import (
-                TFLJamCamIngestor, LTASingaporeIngestor, AustinTXIngestor,
-                NYCDOTIngestor, CaltransIngestor, ColoradoDOTIngestor,
-                WSDOTIngestor, GeorgiaDOTIngestor, IllinoisDOTIngestor,
-                MichiganDOTIngestor, WindyWebcamsIngestor, DGTNationalIngestor,
-                MadridCityIngestor, OSMTrafficCameraIngestor, get_all_cameras,
-            )
-            from services.cctv_pipeline import OSMALPRCameraIngestor
-            _startup_ingestors = [
-                TFLJamCamIngestor(), LTASingaporeIngestor(), AustinTXIngestor(),
-                NYCDOTIngestor(), CaltransIngestor(), ColoradoDOTIngestor(),
-                WSDOTIngestor(), GeorgiaDOTIngestor(), IllinoisDOTIngestor(),
-                MichiganDOTIngestor(), WindyWebcamsIngestor(), DGTNationalIngestor(),
-                MadridCityIngestor(), OSMTrafficCameraIngestor(),
-                OSMALPRCameraIngestor(),
-            ]
+            from services.cctv_pipeline import get_all_cameras, scheduled_cctv_ingestors
+
+            _startup_ingestors = [ing for ing, _name in scheduled_cctv_ingestors()]
             logger.info("Running CCTV ingest at startup (%d ingestors)...", len(_startup_ingestors))
             ingest_futures = {
                 _SHARED_EXECUTOR.submit(ing.ingest): ing.__class__.__name__
@@ -747,6 +811,49 @@ def start_scheduler():
         misfire_grace_time=120,
     )
 
+    # Telegram OSINT — hourly t.me/s channel scrape (kept off the 5-minute slow tier).
+    _telegram_interval_m = max(15, int(os.environ.get("TELEGRAM_OSINT_INTERVAL_MINUTES", "60")))
+
+    def _fetch_telegram_osint_with_gt():
+        fetch_telegram_osint()
+        try:
+            from analytics.integration import maybe_refresh_gt_analytics
+
+            maybe_refresh_gt_analytics()
+        except Exception as exc:
+            logger.error("GT analytics refresh after telegram failed: %s", exc)
+
+    _scheduler.add_job(
+        lambda: _run_task_with_health(_fetch_telegram_osint_with_gt, "fetch_telegram_osint"),
+        "interval",
+        minutes=_telegram_interval_m,
+        next_run_time=datetime.utcnow() + timedelta(seconds=45),
+        id="telegram_osint",
+        max_instances=1,
+        misfire_grace_time=600,
+    )
+
+    # Prediction markets — own jittered cadence (Polymarket/Kalshi clearnet egress).
+    # Kept off the fixed 5-minute slow tier so poll timing is less fingerprintable.
+    from services.fetchers.prediction_markets import fetch_prediction_markets
+
+    _pm_interval_m = max(5, int(os.environ.get("PREDICTION_MARKETS_INTERVAL_MINUTES", "7")))
+    _pm_jitter_s = max(0, int(os.environ.get("PREDICTION_MARKETS_SCHEDULER_JITTER_S", "240")))
+    _pm_initial_max_s = max(0, int(os.environ.get("PREDICTION_MARKETS_INITIAL_DELAY_MAX_S", "180")))
+    _pm_first_run = datetime.utcnow() + timedelta(
+        seconds=random.randint(30, max(30, _pm_initial_max_s))
+    )
+    _scheduler.add_job(
+        lambda: _run_task_with_health(fetch_prediction_markets, "fetch_prediction_markets"),
+        "interval",
+        minutes=_pm_interval_m,
+        jitter=_pm_jitter_s,
+        next_run_time=_pm_first_run,
+        id="prediction_markets",
+        max_instances=1,
+        misfire_grace_time=300,
+    )
+
     # Weather alerts — every 5 minutes (time-critical, separate from slow tier)
     _scheduler.add_job(
         lambda: _run_task_with_health(fetch_weather_alerts, "fetch_weather_alerts"),
@@ -843,16 +950,71 @@ def start_scheduler():
     )
 
     # GDELT — every 30 minutes (downloads 32 ZIP files per call, avoid rate limits)
+    def _fetch_gdelt_with_gt():
+        fetch_gdelt()
+        try:
+            from analytics.integration import maybe_refresh_gt_analytics
+
+            maybe_refresh_gt_analytics()
+        except Exception as exc:
+            logger.error("GT analytics refresh after gdelt failed: %s", exc)
+
     _scheduler.add_job(
-        lambda: _run_task_with_health(fetch_gdelt, "fetch_gdelt"),
+        lambda: _run_task_with_health_on_executor(_SLOW_EXECUTOR, _fetch_gdelt_with_gt, "fetch_gdelt"),
         "interval",
         minutes=30,
         id="gdelt",
         max_instances=1,
         misfire_grace_time=120,
     )
+
+    # GT analytics — Louvain herding/coordination clusters (feature-flagged).
+    def _recompute_gt_clusters():
+        try:
+            from analytics.integration import recompute_gt_herding_clusters
+
+            recompute_gt_herding_clusters()
+        except Exception as exc:
+            logger.error("GT Louvain recompute failed: %s", exc)
+
+    def _freeze_gt_weekly_snapshot():
+        try:
+            from analytics.integration import maybe_freeze_gt_weekly_snapshot
+
+            maybe_freeze_gt_weekly_snapshot()
+        except Exception as exc:
+            logger.error("GT rolling weekly freeze failed: %s", exc)
+
+    try:
+        from analytics.settings import get_gt_settings, gt_engine_operational
+
+        _gt_settings = get_gt_settings()
+        if gt_engine_operational():
+            _scheduler.add_job(
+                _recompute_gt_clusters,
+                "interval",
+                minutes=_gt_settings.louvain_interval_minutes,
+                id="gt_analytics_louvain",
+                max_instances=1,
+                misfire_grace_time=300,
+                next_run_time=datetime.utcnow() + timedelta(minutes=3),
+            )
+            _scheduler.add_job(
+                _freeze_gt_weekly_snapshot,
+                "cron",
+                day_of_week="mon",
+                hour=0,
+                minute=5,
+                id="gt_rolling_weekly_freeze",
+                max_instances=1,
+                misfire_grace_time=3600,
+            )
+    except Exception as exc:
+        logger.warning("GT Louvain scheduler not registered: %s", exc)
     _scheduler.add_job(
-        lambda: _run_task_with_health(update_liveuamap, "update_liveuamap"),
+        lambda: _run_task_with_health_on_executor(
+            _SLOW_EXECUTOR, update_liveuamap, "update_liveuamap"
+        ),
         "interval",
         minutes=30,
         id="liveuamap",
@@ -862,39 +1024,9 @@ def start_scheduler():
 
     # CCTV pipeline refresh — runs all ingestors, then refreshes in-memory data.
     # Delay the first run slightly so startup serves cached/DB-backed data first.
-    from services.cctv_pipeline import (
-        TFLJamCamIngestor,
-        LTASingaporeIngestor,
-        AustinTXIngestor,
-        NYCDOTIngestor,
-        CaltransIngestor,
-        ColoradoDOTIngestor,
-        WSDOTIngestor,
-        GeorgiaDOTIngestor,
-        IllinoisDOTIngestor,
-        MichiganDOTIngestor,
-        WindyWebcamsIngestor,
-        DGTNationalIngestor,
-        MadridCityIngestor,
-        OSMTrafficCameraIngestor,
-    )
+    from services.cctv_pipeline import scheduled_cctv_ingestors
 
-    _cctv_ingestors = [
-        (TFLJamCamIngestor(), "cctv_tfl"),
-        (LTASingaporeIngestor(), "cctv_lta"),
-        (AustinTXIngestor(), "cctv_atx"),
-        (NYCDOTIngestor(), "cctv_nyc"),
-        (CaltransIngestor(), "cctv_caltrans"),
-        (ColoradoDOTIngestor(), "cctv_codot"),
-        (WSDOTIngestor(), "cctv_wsdot"),
-        (GeorgiaDOTIngestor(), "cctv_gdot"),
-        (IllinoisDOTIngestor(), "cctv_idot"),
-        (MichiganDOTIngestor(), "cctv_mdot"),
-        (WindyWebcamsIngestor(), "cctv_windy"),
-        (DGTNationalIngestor(), "cctv_dgt"),
-        (MadridCityIngestor(), "cctv_madrid"),
-        (OSMTrafficCameraIngestor(), "cctv_osm"),
-    ]
+    _cctv_ingestors = scheduled_cctv_ingestors()
 
     def _run_cctv_ingest_cycle():
         from services.fetchers._store import is_any_active
@@ -913,7 +1045,9 @@ def start_scheduler():
             logger.warning(f"CCTV post-ingest refresh failed: {e}")
 
     _scheduler.add_job(
-        _run_cctv_ingest_cycle,
+        lambda: _run_task_with_health_on_executor(
+            _SLOW_EXECUTOR, _run_cctv_ingest_cycle, "cctv_ingest_cycle"
+        ),
         "interval",
         minutes=10,
         id="cctv_ingest",
@@ -983,6 +1117,16 @@ def start_scheduler():
         misfire_grace_time=600,
     )
 
+    # Sentinel-2 road corridor freight trends — daily (opt-in, heavy CDSE usage)
+    _scheduler.add_job(
+        lambda: _run_task_with_health(fetch_road_corridor_trends, "fetch_road_corridor_trends"),
+        "interval",
+        hours=24,
+        id="road_corridor_trends",
+        max_instances=1,
+        misfire_grace_time=3600,
+    )
+
     # FIMI disinformation index — every 12 hours (weekly editorial feed)
     _scheduler.add_job(
         lambda: _run_task_with_health(fetch_fimi, "fetch_fimi"),
@@ -993,9 +1137,9 @@ def start_scheduler():
         misfire_grace_time=600,
     )
 
-    # UAP sightings (NUFORC) — weekly on Mondays at 12:00 UTC. The layer is a
-    # rolling last-60-days digest; refreshing once a week is enough cadence
-    # for human-readable map exploration and keeps load on nuforc.org light.
+    # UAP sightings (NUFORC) — weekly Mondays 12:00 UTC. Rolling ~60-day window;
+    # each self-hosted install pulls live nuforc.org so operators see current
+    # reports (typically ~400–500 mappable pins). Disk cache TTL defaults to 7d.
     _scheduler.add_job(
         lambda: _run_task_with_health(
             lambda: fetch_uap_sightings(force_refresh=True),
@@ -1130,7 +1274,10 @@ def start_scheduler():
 def stop_scheduler():
     if _scheduler:
         _scheduler.shutdown(wait=False)
+    _SLOW_EXECUTOR.shutdown(wait=False, cancel_futures=True)
 
 
 def get_latest_data():
-    return get_latest_data_subset(*latest_data.keys())
+    from services.fetchers._store import get_latest_data_deepcopy_snapshot
+
+    return get_latest_data_deepcopy_snapshot()

backend/services/env_check.py

@@ -46,6 +46,7 @@ _CRITICAL_WARN = {
 
 _OPTIONAL = {
     "AIS_API_KEY": "AIS vessel streaming (ships layer will be empty without it)",
+    "GFW_API_TOKEN": "Global Fishing Watch fishing-vessel activity (fishing_activity layer)",
     "LTA_ACCOUNT_KEY": "Singapore LTA traffic cameras (CCTV layer)",
     "PUBLIC_API_KEY": "Optional client auth for public endpoints (recommended for exposed deployments)",
 }

backend/services/fetchers/_store.py

@@ -69,6 +69,12 @@ class DashboardData(TypedDict, total=False):
     sar_scenes: List[Dict[str, Any]]
     sar_anomalies: List[Dict[str, Any]]
     sar_aoi_coverage: List[Dict[str, Any]]
+    road_corridor_trends: Dict[str, Any]
+    malware_threats: Dict[str, Any]
+    cyber_threats: Dict[str, Any]
+    scm_suppliers: Dict[str, Any]
+    telegram_osint: Dict[str, Any]
+    gt_risk: Dict[str, Any]
 
 
 # In-memory store
@@ -119,6 +125,18 @@ latest_data: DashboardData = {
     "sar_scenes": [],
     "sar_anomalies": [],
     "sar_aoi_coverage": [],
+    "road_corridor_trends": {"updated_at": None, "corridors": []},
+    "malware_threats": {"threats": [], "total": 0, "timestamp": None},
+    "cyber_threats": {"threats": [], "stats": {}},
+    "scm_suppliers": {"suppliers": [], "total": 0, "critical_count": 0},
+    "telegram_osint": {"posts": [], "total": 0, "geolocated": 0, "timestamp": None},
+    "gt_risk": {
+        "enabled": False,
+        "heatmap": {"type": "FeatureCollection", "features": []},
+        "clusters": [],
+        "processed": 0,
+        "timestamp": None,
+    },
 }
 
 # Per-source freshness timestamps
@@ -230,27 +248,52 @@ _active_layers_version: int = 0
 def bump_active_layers_version() -> None:
     """Increment the active-layer version when frontend toggles change response shape."""
     global _active_layers_version
-    _active_layers_version += 1
+    with _data_lock:
+        _active_layers_version += 1
 
 
 def get_active_layers_version() -> int:
     """Return the current active-layer version (for ETag generation)."""
-    return _active_layers_version
+    with _data_lock:
+        return _active_layers_version
 
 
 def get_latest_data_subset(*keys: str) -> DashboardData:
     """Return a deep snapshot of only the requested top-level keys.
 
-    This avoids cloning the entire dashboard store for endpoints that only need
-    a small tier-specific subset.  Deep copy ensures callers cannot mutate
-    nested structures (e.g. individual flight dicts) and affect the live store.
+    Grabs references under the lock, then deep-copies outside it so fetcher
+    writers are not blocked for the duration of a large clone (#375).
     """
     with _data_lock:
-        snap: DashboardData = {}
-        for key in keys:
-            value = latest_data.get(key)
-            snap[key] = copy.deepcopy(value)
-        return snap
+        items = [(key, latest_data.get(key)) for key in keys]
+    snap: DashboardData = {}
+    for key, value in items:
+        snap[key] = copy.deepcopy(value)
+    return snap
+
+
+def get_latest_data_deepcopy_snapshot() -> DashboardData:
+    """Deep-copy the full dashboard for /api/health and legacy /api/live-data.
+
+    The per-value deepcopy runs OUTSIDE ``_data_lock`` so a large clone cannot
+    block fetcher writers (#375). The store contract is replace-don't-mutate,
+    but a writer that mutates a nested object in place (e.g. a live bridge
+    updating an entry that is also published in this store) can race the
+    deepcopy and raise ``RuntimeError: dictionary changed size during
+    iteration`` — surfacing a 500 on the health/live-data path. The racing
+    mutation window is tiny, so retry a few times rather than fail; a fresh
+    attempt almost always lands on a quiescent moment. Defense-in-depth on top
+    of fixing the offending writers, not a substitute for it.
+    """
+    attempts = 4
+    for attempt in range(attempts):
+        with _data_lock:
+            items = list(latest_data.items())
+        try:
+            return {key: copy.deepcopy(value) for key, value in items}
+        except RuntimeError:
+            if attempt == attempts - 1:
+                raise
 
 
 def get_latest_data_subset_refs(*keys: str) -> DashboardData:
@@ -320,6 +363,13 @@ active_layers: dict[str, bool] = {
     "ai_intel": True,
     "crowdthreat": False,
     "sar": True,
+    "road_corridor_trends": False,
+    "malware_c2": False,
+    "submarine_cables": False,
+    "scm_suppliers": False,
+    "cyber_threats": False,
+    "telegram_osint": True,
+    "gt_risk": False,
 }
 
 

backend/services/fetchers/aircraft_database.py

@@ -38,8 +38,6 @@ _S3_NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
 _REFRESH_INTERVAL_S = 5 * 24 * 3600
 _LIST_TIMEOUT_S = 30
 _DOWNLOAD_TIMEOUT_S = 600
-from services.network_utils import DEFAULT_USER_AGENT as _USER_AGENT
-
 _lock = threading.RLock()
 _aircraft_by_hex: dict[str, dict[str, str]] = {}
 _last_refresh = 0.0

backend/services/fetchers/cyber_status.py

@@ -0,0 +1,62 @@
+"""CISA KEV + cyber threat stats (Osiris port)."""
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+from typing import Any
+
+from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
+from services.network_utils import fetch_with_curl
+
+logger = logging.getLogger(__name__)
+
+
+def fetch_cyber_threats() -> dict[str, Any]:
+    if not is_any_active("cyber_threats"):
+        return latest_data.get("cyber_threats") or {"threats": [], "stats": {}}
+
+    results: dict[str, Any] = {"threats": [], "stats": {}, "timestamp": datetime.now(timezone.utc).isoformat()}
+    try:
+        resp = fetch_with_curl(
+            "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json",
+            timeout=15,
+        )
+        if resp.status_code == 200:
+            data = resp.json()
+            vulns = data.get("vulnerabilities") or []
+            results["stats"]["cisa_total"] = len(vulns)
+            now = datetime.now(timezone.utc)
+            recent = []
+            for v in vulns:
+                try:
+                    added = datetime.fromisoformat(v.get("dateAdded", "").replace("Z", "+00:00"))
+                    days = (now - added).total_seconds() / 86400
+                except Exception:
+                    continue
+                if days <= 30:
+                    recent.append(v)
+            recent = recent[:10]
+            results["threats"] = [
+                {
+                    "id": v.get("cveID"),
+                    "name": v.get("vulnerabilityName"),
+                    "vendor": v.get("vendorProject"),
+                    "product": v.get("product"),
+                    "severity": "CRITICAL",
+                    "date": v.get("dateAdded"),
+                    "due": v.get("dueDate"),
+                    "source": "CISA KEV",
+                }
+                for v in recent
+            ]
+    except Exception as exc:
+        logger.warning("CISA KEV fetch failed: %s", exc)
+
+    count = len(results["threats"])
+    results["stats"]["active_cves"] = count
+    results["stats"]["threat_level"] = "CRITICAL" if count >= 8 else "HIGH" if count >= 4 else "ELEVATED"
+
+    with _data_lock:
+        latest_data["cyber_threats"] = results
+    _mark_fresh("cyber_threats")
+    return results

backend/services/fetchers/earth_observation.py

@@ -692,7 +692,8 @@ _NUFORC_TILESET = "nuforc.cmm18aqea06bu1mmselhpnano-0ce5v"
 _NUFORC_TOKEN = os.environ.get("NUFORC_MAPBOX_TOKEN", "").strip()
 _NUFORC_RADIUS_M = 200_000  # 200 km query radius
 _NUFORC_LIMIT = 50  # max features per tilequery call
-_NUFORC_RECENT_DAYS = int(os.environ.get("NUFORC_RECENT_DAYS", "60"))
+# Rolling window shown on the map (~2 calendar months). Override via NUFORC_RECENT_DAYS.
+_NUFORC_RECENT_DAYS = max(1, int(os.environ.get("NUFORC_RECENT_DAYS", "60")))
 _NUFORC_HF_FALLBACK_LIMIT = max(25, int(os.environ.get("NUFORC_HF_FALLBACK_LIMIT", "250")))
 _NUFORC_HF_GEOCODE_LIMIT = max(25, int(os.environ.get("NUFORC_HF_GEOCODE_LIMIT", "150")))
 _NUFORC_GEOCODE_WORKERS = max(1, int(os.environ.get("NUFORC_GEOCODE_WORKERS", "1")))
@@ -700,6 +701,12 @@ _NUFORC_GEOCODE_WORKERS = max(1, int(os.environ.get("NUFORC_GEOCODE_WORKERS", "1
 # practice, so a 0.3s spacing keeps us well under any soft throttle while
 # still rebuilding a full 12-month window in ~10 minutes.
 _NUFORC_GEOCODE_SPACING_S = float(os.environ.get("NUFORC_GEOCODE_SPACING_S", "0.3"))
+# Disk cache TTL — match the weekly scheduler so restarts between fetches still
+# serve the same rolling 60-day snapshot without hammering nuforc.org daily.
+_NUFORC_CACHE_TTL_S = max(
+    3600,
+    int(os.environ.get("NUFORC_CACHE_TTL_HOURS", "168")) * 3600,
+)
 _NUFORC_DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"
 _NUFORC_SIGHTINGS_CACHE_FILE = _NUFORC_DATA_DIR / "nuforc_recent_sightings.json"
 _NUFORC_LOCATION_CACHE_FILE = _NUFORC_DATA_DIR / "nuforc_location_cache.json"
@@ -766,6 +773,35 @@ def _fetch_nuforc_tilequery(lng: float, lat: float) -> list[dict]:
     return []
 
 
+def _uap_cutoff_date_str() -> str:
+    return (datetime.utcnow() - timedelta(days=_NUFORC_RECENT_DAYS)).strftime("%Y-%m-%d")
+
+
+def _uap_sighting_date_str(sighting: dict) -> str | None:
+    """Normalize a sighting row to YYYY-MM-DD for window filtering."""
+    from services.fetchers.nuforc_enrichment import _parse_date
+
+    raw = str(sighting.get("date_time") or sighting.get("occurred") or "").strip()
+    if not raw:
+        return None
+    parsed = _parse_date(raw)
+    if parsed:
+        return parsed
+    if len(raw) >= 10 and raw[4] == "-" and raw[7] == "-":
+        return raw[:10]
+    return None
+
+
+def _filter_uap_sightings_recent(sightings: list[dict]) -> list[dict]:
+    """Drop anything outside the rolling NUFORC_RECENT_DAYS window."""
+    cutoff = _uap_cutoff_date_str()
+    return [
+        sighting
+        for sighting in sightings
+        if (_uap_sighting_date_str(sighting) or "") >= cutoff
+    ]
+
+
 def _parse_nuforc_tile_date(value: str) -> datetime | None:
     raw = str(value or "").strip()
     if not raw:
@@ -802,19 +838,41 @@ def _load_nuforc_sightings_cache(*, force_refresh: bool = False) -> list[dict] |
         built_dt = datetime.fromisoformat(built) if built else None
         if built_dt is None:
             return None
-        if (datetime.utcnow() - built_dt).total_seconds() > 86400:
+        if (datetime.utcnow() - built_dt).total_seconds() > _NUFORC_CACHE_TTL_S:
+            return None
+        if raw.get("cutoff_days") != _NUFORC_RECENT_DAYS:
+            logger.info(
+                "UAP sightings: cache cutoff_days mismatch (%s != %s); rebuilding",
+                raw.get("cutoff_days"),
+                _NUFORC_RECENT_DAYS,
+            )
             return None
         sightings = raw.get("sightings")
         if isinstance(sightings, list):
             if len(sightings) <= 0:
                 logger.info("UAP sightings: cache is fresh but empty; rebuilding")
                 return None
+            filtered = _filter_uap_sightings_recent(sightings)
+            if not filtered:
+                logger.warning(
+                    "UAP sightings: cache had %d rows but none within last %d days; rebuilding",
+                    len(sightings),
+                    _NUFORC_RECENT_DAYS,
+                )
+                return None
+            if len(filtered) < len(sightings):
+                logger.info(
+                    "UAP sightings: dropped %d stale cached rows outside %d-day window",
+                    len(sightings) - len(filtered),
+                    _NUFORC_RECENT_DAYS,
+                )
             logger.info(
-                "UAP sightings: loaded %d cached reports from %s",
-                len(sightings),
+                "UAP sightings: loaded %d cached reports from %s (within %d-day window)",
+                len(filtered),
                 built,
+                _NUFORC_RECENT_DAYS,
             )
-            return sightings
+            return filtered
     except Exception as e:
         logger.warning("UAP sightings: cache load error: %s", e)
     return None
@@ -828,6 +886,7 @@ def _save_nuforc_sightings_cache(sightings: list[dict]) -> None:
         _NUFORC_DATA_DIR.mkdir(parents=True, exist_ok=True)
         payload = {
             "built": datetime.utcnow().isoformat(),
+            "cutoff_days": _NUFORC_RECENT_DAYS,
             "count": len(sightings),
             "sightings": sightings,
         }
@@ -1035,27 +1094,128 @@ def _nuforc_months_for_window(days: int) -> list[str]:
     return months
 
 
-def _nuforc_fetch_month_live(yyyymm: str, cookie_jar: Path) -> list[dict]:
-    """Pull one month of NUFORC sightings via the live wpDataTables AJAX.
-
-    Returns a list of raw row dicts with the fields we care about:
-    id, occurred (YYYY-MM-DD), posted (YYYY-MM-DD), city, state, country,
-    shape_raw, summary, explanation. Empty list on any failure — caller
-    decides whether a failure is fatal.
-    """
+def _parse_nuforc_live_datatables_rows(raw_rows: list) -> list[dict]:
+    """Parse wpDataTables ``data`` array into normalized row dicts."""
     from services.fetchers.nuforc_enrichment import _parse_date
 
-    curl_bin = shutil.which("curl") or "curl"
+    out: list[dict] = []
+    for raw in raw_rows:
+        if not isinstance(raw, list) or len(raw) < 8:
+            continue
+        link_html = str(raw[0] or "")
+        occurred_raw = str(raw[1] or "")
+        city = str(raw[2] or "").strip()
+        state = str(raw[3] or "").strip()
+        country = str(raw[4] or "").strip()
+        shape_raw = (str(raw[5] or "").strip() or "Unknown")
+        summary = str(raw[6] or "").strip()
+        reported_raw = str(raw[7] or "")
+        explanation = str(raw[9] or "").strip() if len(raw) > 9 and raw[9] else ""
+
+        occurred_ymd = _parse_date(occurred_raw)
+        if not occurred_ymd:
+            continue
+        if not city and not state and not country:
+            continue
+
+        id_match = _NUFORC_LIVE_SIGHTING_ID_RE.search(link_html)
+        if id_match:
+            sighting_id = f"NUFORC-{id_match.group(1)}"
+        else:
+            digest = hashlib.sha1(
+                f"{occurred_ymd}|{city}|{state}|{summary}".encode("utf-8", "ignore")
+            ).hexdigest()[:12]
+            sighting_id = f"NUFORC-{digest}"
+
+        if summary and len(summary) > 280:
+            summary = summary[:277] + "..."
+        if not summary:
+            summary = "Sighting reported"
+
+        out.append({
+            "id": sighting_id,
+            "occurred": occurred_ymd,
+            "posted": _parse_date(reported_raw) or occurred_ymd,
+            "city": city,
+            "state": state,
+            "country": country,
+            "shape_raw": shape_raw,
+            "summary": summary,
+            "explanation": explanation,
+        })
+    return out
+
+
+def _nuforc_fetch_month_live_requests(yyyymm: str) -> list[dict]:
+    """Live NUFORC month fetch via requests (Windows-safe when curl is disabled)."""
+    import requests
+
     index_url = _NUFORC_LIVE_INDEX_URL.format(yyyymm=yyyymm)
     ajax_url = _NUFORC_LIVE_AJAX_URL.format(yyyymm=yyyymm)
-
-    if not external_curl_fallback_enabled():
+    headers = {"User-Agent": _nuforc_live_user_agent()}
+    session = requests.Session()
+    session.headers.update(headers)
+    try:
+        index_res = session.get(index_url, timeout=60)
+    except requests.RequestException as e:
+        logger.warning("NUFORC live (requests): index fetch failed for %s: %s", yyyymm, e)
+        return []
+    if index_res.status_code != 200 or not index_res.text:
         logger.warning(
-            "NUFORC live: external curl disabled on Windows for %s; "
-            "set SHADOWBROKER_ENABLE_WINDOWS_CURL_FALLBACK=1 to opt in.",
+            "NUFORC live (requests): index HTTP %s for %s",
+            index_res.status_code,
             yyyymm,
         )
         return []
+    nonce_match = _NUFORC_LIVE_NONCE_RE.search(index_res.text)
+    if not nonce_match:
+        logger.warning("NUFORC live (requests): wdtNonce not found for %s", yyyymm)
+        return []
+    nonce = nonce_match.group(1)
+    post_data = (
+        "draw=1"
+        "&columns%5B0%5D%5Bdata%5D=0&columns%5B0%5D%5Bsearchable%5D=true&columns%5B0%5D%5Borderable%5D=false"
+        "&columns%5B1%5D%5Bdata%5D=1&columns%5B1%5D%5Bsearchable%5D=true&columns%5B1%5D%5Borderable%5D=true"
+        "&order%5B0%5D%5Bcolumn%5D=1&order%5B0%5D%5Bdir%5D=desc"
+        "&start=0&length=-1"
+        "&search%5Bvalue%5D=&search%5Bregex%5D=false"
+        f"&wdtNonce={nonce}"
+    )
+    try:
+        ajax_res = session.post(
+            ajax_url,
+            data=post_data,
+            headers={
+                **headers,
+                "Referer": index_url,
+                "X-Requested-With": "XMLHttpRequest",
+                "Content-Type": "application/x-www-form-urlencoded",
+            },
+            timeout=120,
+        )
+    except requests.RequestException as e:
+        logger.warning("NUFORC live (requests): ajax failed for %s: %s", yyyymm, e)
+        return []
+    if ajax_res.status_code != 200 or not ajax_res.text:
+        logger.warning(
+            "NUFORC live (requests): ajax HTTP %s for %s",
+            ajax_res.status_code,
+            yyyymm,
+        )
+        return []
+    try:
+        payload = ajax_res.json()
+    except json.JSONDecodeError as e:
+        logger.warning("NUFORC live (requests): ajax JSON decode failed for %s: %s", yyyymm, e)
+        return []
+    return _parse_nuforc_live_datatables_rows(payload.get("data") or [])
+
+
+def _nuforc_fetch_month_live_curl(yyyymm: str, cookie_jar: Path) -> list[dict]:
+    """Pull one month of NUFORC sightings via curl + wpDataTables AJAX."""
+    curl_bin = shutil.which("curl") or "curl"
+    index_url = _NUFORC_LIVE_INDEX_URL.format(yyyymm=yyyymm)
+    ajax_url = _NUFORC_LIVE_AJAX_URL.format(yyyymm=yyyymm)
 
     # Step 1: GET the month index to capture session cookies + fresh nonce.
     try:
@@ -1125,65 +1285,27 @@ def _nuforc_fetch_month_live(yyyymm: str, cookie_jar: Path) -> list[dict]:
         logger.warning("NUFORC live: ajax JSON decode failed for %s: %s", yyyymm, e)
         return []
 
-    raw_rows = payload.get("data") or []
-    out: list[dict] = []
-    for raw in raw_rows:
-        if not isinstance(raw, list) or len(raw) < 8:
-            continue
-        link_html = str(raw[0] or "")
-        occurred_raw = str(raw[1] or "")
-        city = str(raw[2] or "").strip()
-        state = str(raw[3] or "").strip()
-        country = str(raw[4] or "").strip()
-        shape_raw = (str(raw[5] or "").strip() or "Unknown")
-        summary = str(raw[6] or "").strip()
-        reported_raw = str(raw[7] or "")
-        explanation = str(raw[9] or "").strip() if len(raw) > 9 and raw[9] else ""
+    return _parse_nuforc_live_datatables_rows(payload.get("data") or [])
 
-        occurred_ymd = _parse_date(occurred_raw)
-        if not occurred_ymd:
-            continue
-        if not city and not state and not country:
-            continue
 
-        id_match = _NUFORC_LIVE_SIGHTING_ID_RE.search(link_html)
-        if id_match:
-            sighting_id = f"NUFORC-{id_match.group(1)}"
-        else:
-            digest = hashlib.sha1(
-                f"{occurred_ymd}|{city}|{state}|{summary}".encode("utf-8", "ignore")
-            ).hexdigest()[:12]
-            sighting_id = f"NUFORC-{digest}"
-
-        if summary and len(summary) > 280:
-            summary = summary[:277] + "..."
-        if not summary:
-            summary = "Sighting reported"
-
-        out.append({
-            "id": sighting_id,
-            "occurred": occurred_ymd,
-            "posted": _parse_date(reported_raw) or occurred_ymd,
-            "city": city,
-            "state": state,
-            "country": country,
-            "shape_raw": shape_raw,
-            "summary": summary,
-            "explanation": explanation,
-        })
-    return out
+def _nuforc_fetch_month_live(yyyymm: str, cookie_jar: Path) -> list[dict]:
+    """Pull one month of NUFORC sightings via live wpDataTables AJAX."""
+    if external_curl_fallback_enabled():
+        rows = _nuforc_fetch_month_live_curl(yyyymm, cookie_jar)
+        if rows:
+            return rows
+    return _nuforc_fetch_month_live_requests(yyyymm)
 
 
 def _build_recent_uap_sightings() -> list[dict]:
-    """Build the rolling 1-year UAP sightings layer from live NUFORC data.
+    """Build the rolling UAP sightings layer from live NUFORC data.
 
     Hits nuforc.org's public sub-index once per month in the window, drops
     anything outside the exact day-precision cutoff, dedupes by sighting id,
     geocodes city+state via the existing location cache, and returns rows
     keyed to the same schema the frontend already renders.
     """
-    cutoff_dt = datetime.utcnow() - timedelta(days=_NUFORC_RECENT_DAYS)
-    cutoff_str = cutoff_dt.strftime("%Y-%m-%d")
+    cutoff_str = _uap_cutoff_date_str()
     months = _nuforc_months_for_window(_NUFORC_RECENT_DAYS)
 
     try:
@@ -1530,11 +1652,12 @@ def _build_uap_sightings_from_hf_mirror() -> list[dict]:
 
 @with_retry(max_retries=1, base_delay=5)
 def fetch_uap_sightings(*, force_refresh: bool = False):
-    """Fetch last-year UAP sightings from NUFORC.
+    """Fetch rolling-window UAP sightings from live NUFORC.
 
-    Startup reads the cached daily snapshot when it is still fresh. The daily
-    scheduler forces a rebuild so this layer updates once per day instead of
-    churning continuously.
+    Startup reads the cached snapshot when still within NUFORC_CACHE_TTL_HOURS
+    (default 168h / one week). The weekly scheduler forces a rebuild so every
+    install refreshes the same ~60-day layer without daily load on nuforc.org.
+    Operators can also POST /api/refresh (admin) to pull immediately.
     """
     from services.fetchers._store import is_any_active
 
@@ -1567,12 +1690,16 @@ def fetch_uap_sightings(*, force_refresh: bool = False):
                 live_error,
             )
 
+    if sightings:
+        sightings = _filter_uap_sightings_recent(sightings)
+
     with _data_lock:
         latest_data["uap_sightings"] = sightings or []
     if sightings:
         _mark_fresh("uap_sightings")
     return
 
+    # Unreachable legacy Mapbox tilequery path (kept for reference).
     cutoff = datetime.utcnow() - timedelta(days=_NUFORC_RECENT_DAYS)
 
     # Query the grid concurrently (up to 8 threads)

backend/services/fetchers/geo.py

@@ -20,17 +20,9 @@ def _env_flag(name: str) -> str:
 
 
 def liveuamap_scraper_enabled() -> bool:
-    """Return whether the Playwright-based LiveUAMap scraper should run.
+    from services.liveuamap_settings import liveuamap_scraper_enabled as _enabled
 
-    It is useful enrichment, but it starts a browser/Node driver and must not be
-    allowed to destabilize Windows local startup.
-    """
-    setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
-    if setting in {"1", "true", "yes", "on"}:
-        return True
-    if setting in {"0", "false", "no", "off"}:
-        return False
-    return os.name != "nt"
+    return _enabled()
 
 
 # ---------------------------------------------------------------------------
@@ -210,10 +202,17 @@ def update_liveuamap():
     if not is_any_active("global_incidents"):
         return
     if not liveuamap_scraper_enabled():
-        logger.info(
-            "Liveuamap scraper disabled for this runtime; set "
-            "SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=1 to opt in."
-        )
+        from services.liveuamap_settings import liveuamap_requires_ui_opt_in
+
+        if liveuamap_requires_ui_opt_in():
+            logger.info(
+                "Liveuamap scraper disabled: enable Global Incidents in the UI to "
+                "consent, or set SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=1."
+            )
+        else:
+            logger.info(
+                "Liveuamap scraper disabled; set SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=1 to opt in."
+            )
         return
     logger.info("Running scheduled Liveuamap scraper...")
     try:
@@ -279,6 +278,16 @@ _FISHING_FETCH_INTERVAL_S = 3600  # once per hour — GFW data has ~5 day lag
 _last_fishing_fetch_ts: float = 0.0
 
 
+def _gfw_int_env(name: str, default: int, *, minimum: int = 1, maximum: int | None = None) -> int:
+    try:
+        value = int(os.environ.get(name, str(default)) or default)
+    except (TypeError, ValueError):
+        value = default
+    if maximum is not None:
+        value = min(maximum, value)
+    return max(minimum, value)
+
+
 @with_retry(max_retries=1, base_delay=5)
 def fetch_fishing_activity():
     """Fetch recent fishing events from Global Fishing Watch (~5 day lag)."""
@@ -301,10 +310,16 @@ def fetch_fishing_activity():
     try:
         import datetime as _dt
 
+        # GFW publishes with ~5 day lag; windows shorter than ~7 days often return 0 events.
+        lookback_days = _gfw_int_env("GFW_EVENTS_LOOKBACK_DAYS", 7, minimum=1, maximum=14)
+        max_pages = _gfw_int_env("GFW_EVENTS_MAX_PAGES", 10, minimum=1, maximum=100)
+        timeout_s = _gfw_int_env("GFW_EVENTS_TIMEOUT_S", 90, minimum=30, maximum=180)
         _end = _dt.date.today().isoformat()
-        _start = (_dt.date.today() - _dt.timedelta(days=7)).isoformat()
-        page_size = max(1, int(os.environ.get("GFW_EVENTS_PAGE_SIZE", "500") or "500"))
+        _start = (_dt.date.today() - _dt.timedelta(days=lookback_days)).isoformat()
+        page_size = _gfw_int_env("GFW_EVENTS_PAGE_SIZE", 500, minimum=1, maximum=1000)
         offset = 0
+        pages_fetched = 0
+        total_available: int | None = None
         seen_offsets: set[int] = set()
         seen_ids: set[str] = set()
         headers = {"Authorization": f"Bearer {token}"}
@@ -325,7 +340,7 @@ def fetch_fishing_activity():
                 }
             )
             url = f"https://gateway.api.globalfishingwatch.org/v3/events?{query}"
-            response = fetch_with_curl(url, timeout=30, headers=headers)
+            response = fetch_with_curl(url, timeout=timeout_s, headers=headers)
             if response.status_code != 200:
                 logger.warning(
                     "Fishing activity fetch failed at offset=%s: HTTP %s",
@@ -335,10 +350,16 @@ def fetch_fishing_activity():
                 break
 
             payload = response.json() or {}
+            if total_available is None:
+                try:
+                    total_available = int(payload.get("total")) if payload.get("total") is not None else None
+                except (TypeError, ValueError):
+                    total_available = None
             entries = payload.get("entries", [])
             if not entries:
                 break
 
+            pages_fetched += 1
             added_this_page = 0
             for e in entries:
                 pos = e.get("position", {})
@@ -373,6 +394,15 @@ def fetch_fishing_activity():
             if len(entries) < page_size:
                 break
 
+            if pages_fetched >= max_pages:
+                logger.info(
+                    "Fishing activity: capped at %s pages (%s events fetched; GFW total=%s)",
+                    max_pages,
+                    len(events),
+                    total_available if total_available is not None else "unknown",
+                )
+                break
+
             next_offset = payload.get("nextOffset")
             if next_offset is None:
                 next_offset = (payload.get("pagination") or {}).get("nextOffset")

backend/services/fetchers/infrastructure.py

@@ -235,11 +235,11 @@ _DC_GEOCODED_PATH = Path(__file__).parent.parent.parent / "data" / "datacenters_
 
 
 def fetch_datacenters():
-    """Load geocoded data centers (5K+ street-level precise locations)."""
-    from services.fetchers._store import is_any_active
+    """Load geocoded data centers (5K+ street-level precise locations).
 
-    if not is_any_active("datacenters"):
-        return
+    Always loads from disk; /api/live-data/slow gates the payload on the
+    datacenters layer toggle so enabling the layer can render immediately.
+    """
     dcs = []
     try:
         if not _DC_GEOCODED_PATH.exists():

backend/services/fetchers/malware.py

@@ -0,0 +1,107 @@
+"""Malware C2 / URLhaus feed (abuse.ch, Osiris port)."""
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+from typing import Any
+
+from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
+from services.network_utils import fetch_with_curl
+
+logger = logging.getLogger(__name__)
+
+COUNTRY_CENTROIDS: dict[str, tuple[float, float]] = {
+    "AF": (65, 33), "AL": (20, 41), "DZ": (3, 28), "AR": (-64, -34), "AU": (134, -25),
+    "AT": (14, 47.5), "BE": (4, 50.8), "BR": (-51, -10), "CA": (-96, 62), "CN": (105, 35),
+    "DE": (10, 51), "FR": (2, 46), "GB": (-2, 54), "IN": (79, 22), "IR": (53, 32),
+    "IT": (12.5, 42.8), "JP": (138, 36), "KR": (128, 36), "MX": (-102, 23.5), "NL": (5.5, 52.5),
+    "PL": (19.5, 52), "RU": (100, 60), "SG": (103.8, 1.35), "TW": (121, 23.7), "UA": (32, 49),
+    "US": (-97, 38), "VN": (106, 16),
+}
+
+
+def fetch_malware_threats() -> list[dict[str, Any]]:
+    if not is_any_active("malware_c2"):
+        return latest_data.get("malware_threats") or []
+
+    threats: list[dict[str, Any]] = []
+    threat_id = 0
+
+    try:
+        resp = fetch_with_curl(
+            "https://feodotracker.abuse.ch/downloads/ipblocklist.json",
+            timeout=10,
+            headers={"User-Agent": "Shadowbroker/1.0", "Accept": "application/json"},
+        )
+        if resp.status_code == 200:
+            entries = resp.json()
+            if not isinstance(entries, list):
+                entries = []
+            for entry in entries[:200]:
+                cc = entry.get("country")
+                if not cc or cc not in COUNTRY_CENTROIDS:
+                    continue
+                lng, lat = COUNTRY_CENTROIDS[cc]
+                j_lng = ((threat_id * 173.7) % 200 - 100) / 100 * 4
+                j_lat = ((threat_id * 293.1) % 200 - 100) / 100 * 4
+                threats.append(
+                    {
+                        "id": f"feodo-{threat_id}",
+                        "lat": lat + j_lat,
+                        "lng": lng + j_lng,
+                        "ip": entry.get("ip_address") or "unknown",
+                        "port": entry.get("dst_port") or 0,
+                        "malware": entry.get("malware") or "unknown",
+                        "status": entry.get("status") or "active",
+                        "first_seen": entry.get("first_seen"),
+                        "last_online": entry.get("last_online"),
+                        "country": cc,
+                        "threat_type": "botnet_c2",
+                    }
+                )
+                threat_id += 1
+    except Exception as exc:
+        logger.warning("Feodo fetch failed: %s", exc)
+
+    try:
+        resp = fetch_with_curl(
+            "https://urlhaus-api.abuse.ch/v1/urls/recent/limit/100/",
+            timeout=8,
+        )
+        if resp.status_code == 200:
+            urls = (resp.json() or {}).get("urls") or []
+            for u in urls:
+                cc = u.get("country")
+                if not cc or cc not in COUNTRY_CENTROIDS:
+                    cc = next(iter(COUNTRY_CENTROIDS))
+                lng, lat = COUNTRY_CENTROIDS[cc]
+                j_lng = ((threat_id * 137.3) % 200 - 100) / 100 * 5
+                j_lat = ((threat_id * 211.7) % 200 - 100) / 100 * 5
+                threats.append(
+                    {
+                        "id": f"urlhaus-{threat_id}",
+                        "lat": lat + j_lat,
+                        "lng": lng + j_lng,
+                        "ip": u.get("host") or "unknown",
+                        "port": 0,
+                        "malware": ", ".join(u.get("tags") or []) or u.get("threat") or "malware",
+                        "status": u.get("url_status") or "online",
+                        "first_seen": u.get("dateadded"),
+                        "country": cc,
+                        "threat_type": "malware_url",
+                    }
+                )
+                threat_id += 1
+    except Exception as exc:
+        logger.debug("URLhaus supplement failed: %s", exc)
+
+    payload = {
+        "threats": threats,
+        "total": len(threats),
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "source": "abuse.ch Feodo Tracker + URLhaus",
+    }
+    with _data_lock:
+        latest_data["malware_threats"] = payload
+    _mark_fresh("malware_threats")
+    return threats

backend/services/fetchers/meshtastic_map.py

@@ -188,8 +188,8 @@ def fetch_meshtastic_nodes():
         callsign = ""
 
     send_callsign_header = str(
-        _os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "true")
-    ).strip().lower() not in {"0", "false", "no", "off", ""}
+        _os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "false")
+    ).strip().lower() in {"1", "true", "yes", "on"}
 
     # Round 7a: outbound_user_agent already includes the per-install handle.
     # The optional Meshtastic callsign is appended as additional context so

backend/services/fetchers/news.py

@@ -158,21 +158,26 @@ _KEYWORD_COORDS = {
 _SORTED_KEYWORDS = sorted(_KEYWORD_COORDS.items(), key=lambda x: len(x[0]), reverse=True)
 
 
+def resolve_coords_match(text: str) -> tuple[tuple[float, float], str] | None:
+    """Return ((lat, lng), matched_keyword) for the most specific keyword hit."""
+    padded_text = f" {text} "
+    for kw, coords in _SORTED_KEYWORDS:
+        if kw.startswith(" ") or kw.endswith(" "):
+            if kw in padded_text:
+                return coords, kw
+        elif re.search(r"\b" + re.escape(kw) + r"\b", text):
+            return coords, kw
+    return None
+
+
 def _resolve_coords(text: str) -> tuple[float, float] | None:
     """Return (lat, lng) for the most specific keyword match, or None.
 
     Longer keywords are tried first. Space-padded keywords (" us ", " uk ")
     use substring matching on padded text; all others use word-boundary regex.
     """
-    padded_text = f" {text} "
-    for kw, coords in _SORTED_KEYWORDS:
-        if kw.startswith(" ") or kw.endswith(" "):
-            if kw in padded_text:
-                return coords
-        else:
-            if re.search(r'\b' + re.escape(kw) + r'\b', text):
-                return coords
-    return None
+    match = resolve_coords_match(text)
+    return match[0] if match else None
 
 
 @with_retry(max_retries=1, base_delay=2)

backend/services/fetchers/prediction_markets.py

@@ -9,6 +9,7 @@ import json
 import logging
 import math
 import os
+import random
 import threading
 import time
 from urllib.parse import urlencode
@@ -21,23 +22,34 @@ _prev_probabilities: dict[str, float] = {}
 _market_cache = TTLCache(maxsize=1, ttl=300)
 _POLYMARKET_PAGE_DELAY_S = float(os.environ.get("MESH_POLYMARKET_PAGE_DELAY_S", "0.02"))
 _KALSHI_PAGE_DELAY_S = float(os.environ.get("MESH_KALSHI_PAGE_DELAY_S", "0.08"))
+_POLYMARKET_PAGE_DELAY_JITTER_S = float(os.environ.get("MESH_POLYMARKET_PAGE_DELAY_JITTER_S", "0.08"))
+_KALSHI_PAGE_DELAY_JITTER_S = float(os.environ.get("MESH_KALSHI_PAGE_DELAY_JITTER_S", "0.2"))
+# Random delay before each full Polymarket+Kalshi cycle (decorrelates from other slow-tier jobs).
+_PRE_FETCH_JITTER_S = float(os.environ.get("PREDICTION_MARKETS_PRE_FETCH_JITTER_S", "90"))
+# Random pause between finishing Polymarket pagination and starting Kalshi.
+_PROVIDER_GAP_JITTER_S = float(os.environ.get("PREDICTION_MARKETS_PROVIDER_GAP_JITTER_S", "45"))
 _provider_pace_lock = threading.Lock()
 _provider_last_request_at: dict[str, float] = {}
 
 
 def prediction_markets_fetch_enabled() -> bool:
-    """Return True only when the operator explicitly opts into Polymarket/Kalshi pulls."""
-    return str(os.environ.get("PREDICTION_MARKETS_ENABLED", "")).strip().lower() in {
-        "1",
-        "true",
-        "yes",
-        "on",
-    }
+    """Return True when UI opt-in or PREDICTION_MARKETS_ENABLED enables pulls."""
+    from services.prediction_markets_settings import prediction_markets_fetch_enabled as _enabled
+
+    return _enabled()
 
 
 def _pace_provider(provider: str, min_interval_s: float) -> None:
     if min_interval_s <= 0:
         return
+    jitter_s = (
+        _POLYMARKET_PAGE_DELAY_JITTER_S
+        if provider == "polymarket"
+        else _KALSHI_PAGE_DELAY_JITTER_S
+        if provider == "kalshi"
+        else 0.0
+    )
+    min_interval_s += random.uniform(0.0, jitter_s) if jitter_s > 0 else 0.0
     with _provider_pace_lock:
         now = time.monotonic()
         wait_s = min_interval_s - (now - _provider_last_request_at.get(provider, 0.0))
@@ -47,6 +59,24 @@ def _pace_provider(provider: str, min_interval_s: float) -> None:
         _provider_last_request_at[provider] = now
 
 
+def _apply_pre_fetch_jitter() -> None:
+    if _PRE_FETCH_JITTER_S <= 0:
+        return
+    delay = random.uniform(0.0, _PRE_FETCH_JITTER_S)
+    if delay >= 1.0:
+        logger.debug("Prediction markets: pre-fetch jitter %.1fs", delay)
+    time.sleep(delay)
+
+
+def _apply_provider_gap_jitter() -> None:
+    if _PROVIDER_GAP_JITTER_S <= 0:
+        return
+    delay = random.uniform(0.0, _PROVIDER_GAP_JITTER_S)
+    if delay >= 1.0:
+        logger.debug("Prediction markets: provider gap jitter %.1fs", delay)
+    time.sleep(delay)
+
+
 def _finite_or_none(value):
     try:
         n = float(value)
@@ -750,7 +780,9 @@ def _merge_markets(poly_events: list[dict], kalshi_events: list[dict]) -> list[d
 @cached(_market_cache)
 def fetch_prediction_markets_raw() -> list[dict]:
     """Fetch and merge prediction markets from both sources. Cached 5 min."""
+    _apply_pre_fetch_jitter()
     poly = _fetch_polymarket_events()
+    _apply_provider_gap_jitter()
     kalshi = _fetch_kalshi_events()
     merged = _merge_markets(poly, kalshi)
     logger.info(

backend/services/fetchers/retry.py

@@ -11,15 +11,20 @@ import random
 import logging
 import functools
 import requests
+from requests.exceptions import ChunkedEncodingError, ConnectionError as RequestsConnectionError
+from requests.exceptions import Timeout as RequestsTimeout
 
 logger = logging.getLogger(__name__)
 
-# Only retry on transient network/OS errors — not on parse errors, key errors, etc.
+# Only retry on transient network/OS errors — not parse/key errors or HTTP 4xx/5xx.
+# requests.HTTPError (from raise_for_status) is intentionally excluded.
 TRANSIENT_ERRORS = (
     TimeoutError,
     ConnectionError,
     OSError,
-    requests.RequestException,
+    RequestsConnectionError,
+    RequestsTimeout,
+    ChunkedEncodingError,
 )
 
 
@@ -43,6 +48,8 @@ def with_retry(max_retries: int = 3, base_delay: float = 2.0, max_delay: float =
             for attempt in range(1 + max_retries):
                 try:
                     return func(*args, **kwargs)
+                except requests.HTTPError:
+                    raise
                 except TRANSIENT_ERRORS as exc:
                     last_exc = exc
                     if attempt < max_retries:

backend/services/fetchers/road_corridor_sat.py

@@ -0,0 +1,84 @@
+"""Scheduled Sentinel-2 road corridor freight trend fetcher (opt-in, slow tier)."""
+from __future__ import annotations
+
+import logging
+import os
+from datetime import datetime, timezone
+
+from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
+
+logger = logging.getLogger(__name__)
+
+_REFRESH_HOURS = float(os.environ.get("ROAD_CORRIDOR_REFRESH_HOURS", "24"))
+
+
+def _hours_since(iso_ts: str) -> float | None:
+    try:
+        dt = datetime.fromisoformat(iso_ts.replace("Z", "+00:00"))
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        return (datetime.now(timezone.utc) - dt).total_seconds() / 3600.0
+    except ValueError:
+        return None
+
+
+def _feature_ready() -> bool:
+    from services.road_corridor_sat.config import optional_deps_available, road_corridor_sat_enabled
+    from services.road_corridor_sat.credentials import sentinel_credentials_configured
+
+    if not road_corridor_sat_enabled():
+        return False
+    if not optional_deps_available():
+        logger.debug("road_corridor_trends skipped — optional deps not installed")
+        return False
+    if not sentinel_credentials_configured():
+        logger.debug("road_corridor_trends skipped — Sentinel credentials missing")
+        return False
+    return True
+
+
+def refresh_road_corridor_store() -> None:
+    from services.road_corridor_sat.storage import build_trends_payload
+
+    payload = build_trends_payload()
+    with _data_lock:
+        latest_data["road_corridor_trends"] = payload
+    _mark_fresh("road_corridor_trends")
+
+
+def fetch_road_corridor_trends(force: bool = False) -> None:
+    """Refresh scheduled corridor presets (default: laredo_i35 every 24h)."""
+    if not is_any_active("road_corridor_trends"):
+        return
+    if not _feature_ready():
+        return
+
+    from services.road_corridor_sat.config import SCHEDULED_PRESET_IDS
+    from services.road_corridor_sat.pipeline import analyze_preset
+    from services.road_corridor_sat.presets import get_preset
+    from services.road_corridor_sat.storage import load_refresh_state
+
+    state = load_refresh_state()
+    for preset_id in SCHEDULED_PRESET_IDS:
+        preset = get_preset(preset_id)
+        if preset is None:
+            logger.warning("Unknown scheduled road corridor preset: %s", preset_id)
+            continue
+        last = state.get(preset_id)
+        if last and not force:
+            age_h = _hours_since(last)
+            if age_h is not None and age_h < _REFRESH_HOURS:
+                logger.info(
+                    "road_corridor %s fresh (%.1fh < %.1fh) — skipping",
+                    preset_id,
+                    age_h,
+                    _REFRESH_HOURS,
+                )
+                continue
+        try:
+            logger.info("road_corridor analysis starting for %s", preset_id)
+            analyze_preset(preset_id)
+        except Exception as exc:
+            logger.exception("road_corridor analysis failed for %s: %s", preset_id, exc)
+
+    refresh_road_corridor_store()

backend/services/fetchers/route_database.py

@@ -30,8 +30,6 @@ _AIRPORTS_URL = "https://vrs-standing-data.adsb.lol/airports.csv.gz"
 _REFRESH_INTERVAL_S = 5 * 24 * 3600
 _HTTP_TIMEOUT_S = 60
 
-from services.network_utils import DEFAULT_USER_AGENT as _USER_AGENT
-
 _lock = threading.RLock()
 _routes_by_callsign: dict[str, dict[str, Any]] = {}
 _airports_by_icao: dict[str, dict[str, Any]] = {}

backend/services/fetchers/sigint.py

@@ -21,12 +21,21 @@ def _merge_sigint_snapshot(
     because they include fresher region/channel metadata.
     """
 
-    merged = list(live_signals)
+    # Shallow-copy every entry so the published list owns its own dicts. The
+    # inputs alias objects that other threads keep mutating in place: live
+    # signals are the SIGINT bridge's own dicts (updated as packets arrive),
+    # and api_nodes are the same objects published under latest_data
+    # ["meshtastic_map_nodes"]. Publishing those references into
+    # latest_data["sigint"] lets a concurrent mutation race the lock-free
+    # deepcopy in get_latest_data_deepcopy_snapshot() (/api/health, /api/live-
+    # data) and raise "dictionary changed size during iteration". Copying
+    # honors the replace-don't-mutate contract in fetchers/_store.py.
+    merged = [dict(s) for s in live_signals]
     live_callsigns = {s["callsign"] for s in merged if s.get("source") == "meshtastic"}
     for node in api_nodes:
         if node.get("callsign") in live_callsigns:
             continue
-        merged.append(node)
+        merged.append(dict(node))
     merged.sort(key=lambda item: str(item.get("timestamp", "") or ""), reverse=True)
     return merged
 

backend/services/fetchers/telegram_osint.py

@@ -0,0 +1,377 @@
+"""Telegram OSINT — public channel web previews (t.me/s) with keyword geoparsing."""
+from __future__ import annotations
+
+import hashlib
+import html
+import logging
+import os
+import re
+from datetime import datetime, timezone
+from typing import Any
+
+from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
+from services.fetchers.news import resolve_coords_match
+from services.network_utils import fetch_with_curl, outbound_user_agent
+from services.telegram_translate import apply_post_translation, apply_posts_translations
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_CHANNELS = (
+    "osintdefender",
+    "insiderpaper",
+    "aljazeeraenglish",
+    "nexta_live",
+    "war_monitor",
+    "OSINTtechnical",
+    "Liveuamap",
+)
+
+_MESSAGE_BLOCK_RE = re.compile(
+    r'<div class="tgme_widget_message_wrap js-widget_message_wrap"[\s\S]*?</div>\s*</div>\s*</div>',
+    re.IGNORECASE,
+)
+_TEXT_RE = re.compile(
+    r'<div class="tgme_widget_message_text[^>]*>([\s\S]*?)</div>',
+    re.IGNORECASE,
+)
+_DATE_RE = re.compile(
+    r'<a class="tgme_widget_message_date" href="(https://t\.me/[^"]+)".*?<time datetime="([^"]+)"',
+    re.IGNORECASE,
+)
+_HAS_VIDEO_RE = re.compile(
+    r'tgme_widget_message_video|js-message_video|<video\s',
+    re.IGNORECASE,
+)
+_HAS_PHOTO_RE = re.compile(r'tgme_widget_message_photo_wrap', re.IGNORECASE)
+_VIDEO_SRC_RE = re.compile(r'<video[^>]+src="([^"]+)"', re.IGNORECASE)
+_BG_IMAGE_RE = re.compile(r"background-image:url\('([^']+)'\)", re.IGNORECASE)
+
+_TELEGRAM_MEDIA_HOST_SUFFIXES = (".telesco.pe", ".telegram-cdn.org")
+
+# Cyrillic / Arabic aliases for war-reporting channels (merged after English resolver).
+_EXTRA_PLACE_KEYWORDS: dict[str, tuple[float, float]] = {
+    "киев": (50.450, 30.523),
+    "київ": (50.450, 30.523),
+    "харьков": (49.993, 36.231),
+    "харків": (49.993, 36.231),
+    "одесса": (46.482, 30.724),
+    "одеса": (46.482, 30.724),
+    "донецк": (48.015, 37.803),
+    "донецьк": (48.015, 37.803),
+    "луганск": (48.574, 39.307),
+    "луганськ": (48.574, 39.307),
+    "москва": (55.755, 37.617),
+    "крым": (45.000, 34.000),
+    "крим": (45.000, 34.000),
+    "бахмут": (48.595, 38.000),
+    "запорожье": (47.838, 35.139),
+    "запоріжжя": (47.838, 35.139),
+    "غزة": (31.416, 34.333),
+    "دمشق": (33.513, 36.276),
+    "بيروت": (33.893, 35.501),
+    "tel aviv": (32.085, 34.781),
+    "תל אביב": (32.085, 34.781),
+}
+
+# Country-level news geocodes sit on national centroids that stack with threat alerts.
+# Telegram uses major metro anchors so pins land on a different map cell than news.
+_TELEGRAM_ANCHOR_OVERRIDES: dict[str, tuple[float, float]] = {
+    "israel": (32.085, 34.781),  # Tel Aviv (news uses central Israel ~Jerusalem corridor)
+    "middle east": (32.085, 34.781),
+    "china": (39.904, 116.407),  # Beijing (news uses country centroid)
+    "united states": (40.712, -74.006),  # New York (news uses Washington DC)
+    "usa": (40.712, -74.006),
+    "us": (40.712, -74.006),
+    "america": (40.712, -74.006),
+    "uk": (51.507, -0.127),  # London
+    "iran": (35.689, 51.389),  # Tehran
+    "russia": (55.755, 37.617),  # Moscow
+    "ukraine": (50.450, 30.523),  # Kyiv
+    "france": (48.856, 2.352),  # Paris
+    "germany": (52.520, 13.405),  # Berlin
+    "lebanon": (34.433, 35.844),  # Tripoli (news uses Beirut corridor)
+}
+
+_RISK_KEYWORDS = (
+    "war",
+    "missile",
+    "strike",
+    "attack",
+    "crisis",
+    "tension",
+    "military",
+    "conflict",
+    "defense",
+    "clash",
+    "nuclear",
+    "invasion",
+    "bomb",
+    "drone",
+    "weapon",
+    "sanctions",
+    "ceasefire",
+    "escalation",
+    "killed",
+    "destroyed",
+    "operation",
+    "casualty",
+    "frontline",
+    "threat",
+    "explosion",
+    "shelling",
+)
+
+
+def telegram_osint_enabled() -> bool:
+    return str(os.environ.get("TELEGRAM_OSINT_ENABLED", "true")).strip().lower() not in {
+        "0",
+        "false",
+        "no",
+        "off",
+        "",
+    }
+
+
+def _configured_channels() -> list[str]:
+    raw = str(os.environ.get("TELEGRAM_OSINT_CHANNELS", "")).strip()
+    if raw:
+        return [part.strip().lstrip("@") for part in raw.split(",") if part.strip()]
+    return list(_DEFAULT_CHANNELS)
+
+
+def telegram_media_host_allowed(hostname: str | None) -> bool:
+    host = str(hostname or "").strip().lower()
+    if not host:
+        return False
+    return any(host.endswith(suffix) for suffix in _TELEGRAM_MEDIA_HOST_SUFFIXES)
+
+
+def _extract_media(block: str, link: str) -> dict[str, Any]:
+    has_video = bool(_HAS_VIDEO_RE.search(block))
+    has_photo = bool(_HAS_PHOTO_RE.search(block))
+    media_type: str | None = None
+    media_url: str | None = None
+    if has_video:
+        media_type = "video"
+        video_match = _VIDEO_SRC_RE.search(block)
+        if video_match:
+            media_url = video_match.group(1).strip()
+    elif has_photo:
+        media_type = "photo"
+        photo_match = _BG_IMAGE_RE.search(block)
+        if photo_match:
+            media_url = photo_match.group(1).strip()
+
+    embed_url: str | None = None
+    if media_type and link:
+        embed_url = f"{link}?embed=1"
+
+    return {
+        "media_type": media_type,
+        "media_url": media_url,
+        "embed_url": embed_url,
+    }
+
+
+def _strip_html(text: str) -> str:
+    cleaned = re.sub(r"<br\s*/?>", "\n", text, flags=re.IGNORECASE)
+    cleaned = re.sub(r"<[^>]+>", "", cleaned)
+    return html.unescape(cleaned).strip()
+
+
+def _score_risk(text: str) -> int:
+    lower = text.lower()
+    score = 1
+    for kw in _RISK_KEYWORDS:
+        if kw in lower:
+            score += 2
+    return min(10, score)
+
+
+def _refresh_post_coords(post: dict[str, Any]) -> dict[str, Any]:
+    """Re-apply geoparsing so stored posts pick up anchor updates."""
+    text = "\n".join(
+        str(part).strip()
+        for part in (post.get("title"), post.get("description"))
+        if part and str(part).strip()
+    )
+    if not text:
+        return post
+    coords = _resolve_telegram_coords(text)
+    if not coords:
+        return post
+    updated = dict(post)
+    updated["coords"] = [coords[0], coords[1]]
+    return updated
+
+
+def _resolve_telegram_coords(text: str) -> tuple[float, float] | None:
+    lower = text.lower()
+    match = resolve_coords_match(lower)
+    if match:
+        _coords, keyword = match
+        anchor = _TELEGRAM_ANCHOR_OVERRIDES.get(keyword.strip().lower())
+        if anchor:
+            return anchor
+        return _coords
+    for keyword, coords in sorted(_EXTRA_PLACE_KEYWORDS.items(), key=lambda x: len(x[0]), reverse=True):
+        if keyword in lower:
+            return coords
+    return None
+
+
+def _post_link(post: dict[str, Any]) -> str:
+    return str(post.get("link") or "").strip()
+
+
+def _extract_new_channel_posts(
+    html: str,
+    channel: str,
+    known_links: set[str],
+    *,
+    bootstrap_limit: int = 12,
+) -> list[dict[str, Any]]:
+    """Return unseen posts from a channel page; stop once we hit a stored link."""
+    parsed = parse_telegram_channel_html(html, channel)
+    if not parsed:
+        return []
+    if not known_links:
+        return parsed[-bootstrap_limit:]
+
+    fresh: list[dict[str, Any]] = []
+    for post in reversed(parsed):
+        link = _post_link(post)
+        if not link:
+            continue
+        if link in known_links:
+            break
+        fresh.append(post)
+    fresh.reverse()
+    return fresh
+
+
+def _merge_telegram_posts(
+    existing: list[dict[str, Any]],
+    incoming: list[dict[str, Any]],
+    *,
+    max_posts: int = 120,
+) -> tuple[list[dict[str, Any]], int]:
+    known_links = {_post_link(post) for post in existing if _post_link(post)}
+    added = 0
+    for post in incoming:
+        link = _post_link(post)
+        if not link or link in known_links:
+            continue
+        known_links.add(link)
+        existing.append(post)
+        added += 1
+    existing.sort(key=lambda p: str(p.get("published") or ""), reverse=True)
+    return existing[:max_posts], added
+
+
+def parse_telegram_channel_html(html: str, channel: str) -> list[dict[str, Any]]:
+    """Parse public t.me/s channel preview HTML into post dicts."""
+    posts: list[dict[str, Any]] = []
+    for block in _MESSAGE_BLOCK_RE.findall(html or ""):
+        text_match = _TEXT_RE.search(block)
+        if not text_match:
+            continue
+        text = _strip_html(text_match.group(1))
+        if len(text) < 10:
+            continue
+
+        date_match = _DATE_RE.search(block)
+        link = date_match.group(1) if date_match else f"https://t.me/{channel}"
+        published = date_match.group(2) if date_match else datetime.now(timezone.utc).isoformat()
+        title = text.split("\n", 1)[0][:160]
+        risk_score = _score_risk(text)
+        coords = _resolve_telegram_coords(text)
+        post_id = hashlib.sha1(f"{link}|{published}".encode("utf-8")).hexdigest()[:16]
+
+        media = _extract_media(block, link)
+        post = {
+            "id": post_id,
+            "title": title,
+            "description": text[:1200],
+            "link": link,
+            "published": published,
+            "source": f"t.me/{channel}",
+            "channel": channel,
+            "risk_score": risk_score,
+            "coords": [coords[0], coords[1]] if coords else None,
+            **media,
+        }
+        posts.append(apply_post_translation(post))
+    return posts
+
+
+def fetch_telegram_osint() -> dict[str, Any]:
+    if not is_any_active("telegram_osint"):
+        return latest_data.get("telegram_osint") or {"posts": [], "total": 0, "timestamp": None}
+
+    if not telegram_osint_enabled():
+        with _data_lock:
+            latest_data["telegram_osint"] = {"posts": [], "total": 0, "timestamp": None, "disabled": True}
+        _mark_fresh("telegram_osint")
+        return latest_data["telegram_osint"]
+
+    headers = {
+        "User-Agent": (
+            f"Mozilla/5.0 (compatible; {outbound_user_agent('telegram-osint')}) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+        ),
+        "Accept": "text/html,application/xhtml+xml",
+    }
+
+    with _data_lock:
+        prior = latest_data.get("telegram_osint") or {}
+        existing_posts = list(prior.get("posts") or [])
+
+    known_links = {_post_link(post) for post in existing_posts if _post_link(post)}
+    incoming: list[dict[str, Any]] = []
+
+    for channel in _configured_channels():
+        url = f"https://t.me/s/{channel}"
+        try:
+            resp = fetch_with_curl(url, timeout=15, headers=headers)
+            if not resp or resp.status_code != 200:
+                logger.warning(
+                    "Telegram channel %s fetch failed: HTTP %s",
+                    channel,
+                    resp.status_code if resp else "no response",
+                )
+                continue
+            channel_new = _extract_new_channel_posts(resp.text, channel, known_links)
+            for post in channel_new:
+                link = _post_link(post)
+                if not link or link in known_links:
+                    continue
+                known_links.add(link)
+                incoming.append(post)
+        except Exception as exc:
+            logger.warning("Telegram channel %s parse failed: %s", channel, exc)
+
+    merged_posts, added = _merge_telegram_posts(existing_posts, incoming)
+    merged_posts = [_refresh_post_coords(post) for post in merged_posts]
+    merged_posts = apply_posts_translations(merged_posts)
+    geolocated = sum(1 for p in merged_posts if p.get("coords"))
+
+    payload = {
+        "posts": merged_posts,
+        "total": len(merged_posts),
+        "geolocated": geolocated,
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "channels": _configured_channels(),
+        "last_fetch_new": added,
+    }
+
+    with _data_lock:
+        latest_data["telegram_osint"] = payload
+    _mark_fresh("telegram_osint")
+    logger.info(
+        "Telegram OSINT: +%s new, %s retained (%s geolocated)",
+        added,
+        len(merged_posts),
+        geolocated,
+    )
+    return payload

backend/services/geopolitics.py

@@ -1,3 +1,4 @@
+import os
 import requests
 import logging
 import zipfile
@@ -20,6 +21,50 @@ logger = logging.getLogger(__name__)
 # Cache Frontline data for 30 minutes, it doesn't move that fast
 frontline_cache = TTLCache(maxsize=1, ttl=1800)
 
+_DEFAULT_DEEPSTATE_MIRROR_REPO = "cyterat/deepstate-map-data"
+
+
+def _deepstate_mirror_ref() -> tuple[str, str]:
+    """Return (github_repo_slug, git_ref) for the DeepState mirror.
+
+    When ``DEEPSTATE_MIRROR_COMMIT`` is set, ingest is pinned to that immutable
+    SHA instead of following the mutable ``main`` branch (#362).
+    """
+    repo = (os.environ.get("DEEPSTATE_MIRROR_REPO") or _DEFAULT_DEEPSTATE_MIRROR_REPO).strip()
+    if repo.count("/") != 1:
+        repo = _DEFAULT_DEEPSTATE_MIRROR_REPO
+    commit = (os.environ.get("DEEPSTATE_MIRROR_COMMIT") or "").strip()
+    ref = commit if commit else "main"
+    return repo, ref
+
+
+def _latest_deepstate_geo_path(tree_items: list) -> str | None:
+    geo_files = [
+        item["path"]
+        for item in tree_items
+        if isinstance(item, dict)
+        and str(item.get("path", "")).startswith("data/deepstatemap_data_")
+        and str(item.get("path", "")).endswith(".geojson")
+    ]
+    return sorted(geo_files)[-1] if geo_files else None
+
+
+def _annotate_deepstate_geojson(data: dict) -> dict:
+    name_map = {
+        0: "Russian-occupied areas",
+        1: "Russian advance",
+        2: "Liberated area",
+        3: "Russian-occupied areas",  # Crimea / LPR / DPR
+        4: "Directions of UA attacks",
+    }
+    if "features" in data:
+        for idx, feature in enumerate(data["features"]):
+            if "properties" not in feature or feature["properties"] is None:
+                feature["properties"] = {}
+            feature["properties"]["name"] = name_map.get(idx, "Russian-occupied areas")
+            feature["properties"]["zone_id"] = idx
+    return data
+
 
 @cached(frontline_cache)
 def fetch_ukraine_frontlines():
@@ -27,67 +72,34 @@ def fetch_ukraine_frontlines():
     Fetches the latest GeoJSON data representing the Ukraine frontline.
     We use the cyterat/deepstate-map-data github mirror since the public API is locked.
     """
+    repo, ref = _deepstate_mirror_ref()
     try:
-        logger.info("Fetching DeepStateMap from GitHub mirror...")
+        logger.info("Fetching DeepStateMap from GitHub mirror (%s @ %s)...", repo, ref)
 
-        # First, query the repo tree to find the latest file name
-        tree_url = (
-            "https://api.github.com/repos/cyterat/deepstate-map-data/git/trees/main?recursive=1"
-        )
+        tree_url = f"https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1"
         res_tree = requests.get(tree_url, timeout=10)
 
         if res_tree.status_code == 200:
-            tree_data = res_tree.json().get("tree", [])
-            # Filter for geojson files in data folder
-            geo_files = [
-                item["path"]
-                for item in tree_data
-                if item["path"].startswith("data/deepstatemap_data_")
-                and item["path"].endswith(".geojson")
-            ]
-
-            if geo_files:
-                # Get the alphabetically latest file (since it's named with YYYYMMDD)
-                latest_file = sorted(geo_files)[-1]
-
-                raw_url = f"https://raw.githubusercontent.com/cyterat/deepstate-map-data/main/{latest_file}"
-                logger.info(f"Downloading latest DeepStateMap: {raw_url}")
+            latest_file = _latest_deepstate_geo_path(res_tree.json().get("tree", []))
+            if latest_file:
+                raw_url = f"https://raw.githubusercontent.com/{repo}/{ref}/{latest_file}"
+                logger.info("Downloading DeepStateMap: %s", raw_url)
 
                 res_geo = requests.get(raw_url, timeout=20)
                 if res_geo.status_code == 200:
-                    data = res_geo.json()
-
-                    # The Cyterat GitHub mirror strips all properties and just provides a raw array of Feature polygons.
-                    # Based on DeepStateMap's frontend mapping, the array index corresponds to the zone type:
-                    # 0: Russian-occupied areas
-                    # 1: Russian advance
-                    # 2: Liberated area
-                    # 3: Uncontested/Crimea (often folded into occupied)
-                    name_map = {
-                        0: "Russian-occupied areas",
-                        1: "Russian advance",
-                        2: "Liberated area",
-                        3: "Russian-occupied areas",  # Crimea / LPR / DPR
-                        4: "Directions of UA attacks",
-                    }
-
-                    if "features" in data:
-                        for idx, feature in enumerate(data["features"]):
-                            if "properties" not in feature or feature["properties"] is None:
-                                feature["properties"] = {}
-
-                            feature["properties"]["name"] = name_map.get(
-                                idx, "Russian-occupied areas"
-                            )
-                            feature["properties"]["zone_id"] = idx
-
-                    return data
-                else:
-                    logger.error(
-                        f"Failed to fetch parsed Github Raw GeoJSON: {res_geo.status_code}"
-                    )
+                    return _annotate_deepstate_geojson(res_geo.json())
+                logger.error(
+                    "Failed to fetch parsed Github Raw GeoJSON: %s", res_geo.status_code
+                )
+            else:
+                logger.error("No deepstatemap_data_*.geojson files in mirror tree at %s", ref)
         else:
-            logger.error(f"Failed to fetch Github Tree for Deepstatemap: {res_tree.status_code}")
+            logger.error(
+                "Failed to fetch Github tree for Deepstatemap (%s @ %s): %s",
+                repo,
+                ref,
+                res_tree.status_code,
+            )
     except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
         logger.error(f"Error fetching DeepStateMap: {e}")
     return None
@@ -594,8 +606,19 @@ def _build_feature_html(features, fetched_titles=None):
 
 
 def _enrich_gdelt_titles_background(features, all_article_urls):
-    """Background thread: fetch real article titles then update features in-place."""
+    """Background thread: fetch real article titles, then publish enriched COPIES.
+
+    The ``features`` handed to us were already published into
+    ``latest_data["gdelt"]`` by ``fetch_gdelt()``. Per the store's thread-safety
+    contract (see ``get_latest_data_subset_refs`` in fetchers/_store.py), HTTP
+    readers hold live references to these nested ``properties`` dicts and
+    serialize them OUTSIDE the data lock. Mutating the published dicts in place
+    here races that serialization and raises
+    ``RuntimeError: dictionary changed size during iteration``. So we enrich
+    copies and atomically swap the top-level key under the lock instead.
+    """
     import html as html_mod
+    from services.fetchers._store import latest_data, _data_lock, _mark_fresh
 
     try:
         logger.info(f"[BG] Fetching real article titles for {len(all_article_urls)} URLs...")
@@ -603,28 +626,44 @@ def _enrich_gdelt_titles_background(features, all_article_urls):
         fetched_count = sum(1 for v in fetched_titles.values() if v)
         logger.info(f"[BG] Resolved {fetched_count}/{len(all_article_urls)} article titles")
 
-        # Update features in-place with real titles and snippets
+        # Build enriched copies — never touch the already-published objects.
+        enriched = []
         for f in features:
-            urls = f["properties"].get("_urls_list", [])
-            if not urls:
-                continue
-            headlines = []
-            snippets = []
-            for u in urls:
-                real_title = fetched_titles.get(u)
-                headlines.append(real_title if real_title else _url_to_headline(u))
-                snippets.append(_article_snippet_cache.get(u) or "")
-            f["properties"]["_headlines_list"] = headlines
-            f["properties"]["_snippets_list"] = snippets
-            links = []
-            for u, h in zip(urls, headlines):
-                safe_url = u if u.startswith(("http://", "https://")) else "about:blank"
-                safe_h = html_mod.escape(h)
-                links.append(
-                    f'<div style="margin-bottom:6px;"><a href="{safe_url}" target="_blank" rel="noopener noreferrer">{safe_h}</a></div>'
-                )
-            f["properties"]["html"] = "".join(links)
-        logger.info(f"[BG] GDELT title enrichment complete")
+            nf = dict(f)
+            props = dict(f.get("properties", {}))
+            urls = props.get("_urls_list", [])
+            if urls:
+                headlines = []
+                snippets = []
+                for u in urls:
+                    real_title = fetched_titles.get(u)
+                    headlines.append(real_title if real_title else _url_to_headline(u))
+                    snippets.append(_article_snippet_cache.get(u) or "")
+                props["_headlines_list"] = headlines
+                props["_snippets_list"] = snippets
+                links = []
+                for u, h in zip(urls, headlines):
+                    safe_url = u if u.startswith(("http://", "https://")) else "about:blank"
+                    safe_h = html_mod.escape(h)
+                    links.append(
+                        f'<div style="margin-bottom:6px;"><a href="{safe_url}" target="_blank" rel="noopener noreferrer">{safe_h}</a></div>'
+                    )
+                props["html"] = "".join(links)
+            nf["properties"] = props
+            enriched.append(nf)
+
+        # Atomically publish — but only if a newer fetch_gdelt() hasn't already
+        # replaced the layer while we were fetching titles (identity guard).
+        published = False
+        with _data_lock:
+            if latest_data.get("gdelt") is features:
+                latest_data["gdelt"] = enriched
+                published = True
+        if published:
+            _mark_fresh("gdelt")
+            logger.info(f"[BG] GDELT title enrichment complete ({len(enriched)} features)")
+        else:
+            logger.info("[BG] GDELT layer changed under us; skipping stale enrichment swap")
     except Exception as e:
         logger.error(f"[BG] GDELT title enrichment failed: {e}")
 

backend/services/infonet/privacy/function_keys/__init__.py

@@ -1,14 +1,20 @@
-"""Function Keys — anonymous citizenship proof.
+"""Function Keys — anonymous credential scaffolding.
 
 Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.4,
 ``infonet-economy/BRAINDUMP.md`` §11 item 9.
 
-A citizen should be able to prove "I am a UBI-eligible Infonet
-citizen" to a real-world operator (food bank, community service)
-**without revealing their Infonet identity**. The naive approach
-(scramble a public key, record each redemption on chain) leaks
-identity through metadata correlation (time, location, operator,
-frequency).
+A citizen should eventually be able to prove "I am a UBI-eligible
+Infonet citizen" to a real-world operator (food bank, community
+service) **without revealing their Infonet identity**. The current
+Python implementation wires the accounting, nullifier, receipt, and
+operator flows, but its HMAC challenge-response is a placeholder for
+integration tests. It is not a production anonymous or zero-knowledge
+citizenship proof until blind signatures or anonymous credentials are
+selected and wired.
+
+The naive approach (scramble a public key, record each redemption on
+chain) leaks identity through metadata correlation (time, location,
+operator, frequency).
 
 The full design has six pieces; five are implemented in pure Python
 here. The remaining piece — issuance via blind signatures or
@@ -27,7 +33,8 @@ Pieces:
    operator: tracked via ``NullifierTracker``.
 3. **Challenge-response** (`challenge_response.py`) — operator
    issues a fresh nonce, key-holder signs with the Function Key's
-   secret. Prevents screenshot attacks, key sharing, replay.
+   secret. This is HMAC placeholder plumbing for screenshot/replay
+   resistance, not the final anonymous credential proof.
 4. **Two-phase commit receipts** (`receipt.py`) — Phase 1
    verification receipt (operator-signed, day-level date NOT
    timestamp, no node_id). Phase 2 fulfillment receipt (citizen

backend/services/intel_feeds/country_risk.py

@@ -0,0 +1,94 @@
+"""Country risk index (static scores + USGS quake enrichment)."""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+from zoneinfo import ZoneInfo
+
+from services.network_utils import fetch_with_curl
+
+RISK_FACTORS: dict[str, dict[str, Any]] = {
+    "UA": {"base": 85, "tags": ["active_conflict", "infrastructure_damage"]},
+    "RU": {"base": 72, "tags": ["sanctions", "military_mobilization"]},
+    "IL": {"base": 78, "tags": ["active_conflict", "regional_instability"]},
+    "PS": {"base": 90, "tags": ["active_conflict", "humanitarian_crisis"]},
+    "SY": {"base": 82, "tags": ["post_conflict", "infrastructure_damage"]},
+    "YE": {"base": 88, "tags": ["active_conflict", "humanitarian_crisis"]},
+    "MM": {"base": 76, "tags": ["civil_unrest", "military_junta"]},
+    "SD": {"base": 84, "tags": ["active_conflict", "humanitarian_crisis"]},
+    "AF": {"base": 80, "tags": ["post_conflict", "governance_collapse"]},
+    "KP": {"base": 70, "tags": ["nuclear_risk", "isolation"]},
+    "IR": {"base": 68, "tags": ["sanctions", "nuclear_program", "regional_proxy"]},
+    "CN": {"base": 35, "tags": ["strategic_competition", "taiwan_tensions"]},
+    "TW": {"base": 45, "tags": ["invasion_risk", "semiconductor_dependency"]},
+    "VE": {"base": 60, "tags": ["economic_collapse", "political_instability"]},
+    "HT": {"base": 85, "tags": ["gang_violence", "governance_collapse"]},
+    "LB": {"base": 65, "tags": ["economic_crisis", "political_deadlock"]},
+    "PK": {"base": 55, "tags": ["terrorism", "political_instability"]},
+    "SO": {"base": 82, "tags": ["terrorism", "state_fragility"]},
+    "LY": {"base": 72, "tags": ["divided_government", "militia_control"]},
+    "ET": {"base": 62, "tags": ["ethnic_tensions", "regional_conflicts"]},
+}
+
+EXCHANGES = [
+    {"name": "NYSE", "tz": "America/New_York", "open": 9.5, "close": 16, "country": "US"},
+    {"name": "NASDAQ", "tz": "America/New_York", "open": 9.5, "close": 16, "country": "US"},
+    {"name": "LSE", "tz": "Europe/London", "open": 8, "close": 16.5, "country": "GB"},
+    {"name": "TSE", "tz": "Asia/Tokyo", "open": 9, "close": 15, "country": "JP"},
+    {"name": "SSE", "tz": "Asia/Shanghai", "open": 9.5, "close": 15, "country": "CN"},
+    {"name": "HKEX", "tz": "Asia/Hong_Kong", "open": 9.5, "close": 16, "country": "HK"},
+    {"name": "FRA", "tz": "Europe/Berlin", "open": 8, "close": 20, "country": "DE"},
+    {"name": "TSX", "tz": "America/Toronto", "open": 9.5, "close": 16, "country": "CA"},
+    {"name": "MOEX", "tz": "Europe/Moscow", "open": 10, "close": 18.5, "country": "RU"},
+]
+
+
+def _exchange_open(ex: dict[str, Any]) -> bool:
+    try:
+        now = datetime.now(ZoneInfo(ex["tz"]))
+        if now.weekday() >= 5:
+            return False
+        decimal = now.hour + now.minute / 60
+        return ex["open"] <= decimal < ex["close"]
+    except Exception:
+        return False
+
+
+def build_country_risk_payload() -> dict[str, Any]:
+    quake_risks: dict[str, float] = {}
+    try:
+        resp = fetch_with_curl(
+            "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/4.5_day.geojson",
+            timeout=5,
+        )
+        if resp.status_code == 200:
+            for f in resp.json().get("features") or []:
+                place = (f.get("properties") or {}).get("place") or ""
+                mag = (f.get("properties") or {}).get("mag") or 0
+                for code in RISK_FACTORS:
+                    if code.lower() in place.lower():
+                        quake_risks[code] = quake_risks.get(code, 0) + mag
+    except Exception:
+        pass
+
+    countries = []
+    for code, data in RISK_FACTORS.items():
+        base = data["base"]
+        score = min(100, base + quake_risks.get(code, 0))
+        countries.append(
+            {
+                "code": code,
+                "risk_score": score,
+                "risk_level": "CRITICAL" if base >= 80 else "HIGH" if base >= 60 else "ELEVATED" if base >= 40 else "LOW",
+                "tags": data["tags"],
+            }
+        )
+    countries.sort(key=lambda c: c["risk_score"], reverse=True)
+    exchanges = [{"name": e["name"], "country": e["country"], "open": _exchange_open(e)} for e in EXCHANGES]
+    return {
+        "countries": countries,
+        "exchanges": exchanges,
+        "open_exchanges": sum(1 for e in exchanges if e["open"]),
+        "total_exchanges": len(exchanges),
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+    }

backend/services/kiwisdr_fetcher.py

@@ -32,14 +32,14 @@ logger = logging.getLogger(__name__)
 _REFRESH_SECONDS = 24 * 3600
 kiwisdr_cache: TTLCache = TTLCache(maxsize=1, ttl=_REFRESH_SECONDS)
 
-_SOURCE_URL = "http://rx.linkfanel.net/kiwisdr_com.js"
+_SOURCE_URL_HTTP = "http://rx.linkfanel.net/kiwisdr_com.js"
+_SOURCE_URL_HTTPS = "https://rx.linkfanel.net/kiwisdr_com.js"
 _CACHE_FILE = Path(__file__).resolve().parent.parent / "data" / "kiwisdr_cache.json"
 # Bundled fallback — shipped with the codebase so the KiwiSDR layer always
 # has something to render even when the upstream is unreachable, returns
-# garbage, or appears to have been tampered with. Issue #206: the upstream
-# only speaks HTTP, so we can't rely on TLS for integrity — instead we
-# validate the response's shape and fall back to this bundle if it doesn't
-# look right.
+# garbage, or appears to have been tampered with. Issue #206 / #364: try HTTPS
+# first, then HTTP; we still validate shape and fall back to this bundle if the
+# payload does not look right.
 _BUNDLED_FALLBACK = Path(__file__).resolve().parent.parent / "data" / "kiwisdr_directory.json"
 
 # Minimum number of receivers we expect from a healthy upstream response.
@@ -184,6 +184,29 @@ def _validate_fetched_nodes(nodes: list[dict]) -> bool:
     return True
 
 
+def _fetch_mirror_payload_text() -> str | None:
+    """Try HTTPS first, then HTTP. Shape validation still applies (#364)."""
+    from services.network_utils import fetch_with_curl
+
+    last_error: Exception | None = None
+    for url in (_SOURCE_URL_HTTPS, _SOURCE_URL_HTTP):
+        try:
+            res = fetch_with_curl(url, timeout=20)
+            if res and res.status_code == 200:
+                if url == _SOURCE_URL_HTTP:
+                    logger.info(
+                        "KiwiSDR: HTTPS mirror unavailable; using HTTP with shape validation"
+                    )
+                return res.text
+            last_error = RuntimeError(f"HTTP {getattr(res, 'status_code', 'unknown')}")
+        except Exception as e:
+            last_error = e
+            logger.debug("KiwiSDR mirror fetch failed for %s: %s", url, e)
+    if last_error is not None:
+        logger.warning("KiwiSDR mirror fetch failed: %s", last_error)
+    return None
+
+
 def _load_bundled_fallback() -> list[dict]:
     """Last-resort directory shipped with the codebase. Always returns a
     list (may be empty if the bundle is missing in older deployments)."""
@@ -202,9 +225,8 @@ def _load_bundled_fallback() -> list[dict]:
 def fetch_kiwisdr_nodes() -> list[dict]:
     """Return the KiwiSDR receiver list, refreshed at most once per day.
 
-    Layered fallback (issue #206 — upstream is HTTP-only, so we defend with
-    content validation + bundled static directory rather than trying to
-    upgrade the transport):
+    Layered fallback (issue #206 / #364 — HTTPS first, HTTP fallback, plus
+    content validation + bundled static directory):
 
       1. In-memory cache (handled by @cached on this function)
       2. On-disk cache if <24h old
@@ -216,8 +238,6 @@ def fetch_kiwisdr_nodes() -> list[dict]:
     tampered upstream returning garbage is caught by _validate_fetched_nodes()
     and falls through to whatever previously-trusted snapshot we have.
     """
-    from services.network_utils import fetch_with_curl
-
     # 1. Trust on-disk cache if fresh.
     cached_nodes = _load_disk_cache()
     if cached_nodes is not None:
@@ -230,14 +250,12 @@ def fetch_kiwisdr_nodes() -> list[dict]:
     fresh_nodes: list[dict] = []
     fetch_succeeded = False
     try:
-        res = fetch_with_curl(_SOURCE_URL, timeout=20)
-        if res and res.status_code == 200:
-            fresh_nodes = _parse_mirror_payload(res.text)
+        body = _fetch_mirror_payload_text()
+        if body:
+            fresh_nodes = _parse_mirror_payload(body)
             fetch_succeeded = True
         else:
-            logger.warning(
-                f"KiwiSDR fetch returned HTTP {res.status_code if res else 'no response'}"
-            )
+            logger.warning("KiwiSDR fetch returned no usable mirror payload")
     except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
         logger.warning(f"KiwiSDR fetch exception: {e}")
 

backend/services/liveuamap_scraper.py

@@ -27,11 +27,21 @@ def fetch_liveuamap():
         browser = p.chromium.launch(
             headless=True, args=["--disable-blink-features=AutomationControlled"]
         )
+        from services.network_utils import outbound_user_agent
+
+        # Per-install handle (no shared Shadowbroker product token). Stealth remains
+        # for Turnstile; see docs/OUTBOUND_DATA.md #348.
+        playwright_ua = (
+            f"Mozilla/5.0 (compatible; {outbound_user_agent('liveuamap')})"
+        )
         context = browser.new_context(
-            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+            user_agent=playwright_ua,
             viewport={"width": 1920, "height": 1080},
             color_scheme="dark",
         )
+        # Bound navigation and script evaluation so a stuck region cannot hang the slow pool.
+        context.set_default_navigation_timeout(60_000)
+        context.set_default_timeout(30_000)
         page = context.new_page()
         stealth_sync(page)
 

backend/services/liveuamap_settings.py

@@ -0,0 +1,73 @@
+"""LiveUAMap Playwright scraper opt-in (#348) — UI consent on Windows."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_OPT_IN_FILE = Path(__file__).resolve().parent.parent / "data" / "liveuamap_scraper_opt_in.json"
+_OPT_IN_LOCK = threading.Lock()
+
+
+def _env_flag(name: str) -> str:
+    return str(os.getenv(name, "")).strip().lower()
+
+
+def liveuamap_requires_ui_opt_in() -> bool:
+    """Windows local installs need explicit consent before Playwright contacts LiveUAMap."""
+    return os.name == "nt"
+
+
+def get_liveuamap_ui_opt_in() -> bool:
+    if not _OPT_IN_FILE.exists():
+        return False
+    try:
+        payload = json.loads(_OPT_IN_FILE.read_text(encoding="utf-8"))
+        return bool(payload.get("opted_in"))
+    except (OSError, json.JSONDecodeError, TypeError) as e:
+        logger.warning("LiveUAMap opt-in file unreadable: %s", e)
+        return False
+
+
+def set_liveuamap_ui_opt_in(opted_in: bool) -> None:
+    _OPT_IN_FILE.parent.mkdir(parents=True, exist_ok=True)
+    with _OPT_IN_LOCK:
+        _OPT_IN_FILE.write_text(
+            json.dumps({"opted_in": bool(opted_in)}, indent=2),
+            encoding="utf-8",
+        )
+
+
+def liveuamap_scraper_enabled() -> bool:
+    """Whether the Playwright LiveUAMap scraper may run on this backend."""
+    setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
+    if setting in {"1", "true", "yes", "on"}:
+        return True
+    if setting in {"0", "false", "no", "off"}:
+        return False
+    if not liveuamap_requires_ui_opt_in():
+        return True
+    return get_liveuamap_ui_opt_in()
+
+
+def liveuamap_scraper_status() -> dict[str, Any]:
+    setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
+    env_override = None
+    if setting in {"1", "true", "yes", "on"}:
+        env_override = "on"
+    elif setting in {"0", "false", "no", "off"}:
+        env_override = "off"
+    ui_opted_in = get_liveuamap_ui_opt_in()
+    requires = liveuamap_requires_ui_opt_in()
+    return {
+        "platform_requires_opt_in": requires,
+        "ui_opted_in": ui_opted_in,
+        "scraper_enabled": liveuamap_scraper_enabled(),
+        "env_override": env_override,
+    }

backend/services/mesh/mesh_bootstrap_manifest.py

@@ -287,28 +287,18 @@ def write_signed_bootstrap_manifest(
     return manifest
 
 
-def load_bootstrap_manifest(
-    path: str | Path,
+def parse_bootstrap_manifest_dict(
+    raw: dict[str, Any],
     *,
     signer_public_key_b64: str,
     now: float | None = None,
 ) -> BootstrapManifest:
-    manifest_path = _resolve_manifest_path(str(path))
-    try:
-        raw = json.loads(manifest_path.read_text(encoding="utf-8"))
-    except FileNotFoundError as exc:
-        raise BootstrapManifestError(f"bootstrap manifest not found: {manifest_path}") from exc
-    except json.JSONDecodeError as exc:
-        raise BootstrapManifestError("bootstrap manifest is not valid JSON") from exc
-
     if not isinstance(raw, dict):
         raise BootstrapManifestError("bootstrap manifest root must be an object")
-
     signature = str(raw.get("signature", "") or "").strip()
     payload = {key: value for key, value in raw.items() if key != "signature"}
     if not signature:
         raise BootstrapManifestError("bootstrap manifest signature is required")
-
     _verify_manifest_signature(
         payload,
         signature_b64=signature,
@@ -325,11 +315,36 @@ def load_bootstrap_manifest(
     )
 
 
+def load_bootstrap_manifest(
+    path: str | Path,
+    *,
+    signer_public_key_b64: str,
+    now: float | None = None,
+) -> BootstrapManifest:
+    manifest_path = _resolve_manifest_path(str(path))
+    try:
+        raw = json.loads(manifest_path.read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise BootstrapManifestError(f"bootstrap manifest not found: {manifest_path}") from exc
+    except json.JSONDecodeError as exc:
+        raise BootstrapManifestError("bootstrap manifest is not valid JSON") from exc
+
+    if not isinstance(raw, dict):
+        raise BootstrapManifestError("bootstrap manifest root must be an object")
+    return parse_bootstrap_manifest_dict(
+        raw,
+        signer_public_key_b64=signer_public_key_b64,
+        now=now,
+    )
+
+
 def load_bootstrap_manifest_from_settings(*, now: float | None = None) -> BootstrapManifest | None:
     settings = get_settings()
     if bool(getattr(settings, "MESH_BOOTSTRAP_DISABLED", False)):
         return None
-    signer_public_key_b64 = str(getattr(settings, "MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY", "") or "").strip()
+    from services.mesh.mesh_fleet_defaults import effective_bootstrap_signer_public_key_b64
+
+    signer_public_key_b64 = effective_bootstrap_signer_public_key_b64()
     if not signer_public_key_b64:
         return None
     manifest_path = _resolve_manifest_path(str(getattr(settings, "MESH_BOOTSTRAP_MANIFEST_PATH", "") or ""))

backend/services/mesh/mesh_crypto.py

@@ -168,9 +168,9 @@ def resolve_peer_key_for_url(peer_url: str) -> bytes:
     try:
         from services.config import get_settings
 
-        global_secret = str(
-            getattr(get_settings(), "MESH_PEER_PUSH_SECRET", "") or ""
-        ).strip()
+        from services.mesh.mesh_fleet_defaults import effective_peer_push_secret
+
+        global_secret = effective_peer_push_secret()
     except Exception:
         return b""
     if not global_secret:

backend/services/mesh/mesh_dm_connect_delivery.py

@@ -0,0 +1,179 @@
+"""Invite-scoped DM connect delivery: auto relay release and contact severance."""
+
+from __future__ import annotations
+
+from typing import Any
+
+CONNECT_AUTO_RELEASE_INTENTS = frozenset(
+    {
+        "invite_short_address",
+        "invite_import",
+        "contact_request",
+        "contact_accept",
+        "contact_offer",
+    }
+)
+
+INVITE_CONNECT_TRUST_LEVELS = frozenset({"invite_pinned", "sas_verified"})
+
+
+def _release_profile() -> str:
+    try:
+        from services.release_profiles import current_release_profile
+
+        return str(current_release_profile() or "dev")
+    except Exception:
+        return "dev"
+
+
+def grant_connect_relay_policy(
+    recipient_id: str,
+    *,
+    reason: str = "connect_scoped_auto_release",
+) -> dict[str, Any]:
+    """Pre-authorize hidden relay delivery for an explicit connect target."""
+    peer_key = str(recipient_id or "").strip()
+    if not peer_key:
+        return {"ok": False, "detail": "recipient_id required"}
+    try:
+        from services.mesh.mesh_relay_policy import grant_relay_policy
+
+        return grant_relay_policy(
+            scope_type="dm_contact",
+            scope_id=peer_key,
+            profile=_release_profile(),
+            hidden_transport_required=True,
+            reason=str(reason or "connect_scoped_auto_release"),
+        )
+    except Exception as exc:
+        return {"ok": False, "detail": str(exc) or type(exc).__name__}
+
+
+def revoke_connect_relay_policy(recipient_id: str) -> dict[str, Any]:
+    peer_key = str(recipient_id or "").strip()
+    if not peer_key:
+        return {"ok": False, "detail": "recipient_id required"}
+    try:
+        from services.mesh.mesh_relay_policy import revoke_relay_policy
+
+        revoked = int(
+            revoke_relay_policy(
+                scope_type="dm_contact",
+                scope_id=peer_key,
+                profile=_release_profile(),
+            )
+            or 0
+        )
+        return {"ok": True, "revoked": revoked}
+    except Exception as exc:
+        return {"ok": False, "detail": str(exc) or type(exc).__name__}
+
+
+def recipient_has_invite_connect_scope(recipient_id: str) -> bool:
+    peer_key = str(recipient_id or "").strip()
+    if not peer_key:
+        return False
+    try:
+        from services.mesh.mesh_wormhole_contacts import get_wormhole_dm_contact
+
+        contact = get_wormhole_dm_contact(peer_key) or {}
+    except Exception:
+        return False
+    if str(contact.get("invitePinnedPrekeyLookupHandle", "") or "").strip():
+        return True
+    if str(contact.get("invitePinnedLookupPeerUrl", "") or "").strip():
+        return True
+    trust = str(contact.get("trust_level", "") or "").strip().lower()
+    return trust in INVITE_CONNECT_TRUST_LEVELS
+
+
+def relay_push_peer_urls_for_payload(payload: dict[str, Any]) -> list[str]:
+    urls: list[str] = []
+    for raw in list(payload.get("relay_push_peer_urls") or []):
+        normalized = str(raw or "").strip().rstrip("/")
+        if normalized and normalized not in urls:
+            urls.append(normalized)
+    lookup_peer_url = str(payload.get("lookup_peer_url", "") or "").strip().rstrip("/")
+    if lookup_peer_url:
+        urls = [url for url in urls if url != lookup_peer_url]
+        urls.insert(0, lookup_peer_url)
+    recipient_id = str(payload.get("recipient_id", "") or "").strip()
+    if recipient_id and not urls:
+        try:
+            from services.mesh.mesh_wormhole_contacts import get_wormhole_dm_contact
+
+            contact = get_wormhole_dm_contact(recipient_id) or {}
+            pinned = str(contact.get("invitePinnedLookupPeerUrl", "") or "").strip().rstrip("/")
+            if pinned:
+                urls.append(pinned)
+        except Exception:
+            pass
+    return urls
+
+
+def should_auto_release_dm_payload(payload: dict[str, Any]) -> bool:
+    if str(payload.get("delivery_class", "") or "").strip().lower() != "request":
+        return False
+    intent = str(payload.get("connect_intent", "") or "").strip().lower()
+    if intent in CONNECT_AUTO_RELEASE_INTENTS:
+        return True
+    if str(payload.get("lookup_peer_url", "") or "").strip():
+        return True
+    recipient_id = str(payload.get("recipient_id", "") or "").strip()
+    return bool(recipient_id and recipient_has_invite_connect_scope(recipient_id))
+
+
+def enrich_connect_release_payload(payload: dict[str, Any]) -> dict[str, Any]:
+    """Attach invite-owner relay hints used during private release."""
+    enriched = dict(payload or {})
+    recipient_id = str(enriched.get("recipient_id", "") or "").strip()
+    lookup_peer_url = str(enriched.get("lookup_peer_url", "") or "").strip().rstrip("/")
+    if not lookup_peer_url and recipient_id:
+        try:
+            from services.mesh.mesh_wormhole_contacts import get_wormhole_dm_contact
+
+            contact = get_wormhole_dm_contact(recipient_id) or {}
+            lookup_peer_url = str(contact.get("invitePinnedLookupPeerUrl", "") or "").strip().rstrip("/")
+        except Exception:
+            lookup_peer_url = ""
+    if lookup_peer_url:
+        enriched["lookup_peer_url"] = lookup_peer_url
+    push_urls = relay_push_peer_urls_for_payload(enriched)
+    if push_urls:
+        enriched["relay_push_peer_urls"] = push_urls
+    return enriched
+
+
+def auto_release_connect_dm_outbox(*, outbox_id: str, payload: dict[str, Any]) -> dict[str, Any]:
+    """Grant scoped relay policy and approve release for invite-scoped connect traffic."""
+    normalized_outbox = str(outbox_id or "").strip()
+    enriched = enrich_connect_release_payload(payload)
+    if not normalized_outbox:
+        return {"ok": False, "detail": "missing outbox_id"}
+    if not should_auto_release_dm_payload(enriched):
+        return {"ok": True, "skipped": True, "reason": "not_connect_scoped"}
+    recipient_id = str(enriched.get("recipient_id", "") or "").strip()
+    if not recipient_id:
+        return {"ok": False, "detail": "missing recipient_id"}
+    grant = grant_connect_relay_policy(recipient_id)
+    try:
+        from services.mesh.mesh_private_outbox import private_delivery_outbox
+        from services.mesh.mesh_private_release_worker import private_release_worker
+
+        private_delivery_outbox.approve_relay_release(normalized_outbox)
+        private_release_worker.ensure_started()
+        private_release_worker.wake()
+    except Exception as exc:
+        return {
+            "ok": False,
+            "detail": str(exc) or type(exc).__name__,
+            "grant": grant,
+        }
+    return {
+        "ok": True,
+        "auto_released": True,
+        "outbox_id": normalized_outbox,
+        "recipient_id": recipient_id,
+        "grant": grant,
+        "relay_push_peer_urls": relay_push_peer_urls_for_payload(enriched),
+    }

backend/services/mesh/mesh_dm_relay.py

@@ -1506,6 +1506,7 @@ class DMRelay:
         sender_token_hash: str = "",
         payload_format: str = "dm1",
         session_welcome: str = "",
+        replication_peer_urls: list[str] | None = None,
     ) -> dict[str, Any]:
         with self._lock:
             self._refresh_from_shared_relay()
@@ -1573,46 +1574,214 @@ class DMRelay:
                 }
             if not msg_id:
                 msg_id = f"dm_{int(time.time() * 1000)}_{secrets.token_hex(6)}"
-            elif any(m.msg_id == msg_id for m in self._mailboxes[mailbox_key]):
-                return {"ok": True, "msg_id": msg_id}
-            relay_sender_id = (
-                f"sender_token:{sender_token_hash}"
-                if sender_token_hash
-                else sender_id
-            )
-            self._mailboxes[mailbox_key].append(
-                DMMessage(
-                    sender_id=relay_sender_id,
-                    ciphertext=ciphertext,
-                    timestamp=time.time(),
-                    msg_id=msg_id,
-                    delivery_class=delivery_class,
-                    sender_seal=sender_seal,
-                    sender_block_ref=sender_block_ref,
-                    payload_format=str(payload_format or "dm1"),
-                    session_welcome=str(session_welcome or ""),
+            duplicate_hit = any(m.msg_id == msg_id for m in self._mailboxes[mailbox_key])
+            if not duplicate_hit:
+                relay_sender_id = (
+                    f"sender_token:{sender_token_hash}"
+                    if sender_token_hash
+                    else sender_id
                 )
-            )
-            self._stats["messages_in_memory"] = sum(len(v) for v in self._mailboxes.values())
-            self._save()
-            # Cross-node mailbox replication: push the freshly-stored
-            # envelope to every authenticated relay peer so the recipient
-            # can log into ANY node and find their messages. The push is
-            # async (fire-and-forget thread) so deposit() returns
-            # immediately — slow Tor peers can't block the sender's UX.
-            # Each receiving peer re-enforces the per-sender cap on
-            # acceptance, so hostile relays can't widen the cap.
+                self._mailboxes[mailbox_key].append(
+                    DMMessage(
+                        sender_id=relay_sender_id,
+                        ciphertext=ciphertext,
+                        timestamp=time.time(),
+                        msg_id=msg_id,
+                        delivery_class=delivery_class,
+                        sender_seal=sender_seal,
+                        sender_block_ref=sender_block_ref,
+                        payload_format=str(payload_format or "dm1"),
+                        session_welcome=str(session_welcome or ""),
+                    )
+                )
+                self._stats["messages_in_memory"] = sum(len(v) for v in self._mailboxes.values())
+                self._save()
+            preferred_urls = list(replication_peer_urls or [])
+            envelope_for_push: dict[str, Any] | None = None
             try:
                 envelope_for_push = self.envelope_for_replication(
-                    mailbox_key=mailbox_key, msg_id=msg_id,
+                    mailbox_key=mailbox_key,
+                    msg_id=msg_id,
+                    recipient_id=recipient_id,
+                    recipient_token=recipient_token,
                 )
-                if envelope_for_push:
-                    self._replicate_envelope_to_peers_async(
-                        envelope=envelope_for_push,
-                    )
             except Exception:
                 metrics_inc("dm_replication_push_error")
-            return {"ok": True, "msg_id": msg_id}
+            deposit_result = {"ok": True, "msg_id": msg_id}
+            if duplicate_hit:
+                deposit_result["duplicate"] = True
+
+        if envelope_for_push:
+            # Invite-scoped connect traffic names an explicit recipient relay
+            # (lookup_peer_url). Block until that push completes so the
+            # recipient can poll their own node; fleet-wide fan-out stays
+            # async so dead manifest peers cannot wedge deposit().
+            if preferred_urls:
+                logger.info(
+                    "DM deposit awaiting scoped replicate to %d peer(s)",
+                    len(preferred_urls),
+                )
+                deposit_result["replicate"] = self._replicate_envelope_to_peers(
+                    envelope=envelope_for_push,
+                    preferred_peer_urls=preferred_urls,
+                )
+            else:
+                self._replicate_envelope_to_peers_async(
+                    envelope=envelope_for_push,
+                    preferred_peer_urls=[],
+                )
+        elif preferred_urls:
+            logger.warning(
+                "DM deposit skipped scoped replicate: envelope missing for msg_id=%s",
+                msg_id,
+            )
+        return deposit_result
+
+    def _replicate_envelope_to_peers(
+        self,
+        *,
+        envelope: dict[str, Any],
+        preferred_peer_urls: list[str] | None = None,
+    ) -> dict[str, Any]:
+        """Push an envelope to relay peers. Returns per-peer results."""
+        import hashlib
+        import hmac
+        import requests as _requests
+
+        from services.mesh.mesh_crypto import (
+            normalize_peer_url,
+            resolve_peer_key_for_url,
+        )
+        from services.mesh.mesh_router import authenticated_push_peer_urls
+
+        peers: list[str] = []
+        for raw_url in list(preferred_peer_urls or []):
+            normalized_preferred = normalize_peer_url(str(raw_url or "").strip())
+            if normalized_preferred and normalized_preferred not in peers:
+                peers.append(normalized_preferred)
+        if not peers:
+            for peer_url in authenticated_push_peer_urls():
+                normalized_peer = normalize_peer_url(str(peer_url or "").strip())
+                if normalized_peer and normalized_peer not in peers:
+                    peers.append(normalized_peer)
+        if not peers:
+            return {"ok": False, "detail": "no_relay_peers", "pushed": [], "failed": []}
+
+        logger.info(
+            "DM replicate push starting for %d peer(s): %s",
+            len(peers),
+            ", ".join(peers[:3]) + ("..." if len(peers) > 3 else ""),
+        )
+
+        payload = json.dumps(
+            {"envelope": envelope},
+            separators=(",", ":"),
+            ensure_ascii=False,
+        ).encode("utf-8")
+
+        base_timeout = max(
+            1,
+            int(getattr(self._settings(), "MESH_RELAY_PUSH_TIMEOUT_S", 10) or 10),
+        )
+
+        from main import _infonet_peer_requests_proxies
+
+        preferred_set = {
+            normalize_peer_url(str(raw_url or "").strip())
+            for raw_url in list(preferred_peer_urls or [])
+        }
+        preferred_set.discard("")
+
+        pushed: list[str] = []
+        failed: list[dict[str, str]] = []
+        for peer_url in peers:
+            try:
+                normalized = normalize_peer_url(peer_url)
+                timeout = max(180 if ".onion" in normalized else 1, base_timeout)
+                headers = {"Content-Type": "application/json"}
+                peer_key = resolve_peer_key_for_url(normalized)
+                if peer_key:
+                    headers["X-Peer-Url"] = normalized
+                    headers["X-Peer-HMAC"] = hmac.new(
+                        peer_key, payload, hashlib.sha256
+                    ).hexdigest()
+                url = f"{peer_url}/api/mesh/dm/replicate-envelope"
+                request_kwargs: dict[str, Any] = {
+                    "data": payload,
+                    "timeout": timeout,
+                    "headers": headers,
+                }
+                proxies = _infonet_peer_requests_proxies(normalized)
+                if proxies:
+                    request_kwargs["proxies"] = proxies
+                resp = None
+                max_attempts = 3 if normalized in preferred_set else 2
+                last_exc = ""
+                for attempt in range(max_attempts):
+                    try:
+                        resp = _requests.post(url, **request_kwargs)
+                        break
+                    except Exception as exc:
+                        last_exc = str(exc) or type(exc).__name__
+                        if attempt + 1 < max_attempts:
+                            time.sleep(5.0 * (attempt + 1))
+                            continue
+                        logger.warning(
+                            "DM replicate push to %s failed: %s",
+                            peer_url,
+                            last_exc,
+                        )
+                        metrics_inc("dm_replication_push_error")
+                        resp = None
+                        break
+                if resp is None:
+                    failed.append({"url": peer_url, "detail": last_exc or "request_failed"})
+                    continue
+                if resp.status_code == 200:
+                    body_ok = True
+                    detail = ""
+                    try:
+                        body = resp.json()
+                        if isinstance(body, dict) and body.get("ok") is False:
+                            body_ok = False
+                            detail = str(body.get("detail", "") or "replicate rejected")[:200]
+                    except Exception:
+                        body_ok = True
+                    if body_ok:
+                        logger.info("DM replicate push to %s succeeded", peer_url)
+                        metrics_inc("dm_replication_push_ok")
+                        pushed.append(peer_url)
+                    else:
+                        logger.warning(
+                            "DM replicate push to %s rejected: %s",
+                            peer_url,
+                            detail,
+                        )
+                        metrics_inc("dm_replication_push_rejected")
+                        failed.append({"url": peer_url, "detail": detail or "replicate_rejected"})
+                else:
+                    detail = (resp.text or "")[:200]
+                    logger.warning(
+                        "DM replicate push to %s -> %s: %s",
+                        peer_url,
+                        resp.status_code,
+                        detail,
+                    )
+                    metrics_inc("dm_replication_push_rejected")
+                    failed.append({"url": peer_url, "detail": f"http_{resp.status_code}: {detail}"})
+            except Exception as exc:
+                logger.warning("DM replicate push outer failure for %s: %s", peer_url, exc)
+                metrics_inc("dm_replication_push_error")
+                failed.append({"url": peer_url, "detail": str(exc) or type(exc).__name__})
+
+        scoped = bool(preferred_set)
+        ok = bool(pushed) if scoped else bool(pushed) or not failed
+        return {
+            "ok": ok,
+            "scoped": scoped,
+            "pushed": pushed,
+            "failed": failed,
+        }
 
     def accept_replica(
         self,
@@ -1645,6 +1814,33 @@ class DMRelay:
         mailbox_key = str(envelope.get("mailbox_key", "") or "").strip()
         sender_block_ref = str(envelope.get("sender_block_ref", "") or "").strip()
         ciphertext = str(envelope.get("ciphertext", "") or "")
+        delivery_class = str(envelope.get("delivery_class", "") or "").strip().lower()
+        recipient_id = str(envelope.get("recipient_id", "") or "").strip()
+        recipient_token = str(envelope.get("recipient_token", "") or "").strip()
+        if delivery_class not in ("request", "shared", "self"):
+            if recipient_id and not recipient_token:
+                delivery_class = "request"
+            elif recipient_token:
+                delivery_class = "shared"
+        if delivery_class == "request":
+            if not recipient_id:
+                try:
+                    from services.mesh.mesh_wormhole_persona import get_dm_identity
+
+                    recipient_id = str((get_dm_identity() or {}).get("node_id") or "").strip()
+                except Exception:
+                    recipient_id = ""
+            if recipient_id:
+                mailbox_key = self.mailbox_key_for_delivery(
+                    recipient_id=recipient_id,
+                    delivery_class="request",
+                )
+        elif delivery_class == "shared" and recipient_token:
+            mailbox_key = self.mailbox_key_for_delivery(
+                recipient_id=recipient_id,
+                delivery_class="shared",
+                recipient_token=recipient_token,
+            )
         if not msg_id or not mailbox_key or not sender_block_ref or not ciphertext:
             return {"ok": False, "detail": "envelope missing required fields"}
 
@@ -1662,7 +1858,6 @@ class DMRelay:
             # Same per-class cap as the deposit path — defense in depth
             # against a peer that wraps a "deposit" as a "replica" to
             # bypass the class limit.
-            delivery_class = str(envelope.get("delivery_class", "") or "")
             if delivery_class in ("request", "shared", "self"):
                 class_limit = self._mailbox_limit_for_class(delivery_class)
             else:
@@ -1716,82 +1911,18 @@ class DMRelay:
         self,
         *,
         envelope: dict[str, Any],
+        preferred_peer_urls: list[str] | None = None,
     ) -> None:
-        """Push an outbound DM envelope to every authenticated relay peer.
-
-        Fire-and-forget: spawned in a background thread so ``deposit``
-        returns to the caller immediately. Per-peer errors are logged
-        and swallowed — the sender's UX must not block on slow Tor
-        peers, and a peer that's down today gets the next message
-        whenever it comes back. Inbound recipient polling from a healthy
-        peer keeps the system functional during peer failures.
-
-        Each peer is authed with the existing per-peer HMAC pattern
-        (#256) — same headers and key resolver gate-message replication
-        uses, so a hostile node that doesn't know any peer's HMAC key
-        can't impersonate a legitimate relay.
-        """
+        """Fire-and-forget fleet-wide replicate push (non-scoped traffic)."""
         import threading
 
-        def _do_push():
+        def _do_push() -> None:
             try:
-                import hashlib
-                import hmac
-                import requests as _requests
-
-                from services.mesh.mesh_crypto import (
-                    normalize_peer_url,
-                    resolve_peer_key_for_url,
+                self._replicate_envelope_to_peers(
+                    envelope=envelope,
+                    preferred_peer_urls=preferred_peer_urls,
                 )
-                from services.mesh.mesh_router import (
-                    authenticated_push_peer_urls,
-                )
-
-                peers = authenticated_push_peer_urls()
-                if not peers:
-                    return
-
-                payload = json.dumps(
-                    {"envelope": envelope},
-                    separators=(",", ":"),
-                    ensure_ascii=False,
-                ).encode("utf-8")
-
-                timeout = max(
-                    1,
-                    int(getattr(self._settings(), "MESH_RELAY_PUSH_TIMEOUT_S", 10) or 10),
-                )
-
-                for peer_url in peers:
-                    try:
-                        normalized = normalize_peer_url(peer_url)
-                        headers = {"Content-Type": "application/json"}
-                        peer_key = resolve_peer_key_for_url(normalized)
-                        if peer_key:
-                            headers["X-Peer-Url"] = normalized
-                            headers["X-Peer-HMAC"] = hmac.new(
-                                peer_key, payload, hashlib.sha256
-                            ).hexdigest()
-                        url = f"{peer_url}/api/mesh/dm/replicate-envelope"
-                        resp = _requests.post(
-                            url, data=payload, timeout=timeout, headers=headers,
-                        )
-                        if resp.status_code == 200:
-                            metrics_inc("dm_replication_push_ok")
-                        else:
-                            # 4xx including the structured cap_violation
-                            # rejection from accept_replica — sender's
-                            # relay learns and stops retrying this msg_id.
-                            metrics_inc("dm_replication_push_rejected")
-                    except Exception:
-                        # Per-peer failure is non-fatal — log to metrics
-                        # but don't break the loop. Other peers and a
-                        # future retry can still propagate the envelope.
-                        metrics_inc("dm_replication_push_error")
-                        continue
             except Exception:
-                # Outer guard — never let replication errors propagate
-                # back to the sender's deposit() caller.
                 metrics_inc("dm_replication_push_error")
 
         thread = threading.Thread(
@@ -1806,6 +1937,8 @@ class DMRelay:
         *,
         mailbox_key: str,
         msg_id: str,
+        recipient_id: str = "",
+        recipient_token: str | None = None,
     ) -> dict[str, Any] | None:
         """Return the wire-form envelope for a stored message, suitable
         for POSTing to a peer relay's replicate-envelope endpoint.
@@ -1822,6 +1955,8 @@ class DMRelay:
                     return {
                         "msg_id": m.msg_id,
                         "mailbox_key": mailbox_key,
+                        "recipient_id": str(recipient_id or "").strip(),
+                        "recipient_token": str(recipient_token or "").strip(),
                         "sender_id": m.sender_id,
                         "sender_block_ref": m.sender_block_ref,
                         "sender_seal": m.sender_seal,

backend/services/mesh/mesh_fleet_defaults.py

@@ -0,0 +1,64 @@
+"""Public Infonet fleet defaults for sb-testnet-0 participants.
+
+Operators who run private single-node installs can set ``MESH_INFONET_FLEET_JOIN=false``
+and provide their own signer keys / peer secrets.
+"""
+
+from __future__ import annotations
+
+FLEET_NETWORK_ID = "sb-testnet-0"
+FLEET_SEED_ONION_URL = (
+    "http://gqpbunqbgtkcqilvclm3xrkt3zowjyl3s62kkktvojgvxzizamvbrqid.onion:8000"
+)
+FLEET_BOOTSTRAP_SIGNER_PUBLIC_KEY_B64 = (
+    "ul1d0kj/ODPIp0OhHzX8eLAVXzJ3CVvzW1vn2IC6q3I="
+)
+# Shared fleet HMAC for sb-testnet peer announce/push/sync. Public testnet join model.
+FLEET_PEER_PUSH_SECRET = "b7GoqsvoUD9MV7tyt0ZOzMptLA84QG6KCfaV9nDqz5Y"
+
+
+def infonet_fleet_join_enabled() -> bool:
+    try:
+        from services.config import get_settings
+
+        if bool(getattr(get_settings(), "MESH_INFONET_FLEET_JOIN_DISABLED", False)):
+            return False
+        return bool(getattr(get_settings(), "MESH_INFONET_FLEET_JOIN", True))
+    except Exception:
+        return True
+
+
+def effective_bootstrap_signer_public_key_b64() -> str:
+    try:
+        from services.config import get_settings
+
+        configured = str(getattr(get_settings(), "MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY", "") or "").strip()
+        if configured:
+            return configured
+    except Exception:
+        pass
+    if infonet_fleet_join_enabled():
+        return FLEET_BOOTSTRAP_SIGNER_PUBLIC_KEY_B64
+    return ""
+
+
+def effective_peer_push_secret() -> str:
+    try:
+        from services.config import get_settings
+
+        configured = str(getattr(get_settings(), "MESH_PEER_PUSH_SECRET", "") or "").strip()
+        if configured:
+            return configured
+    except Exception:
+        pass
+    if infonet_fleet_join_enabled():
+        return FLEET_PEER_PUSH_SECRET
+    return ""
+
+
+def configured_bootstrap_seed_peers_with_fleet_default(peers: list[str]) -> list[str]:
+    if peers:
+        return peers
+    if infonet_fleet_join_enabled():
+        return [FLEET_SEED_ONION_URL]
+    return []

backend/services/mesh/mesh_hashchain.py

@@ -33,8 +33,9 @@ Each event contains:
 
 Persistence: JSON file at backend/data/infonet.json
 
-Encrypted gate chat events are intentionally kept off the public chain and
-persisted separately via GateMessageStore.
+Encrypted gate chat events are private-chain ciphertext records. They are
+excluded from public read surfaces and replicated only over private Infonet
+transports.
 """
 
 import json
@@ -64,6 +65,8 @@ from services.mesh.mesh_schema import (
     ACTIVE_PUBLIC_LEDGER_EVENT_TYPES,
     PUBLIC_LEDGER_EVENT_TYPES,
     validate_event_payload,
+    validate_private_dm_ledger_payload,
+    validate_private_gate_ledger_payload,
     validate_protocol_fields,
     validate_public_ledger_payload,
 )
@@ -127,6 +130,12 @@ GATE_SEGMENT_MAX_COMPRESSED_BYTES = max(
     int(os.environ.get("MESH_GATE_SEGMENT_MAX_COMPRESSED_BYTES", str(2 * 1024 * 1024)) or str(2 * 1024 * 1024)),
 )
 GATE_SEGMENT_STORAGE_VERSION = 1
+DM_HASHCHAIN_SPOOL_LIMIT = max(1, int(os.environ.get("MESH_DM_HASHCHAIN_SPOOL_LIMIT", "2") or "2"))
+DM_HASHCHAIN_SPOOL_SENDER_LIMIT = max(
+    1,
+    int(os.environ.get("MESH_DM_HASHCHAIN_SPOOL_SENDER_LIMIT", "1") or "1"),
+)
+DM_HASHCHAIN_SPOOL_TTL_S = max(60, int(os.environ.get("MESH_DM_HASHCHAIN_SPOOL_TTL_S", "3600") or "3600"))
 _PUBLIC_EVENT_APPEND_HOOKS: list[Any] = []
 _PUBLIC_EVENT_APPEND_HOOKS_LOCK = threading.Lock()
 
@@ -340,6 +349,32 @@ def _private_gate_event_id(
     ).hexdigest()
 
 
+def _private_gate_signature_payload_variants(gate_id: str, event: dict[str, Any]) -> list[dict[str, Any]]:
+    payload = _private_gate_signature_payload(gate_id, event)
+    variants: list[dict[str, Any]] = [payload]
+    event_payload = event.get("payload") if isinstance(event.get("payload"), dict) else {}
+    reply_to = str(event_payload.get("reply_to", "") or "").strip()
+    if reply_to:
+        variants.append(_private_gate_signature_payload(gate_id, event, include_reply_to=False))
+    if "epoch" in payload:
+        no_epoch = dict(payload)
+        no_epoch.pop("epoch", None)
+        variants.append(no_epoch)
+        if reply_to:
+            no_epoch_no_reply = _private_gate_signature_payload(gate_id, event, include_reply_to=False)
+            no_epoch_no_reply.pop("epoch", None)
+            variants.append(no_epoch_no_reply)
+    deduped: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for variant in variants:
+        material = json.dumps(variant, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+        if material in seen:
+            continue
+        seen.add(material)
+        deduped.append(variant)
+    return deduped
+
+
 def _sanitize_private_gate_event(gate_id: str, event: dict[str, Any]) -> dict[str, Any]:
     payload = event.get("payload") if isinstance(event.get("payload"), dict) else {}
     sanitized = {
@@ -1568,11 +1603,18 @@ class Infonet:
     def _rebuild_state(self) -> None:
         self.event_index = {}
         self.node_sequences = {}
-        # Keep private signed-write replay domains across public-chain
-        # rebuilds; these domains protect local side effects that are not
-        # represented as public Infonet events.
-        if not isinstance(getattr(self, "sequence_domains", None), dict):
-            self.sequence_domains = {}
+        # Keep private signed-write replay domains that are not represented
+        # on-chain, but rebuild the gate_message sequence domain from chain
+        # events so reloads/fork application do not mix it with public
+        # per-node message sequences.
+        preserved_domains = {}
+        if isinstance(getattr(self, "sequence_domains", None), dict):
+            preserved_domains = {
+                key: value
+                for key, value in self.sequence_domains.items()
+                if not str(key or "").endswith("|gate_message")
+            }
+        self.sequence_domains = dict(preserved_domains)
         self.public_key_bindings = {}
         self.revocations = {}
         self._replay_filter = ReplayFilter()
@@ -1584,9 +1626,12 @@ class Infonet:
             node_id = evt.get("node_id", "")
             sequence = _safe_int(evt.get("sequence", 0) or 0, 0)
             if node_id and sequence:
-                last = self.node_sequences.get(node_id, 0)
+                sequence_table, sequence_key = self._sequence_table_for_event(
+                    evt.get("event_type", ""), node_id
+                )
+                last = sequence_table.get(sequence_key, 0)
                 if sequence > last:
-                    self.node_sequences[node_id] = sequence
+                    sequence_table[sequence_key] = sequence
             public_key = str(evt.get("public_key", "") or "")
             if public_key and node_id:
                 existing = self.public_key_bindings.get(public_key)
@@ -1898,6 +1943,295 @@ class Infonet:
         self._save()
         return True, "ok"
 
+    def _sequence_table_for_event(self, event_type: str, node_id: str) -> tuple[dict[str, int], str]:
+        normalized = str(event_type or "").strip().lower()
+        if normalized == "gate_message":
+            return self.sequence_domains, f"{node_id}|gate_message"
+        if normalized == "dm_message":
+            return self.sequence_domains, f"{node_id}|dm_message"
+        return self.node_sequences, node_id
+
+    def _dm_spool_target_key(self, payload: dict[str, Any]) -> tuple[str, str]:
+        delivery_class = str(payload.get("delivery_class", "") or "").strip().lower()
+        if delivery_class == "shared":
+            key = str(payload.get("recipient_token", "") or "").strip()
+        else:
+            key = str(payload.get("recipient_id", "") or "").strip()
+        return delivery_class, key
+
+    def _dm_spool_active_counts(
+        self,
+        payload: dict[str, Any],
+        *,
+        sender_id: str = "",
+        now: float | None = None,
+    ) -> tuple[int, int]:
+        delivery_class, key = self._dm_spool_target_key(payload)
+        if not key:
+            return 0, 0
+        sender_id = str(sender_id or "").strip()
+        current = time.time() if now is None else float(now)
+        total_count = 0
+        sender_count = 0
+        for evt in reversed(self.events):
+            if evt.get("event_type") != "dm_message":
+                continue
+            evt_payload = evt.get("payload") if isinstance(evt.get("payload"), dict) else {}
+            evt_delivery_class, evt_key = self._dm_spool_target_key(evt_payload)
+            if evt_delivery_class != delivery_class:
+                continue
+            if evt_key != key:
+                continue
+            evt_ts = float(evt_payload.get("timestamp", evt.get("timestamp", 0)) or 0)
+            if evt_ts > 0 and current - evt_ts > DM_HASHCHAIN_SPOOL_TTL_S:
+                continue
+            total_count += 1
+            if sender_id and str(evt.get("node_id", "") or "").strip() == sender_id:
+                sender_count += 1
+            if total_count >= DM_HASHCHAIN_SPOOL_LIMIT and (
+                not sender_id or sender_count >= DM_HASHCHAIN_SPOOL_SENDER_LIMIT
+            ):
+                break
+        return total_count, sender_count
+
+    def _dm_spool_active_count(self, payload: dict[str, Any], *, now: float | None = None) -> int:
+        total_count, _sender_count = self._dm_spool_active_counts(payload, now=now)
+        return total_count
+
+    def append_private_dm_message(
+        self,
+        *,
+        node_id: str,
+        payload: dict,
+        signature: str,
+        sequence: int,
+        public_key: str,
+        public_key_algo: str,
+        protocol_version: str = "",
+        timestamp: float = 0,
+    ) -> dict:
+        """Append an encrypted DM dead-drop message to the private Infonet ledger.
+
+        The event is a small offline spool, capped per mailbox target, so the
+        hashchain can carry a couple of sealed DMs without becoming an
+        unbounded global mailbox.
+        """
+        event_type = "dm_message"
+        if sequence <= 0:
+            raise ValueError("sequence is required and must be > 0")
+        sequence_table, sequence_key = self._sequence_table_for_event(event_type, node_id)
+        last = sequence_table.get(sequence_key, 0)
+        if sequence <= last:
+            raise ValueError(f"Replay detected: sequence {sequence} <= last {last}")
+
+        raw_payload = dict(payload or {})
+        if "message" in raw_payload or "plaintext" in raw_payload or "_local_plaintext" in raw_payload:
+            raise ValueError("private DM ledger payload must not contain plaintext")
+        if str(raw_payload.get("transport_lock", "") or "").strip().lower() != "private_strong":
+            raise ValueError("DM hashchain spool requires private_strong transport_lock")
+
+        payload = normalize_payload(event_type, raw_payload)
+        ok, reason = validate_private_dm_ledger_payload(payload)
+        if not ok:
+            raise ValueError(reason)
+        total_count, sender_count = self._dm_spool_active_counts(payload, sender_id=node_id)
+        if sender_count >= DM_HASHCHAIN_SPOOL_SENDER_LIMIT:
+            raise ValueError("DM hashchain sender spool full for recipient")
+        if total_count >= DM_HASHCHAIN_SPOOL_LIMIT:
+            raise ValueError("DM hashchain spool full for recipient")
+
+        payload_json = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+        if len(payload_json.encode("utf-8")) > MAX_PAYLOAD_BYTES:
+            raise ValueError("payload exceeds max size")
+
+        protocol_version = str(protocol_version or PROTOCOL_VERSION)
+        ok, reason = validate_protocol_fields(protocol_version, NETWORK_ID)
+        if not ok:
+            raise ValueError(reason)
+
+        if not (signature and public_key and public_key_algo):
+            raise ValueError("Missing signature fields")
+        algo = parse_public_key_algo(public_key_algo)
+        if not algo:
+            raise ValueError("Unsupported public_key_algo")
+        if not verify_node_binding(node_id, public_key):
+            raise ValueError("node_id mismatch")
+        bound, bind_reason = self._bind_public_key(public_key, node_id)
+        if not bound:
+            raise ValueError(bind_reason)
+        sig_payload = build_signature_payload(
+            event_type=event_type,
+            node_id=node_id,
+            sequence=sequence,
+            payload=payload,
+        )
+        if not verify_signature(
+            public_key_b64=public_key,
+            public_key_algo=public_key_algo,
+            signature_hex=signature,
+            payload=sig_payload,
+        ):
+            raise ValueError("Invalid signature")
+
+        revoked, _info = self._revocation_status(public_key)
+        if revoked:
+            raise ValueError("public key is revoked")
+
+        event = ChainEvent(
+            prev_hash=self.head_hash,
+            event_type=event_type,
+            node_id=node_id,
+            payload=payload,
+            timestamp=float(timestamp or time.time()),
+            sequence=sequence,
+            signature=signature,
+            public_key=public_key,
+            public_key_algo=public_key_algo,
+            protocol_version=protocol_version,
+        )
+        event_dict = event.to_dict()
+        self._write_wal(event_dict)
+        self.events.append(event_dict)
+        self.event_index[event.event_id] = len(self.events) - 1
+        self.head_hash = event.event_id
+        sequence_table[sequence_key] = sequence
+        self._replay_filter.add(event.event_id)
+        self._invalidate_merkle_cache()
+        self._update_counters_for_event(event_dict)
+        self._save()
+
+        try:
+            from services.mesh.mesh_rns import rns_bridge
+
+            rns_bridge.publish_event(event_dict)
+        except Exception:
+            pass
+        _notify_public_event_append_hooks(event_dict)
+        logger.info(
+            f"Infonet append [dm_message] by {_redact_node(node_id)} seq={sequence} "
+            f"id={event.event_id[:16]}..."
+        )
+        return event_dict
+
+    def append_private_gate_message(
+        self,
+        *,
+        node_id: str,
+        payload: dict,
+        signature: str,
+        sequence: int,
+        public_key: str,
+        public_key_algo: str,
+        protocol_version: str = "",
+        timestamp: float = 0,
+    ) -> dict:
+        """Append an encrypted gate message to the private Infonet ledger.
+
+        Gate messages use their own sequence domain so a gate post cannot
+        consume or replay-block the author's public broadcast sequence.
+        """
+        event_type = "gate_message"
+        if sequence <= 0:
+            raise ValueError("sequence is required and must be > 0")
+        sequence_table, sequence_key = self._sequence_table_for_event(event_type, node_id)
+        last = sequence_table.get(sequence_key, 0)
+        if sequence <= last:
+            raise ValueError(f"Replay detected: sequence {sequence} <= last {last}")
+
+        raw_payload = dict(payload or {})
+        if "message" in raw_payload or "_local_plaintext" in raw_payload or "_local_reply_to" in raw_payload:
+            raise ValueError("private gate ledger payload must not contain plaintext")
+        if str(raw_payload.get("transport_lock", "") or "").strip().lower() != "private_strong":
+            raise ValueError("gate messages require private_strong transport_lock")
+
+        payload = normalize_payload(event_type, raw_payload)
+        ok, reason = validate_private_gate_ledger_payload(payload)
+        if not ok:
+            raise ValueError(reason)
+
+        payload_json = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+        if len(payload_json.encode("utf-8")) > MAX_PAYLOAD_BYTES:
+            raise ValueError("payload exceeds max size")
+
+        protocol_version = str(protocol_version or PROTOCOL_VERSION)
+        ok, reason = validate_protocol_fields(protocol_version, NETWORK_ID)
+        if not ok:
+            raise ValueError(reason)
+
+        if not (signature and public_key and public_key_algo):
+            raise ValueError("Missing signature fields")
+        algo = parse_public_key_algo(public_key_algo)
+        if not algo:
+            raise ValueError("Unsupported public_key_algo")
+        if not verify_node_binding(node_id, public_key):
+            raise ValueError("node_id mismatch")
+        bound, bind_reason = self._bind_public_key(public_key, node_id)
+        if not bound:
+            raise ValueError(bind_reason)
+        event_for_signature = {"payload": payload}
+        signature_ok = False
+        for signature_payload in _private_gate_signature_payload_variants(
+            str(payload.get("gate", "") or ""),
+            event_for_signature,
+        ):
+            sig_payload = build_signature_payload(
+                event_type=event_type,
+                node_id=node_id,
+                sequence=sequence,
+                payload=signature_payload,
+            )
+            if verify_signature(
+                public_key_b64=public_key,
+                public_key_algo=public_key_algo,
+                signature_hex=signature,
+                payload=sig_payload,
+            ):
+                signature_ok = True
+                break
+        if not signature_ok:
+            raise ValueError("Invalid signature")
+
+        revoked, _info = self._revocation_status(public_key)
+        if revoked:
+            raise ValueError("public key is revoked")
+
+        event = ChainEvent(
+            prev_hash=self.head_hash,
+            event_type=event_type,
+            node_id=node_id,
+            payload=payload,
+            timestamp=float(timestamp or time.time()),
+            sequence=sequence,
+            signature=signature,
+            public_key=public_key,
+            public_key_algo=public_key_algo,
+            protocol_version=protocol_version,
+        )
+        event_dict = event.to_dict()
+        self._write_wal(event_dict)
+        self.events.append(event_dict)
+        self.event_index[event.event_id] = len(self.events) - 1
+        self.head_hash = event.event_id
+        sequence_table[sequence_key] = sequence
+        self._replay_filter.add(event.event_id)
+        self._invalidate_merkle_cache()
+        self._update_counters_for_event(event_dict)
+        self._save()
+
+        try:
+            from services.mesh.mesh_rns import rns_bridge
+
+            rns_bridge.publish_event(event_dict)
+        except Exception:
+            pass
+        _notify_public_event_append_hooks(event_dict)
+
+        logger.info(
+            f"Infonet append [gate_message] by {_redact_node(node_id)} seq={sequence} "
+            f"id={event.event_id[:16]}..."
+        )
+        return event_dict
+
     def append(
         self,
         event_type: str,
@@ -2078,6 +2412,18 @@ class Infonet:
             if not event_id or not prev_hash:
                 rejected.append({"index": idx, "reason": "Missing event_id or prev_hash"})
                 continue
+            if event_id in self.event_index:
+                duplicates += 1
+                continue
+            if self._replay_filter.seen(event_id):
+                try:
+                    from services.mesh.mesh_metrics import increment as metrics_inc
+
+                    metrics_inc("ingest_replay_seen")
+                except Exception:
+                    pass
+                duplicates += 1
+                continue
             if prev_hash != expected_prev:
                 try:
                     from services.mesh.mesh_metrics import increment as metrics_inc
@@ -2096,25 +2442,14 @@ class Infonet:
                     pass
                 rejected.append({"index": idx, "reason": "network_id mismatch"})
                 continue
-            if event_id in self.event_index:
-                duplicates += 1
-                continue
-            if self._replay_filter.seen(event_id):
-                try:
-                    from services.mesh.mesh_metrics import increment as metrics_inc
-
-                    metrics_inc("ingest_replay_seen")
-                except Exception:
-                    pass
-                duplicates += 1
-                continue
             if prev_hash != self.head_hash:
                 rejected.append({"index": idx, "reason": "prev_hash does not match head"})
                 continue
             if sequence <= 0:
                 rejected.append({"index": idx, "reason": "Invalid sequence"})
                 continue
-            last = self.node_sequences.get(node_id, 0)
+            sequence_table, sequence_key = self._sequence_table_for_event(event_type, node_id)
+            last = sequence_table.get(sequence_key, 0)
             if sequence <= last:
                 rejected.append({"index": idx, "reason": "Replay detected"})
                 continue
@@ -2149,7 +2484,18 @@ class Infonet:
             if not ok:
                 rejected.append({"index": idx, "reason": reason})
                 continue
-            ok, reason = validate_public_ledger_payload(event_type, payload)
+            if event_type == "gate_message":
+                ok, reason = validate_private_gate_ledger_payload(payload)
+            elif event_type == "dm_message":
+                ok, reason = validate_private_dm_ledger_payload(payload)
+                if ok:
+                    total_count, sender_count = self._dm_spool_active_counts(payload, sender_id=str(evt.get("node_id", "") or ""))
+                    if sender_count >= DM_HASHCHAIN_SPOOL_SENDER_LIMIT:
+                        ok, reason = False, "DM hashchain sender spool full for recipient"
+                    elif total_count >= DM_HASHCHAIN_SPOOL_LIMIT:
+                        ok, reason = False, "DM hashchain spool full for recipient"
+            else:
+                ok, reason = validate_public_ledger_payload(event_type, payload)
             if not ok:
                 rejected.append({"index": idx, "reason": reason})
                 continue
@@ -2225,7 +2571,7 @@ class Infonet:
                     pass
                 rejected.append({"index": idx, "reason": "public key is revoked"})
                 continue
-            last_seq = self.node_sequences.get(node_id, 0)
+            last_seq = sequence_table.get(sequence_key, 0)
             if sequence <= last_seq:
                 try:
                     from services.mesh.mesh_metrics import increment as metrics_inc
@@ -2261,18 +2607,30 @@ class Infonet:
                 rejected.append({"index": idx, "reason": bind_reason})
                 continue
 
-            sig_payload = build_signature_payload(
-                event_type=event_type,
-                node_id=node_id,
-                sequence=sequence,
-                payload=payload,
-            )
-            if not verify_signature(
-                public_key_b64=public_key,
-                public_key_algo=public_key_algo,
-                signature_hex=signature,
-                payload=sig_payload,
-            ):
+            if event_type == "gate_message":
+                signature_payloads = _private_gate_signature_payload_variants(
+                    str(payload.get("gate", "") or ""),
+                    evt,
+                )
+            else:
+                signature_payloads = [payload]
+            signature_ok = False
+            for signature_payload in signature_payloads:
+                sig_payload = build_signature_payload(
+                    event_type=event_type,
+                    node_id=node_id,
+                    sequence=sequence,
+                    payload=signature_payload,
+                )
+                if verify_signature(
+                    public_key_b64=public_key,
+                    public_key_algo=public_key_algo,
+                    signature_hex=signature,
+                    payload=sig_payload,
+                ):
+                    signature_ok = True
+                    break
+            if not signature_ok:
                 try:
                     from services.mesh.mesh_metrics import increment as metrics_inc
 
@@ -2302,7 +2660,7 @@ class Infonet:
             self.events.append(evt)
             self.event_index[event_id] = len(self.events) - 1
             self.head_hash = event_id
-            self.node_sequences[node_id] = sequence
+            sequence_table[sequence_key] = sequence
             self._update_counters_for_event(evt)
             accepted += 1
             expected_prev = event_id
@@ -2365,6 +2723,7 @@ class Infonet:
                     verify_node_binding,
                 )
 
+                event_type = evt_dict.get("event_type", "")
                 node_id = evt_dict.get("node_id", "")
                 if not parse_public_key_algo(public_key_algo):
                     return False, f"Unsupported public_key_algo at index {i}"
@@ -2375,21 +2734,41 @@ class Infonet:
                     return False, f"public key binding conflict at index {i}"
                 seen_public_keys[public_key] = node_id
 
-                normalized = normalize_payload(
-                    evt_dict.get("event_type", ""), evt_dict.get("payload", {})
-                )
-                sig_payload = build_signature_payload(
-                    event_type=evt_dict.get("event_type", ""),
-                    node_id=node_id,
-                    sequence=_safe_int(evt_dict.get("sequence", 0) or 0, 0),
-                    payload=normalized,
-                )
-                if not verify_signature(
-                    public_key_b64=public_key,
-                    public_key_algo=public_key_algo,
-                    signature_hex=signature,
-                    payload=sig_payload,
-                ):
+                payload = evt_dict.get("payload", {})
+                if event_type == "gate_message":
+                    ok, reason = validate_private_gate_ledger_payload(payload)
+                    if not ok:
+                        return False, f"Invalid gate_message payload at index {i}: {reason}"
+                    signature_payloads = _private_gate_signature_payload_variants(
+                        str(payload.get("gate", "") or ""),
+                        evt_dict,
+                    )
+                elif event_type == "dm_message":
+                    ok, reason = validate_private_dm_ledger_payload(payload)
+                    if not ok:
+                        return False, f"Invalid dm_message payload at index {i}: {reason}"
+                    signature_payloads = [normalize_payload(event_type, payload)]
+                else:
+                    signature_payloads = [
+                        normalize_payload(event_type, payload)
+                    ]
+                signature_ok = False
+                for signature_payload in signature_payloads:
+                    sig_payload = build_signature_payload(
+                        event_type=event_type,
+                        node_id=node_id,
+                        sequence=_safe_int(evt_dict.get("sequence", 0) or 0, 0),
+                        payload=signature_payload,
+                    )
+                    if verify_signature(
+                        public_key_b64=public_key,
+                        public_key_algo=public_key_algo,
+                        signature_hex=signature,
+                        payload=sig_payload,
+                    ):
+                        signature_ok = True
+                        break
+                if not signature_ok:
                     return False, f"Invalid signature at index {i}"
 
             prev = evt_dict["event_id"]
@@ -2454,27 +2833,48 @@ class Infonet:
                     verify_node_binding,
                 )
 
+                event_type = evt_dict.get("event_type", "")
                 node_id = evt_dict.get("node_id", "")
                 if not parse_public_key_algo(public_key_algo):
                     return False, f"Unsupported public_key_algo at index {i}"
                 if not verify_node_binding(node_id, public_key):
                     return False, f"node_id mismatch at index {i}"
 
-                normalized = normalize_payload(
-                    evt_dict.get("event_type", ""), evt_dict.get("payload", {})
-                )
-                sig_payload = build_signature_payload(
-                    event_type=evt_dict.get("event_type", ""),
-                    node_id=node_id,
-                    sequence=_safe_int(evt_dict.get("sequence", 0) or 0, 0),
-                    payload=normalized,
-                )
-                if not verify_signature(
-                    public_key_b64=public_key,
-                    public_key_algo=public_key_algo,
-                    signature_hex=signature,
-                    payload=sig_payload,
-                ):
+                payload = evt_dict.get("payload", {})
+                if event_type == "gate_message":
+                    ok, reason = validate_private_gate_ledger_payload(payload)
+                    if not ok:
+                        return False, f"Invalid gate_message payload at index {i}: {reason}"
+                    signature_payloads = _private_gate_signature_payload_variants(
+                        str(payload.get("gate", "") or ""),
+                        evt_dict,
+                    )
+                elif event_type == "dm_message":
+                    ok, reason = validate_private_dm_ledger_payload(payload)
+                    if not ok:
+                        return False, f"Invalid dm_message payload at index {i}: {reason}"
+                    signature_payloads = [normalize_payload(event_type, payload)]
+                else:
+                    signature_payloads = [
+                        normalize_payload(event_type, payload)
+                    ]
+                signature_ok = False
+                for signature_payload in signature_payloads:
+                    sig_payload = build_signature_payload(
+                        event_type=event_type,
+                        node_id=node_id,
+                        sequence=_safe_int(evt_dict.get("sequence", 0) or 0, 0),
+                        payload=signature_payload,
+                    )
+                    if verify_signature(
+                        public_key_b64=public_key,
+                        public_key_algo=public_key_algo,
+                        signature_hex=signature,
+                        payload=sig_payload,
+                    ):
+                        signature_ok = True
+                        break
+                if not signature_ok:
                     return False, f"Invalid signature at index {i}"
             prev = evt_dict["event_id"]
 
@@ -2538,7 +2938,14 @@ class Infonet:
             node_id = evt.get("node_id", "")
             sequence = _safe_int(evt.get("sequence", 0) or 0, 0)
             if node_id and sequence:
-                last_seq[node_id] = max(last_seq.get(node_id, 0), sequence)
+                sequence_key = (
+                    f"{node_id}|gate_message"
+                    if str(evt.get("event_type", "") or "").strip().lower() == "gate_message"
+                    else f"{node_id}|dm_message"
+                    if str(evt.get("event_type", "") or "").strip().lower() == "dm_message"
+                    else node_id
+                )
+                last_seq[sequence_key] = max(last_seq.get(sequence_key, 0), sequence)
             public_key = str(evt.get("public_key", "") or "")
             if public_key and node_id:
                 seen_public_keys.setdefault(public_key, node_id)
@@ -2558,8 +2965,21 @@ class Infonet:
             existing_idx = self.event_index.get(event_id)
             if existing_idx is not None and existing_idx <= prev_index:
                 return False, "duplicate event_id"
-            payload = normalize_payload(event_type, dict(payload or {}))
+            if event_type == "gate_message":
+                payload = dict(payload or {})
+            elif event_type == "dm_message":
+                payload = normalize_payload(event_type, dict(payload or {}))
+            else:
+                payload = normalize_payload(event_type, dict(payload or {}))
             ok, reason = validate_event_payload(event_type, payload)
+            if not ok:
+                return False, reason
+            if event_type == "gate_message":
+                ok, reason = validate_private_gate_ledger_payload(payload)
+            elif event_type == "dm_message":
+                ok, reason = validate_private_dm_ledger_payload(payload)
+            else:
+                ok, reason = validate_public_ledger_payload(event_type, payload)
             if not ok:
                 return False, reason
             proto = evt.get("protocol_version") or PROTOCOL_VERSION
@@ -2573,7 +2993,14 @@ class Infonet:
             revoked, _info = self._revocation_status(public_key)
             if revoked and event_type != "key_revoke":
                 return False, "public key revoked"
-            last = last_seq.get(node_id, 0)
+            sequence_key = (
+                f"{node_id}|gate_message"
+                if event_type == "gate_message"
+                else f"{node_id}|dm_message"
+                if event_type == "dm_message"
+                else node_id
+            )
+            last = last_seq.get(sequence_key, 0)
             if sequence <= last:
                 return False, "sequence replay"
             from services.mesh.mesh_crypto import (
@@ -2591,23 +3018,35 @@ class Infonet:
             if existing and existing != node_id:
                 return False, "public key binding conflict"
             seen_public_keys[public_key] = node_id
-            sig_payload = build_signature_payload(
-                event_type=event_type,
-                node_id=node_id,
-                sequence=sequence,
-                payload=payload,
-            )
-            if not verify_signature(
-                public_key_b64=public_key,
-                public_key_algo=public_key_algo,
-                signature_hex=signature,
-                payload=sig_payload,
-            ):
+            if event_type == "gate_message":
+                signature_payloads = _private_gate_signature_payload_variants(
+                    str(payload.get("gate", "") or ""),
+                    evt,
+                )
+            else:
+                signature_payloads = [payload]
+            signature_ok = False
+            for signature_payload in signature_payloads:
+                sig_payload = build_signature_payload(
+                    event_type=event_type,
+                    node_id=node_id,
+                    sequence=sequence,
+                    payload=signature_payload,
+                )
+                if verify_signature(
+                    public_key_b64=public_key,
+                    public_key_algo=public_key_algo,
+                    signature_hex=signature,
+                    payload=sig_payload,
+                ):
+                    signature_ok = True
+                    break
+            if not signature_ok:
                 return False, "invalid signature"
             computed = ChainEvent.from_dict(evt).event_id
             if computed != event_id:
                 return False, "event_id mismatch"
-            last_seq[node_id] = sequence
+            last_seq[sequence_key] = sequence
 
         # Apply fork
         self.events = prefix + ordered

backend/services/mesh/mesh_infonet_relay_bootstrap.py

@@ -0,0 +1,86 @@
+"""Auto-enable Tor wormhole transport on Infonet relay/seed nodes."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from services.config import get_settings
+from services.wormhole_settings import read_wormhole_settings, write_wormhole_settings
+
+logger = logging.getLogger(__name__)
+
+
+def infonet_relay_auto_wormhole_requested() -> bool:
+    settings = get_settings()
+    if bool(settings.MESH_INFONET_RELAY_AUTO_WORMHOLE_DISABLED):
+        return False
+    if bool(settings.MESH_INFONET_RELAY_AUTO_WORMHOLE):
+        return True
+    if str(settings.MESH_BOOTSTRAP_SIGNER_PRIVATE_KEY or "").strip():
+        return True
+    return False
+
+
+def _relay_tor_wormhole_target_settings() -> dict[str, Any]:
+    settings = get_settings()
+    socks_port = int(settings.MESH_ARTI_SOCKS_PORT or 9050)
+    return {
+        "enabled": True,
+        "transport": "tor_arti",
+        "socks_proxy": f"socks5h://127.0.0.1:{socks_port}",
+        "socks_dns": True,
+        "anonymous_mode": True,
+    }
+
+
+def _wormhole_settings_match(existing: dict[str, Any], target: dict[str, Any]) -> bool:
+    return (
+        bool(existing.get("enabled")) is bool(target["enabled"])
+        and str(existing.get("transport", "")) == str(target["transport"])
+        and str(existing.get("socks_proxy", "")) == str(target["socks_proxy"])
+        and bool(existing.get("socks_dns", True)) is bool(target["socks_dns"])
+        and bool(existing.get("anonymous_mode", False)) is bool(target["anonymous_mode"])
+    )
+
+
+def ensure_infonet_relay_wormhole_ready(*, reason: str = "relay_auto") -> dict[str, Any]:
+    """Persist Tor wormhole settings and connect on relay/seed startup."""
+    if not infonet_relay_auto_wormhole_requested():
+        return {"ok": True, "skipped": True, "reason": "not_requested"}
+
+    from routers.ai_intel import _write_env_value
+    from services.tor_hidden_service import tor_service
+    from services.wormhole_supervisor import connect_wormhole, restart_wormhole
+
+    existing = read_wormhole_settings()
+    target = _relay_tor_wormhole_target_settings()
+    settings_updated = not _wormhole_settings_match(existing, target)
+    updated = write_wormhole_settings(**target) if settings_updated else existing
+
+    tor_result: dict[str, Any] = {"ok": False, "detail": "not started"}
+    try:
+        tor_result = tor_service.start(target_port=8000)
+        if tor_result.get("ok"):
+            _write_env_value("MESH_ARTI_ENABLED", "true")
+            get_settings.cache_clear()
+    except Exception as exc:
+        tor_result = {"ok": False, "detail": str(exc or type(exc).__name__)}
+
+    runtime = (
+        restart_wormhole(reason=reason)
+        if settings_updated
+        else connect_wormhole(reason=reason)
+    )
+
+    if settings_updated:
+        logger.info("Infonet relay auto-wormhole enabled (%s)", reason)
+
+    return {
+        "ok": True,
+        "skipped": False,
+        "settings_updated": settings_updated,
+        "tor": tor_result,
+        "runtime": runtime,
+        "settings": updated,
+    }

backend/services/mesh/mesh_infonet_sync_support.py

@@ -276,5 +276,6 @@ def should_run_sync(
 ) -> bool:
     current_time = int(now if now is not None else time.time())
     if state.last_outcome == "running":
-        return False
+        started_at = int(state.last_sync_started_at or 0)
+        return started_at <= 0 or current_time - started_at >= 300
     return int(state.next_sync_due_at or 0) <= current_time

backend/services/mesh/mesh_metadata_exposure.py

@@ -125,8 +125,8 @@ def dm_lookup_response_view(
             view.pop("lookup_mode", None)
             view.pop("removal_target", None)
             return view
-        if invite_lookup:
-            view.pop("agent_id", None)
+        # Successful invite lookups keep agent_id: the handle is the capability and
+        # first-contact messaging needs a delivery target. Failures stay generic.
     return view
 
 

backend/services/mesh/mesh_peer_registry.py

@@ -0,0 +1,152 @@
+"""Operator-signed peer registry for private Infonet swarm discovery."""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+from services.mesh.mesh_crypto import normalize_peer_url
+from services.mesh.mesh_router import peer_transport_kind
+
+BACKEND_DIR = Path(__file__).resolve().parents[2]
+DATA_DIR = BACKEND_DIR / "data"
+DEFAULT_PEER_REGISTRY_PATH = DATA_DIR / "peer_registry.json"
+REGISTRY_VERSION = 1
+ALLOWED_REGISTRY_ROLES = {"participant", "relay", "seed"}
+
+
+@dataclass
+class RegistryPeer:
+    peer_url: str
+    transport: str
+    role: str
+    node_id: str = ""
+    label: str = ""
+    announced_at: int = 0
+    last_seen_at: int = 0
+    failure_count: int = 0
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    def manifest_peer(self) -> dict[str, str]:
+        return {
+            "peer_url": self.peer_url,
+            "transport": self.transport,
+            "role": self.role,
+            "label": self.label or self.node_id[:16],
+        }
+
+
+class PeerRegistry:
+    def __init__(self, path: str | Path = DEFAULT_PEER_REGISTRY_PATH):
+        self.path = Path(path)
+        self._peers: dict[str, RegistryPeer] = {}
+
+    def load(self) -> list[RegistryPeer]:
+        if not self.path.exists():
+            self._peers = {}
+            return []
+        raw = json.loads(self.path.read_text(encoding="utf-8"))
+        if not isinstance(raw, dict):
+            raise ValueError("peer registry root must be an object")
+        version = int(raw.get("version", 0) or 0)
+        if version != REGISTRY_VERSION:
+            raise ValueError(f"unsupported peer registry version: {version}")
+        entries = raw.get("peers", [])
+        if not isinstance(entries, list):
+            raise ValueError("peer registry peers must be a list")
+        peers: dict[str, RegistryPeer] = {}
+        for entry in entries:
+            if not isinstance(entry, dict):
+                continue
+            peer = self._normalize_entry(entry)
+            peers[peer.peer_url] = peer
+        self._peers = peers
+        return self.records()
+
+    def save(self) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        payload = {
+            "version": REGISTRY_VERSION,
+            "updated_at": int(time.time()),
+            "peers": [peer.to_dict() for peer in self.records()],
+        }
+        self.path.write_text(
+            json.dumps(payload, sort_keys=True, indent=2) + "\n",
+            encoding="utf-8",
+        )
+
+    def records(self) -> list[RegistryPeer]:
+        return sorted(self._peers.values(), key=lambda item: (item.role, item.peer_url))
+
+    def upsert_announcement(
+        self,
+        *,
+        peer_url: str,
+        transport: str,
+        role: str,
+        node_id: str = "",
+        label: str = "",
+        now: float | None = None,
+    ) -> RegistryPeer:
+        normalized = normalize_peer_url(peer_url)
+        if not normalized:
+            raise ValueError("peer_url is required")
+        resolved_transport = str(transport or "").strip().lower() or str(peer_transport_kind(normalized) or "")
+        if resolved_transport not in {"onion", "clearnet"}:
+            raise ValueError("unsupported peer transport")
+        resolved_role = str(role or "participant").strip().lower()
+        if resolved_role not in ALLOWED_REGISTRY_ROLES:
+            raise ValueError("unsupported peer role")
+        timestamp = int(now if now is not None else time.time())
+        existing = self._peers.get(normalized)
+        peer = RegistryPeer(
+            peer_url=normalized,
+            transport=resolved_transport,
+            role=resolved_role,
+            node_id=str(node_id or (existing.node_id if existing else "") or "").strip(),
+            label=str(label or (existing.label if existing else "") or "").strip(),
+            announced_at=int(existing.announced_at if existing and existing.announced_at else timestamp),
+            last_seen_at=timestamp,
+            failure_count=int(existing.failure_count if existing else 0),
+        )
+        self._peers[normalized] = peer
+        return peer
+
+    def prune_stale(self, *, max_age_s: int, now: float | None = None) -> int:
+        timestamp = int(now if now is not None else time.time())
+        removed = 0
+        for peer_url, peer in list(self._peers.items()):
+            if peer.role == "seed":
+                continue
+            last_seen = int(peer.last_seen_at or peer.announced_at or 0)
+            if last_seen > 0 and timestamp - last_seen > max(60, int(max_age_s or 0)):
+                del self._peers[peer_url]
+                removed += 1
+        return removed
+
+    def manifest_peers(self) -> list[dict[str, str]]:
+        return [peer.manifest_peer() for peer in self.records()]
+
+    def _normalize_entry(self, entry: dict[str, Any]) -> RegistryPeer:
+        peer_url = normalize_peer_url(str(entry.get("peer_url", "") or ""))
+        if not peer_url:
+            raise ValueError("registry peer_url is required")
+        transport = str(entry.get("transport", "") or peer_transport_kind(peer_url) or "").strip().lower()
+        role = str(entry.get("role", "participant") or "participant").strip().lower()
+        if role not in ALLOWED_REGISTRY_ROLES:
+            raise ValueError("registry role unsupported")
+        return RegistryPeer(
+            peer_url=peer_url,
+            transport=transport,
+            role=role,
+            node_id=str(entry.get("node_id", "") or "").strip(),
+            label=str(entry.get("label", "") or "").strip(),
+            announced_at=int(entry.get("announced_at", 0) or 0),
+            last_seen_at=int(entry.get("last_seen_at", 0) or entry.get("announced_at", 0) or 0),
+            failure_count=int(entry.get("failure_count", 0) or 0),
+        )

backend/services/mesh/mesh_peer_store.py

@@ -16,7 +16,7 @@ DATA_DIR = BACKEND_DIR / "data"
 DEFAULT_PEER_STORE_PATH = DATA_DIR / "peer_store.json"
 PEER_STORE_VERSION = 1
 ALLOWED_PEER_BUCKETS = {"bootstrap", "sync", "push"}
-ALLOWED_PEER_SOURCES = {"bundle", "operator", "bootstrap_promoted", "runtime"}
+ALLOWED_PEER_SOURCES = {"bundle", "operator", "bootstrap_promoted", "runtime", "swarm"}
 ALLOWED_PEER_TRANSPORTS = {"clearnet", "onion"}
 ALLOWED_PEER_ROLES = {"participant", "relay", "seed"}
 

backend/services/mesh/mesh_privacy_policy.py

@@ -140,10 +140,24 @@ def transport_tier_from_state(state: dict[str, Any] | None) -> str:
     snapshot = state or {}
     if not bool(snapshot.get("configured")):
         return "public_degraded"
-    if not bool(snapshot.get("ready")):
-        return "public_degraded"
     arti_ready = bool(snapshot.get("arti_ready"))
     rns_ready = bool(snapshot.get("rns_ready"))
+    running = bool(snapshot.get("running"))
+    transport_usable = bool(snapshot.get("ready"))
+    if not transport_usable:
+        try:
+            from services.config import get_settings
+
+            if (
+                bool(getattr(get_settings(), "MESH_WORMHOLE_TRUST_FILE_READY", False))
+                and running
+                and arti_ready
+            ):
+                transport_usable = True
+        except Exception:
+            pass
+    if not transport_usable:
+        return "public_degraded"
     if arti_ready and rns_ready:
         return "private_strong"
     if arti_ready or rns_ready:
@@ -157,8 +171,45 @@ def transport_tier_is_sufficient(current_tier: str | None, required_tier: str |
     return TRANSPORT_TIER_ORDER[current] >= TRANSPORT_TIER_ORDER[required]
 
 
-def release_lane_required_tier(lane: str) -> str:
-    return network_release_required_tier(lane)
+_DM_RUNTIME_ENFORCEMENT_ROUTES = {
+    ("POST", "/api/mesh/dm/send"),
+    ("POST", "/api/mesh/dm/poll"),
+    ("GET", "/api/mesh/dm/poll"),
+    ("GET", "/api/mesh/dm/count"),
+    ("POST", "/api/mesh/dm/count"),
+}
+
+
+def runtime_route_enforcement_tier(path: str, method: str, *, static_tier: str) -> str:
+    """Adjust static route tiers for Tor-only nodes that never reach private_strong."""
+    normalized_path = str(path or "").strip()
+    normalized_method = str(method or "").strip().upper()
+    static = normalize_transport_tier(static_tier)
+    if (normalized_method, normalized_path) not in _DM_RUNTIME_ENFORCEMENT_ROUTES:
+        return static
+    if static != "private_strong":
+        return static
+    return release_lane_required_tier("dm")
+
+
+def release_lane_required_tier(lane: str, *, wormhole_state: dict[str, Any] | None = None) -> str:
+    normalized_lane = str(lane or "").strip().lower()
+    required = network_release_required_tier(normalized_lane)
+    if normalized_lane != "dm":
+        return required
+    state = wormhole_state
+    if state is None:
+        try:
+            from services.wormhole_supervisor import get_wormhole_state
+
+            state = get_wormhole_state()
+        except Exception:
+            state = {}
+    # Tor-only nodes never reach private_strong (needs Arti + RNS). Encrypted
+    # relay over Arti still preserves ciphertext privacy for offline delivery.
+    if not bool((state or {}).get("rns_enabled")):
+        return "private_transitional"
+    return required
 
 
 def private_delivery_status(status_code: str, *, reason_code: str = "", plain_reason: str = "") -> dict[str, str]:

backend/services/mesh/mesh_private_dispatcher.py

@@ -386,6 +386,20 @@ def _dispatch_dm(
             sampled=sampled,
         )
 
+    replication_peer_urls: list[str] = []
+    try:
+        from services.mesh.mesh_dm_connect_delivery import relay_push_peer_urls_for_payload
+
+        replication_peer_urls = [
+            str(raw or "").strip().rstrip("/")
+            for raw in list(payload.get("relay_push_peer_urls") or [])
+            if str(raw or "").strip()
+        ]
+        if not replication_peer_urls:
+            replication_peer_urls = relay_push_peer_urls_for_payload(payload)
+    except Exception:
+        replication_peer_urls = []
+
     apply_dm_relay_jitter()
     relay_result = dm_relay.deposit(
         sender_id=relay_sender_id,
@@ -399,7 +413,25 @@ def _dispatch_dm(
         sender_token_hash=sender_token_hash,
         payload_format=payload_format,
         session_welcome=session_welcome,
+        replication_peer_urls=replication_peer_urls,
     )
+    replicate_info = dict(relay_result.get("replicate") or {})
+    if replication_peer_urls and not replicate_info.get("ok"):
+        return _dispatch_result(
+            ok=False,
+            lane="dm",
+            selected_transport="relay",
+            selected_carrier="relay",
+            dispatch_reason="scoped_relay_replicate_failed",
+            hidden_transport_effective=bool(hidden_relay),
+            no_acceptable_path=False,
+            detail=(
+                "Scoped relay replicate did not reach the recipient node: "
+                + str(replicate_info.get("failed") or replicate_info.get("detail") or "unknown")
+            ),
+            msg_id=msg_id,
+            replicate=replicate_info,
+        )
     if not relay_result.get("ok"):
         return _dispatch_result(
             ok=False,
@@ -436,6 +468,7 @@ def _dispatch_dm(
             else str(relay_result.get("detail", "") or "Delivered privately")
         ),
         msg_id=str(relay_result.get("msg_id", "") or msg_id),
+        replicate=replicate_info,
     )
 
 
@@ -600,8 +633,15 @@ def attempt_private_release(
             policy_reason_code=str(decision.reason_code or ""),
         )
     if normalized_lane == "dm":
+        dm_payload = dict(payload or {})
+        try:
+            from services.mesh.mesh_dm_connect_delivery import enrich_connect_release_payload
+
+            dm_payload = enrich_connect_release_payload(dm_payload)
+        except Exception:
+            pass
         return _dispatch_dm(
-            dict(payload or {}),
+            dm_payload,
             secure_dm_enabled=secure_dm_enabled or _secure_dm_enabled,
             rns_private_dm_ready=rns_private_dm_ready or _rns_private_dm_ready,
             anonymous_dm_hidden_transport_enforced=(

backend/services/mesh/mesh_schema.py

@@ -2,6 +2,9 @@
 
 from __future__ import annotations
 
+import base64
+import binascii
+import math
 from dataclasses import dataclass
 from typing import Any, Callable
 
@@ -33,6 +36,88 @@ def _require_fields(payload: dict[str, Any], fields: tuple[str, ...]) -> tuple[b
     return True, "ok"
 
 
+_SEALED_CIPHERTEXT_PREFIXES = ("x3dh1:", "dm1:", "mls1:", "sealed:")
+
+
+def _strip_sealed_ciphertext_prefix(value: str) -> str:
+    lowered = value.lower()
+    for prefix in _SEALED_CIPHERTEXT_PREFIXES:
+        if lowered.startswith(prefix):
+            return value[len(prefix) :]
+    return value
+
+
+def _sealed_ciphertext_has_known_prefix(value: str) -> bool:
+    lowered = str(value or "").strip().lower()
+    return any(lowered.startswith(prefix) for prefix in _SEALED_CIPHERTEXT_PREFIXES)
+
+
+def _decode_base64ish(value: Any) -> bytes | None:
+    raw = str(value or "").strip()
+    if not raw or any(ch.isspace() for ch in raw):
+        return None
+    padded = raw + ("=" * (-len(raw) % 4))
+    for altchars in (None, b"-_"):
+        try:
+            return base64.b64decode(padded.encode("ascii"), altchars=altchars, validate=True)
+        except (binascii.Error, UnicodeEncodeError, ValueError):
+            continue
+    return None
+
+
+def _decode_sealed_ciphertext_value(value: Any) -> bytes | None:
+    raw = str(value or "").strip()
+    if not raw:
+        return None
+    return _decode_base64ish(_strip_sealed_ciphertext_prefix(raw))
+
+
+def _byte_entropy(data: bytes) -> float:
+    if not data:
+        return 0.0
+    counts = [0] * 256
+    for byte in data:
+        counts[byte] += 1
+    total = float(len(data))
+    return -sum((count / total) * math.log2(count / total) for count in counts if count)
+
+
+def _validate_sealed_bytes_field(
+    payload: dict[str, Any],
+    field: str,
+    *,
+    min_bytes: int = 8,
+    entropy_floor: float = 2.5,
+) -> tuple[bool, str]:
+    raw = str(payload.get(field, "") or "").strip()
+    prefixed = _sealed_ciphertext_has_known_prefix(raw)
+    data = _decode_sealed_ciphertext_value(raw)
+    if data is None:
+        return False, f"{field} must be base64-encoded sealed bytes"
+    if len(data) < min_bytes:
+        return False, f"{field} is too short"
+
+    # X3DH / MLS envelopes are structured JSON or ratchet frames — skip
+    # plaintext heuristics once a known wire prefix is present.
+    if prefixed:
+        return True, "ok"
+
+    # Short test vectors and compact envelopes can be low entropy; only apply
+    # heuristics once there is enough material to distinguish a sealed blob
+    # from accidental base64-encoded plaintext.
+    if len(data) >= 32:
+        printable = sum(1 for byte in data if 32 <= byte <= 126 or byte in (9, 10, 13))
+        if printable / len(data) > 0.9:
+            try:
+                data.decode("utf-8")
+                return False, f"{field} looks like encoded plaintext"
+            except UnicodeDecodeError:
+                pass
+        if _byte_entropy(data) < entropy_floor:
+            return False, f"{field} entropy is too low for sealed bytes"
+    return True, "ok"
+
+
 def _validate_message(payload: dict[str, Any]) -> tuple[bool, str]:
     ok, reason = _require_fields(
         payload, ("message", "destination", "channel", "priority", "ephemeral")
@@ -331,6 +416,7 @@ ACTIVE_PUBLIC_LEDGER_EVENT_TYPES: frozenset[str] = frozenset(
 LEGACY_PUBLIC_LEDGER_EVENT_TYPES: frozenset[str] = frozenset(
     {
         "gate_message",
+        "dm_message",
     }
 )
 """Event types that exist historically on the public chain and must remain
@@ -425,6 +511,8 @@ def validate_event_payload(event_type: str, payload: dict[str, Any]) -> tuple[bo
 
 
 def validate_public_ledger_payload(event_type: str, payload: dict[str, Any]) -> tuple[bool, str]:
+    if event_type == "gate_message":
+        return validate_private_gate_ledger_payload(payload)
     if event_type not in PUBLIC_LEDGER_EVENT_TYPES and event_type not in _EXTENSION_VALIDATORS:
         return False, f"{event_type} is not allowed on the public ledger"
     forbidden = sorted(
@@ -441,6 +529,92 @@ def validate_public_ledger_payload(event_type: str, payload: dict[str, Any]) ->
     return True, "ok"
 
 
+_PRIVATE_GATE_LEDGER_ALLOWED_FIELDS: frozenset[str] = frozenset(
+    {
+        "gate",
+        "ciphertext",
+        "nonce",
+        "sender_ref",
+        "format",
+        "epoch",
+        "gate_envelope",
+        "envelope_hash",
+        "reply_to",
+        "transport_lock",
+        "signed_context",
+    }
+)
+
+
+def validate_private_gate_ledger_payload(payload: dict[str, Any]) -> tuple[bool, str]:
+    """Validate ciphertext-only gate events for private Infonet replication."""
+    ok, reason = validate_event_payload("gate_message", payload)
+    if not ok:
+        return ok, reason
+    unexpected = sorted(
+        key
+        for key in payload.keys()
+        if str(key or "").strip().lower() not in _PRIVATE_GATE_LEDGER_ALLOWED_FIELDS
+    )
+    if unexpected:
+        return False, f"private gate ledger payload contains unsupported fields: {', '.join(unexpected)}"
+    if "message" in payload or "_local_plaintext" in payload or "_local_reply_to" in payload:
+        return False, "private gate ledger payload must not contain plaintext"
+    transport_lock = str(payload.get("transport_lock", "") or "").strip().lower()
+    if transport_lock and transport_lock not in {"private", "private_strong", "rns", "onion"}:
+        return False, "gate messages require private transport_lock"
+    ok, reason = _validate_sealed_bytes_field(payload, "ciphertext")
+    if not ok:
+        return ok, reason
+    ok, reason = _validate_sealed_bytes_field(payload, "nonce")
+    if not ok:
+        return ok, reason
+    return True, "ok"
+
+
+_PRIVATE_DM_LEDGER_ALLOWED_FIELDS: frozenset[str] = frozenset(
+    {
+        "recipient_id",
+        "delivery_class",
+        "recipient_token",
+        "ciphertext",
+        "msg_id",
+        "timestamp",
+        "format",
+        "session_welcome",
+        "sender_seal",
+        "relay_salt",
+        "transport_lock",
+        "signed_context",
+    }
+)
+
+
+def validate_private_dm_ledger_payload(payload: dict[str, Any]) -> tuple[bool, str]:
+    """Validate ciphertext-only DM dead-drop events for private Infonet replication."""
+    ok, reason = validate_event_payload("dm_message", payload)
+    if not ok:
+        return ok, reason
+    unexpected = sorted(
+        key
+        for key in payload.keys()
+        if str(key or "").strip().lower() not in _PRIVATE_DM_LEDGER_ALLOWED_FIELDS
+    )
+    if unexpected:
+        return False, f"private DM ledger payload contains unsupported fields: {', '.join(unexpected)}"
+    if "message" in payload or "plaintext" in payload or "_local_plaintext" in payload:
+        return False, "private DM ledger payload must not contain plaintext"
+    transport_lock = str(payload.get("transport_lock", "") or "").strip().lower()
+    if transport_lock != "private_strong":
+        return False, "DM hashchain spool requires private_strong transport_lock"
+    if not str(payload.get("ciphertext", "") or "").strip():
+        return False, "ciphertext cannot be empty"
+    ok, reason = _validate_sealed_bytes_field(payload, "ciphertext")
+    if not ok:
+        return ok, reason
+    return True, "ok"
+
+
 def validate_protocol_fields(protocol_version: str, network_id: str) -> tuple[bool, str]:
     if protocol_version != PROTOCOL_VERSION:
         return False, "Unsupported protocol_version"

backend/services/mesh/mesh_signed_events.py

@@ -38,6 +38,11 @@ _REVOCATION_TTL_CACHE: dict[str, dict[str, Any]] = {}
 _REVOCATION_TTL_LOCK = threading.Lock()
 _REVOCATION_REFRESH_LOCK = threading.Lock()
 _REVOCATION_REFRESH_FAIL_FAST_WINDOW_S = 5.0
+
+
+def _request_scope_path(request: Request) -> str:
+    scope = getattr(request, "scope", {}) or {}
+    return str(scope.get("path") or "")
 _REVOCATION_REFRESH_RETRY_AFTER_S = 5
 _REVOCATION_PRECHECK_UNAVAILABLE_DETAIL = "Signed event integrity preflight unavailable"
 
@@ -166,7 +171,7 @@ def _canonical_signed_write_retry_payload(
     signed_context = build_signed_context(
         event_type=prepared.event_type,
         kind=prepared.kind.value,
-        endpoint=str(request.url.path or ""),
+        endpoint=_request_scope_path(request),
         lane_floor=_content_private_required_transport_tier(prepared.kind),
         sequence_domain=_signed_context_sequence_domain(prepared),
         node_id=prepared.node_id,
@@ -458,8 +463,26 @@ def _apply_content_private_transport_lock_policy(prepared: "PreparedSignedWrite"
     except Exception:
         current_tier = "public_degraded"
 
+    lock_to_satisfy = normalized
+    if prepared.kind in {
+        SignedWriteKind.DM_POLL,
+        SignedWriteKind.DM_COUNT,
+        SignedWriteKind.DM_SEND,
+        SignedWriteKind.DM_REGISTER,
+        SignedWriteKind.DM_BLOCK,
+        SignedWriteKind.DM_WITNESS,
+    }:
+        from services.mesh.mesh_privacy_policy import release_lane_required_tier
+
+        lane_cap = release_lane_required_tier("dm")
+        # Clients sign private_strong; Tor-only nodes cap DM at
+        # private_transitional. Accept when live transport meets the
+        # strongest tier this node can offer on the DM lane.
+        if not transport_tier_is_sufficient(lane_cap, normalized):
+            lock_to_satisfy = lane_cap
+
     if (
-        not transport_tier_is_sufficient(current_tier, normalized)
+        not transport_tier_is_sufficient(current_tier, lock_to_satisfy)
         and prepared.kind not in _QUEUEABLE_CONTENT_PRIVATE_KINDS
     ):
         metrics_inc("signed_write_transport_lock_tier_mismatch")
@@ -540,7 +563,7 @@ def _apply_signed_context_policy(prepared: "PreparedSignedWrite", request: Reque
     ok, reason = validate_signed_context(
         event_type=prepared.event_type,
         kind=prepared.kind.value,
-        endpoint=str(request.url.path or ""),
+        endpoint=_request_scope_path(request),
         lane_floor=_content_private_required_transport_tier(prepared.kind),
         sequence_domain=_signed_context_sequence_domain(prepared),
         node_id=prepared.node_id,

backend/services/mesh/mesh_swarm_runtime.py

@@ -0,0 +1,507 @@
+"""Private Infonet swarm discovery and immediate ledger propagation."""
+
+from __future__ import annotations
+
+import json
+import logging
+import threading
+import time
+from typing import Any
+
+from services.config import get_settings
+from services.mesh.mesh_bootstrap_manifest import (
+    BootstrapManifest,
+    BootstrapManifestError,
+    BootstrapPeer,
+    build_bootstrap_manifest_payload,
+    load_bootstrap_manifest,
+    parse_bootstrap_manifest_dict,
+    sign_bootstrap_manifest_payload,
+    write_signed_bootstrap_manifest,
+)
+from services.mesh.mesh_crypto import normalize_peer_url, resolve_peer_key_for_url
+from services.mesh.mesh_peer_registry import DEFAULT_PEER_REGISTRY_PATH, PeerRegistry, RegistryPeer
+from services.mesh.mesh_peer_store import (
+    DEFAULT_PEER_STORE_PATH,
+    PeerStore,
+    make_push_peer_record,
+    make_sync_peer_record,
+)
+from services.mesh.mesh_router import parse_configured_relay_peers, peer_transport_kind
+
+logger = logging.getLogger(__name__)
+
+_SWARM_LOCK = threading.Lock()
+_LAST_MANIFEST_PULL_AT = 0.0
+_LAST_ANNOUNCE_AT = 0.0
+
+
+def peer_registry_enabled() -> bool:
+    settings = get_settings()
+    if bool(getattr(settings, "MESH_PEER_REGISTRY_DISABLED", False)):
+        return False
+    if str(getattr(settings, "MESH_BOOTSTRAP_SIGNER_PRIVATE_KEY", "") or "").strip():
+        return True
+    return bool(getattr(settings, "MESH_PEER_REGISTRY_ENABLED", False))
+
+
+def _manifest_path() -> str:
+    return str(getattr(get_settings(), "MESH_BOOTSTRAP_MANIFEST_PATH", "") or "data/bootstrap_peers.json")
+
+
+def _signer_public_key_b64() -> str:
+    from services.mesh.mesh_fleet_defaults import effective_bootstrap_signer_public_key_b64
+
+    return effective_bootstrap_signer_public_key_b64()
+
+
+def _signer_private_key_b64() -> str:
+    return str(getattr(settings, "MESH_BOOTSTRAP_SIGNER_PRIVATE_KEY", "") or "").strip() if (settings := get_settings()) else ""
+
+
+def _signer_id() -> str:
+    configured = str(getattr(get_settings(), "MESH_BOOTSTRAP_SIGNER_ID", "") or "").strip()
+    return configured or "shadowbroker-seed"
+
+
+def _private_transport_required() -> bool:
+    return not bool(getattr(get_settings(), "MESH_INFONET_ALLOW_CLEARNET_SYNC", False))
+
+
+def _configured_seed_peer_urls() -> list[str]:
+    from services.mesh.mesh_fleet_defaults import configured_bootstrap_seed_peers_with_fleet_default
+
+    settings = get_settings()
+    primary = str(getattr(settings, "MESH_BOOTSTRAP_SEED_PEERS", "") or "").strip()
+    legacy = str(getattr(settings, "MESH_DEFAULT_SYNC_PEERS", "") or "").strip()
+    return configured_bootstrap_seed_peers_with_fleet_default(
+        parse_configured_relay_peers(primary or legacy)
+    )
+
+
+def _seed_manifest_peers() -> list[dict[str, str]]:
+    peers: list[dict[str, str]] = []
+    for peer_url in _configured_seed_peer_urls():
+        transport = str(peer_transport_kind(peer_url) or "")
+        if _private_transport_required() and transport != "onion":
+            continue
+        peers.append(
+            {
+                "peer_url": peer_url,
+                "transport": transport,
+                "role": "seed",
+                "label": "ShadowBroker bootstrap seed",
+            }
+        )
+    return peers
+
+
+def publish_registry_manifest(*, now: float | None = None, persist: bool = True) -> BootstrapManifest:
+    private_key = _signer_private_key_b64()
+    public_key = _signer_public_key_b64()
+    if not private_key or not public_key:
+        raise BootstrapManifestError("bootstrap signer keys are required to publish swarm manifest")
+
+    timestamp = int(now if now is not None else time.time())
+    registry = PeerRegistry(DEFAULT_PEER_REGISTRY_PATH)
+    try:
+        registry.load()
+    except Exception:
+        registry = PeerRegistry(DEFAULT_PEER_REGISTRY_PATH)
+    stale_s = int(getattr(get_settings(), "MESH_PEER_REGISTRY_STALE_S", 0) or 7 * 86400)
+    if stale_s > 0:
+        registry.prune_stale(max_age_s=stale_s, now=timestamp)
+
+    peers = _seed_manifest_peers() + registry.manifest_peers()
+    ttl_s = int(getattr(get_settings(), "MESH_SWARM_MANIFEST_TTL_S", 0) or 4 * 3600)
+    payload = build_bootstrap_manifest_payload(
+        signer_id=_signer_id(),
+        peers=peers,
+        issued_at=timestamp,
+        valid_until=timestamp + max(300, ttl_s),
+    )
+    signature = sign_bootstrap_manifest_payload(payload, signer_private_key_b64=private_key)
+    manifest = BootstrapManifest(
+        version=int(payload["version"]),
+        issued_at=int(payload["issued_at"]),
+        valid_until=int(payload["valid_until"]),
+        signer_id=str(payload["signer_id"]),
+        peers=tuple(BootstrapPeer(**dict(peer)) for peer in peers),
+        signature=signature,
+    )
+    if persist:
+        registry.save()
+        write_signed_bootstrap_manifest(
+            _manifest_path(),
+            signer_id=manifest.signer_id,
+            signer_private_key_b64=private_key,
+            peers=[peer.to_dict() for peer in manifest.peers],
+            issued_at=manifest.issued_at,
+            valid_until=manifest.valid_until,
+        )
+    return manifest
+
+
+def load_live_bootstrap_manifest(*, now: float | None = None) -> BootstrapManifest | None:
+    public_key = _signer_public_key_b64()
+    if not public_key:
+        return None
+    if peer_registry_enabled():
+        try:
+            return publish_registry_manifest(now=now, persist=False)
+        except BootstrapManifestError:
+            logger.warning("live registry manifest unavailable", exc_info=True)
+    try:
+        return load_bootstrap_manifest(_manifest_path(), signer_public_key_b64=public_key, now=now)
+    except BootstrapManifestError:
+        return None
+
+
+def _upsert_swarm_peer_into_store(
+    *,
+    peer_url: str,
+    transport: str,
+    role: str,
+    label: str = "",
+    signer_id: str = "",
+    now: float | None = None,
+) -> None:
+    timestamp = int(now if now is not None else time.time())
+    if _private_transport_required() and transport != "onion":
+        return
+    store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    try:
+        store.load()
+    except Exception:
+        store = PeerStore(DEFAULT_PEER_STORE_PATH)
+    store.upsert(
+        make_sync_peer_record(
+            peer_url=peer_url,
+            transport=transport,
+            role=role,
+            source="swarm",
+            label=label,
+            signer_id=signer_id,
+            now=timestamp,
+        )
+    )
+    store.upsert(
+        make_push_peer_record(
+            peer_url=peer_url,
+            transport=transport,
+            role=role if role != "seed" else "relay",
+            source="swarm",
+            label=label,
+            now=timestamp,
+        )
+    )
+    store.save()
+
+
+def record_peer_announcement(body: dict[str, Any], *, now: float | None = None) -> RegistryPeer:
+    if not peer_registry_enabled():
+        raise ValueError("peer registry is not enabled on this node")
+    registry = PeerRegistry(DEFAULT_PEER_REGISTRY_PATH)
+    try:
+        registry.load()
+    except Exception:
+        registry = PeerRegistry(DEFAULT_PEER_REGISTRY_PATH)
+    peer = registry.upsert_announcement(
+        peer_url=str(body.get("peer_url", "") or ""),
+        transport=str(body.get("transport", "") or ""),
+        role=str(body.get("role", "participant") or "participant"),
+        node_id=str(body.get("node_id", "") or ""),
+        label=str(body.get("label", "") or ""),
+        now=now,
+    )
+    registry.save()
+    _upsert_swarm_peer_into_store(
+        peer_url=peer.peer_url,
+        transport=peer.transport,
+        role=peer.role,
+        label=peer.label,
+        signer_id=_signer_id(),
+        now=now,
+    )
+    try:
+        publish_registry_manifest(now=now, persist=True)
+    except Exception:
+        logger.warning("failed to republish swarm manifest after announce", exc_info=True)
+    return peer
+
+
+def merge_manifest_into_peer_store(manifest: BootstrapManifest, *, now: float | None = None) -> int:
+    timestamp = int(now if now is not None else time.time())
+    merged = 0
+    for peer in manifest.peers:
+        if _private_transport_required() and peer.transport != "onion":
+            continue
+        _upsert_swarm_peer_into_store(
+            peer_url=peer.peer_url,
+            transport=peer.transport,
+            role=peer.role,
+            label=peer.label,
+            signer_id=manifest.signer_id,
+            now=timestamp,
+        )
+        merged += 1
+    return merged
+
+
+def fetch_remote_bootstrap_manifest(seed_peer_url: str, *, now: float | None = None) -> BootstrapManifest | None:
+    import requests
+
+    public_key = _signer_public_key_b64()
+    if not public_key:
+        return None
+    normalized = normalize_peer_url(seed_peer_url)
+    if not normalized:
+        return None
+
+    from main import _infonet_peer_requests_proxies
+
+    proxies = _infonet_peer_requests_proxies(normalized)
+    timeout = int(getattr(get_settings(), "MESH_SYNC_TIMEOUT_S", 0) or 45)
+    request_kwargs: dict[str, Any] = {"timeout": timeout}
+    if proxies:
+        request_kwargs["proxies"] = proxies
+    try:
+        response = requests.get(f"{normalized}/api/mesh/infonet/bootstrap-manifest", **request_kwargs)
+    except Exception as exc:
+        logger.debug("swarm manifest fetch failed for %s: %s", normalized, exc)
+        return None
+    if response.status_code != 200:
+        return None
+    try:
+        raw = response.json()
+    except Exception:
+        return None
+    if not isinstance(raw, dict) or raw.get("ok") is False:
+        return None
+    manifest_body = dict(raw.get("manifest") or raw)
+    try:
+        return parse_bootstrap_manifest_dict(
+            manifest_body,
+            signer_public_key_b64=public_key,
+            now=now,
+        )
+    except BootstrapManifestError:
+        return None
+
+
+def refresh_swarm_manifest_from_seeds(*, now: float | None = None, force: bool = False) -> dict[str, Any]:
+    global _LAST_MANIFEST_PULL_AT
+    interval_s = int(getattr(get_settings(), "MESH_SWARM_MANIFEST_PULL_INTERVAL_S", 0) or 300)
+    timestamp = float(now if now is not None else time.time())
+    with _SWARM_LOCK:
+        if not force and _LAST_MANIFEST_PULL_AT and timestamp - _LAST_MANIFEST_PULL_AT < max(30, interval_s):
+            return {"ok": True, "skipped": True, "reason": "manifest_pull_interval"}
+        _LAST_MANIFEST_PULL_AT = timestamp
+
+    if not _signer_public_key_b64():
+        return {"ok": False, "detail": "MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY is not configured"}
+
+    last_error = "manifest fetch failed"
+    for seed_url in _configured_seed_peer_urls():
+        manifest = fetch_remote_bootstrap_manifest(seed_url, now=timestamp)
+        if manifest is None:
+            continue
+        try:
+            merged = merge_manifest_into_peer_store(manifest, now=timestamp)
+            return {
+                "ok": True,
+                "seed_peer_url": seed_url,
+                "peer_count": len(manifest.peers),
+                "merged_peer_count": merged,
+            }
+        except Exception as exc:
+            last_error = str(exc or type(exc).__name__)
+    return {"ok": False, "detail": last_error}
+
+
+def announce_local_peer_to_seeds(*, now: float | None = None, force: bool = False) -> dict[str, Any]:
+    global _LAST_ANNOUNCE_AT
+    import hashlib as _hashlib_mod
+    import hmac as _hmac_mod
+    import requests
+
+    from main import _infonet_peer_requests_proxies, _local_infonet_peer_url, _participant_node_enabled
+
+    if not _participant_node_enabled():
+        return {"ok": False, "detail": "participant node disabled"}
+    peer_url = _local_infonet_peer_url()
+    if not peer_url:
+        return {"ok": False, "detail": "local peer URL is not ready"}
+    peer_key = resolve_peer_key_for_url(peer_url)
+    if not peer_key:
+        return {"ok": False, "detail": "peer HMAC secret is not configured"}
+
+    timestamp = float(now if now is not None else time.time())
+    with _SWARM_LOCK:
+        if not force and _LAST_ANNOUNCE_AT and timestamp - _LAST_ANNOUNCE_AT < 300:
+            return {"ok": True, "skipped": True, "reason": "announce_interval"}
+        _LAST_ANNOUNCE_AT = timestamp
+
+    transport = str(peer_transport_kind(peer_url) or "onion")
+    body = {
+        "peer_url": peer_url,
+        "transport": transport,
+        "role": "participant",
+        "node_id": "",
+        "label": "",
+        "ts": int(timestamp),
+    }
+    body_bytes = json.dumps(body, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
+    hmac_hex = _hmac_mod.new(peer_key, body_bytes, _hashlib_mod.sha256).hexdigest()
+    timeout = int(getattr(get_settings(), "MESH_RELAY_PUSH_TIMEOUT_S", 0) or 45)
+    results: list[dict[str, Any]] = []
+    for seed_url in _configured_seed_peer_urls():
+        normalized = normalize_peer_url(seed_url)
+        if not normalized:
+            continue
+        proxies = _infonet_peer_requests_proxies(normalized)
+        request_kwargs: dict[str, Any] = {
+            "data": body_bytes,
+            "headers": {
+                "Content-Type": "application/json",
+                "X-Peer-Url": peer_url,
+                "X-Peer-HMAC": hmac_hex,
+            },
+            "timeout": timeout,
+        }
+        if proxies:
+            request_kwargs["proxies"] = proxies
+        try:
+            response = requests.post(
+                f"{normalized}/api/mesh/infonet/peer-announce",
+                **request_kwargs,
+            )
+            results.append(
+                {
+                    "seed_peer_url": normalized,
+                    "status_code": int(response.status_code),
+                    "ok": response.status_code == 200,
+                }
+            )
+        except Exception as exc:
+            results.append({"seed_peer_url": normalized, "ok": False, "detail": str(exc)})
+    ok = any(bool(item.get("ok")) for item in results)
+    return {"ok": ok, "peer_url": peer_url, "results": results}
+
+
+def _announce_succeeded(announce: dict[str, Any]) -> bool:
+    if not bool(announce.get("ok")):
+        return False
+    results = announce.get("results") or []
+    return any(bool(item.get("ok")) and int(item.get("status_code") or 0) == 200 for item in results)
+
+
+def _manifest_succeeded(manifest: dict[str, Any]) -> bool:
+    if not bool(manifest.get("ok")):
+        return False
+    peer_count = int(manifest.get("merged_peer_count") or manifest.get("peer_count") or 0)
+    return peer_count >= 1
+
+
+def join_swarm_with_retries(
+    *,
+    attempts: int = 6,
+    delay_s: float = 15.0,
+    force: bool = True,
+) -> dict[str, Any]:
+    """Announce to seed and pull manifest, retrying while Tor circuits warm up."""
+    last_announce: dict[str, Any] = {"ok": False, "detail": "not attempted"}
+    last_manifest: dict[str, Any] = {"ok": False, "detail": "not attempted"}
+    tries = max(1, int(attempts))
+    pause_s = max(1.0, float(delay_s))
+    for attempt in range(tries):
+        last_announce = announce_local_peer_to_seeds(force=force)
+        last_manifest = refresh_swarm_manifest_from_seeds(force=force)
+        if _announce_succeeded(last_announce) and _manifest_succeeded(last_manifest):
+            return {
+                "ok": True,
+                "attempts": attempt + 1,
+                "announce": last_announce,
+                "manifest_pull": last_manifest,
+            }
+        if attempt + 1 < tries:
+            time.sleep(pause_s)
+    return {
+        "ok": False,
+        "attempts": tries,
+        "announce": last_announce,
+        "manifest_pull": last_manifest,
+        "detail": "swarm join incomplete after retries",
+    }
+
+
+def push_infonet_events_to_http_peers(events: list[dict[str, Any]]) -> dict[str, Any]:
+    import hashlib as _hashlib_mod
+    import hmac as _hmac_mod
+    import requests
+
+    from main import (
+        _filter_infonet_peer_urls,
+        _infonet_peer_requests_proxies,
+        _local_infonet_peer_url,
+        _participant_node_enabled,
+        _record_public_push_result,
+    )
+    from services.mesh.mesh_router import authenticated_push_peer_urls
+
+    if not _participant_node_enabled() or not events:
+        return {"ok": False, "detail": "nothing to push"}
+    peers = _filter_infonet_peer_urls(authenticated_push_peer_urls())
+    if not peers:
+        return {"ok": False, "detail": "no push peers configured"}
+
+    sender_url = _local_infonet_peer_url()
+    peer_key = resolve_peer_key_for_url(sender_url)
+    if not peer_key:
+        return {"ok": False, "detail": "peer HMAC secret is not configured"}
+
+    body_bytes = json.dumps(
+        {"events": events},
+        sort_keys=True,
+        separators=(",", ":"),
+        ensure_ascii=False,
+    ).encode("utf-8")
+    hmac_hex = _hmac_mod.new(peer_key, body_bytes, _hashlib_mod.sha256).hexdigest()
+    timeout = int(getattr(get_settings(), "MESH_RELAY_PUSH_TIMEOUT_S", 0) or 45)
+    results: list[dict[str, Any]] = []
+    for peer_url in peers:
+        normalized = normalize_peer_url(peer_url)
+        if not normalized:
+            continue
+        proxies = _infonet_peer_requests_proxies(normalized)
+        request_kwargs: dict[str, Any] = {
+            "data": body_bytes,
+            "headers": {
+                "Content-Type": "application/json",
+                "X-Peer-Url": sender_url,
+                "X-Peer-HMAC": hmac_hex,
+            },
+            "timeout": timeout,
+        }
+        if proxies:
+            request_kwargs["proxies"] = proxies
+        try:
+            response = requests.post(f"{normalized}/api/mesh/infonet/peer-push", **request_kwargs)
+            results.append(
+                {
+                    "peer_url": normalized,
+                    "ok": response.status_code == 200,
+                    "status_code": int(response.status_code),
+                }
+            )
+        except Exception as exc:
+            results.append({"peer_url": normalized, "ok": False, "detail": str(exc)})
+    ok = any(bool(item.get("ok")) for item in results)
+    event_id = str((events[-1] or {}).get("event_id", "") or "")
+    _record_public_push_result(
+        event_id,
+        ok=ok,
+        error="" if ok else "immediate peer push failed",
+        results=results,
+    )
+    return {"ok": ok, "results": results}

backend/services/mesh/mesh_wormhole_contacts.py

@@ -929,6 +929,85 @@ def list_wormhole_dm_contacts() -> dict[str, dict[str, Any]]:
     return _read_contacts()
 
 
+def get_wormhole_dm_contact(peer_id: str) -> dict[str, Any] | None:
+    peer_key = str(peer_id or "").strip()
+    if not peer_key:
+        return None
+    contacts = _read_contacts()
+    if peer_key not in contacts:
+        return None
+    return dict(_normalize_contact(contacts[peer_key]))
+
+
+def sever_wormhole_dm_contact(peer_id: str, *, block: bool = False) -> dict[str, Any]:
+    """Close the shared DM lane; a fresh contact request + accept is required to reopen."""
+    peer_key = str(peer_id or "").strip()
+    if not peer_key:
+        return {"ok": False, "detail": "peer_id required"}
+
+    contacts = _read_contacts()
+    current = _normalize_contact(contacts.get(peer_key))
+    now = int(time.time())
+    current["sharedAlias"] = ""
+    current["sharedAliasCounter"] = 0
+    current["sharedAliasPublicKey"] = ""
+    current["sharedAliasPublicKeyAlgo"] = "Ed25519"
+    current["previousSharedAliases"] = []
+    current["pendingSharedAlias"] = ""
+    current["pendingSharedAliasCounter"] = 0
+    current["pendingSharedAliasPublicKey"] = ""
+    current["pendingSharedAliasPublicKeyAlgo"] = "Ed25519"
+    current["pendingSharedAliasGraceMs"] = 0
+    current["sharedAliasGraceUntil"] = 0
+    current["sharedAliasRotatedAt"] = 0
+    current["acceptedPreviousAlias"] = ""
+    current["acceptedPreviousAliasCounter"] = 0
+    current["acceptedPreviousAliasPublicKey"] = ""
+    current["acceptedPreviousAliasPublicKeyAlgo"] = "Ed25519"
+    current["acceptedPreviousGraceUntil"] = 0
+    current["acceptedPreviousHardGraceUntil"] = 0
+    current["acceptedPreviousAwaitingReply"] = False
+    current["aliasBindingSeq"] = 0
+    current["aliasBindingPendingReason"] = ""
+    current["aliasBindingPreparedAt"] = 0
+    current["aliasGateJoinAppliedSeq"] = 0
+    if block:
+        current["blocked"] = True
+    current["updated_at"] = now
+    contacts[peer_key] = _normalize_contact(current)
+    _write_contacts(contacts)
+
+    relay_policy = {}
+    try:
+        from services.mesh.mesh_dm_connect_delivery import revoke_connect_relay_policy
+
+        relay_policy = revoke_connect_relay_policy(peer_key)
+    except Exception:
+        relay_policy = {"ok": False}
+
+    relay_block = {"ok": False}
+    if block:
+        try:
+            from services.mesh.mesh_dm_relay import dm_relay
+            from services.mesh.mesh_wormhole_persona import get_dm_identity
+
+            local_id = str(get_dm_identity().get("node_id", "") or "").strip()
+            if local_id:
+                dm_relay.block(local_id, peer_key)
+                relay_block = {"ok": True, "local_id": local_id}
+        except Exception as exc:
+            relay_block = {"ok": False, "detail": str(exc) or type(exc).__name__}
+
+    return {
+        "ok": True,
+        "peer_id": peer_key,
+        "severed": True,
+        "blocked": bool(block),
+        "relay_policy": relay_policy,
+        "relay_block": relay_block,
+    }
+
+
 def _promote_invite_lookup_mode(contact: dict[str, Any], *, now: int | None = None) -> bool:
     current = dict(contact or {})
     lookup_handle = str(current.get("invitePinnedPrekeyLookupHandle", "") or "").strip()
@@ -1070,11 +1149,14 @@ def pin_wormhole_dm_invite(
     identity_dh_pub_key = str(payload.get("identity_dh_pub_key", "") or "")
     dh_algo = str(payload.get("dh_algo", "X25519") or "X25519")
     prekey_lookup_handle = str(payload.get("prekey_lookup_handle", "") or "")
+    lookup_peer_url = str(payload.get("lookup_peer_url", "") or "").strip().rstrip("/")
     if str(alias or "").strip():
         current["alias"] = str(alias or "").strip()
     current["dhPubKey"] = identity_dh_pub_key
     current["dhAlgo"] = dh_algo
     current["invitePinnedPrekeyLookupHandle"] = prekey_lookup_handle
+    if lookup_peer_url:
+        current["invitePinnedLookupPeerUrl"] = lookup_peer_url
     current["invitePinnedRootFingerprint"] = str(payload.get("root_fingerprint", "") or "").strip().lower()
     current["invitePinnedRootManifestFingerprint"] = str(
         payload.get("root_manifest_fingerprint", "") or ""
@@ -1170,6 +1252,12 @@ def pin_wormhole_dm_invite(
     current["updated_at"] = now
     contacts[peer_key] = _normalize_contact(current)
     _write_contacts(contacts)
+    try:
+        from services.mesh.mesh_dm_connect_delivery import grant_connect_relay_policy
+
+        grant_connect_relay_policy(peer_key, reason="invite_import")
+    except Exception:
+        pass
     return contacts[peer_key]
 
 

backend/services/mesh/mesh_wormhole_identity.py

@@ -549,6 +549,27 @@ def invite_identity_commitment_for_identity_material(
     return hashlib.sha256(_stable_json(material).encode("utf-8")).hexdigest()
 
 
+def _local_dm_lookup_peer_url() -> str:
+    """Return this node's fleet-reachable URL for invite-scoped prekey lookup."""
+    try:
+        from services.config import get_settings
+        from services.mesh.mesh_crypto import normalize_peer_url
+
+        configured = normalize_peer_url(str(getattr(get_settings(), "MESH_PUBLIC_PEER_URL", "") or ""))
+        if configured:
+            return configured
+        from services.tor_hidden_service import tor_service
+
+        onion = str(getattr(tor_service, "onion_address", "") or "").strip()
+        if onion:
+            if "://" not in onion:
+                onion = f"http://{onion}:8000"
+            return normalize_peer_url(onion)
+    except Exception:
+        pass
+    return ""
+
+
 def _dm_invite_payload(
     data: dict[str, Any],
     *,
@@ -930,6 +951,9 @@ def export_wormhole_dm_invite(*, label: str = "", expires_in_s: int = 0) -> dict
     # fetch our prekey bundle without using our stable agent_id.
     lookup_handle = secrets.token_hex(24)
     payload["prekey_lookup_handle"] = lookup_handle
+    lookup_peer_url = _local_dm_lookup_peer_url()
+    if lookup_peer_url:
+        payload["lookup_peer_url"] = lookup_peer_url
 
     # Persist the handle so it is included in future prekey registrations.
     existing_handles, _ = _normalize_prekey_lookup_handles(

backend/services/mesh/mesh_wormhole_prekey.py

@@ -79,6 +79,164 @@ def _warn_legacy_prekey_lookup(agent_id: str) -> None:
     )
 
 
+def _fleet_peer_lookup_user_agent() -> str:
+    custom = str(os.environ.get("SHADOWBROKER_MESH_PEER_USER_AGENT") or "").strip()
+    if custom:
+        return custom
+    return "Mozilla/5.0 (compatible; ShadowbrokerMesh/1.0)"
+
+
+_INVITE_LOOKUP_MAX_ELAPSED_S = 120
+_INVITE_LOOKUP_MAX_BOOTSTRAP_PEERS = 3
+_INVITE_LOOKUP_MAX_PUSH_PEERS = 16
+_INVITE_LOOKUP_PARALLEL_WORKERS = 8
+
+
+def _invite_lookup_request_timeout(peer_url: str) -> tuple[int, int]:
+    from services.mesh.mesh_router import peer_transport_kind
+
+    if peer_transport_kind(peer_url) == "onion":
+        return (10, 35)
+    return (5, 15)
+
+
+def _bootstrap_seed_peer_urls() -> set[str]:
+    try:
+        from services.config import get_settings
+        from services.mesh.mesh_router import parse_configured_relay_peers
+
+        seeds: set[str] = set()
+        raw = str(getattr(get_settings(), "MESH_BOOTSTRAP_SEED_PEERS", "") or "")
+        for peer in parse_configured_relay_peers(raw):
+            normalized = str(peer or "").strip().rstrip("/")
+            if normalized:
+                seeds.add(normalized)
+        return seeds
+    except Exception:
+        return set()
+
+
+def _discovered_push_peer_urls(*, limit: int = _INVITE_LOOKUP_MAX_PUSH_PEERS) -> list[str]:
+    try:
+        from services.mesh.mesh_router import authenticated_push_peer_urls
+
+        seeds = _bootstrap_seed_peer_urls()
+        peers: list[str] = []
+        for peer in authenticated_push_peer_urls():
+            normalized = str(peer or "").strip().rstrip("/")
+            if not normalized or normalized in seeds:
+                continue
+            peers.append(normalized)
+            if len(peers) >= max(1, int(limit or 1)):
+                break
+        return peers
+    except Exception:
+        return []
+
+
+def _prioritized_invite_lookup_peer_urls(*, preferred: list[str] | None = None) -> list[str]:
+    preferred_urls = [
+        str(peer or "").strip().rstrip("/")
+        for peer in list(preferred or [])
+        if str(peer or "").strip()
+    ]
+    configured = _configured_public_lookup_peer_urls()
+    seeds = _bootstrap_seed_peer_urls()
+    active: list[str] = []
+    bootstrap: list[str] = []
+    push_discovery: list[str] = []
+    seen = set(preferred_urls)
+    for peer in configured:
+        if peer in seen:
+            continue
+        seen.add(peer)
+        if peer in seeds:
+            bootstrap.append(peer)
+        else:
+            active.append(peer)
+    for peer in _discovered_push_peer_urls():
+        if peer in seen:
+            continue
+        seen.add(peer)
+        push_discovery.append(peer)
+    ordered = list(preferred_urls)
+    ordered.extend(active)
+    ordered.extend(push_discovery)
+    ordered.extend(bootstrap[:_INVITE_LOOKUP_MAX_BOOTSTRAP_PEERS])
+    return ordered
+
+
+def _preferred_invite_lookup_peer_urls(lookup_token: str) -> list[str]:
+    token = str(lookup_token or "").strip()
+    if not token:
+        return []
+    try:
+        from services.mesh.mesh_wormhole_contacts import list_wormhole_dm_contacts
+    except Exception:
+        return []
+    peers: list[str] = []
+    for contact in list_wormhole_dm_contacts() or []:
+        if not isinstance(contact, dict):
+            continue
+        if str(contact.get("invitePinnedPrekeyLookupHandle", "") or "").strip() != token:
+            continue
+        peer_url = str(contact.get("invitePinnedLookupPeerUrl", "") or "").strip().rstrip("/")
+        if peer_url and peer_url not in peers:
+            peers.append(peer_url)
+    return peers
+
+
+def _peer_http_request(
+    method: str,
+    peer_url: str,
+    *,
+    body_bytes: bytes | None = None,
+    headers: dict[str, str] | None = None,
+    timeout: int | tuple[int, int] = 45,
+):
+    """HTTP to a fleet peer, using Tor SOCKS when the URL is an onion address."""
+    import requests
+
+    from services.mesh.mesh_crypto import normalize_peer_url
+    from urllib.parse import urlparse
+
+    raw_peer_url = str(peer_url or "").strip()
+    parsed = urlparse(raw_peer_url)
+    if parsed.path and parsed.path not in {"", "/"}:
+        # Full request URLs include invite lookup query params; do not
+        # normalize them away when deriving the peer base URL.
+        normalized = raw_peer_url
+    else:
+        normalized = normalize_peer_url(raw_peer_url)
+    if not normalized:
+        raise OSError("invalid peer url")
+    if isinstance(timeout, tuple):
+        connect_timeout, read_timeout = timeout
+        resolved_timeout: int | tuple[int, int] = (
+            max(1, int(connect_timeout or 5)),
+            max(1, int(read_timeout or 15)),
+        )
+    else:
+        resolved_timeout = max(1, int(timeout or 45))
+    request_kwargs: dict[str, Any] = {
+        "headers": dict(headers or {}),
+        "timeout": resolved_timeout,
+    }
+    try:
+        from main import _infonet_peer_requests_proxies
+
+        proxy_peer_url = normalize_peer_url(f"{parsed.scheme}://{parsed.netloc}")
+        proxies = _infonet_peer_requests_proxies(proxy_peer_url)
+        if proxies:
+            request_kwargs["proxies"] = proxies
+    except Exception:
+        pass
+    if method.upper() == "GET":
+        return requests.get(normalized, **request_kwargs)
+    request_kwargs["data"] = body_bytes or b""
+    return requests.post(normalized, **request_kwargs)
+
+
 def _fetch_dm_prekey_bundle_from_peer_lookup(lookup_token: str) -> dict[str, Any]:
     """Fetch an invite-scoped prekey bundle from configured authenticated peers.
 
@@ -95,12 +253,12 @@ def _fetch_dm_prekey_bundle_from_peer_lookup(lookup_token: str) -> dict[str, Any
             normalize_peer_url,
             resolve_peer_key_for_url,
         )
-        from services.mesh.mesh_router import configured_relay_peer_urls
+        from services.mesh.mesh_router import authenticated_push_peer_urls
 
         settings = get_settings()
         # Issue #256: secret check moved per-peer below. We still bail out
         # cleanly when there are no peers configured at all.
-        peers = configured_relay_peer_urls()
+        peers = authenticated_push_peer_urls()
         if not peers:
             return {"ok": False, "detail": "peer prekey lookup unavailable"}
         timeout = max(1, _safe_int(getattr(settings, "MESH_RELAY_PUSH_TIMEOUT_S", 10) or 10, 10))
@@ -132,17 +290,17 @@ def _fetch_dm_prekey_bundle_from_peer_lookup(lookup_token: str) -> dict[str, Any
             "X-Peer-Url": sender_peer_url,
             "X-Peer-HMAC": hmac.new(peer_key, body, hashlib.sha256).hexdigest(),
         }
-        request = urllib.request.Request(
-            f"{normalized_peer_url}/api/mesh/dm/prekey-peer-lookup",
-            data=body,
-            headers=headers,
-            method="POST",
-        )
         try:
-            with urllib.request.urlopen(request, timeout=timeout) as response:
-                raw = response.read(256 * 1024)
+            response = _peer_http_request(
+                "POST",
+                f"{normalized_peer_url}/api/mesh/dm/prekey-peer-lookup",
+                body_bytes=body,
+                headers=headers,
+                timeout=timeout,
+            )
+            raw = response.content[: 256 * 1024]
             payload = json.loads(raw.decode("utf-8"))
-        except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, OSError) as exc:
+        except (json.JSONDecodeError, OSError, Exception) as exc:
             last_detail = str(exc) or type(exc).__name__
             continue
         if isinstance(payload, dict) and payload.get("ok"):
@@ -161,12 +319,18 @@ def _configured_public_lookup_peer_urls() -> list[str]:
 
         settings = get_settings()
         candidates: list[str] = []
+        # Operator-configured peers first, then recently active fleet nodes.
+        # Invite handles are minted on a specific node; cold bootstrap seeds
+        # rarely have them cached and should not be tried before contacts.
         for raw in (
-            getattr(settings, "MESH_BOOTSTRAP_SEED_PEERS", ""),
             getattr(settings, "MESH_DEFAULT_SYNC_PEERS", ""),
         ):
             candidates.extend(parse_configured_relay_peers(str(raw or "")))
         candidates.extend(active_sync_peer_urls())
+        for raw in (
+            getattr(settings, "MESH_BOOTSTRAP_SEED_PEERS", ""),
+        ):
+            candidates.extend(parse_configured_relay_peers(str(raw or "")))
     except Exception:
         return []
 
@@ -204,7 +368,50 @@ def _normalize_remote_lookup_bundle(payload: dict[str, Any]) -> dict[str, Any]:
     return data
 
 
-def _fetch_dm_prekey_bundle_from_public_lookup(lookup_token: str) -> dict[str, Any]:
+def _try_public_prekey_lookup_peer(
+    peer_url: str,
+    encoded: str,
+    *,
+    timeout: int | tuple[int, int] | None = None,
+) -> dict[str, Any]:
+    normalized_peer_url = str(peer_url or "").strip().rstrip("/")
+    if not normalized_peer_url:
+        return {"ok": False, "detail": "invalid peer url"}
+    resolved_timeout = timeout or _invite_lookup_request_timeout(normalized_peer_url)
+    try:
+        response = _peer_http_request(
+            "GET",
+            f"{normalized_peer_url}/api/mesh/dm/prekey-bundle?{encoded}",
+            headers={
+                "Accept": "application/json",
+                "User-Agent": _fleet_peer_lookup_user_agent(),
+            },
+            timeout=resolved_timeout,
+        )
+        raw = response.content[: 256 * 1024]
+        payload = json.loads(raw.decode("utf-8"))
+    except (json.JSONDecodeError, OSError, Exception) as exc:
+        logger.debug("public prekey lookup failed for %s: %s", normalized_peer_url, type(exc).__name__)
+        return {"ok": False, "detail": "peer prekey lookup unavailable"}
+    if not isinstance(payload, dict):
+        return {"ok": False, "detail": "invalid peer response"}
+    if payload.get("pending") or str(payload.get("status", "") or "") == "preparing_private_lane":
+        return {"ok": False, "detail": "peer prekey lookup still preparing"}
+    if not payload.get("ok"):
+        return {
+            "ok": False,
+            "detail": str(payload.get("detail", "") or "Prekey bundle not found"),
+        }
+    if not isinstance(payload.get("bundle"), dict):
+        return {"ok": False, "detail": "Prekey bundle not found"}
+    return _normalize_remote_lookup_bundle(payload)
+
+
+def _fetch_dm_prekey_bundle_from_public_lookup(
+    lookup_token: str,
+    *,
+    extra_preferred_peer_urls: list[str] | None = None,
+) -> dict[str, Any]:
     """Fetch an invite-scoped prekey bundle from bootstrap/sync peers.
 
     The token is high-entropy and invite-scoped. This path does not expose a
@@ -212,61 +419,69 @@ def _fetch_dm_prekey_bundle_from_public_lookup(lookup_token: str) -> dict[str, A
     derive it from the signed identity public key and validate the bundle before
     accepting it.
     """
+    from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
+
     token = str(lookup_token or "").strip()
     if not token:
         return {"ok": False, "detail": "lookup token required"}
-    peers = _configured_public_lookup_peer_urls()
+    preferred = list(_preferred_invite_lookup_peer_urls(token))
+    for peer in list(extra_preferred_peer_urls or []):
+        normalized = str(peer or "").strip().rstrip("/")
+        if normalized and normalized not in preferred:
+            preferred.insert(0, normalized)
+    peers = _prioritized_invite_lookup_peer_urls(preferred=preferred)
     if not peers:
         return {"ok": False, "detail": "peer prekey lookup unavailable"}
-    try:
-        from services.config import get_settings
-
-        timeout = max(1, _safe_int(getattr(get_settings(), "MESH_SYNC_TIMEOUT_S", 5) or 5, 5))
-    except Exception:
-        timeout = 5
 
     encoded = urllib.parse.urlencode({"lookup_token": token})
     last_detail = ""
-    for peer_url in peers:
-        normalized_peer_url = str(peer_url or "").strip().rstrip("/")
-        if not normalized_peer_url:
-            continue
-        # Generic UA: any peer-facing crypto request should not carry a
-        # fork-specific identifier — that turns prekey lookups into a
-        # software-fingerprinting beacon.
-        from services.network_utils import DEFAULT_USER_AGENT
-        request = urllib.request.Request(
-            f"{normalized_peer_url}/api/mesh/dm/prekey-bundle?{encoded}",
-            headers={
-                "Accept": "application/json",
-                "User-Agent": DEFAULT_USER_AGENT,
-            },
-            method="GET",
+    hinted_only = bool(list(extra_preferred_peer_urls or []))
+    hint_timeout = (5, 20)
+    for peer_url in preferred:
+        hinted = _try_public_prekey_lookup_peer(
+            peer_url,
+            encoded,
+            timeout=hint_timeout if hinted_only else None,
         )
-        try:
-            with urllib.request.urlopen(request, timeout=timeout) as response:
-                raw = response.read(256 * 1024)
-            payload = json.loads(raw.decode("utf-8"))
-        except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, OSError) as exc:
-            logger.debug("public prekey lookup failed for %s: %s", normalized_peer_url, type(exc).__name__)
-            last_detail = "peer prekey lookup unavailable"
-            continue
-        if not isinstance(payload, dict):
-            last_detail = "invalid peer response"
-            continue
-        if payload.get("pending") or str(payload.get("status", "") or "") == "preparing_private_lane":
-            last_detail = "peer prekey lookup still preparing"
-            continue
-        if not payload.get("ok"):
-            last_detail = str(payload.get("detail", "") or last_detail or "Prekey bundle not found")
-            continue
-        if not isinstance(payload.get("bundle"), dict):
-            last_detail = "Prekey bundle not found"
-            continue
-        normalized = _normalize_remote_lookup_bundle(payload)
-        if normalized.get("ok"):
-            return normalized
-        last_detail = str(normalized.get("detail", "") or last_detail)
+        if hinted.get("ok"):
+            return hinted
+        if isinstance(hinted, dict):
+            last_detail = str(hinted.get("detail", "") or last_detail)
+    remaining_peers = [peer for peer in peers if peer not in set(preferred)]
+    if not remaining_peers:
+        return {"ok": False, "detail": last_detail or "Prekey bundle not found"}
+    if hinted_only:
+        return {"ok": False, "detail": last_detail or "Prekey bundle not found"}
+    deadline = time.time() + _INVITE_LOOKUP_MAX_ELAPSED_S
+    workers = min(_INVITE_LOOKUP_PARALLEL_WORKERS, max(1, len(remaining_peers)))
+    with ThreadPoolExecutor(max_workers=workers) as executor:
+        futures = {
+            executor.submit(_try_public_prekey_lookup_peer, peer_url, encoded): peer_url
+            for peer_url in remaining_peers
+        }
+        while futures and time.time() < deadline:
+            done, _ = wait(
+                futures,
+                timeout=max(0.1, deadline - time.time()),
+                return_when=FIRST_COMPLETED,
+            )
+            if not done:
+                break
+            for future in done:
+                futures.pop(future, None)
+                try:
+                    result = future.result()
+                except Exception as exc:
+                    last_detail = str(exc) or type(exc).__name__
+                    continue
+                if isinstance(result, dict) and result.get("ok"):
+                    for pending in futures:
+                        pending.cancel()
+                    return result
+                if isinstance(result, dict):
+                    last_detail = str(result.get("detail", "") or last_detail)
+        for pending in futures:
+            pending.cancel()
     return {"ok": False, "detail": last_detail or "Prekey bundle not found"}
 
 
@@ -1019,6 +1234,7 @@ def fetch_dm_prekey_bundle(
     lookup_token: str = "",
     *,
     allow_peer_lookup: bool = True,
+    lookup_peer_urls: list[str] | None = None,
 ) -> dict[str, Any]:
     from services.mesh.mesh_dm_relay import dm_relay
 
@@ -1043,12 +1259,18 @@ def fetch_dm_prekey_bundle(
             resolved_id = found_id
             lookup_mode = "invite_lookup_handle"
         elif allow_peer_lookup:
-            peer_found = _fetch_dm_prekey_bundle_from_peer_lookup(resolved_lookup)
-            if peer_found.get("ok"):
-                return peer_found
-            public_found = _fetch_dm_prekey_bundle_from_public_lookup(resolved_lookup)
+            preferred_peer_urls = list(lookup_peer_urls or [])
+            public_found = _fetch_dm_prekey_bundle_from_public_lookup(
+                resolved_lookup,
+                extra_preferred_peer_urls=preferred_peer_urls,
+            )
             if public_found.get("ok"):
                 return public_found
+            peer_found: dict[str, Any] = {"ok": False, "detail": ""}
+            if not preferred_peer_urls:
+                peer_found = _fetch_dm_prekey_bundle_from_peer_lookup(resolved_lookup)
+                if peer_found.get("ok"):
+                    return peer_found
             if str(public_found.get("detail", "") or "").strip():
                 return {"ok": False, "detail": str(public_found.get("detail", "") or "Prekey bundle not found")}
             return {"ok": False, "detail": str(peer_found.get("detail", "") or "Prekey bundle not found")}
@@ -1134,12 +1356,24 @@ def _classify_root_attestation_failure(peer_id: str) -> tuple[str, bool]:
     return "", False
 
 
-def bootstrap_encrypt_for_peer(peer_id: str, plaintext: str) -> dict[str, Any]:
-    fetched_bundle = fetch_dm_prekey_bundle(str(peer_id or "").strip())
+def bootstrap_encrypt_for_peer(
+    peer_id: str,
+    plaintext: str,
+    *,
+    lookup_token: str = "",
+    fetched_bundle: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    token = str(lookup_token or "").strip()
+    peer = str(peer_id or "").strip()
+    if fetched_bundle is None:
+        fetched_bundle = fetch_dm_prekey_bundle(
+            agent_id=peer if not token else "",
+            lookup_token=token,
+        )
     if not fetched_bundle.get("ok"):
         detail = str(fetched_bundle.get("detail", "") or "")
         if "root attestation" in detail.lower():
-            trust_level, trust_changed = _classify_root_attestation_failure(str(peer_id or "").strip())
+            trust_level, trust_changed = _classify_root_attestation_failure(peer or token)
             if trust_level:
                 return {
                     "ok": False,
@@ -1152,32 +1386,68 @@ def bootstrap_encrypt_for_peer(peer_id: str, plaintext: str) -> dict[str, Any]:
 
     from services.mesh.mesh_dm_relay import dm_relay
 
-    resolved_peer_id = str(fetched_bundle.get("agent_id", peer_id) or peer_id).strip()
+    resolved_peer_id = str(fetched_bundle.get("agent_id", peer) or peer).strip()
     stored = dm_relay.get_prekey_bundle(resolved_peer_id)
     if not stored:
-        return {"ok": False, "detail": "Peer prekey bundle not found"}
+        remote_bundle = dict(fetched_bundle.get("bundle") or {})
+        if not remote_bundle and fetched_bundle.get("identity_dh_pub_key"):
+            remote_bundle = fetched_bundle
+        if remote_bundle:
+            stored = {
+                "bundle": remote_bundle,
+                "signature": str(fetched_bundle.get("signature", "") or ""),
+                "public_key": str(fetched_bundle.get("public_key", "") or ""),
+                "public_key_algo": str(fetched_bundle.get("public_key_algo", "") or ""),
+                "sequence": _safe_int(fetched_bundle.get("sequence", 0) or 0),
+            }
+        else:
+            return {"ok": False, "detail": "Peer prekey bundle not found"}
     validated_record = {**dict(stored), "agent_id": resolved_peer_id}
     ok, reason = _validate_bundle_record(validated_record)
     if not ok:
         return {"ok": False, "detail": reason}
     trust_state = observe_remote_prekey_bundle(resolved_peer_id, validated_record)
     trust_level = str(trust_state.get("trust_level", "") or "")
-    from services.mesh.mesh_wormhole_contacts import verified_first_contact_requirement
+    consent_handshake = False
+    try:
+        from services.mesh.mesh_wormhole_dead_drop import parse_contact_consent
 
-    verified_first_contact = verified_first_contact_requirement(
-        resolved_peer_id,
-        trust_level=trust_level,
-    )
-    if not verified_first_contact.get("ok"):
-        return {
-            "ok": False,
-            "peer_id": resolved_peer_id,
-            "detail": str(verified_first_contact.get("detail", "") or "verified first contact required"),
-            "trust_changed": trust_level in ("mismatch", "continuity_broken"),
-            "trust_level": str(verified_first_contact.get("trust_level", "") or trust_level or "unpinned"),
+        consent = parse_contact_consent(str(plaintext or "")) or {}
+        consent_handshake = str(consent.get("kind", "") or "") in {
+            "contact_offer",
+            "contact_accept",
+            "contact_deny",
         }
+    except Exception:
+        consent_handshake = False
+    if not consent_handshake:
+        from services.mesh.mesh_wormhole_contacts import verified_first_contact_requirement
+
+        verified_first_contact = verified_first_contact_requirement(
+            resolved_peer_id,
+            trust_level=trust_level,
+        )
+        if not verified_first_contact.get("ok"):
+            return {
+                "ok": False,
+                "peer_id": resolved_peer_id,
+                "detail": str(
+                    verified_first_contact.get("detail", "") or "verified first contact required"
+                ),
+                "trust_changed": trust_level in ("mismatch", "continuity_broken"),
+                "trust_level": str(
+                    verified_first_contact.get("trust_level", "") or trust_level or "unpinned"
+                ),
+            }
     peer_bundle_stored = dm_relay.consume_one_time_prekey(resolved_peer_id)
     if not peer_bundle_stored:
+        remote_bundle = dict(stored.get("bundle") or {})
+        otks = list(remote_bundle.get("one_time_prekeys") or [])
+        peer_bundle_stored = {
+            "bundle": remote_bundle,
+            "claimed_one_time_prekey": dict(otks[0] or {}) if otks else {},
+        }
+    if not peer_bundle_stored.get("bundle"):
         return {"ok": False, "detail": "Peer prekey bundle not found"}
     peer_bundle = dict(peer_bundle_stored.get("bundle") or {})
     peer_static = str(peer_bundle.get("identity_dh_pub_key", "") or "")

backend/services/network_utils.py

@@ -34,9 +34,9 @@ _session.mount("http://", HTTPAdapter(max_retries=_retry, pool_maxsize=10))
 # upstream's only recourse was to block "Shadowbroker" as a whole — which
 # would take out every other install too.
 #
-# Fix: give each install a stable pseudonymous handle and include it in
-# the User-Agent. Now an upstream can rate-limit or block the offending
-# operator without affecting anyone else.
+# Fix: give each install a stable pseudonymous handle used as the entire
+# User-Agent product token (no shared "Shadowbroker" label). Upstreams see
+# ``operator-7f3a92`` (or ``OPERATOR_HANDLE``), not one monolithic app name.
 #
 # The handle:
 #
@@ -51,7 +51,6 @@ _session.mount("http://", HTTPAdapter(max_retries=_retry, pool_maxsize=10))
 # - Is NEVER mixed into mesh / Wormhole / Infonet identity. This layer is
 #   strictly for public third-party API attribution.
 
-_SHADOWBROKER_VERSION = "0.9"
 _OPERATOR_HANDLE_FILE = (
     Path(__file__).parent.parent / "data" / "operator_handle.json"
 )
@@ -146,7 +145,12 @@ def get_operator_handle() -> str:
         # 3. On-disk handle from a previous run.
         persisted = _load_persisted_operator_handle()
         if persisted:
-            _OPERATOR_HANDLE_CACHE = _normalize_handle(persisted)
+            normalized = _normalize_handle(persisted)
+            # Migrate legacy auto-generated handles (pre-Round-7a ``shadow-`` prefix).
+            if normalized.startswith("shadow-"):
+                normalized = f"operator-{normalized[len('shadow-'):]}"
+                _persist_operator_handle(normalized)
+            _OPERATOR_HANDLE_CACHE = normalized
             return _OPERATOR_HANDLE_CACHE
 
         # 4. Generate, persist, return.
@@ -170,41 +174,21 @@ def _normalize_handle(raw: str) -> str:
     return safe[:48] if safe else "anonymous"
 
 
-_CONTACT_URL = "https://github.com/BigBodyCobain/Shadowbroker/issues"
-
-
 def outbound_user_agent(purpose: str = "") -> str:
     """Build a User-Agent for an outbound third-party HTTP request.
 
-    Returns something like::
+    Returns the per-install handle only, e.g. ``operator-7f3a92`` or
+    ``operator-7f3a92 (purpose: wikipedia)``. No shared project name — so
+    upstream abuse teams cannot block every install with one ``Shadowbroker``
+    rule.
 
-        Shadowbroker/0.9 (operator: shadow-7f3a92; purpose: wikipedia;
-         +https://github.com/BigBodyCobain/Shadowbroker/issues)
-
-    The ``purpose`` is optional but recommended — it tells the upstream
-    what feature of ours is making the call (``wikipedia``, ``openmhz``,
-    ``nominatim``, etc.), which makes their logs and our complaints
-    actionable.
-
-    Every outbound call in the backend that previously sent a custom
-    User-Agent should call this helper instead. Centralizing here means:
-      - one place to change the contact URL,
-      - one place to bump the version on release,
-      - one place a Wikimedia / OpenMHz operator can reach to ask for
-        the project to back off, with a per-install handle so they can
-        target the specific install instead of the project as a whole.
+    Set ``SHADOWBROKER_USER_AGENT`` to override the entire string if needed.
     """
     handle = get_operator_handle()
     if purpose:
         purpose_clean = _normalize_handle(purpose)
-        return (
-            f"Shadowbroker/{_SHADOWBROKER_VERSION} "
-            f"(operator: {handle}; purpose: {purpose_clean}; +{_CONTACT_URL})"
-        )
-    return (
-        f"Shadowbroker/{_SHADOWBROKER_VERSION} "
-        f"(operator: {handle}; +{_CONTACT_URL})"
-    )
+        return f"{handle} (purpose: {purpose_clean})"
+    return handle
 
 
 def _reset_operator_handle_cache_for_tests() -> None:
@@ -215,19 +199,13 @@ def _reset_operator_handle_cache_for_tests() -> None:
         _OPERATOR_HANDLE_CACHE = ""
 
 
-# Default outbound User-Agent. Retained for backwards compatibility with
-# call sites that haven't been migrated to ``outbound_user_agent()`` yet.
-# Operators who want full per-install attribution should set the
-# ``OPERATOR_HANDLE`` setting and migrate call sites incrementally.
-#
-# Operators who run a public-facing relay can also override the whole UA
-# string via the ``SHADOWBROKER_USER_AGENT`` env var. That override
-# completely bypasses the per-operator helper; only use it if you know
-# what you're doing.
-DEFAULT_USER_AGENT = os.environ.get(
-    "SHADOWBROKER_USER_AGENT",
-    f"Shadowbroker/{_SHADOWBROKER_VERSION}",
-)
+def default_user_agent() -> str:
+    """Default User-Agent for ``fetch_with_curl`` and legacy call sites."""
+    custom = (os.environ.get("SHADOWBROKER_USER_AGENT") or "").strip()
+    if custom:
+        return custom
+    return outbound_user_agent()
+
 
 # Find bash for curl fallback — Git bash's curl has the TLS features
 # needed to pass CDN fingerprint checks (brotli, zstd, libpsl)
@@ -283,7 +261,7 @@ def fetch_with_curl(url, method="GET", json_data=None, timeout=15, headers=None,
     both Python requests and the barebones Windows system curl.
     """
     default_headers = {
-        "User-Agent": DEFAULT_USER_AGENT,
+        "User-Agent": default_user_agent(),
     }
     if headers:
         default_headers.update(headers)

backend/services/news_feed_config.py

@@ -12,6 +12,8 @@ logger = logging.getLogger(__name__)
 CONFIG_PATH = Path(__file__).parent.parent / "config" / "news_feeds.json"
 MAX_FEEDS = 50
 _FEED_URL_REPLACEMENTS = {
+    "http://feeds.bbci.co.uk/news/world/rss.xml": "https://feeds.bbci.co.uk/news/world/rss.xml",
+    "http://www.news.cn/english/rss/worldrss.xml": "https://www.news.cn/english/rss/worldrss.xml",
     "https://www.channelnewsasia.com/rssfeed/8395986": "https://www.channelnewsasia.com/api/v1/rss-outbound-feed?_format=xml",
 }
 _DEAD_FEED_URLS = {
@@ -27,7 +29,7 @@ _DEAD_FEED_URLS = {
 
 DEFAULT_FEEDS = [
     {"name": "NPR", "url": "https://feeds.npr.org/1004/rss.xml", "weight": 4},
-    {"name": "BBC", "url": "http://feeds.bbci.co.uk/news/world/rss.xml", "weight": 3},
+    {"name": "BBC", "url": "https://feeds.bbci.co.uk/news/world/rss.xml", "weight": 3},
     {"name": "AlJazeera", "url": "https://www.aljazeera.com/xml/rss/all.xml", "weight": 2},
     {"name": "NYT", "url": "https://rss.nytimes.com/services/xml/rss/nyt/World.xml", "weight": 1},
     {"name": "GDACS", "url": "https://www.gdacs.org/xml/rss.xml", "weight": 5},
@@ -35,7 +37,7 @@ DEFAULT_FEEDS = [
     {"name": "Bellingcat", "url": "https://www.bellingcat.com/feed/", "weight": 4},
     {"name": "Guardian", "url": "https://www.theguardian.com/world/rss", "weight": 3},
     {"name": "TASS", "url": "https://tass.com/rss/v2.xml", "weight": 2},
-    {"name": "Xinhua", "url": "http://www.news.cn/english/rss/worldrss.xml", "weight": 2},
+    {"name": "Xinhua", "url": "https://www.news.cn/english/rss/worldrss.xml", "weight": 2},
     {"name": "CNA", "url": "https://www.channelnewsasia.com/api/v1/rss-outbound-feed?_format=xml", "weight": 3},
     {"name": "Mercopress", "url": "https://en.mercopress.com/rss/", "weight": 3},
     {"name": "SCMP", "url": "https://www.scmp.com/rss/91/feed", "weight": 4},

backend/services/openclaw_channel.py

@@ -83,6 +83,27 @@ READ_COMMANDS = frozenset({
     "sar_pin_click",
     # Analysis zones (OpenClaw map overlays)
     "list_analysis_zones",
+    # Recon / OSINT toolkit (server-side proxies, SSRF guarded)
+    "osint_lookup",
+    "osint_tools",
+    "entity_expand",
+    # Agent routing helpers
+    "route_query",
+    "run_playbook",
+    "gt_risk_heatmap",
+    "gt_dossier",
+    "gt_analyze",
+    "gt_backtest",
+    "gt_rolling_freeze",
+    "gt_rolling_label",
+    "gt_rolling_backtest",
+    "gt_micro_rolling",
+    "gt_top_alerts",
+    # Private Infonet reads (operator-delegated)
+    "infonet_status",
+    "list_gates",
+    "read_gate_messages",
+    "poll_dms",
 })
 
 WRITE_COMMANDS = frozenset({
@@ -112,6 +133,14 @@ WRITE_COMMANDS = frozenset({
     "place_analysis_zone",
     "delete_analysis_zone",
     "clear_analysis_zones",
+    # Active recon (subnet device discovery)
+    "osint_sweep",
+    # Private Infonet writes (operator wormhole identity)
+    "ensure_infonet_ready",
+    "join_infonet_swarm",
+    "post_gate_message",
+    "cast_vote",
+    "send_dm",
 })
 
 
@@ -637,6 +666,19 @@ def _compact_query_result(result: Any) -> Any:
 # Command dispatcher
 # ---------------------------------------------------------------------------
 
+def _expensive_gate(cmd: str, args: dict[str, Any]) -> dict[str, Any] | None:
+    from services.openclaw_routing import EXPENSIVE_GATE_MESSAGE, requires_expensive_confirm
+
+    if requires_expensive_confirm(cmd, args):
+        return {
+            "ok": False,
+            "detail": EXPENSIVE_GATE_MESSAGE,
+            "code": "expensive_command_blocked",
+            "hint": "route_query",
+        }
+    return None
+
+
 def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
     """Route a command to the appropriate AI Intel function.
 
@@ -644,6 +686,43 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
     Commands run in an isolated thread (via _execute_command) so they
     do not need or touch the caller's event loop.
     """
+    blocked = _expensive_gate(cmd, args)
+    if blocked is not None:
+        return blocked
+
+    if cmd == "route_query":
+        from services.openclaw_routing import route_query
+
+        result = route_query(
+            text=str(args.get("text", "") or args.get("query", "") or ""),
+            lat=args.get("lat"),
+            lng=args.get("lng"),
+            radius_km=float(args.get("radius_km", 50) or 50),
+            compact=bool(args.get("compact", True)),
+        )
+        return {"ok": True, "data": result}
+
+    if cmd == "run_playbook":
+        from services.openclaw_routing import plan_playbook
+
+        plan = plan_playbook(str(args.get("name", "") or args.get("playbook", "")), args)
+        if not plan.get("ok"):
+            return plan
+        batch_results: list[dict[str, Any]] = []
+        for item in plan.get("batch", []):
+            inner_cmd = str(item.get("cmd", "")).strip().lower()
+            inner_args = item.get("args") or {}
+            inner_result = _dispatch_command(inner_cmd, inner_args)
+            batch_results.append({"cmd": inner_cmd, **inner_result})
+        return {
+            "ok": True,
+            "data": {
+                "playbook": plan.get("playbook"),
+                "description": plan.get("description", ""),
+                "results": batch_results,
+            },
+        }
+
     if cmd == "get_telemetry":
         from services.telemetry import get_cached_telemetry_refs
         data = get_cached_telemetry_refs()
@@ -725,6 +804,7 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
             owner=str(args.get("owner", "") or args.get("operator", "") or ""),
             layers=args.get("layers") if isinstance(args.get("layers"), (list, tuple)) else None,
             limit=args.get("limit", 10),
+            fallback_search=bool(args.get("fallback_search") or args.get("confirm_fuzzy")),
         )
         if _wants_compact(args):
             compact = dict(result)
@@ -780,11 +860,290 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
             query=str(args.get("query", "") or ""),
             limit=args.get("limit", 10),
             include_gdelt=bool(args.get("include_gdelt", True)),
+            include_telegram=bool(args.get("include_telegram", True)),
         )
         if _wants_compact(args):
             return {"ok": True, "data": _compact_query_result(result), "format": "compressed_v1"}
         return {"ok": True, "data": result}
 
+    if cmd == "gt_risk_heatmap":
+        from analytics.settings import gt_analytics_enabled
+        from analytics.integration import get_gt_engine
+        from services.fetchers._store import get_latest_data_subset_refs
+
+        if not gt_analytics_enabled():
+            return {"ok": True, "data": {"enabled": False, "features": [], "clusters": []}}
+        snap = get_latest_data_subset_refs("gt_risk")
+        payload = dict(snap.get("gt_risk") or {})
+        engine = get_gt_engine()
+        if engine is not None and not payload.get("heatmap"):
+            payload["heatmap"] = engine.get_risk_heatmap()
+        return {"ok": True, "data": payload}
+
+    if cmd == "gt_dossier":
+        from analytics.settings import gt_analytics_enabled
+        from analytics.integration import get_gt_engine
+
+        region = str(args.get("region", "") or args.get("area", "") or "").strip().lower()
+        if not region:
+            return {"ok": False, "detail": "region required (e.g. ukraine, uk, europe)"}
+        if not gt_analytics_enabled():
+            return {
+                "ok": True,
+                "data": {
+                    "enabled": False,
+                    "region": region,
+                    "interpretation": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
+                },
+            }
+        engine = get_gt_engine()
+        if engine is None:
+            return {"ok": False, "detail": "GT analytics engine unavailable"}
+        return {"ok": True, "data": engine.get_dossier(region)}
+
+    if cmd == "gt_analyze":
+        from analytics.settings import gt_analytics_enabled
+        from analytics.integration import get_gt_engine, refresh_from_latest_data
+        from services.fetchers._store import _data_lock, latest_data
+
+        if not gt_analytics_enabled():
+            return {"ok": False, "detail": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED)"}
+        engine = get_gt_engine()
+        if engine is None:
+            return {"ok": False, "detail": "GT analytics engine unavailable"}
+
+        feeds = args.get("feeds") if isinstance(args.get("feeds"), (list, tuple)) else None
+        if feeds:
+            from analytics.feed_adapter import normalize_feed_item
+
+            ingested = 0
+            for raw in feeds:
+                if not isinstance(raw, dict):
+                    continue
+                item = normalize_feed_item(raw, source_type=str(raw.get("source_type") or "openclaw"))
+                result = engine.process_feed_item(item)
+                if result and not result.get("skipped"):
+                    ingested += 1
+            summary = {"ingested": ingested, "enabled": True}
+        else:
+            with _data_lock:
+                snapshot = dict(latest_data)
+            summary = refresh_from_latest_data(snapshot, persist=True)
+
+        region = str(args.get("region", "") or "").strip().lower()
+        data = {
+            "refresh": summary,
+            "heatmap_features": len((summary.get("sample") or [])),
+        }
+        if region:
+            data["dossier"] = engine.get_dossier(region)
+        else:
+            data["heatmap"] = engine.get_risk_heatmap()
+            data["clusters"] = engine.compute_herding_clusters()[:5]
+        return {"ok": True, "data": data}
+
+    if cmd == "gt_backtest":
+        from analytics.backtest import (
+            DEFAULT_BACKTEST_ALERT_THRESHOLD,
+            run_historical_backtest,
+            tune_alert_threshold,
+        )
+        from analytics.historical_events import default_historical_cases, expanded_historical_cases
+        from analytics.settings import gt_analytics_enabled
+
+        if not gt_analytics_enabled():
+            return {
+                "ok": True,
+                "data": {
+                    "enabled": False,
+                    "message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
+                },
+            }
+
+        expanded = bool(args.get("expanded", True))
+        tune = bool(args.get("tune", False))
+        include_cases = bool(args.get("include_cases", False))
+        try:
+            target_confidence = float(args.get("target_confidence", 0.95))
+        except (TypeError, ValueError):
+            target_confidence = 0.95
+
+        if tune:
+            suite = expanded_historical_cases() if expanded else default_historical_cases()
+            threshold, report = tune_alert_threshold(
+                suite,
+                target_confidence=target_confidence,
+            )
+        else:
+            raw_threshold = args.get("alert_threshold")
+            threshold = (
+                float(raw_threshold)
+                if raw_threshold is not None
+                else DEFAULT_BACKTEST_ALERT_THRESHOLD
+            )
+            report = run_historical_backtest(
+                use_expanded_suite=expanded,
+                alert_threshold=threshold,
+                target_confidence=target_confidence,
+            )
+
+        data = report.to_dict()
+        data["enabled"] = True
+        data["expanded_suite"] = expanded
+        data["tuned"] = tune
+        data["recommended_alert_threshold"] = threshold
+        if _wants_compact(args) or not include_cases:
+            data.pop("cases", None)
+        return {"ok": True, "data": data}
+
+    if cmd == "gt_rolling_freeze":
+        from analytics.rolling_backtest import freeze_weekly_snapshot
+        from analytics.settings import gt_analytics_enabled
+
+        if not gt_analytics_enabled():
+            return {
+                "ok": True,
+                "data": {
+                    "enabled": False,
+                    "message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
+                },
+            }
+
+        week_id = str(args.get("week_id", "") or "").strip() or None
+        force = bool(args.get("force", False))
+        result = freeze_weekly_snapshot(
+            week_id=week_id,
+            force=force,
+            frozen_by="openclaw",
+        )
+        if not result.get("ok"):
+            return {"ok": False, "detail": result.get("detail", "Freeze failed")}
+        data = dict(result)
+        data["enabled"] = True
+        if _wants_compact(args):
+            data.pop("snapshot", None)
+        return {"ok": True, "data": data}
+
+    if cmd == "gt_rolling_label":
+        from analytics.rolling_backtest import label_region, label_regions
+        from analytics.settings import gt_analytics_enabled
+
+        if not gt_analytics_enabled():
+            return {
+                "ok": True,
+                "data": {
+                    "enabled": False,
+                    "message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
+                },
+            }
+
+        week_id = str(args.get("week_id", "") or "").strip()
+        if not week_id:
+            return {"ok": False, "detail": "week_id required"}
+
+        labels = args.get("labels")
+        if isinstance(labels, list) and labels:
+            result = label_regions(week_id, labels, labeled_by="openclaw")
+        else:
+            region = str(args.get("region", "") or "").strip().lower()
+            label = str(args.get("label", "") or "").strip().lower()
+            if not region or not label:
+                return {"ok": False, "detail": "region and label required (or labels batch)"}
+            result = label_region(
+                week_id,
+                region,
+                label,  # type: ignore[arg-type]
+                notes=str(args.get("notes", "") or ""),
+                labeled_by="openclaw",
+            )
+
+        if not result.get("ok"):
+            return {"ok": False, "detail": result.get("detail", "Label failed")}
+        data = dict(result)
+        data["enabled"] = True
+        return {"ok": True, "data": data}
+
+    if cmd == "gt_rolling_backtest":
+        from analytics.rolling_backtest import rolling_report
+        from analytics.settings import gt_analytics_enabled
+
+        if not gt_analytics_enabled():
+            return {
+                "ok": True,
+                "data": {
+                    "enabled": False,
+                    "message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
+                },
+            }
+
+        try:
+            weeks = int(args.get("weeks", 8))
+        except (TypeError, ValueError):
+            weeks = 8
+        try:
+            target_confidence = float(args.get("target_confidence", 0.80))
+        except (TypeError, ValueError):
+            target_confidence = 0.80
+
+        data = rolling_report(weeks=weeks, target_confidence=target_confidence)
+        data["enabled"] = True
+        if _wants_compact(args):
+            for row in data.get("trend") or []:
+                if isinstance(row, dict):
+                    row.pop("frozen_at", None)
+        return {"ok": True, "data": data}
+
+    if cmd == "gt_top_alerts":
+        from analytics.gt_alerts import top_gt_alerts
+        from analytics.settings import gt_analytics_enabled
+
+        if not gt_analytics_enabled():
+            return {
+                "ok": True,
+                "data": {
+                    "enabled": False,
+                    "message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
+                },
+            }
+
+        try:
+            limit = int(args.get("limit", 8))
+        except (TypeError, ValueError):
+            limit = 8
+
+        data = top_gt_alerts(limit=limit)
+        data["enabled"] = True
+        return {"ok": True, "data": data}
+
+    if cmd == "gt_micro_rolling":
+        from analytics.micro_rolling import micro_rolling_report
+        from analytics.settings import gt_analytics_enabled
+
+        if not gt_analytics_enabled():
+            return {
+                "ok": True,
+                "data": {
+                    "enabled": False,
+                    "message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
+                },
+            }
+
+        try:
+            window_days = int(args.get("window_days", 3))
+        except (TypeError, ValueError):
+            window_days = 3
+        try:
+            limit = int(args.get("limit", 15))
+        except (TypeError, ValueError):
+            limit = 15
+
+        data = micro_rolling_report(window_days=window_days, limit=limit)
+        data["enabled"] = True
+        if _wants_compact(args):
+            data.pop("top_regions", None)
+            data["ignitions"] = (data.get("ignitions") or [])[:5]
+        return {"ok": True, "data": data}
+
     if cmd == "brief_area":
         from services.telemetry import entities_near, search_news, get_layer_slice
         lat = args.get("lat")
@@ -846,6 +1205,26 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
             return {"ok": True, "data": _compact_query_result(result), "format": "compressed_v1"}
         return {"ok": True, "data": result}
 
+    if cmd == "osint_lookup":
+        from services.osint.openclaw_recon import run_osint_lookup
+        tool = str(args.get("tool", "") or args.get("lookup", "") or args.get("type", "") or "")
+        result = run_osint_lookup(tool, args)
+        return {"ok": True, "data": result, "tool": tool.strip().lower()}
+
+    if cmd == "osint_tools":
+        from services.osint.openclaw_recon import osint_tool_help
+        return {"ok": True, "data": osint_tool_help()}
+
+    if cmd == "osint_sweep":
+        from services.osint.openclaw_recon import run_osint_sweep
+        result = run_osint_sweep(args)
+        return {"ok": True, "data": result}
+
+    if cmd == "entity_expand":
+        from services.osint.openclaw_recon import run_entity_expand
+        result = run_entity_expand(args)
+        return {"ok": True, "data": result}
+
     if cmd == "get_report":
         from services.telemetry import get_cached_telemetry_refs, get_cached_slow_telemetry_refs
         fast = get_cached_telemetry_refs()
@@ -1039,7 +1418,7 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
         from services.openclaw_watchdog import add_watch
         watch_type = str(args.get("type", "")).strip()
         if not watch_type:
-            return {"ok": False, "detail": "watch type required (track_aircraft, track_callsign, track_registration, track_ship, track_entity, geofence, keyword, prediction_market)"}
+            return {"ok": False, "detail": "watch type required (track_aircraft, track_callsign, track_registration, track_ship, track_entity, geofence, keyword, telegram_rhetoric, prediction_market)"}
         watch_params = args.get("params", {})
         if not watch_params:
             # Allow flat args (e.g. {type: "track_callsign", callsign: "N189AM"})
@@ -1065,6 +1444,7 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
             owner=str(args.get("owner", "") or args.get("operator", "") or ""),
             layers=args.get("layers") if isinstance(args.get("layers"), (list, tuple)) else None,
             limit=5,
+            fallback_search=True,
         )
         best = lookup.get("best_match") if isinstance(lookup.get("best_match"), dict) else {}
         group = str(best.get("group", "") or entity_type).lower()
@@ -1516,6 +1896,85 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
         count = clear_zones(source="openclaw")
         return {"ok": True, "data": {"removed_count": count}}
 
+    # -- Infonet / gate / DM (operator-delegated, full tier for writes) ------
+
+    if cmd == "infonet_status":
+        from services.openclaw_infonet import get_infonet_status
+
+        return get_infonet_status()
+
+    if cmd == "ensure_infonet_ready":
+        from services.openclaw_infonet import ensure_infonet_ready
+
+        return ensure_infonet_ready(join_swarm=bool(args.get("join_swarm", True)))
+
+    if cmd == "join_infonet_swarm":
+        from services.openclaw_infonet import join_infonet_swarm
+
+        return join_infonet_swarm()
+
+    if cmd == "list_gates":
+        from services.openclaw_infonet import list_gates
+
+        return list_gates()
+
+    if cmd == "read_gate_messages":
+        from services.openclaw_infonet import read_gate_messages
+
+        gate_id = str(args.get("gate_id", "") or args.get("gate", "")).strip()
+        return read_gate_messages(
+            gate_id,
+            limit=int(args.get("limit", 20) or 20),
+            decrypt=bool(args.get("decrypt", False)),
+        )
+
+    if cmd == "post_gate_message":
+        from services.openclaw_infonet import post_gate_message
+
+        gate_id = str(args.get("gate_id", "") or args.get("gate", "")).strip()
+        plaintext = str(args.get("plaintext", "") or args.get("message", "")).strip()
+        return post_gate_message(
+            gate_id,
+            plaintext,
+            reply_to=str(args.get("reply_to", "") or ""),
+        )
+
+    if cmd == "cast_vote":
+        from services.openclaw_infonet import cast_vote
+
+        target_id = str(args.get("target_id", "") or args.get("target", "")).strip()
+        vote_raw = args.get("vote", args.get("direction"))
+        try:
+            vote_val = int(vote_raw)
+        except (TypeError, ValueError):
+            return {"ok": False, "detail": "vote must be 1 or -1"}
+        return cast_vote(
+            target_id,
+            vote_val,
+            gate=str(args.get("gate", "") or args.get("gate_id", "")).strip(),
+        )
+
+    if cmd == "send_dm":
+        from services.openclaw_infonet import send_dm
+
+        peer_id = str(
+            args.get("peer_id", "")
+            or args.get("recipient_id", "")
+            or args.get("recipient", "")
+        ).strip()
+        plaintext = str(args.get("plaintext", "") or args.get("message", "")).strip()
+        return send_dm(
+            peer_id,
+            plaintext,
+            delivery_class=str(args.get("delivery_class", "shared") or "shared"),
+            recipient_token=str(args.get("recipient_token", "") or ""),
+        )
+
+    if cmd == "poll_dms":
+        from services.openclaw_infonet import poll_dms
+
+        return poll_dms(limit=int(args.get("limit", 20) or 20))
+
     return {"ok": False, "detail": f"unhandled command: {cmd}"}
 
 

backend/services/openclaw_infonet.py

@@ -0,0 +1,796 @@
+"""OpenClaw agent delegation for private Infonet / gate / DM actions.
+
+Agents authenticate with OpenClaw HMAC on the command channel. Write
+commands require ``OPENCLAW_ACCESS_TIER=full``. Actions use the operator's
+local wormhole persona and node runtime — the agent posts on behalf of the
+user who configured the skill, not as a separate fleet identity.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import secrets
+import time
+from typing import Any
+
+from starlette.requests import Request
+
+logger = logging.getLogger(__name__)
+
+
+def _run_async(coro):
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        return asyncio.run(coro)
+    return asyncio.run(coro)
+
+
+def _local_agent_request(path: str, *, method: str = "POST") -> Request:
+    scope = {
+        "type": "http",
+        "method": method.upper(),
+        "path": path,
+        "headers": [],
+        "client": ("127.0.0.1", 52421),
+    }
+    request = Request(scope)
+    request.state._private_lane_current_tier = "private_strong"
+    request.state._transport_tier = "private_strong"
+    return request
+
+
+def ensure_infonet_ready(*, join_swarm: bool = True) -> dict[str, Any]:
+    """Warm Tor, enable the participant node, and optionally join the swarm."""
+    from routers.ai_intel import _write_env_value
+    from services.config import get_settings
+    from services.mesh.mesh_swarm_runtime import join_swarm_with_retries
+    from services.node_settings import read_node_settings, write_node_settings
+    from services.tor_hidden_service import tor_service
+    from services.wormhole_supervisor import _check_arti_ready
+
+    steps: dict[str, Any] = {}
+
+    tor_result = tor_service.start(target_port=8000)
+    steps["tor"] = tor_result
+    if tor_result.get("ok"):
+        try:
+            _write_env_value("MESH_ARTI_ENABLED", "true")
+            get_settings.cache_clear()
+        except Exception as exc:
+            logger.debug("failed to persist MESH_ARTI_ENABLED: %s", exc)
+
+    if not _check_arti_ready():
+        return {
+            "ok": False,
+            "detail": "Tor/Arti transport is not ready yet",
+            "steps": steps,
+        }
+
+    if not bool(read_node_settings().get("enabled")):
+        write_node_settings(enabled=True)
+        steps["node_enabled"] = True
+        try:
+            import main as main_mod
+
+            main_mod._refresh_node_peer_store()
+            main_mod._start_infonet_node_runtime("openclaw_agent")
+        except Exception as exc:
+            logger.warning("node runtime start after agent enable failed: %s", exc)
+    else:
+        steps["node_enabled"] = True
+
+    if join_swarm:
+        joined = join_swarm_with_retries()
+        steps["announce"] = joined.get("announce") or {}
+        steps["manifest_pull"] = joined.get("manifest_pull") or {}
+        steps["swarm_attempts"] = joined.get("attempts")
+        ok = bool(joined.get("ok"))
+    else:
+        ok = True
+
+    return {
+        "ok": ok,
+        "detail": "Infonet participant runtime ready" if ok else "swarm join incomplete",
+        "steps": steps,
+        "onion_address": str(tor_result.get("onion_address") or ""),
+    }
+
+
+def join_infonet_swarm() -> dict[str, Any]:
+    from services.mesh.mesh_swarm_runtime import join_swarm_with_retries
+
+    joined = join_swarm_with_retries()
+    return {
+        "ok": bool(joined.get("ok")),
+        "announce": joined.get("announce") or {},
+        "manifest_pull": joined.get("manifest_pull") or {},
+        "attempts": joined.get("attempts"),
+        "detail": joined.get("detail"),
+    }
+
+
+def get_infonet_status() -> dict[str, Any]:
+    from services.mesh.mesh_hashchain import infonet
+    from services.wormhole_supervisor import get_wormhole_state
+
+    info = infonet.get_info()
+    valid, reason = infonet.validate_chain(verify_signatures=False)
+    try:
+        wormhole = get_wormhole_state()
+    except Exception:
+        wormhole = {"configured": False, "ready": False, "arti_ready": False, "rns_ready": False}
+    try:
+        import main as main_mod
+
+        runtime = main_mod._node_runtime_snapshot()
+        private_tier = main_mod._current_private_lane_tier(wormhole)
+    except Exception:
+        runtime = {}
+        private_tier = "public_degraded"
+
+    return {
+        "ok": True,
+        "chain": info,
+        "valid": valid,
+        "validation": reason,
+        "private_lane_tier": private_tier,
+        "wormhole": wormhole,
+        "runtime": runtime,
+    }
+
+
+def list_gates() -> dict[str, Any]:
+    from services.mesh.mesh_reputation import gate_manager
+
+    return {"ok": True, "gates": gate_manager.list_gates()}
+
+
+def read_gate_messages(
+    gate_id: str,
+    *,
+    limit: int = 20,
+    decrypt: bool = False,
+) -> dict[str, Any]:
+    from services.mesh.mesh_hashchain import gate_store
+
+    gate_key = str(gate_id or "").strip().lower()
+    if not gate_key:
+        return {"ok": False, "detail": "gate_id required"}
+
+    messages, cursor = gate_store.get_messages_with_cursor(gate_key, limit=max(1, min(int(limit), 100)))
+    out = []
+    if decrypt:
+        from services.mesh.mesh_gate_repair import decrypt_gate_message_with_repair
+
+        for msg in messages:
+            item = dict(msg)
+            try:
+                decrypted = decrypt_gate_message_with_repair(
+                    gate_id=gate_key,
+                    epoch=int(item.get("epoch") or 0),
+                    ciphertext=str(item.get("ciphertext") or ""),
+                    nonce=str(item.get("nonce") or item.get("iv") or ""),
+                    sender_ref=str(item.get("sender_ref") or ""),
+                    gate_envelope=str(item.get("gate_envelope") or ""),
+                    envelope_hash=str(item.get("envelope_hash") or ""),
+                    event_id=str(item.get("event_id") or ""),
+                )
+                if decrypted.get("ok"):
+                    item["plaintext"] = decrypted.get("plaintext", "")
+            except Exception as exc:
+                item["decrypt_error"] = str(exc)
+            out.append(item)
+    else:
+        out = [dict(m) for m in messages]
+
+    return {
+        "ok": True,
+        "gate": gate_key,
+        "count": len(out),
+        "cursor": cursor,
+        "messages": out,
+    }
+
+
+def post_gate_message(
+    gate_id: str,
+    plaintext: str,
+    *,
+    reply_to: str = "",
+) -> dict[str, Any]:
+    """Compose, sign, and post an MLS gate message using the operator persona."""
+    from services.mesh.mesh_gate_repair import (
+        compose_gate_message_with_repair,
+        sign_gate_message_with_repair,
+    )
+    from services.mesh.mesh_wormhole_persona import bootstrap_wormhole_persona_state, create_gate_persona
+
+    gate_key = str(gate_id or "").strip().lower()
+    if not gate_key:
+        return {"ok": False, "detail": "gate_id required"}
+    if not str(plaintext or "").strip():
+        return {"ok": False, "detail": "plaintext required"}
+
+    bootstrap_wormhole_persona_state(force=False)
+    try:
+        create_gate_persona(gate_key, label="openclaw-agent")
+    except Exception:
+        pass
+
+    composed = compose_gate_message_with_repair(
+        gate_id=gate_key,
+        plaintext=str(plaintext),
+        reply_to=str(reply_to or ""),
+    )
+    if not composed.get("ok"):
+        return composed
+
+    signed = sign_gate_message_with_repair(
+        gate_id=gate_key,
+        epoch=int(composed.get("epoch") or 0),
+        ciphertext=str(composed.get("ciphertext") or ""),
+        nonce=str(composed.get("nonce") or ""),
+        payload_format=str(composed.get("format") or "mls1"),
+        reply_to=str(reply_to or ""),
+        envelope_hash=str(composed.get("envelope_hash") or ""),
+        transport_lock="private_strong",
+    )
+    if not signed.get("ok"):
+        return signed
+
+    body = {
+        "sender_id": str(signed.get("sender_id") or composed.get("sender_id") or ""),
+        "public_key": str(signed.get("public_key") or composed.get("public_key") or ""),
+        "public_key_algo": str(signed.get("public_key_algo") or composed.get("public_key_algo") or ""),
+        "signature": str(signed.get("signature") or ""),
+        "sequence": int(signed.get("sequence") or composed.get("sequence") or 0),
+        "protocol_version": str(signed.get("protocol_version") or composed.get("protocol_version") or ""),
+        "epoch": int(signed.get("epoch") or composed.get("epoch") or 0),
+        "ciphertext": str(signed.get("ciphertext") or composed.get("ciphertext") or ""),
+        "nonce": str(signed.get("nonce") or composed.get("nonce") or ""),
+        "sender_ref": str(signed.get("sender_ref") or composed.get("sender_ref") or ""),
+        "format": str(signed.get("format") or composed.get("format") or "mls1"),
+        "gate_envelope": str(signed.get("gate_envelope") or composed.get("gate_envelope") or ""),
+        "envelope_hash": str(signed.get("envelope_hash") or composed.get("envelope_hash") or ""),
+        "transport_lock": "private_strong",
+        "reply_to": str(signed.get("reply_to") or reply_to or ""),
+    }
+
+    import main as main_mod
+
+    path = f"/api/mesh/gate/{gate_key}/message"
+    request = _local_agent_request(path)
+    return main_mod._submit_gate_message_envelope(request, gate_key, body)
+
+
+def cast_vote(
+    target_id: str,
+    vote: int,
+    *,
+    gate: str = "",
+) -> dict[str, Any]:
+    """Cast a signed reputation vote using the operator gate/transport persona."""
+    from services.mesh.mesh_hashchain import infonet
+    from services.mesh.mesh_protocol import PROTOCOL_VERSION, normalize_payload
+    from services.mesh.mesh_reputation import gate_manager, reputation_ledger
+    from services.mesh.mesh_wormhole_persona import (
+        bootstrap_wormhole_persona_state,
+        sign_gate_wormhole_event,
+        sign_public_wormhole_event,
+    )
+
+    voter_gate = str(gate or "").strip().lower()
+    target = str(target_id or "").strip()
+    vote_val = int(vote)
+    if not target:
+        return {"ok": False, "detail": "target_id required"}
+    if vote_val not in (1, -1):
+        return {"ok": False, "detail": "vote must be 1 or -1"}
+
+    bootstrap_wormhole_persona_state(force=False)
+    vote_payload = {"target_id": target, "vote": vote_val, "gate": voter_gate}
+    normalized = normalize_payload("vote", vote_payload)
+    ok_payload, reason = True, "ok"
+    from services.mesh.mesh_schema import validate_event_payload
+
+    ok_payload, reason = validate_event_payload("vote", normalized)
+    if not ok_payload:
+        return {"ok": False, "detail": reason}
+
+    if voter_gate:
+        signed = sign_gate_wormhole_event(
+            gate_id=voter_gate,
+            event_type="vote",
+            payload=normalized,
+        )
+    else:
+        signed = sign_public_wormhole_event(event_type="vote", payload=normalized)
+
+    if not signed.get("ok", True):
+        return signed
+
+    voter_id = str(signed.get("node_id") or "")
+    public_key = str(signed.get("public_key") or "")
+    public_key_algo = str(signed.get("public_key_algo") or "")
+    signature = str(signed.get("signature") or "")
+    sequence = int(signed.get("sequence") or 0)
+
+    if voter_gate:
+        can_enter, enter_reason = gate_manager.can_enter(voter_id, voter_gate)
+        if not can_enter:
+            return {"ok": False, "detail": f"Gate vote denied: {enter_reason}"}
+
+    reputation_ledger.register_node(voter_id, public_key, public_key_algo)
+    stable_voter_id = voter_id
+    try:
+        import main as main_mod
+
+        root_nid = main_mod._cached_root_node_id()
+        if root_nid:
+            stable_voter_id = root_nid
+    except Exception:
+        pass
+
+    ok, cast_reason, weight = reputation_ledger.cast_vote(
+        stable_voter_id,
+        target,
+        vote_val,
+        voter_gate,
+    )
+    if ok:
+        try:
+            infonet.append(
+                event_type="vote",
+                node_id=voter_id,
+                payload=normalized,
+                signature=signature,
+                sequence=sequence,
+                public_key=public_key,
+                public_key_algo=public_key_algo,
+                protocol_version=str(signed.get("protocol_version") or PROTOCOL_VERSION),
+            )
+        except Exception as exc:
+            logger.warning("vote recorded in ledger but infonet append failed: %s", exc)
+
+    return {"ok": ok, "detail": cast_reason, "weight": round(float(weight or 0), 2)}
+
+
+def _http_post_json(
+    url: str,
+    body: dict[str, Any],
+    *,
+    extra_headers: dict[str, str] | None = None,
+    timeout: int = 120,
+) -> dict[str, Any]:
+    import urllib.error
+    import urllib.request
+
+    payload_bytes = json.dumps(body, separators=(",", ":"), sort_keys=True).encode("utf-8")
+    headers = {"Content-Type": "application/json"}
+    if extra_headers:
+        headers.update(extra_headers)
+    req = urllib.request.Request(url, data=payload_bytes, headers=headers, method="POST")
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            raw = resp.read().decode("utf-8")
+    except urllib.error.HTTPError as exc:
+        detail = exc.read().decode("utf-8", errors="replace")
+        try:
+            parsed = json.loads(detail)
+            if isinstance(parsed, dict):
+                return parsed
+        except Exception:
+            pass
+        return {"ok": False, "detail": detail or f"http {exc.code}"}
+    if not raw:
+        return {}
+    parsed = json.loads(raw)
+    return parsed if isinstance(parsed, dict) else {"ok": False, "detail": "invalid json response"}
+
+
+def _issue_sender_token_for_http_send(
+    api_base: str,
+    *,
+    recipient: str,
+    delivery: str,
+    recipient_token: str,
+) -> dict[str, Any]:
+    extra_headers: dict[str, str] = {}
+    admin_key = str(os.environ.get("ADMIN_KEY") or "").strip()
+    if admin_key:
+        extra_headers["X-Admin-Key"] = admin_key
+    return _http_post_json(
+        f"{api_base}/api/wormhole/dm/sender-token",
+        {
+            "recipient_id": recipient,
+            "delivery_class": delivery,
+            "recipient_token": recipient_token,
+        },
+        extra_headers=extra_headers or None,
+    )
+
+
+def _submit_signed_dm_send(
+    *,
+    recipient: str,
+    delivery_class: str,
+    recipient_token: str,
+    ciphertext: str,
+    payload_format: str,
+    session_welcome: str = "",
+    connect_intent: str = "",
+    lookup_peer_url: str = "",
+    peer_dh_pub: str = "",
+) -> dict[str, Any]:
+    import main as main_mod
+    from services.mesh.mesh_protocol import (
+        PROTOCOL_VERSION,
+        SIGNED_CONTEXT_FIELD,
+        build_signed_context,
+    )
+    from services.mesh.mesh_schema import validate_event_payload
+    from services.mesh.mesh_wormhole_persona import get_dm_identity, sign_dm_wormhole_event
+    from services.mesh.mesh_wormhole_sender_token import issue_wormhole_dm_sender_token
+
+    delivery = str(delivery_class or "shared").strip().lower()
+    identity = get_dm_identity()
+    sender_id = str(identity.get("node_id") or "")
+    msg_id = secrets.token_hex(16)
+    timestamp = int(time.time())
+    sequence = int(identity.get("sequence", 0) or 0) + 1
+
+    dm_payload: dict[str, Any] = {
+        "recipient_id": recipient,
+        "delivery_class": delivery,
+        "recipient_token": str(recipient_token or ""),
+        "ciphertext": str(ciphertext or ""),
+        "msg_id": msg_id,
+        "timestamp": timestamp,
+        "format": str(payload_format or "mls1"),
+        "transport_lock": "private_strong",
+    }
+    if session_welcome:
+        dm_payload["session_welcome"] = str(session_welcome)
+
+    try:
+        from services.config import get_settings
+        from services.mesh.mesh_wormhole_seal import build_sender_seal
+
+        if (
+            delivery == "shared"
+            and bool(get_settings().MESH_DM_REQUIRE_SENDER_SEAL_SHARED)
+            and not str(dm_payload.get("sender_seal", "") or "").strip()
+        ):
+            seal = build_sender_seal(
+                recipient_id=recipient,
+                recipient_dh_pub=str(peer_dh_pub or ""),
+                msg_id=msg_id,
+                timestamp=timestamp,
+            )
+            if seal.get("ok"):
+                dm_payload["sender_seal"] = str(seal.get("sender_seal") or "")
+    except Exception:
+        pass
+
+    ok_payload, reason = validate_event_payload("dm_message", dm_payload)
+    if not ok_payload:
+        return {"ok": False, "detail": reason}
+
+    dm_payload[SIGNED_CONTEXT_FIELD] = build_signed_context(
+        event_type="dm_message",
+        kind="dm_send",
+        endpoint="/api/mesh/dm/send",
+        lane_floor="private_strong",
+        sequence_domain="dm_send",
+        node_id=sender_id,
+        sequence=sequence,
+        payload=dm_payload,
+        recipient_id=recipient,
+    )
+    signed = sign_dm_wormhole_event(
+        event_type="dm_message",
+        payload=dm_payload,
+        sequence=sequence,
+    )
+    if not signed.get("ok", True):
+        return signed
+
+    body = {
+        "sender_id": sender_id,
+        "sender_token": "",
+        "recipient_id": recipient,
+        "delivery_class": delivery,
+        "recipient_token": str(recipient_token or ""),
+        "ciphertext": str(ciphertext or ""),
+        "format": str(payload_format or "mls1"),
+        "transport_lock": "private_strong",
+        "session_welcome": str(session_welcome or ""),
+        "msg_id": msg_id,
+        "timestamp": timestamp,
+        "sender_seal": str(dm_payload.get("sender_seal") or ""),
+        "public_key": str(signed.get("public_key") or ""),
+        "public_key_algo": str(signed.get("public_key_algo") or ""),
+        "signature": str(signed.get("signature") or ""),
+        "sequence": int(signed.get("sequence") or 0),
+        "protocol_version": str(signed.get("protocol_version") or PROTOCOL_VERSION),
+        "signed_context": dict(dm_payload.get(SIGNED_CONTEXT_FIELD) or {}),
+    }
+    normalized_intent = str(connect_intent or "").strip().lower()
+    normalized_lookup_peer = str(lookup_peer_url or "").strip().rstrip("/")
+    if normalized_intent:
+        body["connect_intent"] = normalized_intent
+    if normalized_lookup_peer:
+        body["lookup_peer_url"] = normalized_lookup_peer
+
+    api_base = str(os.environ.get("SB_API_BASE", "http://127.0.0.1:8000") or "http://127.0.0.1:8000").rstrip("/")
+    result: dict[str, Any] = {"ok": False, "detail": "dm send failed"}
+    try:
+        import urllib.error
+
+        if delivery in ("request", "shared"):
+            issued = _issue_sender_token_for_http_send(
+                api_base,
+                recipient=recipient,
+                delivery=delivery,
+                recipient_token=str(recipient_token or ""),
+            )
+            if not issued.get("ok"):
+                return issued
+            body["sender_token"] = str(issued.get("sender_token") or "")
+
+        result = _http_post_json(f"{api_base}/api/mesh/dm/send", body)
+    except (urllib.error.URLError, TimeoutError):
+        if delivery in ("request", "shared"):
+            issued = issue_wormhole_dm_sender_token(
+                recipient_id=recipient,
+                delivery_class=delivery,
+                recipient_token=str(recipient_token or ""),
+            )
+            if not issued.get("ok"):
+                return issued
+            body["sender_token"] = str(issued.get("sender_token") or "")
+
+        async def _send():
+            import json as _json
+
+            raw = _json.dumps(body).encode("utf-8")
+
+            async def receive():
+                return {"type": "http.request", "body": raw, "more_body": False}
+
+            req = Request(
+                {
+                    "type": "http",
+                    "method": "POST",
+                    "path": "/api/mesh/dm/send",
+                    "headers": [(b"content-type", b"application/json")],
+                    "client": ("127.0.0.1", 52421),
+                },
+                receive,
+            )
+            req.state._private_lane_current_tier = "private_strong"
+            req.state._transport_tier = "private_strong"
+            return await main_mod.dm_send(req)
+
+        result = _run_async(_send())
+    except Exception as exc:
+        result = {"ok": False, "detail": str(exc) or type(exc).__name__}
+    if isinstance(result, dict):
+        result.setdefault("msg_id", msg_id)
+        result.setdefault("sender_id", sender_id)
+        result.setdefault("recipient_id", recipient)
+    return result
+
+
+def send_contact_request(
+    *,
+    lookup_token: str = "",
+    peer_id: str = "",
+    note: str = "",
+    lookup_peer_url: str = "",
+    cached_prekey_bundle: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Send a first-contact request using a short address or peer id."""
+    from services.mesh.mesh_wormhole_dead_drop import build_contact_offer
+    from services.mesh.mesh_wormhole_persona import get_dm_identity
+    from services.mesh.mesh_wormhole_prekey import bootstrap_encrypt_for_peer, fetch_dm_prekey_bundle
+
+    token = str(lookup_token or "").strip()
+    peer = str(peer_id or "").strip()
+    if not token and not peer:
+        return {"ok": False, "detail": "lookup_token or peer_id required"}
+
+    preferred_peer = str(lookup_peer_url or "").strip().rstrip("/")
+    if cached_prekey_bundle and cached_prekey_bundle.get("ok"):
+        bundle = dict(cached_prekey_bundle)
+    else:
+        bundle = fetch_dm_prekey_bundle(
+            agent_id=peer if not token else "",
+            lookup_token=token,
+            lookup_peer_urls=[preferred_peer] if preferred_peer else None,
+        )
+    if not bundle.get("ok"):
+        return bundle
+    recipient = str(bundle.get("agent_id") or peer).strip()
+    if not recipient:
+        return {"ok": False, "detail": "recipient unresolved"}
+
+    identity = get_dm_identity()
+    offer = build_contact_offer(
+        dh_pub_key=str(identity.get("dh_pub_key") or ""),
+        dh_algo=str(identity.get("dh_algo") or "X25519"),
+        geo_hint=str(note or ""),
+    )
+    encrypted = bootstrap_encrypt_for_peer(
+        recipient,
+        offer,
+        fetched_bundle=bundle,
+    )
+    if not encrypted.get("ok"):
+        return encrypted
+
+    return _submit_signed_dm_send(
+        recipient=recipient,
+        delivery_class="request",
+        recipient_token="",
+        ciphertext=str(encrypted.get("result") or ""),
+        payload_format="mls1",
+        connect_intent="contact_request",
+        lookup_peer_url=preferred_peer,
+    )
+
+
+def send_contact_accept(
+    *,
+    peer_id: str,
+    peer_dh_pub: str = "",
+    lookup_token: str = "",
+    lookup_peer_url: str = "",
+) -> dict[str, Any]:
+    """Accept a pending contact request and open the shared DM lane."""
+    from services.mesh.mesh_wormhole_dead_drop import build_contact_accept, issue_pairwise_dm_alias
+    from services.mesh.mesh_wormhole_prekey import bootstrap_encrypt_for_peer, fetch_dm_prekey_bundle
+
+    peer = str(peer_id or "").strip()
+    if not peer:
+        return {"ok": False, "detail": "peer_id required"}
+
+    token = str(lookup_token or "").strip()
+    preferred_peer = str(lookup_peer_url or "").strip().rstrip("/")
+    dh_pub = str(peer_dh_pub or "").strip()
+    if not dh_pub:
+        bundle = fetch_dm_prekey_bundle(
+            agent_id=peer if not token else "",
+            lookup_token=token,
+            lookup_peer_urls=[preferred_peer] if preferred_peer else None,
+        )
+        if not bundle.get("ok"):
+            return bundle
+        dh_pub = str(bundle.get("dh_pub_key") or "").strip()
+    if not dh_pub:
+        return {"ok": False, "detail": "peer dh_pub_key unavailable"}
+
+    alias = issue_pairwise_dm_alias(peer_id=peer, peer_dh_pub=dh_pub)
+    if not alias.get("ok"):
+        return alias
+    shared_alias = str(alias.get("shared_alias") or "").strip()
+    if not shared_alias:
+        return {"ok": False, "detail": "shared_alias unavailable"}
+
+    accept_plain = build_contact_accept(shared_alias=shared_alias)
+    encrypted = bootstrap_encrypt_for_peer(peer, accept_plain, lookup_token=token)
+    if not encrypted.get("ok"):
+        return encrypted
+
+    sent = _submit_signed_dm_send(
+        recipient=peer,
+        delivery_class="request",
+        recipient_token="",
+        ciphertext=str(encrypted.get("result") or ""),
+        payload_format="mls1",
+        connect_intent="contact_accept",
+        lookup_peer_url=preferred_peer,
+    )
+    if isinstance(sent, dict):
+        sent.setdefault("shared_alias", shared_alias)
+    return sent
+
+
+def send_dm(
+    peer_id: str,
+    plaintext: str,
+    *,
+    delivery_class: str = "shared",
+    recipient_token: str = "",
+) -> dict[str, Any]:
+    """Compose and send an encrypted DM on behalf of the operator."""
+    import main as main_mod
+
+    recipient = str(peer_id or "").strip()
+    if not recipient:
+        return {"ok": False, "detail": "peer_id required"}
+    if not str(plaintext or "").strip():
+        return {"ok": False, "detail": "plaintext required"}
+
+    delivery = str(delivery_class or "shared").strip().lower()
+    if delivery not in ("shared", "request"):
+        return {"ok": False, "detail": "delivery_class must be shared or request"}
+
+    composed = main_mod.compose_wormhole_dm(
+        peer_id=recipient,
+        peer_dh_pub="",
+        plaintext=str(plaintext),
+    )
+    if not composed.get("ok"):
+        return composed
+
+    return _submit_signed_dm_send(
+        recipient=recipient,
+        delivery_class=delivery,
+        recipient_token=str(recipient_token or ""),
+        ciphertext=str(composed.get("ciphertext") or ""),
+        payload_format=str(composed.get("format") or "mls1"),
+        session_welcome=str(composed.get("session_welcome") or ""),
+    )
+
+
+def poll_dms(*, limit: int = 20) -> dict[str, Any]:
+    """Poll encrypted DMs for the operator DM identity."""
+    import json
+
+    import main as main_mod
+    from services.mesh.mesh_protocol import PROTOCOL_VERSION
+    from services.mesh.mesh_wormhole_persona import get_dm_identity, sign_dm_wormhole_event
+
+    identity = get_dm_identity()
+    agent_id = str(identity.get("node_id") or "")
+    if not agent_id:
+        return {"ok": False, "detail": "dm identity is not configured"}
+
+    poll_payload = {"mailbox_claims": [], "agent_id": agent_id}
+    signed = sign_dm_wormhole_event(event_type="dm_poll", payload=poll_payload)
+    if not signed.get("ok", True):
+        return signed
+
+    body = {
+        "agent_id": agent_id,
+        "mailbox_claims": [],
+        "timestamp": int(time.time()),
+        "nonce": secrets.token_hex(8),
+        "public_key": str(signed.get("public_key") or ""),
+        "public_key_algo": str(signed.get("public_key_algo") or ""),
+        "signature": str(signed.get("signature") or ""),
+        "sequence": int(signed.get("sequence") or 0),
+        "protocol_version": str(signed.get("protocol_version") or PROTOCOL_VERSION),
+    }
+
+    raw = json.dumps(body).encode("utf-8")
+
+    async def _poll():
+        async def receive():
+            return {"type": "http.request", "body": raw, "more_body": False}
+
+        req = Request(
+            {
+                "type": "http",
+                "method": "POST",
+                "path": "/api/mesh/dm/poll",
+                "headers": [(b"content-type", b"application/json")],
+                "client": ("127.0.0.1", 52421),
+            },
+            receive,
+        )
+        return await main_mod.dm_poll_secure(req)
+
+    result = _run_async(_poll())
+    if isinstance(result, dict):
+        messages = list(result.get("messages") or [])
+        if limit and len(messages) > int(limit):
+            result = dict(result)
+            result["messages"] = messages[: int(limit)]
+            result["count"] = len(result["messages"])
+    return result if isinstance(result, dict) else {"ok": False, "detail": "dm poll failed"}

backend/services/openclaw_routing.py

@@ -0,0 +1,705 @@
+"""Deterministic OpenClaw routing — intent → fastest command.
+
+Keeps expensive fuzzy scans and full-layer dumps out of the default agent path.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+EXPENSIVE_COMMANDS = frozenset({
+    "search_telemetry",
+    "get_telemetry",
+    "get_slow_telemetry",
+    "get_report",
+})
+
+EXPENSIVE_GATE_MESSAGE = (
+    "expensive command blocked — use route_query, find_entity, run_playbook, or targeted reads. "
+    "Pass confirm_expensive=true only when fuzzy search or full dumps are intentional."
+)
+
+LATENCY_TIER_MS: dict[str, int] = {
+    "channel_status": 5,
+    "route_query": 5,
+    "get_summary": 10,
+    "what_changed": 15,
+    "search_news": 15,
+    "find_flights": 25,
+    "find_ships": 25,
+    "find_entity": 30,
+    "entities_near": 30,
+    "brief_area": 30,
+    "get_layer_slice": 50,
+    "correlate_entity": 15,
+    "entity_expand": 40,
+    "osint_lookup": 200,
+    "run_playbook": 120,
+    "gt_risk_heatmap": 20,
+    "gt_dossier": 25,
+    "gt_analyze": 80,
+    "gt_backtest": 120,
+    "gt_rolling_freeze": 30,
+    "gt_rolling_label": 20,
+    "gt_rolling_backtest": 30,
+    "gt_micro_rolling": 20,
+    "infonet_status": 20,
+    "list_gates": 15,
+    "read_gate_messages": 40,
+    "poll_dms": 80,
+    "ensure_infonet_ready": 120000,
+    "join_infonet_swarm": 90000,
+    "post_gate_message": 15000,
+    "cast_vote": 5000,
+    "send_dm": 20000,
+    "search_telemetry": 8000,
+    "get_telemetry": 3500,
+    "get_slow_telemetry": 1500,
+    "get_report": 5000,
+}
+
+RE_N_NUMBER = re.compile(r"\bN\d{1,5}[A-Z]{0,2}\b", re.I)
+RE_CALLSIGN = re.compile(r"\b[A-Z]{2,4}\d{1,4}[A-Z]?\b")
+RE_MMSI = re.compile(r"\b\d{9}\b")
+RE_CVE = re.compile(r"\bCVE-\d{4}-\d+\b", re.I)
+RE_IPV4 = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b")
+RE_DOMAIN = re.compile(
+    r"\b(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+(?:[a-z]{2,})\b",
+    re.I,
+)
+
+KNOWN_CALLSIGNS = frozenset({
+    "AF1", "AF2", "EXEC1", "EXEC2", "SAM", "STALK52", "SPAR19", "SPAR20",
+})
+
+PLAYBOOKS: dict[str, dict[str, Any]] = {
+    "hot_snapshot": {
+        "description": "Summary + hot layers + what changed (one batch)",
+        "batch": [
+            {"cmd": "get_summary", "args": {"compact": True}},
+            {
+                "cmd": "get_layer_slice",
+                "args": {
+                    "layers": [
+                        "news",
+                        "telegram_osint",
+                        "military_flights",
+                        "private_jets",
+                        "earthquakes",
+                    ],
+                    "limit_per_layer": 10,
+                    "compact": True,
+                },
+            },
+            {"cmd": "what_changed", "args": {"compact": True}},
+        ],
+    },
+    "status_check": {
+        "description": "Channel health + layer counts",
+        "batch": [
+            {"cmd": "channel_status", "args": {}},
+            {"cmd": "get_summary", "args": {"compact": True}},
+        ],
+    },
+    "morning_brief": {
+        "description": "Operator morning digest layers",
+        "batch": [
+            {"cmd": "get_summary", "args": {"compact": True}},
+            {"cmd": "what_changed", "args": {"compact": True}},
+            {
+                "cmd": "get_layer_slice",
+                "args": {
+                    "layers": [
+                        "news",
+                        "telegram_osint",
+                        "gdelt",
+                        "earthquakes",
+                        "crowdthreat",
+                        "military_flights",
+                    ],
+                    "limit_per_layer": 15,
+                    "compact": True,
+                },
+            },
+        ],
+    },
+    "monitor_heartbeat": {
+        "description": "Low-latency monitor poll (replaces full telemetry pull)",
+        "batch": [
+            {"cmd": "what_changed", "args": {"compact": True}},
+            {
+                "cmd": "get_layer_slice",
+                "args": {
+                    "layers": [
+                        "military_flights",
+                        "ships",
+                        "earthquakes",
+                        "liveuamap",
+                        "crowdthreat",
+                        "uap_sightings",
+                        "firms_fires",
+                        "gps_jamming",
+                        "wastewater",
+                    ],
+                    "limit_per_layer": 200,
+                    "compact": True,
+                },
+            },
+        ],
+    },
+}
+
+
+def routing_manifest() -> dict[str, Any]:
+    """Machine-readable routing hints for /api/ai/capabilities."""
+    return {
+        "default_read": "find_entity",
+        "preferred_entry": "route_query",
+        "client_wrapper": "ShadowBrokerClient.ask",
+        "batch_playbook": "run_playbook",
+        "last_resort": "search_telemetry",
+        "expensive_commands": sorted(EXPENSIVE_COMMANDS),
+        "latency_tier_ms": LATENCY_TIER_MS,
+        "anti_patterns": [
+            "search_telemetry for known tail numbers, callsigns, owners, or MMSI",
+            "get_telemetry for routine reads — use get_layer_slice or run_playbook hot_snapshot",
+            "sequential send_command loops — use send_batch or run_playbook",
+            "/api/health for liveness — use channel_status",
+            "empty layers: [] on get_layer_slice — pass explicit layer names",
+        ],
+        "recipes": [
+            {
+                "intent": "natural language question",
+                "use": "route_query → recommended cmd, or ShadowBrokerClient.ask()",
+            },
+            {
+                "intent": "known person/aircraft",
+                "use": "find_entity(query=...) or find_flights(owner=...)",
+            },
+            {
+                "intent": "news / telegram topic",
+                "use": "search_news(query=...)",
+            },
+            {
+                "intent": "near a point",
+                "use": "entities_near or brief_area",
+            },
+            {
+                "intent": "hot snapshot",
+                "use": "run_playbook(name=hot_snapshot)",
+            },
+            {
+                "intent": "post to infonet gate / join swarm",
+                "use": "ensure_infonet_ready then post_gate_message (full tier)",
+            },
+            {
+                "intent": "read encrypted gate traffic",
+                "use": "read_gate_messages(gate_id=infonet, decrypt=true)",
+            },
+            {
+                "intent": "dm another node",
+                "use": "send_dm(peer_id=..., plaintext=...) (full tier)",
+            },
+        ],
+        "playbooks": {
+            name: {"description": spec.get("description", "")}
+            for name, spec in PLAYBOOKS.items()
+        },
+        "agent_surface": {
+            "primary": ["ask", "send_batch", "channel_status"],
+            "writes": [
+                "place_pin",
+                "add_watch",
+                "inject_data",
+                "place_analysis_zone",
+                "ensure_infonet_ready",
+                "post_gate_message",
+                "cast_vote",
+                "send_dm",
+            ],
+            "infonet_reads": [
+                "infonet_status",
+                "list_gates",
+                "read_gate_messages",
+                "poll_dms",
+            ],
+        },
+    }
+
+
+def requires_expensive_confirm(cmd: str, args: dict[str, Any] | None) -> bool:
+    if cmd not in EXPENSIVE_COMMANDS:
+        return False
+    if isinstance(args, dict) and args.get("confirm_expensive") is True:
+        return False
+    return True
+
+
+def _compact_args(args: dict[str, Any], *, compact: bool) -> dict[str, Any]:
+    out = dict(args)
+    if compact and "compact" not in out:
+        out["compact"] = True
+    return out
+
+
+def _estimate_ms(cmd: str) -> int:
+    return int(LATENCY_TIER_MS.get(cmd, 100))
+
+
+def _news_query(text: str) -> str:
+    cleaned = text
+    for prefix in (
+        "news about",
+        "news on",
+        "telegram",
+        "headlines about",
+        "headlines on",
+        "latest on",
+        "search news for",
+    ):
+        if cleaned.lower().startswith(prefix):
+            cleaned = cleaned[len(prefix):].strip()
+    return cleaned.strip(" ?.")
+
+
+def _gt_region_hint(text: str) -> str:
+    lowered = str(text or "").lower()
+    hints = (
+        "ukraine",
+        "middle east",
+        "eastern europe",
+        "baltics",
+        "israel",
+        "iran",
+        "russia",
+        "china",
+        "europe",
+        "united kingdom",
+        "uk",
+        "usa",
+        "united states",
+    )
+    for hint in hints:
+        if hint in lowered:
+            return "uk" if hint == "united kingdom" else hint
+    match = re.search(r"\bon\s+([a-z][a-z\s]{2,30})\b", lowered)
+    if match:
+        return match.group(1).strip()
+    return ""
+
+
+def route_query(
+    text: str = "",
+    *,
+    lat: float | None = None,
+    lng: float | None = None,
+    radius_km: float = 50,
+    compact: bool = True,
+) -> dict[str, Any]:
+    """Map natural-language intent to the fastest command (no LLM)."""
+    raw = str(text or "").strip()
+    lowered = raw.lower()
+    avoid = ["search_telemetry", "get_telemetry", "get_slow_telemetry"]
+    alternates: list[dict[str, Any]] = []
+
+    if not raw and lat is not None and lng is not None:
+        recommended = {
+            "cmd": "brief_area",
+            "args": _compact_args(
+                {"lat": lat, "lng": lng, "radius_km": radius_km},
+                compact=compact,
+            ),
+        }
+        return {
+            "intent": "area_brief",
+            "recommended": recommended,
+            "alternates": [{"cmd": "entities_near", "args": recommended["args"]}],
+            "avoid": avoid,
+            "estimated_ms": _estimate_ms("brief_area"),
+        }
+
+    if not raw:
+        recommended = {"cmd": "get_summary", "args": _compact_args({}, compact=compact)}
+        return {
+            "intent": "discovery",
+            "recommended": recommended,
+            "alternates": [{"cmd": "channel_status", "args": {}}],
+            "avoid": avoid,
+            "estimated_ms": _estimate_ms("get_summary"),
+        }
+
+    cve_match = RE_CVE.search(raw)
+    if cve_match:
+        recommended = {
+            "cmd": "osint_lookup",
+            "args": _compact_args({"tool": "cve", "cve": cve_match.group(0).upper()}, compact=compact),
+        }
+        return _route_result("cve_lookup", recommended, avoid, alternates)
+
+    ip_match = RE_IPV4.search(raw)
+    if ip_match and ("ip" in lowered or "address" in lowered or lowered.count(".") >= 3):
+        recommended = {
+            "cmd": "osint_lookup",
+            "args": _compact_args({"tool": "ip", "ip": ip_match.group(0)}, compact=compact),
+        }
+        alternates.append({"cmd": "entity_expand", "args": {"type": "ip", "id": ip_match.group(0)}})
+        return _route_result("ip_lookup", recommended, avoid, alternates)
+
+    if "whois" in lowered or ("dns" in lowered and RE_DOMAIN.search(raw)):
+        domain = (RE_DOMAIN.search(raw) or re.search(r"\b([a-z0-9-]+\.[a-z]{2,})\b", raw, re.I))
+        tool = "whois" if "whois" in lowered else "dns"
+        domain_value = domain.group(0) if domain else raw
+        recommended = {
+            "cmd": "osint_lookup",
+            "args": _compact_args({"tool": tool, "domain": domain_value}, compact=compact),
+        }
+        return _route_result("domain_lookup", recommended, avoid, alternates)
+
+    if "sanction" in lowered or "ofac" in lowered:
+        recommended = {
+            "cmd": "osint_lookup",
+            "args": _compact_args({"tool": "sanctions", "query": raw}, compact=compact),
+        }
+        return _route_result("sanctions_lookup", recommended, avoid, alternates)
+
+    mmsi_match = RE_MMSI.search(raw)
+    if mmsi_match and any(k in lowered for k in ("mmsi", "ship", "vessel", "yacht", "boat", "maritime")):
+        recommended = {
+            "cmd": "find_ships",
+            "args": _compact_args({"mmsi": mmsi_match.group(0)}, compact=compact),
+        }
+        alternates.append({"cmd": "find_entity", "args": {"mmsi": mmsi_match.group(0), "entity_type": "ship"}})
+        return _route_result("maritime_identifier", recommended, avoid, alternates)
+
+    n_match = RE_N_NUMBER.search(raw)
+    if n_match:
+        reg = n_match.group(0).upper()
+        recommended = {
+            "cmd": "find_flights",
+            "args": _compact_args({"registration": reg}, compact=compact),
+        }
+        alternates.append({"cmd": "find_entity", "args": {"registration": reg, "entity_type": "aircraft"}})
+        return _route_result("tail_number", recommended, avoid, alternates)
+
+    # callsign tokens
+    tokens = re.findall(r"\b[A-Z0-9]{2,8}\b", raw.upper())
+    for token in tokens:
+        if token in KNOWN_CALLSIGNS or RE_CALLSIGN.fullmatch(token):
+            recommended = {
+                "cmd": "find_flights",
+                "args": _compact_args({"callsign": token}, compact=compact),
+            }
+            alternates.append({"cmd": "find_entity", "args": {"callsign": token, "entity_type": "aircraft"}})
+            return _route_result("callsign", recommended, avoid, alternates)
+
+    if any(k in lowered for k in ("news", "telegram", "headline", "headlines", "gdelt")):
+        recommended = {
+            "cmd": "search_news",
+            "args": _compact_args({"query": _news_query(raw), "limit": 10}, compact=compact),
+        }
+        alternates.append({
+            "cmd": "get_layer_slice",
+            "args": {"layers": ["telegram_osint", "news"], "limit_per_layer": 10, "compact": compact},
+        })
+        return _route_result("news_search", recommended, avoid, alternates)
+
+    if any(
+        k in lowered
+        for k in (
+            "gt backtest",
+            "backtest gt",
+            "historical backtest",
+            "wilson confidence",
+            "confidence rate",
+            "gt benchmark",
+            "validate gt",
+        )
+    ):
+        tune = any(k in lowered for k in ("tune", "grid search", "optimize threshold"))
+        expanded = "base" not in lowered
+        recommended = {
+            "cmd": "gt_backtest",
+            "args": _compact_args(
+                {
+                    "expanded": expanded,
+                    "tune": tune,
+                    "target_confidence": 0.95,
+                },
+                compact=compact,
+            ),
+        }
+        alternates.append({"cmd": "gt_risk_heatmap", "args": {}})
+        return _route_result("gt_backtest", recommended, avoid, alternates)
+
+    if any(
+        k in lowered
+        for k in (
+            "rolling backtest",
+            "rolling validation",
+            "weekly validation",
+            "operational validation",
+            "operational backtest",
+            "week over week",
+            "week-over-week",
+            "gt rolling",
+            "rolling gt",
+            "weekly gt",
+            "weekly gt score",
+            "gt weekly",
+            "gt snapshot",
+            "freeze weekly gt",
+        )
+    ):
+        micro = any(
+            k in lowered
+            for k in (
+                "3 day",
+                "3-day",
+                "three day",
+                "micro rolling",
+                "rolling average",
+                "ignition",
+                "micro gt",
+            )
+        )
+        freeze = any(
+            k in lowered
+            for k in ("freeze", "gt snapshot", "weekly snapshot", "capture week")
+        )
+        label = any(k in lowered for k in ("label", "outcome", "escalation"))
+        if micro and not freeze and not label:
+            recommended = {
+                "cmd": "gt_micro_rolling",
+                "args": _compact_args({"window_days": 3}, compact=compact),
+            }
+            intent = "gt_micro_rolling"
+        elif freeze:
+            recommended = {
+                "cmd": "gt_rolling_freeze",
+                "args": _compact_args({"force": "force" in lowered}, compact=compact),
+            }
+            intent = "gt_rolling_freeze"
+        elif label:
+            recommended = {
+                "cmd": "gt_rolling_label",
+                "args": _compact_args({}, compact=compact),
+            }
+            intent = "gt_rolling_label"
+        else:
+            recommended = {
+                "cmd": "gt_rolling_backtest",
+                "args": _compact_args({"weeks": 8, "target_confidence": 0.80}, compact=compact),
+            }
+            intent = "gt_rolling_backtest"
+        alternates.append({"cmd": "gt_micro_rolling", "args": {"window_days": 3}})
+        alternates.append({"cmd": "gt_backtest", "args": {"expanded": True, "compact": True}})
+        return _route_result(intent, recommended, avoid, alternates)
+
+    if any(
+        k in lowered
+        for k in (
+            "3 day average",
+            "3-day average",
+            "rolling 3 day",
+            "micro risk",
+            "risk ignition",
+        )
+    ):
+        recommended = {
+            "cmd": "gt_micro_rolling",
+            "args": _compact_args({"window_days": 3}, compact=compact),
+        }
+        alternates.append({"cmd": "gt_rolling_backtest", "args": {"weeks": 8}})
+        return _route_result("gt_micro_rolling", recommended, avoid, alternates)
+
+    if any(
+        k in lowered
+        for k in (
+            "gt analysis",
+            "game theoretic",
+            "game-theoretic",
+            "strategic risk",
+            "early warning",
+            "risk heatmap",
+            "costly signal",
+            "gt rationale",
+        )
+    ):
+        region_hint = _gt_region_hint(raw)
+        if region_hint and any(k in lowered for k in ("dossier", "rationale", "scenario")):
+            recommended = {
+                "cmd": "gt_dossier",
+                "args": _compact_args({"region": region_hint}, compact=compact),
+            }
+            alternates.append({"cmd": "gt_risk_heatmap", "args": {}})
+            return _route_result("gt_dossier", recommended, avoid, alternates)
+        recommended = {
+            "cmd": "gt_analyze",
+            "args": _compact_args(
+                {"refresh": True, "region": region_hint} if region_hint else {"refresh": True},
+                compact=compact,
+            ),
+        }
+        alternates.append({"cmd": "gt_risk_heatmap", "args": {}})
+        return _route_result("gt_analyze", recommended, avoid, alternates)
+
+    if lat is not None and lng is not None and any(
+        k in lowered for k in ("near", "around", "within", "radius", "brief", "aoi")
+    ):
+        recommended = {
+            "cmd": "brief_area",
+            "args": _compact_args(
+                {"lat": lat, "lng": lng, "radius_km": radius_km, "query": raw},
+                compact=compact,
+            ),
+        }
+        alternates.append({
+            "cmd": "entities_near",
+            "args": {"lat": lat, "lng": lng, "radius_km": radius_km, "compact": compact},
+        })
+        return _route_result("area_brief", recommended, avoid, alternates)
+
+    if any(k in lowered for k in ("what changed", "updates", "delta", "since last")):
+        recommended = {"cmd": "what_changed", "args": _compact_args({}, compact=compact)}
+        return _route_result("incremental_poll", recommended, avoid, alternates)
+
+    if any(k in lowered for k in ("summary", "status", "layers populated", "what data")):
+        recommended = {"cmd": "get_summary", "args": _compact_args({}, compact=compact)}
+        alternates.append({"cmd": "channel_status", "args": {}})
+        return _route_result("discovery", recommended, avoid, alternates)
+
+    if any(k in lowered for k in ("recon", "whois", "dns lookup", "cve", "mac address")):
+        recommended = {
+            "cmd": "osint_tools",
+            "args": {},
+        }
+        return _route_result("recon_discovery", recommended, avoid, alternates)
+
+    entity_type = ""
+    if any(k in lowered for k in ("ship", "vessel", "yacht", "boat", "maritime", "carrier")):
+        entity_type = "ship"
+    elif any(k in lowered for k in ("jet", "plane", "flight", "aircraft", "helicopter", "tail")):
+        entity_type = "aircraft"
+
+    owner_hint = ""
+    if any(k in lowered for k in ("owner", "operated by", "'s jet", "'s yacht", "belongs to")):
+        owner_hint = raw
+        for phrase in ("where is", "find", "track", "locate", "jet", "yacht", "plane", "flight", "ship"):
+            owner_hint = re.sub(rf"\b{phrase}\b", "", owner_hint, flags=re.I).strip()
+
+    entity_args: dict[str, Any] = {"query": raw, "compact": compact}
+    if entity_type:
+        entity_args["entity_type"] = entity_type
+    if owner_hint and len(owner_hint) >= 3:
+        entity_args["owner"] = owner_hint
+
+    recommended = {
+        "cmd": "find_entity",
+        "args": _compact_args(entity_args, compact=compact),
+    }
+    alternates = [
+        {"cmd": "search_news", "args": {"query": raw, "limit": 10, "compact": compact}},
+    ]
+    if any(k in lowered for k in ("near", "around")):
+        alternates.append({
+            "cmd": "search_telemetry",
+            "args": {"query": raw, "limit": 10, "confirm_expensive": True, "compact": compact},
+        })
+
+    return _route_result("entity_lookup", recommended, avoid, alternates)
+
+
+def _route_result(
+    intent: str,
+    recommended: dict[str, Any],
+    avoid: list[str],
+    alternates: list[dict[str, Any]],
+) -> dict[str, Any]:
+    cmd = str(recommended.get("cmd", ""))
+    return {
+        "intent": intent,
+        "recommended": recommended,
+        "alternates": alternates,
+        "avoid": avoid,
+        "estimated_ms": _estimate_ms(cmd),
+    }
+
+
+def plan_playbook(name: str, args: dict[str, Any] | None = None) -> dict[str, Any]:
+    """Resolve a named playbook to a command batch."""
+    playbook = str(name or "").strip().lower()
+    params = dict(args or {})
+    if not playbook:
+        return {"ok": False, "detail": "playbook name required"}
+
+    if playbook == "track_snapshot":
+        query = str(params.get("query", "") or params.get("name", "") or "").strip()
+        if not query:
+            return {"ok": False, "detail": "track_snapshot requires query"}
+        return {
+            "ok": True,
+            "playbook": playbook,
+            "description": "Resolve entity for tracking",
+            "batch": [
+                {
+                    "cmd": "find_entity",
+                    "args": {
+                        "query": query,
+                        "entity_type": params.get("entity_type", ""),
+                        "fallback_search": True,
+                        "compact": True,
+                    },
+                }
+            ],
+        }
+
+    if playbook == "area_brief":
+        lat = params.get("lat")
+        lng = params.get("lng")
+        if lat is None or lng is None:
+            return {"ok": False, "detail": "area_brief requires lat and lng"}
+        return {
+            "ok": True,
+            "playbook": playbook,
+            "description": "Brief an area of interest",
+            "batch": [
+                {
+                    "cmd": "brief_area",
+                    "args": {
+                        "lat": lat,
+                        "lng": lng,
+                        "radius_km": params.get("radius_km", 50),
+                        "query": params.get("query", ""),
+                        "compact": True,
+                    },
+                }
+            ],
+        }
+
+    if playbook == "entity_recon":
+        query = str(params.get("query", "") or params.get("ip", "") or "").strip()
+        ip_match = RE_IPV4.search(query)
+        if not ip_match:
+            return {"ok": False, "detail": "entity_recon requires an IP in query"}
+        return {
+            "ok": True,
+            "playbook": playbook,
+            "description": "IP recon + entity graph",
+            "batch": [
+                {"cmd": "osint_lookup", "args": {"tool": "ip", "ip": ip_match.group(0), "compact": True}},
+                {"cmd": "entity_expand", "args": {"type": "ip", "id": ip_match.group(0)}},
+            ],
+        }
+
+    spec = PLAYBOOKS.get(playbook)
+    if not spec:
+        known = sorted(PLAYBOOKS) + ["track_snapshot", "area_brief", "entity_recon"]
+        return {"ok": False, "detail": f"unknown playbook: {playbook}", "known": known}
+
+    return {
+        "ok": True,
+        "playbook": playbook,
+        "description": spec.get("description", ""),
+        "batch": [dict(item) for item in spec.get("batch", [])],
+    }

backend/services/openclaw_watchdog.py

@@ -22,9 +22,12 @@ logger = logging.getLogger(__name__)
 _lock = threading.Lock()
 _watches: dict[str, dict[str, Any]] = {}  # watch_id -> watch definition
 _fired: dict[str, float] = {}  # watch_id -> last fire timestamp (debounce)
+_seen_posts: dict[str, set[str]] = {}  # watch_id -> seen Telegram post ids/links
 _running = False
 _stop_event = threading.Event()
 
+_TELEGRAM_SEEN_MAX = 500
+
 # Minimum seconds between re-firing the same watch
 DEBOUNCE_S = 60.0
 # How often the watchdog checks telemetry
@@ -73,6 +76,7 @@ def remove_watch(watch_id: str) -> dict[str, Any]:
     with _lock:
         removed = _watches.pop(watch_id, None)
         _fired.pop(watch_id, None)
+        _seen_posts.pop(watch_id, None)
     if removed:
         return {"ok": True, "removed": removed}
     return {"ok": False, "detail": f"watch '{watch_id}' not found"}
@@ -90,6 +94,7 @@ def clear_watches() -> dict[str, Any]:
         count = len(_watches)
         _watches.clear()
         _fired.clear()
+        _seen_posts.clear()
     return {"ok": True, "cleared": count}
 
 
@@ -157,7 +162,9 @@ def _check_watch(watch: dict, fast: dict, slow: dict) -> dict[str, Any] | None:
     if wtype == "geofence":
         return _check_geofence(params, fast)
     if wtype == "keyword":
-        return _check_keyword(params, fast, slow)
+        return _check_keyword(watch["id"], params, fast, slow)
+    if wtype == "telegram_rhetoric":
+        return _check_telegram_rhetoric(watch["id"], params, slow)
     if wtype == "prediction_market":
         return _check_prediction_market(params, slow)
 
@@ -390,15 +397,41 @@ def _check_geofence(params: dict, fast: dict) -> dict | None:
     return None
 
 
-def _check_keyword(params: dict, fast: dict, slow: dict) -> dict | None:
-    """Alert when a keyword appears in news/GDELT."""
+def _telegram_post_id(post: dict[str, Any]) -> str:
+    return str(post.get("id") or post.get("link") or "").strip()
+
+
+def _mark_seen_posts(watch_id: str, post_ids: list[str]) -> None:
+    clean = [pid for pid in post_ids if pid]
+    if not clean:
+        return
+    with _lock:
+        seen = _seen_posts.setdefault(watch_id, set())
+        seen.update(clean)
+        if len(seen) > _TELEGRAM_SEEN_MAX:
+            _seen_posts[watch_id] = set(list(seen)[-_TELEGRAM_SEEN_MAX:])
+
+
+def _is_seen_post(watch_id: str, post_id: str) -> bool:
+    if not post_id:
+        return False
+    with _lock:
+        return post_id in _seen_posts.get(watch_id, set())
+
+
+def _check_keyword(watch_id: str, params: dict, fast: dict, slow: dict) -> dict | None:
+    """Alert when a keyword appears in news, GDELT, or Telegram OSINT."""
     keyword = str(params.get("keyword", "")).lower().strip()
     if not keyword:
         return None
 
-    matches = []
+    include_telegram = params.get("include_telegram", True)
+    if isinstance(include_telegram, str):
+        include_telegram = include_telegram.strip().lower() not in {"0", "false", "no", "off"}
+
+    matches = []
+    new_telegram_ids: list[str] = []
 
-    # Check news articles
     for article in slow.get("news", []):
         title = str(article.get("title", "") or "").lower()
         desc = str(article.get("description", "") or article.get("summary", "") or "").lower()
@@ -409,7 +442,6 @@ def _check_keyword(params: dict, fast: dict, slow: dict) -> dict | None:
                 "url": article.get("url") or article.get("link"),
             })
 
-    # Check GDELT
     for event in slow.get("gdelt", []):
         text = str(event.get("title", "") or event.get("sourceurl", "") or "").lower()
         if keyword in text:
@@ -419,14 +451,103 @@ def _check_keyword(params: dict, fast: dict, slow: dict) -> dict | None:
                 "url": event.get("sourceurl"),
             })
 
+    if include_telegram:
+        from services.telegram_osint_text import (
+            iter_telegram_posts,
+            keyword_matches_telegram_post,
+            telegram_post_match_entry,
+        )
+
+        for post in iter_telegram_posts(slow.get("telegram_osint")):
+            if not keyword_matches_telegram_post(post, keyword):
+                continue
+            post_id = _telegram_post_id(post)
+            if _is_seen_post(watch_id, post_id):
+                continue
+            entry = telegram_post_match_entry(post)
+            matches.append(entry)
+            if post_id:
+                new_telegram_ids.append(post_id)
+
     if matches:
+        if new_telegram_ids:
+            _mark_seen_posts(watch_id, new_telegram_ids)
+        sources = sorted({str(match.get("source") or "unknown") for match in matches})
         return {
-            "alert": f"Keyword '{keyword}' found in {len(matches)} articles",
+            "alert": f"Keyword '{keyword}' found in {len(matches)} items ({', '.join(sources)})",
             "data": {"keyword": keyword, "matches": matches[:10]},
         }
     return None
 
 
+def _check_telegram_rhetoric(watch_id: str, params: dict, slow: dict) -> dict | None:
+    """Alert on new high-risk Telegram OSINT posts (optionally keyword/channel filtered)."""
+    min_risk = int(params.get("min_risk_score", 7) or 7)
+    min_risk = max(1, min(min_risk, 10))
+
+    raw_keywords = params.get("keywords") or params.get("keyword") or []
+    if isinstance(raw_keywords, str):
+        raw_keywords = [part.strip() for part in raw_keywords.split(",") if part.strip()]
+    keywords = [str(item).lower().strip() for item in raw_keywords if str(item).strip()]
+
+    raw_channels = params.get("channels") or params.get("channel") or []
+    if isinstance(raw_channels, str):
+        raw_channels = [part.strip() for part in raw_channels.split(",") if part.strip()]
+    channels = [str(item).lower().strip().lstrip("@") for item in raw_channels if str(item).strip()]
+
+    from services.telegram_osint_text import (
+        iter_telegram_posts,
+        keyword_matches_telegram_post,
+        telegram_post_match_entry,
+    )
+
+    matches = []
+    new_post_ids: list[str] = []
+
+    for post in iter_telegram_posts(slow.get("telegram_osint")):
+        try:
+            risk = int(post.get("risk_score") or 0)
+        except (TypeError, ValueError):
+            risk = 0
+        if risk < min_risk:
+            continue
+
+        channel = str(post.get("channel") or "").lower().strip()
+        source = str(post.get("source") or "").lower().strip()
+        if channels and channel not in channels and not any(ch in source for ch in channels):
+            continue
+
+        if keywords and not any(keyword_matches_telegram_post(post, kw) for kw in keywords):
+            continue
+
+        post_id = _telegram_post_id(post)
+        if _is_seen_post(watch_id, post_id):
+            continue
+
+        entry = telegram_post_match_entry(post)
+        matches.append(entry)
+        if post_id:
+            new_post_ids.append(post_id)
+
+    if not matches:
+        return None
+
+    _mark_seen_posts(watch_id, new_post_ids)
+    top = max(int(match.get("risk_score") or 0) for match in matches)
+    return {
+        "alert": (
+            f"Telegram rhetoric alert: {len(matches)} new post(s) at LVL {top}/10"
+            + (f" (min {min_risk})" if min_risk > 1 else "")
+        ),
+        "data": {
+            "min_risk_score": min_risk,
+            "keywords": keywords,
+            "channels": channels,
+            "matches": matches[:10],
+        },
+    }
+
+
 def _check_prediction_market(params: dict, slow: dict) -> dict | None:
     """Alert on prediction market movements."""
     query = str(params.get("query", "")).lower().strip()

backend/services/osint/__init__.py

@@ -0,0 +1 @@
+"""Operator-initiated OSINT lookups (server-side proxies)."""

backend/services/osint/lookups.py

@@ -0,0 +1,492 @@
+"""Server-side OSINT lookups (Osiris port, HTTPS outbound only)."""
+from __future__ import annotations
+
+import ipaddress
+import json
+import logging
+import re
+import socket
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timezone
+from typing import Any
+from urllib.parse import quote
+
+from services.network_utils import fetch_with_curl
+from services.sanctions.ofac import match_exact, search_sanctions
+from services.ssrf_guard import safe_get, validate_domain, validate_host
+
+logger = logging.getLogger(__name__)
+
+_IPV4_RE = re.compile(r"^(\d{1,3}\.){3}\d{1,3}$")
+_IPV6_RE = re.compile(r"^[0-9a-fA-F:]+$")
+_CVE_RE = re.compile(r"^CVE-\d{4}-\d{4,}$", re.I)
+_ASN_RE = re.compile(r"^(AS)?\d+$", re.I)
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _json_get(url: str, *, timeout: float = 8.0, headers: dict[str, str] | None = None) -> Any:
+    resp = fetch_with_curl(url, timeout=timeout, headers=headers or {"Accept": "application/json"})
+    if resp.status_code != 200:
+        return None
+    try:
+        return resp.json()
+    except Exception:
+        return None
+
+
+def _sanctions_hits(*values: str) -> list[dict[str, Any]] | None:
+    hits: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for value in values:
+        if not value or value in seen:
+            continue
+        seen.add(value)
+        entries = match_exact(value)
+        if entries:
+            hits.append({"matched_value": value, "entries": entries})
+    return hits or None
+
+
+def lookup_ip(ip: str) -> dict[str, Any]:
+    if not _IPV4_RE.match(ip) and not _IPV6_RE.match(ip):
+        raise ValueError("Invalid IP format")
+    check = validate_host(ip.strip("[]"))
+    if not check.get("ok"):
+        raise ValueError(check.get("reason", "blocked IP"))
+
+    results: dict[str, Any] = {"ip": ip, "timestamp": _now_iso()}
+    fields = (
+        "status,message,continent,country,countryCode,region,regionName,city,zip,"
+        "lat,lon,timezone,isp,org,as,asname,mobile,proxy,hosting,query"
+    )
+    geo = _json_get(f"https://ip-api.com/json/{quote(ip)}?fields={fields}", timeout=5)
+    if isinstance(geo, dict) and geo.get("status") == "success":
+        results["geo"] = {
+            "country": geo.get("country"),
+            "country_code": geo.get("countryCode"),
+            "region": geo.get("regionName"),
+            "city": geo.get("city"),
+            "lat": geo.get("lat"),
+            "lon": geo.get("lon"),
+            "timezone": geo.get("timezone"),
+            "isp": geo.get("isp"),
+            "org": geo.get("org"),
+            "as_number": geo.get("as"),
+            "as_name": geo.get("asname"),
+            "is_mobile": geo.get("mobile"),
+            "is_proxy": geo.get("proxy"),
+            "is_hosting": geo.get("hosting"),
+        }
+        results["reputation"] = {
+            "is_proxy": bool(geo.get("proxy")),
+            "is_hosting": bool(geo.get("hosting")),
+            "is_mobile": bool(geo.get("mobile")),
+            "risk_level": "HIGH" if geo.get("proxy") else "MEDIUM" if geo.get("hosting") else "LOW",
+        }
+        sm = _sanctions_hits(geo.get("org") or "", geo.get("isp") or "", geo.get("asname") or "")
+        if sm:
+            results["sanctions_match"] = {"source": "OFAC SDN", "hits": sm}
+    return results
+
+
+def lookup_dns(domain: str) -> dict[str, Any]:
+    if not validate_domain(domain):
+        raise ValueError("Invalid domain format")
+    results: dict[str, Any] = {"domain": domain, "records": {}, "timestamp": _now_iso()}
+    for rtype in ("A", "AAAA", "MX", "NS", "TXT", "CNAME", "SOA"):
+        data = _json_get(
+            f"https://dns.google/resolve?name={quote(domain)}&type={rtype}",
+            timeout=5,
+        )
+        answers = []
+        if isinstance(data, dict):
+            for ans in data.get("Answer") or []:
+                answers.append(
+                    {
+                        "name": ans.get("name"),
+                        "type": ans.get("type"),
+                        "ttl": ans.get("TTL"),
+                        "data": ans.get("data"),
+                    }
+                )
+        results["records"][rtype] = answers
+    a_records = results["records"].get("A") or []
+    mx_records = results["records"].get("MX") or []
+    ns_records = results["records"].get("NS") or []
+    results["summary"] = {
+        "ip_addresses": [r["data"] for r in a_records if r.get("data")],
+        "mail_servers": [r["data"] for r in mx_records if r.get("data")],
+        "nameservers": [r["data"] for r in ns_records if r.get("data")],
+        "total_records": sum(len(v) for v in results["records"].values()),
+    }
+    return results
+
+
+def lookup_whois(domain: str) -> dict[str, Any]:
+    if not validate_domain(domain):
+        raise ValueError("Invalid domain format")
+    results: dict[str, Any] = {"domain": domain, "timestamp": _now_iso()}
+    rdap = _json_get(f"https://rdap.org/domain/{quote(domain)}", timeout=8)
+    if isinstance(rdap, dict):
+        entities = []
+        for ent in rdap.get("entities") or []:
+            vcard = ent.get("vcardArray")
+            name = org = None
+            if isinstance(vcard, list) and len(vcard) > 1:
+                for row in vcard[1]:
+                    if row[0] == "fn":
+                        name = row[3]
+                    if row[0] == "org":
+                        org = row[3]
+            if name or org:
+                entities.append({"handle": ent.get("handle"), "roles": ent.get("roles"), "name": name, "org": org})
+        events = [
+            {"action": e.get("eventAction"), "date": e.get("eventDate")}
+            for e in (rdap.get("events") or [])
+        ]
+        results["rdap"] = {
+            "handle": rdap.get("handle"),
+            "name": rdap.get("ldhName"),
+            "status": rdap.get("status"),
+            "events": events,
+            "nameservers": [ns.get("ldhName") for ns in (rdap.get("nameservers") or [])],
+            "entities": entities,
+        }
+        results["registration"] = next((e["date"] for e in events if e["action"] == "registration"), None)
+        results["expiration"] = next((e["date"] for e in events if e["action"] == "expiration"), None)
+        results["last_changed"] = next((e["date"] for e in events if e["action"] == "last changed"), None)
+        sm = _sanctions_hits(*(e.get("name") or "" for e in entities), *(e.get("org") or "" for e in entities))
+        if sm:
+            results["sanctions_match"] = {"source": "OFAC SDN", "hits": sm}
+
+    try:
+        res = safe_get(f"https://{domain}", timeout=5, headers={"User-Agent": "Shadowbroker-OSINT/1.0"})
+        headers = {}
+        for h in (
+            "server",
+            "x-powered-by",
+            "x-frame-options",
+            "strict-transport-security",
+            "content-security-policy",
+            "x-content-type-options",
+            "x-xss-protection",
+            "referrer-policy",
+            "permissions-policy",
+        ):
+            val = res.headers.get(h)
+            if val:
+                headers[h] = val
+        score = sum(
+            1
+            for k in (
+                "strict-transport-security",
+                "content-security-policy",
+                "x-frame-options",
+                "x-content-type-options",
+                "referrer-policy",
+            )
+            if k in headers
+        ) + (2 if "strict-transport-security" in headers else 0) + (2 if "content-security-policy" in headers else 0)
+        results["http"] = {"status": res.status_code, "headers": headers, "final_url": res.url}
+        results["security_score"] = {
+            "score": score,
+            "max": 7,
+            "grade": "A" if score >= 5 else "B" if score >= 3 else "C" if score >= 1 else "F",
+        }
+    except Exception as exc:
+        logger.debug("WHOIS header probe failed for %s: %s", domain, exc)
+    return results
+
+
+def lookup_certs(domain: str) -> dict[str, Any]:
+    if not validate_domain(domain):
+        raise ValueError("Invalid domain format")
+    resp = fetch_with_curl(
+        f"https://crt.sh/?q=%25.{quote(domain)}&output=json",
+        timeout=10,
+        headers={"User-Agent": "Shadowbroker-OSINT/1.0"},
+    )
+    if resp.status_code != 200:
+        return {"domain": domain, "certificates": [], "error": "crt.sh unavailable"}
+    try:
+        certs = resp.json()
+    except Exception:
+        certs = []
+    seen: set[str] = set()
+    subdomains: set[str] = set()
+    unique: list[dict[str, Any]] = []
+    for cert in (certs or [])[:200]:
+        key = f"{cert.get('common_name')}-{cert.get('serial_number')}"
+        if key in seen:
+            continue
+        seen.add(key)
+        for name in (cert.get("name_value") or "").split("\n"):
+            clean = name.strip().replace("*.", "")
+            if clean.endswith(domain):
+                subdomains.add(clean)
+        unique.append(
+            {
+                "id": cert.get("id"),
+                "issuer": cert.get("issuer_name"),
+                "common_name": cert.get("common_name"),
+                "not_before": cert.get("not_before"),
+                "not_after": cert.get("not_after"),
+            }
+        )
+    return {
+        "domain": domain,
+        "certificates": unique[:50],
+        "subdomains": sorted(subdomains)[:100],
+        "total_found": len(certs or []),
+        "timestamp": _now_iso(),
+    }
+
+
+def lookup_threats(query: str | None = None) -> dict[str, Any]:
+    results: dict[str, Any] = {"timestamp": _now_iso()}
+    pulses = _json_get("https://otx.alienvault.com/api/v1/pulses/activity?limit=10", timeout=8)
+    if isinstance(pulses, dict):
+        results["pulses"] = [
+            {
+                "name": p.get("name"),
+                "description": (p.get("description") or "")[:200],
+                "created": p.get("created"),
+                "tags": (p.get("tags") or [])[:5],
+                "adversary": p.get("adversary"),
+                "indicators_count": p.get("indicator_count"),
+            }
+            for p in (pulses.get("results") or [])[:10]
+        ]
+    if query:
+        if _IPV4_RE.match(query):
+            try:
+                tor_resp = fetch_with_curl("https://check.torproject.org/torbulkexitlist", timeout=5)
+                results["tor_exit_node"] = query in (tor_resp.text or "").splitlines() if tor_resp.status_code == 200 else None
+            except Exception:
+                results["tor_exit_node"] = None
+            otx = _json_get(f"https://otx.alienvault.com/api/v1/indicators/IPv4/{quote(query)}/general", timeout=5)
+            if isinstance(otx, dict):
+                results["otx"] = {
+                    "reputation": otx.get("reputation"),
+                    "pulse_count": (otx.get("pulse_info") or {}).get("count", 0),
+                    "country": otx.get("country_name"),
+                    "asn": otx.get("asn"),
+                }
+        elif validate_domain(query):
+            otx = _json_get(f"https://otx.alienvault.com/api/v1/indicators/domain/{quote(query)}/general", timeout=5)
+            if isinstance(otx, dict):
+                results["otx"] = {"pulse_count": (otx.get("pulse_info") or {}).get("count", 0)}
+    pulse_count = (results.get("otx") or {}).get("pulse_count", 0)
+    results["threat_level"] = "HIGH" if pulse_count > 5 else "MEDIUM" if pulse_count > 0 else "LOW"
+    return results
+
+
+def lookup_bgp(query: str) -> dict[str, Any]:
+    results: dict[str, Any] = {"query": query, "timestamp": _now_iso()}
+    if _IPV4_RE.match(query):
+        data = _json_get(f"https://api.bgpview.io/ip/{quote(query)}", timeout=8)
+        if isinstance(data, dict) and data.get("status") == "ok":
+            results["ip"] = data.get("data")
+            results["type"] = "ip"
+        return results
+    if _ASN_RE.match(query):
+        asn_num = re.sub(r"^AS", "", query, flags=re.I)
+        asn = _json_get(f"https://api.bgpview.io/asn/{asn_num}", timeout=8)
+        prefixes = _json_get(f"https://api.bgpview.io/asn/{asn_num}/prefixes", timeout=8)
+        peers = _json_get(f"https://api.bgpview.io/asn/{asn_num}/peers", timeout=8)
+        if isinstance(asn, dict) and asn.get("status") == "ok":
+            results["asn"] = asn.get("data")
+        if isinstance(prefixes, dict) and prefixes.get("status") == "ok":
+            pdata = prefixes.get("data") or {}
+            results["prefixes"] = {
+                "ipv4": (pdata.get("ipv4_prefixes") or [])[:20],
+                "ipv6": (pdata.get("ipv6_prefixes") or [])[:10],
+                "total_v4": len(pdata.get("ipv4_prefixes") or []),
+                "total_v6": len(pdata.get("ipv6_prefixes") or []),
+            }
+        if isinstance(peers, dict) and peers.get("status") == "ok":
+            pdata = peers.get("data") or {}
+            results["peers"] = {
+                "upstream": (pdata.get("ipv4_peers") or [])[:10],
+                "total": len(pdata.get("ipv4_peers") or []),
+            }
+        results["type"] = "asn"
+        return results
+    raise ValueError("Unrecognized query format. Use IP address or AS number.")
+
+
+def lookup_sanctions(query: str, *, schema: str | None = None, limit: int = 25) -> dict[str, Any]:
+    matches = search_sanctions(query, schema=schema, limit=limit)
+    return {
+        "query": query,
+        "schema": schema,
+        "total": len(matches),
+        "matches": matches,
+        "source": "OpenSanctions / US OFAC SDN",
+        "timestamp": _now_iso(),
+    }
+
+
+def lookup_cve(cve: str) -> dict[str, Any]:
+    if not _CVE_RE.match(cve):
+        raise ValueError("Invalid CVE format")
+    cve_id = cve.upper()
+    data = _json_get(f"https://cveawg.mitre.org/api/cve/{quote(cve_id)}", timeout=8)
+    if isinstance(data, dict) and data.get("cveMetadata"):
+        meta = data["cveMetadata"]
+        desc = ""
+        for block in (data.get("containers") or {}).get("cna", {}).get("descriptions") or []:
+            if block.get("lang") == "en":
+                desc = block.get("value") or desc
+        return {"id": meta.get("cveId", cve_id), "description": desc or "No description.", "timestamp": _now_iso()}
+    fallback = _json_get(f"https://cve.circl.lu/api/cve/{quote(cve_id)}", timeout=8)
+    if isinstance(fallback, dict):
+        return {
+            "id": fallback.get("id", cve_id),
+            "description": fallback.get("summary") or "No description.",
+            "cvss": fallback.get("cvss"),
+            "references": (fallback.get("references") or [])[:5],
+            "timestamp": _now_iso(),
+        }
+    raise ValueError("CVE not found")
+
+
+def lookup_mac(mac: str) -> dict[str, Any]:
+    clean = mac.strip().upper()
+    clean = re.sub(r"[^A-F0-9:-]", "", clean)
+    data = _json_get(f"https://api.macvendors.com/{quote(clean)}", timeout=8)
+    if isinstance(data, dict):
+        return {"mac": clean, "vendor": data.get("company") or data.get("organization") or "Not Found"}
+    if isinstance(data, str) and data:
+        return {"mac": clean, "vendor": data}
+    return {"mac": clean, "vendor": "Not Found"}
+
+
+def lookup_github(username: str) -> dict[str, Any]:
+    user = _json_get(f"https://api.github.com/users/{quote(username)}", timeout=8)
+    if not isinstance(user, dict) or user.get("message") == "Not Found":
+        raise ValueError("GitHub user not found")
+    repos = _json_get(f"https://api.github.com/users/{quote(username)}/repos?per_page=10&sort=updated", timeout=8)
+    return {
+        "username": username,
+        "profile": {
+            "name": user.get("name"),
+            "bio": user.get("bio"),
+            "company": user.get("company"),
+            "location": user.get("location"),
+            "public_repos": user.get("public_repos"),
+            "followers": user.get("followers"),
+            "created_at": user.get("created_at"),
+            "html_url": user.get("html_url"),
+        },
+        "repos": [
+            {"name": r.get("name"), "language": r.get("language"), "stars": r.get("stargazers_count")}
+            for r in (repos or [])[:10]
+            if isinstance(r, dict)
+        ],
+        "timestamp": _now_iso(),
+    }
+
+
+def lookup_leaks(email: str) -> dict[str, Any]:
+    if "@" not in email or len(email) < 5:
+        raise ValueError("Invalid email")
+    # HIBP requires API key for v3; use public breach directory style via leak-lookup (rate limited)
+    data = _json_get(f"https://leakcheck.io/api/public?check={quote(email)}", timeout=8)
+    if isinstance(data, dict):
+        return {
+            "email": email,
+            "found": bool(data.get("found")),
+            "sources": data.get("sources") or [],
+            "timestamp": _now_iso(),
+        }
+    return {"email": email, "found": False, "sources": [], "timestamp": _now_iso()}
+
+
+def sweep_init(ip: str, cidr: int = 24) -> dict[str, Any]:
+    try:
+        addr = ipaddress.IPv4Address(ip)
+    except ValueError as exc:
+        raise ValueError("Invalid IPv4 address format") from exc
+    if addr.is_private or addr.is_loopback or addr.is_link_local or addr.is_reserved:
+        raise ValueError("Private and reserved IP ranges are not allowed")
+    if cidr < 24 or cidr > 32:
+        raise ValueError("CIDR must be between 24 and 32")
+
+    fields = "status,message,country,countryCode,region,regionName,city,lat,lon,isp,org,as,proxy,hosting"
+    geo = _json_get(f"https://ip-api.com/json/{quote(ip)}?fields={fields}", timeout=5)
+    if not isinstance(geo, dict) or geo.get("status") != "success":
+        raise ValueError(f"Geolocation failed: {(geo or {}).get('message', 'unknown')}")
+    return {
+        "center": {
+            "lat": geo.get("lat"),
+            "lng": geo.get("lon"),
+            "city": geo.get("city"),
+            "region": geo.get("regionName"),
+            "country": geo.get("country"),
+            "countryCode": geo.get("countryCode"),
+            "isp": geo.get("isp"),
+            "asn": geo.get("as") or "",
+            "org": geo.get("org") or "",
+        },
+        "target_ip": ip,
+        "cidr": cidr,
+    }
+
+
+def _internetdb_lookup(ip: str) -> dict[str, Any] | None:
+    try:
+        resp = fetch_with_curl(
+            f"https://internetdb.shodan.io/{quote(ip)}",
+            timeout=4,
+            headers={"Accept": "application/json"},
+        )
+        if resp.status_code == 404:
+            return None
+        if resp.status_code != 200:
+            return None
+        return resp.json()
+    except Exception:
+        return None
+
+
+def sweep_scan(subnet_start: str, cidr: int, *, max_workers: int = 12) -> dict[str, Any]:
+    """Scan a /24-/32 via Shodan InternetDB (server-side proxy)."""
+    base = int(ipaddress.IPv4Address(subnet_start))
+    host_count = 2 ** (32 - cidr)
+    if host_count > 256:
+        raise ValueError("Subnet too large")
+    ips = [str(ipaddress.IPv4Address(base + i)) for i in range(host_count)]
+    devices: list[dict[str, Any]] = []
+    t0 = time.time()
+    with ThreadPoolExecutor(max_workers=max_workers) as pool:
+        futures = {pool.submit(_internetdb_lookup, ip): ip for ip in ips}
+        for fut in as_completed(futures):
+            ip = futures[fut]
+            data = fut.result()
+            if not data:
+                continue
+            devices.append(
+                {
+                    "ip": data.get("ip") or ip,
+                    "ports": data.get("ports") or [],
+                    "hostnames": data.get("hostnames") or [],
+                    "cpes": data.get("cpes") or [],
+                    "vulns": data.get("vulns") or [],
+                    "tags": data.get("tags") or [],
+                }
+            )
+    return {
+        "devices": devices,
+        "summary": {"total_hosts": host_count, "total_responsive": len(devices)},
+        "sweep_time_ms": int((time.time() - t0) * 1000),
+    }
+
+
+def subnet_start_for(ip: str, cidr: int) -> str:
+    net = ipaddress.IPv4Network(f"{ip}/{cidr}", strict=False)
+    return str(net.network_address)

backend/services/osint/openclaw_recon.py

@@ -0,0 +1,135 @@
+"""OpenClaw dispatch for the operator recon / OSINT lookup toolkit."""
+from __future__ import annotations
+
+from typing import Any
+
+from services.osint import lookups
+from services.osint_intel.resolve import ALLOWED_TYPES, resolve_entity
+
+_OSINT_TOOLS: dict[str, str] = {
+    "ip": "ip",
+    "dns": "domain",
+    "whois": "domain",
+    "certs": "domain",
+    "threats": "query",
+    "bgp": "query",
+    "sanctions": "query",
+    "cve": "cve",
+    "mac": "mac",
+    "github": "username",
+    "leaks": "email",
+    "sweep_init": "ip",
+}
+
+_ENTITY_SCHEMAS = frozenset({
+    "Person",
+    "Organization",
+    "Company",
+    "Vessel",
+    "Airplane",
+    "LegalEntity",
+})
+
+
+def _require_str(args: dict[str, Any], *keys: str) -> str:
+    for key in keys:
+        value = str(args.get(key, "") or "").strip()
+        if value:
+            return value
+    joined = "/".join(keys)
+    raise ValueError(f"Missing required argument: {joined}")
+
+
+def run_osint_lookup(tool: str, args: dict[str, Any]) -> dict[str, Any]:
+    """Run a passive OSINT lookup (same backends as /api/osint/*)."""
+    name = str(tool or "").strip().lower().replace("-", "_")
+    if name not in _OSINT_TOOLS:
+        allowed = ", ".join(sorted(_OSINT_TOOLS))
+        raise ValueError(f"Unknown OSINT tool '{tool}'. Allowed: {allowed}")
+
+    if name == "ip":
+        return lookups.lookup_ip(_require_str(args, "ip", "query", "value"))
+    if name == "dns":
+        return lookups.lookup_dns(_require_str(args, "domain", "query", "value"))
+    if name == "whois":
+        return lookups.lookup_whois(_require_str(args, "domain", "query", "value"))
+    if name == "certs":
+        return lookups.lookup_certs(_require_str(args, "domain", "query", "value"))
+    if name == "threats":
+        query = str(args.get("query", "") or args.get("value", "") or "").strip() or None
+        return lookups.lookup_threats(query)
+    if name == "bgp":
+        return lookups.lookup_bgp(_require_str(args, "query", "asn", "value"))
+    if name == "sanctions":
+        query = _require_str(args, "query", "name", "value")
+        schema = str(args.get("schema", "") or "").strip() or None
+        if schema and schema not in _ENTITY_SCHEMAS:
+            allowed = ", ".join(sorted(_ENTITY_SCHEMAS))
+            raise ValueError(f"Invalid schema. Allowed: {allowed}")
+        limit = args.get("limit", 25)
+        try:
+            limit = int(limit)
+        except (TypeError, ValueError):
+            limit = 25
+        limit = max(1, min(100, limit))
+        return lookups.lookup_sanctions(query, schema=schema, limit=limit)
+    if name == "cve":
+        return lookups.lookup_cve(_require_str(args, "cve", "query", "value"))
+    if name == "mac":
+        return lookups.lookup_mac(_require_str(args, "mac", "query", "value"))
+    if name == "github":
+        return lookups.lookup_github(_require_str(args, "username", "user", "query", "value"))
+    if name == "leaks":
+        return lookups.lookup_leaks(_require_str(args, "email", "query", "value"))
+    if name == "sweep_init":
+        ip = _require_str(args, "ip", "query", "value")
+        cidr = args.get("cidr", 24)
+        try:
+            cidr = int(cidr)
+        except (TypeError, ValueError):
+            cidr = 24
+        return lookups.sweep_init(ip, cidr)
+
+    raise ValueError(f"Unhandled OSINT tool: {name}")
+
+
+def run_osint_sweep(args: dict[str, Any]) -> dict[str, Any]:
+    """Run subnet device discovery (Shodan InternetDB proxy). Requires full access tier."""
+    ip = _require_str(args, "ip", "query", "value")
+    cidr = args.get("cidr", 24)
+    try:
+        cidr = int(cidr)
+    except (TypeError, ValueError):
+        cidr = 24
+    subnet = lookups.subnet_start_for(ip, cidr)
+    scan = lookups.sweep_scan(subnet, cidr)
+    init = lookups.sweep_init(ip, cidr)
+    return {**init, **scan, "subnet": f"{subnet}/{cidr}"}
+
+
+def run_entity_expand(args: dict[str, Any]) -> dict[str, Any]:
+    """Expand an entity graph node (aircraft, vessel, IP, company, person, country)."""
+    entity_type = _require_str(args, "type", "entity_type")
+    entity_id = _require_str(args, "id", "entity_id", "query", "value")
+    props = {
+        "label": entity_id,
+        "registration": str(args.get("registration", "") or "").strip() or None,
+        "model": str(args.get("model", "") or "").strip() or None,
+        "icao24": str(args.get("icao24", "") or "").strip() or None,
+    }
+    props = {key: value for key, value in props.items() if value is not None}
+    return resolve_entity(entity_type, entity_id, props)
+
+
+def osint_tool_help() -> dict[str, Any]:
+    """Discovery metadata for agents."""
+    return {
+        "tools": sorted(_OSINT_TOOLS),
+        "entity_types": sorted(ALLOWED_TYPES),
+        "sanctions_schemas": sorted(_ENTITY_SCHEMAS),
+        "notes": {
+            "osint_lookup": "Passive lookups — same data as the Recon panel /api/osint/* routes.",
+            "osint_sweep": "Active subnet scan via Shodan InternetDB — requires full OpenClaw access tier.",
+            "entity_expand": "Build a relationship graph around aircraft, vessels, IPs, companies, people, or countries.",
+        },
+    }