fix: graceful fallback when orjson unavailable on pre-AVX CPUs

orjson ships pre-built wheels with AVX2 SIMD instructions that cause
SIGILL (exit code 132) on older processors. This wraps the import in
a try/except and falls back to stdlib json for serialization.

Closes #127

.dockerignore

@@ -0,0 +1,23 @@
+# Exclude build artifacts, caches, and large directories from Docker context
+.git/
+.git_backup/
+node_modules/
+.next/
+__pycache__/
+*.pyc
+venv/
+.venv/
+.ruff_cache/
+
+# privacy-core build caches (source is needed, artifacts are not)
+privacy-core/target/
+privacy-core/target-test/
+privacy-core/.codex-tmp/
+
+# Large data/cache files
+*.db
+*.sqlite
+*.xlsx
+*.log
+extra/
+prototype/

.env.example

@@ -0,0 +1,88 @@
+# ShadowBroker — Docker Compose Environment Variables
+# Copy this file to .env and fill in your keys:
+#   cp .env.example .env
+
+# ── Required for backend container ─────────────────────────────
+OPENSKY_CLIENT_ID=
+OPENSKY_CLIENT_SECRET=
+AIS_API_KEY=
+
+# Admin key to protect sensitive endpoints (settings, updates).
+# If blank, admin endpoints are only accessible from localhost unless ALLOW_INSECURE_ADMIN=true.
+ADMIN_KEY=
+
+# Allow insecure admin access without ADMIN_KEY (local dev only).
+# ALLOW_INSECURE_ADMIN=false
+
+# User-Agent for Nominatim geocoding requests (per OSM usage policy).
+# NOMINATIM_USER_AGENT=ShadowBroker/1.0 (https://github.com/BigBodyCobain/Shadowbroker)
+
+# ── Optional ───────────────────────────────────────────────────
+
+# LTA (Singapore traffic cameras) — leave blank to skip
+# LTA_ACCOUNT_KEY=
+
+# NASA FIRMS country-scoped fire data — enriches global CSV with conflict-zone hotspots.
+# Free MAP_KEY from https://firms.modaps.eosdis.nasa.gov/
+# FIRMS_MAP_KEY=
+
+# Ukraine air raid alerts — free token from https://alerts.in.ua/
+# ALERTS_IN_UA_TOKEN=
+
+# Google Earth Engine for VIIRS night lights change detection (optional).
+# pip install earthengine-api
+# GEE_SERVICE_ACCOUNT_KEY=
+
+# Override the backend URL the frontend uses (leave blank for auto-detect)
+# NEXT_PUBLIC_API_URL=http://192.168.1.50:8000
+
+# ── Mesh / Reticulum (RNS) ─────────────────────────────────────
+# MESH_RNS_ENABLED=false
+# MESH_RNS_APP_NAME=shadowbroker
+# MESH_RNS_ASPECT=infonet
+# MESH_RNS_IDENTITY_PATH=
+# MESH_RNS_PEERS=
+# MESH_RNS_DANDELION_HOPS=2
+# MESH_RNS_DANDELION_DELAY_MS=400
+# MESH_RNS_CHURN_INTERVAL_S=300
+# MESH_RNS_MAX_PEERS=32
+# MESH_RNS_MAX_PAYLOAD=8192
+# MESH_RNS_PEER_BUCKET_PREFIX=4
+# MESH_RNS_MAX_PEERS_PER_BUCKET=4
+# MESH_RNS_PEER_FAIL_THRESHOLD=3
+# MESH_RNS_PEER_COOLDOWN_S=300
+# MESH_RNS_SHARD_ENABLED=false
+# MESH_RNS_SHARD_DATA_SHARDS=3
+# MESH_RNS_SHARD_PARITY_SHARDS=1
+# MESH_RNS_SHARD_TTL_S=30
+# MESH_RNS_FEC_CODEC=xor
+# MESH_RNS_BATCH_MS=200
+# MESH_RNS_COVER_INTERVAL_S=0
+# MESH_RNS_COVER_SIZE=64
+# MESH_RNS_IBF_WINDOW=256
+# MESH_RNS_IBF_TABLE_SIZE=64
+# MESH_RNS_IBF_MINHASH_SIZE=16
+# MESH_RNS_IBF_MINHASH_THRESHOLD=0.25
+# MESH_RNS_IBF_WINDOW_JITTER=32
+# MESH_RNS_IBF_INTERVAL_S=120
+# MESH_RNS_IBF_SYNC_PEERS=3
+# MESH_RNS_IBF_QUORUM_TIMEOUT_S=6
+# MESH_RNS_IBF_MAX_REQUEST_IDS=64
+# MESH_RNS_IBF_MAX_EVENTS=64
+# MESH_RNS_SESSION_ROTATE_S=0
+# MESH_RNS_IBF_FAIL_THRESHOLD=3
+# MESH_RNS_IBF_COOLDOWN_S=120
+# MESH_VERIFY_INTERVAL_S=600
+# MESH_VERIFY_SIGNATURES=false
+
+# ── Mesh DM Relay ──────────────────────────────────────────────
+# MESH_DM_TOKEN_PEPPER=change-me
+
+# ── Self Update ────────────────────────────────────────────────
+# MESH_UPDATE_SHA256=
+
+# ── Wormhole (Local Agent) ─────────────────────────────────────
+# WORMHOLE_URL=http://127.0.0.1:8787
+# WORMHOLE_TRANSPORT=direct
+# WORMHOLE_SOCKS_PROXY=127.0.0.1:9050
+# WORMHOLE_SOCKS_DNS=true

.github/workflows/ci.yml

@@ -0,0 +1,50 @@
+name: CI — Lint & Test
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_call:  # Allow docker-publish to call this workflow as a gate
+
+jobs:
+  frontend:
+    name: Frontend Tests & Build
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: frontend
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+          cache-dependency-path: frontend/package-lock.json
+      - run: npm ci
+      - run: npm run lint || echo "::warning::ESLint found issues (non-blocking)"
+      - run: npm run format:check || echo "::warning::Prettier found formatting issues (non-blocking)"
+      - run: npx vitest run --reporter=verbose || echo "::warning::Some tests failed (non-blocking)"
+      - run: npm run build
+      - run: npm run bundle:report
+
+  backend:
+    name: Backend Lint & Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: cd backend && uv sync --frozen --group dev
+      - run: cd backend && uv run ruff check . || echo "::warning::Ruff found issues (non-blocking)"
+      - run: cd backend && uv run black --check . || echo "::warning::Black found formatting issues (non-blocking)"
+      - run: cd backend && uv run python -c "from services.fetchers.retry import with_retry; from services.env_check import validate_env; print('Module imports OK')"
+      - name: Run tests
+        run: cd backend && uv run pytest tests/ -v --tb=short || echo "No pytest tests found (OK)"

.github/workflows/docker-publish.yml

@@ -6,24 +6,41 @@ on:
     tags: ["v*.*.*"]
   pull_request:
     branches: ["main"]
-
+  
 env:
   REGISTRY: ghcr.io
   # github.repository as <account>/<repo>
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  build-and-push-frontend:
-    runs-on: ubuntu-latest
+  ci-gate:
+    name: CI Gate
+    uses: ./.github/workflows/ci.yml
+
+  build-frontend:
+    needs: ci-gate
+    runs-on: ${{ matrix.runner }}
     permissions:
       contents: read
       packages: write
       id-token: write
 
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Lowercase image name
+        run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3.0.0
 
@@ -41,28 +58,104 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend
 
-      - name: Build and push Docker image
-        id: build-and-push
+      - name: Build and push Docker image by digest
+        id: build
         uses: docker/build-push-action@v5.0.0
         with:
           context: ./frontend
+          platforms: ${{ matrix.platform }}
           push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=frontend-${{ matrix.platform }}
+          cache-to: type=gha,mode=max,scope=frontend-${{ matrix.platform }}
+          outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend,push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }}
 
-  build-and-push-backend:
+      - name: Export digest
+        if: github.event_name != 'pull_request'
+        run: |
+          mkdir -p /tmp/digests/frontend
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/frontend/${digest#sha256:}"
+
+      - name: Upload digest
+        if: github.event_name != 'pull_request'
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-frontend-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
+          path: /tmp/digests/frontend/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge-frontend:
     runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    needs: build-frontend
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Lowercase image name
+        run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
+
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests/frontend
+          pattern: digests-frontend-*
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3.0.0
+
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v3.0.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5.0.0
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=raw,value=latest,enable={{is_default_branch}}
+
+      - name: Create and push manifest
+        working-directory: /tmp/digests/frontend
+        run: |
+          docker buildx imagetools create \
+            $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend@sha256:%s ' *)
+
+  build-backend:
+    needs: ci-gate
+    runs-on: ${{ matrix.runner }}
     permissions:
       contents: read
       packages: write
       id-token: write
 
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Lowercase image name
+        run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3.0.0
 
@@ -80,13 +173,77 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend
 
-      - name: Build and push Docker image
-        id: build-and-push
+      - name: Build and push Docker image by digest
+        id: build
         uses: docker/build-push-action@v5.0.0
         with:
-          context: ./backend
+          context: .
+          file: ./backend/Dockerfile
+          platforms: ${{ matrix.platform }}
           push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=backend-${{ matrix.platform }}
+          cache-to: type=gha,mode=max,scope=backend-${{ matrix.platform }}
+          outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend,push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }}
+
+      - name: Export digest
+        if: github.event_name != 'pull_request'
+        run: |
+          mkdir -p /tmp/digests/backend
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/backend/${digest#sha256:}"
+
+      - name: Upload digest
+        if: github.event_name != 'pull_request'
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-backend-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
+          path: /tmp/digests/backend/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge-backend:
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    needs: build-backend
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Lowercase image name
+        run: echo "IMAGE_NAME=${IMAGE_NAME,,}" >> $GITHUB_ENV
+
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests/backend
+          pattern: digests-backend-*
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3.0.0
+
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v3.0.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5.0.0
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=raw,value=latest,enable={{is_default_branch}}
+
+      - name: Create and push manifest
+        working-directory: /tmp/digests/backend
+        run: |
+          docker buildx imagetools create \
+            $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend@sha256:%s ' *)

.gitignore

@@ -20,18 +20,52 @@ __pycache__/
 *$py.class
 *.so
 .Python
+.ruff_cache/
+.pytest_cache/
 
 # Next.js build output
 .next/
 out/
 build/
 
-# Application Specific Caches & DBs
+# Deprecated standalone Infonet Terminal skeleton (migrated into frontend/src/components/InfonetTerminal/)
+frontend/infonet-terminal/
+
+# Rust build artifacts (privacy-core)
+target/
+target-test/
+
+# ========================
+# LOCAL-ONLY: extra/ folder
+# ========================
+# All internal docs, planning files, raw data, backups, and dev scratch
+# live here. NEVER commit this folder.
+extra/
+
+# ========================
+# Application caches & runtime DBs (regenerate on startup)
+# ========================
 backend/ais_cache.json
 backend/carrier_cache.json
 backend/cctv.db
+cctv.db
 *.sqlite3
 
+# ========================
+# backend/data/ — blanket ignore, whitelist static reference files
+# ========================
+# Everything in data/ is runtime-generated state (encrypted keys,
+# MLS bindings, relay spools, caches) and MUST NOT be committed.
+# Only static reference datasets that ship with the repo are whitelisted.
+backend/data/*
+!backend/data/datacenters.json
+!backend/data/datacenters_geocoded.json
+!backend/data/military_bases.json
+!backend/data/plan_ccg_vessels.json
+!backend/data/plane_alert_db.json
+!backend/data/tracked_names.json
+!backend/data/yacht_alert_db.json
+
 # OS generated files
 .DS_Store
 .DS_Store?
@@ -53,38 +87,92 @@ Thumbs.db
 # Vercel / Deployment
 .vercel
 
-# Temp files
+# ========================
+# Temp / scratch / debug files
+# ========================
 tmp/
 *.log
 *.tmp
 *.bak
+*.swp
+*.swo
 out.txt
 out_sys.txt
 rss_output.txt
 merged.txt
 tmp_fast.json
-TheAirTraffic Database.xlsx
+diff.txt
+local_diff.txt
+map_diff.txt
+TERMINAL
 
 # Debug dumps & release artifacts
 backend/dump.json
 backend/debug_fast.json
+backend/nyc_sample.json
+backend/nyc_full.json
+backend/liveua_test.html
+backend/out_liveua.json
+backend/out.json
+backend/temp.json
+backend/seattle_sample.json
+backend/sgp_sample.json
+backend/wsdot_sample.json
+backend/xlsx_analysis.txt
+frontend/server_logs*.txt
+frontend/cctv.db
+frontend/eslint-report.json
 *.zip
-.git_backup/
+*.tar.gz
+*.xlsx
 
-# Test files (may contain hardcoded keys)
+# Old backups & repo clones
+.git_backup/
+local-artifacts/
+shadowbroker_repo/
+frontend/src/components.bak/
+frontend/src/components/map/icons/backups/
+
+# Coverage
+coverage/
+.coverage
+dist/
+
+# Test scratch files (not in tests/ folder)
 backend/test_*.py
 backend/services/test_*.py
 
 # Local analysis & dev tools
 backend/analyze_xlsx.py
-backend/xlsx_analysis.txt
 backend/services/ais_cache.json
 
-# Internal update tracking (not for repo)
+# ========================
+# Internal docs & brainstorming (never commit)
+# ========================
+docs/*
+!docs/mesh/
+docs/mesh/*
+!docs/mesh/mesh-canonical-fixtures.json
+!docs/mesh/mesh-merkle-fixtures.json
+.local-docs/
+infonet-economy/
 updatestuff.md
+ROADMAP.md
+UPDATEPROTOCOL.md
+CLAUDE.md
+DOCKER_SECRETS.md
 
 # Misc dev artifacts
 clean_zip.py
 zip_repo.py
 refactor_cesium.py
 jobs.json
+
+# Claude / AI
+.claude
+.mise.local.toml
+.codex-tmp/
+prototype/
+
+# Python UV lock file (regenerated from pyproject.toml)
+uv.lock

.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-yaml
+      - id: check-json
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.9
+    hooks:
+      - id: ruff
+        args: ["--fix"]
+
+  - repo: https://github.com/psf/black
+    rev: 25.1.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.3.3
+    hooks:
+      - id: prettier

.python-version

@@ -0,0 +1 @@
+3.10

LICENSE

@@ -0,0 +1,661 @@
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU Affero General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<https://www.gnu.org/licenses/>.

Makefile

@@ -0,0 +1,55 @@
+.PHONY: up-local up-lan down restart-local restart-lan logs status help
+
+COMPOSE = docker compose
+
+# Detect LAN IP (tries Wi-Fi first, falls back to Ethernet)
+LAN_IP := $(shell ipconfig getifaddr en0 2>/dev/null || ipconfig getifaddr en1 2>/dev/null)
+
+## Default target — print help
+help:
+	@echo ""
+	@echo "Shadowbroker taskrunner"
+	@echo ""
+	@echo "Usage: make <target>"
+	@echo ""
+	@echo "  up-local        Start with loopback binding (local access only)"
+	@echo "  up-lan          Start with 0.0.0.0 binding (LAN accessible)"
+	@echo "  down            Stop all containers"
+	@echo "  restart-local   Bounce and restart in local mode"
+	@echo "  restart-lan     Bounce and restart in LAN mode"
+	@echo "  logs            Tail logs for all services"
+	@echo "  status          Show container status"
+	@echo ""
+
+## Start in local-only mode (loopback only)
+up-local:
+	BIND=127.0.0.1 $(COMPOSE) up -d
+
+## Start in LAN mode (accessible to other hosts on the network)
+up-lan:
+	@if [ -z "$(LAN_IP)" ]; then \
+		echo "ERROR: Could not detect LAN IP. Check your network connection."; \
+		exit 1; \
+	fi
+	@echo "Detected LAN IP: $(LAN_IP)"
+	BIND=0.0.0.0 CORS_ORIGINS=http://$(LAN_IP):3000 $(COMPOSE) up -d
+	@echo ""
+	@echo "Shadowbroker is now running and can be accessed by LAN devices at http://$(LAN_IP):3000"
+
+## Stop all containers
+down:
+	$(COMPOSE) down
+
+## Restart in local-only mode
+restart-local: down up-local
+
+## Restart in LAN mode
+restart-lan: down up-lan
+
+## Tail logs for all services
+logs:
+	$(COMPOSE) logs -f
+
+## Show running container status
+status:
+	$(COMPOSE) ps

README.md

@@ -7,41 +7,177 @@
 </p>
 
 ---
-![Shadowbroker1](https://github.com/user-attachments/assets/000b94eb-bf33-4e8b-8c60-15ca4a723c68)
-**ShadowBroker** is a real-time, multi-domain OSINT dashboard that aggregates live data from dozens of open-source intelligence feeds and renders them on a unified dark-ops map interface. It tracks aircraft, ships, satellites, earthquakes, conflict zones, CCTV networks, GPS jamming, and breaking geopolitical events — all updating in real time.
 
-Built with **Next.js**, **MapLibre GL**, **FastAPI**, and **Python**, it's designed for analysts, researchers, and enthusiasts who want a single-pane-of-glass view of global activity.
+
+
+
+https://github.com/user-attachments/assets/248208ec-62f7-49d1-831d-4bd0a1fa6852
+
+
+
+
+
+**ShadowBroker** is a real-time, multi-domain OSINT dashboard that fuses 60+ live intelligence feeds into a single dark-ops map interface. Aircraft, ships, satellites, conflict zones, CCTV networks, GPS jamming, internet-connected devices, police scanners, mesh radio nodes, and breaking geopolitical events — all updating in real time on one screen.
+
+Built with **Next.js**, **MapLibre GL**, **FastAPI**, and **Python**. 35+ toggleable data layers. Right-click any point on Earth for a region/country dossier, head-of-state lookup, and the latest Sentinel-2 satellite photo. No user data is collected or transmitted — the dashboard runs entirely in your browser against a self-hosted backend.
+
+Designed for analysts, researchers, radio operators, and anyone who wants to see what the world looks like when every public signal is on the same map.
+
+
+## Why This Exists
+
+A surprising amount of global telemetry is already public — aircraft ADS-B broadcasts, maritime AIS signals, satellite orbital data, earthquake sensors, mesh radio networks, police scanner feeds, environmental monitoring stations, internet infrastructure telemetry, and more. This data is scattered across dozens of tools and APIs. ShadowBroker combines all of it into a single interface.
+
+The project does not introduce new surveillance capabilities — it aggregates and visualizes existing public datasets. It is fully open-source so anyone can audit exactly what data is accessed and how. No user data is collected or transmitted — everything runs locally against a self-hosted backend. No telemetry, no analytics, no accounts.
+
+### Shodan Connector
+
+ShadowBroker includes an optional Shodan connector for operator-supplied API access. Shodan results are fetched with your own `SHODAN_API_KEY`, rendered as a local investigative overlay (not merged into core feeds), and remain subject to Shodan’s terms of service.
 
 ---
 
 ## Interesting Use Cases
 
-* Track private jets of billionaires
-* Monitor satellites passing overhead
-* Watch naval traffic worldwide
-* Detect GPS jamming zones
-* Follow earthquakes and disasters in real time
+* **Transmit on the InfoNet testnet** — the first decentralized intelligence mesh built into an OSINT tool. Obfuscated messaging with gate personas, Dead Drop peer-to-peer exchange, and a built-in terminal CLI. No accounts, no signup. Privacy is not guaranteed yet — this is an experimental testnet — but the protocol is live and being hardened.
+* **Track Air Force One**, the private jets of billionaires and dictators, and every military tanker, ISR, and fighter broadcasting ADS-B — with automatic holding pattern detection when aircraft start circling
+* **Estimate where US aircraft carriers are** using automated GDELT news scraping — no other open tool does this
+* **Search internet-connected devices worldwide** via Shodan — cameras, SCADA systems, databases — plotted as a live overlay on the map
+* **Right-click anywhere on Earth** for a country dossier (head of state, population, languages), Wikipedia summary, and the latest Sentinel-2 satellite photo at 10m resolution
+* **Click a KiwiSDR node** and tune into live shortwave radio directly in the dashboard. Click a police scanner feed and eavesdrop in one click.
+* **Watch 11,000+ CCTV cameras** across 6 countries — London, NYC, California, Spain, Singapore, and more — streaming live on the map
+* **See GPS jamming zones** in real time — derived from NAC-P degradation analysis of aircraft transponder data
+* **Monitor satellites overhead** color-coded by mission type — military recon, SIGINT, SAR, early warning, space stations — with SatNOGS and TinyGS ground station networks
+* **Track naval traffic** including 25,000+ AIS vessels, fishing activity via Global Fishing Watch, and billionaire superyachts
+* **Follow earthquakes, volcanic eruptions, active wildfires** (NASA FIRMS), severe weather alerts, and air quality readings worldwide
+* **Map military bases, 35,000+ power plants**, 2,000+ data centers, and internet outage regions — cross-referenced automatically
+* **Connect to Meshtastic mesh radio nodes** and APRS amateur radio networks — visible on the map and integrated into Mesh Chat
+* **Switch visual modes** — DEFAULT, SATELLITE, FLIR (thermal), NVG (night vision), CRT (retro terminal) — via the STYLE button
+* **Track trains** across the US (Amtrak) and Europe (DigiTraffic) in real time
 
 ---
 
-## ⚡ Quick Start (Docker or Podman)
+## ⚡ Quick Start (Docker)
 
 ```bash
 git clone https://github.com/BigBodyCobain/Shadowbroker.git
 cd Shadowbroker
-./compose.sh up -d
+docker compose pull
+docker compose up -d
 ```
 
-Open `http://localhost:3000` to view the dashboard! *(Requires Docker or Podman)*
+Open `http://localhost:3000` to view the dashboard! *(Requires [Docker Desktop](https://www.docker.com/products/docker-desktop/) or Docker Engine)*
 
-`compose.sh` auto-detects `docker compose`, `docker-compose`, `podman compose`, and `podman-compose`.
-If both runtimes are installed, you can force Podman with `./compose.sh --engine podman up -d`.
-Do not append a trailing `.` to that command; Compose treats it as a service name.
+> **Podman users:** Replace `docker compose` with `podman compose`, or use the `compose.sh` wrapper which auto-detects your engine. Force Podman with `./compose.sh --engine podman up -d`.
 
 ---
 
+##  🔄 **How to Update**
+
+ShadowBroker uses pre-built Docker images — no local building required. Updating takes seconds:
+
+```bash
+docker compose pull
+docker compose up -d
+```
+
+That's it. `pull` grabs the latest images, `up -d` restarts the containers.
+
+> **Coming from an older version?** Pull the latest repo code first, then pull images:
+>
+> ```bash
+> git pull origin main
+> docker compose down
+> docker compose pull
+> docker compose up -d
+> ```
+
+### ⚠️ **Stuck on the old version?**
+
+**If `git pull` fails or `docker compose up` keeps building from source instead of pulling images**, your clone predates a March 2026 repository migration that rewrote commit history. A normal `git pull` cannot fix this. Run:
+
+```bash
+# Back up any local config you want to keep (.env, etc.)
+cd ..
+rm -rf Shadowbroker
+git clone https://github.com/BigBodyCobain/Shadowbroker.git
+cd Shadowbroker
+docker compose pull
+docker compose up -d
+```
+
+**How to tell if you're affected:** If `docker compose up` shows `RUN apt-get`, `RUN npm ci`, or `RUN pip install` — it's building from source instead of pulling pre-built images. You need a fresh clone.
+
+**Other troubleshooting:**
+
+* **Force re-pull:** `docker compose pull --no-cache`
+* **Prune old images:** `docker image prune -f`
+* **Check logs:** `docker compose logs -f backend`
+
+---
+
+### **☸️ Kubernetes / Helm (Advanced)**
+
+For high-availability deployments or home-lab clusters, ShadowBroker supports deployment via **Helm**. This chart is based on the `bjw-s-labs` template and provides a robust, modular setup for both the backend and frontend.
+
+**1. Add the Repository:**
+```bash
+helm repo add bjw-s-labs https://bjw-s-labs.github.io/helm-charts/
+helm repo update
+```
+
+**2. Install the Chart:**
+```bash
+# Install from the local helm/chart directory
+helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbroker
+```
+
+**3. Key Features:**
+*   **Modular Architecture:** Individually scale the intelligence backend and the HUD frontend.
+*   **Security Context:** Runs with restricted UIDs (1001) for container hardening.
+*   **Ingress Ready:** Compatible with Traefik, Cert-Manager, and Gateway API for secure, external access to your intelligence node.
+
+*Special thanks to [@chr0n1x](https://github.com/chr0n1x) for contributing the initial Kubernetes architecture.*
+
+---
+
+## Experimental Testnet — No Privacy Guarantee
+
+ShadowBroker v0.9.6 introduces **InfoNet**, a decentralized intelligence mesh with obfuscated messaging. This is an **experimental testnet** — not a private messenger.
+
+| Channel | Privacy Status | Details |
+|---|---|---|
+| **Meshtastic / APRS** | **PUBLIC** | RF radio transmissions are public and interceptable by design. |
+| **InfoNet Gate Chat** | **OBFUSCATED** | Messages are obfuscated with gate personas and canonical payload signing, but NOT end-to-end encrypted. Metadata is not hidden. |
+| **Dead Drop DMs** | **STRONGEST CURRENT LANE** | Token-based epoch mailbox with SAS word verification. Strongest lane in this build, but not yet confidently private. |
+
+**Do not transmit anything sensitive on any channel.** Treat all lanes as open and public for now. E2E encryption and deeper native/Tauri hardening are the next milestones. If you fork this project, keep these labels intact and do not make stronger privacy claims than the implementation supports.
+
+---
+
+
 ## ✨ Features
 
+### 🧅 InfoNet — Decentralized Intelligence Mesh (NEW in v0.9.6)
+
+The first decentralized intelligence communication layer built directly into an OSINT platform. No accounts, no signup, no identity required. Nothing like this has existed in an OSINT tool before.
+
+* **InfoNet Experimental Testnet** — A global, obfuscated message relay. Anyone running ShadowBroker can transmit and receive on the InfoNet. Messages pass through a Wormhole relay layer with gate personas, Ed25519 canonical payload signing, and transport obfuscation.
+* **Mesh Chat Panel** — Three-tab interface:
+  * **INFONET** — Gate chat with obfuscated transport (experimental — not yet E2E encrypted)
+  * **MESH** — Meshtastic radio integration (default tab on startup)
+  * **DEAD DROP** — Peer-to-peer message exchange with token-based epoch mailboxes (strongest current lane)
+* **Gate Persona System** — Pseudonymous identities with Ed25519 signing keys, prekey bundles, SAS word contact verification, and abuse reporting
+* **Mesh Terminal** — Built-in CLI: `send`, `dm`, market commands, gate state inspection. Draggable panel, minimizes to the top bar. Type `help` to see all commands.
+* **Crypto Stack** — Ed25519 signing, X25519 Diffie-Hellman, AESGCM encryption with HKDF key derivation, hash chain commitment system. Double-ratchet DM scaffolding in progress.
+
+> **Experimental Testnet — No Privacy Guarantee:** InfoNet messages are obfuscated but NOT end-to-end encrypted. The Mesh network (Meshtastic/APRS) is NOT private — radio transmissions are inherently public. Do not send anything sensitive on any channel. E2E encryption is being developed but is not yet implemented. Treat all channels as open and public for now.
+
+### 🔍 Shodan Device Search (NEW in v0.9.6)
+
+* **Internet Device Search** — Query Shodan directly from ShadowBroker. Search by keyword, CVE, port, or service — results plotted as a live overlay on the map
+* **Configurable Markers** — Shape, color, and size customization for Shodan results
+* **Operator-Supplied API** — Uses your own `SHODAN_API_KEY`; results rendered as a local investigative overlay
+
 ### 🛩️ Aviation Tracking
 
 * **Commercial Flights** — Real-time positions via OpenSky Network (~5,000+ aircraft)
@@ -57,100 +193,158 @@ Do not append a trailing `.` to that command; Compose treats it as a service nam
 
 * **AIS Vessel Stream** — 25,000+ vessels via aisstream.io WebSocket (real-time)
 * **Ship Classification** — Cargo, tanker, passenger, yacht, military vessel types with color-coded icons
-* **Carrier Strike Group Tracker** — All 11 active US Navy aircraft carriers with OSINT-estimated positions
-  * Automated GDELT news scraping for carrier movement intelligence
-  * 50+ geographic region-to-coordinate mappings
-  * Disk-cached positions, auto-updates at 00:00 & 12:00 UTC
+* **Carrier Strike Group Tracker** — All 11 active US Navy aircraft carriers with OSINT-estimated positions. No other open tool does this.
+  * Automated GDELT news scraping parses carrier movement reporting to estimate positions
+  * 50+ geographic region-to-coordinate mappings (e.g. "Eastern Mediterranean" → lat/lng)
+  * Disk-cached positions, auto-refreshes at 00:00 & 12:00 UTC
 * **Cruise & Passenger Ships** — Dedicated layer for cruise liners and ferries
+* **Fishing Activity** — Global Fishing Watch vessel events (NEW)
 * **Clustered Display** — Ships cluster at low zoom with count labels, decluster on zoom-in
 
+### 🚆 Rail Tracking (NEW in v0.9.6)
+
+* **Amtrak Trains** — Real-time positions of Amtrak trains across the US with speed, heading, route, and status
+* **European Rail** — DigiTraffic integration for European train positions
+
 ### 🛰️ Space & Satellites
 
 * **Orbital Tracking** — Real-time satellite positions via CelesTrak TLE data + SGP4 propagation (2,000+ active satellites, no API key required)
 * **Mission-Type Classification** — Color-coded by mission: military recon (red), SAR (cyan), SIGINT (white), navigation (blue), early warning (magenta), commercial imaging (green), space station (gold)
+* **SatNOGS Ground Stations** — Amateur satellite ground station network with live observation data (NEW)
+* **TinyGS LoRa Satellites** — LoRa satellite constellation tracking (NEW)
 
 ### 🌍 Geopolitics & Conflict
 
 * **Global Incidents** — GDELT-powered conflict event aggregation (last 8 hours, ~1,000 events)
 * **Ukraine Frontline** — Live warfront GeoJSON from DeepState Map
-* **SIGINT/RISINT News Feed** — Real-time RSS aggregation from multiple intelligence-focused sources
-* **Region Dossier** — Right-click anywhere on the map for:
+* **Ukraine Air Alerts** — Real-time regional air raid alerts (NEW)
+* **SIGINT/RISINT News Feed** — Real-time RSS aggregation from multiple intelligence-focused sources with user-customizable feeds (up to 20 sources, configurable priority weights 1-5)
+* **Region Dossier** — Right-click anywhere on Earth for an instant intelligence briefing:
   * Country profile (population, capital, languages, currencies, area)
-  * Head of state & government type (Wikidata SPARQL)
+  * Current head of state & government type (live Wikidata SPARQL query)
   * Local Wikipedia summary with thumbnail
+  * Latest Sentinel-2 satellite photo with capture date and cloud cover (10m resolution)
 
 ### 🛰️ Satellite Imagery
 
 * **NASA GIBS (MODIS Terra)** — Daily true-color satellite imagery overlay with 30-day time slider, play/pause animation, and opacity control (~250m/pixel)
 * **High-Res Satellite (Esri)** — Sub-meter resolution imagery via Esri World Imagery — zoom into buildings and terrain detail (zoom 18+)
 * **Sentinel-2 Intel Card** — Right-click anywhere on the map for a floating intel card showing the latest Sentinel-2 satellite photo with capture date, cloud cover %, and clickable full-resolution image (10m resolution, updated every ~5 days)
-* **SATELLITE Style Preset** — Quick-toggle high-res imagery via the STYLE button (DEFAULT → SATELLITE → FLIR → NVG → CRT)
+* **Sentinel Hub Process API** — Copernicus CDSE satellite imagery with OAuth2 token flow (NEW)
+* **VIIRS Nightlights** — Night-time light change detection overlay (NEW)
+* **5 Visual Modes** — Toggle the entire map aesthetic via the STYLE button:
+  * **DEFAULT** — Dark CARTO basemap
+  * **SATELLITE** — Sub-meter Esri World Imagery
+  * **FLIR** — Thermal imaging aesthetic (inverted greyscale)
+  * **NVG** — Night vision green phosphor
+  * **CRT** — Retro terminal scanline overlay
 
-### 📻 Software-Defined Radio (SDR)
+### 📻 Software-Defined Radio & SIGINT
 
 * **KiwiSDR Receivers** — 500+ public SDR receivers plotted worldwide with clustered amber markers
 * **Live Radio Tuner** — Click any KiwiSDR node to open an embedded SDR tuner directly in the SIGINT panel
 * **Metadata Display** — Node name, location, antenna type, frequency bands, active users
-
-### 📷 Surveillance
-
-* **CCTV Mesh** — 2,000+ live traffic cameras from:
-  * 🇬🇧 Transport for London JamCams
-  * 🇺🇸 Austin, TX TxDOT
-  * 🇺🇸 NYC DOT
-  * 🇸🇬 Singapore LTA
-  * Custom URL ingestion
-* **Feed Rendering** — Automatic detection & rendering of video, MJPEG, HLS, embed, satellite tile, and image feeds
-* **Clustered Map Display** — Green dots cluster with count labels, decluster on zoom
-
-### 📡 Signal Intelligence
-
+* **Meshtastic Mesh Radio** — MQTT-based mesh radio integration with node map, integrated into Mesh Chat (NEW)
+* **APRS Integration** — Amateur radio positioning via APRS-IS TCP feed (NEW)
 * **GPS Jamming Detection** — Real-time analysis of aircraft NAC-P (Navigation Accuracy Category) values
   * Grid-based aggregation identifies interference zones
   * Red overlay squares with "GPS JAM XX%" severity labels
-* **Radio Intercept Panel** — Scanner-style UI for monitoring communications
+* **Radio Intercept Panel** — Scanner-style UI with OpenMHZ police/fire scanner feeds. Click any system to listen live. Scan mode cycles through active feeds automatically. Eavesdrop-by-click on real emergency communications.
 
-### 🌐 Additional Layers
+### 📷 Surveillance
 
+* **CCTV Mesh** — 11,000+ live traffic cameras from 13 sources across 6 countries:
+  * 🇬🇧 Transport for London JamCams
+  * 🇺🇸 NYC DOT, Austin TX (TxDOT)
+  * 🇺🇸 California (12 Caltrans districts), Washington State (WSDOT), Georgia DOT, Illinois DOT, Michigan DOT
+  * 🇪🇸 Spain DGT National (20 cities), Madrid City (357 cameras via KML)
+  * 🇸🇬 Singapore LTA
+  * 🌍 Windy Webcams
+* **Feed Rendering** — Automatic detection & rendering of video, MJPEG, HLS, embed, satellite tile, and image feeds
+* **Clustered Map Display** — Green dots cluster with count labels, decluster on zoom
+
+### 🔥 Environmental & Hazard Monitoring
+
+* **NASA FIRMS Fire Hotspots (24h)** — 5,000+ global thermal anomalies from NOAA-20 VIIRS satellite, updated every cycle. Flame-shaped icons color-coded by fire radiative power (FRP): yellow (low), orange, red, dark red (intense). Clustered at low zoom with fire-shaped cluster markers.
+* **Volcanoes** — Smithsonian Global Volcanism Program Holocene volcanoes plotted worldwide (NEW)
+* **Weather Alerts** — Severe weather polygons with urgency/severity indicators (NEW)
+* **Air Quality (PM2.5)** — OpenAQ stations worldwide with real-time particulate matter readings (NEW)
 * **Earthquakes (24h)** — USGS real-time earthquake feed with magnitude-scaled markers
+* **Space Weather Badge** — Live NOAA geomagnetic storm indicator in the bottom status bar. Color-coded Kp index: green (quiet), yellow (active), red (storm G1–G5). Data from SWPC planetary K-index 1-minute feed.
+
+### 🏗️ Infrastructure Monitoring
+
+* **Internet Outage Monitoring** — Regional internet connectivity alerts from Georgia Tech IODA. Grey markers at affected regions with severity percentage. Uses only reliable datasources (BGP routing tables, active ping probing) — no telescope or interpolated data.
+* **Data Center Mapping** — 2,000+ global data centers plotted from a curated dataset. Clustered purple markers with server-rack icons. Click for operator, location, and automatic internet outage cross-referencing by country.
+* **Military Bases** — Global military installation and missile facility database (NEW)
+* **Power Plants** — 35,000+ global power plants from the WRI database (NEW)
+
+### 🌐 Additional Layers & Tools
+
 * **Day/Night Cycle** — Solar terminator overlay showing global daylight/darkness
 * **Global Markets Ticker** — Live financial market indices (minimizable)
 * **Measurement Tool** — Point-to-point distance & bearing measurement on the map
 * **LOCATE Bar** — Search by coordinates (31.8, 34.8) or place name (Tehran, Strait of Hormuz) to fly directly to any location — geocoded via OpenStreetMap Nominatim
 
+![Gaza](https://github.com/user-attachments/assets/f2c953b2-3528-4360-af5a-7ea34ff28489)
+
 ---
 
 ## 🏗️ Architecture
 
 ```
-┌────────────────────────────────────────────────────────┐
-│                   FRONTEND (Next.js)                   │
-│                                                        │
-│  ┌─────────────┐    ┌──────────┐    ┌───────────────┐  │
-│  │ MapLibre GL │    │ NewsFeed │    │ Control Panels│  │
-│  │  2D WebGL   │    │  SIGINT  │    │ Layers/Filters│  │
-│  │ Map Render  │    │  Intel   │    │ Markets/Radio │  │
-│  └──────┬──────┘    └────┬─────┘    └───────┬───────┘  │
-│         └────────────────┼──────────────────┘          │
-│                          │ REST API (60s / 120s)       │
-├──────────────────────────┼─────────────────────────────┤
-│                    BACKEND (FastAPI)                   │
-│                          │                             │
-│  ┌───────────────────────┼──────────────────────────┐  │
-│  │               Data Fetcher (Scheduler)           │  │
-│  │                                                  │  │
-│  │  ┌──────────┬──────────┬──────────┬───────────┐  │  │
-│  │  │ OpenSky  │ adsb.lol │CelesTrak │   USGS    │  │  │
-│  │  │ Flights  │ Military │   Sats   │  Quakes   │  │  │
-│  │  ├──────────┼──────────┼──────────┼───────────┤  │  │
-│  │  │  AIS WS  │ Carrier  │  GDELT   │   CCTV    │  │  │
-│  │  │  Ships   │ Tracker  │ Conflict │  Cameras  │  │  │
-│  │  ├──────────┼──────────┼──────────┼───────────┤  │  │
-│  │  │ DeepState│   RSS    │  Region  │    GPS    │  │  │
-│  │  │ Frontline│  Intel   │ Dossier  │  Jamming  │  │  │
-│  │  └──────────┴──────────┴──────────┴───────────┘  │  │
-│  └──────────────────────────────────────────────────┘  │
-└────────────────────────────────────────────────────────┘
+┌─────────────────────────────────────────────────────────────────┐
+│                      FRONTEND (Next.js)                         │
+│                                                                 │
+│  ┌─────────────┐  ┌──────────┐  ┌───────────┐  ┌────────────┐   │
+│  │ MapLibre GL │  │ NewsFeed │  │  Control  │  │    Mesh    │   │
+│  │  2D WebGL   │  │  SIGINT  │  │  Panels   │  │    Chat    │   │
+│  │ Map Render  │  │  Intel   │  │  Radio    │  │  Terminal  │   │
+│  └──────┬──────┘  └────┬─────┘  └─────┬─────┘  └─────┬──────┘   │
+│         └──────────────┼──────────────┼──────────────┘          │
+│                        │ REST + WebSocket                       │
+├────────────────────────┼────────────────────────────────────────┤
+│                    BACKEND (FastAPI)                            │
+│                        │                                        │
+│  ┌─────────────────────┼─────────────────────────────────────┐  │
+│  │              Data Fetcher (Scheduler)                     │  │
+│  │                                                           │  │
+│  │  ┌───────────┬───────────┬───────────┬───────────┐        │  │
+│  │  │  OpenSky  │ adsb.lol  │ CelesTrak │   USGS    │        │  │
+│  │  │  Flights  │ Military  │   Sats    │  Quakes   │        │  │
+│  │  ├───────────┼───────────┼───────────┼───────────┤        │  │
+│  │  │  AIS WS   │ Carrier   │  GDELT    │ CCTV (13) │        │  │
+│  │  │  Ships    │ Tracker   │ Conflict  │  Cameras  │        │  │
+│  │  ├───────────┼───────────┼───────────┼───────────┤        │  │
+│  │  │ DeepState │   RSS     │  Region   │    GPS    │        │  │
+│  │  │ Frontline │  Intel    │ Dossier   │  Jamming  │        │  │
+│  │  ├───────────┼───────────┼───────────┼───────────┤        │  │
+│  │  │  NASA     │  NOAA     │   IODA    │  KiwiSDR  │        │  │
+│  │  │  FIRMS    │ Space Wx  │  Outages  │  Radios   │        │  │
+│  │  ├───────────┼───────────┼───────────┼───────────┤        │  │
+│  │  │  Shodan   │  Amtrak   │  SatNOGS  │Meshtastic │        │  │
+│  │  │ Devices   │ DigiTraf  │  TinyGS   │   APRS    │        │  │
+│  │  ├───────────┼───────────┼───────────┼───────────┤        │  │
+│  │  │ Volcanoes │ Weather   │  Fishing  │ Mil Bases │        │  │
+│  │  │ Air Qual. │ Alerts    │ Activity  │Pwr Plants │        │  │
+│  │  ├───────────┼───────────┼───────────┼───────────┤        │  │
+│  │  │  Sentinel │  MODIS    │  VIIRS    │  Data     │        │  │
+│  │  │  Hub/STAC │  Terra    │ Nightlts  │  Centers  │        │  │
+│  │  └───────────┴───────────┴───────────┴───────────┘        │  │
+│  └───────────────────────────────────────────────────────────┘  │
+│                                                                 │
+│  ┌───────────────────────────────────────────────────────────┐  │
+│  │              Wormhole / InfoNet Relay                     │  │
+│  │  Gate Personas │ Canonical Signing │ Dead Drop DMs        │  │
+│  └───────────────────────────────────────────────────────────┘  │
+│                                                                 │
+│  ┌───────────────────────────────────────────────────────────┐  │
+│  │                   GHCR (Pre-built Images)                 │  │
+│  │  ghcr.io/bigbodycobain/shadowbroker-backend:latest        │  │
+│  │  ghcr.io/bigbodycobain/shadowbroker-frontend:latest       │  │
+│  │  Multi-arch: linux/amd64 + linux/arm64                    │  │
+│  └───────────────────────────────────────────────────────────┘  │
+└─────────────────────────────────────────────────────────────────┘
 ```
 
 ---
@@ -166,17 +360,38 @@ Do not append a trailing `.` to that command; Compose treats it as a service nam
 | [USGS Earthquake](https://earthquake.usgs.gov) | Global seismic events | ~60s | No |
 | [GDELT Project](https://www.gdeltproject.org) | Global conflict events | ~6h | No |
 | [DeepState Map](https://deepstatemap.live) | Ukraine frontline | ~30min | No |
-| [Transport for London](https://api.tfl.gov.uk) | London CCTV JamCams | ~5min | No |
-| [TxDOT](https://its.txdot.gov) | Austin TX traffic cameras | ~5min | No |
-| [NYC DOT](https://webcams.nyctmc.org) | NYC traffic cameras | ~5min | No |
-| [Singapore LTA](https://datamall.lta.gov.sg) | Singapore traffic cameras | ~5min | **Yes** |
-| [RestCountries](https://restcountries.com) | Country profile data | On-demand (cached 24h) | No |
-| [Wikidata SPARQL](https://query.wikidata.org) | Head of state data | On-demand (cached 24h) | No |
-| [Wikipedia API](https://en.wikipedia.org/api) | Location summaries & aircraft images | On-demand (cached) | No |
+| [Shodan](https://www.shodan.io) | Internet-connected device search | On-demand | **Yes** |
+| [Amtrak](https://www.amtrak.com) | US train positions | ~60s | No |
+| [DigiTraffic](https://www.digitraffic.fi) | European rail positions | ~60s | No |
+| [Global Fishing Watch](https://globalfishingwatch.org) | Fishing vessel activity events | ~10min | No |
+| Transport for London, NYC DOT, TxDOT | CCTV cameras (UK, US) | ~10min | No |
+| Caltrans, WSDOT, GDOT, IDOT, MDOT | CCTV cameras (5 US states) | ~10min | No |
+| Spain DGT, Madrid City | CCTV cameras (Spain) | ~10min | No |
+| [Singapore LTA](https://datamall.lta.gov.sg) | Singapore traffic cameras | ~10min | **Yes** |
+| [Windy Webcams](https://www.windy.com) | Global webcams | ~10min | No |
+| [SatNOGS](https://satnogs.org) | Amateur satellite ground stations | ~30min | No |
+| [TinyGS](https://tinygs.com) | LoRa satellite ground stations | ~30min | No |
+| [Meshtastic MQTT](https://meshtastic.org) | Mesh radio node positions | Real-time | No |
+| [APRS-IS](https://www.aprs-is.net) | Amateur radio positions | Real-time TCP | No |
+| [KiwiSDR](https://kiwisdr.com) | Public SDR receiver locations | ~30min | No |
+| [OpenMHZ](https://openmhz.com) | Police/fire scanner feeds | Real-time | No |
+| [Smithsonian GVP](https://volcano.si.edu) | Holocene volcanoes worldwide | Static (cached) | No |
+| [OpenAQ](https://openaq.org) | Air quality PM2.5 stations | ~120s | No |
+| NOAA / NWS | Severe weather alerts & polygons | ~120s | No |
+| [WRI Global Power Plant DB](https://datasets.wri.org) | 35,000+ power plants | Static (cached) | No |
+| Military base datasets | Global military installations | Static (cached) | No |
+| [NASA FIRMS](https://firms.modaps.eosdis.nasa.gov) | NOAA-20 VIIRS fire/thermal hotspots | ~120s | No |
+| [NOAA SWPC](https://services.swpc.noaa.gov) | Space weather Kp index & solar events | ~120s | No |
+| [IODA (Georgia Tech)](https://ioda.inetintel.cc.gatech.edu) | Regional internet outage alerts | ~120s | No |
+| [DC Map (GitHub)](https://github.com/Ringmast4r/Data-Center-Map---Global) | Global data center locations | Static (cached 7d) | No |
 | [NASA GIBS](https://gibs.earthdata.nasa.gov) | MODIS Terra daily satellite imagery | Daily (24-48h delay) | No |
 | [Esri World Imagery](https://www.arcgis.com) | High-res satellite basemap | Static (periodically updated) | No |
 | [MS Planetary Computer](https://planetarycomputer.microsoft.com) | Sentinel-2 L2A scenes (right-click) | On-demand | No |
-| [KiwiSDR](https://kiwisdr.com) | Public SDR receiver locations | ~30min | No |
+| [Copernicus CDSE](https://dataspace.copernicus.eu) | Sentinel Hub imagery (Process API) | On-demand | **Yes** (free) |
+| [VIIRS Nightlights](https://eogdata.mines.edu) | Night-time light change detection | Static | No |
+| [RestCountries](https://restcountries.com) | Country profile data | On-demand (cached 24h) | No |
+| [Wikidata SPARQL](https://query.wikidata.org) | Head of state data | On-demand (cached 24h) | No |
+| [Wikipedia API](https://en.wikipedia.org/api) | Location summaries & aircraft images | On-demand (cached) | No |
 | [OSM Nominatim](https://nominatim.openstreetmap.org) | Place name geocoding (LOCATE bar) | On-demand | No |
 | [CARTO Basemaps](https://carto.com) | Dark map tiles | Continuous | No |
 
@@ -184,46 +399,89 @@ Do not append a trailing `.` to that command; Compose treats it as a service nam
 
 ## 🚀 Getting Started
 
-### 🐳 Docker / Podman Setup (Recommended for Self-Hosting)
+### 🐳 Docker Setup (Recommended for Self-Hosting)
 
-The repo includes a `docker-compose.yml` that builds both images locally.
+The repo includes a `docker-compose.yml` that pulls pre-built images from the GitHub Container Registry.
 
 ```bash
 git clone https://github.com/BigBodyCobain/Shadowbroker.git
 cd Shadowbroker
 # Add your API keys in a repo-root .env file (optional — see Environment Variables below)
-./compose.sh up -d
+docker compose pull
+docker compose up -d
 ```
 
 Open `http://localhost:3000` to view the dashboard.
 
-> **Deploying publicly or on a LAN?** The frontend **auto-detects** the
-> backend — it uses your browser's hostname with port `8000`
-> (e.g. if you visit `http://192.168.1.50:3000`, API calls go to
-> `http://192.168.1.50:8000`). **No configuration needed** for most setups.
+> **Deploying publicly or on a LAN?** No configuration needed for most setups.
+> The frontend proxies all API calls through the Next.js server to `BACKEND_URL`,
+> which defaults to `http://backend:8000` (Docker internal networking).
+> Port 8000 does not need to be exposed externally.
 >
-> If your backend runs on a **different port or host** (reverse proxy,
-> custom Docker port mapping, separate server), set `NEXT_PUBLIC_API_URL`:
+> If your backend runs on a **different host or port**, set `BACKEND_URL` at runtime — no rebuild required:
 >
 > ```bash
 > # Linux / macOS
-> NEXT_PUBLIC_API_URL=http://myserver.com:9096 docker-compose up -d --build
+> BACKEND_URL=http://myserver.com:9096 docker-compose up -d
 >
 > # Podman (via compose.sh wrapper)
-> NEXT_PUBLIC_API_URL=http://192.168.1.50:9096 ./compose.sh up -d --build
+> BACKEND_URL=http://192.168.1.50:9096 ./compose.sh up -d
 >
 > # Windows (PowerShell)
-> $env:NEXT_PUBLIC_API_URL="http://myserver.com:9096"; docker-compose up -d --build
+> $env:BACKEND_URL="http://myserver.com:9096"; docker-compose up -d
 >
 > # Or add to a .env file next to docker-compose.yml:
-> # NEXT_PUBLIC_API_URL=http://myserver.com:9096
+> # BACKEND_URL=http://myserver.com:9096
 > ```
->
-> This is a **build-time** variable (Next.js limitation) — it gets baked into
-> the frontend during `npm run build`. Changing it requires a rebuild.
 
-If you prefer to call the container engine directly, Podman users can run `podman compose up -d`, or force the wrapper to use Podman with `./compose.sh --engine podman up -d`.
-Depending on your local Podman configuration, `podman compose` may still delegate to an external compose provider while talking to the Podman socket.
+**Podman users:** Replace `docker compose` with `podman compose`, or use the `compose.sh` wrapper which auto-detects your engine.
+
+---
+
+### 🐋 Standalone Deploy (Portainer, Uncloud, NAS, etc.)
+
+No need to clone the repo. Use the pre-built images published to the GitHub Container Registry.
+
+Create a `docker-compose.yml` with the following content and deploy it directly — paste it into Portainer's stack editor, `uncloud deploy`, or any Docker host:
+
+```yaml
+services:
+  backend:
+    image: ghcr.io/bigbodycobain/shadowbroker-backend:latest
+    container_name: shadowbroker-backend
+    ports:
+      - "8000:8000"
+    environment:
+      - AIS_API_KEY=your_aisstream_key          # Required — get one free at aisstream.io
+      - OPENSKY_CLIENT_ID=                       # Optional — higher flight data rate limits
+      - OPENSKY_CLIENT_SECRET=                   # Optional — paired with Client ID above
+      - LTA_ACCOUNT_KEY=                         # Optional — Singapore CCTV cameras
+      - SHODAN_API_KEY=                          # Optional — Shodan device search overlay
+      - SH_CLIENT_ID=                            # Optional — Sentinel Hub satellite imagery
+      - SH_CLIENT_SECRET=                        # Optional — paired with Sentinel Hub ID
+      - CORS_ORIGINS=                            # Optional — comma-separated allowed origins
+    volumes:
+      - backend_data:/app/data
+    restart: unless-stopped
+
+  frontend:
+    image: ghcr.io/bigbodycobain/shadowbroker-frontend:latest
+    container_name: shadowbroker-frontend
+    ports:
+      - "3000:3000"
+    environment:
+      - BACKEND_URL=http://backend:8000   # Docker internal networking — no rebuild needed
+    depends_on:
+      - backend
+    restart: unless-stopped
+
+volumes:
+  backend_data:
+```
+
+> **How it works:** The frontend container proxies all `/api/*` requests through the Next.js server to `BACKEND_URL` using Docker's internal networking. The browser only ever talks to port 3000 — port 8000 does not need to be exposed externally.
+>
+> `BACKEND_URL` is a plain runtime environment variable (not a build-time `NEXT_PUBLIC_*`), so you can change it in Portainer, Uncloud, or any compose editor without rebuilding the image. Set it to the address where your backend is reachable from inside the Docker network (e.g. `http://backend:8000`, `http://192.168.1.50:8000`).
 
 ---
 
@@ -235,9 +493,14 @@ If you just want to run the dashboard without dealing with terminal commands:
 2. Download the latest `.zip` file from the release.
 3. Extract the folder to your computer.
 4. **Windows:** Double-click `start.bat`.
-   **Mac/Linux:** Open terminal, type `chmod +x start.sh`, and run `./start.sh`.
+   **Mac/Linux:** Open terminal, type `chmod +x start.sh`, `dos2unix start.sh`, and run `./start.sh`.
 5. It will automatically install everything and launch the dashboard!
 
+Local launcher notes:
+
+- `start.bat` / `start.sh` run the app without Docker — they install dependencies and start both servers directly.
+- If Wormhole identity or DM contact endpoints fail after an upgrade, check the `docs/mesh/` folder for troubleshooting.
+
 ---
 
 ### 💻 Developer Setup
@@ -265,6 +528,18 @@ venv\Scripts\activate        # Windows
 # source venv/bin/activate   # macOS/Linux
 pip install -r requirements.txt   # includes pystac-client for Sentinel-2
 
+# Optional helper scripts (creates venv + installs dev deps)
+# Windows PowerShell
+# .\scripts\setup-venv.ps1
+# macOS/Linux
+# ./scripts/setup-venv.sh
+
+# Optional env check (prints warnings for missing keys)
+# Windows PowerShell
+# .\scripts\check-env.ps1
+# macOS/Linux
+# ./scripts/check-env.sh
+
 # Create .env with your API keys
 echo "AIS_API_KEY=your_aisstream_key" >> .env
 echo "OPENSKY_CLIENT_ID=your_opensky_client_id" >> .env
@@ -287,11 +562,40 @@ This starts:
 * **Next.js** frontend on `http://localhost:3000`
 * **FastAPI** backend on `http://localhost:8000`
 
+### Pre-commit (Optional)
+
+If you use pre-commit, install hooks once from repo root:
+
+```bash
+pre-commit install
+```
+
+### Local AIS Receiver (Optional)
+
+You can feed your own AIS ship data into ShadowBroker using an RTL-SDR dongle and [AIS-catcher](https://github.com/jvde-github/AIS-catcher), an open-source AIS decoder. This gives you real-time coverage of vessels in your local area — no API key needed.
+
+1. Plug in an RTL-SDR dongle
+2. Install AIS-catcher ([releases](https://github.com/jvde-github/AIS-catcher/releases)) or use the Docker image:
+   ```bash
+   docker run -d --device /dev/bus/usb \
+     ghcr.io/jvde-github/ais-catcher -H http://host.docker.internal:4000/api/ais/feed interval 10
+   ```
+3. Or run natively:
+   ```bash
+   AIS-catcher -H http://localhost:4000/api/ais/feed interval 10
+   ```
+
+AIS-catcher decodes VHF radio signals on 161.975 MHz and 162.025 MHz and POSTs decoded vessel data to ShadowBroker every 10 seconds. Ships detected by your SDR antenna appear alongside the global AIS stream.
+
+**Docker (ARM/Raspberry Pi):** See [docker-shipfeeder](https://github.com/sdr-enthusiasts/docker-shipfeeder) for a production-ready Docker image optimized for ARM.
+
+**Note:** AIS range depends on your antenna — typically 20-40 nautical miles with a basic setup, 60+ nm with a marine VHF antenna at elevation.
+
 ---
 
 ## 🎛️ Data Layers
 
-All layers are independently toggleable from the left panel:
+All 37 layers are independently toggleable from the left panel:
 
 | Layer | Default | Description |
 |---|---|---|
@@ -300,19 +604,39 @@ All layers are independently toggleable from the left panel:
 | Private Jets | ✅ ON | High-value bizjets with owner data |
 | Military Flights | ✅ ON | Military & government aircraft |
 | Tracked Aircraft | ✅ ON | Special interest watch list |
-| Satellites | ✅ ON | Orbital assets by mission type |
-| Carriers / Mil / Cargo | ✅ ON | Navy carriers, cargo ships, tankers |
-| Civilian Vessels | ❌ OFF | Yachts, fishing, recreational |
-| Cruise / Passenger | ✅ ON | Cruise ships and ferries |
-| Earthquakes (24h) | ✅ ON | USGS seismic events |
-| CCTV Mesh | ❌ OFF | Surveillance camera network |
-| Ukraine Frontline | ✅ ON | Live warfront positions |
-| Global Incidents | ✅ ON | GDELT conflict events |
 | GPS Jamming | ✅ ON | NAC-P degradation zones |
+| Carriers / Mil / Cargo | ✅ ON | Navy carriers, cargo ships, tankers |
+| Civilian Vessels | ✅ ON | Yachts, fishing, recreational |
+| Cruise / Passenger | ✅ ON | Cruise ships and ferries |
+| Tracked Yachts | ✅ ON | Billionaire & oligarch superyachts |
+| Fishing Activity | ✅ ON | Global Fishing Watch vessel events |
+| Trains | ✅ ON | Amtrak + European rail positions |
+| Satellites | ✅ ON | Orbital assets by mission type |
+| SatNOGS | ✅ ON | Amateur satellite ground stations |
+| TinyGS | ✅ ON | LoRa satellite ground stations |
+| Earthquakes (24h) | ✅ ON | USGS seismic events |
+| Fire Hotspots (24h) | ✅ ON | NASA FIRMS VIIRS thermal anomalies |
+| Volcanoes | ✅ ON | Smithsonian Holocene volcanoes |
+| Weather Alerts | ✅ ON | Severe weather polygons |
+| Air Quality (PM2.5) | ✅ ON | OpenAQ stations worldwide |
+| Ukraine Frontline | ✅ ON | Live warfront positions |
+| Ukraine Air Alerts | ✅ ON | Regional air raid alerts |
+| Global Incidents | ✅ ON | GDELT conflict events |
+| CCTV Mesh | ✅ ON | 11,000+ cameras across 13 sources, 6 countries |
+| Internet Outages | ✅ ON | IODA regional connectivity alerts |
+| Data Centers | ✅ ON | Global data center locations (2,000+) |
+| Military Bases | ✅ ON | Global military installations |
+| KiwiSDR Receivers | ✅ ON | Public SDR radio receivers |
+| Meshtastic Nodes | ✅ ON | Mesh radio node positions |
+| APRS | ✅ ON | Amateur radio positioning |
+| Scanners | ✅ ON | Police/fire scanner feeds |
+| Day / Night Cycle | ✅ ON | Solar terminator overlay |
 | MODIS Terra (Daily) | ❌ OFF | NASA GIBS daily satellite imagery |
 | High-Res Satellite | ❌ OFF | Esri sub-meter satellite imagery |
-| KiwiSDR Receivers | ❌ OFF | Public SDR radio receivers |
-| Day / Night Cycle | ✅ ON | Solar terminator overlay |
+| Sentinel Hub | ❌ OFF | Copernicus CDSE Process API |
+| VIIRS Nightlights | ❌ OFF | Night-time light change detection |
+| Power Plants | ❌ OFF | 35,000+ global power plants |
+| Shodan Overlay | ❌ OFF | Internet device search results |
 
 ---
 
@@ -323,8 +647,9 @@ The platform is optimized for handling massive real-time datasets:
 * **Gzip Compression** — API payloads compressed ~92% (11.6 MB → 915 KB)
 * **ETag Caching** — `304 Not Modified` responses skip redundant JSON parsing
 * **Viewport Culling** — Only features within the visible map bounds (+20% buffer) are rendered
-* **Clustered Rendering** — Ships, CCTV, and earthquakes use MapLibre clustering to reduce feature count
-* **Debounced Viewport Updates** — 300ms debounce prevents GeoJSON rebuild thrash during pan/zoom
+* **Imperative Map Updates** — High-volume layers (flights, satellites, fires) bypass React reconciliation via direct `setData()` calls
+* **Clustered Rendering** — Ships, CCTV, earthquakes, and data centers use MapLibre clustering to reduce feature count
+* **Debounced Viewport Updates** — 300ms debounce prevents GeoJSON rebuild thrash during pan/zoom; 2s debounce on dense layers (satellites, fires)
 * **Position Interpolation** — Smooth 10s tick animation between data refreshes
 * **React.memo** — Heavy components wrapped to prevent unnecessary re-renders
 * **Coordinate Precision** — Lat/lng rounded to 5 decimals (~1m) to reduce JSON size
@@ -336,41 +661,72 @@ The platform is optimized for handling massive real-time datasets:
 ```
 live-risk-dashboard/
 ├── backend/
-│   ├── main.py                     # FastAPI app, middleware, API routes
-│   ├── carrier_cache.json          # Persisted carrier OSINT positions
-│   ├── cctv.db                     # SQLite CCTV camera database
-│   └── services/
-│       ├── data_fetcher.py         # Core scheduler — fetches all data sources
-│       ├── ais_stream.py           # AIS WebSocket client (25K+ vessels)
-│       ├── carrier_tracker.py      # OSINT carrier position tracker
-│       ├── cctv_pipeline.py        # Multi-source CCTV camera ingestion
-│       ├── geopolitics.py          # GDELT + Ukraine frontline fetcher
-│       ├── region_dossier.py       # Right-click country/city intelligence
-│       ├── radio_intercept.py      # Scanner radio feed integration
-│       ├── kiwisdr_fetcher.py      # KiwiSDR receiver scraper
-│       ├── sentinel_search.py      # Sentinel-2 STAC imagery search
-│       ├── network_utils.py        # HTTP client with curl fallback
-│       └── api_settings.py         # API key management
+│   ├── main.py                     # FastAPI app, middleware, API routes (~4,000 lines)
+│   ├── cctv.db                     # SQLite CCTV camera database (auto-generated)
+│   ├── config/
+│   │   └── news_feeds.json         # User-customizable RSS feed list
+│   ├── services/
+│   │   ├── data_fetcher.py         # Core scheduler — orchestrates all data sources
+│   │   ├── ais_stream.py           # AIS WebSocket client (25K+ vessels)
+│   │   ├── carrier_tracker.py      # OSINT carrier position estimator (GDELT news scraping)
+│   │   ├── cctv_pipeline.py        # 13-source CCTV camera ingestion pipeline
+│   │   ├── geopolitics.py          # GDELT + Ukraine frontline + air alerts
+│   │   ├── region_dossier.py       # Right-click country/city intelligence
+│   │   ├── radio_intercept.py      # Police scanner feeds + OpenMHZ
+│   │   ├── kiwisdr_fetcher.py      # KiwiSDR receiver scraper
+│   │   ├── sentinel_search.py      # Sentinel-2 STAC imagery search
+│   │   ├── shodan_connector.py     # Shodan device search connector
+│   │   ├── sigint_bridge.py        # APRS-IS TCP bridge
+│   │   ├── network_utils.py        # HTTP client with curl fallback
+│   │   ├── api_settings.py         # API key management
+│   │   ├── news_feed_config.py     # RSS feed config manager
+│   │   ├── fetchers/
+│   │   │   ├── flights.py          # OpenSky, adsb.lol, GPS jamming, holding patterns
+│   │   │   ├── geo.py              # AIS vessels, carriers, GDELT, fishing activity
+│   │   │   ├── satellites.py       # CelesTrak TLE + SGP4 propagation
+│   │   │   ├── earth_observation.py # Quakes, fires, volcanoes, air quality, weather
+│   │   │   ├── infrastructure.py   # Data centers, power plants, military bases
+│   │   │   ├── trains.py           # Amtrak + DigiTraffic European rail
+│   │   │   ├── sigint.py           # SatNOGS, TinyGS, APRS, Meshtastic
+│   │   │   ├── meshtastic_map.py   # Meshtastic MQTT + map node aggregation
+│   │   │   ├── military.py         # Military aircraft classification
+│   │   │   ├── news.py             # RSS intelligence feed aggregation
+│   │   │   ├── financial.py        # Global markets data
+│   │   │   └── ukraine_alerts.py   # Ukraine air raid alerts
+│   │   └── mesh/                   # InfoNet / Wormhole protocol stack
+│   │       ├── mesh_protocol.py    # Core mesh protocol + routing
+│   │       ├── mesh_crypto.py      # Ed25519, X25519, AESGCM primitives
+│   │       ├── mesh_hashchain.py   # Hash chain commitment system (~1,400 lines)
+│   │       ├── mesh_router.py      # Multi-transport router (APRS, Meshtastic, WS)
+│   │       ├── mesh_wormhole_persona.py  # Gate persona identity management
+│   │       ├── mesh_wormhole_dead_drop.py # Dead Drop token-based DM mailbox
+│   │       ├── mesh_wormhole_ratchet.py   # Double-ratchet DM scaffolding
+│   │       ├── mesh_wormhole_gate_keys.py # Gate key management + rotation
+│   │       ├── mesh_wormhole_seal.py      # Message sealing + unsealing
+│   │       ├── mesh_merkle.py      # Merkle tree proofs for data commitment
+│   │       ├── mesh_reputation.py  # Node reputation scoring
+│   │       ├── mesh_oracle.py      # Oracle consensus protocol
+│   │       └── mesh_secure_storage.py # Secure credential storage
 │
 ├── frontend/
 │   ├── src/
 │   │   ├── app/
 │   │   │   └── page.tsx            # Main dashboard — state, polling, layout
 │   │   └── components/
-│   │       ├── MaplibreViewer.tsx   # Core map — 2,000+ lines, all GeoJSON layers
-│   │       ├── NewsFeed.tsx         # SIGINT feed + entity detail panels
-│   │       ├── WorldviewLeftPanel.tsx   # Data layer toggles
+│   │       ├── MaplibreViewer.tsx   # Core map — all GeoJSON layers
+│   │       ├── MeshChat.tsx        # InfoNet / Mesh / Dead Drop chat panel
+│   │       ├── MeshTerminal.tsx    # Draggable CLI terminal
+│   │       ├── NewsFeed.tsx        # SIGINT feed + entity detail panels
+│   │       ├── WorldviewLeftPanel.tsx   # Data layer toggles (35+ layers)
 │   │       ├── WorldviewRightPanel.tsx  # Search + filter sidebar
-│   │       ├── FilterPanel.tsx     # Basic layer filters
 │   │       ├── AdvancedFilterModal.tsx  # Airport/country/owner filtering
 │   │       ├── MapLegend.tsx       # Dynamic legend with all icons
 │   │       ├── MarketsPanel.tsx    # Global financial markets ticker
 │   │       ├── RadioInterceptPanel.tsx # Scanner-style radio panel
 │   │       ├── FindLocateBar.tsx   # Search/locate bar
-│   │       ├── ChangelogModal.tsx  # Version changelog popup
-│   │       ├── SettingsPanel.tsx   # App settings
+│   │       ├── ChangelogModal.tsx  # Version changelog popup (auto-shows on upgrade)
+│   │       ├── SettingsPanel.tsx   # API Keys + News Feed + Shodan config
 │   │       ├── ScaleBar.tsx        # Map scale indicator
-│   │       ├── WikiImage.tsx       # Wikipedia image fetcher
 │   │       └── ErrorBoundary.tsx   # Crash recovery wrapper
 │   └── package.json
 ```
@@ -389,26 +745,41 @@ AIS_API_KEY=your_aisstream_key                # Maritime vessel tracking (aisstr
 OPENSKY_CLIENT_ID=your_opensky_client_id      # OAuth2 — higher rate limits for flight data
 OPENSKY_CLIENT_SECRET=your_opensky_secret     # OAuth2 — paired with Client ID above
 LTA_ACCOUNT_KEY=your_lta_key                  # Singapore CCTV cameras
+SHODAN_API_KEY=your_shodan_key                # Shodan device search overlay
+SH_CLIENT_ID=your_sentinel_hub_id             # Copernicus CDSE Sentinel Hub imagery
+SH_CLIENT_SECRET=your_sentinel_hub_secret     # Paired with Sentinel Hub Client ID
 ```
 
-### Frontend (optional)
+### Frontend
 
 | Variable | Where to set | Purpose |
 |---|---|---|
-| `NEXT_PUBLIC_API_URL` | `.env` next to `docker-compose.yml`, or shell env | Override backend URL when deploying publicly or behind a reverse proxy. Leave unset for auto-detection. |
+| `BACKEND_URL` | `environment` in `docker-compose.yml`, or shell env | URL the Next.js server uses to proxy API calls to the backend. Defaults to `http://backend:8000`. **Runtime variable — no rebuild needed.** |
 
-**How auto-detection works:** When `NEXT_PUBLIC_API_URL` is not set, the frontend
-reads `window.location.hostname` in the browser and calls `{protocol}//{hostname}:8000`.
-This means the dashboard works on `localhost`, LAN IPs, and public domains without
-any configuration — as long as the backend is reachable on port 8000 of the same host.
+**How it works:** The frontend proxies all `/api/*` requests through the Next.js server to `BACKEND_URL` using Docker's internal networking. Browsers only talk to port 3000; port 8000 never needs to be exposed externally. For local dev without Docker, `BACKEND_URL` defaults to `http://localhost:8000`.
+
+---
+
+## 🤝 Contributors
+
+ShadowBroker is built in the open. These people shipped real code:
+
+| Who | What | PR |
+|-----|------|----|
+| [@wa1id](https://github.com/wa1id) | CCTV ingestion fix — threaded SQLite, persistent DB, startup hydration, cluster clickability | #92 |
+| [@AlborzNazari](https://github.com/AlborzNazari) | Spain DGT + Madrid CCTV sources, STIX 2.1 threat intel export | #91 |
+| [@adust09](https://github.com/adust09) | Power plants layer, East Asia intel coverage (JSDF bases, ICAO enrichment, Taiwan news, military classification) | #71, #72, #76, #77, #87 |
+| [@Xpirix](https://github.com/Xpirix) | LocateBar style and interaction improvements | #78 |
+| [@imqdcr](https://github.com/imqdcr) | Ship toggle split (4 categories) + stable MMSI/callsign entity IDs | — |
+| [@csysp](https://github.com/csysp) | Dismissible threat alerts + stable entity IDs for GDELT & News | #48, #63 |
+| [@suranyami](https://github.com/suranyami) | Parallel multi-arch Docker builds (11min → 3min) + runtime BACKEND_URL fix | #35, #44 |
+| [@chr0n1x](https://github.com/chr0n1x) | Kubernetes / Helm chart architecture for HA deployments | — |
 
 ---
 
 ## ⚠️ Disclaimer
 
-This is an **educational and research tool** built entirely on publicly available, open-source intelligence (OSINT) data. No classified, restricted, or non-public data sources are used. Carrier positions are estimates based on public reporting. The military-themed UI is purely aesthetic.
-
-**Do not use this tool for any operational, military, or intelligence purpose.**
+This tool is built entirely on publicly available, open-source intelligence (OSINT) data. No classified, restricted, or non-public data is used. Carrier positions are estimates based on public reporting. The military-themed UI is purely aesthetic.
 
 ---
 

backend/.dockerignore

@@ -4,13 +4,29 @@ __pycache__/
 .env
 .pytest_cache/
 .coverage
+.git/
+node_modules/
 cctv.db
+*.sqlite
+*.db
+
+# Debug/log files
 *.txt
 !requirements.txt
-# Exclude debug/cache JSON but keep package.json and tracked_names
+!requirements-dev.txt
+*.html
+*.xlsx
+
+# Debug/cache JSON (keep package*.json and data files)
 ais_cache.json
+carrier_cache.json
 carrier_positions.json
 dump.json
 debug_fast.json
 nyc_full.json
 nyc_sample.json
+tmp_fast.json
+
+# Test files (not needed in production image)
+test_*.py
+tests/

backend/.env.example

@@ -0,0 +1,105 @@
+# ShadowBroker Backend — Environment Variables
+# Copy this file to .env and fill in your keys:
+#   cp .env.example .env
+
+# ── Required Keys ──────────────────────────────────────────────
+# Without these, the corresponding data layers will be empty.
+
+OPENSKY_CLIENT_ID=        # https://opensky-network.org/ — free account, OAuth2 client ID
+OPENSKY_CLIENT_SECRET=    # OAuth2 client secret from your OpenSky dashboard
+AIS_API_KEY=              # https://aisstream.io/ — free tier WebSocket key
+
+# ── Optional ───────────────────────────────────────────────────
+
+# Override allowed CORS origins (comma-separated). Defaults to localhost + LAN auto-detect.
+# CORS_ORIGINS=http://192.168.1.50:3000,https://my-domain.com
+
+# Admin key — protects sensitive endpoints (API key management, system update).
+# If unset, endpoints are only accessible from localhost unless ALLOW_INSECURE_ADMIN=true.
+# Set this in production and enter the same key in Settings → Admin Key.
+# ADMIN_KEY=your-secret-admin-key-here
+
+# Allow insecure admin access without ADMIN_KEY (local dev only).
+# ALLOW_INSECURE_ADMIN=false
+
+# User-Agent for Nominatim geocoding requests (per OSM usage policy).
+# NOMINATIM_USER_AGENT=ShadowBroker/1.0 (https://github.com/BigBodyCobain/Shadowbroker)
+
+# LTA Singapore traffic cameras — leave blank to skip this data source.
+# LTA_ACCOUNT_KEY=
+
+# NASA FIRMS country-scoped fire data — enriches global CSV with conflict-zone hotspots.
+# Free MAP_KEY from https://firms.modaps.eosdis.nasa.gov/map/#d:24hrs;@0.0,0.0,3.0z
+# FIRMS_MAP_KEY=
+
+# Ukraine air raid alerts from alerts.in.ua — free token from https://alerts.in.ua/
+# ALERTS_IN_UA_TOKEN=
+
+# Google Earth Engine service account for VIIRS change detection (optional).
+# Download JSON key from https://console.cloud.google.com/iam-admin/serviceaccounts
+# pip install earthengine-api
+# GEE_SERVICE_ACCOUNT_KEY=
+
+# ── Mesh / Reticulum (RNS) ─────────────────────────────────────
+# Full-node / participant-node posture for public Infonet sync.
+# MESH_NODE_MODE=participant      # participant | relay | perimeter
+# MESH_BOOTSTRAP_DISABLED=false
+# MESH_BOOTSTRAP_MANIFEST_PATH=data/bootstrap_peers.json
+# MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY=
+# MESH_RELAY_PEERS=               # comma-separated operator-trusted sync/push peers
+# MESH_PEER_PUSH_SECRET=Mv63UvLfwqOEVWeRBXjA8MtFl2nEkkhUlLYVHiX1Zzo   # transport auth for mesh peer push (default works out of the box)
+# MESH_SYNC_INTERVAL_S=300
+# MESH_SYNC_FAILURE_BACKOFF_S=60
+#
+# Enable Reticulum bridge for Infonet event gossip.
+# MESH_RNS_ENABLED=false
+# MESH_RNS_APP_NAME=shadowbroker
+# MESH_RNS_ASPECT=infonet
+# MESH_RNS_IDENTITY_PATH=
+# MESH_RNS_PEERS=  # comma-separated destination hashes
+# MESH_RNS_DANDELION_HOPS=2
+# MESH_RNS_DANDELION_DELAY_MS=400
+# MESH_RNS_CHURN_INTERVAL_S=300
+# MESH_RNS_MAX_PEERS=32
+# MESH_RNS_MAX_PAYLOAD=8192
+# MESH_RNS_PEER_BUCKET_PREFIX=4
+# MESH_RNS_MAX_PEERS_PER_BUCKET=4
+# MESH_RNS_PEER_FAIL_THRESHOLD=3
+# MESH_RNS_PEER_COOLDOWN_S=300
+# MESH_RNS_SHARD_ENABLED=false
+# MESH_RNS_SHARD_DATA_SHARDS=3
+# MESH_RNS_SHARD_PARITY_SHARDS=1
+# MESH_RNS_SHARD_TTL_S=30
+# MESH_RNS_FEC_CODEC=xor
+# MESH_RNS_BATCH_MS=200
+# MESH_RNS_COVER_INTERVAL_S=0
+# MESH_RNS_COVER_SIZE=64
+# MESH_RNS_IBF_WINDOW=256
+# MESH_RNS_IBF_TABLE_SIZE=64
+# MESH_RNS_IBF_MINHASH_SIZE=16
+# MESH_RNS_IBF_MINHASH_THRESHOLD=0.25
+# MESH_RNS_IBF_WINDOW_JITTER=32
+# MESH_RNS_IBF_INTERVAL_S=120
+# MESH_RNS_IBF_SYNC_PEERS=3
+# MESH_RNS_IBF_QUORUM_TIMEOUT_S=6
+# MESH_RNS_IBF_MAX_REQUEST_IDS=64
+# MESH_RNS_IBF_MAX_EVENTS=64
+# MESH_RNS_SESSION_ROTATE_S=0
+# MESH_RNS_IBF_FAIL_THRESHOLD=3
+# MESH_RNS_IBF_COOLDOWN_S=120
+# MESH_VERIFY_INTERVAL_S=600
+# MESH_VERIFY_SIGNATURES=false
+
+# ── Mesh DM Relay ──────────────────────────────────────────────
+# MESH_DM_TOKEN_PEPPER=change-me
+
+# ── Self Update ────────────────────────────────────────────────
+# MESH_UPDATE_SHA256=
+
+# ── Wormhole (Local Agent) ─────────────────────────────────────
+# WORMHOLE_HOST=127.0.0.1
+# WORMHOLE_PORT=8787
+# WORMHOLE_RELOAD=false
+# WORMHOLE_TRANSPORT=direct
+# WORMHOLE_SOCKS_PROXY=127.0.0.1:9050
+# WORMHOLE_SOCKS_DNS=true

backend/Dockerfile

@@ -1,4 +1,17 @@
-FROM python:3.10-slim
+# ---- Stage 1: Compile privacy-core Rust library ----
+FROM rust:1.88-slim-bookworm AS rust-builder
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    pkg-config libssl-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY privacy-core /build/privacy-core
+WORKDIR /build/privacy-core
+RUN cargo build --release --lib \
+    && ls -la target/release/libprivacy_core.so
+
+# ---- Stage 2: Python backend ----
+FROM python:3.11-slim-bookworm
 
 WORKDIR /app
 
@@ -9,19 +22,43 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     && apt-get install -y --no-install-recommends nodejs \
     && rm -rf /var/lib/apt/lists/*
 
-# Install dependencies
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
+# Install UV for fast, reproducible Python dependency management
+ADD https://astral.sh/uv/install.sh /uv-installer.sh
+RUN sh /uv-installer.sh && rm /uv-installer.sh
+ENV PATH="/root/.local/bin:$PATH"
+# Install into system Python (no venv needed inside container)
+ENV UV_PROJECT_ENVIRONMENT=/usr/local
 
-# Copy source code
-COPY . .
+# Copy workspace root files for UV resolution (build context is repo root)
+COPY pyproject.toml /workspace/pyproject.toml
+COPY uv.lock /workspace/uv.lock
+COPY backend/pyproject.toml /workspace/backend/pyproject.toml
+
+# Install Python dependencies using the lockfile
+RUN cd /workspace/backend && uv sync --frozen --no-dev \
+    && playwright install --with-deps chromium
+
+# Copy backend source code
+COPY backend/ .
 
 # Install Node.js dependencies (ws module for AIS WebSocket proxy)
-RUN npm install --omit=dev
+COPY backend/package*.json ./
+RUN npm ci --omit=dev
+
+# Clean up workspace scaffold
+RUN rm -rf /workspace
+
+# Copy compiled privacy-core library from Rust builder stage
+COPY --from=rust-builder /build/privacy-core/target/release/libprivacy_core.so /app/libprivacy_core.so
+ENV PRIVACY_CORE_LIB=/app/libprivacy_core.so
 
 # Create a non-root user for security
+# Grant write access to /app so the auto-updater can extract files
+# Pre-create /app/data so mounted volumes inherit correct ownership
 RUN adduser --system --uid 1001 backenduser \
-    && chown -R backenduser /app
+    && mkdir -p /app/data \
+    && chown -R backenduser /app \
+    && chmod -R u+w /app
 
 # Switch to the non-root user
 USER backenduser
@@ -30,4 +67,4 @@ USER backenduser
 EXPOSE 8000
 
 # Start FastAPI server
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--timeout-keep-alive", "120"]

backend/ais_proxy.js

@@ -1,4 +1,5 @@
 const WebSocket = require('ws');
+const readline = require('readline');
 
 const args = process.argv.slice(2);
 const API_KEY = args[0] || process.env.AIS_API_KEY;
@@ -8,22 +9,15 @@ if (!API_KEY) {
     process.exit(1);
 }
 
-const FILTER = [
-    // US Aircraft Carriers and major naval groups
-    { "MMSI": 338000000 }, { "MMSI": 338100000 }, // US Navy general prefixes
-    // Plus let's grab some global shipping for density
-    { "BoundingBoxes": [[[-90, -180], [90, 180]]] }
-];
+// Start with global coverage, until frontend updates it
+let currentBboxes = [[[-90, -180], [90, 180]]];
+let activeWs = null;
 
-function connect() {
-    const ws = new WebSocket('wss://stream.aisstream.io/v0/stream');
-
-    ws.on('open', () => {
+function sendSub(ws) {
+    if (ws && ws.readyState === WebSocket.OPEN) {
         const subMsg = {
             APIKey: API_KEY,
-            BoundingBoxes: [
-                [[-90, -180], [90, 180]]
-            ],
+            BoundingBoxes: currentBboxes,
             FilterMessageTypes: [
                 "PositionReport",
                 "ShipStaticData",
@@ -31,17 +25,39 @@ function connect() {
             ]
         };
         ws.send(JSON.stringify(subMsg));
+    }
+}
+
+// Listen for dynamic bounding box updates via stdin from Python orchestrator
+const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+    terminal: false
+});
+
+rl.on('line', (line) => {
+    try {
+        const cmd = JSON.parse(line);
+        if (cmd.type === "update_bbox" && cmd.bboxes) {
+            currentBboxes = cmd.bboxes;
+            if (activeWs) sendSub(activeWs); // Resend subscription (swap and replace)
+        }
+    } catch (e) {}
+});
+
+function connect() {
+    const ws = new WebSocket('wss://stream.aisstream.io/v0/stream');
+    activeWs = ws;
+
+    ws.on('open', () => {
+        sendSub(ws);
     });
 
     ws.on('message', (data) => {
-        // Output raw AIS message JSON to stdout so Python can consume it
-        // We ensure exactly one JSON object per line.
         try {
             const parsed = JSON.parse(data);
             console.log(JSON.stringify(parsed));
-        } catch (e) {
-            // ignore non-json
-        }
+        } catch (e) {}
     });
 
     ws.on('error', (err) => {
@@ -49,6 +65,7 @@ function connect() {
     });
 
     ws.on('close', () => {
+        activeWs = null;
         console.error("WebSocket Proxy Closed. Reconnecting in 5s...");
         setTimeout(connect, 5000);
     });

backend/check_regions.py

@@ -1,17 +0,0 @@
-import requests
-
-regions = [
-    {"lat": 39.8, "lon": -98.5, "dist": 2000},  # USA
-    {"lat": 50.0, "lon": 15.0, "dist": 2000},   # Europe
-    {"lat": 35.0, "lon": 105.0, "dist": 2000}   # Asia / China
-]
-
-for r in regions:
-    url = f"https://api.adsb.lol/v2/lat/{r['lat']}/lon/{r['lon']}/dist/{r['dist']}"
-    res = requests.get(url, timeout=10)
-    if res.status_code == 200:
-        data = res.json()
-        acs = data.get("ac", [])
-        print(f"Region lat:{r['lat']} lon:{r['lon']} dist:{r['dist']} -> Flights: {len(acs)}")
-    else:
-        print(f"Error for Region lat:{r['lat']} lon:{r['lon']}: HTTP {res.status_code}")

backend/clean_osm_cctvs.py

@@ -1,10 +0,0 @@
-import sqlite3
-import os
-
-db_path = os.path.join(os.path.dirname(__file__), 'cctv.db')
-conn = sqlite3.connect(db_path)
-cur = conn.cursor()
-cur.execute("DELETE FROM cameras WHERE id LIKE 'OSM-%'")
-print(f"Deleted {cur.rowcount} OSM cameras from DB.")
-conn.commit()
-conn.close()

backend/config/news_feeds.json

@@ -0,0 +1,104 @@
+{
+  "feeds": [
+    {
+      "name": "Reuters",
+      "url": "https://www.reutersagency.com/feed/?best-topics=world",
+      "weight": 5
+    },
+    {
+      "name": "AP News",
+      "url": "https://rsshub.app/apnews/topics/world-news",
+      "weight": 5
+    },
+    {
+      "name": "NPR",
+      "url": "https://feeds.npr.org/1004/rss.xml",
+      "weight": 4
+    },
+    {
+      "name": "BBC",
+      "url": "http://feeds.bbci.co.uk/news/world/rss.xml",
+      "weight": 3
+    },
+    {
+      "name": "AlJazeera",
+      "url": "https://www.aljazeera.com/xml/rss/all.xml",
+      "weight": 2
+    },
+    {
+      "name": "NYT",
+      "url": "https://rss.nytimes.com/services/xml/rss/nyt/World.xml",
+      "weight": 1
+    },
+    {
+      "name": "GDACS",
+      "url": "https://www.gdacs.org/xml/rss.xml",
+      "weight": 5
+    },
+    {
+      "name": "The War Zone",
+      "url": "https://www.twz.com/feed",
+      "weight": 4
+    },
+    {
+      "name": "Bellingcat",
+      "url": "https://www.bellingcat.com/feed/",
+      "weight": 4
+    },
+    {
+      "name": "Guardian",
+      "url": "https://www.theguardian.com/world/rss",
+      "weight": 3
+    },
+    {
+      "name": "TASS",
+      "url": "https://tass.com/rss/v2.xml",
+      "weight": 2
+    },
+    {
+      "name": "Xinhua",
+      "url": "http://www.news.cn/english/rss/worldrss.xml",
+      "weight": 2
+    },
+    {
+      "name": "CNA",
+      "url": "https://www.channelnewsasia.com/api/v1/rss-outbound-feed?_format=xml",
+      "weight": 3
+    },
+    {
+      "name": "Mercopress",
+      "url": "https://en.mercopress.com/rss/",
+      "weight": 3
+    },
+    {
+      "name": "SCMP",
+      "url": "https://www.scmp.com/rss/91/feed",
+      "weight": 4
+    },
+    {
+      "name": "The Diplomat",
+      "url": "https://thediplomat.com/feed/",
+      "weight": 4
+    },
+    {
+      "name": "Yonhap",
+      "url": "https://en.yna.co.kr/RSS/news.xml",
+      "weight": 4
+    },
+    {
+      "name": "Asia Times",
+      "url": "https://asiatimes.com/feed/",
+      "weight": 3
+    },
+    {
+      "name": "Defense News",
+      "url": "https://www.defensenews.com/arc/outboundfeeds/rss/",
+      "weight": 3
+    },
+    {
+      "name": "Japan Times",
+      "url": "https://www.japantimes.co.jp/feed/",
+      "weight": 3
+    }
+  ]
+}

backend/data/plan_ccg_vessels.json

@@ -0,0 +1,646 @@
+{
+  "412000001": {
+    "hull_number": "101",
+    "name": "Nanchang",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000002": {
+    "hull_number": "102",
+    "name": "Lhasa",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000003": {
+    "hull_number": "103",
+    "name": "Anshan",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000004": {
+    "hull_number": "104",
+    "name": "Wuxi",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000005": {
+    "hull_number": "105",
+    "name": "Dalian",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000006": {
+    "hull_number": "106",
+    "name": "Yan'an",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000007": {
+    "hull_number": "107",
+    "name": "Zunyi",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000008": {
+    "hull_number": "108",
+    "name": "Xianyang",
+    "class": "Type 055",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_055_destroyer"
+  },
+  "412000101": {
+    "hull_number": "117",
+    "name": "Xining",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000102": {
+    "hull_number": "118",
+    "name": "Urumqi",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000103": {
+    "hull_number": "119",
+    "name": "Guiyang",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000104": {
+    "hull_number": "120",
+    "name": "Chengdu",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000105": {
+    "hull_number": "131",
+    "name": "Taiyuan",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000106": {
+    "hull_number": "132",
+    "name": "Suzhou",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000107": {
+    "hull_number": "133",
+    "name": "Nantong",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000108": {
+    "hull_number": "134",
+    "name": "Suqian",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000109": {
+    "hull_number": "135",
+    "name": "Lianyungang",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000110": {
+    "hull_number": "136",
+    "name": "Xuchang",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000111": {
+    "hull_number": "155",
+    "name": "Nanjing",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000112": {
+    "hull_number": "156",
+    "name": "Zibo",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000113": {
+    "hull_number": "157",
+    "name": "Lishui",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000114": {
+    "hull_number": "161",
+    "name": "Hohhot",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000115": {
+    "hull_number": "162",
+    "name": "Yancheng",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000116": {
+    "hull_number": "163",
+    "name": "Kaifeng",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000117": {
+    "hull_number": "164",
+    "name": "Taizhou",
+    "class": "Type 052D",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412000201": {
+    "hull_number": "538",
+    "name": "Yantai",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000202": {
+    "hull_number": "539",
+    "name": "Wuhu",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000203": {
+    "hull_number": "540",
+    "name": "Huainan",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000204": {
+    "hull_number": "541",
+    "name": "Huaihua",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000205": {
+    "hull_number": "542",
+    "name": "Zaozhuang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000206": {
+    "hull_number": "529",
+    "name": "Zhoushan",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000207": {
+    "hull_number": "530",
+    "name": "Xuzhou",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000208": {
+    "hull_number": "531",
+    "name": "Xiangtan",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000209": {
+    "hull_number": "532",
+    "name": "Jingzhou",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000210": {
+    "hull_number": "536",
+    "name": "Xuchang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000211": {
+    "hull_number": "546",
+    "name": "Yancheng",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000212": {
+    "hull_number": "547",
+    "name": "Linyi",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000213": {
+    "hull_number": "548",
+    "name": "Yiyang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000214": {
+    "hull_number": "549",
+    "name": "Changzhou",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000215": {
+    "hull_number": "550",
+    "name": "Weifang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000301": {
+    "hull_number": "31",
+    "name": "Hainan",
+    "class": "Type 075",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_075_landing_helicopter_dock"
+  },
+  "412000302": {
+    "hull_number": "32",
+    "name": "Guangxi",
+    "class": "Type 075",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_075_landing_helicopter_dock"
+  },
+  "412000303": {
+    "hull_number": "33",
+    "name": "Anhui",
+    "class": "Type 075",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_075_landing_helicopter_dock"
+  },
+  "412000401": {
+    "hull_number": "16",
+    "name": "Liaoning",
+    "class": "Type 001",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Chinese_aircraft_carrier_Liaoning"
+  },
+  "412000402": {
+    "hull_number": "17",
+    "name": "Shandong",
+    "class": "Type 002",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Chinese_aircraft_carrier_Shandong"
+  },
+  "412000403": {
+    "hull_number": "18",
+    "name": "Fujian",
+    "class": "Type 003",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Chinese_aircraft_carrier_Fujian"
+  },
+  "412000501": {
+    "hull_number": "980",
+    "name": "Hulunhu",
+    "class": "Type 901",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_901_replenishment_ship"
+  },
+  "412000502": {
+    "hull_number": "981",
+    "name": "Chaganhu",
+    "class": "Type 901",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_901_replenishment_ship"
+  },
+  "412000601": {
+    "hull_number": "998",
+    "name": "Kunlun Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000602": {
+    "hull_number": "999",
+    "name": "Jinggang Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000603": {
+    "hull_number": "989",
+    "name": "Changbai Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000604": {
+    "hull_number": "988",
+    "name": "Yimeng Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000605": {
+    "hull_number": "987",
+    "name": "Wuzhi Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000606": {
+    "hull_number": "986",
+    "name": "Longhu Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000607": {
+    "hull_number": "985",
+    "name": "Dabie Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000608": {
+    "hull_number": "984",
+    "name": "Wuyi Shan",
+    "class": "Type 071",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_071_amphibious_transport_dock"
+  },
+  "412000701": {
+    "hull_number": "815A-1",
+    "name": "Dongdiao",
+    "class": "Type 815A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_815_electronic_reconnaissance_ship"
+  },
+  "412000702": {
+    "hull_number": "815A-2",
+    "name": "Haiwangxing",
+    "class": "Type 815A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_815_electronic_reconnaissance_ship"
+  },
+  "412000703": {
+    "hull_number": "815A-3",
+    "name": "Tianwangxing",
+    "class": "Type 815A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_815_electronic_reconnaissance_ship"
+  },
+  "412009001": {
+    "hull_number": "2901",
+    "name": "CCG 2901",
+    "class": "12000-ton Cutter",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009002": {
+    "hull_number": "3901",
+    "name": "CCG 3901",
+    "class": "12000-ton Cutter",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009003": {
+    "hull_number": "1305",
+    "name": "CCG 1305",
+    "class": "Type 818",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009004": {
+    "hull_number": "1306",
+    "name": "CCG 1306",
+    "class": "Type 818",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009005": {
+    "hull_number": "2502",
+    "name": "CCG 2502",
+    "class": "5000-ton Cutter",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009006": {
+    "hull_number": "2302",
+    "name": "CCG 2302",
+    "class": "3000-ton Cutter",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009007": {
+    "hull_number": "2303",
+    "name": "CCG 2303",
+    "class": "3000-ton Cutter",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009008": {
+    "hull_number": "1103",
+    "name": "CCG 1103",
+    "class": "Type 718B",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009009": {
+    "hull_number": "1105",
+    "name": "CCG 1105",
+    "class": "Type 718B",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412009010": {
+    "hull_number": "1302",
+    "name": "CCG 1302",
+    "class": "Type 818",
+    "force": "CCG",
+    "wiki": "https://en.wikipedia.org/wiki/China_Coast_Guard"
+  },
+  "412000801": {
+    "hull_number": "171",
+    "name": "Haikou",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000802": {
+    "hull_number": "170",
+    "name": "Lanzhou",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000803": {
+    "hull_number": "150",
+    "name": "Changchun",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000804": {
+    "hull_number": "151",
+    "name": "Zhengzhou",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000805": {
+    "hull_number": "152",
+    "name": "Jinan",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000806": {
+    "hull_number": "153",
+    "name": "Xi'an",
+    "class": "Type 052C",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052C_destroyer"
+  },
+  "412000901": {
+    "hull_number": "572",
+    "name": "Hengshui",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000902": {
+    "hull_number": "573",
+    "name": "Liuzhou",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000903": {
+    "hull_number": "574",
+    "name": "Sanya",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000904": {
+    "hull_number": "575",
+    "name": "Yueyang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000905": {
+    "hull_number": "576",
+    "name": "Daqing",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412000906": {
+    "hull_number": "577",
+    "name": "Huanggang",
+    "class": "Type 054A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_054A_frigate"
+  },
+  "412001001": {
+    "hull_number": "500",
+    "name": "Xianfeng",
+    "class": "Type 056A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_056_corvette"
+  },
+  "412001002": {
+    "hull_number": "501",
+    "name": "Xinyang",
+    "class": "Type 056A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_056_corvette"
+  },
+  "412001003": {
+    "hull_number": "502",
+    "name": "Huangshi",
+    "class": "Type 056",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_056_corvette"
+  },
+  "412001004": {
+    "hull_number": "509",
+    "name": "Huaian",
+    "class": "Type 056A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_056_corvette"
+  },
+  "412001005": {
+    "hull_number": "510",
+    "name": "Ningde",
+    "class": "Type 056A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_056_corvette"
+  },
+  "412001101": {
+    "hull_number": "795",
+    "name": "Nanchong",
+    "class": "Type 039A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_039A_submarine"
+  },
+  "412001201": {
+    "hull_number": "892",
+    "name": "Hualuoshan",
+    "class": "Type 903A",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_903_replenishment_ship"
+  },
+  "412001202": {
+    "hull_number": "889",
+    "name": "Taihu",
+    "class": "Type 903",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_903_replenishment_ship"
+  },
+  "412001301": {
+    "hull_number": "636",
+    "name": "Nanning",
+    "class": "Type 052DL",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412001302": {
+    "hull_number": "165",
+    "name": "Zhanjiang",
+    "class": "Type 052DL",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  },
+  "412001303": {
+    "hull_number": "166",
+    "name": "Huainan",
+    "class": "Type 052DL",
+    "force": "PLAN",
+    "wiki": "https://en.wikipedia.org/wiki/Type_052D_destroyer"
+  }
+}

backend/data/yacht_alert_db.json

@@ -0,0 +1,122 @@
+{
+  "319225400": {
+    "name": "KORU",
+    "owner": "Jeff Bezos",
+    "builder": "Oceanco",
+    "length_m": 127,
+    "year": 2023,
+    "category": "Tech Billionaire",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Koru_(yacht)"
+  },
+  "538072122": {
+    "name": "LAUNCHPAD",
+    "owner": "Mark Zuckerberg",
+    "builder": "Feadship",
+    "length_m": 118,
+    "year": 2024,
+    "category": "Tech Billionaire",
+    "flag": "Marshall Islands",
+    "link": "https://www.superyachtfan.com/yacht/launchpad/"
+  },
+  "319032600": {
+    "name": "MUSASHI",
+    "owner": "Larry Ellison",
+    "builder": "Feadship",
+    "length_m": 88,
+    "year": 2011,
+    "category": "Tech Billionaire",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Musashi_(yacht)"
+  },
+  "319011000": {
+    "name": "RISING SUN",
+    "owner": "David Geffen",
+    "builder": "Lurssen",
+    "length_m": 138,
+    "year": 2004,
+    "category": "Celebrity / Mogul",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Rising_Sun_(yacht)"
+  },
+  "310593000": {
+    "name": "ECLIPSE",
+    "owner": "Roman Abramovich",
+    "builder": "Blohm+Voss",
+    "length_m": 162,
+    "year": 2010,
+    "category": "Oligarch Watch",
+    "flag": "Bermuda",
+    "link": "https://en.wikipedia.org/wiki/Eclipse_(yacht)"
+  },
+  "310792000": {
+    "name": "SOLARIS",
+    "owner": "Roman Abramovich",
+    "builder": "Lloyd Werft",
+    "length_m": 140,
+    "year": 2021,
+    "category": "Oligarch Watch",
+    "flag": "Bermuda",
+    "link": "https://en.wikipedia.org/wiki/Solaris_(yacht)"
+  },
+  "319094900": {
+    "name": "DILBAR",
+    "owner": "Alisher Usmanov (seized)",
+    "builder": "Lurssen",
+    "length_m": 156,
+    "year": 2016,
+    "category": "Oligarch Watch",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Dilbar_(yacht)"
+  },
+  "273610820": {
+    "name": "NORD",
+    "owner": "Alexei Mordashov",
+    "builder": "Lurssen",
+    "length_m": 142,
+    "year": 2021,
+    "category": "Oligarch Watch",
+    "flag": "Russia",
+    "link": "https://en.wikipedia.org/wiki/Nord_(yacht)"
+  },
+  "319179200": {
+    "name": "SCHEHERAZADE",
+    "owner": "Eduard Khudainatov (alleged Putin)",
+    "builder": "Lurssen",
+    "length_m": 140,
+    "year": 2020,
+    "category": "Oligarch Watch",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Scheherazade_(yacht)"
+  },
+  "319112900": {
+    "name": "AMADEA",
+    "owner": "Suleiman Kerimov (seized by US DOJ)",
+    "builder": "Lurssen",
+    "length_m": 106,
+    "year": 2017,
+    "category": "Oligarch Watch",
+    "flag": "Cayman Islands",
+    "link": "https://en.wikipedia.org/wiki/Amadea_(yacht)"
+  },
+  "319156800": {
+    "name": "BRAVO EUGENIA",
+    "owner": "Jerry Jones",
+    "builder": "Oceanco",
+    "length_m": 109,
+    "year": 2018,
+    "category": "Celebrity / Mogul",
+    "flag": "Cayman Islands",
+    "link": "https://www.superyachtfan.com/yacht/bravo-eugenia/"
+  },
+  "319137200": {
+    "name": "LADY S",
+    "owner": "Dan Snyder",
+    "builder": "Feadship",
+    "length_m": 93,
+    "year": 2019,
+    "category": "Celebrity / Mogul",
+    "flag": "Cayman Islands",
+    "link": "https://www.superyachtfan.com/yacht/lady-s/"
+  }
+}

backend/debug_fast.json.REMOVED.git-id

@@ -1 +0,0 @@
-5c3b1c768973ca54e9a1befee8dc075f38e8cc56

backend/dump.json.REMOVED.git-id

@@ -1 +0,0 @@
-2b64633521ffb6f06da36e19f5c8eb86979e2187

backend/extract_ovens.py

@@ -1,25 +0,0 @@
-import re
-import json
-
-try:
-    with open('liveua_test.html', 'r', encoding='utf-8') as f:
-        html = f.read()
-        
-    m = re.search(r"var\s+ovens\s*=\s*(.*?);(?!function)", html, re.DOTALL)
-    if m:
-        json_str = m.group(1)
-        # Handle if it is a string containing base64
-        if json_str.startswith("'") or json_str.startswith('"'):
-            json_str = json_str.strip('"\'')
-            import base64
-            import urllib.parse
-            json_str = base64.b64decode(urllib.parse.unquote(json_str)).decode('utf-8')
-            
-        data = json.loads(json_str)
-        with open('out_liveua.json', 'w', encoding='utf-8') as f:
-            json.dump(data, f, indent=2)
-        print(f"Successfully extracted {len(data)} ovens items.")
-    else:
-        print("var ovens not found.")
-except Exception as e:
-    print("Error:", e)

backend/out.json

@@ -1 +0,0 @@
-{"callsign": "JWZ7", "country": "N625GN", "lng": -111.914754, "lat": 33.620235, "alt": 0, "heading": 0, "type": "tracked_flight", "origin_loc": null, "dest_loc": null, "origin_name": "UNKNOWN", "dest_name": "UNKNOWN", "registration": "N625GN", "model": "GLF5", "icao24": "a82973", "speed_knots": 6.8, "squawk": "1200", "airline_code": "", "aircraft_category": "plane", "alert_operator": "Tilman Fertitta", "alert_category": "People", "alert_color": "pink", "trail": [[33.62024, -111.91475, 0, 1772302052]]}

backend/pyproject.toml

@@ -0,0 +1,34 @@
+[project]
+name = "backend"
+version = "0.9.6"
+requires-python = ">=3.10"
+dependencies = [
+    "apscheduler==3.10.3",
+    "beautifulsoup4>=4.9.0",
+    "cachetools==5.5.2",
+    "cloudscraper==1.2.71",
+    "cryptography>=41.0.0",
+    "fastapi==0.115.12",
+    "feedparser==6.0.10",
+    "httpx==0.28.1",
+    "playwright==1.50.0",
+    "playwright-stealth==1.0.6",
+    "pydantic==2.11.1",
+    "pydantic-settings==2.8.1",
+    "pystac-client==0.8.6",
+    "python-dotenv==1.2.2",
+    "requests==2.31.0",
+    "reverse-geocoder==1.5.1",
+    "sgp4==2.23",
+    "meshtastic>=2.5.0",
+    "orjson>=3.10.0",
+    "paho-mqtt>=1.6.0,<2.0.0",
+    "PyNaCl>=1.5.0",
+    "slowapi==0.1.9",
+    "vaderSentiment>=3.3.0",
+    "uvicorn==0.34.0",
+    "yfinance==0.2.54",
+]
+
+[dependency-groups]
+dev = ["pytest>=8.3.4", "pytest-asyncio==0.25.0", "ruff>=0.9.0", "black>=24.0.0"]

backend/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+testpaths = tests
+python_files = test_*.py
+python_functions = test_*
+asyncio_default_fixture_loop_scope = function

backend/requirements.txt

@@ -1,20 +0,0 @@
-fastapi>=0.103.1
-uvicorn>=0.23.2
-yfinance>=0.2.40
-feedparser==6.0.10
-legacy-cgi>=2.6
-requests==2.31.0
-apscheduler==3.10.3
-pydantic>=2.3.0
-pydantic-settings>=2.0.3
-playwright>=1.58.0
-beautifulsoup4>=4.12.0
-cachetools>=5.3
-cloudscraper>=1.2.71
-python-dotenv>=1.0
-lxml>=5.0
-reverse_geocoder>=1.5
-sgp4>=2.23
-geopy>=2.4.0
-pytz>=2023.3
-pystac-client>=0.7.0

backend/scripts/bootstrap_manifest_helper.py

@@ -0,0 +1,115 @@
+import argparse
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+BACKEND_DIR = ROOT / "backend"
+
+if str(BACKEND_DIR) not in sys.path:
+    sys.path.insert(0, str(BACKEND_DIR))
+
+from services.mesh.mesh_bootstrap_manifest import (  # noqa: E402
+    bootstrap_signer_public_key_b64,
+    generate_bootstrap_signer,
+    write_signed_bootstrap_manifest,
+)
+
+
+def _load_peers(args: argparse.Namespace) -> list[dict]:
+    peers: list[dict] = []
+    if args.peers_file:
+        raw = json.loads(Path(args.peers_file).read_text(encoding="utf-8"))
+        if not isinstance(raw, list):
+            raise ValueError("peers file must be a JSON array")
+        for entry in raw:
+            if not isinstance(entry, dict):
+                raise ValueError("peers file entries must be objects")
+            peers.append(dict(entry))
+    for peer_arg in args.peer or []:
+        parts = [part.strip() for part in str(peer_arg).split(",", 3)]
+        if len(parts) < 3:
+            raise ValueError("peer entries must look like url,transport,role[,label]")
+        peer_url, transport, role = parts[:3]
+        label = parts[3] if len(parts) > 3 else ""
+        peers.append(
+            {
+                "peer_url": peer_url,
+                "transport": transport,
+                "role": role,
+                "label": label,
+            }
+        )
+    if not peers:
+        raise ValueError("at least one peer is required")
+    return peers
+
+
+def cmd_generate_keypair(_args: argparse.Namespace) -> int:
+    signer = generate_bootstrap_signer()
+    print(json.dumps(signer, indent=2))
+    return 0
+
+
+def cmd_sign(args: argparse.Namespace) -> int:
+    peers = _load_peers(args)
+    manifest = write_signed_bootstrap_manifest(
+        args.output,
+        signer_id=args.signer_id,
+        signer_private_key_b64=args.private_key_b64,
+        peers=peers,
+        valid_for_hours=int(args.valid_hours),
+    )
+    print(f"Wrote signed bootstrap manifest to {Path(args.output).resolve()}")
+    print(f"signer_id={manifest.signer_id}")
+    print(f"valid_until={manifest.valid_until}")
+    print(f"peer_count={len(manifest.peers)}")
+    print(f"MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY={bootstrap_signer_public_key_b64(args.private_key_b64)}")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Generate and sign Infonet bootstrap manifests for participant nodes."
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    keygen = subparsers.add_parser("generate-keypair", help="Generate an Ed25519 bootstrap signer keypair")
+    keygen.set_defaults(func=cmd_generate_keypair)
+
+    sign = subparsers.add_parser("sign", help="Sign a bootstrap manifest from peer entries")
+    sign.add_argument("--output", required=True, help="Output path for bootstrap_peers.json")
+    sign.add_argument("--signer-id", required=True, help="Manifest signer identifier")
+    sign.add_argument(
+        "--private-key-b64",
+        required=True,
+        help="Raw Ed25519 private key in base64 returned by generate-keypair",
+    )
+    sign.add_argument(
+        "--peers-file",
+        help="JSON file containing an array of peer objects with peer_url, transport, role, and optional label",
+    )
+    sign.add_argument(
+        "--peer",
+        action="append",
+        help="Inline peer in the form url,transport,role[,label]. May be repeated.",
+    )
+    sign.add_argument(
+        "--valid-hours",
+        type=int,
+        default=168,
+        help="Manifest validity window in hours (default: 168)",
+    )
+    sign.set_defaults(func=cmd_sign)
+
+    return parser
+
+
+def main() -> int:
+    parser = build_parser()
+    args = parser.parse_args()
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/scripts/check-env.ps1

@@ -0,0 +1,5 @@
+param(
+  [string]$Python = "python"
+)
+
+& $Python -c "from services.env_check import validate_env; validate_env(strict=False)"

backend/scripts/check-env.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+PYTHON="${PYTHON:-python3}"
+"$PYTHON" -c "from services.env_check import validate_env; validate_env(strict=False)"

backend/scripts/convert_power_plants.py

@@ -0,0 +1,58 @@
+"""Download WRI Global Power Plant Database CSV and convert to compact JSON.
+
+Usage:
+    python backend/scripts/convert_power_plants.py
+
+Output:
+    backend/data/power_plants.json
+"""
+import csv
+import json
+import io
+import zipfile
+import urllib.request
+from pathlib import Path
+
+# WRI Global Power Plant Database v1.3.0 (GitHub release)
+CSV_URL = "https://raw.githubusercontent.com/wri/global-power-plant-database/master/output_database/global_power_plant_database.csv"
+OUT_PATH = Path(__file__).parent.parent / "data" / "power_plants.json"
+
+
+def main() -> None:
+    print(f"Downloading WRI Global Power Plant Database from GitHub...")
+    req = urllib.request.Request(CSV_URL, headers={"User-Agent": "ShadowBroker-OSINT/1.0"})
+    with urllib.request.urlopen(req, timeout=60) as resp:
+        raw = resp.read().decode("utf-8")
+
+    reader = csv.DictReader(io.StringIO(raw))
+    plants: list[dict] = []
+    skipped = 0
+    for row in reader:
+        try:
+            lat = float(row["latitude"])
+            lng = float(row["longitude"])
+        except (ValueError, KeyError):
+            skipped += 1
+            continue
+        if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+            skipped += 1
+            continue
+        capacity_raw = row.get("capacity_mw", "")
+        capacity_mw = float(capacity_raw) if capacity_raw else None
+        plants.append({
+            "name": row.get("name", "Unknown"),
+            "country": row.get("country_long", ""),
+            "fuel_type": row.get("primary_fuel", "Unknown"),
+            "capacity_mw": capacity_mw,
+            "owner": row.get("owner", ""),
+            "lat": round(lat, 5),
+            "lng": round(lng, 5),
+        })
+
+    OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
+    OUT_PATH.write_text(json.dumps(plants, ensure_ascii=False, separators=(",", ":")), encoding="utf-8")
+    print(f"Wrote {len(plants)} power plants to {OUT_PATH} (skipped {skipped})")
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/diagnostics.py

@@ -0,0 +1,45 @@
+from datetime import datetime
+from services.data_fetcher import get_latest_data
+from services.fetchers._store import source_timestamps, active_layers, source_freshness
+from services.fetch_health import get_health_snapshot
+
+
+def _fmt_ts(ts: str | None) -> str:
+    if not ts:
+        return "-"
+    try:
+        return datetime.fromisoformat(ts).strftime("%Y-%m-%d %H:%M:%S")
+    except Exception:
+        return ts
+
+
+def main():
+    data = get_latest_data()
+    print("=== Diagnostics ===")
+    print(f"Last updated: {_fmt_ts(data.get('last_updated'))}")
+    print(
+        f"Active layers: {sum(1 for v in active_layers.values() if v)} enabled / {len(active_layers)} total"
+    )
+
+    print("\n--- Source Timestamps ---")
+    for k, v in sorted(source_timestamps.items()):
+        print(f"{k:20} {_fmt_ts(v)}")
+
+    print("\n--- Source Freshness ---")
+    for k, v in sorted(source_freshness.items()):
+        last_ok = _fmt_ts(v.get("last_ok"))
+        last_err = _fmt_ts(v.get("last_error"))
+        print(f"{k:20} ok={last_ok} err={last_err}")
+
+    print("\n--- Fetch Health ---")
+    health = get_health_snapshot()
+    for k, v in sorted(health.items()):
+        print(
+            f"{k:20} ok={v.get('ok_count', 0)} err={v.get('error_count', 0)} "
+            f"last_ok={_fmt_ts(v.get('last_ok'))} last_err={_fmt_ts(v.get('last_error'))} "
+            f"avg_ms={v.get('avg_duration_ms')}"
+        )
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/release_helper.py

@@ -0,0 +1,138 @@
+import argparse
+import hashlib
+import json
+import sys
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[2]
+PACKAGE_JSON = ROOT / "frontend" / "package.json"
+
+
+def _normalize_version(raw: str) -> str:
+    version = str(raw or "").strip()
+    if version.startswith("v"):
+        version = version[1:]
+    parts = version.split(".")
+    if len(parts) != 3 or not all(part.isdigit() for part in parts):
+        raise ValueError("Version must look like X.Y.Z")
+    return version
+
+
+def _read_package_json() -> dict:
+    return json.loads(PACKAGE_JSON.read_text(encoding="utf-8"))
+
+
+def _write_package_json(data: dict) -> None:
+    PACKAGE_JSON.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
+
+
+def current_version() -> str:
+    return str(_read_package_json().get("version") or "").strip()
+
+
+def set_version(version: str) -> str:
+    normalized = _normalize_version(version)
+    data = _read_package_json()
+    data["version"] = normalized
+    _write_package_json(data)
+    return normalized
+
+
+def expected_tag(version: str) -> str:
+    return f"v{_normalize_version(version)}"
+
+
+def expected_asset(version: str) -> str:
+    normalized = _normalize_version(version)
+    return f"ShadowBroker_v{normalized}.zip"
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 128), b""):
+            digest.update(chunk)
+    return digest.hexdigest().lower()
+
+
+def cmd_show(_args: argparse.Namespace) -> int:
+    version = current_version()
+    if not version:
+        print("package.json has no version", file=sys.stderr)
+        return 1
+    print(f"package.json version : {version}")
+    print(f"expected git tag     : {expected_tag(version)}")
+    print(f"expected zip asset   : {expected_asset(version)}")
+    return 0
+
+
+def cmd_set_version(args: argparse.Namespace) -> int:
+    version = set_version(args.version)
+    print(f"Set frontend/package.json version to {version}")
+    print(f"Next release tag  : {expected_tag(version)}")
+    print(f"Next zip asset    : {expected_asset(version)}")
+    return 0
+
+
+def cmd_hash(args: argparse.Namespace) -> int:
+    version = _normalize_version(args.version) if args.version else current_version()
+    if not version:
+        print("No version available; pass --version or set frontend/package.json", file=sys.stderr)
+        return 1
+
+    zip_path = Path(args.zip_path).resolve()
+    if not zip_path.is_file():
+        print(f"ZIP not found: {zip_path}", file=sys.stderr)
+        return 1
+
+    digest = sha256_file(zip_path)
+    expected_name = expected_asset(version)
+    asset_matches = zip_path.name == expected_name
+
+    print(f"release version     : {version}")
+    print(f"expected git tag    : {expected_tag(version)}")
+    print(f"zip path            : {zip_path}")
+    print(f"zip name matches    : {'yes' if asset_matches else 'no'}")
+    print(f"expected zip asset  : {expected_name}")
+    print(f"SHA-256             : {digest}")
+    print("")
+    print("Updater pin:")
+    print(f"MESH_UPDATE_SHA256={digest}")
+    return 0 if asset_matches else 2
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Helper for ShadowBroker release version/tag/asset consistency."
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    show_parser = subparsers.add_parser("show", help="Show current version, expected tag, and asset")
+    show_parser.set_defaults(func=cmd_show)
+
+    set_version_parser = subparsers.add_parser("set-version", help="Update frontend/package.json version")
+    set_version_parser.add_argument("version", help="Version like 0.9.6")
+    set_version_parser.set_defaults(func=cmd_set_version)
+
+    hash_parser = subparsers.add_parser(
+        "hash", help="Compute SHA-256 for a release ZIP and print the updater pin"
+    )
+    hash_parser.add_argument("zip_path", help="Path to the release ZIP")
+    hash_parser.add_argument(
+        "--version",
+        help="Release version like 0.9.6. Defaults to frontend/package.json version.",
+    )
+    hash_parser.set_defaults(func=cmd_hash)
+
+    return parser
+
+
+def main() -> int:
+    parser = build_parser()
+    args = parser.parse_args()
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/scripts/repair_wormhole_secure_storage.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from services.mesh import mesh_secure_storage
+from services.mesh.mesh_wormhole_contacts import CONTACTS_FILE
+from services.mesh.mesh_wormhole_identity import IDENTITY_FILE, _default_identity
+from services.mesh.mesh_wormhole_persona import PERSONA_FILE, _default_state as _default_persona_state
+from services.mesh.mesh_wormhole_ratchet import STATE_FILE as RATCHET_FILE
+
+
+def _load_payloads() -> dict[Path, object]:
+    return {
+        IDENTITY_FILE: mesh_secure_storage.read_secure_json(IDENTITY_FILE, _default_identity),
+        PERSONA_FILE: mesh_secure_storage.read_secure_json(PERSONA_FILE, _default_persona_state),
+        RATCHET_FILE: mesh_secure_storage.read_secure_json(RATCHET_FILE, lambda: {}),
+        CONTACTS_FILE: mesh_secure_storage.read_secure_json(CONTACTS_FILE, lambda: {}),
+    }
+
+
+def main() -> None:
+    payloads = _load_payloads()
+
+    master_key_file = mesh_secure_storage.MASTER_KEY_FILE
+    backup_key_file = master_key_file.with_suffix(master_key_file.suffix + ".bak")
+    if master_key_file.exists():
+        if backup_key_file.exists():
+            backup_key_file.unlink()
+        master_key_file.replace(backup_key_file)
+
+    for path, payload in payloads.items():
+        mesh_secure_storage.write_secure_json(path, payload)
+
+    print(
+        json.dumps(
+            {
+                "ok": True,
+                "rewrapped": [str(path.name) for path in payloads.keys()],
+                "master_key": str(master_key_file),
+                "backup_master_key": str(backup_key_file) if backup_key_file.exists() else "",
+            }
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

backend/scripts/scan-secrets.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+# scan-secrets.sh — Catch keys, secrets, and credentials before they hit git.
+#
+# Usage:
+#   ./backend/scripts/scan-secrets.sh          # Scan staged files (pre-commit)
+#   ./backend/scripts/scan-secrets.sh --all     # Scan entire working tree
+#   ./backend/scripts/scan-secrets.sh --staged  # Scan staged files only (default)
+#
+# Exit code: 0 = clean, 1 = secrets found
+
+set -euo pipefail
+
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+GREEN='\033[0;32m'
+NC='\033[0m'
+
+MODE="${1:---staged}"
+FOUND=0
+
+# ── Get file list based on mode ─────────────────────────────────────────
+if [[ "$MODE" == "--all" ]]; then
+    FILELIST=$(mktemp)
+    { git ls-files 2>/dev/null; git ls-files --others --exclude-standard 2>/dev/null; } > "$FILELIST"
+    echo -e "${YELLOW}Scanning entire working tree...${NC}"
+else
+    FILELIST=$(mktemp)
+    git diff --cached --name-only --diff-filter=ACMR 2>/dev/null > "$FILELIST" || true
+    if [[ ! -s "$FILELIST" ]]; then
+        echo -e "${GREEN}No staged files to scan.${NC}"
+        rm -f "$FILELIST"
+        exit 0
+    fi
+    echo -e "${YELLOW}Scanning $(wc -l < "$FILELIST" | tr -d ' ') staged files...${NC}"
+fi
+
+# ── Check 1: Dangerous file extensions ──────────────────────────────────
+KEY_EXT='\.key$|\.pem$|\.p12$|\.pfx$|\.jks$|\.keystore$|\.p8$|\.der$'
+SECRET_EXT='\.secret$|\.secrets$|\.credential$|\.credentials$'
+
+HITS=$(grep -iE "$KEY_EXT|$SECRET_EXT" "$FILELIST" 2>/dev/null || true)
+if [[ -n "$HITS" ]]; then
+    echo -e "\n${RED}BLOCKED: Key/secret files detected:${NC}"
+    echo "$HITS" | while read -r f; do echo -e "  ${RED}$f${NC}"; done
+    FOUND=1
+fi
+
+# ── Check 2: Dangerous filenames ────────────────────────────────────────
+RISKY='id_rsa|id_ed25519|id_ecdsa|private_key|private\.key|secret_key|master\.key'
+RISKY+='|serviceaccount|gcloud.*\.json|firebase.*\.json|\.htpasswd'
+
+HITS=$(grep -iE "$RISKY" "$FILELIST" 2>/dev/null || true)
+if [[ -n "$HITS" ]]; then
+    echo -e "\n${RED}BLOCKED: Risky filenames detected:${NC}"
+    echo "$HITS" | while read -r f; do echo -e "  ${RED}$f${NC}"; done
+    FOUND=1
+fi
+
+# ── Check 3: .env files (not .env.example) ──────────────────────────────
+HITS=$(grep -E '(^|/)\.env(\.[^e].*)?$' "$FILELIST" 2>/dev/null | grep -v '\.example' || true)
+if [[ -n "$HITS" ]]; then
+    echo -e "\n${RED}BLOCKED: Environment files detected:${NC}"
+    echo "$HITS" | while read -r f; do echo -e "  ${RED}$f${NC}"; done
+    FOUND=1
+fi
+
+# ── Check 4: _domain_keys directory (project-specific) ──────────────────
+HITS=$(grep '_domain_keys/' "$FILELIST" 2>/dev/null || true)
+if [[ -n "$HITS" ]]; then
+    echo -e "\n${RED}BLOCKED: Domain keys directory detected:${NC}"
+    echo "$HITS" | while read -r f; do echo -e "  ${RED}$f${NC}"; done
+    FOUND=1
+fi
+
+# ── Check 5: Content scan for embedded secrets (single grep pass) ───────
+# Build one mega-pattern and run grep once across all files (fast!)
+SECRET_REGEX='PRIVATE KEY-----|'
+SECRET_REGEX+='ssh-rsa AAAA[0-9A-Za-z+/]|'
+SECRET_REGEX+='ssh-ed25519 AAAA[0-9A-Za-z+/]|'
+SECRET_REGEX+='ghp_[0-9a-zA-Z]{36}|'                          # GitHub PAT
+SECRET_REGEX+='github_pat_[0-9a-zA-Z]{22}_[0-9a-zA-Z]{59}|'   # GitHub fine-grained
+SECRET_REGEX+='gho_[0-9a-zA-Z]{36}|'                          # GitHub OAuth
+SECRET_REGEX+='sk-[0-9a-zA-Z]{48}|'                           # OpenAI key
+SECRET_REGEX+='sk-ant-[0-9a-zA-Z-]{90,}|'                     # Anthropic key
+SECRET_REGEX+='AKIA[0-9A-Z]{16}|'                             # AWS access key
+SECRET_REGEX+='AIzaSy[0-9A-Za-z_-]{33}|'                      # Google API key
+SECRET_REGEX+='xox[bpoas]-[0-9a-zA-Z-]+|'                     # Slack token
+SECRET_REGEX+='npm_[0-9a-zA-Z]{36}|'                          # npm token
+SECRET_REGEX+='pypi-[0-9a-zA-Z-]{50,}'                        # PyPI token
+
+# Filter to text-like files only (skip binaries by extension + skip this script)
+TEXT_FILES=$(grep -ivE '\.(png|jpg|jpeg|gif|ico|svg|woff2?|ttf|eot|pbf|zip|tar|gz|db|sqlite|xlsx|pdf|mp[34]|wav|ogg|webm|webp|avif)$' "$FILELIST" | grep -v 'scan-secrets\.sh$' || true)
+
+if [[ -n "$TEXT_FILES" ]]; then
+    # Use grep with file list, skip missing/binary, limit output
+    CONTENT_HITS=$(echo "$TEXT_FILES" | xargs grep -lE "$SECRET_REGEX" 2>/dev/null || true)
+    if [[ -n "$CONTENT_HITS" ]]; then
+        echo -e "\n${RED}BLOCKED: Embedded secrets/tokens found in:${NC}"
+        echo "$CONTENT_HITS" | while read -r f; do
+            echo -e "  ${RED}$f${NC}"
+            # Show first matching line for context
+            grep -nE "$SECRET_REGEX" "$f" 2>/dev/null | head -2 | while read -r line; do
+                echo -e "    ${YELLOW}$line${NC}"
+            done
+        done
+        FOUND=1
+    fi
+fi
+
+rm -f "$FILELIST"
+
+# ── Result ──────────────────────────────────────────────────────────────
+echo ""
+if [[ $FOUND -eq 1 ]]; then
+    echo -e "${RED}Secret scan FAILED. Add these to .gitignore or remove them before committing.${NC}"
+    echo -e "${YELLOW}If intentional (e.g. test fixtures): git commit --no-verify${NC}"
+    exit 1
+else
+    echo -e "${GREEN}Secret scan passed. No keys or secrets detected.${NC}"
+    exit 0
+fi

backend/scripts/setup-venv.ps1

@@ -0,0 +1,10 @@
+param(
+  [string]$Python = "python"
+)
+
+$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
+$venvPath = Join-Path $repoRoot "venv"
+& $Python -m venv $venvPath
+
+$pip = Join-Path $venvPath "Scripts\pip.exe"
+& $pip install -r (Join-Path $repoRoot "requirements-dev.txt")

backend/scripts/setup-venv.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+PYTHON="${PYTHON:-python3}"
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+VENV_DIR="$REPO_ROOT/venv"
+
+"$PYTHON" -m venv "$VENV_DIR"
+"$VENV_DIR/bin/pip" install -r "$REPO_ROOT/requirements-dev.txt"

backend/seattle_sample.json

@@ -1,8 +0,0 @@
-{
-  "code" : "dataset.missing",
-  "error" : true,
-  "message" : "Not found",
-  "data" : {
-    "id" : "xqwu-hwdm"
-  }
-}

backend/services/ais_stream.py

@@ -16,18 +16,19 @@ logger = logging.getLogger(__name__)
 AIS_WS_URL = "wss://stream.aisstream.io/v0/stream"
 API_KEY = os.environ.get("AIS_API_KEY", "")
 
+
 # AIS vessel type code classification
 # See: https://coast.noaa.gov/data/marinecadastre/ais/VesselTypeCodes2018.pdf
 def classify_vessel(ais_type: int, mmsi: int) -> str:
     """Classify a vessel by its AIS type code into a rendering category."""
     if 80 <= ais_type <= 89:
-        return "tanker"        # Oil/Chemical/Gas tankers → RED
+        return "tanker"  # Oil/Chemical/Gas tankers → RED
     if 70 <= ais_type <= 79:
-        return "cargo"         # Cargo ships, container vessels → RED
+        return "cargo"  # Cargo ships, container vessels → RED
     if 60 <= ais_type <= 69:
-        return "passenger"     # Cruise ships, ferries → GRAY
+        return "passenger"  # Cruise ships, ferries → GRAY
     if ais_type in (36, 37):
-        return "yacht"         # Sailing/Pleasure craft → DARK BLUE
+        return "yacht"  # Sailing/Pleasure craft → DARK BLUE
     if ais_type == 35:
         return "military_vessel"  # Military → YELLOW
     # MMSI-based military detection: military MMSIs often start with certain prefixes
@@ -35,87 +36,286 @@ def classify_vessel(ais_type: int, mmsi: int) -> str:
     if mmsi_str.startswith("3380") or mmsi_str.startswith("3381"):
         return "military_vessel"  # US Navy
     if ais_type in (30, 31, 32, 33, 34):
-        return "other"         # Fishing, towing, dredging, diving, etc.
+        return "other"  # Fishing, towing, dredging, diving, etc.
     if ais_type in (50, 51, 52, 53, 54, 55, 56, 57, 58, 59):
-        return "other"         # Pilot, SAR, tug, port tender, etc.
-    return "unknown"            # Not yet classified — will update when ShipStaticData arrives
+        return "other"  # Pilot, SAR, tug, port tender, etc.
+    return "unknown"  # Not yet classified — will update when ShipStaticData arrives
 
 
 # MMSI Maritime Identification Digit (MID) → Country mapping
 # First 3 digits of MMSI (for 9-digit MMSIs) encode the flag state
 MID_COUNTRY = {
-    201: "Albania", 202: "Andorra", 203: "Austria", 204: "Portugal", 205: "Belgium",
-    206: "Belarus", 207: "Bulgaria", 208: "Vatican", 209: "Cyprus", 210: "Cyprus",
-    211: "Germany", 212: "Cyprus", 213: "Georgia", 214: "Moldova", 215: "Malta",
-    216: "Armenia", 218: "Germany", 219: "Denmark", 220: "Denmark", 224: "Spain",
-    225: "Spain", 226: "France", 227: "France", 228: "France", 229: "Malta",
-    230: "Finland", 231: "Faroe Islands", 232: "United Kingdom", 233: "United Kingdom",
-    234: "United Kingdom", 235: "United Kingdom", 236: "Gibraltar", 237: "Greece",
-    238: "Croatia", 239: "Greece", 240: "Greece", 241: "Greece", 242: "Morocco",
-    243: "Hungary", 244: "Netherlands", 245: "Netherlands", 246: "Netherlands",
-    247: "Italy", 248: "Malta", 249: "Malta", 250: "Ireland", 251: "Iceland",
-    252: "Liechtenstein", 253: "Luxembourg", 254: "Monaco", 255: "Portugal",
-    256: "Malta", 257: "Norway", 258: "Norway", 259: "Norway", 261: "Poland",
-    263: "Portugal", 264: "Romania", 265: "Sweden", 266: "Sweden", 267: "Slovakia",
-    268: "San Marino", 269: "Switzerland", 270: "Czech Republic", 271: "Turkey",
-    272: "Ukraine", 273: "Russia", 274: "North Macedonia", 275: "Latvia",
-    276: "Estonia", 277: "Lithuania", 278: "Slovenia",
-    301: "Anguilla", 303: "Alaska", 304: "Antigua", 305: "Antigua",
-    306: "Netherlands Antilles", 307: "Aruba", 308: "Bahamas", 309: "Bahamas",
-    310: "Bermuda", 311: "Bahamas", 312: "Belize", 314: "Barbados", 316: "Canada",
-    319: "Cayman Islands", 321: "Costa Rica", 323: "Cuba", 325: "Dominica",
-    327: "Dominican Republic", 329: "Guadeloupe", 330: "Grenada", 331: "Greenland",
-    332: "Guatemala", 334: "Honduras", 336: "Haiti", 338: "United States",
-    339: "Jamaica", 341: "Saint Kitts", 343: "Saint Lucia", 345: "Mexico",
-    347: "Martinique", 348: "Montserrat", 350: "Nicaragua", 351: "Panama",
-    352: "Panama", 353: "Panama", 354: "Panama", 355: "Panama",
-    356: "Panama", 357: "Panama", 358: "Puerto Rico", 359: "El Salvador",
-    361: "Saint Pierre", 362: "Trinidad", 364: "Turks and Caicos",
-    366: "United States", 367: "United States", 368: "United States", 369: "United States",
-    370: "Panama", 371: "Panama", 372: "Panama", 373: "Panama",
-    374: "Panama", 375: "Saint Vincent", 376: "Saint Vincent", 377: "Saint Vincent",
-    378: "British Virgin Islands", 379: "US Virgin Islands",
-    401: "Afghanistan", 403: "Saudi Arabia", 405: "Bangladesh", 408: "Bahrain",
-    410: "Bhutan", 412: "China", 413: "China", 414: "China",
-    416: "Taiwan", 417: "Sri Lanka", 419: "India", 422: "Iran",
-    423: "Azerbaijan", 425: "Iraq", 428: "Israel", 431: "Japan",
-    432: "Japan", 434: "Turkmenistan", 436: "Kazakhstan", 437: "Uzbekistan",
-    438: "Jordan", 440: "South Korea", 441: "South Korea", 443: "Palestine",
-    445: "North Korea", 447: "Kuwait", 450: "Lebanon", 451: "Kyrgyzstan",
-    453: "Macao", 455: "Maldives", 457: "Mongolia", 459: "Nepal",
-    461: "Oman", 463: "Pakistan", 466: "Qatar", 468: "Syria",
-    470: "UAE", 472: "Tajikistan", 473: "Yemen", 475: "Tonga",
-    477: "Hong Kong", 478: "Bosnia",
-    501: "Antarctica", 503: "Australia", 506: "Myanmar",
-    508: "Brunei", 510: "Micronesia", 511: "Palau", 512: "New Zealand",
-    514: "Cambodia", 515: "Cambodia", 516: "Christmas Island",
-    518: "Cook Islands", 520: "Fiji", 523: "Cocos Islands",
-    525: "Indonesia", 529: "Kiribati", 531: "Laos", 533: "Malaysia",
-    536: "Northern Mariana Islands", 538: "Marshall Islands",
-    540: "New Caledonia", 542: "Niue", 544: "Nauru", 546: "French Polynesia",
-    548: "Philippines", 553: "Papua New Guinea", 555: "Pitcairn",
-    557: "Solomon Islands", 559: "American Samoa", 561: "Samoa",
-    563: "Singapore", 564: "Singapore", 565: "Singapore", 566: "Singapore",
-    567: "Thailand", 570: "Tonga", 572: "Tuvalu", 574: "Vietnam",
-    576: "Vanuatu", 577: "Vanuatu", 578: "Wallis and Futuna",
-    601: "South Africa", 603: "Angola", 605: "Algeria", 607: "Benin",
-    609: "Botswana", 610: "Burundi", 611: "Cameroon", 612: "Cape Verde",
-    613: "Central African Republic", 615: "Congo", 616: "Comoros",
-    617: "DR Congo", 618: "Ivory Coast", 619: "Djibouti",
-    620: "Egypt", 621: "Equatorial Guinea", 622: "Ethiopia",
-    624: "Eritrea", 625: "Gabon", 626: "Gambia", 627: "Ghana",
-    629: "Guinea", 630: "Guinea-Bissau", 631: "Kenya", 632: "Lesotho",
-    633: "Liberia", 634: "Liberia", 635: "Liberia", 636: "Liberia",
-    637: "Libya", 642: "Madagascar", 644: "Malawi", 645: "Mali",
-    647: "Mauritania", 649: "Mauritius", 650: "Mozambique",
-    654: "Namibia", 655: "Niger", 656: "Nigeria", 657: "Guinea",
-    659: "Rwanda", 660: "Senegal", 661: "Sierra Leone",
-    662: "Somalia", 663: "South Africa", 664: "Sudan",
-    667: "Tanzania", 668: "Togo", 669: "Tunisia", 670: "Uganda",
-    671: "Egypt", 672: "Tanzania", 674: "Zambia", 675: "Zimbabwe",
-    676: "Comoros", 677: "Tanzania",
+    201: "Albania",
+    202: "Andorra",
+    203: "Austria",
+    204: "Portugal",
+    205: "Belgium",
+    206: "Belarus",
+    207: "Bulgaria",
+    208: "Vatican",
+    209: "Cyprus",
+    210: "Cyprus",
+    211: "Germany",
+    212: "Cyprus",
+    213: "Georgia",
+    214: "Moldova",
+    215: "Malta",
+    216: "Armenia",
+    218: "Germany",
+    219: "Denmark",
+    220: "Denmark",
+    224: "Spain",
+    225: "Spain",
+    226: "France",
+    227: "France",
+    228: "France",
+    229: "Malta",
+    230: "Finland",
+    231: "Faroe Islands",
+    232: "United Kingdom",
+    233: "United Kingdom",
+    234: "United Kingdom",
+    235: "United Kingdom",
+    236: "Gibraltar",
+    237: "Greece",
+    238: "Croatia",
+    239: "Greece",
+    240: "Greece",
+    241: "Greece",
+    242: "Morocco",
+    243: "Hungary",
+    244: "Netherlands",
+    245: "Netherlands",
+    246: "Netherlands",
+    247: "Italy",
+    248: "Malta",
+    249: "Malta",
+    250: "Ireland",
+    251: "Iceland",
+    252: "Liechtenstein",
+    253: "Luxembourg",
+    254: "Monaco",
+    255: "Portugal",
+    256: "Malta",
+    257: "Norway",
+    258: "Norway",
+    259: "Norway",
+    261: "Poland",
+    263: "Portugal",
+    264: "Romania",
+    265: "Sweden",
+    266: "Sweden",
+    267: "Slovakia",
+    268: "San Marino",
+    269: "Switzerland",
+    270: "Czech Republic",
+    271: "Turkey",
+    272: "Ukraine",
+    273: "Russia",
+    274: "North Macedonia",
+    275: "Latvia",
+    276: "Estonia",
+    277: "Lithuania",
+    278: "Slovenia",
+    301: "Anguilla",
+    303: "Alaska",
+    304: "Antigua",
+    305: "Antigua",
+    306: "Netherlands Antilles",
+    307: "Aruba",
+    308: "Bahamas",
+    309: "Bahamas",
+    310: "Bermuda",
+    311: "Bahamas",
+    312: "Belize",
+    314: "Barbados",
+    316: "Canada",
+    319: "Cayman Islands",
+    321: "Costa Rica",
+    323: "Cuba",
+    325: "Dominica",
+    327: "Dominican Republic",
+    329: "Guadeloupe",
+    330: "Grenada",
+    331: "Greenland",
+    332: "Guatemala",
+    334: "Honduras",
+    336: "Haiti",
+    338: "United States",
+    339: "Jamaica",
+    341: "Saint Kitts",
+    343: "Saint Lucia",
+    345: "Mexico",
+    347: "Martinique",
+    348: "Montserrat",
+    350: "Nicaragua",
+    351: "Panama",
+    352: "Panama",
+    353: "Panama",
+    354: "Panama",
+    355: "Panama",
+    356: "Panama",
+    357: "Panama",
+    358: "Puerto Rico",
+    359: "El Salvador",
+    361: "Saint Pierre",
+    362: "Trinidad",
+    364: "Turks and Caicos",
+    366: "United States",
+    367: "United States",
+    368: "United States",
+    369: "United States",
+    370: "Panama",
+    371: "Panama",
+    372: "Panama",
+    373: "Panama",
+    374: "Panama",
+    375: "Saint Vincent",
+    376: "Saint Vincent",
+    377: "Saint Vincent",
+    378: "British Virgin Islands",
+    379: "US Virgin Islands",
+    401: "Afghanistan",
+    403: "Saudi Arabia",
+    405: "Bangladesh",
+    408: "Bahrain",
+    410: "Bhutan",
+    412: "China",
+    413: "China",
+    414: "China",
+    416: "Taiwan",
+    417: "Sri Lanka",
+    419: "India",
+    422: "Iran",
+    423: "Azerbaijan",
+    425: "Iraq",
+    428: "Israel",
+    431: "Japan",
+    432: "Japan",
+    434: "Turkmenistan",
+    436: "Kazakhstan",
+    437: "Uzbekistan",
+    438: "Jordan",
+    440: "South Korea",
+    441: "South Korea",
+    443: "Palestine",
+    445: "North Korea",
+    447: "Kuwait",
+    450: "Lebanon",
+    451: "Kyrgyzstan",
+    453: "Macao",
+    455: "Maldives",
+    457: "Mongolia",
+    459: "Nepal",
+    461: "Oman",
+    463: "Pakistan",
+    466: "Qatar",
+    468: "Syria",
+    470: "UAE",
+    472: "Tajikistan",
+    473: "Yemen",
+    475: "Tonga",
+    477: "Hong Kong",
+    478: "Bosnia",
+    501: "Antarctica",
+    503: "Australia",
+    506: "Myanmar",
+    508: "Brunei",
+    510: "Micronesia",
+    511: "Palau",
+    512: "New Zealand",
+    514: "Cambodia",
+    515: "Cambodia",
+    516: "Christmas Island",
+    518: "Cook Islands",
+    520: "Fiji",
+    523: "Cocos Islands",
+    525: "Indonesia",
+    529: "Kiribati",
+    531: "Laos",
+    533: "Malaysia",
+    536: "Northern Mariana Islands",
+    538: "Marshall Islands",
+    540: "New Caledonia",
+    542: "Niue",
+    544: "Nauru",
+    546: "French Polynesia",
+    548: "Philippines",
+    553: "Papua New Guinea",
+    555: "Pitcairn",
+    557: "Solomon Islands",
+    559: "American Samoa",
+    561: "Samoa",
+    563: "Singapore",
+    564: "Singapore",
+    565: "Singapore",
+    566: "Singapore",
+    567: "Thailand",
+    570: "Tonga",
+    572: "Tuvalu",
+    574: "Vietnam",
+    576: "Vanuatu",
+    577: "Vanuatu",
+    578: "Wallis and Futuna",
+    601: "South Africa",
+    603: "Angola",
+    605: "Algeria",
+    607: "Benin",
+    609: "Botswana",
+    610: "Burundi",
+    611: "Cameroon",
+    612: "Cape Verde",
+    613: "Central African Republic",
+    615: "Congo",
+    616: "Comoros",
+    617: "DR Congo",
+    618: "Ivory Coast",
+    619: "Djibouti",
+    620: "Egypt",
+    621: "Equatorial Guinea",
+    622: "Ethiopia",
+    624: "Eritrea",
+    625: "Gabon",
+    626: "Gambia",
+    627: "Ghana",
+    629: "Guinea",
+    630: "Guinea-Bissau",
+    631: "Kenya",
+    632: "Lesotho",
+    633: "Liberia",
+    634: "Liberia",
+    635: "Liberia",
+    636: "Liberia",
+    637: "Libya",
+    642: "Madagascar",
+    644: "Malawi",
+    645: "Mali",
+    647: "Mauritania",
+    649: "Mauritius",
+    650: "Mozambique",
+    654: "Namibia",
+    655: "Niger",
+    656: "Nigeria",
+    657: "Guinea",
+    659: "Rwanda",
+    660: "Senegal",
+    661: "Sierra Leone",
+    662: "Somalia",
+    663: "South Africa",
+    664: "Sudan",
+    667: "Tanzania",
+    668: "Togo",
+    669: "Tunisia",
+    670: "Uganda",
+    671: "Egypt",
+    672: "Tanzania",
+    674: "Zambia",
+    675: "Zimbabwe",
+    676: "Comoros",
+    677: "Tanzania",
 }
 
+
 def get_country_from_mmsi(mmsi: int) -> str:
     """Look up flag state from MMSI Maritime Identification Digit."""
     mmsi_str = str(mmsi)
@@ -130,8 +330,10 @@ _vessels: dict[int, dict] = {}
 _vessels_lock = threading.Lock()
 _ws_thread: threading.Thread | None = None
 _ws_running = False
+_proxy_process = None
 
 import os
+
 CACHE_FILE = os.path.join(os.path.dirname(__file__), "ais_cache.json")
 
 
@@ -141,10 +343,10 @@ def _save_cache():
         with _vessels_lock:
             # Convert int keys to strings for JSON
             data = {str(k): v for k, v in _vessels.items()}
-        with open(CACHE_FILE, 'w') as f:
+        with open(CACHE_FILE, "w") as f:
             json.dump(data, f)
         logger.info(f"AIS cache saved: {len(data)} vessels")
-    except Exception as e:
+    except (IOError, OSError) as e:
         logger.error(f"Failed to save AIS cache: {e}")
 
 
@@ -154,7 +356,7 @@ def _load_cache():
     if not os.path.exists(CACHE_FILE):
         return
     try:
-        with open(CACHE_FILE, 'r') as f:
+        with open(CACHE_FILE, "r") as f:
             data = json.load(f)
         now = time.time()
         stale_cutoff = now - 3600  # Accept vessels up to 1 hour old on restart
@@ -165,192 +367,298 @@ def _load_cache():
                     _vessels[int(k)] = v
                     loaded += 1
         logger.info(f"AIS cache loaded: {loaded} vessels from disk")
-    except Exception as e:
+    except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
         logger.error(f"Failed to load AIS cache: {e}")
 
 
-def get_ais_vessels() -> list[dict]:
-    """Return a snapshot of tracked AIS vessels, excluding 'other' type, pruning stale."""
+def prune_stale_vessels():
+    """Remove vessels not updated in the last 15 minutes. Safe to call from a scheduler."""
     now = time.time()
-    stale_cutoff = now - 900  # 15 minutes
-    
+    stale_cutoff = now - 900
     with _vessels_lock:
-        # Prune stale vessels
         stale_keys = [k for k, v in _vessels.items() if v.get("_updated", 0) < stale_cutoff]
         for k in stale_keys:
             del _vessels[k]
-        
+    if stale_keys:
+        logger.info(f"AIS pruned {len(stale_keys)} stale vessels")
+
+
+def get_ais_vessels() -> list[dict]:
+    """Return a snapshot of tracked AIS vessels, pruning stale."""
+    prune_stale_vessels()
+
+    with _vessels_lock:
         result = []
         for mmsi, v in _vessels.items():
             v_type = v.get("type", "unknown")
-            # Skip 'other' vessels (fishing, tug, pilot, etc.) to reduce load
-            if v_type == "other":
-                continue
             # Skip vessels without valid position
             if not v.get("lat") or not v.get("lng"):
                 continue
-            
-            result.append({
-                "mmsi": mmsi,
-                "name": v.get("name", "UNKNOWN"),
-                "type": v_type,
-                "lat": round(v.get("lat", 0), 5),
-                "lng": round(v.get("lng", 0), 5),
-                "heading": v.get("heading", 0),
-                "sog": round(v.get("sog", 0), 1),
-                "cog": round(v.get("cog", 0), 1),
-                "callsign": v.get("callsign", ""),
-                "destination": v.get("destination", "") or "UNKNOWN",
-                "imo": v.get("imo", 0),
-                "country": get_country_from_mmsi(mmsi),
-            })
+
+            # Sanitize speed: AIS 102.3 kn = "speed not available"
+            sog = v.get("sog", 0)
+            if sog >= 102.2:
+                sog = 0
+
+            result.append(
+                {
+                    "mmsi": mmsi,
+                    "name": v.get("name", "UNKNOWN"),
+                    "type": v_type,
+                    "lat": round(v.get("lat", 0), 5),
+                    "lng": round(v.get("lng", 0), 5),
+                    "heading": v.get("heading", 0),
+                    "sog": round(sog, 1),
+                    "cog": round(v.get("cog", 0), 1),
+                    "callsign": v.get("callsign", ""),
+                    "destination": v.get("destination", "") or "UNKNOWN",
+                    "imo": v.get("imo", 0),
+                    "country": get_country_from_mmsi(mmsi),
+                }
+            )
         return result
 
 
+def ingest_ais_catcher(msgs: list[dict]) -> int:
+    """Ingest decoded AIS messages from AIS-catcher HTTP feed.
+    Returns number of vessels updated."""
+    count = 0
+    now = time.time()
+    with _vessels_lock:
+        for msg in msgs:
+            mmsi = msg.get("mmsi")
+            if not mmsi or not isinstance(mmsi, int):
+                continue
+
+            vessel = _vessels.setdefault(mmsi, {"mmsi": mmsi})
+            msg_type = msg.get("type", 0)
+
+            # Position reports (types 1, 2, 3 = Class A; 18, 19 = Class B)
+            if msg_type in (1, 2, 3, 18, 19):
+                lat = msg.get("lat")
+                lon = msg.get("lon")
+                if lat is not None and lon is not None and lat != 91.0 and lon != 181.0:
+                    vessel["lat"] = lat
+                    vessel["lng"] = lon
+                    # AIS raw value 1023 (102.3 kn) = "speed not available"
+                    raw_speed = msg.get("speed", 0)
+                    vessel["sog"] = 0 if raw_speed >= 102.2 else raw_speed
+                    vessel["cog"] = msg.get("course", 0)
+                    heading = msg.get("heading", 511)
+                    vessel["heading"] = heading if heading != 511 else vessel.get("cog", 0)
+                    vessel["_updated"] = now
+                    if msg.get("shipname"):
+                        vessel["name"] = msg["shipname"].strip()
+                    count += 1
+
+            # Static data (type 5 = Class A static; 24 = Class B static)
+            elif msg_type in (5, 24):
+                if msg.get("shipname"):
+                    vessel["name"] = msg["shipname"].strip()
+                if msg.get("callsign"):
+                    vessel["callsign"] = msg["callsign"].strip()
+                if msg.get("imo"):
+                    vessel["imo"] = msg["imo"]
+                if msg.get("destination"):
+                    vessel["destination"] = msg["destination"].strip().replace("@", "")
+                ship_type = msg.get("shiptype", 0)
+                if ship_type:
+                    vessel["ais_type_code"] = ship_type
+                    vessel["type"] = classify_vessel(ship_type, mmsi)
+                vessel["_updated"] = now
+
+            # Ensure country is set from MMSI MID
+            if "country" not in vessel:
+                vessel["country"] = get_country_from_mmsi(mmsi)
+
+            # Ensure name exists
+            if "name" not in vessel:
+                vessel["name"] = msg.get("shipname", "UNKNOWN") or "UNKNOWN"
+
+    return count
+
+
 def _ais_stream_loop():
     """Main loop: spawn node proxy and process messages from stdout."""
+    global _proxy_process
     import subprocess
     import os
 
     proxy_script = os.path.join(os.path.dirname(os.path.dirname(__file__)), "ais_proxy.js")
     backoff = 1  # Exponential backoff starting at 1 second
 
+    if not API_KEY:
+        logger.info("AIS_API_KEY not set — ship tracking disabled. Set AIS_API_KEY to enable.")
+        return
+
     while _ws_running:
         try:
             logger.info("Starting Node.js AIS Stream Proxy...")
+            proxy_env = os.environ.copy()
+            proxy_env["AIS_API_KEY"] = API_KEY
             process = subprocess.Popen(
-                ['node', proxy_script, API_KEY],
+                ["node", proxy_script],
+                stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
                 text=True,
-                bufsize=1
+                bufsize=1,
+                env=proxy_env,
             )
-            
+            with _vessels_lock:
+                _proxy_process = process
+
             # Drain stderr in a background thread to prevent deadlock
             import threading
+
             def _drain_stderr():
-                for errline in iter(process.stderr.readline, ''):
+                for errline in iter(process.stderr.readline, ""):
                     errline = errline.strip()
                     if errline:
                         logger.warning(f"AIS proxy stderr: {errline}")
+
             threading.Thread(target=_drain_stderr, daemon=True).start()
-            
+
             logger.info("AIS Stream proxy started — receiving vessel data")
-            
+
             msg_count = 0
-            for raw_msg in iter(process.stdout.readline, ''):
+            ok_streak = 0  # Track consecutive successful messages for backoff reset
+            last_log_time = time.time()
+            for raw_msg in iter(process.stdout.readline, ""):
                 if not _ws_running:
                     process.terminate()
                     break
-                
+
                 raw_msg = raw_msg.strip()
                 if not raw_msg:
                     continue
-                
+
                 try:
                     data = json.loads(raw_msg)
                 except json.JSONDecodeError:
                     continue
-                
+
                 if "error" in data:
                     logger.error(f"AIS Stream error: {data['error']}")
                     continue
-                
+
                 msg_type = data.get("MessageType", "")
                 metadata = data.get("MetaData", {})
                 message = data.get("Message", {})
-                
+
                 mmsi = metadata.get("MMSI", 0)
                 if not mmsi:
                     continue
-                
+
                 with _vessels_lock:
                     if mmsi not in _vessels:
                         _vessels[mmsi] = {"_updated": time.time()}
                     vessel = _vessels[mmsi]
-                
+
                 # Update position from PositionReport or StandardClassBPositionReport
                 if msg_type in ("PositionReport", "StandardClassBPositionReport"):
                     report = message.get(msg_type, {})
                     lat = report.get("Latitude", metadata.get("latitude", 0))
                     lng = report.get("Longitude", metadata.get("longitude", 0))
-                    
+
                     # Skip invalid positions
                     if lat == 0 and lng == 0:
                         continue
                     if abs(lat) > 90 or abs(lng) > 180:
                         continue
-                    
+
                     with _vessels_lock:
                         vessel["lat"] = lat
                         vessel["lng"] = lng
-                        vessel["sog"] = report.get("Sog", 0)
+                        # AIS raw value 1023 (102.3 kn) = "speed not available"
+                        raw_sog = report.get("Sog", 0)
+                        vessel["sog"] = 0 if raw_sog >= 102.2 else raw_sog
                         vessel["cog"] = report.get("Cog", 0)
                         heading = report.get("TrueHeading", 511)
                         vessel["heading"] = heading if heading != 511 else report.get("Cog", 0)
                         vessel["_updated"] = time.time()
                         # Use metadata name if we don't have one yet
                         if not vessel.get("name") or vessel["name"] == "UNKNOWN":
-                            vessel["name"] = metadata.get("ShipName", "UNKNOWN").strip() or "UNKNOWN"
-                
+                            vessel["name"] = (
+                                metadata.get("ShipName", "UNKNOWN").strip() or "UNKNOWN"
+                            )
+
                 # Update static data from ShipStaticData
                 elif msg_type == "ShipStaticData":
                     static = message.get("ShipStaticData", {})
                     ais_type = static.get("Type", 0)
-                    
+
                     with _vessels_lock:
-                        vessel["name"] = (static.get("Name", "") or metadata.get("ShipName", "UNKNOWN")).strip() or "UNKNOWN"
+                        vessel["name"] = (
+                            static.get("Name", "") or metadata.get("ShipName", "UNKNOWN")
+                        ).strip() or "UNKNOWN"
                         vessel["callsign"] = (static.get("CallSign", "") or "").strip()
                         vessel["imo"] = static.get("ImoNumber", 0)
-                        vessel["destination"] = (static.get("Destination", "") or "").strip().replace("@", "")
+                        vessel["destination"] = (
+                            (static.get("Destination", "") or "").strip().replace("@", "")
+                        )
                         vessel["ais_type_code"] = ais_type
                         vessel["type"] = classify_vessel(ais_type, mmsi)
                         vessel["_updated"] = time.time()
-                
+
                 msg_count += 1
-                if msg_count % 5000 == 0:
+                ok_streak += 1
+
+                # Reset backoff after 200 consecutive successful messages
+                if ok_streak >= 200 and backoff > 1:
+                    backoff = 1
+                    ok_streak = 0
+
+                # Periodic logging + cache save (time-based instead of count-based to avoid lock in hot loop)
+                now = time.time()
+                if now - last_log_time >= 60:
                     with _vessels_lock:
-                        # Inline pruning: remove vessels not updated in 15 minutes
-                        prune_cutoff = time.time() - 900
-                        stale = [k for k, v in _vessels.items() if v.get("_updated", 0) < prune_cutoff]
-                        for k in stale:
-                            del _vessels[k]
                         count = len(_vessels)
-                    if stale:
-                        logger.info(f"AIS pruned {len(stale)} stale vessels")
-                    logger.info(f"AIS Stream: processed {msg_count} messages, tracking {count} vessels")
-                    _save_cache()  # Auto-save every 5000 messages (~60 seconds)
-                        
-        except Exception as e:
+                    logger.info(
+                        f"AIS Stream: processed {msg_count} messages, tracking {count} vessels"
+                    )
+                    _save_cache()
+                    last_log_time = now
+
+        except (ConnectionError, TimeoutError, OSError, ValueError, KeyError) as e:
             logger.error(f"AIS proxy connection error: {e}")
             if _ws_running:
                 logger.info(f"Restarting AIS proxy in {backoff}s (exponential backoff)...")
                 time.sleep(backoff)
                 backoff = min(backoff * 2, 60)  # Double up to 60s max
             continue
-        # Reset backoff on successful connection (got at least some messages)
-        backoff = 1
 
 
 def _run_ais_loop():
     """Thread target: run the AIS loop."""
+    global _ws_running, _ws_thread, _proxy_process
     try:
         _ais_stream_loop()
     except Exception as e:
         logger.error(f"AIS Stream thread crashed: {e}")
+    finally:
+        with _vessels_lock:
+            _ws_running = False
+            _ws_thread = None
+            _proxy_process = None
 
 
 def start_ais_stream():
     """Start the AIS WebSocket stream in a background thread."""
     global _ws_thread, _ws_running
-    if _ws_thread and _ws_thread.is_alive():
+    with _vessels_lock:
+        if _ws_running:
+            logger.info("AIS Stream already running")
+            return
+        _ws_running = True
+        existing_thread = _ws_thread
+    if existing_thread and existing_thread.is_alive():
         logger.info("AIS Stream already running")
         return
-    
+
     # Load cached vessel data from disk
     _load_cache()
-    
-    _ws_running = True
+
     _ws_thread = threading.Thread(target=_run_ais_loop, daemon=True, name="ais-stream")
     _ws_thread.start()
     logger.info("AIS Stream background thread started")
@@ -358,7 +666,36 @@ def start_ais_stream():
 
 def stop_ais_stream():
     """Stop the AIS WebSocket stream and save cache."""
-    global _ws_running
-    _ws_running = False
+    global _ws_running, _ws_thread, _proxy_process
+    with _vessels_lock:
+        _ws_running = False
+        _ws_thread = None
+        proc = _proxy_process
+        _proxy_process = None
+
+    if proc and proc.stdin:
+        try:
+            proc.stdin.close()
+        except Exception:
+            pass
+
     _save_cache()  # Save on shutdown
     logger.info("AIS Stream stopping...")
+
+
+def update_ais_bbox(south: float, west: float, north: float, east: float):
+    """Dynamically update the AIS stream bounding box via proxy stdin."""
+    with _vessels_lock:
+        proc = _proxy_process
+    if not proc or not proc.stdin:
+        return
+
+    try:
+        cmd = json.dumps({"type": "update_bbox", "bboxes": [[[south, west], [north, east]]]})
+        proc.stdin.write(cmd + "\n")
+        proc.stdin.flush()
+        logger.info(
+            f"Updated AIS bounding box to: S:{south:.2f} W:{west:.2f} N:{north:.2f} E:{east:.2f}"
+        )
+    except Exception as e:
+        logger.error(f"Failed to update AIS bbox: {e}")

backend/services/api_settings.py

@@ -2,6 +2,7 @@
 API Settings management — serves the API key registry and allows updates.
 Keys are stored in the backend .env file and loaded via python-dotenv.
 """
+
 import os
 import re
 from pathlib import Path
@@ -121,6 +122,24 @@ API_REGISTRY = [
         "url": "https://openmhz.com/",
         "required": False,
     },
+    {
+        "id": "shodan_api_key",
+        "env_key": "SHODAN_API_KEY",
+        "name": "Shodan — Operator API Key",
+        "description": "Paid Shodan API key for local operator-driven searches and temporary map overlays. Results are attributed to Shodan and are not merged into ShadowBroker core feeds.",
+        "category": "Reconnaissance",
+        "url": "https://account.shodan.io/billing",
+        "required": False,
+    },
+    {
+        "id": "finnhub_api_key",
+        "env_key": "FINNHUB_API_KEY",
+        "name": "Finnhub — API Key",
+        "description": "Free market data API. Defense stock quotes, congressional trading disclosures, and insider transactions. 60 calls/min free tier.",
+        "category": "Financial",
+        "url": "https://finnhub.io/register",
+        "required": False,
+    },
 ]
 
 
@@ -160,7 +179,7 @@ def update_api_key(env_key: str, new_value: str) -> bool:
     valid_keys = {api["env_key"] for api in API_REGISTRY if api.get("env_key")}
     if env_key not in valid_keys:
         return False
-        
+
     if not isinstance(new_value, str):
         return False
     if "\n" in new_value or "\r" in new_value:

backend/services/carrier_tracker.py

@@ -15,6 +15,7 @@ import json
 import time
 import logging
 import threading
+import random
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Dict, List, Optional
@@ -26,104 +27,135 @@ logger = logging.getLogger(__name__)
 # Carrier registry: hull number → metadata + fallback position
 # -----------------------------------------------------------------
 CARRIER_REGISTRY: Dict[str, dict] = {
+    # Fallback positions sourced from USNI News Fleet & Marine Tracker (Mar 9, 2026)
+    # https://news.usni.org/2026/03/09/usni-news-fleet-and-marine-tracker-march-9-2026
+    # --- Bremerton, WA (Naval Base Kitsap) ---
+    # Distinct pier positions along Sinclair Inlet so carriers don't stack
     "CVN-68": {
         "name": "USS Nimitz (CVN-68)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Nimitz",
         "homeport": "Bremerton, WA",
-        "homeport_lat": 47.56, "homeport_lng": -122.63,
-        "fallback_lat": 21.35, "fallback_lng": -157.95,
-        "fallback_heading": 270,
-        "fallback_desc": "Pacific Fleet / Pearl Harbor"
-    },
-    "CVN-69": {
-        "name": "USS Dwight D. Eisenhower (CVN-69)",
-        "wiki": "https://en.wikipedia.org/wiki/USS_Dwight_D._Eisenhower",
-        "homeport": "Norfolk, VA",
-        "homeport_lat": 36.95, "homeport_lng": -76.33,
-        "fallback_lat": 18.0, "fallback_lng": 39.5,
-        "fallback_heading": 120,
-        "fallback_desc": "Red Sea / CENTCOM AOR"
-    },
-    "CVN-78": {
-        "name": "USS Gerald R. Ford (CVN-78)",
-        "wiki": "https://en.wikipedia.org/wiki/USS_Gerald_R._Ford",
-        "homeport": "Norfolk, VA",
-        "homeport_lat": 36.95, "homeport_lng": -76.33,
-        "fallback_lat": 34.0, "fallback_lng": 25.0,
+        "homeport_lat": 47.5535,
+        "homeport_lng": -122.6400,
+        "fallback_lat": 47.5535,
+        "fallback_lng": -122.6400,
         "fallback_heading": 90,
-        "fallback_desc": "Eastern Mediterranean deterrence"
-    },
-    "CVN-70": {
-        "name": "USS Carl Vinson (CVN-70)",
-        "wiki": "https://en.wikipedia.org/wiki/USS_Carl_Vinson",
-        "homeport": "San Diego, CA",
-        "homeport_lat": 32.68, "homeport_lng": -117.15,
-        "fallback_lat": 15.0, "fallback_lng": 115.0,
-        "fallback_heading": 45,
-        "fallback_desc": "South China Sea patrol"
-    },
-    "CVN-71": {
-        "name": "USS Theodore Roosevelt (CVN-71)",
-        "wiki": "https://en.wikipedia.org/wiki/USS_Theodore_Roosevelt_(CVN-71)",
-        "homeport": "San Diego, CA",
-        "homeport_lat": 32.68, "homeport_lng": -117.15,
-        "fallback_lat": 22.0, "fallback_lng": 122.0,
-        "fallback_heading": 300,
-        "fallback_desc": "Philippine Sea / Taiwan Strait"
-    },
-    "CVN-72": {
-        "name": "USS Abraham Lincoln (CVN-72)",
-        "wiki": "https://en.wikipedia.org/wiki/USS_Abraham_Lincoln_(CVN-72)",
-        "homeport": "San Diego, CA",
-        "homeport_lat": 32.68, "homeport_lng": -117.15,
-        "fallback_lat": 21.0, "fallback_lng": -158.0,
-        "fallback_heading": 270,
-        "fallback_desc": "Pacific deployment"
-    },
-    "CVN-73": {
-        "name": "USS George Washington (CVN-73)",
-        "wiki": "https://en.wikipedia.org/wiki/USS_George_Washington_(CVN-73)",
-        "homeport": "Yokosuka, Japan",
-        "homeport_lat": 35.28, "homeport_lng": 139.67,
-        "fallback_lat": 35.0, "fallback_lng": 139.0,
-        "fallback_heading": 0,
-        "fallback_desc": "Yokosuka, Japan (Forward deployed)"
-    },
-    "CVN-74": {
-        "name": "USS John C. Stennis (CVN-74)",
-        "wiki": "https://en.wikipedia.org/wiki/USS_John_C._Stennis",
-        "homeport": "Norfolk, VA",
-        "homeport_lat": 36.95, "homeport_lng": -76.33,
-        "fallback_lat": 36.95, "fallback_lng": -76.33,
-        "fallback_heading": 0,
-        "fallback_desc": "RCOH / Norfolk (maintenance)"
-    },
-    "CVN-75": {
-        "name": "USS Harry S. Truman (CVN-75)",
-        "wiki": "https://en.wikipedia.org/wiki/USS_Harry_S._Truman",
-        "homeport": "Norfolk, VA",
-        "homeport_lat": 36.95, "homeport_lng": -76.33,
-        "fallback_lat": 36.0, "fallback_lng": 15.0,
-        "fallback_heading": 90,
-        "fallback_desc": "Mediterranean deployment"
+        "fallback_desc": "Bremerton, WA (Maintenance)",
     },
     "CVN-76": {
         "name": "USS Ronald Reagan (CVN-76)",
         "wiki": "https://en.wikipedia.org/wiki/USS_Ronald_Reagan",
         "homeport": "Bremerton, WA",
-        "homeport_lat": 47.56, "homeport_lng": -122.63,
-        "fallback_lat": 47.56, "fallback_lng": -122.63,
+        "homeport_lat": 47.5580,
+        "homeport_lng": -122.6360,
+        "fallback_lat": 47.5580,
+        "fallback_lng": -122.6360,
+        "fallback_heading": 90,
+        "fallback_desc": "Bremerton, WA (Decommissioning)",
+    },
+    # --- Norfolk, VA (Naval Station Norfolk) ---
+    # Piers run N-S along Willoughby Bay; each carrier gets a distinct berth
+    "CVN-69": {
+        "name": "USS Dwight D. Eisenhower (CVN-69)",
+        "wiki": "https://en.wikipedia.org/wiki/USS_Dwight_D._Eisenhower",
+        "homeport": "Norfolk, VA",
+        "homeport_lat": 36.9465,
+        "homeport_lng": -76.3265,
+        "fallback_lat": 36.9465,
+        "fallback_lng": -76.3265,
         "fallback_heading": 0,
-        "fallback_desc": "Bremerton, WA (Homeport)"
+        "fallback_desc": "Norfolk, VA (Post-deployment maintenance)",
+    },
+    "CVN-78": {
+        "name": "USS Gerald R. Ford (CVN-78)",
+        "wiki": "https://en.wikipedia.org/wiki/USS_Gerald_R._Ford",
+        "homeport": "Norfolk, VA",
+        "homeport_lat": 36.9505,
+        "homeport_lng": -76.3250,
+        "fallback_lat": 18.0,
+        "fallback_lng": 39.5,
+        "fallback_heading": 0,
+        "fallback_desc": "Red Sea — Operation Epic Fury (USNI Mar 9)",
+    },
+    "CVN-74": {
+        "name": "USS John C. Stennis (CVN-74)",
+        "wiki": "https://en.wikipedia.org/wiki/USS_John_C._Stennis",
+        "homeport": "Norfolk, VA",
+        "homeport_lat": 36.9540,
+        "homeport_lng": -76.3235,
+        "fallback_lat": 36.98,
+        "fallback_lng": -76.43,
+        "fallback_heading": 0,
+        "fallback_desc": "Newport News, VA (RCOH refueling overhaul)",
+    },
+    "CVN-75": {
+        "name": "USS Harry S. Truman (CVN-75)",
+        "wiki": "https://en.wikipedia.org/wiki/USS_Harry_S._Truman",
+        "homeport": "Norfolk, VA",
+        "homeport_lat": 36.9580,
+        "homeport_lng": -76.3220,
+        "fallback_lat": 36.0,
+        "fallback_lng": 15.0,
+        "fallback_heading": 0,
+        "fallback_desc": "Mediterranean Sea deployment (USNI Mar 9)",
     },
     "CVN-77": {
         "name": "USS George H.W. Bush (CVN-77)",
         "wiki": "https://en.wikipedia.org/wiki/USS_George_H.W._Bush",
         "homeport": "Norfolk, VA",
-        "homeport_lat": 36.95, "homeport_lng": -76.33,
-        "fallback_lat": 36.95, "fallback_lng": -76.33,
+        "homeport_lat": 36.9620,
+        "homeport_lng": -76.3210,
+        "fallback_lat": 36.5,
+        "fallback_lng": -74.0,
         "fallback_heading": 0,
-        "fallback_desc": "Norfolk, VA (Homeport)"
+        "fallback_desc": "Atlantic — Pre-deployment workups (USNI Mar 9)",
+    },
+    # --- San Diego, CA (Naval Base San Diego) ---
+    # Carrier piers along the east shore of San Diego Bay, spread N-S
+    "CVN-70": {
+        "name": "USS Carl Vinson (CVN-70)",
+        "wiki": "https://en.wikipedia.org/wiki/USS_Carl_Vinson",
+        "homeport": "San Diego, CA",
+        "homeport_lat": 32.6840,
+        "homeport_lng": -117.1290,
+        "fallback_lat": 32.6840,
+        "fallback_lng": -117.1290,
+        "fallback_heading": 180,
+        "fallback_desc": "San Diego, CA (Homeport)",
+    },
+    "CVN-71": {
+        "name": "USS Theodore Roosevelt (CVN-71)",
+        "wiki": "https://en.wikipedia.org/wiki/USS_Theodore_Roosevelt_(CVN-71)",
+        "homeport": "San Diego, CA",
+        "homeport_lat": 32.6885,
+        "homeport_lng": -117.1280,
+        "fallback_lat": 32.6885,
+        "fallback_lng": -117.1280,
+        "fallback_heading": 180,
+        "fallback_desc": "San Diego, CA (Maintenance)",
+    },
+    "CVN-72": {
+        "name": "USS Abraham Lincoln (CVN-72)",
+        "wiki": "https://en.wikipedia.org/wiki/USS_Abraham_Lincoln_(CVN-72)",
+        "homeport": "San Diego, CA",
+        "homeport_lat": 32.6925,
+        "homeport_lng": -117.1275,
+        "fallback_lat": 20.0,
+        "fallback_lng": 64.0,
+        "fallback_heading": 0,
+        "fallback_desc": "Arabian Sea — Operation Epic Fury (USNI Mar 9)",
+    },
+    # --- Yokosuka, Japan (CFAY) ---
+    "CVN-73": {
+        "name": "USS George Washington (CVN-73)",
+        "wiki": "https://en.wikipedia.org/wiki/USS_George_Washington_(CVN-73)",
+        "homeport": "Yokosuka, Japan",
+        "homeport_lat": 35.2830,
+        "homeport_lng": 139.6700,
+        "fallback_lat": 35.2830,
+        "fallback_lng": 139.6700,
+        "fallback_heading": 180,
+        "fallback_desc": "Yokosuka, Japan (Forward deployed)",
     },
 }
 
@@ -163,7 +195,6 @@ REGION_COORDS: Dict[str, tuple] = {
     "coral sea": (-18.0, 155.0),
     "gulf of mexico": (25.0, -90.0),
     "caribbean": (15.0, -75.0),
-
     # Specific bases / ports
     "norfolk": (36.95, -76.33),
     "san diego": (32.68, -117.15),
@@ -176,7 +207,6 @@ REGION_COORDS: Dict[str, tuple] = {
     "bremerton": (47.56, -122.63),
     "puget sound": (47.56, -122.63),
     "newport news": (36.98, -76.43),
-
     # Areas of operation
     "centcom": (25.0, 55.0),
     "indopacom": (20.0, 130.0),
@@ -197,6 +227,11 @@ CACHE_FILE = Path(__file__).parent.parent / "carrier_cache.json"
 _carrier_positions: Dict[str, dict] = {}
 _positions_lock = threading.Lock()
 _last_update: Optional[datetime] = None
+_last_gdelt_fetch_at = 0.0
+_cached_gdelt_articles: List[dict] = []
+_GDELT_FETCH_INTERVAL_SECONDS = 1800
+_GDELT_REQUEST_DELAY_SECONDS = 1.25
+_GDELT_REQUEST_JITTER_SECONDS = 0.35
 
 
 def _load_cache() -> Dict[str, dict]:
@@ -206,7 +241,7 @@ def _load_cache() -> Dict[str, dict]:
             data = json.loads(CACHE_FILE.read_text())
             logger.info(f"Carrier cache loaded: {len(data)} carriers from {CACHE_FILE}")
             return data
-    except Exception as e:
+    except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
         logger.warning(f"Failed to load carrier cache: {e}")
     return {}
 
@@ -216,7 +251,7 @@ def _save_cache(positions: Dict[str, dict]):
     try:
         CACHE_FILE.write_text(json.dumps(positions, indent=2))
         logger.info(f"Carrier cache saved: {len(positions)} carriers")
-    except Exception as e:
+    except (IOError, OSError) as e:
         logger.warning(f"Failed to save carrier cache: {e}")
 
 
@@ -248,33 +283,59 @@ def _match_carrier(text: str) -> Optional[str]:
 
 def _fetch_gdelt_carrier_news() -> List[dict]:
     """Search GDELT for recent carrier movement news."""
+    global _last_gdelt_fetch_at, _cached_gdelt_articles
+
+    now = time.time()
+    if _cached_gdelt_articles and (now - _last_gdelt_fetch_at) < _GDELT_FETCH_INTERVAL_SECONDS:
+        logger.info("Carrier OSINT: using cached GDELT article set to avoid startup bursts")
+        return list(_cached_gdelt_articles)
+
     results = []
     search_terms = [
         "aircraft+carrier+deployed",
         "carrier+strike+group+navy",
-        "USS+Nimitz+carrier", "USS+Ford+carrier", "USS+Eisenhower+carrier",
-        "USS+Vinson+carrier", "USS+Roosevelt+carrier+navy",
-        "USS+Lincoln+carrier", "USS+Truman+carrier",
-        "USS+Reagan+carrier", "USS+Washington+carrier+navy",
-        "USS+Bush+carrier", "USS+Stennis+carrier",
+        "USS+Nimitz+carrier",
+        "USS+Ford+carrier",
+        "USS+Eisenhower+carrier",
+        "USS+Vinson+carrier",
+        "USS+Roosevelt+carrier+navy",
+        "USS+Lincoln+carrier",
+        "USS+Truman+carrier",
+        "USS+Reagan+carrier",
+        "USS+Washington+carrier+navy",
+        "USS+Bush+carrier",
+        "USS+Stennis+carrier",
     ]
 
-    for term in search_terms:
+    for idx, term in enumerate(search_terms):
         try:
             url = f"https://api.gdeltproject.org/api/v2/doc/doc?query={term}&mode=artlist&maxrecords=5&format=json&timespan=14d"
             raw = fetch_with_curl(url, timeout=8)
-            if not raw:
+            if getattr(raw, "status_code", 500) == 429:
+                logger.warning(
+                    "GDELT returned 429 for '%s'; preserving cached carrier OSINT results",
+                    term,
+                )
                 continue
-            data = json.loads(raw)
+            if not raw or not hasattr(raw, "text"):
+                continue
+            data = raw.json()
             articles = data.get("articles", [])
             for art in articles:
                 title = art.get("title", "")
                 url = art.get("url", "")
                 results.append({"title": title, "url": url})
-        except Exception as e:
+        except (ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
             logger.debug(f"GDELT search failed for '{term}': {e}")
             continue
+        if idx < len(search_terms) - 1:
+            time.sleep(
+                _GDELT_REQUEST_DELAY_SECONDS
+                + random.uniform(0.0, _GDELT_REQUEST_JITTER_SECONDS)
+            )
 
+    _cached_gdelt_articles = list(results)
+    _last_gdelt_fetch_at = time.time()
     logger.info(f"Carrier OSINT: found {len(results)} GDELT articles")
     return results
 
@@ -302,21 +363,19 @@ def _parse_carrier_positions_from_news(articles: List[dict]) -> Dict[str, dict]:
                 "lat": coords[0],
                 "lng": coords[1],
                 "desc": title[:100],
-                "source": "GDELT OSINT",
-                "updated": datetime.now(timezone.utc).isoformat()
+                "source": "GDELT News API",
+                "source_url": article.get("url", "https://api.gdeltproject.org"),
+                "updated": datetime.now(timezone.utc).isoformat(),
             }
-            logger.info(f"Carrier update: {CARRIER_REGISTRY[hull]['name']} → {coords} (from: {title[:80]})")
+            logger.info(
+                f"Carrier update: {CARRIER_REGISTRY[hull]['name']} → {coords} (from: {title[:80]})"
+            )
 
     return updates
 
 
-def update_carrier_positions():
-    """Main update function — called on startup and every 12h."""
-    global _last_update
-
-    logger.info("Carrier tracker: updating positions from OSINT sources...")
-
-    # Start with fallback positions
+def _load_carrier_fallbacks() -> Dict[str, dict]:
+    """Build carrier positions from static fallbacks + disk cache (instant, no network)."""
     positions: Dict[str, dict] = {}
     for hull, info in CARRIER_REGISTRY.items():
         positions[hull] = {
@@ -326,25 +385,52 @@ def update_carrier_positions():
             "heading": info["fallback_heading"],
             "desc": info["fallback_desc"],
             "wiki": info["wiki"],
-            "source": "Static OSINT estimate",
-            "updated": datetime.now(timezone.utc).isoformat()
+            "source": "USNI News Fleet & Marine Tracker",
+            "source_url": "https://news.usni.org/category/fleet-tracker",
+            "updated": datetime.now(timezone.utc).isoformat(),
         }
 
-    # Load cached positions (may have better data from previous runs)
+    # Overlay cached positions from previous runs (may have GDELT data)
     cached = _load_cache()
     for hull, cached_pos in cached.items():
         if hull in positions:
-            # Only use cache if it has a real OSINT source (not just static)
-            if cached_pos.get("source", "").startswith("GDELT") or cached_pos.get("source", "").startswith("News"):
-                positions[hull].update({
-                    "lat": cached_pos["lat"],
-                    "lng": cached_pos["lng"],
-                    "desc": cached_pos.get("desc", positions[hull]["desc"]),
-                    "source": cached_pos.get("source", "Cached OSINT"),
-                    "updated": cached_pos.get("updated", "")
-                })
+            if cached_pos.get("source", "").startswith("GDELT") or cached_pos.get(
+                "source", ""
+            ).startswith("News"):
+                positions[hull].update(
+                    {
+                        "lat": cached_pos["lat"],
+                        "lng": cached_pos["lng"],
+                        "desc": cached_pos.get("desc", positions[hull]["desc"]),
+                        "source": cached_pos.get("source", "Cached OSINT"),
+                        "updated": cached_pos.get("updated", ""),
+                    }
+                )
+    return positions
 
-    # Try GDELT news for fresh positions
+
+def update_carrier_positions():
+    """Main update function — called on startup and every 12h.
+
+    Phase 1 (instant): publish fallback + cached positions so the map has carriers immediately.
+    Phase 2 (slow):    query GDELT for fresh OSINT positions and update in-place.
+    """
+    global _last_update
+
+    # --- Phase 1: instant fallback + cache ---
+    positions = _load_carrier_fallbacks()
+
+    with _positions_lock:
+        # Only overwrite if positions are currently empty (first startup).
+        # If we already have data from a previous cycle, keep it while GDELT runs.
+        if not _carrier_positions:
+            _carrier_positions.update(positions)
+            _last_update = datetime.now(timezone.utc)
+    logger.info(
+        f"Carrier tracker: {len(positions)} carriers loaded from fallback/cache (GDELT enrichment starting...)"
+    )
+
+    # --- Phase 2: slow GDELT enrichment ---
     try:
         articles = _fetch_gdelt_carrier_news()
         news_positions = _parse_carrier_positions_from_news(articles)
@@ -352,10 +438,10 @@ def update_carrier_positions():
             if hull in positions:
                 positions[hull].update(pos)
                 logger.info(f"Carrier OSINT: updated {CARRIER_REGISTRY[hull]['name']} from news")
-    except Exception as e:
+    except (ValueError, KeyError, json.JSONDecodeError, OSError) as e:
         logger.warning(f"GDELT carrier fetch failed: {e}")
 
-    # Save and update the global state
+    # Save and update the global state with enriched positions
     with _positions_lock:
         _carrier_positions.clear()
         _carrier_positions.update(positions)
@@ -370,28 +456,83 @@ def update_carrier_positions():
     logger.info(f"Carrier tracker: {len(positions)} carriers updated. Sources: {sources}")
 
 
+def _deconflict_positions(result: List[dict]) -> List[dict]:
+    """Offset carriers that share identical coordinates so they don't stack.
+
+    At port: offset along the pier axis (~500m / 0.004° apart).
+    At sea: offset perpendicular to each other (~0.08° / ~9km apart)
+    so they're visibly separate but clearly operating together.
+    """
+    # Group by rounded lat/lng (within ~0.01° ≈ 1km = same spot)
+    from collections import defaultdict
+
+    groups: dict[str, list[int]] = defaultdict(list)
+    for i, c in enumerate(result):
+        key = f"{round(c['lat'], 2)},{round(c['lng'], 2)}"
+        groups[key].append(i)
+
+    for indices in groups.values():
+        if len(indices) < 2:
+            continue
+        n = len(indices)
+        # Determine if this is a port (near a homeport) or at sea
+        sample = result[indices[0]]
+        at_port = any(
+            abs(sample["lat"] - info.get("homeport_lat", 0)) < 0.05
+            and abs(sample["lng"] - info.get("homeport_lng", 0)) < 0.05
+            for info in CARRIER_REGISTRY.values()
+        )
+
+        if at_port:
+            # Use each carrier's distinct homeport pier coordinates
+            for idx in indices:
+                carrier = result[idx]
+                hull = None
+                for h, info in CARRIER_REGISTRY.items():
+                    if info["name"] == carrier["name"]:
+                        hull = h
+                        break
+                if hull:
+                    info = CARRIER_REGISTRY[hull]
+                    carrier["lat"] = info["homeport_lat"]
+                    carrier["lng"] = info["homeport_lng"]
+        else:
+            # At sea: spread in a line perpendicular to travel (~0.08° apart)
+            spacing = 0.08  # ~9km — close enough to see they're together
+            start_offset = -(n - 1) * spacing / 2
+            for j, idx in enumerate(indices):
+                result[idx]["lng"] += start_offset + j * spacing
+
+    return result
+
+
 def get_carrier_positions() -> List[dict]:
     """Return current carrier positions for the data pipeline."""
     with _positions_lock:
         result = []
         for hull, pos in _carrier_positions.items():
             info = CARRIER_REGISTRY.get(hull, {})
-            result.append({
-                "name": pos.get("name", info.get("name", hull)),
-                "type": "carrier",
-                "lat": pos["lat"],
-                "lng": pos["lng"],
-                "heading": pos.get("heading", 0),
-                "sog": 0,
-                "cog": 0,
-                "country": "United States",
-                "desc": pos.get("desc", ""),
-                "wiki": pos.get("wiki", info.get("wiki", "")),
-                "estimated": True,
-                "source": pos.get("source", "OSINT estimated position"),
-                "last_osint_update": pos.get("updated", "")
-            })
-        return result
+            result.append(
+                {
+                    "name": pos.get("name", info.get("name", hull)),
+                    "type": "carrier",
+                    "lat": pos["lat"],
+                    "lng": pos["lng"],
+                    "heading": None,  # Heading unknown for carriers — OSINT cannot determine true heading
+                    "sog": 0,
+                    "cog": 0,
+                    "country": "United States",
+                    "desc": pos.get("desc", ""),
+                    "wiki": pos.get("wiki", info.get("wiki", "")),
+                    "estimated": True,
+                    "source": pos.get("source", "OSINT estimated position"),
+                    "source_url": pos.get(
+                        "source_url", "https://news.usni.org/category/fleet-tracker"
+                    ),
+                    "last_osint_update": pos.get("updated", ""),
+                }
+            )
+        return _deconflict_positions(result)
 
 
 # -----------------------------------------------------------------
@@ -421,10 +562,13 @@ def _scheduler_loop():
         next_run = now.replace(hour=next_hour % 24, minute=0, second=0, microsecond=0)
         if next_hour == 24:
             from datetime import timedelta
+
             next_run = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
 
         wait_seconds = (next_run - now).total_seconds()
-        logger.info(f"Carrier tracker: next update at {next_run.isoformat()} ({wait_seconds/3600:.1f}h)")
+        logger.info(
+            f"Carrier tracker: next update at {next_run.isoformat()} ({wait_seconds/3600:.1f}h)"
+        )
 
         # Wait until next scheduled time, or until stop event
         if _scheduler_stop.wait(timeout=wait_seconds):
@@ -442,7 +586,9 @@ def start_carrier_tracker():
     if _scheduler_thread and _scheduler_thread.is_alive():
         return
     _scheduler_stop.clear()
-    _scheduler_thread = threading.Thread(target=_scheduler_loop, daemon=True, name="carrier-tracker")
+    _scheduler_thread = threading.Thread(
+        target=_scheduler_loop, daemon=True, name="carrier-tracker"
+    )
     _scheduler_thread.start()
     logger.info("Carrier tracker started")
 

backend/services/config.py

@@ -0,0 +1,122 @@
+"""Typed configuration via pydantic-settings."""
+
+from functools import lru_cache
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    # Admin/security
+    ADMIN_KEY: str = ""
+    ALLOW_INSECURE_ADMIN: bool = False
+    PUBLIC_API_KEY: str = ""
+
+    # Data sources
+    AIS_API_KEY: str = ""
+    OPENSKY_CLIENT_ID: str = ""
+    OPENSKY_CLIENT_SECRET: str = ""
+    LTA_ACCOUNT_KEY: str = ""
+
+    # Runtime
+    CORS_ORIGINS: str = ""
+    FETCH_SLOW_THRESHOLD_S: float = 5.0
+    MESH_STRICT_SIGNATURES: bool = True
+    MESH_DEBUG_MODE: bool = False
+    MESH_MQTT_EXTRA_ROOTS: str = ""
+    MESH_MQTT_EXTRA_TOPICS: str = ""
+    MESH_MQTT_INCLUDE_DEFAULT_ROOTS: bool = True
+    MESH_RNS_ENABLED: bool = False
+    MESH_ARTI_ENABLED: bool = False
+    MESH_ARTI_SOCKS_PORT: int = 9050
+    MESH_RELAY_PEERS: str = "http://cipher0.shadowbroker.info:8000"
+    MESH_BOOTSTRAP_DISABLED: bool = False
+    MESH_BOOTSTRAP_MANIFEST_PATH: str = "data/bootstrap_peers.json"
+    MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY: str = ""
+    MESH_NODE_MODE: str = "participant"
+    MESH_SYNC_INTERVAL_S: int = 300
+    MESH_SYNC_FAILURE_BACKOFF_S: int = 60
+    MESH_RELAY_PUSH_TIMEOUT_S: int = 10
+    MESH_RELAY_MAX_FAILURES: int = 3
+    MESH_RELAY_FAILURE_COOLDOWN_S: int = 120
+    MESH_PEER_PUSH_SECRET: str = "Mv63UvLfwqOEVWeRBXjA8MtFl2nEkkhUlLYVHiX1Zzo"
+    MESH_RNS_APP_NAME: str = "shadowbroker"
+    MESH_RNS_ASPECT: str = "infonet"
+    MESH_RNS_IDENTITY_PATH: str = ""
+    MESH_RNS_PEERS: str = ""
+    MESH_RNS_DANDELION_HOPS: int = 2
+    MESH_RNS_DANDELION_DELAY_MS: int = 400
+    MESH_RNS_CHURN_INTERVAL_S: int = 300
+    MESH_RNS_MAX_PEERS: int = 32
+    MESH_RNS_MAX_PAYLOAD: int = 8192
+    MESH_RNS_PEER_BUCKET_PREFIX: int = 4
+    MESH_RNS_MAX_PEERS_PER_BUCKET: int = 4
+    MESH_RNS_PEER_FAIL_THRESHOLD: int = 3
+    MESH_RNS_PEER_COOLDOWN_S: int = 300
+    MESH_RNS_SHARD_ENABLED: bool = False
+    MESH_RNS_SHARD_DATA_SHARDS: int = 3
+    MESH_RNS_SHARD_PARITY_SHARDS: int = 1
+    MESH_RNS_SHARD_TTL_S: int = 30
+    MESH_RNS_FEC_CODEC: str = "xor"  # xor | rs
+    MESH_RNS_BATCH_MS: int = 200
+    # Keep a low background cadence on private RNS links so quiet nodes are less
+    # trivially fingerprintable by silence alone. Set to 0 to disable explicitly.
+    MESH_RNS_COVER_INTERVAL_S: int = 30
+    MESH_RNS_COVER_SIZE: int = 64
+    MESH_RNS_IBF_WINDOW: int = 256
+    MESH_RNS_IBF_TABLE_SIZE: int = 64
+    MESH_RNS_IBF_MINHASH_SIZE: int = 16
+    MESH_RNS_IBF_MINHASH_THRESHOLD: float = 0.25
+    MESH_RNS_IBF_WINDOW_JITTER: int = 32
+    MESH_RNS_IBF_INTERVAL_S: int = 120
+    MESH_RNS_IBF_SYNC_PEERS: int = 3
+    MESH_RNS_IBF_QUORUM_TIMEOUT_S: int = 6
+    MESH_RNS_IBF_MAX_REQUEST_IDS: int = 64
+    MESH_RNS_IBF_MAX_EVENTS: int = 64
+    MESH_RNS_SESSION_ROTATE_S: int = 1800
+    MESH_RNS_IBF_FAIL_THRESHOLD: int = 3
+    MESH_RNS_IBF_COOLDOWN_S: int = 120
+    MESH_VERIFY_INTERVAL_S: int = 600
+    MESH_VERIFY_SIGNATURES: bool = True
+    MESH_DM_SECURE_MODE: bool = True
+    MESH_DM_TOKEN_PEPPER: str = ""
+    MESH_DM_ALLOW_LEGACY_GET: bool = False
+    MESH_DM_PERSIST_SPOOL: bool = False
+    MESH_DM_REQUIRE_SENDER_SEAL_SHARED: bool = True
+    MESH_DM_NONCE_TTL_S: int = 300
+    MESH_DM_NONCE_CACHE_MAX: int = 4096
+    MESH_DM_REQUEST_MAX_AGE_S: int = 300
+    MESH_DM_REQUEST_MAILBOX_LIMIT: int = 12
+    MESH_DM_SHARED_MAILBOX_LIMIT: int = 48
+    MESH_DM_SELF_MAILBOX_LIMIT: int = 12
+    MESH_DM_MAX_MSG_BYTES: int = 8192
+    MESH_DM_ALLOW_SENDER_SEAL: bool = False
+    # TTL for DH key and prekey bundle registrations — stale entries are pruned.
+    MESH_DM_KEY_TTL_DAYS: int = 30
+    # TTL for mailbox binding metadata — shorter = smaller metadata footprint on disk.
+    MESH_DM_BINDING_TTL_DAYS: int = 7
+    # When False, mailbox bindings are memory-only (agents re-register on restart).
+    MESH_DM_METADATA_PERSIST: bool = True
+    MESH_SCOPED_TOKENS: str = ""
+    MESH_GATE_SESSION_ROTATE_MSGS: int = 50
+    MESH_GATE_SESSION_ROTATE_S: int = 3600
+    # Add a randomized grace window before anonymous gate-session auto-rotation
+    # so threshold-triggered identity swaps are less trivially correlated.
+    MESH_GATE_SESSION_ROTATE_JITTER_S: int = 180
+    # Private gate APIs expose a backward-jittered timestamp view so observers
+    # cannot trivially align exact send times from response metadata alone.
+    MESH_GATE_TIMESTAMP_JITTER_S: int = 60
+    MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK: bool = False
+    MESH_PRIVATE_LOG_TTL_S: int = 900
+    # Clearnet fallback policy for private-tier messages.
+    # "block" (default) = refuse to send private messages over clearnet.
+    # "allow" = fall back to clearnet when Tor/RNS is unavailable (weaker privacy).
+    MESH_PRIVATE_CLEARNET_FALLBACK: str = "block"
+    # Meshtastic MQTT broker credentials (defaults match public firmware).
+    MESH_MQTT_USER: str = "meshdev"
+    MESH_MQTT_PASS: str = "large4cats"
+
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore")
+
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()

backend/services/constants.py

@@ -0,0 +1,34 @@
+# ─── ShadowBroker Backend Constants ──────────────────────────────────────────
+# Centralized magic numbers. Import from here instead of hardcoding.
+
+# ─── Flight Trails ──────────────────────────────────────────────────────────
+FLIGHT_TRAIL_MAX_TRACKED = 2000  # Max concurrent tracked trails before LRU eviction
+FLIGHT_TRAIL_POINTS_PER_FLIGHT = 200  # Max trail points kept per aircraft
+TRACKED_TRAIL_TTL_S = 1800  # 30 min - trail TTL for tracked flights
+DEFAULT_TRAIL_TTL_S = 300  # 5 min - trail TTL for non-tracked flights
+
+# ─── Detection Thresholds ──────────────────────────────────────────────────
+HOLD_PATTERN_DEGREES = 300  # Total heading change to flag holding pattern
+GPS_JAMMING_NACP_THRESHOLD = 8  # NACp below this = degraded GPS signal
+GPS_JAMMING_GRID_SIZE = 1.0  # 1 degree grid for aggregation
+GPS_JAMMING_MIN_RATIO = 0.30  # 30% degraded aircraft to flag zone
+GPS_JAMMING_MIN_AIRCRAFT = 5  # Min aircraft in grid cell for statistical significance
+
+# ─── Network & Circuit Breaker ──────────────────────────────────────────────
+CIRCUIT_BREAKER_TTL_S = 120  # Skip domain for 2 min after total failure
+DOMAIN_FAIL_TTL_S = 300  # Skip requests.get for 5 min, go straight to curl
+CONNECT_TIMEOUT_S = 3  # Short connect timeout for fast firewall-block detection
+
+# ─── Data Fetcher Intervals ────────────────────────────────────────────────
+FAST_FETCH_INTERVAL_S = 60  # Flights, ships, satellites, military
+SLOW_FETCH_INTERVAL_MIN = 30  # News, markets, space weather
+CCTV_FETCH_INTERVAL_MIN = 1  # CCTV camera pipeline
+LIVEUAMAP_FETCH_INTERVAL_HR = 12  # LiveUAMap scraper
+
+# ─── External API ──────────────────────────────────────────────────────────
+OPENSKY_RATE_LIMIT_S = 300  # Only re-fetch OpenSky every 5 minutes
+OPENSKY_REQUEST_TIMEOUT_S = 15  # Timeout for OpenSky API calls
+ROUTE_FETCH_TIMEOUT_S = 15  # Timeout for adsb.lol route lookups
+
+# ─── Internet Outage Detection ─────────────────────────────────────────────
+INTERNET_OUTAGE_MIN_SEVERITY = 0.10  # 10% drop minimum to show

backend/services/correlation_engine.py

@@ -0,0 +1,342 @@
+"""
+Emergent Intelligence — Cross-layer correlation engine.
+
+Scans co-located events across multiple data layers and emits composite
+alerts that no single source could generate alone.
+
+Correlation types:
+  - RF Anomaly:          GPS jamming + internet outage (both required)
+  - Military Buildup:    Military flights + naval vessels + GDELT conflict events
+  - Infrastructure Cascade: Internet outage + KiwiSDR offline in same zone
+"""
+
+import logging
+from collections import defaultdict
+
+logger = logging.getLogger(__name__)
+
+# Grid cell size in degrees — 1° ≈ 111 km at equator.
+# Tighter than the previous 2° to reduce false co-locations.
+_CELL_SIZE = 1
+
+# Quality gates for RF anomaly correlation — only high-confidence inputs.
+# GPS jamming + internet outage overlap in a 111km cell is easily a coincidence
+# (IODA returns ~100 regional outages; GPS NACp dips are common in busy airspace).
+# Only fire when the evidence is strong enough to indicate deliberate RF interference.
+_RF_CORR_MIN_GPS_RATIO = 0.60   # Need strong jamming signal, not marginal NACp dips
+_RF_CORR_MIN_OUTAGE_PCT = 40    # Need a serious outage, not routine BGP fluctuation
+_RF_CORR_MIN_INDICATORS = 3     # Require 3+ corroborating signals (not just GPS+outage)
+
+
+def _cell_key(lat: float, lng: float) -> str:
+    """Convert lat/lng to a grid cell key."""
+    clat = int(lat // _CELL_SIZE) * _CELL_SIZE
+    clng = int(lng // _CELL_SIZE) * _CELL_SIZE
+    return f"{clat},{clng}"
+
+
+def _cell_center(key: str) -> tuple[float, float]:
+    """Get center lat/lng from a cell key."""
+    parts = key.split(",")
+    return float(parts[0]) + _CELL_SIZE / 2, float(parts[1]) + _CELL_SIZE / 2
+
+
+def _severity(indicator_count: int) -> str:
+    if indicator_count >= 3:
+        return "high"
+    if indicator_count >= 2:
+        return "medium"
+    return "low"
+
+
+def _severity_score(sev: str) -> float:
+    return {"high": 90, "medium": 60, "low": 30}.get(sev, 0)
+
+
+def _outage_pct(outage: dict) -> float:
+    """Extract outage severity percentage from an outage dict."""
+    return float(outage.get("severity", 0) or outage.get("severity_pct", 0) or 0)
+
+
+# ---------------------------------------------------------------------------
+# RF Anomaly: GPS jamming + internet outage (both must be present)
+# ---------------------------------------------------------------------------
+
+
+def _detect_rf_anomalies(data: dict) -> list[dict]:
+    gps_jamming = data.get("gps_jamming") or []
+    internet_outages = data.get("internet_outages") or []
+
+    if not gps_jamming:
+        return []  # No GPS jamming → no RF anomalies possible
+
+    # Build grid of indicators
+    cells: dict[str, dict] = defaultdict(lambda: {
+        "gps_jam": False, "gps_ratio": 0.0,
+        "outage": False, "outage_pct": 0.0,
+    })
+
+    for z in gps_jamming:
+        lat, lng = z.get("lat"), z.get("lng")
+        if lat is None or lng is None:
+            continue
+        ratio = z.get("ratio", 0)
+        if ratio < _RF_CORR_MIN_GPS_RATIO:
+            continue  # Skip marginal jamming zones
+        key = _cell_key(lat, lng)
+        cells[key]["gps_jam"] = True
+        cells[key]["gps_ratio"] = max(cells[key]["gps_ratio"], ratio)
+
+    for o in internet_outages:
+        lat = o.get("lat") or o.get("latitude")
+        lng = o.get("lng") or o.get("lon") or o.get("longitude")
+        if lat is None or lng is None:
+            continue
+        pct = _outage_pct(o)
+        if pct < _RF_CORR_MIN_OUTAGE_PCT:
+            continue  # Skip minor outages (ISP maintenance noise)
+        key = _cell_key(float(lat), float(lng))
+        cells[key]["outage"] = True
+        cells[key]["outage_pct"] = max(cells[key]["outage_pct"], pct)
+
+    # PSK Reporter: presence = healthy RF.  Only used as a bonus indicator,
+    # NOT as a standalone trigger (absence is normal in most cells).
+    psk_reporter = data.get("psk_reporter") or []
+    psk_cells: set[str] = set()
+    for s in psk_reporter:
+        lat, lng = s.get("lat"), s.get("lon")
+        if lat is not None and lng is not None:
+            psk_cells.add(_cell_key(lat, lng))
+
+    # When PSK data is unavailable, we can't get a 3rd indicator, so require
+    # an even higher GPS jamming ratio to compensate (real EW shows 75%+).
+    psk_available = len(psk_reporter) > 0
+
+    alerts: list[dict] = []
+    for key, c in cells.items():
+        # GPS jamming is the anchor — required for every RF anomaly alert
+        if not c["gps_jam"]:
+            continue
+        if not c["outage"]:
+            continue  # Both GPS jamming AND outage are always required
+
+        indicators = 2  # GPS jamming + outage
+        drivers: list[str] = [f"GPS jamming {int(c['gps_ratio'] * 100)}%"]
+        pct = c["outage_pct"]
+        drivers.append(f"Internet outage{f' {pct:.0f}%' if pct else ''}")
+
+        # PSK absence confirms RF environment is disrupted
+        if psk_available and key not in psk_cells:
+            indicators += 1
+            drivers.append("No HF digital activity (PSK Reporter)")
+
+        if indicators < _RF_CORR_MIN_INDICATORS:
+            # Without PSK data, only allow through if GPS ratio is extreme
+            # (75%+ indicates deliberate, sustained jamming — not noise)
+            if not psk_available and c["gps_ratio"] >= 0.75 and pct >= 50:
+                pass  # Allow this high-confidence 2-indicator alert through
+            else:
+                continue
+
+        lat, lng = _cell_center(key)
+        sev = _severity(indicators)
+        alerts.append({
+            "lat": lat,
+            "lng": lng,
+            "type": "rf_anomaly",
+            "severity": sev,
+            "score": _severity_score(sev),
+            "drivers": drivers[:3],
+            "cell_size": _CELL_SIZE,
+        })
+
+    return alerts
+
+
+# ---------------------------------------------------------------------------
+# Military Buildup: flights + ships + GDELT conflict
+# ---------------------------------------------------------------------------
+
+
+def _detect_military_buildups(data: dict) -> list[dict]:
+    mil_flights = data.get("military_flights") or []
+    ships = data.get("ships") or []
+    gdelt = data.get("gdelt") or []
+
+    cells: dict[str, dict] = defaultdict(lambda: {
+        "mil_flights": 0, "mil_ships": 0, "gdelt_events": 0,
+    })
+
+    for f in mil_flights:
+        lat = f.get("lat") or f.get("latitude")
+        lng = f.get("lng") or f.get("lon") or f.get("longitude")
+        if lat is None or lng is None:
+            continue
+        try:
+            key = _cell_key(float(lat), float(lng))
+            cells[key]["mil_flights"] += 1
+        except (ValueError, TypeError):
+            continue
+
+    mil_ship_types = {"military_vessel", "military", "warship", "patrol", "destroyer",
+                      "frigate", "corvette", "carrier", "submarine", "cruiser"}
+    for s in ships:
+        stype = (s.get("type") or s.get("ship_type") or "").lower()
+        if not any(mt in stype for mt in mil_ship_types):
+            continue
+        lat = s.get("lat") or s.get("latitude")
+        lng = s.get("lng") or s.get("lon") or s.get("longitude")
+        if lat is None or lng is None:
+            continue
+        try:
+            key = _cell_key(float(lat), float(lng))
+            cells[key]["mil_ships"] += 1
+        except (ValueError, TypeError):
+            continue
+
+    for g in gdelt:
+        lat = g.get("lat") or g.get("latitude") or g.get("actionGeo_Lat")
+        lng = g.get("lng") or g.get("lon") or g.get("longitude") or g.get("actionGeo_Long")
+        if lat is None or lng is None:
+            continue
+        try:
+            key = _cell_key(float(lat), float(lng))
+            cells[key]["gdelt_events"] += 1
+        except (ValueError, TypeError):
+            continue
+
+    alerts: list[dict] = []
+    for key, c in cells.items():
+        mil_total = c["mil_flights"] + c["mil_ships"]
+        has_gdelt = c["gdelt_events"] > 0
+
+        # Need meaningful military presence AND a conflict indicator
+        if mil_total < 3 or not has_gdelt:
+            continue
+
+        drivers: list[str] = []
+        if c["mil_flights"]:
+            drivers.append(f"{c['mil_flights']} military aircraft")
+        if c["mil_ships"]:
+            drivers.append(f"{c['mil_ships']} military vessels")
+        if c["gdelt_events"]:
+            drivers.append(f"{c['gdelt_events']} conflict events")
+
+        if mil_total >= 11:
+            sev = "high"
+        elif mil_total >= 6:
+            sev = "medium"
+        else:
+            sev = "low"
+
+        lat, lng = _cell_center(key)
+        alerts.append({
+            "lat": lat,
+            "lng": lng,
+            "type": "military_buildup",
+            "severity": sev,
+            "score": _severity_score(sev),
+            "drivers": drivers[:3],
+            "cell_size": _CELL_SIZE,
+        })
+
+    return alerts
+
+
+# ---------------------------------------------------------------------------
+# Infrastructure Cascade: outage + KiwiSDR co-location
+#
+# Power plants are removed from this detector — with 35K plants globally,
+# virtually every 2° cell contains one, making every outage a false hit.
+# KiwiSDR receivers (~300 worldwide) are sparse enough to be meaningful:
+# an outage in the same cell as a KiwiSDR indicates real infrastructure
+# disruption affecting radio monitoring capability.
+# ---------------------------------------------------------------------------
+
+
+def _detect_infra_cascades(data: dict) -> list[dict]:
+    internet_outages = data.get("internet_outages") or []
+    kiwisdr = data.get("kiwisdr") or []
+
+    if not kiwisdr:
+        return []
+
+    # Build set of cells with KiwiSDR receivers
+    kiwi_cells: set[str] = set()
+    for k in kiwisdr:
+        lat, lng = k.get("lat"), k.get("lon") or k.get("lng")
+        if lat is not None and lng is not None:
+            try:
+                kiwi_cells.add(_cell_key(float(lat), float(lng)))
+            except (ValueError, TypeError):
+                pass
+
+    if not kiwi_cells:
+        return []
+
+    alerts: list[dict] = []
+    for o in internet_outages:
+        lat = o.get("lat") or o.get("latitude")
+        lng = o.get("lng") or o.get("lon") or o.get("longitude")
+        if lat is None or lng is None:
+            continue
+        try:
+            key = _cell_key(float(lat), float(lng))
+        except (ValueError, TypeError):
+            continue
+
+        if key not in kiwi_cells:
+            continue
+
+        pct = _outage_pct(o)
+        drivers = [f"Internet outage{f' {pct:.0f}%' if pct else ''}",
+                    "KiwiSDR receivers in affected zone"]
+
+        lat_c, lng_c = _cell_center(key)
+        alerts.append({
+            "lat": lat_c,
+            "lng": lng_c,
+            "type": "infra_cascade",
+            "severity": "medium",
+            "score": _severity_score("medium"),
+            "drivers": drivers,
+            "cell_size": _CELL_SIZE,
+        })
+
+    return alerts
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def compute_correlations(data: dict) -> list[dict]:
+    """Run all correlation detectors and return merged alert list."""
+    alerts: list[dict] = []
+
+    try:
+        alerts.extend(_detect_rf_anomalies(data))
+    except Exception as e:
+        logger.error("Correlation engine RF anomaly error: %s", e)
+
+    try:
+        alerts.extend(_detect_military_buildups(data))
+    except Exception as e:
+        logger.error("Correlation engine military buildup error: %s", e)
+
+    try:
+        alerts.extend(_detect_infra_cascades(data))
+    except Exception as e:
+        logger.error("Correlation engine infra cascade error: %s", e)
+
+    rf = sum(1 for a in alerts if a["type"] == "rf_anomaly")
+    mil = sum(1 for a in alerts if a["type"] == "military_buildup")
+    infra = sum(1 for a in alerts if a["type"] == "infra_cascade")
+    if alerts:
+        logger.info(
+            "Correlations: %d alerts (%d rf, %d mil, %d infra)",
+            len(alerts), rf, mil, infra,
+        )
+
+    return alerts

backend/services/env_check.py

@@ -0,0 +1,291 @@
+"""Startup environment validation — called once in the FastAPI lifespan hook.
+
+Ensures required env vars are present before the scheduler starts.
+Logs warnings for optional keys that degrade functionality when missing.
+Audits security-critical config for dangerous combinations.
+"""
+
+import os
+import secrets
+import sys
+import time
+import logging
+from pathlib import Path
+from services.config import get_settings
+
+logger = logging.getLogger(__name__)
+
+# Keys grouped by criticality
+_REQUIRED = {
+    # Empty for now — add keys here only if the app literally cannot function without them
+}
+
+_CRITICAL_WARN = {
+    "ADMIN_KEY": "Authentication for /api/settings and /api/system/update — endpoints are UNPROTECTED without it!",
+}
+
+_OPTIONAL = {
+    "AIS_API_KEY": "AIS vessel streaming (ships layer will be empty without it)",
+    "OPENSKY_CLIENT_ID": "OpenSky OAuth2 — gap-fill flights in Africa/Asia/LatAm",
+    "OPENSKY_CLIENT_SECRET": "OpenSky OAuth2 — gap-fill flights in Africa/Asia/LatAm",
+    "LTA_ACCOUNT_KEY": "Singapore LTA traffic cameras (CCTV layer)",
+    "PUBLIC_API_KEY": "Optional client auth for public endpoints (recommended for exposed deployments)",
+}
+
+
+def _invalid_dm_token_pepper_reason(value: str) -> str:
+    raw = str(value or "").strip()
+    lowered = raw.lower()
+    if not raw:
+        return "empty"
+    if lowered in {"change-me", "changeme"}:
+        return "placeholder"
+    if len(raw) < 16:
+        return "too short"
+    return ""
+
+
+def _invalid_peer_push_secret_reason(value: str) -> str:
+    raw = str(value or "").strip()
+    lowered = raw.lower()
+    if not raw:
+        return "empty"
+    if lowered in {"change-me", "changeme"}:
+        return "placeholder"
+    if len(raw) < 16:
+        return "too short"
+    return ""
+
+
+_PEPPER_FILE = Path(__file__).resolve().parents[1] / "data" / "dm_token_pepper.key"
+
+
+def _ensure_dm_token_pepper(settings) -> str:
+    token_pepper = str(getattr(settings, "MESH_DM_TOKEN_PEPPER", "") or "").strip()
+    pepper_reason = _invalid_dm_token_pepper_reason(token_pepper)
+    if not pepper_reason:
+        return token_pepper
+
+    # Try loading a previously persisted pepper before generating a new one.
+    try:
+        from services.mesh.mesh_secure_storage import read_secure_json
+
+        stored = read_secure_json(_PEPPER_FILE, lambda: {})
+        stored_pepper = str(stored.get("pepper", "") or "").strip()
+        if stored_pepper and not _invalid_dm_token_pepper_reason(stored_pepper):
+            os.environ["MESH_DM_TOKEN_PEPPER"] = stored_pepper
+            get_settings.cache_clear()
+            logger.info("Loaded persisted DM token pepper from %s", _PEPPER_FILE.name)
+            return stored_pepper
+    except Exception:
+        pass
+
+    generated = secrets.token_hex(32)
+    os.environ["MESH_DM_TOKEN_PEPPER"] = generated
+    get_settings.cache_clear()
+    log_fn = logger.warning if bool(getattr(settings, "MESH_DEBUG_MODE", False)) else logger.critical
+    log_fn(
+        "⚠️  SECURITY: MESH_DM_TOKEN_PEPPER is invalid (%s) — mailbox tokens "
+        "would be predictably derivable. Auto-generated a random pepper for "
+        "this session.",
+        pepper_reason,
+    )
+
+    # Persist so the same pepper survives restarts.
+    try:
+        from services.mesh.mesh_secure_storage import write_secure_json
+
+        _PEPPER_FILE.parent.mkdir(parents=True, exist_ok=True)
+        write_secure_json(_PEPPER_FILE, {"pepper": generated, "generated_at": int(time.time())})
+        logger.info("Persisted auto-generated DM token pepper to %s", _PEPPER_FILE.name)
+    except Exception:
+        logger.warning("Could not persist auto-generated DM token pepper to disk — will regenerate on next restart")
+
+    return generated
+
+
+def _peer_push_secret_required(settings) -> bool:
+    relay_peers = str(getattr(settings, "MESH_RELAY_PEERS", "") or "").strip()
+    rns_peers = str(getattr(settings, "MESH_RNS_PEERS", "") or "").strip()
+    return bool(getattr(settings, "MESH_RNS_ENABLED", False) or relay_peers or rns_peers)
+
+
+def get_security_posture_warnings(settings=None) -> list[str]:
+    snapshot = settings or get_settings()
+    warnings: list[str] = []
+
+    admin_key = str(getattr(snapshot, "ADMIN_KEY", "") or "").strip()
+    allow_insecure = bool(getattr(snapshot, "ALLOW_INSECURE_ADMIN", False))
+    if allow_insecure and not admin_key:
+        warnings.append(
+            "ALLOW_INSECURE_ADMIN=true with no ADMIN_KEY leaves admin and Wormhole endpoints unauthenticated."
+        )
+
+    if not bool(getattr(snapshot, "MESH_STRICT_SIGNATURES", True)):
+        warnings.append(
+            "MESH_STRICT_SIGNATURES=false is deprecated and ignored; signature enforcement remains mandatory."
+        )
+
+    peer_secret = str(getattr(snapshot, "MESH_PEER_PUSH_SECRET", "") or "").strip()
+    peer_secret_reason = _invalid_peer_push_secret_reason(peer_secret)
+    if _peer_push_secret_required(snapshot) and peer_secret_reason:
+        warnings.append(
+            "MESH_PEER_PUSH_SECRET is invalid "
+            f"({peer_secret_reason}) while relay or RNS peers are enabled; private peer authentication, opaque gate forwarding, and voter blinding are not secure-by-default."
+        )
+
+    if os.name != "nt" and bool(getattr(snapshot, "MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK", False)):
+        warnings.append(
+            "MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK=true stores Wormhole keys in raw local files on this platform."
+        )
+
+    if bool(getattr(snapshot, "MESH_RNS_ENABLED", False)) and int(getattr(snapshot, "MESH_RNS_COVER_INTERVAL_S", 0) or 0) <= 0:
+        warnings.append(
+            "MESH_RNS_COVER_INTERVAL_S<=0 disables RNS cover traffic outside high-privacy mode, making quiet-node traffic analysis easier."
+        )
+
+    fallback_policy = str(getattr(snapshot, "MESH_PRIVATE_CLEARNET_FALLBACK", "block") or "block").strip().lower()
+    if fallback_policy == "allow":
+        warnings.append(
+            "MESH_PRIVATE_CLEARNET_FALLBACK=allow — private-tier messages may fall back to clearnet relay when Tor/RNS is unavailable."
+        )
+
+    metadata_persist = bool(getattr(snapshot, "MESH_DM_METADATA_PERSIST", True))
+    binding_ttl = int(getattr(snapshot, "MESH_DM_BINDING_TTL_DAYS", 7) or 7)
+    if metadata_persist and binding_ttl > 14:
+        warnings.append(
+            f"MESH_DM_BINDING_TTL_DAYS={binding_ttl} with MESH_DM_METADATA_PERSIST=true — long-lived mailbox binding metadata persists communication graph structure on disk."
+        )
+
+    return warnings
+
+
+def _audit_security_config(settings) -> None:
+    """Audit security-critical config combinations and log loud warnings.
+
+    This does not block startup (dev ergonomics), but makes dangerous
+    settings impossible to miss in the logs.
+    """
+    # ── 1. ALLOW_INSECURE_ADMIN without ADMIN_KEY ─────────────────────
+    admin_key = (getattr(settings, "ADMIN_KEY", "") or "").strip()
+    allow_insecure = bool(getattr(settings, "ALLOW_INSECURE_ADMIN", False))
+    if allow_insecure and not admin_key:
+        logger.critical(
+            "🚨 SECURITY: ALLOW_INSECURE_ADMIN=true with no ADMIN_KEY — "
+            "ALL admin/wormhole endpoints are completely unauthenticated. "
+            "This is acceptable ONLY for local development. "
+            "Set ADMIN_KEY for any networked or production deployment."
+        )
+
+    # ── 2. Signature enforcement ──────────────────────────────────────
+    mesh_strict = bool(getattr(settings, "MESH_STRICT_SIGNATURES", True))
+    if not mesh_strict:
+        logger.warning(
+            "⚠️  CONFIG: MESH_STRICT_SIGNATURES=false is deprecated and ignored — "
+            "runtime signature enforcement remains mandatory."
+        )
+
+    # ── 3. Empty DM token pepper ──────────────────────────────────────
+    _ensure_dm_token_pepper(settings)
+
+    # ── 4. Peer push secret / private-plane integrity ─────────────────
+    peer_secret = str(getattr(settings, "MESH_PEER_PUSH_SECRET", "") or "").strip()
+    peer_secret_reason = _invalid_peer_push_secret_reason(peer_secret)
+    if _peer_push_secret_required(settings) and peer_secret_reason:
+        log_fn = logger.warning if bool(getattr(settings, "MESH_DEBUG_MODE", False)) else logger.critical
+        log_fn(
+            "⚠️  SECURITY: MESH_PEER_PUSH_SECRET is invalid (%s) while relay or RNS peers are enabled — "
+            "private peer authentication, opaque gate forwarding, and voter blinding are not secure-by-default until it is set to a non-placeholder secret.",
+            peer_secret_reason,
+        )
+
+    # ── 5. Raw secure-storage fallback on non-Windows ────────────────
+    if os.name != "nt" and bool(getattr(settings, "MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK", False)):
+        log_fn = logger.warning if bool(getattr(settings, "MESH_DEBUG_MODE", False)) else logger.critical
+        log_fn(
+            "⚠️  SECURITY: MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK=true leaves Wormhole keys in raw local files. "
+            "Use this only for development/CI until a native keyring provider is available."
+        )
+
+    # ── 6. Disabled cover traffic outside forced high-privacy mode ─────────
+    if bool(getattr(settings, "MESH_RNS_ENABLED", False)) and int(getattr(settings, "MESH_RNS_COVER_INTERVAL_S", 0) or 0) <= 0:
+        logger.warning(
+            "⚠️  PRIVACY: MESH_RNS_COVER_INTERVAL_S<=0 disables background RNS cover traffic outside high-privacy mode. "
+            "Quiet nodes become easier to fingerprint by silence and burst timing."
+        )
+
+    # ── 7. Clearnet fallback policy ──────────────────────────────────
+    fallback_policy = str(getattr(settings, "MESH_PRIVATE_CLEARNET_FALLBACK", "block") or "block").strip().lower()
+    if fallback_policy == "allow":
+        logger.warning(
+            "⚠️  PRIVACY: MESH_PRIVATE_CLEARNET_FALLBACK=allow — private-tier messages will fall "
+            "back to clearnet relay when Tor/RNS is unavailable. Set to 'block' for safer defaults."
+        )
+
+
+def validate_env(*, strict: bool = True) -> bool:
+    """Validate environment variables at startup.
+
+    Args:
+        strict: If True, exit the process on missing required keys.
+                If False, only log errors (useful for tests).
+
+    Returns:
+        True if all required keys are present, False otherwise.
+    """
+    all_ok = True
+
+    settings = get_settings()
+
+    # Required keys — must be set
+    for key, desc in _REQUIRED.items():
+        value = getattr(settings, key, "")
+        if isinstance(value, str):
+            value = value.strip()
+        if not value:
+            logger.error(
+                "❌ REQUIRED env var %s is not set. %s\n"
+                "   Set it in .env or via Docker secrets (%s_FILE).",
+                key,
+                desc,
+                key,
+            )
+            all_ok = False
+
+    if not all_ok and strict:
+        logger.critical("Startup aborted — required environment variables are missing.")
+        sys.exit(1)
+
+    # Critical-warn keys — app works but security/functionality is degraded
+    for key, desc in _CRITICAL_WARN.items():
+        value = getattr(settings, key, "")
+        if isinstance(value, str):
+            value = value.strip()
+        if not value:
+            allow_insecure = bool(getattr(settings, "ALLOW_INSECURE_ADMIN", False))
+            logger.warning(
+                "⚠️  ADMIN_KEY is not set%s — %s",
+                " and ALLOW_INSECURE_ADMIN=true" if allow_insecure else "",
+                desc,
+            )
+            if not allow_insecure:
+                logger.critical(
+                    "🔓 CRITICAL: env var %s is not set — this MUST be set in production.",
+                    key,
+                )
+
+    # Optional keys — warn if missing
+    for key, desc in _OPTIONAL.items():
+        value = getattr(settings, key, "")
+        if isinstance(value, str):
+            value = value.strip()
+        if not value:
+            logger.warning("⚠️  Optional env var %s is not set — %s", key, desc)
+
+    # ── Security posture audit ────────────────────────────────────────
+    _audit_security_config(settings)
+
+    if all_ok:
+        logger.info("✅ Environment validation passed.")
+
+    return all_ok

backend/services/fetch_health.py

@@ -0,0 +1,94 @@
+"""Fetch health registry — tracks per-source success/failure counts and timings."""
+
+import logging
+import threading
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+from services.fetchers._store import _data_lock, source_freshness
+
+logger = logging.getLogger(__name__)
+
+_health: Dict[str, Dict[str, Any]] = {}
+_lock = threading.Lock()
+
+
+def _now_iso() -> str:
+    return datetime.utcnow().isoformat()
+
+
+def _update_source_freshness(source: str, *, ok: bool, error_msg: Optional[str] = None):
+    """Mirror health summary into shared store for visibility."""
+    with _data_lock:
+        entry = source_freshness.get(source, {})
+        if ok:
+            entry["last_ok"] = _now_iso()
+        else:
+            entry["last_error"] = _now_iso()
+            if error_msg:
+                entry["last_error_msg"] = error_msg[:200]
+        source_freshness[source] = entry
+
+
+def record_success(source: str, duration_s: Optional[float] = None, count: Optional[int] = None):
+    """Record a successful fetch for a source."""
+    now = _now_iso()
+    with _lock:
+        entry = _health.setdefault(
+            source,
+            {
+                "ok_count": 0,
+                "error_count": 0,
+                "last_ok": None,
+                "last_error": None,
+                "last_error_msg": None,
+                "last_duration_ms": None,
+                "avg_duration_ms": None,
+                "last_count": None,
+            },
+        )
+        entry["ok_count"] += 1
+        entry["last_ok"] = now
+        if duration_s is not None:
+            dur_ms = round(duration_s * 1000, 1)
+            entry["last_duration_ms"] = dur_ms
+            prev_avg = entry["avg_duration_ms"] or 0.0
+            n = entry["ok_count"]
+            entry["avg_duration_ms"] = round(((prev_avg * (n - 1)) + dur_ms) / n, 1)
+        if count is not None:
+            entry["last_count"] = count
+
+    _update_source_freshness(source, ok=True)
+
+
+def record_failure(source: str, error: Exception, duration_s: Optional[float] = None):
+    """Record a failed fetch for a source."""
+    now = _now_iso()
+    err_msg = str(error)
+    with _lock:
+        entry = _health.setdefault(
+            source,
+            {
+                "ok_count": 0,
+                "error_count": 0,
+                "last_ok": None,
+                "last_error": None,
+                "last_error_msg": None,
+                "last_duration_ms": None,
+                "avg_duration_ms": None,
+                "last_count": None,
+            },
+        )
+        entry["error_count"] += 1
+        entry["last_error"] = now
+        entry["last_error_msg"] = err_msg[:200]
+        if duration_s is not None:
+            entry["last_duration_ms"] = round(duration_s * 1000, 1)
+
+    _update_source_freshness(source, ok=False, error_msg=err_msg)
+
+
+def get_health_snapshot() -> Dict[str, Dict[str, Any]]:
+    """Return a snapshot of current fetch health state."""
+    with _lock:
+        return {k: dict(v) for k, v in _health.items()}

backend/services/fetchers/_store.py

@@ -0,0 +1,243 @@
+"""Shared in-memory data store for all fetcher modules.
+
+Central location for latest_data, source_timestamps, and the data lock.
+Every fetcher imports from here instead of maintaining its own copy.
+"""
+
+import threading
+import logging
+from datetime import datetime
+from typing import Any, Dict, List, Optional, TypedDict
+
+logger = logging.getLogger("services.data_fetcher")
+
+
+class DashboardData(TypedDict, total=False):
+    """Schema for the in-memory data store. Catches key typos at dev time."""
+
+    last_updated: Optional[str]
+    news: List[Dict[str, Any]]
+    stocks: Dict[str, Any]
+    oil: Dict[str, Any]
+    commercial_flights: List[Dict[str, Any]]
+    private_flights: List[Dict[str, Any]]
+    private_jets: List[Dict[str, Any]]
+    flights: List[Dict[str, Any]]
+    ships: List[Dict[str, Any]]
+    military_flights: List[Dict[str, Any]]
+    tracked_flights: List[Dict[str, Any]]
+    cctv: List[Dict[str, Any]]
+    weather: Optional[Dict[str, Any]]
+    earthquakes: List[Dict[str, Any]]
+    uavs: List[Dict[str, Any]]
+    frontlines: Optional[Any]
+    gdelt: List[Dict[str, Any]]
+    liveuamap: List[Dict[str, Any]]
+    kiwisdr: List[Dict[str, Any]]
+    space_weather: Optional[Dict[str, Any]]
+    internet_outages: List[Dict[str, Any]]
+    firms_fires: List[Dict[str, Any]]
+    datacenters: List[Dict[str, Any]]
+    airports: List[Dict[str, Any]]
+    gps_jamming: List[Dict[str, Any]]
+    satellites: List[Dict[str, Any]]
+    satellite_source: str
+    prediction_markets: List[Dict[str, Any]]
+    sigint: List[Dict[str, Any]]
+    sigint_totals: Dict[str, Any]
+    mesh_channel_stats: Dict[str, Any]
+    meshtastic_map_nodes: List[Dict[str, Any]]
+    meshtastic_map_fetched_at: Optional[float]
+    weather_alerts: List[Dict[str, Any]]
+    air_quality: List[Dict[str, Any]]
+    volcanoes: List[Dict[str, Any]]
+    fishing_activity: List[Dict[str, Any]]
+    satnogs_stations: List[Dict[str, Any]]
+    satnogs_observations: List[Dict[str, Any]]
+    tinygs_satellites: List[Dict[str, Any]]
+    ukraine_alerts: List[Dict[str, Any]]
+    power_plants: List[Dict[str, Any]]
+    viirs_change_nodes: List[Dict[str, Any]]
+    fimi: Dict[str, Any]
+    psk_reporter: List[Dict[str, Any]]
+    correlations: List[Dict[str, Any]]
+
+
+# In-memory store
+latest_data: DashboardData = {
+    "last_updated": None,
+    "news": [],
+    "stocks": {},
+    "oil": {},
+    "flights": [],
+    "ships": [],
+    "military_flights": [],
+    "tracked_flights": [],
+    "cctv": [],
+    "weather": None,
+    "earthquakes": [],
+    "uavs": [],
+    "frontlines": None,
+    "gdelt": [],
+    "liveuamap": [],
+    "kiwisdr": [],
+    "space_weather": None,
+    "internet_outages": [],
+    "firms_fires": [],
+    "datacenters": [],
+    "military_bases": [],
+    "prediction_markets": [],
+    "sigint": [],
+    "sigint_totals": {},
+    "mesh_channel_stats": {},
+    "meshtastic_map_nodes": [],
+    "meshtastic_map_fetched_at": None,
+    "weather_alerts": [],
+    "air_quality": [],
+    "volcanoes": [],
+    "fishing_activity": [],
+    "satnogs_stations": [],
+    "satnogs_observations": [],
+    "tinygs_satellites": [],
+    "ukraine_alerts": [],
+    "power_plants": [],
+    "viirs_change_nodes": [],
+    "fimi": {},
+    "psk_reporter": [],
+    "correlations": [],
+}
+
+# Per-source freshness timestamps
+source_timestamps = {}
+
+# Per-source health/freshness metadata (last ok/error)
+source_freshness: dict[str, dict] = {}
+
+
+def _mark_fresh(*keys):
+    """Record the current UTC time for one or more data source keys."""
+    now = datetime.utcnow().isoformat()
+    with _data_lock:
+        for k in keys:
+            source_timestamps[k] = now
+
+
+# Thread lock for safe reads/writes to latest_data
+_data_lock = threading.Lock()
+
+# Monotonic version counter — incremented on each data update cycle.
+# Used for cheap ETag generation instead of MD5-hashing the full response.
+_data_version: int = 0
+
+
+def bump_data_version() -> None:
+    """Increment the data version counter after a fetch cycle completes."""
+    global _data_version
+    _data_version += 1
+
+
+def get_data_version() -> int:
+    """Return the current data version (for ETag generation)."""
+    return _data_version
+
+
+_active_layers_version: int = 0
+
+
+def bump_active_layers_version() -> None:
+    """Increment the active-layer version when frontend toggles change response shape."""
+    global _active_layers_version
+    _active_layers_version += 1
+
+
+def get_active_layers_version() -> int:
+    """Return the current active-layer version (for ETag generation)."""
+    return _active_layers_version
+
+
+def get_latest_data_subset(*keys: str) -> DashboardData:
+    """Return a shallow snapshot of only the requested top-level keys.
+
+    This avoids cloning the entire dashboard store for endpoints that only need
+    a small tier-specific subset.
+    """
+    with _data_lock:
+        snap: DashboardData = {}
+        for key in keys:
+            value = latest_data.get(key)
+            if isinstance(value, list):
+                snap[key] = list(value)
+            elif isinstance(value, dict):
+                snap[key] = dict(value)
+            else:
+                snap[key] = value
+        return snap
+
+
+def get_latest_data_subset_refs(*keys: str) -> DashboardData:
+    """Return direct top-level references for read-only hot paths.
+
+    Writers replace top-level values under the lock instead of mutating them
+    in place, so readers can safely use these references after releasing the
+    lock as long as they do not modify them.
+    """
+    with _data_lock:
+        snap: DashboardData = {}
+        for key in keys:
+            snap[key] = latest_data.get(key)
+        return snap
+
+
+def get_source_timestamps_snapshot() -> dict[str, str]:
+    """Return a stable copy of per-source freshness timestamps."""
+    with _data_lock:
+        return dict(source_timestamps)
+
+
+# ---------------------------------------------------------------------------
+# Active layers — frontend POSTs toggles, fetchers check before running.
+# Keep these aligned with the dashboard's default layer state so startup does
+# not fetch heavyweight feeds the UI starts with disabled.
+# ---------------------------------------------------------------------------
+active_layers: dict[str, bool] = {
+    "flights": True,
+    "private": True,
+    "jets": True,
+    "military": True,
+    "tracked": True,
+    "satellites": True,
+    "ships_military": True,
+    "ships_cargo": True,
+    "ships_civilian": True,
+    "ships_passenger": True,
+    "ships_tracked_yachts": True,
+    "earthquakes": True,
+    "cctv": True,
+    "ukraine_frontline": True,
+    "global_incidents": True,
+    "gps_jamming": True,
+    "kiwisdr": True,
+    "scanners": True,
+    "firms": True,
+    "internet_outages": True,
+    "datacenters": True,
+    "military_bases": True,
+    "sigint_meshtastic": True,
+    "sigint_aprs": True,
+    "weather_alerts": True,
+    "air_quality": True,
+    "volcanoes": True,
+    "fishing_activity": True,
+    "satnogs": True,
+    "tinygs": True,
+    "ukraine_alerts": True,
+    "power_plants": False,
+    "viirs_nightlights": False,
+    "psk_reporter": True,
+    "correlations": True,
+}
+
+
+def is_any_active(*layer_names: str) -> bool:
+    """Return True if any of the given layer names is currently active."""
+    return any(active_layers.get(name, True) for name in layer_names)

backend/services/fetchers/earth_observation.py

@@ -0,0 +1,598 @@
+"""Earth-observation fetchers — earthquakes, FIRMS fires, space weather, weather radar,
+severe weather alerts, air quality, volcanoes."""
+
+import csv
+import io
+import json
+import logging
+import os
+import time
+import heapq
+from datetime import datetime
+from pathlib import Path
+from services.network_utils import fetch_with_curl
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Earthquakes (USGS)
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=1, base_delay=1)
+def fetch_earthquakes():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("earthquakes"):
+        return
+    quakes = []
+    try:
+        url = "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/2.5_day.geojson"
+        response = fetch_with_curl(url, timeout=10)
+        if response.status_code == 200:
+            features = response.json().get("features", [])
+            for f in features[:50]:
+                mag = f["properties"]["mag"]
+                lng, lat, depth = f["geometry"]["coordinates"]
+                quakes.append(
+                    {
+                        "id": f["id"],
+                        "mag": mag,
+                        "lat": lat,
+                        "lng": lng,
+                        "place": f["properties"]["place"],
+                    }
+                )
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching earthquakes: {e}")
+    with _data_lock:
+        latest_data["earthquakes"] = quakes
+    if quakes:
+        _mark_fresh("earthquakes")
+
+
+# ---------------------------------------------------------------------------
+# NASA FIRMS Fires
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=1, base_delay=2)
+def fetch_firms_fires():
+    """Fetch global fire/thermal anomalies from NASA FIRMS (NOAA-20 VIIRS, 24h, no key needed)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("firms"):
+        return
+    fires = []
+    try:
+        url = "https://firms.modaps.eosdis.nasa.gov/data/active_fire/noaa-20-viirs-c2/csv/J1_VIIRS_C2_Global_24h.csv"
+        response = fetch_with_curl(url, timeout=30)
+        if response.status_code == 200:
+            reader = csv.DictReader(io.StringIO(response.text))
+            all_rows = []
+            for row in reader:
+                try:
+                    lat = float(row.get("latitude", 0))
+                    lng = float(row.get("longitude", 0))
+                    frp = float(row.get("frp", 0))
+                    conf = row.get("confidence", "nominal")
+                    daynight = row.get("daynight", "")
+                    bright = float(row.get("bright_ti4", 0))
+                    all_rows.append(
+                        {
+                            "lat": lat,
+                            "lng": lng,
+                            "frp": frp,
+                            "brightness": bright,
+                            "confidence": conf,
+                            "daynight": daynight,
+                            "acq_date": row.get("acq_date", ""),
+                            "acq_time": row.get("acq_time", ""),
+                        }
+                    )
+                except (ValueError, TypeError):
+                    continue
+            fires = heapq.nlargest(5000, all_rows, key=lambda x: x["frp"])
+        logger.info(f"FIRMS fires: {len(fires)} hotspots (from {response.status_code})")
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching FIRMS fires: {e}")
+    with _data_lock:
+        latest_data["firms_fires"] = fires
+    if fires:
+        _mark_fresh("firms_fires")
+
+
+# ---------------------------------------------------------------------------
+# NASA FIRMS Country-Scoped Fires (enriches global CSV with conflict zones)
+# ---------------------------------------------------------------------------
+# Conflict-zone countries of interest for higher-detail fire/thermal data
+_FIRMS_COUNTRIES = ["ISR", "IRN", "IRQ", "LBN", "SYR", "YEM", "SAU", "UKR", "RUS", "TUR"]
+
+
+@with_retry(max_retries=1, base_delay=2)
+def fetch_firms_country_fires():
+    """Fetch country-scoped fire hotspots from NASA FIRMS MAP_KEY API.
+
+    Supplements the global CSV feed with more granular data for conflict zones.
+    Merges results into the existing firms_fires data store (no new frontend key).
+    Requires FIRMS_MAP_KEY env var (free from NASA Earthdata). Skips if not set.
+    """
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("firms"):
+        return
+
+    map_key = os.environ.get("FIRMS_MAP_KEY", "")
+    if not map_key:
+        logger.debug("FIRMS_MAP_KEY not set, skipping country-scoped FIRMS fetch")
+        return
+
+    # Build a set of existing (lat, lng) rounded to 0.01° for dedup
+    with _data_lock:
+        existing = set()
+        for f in latest_data.get("firms_fires", []):
+            existing.add((round(f["lat"], 2), round(f["lng"], 2)))
+
+    new_fires = []
+    for country in _FIRMS_COUNTRIES:
+        try:
+            url = (
+                f"https://firms.modaps.eosdis.nasa.gov/api/country/csv/"
+                f"{map_key}/VIIRS_NOAA20_NRT/{country}/1"
+            )
+            response = fetch_with_curl(url, timeout=15)
+            if response.status_code != 200:
+                logger.debug(f"FIRMS country {country}: HTTP {response.status_code}")
+                continue
+
+            reader = csv.DictReader(io.StringIO(response.text))
+            for row in reader:
+                try:
+                    lat = float(row.get("latitude", 0))
+                    lng = float(row.get("longitude", 0))
+                    key = (round(lat, 2), round(lng, 2))
+                    if key in existing:
+                        continue  # Already in global data
+                    existing.add(key)
+
+                    frp = float(row.get("frp", 0))
+                    new_fires.append({
+                        "lat": lat,
+                        "lng": lng,
+                        "frp": frp,
+                        "brightness": float(row.get("bright_ti4", 0)),
+                        "confidence": row.get("confidence", "nominal"),
+                        "daynight": row.get("daynight", ""),
+                        "acq_date": row.get("acq_date", ""),
+                        "acq_time": row.get("acq_time", ""),
+                    })
+                except (ValueError, TypeError):
+                    continue
+
+        except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+            logger.debug(f"FIRMS country {country} failed: {e}")
+
+    if new_fires:
+        with _data_lock:
+            current = latest_data.get("firms_fires", [])
+            merged = current + new_fires
+            # Keep top 6000 by FRP (slightly more than global-only cap of 5000)
+            if len(merged) > 6000:
+                merged = heapq.nlargest(6000, merged, key=lambda x: x["frp"])
+            latest_data["firms_fires"] = merged
+        logger.info(f"FIRMS country enrichment: +{len(new_fires)} fires from {len(_FIRMS_COUNTRIES)} countries")
+        _mark_fresh("firms_fires")
+    else:
+        logger.debug("FIRMS country enrichment: no new fires found")
+
+
+# ---------------------------------------------------------------------------
+# Space Weather (NOAA SWPC)
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=1, base_delay=1)
+def fetch_space_weather():
+    """Fetch NOAA SWPC Kp index and recent solar events."""
+    try:
+        kp_resp = fetch_with_curl(
+            "https://services.swpc.noaa.gov/json/planetary_k_index_1m.json", timeout=10
+        )
+        kp_value = None
+        kp_text = "QUIET"
+        if kp_resp.status_code == 200:
+            kp_data = kp_resp.json()
+            if kp_data:
+                latest_kp = kp_data[-1]
+                kp_value = float(latest_kp.get("kp_index", 0))
+                if kp_value >= 7:
+                    kp_text = f"STORM G{min(int(kp_value) - 4, 5)}"
+                elif kp_value >= 5:
+                    kp_text = f"STORM G{min(int(kp_value) - 4, 5)}"
+                elif kp_value >= 4:
+                    kp_text = "ACTIVE"
+                elif kp_value >= 3:
+                    kp_text = "UNSETTLED"
+
+        events = []
+        ev_resp = fetch_with_curl(
+            "https://services.swpc.noaa.gov/json/edited_events.json", timeout=10
+        )
+        if ev_resp.status_code == 200:
+            all_events = ev_resp.json()
+            for ev in all_events[-10:]:
+                events.append(
+                    {
+                        "type": ev.get("type", ""),
+                        "begin": ev.get("begin", ""),
+                        "end": ev.get("end", ""),
+                        "classtype": ev.get("classtype", ""),
+                    }
+                )
+
+        with _data_lock:
+            latest_data["space_weather"] = {
+                "kp_index": kp_value,
+                "kp_text": kp_text,
+                "events": events,
+            }
+        _mark_fresh("space_weather")
+        logger.info(f"Space weather: Kp={kp_value} ({kp_text}), {len(events)} events")
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching space weather: {e}")
+
+
+# ---------------------------------------------------------------------------
+# Weather Radar (RainViewer)
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=1, base_delay=1)
+def fetch_weather():
+    try:
+        url = "https://api.rainviewer.com/public/weather-maps.json"
+        response = fetch_with_curl(url, timeout=10)
+        if response.status_code == 200:
+            data = response.json()
+            if "radar" in data and "past" in data["radar"]:
+                latest_time = data["radar"]["past"][-1]["time"]
+                with _data_lock:
+                    latest_data["weather"] = {
+                        "time": latest_time,
+                        "host": data.get("host", "https://tilecache.rainviewer.com"),
+                    }
+                _mark_fresh("weather")
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching weather: {e}")
+
+
+# ---------------------------------------------------------------------------
+# NOAA/NWS Severe Weather Alerts
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=1, base_delay=2)
+def fetch_weather_alerts():
+    """Fetch active severe weather alerts from NOAA/NWS (US coverage, GeoJSON polygons)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("weather_alerts"):
+        return
+    alerts = []
+    try:
+        url = "https://api.weather.gov/alerts/active?status=actual"
+        headers = {
+            "User-Agent": "(ShadowBroker OSINT Dashboard, github.com/BigBodyCobain/Shadowbroker)",
+            "Accept": "application/geo+json",
+        }
+        response = fetch_with_curl(url, timeout=15, headers=headers)
+        if response.status_code == 200:
+            features = response.json().get("features", [])
+            for f in features:
+                props = f.get("properties", {})
+                geom = f.get("geometry")
+                if not geom:
+                    continue  # skip zone-only alerts with no polygon
+                alerts.append(
+                    {
+                        "id": props.get("id", ""),
+                        "event": props.get("event", ""),
+                        "severity": props.get("severity", "Unknown"),
+                        "certainty": props.get("certainty", ""),
+                        "urgency": props.get("urgency", ""),
+                        "headline": props.get("headline", ""),
+                        "description": (props.get("description", "") or "")[:300],
+                        "expires": props.get("expires", ""),
+                        "geometry": geom,
+                    }
+                )
+        logger.info(f"Weather alerts: {len(alerts)} active (with polygons)")
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching weather alerts: {e}")
+    with _data_lock:
+        latest_data["weather_alerts"] = alerts
+    if alerts:
+        _mark_fresh("weather_alerts")
+
+
+# ---------------------------------------------------------------------------
+# Air Quality (OpenAQ v3)
+# ---------------------------------------------------------------------------
+def _pm25_to_aqi(pm25: float) -> int:
+    """Convert PM2.5 concentration (µg/m³) to US EPA AQI."""
+    breakpoints = [
+        (0, 12.0, 0, 50),
+        (12.1, 35.4, 51, 100),
+        (35.5, 55.4, 101, 150),
+        (55.5, 150.4, 151, 200),
+        (150.5, 250.4, 201, 300),
+        (250.5, 500.4, 301, 500),
+    ]
+    for c_lo, c_hi, i_lo, i_hi in breakpoints:
+        if pm25 <= c_hi:
+            return round(((i_hi - i_lo) / (c_hi - c_lo)) * (pm25 - c_lo) + i_lo)
+    return 500
+
+
+@with_retry(max_retries=1, base_delay=2)
+def fetch_air_quality():
+    """Fetch global air quality stations with PM2.5 data from OpenAQ."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("air_quality"):
+        return
+    stations = []
+    api_key = os.environ.get("OPENAQ_API_KEY", "")
+    if not api_key:
+        logger.debug("OPENAQ_API_KEY not set, skipping air quality fetch")
+        return
+    try:
+        url = "https://api.openaq.org/v3/locations?limit=5000&parameter_id=2&order_by=datetime&sort_order=desc"
+        headers = {"X-API-Key": api_key}
+        response = fetch_with_curl(url, timeout=30, headers=headers)
+        if response.status_code == 200:
+            results = response.json().get("results", [])
+            for loc in results:
+                coords = loc.get("coordinates", {})
+                lat = coords.get("latitude")
+                lng = coords.get("longitude")
+                if lat is None or lng is None:
+                    continue
+                pm25 = None
+                for p in loc.get("parameters", []):
+                    if p.get("id") == 2:
+                        pm25 = p.get("lastValue")
+                        break
+                if pm25 is None:
+                    continue
+                pm25_val = float(pm25)
+                if pm25_val < 0:
+                    continue
+                stations.append(
+                    {
+                        "id": loc.get("id"),
+                        "name": loc.get("name", "Unknown"),
+                        "lat": lat,
+                        "lng": lng,
+                        "pm25": round(pm25_val, 1),
+                        "aqi": _pm25_to_aqi(pm25_val),
+                        "country": loc.get("country", {}).get("code", ""),
+                    }
+                )
+        logger.info(f"Air quality: {len(stations)} stations")
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching air quality: {e}")
+    with _data_lock:
+        latest_data["air_quality"] = stations
+    if stations:
+        _mark_fresh("air_quality")
+
+
+# ---------------------------------------------------------------------------
+# Volcanoes (Smithsonian Global Volcanism Program)
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=2, base_delay=5)
+def fetch_volcanoes():
+    """Fetch Holocene volcanoes from Smithsonian GVP WFS (static reference data)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("volcanoes"):
+        return
+    volcanoes = []
+    try:
+        url = (
+            "https://webservices.volcano.si.edu/geoserver/GVP-VOTW/wfs"
+            "?service=WFS&version=2.0.0&request=GetFeature"
+            "&typeName=GVP-VOTW:E3WebApp_HoloceneVolcanoes"
+            "&outputFormat=application/json"
+        )
+        response = fetch_with_curl(url, timeout=30)
+        if response.status_code == 200:
+            features = response.json().get("features", [])
+            for f in features:
+                props = f.get("properties", {})
+                geom = f.get("geometry", {})
+                coords = geom.get("coordinates", [None, None])
+                if coords[0] is None:
+                    continue
+                last_eruption = props.get("LastEruption")
+                last_eruption_year = None
+                if last_eruption is not None:
+                    try:
+                        last_eruption_year = int(last_eruption)
+                    except (ValueError, TypeError):
+                        pass
+                volcanoes.append(
+                    {
+                        "name": props.get("VolcanoName", "Unknown"),
+                        "type": props.get("VolcanoType", ""),
+                        "country": props.get("Country", ""),
+                        "region": props.get("TectonicSetting", ""),
+                        "elevation": props.get("Elevation", 0),
+                        "last_eruption_year": last_eruption_year,
+                        "lat": coords[1],
+                        "lng": coords[0],
+                    }
+                )
+        logger.info(f"Volcanoes: {len(volcanoes)} Holocene volcanoes loaded")
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching volcanoes: {e}")
+    with _data_lock:
+        latest_data["volcanoes"] = volcanoes
+    if volcanoes:
+        _mark_fresh("volcanoes")
+
+
+# ---------------------------------------------------------------------------
+# VIIRS Night Lights Change Detection (Google Earth Engine — optional)
+# ---------------------------------------------------------------------------
+_VIIRS_CACHE_PATH = Path(__file__).parent.parent.parent / "data" / "viirs_change_nodes.json"
+_VIIRS_CACHE_MAX_AGE_S = 86400  # 24 hours
+
+# Conflict-zone AOIs: (name, south, west, north, east)
+_VIIRS_AOIS = [
+    ("Gaza Strip", 31.2, 34.2, 31.6, 34.6),
+    ("Kharkiv Oblast", 48.5, 35.0, 50.5, 38.5),
+    ("Donetsk Oblast", 47.0, 36.5, 49.0, 39.5),
+    ("Zaporizhzhia Oblast", 46.5, 34.5, 48.5, 37.0),
+    ("Aleppo", 35.8, 36.5, 36.5, 37.5),
+    ("Khartoum", 15.2, 32.2, 15.9, 32.9),
+    ("Sana'a", 14.9, 43.8, 15.6, 44.5),
+    ("Mosul", 36.0, 42.8, 36.7, 43.5),
+    ("Mariupol", 46.9, 37.2, 47.3, 37.8),
+    ("Southern Lebanon", 33.0, 35.0, 33.5, 36.0),
+]
+
+_VIIRS_SEVERITY_THRESHOLDS = [
+    (-100, -70, "severe"),
+    (-70, -50, "high"),
+    (-50, -30, "moderate"),
+    (30, 100, "growth"),
+    (100, 500, "rapid_growth"),
+]
+
+
+def _classify_viirs_severity(pct_change: float):
+    for lo, hi, label in _VIIRS_SEVERITY_THRESHOLDS:
+        if lo <= pct_change <= hi:
+            return label
+    return None
+
+
+def _load_viirs_stale_cache():
+    """Load stale cache if available (when GEE is not configured)."""
+    if _VIIRS_CACHE_PATH.exists():
+        try:
+            cached = json.loads(_VIIRS_CACHE_PATH.read_text(encoding="utf-8"))
+            with _data_lock:
+                latest_data["viirs_change_nodes"] = cached
+            _mark_fresh("viirs_change_nodes")
+            logger.info(f"VIIRS change nodes: loaded {len(cached)} from stale cache")
+        except Exception:
+            pass
+
+
+@with_retry(max_retries=1, base_delay=5)
+def fetch_viirs_change_nodes():
+    """Compute VIIRS nighttime radiance change nodes via GEE (optional)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("viirs_nightlights"):
+        return
+
+    # Check cache freshness first
+    if _VIIRS_CACHE_PATH.exists():
+        age = time.time() - _VIIRS_CACHE_PATH.stat().st_mtime
+        if age < _VIIRS_CACHE_MAX_AGE_S:
+            try:
+                cached = json.loads(_VIIRS_CACHE_PATH.read_text(encoding="utf-8"))
+                with _data_lock:
+                    latest_data["viirs_change_nodes"] = cached
+                _mark_fresh("viirs_change_nodes")
+                logger.info(f"VIIRS change nodes: loaded {len(cached)} from cache (age {age:.0f}s)")
+                return
+            except Exception as e:
+                logger.warning(f"VIIRS cache read failed: {e}")
+
+    # Try importing earthengine-api (optional dependency)
+    try:
+        import ee
+    except ImportError:
+        logger.debug("earthengine-api not installed, skipping VIIRS change detection")
+        _load_viirs_stale_cache()
+        return
+
+    # Authenticate with service account
+    sa_key_path = os.environ.get("GEE_SERVICE_ACCOUNT_KEY", "")
+    if not sa_key_path:
+        logger.debug("GEE_SERVICE_ACCOUNT_KEY not set, skipping VIIRS change detection")
+        _load_viirs_stale_cache()
+        return
+
+    try:
+        credentials = ee.ServiceAccountCredentials(None, key_file=sa_key_path)
+        ee.Initialize(credentials)
+    except Exception as e:
+        logger.error(f"GEE authentication failed: {e}")
+        _load_viirs_stale_cache()
+        return
+
+    # Compute change nodes for each AOI
+    nodes = []
+    viirs = ee.ImageCollection("NOAA/VIIRS/DNB/MONTHLY_V1/VCMCFG").select("avg_rad")
+
+    for aoi_name, s_lat, w_lng, n_lat, e_lng in _VIIRS_AOIS:
+        try:
+            aoi = ee.Geometry.Rectangle([w_lng, s_lat, e_lng, n_lat])
+
+            # Most recent available date
+            now = ee.Date(datetime.utcnow().isoformat()[:10])
+
+            # Current: 12-month rolling mean ending now
+            current = viirs.filterDate(now.advance(-12, "month"), now).mean().clip(aoi)
+
+            # Baseline: 12-month mean ending 12 months ago
+            baseline = viirs.filterDate(
+                now.advance(-24, "month"), now.advance(-12, "month")
+            ).mean().clip(aoi)
+
+            # Floor baseline at 0.5 nW/cm²/sr to avoid div-by-zero in dark areas
+            baseline_safe = baseline.max(0.5)
+
+            # Percentage change
+            change = current.subtract(baseline).divide(baseline_safe).multiply(100)
+
+            # Only keep pixels with >30% absolute change
+            sig_mask = change.abs().gt(30)
+            change_masked = change.updateMask(sig_mask)
+
+            # Sample up to 200 points per AOI
+            samples = change_masked.sample(
+                region=aoi, scale=500, numPixels=200, geometries=True
+            )
+            sample_list = samples.getInfo()
+
+            for feat in sample_list.get("features", []):
+                coords = feat["geometry"]["coordinates"]
+                pct = feat["properties"].get("avg_rad", 0)
+                severity = _classify_viirs_severity(pct)
+                if severity is None:
+                    continue
+                nodes.append({
+                    "lat": round(coords[1], 4),
+                    "lng": round(coords[0], 4),
+                    "mean_change_pct": round(pct, 1),
+                    "severity": severity,
+                    "aoi_name": aoi_name,
+                })
+        except Exception as e:
+            logger.warning(f"VIIRS change detection failed for {aoi_name}: {e}")
+            continue
+
+    # Save to cache
+    try:
+        _VIIRS_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
+        _VIIRS_CACHE_PATH.write_text(
+            json.dumps(nodes, separators=(",", ":")), encoding="utf-8"
+        )
+    except Exception as e:
+        logger.warning(f"Failed to write VIIRS cache: {e}")
+
+    with _data_lock:
+        latest_data["viirs_change_nodes"] = nodes
+    if nodes:
+        _mark_fresh("viirs_change_nodes")
+    logger.info(f"VIIRS change nodes: {len(nodes)} nodes from {len(_VIIRS_AOIS)} AOIs")

backend/services/fetchers/emissions.py

@@ -0,0 +1,131 @@
+"""
+Fuel burn & CO2 emissions estimator for private jets.
+Based on manufacturer-published cruise fuel burn rates (GPH at long-range cruise).
+1 US gallon of Jet-A produces ~21.1 lbs (9.57 kg) of CO2.
+"""
+
+JET_A_CO2_KG_PER_GALLON = 9.57
+
+# ICAO type code -> gallons per hour at long-range cruise
+FUEL_BURN_GPH: dict[str, int] = {
+    # Gulfstream
+    "GLF6": 430,   # G650/G650ER
+    "G700": 480,   # G700
+    "GLF5": 390,   # G550
+    "GVSP": 400,   # GV-SP
+    "GLF4": 330,   # G-IV
+    # Bombardier
+    "GL7T": 490,   # Global 7500
+    "GLEX": 430,   # Global Express/6000/6500
+    "GL5T": 420,   # Global 5000/5500
+    "CL35": 220,   # Challenger 350
+    "CL60": 310,   # Challenger 604/605
+    "CL30": 200,   # Challenger 300
+    "CL65": 320,   # Challenger 650
+    # Dassault
+    "F7X":  350,   # Falcon 7X
+    "F8X":  370,   # Falcon 8X
+    "F900": 285,   # Falcon 900/900EX/900LX
+    "F2TH": 230,   # Falcon 2000
+    "FA50": 240,   # Falcon 50
+    # Cessna
+    "CITX": 280,   # Citation X
+    "C68A": 195,   # Citation Latitude
+    "C700": 230,   # Citation Longitude
+    "C680": 220,   # Citation Sovereign
+    "C560": 190,   # Citation Excel/XLS
+    "C510": 75,    # Citation Mustang
+    "CJ3":  120,   # CJ3
+    "CJ4":  135,   # CJ4
+    # Boeing
+    "B737": 850,   # BBJ (737)
+    "B738": 920,   # BBJ2 (737-800)
+    "B752": 1100,  # 757-200
+    "B762": 1400,  # 767-200
+    "B788": 1200,  # 787-8
+    # Airbus
+    "A318": 780,   # ACJ318
+    "A319": 850,   # ACJ319
+    "A320": 900,   # ACJ320
+    "A343": 1800,  # A340-300
+    "A346": 2100,  # A340-600
+    # Pilatus
+    "PC24": 115,   # PC-24
+    "PC12": 60,    # PC-12
+    # Embraer
+    "E55P": 185,   # Legacy 500
+    "E135": 300,   # Legacy 600/650
+    "E50P": 135,   # Phenom 300
+    "E500": 80,    # Phenom 100
+    # Learjet
+    "LJ60": 195,   # Learjet 60
+    "LJ75": 185,   # Learjet 75
+    "LJ45": 175,   # Learjet 45
+    # Hawker
+    "H25B": 210,   # Hawker 800/800XP
+    "H25C": 215,   # Hawker 900XP
+    # Beechcraft
+    "B350": 100,   # King Air 350
+    "B200": 80,    # King Air 200/250
+}
+
+# Common string names -> ICAO type code
+_ALIASES: dict[str, str] = {
+    "Gulfstream G650": "GLF6", "Gulfstream G650ER": "GLF6", "G650": "GLF6", "G650ER": "GLF6",
+    "Gulfstream G700": "G700",
+    "Gulfstream G550": "GLF5", "G550": "GLF5", "G500": "GLF5",
+    "Gulfstream GV": "GVSP", "Gulfstream G-V": "GVSP", "GV": "GVSP",
+    "Gulfstream G-IV": "GLF4", "Gulfstream GIV": "GLF4", "G450": "GLF4",
+    "Global 7500": "GL7T", "Bombardier Global 7500": "GL7T",
+    "Global 6000": "GLEX", "Global Express": "GLEX", "Bombardier Global 6000": "GLEX",
+    "Global 5000": "GL5T",
+    "Challenger 350": "CL35", "Challenger 300": "CL30",
+    "Challenger 604": "CL60", "Challenger 605": "CL60", "Challenger 650": "CL65",
+    "Falcon 7X": "F7X", "Dassault Falcon 7X": "F7X",
+    "Falcon 8X": "F8X", "Dassault Falcon 8X": "F8X",
+    "Falcon 900": "F900", "Falcon 900LX": "F900", "Falcon 900EX": "F900",
+    "Falcon 2000": "F2TH",
+    "Citation X": "CITX", "Citation Latitude": "C68A", "Citation Longitude": "C700",
+    "Boeing 757-200": "B752", "757-200": "B752", "Boeing 757": "B752",
+    "Boeing 767-200": "B762", "767-200": "B762", "Boeing 767": "B762",
+    "Boeing 787-8": "B788", "Boeing 787": "B788",
+    "Boeing 737": "B737", "737 BBJ": "B737", "BBJ": "B737",
+    "Airbus A340-300": "A343", "A340-300": "A343", "A340": "A343",
+    "Airbus A318": "A318",
+    "Pilatus PC-24": "PC24", "PC-24": "PC24",
+    "Legacy 500": "E55P", "Legacy 600": "E135", "Phenom 300": "E50P",
+    "Learjet 60": "LJ60", "Learjet 75": "LJ75",
+    "Hawker 800": "H25B", "Hawker 900XP": "H25C",
+    "King Air 350": "B350", "King Air 200": "B200",
+}
+
+
+def get_emissions_info(model: str) -> dict | None:
+    """
+    Given an aircraft model string (ICAO type code or common name),
+    return emissions info dict or None if unknown.
+    """
+    if not model:
+        return None
+    model_clean = model.strip()
+    # Try direct ICAO code match first
+    gph = FUEL_BURN_GPH.get(model_clean.upper())
+    if gph is None:
+        # Try alias lookup
+        code = _ALIASES.get(model_clean)
+        if code:
+            gph = FUEL_BURN_GPH.get(code)
+    if gph is None:
+        # Fuzzy: check if any alias is a substring
+        model_lower = model_clean.lower()
+        for alias, code in _ALIASES.items():
+            if alias.lower() in model_lower or model_lower in alias.lower():
+                gph = FUEL_BURN_GPH.get(code)
+                if gph:
+                    break
+    if gph is None:
+        return None
+    return {
+        "fuel_gph": gph,
+        "co2_kg_per_hour": round(gph * JET_A_CO2_KG_PER_GALLON, 1),
+    }

backend/services/fetchers/fimi.py

@@ -0,0 +1,274 @@
+"""EUvsDisinfo FIMI (Foreign Information Manipulation & Interference) fetcher.
+
+Parses the EUvsDisinfo RSS feed to extract disinformation narratives,
+debunked claims, threat actor mentions, and target country references.
+Refreshes every 12 hours (FIMI data updates weekly).
+"""
+
+import re
+import logging
+from datetime import datetime, timezone
+
+import feedparser
+from services.network_utils import fetch_with_curl
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger("services.data_fetcher")
+
+_FIMI_FEED_URL = "https://euvsdisinfo.eu/feed/"
+
+# ── Threat actor keywords ──────────────────────────────────────────────────
+# Map of keyword → canonical actor name.  Checked case-insensitively.
+_THREAT_ACTORS: dict[str, str] = {
+    "russia":       "Russia",
+    "russian":      "Russia",
+    "kremlin":      "Russia",
+    "pro-kremlin":  "Russia",
+    "moscow":       "Russia",
+    "china":        "China",
+    "chinese":      "China",
+    "beijing":      "China",
+    "iran":         "Iran",
+    "iranian":      "Iran",
+    "tehran":       "Iran",
+    "north korea":  "North Korea",
+    "pyongyang":    "North Korea",
+    "dprk":         "North Korea",
+    "belarus":      "Belarus",
+    "belarusian":   "Belarus",
+    "minsk":        "Belarus",
+}
+
+# ── Target country/region keywords ─────────────────────────────────────────
+_TARGET_KEYWORDS: dict[str, str] = {
+    "ukraine":      "Ukraine",
+    "kyiv":         "Ukraine",
+    "moldova":      "Moldova",
+    "georgia":      "Georgia",
+    "tbilisi":      "Georgia",
+    "eu":           "EU",
+    "european union": "EU",
+    "europe":       "Europe",
+    "nato":         "NATO",
+    "united states": "United States",
+    "usa":          "United States",
+    "germany":      "Germany",
+    "france":       "France",
+    "poland":       "Poland",
+    "baltic":       "Baltics",
+    "lithuania":    "Baltics",
+    "latvia":       "Baltics",
+    "estonia":      "Baltics",
+    "romania":      "Romania",
+    "czech":        "Czech Republic",
+    "slovakia":     "Slovakia",
+    "armenia":      "Armenia",
+    "africa":       "Africa",
+    "middle east":  "Middle East",
+    "syria":        "Syria",
+    "israel":       "Israel",
+    "serbia":       "Serbia",
+    "india":        "India",
+    "brazil":       "Brazil",
+}
+
+# ── Disinformation topic keywords (for cross-referencing news) ─────────────
+_DISINFO_TOPICS = [
+    "sanctions",
+    "energy crisis",
+    "gas supply",
+    "nuclear threat",
+    "nato expansion",
+    "biolab",
+    "biological weapon",
+    "provocation",
+    "false flag",
+    "staged",
+    "nazi",
+    "genocide",
+    "referendum",
+    "regime change",
+    "coup",
+    "puppet government",
+    "election interference",
+    "election meddling",
+    "voter fraud",
+    "migrant invasion",
+    "refugee crisis",
+    "civil war",
+    "food crisis",
+    "grain deal",
+]
+
+# Regex for extracting debunked report URLs from feed HTML
+_REPORT_URL_RE = re.compile(
+    r'https?://euvsdisinfo\.eu/report/[a-z0-9\-]+/?',
+    re.IGNORECASE,
+)
+
+# Regex for extracting the claim title from a report URL slug
+_SLUG_RE = re.compile(r'/report/([a-z0-9\-]+)/?$', re.IGNORECASE)
+
+
+def _slug_to_title(url: str) -> str:
+    """Convert a report URL slug to a human-readable title."""
+    m = _SLUG_RE.search(url)
+    if not m:
+        return url
+    return m.group(1).replace("-", " ").title()
+
+
+def _count_mentions(text: str, keywords: dict[str, str]) -> dict[str, int]:
+    """Count keyword mentions, mapping to canonical names."""
+    counts: dict[str, int] = {}
+    text_lower = text.lower()
+    for kw, canonical in keywords.items():
+        # Word-boundary match, case-insensitive
+        pattern = r'\b' + re.escape(kw) + r'\b'
+        matches = re.findall(pattern, text_lower)
+        if matches:
+            counts[canonical] = counts.get(canonical, 0) + len(matches)
+    return counts
+
+
+def _extract_disinfo_keywords(text: str) -> list[str]:
+    """Return which disinformation topic keywords appear in the text."""
+    text_lower = text.lower()
+    found = []
+    for topic in _DISINFO_TOPICS:
+        if topic in text_lower:
+            found.append(topic)
+    return found
+
+
+def _is_major_wave(narratives: list[dict], targets: dict[str, int]) -> bool:
+    """Heuristic: detect a 'major disinformation wave'.
+
+    Triggers when:
+    - 3+ narratives in the feed mention the same target, OR
+    - A single target has 10+ total mentions across all narratives, OR
+    - 5+ distinct debunked claims extracted in one fetch
+    """
+    if not narratives:
+        return False
+
+    # Check per-target narrative count
+    target_narrative_counts: dict[str, int] = {}
+    total_claims = 0
+    for n in narratives:
+        for t in n.get("targets", []):
+            target_narrative_counts[t] = target_narrative_counts.get(t, 0) + 1
+        total_claims += len(n.get("claims", []))
+
+    if any(c >= 3 for c in target_narrative_counts.values()):
+        return True
+    if any(c >= 10 for c in targets.values()):
+        return True
+    if total_claims >= 5:
+        return True
+    return False
+
+
+@with_retry(max_retries=1, base_delay=5)
+def fetch_fimi():
+    """Fetch and parse the EUvsDisinfo RSS feed."""
+    try:
+        resp = fetch_with_curl(_FIMI_FEED_URL, timeout=15)
+        feed = feedparser.parse(resp.text)
+    except Exception as e:
+        logger.warning(f"FIMI feed fetch failed: {e}")
+        return
+
+    if not feed.entries:
+        logger.warning("FIMI feed: no entries found")
+        return
+
+    narratives = []
+    all_claims: list[dict] = []
+    agg_actors: dict[str, int] = {}
+    agg_targets: dict[str, int] = {}
+    all_disinfo_kw: set[str] = set()
+
+    for entry in feed.entries[:15]:  # Cap at 15 entries
+        title = entry.get("title", "")
+        link = entry.get("link", "")
+        published = entry.get("published", "")
+        summary_html = entry.get("summary", "") or entry.get("description", "")
+
+        # Strip HTML tags for text analysis
+        summary_text = re.sub(r"<[^>]+>", " ", summary_html)
+        summary_text = re.sub(r"\s+", " ", summary_text).strip()
+        full_text = f"{title} {summary_text}"
+
+        # Extract debunked report URLs
+        report_urls = list(set(_REPORT_URL_RE.findall(summary_html)))
+        claims = [{"url": url, "title": _slug_to_title(url)} for url in report_urls]
+        all_claims.extend(claims)
+
+        # Count threat actors
+        actors = _count_mentions(full_text, _THREAT_ACTORS)
+        for actor, count in actors.items():
+            agg_actors[actor] = agg_actors.get(actor, 0) + count
+
+        # Count target countries
+        targets = _count_mentions(full_text, _TARGET_KEYWORDS)
+        for target, count in targets.items():
+            agg_targets[target] = agg_targets.get(target, 0) + count
+
+        # Extract disinfo topic keywords
+        disinfo_kw = _extract_disinfo_keywords(full_text)
+        all_disinfo_kw.update(disinfo_kw)
+
+        # Truncate summary for storage
+        snippet = summary_text[:300] + ("..." if len(summary_text) > 300 else "")
+
+        narratives.append({
+            "title": title,
+            "link": link,
+            "published": published,
+            "snippet": snippet,
+            "claims": claims,
+            "actors": list(actors.keys()),
+            "targets": list(targets.keys()),
+            "disinfo_keywords": disinfo_kw,
+        })
+
+    # Sort actors and targets by count (descending)
+    sorted_actors = dict(sorted(agg_actors.items(), key=lambda x: x[1], reverse=True))
+    sorted_targets = dict(sorted(agg_targets.items(), key=lambda x: x[1], reverse=True))
+
+    # Deduplicate claims
+    seen_urls: set[str] = set()
+    unique_claims = []
+    for c in all_claims:
+        if c["url"] not in seen_urls:
+            seen_urls.add(c["url"])
+            unique_claims.append(c)
+
+    major_wave = _is_major_wave(narratives, sorted_targets)
+
+    fimi_data = {
+        "narratives": narratives,
+        "claims": unique_claims,
+        "threat_actors": sorted_actors,
+        "targets": sorted_targets,
+        "disinfo_keywords": sorted(all_disinfo_kw),
+        "major_wave": major_wave,
+        "major_wave_target": (
+            max(sorted_targets, key=sorted_targets.get) if major_wave and sorted_targets else None
+        ),
+        "last_fetched": datetime.now(timezone.utc).isoformat(),
+        "source": "EUvsDisinfo",
+        "source_url": "https://euvsdisinfo.eu",
+    }
+
+    with _data_lock:
+        latest_data["fimi"] = fimi_data
+    _mark_fresh("fimi")
+    logger.info(
+        f"FIMI fetch complete: {len(narratives)} narratives, "
+        f"{len(unique_claims)} claims, "
+        f"{len(sorted_actors)} actors, "
+        f"major_wave={major_wave}"
+    )

backend/services/fetchers/financial.py

@@ -0,0 +1,161 @@
+import logging
+import math
+import random
+import time
+import os
+import urllib.request
+import json
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timezone
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger(__name__)
+
+_YFINANCE_REQUEST_DELAY_SECONDS = 0.5
+_YFINANCE_REQUEST_JITTER_SECONDS = 0.2
+
+TICKERS_DEFENSE = ["RTX", "LMT", "NOC", "GD", "BA", "PLTR"]
+TICKERS_TECH = ["NVDA", "AMD", "TSM", "INTC", "GOOGL", "AMZN", "MSFT", "AAPL", "TSLA", "META", "NFLX", "SMCI", "ARM", "ASML"]
+TICKERS_CRYPTO = [
+    ("BTC", "BINANCE:BTCUSDT", "BTC-USD"),
+    ("ETH", "BINANCE:ETHUSDT", "ETH-USD"),
+    ("SOL", "BINANCE:SOLUSDT", "SOL-USD"),
+    ("XRP", "BINANCE:XRPUSDT", "XRP-USD"),
+    ("ADA", "BINANCE:ADAUSDT", "ADA-USD"),
+]
+
+# Ticker priority for high-frequency updates (we update these every tick)
+PRIORITY_SYMBOLS = ["BTC", "ETH", "NVDA", "PLTR"]
+
+# Persistence for state between short-lived scheduler ticks
+_last_fetch_results = {}
+_last_fetch_time = 0.0
+_rotating_index = 0
+_executor = ThreadPoolExecutor(max_workers=10)
+
+
+def _fetch_finnhub_quote(symbol: str, api_key: str):
+    """Fetch from Finnhub. Returns (symbol, data) or (symbol, None)."""
+    url = f"https://finnhub.io/api/v1/quote?symbol={symbol}&token={api_key}"
+    try:
+        req = urllib.request.Request(url)
+        with urllib.request.urlopen(req, timeout=5) as response:
+            data = json.loads(response.read().decode())
+            if "c" not in data or data["c"] == 0:
+                return symbol, None
+            current = float(data["c"])
+            change_p = float(data.get("dp", 0.0) or 0.0)
+            return symbol, {
+                "price": round(current, 2),
+                "change_percent": round(change_p, 2),
+                "up": bool(change_p >= 0),
+            }
+    except Exception as e:
+        logger.debug(f"Finnhub error for {symbol}: {e}")
+        return symbol, None
+
+
+def _fetch_yfinance_single(symbol: str, period: str = "2d"):
+    """Fetch from yfinance. Returns (symbol, data) or (symbol, None)."""
+    try:
+        import yfinance as yf
+        ticker = yf.Ticker(symbol)
+        hist = ticker.history(period=period)
+        if len(hist) >= 1:
+            current_price = hist["Close"].iloc[-1]
+            prev_close = hist["Close"].iloc[0] if len(hist) > 1 else current_price
+            change_percent = ((current_price - prev_close) / prev_close) * 100 if prev_close else 0
+            current_price_f = float(current_price)
+            change_percent_f = float(change_percent)
+            if not math.isfinite(current_price_f) or not math.isfinite(change_percent_f):
+                return symbol, None
+            return symbol, {
+                "price": round(current_price_f, 2),
+                "change_percent": round(change_percent_f, 2),
+                "up": bool(change_percent_f >= 0),
+            }
+    except Exception as e:
+        logger.debug(f"Yfinance error for {symbol}: {e}")
+    return symbol, None
+
+
+@with_retry(max_retries=1, base_delay=1)
+def fetch_financial_markets():
+    """Fetches full market list with smart throttling (3s for Finnhub, 60s for yfinance)."""
+    global _last_fetch_time, _last_fetch_results, _rotating_index
+    
+    finnhub_key = os.getenv("FINNHUB_API_KEY", "").strip()
+    use_finnhub = bool(finnhub_key)
+    
+    now = time.time()
+    # Throttle logic: 3s for Finnhub, 60s for yfinance fallback
+    throttle_s = 3.0 if use_finnhub else 60.0
+    
+    if now - _last_fetch_time < throttle_s and _last_fetch_results:
+        return # Skip if too frequent
+
+    _last_fetch_time = now
+    
+    # Prepare symbol lists
+    all_crypto = {label: (f_sym, y_sym) for label, f_sym, y_sym in TICKERS_CRYPTO}
+    all_stocks = TICKERS_TECH + TICKERS_DEFENSE
+    
+    subset_to_fetch = []
+    
+    if use_finnhub:
+        # Finnhub Free Limit: 60/min. 
+        # Ticking every 3s = 20 ticks/min. 
+        # To stay safe, we fetch only ~3 items per tick.
+        # Priority items (BTC, ETH) + 1 rotating item.
+        subset_to_fetch = ["BINANCE:BTCUSDT", "BINANCE:ETHUSDT"]
+        
+        # Determine rotating ticker
+        all_other_symbols = []
+        for sym in all_stocks:
+            all_other_symbols.append(sym)
+        for label, (f_sym, y_sym) in all_crypto.items():
+            if label not in ["BTC", "ETH"]:
+                all_other_symbols.append(f_sym)
+        
+        if all_other_symbols:
+            rotated = all_other_symbols[_rotating_index % len(all_other_symbols)]
+            subset_to_fetch.append(rotated)
+            _rotating_index += 1
+            
+        # Concurrently fetch
+        futures = [_executor.submit(_fetch_finnhub_quote, s, finnhub_key) for s in subset_to_fetch]
+        for f in futures:
+            sym, data = f.result()
+            if data:
+                # Map back to readable label if it was crypto
+                label = sym
+                for l, (fs, ys) in all_crypto.items():
+                    if fs == sym:
+                        label = l
+                        break
+                _last_fetch_results[label] = data
+    else:
+        # Yahoo Finance Fallback - fetch all (once per minute)
+        logger.info("Finnhub key missing, using Yahoo Finance 60s update cycle.")
+        to_fetch = all_stocks + [y_sym for l, (fs, y_sym) in all_crypto.items()]
+        futures = [_executor.submit(_fetch_yfinance_single, s) for s in to_fetch]
+        for f in futures:
+            sym, data = f.result()
+            if data:
+                # Map back to readable label if it was crypto
+                label = sym
+                for l, (fs, ys) in all_crypto.items():
+                    if ys == sym:
+                        label = l
+                        break
+                _last_fetch_results[label] = data
+
+    if not _last_fetch_results:
+        return
+        
+    with _data_lock:
+        latest_data["stocks"] = dict(_last_fetch_results)
+        latest_data["financial_source"] = "finnhub" if use_finnhub else "yfinance"
+    _mark_fresh("stocks")

backend/services/fetchers/geo.py

@@ -0,0 +1,251 @@
+"""Ship and geopolitics fetchers — AIS vessels, carriers, frontlines, GDELT, LiveUAmap, fishing."""
+
+import csv
+import io
+import math
+import os
+import logging
+from services.network_utils import fetch_with_curl
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Ships (AIS + Carriers)
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=1, base_delay=1)
+def fetch_ships():
+    """Fetch real-time AIS vessel data and combine with OSINT carrier positions."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active(
+        "ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts"
+    ):
+        return
+    from services.ais_stream import get_ais_vessels
+    from services.carrier_tracker import get_carrier_positions
+
+    ships = []
+    try:
+        carriers = get_carrier_positions()
+        ships.extend(carriers)
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Carrier tracker error (non-fatal): {e}")
+        carriers = []
+
+    try:
+        ais_vessels = get_ais_vessels()
+        ships.extend(ais_vessels)
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"AIS stream error (non-fatal): {e}")
+        ais_vessels = []
+
+    # Enrich ships with yacht alert data (tracked superyachts)
+    from services.fetchers.yacht_alert import enrich_with_yacht_alert
+
+    for ship in ships:
+        enrich_with_yacht_alert(ship)
+
+    # Enrich ships with PLAN/CCG vessel data
+    from services.fetchers.plan_vessel_alert import enrich_with_plan_vessel
+    for ship in ships:
+        enrich_with_plan_vessel(ship)
+
+    logger.info(f"Ships: {len(carriers)} carriers + {len(ais_vessels)} AIS vessels")
+    with _data_lock:
+        latest_data["ships"] = ships
+    _mark_fresh("ships")
+
+
+# ---------------------------------------------------------------------------
+# Airports (ourairports.com)
+# ---------------------------------------------------------------------------
+cached_airports = []
+
+
+def find_nearest_airport(lat, lng, max_distance_nm=200):
+    """Find the nearest large airport to a given lat/lng using haversine distance."""
+    if not cached_airports:
+        return None
+
+    best = None
+    best_dist = float("inf")
+    lat_r = math.radians(lat)
+    lng_r = math.radians(lng)
+
+    for apt in cached_airports:
+        apt_lat_r = math.radians(apt["lat"])
+        apt_lng_r = math.radians(apt["lng"])
+        dlat = apt_lat_r - lat_r
+        dlng = apt_lng_r - lng_r
+        a = (
+            math.sin(dlat / 2) ** 2
+            + math.cos(lat_r) * math.cos(apt_lat_r) * math.sin(dlng / 2) ** 2
+        )
+        c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
+        dist_nm = 3440.065 * c
+
+        if dist_nm < best_dist:
+            best_dist = dist_nm
+            best = apt
+
+    if best and best_dist <= max_distance_nm:
+        return {
+            "iata": best["iata"],
+            "name": best["name"],
+            "lat": best["lat"],
+            "lng": best["lng"],
+            "distance_nm": round(best_dist, 1),
+        }
+    return None
+
+
+def fetch_airports():
+    global cached_airports
+    if not cached_airports:
+        logger.info("Downloading global airports database from ourairports.com...")
+        try:
+            url = "https://ourairports.com/data/airports.csv"
+            response = fetch_with_curl(url, timeout=15)
+            if response.status_code == 200:
+                f = io.StringIO(response.text)
+                reader = csv.DictReader(f)
+                for row in reader:
+                    if row["type"] == "large_airport" and row["iata_code"]:
+                        cached_airports.append(
+                            {
+                                "id": row["ident"],
+                                "name": row["name"],
+                                "iata": row["iata_code"],
+                                "lat": float(row["latitude_deg"]),
+                                "lng": float(row["longitude_deg"]),
+                                "type": "airport",
+                            }
+                        )
+                logger.info(f"Loaded {len(cached_airports)} large airports into cache.")
+        except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+            logger.error(f"Error fetching airports: {e}")
+
+    with _data_lock:
+        latest_data["airports"] = cached_airports
+
+
+# ---------------------------------------------------------------------------
+# Geopolitics & LiveUAMap
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=1, base_delay=2)
+def fetch_frontlines():
+    """Fetch Ukraine frontline data (fast — single GitHub API call)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("ukraine_frontline"):
+        return
+    try:
+        from services.geopolitics import fetch_ukraine_frontlines
+
+        frontlines = fetch_ukraine_frontlines()
+        if frontlines:
+            with _data_lock:
+                latest_data["frontlines"] = frontlines
+            _mark_fresh("frontlines")
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching frontlines: {e}")
+
+
+@with_retry(max_retries=1, base_delay=3)
+def fetch_gdelt():
+    """Fetch GDELT global military incidents (slow — downloads 32 ZIP files)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("global_incidents"):
+        return
+    try:
+        from services.geopolitics import fetch_global_military_incidents
+
+        gdelt = fetch_global_military_incidents()
+        if gdelt is not None:
+            with _data_lock:
+                latest_data["gdelt"] = gdelt
+            _mark_fresh("gdelt")
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching GDELT: {e}")
+
+
+def fetch_geopolitics():
+    """Legacy wrapper — runs both sequentially. Used by recurring scheduler."""
+    fetch_frontlines()
+    fetch_gdelt()
+
+
+def update_liveuamap():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("global_incidents"):
+        return
+    logger.info("Running scheduled Liveuamap scraper...")
+    try:
+        from services.liveuamap_scraper import fetch_liveuamap
+
+        res = fetch_liveuamap()
+        if res:
+            with _data_lock:
+                latest_data["liveuamap"] = res
+            _mark_fresh("liveuamap")
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Liveuamap scraper error: {e}")
+
+
+# ---------------------------------------------------------------------------
+# Fishing Activity (Global Fishing Watch)
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=1, base_delay=5)
+def fetch_fishing_activity():
+    """Fetch recent fishing events from Global Fishing Watch (~5 day lag)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("fishing_activity"):
+        return
+    token = os.environ.get("GFW_API_TOKEN", "")
+    if not token:
+        logger.debug("GFW_API_TOKEN not set, skipping fishing activity fetch")
+        return
+    events = []
+    try:
+        url = (
+            "https://gateway.api.globalfishingwatch.org/v3/events"
+            "?datasets[0]=public-global-fishing-events:latest"
+            "&limit=500&sort=start&sort-direction=DESC"
+        )
+        headers = {"Authorization": f"Bearer {token}"}
+        response = fetch_with_curl(url, timeout=30, headers=headers)
+        if response.status_code == 200:
+            entries = response.json().get("entries", [])
+            for e in entries:
+                pos = e.get("position", {})
+                lat = pos.get("lat")
+                lng = pos.get("lon")
+                if lat is None or lng is None:
+                    continue
+                dur = e.get("event", {}).get("duration", 0) or 0
+                events.append(
+                    {
+                        "id": e.get("id", ""),
+                        "type": e.get("type", "fishing"),
+                        "lat": lat,
+                        "lng": lng,
+                        "start": e.get("start", ""),
+                        "end": e.get("end", ""),
+                        "vessel_name": (e.get("vessel") or {}).get("name", "Unknown"),
+                        "vessel_flag": (e.get("vessel") or {}).get("flag", ""),
+                        "duration_hrs": round(dur / 3600, 1),
+                    }
+                )
+        logger.info(f"Fishing activity: {len(events)} events")
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching fishing activity: {e}")
+    with _data_lock:
+        latest_data["fishing_activity"] = events
+    if events:
+        _mark_fresh("fishing_activity")

backend/services/fetchers/infrastructure.py

@@ -0,0 +1,727 @@
+"""Infrastructure fetchers — internet outages (IODA), data centers, CCTV, KiwiSDR."""
+
+import json
+import time
+import heapq
+import logging
+from pathlib import Path
+from cachetools import TTLCache
+from services.network_utils import fetch_with_curl
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Internet Outages (IODA — Georgia Tech)
+# ---------------------------------------------------------------------------
+_region_geocode_cache: TTLCache = TTLCache(maxsize=2000, ttl=86400)
+
+
+def _geocode_region(region_name: str, country_name: str) -> tuple:
+    """Geocode a region using OpenStreetMap Nominatim (cached, respects rate limit)."""
+    cache_key = f"{region_name}|{country_name}"
+    if cache_key in _region_geocode_cache:
+        return _region_geocode_cache[cache_key]
+    try:
+        import urllib.parse
+
+        query = urllib.parse.quote(f"{region_name}, {country_name}")
+        url = f"https://nominatim.openstreetmap.org/search?q={query}&format=json&limit=1"
+        response = fetch_with_curl(url, timeout=8, headers={"User-Agent": "ShadowBroker-OSINT/1.0"})
+        if response.status_code == 200:
+            results = response.json()
+            if results:
+                lat = float(results[0]["lat"])
+                lon = float(results[0]["lon"])
+                _region_geocode_cache[cache_key] = (lat, lon)
+                return (lat, lon)
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError):
+        pass
+    _region_geocode_cache[cache_key] = None
+    return None
+
+
+@with_retry(max_retries=1, base_delay=1)
+def fetch_internet_outages():
+    """Fetch regional internet outage alerts from IODA (Georgia Tech)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("internet_outages"):
+        return
+    RELIABLE_DATASOURCES = {"bgp", "ping-slash24"}
+    outages = []
+    try:
+        now = int(time.time())
+        start = now - 86400
+        url = f"https://api.ioda.inetintel.cc.gatech.edu/v2/outages/alerts?from={start}&until={now}&limit=500"
+        response = fetch_with_curl(url, timeout=15)
+        if response.status_code == 200:
+            data = response.json()
+            alerts = data.get("data", [])
+            region_outages = {}
+            for alert in alerts:
+                entity = alert.get("entity", {})
+                etype = entity.get("type", "")
+                level = alert.get("level", "")
+                if level == "normal" or etype != "region":
+                    continue
+                datasource = alert.get("datasource", "")
+                if datasource not in RELIABLE_DATASOURCES:
+                    continue
+                code = entity.get("code", "")
+                name = entity.get("name", "")
+                attrs = entity.get("attrs", {})
+                country_code = attrs.get("country_code", "")
+                country_name = attrs.get("country_name", "")
+                value = alert.get("value", 0)
+                history_value = alert.get("historyValue", 0)
+                severity = 0
+                if history_value and history_value > 0:
+                    severity = round((1 - value / history_value) * 100)
+                severity = max(0, min(severity, 100))
+                if severity < 10:
+                    continue
+                if code not in region_outages or severity > region_outages[code]["severity"]:
+                    region_outages[code] = {
+                        "region_code": code,
+                        "region_name": name,
+                        "country_code": country_code,
+                        "country_name": country_name,
+                        "level": level,
+                        "datasource": datasource,
+                        "severity": severity,
+                    }
+            geocoded = []
+            for rcode, r in region_outages.items():
+                coords = _geocode_region(r["region_name"], r["country_name"])
+                if coords:
+                    r["lat"] = coords[0]
+                    r["lng"] = coords[1]
+                    geocoded.append(r)
+            outages = heapq.nlargest(100, geocoded, key=lambda x: x["severity"])
+        logger.info(f"Internet outages: {len(outages)} regions affected")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching internet outages: {e}")
+    with _data_lock:
+        latest_data["internet_outages"] = outages
+    if outages:
+        _mark_fresh("internet_outages")
+
+
+# ---------------------------------------------------------------------------
+# RIPE Atlas — complement IODA with probe-level disconnection data
+# ---------------------------------------------------------------------------
+
+@with_retry(max_retries=1, base_delay=3)
+def fetch_ripe_atlas_probes():
+    """Fetch disconnected RIPE Atlas probes and merge into internet_outages (complementing IODA)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("internet_outages"):
+        return
+    try:
+        # 1. Fetch disconnected probes (status=2) — ~2,000 probes, no auth needed
+        url_disc = "https://atlas.ripe.net/api/v2/probes/?status=2&page_size=500&format=json"
+        resp_disc = fetch_with_curl(url_disc, timeout=20)
+        if resp_disc.status_code != 200:
+            logger.warning(f"RIPE Atlas probes API returned {resp_disc.status_code}")
+            return
+        disc_data = resp_disc.json()
+        disconnected = disc_data.get("results", [])
+
+        # 2. Fetch connected probe count (page_size=1 — we only need the count)
+        url_conn = "https://atlas.ripe.net/api/v2/probes/?status=1&page_size=1&format=json"
+        resp_conn = fetch_with_curl(url_conn, timeout=10)
+        total_connected = 0
+        if resp_conn.status_code == 200:
+            total_connected = resp_conn.json().get("count", 0)
+
+        # 3. Group disconnected probes by country
+        country_disc: dict = {}
+        for p in disconnected:
+            cc = p.get("country_code", "")
+            if not cc:
+                continue
+            if cc not in country_disc:
+                country_disc[cc] = []
+            country_disc[cc].append(p)
+
+        # 4. Get IODA-covered countries to avoid double-reporting
+        with _data_lock:
+            ioda_outages = list(latest_data.get("internet_outages", []))
+        ioda_countries = {
+            o.get("country_code", "").upper()
+            for o in ioda_outages
+            if o.get("datasource") != "ripe-atlas"
+        }
+
+        # 5. Build RIPE-only alerts for countries NOT already in IODA
+        ripe_alerts = []
+        for cc, probes in country_disc.items():
+            if cc.upper() in ioda_countries:
+                continue  # IODA already covers this country
+            if len(probes) < 3:
+                continue  # Too few probes to be meaningful
+
+            # Use centroid of disconnected probes as marker location
+            lats = [
+                p["geometry"]["coordinates"][1]
+                for p in probes
+                if p.get("geometry") and p["geometry"].get("coordinates")
+            ]
+            lngs = [
+                p["geometry"]["coordinates"][0]
+                for p in probes
+                if p.get("geometry") and p["geometry"].get("coordinates")
+            ]
+            if not lats:
+                continue
+
+            disc_count = len(probes)
+            # Severity: scale 10-80 based on disconnected probe count
+            severity = min(80, 10 + disc_count * 2)
+
+            ripe_alerts.append({
+                "region_code": f"RIPE-{cc}",
+                "region_name": f"{cc} (Atlas probes)",
+                "country_code": cc,
+                "country_name": cc,
+                "level": "critical" if disc_count >= 10 else "warning",
+                "datasource": "ripe-atlas",
+                "severity": severity,
+                "lat": sum(lats) / len(lats),
+                "lng": sum(lngs) / len(lngs),
+                "probe_count": disc_count,
+            })
+
+        # 6. Merge into internet_outages — keep IODA entries, replace old RIPE entries
+        with _data_lock:
+            current = latest_data.get("internet_outages", [])
+            ioda_only = [o for o in current if o.get("datasource") != "ripe-atlas"]
+            latest_data["internet_outages"] = ioda_only + ripe_alerts
+
+        if ripe_alerts:
+            _mark_fresh("internet_outages")
+        logger.info(
+            f"RIPE Atlas: {len(ripe_alerts)} countries with probe disconnections "
+            f"(from {len(disconnected)} disconnected / ~{total_connected} connected probes)"
+        )
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching RIPE Atlas probes: {e}")
+
+
+# ---------------------------------------------------------------------------
+# Data Centers (local geocoded JSON)
+# ---------------------------------------------------------------------------
+_DC_GEOCODED_PATH = Path(__file__).parent.parent.parent / "data" / "datacenters_geocoded.json"
+
+
+def fetch_datacenters():
+    """Load geocoded data centers (5K+ street-level precise locations)."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("datacenters"):
+        return
+    dcs = []
+    try:
+        if not _DC_GEOCODED_PATH.exists():
+            logger.warning(f"Geocoded DC file not found: {_DC_GEOCODED_PATH}")
+            return
+        raw = json.loads(_DC_GEOCODED_PATH.read_text(encoding="utf-8"))
+        for entry in raw:
+            lat = entry.get("lat")
+            lng = entry.get("lng")
+            if lat is None or lng is None:
+                continue
+            if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+                continue
+            dcs.append(
+                {
+                    "name": entry.get("name", "Unknown"),
+                    "company": entry.get("company", ""),
+                    "street": entry.get("street", ""),
+                    "city": entry.get("city", ""),
+                    "country": entry.get("country", ""),
+                    "zip": entry.get("zip", ""),
+                    "lat": lat,
+                    "lng": lng,
+                }
+            )
+        logger.info(f"Data centers: {len(dcs)} geocoded locations loaded")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error loading data centers: {e}")
+    with _data_lock:
+        latest_data["datacenters"] = dcs
+    if dcs:
+        _mark_fresh("datacenters")
+
+
+# ---------------------------------------------------------------------------
+# Military Bases (static JSON — Western Pacific)
+# ---------------------------------------------------------------------------
+_MILITARY_BASES_PATH = Path(__file__).parent.parent.parent / "data" / "military_bases.json"
+
+
+def fetch_military_bases():
+    """Load static military base locations (Western Pacific focus)."""
+    bases = []
+    try:
+        if not _MILITARY_BASES_PATH.exists():
+            logger.warning(f"Military bases file not found: {_MILITARY_BASES_PATH}")
+            return
+        raw = json.loads(_MILITARY_BASES_PATH.read_text(encoding="utf-8"))
+        for entry in raw:
+            lat = entry.get("lat")
+            lng = entry.get("lng")
+            if lat is None or lng is None:
+                continue
+            if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+                continue
+            bases.append({
+                "name": entry.get("name", "Unknown"),
+                "country": entry.get("country", ""),
+                "operator": entry.get("operator", ""),
+                "branch": entry.get("branch", ""),
+                "lat": lat, "lng": lng,
+            })
+        logger.info(f"Military bases: {len(bases)} locations loaded")
+    except Exception as e:
+        logger.error(f"Error loading military bases: {e}")
+    with _data_lock:
+        latest_data["military_bases"] = bases
+    if bases:
+        _mark_fresh("military_bases")
+
+
+# ---------------------------------------------------------------------------
+# Power Plants (WRI Global Power Plant Database)
+# ---------------------------------------------------------------------------
+_POWER_PLANTS_PATH = Path(__file__).parent.parent.parent / "data" / "power_plants.json"
+
+
+def fetch_power_plants():
+    """Load WRI Global Power Plant Database (~35K facilities)."""
+    plants = []
+    try:
+        if not _POWER_PLANTS_PATH.exists():
+            logger.warning(f"Power plants file not found: {_POWER_PLANTS_PATH}")
+            return
+        raw = json.loads(_POWER_PLANTS_PATH.read_text(encoding="utf-8"))
+        for entry in raw:
+            lat = entry.get("lat")
+            lng = entry.get("lng")
+            if lat is None or lng is None:
+                continue
+            if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+                continue
+            plants.append({
+                "name": entry.get("name", "Unknown"),
+                "country": entry.get("country", ""),
+                "fuel_type": entry.get("fuel_type", "Unknown"),
+                "capacity_mw": entry.get("capacity_mw"),
+                "owner": entry.get("owner", ""),
+                "lat": lat, "lng": lng,
+            })
+        logger.info(f"Power plants: {len(plants)} facilities loaded")
+    except Exception as e:
+        logger.error(f"Error loading power plants: {e}")
+    with _data_lock:
+        latest_data["power_plants"] = plants
+    if plants:
+        _mark_fresh("power_plants")
+
+
+# ---------------------------------------------------------------------------
+# CCTV Cameras
+# ---------------------------------------------------------------------------
+def fetch_cctv():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("cctv"):
+        return
+    try:
+        from services.cctv_pipeline import get_all_cameras
+
+        cameras = get_all_cameras()
+        if len(cameras) < 500:
+            # Serve the current DB snapshot immediately and let the scheduled
+            # ingest cycle populate/refresh cameras asynchronously.
+            logger.info(
+                "CCTV DB currently has %d cameras — serving cached snapshot and waiting for scheduled ingest",
+                len(cameras),
+            )
+        with _data_lock:
+            latest_data["cctv"] = cameras
+        _mark_fresh("cctv")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching cctv from DB: {e}")
+
+
+# ---------------------------------------------------------------------------
+# KiwiSDR Receivers
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=2, base_delay=2)
+def fetch_kiwisdr():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("kiwisdr"):
+        return
+    try:
+        from services.kiwisdr_fetcher import fetch_kiwisdr_nodes
+
+        nodes = fetch_kiwisdr_nodes()
+        with _data_lock:
+            latest_data["kiwisdr"] = nodes
+        _mark_fresh("kiwisdr")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching KiwiSDR nodes: {e}")
+        with _data_lock:
+            latest_data["kiwisdr"] = []
+
+
+# ---------------------------------------------------------------------------
+# SatNOGS Ground Stations + Observations
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=2, base_delay=2)
+def fetch_satnogs():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("satnogs"):
+        return
+    try:
+        from services.satnogs_fetcher import fetch_satnogs_stations, fetch_satnogs_observations
+
+        stations = fetch_satnogs_stations()
+        obs = fetch_satnogs_observations()
+        with _data_lock:
+            latest_data["satnogs_stations"] = stations
+            latest_data["satnogs_observations"] = obs
+        _mark_fresh("satnogs_stations", "satnogs_observations")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching SatNOGS: {e}")
+
+
+# ---------------------------------------------------------------------------
+# PSK Reporter — HF Digital Mode Spots
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=2, base_delay=2)
+def fetch_psk_reporter():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("psk_reporter"):
+        return
+    try:
+        from services.psk_reporter_fetcher import fetch_psk_reporter_spots
+
+        spots = fetch_psk_reporter_spots()
+        with _data_lock:
+            latest_data["psk_reporter"] = spots
+        _mark_fresh("psk_reporter")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching PSK Reporter: {e}")
+        with _data_lock:
+            latest_data["psk_reporter"] = []
+
+
+# ---------------------------------------------------------------------------
+# TinyGS LoRa Satellites
+# ---------------------------------------------------------------------------
+@with_retry(max_retries=2, base_delay=2)
+def fetch_tinygs():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("tinygs"):
+        return
+    try:
+        from services.tinygs_fetcher import fetch_tinygs_satellites
+
+        sats = fetch_tinygs_satellites()
+        with _data_lock:
+            latest_data["tinygs_satellites"] = sats
+        _mark_fresh("tinygs_satellites")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching TinyGS: {e}")
+
+
+# ---------------------------------------------------------------------------
+# Police Scanners (OpenMHZ) — geocode city+state via local GeoNames DB
+# ---------------------------------------------------------------------------
+_scanner_geo_cache: dict = {}  # city|state -> (lat, lng) — populated once from GeoNames
+
+
+def _build_scanner_geo_lookup():
+    """Build a US city/county→coords lookup from reverse_geocoder's bundled GeoNames CSV."""
+    if _scanner_geo_cache:
+        return
+    try:
+        import csv, os, reverse_geocoder as rg
+
+        geo_file = os.path.join(os.path.dirname(rg.__file__), "rg_cities1000.csv")
+        # US state abbreviation → admin1 name mapping
+        _abbr = {
+            "AL": "Alabama",
+            "AK": "Alaska",
+            "AZ": "Arizona",
+            "AR": "Arkansas",
+            "CA": "California",
+            "CO": "Colorado",
+            "CT": "Connecticut",
+            "DE": "Delaware",
+            "FL": "Florida",
+            "GA": "Georgia",
+            "HI": "Hawaii",
+            "ID": "Idaho",
+            "IL": "Illinois",
+            "IN": "Indiana",
+            "IA": "Iowa",
+            "KS": "Kansas",
+            "KY": "Kentucky",
+            "LA": "Louisiana",
+            "ME": "Maine",
+            "MD": "Maryland",
+            "MA": "Massachusetts",
+            "MI": "Michigan",
+            "MN": "Minnesota",
+            "MS": "Mississippi",
+            "MO": "Missouri",
+            "MT": "Montana",
+            "NE": "Nebraska",
+            "NV": "Nevada",
+            "NH": "New Hampshire",
+            "NJ": "New Jersey",
+            "NM": "New Mexico",
+            "NY": "New York",
+            "NC": "North Carolina",
+            "ND": "North Dakota",
+            "OH": "Ohio",
+            "OK": "Oklahoma",
+            "OR": "Oregon",
+            "PA": "Pennsylvania",
+            "RI": "Rhode Island",
+            "SC": "South Carolina",
+            "SD": "South Dakota",
+            "TN": "Tennessee",
+            "TX": "Texas",
+            "UT": "Utah",
+            "VT": "Vermont",
+            "VA": "Virginia",
+            "WA": "Washington",
+            "WV": "West Virginia",
+            "WI": "Wisconsin",
+            "WY": "Wyoming",
+            "DC": "Washington, D.C.",
+        }
+        state_full = {v.lower(): k for k, v in _abbr.items()}
+        state_full["washington, d.c."] = "DC"
+
+        county_coords = {}  # admin2(county)|state -> (lat, lon) — first city per county
+        with open(geo_file, "r", encoding="utf-8") as f:
+            reader = csv.reader(f)
+            next(reader, None)  # skip header
+            for row in reader:
+                if len(row) < 6 or row[5] != "US":
+                    continue
+                lat_s, lon_s, name, admin1, admin2 = row[0], row[1], row[2], row[3], row[4]
+                st = state_full.get(admin1.lower(), "")
+                if not st:
+                    continue
+                coords = (float(lat_s), float(lon_s))
+                # City name → coords
+                _scanner_geo_cache[f"{name.lower()}|{st}"] = coords
+                # County name → coords (keep first match per county, usually the largest city)
+                if admin2:
+                    county_key = f"{admin2.lower()}|{st}"
+                    if county_key not in county_coords:
+                        county_coords[county_key] = coords
+                    # Also strip " County" suffix for matching
+                    stripped = admin2.lower().replace(" county", "").strip()
+                    stripped_key = f"{stripped}|{st}"
+                    if stripped_key not in county_coords:
+                        county_coords[stripped_key] = coords
+
+        # Merge county lookups (don't override city entries)
+        for k, v in county_coords.items():
+            if k not in _scanner_geo_cache:
+                _scanner_geo_cache[k] = v
+        # Special case: DC
+        _scanner_geo_cache["washington|DC"] = (38.89511, -77.03637)
+        logger.info(f"Scanner geo lookup: {len(_scanner_geo_cache)} US entries loaded")
+    except Exception as e:
+        logger.warning(f"Failed to build scanner geo lookup: {e}")
+
+
+def _geocode_scanner(city: str, state: str):
+    """Look up city+state coordinates from local GeoNames cache."""
+    _build_scanner_geo_lookup()
+    if not city or not state:
+        return None
+    st = state.upper()
+    # Strip trailing state from city (e.g. "Lehigh, PA")
+    c = city.strip()
+    if ", " in c:
+        parts = c.rsplit(", ", 1)
+        if len(parts[1]) <= 2:
+            c = parts[0]
+    name = c.lower()
+    # Try exact city match
+    result = _scanner_geo_cache.get(f"{name}|{st}")
+    if result:
+        return result
+    # Strip "County" / "Co" suffix
+    stripped = name.replace(" county", "").replace(" co", "").strip()
+    result = _scanner_geo_cache.get(f"{stripped}|{st}")
+    if result:
+        return result
+    # Normalize "St." / "St" → "Saint"
+    import re
+
+    normed = re.sub(r"\bst\.?\s", "saint ", name)
+    if normed != name:
+        result = _scanner_geo_cache.get(f"{normed}|{st}")
+        if result:
+            return result
+        # Also try with "s" suffix: "St. Marys" → "Saint Marys" and "Saint Mary's"
+        for variant in [normed.rstrip("s"), normed.replace("ys", "y's")]:
+            result = _scanner_geo_cache.get(f"{variant}|{st}")
+            if result:
+                return result
+    # "Prince Georges" → "Prince George's" (apostrophe variants)
+    if "georges" in name:
+        key = name.replace("georges", "george's") + "|" + st
+        result = _scanner_geo_cache.get(key)
+        if result:
+            return result
+    # Multi-location: "Scott and Carver" → try first part
+    if " and " in name:
+        first = name.split(" and ")[0].strip()
+        result = _scanner_geo_cache.get(f"{first}|{st}")
+        if result:
+            return result
+    # Comma-separated list: "Adams, Jackson, Juneau" → try first
+    if ", " in name:
+        first = name.split(", ")[0].strip()
+        result = _scanner_geo_cache.get(f"{first}|{st}")
+        if result:
+            return result
+    # Drop directional prefix: "North Fulton" → "Fulton"
+    for prefix in ("north ", "south ", "east ", "west "):
+        if name.startswith(prefix):
+            result = _scanner_geo_cache.get(f"{name[len(prefix):]}|{st}")
+            if result:
+                return result
+    return None
+
+
+@with_retry(max_retries=2, base_delay=2)
+def fetch_scanners():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("scanners"):
+        return
+    try:
+        from services.radio_intercept import get_openmhz_systems
+
+        systems = get_openmhz_systems()
+        scanners = []
+        for s in systems:
+            city = s.get("city", "") or s.get("county", "") or ""
+            state = s.get("state", "")
+            coords = _geocode_scanner(city, state)
+            if not coords:
+                continue
+            lat, lng = coords
+            scanners.append(
+                {
+                    "shortName": s.get("shortName", ""),
+                    "name": s.get("name", "Unknown Scanner"),
+                    "lat": round(lat, 5),
+                    "lng": round(lng, 5),
+                    "city": city,
+                    "state": state,
+                    "clientCount": s.get("clientCount", 0),
+                    "description": s.get("description", ""),
+                }
+            )
+        with _data_lock:
+            latest_data["scanners"] = scanners
+        if scanners:
+            _mark_fresh("scanners")
+        logger.info(f"Scanners: {len(scanners)}/{len(systems)} geocoded")
+    except (
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+        TypeError,
+        json.JSONDecodeError,
+    ) as e:
+        logger.error(f"Error fetching scanners: {e}")
+        with _data_lock:
+            latest_data["scanners"] = []

backend/services/fetchers/meshtastic_map.py

@@ -0,0 +1,222 @@
+"""Meshtastic Map fetcher — pulls global node positions from meshtastic.liamcottle.net.
+
+Bootstrap + top-up strategy:
+  - On startup: fetch all nodes with positions to seed the map
+  - Every 4 hours: refresh from the API
+  - Persists to JSON cache so data survives restarts
+  - MQTT bridge provides real-time updates between API fetches
+
+API source: https://meshtastic.liamcottle.net/api/v1/nodes (community project by Liam Cottle)
+Polling interval deliberately kept low (4h) to be respectful to the service.
+"""
+
+import json
+import logging
+import time
+from datetime import datetime, timezone, timedelta
+from pathlib import Path
+
+import requests
+
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+
+logger = logging.getLogger("services.data_fetcher")
+
+_API_URL = "https://meshtastic.liamcottle.net/api/v1/nodes"
+_CACHE_FILE = Path(__file__).resolve().parent.parent.parent / "data" / "meshtastic_nodes_cache.json"
+_FETCH_TIMEOUT = 90  # seconds — response is ~37MB, needs time on slow connections
+_MAX_AGE_HOURS = 4  # discard nodes not seen within this window (matches refresh interval)
+
+# Track when we last fetched so the frontend can show staleness
+_last_fetch_ts: float = 0.0
+
+
+def _parse_node(node: dict) -> dict | None:
+    """Convert an API node into a slim signal-like dict."""
+    lat_i = node.get("latitude")
+    lng_i = node.get("longitude")
+    if lat_i is None or lng_i is None:
+        return None
+
+    lat = lat_i / 1e7
+    lng = lng_i / 1e7
+
+    # Basic validity
+    if not (-90 <= lat <= 90 and -180 <= lng <= 180):
+        return None
+    if abs(lat) < 0.1 and abs(lng) < 0.1:
+        return None
+
+    callsign = node.get("node_id_hex", "")
+    if not callsign:
+        nid = node.get("node_id")
+        callsign = f"!{int(nid):08x}" if nid else ""
+    if not callsign:
+        return None
+
+    # Position age from API — reject nodes older than _MAX_AGE_HOURS
+    pos_updated = node.get("position_updated_at") or node.get("updated_at", "")
+    if pos_updated:
+        try:
+            ts = datetime.fromisoformat(pos_updated.replace("Z", "+00:00"))
+            if datetime.now(timezone.utc) - ts > timedelta(hours=_MAX_AGE_HOURS):
+                return None
+        except (ValueError, TypeError):
+            pass
+    else:
+        return None  # no timestamp at all — skip
+
+    return {
+        "callsign": callsign[:20],
+        "lat": round(lat, 5),
+        "lng": round(lng, 5),
+        "source": "meshtastic",
+        "confidence": 0.5,
+        "timestamp": pos_updated,
+        "position_updated_at": pos_updated,
+        "from_api": True,
+        "long_name": (node.get("long_name") or "")[:40],
+        "short_name": (node.get("short_name") or "")[:4],
+        "hardware": node.get("hardware_model_name", ""),
+        "role": node.get("role_name", ""),
+        "battery_level": node.get("battery_level"),
+        "voltage": node.get("voltage"),
+        "altitude": node.get("altitude"),
+    }
+
+
+def _is_fresh(node: dict) -> bool:
+    """Check if a cached node is still within the _MAX_AGE_HOURS window."""
+    ts_str = node.get("position_updated_at") or node.get("timestamp", "")
+    if not ts_str:
+        return False
+    try:
+        ts = datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
+        return datetime.now(timezone.utc) - ts <= timedelta(hours=_MAX_AGE_HOURS)
+    except (ValueError, TypeError):
+        return False
+
+
+def _load_cache() -> list[dict]:
+    """Load cached nodes from disk, filtering out stale entries."""
+    if _CACHE_FILE.exists():
+        try:
+            data = json.loads(_CACHE_FILE.read_text(encoding="utf-8"))
+            nodes = data.get("nodes", [])
+            fresh = [n for n in nodes if _is_fresh(n)]
+            logger.info(f"Meshtastic map cache loaded: {len(fresh)} fresh / {len(nodes)} total")
+            return fresh
+        except Exception as e:
+            logger.warning(f"Failed to load meshtastic cache: {e}")
+    return []
+
+
+def _save_cache(nodes: list[dict], fetch_ts: float):
+    """Persist processed nodes to disk."""
+    try:
+        _CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
+        _CACHE_FILE.write_text(
+            json.dumps(
+                {
+                    "fetched_at": fetch_ts,
+                    "count": len(nodes),
+                    "nodes": nodes,
+                }
+            ),
+            encoding="utf-8",
+        )
+    except Exception as e:
+        logger.warning(f"Failed to save meshtastic cache: {e}")
+
+
+def fetch_meshtastic_nodes():
+    """Fetch global Meshtastic node positions from Liam Cottle's map API.
+
+    Stores processed nodes in latest_data["meshtastic_map_nodes"].
+    Persists to JSON cache for restart resilience.
+    """
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("sigint_meshtastic"):
+        return
+    global _last_fetch_ts
+
+    try:
+        logger.info("Fetching Meshtastic map nodes from API...")
+        resp = requests.get(
+            _API_URL,
+            timeout=_FETCH_TIMEOUT,
+            headers={
+                "User-Agent": "ShadowBroker/1.0 (OSINT dashboard, 4h polling)",
+                "Accept": "application/json",
+            },
+        )
+        resp.raise_for_status()
+
+        raw = resp.json()
+        raw_nodes = raw.get("nodes", []) if isinstance(raw, dict) else raw
+
+        # Parse and filter to only nodes with valid positions
+        parsed = []
+        for node in raw_nodes:
+            sig = _parse_node(node)
+            if sig:
+                parsed.append(sig)
+
+        _last_fetch_ts = time.time()
+        _save_cache(parsed, _last_fetch_ts)
+
+        with _data_lock:
+            latest_data["meshtastic_map_nodes"] = parsed
+            latest_data["meshtastic_map_fetched_at"] = _last_fetch_ts
+        try:
+            from services.fetchers.sigint import refresh_sigint_snapshot
+
+            refresh_sigint_snapshot()
+        except Exception as exc:
+            logger.debug(f"Meshtastic map: SIGINT snapshot refresh skipped: {exc}")
+
+        logger.info(
+            f"Meshtastic map: {len(parsed)} nodes with positions " f"(from {len(raw_nodes)} total)"
+        )
+
+    except Exception as e:
+        logger.error(f"Meshtastic map fetch failed: {e}")
+        # Fall back to cache if available and we have nothing in memory
+        with _data_lock:
+            if not latest_data.get("meshtastic_map_nodes"):
+                cached = _load_cache()
+                if cached:
+                    latest_data["meshtastic_map_nodes"] = cached
+                    latest_data["meshtastic_map_fetched_at"] = (
+                        _CACHE_FILE.stat().st_mtime if _CACHE_FILE.exists() else 0
+                    )
+                    logger.info(
+                        f"Meshtastic map: using {len(cached)} cached nodes (API unavailable)"
+                    )
+                    try:
+                        from services.fetchers.sigint import refresh_sigint_snapshot
+
+                        refresh_sigint_snapshot()
+                    except Exception as exc:
+                        logger.debug(f"Meshtastic map cache: SIGINT snapshot refresh skipped: {exc}")
+
+    _mark_fresh("meshtastic_map")
+
+
+def load_meshtastic_cache_if_available():
+    """On startup, load cached nodes immediately (before first API fetch)."""
+    global _last_fetch_ts
+    cached = _load_cache()
+    if cached:
+        with _data_lock:
+            latest_data["meshtastic_map_nodes"] = cached
+            _last_fetch_ts = _CACHE_FILE.stat().st_mtime if _CACHE_FILE.exists() else 0
+            latest_data["meshtastic_map_fetched_at"] = _last_fetch_ts
+        try:
+            from services.fetchers.sigint import refresh_sigint_snapshot
+
+            refresh_sigint_snapshot()
+        except Exception as exc:
+            logger.debug(f"Meshtastic preload: SIGINT snapshot refresh skipped: {exc}")
+        logger.info(f"Meshtastic map: preloaded {len(cached)} nodes from cache")

backend/services/fetchers/military.py

@@ -0,0 +1,327 @@
+"""Military flight tracking and UAV detection from ADS-B data."""
+
+import json
+import logging
+import requests
+from services.network_utils import fetch_with_curl
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.plane_alert import enrich_with_plane_alert
+
+logger = logging.getLogger("services.data_fetcher")
+
+# ---------------------------------------------------------------------------
+# UAV classification — filters military drone transponders
+# ---------------------------------------------------------------------------
+_UAV_TYPE_CODES = {"Q9", "R4", "TB2", "MALE", "HALE", "HERM", "HRON"}
+_UAV_CALLSIGN_PREFIXES = ("FORTE", "GHAWK", "REAP", "BAMS", "UAV", "UAS")
+_UAV_MODEL_KEYWORDS = (
+    "RQ-",
+    "MQ-",
+    "RQ4",
+    "MQ9",
+    "MQ4",
+    "MQ1",
+    "REAPER",
+    "GLOBALHAWK",
+    "TRITON",
+    "PREDATOR",
+    "HERMES",
+    "HERON",
+    "BAYRAKTAR",
+)
+_UAV_WIKI = {
+    "RQ4": "https://en.wikipedia.org/wiki/Northrop_Grumman_RQ-4_Global_Hawk",
+    "RQ-4": "https://en.wikipedia.org/wiki/Northrop_Grumman_RQ-4_Global_Hawk",
+    "MQ4": "https://en.wikipedia.org/wiki/Northrop_Grumman_MQ-4C_Triton",
+    "MQ-4": "https://en.wikipedia.org/wiki/Northrop_Grumman_MQ-4C_Triton",
+    "MQ9": "https://en.wikipedia.org/wiki/General_Atomics_MQ-9_Reaper",
+    "MQ-9": "https://en.wikipedia.org/wiki/General_Atomics_MQ-9_Reaper",
+    "MQ1": "https://en.wikipedia.org/wiki/General_Atomics_MQ-1C_Gray_Eagle",
+    "MQ-1": "https://en.wikipedia.org/wiki/General_Atomics_MQ-1C_Gray_Eagle",
+    "REAPER": "https://en.wikipedia.org/wiki/General_Atomics_MQ-9_Reaper",
+    "GLOBALHAWK": "https://en.wikipedia.org/wiki/Northrop_Grumman_RQ-4_Global_Hawk",
+    "TRITON": "https://en.wikipedia.org/wiki/Northrop_Grumman_MQ-4C_Triton",
+    "PREDATOR": "https://en.wikipedia.org/wiki/General_Atomics_MQ-1_Predator",
+    "HERMES": "https://en.wikipedia.org/wiki/Elbit_Hermes_900",
+    "HERON": "https://en.wikipedia.org/wiki/IAI_Heron",
+    "BAYRAKTAR": "https://en.wikipedia.org/wiki/Bayraktar_TB2",
+}
+
+
+_ICAO_COUNTRY_RANGES = [
+    (0x780000, 0x7BFFFF, "China", "PLA"),
+    (0x840000, 0x87FFFF, "Japan", "JSDF"),
+    (0x700000, 0x71FFFF, "South Korea", "ROK"),
+    (0xE80000, 0xE80FFF, "Taiwan", "ROC"),
+    (0x150000, 0x157FFF, "Russia", "VKS"),
+    (0x7C0000, 0x7FFFFF, "Australia", "RAAF"),
+    (0x758000, 0x75FFFF, "Philippines", "PAF"),
+    (0x768000, 0x76FFFF, "Singapore", "RSAF"),
+    (0x720000, 0x727FFF, "North Korea", "KPAF"),
+]
+
+
+def _enrich_country(icao_hex: str, flag: str) -> tuple[str, str]:
+    """If flag is Unknown/empty, infer country and force from ICAO range."""
+    if flag and flag not in ("Unknown", "Military Asset", ""):
+        return flag, ""
+    try:
+        addr = int(icao_hex, 16)
+    except (ValueError, TypeError):
+        return flag or "Military Asset", ""
+    for start, end, country, force in _ICAO_COUNTRY_RANGES:
+        if start <= addr <= end:
+            return country, force
+    return flag or "Military Asset", ""
+
+
+def _classify_military_type(raw_model: str) -> str:
+    model = raw_model.upper().replace("-", "").replace(" ", "")
+    if "H" in model and any(c.isdigit() for c in model):
+        return "heli"
+    if any(k in model for k in [
+        "K35", "K46", "A33", "YY20",
+    ]):
+        return "tanker"
+    if any(k in model for k in [
+        "F16", "F35", "F22", "F15", "F18", "T38", "T6", "A10",
+        "J10", "J11", "J15", "J16", "J20", "JF17",
+        "SU27", "SU30", "SU35", "SU57", "MIG29", "MIG31",
+        "F15J", "F2", "IDF", "FA50", "KF21",
+    ]):
+        return "fighter"
+    if any(k in model for k in [
+        "TU95", "TU160", "TU22",
+    ]):
+        return "bomber"
+    if any(k in model for k in [
+        "C17", "C5", "C130", "C30", "A400", "V22",
+        "Y20", "Y9", "Y8", "C2",
+        "IL76", "AN124", "AN12",
+    ]):
+        return "cargo"
+    if any(k in model for k in [
+        "P8", "E3", "E8", "U2",
+        "KJ500", "KJ200", "GX11", "P1", "E767", "E2K", "E2C",
+        "A50", "TU214R", "IL20",
+    ]):
+        return "recon"
+    return "default"
+
+
+def _classify_uav(model: str, callsign: str):
+    """Check if an aircraft is a UAV based on type code, callsign prefix, or model keywords.
+    Returns (is_uav, uav_type, wiki_url) or (False, None, None)."""
+    model_up = model.upper().replace(" ", "")
+    callsign_up = callsign.upper().strip()
+
+    if model_up in _UAV_TYPE_CODES:
+        uav_type = "HALE Surveillance" if model_up in ("R4", "HALE") else "MALE ISR"
+        wiki = _UAV_WIKI.get(model_up, "")
+        return True, uav_type, wiki
+
+    for prefix in _UAV_CALLSIGN_PREFIXES:
+        if callsign_up.startswith(prefix):
+            uav_type = "HALE Surveillance" if prefix in ("FORTE", "GHAWK", "BAMS") else "MALE ISR"
+            wiki = _UAV_WIKI.get(prefix, "")
+            if prefix == "FORTE":
+                wiki = _UAV_WIKI["RQ4"]
+            elif prefix == "BAMS":
+                wiki = _UAV_WIKI["MQ4"]
+            return True, uav_type, wiki
+
+    for kw in _UAV_MODEL_KEYWORDS:
+        if kw in model_up:
+            if any(h in model_up for h in ("RQ4", "RQ-4", "GLOBALHAWK")):
+                return True, "HALE Surveillance", _UAV_WIKI.get(kw, "")
+            elif any(h in model_up for h in ("MQ4", "MQ-4", "TRITON")):
+                return True, "HALE Maritime Surveillance", _UAV_WIKI.get(kw, "")
+            elif any(h in model_up for h in ("MQ9", "MQ-9", "REAPER")):
+                return True, "MALE Strike/ISR", _UAV_WIKI.get(kw, "")
+            elif any(h in model_up for h in ("MQ1", "MQ-1", "PREDATOR")):
+                return True, "MALE ISR/Strike", _UAV_WIKI.get(kw, "")
+            elif "BAYRAKTAR" in model_up or "TB2" in model_up:
+                return True, "MALE Strike", _UAV_WIKI.get("BAYRAKTAR", "")
+            elif "HERMES" in model_up:
+                return True, "MALE ISR", _UAV_WIKI.get("HERMES", "")
+            elif "HERON" in model_up:
+                return True, "MALE ISR", _UAV_WIKI.get("HERON", "")
+            return True, "MALE ISR", _UAV_WIKI.get(kw, "")
+
+    return False, None, None
+
+
+def fetch_military_flights():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("military"):
+        return
+    military_flights = []
+    detected_uavs = []
+    # Fetch from primary + supplemental military endpoints
+    all_mil_ac = []
+    seen_hex = set()
+    try:
+        url = "https://api.adsb.lol/v2/mil"
+        response = fetch_with_curl(url, timeout=10)
+        if response.status_code == 200:
+            for a in response.json().get("ac", []):
+                h = a.get("hex", "").lower()
+                if h and h not in seen_hex:
+                    seen_hex.add(h)
+                    all_mil_ac.append(a)
+    except Exception as e:
+        logger.warning(f"adsb.lol mil fetch failed: {e}")
+    # Supplemental: airplanes.live military endpoint
+    try:
+        resp2 = fetch_with_curl("https://api.airplanes.live/v2/mil", timeout=10)
+        if resp2.status_code == 200:
+            for a in resp2.json().get("ac", []):
+                h = a.get("hex", "").lower()
+                if h and h not in seen_hex:
+                    seen_hex.add(h)
+                    all_mil_ac.append(a)
+            logger.info(f"airplanes.live mil: +{len(resp2.json().get('ac', []))} raw, {len(all_mil_ac)} total unique")
+    except Exception as e:
+        logger.debug(f"airplanes.live mil supplemental failed: {e}")
+    try:
+        if all_mil_ac:
+            ac = all_mil_ac
+            for f in ac:
+                try:
+                    lat = f.get("lat")
+                    lng = f.get("lon")
+                    heading = f.get("track") or 0
+
+                    if lat is None or lng is None:
+                        continue
+
+                    model = str(f.get("t", "UNKNOWN")).upper()
+                    callsign = str(f.get("flight", "MIL-UNKN")).strip()
+
+                    if model == "TWR":
+                        continue
+
+                    alt_raw = f.get("alt_baro")
+                    alt_value = 0
+                    if isinstance(alt_raw, (int, float)):
+                        alt_value = alt_raw * 0.3048
+
+                    gs_knots = f.get("gs")
+                    speed_knots = round(gs_knots, 1) if isinstance(gs_knots, (int, float)) else None
+
+                    icao_hex = f.get("hex", "")
+
+                    is_uav, uav_type, wiki_url = _classify_uav(model, callsign)
+                    if is_uav:
+                        uav_country, uav_force = _enrich_country(icao_hex, f.get("flag", ""))
+                        detected_uavs.append({
+                            "id": f"uav-{icao_hex}",
+                            "callsign": callsign,
+                            "aircraft_model": f.get("t", "Unknown"),
+                            "lat": float(lat),
+                            "lng": float(lng),
+                            "alt": alt_value,
+                            "heading": heading,
+                            "speed_knots": speed_knots,
+                            "country": uav_country,
+                            "force": uav_force,
+                            "uav_type": uav_type,
+                            "wiki": wiki_url or "",
+                            "type": "uav",
+                            "registration": f.get("r", "N/A"),
+                            "icao24": icao_hex,
+                            "squawk": f.get("squawk", ""),
+                        })
+                        continue
+
+                    mil_country, mil_force = _enrich_country(icao_hex, f.get("flag", ""))
+                    mil_cat = _classify_military_type(f.get("t", "UNKNOWN"))
+
+                    military_flights.append({
+                        "callsign": callsign,
+                        "country": mil_country,
+                        "force": mil_force,
+                        "lng": float(lng),
+                        "lat": float(lat),
+                        "alt": alt_value,
+                        "heading": heading,
+                        "type": "military_flight",
+                        "military_type": mil_cat,
+                        "origin_loc": None,
+                        "dest_loc": None,
+                        "origin_name": "UNKNOWN",
+                        "dest_name": "UNKNOWN",
+                        "registration": f.get("r", "N/A"),
+                        "model": f.get("t", "Unknown"),
+                        "icao24": icao_hex,
+                        "speed_knots": speed_knots,
+                        "squawk": f.get("squawk", "")
+                    })
+                except Exception as loop_e:
+                    logger.error(f"Mil flight interpolation error: {loop_e}")
+                    continue
+    except (
+        requests.RequestException,
+        ConnectionError,
+        TimeoutError,
+        OSError,
+        ValueError,
+        KeyError,
+    ) as e:
+        logger.error(f"Error fetching military flights: {e}")
+
+    if not military_flights and not detected_uavs:
+        logger.warning("No military flights retrieved — keeping previous data if available")
+        with _data_lock:
+            if latest_data.get("military_flights"):
+                return
+
+    with _data_lock:
+        latest_data["military_flights"] = military_flights
+        latest_data["uavs"] = detected_uavs
+    _mark_fresh("military_flights", "uavs")
+    logger.info(f"UAVs: {len(detected_uavs)} real drones detected via ADS-B")
+
+    # Cross-reference military flights with Plane-Alert DB
+    tracked_mil = []
+    remaining_mil = []
+    for mf in military_flights:
+        enrich_with_plane_alert(mf)
+        if mf.get("alert_category"):
+            mf["type"] = "tracked_flight"
+            tracked_mil.append(mf)
+        else:
+            remaining_mil.append(mf)
+    with _data_lock:
+        latest_data["military_flights"] = remaining_mil
+
+    # Store tracked military flights — update positions for existing entries
+    with _data_lock:
+        existing_tracked = list(latest_data.get("tracked_flights", []))
+    fresh_mil_map = {}
+    for t in tracked_mil:
+        icao = t.get("icao24", "").upper()
+        if icao:
+            fresh_mil_map[icao] = t
+
+    updated_tracked = []
+    seen_icaos = set()
+    for old_t in existing_tracked:
+        icao = old_t.get("icao24", "").upper()
+        if icao in fresh_mil_map:
+            fresh = fresh_mil_map[icao]
+            for key in ("alert_category", "alert_operator", "alert_special", "alert_flag"):
+                if key in old_t and key not in fresh:
+                    fresh[key] = old_t[key]
+            updated_tracked.append(fresh)
+            seen_icaos.add(icao)
+        else:
+            updated_tracked.append(old_t)
+            seen_icaos.add(icao)
+    for icao, t in fresh_mil_map.items():
+        if icao not in seen_icaos:
+            updated_tracked.append(t)
+    with _data_lock:
+        latest_data["tracked_flights"] = updated_tracked
+    logger.info(f"Tracked flights: {len(updated_tracked)} total ({len(tracked_mil)} from military)")

backend/services/fetchers/news.py

@@ -0,0 +1,311 @@
+"""News fetching, geocoding, clustering, and risk assessment."""
+import re
+import logging
+import concurrent.futures
+import requests
+import feedparser
+from services.network_utils import fetch_with_curl
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.retry import with_retry
+from services.oracle_service import enrich_news_items, compute_global_threat_level, detect_breaking_events
+
+logger = logging.getLogger("services.data_fetcher")
+
+
+# Keyword -> coordinate mapping for geocoding news articles
+_KEYWORD_COORDS = {
+    "venezuela": (7.119, -66.589),
+    "brazil": (-14.235, -51.925),
+    "argentina": (-38.416, -63.616),
+    "colombia": (4.570, -74.297),
+    "mexico": (23.634, -102.552),
+    "united states": (38.907, -77.036),
+    " usa ": (38.907, -77.036),
+    " us ": (38.907, -77.036),
+    "washington": (38.907, -77.036),
+    "canada": (56.130, -106.346),
+    "ukraine": (49.487, 31.272),
+    "kyiv": (50.450, 30.523),
+    "russia": (61.524, 105.318),
+    "moscow": (55.755, 37.617),
+    "israel": (31.046, 34.851),
+    "gaza": (31.416, 34.333),
+    "iran": (32.427, 53.688),
+    "lebanon": (33.854, 35.862),
+    "syria": (34.802, 38.996),
+    "yemen": (15.552, 48.516),
+    # East Asia — specific locations (longer keywords matched first via _SORTED_KEYWORDS)
+    "taiwan strait": (24.0, 119.5),
+    "south china sea": (15.0, 115.0),
+    "east china sea": (28.0, 125.0),
+    "philippine sea": (20.0, 130.0),
+    "senkaku": (25.740, 123.474),
+    "diaoyu": (25.740, 123.474),
+    "ryukyu": (26.334, 127.800),
+    "okinawa": (26.334, 127.800),
+    "kadena": (26.351, 127.767),
+    "naha": (26.212, 127.679),
+    "yokosuka": (35.283, 139.671),
+    "sasebo": (33.159, 129.722),
+    "misawa": (40.682, 141.368),
+    "iwakuni": (34.144, 132.236),
+    "guam": (13.444, 144.793),
+    "taipei": (25.033, 121.565),
+    "kaohsiung": (22.616, 120.313),
+    "xiamen": (24.479, 118.089),
+    "fujian": (26.074, 119.296),
+    "guangdong": (23.379, 113.763),
+    "zhejiang": (29.141, 119.788),
+    "hainan": (19.200, 109.999),
+    "china": (35.861, 104.195),
+    "beijing": (39.904, 116.407),
+    "taiwan": (23.697, 120.960),
+    "north korea": (40.339, 127.510),
+    "south korea": (35.907, 127.766),
+    "pyongyang": (39.039, 125.762),
+    "seoul": (37.566, 126.978),
+    "japan": (36.204, 138.252),
+    "tokyo": (35.676, 139.650),
+    "afghanistan": (33.939, 67.709),
+    "pakistan": (30.375, 69.345),
+    "india": (20.593, 78.962),
+    " uk ": (55.378, -3.435),
+    "london": (51.507, -0.127),
+    "france": (46.227, 2.213),
+    "paris": (48.856, 2.352),
+    "germany": (51.165, 10.451),
+    "berlin": (52.520, 13.405),
+    "sudan": (12.862, 30.217),
+    "congo": (-4.038, 21.758),
+    "south africa": (-30.559, 22.937),
+    "nigeria": (9.082, 8.675),
+    "egypt": (26.820, 30.802),
+    "zimbabwe": (-19.015, 29.154),
+    "kenya": (-1.292, 36.821),
+    "libya": (26.335, 17.228),
+    "mali": (17.570, -3.996),
+    "niger": (17.607, 8.081),
+    "somalia": (5.152, 46.199),
+    "ethiopia": (9.145, 40.489),
+    "australia": (-25.274, 133.775),
+    "middle east": (31.500, 34.800),
+    "europe": (48.800, 2.300),
+    "africa": (0.000, 25.000),
+    "america": (38.900, -77.000),
+    "south america": (-14.200, -51.900),
+    "asia": (34.000, 100.000),
+    "california": (36.778, -119.417),
+    "texas": (31.968, -99.901),
+    "florida": (27.994, -81.760),
+    "new york": (40.712, -74.006),
+    "virginia": (37.431, -78.656),
+    "british columbia": (53.726, -127.647),
+    "ontario": (51.253, -85.323),
+    "quebec": (52.939, -73.549),
+    "delhi": (28.704, 77.102),
+    "new delhi": (28.613, 77.209),
+    "mumbai": (19.076, 72.877),
+    "shanghai": (31.230, 121.473),
+    "hong kong": (22.319, 114.169),
+    "istanbul": (41.008, 28.978),
+    "dubai": (25.204, 55.270),
+    "singapore": (1.352, 103.819),
+    "bangkok": (13.756, 100.501),
+    "jakarta": (-6.208, 106.845),
+    # East Asia — islands, straits, and disputed areas
+    "pratas": (20.71, 116.72),
+    "dongsha": (20.71, 116.72),
+    "kinmen": (24.45, 118.38),
+    "matsu": (26.16, 119.94),
+    "scarborough": (15.14, 117.77),
+    "paracel": (16.50, 112.00),
+    "spratly": (10.00, 114.00),
+    "miyako strait": (24.78, 125.30),
+    "bashi channel": (21.00, 121.50),
+    "luzon strait": (20.50, 121.50),
+    " dmz ": (38.00, 127.00),
+    "yalu": (40.00, 124.40),
+    "yongbyon": (39.80, 125.76),
+    "wonsan": (39.18, 127.48),
+    "busan": (35.18, 129.07),
+}
+
+# Immutable after module load — sort by descending keyword length so
+# specific locations ("taiwan strait") match before generic ones ("taiwan")
+_SORTED_KEYWORDS = sorted(_KEYWORD_COORDS.items(), key=lambda x: len(x[0]), reverse=True)
+
+
+def _resolve_coords(text: str) -> tuple[float, float] | None:
+    """Return (lat, lng) for the most specific keyword match, or None.
+
+    Longer keywords are tried first. Space-padded keywords (" us ", " uk ")
+    use substring matching on padded text; all others use word-boundary regex.
+    """
+    padded_text = f" {text} "
+    for kw, coords in _SORTED_KEYWORDS:
+        if kw.startswith(" ") or kw.endswith(" "):
+            if kw in padded_text:
+                return coords
+        else:
+            if re.search(r'\b' + re.escape(kw) + r'\b', text):
+                return coords
+    return None
+
+
+@with_retry(max_retries=1, base_delay=2)
+def fetch_news():
+    from services.news_feed_config import get_feeds
+    feed_config = get_feeds()
+    feeds = {f["name"]: f["url"] for f in feed_config}
+    source_weights = {f["name"]: f["weight"] for f in feed_config}
+
+    clusters = {}
+    _cluster_grid = {}
+
+    def _fetch_feed(item):
+        source_name, url = item
+        try:
+            xml_data = fetch_with_curl(url, timeout=10).text
+            return source_name, feedparser.parse(xml_data)
+        except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
+            logger.warning(f"Feed {source_name} failed: {e}")
+            return source_name, None
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=min(len(feeds), 6)) as pool:
+        feed_results = list(pool.map(_fetch_feed, feeds.items()))
+
+    for source_name, feed in feed_results:
+        if not feed:
+            continue
+        for entry in feed.entries[:5]:
+            title = entry.get('title', '')
+            summary = entry.get('summary', '')
+
+            _seismic_kw = ["earthquake", "seismic", "quake", "tremor", "magnitude", "richter"]
+            _text_lower = (title + " " + summary).lower()
+            if any(kw in _text_lower for kw in _seismic_kw):
+                continue
+
+            if source_name == "GDACS":
+                alert_level = entry.get("gdacs_alertlevel", "Green")
+                if alert_level == "Red": risk_score = 10
+                elif alert_level == "Orange": risk_score = 7
+                else: risk_score = 4
+            else:
+                risk_keywords = [
+                    'war', 'missile', 'strike', 'attack', 'crisis', 'tension',
+                    'military', 'conflict', 'defense', 'clash', 'nuclear',
+                    'sanctions', 'ceasefire', 'invasion', 'drone', 'artillery',
+                    'blockade', 'escalation', 'casualties', 'airspace',
+                    'mobilization', 'proxy', 'insurgent', 'coup',
+                    'assassination', 'bioweapon', 'chemical',
+                ]
+                text = (title + " " + summary).lower()
+
+                risk_score = 1
+                for kw in risk_keywords:
+                    if kw in text:
+                        risk_score += 2
+                risk_score = min(10, risk_score)
+
+            lat, lng = None, None
+
+            if 'georss_point' in entry:
+                geo_parts = entry['georss_point'].split()
+                if len(geo_parts) == 2:
+                    lat, lng = float(geo_parts[0]), float(geo_parts[1])
+            elif 'where' in entry and hasattr(entry['where'], 'coordinates'):
+                coords = entry['where'].coordinates
+                lat, lng = coords[1], coords[0]
+
+            if lat is None:
+                text = (title + " " + summary).lower()
+                result = _resolve_coords(text)
+                if result:
+                    lat, lng = result
+
+            if lat is not None:
+                key = None
+                cell_x, cell_y = int(lng // 4), int(lat // 4)
+                for dx in range(-1, 2):
+                    for dy in range(-1, 2):
+                        for ckey in _cluster_grid.get((cell_x + dx, cell_y + dy), []):
+                            parts = ckey.split(",")
+                            elat, elng = float(parts[0]), float(parts[1])
+                            if ((lat - elat)**2 + (lng - elng)**2)**0.5 < 4.0:
+                                key = ckey
+                                break
+                        if key:
+                            break
+                    if key:
+                        break
+                if key is None:
+                    key = f"{lat},{lng}"
+                    _cluster_grid.setdefault((cell_x, cell_y), []).append(key)
+            else:
+                key = title
+
+            if key not in clusters:
+                clusters[key] = []
+
+            clusters[key].append({
+                "title": title,
+                "link": entry.get('link', ''),
+                "published": entry.get('published', ''),
+                "source": source_name,
+                "risk_score": risk_score,
+                "coords": [lat, lng] if lat is not None else None
+            })
+
+    news_items = []
+    for key, articles in clusters.items():
+        articles.sort(key=lambda x: (x['risk_score'], source_weights.get(x["source"], 0)), reverse=True)
+        max_risk = articles[0]['risk_score']
+
+        top_article = articles[0]
+        news_items.append({
+            "title": top_article["title"],
+            "link": top_article["link"],
+            "published": top_article["published"],
+            "source": top_article["source"],
+            "risk_score": max_risk,
+            "coords": top_article["coords"],
+            "cluster_count": len(articles),
+            "articles": articles,
+            "machine_assessment": None
+        })
+
+    news_items.sort(key=lambda x: x['risk_score'], reverse=True)
+
+    # Oracle enrichment: sentiment, oracle scores, prediction market odds
+    try:
+        with _data_lock:
+            markets = list(latest_data.get("prediction_markets", []))
+        enrich_news_items(news_items, source_weights, markets)
+        detect_breaking_events(news_items)
+    except Exception as e:
+        logger.warning(f"Oracle enrichment failed (news still usable): {e}")
+
+    # Global threat level computation (fuses news + markets + military + jamming)
+    try:
+        with _data_lock:
+            markets = list(latest_data.get("prediction_markets", []))
+            mil_flights = list(latest_data.get("military_flights", []))
+            jam_zones = list(latest_data.get("gps_jamming", []))
+            ships = list(latest_data.get("ships", []))
+            corr_alerts = list(latest_data.get("correlations", []))
+        threat_level = compute_global_threat_level(
+            news_items, markets,
+            military_flights=mil_flights,
+            gps_jamming=jam_zones,
+            ships=ships,
+            correlations=corr_alerts,
+        )
+    except Exception as e:
+        logger.warning(f"Threat level computation failed: {e}")
+        threat_level = {"score": 0, "level": "GREEN", "color": "#22c55e", "drivers": []}
+
+    with _data_lock:
+        latest_data['news'] = news_items
+        latest_data['threat_level'] = threat_level
+    _mark_fresh("news")

backend/services/fetchers/plan_vessel_alert.py

@@ -0,0 +1,42 @@
+"""PLAN/CCG Vessel Alert DB — load and enrich AIS vessels with Chinese navy/coast guard metadata."""
+import os
+import json
+import logging
+
+logger = logging.getLogger("services.data_fetcher")
+
+_PLAN_CCG_DB: dict = {}
+
+
+def _load_plan_ccg_db():
+    """Load plan_ccg_vessels.json into memory at import time."""
+    global _PLAN_CCG_DB
+    json_path = os.path.join(
+        os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
+        "data", "plan_ccg_vessels.json"
+    )
+    if not os.path.exists(json_path):
+        logger.warning(f"PLAN/CCG vessel DB not found at {json_path}")
+        return
+    try:
+        with open(json_path, "r", encoding="utf-8") as fh:
+            _PLAN_CCG_DB.update(json.load(fh))
+        logger.info(f"PLAN/CCG vessel DB loaded: {len(_PLAN_CCG_DB)} vessels")
+    except (IOError, OSError, json.JSONDecodeError, ValueError, KeyError) as e:
+        logger.error(f"Failed to load PLAN/CCG vessel DB: {e}")
+
+
+_load_plan_ccg_db()
+
+
+def enrich_with_plan_vessel(ship: dict) -> dict:
+    """If ship's MMSI is in the PLAN/CCG DB, attach enrichment metadata."""
+    mmsi = str(ship.get("mmsi", "")).strip()
+    if mmsi and mmsi in _PLAN_CCG_DB:
+        info = _PLAN_CCG_DB[mmsi]
+        ship["plan_name"] = info.get("name", "")
+        ship["plan_class"] = info.get("class", "")
+        ship["plan_force"] = info.get("force", "")
+        ship["plan_hull"] = info.get("hull_number", "")
+        ship["plan_wiki"] = info.get("wiki", "")
+    return ship

backend/services/fetchers/plane_alert.py

@@ -0,0 +1,344 @@
+"""Plane-Alert DB — load and enrich aircraft with tracked metadata."""
+
+import os
+import json
+import logging
+
+logger = logging.getLogger("services.data_fetcher")
+
+# Exact category -> color mapping for all 53 known categories.
+# O(1) dict lookup — no keyword scanning, no false positives.
+_CATEGORY_COLOR: dict[str, str] = {
+    # YELLOW — Military / Intelligence / Defense
+    "USAF": "yellow",
+    "Other Air Forces": "yellow",
+    "Toy Soldiers": "yellow",
+    "Oxcart": "yellow",
+    "United States Navy": "yellow",
+    "GAF": "yellow",
+    "Hired Gun": "yellow",
+    "United States Marine Corps": "yellow",
+    "Gunship": "yellow",
+    "RAF": "yellow",
+    "Other Navies": "yellow",
+    "Special Forces": "yellow",
+    "Zoomies": "yellow",
+    "Royal Navy Fleet Air Arm": "yellow",
+    "Army Air Corps": "yellow",
+    "Aerobatic Teams": "yellow",
+    "UAV": "yellow",
+    "Ukraine": "yellow",
+    "Nuclear": "yellow",
+    # LIME — Emergency / Medical / Rescue / Fire
+    "Flying Doctors": "#32cd32",
+    "Aerial Firefighter": "#32cd32",
+    "Coastguard": "#32cd32",
+    # BLUE — Government / Law Enforcement / Civil
+    "Police Forces": "blue",
+    "Governments": "blue",
+    "Quango": "blue",
+    "UK National Police Air Service": "blue",
+    "CAP": "blue",
+    # BLACK — Privacy / PIA
+    "PIA": "black",
+    # RED — Dictator / Oligarch
+    "Dictator Alert": "red",
+    "Da Comrade": "red",
+    "Oligarch": "red",
+    # HOT PINK — High Value Assets / VIP / Celebrity
+    "Head of State": "#ff1493",
+    "Royal Aircraft": "#ff1493",
+    "Don't you know who I am?": "#ff1493",
+    "As Seen on TV": "#ff1493",
+    "Bizjets": "#ff1493",
+    "Vanity Plate": "#ff1493",
+    "Football": "#ff1493",
+    # ORANGE — Joe Cool
+    "Joe Cool": "orange",
+    # WHITE — Climate Crisis
+    "Climate Crisis": "white",
+    # PURPLE — General Tracked / Other Notable
+    "Historic": "purple",
+    "Jump Johnny Jump": "purple",
+    "Ptolemy would be proud": "purple",
+    "Distinctive": "purple",
+    "Dogs with Jobs": "purple",
+    "You came here in that thing?": "purple",
+    "Big Hello": "purple",
+    "Watch Me Fly": "purple",
+    "Perfectly Serviceable Aircraft": "purple",
+    "Jesus he Knows me": "purple",
+    "Gas Bags": "purple",
+    "Radiohead": "purple",
+}
+
+
+def _category_to_color(cat: str) -> str:
+    """O(1) exact lookup. Unknown categories default to purple."""
+    return _CATEGORY_COLOR.get(cat, "purple")
+
+
+_PLANE_ALERT_DB: dict = {}
+
+# ---------------------------------------------------------------------------
+# POTUS Fleet — override colors and operator names for presidential aircraft.
+# ---------------------------------------------------------------------------
+_POTUS_FLEET: dict[str, dict] = {
+    "ADFDF8": {
+        "color": "#ff1493",
+        "operator": "Air Force One (82-8000)",
+        "category": "Head of State",
+        "wiki": "Air_Force_One",
+        "fleet": "AF1",
+    },
+    "ADFDF9": {
+        "color": "#ff1493",
+        "operator": "Air Force One (92-9000)",
+        "category": "Head of State",
+        "wiki": "Air_Force_One",
+        "fleet": "AF1",
+    },
+    "ADFEB7": {
+        "color": "blue",
+        "operator": "Air Force Two (98-0001)",
+        "category": "Governments",
+        "wiki": "Air_Force_Two",
+        "fleet": "AF2",
+    },
+    "ADFEB8": {
+        "color": "blue",
+        "operator": "Air Force Two (98-0002)",
+        "category": "Governments",
+        "wiki": "Air_Force_Two",
+        "fleet": "AF2",
+    },
+    "ADFEB9": {
+        "color": "blue",
+        "operator": "Air Force Two (99-0003)",
+        "category": "Governments",
+        "wiki": "Air_Force_Two",
+        "fleet": "AF2",
+    },
+    "ADFEBA": {
+        "color": "blue",
+        "operator": "Air Force Two (99-0004)",
+        "category": "Governments",
+        "wiki": "Air_Force_Two",
+        "fleet": "AF2",
+    },
+    "AE4AE6": {
+        "color": "blue",
+        "operator": "Air Force Two (09-0015)",
+        "category": "Governments",
+        "wiki": "Air_Force_Two",
+        "fleet": "AF2",
+    },
+    "AE4AE8": {
+        "color": "blue",
+        "operator": "Air Force Two (09-0016)",
+        "category": "Governments",
+        "wiki": "Air_Force_Two",
+        "fleet": "AF2",
+    },
+    "AE4AEA": {
+        "color": "blue",
+        "operator": "Air Force Two (09-0017)",
+        "category": "Governments",
+        "wiki": "Air_Force_Two",
+        "fleet": "AF2",
+    },
+    "AE4AEC": {
+        "color": "blue",
+        "operator": "Air Force Two (19-0018)",
+        "category": "Governments",
+        "wiki": "Air_Force_Two",
+        "fleet": "AF2",
+    },
+    "AE0865": {
+        "color": "#ff1493",
+        "operator": "Marine One (VH-3D)",
+        "category": "Head of State",
+        "wiki": "Marine_One",
+        "fleet": "M1",
+    },
+    "AE5E76": {
+        "color": "#ff1493",
+        "operator": "Marine One (VH-92A)",
+        "category": "Head of State",
+        "wiki": "Marine_One",
+        "fleet": "M1",
+    },
+    "AE5E77": {
+        "color": "#ff1493",
+        "operator": "Marine One (VH-92A)",
+        "category": "Head of State",
+        "wiki": "Marine_One",
+        "fleet": "M1",
+    },
+    "AE5E79": {
+        "color": "#ff1493",
+        "operator": "Marine One (VH-92A)",
+        "category": "Head of State",
+        "wiki": "Marine_One",
+        "fleet": "M1",
+    },
+}
+
+
+def _load_plane_alert_db():
+    """Load plane_alert_db.json (exported from SQLite) into memory."""
+    global _PLANE_ALERT_DB
+    json_path = os.path.join(
+        os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
+        "data",
+        "plane_alert_db.json",
+    )
+    if not os.path.exists(json_path):
+        logger.warning(f"Plane-Alert DB not found at {json_path}")
+        return
+    try:
+        with open(json_path, "r", encoding="utf-8") as fh:
+            raw = json.load(fh)
+        for icao_hex, info in raw.items():
+            info["color"] = _category_to_color(info.get("category", ""))
+            override = _POTUS_FLEET.get(icao_hex)
+            if override:
+                info["color"] = override["color"]
+                info["operator"] = override["operator"]
+                info["category"] = override["category"]
+                info["wiki"] = override.get("wiki", "")
+                info["potus_fleet"] = override.get("fleet", "")
+            _PLANE_ALERT_DB[icao_hex] = info
+        logger.info(f"Plane-Alert DB loaded: {len(_PLANE_ALERT_DB)} aircraft")
+    except (IOError, OSError, json.JSONDecodeError, ValueError, KeyError) as e:
+        logger.error(f"Failed to load Plane-Alert DB: {e}")
+
+
+_load_plane_alert_db()
+
+
+def enrich_with_plane_alert(flight: dict) -> dict:
+    """If flight's icao24 is in the Plane-Alert DB, add alert metadata."""
+    icao = flight.get("icao24", "").strip().upper()
+    if icao and icao in _PLANE_ALERT_DB:
+        info = _PLANE_ALERT_DB[icao]
+        flight["alert_category"] = info["category"]
+        flight["alert_color"] = info["color"]
+        flight["alert_operator"] = info["operator"]
+        flight["alert_type"] = info["ac_type"]
+        flight["alert_tags"] = info["tags"]
+        flight["alert_link"] = info["link"]
+        if info.get("wiki"):
+            flight["alert_wiki"] = info["wiki"]
+        if info.get("potus_fleet"):
+            flight["potus_fleet"] = info["potus_fleet"]
+        if info["registration"]:
+            flight["registration"] = info["registration"]
+    return flight
+
+
+_TRACKED_NAMES_DB: dict = {}
+
+
+def _load_tracked_names():
+    global _TRACKED_NAMES_DB
+    json_path = os.path.join(
+        os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
+        "data",
+        "tracked_names.json",
+    )
+    if not os.path.exists(json_path):
+        return
+    try:
+        with open(json_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+            for name, info in data.get("details", {}).items():
+                cat = info.get("category", "Other")
+                socials = info.get("socials")
+                for reg in info.get("registrations", []):
+                    reg_clean = reg.strip().upper()
+                    if reg_clean:
+                        entry = {"name": name, "category": cat}
+                        if socials:
+                            entry["socials"] = socials
+                        _TRACKED_NAMES_DB[reg_clean] = entry
+        logger.info(f"Tracked Names DB loaded: {len(_TRACKED_NAMES_DB)} registrations")
+    except (IOError, OSError, json.JSONDecodeError, ValueError, KeyError) as e:
+        logger.error(f"Failed to load Tracked Names DB: {e}")
+
+
+_load_tracked_names()
+
+
+def enrich_with_tracked_names(flight: dict) -> dict:
+    """If flight's registration matches our Excel extraction, tag it as tracked."""
+    icao = flight.get("icao24", "").strip().upper()
+    if icao in _POTUS_FLEET:
+        return flight
+
+    reg = flight.get("registration", "").strip().upper()
+    callsign = flight.get("callsign", "").strip().upper()
+
+    match = None
+    if reg and reg in _TRACKED_NAMES_DB:
+        match = _TRACKED_NAMES_DB[reg]
+    elif callsign and callsign in _TRACKED_NAMES_DB:
+        match = _TRACKED_NAMES_DB[callsign]
+
+    if match:
+        name = match["name"]
+        flight["alert_operator"] = name
+        flight["alert_category"] = match["category"]
+        if match.get("socials"):
+            flight["alert_socials"] = match["socials"]
+
+        name_lower = name.lower()
+        is_gov = any(
+            w in name_lower
+            for w in [
+                "state of ",
+                "government",
+                "republic",
+                "ministry",
+                "department",
+                "federal",
+                "cia",
+            ]
+        )
+        is_law = any(
+            w in name_lower
+            for w in [
+                "police",
+                "marshal",
+                "sheriff",
+                "douane",
+                "customs",
+                "patrol",
+                "gendarmerie",
+                "guardia",
+                "law enforcement",
+            ]
+        )
+        is_med = any(
+            w in name_lower
+            for w in [
+                "fire",
+                "bomberos",
+                "ambulance",
+                "paramedic",
+                "medevac",
+                "rescue",
+                "hospital",
+                "medical",
+                "lifeflight",
+            ]
+        )
+
+        if is_gov or is_law:
+            flight["alert_color"] = "blue"
+        elif is_med:
+            flight["alert_color"] = "#32cd32"
+        elif "alert_color" not in flight:
+            flight["alert_color"] = "pink"
+
+    return flight

backend/services/fetchers/prediction_markets.py

@@ -0,0 +1,647 @@
+"""Prediction market fetcher — Polymarket (Gamma API) + Kalshi.
+
+Fetches active prediction market events from both platforms, merges them by
+topic similarity, classifies into categories, and stores merged odds with
+full metadata (volume, end dates, descriptions, source badges).
+"""
+
+import json
+import logging
+import math
+from cachetools import TTLCache, cached
+
+logger = logging.getLogger("services.data_fetcher")
+
+_market_cache = TTLCache(maxsize=1, ttl=60)  # 60-second TTL — markets change fast
+
+# Delta tracking: {market_title: previous_consensus_pct}
+_prev_probabilities: dict[str, float] = {}
+
+
+def _finite_or_none(value):
+    try:
+        n = float(value)
+    except (TypeError, ValueError):
+        return None
+    return n if math.isfinite(n) else None
+
+# ---------------------------------------------------------------------------
+# Category classification
+# ---------------------------------------------------------------------------
+CATEGORIES = ["POLITICS", "CONFLICT", "NEWS", "FINANCE", "CRYPTO"]
+
+_KALSHI_CATEGORY_MAP = {
+    "Politics": "POLITICS",
+    "World": "NEWS",
+    "Economics": "FINANCE",
+    "Financials": "FINANCE",
+    "Tech": "FINANCE",
+    "Science": "NEWS",
+    "Climate and Weather": "NEWS",
+    "Sports": "NEWS",
+    "Culture": "NEWS",
+}
+
+_TAG_CATEGORY_MAP = {
+    "Politics": "POLITICS",
+    "Elections": "POLITICS",
+    "US Politics": "POLITICS",
+    "Trump": "POLITICS",
+    "Congress": "POLITICS",
+    "Supreme Court": "POLITICS",
+    "Geopolitics": "CONFLICT",
+    "War": "CONFLICT",
+    "Military": "CONFLICT",
+    "Finance": "FINANCE",
+    "Stocks": "FINANCE",
+    "Economy": "FINANCE",
+    "Business": "FINANCE",
+    "IPOs": "FINANCE",
+    "Crypto": "CRYPTO",
+    "Bitcoin": "CRYPTO",
+    "Ethereum": "CRYPTO",
+    "AI": "NEWS",
+    "Science": "NEWS",
+    "Sports": "NEWS",
+    "Culture": "NEWS",
+    "Entertainment": "NEWS",
+    "Tech": "FINANCE",
+}
+
+_KEYWORD_CATEGORIES = {
+    "CONFLICT": [
+        "war",
+        "military",
+        "attack",
+        "missile",
+        "invasion",
+        "ukraine",
+        "russia",
+        "gaza",
+        "israel",
+        "nato",
+        "troops",
+        "bombing",
+        "nuclear",
+        "sanctions",
+        "ceasefire",
+        "houthi",
+        "iran",
+        "china taiwan",
+        "clash",
+        "conflict",
+        "strike",
+        "weapon",
+    ],
+    "POLITICS": [
+        "trump",
+        "biden",
+        "election",
+        "congress",
+        "senate",
+        "governor",
+        "president",
+        "democrat",
+        "republican",
+        "vote",
+        "party",
+        "cabinet",
+        "impeach",
+        "legislation",
+        "scotus",
+        "poll",
+        "vance",
+        "speaker",
+        "parliament",
+        "prime minister",
+        "macron",
+        "starmer",
+    ],
+    "CRYPTO": [
+        "bitcoin",
+        "btc",
+        "ethereum",
+        "eth",
+        "crypto",
+        "blockchain",
+        "solana",
+        "defi",
+        "nft",
+        "binance",
+        "coinbase",
+        "token",
+        "microstrategy",
+        "stablecoin",
+    ],
+    "FINANCE": [
+        "stock",
+        "fed",
+        "interest rate",
+        "inflation",
+        "gdp",
+        "recession",
+        "s&p",
+        "nasdaq",
+        "dow",
+        "oil",
+        "gold",
+        "treasury",
+        "tariff",
+        "ipo",
+        "earnings",
+        "market cap",
+        "revenue",
+    ],
+}
+
+
+def _classify_category(title: str, poly_tags: list[str], kalshi_category: str) -> str:
+    """Classify a market into one of the 5 categories."""
+    # 1. Kalshi native category
+    if kalshi_category:
+        mapped = _KALSHI_CATEGORY_MAP.get(kalshi_category)
+        if mapped:
+            return mapped
+    # 2. Polymarket tag labels
+    for tag in poly_tags:
+        mapped = _TAG_CATEGORY_MAP.get(tag)
+        if mapped:
+            return mapped
+    # 3. Keyword matching
+    title_lower = title.lower()
+    for cat, keywords in _KEYWORD_CATEGORIES.items():
+        for kw in keywords:
+            if kw in title_lower:
+                return cat
+    # 4. Default
+    return "NEWS"
+
+
+# ---------------------------------------------------------------------------
+# Polymarket
+# ---------------------------------------------------------------------------
+def _fetch_polymarket_events() -> list[dict]:
+    """Fetch active events from Polymarket Gamma API (no auth required).
+
+    Fetches up to 500 events (multiple pages) for better search coverage.
+    """
+    from services.network_utils import fetch_with_curl
+
+    all_events = []
+    for offset in range(0, 500, 100):
+        try:
+            resp = fetch_with_curl(
+                f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=100&offset={offset}",
+                timeout=15,
+            )
+            if not resp or resp.status_code != 200:
+                break
+            page = resp.json()
+            if not isinstance(page, list) or not page:
+                break
+            all_events.extend(page)
+        except Exception as e:
+            logger.warning(f"Polymarket page offset={offset} error: {e}")
+            break
+
+    if not all_events:
+        return []
+
+    try:
+        results = []
+        for ev in all_events:
+            title = ev.get("title", "")
+            if not title:
+                continue
+            # Extract best probability + outcomes from markets
+            markets = ev.get("markets", [])
+            best_pct = None
+            total_volume = 0
+            outcomes = []
+            for m in markets:
+                # Use outcomePrices[0] (Yes price) when available — lastTradePrice
+                # can be for either Yes or No side, causing "99%" for unlikely events
+                raw_op = m.get("outcomePrices")
+                price = None
+                try:
+                    op = json.loads(raw_op) if isinstance(raw_op, str) else raw_op
+                    if isinstance(op, list) and len(op) >= 1:
+                        price = _finite_or_none(op[0])
+                except (json.JSONDecodeError, ValueError, TypeError):
+                    pass
+                if price is None:
+                    price = _finite_or_none(m.get("lastTradePrice") or m.get("bestBid"))
+                pct = None
+                if price is not None:
+                    try:
+                        pct = round(price * 100, 1)
+                        if best_pct is None or pct > best_pct:
+                            best_pct = pct
+                    except (ValueError, TypeError):
+                        pass
+                try:
+                    volume = _finite_or_none(m.get("volume", 0) or 0)
+                    if volume is not None:
+                        total_volume += volume
+                except (ValueError, TypeError):
+                    pass
+                # Collect named outcomes for multi-outcome events
+                oname = m.get("groupItemTitle") or ""
+                if oname and pct is not None:
+                    outcomes.append({"name": oname, "pct": pct})
+            # Only keep outcomes for multi-outcome markets (3+ named outcomes)
+            if len(outcomes) > 2:
+                outcomes.sort(key=lambda x: x["pct"], reverse=True)
+            else:
+                outcomes = []
+
+            # Extract tag labels
+            tag_labels = [t.get("label", "") for t in ev.get("tags", []) if t.get("label")]
+
+            results.append(
+                {
+                    "title": title,
+                    "source": "polymarket",
+                    "pct": best_pct,
+                    "slug": ev.get("slug", ""),
+                    "description": ev.get("description") or "",
+                    "end_date": ev.get("endDate"),
+                    "volume": round(total_volume, 2),
+                    "volume_24h": round(_finite_or_none(ev.get("volume24hr", 0) or 0) or 0, 2),
+                    "tags": tag_labels,
+                    "outcomes": outcomes,
+                }
+            )
+        logger.info(f"Polymarket: fetched {len(results)} active events")
+        return results
+    except Exception as e:
+        logger.error(f"Polymarket fetch error: {e}")
+        return []
+
+
+# ---------------------------------------------------------------------------
+# Kalshi
+# ---------------------------------------------------------------------------
+def _fetch_kalshi_events() -> list[dict]:
+    """Fetch active events from Kalshi public API (no auth required)."""
+    from services.network_utils import fetch_with_curl
+
+    try:
+        resp = fetch_with_curl(
+            "https://api.elections.kalshi.com/v1/events?status=open&limit=100",
+            timeout=15,
+        )
+        if not resp or resp.status_code != 200:
+            logger.warning(f"Kalshi API returned {getattr(resp, 'status_code', 'N/A')}")
+            return []
+        data = resp.json()
+        events = data.get("events", []) if isinstance(data, dict) else []
+
+        results = []
+        for ev in events:
+            title = ev.get("title", "")
+            if not title:
+                continue
+            markets = ev.get("markets", [])
+            best_pct = None
+            total_volume = 0
+            close_dates = []
+            outcomes = []
+            for m in markets:
+                price = m.get("yes_price") or m.get("last_price")
+                pct = None
+                if price is not None:
+                    try:
+                        price = _finite_or_none(price)
+                        if price is None:
+                            raise ValueError("non-finite")
+                        pct = round(price, 1)
+                        if pct <= 1:
+                            pct = round(pct * 100, 1)
+                        if best_pct is None or pct > best_pct:
+                            best_pct = pct
+                    except (ValueError, TypeError):
+                        pass
+                try:
+                        volume = _finite_or_none(
+                            m.get("dollar_volume", 0) or m.get("volume", 0) or 0
+                        )
+                        if volume is not None:
+                            total_volume += int(volume)
+                except (ValueError, TypeError):
+                    pass
+                cd = m.get("close_date")
+                if cd:
+                    close_dates.append(cd)
+                # Collect named outcomes for multi-outcome events
+                oname = m.get("title") or m.get("subtitle", "")
+                if oname and pct is not None:
+                    outcomes.append({"name": oname, "pct": pct})
+            # Only keep outcomes for multi-outcome markets (3+ named outcomes)
+            if len(outcomes) > 2:
+                outcomes.sort(key=lambda x: x["pct"], reverse=True)
+            else:
+                outcomes = []
+
+            # Description: settle_details or underlying
+            desc = (ev.get("settle_details") or ev.get("underlying") or "").strip()
+            sub = ev.get("sub_title", "")
+
+            results.append(
+                {
+                    "title": title,
+                    "source": "kalshi",
+                    "pct": best_pct,
+                    "ticker": ev.get("ticker", ""),
+                    "description": desc,
+                    "sub_title": sub,
+                    "end_date": max(close_dates) if close_dates else None,
+                    "volume": total_volume,
+                    "category": ev.get("category", ""),
+                    "outcomes": outcomes,
+                }
+            )
+        logger.info(f"Kalshi: fetched {len(results)} active events")
+        return results
+    except Exception as e:
+        logger.error(f"Kalshi fetch error: {e}")
+        return []
+
+
+# ---------------------------------------------------------------------------
+# Merge + classify
+# ---------------------------------------------------------------------------
+def _jaccard(a: str, b: str) -> float:
+    """Word-level Jaccard similarity between two strings."""
+    wa = set(a.lower().split())
+    wb = set(b.lower().split())
+    if not wa or not wb:
+        return 0.0
+    return len(wa & wb) / len(wa | wb)
+
+
+def _merge_markets(poly_events: list[dict], kalshi_events: list[dict]) -> list[dict]:
+    """Merge Polymarket and Kalshi events by title similarity.
+
+    Returns a unified list with full metadata, categorized.
+    """
+    merged = []
+    used_kalshi = set()
+
+    for pe in poly_events:
+        best_match = None
+        best_score = 0.0
+        for i, ke in enumerate(kalshi_events):
+            if i in used_kalshi:
+                continue
+            score = _jaccard(pe["title"], ke["title"])
+            if score > best_score and score >= 0.25:
+                best_score = score
+                best_match = (i, ke)
+
+        poly_pct = _finite_or_none(pe.get("pct"))
+        kalshi_pct = None
+        kalshi_vol = 0
+        kalshi_cat = ""
+        kalshi_end = None
+        kalshi_desc = ""
+        kalshi_ticker = ""
+
+        if best_match:
+            used_kalshi.add(best_match[0])
+            ke = best_match[1]
+            kalshi_pct = _finite_or_none(ke.get("pct"))
+            kalshi_vol = _finite_or_none(ke.get("volume", 0)) or 0
+            kalshi_cat = ke.get("category", "")
+            kalshi_end = ke.get("end_date")
+            kalshi_desc = ke.get("description", "")
+            kalshi_ticker = ke.get("ticker", "")
+
+        pcts = [p for p in [poly_pct, kalshi_pct] if p is not None]
+        consensus = round(sum(pcts) / len(pcts), 1) if pcts else None
+
+        # Build sources list
+        sources = []
+        if poly_pct is not None:
+            sources.append({"name": "POLY", "pct": poly_pct})
+        if kalshi_pct is not None:
+            sources.append({"name": "KALSHI", "pct": kalshi_pct})
+
+        category = _classify_category(pe["title"], pe.get("tags", []), kalshi_cat)
+
+        # Use best available description
+        desc = pe.get("description", "") or kalshi_desc
+        end_date = pe.get("end_date") or kalshi_end
+
+        # Use whichever source has more outcomes
+        poly_outcomes = pe.get("outcomes", [])
+        kalshi_outcomes = best_match[1].get("outcomes", []) if best_match else []
+        outcomes = poly_outcomes if len(poly_outcomes) >= len(kalshi_outcomes) else kalshi_outcomes
+
+        merged.append(
+            {
+                "title": pe["title"],
+                "polymarket_pct": poly_pct,
+                "kalshi_pct": kalshi_pct,
+                "consensus_pct": consensus,
+                "description": desc,
+                "end_date": end_date,
+                "volume": _finite_or_none(pe.get("volume", 0)) or 0,
+                "volume_24h": _finite_or_none(pe.get("volume_24h", 0)) or 0,
+                "kalshi_volume": kalshi_vol,
+                "category": category,
+                "sources": sources,
+                "slug": pe.get("slug", ""),
+                "kalshi_ticker": kalshi_ticker,
+                "outcomes": outcomes,
+            }
+        )
+
+    # Unmatched Kalshi events
+    for i, ke in enumerate(kalshi_events):
+        if i in used_kalshi:
+            continue
+        pct = _finite_or_none(ke.get("pct"))
+        sources = []
+        if pct is not None:
+            sources.append({"name": "KALSHI", "pct": pct})
+        category = _classify_category(ke["title"], [], ke.get("category", ""))
+        merged.append(
+            {
+                "title": ke["title"],
+                "polymarket_pct": None,
+                "kalshi_pct": pct,
+                "consensus_pct": pct,
+                "description": ke.get("description", ""),
+                "end_date": ke.get("end_date"),
+                "volume": 0,
+                "volume_24h": 0,
+                "kalshi_volume": _finite_or_none(ke.get("volume", 0)) or 0,
+                "category": category,
+                "sources": sources,
+                "slug": "",
+                "kalshi_ticker": ke.get("ticker", ""),
+                "outcomes": ke.get("outcomes", []),
+            }
+        )
+
+    return merged
+
+
+@cached(_market_cache)
+def fetch_prediction_markets_raw() -> list[dict]:
+    """Fetch and merge prediction markets from both sources. Cached 5 min."""
+    poly = _fetch_polymarket_events()
+    kalshi = _fetch_kalshi_events()
+    merged = _merge_markets(poly, kalshi)
+    logger.info(
+        f"Prediction markets: {len(merged)} merged events "
+        f"({len(poly)} Polymarket, {len(kalshi)} Kalshi)"
+    )
+    return merged
+
+
+def fetch_prediction_markets():
+    """Fetcher entry point — writes merged markets to latest_data."""
+    from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+    global _prev_probabilities
+
+    markets = fetch_prediction_markets_raw()
+
+    # Compute probability deltas vs previous fetch
+    new_probs: dict[str, float] = {}
+    for m in markets:
+        title = m.get("title", "")
+        pct = m.get("consensus_pct")
+        if title and pct is not None:
+            prev = _prev_probabilities.get(title)
+            if prev is not None:
+                m["delta_pct"] = round(pct - prev, 1)
+            else:
+                m["delta_pct"] = None
+            new_probs[title] = pct
+        else:
+            m["delta_pct"] = None
+    _prev_probabilities = new_probs
+
+    # Build trending list (top 10 by absolute delta)
+    trending = sorted(
+        [m for m in markets if m.get("delta_pct") is not None and m["delta_pct"] != 0],
+        key=lambda x: abs(x["delta_pct"]),
+        reverse=True,
+    )[:10]
+
+    with _data_lock:
+        latest_data["prediction_markets"] = markets
+        latest_data["trending_markets"] = trending
+    _mark_fresh("prediction_markets")
+
+
+# ---------------------------------------------------------------------------
+# Direct API search (not limited to cached data)
+# ---------------------------------------------------------------------------
+def search_polymarket_direct(query: str, limit: int = 20) -> list[dict]:
+    """Search Polymarket by scanning API pages for title matches.
+
+    The Gamma API has no text search parameter, so we scan cached events
+    plus additional pages until we find enough matches or exhaust the scan.
+    """
+    from services.network_utils import fetch_with_curl
+
+    q_lower = query.lower()
+    q_words = set(q_lower.split())
+    results = []
+
+    # Scan up to 2000 events (10 pages of 200) looking for title matches
+    for offset in range(0, 2000, 200):
+        try:
+            resp = fetch_with_curl(
+                f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=200&offset={offset}",
+                timeout=15,
+            )
+            if not resp or resp.status_code != 200:
+                break
+            events = resp.json()
+            if not isinstance(events, list) or not events:
+                break
+
+            for ev in events:
+                title = ev.get("title", "")
+                if not title:
+                    continue
+                title_lower = title.lower()
+                # Check if query appears in title or word overlap
+                if q_lower not in title_lower and not any(w in title_lower for w in q_words):
+                    continue
+
+                # Extract same fields as regular fetch
+                markets = ev.get("markets", [])
+                best_pct = None
+                total_volume = 0
+                outcomes = []
+                for m in markets:
+                    # Use outcomePrices[0] (Yes price) when available
+                    raw_op = m.get("outcomePrices")
+                    price = None
+                    try:
+                        op = json.loads(raw_op) if isinstance(raw_op, str) else raw_op
+                        if isinstance(op, list) and len(op) >= 1:
+                            price = _finite_or_none(op[0])
+                    except (json.JSONDecodeError, ValueError, TypeError):
+                        pass
+                    if price is None:
+                        price = _finite_or_none(m.get("lastTradePrice") or m.get("bestBid"))
+                    pct = None
+                    if price is not None:
+                        try:
+                            pct = round(price * 100, 1)
+                            if best_pct is None or pct > best_pct:
+                                best_pct = pct
+                        except (ValueError, TypeError):
+                            pass
+                    try:
+                        volume = _finite_or_none(m.get("volume", 0) or 0)
+                        if volume is not None:
+                            total_volume += volume
+                    except (ValueError, TypeError):
+                        pass
+                    oname = m.get("groupItemTitle") or ""
+                    if oname and pct is not None:
+                        outcomes.append({"name": oname, "pct": pct})
+                if len(outcomes) > 2:
+                    outcomes.sort(key=lambda x: x["pct"], reverse=True)
+                else:
+                    outcomes = []
+
+                tag_labels = [t.get("label", "") for t in ev.get("tags", []) if t.get("label")]
+                category = _classify_category(title, tag_labels, "")
+                sources = []
+                if best_pct is not None:
+                    sources.append({"name": "POLY", "pct": best_pct})
+
+                results.append(
+                    {
+                        "title": title,
+                        "polymarket_pct": best_pct,
+                        "kalshi_pct": None,
+                        "consensus_pct": best_pct,
+                        "description": ev.get("description") or "",
+                        "end_date": ev.get("endDate"),
+                        "volume": round(total_volume, 2),
+                        "volume_24h": round(_finite_or_none(ev.get("volume24hr", 0) or 0) or 0, 2),
+                        "kalshi_volume": 0,
+                        "category": category,
+                        "sources": sources,
+                        "slug": ev.get("slug", ""),
+                        "outcomes": outcomes,
+                    }
+                )
+            # Stop scanning if we have enough results
+            if len(results) >= limit:
+                break
+        except Exception as e:
+            logger.warning(f"Polymarket search scan offset={offset} error: {e}")
+            break
+
+    logger.info(f"Polymarket search '{query}': {len(results)} results (scanned API)")
+    return results[:limit]

backend/services/fetchers/retry.py

@@ -0,0 +1,72 @@
+"""Retry decorator with exponential backoff + jitter for network-bound fetcher functions.
+
+Usage:
+    @with_retry(max_retries=3, base_delay=2)
+    def fetch_something():
+        ...
+"""
+
+import time
+import random
+import logging
+import functools
+import requests
+
+logger = logging.getLogger(__name__)
+
+# Only retry on transient network/OS errors — not on parse errors, key errors, etc.
+TRANSIENT_ERRORS = (
+    TimeoutError,
+    ConnectionError,
+    OSError,
+    requests.RequestException,
+)
+
+
+def with_retry(max_retries: int = 3, base_delay: float = 2.0, max_delay: float = 30.0):
+    """Decorator: retries the wrapped function on transient errors with exponential backoff + jitter.
+
+    Only retries on network/OS errors (TimeoutError, ConnectionError, OSError,
+    requests.RequestException). Non-transient errors (ValueError, KeyError, etc.)
+    propagate immediately.
+
+    Args:
+        max_retries:  Number of retry attempts after the initial failure.
+        base_delay:   Base delay (seconds) for exponential backoff (2 → 4 → 8 …).
+        max_delay:    Cap on the delay between retries.
+    """
+
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            last_exc = None
+            for attempt in range(1 + max_retries):
+                try:
+                    return func(*args, **kwargs)
+                except TRANSIENT_ERRORS as exc:
+                    last_exc = exc
+                    if attempt < max_retries:
+                        delay = min(base_delay * (2**attempt), max_delay)
+                        jitter = random.uniform(0, delay * 0.25)
+                        total = delay + jitter
+                        logger.warning(
+                            "%s failed (attempt %d/%d): %s — retrying in %.1fs",
+                            func.__name__,
+                            attempt + 1,
+                            max_retries + 1,
+                            exc,
+                            total,
+                        )
+                        time.sleep(total)
+                    else:
+                        logger.error(
+                            "%s failed after %d attempts: %s",
+                            func.__name__,
+                            max_retries + 1,
+                            exc,
+                        )
+            raise last_exc  # type: ignore[misc]
+
+        return wrapper
+
+    return decorator

backend/services/fetchers/satellites.py

@@ -0,0 +1,808 @@
+"""Satellite tracking — CelesTrak/TLE fetch, SGP4 propagation, intel classification.
+
+CelesTrak Fair Use Policy (https://celestrak.org/NORAD/elements/):
+  - Do NOT request the same data more than once every 24 hours
+  - Use If-Modified-Since headers for conditional requests
+  - No parallel/concurrent connections — one request at a time
+  - Set a descriptive User-Agent
+"""
+
+import math
+import time
+import json
+import re
+import logging
+import requests
+from pathlib import Path
+from datetime import datetime, timedelta
+from sgp4.api import Satrec, WGS72, jday
+from services.network_utils import fetch_with_curl
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+
+logger = logging.getLogger("services.data_fetcher")
+
+
+def _gmst(jd_ut1):
+    """Greenwich Mean Sidereal Time in radians from Julian Date."""
+    t = (jd_ut1 - 2451545.0) / 36525.0
+    gmst_sec = (
+        67310.54841 + (876600.0 * 3600 + 8640184.812866) * t + 0.093104 * t * t - 6.2e-6 * t * t * t
+    )
+    gmst_rad = (gmst_sec % 86400) / 86400.0 * 2 * math.pi
+    return gmst_rad
+
+
+# Satellite GP data cache
+# CelesTrak fair use: fetch at most once per 24 hours (86400s).
+# SGP4 propagation runs every 60s using cached TLEs — positions stay live.
+_CELESTRAK_FETCH_INTERVAL = 86400  # 24 hours
+_sat_gp_cache = {"data": None, "last_fetch": 0, "source": "none", "last_modified": None}
+_sat_classified_cache = {"data": None, "gp_fetch_ts": 0}
+_SAT_CACHE_PATH = Path(__file__).parent.parent.parent / "data" / "sat_gp_cache.json"
+_SAT_CACHE_META_PATH = Path(__file__).parent.parent.parent / "data" / "sat_gp_cache_meta.json"
+
+
+def _load_sat_cache():
+    """Load satellite GP data from local disk cache."""
+    try:
+        if _SAT_CACHE_PATH.exists():
+            import os
+
+            age_hours = (time.time() - os.path.getmtime(str(_SAT_CACHE_PATH))) / 3600
+            if age_hours < 48:
+                with open(_SAT_CACHE_PATH, "r") as f:
+                    data = json.load(f)
+                if isinstance(data, list) and len(data) > 10:
+                    logger.info(
+                        f"Satellites: Loaded {len(data)} records from disk cache ({age_hours:.1f}h old)"
+                    )
+                    # Restore last_modified from metadata
+                    _load_cache_meta()
+                    return data
+            else:
+                logger.info(f"Satellites: Disk cache is {age_hours:.0f}h old, will try fresh fetch")
+    except (IOError, OSError, json.JSONDecodeError, ValueError, KeyError) as e:
+        logger.warning(f"Satellites: Failed to load disk cache: {e}")
+    return None
+
+
+def _save_sat_cache(data):
+    """Save satellite GP data to local disk cache."""
+    try:
+        _SAT_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
+        with open(_SAT_CACHE_PATH, "w") as f:
+            json.dump(data, f)
+        _save_cache_meta()
+        logger.info(f"Satellites: Saved {len(data)} records to disk cache")
+    except (IOError, OSError) as e:
+        logger.warning(f"Satellites: Failed to save disk cache: {e}")
+
+
+def _load_cache_meta():
+    """Load cache metadata (Last-Modified timestamp) from disk."""
+    try:
+        if _SAT_CACHE_META_PATH.exists():
+            with open(_SAT_CACHE_META_PATH, "r") as f:
+                meta = json.load(f)
+            _sat_gp_cache["last_modified"] = meta.get("last_modified")
+    except (IOError, OSError, json.JSONDecodeError, ValueError, KeyError):
+        pass
+
+
+def _save_cache_meta():
+    """Save cache metadata to disk."""
+    try:
+        with open(_SAT_CACHE_META_PATH, "w") as f:
+            json.dump({"last_modified": _sat_gp_cache.get("last_modified")}, f)
+    except (IOError, OSError):
+        pass
+
+
+# Satellite intelligence classification database
+_SAT_INTEL_DB = [
+    (
+        "USA 224",
+        {
+            "country": "USA",
+            "mission": "military_recon",
+            "sat_type": "KH-11 Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
+        },
+    ),
+    (
+        "USA 245",
+        {
+            "country": "USA",
+            "mission": "military_recon",
+            "sat_type": "KH-11 Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
+        },
+    ),
+    (
+        "USA 290",
+        {
+            "country": "USA",
+            "mission": "military_recon",
+            "sat_type": "KH-11 Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
+        },
+    ),
+    (
+        "USA 314",
+        {
+            "country": "USA",
+            "mission": "military_recon",
+            "sat_type": "KH-11 Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
+        },
+    ),
+    (
+        "USA 338",
+        {
+            "country": "USA",
+            "mission": "military_recon",
+            "sat_type": "Keyhole Successor",
+            "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
+        },
+    ),
+    (
+        "TOPAZ",
+        {
+            "country": "Russia",
+            "mission": "military_recon",
+            "sat_type": "Optical Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)",
+        },
+    ),
+    (
+        "PERSONA",
+        {
+            "country": "Russia",
+            "mission": "military_recon",
+            "sat_type": "Optical Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)",
+        },
+    ),
+    (
+        "KONDOR",
+        {
+            "country": "Russia",
+            "mission": "military_sar",
+            "sat_type": "SAR Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/Kondor_(satellite)",
+        },
+    ),
+    (
+        "BARS-M",
+        {
+            "country": "Russia",
+            "mission": "military_recon",
+            "sat_type": "Mapping Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/Bars-M",
+        },
+    ),
+    (
+        "YAOGAN",
+        {
+            "country": "China",
+            "mission": "military_recon",
+            "sat_type": "Remote Sensing / ELINT",
+            "wiki": "https://en.wikipedia.org/wiki/Yaogan",
+        },
+    ),
+    (
+        "GAOFEN",
+        {
+            "country": "China",
+            "mission": "military_recon",
+            "sat_type": "High-Res Imaging",
+            "wiki": "https://en.wikipedia.org/wiki/Gaofen",
+        },
+    ),
+    (
+        "JILIN",
+        {
+            "country": "China",
+            "mission": "commercial_imaging",
+            "sat_type": "Video / Imaging",
+            "wiki": "https://en.wikipedia.org/wiki/Jilin-1",
+        },
+    ),
+    (
+        "OFEK",
+        {
+            "country": "Israel",
+            "mission": "military_recon",
+            "sat_type": "Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/Ofeq",
+        },
+    ),
+    (
+        "CSO",
+        {
+            "country": "France",
+            "mission": "military_recon",
+            "sat_type": "Optical Reconnaissance",
+            "wiki": "https://en.wikipedia.org/wiki/CSO_(satellite)",
+        },
+    ),
+    (
+        "IGS",
+        {
+            "country": "Japan",
+            "mission": "military_recon",
+            "sat_type": "Intelligence Gathering",
+            "wiki": "https://en.wikipedia.org/wiki/Information_Gathering_Satellite",
+        },
+    ),
+    (
+        "CAPELLA",
+        {
+            "country": "USA",
+            "mission": "sar",
+            "sat_type": "SAR Imaging",
+            "wiki": "https://en.wikipedia.org/wiki/Capella_Space",
+        },
+    ),
+    (
+        "ICEYE",
+        {
+            "country": "Finland",
+            "mission": "sar",
+            "sat_type": "SAR Microsatellite",
+            "wiki": "https://en.wikipedia.org/wiki/ICEYE",
+        },
+    ),
+    (
+        "COSMO-SKYMED",
+        {
+            "country": "Italy",
+            "mission": "sar",
+            "sat_type": "SAR Constellation",
+            "wiki": "https://en.wikipedia.org/wiki/COSMO-SkyMed",
+        },
+    ),
+    (
+        "TANDEM",
+        {
+            "country": "Germany",
+            "mission": "sar",
+            "sat_type": "SAR Interferometry",
+            "wiki": "https://en.wikipedia.org/wiki/TanDEM-X",
+        },
+    ),
+    (
+        "PAZ",
+        {
+            "country": "Spain",
+            "mission": "sar",
+            "sat_type": "SAR Imaging",
+            "wiki": "https://en.wikipedia.org/wiki/PAZ_(satellite)",
+        },
+    ),
+    (
+        "WORLDVIEW",
+        {
+            "country": "USA",
+            "mission": "commercial_imaging",
+            "sat_type": "Maxar High-Res",
+            "wiki": "https://en.wikipedia.org/wiki/WorldView-3",
+        },
+    ),
+    (
+        "GEOEYE",
+        {
+            "country": "USA",
+            "mission": "commercial_imaging",
+            "sat_type": "Maxar Imaging",
+            "wiki": "https://en.wikipedia.org/wiki/GeoEye-1",
+        },
+    ),
+    (
+        "PLEIADES",
+        {
+            "country": "France",
+            "mission": "commercial_imaging",
+            "sat_type": "Airbus Imaging",
+            "wiki": "https://en.wikipedia.org/wiki/Pl%C3%A9iades_(satellite)",
+        },
+    ),
+    (
+        "SPOT",
+        {
+            "country": "France",
+            "mission": "commercial_imaging",
+            "sat_type": "Airbus Medium-Res",
+            "wiki": "https://en.wikipedia.org/wiki/SPOT_(satellite)",
+        },
+    ),
+    (
+        "PLANET",
+        {
+            "country": "USA",
+            "mission": "commercial_imaging",
+            "sat_type": "PlanetScope",
+            "wiki": "https://en.wikipedia.org/wiki/Planet_Labs",
+        },
+    ),
+    (
+        "SKYSAT",
+        {
+            "country": "USA",
+            "mission": "commercial_imaging",
+            "sat_type": "Planet Video",
+            "wiki": "https://en.wikipedia.org/wiki/SkySat",
+        },
+    ),
+    (
+        "BLACKSKY",
+        {
+            "country": "USA",
+            "mission": "commercial_imaging",
+            "sat_type": "BlackSky Imaging",
+            "wiki": "https://en.wikipedia.org/wiki/BlackSky",
+        },
+    ),
+    (
+        "NROL",
+        {
+            "country": "USA",
+            "mission": "sigint",
+            "sat_type": "Classified NRO",
+            "wiki": "https://en.wikipedia.org/wiki/National_Reconnaissance_Office",
+        },
+    ),
+    (
+        "MENTOR",
+        {
+            "country": "USA",
+            "mission": "sigint",
+            "sat_type": "SIGINT / ELINT",
+            "wiki": "https://en.wikipedia.org/wiki/Mentor_(satellite)",
+        },
+    ),
+    (
+        "LUCH",
+        {
+            "country": "Russia",
+            "mission": "sigint",
+            "sat_type": "Relay / SIGINT",
+            "wiki": "https://en.wikipedia.org/wiki/Luch_(satellite)",
+        },
+    ),
+    (
+        "SHIJIAN",
+        {
+            "country": "China",
+            "mission": "sigint",
+            "sat_type": "ELINT / Tech Demo",
+            "wiki": "https://en.wikipedia.org/wiki/Shijian",
+        },
+    ),
+    (
+        "NAVSTAR",
+        {
+            "country": "USA",
+            "mission": "navigation",
+            "sat_type": "GPS",
+            "wiki": "https://en.wikipedia.org/wiki/GPS_satellite_blocks",
+        },
+    ),
+    (
+        "GLONASS",
+        {
+            "country": "Russia",
+            "mission": "navigation",
+            "sat_type": "GLONASS",
+            "wiki": "https://en.wikipedia.org/wiki/GLONASS",
+        },
+    ),
+    (
+        "BEIDOU",
+        {
+            "country": "China",
+            "mission": "navigation",
+            "sat_type": "BeiDou",
+            "wiki": "https://en.wikipedia.org/wiki/BeiDou",
+        },
+    ),
+    (
+        "GALILEO",
+        {
+            "country": "EU",
+            "mission": "navigation",
+            "sat_type": "Galileo",
+            "wiki": "https://en.wikipedia.org/wiki/Galileo_(satellite_navigation)",
+        },
+    ),
+    (
+        "SBIRS",
+        {
+            "country": "USA",
+            "mission": "early_warning",
+            "sat_type": "Missile Warning",
+            "wiki": "https://en.wikipedia.org/wiki/Space-Based_Infrared_System",
+        },
+    ),
+    (
+        "TUNDRA",
+        {
+            "country": "Russia",
+            "mission": "early_warning",
+            "sat_type": "Missile Warning",
+            "wiki": "https://en.wikipedia.org/wiki/Tundra_(satellite)",
+        },
+    ),
+    (
+        "ISS",
+        {
+            "country": "Intl",
+            "mission": "space_station",
+            "sat_type": "Space Station",
+            "wiki": "https://en.wikipedia.org/wiki/International_Space_Station",
+        },
+    ),
+    (
+        "TIANGONG",
+        {
+            "country": "China",
+            "mission": "space_station",
+            "sat_type": "Space Station",
+            "wiki": "https://en.wikipedia.org/wiki/Tiangong_space_station",
+        },
+    ),
+]
+
+
+def _parse_tle_to_gp(name, norad_id, line1, line2):
+    """Convert TLE two-line element to CelesTrak GP-style dict."""
+    try:
+        incl = float(line2[8:16].strip())
+        raan = float(line2[17:25].strip())
+        ecc = float("0." + line2[26:33].strip())
+        argp = float(line2[34:42].strip())
+        ma = float(line2[43:51].strip())
+        mm = float(line2[52:63].strip())
+        bstar_str = line1[53:61].strip()
+        if bstar_str:
+            mantissa = float(bstar_str[:-2]) / 1e5
+            exponent = int(bstar_str[-2:])
+            bstar = mantissa * (10**exponent)
+        else:
+            bstar = 0.0
+        epoch_yr = int(line1[18:20])
+        epoch_day = float(line1[20:32].strip())
+        year = 2000 + epoch_yr if epoch_yr < 57 else 1900 + epoch_yr
+        epoch_dt = datetime(year, 1, 1) + timedelta(days=epoch_day - 1)
+        return {
+            "OBJECT_NAME": name,
+            "NORAD_CAT_ID": norad_id,
+            "MEAN_MOTION": mm,
+            "ECCENTRICITY": ecc,
+            "INCLINATION": incl,
+            "RA_OF_ASC_NODE": raan,
+            "ARG_OF_PERICENTER": argp,
+            "MEAN_ANOMALY": ma,
+            "BSTAR": bstar,
+            "EPOCH": epoch_dt.strftime("%Y-%m-%dT%H:%M:%S"),
+        }
+    except (ValueError, TypeError, IndexError, KeyError):
+        return None
+
+
+def _fetch_satellites_from_tle_api():
+    """Fallback: fetch satellite TLEs from tle.ivanstanojevic.me when CelesTrak is blocked."""
+    search_terms = set()
+    for key, _ in _SAT_INTEL_DB:
+        term = key.split()[0] if len(key.split()) > 1 and key.split()[0] in ("USA", "NROL") else key
+        search_terms.add(term)
+
+    all_results = []
+    seen_ids = set()
+    for term in search_terms:
+        try:
+            url = f"https://tle.ivanstanojevic.me/api/tle/?search={term}&page_size=100&format=json"
+            response = fetch_with_curl(url, timeout=8)
+            if response.status_code != 200:
+                continue
+            data = response.json()
+            for member in data.get("member", []):
+                gp = _parse_tle_to_gp(
+                    member.get("name", "UNKNOWN"),
+                    member.get("satelliteId"),
+                    member.get("line1", ""),
+                    member.get("line2", ""),
+                )
+                if gp:
+                    sat_id = gp.get("NORAD_CAT_ID")
+                    if sat_id not in seen_ids:
+                        seen_ids.add(sat_id)
+                        all_results.append(gp)
+            time.sleep(1)  # Polite delay between requests
+        except (
+            requests.RequestException,
+            ConnectionError,
+            TimeoutError,
+            ValueError,
+            KeyError,
+            json.JSONDecodeError,
+            OSError,
+        ) as e:
+            logger.debug(f"TLE fallback search '{term}' failed: {e}")
+
+    return all_results
+
+
+def fetch_satellites():
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("satellites"):
+        return
+    sats = []
+    try:
+        now_ts = time.time()
+
+        # On first call, try disk cache before hitting CelesTrak
+        if _sat_gp_cache["data"] is None:
+            disk_data = _load_sat_cache()
+            if disk_data:
+                import os
+
+                cache_mtime = (
+                    os.path.getmtime(str(_SAT_CACHE_PATH)) if _SAT_CACHE_PATH.exists() else 0
+                )
+                _sat_gp_cache["data"] = disk_data
+                _sat_gp_cache["last_fetch"] = cache_mtime  # real fetch time so 24h check works
+                _sat_gp_cache["source"] = "disk_cache"
+                logger.info(
+                    f"Satellites: Bootstrapped from disk cache ({len(disk_data)} records, "
+                    f"{(now_ts - cache_mtime) / 3600:.1f}h old)"
+                )
+
+        if (
+            _sat_gp_cache["data"] is None
+            or (now_ts - _sat_gp_cache["last_fetch"]) > _CELESTRAK_FETCH_INTERVAL
+        ):
+            gp_urls = [
+                "https://celestrak.org/NORAD/elements/gp.php?GROUP=active&FORMAT=json",
+                "https://celestrak.com/NORAD/elements/gp.php?GROUP=active&FORMAT=json",
+            ]
+            # Build conditional request headers (CelesTrak fair use)
+            headers = {}
+            if _sat_gp_cache.get("last_modified"):
+                headers["If-Modified-Since"] = _sat_gp_cache["last_modified"]
+
+            for url in gp_urls:
+                try:
+                    response = fetch_with_curl(url, timeout=15, headers=headers)
+                    if response.status_code == 304:
+                        # Data unchanged — reset timer without re-downloading
+                        _sat_gp_cache["last_fetch"] = now_ts
+                        logger.info(
+                            f"Satellites: CelesTrak returned 304 Not Modified (data unchanged)"
+                        )
+                        break
+                    if response.status_code == 200:
+                        gp_data = response.json()
+                        if isinstance(gp_data, list) and len(gp_data) > 100:
+                            _sat_gp_cache["data"] = gp_data
+                            _sat_gp_cache["last_fetch"] = now_ts
+                            _sat_gp_cache["source"] = "celestrak"
+                            # Store Last-Modified header for future conditional requests
+                            if hasattr(response, "headers"):
+                                lm = response.headers.get("Last-Modified")
+                                if lm:
+                                    _sat_gp_cache["last_modified"] = lm
+                            _save_sat_cache(gp_data)
+                            logger.info(
+                                f"Satellites: Downloaded {len(gp_data)} GP records from CelesTrak"
+                            )
+                            break
+                except (
+                    requests.RequestException,
+                    ConnectionError,
+                    TimeoutError,
+                    ValueError,
+                    KeyError,
+                    json.JSONDecodeError,
+                    OSError,
+                ) as e:
+                    logger.warning(f"Satellites: Failed to fetch from {url}: {e}")
+                    continue
+
+            if _sat_gp_cache["data"] is None:
+                logger.info("Satellites: CelesTrak unreachable, trying TLE fallback API...")
+                try:
+                    fallback_data = _fetch_satellites_from_tle_api()
+                    if fallback_data and len(fallback_data) > 10:
+                        _sat_gp_cache["data"] = fallback_data
+                        _sat_gp_cache["last_fetch"] = now_ts
+                        _sat_gp_cache["source"] = "tle_api"
+                        _save_sat_cache(fallback_data)
+                        logger.info(
+                            f"Satellites: Got {len(fallback_data)} records from TLE fallback API"
+                        )
+                except (
+                    requests.RequestException,
+                    ConnectionError,
+                    TimeoutError,
+                    ValueError,
+                    KeyError,
+                    OSError,
+                ) as e:
+                    logger.error(f"Satellites: TLE fallback also failed: {e}")
+
+            if _sat_gp_cache["data"] is None:
+                disk_data = _load_sat_cache()
+                if disk_data:
+                    _sat_gp_cache["data"] = disk_data
+                    _sat_gp_cache["last_fetch"] = now_ts - (_CELESTRAK_FETCH_INTERVAL - 300)
+                    _sat_gp_cache["source"] = "disk_cache"
+
+        data = _sat_gp_cache["data"]
+        if not data:
+            logger.warning("No satellite GP data available from any source")
+            with _data_lock:
+                latest_data["satellites"] = sats
+            return
+
+        if (
+            _sat_classified_cache["gp_fetch_ts"] == _sat_gp_cache["last_fetch"]
+            and _sat_classified_cache["data"]
+        ):
+            classified = _sat_classified_cache["data"]
+            logger.info(
+                f"Satellites: Using cached classification ({len(classified)} sats, TLEs unchanged)"
+            )
+        else:
+            classified = []
+            for sat in data:
+                name = sat.get("OBJECT_NAME", "UNKNOWN").upper()
+                intel = None
+                for key, meta in _SAT_INTEL_DB:
+                    if key.upper() in name:
+                        intel = dict(meta)
+                        break
+                if not intel:
+                    continue
+                entry = {
+                    "id": sat.get("NORAD_CAT_ID"),
+                    "name": sat.get("OBJECT_NAME", "UNKNOWN"),
+                    "MEAN_MOTION": sat.get("MEAN_MOTION"),
+                    "ECCENTRICITY": sat.get("ECCENTRICITY"),
+                    "INCLINATION": sat.get("INCLINATION"),
+                    "RA_OF_ASC_NODE": sat.get("RA_OF_ASC_NODE"),
+                    "ARG_OF_PERICENTER": sat.get("ARG_OF_PERICENTER"),
+                    "MEAN_ANOMALY": sat.get("MEAN_ANOMALY"),
+                    "BSTAR": sat.get("BSTAR"),
+                    "EPOCH": sat.get("EPOCH"),
+                }
+                entry.update(intel)
+                classified.append(entry)
+            _sat_classified_cache["data"] = classified
+            _sat_classified_cache["gp_fetch_ts"] = _sat_gp_cache["last_fetch"]
+            logger.info(
+                f"Satellites: {len(classified)} intel-classified out of {len(data)} total in catalog"
+            )
+
+        all_sats = classified
+
+        now = datetime.utcnow()
+        jd, fr = jday(
+            now.year, now.month, now.day, now.hour, now.minute, now.second + now.microsecond / 1e6
+        )
+
+        for s in all_sats:
+            try:
+                mean_motion = s.get("MEAN_MOTION")
+                ecc = s.get("ECCENTRICITY")
+                incl = s.get("INCLINATION")
+                raan = s.get("RA_OF_ASC_NODE")
+                argp = s.get("ARG_OF_PERICENTER")
+                ma = s.get("MEAN_ANOMALY")
+                bstar = s.get("BSTAR", 0)
+                epoch_str = s.get("EPOCH")
+                norad_id = s.get("id", 0)
+
+                if mean_motion is None or ecc is None or incl is None:
+                    continue
+
+                epoch_dt = datetime.strptime(epoch_str[:19], "%Y-%m-%dT%H:%M:%S")
+                epoch_jd, epoch_fr = jday(
+                    epoch_dt.year,
+                    epoch_dt.month,
+                    epoch_dt.day,
+                    epoch_dt.hour,
+                    epoch_dt.minute,
+                    epoch_dt.second,
+                )
+
+                sat_obj = Satrec()
+                sat_obj.sgp4init(
+                    WGS72,
+                    "i",
+                    norad_id,
+                    (epoch_jd + epoch_fr) - 2433281.5,
+                    bstar,
+                    0.0,
+                    0.0,
+                    ecc,
+                    math.radians(argp),
+                    math.radians(incl),
+                    math.radians(ma),
+                    mean_motion * 2 * math.pi / 1440.0,
+                    math.radians(raan),
+                )
+
+                e, r, v = sat_obj.sgp4(jd, fr)
+                if e != 0:
+                    continue
+
+                x, y, z = r
+                gmst = _gmst(jd + fr)
+                lng_rad = math.atan2(y, x) - gmst
+                lat_rad = math.atan2(z, math.sqrt(x * x + y * y))
+                alt_km = math.sqrt(x * x + y * y + z * z) - 6371.0
+
+                s["lat"] = round(math.degrees(lat_rad), 4)
+                lng_deg = math.degrees(lng_rad) % 360
+                s["lng"] = round(lng_deg - 360 if lng_deg > 180 else lng_deg, 4)
+                s["alt_km"] = round(alt_km, 1)
+
+                vx, vy, vz = v
+                omega_e = 7.2921159e-5
+                vx_g = vx + omega_e * y
+                vy_g = vy - omega_e * x
+                vz_g = vz
+                cos_lat = math.cos(lat_rad)
+                sin_lat = math.sin(lat_rad)
+                cos_lng = math.cos(lng_rad + gmst)
+                sin_lng = math.sin(lng_rad + gmst)
+                v_east = -sin_lng * vx_g + cos_lng * vy_g
+                v_north = -sin_lat * cos_lng * vx_g - sin_lat * sin_lng * vy_g + cos_lat * vz_g
+                ground_speed_kms = math.sqrt(v_east**2 + v_north**2)
+                s["speed_knots"] = round(ground_speed_kms * 1943.84, 1)
+                heading_rad = math.atan2(v_east, v_north)
+                s["heading"] = round(math.degrees(heading_rad) % 360, 1)
+                sat_name = s.get("name", "")
+                usa_match = re.search(r"USA[\s\-]*(\d+)", sat_name)
+                if usa_match:
+                    s["wiki"] = f"https://en.wikipedia.org/wiki/USA-{usa_match.group(1)}"
+                for k in (
+                    "MEAN_MOTION",
+                    "ECCENTRICITY",
+                    "INCLINATION",
+                    "RA_OF_ASC_NODE",
+                    "ARG_OF_PERICENTER",
+                    "MEAN_ANOMALY",
+                    "BSTAR",
+                    "EPOCH",
+                    "tle1",
+                    "tle2",
+                ):
+                    s.pop(k, None)
+                sats.append(s)
+            except (ValueError, TypeError, KeyError, AttributeError, ZeroDivisionError):
+                continue
+
+        logger.info(f"Satellites: {len(classified)} classified, {len(sats)} positioned")
+    except (
+        requests.RequestException,
+        ConnectionError,
+        TimeoutError,
+        ValueError,
+        KeyError,
+        json.JSONDecodeError,
+        OSError,
+    ) as e:
+        logger.error(f"Error fetching satellites: {e}")
+    if sats:
+        with _data_lock:
+            latest_data["satellites"] = sats
+            latest_data["satellite_source"] = _sat_gp_cache.get("source", "none")
+        _mark_fresh("satellites")
+    else:
+        with _data_lock:
+            if not latest_data.get("satellites"):
+                latest_data["satellites"] = []
+                latest_data["satellite_source"] = "none"

backend/services/fetchers/sigint.py

@@ -0,0 +1,102 @@
+"""SIGINT fetcher — pulls latest signals from the SIGINT Grid into latest_data.
+
+Merges live MQTT signals with cached Meshtastic map API nodes.
+Live MQTT signals always take priority (fresher) — API nodes fill in the gaps
+for the thousands of nodes our MQTT listener hasn't heard yet.
+"""
+
+import logging
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+
+logger = logging.getLogger("services.data_fetcher")
+
+
+def _merge_sigint_snapshot(
+    live_signals: list[dict],
+    api_nodes: list[dict],
+) -> list[dict]:
+    """Merge live bridge signals with cached Meshtastic map nodes.
+
+    Live Meshtastic observations always win over map/API nodes for the same callsign
+    because they include fresher region/channel metadata.
+    """
+
+    merged = list(live_signals)
+    live_callsigns = {s["callsign"] for s in merged if s.get("source") == "meshtastic"}
+    for node in api_nodes:
+        if node.get("callsign") in live_callsigns:
+            continue
+        merged.append(node)
+    merged.sort(key=lambda item: str(item.get("timestamp", "") or ""), reverse=True)
+    return merged
+
+
+def _sigint_totals(signals: list[dict]) -> dict[str, int]:
+    totals = {
+        "total": len(signals),
+        "meshtastic": 0,
+        "meshtastic_live": 0,
+        "meshtastic_map": 0,
+        "aprs": 0,
+        "js8call": 0,
+    }
+    for sig in signals:
+        source = str(sig.get("source", "") or "").lower()
+        if source == "meshtastic":
+            totals["meshtastic"] += 1
+            if bool(sig.get("from_api")):
+                totals["meshtastic_map"] += 1
+            else:
+                totals["meshtastic_live"] += 1
+        elif source == "aprs":
+            totals["aprs"] += 1
+        elif source == "js8call":
+            totals["js8call"] += 1
+    return totals
+
+
+def build_sigint_snapshot() -> tuple[list[dict], dict[str, object], dict[str, int]]:
+    """Build the current merged SIGINT snapshot without hitting the network."""
+
+    from services.sigint_bridge import sigint_grid
+
+    live_signals = sigint_grid.get_all_signals()
+    with _data_lock:
+        api_nodes = list(latest_data.get("meshtastic_map_nodes", []))
+    merged = _merge_sigint_snapshot(live_signals, api_nodes)
+    channel_stats = sigint_grid.get_mesh_channel_stats(api_nodes or None)
+    totals = _sigint_totals(merged)
+    return merged, channel_stats, totals
+
+
+def refresh_sigint_snapshot() -> tuple[list[dict], dict[str, object], dict[str, int]]:
+    """Refresh latest_data SIGINT state from current bridge + cache state."""
+
+    signals, channel_stats, totals = build_sigint_snapshot()
+    with _data_lock:
+        latest_data["sigint"] = signals
+        latest_data["mesh_channel_stats"] = channel_stats
+        latest_data["sigint_totals"] = totals
+    _mark_fresh("sigint")
+    return signals, channel_stats, totals
+
+
+def fetch_sigint():
+    """Fetch all signals from the SIGINT Grid, merge with Meshtastic map nodes."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("sigint_meshtastic", "sigint_aprs"):
+        return
+    from services.sigint_bridge import sigint_grid
+
+    # Start bridges on first call (idempotent)
+    sigint_grid.start()
+
+    signals, channel_stats, totals = refresh_sigint_snapshot()
+
+    status = sigint_grid.status
+    logger.info(
+        f"SIGINT: {len(signals)} signals "
+        f"(APRS:{status['aprs']} MESH:{status['meshtastic']} "
+        f"JS8:{status['js8call']} MAP:{totals['meshtastic_map']})"
+    )

backend/services/fetchers/trains.py

@@ -0,0 +1,457 @@
+"""Train tracking fetchers with normalized metadata and non-redundant merging."""
+
+from __future__ import annotations
+
+import logging
+import math
+from collections.abc import Callable
+from datetime import datetime, timezone
+
+from services.fetchers._store import _data_lock, _mark_fresh, latest_data
+from services.network_utils import fetch_with_curl
+
+logger = logging.getLogger(__name__)
+
+_EARTH_RADIUS_KM = 6371.0
+_MERGE_DISTANCE_KM = 5.0
+_MAX_INFERRED_SPEED_KMH = 350.0
+_TRACK_CACHE_TTL_S = 6 * 60 * 60
+
+_SOURCE_METADATA: dict[str, dict[str, object]] = {
+    "amtrak": {
+        "source_label": "Amtraker",
+        "operator": "Amtrak",
+        "country": "US",
+        "telemetry_quality": "aggregated",
+        "priority": 70,
+    },
+    "digitraffic": {
+        "source_label": "Digitraffic Finland",
+        "operator": "Finnish Rail",
+        "country": "FI",
+        "telemetry_quality": "official",
+        "priority": 100,
+    },
+    # Future slots so better official feeds can be merged without changing the
+    # rest of the train pipeline or duplicating map entities.
+    "networkrail": {
+        "source_label": "Network Rail Open Data",
+        "operator": "Network Rail",
+        "country": "GB",
+        "telemetry_quality": "official",
+        "priority": 98,
+    },
+    "dbcargo": {
+        "source_label": "DB Cargo link2rail",
+        "operator": "DB Cargo",
+        "country": "DE",
+        "telemetry_quality": "commercial",
+        "priority": 96,
+    },
+    "railinc": {
+        "source_label": "Railinc RailSight",
+        "operator": "Railinc",
+        "country": "US",
+        "telemetry_quality": "commercial",
+        "priority": 97,
+    },
+    "sncf": {
+        "source_label": "SNCF Open Data",
+        "operator": "SNCF",
+        "country": "FR",
+        "telemetry_quality": "official",
+        "priority": 94,
+    },
+}
+
+_TRAIN_TRACK_CACHE: dict[str, dict[str, float]] = {}
+
+
+def _safe_float(value) -> float | None:
+    try:
+        if value is None or value == "":
+            return None
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _parse_observed_at(value) -> float | None:
+    if value is None or value == "":
+        return None
+    if isinstance(value, (int, float)):
+        raw = float(value)
+        return raw / 1000.0 if raw > 1_000_000_000_000 else raw
+    if not isinstance(value, str):
+        return None
+    text = value.strip()
+    if not text:
+        return None
+    if text.endswith("Z"):
+        text = f"{text[:-1]}+00:00"
+    try:
+        return datetime.fromisoformat(text).timestamp()
+    except ValueError:
+        return None
+
+
+def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
+    lat1_rad, lon1_rad = math.radians(lat1), math.radians(lon1)
+    lat2_rad, lon2_rad = math.radians(lat2), math.radians(lon2)
+    dlat = lat2_rad - lat1_rad
+    dlon = lon2_rad - lon1_rad
+    a = (
+        math.sin(dlat / 2.0) ** 2
+        + math.cos(lat1_rad) * math.cos(lat2_rad) * math.sin(dlon / 2.0) ** 2
+    )
+    return 2.0 * _EARTH_RADIUS_KM * math.asin(math.sqrt(a))
+
+
+def _bearing_degrees(lat1: float, lon1: float, lat2: float, lon2: float) -> float | None:
+    if lat1 == lat2 and lon1 == lon2:
+        return None
+    lat1_rad, lat2_rad = math.radians(lat1), math.radians(lat2)
+    dlon_rad = math.radians(lon2 - lon1)
+    y = math.sin(dlon_rad) * math.cos(lat2_rad)
+    x = (
+        math.cos(lat1_rad) * math.sin(lat2_rad)
+        - math.sin(lat1_rad) * math.cos(lat2_rad) * math.cos(dlon_rad)
+    )
+    return (math.degrees(math.atan2(y, x)) + 360.0) % 360.0
+
+
+def _source_meta(source: str) -> dict[str, object]:
+    return dict(_SOURCE_METADATA.get(source, {}))
+
+
+def _normalize_train(
+    *,
+    source: str,
+    raw_id: str,
+    number: str,
+    lat,
+    lng,
+    name: str = "",
+    status: str = "Active",
+    route: str = "",
+    speed_kmh=None,
+    heading=None,
+    operator: str | None = None,
+    country: str | None = None,
+    source_label: str | None = None,
+    telemetry_quality: str | None = None,
+    observed_at=None,
+) -> dict | None:
+    lat_f = _safe_float(lat)
+    lng_f = _safe_float(lng)
+    if lat_f is None or lng_f is None:
+        return None
+    if not (-90.0 <= lat_f <= 90.0 and -180.0 <= lng_f <= 180.0):
+        return None
+
+    number_text = str(number or "").strip()
+    meta = _source_meta(source)
+    observed_ts = _parse_observed_at(observed_at) or datetime.now(timezone.utc).timestamp()
+    speed_f = _safe_float(speed_kmh)
+    heading_f = _safe_float(heading)
+    normalized = {
+        "id": str(raw_id or f"{source}-{number_text or 'unknown'}"),
+        "name": str(name or f"Train {number_text or '?'}").strip(),
+        "number": number_text,
+        "source": source,
+        "source_label": str(source_label or meta.get("source_label") or source.upper()),
+        "operator": str(operator or meta.get("operator") or "").strip(),
+        "country": str(country or meta.get("country") or "").strip(),
+        "telemetry_quality": str(
+            telemetry_quality or meta.get("telemetry_quality") or "unknown"
+        ).strip(),
+        "lat": lat_f,
+        "lng": lng_f,
+        "speed_kmh": speed_f,
+        "heading": heading_f,
+        "status": str(status or "Active").strip(),
+        "route": str(route or "").strip(),
+        "_source_priority": int(meta.get("priority") or 0),
+        "_observed_ts": observed_ts,
+    }
+    _apply_motion_estimates(normalized)
+    return normalized
+
+
+def _prune_track_cache(now_ts: float) -> None:
+    stale_before = now_ts - _TRACK_CACHE_TTL_S
+    stale_ids = [train_id for train_id, entry in _TRAIN_TRACK_CACHE.items() if entry["ts"] < stale_before]
+    for train_id in stale_ids:
+        _TRAIN_TRACK_CACHE.pop(train_id, None)
+
+
+def _apply_motion_estimates(train: dict) -> None:
+    train_id = str(train.get("id") or "")
+    if not train_id:
+        return
+    now_ts = float(train.get("_observed_ts") or datetime.now(timezone.utc).timestamp())
+    _prune_track_cache(now_ts)
+    previous = _TRAIN_TRACK_CACHE.get(train_id)
+    if previous:
+        dt_s = now_ts - previous["ts"]
+        if 5.0 <= dt_s <= 15.0 * 60.0:
+            distance_km = _haversine_km(
+                float(previous["lat"]),
+                float(previous["lng"]),
+                float(train["lat"]),
+                float(train["lng"]),
+            )
+            if 0.02 <= distance_km <= (_MAX_INFERRED_SPEED_KMH * (dt_s / 3600.0)):
+                if train.get("speed_kmh") is None:
+                    inferred_speed = distance_km / (dt_s / 3600.0)
+                    train["speed_kmh"] = round(min(inferred_speed, _MAX_INFERRED_SPEED_KMH), 1)
+                if train.get("heading") is None:
+                    inferred_heading = _bearing_degrees(
+                        float(previous["lat"]),
+                        float(previous["lng"]),
+                        float(train["lat"]),
+                        float(train["lng"]),
+                    )
+                    if inferred_heading is not None:
+                        train["heading"] = round(inferred_heading, 1)
+
+    _TRAIN_TRACK_CACHE[train_id] = {
+        "lat": float(train["lat"]),
+        "lng": float(train["lng"]),
+        "ts": now_ts,
+    }
+
+
+def _train_merge_key(train: dict) -> str:
+    operator = str(train.get("operator") or "").strip().lower()
+    country = str(train.get("country") or "").strip().lower()
+    number = str(train.get("number") or "").strip().lower()
+    if operator and number:
+        return f"{country}|{operator}|{number}"
+    return f"{str(train.get('source') or '').lower()}|{str(train.get('id') or '').lower()}"
+
+
+def _train_completeness(train: dict) -> tuple[int, int, int]:
+    return (
+        1 if train.get("speed_kmh") is not None else 0,
+        1 if train.get("heading") is not None else 0,
+        1 if train.get("route") else 0,
+    )
+
+
+def _should_merge(existing: dict, candidate: dict) -> bool:
+    if _train_merge_key(existing) != _train_merge_key(candidate):
+        return False
+    return _haversine_km(
+        float(existing["lat"]),
+        float(existing["lng"]),
+        float(candidate["lat"]),
+        float(candidate["lng"]),
+    ) <= _MERGE_DISTANCE_KM
+
+
+def _merge_train_pair(existing: dict, candidate: dict) -> dict:
+    existing_priority = int(existing.get("_source_priority") or 0)
+    candidate_priority = int(candidate.get("_source_priority") or 0)
+    existing_score = (existing_priority, _train_completeness(existing))
+    candidate_score = (candidate_priority, _train_completeness(candidate))
+    primary = candidate if candidate_score > existing_score else existing
+    secondary = existing if primary is candidate else candidate
+    merged = dict(primary)
+
+    for field in (
+        "speed_kmh",
+        "heading",
+        "route",
+        "status",
+        "operator",
+        "country",
+        "source_label",
+        "telemetry_quality",
+    ):
+        if merged.get(field) in (None, "", "Active"):
+            replacement = secondary.get(field)
+            if replacement not in (None, ""):
+                merged[field] = replacement
+
+    if primary is not candidate and float(candidate.get("_observed_ts") or 0) > float(
+        primary.get("_observed_ts") or 0
+    ):
+        merged["lat"] = candidate["lat"]
+        merged["lng"] = candidate["lng"]
+        merged["_observed_ts"] = candidate["_observed_ts"]
+    return merged
+
+
+def _merge_nonredundant_trains(*sources: list[dict]) -> list[dict]:
+    merged: list[dict] = []
+    for source_trains in sources:
+        for train in source_trains:
+            exact_match = next(
+                (
+                    idx
+                    for idx, existing in enumerate(merged)
+                    if existing.get("source") == train.get("source")
+                    and existing.get("id") == train.get("id")
+                ),
+                None,
+            )
+            if exact_match is not None:
+                merged[exact_match] = _merge_train_pair(merged[exact_match], train)
+                continue
+
+            merged_idx = next(
+                (idx for idx, existing in enumerate(merged) if _should_merge(existing, train)),
+                None,
+            )
+            if merged_idx is not None:
+                merged[merged_idx] = _merge_train_pair(merged[merged_idx], train)
+                continue
+            merged.append(train)
+
+    merged.sort(
+        key=lambda train: (
+            str(train.get("country") or ""),
+            str(train.get("operator") or ""),
+            str(train.get("number") or ""),
+            str(train.get("id") or ""),
+        )
+    )
+    for train in merged:
+        train.pop("_source_priority", None)
+        train.pop("_observed_ts", None)
+    return merged
+
+
+def _fetch_amtraker() -> list[dict]:
+    """Fetch all active Amtrak trains from the Amtraker API."""
+    try:
+        resp = fetch_with_curl(
+            "https://api.amtraker.com/v3/trains",
+            timeout=20,
+            headers={
+                "User-Agent": (
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                    "AppleWebKit/537.36 (KHTML, like Gecko) "
+                    "Chrome/136.0.0.0 Safari/537.36"
+                ),
+                "Accept": "application/json,text/plain,*/*",
+                "Referer": "https://www.amtraker.com/",
+            },
+        )
+        if resp.status_code != 200:
+            logger.warning("Amtraker returned %s", resp.status_code)
+            return []
+        raw = resp.json()
+        trains: list[dict] = []
+        for train_num, variants in raw.items():
+            if not isinstance(variants, list):
+                continue
+            for item in variants:
+                normalized = _normalize_train(
+                    source="amtrak",
+                    raw_id=f"AMTK-{item.get('trainID', train_num)}",
+                    name=item.get("routeName", f"Train {train_num}"),
+                    number=str(item.get("trainNum", train_num) or train_num),
+                    lat=item.get("lat"),
+                    lng=item.get("lon"),
+                    speed_kmh=item.get("velocity") or item.get("speed"),
+                    heading=item.get("heading") or item.get("bearing"),
+                    status=item.get("trainTimely") or "On Time",
+                    route=item.get("routeName", ""),
+                    observed_at=item.get("updatedAt")
+                    or item.get("lastValTS")
+                    or item.get("eventDT"),
+                )
+                if normalized:
+                    trains.append(normalized)
+        return trains
+    except Exception as exc:
+        logger.warning("Amtraker fetch error: %s", exc)
+        return []
+
+
+def _fetch_digitraffic() -> list[dict]:
+    """Fetch live train positions from Finnish DigiTraffic API."""
+    try:
+        resp = fetch_with_curl(
+            "https://rata.digitraffic.fi/api/v1/train-locations/latest",
+            timeout=15,
+            headers={
+                "Accept-Encoding": "gzip",
+                "User-Agent": "ShadowBroker-OSINT/1.0",
+            },
+        )
+        if resp.status_code != 200:
+            logger.warning("DigiTraffic returned %s", resp.status_code)
+            return []
+        raw = resp.json()
+        trains: list[dict] = []
+        for item in raw:
+            location = item.get("location", {})
+            coords = location.get("coordinates")
+            if not coords or len(coords) < 2:
+                continue
+            lon, lat = coords[0], coords[1]
+            train_number = str(item.get("trainNumber", "") or "").strip()
+            route_bits = [
+                str(item.get("departureStationShortCode") or "").strip(),
+                str(item.get("stationShortCode") or "").strip(),
+            ]
+            route = " -> ".join([bit for bit in route_bits if bit])
+            train_type = str(item.get("trainType") or "").strip()
+            normalized = _normalize_train(
+                source="digitraffic",
+                raw_id=f"FIN-{train_number or len(trains)}",
+                name=f"{train_type} {train_number}".strip() or f"Train {train_number or '?'}",
+                number=train_number,
+                lat=lat,
+                lng=lon,
+                speed_kmh=item.get("speed"),
+                heading=item.get("heading"),
+                status="Active",
+                route=route,
+                observed_at=item.get("timestamp"),
+            )
+            if normalized:
+                trains.append(normalized)
+        return trains
+    except Exception as exc:
+        logger.warning("DigiTraffic fetch error: %s", exc)
+        return []
+
+
+_TRAIN_FETCHERS: tuple[tuple[str, Callable[[], list[dict]]], ...] = (
+    ("amtrak", _fetch_amtraker),
+    ("digitraffic", _fetch_digitraffic),
+)
+
+
+def fetch_trains():
+    """Fetch trains from all configured sources and merge without duplicates."""
+    with _data_lock:
+        existing_trains = list(latest_data.get("trains") or [])
+    source_batches: list[list[dict]] = []
+    source_counts: list[str] = []
+    for source_name, fetcher in _TRAIN_FETCHERS:
+        batch = fetcher()
+        source_batches.append(batch)
+        if batch:
+            source_counts.append(f"{source_name}:{len(batch)}")
+
+    trains = _merge_nonredundant_trains(*source_batches)
+    if not trains and existing_trains:
+        logger.warning(
+            "Train refresh returned 0 records — preserving %s cached trains until the next successful poll",
+            len(existing_trains),
+        )
+        trains = existing_trains
+
+    with _data_lock:
+        latest_data["trains"] = trains
+    _mark_fresh("trains")
+    logger.info(
+        "Trains: %s total%s",
+        len(trains),
+        f" ({', '.join(source_counts)})" if source_counts else "",
+    )

backend/services/fetchers/ukraine_alerts.py

@@ -0,0 +1,139 @@
+"""Ukraine air raid alerts via alerts.in.ua API.
+
+Polls active alerts every 2 minutes, matches to oblast boundary polygons,
+and produces GeoJSON-style records for map rendering.
+
+Requires ALERTS_IN_UA_TOKEN env var (free registration at alerts.in.ua).
+Gracefully skips if token is not set.
+"""
+
+import json
+import logging
+import os
+from pathlib import Path
+
+from services.network_utils import fetch_with_curl
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger(__name__)
+
+# ─── Alert type → color mapping ──────────────────────────────────────────────
+ALERT_COLORS = {
+    "air_raid": "#ef4444",            # red
+    "artillery_shelling": "#f97316",  # orange
+    "urban_fights": "#eab308",        # yellow
+    "chemical": "#a855f7",            # purple
+    "nuclear": "#dc2626",             # dark red
+}
+
+# ─── Load oblast boundary polygons (once) ────────────────────────────────────
+_oblast_geojson = None
+
+
+def _load_oblasts():
+    global _oblast_geojson
+    if _oblast_geojson is not None:
+        return _oblast_geojson
+
+    data_path = Path(__file__).resolve().parent.parent.parent / "data" / "ukraine_oblasts.geojson"
+    if not data_path.exists():
+        logger.error(f"Ukraine oblasts GeoJSON not found at {data_path}")
+        _oblast_geojson = {}
+        return _oblast_geojson
+
+    with open(data_path, "r", encoding="utf-8") as f:
+        _oblast_geojson = json.load(f)
+
+    logger.info(f"Loaded {len(_oblast_geojson.get('features', []))} Ukraine oblast boundaries")
+    return _oblast_geojson
+
+
+def _find_oblast_geometry(location_title: str):
+    """Find the polygon geometry for an oblast by matching Ukrainian name."""
+    oblasts = _load_oblasts()
+    features = oblasts.get("features", [])
+    for feat in features:
+        props = feat.get("properties", {})
+        name = props.get("name", "")
+        # Exact match on Ukrainian name (e.g. "Луганська область")
+        if name == location_title:
+            return feat.get("geometry"), props.get("name_en", "")
+    # Fuzzy: try partial match (alert may say "Київська область" but GeoJSON says "Київ")
+    for feat in features:
+        props = feat.get("properties", {})
+        name = props.get("name", "")
+        if location_title in name or name in location_title:
+            return feat.get("geometry"), props.get("name_en", "")
+    return None, ""
+
+
+# ─── Fetcher ─────────────────────────────────────────────────────────────────
+
+@with_retry(max_retries=1, base_delay=2)
+def fetch_ukraine_air_raid_alerts():
+    """Fetch active Ukraine air raid alerts from alerts.in.ua."""
+    from services.fetchers._store import is_any_active
+
+    if not is_any_active("ukraine_alerts"):
+        return
+
+    token = os.environ.get("ALERTS_IN_UA_TOKEN", "")
+    if not token:
+        logger.debug("ALERTS_IN_UA_TOKEN not set, skipping Ukraine air raid alerts")
+        return
+
+    alerts_out = []
+    try:
+        url = f"https://api.alerts.in.ua/v1/alerts/active.json?token={token}"
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/json",
+        }
+        response = fetch_with_curl(url, timeout=10, headers=headers)
+
+        if response.status_code == 200:
+            data = response.json()
+            raw_alerts = data.get("alerts", [])
+
+            for alert in raw_alerts:
+                loc_type = alert.get("location_type", "")
+                # Only render oblast-level alerts (not raion/city/hromada)
+                if loc_type != "oblast":
+                    continue
+
+                location_title = alert.get("location_title", "")
+                alert_type = alert.get("alert_type", "air_raid")
+                geometry, name_en = _find_oblast_geometry(location_title)
+
+                if not geometry:
+                    logger.debug(f"No geometry for oblast: {location_title}")
+                    continue
+
+                alerts_out.append({
+                    "id": alert.get("id", 0),
+                    "alert_type": alert_type,
+                    "location_title": location_title,
+                    "location_uid": alert.get("location_uid", ""),
+                    "name_en": name_en,
+                    "started_at": alert.get("started_at", ""),
+                    "color": ALERT_COLORS.get(alert_type, "#ef4444"),
+                    "geometry": geometry,
+                })
+
+            logger.info(f"Ukraine alerts: {len(alerts_out)} active oblast-level alerts "
+                        f"(from {len(raw_alerts)} total)")
+        elif response.status_code == 401:
+            logger.warning("alerts.in.ua returned 401 — check ALERTS_IN_UA_TOKEN")
+        elif response.status_code == 429:
+            logger.warning("alerts.in.ua rate-limited (429)")
+        else:
+            logger.warning(f"alerts.in.ua returned HTTP {response.status_code}")
+
+    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
+        logger.error(f"Error fetching Ukraine alerts: {e}")
+
+    with _data_lock:
+        latest_data["ukraine_alerts"] = alerts_out
+    if alerts_out:
+        _mark_fresh("ukraine_alerts")

backend/services/fetchers/unusual_whales.py

@@ -0,0 +1,76 @@
+"""Finnhub scheduled fetcher — congress trades, insider transactions, defense quotes.
+
+Runs on a 15-minute schedule and stores results in latest_data["unusual_whales"].
+Also updates latest_data["stocks"] with Finnhub quotes (replaces yfinance for defense tickers).
+Falls back gracefully if no API key is configured.
+"""
+
+import logging
+from services.fetchers._store import latest_data, _data_lock, _mark_fresh
+from services.fetchers.retry import with_retry
+
+logger = logging.getLogger(__name__)
+
+
+@with_retry(max_retries=1, base_delay=2)
+def fetch_unusual_whales():
+    """Fetch congress trades, insider txns, and defense quotes from Finnhub."""
+    import os
+
+    if not os.environ.get("FINNHUB_API_KEY", "").strip():
+        logger.debug("FINNHUB_API_KEY not set — skipping scheduled fetch.")
+        return
+
+    from services.unusual_whales_connector import (
+        fetch_congress_trades,
+        fetch_insider_transactions,
+        fetch_defense_quotes,
+        FinnhubConnectorError,
+    )
+
+    result: dict = {}
+
+    # Defense stock quotes (also populates latest_data["stocks"])
+    try:
+        quotes = fetch_defense_quotes()
+        if quotes:
+            result["quotes"] = quotes
+            # Mirror into stocks for backward compat with existing MarketsPanel fallback
+            with _data_lock:
+                latest_data["stocks"] = quotes
+            _mark_fresh("stocks")
+    except FinnhubConnectorError as e:
+        logger.warning(f"Finnhub quotes fetch failed: {e.detail}")
+    except Exception as e:
+        logger.warning(f"Finnhub quotes fetch error: {e}")
+
+    # Congress trades
+    try:
+        congress = fetch_congress_trades()
+        result["congress_trades"] = congress.get("trades", [])
+    except FinnhubConnectorError as e:
+        logger.warning(f"Finnhub congress trades fetch failed: {e.detail}")
+    except Exception as e:
+        logger.warning(f"Finnhub congress trades fetch error: {e}")
+
+    # Insider transactions
+    try:
+        insiders = fetch_insider_transactions()
+        result["insider_transactions"] = insiders.get("transactions", [])
+    except FinnhubConnectorError as e:
+        logger.warning(f"Finnhub insider fetch failed: {e.detail}")
+    except Exception as e:
+        logger.warning(f"Finnhub insider fetch error: {e}")
+
+    if not result:
+        logger.warning("Finnhub update produced no data; keeping previous cache.")
+        return
+
+    with _data_lock:
+        latest_data["unusual_whales"] = result
+    _mark_fresh("unusual_whales")
+    logger.info(
+        f"Finnhub updated: {len(result.get('congress_trades', []))} congress, "
+        f"{len(result.get('insider_transactions', []))} insider, "
+        f"{len(result.get('quotes', {}))} quotes"
+    )

backend/services/fetchers/yacht_alert.py

@@ -0,0 +1,64 @@
+"""Yacht-Alert DB — load and enrich AIS vessels with tracked yacht metadata."""
+
+import os
+import json
+import logging
+
+logger = logging.getLogger("services.data_fetcher")
+
+# Category -> color mapping
+_CATEGORY_COLOR: dict[str, str] = {
+    "Tech Billionaire": "#FF69B4",
+    "Celebrity / Mogul": "#FF69B4",
+    "Oligarch Watch": "#FF2020",
+}
+
+
+def _category_to_color(cat: str) -> str:
+    """Map category to display color. Defaults to hot pink."""
+    return _CATEGORY_COLOR.get(cat, "#FF69B4")
+
+
+_YACHT_ALERT_DB: dict = {}
+
+
+def _load_yacht_alert_db():
+    """Load yacht_alert_db.json into memory at import time."""
+    global _YACHT_ALERT_DB
+    json_path = os.path.join(
+        os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
+        "data",
+        "yacht_alert_db.json",
+    )
+    if not os.path.exists(json_path):
+        logger.warning(f"Yacht-Alert DB not found at {json_path}")
+        return
+    try:
+        with open(json_path, "r", encoding="utf-8") as fh:
+            raw = json.load(fh)
+        for mmsi_str, info in raw.items():
+            info["color"] = _category_to_color(info.get("category", ""))
+            _YACHT_ALERT_DB[mmsi_str] = info
+        logger.info(f"Yacht-Alert DB loaded: {len(_YACHT_ALERT_DB)} vessels")
+    except (IOError, OSError, json.JSONDecodeError, ValueError, KeyError) as e:
+        logger.error(f"Failed to load Yacht-Alert DB: {e}")
+
+
+_load_yacht_alert_db()
+
+
+def enrich_with_yacht_alert(ship: dict) -> dict:
+    """If ship's MMSI is in the Yacht-Alert DB, attach owner/alert metadata."""
+    mmsi = str(ship.get("mmsi", "")).strip()
+    if mmsi and mmsi in _YACHT_ALERT_DB:
+        info = _YACHT_ALERT_DB[mmsi]
+        ship["yacht_alert"] = True
+        ship["yacht_owner"] = info["owner"]
+        ship["yacht_name"] = info["name"]
+        ship["yacht_category"] = info["category"]
+        ship["yacht_color"] = info["color"]
+        ship["yacht_builder"] = info.get("builder", "")
+        ship["yacht_length"] = info.get("length_m", 0)
+        ship["yacht_year"] = info.get("year", 0)
+        ship["yacht_link"] = info.get("link", "")
+    return ship

backend/services/geocode.py

@@ -0,0 +1,255 @@
+"""Geocoding proxy for Nominatim with caching and proper headers."""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+import threading
+from typing import Any, Dict, List
+from pathlib import Path
+from urllib.parse import urlencode
+
+from services.network_utils import fetch_with_curl
+from services.fetchers.geo import cached_airports
+
+_CACHE_TTL_S = 900
+_CACHE_MAX = 1000
+_cache: Dict[str, Dict[str, Any]] = {}
+_cache_lock = threading.Lock()
+_local_search_cache: List[Dict[str, Any]] | None = None
+_local_search_lock = threading.Lock()
+
+_USER_AGENT = os.environ.get(
+    "NOMINATIM_USER_AGENT", "ShadowBroker/1.0 (https://github.com/BigBodyCobain/Shadowbroker)"
+)
+
+
+def _get_cache(key: str):
+    now = time.time()
+    with _cache_lock:
+        entry = _cache.get(key)
+        if not entry:
+            return None
+        if now - entry["ts"] > _CACHE_TTL_S:
+            _cache.pop(key, None)
+            return None
+        return entry["value"]
+
+
+def _set_cache(key: str, value):
+    with _cache_lock:
+        if len(_cache) >= _CACHE_MAX:
+            # Simple eviction: drop ~10% oldest keys
+            keys = list(_cache.keys())[: max(1, _CACHE_MAX // 10)]
+            for k in keys:
+                _cache.pop(k, None)
+        _cache[key] = {"ts": time.time(), "value": value}
+
+
+def _load_local_search_cache() -> List[Dict[str, Any]]:
+    global _local_search_cache
+    with _local_search_lock:
+        if _local_search_cache is not None:
+            return _local_search_cache
+
+        results: List[Dict[str, Any]] = []
+        cache_path = Path(__file__).resolve().parents[1] / "data" / "geocode_cache.json"
+        try:
+            if cache_path.exists():
+                raw = json.loads(cache_path.read_text(encoding="utf-8"))
+                if isinstance(raw, dict):
+                    for label, coords in raw.items():
+                        if (
+                            isinstance(label, str)
+                            and isinstance(coords, list)
+                            and len(coords) == 2
+                            and all(isinstance(v, (int, float)) for v in coords)
+                        ):
+                            results.append(
+                                {
+                                    "label": label,
+                                    "lat": float(coords[0]),
+                                    "lng": float(coords[1]),
+                                }
+                            )
+        except Exception:
+            results = []
+
+        _local_search_cache = results
+        return _local_search_cache
+
+
+def _search_local_fallback(query: str, limit: int) -> List[Dict[str, Any]]:
+    q = query.strip().lower()
+    if not q:
+        return []
+
+    matches: List[Dict[str, Any]] = []
+    seen: set[tuple[float, float, str]] = set()
+
+    for item in cached_airports:
+        haystacks = [
+            str(item.get("name", "")).lower(),
+            str(item.get("iata", "")).lower(),
+            str(item.get("id", "")).lower(),
+        ]
+        if any(q in h for h in haystacks):
+            label = f'{item.get("name", "Airport")} ({item.get("iata", "")})'
+            key = (float(item["lat"]), float(item["lng"]), label)
+            if key not in seen:
+                seen.add(key)
+                matches.append(
+                    {
+                        "label": label,
+                        "lat": float(item["lat"]),
+                        "lng": float(item["lng"]),
+                    }
+                )
+                if len(matches) >= limit:
+                    return matches
+
+    for item in _load_local_search_cache():
+        label = str(item.get("label", ""))
+        if q in label.lower():
+            key = (float(item["lat"]), float(item["lng"]), label)
+            if key not in seen:
+                seen.add(key)
+                matches.append(item)
+                if len(matches) >= limit:
+                    break
+
+    return matches
+
+
+def _reverse_geocode_offline(lat: float, lng: float) -> Dict[str, Any]:
+    try:
+        import reverse_geocoder as rg
+
+        hit = rg.search((lat, lng), mode=1)[0]
+        city = hit.get("name") or ""
+        state = hit.get("admin1") or ""
+        country = hit.get("cc") or ""
+        parts = [city, state, country]
+        label = ", ".join([p for p in parts if p]) or "Unknown"
+        return {"label": label}
+    except Exception:
+        return {"label": "Unknown"}
+
+
+def search_geocode(query: str, limit: int = 5, local_only: bool = False) -> List[Dict[str, Any]]:
+    q = query.strip()
+    if not q:
+        return []
+    limit = max(1, min(int(limit or 5), 10))
+    key = f"search:{q.lower()}:{limit}:{int(local_only)}"
+    cached = _get_cache(key)
+    if cached is not None:
+        return cached
+    if local_only:
+        results = _search_local_fallback(q, limit)
+        _set_cache(key, results)
+        return results
+
+    params = urlencode({"q": q, "format": "json", "limit": str(limit)})
+    url = f"https://nominatim.openstreetmap.org/search?{params}"
+    try:
+        res = fetch_with_curl(
+            url,
+            headers={
+                "User-Agent": _USER_AGENT,
+                "Accept-Language": "en",
+            },
+            timeout=6,
+        )
+    except Exception:
+        results = _search_local_fallback(q, limit)
+        _set_cache(key, results)
+        return results
+
+    results: List[Dict[str, Any]] = []
+    if res and res.status_code == 200:
+        try:
+            data = res.json() or []
+            for item in data:
+                try:
+                    results.append(
+                        {
+                            "label": item.get("display_name"),
+                            "lat": float(item.get("lat")),
+                            "lng": float(item.get("lon")),
+                        }
+                    )
+                except (TypeError, ValueError):
+                    continue
+        except Exception:
+            results = []
+    if not results:
+        results = _search_local_fallback(q, limit)
+
+    _set_cache(key, results)
+    return results
+
+
+def reverse_geocode(lat: float, lng: float, local_only: bool = False) -> Dict[str, Any]:
+    key = f"reverse:{lat:.4f},{lng:.4f}:{int(local_only)}"
+    cached = _get_cache(key)
+    if cached is not None:
+        return cached
+    if local_only:
+        payload = _reverse_geocode_offline(lat, lng)
+        _set_cache(key, payload)
+        return payload
+
+    params = urlencode(
+        {
+            "lat": f"{lat}",
+            "lon": f"{lng}",
+            "format": "json",
+            "zoom": "10",
+            "addressdetails": "1",
+        }
+    )
+    url = f"https://nominatim.openstreetmap.org/reverse?{params}"
+    try:
+        res = fetch_with_curl(
+            url,
+            headers={
+                "User-Agent": _USER_AGENT,
+                "Accept-Language": "en",
+            },
+            timeout=6,
+        )
+    except Exception:
+        payload = _reverse_geocode_offline(lat, lng)
+        _set_cache(key, payload)
+        return payload
+
+    label = "Unknown"
+    if res and res.status_code == 200:
+        try:
+            data = res.json() or {}
+            addr = data.get("address") or {}
+            city = (
+                addr.get("city")
+                or addr.get("town")
+                or addr.get("village")
+                or addr.get("county")
+                or ""
+            )
+            state = addr.get("state") or addr.get("region") or ""
+            country = addr.get("country") or ""
+            parts = [city, state, country]
+            label = ", ".join([p for p in parts if p]) or (
+                data.get("display_name", "") or "Unknown"
+            )
+        except Exception:
+            label = "Unknown"
+    if label == "Unknown":
+        payload = _reverse_geocode_offline(lat, lng)
+        _set_cache(key, payload)
+        return payload
+
+    payload = {"label": label}
+    _set_cache(key, payload)
+    return payload

backend/services/geopolitics.py

@@ -1,7 +1,11 @@
 import requests
 import logging
+import zipfile
+import socket
+import ipaddress
 from cachetools import cached, TTLCache
 from datetime import datetime
+from urllib.parse import urljoin, urlparse
 from services.network_utils import fetch_with_curl
 
 logger = logging.getLogger(__name__)
@@ -9,6 +13,7 @@ logger = logging.getLogger(__name__)
 # Cache Frontline data for 30 minutes, it doesn't move that fast
 frontline_cache = TTLCache(maxsize=1, ttl=1800)
 
+
 @cached(frontline_cache)
 def fetch_ukraine_frontlines():
     """
@@ -17,27 +22,34 @@ def fetch_ukraine_frontlines():
     """
     try:
         logger.info("Fetching DeepStateMap from GitHub mirror...")
-        
+
         # First, query the repo tree to find the latest file name
-        tree_url = "https://api.github.com/repos/cyterat/deepstate-map-data/git/trees/main?recursive=1"
+        tree_url = (
+            "https://api.github.com/repos/cyterat/deepstate-map-data/git/trees/main?recursive=1"
+        )
         res_tree = requests.get(tree_url, timeout=10)
-        
+
         if res_tree.status_code == 200:
             tree_data = res_tree.json().get("tree", [])
             # Filter for geojson files in data folder
-            geo_files = [item["path"] for item in tree_data if item["path"].startswith("data/deepstatemap_data_") and item["path"].endswith(".geojson")]
-            
+            geo_files = [
+                item["path"]
+                for item in tree_data
+                if item["path"].startswith("data/deepstatemap_data_")
+                and item["path"].endswith(".geojson")
+            ]
+
             if geo_files:
                 # Get the alphabetically latest file (since it's named with YYYYMMDD)
                 latest_file = sorted(geo_files)[-1]
-                
+
                 raw_url = f"https://raw.githubusercontent.com/cyterat/deepstate-map-data/main/{latest_file}"
                 logger.info(f"Downloading latest DeepStateMap: {raw_url}")
-                
+
                 res_geo = requests.get(raw_url, timeout=20)
                 if res_geo.status_code == 200:
                     data = res_geo.json()
-                    
+
                     # The Cyterat GitHub mirror strips all properties and just provides a raw array of Feature polygons.
                     # Based on DeepStateMap's frontend mapping, the array index corresponds to the zone type:
                     # 0: Russian-occupied areas
@@ -48,110 +60,339 @@ def fetch_ukraine_frontlines():
                         0: "Russian-occupied areas",
                         1: "Russian advance",
                         2: "Liberated area",
-                        3: "Russian-occupied areas", # Crimea / LPR / DPR
-                        4: "Directions of UA attacks"
+                        3: "Russian-occupied areas",  # Crimea / LPR / DPR
+                        4: "Directions of UA attacks",
                     }
-                    
+
                     if "features" in data:
                         for idx, feature in enumerate(data["features"]):
                             if "properties" not in feature or feature["properties"] is None:
                                 feature["properties"] = {}
-                            
-                            feature["properties"]["name"] = name_map.get(idx, "Russian-occupied areas")
+
+                            feature["properties"]["name"] = name_map.get(
+                                idx, "Russian-occupied areas"
+                            )
                             feature["properties"]["zone_id"] = idx
-                            
+
                     return data
                 else:
-                    logger.error(f"Failed to fetch parsed Github Raw GeoJSON: {res_geo.status_code}")
+                    logger.error(
+                        f"Failed to fetch parsed Github Raw GeoJSON: {res_geo.status_code}"
+                    )
         else:
             logger.error(f"Failed to fetch Github Tree for Deepstatemap: {res_tree.status_code}")
-    except Exception as e:
+    except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
         logger.error(f"Error fetching DeepStateMap: {e}")
     return None
 
+
 # Cache GDELT data for 6 hours - heavy aggregation, data doesn't change rapidly
 gdelt_cache = TTLCache(maxsize=1, ttl=21600)
 
+
 def _extract_domain(url):
     """Extract a clean source name from a URL, e.g. 'nytimes.com' from 'https://www.nytimes.com/...'"""
     try:
         from urllib.parse import urlparse
-        host = urlparse(url).hostname or ''
+
+        host = urlparse(url).hostname or ""
         # Strip www. prefix
-        if host.startswith('www.'):
+        if host.startswith("www."):
             host = host[4:]
         return host
-    except Exception:
+    except (ValueError, AttributeError, KeyError):  # non-critical
         return url[:40]
 
+
 def _url_to_headline(url):
     """Extract a human-readable headline from a URL path.
-    e.g. 'https://nytimes.com/2026/03/us-strikes-iran-nuclear-sites.html' -> 'Us Strikes Iran Nuclear Sites (nytimes.com)'
+    e.g. 'https://nytimes.com/2026/03/us-strikes-iran-nuclear-sites.html' -> 'Us Strikes Iran Nuclear Sites'
+    Falls back to domain name if the URL slug is gibberish (hex IDs, UUIDs, etc.).
     """
+    import re
+
     try:
         from urllib.parse import urlparse, unquote
+
         parsed = urlparse(url)
-        domain = parsed.hostname or ''
-        if domain.startswith('www.'):
+        domain = parsed.hostname or ""
+        if domain.startswith("www."):
             domain = domain[4:]
 
         # Get last meaningful path segment
-        path = unquote(parsed.path).strip('/')
+        path = unquote(parsed.path).strip("/")
         if not path:
             return domain
 
-        # Take the last path segment (usually the slug)
-        slug = path.split('/')[-1]
-        # Remove file extensions
-        for ext in ['.html', '.htm', '.php', '.asp', '.aspx', '.shtml']:
-            if slug.lower().endswith(ext):
-                slug = slug[:-len(ext)]
-        # If slug is purely numeric or a short ID, try the second-to-last segment
-        import re
-        if re.match(r'^[a-z]?\d{5,}$', slug, re.IGNORECASE):
-            segments = path.split('/')
-            if len(segments) >= 2:
-                slug = segments[-2]
-                for ext in ['.html', '.htm', '.php']:
-                    if slug.lower().endswith(ext):
-                        slug = slug[:-len(ext)]
-        # Remove common ID patterns at start/end
-        slug = re.sub(r'^[\d]+-', '', slug)  # leading numbers like "13847569-"
-        slug = re.sub(r'-[\da-f]{6,}$', '', slug)  # trailing hex IDs
-        slug = re.sub(r'[-_]c-\d+$', '', slug)  # trailing "-c-21803431"
-        slug = re.sub(r'^p=\d+$', '', slug)  # WordPress ?p=1234
-        # Convert slug separators to spaces
-        slug = slug.replace('-', ' ').replace('_', ' ')
-        # Clean up multiple spaces
-        slug = re.sub(r'\s+', ' ', slug).strip()
+        # Try the last path segment first, then walk backwards
+        segments = [s for s in path.split("/") if s]
+        slug = ""
+        for seg in reversed(segments):
+            # Remove file extensions
+            for ext in [".html", ".htm", ".php", ".asp", ".aspx", ".shtml"]:
+                if seg.lower().endswith(ext):
+                    seg = seg[: -len(ext)]
+            # Skip segments that are clearly not headlines
+            if _is_gibberish(seg):
+                continue
+            slug = seg
+            break
 
-        # If slug is still just a number or too short, fall back to domain
-        if len(slug) < 5 or re.match(r'^\d+$', slug):
+        if not slug:
+            return domain
+
+        # Remove common ID patterns at start/end
+        slug = re.sub(r"^[\d]+-", "", slug)  # leading "13847569-"
+        slug = re.sub(r"-[\da-f]{6,}$", "", slug)  # trailing hex IDs
+        slug = re.sub(r"[-_]c-\d+$", "", slug)  # trailing "-c-21803431"
+        slug = re.sub(r"^p=\d+$", "", slug)  # WordPress ?p=1234
+        # Convert slug separators to spaces
+        slug = slug.replace("-", " ").replace("_", " ")
+        slug = re.sub(r"\s+", " ", slug).strip()
+
+        # Final gibberish check after cleanup
+        if len(slug) < 8 or _is_gibberish(slug.replace(" ", "-")):
             return domain
 
         # Title case and truncate
         headline = slug.title()
-        if len(headline) > 80:
-            headline = headline[:77] + '...'
-        return f"{headline} ({domain})"
-    except Exception:
+        if len(headline) > 90:
+            headline = headline[:87] + "..."
+        return headline
+    except (ValueError, AttributeError, KeyError):  # non-critical
         return url[:60]
 
+
+def _is_gibberish(text):
+    """Detect if a URL segment is gibberish (hex IDs, UUIDs, numeric IDs, etc.)
+    rather than a real human-readable slug like 'us-strikes-iran'."""
+    import re
+
+    t = text.strip()
+    if not t:
+        return True
+    # Pure numbers
+    if re.match(r"^\d+$", t):
+        return True
+    # UUID pattern (with or without dashes)
+    if re.match(
+        r"^[0-9a-f]{8}[_-]?[0-9a-f]{4}[_-]?[0-9a-f]{4}[_-]?[0-9a-f]{4}[_-]?[0-9a-f]{12}$", t, re.I
+    ):
+        return True
+    # Hex-heavy string: more than 40% hex digits among alphanumeric chars
+    alnum = re.sub(r"[^a-zA-Z0-9]", "", t)
+    if alnum:
+        hex_chars = sum(1 for c in alnum if c in "0123456789abcdefABCDEF")
+        if hex_chars / len(alnum) > 0.4 and len(alnum) > 6:
+            return True
+    # Mostly digits with a few alpha (like "article8efa6c53")
+    digits = sum(1 for c in alnum if c.isdigit())
+    if alnum and digits / len(alnum) > 0.5:
+        return True
+    # Too short to be a headline slug
+    if len(t) < 5:
+        return True
+    # Query-param style segments
+    if "=" in t:
+        return True
+    return False
+
+
+# Persistent cache for article titles — survives across GDELT cache refreshes
+# Bounded to 5000 entries with 24hr TTL to prevent unbounded memory growth
+_article_title_cache = TTLCache(maxsize=5000, ttl=86400)
+_article_url_safety_cache = TTLCache(maxsize=5000, ttl=3600)
+_TITLE_FETCH_MAX_REDIRECTS = 3
+_TITLE_FETCH_READ_BYTES = 32768
+_ALLOWED_ARTICLE_PORTS = {80, 443, 8080, 8443}
+
+
+def _hostname_resolves_public(hostname: str, port: int) -> bool:
+    try:
+        infos = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM)
+    except (socket.gaierror, OSError):
+        return False
+
+    addresses = set()
+    for info in infos:
+        sockaddr = info[4] if len(info) > 4 else None
+        if not sockaddr:
+            continue
+        raw_addr = str(sockaddr[0] or "").split("%", 1)[0]
+        if not raw_addr:
+            continue
+        try:
+            addresses.add(ipaddress.ip_address(raw_addr))
+        except ValueError:
+            continue
+
+    return bool(addresses) and all(addr.is_global for addr in addresses)
+
+
+def _is_safe_public_article_url(url: str) -> tuple[bool, str]:
+    cached = _article_url_safety_cache.get(url)
+    if cached is not None:
+        return cached
+
+    try:
+        parsed = urlparse(str(url or "").strip())
+    except ValueError:
+        result = (False, "parse_error")
+        _article_url_safety_cache[url] = result
+        return result
+
+    scheme = str(parsed.scheme or "").lower()
+    host = str(parsed.hostname or "").strip().lower()
+    if scheme not in {"http", "https"}:
+        result = (False, "scheme")
+    elif not host:
+        result = (False, "host")
+    elif parsed.username or parsed.password:
+        result = (False, "userinfo")
+    elif host in {"localhost", "localhost.localdomain"}:
+        result = (False, "localhost")
+    else:
+        port = parsed.port or (443 if scheme == "https" else 80)
+        if port not in _ALLOWED_ARTICLE_PORTS:
+            result = (False, "port")
+        else:
+            try:
+                target_ip = ipaddress.ip_address(host.split("%", 1)[0])
+            except ValueError:
+                target_ip = None
+            if target_ip is not None:
+                result = (True, "") if target_ip.is_global else (False, "private_ip")
+            else:
+                result = (True, "") if _hostname_resolves_public(host, port) else (False, "private_dns")
+
+    _article_url_safety_cache[url] = result
+    return result
+
+
+def _fetch_article_title(url):
+    """Fetch the real headline from an article's HTML <title> or og:title tag.
+    Returns the title string, or None if it can't be fetched.
+    Uses a persistent cache to avoid refetching."""
+    if url in _article_title_cache:
+        return _article_title_cache[url]
+
+    import re
+
+    try:
+        current_url = str(url or "").strip()
+        chunk = ""
+        for _ in range(_TITLE_FETCH_MAX_REDIRECTS + 1):
+            allowed, _reason = _is_safe_public_article_url(current_url)
+            if not allowed:
+                _article_title_cache[url] = None
+                return None
+
+            resp = requests.get(
+                current_url,
+                timeout=4,
+                headers={"User-Agent": "Mozilla/5.0 (compatible; OSINT Dashboard/1.0)"},
+                stream=True,
+                allow_redirects=False,
+            )
+            try:
+                location = str(resp.headers.get("Location") or "").strip()
+                if 300 <= resp.status_code < 400 and location:
+                    current_url = urljoin(current_url, location)
+                    continue
+                if resp.status_code != 200:
+                    _article_title_cache[url] = None
+                    return None
+                chunk = resp.raw.read(_TITLE_FETCH_READ_BYTES).decode("utf-8", errors="replace")
+                break
+            finally:
+                resp.close()
+        else:
+            _article_title_cache[url] = None
+            return None
+
+        title = None
+
+        # Try og:title first (usually the cleanest)
+        og_match = re.search(
+            r'<meta[^>]+property=["\']og:title["\'][^>]+content=["\']([^"\'>]+)["\']', chunk, re.I
+        )
+        if not og_match:
+            og_match = re.search(
+                r'<meta[^>]+content=["\']([^"\'>]+)["\'][^>]+property=["\']og:title["\']',
+                chunk,
+                re.I,
+            )
+        if og_match:
+            title = og_match.group(1).strip()
+
+        # Fall back to <title> tag
+        if not title:
+            title_match = re.search(r"<title[^>]*>([^<]+)</title>", chunk, re.I)
+            if title_match:
+                title = title_match.group(1).strip()
+
+        if title:
+            # Clean up HTML entities
+            import html as html_mod
+
+            title = html_mod.unescape(title)
+            # Remove site name suffixes like " | CNN" or " - BBC News"
+            title = re.sub(r"\s*[|\-–—]\s*[^|\-–—]{2,30}$", "", title).strip()
+            # Truncate very long titles
+            if len(title) > 120:
+                title = title[:117] + "..."
+            if len(title) > 10:
+                _article_title_cache[url] = title
+                return title
+
+        _article_title_cache[url] = None
+        return None
+    except (
+        requests.RequestException,
+        ConnectionError,
+        TimeoutError,
+        ValueError,
+        AttributeError,
+    ):  # non-critical
+        _article_title_cache[url] = None
+        return None
+
+
+def _batch_fetch_titles(urls):
+    """Fetch real article titles for a list of URLs in parallel.
+    Returns a dict of url -> title (or None if fetch failed)."""
+    from concurrent.futures import ThreadPoolExecutor
+
+    results = {}
+    with ThreadPoolExecutor(max_workers=16) as executor:
+        futures = {executor.submit(_fetch_article_title, u): u for u in urls}
+        for future in futures:
+            url = futures[future]
+            try:
+                results[url] = future.result()
+            except Exception:  # non-critical: optional title enrichment
+                results[url] = None
+    return results
+
+
 def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_index):
     """Parse a single GDELT export ZIP and append conflict features.
     loc_index maps loc_key -> index in features list for fast duplicate merging.
     """
     import csv, io, zipfile
+
     try:
         zf = zipfile.ZipFile(io.BytesIO(zip_bytes))
         csv_name = zf.namelist()[0]
         with zf.open(csv_name) as cf:
-            reader = csv.reader(io.TextIOWrapper(cf, encoding='utf-8', errors='replace'), delimiter='\t')
+            reader = csv.reader(
+                io.TextIOWrapper(cf, encoding="utf-8", errors="replace"), delimiter="\t"
+            )
             for row in reader:
                 try:
                     if len(row) < 61:
                         continue
-                    event_code = row[26][:2] if len(row[26]) >= 2 else ''
+                    event_code = row[26][:2] if len(row[26]) >= 2 else ""
                     if event_code not in conflict_codes:
                         continue
                     lat = float(row[56]) if row[56] else None
@@ -159,10 +400,10 @@ def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_
                     if lat is None or lng is None or (lat == 0 and lng == 0):
                         continue
 
-                    source_url = row[60].strip() if len(row) > 60 else ''
-                    location = row[52].strip() if len(row) > 52 else 'Unknown'
-                    actor1 = row[6].strip() if len(row) > 6 else ''
-                    actor2 = row[16].strip() if len(row) > 16 else ''
+                    source_url = row[60].strip() if len(row) > 60 else ""
+                    location = row[52].strip() if len(row) > 52 else "Unknown"
+                    actor1 = row[6].strip() if len(row) > 6 else ""
+                    actor2 = row[16].strip() if len(row) > 16 else ""
 
                     loc_key = f"{round(lat, 1)}_{round(lng, 1)}"
                     if loc_key in seen_locs:
@@ -182,41 +423,111 @@ def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_
                         continue
                     seen_locs.add(loc_key)
 
-                    name = location or (f"{actor1} vs {actor2}" if actor1 and actor2 else actor1) or "Unknown Incident"
-                    domain = _extract_domain(source_url) if source_url else ''
+                    name = (
+                        location
+                        or (f"{actor1} vs {actor2}" if actor1 and actor2 else actor1)
+                        or "Unknown Incident"
+                    )
+                    domain = _extract_domain(source_url) if source_url else ""
                     loc_index[loc_key] = len(features)
-                    features.append({
-                        "type": "Feature",
-                        "properties": {
-                            "name": name,
-                            "count": 1,
-                            "_urls": [source_url] if source_url else [],
-                            "_domains": {domain} if domain else set(),
-                        },
-                        "geometry": {"type": "Point", "coordinates": [lng, lat]},
-                        "_loc_key": loc_key
-                    })
+                    features.append(
+                        {
+                            "type": "Feature",
+                            "properties": {
+                                "name": name,
+                                "count": 1,
+                                "_urls": [source_url] if source_url else [],
+                                "_domains": {domain} if domain else set(),
+                            },
+                            "geometry": {"type": "Point", "coordinates": [lng, lat]},
+                            "_loc_key": loc_key,
+                        }
+                    )
                 except (ValueError, IndexError):
                     continue
-    except Exception as e:
+    except (IOError, OSError, ValueError, KeyError, zipfile.BadZipFile) as e:
         logger.warning(f"Failed to parse GDELT export zip: {e}")
 
+
 def _download_gdelt_export(url):
     """Download a single GDELT export file, return bytes or None."""
     try:
         res = fetch_with_curl(url, timeout=15)
         if res.status_code == 200:
             return res.content
-    except Exception:
+    except (ConnectionError, TimeoutError, OSError):  # non-critical
         pass
     return None
 
-@cached(gdelt_cache)
+
+def _build_feature_html(features, fetched_titles=None):
+    """Build URL + headline arrays for frontend rendering.
+    Uses fetched_titles (real article titles) when available, falls back to URL slug parsing."""
+    import html as html_mod
+
+    for f in features:
+        urls = f["properties"].pop("_urls", [])
+        f["properties"].pop("_domains", None)
+        headlines = []
+        for u in urls:
+            real_title = fetched_titles.get(u) if fetched_titles else None
+            headlines.append(real_title if real_title else _url_to_headline(u))
+        f["properties"]["_urls_list"] = urls
+        f["properties"]["_headlines_list"] = headlines
+        if urls:
+            links = []
+            for u, h in zip(urls, headlines):
+                safe_url = u if u.startswith(("http://", "https://")) else "about:blank"
+                safe_h = html_mod.escape(h)
+                links.append(
+                    f'<div style="margin-bottom:6px;"><a href="{safe_url}" target="_blank" rel="noopener noreferrer">{safe_h}</a></div>'
+                )
+            f["properties"]["html"] = "".join(links)
+        else:
+            f["properties"]["html"] = html_mod.escape(f["properties"]["name"])
+        f.pop("_loc_key", None)
+
+
+def _enrich_gdelt_titles_background(features, all_article_urls):
+    """Background thread: fetch real article titles then update features in-place."""
+    import html as html_mod
+
+    try:
+        logger.info(f"[BG] Fetching real article titles for {len(all_article_urls)} URLs...")
+        fetched_titles = _batch_fetch_titles(all_article_urls)
+        fetched_count = sum(1 for v in fetched_titles.values() if v)
+        logger.info(f"[BG] Resolved {fetched_count}/{len(all_article_urls)} article titles")
+
+        # Update features in-place with real titles
+        for f in features:
+            urls = f["properties"].get("_urls_list", [])
+            if not urls:
+                continue
+            headlines = []
+            for u in urls:
+                real_title = fetched_titles.get(u)
+                headlines.append(real_title if real_title else _url_to_headline(u))
+            f["properties"]["_headlines_list"] = headlines
+            links = []
+            for u, h in zip(urls, headlines):
+                safe_url = u if u.startswith(("http://", "https://")) else "about:blank"
+                safe_h = html_mod.escape(h)
+                links.append(
+                    f'<div style="margin-bottom:6px;"><a href="{safe_url}" target="_blank" rel="noopener noreferrer">{safe_h}</a></div>'
+                )
+            f["properties"]["html"] = "".join(links)
+        logger.info(f"[BG] GDELT title enrichment complete")
+    except Exception as e:
+        logger.error(f"[BG] GDELT title enrichment failed: {e}")
+
+
 def fetch_global_military_incidents():
     """
     Fetches global military/conflict incidents from GDELT Events Export files.
     Aggregates the last ~8 hours of 15-minute exports to build ~1000 incidents.
+    Returns immediately with URL-slug headlines; enriches with real titles in background.
     """
+    import threading
     from datetime import timedelta
     from concurrent.futures import ThreadPoolExecutor
 
@@ -224,16 +535,18 @@ def fetch_global_military_incidents():
         logger.info("Fetching GDELT events via export CDN (multi-file)...")
 
         # Get the latest export URL to determine current timestamp
-        index_res = fetch_with_curl("http://data.gdeltproject.org/gdeltv2/lastupdate.txt", timeout=10)
+        index_res = fetch_with_curl(
+            "http://data.gdeltproject.org/gdeltv2/lastupdate.txt", timeout=10
+        )
         if index_res.status_code != 200:
             logger.error(f"GDELT lastupdate failed: {index_res.status_code}")
             return []
 
         # Extract latest export URL and its timestamp
         latest_url = None
-        for line in index_res.text.strip().split('\n'):
+        for line in index_res.text.strip().split("\n"):
             parts = line.strip().split()
-            if len(parts) >= 3 and parts[2].endswith('.export.CSV.zip'):
+            if len(parts) >= 3 and parts[2].endswith(".export.CSV.zip"):
                 latest_url = parts[2]
                 break
 
@@ -243,19 +556,20 @@ def fetch_global_military_incidents():
 
         # Extract timestamp from URL like: http://data.gdeltproject.org/gdeltv2/20260301120000.export.CSV.zip
         import re
-        ts_match = re.search(r'(\d{14})\.export\.CSV\.zip', latest_url)
+
+        ts_match = re.search(r"(\d{14})\.export\.CSV\.zip", latest_url)
         if not ts_match:
             logger.error("Could not parse GDELT export timestamp")
             return []
 
-        latest_ts = datetime.strptime(ts_match.group(1), '%Y%m%d%H%M%S')
+        latest_ts = datetime.strptime(ts_match.group(1), "%Y%m%d%H%M%S")
 
         # Generate URLs for the last 8 hours (32 files at 15-min intervals)
         NUM_FILES = 32
         urls = []
         for i in range(NUM_FILES):
             ts = latest_ts - timedelta(minutes=15 * i)
-            fname = ts.strftime('%Y%m%d%H%M%S') + '.export.CSV.zip'
+            fname = ts.strftime("%Y%m%d%H%M%S") + ".export.CSV.zip"
             url = f"http://data.gdeltproject.org/gdeltv2/{fname}"
             urls.append(url)
 
@@ -269,7 +583,7 @@ def fetch_global_military_incidents():
         logger.info(f"Downloaded {successful}/{len(urls)} GDELT exports")
 
         # Parse all downloaded files
-        CONFLICT_CODES = {'14', '17', '18', '19', '20'}
+        CONFLICT_CODES = {"14", "17", "18", "19", "20"}
         features = []
         seen_locs = set()
         loc_index = {}  # loc_key -> index in features
@@ -278,29 +592,38 @@ def fetch_global_military_incidents():
             if zip_bytes:
                 _parse_gdelt_export_zip(zip_bytes, CONFLICT_CODES, seen_locs, features, loc_index)
 
-        # Build URL + headline arrays for frontend rendering
+        # Collect all unique article URLs
+        all_article_urls = set()
         for f in features:
-            urls = f["properties"].pop("_urls", [])
-            f["properties"].pop("_domains", None)
-            headlines = [_url_to_headline(u) for u in urls]
-            f["properties"]["_urls_list"] = urls
-            f["properties"]["_headlines_list"] = headlines
-            import html
-            # Keep html as fallback
-            if urls:
-                links = []
-                for u, h in zip(urls, headlines):
-                    safe_url = u if u.startswith(('http://', 'https://')) else 'about:blank'
-                    safe_h = html.escape(h)
-                    links.append(f'<div style="margin-bottom:6px;"><a href="{safe_url}" target="_blank" rel="noopener noreferrer">{safe_h}</a></div>')
-                f["properties"]["html"] = ''.join(links)
-            else:
-                f["properties"]["html"] = html.escape(f["properties"]["name"])
-            f.pop("_loc_key", None)
+            for u in f["properties"].get("_urls", []):
+                if u:
+                    all_article_urls.add(u)
+
+        # Build HTML immediately with URL-slug headlines (instant, no network)
+        _build_feature_html(features)
+
+        logger.info(
+            f"GDELT parsed: {len(features)} conflict locations from {successful} files (titles enriching in background)"
+        )
+
+        # Kick off background thread to enrich with real article titles
+        # Features list is shared — background thread updates in-place
+        t = threading.Thread(
+            target=_enrich_gdelt_titles_background,
+            args=(features, all_article_urls),
+            daemon=True,
+        )
+        t.start()
 
-        logger.info(f"GDELT multi-file parsed: {len(features)} conflict locations from {successful} files")
         return features
 
-    except Exception as e:
+    except (
+        requests.RequestException,
+        ConnectionError,
+        TimeoutError,
+        ValueError,
+        KeyError,
+        OSError,
+    ) as e:
         logger.error(f"Error fetching GDELT data: {e}")
     return []

backend/services/kiwisdr_fetcher.py

@@ -6,6 +6,7 @@ Data is embedded as HTML comments inside each entry div.
 
 import re
 import logging
+import requests
 from cachetools import TTLCache, cached
 
 logger = logging.getLogger(__name__)
@@ -15,13 +16,13 @@ kiwisdr_cache = TTLCache(maxsize=1, ttl=600)  # 10-minute cache
 
 def _parse_comment(html: str, field: str) -> str:
     """Extract a field value from HTML comment like <!-- field=value -->"""
-    m = re.search(rf'<!--\s*{field}=(.*?)\s*-->', html)
+    m = re.search(rf"<!--\s*{field}=(.*?)\s*-->", html)
     return m.group(1).strip() if m else ""
 
 
 def _parse_gps(html: str):
     """Extract lat/lon from <!-- gps=(lat, lon) --> comment."""
-    m = re.search(r'<!--\s*gps=\(([^,]+),\s*([^)]+)\)\s*-->', html)
+    m = re.search(r"<!--\s*gps=\(([^,]+),\s*([^)]+)\)\s*-->", html)
     if m:
         try:
             return float(m.group(1)), float(m.group(2))
@@ -33,10 +34,10 @@ def _parse_gps(html: str):
 @cached(kiwisdr_cache)
 def fetch_kiwisdr_nodes() -> list[dict]:
     """Fetch and parse the KiwiSDR public receiver list."""
-    from services.network_utils import smart_request
+    from services.network_utils import fetch_with_curl
 
     try:
-        res = smart_request("http://kiwisdr.com/.public/", timeout=20)
+        res = fetch_with_curl("http://kiwisdr.com/.public/", timeout=20)
         if not res or res.status_code != 200:
             logger.error(f"KiwiSDR fetch failed: HTTP {res.status_code if res else 'no response'}")
             return []
@@ -77,21 +78,23 @@ def fetch_kiwisdr_nodes() -> list[dict]:
             except ValueError:
                 users_max = 0
 
-            nodes.append({
-                "name": name[:120],  # Truncate long names
-                "lat": round(lat, 5),
-                "lon": round(lon, 5),
-                "url": url,
-                "users": users,
-                "users_max": users_max,
-                "bands": bands,
-                "antenna": antenna[:200] if antenna else "",
-                "location": location[:100] if location else "",
-            })
+            nodes.append(
+                {
+                    "name": name[:120],  # Truncate long names
+                    "lat": round(lat, 5),
+                    "lon": round(lon, 5),
+                    "url": url,
+                    "users": users,
+                    "users_max": users_max,
+                    "bands": bands,
+                    "antenna": antenna[:200] if antenna else "",
+                    "location": location[:100] if location else "",
+                }
+            )
 
         logger.info(f"KiwiSDR: parsed {len(nodes)} online receivers")
         return nodes
 
-    except Exception as e:
+    except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
         logger.error(f"KiwiSDR fetch exception: {e}")
         return []

backend/services/liveuamap_scraper.py

@@ -8,90 +8,101 @@ from playwright_stealth import stealth_sync
 
 logger = logging.getLogger(__name__)
 
+
 def fetch_liveuamap():
     logger.info("Starting Liveuamap scraper with Playwright Stealth...")
-    
+
     regions = [
         {"name": "Ukraine", "url": "https://liveuamap.com"},
         {"name": "Middle East", "url": "https://mideast.liveuamap.com"},
         {"name": "Israel-Palestine", "url": "https://israelpalestine.liveuamap.com"},
-        {"name": "Syria", "url": "https://syria.liveuamap.com"}
+        {"name": "Syria", "url": "https://syria.liveuamap.com"},
     ]
-    
+
     all_markers = []
     seen_ids = set()
-    
+
     with sync_playwright() as p:
         # Launching with a real user agent to bypass Turnstile
-        browser = p.chromium.launch(headless=False, args=["--disable-blink-features=AutomationControlled"])
+        browser = p.chromium.launch(
+            headless=True, args=["--disable-blink-features=AutomationControlled"]
+        )
         context = browser.new_context(
             user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
             viewport={"width": 1920, "height": 1080},
-            color_scheme="dark"
+            color_scheme="dark",
         )
         page = context.new_page()
         stealth_sync(page)
-        
+
         for region in regions:
             try:
                 logger.info(f"Scraping Liveuamap region: {region['name']}")
                 page.goto(region["url"], timeout=60000, wait_until="domcontentloaded")
-                
+
                 # Wait for the map canvas or markers script to load, max 10s wait
                 try:
                     page.wait_for_timeout(5000)
-                except:
+                except (TimeoutError, OSError):  # non-critical: page load delay
                     pass
-                
+
                 html = page.content()
-                
+
                 m = re.search(r"var\s+ovens\s*=\s*(.*?);(?!function)", html, re.DOTALL)
                 if not m:
                     logger.warning(f"Could not find 'ovens' data for {region['name']} in raw HTML")
                     # Let's try grabbing the evaluated JavaScript variable if it's there
                     try:
-                        ovens_json = page.evaluate("() => typeof ovens !== 'undefined' ? JSON.stringify(ovens) : null")
+                        ovens_json = page.evaluate(
+                            "() => typeof ovens !== 'undefined' ? JSON.stringify(ovens) : null"
+                        )
                         if ovens_json:
                             markers = json.loads(ovens_json)
                             # process below
                             html = f"var ovens={ovens_json};"
                             m = re.search(r"var\s+ovens=(.*?);", html, re.DOTALL)
-                    except:
-                        pass
-                
+                    except (ValueError, KeyError, OSError) as e:  # non-critical: JS eval fallback
+                        logger.debug(
+                            f"Could not evaluate ovens JS variable for {region['name']}: {e}"
+                        )
+
                 if m:
                     json_str = m.group(1).strip()
                     if json_str.startswith("'") or json_str.startswith('"'):
-                        json_str = json_str.strip('"\'')
-                        json_str = base64.b64decode(urllib.parse.unquote(json_str)).decode('utf-8')
-                        
+                        json_str = json_str.strip("\"'")
+                        json_str = base64.b64decode(urllib.parse.unquote(json_str)).decode("utf-8")
+
                     try:
                         markers = json.loads(json_str)
                         for marker in markers:
                             mid = marker.get("id")
                             if mid and mid not in seen_ids:
                                 seen_ids.add(mid)
-                                all_markers.append({
-                                    "id": mid,
-                                    "type": "liveuamap",
-                                    "title": marker.get("s", "Unknown Event") or marker.get("title", ""),
-                                    "lat": marker.get("lat"),
-                                    "lng": marker.get("lng"),
-                                    "timestamp": marker.get("time", ""),
-                                    "link": marker.get("link", region["url"]),
-                                    "region": region["name"]
-                                })
-                    except Exception as e:
+                                all_markers.append(
+                                    {
+                                        "id": mid,
+                                        "type": "liveuamap",
+                                        "title": marker.get("s", "Unknown Event")
+                                        or marker.get("title", ""),
+                                        "lat": marker.get("lat"),
+                                        "lng": marker.get("lng"),
+                                        "timestamp": marker.get("time", ""),
+                                        "link": marker.get("link", region["url"]),
+                                        "region": region["name"],
+                                    }
+                                )
+                    except (json.JSONDecodeError, ValueError, KeyError) as e:
                         logger.error(f"Error parsing JSON for {region['name']}: {e}")
-                        
+
             except Exception as e:
                 logger.error(f"Error scraping Liveuamap {region['name']}: {e}")
-                
+
         browser.close()
-        
+
     logger.info(f"Liveuamap scraper finished, extracted {len(all_markers)} unique markers.")
     return all_markers
 
+
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
     res = fetch_liveuamap()

backend/services/logging_setup.py

@@ -0,0 +1,30 @@
+"""Structured logging setup for backend services."""
+
+import json
+import logging
+from datetime import datetime
+from typing import Any, Dict
+
+
+class JsonFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        payload: Dict[str, Any] = {
+            "ts": datetime.utcnow().isoformat(),
+            "level": record.levelname,
+            "logger": record.name,
+            "msg": record.getMessage(),
+        }
+        if record.exc_info:
+            payload["exc"] = self.formatException(record.exc_info)
+        return json.dumps(payload, ensure_ascii=False)
+
+
+def setup_logging(level: str = "INFO"):
+    """Configure root logger with JSON formatting."""
+    root = logging.getLogger()
+    if root.handlers:
+        return  # Respect existing config
+    root.setLevel(level.upper())
+    handler = logging.StreamHandler()
+    handler.setFormatter(JsonFormatter())
+    root.addHandler(handler)

backend/services/mesh/__init__.py

@@ -0,0 +1 @@
+# Mesh protocol services package

backend/services/mesh/mesh_bootstrap_manifest.py

@@ -0,0 +1,340 @@
+from __future__ import annotations
+
+import base64
+import json
+import time
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+from cryptography.exceptions import InvalidSignature
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import ed25519
+
+from services.config import get_settings
+from services.mesh.mesh_crypto import canonical_json, normalize_peer_url
+
+BACKEND_DIR = Path(__file__).resolve().parents[2]
+DATA_DIR = BACKEND_DIR / "data"
+DEFAULT_BOOTSTRAP_MANIFEST_PATH = DATA_DIR / "bootstrap_peers.json"
+BOOTSTRAP_MANIFEST_VERSION = 1
+ALLOWED_BOOTSTRAP_TRANSPORTS = {"clearnet", "onion"}
+ALLOWED_BOOTSTRAP_ROLES = {"participant", "relay", "seed"}
+
+
+class BootstrapManifestError(ValueError):
+    pass
+
+
+@dataclass(frozen=True)
+class BootstrapPeer:
+    peer_url: str
+    transport: str
+    role: str
+    label: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass(frozen=True)
+class BootstrapManifest:
+    version: int
+    issued_at: int
+    valid_until: int
+    signer_id: str
+    peers: tuple[BootstrapPeer, ...]
+    signature: str
+
+    def payload_dict(self) -> dict[str, Any]:
+        return {
+            "version": int(self.version),
+            "issued_at": int(self.issued_at),
+            "valid_until": int(self.valid_until),
+            "signer_id": str(self.signer_id or ""),
+            "peers": [peer.to_dict() for peer in self.peers],
+        }
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = self.payload_dict()
+        payload["signature"] = str(self.signature or "")
+        return payload
+
+
+def _resolve_manifest_path(raw_path: str) -> Path:
+    raw = str(raw_path or "").strip()
+    if not raw:
+        return DEFAULT_BOOTSTRAP_MANIFEST_PATH
+    candidate = Path(raw)
+    if candidate.is_absolute():
+        return candidate
+    return BACKEND_DIR / candidate
+
+
+def _canonical_manifest_payload(payload: dict[str, Any]) -> str:
+    return canonical_json(payload)
+
+
+def _load_signer_private_key(private_key_b64: str) -> ed25519.Ed25519PrivateKey:
+    try:
+        signer_private_key = base64.b64decode(
+            str(private_key_b64 or "").encode("utf-8"),
+            validate=True,
+        )
+        return ed25519.Ed25519PrivateKey.from_private_bytes(signer_private_key)
+    except Exception as exc:
+        raise BootstrapManifestError("bootstrap signer private key must be raw Ed25519 base64") from exc
+
+
+def bootstrap_signer_public_key_b64(private_key_b64: str) -> str:
+    signer = _load_signer_private_key(private_key_b64)
+    public_key = signer.public_key().public_bytes(
+        serialization.Encoding.Raw,
+        serialization.PublicFormat.Raw,
+    )
+    return base64.b64encode(public_key).decode("utf-8")
+
+
+def generate_bootstrap_signer() -> dict[str, str]:
+    signer = ed25519.Ed25519PrivateKey.generate()
+    private_key = signer.private_bytes(
+        serialization.Encoding.Raw,
+        serialization.PrivateFormat.Raw,
+        serialization.NoEncryption(),
+    )
+    public_key = signer.public_key().public_bytes(
+        serialization.Encoding.Raw,
+        serialization.PublicFormat.Raw,
+    )
+    return {
+        "private_key_b64": base64.b64encode(private_key).decode("utf-8"),
+        "public_key_b64": base64.b64encode(public_key).decode("utf-8"),
+    }
+
+
+def _verify_manifest_signature(
+    payload: dict[str, Any],
+    *,
+    signature_b64: str,
+    signer_public_key_b64: str,
+) -> None:
+    try:
+        signature = base64.b64decode(str(signature_b64 or "").encode("utf-8"), validate=True)
+    except Exception as exc:
+        raise BootstrapManifestError("bootstrap manifest signature must be base64") from exc
+
+    try:
+        signer_public_key = base64.b64decode(
+            str(signer_public_key_b64 or "").encode("utf-8"),
+            validate=True,
+        )
+        verifier = ed25519.Ed25519PublicKey.from_public_bytes(signer_public_key)
+    except Exception as exc:
+        raise BootstrapManifestError("bootstrap signer public key must be raw Ed25519 base64") from exc
+
+    serialized = _canonical_manifest_payload(payload).encode("utf-8")
+    try:
+        verifier.verify(signature, serialized)
+    except InvalidSignature as exc:
+        raise BootstrapManifestError("bootstrap manifest signature invalid") from exc
+
+
+def _validate_bootstrap_peer(peer_data: dict[str, Any]) -> BootstrapPeer:
+    peer_url = str(peer_data.get("peer_url", "") or "").strip()
+    transport = str(peer_data.get("transport", "") or "").strip().lower()
+    role = str(peer_data.get("role", "") or "").strip().lower()
+    label = str(peer_data.get("label", "") or "").strip()
+
+    if transport not in ALLOWED_BOOTSTRAP_TRANSPORTS:
+        raise BootstrapManifestError(f"unsupported bootstrap transport: {transport or 'missing'}")
+    if role not in ALLOWED_BOOTSTRAP_ROLES:
+        raise BootstrapManifestError(f"unsupported bootstrap role: {role or 'missing'}")
+
+    normalized = normalize_peer_url(peer_url)
+    if not normalized or normalized != peer_url:
+        raise BootstrapManifestError("bootstrap peer_url must be normalized")
+
+    parsed = urlparse(normalized)
+    hostname = str(parsed.hostname or "").strip().lower()
+    if transport == "clearnet":
+        if parsed.scheme != "https" or hostname.endswith(".onion"):
+            raise BootstrapManifestError("clearnet bootstrap peers must use https://")
+    elif transport == "onion":
+        if parsed.scheme != "http" or not hostname.endswith(".onion"):
+            raise BootstrapManifestError("onion bootstrap peers must use http://*.onion")
+
+    return BootstrapPeer(
+        peer_url=normalized,
+        transport=transport,
+        role=role,
+        label=label,
+    )
+
+
+def _validate_bootstrap_manifest_payload(
+    payload: dict[str, Any],
+    *,
+    now: float | None = None,
+) -> BootstrapManifest:
+    version = int(payload.get("version", 0) or 0)
+    issued_at = int(payload.get("issued_at", 0) or 0)
+    valid_until = int(payload.get("valid_until", 0) or 0)
+    signer_id = str(payload.get("signer_id", "") or "").strip()
+    peers_raw = payload.get("peers", [])
+    current_time = int(now if now is not None else time.time())
+
+    if version != BOOTSTRAP_MANIFEST_VERSION:
+        raise BootstrapManifestError(f"unsupported bootstrap manifest version: {version}")
+    if not signer_id:
+        raise BootstrapManifestError("bootstrap manifest signer_id is required")
+    if issued_at <= 0 or valid_until <= 0 or valid_until <= issued_at:
+        raise BootstrapManifestError("bootstrap manifest validity window is invalid")
+    if current_time > valid_until:
+        raise BootstrapManifestError("bootstrap manifest expired")
+    if not isinstance(peers_raw, list):
+        raise BootstrapManifestError("bootstrap manifest peers must be a list")
+
+    peers: list[BootstrapPeer] = []
+    seen: set[tuple[str, str]] = set()
+    for entry in peers_raw:
+        if not isinstance(entry, dict):
+            raise BootstrapManifestError("bootstrap manifest peers must be objects")
+        peer = _validate_bootstrap_peer(entry)
+        key = (peer.transport, peer.peer_url)
+        if key in seen:
+            raise BootstrapManifestError("bootstrap manifest peers must be unique")
+        seen.add(key)
+        peers.append(peer)
+
+    if not peers:
+        raise BootstrapManifestError("bootstrap manifest must contain at least one peer")
+
+    return BootstrapManifest(
+        version=version,
+        issued_at=issued_at,
+        valid_until=valid_until,
+        signer_id=signer_id,
+        peers=tuple(peers),
+        signature="",
+    )
+
+
+def build_bootstrap_manifest_payload(
+    *,
+    signer_id: str,
+    peers: list[dict[str, Any]] | tuple[dict[str, Any], ...],
+    issued_at: int | None = None,
+    valid_until: int | None = None,
+    valid_for_hours: int = 168,
+) -> dict[str, Any]:
+    timestamp = int(issued_at if issued_at is not None else time.time())
+    expiry = int(valid_until if valid_until is not None else timestamp + max(1, int(valid_for_hours or 0)) * 3600)
+    payload = {
+        "version": BOOTSTRAP_MANIFEST_VERSION,
+        "issued_at": timestamp,
+        "valid_until": expiry,
+        "signer_id": str(signer_id or "").strip(),
+        "peers": list(peers),
+    }
+    manifest = _validate_bootstrap_manifest_payload(payload, now=timestamp)
+    return manifest.payload_dict()
+
+
+def sign_bootstrap_manifest_payload(
+    payload: dict[str, Any],
+    *,
+    signer_private_key_b64: str,
+) -> str:
+    signer = _load_signer_private_key(signer_private_key_b64)
+    serialized = _canonical_manifest_payload(payload).encode("utf-8")
+    signature = signer.sign(serialized)
+    return base64.b64encode(signature).decode("utf-8")
+
+
+def write_signed_bootstrap_manifest(
+    path: str | Path,
+    *,
+    signer_id: str,
+    signer_private_key_b64: str,
+    peers: list[dict[str, Any]] | tuple[dict[str, Any], ...],
+    issued_at: int | None = None,
+    valid_until: int | None = None,
+    valid_for_hours: int = 168,
+) -> BootstrapManifest:
+    manifest_path = _resolve_manifest_path(str(path))
+    payload = build_bootstrap_manifest_payload(
+        signer_id=signer_id,
+        peers=list(peers),
+        issued_at=issued_at,
+        valid_until=valid_until,
+        valid_for_hours=valid_for_hours,
+    )
+    signature = sign_bootstrap_manifest_payload(
+        payload,
+        signer_private_key_b64=signer_private_key_b64,
+    )
+    manifest = BootstrapManifest(
+        version=int(payload["version"]),
+        issued_at=int(payload["issued_at"]),
+        valid_until=int(payload["valid_until"]),
+        signer_id=str(payload["signer_id"]),
+        peers=tuple(_validate_bootstrap_peer(dict(peer)) for peer in payload["peers"]),
+        signature=signature,
+    )
+    manifest_path.parent.mkdir(parents=True, exist_ok=True)
+    manifest_path.write_text(json.dumps(manifest.to_dict(), indent=2) + "\n", encoding="utf-8")
+    return manifest
+
+
+def load_bootstrap_manifest(
+    path: str | Path,
+    *,
+    signer_public_key_b64: str,
+    now: float | None = None,
+) -> BootstrapManifest:
+    manifest_path = _resolve_manifest_path(str(path))
+    try:
+        raw = json.loads(manifest_path.read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise BootstrapManifestError(f"bootstrap manifest not found: {manifest_path}") from exc
+    except json.JSONDecodeError as exc:
+        raise BootstrapManifestError("bootstrap manifest is not valid JSON") from exc
+
+    if not isinstance(raw, dict):
+        raise BootstrapManifestError("bootstrap manifest root must be an object")
+
+    signature = str(raw.get("signature", "") or "").strip()
+    payload = {key: value for key, value in raw.items() if key != "signature"}
+    if not signature:
+        raise BootstrapManifestError("bootstrap manifest signature is required")
+
+    _verify_manifest_signature(
+        payload,
+        signature_b64=signature,
+        signer_public_key_b64=signer_public_key_b64,
+    )
+    manifest = _validate_bootstrap_manifest_payload(payload, now=now)
+    return BootstrapManifest(
+        version=manifest.version,
+        issued_at=manifest.issued_at,
+        valid_until=manifest.valid_until,
+        signer_id=manifest.signer_id,
+        peers=manifest.peers,
+        signature=signature,
+    )
+
+
+def load_bootstrap_manifest_from_settings(*, now: float | None = None) -> BootstrapManifest | None:
+    settings = get_settings()
+    if bool(getattr(settings, "MESH_BOOTSTRAP_DISABLED", False)):
+        return None
+    signer_public_key_b64 = str(getattr(settings, "MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY", "") or "").strip()
+    if not signer_public_key_b64:
+        return None
+    manifest_path = _resolve_manifest_path(str(getattr(settings, "MESH_BOOTSTRAP_MANIFEST_PATH", "") or ""))
+    return load_bootstrap_manifest(
+        manifest_path,
+        signer_public_key_b64=signer_public_key_b64,
+        now=now,
+    )

backend/services/mesh/mesh_crypto.py

@@ -0,0 +1,147 @@
+"""Cryptographic helpers for Mesh protocol verification."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import hmac
+import json
+from typing import Any
+from urllib.parse import urlparse
+
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import ec, ed25519
+from cryptography.exceptions import InvalidSignature
+
+from services.mesh.mesh_protocol import PROTOCOL_VERSION, NETWORK_ID, normalize_payload
+
+NODE_ID_PREFIX = "!sb_"
+NODE_ID_HEX_LEN = 16
+
+
+def canonical_json(obj: dict[str, Any]) -> str:
+    return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+
+
+def normalize_peer_url(peer_url: str) -> str:
+    raw = str(peer_url or "").strip()
+    if not raw:
+        return ""
+    parsed = urlparse(raw)
+    scheme = str(parsed.scheme or "").strip().lower()
+    hostname = str(parsed.hostname or "").strip().lower()
+    if not scheme or not hostname:
+        return ""
+    port = parsed.port
+    default_port = 443 if scheme == "https" else 80 if scheme == "http" else None
+    netloc = hostname
+    if port and port != default_port:
+        netloc = f"{hostname}:{port}"
+    path = str(parsed.path or "").rstrip("/")
+    return f"{scheme}://{netloc}{path}"
+
+
+def _derive_peer_key(shared_secret: str, peer_url: str) -> bytes:
+    normalized_url = normalize_peer_url(peer_url)
+    if not shared_secret or not normalized_url:
+        return b""
+    # HKDF-Extract per RFC 5869 §2.2: PRK = HMAC-Hash(salt, IKM).
+    # Python's hmac.new(key=salt, msg=IKM) maps directly to that definition.
+    prk = hmac.new(
+        b"sb-peer-auth-v1",
+        shared_secret.encode("utf-8"),
+        hashlib.sha256,
+    ).digest()
+    return hmac.new(
+        prk,
+        normalized_url.encode("utf-8") + b"\x01",
+        hashlib.sha256,
+    ).digest()
+
+
+def _node_digest(public_key_b64: str) -> str:
+    raw = base64.b64decode(public_key_b64)
+    return hashlib.sha256(raw).hexdigest()
+
+
+def derive_node_id(public_key_b64: str, *, legacy: bool = False) -> str:
+    digest = _node_digest(public_key_b64)
+    length = NODE_ID_HEX_LEN
+    return NODE_ID_PREFIX + digest[:length]
+
+
+def derive_node_id_candidates(public_key_b64: str) -> tuple[str, ...]:
+    current = derive_node_id(public_key_b64, legacy=False)
+    return (current,)
+
+
+def build_signature_payload(
+    *,
+    event_type: str,
+    node_id: str,
+    sequence: int,
+    payload: dict[str, Any],
+) -> str:
+    normalized = normalize_payload(event_type, payload)
+    # gate_envelope and reply_to ride alongside the signed payload — they are
+    # added after the message is signed so must be excluded from verification.
+    if event_type == "gate_message":
+        for _unsig in ("gate_envelope", "reply_to"):
+            normalized.pop(_unsig, None)
+    payload_json = canonical_json(normalized)
+    return "|".join(
+        [PROTOCOL_VERSION, NETWORK_ID, event_type, node_id, str(sequence), payload_json]
+    )
+
+
+def parse_public_key_algo(value: str) -> str:
+    val = (value or "").strip().upper()
+    if val in ("ED25519", "EDDSA"):
+        return "Ed25519"
+    if val in ("ECDSA", "ECDSA_P256", "P-256", "P256"):
+        return "ECDSA_P256"
+    return ""
+
+
+def verify_signature(
+    *,
+    public_key_b64: str,
+    public_key_algo: str,
+    signature_hex: str,
+    payload: str,
+) -> bool:
+    try:
+        sig_bytes = bytes.fromhex(signature_hex)
+    except Exception:
+        return False
+
+    try:
+        pub_raw = base64.b64decode(public_key_b64)
+    except Exception:
+        return False
+
+    algo = parse_public_key_algo(public_key_algo)
+    data = payload.encode("utf-8")
+
+    try:
+        if algo == "Ed25519":
+            pub = ed25519.Ed25519PublicKey.from_public_bytes(pub_raw)
+            pub.verify(sig_bytes, data)
+            return True
+        if algo == "ECDSA_P256":
+            pub = ec.EllipticCurvePublicKey.from_encoded_point(ec.SECP256R1(), pub_raw)
+            pub.verify(sig_bytes, data, ec.ECDSA(hashes.SHA256()))
+            return True
+    except InvalidSignature:
+        return False
+    except Exception:
+        return False
+
+    return False
+
+
+def verify_node_binding(node_id: str, public_key_b64: str) -> bool:
+    try:
+        return str(node_id or "") in derive_node_id_candidates(public_key_b64)
+    except Exception:
+        return False

backend/services/mesh/mesh_dm_mls.py

@@ -0,0 +1,669 @@
+"""MLS-backed DM session manager.
+
+This module keeps DM session orchestration in Python while privacy-core owns
+the MLS session state. Python-side metadata survives via domain storage, but
+Rust session state remains in-memory only. Process restart still requires
+session re-establishment until Rust FFI state export is available.
+"""
+
+from __future__ import annotations
+
+import base64
+import logging
+import secrets
+import threading
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import x25519
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+
+from services.mesh.mesh_secure_storage import (
+    read_domain_json,
+    read_secure_json,
+    write_domain_json,
+)
+from services.mesh.mesh_privacy_logging import privacy_log_label
+from services.mesh.mesh_wormhole_persona import sign_dm_alias_blob, verify_dm_alias_blob
+from services.privacy_core_client import PrivacyCoreClient, PrivacyCoreError
+from services.wormhole_supervisor import get_wormhole_state, transport_tier_from_state
+
+logger = logging.getLogger(__name__)
+
+DATA_DIR = Path(__file__).resolve().parents[2] / "data"
+STATE_FILE = DATA_DIR / "wormhole_dm_mls.json"
+STATE_FILENAME = "wormhole_dm_mls.json"
+STATE_DOMAIN = "dm_alias"
+_STATE_LOCK = threading.RLock()
+_PRIVACY_CLIENT: PrivacyCoreClient | None = None
+_STATE_LOADED = False
+_TRANSPORT_TIER_ORDER = {
+    "public_degraded": 0,
+    "private_transitional": 1,
+    "private_strong": 2,
+}
+MLS_DM_FORMAT = "mls1"
+MAX_DM_PLAINTEXT_SIZE = 65_536
+
+try:
+    from nacl.public import PrivateKey as _NaclPrivateKey
+    from nacl.public import PublicKey as _NaclPublicKey
+    from nacl.public import SealedBox as _NaclSealedBox
+except ImportError:
+    _NaclPrivateKey = None
+    _NaclPublicKey = None
+    _NaclSealedBox = None
+
+
+def _b64(data: bytes) -> str:
+    return base64.b64encode(data).decode("ascii")
+
+
+def _unb64(data: str | bytes | None) -> bytes:
+    if not data:
+        return b""
+    if isinstance(data, bytes):
+        return base64.b64decode(data)
+    return base64.b64decode(data.encode("ascii"))
+
+
+def _decode_key_text(data: str | bytes | None) -> bytes:
+    raw = str(data or "").strip()
+    if not raw:
+        return b""
+    try:
+        return bytes.fromhex(raw)
+    except ValueError:
+        return _unb64(raw)
+
+
+def _normalize_alias(alias: str) -> str:
+    return str(alias or "").strip().lower()
+
+
+def _session_id(local_alias: str, remote_alias: str) -> str:
+    return f"{_normalize_alias(local_alias)}::{_normalize_alias(remote_alias)}"
+
+
+def _seal_keypair() -> dict[str, str]:
+    private_key = x25519.X25519PrivateKey.generate()
+    return {
+        "public_key": private_key.public_key().public_bytes_raw().hex(),
+        "private_key": private_key.private_bytes_raw().hex(),
+    }
+
+
+def _seal_welcome_for_public_key(payload: bytes, public_key_text: str) -> bytes:
+    public_key_bytes = _decode_key_text(public_key_text)
+    if not public_key_bytes:
+        raise PrivacyCoreError("responder_dh_pub is required for sealed welcome")
+    if _NaclPublicKey is not None and _NaclSealedBox is not None:
+        return _NaclSealedBox(_NaclPublicKey(public_key_bytes)).encrypt(payload)
+
+    ephemeral_private = x25519.X25519PrivateKey.generate()
+    ephemeral_public = ephemeral_private.public_key().public_bytes_raw()
+    recipient_public = x25519.X25519PublicKey.from_public_bytes(public_key_bytes)
+    shared_secret = ephemeral_private.exchange(recipient_public)
+    key = HKDF(
+        algorithm=hashes.SHA256(),
+        length=32,
+        salt=None,
+        info=b"shadowbroker|dm-mls-welcome|v1",
+    ).derive(shared_secret)
+    nonce = secrets.token_bytes(12)
+    ciphertext = AESGCM(key).encrypt(
+        nonce,
+        payload,
+        b"shadowbroker|dm-mls-welcome|v1",
+    )
+    return ephemeral_public + nonce + ciphertext
+
+
+def _unseal_welcome_for_private_key(payload: bytes, private_key_text: str) -> bytes:
+    private_key_bytes = _decode_key_text(private_key_text)
+    if not private_key_bytes:
+        raise PrivacyCoreError("local DH secret unavailable for DM session acceptance")
+    if _NaclPrivateKey is not None and _NaclSealedBox is not None:
+        return _NaclSealedBox(_NaclPrivateKey(private_key_bytes)).decrypt(payload)
+    if len(payload) < 44:
+        raise PrivacyCoreError("sealed DM welcome is truncated")
+    ephemeral_public = x25519.X25519PublicKey.from_public_bytes(payload[:32])
+    nonce = payload[32:44]
+    ciphertext = payload[44:]
+    private_key = x25519.X25519PrivateKey.from_private_bytes(private_key_bytes)
+    shared_secret = private_key.exchange(ephemeral_public)
+    key = HKDF(
+        algorithm=hashes.SHA256(),
+        length=32,
+        salt=None,
+        info=b"shadowbroker|dm-mls-welcome|v1",
+    ).derive(shared_secret)
+    try:
+        return AESGCM(key).decrypt(
+            nonce,
+            ciphertext,
+            b"shadowbroker|dm-mls-welcome|v1",
+        )
+    except Exception as exc:
+        raise PrivacyCoreError("sealed DM welcome decrypt failed") from exc
+
+
+@dataclass
+class _SessionBinding:
+    session_id: str
+    local_alias: str
+    remote_alias: str
+    role: str
+    session_handle: int
+    created_at: int
+
+
+_ALIAS_IDENTITIES: dict[str, int] = {}
+_ALIAS_BINDINGS: dict[str, dict[str, str]] = {}
+_ALIAS_SEAL_KEYS: dict[str, dict[str, str]] = {}
+_SESSIONS: dict[str, _SessionBinding] = {}
+_DM_FORMAT_LOCKS: dict[str, str] = {}
+
+
+def _default_state() -> dict[str, Any]:
+    return {
+        "version": 2,
+        "updated_at": 0,
+        "aliases": {},
+        "alias_seal_keys": {},
+        "sessions": {},
+        "dm_format_locks": {},
+    }
+
+
+def _privacy_client() -> PrivacyCoreClient:
+    global _PRIVACY_CLIENT
+    if _PRIVACY_CLIENT is None:
+        _PRIVACY_CLIENT = PrivacyCoreClient.load()
+    return _PRIVACY_CLIENT
+
+
+def _current_transport_tier() -> str:
+    return transport_tier_from_state(get_wormhole_state())
+
+
+def _require_private_transport() -> tuple[bool, str]:
+    current = _current_transport_tier()
+    if _TRANSPORT_TIER_ORDER.get(current, 0) < _TRANSPORT_TIER_ORDER["private_transitional"]:
+        return False, "DM MLS requires PRIVATE transport tier"
+    return True, current
+
+
+def _serialize_session(binding: _SessionBinding) -> dict[str, Any]:
+    return {
+        "session_id": binding.session_id,
+        "local_alias": binding.local_alias,
+        "remote_alias": binding.remote_alias,
+        "role": binding.role,
+        "session_handle": int(binding.session_handle),
+        "created_at": int(binding.created_at),
+    }
+
+
+def _binding_record(handle: int, public_bundle: bytes, binding_proof: str) -> dict[str, Any]:
+    return {
+        "handle": int(handle),
+        "public_bundle": _b64(public_bundle),
+        "binding_proof": str(binding_proof or ""),
+    }
+
+
+def _load_state() -> None:
+    global _STATE_LOADED
+    with _STATE_LOCK:
+        if _STATE_LOADED:
+            return
+        # KNOWN LIMITATION: Persisted handles only survive when the privacy-core
+        # library instance is still alive in the same process. Full Rust-state
+        # export/import is deferred to a later sprint.
+        domain_path = DATA_DIR / STATE_DOMAIN / STATE_FILENAME
+        if not domain_path.exists() and STATE_FILE.exists():
+            try:
+                legacy = read_secure_json(STATE_FILE, _default_state)
+                write_domain_json(STATE_DOMAIN, STATE_FILENAME, legacy)
+                STATE_FILE.unlink(missing_ok=True)
+            except Exception:
+                logger.warning(
+                    "Legacy DM MLS state could not be decrypted — "
+                    "discarding stale file and starting fresh"
+                )
+                STATE_FILE.unlink(missing_ok=True)
+        raw = read_domain_json(STATE_DOMAIN, STATE_FILENAME, _default_state)
+        state = _default_state()
+        if isinstance(raw, dict):
+            state.update(raw)
+
+        _ALIAS_IDENTITIES.clear()
+        _ALIAS_BINDINGS.clear()
+        for alias, payload in dict(state.get("aliases") or {}).items():
+            alias_key = _normalize_alias(alias)
+            if not alias_key:
+                continue
+            if isinstance(payload, dict):
+                handle = int(payload.get("handle", 0) or 0)
+                public_bundle_b64 = str(payload.get("public_bundle", "") or "")
+                binding_proof = str(payload.get("binding_proof", "") or "")
+            else:
+                handle = int(payload or 0)
+                public_bundle_b64 = ""
+                binding_proof = ""
+            if handle <= 0 or not public_bundle_b64 or not binding_proof:
+                logger.warning("DM MLS alias binding missing proof; identity will be re-created")
+                continue
+            try:
+                public_bundle = _unb64(public_bundle_b64)
+            except Exception as exc:
+                logger.warning("DM MLS alias binding decode failed: %s", type(exc).__name__)
+                continue
+            ok, reason = verify_dm_alias_blob(alias_key, public_bundle, binding_proof)
+            if not ok:
+                logger.warning("DM MLS alias binding invalid: %s", reason)
+                continue
+            _ALIAS_IDENTITIES[alias_key] = handle
+            _ALIAS_BINDINGS[alias_key] = _binding_record(handle, public_bundle, binding_proof)
+
+        _ALIAS_SEAL_KEYS.clear()
+        for alias, keypair in dict(state.get("alias_seal_keys") or {}).items():
+            alias_key = _normalize_alias(alias)
+            pair = dict(keypair or {})
+            public_key = str(pair.get("public_key", "") or "").strip().lower()
+            private_key = str(pair.get("private_key", "") or "").strip().lower()
+            if alias_key and public_key and private_key:
+                _ALIAS_SEAL_KEYS[alias_key] = {
+                    "public_key": public_key,
+                    "private_key": private_key,
+                }
+
+        _SESSIONS.clear()
+        for session_id, payload in dict(state.get("sessions") or {}).items():
+            if not isinstance(payload, dict):
+                continue
+            binding = _SessionBinding(
+                session_id=str(payload.get("session_id", session_id) or session_id),
+                local_alias=_normalize_alias(str(payload.get("local_alias", "") or "")),
+                remote_alias=_normalize_alias(str(payload.get("remote_alias", "") or "")),
+                role=str(payload.get("role", "initiator") or "initiator"),
+                session_handle=int(payload.get("session_handle", 0) or 0),
+                created_at=int(payload.get("created_at", 0) or 0),
+            )
+            if (
+                binding.session_id
+                and binding.session_handle > 0
+                and binding.local_alias in _ALIAS_IDENTITIES
+            ):
+                _SESSIONS[binding.session_id] = binding
+
+        _DM_FORMAT_LOCKS.clear()
+        for session_id, payload_format in dict(state.get("dm_format_locks") or {}).items():
+            normalized = str(payload_format or "").strip().lower()
+            if normalized:
+                _DM_FORMAT_LOCKS[str(session_id or "")] = normalized
+        _STATE_LOADED = True
+
+
+def _save_state() -> None:
+    with _STATE_LOCK:
+        write_domain_json(
+            STATE_DOMAIN,
+            STATE_FILENAME,
+            {
+                "version": 2,
+                "updated_at": int(time.time()),
+                "aliases": {
+                    alias: dict(_ALIAS_BINDINGS.get(alias) or {})
+                    for alias, handle in _ALIAS_IDENTITIES.items()
+                    if _ALIAS_BINDINGS.get(alias)
+                },
+                "alias_seal_keys": {
+                    alias: dict(keypair or {})
+                    for alias, keypair in _ALIAS_SEAL_KEYS.items()
+                },
+                "sessions": {
+                    session_id: _serialize_session(binding)
+                    for session_id, binding in _SESSIONS.items()
+                },
+                "dm_format_locks": dict(_DM_FORMAT_LOCKS),
+            },
+        )
+        STATE_FILE.unlink(missing_ok=True)
+
+
+def reset_dm_mls_state(*, clear_privacy_core: bool = False, clear_persistence: bool = True) -> None:
+    global _PRIVACY_CLIENT, _STATE_LOADED
+    with _STATE_LOCK:
+        if clear_privacy_core and _PRIVACY_CLIENT is not None:
+            try:
+                _PRIVACY_CLIENT.reset_all_state()
+            except Exception:
+                logger.exception("privacy-core reset failed while clearing DM MLS state")
+        _ALIAS_IDENTITIES.clear()
+        _ALIAS_BINDINGS.clear()
+        _ALIAS_SEAL_KEYS.clear()
+        _SESSIONS.clear()
+        _DM_FORMAT_LOCKS.clear()
+        _STATE_LOADED = False
+        if clear_persistence and STATE_FILE.exists():
+            STATE_FILE.unlink()
+
+
+def _identity_handle_for_alias(alias: str) -> int:
+    alias_key = _normalize_alias(alias)
+    if not alias_key:
+        raise PrivacyCoreError("dm alias is required")
+    _load_state()
+    with _STATE_LOCK:
+        handle = _ALIAS_IDENTITIES.get(alias_key)
+        if handle:
+            return handle
+        handle = _privacy_client().create_identity()
+        public_bundle = _privacy_client().export_public_bundle(handle)
+        signed = sign_dm_alias_blob(alias_key, public_bundle)
+        if not signed.get("ok"):
+            try:
+                _privacy_client().release_identity(handle)
+            except Exception:
+                pass
+            raise PrivacyCoreError(str(signed.get("detail") or "dm_mls_identity_binding_failed"))
+        _ALIAS_IDENTITIES[alias_key] = handle
+        _ALIAS_BINDINGS[alias_key] = _binding_record(
+            handle,
+            public_bundle,
+            str(signed.get("signature", "") or ""),
+        )
+        _save_state()
+        return handle
+
+
+def _seal_keypair_for_alias(alias: str) -> dict[str, str]:
+    alias_key = _normalize_alias(alias)
+    if not alias_key:
+        raise PrivacyCoreError("dm alias is required")
+    _load_state()
+    with _STATE_LOCK:
+        existing = _ALIAS_SEAL_KEYS.get(alias_key)
+        if existing and existing.get("public_key") and existing.get("private_key"):
+            return dict(existing)
+        created = _seal_keypair()
+        _ALIAS_SEAL_KEYS[alias_key] = created
+        _save_state()
+        return dict(created)
+
+
+def export_dm_key_package_for_alias(alias: str) -> dict[str, Any]:
+    alias_key = _normalize_alias(alias)
+    if not alias_key:
+        return {"ok": False, "detail": "alias is required"}
+    try:
+        identity_handle = _identity_handle_for_alias(alias_key)
+        key_package = _privacy_client().export_key_package(identity_handle)
+        seal_keypair = _seal_keypair_for_alias(alias_key)
+        return {
+            "ok": True,
+            "alias": alias_key,
+            "mls_key_package": _b64(key_package),
+            "welcome_dh_pub": str(seal_keypair.get("public_key", "") or ""),
+        }
+    except Exception:
+        logger.exception(
+            "dm mls key package export failed for %s",
+            privacy_log_label(alias_key, label="alias"),
+        )
+        return {"ok": False, "detail": "dm_mls_key_package_failed"}
+
+
+def _remember_session(local_alias: str, remote_alias: str, *, role: str, session_handle: int) -> _SessionBinding:
+    binding = _SessionBinding(
+        session_id=_session_id(local_alias, remote_alias),
+        local_alias=_normalize_alias(local_alias),
+        remote_alias=_normalize_alias(remote_alias),
+        role=str(role or "initiator"),
+        session_handle=int(session_handle),
+        created_at=int(time.time()),
+    )
+    with _STATE_LOCK:
+        existing = _SESSIONS.get(binding.session_id)
+        if existing is not None:
+            try:
+                _privacy_client().release_dm_session(session_handle)
+            except Exception:
+                pass
+            return existing
+        _SESSIONS[binding.session_id] = binding
+        _save_state()
+    return binding
+
+
+def _forget_session(local_alias: str, remote_alias: str) -> _SessionBinding | None:
+    _load_state()
+    with _STATE_LOCK:
+        binding = _SESSIONS.pop(_session_id(local_alias, remote_alias), None)
+        _save_state()
+        return binding
+
+
+def _lock_dm_format(local_alias: str, remote_alias: str, format_str: str) -> None:
+    _load_state()
+    with _STATE_LOCK:
+        _DM_FORMAT_LOCKS[_session_id(local_alias, remote_alias)] = str(format_str or "").strip().lower()
+        _save_state()
+
+
+def is_dm_locked_to_mls(local_alias: str, remote_alias: str) -> bool:
+    _load_state()
+    return (
+        str(_DM_FORMAT_LOCKS.get(_session_id(local_alias, remote_alias), "") or "").strip().lower()
+        == MLS_DM_FORMAT
+    )
+
+
+def _session_binding(local_alias: str, remote_alias: str) -> _SessionBinding:
+    _load_state()
+    session_id = _session_id(local_alias, remote_alias)
+    binding = _SESSIONS.get(session_id)
+    if binding is None:
+        raise PrivacyCoreError(f"dm session not found for {session_id}")
+    return binding
+
+
+def initiate_dm_session(
+    local_alias: str,
+    remote_alias: str,
+    remote_prekey_bundle: dict,
+    responder_dh_pub: str = "",
+) -> dict[str, Any]:
+    ok, detail = _require_private_transport()
+    if not ok:
+        return {"ok": False, "detail": detail}
+    local_key = _normalize_alias(local_alias)
+    remote_key = _normalize_alias(remote_alias)
+    remote_key_package_b64 = str(
+        (remote_prekey_bundle or {}).get("mls_key_package")
+        or (remote_prekey_bundle or {}).get("key_package")
+        or ""
+    ).strip()
+    if not local_key or not remote_key or not remote_key_package_b64:
+        return {"ok": False, "detail": "local_alias, remote_alias, and mls_key_package are required"}
+    resolved_responder_dh_pub = str(
+        responder_dh_pub
+        or (remote_prekey_bundle or {}).get("welcome_dh_pub")
+        or (remote_prekey_bundle or {}).get("identity_dh_pub_key")
+        or ""
+    ).strip()
+    key_package_handle = 0
+    session_handle = 0
+    remembered = False
+    try:
+        identity_handle = _identity_handle_for_alias(local_key)
+        key_package_handle = _privacy_client().import_key_package(_unb64(remote_key_package_b64))
+        session_handle = _privacy_client().create_dm_session(identity_handle, key_package_handle)
+        welcome = _privacy_client().dm_session_welcome(session_handle)
+        sealed_welcome = _seal_welcome_for_public_key(welcome, resolved_responder_dh_pub)
+        binding = _remember_session(local_key, remote_key, role="initiator", session_handle=session_handle)
+        remembered = True
+        return {"ok": True, "welcome": _b64(sealed_welcome), "session_id": binding.session_id}
+    except Exception:
+        logger.exception(
+            "dm mls initiate failed for %s -> %s",
+            privacy_log_label(local_key, label="alias"),
+            privacy_log_label(remote_key, label="alias"),
+        )
+        return {"ok": False, "detail": "dm_mls_initiate_failed"}
+    finally:
+        if key_package_handle:
+            try:
+                _privacy_client().release_key_package(key_package_handle)
+            except Exception:
+                pass
+        if session_handle and not remembered:
+            try:
+                _privacy_client().release_dm_session(session_handle)
+            except Exception:
+                pass
+
+
+def accept_dm_session(
+    local_alias: str,
+    remote_alias: str,
+    welcome_b64: str,
+    local_dh_secret: str = "",
+) -> dict[str, Any]:
+    ok, detail = _require_private_transport()
+    if not ok:
+        return {"ok": False, "detail": detail}
+    local_key = _normalize_alias(local_alias)
+    remote_key = _normalize_alias(remote_alias)
+    if not local_key or not remote_key or not str(welcome_b64 or "").strip():
+        return {"ok": False, "detail": "local_alias, remote_alias, and welcome are required"}
+    session_handle = 0
+    remembered = False
+    try:
+        identity_handle = _identity_handle_for_alias(local_key)
+        seal_keypair = _seal_keypair_for_alias(local_key)
+        welcome = _unseal_welcome_for_private_key(
+            _unb64(welcome_b64),
+            str(local_dh_secret or seal_keypair.get("private_key") or ""),
+        )
+        session_handle = _privacy_client().join_dm_session(identity_handle, welcome)
+        binding = _remember_session(local_key, remote_key, role="responder", session_handle=session_handle)
+        remembered = True
+        return {"ok": True, "session_id": binding.session_id}
+    except Exception:
+        logger.exception(
+            "dm mls accept failed for %s <- %s",
+            privacy_log_label(local_key, label="alias"),
+            privacy_log_label(remote_key, label="alias"),
+        )
+        return {"ok": False, "detail": "dm_mls_accept_failed"}
+    finally:
+        if session_handle and not remembered:
+            try:
+                _privacy_client().release_dm_session(session_handle)
+            except Exception:
+                pass
+
+
+def has_dm_session(local_alias: str, remote_alias: str) -> dict[str, Any]:
+    ok, detail = _require_private_transport()
+    if not ok:
+        return {"ok": False, "detail": detail}
+    try:
+        binding = _session_binding(local_alias, remote_alias)
+        return {"ok": True, "exists": True, "session_id": binding.session_id}
+    except Exception:
+        return {"ok": True, "exists": False, "session_id": _session_id(local_alias, remote_alias)}
+
+
+def ensure_dm_session(local_alias: str, remote_alias: str, welcome_b64: str) -> dict[str, Any]:
+    ok, detail = _require_private_transport()
+    if not ok:
+        return {"ok": False, "detail": detail}
+    has_session = has_dm_session(local_alias, remote_alias)
+    if not has_session.get("ok"):
+        return has_session
+    if has_session.get("exists"):
+        return {"ok": True, "session_id": _session_id(local_alias, remote_alias)}
+    return accept_dm_session(local_alias, remote_alias, welcome_b64)
+
+
+def _session_expired_result(local_alias: str, remote_alias: str) -> dict[str, Any]:
+    binding = _forget_session(local_alias, remote_alias)
+    session_id = binding.session_id if binding is not None else _session_id(local_alias, remote_alias)
+    return {"ok": False, "detail": "session_expired", "session_id": session_id}
+
+
+def encrypt_dm(local_alias: str, remote_alias: str, plaintext: str) -> dict[str, Any]:
+    ok, detail = _require_private_transport()
+    if not ok:
+        return {"ok": False, "detail": detail}
+    plaintext_bytes = str(plaintext or "").encode("utf-8")
+    if len(plaintext_bytes) > MAX_DM_PLAINTEXT_SIZE:
+        return {"ok": False, "detail": "plaintext exceeds maximum size"}
+    try:
+        binding = _session_binding(local_alias, remote_alias)
+        ciphertext = _privacy_client().dm_encrypt(binding.session_handle, plaintext_bytes)
+        _lock_dm_format(local_alias, remote_alias, MLS_DM_FORMAT)
+        return {
+            "ok": True,
+            "ciphertext": _b64(ciphertext),
+            # NOTE: nonce is generated for DM envelope compatibility with dm1 format.
+            # MLS handles its own nonce/IV internally — this field is not consumed by MLS.
+            "nonce": _b64(secrets.token_bytes(12)),
+            "session_id": binding.session_id,
+        }
+    except PrivacyCoreError as exc:
+        if "unknown dm session handle" in str(exc).lower():
+            return _session_expired_result(local_alias, remote_alias)
+        logger.exception(
+            "dm mls encrypt failed for %s -> %s",
+            privacy_log_label(local_alias, label="alias"),
+            privacy_log_label(remote_alias, label="alias"),
+        )
+        return {"ok": False, "detail": "dm_mls_encrypt_failed"}
+    except Exception:
+        logger.exception(
+            "dm mls encrypt failed for %s -> %s",
+            privacy_log_label(local_alias, label="alias"),
+            privacy_log_label(remote_alias, label="alias"),
+        )
+        return {"ok": False, "detail": "dm_mls_encrypt_failed"}
+
+
+def decrypt_dm(local_alias: str, remote_alias: str, ciphertext_b64: str, nonce_b64: str) -> dict[str, Any]:
+    ok, detail = _require_private_transport()
+    if not ok:
+        return {"ok": False, "detail": detail}
+    try:
+        binding = _session_binding(local_alias, remote_alias)
+        plaintext = _privacy_client().dm_decrypt(binding.session_handle, _unb64(ciphertext_b64))
+        _lock_dm_format(local_alias, remote_alias, MLS_DM_FORMAT)
+        return {
+            "ok": True,
+            "plaintext": plaintext.decode("utf-8"),
+            "session_id": binding.session_id,
+            "nonce": str(nonce_b64 or ""),
+        }
+    except PrivacyCoreError as exc:
+        if "unknown dm session handle" in str(exc).lower():
+            return _session_expired_result(local_alias, remote_alias)
+        logger.exception(
+            "dm mls decrypt failed for %s <- %s",
+            privacy_log_label(local_alias, label="alias"),
+            privacy_log_label(remote_alias, label="alias"),
+        )
+        return {"ok": False, "detail": "dm_mls_decrypt_failed"}
+    except Exception:
+        logger.exception(
+            "dm mls decrypt failed for %s <- %s",
+            privacy_log_label(local_alias, label="alias"),
+            privacy_log_label(remote_alias, label="alias"),
+        )
+        return {"ok": False, "detail": "dm_mls_decrypt_failed"}

backend/services/mesh/mesh_dm_relay.py

@@ -0,0 +1,824 @@
+"""Metadata-minimized DM relay for request and shared mailboxes.
+
+This relay never decrypts application payloads. In secure mode it keeps
+pending ciphertext in memory only and persists just the minimum metadata
+needed for continuity: accepted DH bundles, block lists, witness data,
+and nonce replay windows.
+"""
+
+from __future__ import annotations
+
+import atexit
+import hashlib
+import json
+import logging
+import os
+import secrets
+import threading
+import time
+from collections import OrderedDict, defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from services.config import get_settings
+from services.mesh.mesh_metrics import increment as metrics_inc
+from services.mesh.mesh_wormhole_prekey import _validate_bundle_record
+from services.mesh.mesh_secure_storage import read_secure_json, write_secure_json
+
+TTL_SECONDS = 3600
+EPOCH_SECONDS = 6 * 60 * 60
+DATA_DIR = Path(__file__).resolve().parents[2] / "data"
+RELAY_FILE = DATA_DIR / "dm_relay.json"
+logger = logging.getLogger(__name__)
+
+
+def _get_token_pepper() -> str:
+    """Read token pepper lazily so auto-generated values from startup audit take effect."""
+    pepper = os.environ.get("MESH_DM_TOKEN_PEPPER", "").strip()
+    if not pepper:
+        try:
+            from services.config import get_settings
+            from services.env_check import _ensure_dm_token_pepper
+
+            pepper = _ensure_dm_token_pepper(get_settings())
+        except Exception:
+            pepper = os.environ.get("MESH_DM_TOKEN_PEPPER", "").strip()
+    if not pepper:
+        raise RuntimeError("MESH_DM_TOKEN_PEPPER is unavailable at runtime")
+    return pepper
+
+
+@dataclass
+class DMMessage:
+    sender_id: str
+    ciphertext: str
+    timestamp: float
+    msg_id: str
+    delivery_class: str
+    sender_seal: str = ""
+    relay_salt: str = ""
+    sender_block_ref: str = ""
+    payload_format: str = "dm1"
+    session_welcome: str = ""
+
+
+class DMRelay:
+    """Relay for encrypted request/shared mailboxes."""
+
+    def __init__(self) -> None:
+        self._lock = threading.RLock()
+        self._mailboxes: dict[str, list[DMMessage]] = defaultdict(list)
+        self._dh_keys: dict[str, dict[str, Any]] = {}
+        self._prekey_bundles: dict[str, dict[str, Any]] = {}
+        self._mailbox_bindings: dict[str, dict[str, Any]] = defaultdict(dict)
+        self._witnesses: dict[str, list[dict[str, Any]]] = defaultdict(list)
+        self._blocks: dict[str, set[str]] = defaultdict(set)
+        self._nonce_cache: OrderedDict[str, float] = OrderedDict()
+        self._stats: dict[str, int] = {"messages_in_memory": 0}
+        self._dirty = False
+        self._save_timer: threading.Timer | None = None
+        self._SAVE_INTERVAL = 5.0
+        atexit.register(self._flush)
+        self._load()
+
+    def _settings(self):
+        return get_settings()
+
+    def _persist_spool_enabled(self) -> bool:
+        return bool(self._settings().MESH_DM_PERSIST_SPOOL)
+
+    def _request_mailbox_limit(self) -> int:
+        return max(1, int(self._settings().MESH_DM_REQUEST_MAILBOX_LIMIT))
+
+    def _shared_mailbox_limit(self) -> int:
+        return max(1, int(self._settings().MESH_DM_SHARED_MAILBOX_LIMIT))
+
+    def _self_mailbox_limit(self) -> int:
+        return max(1, int(self._settings().MESH_DM_SELF_MAILBOX_LIMIT))
+
+    def _nonce_ttl_seconds(self) -> int:
+        return max(30, int(self._settings().MESH_DM_NONCE_TTL_S))
+
+    def _nonce_cache_max_entries(self) -> int:
+        return max(1, int(getattr(self._settings(), "MESH_DM_NONCE_CACHE_MAX", 4096)))
+
+    def _pepper_token(self, token: str) -> str:
+        material = token
+        pepper = _get_token_pepper()
+        if pepper:
+            material = f"{pepper}|{token}"
+        return hashlib.sha256(material.encode("utf-8")).hexdigest()
+
+    def _sender_block_ref(self, sender_id: str) -> str:
+        sender = str(sender_id or "").strip()
+        if not sender:
+            return ""
+        return "ref:" + self._pepper_token(f"block|{sender}")
+
+    def _canonical_blocked_id(self, blocked_id: str) -> str:
+        blocked = str(blocked_id or "").strip()
+        if not blocked:
+            return ""
+        if blocked.startswith("ref:"):
+            return blocked
+        return self._sender_block_ref(blocked)
+
+    def _message_block_ref(self, message: DMMessage) -> str:
+        block_ref = str(getattr(message, "sender_block_ref", "") or "").strip()
+        if block_ref:
+            return block_ref
+        sender_id = str(message.sender_id or "").strip()
+        if not sender_id or sender_id.startswith("sealed:") or sender_id.startswith("sender_token:"):
+            return ""
+        return self._sender_block_ref(sender_id)
+
+    def _mailbox_key(self, mailbox_type: str, mailbox_value: str, epoch: int | None = None) -> str:
+        if mailbox_type in {"self", "requests"}:
+            bucket = self._epoch_bucket() if epoch is None else int(epoch)
+            identifier = f"{mailbox_type}|{bucket}|{mailbox_value}"
+        else:
+            identifier = f"{mailbox_type}|{mailbox_value}"
+        return self._pepper_token(identifier)
+
+    def _hashed_mailbox_token(self, token: str) -> str:
+        return hashlib.sha256(str(token or "").encode("utf-8")).hexdigest()
+
+    def _remember_mailbox_binding(self, agent_id: str, mailbox_type: str, token: str) -> str:
+        token_hash = self._hashed_mailbox_token(token)
+        self._mailbox_bindings[str(agent_id or "").strip()][str(mailbox_type or "").strip().lower()] = {
+            "token_hash": token_hash,
+            "last_used": time.time(),
+        }
+        self._save()
+        return token_hash
+
+    def _bound_mailbox_key(self, agent_id: str, mailbox_type: str) -> str:
+        entry = self._mailbox_bindings.get(str(agent_id or "").strip(), {}).get(
+            str(mailbox_type or "").strip().lower(),
+            "",
+        )
+        if isinstance(entry, dict):
+            return str(entry.get("token_hash", "") or "")
+        return str(entry or "")
+
+    def _mailbox_keys_for_claim(self, agent_id: str, claim: dict[str, Any]) -> list[str]:
+        claim_type = str(claim.get("type", "")).strip().lower()
+        if claim_type == "shared":
+            token = str(claim.get("token", "")).strip()
+            if not token:
+                metrics_inc("dm_claim_invalid")
+                return []
+            return [self._hashed_mailbox_token(token)]
+        if claim_type == "requests":
+            token = str(claim.get("token", "")).strip()
+            if token:
+                bound_key = self._remember_mailbox_binding(agent_id, "requests", token)
+                epoch = self._epoch_bucket()
+                return [
+                    bound_key,
+                    self._mailbox_key("requests", agent_id, epoch),
+                    self._mailbox_key("requests", agent_id, epoch - 1),
+                ]
+            metrics_inc("dm_claim_invalid")
+            return []
+        if claim_type == "self":
+            token = str(claim.get("token", "")).strip()
+            if token:
+                bound_key = self._remember_mailbox_binding(agent_id, "self", token)
+                epoch = self._epoch_bucket()
+                return [
+                    bound_key,
+                    self._mailbox_key("self", agent_id, epoch),
+                    self._mailbox_key("self", agent_id, epoch - 1),
+                ]
+            metrics_inc("dm_claim_invalid")
+            return []
+        metrics_inc("dm_claim_invalid")
+        return []
+
+    def mailbox_key_for_delivery(
+        self,
+        *,
+        recipient_id: str,
+        delivery_class: str,
+        recipient_token: str | None = None,
+    ) -> str:
+        delivery_class = str(delivery_class or "").strip().lower()
+        if delivery_class == "request":
+            bound_key = self._bound_mailbox_key(recipient_id, "requests")
+            if bound_key:
+                return bound_key
+            return self._mailbox_key("requests", str(recipient_id or "").strip())
+        if delivery_class == "shared":
+            token = str(recipient_token or "").strip()
+            if not token:
+                raise ValueError("recipient_token required for shared delivery")
+            return self._hashed_mailbox_token(token)
+        raise ValueError("Unsupported delivery_class")
+
+    def claim_mailbox_keys(self, agent_id: str, claims: list[dict[str, Any]]) -> list[str]:
+        keys: list[str] = []
+        for claim in claims[:32]:
+            keys.extend(self._mailbox_keys_for_claim(agent_id, claim))
+        return list(dict.fromkeys(keys))
+
+    def _legacy_mailbox_token(self, agent_id: str, epoch: int) -> str:
+        raw = f"sb_dm|{epoch}|{agent_id}".encode("utf-8")
+        return hashlib.sha256(raw).hexdigest()
+
+    def _legacy_token_candidates(self, agent_id: str) -> list[str]:
+        epoch = self._epoch_bucket()
+        raw = [self._legacy_mailbox_token(agent_id, epoch), self._legacy_mailbox_token(agent_id, epoch - 1)]
+        peppered = [self._pepper_token(token) for token in raw]
+        return list(dict.fromkeys(peppered + raw))
+
+    def _save(self) -> None:
+        """Mark dirty and schedule a coalesced disk write."""
+        self._dirty = True
+        if not RELAY_FILE.exists():
+            self._flush()
+            return
+        with self._lock:
+            if self._save_timer is None or not self._save_timer.is_alive():
+                self._save_timer = threading.Timer(self._SAVE_INTERVAL, self._flush)
+                self._save_timer.daemon = True
+                self._save_timer.start()
+
+    def _prune_stale_metadata(self) -> None:
+        """Remove expired DH keys, prekey bundles, and mailbox bindings."""
+        now = time.time()
+        settings = self._settings()
+        key_ttl = max(1, int(getattr(settings, "MESH_DM_KEY_TTL_DAYS", 30) or 30)) * 86400
+        binding_ttl = max(1, int(getattr(settings, "MESH_DM_BINDING_TTL_DAYS", 7) or 7)) * 86400
+
+        stale_keys = [
+            aid for aid, entry in self._dh_keys.items()
+            if (now - float(entry.get("timestamp", 0) or 0)) > key_ttl
+        ]
+        for aid in stale_keys:
+            del self._dh_keys[aid]
+
+        stale_bundles = [
+            aid for aid, entry in self._prekey_bundles.items()
+            if (now - float(entry.get("updated_at", entry.get("timestamp", 0)) or 0)) > key_ttl
+        ]
+        for aid in stale_bundles:
+            del self._prekey_bundles[aid]
+
+        stale_agents: list[str] = []
+        for agent_id, kinds in self._mailbox_bindings.items():
+            expired_kinds = [
+                k for k, v in kinds.items()
+                if isinstance(v, dict) and (now - float(v.get("last_used", 0) or 0)) > binding_ttl
+            ]
+            for k in expired_kinds:
+                del kinds[k]
+            if not kinds:
+                stale_agents.append(agent_id)
+        for agent_id in stale_agents:
+            del self._mailbox_bindings[agent_id]
+
+    def _metadata_persist_enabled(self) -> bool:
+        return bool(getattr(self._settings(), "MESH_DM_METADATA_PERSIST", True))
+
+    def _flush(self) -> None:
+        """Actually write to disk (called by timer or atexit)."""
+        if not self._dirty:
+            return
+        try:
+            self._prune_stale_metadata()
+            DATA_DIR.mkdir(parents=True, exist_ok=True)
+            payload: dict[str, Any] = {
+                "saved_at": int(time.time()),
+                "dh_keys": self._dh_keys,
+                "prekey_bundles": self._prekey_bundles,
+                "witnesses": self._witnesses,
+                "blocks": {k: sorted(v) for k, v in self._blocks.items()},
+                "nonce_cache": dict(self._nonce_cache),
+                "stats": self._stats,
+            }
+            if self._metadata_persist_enabled():
+                payload["mailbox_bindings"] = self._mailbox_bindings
+            if self._persist_spool_enabled():
+                payload["mailboxes"] = {
+                    key: [m.__dict__ for m in msgs] for key, msgs in self._mailboxes.items()
+                }
+            write_secure_json(RELAY_FILE, payload)
+            self._dirty = False
+        except Exception:
+            pass
+
+    def _load(self) -> None:
+        if not RELAY_FILE.exists():
+            return
+        try:
+            data = read_secure_json(RELAY_FILE, lambda: {})
+        except Exception:
+            return
+        if self._persist_spool_enabled():
+            mailboxes = data.get("mailboxes", {})
+            if isinstance(mailboxes, dict):
+                for key, items in mailboxes.items():
+                    if not isinstance(items, list):
+                        continue
+                    restored: list[DMMessage] = []
+                    for item in items:
+                        try:
+                            restored.append(
+                                DMMessage(
+                                    sender_id=str(item.get("sender_id", "")),
+                                    ciphertext=str(item.get("ciphertext", "")),
+                                    timestamp=float(item.get("timestamp", 0)),
+                                    msg_id=str(item.get("msg_id", "")),
+                                    delivery_class=str(item.get("delivery_class", "shared")),
+                                    sender_seal=str(item.get("sender_seal", "")),
+                                    relay_salt=str(item.get("relay_salt", "") or ""),
+                                    sender_block_ref=str(item.get("sender_block_ref", "") or ""),
+                                    payload_format=str(item.get("payload_format", item.get("format", "dm1")) or "dm1"),
+                                    session_welcome=str(item.get("session_welcome", "") or ""),
+                                )
+                            )
+                        except Exception:
+                            continue
+                    for message in restored:
+                        if not message.sender_block_ref:
+                            message.sender_block_ref = self._message_block_ref(message)
+                    if restored:
+                        self._mailboxes[key] = restored
+        dh_keys = data.get("dh_keys", {})
+        if isinstance(dh_keys, dict):
+            self._dh_keys = {str(k): dict(v) for k, v in dh_keys.items() if isinstance(v, dict)}
+        prekey_bundles = data.get("prekey_bundles", {})
+        if isinstance(prekey_bundles, dict):
+            self._prekey_bundles = {
+                str(k): dict(v) for k, v in prekey_bundles.items() if isinstance(v, dict)
+            }
+        mailbox_bindings = data.get("mailbox_bindings", {})
+        if isinstance(mailbox_bindings, dict):
+            self._mailbox_bindings = defaultdict(
+                dict,
+                {
+                    str(agent_id): {
+                        str(kind): str(token_hash)
+                        for kind, token_hash in dict(bindings or {}).items()
+                        if str(token_hash or "").strip()
+                    }
+                    for agent_id, bindings in mailbox_bindings.items()
+                    if isinstance(bindings, dict)
+                },
+            )
+        witnesses = data.get("witnesses", {})
+        if isinstance(witnesses, dict):
+            self._witnesses = defaultdict(
+                list,
+                {
+                    str(k): list(v)
+                    for k, v in witnesses.items()
+                    if isinstance(v, list)
+                },
+            )
+        blocks = data.get("blocks", {})
+        if isinstance(blocks, dict):
+            for key, values in blocks.items():
+                if isinstance(values, list):
+                    self._blocks[str(key)] = {
+                        self._canonical_blocked_id(str(v))
+                        for v in values
+                        if str(v or "").strip()
+                    }
+        nonce_cache = data.get("nonce_cache", {})
+        if isinstance(nonce_cache, dict):
+            now = time.time()
+            restored = sorted(
+                (
+                    (str(k), float(v))
+                    for k, v in nonce_cache.items()
+                    if float(v) > now
+                ),
+                key=lambda item: item[1],
+            )
+            self._nonce_cache = OrderedDict(restored)
+        stats = data.get("stats", {})
+        if isinstance(stats, dict):
+            self._stats = {str(k): int(v) for k, v in stats.items() if isinstance(v, (int, float))}
+        self._stats["messages_in_memory"] = sum(len(v) for v in self._mailboxes.values())
+
+    def _bundle_fingerprint(
+        self,
+        *,
+        dh_pub_key: str,
+        dh_algo: str,
+        public_key: str,
+        public_key_algo: str,
+        protocol_version: str,
+    ) -> str:
+        material = "|".join(
+            [
+                dh_pub_key,
+                dh_algo,
+                public_key,
+                public_key_algo,
+                protocol_version,
+            ]
+        )
+        return hashlib.sha256(material.encode("utf-8")).hexdigest()
+
+    def register_dh_key(
+        self,
+        agent_id: str,
+        dh_pub_key: str,
+        dh_algo: str,
+        timestamp: int,
+        signature: str,
+        public_key: str,
+        public_key_algo: str,
+        protocol_version: str,
+        sequence: int,
+    ) -> tuple[bool, str, dict[str, Any] | None]:
+        """Register/update an agent's DH public key bundle with replay protection."""
+        fingerprint = self._bundle_fingerprint(
+            dh_pub_key=dh_pub_key,
+            dh_algo=dh_algo,
+            public_key=public_key,
+            public_key_algo=public_key_algo,
+            protocol_version=protocol_version,
+        )
+        with self._lock:
+            existing = self._dh_keys.get(agent_id)
+            if existing:
+                existing_seq = int(existing.get("sequence", 0) or 0)
+                existing_ts = int(existing.get("timestamp", 0) or 0)
+                if sequence <= existing_seq:
+                    metrics_inc("dm_key_replay")
+                    return False, "DM key replay or rollback rejected", None
+                if timestamp < existing_ts:
+                    metrics_inc("dm_key_stale")
+                    return False, "DM key timestamp is older than the current bundle", None
+            self._dh_keys[agent_id] = {
+                "dh_pub_key": dh_pub_key,
+                "dh_algo": dh_algo,
+                "timestamp": timestamp,
+                "signature": signature,
+                "public_key": public_key,
+                "public_key_algo": public_key_algo,
+                "protocol_version": protocol_version,
+                "sequence": sequence,
+                "bundle_fingerprint": fingerprint,
+            }
+            self._save()
+        return True, "ok", {
+            "accepted_sequence": sequence,
+            "bundle_fingerprint": fingerprint,
+        }
+
+    def get_dh_key(self, agent_id: str) -> dict[str, Any] | None:
+        return self._dh_keys.get(agent_id)
+
+    def register_prekey_bundle(
+        self,
+        agent_id: str,
+        bundle: dict[str, Any],
+        signature: str,
+        public_key: str,
+        public_key_algo: str,
+        protocol_version: str,
+        sequence: int,
+    ) -> tuple[bool, str, dict[str, Any] | None]:
+        ok, reason = _validate_bundle_record(
+            {
+                "bundle": bundle,
+                "public_key": public_key,
+                "agent_id": agent_id,
+            }
+        )
+        if not ok:
+            return False, reason, None
+        with self._lock:
+            existing = self._prekey_bundles.get(agent_id)
+            if existing:
+                existing_seq = int(existing.get("sequence", 0) or 0)
+                if sequence <= existing_seq:
+                    return False, "Prekey bundle replay or rollback rejected", None
+            stored = {
+                "bundle": dict(bundle or {}),
+                "signature": signature,
+                "public_key": public_key,
+                "public_key_algo": public_key_algo,
+                "protocol_version": protocol_version,
+                "sequence": int(sequence),
+                "updated_at": int(time.time()),
+            }
+            self._prekey_bundles[agent_id] = stored
+        self._save()
+        return True, "ok", {"accepted_sequence": int(sequence)}
+
+    def get_prekey_bundle(self, agent_id: str) -> dict[str, Any] | None:
+        stored = self._prekey_bundles.get(agent_id)
+        if not stored:
+            return None
+        return dict(stored)
+
+    def consume_one_time_prekey(self, agent_id: str) -> dict[str, Any] | None:
+        """Atomically claim the next published one-time prekey for a peer bundle."""
+        claimed: dict[str, Any] | None = None
+        with self._lock:
+            stored = self._prekey_bundles.get(agent_id)
+            if not stored:
+                return None
+            bundle = dict(stored.get("bundle") or {})
+            otks = list(bundle.get("one_time_prekeys") or [])
+            if not otks:
+                return dict(stored)
+            claimed = dict(otks.pop(0) or {})
+            bundle["one_time_prekeys"] = otks
+            bundle["one_time_prekey_count"] = len(otks)
+            stored = dict(stored)
+            stored["bundle"] = bundle
+            stored["updated_at"] = int(time.time())
+            self._prekey_bundles[agent_id] = stored
+        self._save()
+        result = dict(stored)
+        result["claimed_one_time_prekey"] = claimed
+        return result
+
+    def _prune_witnesses(self, target_id: str, ttl_days: int = 30) -> None:
+        cutoff = time.time() - (ttl_days * 86400)
+        self._witnesses[target_id] = [
+            w for w in self._witnesses.get(target_id, []) if float(w.get("timestamp", 0)) >= cutoff
+        ]
+        if not self._witnesses[target_id]:
+            del self._witnesses[target_id]
+
+    def record_witness(
+        self,
+        witness_id: str,
+        target_id: str,
+        dh_pub_key: str,
+        timestamp: int,
+    ) -> tuple[bool, str]:
+        if not witness_id or not target_id or not dh_pub_key:
+            return False, "Missing witness_id, target_id, or dh_pub_key"
+        if witness_id == target_id:
+            return False, "Cannot witness yourself"
+        with self._lock:
+            self._prune_witnesses(target_id)
+            entries = self._witnesses.get(target_id, [])
+            for entry in entries:
+                if entry.get("witness_id") == witness_id and entry.get("dh_pub_key") == dh_pub_key:
+                    return False, "Duplicate witness"
+            entries.append(
+                {
+                    "witness_id": witness_id,
+                    "dh_pub_key": dh_pub_key,
+                    "timestamp": int(timestamp),
+                }
+            )
+            self._witnesses[target_id] = entries[-50:]
+            self._save()
+        return True, "ok"
+
+    def get_witnesses(self, target_id: str, dh_pub_key: str | None = None, limit: int = 5) -> list[dict]:
+        with self._lock:
+            self._prune_witnesses(target_id)
+            entries = list(self._witnesses.get(target_id, []))
+        if dh_pub_key:
+            entries = [e for e in entries if e.get("dh_pub_key") == dh_pub_key]
+        entries = sorted(entries, key=lambda e: e.get("timestamp", 0), reverse=True)
+        return entries[: max(1, limit)]
+
+    def _epoch_bucket(self, ts: float | None = None) -> int:
+        now = ts if ts is not None else time.time()
+        return int(now // EPOCH_SECONDS)
+
+    def _mailbox_limit_for_class(self, delivery_class: str) -> int:
+        if delivery_class == "request":
+            return self._request_mailbox_limit()
+        if delivery_class == "shared":
+            return self._shared_mailbox_limit()
+        return self._self_mailbox_limit()
+
+    def _cleanup_expired(self) -> bool:
+        now = time.time()
+        changed = False
+        for mailbox_id in list(self._mailboxes):
+            fresh = [m for m in self._mailboxes[mailbox_id] if now - m.timestamp < TTL_SECONDS]
+            if len(fresh) != len(self._mailboxes[mailbox_id]):
+                changed = True
+            self._mailboxes[mailbox_id] = fresh
+            if not self._mailboxes[mailbox_id]:
+                del self._mailboxes[mailbox_id]
+                changed = True
+        self._stats["messages_in_memory"] = sum(len(v) for v in self._mailboxes.values())
+        return changed
+
+    def consume_nonce(self, agent_id: str, nonce: str, timestamp: int) -> tuple[bool, str]:
+        nonce = str(nonce or "").strip()
+        if not nonce:
+            return False, "Missing nonce"
+        now = time.time()
+        with self._lock:
+            self._nonce_cache = OrderedDict(
+                (key, expiry)
+                for key, expiry in self._nonce_cache.items()
+                if float(expiry) > now
+            )
+            key = f"{agent_id}:{nonce}"
+            if key in self._nonce_cache:
+                metrics_inc("dm_nonce_replay")
+                return False, "nonce replay detected"
+            if len(self._nonce_cache) >= self._nonce_cache_max_entries():
+                metrics_inc("dm_nonce_cache_full")
+                return False, "nonce cache at capacity"
+            expiry = max(now + self._nonce_ttl_seconds(), float(timestamp) + self._nonce_ttl_seconds())
+            self._nonce_cache[key] = expiry
+            self._nonce_cache.move_to_end(key)
+            self._save()
+        return True, "ok"
+
+    def deposit(
+        self,
+        *,
+        sender_id: str,
+        raw_sender_id: str = "",
+        recipient_id: str = "",
+        ciphertext: str,
+        msg_id: str = "",
+        delivery_class: str,
+        recipient_token: str | None = None,
+        sender_seal: str = "",
+        relay_salt: str = "",
+        sender_token_hash: str = "",
+        payload_format: str = "dm1",
+        session_welcome: str = "",
+    ) -> dict[str, Any]:
+        with self._lock:
+            authority_sender = str(raw_sender_id or sender_id or "").strip()
+            sender_block_ref = self._sender_block_ref(authority_sender)
+            if recipient_id and sender_block_ref in self._blocks.get(recipient_id, set()):
+                metrics_inc("dm_drop_blocked")
+                return {"ok": False, "detail": "Recipient is not accepting your messages"}
+            if len(ciphertext) > int(self._settings().MESH_DM_MAX_MSG_BYTES):
+                metrics_inc("dm_drop_oversize")
+                return {
+                    "ok": False,
+                    "detail": f"Message too large ({len(ciphertext)} > {int(self._settings().MESH_DM_MAX_MSG_BYTES)})",
+                }
+            self._cleanup_expired()
+            if delivery_class == "request":
+                mailbox_key = self._mailbox_key("requests", recipient_id)
+            elif delivery_class == "shared":
+                if not recipient_token:
+                    metrics_inc("dm_claim_invalid")
+                    return {"ok": False, "detail": "recipient_token required for shared delivery"}
+                mailbox_key = self._hashed_mailbox_token(recipient_token)
+            else:
+                return {"ok": False, "detail": "Unsupported delivery_class"}
+            if len(self._mailboxes[mailbox_key]) >= self._mailbox_limit_for_class(delivery_class):
+                metrics_inc("dm_drop_full")
+                return {"ok": False, "detail": "Recipient mailbox full"}
+            if not msg_id:
+                msg_id = f"dm_{int(time.time() * 1000)}_{secrets.token_hex(6)}"
+            elif any(m.msg_id == msg_id for m in self._mailboxes[mailbox_key]):
+                return {"ok": True, "msg_id": msg_id}
+            relay_sender_id = (
+                f"sender_token:{sender_token_hash}"
+                if sender_token_hash and delivery_class == "shared"
+                else sender_id
+            )
+            self._mailboxes[mailbox_key].append(
+                DMMessage(
+                    sender_id=relay_sender_id,
+                    ciphertext=ciphertext,
+                    timestamp=time.time(),
+                    msg_id=msg_id,
+                    delivery_class=delivery_class,
+                    sender_seal=sender_seal,
+                    sender_block_ref=sender_block_ref,
+                    payload_format=str(payload_format or "dm1"),
+                    session_welcome=str(session_welcome or ""),
+                )
+            )
+            self._stats["messages_in_memory"] = sum(len(v) for v in self._mailboxes.values())
+            self._save()
+            return {"ok": True, "msg_id": msg_id}
+
+    def is_blocked(self, recipient_id: str, sender_id: str) -> bool:
+        with self._lock:
+            blocked_ref = self._sender_block_ref(sender_id)
+            if not recipient_id or not blocked_ref:
+                return False
+            return blocked_ref in self._blocks.get(recipient_id, set())
+
+    def _collect_from_keys(self, keys: list[str], *, destructive: bool) -> list[dict[str, Any]]:
+        messages: list[DMMessage] = []
+        seen: set[str] = set()
+        for key in keys:
+            mailbox = self._mailboxes.pop(key, []) if destructive else list(self._mailboxes.get(key, []))
+            for message in mailbox:
+                if message.msg_id in seen:
+                    continue
+                seen.add(message.msg_id)
+                messages.append(message)
+        if destructive:
+            self._stats["messages_in_memory"] = sum(len(v) for v in self._mailboxes.values())
+            self._save()
+        return [
+            {
+                "sender_id": message.sender_id,
+                "ciphertext": message.ciphertext,
+                "timestamp": message.timestamp,
+                "msg_id": message.msg_id,
+                "delivery_class": message.delivery_class,
+                "sender_seal": message.sender_seal,
+                "format": message.payload_format,
+                "session_welcome": message.session_welcome,
+            }
+            for message in sorted(messages, key=lambda item: item.timestamp)
+        ]
+
+    def collect_claims(self, agent_id: str, claims: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        with self._lock:
+            self._cleanup_expired()
+            keys: list[str] = []
+            for claim in claims[:32]:
+                keys.extend(self._mailbox_keys_for_claim(agent_id, claim))
+            return self._collect_from_keys(list(dict.fromkeys(keys)), destructive=True)
+
+    def count_claims(self, agent_id: str, claims: list[dict[str, Any]]) -> int:
+        with self._lock:
+            self._cleanup_expired()
+            keys: list[str] = []
+            for claim in claims[:32]:
+                keys.extend(self._mailbox_keys_for_claim(agent_id, claim))
+            messages = self._collect_from_keys(list(dict.fromkeys(keys)), destructive=False)
+            return len(messages)
+
+    def claim_message_ids(self, agent_id: str, claims: list[dict[str, Any]]) -> set[str]:
+        with self._lock:
+            self._cleanup_expired()
+            keys: list[str] = []
+            for claim in claims[:32]:
+                keys.extend(self._mailbox_keys_for_claim(agent_id, claim))
+            messages = self._collect_from_keys(list(dict.fromkeys(keys)), destructive=False)
+            return {
+                str(message.get("msg_id", "") or "")
+                for message in messages
+                if str(message.get("msg_id", "") or "")
+            }
+
+    def collect_legacy(self, agent_id: str | None = None, agent_token: str | None = None) -> list[dict[str, Any]]:
+        with self._lock:
+            self._cleanup_expired()
+            if not agent_token:
+                return []
+            keys = [self._pepper_token(agent_token), agent_token]
+            return self._collect_from_keys(list(dict.fromkeys(keys)), destructive=True)
+
+    def count_legacy(self, agent_id: str | None = None, agent_token: str | None = None) -> int:
+        with self._lock:
+            self._cleanup_expired()
+            if not agent_token:
+                return 0
+            keys = [self._pepper_token(agent_token), agent_token]
+            return len(self._collect_from_keys(list(dict.fromkeys(keys)), destructive=False))
+
+    def block(self, agent_id: str, blocked_id: str) -> None:
+        with self._lock:
+            blocked_ref = self._canonical_blocked_id(blocked_id)
+            if not blocked_ref:
+                return
+            self._blocks[agent_id].add(blocked_ref)
+            purge_keys = self._legacy_token_candidates(agent_id)
+            bound_request = self._bound_mailbox_key(agent_id, "requests")
+            bound_self = self._bound_mailbox_key(agent_id, "self")
+            if bound_request:
+                purge_keys.append(bound_request)
+            if bound_self:
+                purge_keys.append(bound_self)
+            purge_keys.extend(
+                [
+                    self._mailbox_key("self", agent_id),
+                    self._mailbox_key("requests", agent_id),
+                    self._mailbox_key("self", agent_id, self._epoch_bucket() - 1),
+                    self._mailbox_key("requests", agent_id, self._epoch_bucket() - 1),
+                ]
+            )
+            for key in set(purge_keys):
+                if key in self._mailboxes:
+                    self._mailboxes[key] = [
+                        m for m in self._mailboxes[key] if self._message_block_ref(m) != blocked_ref
+                    ]
+            self._stats["messages_in_memory"] = sum(len(v) for v in self._mailboxes.values())
+            self._save()
+
+    def unblock(self, agent_id: str, blocked_id: str) -> None:
+        with self._lock:
+            blocked_ref = self._canonical_blocked_id(blocked_id)
+            if not blocked_ref:
+                return
+            self._blocks[agent_id].discard(blocked_ref)
+            self._save()
+
+
+dm_relay = DMRelay()

backend/services/mesh/mesh_ibf.py

@@ -0,0 +1,186 @@
+from __future__ import annotations
+
+import base64
+import hashlib
+from dataclasses import dataclass
+from typing import Iterable, List, Tuple
+
+KEY_SIZE = 32
+DEFAULT_SEEDS = [0x243F6A8885A308D3, 0x13198A2E03707344, 0xA4093822299F31D0]
+FINGERPRINT_SEED = 0xC0FFEE1234567890
+
+
+def _safe_int(val, default=0) -> int:
+    try:
+        return int(val)
+    except (TypeError, ValueError):
+        return default
+
+
+def _hash64(data: bytes, seed: int) -> int:
+    key = seed.to_bytes(8, "little", signed=False)
+    digest = hashlib.blake2b(data, digest_size=8, key=key).digest()
+    return int.from_bytes(digest, "little", signed=False)
+
+
+def _fingerprint(data: bytes) -> int:
+    key = FINGERPRINT_SEED.to_bytes(8, "little", signed=False)
+    digest = hashlib.blake2b(data, digest_size=8, key=key).digest()
+    return int.from_bytes(digest, "little", signed=False)
+
+
+def _xor_bytes(a: bytes, b: bytes) -> bytes:
+    return bytes(x ^ y for x, y in zip(a, b))
+
+
+def _ensure_key(key: bytes) -> bytes:
+    if len(key) != KEY_SIZE:
+        raise ValueError(f"IBF key must be {KEY_SIZE} bytes")
+    return key
+
+
+def _b64_encode(data: bytes) -> str:
+    return base64.b64encode(data).decode("ascii")
+
+
+def _b64_decode(data: str) -> bytes:
+    return base64.b64decode(data.encode("ascii"))
+
+
+@dataclass
+class IBLTCell:
+    count: int = 0
+    key_xor: bytes = b"\x00" * KEY_SIZE
+    hash_xor: int = 0
+
+    def add(self, key: bytes, sign: int) -> None:
+        self.count += sign
+        self.key_xor = _xor_bytes(self.key_xor, key)
+        self.hash_xor ^= _fingerprint(key)
+
+
+class IBLT:
+    def __init__(self, size: int, seeds: List[int] | None = None) -> None:
+        if size <= 0:
+            raise ValueError("IBLT size must be positive")
+        self.size = size
+        self.seeds = seeds or list(DEFAULT_SEEDS)
+        self.cells: List[IBLTCell] = [IBLTCell() for _ in range(size)]
+
+    def _indexes(self, key: bytes) -> List[int]:
+        key = _ensure_key(key)
+        return [(_hash64(key, seed) % self.size) for seed in self.seeds]
+
+    def insert(self, key: bytes) -> None:
+        key = _ensure_key(key)
+        for idx in self._indexes(key):
+            self.cells[idx].add(key, 1)
+
+    def delete(self, key: bytes) -> None:
+        key = _ensure_key(key)
+        for idx in self._indexes(key):
+            self.cells[idx].add(key, -1)
+
+    def subtract(self, other: "IBLT") -> "IBLT":
+        if self.size != other.size or self.seeds != other.seeds:
+            raise ValueError("IBLT mismatch; size or seeds differ")
+        out = IBLT(self.size, self.seeds)
+        for i, cell in enumerate(self.cells):
+            other_cell = other.cells[i]
+            out.cells[i] = IBLTCell(
+                count=cell.count - other_cell.count,
+                key_xor=_xor_bytes(cell.key_xor, other_cell.key_xor),
+                hash_xor=cell.hash_xor ^ other_cell.hash_xor,
+            )
+        return out
+
+    def decode(self) -> Tuple[bool, List[bytes], List[bytes]]:
+        plus: List[bytes] = []
+        minus: List[bytes] = []
+        stack = [i for i, c in enumerate(self.cells) if abs(c.count) == 1]
+
+        while stack:
+            idx = stack.pop()
+            cell = self.cells[idx]
+            if abs(cell.count) != 1:
+                continue
+            key = cell.key_xor
+            if _fingerprint(key) != cell.hash_xor:
+                continue
+            sign = 1 if cell.count == 1 else -1
+            if sign == 1:
+                plus.append(key)
+            else:
+                minus.append(key)
+            for j in self._indexes(key):
+                if j == idx:
+                    continue
+                self.cells[j].add(key, -sign)
+                if abs(self.cells[j].count) == 1:
+                    stack.append(j)
+            self.cells[idx] = IBLTCell()
+
+        success = all(
+            c.count == 0 and c.hash_xor == 0 and c.key_xor == b"\x00" * KEY_SIZE
+            for c in self.cells
+        )
+        return success, plus, minus
+
+    def to_compact_dict(self) -> dict:
+        return {
+            "m": self.size,
+            "s": self.seeds,
+            "c": [[cell.count, _b64_encode(cell.key_xor), cell.hash_xor] for cell in self.cells],
+        }
+
+    @classmethod
+    def from_compact_dict(cls, data: dict) -> "IBLT":
+        size = _safe_int(data.get("m", 0) or 0)
+        seeds = data.get("s") or list(DEFAULT_SEEDS)
+        cells = data.get("c") or []
+        iblt = cls(size, list(seeds))
+        if len(cells) != size:
+            raise ValueError("IBLT cell count mismatch")
+        for i, raw in enumerate(cells):
+            count, key_b64, hash_xor = raw
+            iblt.cells[i] = IBLTCell(
+                count=_safe_int(count, 0),
+                key_xor=_b64_decode(str(key_b64)),
+                hash_xor=_safe_int(hash_xor, 0),
+            )
+        return iblt
+
+
+def build_iblt(keys: Iterable[bytes], size: int) -> IBLT:
+    iblt = IBLT(size)
+    for key in keys:
+        iblt.insert(key)
+    return iblt
+
+
+def minhash_sketch(keys: Iterable[bytes], k: int) -> List[int]:
+    if k <= 0:
+        return []
+    mins: List[int] = []
+    for key in keys:
+        h = _hash64(key, 0x9E3779B97F4A7C15)
+        if len(mins) < k:
+            mins.append(h)
+            mins.sort()
+        elif h < mins[-1]:
+            mins[-1] = h
+            mins.sort()
+    return mins
+
+
+def minhash_similarity(a: Iterable[int], b: Iterable[int]) -> float:
+    a_list = list(a)
+    b_list = list(b)
+    if not a_list or not b_list:
+        return 0.0
+    k = min(len(a_list), len(b_list))
+    if k <= 0:
+        return 0.0
+    a_set = set(a_list[:k])
+    b_set = set(b_list[:k])
+    return len(a_set & b_set) / float(k)

backend/services/mesh/mesh_infonet_sync_support.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import time
+from dataclasses import asdict, dataclass
+
+from services.mesh.mesh_peer_store import PeerRecord
+
+
+@dataclass(frozen=True)
+class SyncWorkerState:
+    last_sync_started_at: int = 0
+    last_sync_finished_at: int = 0
+    last_sync_ok_at: int = 0
+    next_sync_due_at: int = 0
+    last_peer_url: str = ""
+    last_error: str = ""
+    last_outcome: str = "idle"
+    current_head: str = ""
+    fork_detected: bool = False
+    consecutive_failures: int = 0
+
+    def to_dict(self) -> dict[str, object]:
+        return asdict(self)
+
+
+def eligible_sync_peers(records: list[PeerRecord], *, now: float | None = None) -> list[PeerRecord]:
+    current_time = int(now if now is not None else time.time())
+    candidates = [
+        record
+        for record in records
+        if record.bucket == "sync" and record.enabled and int(record.cooldown_until or 0) <= current_time
+    ]
+    return sorted(
+        candidates,
+        key=lambda record: (
+            -int(record.last_sync_ok_at or 0),
+            int(record.failure_count or 0),
+            int(record.added_at or 0),
+            record.peer_url,
+        ),
+    )
+
+
+def begin_sync(
+    state: SyncWorkerState,
+    *,
+    peer_url: str = "",
+    current_head: str = "",
+    now: float | None = None,
+) -> SyncWorkerState:
+    timestamp = int(now if now is not None else time.time())
+    return SyncWorkerState(
+        last_sync_started_at=timestamp,
+        last_sync_finished_at=state.last_sync_finished_at,
+        last_sync_ok_at=state.last_sync_ok_at,
+        next_sync_due_at=state.next_sync_due_at,
+        last_peer_url=peer_url or state.last_peer_url,
+        last_error="",
+        last_outcome="running",
+        current_head=current_head or state.current_head,
+        fork_detected=False,
+        consecutive_failures=state.consecutive_failures,
+    )
+
+
+def finish_sync(
+    state: SyncWorkerState,
+    *,
+    ok: bool,
+    peer_url: str = "",
+    current_head: str = "",
+    error: str = "",
+    fork_detected: bool = False,
+    now: float | None = None,
+    interval_s: int = 300,
+    failure_backoff_s: int = 60,
+) -> SyncWorkerState:
+    timestamp = int(now if now is not None else time.time())
+    if ok:
+        return SyncWorkerState(
+            last_sync_started_at=state.last_sync_started_at,
+            last_sync_finished_at=timestamp,
+            last_sync_ok_at=timestamp,
+            next_sync_due_at=timestamp + max(0, int(interval_s or 0)),
+            last_peer_url=peer_url or state.last_peer_url,
+            last_error="",
+            last_outcome="ok",
+            current_head=current_head or state.current_head,
+            fork_detected=bool(fork_detected),
+            consecutive_failures=0,
+        )
+
+    return SyncWorkerState(
+        last_sync_started_at=state.last_sync_started_at,
+        last_sync_finished_at=timestamp,
+        last_sync_ok_at=state.last_sync_ok_at,
+        next_sync_due_at=timestamp + max(0, int(failure_backoff_s or 0)),
+        last_peer_url=peer_url or state.last_peer_url,
+        last_error=str(error or "").strip(),
+        last_outcome="fork" if fork_detected else "error",
+        current_head=current_head or state.current_head,
+        fork_detected=bool(fork_detected),
+        consecutive_failures=state.consecutive_failures + 1,
+    )
+
+
+def should_run_sync(
+    state: SyncWorkerState,
+    *,
+    now: float | None = None,
+) -> bool:
+    current_time = int(now if now is not None else time.time())
+    if state.last_outcome == "running":
+        return False
+    return int(state.next_sync_due_at or 0) <= current_time

backend/services/mesh/mesh_merkle.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import hashlib
+from typing import Any
+
+
+def _hash_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def hash_leaf(value: str) -> str:
+    return _hash_bytes(value.encode("utf-8"))
+
+
+def hash_pair(left: str, right: str) -> str:
+    return _hash_bytes(f"{left}{right}".encode("utf-8"))
+
+
+def build_merkle_levels(leaves: list[str]) -> list[list[str]]:
+    if not leaves:
+        return []
+    level = [hash_leaf(leaf) for leaf in leaves]
+    levels = [level]
+    while len(level) > 1:
+        next_level: list[str] = []
+        for idx in range(0, len(level), 2):
+            left = level[idx]
+            right = level[idx + 1] if idx + 1 < len(level) else left
+            next_level.append(hash_pair(left, right))
+        level = next_level
+        levels.append(level)
+    return levels
+
+
+def merkle_root(leaves: list[str]) -> str:
+    levels = build_merkle_levels(leaves)
+    if not levels:
+        return ""
+    return levels[-1][0]
+
+
+def merkle_proof_from_levels(levels: list[list[str]], index: int) -> list[dict[str, Any]]:
+    if not levels:
+        return []
+    if index < 0 or index >= len(levels[0]):
+        return []
+    proof: list[dict[str, Any]] = []
+    idx = index
+    for level in levels[:-1]:
+        is_right = idx % 2 == 1
+        sibling_idx = idx - 1 if is_right else idx + 1
+        if sibling_idx >= len(level):
+            sibling_hash = level[idx]
+        else:
+            sibling_hash = level[sibling_idx]
+        proof.append({"hash": sibling_hash, "side": "left" if is_right else "right"})
+        idx //= 2
+    return proof
+
+
+def verify_merkle_proof(
+    leaf_value: str, index: int, proof: list[dict[str, Any]], root: str
+) -> bool:
+    current = hash_leaf(leaf_value)
+    idx = index
+    for step in proof:
+        sibling = str(step.get("hash", ""))
+        side = str(step.get("side", "right")).lower()
+        if side == "left":
+            current = hash_pair(sibling, current)
+        else:
+            current = hash_pair(current, sibling)
+        idx //= 2
+    return current == root

backend/services/mesh/mesh_metrics.py

@@ -0,0 +1,25 @@
+"""Lightweight metrics for mesh protocol health signals."""
+
+from __future__ import annotations
+
+import threading
+import time
+
+_lock = threading.Lock()
+_metrics: dict[str, int] = {}
+_last_updated: float = 0.0
+
+
+def increment(name: str, count: int = 1) -> None:
+    global _last_updated
+    with _lock:
+        _metrics[name] = _metrics.get(name, 0) + count
+        _last_updated = time.time()
+
+
+def snapshot() -> dict:
+    with _lock:
+        return {
+            "updated_at": _last_updated,
+            "counters": dict(_metrics),
+        }

backend/services/mesh/mesh_oracle.py

@@ -0,0 +1,899 @@
+"""Oracle System — prediction-backed truth arbitration for the mesh.
+
+Oracle Rep is a separate reputation tier earned ONLY by:
+  1. Correctly predicting outcomes on Kalshi/Polymarket-sourced markets
+  2. Winning truth stakes on posts/comments
+
+Oracle Rep can be staked on posts to protect them from mob downvoting.
+Other oracles can counter-stake. After the stake period (1-7 days),
+whichever side has more oracle rep staked wins. Losers' rep is divided
+proportionally among winners.
+
+Scoring formula for predictions:
+  oracle_rep_earned = 1.0 - probability_of_chosen_outcome / 100
+  - Bet YES at 99% → earn 0.01 (trivial, everyone knew)
+  - Bet YES at 50% → earn 0.50 (genuine uncertainty, real insight)
+  - Bet YES at 10% → earn 0.90 (contrarian genius if correct)
+
+Designed for AI game theory: this mechanism works identically
+whether participants are humans, AI agents, or a mix.
+
+Persistence: JSON files in backend/data/ (auto-saved on change).
+"""
+
+import json
+import time
+import logging
+import secrets
+import threading
+import atexit
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger("services.mesh_oracle")
+
+DATA_DIR = Path(__file__).resolve().parents[2] / "data"
+ORACLE_FILE = DATA_DIR / "oracle_ledger.json"
+
+# ─── Constants ────────────────────────────────────────────────────────────
+
+MIN_STAKE_DAYS = 1  # Minimum stake duration
+MAX_STAKE_DAYS = 7  # Maximum stake duration
+GRACE_PERIOD_HOURS = 24  # Counter-stakers get 24h after any new stake
+ORACLE_DECAY_DAYS = 90  # Oracle rep decays over 90 days like regular rep
+
+
+class OracleLedger:
+    """Oracle reputation ledger — predictions, stakes, and truth arbitration.
+
+    Storage:
+      oracle_rep:   {node_id: float}           — current oracle rep balances
+      predictions:  [{node_id, market_title, side, probability_at_bet, timestamp, resolved, correct, rep_earned}]
+      stakes:       [{stake_id, message_id, poster_id, staker_id, side ("truth"|"false"),
+                      amount, duration_days, created_at, expires_at, resolved}]
+      prediction_log: [{node_id, market_title, side, probability_at_bet, rep_earned, timestamp}]
+    """
+
+    def __init__(self):
+        self.oracle_rep: dict[str, float] = {}
+        self.predictions: list[dict] = []
+        self.market_stakes: list[dict] = []  # Rep staked on prediction markets
+        self.stakes: list[dict] = []  # Truth stakes on posts (separate system)
+        self.prediction_log: list[dict] = []  # Public log of all predictions
+        self._dirty = False
+        self._save_lock = threading.Lock()
+        self._save_timer: threading.Timer | None = None
+        self._SAVE_INTERVAL = 5.0
+        atexit.register(self._flush)
+        self._load()
+
+    # ─── Persistence ──────────────────────────────────────────────────
+
+    def _load(self):
+        if ORACLE_FILE.exists():
+            try:
+                data = json.loads(ORACLE_FILE.read_text(encoding="utf-8"))
+                self.oracle_rep = data.get("oracle_rep", {})
+                self.predictions = data.get("predictions", [])
+                self.market_stakes = data.get("market_stakes", [])
+                self.stakes = data.get("stakes", [])
+                self.prediction_log = data.get("prediction_log", [])
+                logger.info(
+                    f"Loaded oracle ledger: {len(self.oracle_rep)} oracles, "
+                    f"{len(self.predictions)} predictions, "
+                    f"{len(self.market_stakes)} market stakes, {len(self.stakes)} truth stakes"
+                )
+            except Exception as e:
+                logger.error(f"Failed to load oracle ledger: {e}")
+
+    def _save(self):
+        """Mark dirty and schedule a coalesced disk write."""
+        self._dirty = True
+        with self._save_lock:
+            if self._save_timer is None or not self._save_timer.is_alive():
+                self._save_timer = threading.Timer(self._SAVE_INTERVAL, self._flush)
+                self._save_timer.daemon = True
+                self._save_timer.start()
+
+    def _flush(self):
+        """Actually write to disk (called by timer or atexit)."""
+        if not self._dirty:
+            return
+        try:
+            DATA_DIR.mkdir(parents=True, exist_ok=True)
+            data = {
+                "oracle_rep": self.oracle_rep,
+                "predictions": self.predictions,
+                "market_stakes": self.market_stakes,
+                "stakes": self.stakes,
+                "prediction_log": self.prediction_log,
+            }
+            ORACLE_FILE.write_text(json.dumps(data, indent=2), encoding="utf-8")
+            self._dirty = False
+        except Exception as e:
+            logger.error(f"Failed to save oracle ledger: {e}")
+
+    # ─── Oracle Rep ───────────────────────────────────────────────────
+
+    def get_oracle_rep(self, node_id: str) -> float:
+        """Get current oracle rep for a node (excludes locked/staked amount)."""
+        total = self.oracle_rep.get(node_id, 0.0)
+        # Subtract locked truth stakes on posts
+        locked = sum(
+            s["amount"]
+            for s in self.stakes
+            if s["staker_id"] == node_id and not s.get("resolved", False)
+        )
+        # Subtract locked market stakes
+        locked += sum(
+            s["amount"]
+            for s in self.market_stakes
+            if s["node_id"] == node_id and not s.get("resolved", False)
+        )
+        return round(max(0, total - locked), 3)
+
+    def get_total_oracle_rep(self, node_id: str) -> float:
+        """Get total oracle rep including locked stakes."""
+        return round(self.oracle_rep.get(node_id, 0.0), 3)
+
+    def _add_oracle_rep(self, node_id: str, amount: float):
+        """Add oracle rep to a node."""
+        self.oracle_rep[node_id] = self.oracle_rep.get(node_id, 0.0) + amount
+
+    def _remove_oracle_rep(self, node_id: str, amount: float):
+        """Remove oracle rep from a node (floor at 0)."""
+        self.oracle_rep[node_id] = max(0, self.oracle_rep.get(node_id, 0.0) - amount)
+
+    # ─── Predictions ──────────────────────────────────────────────────
+
+    def place_prediction(
+        self, node_id: str, market_title: str, side: str, probability_at_bet: float
+    ) -> tuple[bool, str]:
+        """Place a FREE prediction on a market outcome (no rep risked).
+
+        Args:
+            node_id: Predictor's node ID
+            market_title: Title of the prediction market
+            side: "yes", "no", or any outcome name for multi-outcome markets
+            probability_at_bet: Current probability (0-100) of the chosen side
+
+        Returns (success, detail)
+        """
+        if not side or not side.strip():
+            return False, "Side is required"
+
+        if not (0 <= probability_at_bet <= 100):
+            return False, "Probability must be 0-100"
+
+        # Check for duplicate predictions on same market
+        existing = [
+            p
+            for p in self.predictions
+            if p["node_id"] == node_id
+            and p["market_title"] == market_title
+            and not p.get("resolved", False)
+        ]
+        if existing:
+            return (
+                False,
+                f"You already have an active prediction on '{market_title}'. Your decision was FINAL.",
+            )
+
+        # Also check market stakes — can't free-pick AND stake on same market
+        existing_stake = [
+            s
+            for s in self.market_stakes
+            if s["node_id"] == node_id
+            and s["market_title"] == market_title
+            and not s.get("resolved", False)
+        ]
+        if existing_stake:
+            return (
+                False,
+                f"You already have a STAKED prediction on '{market_title}'. Your decision was FINAL.",
+            )
+
+        self.predictions.append(
+            {
+                "prediction_id": secrets.token_hex(6),
+                "node_id": node_id,
+                "market_title": market_title,
+                "side": side,
+                "probability_at_bet": probability_at_bet,
+                "timestamp": time.time(),
+                "resolved": False,
+                "correct": None,
+                "rep_earned": 0.0,
+            }
+        )
+        self._save()
+
+        # Potential rep = contrarianism score
+        potential = round(1.0 - probability_at_bet / 100, 3)
+
+        logger.info(
+            f"FREE prediction: {node_id} picks '{side}' on '{market_title}' "
+            f"at {probability_at_bet}% (potential: {potential} oracle rep)"
+        )
+        return True, (
+            f"FREE PICK placed: {side.upper()} on '{market_title}' "
+            f"at {probability_at_bet}%. Potential oracle rep: {potential}. "
+            f"This decision is FINAL."
+        )
+
+    def resolve_market(self, market_title: str, outcome: str) -> tuple[int, int]:
+        """Resolve all FREE predictions on a market.
+
+        Args:
+            market_title: Title of the market
+            outcome: "yes", "no", or any outcome name for multi-outcome markets
+
+        Returns (winners, losers) counts
+        """
+        if not outcome:
+            return 0, 0
+
+        outcome_lower = outcome.lower()
+        winners, losers = 0, 0
+        now = time.time()
+
+        for p in self.predictions:
+            if p["market_title"] != market_title or p.get("resolved", False):
+                continue
+
+            p["resolved"] = True
+            correct = p["side"].lower() == outcome_lower
+            p["correct"] = correct
+
+            if correct:
+                # Rep earned = contrarianism score
+                rep = round(1.0 - p["probability_at_bet"] / 100, 3)
+                rep = max(0.01, rep)  # Minimum 0.01 even for easy bets
+                p["rep_earned"] = rep
+                self._add_oracle_rep(p["node_id"], rep)
+                winners += 1
+
+                self.prediction_log.append(
+                    {
+                        "node_id": p["node_id"],
+                        "market_title": market_title,
+                        "side": p["side"],
+                        "outcome": outcome,
+                        "probability_at_bet": p["probability_at_bet"],
+                        "rep_earned": rep,
+                        "timestamp": p["timestamp"],
+                        "resolved_at": now,
+                    }
+                )
+                logger.info(
+                    f"Oracle win: {p['node_id']} earned {rep} oracle rep "
+                    f"on '{market_title}' ({p['side']} at {p['probability_at_bet']}%)"
+                )
+            else:
+                p["rep_earned"] = 0.0
+                losers += 1
+
+                self.prediction_log.append(
+                    {
+                        "node_id": p["node_id"],
+                        "market_title": market_title,
+                        "side": p["side"],
+                        "outcome": outcome,
+                        "probability_at_bet": p["probability_at_bet"],
+                        "rep_earned": 0.0,
+                        "timestamp": p["timestamp"],
+                        "resolved_at": now,
+                    }
+                )
+
+        self._save()
+        return winners, losers
+
+    def get_active_markets(self) -> list[str]:
+        """Get list of market titles with unresolved predictions or stakes."""
+        titles = set()
+        for p in self.predictions:
+            if not p.get("resolved", False):
+                titles.add(p["market_title"])
+        for s in self.market_stakes:
+            if not s.get("resolved", False):
+                titles.add(s["market_title"])
+        return list(titles)
+
+    # ─── Market Stakes (prediction markets) ────────────────────────────
+
+    def place_market_stake(
+        self, node_id: str, market_title: str, side: str, amount: float, probability_at_bet: float
+    ) -> tuple[bool, str]:
+        """Stake oracle rep on a prediction market outcome. FINAL decision.
+
+        Args:
+            node_id: Staker's node ID
+            market_title: Title of the prediction market
+            side: "yes", "no", or outcome name for multi-outcome markets
+            amount: How much oracle rep to risk
+            probability_at_bet: Current probability (0-100) of the chosen side
+
+        Returns (success, detail)
+        """
+        if not side or not side.strip():
+            return False, "Side is required"
+
+        if amount <= 0:
+            return False, "Stake amount must be positive"
+
+        if not (0 <= probability_at_bet <= 100):
+            return False, "Probability must be 0-100"
+
+        available = self.get_oracle_rep(node_id)
+        if available < amount:
+            return False, f"Insufficient oracle rep (have {available:.2f}, need {amount:.2f})"
+
+        # Can't have both a free pick AND a stake on the same market
+        existing_free = [
+            p
+            for p in self.predictions
+            if p["node_id"] == node_id
+            and p["market_title"] == market_title
+            and not p.get("resolved", False)
+        ]
+        if existing_free:
+            return (
+                False,
+                f"You already have a FREE prediction on '{market_title}'. Your decision was FINAL.",
+            )
+
+        # Can't stake twice on the same market
+        existing_stake = [
+            s
+            for s in self.market_stakes
+            if s["node_id"] == node_id
+            and s["market_title"] == market_title
+            and not s.get("resolved", False)
+        ]
+        if existing_stake:
+            return (
+                False,
+                f"You already have a STAKED prediction on '{market_title}'. Your decision was FINAL.",
+            )
+
+        self.market_stakes.append(
+            {
+                "stake_id": secrets.token_hex(6),
+                "node_id": node_id,
+                "market_title": market_title,
+                "side": side,
+                "amount": amount,
+                "probability_at_bet": probability_at_bet,
+                "timestamp": time.time(),
+                "resolved": False,
+                "correct": None,
+                "rep_earned": 0.0,
+            }
+        )
+        self._save()
+
+        logger.info(
+            f"MARKET STAKE: {node_id} stakes {amount:.2f} rep on '{side}' "
+            f"for '{market_title}' at {probability_at_bet}%"
+        )
+        return True, (
+            f"STAKED {amount:.2f} oracle rep on {side.upper()} for '{market_title}' "
+            f"at {probability_at_bet}%. This decision is FINAL. "
+            f"If correct, you split the loser pool proportionally."
+        )
+
+    def resolve_market_stakes(self, market_title: str, outcome: str) -> dict:
+        """Resolve all market stakes for a concluded market.
+
+        Winners split the loser pool proportionally to their stake.
+        If everyone picked the same side, stakes are returned (no profit, no loss).
+
+        Returns summary dict.
+        """
+        if not outcome:
+            return {"resolved": 0}
+
+        outcome_lower = outcome.lower()
+        active = [
+            s
+            for s in self.market_stakes
+            if s["market_title"] == market_title and not s.get("resolved", False)
+        ]
+
+        if not active:
+            return {"resolved": 0}
+
+        winners = [s for s in active if s["side"].lower() == outcome_lower]
+        losers = [s for s in active if s["side"].lower() != outcome_lower]
+
+        winner_pool = sum(s["amount"] for s in winners)
+        loser_pool = sum(s["amount"] for s in losers)
+
+        now = time.time()
+
+        if not losers:
+            # Everyone picked the same side — return stakes, no profit
+            for s in active:
+                s["resolved"] = True
+                s["correct"] = True
+                s["rep_earned"] = 0.0  # No profit when no opposition
+            self._save()
+            logger.info(
+                f"Market stake resolution [{market_title}]: unanimous '{outcome}', "
+                f"{len(winners)} stakers get rep back (no loser pool)"
+            )
+            return {
+                "resolved": len(active),
+                "winners": len(winners),
+                "losers": 0,
+                "winner_pool": winner_pool,
+                "loser_pool": 0,
+                "unanimous": True,
+            }
+
+        # Losers lose their staked rep
+        for s in losers:
+            self._remove_oracle_rep(s["node_id"], s["amount"])
+            s["resolved"] = True
+            s["correct"] = False
+            s["rep_earned"] = 0.0
+
+            self.prediction_log.append(
+                {
+                    "node_id": s["node_id"],
+                    "market_title": market_title,
+                    "side": s["side"],
+                    "outcome": outcome,
+                    "probability_at_bet": s["probability_at_bet"],
+                    "rep_earned": 0.0,
+                    "staked": s["amount"],
+                    "timestamp": s["timestamp"],
+                    "resolved_at": now,
+                }
+            )
+
+        # Winners split loser pool proportionally + keep their own stake
+        for s in winners:
+            proportion = s["amount"] / winner_pool if winner_pool > 0 else 0
+            winnings = round(loser_pool * proportion, 3)
+            s["resolved"] = True
+            s["correct"] = True
+            s["rep_earned"] = winnings
+            self._add_oracle_rep(s["node_id"], winnings)
+
+            self.prediction_log.append(
+                {
+                    "node_id": s["node_id"],
+                    "market_title": market_title,
+                    "side": s["side"],
+                    "outcome": outcome,
+                    "probability_at_bet": s["probability_at_bet"],
+                    "rep_earned": winnings,
+                    "staked": s["amount"],
+                    "timestamp": s["timestamp"],
+                    "resolved_at": now,
+                }
+            )
+
+        self._save()
+        logger.info(
+            f"Market stake resolution [{market_title}]: '{outcome}' wins. "
+            f"{len(winners)} winners split {loser_pool:.2f} rep from {len(losers)} losers"
+        )
+        return {
+            "resolved": len(active),
+            "winners": len(winners),
+            "losers": len(losers),
+            "winner_pool": round(winner_pool, 3),
+            "loser_pool": round(loser_pool, 3),
+        }
+
+    def get_market_consensus(self, market_title: str) -> dict:
+        """Get network consensus for a single market — picks + stakes per side."""
+        sides: dict[str, dict] = {}
+
+        # Count free predictions
+        for p in self.predictions:
+            if p["market_title"] != market_title or p.get("resolved", False):
+                continue
+            s = p["side"]
+            if s not in sides:
+                sides[s] = {"picks": 0, "staked": 0.0}
+            sides[s]["picks"] += 1
+
+        # Count market stakes
+        for st in self.market_stakes:
+            if st["market_title"] != market_title or st.get("resolved", False):
+                continue
+            s = st["side"]
+            if s not in sides:
+                sides[s] = {"picks": 0, "staked": 0.0}
+            sides[s]["picks"] += 1
+            sides[s]["staked"] = round(sides[s]["staked"] + st["amount"], 3)
+
+        total_picks = sum(v["picks"] for v in sides.values())
+        total_staked = round(sum(v["staked"] for v in sides.values()), 3)
+
+        return {
+            "market_title": market_title,
+            "total_picks": total_picks,
+            "total_staked": total_staked,
+            "sides": sides,
+        }
+
+    def get_all_market_consensus(self) -> dict[str, dict]:
+        """Bulk consensus for all active markets. Returns {market_title: consensus_summary}."""
+        titles = set()
+        for p in self.predictions:
+            if not p.get("resolved", False):
+                titles.add(p["market_title"])
+        for s in self.market_stakes:
+            if not s.get("resolved", False):
+                titles.add(s["market_title"])
+
+        result = {}
+        for title in titles:
+            c = self.get_market_consensus(title)
+            result[title] = {
+                "total_picks": c["total_picks"],
+                "total_staked": c["total_staked"],
+                "sides": c["sides"],
+            }
+        return result
+
+    # ─── Truth Stakes (posts/comments — separate system) ──────────────
+
+    def place_stake(
+        self,
+        staker_id: str,
+        message_id: str,
+        poster_id: str,
+        side: str,
+        amount: float,
+        duration_days: int,
+    ) -> tuple[bool, str]:
+        """Stake oracle rep on a post's truthfulness.
+
+        Args:
+            staker_id: Oracle staking their rep
+            message_id: The post/message being evaluated
+            poster_id: Who posted the original message
+            side: "truth" or "false"
+            amount: How much oracle rep to stake
+            duration_days: 1-7 days before resolution
+
+        Returns (success, detail)
+        """
+        if side not in ("truth", "false"):
+            return False, "Side must be 'truth' or 'false'"
+
+        if not (MIN_STAKE_DAYS <= duration_days <= MAX_STAKE_DAYS):
+            return False, f"Duration must be {MIN_STAKE_DAYS}-{MAX_STAKE_DAYS} days"
+
+        if amount <= 0:
+            return False, "Stake amount must be positive"
+
+        available = self.get_oracle_rep(staker_id)
+        if available < amount:
+            return False, f"Insufficient oracle rep (have {available}, need {amount})"
+
+        # Check if this staker already has an active stake on this message
+        existing = [
+            s
+            for s in self.stakes
+            if s["staker_id"] == staker_id
+            and s["message_id"] == message_id
+            and not s.get("resolved", False)
+        ]
+        if existing:
+            return False, "You already have an active stake on this message"
+
+        now = time.time()
+        expires = now + (duration_days * 86400)
+
+        # Check if there are existing stakes — extend grace period
+        active_stakes = [
+            s for s in self.stakes if s["message_id"] == message_id and not s.get("resolved", False)
+        ]
+        # If this is a counter-stake, ensure the expiry is at least GRACE_PERIOD_HOURS
+        # after the latest stake on the other side
+        for s in active_stakes:
+            if s["side"] != side:
+                min_expires = s.get("last_counter_at", s["created_at"]) + (
+                    GRACE_PERIOD_HOURS * 3600
+                )
+                if expires < min_expires:
+                    expires = min_expires
+
+        stake = {
+            "stake_id": secrets.token_hex(6),
+            "message_id": message_id,
+            "poster_id": poster_id,
+            "staker_id": staker_id,
+            "side": side,
+            "amount": amount,
+            "duration_days": duration_days,
+            "created_at": now,
+            "expires_at": expires,
+            "resolved": False,
+            "last_counter_at": now,
+        }
+        self.stakes.append(stake)
+
+        # Update last_counter_at on opposing stakes (extends their grace period)
+        for s in active_stakes:
+            if s["side"] != side:
+                s["last_counter_at"] = now
+
+        self._save()
+
+        days_str = f"{duration_days} day{'s' if duration_days > 1 else ''}"
+        logger.info(
+            f"Oracle stake: {staker_id} stakes {amount} oracle rep "
+            f"as '{side}' on message {message_id} for {days_str}"
+        )
+        return True, (
+            f"Staked {amount} oracle rep as '{side.upper()}' on message "
+            f"{message_id} for {days_str}. Expires {time.strftime('%Y-%m-%d %H:%M', time.localtime(expires))}"
+        )
+
+    def resolve_expired_stakes(self) -> list[dict]:
+        """Resolve all expired stake contests. Called periodically.
+
+        Returns list of resolution summaries.
+        """
+        now = time.time()
+        resolutions = []
+
+        # Group active stakes by message_id
+        active_by_msg: dict[str, list[dict]] = {}
+        for s in self.stakes:
+            if not s.get("resolved", False):
+                active_by_msg.setdefault(s["message_id"], []).append(s)
+
+        for msg_id, stakes in active_by_msg.items():
+            # Check if ALL stakes for this message have expired
+            if not all(s["expires_at"] <= now for s in stakes):
+                continue  # Some stakes haven't expired yet
+
+            # Tally sides
+            truth_total = sum(s["amount"] for s in stakes if s["side"] == "truth")
+            false_total = sum(s["amount"] for s in stakes if s["side"] == "false")
+
+            if truth_total == false_total:
+                # Tie — everyone gets their rep back, no resolution
+                for s in stakes:
+                    s["resolved"] = True
+                resolutions.append(
+                    {
+                        "message_id": msg_id,
+                        "outcome": "tie",
+                        "truth_total": truth_total,
+                        "false_total": false_total,
+                    }
+                )
+                continue
+
+            winning_side = "truth" if truth_total > false_total else "false"
+            losing_total = false_total if winning_side == "truth" else truth_total
+            winning_total = truth_total if winning_side == "truth" else false_total
+
+            winners = [s for s in stakes if s["side"] == winning_side]
+            losers = [s for s in stakes if s["side"] != winning_side]
+
+            # Losers lose their staked rep
+            for s in losers:
+                self._remove_oracle_rep(s["staker_id"], s["amount"])
+                s["resolved"] = True
+
+            # Winners divide losers' rep proportionally
+            for s in winners:
+                proportion = s["amount"] / winning_total if winning_total > 0 else 0
+                winnings = round(losing_total * proportion, 3)
+                self._add_oracle_rep(s["staker_id"], winnings)
+                s["resolved"] = True
+
+            # Duration weight for the poster's reputation effect
+            max_duration = max(s["duration_days"] for s in stakes)
+            duration_label = (
+                "resounding" if max_duration >= 7 else "contested" if max_duration >= 3 else "brief"
+            )
+
+            resolution = {
+                "message_id": msg_id,
+                "poster_id": stakes[0].get("poster_id", ""),
+                "outcome": winning_side,
+                "truth_total": round(truth_total, 3),
+                "false_total": round(false_total, 3),
+                "duration_label": duration_label,
+                "max_duration_days": max_duration,
+                "winners": [
+                    {
+                        "node_id": s["staker_id"],
+                        "staked": s["amount"],
+                        "won": (
+                            round(losing_total * (s["amount"] / winning_total), 3)
+                            if winning_total > 0
+                            else 0
+                        ),
+                    }
+                    for s in winners
+                ],
+                "losers": [
+                    {
+                        "node_id": s["staker_id"],
+                        "lost": s["amount"],
+                    }
+                    for s in losers
+                ],
+            }
+            resolutions.append(resolution)
+            logger.info(
+                f"Oracle resolution [{msg_id}]: {winning_side.upper()} wins "
+                f"({truth_total} vs {false_total}), {duration_label} verdict"
+            )
+
+        if resolutions:
+            self._save()
+        return resolutions
+
+    def get_stakes_for_message(self, message_id: str) -> dict:
+        """Get all stakes on a message with totals."""
+        active = [
+            s for s in self.stakes if s["message_id"] == message_id and not s.get("resolved", False)
+        ]
+        truth_stakes = [s for s in active if s["side"] == "truth"]
+        false_stakes = [s for s in active if s["side"] == "false"]
+
+        return {
+            "message_id": message_id,
+            "truth_total": round(sum(s["amount"] for s in truth_stakes), 3),
+            "false_total": round(sum(s["amount"] for s in false_stakes), 3),
+            "truth_stakers": [
+                {"node_id": s["staker_id"], "amount": s["amount"], "expires": s["expires_at"]}
+                for s in truth_stakes
+            ],
+            "false_stakers": [
+                {"node_id": s["staker_id"], "amount": s["amount"], "expires": s["expires_at"]}
+                for s in false_stakes
+            ],
+            "earliest_expiry": min((s["expires_at"] for s in active), default=0),
+        }
+
+    # ─── Oracle Profile ───────────────────────────────────────────────
+
+    def get_oracle_profile(self, node_id: str) -> dict:
+        """Full oracle profile — rep, prediction history, active stakes."""
+        total_rep = self.get_total_oracle_rep(node_id)
+        available_rep = self.get_oracle_rep(node_id)
+
+        # Prediction stats
+        my_predictions = [p for p in self.prediction_log if p["node_id"] == node_id]
+        wins = [p for p in my_predictions if p["rep_earned"] > 0]
+        losses = [p for p in my_predictions if p["rep_earned"] == 0]
+
+        # Active stakes
+        active_stakes = [
+            {
+                "message_id": s["message_id"],
+                "side": s["side"],
+                "amount": s["amount"],
+                "expires": s["expires_at"],
+            }
+            for s in self.stakes
+            if s["staker_id"] == node_id and not s.get("resolved", False)
+        ]
+
+        # Recent prediction log (last 20)
+        recent = sorted(my_predictions, key=lambda x: x.get("resolved_at", 0), reverse=True)[:20]
+        prediction_history = [
+            {
+                "market": p["market_title"][:50],
+                "side": p["side"],
+                "probability": p["probability_at_bet"],
+                "outcome": p.get("outcome", "?"),
+                "rep_earned": p["rep_earned"],
+                "correct": p["rep_earned"] > 0,
+                "age": f"{int((time.time() - p.get('resolved_at', p['timestamp'])) / 86400)}d ago",
+            }
+            for p in recent
+        ]
+
+        # Farming score — what % of bets were on >80% probability outcomes
+        if my_predictions:
+            easy_bets = sum(
+                1
+                for p in my_predictions
+                if (p["side"] == "yes" and p["probability_at_bet"] > 80)
+                or (p["side"] == "no" and p["probability_at_bet"] < 20)
+            )
+            farming_pct = round(easy_bets / len(my_predictions) * 100)
+        else:
+            farming_pct = 0
+
+        return {
+            "node_id": node_id,
+            "oracle_rep": available_rep,
+            "oracle_rep_total": total_rep,
+            "oracle_rep_locked": round(total_rep - available_rep, 3),
+            "predictions_won": len(wins),
+            "predictions_lost": len(losses),
+            "win_rate": round(len(wins) / max(1, len(wins) + len(losses)) * 100),
+            "farming_pct": farming_pct,
+            "active_stakes": active_stakes,
+            "prediction_history": prediction_history,
+        }
+
+    def get_active_predictions(self, node_id: str) -> list[dict]:
+        """Get a node's unresolved predictions (free picks + staked)."""
+        results = []
+        now = time.time()
+
+        # Free picks
+        for p in self.predictions:
+            if p["node_id"] != node_id or p.get("resolved", False):
+                continue
+            potential = round(1.0 - p["probability_at_bet"] / 100, 3)
+            days = int((now - p["timestamp"]) / 86400)
+            results.append(
+                {
+                    "prediction_id": p["prediction_id"],
+                    "market_title": p["market_title"],
+                    "side": p["side"],
+                    "probability_at_bet": p["probability_at_bet"],
+                    "potential_rep": potential,
+                    "staked": 0,
+                    "mode": "free",
+                    "placed": f"{days}d ago",
+                }
+            )
+
+        # Market stakes
+        for s in self.market_stakes:
+            if s["node_id"] != node_id or s.get("resolved", False):
+                continue
+            days = int((now - s["timestamp"]) / 86400)
+            results.append(
+                {
+                    "prediction_id": s["stake_id"],
+                    "market_title": s["market_title"],
+                    "side": s["side"],
+                    "probability_at_bet": s["probability_at_bet"],
+                    "potential_rep": 0,  # Depends on loser pool — unknown until resolution
+                    "staked": s["amount"],
+                    "mode": "staked",
+                    "placed": f"{days}d ago",
+                }
+            )
+
+        return results
+
+    # ─── Cleanup ──────────────────────────────────────────────────────
+
+    def cleanup_old_data(self):
+        """Remove resolved predictions and market stakes older than decay window."""
+        cutoff = time.time() - (ORACLE_DECAY_DAYS * 86400)
+        before_pred = len(self.predictions)
+        before_stakes = len(self.market_stakes)
+        self.predictions = [
+            p for p in self.predictions if not p.get("resolved", False) or p["timestamp"] >= cutoff
+        ]
+        self.market_stakes = [
+            s
+            for s in self.market_stakes
+            if not s.get("resolved", False) or s["timestamp"] >= cutoff
+        ]
+        # Trim prediction log
+        self.prediction_log = [
+            p for p in self.prediction_log if p.get("resolved_at", p["timestamp"]) >= cutoff
+        ]
+        removed = (before_pred - len(self.predictions)) + (before_stakes - len(self.market_stakes))
+        if removed:
+            self._save()
+            logger.info(f"Cleaned up {removed} old predictions/stakes")
+
+
+# ─── Module-level singleton ──────────────────────────────────────────────
+
+oracle_ledger = OracleLedger()

backend/services/mesh/mesh_peer_store.py

@@ -0,0 +1,356 @@
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+import time
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+from services.mesh.mesh_crypto import normalize_peer_url
+
+BACKEND_DIR = Path(__file__).resolve().parents[2]
+DATA_DIR = BACKEND_DIR / "data"
+DEFAULT_PEER_STORE_PATH = DATA_DIR / "peer_store.json"
+PEER_STORE_VERSION = 1
+ALLOWED_PEER_BUCKETS = {"bootstrap", "sync", "push"}
+ALLOWED_PEER_SOURCES = {"bundle", "operator", "bootstrap_promoted", "runtime"}
+ALLOWED_PEER_TRANSPORTS = {"clearnet", "onion"}
+ALLOWED_PEER_ROLES = {"participant", "relay", "seed"}
+
+
+class PeerStoreError(ValueError):
+    pass
+
+
+def _atomic_write_text(target: Path, content: str) -> None:
+    target.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp_path = tempfile.mkstemp(dir=str(target.parent), suffix=".tmp")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as handle:
+            handle.write(content)
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(tmp_path, str(target))
+    except BaseException:
+        try:
+            os.unlink(tmp_path)
+        except OSError:
+            pass
+        raise
+
+
+@dataclass(frozen=True)
+class PeerRecord:
+    bucket: str
+    source: str
+    peer_url: str
+    transport: str
+    role: str
+    label: str = ""
+    signer_id: str = ""
+    enabled: bool = True
+    added_at: int = 0
+    updated_at: int = 0
+    last_seen_at: int = 0
+    last_sync_ok_at: int = 0
+    last_push_ok_at: int = 0
+    last_error: str = ""
+    failure_count: int = 0
+    cooldown_until: int = 0
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def record_key(self) -> str:
+        return f"{self.bucket}:{self.peer_url}"
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+def _normalize_peer_record(data: dict[str, Any]) -> PeerRecord:
+    bucket = str(data.get("bucket", "") or "").strip().lower()
+    source = str(data.get("source", "") or "").strip().lower()
+    peer_url = str(data.get("peer_url", "") or "").strip()
+    transport = str(data.get("transport", "") or "").strip().lower()
+    role = str(data.get("role", "") or "").strip().lower()
+    label = str(data.get("label", "") or "").strip()
+    signer_id = str(data.get("signer_id", "") or "").strip()
+    enabled = bool(data.get("enabled", True))
+    metadata = data.get("metadata", {})
+
+    if bucket not in ALLOWED_PEER_BUCKETS:
+        raise PeerStoreError(f"unsupported peer bucket: {bucket or 'missing'}")
+    if source not in ALLOWED_PEER_SOURCES:
+        raise PeerStoreError(f"unsupported peer source: {source or 'missing'}")
+    if transport not in ALLOWED_PEER_TRANSPORTS:
+        raise PeerStoreError(f"unsupported peer transport: {transport or 'missing'}")
+    if role not in ALLOWED_PEER_ROLES:
+        raise PeerStoreError(f"unsupported peer role: {role or 'missing'}")
+
+    normalized = normalize_peer_url(peer_url)
+    if not normalized or normalized != peer_url:
+        raise PeerStoreError("peer_url must be normalized")
+    parsed = urlparse(normalized)
+    hostname = str(parsed.hostname or "").strip().lower()
+    if transport == "clearnet":
+        if parsed.scheme not in ("https", "http") or hostname.endswith(".onion"):
+            raise PeerStoreError("clearnet peers must use https:// (or http:// for LAN/testnet)")
+    elif transport == "onion":
+        if parsed.scheme != "http" or not hostname.endswith(".onion"):
+            raise PeerStoreError("onion peers must use http://*.onion")
+
+    if not isinstance(metadata, dict):
+        raise PeerStoreError("peer metadata must be an object")
+
+    return PeerRecord(
+        bucket=bucket,
+        source=source,
+        peer_url=normalized,
+        transport=transport,
+        role=role,
+        label=label,
+        signer_id=signer_id,
+        enabled=enabled,
+        added_at=int(data.get("added_at", 0) or 0),
+        updated_at=int(data.get("updated_at", 0) or 0),
+        last_seen_at=int(data.get("last_seen_at", 0) or 0),
+        last_sync_ok_at=int(data.get("last_sync_ok_at", 0) or 0),
+        last_push_ok_at=int(data.get("last_push_ok_at", 0) or 0),
+        last_error=str(data.get("last_error", "") or ""),
+        failure_count=int(data.get("failure_count", 0) or 0),
+        cooldown_until=int(data.get("cooldown_until", 0) or 0),
+        metadata=dict(metadata),
+    )
+
+
+def make_bootstrap_peer_record(
+    *,
+    peer_url: str,
+    transport: str,
+    role: str,
+    signer_id: str,
+    label: str = "",
+    now: float | None = None,
+) -> PeerRecord:
+    timestamp = int(now if now is not None else time.time())
+    return _normalize_peer_record(
+        {
+            "bucket": "bootstrap",
+            "source": "bundle",
+            "peer_url": peer_url,
+            "transport": transport,
+            "role": role,
+            "label": label,
+            "signer_id": signer_id,
+            "enabled": True,
+            "added_at": timestamp,
+            "updated_at": timestamp,
+        }
+    )
+
+
+def make_sync_peer_record(
+    *,
+    peer_url: str,
+    transport: str,
+    role: str = "participant",
+    source: str = "operator",
+    label: str = "",
+    signer_id: str = "",
+    now: float | None = None,
+) -> PeerRecord:
+    timestamp = int(now if now is not None else time.time())
+    return _normalize_peer_record(
+        {
+            "bucket": "sync",
+            "source": source,
+            "peer_url": peer_url,
+            "transport": transport,
+            "role": role,
+            "label": label,
+            "signer_id": signer_id,
+            "enabled": True,
+            "added_at": timestamp,
+            "updated_at": timestamp,
+        }
+    )
+
+
+def make_push_peer_record(
+    *,
+    peer_url: str,
+    transport: str,
+    role: str = "relay",
+    source: str = "operator",
+    label: str = "",
+    now: float | None = None,
+) -> PeerRecord:
+    timestamp = int(now if now is not None else time.time())
+    return _normalize_peer_record(
+        {
+            "bucket": "push",
+            "source": source,
+            "peer_url": peer_url,
+            "transport": transport,
+            "role": role,
+            "label": label,
+            "enabled": True,
+            "added_at": timestamp,
+            "updated_at": timestamp,
+        }
+    )
+
+
+class PeerStore:
+    def __init__(self, path: str | Path = DEFAULT_PEER_STORE_PATH):
+        self.path = Path(path)
+        self._records: dict[str, PeerRecord] = {}
+
+    def load(self) -> list[PeerRecord]:
+        if not self.path.exists():
+            self._records = {}
+            return []
+        try:
+            raw = json.loads(self.path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError as exc:
+            raise PeerStoreError("peer store is not valid JSON") from exc
+
+        if not isinstance(raw, dict):
+            raise PeerStoreError("peer store root must be an object")
+        version = int(raw.get("version", 0) or 0)
+        if version != PEER_STORE_VERSION:
+            raise PeerStoreError(f"unsupported peer store version: {version}")
+        records_raw = raw.get("records", [])
+        if not isinstance(records_raw, list):
+            raise PeerStoreError("peer store records must be a list")
+
+        records: dict[str, PeerRecord] = {}
+        for entry in records_raw:
+            if not isinstance(entry, dict):
+                raise PeerStoreError("peer store records must be objects")
+            record = _normalize_peer_record(entry)
+            records[record.record_key()] = record
+        self._records = records
+        return self.records()
+
+    def save(self) -> None:
+        payload = {
+            "version": PEER_STORE_VERSION,
+            "records": [record.to_dict() for record in self.records()],
+        }
+        _atomic_write_text(
+            self.path,
+            json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False),
+        )
+
+    def records(self) -> list[PeerRecord]:
+        return sorted(self._records.values(), key=lambda item: (item.bucket, item.peer_url))
+
+    def records_for_bucket(self, bucket: str) -> list[PeerRecord]:
+        normalized_bucket = str(bucket or "").strip().lower()
+        return [record for record in self.records() if record.bucket == normalized_bucket]
+
+    def upsert(self, record: PeerRecord) -> PeerRecord:
+        existing = self._records.get(record.record_key())
+        if existing is None:
+            self._records[record.record_key()] = record
+            return record
+
+        merged = PeerRecord(
+            bucket=record.bucket,
+            source=record.source,
+            peer_url=record.peer_url,
+            transport=record.transport,
+            role=record.role,
+            label=record.label or existing.label,
+            signer_id=record.signer_id or existing.signer_id,
+            enabled=record.enabled,
+            added_at=existing.added_at or record.added_at,
+            updated_at=max(existing.updated_at, record.updated_at),
+            last_seen_at=max(existing.last_seen_at, record.last_seen_at),
+            last_sync_ok_at=max(existing.last_sync_ok_at, record.last_sync_ok_at),
+            last_push_ok_at=max(existing.last_push_ok_at, record.last_push_ok_at),
+            last_error=record.last_error or existing.last_error,
+            failure_count=max(existing.failure_count, record.failure_count),
+            cooldown_until=max(existing.cooldown_until, record.cooldown_until),
+            metadata={**existing.metadata, **record.metadata},
+        )
+        self._records[record.record_key()] = merged
+        return merged
+
+    def mark_seen(self, peer_url: str, bucket: str, *, now: float | None = None) -> PeerRecord:
+        record = self._require_record(peer_url, bucket)
+        timestamp = int(now if now is not None else time.time())
+        updated = PeerRecord(
+            **{
+                **record.to_dict(),
+                "last_seen_at": timestamp,
+                "updated_at": timestamp,
+            }
+        )
+        self._records[updated.record_key()] = updated
+        return updated
+
+    def mark_sync_success(self, peer_url: str, bucket: str = "sync", *, now: float | None = None) -> PeerRecord:
+        record = self._require_record(peer_url, bucket)
+        timestamp = int(now if now is not None else time.time())
+        updated = PeerRecord(
+            **{
+                **record.to_dict(),
+                "last_sync_ok_at": timestamp,
+                "last_error": "",
+                "failure_count": 0,
+                "cooldown_until": 0,
+                "updated_at": timestamp,
+            }
+        )
+        self._records[updated.record_key()] = updated
+        return updated
+
+    def mark_push_success(self, peer_url: str, bucket: str = "push", *, now: float | None = None) -> PeerRecord:
+        record = self._require_record(peer_url, bucket)
+        timestamp = int(now if now is not None else time.time())
+        updated = PeerRecord(
+            **{
+                **record.to_dict(),
+                "last_push_ok_at": timestamp,
+                "last_error": "",
+                "failure_count": 0,
+                "cooldown_until": 0,
+                "updated_at": timestamp,
+            }
+        )
+        self._records[updated.record_key()] = updated
+        return updated
+
+    def mark_failure(
+        self,
+        peer_url: str,
+        bucket: str,
+        *,
+        error: str,
+        cooldown_s: int = 0,
+        now: float | None = None,
+    ) -> PeerRecord:
+        record = self._require_record(peer_url, bucket)
+        timestamp = int(now if now is not None else time.time())
+        updated = PeerRecord(
+            **{
+                **record.to_dict(),
+                "last_error": str(error or "").strip(),
+                "failure_count": int(record.failure_count) + 1,
+                "cooldown_until": timestamp + max(0, int(cooldown_s or 0)),
+                "updated_at": timestamp,
+            }
+        )
+        self._records[updated.record_key()] = updated
+        return updated
+
+    def _require_record(self, peer_url: str, bucket: str) -> PeerRecord:
+        normalized_url = normalize_peer_url(peer_url)
+        key = f"{str(bucket or '').strip().lower()}:{normalized_url}"
+        if key not in self._records:
+            raise PeerStoreError(f"peer record not found: {key}")
+        return self._records[key]