mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-06-05 22:06:40 +02:00
Compare commits
47 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 401f114e4f | |||
| 79b39e8985 | |||
| 2da739c9e8 | |||
| eca7f24e2c | |||
| e3efcfd476 | |||
| bc70cc3527 | |||
| 44e9b38ac2 | |||
| b01a69c172 | |||
| c54ea7fd9f | |||
| a3aa7b4dec | |||
| 19fb7f0b1e | |||
| 35cd4e4c71 | |||
| 31f79fd8e2 | |||
| fd7d6fa401 | |||
| 49621824b1 | |||
| 76750caa92 | |||
| c3ef9f4b9e | |||
| 5e6bb8511a | |||
| 0fee36e8f7 | |||
| e125467721 | |||
| 2b03b808ac | |||
| 2e14e75a0e | |||
| 084e563412 | |||
| 9ef6213284 | |||
| fb11e0881f | |||
| 7f96151e56 | |||
| d0299fc0a0 | |||
| 87ba70acd6 | |||
| bcc2d036b3 | |||
| 729ea78cb2 | |||
| 459178f283 | |||
| 8e27658157 | |||
| e36d1fc79c | |||
| d00c63abed | |||
| e3297e9bc0 | |||
| 9ae0b189ba | |||
| dd7706f17f | |||
| 30f0360ef8 | |||
| 421682c447 | |||
| 40734e310b | |||
| 71a9d9e144 | |||
| de27d119f9 | |||
| b8384d6d91 | |||
| 11ea345518 | |||
| 25a98a9869 | |||
| 2ce0e43ee5 | |||
| b86a258535 |
@@ -67,6 +67,12 @@ ADMIN_KEY=
|
|||||||
# SHADOWBROKER_SLOW_FETCH_CONCURRENCY=4
|
# SHADOWBROKER_SLOW_FETCH_CONCURRENCY=4
|
||||||
# SHADOWBROKER_STARTUP_HEAVY_CONCURRENCY=2
|
# SHADOWBROKER_STARTUP_HEAVY_CONCURRENCY=2
|
||||||
|
|
||||||
|
# Infonet bootstrap/sync responsiveness. Defaults favor fast seed failure
|
||||||
|
# detection so stale onion peers do not make the terminal look hung.
|
||||||
|
# MESH_SYNC_TIMEOUT_S=5
|
||||||
|
# MESH_SYNC_MAX_PEERS_PER_CYCLE=3
|
||||||
|
# MESH_BOOTSTRAP_SEED_FAILURE_COOLDOWN_S=15
|
||||||
|
|
||||||
# Google Earth Engine for VIIRS night lights change detection (optional).
|
# Google Earth Engine for VIIRS night lights change detection (optional).
|
||||||
# pip install earthengine-api
|
# pip install earthengine-api
|
||||||
# GEE_SERVICE_ACCOUNT_KEY=
|
# GEE_SERVICE_ACCOUNT_KEY=
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
# CODEOWNERS — assigns required reviewers for sensitive paths.
|
||||||
|
# Format: <path glob> <user-or-team> [<user-or-team> ...]
|
||||||
|
# See https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
|
||||||
|
#
|
||||||
|
# Owners listed here are auto-requested for review when matching files
|
||||||
|
# change in a PR. If branch protection requires CODEOWNERS approval, the
|
||||||
|
# PR cannot be merged until an owner approves.
|
||||||
|
|
||||||
|
# ── Internationalization / translations ──
|
||||||
|
# Translation contributions are held to a stricter neutrality standard
|
||||||
|
# than most code changes — see CONTRIBUTING.md "Translation contributions".
|
||||||
|
# The i18n layer itself (no network calls, no telemetry, static JSON
|
||||||
|
# bundled at build) is the structural guarantee that makes this safe;
|
||||||
|
# changes to it need owner review.
|
||||||
|
/frontend/src/i18n/ @BigBodyCobain
|
||||||
|
|
||||||
|
# ── Security-sensitive code paths ──
|
||||||
|
/backend/auth.py @BigBodyCobain
|
||||||
|
/backend/routers/wormhole.py @BigBodyCobain
|
||||||
|
/backend/services/mesh/ @BigBodyCobain
|
||||||
|
/backend/services/fetchers/ @BigBodyCobain
|
||||||
|
|
||||||
|
# ── CI / build / deploy infra ──
|
||||||
|
/.github/workflows/ @BigBodyCobain
|
||||||
|
/.gitlab-ci.yml @BigBodyCobain
|
||||||
|
/docker-compose.yml @BigBodyCobain
|
||||||
|
/docker-compose.gitlab.yml @BigBodyCobain
|
||||||
|
/helm/ @BigBodyCobain
|
||||||
|
|
||||||
|
# ── This file and policy docs ──
|
||||||
|
/.github/CODEOWNERS @BigBodyCobain
|
||||||
|
/CONTRIBUTING.md @BigBodyCobain
|
||||||
@@ -7,6 +7,28 @@ on:
|
|||||||
branches: [main]
|
branches: [main]
|
||||||
workflow_call:
|
workflow_call:
|
||||||
|
|
||||||
|
# CI flake mitigation:
|
||||||
|
# ci.yml is triggered TWICE per PR on the same commit — once directly via
|
||||||
|
# the `pull_request` trigger above ("Frontend Tests & Build" check) and once
|
||||||
|
# via `workflow_call` from docker-publish.yml ("CI Gate / Frontend Tests &
|
||||||
|
# Build" check). Both jobs land on the same Actions runner pool at the same
|
||||||
|
# time and fight for CPU/RAM. Under contention, React's reconciliation in
|
||||||
|
# `messagesViewFirstContact.test.tsx > removes an approved contact …`
|
||||||
|
# overruns its 5s waitFor timeout — that's the single failure mode we've
|
||||||
|
# seen flake on PRs #226, #237, #261, #262, #265, #294, #303, and the
|
||||||
|
# fd7d6fa push. Backend tests and every other frontend test pass under
|
||||||
|
# the same conditions, which is what made this look random.
|
||||||
|
#
|
||||||
|
# Pinning a concurrency group on the SHA (PR head, or the pushed commit
|
||||||
|
# for main) serializes the two invocations so neither starves the other.
|
||||||
|
# We use cancel-in-progress: false so the second one queues instead of
|
||||||
|
# cancelling — cancelling could leave the PR check stuck "Expected" if
|
||||||
|
# only one of the two ever finishes. Total CI time grows by ~2 min in
|
||||||
|
# exchange for deterministic outcomes.
|
||||||
|
concurrency:
|
||||||
|
group: ci-${{ github.event.pull_request.head.sha || github.sha }}
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
frontend:
|
frontend:
|
||||||
name: Frontend Tests & Build
|
name: Frontend Tests & Build
|
||||||
|
|||||||
+49
@@ -91,6 +91,24 @@ backend/data/*
|
|||||||
!backend/data/power_plants.json
|
!backend/data/power_plants.json
|
||||||
!backend/data/tracked_names.json
|
!backend/data/tracked_names.json
|
||||||
!backend/data/yacht_alert_db.json
|
!backend/data/yacht_alert_db.json
|
||||||
|
# Issue #206: bundled KiwiSDR receiver directory used as last-resort
|
||||||
|
# fallback when rx.linkfanel.net (HTTP-only upstream) is unreachable
|
||||||
|
# or returns content that fails our integrity validation.
|
||||||
|
!backend/data/kiwisdr_directory.json
|
||||||
|
# Issue #201: pinned SHA-256 digests for known Tor Expert Bundle URLs.
|
||||||
|
# Used as a second verification source when upstream .sha256sum fails.
|
||||||
|
!backend/data/tor_bundle_digests.json
|
||||||
|
# Issue #258: SPKI pins for stream.aisstream.io so we can survive upstream
|
||||||
|
# Let's Encrypt renewal failures without disabling TLS validation entirely.
|
||||||
|
!backend/data/aisstream_spki_pins.json
|
||||||
|
# Issue #231: pinned SHA-256 digests for known release archives. Used by
|
||||||
|
# the self-updater as a second-line integrity check when the release's
|
||||||
|
# SHA256SUMS.txt asset can't be fetched.
|
||||||
|
!backend/data/release_digests.json
|
||||||
|
# Issue #244/#245/#246: one-shot carrier-position seed shipped with each
|
||||||
|
# release. Used ONLY on first-ever startup to bootstrap carrier_cache.json;
|
||||||
|
# after that the cache reflects this install's own GDELT observations.
|
||||||
|
!backend/data/carrier_seed.json
|
||||||
|
|
||||||
# OS generated files
|
# OS generated files
|
||||||
.DS_Store
|
.DS_Store
|
||||||
@@ -173,6 +191,8 @@ backend/services/test_*.py
|
|||||||
# Local analysis & dev tools
|
# Local analysis & dev tools
|
||||||
backend/analyze_xlsx.py
|
backend/analyze_xlsx.py
|
||||||
backend/services/ais_cache.json
|
backend/services/ais_cache.json
|
||||||
|
graphify/
|
||||||
|
graphify-out/
|
||||||
|
|
||||||
# ========================
|
# ========================
|
||||||
# Internal docs & brainstorming (never commit)
|
# Internal docs & brainstorming (never commit)
|
||||||
@@ -241,3 +261,32 @@ backend/data/wormhole_stdout.log
|
|||||||
|
|
||||||
# Compressed snapshot archives (can be 100 MB+)
|
# Compressed snapshot archives (can be 100 MB+)
|
||||||
*.json.gz
|
*.json.gz
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────────────────────────────
|
||||||
|
# AI assistant / coding-agent scratch
|
||||||
|
# ──────────────────────────────────────────────────────────────────────
|
||||||
|
# Per-tool config + scratch directories. These are private to whichever
|
||||||
|
# coding agent the operator happens to be using and have no business in
|
||||||
|
# the repo. If a tool's instructions need to be canonical for the project,
|
||||||
|
# we'll put them in docs/ explicitly — not let the agent dump them at the
|
||||||
|
# repo root.
|
||||||
|
|
||||||
|
# OpenAI Codex CLI
|
||||||
|
.codex/
|
||||||
|
.codex-app-schema/
|
||||||
|
.codex-app-ts/
|
||||||
|
|
||||||
|
# Per-agent instruction files dropped at repo root by various tools.
|
||||||
|
# These are operator-side preferences, not part of the project contract.
|
||||||
|
AGENTS.md
|
||||||
|
GEMINI.md
|
||||||
|
CLAUDE.md
|
||||||
|
.github/copilot-instructions.md
|
||||||
|
|
||||||
|
# Stale AI-generated test file that referenced fields that don't exist in
|
||||||
|
# the current `_parse_carrier_positions_from_news` implementation. Kept
|
||||||
|
# ignored so it doesn't accidentally get committed if it shows up again
|
||||||
|
# from a tool that's working off an out-of-date understanding of the
|
||||||
|
# module. If a real test for that function is needed, write it under a
|
||||||
|
# meaningful name in tests/test_carrier_tracker_quality.py.
|
||||||
|
backend/tests/test_carrier_tracker_region_centers.py
|
||||||
|
|||||||
+121
@@ -0,0 +1,121 @@
|
|||||||
|
# GitLab CI/CD for Shadowbroker
|
||||||
|
#
|
||||||
|
# Mirror of .github/workflows/docker-publish.yml — keeps the GitLab install
|
||||||
|
# path (image registry + source) at parity with GitHub so users who prefer
|
||||||
|
# GitLab get the same experience.
|
||||||
|
#
|
||||||
|
# What this does on every push to main:
|
||||||
|
# 1. Builds multi-arch (amd64 + arm64) Docker images for the backend and
|
||||||
|
# frontend, pushes them to the project's GitLab Container Registry:
|
||||||
|
# registry.gitlab.com/bigbodycobain/shadowbroker/backend:latest
|
||||||
|
# registry.gitlab.com/bigbodycobain/shadowbroker/frontend:latest
|
||||||
|
# Both also get a :$CI_COMMIT_SHORT_SHA tag for traceability.
|
||||||
|
# 2. Reverse-mirrors main back to GitHub (only if commits land directly
|
||||||
|
# on GitLab) so the two sources stay in sync.
|
||||||
|
#
|
||||||
|
# Auth notes:
|
||||||
|
# - The image build/push uses $CI_JOB_TOKEN, which GitLab provides
|
||||||
|
# automatically. No credentials need to be configured.
|
||||||
|
# - The reverse mirror requires a GitHub personal access token stored
|
||||||
|
# as the GitLab CI/CD variable GITHUB_MIRROR_TOKEN (Protected + Masked).
|
||||||
|
# Scope: public_repo (or repo for private). If the variable isn't
|
||||||
|
# set the mirror job is skipped — image builds still run.
|
||||||
|
|
||||||
|
stages:
|
||||||
|
- build
|
||||||
|
- mirror
|
||||||
|
|
||||||
|
variables:
|
||||||
|
# Use the dind service for buildx multi-arch builds.
|
||||||
|
DOCKER_HOST: tcp://docker:2376
|
||||||
|
DOCKER_TLS_CERTDIR: "/certs"
|
||||||
|
DOCKER_DRIVER: overlay2
|
||||||
|
# QEMU is what lets a single x86 runner build arm64 images. dind doesn't
|
||||||
|
# install it by default; we install via tonistiigi/binfmt below.
|
||||||
|
BUILDX_VERSION: "v0.14.1"
|
||||||
|
# Repository-relative paths.
|
||||||
|
BACKEND_IMAGE: $CI_REGISTRY_IMAGE/backend
|
||||||
|
FRONTEND_IMAGE: $CI_REGISTRY_IMAGE/frontend
|
||||||
|
|
||||||
|
# Shared template: bootstraps buildx + QEMU on the dind service so a single
|
||||||
|
# runner can produce both amd64 and arm64 manifests in one push.
|
||||||
|
.buildx-setup: &buildx-setup
|
||||||
|
image: docker:24
|
||||||
|
services:
|
||||||
|
- name: docker:24-dind
|
||||||
|
command: ["--tls=true"]
|
||||||
|
before_script:
|
||||||
|
- docker info
|
||||||
|
- docker login -u "$CI_REGISTRY_USER" -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
|
||||||
|
- docker run --privileged --rm tonistiigi/binfmt --install all
|
||||||
|
- docker buildx create --use --name multiarch --driver docker-container
|
||||||
|
|
||||||
|
# ── Backend image ────────────────────────────────────────────────────────
|
||||||
|
build-backend:
|
||||||
|
<<: *buildx-setup
|
||||||
|
stage: build
|
||||||
|
script:
|
||||||
|
- >
|
||||||
|
docker buildx build
|
||||||
|
--platform linux/amd64,linux/arm64
|
||||||
|
--file backend/Dockerfile
|
||||||
|
--tag $BACKEND_IMAGE:latest
|
||||||
|
--tag $BACKEND_IMAGE:$CI_COMMIT_SHORT_SHA
|
||||||
|
--push
|
||||||
|
.
|
||||||
|
rules:
|
||||||
|
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||||||
|
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "schedule"
|
||||||
|
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||||
|
changes:
|
||||||
|
- backend/**/*
|
||||||
|
- .gitlab-ci.yml
|
||||||
|
|
||||||
|
# ── Frontend image ───────────────────────────────────────────────────────
|
||||||
|
build-frontend:
|
||||||
|
<<: *buildx-setup
|
||||||
|
stage: build
|
||||||
|
script:
|
||||||
|
- cd frontend
|
||||||
|
- >
|
||||||
|
docker buildx build
|
||||||
|
--platform linux/amd64,linux/arm64
|
||||||
|
--tag $FRONTEND_IMAGE:latest
|
||||||
|
--tag $FRONTEND_IMAGE:$CI_COMMIT_SHORT_SHA
|
||||||
|
--push
|
||||||
|
.
|
||||||
|
rules:
|
||||||
|
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||||||
|
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "schedule"
|
||||||
|
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||||
|
changes:
|
||||||
|
- frontend/**/*
|
||||||
|
- .gitlab-ci.yml
|
||||||
|
|
||||||
|
# ── Reverse mirror to GitHub ─────────────────────────────────────────────
|
||||||
|
# Pushes refs/heads/main to github.com/BigBodyCobain/Shadowbroker.
|
||||||
|
# Fast-forward-only — if GitLab main and GitHub main have diverged, this
|
||||||
|
# fails loudly rather than silently overwriting either side.
|
||||||
|
#
|
||||||
|
# Only runs if GITHUB_MIRROR_TOKEN is set as a CI/CD variable. See the
|
||||||
|
# header comment of this file for setup instructions.
|
||||||
|
mirror-to-github:
|
||||||
|
stage: mirror
|
||||||
|
image: alpine:3.20
|
||||||
|
needs: []
|
||||||
|
before_script:
|
||||||
|
- apk add --no-cache git openssh-client ca-certificates
|
||||||
|
script:
|
||||||
|
- git config --global user.email "ci-mirror@gitlab.com"
|
||||||
|
- git config --global user.name "GitLab CI Mirror"
|
||||||
|
- >
|
||||||
|
git clone --depth=50 --branch main
|
||||||
|
"https://oauth2:${CI_JOB_TOKEN}@gitlab.com/${CI_PROJECT_PATH}.git"
|
||||||
|
repo
|
||||||
|
- cd repo
|
||||||
|
- >
|
||||||
|
git push
|
||||||
|
"https://x-access-token:${GITHUB_MIRROR_TOKEN}@github.com/BigBodyCobain/Shadowbroker.git"
|
||||||
|
"${CI_COMMIT_SHA}:refs/heads/main"
|
||||||
|
rules:
|
||||||
|
- if: $CI_COMMIT_BRANCH == "main" && $GITHUB_MIRROR_TOKEN
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
# Contributing to Shadowbroker
|
||||||
|
|
||||||
|
Thank you for taking the time to contribute. This document covers things specific to this project — for general open-source contribution etiquette, see the GitHub docs.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Code contributions
|
||||||
|
|
||||||
|
1. Fork the repo on GitHub (`bigbodycobain/Shadowbroker`) or GitLab (`bigbodycobain/Shadowbroker` mirror).
|
||||||
|
2. Make your changes on a feature branch.
|
||||||
|
3. Run the local test suite:
|
||||||
|
- Backend: `pytest backend/tests/`
|
||||||
|
- Frontend: `cd frontend && npx vitest run`
|
||||||
|
4. Open a Pull Request against `main`.
|
||||||
|
|
||||||
|
CI runs on every PR. If CI fails, that's blocking — please push fixes rather than asking for it to be merged anyway.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Reporting security issues
|
||||||
|
|
||||||
|
Do **not** file security issues as public GitHub issues. Email the maintainer or use a private security advisory on GitHub. Public disclosure of an exploitable vulnerability without prior coordination will be rejected from the project.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Translation contributions
|
||||||
|
|
||||||
|
Shadowbroker supports UI localization (`frontend/src/i18n/`). Translation contributions are welcome but held to a stricter standard than most code changes, because translations can subtly reshape user perception in ways that are hard to spot during review. Read this section before submitting one.
|
||||||
|
|
||||||
|
### The neutrality requirement
|
||||||
|
|
||||||
|
**Translations must be technically faithful to the English source.** That means:
|
||||||
|
|
||||||
|
- Each `t('key')` entry should mean approximately the same thing in the target language as in English, modulo idiom.
|
||||||
|
- Technical terms with established meanings (e.g. "GPS jamming," "military flight," "Tor," "onion routing," "encryption") should be translated using the corresponding established technical term in the target language — **not** softened, rebranded, or politically reframed.
|
||||||
|
- The set of UI strings should be **the same** between languages. Don't omit features from one locale that are visible in another.
|
||||||
|
|
||||||
|
### What will get a translation PR rejected
|
||||||
|
|
||||||
|
Translation choices that align the project with the framing or terminology of state propaganda — from **any** country — will be rejected. This applies symmetrically:
|
||||||
|
|
||||||
|
| Country / source | Examples of substitutions we will reject |
|
||||||
|
|---|---|
|
||||||
|
| **PRC / CCP** | Calling Taiwan a "province" or "renegade province"; reframing protest layers as "riots"; using softened or euphemistic terms for surveillance, internment, or jamming when the source text is direct |
|
||||||
|
| **Russia** | Calling the Ukraine war a "special military operation"; relabeling occupied territories as Russian; softening sanctions/jamming/disinfo terminology |
|
||||||
|
| **United States / EU** | Reframing adversaries with editorial labels not in the source (e.g. inserting "regime" where the English says "government"); applying labels like "terrorist" or "rogue state" to entities the English source describes neutrally |
|
||||||
|
| **Israel / Palestine / any active conflict** | Substituting one side's preferred terminology when the source uses the other side's or a neutral term |
|
||||||
|
| **Any government** | Adding political slogans, omitting features that government finds inconvenient, or inserting terminology associated with a specific political faction |
|
||||||
|
|
||||||
|
The test is **"would a translator working strictly from the English source produce this rendering?"** If the answer requires assuming a political stance the source does not take, the substitution does not belong in the translation.
|
||||||
|
|
||||||
|
### How translation PRs are reviewed
|
||||||
|
|
||||||
|
Changes to `frontend/src/i18n/**` are owned by the maintainer (see `CODEOWNERS`) and require explicit approval. We will:
|
||||||
|
|
||||||
|
1. Diff the translation against the English source key-by-key.
|
||||||
|
2. Spot-check a sample of entries with a native speaker of the target language when possible.
|
||||||
|
3. Look for the patterns above.
|
||||||
|
4. Look for suspicious additions to the i18n infrastructure itself (e.g. a remote translation fetcher, telemetry on language choice) — the i18n layer is supposed to be 100% client-side static JSON.
|
||||||
|
|
||||||
|
A PR that adds a new language is harder to review than one that fixes typos in an existing language. For new languages, please be patient and expect a real review window. For typo fixes, please describe each change in the PR body so the reviewer can verify intent.
|
||||||
|
|
||||||
|
### What about adding a new language?
|
||||||
|
|
||||||
|
We welcome new languages. The mechanical setup is documented in the header comment of `frontend/src/i18n/index.ts`. Beyond that:
|
||||||
|
|
||||||
|
- We are more likely to merge a new language quickly if at least one reviewer in the maintainer's network speaks it.
|
||||||
|
- If you are the *only* speaker of the target language reading this repo, your translation is welcome but the merge timeline will be longer while a reviewer is found.
|
||||||
|
- Partial translations are fine — the system falls back to English for any missing key.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Anything else
|
||||||
|
|
||||||
|
If you have a question that isn't a security report, opening a GitHub Discussion or a draft PR with a question in the body is the fastest way to get a response. Direct emails are read but not always replied to promptly.
|
||||||
@@ -61,6 +61,8 @@ ShadowBroker includes an optional Shodan connector for operator-supplied API acc
|
|||||||
|
|
||||||
## ⚡ Quick Start (Docker)
|
## ⚡ Quick Start (Docker)
|
||||||
|
|
||||||
|
### From GitHub (default — uses GHCR images)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone https://github.com/bigbodycobain/Shadowbroker.git
|
git clone https://github.com/bigbodycobain/Shadowbroker.git
|
||||||
cd Shadowbroker
|
cd Shadowbroker
|
||||||
@@ -68,6 +70,17 @@ docker compose pull
|
|||||||
docker compose up -d
|
docker compose up -d
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### From GitLab (uses GitLab Container Registry)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git clone https://gitlab.com/bigbodycobain/Shadowbroker.git
|
||||||
|
cd Shadowbroker
|
||||||
|
docker compose -f docker-compose.yml -f docker-compose.gitlab.yml pull
|
||||||
|
docker compose -f docker-compose.yml -f docker-compose.gitlab.yml up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
Both paths produce identical containers — same source, same CI, same images byte-for-byte. Pick whichever ecosystem you already use.
|
||||||
|
|
||||||
Open `http://localhost:3000` to view the dashboard! *(Requires [Docker Desktop](https://www.docker.com/products/docker-desktop/) or Docker Engine)*
|
Open `http://localhost:3000` to view the dashboard! *(Requires [Docker Desktop](https://www.docker.com/products/docker-desktop/) or Docker Engine)*
|
||||||
|
|
||||||
> **Backend port already in use?** The browser only needs port `3000`, but the backend API is also published on host port `8000` for local diagnostics. If another app already uses `8000`, create or edit `.env` next to `docker-compose.yml` and set `BACKEND_PORT=8001`, then run `docker compose up -d`.
|
> **Backend port already in use?** The browser only needs port `3000`, but the backend API is also published on host port `8000` for local diagnostics. If another app already uses `8000`, create or edit `.env` next to `docker-compose.yml` and set `BACKEND_PORT=8001`, then run `docker compose up -d`.
|
||||||
@@ -136,8 +149,13 @@ helm repo update
|
|||||||
|
|
||||||
**2. Install the Chart:**
|
**2. Install the Chart:**
|
||||||
```bash
|
```bash
|
||||||
# Install from the local helm/chart directory
|
# Default — pulls images from GHCR
|
||||||
helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbroker
|
helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbroker
|
||||||
|
|
||||||
|
# GitLab registry variant
|
||||||
|
helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbroker \
|
||||||
|
-f helm/chart/values.yaml \
|
||||||
|
-f helm/chart/values-gitlab.yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
**3. Key Features:**
|
**3. Key Features:**
|
||||||
|
|||||||
+53
-3
@@ -24,8 +24,54 @@ AIS_API_KEY= # https://aisstream.io/ — free tier WebSocket key
|
|||||||
# Requires MESH_DEBUG_MODE=true; do not enable this for ordinary use.
|
# Requires MESH_DEBUG_MODE=true; do not enable this for ordinary use.
|
||||||
# ALLOW_INSECURE_ADMIN=false
|
# ALLOW_INSECURE_ADMIN=false
|
||||||
|
|
||||||
# User-Agent for Nominatim geocoding requests (per OSM usage policy).
|
# Per-install operator handle. Round 7a: every outbound third-party API
|
||||||
# NOMINATIM_USER_AGENT=ShadowBroker/1.0 (https://github.com/BigBodyCobain/Shadowbroker)
|
# call (Wikipedia, Wikidata, Nominatim, GDELT, OpenMHz, Broadcastify,
|
||||||
|
# weather.gov, NUFORC, etc.) includes this handle in the User-Agent so
|
||||||
|
# upstreams can rate-limit / contact the specific install instead of
|
||||||
|
# treating every Shadowbroker user as one entity.
|
||||||
|
#
|
||||||
|
# Default empty -> a stable pseudonymous handle (e.g. "operator-7f3a92") is
|
||||||
|
# auto-generated on first run and persisted to backend/data/operator_handle.json.
|
||||||
|
# Operators who want a meaningful handle (real name, org, GitHub login) can
|
||||||
|
# set it here. Special characters are sanitized to dashes.
|
||||||
|
# OPERATOR_HANDLE=
|
||||||
|
|
||||||
|
# Default outbound User-Agent for all third-party HTTP fetchers. Operators
|
||||||
|
# who run a public relay and want a completely custom UA can set this; it
|
||||||
|
# bypasses the per-operator helper entirely. Most installs should leave it
|
||||||
|
# unset and use OPERATOR_HANDLE instead.
|
||||||
|
# SHADOWBROKER_USER_AGENT=
|
||||||
|
|
||||||
|
# Nominatim-specific User-Agent override (OSM usage policy). Leave unset to
|
||||||
|
# use the per-install handle (default) — set only if you have a registered
|
||||||
|
# Nominatim relay identity.
|
||||||
|
# NOMINATIM_USER_AGENT=
|
||||||
|
|
||||||
|
# ── Third-party fetcher opt-ins ────────────────────────────────
|
||||||
|
# These data sources phone home to politically/commercially sensitive
|
||||||
|
# upstreams. Disabled by default; set to "true" only if the operator
|
||||||
|
# explicitly wants the node's IP to contact these services.
|
||||||
|
#
|
||||||
|
# CrowdThreat — backend.crowdthreat.world (paid threat-intel aggregator).
|
||||||
|
# CROWDTHREAT_ENABLED=false
|
||||||
|
#
|
||||||
|
# EUvsDisinfo FIMI — euvsdisinfo.eu (EU disinformation tracker).
|
||||||
|
# FIMI_ENABLED=false
|
||||||
|
#
|
||||||
|
# Polymarket + Kalshi — US political/election prediction markets.
|
||||||
|
# PREDICTION_MARKETS_ENABLED=false
|
||||||
|
#
|
||||||
|
# Finnhub fallback / yfinance — financial market data.
|
||||||
|
# Set FINNHUB_API_KEY to enable Finnhub, or set FINANCIAL_ENABLED=true to allow
|
||||||
|
# the unauthenticated yfinance fallback to call Yahoo Finance.
|
||||||
|
# FINANCIAL_ENABLED=false
|
||||||
|
#
|
||||||
|
# NUFORC UAP sightings — huggingface.co dataset download.
|
||||||
|
# NUFORC_ENABLED=false
|
||||||
|
#
|
||||||
|
# News RSS aggregator — defaults ON. Set to "false" to disable all
|
||||||
|
# configured news feeds (kill switch for the news layer).
|
||||||
|
# NEWS_ENABLED=true
|
||||||
|
|
||||||
# LTA Singapore traffic cameras — leave blank to skip this data source.
|
# LTA Singapore traffic cameras — leave blank to skip this data source.
|
||||||
# LTA_ACCOUNT_KEY=
|
# LTA_ACCOUNT_KEY=
|
||||||
@@ -61,8 +107,12 @@ AIS_API_KEY= # https://aisstream.io/ — free tier WebSocket key
|
|||||||
# Optional Meshtastic node ID (e.g. "!abcd1234"). When set, included in the
|
# Optional Meshtastic node ID (e.g. "!abcd1234"). When set, included in the
|
||||||
# User-Agent sent to meshtastic.liamcottle.net so the upstream service operator
|
# User-Agent sent to meshtastic.liamcottle.net so the upstream service operator
|
||||||
# can identify per-install traffic instead of aggregated "ShadowBroker" hits.
|
# can identify per-install traffic instead of aggregated "ShadowBroker" hits.
|
||||||
# Leave blank to send a generic UA with the project contact email only.
|
# Leave blank to send a generic UA. If you set MESHTASTIC_OPERATOR_CALLSIGN,
|
||||||
|
# it is included in outbound headers to meshtastic.org by default so they
|
||||||
|
# can rate-limit per-operator. Set MESHTASTIC_SEND_CALLSIGN_HEADER=false to
|
||||||
|
# suppress the callsign while still using it locally (e.g. for APRS).
|
||||||
# MESHTASTIC_OPERATOR_CALLSIGN=
|
# MESHTASTIC_OPERATOR_CALLSIGN=
|
||||||
|
# MESHTASTIC_SEND_CALLSIGN_HEADER=true
|
||||||
# MESH_MQTT_PSK= # hex-encoded, empty = default LongFast key
|
# MESH_MQTT_PSK= # hex-encoded, empty = default LongFast key
|
||||||
|
|
||||||
# ── Mesh / Reticulum (RNS) ─────────────────────────────────────
|
# ── Mesh / Reticulum (RNS) ─────────────────────────────────────
|
||||||
|
|||||||
+234
-4
@@ -1,5 +1,37 @@
|
|||||||
|
// AIS Stream WebSocket proxy.
|
||||||
|
//
|
||||||
|
// Reads AIS_API_KEY from argv or env, opens a wss:// connection to
|
||||||
|
// stream.aisstream.io, subscribes for vessel position reports inside the
|
||||||
|
// active map bounding boxes, and pipes JSON messages to stdout for the
|
||||||
|
// Python backend to ingest.
|
||||||
|
//
|
||||||
|
// Issue #258 — SPKI pinning fallback for upstream cert outages
|
||||||
|
// -------------------------------------------------------------
|
||||||
|
// AISStream uses Let's Encrypt and their renewal pipeline has been observed
|
||||||
|
// to fail (cert expired on 2026-05-20). The naive fix the issue reporter
|
||||||
|
// applied — passing { rejectUnauthorized: false } — turns off TLS validation
|
||||||
|
// entirely, which lets any network attacker MITM the WebSocket and inject
|
||||||
|
// fake ship positions onto the operator's map. Same class as the GDELT
|
||||||
|
// plaintext-HTTP MITM issue (#199).
|
||||||
|
//
|
||||||
|
// Instead, when the normal TLS handshake fails with CERT_HAS_EXPIRED, we
|
||||||
|
// do a custom TLS connection that ignores ONLY the expiry check, capture
|
||||||
|
// the leaf certificate, and compare its public-key SPKI hash against a
|
||||||
|
// pinned list (backend/data/aisstream_spki_pins.json). If the SPKI matches,
|
||||||
|
// the upstream is still the genuine AISStream — just with an expired cert —
|
||||||
|
// and we proceed in "degraded TLS" mode. If the SPKI does not match, we
|
||||||
|
// refuse the connection and log loudly: an actual MITM is in progress.
|
||||||
|
//
|
||||||
|
// Let's Encrypt renewals keep the same public key by default, so the pinned
|
||||||
|
// SPKI survives normal cert rotation. The pin list MUST be updated before
|
||||||
|
// the operator's pinned key is rotated upstream.
|
||||||
|
|
||||||
const WebSocket = require('ws');
|
const WebSocket = require('ws');
|
||||||
const readline = require('readline');
|
const readline = require('readline');
|
||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
const tls = require('tls');
|
||||||
|
const crypto = require('crypto');
|
||||||
|
|
||||||
const args = process.argv.slice(2);
|
const args = process.argv.slice(2);
|
||||||
const API_KEY = args[0] || process.env.AIS_API_KEY;
|
const API_KEY = args[0] || process.env.AIS_API_KEY;
|
||||||
@@ -9,6 +41,135 @@ if (!API_KEY) {
|
|||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── SPKI pin support (issue #258) ─────────────────────────────────────────
|
||||||
|
|
||||||
|
const AIS_HOST = 'stream.aisstream.io';
|
||||||
|
const AIS_PORT = 443;
|
||||||
|
const AIS_WS_URL = `wss://${AIS_HOST}/v0/stream`;
|
||||||
|
|
||||||
|
// Pin file is looked up in several layouts so the same JS works in:
|
||||||
|
// - the Docker backend image (PIN_FILE_CANDIDATES[0])
|
||||||
|
// - the Tauri desktop runtime (PIN_FILE_CANDIDATES[1])
|
||||||
|
// - a future relocated layout (operator can drop a file at
|
||||||
|
// SHADOWBROKER_AIS_PINS env var)
|
||||||
|
const PIN_FILE_CANDIDATES = [
|
||||||
|
process.env.SHADOWBROKER_AIS_PINS || '',
|
||||||
|
path.join(__dirname, 'data', 'aisstream_spki_pins.json'),
|
||||||
|
path.join(__dirname, 'aisstream_spki_pins.json'),
|
||||||
|
].filter(Boolean);
|
||||||
|
|
||||||
|
// Embedded fallback. Used when no external pin file is reachable so the
|
||||||
|
// SPKI fallback still works on minimal/portable installs. The external
|
||||||
|
// file (when present) takes priority so operators can update pins without
|
||||||
|
// needing a new build.
|
||||||
|
const EMBEDDED_PINS = {
|
||||||
|
[AIS_HOST]: [
|
||||||
|
// Captured 2026-05-20 from AISStream's leaf cert (Let's Encrypt R12).
|
||||||
|
// Replace when AISStream rotates server keys.
|
||||||
|
'GJ10H0UPgLrO+2d3ZXROR/TXSVFXKUfRC3QEI2ibEg4=',
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
let aisDegradedMode = false; // surfaced via stdout status_query marker
|
||||||
|
|
||||||
|
function loadSpkiPins() {
|
||||||
|
for (const candidate of PIN_FILE_CANDIDATES) {
|
||||||
|
try {
|
||||||
|
const raw = fs.readFileSync(candidate, 'utf-8');
|
||||||
|
const parsed = JSON.parse(raw);
|
||||||
|
const pins = Array.isArray(parsed[AIS_HOST]) ? parsed[AIS_HOST] : [];
|
||||||
|
const cleaned = pins
|
||||||
|
.filter((p) => typeof p === 'string' && p.length > 0)
|
||||||
|
.map((p) => p.trim());
|
||||||
|
if (cleaned.length > 0) {
|
||||||
|
return cleaned;
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
// Try the next candidate — file may not exist in this layout.
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const embedded = (EMBEDDED_PINS[AIS_HOST] || []).slice();
|
||||||
|
if (embedded.length > 0) {
|
||||||
|
console.error(
|
||||||
|
'[AIS Proxy] No external SPKI pin file found; using embedded fallback. '
|
||||||
|
+ `(Set SHADOWBROKER_AIS_PINS or drop ${PIN_FILE_CANDIDATES[1]} to override.)`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return embedded;
|
||||||
|
}
|
||||||
|
|
||||||
|
function spkiHashFromPeerCert(peerCert) {
|
||||||
|
// tls.TLSSocket.getPeerCertificate() exposes .pubkey when called with
|
||||||
|
// detailed=true. The pubkey buffer is the DER-encoded SubjectPublicKeyInfo,
|
||||||
|
// which is exactly the value we hash for SPKI pinning.
|
||||||
|
if (!peerCert || !peerCert.pubkey) return null;
|
||||||
|
return crypto.createHash('sha256').update(peerCert.pubkey).digest('base64');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Probe the upstream when normal TLS failed with CERT_HAS_EXPIRED. We open
|
||||||
|
// a raw TLS connection with rejectUnauthorized=false ONLY to inspect the
|
||||||
|
// leaf cert; we do NOT use this socket for the actual WebSocket traffic.
|
||||||
|
// Returns { ok: true } if the leaf SPKI matches the pin list, { ok: false }
|
||||||
|
// with a reason otherwise.
|
||||||
|
function verifyExpiredCertAgainstPins() {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const pins = loadSpkiPins();
|
||||||
|
if (pins.length === 0) {
|
||||||
|
resolve({ ok: false, reason: 'no SPKI pins configured' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const sock = tls.connect(
|
||||||
|
{
|
||||||
|
host: AIS_HOST,
|
||||||
|
port: AIS_PORT,
|
||||||
|
servername: AIS_HOST,
|
||||||
|
// Allow the handshake to complete despite the expired cert
|
||||||
|
// so we can inspect the leaf. We do NOT trust this connection
|
||||||
|
// for any application data.
|
||||||
|
rejectUnauthorized: false,
|
||||||
|
},
|
||||||
|
() => {
|
||||||
|
const peer = sock.getPeerCertificate(true);
|
||||||
|
sock.end();
|
||||||
|
if (!peer || Object.keys(peer).length === 0) {
|
||||||
|
resolve({ ok: false, reason: 'no peer certificate returned' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (peer.subject && peer.subject.CN !== AIS_HOST) {
|
||||||
|
resolve({
|
||||||
|
ok: false,
|
||||||
|
reason: `cert CN mismatch (got ${peer.subject.CN}, expected ${AIS_HOST})`,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const hash = spkiHashFromPeerCert(peer);
|
||||||
|
if (!hash) {
|
||||||
|
resolve({ ok: false, reason: 'could not compute SPKI hash from peer cert' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (pins.includes(hash)) {
|
||||||
|
resolve({ ok: true, hash });
|
||||||
|
} else {
|
||||||
|
resolve({
|
||||||
|
ok: false,
|
||||||
|
reason: `SPKI ${hash} not in pin list (possible MITM)`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
sock.setTimeout(10000, () => {
|
||||||
|
sock.destroy();
|
||||||
|
resolve({ ok: false, reason: 'TLS probe timeout' });
|
||||||
|
});
|
||||||
|
sock.on('error', (err) => {
|
||||||
|
resolve({ ok: false, reason: `TLS probe error: ${err.message}` });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Subscription state ───────────────────────────────────────────────────
|
||||||
|
|
||||||
// Start with global coverage, until frontend updates it
|
// Start with global coverage, until frontend updates it
|
||||||
let currentBboxes = [[[-90, -180], [90, 180]]];
|
let currentBboxes = [[[-90, -180], [90, 180]]];
|
||||||
let activeWs = null;
|
let activeWs = null;
|
||||||
@@ -42,14 +203,34 @@ rl.on('line', (line) => {
|
|||||||
currentBboxes = cmd.bboxes;
|
currentBboxes = cmd.bboxes;
|
||||||
if (activeWs) sendSub(activeWs); // Resend subscription (swap and replace)
|
if (activeWs) sendSub(activeWs); // Resend subscription (swap and replace)
|
||||||
}
|
}
|
||||||
|
if (cmd.type === "status_query") {
|
||||||
|
// Allow the Python side to probe degraded-mode state by sending
|
||||||
|
// {"type": "status_query"} on stdin. Reply on stdout as a marker.
|
||||||
|
process.stdout.write(JSON.stringify({
|
||||||
|
__ais_proxy_status: { degraded_tls: aisDegradedMode }
|
||||||
|
}) + '\n');
|
||||||
|
}
|
||||||
} catch (e) {}
|
} catch (e) {}
|
||||||
});
|
});
|
||||||
|
|
||||||
function connect() {
|
function attachWsHandlers(ws, { degraded } = { degraded: false }) {
|
||||||
const ws = new WebSocket('wss://stream.aisstream.io/v0/stream');
|
|
||||||
activeWs = ws;
|
activeWs = ws;
|
||||||
|
|
||||||
ws.on('open', () => {
|
ws.on('open', () => {
|
||||||
|
if (degraded) {
|
||||||
|
console.error(
|
||||||
|
'[AIS Proxy] Connected in DEGRADED TLS MODE — upstream cert is expired '
|
||||||
|
+ 'but SPKI matches the pinned key, so identity is still verified. '
|
||||||
|
+ 'AISStream needs to renew their cert; until then MITM protection '
|
||||||
|
+ 'depends only on the SPKI match. Watch backend logs for resolution.'
|
||||||
|
);
|
||||||
|
aisDegradedMode = true;
|
||||||
|
} else {
|
||||||
|
if (aisDegradedMode) {
|
||||||
|
console.error('[AIS Proxy] Reconnected with full TLS validation — degraded mode cleared.');
|
||||||
|
}
|
||||||
|
aisDegradedMode = false;
|
||||||
|
}
|
||||||
sendSub(ws);
|
sendSub(ws);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -61,14 +242,63 @@ function connect() {
|
|||||||
});
|
});
|
||||||
|
|
||||||
ws.on('error', (err) => {
|
ws.on('error', (err) => {
|
||||||
console.error("WebSocket Proxy Error:", err.message);
|
console.error('WebSocket Proxy Error:', err.message);
|
||||||
});
|
});
|
||||||
|
|
||||||
ws.on('close', () => {
|
ws.on('close', () => {
|
||||||
activeWs = null;
|
activeWs = null;
|
||||||
console.error("WebSocket Proxy Closed. Reconnecting in 5s...");
|
console.error('WebSocket Proxy Closed. Reconnecting in 5s...');
|
||||||
setTimeout(connect, 5000);
|
setTimeout(connect, 5000);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function connect() {
|
||||||
|
// Path A: normal TLS validation (the 99.9% case). If this succeeds we
|
||||||
|
// never touch the SPKI fallback.
|
||||||
|
const ws = new WebSocket(AIS_WS_URL);
|
||||||
|
|
||||||
|
let openedOk = false;
|
||||||
|
ws.on('open', () => { openedOk = true; });
|
||||||
|
|
||||||
|
ws.on('error', async (err) => {
|
||||||
|
// Only the CERT_HAS_EXPIRED case triggers SPKI verification. Any
|
||||||
|
// other TLS or network error gets the standard reconnect path so we
|
||||||
|
// don't accidentally cover up legitimate problems.
|
||||||
|
if (!openedOk && err && err.code === 'CERT_HAS_EXPIRED') {
|
||||||
|
console.error(
|
||||||
|
'[AIS Proxy] Upstream certificate is expired. Verifying SPKI '
|
||||||
|
+ 'against pinned keys before deciding whether to proceed in '
|
||||||
|
+ 'degraded mode...'
|
||||||
|
);
|
||||||
|
const verdict = await verifyExpiredCertAgainstPins();
|
||||||
|
if (verdict.ok) {
|
||||||
|
console.error(
|
||||||
|
`[AIS Proxy] SPKI ${verdict.hash} matches pinned key — `
|
||||||
|
+ 'identity is verified, proceeding in DEGRADED TLS mode.'
|
||||||
|
);
|
||||||
|
const insecureWs = new WebSocket(AIS_WS_URL, {
|
||||||
|
rejectUnauthorized: false,
|
||||||
|
});
|
||||||
|
attachWsHandlers(insecureWs, { degraded: true });
|
||||||
|
} else {
|
||||||
|
console.error(
|
||||||
|
`[AIS Proxy] SPKI verification FAILED (${verdict.reason}). `
|
||||||
|
+ 'Refusing to connect — this would normally indicate an active '
|
||||||
|
+ 'MITM attack. If AISStream rotated their server key, update '
|
||||||
|
+ 'backend/data/aisstream_spki_pins.json with the new SPKI hash.'
|
||||||
|
);
|
||||||
|
// Schedule a retry — operator may have updated the pin file.
|
||||||
|
setTimeout(connect, 60000);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Default: surface the error and let the close handler reconnect.
|
||||||
|
console.error('WebSocket Proxy Error:', err.message);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Wire normal handlers — these apply unless the error handler above
|
||||||
|
// takes over and replaces activeWs with an insecure socket.
|
||||||
|
attachWsHandlers(ws, { degraded: false });
|
||||||
|
}
|
||||||
|
|
||||||
connect();
|
connect();
|
||||||
|
|||||||
+91
-9
@@ -45,6 +45,7 @@ from services.mesh.mesh_compatibility import (
|
|||||||
from services.mesh.mesh_crypto import (
|
from services.mesh.mesh_crypto import (
|
||||||
_derive_peer_key,
|
_derive_peer_key,
|
||||||
normalize_peer_url,
|
normalize_peer_url,
|
||||||
|
resolve_peer_key_for_url,
|
||||||
verify_signature,
|
verify_signature,
|
||||||
verify_node_binding,
|
verify_node_binding,
|
||||||
parse_public_key_algo,
|
parse_public_key_algo,
|
||||||
@@ -245,15 +246,90 @@ def _docker_bridge_local_operator_enabled() -> bool:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Issue #250 (tg12): the previous implementation returned True for any IP
|
||||||
|
# in the entire 172.16.0.0/12 range. Anyone with `docker run` access on
|
||||||
|
# the same daemon could spin up a container that automatically passed
|
||||||
|
# local-operator auth. The fix narrows trust to ONLY connections whose
|
||||||
|
# source IP matches the configured frontend container's hostname.
|
||||||
|
#
|
||||||
|
# Docker DNS resolves both the compose service name (``frontend``) and
|
||||||
|
# the explicit ``container_name`` (``shadowbroker-frontend``) to the
|
||||||
|
# frontend container's bridge IP. We forward-resolve both, cache the
|
||||||
|
# result for 30s, and only trust connections from those exact IPs.
|
||||||
|
#
|
||||||
|
# Operators on shared Docker hosts get the benefit of the narrower
|
||||||
|
# surface. Operators on single-user installs see no behavior change —
|
||||||
|
# their frontend container still resolves and is still trusted.
|
||||||
|
_DOCKER_BRIDGE_TRUST_CACHE: dict = {"ips": frozenset(), "expires": 0.0}
|
||||||
|
_DOCKER_BRIDGE_TRUST_TTL = 30.0
|
||||||
|
|
||||||
|
|
||||||
|
def _trusted_bridge_frontend_hostnames() -> list[str]:
|
||||||
|
"""Container hostnames whose IPs we treat as local-operator on the bridge.
|
||||||
|
|
||||||
|
Default covers both Docker Compose service name (``frontend``) and the
|
||||||
|
explicit ``container_name`` from the shipped docker-compose.yml
|
||||||
|
(``shadowbroker-frontend``). Operators with non-default names can
|
||||||
|
override via the ``SHADOWBROKER_TRUSTED_FRONTEND_HOSTS`` env var
|
||||||
|
(comma-separated, no spaces).
|
||||||
|
"""
|
||||||
|
raw = str(
|
||||||
|
os.environ.get(
|
||||||
|
"SHADOWBROKER_TRUSTED_FRONTEND_HOSTS",
|
||||||
|
"frontend,shadowbroker-frontend",
|
||||||
|
)
|
||||||
|
).strip()
|
||||||
|
return [h.strip() for h in raw.split(",") if h.strip()]
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_trusted_bridge_ips() -> frozenset[str]:
|
||||||
|
"""Resolve trusted frontend hostnames to a set of IPs, with caching.
|
||||||
|
|
||||||
|
Cached for 30s so we don't hit DNS on every request. The cache is
|
||||||
|
process-local — frontend container IP rotations during a backend's
|
||||||
|
lifetime will be picked up within 30s.
|
||||||
|
|
||||||
|
Returns frozenset() if Docker DNS can't resolve any of the configured
|
||||||
|
hostnames (fail-closed — when in doubt, refuse to trust the bridge).
|
||||||
|
"""
|
||||||
|
import socket
|
||||||
|
import time as _time
|
||||||
|
|
||||||
|
now = _time.time()
|
||||||
|
cache = _DOCKER_BRIDGE_TRUST_CACHE
|
||||||
|
if cache["expires"] > now:
|
||||||
|
return cache["ips"]
|
||||||
|
|
||||||
|
ips: set[str] = set()
|
||||||
|
for hostname in _trusted_bridge_frontend_hostnames():
|
||||||
|
try:
|
||||||
|
_, _, addrs = socket.gethostbyname_ex(hostname)
|
||||||
|
except (OSError, socket.gaierror):
|
||||||
|
continue
|
||||||
|
for addr in addrs:
|
||||||
|
ips.add(addr)
|
||||||
|
|
||||||
|
resolved = frozenset(ips)
|
||||||
|
cache["ips"] = resolved
|
||||||
|
cache["expires"] = now + _DOCKER_BRIDGE_TRUST_TTL
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
def _is_docker_bridge_host(host: str) -> bool:
|
def _is_docker_bridge_host(host: str) -> bool:
|
||||||
|
"""Return True only when the source IP matches our trusted frontend
|
||||||
|
container hostname(s).
|
||||||
|
|
||||||
|
Previously trusted any 172.16.0.0/12 IP unconditionally. See the
|
||||||
|
block comment above for the security rationale.
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
ip = ipaddress.ip_address(host)
|
ip = ipaddress.ip_address(host)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
return False
|
return False
|
||||||
# Docker Desktop and the default compose bridge normally sit inside
|
# Public IPs are never our frontend container — skip DNS work for them.
|
||||||
# 172.16.0.0/12. Keep this narrower than "any private IP" so a user who
|
if not ip.is_private:
|
||||||
# intentionally binds the backend to LAN does not silently trust LAN clients.
|
return False
|
||||||
return ip in ipaddress.ip_network("172.16.0.0/12")
|
return host in _resolve_trusted_bridge_ips()
|
||||||
|
|
||||||
|
|
||||||
def _is_trusted_local_runtime_host(host: str) -> bool:
|
def _is_trusted_local_runtime_host(host: str) -> bool:
|
||||||
@@ -361,6 +437,8 @@ async def _verify_openclaw_hmac(request: Request) -> bool:
|
|||||||
# Bind request body: digest the raw bytes so any body tampering
|
# Bind request body: digest the raw bytes so any body tampering
|
||||||
# invalidates the signature. Empty/absent bodies hash as sha256(b"").
|
# invalidates the signature. Empty/absent bodies hash as sha256(b"").
|
||||||
body_bytes = await request.body()
|
body_bytes = await request.body()
|
||||||
|
# Keep the cached body available for downstream handlers that call request.json().
|
||||||
|
request._body = body_bytes
|
||||||
body_digest = _hashlib_mod.sha256(body_bytes).hexdigest()
|
body_digest = _hashlib_mod.sha256(body_bytes).hexdigest()
|
||||||
|
|
||||||
# Compute expected signature: HMAC-SHA256(secret, METHOD|path|ts|nonce|body_digest)
|
# Compute expected signature: HMAC-SHA256(secret, METHOD|path|ts|nonce|body_digest)
|
||||||
@@ -1326,11 +1404,15 @@ def _peer_hmac_url_from_request(request: Request) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def _verify_peer_push_hmac(request: Request, body_bytes: bytes) -> bool:
|
def _verify_peer_push_hmac(request: Request, body_bytes: bytes) -> bool:
|
||||||
"""Verify HMAC-SHA256 peer authentication on push requests."""
|
"""Verify HMAC-SHA256 peer authentication on push requests.
|
||||||
secret = str(get_settings().MESH_PEER_PUSH_SECRET or "").strip()
|
|
||||||
if not secret:
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
Issue #256: ``resolve_peer_key_for_url`` looks up a per-peer secret
|
||||||
|
in ``MESH_PEER_SECRETS`` first, then falls back to the global
|
||||||
|
``MESH_PEER_PUSH_SECRET``. When a peer URL is listed in the per-peer
|
||||||
|
map, only the listed secret is accepted for it — the global secret
|
||||||
|
is ignored, so any peer that knows only the global secret cannot
|
||||||
|
forge a request claiming to be that peer.
|
||||||
|
"""
|
||||||
provided = str(request.headers.get("x-peer-hmac", "") or "").strip()
|
provided = str(request.headers.get("x-peer-hmac", "") or "").strip()
|
||||||
if not provided:
|
if not provided:
|
||||||
return False
|
return False
|
||||||
@@ -1339,7 +1421,7 @@ def _verify_peer_push_hmac(request: Request, body_bytes: bytes) -> bool:
|
|||||||
allowed_peers = set(authenticated_push_peer_urls())
|
allowed_peers = set(authenticated_push_peer_urls())
|
||||||
if not peer_url or peer_url not in allowed_peers:
|
if not peer_url or peer_url not in allowed_peers:
|
||||||
return False
|
return False
|
||||||
peer_key = _derive_peer_key(secret, peer_url)
|
peer_key = resolve_peer_key_for_url(peer_url)
|
||||||
if not peer_key:
|
if not peer_key:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
{
|
||||||
|
"_comment": [
|
||||||
|
"SPKI (Subject Public Key Info) pin list for stream.aisstream.io.",
|
||||||
|
"",
|
||||||
|
"Issue #258: AISStream's Let's Encrypt cert expired on 2026-05-20 due to an",
|
||||||
|
"upstream renewal-pipeline failure. Disabling TLS verification entirely",
|
||||||
|
"would let any network attacker MITM the AIS WebSocket and inject fake",
|
||||||
|
"ship positions onto the operator's map (same class as #199 GDELT MITM).",
|
||||||
|
"Instead we pin the leaf certificate's public-key SPKI hash: if normal",
|
||||||
|
"TLS validation fails specifically with CERT_HAS_EXPIRED, ais_proxy.js",
|
||||||
|
"re-checks the leaf cert's SPKI against this list. A match means the",
|
||||||
|
"key is still the genuine AISStream key (Let's Encrypt renewals keep the",
|
||||||
|
"same key unless rekey is requested), so we proceed in 'degraded TLS'",
|
||||||
|
"mode. A mismatch means a real MITM attempt and we refuse the connection.",
|
||||||
|
"",
|
||||||
|
"Format: each entry is a SHA-256 hash of the DER-encoded SPKI bytes,",
|
||||||
|
"encoded as standard base64 (matches the format produced by:",
|
||||||
|
" openssl s_client -connect host:443 | \\",
|
||||||
|
" openssl x509 -pubkey -noout | openssl pkey -pubin -outform DER | \\",
|
||||||
|
" openssl dgst -sha256 -binary | openssl base64",
|
||||||
|
").",
|
||||||
|
"",
|
||||||
|
"When AISStream rotates their server key (rare — Let's Encrypt renewals",
|
||||||
|
"default to keeping the same key), capture the new SPKI and add it to",
|
||||||
|
"this list BEFORE removing the old one. That way operators on the old",
|
||||||
|
"code still validate against the previous key during the transition."
|
||||||
|
],
|
||||||
|
"stream.aisstream.io": [
|
||||||
|
"GJ10H0UPgLrO+2d3ZXROR/TXSVFXKUfRC3QEI2ibEg4="
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
{
|
||||||
|
"_meta": {
|
||||||
|
"as_of": "2026-03-09",
|
||||||
|
"source": "USNI News Fleet & Marine Tracker",
|
||||||
|
"source_url": "https://news.usni.org/2026/03/09/usni-news-fleet-and-marine-tracker-march-9-2026",
|
||||||
|
"note": "One-shot bootstrap for first-run carrier positions. Once carrier_cache.json exists in the runtime data volume, this seed file is never read again. All subsequent updates come from GDELT (and any future sources) and are written to carrier_cache.json. A year from now, your runtime cache reflects whatever your install has observed since first launch — not these snapshot positions."
|
||||||
|
},
|
||||||
|
"carriers": {
|
||||||
|
"CVN-68": {
|
||||||
|
"lat": 47.5535,
|
||||||
|
"lng": -122.6400,
|
||||||
|
"heading": 90,
|
||||||
|
"desc": "Bremerton, WA (Maintenance)",
|
||||||
|
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||||
|
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||||
|
"position_source_at": "2026-03-09T00:00:00Z",
|
||||||
|
"position_confidence": "seed"
|
||||||
|
},
|
||||||
|
"CVN-76": {
|
||||||
|
"lat": 47.5580,
|
||||||
|
"lng": -122.6360,
|
||||||
|
"heading": 90,
|
||||||
|
"desc": "Bremerton, WA (Decommissioning)",
|
||||||
|
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||||
|
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||||
|
"position_source_at": "2026-03-09T00:00:00Z",
|
||||||
|
"position_confidence": "seed"
|
||||||
|
},
|
||||||
|
"CVN-69": {
|
||||||
|
"lat": 36.9465,
|
||||||
|
"lng": -76.3265,
|
||||||
|
"heading": 0,
|
||||||
|
"desc": "Norfolk, VA (Post-deployment maintenance)",
|
||||||
|
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||||
|
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||||
|
"position_source_at": "2026-03-09T00:00:00Z",
|
||||||
|
"position_confidence": "seed"
|
||||||
|
},
|
||||||
|
"CVN-78": {
|
||||||
|
"lat": 18.0,
|
||||||
|
"lng": 39.5,
|
||||||
|
"heading": 0,
|
||||||
|
"desc": "Red Sea — Operation Epic Fury (USNI Mar 9)",
|
||||||
|
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||||
|
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||||
|
"position_source_at": "2026-03-09T00:00:00Z",
|
||||||
|
"position_confidence": "seed"
|
||||||
|
},
|
||||||
|
"CVN-74": {
|
||||||
|
"lat": 36.98,
|
||||||
|
"lng": -76.43,
|
||||||
|
"heading": 0,
|
||||||
|
"desc": "Newport News, VA (RCOH refueling overhaul)",
|
||||||
|
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||||
|
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||||
|
"position_source_at": "2026-03-09T00:00:00Z",
|
||||||
|
"position_confidence": "seed"
|
||||||
|
},
|
||||||
|
"CVN-75": {
|
||||||
|
"lat": 36.0,
|
||||||
|
"lng": 15.0,
|
||||||
|
"heading": 0,
|
||||||
|
"desc": "Mediterranean Sea deployment (USNI Mar 9)",
|
||||||
|
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||||
|
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||||
|
"position_source_at": "2026-03-09T00:00:00Z",
|
||||||
|
"position_confidence": "seed"
|
||||||
|
},
|
||||||
|
"CVN-77": {
|
||||||
|
"lat": 36.5,
|
||||||
|
"lng": -74.0,
|
||||||
|
"heading": 0,
|
||||||
|
"desc": "Atlantic — Pre-deployment workups (USNI Mar 9)",
|
||||||
|
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||||
|
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||||
|
"position_source_at": "2026-03-09T00:00:00Z",
|
||||||
|
"position_confidence": "seed"
|
||||||
|
},
|
||||||
|
"CVN-70": {
|
||||||
|
"lat": 32.6840,
|
||||||
|
"lng": -117.1290,
|
||||||
|
"heading": 180,
|
||||||
|
"desc": "San Diego, CA (Homeport)",
|
||||||
|
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||||
|
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||||
|
"position_source_at": "2026-03-09T00:00:00Z",
|
||||||
|
"position_confidence": "seed"
|
||||||
|
},
|
||||||
|
"CVN-71": {
|
||||||
|
"lat": 32.6885,
|
||||||
|
"lng": -117.1280,
|
||||||
|
"heading": 180,
|
||||||
|
"desc": "San Diego, CA (Maintenance)",
|
||||||
|
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||||
|
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||||
|
"position_source_at": "2026-03-09T00:00:00Z",
|
||||||
|
"position_confidence": "seed"
|
||||||
|
},
|
||||||
|
"CVN-72": {
|
||||||
|
"lat": 20.0,
|
||||||
|
"lng": 64.0,
|
||||||
|
"heading": 0,
|
||||||
|
"desc": "Arabian Sea — Operation Epic Fury (USNI Mar 9)",
|
||||||
|
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||||
|
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||||
|
"position_source_at": "2026-03-09T00:00:00Z",
|
||||||
|
"position_confidence": "seed"
|
||||||
|
},
|
||||||
|
"CVN-73": {
|
||||||
|
"lat": 35.2830,
|
||||||
|
"lng": 139.6700,
|
||||||
|
"heading": 180,
|
||||||
|
"desc": "Yokosuka, Japan (Forward deployed)",
|
||||||
|
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||||
|
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||||
|
"position_source_at": "2026-03-09T00:00:00Z",
|
||||||
|
"position_confidence": "seed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,40 @@
|
|||||||
|
{
|
||||||
|
"_comment": [
|
||||||
|
"Baked-in SHA-256 digests for known Shadowbroker release archives.",
|
||||||
|
"",
|
||||||
|
"Issue #231: the self-updater previously skipped integrity verification",
|
||||||
|
"entirely whenever the MESH_UPDATE_SHA256 env var was unset (which is the",
|
||||||
|
"default — nothing in the install docs tells operators to set it). That",
|
||||||
|
"made the auto-update a supply-chain RCE on any compromise of the GitHub",
|
||||||
|
"release pipeline.",
|
||||||
|
"",
|
||||||
|
"The fix uses a multi-source verification chain mirroring the Tor bundle",
|
||||||
|
"digest approach in #201:",
|
||||||
|
"",
|
||||||
|
" 1. MESH_UPDATE_SHA256 env var (operator override, preserved)",
|
||||||
|
" 2. SHA256SUMS.txt asset published alongside each release (primary —",
|
||||||
|
" the maintainer's release process already publishes this)",
|
||||||
|
" 3. This baked-in digest list (second line of defense for releases",
|
||||||
|
" missing a SHA256SUMS asset, or when the asset can't be fetched)",
|
||||||
|
" 4. HTTPS-only fallback with a loud warning (preserves auto-update",
|
||||||
|
" flow during transient outages so users don't get stuck)",
|
||||||
|
"",
|
||||||
|
"Mismatch from a source that DID respond is fatal — the update is",
|
||||||
|
"refused and the existing install keeps running. Only the 'no source",
|
||||||
|
"reachable at all' case falls back to HTTPS-only.",
|
||||||
|
"",
|
||||||
|
"Format: each entry is keyed by release tag and maps asset filenames",
|
||||||
|
"to their canonical SHA-256 digest (hex, lowercase). The updater",
|
||||||
|
"compares the locally-computed digest of the downloaded asset against",
|
||||||
|
"the value here.",
|
||||||
|
"",
|
||||||
|
"When the maintainer ships a new release, add its digests here BEFORE",
|
||||||
|
"removing the old ones so operators on the old code still validate",
|
||||||
|
"against the previous entries during the transition."
|
||||||
|
],
|
||||||
|
"v0.9.79": {
|
||||||
|
"ShadowBroker_v0.9.79.zip": "f6877c1d66614525315ea82636ce9f7b41178332c4dbf90d27431a1ea1d9cd47",
|
||||||
|
"ShadowBroker_0.9.79_x64-setup.exe": "f7b676ada45cac7da05868b0a353678c9ee700e3abcf456a7c0c038c36da446f",
|
||||||
|
"ShadowBroker_0.9.79_x64_en-US.msi": "e0713c3cdda184cfbea750bfac0d62a35678fec00847e6476f2cac8e7e42046e"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
{
|
||||||
|
"_comment": [
|
||||||
|
"Pinned SHA-256 digests for the Tor Expert Bundle archives we know how to install.",
|
||||||
|
"Used as the LAST-RESORT verification source when the upstream .sha256sum file is",
|
||||||
|
"unreachable, MITM'd, or doesn't match what we downloaded. Issue #201.",
|
||||||
|
"",
|
||||||
|
"Each entry is keyed by the archive URL (so multiple platforms / versions",
|
||||||
|
"can share this one file) and contains the canonical SHA-256 we trust.",
|
||||||
|
"",
|
||||||
|
"When the project tests a new Tor release, add its digest here in the same",
|
||||||
|
"PR that bumps _TOR_EXPERT_BUNDLE_URLS. Old entries are kept indefinitely so",
|
||||||
|
"users on older versions keep working — we only ever ADD here, never remove."
|
||||||
|
],
|
||||||
|
"https://dist.torproject.org/torbrowser/15.0.11/tor-expert-bundle-windows-x86_64-15.0.11.tar.gz": "PLACEHOLDER_REPLACE_BEFORE_RELEASE",
|
||||||
|
"https://dist.torproject.org/torbrowser/15.0.8/tor-expert-bundle-windows-x86_64-15.0.8.tar.gz": "PLACEHOLDER_REPLACE_BEFORE_RELEASE"
|
||||||
|
}
|
||||||
@@ -14,4 +14,9 @@ if [ -d /app/image-data ]; then
|
|||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [ -z "${PRIVACY_CORE_ALLOWED_SHA256:-}" ] && [ -f /app/libprivacy_core.so ]; then
|
||||||
|
PRIVACY_CORE_ALLOWED_SHA256="$(sha256sum /app/libprivacy_core.so | awk '{print $1}')"
|
||||||
|
export PRIVACY_CORE_ALLOWED_SHA256
|
||||||
|
fi
|
||||||
|
|
||||||
exec "$@"
|
exec "$@"
|
||||||
|
|||||||
+105
-1
@@ -1,4 +1,108 @@
|
|||||||
|
"""Rate-limit key function for slowapi.
|
||||||
|
|
||||||
|
Issue #287 (tg12): the previous implementation used
|
||||||
|
``slowapi.util.get_remote_address`` which only ever returns
|
||||||
|
``request.client.host``. Behind the bundled Next.js proxy (or any other
|
||||||
|
reverse proxy), every connected operator's ``client.host`` is the
|
||||||
|
frontend container's bridge IP. ``@limiter.limit("120/minute")`` then
|
||||||
|
collapses into one shared bucket for everybody on the same backend —
|
||||||
|
one heavy tab can starve every other operator on the node.
|
||||||
|
|
||||||
|
This module replaces that key function with one that:
|
||||||
|
|
||||||
|
* Reads ``X-Forwarded-For`` ONLY when the immediate peer is a trusted
|
||||||
|
frontend container (same allowlist used by the Docker bridge
|
||||||
|
local-operator trust path — see ``backend/auth.py`` ``#250``).
|
||||||
|
* Picks the FIRST entry in the XFF chain. That's the client end of
|
||||||
|
the proxy chain, which is the operator we want to bucket on.
|
||||||
|
* Falls back to ``request.client.host`` for any peer that isn't on
|
||||||
|
the trusted-frontend allowlist. Direct hits, unrelated containers,
|
||||||
|
and unknown hosts are bucketed exactly like before — there is no
|
||||||
|
way for an untrusted caller to spoof XFF and steal another
|
||||||
|
operator's rate-limit bucket.
|
||||||
|
|
||||||
|
Single-operator nodes are unaffected: the frontend resolves to one IP,
|
||||||
|
that IP is on the trust list, the XFF header is read, and you get one
|
||||||
|
bucket per operator (i.e. you).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from slowapi import Limiter
|
from slowapi import Limiter
|
||||||
from slowapi.util import get_remote_address
|
from slowapi.util import get_remote_address
|
||||||
|
|
||||||
limiter = Limiter(key_func=get_remote_address)
|
|
||||||
|
def _client_host(request: Any) -> str:
|
||||||
|
"""Return the immediate peer's IP, normalised to a lowercase string."""
|
||||||
|
client = getattr(request, "client", None)
|
||||||
|
if client is None:
|
||||||
|
return ""
|
||||||
|
host = getattr(client, "host", "") or ""
|
||||||
|
return host.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def _first_forwarded_for(value: str) -> str:
|
||||||
|
"""Return the first non-empty entry from an ``X-Forwarded-For`` header.
|
||||||
|
|
||||||
|
RFC 7239 / de-facto XFF format is ``client, proxy1, proxy2, …``. The
|
||||||
|
client end is what we want to bucket on. Empty parts (which appear
|
||||||
|
in some malformed headers) are skipped so we don't end up keying on
|
||||||
|
an empty string.
|
||||||
|
"""
|
||||||
|
for raw in value.split(","):
|
||||||
|
candidate = raw.strip()
|
||||||
|
if candidate:
|
||||||
|
return candidate.lower()
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def _is_trusted_frontend_peer(host: str) -> bool:
|
||||||
|
"""True iff ``host`` is one of the resolved trusted-frontend IPs.
|
||||||
|
|
||||||
|
Imported lazily so this module stays usable in unit tests that
|
||||||
|
don't want to pull the whole auth module into scope.
|
||||||
|
"""
|
||||||
|
if not host:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
from auth import _resolve_trusted_bridge_ips
|
||||||
|
except Exception: # pragma: no cover - defensive
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
trusted_ips = _resolve_trusted_bridge_ips()
|
||||||
|
except Exception: # pragma: no cover - defensive
|
||||||
|
return False
|
||||||
|
return host in trusted_ips
|
||||||
|
|
||||||
|
|
||||||
|
def shadowbroker_rate_limit_key(request: Any) -> str:
|
||||||
|
"""slowapi key_func that is proxy-aware on trusted frontend peers only.
|
||||||
|
|
||||||
|
Behaviour matrix:
|
||||||
|
|
||||||
|
* Direct loopback / unknown peer → ``request.client.host``
|
||||||
|
(identical to slowapi's default ``get_remote_address``).
|
||||||
|
* Peer is a trusted frontend container AND ``X-Forwarded-For`` is
|
||||||
|
present → first XFF entry (the actual operator).
|
||||||
|
* Peer is a trusted frontend container but no XFF → fall back to
|
||||||
|
``request.client.host`` (the bridge IP). One shared bucket for
|
||||||
|
everyone in that case, same as before — but you only get there
|
||||||
|
if the trusted frontend forgot to forward XFF, which it won't.
|
||||||
|
"""
|
||||||
|
peer = _client_host(request)
|
||||||
|
if _is_trusted_frontend_peer(peer):
|
||||||
|
headers = getattr(request, "headers", None)
|
||||||
|
if headers is not None:
|
||||||
|
xff = headers.get("x-forwarded-for") or headers.get("X-Forwarded-For")
|
||||||
|
if xff:
|
||||||
|
first = _first_forwarded_for(xff)
|
||||||
|
if first:
|
||||||
|
return first
|
||||||
|
# Untrusted peer (or trusted peer without XFF): match the original
|
||||||
|
# get_remote_address behaviour byte-for-byte.
|
||||||
|
return get_remote_address(request)
|
||||||
|
|
||||||
|
|
||||||
|
limiter = Limiter(key_func=shadowbroker_rate_limit_key)
|
||||||
|
|||||||
+193
-44
@@ -14,7 +14,7 @@ from dataclasses import dataclass, field
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
from json import JSONDecodeError
|
from json import JSONDecodeError
|
||||||
|
|
||||||
APP_VERSION = "0.9.75"
|
APP_VERSION = "0.9.79"
|
||||||
|
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -220,6 +220,7 @@ from services.mesh.mesh_crypto import (
|
|||||||
_derive_peer_key,
|
_derive_peer_key,
|
||||||
derive_node_id,
|
derive_node_id,
|
||||||
normalize_peer_url,
|
normalize_peer_url,
|
||||||
|
resolve_peer_key_for_url,
|
||||||
verify_node_binding,
|
verify_node_binding,
|
||||||
parse_public_key_algo,
|
parse_public_key_algo,
|
||||||
)
|
)
|
||||||
@@ -1079,8 +1080,18 @@ def _public_mesh_log_size(entries: list[dict[str, Any]]) -> int:
|
|||||||
return sum(1 for item in entries if _public_mesh_log_entry(item) is not None)
|
return sum(1 for item in entries if _public_mesh_log_entry(item) is not None)
|
||||||
|
|
||||||
|
|
||||||
_WORMHOLE_PUBLIC_SETTINGS_FIELDS = {"enabled", "transport", "anonymous_mode"}
|
# Issue #243 (tg12): the public redaction now exposes only the bare
|
||||||
_WORMHOLE_PUBLIC_PROFILE_FIELDS = {"profile", "wormhole_enabled"}
|
# "is Wormhole on?" boolean. Transport choice (tor/i2p/mixnet/direct),
|
||||||
|
# anonymous-mode state, and the named privacy profile are all
|
||||||
|
# operational posture and were leaking actionable recon to any
|
||||||
|
# unauthenticated caller. They are now gated behind authenticated reads
|
||||||
|
# (admin key or scoped-view token). Loopback Tauri shells and Docker
|
||||||
|
# bridge frontend containers continue to see full status because the
|
||||||
|
# Next.js catch-all proxy injects the configured ADMIN_KEY for
|
||||||
|
# same-origin/non-browser callers (see PR #263), so legitimate operator
|
||||||
|
# UX is unaffected.
|
||||||
|
_WORMHOLE_PUBLIC_SETTINGS_FIELDS = {"enabled"}
|
||||||
|
_WORMHOLE_PUBLIC_PROFILE_FIELDS = {"wormhole_enabled"}
|
||||||
_PRIVATE_LANE_CONTROL_FIELDS = {"private_lane_tier", "private_lane_policy"}
|
_PRIVATE_LANE_CONTROL_FIELDS = {"private_lane_tier", "private_lane_policy"}
|
||||||
_PUBLIC_RNS_STATUS_FIELDS = {"enabled", "ready", "configured_peers", "active_peers"}
|
_PUBLIC_RNS_STATUS_FIELDS = {"enabled", "ready", "configured_peers", "active_peers"}
|
||||||
_NODE_PUBLIC_EVENT_HOOK_REGISTERED = False
|
_NODE_PUBLIC_EVENT_HOOK_REGISTERED = False
|
||||||
@@ -1386,7 +1397,12 @@ def _peer_sync_response(peer_url: str, body: dict[str, Any]) -> dict[str, Any]:
|
|||||||
if _infonet_private_transport_required() and not _is_private_infonet_transport(transport):
|
if _infonet_private_transport_required() and not _is_private_infonet_transport(transport):
|
||||||
raise RuntimeError(_infonet_private_transport_error())
|
raise RuntimeError(_infonet_private_transport_error())
|
||||||
|
|
||||||
timeout = int(get_settings().MESH_RELAY_PUSH_TIMEOUT_S or 10)
|
settings = get_settings()
|
||||||
|
timeout = int(
|
||||||
|
getattr(settings, "MESH_SYNC_TIMEOUT_S", 0)
|
||||||
|
or getattr(settings, "MESH_RELAY_PUSH_TIMEOUT_S", 0)
|
||||||
|
or 10
|
||||||
|
)
|
||||||
kwargs: dict[str, Any] = {
|
kwargs: dict[str, Any] = {
|
||||||
"json": body,
|
"json": body,
|
||||||
"timeout": timeout,
|
"timeout": timeout,
|
||||||
@@ -1509,6 +1525,8 @@ def _run_public_sync_cycle() -> SyncWorkerState:
|
|||||||
|
|
||||||
records = _filter_infonet_sync_records(store.records())
|
records = _filter_infonet_sync_records(store.records())
|
||||||
peers = eligible_sync_peers(records, now=time.time())
|
peers = eligible_sync_peers(records, now=time.time())
|
||||||
|
max_peers = max(1, int(getattr(get_settings(), "MESH_SYNC_MAX_PEERS_PER_CYCLE", 0) or 3))
|
||||||
|
peers = peers[:max_peers]
|
||||||
with _NODE_RUNTIME_LOCK:
|
with _NODE_RUNTIME_LOCK:
|
||||||
current_state = get_sync_state()
|
current_state = get_sync_state()
|
||||||
if not peers:
|
if not peers:
|
||||||
@@ -1571,14 +1589,25 @@ def _run_public_sync_cycle() -> SyncWorkerState:
|
|||||||
return updated
|
return updated
|
||||||
|
|
||||||
last_error = error
|
last_error = error
|
||||||
|
settings = get_settings()
|
||||||
|
is_seed_peer = str(getattr(record, "role", "") or "").strip().lower() == "seed"
|
||||||
|
cooldown_s = int(getattr(settings, "MESH_RELAY_FAILURE_COOLDOWN_S", 120) or 120)
|
||||||
|
if is_seed_peer:
|
||||||
|
cooldown_s = int(
|
||||||
|
getattr(settings, "MESH_BOOTSTRAP_SEED_FAILURE_COOLDOWN_S", cooldown_s)
|
||||||
|
or cooldown_s
|
||||||
|
)
|
||||||
store.mark_failure(
|
store.mark_failure(
|
||||||
record.peer_url,
|
record.peer_url,
|
||||||
"sync",
|
"sync",
|
||||||
error=error,
|
error=error,
|
||||||
cooldown_s=int(get_settings().MESH_RELAY_FAILURE_COOLDOWN_S or 120),
|
cooldown_s=cooldown_s,
|
||||||
now=time.time(),
|
now=time.time(),
|
||||||
)
|
)
|
||||||
store.save()
|
store.save()
|
||||||
|
failure_backoff_s = int(settings.MESH_SYNC_FAILURE_BACKOFF_S or 60)
|
||||||
|
if is_seed_peer:
|
||||||
|
failure_backoff_s = min(failure_backoff_s, max(1, cooldown_s))
|
||||||
updated = finish_sync(
|
updated = finish_sync(
|
||||||
started,
|
started,
|
||||||
ok=False,
|
ok=False,
|
||||||
@@ -1588,7 +1617,7 @@ def _run_public_sync_cycle() -> SyncWorkerState:
|
|||||||
fork_detected=forked,
|
fork_detected=forked,
|
||||||
now=time.time(),
|
now=time.time(),
|
||||||
interval_s=int(get_settings().MESH_SYNC_INTERVAL_S or 300),
|
interval_s=int(get_settings().MESH_SYNC_INTERVAL_S or 300),
|
||||||
failure_backoff_s=int(get_settings().MESH_SYNC_FAILURE_BACKOFF_S or 60),
|
failure_backoff_s=failure_backoff_s,
|
||||||
)
|
)
|
||||||
with _NODE_RUNTIME_LOCK:
|
with _NODE_RUNTIME_LOCK:
|
||||||
set_sync_state(updated)
|
set_sync_state(updated)
|
||||||
@@ -1727,10 +1756,12 @@ def _http_peer_push_loop() -> None:
|
|||||||
_NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S)
|
_NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
secret = str(get_settings().MESH_PEER_PUSH_SECRET or "").strip()
|
# Issue #256: resolve_peer_key_for_url() handles both the
|
||||||
if not secret:
|
# legacy global MESH_PEER_PUSH_SECRET path and the per-peer
|
||||||
_NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S)
|
# MESH_PEER_SECRETS map. The per-peer skip happens below
|
||||||
continue
|
# ("if not peer_key: continue"), so we don't gate the whole
|
||||||
|
# loop on the global secret being set — an install that only
|
||||||
|
# configures per-peer secrets is now valid.
|
||||||
|
|
||||||
peers = authenticated_push_peer_urls()
|
peers = authenticated_push_peer_urls()
|
||||||
if not peers:
|
if not peers:
|
||||||
@@ -1760,7 +1791,7 @@ def _http_peer_push_loop() -> None:
|
|||||||
ensure_ascii=False,
|
ensure_ascii=False,
|
||||||
).encode("utf-8")
|
).encode("utf-8")
|
||||||
|
|
||||||
peer_key = _derive_peer_key(secret, normalized)
|
peer_key = resolve_peer_key_for_url(normalized)
|
||||||
if not peer_key:
|
if not peer_key:
|
||||||
continue
|
continue
|
||||||
import hmac as _hmac_mod2
|
import hmac as _hmac_mod2
|
||||||
@@ -1813,10 +1844,7 @@ def _http_gate_pull_loop() -> None:
|
|||||||
_NODE_SYNC_STOP.wait(_GATE_PULL_INTERVAL_S)
|
_NODE_SYNC_STOP.wait(_GATE_PULL_INTERVAL_S)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
secret = str(get_settings().MESH_PEER_PUSH_SECRET or "").strip()
|
# Issue #256: per-peer key resolution; see _http_peer_push_loop.
|
||||||
if not secret:
|
|
||||||
_NODE_SYNC_STOP.wait(_GATE_PULL_INTERVAL_S)
|
|
||||||
continue
|
|
||||||
|
|
||||||
peers = authenticated_push_peer_urls()
|
peers = authenticated_push_peer_urls()
|
||||||
if not peers:
|
if not peers:
|
||||||
@@ -1828,7 +1856,7 @@ def _http_gate_pull_loop() -> None:
|
|||||||
if not normalized:
|
if not normalized:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
peer_key = _derive_peer_key(secret, normalized)
|
peer_key = resolve_peer_key_for_url(normalized)
|
||||||
if not peer_key:
|
if not peer_key:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -1941,10 +1969,7 @@ def _http_gate_push_loop() -> None:
|
|||||||
_NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S)
|
_NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
secret = str(get_settings().MESH_PEER_PUSH_SECRET or "").strip()
|
# Issue #256: per-peer key resolution; see _http_peer_push_loop.
|
||||||
if not secret:
|
|
||||||
_NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S)
|
|
||||||
continue
|
|
||||||
|
|
||||||
peers = authenticated_push_peer_urls()
|
peers = authenticated_push_peer_urls()
|
||||||
if not peers:
|
if not peers:
|
||||||
@@ -1959,7 +1984,7 @@ def _http_gate_push_loop() -> None:
|
|||||||
if not normalized:
|
if not normalized:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
peer_key = _derive_peer_key(secret, normalized)
|
peer_key = resolve_peer_key_for_url(normalized)
|
||||||
if not peer_key:
|
if not peer_key:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -3043,6 +3068,17 @@ def _request_private_surface_warmup(*, path: str, method: str, current_tier: str
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_invite_scoped_prekey_bundle_lookup(request: Request, path: str) -> bool:
|
||||||
|
if request.method.upper() != "GET" or str(path or "").strip() != "/api/mesh/dm/prekey-bundle":
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
lookup_token = str(request.query_params.get("lookup_token", "") or "").strip()
|
||||||
|
agent_id = str(request.query_params.get("agent_id", "") or "").strip()
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
return bool(lookup_token) and not agent_id
|
||||||
|
|
||||||
|
|
||||||
def _resume_private_delivery_background_work(*, current_tier: str, reason: str) -> None:
|
def _resume_private_delivery_background_work(*, current_tier: str, reason: str) -> None:
|
||||||
pending_items = private_delivery_outbox.pending_items()
|
pending_items = private_delivery_outbox.pending_items()
|
||||||
if not pending_items:
|
if not pending_items:
|
||||||
@@ -3061,6 +3097,24 @@ def _resume_private_delivery_background_work(*, current_tier: str, reason: str)
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_public_meshtastic_lane_path(path: str, method: str) -> bool:
|
||||||
|
"""Routes for the public Meshtastic MQTT lane.
|
||||||
|
|
||||||
|
These are intentionally outside the Wormhole/Infonet private transport
|
||||||
|
lifecycle. Polling public MeshChat must not wake or re-enable Wormhole.
|
||||||
|
"""
|
||||||
|
normalized_path = str(path or "").strip()
|
||||||
|
method_name = str(method or "").upper()
|
||||||
|
if method_name == "POST" and normalized_path == "/api/mesh/meshtastic/send":
|
||||||
|
return True
|
||||||
|
if method_name == "GET" and normalized_path in {
|
||||||
|
"/api/mesh/messages",
|
||||||
|
"/api/mesh/channels",
|
||||||
|
}:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def _upgrade_invite_scoped_contact_preferences_background() -> dict[str, Any]:
|
def _upgrade_invite_scoped_contact_preferences_background() -> dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
from services.mesh.mesh_wormhole_contacts import upgrade_invite_scoped_contact_preferences
|
from services.mesh.mesh_wormhole_contacts import upgrade_invite_scoped_contact_preferences
|
||||||
@@ -3092,7 +3146,11 @@ def _refresh_lookup_handle_rotation_background(*, reason: str) -> dict[str, Any]
|
|||||||
@app.middleware("http")
|
@app.middleware("http")
|
||||||
async def enforce_high_privacy_mesh(request: Request, call_next):
|
async def enforce_high_privacy_mesh(request: Request, call_next):
|
||||||
path = request.url.path
|
path = request.url.path
|
||||||
if path.startswith("/api/mesh") or path.startswith("/api/wormhole/gate/") or path.startswith("/api/wormhole/dm/"):
|
private_mesh_path = path.startswith("/api/mesh") and not _is_public_meshtastic_lane_path(
|
||||||
|
path,
|
||||||
|
request.method,
|
||||||
|
)
|
||||||
|
if private_mesh_path or path.startswith("/api/wormhole/gate/") or path.startswith("/api/wormhole/dm/"):
|
||||||
request.state._private_lane_started_at = time.perf_counter()
|
request.state._private_lane_started_at = time.perf_counter()
|
||||||
current_tier = "public_degraded"
|
current_tier = "public_degraded"
|
||||||
try:
|
try:
|
||||||
@@ -3151,6 +3209,17 @@ async def enforce_high_privacy_mesh(request: Request, call_next):
|
|||||||
# transport has not finished coming up yet.
|
# transport has not finished coming up yet.
|
||||||
request.state._private_control_transport_pending = current_tier == "public_degraded"
|
request.state._private_control_transport_pending = current_tier == "public_degraded"
|
||||||
request.state._private_lane_current_tier = current_tier
|
request.state._private_lane_current_tier = current_tier
|
||||||
|
elif _is_invite_scoped_prekey_bundle_lookup(request, path):
|
||||||
|
# A copied DM address carries a high-entropy invite lookup
|
||||||
|
# handle. Returning the public prekey bundle for that
|
||||||
|
# handle is the bootstrap step that lets first contact get
|
||||||
|
# saved; blocking it behind the full private lane creates a
|
||||||
|
# circular warm-up failure. Stable agent_id lookup still
|
||||||
|
# follows the normal transport-tier policy.
|
||||||
|
request.state._invite_prekey_lookup_transport_pending = (
|
||||||
|
current_tier == "public_degraded"
|
||||||
|
)
|
||||||
|
request.state._private_lane_current_tier = current_tier
|
||||||
else:
|
else:
|
||||||
# Tor-style: instead of failing, keep trying in the
|
# Tor-style: instead of failing, keep trying in the
|
||||||
# background and return an ok:True "preparing" response
|
# background and return an ok:True "preparing" response
|
||||||
@@ -3193,7 +3262,7 @@ async def enforce_high_privacy_mesh(request: Request, call_next):
|
|||||||
# Don't block the request on the upgrade — the transport
|
# Don't block the request on the upgrade — the transport
|
||||||
# manager will converge in the background.
|
# manager will converge in the background.
|
||||||
if (
|
if (
|
||||||
path.startswith("/api/mesh")
|
private_mesh_path
|
||||||
and str(data.get("privacy_profile", "default")).lower() == "high"
|
and str(data.get("privacy_profile", "default")).lower() == "high"
|
||||||
and not bool(data.get("enabled"))
|
and not bool(data.get("enabled"))
|
||||||
):
|
):
|
||||||
@@ -3283,7 +3352,7 @@ async def force_refresh(request: Request):
|
|||||||
return {"status": "refreshing in background"}
|
return {"status": "refreshing in background"}
|
||||||
|
|
||||||
|
|
||||||
@app.post("/api/ais/feed")
|
@app.post("/api/ais/feed", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("60/minute")
|
@limiter.limit("60/minute")
|
||||||
async def ais_feed(request: Request):
|
async def ais_feed(request: Request):
|
||||||
"""Accept AIS-catcher HTTP JSON feed (POST decoded AIS messages)."""
|
"""Accept AIS-catcher HTTP JSON feed (POST decoded AIS messages)."""
|
||||||
@@ -3378,7 +3447,7 @@ class LayerUpdate(BaseModel):
|
|||||||
layers: dict[str, bool]
|
layers: dict[str, bool]
|
||||||
|
|
||||||
|
|
||||||
@app.post("/api/layers")
|
@app.post("/api/layers", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def update_layers(update: LayerUpdate, request: Request):
|
async def update_layers(update: LayerUpdate, request: Request):
|
||||||
"""Receive frontend layer toggle state. Starts/stops streams accordingly."""
|
"""Receive frontend layer toggle state. Starts/stops streams accordingly."""
|
||||||
@@ -3426,8 +3495,16 @@ async def update_layers(update: LayerUpdate, request: Request):
|
|||||||
from services.sigint_bridge import sigint_grid
|
from services.sigint_bridge import sigint_grid
|
||||||
|
|
||||||
if old_mesh and not new_mesh:
|
if old_mesh and not new_mesh:
|
||||||
sigint_grid.mesh.stop()
|
try:
|
||||||
logger.info("Meshtastic MQTT bridge stopped (layer disabled)")
|
from services.meshtastic_mqtt_settings import mqtt_bridge_enabled
|
||||||
|
keep_chat_running = mqtt_bridge_enabled()
|
||||||
|
except Exception:
|
||||||
|
keep_chat_running = False
|
||||||
|
if keep_chat_running:
|
||||||
|
logger.info("Meshtastic map layer disabled; MQTT bridge kept running for MeshChat")
|
||||||
|
else:
|
||||||
|
sigint_grid.mesh.stop()
|
||||||
|
logger.info("Meshtastic MQTT bridge stopped (layer disabled)")
|
||||||
elif not old_mesh and new_mesh:
|
elif not old_mesh and new_mesh:
|
||||||
# Respect the global MESH_MQTT_ENABLED gate even when the UI layer is
|
# Respect the global MESH_MQTT_ENABLED gate even when the UI layer is
|
||||||
# toggled on. The layer toggle should not bypass the opt-in flag that
|
# toggled on. The layer toggle should not bypass the opt-in flag that
|
||||||
@@ -4361,9 +4438,11 @@ async def mesh_send(request: Request):
|
|||||||
any_ok = any(r.ok for r in results)
|
any_ok = any(r.ok for r in results)
|
||||||
|
|
||||||
# ─── Mirror to Meshtastic bridge feed ────────────────────────
|
# ─── Mirror to Meshtastic bridge feed ────────────────────────
|
||||||
# The MQTT broker won't echo our own publishes back to our subscriber,
|
# The MQTT broker won't echo our own publishes back to our subscriber, so
|
||||||
# so inject successfully-sent messages into the bridge's deque directly.
|
# inject successfully-sent channel broadcasts into the bridge directly.
|
||||||
if any_ok and envelope.routed_via == "meshtastic":
|
# Node-targeted packets must not appear in the public channel feed.
|
||||||
|
is_direct_destination = MeshtasticTransport._parse_node_id(destination) is not None
|
||||||
|
if any_ok and envelope.routed_via == "meshtastic" and not is_direct_destination:
|
||||||
try:
|
try:
|
||||||
from services.sigint_bridge import sigint_grid
|
from services.sigint_bridge import sigint_grid
|
||||||
|
|
||||||
@@ -4371,16 +4450,22 @@ async def mesh_send(request: Request):
|
|||||||
if bridge:
|
if bridge:
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
bridge.messages.appendleft(
|
append_text = getattr(bridge, "append_text_message", None)
|
||||||
|
message_record = (
|
||||||
{
|
{
|
||||||
"from": MeshtasticTransport.mesh_address_for_sender(node_id),
|
"from": MeshtasticTransport.mesh_address_for_sender(node_id),
|
||||||
"to": destination if MeshtasticTransport._parse_node_id(destination) is not None else "broadcast",
|
"to": "broadcast",
|
||||||
"text": message,
|
"text": message,
|
||||||
"region": credentials.get("mesh_region", "US"),
|
"region": credentials.get("mesh_region", "US"),
|
||||||
|
"root": credentials.get("mesh_region", "US"),
|
||||||
"channel": body.get("channel", "LongFast"),
|
"channel": body.get("channel", "LongFast"),
|
||||||
"timestamp": datetime.utcnow().isoformat() + "Z",
|
"timestamp": datetime.utcnow().isoformat() + "Z",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
if callable(append_text):
|
||||||
|
append_text(message_record)
|
||||||
|
else:
|
||||||
|
bridge.messages.appendleft(message_record)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass # Non-critical
|
pass # Non-critical
|
||||||
|
|
||||||
@@ -4390,6 +4475,8 @@ async def mesh_send(request: Request):
|
|||||||
"event_id": "",
|
"event_id": "",
|
||||||
"routed_via": envelope.routed_via,
|
"routed_via": envelope.routed_via,
|
||||||
"route_reason": envelope.route_reason,
|
"route_reason": envelope.route_reason,
|
||||||
|
"direct": is_direct_destination,
|
||||||
|
"channel_echo": not is_direct_destination,
|
||||||
"results": [r.to_dict() for r in results],
|
"results": [r.to_dict() for r in results],
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4488,6 +4575,7 @@ async def mesh_messages(
|
|||||||
root: str = "",
|
root: str = "",
|
||||||
channel: str = "",
|
channel: str = "",
|
||||||
limit: int = 30,
|
limit: int = 30,
|
||||||
|
include_direct: bool = False,
|
||||||
):
|
):
|
||||||
"""Get recent Meshtastic text messages from the MQTT bridge."""
|
"""Get recent Meshtastic text messages from the MQTT bridge."""
|
||||||
from services.sigint_bridge import sigint_grid
|
from services.sigint_bridge import sigint_grid
|
||||||
@@ -4509,6 +4597,12 @@ async def mesh_messages(
|
|||||||
msgs = [m for m in msgs if m.get("root", "").upper() == root_filter]
|
msgs = [m for m in msgs if m.get("root", "").upper() == root_filter]
|
||||||
if channel:
|
if channel:
|
||||||
msgs = [m for m in msgs if m.get("channel", "").lower() == channel.lower()]
|
msgs = [m for m in msgs if m.get("channel", "").lower() == channel.lower()]
|
||||||
|
if not include_direct:
|
||||||
|
msgs = [
|
||||||
|
m
|
||||||
|
for m in msgs
|
||||||
|
if str(m.get("to") or "broadcast").strip().lower() in {"", "broadcast", "^all"}
|
||||||
|
]
|
||||||
return msgs[: min(limit, 100)]
|
return msgs[: min(limit, 100)]
|
||||||
|
|
||||||
|
|
||||||
@@ -8054,8 +8148,12 @@ def _cctv_proxy_profile_for_url(target_url: str) -> _CCTVProxyProfile:
|
|||||||
|
|
||||||
|
|
||||||
def _cctv_upstream_headers(request: Request, profile: _CCTVProxyProfile) -> dict[str, str]:
|
def _cctv_upstream_headers(request: Request, profile: _CCTVProxyProfile) -> dict[str, str]:
|
||||||
|
# Round 7a: per-install operator handle. See routers/cctv.py for the
|
||||||
|
# canonical handler; this duplicate stays in lockstep until the #239
|
||||||
|
# dedup ladder removes it.
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
headers = {
|
headers = {
|
||||||
"User-Agent": "Mozilla/5.0 (compatible; ShadowBroker CCTV proxy)",
|
"User-Agent": f"Mozilla/5.0 (compatible; {outbound_user_agent('cctv-proxy')})",
|
||||||
**profile.headers,
|
**profile.headers,
|
||||||
}
|
}
|
||||||
range_header = request.headers.get("range")
|
range_header = request.headers.get("range")
|
||||||
@@ -8726,9 +8824,14 @@ async def api_uw_flow(request: Request):
|
|||||||
from services.news_feed_config import get_feeds, save_feeds, reset_feeds
|
from services.news_feed_config import get_feeds, save_feeds, reset_feeds
|
||||||
|
|
||||||
|
|
||||||
@app.get("/api/settings/news-feeds")
|
@app.get(
|
||||||
|
"/api/settings/news-feeds",
|
||||||
|
dependencies=[Depends(require_local_operator)],
|
||||||
|
)
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def api_get_news_feeds(request: Request):
|
async def api_get_news_feeds(request: Request):
|
||||||
|
"""Issue #252 (tg12): gated on local-operator. See the canonical
|
||||||
|
handler in backend/routers/admin.py for the full rationale."""
|
||||||
return get_feeds()
|
return get_feeds()
|
||||||
|
|
||||||
|
|
||||||
@@ -8789,6 +8892,16 @@ export_wormhole_dm_invite = getattr(
|
|||||||
"export_wormhole_dm_invite",
|
"export_wormhole_dm_invite",
|
||||||
_wormhole_identity_unavailable,
|
_wormhole_identity_unavailable,
|
||||||
)
|
)
|
||||||
|
list_prekey_lookup_handle_records_for_ui = getattr(
|
||||||
|
_mesh_wormhole_identity,
|
||||||
|
"list_prekey_lookup_handle_records_for_ui",
|
||||||
|
_wormhole_identity_unavailable,
|
||||||
|
)
|
||||||
|
revoke_prekey_lookup_handle = getattr(
|
||||||
|
_mesh_wormhole_identity,
|
||||||
|
"revoke_prekey_lookup_handle",
|
||||||
|
_wormhole_identity_unavailable,
|
||||||
|
)
|
||||||
import_wormhole_dm_invite = getattr(
|
import_wormhole_dm_invite = getattr(
|
||||||
_mesh_wormhole_identity,
|
_mesh_wormhole_identity,
|
||||||
"import_wormhole_dm_invite",
|
"import_wormhole_dm_invite",
|
||||||
@@ -8921,9 +9034,22 @@ class NodeSettingsUpdate(BaseModel):
|
|||||||
@app.get("/api/settings/node")
|
@app.get("/api/settings/node")
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def api_get_node_settings(request: Request):
|
async def api_get_node_settings(request: Request):
|
||||||
|
"""Issue #243 (tg12): node mode and participant state are
|
||||||
|
operational posture. Anonymous callers receive an empty stub —
|
||||||
|
enough for the UI to know the endpoint exists but nothing
|
||||||
|
fingerprintable. Authenticated callers see the full state.
|
||||||
|
|
||||||
|
Authenticated == local-operator (loopback / Docker bridge) OR an
|
||||||
|
admin / scoped-view token. The Tauri shell and Docker frontend
|
||||||
|
container both qualify via their existing transport (PR #263 +
|
||||||
|
PR #278), so legitimate operator UX is unchanged.
|
||||||
|
"""
|
||||||
from services.node_settings import read_node_settings
|
from services.node_settings import read_node_settings
|
||||||
|
|
||||||
data = await asyncio.to_thread(read_node_settings)
|
data = await asyncio.to_thread(read_node_settings)
|
||||||
|
authenticated = _scoped_view_authenticated(request, "node")
|
||||||
|
if not authenticated:
|
||||||
|
return {}
|
||||||
return {
|
return {
|
||||||
**data,
|
**data,
|
||||||
"node_mode": _current_node_mode(),
|
"node_mode": _current_node_mode(),
|
||||||
@@ -8935,6 +9061,13 @@ async def api_get_node_settings(request: Request):
|
|||||||
@limiter.limit("10/minute")
|
@limiter.limit("10/minute")
|
||||||
async def api_set_node_settings(request: Request, body: NodeSettingsUpdate):
|
async def api_set_node_settings(request: Request, body: NodeSettingsUpdate):
|
||||||
_refresh_node_peer_store()
|
_refresh_node_peer_store()
|
||||||
|
if bool(body.enabled):
|
||||||
|
try:
|
||||||
|
from services.transport_lane_isolation import disable_public_mesh_lane
|
||||||
|
|
||||||
|
disable_public_mesh_lane(reason="private_node_enabled")
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("Failed to disable public Mesh while enabling private node: %s", exc)
|
||||||
result = _set_participant_node_enabled(bool(body.enabled))
|
result = _set_participant_node_enabled(bool(body.enabled))
|
||||||
if bool(body.enabled):
|
if bool(body.enabled):
|
||||||
_kick_public_sync_background("operator_enable")
|
_kick_public_sync_background("operator_enable")
|
||||||
@@ -9659,7 +9792,7 @@ async def api_get_wormhole_status(request: Request):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.post("/api/wormhole/join", dependencies=[Depends(require_local_operator)])
|
@app.post("/api/wormhole/join")
|
||||||
@limiter.limit("10/minute")
|
@limiter.limit("10/minute")
|
||||||
async def api_wormhole_join(request: Request):
|
async def api_wormhole_join(request: Request):
|
||||||
existing = read_wormhole_settings()
|
existing = read_wormhole_settings()
|
||||||
@@ -9713,7 +9846,7 @@ async def api_wormhole_join(request: Request):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@app.post("/api/wormhole/leave", dependencies=[Depends(require_local_operator)])
|
@app.post("/api/wormhole/leave")
|
||||||
@limiter.limit("10/minute")
|
@limiter.limit("10/minute")
|
||||||
async def api_wormhole_leave(request: Request):
|
async def api_wormhole_leave(request: Request):
|
||||||
updated = write_wormhole_settings(enabled=False)
|
updated = write_wormhole_settings(enabled=False)
|
||||||
@@ -9776,11 +9909,27 @@ async def api_wormhole_dm_identity(request: Request):
|
|||||||
|
|
||||||
@app.get("/api/wormhole/dm/invite", dependencies=[Depends(require_local_operator)])
|
@app.get("/api/wormhole/dm/invite", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def api_wormhole_dm_invite(request: Request):
|
async def api_wormhole_dm_invite(
|
||||||
return export_wormhole_dm_invite()
|
request: Request,
|
||||||
|
label: str = Query("", max_length=96),
|
||||||
|
expires_in_s: int = Query(0, ge=0, le=2_592_000),
|
||||||
|
):
|
||||||
|
return export_wormhole_dm_invite(label=label, expires_in_s=expires_in_s)
|
||||||
|
|
||||||
|
|
||||||
@app.post("/api/wormhole/dm/invite/import", dependencies=[Depends(require_admin)])
|
@app.get("/api/wormhole/dm/invite/handles", dependencies=[Depends(require_local_operator)])
|
||||||
|
@limiter.limit("30/minute")
|
||||||
|
async def api_wormhole_dm_invite_handles(request: Request):
|
||||||
|
return list_prekey_lookup_handle_records_for_ui()
|
||||||
|
|
||||||
|
|
||||||
|
@app.delete("/api/wormhole/dm/invite/handles/{handle}", dependencies=[Depends(require_local_operator)])
|
||||||
|
@limiter.limit("30/minute")
|
||||||
|
async def api_wormhole_dm_invite_handle_revoke(request: Request, handle: str):
|
||||||
|
return revoke_prekey_lookup_handle(handle)
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/wormhole/dm/invite/import", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def api_wormhole_dm_invite_import(request: Request, body: WormholeDmInviteImportRequest):
|
async def api_wormhole_dm_invite_import(request: Request, body: WormholeDmInviteImportRequest):
|
||||||
return import_wormhole_dm_invite(
|
return import_wormhole_dm_invite(
|
||||||
@@ -10527,19 +10676,19 @@ async def api_wormhole_gate_leave(request: Request, body: WormholeGateRequest):
|
|||||||
return leave_gate(str(body.gate_id or ""))
|
return leave_gate(str(body.gate_id or ""))
|
||||||
|
|
||||||
|
|
||||||
@app.get("/api/wormhole/gate/{gate_id}/identity", dependencies=[Depends(require_local_operator)])
|
@app.get("/api/wormhole/gate/{gate_id}/identity")
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def api_wormhole_gate_identity(request: Request, gate_id: str):
|
async def api_wormhole_gate_identity(request: Request, gate_id: str):
|
||||||
return get_active_gate_identity(gate_id)
|
return get_active_gate_identity(gate_id)
|
||||||
|
|
||||||
|
|
||||||
@app.get("/api/wormhole/gate/{gate_id}/personas", dependencies=[Depends(require_local_operator)])
|
@app.get("/api/wormhole/gate/{gate_id}/personas")
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def api_wormhole_gate_personas(request: Request, gate_id: str):
|
async def api_wormhole_gate_personas(request: Request, gate_id: str):
|
||||||
return list_gate_personas(gate_id)
|
return list_gate_personas(gate_id)
|
||||||
|
|
||||||
|
|
||||||
@app.get("/api/wormhole/gate/{gate_id}/key", dependencies=[Depends(require_local_operator)])
|
@app.get("/api/wormhole/gate/{gate_id}/key")
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def api_wormhole_gate_key_status(request: Request, gate_id: str):
|
async def api_wormhole_gate_key_status(request: Request, gate_id: str):
|
||||||
exposure = metadata_exposure_for_request(request, authenticated=True)
|
exposure = metadata_exposure_for_request(request, authenticated=True)
|
||||||
@@ -10722,7 +10871,7 @@ async def api_wormhole_gate_message_sign_encrypted(
|
|||||||
return signed
|
return signed
|
||||||
|
|
||||||
|
|
||||||
@app.post("/api/wormhole/gate/message/post-encrypted", dependencies=[Depends(require_local_operator)])
|
@app.post("/api/wormhole/gate/message/post-encrypted")
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def api_wormhole_gate_message_post_encrypted(
|
async def api_wormhole_gate_message_post_encrypted(
|
||||||
request: Request,
|
request: Request,
|
||||||
@@ -11455,7 +11604,7 @@ async def api_wormhole_health(request: Request):
|
|||||||
return _redact_wormhole_status(full_state, authenticated=ok)
|
return _redact_wormhole_status(full_state, authenticated=ok)
|
||||||
|
|
||||||
|
|
||||||
@app.post("/api/wormhole/connect", dependencies=[Depends(require_admin)])
|
@app.post("/api/wormhole/connect", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("10/minute")
|
@limiter.limit("10/minute")
|
||||||
async def api_wormhole_connect(request: Request):
|
async def api_wormhole_connect(request: Request):
|
||||||
settings = read_wormhole_settings()
|
settings = read_wormhole_settings()
|
||||||
|
|||||||
@@ -7,14 +7,14 @@ py-modules = []
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "backend"
|
name = "backend"
|
||||||
version = "0.9.75"
|
version = "0.9.79"
|
||||||
requires-python = ">=3.10"
|
requires-python = ">=3.10"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"apscheduler==3.10.3",
|
"apscheduler==3.10.3",
|
||||||
"beautifulsoup4>=4.9.0",
|
"beautifulsoup4>=4.9.0",
|
||||||
"cachetools==5.5.2",
|
"cachetools==5.5.2",
|
||||||
"cloudscraper==1.2.71",
|
|
||||||
"cryptography>=41.0.0",
|
"cryptography>=41.0.0",
|
||||||
|
"defusedxml>=0.7.1",
|
||||||
"fastapi==0.115.12",
|
"fastapi==0.115.12",
|
||||||
"feedparser==6.0.10",
|
"feedparser==6.0.10",
|
||||||
"httpx==0.28.1",
|
"httpx==0.28.1",
|
||||||
@@ -43,7 +43,7 @@ dev = ["pytest>=8.3.4", "pytest-asyncio==0.25.0", "ruff>=0.9.0", "black>=24.0.0"
|
|||||||
|
|
||||||
[tool.ruff.lint]
|
[tool.ruff.lint]
|
||||||
# The current backend carries historical style debt in large legacy modules.
|
# The current backend carries historical style debt in large legacy modules.
|
||||||
# Keep CI focused on actionable correctness checks for the v0.9.75 release.
|
# Keep CI focused on actionable correctness checks for the v0.9.79 release.
|
||||||
ignore = ["E401", "E402", "E701", "E731", "E741", "F401", "F402", "F541", "F811", "F841"]
|
ignore = ["E401", "E402", "E701", "E731", "E741", "F401", "F402", "F541", "F811", "F841"]
|
||||||
|
|
||||||
[tool.black]
|
[tool.black]
|
||||||
|
|||||||
@@ -82,9 +82,40 @@ async def api_get_keys_meta(request: Request):
|
|||||||
return get_env_path_info()
|
return get_env_path_info()
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api/settings/news-feeds")
|
@router.get(
|
||||||
|
"/api/settings/operator-handle",
|
||||||
|
dependencies=[Depends(require_local_operator)],
|
||||||
|
)
|
||||||
|
@limiter.limit("60/minute")
|
||||||
|
async def api_get_operator_handle(request: Request):
|
||||||
|
"""Round 7a: return the per-install operator handle so the frontend
|
||||||
|
can include it in browser-direct third-party API calls (Wikipedia /
|
||||||
|
Wikidata via lib/wikimediaClient). The handle is auto-generated on
|
||||||
|
first use; operators can override it via the OPERATOR_HANDLE setting
|
||||||
|
or the env var of the same name.
|
||||||
|
|
||||||
|
Gated on local-operator: legitimate browser usage goes through the
|
||||||
|
Next.js proxy which auto-attaches the admin key; remote scanners get
|
||||||
|
403. The handle itself isn't a secret (it's sent to every third-party
|
||||||
|
API the operator touches), but admin-gating it matches the rest of
|
||||||
|
the settings endpoints and follows least-privilege.
|
||||||
|
"""
|
||||||
|
from services.network_utils import get_operator_handle
|
||||||
|
return {"handle": get_operator_handle()}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/api/settings/news-feeds",
|
||||||
|
dependencies=[Depends(require_local_operator)],
|
||||||
|
)
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def api_get_news_feeds(request: Request):
|
async def api_get_news_feeds(request: Request):
|
||||||
|
"""Issue #252 (tg12): the curated feed inventory is configuration
|
||||||
|
state, not a public data feed. Gated on local-operator so the
|
||||||
|
Tauri shell, the Docker bridge frontend, and any caller with an
|
||||||
|
admin key all see the full list; anonymous LAN/internet callers
|
||||||
|
can no longer enumerate operator source URLs.
|
||||||
|
"""
|
||||||
from services.news_feed_config import get_feeds
|
from services.news_feed_config import get_feeds
|
||||||
return get_feeds()
|
return get_feeds()
|
||||||
|
|
||||||
@@ -118,9 +149,18 @@ async def api_reset_news_feeds(request: Request):
|
|||||||
@router.get("/api/settings/node")
|
@router.get("/api/settings/node")
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def api_get_node_settings(request: Request):
|
async def api_get_node_settings(request: Request):
|
||||||
|
"""Issue #243 (tg12): node_mode and node_enabled are operational
|
||||||
|
posture. Anonymous callers receive an empty stub; authenticated
|
||||||
|
callers (local-operator or admin/scoped token) see the full
|
||||||
|
state. See the canonical handler in backend/main.py for the full
|
||||||
|
rationale.
|
||||||
|
"""
|
||||||
import asyncio
|
import asyncio
|
||||||
|
from auth import _scoped_view_authenticated
|
||||||
from services.node_settings import read_node_settings
|
from services.node_settings import read_node_settings
|
||||||
data = await asyncio.to_thread(read_node_settings)
|
data = await asyncio.to_thread(read_node_settings)
|
||||||
|
if not _scoped_view_authenticated(request, "node"):
|
||||||
|
return {}
|
||||||
return {
|
return {
|
||||||
**data,
|
**data,
|
||||||
"node_mode": _current_node_mode(),
|
"node_mode": _current_node_mode(),
|
||||||
@@ -132,6 +172,13 @@ async def api_get_node_settings(request: Request):
|
|||||||
@limiter.limit("10/minute")
|
@limiter.limit("10/minute")
|
||||||
async def api_set_node_settings(request: Request, body: NodeSettingsUpdate):
|
async def api_set_node_settings(request: Request, body: NodeSettingsUpdate):
|
||||||
_refresh_node_peer_store()
|
_refresh_node_peer_store()
|
||||||
|
if bool(body.enabled):
|
||||||
|
try:
|
||||||
|
from services.transport_lane_isolation import disable_public_mesh_lane
|
||||||
|
|
||||||
|
disable_public_mesh_lane(reason="private_node_enabled")
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("Failed to disable public Mesh while enabling private node: %s", exc)
|
||||||
result = _set_participant_node_enabled(bool(body.enabled))
|
result = _set_participant_node_enabled(bool(body.enabled))
|
||||||
if bool(body.enabled):
|
if bool(body.enabled):
|
||||||
try:
|
try:
|
||||||
@@ -174,17 +221,22 @@ async def api_set_meshtastic_mqtt_settings(request: Request, body: MeshtasticMqt
|
|||||||
|
|
||||||
enabled_requested = updates.get("enabled")
|
enabled_requested = updates.get("enabled")
|
||||||
settings = write_meshtastic_mqtt_settings(**updates)
|
settings = write_meshtastic_mqtt_settings(**updates)
|
||||||
|
if isinstance(enabled_requested, bool):
|
||||||
|
logger.info("Meshtastic MQTT settings update: enabled=%s", enabled_requested)
|
||||||
|
|
||||||
if enabled_requested is True:
|
if enabled_requested is True:
|
||||||
# Public MQTT and Wormhole are intentionally mutually exclusive lanes.
|
# Public MQTT and Wormhole are intentionally mutually exclusive lanes.
|
||||||
try:
|
try:
|
||||||
|
from services.node_settings import write_node_settings
|
||||||
from services.wormhole_settings import write_wormhole_settings
|
from services.wormhole_settings import write_wormhole_settings
|
||||||
from services.wormhole_supervisor import disconnect_wormhole
|
from services.wormhole_supervisor import disconnect_wormhole
|
||||||
|
|
||||||
write_wormhole_settings(enabled=False)
|
write_wormhole_settings(enabled=False)
|
||||||
disconnect_wormhole(reason="public_mesh_enabled")
|
disconnect_wormhole(reason="public_mesh_enabled")
|
||||||
|
write_node_settings(enabled=False)
|
||||||
|
_set_participant_node_enabled(False)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.warning("Failed to disable Wormhole while enabling public mesh: %s", exc)
|
logger.warning("Failed to disable private mesh lane while enabling public mesh: %s", exc)
|
||||||
|
|
||||||
if bool(settings.get("enabled")):
|
if bool(settings.get("enabled")):
|
||||||
if sigint_grid.mesh.is_running():
|
if sigint_grid.mesh.is_running():
|
||||||
@@ -198,9 +250,19 @@ async def api_set_meshtastic_mqtt_settings(request: Request, body: MeshtasticMqt
|
|||||||
return _meshtastic_runtime_snapshot()
|
return _meshtastic_runtime_snapshot()
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api/settings/timemachine")
|
@router.get(
|
||||||
|
"/api/settings/timemachine",
|
||||||
|
dependencies=[Depends(require_local_operator)],
|
||||||
|
)
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def api_get_timemachine_settings(request: Request):
|
async def api_get_timemachine_settings(request: Request):
|
||||||
|
"""Issue #253 (tg12): archival-capture posture is operationally
|
||||||
|
sensitive — it tells a remote caller whether this deployment is
|
||||||
|
retaining replayable historical surveillance data. Gated on
|
||||||
|
local-operator so the Tauri shell and Docker bridge frontend
|
||||||
|
still see the toggle state, but anonymous LAN/internet callers
|
||||||
|
can no longer fingerprint Time Machine state.
|
||||||
|
"""
|
||||||
import asyncio
|
import asyncio
|
||||||
from services.node_settings import read_node_settings
|
from services.node_settings import read_node_settings
|
||||||
data = await asyncio.to_thread(read_node_settings)
|
data = await asyncio.to_thread(read_node_settings)
|
||||||
@@ -357,8 +419,8 @@ async def api_reset_all_agent_credentials(request: Request):
|
|||||||
|
|
||||||
return {
|
return {
|
||||||
"ok": True,
|
"ok": True,
|
||||||
"new_hmac_secret": new_secret,
|
"hmac_regenerated": True,
|
||||||
"detail": "All agent credentials have been reset. Reconfigure your agent with the new credentials.",
|
"detail": "All agent credentials have been reset. Use the agent connection screen to generate or reveal replacement credentials.",
|
||||||
**results,
|
**results,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -18,6 +18,12 @@ from auth import require_local_operator, require_openclaw_or_local
|
|||||||
from limiter import limiter
|
from limiter import limiter
|
||||||
from services.fetchers._store import latest_data as _latest_data
|
from services.fetchers._store import latest_data as _latest_data
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def _ai_intel_user_agent() -> str:
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
|
return outbound_user_agent("ai-intel")
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
@@ -379,14 +385,13 @@ async def api_refresh_layer_feed(request: Request, layer_id: str):
|
|||||||
# Agent Actions endpoint — frontend polls this for UI commands from the agent
|
# Agent Actions endpoint — frontend polls this for UI commands from the agent
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
@router.get("/api/ai/agent-actions")
|
@router.get("/api/ai/agent-actions", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("120/minute")
|
@limiter.limit("120/minute")
|
||||||
async def get_agent_actions(request: Request):
|
async def get_agent_actions(request: Request):
|
||||||
"""Frontend polls for pending agent display actions (destructive read).
|
"""Frontend polls for pending agent display actions (destructive read).
|
||||||
|
|
||||||
No auth required — this only contains display directives (show image,
|
Local operator access is required because polling destructively drains
|
||||||
fly to location), not sensitive data. The agent authenticates when
|
the shared operator action queue.
|
||||||
pushing actions through the command channel.
|
|
||||||
"""
|
"""
|
||||||
actions = pop_agent_actions()
|
actions = pop_agent_actions()
|
||||||
return {"ok": True, "actions": actions}
|
return {"ok": True, "actions": actions}
|
||||||
@@ -448,7 +453,7 @@ async def ai_satellite_images(
|
|||||||
"https://planetarycomputer.microsoft.com/api/stac/v1/search",
|
"https://planetarycomputer.microsoft.com/api/stac/v1/search",
|
||||||
json=search_payload,
|
json=search_payload,
|
||||||
timeout=10,
|
timeout=10,
|
||||||
headers={"User-Agent": "ShadowBroker-OSINT/1.0 (ai-intel)"},
|
headers={"User-Agent": _ai_intel_user_agent()},
|
||||||
)
|
)
|
||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
features = resp.json().get("features", [])
|
features = resp.json().get("features", [])
|
||||||
@@ -1585,7 +1590,7 @@ async def agent_tool_manifest(request: Request):
|
|||||||
|
|
||||||
return {
|
return {
|
||||||
"ok": True,
|
"ok": True,
|
||||||
"version": "0.9.75",
|
"version": "0.9.79",
|
||||||
"access_tier": access_tier,
|
"access_tier": access_tier,
|
||||||
"available_commands": available_commands,
|
"available_commands": available_commands,
|
||||||
"transport": {
|
"transport": {
|
||||||
@@ -2221,7 +2226,7 @@ async def api_capabilities(request: Request):
|
|||||||
access_tier = str(get_settings().OPENCLAW_ACCESS_TIER or "restricted").strip().lower()
|
access_tier = str(get_settings().OPENCLAW_ACCESS_TIER or "restricted").strip().lower()
|
||||||
return {
|
return {
|
||||||
"ok": True,
|
"ok": True,
|
||||||
"version": "0.9.75",
|
"version": "0.9.79",
|
||||||
"auth": {
|
"auth": {
|
||||||
"method": "HMAC-SHA256",
|
"method": "HMAC-SHA256",
|
||||||
"headers": ["X-SB-Timestamp", "X-SB-Nonce", "X-SB-Signature"],
|
"headers": ["X-SB-Timestamp", "X-SB-Nonce", "X-SB-Signature"],
|
||||||
|
|||||||
+65
-2
@@ -165,7 +165,13 @@ def _cctv_proxy_profile_for_url(target_url: str) -> _CCTVProxyProfile:
|
|||||||
|
|
||||||
|
|
||||||
def _cctv_upstream_headers(request: Request, profile: _CCTVProxyProfile) -> dict:
|
def _cctv_upstream_headers(request: Request, profile: _CCTVProxyProfile) -> dict:
|
||||||
headers = {"User-Agent": "Mozilla/5.0 (compatible; ShadowBroker CCTV proxy)", **profile.headers}
|
# Round 7a: per-install operator handle. Mozilla/5.0 prefix retained
|
||||||
|
# because many CCTV endpoints sniff for a browser-like prefix.
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
|
headers = {
|
||||||
|
"User-Agent": f"Mozilla/5.0 (compatible; {outbound_user_agent('cctv-proxy')})",
|
||||||
|
**profile.headers,
|
||||||
|
}
|
||||||
range_header = request.headers.get("range")
|
range_header = request.headers.get("range")
|
||||||
if range_header:
|
if range_header:
|
||||||
headers["Range"] = range_header
|
headers["Range"] = range_header
|
||||||
@@ -191,11 +197,68 @@ def _cctv_response_headers(resp, cache_seconds: int, include_length: bool = True
|
|||||||
return headers
|
return headers
|
||||||
|
|
||||||
|
|
||||||
|
# Maximum number of redirects we'll follow on the CCTV upstream. Each hop is
|
||||||
|
# re-validated against _cctv_host_allowed() before continuing, so this caps
|
||||||
|
# the redirect-chain SSRF blast radius.
|
||||||
|
_CCTV_MAX_REDIRECTS = 5
|
||||||
|
|
||||||
|
|
||||||
def _fetch_cctv_upstream_response(request: Request, target_url: str, profile: _CCTVProxyProfile):
|
def _fetch_cctv_upstream_response(request: Request, target_url: str, profile: _CCTVProxyProfile):
|
||||||
|
"""Fetch an upstream CCTV URL, following redirects manually with host re-validation.
|
||||||
|
|
||||||
|
Why manual redirect following:
|
||||||
|
The original code used ``allow_redirects=True``, which only validated
|
||||||
|
the initial caller-supplied URL host against the allowlist. An attacker
|
||||||
|
could submit an allowed host that 302-redirected to an internal address
|
||||||
|
(e.g. ``http://localhost:8000/api/...`` or a private RFC1918 range),
|
||||||
|
and the backend would dutifully follow and proxy the response — a
|
||||||
|
classic open-redirect-to-SSRF chain.
|
||||||
|
|
||||||
|
With this loop, we re-run ``_cctv_host_allowed()`` on every hop's
|
||||||
|
``Location`` header. A redirect to a host that isn't on the allowlist
|
||||||
|
is rejected with 502 rather than silently followed.
|
||||||
|
"""
|
||||||
import requests as _req
|
import requests as _req
|
||||||
|
from urllib.parse import urlparse, urljoin
|
||||||
|
|
||||||
headers = _cctv_upstream_headers(request, profile)
|
headers = _cctv_upstream_headers(request, profile)
|
||||||
|
current_url = target_url
|
||||||
|
hops = 0
|
||||||
try:
|
try:
|
||||||
resp = _req.get(target_url, timeout=profile.timeout, stream=True, allow_redirects=True, headers=headers)
|
while True:
|
||||||
|
resp = _req.get(
|
||||||
|
current_url,
|
||||||
|
timeout=profile.timeout,
|
||||||
|
stream=True,
|
||||||
|
allow_redirects=False,
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
# Redirect handling — re-validate the next-hop host before following.
|
||||||
|
if resp.is_redirect or resp.status_code in (301, 302, 303, 307, 308):
|
||||||
|
location = resp.headers.get("Location", "")
|
||||||
|
resp.close()
|
||||||
|
if hops >= _CCTV_MAX_REDIRECTS:
|
||||||
|
logger.warning(
|
||||||
|
"CCTV upstream redirect chain exceeded limit [%s] %s",
|
||||||
|
profile.name, target_url,
|
||||||
|
)
|
||||||
|
raise HTTPException(status_code=502, detail="Upstream redirect chain too long")
|
||||||
|
if not location:
|
||||||
|
raise HTTPException(status_code=502, detail="Upstream redirect missing Location")
|
||||||
|
next_url = urljoin(current_url, location)
|
||||||
|
next_parsed = urlparse(next_url)
|
||||||
|
if next_parsed.scheme not in ("http", "https"):
|
||||||
|
raise HTTPException(status_code=502, detail="Upstream redirect to non-HTTP scheme")
|
||||||
|
if not _cctv_host_allowed(next_parsed.hostname):
|
||||||
|
logger.warning(
|
||||||
|
"CCTV upstream redirect to disallowed host [%s] %s -> %s",
|
||||||
|
profile.name, current_url, next_url,
|
||||||
|
)
|
||||||
|
raise HTTPException(status_code=502, detail="Upstream redirect to disallowed host")
|
||||||
|
current_url = next_url
|
||||||
|
hops += 1
|
||||||
|
continue
|
||||||
|
break
|
||||||
except _req.exceptions.Timeout as exc:
|
except _req.exceptions.Timeout as exc:
|
||||||
logger.warning("CCTV upstream timeout [%s] %s", profile.name, target_url)
|
logger.warning("CCTV upstream timeout [%s] %s", profile.name, target_url)
|
||||||
raise HTTPException(status_code=504, detail="Upstream timeout") from exc
|
raise HTTPException(status_code=504, detail="Upstream timeout") from exc
|
||||||
|
|||||||
+145
-15
@@ -98,6 +98,88 @@ def _current_etag(prefix: str = "") -> str:
|
|||||||
return f"{prefix}v{get_data_version()}-l{get_active_layers_version()}"
|
return f"{prefix}v{get_data_version()}-l{get_active_layers_version()}"
|
||||||
|
|
||||||
|
|
||||||
|
# ── Issue #288: viewport-aware payloads ─────────────────────────────────────
|
||||||
|
# Heavy, density-driven, time-sensitive layers that benefit from bbox
|
||||||
|
# filtering. Light reference layers (datacenters, military_bases,
|
||||||
|
# power_plants, satellites, weather, news, etc.) are intentionally NOT
|
||||||
|
# in these sets — they ship world-scale even when bounds are supplied so
|
||||||
|
# panning never reveals an "empty world" of static infrastructure.
|
||||||
|
#
|
||||||
|
# When the caller does NOT pass s/w/n/e, none of this runs and the response
|
||||||
|
# is byte-for-byte identical to the pre-#288 behavior.
|
||||||
|
_FAST_BBOX_HEAVY_KEYS: tuple[str, ...] = (
|
||||||
|
"commercial_flights",
|
||||||
|
"military_flights",
|
||||||
|
"private_flights",
|
||||||
|
"private_jets",
|
||||||
|
"tracked_flights",
|
||||||
|
"ships",
|
||||||
|
"cctv",
|
||||||
|
"uavs",
|
||||||
|
"liveuamap",
|
||||||
|
"gps_jamming",
|
||||||
|
"sigint",
|
||||||
|
"trains",
|
||||||
|
)
|
||||||
|
_SLOW_BBOX_HEAVY_KEYS: tuple[str, ...] = (
|
||||||
|
"gdelt",
|
||||||
|
"firms_fires",
|
||||||
|
"kiwisdr",
|
||||||
|
"scanners",
|
||||||
|
"psk_reporter",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _has_full_bbox(s, w, n, e) -> bool:
|
||||||
|
return None not in (s, w, n, e)
|
||||||
|
|
||||||
|
|
||||||
|
def _bbox_etag_suffix(s, w, n, e) -> str:
|
||||||
|
"""Quantize bbox to 1° before mixing into the ETag.
|
||||||
|
|
||||||
|
The 20% padding inside _bbox_filter already absorbs sub-degree pans;
|
||||||
|
quantizing here means small mouse drags don't blow the ETag cache
|
||||||
|
on the client. Full-world bounds collapse to a single suffix.
|
||||||
|
"""
|
||||||
|
if not _has_full_bbox(s, w, n, e):
|
||||||
|
return ""
|
||||||
|
try:
|
||||||
|
ss = math.floor(float(s))
|
||||||
|
ww = math.floor(float(w))
|
||||||
|
nn = math.ceil(float(n))
|
||||||
|
ee = math.ceil(float(e))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return ""
|
||||||
|
# If the requested window covers basically the whole world, treat it as
|
||||||
|
# "no bbox" for caching purposes so world-zoomed clients all hit the
|
||||||
|
# same ETag and benefit from the existing 304 path.
|
||||||
|
lat_span, lng_span = _bbox_spans(s, w, n, e)
|
||||||
|
if lng_span >= 300 or lat_span >= 120:
|
||||||
|
return ""
|
||||||
|
return f"|bbox={ss},{ww},{nn},{ee}"
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_bbox_to_payload(payload: dict, heavy_keys: tuple[str, ...],
|
||||||
|
s: float, w: float, n: float, e: float) -> dict:
|
||||||
|
"""In-place filter the heavy-key collections in *payload* to a viewport.
|
||||||
|
|
||||||
|
Items without lat/lng are passed through (so e.g. summary blobs aren't
|
||||||
|
accidentally dropped). The existing _bbox_filter helper applies a 20%
|
||||||
|
pad and handles antimeridian crossings.
|
||||||
|
"""
|
||||||
|
lat_span, lng_span = _bbox_spans(s, w, n, e)
|
||||||
|
# World-scale request → skip filtering entirely. Spares the CPU and
|
||||||
|
# guarantees the response matches the no-params shape.
|
||||||
|
if lng_span >= 300 or lat_span >= 120:
|
||||||
|
return payload
|
||||||
|
for key in heavy_keys:
|
||||||
|
items = payload.get(key)
|
||||||
|
if not isinstance(items, list) or not items:
|
||||||
|
continue
|
||||||
|
payload[key] = _bbox_filter(items, s, w, n, e)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
def _json_safe(value):
|
def _json_safe(value):
|
||||||
if isinstance(value, float):
|
if isinstance(value, float):
|
||||||
return value if math.isfinite(value) else None
|
return value if math.isfinite(value) else None
|
||||||
@@ -266,7 +348,7 @@ async def force_refresh(request: Request):
|
|||||||
return {"status": "refreshing in background"}
|
return {"status": "refreshing in background"}
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/ais/feed")
|
@router.post("/api/ais/feed", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("60/minute")
|
@limiter.limit("60/minute")
|
||||||
async def ais_feed(request: Request):
|
async def ais_feed(request: Request):
|
||||||
"""Accept AIS-catcher HTTP JSON feed (POST decoded AIS messages)."""
|
"""Accept AIS-catcher HTTP JSON feed (POST decoded AIS messages)."""
|
||||||
@@ -304,7 +386,7 @@ async def update_viewport(vp: ViewportUpdate, request: Request): # noqa: ARG001
|
|||||||
return {"status": "ok"}
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/layers")
|
@router.post("/api/layers", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def update_layers(update: LayerUpdate, request: Request):
|
async def update_layers(update: LayerUpdate, request: Request):
|
||||||
"""Receive frontend layer toggle state. Starts/stops streams accordingly."""
|
"""Receive frontend layer toggle state. Starts/stops streams accordingly."""
|
||||||
@@ -335,8 +417,16 @@ async def update_layers(update: LayerUpdate, request: Request):
|
|||||||
logger.info("AIS stream started (ship layer enabled)")
|
logger.info("AIS stream started (ship layer enabled)")
|
||||||
from services.sigint_bridge import sigint_grid
|
from services.sigint_bridge import sigint_grid
|
||||||
if old_mesh and not new_mesh:
|
if old_mesh and not new_mesh:
|
||||||
sigint_grid.mesh.stop()
|
try:
|
||||||
logger.info("Meshtastic MQTT bridge stopped (layer disabled)")
|
from services.meshtastic_mqtt_settings import mqtt_bridge_enabled
|
||||||
|
keep_chat_running = mqtt_bridge_enabled()
|
||||||
|
except Exception:
|
||||||
|
keep_chat_running = False
|
||||||
|
if keep_chat_running:
|
||||||
|
logger.info("Meshtastic map layer disabled; MQTT bridge kept running for MeshChat")
|
||||||
|
else:
|
||||||
|
sigint_grid.mesh.stop()
|
||||||
|
logger.info("Meshtastic MQTT bridge stopped (layer disabled)")
|
||||||
elif not old_mesh and new_mesh:
|
elif not old_mesh and new_mesh:
|
||||||
try:
|
try:
|
||||||
from services.meshtastic_mqtt_settings import mqtt_bridge_enabled
|
from services.meshtastic_mqtt_settings import mqtt_bridge_enabled
|
||||||
@@ -471,13 +561,14 @@ async def bootstrap_critical(request: Request):
|
|||||||
@limiter.limit("120/minute")
|
@limiter.limit("120/minute")
|
||||||
async def live_data_fast(
|
async def live_data_fast(
|
||||||
request: Request,
|
request: Request,
|
||||||
s: float = Query(None, description="South bound (ignored)", ge=-90, le=90),
|
s: float = Query(None, description="South bound — when all four bounds are supplied, heavy/dense layers (vessels, aircraft, sigint, CCTV, …) are filtered to this viewport with 20% padding. Static reference layers (satellites, etc.) always ship world-scale.", ge=-90, le=90),
|
||||||
w: float = Query(None, description="West bound (ignored)", ge=-180, le=180),
|
w: float = Query(None, description="West bound (see s)", ge=-180, le=180),
|
||||||
n: float = Query(None, description="North bound (ignored)", ge=-90, le=90),
|
n: float = Query(None, description="North bound (see s)", ge=-90, le=90),
|
||||||
e: float = Query(None, description="East bound (ignored)", ge=-180, le=180),
|
e: float = Query(None, description="East bound (see s)", ge=-180, le=180),
|
||||||
initial: bool = Query(False, description="Return a capped startup payload for first paint"),
|
initial: bool = Query(False, description="Return a capped startup payload for first paint"),
|
||||||
):
|
):
|
||||||
etag = _current_etag(prefix="fast|initial|" if initial else "fast|full|")
|
bbox_suffix = _bbox_etag_suffix(s, w, n, e)
|
||||||
|
etag = _current_etag(prefix=("fast|initial|" if initial else "fast|full|") + bbox_suffix.lstrip("|") + ("|" if bbox_suffix else ""))
|
||||||
if request.headers.get("if-none-match") == etag:
|
if request.headers.get("if-none-match") == etag:
|
||||||
return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
|
return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
|
||||||
from services.fetchers._store import (active_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot)
|
from services.fetchers._store import (active_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot)
|
||||||
@@ -517,6 +608,11 @@ async def live_data_fast(
|
|||||||
payload = _cap_fast_startup_payload(payload)
|
payload = _cap_fast_startup_payload(payload)
|
||||||
else:
|
else:
|
||||||
payload = _cap_fast_dashboard_payload(payload)
|
payload = _cap_fast_dashboard_payload(payload)
|
||||||
|
# Issue #288: bbox filter heavy/dense layers only when all four bounds
|
||||||
|
# are supplied. Without bounds, behaviour is byte-for-byte identical
|
||||||
|
# to the pre-#288 implementation.
|
||||||
|
if _has_full_bbox(s, w, n, e):
|
||||||
|
payload = _apply_bbox_to_payload(payload, _FAST_BBOX_HEAVY_KEYS, s, w, n, e)
|
||||||
return Response(content=orjson.dumps(_sanitize_payload(payload)), media_type="application/json",
|
return Response(content=orjson.dumps(_sanitize_payload(payload)), media_type="application/json",
|
||||||
headers={"ETag": etag, "Cache-Control": "no-cache"})
|
headers={"ETag": etag, "Cache-Control": "no-cache"})
|
||||||
|
|
||||||
@@ -525,12 +621,13 @@ async def live_data_fast(
|
|||||||
@limiter.limit("60/minute")
|
@limiter.limit("60/minute")
|
||||||
async def live_data_slow(
|
async def live_data_slow(
|
||||||
request: Request,
|
request: Request,
|
||||||
s: float = Query(None, description="South bound (ignored)", ge=-90, le=90),
|
s: float = Query(None, description="South bound — when all four bounds are supplied, heavy/dense layers (gdelt, firms_fires, kiwisdr, scanners, psk_reporter) are filtered to this viewport with 20% padding. Static reference layers (datacenters, military bases, power plants, weather, news, …) always ship world-scale.", ge=-90, le=90),
|
||||||
w: float = Query(None, description="West bound (ignored)", ge=-180, le=180),
|
w: float = Query(None, description="West bound (see s)", ge=-180, le=180),
|
||||||
n: float = Query(None, description="North bound (ignored)", ge=-90, le=90),
|
n: float = Query(None, description="North bound (see s)", ge=-90, le=90),
|
||||||
e: float = Query(None, description="East bound (ignored)", ge=-180, le=180),
|
e: float = Query(None, description="East bound (see s)", ge=-180, le=180),
|
||||||
):
|
):
|
||||||
etag = _current_etag(prefix="slow|full|")
|
bbox_suffix = _bbox_etag_suffix(s, w, n, e)
|
||||||
|
etag = _current_etag(prefix="slow|full|" + bbox_suffix.lstrip("|") + ("|" if bbox_suffix else ""))
|
||||||
if request.headers.get("if-none-match") == etag:
|
if request.headers.get("if-none-match") == etag:
|
||||||
return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
|
return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
|
||||||
from services.fetchers._store import (active_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot)
|
from services.fetchers._store import (active_layers, get_latest_data_subset_refs, get_source_timestamps_snapshot)
|
||||||
@@ -584,6 +681,12 @@ async def live_data_slow(
|
|||||||
"crowdthreat": (d.get("crowdthreat") or []) if active_layers.get("crowdthreat", True) else [],
|
"crowdthreat": (d.get("crowdthreat") or []) if active_layers.get("crowdthreat", True) else [],
|
||||||
"freshness": freshness,
|
"freshness": freshness,
|
||||||
}
|
}
|
||||||
|
# Issue #288: bbox filter heavy/dense layers only when all four bounds
|
||||||
|
# are supplied. Static reference layers (datacenters, military bases,
|
||||||
|
# power_plants, etc.) deliberately stay world-scale so panning never
|
||||||
|
# hides the infrastructure overlay the operator already has on screen.
|
||||||
|
if _has_full_bbox(s, w, n, e):
|
||||||
|
payload = _apply_bbox_to_payload(payload, _SLOW_BBOX_HEAVY_KEYS, s, w, n, e)
|
||||||
return Response(
|
return Response(
|
||||||
content=orjson.dumps(_sanitize_payload(payload), default=str, option=orjson.OPT_NON_STR_KEYS),
|
content=orjson.dumps(_sanitize_payload(payload), default=str, option=orjson.OPT_NON_STR_KEYS),
|
||||||
media_type="application/json",
|
media_type="application/json",
|
||||||
@@ -603,6 +706,23 @@ class OverflightRequest(BaseModel):
|
|||||||
hours: int = 24
|
hours: int = 24
|
||||||
|
|
||||||
|
|
||||||
|
# Issue #202: compute_overflights() is O(catalog_size × timesteps), where
|
||||||
|
# timesteps grows linearly with `hours`. An unbounded `hours` value is a
|
||||||
|
# trivial CPU-exhaustion vector. We clamp silently rather than raising 422 —
|
||||||
|
# the response shape is unchanged, callers asking for too many hours just
|
||||||
|
# get a shorter window, which is friendlier than a hostile error.
|
||||||
|
#
|
||||||
|
# Override via OVERFLIGHTS_MAX_HOURS env var if you legitimately need a
|
||||||
|
# longer window (e.g. a planning use case that wants a full week).
|
||||||
|
def _overflight_max_hours() -> int:
|
||||||
|
import os as _os
|
||||||
|
try:
|
||||||
|
raw = int(str(_os.environ.get("OVERFLIGHTS_MAX_HOURS", "72")).strip())
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
raw = 72
|
||||||
|
return max(1, raw)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/satellites/overflights")
|
@router.post("/api/satellites/overflights")
|
||||||
@limiter.limit("10/minute")
|
@limiter.limit("10/minute")
|
||||||
async def satellite_overflights(request: Request, body: OverflightRequest):
|
async def satellite_overflights(request: Request, body: OverflightRequest):
|
||||||
@@ -611,5 +731,15 @@ async def satellite_overflights(request: Request, body: OverflightRequest):
|
|||||||
if not gp_data:
|
if not gp_data:
|
||||||
return JSONResponse({"total": 0, "by_mission": {}, "satellites": [], "error": "No GP data cached yet"})
|
return JSONResponse({"total": 0, "by_mission": {}, "satellites": [], "error": "No GP data cached yet"})
|
||||||
bbox = {"s": body.s, "w": body.w, "n": body.n, "e": body.e}
|
bbox = {"s": body.s, "w": body.w, "n": body.n, "e": body.e}
|
||||||
result = compute_overflights(gp_data, bbox, hours=body.hours)
|
|
||||||
|
# Silent clamp — see comment on _overflight_max_hours().
|
||||||
|
requested_hours = max(1, int(body.hours or 0))
|
||||||
|
effective_hours = min(requested_hours, _overflight_max_hours())
|
||||||
|
|
||||||
|
result = compute_overflights(gp_data, bbox, hours=effective_hours)
|
||||||
|
# If we clamped, surface the effective window in the response so the
|
||||||
|
# caller can detect it if they care, without it being an error.
|
||||||
|
if isinstance(result, dict) and effective_hours != requested_hours:
|
||||||
|
result.setdefault("requested_hours", requested_hours)
|
||||||
|
result.setdefault("effective_hours", effective_hours)
|
||||||
return JSONResponse(result)
|
return JSONResponse(result)
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from services.data_fetcher import get_latest_data
|
|||||||
from services.schemas import HealthResponse
|
from services.schemas import HealthResponse
|
||||||
import os
|
import os
|
||||||
|
|
||||||
APP_VERSION = os.environ.get("_HEALTH_APP_VERSION", "0.9.75")
|
APP_VERSION = os.environ.get("_HEALTH_APP_VERSION", "0.9.79")
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
@@ -54,6 +54,22 @@ async def health_check(request: Request):
|
|||||||
top_status = "error"
|
top_status = "error"
|
||||||
elif slo_summary.get("yellow", 0) > 0:
|
elif slo_summary.get("yellow", 0) > 0:
|
||||||
top_status = "degraded"
|
top_status = "degraded"
|
||||||
|
|
||||||
|
# Issue #258: surface AIS proxy degraded TLS state so operators can see
|
||||||
|
# when the SPKI-pinned fallback is in effect. The data plane keeps
|
||||||
|
# flowing (this is by design — see ais_proxy.js comments) but observers
|
||||||
|
# who care about MITM-protection posture deserve a visible signal.
|
||||||
|
ais_status: dict = {}
|
||||||
|
try:
|
||||||
|
from services.ais_stream import ais_proxy_status
|
||||||
|
ais_status = ais_proxy_status() or {}
|
||||||
|
except Exception:
|
||||||
|
ais_status = {}
|
||||||
|
if ais_status.get("degraded_tls") and top_status == "ok":
|
||||||
|
# Don't override a worse top-level status if SLOs already failed,
|
||||||
|
# but escalate ok -> degraded so the field surfaces in dashboards.
|
||||||
|
top_status = "degraded"
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"status": top_status,
|
"status": top_status,
|
||||||
"version": _get_app_version(),
|
"version": _get_app_version(),
|
||||||
@@ -76,6 +92,7 @@ async def health_check(request: Request):
|
|||||||
"uptime_seconds": round(_time_mod.time() - _get_start_time()),
|
"uptime_seconds": round(_time_mod.time() - _get_start_time()),
|
||||||
"slo": slo_statuses,
|
"slo": slo_statuses,
|
||||||
"slo_summary": slo_summary,
|
"slo_summary": slo_summary,
|
||||||
|
"ais_proxy": ais_status,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -223,11 +223,21 @@ async def oracle_markets_more(request: Request, category: str = "NEWS", offset:
|
|||||||
"has_more": offset + limit < len(cat_markets), "total": len(cat_markets)}
|
"has_more": offset + limit < len(cat_markets), "total": len(cat_markets)}
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/mesh/oracle/resolve")
|
@router.post(
|
||||||
|
"/api/mesh/oracle/resolve",
|
||||||
|
dependencies=[Depends(require_admin)],
|
||||||
|
)
|
||||||
@limiter.limit("5/minute")
|
@limiter.limit("5/minute")
|
||||||
@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
|
@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
|
||||||
async def oracle_resolve(request: Request):
|
async def oracle_resolve(request: Request):
|
||||||
"""Resolve a prediction market."""
|
"""Resolve a prediction market.
|
||||||
|
|
||||||
|
Issue #240 (tg12): requires admin authentication. The
|
||||||
|
``mesh_write_exempt`` decorator below is **metadata only** — it tags
|
||||||
|
the route as not requiring a mesh signed-write envelope, it does
|
||||||
|
NOT itself enforce caller authorization. The ``Depends(require_admin)``
|
||||||
|
on the route decorator is what actually gates access.
|
||||||
|
"""
|
||||||
from services.mesh.mesh_oracle import oracle_ledger
|
from services.mesh.mesh_oracle import oracle_ledger
|
||||||
body = await request.json()
|
body = await request.json()
|
||||||
market_title = body.get("market_title", "")
|
market_title = body.get("market_title", "")
|
||||||
@@ -327,11 +337,18 @@ async def oracle_predictions(request: Request, node_id: str = ""):
|
|||||||
active_predictions, authenticated=_scoped_view_authenticated(request, "mesh.audit"))
|
active_predictions, authenticated=_scoped_view_authenticated(request, "mesh.audit"))
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/mesh/oracle/resolve-stakes")
|
@router.post(
|
||||||
|
"/api/mesh/oracle/resolve-stakes",
|
||||||
|
dependencies=[Depends(require_admin)],
|
||||||
|
)
|
||||||
@limiter.limit("5/minute")
|
@limiter.limit("5/minute")
|
||||||
@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
|
@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
|
||||||
async def oracle_resolve_stakes(request: Request):
|
async def oracle_resolve_stakes(request: Request):
|
||||||
"""Resolve all expired stake contests."""
|
"""Resolve all expired stake contests.
|
||||||
|
|
||||||
|
Issue #241 (tg12): requires admin authentication. See the note on
|
||||||
|
``oracle_resolve`` above — ``mesh_write_exempt`` is metadata only.
|
||||||
|
"""
|
||||||
from services.mesh.mesh_oracle import oracle_ledger
|
from services.mesh.mesh_oracle import oracle_ledger
|
||||||
resolutions = oracle_ledger.resolve_expired_stakes()
|
resolutions = oracle_ledger.resolve_expired_stakes()
|
||||||
return {"ok": True, "resolutions": resolutions, "count": len(resolutions)}
|
return {"ok": True, "resolutions": resolutions, "count": len(resolutions)}
|
||||||
|
|||||||
@@ -85,6 +85,64 @@ async def infonet_peer_push(request: Request):
|
|||||||
return {"ok": True, **result}
|
return {"ok": True, **result}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/api/mesh/dm/replicate-envelope")
|
||||||
|
@limiter.limit("60/minute")
|
||||||
|
async def dm_replicate_envelope(request: Request):
|
||||||
|
"""Accept a DM envelope replicated from a peer relay (cross-node mailbox).
|
||||||
|
|
||||||
|
Companion endpoint to ``DMRelay.replicate_to_peers`` (outbound, in
|
||||||
|
``mesh_dm_relay.py``). The sender's relay POSTs an encrypted DM
|
||||||
|
envelope here after a successful local ``deposit``; this endpoint
|
||||||
|
re-enforces the per-(sender, recipient) anti-spam cap and stores
|
||||||
|
the envelope in the local mailbox if accepted.
|
||||||
|
|
||||||
|
The cap is the network rule: a hostile sender's relay can spool
|
||||||
|
extras locally, but every honest peer enforces the cap on inbound
|
||||||
|
replication. Recipient polling from any honest peer therefore
|
||||||
|
never sees more than ``MESH_DM_PENDING_PER_SENDER_LIMIT`` pending
|
||||||
|
from any one sender, no matter how many spam attempts were tried.
|
||||||
|
|
||||||
|
Same HMAC auth pattern as ``infonet_peer_push`` and ``gate_peer_push``.
|
||||||
|
"""
|
||||||
|
content_length = request.headers.get("content-length")
|
||||||
|
if content_length:
|
||||||
|
try:
|
||||||
|
# DM envelopes are bounded by MESH_DM_MAX_MSG_BYTES + envelope
|
||||||
|
# overhead; 64 KB is a generous ceiling.
|
||||||
|
if int(content_length) > 65_536:
|
||||||
|
return Response(
|
||||||
|
content='{"ok":false,"detail":"Request body too large (max 64KB)"}',
|
||||||
|
status_code=413, media_type="application/json",
|
||||||
|
)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
pass
|
||||||
|
body_bytes = await request.body()
|
||||||
|
if not _verify_peer_push_hmac(request, body_bytes):
|
||||||
|
return Response(
|
||||||
|
content='{"ok":false,"detail":"Invalid or missing peer HMAC"}',
|
||||||
|
status_code=403, media_type="application/json",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
body = json_mod.loads(body_bytes or b"{}")
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
return Response(
|
||||||
|
content='{"ok":false,"detail":"Invalid JSON body"}',
|
||||||
|
status_code=400, media_type="application/json",
|
||||||
|
)
|
||||||
|
envelope = body.get("envelope")
|
||||||
|
if not isinstance(envelope, dict):
|
||||||
|
return {"ok": False, "detail": "envelope must be an object"}
|
||||||
|
|
||||||
|
originating_peer = _peer_hmac_url_from_request(request) or ""
|
||||||
|
|
||||||
|
from services.mesh.mesh_dm_relay import dm_relay
|
||||||
|
result = dm_relay.accept_replica(
|
||||||
|
envelope=envelope,
|
||||||
|
originating_peer_url=originating_peer,
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/mesh/gate/peer-push")
|
@router.post("/api/mesh/gate/peer-push")
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def gate_peer_push(request: Request):
|
async def gate_peer_push(request: Request):
|
||||||
|
|||||||
@@ -721,9 +721,11 @@ async def mesh_send(request: Request):
|
|||||||
any_ok = any(r.ok for r in results)
|
any_ok = any(r.ok for r in results)
|
||||||
|
|
||||||
# ─── Mirror to Meshtastic bridge feed ────────────────────────
|
# ─── Mirror to Meshtastic bridge feed ────────────────────────
|
||||||
# The MQTT broker won't echo our own publishes back to our subscriber,
|
# The MQTT broker won't echo our own publishes back to our subscriber, so
|
||||||
# so inject successfully-sent messages into the bridge's deque directly.
|
# inject successfully-sent channel broadcasts into the bridge directly.
|
||||||
if any_ok and envelope.routed_via == "meshtastic":
|
# Node-targeted packets must not appear in the public channel feed.
|
||||||
|
is_direct_destination = MeshtasticTransport._parse_node_id(destination) is not None
|
||||||
|
if any_ok and envelope.routed_via == "meshtastic" and not is_direct_destination:
|
||||||
try:
|
try:
|
||||||
from services.sigint_bridge import sigint_grid
|
from services.sigint_bridge import sigint_grid
|
||||||
|
|
||||||
@@ -734,7 +736,7 @@ async def mesh_send(request: Request):
|
|||||||
bridge.messages.appendleft(
|
bridge.messages.appendleft(
|
||||||
{
|
{
|
||||||
"from": MeshtasticTransport.mesh_address_for_sender(node_id),
|
"from": MeshtasticTransport.mesh_address_for_sender(node_id),
|
||||||
"to": destination if MeshtasticTransport._parse_node_id(destination) is not None else "broadcast",
|
"to": "broadcast",
|
||||||
"text": message,
|
"text": message,
|
||||||
"region": credentials.get("mesh_region", "US"),
|
"region": credentials.get("mesh_region", "US"),
|
||||||
"channel": body.get("channel", "LongFast"),
|
"channel": body.get("channel", "LongFast"),
|
||||||
@@ -750,6 +752,8 @@ async def mesh_send(request: Request):
|
|||||||
"event_id": "",
|
"event_id": "",
|
||||||
"routed_via": envelope.routed_via,
|
"routed_via": envelope.routed_via,
|
||||||
"route_reason": envelope.route_reason,
|
"route_reason": envelope.route_reason,
|
||||||
|
"direct": is_direct_destination,
|
||||||
|
"channel_echo": not is_direct_destination,
|
||||||
"results": [r.to_dict() for r in results],
|
"results": [r.to_dict() for r in results],
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -818,9 +822,10 @@ async def meshtastic_public_send(request: Request):
|
|||||||
if not cb_ok:
|
if not cb_ok:
|
||||||
results = [TransportResult(False, "meshtastic", cb_reason)]
|
results = [TransportResult(False, "meshtastic", cb_reason)]
|
||||||
else:
|
else:
|
||||||
|
is_direct_destination = MeshtasticTransport._parse_node_id(destination) is not None
|
||||||
envelope.route_reason = (
|
envelope.route_reason = (
|
||||||
"Local public Meshtastic MQTT path"
|
"Local public Meshtastic MQTT path"
|
||||||
if MeshtasticTransport._parse_node_id(destination) is None
|
if not is_direct_destination
|
||||||
else "Local public Meshtastic direct node path"
|
else "Local public Meshtastic direct node path"
|
||||||
)
|
)
|
||||||
credentials = {"mesh_region": str(body.get("mesh_region", "US") or "US")}
|
credentials = {"mesh_region": str(body.get("mesh_region", "US") or "US")}
|
||||||
@@ -830,23 +835,28 @@ async def meshtastic_public_send(request: Request):
|
|||||||
results = [result]
|
results = [result]
|
||||||
|
|
||||||
any_ok = any(r.ok for r in results)
|
any_ok = any(r.ok for r in results)
|
||||||
if any_ok and envelope.routed_via == "meshtastic":
|
is_direct_destination = MeshtasticTransport._parse_node_id(destination) is not None
|
||||||
|
if any_ok and envelope.routed_via == "meshtastic" and not is_direct_destination:
|
||||||
try:
|
try:
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from services.sigint_bridge import sigint_grid
|
from services.sigint_bridge import sigint_grid
|
||||||
|
|
||||||
bridge = sigint_grid.mesh
|
bridge = sigint_grid.mesh
|
||||||
if bridge:
|
if bridge:
|
||||||
bridge.messages.appendleft(
|
record = {
|
||||||
{
|
"from": MeshtasticTransport.mesh_address_for_sender(sender_id),
|
||||||
"from": MeshtasticTransport.mesh_address_for_sender(sender_id),
|
"to": "broadcast",
|
||||||
"to": destination if MeshtasticTransport._parse_node_id(destination) is not None else "broadcast",
|
"text": message,
|
||||||
"text": message,
|
"region": str(body.get("mesh_region", "US") or "US"),
|
||||||
"region": str(body.get("mesh_region", "US") or "US"),
|
"root": str(body.get("mesh_region", "US") or "US"),
|
||||||
"channel": str(body.get("channel", "LongFast") or "LongFast"),
|
"channel": str(body.get("channel", "LongFast") or "LongFast"),
|
||||||
"timestamp": datetime.utcnow().isoformat() + "Z",
|
"timestamp": datetime.utcnow().isoformat() + "Z",
|
||||||
}
|
}
|
||||||
)
|
append_text = getattr(bridge, "append_text_message", None)
|
||||||
|
if callable(append_text):
|
||||||
|
append_text(record)
|
||||||
|
else:
|
||||||
|
bridge.messages.appendleft(record)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@@ -856,6 +866,8 @@ async def meshtastic_public_send(request: Request):
|
|||||||
"event_id": "",
|
"event_id": "",
|
||||||
"routed_via": envelope.routed_via,
|
"routed_via": envelope.routed_via,
|
||||||
"route_reason": envelope.route_reason,
|
"route_reason": envelope.route_reason,
|
||||||
|
"direct": is_direct_destination,
|
||||||
|
"channel_echo": not is_direct_destination,
|
||||||
"results": [r.to_dict() for r in results],
|
"results": [r.to_dict() for r in results],
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -954,6 +966,7 @@ async def mesh_messages(
|
|||||||
root: str = "",
|
root: str = "",
|
||||||
channel: str = "",
|
channel: str = "",
|
||||||
limit: int = 30,
|
limit: int = 30,
|
||||||
|
include_direct: bool = False,
|
||||||
):
|
):
|
||||||
"""Get recent Meshtastic text messages from the MQTT bridge."""
|
"""Get recent Meshtastic text messages from the MQTT bridge."""
|
||||||
from services.sigint_bridge import sigint_grid
|
from services.sigint_bridge import sigint_grid
|
||||||
@@ -975,6 +988,12 @@ async def mesh_messages(
|
|||||||
msgs = [m for m in msgs if m.get("root", "").upper() == root_filter]
|
msgs = [m for m in msgs if m.get("root", "").upper() == root_filter]
|
||||||
if channel:
|
if channel:
|
||||||
msgs = [m for m in msgs if m.get("channel", "").lower() == channel.lower()]
|
msgs = [m for m in msgs if m.get("channel", "").lower() == channel.lower()]
|
||||||
|
if not include_direct:
|
||||||
|
msgs = [
|
||||||
|
m
|
||||||
|
for m in msgs
|
||||||
|
if str(m.get("to") or "broadcast").strip().lower() in {"", "broadcast", "^all"}
|
||||||
|
]
|
||||||
return msgs[: min(limit, 100)]
|
return msgs[: min(limit, 100)]
|
||||||
|
|
||||||
|
|
||||||
@@ -1448,25 +1467,37 @@ def _submit_gate_message_envelope(request: Request, gate_id: str, body: dict[str
|
|||||||
@router.get("/api/mesh/infonet/status")
|
@router.get("/api/mesh/infonet/status")
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def infonet_status(request: Request, verify_signatures: bool = False):
|
async def infonet_status(request: Request, verify_signatures: bool = False):
|
||||||
"""Get Infonet metadata — event counts, head hash, chain size."""
|
"""Get Infonet metadata — event counts, head hash, chain size.
|
||||||
|
|
||||||
|
The ``verify_signatures`` query parameter is honored ONLY when the
|
||||||
|
caller has authenticated via scoped auth or local-operator credentials.
|
||||||
|
Verifying every signature in a long chain is O(n_events) work — letting
|
||||||
|
anonymous callers trigger it is a DoS surface (issue #207). For
|
||||||
|
anonymous callers we silently fall back to the cheap path; the response
|
||||||
|
structure is identical so legitimate frontends see no behavior change.
|
||||||
|
"""
|
||||||
from services.mesh.mesh_hashchain import infonet
|
from services.mesh.mesh_hashchain import infonet
|
||||||
from services.wormhole_supervisor import get_wormhole_state
|
from services.wormhole_supervisor import get_wormhole_state
|
||||||
|
|
||||||
|
# Silently downgrade for unauthenticated callers — no error surfaced.
|
||||||
|
authenticated = _scoped_view_authenticated(request, "mesh.audit")
|
||||||
|
effective_verify_signatures = bool(verify_signatures) and authenticated
|
||||||
|
|
||||||
info = infonet.get_info()
|
info = infonet.get_info()
|
||||||
valid, reason = infonet.validate_chain(verify_signatures=verify_signatures)
|
valid, reason = infonet.validate_chain(verify_signatures=effective_verify_signatures)
|
||||||
try:
|
try:
|
||||||
wormhole = get_wormhole_state()
|
wormhole = get_wormhole_state()
|
||||||
except Exception:
|
except Exception:
|
||||||
wormhole = {"configured": False, "ready": False, "rns_ready": False}
|
wormhole = {"configured": False, "ready": False, "rns_ready": False}
|
||||||
info["valid"] = valid
|
info["valid"] = valid
|
||||||
info["validation"] = reason
|
info["validation"] = reason
|
||||||
info["verify_signatures"] = verify_signatures
|
info["verify_signatures"] = effective_verify_signatures
|
||||||
info["private_lane_tier"] = _current_private_lane_tier(wormhole)
|
info["private_lane_tier"] = _current_private_lane_tier(wormhole)
|
||||||
info["private_lane_policy"] = _private_infonet_policy_snapshot()
|
info["private_lane_policy"] = _private_infonet_policy_snapshot()
|
||||||
info.update(_node_runtime_snapshot())
|
info.update(_node_runtime_snapshot())
|
||||||
return _redact_private_lane_control_fields(
|
return _redact_private_lane_control_fields(
|
||||||
info,
|
info,
|
||||||
authenticated=_scoped_view_authenticated(request, "mesh.audit"),
|
authenticated=authenticated,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -21,14 +21,30 @@ async def api_get_openmhz_systems(request: Request):
|
|||||||
return get_openmhz_systems()
|
return get_openmhz_systems()
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api/radio/openmhz/calls/{sys_name}")
|
# Issue #213: rotating sys_name bypasses the 20s TTL cache and lets an
|
||||||
|
# anonymous caller hammer api.openmhz.com through this proxy, risking an
|
||||||
|
# IP-ban for the project. require_local_operator scopes this to the local
|
||||||
|
# UI (which goes through the Next.js proxy with admin-key injection) and
|
||||||
|
# scoped agent tokens.
|
||||||
|
@router.get(
|
||||||
|
"/api/radio/openmhz/calls/{sys_name}",
|
||||||
|
dependencies=[Depends(require_local_operator)],
|
||||||
|
)
|
||||||
@limiter.limit("60/minute")
|
@limiter.limit("60/minute")
|
||||||
async def api_get_openmhz_calls(request: Request, sys_name: str):
|
async def api_get_openmhz_calls(request: Request, sys_name: str):
|
||||||
from services.radio_intercept import get_recent_openmhz_calls
|
from services.radio_intercept import get_recent_openmhz_calls
|
||||||
return get_recent_openmhz_calls(sys_name)
|
return get_recent_openmhz_calls(sys_name)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api/radio/openmhz/audio")
|
# Issue #214: this is a streaming bandwidth relay. An anonymous caller can
|
||||||
|
# stream audio through the backend, saturating the operator's outbound
|
||||||
|
# bandwidth. Scope to local operator; the legitimate browser UI still
|
||||||
|
# works because relative /api/... paths go through the Next.js proxy
|
||||||
|
# which injects the admin key automatically.
|
||||||
|
@router.get(
|
||||||
|
"/api/radio/openmhz/audio",
|
||||||
|
dependencies=[Depends(require_local_operator)],
|
||||||
|
)
|
||||||
@limiter.limit("120/minute")
|
@limiter.limit("120/minute")
|
||||||
async def api_get_openmhz_audio(request: Request, url: str = Query(..., min_length=10)):
|
async def api_get_openmhz_audio(request: Request, url: str = Query(..., min_length=10)):
|
||||||
from services.radio_intercept import openmhz_audio_response
|
from services.radio_intercept import openmhz_audio_response
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ async def oracle_region_intel(
|
|||||||
return get_region_oracle_intel(lat, lng, news_items)
|
return get_region_oracle_intel(lat, lng, news_items)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api/thermal/verify")
|
@router.get("/api/thermal/verify", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("10/minute")
|
@limiter.limit("10/minute")
|
||||||
async def thermal_verify(
|
async def thermal_verify(
|
||||||
request: Request,
|
request: Request,
|
||||||
@@ -35,7 +35,7 @@ async def thermal_verify(
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/sigint/transmit")
|
@router.post("/api/sigint/transmit", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("5/minute")
|
@limiter.limit("5/minute")
|
||||||
async def sigint_transmit(request: Request):
|
async def sigint_transmit(request: Request):
|
||||||
"""Send an APRS-IS message to a specific callsign. Requires ham radio credentials."""
|
"""Send an APRS-IS message to a specific callsign. Requires ham radio credentials."""
|
||||||
|
|||||||
@@ -85,7 +85,30 @@ async def api_geocode_reverse(
|
|||||||
return await asyncio.to_thread(reverse_geocode, lat, lng, local_only)
|
return await asyncio.to_thread(reverse_geocode, lat, lng, local_only)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api/sentinel2/search")
|
# ── Sentinel proxy routes (Issue #299/#300/#301, reported by tg12) ──────────
|
||||||
|
# These three endpoints relay external Sentinel / Planetary Computer
|
||||||
|
# requests through the backend to avoid browser CORS blocks. They are
|
||||||
|
# operator-only helpers — they MUST NOT be callable by anonymous remote
|
||||||
|
# users, because:
|
||||||
|
#
|
||||||
|
# * /api/sentinel/token — caller supplies their own Sentinel client_id +
|
||||||
|
# client_secret. Without operator gating, the backend becomes a free
|
||||||
|
# anonymous OAuth-mint relay for any Copernicus account.
|
||||||
|
# * /api/sentinel/tile — same shape as the token route but for tile
|
||||||
|
# imagery. Without gating, the backend acts as an anonymous quota and
|
||||||
|
# bandwidth relay for Sentinel Hub Process API calls.
|
||||||
|
# * /api/sentinel2/search — hits the Planetary Computer STAC search API
|
||||||
|
# and falls back to Esri imagery. No caller credentials are involved,
|
||||||
|
# but the route is still an anonymous external-search relay. We gate
|
||||||
|
# it the same way for consistency with the rest of the operator-only
|
||||||
|
# helper surface.
|
||||||
|
#
|
||||||
|
# Gating is via require_local_operator (loopback / bridge / admin key),
|
||||||
|
# matching the same allowlist already used by /api/region-dossier and
|
||||||
|
# the other operator helpers further up this file. Single-operator nodes
|
||||||
|
# see no behavior change — their dashboard already lives on loopback or
|
||||||
|
# the trusted Docker bridge, so it still resolves.
|
||||||
|
@router.get("/api/sentinel2/search", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
def api_sentinel2_search(
|
def api_sentinel2_search(
|
||||||
request: Request,
|
request: Request,
|
||||||
@@ -97,7 +120,7 @@ def api_sentinel2_search(
|
|||||||
return search_sentinel2_scene(lat, lng)
|
return search_sentinel2_scene(lat, lng)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/sentinel/token")
|
@router.post("/api/sentinel/token", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("60/minute")
|
@limiter.limit("60/minute")
|
||||||
async def api_sentinel_token(request: Request):
|
async def api_sentinel_token(request: Request):
|
||||||
"""Proxy Copernicus CDSE OAuth2 token request (avoids browser CORS block)."""
|
"""Proxy Copernicus CDSE OAuth2 token request (avoids browser CORS block)."""
|
||||||
@@ -120,10 +143,39 @@ async def api_sentinel_token(request: Request):
|
|||||||
raise HTTPException(502, "Token request failed")
|
raise HTTPException(502, "Token request failed")
|
||||||
|
|
||||||
|
|
||||||
_sh_token_cache: dict = {"token": None, "expiry": 0, "client_id": ""}
|
# Cache key is an HMAC of (client_id, client_secret) — a caller cannot hit
|
||||||
|
# this cache without knowing the same secret that originally populated it.
|
||||||
|
# Without this binding, the lookup only checked client_id, so anyone who
|
||||||
|
# knew a valid client_id could reuse another caller's cached token (and
|
||||||
|
# burn their Copernicus quota / access tiles on their account).
|
||||||
|
_sh_token_cache: dict = {"token": None, "expiry": 0, "credential_fp": ""}
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/sentinel/tile")
|
def _credential_fingerprint(client_id: str, client_secret: str) -> str:
|
||||||
|
"""Return a stable, secret-binding fingerprint for the Sentinel cache key.
|
||||||
|
|
||||||
|
Uses HMAC-SHA256 so the raw secret is never stored in process memory as
|
||||||
|
a cache key. The HMAC key is a per-process random value, which means the
|
||||||
|
fingerprint cannot be precomputed across restarts (additional defense
|
||||||
|
against an attacker who learned a valid client_id but not the secret).
|
||||||
|
"""
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
|
||||||
|
return hmac.new(
|
||||||
|
_SH_TOKEN_CACHE_HMAC_KEY,
|
||||||
|
f"{client_id}\x00{client_secret}".encode("utf-8"),
|
||||||
|
hashlib.sha256,
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
# Per-process random HMAC key. Regenerated on each backend startup so cached
|
||||||
|
# fingerprints don't survive restarts.
|
||||||
|
import os as _os
|
||||||
|
_SH_TOKEN_CACHE_HMAC_KEY = _os.urandom(32)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/api/sentinel/tile", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("300/minute")
|
@limiter.limit("300/minute")
|
||||||
async def api_sentinel_tile(request: Request):
|
async def api_sentinel_tile(request: Request):
|
||||||
"""Proxy Sentinel Hub Process API tile request (avoids CORS block)."""
|
"""Proxy Sentinel Hub Process API tile request (avoids CORS block)."""
|
||||||
@@ -146,7 +198,9 @@ async def api_sentinel_tile(request: Request):
|
|||||||
raise HTTPException(400, "client_id, client_secret, and date required")
|
raise HTTPException(400, "client_id, client_secret, and date required")
|
||||||
|
|
||||||
now = _time.time()
|
now = _time.time()
|
||||||
if (_sh_token_cache["token"] and _sh_token_cache["client_id"] == client_id
|
credential_fp = _credential_fingerprint(client_id, client_secret)
|
||||||
|
if (_sh_token_cache["token"]
|
||||||
|
and _sh_token_cache["credential_fp"] == credential_fp
|
||||||
and now < _sh_token_cache["expiry"] - 30):
|
and now < _sh_token_cache["expiry"] - 30):
|
||||||
token = _sh_token_cache["token"]
|
token = _sh_token_cache["token"]
|
||||||
else:
|
else:
|
||||||
@@ -161,7 +215,7 @@ async def api_sentinel_tile(request: Request):
|
|||||||
token = tdata["access_token"]
|
token = tdata["access_token"]
|
||||||
_sh_token_cache["token"] = token
|
_sh_token_cache["token"] = token
|
||||||
_sh_token_cache["expiry"] = now + tdata.get("expires_in", 300)
|
_sh_token_cache["expiry"] = now + tdata.get("expires_in", 300)
|
||||||
_sh_token_cache["client_id"] = client_id
|
_sh_token_cache["credential_fp"] = credential_fp
|
||||||
except HTTPException:
|
except HTTPException:
|
||||||
raise
|
raise
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|||||||
+67
-313
@@ -78,6 +78,21 @@ export_wormhole_dm_invite = getattr(
|
|||||||
"export_wormhole_dm_invite",
|
"export_wormhole_dm_invite",
|
||||||
_wormhole_identity_unavailable,
|
_wormhole_identity_unavailable,
|
||||||
)
|
)
|
||||||
|
list_prekey_lookup_handle_records_for_ui = getattr(
|
||||||
|
_mesh_wormhole_identity,
|
||||||
|
"list_prekey_lookup_handle_records_for_ui",
|
||||||
|
_wormhole_identity_unavailable,
|
||||||
|
)
|
||||||
|
rename_prekey_lookup_handle = getattr(
|
||||||
|
_mesh_wormhole_identity,
|
||||||
|
"rename_prekey_lookup_handle",
|
||||||
|
_wormhole_identity_unavailable,
|
||||||
|
)
|
||||||
|
revoke_prekey_lookup_handle = getattr(
|
||||||
|
_mesh_wormhole_identity,
|
||||||
|
"revoke_prekey_lookup_handle",
|
||||||
|
_wormhole_identity_unavailable,
|
||||||
|
)
|
||||||
import_wormhole_dm_invite = getattr(
|
import_wormhole_dm_invite = getattr(
|
||||||
_mesh_wormhole_identity,
|
_mesh_wormhole_identity,
|
||||||
"import_wormhole_dm_invite",
|
"import_wormhole_dm_invite",
|
||||||
@@ -145,8 +160,13 @@ router = APIRouter()
|
|||||||
|
|
||||||
# --- Constants ---
|
# --- Constants ---
|
||||||
|
|
||||||
_WORMHOLE_PUBLIC_SETTINGS_FIELDS = {"enabled", "transport", "anonymous_mode"}
|
# Issue #243 (tg12): the public redaction now exposes only the bare
|
||||||
_WORMHOLE_PUBLIC_PROFILE_FIELDS = {"profile", "wormhole_enabled"}
|
# "is this on?" boolean. Transport choice, anonymous-mode state, and
|
||||||
|
# the named privacy profile were all leaking actionable recon to
|
||||||
|
# unauthenticated callers and are now gated behind authenticated reads.
|
||||||
|
# See the matching block in backend/main.py for the full rationale.
|
||||||
|
_WORMHOLE_PUBLIC_SETTINGS_FIELDS = {"enabled"}
|
||||||
|
_WORMHOLE_PUBLIC_PROFILE_FIELDS = {"wormhole_enabled"}
|
||||||
_PRIVATE_LANE_CONTROL_FIELDS = {"private_lane_tier", "private_lane_policy"}
|
_PRIVATE_LANE_CONTROL_FIELDS = {"private_lane_tier", "private_lane_policy"}
|
||||||
_PUBLIC_RNS_STATUS_FIELDS = {"enabled", "ready", "configured_peers", "active_peers"}
|
_PUBLIC_RNS_STATUS_FIELDS = {"enabled", "ready", "configured_peers", "active_peers"}
|
||||||
_NODE_PUBLIC_EVENT_HOOK_REGISTERED = False
|
_NODE_PUBLIC_EVENT_HOOK_REGISTERED = False
|
||||||
@@ -311,6 +331,10 @@ class WormholeDmInviteImportRequest(BaseModel):
|
|||||||
alias: str = ""
|
alias: str = ""
|
||||||
|
|
||||||
|
|
||||||
|
class WormholeDmInviteHandleUpdateRequest(BaseModel):
|
||||||
|
label: str = ""
|
||||||
|
|
||||||
|
|
||||||
class WormholeDmSenderTokenRequest(BaseModel):
|
class WormholeDmSenderTokenRequest(BaseModel):
|
||||||
recipient_id: str
|
recipient_id: str
|
||||||
delivery_class: str
|
delivery_class: str
|
||||||
@@ -477,6 +501,7 @@ def decrypt_wormhole_dm_envelope(
|
|||||||
remote_alias: str | None = None,
|
remote_alias: str | None = None,
|
||||||
session_welcome: str | None = None,
|
session_welcome: str | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
|
"""Delegate to main.py, which owns current MLS/alias/legacy gating behavior."""
|
||||||
import main as _m
|
import main as _m
|
||||||
|
|
||||||
return _m.decrypt_wormhole_dm_envelope(
|
return _m.decrypt_wormhole_dm_envelope(
|
||||||
@@ -489,71 +514,13 @@ def decrypt_wormhole_dm_envelope(
|
|||||||
session_welcome=session_welcome,
|
session_welcome=session_welcome,
|
||||||
)
|
)
|
||||||
|
|
||||||
resolved_local, resolved_remote = _resolve_dm_aliases(
|
|
||||||
peer_id=peer_id,
|
|
||||||
local_alias=local_alias,
|
|
||||||
remote_alias=remote_alias,
|
|
||||||
)
|
|
||||||
normalized_format = str(payload_format or "dm1").strip().lower() or "dm1"
|
|
||||||
if normalized_format != "mls1" and is_dm_locked_to_mls(resolved_local, resolved_remote):
|
|
||||||
return {
|
|
||||||
"ok": False,
|
|
||||||
"detail": "DM session is locked to MLS format",
|
|
||||||
"required_format": "mls1",
|
|
||||||
"current_format": normalized_format,
|
|
||||||
}
|
|
||||||
if normalized_format == "mls1":
|
|
||||||
has_session = has_mls_dm_session(resolved_local, resolved_remote)
|
|
||||||
if not has_session.get("ok"):
|
|
||||||
return has_session
|
|
||||||
if not has_session.get("exists"):
|
|
||||||
ensured = ensure_mls_dm_session(resolved_local, resolved_remote, str(session_welcome or ""))
|
|
||||||
if not ensured.get("ok"):
|
|
||||||
return ensured
|
|
||||||
decrypted = decrypt_mls_dm(
|
|
||||||
resolved_local,
|
|
||||||
resolved_remote,
|
|
||||||
str(ciphertext or ""),
|
|
||||||
str(nonce or ""),
|
|
||||||
)
|
|
||||||
if not decrypted.get("ok"):
|
|
||||||
return decrypted
|
|
||||||
return {
|
|
||||||
"ok": True,
|
|
||||||
"peer_id": str(peer_id or "").strip(),
|
|
||||||
"local_alias": resolved_local,
|
|
||||||
"remote_alias": resolved_remote,
|
|
||||||
"plaintext": str(decrypted.get("plaintext", "") or ""),
|
|
||||||
"format": "mls1",
|
|
||||||
}
|
|
||||||
|
|
||||||
from services.wormhole_supervisor import get_transport_tier
|
|
||||||
|
|
||||||
current_tier = get_transport_tier()
|
|
||||||
if str(current_tier or "").startswith("private_"):
|
|
||||||
return {
|
|
||||||
"ok": False,
|
|
||||||
"detail": "MLS format required in private transport mode — legacy DM decrypt blocked",
|
|
||||||
}
|
|
||||||
logger.warning("legacy dm decrypt path used")
|
|
||||||
legacy = decrypt_wormhole_dm(peer_id=str(peer_id or ""), ciphertext=str(ciphertext or ""))
|
|
||||||
if not legacy.get("ok"):
|
|
||||||
return legacy
|
|
||||||
return {
|
|
||||||
"ok": True,
|
|
||||||
"peer_id": str(peer_id or "").strip(),
|
|
||||||
"local_alias": resolved_local,
|
|
||||||
"remote_alias": resolved_remote,
|
|
||||||
"plaintext": str(legacy.get("result", "") or ""),
|
|
||||||
"format": "dm1",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# --- Routes ---
|
# --- Routes ---
|
||||||
|
|
||||||
@router.get("/api/settings/wormhole")
|
@router.get("/api/settings/wormhole")
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("240/minute")
|
||||||
async def api_get_wormhole_settings(request: Request):
|
async def api_get_wormhole_settings(request: Request):
|
||||||
settings = await asyncio.to_thread(read_wormhole_settings)
|
settings = await asyncio.to_thread(read_wormhole_settings)
|
||||||
return _redact_wormhole_settings(settings, authenticated=_scoped_view_authenticated(request, "wormhole"))
|
return _redact_wormhole_settings(settings, authenticated=_scoped_view_authenticated(request, "wormhole"))
|
||||||
@@ -582,248 +549,9 @@ async def api_set_wormhole_settings(request: Request, body: WormholeUpdate):
|
|||||||
return {**updated, "requires_restart": False, "runtime": state}
|
return {**updated, "requires_restart": False, "runtime": state}
|
||||||
|
|
||||||
|
|
||||||
class PrivacyProfileUpdate(BaseModel):
|
|
||||||
profile: str
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeSignRequest(BaseModel):
|
|
||||||
event_type: str
|
|
||||||
payload: dict
|
|
||||||
sequence: int | None = None
|
|
||||||
gate_id: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeSignRawRequest(BaseModel):
|
|
||||||
message: str
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeDmEncryptRequest(BaseModel):
|
|
||||||
peer_id: str
|
|
||||||
peer_dh_pub: str = ""
|
|
||||||
plaintext: str
|
|
||||||
local_alias: str | None = None
|
|
||||||
remote_alias: str | None = None
|
|
||||||
remote_prekey_bundle: dict[str, Any] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeDmComposeRequest(BaseModel):
|
|
||||||
peer_id: str
|
|
||||||
peer_dh_pub: str = ""
|
|
||||||
plaintext: str
|
|
||||||
local_alias: str | None = None
|
|
||||||
remote_alias: str | None = None
|
|
||||||
remote_prekey_bundle: dict[str, Any] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeDmDecryptRequest(BaseModel):
|
|
||||||
peer_id: str
|
|
||||||
ciphertext: str
|
|
||||||
format: str = "dm1"
|
|
||||||
nonce: str = ""
|
|
||||||
local_alias: str | None = None
|
|
||||||
remote_alias: str | None = None
|
|
||||||
session_welcome: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeDmResetRequest(BaseModel):
|
|
||||||
peer_id: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeDmBootstrapEncryptRequest(BaseModel):
|
|
||||||
peer_id: str
|
|
||||||
plaintext: str
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeDmBootstrapDecryptRequest(BaseModel):
|
|
||||||
sender_id: str = ""
|
|
||||||
ciphertext: str
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeDmSenderTokenRequest(BaseModel):
|
|
||||||
recipient_id: str
|
|
||||||
delivery_class: str
|
|
||||||
recipient_token: str = ""
|
|
||||||
count: int = 1
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeOpenSealRequest(BaseModel):
|
|
||||||
sender_seal: str
|
|
||||||
candidate_dh_pub: str = ""
|
|
||||||
recipient_id: str
|
|
||||||
expected_msg_id: str
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeBuildSealRequest(BaseModel):
|
|
||||||
recipient_id: str
|
|
||||||
recipient_dh_pub: str = ""
|
|
||||||
msg_id: str
|
|
||||||
timestamp: int
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeDeadDropTokenRequest(BaseModel):
|
|
||||||
peer_id: str
|
|
||||||
peer_dh_pub: str = ""
|
|
||||||
peer_ref: str = ""
|
|
||||||
|
|
||||||
|
|
||||||
class WormholePairwiseAliasRequest(BaseModel):
|
|
||||||
peer_id: str
|
|
||||||
peer_dh_pub: str = ""
|
|
||||||
|
|
||||||
|
|
||||||
class WormholePairwiseAliasRotateRequest(BaseModel):
|
|
||||||
peer_id: str
|
|
||||||
peer_dh_pub: str = ""
|
|
||||||
grace_ms: int = 45_000
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeDeadDropContactsRequest(BaseModel):
|
|
||||||
contacts: list[dict[str, Any]]
|
|
||||||
limit: int = 24
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeSasRequest(BaseModel):
|
|
||||||
peer_id: str
|
|
||||||
peer_dh_pub: str = ""
|
|
||||||
words: int = 8
|
|
||||||
peer_ref: str = ""
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeGateRequest(BaseModel):
|
|
||||||
gate_id: str
|
|
||||||
rotate: bool = False
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeGatePersonaCreateRequest(BaseModel):
|
|
||||||
gate_id: str
|
|
||||||
label: str = ""
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeGatePersonaActivateRequest(BaseModel):
|
|
||||||
gate_id: str
|
|
||||||
persona_id: str
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeGateKeyGrantRequest(BaseModel):
|
|
||||||
gate_id: str
|
|
||||||
recipient_node_id: str
|
|
||||||
recipient_dh_pub: str
|
|
||||||
recipient_scope: str = "member"
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeGateComposeRequest(BaseModel):
|
|
||||||
gate_id: str
|
|
||||||
plaintext: str
|
|
||||||
reply_to: str = ""
|
|
||||||
compat_plaintext: bool = False
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeGateDecryptRequest(BaseModel):
|
|
||||||
gate_id: str
|
|
||||||
epoch: int = 0
|
|
||||||
ciphertext: str
|
|
||||||
nonce: str = ""
|
|
||||||
sender_ref: str = ""
|
|
||||||
format: str = "mls1"
|
|
||||||
gate_envelope: str = ""
|
|
||||||
envelope_hash: str = ""
|
|
||||||
recovery_envelope: bool = False
|
|
||||||
compat_decrypt: bool = False
|
|
||||||
event_id: str = ""
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeGateDecryptBatchRequest(BaseModel):
|
|
||||||
messages: list[WormholeGateDecryptRequest]
|
|
||||||
|
|
||||||
|
|
||||||
class WormholeGateRotateRequest(BaseModel):
|
|
||||||
gate_id: str
|
|
||||||
reason: str = "manual_rotate"
|
|
||||||
|
|
||||||
def decrypt_wormhole_dm_envelope(
|
|
||||||
*,
|
|
||||||
peer_id: str,
|
|
||||||
ciphertext: str,
|
|
||||||
payload_format: str = "dm1",
|
|
||||||
nonce: str = "",
|
|
||||||
local_alias: str | None = None,
|
|
||||||
remote_alias: str | None = None,
|
|
||||||
session_welcome: str | None = None,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
import main as _m
|
|
||||||
|
|
||||||
return _m.decrypt_wormhole_dm_envelope(
|
|
||||||
peer_id=peer_id,
|
|
||||||
ciphertext=ciphertext,
|
|
||||||
payload_format=payload_format,
|
|
||||||
nonce=nonce,
|
|
||||||
local_alias=local_alias,
|
|
||||||
remote_alias=remote_alias,
|
|
||||||
session_welcome=session_welcome,
|
|
||||||
)
|
|
||||||
|
|
||||||
resolved_local, resolved_remote = _resolve_dm_aliases(
|
|
||||||
peer_id=peer_id,
|
|
||||||
local_alias=local_alias,
|
|
||||||
remote_alias=remote_alias,
|
|
||||||
)
|
|
||||||
normalized_format = str(payload_format or "dm1").strip().lower() or "dm1"
|
|
||||||
if normalized_format != "mls1" and is_dm_locked_to_mls(resolved_local, resolved_remote):
|
|
||||||
return {
|
|
||||||
"ok": False,
|
|
||||||
"detail": "DM session is locked to MLS format",
|
|
||||||
"required_format": "mls1",
|
|
||||||
"current_format": normalized_format,
|
|
||||||
}
|
|
||||||
if normalized_format == "mls1":
|
|
||||||
has_session = has_mls_dm_session(resolved_local, resolved_remote)
|
|
||||||
if not has_session.get("ok"):
|
|
||||||
return has_session
|
|
||||||
if not has_session.get("exists"):
|
|
||||||
ensured = ensure_mls_dm_session(resolved_local, resolved_remote, str(session_welcome or ""))
|
|
||||||
if not ensured.get("ok"):
|
|
||||||
return ensured
|
|
||||||
decrypted = decrypt_mls_dm(
|
|
||||||
resolved_local,
|
|
||||||
resolved_remote,
|
|
||||||
str(ciphertext or ""),
|
|
||||||
str(nonce or ""),
|
|
||||||
)
|
|
||||||
if not decrypted.get("ok"):
|
|
||||||
return decrypted
|
|
||||||
return {
|
|
||||||
"ok": True,
|
|
||||||
"peer_id": str(peer_id or "").strip(),
|
|
||||||
"local_alias": resolved_local,
|
|
||||||
"remote_alias": resolved_remote,
|
|
||||||
"plaintext": str(decrypted.get("plaintext", "") or ""),
|
|
||||||
"format": "mls1",
|
|
||||||
}
|
|
||||||
|
|
||||||
from services.wormhole_supervisor import get_transport_tier
|
|
||||||
|
|
||||||
current_tier = get_transport_tier()
|
|
||||||
if str(current_tier or "").startswith("private_"):
|
|
||||||
return {
|
|
||||||
"ok": False,
|
|
||||||
"detail": "MLS format required in private transport mode — legacy DM decrypt blocked",
|
|
||||||
}
|
|
||||||
logger.warning("legacy dm decrypt path used")
|
|
||||||
legacy = decrypt_wormhole_dm(peer_id=str(peer_id or ""), ciphertext=str(ciphertext or ""))
|
|
||||||
if not legacy.get("ok"):
|
|
||||||
return legacy
|
|
||||||
return {
|
|
||||||
"ok": True,
|
|
||||||
"peer_id": str(peer_id or "").strip(),
|
|
||||||
"local_alias": resolved_local,
|
|
||||||
"remote_alias": resolved_remote,
|
|
||||||
"plaintext": str(legacy.get("result", "") or ""),
|
|
||||||
"format": "dm1",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api/settings/privacy-profile")
|
@router.get("/api/settings/privacy-profile")
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("240/minute")
|
||||||
async def api_get_privacy_profile(request: Request):
|
async def api_get_privacy_profile(request: Request):
|
||||||
data = await asyncio.to_thread(read_wormhole_settings)
|
data = await asyncio.to_thread(read_wormhole_settings)
|
||||||
return _redact_privacy_profile_settings(
|
return _redact_privacy_profile_settings(
|
||||||
@@ -833,7 +561,7 @@ async def api_get_privacy_profile(request: Request):
|
|||||||
|
|
||||||
|
|
||||||
@router.get("/api/settings/wormhole-status")
|
@router.get("/api/settings/wormhole-status")
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("240/minute")
|
||||||
async def api_get_wormhole_status(request: Request):
|
async def api_get_wormhole_status(request: Request):
|
||||||
state = await asyncio.to_thread(get_wormhole_state)
|
state = await asyncio.to_thread(get_wormhole_state)
|
||||||
transport_tier = _current_private_lane_tier(state)
|
transport_tier = _current_private_lane_tier(state)
|
||||||
@@ -907,7 +635,7 @@ async def api_wormhole_join(request: Request):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Enable node participation so the sync/push workers connect to peers.
|
# Enable node participation so the sync/push workers connect to peers.
|
||||||
# This is the voluntary opt-in — the node only joins the network when
|
# This is the voluntary opt-in — the node only joins the network when
|
||||||
# the user explicitly opens the Wormhole.
|
# the user explicitly opens the Wormhole.
|
||||||
from services.node_settings import write_node_settings
|
from services.node_settings import write_node_settings
|
||||||
|
|
||||||
@@ -923,7 +651,7 @@ async def api_wormhole_join(request: Request):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/wormhole/leave", dependencies=[Depends(require_local_operator)])
|
@router.post("/api/wormhole/leave")
|
||||||
@limiter.limit("10/minute")
|
@limiter.limit("10/minute")
|
||||||
async def api_wormhole_leave(request: Request):
|
async def api_wormhole_leave(request: Request):
|
||||||
updated = write_wormhole_settings(enabled=False)
|
updated = write_wormhole_settings(enabled=False)
|
||||||
@@ -941,7 +669,7 @@ async def api_wormhole_leave(request: Request):
|
|||||||
|
|
||||||
|
|
||||||
@router.get("/api/wormhole/identity", dependencies=[Depends(require_local_operator)])
|
@router.get("/api/wormhole/identity", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("240/minute")
|
||||||
async def api_wormhole_identity(request: Request):
|
async def api_wormhole_identity(request: Request):
|
||||||
try:
|
try:
|
||||||
bootstrap_wormhole_persona_state()
|
bootstrap_wormhole_persona_state()
|
||||||
@@ -970,7 +698,7 @@ async def api_wormhole_identity_bootstrap(request: Request):
|
|||||||
|
|
||||||
|
|
||||||
@router.get("/api/wormhole/dm/identity", dependencies=[Depends(require_local_operator)])
|
@router.get("/api/wormhole/dm/identity", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("240/minute")
|
||||||
async def api_wormhole_dm_identity(request: Request):
|
async def api_wormhole_dm_identity(request: Request):
|
||||||
try:
|
try:
|
||||||
bootstrap_wormhole_persona_state()
|
bootstrap_wormhole_persona_state()
|
||||||
@@ -982,11 +710,37 @@ async def api_wormhole_dm_identity(request: Request):
|
|||||||
|
|
||||||
@router.get("/api/wormhole/dm/invite", dependencies=[Depends(require_local_operator)])
|
@router.get("/api/wormhole/dm/invite", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def api_wormhole_dm_invite(request: Request):
|
async def api_wormhole_dm_invite(
|
||||||
return export_wormhole_dm_invite()
|
request: Request,
|
||||||
|
label: str = Query("", max_length=96),
|
||||||
|
expires_in_s: int = Query(0, ge=0, le=2_592_000),
|
||||||
|
):
|
||||||
|
return export_wormhole_dm_invite(label=label, expires_in_s=expires_in_s)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/wormhole/dm/invite/import", dependencies=[Depends(require_admin)])
|
@router.get("/api/wormhole/dm/invite/handles", dependencies=[Depends(require_local_operator)])
|
||||||
|
@limiter.limit("240/minute")
|
||||||
|
async def api_wormhole_dm_invite_handles(request: Request):
|
||||||
|
return list_prekey_lookup_handle_records_for_ui()
|
||||||
|
|
||||||
|
|
||||||
|
@router.patch("/api/wormhole/dm/invite/handles/{handle}", dependencies=[Depends(require_local_operator)])
|
||||||
|
@limiter.limit("60/minute")
|
||||||
|
async def api_wormhole_dm_invite_handle_update(
|
||||||
|
request: Request,
|
||||||
|
handle: str,
|
||||||
|
body: WormholeDmInviteHandleUpdateRequest,
|
||||||
|
):
|
||||||
|
return rename_prekey_lookup_handle(handle, str(body.label or "").strip())
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/api/wormhole/dm/invite/handles/{handle}", dependencies=[Depends(require_local_operator)])
|
||||||
|
@limiter.limit("30/minute")
|
||||||
|
async def api_wormhole_dm_invite_handle_revoke(request: Request, handle: str):
|
||||||
|
return revoke_prekey_lookup_handle(handle)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/api/wormhole/dm/invite/import", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def api_wormhole_dm_invite_import(request: Request, body: WormholeDmInviteImportRequest):
|
async def api_wormhole_dm_invite_import(request: Request, body: WormholeDmInviteImportRequest):
|
||||||
return import_wormhole_dm_invite(
|
return import_wormhole_dm_invite(
|
||||||
@@ -1205,7 +959,7 @@ async def api_wormhole_gate_message_sign_encrypted(
|
|||||||
return await _m.api_wormhole_gate_message_sign_encrypted(request, body)
|
return await _m.api_wormhole_gate_message_sign_encrypted(request, body)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/wormhole/gate/message/post-encrypted", dependencies=[Depends(require_local_operator)])
|
@router.post("/api/wormhole/gate/message/post-encrypted")
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def api_wormhole_gate_message_post_encrypted(
|
async def api_wormhole_gate_message_post_encrypted(
|
||||||
request: Request,
|
request: Request,
|
||||||
@@ -1547,7 +1301,7 @@ class PrivateDeliveryActionRequest(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
@router.get("/api/wormhole/status")
|
@router.get("/api/wormhole/status")
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("240/minute")
|
||||||
async def api_wormhole_status(request: Request):
|
async def api_wormhole_status(request: Request):
|
||||||
import main as _m
|
import main as _m
|
||||||
|
|
||||||
@@ -1590,7 +1344,7 @@ async def api_wormhole_private_delivery_action(
|
|||||||
|
|
||||||
|
|
||||||
@router.get("/api/wormhole/health")
|
@router.get("/api/wormhole/health")
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("240/minute")
|
||||||
async def api_wormhole_health(request: Request):
|
async def api_wormhole_health(request: Request):
|
||||||
state = get_wormhole_state()
|
state = get_wormhole_state()
|
||||||
transport_tier = _current_private_lane_tier(state)
|
transport_tier = _current_private_lane_tier(state)
|
||||||
@@ -1611,7 +1365,7 @@ async def api_wormhole_health(request: Request):
|
|||||||
return _redact_wormhole_status(full_state, authenticated=ok)
|
return _redact_wormhole_status(full_state, authenticated=ok)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/wormhole/connect", dependencies=[Depends(require_admin)])
|
@router.post("/api/wormhole/connect", dependencies=[Depends(require_local_operator)])
|
||||||
@limiter.limit("10/minute")
|
@limiter.limit("10/minute")
|
||||||
async def api_wormhole_connect(request: Request):
|
async def api_wormhole_connect(request: Request):
|
||||||
settings = read_wormhole_settings()
|
settings = read_wormhole_settings()
|
||||||
|
|||||||
@@ -20,7 +20,17 @@ OUT_PATH = Path(__file__).parent.parent / "data" / "power_plants.json"
|
|||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
print(f"Downloading WRI Global Power Plant Database from GitHub...")
|
print(f"Downloading WRI Global Power Plant Database from GitHub...")
|
||||||
req = urllib.request.Request(CSV_URL, headers={"User-Agent": "ShadowBroker-OSINT/1.0"})
|
# Round 7a: release-time data refresher. Uses the per-operator UA if
|
||||||
|
# available, otherwise a release-script-specific identifier. This
|
||||||
|
# script is run by the maintainer at release time, NOT at runtime,
|
||||||
|
# so an aggregate UA is acceptable; we still use the helper so the
|
||||||
|
# behavior matches the rest of the project.
|
||||||
|
try:
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
|
ua = outbound_user_agent("release-script-power-plants")
|
||||||
|
except Exception:
|
||||||
|
ua = "Shadowbroker/0.9 (release-script-power-plants; +https://github.com/BigBodyCobain/Shadowbroker/issues)"
|
||||||
|
req = urllib.request.Request(CSV_URL, headers={"User-Agent": ua})
|
||||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||||
raw = resp.read().decode("utf-8")
|
raw = resp.read().decode("utf-8")
|
||||||
|
|
||||||
|
|||||||
@@ -344,9 +344,26 @@ _vessels_lock = threading.Lock()
|
|||||||
_ws_thread: threading.Thread | None = None
|
_ws_thread: threading.Thread | None = None
|
||||||
_ws_running = False
|
_ws_running = False
|
||||||
_proxy_process = None
|
_proxy_process = None
|
||||||
|
# Issue #258: latest status snapshot emitted by ais_proxy.js. Populated when
|
||||||
|
# the proxy reports e.g. {"__ais_proxy_status": {"degraded_tls": true}} on
|
||||||
|
# stdout, which it does when it falls back to the SPKI-pinned insecure-date
|
||||||
|
# path during an upstream cert outage. Surfaced via ais_proxy_status() for
|
||||||
|
# /api/health.
|
||||||
|
_proxy_status: dict = {}
|
||||||
_VESSEL_TRAIL_INTERVAL_S = 120
|
_VESSEL_TRAIL_INTERVAL_S = 120
|
||||||
_VESSEL_TRAIL_MAX_POINTS = 240
|
_VESSEL_TRAIL_MAX_POINTS = 240
|
||||||
|
|
||||||
|
|
||||||
|
def ais_proxy_status() -> dict:
|
||||||
|
"""Return a copy of the latest ais_proxy.js status (issue #258).
|
||||||
|
|
||||||
|
Currently surfaces ``degraded_tls`` (bool) which is true when the
|
||||||
|
proxy is using SPKI-pinned fallback because AISStream's cert expired.
|
||||||
|
Returns an empty dict when no status has been received yet.
|
||||||
|
"""
|
||||||
|
with _vessels_lock:
|
||||||
|
return dict(_proxy_status)
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
CACHE_FILE = os.path.join(os.path.dirname(__file__), "ais_cache.json")
|
CACHE_FILE = os.path.join(os.path.dirname(__file__), "ais_cache.json")
|
||||||
@@ -608,6 +625,18 @@ def _ais_stream_loop():
|
|||||||
logger.error(f"AIS Stream error: {data['error']}")
|
logger.error(f"AIS Stream error: {data['error']}")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# Issue #258: ais_proxy.js emits status markers (e.g.
|
||||||
|
# {"__ais_proxy_status": {"degraded_tls": true}}) when the
|
||||||
|
# SPKI-pinned fallback is in use. We snapshot the latest
|
||||||
|
# status so the backend can expose it on /api/health.
|
||||||
|
if isinstance(data, dict) and "__ais_proxy_status" in data:
|
||||||
|
status = data.get("__ais_proxy_status") or {}
|
||||||
|
if isinstance(status, dict):
|
||||||
|
with _vessels_lock:
|
||||||
|
_proxy_status.clear()
|
||||||
|
_proxy_status.update(status)
|
||||||
|
continue
|
||||||
|
|
||||||
msg_type = data.get("MessageType", "")
|
msg_type = data.get("MessageType", "")
|
||||||
metadata = data.get("MetaData", {})
|
metadata = data.get("MetaData", {})
|
||||||
message = data.get("Message", {})
|
message = data.get("Message", {})
|
||||||
|
|||||||
+407
-173
@@ -1,46 +1,90 @@
|
|||||||
"""
|
"""
|
||||||
Carrier Strike Group OSINT Tracker
|
Carrier Strike Group OSINT Tracker
|
||||||
===================================
|
===================================
|
||||||
Scrapes multiple OSINT sources to maintain current estimated positions
|
Maintains estimated positions for US Navy Carrier Strike Groups with
|
||||||
for US Navy Carrier Strike Groups. Updates on startup + 00:00 & 12:00 UTC.
|
honest provenance and freshness signals.
|
||||||
|
|
||||||
Sources:
|
Issues #244 / #245 / #246 (tg12 external audit):
|
||||||
1. GDELT News API — recent carrier movement headlines
|
|
||||||
2. WikiVoyage / public port-call databases
|
The previous implementation baked a snapshot of USNI News Fleet &
|
||||||
3. Fallback — last-known or static OSINT estimates
|
Marine Tracker positions (March 9, 2026) into the registry as
|
||||||
|
``fallback_lat``/``fallback_lng`` and stamped ``updated = now()``
|
||||||
|
every time the dossier was rendered. That presented stale editorial
|
||||||
|
data as live state. It also persisted GDELT-derived positions to the
|
||||||
|
on-disk cache with no freshness signal, so a single news mention from
|
||||||
|
months ago could keep overriding the (already-stale) registry default
|
||||||
|
indefinitely.
|
||||||
|
|
||||||
|
Architecture after this PR:
|
||||||
|
|
||||||
|
::
|
||||||
|
|
||||||
|
backend/data/carrier_seed.json read-only, shipped with image,
|
||||||
|
used ONCE on first-ever startup
|
||||||
|
to bootstrap carrier_cache.json.
|
||||||
|
|
||||||
|
backend/data/carrier_cache.json mutable, lives in the runtime data
|
||||||
|
volume, written by every GDELT
|
||||||
|
refresh + any future source.
|
||||||
|
|
||||||
|
Startup flow:
|
||||||
|
|
||||||
|
1. ``carrier_cache.json`` exists? → load it.
|
||||||
|
2. Otherwise, copy ``carrier_seed.json`` → ``carrier_cache.json``,
|
||||||
|
then load it. (This happens once, ever, per install.)
|
||||||
|
3. Background: GDELT fetch runs. Any carrier mentioned in fresh news
|
||||||
|
gets its entry replaced with the news-derived position.
|
||||||
|
``position_source_at`` is set to the news article timestamp.
|
||||||
|
|
||||||
|
Freshness is a *labelling* decision, not an eviction decision:
|
||||||
|
|
||||||
|
- ``position_source_at`` within the configurable freshness window
|
||||||
|
(default 14 days) → ``position_confidence = "recent"``.
|
||||||
|
- Older than that → ``position_confidence = "stale"``.
|
||||||
|
- Bootstrapped from the seed file (never updated) → ``"seed"``.
|
||||||
|
- No cache entry at all (e.g. a carrier added to the registry after
|
||||||
|
first install) → carrier renders at its homeport with
|
||||||
|
``"homeport_default"``.
|
||||||
|
|
||||||
|
Carriers are never hidden, never teleported, never disappeared. The
|
||||||
|
position the user sees is always the last position the system actually
|
||||||
|
observed, with an honest "as-of" timestamp the UI can render however
|
||||||
|
it likes. A year from now, the runtime cache reflects whatever this
|
||||||
|
install has observed via GDELT — not the seed snapshot.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import re
|
import os
|
||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
import logging
|
import logging
|
||||||
import threading
|
import threading
|
||||||
import random
|
import random
|
||||||
from datetime import datetime, timezone
|
import shutil
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Dict, List, Optional
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
from services.network_utils import fetch_with_curl
|
from services.network_utils import fetch_with_curl
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# -----------------------------------------------------------------
|
# -----------------------------------------------------------------
|
||||||
# Carrier registry: hull number → metadata + fallback position
|
# Carrier registry: hull number → identity only.
|
||||||
|
#
|
||||||
|
# Issue #244 (tg12): the previous registry carried hard-coded
|
||||||
|
# ``fallback_lat``/``fallback_lng`` that were dated editorial
|
||||||
|
# snapshots from a 2026-03-09 article. Those fields are DELETED. The
|
||||||
|
# registry is now identity + homeport only; positions are sourced
|
||||||
|
# exclusively from carrier_cache.json (and via that, from the
|
||||||
|
# bootstrap seed or live OSINT).
|
||||||
# -----------------------------------------------------------------
|
# -----------------------------------------------------------------
|
||||||
CARRIER_REGISTRY: Dict[str, dict] = {
|
CARRIER_REGISTRY: Dict[str, dict] = {
|
||||||
# Fallback positions sourced from USNI News Fleet & Marine Tracker (Mar 9, 2026)
|
|
||||||
# https://news.usni.org/2026/03/09/usni-news-fleet-and-marine-tracker-march-9-2026
|
|
||||||
# --- Bremerton, WA (Naval Base Kitsap) ---
|
# --- Bremerton, WA (Naval Base Kitsap) ---
|
||||||
# Distinct pier positions along Sinclair Inlet so carriers don't stack
|
|
||||||
"CVN-68": {
|
"CVN-68": {
|
||||||
"name": "USS Nimitz (CVN-68)",
|
"name": "USS Nimitz (CVN-68)",
|
||||||
"wiki": "https://en.wikipedia.org/wiki/USS_Nimitz",
|
"wiki": "https://en.wikipedia.org/wiki/USS_Nimitz",
|
||||||
"homeport": "Bremerton, WA",
|
"homeport": "Bremerton, WA",
|
||||||
"homeport_lat": 47.5535,
|
"homeport_lat": 47.5535,
|
||||||
"homeport_lng": -122.6400,
|
"homeport_lng": -122.6400,
|
||||||
"fallback_lat": 47.5535,
|
|
||||||
"fallback_lng": -122.6400,
|
|
||||||
"fallback_heading": 90,
|
|
||||||
"fallback_desc": "Bremerton, WA (Maintenance)",
|
|
||||||
},
|
},
|
||||||
"CVN-76": {
|
"CVN-76": {
|
||||||
"name": "USS Ronald Reagan (CVN-76)",
|
"name": "USS Ronald Reagan (CVN-76)",
|
||||||
@@ -48,23 +92,14 @@ CARRIER_REGISTRY: Dict[str, dict] = {
|
|||||||
"homeport": "Bremerton, WA",
|
"homeport": "Bremerton, WA",
|
||||||
"homeport_lat": 47.5580,
|
"homeport_lat": 47.5580,
|
||||||
"homeport_lng": -122.6360,
|
"homeport_lng": -122.6360,
|
||||||
"fallback_lat": 47.5580,
|
|
||||||
"fallback_lng": -122.6360,
|
|
||||||
"fallback_heading": 90,
|
|
||||||
"fallback_desc": "Bremerton, WA (Decommissioning)",
|
|
||||||
},
|
},
|
||||||
# --- Norfolk, VA (Naval Station Norfolk) ---
|
# --- Norfolk, VA (Naval Station Norfolk) ---
|
||||||
# Piers run N-S along Willoughby Bay; each carrier gets a distinct berth
|
|
||||||
"CVN-69": {
|
"CVN-69": {
|
||||||
"name": "USS Dwight D. Eisenhower (CVN-69)",
|
"name": "USS Dwight D. Eisenhower (CVN-69)",
|
||||||
"wiki": "https://en.wikipedia.org/wiki/USS_Dwight_D._Eisenhower",
|
"wiki": "https://en.wikipedia.org/wiki/USS_Dwight_D._Eisenhower",
|
||||||
"homeport": "Norfolk, VA",
|
"homeport": "Norfolk, VA",
|
||||||
"homeport_lat": 36.9465,
|
"homeport_lat": 36.9465,
|
||||||
"homeport_lng": -76.3265,
|
"homeport_lng": -76.3265,
|
||||||
"fallback_lat": 36.9465,
|
|
||||||
"fallback_lng": -76.3265,
|
|
||||||
"fallback_heading": 0,
|
|
||||||
"fallback_desc": "Norfolk, VA (Post-deployment maintenance)",
|
|
||||||
},
|
},
|
||||||
"CVN-78": {
|
"CVN-78": {
|
||||||
"name": "USS Gerald R. Ford (CVN-78)",
|
"name": "USS Gerald R. Ford (CVN-78)",
|
||||||
@@ -72,10 +107,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
|
|||||||
"homeport": "Norfolk, VA",
|
"homeport": "Norfolk, VA",
|
||||||
"homeport_lat": 36.9505,
|
"homeport_lat": 36.9505,
|
||||||
"homeport_lng": -76.3250,
|
"homeport_lng": -76.3250,
|
||||||
"fallback_lat": 18.0,
|
|
||||||
"fallback_lng": 39.5,
|
|
||||||
"fallback_heading": 0,
|
|
||||||
"fallback_desc": "Red Sea — Operation Epic Fury (USNI Mar 9)",
|
|
||||||
},
|
},
|
||||||
"CVN-74": {
|
"CVN-74": {
|
||||||
"name": "USS John C. Stennis (CVN-74)",
|
"name": "USS John C. Stennis (CVN-74)",
|
||||||
@@ -83,10 +114,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
|
|||||||
"homeport": "Norfolk, VA",
|
"homeport": "Norfolk, VA",
|
||||||
"homeport_lat": 36.9540,
|
"homeport_lat": 36.9540,
|
||||||
"homeport_lng": -76.3235,
|
"homeport_lng": -76.3235,
|
||||||
"fallback_lat": 36.98,
|
|
||||||
"fallback_lng": -76.43,
|
|
||||||
"fallback_heading": 0,
|
|
||||||
"fallback_desc": "Newport News, VA (RCOH refueling overhaul)",
|
|
||||||
},
|
},
|
||||||
"CVN-75": {
|
"CVN-75": {
|
||||||
"name": "USS Harry S. Truman (CVN-75)",
|
"name": "USS Harry S. Truman (CVN-75)",
|
||||||
@@ -94,10 +121,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
|
|||||||
"homeport": "Norfolk, VA",
|
"homeport": "Norfolk, VA",
|
||||||
"homeport_lat": 36.9580,
|
"homeport_lat": 36.9580,
|
||||||
"homeport_lng": -76.3220,
|
"homeport_lng": -76.3220,
|
||||||
"fallback_lat": 36.0,
|
|
||||||
"fallback_lng": 15.0,
|
|
||||||
"fallback_heading": 0,
|
|
||||||
"fallback_desc": "Mediterranean Sea deployment (USNI Mar 9)",
|
|
||||||
},
|
},
|
||||||
"CVN-77": {
|
"CVN-77": {
|
||||||
"name": "USS George H.W. Bush (CVN-77)",
|
"name": "USS George H.W. Bush (CVN-77)",
|
||||||
@@ -105,23 +128,14 @@ CARRIER_REGISTRY: Dict[str, dict] = {
|
|||||||
"homeport": "Norfolk, VA",
|
"homeport": "Norfolk, VA",
|
||||||
"homeport_lat": 36.9620,
|
"homeport_lat": 36.9620,
|
||||||
"homeport_lng": -76.3210,
|
"homeport_lng": -76.3210,
|
||||||
"fallback_lat": 36.5,
|
|
||||||
"fallback_lng": -74.0,
|
|
||||||
"fallback_heading": 0,
|
|
||||||
"fallback_desc": "Atlantic — Pre-deployment workups (USNI Mar 9)",
|
|
||||||
},
|
},
|
||||||
# --- San Diego, CA (Naval Base San Diego) ---
|
# --- San Diego, CA (Naval Base San Diego) ---
|
||||||
# Carrier piers along the east shore of San Diego Bay, spread N-S
|
|
||||||
"CVN-70": {
|
"CVN-70": {
|
||||||
"name": "USS Carl Vinson (CVN-70)",
|
"name": "USS Carl Vinson (CVN-70)",
|
||||||
"wiki": "https://en.wikipedia.org/wiki/USS_Carl_Vinson",
|
"wiki": "https://en.wikipedia.org/wiki/USS_Carl_Vinson",
|
||||||
"homeport": "San Diego, CA",
|
"homeport": "San Diego, CA",
|
||||||
"homeport_lat": 32.6840,
|
"homeport_lat": 32.6840,
|
||||||
"homeport_lng": -117.1290,
|
"homeport_lng": -117.1290,
|
||||||
"fallback_lat": 32.6840,
|
|
||||||
"fallback_lng": -117.1290,
|
|
||||||
"fallback_heading": 180,
|
|
||||||
"fallback_desc": "San Diego, CA (Homeport)",
|
|
||||||
},
|
},
|
||||||
"CVN-71": {
|
"CVN-71": {
|
||||||
"name": "USS Theodore Roosevelt (CVN-71)",
|
"name": "USS Theodore Roosevelt (CVN-71)",
|
||||||
@@ -129,10 +143,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
|
|||||||
"homeport": "San Diego, CA",
|
"homeport": "San Diego, CA",
|
||||||
"homeport_lat": 32.6885,
|
"homeport_lat": 32.6885,
|
||||||
"homeport_lng": -117.1280,
|
"homeport_lng": -117.1280,
|
||||||
"fallback_lat": 32.6885,
|
|
||||||
"fallback_lng": -117.1280,
|
|
||||||
"fallback_heading": 180,
|
|
||||||
"fallback_desc": "San Diego, CA (Maintenance)",
|
|
||||||
},
|
},
|
||||||
"CVN-72": {
|
"CVN-72": {
|
||||||
"name": "USS Abraham Lincoln (CVN-72)",
|
"name": "USS Abraham Lincoln (CVN-72)",
|
||||||
@@ -140,10 +150,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
|
|||||||
"homeport": "San Diego, CA",
|
"homeport": "San Diego, CA",
|
||||||
"homeport_lat": 32.6925,
|
"homeport_lat": 32.6925,
|
||||||
"homeport_lng": -117.1275,
|
"homeport_lng": -117.1275,
|
||||||
"fallback_lat": 20.0,
|
|
||||||
"fallback_lng": 64.0,
|
|
||||||
"fallback_heading": 0,
|
|
||||||
"fallback_desc": "Arabian Sea — Operation Epic Fury (USNI Mar 9)",
|
|
||||||
},
|
},
|
||||||
# --- Yokosuka, Japan (CFAY) ---
|
# --- Yokosuka, Japan (CFAY) ---
|
||||||
"CVN-73": {
|
"CVN-73": {
|
||||||
@@ -152,16 +158,18 @@ CARRIER_REGISTRY: Dict[str, dict] = {
|
|||||||
"homeport": "Yokosuka, Japan",
|
"homeport": "Yokosuka, Japan",
|
||||||
"homeport_lat": 35.2830,
|
"homeport_lat": 35.2830,
|
||||||
"homeport_lng": 139.6700,
|
"homeport_lng": 139.6700,
|
||||||
"fallback_lat": 35.2830,
|
|
||||||
"fallback_lng": 139.6700,
|
|
||||||
"fallback_heading": 180,
|
|
||||||
"fallback_desc": "Yokosuka, Japan (Forward deployed)",
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
# -----------------------------------------------------------------
|
# -----------------------------------------------------------------
|
||||||
# Region → approximate center coordinates
|
# Region → approximate center coordinates.
|
||||||
# Used to map textual geographic descriptions to lat/lng
|
#
|
||||||
|
# Issue #245 (tg12): converting a region name straight into precise
|
||||||
|
# map coordinates is false precision. We still use this table to
|
||||||
|
# infer a coarse position from a headline mention, but the resulting
|
||||||
|
# carrier object is now stamped ``position_confidence = "approximate"``
|
||||||
|
# so the UI can render an uncertainty radius / dimmed icon. The
|
||||||
|
# centroid is a best-effort midpoint of the named body of water.
|
||||||
# -----------------------------------------------------------------
|
# -----------------------------------------------------------------
|
||||||
REGION_COORDS: Dict[str, tuple] = {
|
REGION_COORDS: Dict[str, tuple] = {
|
||||||
# Oceans & Seas
|
# Oceans & Seas
|
||||||
@@ -220,9 +228,39 @@ REGION_COORDS: Dict[str, tuple] = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# -----------------------------------------------------------------
|
# -----------------------------------------------------------------
|
||||||
# Cache file for persisting positions between restarts
|
# Files
|
||||||
# -----------------------------------------------------------------
|
# -----------------------------------------------------------------
|
||||||
CACHE_FILE = Path(__file__).parent.parent / "carrier_cache.json"
|
#
|
||||||
|
# The seed lives in the read-only image data dir (it ships with each
|
||||||
|
# release). The cache lives in the same data dir but is written at
|
||||||
|
# runtime; under Docker compose this dir is volume-mounted so the
|
||||||
|
# cache persists across container restarts, which is the whole point
|
||||||
|
# of the seed-then-observe model — the user's runtime observations
|
||||||
|
# survive image upgrades.
|
||||||
|
SEED_FILE = Path(__file__).parent.parent / "data" / "carrier_seed.json"
|
||||||
|
CACHE_FILE = Path(__file__).parent.parent / "data" / "carrier_cache.json"
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------
|
||||||
|
# Freshness window for position_confidence labeling. Issue #246 (tg12):
|
||||||
|
# previously persisted cache entries had no freshness signal at all.
|
||||||
|
# After this change, the position itself is preserved (we never lose
|
||||||
|
# what was last observed) but the confidence label flips from
|
||||||
|
# "recent" to "stale" once the underlying source is older than this
|
||||||
|
# window. Operator-overridable via env var.
|
||||||
|
# -----------------------------------------------------------------
|
||||||
|
_DEFAULT_FRESHNESS_WINDOW_DAYS = 14
|
||||||
|
|
||||||
|
|
||||||
|
def _freshness_window_days() -> int:
|
||||||
|
raw = str(os.environ.get("SHADOWBROKER_CARRIER_FRESHNESS_DAYS", "") or "").strip()
|
||||||
|
if not raw:
|
||||||
|
return _DEFAULT_FRESHNESS_WINDOW_DAYS
|
||||||
|
try:
|
||||||
|
n = int(raw)
|
||||||
|
return n if n > 0 else _DEFAULT_FRESHNESS_WINDOW_DAYS
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return _DEFAULT_FRESHNESS_WINDOW_DAYS
|
||||||
|
|
||||||
|
|
||||||
_carrier_positions: Dict[str, dict] = {}
|
_carrier_positions: Dict[str, dict] = {}
|
||||||
_positions_lock = threading.Lock()
|
_positions_lock = threading.Lock()
|
||||||
@@ -234,25 +272,159 @@ _GDELT_REQUEST_DELAY_SECONDS = 1.25
|
|||||||
_GDELT_REQUEST_JITTER_SECONDS = 0.35
|
_GDELT_REQUEST_JITTER_SECONDS = 0.35
|
||||||
|
|
||||||
|
|
||||||
|
def _now_iso() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_iso(ts: str) -> Optional[datetime]:
|
||||||
|
if not ts:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
# Python's fromisoformat accepts +00:00 but not 'Z' until 3.11.
|
||||||
|
normalized = ts.replace("Z", "+00:00")
|
||||||
|
dt = datetime.fromisoformat(normalized)
|
||||||
|
if dt.tzinfo is None:
|
||||||
|
dt = dt.replace(tzinfo=timezone.utc)
|
||||||
|
return dt
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _compute_position_confidence(entry: dict, *, now: Optional[datetime] = None) -> str:
|
||||||
|
"""Return the public confidence label for a carrier cache entry.
|
||||||
|
|
||||||
|
Order of precedence:
|
||||||
|
- explicit "homeport_default" / "seed" labels are preserved.
|
||||||
|
- dated entries (with position_source_at) are "recent" if within
|
||||||
|
the configured freshness window, else "stale".
|
||||||
|
- missing position_source_at falls through to "stale".
|
||||||
|
"""
|
||||||
|
raw_label = str(entry.get("position_confidence", "") or "").strip()
|
||||||
|
# Explicit "kind of provenance" labels are preserved as-is. They
|
||||||
|
# describe HOW we got the position, not WHEN — a fresh headline-to-
|
||||||
|
# centroid match (#245) is still imprecise no matter how recently
|
||||||
|
# it was observed, and the seed (#244) is always the seed.
|
||||||
|
if raw_label in {"seed", "homeport_default", "approximate"}:
|
||||||
|
# Approximate entries can still age into "stale_approximate" if
|
||||||
|
# they fall out of the freshness window — that distinction lets
|
||||||
|
# the UI render a different badge for old-and-imprecise vs
|
||||||
|
# recent-and-imprecise. seed/homeport_default never age (they
|
||||||
|
# were never timestamped against real observations).
|
||||||
|
if raw_label == "approximate":
|
||||||
|
source_at = _parse_iso(str(entry.get("position_source_at", "") or ""))
|
||||||
|
if source_at is not None:
|
||||||
|
reference = now or datetime.now(timezone.utc)
|
||||||
|
if reference - source_at > timedelta(days=_freshness_window_days()):
|
||||||
|
return "stale_approximate"
|
||||||
|
return raw_label
|
||||||
|
|
||||||
|
source_at = _parse_iso(str(entry.get("position_source_at", "") or ""))
|
||||||
|
if not source_at:
|
||||||
|
return "stale"
|
||||||
|
|
||||||
|
reference = now or datetime.now(timezone.utc)
|
||||||
|
window = timedelta(days=_freshness_window_days())
|
||||||
|
if reference - source_at <= window:
|
||||||
|
return "recent"
|
||||||
|
return "stale"
|
||||||
|
|
||||||
|
|
||||||
|
def _load_seed() -> Dict[str, dict]:
|
||||||
|
"""Load the read-only seed file shipped with the image.
|
||||||
|
|
||||||
|
Returns a hull→entry dict (no _meta wrapper). Missing or malformed
|
||||||
|
seed files yield an empty dict — the caller falls back to homeport
|
||||||
|
defaults.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
if not SEED_FILE.exists():
|
||||||
|
logger.info("Carrier seed file not present at %s; first-run will fall back to homeport defaults", SEED_FILE)
|
||||||
|
return {}
|
||||||
|
raw = json.loads(SEED_FILE.read_text(encoding="utf-8"))
|
||||||
|
carriers = raw.get("carriers", {}) if isinstance(raw, dict) else {}
|
||||||
|
if not isinstance(carriers, dict):
|
||||||
|
return {}
|
||||||
|
logger.info("Carrier seed loaded: %d entries from %s", len(carriers), SEED_FILE)
|
||||||
|
return carriers
|
||||||
|
except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
|
||||||
|
logger.warning("Failed to load carrier seed file %s: %s", SEED_FILE, e)
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
def _load_cache() -> Dict[str, dict]:
|
def _load_cache() -> Dict[str, dict]:
|
||||||
"""Load cached carrier positions from disk."""
|
"""Load the mutable cache (last-known positions persisted between restarts)."""
|
||||||
try:
|
try:
|
||||||
if CACHE_FILE.exists():
|
if CACHE_FILE.exists():
|
||||||
data = json.loads(CACHE_FILE.read_text())
|
data = json.loads(CACHE_FILE.read_text(encoding="utf-8"))
|
||||||
logger.info(f"Carrier cache loaded: {len(data)} carriers from {CACHE_FILE}")
|
if isinstance(data, dict):
|
||||||
return data
|
logger.info("Carrier cache loaded: %d carriers from %s", len(data), CACHE_FILE)
|
||||||
|
return data
|
||||||
except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
|
except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
|
||||||
logger.warning(f"Failed to load carrier cache: {e}")
|
logger.warning("Failed to load carrier cache: %s", e)
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
|
||||||
def _save_cache(positions: Dict[str, dict]):
|
def _save_cache(positions: Dict[str, dict]) -> None:
|
||||||
"""Persist carrier positions to disk."""
|
"""Persist the mutable cache. Atomic write (temp + rename) so a crash
|
||||||
|
mid-write can't leave the file truncated."""
|
||||||
try:
|
try:
|
||||||
CACHE_FILE.write_text(json.dumps(positions, indent=2))
|
CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||||
logger.info(f"Carrier cache saved: {len(positions)} carriers")
|
tmp = CACHE_FILE.with_suffix(CACHE_FILE.suffix + ".tmp")
|
||||||
|
tmp.write_text(json.dumps(positions, indent=2), encoding="utf-8")
|
||||||
|
# On Windows os.replace is atomic and overwrites existing files.
|
||||||
|
os.replace(tmp, CACHE_FILE)
|
||||||
|
logger.info("Carrier cache saved: %d carriers", len(positions))
|
||||||
except (IOError, OSError) as e:
|
except (IOError, OSError) as e:
|
||||||
logger.warning(f"Failed to save carrier cache: {e}")
|
logger.warning("Failed to save carrier cache: %s", e)
|
||||||
|
|
||||||
|
|
||||||
|
def _homeport_entry_for(hull: str) -> Optional[dict]:
|
||||||
|
"""Return a homeport-default cache entry for a hull, or None if the
|
||||||
|
hull is not in the registry."""
|
||||||
|
info = CARRIER_REGISTRY.get(hull)
|
||||||
|
if not info:
|
||||||
|
return None
|
||||||
|
return {
|
||||||
|
"lat": info["homeport_lat"],
|
||||||
|
"lng": info["homeport_lng"],
|
||||||
|
"heading": 0,
|
||||||
|
"desc": f"{info['homeport']} (no observations yet)",
|
||||||
|
"source": f"Homeport default ({info['homeport']})",
|
||||||
|
"source_url": info.get("wiki", ""),
|
||||||
|
"position_source_at": _now_iso(),
|
||||||
|
"position_confidence": "homeport_default",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _bootstrap_cache_if_missing() -> Dict[str, dict]:
|
||||||
|
"""One-shot: if no cache exists, materialize one from the seed file.
|
||||||
|
|
||||||
|
Returns the cache contents (hull→entry). On first-ever startup,
|
||||||
|
this writes ``carrier_cache.json`` so subsequent restarts skip the
|
||||||
|
seed entirely. Operator-deleted caches re-bootstrap the same way —
|
||||||
|
operators can use that to "reset" carrier positions, but it's an
|
||||||
|
explicit operator action.
|
||||||
|
"""
|
||||||
|
if CACHE_FILE.exists():
|
||||||
|
return _load_cache()
|
||||||
|
|
||||||
|
seed = _load_seed()
|
||||||
|
if not seed:
|
||||||
|
# No seed file either. Build a homeport-default cache so the
|
||||||
|
# first save_cache call still produces something honest.
|
||||||
|
homeports: Dict[str, dict] = {}
|
||||||
|
for hull in CARRIER_REGISTRY:
|
||||||
|
entry = _homeport_entry_for(hull)
|
||||||
|
if entry is not None:
|
||||||
|
homeports[hull] = entry
|
||||||
|
if homeports:
|
||||||
|
_save_cache(homeports)
|
||||||
|
return homeports
|
||||||
|
|
||||||
|
# Persist the seed as the first cache so subsequent runs skip this branch.
|
||||||
|
_save_cache(seed)
|
||||||
|
logger.info("Carrier cache bootstrapped from seed (first-ever startup)")
|
||||||
|
return dict(seed)
|
||||||
|
|
||||||
|
|
||||||
def _match_region(text: str) -> Optional[tuple]:
|
def _match_region(text: str) -> Optional[tuple]:
|
||||||
@@ -270,10 +442,8 @@ def _match_carrier(text: str) -> Optional[str]:
|
|||||||
for hull, info in CARRIER_REGISTRY.items():
|
for hull, info in CARRIER_REGISTRY.items():
|
||||||
hull_check = hull.lower().replace("-", "")
|
hull_check = hull.lower().replace("-", "")
|
||||||
name_parts = info["name"].lower()
|
name_parts = info["name"].lower()
|
||||||
# Match hull number (e.g., "CVN-78", "CVN78")
|
|
||||||
if hull.lower() in text_lower or hull_check in text_lower.replace("-", ""):
|
if hull.lower() in text_lower or hull_check in text_lower.replace("-", ""):
|
||||||
return hull
|
return hull
|
||||||
# Match ship name (e.g., "Ford", "Eisenhower", "Vinson")
|
|
||||||
ship_name = name_parts.split("(")[0].strip()
|
ship_name = name_parts.split("(")[0].strip()
|
||||||
last_name = ship_name.split()[-1] if ship_name else ""
|
last_name = ship_name.split()[-1] if ship_name else ""
|
||||||
if last_name and len(last_name) > 3 and last_name in text_lower:
|
if last_name and len(last_name) > 3 and last_name in text_lower:
|
||||||
@@ -323,8 +493,9 @@ def _fetch_gdelt_carrier_news() -> List[dict]:
|
|||||||
articles = data.get("articles", [])
|
articles = data.get("articles", [])
|
||||||
for art in articles:
|
for art in articles:
|
||||||
title = art.get("title", "")
|
title = art.get("title", "")
|
||||||
url = art.get("url", "")
|
article_url = art.get("url", "")
|
||||||
results.append({"title": title, "url": url})
|
article_at = art.get("seendate") or art.get("date") or ""
|
||||||
|
results.append({"title": title, "url": article_url, "seendate": article_at})
|
||||||
except (ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
|
except (ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
|
||||||
logger.debug(f"GDELT search failed for '{term}': {e}")
|
logger.debug(f"GDELT search failed for '{term}': {e}")
|
||||||
continue
|
continue
|
||||||
@@ -340,108 +511,175 @@ def _fetch_gdelt_carrier_news() -> List[dict]:
|
|||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def _gdelt_seendate_to_iso(seendate: str) -> Optional[str]:
|
||||||
|
"""GDELT returns YYYYMMDDhhmmss (UTC). Convert to ISO8601 for
|
||||||
|
position_source_at. Returns None if the input is unparseable."""
|
||||||
|
raw = (seendate or "").strip()
|
||||||
|
if len(raw) < 8 or not raw.isdigit():
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
dt = datetime.strptime(raw[:14] if len(raw) >= 14 else raw[:8] + "000000", "%Y%m%d%H%M%S")
|
||||||
|
return dt.replace(tzinfo=timezone.utc).isoformat()
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _parse_carrier_positions_from_news(articles: List[dict]) -> Dict[str, dict]:
|
def _parse_carrier_positions_from_news(articles: List[dict]) -> Dict[str, dict]:
|
||||||
"""Parse carrier positions from news article titles and descriptions."""
|
"""Parse carrier positions from news article titles.
|
||||||
|
|
||||||
|
Issue #245 (tg12): the position is a region centroid, which is
|
||||||
|
coarse — we now stamp ``position_confidence = "approximate"`` so
|
||||||
|
the UI can render that uncertainty. Issue #244: the
|
||||||
|
``position_source_at`` field is the news article's actual seen
|
||||||
|
date, NOT now(), so the freshness check correctly flips entries
|
||||||
|
to "stale" once they age past the configured window.
|
||||||
|
"""
|
||||||
updates: Dict[str, dict] = {}
|
updates: Dict[str, dict] = {}
|
||||||
|
|
||||||
for article in articles:
|
for article in articles:
|
||||||
title = article.get("title", "")
|
title = article.get("title", "")
|
||||||
|
|
||||||
# Try to match a carrier from the title
|
|
||||||
hull = _match_carrier(title)
|
hull = _match_carrier(title)
|
||||||
if not hull:
|
if not hull:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Try to match a region from the title
|
|
||||||
coords = _match_region(title)
|
coords = _match_region(title)
|
||||||
if not coords:
|
if not coords:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Only update if we haven't seen this carrier yet (first match wins — most recent)
|
# First match wins (most recent article, GDELT returns newest first
|
||||||
|
# per term).
|
||||||
if hull not in updates:
|
if hull not in updates:
|
||||||
|
iso_at = _gdelt_seendate_to_iso(str(article.get("seendate", ""))) or _now_iso()
|
||||||
updates[hull] = {
|
updates[hull] = {
|
||||||
"lat": coords[0],
|
"lat": coords[0],
|
||||||
"lng": coords[1],
|
"lng": coords[1],
|
||||||
|
"heading": 0,
|
||||||
"desc": title[:100],
|
"desc": title[:100],
|
||||||
"source": "GDELT News API",
|
"source": "GDELT News API (headline region match — approximate)",
|
||||||
"source_url": article.get("url", "https://api.gdeltproject.org"),
|
"source_url": article.get("url", "https://api.gdeltproject.org"),
|
||||||
"updated": datetime.now(timezone.utc).isoformat(),
|
"position_source_at": iso_at,
|
||||||
|
# Headline-to-centroid match is explicitly approximate.
|
||||||
|
"position_confidence": "approximate",
|
||||||
}
|
}
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Carrier update: {CARRIER_REGISTRY[hull]['name']} → {coords} (from: {title[:80]})"
|
"Carrier update: %s → %s (from: %s)",
|
||||||
|
CARRIER_REGISTRY[hull]["name"],
|
||||||
|
coords,
|
||||||
|
title[:80],
|
||||||
)
|
)
|
||||||
|
|
||||||
return updates
|
return updates
|
||||||
|
|
||||||
|
|
||||||
def _load_carrier_fallbacks() -> Dict[str, dict]:
|
def _enrich_for_rendering(hull: str, entry: dict, *, now: Optional[datetime] = None) -> dict:
|
||||||
"""Build carrier positions from static fallbacks + disk cache (instant, no network)."""
|
"""Add live computed fields (confidence label, last_osint_update)
|
||||||
positions: Dict[str, dict] = {}
|
on top of the persisted cache entry. The persisted entry is left
|
||||||
for hull, info in CARRIER_REGISTRY.items():
|
untouched; this function builds the public-facing object.
|
||||||
positions[hull] = {
|
"""
|
||||||
"name": info["name"],
|
info = CARRIER_REGISTRY.get(hull, {})
|
||||||
"lat": info["fallback_lat"],
|
confidence = _compute_position_confidence(entry, now=now)
|
||||||
"lng": info["fallback_lng"],
|
return {
|
||||||
"heading": info["fallback_heading"],
|
"name": entry.get("name", info.get("name", hull)),
|
||||||
"desc": info["fallback_desc"],
|
"lat": entry["lat"],
|
||||||
"wiki": info["wiki"],
|
"lng": entry["lng"],
|
||||||
"source": "USNI News Fleet & Marine Tracker",
|
"heading": entry.get("heading", 0),
|
||||||
"source_url": "https://news.usni.org/category/fleet-tracker",
|
"desc": entry.get("desc", ""),
|
||||||
"updated": datetime.now(timezone.utc).isoformat(),
|
"wiki": entry.get("wiki", info.get("wiki", "")),
|
||||||
}
|
"source": entry.get("source", "OSINT estimated position"),
|
||||||
|
"source_url": entry.get("source_url", ""),
|
||||||
# Overlay cached positions from previous runs (may have GDELT data)
|
"position_source_at": entry.get("position_source_at", ""),
|
||||||
cached = _load_cache()
|
"position_confidence": confidence,
|
||||||
for hull, cached_pos in cached.items():
|
# Existing field preserved for backward compatibility with the
|
||||||
if hull in positions:
|
# current frontend ShipPopup; now reflects the SOURCE's observed
|
||||||
if cached_pos.get("source", "").startswith("GDELT") or cached_pos.get(
|
# time (not now()), so "last reported X days ago" is honest.
|
||||||
"source", ""
|
"last_osint_update": entry.get("position_source_at", ""),
|
||||||
).startswith("News"):
|
# Convenience boolean for the UI: true when the position is
|
||||||
positions[hull].update(
|
# NOT live OSINT (used to render dimmed icons / badges).
|
||||||
{
|
"is_fallback": confidence in {"seed", "stale", "stale_approximate", "homeport_default"},
|
||||||
"lat": cached_pos["lat"],
|
}
|
||||||
"lng": cached_pos["lng"],
|
|
||||||
"desc": cached_pos.get("desc", positions[hull]["desc"]),
|
|
||||||
"source": cached_pos.get("source", "Cached OSINT"),
|
|
||||||
"updated": cached_pos.get("updated", ""),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
return positions
|
|
||||||
|
|
||||||
|
|
||||||
def update_carrier_positions():
|
def update_carrier_positions() -> None:
|
||||||
"""Main update function — called on startup and every 12h.
|
"""Refresh carrier positions.
|
||||||
|
|
||||||
Phase 1 (instant): publish fallback + cached positions so the map has carriers immediately.
|
Phase 1 (instant): publish whatever's in carrier_cache.json (or
|
||||||
Phase 2 (slow): query GDELT for fresh OSINT positions and update in-place.
|
bootstrap from seed on first-ever run), so the map has carriers
|
||||||
|
immediately.
|
||||||
|
|
||||||
|
Phase 2 (slow): query GDELT and replace position entries for any
|
||||||
|
carrier mentioned in fresh news. Persist back to cache.
|
||||||
"""
|
"""
|
||||||
global _last_update
|
global _last_update
|
||||||
|
|
||||||
# --- Phase 1: instant fallback + cache ---
|
# --- Phase 1: instant cache (bootstrap from seed on first-ever run) ---
|
||||||
positions = _load_carrier_fallbacks()
|
positions = _bootstrap_cache_if_missing()
|
||||||
|
|
||||||
|
# Ensure every registered hull has SOMETHING in the cache. A hull
|
||||||
|
# the seed didn't cover (e.g. added after install) renders at its
|
||||||
|
# homeport with "homeport_default" confidence.
|
||||||
|
for hull in CARRIER_REGISTRY:
|
||||||
|
if hull not in positions:
|
||||||
|
entry = _homeport_entry_for(hull)
|
||||||
|
if entry is not None:
|
||||||
|
positions[hull] = entry
|
||||||
|
|
||||||
with _positions_lock:
|
with _positions_lock:
|
||||||
# Only overwrite if positions are currently empty (first startup).
|
|
||||||
# If we already have data from a previous cycle, keep it while GDELT runs.
|
|
||||||
if not _carrier_positions:
|
if not _carrier_positions:
|
||||||
_carrier_positions.update(positions)
|
_carrier_positions.update(positions)
|
||||||
_last_update = datetime.now(timezone.utc)
|
_last_update = datetime.now(timezone.utc)
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Carrier tracker: {len(positions)} carriers loaded from fallback/cache (GDELT enrichment starting...)"
|
"Carrier tracker: %d carriers loaded from cache (USNI + GDELT enrichment starting...)",
|
||||||
|
len(positions),
|
||||||
)
|
)
|
||||||
|
|
||||||
# --- Phase 2: slow GDELT enrichment ---
|
# --- Phase 2: USNI Fleet & Marine Tracker (PRIMARY source) ---
|
||||||
|
#
|
||||||
|
# USNI publishes a weekly editorial tracker with each carrier's
|
||||||
|
# actual operating area, parsed from explicit prose like
|
||||||
|
# "The Gerald R. Ford Carrier Strike Group is operating in the Red Sea"
|
||||||
|
# These positions are tagged ``position_confidence: "recent"`` because
|
||||||
|
# they reflect actual reporting, not headline-keyword centroids.
|
||||||
|
# USNI updates are preferred over GDELT — they're authoritative on
|
||||||
|
# US Navy positions where GDELT is just article-title text mining.
|
||||||
|
try:
|
||||||
|
from services.fetchers.usni_fleet_tracker import (
|
||||||
|
fetch_latest_fleet_tracker_positions,
|
||||||
|
)
|
||||||
|
usni_positions = fetch_latest_fleet_tracker_positions()
|
||||||
|
for hull, pos in usni_positions.items():
|
||||||
|
positions[hull] = pos
|
||||||
|
logger.info(
|
||||||
|
"Carrier USNI update: %s → %s",
|
||||||
|
CARRIER_REGISTRY[hull]["name"],
|
||||||
|
pos.get("desc", ""),
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("USNI fleet-tracker fetch failed: %s", e)
|
||||||
|
|
||||||
|
# --- Phase 3: GDELT enrichment (SECONDARY — fills gaps) ---
|
||||||
|
#
|
||||||
|
# Used only to backfill carriers USNI didn't mention this week. The
|
||||||
|
# position is stamped ``approximate`` so the UI knows it's a
|
||||||
|
# headline-centroid match (Issue #245).
|
||||||
try:
|
try:
|
||||||
articles = _fetch_gdelt_carrier_news()
|
articles = _fetch_gdelt_carrier_news()
|
||||||
news_positions = _parse_carrier_positions_from_news(articles)
|
news_positions = _parse_carrier_positions_from_news(articles)
|
||||||
for hull, pos in news_positions.items():
|
for hull, pos in news_positions.items():
|
||||||
if hull in positions:
|
# Only overwrite if the existing entry is NOT a recent USNI
|
||||||
positions[hull].update(pos)
|
# observation. A "recent" USNI position is higher-confidence
|
||||||
logger.info(f"Carrier OSINT: updated {CARRIER_REGISTRY[hull]['name']} from news")
|
# than a GDELT headline-centroid match — don't let GDELT
|
||||||
|
# demote a real position to an approximate one.
|
||||||
|
existing = positions.get(hull, {})
|
||||||
|
existing_conf = _compute_position_confidence(existing)
|
||||||
|
if existing_conf == "recent":
|
||||||
|
continue
|
||||||
|
positions[hull] = pos
|
||||||
|
logger.info(
|
||||||
|
"Carrier OSINT: updated %s from GDELT news",
|
||||||
|
CARRIER_REGISTRY[hull]["name"],
|
||||||
|
)
|
||||||
except (ValueError, KeyError, json.JSONDecodeError, OSError) as e:
|
except (ValueError, KeyError, json.JSONDecodeError, OSError) as e:
|
||||||
logger.warning(f"GDELT carrier fetch failed: {e}")
|
logger.warning("GDELT carrier fetch failed: %s", e)
|
||||||
|
|
||||||
# Save and update the global state with enriched positions
|
|
||||||
with _positions_lock:
|
with _positions_lock:
|
||||||
_carrier_positions.clear()
|
_carrier_positions.clear()
|
||||||
_carrier_positions.update(positions)
|
_carrier_positions.update(positions)
|
||||||
@@ -449,21 +687,15 @@ def update_carrier_positions():
|
|||||||
|
|
||||||
_save_cache(positions)
|
_save_cache(positions)
|
||||||
|
|
||||||
sources = {}
|
confidences: Dict[str, int] = {}
|
||||||
for p in positions.values():
|
for entry in positions.values():
|
||||||
src = p.get("source", "unknown")
|
label = _compute_position_confidence(entry)
|
||||||
sources[src] = sources.get(src, 0) + 1
|
confidences[label] = confidences.get(label, 0) + 1
|
||||||
logger.info(f"Carrier tracker: {len(positions)} carriers updated. Sources: {sources}")
|
logger.info("Carrier tracker: %d carriers updated. Confidence: %s", len(positions), confidences)
|
||||||
|
|
||||||
|
|
||||||
def _deconflict_positions(result: List[dict]) -> List[dict]:
|
def _deconflict_positions(result: List[dict]) -> List[dict]:
|
||||||
"""Offset carriers that share identical coordinates so they don't stack.
|
"""Offset carriers that share identical coordinates so they don't stack."""
|
||||||
|
|
||||||
At port: offset along the pier axis (~500m / 0.004° apart).
|
|
||||||
At sea: offset perpendicular to each other (~0.08° / ~9km apart)
|
|
||||||
so they're visibly separate but clearly operating together.
|
|
||||||
"""
|
|
||||||
# Group by rounded lat/lng (within ~0.01° ≈ 1km = same spot)
|
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
|
||||||
groups: dict[str, list[int]] = defaultdict(list)
|
groups: dict[str, list[int]] = defaultdict(list)
|
||||||
@@ -475,7 +707,6 @@ def _deconflict_positions(result: List[dict]) -> List[dict]:
|
|||||||
if len(indices) < 2:
|
if len(indices) < 2:
|
||||||
continue
|
continue
|
||||||
n = len(indices)
|
n = len(indices)
|
||||||
# Determine if this is a port (near a homeport) or at sea
|
|
||||||
sample = result[indices[0]]
|
sample = result[indices[0]]
|
||||||
at_port = any(
|
at_port = any(
|
||||||
abs(sample["lat"] - info.get("homeport_lat", 0)) < 0.05
|
abs(sample["lat"] - info.get("homeport_lat", 0)) < 0.05
|
||||||
@@ -484,7 +715,6 @@ def _deconflict_positions(result: List[dict]) -> List[dict]:
|
|||||||
)
|
)
|
||||||
|
|
||||||
if at_port:
|
if at_port:
|
||||||
# Use each carrier's distinct homeport pier coordinates
|
|
||||||
for idx in indices:
|
for idx in indices:
|
||||||
carrier = result[idx]
|
carrier = result[idx]
|
||||||
hull = None
|
hull = None
|
||||||
@@ -497,8 +727,7 @@ def _deconflict_positions(result: List[dict]) -> List[dict]:
|
|||||||
carrier["lat"] = info["homeport_lat"]
|
carrier["lat"] = info["homeport_lat"]
|
||||||
carrier["lng"] = info["homeport_lng"]
|
carrier["lng"] = info["homeport_lng"]
|
||||||
else:
|
else:
|
||||||
# At sea: spread in a line perpendicular to travel (~0.08° apart)
|
spacing = 0.08
|
||||||
spacing = 0.08 # ~9km — close enough to see they're together
|
|
||||||
start_offset = -(n - 1) * spacing / 2
|
start_offset = -(n - 1) * spacing / 2
|
||||||
for j, idx in enumerate(indices):
|
for j, idx in enumerate(indices):
|
||||||
result[idx]["lng"] += start_offset + j * spacing
|
result[idx]["lng"] += start_offset + j * spacing
|
||||||
@@ -507,36 +736,44 @@ def _deconflict_positions(result: List[dict]) -> List[dict]:
|
|||||||
|
|
||||||
|
|
||||||
def get_carrier_positions() -> List[dict]:
|
def get_carrier_positions() -> List[dict]:
|
||||||
"""Return current carrier positions for the data pipeline."""
|
"""Return current carrier positions for the data pipeline.
|
||||||
|
|
||||||
|
Each entry has the full provenance + freshness fields; the UI can
|
||||||
|
decide how to render them. Carriers are never hidden — only
|
||||||
|
labeled.
|
||||||
|
"""
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
with _positions_lock:
|
with _positions_lock:
|
||||||
result = []
|
result: List[dict] = []
|
||||||
for hull, pos in _carrier_positions.items():
|
for hull, entry in _carrier_positions.items():
|
||||||
info = CARRIER_REGISTRY.get(hull, {})
|
enriched = _enrich_for_rendering(hull, entry, now=now)
|
||||||
result.append(
|
result.append(
|
||||||
{
|
{
|
||||||
"name": pos.get("name", info.get("name", hull)),
|
"name": enriched["name"],
|
||||||
"type": "carrier",
|
"type": "carrier",
|
||||||
"lat": pos["lat"],
|
"lat": enriched["lat"],
|
||||||
"lng": pos["lng"],
|
"lng": enriched["lng"],
|
||||||
"heading": None, # Heading unknown for carriers — OSINT cannot determine true heading
|
"heading": None, # OSINT cannot determine true heading.
|
||||||
"sog": 0,
|
"sog": 0,
|
||||||
"cog": 0,
|
"cog": 0,
|
||||||
"country": "United States",
|
"country": "United States",
|
||||||
"desc": pos.get("desc", ""),
|
"desc": enriched["desc"],
|
||||||
"wiki": pos.get("wiki", info.get("wiki", "")),
|
"wiki": enriched["wiki"],
|
||||||
"estimated": True,
|
"estimated": True,
|
||||||
"source": pos.get("source", "OSINT estimated position"),
|
"source": enriched["source"],
|
||||||
"source_url": pos.get(
|
"source_url": enriched["source_url"],
|
||||||
"source_url", "https://news.usni.org/category/fleet-tracker"
|
"last_osint_update": enriched["last_osint_update"],
|
||||||
),
|
# New fields (additive — existing UI continues to work):
|
||||||
"last_osint_update": pos.get("updated", ""),
|
"position_source_at": enriched["position_source_at"],
|
||||||
|
"position_confidence": enriched["position_confidence"],
|
||||||
|
"is_fallback": enriched["is_fallback"],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
return _deconflict_positions(result)
|
return _deconflict_positions(result)
|
||||||
|
|
||||||
|
|
||||||
# -----------------------------------------------------------------
|
# -----------------------------------------------------------------
|
||||||
# Scheduler: runs at startup, then at 00:00 and 12:00 UTC daily
|
# Scheduler: runs at startup, then at 00:00 and 12:00 UTC daily.
|
||||||
# -----------------------------------------------------------------
|
# -----------------------------------------------------------------
|
||||||
_scheduler_thread: Optional[threading.Thread] = None
|
_scheduler_thread: Optional[threading.Thread] = None
|
||||||
_scheduler_stop = threading.Event()
|
_scheduler_stop = threading.Event()
|
||||||
@@ -544,7 +781,6 @@ _scheduler_stop = threading.Event()
|
|||||||
|
|
||||||
def _scheduler_loop():
|
def _scheduler_loop():
|
||||||
"""Background thread that triggers updates at 00:00 and 12:00 UTC."""
|
"""Background thread that triggers updates at 00:00 and 12:00 UTC."""
|
||||||
# Initial update on startup
|
|
||||||
try:
|
try:
|
||||||
update_carrier_positions()
|
update_carrier_positions()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -552,7 +788,6 @@ def _scheduler_loop():
|
|||||||
|
|
||||||
while not _scheduler_stop.is_set():
|
while not _scheduler_stop.is_set():
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
# Next target: 00:00 or 12:00 UTC, whichever is sooner
|
|
||||||
hour = now.hour
|
hour = now.hour
|
||||||
if hour < 12:
|
if hour < 12:
|
||||||
next_hour = 12
|
next_hour = 12
|
||||||
@@ -561,18 +796,17 @@ def _scheduler_loop():
|
|||||||
|
|
||||||
next_run = now.replace(hour=next_hour % 24, minute=0, second=0, microsecond=0)
|
next_run = now.replace(hour=next_hour % 24, minute=0, second=0, microsecond=0)
|
||||||
if next_hour == 24:
|
if next_hour == 24:
|
||||||
from datetime import timedelta
|
|
||||||
|
|
||||||
next_run = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
|
next_run = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||||
|
|
||||||
wait_seconds = (next_run - now).total_seconds()
|
wait_seconds = (next_run - now).total_seconds()
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Carrier tracker: next update at {next_run.isoformat()} ({wait_seconds/3600:.1f}h)"
|
"Carrier tracker: next update at %s (%.1fh)",
|
||||||
|
next_run.isoformat(),
|
||||||
|
wait_seconds / 3600,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Wait until next scheduled time, or until stop event
|
|
||||||
if _scheduler_stop.wait(timeout=wait_seconds):
|
if _scheduler_stop.wait(timeout=wait_seconds):
|
||||||
break # Stop event was set
|
break
|
||||||
|
|
||||||
try:
|
try:
|
||||||
update_carrier_positions()
|
update_carrier_positions()
|
||||||
|
|||||||
@@ -987,7 +987,7 @@ _KML_NS = {"kml": "http://www.opengis.net/kml/2.2"}
|
|||||||
|
|
||||||
def _find_kml_element(element, tag):
|
def _find_kml_element(element, tag):
|
||||||
"""Find first descendant matching tag, ignoring XML namespace prefix."""
|
"""Find first descendant matching tag, ignoring XML namespace prefix."""
|
||||||
import xml.etree.ElementTree as ET
|
import defusedxml.ElementTree as ET
|
||||||
el = element.find(f".//{tag}")
|
el = element.find(f".//{tag}")
|
||||||
if el is not None:
|
if el is not None:
|
||||||
return el
|
return el
|
||||||
@@ -1015,7 +1015,7 @@ class MadridCityIngestor(BaseCCTVIngestor):
|
|||||||
KML_URL = "http://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml"
|
KML_URL = "http://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml"
|
||||||
|
|
||||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||||
import xml.etree.ElementTree as ET
|
import defusedxml.ElementTree as ET
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = fetch_with_curl(self.KML_URL, timeout=20)
|
response = fetch_with_curl(self.KML_URL, timeout=20)
|
||||||
|
|||||||
@@ -46,10 +46,19 @@ class Settings(BaseSettings):
|
|||||||
MESH_NODE_MODE: str = "participant"
|
MESH_NODE_MODE: str = "participant"
|
||||||
MESH_SYNC_INTERVAL_S: int = 300
|
MESH_SYNC_INTERVAL_S: int = 300
|
||||||
MESH_SYNC_FAILURE_BACKOFF_S: int = 60
|
MESH_SYNC_FAILURE_BACKOFF_S: int = 60
|
||||||
|
MESH_SYNC_TIMEOUT_S: int = 5
|
||||||
|
MESH_SYNC_MAX_PEERS_PER_CYCLE: int = 3
|
||||||
MESH_RELAY_PUSH_TIMEOUT_S: int = 10
|
MESH_RELAY_PUSH_TIMEOUT_S: int = 10
|
||||||
MESH_RELAY_MAX_FAILURES: int = 3
|
MESH_RELAY_MAX_FAILURES: int = 3
|
||||||
MESH_RELAY_FAILURE_COOLDOWN_S: int = 120
|
MESH_RELAY_FAILURE_COOLDOWN_S: int = 120
|
||||||
|
MESH_BOOTSTRAP_SEED_FAILURE_COOLDOWN_S: int = 15
|
||||||
MESH_PEER_PUSH_SECRET: str = ""
|
MESH_PEER_PUSH_SECRET: str = ""
|
||||||
|
# Issue #256 (tg12): optional per-peer HMAC secret map. Comma-separated
|
||||||
|
# `url=secret` pairs. When a peer URL appears here, only that per-peer
|
||||||
|
# secret is accepted for it — the global MESH_PEER_PUSH_SECRET above is
|
||||||
|
# ignored for that specific URL. Single-peer installs and unmigrated
|
||||||
|
# multi-peer installs leave this empty and behavior is unchanged.
|
||||||
|
MESH_PEER_SECRETS: str = ""
|
||||||
MESH_RNS_APP_NAME: str = "shadowbroker"
|
MESH_RNS_APP_NAME: str = "shadowbroker"
|
||||||
MESH_RNS_ASPECT: str = "infonet"
|
MESH_RNS_ASPECT: str = "infonet"
|
||||||
MESH_RNS_IDENTITY_PATH: str = ""
|
MESH_RNS_IDENTITY_PATH: str = ""
|
||||||
@@ -107,6 +116,21 @@ class Settings(BaseSettings):
|
|||||||
MESH_DM_REQUEST_MAILBOX_LIMIT: int = 12
|
MESH_DM_REQUEST_MAILBOX_LIMIT: int = 12
|
||||||
MESH_DM_SHARED_MAILBOX_LIMIT: int = 48
|
MESH_DM_SHARED_MAILBOX_LIMIT: int = 48
|
||||||
MESH_DM_SELF_MAILBOX_LIMIT: int = 12
|
MESH_DM_SELF_MAILBOX_LIMIT: int = 12
|
||||||
|
# Anti-spam: cap on distinct UNACKED messages a single sender can have
|
||||||
|
# parked in a single recipient's mailbox at any one time. Once the
|
||||||
|
# recipient pulls (acks) a message, the sender's quota for that pair
|
||||||
|
# frees up. Default 2 — a sender who wants to deliver more must wait
|
||||||
|
# for the recipient to actually read the prior messages.
|
||||||
|
#
|
||||||
|
# This cap is enforced TWICE: once on the local deposit path (the
|
||||||
|
# sender's own node refuses to spool the 3rd message) AND once on
|
||||||
|
# the replication-acceptance path (honest peer relays refuse to
|
||||||
|
# accept inbound replicas that would put them over the cap). The
|
||||||
|
# double enforcement makes the rule a NETWORK rule — patching out
|
||||||
|
# the local check on a hostile sender's relay doesn't let extras
|
||||||
|
# propagate, because every honest peer enforces the same cap on
|
||||||
|
# inbound replication.
|
||||||
|
MESH_DM_PENDING_PER_SENDER_LIMIT: int = 2
|
||||||
MESH_BLOCK_LEGACY_AGENT_ID_LOOKUP: bool = True
|
MESH_BLOCK_LEGACY_AGENT_ID_LOOKUP: bool = True
|
||||||
MESH_ALLOW_COMPAT_DM_INVITE_IMPORT: bool = False
|
MESH_ALLOW_COMPAT_DM_INVITE_IMPORT: bool = False
|
||||||
MESH_ALLOW_COMPAT_DM_INVITE_IMPORT_UNTIL: str = ""
|
MESH_ALLOW_COMPAT_DM_INVITE_IMPORT_UNTIL: str = ""
|
||||||
@@ -286,6 +310,19 @@ class Settings(BaseSettings):
|
|||||||
# service operator can identify per-install traffic instead of a generic
|
# service operator can identify per-install traffic instead of a generic
|
||||||
# "ShadowBroker" aggregate.
|
# "ShadowBroker" aggregate.
|
||||||
MESHTASTIC_OPERATOR_CALLSIGN: str = ""
|
MESHTASTIC_OPERATOR_CALLSIGN: str = ""
|
||||||
|
# Per-install operator handle used in the User-Agent for EVERY third-party
|
||||||
|
# API the backend calls (Wikipedia, Wikidata, Nominatim, GDELT, OpenMHz,
|
||||||
|
# Broadcastify, weather.gov, NUFORC, etc.). The default is empty, in which
|
||||||
|
# case backend/services/network_utils.py auto-generates a stable
|
||||||
|
# pseudonymous handle like "operator-7f3a92" on first use and caches it.
|
||||||
|
# Operators who want to identify themselves with a real handle can set
|
||||||
|
# this; operators who want to stay pseudonymous can leave it empty.
|
||||||
|
#
|
||||||
|
# The handle is sent ONLY to public third-party APIs. It is NEVER mixed
|
||||||
|
# into mesh / Wormhole / Infonet identity (those have their own crypto
|
||||||
|
# identity layer; conflating the two would leak public attribution into
|
||||||
|
# private mesh state).
|
||||||
|
OPERATOR_HANDLE: str = ""
|
||||||
|
|
||||||
# SAR (Synthetic Aperture Radar) data layer
|
# SAR (Synthetic Aperture Radar) data layer
|
||||||
# Mode A — free catalog metadata, no account, default-on
|
# Mode A — free catalog metadata, no account, default-on
|
||||||
|
|||||||
@@ -16,8 +16,15 @@ from typing import Any
|
|||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _feed_ingester_user_agent() -> str:
|
||||||
|
# Round 7a: per-install attribution for operator-curated feed URLs.
|
||||||
|
return outbound_user_agent("feed-ingester")
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# State
|
# State
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -157,7 +164,7 @@ def _fetch_layer_feed(layer: dict[str, Any]) -> None:
|
|||||||
resp = requests.get(
|
resp = requests.get(
|
||||||
feed_url,
|
feed_url,
|
||||||
timeout=_FETCH_TIMEOUT,
|
timeout=_FETCH_TIMEOUT,
|
||||||
headers={"User-Agent": "ShadowBroker-FeedIngester/1.0"},
|
headers={"User-Agent": _feed_ingester_user_agent()},
|
||||||
)
|
)
|
||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
data = resp.json()
|
data = resp.json()
|
||||||
|
|||||||
@@ -318,7 +318,7 @@ active_layers: dict[str, bool] = {
|
|||||||
"uap_sightings": True,
|
"uap_sightings": True,
|
||||||
"wastewater": True,
|
"wastewater": True,
|
||||||
"ai_intel": True,
|
"ai_intel": True,
|
||||||
"crowdthreat": True,
|
"crowdthreat": False,
|
||||||
"sar": True,
|
"sar": True,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,11 +16,18 @@ import csv
|
|||||||
import logging
|
import logging
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
import xml.etree.ElementTree as ET
|
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
import defusedxml.ElementTree as ET
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def _aircraft_db_user_agent() -> str:
|
||||||
|
"""Round 7a: lazy import so the per-install operator handle is included."""
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
|
return outbound_user_agent("aircraft-database")
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
_BUCKET_LIST_URL = (
|
_BUCKET_LIST_URL = (
|
||||||
@@ -31,11 +38,7 @@ _S3_NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
|
|||||||
_REFRESH_INTERVAL_S = 5 * 24 * 3600
|
_REFRESH_INTERVAL_S = 5 * 24 * 3600
|
||||||
_LIST_TIMEOUT_S = 30
|
_LIST_TIMEOUT_S = 30
|
||||||
_DOWNLOAD_TIMEOUT_S = 600
|
_DOWNLOAD_TIMEOUT_S = 600
|
||||||
_USER_AGENT = (
|
from services.network_utils import DEFAULT_USER_AGENT as _USER_AGENT
|
||||||
"ShadowBroker-OSINT/0.9.75 "
|
|
||||||
"(+https://github.com/BigBodyCobain/Shadowbroker; "
|
|
||||||
"contact: bigbodycobain@gmail.com)"
|
|
||||||
)
|
|
||||||
|
|
||||||
_lock = threading.RLock()
|
_lock = threading.RLock()
|
||||||
_aircraft_by_hex: dict[str, dict[str, str]] = {}
|
_aircraft_by_hex: dict[str, dict[str, str]] = {}
|
||||||
@@ -48,7 +51,7 @@ def _latest_snapshot_key() -> str:
|
|||||||
response = requests.get(
|
response = requests.get(
|
||||||
_BUCKET_LIST_URL,
|
_BUCKET_LIST_URL,
|
||||||
timeout=_LIST_TIMEOUT_S,
|
timeout=_LIST_TIMEOUT_S,
|
||||||
headers={"User-Agent": _USER_AGENT},
|
headers={"User-Agent": _aircraft_db_user_agent()},
|
||||||
)
|
)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
root = ET.fromstring(response.text)
|
root = ET.fromstring(response.text)
|
||||||
@@ -75,7 +78,7 @@ def _stream_csv_index(url: str) -> dict[str, dict[str, str]]:
|
|||||||
url,
|
url,
|
||||||
timeout=_DOWNLOAD_TIMEOUT_S,
|
timeout=_DOWNLOAD_TIMEOUT_S,
|
||||||
stream=True,
|
stream=True,
|
||||||
headers={"User-Agent": _USER_AGENT},
|
headers={"User-Agent": _aircraft_db_user_agent()},
|
||||||
) as response:
|
) as response:
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
line_iter = (
|
line_iter = (
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ No API key required — the /threats endpoint is unauthenticated.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
|
|
||||||
from services.network_utils import fetch_with_curl
|
from services.network_utils import fetch_with_curl
|
||||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh, is_any_active
|
from services.fetchers._store import latest_data, _data_lock, _mark_fresh, is_any_active
|
||||||
@@ -16,6 +17,16 @@ logger = logging.getLogger("services.data_fetcher")
|
|||||||
|
|
||||||
_CT_BASE = "https://backend.crowdthreat.world"
|
_CT_BASE = "https://backend.crowdthreat.world"
|
||||||
|
|
||||||
|
|
||||||
|
def crowdthreat_fetch_enabled() -> bool:
|
||||||
|
"""Return True only when the operator explicitly opts into CrowdThreat pulls."""
|
||||||
|
return str(os.environ.get("CROWDTHREAT_ENABLED", "")).strip().lower() in {
|
||||||
|
"1",
|
||||||
|
"true",
|
||||||
|
"yes",
|
||||||
|
"on",
|
||||||
|
}
|
||||||
|
|
||||||
# CrowdThreat category_id → icon ID used on the MapLibre layer
|
# CrowdThreat category_id → icon ID used on the MapLibre layer
|
||||||
_CATEGORY_ICON = {
|
_CATEGORY_ICON = {
|
||||||
1: "ct-security", # Security & Conflict (red)
|
1: "ct-security", # Security & Conflict (red)
|
||||||
@@ -43,6 +54,12 @@ _CATEGORY_COLOUR = {
|
|||||||
@with_retry(max_retries=2, base_delay=5)
|
@with_retry(max_retries=2, base_delay=5)
|
||||||
def fetch_crowdthreat():
|
def fetch_crowdthreat():
|
||||||
"""Fetch verified threat reports from CrowdThreat public API."""
|
"""Fetch verified threat reports from CrowdThreat public API."""
|
||||||
|
if not crowdthreat_fetch_enabled():
|
||||||
|
logger.debug("CrowdThreat fetch skipped; set CROWDTHREAT_ENABLED=true to opt in")
|
||||||
|
with _data_lock:
|
||||||
|
latest_data["crowdthreat"] = []
|
||||||
|
_mark_fresh("crowdthreat")
|
||||||
|
return
|
||||||
if not is_any_active("crowdthreat"):
|
if not is_any_active("crowdthreat"):
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,11 @@ import time
|
|||||||
import heapq
|
import heapq
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from services.network_utils import external_curl_fallback_enabled, fetch_with_curl
|
from services.network_utils import (
|
||||||
|
external_curl_fallback_enabled,
|
||||||
|
fetch_with_curl,
|
||||||
|
outbound_user_agent,
|
||||||
|
)
|
||||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||||
from services.fetchers.nuforc_enrichment import enrich_sighting
|
from services.fetchers.nuforc_enrichment import enrich_sighting
|
||||||
from services.fetchers.retry import with_retry
|
from services.fetchers.retry import with_retry
|
||||||
@@ -279,9 +283,13 @@ def fetch_weather_alerts():
|
|||||||
return
|
return
|
||||||
alerts = []
|
alerts = []
|
||||||
try:
|
try:
|
||||||
|
# weather.gov requires a User-Agent per their API policy. Round 7a:
|
||||||
|
# send the per-install operator handle so they can rate-limit per
|
||||||
|
# operator instead of treating "Shadowbroker" as one entity.
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
url = "https://api.weather.gov/alerts/active?status=actual"
|
url = "https://api.weather.gov/alerts/active?status=actual"
|
||||||
headers = {
|
headers = {
|
||||||
"User-Agent": "(ShadowBroker OSINT Dashboard, github.com/BigBodyCobain/Shadowbroker)",
|
"User-Agent": outbound_user_agent("weather-gov"),
|
||||||
"Accept": "application/geo+json",
|
"Accept": "application/geo+json",
|
||||||
}
|
}
|
||||||
response = fetch_with_curl(url, timeout=15, headers=headers)
|
response = fetch_with_curl(url, timeout=15, headers=headers)
|
||||||
@@ -709,7 +717,12 @@ _NUFORC_LIVE_NONCE_RE = re.compile(
|
|||||||
r'id=["\']wdtNonceFrontendServerSide_1["\'][^>]*value=["\']([a-f0-9]+)["\']'
|
r'id=["\']wdtNonceFrontendServerSide_1["\'][^>]*value=["\']([a-f0-9]+)["\']'
|
||||||
)
|
)
|
||||||
_NUFORC_LIVE_SIGHTING_ID_RE = re.compile(r"id=(\d+)")
|
_NUFORC_LIVE_SIGHTING_ID_RE = re.compile(r"id=(\d+)")
|
||||||
_NUFORC_LIVE_USER_AGENT = "Mozilla/5.0 (ShadowBroker-OSINT NUFORC-fetcher)"
|
# Round 7a: NUFORC's site is sensitive to non-browser UAs but we send a
|
||||||
|
# per-install operator handle prefixed by Mozilla/5.0 so we're identifiable
|
||||||
|
# without being aggregately blocked. Operators who want stricter privacy
|
||||||
|
# can override the entire UA via SHADOWBROKER_USER_AGENT.
|
||||||
|
def _nuforc_live_user_agent() -> str:
|
||||||
|
return f"Mozilla/5.0 ({outbound_user_agent('nuforc-live')})"
|
||||||
_NUFORC_LIVE_SESSION_COOKIES = _NUFORC_DATA_DIR / "nuforc_session.cookies"
|
_NUFORC_LIVE_SESSION_COOKIES = _NUFORC_DATA_DIR / "nuforc_session.cookies"
|
||||||
|
|
||||||
# Sample grid covering continental US, Alaska, Hawaii, Canada, UK, Australia
|
# Sample grid covering continental US, Alaska, Hawaii, Canada, UK, Australia
|
||||||
@@ -953,7 +966,7 @@ def _photon_lookup(query: str) -> list[float] | None:
|
|||||||
res = fetch_with_curl(
|
res = fetch_with_curl(
|
||||||
url,
|
url,
|
||||||
headers={
|
headers={
|
||||||
"User-Agent": "ShadowBroker-OSINT/1.0 (NUFORC-UAP-layer)",
|
"User-Agent": outbound_user_agent("nuforc-uap-geocode"),
|
||||||
"Accept-Language": "en",
|
"Accept-Language": "en",
|
||||||
},
|
},
|
||||||
timeout=10,
|
timeout=10,
|
||||||
@@ -1049,7 +1062,7 @@ def _nuforc_fetch_month_live(yyyymm: str, cookie_jar: Path) -> list[dict]:
|
|||||||
index_res = subprocess.run(
|
index_res = subprocess.run(
|
||||||
[
|
[
|
||||||
curl_bin, "-sL",
|
curl_bin, "-sL",
|
||||||
"-A", _NUFORC_LIVE_USER_AGENT,
|
"-A", _nuforc_live_user_agent(),
|
||||||
"-c", str(cookie_jar),
|
"-c", str(cookie_jar),
|
||||||
"-b", str(cookie_jar),
|
"-b", str(cookie_jar),
|
||||||
index_url,
|
index_url,
|
||||||
@@ -1085,7 +1098,7 @@ def _nuforc_fetch_month_live(yyyymm: str, cookie_jar: Path) -> list[dict]:
|
|||||||
ajax_res = subprocess.run(
|
ajax_res = subprocess.run(
|
||||||
[
|
[
|
||||||
curl_bin, "-sL",
|
curl_bin, "-sL",
|
||||||
"-A", _NUFORC_LIVE_USER_AGENT,
|
"-A", _nuforc_live_user_agent(),
|
||||||
"-c", str(cookie_jar),
|
"-c", str(cookie_jar),
|
||||||
"-b", str(cookie_jar),
|
"-b", str(cookie_jar),
|
||||||
"-X", "POST",
|
"-X", "POST",
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ debunked claims, threat actor mentions, and target country references.
|
|||||||
Refreshes every 12 hours (FIMI data updates weekly).
|
Refreshes every 12 hours (FIMI data updates weekly).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
import logging
|
import logging
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
@@ -18,6 +19,16 @@ logger = logging.getLogger("services.data_fetcher")
|
|||||||
|
|
||||||
_FIMI_FEED_URL = "https://euvsdisinfo.eu/feed/"
|
_FIMI_FEED_URL = "https://euvsdisinfo.eu/feed/"
|
||||||
|
|
||||||
|
|
||||||
|
def fimi_fetch_enabled() -> bool:
|
||||||
|
"""Return True only when the operator explicitly opts into FIMI pulls."""
|
||||||
|
return str(os.environ.get("FIMI_ENABLED", "")).strip().lower() in {
|
||||||
|
"1",
|
||||||
|
"true",
|
||||||
|
"yes",
|
||||||
|
"on",
|
||||||
|
}
|
||||||
|
|
||||||
# ── Threat actor keywords ──────────────────────────────────────────────────
|
# ── Threat actor keywords ──────────────────────────────────────────────────
|
||||||
# Map of keyword → canonical actor name. Checked case-insensitively.
|
# Map of keyword → canonical actor name. Checked case-insensitively.
|
||||||
_THREAT_ACTORS: dict[str, str] = {
|
_THREAT_ACTORS: dict[str, str] = {
|
||||||
@@ -173,6 +184,12 @@ def _is_major_wave(narratives: list[dict], targets: dict[str, int]) -> bool:
|
|||||||
@with_retry(max_retries=1, base_delay=5)
|
@with_retry(max_retries=1, base_delay=5)
|
||||||
def fetch_fimi():
|
def fetch_fimi():
|
||||||
"""Fetch and parse the EUvsDisinfo RSS feed."""
|
"""Fetch and parse the EUvsDisinfo RSS feed."""
|
||||||
|
if not fimi_fetch_enabled():
|
||||||
|
logger.debug("FIMI fetch skipped; set FIMI_ENABLED=true to opt in")
|
||||||
|
with _data_lock:
|
||||||
|
latest_data["fimi"] = []
|
||||||
|
_mark_fresh("fimi")
|
||||||
|
return
|
||||||
try:
|
try:
|
||||||
resp = fetch_with_curl(_FIMI_FEED_URL, timeout=15)
|
resp = fetch_with_curl(_FIMI_FEED_URL, timeout=15)
|
||||||
feed = feedparser.parse(resp.text)
|
feed = feedparser.parse(resp.text)
|
||||||
|
|||||||
@@ -82,10 +82,37 @@ def _fetch_yfinance_single(symbol: str, period: str = "2d"):
|
|||||||
|
|
||||||
|
|
||||||
@with_retry(max_retries=1, base_delay=1)
|
@with_retry(max_retries=1, base_delay=1)
|
||||||
|
def financial_fetch_enabled() -> bool:
|
||||||
|
"""Return True only when the operator explicitly opts into financial pulls.
|
||||||
|
|
||||||
|
Either ``FINANCIAL_ENABLED=true`` or the presence of ``FINNHUB_API_KEY``
|
||||||
|
counts as an explicit opt-in. Without either, the default yfinance path
|
||||||
|
is disabled to avoid silent outbound calls to finance.yahoo.com.
|
||||||
|
"""
|
||||||
|
if os.getenv("FINNHUB_API_KEY", "").strip():
|
||||||
|
return True
|
||||||
|
return str(os.environ.get("FINANCIAL_ENABLED", "")).strip().lower() in {
|
||||||
|
"1",
|
||||||
|
"true",
|
||||||
|
"yes",
|
||||||
|
"on",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def fetch_financial_markets():
|
def fetch_financial_markets():
|
||||||
"""Fetches full market list with smart throttling (3s for Finnhub, 60s for yfinance)."""
|
"""Fetches full market list with smart throttling (3s for Finnhub, 60s for yfinance)."""
|
||||||
global _last_fetch_time, _last_fetch_results, _rotating_index
|
global _last_fetch_time, _last_fetch_results, _rotating_index
|
||||||
|
|
||||||
|
if not financial_fetch_enabled():
|
||||||
|
logger.debug(
|
||||||
|
"Financial fetch skipped; set FINANCIAL_ENABLED=true or supply "
|
||||||
|
"FINNHUB_API_KEY to opt in"
|
||||||
|
)
|
||||||
|
with _data_lock:
|
||||||
|
latest_data["financial"] = {}
|
||||||
|
_mark_fresh("financial")
|
||||||
|
return
|
||||||
|
|
||||||
finnhub_key = os.getenv("FINNHUB_API_KEY", "").strip()
|
finnhub_key = os.getenv("FINNHUB_API_KEY", "").strip()
|
||||||
use_finnhub = bool(finnhub_key)
|
use_finnhub = bool(finnhub_key)
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import heapq
|
|||||||
import logging
|
import logging
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from cachetools import TTLCache
|
from cachetools import TTLCache
|
||||||
from services.network_utils import fetch_with_curl
|
from services.network_utils import fetch_with_curl, outbound_user_agent
|
||||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||||
from services.fetchers.retry import with_retry
|
from services.fetchers.retry import with_retry
|
||||||
|
|
||||||
@@ -29,7 +29,7 @@ def _geocode_region(region_name: str, country_name: str) -> tuple:
|
|||||||
|
|
||||||
query = urllib.parse.quote(f"{region_name}, {country_name}")
|
query = urllib.parse.quote(f"{region_name}, {country_name}")
|
||||||
url = f"https://nominatim.openstreetmap.org/search?q={query}&format=json&limit=1"
|
url = f"https://nominatim.openstreetmap.org/search?q={query}&format=json&limit=1"
|
||||||
response = fetch_with_curl(url, timeout=8, headers={"User-Agent": "ShadowBroker-OSINT/1.0"})
|
response = fetch_with_curl(url, timeout=8, headers={"User-Agent": outbound_user_agent("infrastructure-data")})
|
||||||
if response.status_code == 200:
|
if response.status_code == 200:
|
||||||
results = response.json()
|
results = response.json()
|
||||||
if results:
|
if results:
|
||||||
|
|||||||
@@ -174,16 +174,34 @@ def fetch_meshtastic_nodes():
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug(f"Meshtastic cache freshness check failed: {e}")
|
logger.debug(f"Meshtastic cache freshness check failed: {e}")
|
||||||
|
|
||||||
# Build a polite User-Agent. Include the operator callsign when set so
|
# Build a polite User-Agent. Historically this included the operator
|
||||||
# the upstream service can correlate per-install traffic if needed.
|
# callsign so meshtastic.org could rate-limit per-install; that's still
|
||||||
|
# the default behavior for backward compatibility. Operators who want
|
||||||
|
# stricter outbound privacy can suppress the callsign by setting
|
||||||
|
# MESHTASTIC_SEND_CALLSIGN_HEADER=false. Issue #203.
|
||||||
|
import os as _os
|
||||||
try:
|
try:
|
||||||
from services.config import get_settings
|
from services.config import get_settings
|
||||||
|
|
||||||
callsign = str(getattr(get_settings(), "MESHTASTIC_OPERATOR_CALLSIGN", "") or "").strip()
|
callsign = str(getattr(get_settings(), "MESHTASTIC_OPERATOR_CALLSIGN", "") or "").strip()
|
||||||
except Exception:
|
except Exception:
|
||||||
callsign = ""
|
callsign = ""
|
||||||
ua_base = "ShadowBroker-OSINT/0.9.75 (+https://github.com/BigBodyCobain/Shadowbroker; contact: bigbodycobain@gmail.com; 24h polling)"
|
|
||||||
user_agent = f"{ua_base}; node={callsign}" if callsign else ua_base
|
send_callsign_header = str(
|
||||||
|
_os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "true")
|
||||||
|
).strip().lower() not in {"0", "false", "no", "off", ""}
|
||||||
|
|
||||||
|
# Round 7a: outbound_user_agent already includes the per-install handle.
|
||||||
|
# The optional Meshtastic callsign is appended as additional context so
|
||||||
|
# meshtastic.liamcottle.net's operator can identify both the install AND
|
||||||
|
# the registered radio operator (when MESHTASTIC_OPERATOR_CALLSIGN is set
|
||||||
|
# and MESHTASTIC_SEND_CALLSIGN_HEADER is true; see issue #203).
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
|
ua_base = f"{outbound_user_agent('meshtastic-map')}; 24h polling"
|
||||||
|
if callsign and send_callsign_header:
|
||||||
|
user_agent = f"{ua_base}; node={callsign}"
|
||||||
|
else:
|
||||||
|
user_agent = ua_base
|
||||||
|
|
||||||
try:
|
try:
|
||||||
logger.info("Fetching Meshtastic map nodes from API...")
|
logger.info("Fetching Meshtastic map nodes from API...")
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
"""News fetching, geocoding, clustering, and risk assessment."""
|
"""News fetching, geocoding, clustering, and risk assessment."""
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
import logging
|
import logging
|
||||||
@@ -11,6 +12,22 @@ from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
|||||||
from services.fetchers.retry import with_retry
|
from services.fetchers.retry import with_retry
|
||||||
from services.oracle_service import enrich_news_items, compute_global_threat_level, detect_breaking_events
|
from services.oracle_service import enrich_news_items, compute_global_threat_level, detect_breaking_events
|
||||||
|
|
||||||
|
|
||||||
|
def news_fetch_enabled() -> bool:
|
||||||
|
"""Return True only when the operator explicitly opts into news RSS pulls.
|
||||||
|
|
||||||
|
Defaults to **on** for backward compatibility (this is the only fetcher
|
||||||
|
where opting out is the new behavior, not the old one). Set
|
||||||
|
``NEWS_ENABLED=false`` to disable all outbound RSS feed traffic.
|
||||||
|
"""
|
||||||
|
return str(os.environ.get("NEWS_ENABLED", "true")).strip().lower() not in {
|
||||||
|
"0",
|
||||||
|
"false",
|
||||||
|
"no",
|
||||||
|
"off",
|
||||||
|
"",
|
||||||
|
}
|
||||||
|
|
||||||
logger = logging.getLogger("services.data_fetcher")
|
logger = logging.getLogger("services.data_fetcher")
|
||||||
|
|
||||||
# Maximum article age in seconds. Anything older than this is dropped
|
# Maximum article age in seconds. Anything older than this is dropped
|
||||||
@@ -160,6 +177,12 @@ def _resolve_coords(text: str) -> tuple[float, float] | None:
|
|||||||
|
|
||||||
@with_retry(max_retries=1, base_delay=2)
|
@with_retry(max_retries=1, base_delay=2)
|
||||||
def fetch_news():
|
def fetch_news():
|
||||||
|
if not news_fetch_enabled():
|
||||||
|
logger.debug("News fetch skipped; unset NEWS_ENABLED=false to re-enable")
|
||||||
|
with _data_lock:
|
||||||
|
latest_data["news"] = []
|
||||||
|
_mark_fresh("news")
|
||||||
|
return
|
||||||
from services.news_feed_config import get_feeds
|
from services.news_feed_config import get_feeds
|
||||||
feed_config = get_feeds()
|
feed_config = get_feeds()
|
||||||
feeds = {f["name"]: f["url"] for f in feed_config}
|
feeds = {f["name"]: f["url"] for f in feed_config}
|
||||||
|
|||||||
@@ -49,6 +49,16 @@ _HF_CSV_URL = (
|
|||||||
"https://huggingface.co/datasets/kcimc/NUFORC/resolve/main/nuforc_str.csv"
|
"https://huggingface.co/datasets/kcimc/NUFORC/resolve/main/nuforc_str.csv"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def nuforc_fetch_enabled() -> bool:
|
||||||
|
"""Return True only when the operator explicitly opts into NUFORC pulls."""
|
||||||
|
return str(os.environ.get("NUFORC_ENABLED", "")).strip().lower() in {
|
||||||
|
"1",
|
||||||
|
"true",
|
||||||
|
"yes",
|
||||||
|
"on",
|
||||||
|
}
|
||||||
|
|
||||||
# Only keep sightings from the last N years for the enrichment index
|
# Only keep sightings from the last N years for the enrichment index
|
||||||
_KEEP_YEARS = 5
|
_KEEP_YEARS = 5
|
||||||
|
|
||||||
@@ -160,6 +170,12 @@ def _download_and_build() -> dict | None:
|
|||||||
|
|
||||||
Returns the index dict or None on failure.
|
Returns the index dict or None on failure.
|
||||||
"""
|
"""
|
||||||
|
if not nuforc_fetch_enabled():
|
||||||
|
logger.debug(
|
||||||
|
"NUFORC enrichment skipped; set NUFORC_ENABLED=true to opt in"
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
cutoff = datetime.utcnow() - timedelta(days=_KEEP_YEARS * 365)
|
cutoff = datetime.utcnow() - timedelta(days=_KEEP_YEARS * 365)
|
||||||
cutoff_str = cutoff.strftime("%Y-%m-%d")
|
cutoff_str = cutoff.strftime("%Y-%m-%d")
|
||||||
|
|
||||||
|
|||||||
@@ -25,6 +25,16 @@ _provider_pace_lock = threading.Lock()
|
|||||||
_provider_last_request_at: dict[str, float] = {}
|
_provider_last_request_at: dict[str, float] = {}
|
||||||
|
|
||||||
|
|
||||||
|
def prediction_markets_fetch_enabled() -> bool:
|
||||||
|
"""Return True only when the operator explicitly opts into Polymarket/Kalshi pulls."""
|
||||||
|
return str(os.environ.get("PREDICTION_MARKETS_ENABLED", "")).strip().lower() in {
|
||||||
|
"1",
|
||||||
|
"true",
|
||||||
|
"yes",
|
||||||
|
"on",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _pace_provider(provider: str, min_interval_s: float) -> None:
|
def _pace_provider(provider: str, min_interval_s: float) -> None:
|
||||||
if min_interval_s <= 0:
|
if min_interval_s <= 0:
|
||||||
return
|
return
|
||||||
@@ -755,6 +765,16 @@ def fetch_prediction_markets():
|
|||||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||||
global _prev_probabilities
|
global _prev_probabilities
|
||||||
|
|
||||||
|
if not prediction_markets_fetch_enabled():
|
||||||
|
logger.debug(
|
||||||
|
"Prediction markets fetch skipped; set "
|
||||||
|
"PREDICTION_MARKETS_ENABLED=true to opt in"
|
||||||
|
)
|
||||||
|
with _data_lock:
|
||||||
|
latest_data["prediction_markets"] = []
|
||||||
|
_mark_fresh("prediction_markets")
|
||||||
|
return
|
||||||
|
|
||||||
markets = fetch_prediction_markets_raw()
|
markets = fetch_prediction_markets_raw()
|
||||||
|
|
||||||
# Compute probability deltas vs previous fetch
|
# Compute probability deltas vs previous fetch
|
||||||
|
|||||||
@@ -17,6 +17,12 @@ from typing import Any
|
|||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def _route_db_user_agent() -> str:
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
|
return outbound_user_agent("route-database")
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
_ROUTES_URL = "https://vrs-standing-data.adsb.lol/routes.csv.gz"
|
_ROUTES_URL = "https://vrs-standing-data.adsb.lol/routes.csv.gz"
|
||||||
@@ -24,11 +30,7 @@ _AIRPORTS_URL = "https://vrs-standing-data.adsb.lol/airports.csv.gz"
|
|||||||
_REFRESH_INTERVAL_S = 5 * 24 * 3600
|
_REFRESH_INTERVAL_S = 5 * 24 * 3600
|
||||||
_HTTP_TIMEOUT_S = 60
|
_HTTP_TIMEOUT_S = 60
|
||||||
|
|
||||||
_USER_AGENT = (
|
from services.network_utils import DEFAULT_USER_AGENT as _USER_AGENT
|
||||||
"ShadowBroker-OSINT/0.9.75 "
|
|
||||||
"(+https://github.com/BigBodyCobain/Shadowbroker; "
|
|
||||||
"contact: bigbodycobain@gmail.com)"
|
|
||||||
)
|
|
||||||
|
|
||||||
_lock = threading.RLock()
|
_lock = threading.RLock()
|
||||||
_routes_by_callsign: dict[str, dict[str, Any]] = {}
|
_routes_by_callsign: dict[str, dict[str, Any]] = {}
|
||||||
@@ -41,7 +43,7 @@ def _fetch_csv_gz(url: str) -> list[dict[str, str]]:
|
|||||||
response = requests.get(
|
response = requests.get(
|
||||||
url,
|
url,
|
||||||
timeout=_HTTP_TIMEOUT_S,
|
timeout=_HTTP_TIMEOUT_S,
|
||||||
headers={"User-Agent": _USER_AGENT, "Accept-Encoding": "gzip"},
|
headers={"User-Agent": _route_db_user_agent(), "Accept-Encoding": "gzip"},
|
||||||
)
|
)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
text = gzip.decompress(response.content).decode("utf-8-sig")
|
text = gzip.decompress(response.content).decode("utf-8-sig")
|
||||||
|
|||||||
@@ -10,6 +10,12 @@ from datetime import datetime, timezone
|
|||||||
from services.fetchers._store import _data_lock, _mark_fresh, latest_data
|
from services.fetchers._store import _data_lock, _mark_fresh, latest_data
|
||||||
from services.network_utils import fetch_with_curl
|
from services.network_utils import fetch_with_curl
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def _trains_user_agent() -> str:
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
|
return outbound_user_agent("trains")
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
_EARTH_RADIUS_KM = 6371.0
|
_EARTH_RADIUS_KM = 6371.0
|
||||||
@@ -379,7 +385,7 @@ def _fetch_digitraffic() -> list[dict]:
|
|||||||
timeout=15,
|
timeout=15,
|
||||||
headers={
|
headers={
|
||||||
"Accept-Encoding": "gzip",
|
"Accept-Encoding": "gzip",
|
||||||
"User-Agent": "ShadowBroker-OSINT/1.0",
|
"User-Agent": _trains_user_agent(),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
if resp.status_code != 200:
|
if resp.status_code != 200:
|
||||||
|
|||||||
@@ -0,0 +1,457 @@
|
|||||||
|
"""USNI News Fleet & Marine Tracker — authoritative weekly carrier
|
||||||
|
position publication.
|
||||||
|
|
||||||
|
Why this exists
|
||||||
|
---------------
|
||||||
|
The previous carrier_tracker pipeline relied on GDELT headline matching
|
||||||
|
(``api.gdeltproject.org``) to derive positions from text like "USS Ford
|
||||||
|
in the Mediterranean" → centroid of "Mediterranean Sea". That was
|
||||||
|
- low-precision (audit issue #245 — false precision from text mentions),
|
||||||
|
- unreliable (``api.gdeltproject.org`` is sometimes unreachable from
|
||||||
|
certain network paths, including Docker Desktop on some Windows hosts).
|
||||||
|
|
||||||
|
USNI publishes a weekly tracker that explicitly lists where every U.S.
|
||||||
|
carrier is operating. The article body uses extremely consistent phrasing:
|
||||||
|
|
||||||
|
"The Gerald R. Ford Carrier Strike Group is operating in the Red Sea"
|
||||||
|
"Aircraft carrier USS George Washington (CVN-73) is in port in
|
||||||
|
Yokosuka, Japan."
|
||||||
|
"USS Dwight D. Eisenhower (CVN-69) sails down the Elizabeth River"
|
||||||
|
|
||||||
|
Those are deterministic to parse. This module:
|
||||||
|
|
||||||
|
1. Pulls the WordPress RSS feeds (both site-wide and category) — the
|
||||||
|
site-wide feed often has fresher posts before the category feed
|
||||||
|
catches up, so we union them.
|
||||||
|
2. Picks the most recent post by parsed ``pubDate``.
|
||||||
|
3. For each carrier in the registry, scans the article body for a
|
||||||
|
"is operating in / is in port in / departed from" pattern near
|
||||||
|
the carrier's name.
|
||||||
|
4. Maps the extracted region phrase to coordinates via the carrier
|
||||||
|
tracker's existing REGION_COORDS.
|
||||||
|
|
||||||
|
The result is a ``{hull: position_entry}`` dict that the carrier tracker
|
||||||
|
consumes as a high-confidence source — ``position_confidence: "recent"``
|
||||||
|
with ``position_source_at`` set to the article's actual publication
|
||||||
|
timestamp (not ``now()``).
|
||||||
|
|
||||||
|
Politeness
|
||||||
|
----------
|
||||||
|
We send the per-install operator handle via ``outbound_user_agent``
|
||||||
|
(Round 7a) so USNI can rate-limit / contact the specific install if
|
||||||
|
needed. Article-body pages return 403 to non-browser UAs (Cloudflare),
|
||||||
|
but WordPress RSS feeds are open and serve the full article in
|
||||||
|
``<content:encoded>`` — that's the supported path for aggregators and
|
||||||
|
the one we use. We do not spoof browser headers.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from email.utils import parsedate_to_datetime
|
||||||
|
from typing import Iterable
|
||||||
|
|
||||||
|
from services.network_utils import fetch_with_curl, outbound_user_agent
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_RSS_URLS: tuple[str, ...] = (
|
||||||
|
# Site-wide feed often has the freshest posts before the category
|
||||||
|
# feed catches up. We try this first.
|
||||||
|
"https://news.usni.org/feed",
|
||||||
|
# Category feed has older fleet trackers for backfill.
|
||||||
|
"https://news.usni.org/category/fleet-tracker/feed",
|
||||||
|
)
|
||||||
|
|
||||||
|
_RSS_NS = {"content": "http://purl.org/rss/1.0/modules/content/"}
|
||||||
|
|
||||||
|
_FLEET_TRACKER_TITLE_RE = re.compile(
|
||||||
|
r"fleet\s+and\s+marine\s+tracker", re.IGNORECASE
|
||||||
|
)
|
||||||
|
|
||||||
|
_TAG_STRIP_RE = re.compile(r"<[^>]+>")
|
||||||
|
_WHITESPACE_RE = re.compile(r"\s+")
|
||||||
|
|
||||||
|
|
||||||
|
def _strip_html(html: str) -> str:
|
||||||
|
text = _TAG_STRIP_RE.sub(" ", html or "")
|
||||||
|
return _WHITESPACE_RE.sub(" ", text).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _request_headers() -> dict[str, str]:
|
||||||
|
"""Headers USNI's WordPress feed accepts from a legitimate aggregator.
|
||||||
|
|
||||||
|
The ``Referer`` is the category index page — that's where a real
|
||||||
|
feed reader navigates from. ``Accept`` declares RSS preference but
|
||||||
|
falls back to HTML. No browser UA spoofing.
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"User-Agent": outbound_user_agent("usni-fleet-tracker"),
|
||||||
|
"Accept": "application/rss+xml, application/xml;q=0.9, */*;q=0.1",
|
||||||
|
"Accept-Language": "en-US,en;q=0.5",
|
||||||
|
"Referer": "https://news.usni.org/category/fleet-tracker",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_pubdate(raw: str) -> datetime | None:
|
||||||
|
if not raw:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
dt = parsedate_to_datetime(raw)
|
||||||
|
if dt.tzinfo is None:
|
||||||
|
dt = dt.replace(tzinfo=timezone.utc)
|
||||||
|
return dt
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_fleet_tracker_items(rss_urls: Iterable[str]) -> list[dict]:
|
||||||
|
"""Pull every fleet-tracker post visible across the given RSS feeds.
|
||||||
|
|
||||||
|
De-duplicates by article link. Returns a list of dicts:
|
||||||
|
{"title", "link", "pub_date" (datetime), "body" (plain text)}
|
||||||
|
"""
|
||||||
|
items_by_link: dict[str, dict] = {}
|
||||||
|
for url in rss_urls:
|
||||||
|
try:
|
||||||
|
r = fetch_with_curl(url, timeout=15, headers=_request_headers())
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("USNI RSS %s exception: %s", url, exc)
|
||||||
|
continue
|
||||||
|
if not r or r.status_code != 200 or not r.text:
|
||||||
|
logger.debug(
|
||||||
|
"USNI RSS %s returned status=%s body=%d",
|
||||||
|
url,
|
||||||
|
getattr(r, "status_code", "?"),
|
||||||
|
len(getattr(r, "text", "") or ""),
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
root = ET.fromstring(r.text)
|
||||||
|
except ET.ParseError as exc:
|
||||||
|
logger.warning("USNI RSS parse error from %s: %s", url, exc)
|
||||||
|
continue
|
||||||
|
for item in root.findall(".//item"):
|
||||||
|
title = (item.findtext("title") or "").strip()
|
||||||
|
if not _FLEET_TRACKER_TITLE_RE.search(title):
|
||||||
|
continue
|
||||||
|
link = (item.findtext("link") or "").strip()
|
||||||
|
if not link or link in items_by_link:
|
||||||
|
continue
|
||||||
|
pub_dt = _parse_pubdate(item.findtext("pubDate") or "")
|
||||||
|
body_html = (
|
||||||
|
item.findtext("content:encoded", default="", namespaces=_RSS_NS)
|
||||||
|
or item.findtext("description", default="")
|
||||||
|
or ""
|
||||||
|
)
|
||||||
|
items_by_link[link] = {
|
||||||
|
"title": title,
|
||||||
|
"link": link,
|
||||||
|
"pub_date": pub_dt,
|
||||||
|
"body": _strip_html(body_html),
|
||||||
|
}
|
||||||
|
return list(items_by_link.values())
|
||||||
|
|
||||||
|
|
||||||
|
# Map USNI region phrases to keys in carrier_tracker.REGION_COORDS.
|
||||||
|
# The carrier_tracker table already covers most named bodies of water and
|
||||||
|
# major ports — we just need to teach this module to RECOGNIZE the
|
||||||
|
# specific phrases USNI's editorial style uses, which sometimes spell
|
||||||
|
# the same body of water differently.
|
||||||
|
_USNI_REGION_ALIASES: tuple[tuple[str, str], ...] = (
|
||||||
|
# USNI phrase (lowercase) -> REGION_COORDS key
|
||||||
|
("eastern mediterranean", "eastern mediterranean"),
|
||||||
|
("western mediterranean", "western mediterranean"),
|
||||||
|
("mediterranean sea", "mediterranean"),
|
||||||
|
("the mediterranean", "mediterranean"),
|
||||||
|
("red sea", "red sea"),
|
||||||
|
("arabian sea area of responsibility", "arabian sea"),
|
||||||
|
("north arabian sea", "north arabian sea"),
|
||||||
|
("arabian sea", "arabian sea"),
|
||||||
|
("persian gulf", "persian gulf"),
|
||||||
|
("gulf of oman", "gulf of oman"),
|
||||||
|
("strait of hormuz", "strait of hormuz"),
|
||||||
|
("south china sea", "south china sea"),
|
||||||
|
("east china sea", "east china sea"),
|
||||||
|
("philippine sea", "philippine sea"),
|
||||||
|
("sea of japan", "sea of japan"),
|
||||||
|
("taiwan strait", "taiwan strait"),
|
||||||
|
("western pacific", "western pacific"),
|
||||||
|
("pacific ocean", "pacific"),
|
||||||
|
("indian ocean", "indian ocean"),
|
||||||
|
("north atlantic", "north atlantic"),
|
||||||
|
("western atlantic", "atlantic"),
|
||||||
|
("eastern atlantic", "atlantic"),
|
||||||
|
("atlantic ocean", "atlantic"),
|
||||||
|
("gulf of aden", "gulf of aden"),
|
||||||
|
("horn of africa", "horn of africa"),
|
||||||
|
("bab el-mandeb", "bab el-mandeb"),
|
||||||
|
("suez canal", "suez canal"),
|
||||||
|
("baltic sea", "baltic sea"),
|
||||||
|
("north sea", "north sea"),
|
||||||
|
("black sea", "black sea"),
|
||||||
|
("south atlantic", "south atlantic"),
|
||||||
|
("coral sea", "coral sea"),
|
||||||
|
("gulf of mexico", "gulf of mexico"),
|
||||||
|
("caribbean sea", "caribbean"),
|
||||||
|
("caribbean", "caribbean"),
|
||||||
|
# Specific ports
|
||||||
|
("naval station norfolk", "norfolk"),
|
||||||
|
("norfolk naval shipyard", "newport news"),
|
||||||
|
("newport news shipbuilding", "newport news"),
|
||||||
|
("newport news", "newport news"),
|
||||||
|
# USNI tags Norfolk mentions with state suffix; match both.
|
||||||
|
("norfolk, va", "norfolk"),
|
||||||
|
("norfolk", "norfolk"),
|
||||||
|
("naval station everett", "puget sound"),
|
||||||
|
("naval base kitsap", "bremerton"),
|
||||||
|
("bremerton", "bremerton"),
|
||||||
|
("puget sound", "puget sound"),
|
||||||
|
("naval base san diego", "san diego"),
|
||||||
|
("san diego, calif", "san diego"),
|
||||||
|
("san diego", "san diego"),
|
||||||
|
("yokosuka, japan", "yokosuka"),
|
||||||
|
("yokosuka", "yokosuka"),
|
||||||
|
("pearl harbor", "pearl harbor"),
|
||||||
|
("apra harbor, guam", "guam"),
|
||||||
|
("guam", "guam"),
|
||||||
|
("bahrain", "bahrain"),
|
||||||
|
("naval station rota", "rota"),
|
||||||
|
("rota, spain", "rota"),
|
||||||
|
("naples, italy", "naples"),
|
||||||
|
# Fleets / AORs
|
||||||
|
("5th fleet", "5th fleet"),
|
||||||
|
("6th fleet", "6th fleet"),
|
||||||
|
("7th fleet", "7th fleet"),
|
||||||
|
("3rd fleet", "3rd fleet"),
|
||||||
|
("2nd fleet", "2nd fleet"),
|
||||||
|
("centcom", "centcom"),
|
||||||
|
("indo-pacific command", "indopacom"),
|
||||||
|
("eucom", "eucom"),
|
||||||
|
("southcom", "southcom"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_region_phrase(phrase: str) -> tuple[str, str] | None:
|
||||||
|
"""Map a USNI region phrase to a ``(canonical_key, display)`` tuple,
|
||||||
|
or ``None`` if we don't recognize it.
|
||||||
|
|
||||||
|
``canonical_key`` is what ``carrier_tracker.REGION_COORDS`` keys on.
|
||||||
|
``display`` is the phrase we'll show in the dossier description.
|
||||||
|
"""
|
||||||
|
p = (phrase or "").lower().strip()
|
||||||
|
if not p:
|
||||||
|
return None
|
||||||
|
for usni_phrase, canonical in _USNI_REGION_ALIASES:
|
||||||
|
if usni_phrase in p:
|
||||||
|
return canonical, usni_phrase
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# Operating-verb phrases USNI uses, with a capture group for the region
|
||||||
|
# phrase that immediately follows. Each pattern is designed to swallow
|
||||||
|
# the optional editorial filler that often appears between verb and
|
||||||
|
# location (e.g. "returned Friday to Norfolk" — "Friday" goes in the
|
||||||
|
# filler; "Norfolk" is the location).
|
||||||
|
#
|
||||||
|
# Order matters: most-specific patterns first, so e.g. "is in port in"
|
||||||
|
# wins over the generic "is".
|
||||||
|
_DAY_FILLER = r"(?:[A-Z][a-z]+(?:day)?,?\s+)?" # optional "Friday" / "Monday" / etc.
|
||||||
|
_LOC_CAPTURE = r"([A-Za-z][A-Za-z0-9\s,\.\-']{2,80})"
|
||||||
|
|
||||||
|
_OPERATING_PATTERNS: tuple[re.Pattern, ...] = (
|
||||||
|
# "is operating in [the] {REGION}" / "is also operating in [the] {REGION}"
|
||||||
|
re.compile(r"\bis\s+(?:also\s+|now\s+)?operating\s+in\s+(?:the\s+)?" + _LOC_CAPTURE, re.IGNORECASE),
|
||||||
|
# "is conducting <stuff> in [the] {REGION}"
|
||||||
|
re.compile(r"\bis\s+conducting\s+[A-Za-z0-9\-\s]{2,40}\s+in\s+(?:the\s+)?" + _LOC_CAPTURE, re.IGNORECASE),
|
||||||
|
# "is in port in {LOCATION}"
|
||||||
|
re.compile(r"\bis\s+in\s+port\s+in\s+" + _LOC_CAPTURE, re.IGNORECASE),
|
||||||
|
# "is in port" (no location — degenerate, use carrier's homeport via separate path)
|
||||||
|
# → not captured here; falls through to homeport
|
||||||
|
# "is underway in [the] {REGION}"
|
||||||
|
re.compile(r"\bis\s+underway\s+in\s+(?:the\s+)?" + _LOC_CAPTURE, re.IGNORECASE),
|
||||||
|
# "is deployed to [the] {REGION}" / "deployed in"
|
||||||
|
re.compile(r"\bis\s+deployed\s+(?:to|in)\s+(?:the\s+)?" + _LOC_CAPTURE, re.IGNORECASE),
|
||||||
|
# "returned [Day] to {LOCATION}" / "returned [Day] from {REGION}"
|
||||||
|
re.compile(r"\breturned\s+" + _DAY_FILLER + r"to\s+" + _LOC_CAPTURE, re.IGNORECASE),
|
||||||
|
re.compile(r"\breturned\s+" + _DAY_FILLER + r"from\s+(?:the\s+)?" + _LOC_CAPTURE, re.IGNORECASE),
|
||||||
|
# "arrived [Day] in/at {LOCATION}"
|
||||||
|
re.compile(r"\barrived\s+" + _DAY_FILLER + r"(?:in|at)\s+" + _LOC_CAPTURE, re.IGNORECASE),
|
||||||
|
# "departed [Day] from {LOCATION}"
|
||||||
|
re.compile(r"\bdeparted\s+" + _DAY_FILLER + r"(?:from\s+)?" + _LOC_CAPTURE, re.IGNORECASE),
|
||||||
|
# "transiting [the] {REGION}" / "sailing through [the] {REGION}"
|
||||||
|
re.compile(r"\btransiting\s+(?:the\s+)?" + _LOC_CAPTURE, re.IGNORECASE),
|
||||||
|
re.compile(r"\bsailing\s+through\s+(?:the\s+)?" + _LOC_CAPTURE, re.IGNORECASE),
|
||||||
|
# "is homeported at {LOCATION}"
|
||||||
|
re.compile(r"\bis\s+homeported\s+at\s+" + _LOC_CAPTURE, re.IGNORECASE),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_region_for_carrier(
|
||||||
|
body: str,
|
||||||
|
carrier_names: list[str],
|
||||||
|
hull_code: str,
|
||||||
|
) -> str | None:
|
||||||
|
"""Return the best-guess region phrase for one carrier from the
|
||||||
|
article body, or None if no confident match.
|
||||||
|
|
||||||
|
Algorithm:
|
||||||
|
1. Find every mention of the carrier (any name variant or the hull
|
||||||
|
code) in the body.
|
||||||
|
2. For each mention, look in the ~300-char window AFTER it for any
|
||||||
|
of the operating-verb patterns.
|
||||||
|
3. Return the first hit. If a more-confident match later turns up
|
||||||
|
(e.g. "is operating in the X" beats "is homeported at Y"), the
|
||||||
|
first one in document order still wins — USNI's structure puts
|
||||||
|
the position-update sentence near the top of each carrier's
|
||||||
|
section, and the homeport mention later.
|
||||||
|
"""
|
||||||
|
# Build a master mention regex covering every name variant + the hull.
|
||||||
|
candidates: list[str] = []
|
||||||
|
for name in carrier_names:
|
||||||
|
if name and len(name) >= 4:
|
||||||
|
candidates.append(re.escape(name))
|
||||||
|
if hull_code:
|
||||||
|
candidates.append(re.escape(hull_code))
|
||||||
|
if not candidates:
|
||||||
|
return None
|
||||||
|
mention_re = re.compile(r"\b(?:" + "|".join(candidates) + r")\b", re.IGNORECASE)
|
||||||
|
|
||||||
|
window_chars = 320
|
||||||
|
seen_phrases: list[str] = []
|
||||||
|
for mention in mention_re.finditer(body):
|
||||||
|
end = mention.end()
|
||||||
|
window = body[end : end + window_chars]
|
||||||
|
# Cut window at the next sentence break for tighter context.
|
||||||
|
# (We use the LAST period within the window so "Norfolk, Va." isn't
|
||||||
|
# confused for a sentence end — USNI uses ", Va." prolifically.)
|
||||||
|
# Sentence break candidates: ". " followed by uppercase OR newline.
|
||||||
|
sent_break = re.search(r"[\.!?]\s+[A-Z]", window)
|
||||||
|
if sent_break:
|
||||||
|
window = window[: sent_break.start() + 1]
|
||||||
|
# Try patterns in priority order.
|
||||||
|
for pat in _OPERATING_PATTERNS:
|
||||||
|
m = pat.search(window)
|
||||||
|
if not m:
|
||||||
|
continue
|
||||||
|
phrase = m.group(1).strip().rstrip(",.;: ")
|
||||||
|
if not phrase:
|
||||||
|
continue
|
||||||
|
# Strip trailing editorial filler — USNI often writes
|
||||||
|
# "Norfolk, Va., according to ship spotters" or
|
||||||
|
# "Yokosuka, Japan, according to..."
|
||||||
|
phrase = re.split(
|
||||||
|
r",\s+(?:according|as of|for|while|where|in support|in the)",
|
||||||
|
phrase,
|
||||||
|
maxsplit=1,
|
||||||
|
)[0].strip()
|
||||||
|
seen_phrases.append(phrase)
|
||||||
|
return phrase
|
||||||
|
return seen_phrases[0] if seen_phrases else None
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_latest_fleet_tracker_positions(
|
||||||
|
carrier_registry: dict | None = None,
|
||||||
|
region_coords: dict | None = None,
|
||||||
|
) -> dict[str, dict]:
|
||||||
|
"""Return ``{hull: position_entry}`` for the latest USNI fleet tracker.
|
||||||
|
|
||||||
|
Entries look like::
|
||||||
|
|
||||||
|
{
|
||||||
|
"lat": 18.0, "lng": 39.5, "heading": 0,
|
||||||
|
"desc": "Red Sea (USNI May 18, 2026)",
|
||||||
|
"source": "USNI News Fleet & Marine Tracker (May 18, 2026)",
|
||||||
|
"source_url": "https://news.usni.org/2026/05/18/...",
|
||||||
|
"position_source_at": "2026-05-18T18:58:44+00:00",
|
||||||
|
"position_confidence": "recent",
|
||||||
|
}
|
||||||
|
|
||||||
|
Carriers whose section can't be parsed (e.g. an off-week with no
|
||||||
|
mention) are simply absent from the result — the caller keeps
|
||||||
|
whatever position they had before.
|
||||||
|
|
||||||
|
``carrier_registry`` and ``region_coords`` default to the carrier_tracker
|
||||||
|
module's own tables; passed in here for testability.
|
||||||
|
"""
|
||||||
|
if carrier_registry is None or region_coords is None:
|
||||||
|
from services.carrier_tracker import CARRIER_REGISTRY, REGION_COORDS
|
||||||
|
carrier_registry = carrier_registry or CARRIER_REGISTRY
|
||||||
|
region_coords = region_coords or REGION_COORDS
|
||||||
|
|
||||||
|
items = _iter_fleet_tracker_items(_RSS_URLS)
|
||||||
|
if not items:
|
||||||
|
logger.warning("USNI fleet-tracker: no parseable RSS items")
|
||||||
|
return {}
|
||||||
|
|
||||||
|
# Pick the most recent by parsed pubDate. Items without a parseable
|
||||||
|
# date fall to the back of the list.
|
||||||
|
items.sort(
|
||||||
|
key=lambda it: it["pub_date"] or datetime(1970, 1, 1, tzinfo=timezone.utc),
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
latest = items[0]
|
||||||
|
|
||||||
|
pub_dt: datetime | None = latest["pub_date"]
|
||||||
|
pub_iso = pub_dt.isoformat() if pub_dt else ""
|
||||||
|
pub_human = pub_dt.strftime("%b %d, %Y") if pub_dt else "unknown date"
|
||||||
|
|
||||||
|
body = latest["body"]
|
||||||
|
if not body:
|
||||||
|
logger.warning("USNI fleet-tracker: latest item has empty body")
|
||||||
|
return {}
|
||||||
|
|
||||||
|
positions: dict[str, dict] = {}
|
||||||
|
for hull, info in carrier_registry.items():
|
||||||
|
# Build name variants we'll try in the body.
|
||||||
|
full_name = info["name"] # "USS Gerald R. Ford (CVN-78)"
|
||||||
|
without_hull = full_name.split("(")[0].strip() # "USS Gerald R. Ford"
|
||||||
|
last_word = without_hull.split()[-1] # "Ford"
|
||||||
|
ship_only = without_hull[4:] # "Gerald R. Ford"
|
||||||
|
|
||||||
|
# Variants ordered most-specific first.
|
||||||
|
variants: list[str] = []
|
||||||
|
for v in (without_hull, f"USS {ship_only}", ship_only, last_word):
|
||||||
|
if v and v not in variants and len(v) >= 4:
|
||||||
|
variants.append(v)
|
||||||
|
|
||||||
|
phrase = _extract_region_for_carrier(body, variants, hull)
|
||||||
|
if not phrase:
|
||||||
|
continue
|
||||||
|
resolved = _resolve_region_phrase(phrase)
|
||||||
|
if not resolved:
|
||||||
|
logger.debug(
|
||||||
|
"USNI: %s region phrase %r did not match any known region",
|
||||||
|
hull, phrase,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
canonical_key, display_phrase = resolved
|
||||||
|
coords = region_coords.get(canonical_key)
|
||||||
|
if not coords:
|
||||||
|
continue
|
||||||
|
|
||||||
|
positions[hull] = {
|
||||||
|
"lat": coords[0],
|
||||||
|
"lng": coords[1],
|
||||||
|
"heading": 0,
|
||||||
|
"desc": f"{display_phrase.title()} (USNI {pub_human})",
|
||||||
|
"source": f"USNI News Fleet & Marine Tracker ({pub_human})",
|
||||||
|
"source_url": latest["link"],
|
||||||
|
"position_source_at": pub_iso,
|
||||||
|
"position_confidence": "recent",
|
||||||
|
}
|
||||||
|
|
||||||
|
if positions:
|
||||||
|
logger.info(
|
||||||
|
"USNI fleet-tracker: parsed %d/%d carrier positions from %s",
|
||||||
|
len(positions), len(carrier_registry), latest["link"],
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
"USNI fleet-tracker: latest article %s yielded zero parseable carriers",
|
||||||
|
latest["link"],
|
||||||
|
)
|
||||||
|
return positions
|
||||||
@@ -21,9 +21,17 @@ _cache_lock = threading.Lock()
|
|||||||
_local_search_cache: List[Dict[str, Any]] | None = None
|
_local_search_cache: List[Dict[str, Any]] | None = None
|
||||||
_local_search_lock = threading.Lock()
|
_local_search_lock = threading.Lock()
|
||||||
|
|
||||||
_USER_AGENT = os.environ.get(
|
# Round 7a: per-install operator handle threads through every Nominatim
|
||||||
"NOMINATIM_USER_AGENT", "ShadowBroker/1.0 (https://github.com/BigBodyCobain/Shadowbroker)"
|
# call. NOMINATIM_USER_AGENT env override is still honored for operators
|
||||||
)
|
# who run a custom relay / known good identity, but the default uses the
|
||||||
|
# per-install handle so OpenStreetMap can rate-limit per install instead
|
||||||
|
# of treating "Shadowbroker" as one big offender.
|
||||||
|
def _nominatim_user_agent() -> str:
|
||||||
|
override = os.environ.get("NOMINATIM_USER_AGENT", "").strip()
|
||||||
|
if override:
|
||||||
|
return override
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
|
return outbound_user_agent("nominatim")
|
||||||
|
|
||||||
|
|
||||||
def _get_cache(key: str):
|
def _get_cache(key: str):
|
||||||
@@ -178,7 +186,7 @@ def search_geocode(query: str, limit: int = 5, local_only: bool = False) -> List
|
|||||||
res = fetch_with_curl(
|
res = fetch_with_curl(
|
||||||
url,
|
url,
|
||||||
headers={
|
headers={
|
||||||
"User-Agent": _USER_AGENT,
|
"User-Agent": _nominatim_user_agent(),
|
||||||
"Accept-Language": "en",
|
"Accept-Language": "en",
|
||||||
},
|
},
|
||||||
timeout=6,
|
timeout=6,
|
||||||
@@ -241,7 +249,7 @@ def reverse_geocode(lat: float, lng: float, local_only: bool = False) -> Dict[st
|
|||||||
res = fetch_with_curl(
|
res = fetch_with_curl(
|
||||||
url,
|
url,
|
||||||
headers={
|
headers={
|
||||||
"User-Agent": _USER_AGENT,
|
"User-Agent": _nominatim_user_agent(),
|
||||||
"Accept-Language": "en",
|
"Accept-Language": "en",
|
||||||
},
|
},
|
||||||
timeout=6,
|
timeout=6,
|
||||||
|
|||||||
@@ -8,6 +8,13 @@ from datetime import datetime
|
|||||||
from urllib.parse import urljoin, urlparse
|
from urllib.parse import urljoin, urlparse
|
||||||
from services.network_utils import fetch_with_curl
|
from services.network_utils import fetch_with_curl
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def _geopolitics_user_agent() -> str:
|
||||||
|
"""Round 7a: GDELT geopolitics fetcher attribution."""
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
|
return outbound_user_agent("geopolitics-gdelt")
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Cache Frontline data for 30 minutes, it doesn't move that fast
|
# Cache Frontline data for 30 minutes, it doesn't move that fast
|
||||||
@@ -316,7 +323,7 @@ def _fetch_article_title(url):
|
|||||||
resp = requests.get(
|
resp = requests.get(
|
||||||
current_url,
|
current_url,
|
||||||
timeout=4,
|
timeout=4,
|
||||||
headers={"User-Agent": "Mozilla/5.0 (compatible; OSINT Dashboard/1.0)"},
|
headers={"User-Agent": _geopolitics_user_agent()},
|
||||||
stream=True,
|
stream=True,
|
||||||
allow_redirects=False,
|
allow_redirects=False,
|
||||||
)
|
)
|
||||||
@@ -521,10 +528,29 @@ def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_
|
|||||||
logger.warning(f"Failed to parse GDELT export zip: {e}")
|
logger.warning(f"Failed to parse GDELT export zip: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
# GDELT's data.gdeltproject.org is a CNAME to a Google Cloud Storage
|
||||||
|
# bucket of the same name. GCS returns the wildcard ``*.storage.googleapis.com``
|
||||||
|
# certificate, which legitimately does NOT cover the GDELT custom domain
|
||||||
|
# — Python's TLS verification correctly refuses it. Some networks/POPs
|
||||||
|
# happen to route through a path where this works; many do not (notably
|
||||||
|
# Docker Desktop's outbound NAT on local installs).
|
||||||
|
#
|
||||||
|
# Fix: rewrite the URL to hit GCS directly with a path-style bucket
|
||||||
|
# reference, where the standard GCS cert is genuinely valid. Same data,
|
||||||
|
# verified TLS, no operator-side workaround needed.
|
||||||
|
def _gcs_direct_gdelt_url(url: str) -> str:
|
||||||
|
"""If ``url`` points at data.gdeltproject.org, return the equivalent
|
||||||
|
GCS-direct URL. Otherwise return the URL unchanged."""
|
||||||
|
prefix = "://data.gdeltproject.org/"
|
||||||
|
if prefix in url:
|
||||||
|
return url.replace(prefix, "://storage.googleapis.com/data.gdeltproject.org/", 1)
|
||||||
|
return url
|
||||||
|
|
||||||
|
|
||||||
def _download_gdelt_export(url):
|
def _download_gdelt_export(url):
|
||||||
"""Download a single GDELT export file, return bytes or None."""
|
"""Download a single GDELT export file, return bytes or None."""
|
||||||
try:
|
try:
|
||||||
res = fetch_with_curl(url, timeout=15)
|
res = fetch_with_curl(_gcs_direct_gdelt_url(url), timeout=15)
|
||||||
if res.status_code == 200:
|
if res.status_code == 200:
|
||||||
return res.content
|
return res.content
|
||||||
except (ConnectionError, TimeoutError, OSError): # non-critical
|
except (ConnectionError, TimeoutError, OSError): # non-critical
|
||||||
@@ -616,9 +642,16 @@ def fetch_global_military_incidents():
|
|||||||
try:
|
try:
|
||||||
logger.info("Fetching GDELT events via export CDN (multi-file)...")
|
logger.info("Fetching GDELT events via export CDN (multi-file)...")
|
||||||
|
|
||||||
# Get the latest export URL to determine current timestamp
|
# Get the latest export URL to determine current timestamp.
|
||||||
|
# HTTPS is used to prevent passive network observers from injecting
|
||||||
|
# poisoned export records into the global incident map via MITM.
|
||||||
|
# GDELT serves the same content over HTTPS as HTTP.
|
||||||
|
# Use the GCS-direct URL because data.gdeltproject.org's CNAME
|
||||||
|
# serves a wildcard *.storage.googleapis.com cert that legitimately
|
||||||
|
# doesn't cover the GDELT hostname. See _gcs_direct_gdelt_url above.
|
||||||
index_res = fetch_with_curl(
|
index_res = fetch_with_curl(
|
||||||
"http://data.gdeltproject.org/gdeltv2/lastupdate.txt", timeout=10
|
_gcs_direct_gdelt_url("https://data.gdeltproject.org/gdeltv2/lastupdate.txt"),
|
||||||
|
timeout=10,
|
||||||
)
|
)
|
||||||
if index_res.status_code != 200:
|
if index_res.status_code != 200:
|
||||||
logger.error(f"GDELT lastupdate failed: {index_res.status_code}")
|
logger.error(f"GDELT lastupdate failed: {index_res.status_code}")
|
||||||
@@ -636,7 +669,9 @@ def fetch_global_military_incidents():
|
|||||||
logger.error("Could not find GDELT export URL")
|
logger.error("Could not find GDELT export URL")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
# Extract timestamp from URL like: http://data.gdeltproject.org/gdeltv2/20260301120000.export.CSV.zip
|
# Extract timestamp from URL like: https://data.gdeltproject.org/gdeltv2/20260301120000.export.CSV.zip
|
||||||
|
# (GDELT's lastupdate.txt may still list URLs with http:// — we ignore
|
||||||
|
# the scheme there and reconstruct each download URL as https:// below.)
|
||||||
import re
|
import re
|
||||||
|
|
||||||
ts_match = re.search(r"(\d{14})\.export\.CSV\.zip", latest_url)
|
ts_match = re.search(r"(\d{14})\.export\.CSV\.zip", latest_url)
|
||||||
@@ -652,7 +687,7 @@ def fetch_global_military_incidents():
|
|||||||
for i in range(NUM_FILES):
|
for i in range(NUM_FILES):
|
||||||
ts = latest_ts - timedelta(minutes=15 * i)
|
ts = latest_ts - timedelta(minutes=15 * i)
|
||||||
fname = ts.strftime("%Y%m%d%H%M%S") + ".export.CSV.zip"
|
fname = ts.strftime("%Y%m%d%H%M%S") + ".export.CSV.zip"
|
||||||
url = f"http://data.gdeltproject.org/gdeltv2/{fname}"
|
url = f"https://data.gdeltproject.org/gdeltv2/{fname}"
|
||||||
urls.append(url)
|
urls.append(url)
|
||||||
|
|
||||||
logger.info(f"Downloading {len(urls)} GDELT export files...")
|
logger.info(f"Downloading {len(urls)} GDELT export files...")
|
||||||
|
|||||||
@@ -34,6 +34,20 @@ kiwisdr_cache: TTLCache = TTLCache(maxsize=1, ttl=_REFRESH_SECONDS)
|
|||||||
|
|
||||||
_SOURCE_URL = "http://rx.linkfanel.net/kiwisdr_com.js"
|
_SOURCE_URL = "http://rx.linkfanel.net/kiwisdr_com.js"
|
||||||
_CACHE_FILE = Path(__file__).resolve().parent.parent / "data" / "kiwisdr_cache.json"
|
_CACHE_FILE = Path(__file__).resolve().parent.parent / "data" / "kiwisdr_cache.json"
|
||||||
|
# Bundled fallback — shipped with the codebase so the KiwiSDR layer always
|
||||||
|
# has something to render even when the upstream is unreachable, returns
|
||||||
|
# garbage, or appears to have been tampered with. Issue #206: the upstream
|
||||||
|
# only speaks HTTP, so we can't rely on TLS for integrity — instead we
|
||||||
|
# validate the response's shape and fall back to this bundle if it doesn't
|
||||||
|
# look right.
|
||||||
|
_BUNDLED_FALLBACK = Path(__file__).resolve().parent.parent / "data" / "kiwisdr_directory.json"
|
||||||
|
|
||||||
|
# Minimum number of receivers we expect from a healthy upstream response.
|
||||||
|
# The KiwiSDR public network has consistently sat well above this threshold
|
||||||
|
# for years. If we see fewer than this many parsed receivers, treat the
|
||||||
|
# response as suspect and fall back. Tune via env if the upstream shrinks
|
||||||
|
# legitimately.
|
||||||
|
_MIN_HEALTHY_RECEIVER_COUNT = 50
|
||||||
_LINE_COMMENT_RE = re.compile(r"^\s*//.*$", re.MULTILINE)
|
_LINE_COMMENT_RE = re.compile(r"^\s*//.*$", re.MULTILINE)
|
||||||
_VAR_PREFIX_RE = re.compile(r"^\s*var\s+kiwisdr_com\s*=\s*", re.MULTILINE)
|
_VAR_PREFIX_RE = re.compile(r"^\s*var\s+kiwisdr_com\s*=\s*", re.MULTILINE)
|
||||||
_TRAILING_COMMA_RE = re.compile(r",(\s*[\]}])")
|
_TRAILING_COMMA_RE = re.compile(r",(\s*[\]}])")
|
||||||
@@ -135,12 +149,72 @@ def _parse_mirror_payload(body: str) -> list[dict]:
|
|||||||
return nodes
|
return nodes
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_fetched_nodes(nodes: list[dict]) -> bool:
|
||||||
|
"""Sanity-check freshly-fetched receiver data before trusting it.
|
||||||
|
|
||||||
|
The upstream (rx.linkfanel.net) speaks only HTTP — there is no TLS to
|
||||||
|
authenticate the response. A passive MITM could inject doctored
|
||||||
|
receiver positions (false pins on the map) or strip the response down
|
||||||
|
to a tiny subset. We can't prevent the modification at the transport
|
||||||
|
layer, but we can refuse to commit to obviously-bad responses.
|
||||||
|
|
||||||
|
Returns True if the parsed list looks reasonable. False means we
|
||||||
|
should fall back to a previously-cached or bundled directory.
|
||||||
|
"""
|
||||||
|
if not isinstance(nodes, list):
|
||||||
|
return False
|
||||||
|
if len(nodes) < _MIN_HEALTHY_RECEIVER_COUNT:
|
||||||
|
# Either upstream is degraded or someone is feeding us a stripped
|
||||||
|
# response. Either way, the bundled fallback is more useful.
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Spot-check: every entry should have a name, a parsed lat/lon, and a
|
||||||
|
# URL field. If more than 5% of entries are missing core fields, the
|
||||||
|
# parse went sideways.
|
||||||
|
missing_core = 0
|
||||||
|
for entry in nodes:
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
missing_core += 1
|
||||||
|
continue
|
||||||
|
if not entry.get("name") or not isinstance(entry.get("lat"), (int, float)):
|
||||||
|
missing_core += 1
|
||||||
|
if missing_core > max(5, len(nodes) // 20):
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _load_bundled_fallback() -> list[dict]:
|
||||||
|
"""Last-resort directory shipped with the codebase. Always returns a
|
||||||
|
list (may be empty if the bundle is missing in older deployments)."""
|
||||||
|
if not _BUNDLED_FALLBACK.exists():
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
data = json.loads(_BUNDLED_FALLBACK.read_text(encoding="utf-8"))
|
||||||
|
if isinstance(data, list):
|
||||||
|
return data
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"KiwiSDR bundled fallback unreadable: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
@cached(kiwisdr_cache)
|
@cached(kiwisdr_cache)
|
||||||
def fetch_kiwisdr_nodes() -> list[dict]:
|
def fetch_kiwisdr_nodes() -> list[dict]:
|
||||||
"""Return the KiwiSDR receiver list, refreshed at most once per day.
|
"""Return the KiwiSDR receiver list, refreshed at most once per day.
|
||||||
|
|
||||||
Order of preference: in-memory cache (handled by @cached) → on-disk cache
|
Layered fallback (issue #206 — upstream is HTTP-only, so we defend with
|
||||||
if <24h old → network fetch from rx.linkfanel.net.
|
content validation + bundled static directory rather than trying to
|
||||||
|
upgrade the transport):
|
||||||
|
|
||||||
|
1. In-memory cache (handled by @cached on this function)
|
||||||
|
2. On-disk cache if <24h old
|
||||||
|
3. Fresh network fetch from rx.linkfanel.net → validated → committed
|
||||||
|
4. Stale on-disk cache (>24h) if validation fails
|
||||||
|
5. Bundled static directory at backend/data/kiwisdr_directory.json
|
||||||
|
|
||||||
|
The KiwiSDR map layer renders something useful in every case. A
|
||||||
|
tampered upstream returning garbage is caught by _validate_fetched_nodes()
|
||||||
|
and falls through to whatever previously-trusted snapshot we have.
|
||||||
"""
|
"""
|
||||||
from services.network_utils import fetch_with_curl
|
from services.network_utils import fetch_with_curl
|
||||||
|
|
||||||
@@ -153,34 +227,57 @@ def fetch_kiwisdr_nodes() -> list[dict]:
|
|||||||
return cached_nodes
|
return cached_nodes
|
||||||
|
|
||||||
# 2. Cache cold or stale — fetch from network.
|
# 2. Cache cold or stale — fetch from network.
|
||||||
|
fresh_nodes: list[dict] = []
|
||||||
|
fetch_succeeded = False
|
||||||
try:
|
try:
|
||||||
res = fetch_with_curl(_SOURCE_URL, timeout=20)
|
res = fetch_with_curl(_SOURCE_URL, timeout=20)
|
||||||
if not res or res.status_code != 200:
|
if res and res.status_code == 200:
|
||||||
logger.error(
|
fresh_nodes = _parse_mirror_payload(res.text)
|
||||||
f"KiwiSDR fetch failed: HTTP {res.status_code if res else 'no response'}"
|
fetch_succeeded = True
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
f"KiwiSDR fetch returned HTTP {res.status_code if res else 'no response'}"
|
||||||
)
|
)
|
||||||
return []
|
|
||||||
|
|
||||||
nodes = _parse_mirror_payload(res.text)
|
|
||||||
if nodes:
|
|
||||||
_save_disk_cache(nodes)
|
|
||||||
logger.info(
|
|
||||||
f"KiwiSDR: refreshed {len(nodes)} receivers from rx.linkfanel.net "
|
|
||||||
"(next refresh in 24h)"
|
|
||||||
)
|
|
||||||
return nodes
|
|
||||||
|
|
||||||
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
|
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
|
||||||
logger.error(f"KiwiSDR fetch exception: {e}")
|
logger.warning(f"KiwiSDR fetch exception: {e}")
|
||||||
# Fall back to a stale disk cache if one exists, even if >24h old.
|
|
||||||
if _CACHE_FILE.exists():
|
# 3. Validate before committing. If the response looks healthy, save
|
||||||
try:
|
# it as the new cache and return.
|
||||||
stale = json.loads(_CACHE_FILE.read_text(encoding="utf-8"))
|
if fetch_succeeded and _validate_fetched_nodes(fresh_nodes):
|
||||||
if isinstance(stale, list):
|
_save_disk_cache(fresh_nodes)
|
||||||
logger.info(
|
logger.info(
|
||||||
f"KiwiSDR: serving {len(stale)} stale receivers from disk after fetch failure"
|
f"KiwiSDR: refreshed {len(fresh_nodes)} receivers from rx.linkfanel.net "
|
||||||
)
|
"(next refresh in 24h)"
|
||||||
return stale
|
)
|
||||||
except Exception:
|
return fresh_nodes
|
||||||
pass
|
|
||||||
return []
|
if fetch_succeeded:
|
||||||
|
# Network came back, but the payload didn't pass validation —
|
||||||
|
# either upstream is degraded or a MITM is at work. Fall through
|
||||||
|
# to a trusted snapshot rather than committing garbage to disk.
|
||||||
|
logger.warning(
|
||||||
|
"KiwiSDR: upstream response failed validation (%d entries) — "
|
||||||
|
"falling back to trusted snapshot",
|
||||||
|
len(fresh_nodes),
|
||||||
|
)
|
||||||
|
|
||||||
|
# 4. Stale on-disk cache, if any.
|
||||||
|
if _CACHE_FILE.exists():
|
||||||
|
try:
|
||||||
|
stale = json.loads(_CACHE_FILE.read_text(encoding="utf-8"))
|
||||||
|
if isinstance(stale, list) and stale:
|
||||||
|
logger.info(
|
||||||
|
f"KiwiSDR: serving {len(stale)} stale receivers from disk"
|
||||||
|
)
|
||||||
|
return stale
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# 5. Bundled static directory — last resort, always works.
|
||||||
|
bundled = _load_bundled_fallback()
|
||||||
|
if bundled:
|
||||||
|
logger.info(
|
||||||
|
f"KiwiSDR: serving {len(bundled)} receivers from bundled fallback "
|
||||||
|
"(no fresh fetch + no disk cache available)"
|
||||||
|
)
|
||||||
|
return bundled
|
||||||
|
|||||||
@@ -69,6 +69,115 @@ def _derive_peer_key(shared_secret: str, peer_url: str) -> bytes:
|
|||||||
).digest()
|
).digest()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Issue #256 (tg12): per-peer HMAC secrets
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
#
|
||||||
|
# Before this change, ALL peer-push HMACs were derived from a single
|
||||||
|
# fleet-shared ``MESH_PEER_PUSH_SECRET``. The receiver could prove a
|
||||||
|
# request was signed by *someone who knows the fleet secret*, but it
|
||||||
|
# could NOT prove which peer signed it — any peer could compute the
|
||||||
|
# expected HMAC for any other peer's URL and impersonate that peer.
|
||||||
|
#
|
||||||
|
# Fix: an optional ``MESH_PEER_SECRETS`` env var maps specific peer URLs
|
||||||
|
# to per-peer secrets. When a peer URL is listed there, only that
|
||||||
|
# per-peer secret is accepted for that URL — the global secret is
|
||||||
|
# ignored for that peer. Peer A no longer learns peer B's secret, so
|
||||||
|
# peer A cannot forge a request claiming to be peer B.
|
||||||
|
#
|
||||||
|
# Backwards-compatible by design:
|
||||||
|
#
|
||||||
|
# - Single-peer installs (``MESH_PEER_SECRETS`` empty) keep using the
|
||||||
|
# global secret. Zero behavior change. Zero operator action required.
|
||||||
|
# - Multi-peer installs that haven't migrated yet keep using the global
|
||||||
|
# secret for every peer. Same behavior as before — same exposure.
|
||||||
|
# - Multi-peer installs that have migrated configure
|
||||||
|
# ``MESH_PEER_SECRETS=urlA=secretA,urlB=secretB`` and immediately get
|
||||||
|
# per-peer identity. Migration is incremental: peers not yet listed
|
||||||
|
# continue using the global secret until both sides of that peering
|
||||||
|
# add their entry.
|
||||||
|
|
||||||
|
_PEER_SECRETS_CACHE: dict[str, str] = {}
|
||||||
|
_PEER_SECRETS_CACHE_RAW: str = ""
|
||||||
|
|
||||||
|
|
||||||
|
def _lookup_per_peer_secret(normalized_url: str) -> str:
|
||||||
|
"""Return the per-peer secret for ``normalized_url`` from MESH_PEER_SECRETS.
|
||||||
|
|
||||||
|
Returns "" if no per-peer entry is configured for that URL. The parser
|
||||||
|
is forgiving:
|
||||||
|
|
||||||
|
- Whitespace around items, URLs, and secrets is stripped.
|
||||||
|
- Items without ``=`` or with empty URL/secret halves are skipped.
|
||||||
|
- The URL half is normalized via ``normalize_peer_url`` so config
|
||||||
|
authors don't have to match scheme/port/path quirks exactly.
|
||||||
|
|
||||||
|
The cache is invalidated whenever the env var's raw value changes,
|
||||||
|
which keeps tests' ``monkeypatch.setenv`` calls effective without
|
||||||
|
forcing a process restart.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
|
||||||
|
raw = str(os.environ.get("MESH_PEER_SECRETS", "") or "").strip()
|
||||||
|
|
||||||
|
global _PEER_SECRETS_CACHE, _PEER_SECRETS_CACHE_RAW
|
||||||
|
if raw != _PEER_SECRETS_CACHE_RAW:
|
||||||
|
new_cache: dict[str, str] = {}
|
||||||
|
for chunk in raw.split(","):
|
||||||
|
chunk = chunk.strip()
|
||||||
|
if not chunk or "=" not in chunk:
|
||||||
|
continue
|
||||||
|
url_part, _, secret_part = chunk.partition("=")
|
||||||
|
normalized = normalize_peer_url(url_part.strip())
|
||||||
|
secret = secret_part.strip()
|
||||||
|
if normalized and secret:
|
||||||
|
new_cache[normalized] = secret
|
||||||
|
_PEER_SECRETS_CACHE = new_cache
|
||||||
|
_PEER_SECRETS_CACHE_RAW = raw
|
||||||
|
|
||||||
|
return _PEER_SECRETS_CACHE.get(normalized_url, "")
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_peer_key_for_url(peer_url: str) -> bytes:
|
||||||
|
"""Return the HMAC key for ``peer_url``, preferring per-peer secret.
|
||||||
|
|
||||||
|
Issue #256: this is the function every peer-push call site should
|
||||||
|
use. It looks up the peer-specific secret first, falling back to the
|
||||||
|
fleet-shared ``MESH_PEER_PUSH_SECRET`` only when the URL is NOT
|
||||||
|
listed in ``MESH_PEER_SECRETS``.
|
||||||
|
|
||||||
|
Both sender (computing X-Peer-HMAC) and receiver (verifying it) call
|
||||||
|
this with the SENDER's URL — they must derive the same key, so
|
||||||
|
operators on both ends of a peering need matching MESH_PEER_SECRETS
|
||||||
|
entries for that URL to stay in sync.
|
||||||
|
|
||||||
|
Returns empty bytes when no usable secret exists. Callers must treat
|
||||||
|
that as fail-closed (skip the push, reject the verification).
|
||||||
|
"""
|
||||||
|
normalized_url = normalize_peer_url(peer_url)
|
||||||
|
if not normalized_url:
|
||||||
|
return b""
|
||||||
|
|
||||||
|
per_peer_secret = _lookup_per_peer_secret(normalized_url)
|
||||||
|
if per_peer_secret:
|
||||||
|
return _derive_peer_key(per_peer_secret, normalized_url)
|
||||||
|
|
||||||
|
# No per-peer entry for this URL — fall back to the legacy global
|
||||||
|
# secret. This is what preserves zero-hostility for single-peer
|
||||||
|
# installs and the migration window for multi-peer installs.
|
||||||
|
try:
|
||||||
|
from services.config import get_settings
|
||||||
|
|
||||||
|
global_secret = str(
|
||||||
|
getattr(get_settings(), "MESH_PEER_PUSH_SECRET", "") or ""
|
||||||
|
).strip()
|
||||||
|
except Exception:
|
||||||
|
return b""
|
||||||
|
if not global_secret:
|
||||||
|
return b""
|
||||||
|
return _derive_peer_key(global_secret, normalized_url)
|
||||||
|
|
||||||
|
|
||||||
def _node_digest(public_key_b64: str) -> str:
|
def _node_digest(public_key_b64: str) -> str:
|
||||||
raw = base64.b64decode(public_key_b64)
|
raw = base64.b64decode(public_key_b64)
|
||||||
return hashlib.sha256(raw).hexdigest()
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|||||||
@@ -317,6 +317,39 @@ class DMRelay:
|
|||||||
def _self_mailbox_limit(self) -> int:
|
def _self_mailbox_limit(self) -> int:
|
||||||
return max(1, int(self._settings().MESH_DM_SELF_MAILBOX_LIMIT))
|
return max(1, int(self._settings().MESH_DM_SELF_MAILBOX_LIMIT))
|
||||||
|
|
||||||
|
def _per_sender_pending_limit(self) -> int:
|
||||||
|
"""Anti-spam cap on UNACKED messages a single sender can have parked
|
||||||
|
in a single recipient mailbox at any one time. See ``config.py``
|
||||||
|
``MESH_DM_PENDING_PER_SENDER_LIMIT`` for the threat model — this
|
||||||
|
rule is enforced both at ``deposit`` (local) and at
|
||||||
|
``accept_replica`` (peer push acceptance), making it a network
|
||||||
|
rule rather than a client-side honor system."""
|
||||||
|
try:
|
||||||
|
limit = int(getattr(self._settings(), "MESH_DM_PENDING_PER_SENDER_LIMIT", 2) or 2)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
limit = 2
|
||||||
|
return max(1, limit)
|
||||||
|
|
||||||
|
def _per_sender_pending_count(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
mailbox_key: str,
|
||||||
|
sender_block_ref: str,
|
||||||
|
) -> int:
|
||||||
|
"""Count UNACKED messages from ``sender_block_ref`` currently parked
|
||||||
|
in ``mailbox_key``. Caller already holds ``self._lock``.
|
||||||
|
|
||||||
|
Messages that have been claimed/acked are removed from the mailbox
|
||||||
|
list (see ``claim_message_ids``), so anything still here is by
|
||||||
|
definition unacked. We count by exact ``sender_block_ref`` match
|
||||||
|
— that's the per-pair sender identity used for blocking too, so
|
||||||
|
the cap is naturally per-(sender, recipient).
|
||||||
|
"""
|
||||||
|
if not mailbox_key or not sender_block_ref:
|
||||||
|
return 0
|
||||||
|
messages = self._mailboxes.get(mailbox_key, [])
|
||||||
|
return sum(1 for m in messages if m.sender_block_ref == sender_block_ref)
|
||||||
|
|
||||||
def _nonce_ttl_seconds(self) -> int:
|
def _nonce_ttl_seconds(self) -> int:
|
||||||
return max(30, int(self._settings().MESH_DM_NONCE_TTL_S))
|
return max(30, int(self._settings().MESH_DM_NONCE_TTL_S))
|
||||||
|
|
||||||
@@ -1264,6 +1297,21 @@ class DMRelay:
|
|||||||
)
|
)
|
||||||
self._save()
|
self._save()
|
||||||
|
|
||||||
|
def unregister_prekey_lookup_alias(self, alias: str) -> bool:
|
||||||
|
"""Remove an invite-scoped lookup alias from the local relay."""
|
||||||
|
handle = str(alias or "").strip()
|
||||||
|
if not handle:
|
||||||
|
return False
|
||||||
|
removed = False
|
||||||
|
with self._lock:
|
||||||
|
self._refresh_from_shared_relay()
|
||||||
|
if handle in self._prekey_lookup_aliases:
|
||||||
|
del self._prekey_lookup_aliases[handle]
|
||||||
|
removed = True
|
||||||
|
if removed:
|
||||||
|
self._save()
|
||||||
|
return removed
|
||||||
|
|
||||||
def consume_one_time_prekey(self, agent_id: str) -> dict[str, Any] | None:
|
def consume_one_time_prekey(self, agent_id: str) -> dict[str, Any] | None:
|
||||||
"""Atomically claim the next published one-time prekey for a peer bundle."""
|
"""Atomically claim the next published one-time prekey for a peer bundle."""
|
||||||
claimed: dict[str, Any] | None = None
|
claimed: dict[str, Any] | None = None
|
||||||
@@ -1500,6 +1548,29 @@ class DMRelay:
|
|||||||
if len(self._mailboxes[mailbox_key]) >= self._mailbox_limit_for_class(delivery_class):
|
if len(self._mailboxes[mailbox_key]) >= self._mailbox_limit_for_class(delivery_class):
|
||||||
metrics_inc("dm_drop_full")
|
metrics_inc("dm_drop_full")
|
||||||
return {"ok": False, "detail": "Recipient mailbox full"}
|
return {"ok": False, "detail": "Recipient mailbox full"}
|
||||||
|
# Anti-spam: per-(sender, recipient) cap on unacked messages.
|
||||||
|
# A sender who already has the configured number of messages
|
||||||
|
# parked in this mailbox can't deposit more until the recipient
|
||||||
|
# pulls (acks) at least one. The same cap is re-enforced on
|
||||||
|
# inbound replication in ``accept_replica`` so this rule isn't
|
||||||
|
# bypassable by patching out the local check on a hostile
|
||||||
|
# sender's relay — see config.py
|
||||||
|
# MESH_DM_PENDING_PER_SENDER_LIMIT for the threat model.
|
||||||
|
per_sender_limit = self._per_sender_pending_limit()
|
||||||
|
pending = self._per_sender_pending_count(
|
||||||
|
mailbox_key=mailbox_key,
|
||||||
|
sender_block_ref=sender_block_ref,
|
||||||
|
)
|
||||||
|
if pending >= per_sender_limit:
|
||||||
|
metrics_inc("dm_drop_per_sender_cap")
|
||||||
|
return {
|
||||||
|
"ok": False,
|
||||||
|
"detail": (
|
||||||
|
f"Recipient already has {pending} unread message"
|
||||||
|
f"{'s' if pending != 1 else ''} from you. Wait for "
|
||||||
|
"them to read your messages before sending more."
|
||||||
|
),
|
||||||
|
}
|
||||||
if not msg_id:
|
if not msg_id:
|
||||||
msg_id = f"dm_{int(time.time() * 1000)}_{secrets.token_hex(6)}"
|
msg_id = f"dm_{int(time.time() * 1000)}_{secrets.token_hex(6)}"
|
||||||
elif any(m.msg_id == msg_id for m in self._mailboxes[mailbox_key]):
|
elif any(m.msg_id == msg_id for m in self._mailboxes[mailbox_key]):
|
||||||
@@ -1524,8 +1595,245 @@ class DMRelay:
|
|||||||
)
|
)
|
||||||
self._stats["messages_in_memory"] = sum(len(v) for v in self._mailboxes.values())
|
self._stats["messages_in_memory"] = sum(len(v) for v in self._mailboxes.values())
|
||||||
self._save()
|
self._save()
|
||||||
|
# Cross-node mailbox replication: push the freshly-stored
|
||||||
|
# envelope to every authenticated relay peer so the recipient
|
||||||
|
# can log into ANY node and find their messages. The push is
|
||||||
|
# async (fire-and-forget thread) so deposit() returns
|
||||||
|
# immediately — slow Tor peers can't block the sender's UX.
|
||||||
|
# Each receiving peer re-enforces the per-sender cap on
|
||||||
|
# acceptance, so hostile relays can't widen the cap.
|
||||||
|
try:
|
||||||
|
envelope_for_push = self.envelope_for_replication(
|
||||||
|
mailbox_key=mailbox_key, msg_id=msg_id,
|
||||||
|
)
|
||||||
|
if envelope_for_push:
|
||||||
|
self._replicate_envelope_to_peers_async(
|
||||||
|
envelope=envelope_for_push,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
metrics_inc("dm_replication_push_error")
|
||||||
return {"ok": True, "msg_id": msg_id}
|
return {"ok": True, "msg_id": msg_id}
|
||||||
|
|
||||||
|
def accept_replica(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
envelope: dict[str, Any],
|
||||||
|
originating_peer_url: str = "",
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Receive a DM envelope replicated from a peer relay.
|
||||||
|
|
||||||
|
Cross-node mailbox replication entry point. When a sender's local
|
||||||
|
relay accepts a ``deposit`` and pushes the envelope to
|
||||||
|
``MESH_RELAY_PEERS`` (so the recipient can log into any peer
|
||||||
|
node and find their messages), each receiving peer calls
|
||||||
|
``accept_replica`` to ingest it.
|
||||||
|
|
||||||
|
The per-(sender, recipient) cap is re-enforced HERE. That's what
|
||||||
|
makes the rule a NETWORK rule rather than a client-side honor
|
||||||
|
system: a hostile sender who patches out the local ``deposit``
|
||||||
|
check still can't get a 3rd unacked message to spread, because
|
||||||
|
every honest peer enforces the same cap on inbound replicas.
|
||||||
|
Result: hostile relays can hold extras locally, but those extras
|
||||||
|
never reach any node a legitimate recipient is polling from.
|
||||||
|
|
||||||
|
Returns the same shape as ``deposit`` so the calling endpoint can
|
||||||
|
forward the result back to the originating peer.
|
||||||
|
"""
|
||||||
|
if not isinstance(envelope, dict):
|
||||||
|
return {"ok": False, "detail": "envelope must be an object"}
|
||||||
|
msg_id = str(envelope.get("msg_id", "") or "").strip()
|
||||||
|
mailbox_key = str(envelope.get("mailbox_key", "") or "").strip()
|
||||||
|
sender_block_ref = str(envelope.get("sender_block_ref", "") or "").strip()
|
||||||
|
ciphertext = str(envelope.get("ciphertext", "") or "")
|
||||||
|
if not msg_id or not mailbox_key or not sender_block_ref or not ciphertext:
|
||||||
|
return {"ok": False, "detail": "envelope missing required fields"}
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
self._refresh_from_shared_relay()
|
||||||
|
self._cleanup_expired()
|
||||||
|
|
||||||
|
# Idempotent — if we already hold this exact msg_id, the
|
||||||
|
# replication round-tripped or a peer pushed the same
|
||||||
|
# envelope through multiple paths. Accept silently.
|
||||||
|
if any(m.msg_id == msg_id for m in self._mailboxes.get(mailbox_key, [])):
|
||||||
|
metrics_inc("dm_replica_duplicate")
|
||||||
|
return {"ok": True, "msg_id": msg_id, "duplicate": True}
|
||||||
|
|
||||||
|
# Same per-class cap as the deposit path — defense in depth
|
||||||
|
# against a peer that wraps a "deposit" as a "replica" to
|
||||||
|
# bypass the class limit.
|
||||||
|
delivery_class = str(envelope.get("delivery_class", "") or "")
|
||||||
|
if delivery_class in ("request", "shared", "self"):
|
||||||
|
class_limit = self._mailbox_limit_for_class(delivery_class)
|
||||||
|
else:
|
||||||
|
class_limit = self._shared_mailbox_limit()
|
||||||
|
if len(self._mailboxes.get(mailbox_key, [])) >= class_limit:
|
||||||
|
metrics_inc("dm_replica_drop_full")
|
||||||
|
return {"ok": False, "detail": "Recipient mailbox full"}
|
||||||
|
|
||||||
|
# THE network rule: per-(sender, recipient) anti-spam cap.
|
||||||
|
per_sender_limit = self._per_sender_pending_limit()
|
||||||
|
pending = self._per_sender_pending_count(
|
||||||
|
mailbox_key=mailbox_key,
|
||||||
|
sender_block_ref=sender_block_ref,
|
||||||
|
)
|
||||||
|
if pending >= per_sender_limit:
|
||||||
|
metrics_inc("dm_replica_drop_per_sender_cap")
|
||||||
|
# Returning a structured rejection — the sender's relay
|
||||||
|
# learns its envelope was rejected by an honest peer and
|
||||||
|
# can stop trying to push it.
|
||||||
|
return {
|
||||||
|
"ok": False,
|
||||||
|
"detail": (
|
||||||
|
"Per-sender cap reached on this relay; refusing replica"
|
||||||
|
),
|
||||||
|
"cap_violation": True,
|
||||||
|
"pending": pending,
|
||||||
|
"limit": per_sender_limit,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Accept the replica into the local mailbox.
|
||||||
|
self._mailboxes[mailbox_key].append(
|
||||||
|
DMMessage(
|
||||||
|
sender_id=str(envelope.get("sender_id", "") or ""),
|
||||||
|
ciphertext=ciphertext,
|
||||||
|
timestamp=float(envelope.get("timestamp", time.time()) or time.time()),
|
||||||
|
msg_id=msg_id,
|
||||||
|
delivery_class=str(envelope.get("delivery_class", "shared") or "shared"),
|
||||||
|
sender_seal=str(envelope.get("sender_seal", "") or ""),
|
||||||
|
relay_salt=str(envelope.get("relay_salt", "") or ""),
|
||||||
|
sender_block_ref=sender_block_ref,
|
||||||
|
payload_format=str(envelope.get("payload_format", "dm1") or "dm1"),
|
||||||
|
session_welcome=str(envelope.get("session_welcome", "") or ""),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self._stats["messages_in_memory"] = sum(len(v) for v in self._mailboxes.values())
|
||||||
|
self._save()
|
||||||
|
metrics_inc("dm_replica_accepted")
|
||||||
|
return {"ok": True, "msg_id": msg_id}
|
||||||
|
|
||||||
|
def _replicate_envelope_to_peers_async(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
envelope: dict[str, Any],
|
||||||
|
) -> None:
|
||||||
|
"""Push an outbound DM envelope to every authenticated relay peer.
|
||||||
|
|
||||||
|
Fire-and-forget: spawned in a background thread so ``deposit``
|
||||||
|
returns to the caller immediately. Per-peer errors are logged
|
||||||
|
and swallowed — the sender's UX must not block on slow Tor
|
||||||
|
peers, and a peer that's down today gets the next message
|
||||||
|
whenever it comes back. Inbound recipient polling from a healthy
|
||||||
|
peer keeps the system functional during peer failures.
|
||||||
|
|
||||||
|
Each peer is authed with the existing per-peer HMAC pattern
|
||||||
|
(#256) — same headers and key resolver gate-message replication
|
||||||
|
uses, so a hostile node that doesn't know any peer's HMAC key
|
||||||
|
can't impersonate a legitimate relay.
|
||||||
|
"""
|
||||||
|
import threading
|
||||||
|
|
||||||
|
def _do_push():
|
||||||
|
try:
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import requests as _requests
|
||||||
|
|
||||||
|
from services.mesh.mesh_crypto import (
|
||||||
|
normalize_peer_url,
|
||||||
|
resolve_peer_key_for_url,
|
||||||
|
)
|
||||||
|
from services.mesh.mesh_router import (
|
||||||
|
authenticated_push_peer_urls,
|
||||||
|
)
|
||||||
|
|
||||||
|
peers = authenticated_push_peer_urls()
|
||||||
|
if not peers:
|
||||||
|
return
|
||||||
|
|
||||||
|
payload = json.dumps(
|
||||||
|
{"envelope": envelope},
|
||||||
|
separators=(",", ":"),
|
||||||
|
ensure_ascii=False,
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
timeout = max(
|
||||||
|
1,
|
||||||
|
int(getattr(self._settings(), "MESH_RELAY_PUSH_TIMEOUT_S", 10) or 10),
|
||||||
|
)
|
||||||
|
|
||||||
|
for peer_url in peers:
|
||||||
|
try:
|
||||||
|
normalized = normalize_peer_url(peer_url)
|
||||||
|
headers = {"Content-Type": "application/json"}
|
||||||
|
peer_key = resolve_peer_key_for_url(normalized)
|
||||||
|
if peer_key:
|
||||||
|
headers["X-Peer-Url"] = normalized
|
||||||
|
headers["X-Peer-HMAC"] = hmac.new(
|
||||||
|
peer_key, payload, hashlib.sha256
|
||||||
|
).hexdigest()
|
||||||
|
url = f"{peer_url}/api/mesh/dm/replicate-envelope"
|
||||||
|
resp = _requests.post(
|
||||||
|
url, data=payload, timeout=timeout, headers=headers,
|
||||||
|
)
|
||||||
|
if resp.status_code == 200:
|
||||||
|
metrics_inc("dm_replication_push_ok")
|
||||||
|
else:
|
||||||
|
# 4xx including the structured cap_violation
|
||||||
|
# rejection from accept_replica — sender's
|
||||||
|
# relay learns and stops retrying this msg_id.
|
||||||
|
metrics_inc("dm_replication_push_rejected")
|
||||||
|
except Exception:
|
||||||
|
# Per-peer failure is non-fatal — log to metrics
|
||||||
|
# but don't break the loop. Other peers and a
|
||||||
|
# future retry can still propagate the envelope.
|
||||||
|
metrics_inc("dm_replication_push_error")
|
||||||
|
continue
|
||||||
|
except Exception:
|
||||||
|
# Outer guard — never let replication errors propagate
|
||||||
|
# back to the sender's deposit() caller.
|
||||||
|
metrics_inc("dm_replication_push_error")
|
||||||
|
|
||||||
|
thread = threading.Thread(
|
||||||
|
target=_do_push,
|
||||||
|
name="dm-replicate-push",
|
||||||
|
daemon=True,
|
||||||
|
)
|
||||||
|
thread.start()
|
||||||
|
|
||||||
|
def envelope_for_replication(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
mailbox_key: str,
|
||||||
|
msg_id: str,
|
||||||
|
) -> dict[str, Any] | None:
|
||||||
|
"""Return the wire-form envelope for a stored message, suitable
|
||||||
|
for POSTing to a peer relay's replicate-envelope endpoint.
|
||||||
|
|
||||||
|
Returns ``None`` if the message isn't in the mailbox (already
|
||||||
|
acked, expired, never existed). The caller holds the
|
||||||
|
responsibility for transport security (Tor SOCKS for .onion
|
||||||
|
peers, per-peer HMAC) and for not leaking the envelope to
|
||||||
|
clearnet peers when private transport is required.
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
for m in self._mailboxes.get(mailbox_key, []):
|
||||||
|
if m.msg_id == msg_id:
|
||||||
|
return {
|
||||||
|
"msg_id": m.msg_id,
|
||||||
|
"mailbox_key": mailbox_key,
|
||||||
|
"sender_id": m.sender_id,
|
||||||
|
"sender_block_ref": m.sender_block_ref,
|
||||||
|
"sender_seal": m.sender_seal,
|
||||||
|
"ciphertext": m.ciphertext,
|
||||||
|
"timestamp": m.timestamp,
|
||||||
|
"delivery_class": m.delivery_class,
|
||||||
|
"relay_salt": m.relay_salt,
|
||||||
|
"payload_format": m.payload_format,
|
||||||
|
"session_welcome": m.session_welcome,
|
||||||
|
}
|
||||||
|
return None
|
||||||
|
|
||||||
def is_blocked(self, recipient_id: str, sender_id: str) -> bool:
|
def is_blocked(self, recipient_id: str, sender_id: str) -> bool:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
self._refresh_from_shared_relay()
|
self._refresh_from_shared_relay()
|
||||||
|
|||||||
@@ -216,18 +216,19 @@ def _peer_pair_ref_key(peer_url: str) -> bytes:
|
|||||||
Returns an empty key on misconfiguration so callers fail closed.
|
Returns an empty key on misconfiguration so callers fail closed.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
from services.config import get_settings
|
from services.mesh.mesh_crypto import (
|
||||||
from services.mesh.mesh_crypto import _derive_peer_key, normalize_peer_url
|
normalize_peer_url,
|
||||||
|
resolve_peer_key_for_url,
|
||||||
secret = str(get_settings().MESH_PEER_PUSH_SECRET or "").strip()
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
return b""
|
return b""
|
||||||
if not secret:
|
|
||||||
return b""
|
|
||||||
normalized = normalize_peer_url(peer_url or "")
|
normalized = normalize_peer_url(peer_url or "")
|
||||||
if not normalized:
|
if not normalized:
|
||||||
return b""
|
return b""
|
||||||
peer_key = _derive_peer_key(secret, normalized)
|
# Issue #256: resolve_peer_key_for_url() prefers per-peer secrets
|
||||||
|
# from MESH_PEER_SECRETS and falls back to the global
|
||||||
|
# MESH_PEER_PUSH_SECRET only when the URL has no per-peer entry.
|
||||||
|
peer_key = resolve_peer_key_for_url(normalized)
|
||||||
if not peer_key:
|
if not peer_key:
|
||||||
return b""
|
return b""
|
||||||
# Domain-separate from the transport HMAC key so the two
|
# Domain-separate from the transport HMAC key so the two
|
||||||
@@ -1438,14 +1439,57 @@ class Infonet:
|
|||||||
# Running counters — avoid O(N) scans in get_info()
|
# Running counters — avoid O(N) scans in get_info()
|
||||||
self._type_counts: dict[str, int] = {}
|
self._type_counts: dict[str, int] = {}
|
||||||
self._active_count: int = 0
|
self._active_count: int = 0
|
||||||
|
self._registered_nodes: set[str] = set()
|
||||||
self._chain_bytes: int = 2 # Start with "[]" empty JSON array
|
self._chain_bytes: int = 2 # Start with "[]" empty JSON array
|
||||||
self._dirty = False
|
self._dirty = False
|
||||||
self._save_lock = threading.Lock()
|
self._save_lock = threading.Lock()
|
||||||
self._save_timer: threading.Timer | None = None
|
self._save_timer: threading.Timer | None = None
|
||||||
self._SAVE_INTERVAL = 5.0 # seconds — coalesce writes
|
self._SAVE_INTERVAL = 5.0 # seconds — coalesce writes
|
||||||
|
# Issue #208: Merkle levels cache so get_merkle_proofs() doesn't
|
||||||
|
# rebuild O(n) levels on every public call. Invalidated whenever
|
||||||
|
# self.events mutates. Computed lazily on first read after an
|
||||||
|
# invalidation.
|
||||||
|
self._merkle_levels_cache: list[list[str]] | None = None
|
||||||
|
self._merkle_levels_for_event_count: int = -1
|
||||||
atexit.register(self._flush)
|
atexit.register(self._flush)
|
||||||
self._load()
|
self._load()
|
||||||
|
|
||||||
|
def _invalidate_merkle_cache(self) -> None:
|
||||||
|
"""Clear the precomputed Merkle levels.
|
||||||
|
|
||||||
|
Called whenever ``self.events`` may have mutated (append, rebuild,
|
||||||
|
cleanup, fork resolution). The next call to ``get_merkle_root()``
|
||||||
|
or ``get_merkle_proofs()`` will recompute and re-cache.
|
||||||
|
"""
|
||||||
|
self._merkle_levels_cache = None
|
||||||
|
self._merkle_levels_for_event_count = -1
|
||||||
|
|
||||||
|
def _get_merkle_levels(self) -> list[list[str]]:
|
||||||
|
"""Return Merkle levels for the current chain, recomputing if
|
||||||
|
the cache is invalid or out of date.
|
||||||
|
|
||||||
|
Issue #208: a public endpoint (``/api/mesh/infonet/sync?include_proofs=true``)
|
||||||
|
used to rebuild Merkle levels on every request, which is O(n) in
|
||||||
|
chain length and trivially abusable for CPU exhaustion. By caching
|
||||||
|
the levels and invalidating on mutation, repeated proof requests
|
||||||
|
become O(1) per proof; the rebuild only happens after a genuine
|
||||||
|
append/rebuild/cleanup.
|
||||||
|
"""
|
||||||
|
from services.mesh.mesh_merkle import build_merkle_levels
|
||||||
|
|
||||||
|
current_count = len(self.events)
|
||||||
|
if (
|
||||||
|
self._merkle_levels_cache is not None
|
||||||
|
and self._merkle_levels_for_event_count == current_count
|
||||||
|
):
|
||||||
|
return self._merkle_levels_cache
|
||||||
|
|
||||||
|
leaves = [e["event_id"] for e in self.events]
|
||||||
|
levels = build_merkle_levels(leaves)
|
||||||
|
self._merkle_levels_cache = levels
|
||||||
|
self._merkle_levels_for_event_count = current_count
|
||||||
|
return levels
|
||||||
|
|
||||||
# ─── Persistence ──────────────────────────────────────────────────
|
# ─── Persistence ──────────────────────────────────────────────────
|
||||||
|
|
||||||
def _load(self):
|
def _load(self):
|
||||||
@@ -1518,6 +1562,7 @@ class Infonet:
|
|||||||
self._last_validated_index = 0
|
self._last_validated_index = 0
|
||||||
self._type_counts = {}
|
self._type_counts = {}
|
||||||
self._active_count = 0
|
self._active_count = 0
|
||||||
|
self._registered_nodes = set()
|
||||||
self._chain_bytes = 2
|
self._chain_bytes = 2
|
||||||
|
|
||||||
def _rebuild_state(self) -> None:
|
def _rebuild_state(self) -> None:
|
||||||
@@ -1566,10 +1611,15 @@ class Infonet:
|
|||||||
now = time.time()
|
now = time.time()
|
||||||
self._type_counts = {}
|
self._type_counts = {}
|
||||||
self._active_count = 0
|
self._active_count = 0
|
||||||
|
self._registered_nodes = set()
|
||||||
self._chain_bytes = 2 # "[]"
|
self._chain_bytes = 2 # "[]"
|
||||||
for evt in self.events:
|
for evt in self.events:
|
||||||
t = evt.get("event_type", "unknown")
|
t = evt.get("event_type", "unknown")
|
||||||
self._type_counts[t] = self._type_counts.get(t, 0) + 1
|
self._type_counts[t] = self._type_counts.get(t, 0) + 1
|
||||||
|
if t == "node_register":
|
||||||
|
node_id = str(evt.get("node_id", "") or "")
|
||||||
|
if node_id:
|
||||||
|
self._registered_nodes.add(node_id)
|
||||||
is_eph = evt.get("payload", {}).get("ephemeral") or evt.get("payload", {}).get("_ephemeral")
|
is_eph = evt.get("payload", {}).get("ephemeral") or evt.get("payload", {}).get("_ephemeral")
|
||||||
if not is_eph or (now - evt.get("timestamp", 0)) < EPHEMERAL_TTL:
|
if not is_eph or (now - evt.get("timestamp", 0)) < EPHEMERAL_TTL:
|
||||||
self._active_count += 1
|
self._active_count += 1
|
||||||
@@ -1579,6 +1629,10 @@ class Infonet:
|
|||||||
"""Incrementally update counters when a new event is appended."""
|
"""Incrementally update counters when a new event is appended."""
|
||||||
t = evt.get("event_type", "unknown")
|
t = evt.get("event_type", "unknown")
|
||||||
self._type_counts[t] = self._type_counts.get(t, 0) + 1
|
self._type_counts[t] = self._type_counts.get(t, 0) + 1
|
||||||
|
if t == "node_register":
|
||||||
|
node_id = str(evt.get("node_id", "") or "")
|
||||||
|
if node_id:
|
||||||
|
self._registered_nodes.add(node_id)
|
||||||
self._active_count += 1
|
self._active_count += 1
|
||||||
self._chain_bytes += len(json.dumps(evt)) + 2
|
self._chain_bytes += len(json.dumps(evt)) + 2
|
||||||
|
|
||||||
@@ -1972,6 +2026,8 @@ class Infonet:
|
|||||||
self.head_hash = event.event_id
|
self.head_hash = event.event_id
|
||||||
self.node_sequences[node_id] = sequence
|
self.node_sequences[node_id] = sequence
|
||||||
self._replay_filter.add(event.event_id)
|
self._replay_filter.add(event.event_id)
|
||||||
|
# Issue #208: chain advanced, cached Merkle levels are stale.
|
||||||
|
self._invalidate_merkle_cache()
|
||||||
self._update_counters_for_event(event_dict)
|
self._update_counters_for_event(event_dict)
|
||||||
|
|
||||||
if event_type == "key_revoke":
|
if event_type == "key_revoke":
|
||||||
@@ -2247,6 +2303,7 @@ class Infonet:
|
|||||||
self.event_index[event_id] = len(self.events) - 1
|
self.event_index[event_id] = len(self.events) - 1
|
||||||
self.head_hash = event_id
|
self.head_hash = event_id
|
||||||
self.node_sequences[node_id] = sequence
|
self.node_sequences[node_id] = sequence
|
||||||
|
self._update_counters_for_event(evt)
|
||||||
accepted += 1
|
accepted += 1
|
||||||
expected_prev = event_id
|
expected_prev = event_id
|
||||||
self._replay_filter.add(event_id)
|
self._replay_filter.add(event_id)
|
||||||
@@ -2254,6 +2311,9 @@ class Infonet:
|
|||||||
self._apply_revocation(evt)
|
self._apply_revocation(evt)
|
||||||
|
|
||||||
if accepted:
|
if accepted:
|
||||||
|
# Issue #208: any accepted event invalidates the cached Merkle
|
||||||
|
# levels. One invalidation per batch, not per event.
|
||||||
|
self._invalidate_merkle_cache()
|
||||||
self._save()
|
self._save()
|
||||||
return {"accepted": accepted, "duplicates": duplicates, "rejected": rejected}
|
return {"accepted": accepted, "duplicates": duplicates, "rejected": rejected}
|
||||||
|
|
||||||
@@ -2552,6 +2612,10 @@ class Infonet:
|
|||||||
# Apply fork
|
# Apply fork
|
||||||
self.events = prefix + ordered
|
self.events = prefix + ordered
|
||||||
self._rebuild_state()
|
self._rebuild_state()
|
||||||
|
self._rebuild_revocations()
|
||||||
|
self._rebuild_counters()
|
||||||
|
# Issue #208: chain replaced, cached Merkle levels are stale.
|
||||||
|
self._invalidate_merkle_cache()
|
||||||
self._save()
|
self._save()
|
||||||
try:
|
try:
|
||||||
from services.mesh.mesh_metrics import increment as metrics_inc
|
from services.mesh.mesh_metrics import increment as metrics_inc
|
||||||
@@ -2681,6 +2745,8 @@ class Infonet:
|
|||||||
"head_hash_full": self.head_hash,
|
"head_hash_full": self.head_hash,
|
||||||
"chain_lock": self.chain_lock(),
|
"chain_lock": self.chain_lock(),
|
||||||
"known_nodes": len(self.node_sequences),
|
"known_nodes": len(self.node_sequences),
|
||||||
|
"author_nodes": len(self.node_sequences),
|
||||||
|
"registered_nodes": len(self._registered_nodes),
|
||||||
"event_types": dict(self._type_counts),
|
"event_types": dict(self._type_counts),
|
||||||
"chain_size_kb": round(self._chain_bytes / 1024, 1),
|
"chain_size_kb": round(self._chain_bytes / 1024, 1),
|
||||||
"unsigned_events": 0,
|
"unsigned_events": 0,
|
||||||
@@ -2716,8 +2782,11 @@ class Infonet:
|
|||||||
|
|
||||||
if len(new_events) != before:
|
if len(new_events) != before:
|
||||||
self.events = new_events
|
self.events = new_events
|
||||||
# Rebuild index
|
self._rebuild_state()
|
||||||
self.event_index = {e["event_id"]: i for i, e in enumerate(self.events)}
|
self._rebuild_revocations()
|
||||||
|
self._rebuild_counters()
|
||||||
|
# Issue #208: cleanup may have dropped expired events.
|
||||||
|
self._invalidate_merkle_cache()
|
||||||
self._save()
|
self._save()
|
||||||
logger.info(f"Infonet cleanup: removed {before - len(new_events)} expired events")
|
logger.info(f"Infonet cleanup: removed {before - len(new_events)} expired events")
|
||||||
|
|
||||||
@@ -2726,30 +2795,37 @@ class Infonet:
|
|||||||
def get_merkle_root(self) -> str:
|
def get_merkle_root(self) -> str:
|
||||||
"""Compute a Merkle root hash of the Infonet for sync comparison.
|
"""Compute a Merkle root hash of the Infonet for sync comparison.
|
||||||
|
|
||||||
Two nodes with the same Merkle root have identical chains.
|
Two nodes with the same Merkle root have identical chains. Reads
|
||||||
|
from the cached Merkle levels (issue #208) — O(1) when the chain
|
||||||
|
hasn't changed since the last computation.
|
||||||
"""
|
"""
|
||||||
if not self.events:
|
if not self.events:
|
||||||
return GENESIS_HASH
|
return GENESIS_HASH
|
||||||
|
|
||||||
from services.mesh.mesh_merkle import merkle_root
|
levels = self._get_merkle_levels()
|
||||||
|
if not levels or not levels[-1]:
|
||||||
leaves = [e["event_id"] for e in self.events]
|
return GENESIS_HASH
|
||||||
root = merkle_root(leaves)
|
return levels[-1][0] or GENESIS_HASH
|
||||||
return root or GENESIS_HASH
|
|
||||||
|
|
||||||
def get_merkle_proofs(self, start_index: int, count: int) -> dict:
|
def get_merkle_proofs(self, start_index: int, count: int) -> dict:
|
||||||
"""Return merkle proofs for a contiguous range of events."""
|
"""Return merkle proofs for a contiguous range of events.
|
||||||
leaves = [e["event_id"] for e in self.events]
|
|
||||||
total = len(leaves)
|
Issue #208: uses the cached Merkle levels so this is O(count *
|
||||||
|
log n) per request, not O(n + count * log n). Anonymous peers
|
||||||
|
hitting ``/api/mesh/infonet/sync?include_proofs=true`` no longer
|
||||||
|
force a rebuild on every call.
|
||||||
|
"""
|
||||||
|
total = len(self.events)
|
||||||
if total == 0:
|
if total == 0:
|
||||||
return {"root": GENESIS_HASH, "total": 0, "start": 0, "proofs": []}
|
return {"root": GENESIS_HASH, "total": 0, "start": 0, "proofs": []}
|
||||||
|
|
||||||
from services.mesh.mesh_merkle import build_merkle_levels, merkle_proof_from_levels
|
from services.mesh.mesh_merkle import merkle_proof_from_levels
|
||||||
|
|
||||||
|
leaves = [e["event_id"] for e in self.events]
|
||||||
start = max(0, start_index)
|
start = max(0, start_index)
|
||||||
end = min(total, start + max(0, count))
|
end = min(total, start + max(0, count))
|
||||||
levels = build_merkle_levels(leaves)
|
levels = self._get_merkle_levels()
|
||||||
root = levels[-1][0] if levels else GENESIS_HASH
|
root = levels[-1][0] if levels and levels[-1] else GENESIS_HASH
|
||||||
|
|
||||||
proofs = []
|
proofs = []
|
||||||
for idx in range(start, end):
|
for idx in range(start, end):
|
||||||
|
|||||||
@@ -30,10 +30,19 @@ def eligible_sync_peers(records: list[PeerRecord], *, now: float | None = None)
|
|||||||
for record in records
|
for record in records
|
||||||
if record.bucket == "sync" and record.enabled and int(record.cooldown_until or 0) <= current_time
|
if record.bucket == "sync" and record.enabled and int(record.cooldown_until or 0) <= current_time
|
||||||
]
|
]
|
||||||
|
|
||||||
|
def _seed_priority(record: PeerRecord) -> int:
|
||||||
|
role = str(record.role or "").strip().lower()
|
||||||
|
source = str(record.source or "").strip().lower()
|
||||||
|
if role == "seed" and source in {"bundle", "bootstrap_promoted"}:
|
||||||
|
return 0
|
||||||
|
return 1
|
||||||
|
|
||||||
return sorted(
|
return sorted(
|
||||||
candidates,
|
candidates,
|
||||||
key=lambda record: (
|
key=lambda record: (
|
||||||
-int(record.last_sync_ok_at or 0),
|
-int(record.last_sync_ok_at or 0),
|
||||||
|
_seed_priority(record),
|
||||||
int(record.failure_count or 0),
|
int(record.failure_count or 0),
|
||||||
int(record.added_at or 0),
|
int(record.added_at or 0),
|
||||||
record.peer_url,
|
record.peer_url,
|
||||||
|
|||||||
@@ -258,6 +258,12 @@ class PeerStore:
|
|||||||
self._records[record.record_key()] = record
|
self._records[record.record_key()] = record
|
||||||
return record
|
return record
|
||||||
|
|
||||||
|
explicit_seed_refresh = (
|
||||||
|
record.bucket == "sync"
|
||||||
|
and record.role == "seed"
|
||||||
|
and record.source in {"bundle", "bootstrap_promoted"}
|
||||||
|
)
|
||||||
|
|
||||||
merged = PeerRecord(
|
merged = PeerRecord(
|
||||||
bucket=record.bucket,
|
bucket=record.bucket,
|
||||||
source=record.source,
|
source=record.source,
|
||||||
@@ -272,9 +278,9 @@ class PeerStore:
|
|||||||
last_seen_at=max(existing.last_seen_at, record.last_seen_at),
|
last_seen_at=max(existing.last_seen_at, record.last_seen_at),
|
||||||
last_sync_ok_at=max(existing.last_sync_ok_at, record.last_sync_ok_at),
|
last_sync_ok_at=max(existing.last_sync_ok_at, record.last_sync_ok_at),
|
||||||
last_push_ok_at=max(existing.last_push_ok_at, record.last_push_ok_at),
|
last_push_ok_at=max(existing.last_push_ok_at, record.last_push_ok_at),
|
||||||
last_error=record.last_error or existing.last_error,
|
last_error="" if explicit_seed_refresh else record.last_error or existing.last_error,
|
||||||
failure_count=max(existing.failure_count, record.failure_count),
|
failure_count=0 if explicit_seed_refresh else max(existing.failure_count, record.failure_count),
|
||||||
cooldown_until=max(existing.cooldown_until, record.cooldown_until),
|
cooldown_until=0 if explicit_seed_refresh else max(existing.cooldown_until, record.cooldown_until),
|
||||||
metadata={**existing.metadata, **record.metadata},
|
metadata={**existing.metadata, **record.metadata},
|
||||||
)
|
)
|
||||||
self._records[record.record_key()] = merged
|
self._records[record.record_key()] = merged
|
||||||
|
|||||||
@@ -26,7 +26,11 @@ from enum import Enum
|
|||||||
from typing import Any, Callable, Optional
|
from typing import Any, Callable, Optional
|
||||||
from collections import deque
|
from collections import deque
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
from services.mesh.mesh_crypto import _derive_peer_key, normalize_peer_url
|
from services.mesh.mesh_crypto import (
|
||||||
|
_derive_peer_key,
|
||||||
|
normalize_peer_url,
|
||||||
|
resolve_peer_key_for_url,
|
||||||
|
)
|
||||||
from services.mesh.mesh_metrics import increment as metrics_inc
|
from services.mesh.mesh_metrics import increment as metrics_inc
|
||||||
from services.mesh.mesh_privacy_policy import (
|
from services.mesh.mesh_privacy_policy import (
|
||||||
TRANSPORT_TIER_ORDER as _TIER_RANK,
|
TRANSPORT_TIER_ORDER as _TIER_RANK,
|
||||||
@@ -520,7 +524,7 @@ class MeshtasticTransport:
|
|||||||
|
|
||||||
def _on_connect(client, userdata, flags, rc):
|
def _on_connect(client, userdata, flags, rc):
|
||||||
if rc == 0:
|
if rc == 0:
|
||||||
info = client.publish(topic, payload, qos=0)
|
info = client.publish(topic, payload, qos=1)
|
||||||
info.wait_for_publish(timeout=5)
|
info.wait_for_publish(timeout=5)
|
||||||
published[0] = True
|
published[0] = True
|
||||||
client.disconnect()
|
client.disconnect()
|
||||||
@@ -550,9 +554,9 @@ class MeshtasticTransport:
|
|||||||
True,
|
True,
|
||||||
self.NAME,
|
self.NAME,
|
||||||
(
|
(
|
||||||
f"Published direct to !{to_node:08x} via {region}/{channel}"
|
f"Broker accepted direct publish to !{to_node:08x} via {region}/{channel}"
|
||||||
if direct_node is not None
|
if direct_node is not None
|
||||||
else f"Published to {region}/{channel} ({len(payload)}B protobuf)"
|
else f"Broker accepted channel publish to {region}/{channel} ({len(payload)}B protobuf)"
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -703,7 +707,6 @@ class InternetTransport(_PeerPushTransportMixin):
|
|||||||
endpoint_path, padded = self._build_peer_push_request(envelope, self.NAME)
|
endpoint_path, padded = self._build_peer_push_request(envelope, self.NAME)
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
return TransportResult(False, self.NAME, str(exc))
|
return TransportResult(False, self.NAME, str(exc))
|
||||||
secret = str(settings.MESH_PEER_PUSH_SECRET or "").strip()
|
|
||||||
|
|
||||||
delivered = 0
|
delivered = 0
|
||||||
last_error = ""
|
last_error = ""
|
||||||
@@ -713,10 +716,13 @@ class InternetTransport(_PeerPushTransportMixin):
|
|||||||
try:
|
try:
|
||||||
normalized_peer_url = normalize_peer_url(peer_url)
|
normalized_peer_url = normalize_peer_url(peer_url)
|
||||||
headers = {"Content-Type": "application/json"}
|
headers = {"Content-Type": "application/json"}
|
||||||
if secret:
|
# Issue #256: per-peer secret takes precedence over the
|
||||||
peer_key = _derive_peer_key(secret, normalized_peer_url)
|
# global MESH_PEER_PUSH_SECRET. When neither is set the
|
||||||
if not peer_key:
|
# key is empty and we skip the HMAC header entirely so a
|
||||||
raise ValueError("invalid peer URL for HMAC derivation")
|
# bare (unsigned) push still works on test deployments
|
||||||
|
# that have not yet configured any secret at all.
|
||||||
|
peer_key = resolve_peer_key_for_url(normalized_peer_url)
|
||||||
|
if peer_key:
|
||||||
headers["X-Peer-Url"] = normalized_peer_url
|
headers["X-Peer-Url"] = normalized_peer_url
|
||||||
headers["X-Peer-HMAC"] = hmac.new(
|
headers["X-Peer-HMAC"] = hmac.new(
|
||||||
peer_key,
|
peer_key,
|
||||||
@@ -798,7 +804,6 @@ class TorArtiTransport(_PeerPushTransportMixin):
|
|||||||
endpoint_path, padded = self._build_peer_push_request(envelope, self.NAME)
|
endpoint_path, padded = self._build_peer_push_request(envelope, self.NAME)
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
return TransportResult(False, self.NAME, str(exc))
|
return TransportResult(False, self.NAME, str(exc))
|
||||||
secret = str(settings.MESH_PEER_PUSH_SECRET or "").strip()
|
|
||||||
|
|
||||||
delivered = 0
|
delivered = 0
|
||||||
last_error = ""
|
last_error = ""
|
||||||
@@ -808,10 +813,10 @@ class TorArtiTransport(_PeerPushTransportMixin):
|
|||||||
try:
|
try:
|
||||||
normalized_peer_url = normalize_peer_url(peer_url)
|
normalized_peer_url = normalize_peer_url(peer_url)
|
||||||
headers = {"Content-Type": "application/json"}
|
headers = {"Content-Type": "application/json"}
|
||||||
if secret:
|
# Issue #256: per-peer secret takes precedence; see the
|
||||||
peer_key = _derive_peer_key(secret, normalized_peer_url)
|
# other transport above for the rationale.
|
||||||
if not peer_key:
|
peer_key = resolve_peer_key_for_url(normalized_peer_url)
|
||||||
raise ValueError("invalid peer URL for HMAC derivation")
|
if peer_key:
|
||||||
headers["X-Peer-Url"] = normalized_peer_url
|
headers["X-Peer-Url"] = normalized_peer_url
|
||||||
headers["X-Peer-HMAC"] = hmac.new(
|
headers["X-Peer-HMAC"] = hmac.new(
|
||||||
peer_key,
|
peer_key,
|
||||||
|
|||||||
@@ -11,12 +11,13 @@ import base64
|
|||||||
import hmac
|
import hmac
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
import secrets
|
import secrets
|
||||||
import time
|
import time
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from cryptography.hazmat.primitives import serialization
|
from cryptography.hazmat.primitives import serialization
|
||||||
from cryptography.hazmat.primitives.asymmetric import ed25519
|
from cryptography.hazmat.primitives.asymmetric import ed25519, x25519
|
||||||
|
|
||||||
from services.mesh.mesh_crypto import (
|
from services.mesh.mesh_crypto import (
|
||||||
build_signature_payload,
|
build_signature_payload,
|
||||||
@@ -51,6 +52,8 @@ PREKEY_LOOKUP_ROTATE_BEFORE_REMAINING_USES = 8
|
|||||||
PREKEY_LOOKUP_ROTATION_OVERLAP_S = 12 * 60 * 60
|
PREKEY_LOOKUP_ROTATION_OVERLAP_S = 12 * 60 * 60
|
||||||
PREKEY_LOOKUP_ROTATION_ACTIVE_CAP = 4
|
PREKEY_LOOKUP_ROTATION_ACTIVE_CAP = 4
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def _safe_int(val, default=0) -> int:
|
def _safe_int(val, default=0) -> int:
|
||||||
try:
|
try:
|
||||||
@@ -107,6 +110,7 @@ def _default_identity() -> dict[str, Any]:
|
|||||||
def _prekey_lookup_handle_record(
|
def _prekey_lookup_handle_record(
|
||||||
handle: str,
|
handle: str,
|
||||||
*,
|
*,
|
||||||
|
label: str = "",
|
||||||
issued_at: int = 0,
|
issued_at: int = 0,
|
||||||
expires_at: int = 0,
|
expires_at: int = 0,
|
||||||
max_uses: int = 0,
|
max_uses: int = 0,
|
||||||
@@ -125,6 +129,7 @@ def _prekey_lookup_handle_record(
|
|||||||
bounded_max_uses = max(1, _safe_int(max_uses or PREKEY_LOOKUP_HANDLE_MAX_USES, PREKEY_LOOKUP_HANDLE_MAX_USES))
|
bounded_max_uses = max(1, _safe_int(max_uses or PREKEY_LOOKUP_HANDLE_MAX_USES, PREKEY_LOOKUP_HANDLE_MAX_USES))
|
||||||
return {
|
return {
|
||||||
"handle": str(handle or "").strip(),
|
"handle": str(handle or "").strip(),
|
||||||
|
"label": str(label or "").strip()[:96],
|
||||||
"issued_at": issued,
|
"issued_at": issued,
|
||||||
"expires_at": bounded_expires_at,
|
"expires_at": bounded_expires_at,
|
||||||
"max_uses": bounded_max_uses,
|
"max_uses": bounded_max_uses,
|
||||||
@@ -152,8 +157,10 @@ def _coerce_prekey_lookup_handle_record(
|
|||||||
max_uses = _safe_int(value.get("max_uses", PREKEY_LOOKUP_HANDLE_MAX_USES) or PREKEY_LOOKUP_HANDLE_MAX_USES)
|
max_uses = _safe_int(value.get("max_uses", PREKEY_LOOKUP_HANDLE_MAX_USES) or PREKEY_LOOKUP_HANDLE_MAX_USES)
|
||||||
use_count = _safe_int(value.get("use_count", value.get("uses", 0)) or 0, 0)
|
use_count = _safe_int(value.get("use_count", value.get("uses", 0)) or 0, 0)
|
||||||
last_used_at = _safe_int(value.get("last_used_at", value.get("last_used", 0)) or 0, 0)
|
last_used_at = _safe_int(value.get("last_used_at", value.get("last_used", 0)) or 0, 0)
|
||||||
|
label = str(value.get("label", "") or "").strip()
|
||||||
return _prekey_lookup_handle_record(
|
return _prekey_lookup_handle_record(
|
||||||
handle,
|
handle,
|
||||||
|
label=label,
|
||||||
issued_at=issued_at,
|
issued_at=issued_at,
|
||||||
expires_at=expires_at,
|
expires_at=expires_at,
|
||||||
max_uses=max_uses,
|
max_uses=max_uses,
|
||||||
@@ -228,6 +235,23 @@ def _fresh_prekey_lookup_handle_record(*, now: int | None = None) -> dict[str, A
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _prekey_registration_failure_blocks_dm_invite(detail: str) -> bool:
|
||||||
|
"""Only trust-root failures block address export; transport warm-up can finish later."""
|
||||||
|
lowered = str(detail or "").lower()
|
||||||
|
critical_markers = (
|
||||||
|
"root transparency",
|
||||||
|
"external root witness",
|
||||||
|
"stable root",
|
||||||
|
"witness threshold",
|
||||||
|
"witness finality",
|
||||||
|
"root manifest",
|
||||||
|
"root witness",
|
||||||
|
"manifest_fingerprint",
|
||||||
|
"policy fingerprint",
|
||||||
|
)
|
||||||
|
return any(marker in lowered for marker in critical_markers)
|
||||||
|
|
||||||
|
|
||||||
def _bounded_lookup_handle_records(
|
def _bounded_lookup_handle_records(
|
||||||
records: list[dict[str, Any]],
|
records: list[dict[str, Any]],
|
||||||
*,
|
*,
|
||||||
@@ -440,6 +464,37 @@ def _bundle_fingerprint(data: dict[str, Any]) -> str:
|
|||||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_dm_dh_material(data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
||||||
|
"""Repair legacy/corrupt DM identities that kept signing keys but lost DH material."""
|
||||||
|
if str(data.get("dh_pub_key", "") or "").strip() and str(data.get("dh_private_key", "") or "").strip():
|
||||||
|
return data, False
|
||||||
|
|
||||||
|
dh_priv = x25519.X25519PrivateKey.generate()
|
||||||
|
dh_priv_raw = dh_priv.private_bytes(
|
||||||
|
encoding=serialization.Encoding.Raw,
|
||||||
|
format=serialization.PrivateFormat.Raw,
|
||||||
|
encryption_algorithm=serialization.NoEncryption(),
|
||||||
|
)
|
||||||
|
dh_pub_raw = dh_priv.public_key().public_bytes(
|
||||||
|
encoding=serialization.Encoding.Raw,
|
||||||
|
format=serialization.PublicFormat.Raw,
|
||||||
|
)
|
||||||
|
repaired = {
|
||||||
|
**dict(data or {}),
|
||||||
|
"dh_pub_key": base64.b64encode(dh_pub_raw).decode("ascii"),
|
||||||
|
"dh_algo": "X25519",
|
||||||
|
"dh_private_key": base64.b64encode(dh_priv_raw).decode("ascii"),
|
||||||
|
"last_dh_timestamp": int(time.time()),
|
||||||
|
"bundle_fingerprint": "",
|
||||||
|
"bundle_sequence": 0,
|
||||||
|
"bundle_registered_at": 0,
|
||||||
|
"prekey_bundle_registered_at": 0,
|
||||||
|
"prekey_transparency_head": "",
|
||||||
|
"prekey_transparency_size": 0,
|
||||||
|
}
|
||||||
|
return _write_identity(repaired), True
|
||||||
|
|
||||||
|
|
||||||
def trust_fingerprint_for_identity_material(
|
def trust_fingerprint_for_identity_material(
|
||||||
*,
|
*,
|
||||||
agent_id: str,
|
agent_id: str,
|
||||||
@@ -806,10 +861,11 @@ def _sign_dm_invite_payload(
|
|||||||
|
|
||||||
def register_wormhole_dm_key(force: bool = False) -> dict[str, Any]:
|
def register_wormhole_dm_key(force: bool = False) -> dict[str, Any]:
|
||||||
data = read_wormhole_identity()
|
data = read_wormhole_identity()
|
||||||
|
data, repaired_dh = _ensure_dm_dh_material(data)
|
||||||
|
|
||||||
timestamp = int(time.time())
|
timestamp = int(time.time())
|
||||||
fingerprint = _bundle_fingerprint(data)
|
fingerprint = _bundle_fingerprint(data)
|
||||||
if not force and fingerprint and fingerprint == data.get("bundle_fingerprint"):
|
if not force and not repaired_dh and fingerprint and fingerprint == data.get("bundle_fingerprint"):
|
||||||
return {
|
return {
|
||||||
"ok": True,
|
"ok": True,
|
||||||
**_public_view(data),
|
**_public_view(data),
|
||||||
@@ -884,6 +940,7 @@ def export_wormhole_dm_invite(*, label: str = "", expires_in_s: int = 0) -> dict
|
|||||||
existing_handles.append(
|
existing_handles.append(
|
||||||
_prekey_lookup_handle_record(
|
_prekey_lookup_handle_record(
|
||||||
lookup_handle,
|
lookup_handle,
|
||||||
|
label=str(label or "").strip(),
|
||||||
issued_at=issued_at,
|
issued_at=issued_at,
|
||||||
expires_at=expires_at,
|
expires_at=expires_at,
|
||||||
)
|
)
|
||||||
@@ -920,14 +977,25 @@ def export_wormhole_dm_invite(*, label: str = "", expires_in_s: int = 0) -> dict
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
prekey_registration: dict[str, Any] = {"ok": False, "detail": "prekey bundle publish not attempted"}
|
||||||
try:
|
try:
|
||||||
from services.mesh.mesh_wormhole_prekey import register_wormhole_prekey_bundle
|
from services.mesh.mesh_wormhole_prekey import register_wormhole_prekey_bundle
|
||||||
|
|
||||||
registered = register_wormhole_prekey_bundle()
|
prekey_registration = register_wormhole_prekey_bundle()
|
||||||
if not registered.get("ok"):
|
if not prekey_registration.get("ok"):
|
||||||
return {"ok": False, "detail": str(registered.get("detail", "") or "prekey bundle registration failed")}
|
detail = str(prekey_registration.get("detail", "") or "prekey bundle registration failed")
|
||||||
|
if _prekey_registration_failure_blocks_dm_invite(detail):
|
||||||
|
return {"ok": False, "detail": detail}
|
||||||
|
logger.warning(
|
||||||
|
"DM invite prekey publish pending: %s",
|
||||||
|
detail,
|
||||||
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return {"ok": False, "detail": str(exc) or "prekey bundle registration failed"}
|
prekey_registration = {"ok": False, "detail": str(exc) or "prekey bundle registration failed"}
|
||||||
|
detail = str(prekey_registration.get("detail", "") or "")
|
||||||
|
if _prekey_registration_failure_blocks_dm_invite(detail):
|
||||||
|
return {"ok": False, "detail": detail}
|
||||||
|
logger.warning("DM invite prekey publish pending: %s", prekey_registration["detail"])
|
||||||
|
|
||||||
invite_node_id, invite_public_key, invite_private_key = _generate_invite_signing_identity()
|
invite_node_id, invite_public_key, invite_private_key = _generate_invite_signing_identity()
|
||||||
payload = _attach_dm_invite_root_distribution(payload)
|
payload = _attach_dm_invite_root_distribution(payload)
|
||||||
@@ -958,6 +1026,8 @@ def export_wormhole_dm_invite(*, label: str = "", expires_in_s: int = 0) -> dict
|
|||||||
"peer_id": str(invite_node_id or ""),
|
"peer_id": str(invite_node_id or ""),
|
||||||
"trust_fingerprint": str(payload.get("identity_commitment", "") or ""),
|
"trust_fingerprint": str(payload.get("identity_commitment", "") or ""),
|
||||||
"invite": invite,
|
"invite": invite,
|
||||||
|
"prekey_publish_pending": not bool(prekey_registration.get("ok")),
|
||||||
|
"prekey_registration": prekey_registration,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -980,6 +1050,140 @@ def get_prekey_lookup_handle_records() -> list[dict[str, Any]]:
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def list_prekey_lookup_handle_records_for_ui(*, now: int | None = None) -> dict[str, Any]:
|
||||||
|
"""Return shareable DM address records without exposing local identity secrets."""
|
||||||
|
current_time = _safe_int(now or time.time(), int(time.time()))
|
||||||
|
addresses: list[dict[str, Any]] = []
|
||||||
|
for record in get_prekey_lookup_handle_records():
|
||||||
|
handle = str(record.get("handle", "") or "").strip()
|
||||||
|
if not handle:
|
||||||
|
continue
|
||||||
|
expires_at = _effective_prekey_lookup_handle_expires_at(record)
|
||||||
|
max_uses = max(
|
||||||
|
1,
|
||||||
|
_safe_int(
|
||||||
|
record.get("max_uses", PREKEY_LOOKUP_HANDLE_MAX_USES) or PREKEY_LOOKUP_HANDLE_MAX_USES,
|
||||||
|
PREKEY_LOOKUP_HANDLE_MAX_USES,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
use_count = max(0, _safe_int(record.get("use_count", 0) or 0, 0))
|
||||||
|
addresses.append(
|
||||||
|
{
|
||||||
|
"handle": handle,
|
||||||
|
"label": str(record.get("label", "") or "").strip(),
|
||||||
|
"issued_at": _safe_int(record.get("issued_at", 0) or 0, 0),
|
||||||
|
"expires_at": expires_at,
|
||||||
|
"max_uses": max_uses,
|
||||||
|
"use_count": use_count,
|
||||||
|
"remaining_uses": max(0, max_uses - use_count),
|
||||||
|
"last_used_at": _safe_int(record.get("last_used_at", 0) or 0, 0),
|
||||||
|
"expired": bool(expires_at > 0 and current_time >= expires_at),
|
||||||
|
"exhausted": bool(use_count >= max_uses),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
addresses.sort(key=lambda item: _safe_int(item.get("issued_at", 0) or 0, 0), reverse=True)
|
||||||
|
return {"ok": True, "addresses": addresses}
|
||||||
|
|
||||||
|
|
||||||
|
def rename_prekey_lookup_handle(handle: str, label: str) -> dict[str, Any]:
|
||||||
|
"""Rename an active invite-scoped DM lookup handle without changing the handle."""
|
||||||
|
lookup_handle = str(handle or "").strip()
|
||||||
|
next_label = str(label or "").strip()[:96]
|
||||||
|
if not lookup_handle:
|
||||||
|
return {"ok": False, "detail": "missing_lookup_handle"}
|
||||||
|
|
||||||
|
current_time = int(time.time())
|
||||||
|
data = read_wormhole_identity()
|
||||||
|
existing, _ = _normalize_prekey_lookup_handles(
|
||||||
|
data.get("prekey_lookup_handles", []),
|
||||||
|
fallback_issued_at=current_time,
|
||||||
|
now=current_time,
|
||||||
|
)
|
||||||
|
updated = False
|
||||||
|
next_records: list[dict[str, Any]] = []
|
||||||
|
for record in existing:
|
||||||
|
current = dict(record)
|
||||||
|
if str(current.get("handle", "") or "").strip() == lookup_handle:
|
||||||
|
current["label"] = next_label
|
||||||
|
updated = True
|
||||||
|
next_records.append(current)
|
||||||
|
|
||||||
|
if not updated:
|
||||||
|
return {
|
||||||
|
"ok": False,
|
||||||
|
"handle": lookup_handle,
|
||||||
|
"label": next_label,
|
||||||
|
"updated": False,
|
||||||
|
"detail": "lookup_handle_not_found",
|
||||||
|
}
|
||||||
|
|
||||||
|
normalized_records, _ = _normalize_prekey_lookup_handles(
|
||||||
|
next_records,
|
||||||
|
fallback_issued_at=current_time,
|
||||||
|
now=current_time,
|
||||||
|
)
|
||||||
|
_write_identity({"prekey_lookup_handles": normalized_records})
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"handle": lookup_handle,
|
||||||
|
"label": next_label,
|
||||||
|
"updated": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def revoke_prekey_lookup_handle(handle: str) -> dict[str, Any]:
|
||||||
|
"""Revoke an invite-scoped DM lookup handle for future first-contact attempts."""
|
||||||
|
lookup_handle = str(handle or "").strip()
|
||||||
|
if not lookup_handle:
|
||||||
|
return {"ok": False, "detail": "missing_lookup_handle"}
|
||||||
|
current_time = int(time.time())
|
||||||
|
data = read_wormhole_identity()
|
||||||
|
existing, _ = _normalize_prekey_lookup_handles(
|
||||||
|
data.get("prekey_lookup_handles", []),
|
||||||
|
fallback_issued_at=current_time,
|
||||||
|
now=current_time,
|
||||||
|
)
|
||||||
|
next_records = [
|
||||||
|
dict(record)
|
||||||
|
for record in existing
|
||||||
|
if str(record.get("handle", "") or "").strip() != lookup_handle
|
||||||
|
]
|
||||||
|
identity_removed = len(next_records) != len(existing)
|
||||||
|
if identity_removed:
|
||||||
|
_write_identity({"prekey_lookup_handles": next_records})
|
||||||
|
|
||||||
|
relay_removed = False
|
||||||
|
try:
|
||||||
|
from services.mesh.mesh_dm_relay import dm_relay
|
||||||
|
|
||||||
|
relay_removed = bool(dm_relay.unregister_prekey_lookup_alias(lookup_handle))
|
||||||
|
except Exception:
|
||||||
|
relay_removed = False
|
||||||
|
|
||||||
|
republished = False
|
||||||
|
detail = ""
|
||||||
|
if identity_removed:
|
||||||
|
try:
|
||||||
|
from services.mesh.mesh_wormhole_prekey import register_wormhole_prekey_bundle
|
||||||
|
|
||||||
|
registered = register_wormhole_prekey_bundle()
|
||||||
|
republished = bool(registered.get("ok"))
|
||||||
|
if not republished:
|
||||||
|
detail = str(registered.get("detail", "") or "prekey bundle republish failed")
|
||||||
|
except Exception as exc:
|
||||||
|
detail = str(exc) or "prekey bundle republish failed"
|
||||||
|
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"handle": lookup_handle,
|
||||||
|
"revoked": bool(identity_removed or relay_removed),
|
||||||
|
"identity_removed": identity_removed,
|
||||||
|
"relay_removed": relay_removed,
|
||||||
|
"republished": republished,
|
||||||
|
"detail": detail,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def record_prekey_lookup_handle_use(handle: str, *, now: int | None = None) -> dict[str, Any] | None:
|
def record_prekey_lookup_handle_use(handle: str, *, now: int | None = None) -> dict[str, Any] | None:
|
||||||
lookup_handle = str(handle or "").strip()
|
lookup_handle = str(handle or "").strip()
|
||||||
if not lookup_handle:
|
if not lookup_handle:
|
||||||
@@ -999,6 +1203,7 @@ def record_prekey_lookup_handle_use(handle: str, *, now: int | None = None) -> d
|
|||||||
if str(current.get("handle", "") or "").strip() == lookup_handle:
|
if str(current.get("handle", "") or "").strip() == lookup_handle:
|
||||||
current = _prekey_lookup_handle_record(
|
current = _prekey_lookup_handle_record(
|
||||||
lookup_handle,
|
lookup_handle,
|
||||||
|
label=str(current.get("label", "") or "").strip(),
|
||||||
issued_at=_safe_int(current.get("issued_at", 0) or 0, current_time),
|
issued_at=_safe_int(current.get("issued_at", 0) or 0, current_time),
|
||||||
expires_at=_safe_int(current.get("expires_at", 0) or 0, 0),
|
expires_at=_safe_int(current.get("expires_at", 0) or 0, 0),
|
||||||
max_uses=_safe_int(current.get("max_uses", PREKEY_LOOKUP_HANDLE_MAX_USES) or PREKEY_LOOKUP_HANDLE_MAX_USES),
|
max_uses=_safe_int(current.get("max_uses", PREKEY_LOOKUP_HANDLE_MAX_USES) or PREKEY_LOOKUP_HANDLE_MAX_USES),
|
||||||
@@ -1129,6 +1334,7 @@ def maybe_rotate_prekey_lookup_handles(*, now: int | None = None) -> dict[str, A
|
|||||||
candidate_records.append(
|
candidate_records.append(
|
||||||
_prekey_lookup_handle_record(
|
_prekey_lookup_handle_record(
|
||||||
old_handle,
|
old_handle,
|
||||||
|
label=str(record.get("label", "") or "").strip(),
|
||||||
issued_at=_safe_int(record.get("issued_at", 0) or 0, current_time),
|
issued_at=_safe_int(record.get("issued_at", 0) or 0, current_time),
|
||||||
expires_at=overlap_expires_at,
|
expires_at=overlap_expires_at,
|
||||||
max_uses=_safe_int(record.get("max_uses", PREKEY_LOOKUP_HANDLE_MAX_USES) or PREKEY_LOOKUP_HANDLE_MAX_USES),
|
max_uses=_safe_int(record.get("max_uses", PREKEY_LOOKUP_HANDLE_MAX_USES) or PREKEY_LOOKUP_HANDLE_MAX_USES),
|
||||||
@@ -1351,11 +1557,101 @@ def import_wormhole_dm_invite(invite: dict[str, Any], *, alias: str = "") -> dic
|
|||||||
"detail": "compat dm invite import disabled; ask the sender to re-export a current signed invite",
|
"detail": "compat dm invite import disabled; ask the sender to re-export a current signed invite",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def _prekey_missing_or_pending(detail: str) -> bool:
|
||||||
|
lower = str(detail or "").strip().lower()
|
||||||
|
return any(
|
||||||
|
phrase in lower
|
||||||
|
for phrase in (
|
||||||
|
"prekey bundle not found",
|
||||||
|
"invite prekey bundle not found",
|
||||||
|
"peer prekey lookup unavailable",
|
||||||
|
"peer prekey lookup still preparing",
|
||||||
|
"transport tier insufficient",
|
||||||
|
"preparing_private_lane",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _pin_pending_invite_prekey(detail: str) -> dict[str, Any]:
|
||||||
|
if invite_version < DM_INVITE_VERSION:
|
||||||
|
return {"ok": False, "detail": detail or "invite prekey bundle not found"}
|
||||||
|
invite_root_distribution = _verify_dm_invite_root_distribution(payload)
|
||||||
|
if not invite_root_distribution.get("ok"):
|
||||||
|
return invite_root_distribution
|
||||||
|
attested = _verify_dm_invite_identity_attestation(
|
||||||
|
envelope=envelope,
|
||||||
|
payload=payload,
|
||||||
|
resolved_root_node_id=str(invite_root_distribution.get("root_node_id", "") or ""),
|
||||||
|
resolved_root_public_key=str(invite_root_distribution.get("root_public_key", "") or ""),
|
||||||
|
resolved_root_public_key_algo=str(
|
||||||
|
invite_root_distribution.get("root_public_key_algo", "Ed25519") or "Ed25519"
|
||||||
|
),
|
||||||
|
resolved_root_manifest_fingerprint=str(
|
||||||
|
invite_root_distribution.get("root_manifest_fingerprint", "") or ""
|
||||||
|
).strip().lower(),
|
||||||
|
)
|
||||||
|
if not attested.get("ok"):
|
||||||
|
return attested
|
||||||
|
pending_peer_id = str(verified.get("peer_id", "") or "").strip()
|
||||||
|
trust_fingerprint = str(verified.get("trust_fingerprint", "") or "").strip().lower()
|
||||||
|
contact = pin_wormhole_dm_invite(
|
||||||
|
pending_peer_id,
|
||||||
|
invite_payload={
|
||||||
|
"trust_fingerprint": trust_fingerprint,
|
||||||
|
"public_key": "",
|
||||||
|
"public_key_algo": "Ed25519",
|
||||||
|
"identity_dh_pub_key": "",
|
||||||
|
"dh_algo": "X25519",
|
||||||
|
"prekey_lookup_handle": lookup_handle,
|
||||||
|
"issued_at": int(payload.get("issued_at", 0) or 0),
|
||||||
|
"expires_at": int(payload.get("expires_at", 0) or 0),
|
||||||
|
"label": str(payload.get("label", "") or ""),
|
||||||
|
"root_node_id": str(attested.get("root_node_id", "") or ""),
|
||||||
|
"root_public_key": str(attested.get("root_public_key", "") or ""),
|
||||||
|
"root_public_key_algo": str(attested.get("root_public_key_algo", "Ed25519") or "Ed25519"),
|
||||||
|
"root_fingerprint": str(attested.get("root_fingerprint", "") or ""),
|
||||||
|
"root_manifest_fingerprint": str(invite_root_distribution.get("root_manifest_fingerprint", "") or ""),
|
||||||
|
"root_witness_policy_fingerprint": str(
|
||||||
|
invite_root_distribution.get("root_witness_policy_fingerprint", "") or ""
|
||||||
|
),
|
||||||
|
"root_witness_threshold": _safe_int(
|
||||||
|
invite_root_distribution.get("root_witness_threshold", 0) or 0,
|
||||||
|
0,
|
||||||
|
),
|
||||||
|
"root_witness_count": _safe_int(invite_root_distribution.get("root_witness_count", 0) or 0, 0),
|
||||||
|
"root_witness_domain_count": _safe_int(
|
||||||
|
invite_root_distribution.get("root_witness_domain_count", 0) or 0,
|
||||||
|
0,
|
||||||
|
),
|
||||||
|
"root_manifest_generation": _safe_int(
|
||||||
|
invite_root_distribution.get("root_manifest_generation", 0) or 0,
|
||||||
|
0,
|
||||||
|
),
|
||||||
|
"root_rotation_proven": bool(invite_root_distribution.get("root_rotation_proven")),
|
||||||
|
},
|
||||||
|
alias=resolved_alias,
|
||||||
|
attested=True,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"peer_id": pending_peer_id,
|
||||||
|
"invite_peer_id": pending_peer_id,
|
||||||
|
"trust_fingerprint": trust_fingerprint,
|
||||||
|
"trust_level": str(contact.get("trust_level", "") or ""),
|
||||||
|
"detail": "Contact saved.",
|
||||||
|
"invite_attested": True,
|
||||||
|
"pending_prekey": True,
|
||||||
|
"prekey_detail": detail or "invite prekey bundle not found",
|
||||||
|
"contact": contact,
|
||||||
|
}
|
||||||
|
|
||||||
from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle
|
from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle
|
||||||
|
|
||||||
fetched = fetch_dm_prekey_bundle(lookup_token=lookup_handle)
|
fetched = fetch_dm_prekey_bundle(lookup_token=lookup_handle)
|
||||||
if not fetched.get("ok"):
|
if not fetched.get("ok"):
|
||||||
return {"ok": False, "detail": str(fetched.get("detail", "") or "invite prekey bundle not found")}
|
fetch_detail = str(fetched.get("detail", "") or "invite prekey bundle not found")
|
||||||
|
if _prekey_missing_or_pending(fetch_detail):
|
||||||
|
return _pin_pending_invite_prekey(fetch_detail)
|
||||||
|
return {"ok": False, "detail": fetch_detail}
|
||||||
|
|
||||||
resolved_peer_id = str(fetched.get("agent_id", "") or "").strip()
|
resolved_peer_id = str(fetched.get("agent_id", "") or "").strip()
|
||||||
if not resolved_peer_id:
|
if not resolved_peer_id:
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import os
|
|||||||
import random
|
import random
|
||||||
import time
|
import time
|
||||||
import urllib.error
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
import urllib.request
|
import urllib.request
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -90,13 +91,15 @@ def _fetch_dm_prekey_bundle_from_peer_lookup(lookup_token: str) -> dict[str, Any
|
|||||||
return {"ok": False, "detail": "lookup token required"}
|
return {"ok": False, "detail": "lookup token required"}
|
||||||
try:
|
try:
|
||||||
from services.config import get_settings
|
from services.config import get_settings
|
||||||
from services.mesh.mesh_crypto import _derive_peer_key, normalize_peer_url
|
from services.mesh.mesh_crypto import (
|
||||||
|
normalize_peer_url,
|
||||||
|
resolve_peer_key_for_url,
|
||||||
|
)
|
||||||
from services.mesh.mesh_router import configured_relay_peer_urls
|
from services.mesh.mesh_router import configured_relay_peer_urls
|
||||||
|
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
secret = str(getattr(settings, "MESH_PEER_PUSH_SECRET", "") or "").strip()
|
# Issue #256: secret check moved per-peer below. We still bail out
|
||||||
if not secret:
|
# cleanly when there are no peers configured at all.
|
||||||
return {"ok": False, "detail": "peer prekey lookup unavailable"}
|
|
||||||
peers = configured_relay_peer_urls()
|
peers = configured_relay_peer_urls()
|
||||||
if not peers:
|
if not peers:
|
||||||
return {"ok": False, "detail": "peer prekey lookup unavailable"}
|
return {"ok": False, "detail": "peer prekey lookup unavailable"}
|
||||||
@@ -120,7 +123,8 @@ def _fetch_dm_prekey_bundle_from_peer_lookup(lookup_token: str) -> dict[str, Any
|
|||||||
or os.environ.get("SB_TEST_NODE_URL", "").strip()
|
or os.environ.get("SB_TEST_NODE_URL", "").strip()
|
||||||
or normalized_peer_url
|
or normalized_peer_url
|
||||||
)
|
)
|
||||||
peer_key = _derive_peer_key(secret, sender_peer_url)
|
# Issue #256: prefer per-peer secret keyed by the sender URL.
|
||||||
|
peer_key = resolve_peer_key_for_url(sender_peer_url)
|
||||||
if not peer_key:
|
if not peer_key:
|
||||||
continue
|
continue
|
||||||
headers = {
|
headers = {
|
||||||
@@ -150,6 +154,122 @@ def _fetch_dm_prekey_bundle_from_peer_lookup(lookup_token: str) -> dict[str, Any
|
|||||||
return {"ok": False, "detail": last_detail or "Prekey bundle not found"}
|
return {"ok": False, "detail": last_detail or "Prekey bundle not found"}
|
||||||
|
|
||||||
|
|
||||||
|
def _configured_public_lookup_peer_urls() -> list[str]:
|
||||||
|
try:
|
||||||
|
from services.config import get_settings
|
||||||
|
from services.mesh.mesh_router import active_sync_peer_urls, parse_configured_relay_peers
|
||||||
|
|
||||||
|
settings = get_settings()
|
||||||
|
candidates: list[str] = []
|
||||||
|
for raw in (
|
||||||
|
getattr(settings, "MESH_BOOTSTRAP_SEED_PEERS", ""),
|
||||||
|
getattr(settings, "MESH_DEFAULT_SYNC_PEERS", ""),
|
||||||
|
):
|
||||||
|
candidates.extend(parse_configured_relay_peers(str(raw or "")))
|
||||||
|
candidates.extend(active_sync_peer_urls())
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
|
||||||
|
seen: set[str] = set()
|
||||||
|
peers: list[str] = []
|
||||||
|
for candidate in candidates:
|
||||||
|
peer = str(candidate or "").strip().rstrip("/")
|
||||||
|
if not peer or peer in seen:
|
||||||
|
continue
|
||||||
|
seen.add(peer)
|
||||||
|
peers.append(peer)
|
||||||
|
return peers
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_remote_lookup_bundle(payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
data = dict(payload or {})
|
||||||
|
bundle = dict(data.get("bundle") or {})
|
||||||
|
public_key = str(data.get("public_key", "") or bundle.get("public_key", "") or "").strip()
|
||||||
|
if not public_key:
|
||||||
|
return {"ok": False, "detail": "Prekey bundle missing signing key"}
|
||||||
|
agent_id = str(data.get("agent_id", "") or "").strip() or derive_node_id(public_key)
|
||||||
|
if not agent_id:
|
||||||
|
return {"ok": False, "detail": "Prekey bundle public key binding mismatch"}
|
||||||
|
data["agent_id"] = agent_id
|
||||||
|
data["public_key"] = public_key
|
||||||
|
data["public_key_algo"] = str(data.get("public_key_algo", "") or bundle.get("public_key_algo", "Ed25519") or "Ed25519")
|
||||||
|
data["protocol_version"] = str(data.get("protocol_version", "") or bundle.get("protocol_version", PROTOCOL_VERSION) or PROTOCOL_VERSION)
|
||||||
|
data["bundle"] = bundle
|
||||||
|
ok, reason = _validate_bundle_record(data)
|
||||||
|
if not ok:
|
||||||
|
return {"ok": False, "detail": reason}
|
||||||
|
data["ok"] = True
|
||||||
|
data["lookup_mode"] = "invite_lookup_handle"
|
||||||
|
data["public_lookup"] = True
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_dm_prekey_bundle_from_public_lookup(lookup_token: str) -> dict[str, Any]:
|
||||||
|
"""Fetch an invite-scoped prekey bundle from bootstrap/sync peers.
|
||||||
|
|
||||||
|
The token is high-entropy and invite-scoped. This path does not expose a
|
||||||
|
stable agent_id to the peer; if the ordinary peer response omits agent_id,
|
||||||
|
derive it from the signed identity public key and validate the bundle before
|
||||||
|
accepting it.
|
||||||
|
"""
|
||||||
|
token = str(lookup_token or "").strip()
|
||||||
|
if not token:
|
||||||
|
return {"ok": False, "detail": "lookup token required"}
|
||||||
|
peers = _configured_public_lookup_peer_urls()
|
||||||
|
if not peers:
|
||||||
|
return {"ok": False, "detail": "peer prekey lookup unavailable"}
|
||||||
|
try:
|
||||||
|
from services.config import get_settings
|
||||||
|
|
||||||
|
timeout = max(1, _safe_int(getattr(get_settings(), "MESH_SYNC_TIMEOUT_S", 5) or 5, 5))
|
||||||
|
except Exception:
|
||||||
|
timeout = 5
|
||||||
|
|
||||||
|
encoded = urllib.parse.urlencode({"lookup_token": token})
|
||||||
|
last_detail = ""
|
||||||
|
for peer_url in peers:
|
||||||
|
normalized_peer_url = str(peer_url or "").strip().rstrip("/")
|
||||||
|
if not normalized_peer_url:
|
||||||
|
continue
|
||||||
|
# Generic UA: any peer-facing crypto request should not carry a
|
||||||
|
# fork-specific identifier — that turns prekey lookups into a
|
||||||
|
# software-fingerprinting beacon.
|
||||||
|
from services.network_utils import DEFAULT_USER_AGENT
|
||||||
|
request = urllib.request.Request(
|
||||||
|
f"{normalized_peer_url}/api/mesh/dm/prekey-bundle?{encoded}",
|
||||||
|
headers={
|
||||||
|
"Accept": "application/json",
|
||||||
|
"User-Agent": DEFAULT_USER_AGENT,
|
||||||
|
},
|
||||||
|
method="GET",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||||
|
raw = response.read(256 * 1024)
|
||||||
|
payload = json.loads(raw.decode("utf-8"))
|
||||||
|
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, OSError) as exc:
|
||||||
|
logger.debug("public prekey lookup failed for %s: %s", normalized_peer_url, type(exc).__name__)
|
||||||
|
last_detail = "peer prekey lookup unavailable"
|
||||||
|
continue
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
last_detail = "invalid peer response"
|
||||||
|
continue
|
||||||
|
if payload.get("pending") or str(payload.get("status", "") or "") == "preparing_private_lane":
|
||||||
|
last_detail = "peer prekey lookup still preparing"
|
||||||
|
continue
|
||||||
|
if not payload.get("ok"):
|
||||||
|
last_detail = str(payload.get("detail", "") or last_detail or "Prekey bundle not found")
|
||||||
|
continue
|
||||||
|
if not isinstance(payload.get("bundle"), dict):
|
||||||
|
last_detail = "Prekey bundle not found"
|
||||||
|
continue
|
||||||
|
normalized = _normalize_remote_lookup_bundle(payload)
|
||||||
|
if normalized.get("ok"):
|
||||||
|
return normalized
|
||||||
|
last_detail = str(normalized.get("detail", "") or last_detail)
|
||||||
|
return {"ok": False, "detail": last_detail or "Prekey bundle not found"}
|
||||||
|
|
||||||
|
|
||||||
def _b64(data: bytes) -> str:
|
def _b64(data: bytes) -> str:
|
||||||
return base64.b64encode(data).decode("ascii")
|
return base64.b64encode(data).decode("ascii")
|
||||||
|
|
||||||
@@ -926,6 +1046,11 @@ def fetch_dm_prekey_bundle(
|
|||||||
peer_found = _fetch_dm_prekey_bundle_from_peer_lookup(resolved_lookup)
|
peer_found = _fetch_dm_prekey_bundle_from_peer_lookup(resolved_lookup)
|
||||||
if peer_found.get("ok"):
|
if peer_found.get("ok"):
|
||||||
return peer_found
|
return peer_found
|
||||||
|
public_found = _fetch_dm_prekey_bundle_from_public_lookup(resolved_lookup)
|
||||||
|
if public_found.get("ok"):
|
||||||
|
return public_found
|
||||||
|
if str(public_found.get("detail", "") or "").strip():
|
||||||
|
return {"ok": False, "detail": str(public_found.get("detail", "") or "Prekey bundle not found")}
|
||||||
return {"ok": False, "detail": str(peer_found.get("detail", "") or "Prekey bundle not found")}
|
return {"ok": False, "detail": str(peer_found.get("detail", "") or "Prekey bundle not found")}
|
||||||
else:
|
else:
|
||||||
return {"ok": False, "detail": "Prekey bundle not found"}
|
return {"ok": False, "detail": "Prekey bundle not found"}
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ from __future__ import annotations
|
|||||||
import base64
|
import base64
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
@@ -23,7 +24,7 @@ from cryptography.hazmat.primitives.asymmetric import ed25519
|
|||||||
|
|
||||||
from services.mesh.mesh_crypto import build_signature_payload, derive_node_id, verify_node_binding, verify_signature
|
from services.mesh.mesh_crypto import build_signature_payload, derive_node_id, verify_node_binding, verify_signature
|
||||||
from services.mesh.mesh_protocol import PROTOCOL_VERSION
|
from services.mesh.mesh_protocol import PROTOCOL_VERSION
|
||||||
from services.mesh.mesh_secure_storage import read_domain_json, write_domain_json
|
from services.mesh.mesh_secure_storage import SecureStorageError, read_domain_json, write_domain_json
|
||||||
from services.mesh.mesh_wormhole_identity import root_identity_fingerprint_for_material
|
from services.mesh.mesh_wormhole_identity import root_identity_fingerprint_for_material
|
||||||
from services.mesh.mesh_wormhole_persona import (
|
from services.mesh.mesh_wormhole_persona import (
|
||||||
bootstrap_wormhole_persona_state,
|
bootstrap_wormhole_persona_state,
|
||||||
@@ -51,6 +52,7 @@ DEFAULT_ROOT_WITNESS_THRESHOLD = 2
|
|||||||
DEFAULT_ROOT_WITNESS_MANAGEMENT_SCOPE = "local"
|
DEFAULT_ROOT_WITNESS_MANAGEMENT_SCOPE = "local"
|
||||||
DEFAULT_ROOT_WITNESS_INDEPENDENCE_GROUP = "local_system"
|
DEFAULT_ROOT_WITNESS_INDEPENDENCE_GROUP = "local_system"
|
||||||
DEFAULT_ROOT_EXTERNAL_WITNESS_MAX_AGE_S = 3600
|
DEFAULT_ROOT_EXTERNAL_WITNESS_MAX_AGE_S = 3600
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def _safe_int(val: Any, default: int = 0) -> int:
|
def _safe_int(val: Any, default: int = 0) -> int:
|
||||||
@@ -461,12 +463,22 @@ def witness_policy_fingerprint(policy: dict[str, Any]) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def read_root_distribution_state() -> dict[str, Any]:
|
def read_root_distribution_state() -> dict[str, Any]:
|
||||||
raw = read_domain_json(
|
try:
|
||||||
ROOT_DISTRIBUTION_DOMAIN,
|
raw = read_domain_json(
|
||||||
ROOT_DISTRIBUTION_FILE,
|
ROOT_DISTRIBUTION_DOMAIN,
|
||||||
_default_state,
|
ROOT_DISTRIBUTION_FILE,
|
||||||
base_dir=DATA_DIR,
|
_default_state,
|
||||||
)
|
base_dir=DATA_DIR,
|
||||||
|
)
|
||||||
|
except SecureStorageError as exc:
|
||||||
|
detail = str(exc)
|
||||||
|
if "Failed to decrypt domain JSON" not in detail:
|
||||||
|
raise
|
||||||
|
logger.warning(
|
||||||
|
"Root distribution state could not decrypt; regenerating local witness distribution: %s",
|
||||||
|
detail,
|
||||||
|
)
|
||||||
|
raw = _default_state()
|
||||||
state = {**_default_state(), **dict(raw or {})}
|
state = {**_default_state(), **dict(raw or {})}
|
||||||
state["witness_identity"] = {**_empty_witness_identity(), **dict(state.get("witness_identity") or {})}
|
state["witness_identity"] = {**_empty_witness_identity(), **dict(state.get("witness_identity") or {})}
|
||||||
witness_identities, witness_changed = _normalize_witness_identities(
|
witness_identities, witness_changed = _normalize_witness_identities(
|
||||||
|
|||||||
@@ -108,8 +108,18 @@ def normalize_topic_filter(value: str) -> str | None:
|
|||||||
return "/".join(parts)
|
return "/".join(parts)
|
||||||
|
|
||||||
|
|
||||||
def _default_topic_for_root(root: str) -> str:
|
def _default_topics_for_root(root: str) -> list[str]:
|
||||||
return f"msh/{root}/2/e/{DEFAULT_CHANNEL}/#"
|
"""Return the default LongFast subscriptions for a region root.
|
||||||
|
|
||||||
|
The public broker carries protobuf/encrypted traffic under ``/e/`` and
|
||||||
|
companion decoded JSON traffic under ``/json/``. Positions often arrive on
|
||||||
|
the protobuf path, while public text is commonly easiest to observe on the
|
||||||
|
JSON path.
|
||||||
|
"""
|
||||||
|
return [
|
||||||
|
f"msh/{root}/2/e/{DEFAULT_CHANNEL}/#",
|
||||||
|
f"msh/{root}/2/json/{DEFAULT_CHANNEL}/#",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def build_subscription_topics(
|
def build_subscription_topics(
|
||||||
@@ -124,7 +134,11 @@ def build_subscription_topics(
|
|||||||
# via MESH_MQTT_EXTRA_ROOTS to avoid flooding the public broker.
|
# via MESH_MQTT_EXTRA_ROOTS to avoid flooding the public broker.
|
||||||
roots.extend(root for root in (normalize_root(item) for item in _split_config_values(extra_roots)) if root)
|
roots.extend(root for root in (normalize_root(item) for item in _split_config_values(extra_roots)) if root)
|
||||||
|
|
||||||
topics = [_default_topic_for_root(root) for root in _dedupe(roots)]
|
topics = [
|
||||||
|
topic
|
||||||
|
for root in _dedupe(roots)
|
||||||
|
for topic in _default_topics_for_root(root)
|
||||||
|
]
|
||||||
topics.extend(
|
topics.extend(
|
||||||
topic
|
topic
|
||||||
for topic in (
|
for topic in (
|
||||||
|
|||||||
@@ -5,7 +5,9 @@ import subprocess
|
|||||||
import shutil
|
import shutil
|
||||||
import time
|
import time
|
||||||
import threading
|
import threading
|
||||||
|
import uuid
|
||||||
import requests
|
import requests
|
||||||
|
from pathlib import Path
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
from requests.adapters import HTTPAdapter
|
from requests.adapters import HTTPAdapter
|
||||||
from urllib3.util.retry import Retry
|
from urllib3.util.retry import Retry
|
||||||
@@ -19,6 +21,214 @@ _retry = Retry(total=1, backoff_factor=0.3, status_forcelist=[502, 503, 504])
|
|||||||
_session.mount("https://", HTTPAdapter(max_retries=_retry, pool_maxsize=20))
|
_session.mount("https://", HTTPAdapter(max_retries=_retry, pool_maxsize=20))
|
||||||
_session.mount("http://", HTTPAdapter(max_retries=_retry, pool_maxsize=10))
|
_session.mount("http://", HTTPAdapter(max_retries=_retry, pool_maxsize=10))
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Per-operator outbound identification
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
#
|
||||||
|
# Issues #289 / #290 / #291 and the retrofit of PR #284 (#218 / #219 / #220):
|
||||||
|
# every third-party API the backend calls used to identify itself with a
|
||||||
|
# single "Shadowbroker" aggregate User-Agent. From the upstream's
|
||||||
|
# perspective, that meant every Shadowbroker install in the world looked
|
||||||
|
# like one giant entity hammering them. If one install misbehaved, the
|
||||||
|
# upstream's only recourse was to block "Shadowbroker" as a whole — which
|
||||||
|
# would take out every other install too.
|
||||||
|
#
|
||||||
|
# Fix: give each install a stable pseudonymous handle and include it in
|
||||||
|
# the User-Agent. Now an upstream can rate-limit or block the offending
|
||||||
|
# operator without affecting anyone else.
|
||||||
|
#
|
||||||
|
# The handle:
|
||||||
|
#
|
||||||
|
# - Is auto-generated on first call if no `OPERATOR_HANDLE` is configured
|
||||||
|
# (looks like "operator-7f3a92" — 6 hex chars from uuid4()).
|
||||||
|
# - Is persisted to ``backend/data/operator_handle.json`` so it survives
|
||||||
|
# restarts. Under Docker compose that file lives in the volume mount
|
||||||
|
# alongside `carrier_cache.json` and the other persistent state.
|
||||||
|
# - Can be overridden by the operator via the `OPERATOR_HANDLE` setting
|
||||||
|
# (env var or settings UI). Operators with their own GitHub handle,
|
||||||
|
# organization name, etc. can use that for traceability.
|
||||||
|
# - Is NEVER mixed into mesh / Wormhole / Infonet identity. This layer is
|
||||||
|
# strictly for public third-party API attribution.
|
||||||
|
|
||||||
|
_SHADOWBROKER_VERSION = "0.9"
|
||||||
|
_OPERATOR_HANDLE_FILE = (
|
||||||
|
Path(__file__).parent.parent / "data" / "operator_handle.json"
|
||||||
|
)
|
||||||
|
_OPERATOR_HANDLE_CACHE: str = ""
|
||||||
|
_OPERATOR_HANDLE_LOCK = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def _generate_operator_handle() -> str:
|
||||||
|
"""Produce a stable pseudonymous handle for first-launch installs.
|
||||||
|
|
||||||
|
Format: ``operator-7f3a92`` (6 hex chars from a fresh uuid4()).
|
||||||
|
Distinct per install. Carries no real-world identity by default —
|
||||||
|
operators who want one can override via ``OPERATOR_HANDLE``.
|
||||||
|
|
||||||
|
Note: the prefix is deliberately neutral. Earlier drafts used
|
||||||
|
``shadow-`` which, while accurate to the project name, looks
|
||||||
|
exactly like the kind of pattern a third-party abuse-detection
|
||||||
|
system would auto-block as suspicious. ``operator-`` describes
|
||||||
|
what the value actually is and doesn't pattern-match malware.
|
||||||
|
"""
|
||||||
|
return f"operator-{uuid.uuid4().hex[:6]}"
|
||||||
|
|
||||||
|
|
||||||
|
def _load_persisted_operator_handle() -> str:
|
||||||
|
"""Return the previously-saved handle from disk, or empty if none.
|
||||||
|
|
||||||
|
Reads ``backend/data/operator_handle.json`` if it exists. Any read
|
||||||
|
error returns empty so a fresh handle gets generated rather than
|
||||||
|
crashing the request.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
if _OPERATOR_HANDLE_FILE.exists():
|
||||||
|
data = json.loads(_OPERATOR_HANDLE_FILE.read_text(encoding="utf-8"))
|
||||||
|
return str(data.get("handle", "") or "").strip()
|
||||||
|
except (OSError, json.JSONDecodeError, ValueError):
|
||||||
|
pass
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def _persist_operator_handle(handle: str) -> None:
|
||||||
|
"""Atomically save the auto-generated handle so subsequent restarts
|
||||||
|
use the same one. Failure to persist is non-fatal — the request still
|
||||||
|
succeeds with the in-memory handle, we just may generate a different
|
||||||
|
one on the next process restart."""
|
||||||
|
try:
|
||||||
|
_OPERATOR_HANDLE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
tmp = _OPERATOR_HANDLE_FILE.with_suffix(_OPERATOR_HANDLE_FILE.suffix + ".tmp")
|
||||||
|
tmp.write_text(
|
||||||
|
json.dumps({"handle": handle, "_meta": {
|
||||||
|
"purpose": "Per-install operator handle for outbound third-party API attribution.",
|
||||||
|
"see": "backend/services/network_utils.py:outbound_user_agent",
|
||||||
|
}}, indent=2),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
os.replace(tmp, _OPERATOR_HANDLE_FILE)
|
||||||
|
except OSError as exc:
|
||||||
|
logger.debug("Could not persist operator_handle (continuing in-memory): %s", exc)
|
||||||
|
|
||||||
|
|
||||||
|
def get_operator_handle() -> str:
|
||||||
|
"""Return the stable per-install operator handle.
|
||||||
|
|
||||||
|
Resolution order:
|
||||||
|
1. ``OPERATOR_HANDLE`` setting (env var / settings UI) if non-empty.
|
||||||
|
2. Process-cached value from previous call this run.
|
||||||
|
3. Value persisted to ``operator_handle.json`` (from a previous run).
|
||||||
|
4. Newly generated pseudonymous handle, persisted to disk.
|
||||||
|
|
||||||
|
The handle is normalized: stripped of whitespace, lowercased,
|
||||||
|
non-alphanumeric chars (except ``-`` and ``_``) replaced with ``-``.
|
||||||
|
This both sanitizes any HTTP-header-unsafe characters AND prevents
|
||||||
|
the operator from impersonating real third-party projects via
|
||||||
|
inventive whitespace.
|
||||||
|
"""
|
||||||
|
global _OPERATOR_HANDLE_CACHE
|
||||||
|
with _OPERATOR_HANDLE_LOCK:
|
||||||
|
# 1. Configured override always wins.
|
||||||
|
configured = ""
|
||||||
|
try:
|
||||||
|
from services.config import get_settings
|
||||||
|
|
||||||
|
configured = str(getattr(get_settings(), "OPERATOR_HANDLE", "") or "").strip()
|
||||||
|
except Exception:
|
||||||
|
configured = ""
|
||||||
|
if configured:
|
||||||
|
return _normalize_handle(configured)
|
||||||
|
|
||||||
|
# 2. In-memory cache (fast path for repeated calls).
|
||||||
|
if _OPERATOR_HANDLE_CACHE:
|
||||||
|
return _OPERATOR_HANDLE_CACHE
|
||||||
|
|
||||||
|
# 3. On-disk handle from a previous run.
|
||||||
|
persisted = _load_persisted_operator_handle()
|
||||||
|
if persisted:
|
||||||
|
_OPERATOR_HANDLE_CACHE = _normalize_handle(persisted)
|
||||||
|
return _OPERATOR_HANDLE_CACHE
|
||||||
|
|
||||||
|
# 4. Generate, persist, return.
|
||||||
|
fresh = _generate_operator_handle()
|
||||||
|
_persist_operator_handle(fresh)
|
||||||
|
_OPERATOR_HANDLE_CACHE = fresh
|
||||||
|
return fresh
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_handle(raw: str) -> str:
|
||||||
|
"""Strip whitespace, lowercase, replace unsafe characters with dashes."""
|
||||||
|
safe = "".join(
|
||||||
|
ch if (ch.isalnum() or ch in "-_") else "-"
|
||||||
|
for ch in raw.strip().lower()
|
||||||
|
)
|
||||||
|
# Collapse runs of dashes and trim to a reasonable length so an
|
||||||
|
# operator can't make our outbound logs unreadable.
|
||||||
|
while "--" in safe:
|
||||||
|
safe = safe.replace("--", "-")
|
||||||
|
safe = safe.strip("-")
|
||||||
|
return safe[:48] if safe else "anonymous"
|
||||||
|
|
||||||
|
|
||||||
|
_CONTACT_URL = "https://github.com/BigBodyCobain/Shadowbroker/issues"
|
||||||
|
|
||||||
|
|
||||||
|
def outbound_user_agent(purpose: str = "") -> str:
|
||||||
|
"""Build a User-Agent for an outbound third-party HTTP request.
|
||||||
|
|
||||||
|
Returns something like::
|
||||||
|
|
||||||
|
Shadowbroker/0.9 (operator: shadow-7f3a92; purpose: wikipedia;
|
||||||
|
+https://github.com/BigBodyCobain/Shadowbroker/issues)
|
||||||
|
|
||||||
|
The ``purpose`` is optional but recommended — it tells the upstream
|
||||||
|
what feature of ours is making the call (``wikipedia``, ``openmhz``,
|
||||||
|
``nominatim``, etc.), which makes their logs and our complaints
|
||||||
|
actionable.
|
||||||
|
|
||||||
|
Every outbound call in the backend that previously sent a custom
|
||||||
|
User-Agent should call this helper instead. Centralizing here means:
|
||||||
|
- one place to change the contact URL,
|
||||||
|
- one place to bump the version on release,
|
||||||
|
- one place a Wikimedia / OpenMHz operator can reach to ask for
|
||||||
|
the project to back off, with a per-install handle so they can
|
||||||
|
target the specific install instead of the project as a whole.
|
||||||
|
"""
|
||||||
|
handle = get_operator_handle()
|
||||||
|
if purpose:
|
||||||
|
purpose_clean = _normalize_handle(purpose)
|
||||||
|
return (
|
||||||
|
f"Shadowbroker/{_SHADOWBROKER_VERSION} "
|
||||||
|
f"(operator: {handle}; purpose: {purpose_clean}; +{_CONTACT_URL})"
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
f"Shadowbroker/{_SHADOWBROKER_VERSION} "
|
||||||
|
f"(operator: {handle}; +{_CONTACT_URL})"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_operator_handle_cache_for_tests() -> None:
|
||||||
|
"""Test-only: invalidate the in-memory cache so a test can set a
|
||||||
|
new ``OPERATOR_HANDLE`` env var and see it picked up immediately."""
|
||||||
|
global _OPERATOR_HANDLE_CACHE
|
||||||
|
with _OPERATOR_HANDLE_LOCK:
|
||||||
|
_OPERATOR_HANDLE_CACHE = ""
|
||||||
|
|
||||||
|
|
||||||
|
# Default outbound User-Agent. Retained for backwards compatibility with
|
||||||
|
# call sites that haven't been migrated to ``outbound_user_agent()`` yet.
|
||||||
|
# Operators who want full per-install attribution should set the
|
||||||
|
# ``OPERATOR_HANDLE`` setting and migrate call sites incrementally.
|
||||||
|
#
|
||||||
|
# Operators who run a public-facing relay can also override the whole UA
|
||||||
|
# string via the ``SHADOWBROKER_USER_AGENT`` env var. That override
|
||||||
|
# completely bypasses the per-operator helper; only use it if you know
|
||||||
|
# what you're doing.
|
||||||
|
DEFAULT_USER_AGENT = os.environ.get(
|
||||||
|
"SHADOWBROKER_USER_AGENT",
|
||||||
|
f"Shadowbroker/{_SHADOWBROKER_VERSION}",
|
||||||
|
)
|
||||||
|
|
||||||
# Find bash for curl fallback — Git bash's curl has the TLS features
|
# Find bash for curl fallback — Git bash's curl has the TLS features
|
||||||
# needed to pass CDN fingerprint checks (brotli, zstd, libpsl)
|
# needed to pass CDN fingerprint checks (brotli, zstd, libpsl)
|
||||||
|
|
||||||
@@ -73,7 +283,7 @@ def fetch_with_curl(url, method="GET", json_data=None, timeout=15, headers=None,
|
|||||||
both Python requests and the barebones Windows system curl.
|
both Python requests and the barebones Windows system curl.
|
||||||
"""
|
"""
|
||||||
default_headers = {
|
default_headers = {
|
||||||
"User-Agent": "ShadowBroker-OSINT/0.9.75 (+https://github.com/BigBodyCobain/Shadowbroker; contact: bigbodycobain@gmail.com)",
|
"User-Agent": DEFAULT_USER_AGENT,
|
||||||
}
|
}
|
||||||
if headers:
|
if headers:
|
||||||
default_headers.update(headers)
|
default_headers.update(headers)
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ Docs: https://pskreporter.info/pskdev.html
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import xml.etree.ElementTree as ET
|
|
||||||
|
|
||||||
|
import defusedxml.ElementTree as ET
|
||||||
import requests
|
import requests
|
||||||
from cachetools import TTLCache, cached
|
from cachetools import TTLCache, cached
|
||||||
|
|
||||||
|
|||||||
@@ -2,14 +2,34 @@ import requests
|
|||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
import logging
|
import logging
|
||||||
from cachetools import cached, TTLCache
|
from cachetools import cached, TTLCache
|
||||||
import cloudscraper
|
|
||||||
import reverse_geocoder as rg
|
import reverse_geocoder as rg
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
_OPENMHZ_AUDIO_HOSTS = {"media.openmhz.com", "media2.openmhz.com", "media3.openmhz.com"}
|
_OPENMHZ_AUDIO_HOSTS = {"media.openmhz.com", "media2.openmhz.com", "media3.openmhz.com"}
|
||||||
|
|
||||||
|
|
||||||
|
# Round 7a / Issues #289, #290, #291 (tg12 audit):
|
||||||
|
# We previously sent a spoofed Chrome User-Agent and (for OpenMHz) used
|
||||||
|
# cloudscraper to bypass anti-bot challenges. Both are dishonest and ToS-
|
||||||
|
# unfriendly. We now send the per-install Shadowbroker UA — the upstream
|
||||||
|
# can identify us, rate-limit us per install, and contact us if needed.
|
||||||
|
#
|
||||||
|
# If the upstream actively blocks our honest UA, the feature degrades
|
||||||
|
# gracefully (returns an empty list / cached results) rather than
|
||||||
|
# escalating to deception.
|
||||||
|
|
||||||
|
|
||||||
|
def _broadcastify_user_agent() -> str:
|
||||||
|
return outbound_user_agent("broadcastify")
|
||||||
|
|
||||||
|
|
||||||
|
def _openmhz_user_agent() -> str:
|
||||||
|
return outbound_user_agent("openmhz")
|
||||||
|
|
||||||
# Cache the top feeds for 5 minutes so we don't hammer Broadcastify
|
# Cache the top feeds for 5 minutes so we don't hammer Broadcastify
|
||||||
radio_cache = TTLCache(maxsize=1, ttl=300)
|
radio_cache = TTLCache(maxsize=1, ttl=300)
|
||||||
|
|
||||||
@@ -22,8 +42,12 @@ def get_top_broadcastify_feeds():
|
|||||||
"""
|
"""
|
||||||
logger.info("Scraping Broadcastify Top Feeds (Cache Miss)")
|
logger.info("Scraping Broadcastify Top Feeds (Cache Miss)")
|
||||||
headers = {
|
headers = {
|
||||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
# Issue #289 (tg12) + Round 7a: identify ourselves honestly as a
|
||||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
# per-install Shadowbroker scraper. Broadcastify can rate-limit
|
||||||
|
# us per install or block us; either way we stop pretending to be
|
||||||
|
# a browser. If they block, the panel degrades gracefully.
|
||||||
|
"User-Agent": _broadcastify_user_agent(),
|
||||||
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
"Accept-Language": "en-US,en;q=0.9",
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -89,21 +113,32 @@ openmhz_systems_cache = TTLCache(maxsize=1, ttl=3600)
|
|||||||
|
|
||||||
@cached(openmhz_systems_cache)
|
@cached(openmhz_systems_cache)
|
||||||
def get_openmhz_systems():
|
def get_openmhz_systems():
|
||||||
"""Fetches the full directory of OpenMHZ systems."""
|
"""Fetches the full directory of OpenMHZ systems.
|
||||||
logger.info("Scraping OpenMHZ Systems (Cache Miss)")
|
|
||||||
scraper = cloudscraper.create_scraper(
|
|
||||||
browser={"browser": "chrome", "platform": "windows", "desktop": True}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
Issue #290 (tg12) + Round 7a: replaced cloudscraper-based Chrome
|
||||||
|
impersonation with an honest per-install Shadowbroker User-Agent.
|
||||||
|
If OpenMHz's Cloudflare layer blocks honest traffic, we accept
|
||||||
|
that degradation (return empty list) rather than spoof a browser.
|
||||||
|
"""
|
||||||
|
logger.info("Fetching OpenMHZ Systems (Cache Miss)")
|
||||||
try:
|
try:
|
||||||
res = scraper.get("https://api.openmhz.com/systems", timeout=15)
|
res = requests.get(
|
||||||
|
"https://api.openmhz.com/systems",
|
||||||
|
timeout=15,
|
||||||
|
headers={"User-Agent": _openmhz_user_agent(), "Accept": "application/json"},
|
||||||
|
)
|
||||||
if res.status_code == 200:
|
if res.status_code == 200:
|
||||||
data = res.json()
|
data = res.json()
|
||||||
# Return list of systems
|
|
||||||
return data.get("systems", []) if isinstance(data, dict) else []
|
return data.get("systems", []) if isinstance(data, dict) else []
|
||||||
|
if res.status_code in (403, 503):
|
||||||
|
logger.warning(
|
||||||
|
"OpenMHZ returned %s for systems directory — Cloudflare may "
|
||||||
|
"be blocking our honest UA. Feature degrades to empty result.",
|
||||||
|
res.status_code,
|
||||||
|
)
|
||||||
return []
|
return []
|
||||||
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
|
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
|
||||||
logger.error(f"OpenMHZ Systems Scrape Exception: {e}")
|
logger.error(f"OpenMHZ Systems Fetch Exception: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
@@ -113,45 +148,85 @@ openmhz_calls_cache = TTLCache(maxsize=100, ttl=20)
|
|||||||
|
|
||||||
@cached(openmhz_calls_cache)
|
@cached(openmhz_calls_cache)
|
||||||
def get_recent_openmhz_calls(sys_name: str):
|
def get_recent_openmhz_calls(sys_name: str):
|
||||||
"""Fetches the actual audio burst .m4a URLs for a specific system (e.g., 'wmata')."""
|
"""Fetches the actual audio burst .m4a URLs for a specific system (e.g., 'wmata').
|
||||||
logger.info(f"Fetching OpenMHZ calls for {sys_name} (Cache Miss)")
|
|
||||||
scraper = cloudscraper.create_scraper(
|
|
||||||
browser={"browser": "chrome", "platform": "windows", "desktop": True}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
Issue #290 (tg12) + Round 7a: same honest-UA model as
|
||||||
|
``get_openmhz_systems``.
|
||||||
|
"""
|
||||||
|
logger.info(f"Fetching OpenMHZ calls for {sys_name} (Cache Miss)")
|
||||||
try:
|
try:
|
||||||
url = f"https://api.openmhz.com/{sys_name}/calls"
|
url = f"https://api.openmhz.com/{sys_name}/calls"
|
||||||
res = scraper.get(url, timeout=15)
|
res = requests.get(
|
||||||
|
url,
|
||||||
|
timeout=15,
|
||||||
|
headers={"User-Agent": _openmhz_user_agent(), "Accept": "application/json"},
|
||||||
|
)
|
||||||
if res.status_code == 200:
|
if res.status_code == 200:
|
||||||
data = res.json()
|
data = res.json()
|
||||||
return data.get("calls", []) if isinstance(data, dict) else []
|
return data.get("calls", []) if isinstance(data, dict) else []
|
||||||
return []
|
return []
|
||||||
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
|
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
|
||||||
logger.error(f"OpenMHZ Calls Scrape Exception ({sys_name}): {e}")
|
logger.error(f"OpenMHZ Calls Fetch Exception ({sys_name}): {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
_OPENMHZ_MAX_REDIRECTS = 5
|
||||||
|
|
||||||
|
|
||||||
def openmhz_audio_response(target_url: str):
|
def openmhz_audio_response(target_url: str):
|
||||||
"""Fetch an OpenMHz audio object through the backend with browser-safe headers."""
|
"""Fetch an OpenMHz audio object through the backend with browser-safe headers.
|
||||||
|
|
||||||
|
Redirects are followed manually so each hop's host can be re-validated
|
||||||
|
against ``_OPENMHZ_AUDIO_HOSTS``. Without this, the upstream could
|
||||||
|
302-redirect to an internal address (e.g. ``http://127.0.0.1:8000/...``
|
||||||
|
or an RFC1918 range), and the backend would dutifully fetch and stream
|
||||||
|
that response back to the browser — a classic open-redirect-to-SSRF
|
||||||
|
chain. Same-host redirects (CDN edge selection) still work normally.
|
||||||
|
"""
|
||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
from fastapi.responses import StreamingResponse
|
from fastapi.responses import StreamingResponse
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
parsed = urlparse(str(target_url or ""))
|
parsed = urlparse(str(target_url or ""))
|
||||||
host = (parsed.hostname or "").lower()
|
host = (parsed.hostname or "").lower()
|
||||||
if parsed.scheme != "https" or host not in _OPENMHZ_AUDIO_HOSTS:
|
if parsed.scheme != "https" or host not in _OPENMHZ_AUDIO_HOSTS:
|
||||||
raise HTTPException(status_code=400, detail="Unsupported OpenMHz audio URL")
|
raise HTTPException(status_code=400, detail="Unsupported OpenMHz audio URL")
|
||||||
|
|
||||||
|
current_url = target_url
|
||||||
|
hops = 0
|
||||||
try:
|
try:
|
||||||
upstream = requests.get(
|
while True:
|
||||||
target_url,
|
upstream = requests.get(
|
||||||
stream=True,
|
current_url,
|
||||||
timeout=(5, 20),
|
stream=True,
|
||||||
headers={
|
timeout=(5, 20),
|
||||||
"User-Agent": "Mozilla/5.0",
|
allow_redirects=False,
|
||||||
"Accept": "audio/mpeg,audio/*,*/*;q=0.8",
|
headers={
|
||||||
"Referer": "https://openmhz.com/",
|
# Issue #291 (tg12) + Round 7a: drop spoofed Mozilla
|
||||||
},
|
# UA and the fake first-party Referer. Identify as
|
||||||
)
|
# the per-install Shadowbroker proxy honestly.
|
||||||
|
"User-Agent": _openmhz_user_agent(),
|
||||||
|
"Accept": "audio/mpeg,audio/*,*/*;q=0.8",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if upstream.is_redirect or upstream.status_code in (301, 302, 303, 307, 308):
|
||||||
|
location = upstream.headers.get("Location", "")
|
||||||
|
upstream.close()
|
||||||
|
if hops >= _OPENMHZ_MAX_REDIRECTS or not location:
|
||||||
|
raise HTTPException(status_code=502, detail="OpenMHz redirect rejected")
|
||||||
|
next_url = urljoin(current_url, location)
|
||||||
|
next_parsed = urlparse(next_url)
|
||||||
|
next_host = (next_parsed.hostname or "").lower()
|
||||||
|
# Re-validate the next hop against the same allowlist used for
|
||||||
|
# the original URL. Cross-host redirects to disallowed hosts
|
||||||
|
# are rejected silently; the browser audio element handles
|
||||||
|
# the resulting 502 gracefully and moves on.
|
||||||
|
if next_parsed.scheme != "https" or next_host not in _OPENMHZ_AUDIO_HOSTS:
|
||||||
|
raise HTTPException(status_code=502, detail="OpenMHz redirect rejected")
|
||||||
|
current_url = next_url
|
||||||
|
hops += 1
|
||||||
|
continue
|
||||||
|
break
|
||||||
except requests.RequestException as exc:
|
except requests.RequestException as exc:
|
||||||
raise HTTPException(status_code=502, detail="OpenMHz audio fetch failed") from exc
|
raise HTTPException(status_code=502, detail="OpenMHz audio fetch failed") from exc
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import concurrent.futures
|
|||||||
from urllib.parse import quote
|
from urllib.parse import quote
|
||||||
import requests as _requests
|
import requests as _requests
|
||||||
from cachetools import TTLCache
|
from cachetools import TTLCache
|
||||||
from services.network_utils import fetch_with_curl
|
from services.network_utils import fetch_with_curl, outbound_user_agent
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -15,6 +15,31 @@ dossier_cache = TTLCache(maxsize=500, ttl=86400)
|
|||||||
# Nominatim requires max 1 req/sec — track last call time
|
# Nominatim requires max 1 req/sec — track last call time
|
||||||
_nominatim_last_call = 0.0
|
_nominatim_last_call = 0.0
|
||||||
|
|
||||||
|
# Issues #218 / #219 (tg12): Wikimedia's User-Agent policy requires API
|
||||||
|
# clients to identify themselves with a stable User-Agent that includes
|
||||||
|
# a contact path.
|
||||||
|
#
|
||||||
|
# Round 7a: the original fix in PR #284 used a single project-wide
|
||||||
|
# identifier, which from Wikimedia's perspective made every Shadowbroker
|
||||||
|
# install in the world look like one giant scraper. If one install
|
||||||
|
# misbehaved, their only recourse was to block "Shadowbroker" as a
|
||||||
|
# whole. We now build the headers from ``outbound_user_agent('wikimedia')``
|
||||||
|
# which embeds the per-install operator handle (auto-generated or
|
||||||
|
# operator-chosen), so Wikimedia can rate-limit / contact the specific
|
||||||
|
# install instead of the project.
|
||||||
|
|
||||||
|
|
||||||
|
def _wikimedia_request_headers() -> dict[str, str]:
|
||||||
|
ua = outbound_user_agent("wikimedia")
|
||||||
|
return {
|
||||||
|
"User-Agent": ua,
|
||||||
|
# Browser-JS-style header that Wikimedia's policy explicitly
|
||||||
|
# accepts on top of (or instead of) User-Agent. We send both so
|
||||||
|
# whichever the upstream prefers, the per-operator handle is
|
||||||
|
# always available.
|
||||||
|
"Api-User-Agent": ua,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _reverse_geocode_offline(lat: float, lng: float) -> dict:
|
def _reverse_geocode_offline(lat: float, lng: float) -> dict:
|
||||||
"""Offline fallback via reverse_geocoder when external reverse geocoding is blocked."""
|
"""Offline fallback via reverse_geocoder when external reverse geocoding is blocked."""
|
||||||
@@ -45,9 +70,7 @@ def _reverse_geocode(lat: float, lng: float) -> dict:
|
|||||||
f"https://nominatim.openstreetmap.org/reverse?"
|
f"https://nominatim.openstreetmap.org/reverse?"
|
||||||
f"lat={lat}&lon={lng}&format=json&zoom=10&addressdetails=1&accept-language=en"
|
f"lat={lat}&lon={lng}&format=json&zoom=10&addressdetails=1&accept-language=en"
|
||||||
)
|
)
|
||||||
headers = {
|
headers = {"User-Agent": outbound_user_agent("nominatim")}
|
||||||
"User-Agent": "ShadowBroker-OSINT/1.0 (live-risk-dashboard; contact@shadowbroker.app)"
|
|
||||||
}
|
|
||||||
|
|
||||||
for attempt in range(2):
|
for attempt in range(2):
|
||||||
# Enforce Nominatim's 1 req/sec policy
|
# Enforce Nominatim's 1 req/sec policy
|
||||||
@@ -121,7 +144,13 @@ def _fetch_wikidata_leader(country_name: str) -> dict:
|
|||||||
"""
|
"""
|
||||||
url = f"https://query.wikidata.org/sparql?query={quote(sparql)}&format=json"
|
url = f"https://query.wikidata.org/sparql?query={quote(sparql)}&format=json"
|
||||||
try:
|
try:
|
||||||
res = fetch_with_curl(url, timeout=6)
|
# Issue #218 (tg12): Wikimedia's User-Agent policy requires
|
||||||
|
# outbound API traffic to be identifiable. fetch_with_curl()
|
||||||
|
# sends the project default, and we also add the Wikimedia-
|
||||||
|
# specific Api-User-Agent that the policy specifically asks
|
||||||
|
# for, since this request originates from a backend service
|
||||||
|
# that proxies on behalf of (potentially many) browser users.
|
||||||
|
res = fetch_with_curl(url, timeout=6, headers=_wikimedia_request_headers())
|
||||||
if res.status_code == 200:
|
if res.status_code == 200:
|
||||||
results = res.json().get("results", {}).get("bindings", [])
|
results = res.json().get("results", {}).get("bindings", [])
|
||||||
if results:
|
if results:
|
||||||
@@ -147,7 +176,9 @@ def _fetch_local_wiki_summary(place_name: str, country_name: str = "") -> dict:
|
|||||||
slug = quote(name.replace(" ", "_"))
|
slug = quote(name.replace(" ", "_"))
|
||||||
url = f"https://en.wikipedia.org/api/rest_v1/page/summary/{slug}"
|
url = f"https://en.wikipedia.org/api/rest_v1/page/summary/{slug}"
|
||||||
try:
|
try:
|
||||||
res = fetch_with_curl(url, timeout=5)
|
# Issue #219 (tg12): identify ourselves to Wikimedia per
|
||||||
|
# their UA policy; see _fetch_wikidata_leader above.
|
||||||
|
res = fetch_with_curl(url, timeout=5, headers=_wikimedia_request_headers())
|
||||||
if res.status_code == 200:
|
if res.status_code == 200:
|
||||||
data = res.json()
|
data = res.json()
|
||||||
if data.get("type") != "disambiguation":
|
if data.get("type") != "disambiguation":
|
||||||
|
|||||||
@@ -34,6 +34,11 @@ from services.sar.sar_config import (
|
|||||||
copernicus_token,
|
copernicus_token,
|
||||||
earthdata_token,
|
earthdata_token,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _sar_user_agent() -> str:
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
|
return outbound_user_agent("sar-products")
|
||||||
from services.sar.sar_normalize import (
|
from services.sar.sar_normalize import (
|
||||||
SarAnomaly,
|
SarAnomaly,
|
||||||
evidence_hash_for_payload,
|
evidence_hash_for_payload,
|
||||||
@@ -442,7 +447,7 @@ def _fetch_unosat_packages() -> list[dict[str, Any]]:
|
|||||||
# HDX CKAN returns 406 without explicit Accept + a browser-ish UA.
|
# HDX CKAN returns 406 without explicit Accept + a browser-ish UA.
|
||||||
hdx_headers = {
|
hdx_headers = {
|
||||||
"Accept": "application/json",
|
"Accept": "application/json",
|
||||||
"User-Agent": "Mozilla/5.0 (compatible; ShadowBroker-SAR/1.0)",
|
"User-Agent": _sar_user_agent(),
|
||||||
}
|
}
|
||||||
try:
|
try:
|
||||||
resp = fetch_with_curl(url, timeout=20, headers=hdx_headers)
|
resp = fetch_with_curl(url, timeout=20, headers=hdx_headers)
|
||||||
|
|||||||
@@ -14,6 +14,11 @@ class HealthResponse(BaseModel):
|
|||||||
# ({status, age_s, row_count, slo, stale, empty, description}).
|
# ({status, age_s, row_count, slo, stale, empty, description}).
|
||||||
slo: Optional[Dict[str, Any]] = None
|
slo: Optional[Dict[str, Any]] = None
|
||||||
slo_summary: Optional[Dict[str, int]] = None
|
slo_summary: Optional[Dict[str, int]] = None
|
||||||
|
# Issue #258: AIS proxy status — currently exposes ``degraded_tls``
|
||||||
|
# (bool), true when ais_proxy.js fell back to the SPKI-pinned
|
||||||
|
# insecure-date path because the upstream Let's Encrypt cert is
|
||||||
|
# expired. Empty dict / null means no status reported yet.
|
||||||
|
ais_proxy: Optional[Dict[str, Any]] = None
|
||||||
|
|
||||||
|
|
||||||
class RefreshResponse(BaseModel):
|
class RefreshResponse(BaseModel):
|
||||||
|
|||||||
@@ -11,12 +11,21 @@ import requests
|
|||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from cachetools import TTLCache
|
from cachetools import TTLCache
|
||||||
|
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Cache by rounded lat/lon (0.02° grid ~= 2km), TTL 1 hour
|
# Cache by rounded lat/lon (0.02° grid ~= 2km), TTL 1 hour
|
||||||
_sentinel_cache = TTLCache(maxsize=200, ttl=3600)
|
_sentinel_cache = TTLCache(maxsize=200, ttl=3600)
|
||||||
|
|
||||||
|
|
||||||
|
def _planetary_user_agent() -> str:
|
||||||
|
# Round 7a: per-install handle so Microsoft Planetary Computer can
|
||||||
|
# attribute requests to the specific operator rather than treating
|
||||||
|
# the whole Shadowbroker user base as one entity.
|
||||||
|
return outbound_user_agent("sentinel2-planetary-computer")
|
||||||
|
|
||||||
|
|
||||||
def _esri_imagery_fallback(lat: float, lng: float) -> dict:
|
def _esri_imagery_fallback(lat: float, lng: float) -> dict:
|
||||||
lat_span = 0.18
|
lat_span = 0.18
|
||||||
lng_span = 0.24
|
lng_span = 0.24
|
||||||
@@ -64,7 +73,7 @@ def search_sentinel2_scene(lat: float, lng: float) -> dict:
|
|||||||
"https://planetarycomputer.microsoft.com/api/stac/v1/search",
|
"https://planetarycomputer.microsoft.com/api/stac/v1/search",
|
||||||
json=search_payload,
|
json=search_payload,
|
||||||
timeout=8,
|
timeout=8,
|
||||||
headers={"User-Agent": "ShadowBroker-OSINT/1.0 (live-risk-dashboard)"},
|
headers={"User-Agent": _planetary_user_agent()},
|
||||||
)
|
)
|
||||||
search_res.raise_for_status()
|
search_res.raise_for_status()
|
||||||
data = search_res.json()
|
data = search_res.json()
|
||||||
|
|||||||
@@ -20,7 +20,11 @@ from cachetools import TTLCache
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
_SHODAN_BASE = "https://api.shodan.io"
|
_SHODAN_BASE = "https://api.shodan.io"
|
||||||
_USER_AGENT = "ShadowBroker/0.9.75 local Shodan connector"
|
# Round 7a: per-install attribution. Shodan already has the operator API
|
||||||
|
# key for billing, but the UA still identifies the install.
|
||||||
|
def _shodan_user_agent():
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
|
return outbound_user_agent("shodan")
|
||||||
_REQUEST_TIMEOUT = 15
|
_REQUEST_TIMEOUT = 15
|
||||||
_MIN_INTERVAL_SECONDS = 1.05 # Shodan docs say API plans are rate limited to ~1 req/sec.
|
_MIN_INTERVAL_SECONDS = 1.05 # Shodan docs say API plans are rate limited to ~1 req/sec.
|
||||||
_DEFAULT_SEARCH_PAGES = 1
|
_DEFAULT_SEARCH_PAGES = 1
|
||||||
@@ -179,7 +183,7 @@ def _request(path: str, *, params: dict[str, Any], cache: TTLCache[str, dict[str
|
|||||||
f"{_SHODAN_BASE}{path}",
|
f"{_SHODAN_BASE}{path}",
|
||||||
params=payload,
|
params=payload,
|
||||||
timeout=_REQUEST_TIMEOUT,
|
timeout=_REQUEST_TIMEOUT,
|
||||||
headers={"User-Agent": _USER_AGENT, "Accept": "application/json"},
|
headers={"User-Agent": _shodan_user_agent(), "Accept": "application/json"},
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
_last_request_at = time.monotonic()
|
_last_request_at = time.monotonic()
|
||||||
|
|||||||
@@ -545,6 +545,198 @@ class MeshtasticBridge:
|
|||||||
self._message_dedupe[key] = now
|
self._message_dedupe[key] = now
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _message_dedupe_key(message: dict) -> str:
|
||||||
|
sender = str(message.get("from") or "???").strip().lower()
|
||||||
|
recipient = str(message.get("to") or "broadcast").strip().lower()
|
||||||
|
text = str(message.get("text") or "").strip()
|
||||||
|
channel = str(message.get("channel") or "LongFast").strip().lower()
|
||||||
|
root = str(message.get("root") or message.get("region") or "").strip().lower()
|
||||||
|
if root == "us":
|
||||||
|
root = "us"
|
||||||
|
return f"{sender}:{recipient}:{root}:{channel}:{text}"
|
||||||
|
|
||||||
|
def append_text_message(self, message: dict, *, dedupe_window_s: float = 5.0) -> bool:
|
||||||
|
"""Append a Meshtastic text message unless it is a near-immediate echo."""
|
||||||
|
if not str(message.get("text") or "").strip():
|
||||||
|
return False
|
||||||
|
now = time.time()
|
||||||
|
cutoff = now - max(1.0, dedupe_window_s)
|
||||||
|
next_message = dict(message)
|
||||||
|
next_message.setdefault("to", "broadcast")
|
||||||
|
next_message.setdefault("channel", "LongFast")
|
||||||
|
next_message.setdefault("timestamp", datetime.utcnow().isoformat() + "Z")
|
||||||
|
key = self._message_dedupe_key(next_message)
|
||||||
|
for existing in list(self.messages)[:40]:
|
||||||
|
if self._message_dedupe_key(existing) != key:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
existing_ts_raw = existing.get("timestamp")
|
||||||
|
existing_ts = (
|
||||||
|
datetime.fromisoformat(str(existing_ts_raw).replace("Z", "+00:00")).timestamp()
|
||||||
|
if existing_ts_raw
|
||||||
|
else now
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
existing_ts = now
|
||||||
|
if existing_ts >= cutoff:
|
||||||
|
if not existing.get("root") and next_message.get("root"):
|
||||||
|
existing["root"] = next_message.get("root")
|
||||||
|
if not existing.get("region") and next_message.get("region"):
|
||||||
|
existing["region"] = next_message.get("region")
|
||||||
|
return False
|
||||||
|
self.messages.appendleft(next_message)
|
||||||
|
return True
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _coerce_node_ref(value) -> str:
|
||||||
|
"""Normalize Meshtastic node identifiers into the public !xxxxxxxx form."""
|
||||||
|
if value is None:
|
||||||
|
return ""
|
||||||
|
if isinstance(value, int):
|
||||||
|
return f"!{value & 0xFFFFFFFF:08x}"
|
||||||
|
raw = str(value).strip()
|
||||||
|
if not raw:
|
||||||
|
return ""
|
||||||
|
if raw.startswith("!"):
|
||||||
|
return raw
|
||||||
|
lowered = raw.lower()
|
||||||
|
if lowered.startswith("0x"):
|
||||||
|
try:
|
||||||
|
return f"!{int(lowered, 16) & 0xFFFFFFFF:08x}"
|
||||||
|
except ValueError:
|
||||||
|
return raw
|
||||||
|
if raw.isdigit():
|
||||||
|
try:
|
||||||
|
return f"!{int(raw) & 0xFFFFFFFF:08x}"
|
||||||
|
except ValueError:
|
||||||
|
return raw
|
||||||
|
if len(raw) == 8 and all(ch in "0123456789abcdefABCDEF" for ch in raw):
|
||||||
|
return f"!{raw.lower()}"
|
||||||
|
return raw
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _first_text_value(*values) -> str:
|
||||||
|
for value in values:
|
||||||
|
if isinstance(value, bytes):
|
||||||
|
value = value.decode("utf-8", errors="replace")
|
||||||
|
if isinstance(value, str):
|
||||||
|
text = value.strip()
|
||||||
|
if text:
|
||||||
|
return MeshtasticBridge._repair_text_mojibake(text)
|
||||||
|
return ""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _repair_text_mojibake(text: str) -> str:
|
||||||
|
"""Repair common UTF-8-as-Latin-1 mojibake from MQTT JSON bridges."""
|
||||||
|
if not text or not any(marker in text for marker in ("Ã", "Ð", "Ñ")):
|
||||||
|
return text
|
||||||
|
try:
|
||||||
|
repaired = text.encode("latin-1").decode("utf-8").strip()
|
||||||
|
except UnicodeError:
|
||||||
|
return text
|
||||||
|
if repaired and repaired != text:
|
||||||
|
return repaired
|
||||||
|
return text
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _first_present(*values):
|
||||||
|
for value in values:
|
||||||
|
if value is not None and value != "":
|
||||||
|
return value
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _extract_json_text_message(self, data: dict, topic: str) -> dict | None:
|
||||||
|
"""Extract a public Meshtastic text event from decoded MQTT JSON.
|
||||||
|
|
||||||
|
Meshtastic JSON brokers are not perfectly uniform. Some packets expose
|
||||||
|
text at the top level, some under ``decoded`` or ``payload``. Keep this
|
||||||
|
permissive for receive, but only return messages with non-empty text.
|
||||||
|
"""
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return None
|
||||||
|
topic_meta = parse_topic_metadata(topic)
|
||||||
|
packet = data.get("packet") if isinstance(data.get("packet"), dict) else {}
|
||||||
|
decoded = data.get("decoded") if isinstance(data.get("decoded"), dict) else {}
|
||||||
|
payload_obj = data.get("payload")
|
||||||
|
payload = payload_obj if isinstance(payload_obj, dict) else {}
|
||||||
|
decoded_payload_obj = decoded.get("payload") if decoded else None
|
||||||
|
decoded_payload = decoded_payload_obj if isinstance(decoded_payload_obj, dict) else {}
|
||||||
|
|
||||||
|
text = self._first_text_value(
|
||||||
|
data.get("text"),
|
||||||
|
data.get("message"),
|
||||||
|
data.get("msg"),
|
||||||
|
payload_obj if isinstance(payload_obj, str) else "",
|
||||||
|
payload.get("text"),
|
||||||
|
payload.get("message"),
|
||||||
|
payload.get("msg"),
|
||||||
|
payload.get("payload") if isinstance(payload.get("payload"), str) else "",
|
||||||
|
decoded.get("text"),
|
||||||
|
decoded.get("message"),
|
||||||
|
decoded.get("payload") if isinstance(decoded.get("payload"), str) else "",
|
||||||
|
decoded_payload.get("text"),
|
||||||
|
decoded_payload.get("message"),
|
||||||
|
decoded_payload.get("msg"),
|
||||||
|
)
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
|
||||||
|
sender = self._coerce_node_ref(
|
||||||
|
self._first_present(
|
||||||
|
data.get("from"),
|
||||||
|
data.get("fromId"),
|
||||||
|
data.get("from_id"),
|
||||||
|
data.get("sender"),
|
||||||
|
data.get("senderId"),
|
||||||
|
data.get("sender_id"),
|
||||||
|
packet.get("from"),
|
||||||
|
packet.get("fromId"),
|
||||||
|
packet.get("from_id"),
|
||||||
|
decoded.get("from"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
recipient = self._coerce_node_ref(
|
||||||
|
self._first_present(
|
||||||
|
data.get("to"),
|
||||||
|
data.get("toId"),
|
||||||
|
data.get("to_id"),
|
||||||
|
data.get("recipient"),
|
||||||
|
data.get("recipientId"),
|
||||||
|
data.get("recipient_id"),
|
||||||
|
packet.get("to"),
|
||||||
|
packet.get("toId"),
|
||||||
|
packet.get("to_id"),
|
||||||
|
decoded.get("to"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not recipient or recipient in {"!ffffffff", "broadcast"}:
|
||||||
|
recipient = "broadcast"
|
||||||
|
|
||||||
|
timestamp = datetime.utcnow().isoformat() + "Z"
|
||||||
|
rx_time = self._first_present(
|
||||||
|
data.get("rxTime"),
|
||||||
|
data.get("rx_time"),
|
||||||
|
data.get("timestamp"),
|
||||||
|
packet.get("rxTime"),
|
||||||
|
packet.get("timestamp"),
|
||||||
|
)
|
||||||
|
if isinstance(rx_time, (int, float)) and rx_time > 0:
|
||||||
|
try:
|
||||||
|
timestamp = datetime.fromtimestamp(float(rx_time), tz=timezone.utc).isoformat()
|
||||||
|
except (OSError, ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
return {
|
||||||
|
"from": sender or topic.split("/")[-1],
|
||||||
|
"to": recipient,
|
||||||
|
"text": text[:500],
|
||||||
|
"region": topic_meta["region"],
|
||||||
|
"root": topic_meta["root"],
|
||||||
|
"channel": topic_meta["channel"],
|
||||||
|
"timestamp": timestamp,
|
||||||
|
}
|
||||||
|
|
||||||
def start(self):
|
def start(self):
|
||||||
if self._thread and self._thread.is_alive():
|
if self._thread and self._thread.is_alive():
|
||||||
if not self._stop.is_set():
|
if not self._stop.is_set():
|
||||||
@@ -693,6 +885,9 @@ class MeshtasticBridge:
|
|||||||
if "/json/" in topic:
|
if "/json/" in topic:
|
||||||
try:
|
try:
|
||||||
data = json.loads(payload)
|
data = json.loads(payload)
|
||||||
|
text_message = self._extract_json_text_message(data, topic)
|
||||||
|
if text_message:
|
||||||
|
self.append_text_message(text_message, dedupe_window_s=30.0)
|
||||||
if self._rate_limited():
|
if self._rate_limited():
|
||||||
return
|
return
|
||||||
self._ingest_data(data, topic)
|
self._ingest_data(data, topic)
|
||||||
@@ -715,7 +910,7 @@ class MeshtasticBridge:
|
|||||||
topic_meta["root"],
|
topic_meta["root"],
|
||||||
):
|
):
|
||||||
return
|
return
|
||||||
self.messages.appendleft(
|
self.append_text_message(
|
||||||
{
|
{
|
||||||
"from": data.get("from", "???"),
|
"from": data.get("from", "???"),
|
||||||
"to": recipient,
|
"to": recipient,
|
||||||
|
|||||||
@@ -19,6 +19,13 @@ from pathlib import Path
|
|||||||
import requests
|
import requests
|
||||||
from sgp4.api import Satrec, WGS72, jday
|
from sgp4.api import Satrec, WGS72, jday
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def _tinygs_user_agent(purpose: str) -> str:
|
||||||
|
"""Round 7a: per-install handle for CelesTrak / TinyGS attribution."""
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
|
return outbound_user_agent(f"tinygs-{purpose}")
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -113,7 +120,7 @@ def _fetch_celestrak_tles() -> list[dict]:
|
|||||||
params={"GROUP": group, "FORMAT": "json"},
|
params={"GROUP": group, "FORMAT": "json"},
|
||||||
timeout=20,
|
timeout=20,
|
||||||
headers={
|
headers={
|
||||||
"User-Agent": "ShadowBroker-OSINT/1.0 (CelesTrak fair-use)",
|
"User-Agent": _tinygs_user_agent("celestrak"),
|
||||||
"Accept": "application/json",
|
"Accept": "application/json",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@@ -259,7 +266,7 @@ def _fetch_tinygs_telemetry() -> None:
|
|||||||
timeout=15,
|
timeout=15,
|
||||||
headers={
|
headers={
|
||||||
"Accept": "application/json",
|
"Accept": "application/json",
|
||||||
"User-Agent": "ShadowBroker-OSINT/1.0",
|
"User-Agent": _tinygs_user_agent("tinygs"),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
|
|||||||
@@ -64,6 +64,203 @@ def _find_tor_binary() -> str | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# Baked-in expected digest list. Loaded lazily; populated by maintainers
|
||||||
|
# when a new Tor Expert Bundle URL is added to _TOR_EXPERT_BUNDLE_URLS.
|
||||||
|
# See issue #201 for rationale.
|
||||||
|
_TOR_DIGEST_FILE = Path(__file__).resolve().parent.parent / "data" / "tor_bundle_digests.json"
|
||||||
|
_DIGEST_PLACEHOLDER = "PLACEHOLDER_REPLACE_BEFORE_RELEASE"
|
||||||
|
|
||||||
|
|
||||||
|
def _load_baked_in_digests() -> dict[str, str]:
|
||||||
|
"""Return {url: expected_sha256_lower} for URLs we ship a known digest for.
|
||||||
|
|
||||||
|
Entries whose value is the placeholder sentinel are filtered out — they
|
||||||
|
represent versions the maintainer has not yet pinned, and we don't
|
||||||
|
want to trust them via this layer.
|
||||||
|
"""
|
||||||
|
if not _TOR_DIGEST_FILE.exists():
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
import json as _json
|
||||||
|
raw = _json.loads(_TOR_DIGEST_FILE.read_text(encoding="utf-8"))
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("Tor bundle digests file unreadable: %s", exc)
|
||||||
|
return {}
|
||||||
|
result: dict[str, str] = {}
|
||||||
|
for k, v in raw.items():
|
||||||
|
if not isinstance(k, str) or k.startswith("_"):
|
||||||
|
continue
|
||||||
|
if not isinstance(v, str) or v == _DIGEST_PLACEHOLDER:
|
||||||
|
continue
|
||||||
|
result[k] = v.strip().lower()
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_tor_bundle(archive_path: Path, bundle_url: str) -> tuple[bool, str]:
|
||||||
|
"""Verify the downloaded Tor bundle against any source we trust.
|
||||||
|
|
||||||
|
Returns (verified, reason). The bundle is considered verified if EITHER:
|
||||||
|
|
||||||
|
* The upstream ``.sha256sum`` file is reachable AND its digest matches
|
||||||
|
what we just downloaded, OR
|
||||||
|
* Our baked-in digest list (``backend/data/tor_bundle_digests.json``)
|
||||||
|
contains this URL AND that digest matches.
|
||||||
|
|
||||||
|
If both sources are unavailable (e.g. fresh checkout before the
|
||||||
|
maintainer has populated the digest file AND the upstream
|
||||||
|
``.sha256sum`` is unreachable), we **fall back to HTTPS-only trust**
|
||||||
|
with a warning so first-run onboarding does not break. As soon as the
|
||||||
|
digest file is populated for a shipped Tor version, the secure path
|
||||||
|
activates automatically — no operator action required.
|
||||||
|
|
||||||
|
Issue #201.
|
||||||
|
"""
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
actual_hash = hashlib.sha256(archive_path.read_bytes()).hexdigest().lower()
|
||||||
|
|
||||||
|
# Source 1: upstream .sha256sum
|
||||||
|
upstream_hash: str | None = None
|
||||||
|
sha256_url = bundle_url + ".sha256sum"
|
||||||
|
sha256_file = TOR_INSTALL_DIR / "sha256sum.txt"
|
||||||
|
try:
|
||||||
|
urlretrieve(sha256_url, str(sha256_file))
|
||||||
|
upstream_hash = sha256_file.read_text().strip().split()[0].lower()
|
||||||
|
sha256_file.unlink(missing_ok=True)
|
||||||
|
except Exception as hash_err:
|
||||||
|
logger.info("Tor bundle upstream .sha256sum unreachable: %s", hash_err)
|
||||||
|
sha256_file.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
if upstream_hash and upstream_hash == actual_hash:
|
||||||
|
return True, f"verified via upstream .sha256sum ({actual_hash[:16]}...)"
|
||||||
|
|
||||||
|
# Source 2: baked-in digest list
|
||||||
|
baked = _load_baked_in_digests()
|
||||||
|
baked_hash = baked.get(bundle_url)
|
||||||
|
if baked_hash and baked_hash == actual_hash:
|
||||||
|
return True, f"verified via baked-in digest list ({actual_hash[:16]}...)"
|
||||||
|
|
||||||
|
# If we got an upstream digest AND a baked-in digest AND neither
|
||||||
|
# matched, the bundle is genuinely suspect — refuse it.
|
||||||
|
if upstream_hash and baked_hash:
|
||||||
|
return False, (
|
||||||
|
f"SHA-256 mismatch: archive={actual_hash[:16]}..., "
|
||||||
|
f"upstream={upstream_hash[:16]}..., baked={baked_hash[:16]}..."
|
||||||
|
)
|
||||||
|
if upstream_hash and upstream_hash != actual_hash:
|
||||||
|
return False, (
|
||||||
|
f"SHA-256 mismatch vs upstream: archive={actual_hash[:16]}..., "
|
||||||
|
f"upstream={upstream_hash[:16]}..."
|
||||||
|
)
|
||||||
|
if baked_hash and baked_hash != actual_hash:
|
||||||
|
return False, (
|
||||||
|
f"SHA-256 mismatch vs baked-in digest: archive={actual_hash[:16]}..., "
|
||||||
|
f"expected={baked_hash[:16]}..."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Neither verification source available. This is the fallback path for
|
||||||
|
# the case where the upstream .sha256sum is temporarily unreachable
|
||||||
|
# AND the maintainer hasn't yet pinned this Tor version. Trust HTTPS
|
||||||
|
# only (current behavior pre-#201) with a clear warning. Onboarding
|
||||||
|
# works; once we populate the digest file, the secure path activates.
|
||||||
|
logger.warning(
|
||||||
|
"Tor bundle integrity check fell back to HTTPS-only trust "
|
||||||
|
"(upstream .sha256sum unreachable AND no baked-in digest for %s). "
|
||||||
|
"Add this URL's SHA-256 to backend/data/tor_bundle_digests.json "
|
||||||
|
"to enable the secure path.",
|
||||||
|
bundle_url,
|
||||||
|
)
|
||||||
|
return True, f"https-only (no digest source reachable, archive={actual_hash[:16]}...)"
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_tor_bundle_safely(archive_path: Path, install_dir: Path) -> bool:
|
||||||
|
"""Extract a Tor Expert Bundle tar.gz safely.
|
||||||
|
|
||||||
|
Issue #251: the previous extractor checked tarinfo.name against path
|
||||||
|
traversal but never inspected tarinfo.linkname for symlink/hardlink
|
||||||
|
members. Python 3.11's tarfile honors symlinks during extractall(),
|
||||||
|
so a malicious archive could ship a member like::
|
||||||
|
|
||||||
|
name = "innocent.txt" # passes the path check
|
||||||
|
type = SYMTYPE
|
||||||
|
linkname = "C:\\Windows\\System32\\config\\system"
|
||||||
|
|
||||||
|
and extractall() would then create that symlink. Subsequent reads
|
||||||
|
of innocent.txt deference to a sensitive system file; subsequent
|
||||||
|
writes corrupt one. Tor bundles never legitimately contain symlinks
|
||||||
|
or hardlinks, so we refuse all link members categorically rather
|
||||||
|
than trying to validate linkname targets (which has its own pitfalls
|
||||||
|
around relative path resolution).
|
||||||
|
|
||||||
|
Also refuses non-regular-non-directory members (devices, FIFOs,
|
||||||
|
character/block special files) for completeness — none of those
|
||||||
|
belong in a Tor Expert Bundle and accepting them is a category of
|
||||||
|
bug we don't need to debug later.
|
||||||
|
|
||||||
|
Returns True on success, False on rejection (and logs the reason).
|
||||||
|
The caller is responsible for cleaning up the archive file.
|
||||||
|
"""
|
||||||
|
import tarfile
|
||||||
|
|
||||||
|
install_resolved = install_dir.resolve()
|
||||||
|
|
||||||
|
try:
|
||||||
|
with tarfile.open(str(archive_path), "r:gz") as tar:
|
||||||
|
for member in tar.getmembers():
|
||||||
|
# Reject anything that isn't a regular file or directory.
|
||||||
|
# Symlinks (SYMTYPE) and hardlinks (LNKTYPE) are the
|
||||||
|
# path-traversal vectors; the others (CHRTYPE, BLKTYPE,
|
||||||
|
# FIFOTYPE, CONTTYPE) have no legitimate use in a Tor
|
||||||
|
# Expert Bundle.
|
||||||
|
if member.issym() or member.islnk():
|
||||||
|
logger.error(
|
||||||
|
"Tor bundle extraction blocked: link member %s -> %s "
|
||||||
|
"(symlinks/hardlinks are not allowed in Tor bundles; "
|
||||||
|
"this archive is malformed or hostile)",
|
||||||
|
member.name,
|
||||||
|
member.linkname,
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
if not (member.isfile() or member.isdir()):
|
||||||
|
logger.error(
|
||||||
|
"Tor bundle extraction blocked: unexpected member type "
|
||||||
|
"for %s (only regular files and directories are allowed)",
|
||||||
|
member.name,
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Path traversal check (preserves the original guard).
|
||||||
|
try:
|
||||||
|
member_path = (install_dir / member.name).resolve()
|
||||||
|
except OSError as exc:
|
||||||
|
logger.error(
|
||||||
|
"Tor bundle extraction blocked: cannot resolve member "
|
||||||
|
"path %s: %s",
|
||||||
|
member.name,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
member_path.relative_to(install_resolved)
|
||||||
|
except ValueError:
|
||||||
|
logger.error(
|
||||||
|
"Tor bundle extraction blocked: path traversal on %s "
|
||||||
|
"(resolves to %s, outside install dir %s)",
|
||||||
|
member.name,
|
||||||
|
member_path,
|
||||||
|
install_resolved,
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
# All members validated — extract.
|
||||||
|
tar.extractall(path=str(install_dir))
|
||||||
|
except tarfile.TarError as exc:
|
||||||
|
logger.error("Tor bundle extraction failed: malformed tar (%s)", exc)
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def _auto_install_tor() -> str | None:
|
def _auto_install_tor() -> str | None:
|
||||||
"""Install or download Tor when it is safe to do so."""
|
"""Install or download Tor when it is safe to do so."""
|
||||||
if os.name != "nt":
|
if os.name != "nt":
|
||||||
@@ -79,37 +276,24 @@ def _auto_install_tor() -> str | None:
|
|||||||
logger.info("Downloading Tor Expert Bundle over HTTPS from %s...", bundle_url)
|
logger.info("Downloading Tor Expert Bundle over HTTPS from %s...", bundle_url)
|
||||||
urlretrieve(bundle_url, str(archive_path))
|
urlretrieve(bundle_url, str(archive_path))
|
||||||
|
|
||||||
sha256_url = bundle_url + ".sha256sum"
|
# Issue #201: multi-source verification. If neither upstream
|
||||||
sha256_file = TOR_INSTALL_DIR / "sha256sum.txt"
|
# .sha256sum nor a baked-in digest matches, we refuse this URL
|
||||||
try:
|
# and try the next one in _TOR_EXPERT_BUNDLE_URLS. If neither
|
||||||
urlretrieve(sha256_url, str(sha256_file))
|
# source is reachable at all, we fall back to HTTPS-only trust
|
||||||
expected_hash = sha256_file.read_text().strip().split()[0].lower()
|
# (current behavior) rather than blocking onboarding.
|
||||||
import hashlib
|
verified, reason = _verify_tor_bundle(archive_path, bundle_url)
|
||||||
|
if not verified:
|
||||||
actual_hash = hashlib.sha256(archive_path.read_bytes()).hexdigest().lower()
|
logger.error("Tor bundle verification failed for %s: %s", bundle_url, reason)
|
||||||
sha256_file.unlink(missing_ok=True)
|
archive_path.unlink(missing_ok=True)
|
||||||
if actual_hash != expected_hash:
|
continue
|
||||||
logger.error("SHA-256 mismatch for Tor download. Expected %s, got %s", expected_hash, actual_hash)
|
logger.info("Tor bundle %s", reason)
|
||||||
archive_path.unlink(missing_ok=True)
|
|
||||||
continue
|
|
||||||
logger.info("SHA-256 verified: %s", actual_hash[:16] + "...")
|
|
||||||
except Exception as hash_err:
|
|
||||||
logger.warning(
|
|
||||||
"Could not verify SHA-256 (hash file unavailable): %s; proceeding with HTTPS-only verification",
|
|
||||||
hash_err,
|
|
||||||
)
|
|
||||||
|
|
||||||
logger.info("Download complete, extracting...")
|
logger.info("Download complete, extracting...")
|
||||||
import tarfile
|
import tarfile
|
||||||
|
|
||||||
with tarfile.open(str(archive_path), "r:gz") as tar:
|
if not _extract_tor_bundle_safely(archive_path, TOR_INSTALL_DIR):
|
||||||
for member in tar.getmembers():
|
archive_path.unlink(missing_ok=True)
|
||||||
member_path = (TOR_INSTALL_DIR / member.name).resolve()
|
return None
|
||||||
if not str(member_path).startswith(str(TOR_INSTALL_DIR.resolve())):
|
|
||||||
logger.error("Tar path traversal blocked: %s", member.name)
|
|
||||||
archive_path.unlink(missing_ok=True)
|
|
||||||
return None
|
|
||||||
tar.extractall(path=str(TOR_INSTALL_DIR))
|
|
||||||
|
|
||||||
archive_path.unlink(missing_ok=True)
|
archive_path.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def disable_public_mesh_lane(*, reason: str = "private_lane_enabled") -> dict[str, Any]:
|
||||||
|
"""Disable public Meshtastic MQTT before private Wormhole/Infonet starts."""
|
||||||
|
result: dict[str, Any] = {
|
||||||
|
"ok": True,
|
||||||
|
"reason": reason,
|
||||||
|
"settings_disabled": False,
|
||||||
|
"runtime_stopped": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Scheduled Wormhole prewarm must not mutate the user's explicit public
|
||||||
|
# MeshChat session. Only a deliberate private-lane activation should sever
|
||||||
|
# the public MQTT lane.
|
||||||
|
normalized_reason = str(reason or "").strip().lower()
|
||||||
|
if normalized_reason == "wormhole_scheduled_prewarm" or normalized_reason.endswith(":scheduled_prewarm"):
|
||||||
|
try:
|
||||||
|
from services.meshtastic_mqtt_settings import mqtt_bridge_enabled
|
||||||
|
|
||||||
|
if mqtt_bridge_enabled():
|
||||||
|
logger.info("Keeping public Mesh lane active during Wormhole prewarm: %s", reason)
|
||||||
|
result["skipped"] = True
|
||||||
|
result["skip_reason"] = "public_mesh_user_enabled"
|
||||||
|
return result
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("Could not inspect public Mesh state during %s: %s", reason, exc)
|
||||||
|
|
||||||
|
logger.info("Disabling public Mesh lane: %s", reason)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from services.meshtastic_mqtt_settings import write_meshtastic_mqtt_settings
|
||||||
|
|
||||||
|
settings = write_meshtastic_mqtt_settings(enabled=False)
|
||||||
|
result["settings_disabled"] = not bool(settings.get("enabled"))
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("Failed to disable public Mesh settings during %s: %s", reason, exc)
|
||||||
|
result["ok"] = False
|
||||||
|
result["settings_error"] = str(exc)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from services.sigint_bridge import sigint_grid
|
||||||
|
|
||||||
|
if sigint_grid.mesh.is_running():
|
||||||
|
sigint_grid.mesh.stop()
|
||||||
|
result["runtime_stopped"] = not sigint_grid.mesh.is_running()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("Failed to stop public Mesh runtime during %s: %s", reason, exc)
|
||||||
|
result["ok"] = False
|
||||||
|
result["runtime_error"] = str(exc)
|
||||||
|
|
||||||
|
return result
|
||||||
@@ -24,7 +24,9 @@ from cachetools import TTLCache
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
_FINNHUB_BASE = "https://finnhub.io/api/v1"
|
_FINNHUB_BASE = "https://finnhub.io/api/v1"
|
||||||
_USER_AGENT = "ShadowBroker/0.9.75 Finnhub connector"
|
def _finnhub_user_agent():
|
||||||
|
from services.network_utils import outbound_user_agent
|
||||||
|
return outbound_user_agent("finnhub")
|
||||||
_REQUEST_TIMEOUT = 12
|
_REQUEST_TIMEOUT = 12
|
||||||
_MIN_INTERVAL_SECONDS = 0.35 # Stay well under 60 calls/min
|
_MIN_INTERVAL_SECONDS = 0.35 # Stay well under 60 calls/min
|
||||||
|
|
||||||
@@ -89,7 +91,7 @@ def _request(path: str, params: dict[str, Any] | None = None) -> Any:
|
|||||||
f"{_FINNHUB_BASE}{path}",
|
f"{_FINNHUB_BASE}{path}",
|
||||||
params=payload,
|
params=payload,
|
||||||
timeout=_REQUEST_TIMEOUT,
|
timeout=_REQUEST_TIMEOUT,
|
||||||
headers={"User-Agent": _USER_AGENT, "Accept": "application/json"},
|
headers={"User-Agent": _finnhub_user_agent(), "Accept": "application/json"},
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
_last_request_at = time.monotonic()
|
_last_request_at = time.monotonic()
|
||||||
|
|||||||
+232
-14
@@ -6,9 +6,11 @@ Public API:
|
|||||||
schedule_restart(project_root) (spawn detached start script, then exit)
|
schedule_restart(project_root) (spawn detached start script, then exit)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import logging
|
import logging
|
||||||
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
@@ -29,6 +31,19 @@ DOCKER_UPDATE_COMMANDS = (
|
|||||||
"docker compose pull && docker compose up -d"
|
"docker compose pull && docker compose up -d"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Issue #231: baked-in release digests. Loaded lazily, used as a fallback
|
||||||
|
# verification source when the release's SHA256SUMS.txt asset can't be
|
||||||
|
# fetched (e.g. transient network failure during update).
|
||||||
|
_RELEASE_DIGESTS_FILE = (
|
||||||
|
Path(__file__).resolve().parent.parent / "data" / "release_digests.json"
|
||||||
|
)
|
||||||
|
# Pattern for the maintainer's signed source-archive release asset. This
|
||||||
|
# is the file we prefer over the auto-generated ``zipball_url`` because
|
||||||
|
# the maintainer's build process publishes it with a matching entry in
|
||||||
|
# SHA256SUMS.txt — the zipball does not have a signed digest.
|
||||||
|
_SOURCE_ASSET_PATTERN = re.compile(r"^ShadowBroker_v\d", re.IGNORECASE)
|
||||||
|
_SHA256SUMS_ASSET_NAME = "SHA256SUMS.txt"
|
||||||
|
|
||||||
|
|
||||||
def _is_docker() -> bool:
|
def _is_docker() -> bool:
|
||||||
"""Detect if we're running inside a Docker container."""
|
"""Detect if we're running inside a Docker container."""
|
||||||
@@ -40,7 +55,6 @@ def _is_docker() -> bool:
|
|||||||
except (FileNotFoundError, PermissionError):
|
except (FileNotFoundError, PermissionError):
|
||||||
pass
|
pass
|
||||||
return os.environ.get("container") == "docker"
|
return os.environ.get("container") == "docker"
|
||||||
_EXPECTED_SHA256 = os.environ.get("MESH_UPDATE_SHA256", "").strip().lower()
|
|
||||||
_ALLOWED_UPDATE_HOSTS = {
|
_ALLOWED_UPDATE_HOSTS = {
|
||||||
"api.github.com",
|
"api.github.com",
|
||||||
"codeload.github.com",
|
"codeload.github.com",
|
||||||
@@ -119,7 +133,16 @@ def _validate_update_url(url: str, *, allow_release_page: bool = False) -> str:
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
def _download_release(temp_dir: str) -> tuple:
|
def _download_release(temp_dir: str) -> tuple:
|
||||||
"""Fetch latest release info and download the source zip archive.
|
"""Fetch latest release info and download the source zip archive.
|
||||||
Returns (zip_path, version_tag, download_url, release_url).
|
|
||||||
|
Issue #231: prefer the maintainer's signed release asset (matching
|
||||||
|
``ShadowBroker_v*.zip``) over the auto-generated ``zipball_url``,
|
||||||
|
because the maintainer's release process publishes a matching entry
|
||||||
|
in SHA256SUMS.txt for the named asset but NOT for the zipball.
|
||||||
|
|
||||||
|
Returns (zip_path, version_tag, download_url, release_url, asset_name,
|
||||||
|
sha256sums_url) — the last two are empty strings when the release
|
||||||
|
doesn't publish a signed asset, falling back to the legacy zipball
|
||||||
|
path.
|
||||||
"""
|
"""
|
||||||
logger.info("Fetching latest release info from GitHub...")
|
logger.info("Fetching latest release info from GitHub...")
|
||||||
_validate_update_url(GITHUB_RELEASES_URL)
|
_validate_update_url(GITHUB_RELEASES_URL)
|
||||||
@@ -131,9 +154,42 @@ def _download_release(temp_dir: str) -> tuple:
|
|||||||
tag = release.get("tag_name", "unknown")
|
tag = release.get("tag_name", "unknown")
|
||||||
release_url = str(release.get("html_url") or GITHUB_RELEASES_PAGE_URL).strip()
|
release_url = str(release.get("html_url") or GITHUB_RELEASES_PAGE_URL).strip()
|
||||||
_validate_update_url(release_url, allow_release_page=True)
|
_validate_update_url(release_url, allow_release_page=True)
|
||||||
zip_url = str(release.get("zipball_url") or "").strip()
|
|
||||||
if not zip_url:
|
# Prefer the maintainer-signed release asset. Fall back to the
|
||||||
raise RuntimeError("Latest release is missing a source archive URL")
|
# auto-generated zipball if the release doesn't publish one.
|
||||||
|
assets = release.get("assets") or []
|
||||||
|
asset_name = ""
|
||||||
|
asset_url = ""
|
||||||
|
sha256sums_url = ""
|
||||||
|
for a in assets:
|
||||||
|
name = str(a.get("name") or "").strip()
|
||||||
|
download = str(a.get("browser_download_url") or "").strip()
|
||||||
|
if not name or not download:
|
||||||
|
continue
|
||||||
|
if _SOURCE_ASSET_PATTERN.match(name) and name.lower().endswith(".zip"):
|
||||||
|
asset_name = name
|
||||||
|
asset_url = download
|
||||||
|
elif name == _SHA256SUMS_ASSET_NAME:
|
||||||
|
sha256sums_url = download
|
||||||
|
|
||||||
|
if asset_url:
|
||||||
|
zip_url = asset_url
|
||||||
|
logger.info(
|
||||||
|
"Using signed release asset %s (sha256sums=%s)",
|
||||||
|
asset_name,
|
||||||
|
"yes" if sha256sums_url else "no",
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
zip_url = str(release.get("zipball_url") or "").strip()
|
||||||
|
if not zip_url:
|
||||||
|
raise RuntimeError("Latest release is missing a source archive URL")
|
||||||
|
logger.warning(
|
||||||
|
"Release does not publish a signed ShadowBroker_v*.zip asset — "
|
||||||
|
"falling back to auto-generated zipball_url. Integrity will be "
|
||||||
|
"verified against the baked-in release_digests.json (if present) "
|
||||||
|
"or HTTPS-only otherwise."
|
||||||
|
)
|
||||||
|
|
||||||
_validate_update_url(zip_url)
|
_validate_update_url(zip_url)
|
||||||
|
|
||||||
logger.info(f"Downloading {zip_url} ...")
|
logger.info(f"Downloading {zip_url} ...")
|
||||||
@@ -150,19 +206,174 @@ def _download_release(temp_dir: str) -> tuple:
|
|||||||
|
|
||||||
size_mb = os.path.getsize(zip_path) / (1024 * 1024)
|
size_mb = os.path.getsize(zip_path) / (1024 * 1024)
|
||||||
logger.info(f"Downloaded {size_mb:.1f} MB — ZIP validated OK")
|
logger.info(f"Downloaded {size_mb:.1f} MB — ZIP validated OK")
|
||||||
return zip_path, tag, zip_url, release_url
|
return zip_path, tag, zip_url, release_url, asset_name, sha256sums_url
|
||||||
|
|
||||||
|
|
||||||
def _validate_zip_hash(zip_path: str) -> None:
|
def _compute_sha256(zip_path: str) -> str:
|
||||||
if not _EXPECTED_SHA256:
|
"""Return the hex SHA-256 of the file at ``zip_path`` (lowercase)."""
|
||||||
return
|
|
||||||
h = hashlib.sha256()
|
h = hashlib.sha256()
|
||||||
with open(zip_path, "rb") as f:
|
with open(zip_path, "rb") as f:
|
||||||
for chunk in iter(lambda: f.read(1024 * 128), b""):
|
for chunk in iter(lambda: f.read(1024 * 128), b""):
|
||||||
h.update(chunk)
|
h.update(chunk)
|
||||||
digest = h.hexdigest().lower()
|
return h.hexdigest().lower()
|
||||||
if digest != _EXPECTED_SHA256:
|
|
||||||
raise RuntimeError("Update SHA-256 mismatch")
|
|
||||||
|
def _load_baked_in_release_digests() -> dict:
|
||||||
|
"""Return the ``release_digests.json`` mapping, or an empty dict.
|
||||||
|
|
||||||
|
Schema (issue #231):
|
||||||
|
{
|
||||||
|
"<release_tag>": {
|
||||||
|
"<asset_filename>": "<sha256_hex>",
|
||||||
|
...
|
||||||
|
},
|
||||||
|
...
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
raw = _RELEASE_DIGESTS_FILE.read_text(encoding="utf-8")
|
||||||
|
parsed = json.loads(raw)
|
||||||
|
except (OSError, ValueError) as exc:
|
||||||
|
logger.debug("Release digest file unreadable: %s", exc)
|
||||||
|
return {}
|
||||||
|
if not isinstance(parsed, dict):
|
||||||
|
return {}
|
||||||
|
cleaned: dict[str, dict[str, str]] = {}
|
||||||
|
for k, v in parsed.items():
|
||||||
|
if not isinstance(k, str) or k.startswith("_"):
|
||||||
|
continue
|
||||||
|
if isinstance(v, dict):
|
||||||
|
entries = {
|
||||||
|
fname: digest.strip().lower()
|
||||||
|
for fname, digest in v.items()
|
||||||
|
if isinstance(fname, str) and isinstance(digest, str)
|
||||||
|
}
|
||||||
|
if entries:
|
||||||
|
cleaned[k] = entries
|
||||||
|
return cleaned
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_sha256sums(sha256sums_url: str) -> dict[str, str]:
|
||||||
|
"""Download a SHA256SUMS.txt and return {filename: digest_hex_lower}.
|
||||||
|
|
||||||
|
Standard ``sha256sum`` format: ``<digest> <filename>`` per line. The
|
||||||
|
leading ``*`` binary-mode marker (e.g. ``<digest> *<filename>``) is
|
||||||
|
handled.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
_validate_update_url(sha256sums_url)
|
||||||
|
except RuntimeError as exc:
|
||||||
|
logger.warning("SHA256SUMS URL rejected: %s", exc)
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
resp = requests.get(sha256sums_url, timeout=15)
|
||||||
|
resp.raise_for_status()
|
||||||
|
except requests.RequestException as exc:
|
||||||
|
logger.info("SHA256SUMS fetch failed: %s", exc)
|
||||||
|
return {}
|
||||||
|
out: dict[str, str] = {}
|
||||||
|
for line in resp.text.splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if not line or line.startswith("#"):
|
||||||
|
continue
|
||||||
|
# Tolerant split: handle both `<digest> <name>` and `<digest> *<name>`.
|
||||||
|
parts = line.split(None, 1)
|
||||||
|
if len(parts) != 2:
|
||||||
|
continue
|
||||||
|
digest, fname = parts
|
||||||
|
fname = fname.lstrip("*").strip()
|
||||||
|
digest = digest.strip().lower()
|
||||||
|
if len(digest) == 64 and all(c in "0123456789abcdef" for c in digest) and fname:
|
||||||
|
out[fname] = digest
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_zip_hash(
|
||||||
|
zip_path: str,
|
||||||
|
*,
|
||||||
|
asset_name: str = "",
|
||||||
|
sha256sums_url: str = "",
|
||||||
|
release_tag: str = "",
|
||||||
|
) -> str:
|
||||||
|
"""Verify the downloaded archive against trusted digest sources.
|
||||||
|
|
||||||
|
Issue #231: previously this returned silently when ``MESH_UPDATE_SHA256``
|
||||||
|
was unset, which made the auto-updater a supply-chain RCE vector on any
|
||||||
|
compromise of the GitHub release pipeline. The chain now is:
|
||||||
|
|
||||||
|
1. ``MESH_UPDATE_SHA256`` env var (operator override — preserved for
|
||||||
|
power-users who want to pin an exact digest manually)
|
||||||
|
2. ``SHA256SUMS.txt`` release asset (primary — the maintainer's
|
||||||
|
release process already publishes this)
|
||||||
|
3. Baked-in ``backend/data/release_digests.json`` (second line of
|
||||||
|
defense for releases that lack the SHA256SUMS asset, or when the
|
||||||
|
asset can't be fetched at update time)
|
||||||
|
4. HTTPS-only fallback with a loud warning (preserves the auto-update
|
||||||
|
flow during transient outages — but never silently)
|
||||||
|
|
||||||
|
A mismatch from a source that DID respond is fatal: the update is
|
||||||
|
refused and the existing install keeps running. Only the "no source
|
||||||
|
reachable at all" case falls back to HTTPS-only.
|
||||||
|
|
||||||
|
Returns a short human-readable description of which source verified
|
||||||
|
the archive (used in the update-success message).
|
||||||
|
"""
|
||||||
|
actual = _compute_sha256(zip_path)
|
||||||
|
|
||||||
|
# Source 1: explicit operator override.
|
||||||
|
override = os.environ.get("MESH_UPDATE_SHA256", "").strip().lower()
|
||||||
|
if override:
|
||||||
|
if actual == override:
|
||||||
|
return f"verified via MESH_UPDATE_SHA256 ({actual[:16]}...)"
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Update SHA-256 mismatch vs MESH_UPDATE_SHA256: archive={actual[:16]}..., "
|
||||||
|
f"expected={override[:16]}..."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Source 2: SHA256SUMS.txt asset from the release.
|
||||||
|
sums_map: dict[str, str] = {}
|
||||||
|
if sha256sums_url and asset_name:
|
||||||
|
sums_map = _fetch_sha256sums(sha256sums_url)
|
||||||
|
|
||||||
|
sums_expected = sums_map.get(asset_name) if asset_name else None
|
||||||
|
if sums_expected:
|
||||||
|
if actual == sums_expected:
|
||||||
|
return f"verified via release SHA256SUMS.txt ({actual[:16]}...)"
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Update SHA-256 mismatch vs release SHA256SUMS.txt: "
|
||||||
|
f"archive={actual[:16]}..., expected={sums_expected[:16]}..."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Source 3: baked-in digest list.
|
||||||
|
baked = _load_baked_in_release_digests()
|
||||||
|
baked_expected = ""
|
||||||
|
if release_tag and asset_name:
|
||||||
|
baked_expected = baked.get(release_tag, {}).get(asset_name, "")
|
||||||
|
if baked_expected:
|
||||||
|
if actual == baked_expected:
|
||||||
|
return f"verified via baked-in digest list ({actual[:16]}...)"
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Update SHA-256 mismatch vs baked-in digest list: "
|
||||||
|
f"archive={actual[:16]}..., expected={baked_expected[:16]}..."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Source 4: HTTPS-only fallback. We keep onboarding/auto-update working
|
||||||
|
# during transient outages (no SHA256SUMS reachable AND no baked-in
|
||||||
|
# entry for this release), but surface the degraded posture loudly so
|
||||||
|
# the operator can see it in logs and the maintainer can populate the
|
||||||
|
# digest list on the next release bump.
|
||||||
|
logger.warning(
|
||||||
|
"Update integrity check fell back to HTTPS-only trust "
|
||||||
|
"(no SHA256SUMS.txt response and no baked-in digest for "
|
||||||
|
"release=%s asset=%s). The archive SHA-256 is %s. Once the "
|
||||||
|
"release ships a SHA256SUMS.txt asset OR backend/data/"
|
||||||
|
"release_digests.json is updated with this release, the secure "
|
||||||
|
"path will activate automatically.",
|
||||||
|
release_tag or "unknown",
|
||||||
|
asset_name or "unknown",
|
||||||
|
actual,
|
||||||
|
)
|
||||||
|
return f"https-only (no digest source reachable, archive={actual[:16]}...)"
|
||||||
|
|
||||||
|
|
||||||
def _is_source_checkout(project_root: str) -> bool:
|
def _is_source_checkout(project_root: str) -> bool:
|
||||||
@@ -334,7 +545,7 @@ def perform_update(project_root: str) -> dict:
|
|||||||
temp_dir = tempfile.mkdtemp(prefix="sb_update_")
|
temp_dir = tempfile.mkdtemp(prefix="sb_update_")
|
||||||
manual_url = GITHUB_RELEASES_PAGE_URL
|
manual_url = GITHUB_RELEASES_PAGE_URL
|
||||||
try:
|
try:
|
||||||
zip_path, version, url, release_url = _download_release(temp_dir)
|
zip_path, version, url, release_url, asset_name, sha256sums_url = _download_release(temp_dir)
|
||||||
manual_url = release_url or manual_url
|
manual_url = release_url or manual_url
|
||||||
|
|
||||||
if in_docker:
|
if in_docker:
|
||||||
@@ -366,7 +577,13 @@ def perform_update(project_root: str) -> dict:
|
|||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
_validate_zip_hash(zip_path)
|
verification_note = _validate_zip_hash(
|
||||||
|
zip_path,
|
||||||
|
asset_name=asset_name,
|
||||||
|
sha256sums_url=sha256sums_url,
|
||||||
|
release_tag=version,
|
||||||
|
)
|
||||||
|
logger.info("Update archive %s", verification_note)
|
||||||
backup_path = _backup_current(project_root, temp_dir)
|
backup_path = _backup_current(project_root, temp_dir)
|
||||||
copied = _extract_and_copy(zip_path, project_root, temp_dir)
|
copied = _extract_and_copy(zip_path, project_root, temp_dir)
|
||||||
|
|
||||||
@@ -378,6 +595,7 @@ def perform_update(project_root: str) -> dict:
|
|||||||
"manual_url": manual_url,
|
"manual_url": manual_url,
|
||||||
"release_url": release_url,
|
"release_url": release_url,
|
||||||
"download_url": url,
|
"download_url": url,
|
||||||
|
"integrity": verification_note,
|
||||||
"message": f"Updated to {version} — {copied} files replaced. Restarting...",
|
"message": f"Updated to {version} — {copied} files replaced. Restarting...",
|
||||||
}
|
}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
@@ -243,6 +243,48 @@ def _pid_alive(pid: int) -> bool:
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _find_wormhole_server_pid() -> int:
|
||||||
|
if os.name == "nt":
|
||||||
|
return 0
|
||||||
|
proc_dir = Path("/proc")
|
||||||
|
if not proc_dir.exists():
|
||||||
|
return 0
|
||||||
|
current_pid = os.getpid()
|
||||||
|
script_name = WORMHOLE_SCRIPT.name
|
||||||
|
script_path = str(WORMHOLE_SCRIPT)
|
||||||
|
for entry in proc_dir.iterdir():
|
||||||
|
if not entry.name.isdigit():
|
||||||
|
continue
|
||||||
|
pid = int(entry.name)
|
||||||
|
if pid == current_pid:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
raw = (entry / "cmdline").read_bytes()
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
cmdline = raw.replace(b"\x00", b" ").decode("utf-8", errors="replace")
|
||||||
|
if script_path in cmdline or script_name in cmdline:
|
||||||
|
return pid
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _terminate_pid(pid: int, *, timeout_s: float = 5.0) -> None:
|
||||||
|
if os.name == "nt" or pid <= 0:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
os.kill(pid, signal.SIGTERM)
|
||||||
|
except Exception:
|
||||||
|
return
|
||||||
|
deadline = time.monotonic() + timeout_s
|
||||||
|
while time.monotonic() < deadline and _pid_alive(pid):
|
||||||
|
time.sleep(0.1)
|
||||||
|
if _pid_alive(pid):
|
||||||
|
try:
|
||||||
|
os.kill(pid, signal.SIGKILL)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def _probe_ready(timeout_s: float = 1.5) -> bool:
|
def _probe_ready(timeout_s: float = 1.5) -> bool:
|
||||||
try:
|
try:
|
||||||
with urlopen(f"http://{WORMHOLE_HOST}:{WORMHOLE_PORT}/api/health", timeout=timeout_s) as resp:
|
with urlopen(f"http://{WORMHOLE_HOST}:{WORMHOLE_PORT}/api/health", timeout=timeout_s) as resp:
|
||||||
@@ -266,17 +308,32 @@ def _probe_json(path: str, timeout_s: float = 1.5) -> dict[str, Any] | None:
|
|||||||
def _current_runtime_state() -> dict[str, Any]:
|
def _current_runtime_state() -> dict[str, Any]:
|
||||||
settings = read_wormhole_settings()
|
settings = read_wormhole_settings()
|
||||||
status = read_wormhole_status()
|
status = read_wormhole_status()
|
||||||
|
configured = bool(settings.get("enabled"))
|
||||||
running = False
|
running = False
|
||||||
|
ready = False
|
||||||
pid = int(status.get("pid", 0) or 0)
|
pid = int(status.get("pid", 0) or 0)
|
||||||
if _PROCESS and _PROCESS.poll() is None:
|
if not configured:
|
||||||
|
# Disabled private transport must stay disabled even if a stale local
|
||||||
|
# wormhole process is still answering on the health port. Public
|
||||||
|
# MeshChat relies on this state to keep the MQTT and Wormhole lanes
|
||||||
|
# mutually exclusive.
|
||||||
|
pid = 0
|
||||||
|
ready = False
|
||||||
|
elif _PROCESS and _PROCESS.poll() is None:
|
||||||
running = True
|
running = True
|
||||||
pid = int(_PROCESS.pid or 0)
|
pid = int(_PROCESS.pid or 0)
|
||||||
elif _pid_alive(pid):
|
else:
|
||||||
running = True
|
if _pid_alive(pid):
|
||||||
elif _probe_ready(timeout_s=0.35):
|
running = True
|
||||||
running = True
|
else:
|
||||||
pid = 0
|
discovered_pid = _find_wormhole_server_pid()
|
||||||
ready = running and _probe_ready()
|
if discovered_pid > 0:
|
||||||
|
running = True
|
||||||
|
pid = discovered_pid
|
||||||
|
if not running and _probe_ready(timeout_s=0.35):
|
||||||
|
running = True
|
||||||
|
pid = 0
|
||||||
|
ready = running and _probe_ready()
|
||||||
if not running:
|
if not running:
|
||||||
pid = 0
|
pid = 0
|
||||||
transport_active = status.get("transport_active", "") if ready else ""
|
transport_active = status.get("transport_active", "") if ready else ""
|
||||||
@@ -319,13 +376,13 @@ def _current_runtime_state() -> dict[str, Any]:
|
|||||||
anonymous_mode = bool(settings.get("anonymous_mode"))
|
anonymous_mode = bool(settings.get("anonymous_mode"))
|
||||||
anonymous_mode_ready = bool(
|
anonymous_mode_ready = bool(
|
||||||
anonymous_mode
|
anonymous_mode
|
||||||
and settings.get("enabled")
|
and configured
|
||||||
and ready
|
and ready
|
||||||
and effective_transport in {"tor", "tor_arti", "i2p", "mixnet"}
|
and effective_transport in {"tor", "tor_arti", "i2p", "mixnet"}
|
||||||
)
|
)
|
||||||
snapshot = {
|
snapshot = {
|
||||||
"installed": _installed(),
|
"installed": _installed(),
|
||||||
"configured": bool(settings.get("enabled")),
|
"configured": configured,
|
||||||
"running": running,
|
"running": running,
|
||||||
"ready": ready,
|
"ready": ready,
|
||||||
"transport_configured": str(settings.get("transport", "direct") or "direct"),
|
"transport_configured": str(settings.get("transport", "direct") or "direct"),
|
||||||
@@ -395,6 +452,12 @@ def get_wormhole_state() -> dict[str, Any]:
|
|||||||
def connect_wormhole(*, reason: str = "connect") -> dict[str, Any]:
|
def connect_wormhole(*, reason: str = "connect") -> dict[str, Any]:
|
||||||
with _LOCK:
|
with _LOCK:
|
||||||
_invalidate_state_cache()
|
_invalidate_state_cache()
|
||||||
|
try:
|
||||||
|
from services.transport_lane_isolation import disable_public_mesh_lane
|
||||||
|
|
||||||
|
disable_public_mesh_lane(reason=f"wormhole_{reason}")
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("Failed to enforce public/private lane isolation during %s: %s", reason, exc)
|
||||||
settings = read_wormhole_settings()
|
settings = read_wormhole_settings()
|
||||||
if not settings.get("enabled"):
|
if not settings.get("enabled"):
|
||||||
settings = settings.copy()
|
settings = settings.copy()
|
||||||
@@ -487,8 +550,8 @@ def connect_wormhole(*, reason: str = "connect") -> dict[str, Any]:
|
|||||||
def disconnect_wormhole(*, reason: str = "disconnect") -> dict[str, Any]:
|
def disconnect_wormhole(*, reason: str = "disconnect") -> dict[str, Any]:
|
||||||
with _LOCK:
|
with _LOCK:
|
||||||
_invalidate_state_cache()
|
_invalidate_state_cache()
|
||||||
current = _current_runtime_state()
|
status = read_wormhole_status()
|
||||||
pid = int(current.get("pid", 0) or 0)
|
pid = int(status.get("pid", 0) or 0)
|
||||||
global _PROCESS
|
global _PROCESS
|
||||||
if _PROCESS and _PROCESS.poll() is None:
|
if _PROCESS and _PROCESS.poll() is None:
|
||||||
try:
|
try:
|
||||||
@@ -499,14 +562,15 @@ def disconnect_wormhole(*, reason: str = "disconnect") -> dict[str, Any]:
|
|||||||
_PROCESS.kill()
|
_PROCESS.kill()
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
elif os.name != "nt" and _pid_alive(pid):
|
if os.name != "nt":
|
||||||
try:
|
_terminate_pid(pid)
|
||||||
os.kill(pid, signal.SIGTERM)
|
discovered_pid = _find_wormhole_server_pid()
|
||||||
except Exception:
|
if discovered_pid > 0 and discovered_pid != pid:
|
||||||
pass
|
_terminate_pid(discovered_pid)
|
||||||
_PROCESS = None
|
_PROCESS = None
|
||||||
write_wormhole_status(
|
write_wormhole_status(
|
||||||
reason=reason,
|
reason=reason,
|
||||||
|
configured=False,
|
||||||
running=False,
|
running=False,
|
||||||
ready=False,
|
ready=False,
|
||||||
pid=0,
|
pid=0,
|
||||||
|
|||||||
@@ -0,0 +1,677 @@
|
|||||||
|
{
|
||||||
|
"_meta": {
|
||||||
|
"issue": "#239",
|
||||||
|
"note": "Snapshot of currently-tolerated duplicate route registrations. The test in test_no_new_duplicate_routes.py fails if any NEW (method, path) duplicate appears outside this list. Removing entries (by actually deduping) is fine and the test stays green. New entries here require explicit, reviewed updates.",
|
||||||
|
"generated_with": "python -c 'see tests/test_no_new_duplicate_routes.py'"
|
||||||
|
},
|
||||||
|
"duplicates": {
|
||||||
|
"DELETE /api/mesh/peers": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_operator",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"DELETE /api/wormhole/dm/contact/{peer_id}": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"DELETE /api/wormhole/dm/invite/handles/{handle}": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"GET /api/cctv/media": [
|
||||||
|
"main",
|
||||||
|
"routers.cctv"
|
||||||
|
],
|
||||||
|
"GET /api/debug-latest": [
|
||||||
|
"main",
|
||||||
|
"routers.health"
|
||||||
|
],
|
||||||
|
"GET /api/geocode/reverse": [
|
||||||
|
"main",
|
||||||
|
"routers.tools"
|
||||||
|
],
|
||||||
|
"GET /api/geocode/search": [
|
||||||
|
"main",
|
||||||
|
"routers.tools"
|
||||||
|
],
|
||||||
|
"GET /api/health": [
|
||||||
|
"main",
|
||||||
|
"routers.health"
|
||||||
|
],
|
||||||
|
"GET /api/live-data": [
|
||||||
|
"main",
|
||||||
|
"routers.data"
|
||||||
|
],
|
||||||
|
"GET /api/live-data/fast": [
|
||||||
|
"main",
|
||||||
|
"routers.data"
|
||||||
|
],
|
||||||
|
"GET /api/live-data/slow": [
|
||||||
|
"main",
|
||||||
|
"routers.data"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/channels": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/dm/count": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_dm"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/dm/poll": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_dm"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/dm/prekey-bundle": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_dm"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/dm/pubkey": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_dm"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/dm/witness": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_dm"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/gate/list": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/gate/{gate_id}": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/gate/{gate_id}/messages": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/infonet/event/{event_id}": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/infonet/events": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/infonet/locator": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/infonet/merkle": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/infonet/messages": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/infonet/messages/wait": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/infonet/node/{node_id}": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/infonet/status": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/infonet/sync": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/log": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/messages": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/metrics": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/oracle/consensus": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_oracle"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/oracle/markets": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_oracle"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/oracle/markets/more": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_oracle"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/oracle/predictions": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_oracle"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/oracle/profile": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_oracle"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/oracle/search": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_oracle"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/oracle/stakes/{message_id}": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_oracle"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/peers": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_operator",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/reputation": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/reputation/all": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/reputation/batch": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/rns/status": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/signals": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/status": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"GET /api/mesh/trust/vouches": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_dm"
|
||||||
|
],
|
||||||
|
"GET /api/oracle/region-intel": [
|
||||||
|
"main",
|
||||||
|
"routers.sigint"
|
||||||
|
],
|
||||||
|
"GET /api/radio/nearest": [
|
||||||
|
"main",
|
||||||
|
"routers.radio"
|
||||||
|
],
|
||||||
|
"GET /api/radio/nearest-list": [
|
||||||
|
"main",
|
||||||
|
"routers.radio"
|
||||||
|
],
|
||||||
|
"GET /api/radio/openmhz/audio": [
|
||||||
|
"main",
|
||||||
|
"routers.radio"
|
||||||
|
],
|
||||||
|
"GET /api/radio/openmhz/calls/{sys_name}": [
|
||||||
|
"main",
|
||||||
|
"routers.radio"
|
||||||
|
],
|
||||||
|
"GET /api/radio/openmhz/systems": [
|
||||||
|
"main",
|
||||||
|
"routers.radio"
|
||||||
|
],
|
||||||
|
"GET /api/radio/top": [
|
||||||
|
"main",
|
||||||
|
"routers.radio"
|
||||||
|
],
|
||||||
|
"GET /api/refresh": [
|
||||||
|
"main",
|
||||||
|
"routers.data"
|
||||||
|
],
|
||||||
|
"GET /api/region-dossier": [
|
||||||
|
"main",
|
||||||
|
"routers.tools"
|
||||||
|
],
|
||||||
|
"GET /api/route/{callsign}": [
|
||||||
|
"main",
|
||||||
|
"routers.radio"
|
||||||
|
],
|
||||||
|
"GET /api/sentinel2/search": [
|
||||||
|
"main",
|
||||||
|
"routers.tools"
|
||||||
|
],
|
||||||
|
"GET /api/settings/api-keys": [
|
||||||
|
"main",
|
||||||
|
"routers.admin"
|
||||||
|
],
|
||||||
|
"GET /api/settings/api-keys/meta": [
|
||||||
|
"main",
|
||||||
|
"routers.admin"
|
||||||
|
],
|
||||||
|
"GET /api/settings/news-feeds": [
|
||||||
|
"main",
|
||||||
|
"routers.admin"
|
||||||
|
],
|
||||||
|
"GET /api/settings/node": [
|
||||||
|
"main",
|
||||||
|
"routers.admin"
|
||||||
|
],
|
||||||
|
"GET /api/settings/privacy-profile": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"GET /api/settings/wormhole": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"GET /api/settings/wormhole-status": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"GET /api/sigint/nearest-sdr": [
|
||||||
|
"main",
|
||||||
|
"routers.sigint"
|
||||||
|
],
|
||||||
|
"GET /api/thermal/verify": [
|
||||||
|
"main",
|
||||||
|
"routers.sigint"
|
||||||
|
],
|
||||||
|
"GET /api/tools/shodan/status": [
|
||||||
|
"main",
|
||||||
|
"routers.tools"
|
||||||
|
],
|
||||||
|
"GET /api/tools/uw/status": [
|
||||||
|
"main",
|
||||||
|
"routers.tools"
|
||||||
|
],
|
||||||
|
"GET /api/wormhole/dm/contacts": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"GET /api/wormhole/dm/identity": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"GET /api/wormhole/dm/invite": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"GET /api/wormhole/dm/invite/handles": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"GET /api/wormhole/gate/{gate_id}/identity": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"GET /api/wormhole/gate/{gate_id}/key": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"GET /api/wormhole/gate/{gate_id}/personas": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"GET /api/wormhole/health": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"GET /api/wormhole/identity": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"GET /api/wormhole/status": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"PATCH /api/mesh/peers": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_operator",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"POST /api/ais/feed": [
|
||||||
|
"main",
|
||||||
|
"routers.data"
|
||||||
|
],
|
||||||
|
"POST /api/layers": [
|
||||||
|
"main",
|
||||||
|
"routers.data"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/dm/block": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_dm"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/dm/count": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_dm"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/dm/poll": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_dm"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/dm/register": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_dm"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/dm/send": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_dm"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/dm/witness": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_dm"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/gate/create": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/gate/peer-pull": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_peer_sync"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/gate/peer-push": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_peer_sync"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/gate/{gate_id}/message": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/identity/revoke": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/identity/rotate": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/infonet/ingest": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/infonet/peer-push": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_peer_sync"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/infonet/sync": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/oracle/predict": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_oracle"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/oracle/resolve": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_oracle"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/oracle/resolve-stakes": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_oracle"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/oracle/stake": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_oracle"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/peers": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_operator",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/report": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/send": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/trust/vouch": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_dm"
|
||||||
|
],
|
||||||
|
"POST /api/mesh/vote": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"POST /api/sentinel/tile": [
|
||||||
|
"main",
|
||||||
|
"routers.tools"
|
||||||
|
],
|
||||||
|
"POST /api/sentinel/token": [
|
||||||
|
"main",
|
||||||
|
"routers.tools"
|
||||||
|
],
|
||||||
|
"POST /api/settings/news-feeds/reset": [
|
||||||
|
"main",
|
||||||
|
"routers.admin"
|
||||||
|
],
|
||||||
|
"POST /api/sigint/transmit": [
|
||||||
|
"main",
|
||||||
|
"routers.sigint"
|
||||||
|
],
|
||||||
|
"POST /api/system/update": [
|
||||||
|
"main",
|
||||||
|
"routers.admin"
|
||||||
|
],
|
||||||
|
"POST /api/tools/shodan/count": [
|
||||||
|
"main",
|
||||||
|
"routers.tools"
|
||||||
|
],
|
||||||
|
"POST /api/tools/shodan/host": [
|
||||||
|
"main",
|
||||||
|
"routers.tools"
|
||||||
|
],
|
||||||
|
"POST /api/tools/shodan/search": [
|
||||||
|
"main",
|
||||||
|
"routers.tools"
|
||||||
|
],
|
||||||
|
"POST /api/tools/uw/congress": [
|
||||||
|
"main",
|
||||||
|
"routers.tools"
|
||||||
|
],
|
||||||
|
"POST /api/tools/uw/darkpool": [
|
||||||
|
"main",
|
||||||
|
"routers.tools"
|
||||||
|
],
|
||||||
|
"POST /api/tools/uw/flow": [
|
||||||
|
"main",
|
||||||
|
"routers.tools"
|
||||||
|
],
|
||||||
|
"POST /api/viewport": [
|
||||||
|
"main",
|
||||||
|
"routers.data"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/connect": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/disconnect": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/dm/bootstrap-decrypt": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/dm/bootstrap-encrypt": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/dm/build-seal": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/dm/compose": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/dm/dead-drop-token": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/dm/dead-drop-tokens": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/dm/decrypt": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/dm/encrypt": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/dm/invite/import": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/dm/open-seal": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/dm/pairwise-alias": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/dm/pairwise-alias/rotate": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/dm/prekey/register": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/dm/register-key": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/dm/reset": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/dm/sas": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/dm/sender-token": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/gate/enter": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/gate/key/grant": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/gate/key/rotate": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/gate/leave": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/gate/message/compose": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/gate/message/decrypt": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/gate/message/post": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/gate/message/post-encrypted": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/gate/message/sign-encrypted": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/gate/messages/decrypt": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/gate/persona/activate": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/gate/persona/clear": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/gate/persona/create": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/gate/persona/retire": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/gate/proof": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/gate/state/export": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/identity/bootstrap": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/join": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/leave": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/restart": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/sign": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"POST /api/wormhole/sign-raw": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"PUT /api/mesh/gate/{gate_id}/envelope_policy": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"PUT /api/mesh/gate/{gate_id}/legacy_envelope_fallback": [
|
||||||
|
"main",
|
||||||
|
"routers.mesh_public"
|
||||||
|
],
|
||||||
|
"PUT /api/settings/news-feeds": [
|
||||||
|
"main",
|
||||||
|
"routers.admin"
|
||||||
|
],
|
||||||
|
"PUT /api/settings/node": [
|
||||||
|
"main",
|
||||||
|
"routers.admin"
|
||||||
|
],
|
||||||
|
"PUT /api/settings/privacy-profile": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"PUT /api/settings/wormhole": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
],
|
||||||
|
"PUT /api/wormhole/dm/contact": [
|
||||||
|
"main",
|
||||||
|
"routers.wormhole"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,7 +5,7 @@ from starlette.requests import Request
|
|||||||
from starlette.responses import Response
|
from starlette.responses import Response
|
||||||
|
|
||||||
|
|
||||||
def _request(path: str, method: str = "POST") -> Request:
|
def _request(path: str, method: str = "POST", query_string: bytes = b"") -> Request:
|
||||||
return Request(
|
return Request(
|
||||||
{
|
{
|
||||||
"type": "http",
|
"type": "http",
|
||||||
@@ -13,6 +13,7 @@ def _request(path: str, method: str = "POST") -> Request:
|
|||||||
"client": ("test", 12345),
|
"client": ("test", 12345),
|
||||||
"method": method,
|
"method": method,
|
||||||
"path": path,
|
"path": path,
|
||||||
|
"query_string": query_string,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -504,6 +505,61 @@ def test_private_infonet_gate_write_returns_preparing_state_when_wormhole_not_re
|
|||||||
get_settings.cache_clear()
|
get_settings.cache_clear()
|
||||||
|
|
||||||
|
|
||||||
|
def test_invite_scoped_prekey_lookup_reaches_handler_while_lane_prepares(monkeypatch):
|
||||||
|
"""Copied-address import must not be blocked by private-lane warmup."""
|
||||||
|
import main
|
||||||
|
import auth
|
||||||
|
from services.config import get_settings
|
||||||
|
from services import wormhole_supervisor
|
||||||
|
|
||||||
|
monkeypatch.setenv("MESH_PRIVATE_CLEARNET_FALLBACK", "block")
|
||||||
|
monkeypatch.setenv("MESH_BLOCK_LEGACY_NODE_ID_COMPAT", "true")
|
||||||
|
monkeypatch.setenv("MESH_BLOCK_LEGACY_AGENT_ID_LOOKUP", "true")
|
||||||
|
monkeypatch.setenv("MESH_ALLOW_COMPAT_DM_INVITE_IMPORT", "false")
|
||||||
|
get_settings.cache_clear()
|
||||||
|
monkeypatch.setattr(
|
||||||
|
auth,
|
||||||
|
"_anonymous_mode_state",
|
||||||
|
lambda: {
|
||||||
|
"enabled": False,
|
||||||
|
"wormhole_enabled": True,
|
||||||
|
"ready": False,
|
||||||
|
"effective_transport": "direct",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
wormhole_supervisor,
|
||||||
|
"get_wormhole_state",
|
||||||
|
lambda: {
|
||||||
|
"configured": True,
|
||||||
|
"ready": False,
|
||||||
|
"rns_ready": False,
|
||||||
|
"arti_ready": False,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
called = {"value": False}
|
||||||
|
|
||||||
|
async def call_next(_request: Request) -> Response:
|
||||||
|
called["value"] = True
|
||||||
|
return Response(status_code=200)
|
||||||
|
|
||||||
|
response = asyncio.run(
|
||||||
|
main.enforce_high_privacy_mesh(
|
||||||
|
_request(
|
||||||
|
"/api/mesh/dm/prekey-bundle",
|
||||||
|
method="GET",
|
||||||
|
query_string=b"lookup_token=invite-handle",
|
||||||
|
),
|
||||||
|
call_next,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert called["value"] is True
|
||||||
|
get_settings.cache_clear()
|
||||||
|
|
||||||
|
|
||||||
def test_private_dm_send_blocks_at_transitional_tier(monkeypatch):
|
def test_private_dm_send_blocks_at_transitional_tier(monkeypatch):
|
||||||
import main
|
import main
|
||||||
import auth
|
import auth
|
||||||
|
|||||||
@@ -47,6 +47,11 @@ def test_infonet_ingest_accepts_valid_event(tmp_path, monkeypatch):
|
|||||||
|
|
||||||
assert result["accepted"] == 1
|
assert result["accepted"] == 1
|
||||||
assert inf.head_hash == evt.event_id
|
assert inf.head_hash == evt.event_id
|
||||||
|
info = inf.get_info()
|
||||||
|
assert info["known_nodes"] == 1
|
||||||
|
assert info["author_nodes"] == 1
|
||||||
|
assert info["total_events"] == 1
|
||||||
|
assert info["event_types"]["message"] == 1
|
||||||
|
|
||||||
|
|
||||||
def test_verify_node_binding_accepts_current_and_compat_ids_only(monkeypatch):
|
def test_verify_node_binding_accepts_current_and_compat_ids_only(monkeypatch):
|
||||||
@@ -64,6 +69,8 @@ def test_verify_node_binding_accepts_current_and_compat_ids_only(monkeypatch):
|
|||||||
f"{current[len(mesh_crypto.NODE_ID_PREFIX):len(mesh_crypto.NODE_ID_PREFIX) + 8]}"
|
f"{current[len(mesh_crypto.NODE_ID_PREFIX):len(mesh_crypto.NODE_ID_PREFIX) + 8]}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
monkeypatch.setenv("MESH_DEV_ALLOW_LEGACY_COMPAT", "true")
|
||||||
|
monkeypatch.setenv("MESH_BLOCK_LEGACY_NODE_ID_COMPAT", "false")
|
||||||
monkeypatch.setenv("MESH_ALLOW_LEGACY_NODE_ID_COMPAT_UNTIL", "2099-01-01")
|
monkeypatch.setenv("MESH_ALLOW_LEGACY_NODE_ID_COMPAT_UNTIL", "2099-01-01")
|
||||||
from services.config import get_settings
|
from services.config import get_settings
|
||||||
|
|
||||||
@@ -98,7 +105,7 @@ def test_infonet_append_rejects_missing_signature_fields(tmp_path, monkeypatch):
|
|||||||
assert "signature" in str(exc).lower()
|
assert "signature" in str(exc).lower()
|
||||||
|
|
||||||
|
|
||||||
def test_infonet_load_fails_closed_on_hash_mismatch(tmp_path, monkeypatch):
|
def test_infonet_load_quarantines_and_resets_on_hash_mismatch(tmp_path, monkeypatch):
|
||||||
monkeypatch.setattr(mesh_hashchain, "DATA_DIR", tmp_path)
|
monkeypatch.setattr(mesh_hashchain, "DATA_DIR", tmp_path)
|
||||||
monkeypatch.setattr(mesh_hashchain, "CHAIN_FILE", tmp_path / "infonet.json")
|
monkeypatch.setattr(mesh_hashchain, "CHAIN_FILE", tmp_path / "infonet.json")
|
||||||
|
|
||||||
@@ -135,8 +142,12 @@ def test_infonet_load_fails_closed_on_hash_mismatch(tmp_path, monkeypatch):
|
|||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="Hash mismatch on event load"):
|
inf = mesh_hashchain.Infonet()
|
||||||
mesh_hashchain.Infonet()
|
|
||||||
|
assert inf.events == []
|
||||||
|
assert inf.head_hash == mesh_hashchain.GENESIS_HASH
|
||||||
|
assert not mesh_hashchain.CHAIN_FILE.exists()
|
||||||
|
assert list(tmp_path.glob("infonet.json.quarantine.*"))
|
||||||
|
|
||||||
|
|
||||||
def test_validate_gate_message_payload_rejects_plaintext_shape():
|
def test_validate_gate_message_payload_rejects_plaintext_shape():
|
||||||
|
|||||||
@@ -37,6 +37,30 @@ def test_eligible_sync_peers_filters_bucket_and_cooldown():
|
|||||||
assert [record.peer_url for record in candidates] == ["https://active.example"]
|
assert [record.peer_url for record in candidates] == ["https://active.example"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_eligible_sync_peers_prioritizes_explicit_bootstrap_seed():
|
||||||
|
old_runtime = make_sync_peer_record(
|
||||||
|
peer_url="https://old-runtime.example",
|
||||||
|
transport="clearnet",
|
||||||
|
role="participant",
|
||||||
|
source="runtime",
|
||||||
|
now=100,
|
||||||
|
)
|
||||||
|
seed = make_sync_peer_record(
|
||||||
|
peer_url="https://node.shadowbroker.info",
|
||||||
|
transport="clearnet",
|
||||||
|
role="seed",
|
||||||
|
source="bundle",
|
||||||
|
now=200,
|
||||||
|
)
|
||||||
|
|
||||||
|
candidates = eligible_sync_peers([old_runtime, seed], now=300)
|
||||||
|
|
||||||
|
assert [record.peer_url for record in candidates] == [
|
||||||
|
"https://node.shadowbroker.info",
|
||||||
|
"https://old-runtime.example",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def test_finish_sync_success_updates_schedule():
|
def test_finish_sync_success_updates_schedule():
|
||||||
state = begin_sync(SyncWorkerState(), peer_url="https://seed.example", now=100)
|
state = begin_sync(SyncWorkerState(), peer_url="https://seed.example", now=100)
|
||||||
finished = finish_sync(
|
finished = finish_sync(
|
||||||
|
|||||||
@@ -96,3 +96,38 @@ def test_peer_store_failure_and_success_lifecycle(tmp_path):
|
|||||||
assert recovered.cooldown_until == 0
|
assert recovered.cooldown_until == 0
|
||||||
assert recovered.last_error == ""
|
assert recovered.last_error == ""
|
||||||
assert recovered.last_sync_ok_at == 250
|
assert recovered.last_sync_ok_at == 250
|
||||||
|
|
||||||
|
|
||||||
|
def test_upsert_explicit_seed_clears_stale_cooldown(tmp_path):
|
||||||
|
store = PeerStore(tmp_path / "peer_store.json")
|
||||||
|
store.upsert(
|
||||||
|
make_sync_peer_record(
|
||||||
|
peer_url="https://node.shadowbroker.info",
|
||||||
|
transport="clearnet",
|
||||||
|
role="seed",
|
||||||
|
source="bundle",
|
||||||
|
now=100,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
failed = store.mark_failure(
|
||||||
|
"https://node.shadowbroker.info",
|
||||||
|
"sync",
|
||||||
|
error="timed out",
|
||||||
|
cooldown_s=120,
|
||||||
|
now=110,
|
||||||
|
)
|
||||||
|
assert failed.cooldown_until == 230
|
||||||
|
|
||||||
|
refreshed = store.upsert(
|
||||||
|
make_sync_peer_record(
|
||||||
|
peer_url="https://node.shadowbroker.info",
|
||||||
|
transport="clearnet",
|
||||||
|
role="seed",
|
||||||
|
source="bundle",
|
||||||
|
now=120,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert refreshed.failure_count == 0
|
||||||
|
assert refreshed.cooldown_until == 0
|
||||||
|
assert refreshed.last_error == ""
|
||||||
|
|||||||
@@ -2,15 +2,21 @@ from services.mesh.meshtastic_topics import build_subscription_topics, known_roo
|
|||||||
|
|
||||||
|
|
||||||
def test_default_subscription_is_longfast_only():
|
def test_default_subscription_is_longfast_only():
|
||||||
assert build_subscription_topics() == ["msh/US/2/e/LongFast/#"]
|
assert build_subscription_topics() == [
|
||||||
|
"msh/US/2/e/LongFast/#",
|
||||||
|
"msh/US/2/json/LongFast/#",
|
||||||
|
]
|
||||||
assert known_roots() == ["US"]
|
assert known_roots() == ["US"]
|
||||||
|
|
||||||
|
|
||||||
def test_extra_roots_are_longfast_only():
|
def test_extra_roots_are_longfast_only():
|
||||||
assert build_subscription_topics(extra_roots="EU_868,ANZ") == [
|
assert build_subscription_topics(extra_roots="EU_868,ANZ") == [
|
||||||
"msh/US/2/e/LongFast/#",
|
"msh/US/2/e/LongFast/#",
|
||||||
|
"msh/US/2/json/LongFast/#",
|
||||||
"msh/EU_868/2/e/LongFast/#",
|
"msh/EU_868/2/e/LongFast/#",
|
||||||
|
"msh/EU_868/2/json/LongFast/#",
|
||||||
"msh/ANZ/2/e/LongFast/#",
|
"msh/ANZ/2/e/LongFast/#",
|
||||||
|
"msh/ANZ/2/json/LongFast/#",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ Tests verify:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import json
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from services.config import get_settings
|
from services.config import get_settings
|
||||||
@@ -611,6 +612,99 @@ class TestFetchPrekeyBundleByLookup:
|
|||||||
"peer prekey lookup unavailable",
|
"peer prekey lookup unavailable",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def test_fetch_lookup_token_uses_bootstrap_peer_without_agent_id(self, tmp_path, monkeypatch):
|
||||||
|
"""Invite lookup can resolve through bootstrap peers without exposing agent_id."""
|
||||||
|
_isolated_relay(tmp_path, monkeypatch)
|
||||||
|
record = _valid_bundle_record("test-agent")
|
||||||
|
requested_urls: list[str] = []
|
||||||
|
|
||||||
|
monkeypatch.setenv("MESH_BOOTSTRAP_SEED_PEERS", "https://seed.example")
|
||||||
|
monkeypatch.setenv("MESH_DEFAULT_SYNC_PEERS", "")
|
||||||
|
monkeypatch.setenv("MESH_RELAY_PEERS", "")
|
||||||
|
get_settings.cache_clear()
|
||||||
|
|
||||||
|
class _Response:
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_args):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def read(self, _limit: int = -1):
|
||||||
|
return json.dumps(
|
||||||
|
{
|
||||||
|
"ok": True,
|
||||||
|
"identity_dh_pub_key": record["dh_pub_key"],
|
||||||
|
"dh_algo": record["dh_algo"],
|
||||||
|
"public_key": record["public_key"],
|
||||||
|
"public_key_algo": record["public_key_algo"],
|
||||||
|
"protocol_version": record["protocol_version"],
|
||||||
|
"sequence": 1,
|
||||||
|
"signed_at": int(record["bundle"].get("signed_at", 0) or 0),
|
||||||
|
"bundle": record["bundle"],
|
||||||
|
}
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
def _urlopen(request, timeout=0):
|
||||||
|
requested_urls.append(str(getattr(request, "full_url", "")))
|
||||||
|
return _Response()
|
||||||
|
|
||||||
|
monkeypatch.setattr("services.mesh.mesh_wormhole_prekey.urllib.request.urlopen", _urlopen)
|
||||||
|
|
||||||
|
from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle
|
||||||
|
|
||||||
|
result = fetch_dm_prekey_bundle(agent_id="", lookup_token="bootstrap-handle")
|
||||||
|
|
||||||
|
assert result["ok"] is True
|
||||||
|
assert result["agent_id"] == record["agent_id"]
|
||||||
|
assert result["lookup_mode"] == "invite_lookup_handle"
|
||||||
|
assert result["public_lookup"] is True
|
||||||
|
assert requested_urls
|
||||||
|
assert "lookup_token=bootstrap-handle" in requested_urls[0]
|
||||||
|
assert "agent_id" not in requested_urls[0]
|
||||||
|
|
||||||
|
def test_fetch_lookup_token_does_not_parse_peer_pending_as_bundle(self, tmp_path, monkeypatch):
|
||||||
|
"""A peer's private-lane pending response is not a malformed prekey bundle."""
|
||||||
|
_isolated_relay(tmp_path, monkeypatch)
|
||||||
|
requested_urls: list[str] = []
|
||||||
|
|
||||||
|
monkeypatch.setenv("MESH_BOOTSTRAP_SEED_PEERS", "https://seed.example")
|
||||||
|
monkeypatch.setenv("MESH_DEFAULT_SYNC_PEERS", "")
|
||||||
|
monkeypatch.setenv("MESH_RELAY_PEERS", "")
|
||||||
|
get_settings.cache_clear()
|
||||||
|
|
||||||
|
class _Response:
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_args):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def read(self, _limit: int = -1):
|
||||||
|
return json.dumps(
|
||||||
|
{
|
||||||
|
"ok": True,
|
||||||
|
"pending": True,
|
||||||
|
"status": "preparing_private_lane",
|
||||||
|
"detail": "transport tier insufficient",
|
||||||
|
}
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
def _urlopen(request, timeout=0):
|
||||||
|
requested_urls.append(str(getattr(request, "full_url", "")))
|
||||||
|
return _Response()
|
||||||
|
|
||||||
|
monkeypatch.setattr("services.mesh.mesh_wormhole_prekey.urllib.request.urlopen", _urlopen)
|
||||||
|
|
||||||
|
from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle
|
||||||
|
|
||||||
|
result = fetch_dm_prekey_bundle(agent_id="", lookup_token="bootstrap-handle")
|
||||||
|
|
||||||
|
assert requested_urls
|
||||||
|
assert result["ok"] is False
|
||||||
|
assert result["detail"] == "peer prekey lookup still preparing"
|
||||||
|
assert result["detail"] != "Prekey bundle missing signing key"
|
||||||
|
|
||||||
def test_fetch_agent_id_uses_pinned_contact_lookup_handle(self, tmp_path, monkeypatch):
|
def test_fetch_agent_id_uses_pinned_contact_lookup_handle(self, tmp_path, monkeypatch):
|
||||||
"""Pinned invite lookup handle is used before direct agent_id lookup."""
|
"""Pinned invite lookup handle is used before direct agent_id lookup."""
|
||||||
relay = _isolated_relay(tmp_path, monkeypatch)
|
relay = _isolated_relay(tmp_path, monkeypatch)
|
||||||
|
|||||||
@@ -71,6 +71,38 @@ def _fresh_wormhole_state(tmp_path, monkeypatch):
|
|||||||
return relay, mesh_wormhole_identity, mesh_wormhole_contacts, mesh_wormhole_prekey
|
return relay, mesh_wormhole_identity, mesh_wormhole_contacts, mesh_wormhole_prekey
|
||||||
|
|
||||||
|
|
||||||
|
def test_register_wormhole_dm_key_repairs_missing_local_dh_material(tmp_path, monkeypatch):
|
||||||
|
relay, identity_mod, _contacts_mod, _prekey_mod = _fresh_wormhole_state(tmp_path, monkeypatch)
|
||||||
|
identity = identity_mod.read_wormhole_identity()
|
||||||
|
original_node_id = identity["node_id"]
|
||||||
|
original_public_key = identity["public_key"]
|
||||||
|
original_private_key = identity["private_key"]
|
||||||
|
|
||||||
|
identity_mod.write_dm_identity(
|
||||||
|
{
|
||||||
|
**identity,
|
||||||
|
"dh_pub_key": "",
|
||||||
|
"dh_private_key": "",
|
||||||
|
"bundle_fingerprint": "",
|
||||||
|
"bundle_sequence": 0,
|
||||||
|
"bundle_registered_at": 0,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
registered = identity_mod.register_wormhole_dm_key()
|
||||||
|
repaired = identity_mod.read_wormhole_identity()
|
||||||
|
|
||||||
|
assert registered["ok"] is True
|
||||||
|
assert registered["dh_pub_key"]
|
||||||
|
assert registered["dh_algo"] == "X25519"
|
||||||
|
assert repaired["dh_pub_key"] == registered["dh_pub_key"]
|
||||||
|
assert repaired["dh_private_key"]
|
||||||
|
assert repaired["node_id"] == original_node_id
|
||||||
|
assert repaired["public_key"] == original_public_key
|
||||||
|
assert repaired["private_key"] == original_private_key
|
||||||
|
assert relay.get_dh_key(original_node_id)["dh_pub_key"] == registered["dh_pub_key"]
|
||||||
|
|
||||||
|
|
||||||
def _export_verified_invite(identity_mod):
|
def _export_verified_invite(identity_mod):
|
||||||
exported = identity_mod.export_wormhole_dm_invite()
|
exported = identity_mod.export_wormhole_dm_invite()
|
||||||
assert exported["ok"] is True
|
assert exported["ok"] is True
|
||||||
@@ -460,6 +492,30 @@ def test_imported_dm_invite_pins_contact_as_invite_pinned(tmp_path, monkeypatch)
|
|||||||
assert contacts_mod.list_wormhole_dm_contacts()[imported["peer_id"]]["trust_level"] == "invite_pinned"
|
assert contacts_mod.list_wormhole_dm_contacts()[imported["peer_id"]]["trust_level"] == "invite_pinned"
|
||||||
|
|
||||||
|
|
||||||
|
def test_imported_dm_invite_saves_pending_contact_when_prekey_not_visible(tmp_path, monkeypatch):
|
||||||
|
_relay, identity_mod, contacts_mod, prekey_mod = _fresh_wormhole_state(tmp_path, monkeypatch)
|
||||||
|
exported, verified = _export_verified_invite(identity_mod)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
prekey_mod,
|
||||||
|
"fetch_dm_prekey_bundle",
|
||||||
|
lambda **_kw: {"ok": False, "detail": "Prekey bundle not found"},
|
||||||
|
)
|
||||||
|
|
||||||
|
imported = identity_mod.import_wormhole_dm_invite(exported["invite"], alias="alice")
|
||||||
|
contact = imported["contact"]
|
||||||
|
|
||||||
|
assert imported["ok"] is True
|
||||||
|
assert imported["pending_prekey"] is True
|
||||||
|
assert imported["peer_id"] == verified["peer_id"]
|
||||||
|
assert contact["alias"] == "alice"
|
||||||
|
assert contact["trust_level"] == "invite_pinned"
|
||||||
|
assert contact["invitePinnedPrekeyLookupHandle"] == exported["invite"]["payload"]["prekey_lookup_handle"]
|
||||||
|
assert contact["remotePrekeyLookupMode"] == "invite_lookup_handle"
|
||||||
|
assert contact["remotePrekeyFingerprint"] == verified["trust_fingerprint"]
|
||||||
|
assert contact["dhPubKey"] == ""
|
||||||
|
assert contacts_mod.list_wormhole_dm_contacts()[verified["peer_id"]]["trust_level"] == "invite_pinned"
|
||||||
|
|
||||||
|
|
||||||
def test_imported_dm_invite_requires_root_attested_prekey_bundle(tmp_path, monkeypatch):
|
def test_imported_dm_invite_requires_root_attested_prekey_bundle(tmp_path, monkeypatch):
|
||||||
relay, identity_mod, _contacts_mod, _prekey_mod = _fresh_wormhole_state(tmp_path, monkeypatch)
|
relay, identity_mod, _contacts_mod, _prekey_mod = _fresh_wormhole_state(tmp_path, monkeypatch)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,118 @@
|
|||||||
|
"""Issue #258 — AIS proxy SPKI pinning.
|
||||||
|
|
||||||
|
Most of the SPKI logic lives in ``backend/ais_proxy.js`` (Node) and can't
|
||||||
|
be unit-tested from Python directly. These tests cover the Python-side
|
||||||
|
glue: ``services.ais_stream.ais_proxy_status()`` (the snapshot the proxy
|
||||||
|
populates via stdout markers) and ``routers/health.py`` surfacing the
|
||||||
|
degraded TLS state.
|
||||||
|
|
||||||
|
Additionally, the pin-file structure is validated: it must be parseable
|
||||||
|
JSON, must contain an entry for ``stream.aisstream.io``, and each pin
|
||||||
|
must look like a base64-encoded SHA-256 hash.
|
||||||
|
"""
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from services import ais_stream
|
||||||
|
|
||||||
|
PIN_FILE = (
|
||||||
|
Path(__file__).resolve().parent.parent / "data" / "aisstream_spki_pins.json"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_pin_file_exists_and_is_valid_json():
|
||||||
|
assert PIN_FILE.exists(), f"Expected pin file at {PIN_FILE}"
|
||||||
|
data = json.loads(PIN_FILE.read_text(encoding="utf-8"))
|
||||||
|
assert isinstance(data, dict)
|
||||||
|
|
||||||
|
|
||||||
|
def test_pin_file_has_aisstream_entry():
|
||||||
|
data = json.loads(PIN_FILE.read_text(encoding="utf-8"))
|
||||||
|
pins = data.get("stream.aisstream.io")
|
||||||
|
assert isinstance(pins, list)
|
||||||
|
assert len(pins) >= 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_each_pin_looks_like_a_base64_sha256():
|
||||||
|
"""SPKI pins must be 44-char base64-encoded SHA-256 digests."""
|
||||||
|
data = json.loads(PIN_FILE.read_text(encoding="utf-8"))
|
||||||
|
pins = data["stream.aisstream.io"]
|
||||||
|
for pin in pins:
|
||||||
|
assert isinstance(pin, str), f"pin not a string: {pin!r}"
|
||||||
|
assert len(pin) == 44, f"pin {pin!r} not 44 chars (SHA-256 base64)"
|
||||||
|
# Must base64-decode to exactly 32 bytes (256 bits)
|
||||||
|
try:
|
||||||
|
raw = base64.b64decode(pin)
|
||||||
|
except Exception as exc:
|
||||||
|
pytest.fail(f"pin {pin!r} is not valid base64: {exc}")
|
||||||
|
assert len(raw) == 32, f"pin {pin!r} decodes to {len(raw)} bytes, expected 32"
|
||||||
|
# Should match the canonical base64 alphabet (no URL-safe variants)
|
||||||
|
assert re.match(r"^[A-Za-z0-9+/]+=*$", pin), f"pin {pin!r} contains non-base64 chars"
|
||||||
|
|
||||||
|
|
||||||
|
def test_ais_proxy_status_starts_empty():
|
||||||
|
"""Before the proxy emits any status marker, the snapshot is empty."""
|
||||||
|
# Clear any stale state from other tests
|
||||||
|
with ais_stream._vessels_lock:
|
||||||
|
ais_stream._proxy_status.clear()
|
||||||
|
status = ais_stream.ais_proxy_status()
|
||||||
|
assert status == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_ais_proxy_status_returns_copy_not_reference():
|
||||||
|
"""ais_proxy_status() must return a defensive copy.
|
||||||
|
|
||||||
|
Otherwise a caller could mutate the live dict and confuse later reads.
|
||||||
|
"""
|
||||||
|
with ais_stream._vessels_lock:
|
||||||
|
ais_stream._proxy_status.clear()
|
||||||
|
ais_stream._proxy_status["degraded_tls"] = True
|
||||||
|
|
||||||
|
snapshot = ais_stream.ais_proxy_status()
|
||||||
|
assert snapshot == {"degraded_tls": True}
|
||||||
|
snapshot["degraded_tls"] = False # mutate the returned copy
|
||||||
|
|
||||||
|
# Original should be untouched
|
||||||
|
re_snapshot = ais_stream.ais_proxy_status()
|
||||||
|
assert re_snapshot == {"degraded_tls": True}
|
||||||
|
|
||||||
|
# Cleanup so other tests start clean
|
||||||
|
with ais_stream._vessels_lock:
|
||||||
|
ais_stream._proxy_status.clear()
|
||||||
|
|
||||||
|
|
||||||
|
def test_health_includes_ais_proxy_field(client):
|
||||||
|
"""The /api/health response must include the ais_proxy block."""
|
||||||
|
# Inject a known degraded state
|
||||||
|
with ais_stream._vessels_lock:
|
||||||
|
ais_stream._proxy_status.clear()
|
||||||
|
ais_stream._proxy_status["degraded_tls"] = True
|
||||||
|
|
||||||
|
response = client.get("/api/health")
|
||||||
|
assert response.status_code == 200
|
||||||
|
payload = response.json()
|
||||||
|
|
||||||
|
assert "ais_proxy" in payload
|
||||||
|
assert payload["ais_proxy"] == {"degraded_tls": True}
|
||||||
|
# Top-level status should escalate from ok to degraded when AIS is
|
||||||
|
# in degraded-TLS mode (unless SLOs already report worse).
|
||||||
|
assert payload["status"] in {"degraded", "error"}
|
||||||
|
|
||||||
|
# Cleanup
|
||||||
|
with ais_stream._vessels_lock:
|
||||||
|
ais_stream._proxy_status.clear()
|
||||||
|
|
||||||
|
|
||||||
|
def test_health_ais_proxy_field_when_no_status(client):
|
||||||
|
"""When the proxy hasn't reported anything yet, ais_proxy is empty."""
|
||||||
|
with ais_stream._vessels_lock:
|
||||||
|
ais_stream._proxy_status.clear()
|
||||||
|
|
||||||
|
response = client.get("/api/health")
|
||||||
|
assert response.status_code == 200
|
||||||
|
payload = response.json()
|
||||||
|
assert payload.get("ais_proxy") == {}
|
||||||
@@ -0,0 +1,389 @@
|
|||||||
|
"""Issues #244, #245, #246 (tg12 external audit): carrier tracker
|
||||||
|
quality + provenance + freshness.
|
||||||
|
|
||||||
|
These tests pin the post-fix contract:
|
||||||
|
|
||||||
|
- **#244**: dated editorial snapshot positions no longer live in the
|
||||||
|
registry. They live in a one-shot seed file that is consumed once
|
||||||
|
on first-ever startup. After that, the runtime cache reflects only
|
||||||
|
what THIS install has actually observed.
|
||||||
|
|
||||||
|
- **#245**: headline-derived positions (centroid of a region keyword)
|
||||||
|
are stamped ``position_confidence = "approximate"`` so the UI can
|
||||||
|
render them with appropriate uncertainty.
|
||||||
|
|
||||||
|
- **#246**: freshness is a *labelling* decision, not an eviction
|
||||||
|
decision. Positions older than the configurable freshness window
|
||||||
|
flip from ``"recent"`` to ``"stale"`` but are NEVER replaced with
|
||||||
|
the registry default — that would teleport the carrier. The user
|
||||||
|
always sees the last position the system actually observed.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def fresh_tracker(tmp_path, monkeypatch):
|
||||||
|
"""Isolated carrier_tracker with seed/cache paths redirected to tmp.
|
||||||
|
|
||||||
|
Yields the module so tests can call its functions; resets globals
|
||||||
|
between tests so position caches don't leak across cases.
|
||||||
|
"""
|
||||||
|
from services import carrier_tracker
|
||||||
|
|
||||||
|
seed_path = tmp_path / "data" / "carrier_seed.json"
|
||||||
|
cache_path = tmp_path / "carrier_cache.json"
|
||||||
|
seed_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
monkeypatch.setattr(carrier_tracker, "SEED_FILE", seed_path)
|
||||||
|
monkeypatch.setattr(carrier_tracker, "CACHE_FILE", cache_path)
|
||||||
|
monkeypatch.delenv("SHADOWBROKER_CARRIER_FRESHNESS_DAYS", raising=False)
|
||||||
|
|
||||||
|
# Reset module-level mutable state.
|
||||||
|
carrier_tracker._carrier_positions.clear()
|
||||||
|
carrier_tracker._cached_gdelt_articles.clear()
|
||||||
|
carrier_tracker._last_gdelt_fetch_at = 0.0
|
||||||
|
|
||||||
|
yield carrier_tracker
|
||||||
|
|
||||||
|
# Clean up so subsequent tests start fresh.
|
||||||
|
carrier_tracker._carrier_positions.clear()
|
||||||
|
carrier_tracker._cached_gdelt_articles.clear()
|
||||||
|
|
||||||
|
|
||||||
|
def _write_seed(path: Path, hull: str = "CVN-78", **overrides) -> None:
|
||||||
|
payload = {
|
||||||
|
"_meta": {
|
||||||
|
"as_of": "2026-03-09",
|
||||||
|
"source": "USNI News Fleet & Marine Tracker",
|
||||||
|
"source_url": "https://news.usni.org/...",
|
||||||
|
"note": "test",
|
||||||
|
},
|
||||||
|
"carriers": {
|
||||||
|
hull: {
|
||||||
|
"lat": 18.0,
|
||||||
|
"lng": 39.5,
|
||||||
|
"heading": 0,
|
||||||
|
"desc": "Red Sea — Operation Epic Fury (USNI Mar 9)",
|
||||||
|
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||||
|
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||||
|
"position_source_at": "2026-03-09T00:00:00Z",
|
||||||
|
"position_confidence": "seed",
|
||||||
|
**overrides,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# #244 — first-run seed bootstrap, never re-seeds after that
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestSeedBootstrap:
|
||||||
|
def test_first_ever_startup_bootstraps_from_seed(self, fresh_tracker, tmp_path):
|
||||||
|
_write_seed(fresh_tracker.SEED_FILE)
|
||||||
|
# No cache exists yet.
|
||||||
|
assert not fresh_tracker.CACHE_FILE.exists()
|
||||||
|
|
||||||
|
positions = fresh_tracker._bootstrap_cache_if_missing()
|
||||||
|
|
||||||
|
# The seed entry made it into the cache.
|
||||||
|
assert "CVN-78" in positions
|
||||||
|
assert positions["CVN-78"]["lat"] == 18.0
|
||||||
|
assert positions["CVN-78"]["position_confidence"] == "seed"
|
||||||
|
# And the cache file is now on disk so subsequent runs skip the seed.
|
||||||
|
assert fresh_tracker.CACHE_FILE.exists()
|
||||||
|
|
||||||
|
def test_subsequent_startup_ignores_seed(self, fresh_tracker, tmp_path):
|
||||||
|
# Pre-seed a different position into the cache; the seed file says Red Sea.
|
||||||
|
cache_data = {
|
||||||
|
"CVN-78": {
|
||||||
|
"lat": 25.0,
|
||||||
|
"lng": 55.0,
|
||||||
|
"heading": 0,
|
||||||
|
"desc": "Persian Gulf — operator-observed",
|
||||||
|
"source": "Operator log",
|
||||||
|
"source_url": "",
|
||||||
|
"position_source_at": "2026-04-15T12:00:00Z",
|
||||||
|
"position_confidence": "recent",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fresh_tracker.CACHE_FILE.write_text(json.dumps(cache_data))
|
||||||
|
_write_seed(fresh_tracker.SEED_FILE) # seed is present but should NOT be used
|
||||||
|
|
||||||
|
positions = fresh_tracker._bootstrap_cache_if_missing()
|
||||||
|
|
||||||
|
assert positions["CVN-78"]["lat"] == 25.0
|
||||||
|
assert positions["CVN-78"]["desc"] == "Persian Gulf — operator-observed"
|
||||||
|
|
||||||
|
def test_no_seed_no_cache_falls_back_to_homeport(self, fresh_tracker):
|
||||||
|
# Neither seed nor cache. Must fall back to homeport defaults
|
||||||
|
# (carrier never disappears).
|
||||||
|
assert not fresh_tracker.SEED_FILE.exists()
|
||||||
|
assert not fresh_tracker.CACHE_FILE.exists()
|
||||||
|
|
||||||
|
positions = fresh_tracker._bootstrap_cache_if_missing()
|
||||||
|
|
||||||
|
# Every registered carrier has SOMETHING.
|
||||||
|
assert set(positions.keys()) == set(fresh_tracker.CARRIER_REGISTRY.keys())
|
||||||
|
# All entries are labelled as homeport defaults.
|
||||||
|
for hull, entry in positions.items():
|
||||||
|
assert entry["position_confidence"] == "homeport_default"
|
||||||
|
registry = fresh_tracker.CARRIER_REGISTRY[hull]
|
||||||
|
assert entry["lat"] == registry["homeport_lat"]
|
||||||
|
assert entry["lng"] == registry["homeport_lng"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# #244 — no editorial fallbacks live in the registry
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestRegistryShape:
|
||||||
|
def test_registry_has_no_dated_fallback_fields(self, fresh_tracker):
|
||||||
|
"""The Mar 9 editorial coordinates are gone from the registry.
|
||||||
|
They live only in the seed file."""
|
||||||
|
forbidden = {"fallback_lat", "fallback_lng", "fallback_heading", "fallback_desc"}
|
||||||
|
for hull, entry in fresh_tracker.CARRIER_REGISTRY.items():
|
||||||
|
offending = forbidden & set(entry.keys())
|
||||||
|
assert not offending, f"{hull} still has dated registry fields: {offending}"
|
||||||
|
|
||||||
|
def test_registry_keeps_homeport_for_every_hull(self, fresh_tracker):
|
||||||
|
for hull, entry in fresh_tracker.CARRIER_REGISTRY.items():
|
||||||
|
assert "homeport_lat" in entry, f"{hull} missing homeport_lat"
|
||||||
|
assert "homeport_lng" in entry, f"{hull} missing homeport_lng"
|
||||||
|
assert "name" in entry
|
||||||
|
assert "wiki" in entry
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# #246 — freshness labelling, NOT eviction
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestFreshnessLabelling:
|
||||||
|
def test_recent_observation_labels_recent(self, fresh_tracker):
|
||||||
|
now = datetime(2026, 6, 1, tzinfo=timezone.utc)
|
||||||
|
entry = {
|
||||||
|
"lat": 25.0,
|
||||||
|
"lng": 55.0,
|
||||||
|
"position_source_at": (now - timedelta(days=3)).isoformat(),
|
||||||
|
}
|
||||||
|
assert fresh_tracker._compute_position_confidence(entry, now=now) == "recent"
|
||||||
|
|
||||||
|
def test_aged_observation_flips_to_stale(self, fresh_tracker):
|
||||||
|
now = datetime(2026, 6, 1, tzinfo=timezone.utc)
|
||||||
|
entry = {
|
||||||
|
"lat": 25.0,
|
||||||
|
"lng": 55.0,
|
||||||
|
"position_source_at": (now - timedelta(days=30)).isoformat(),
|
||||||
|
}
|
||||||
|
assert fresh_tracker._compute_position_confidence(entry, now=now) == "stale"
|
||||||
|
|
||||||
|
def test_seed_label_is_preserved_explicitly(self, fresh_tracker):
|
||||||
|
now = datetime(2026, 6, 1, tzinfo=timezone.utc)
|
||||||
|
entry = {
|
||||||
|
"lat": 18.0,
|
||||||
|
"lng": 39.5,
|
||||||
|
"position_source_at": "2026-03-09T00:00:00Z",
|
||||||
|
"position_confidence": "seed",
|
||||||
|
}
|
||||||
|
# Even though the source is months old, the explicit "seed" label wins
|
||||||
|
# so the UI can render the seed-specific badge instead of generic "stale".
|
||||||
|
assert fresh_tracker._compute_position_confidence(entry, now=now) == "seed"
|
||||||
|
|
||||||
|
def test_homeport_default_label_is_preserved(self, fresh_tracker):
|
||||||
|
now = datetime(2026, 6, 1, tzinfo=timezone.utc)
|
||||||
|
entry = {
|
||||||
|
"lat": 36.95,
|
||||||
|
"lng": -76.32,
|
||||||
|
"position_source_at": now.isoformat(),
|
||||||
|
"position_confidence": "homeport_default",
|
||||||
|
}
|
||||||
|
assert fresh_tracker._compute_position_confidence(entry, now=now) == "homeport_default"
|
||||||
|
|
||||||
|
def test_freshness_window_is_env_configurable(self, fresh_tracker, monkeypatch):
|
||||||
|
now = datetime(2026, 6, 1, tzinfo=timezone.utc)
|
||||||
|
entry = {
|
||||||
|
"lat": 25.0,
|
||||||
|
"lng": 55.0,
|
||||||
|
"position_source_at": (now - timedelta(days=20)).isoformat(),
|
||||||
|
}
|
||||||
|
# Default window = 14 days → 20-day-old entry is stale.
|
||||||
|
assert fresh_tracker._compute_position_confidence(entry, now=now) == "stale"
|
||||||
|
# Stretch to 30 days → same entry is now "recent".
|
||||||
|
monkeypatch.setenv("SHADOWBROKER_CARRIER_FRESHNESS_DAYS", "30")
|
||||||
|
assert fresh_tracker._compute_position_confidence(entry, now=now) == "recent"
|
||||||
|
|
||||||
|
def test_aged_cache_entry_keeps_its_position_never_reverts(self, fresh_tracker):
|
||||||
|
"""The core regression test for the user's intent: a year-old
|
||||||
|
cache entry must NOT be replaced with the seed or homeport.
|
||||||
|
The PHYSICAL position the user sees is the last one observed;
|
||||||
|
only the freshness LABEL changes."""
|
||||||
|
a_year_ago = (datetime.now(timezone.utc) - timedelta(days=365)).isoformat()
|
||||||
|
cache_data = {
|
||||||
|
"CVN-78": {
|
||||||
|
"lat": 25.0,
|
||||||
|
"lng": 55.0,
|
||||||
|
"heading": 0,
|
||||||
|
"desc": "Persian Gulf",
|
||||||
|
"source": "GDELT News API",
|
||||||
|
"source_url": "https://news.example/...",
|
||||||
|
"position_source_at": a_year_ago,
|
||||||
|
"position_confidence": "recent", # was recent when written
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fresh_tracker.CACHE_FILE.write_text(json.dumps(cache_data))
|
||||||
|
|
||||||
|
positions = fresh_tracker._bootstrap_cache_if_missing()
|
||||||
|
enriched = fresh_tracker._enrich_for_rendering("CVN-78", positions["CVN-78"])
|
||||||
|
|
||||||
|
# The position is preserved exactly.
|
||||||
|
assert enriched["lat"] == 25.0
|
||||||
|
assert enriched["lng"] == 55.0
|
||||||
|
# But the live label has flipped to stale.
|
||||||
|
assert enriched["position_confidence"] == "stale"
|
||||||
|
assert enriched["is_fallback"] is True
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# #245 — approximate confidence for region-centroid positions
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestApproximateConfidenceForNewsDerivedPositions:
|
||||||
|
def test_news_parsing_stamps_approximate_confidence(self, fresh_tracker):
|
||||||
|
articles = [
|
||||||
|
{
|
||||||
|
"title": "USS Ford carrier deployed in Mediterranean for joint exercise",
|
||||||
|
"url": "https://news.example/ford-mediterranean",
|
||||||
|
"seendate": "20260415120000",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
updates = fresh_tracker._parse_carrier_positions_from_news(articles)
|
||||||
|
assert "CVN-78" in updates
|
||||||
|
entry = updates["CVN-78"]
|
||||||
|
assert entry["position_confidence"] == "approximate"
|
||||||
|
# And the source_at is the article's seen date, not now().
|
||||||
|
assert entry["position_source_at"].startswith("2026-04-15")
|
||||||
|
|
||||||
|
def test_gdelt_seendate_parser_handles_well_formed_input(self, fresh_tracker):
|
||||||
|
iso = fresh_tracker._gdelt_seendate_to_iso("20260415120000")
|
||||||
|
assert iso is not None
|
||||||
|
assert iso.startswith("2026-04-15T12:00:00")
|
||||||
|
|
||||||
|
def test_gdelt_seendate_parser_returns_none_on_garbage(self, fresh_tracker):
|
||||||
|
assert fresh_tracker._gdelt_seendate_to_iso("") is None
|
||||||
|
assert fresh_tracker._gdelt_seendate_to_iso("not-a-date") is None
|
||||||
|
assert fresh_tracker._gdelt_seendate_to_iso("2026") is None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Full enrichment → public API shape
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestEnrichForRendering:
|
||||||
|
def test_seed_entry_produces_expected_public_fields(self, fresh_tracker):
|
||||||
|
seed_entry = {
|
||||||
|
"lat": 18.0,
|
||||||
|
"lng": 39.5,
|
||||||
|
"heading": 0,
|
||||||
|
"desc": "Red Sea (USNI Mar 9)",
|
||||||
|
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||||
|
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||||
|
"position_source_at": "2026-03-09T00:00:00Z",
|
||||||
|
"position_confidence": "seed",
|
||||||
|
}
|
||||||
|
enriched = fresh_tracker._enrich_for_rendering("CVN-78", seed_entry)
|
||||||
|
# Existing UI fields preserved.
|
||||||
|
assert enriched["lat"] == 18.0
|
||||||
|
assert enriched["lng"] == 39.5
|
||||||
|
assert enriched["source"].startswith("USNI")
|
||||||
|
assert enriched["last_osint_update"] == "2026-03-09T00:00:00Z"
|
||||||
|
# New audit-required fields.
|
||||||
|
assert enriched["position_confidence"] == "seed"
|
||||||
|
assert enriched["position_source_at"] == "2026-03-09T00:00:00Z"
|
||||||
|
assert enriched["is_fallback"] is True
|
||||||
|
|
||||||
|
def test_recent_observation_is_not_fallback(self, fresh_tracker):
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
recent_entry = {
|
||||||
|
"lat": 25.0,
|
||||||
|
"lng": 55.0,
|
||||||
|
"heading": 0,
|
||||||
|
"desc": "Persian Gulf",
|
||||||
|
"source": "GDELT News API",
|
||||||
|
"source_url": "https://news.example/...",
|
||||||
|
"position_source_at": (now - timedelta(days=2)).isoformat(),
|
||||||
|
"position_confidence": "approximate",
|
||||||
|
}
|
||||||
|
enriched = fresh_tracker._enrich_for_rendering("CVN-78", recent_entry, now=now)
|
||||||
|
assert enriched["position_confidence"] == "approximate"
|
||||||
|
# Approximate (from a recent headline) is honest precision, but the UI
|
||||||
|
# treats it as live data — is_fallback only flips True for explicit
|
||||||
|
# fallback categories (seed / stale / homeport_default).
|
||||||
|
assert enriched["is_fallback"] is False
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Regression: existing frontend fields are preserved
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestPublicResponseShapeBackwardCompat:
|
||||||
|
"""The frontend ShipPopup expects `estimated`, `source`, `source_url`,
|
||||||
|
`last_osint_update`. The new fields are additive and existing fields
|
||||||
|
keep their meaning so the UI does not need updating to keep working."""
|
||||||
|
|
||||||
|
def test_get_carrier_positions_preserves_existing_keys(self, fresh_tracker):
|
||||||
|
_write_seed(fresh_tracker.SEED_FILE)
|
||||||
|
fresh_tracker._bootstrap_cache_if_missing()
|
||||||
|
with fresh_tracker._positions_lock:
|
||||||
|
fresh_tracker._carrier_positions.update(
|
||||||
|
{
|
||||||
|
"CVN-78": {
|
||||||
|
"lat": 18.0,
|
||||||
|
"lng": 39.5,
|
||||||
|
"heading": 0,
|
||||||
|
"desc": "Red Sea (seed)",
|
||||||
|
"source": "Seed",
|
||||||
|
"source_url": "",
|
||||||
|
"position_source_at": "2026-03-09T00:00:00Z",
|
||||||
|
"position_confidence": "seed",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
out = fresh_tracker.get_carrier_positions()
|
||||||
|
assert len(out) == 1
|
||||||
|
c = out[0]
|
||||||
|
# Old fields the frontend uses.
|
||||||
|
for key in (
|
||||||
|
"name",
|
||||||
|
"type",
|
||||||
|
"lat",
|
||||||
|
"lng",
|
||||||
|
"country",
|
||||||
|
"desc",
|
||||||
|
"wiki",
|
||||||
|
"estimated",
|
||||||
|
"source",
|
||||||
|
"source_url",
|
||||||
|
"last_osint_update",
|
||||||
|
):
|
||||||
|
assert key in c, f"missing legacy field {key!r}"
|
||||||
|
# New fields.
|
||||||
|
for key in ("position_confidence", "position_source_at", "is_fallback"):
|
||||||
|
assert key in c, f"missing audit-required field {key!r}"
|
||||||
|
assert c["type"] == "carrier"
|
||||||
|
assert c["estimated"] is True
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
"""Issue #192 (tg12): CCTV proxy must re-validate the host on every redirect hop.
|
||||||
|
|
||||||
|
Before this fix, the proxy validated only the initial caller-supplied URL
|
||||||
|
host and then used ``requests.get(..., allow_redirects=True)``, which would
|
||||||
|
silently follow a 302 to an arbitrary internal address — an open-redirect-
|
||||||
|
to-SSRF chain.
|
||||||
|
|
||||||
|
These tests assert that:
|
||||||
|
|
||||||
|
1. A redirect to a disallowed host is rejected (502).
|
||||||
|
2. A redirect to an allowed host is followed (200).
|
||||||
|
3. The redirect chain length is bounded.
|
||||||
|
"""
|
||||||
|
import pytest
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
from fastapi import HTTPException
|
||||||
|
|
||||||
|
from routers.cctv import _fetch_cctv_upstream_response, _CCTV_MAX_REDIRECTS
|
||||||
|
|
||||||
|
|
||||||
|
class _Resp:
|
||||||
|
"""Minimal mock for requests.Response that mimics what _fetch needs."""
|
||||||
|
|
||||||
|
def __init__(self, status_code=200, headers=None, is_redirect=False):
|
||||||
|
self.status_code = status_code
|
||||||
|
self.headers = headers or {}
|
||||||
|
self.is_redirect = is_redirect
|
||||||
|
self.closed = False
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.closed = True
|
||||||
|
|
||||||
|
|
||||||
|
def _profile():
|
||||||
|
"""Build a tiny _CCTVProxyProfile-shaped mock the function expects."""
|
||||||
|
p = MagicMock()
|
||||||
|
p.name = "test"
|
||||||
|
p.timeout = 5
|
||||||
|
p.cache_seconds = 60
|
||||||
|
return p
|
||||||
|
|
||||||
|
|
||||||
|
def _request():
|
||||||
|
"""Build a tiny Request-shaped mock — only headers are read."""
|
||||||
|
req = MagicMock()
|
||||||
|
req.headers = {}
|
||||||
|
return req
|
||||||
|
|
||||||
|
|
||||||
|
@patch("routers.cctv._cctv_upstream_headers", return_value={})
|
||||||
|
@patch("routers.cctv._cctv_host_allowed", side_effect=lambda host: host == "allowed.example")
|
||||||
|
@patch("routers.cctv._req" if False else "requests.get") # patched below per-call
|
||||||
|
def test_redirect_to_disallowed_host_is_rejected(mock_get, mock_allow, mock_headers):
|
||||||
|
"""A 302 from allowed.example -> evil.example must be rejected with 502."""
|
||||||
|
# First call: 302 with Location: http://evil.example/path
|
||||||
|
mock_get.side_effect = [
|
||||||
|
_Resp(status_code=302, headers={"Location": "http://evil.example/path"}, is_redirect=True),
|
||||||
|
]
|
||||||
|
with pytest.raises(HTTPException) as exc_info:
|
||||||
|
_fetch_cctv_upstream_response(_request(), "http://allowed.example/cam", _profile())
|
||||||
|
assert exc_info.value.status_code == 502
|
||||||
|
assert "disallowed host" in str(exc_info.value.detail).lower()
|
||||||
|
|
||||||
|
|
||||||
|
@patch("routers.cctv._cctv_upstream_headers", return_value={})
|
||||||
|
@patch("routers.cctv._cctv_host_allowed", side_effect=lambda host: host == "allowed.example")
|
||||||
|
@patch("requests.get")
|
||||||
|
def test_redirect_to_localhost_is_rejected(mock_get, mock_allow, mock_headers):
|
||||||
|
"""A redirect to 127.0.0.1 (internal SSRF target) must be rejected."""
|
||||||
|
mock_get.side_effect = [
|
||||||
|
_Resp(status_code=302, headers={"Location": "http://127.0.0.1:8000/api/secret"}, is_redirect=True),
|
||||||
|
]
|
||||||
|
with pytest.raises(HTTPException) as exc_info:
|
||||||
|
_fetch_cctv_upstream_response(_request(), "http://allowed.example/cam", _profile())
|
||||||
|
assert exc_info.value.status_code == 502
|
||||||
|
|
||||||
|
|
||||||
|
@patch("routers.cctv._cctv_upstream_headers", return_value={})
|
||||||
|
@patch("routers.cctv._cctv_host_allowed", side_effect=lambda host: host in {"allowed.example", "other-allowed.example"})
|
||||||
|
@patch("requests.get")
|
||||||
|
def test_redirect_to_another_allowed_host_is_followed(mock_get, mock_allow, mock_headers):
|
||||||
|
"""A 302 from one allowed host to another allowed host should succeed."""
|
||||||
|
mock_get.side_effect = [
|
||||||
|
_Resp(status_code=302, headers={"Location": "http://other-allowed.example/cam"}, is_redirect=True),
|
||||||
|
_Resp(status_code=200, headers={"Content-Type": "image/jpeg"}),
|
||||||
|
]
|
||||||
|
resp = _fetch_cctv_upstream_response(_request(), "http://allowed.example/cam", _profile())
|
||||||
|
assert resp.status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
@patch("routers.cctv._cctv_upstream_headers", return_value={})
|
||||||
|
@patch("routers.cctv._cctv_host_allowed", return_value=True)
|
||||||
|
@patch("requests.get")
|
||||||
|
def test_redirect_chain_length_is_bounded(mock_get, mock_allow, mock_headers):
|
||||||
|
"""A pathological redirect loop must terminate within _CCTV_MAX_REDIRECTS."""
|
||||||
|
# Generate enough 302s to exceed the cap.
|
||||||
|
mock_get.side_effect = [
|
||||||
|
_Resp(status_code=302, headers={"Location": f"http://allowed.example/{i}"}, is_redirect=True)
|
||||||
|
for i in range(_CCTV_MAX_REDIRECTS + 2)
|
||||||
|
]
|
||||||
|
with pytest.raises(HTTPException) as exc_info:
|
||||||
|
_fetch_cctv_upstream_response(_request(), "http://allowed.example/cam", _profile())
|
||||||
|
assert exc_info.value.status_code == 502
|
||||||
|
assert "too long" in str(exc_info.value.detail).lower()
|
||||||
|
|
||||||
|
|
||||||
|
@patch("routers.cctv._cctv_upstream_headers", return_value={})
|
||||||
|
@patch("routers.cctv._cctv_host_allowed", return_value=True)
|
||||||
|
@patch("requests.get")
|
||||||
|
def test_redirect_to_non_http_scheme_is_rejected(mock_get, mock_allow, mock_headers):
|
||||||
|
"""A 302 to ``file://`` or ``ftp://`` must be rejected even if the host parses cleanly."""
|
||||||
|
mock_get.side_effect = [
|
||||||
|
_Resp(status_code=302, headers={"Location": "file:///etc/passwd"}, is_redirect=True),
|
||||||
|
]
|
||||||
|
with pytest.raises(HTTPException) as exc_info:
|
||||||
|
_fetch_cctv_upstream_response(_request(), "http://allowed.example/cam", _profile())
|
||||||
|
assert exc_info.value.status_code == 502
|
||||||
|
assert "non-http" in str(exc_info.value.detail).lower()
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
"""Regression coverage for operator-only control surfaces."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("method", "path", "payload"),
|
||||||
|
[
|
||||||
|
("get", "/api/wormhole/identity", None),
|
||||||
|
("post", "/api/wormhole/identity/bootstrap", {}),
|
||||||
|
("post", "/api/wormhole/gate/enter", {"gate_id": "general-talk"}),
|
||||||
|
("post", "/api/wormhole/gate/leave", {"gate_id": "general-talk"}),
|
||||||
|
("post", "/api/wormhole/sign", {"event_type": "gate_event", "payload": {"ok": True}}),
|
||||||
|
("post", "/api/wormhole/gate/key/rotate", {"gate_id": "general-talk", "reason": "test"}),
|
||||||
|
(
|
||||||
|
"post",
|
||||||
|
"/api/wormhole/gate/key/grant",
|
||||||
|
{
|
||||||
|
"gate_id": "general-talk",
|
||||||
|
"recipient_node_id": "node-test",
|
||||||
|
"recipient_dh_pub": "dh-test",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
("post", "/api/wormhole/gate/persona/create", {"gate_id": "general-talk", "label": "test"}),
|
||||||
|
(
|
||||||
|
"post",
|
||||||
|
"/api/wormhole/gate/persona/activate",
|
||||||
|
{"gate_id": "general-talk", "persona_id": "persona-test"},
|
||||||
|
),
|
||||||
|
("post", "/api/wormhole/gate/persona/clear", {"gate_id": "general-talk"}),
|
||||||
|
(
|
||||||
|
"post",
|
||||||
|
"/api/wormhole/gate/persona/retire",
|
||||||
|
{"gate_id": "general-talk", "persona_id": "persona-test"},
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"post",
|
||||||
|
"/api/wormhole/gate/message/sign-encrypted",
|
||||||
|
{
|
||||||
|
"gate_id": "general-talk",
|
||||||
|
"epoch": 1,
|
||||||
|
"ciphertext": "ciphertext",
|
||||||
|
"nonce": "nonce",
|
||||||
|
"format": "mls1",
|
||||||
|
"envelope_hash": "hash",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
("post", "/api/wormhole/gate/message/compose", {"gate_id": "general-talk", "plaintext": "hello"}),
|
||||||
|
("post", "/api/wormhole/sign-raw", {"message": "raw"}),
|
||||||
|
("post", "/api/wormhole/gate/state/export", {"gate_id": "general-talk"}),
|
||||||
|
("post", "/api/wormhole/gate/proof", {"gate_id": "general-talk"}),
|
||||||
|
("post", "/api/wormhole/connect", {}),
|
||||||
|
("post", "/api/layers", {"layers": {"viirs_nightlights": True}}),
|
||||||
|
("post", "/api/ais/feed", {"msgs": []}),
|
||||||
|
# Added in post-#227 gap audit:
|
||||||
|
# /api/wormhole/join also calls bootstrap_wormhole_identity() — same
|
||||||
|
# identity-takeover surface as /identity/bootstrap. PR #227 hardened
|
||||||
|
# the latter but missed the former.
|
||||||
|
("post", "/api/wormhole/join", {}),
|
||||||
|
# /api/sigint/transmit relays APRS-IS packets over radio using
|
||||||
|
# operator-supplied credentials. Any caller who reaches this endpoint
|
||||||
|
# could transmit on the operator's authority. Must be local-only.
|
||||||
|
(
|
||||||
|
"post",
|
||||||
|
"/api/sigint/transmit",
|
||||||
|
{
|
||||||
|
"callsign": "N0CALL",
|
||||||
|
"passcode": "12345",
|
||||||
|
"target": "NOCALL",
|
||||||
|
"message": "test",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
# Issue #198 (tg12, May 17): three gate introspection GETs leak the
|
||||||
|
# operator's active persona, persona inventory, and key status for
|
||||||
|
# any gate_id an anonymous caller knows. Defeats the unlinkability
|
||||||
|
# property documented in the privacy threat model.
|
||||||
|
("get", "/api/wormhole/gate/general-talk/identity", None),
|
||||||
|
("get", "/api/wormhole/gate/general-talk/personas", None),
|
||||||
|
("get", "/api/wormhole/gate/general-talk/key", None),
|
||||||
|
# Issue #211 (tg12): /api/thermal/verify fans out into an expensive
|
||||||
|
# STAC search + remote SWIR raster reads. Unauthenticated abuse
|
||||||
|
# could burn Sentinel-Hub quota and outbound bandwidth.
|
||||||
|
("get", "/api/thermal/verify?lat=0&lng=0&radius_km=10", None),
|
||||||
|
# Issue #213 (tg12): /api/radio/openmhz/calls/{sys_name} — rotating
|
||||||
|
# sys_name bypasses the 20s cache and hammers OpenMHZ. Risks an
|
||||||
|
# IP-ban for the project.
|
||||||
|
("get", "/api/radio/openmhz/calls/abc", None),
|
||||||
|
# Issue #214 (tg12): /api/radio/openmhz/audio — anonymous bandwidth
|
||||||
|
# relay through the backend. 60/minute rate limit is not enough on
|
||||||
|
# a streaming endpoint.
|
||||||
|
("get", "/api/radio/openmhz/audio?url=https%3A%2F%2Fmedia.openmhz.com%2Faudio%2Fabc.mp3", None),
|
||||||
|
# Issue #299 (tg12): /api/sentinel/token relays Copernicus CDSE
|
||||||
|
# OAuth token requests for caller-supplied client_id/secret.
|
||||||
|
# Anonymous access turns the backend into a free OAuth-mint relay.
|
||||||
|
(
|
||||||
|
"post",
|
||||||
|
"/api/sentinel/token",
|
||||||
|
None, # body sent via raw form-encoded data — None lets the
|
||||||
|
# remote_client wrapper send an empty body; the auth
|
||||||
|
# check fires before the form parser runs.
|
||||||
|
),
|
||||||
|
# Issue #300 (tg12): /api/sentinel/tile relays Sentinel Hub Process
|
||||||
|
# API tile fetches. Anonymous access is a bandwidth/quota relay
|
||||||
|
# for any caller's Copernicus account.
|
||||||
|
(
|
||||||
|
"post",
|
||||||
|
"/api/sentinel/tile",
|
||||||
|
{
|
||||||
|
"client_id": "ignored",
|
||||||
|
"client_secret": "ignored",
|
||||||
|
"preset": "TRUE-COLOR",
|
||||||
|
"date": "2026-01-01",
|
||||||
|
"z": 6, "x": 30, "y": 20,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
# Issue #301 (tg12): /api/sentinel2/search hits Planetary Computer
|
||||||
|
# STAC + Esri fallback. Anonymous access is a free external-search
|
||||||
|
# relay even though no caller credentials are involved.
|
||||||
|
("get", "/api/sentinel2/search?lat=0&lng=0", None),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_remote_control_surface_rejects_without_local_operator_or_admin(
|
||||||
|
remote_client, method, path, payload
|
||||||
|
):
|
||||||
|
request = getattr(remote_client, method)
|
||||||
|
response = request(path, json=payload) if payload is not None else request(path)
|
||||||
|
|
||||||
|
assert response.status_code == 403
|
||||||
|
|
||||||
|
|
||||||
|
def test_remote_agent_actions_poll_rejects_without_local_operator_or_admin(remote_client):
|
||||||
|
response = remote_client.get("/api/ai/agent-actions")
|
||||||
|
|
||||||
|
assert response.status_code == 403
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
"""CrowdThreat ingestion is operator opt-in only."""
|
||||||
|
|
||||||
|
|
||||||
|
class _CrowdThreatResponse:
|
||||||
|
status_code = 200
|
||||||
|
|
||||||
|
def json(self):
|
||||||
|
return {
|
||||||
|
"data": {
|
||||||
|
"threats": [
|
||||||
|
{
|
||||||
|
"id": "ct-1",
|
||||||
|
"title": "Example report",
|
||||||
|
"location": {
|
||||||
|
"lng_lat": [12.5, 41.9],
|
||||||
|
"name": "Example place",
|
||||||
|
"country": {"name": "Italy"},
|
||||||
|
},
|
||||||
|
"category": {"id": 1, "name": "Security"},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_crowdthreat_disabled_by_default_does_not_call_upstream(monkeypatch):
|
||||||
|
from services.fetchers import _store, crowdthreat
|
||||||
|
|
||||||
|
monkeypatch.delenv("CROWDTHREAT_ENABLED", raising=False)
|
||||||
|
monkeypatch.setitem(_store.latest_data, "crowdthreat", [{"id": "old"}])
|
||||||
|
monkeypatch.setattr(
|
||||||
|
crowdthreat,
|
||||||
|
"fetch_with_curl",
|
||||||
|
lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("upstream called")),
|
||||||
|
)
|
||||||
|
|
||||||
|
crowdthreat.fetch_crowdthreat()
|
||||||
|
|
||||||
|
assert _store.latest_data["crowdthreat"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_crowdthreat_opt_in_fetches_when_layer_is_enabled(monkeypatch):
|
||||||
|
from services.fetchers import _store, crowdthreat
|
||||||
|
|
||||||
|
monkeypatch.setenv("CROWDTHREAT_ENABLED", "true")
|
||||||
|
monkeypatch.setitem(_store.active_layers, "crowdthreat", True)
|
||||||
|
monkeypatch.setattr(crowdthreat, "fetch_with_curl", lambda *args, **kwargs: _CrowdThreatResponse())
|
||||||
|
|
||||||
|
crowdthreat.fetch_crowdthreat()
|
||||||
|
|
||||||
|
assert _store.latest_data["crowdthreat"][0]["id"] == "ct-1"
|
||||||
|
assert _store.latest_data["crowdthreat"][0]["source"] == "CrowdThreat"
|
||||||
@@ -0,0 +1,270 @@
|
|||||||
|
"""Per-(sender, recipient) anti-spam cap on the DM relay.
|
||||||
|
|
||||||
|
The user-stated rule: a single sender can have at most N UNACKED messages
|
||||||
|
parked in a single recipient's mailbox at any one time (N=2 by default).
|
||||||
|
Once the recipient pulls a message, the sender's quota for that pair
|
||||||
|
frees up.
|
||||||
|
|
||||||
|
Network rule, not local rule
|
||||||
|
-----------------------------
|
||||||
|
The cap is enforced TWICE:
|
||||||
|
|
||||||
|
1. ``DMRelay.deposit(...)`` -- local check on the sender's own node.
|
||||||
|
Refuses to spool the (N+1)th message before it can be replicated.
|
||||||
|
|
||||||
|
2. ``DMRelay.accept_replica(...)`` -- replication-acceptance check on
|
||||||
|
every receiving peer. Refuses to accept an inbound replica that
|
||||||
|
would put the local mailbox over the cap, even if the originating
|
||||||
|
peer claims it had cap room.
|
||||||
|
|
||||||
|
The double enforcement matters because cap (1) is client-side -- a
|
||||||
|
hostile relay could patch it out and continue to spool extras locally.
|
||||||
|
Cap (2) means those extras can't propagate: every honest peer rejects
|
||||||
|
them on the way in. A recipient who polls from honest peers therefore
|
||||||
|
never sees more than N pending from any one sender, regardless of how
|
||||||
|
many spam attempts the sender's own relay accepted.
|
||||||
|
|
||||||
|
These tests pin both halves of the rule.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def relay():
|
||||||
|
"""Fresh ``DMRelay`` per test."""
|
||||||
|
from services.mesh.mesh_dm_relay import DMRelay
|
||||||
|
r = DMRelay()
|
||||||
|
r._mailboxes.clear()
|
||||||
|
r._blocks.clear()
|
||||||
|
r._stats = {"messages_in_memory": 0}
|
||||||
|
return r
|
||||||
|
|
||||||
|
|
||||||
|
def _deposit(
|
||||||
|
relay,
|
||||||
|
*,
|
||||||
|
sender: str = "alice",
|
||||||
|
recipient_token: str = "bob_mailbox_token_abc",
|
||||||
|
ciphertext: str = "ciphertext-blob",
|
||||||
|
msg_id: str = "",
|
||||||
|
):
|
||||||
|
"""Convenience wrapper using ``shared`` delivery class."""
|
||||||
|
return relay.deposit(
|
||||||
|
sender_id=sender,
|
||||||
|
raw_sender_id=sender,
|
||||||
|
recipient_id="bob",
|
||||||
|
ciphertext=ciphertext,
|
||||||
|
msg_id=msg_id,
|
||||||
|
delivery_class="shared",
|
||||||
|
recipient_token=recipient_token,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Local cap on ``deposit``
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestDepositCap:
|
||||||
|
def test_two_deposits_from_same_sender_succeed(self, relay):
|
||||||
|
r1 = _deposit(relay)
|
||||||
|
r2 = _deposit(relay)
|
||||||
|
assert r1["ok"] is True
|
||||||
|
assert r2["ok"] is True
|
||||||
|
assert r1["msg_id"] != r2["msg_id"]
|
||||||
|
|
||||||
|
def test_third_deposit_from_same_sender_rejected(self, relay):
|
||||||
|
_deposit(relay)
|
||||||
|
_deposit(relay)
|
||||||
|
r3 = _deposit(relay)
|
||||||
|
assert r3["ok"] is False
|
||||||
|
detail = r3["detail"].lower()
|
||||||
|
assert "unread" in detail or "read your messages" in detail
|
||||||
|
|
||||||
|
def test_different_senders_have_independent_quotas(self, relay):
|
||||||
|
for _ in range(2):
|
||||||
|
assert _deposit(relay, sender="alice")["ok"] is True
|
||||||
|
for _ in range(2):
|
||||||
|
assert _deposit(relay, sender="carol")["ok"] is True
|
||||||
|
assert _deposit(relay, sender="carol")["ok"] is False
|
||||||
|
|
||||||
|
def test_different_recipients_have_independent_quotas(self, relay):
|
||||||
|
for _ in range(2):
|
||||||
|
assert _deposit(relay, sender="alice", recipient_token="bob_token")["ok"] is True
|
||||||
|
for _ in range(2):
|
||||||
|
assert _deposit(relay, sender="alice", recipient_token="dave_token")["ok"] is True
|
||||||
|
|
||||||
|
def test_ack_frees_quota(self, relay):
|
||||||
|
r1 = _deposit(relay)
|
||||||
|
_deposit(relay)
|
||||||
|
assert _deposit(relay)["ok"] is False
|
||||||
|
|
||||||
|
mailbox_key = relay._hashed_mailbox_token("bob_mailbox_token_abc")
|
||||||
|
relay._mailboxes[mailbox_key] = [
|
||||||
|
m for m in relay._mailboxes[mailbox_key]
|
||||||
|
if m.msg_id != r1["msg_id"]
|
||||||
|
]
|
||||||
|
relay._stats["messages_in_memory"] = sum(
|
||||||
|
len(v) for v in relay._mailboxes.values()
|
||||||
|
)
|
||||||
|
|
||||||
|
r3 = _deposit(relay)
|
||||||
|
assert r3["ok"] is True, f"expected quota free after ack, got: {r3}"
|
||||||
|
|
||||||
|
def test_cap_is_env_tunable(self, relay, monkeypatch):
|
||||||
|
import services.mesh.mesh_dm_relay as mdr
|
||||||
|
monkeypatch.setattr(
|
||||||
|
mdr.DMRelay,
|
||||||
|
"_per_sender_pending_limit",
|
||||||
|
lambda self: 1,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert _deposit(relay)["ok"] is True
|
||||||
|
assert _deposit(relay)["ok"] is False
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Replication-acceptance cap (the half that makes this a network rule)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestAcceptReplicaCap:
|
||||||
|
def _envelope(self, *, msg_id: str, sender_block_ref: str, mailbox_key: str):
|
||||||
|
return {
|
||||||
|
"msg_id": msg_id,
|
||||||
|
"mailbox_key": mailbox_key,
|
||||||
|
"sender_block_ref": sender_block_ref,
|
||||||
|
"sender_id": "alice",
|
||||||
|
"sender_seal": "",
|
||||||
|
"ciphertext": f"ciphertext-{msg_id}",
|
||||||
|
"timestamp": time.time(),
|
||||||
|
"delivery_class": "shared",
|
||||||
|
"relay_salt": "",
|
||||||
|
"payload_format": "dm1",
|
||||||
|
"session_welcome": "",
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_replica_accepted_under_cap(self, relay):
|
||||||
|
env = self._envelope(
|
||||||
|
msg_id="dm_replica_1",
|
||||||
|
sender_block_ref="alice_block_ref",
|
||||||
|
mailbox_key="mailbox_xyz",
|
||||||
|
)
|
||||||
|
result = relay.accept_replica(envelope=env)
|
||||||
|
assert result["ok"] is True
|
||||||
|
|
||||||
|
def test_replica_idempotent_on_duplicate_msg_id(self, relay):
|
||||||
|
mailbox_key = "mailbox_xyz"
|
||||||
|
env = self._envelope(
|
||||||
|
msg_id="dm_dup_1",
|
||||||
|
sender_block_ref="alice_block_ref",
|
||||||
|
mailbox_key=mailbox_key,
|
||||||
|
)
|
||||||
|
r1 = relay.accept_replica(envelope=env)
|
||||||
|
r2 = relay.accept_replica(envelope=env)
|
||||||
|
assert r1["ok"] is True
|
||||||
|
assert r2["ok"] is True
|
||||||
|
assert r2.get("duplicate") is True
|
||||||
|
assert len(relay._mailboxes[mailbox_key]) == 1
|
||||||
|
|
||||||
|
def test_replica_rejected_when_local_count_already_at_cap(self, relay):
|
||||||
|
mailbox_key = "mailbox_xyz"
|
||||||
|
for i in (1, 2):
|
||||||
|
relay.accept_replica(envelope=self._envelope(
|
||||||
|
msg_id=f"dm_seeded_{i}",
|
||||||
|
sender_block_ref="alice_block_ref",
|
||||||
|
mailbox_key=mailbox_key,
|
||||||
|
))
|
||||||
|
|
||||||
|
result = relay.accept_replica(envelope=self._envelope(
|
||||||
|
msg_id="dm_overcap_3",
|
||||||
|
sender_block_ref="alice_block_ref",
|
||||||
|
mailbox_key=mailbox_key,
|
||||||
|
))
|
||||||
|
assert result["ok"] is False
|
||||||
|
assert result.get("cap_violation") is True
|
||||||
|
assert result.get("pending") == 2
|
||||||
|
assert result.get("limit") == 2
|
||||||
|
assert len(relay._mailboxes[mailbox_key]) == 2
|
||||||
|
|
||||||
|
def test_replica_from_different_sender_passes_when_one_is_at_cap(self, relay):
|
||||||
|
mailbox_key = "mailbox_xyz"
|
||||||
|
for i in (1, 2):
|
||||||
|
relay.accept_replica(envelope=self._envelope(
|
||||||
|
msg_id=f"dm_alice_{i}",
|
||||||
|
sender_block_ref="alice_block_ref",
|
||||||
|
mailbox_key=mailbox_key,
|
||||||
|
))
|
||||||
|
assert relay.accept_replica(envelope=self._envelope(
|
||||||
|
msg_id="dm_alice_3",
|
||||||
|
sender_block_ref="alice_block_ref",
|
||||||
|
mailbox_key=mailbox_key,
|
||||||
|
))["ok"] is False
|
||||||
|
assert relay.accept_replica(envelope=self._envelope(
|
||||||
|
msg_id="dm_carol_1",
|
||||||
|
sender_block_ref="carol_block_ref",
|
||||||
|
mailbox_key=mailbox_key,
|
||||||
|
))["ok"] is True
|
||||||
|
|
||||||
|
def test_replica_rejects_malformed_envelopes(self, relay):
|
||||||
|
for bad in (
|
||||||
|
{},
|
||||||
|
{"msg_id": "x"},
|
||||||
|
{"msg_id": "x", "mailbox_key": "y"},
|
||||||
|
"not an object at all",
|
||||||
|
):
|
||||||
|
result = relay.accept_replica(envelope=bad)
|
||||||
|
assert result["ok"] is False
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# ``envelope_for_replication`` -- helper for the outbound replication path
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestEnvelopeForReplication:
|
||||||
|
def test_returns_envelope_for_stored_message(self, relay):
|
||||||
|
r = _deposit(relay, ciphertext="hello-ciphertext")
|
||||||
|
msg_id = r["msg_id"]
|
||||||
|
mailbox_key = relay._hashed_mailbox_token("bob_mailbox_token_abc")
|
||||||
|
|
||||||
|
env = relay.envelope_for_replication(mailbox_key=mailbox_key, msg_id=msg_id)
|
||||||
|
assert env is not None
|
||||||
|
assert env["msg_id"] == msg_id
|
||||||
|
assert env["mailbox_key"] == mailbox_key
|
||||||
|
assert env["ciphertext"] == "hello-ciphertext"
|
||||||
|
assert env["delivery_class"] == "shared"
|
||||||
|
for k in ("msg_id", "mailbox_key", "sender_block_ref", "ciphertext"):
|
||||||
|
assert env.get(k), f"envelope missing required field {k!r}"
|
||||||
|
|
||||||
|
def test_returns_none_for_unknown_message(self, relay):
|
||||||
|
env = relay.envelope_for_replication(
|
||||||
|
mailbox_key="never_existed", msg_id="never_existed",
|
||||||
|
)
|
||||||
|
assert env is None
|
||||||
|
|
||||||
|
def test_envelope_round_trips_through_accept_replica(self, relay):
|
||||||
|
from services.mesh.mesh_dm_relay import DMRelay
|
||||||
|
receiver_relay = DMRelay()
|
||||||
|
receiver_relay._mailboxes.clear()
|
||||||
|
receiver_relay._stats = {"messages_in_memory": 0}
|
||||||
|
|
||||||
|
r = _deposit(relay)
|
||||||
|
msg_id = r["msg_id"]
|
||||||
|
mailbox_key = relay._hashed_mailbox_token("bob_mailbox_token_abc")
|
||||||
|
env = relay.envelope_for_replication(
|
||||||
|
mailbox_key=mailbox_key, msg_id=msg_id,
|
||||||
|
)
|
||||||
|
assert env is not None
|
||||||
|
|
||||||
|
result = receiver_relay.accept_replica(envelope=env)
|
||||||
|
assert result["ok"] is True
|
||||||
|
stored = receiver_relay._mailboxes.get(mailbox_key, [])
|
||||||
|
assert len(stored) == 1
|
||||||
|
assert stored[0].msg_id == msg_id
|
||||||
|
assert stored[0].ciphertext == "ciphertext-blob"
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
"""POST /api/mesh/dm/replicate-envelope — receiving side of cross-node DM
|
||||||
|
mailbox replication.
|
||||||
|
|
||||||
|
This is the endpoint that peer relays call when they want to hand off an
|
||||||
|
encrypted DM envelope to us (so the recipient can log into our node and
|
||||||
|
find their messages). It re-enforces the per-(sender, recipient) anti-spam
|
||||||
|
cap so hostile sender relays can't widen the cap by skipping the local
|
||||||
|
check on their own deposit path.
|
||||||
|
|
||||||
|
The endpoint:
|
||||||
|
|
||||||
|
* authenticates the caller via the existing per-peer HMAC pattern
|
||||||
|
(same one /api/mesh/infonet/peer-push and /api/mesh/gate/peer-push
|
||||||
|
use, introduced in #256 — ``X-Peer-Url`` + ``X-Peer-HMAC`` headers
|
||||||
|
keyed off ``resolve_peer_key_for_url``)
|
||||||
|
* rejects bodies > 64 KB (DM envelope size is bounded by
|
||||||
|
``MESH_DM_MAX_MSG_BYTES`` — 64KB ceiling has generous headroom)
|
||||||
|
* rejects requests without a valid peer HMAC with 403
|
||||||
|
* passes the envelope to ``DMRelay.accept_replica`` which enforces
|
||||||
|
the cap
|
||||||
|
|
||||||
|
This file pins the endpoint contract. The cap enforcement itself is
|
||||||
|
tested in ``test_dm_relay_per_sender_cap.py`` against the relay's
|
||||||
|
``accept_replica`` method directly.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import json
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from httpx import ASGITransport, AsyncClient
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def remote_client():
|
||||||
|
"""ASGI client with peer IP 1.2.3.4 — never on the local-operator
|
||||||
|
allowlist. Used to prove the endpoint isn't accidentally reachable
|
||||||
|
by random remote callers without peer HMAC."""
|
||||||
|
from main import app
|
||||||
|
|
||||||
|
class _RemoteClient:
|
||||||
|
def __init__(self):
|
||||||
|
self._loop = asyncio.new_event_loop()
|
||||||
|
self._transport = ASGITransport(app=app, client=("1.2.3.4", 12345))
|
||||||
|
self._base = "http://1.2.3.4:8000"
|
||||||
|
|
||||||
|
def post(self, url, **kw):
|
||||||
|
async def go():
|
||||||
|
async with AsyncClient(transport=self._transport, base_url=self._base) as ac:
|
||||||
|
return await ac.post(url, **kw)
|
||||||
|
return self._loop.run_until_complete(go())
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self._loop.close()
|
||||||
|
|
||||||
|
c = _RemoteClient()
|
||||||
|
yield c
|
||||||
|
c.close()
|
||||||
|
|
||||||
|
|
||||||
|
class TestReplicateEndpointAuth:
|
||||||
|
def test_rejects_request_without_peer_hmac(self, remote_client):
|
||||||
|
"""A peer push that does NOT carry X-Peer-Url + X-Peer-HMAC
|
||||||
|
must be rejected with 403 before the envelope is ever passed
|
||||||
|
to the relay. Same gate the existing infonet/gate peer-push
|
||||||
|
endpoints enforce."""
|
||||||
|
payload = {
|
||||||
|
"envelope": {
|
||||||
|
"msg_id": "dm_unauth_1",
|
||||||
|
"mailbox_key": "mb",
|
||||||
|
"sender_block_ref": "sender",
|
||||||
|
"ciphertext": "x",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
r = remote_client.post(
|
||||||
|
"/api/mesh/dm/replicate-envelope",
|
||||||
|
json=payload,
|
||||||
|
)
|
||||||
|
assert r.status_code == 403
|
||||||
|
assert "peer HMAC" in r.text or "peer hmac" in r.text.lower()
|
||||||
|
|
||||||
|
def test_rejects_wrong_peer_hmac(self, remote_client, monkeypatch):
|
||||||
|
"""A request with a peer HMAC header keyed off the WRONG secret
|
||||||
|
is rejected. Confirms the HMAC is actually verified — a tampered
|
||||||
|
body or a key-substitution attack doesn't sneak through."""
|
||||||
|
# Plant a known peer secret. The request will sign with a
|
||||||
|
# DIFFERENT key, so verification must fail.
|
||||||
|
from services.config import get_settings
|
||||||
|
monkeypatch.setenv("MESH_PEER_PUSH_SECRET", "real-secret-32-chars-min-padding-padding")
|
||||||
|
get_settings.cache_clear()
|
||||||
|
|
||||||
|
body = json.dumps({
|
||||||
|
"envelope": {
|
||||||
|
"msg_id": "dm_wronghmac",
|
||||||
|
"mailbox_key": "mb",
|
||||||
|
"sender_block_ref": "sender",
|
||||||
|
"ciphertext": "x",
|
||||||
|
},
|
||||||
|
}).encode("utf-8")
|
||||||
|
wrong_hmac = hmac.new(b"wrong-key", body, hashlib.sha256).hexdigest()
|
||||||
|
r = remote_client.post(
|
||||||
|
"/api/mesh/dm/replicate-envelope",
|
||||||
|
content=body,
|
||||||
|
headers={
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"X-Peer-Url": "http://example-peer.onion:8000",
|
||||||
|
"X-Peer-HMAC": wrong_hmac,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert r.status_code == 403
|
||||||
|
|
||||||
|
def test_rejects_oversize_body(self, remote_client):
|
||||||
|
"""64 KB ceiling — anything bigger doesn't even get parsed.
|
||||||
|
Defends against memory amplification via giant ciphertexts."""
|
||||||
|
# 100 KB body is well over the 64 KB cap.
|
||||||
|
big = b"{" + b"x" * 100_000 + b"}"
|
||||||
|
r = remote_client.post(
|
||||||
|
"/api/mesh/dm/replicate-envelope",
|
||||||
|
content=big,
|
||||||
|
headers={
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Content-Length": str(len(big)),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert r.status_code in (400, 413), (
|
||||||
|
f"oversize body should be rejected with 400/413, got {r.status_code}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestReplicateEndpointRegistered:
|
||||||
|
def test_route_present_in_app(self):
|
||||||
|
"""Static check that the route is actually wired into the app.
|
||||||
|
Catches a future refactor that drops the router include or
|
||||||
|
deletes the endpoint by accident."""
|
||||||
|
from main import app
|
||||||
|
|
||||||
|
paths_methods = set()
|
||||||
|
for route in app.routes:
|
||||||
|
path = getattr(route, "path", None)
|
||||||
|
methods = getattr(route, "methods", set()) or set()
|
||||||
|
for m in methods:
|
||||||
|
paths_methods.add((m, path))
|
||||||
|
|
||||||
|
assert ("POST", "/api/mesh/dm/replicate-envelope") in paths_methods, (
|
||||||
|
"POST /api/mesh/dm/replicate-envelope is not registered on the app"
|
||||||
|
)
|
||||||
@@ -0,0 +1,196 @@
|
|||||||
|
"""Issue #250 (tg12): Docker bridge local-operator trust must be bound to
|
||||||
|
the frontend container's hostname, not the entire 172.16.0.0/12 range.
|
||||||
|
|
||||||
|
Previous behavior trusted ANY private-RFC1918 source IP on the bridge
|
||||||
|
when ``SHADOWBROKER_TRUST_DOCKER_BRIDGE_LOCAL_OPERATOR=1``. On a shared
|
||||||
|
Docker host this granted local-operator privileges to any other
|
||||||
|
container that could route to the backend's bridge — far broader than
|
||||||
|
intended.
|
||||||
|
|
||||||
|
The fix narrows trust to source IPs that forward-resolve from one of the
|
||||||
|
configured frontend container hostnames (default: the compose service
|
||||||
|
name ``frontend`` plus the explicit ``container_name``
|
||||||
|
``shadowbroker-frontend``). Operators with renamed containers can list
|
||||||
|
the new names in ``SHADOWBROKER_TRUSTED_FRONTEND_HOSTS``.
|
||||||
|
|
||||||
|
These tests exercise the resolution helpers directly so that we don't
|
||||||
|
need a live Docker daemon to validate the contract.
|
||||||
|
"""
|
||||||
|
import socket
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# _trusted_bridge_frontend_hostnames — env parsing
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestTrustedHostnameParsing:
|
||||||
|
def _fn(self):
|
||||||
|
from auth import _trusted_bridge_frontend_hostnames
|
||||||
|
return _trusted_bridge_frontend_hostnames
|
||||||
|
|
||||||
|
def test_default_covers_compose_service_and_container_name(self):
|
||||||
|
with patch.dict("os.environ", {}, clear=False):
|
||||||
|
# Make sure the env var is not set so we exercise the default.
|
||||||
|
import os
|
||||||
|
os.environ.pop("SHADOWBROKER_TRUSTED_FRONTEND_HOSTS", None)
|
||||||
|
assert self._fn()() == ["frontend", "shadowbroker-frontend"]
|
||||||
|
|
||||||
|
def test_custom_list_via_env(self):
|
||||||
|
with patch.dict(
|
||||||
|
"os.environ",
|
||||||
|
{"SHADOWBROKER_TRUSTED_FRONTEND_HOSTS": "my-ui,alt-frontend"},
|
||||||
|
):
|
||||||
|
assert self._fn()() == ["my-ui", "alt-frontend"]
|
||||||
|
|
||||||
|
def test_whitespace_trimmed(self):
|
||||||
|
with patch.dict(
|
||||||
|
"os.environ",
|
||||||
|
{"SHADOWBROKER_TRUSTED_FRONTEND_HOSTS": " my-ui , alt-frontend "},
|
||||||
|
):
|
||||||
|
assert self._fn()() == ["my-ui", "alt-frontend"]
|
||||||
|
|
||||||
|
def test_empty_env_falls_back_to_default(self):
|
||||||
|
# An empty string still falls back to the bundled defaults so a
|
||||||
|
# misconfigured env var doesn't silently dismantle bridge trust.
|
||||||
|
with patch.dict(
|
||||||
|
"os.environ",
|
||||||
|
{"SHADOWBROKER_TRUSTED_FRONTEND_HOSTS": ""},
|
||||||
|
):
|
||||||
|
# Per docs: empty string sets the env var to "" so os.environ.get
|
||||||
|
# returns "" — that string is parsed and yields []. We assert
|
||||||
|
# that empty parse yields [] (caller fail-closes from there).
|
||||||
|
assert self._fn()() == []
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# _resolve_trusted_bridge_ips — DNS resolution with cache + fail-closed
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestResolveTrustedBridgeIps:
|
||||||
|
def setup_method(self):
|
||||||
|
# Reset the module-level cache before each test so prior tests
|
||||||
|
# don't bleed state across cases.
|
||||||
|
from auth import _DOCKER_BRIDGE_TRUST_CACHE
|
||||||
|
_DOCKER_BRIDGE_TRUST_CACHE["ips"] = frozenset()
|
||||||
|
_DOCKER_BRIDGE_TRUST_CACHE["expires"] = 0.0
|
||||||
|
|
||||||
|
def test_resolves_configured_hostnames(self):
|
||||||
|
from auth import _resolve_trusted_bridge_ips
|
||||||
|
|
||||||
|
def fake_gethostbyname_ex(host):
|
||||||
|
mapping = {
|
||||||
|
"frontend": ("frontend", [], ["172.18.0.3"]),
|
||||||
|
"shadowbroker-frontend": ("shadowbroker-frontend", [], ["172.18.0.3", "172.18.0.4"]),
|
||||||
|
}
|
||||||
|
if host not in mapping:
|
||||||
|
raise socket.gaierror("no such host")
|
||||||
|
return mapping[host]
|
||||||
|
|
||||||
|
with patch("socket.gethostbyname_ex", side_effect=fake_gethostbyname_ex):
|
||||||
|
ips = _resolve_trusted_bridge_ips()
|
||||||
|
assert ips == frozenset({"172.18.0.3", "172.18.0.4"})
|
||||||
|
|
||||||
|
def test_fail_closed_when_dns_returns_nothing(self):
|
||||||
|
from auth import _resolve_trusted_bridge_ips
|
||||||
|
|
||||||
|
def always_fail(host):
|
||||||
|
raise socket.gaierror("no resolver")
|
||||||
|
|
||||||
|
with patch("socket.gethostbyname_ex", side_effect=always_fail):
|
||||||
|
ips = _resolve_trusted_bridge_ips()
|
||||||
|
assert ips == frozenset()
|
||||||
|
|
||||||
|
def test_partial_resolution_is_kept(self):
|
||||||
|
"""If one hostname resolves and another fails, we keep the
|
||||||
|
successful one rather than discarding the whole set."""
|
||||||
|
from auth import _resolve_trusted_bridge_ips
|
||||||
|
|
||||||
|
def partial(host):
|
||||||
|
if host == "frontend":
|
||||||
|
return ("frontend", [], ["172.18.0.3"])
|
||||||
|
raise socket.gaierror("missing")
|
||||||
|
|
||||||
|
with patch("socket.gethostbyname_ex", side_effect=partial):
|
||||||
|
ips = _resolve_trusted_bridge_ips()
|
||||||
|
assert ips == frozenset({"172.18.0.3"})
|
||||||
|
|
||||||
|
def test_cache_short_circuits_repeated_dns_calls(self):
|
||||||
|
from auth import _resolve_trusted_bridge_ips
|
||||||
|
|
||||||
|
call_count = {"n": 0}
|
||||||
|
|
||||||
|
def counting(host):
|
||||||
|
call_count["n"] += 1
|
||||||
|
return ("frontend", [], ["172.18.0.3"])
|
||||||
|
|
||||||
|
with patch("socket.gethostbyname_ex", side_effect=counting):
|
||||||
|
_resolve_trusted_bridge_ips()
|
||||||
|
calls_after_first = call_count["n"]
|
||||||
|
_resolve_trusted_bridge_ips()
|
||||||
|
_resolve_trusted_bridge_ips()
|
||||||
|
# Second + third calls hit the cache, not the DNS stub.
|
||||||
|
assert call_count["n"] == calls_after_first
|
||||||
|
|
||||||
|
def test_cache_expires(self):
|
||||||
|
from auth import _resolve_trusted_bridge_ips, _DOCKER_BRIDGE_TRUST_CACHE
|
||||||
|
|
||||||
|
with patch("socket.gethostbyname_ex", return_value=("frontend", [], ["172.18.0.3"])):
|
||||||
|
_resolve_trusted_bridge_ips()
|
||||||
|
# Force expiry.
|
||||||
|
_DOCKER_BRIDGE_TRUST_CACHE["expires"] = 0.0
|
||||||
|
with patch("socket.gethostbyname_ex", return_value=("frontend", [], ["172.18.0.9"])) as stub:
|
||||||
|
ips = _resolve_trusted_bridge_ips()
|
||||||
|
assert stub.called
|
||||||
|
assert "172.18.0.9" in ips
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# _is_docker_bridge_host — composite of the helpers above
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestIsDockerBridgeHost:
|
||||||
|
def setup_method(self):
|
||||||
|
from auth import _DOCKER_BRIDGE_TRUST_CACHE
|
||||||
|
_DOCKER_BRIDGE_TRUST_CACHE["ips"] = frozenset()
|
||||||
|
_DOCKER_BRIDGE_TRUST_CACHE["expires"] = 0.0
|
||||||
|
|
||||||
|
def test_trusts_resolved_frontend_ip(self):
|
||||||
|
from auth import _is_docker_bridge_host
|
||||||
|
|
||||||
|
with patch("auth._resolve_trusted_bridge_ips", return_value=frozenset({"172.18.0.3"})):
|
||||||
|
assert _is_docker_bridge_host("172.18.0.3") is True
|
||||||
|
|
||||||
|
def test_rejects_arbitrary_bridge_ip(self):
|
||||||
|
"""A rogue container on the same bridge but at a different IP
|
||||||
|
must NOT be trusted, even though it falls in 172.16.0.0/12."""
|
||||||
|
from auth import _is_docker_bridge_host
|
||||||
|
|
||||||
|
with patch("auth._resolve_trusted_bridge_ips", return_value=frozenset({"172.18.0.3"})):
|
||||||
|
assert _is_docker_bridge_host("172.18.0.99") is False
|
||||||
|
|
||||||
|
def test_rejects_public_ip_without_dns_work(self):
|
||||||
|
"""Public IPs skip DNS resolution entirely (perf + safety)."""
|
||||||
|
from auth import _is_docker_bridge_host
|
||||||
|
|
||||||
|
with patch("auth._resolve_trusted_bridge_ips") as stub:
|
||||||
|
assert _is_docker_bridge_host("8.8.8.8") is False
|
||||||
|
stub.assert_not_called()
|
||||||
|
|
||||||
|
def test_rejects_non_ip_input(self):
|
||||||
|
from auth import _is_docker_bridge_host
|
||||||
|
|
||||||
|
assert _is_docker_bridge_host("") is False
|
||||||
|
assert _is_docker_bridge_host("not-an-ip") is False
|
||||||
|
assert _is_docker_bridge_host("frontend") is False
|
||||||
|
|
||||||
|
def test_fails_closed_when_dns_returns_empty(self):
|
||||||
|
"""If Docker DNS can't resolve any frontend hostname, the bridge
|
||||||
|
is not trusted — even for IPs that would have been trusted under
|
||||||
|
the old 172.16.0.0/12 blanket policy."""
|
||||||
|
from auth import _is_docker_bridge_host
|
||||||
|
|
||||||
|
with patch("auth._resolve_trusted_bridge_ips", return_value=frozenset()):
|
||||||
|
assert _is_docker_bridge_host("172.18.0.3") is False
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
"""GDELT's ``data.gdeltproject.org`` is a CNAME to a Google Cloud Storage
|
||||||
|
bucket. GCS responds with the wildcard ``*.storage.googleapis.com``
|
||||||
|
certificate, which legitimately does NOT cover the GDELT custom
|
||||||
|
domain, so Python's TLS verification refuses the connection. Some
|
||||||
|
networks happen to route through a path where this works; many
|
||||||
|
(notably Docker Desktop's outbound NAT on local installs) do not.
|
||||||
|
|
||||||
|
The fix in ``services.geopolitics._gcs_direct_gdelt_url`` rewrites any
|
||||||
|
URL pointing at ``data.gdeltproject.org`` to its GCS-direct equivalent
|
||||||
|
(``storage.googleapis.com/data.gdeltproject.org/...``), where the
|
||||||
|
standard GCS certificate is genuinely valid. ``api.gdeltproject.org``
|
||||||
|
and every other host are left untouched.
|
||||||
|
|
||||||
|
These tests pin that behavior so a future refactor that drops the
|
||||||
|
helper or accidentally rewrites the wrong host gets a loud failure.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
def test_rewrites_data_gdeltproject_https():
|
||||||
|
from services.geopolitics import _gcs_direct_gdelt_url
|
||||||
|
|
||||||
|
assert _gcs_direct_gdelt_url(
|
||||||
|
"https://data.gdeltproject.org/gdeltv2/lastupdate.txt"
|
||||||
|
) == "https://storage.googleapis.com/data.gdeltproject.org/gdeltv2/lastupdate.txt"
|
||||||
|
|
||||||
|
|
||||||
|
def test_rewrites_data_gdeltproject_http():
|
||||||
|
"""GDELT's lastupdate.txt sometimes lists URLs with http:// — we
|
||||||
|
rewrite those too (the downstream call upgrades them to https)."""
|
||||||
|
from services.geopolitics import _gcs_direct_gdelt_url
|
||||||
|
|
||||||
|
assert _gcs_direct_gdelt_url(
|
||||||
|
"http://data.gdeltproject.org/gdeltv2/20260301120000.export.CSV.zip"
|
||||||
|
) == "http://storage.googleapis.com/data.gdeltproject.org/gdeltv2/20260301120000.export.CSV.zip"
|
||||||
|
|
||||||
|
|
||||||
|
def test_rewrites_preserve_query_string_and_path():
|
||||||
|
from services.geopolitics import _gcs_direct_gdelt_url
|
||||||
|
|
||||||
|
url = "https://data.gdeltproject.org/some/deep/path?a=1&b=2&c=hello%20world"
|
||||||
|
rewritten = _gcs_direct_gdelt_url(url)
|
||||||
|
assert rewritten == (
|
||||||
|
"https://storage.googleapis.com/data.gdeltproject.org"
|
||||||
|
"/some/deep/path?a=1&b=2&c=hello%20world"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_does_not_touch_api_gdeltproject_org():
|
||||||
|
"""The API host is NOT a CNAME to GCS; rewriting it would break the
|
||||||
|
actual GDELT API endpoint."""
|
||||||
|
from services.geopolitics import _gcs_direct_gdelt_url
|
||||||
|
|
||||||
|
url = "https://api.gdeltproject.org/api/v2/doc/doc?query=carrier"
|
||||||
|
assert _gcs_direct_gdelt_url(url) == url
|
||||||
|
|
||||||
|
|
||||||
|
def test_does_not_touch_other_hosts():
|
||||||
|
from services.geopolitics import _gcs_direct_gdelt_url
|
||||||
|
|
||||||
|
for url in (
|
||||||
|
"https://en.wikipedia.org/wiki/Boeing_747",
|
||||||
|
"https://query.wikidata.org/sparql",
|
||||||
|
"https://storage.googleapis.com/already-correct/path",
|
||||||
|
"https://nominatim.openstreetmap.org/search",
|
||||||
|
):
|
||||||
|
assert _gcs_direct_gdelt_url(url) == url
|
||||||
|
|
||||||
|
|
||||||
|
def test_does_not_partially_match_strings():
|
||||||
|
"""``data.gdeltproject.org`` is matched exactly; URLs that merely
|
||||||
|
contain that substring elsewhere (in a query parameter, for example)
|
||||||
|
are left alone. Otherwise we'd rewrite something like
|
||||||
|
``https://example.com/?ref=data.gdeltproject.org/x`` which is wrong."""
|
||||||
|
from services.geopolitics import _gcs_direct_gdelt_url
|
||||||
|
|
||||||
|
# The match requires ``://`` immediately before the host, so a host
|
||||||
|
# like ``example-data.gdeltproject.org`` would also be left alone
|
||||||
|
# (treated as a different host, which is correct).
|
||||||
|
url = "https://example-data.gdeltproject.org/path"
|
||||||
|
assert _gcs_direct_gdelt_url(url) == url
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
"""Issue #199 (tg12): GDELT military incident ingestion must use HTTPS.
|
||||||
|
|
||||||
|
The previous code fetched ``http://data.gdeltproject.org/gdeltv2/lastupdate.txt``
|
||||||
|
and ~48 export archives over plaintext HTTP, which let a passive observer
|
||||||
|
identify Shadowbroker nodes by their fetch pattern and let an active MITM
|
||||||
|
inject doctored export records into the global incident map.
|
||||||
|
|
||||||
|
These tests assert the URL constants and outbound URL constructor in
|
||||||
|
``services/geopolitics.py`` only use HTTPS.
|
||||||
|
"""
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
_GEOPOLITICS_SRC = Path(__file__).resolve().parent.parent / "services" / "geopolitics.py"
|
||||||
|
|
||||||
|
|
||||||
|
def _read_source() -> str:
|
||||||
|
return _GEOPOLITICS_SRC.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_geopolitics_does_not_use_plaintext_http_for_gdelt():
|
||||||
|
"""No string literal in geopolitics.py should fetch GDELT over plaintext HTTP."""
|
||||||
|
src = _read_source()
|
||||||
|
# Strings that would issue an HTTP request — comments are excluded because
|
||||||
|
# comments include "http://" in example URLs even after the fix.
|
||||||
|
code_lines = [
|
||||||
|
ln for ln in src.split("\n")
|
||||||
|
if "http://data.gdeltproject.org" in ln and not ln.lstrip().startswith("#")
|
||||||
|
]
|
||||||
|
assert code_lines == [], (
|
||||||
|
"Found plaintext http://data.gdeltproject.org usage in geopolitics.py:\n"
|
||||||
|
+ "\n".join(code_lines)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_geopolitics_uses_https_for_gdelt():
|
||||||
|
"""The HTTPS URLs we expect must be present."""
|
||||||
|
src = _read_source()
|
||||||
|
assert "https://data.gdeltproject.org/gdeltv2/lastupdate.txt" in src
|
||||||
|
# The download URL is constructed via f-string with {fname}
|
||||||
|
assert re.search(
|
||||||
|
r'https://data\.gdeltproject\.org/gdeltv2/\{fname\}', src
|
||||||
|
), "expected https URL template for individual GDELT export downloads"
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
"""Issue #207 (tg12): /api/mesh/infonet/status accepted
|
||||||
|
?verify_signatures=true from anonymous callers, triggering O(n_events)
|
||||||
|
signature verification across the entire chain. Trivial DoS.
|
||||||
|
|
||||||
|
The fix silently downgrades the parameter to False for unauthenticated
|
||||||
|
callers — no error surfaced, response structure unchanged, the
|
||||||
|
expensive path runs only when the caller has authenticated.
|
||||||
|
|
||||||
|
These tests focus on the source-level contract because a full
|
||||||
|
FastAPI test client doesn't have an easy hook into the ``_scoped_view_authenticated``
|
||||||
|
helper. They lock in the key invariant: the ``effective_verify_signatures``
|
||||||
|
value seen by ``validate_chain()`` is the AND of the request param and
|
||||||
|
the auth check.
|
||||||
|
"""
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
_ROUTER_PATH = Path(__file__).resolve().parent.parent / "routers" / "mesh_public.py"
|
||||||
|
|
||||||
|
|
||||||
|
def _read_router_source() -> str:
|
||||||
|
return _ROUTER_PATH.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_infonet_status_gates_verify_signatures():
|
||||||
|
"""The infonet_status route must AND verify_signatures with auth."""
|
||||||
|
src = _read_router_source()
|
||||||
|
# The fix introduces an `effective_verify_signatures` variable.
|
||||||
|
assert "effective_verify_signatures" in src
|
||||||
|
|
||||||
|
# It must be computed as the AND of the request param and the
|
||||||
|
# authenticated check.
|
||||||
|
assert "bool(verify_signatures) and authenticated" in src
|
||||||
|
|
||||||
|
# validate_chain() must be called with the effective value, NOT the
|
||||||
|
# raw request param.
|
||||||
|
assert "validate_chain(verify_signatures=effective_verify_signatures)" in src
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_http_error_path_for_anonymous_callers():
|
||||||
|
"""No HTTPException is raised for unauthenticated verify_signatures=true.
|
||||||
|
|
||||||
|
The endpoint should silently downgrade — not return 403 — so existing
|
||||||
|
frontends that happen to pass the param see no behavior change.
|
||||||
|
"""
|
||||||
|
src = _read_router_source()
|
||||||
|
# Within the infonet_status function body, there should be no
|
||||||
|
# HTTPException(403) raised because of the verify_signatures param.
|
||||||
|
# Find the function definition and inspect the body.
|
||||||
|
import re
|
||||||
|
m = re.search(
|
||||||
|
r"async def infonet_status\(.*?\):(.+?)(?=\n@router|\nasync def |\ndef |\Z)",
|
||||||
|
src,
|
||||||
|
re.DOTALL,
|
||||||
|
)
|
||||||
|
assert m, "infonet_status function not found in source"
|
||||||
|
body = m.group(1)
|
||||||
|
# No explicit 403 around the verify_signatures handling.
|
||||||
|
assert "HTTPException(status_code=403" not in body
|
||||||
|
assert "raise HTTPException(403" not in body
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
"""Issue #206 (tg12): KiwiSDR upstream is HTTP-only and cannot be upgraded
|
||||||
|
to TLS. We defend with content validation + a bundled static directory
|
||||||
|
so the layer always renders something useful and a MITM injecting
|
||||||
|
garbage can't corrupt the map.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from services import kiwisdr_fetcher
|
||||||
|
from services.kiwisdr_fetcher import (
|
||||||
|
_MIN_HEALTHY_RECEIVER_COUNT,
|
||||||
|
_load_bundled_fallback,
|
||||||
|
_validate_fetched_nodes,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_bundled_fallback_file_exists_and_is_nonempty():
|
||||||
|
"""The codebase ships a static snapshot for last-resort use."""
|
||||||
|
bundle = _load_bundled_fallback()
|
||||||
|
assert isinstance(bundle, list)
|
||||||
|
assert len(bundle) >= _MIN_HEALTHY_RECEIVER_COUNT
|
||||||
|
|
||||||
|
|
||||||
|
def test_validation_rejects_too_few_entries():
|
||||||
|
too_short = [{"name": "x", "lat": 0.0, "lon": 0.0, "url": ""}] * (_MIN_HEALTHY_RECEIVER_COUNT - 1)
|
||||||
|
assert _validate_fetched_nodes(too_short) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_validation_accepts_healthy_response():
|
||||||
|
healthy = [
|
||||||
|
{"name": f"Receiver {i}", "lat": 50.0, "lon": -1.0, "url": "http://example"}
|
||||||
|
for i in range(_MIN_HEALTHY_RECEIVER_COUNT)
|
||||||
|
]
|
||||||
|
assert _validate_fetched_nodes(healthy) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_validation_rejects_non_list():
|
||||||
|
assert _validate_fetched_nodes(None) is False # type: ignore[arg-type]
|
||||||
|
assert _validate_fetched_nodes("a string") is False # type: ignore[arg-type]
|
||||||
|
assert _validate_fetched_nodes({}) is False # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_validation_rejects_too_many_malformed_entries():
|
||||||
|
"""If more than 5% of entries lack a name or numeric lat, reject."""
|
||||||
|
nodes = []
|
||||||
|
# 100 entries, 20 of them malformed — well over the 5% threshold.
|
||||||
|
for i in range(_MIN_HEALTHY_RECEIVER_COUNT + 50):
|
||||||
|
if i % 5 == 0:
|
||||||
|
nodes.append({}) # missing name + lat
|
||||||
|
else:
|
||||||
|
nodes.append({"name": f"R{i}", "lat": 50.0, "lon": -1.0, "url": ""})
|
||||||
|
assert _validate_fetched_nodes(nodes) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_fallback_used_when_validation_fails(monkeypatch, tmp_path):
|
||||||
|
"""If a fetch returns garbage, the fallback chain reaches the bundle."""
|
||||||
|
# Force disk cache miss
|
||||||
|
fake_cache = tmp_path / "kiwisdr_cache.json"
|
||||||
|
monkeypatch.setattr(kiwisdr_fetcher, "_CACHE_FILE", fake_cache)
|
||||||
|
|
||||||
|
# Make fetch_with_curl return a parseable but UNHEALTHY response
|
||||||
|
# (only 3 entries — well below the validation threshold).
|
||||||
|
class _GarbageResp:
|
||||||
|
status_code = 200
|
||||||
|
text = "var kiwisdr_com = [{\"name\":\"x\",\"gps\":\"(0,0)\"}];"
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"services.network_utils.fetch_with_curl", lambda *a, **kw: _GarbageResp()
|
||||||
|
)
|
||||||
|
|
||||||
|
# Bypass the @cached decorator
|
||||||
|
kiwisdr_fetcher.kiwisdr_cache.clear()
|
||||||
|
|
||||||
|
result = kiwisdr_fetcher.fetch_kiwisdr_nodes()
|
||||||
|
# Should be the bundled fallback (798 entries), not the garbage (1 entry)
|
||||||
|
assert isinstance(result, list)
|
||||||
|
assert len(result) >= _MIN_HEALTHY_RECEIVER_COUNT
|
||||||
@@ -0,0 +1,273 @@
|
|||||||
|
"""Tests for issue #288: viewport bbox filtering on /api/live-data/{fast,slow}.
|
||||||
|
|
||||||
|
Behaviour contract:
|
||||||
|
* Without s/w/n/e params, the response is byte-for-byte identical to the
|
||||||
|
pre-#288 implementation. (No filtering, no extra fields, no ETag change.)
|
||||||
|
* With s/w/n/e supplied, heavy/dense layers are filtered to that viewport
|
||||||
|
with a 20% padding box.
|
||||||
|
* Light reference layers (datacenters, military_bases, power_plants,
|
||||||
|
satellites, news, weather, …) are NEVER filtered, even when bounds are
|
||||||
|
supplied — panning must never reveal an "empty world" of infrastructure.
|
||||||
|
* World-scale bounds (lng_span >= 300 OR lat_span >= 120) short-circuit
|
||||||
|
filtering and share the global ETag.
|
||||||
|
* The ETag includes a 1°-quantized bbox so two viewports never poison each
|
||||||
|
other's 304 cache.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
# ───────────────────────── /api/live-data/fast ─────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestFastBboxFiltering:
|
||||||
|
def _seed_fast(self, monkeypatch):
|
||||||
|
"""Plant deterministic heavy + light fixtures across the globe."""
|
||||||
|
from services.fetchers import _store
|
||||||
|
|
||||||
|
# Heavy collections: dense across the world.
|
||||||
|
commercial = [
|
||||||
|
{"lat": -60.0, "lng": -120.0, "id": "f-sw"}, # south Pacific
|
||||||
|
{"lat": 35.0, "lng": -75.0, "id": "f-ne"}, # eastern US
|
||||||
|
{"lat": 35.0, "lng": 100.0, "id": "f-asia"}, # Asia
|
||||||
|
]
|
||||||
|
ships = [
|
||||||
|
{"lat": -60.0, "lng": -120.0, "id": "s-sw"},
|
||||||
|
{"lat": 35.0, "lng": -75.0, "id": "s-ne"},
|
||||||
|
]
|
||||||
|
cctv = [{"lat": 35.0, "lng": -75.0, "id": "c-1"}]
|
||||||
|
|
||||||
|
# Sigint heavy collection.
|
||||||
|
sigint = [
|
||||||
|
{"source": "meshtastic", "lat": 35.0, "lng": -75.0, "id": "sig-east"},
|
||||||
|
{"source": "meshtastic", "lat": 35.0, "lng": 100.0, "id": "sig-asia"},
|
||||||
|
]
|
||||||
|
|
||||||
|
# Light/reference layer — must NEVER be filtered.
|
||||||
|
satellites = [
|
||||||
|
{"lat": -60.0, "lng": -120.0, "id": "sat-sw"},
|
||||||
|
{"lat": 35.0, "lng": -75.0, "id": "sat-ne"},
|
||||||
|
{"lat": 35.0, "lng": 100.0, "id": "sat-asia"},
|
||||||
|
]
|
||||||
|
|
||||||
|
monkeypatch.setitem(_store.latest_data, "commercial_flights", commercial)
|
||||||
|
monkeypatch.setitem(_store.latest_data, "ships", ships)
|
||||||
|
monkeypatch.setitem(_store.latest_data, "cctv", cctv)
|
||||||
|
monkeypatch.setitem(_store.latest_data, "sigint", sigint)
|
||||||
|
monkeypatch.setitem(_store.latest_data, "satellites", satellites)
|
||||||
|
# Ensure all layers are on so the response includes them.
|
||||||
|
for layer in (
|
||||||
|
"flights", "ships_military", "ships_cargo", "ships_civilian",
|
||||||
|
"ships_passenger", "ships_tracked_yachts", "cctv",
|
||||||
|
"sigint_meshtastic", "sigint_aprs", "satellites",
|
||||||
|
):
|
||||||
|
monkeypatch.setitem(_store.active_layers, layer, True)
|
||||||
|
|
||||||
|
def test_no_bbox_returns_world_data(self, client, monkeypatch):
|
||||||
|
self._seed_fast(monkeypatch)
|
||||||
|
r = client.get("/api/live-data/fast")
|
||||||
|
assert r.status_code == 200
|
||||||
|
data = r.json()
|
||||||
|
# All heavy fixtures pass through unchanged.
|
||||||
|
assert len(data["commercial_flights"]) == 3
|
||||||
|
assert len(data["ships"]) == 2
|
||||||
|
assert len(data["sigint"]) == 2
|
||||||
|
# Light layer also full.
|
||||||
|
assert len(data["satellites"]) == 3
|
||||||
|
|
||||||
|
def test_bbox_filters_heavy_layers(self, client, monkeypatch):
|
||||||
|
self._seed_fast(monkeypatch)
|
||||||
|
# Box tightly around the eastern-US fixture (lat 35, lng -75).
|
||||||
|
# ±5° → after 20% padding inside _bbox_filter, ~±6° window.
|
||||||
|
r = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70")
|
||||||
|
assert r.status_code == 200
|
||||||
|
data = r.json()
|
||||||
|
# Heavy layers: only the eastern-US fixture survives.
|
||||||
|
assert {f["id"] for f in data["commercial_flights"]} == {"f-ne"}
|
||||||
|
assert {s["id"] for s in data["ships"]} == {"s-ne"}
|
||||||
|
assert {c["id"] for c in data["cctv"]} == {"c-1"}
|
||||||
|
assert {s["id"] for s in data["sigint"]} == {"sig-east"}
|
||||||
|
|
||||||
|
def test_bbox_does_not_filter_light_layers(self, client, monkeypatch):
|
||||||
|
self._seed_fast(monkeypatch)
|
||||||
|
r = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70")
|
||||||
|
assert r.status_code == 200
|
||||||
|
data = r.json()
|
||||||
|
# Satellites are a reference layer — must NOT be bbox-filtered.
|
||||||
|
assert len(data["satellites"]) == 3
|
||||||
|
|
||||||
|
def test_world_scale_bbox_skips_filtering(self, client, monkeypatch):
|
||||||
|
self._seed_fast(monkeypatch)
|
||||||
|
# lng_span = 360 → treated as world-scale; same as no bbox.
|
||||||
|
r = client.get("/api/live-data/fast?s=-90&w=-180&n=90&e=180")
|
||||||
|
assert r.status_code == 200
|
||||||
|
data = r.json()
|
||||||
|
assert len(data["commercial_flights"]) == 3
|
||||||
|
assert len(data["ships"]) == 2
|
||||||
|
|
||||||
|
def test_partial_bbox_is_treated_as_no_bbox(self, client, monkeypatch):
|
||||||
|
self._seed_fast(monkeypatch)
|
||||||
|
# Only three of four bounds → filtering must NOT engage.
|
||||||
|
r = client.get("/api/live-data/fast?s=30&w=-80&n=40")
|
||||||
|
assert r.status_code == 200
|
||||||
|
data = r.json()
|
||||||
|
assert len(data["commercial_flights"]) == 3
|
||||||
|
|
||||||
|
def test_etag_changes_with_bbox(self, client, monkeypatch):
|
||||||
|
self._seed_fast(monkeypatch)
|
||||||
|
r_world = client.get("/api/live-data/fast")
|
||||||
|
r_local = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70")
|
||||||
|
assert r_world.status_code == 200
|
||||||
|
assert r_local.status_code == 200
|
||||||
|
etag_world = r_world.headers.get("etag")
|
||||||
|
etag_local = r_local.headers.get("etag")
|
||||||
|
assert etag_world and etag_local
|
||||||
|
assert etag_world != etag_local, (
|
||||||
|
"ETag must differ between world and regional bbox to prevent "
|
||||||
|
"304 cache poisoning across viewports"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_etag_stable_for_subdegree_pan(self, client, monkeypatch):
|
||||||
|
self._seed_fast(monkeypatch)
|
||||||
|
# Sub-degree pan should land in the same 1°-quantized bucket.
|
||||||
|
r_a = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70")
|
||||||
|
r_b = client.get("/api/live-data/fast?s=30.3&w=-79.8&n=39.7&e=-70.4")
|
||||||
|
assert r_a.headers.get("etag") == r_b.headers.get("etag")
|
||||||
|
|
||||||
|
def test_if_none_match_returns_304_for_same_bbox(self, client, monkeypatch):
|
||||||
|
self._seed_fast(monkeypatch)
|
||||||
|
r1 = client.get("/api/live-data/fast?s=30&w=-80&n=40&e=-70")
|
||||||
|
etag = r1.headers.get("etag")
|
||||||
|
r2 = client.get(
|
||||||
|
"/api/live-data/fast?s=30&w=-80&n=40&e=-70",
|
||||||
|
headers={"If-None-Match": etag},
|
||||||
|
)
|
||||||
|
assert r2.status_code == 304
|
||||||
|
|
||||||
|
|
||||||
|
# ───────────────────────── /api/live-data/slow ─────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestSlowBboxFiltering:
|
||||||
|
def _seed_slow(self, monkeypatch):
|
||||||
|
from services.fetchers import _store
|
||||||
|
|
||||||
|
# Heavy collections.
|
||||||
|
gdelt = [
|
||||||
|
{"lat": 35.0, "lng": -75.0, "id": "g-east"},
|
||||||
|
{"lat": 35.0, "lng": 100.0, "id": "g-asia"},
|
||||||
|
]
|
||||||
|
firms_fires = [
|
||||||
|
{"lat": 35.0, "lng": -75.0, "id": "fire-east"},
|
||||||
|
{"lat": -10.0, "lng": 120.0, "id": "fire-ido"},
|
||||||
|
]
|
||||||
|
# Light/reference layers — must always ship in full.
|
||||||
|
datacenters = [
|
||||||
|
{"lat": 35.0, "lng": -75.0, "id": "dc-east"},
|
||||||
|
{"lat": 35.0, "lng": 100.0, "id": "dc-asia"},
|
||||||
|
{"lat": -10.0, "lng": 120.0, "id": "dc-ido"},
|
||||||
|
]
|
||||||
|
military_bases = [
|
||||||
|
{"lat": 35.0, "lng": -75.0, "id": "mb-east"},
|
||||||
|
{"lat": -10.0, "lng": 120.0, "id": "mb-ido"},
|
||||||
|
]
|
||||||
|
power_plants = [
|
||||||
|
{"lat": 35.0, "lng": -75.0, "id": "pp-east"},
|
||||||
|
{"lat": 35.0, "lng": 100.0, "id": "pp-asia"},
|
||||||
|
]
|
||||||
|
|
||||||
|
monkeypatch.setitem(_store.latest_data, "gdelt", gdelt)
|
||||||
|
monkeypatch.setitem(_store.latest_data, "firms_fires", firms_fires)
|
||||||
|
monkeypatch.setitem(_store.latest_data, "datacenters", datacenters)
|
||||||
|
monkeypatch.setitem(_store.latest_data, "military_bases", military_bases)
|
||||||
|
monkeypatch.setitem(_store.latest_data, "power_plants", power_plants)
|
||||||
|
for layer in (
|
||||||
|
"global_incidents", "firms", "datacenters", "military_bases", "power_plants",
|
||||||
|
):
|
||||||
|
monkeypatch.setitem(_store.active_layers, layer, True)
|
||||||
|
|
||||||
|
def test_no_bbox_returns_world_data(self, client, monkeypatch):
|
||||||
|
self._seed_slow(monkeypatch)
|
||||||
|
r = client.get("/api/live-data/slow")
|
||||||
|
assert r.status_code == 200
|
||||||
|
data = r.json()
|
||||||
|
assert len(data["gdelt"]) == 2
|
||||||
|
assert len(data["firms_fires"]) == 2
|
||||||
|
assert len(data["datacenters"]) == 3
|
||||||
|
|
||||||
|
def test_bbox_filters_heavy_layers(self, client, monkeypatch):
|
||||||
|
self._seed_slow(monkeypatch)
|
||||||
|
r = client.get("/api/live-data/slow?s=30&w=-80&n=40&e=-70")
|
||||||
|
assert r.status_code == 200
|
||||||
|
data = r.json()
|
||||||
|
assert {g["id"] for g in data["gdelt"]} == {"g-east"}
|
||||||
|
assert {f["id"] for f in data["firms_fires"]} == {"fire-east"}
|
||||||
|
|
||||||
|
def test_bbox_leaves_reference_layers_untouched(self, client, monkeypatch):
|
||||||
|
"""Datacenters, bases, and power plants are infrastructure overlays —
|
||||||
|
they must remain world-scale so panning never hides them."""
|
||||||
|
self._seed_slow(monkeypatch)
|
||||||
|
r = client.get("/api/live-data/slow?s=30&w=-80&n=40&e=-70")
|
||||||
|
assert r.status_code == 200
|
||||||
|
data = r.json()
|
||||||
|
assert len(data["datacenters"]) == 3
|
||||||
|
assert len(data["military_bases"]) == 2
|
||||||
|
assert len(data["power_plants"]) == 2
|
||||||
|
|
||||||
|
def test_antimeridian_bbox(self, client, monkeypatch):
|
||||||
|
from services.fetchers import _store
|
||||||
|
# Box that straddles the antimeridian (Pacific): w=170, e=-170.
|
||||||
|
gdelt = [
|
||||||
|
{"lat": 0.0, "lng": 175.0, "id": "in-west"},
|
||||||
|
{"lat": 0.0, "lng": -175.0, "id": "in-east"},
|
||||||
|
{"lat": 0.0, "lng": 0.0, "id": "out-mid"},
|
||||||
|
]
|
||||||
|
monkeypatch.setitem(_store.latest_data, "gdelt", gdelt)
|
||||||
|
monkeypatch.setitem(_store.active_layers, "global_incidents", True)
|
||||||
|
r = client.get("/api/live-data/slow?s=-10&w=170&n=10&e=-170")
|
||||||
|
assert r.status_code == 200
|
||||||
|
data = r.json()
|
||||||
|
ids = {g["id"] for g in data["gdelt"]}
|
||||||
|
assert "in-west" in ids
|
||||||
|
assert "in-east" in ids
|
||||||
|
assert "out-mid" not in ids
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────── Direct helper coverage (defensive) ─────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestHelpers:
|
||||||
|
def test_has_full_bbox(self):
|
||||||
|
from routers.data import _has_full_bbox
|
||||||
|
assert _has_full_bbox(1, 2, 3, 4)
|
||||||
|
assert not _has_full_bbox(None, 2, 3, 4)
|
||||||
|
assert not _has_full_bbox(1, None, 3, 4)
|
||||||
|
assert not _has_full_bbox(1, 2, None, 4)
|
||||||
|
assert not _has_full_bbox(1, 2, 3, None)
|
||||||
|
|
||||||
|
def test_bbox_etag_suffix_quantizes(self):
|
||||||
|
from routers.data import _bbox_etag_suffix
|
||||||
|
a = _bbox_etag_suffix(30.1, -79.6, 39.9, -70.1)
|
||||||
|
b = _bbox_etag_suffix(30.4, -79.2, 39.4, -70.8)
|
||||||
|
assert a == b, "Sub-degree pan must collapse to the same ETag suffix"
|
||||||
|
assert a.startswith("|bbox=")
|
||||||
|
|
||||||
|
def test_bbox_etag_suffix_world_collapses(self):
|
||||||
|
from routers.data import _bbox_etag_suffix
|
||||||
|
# World-scale → empty suffix (shares the global ETag).
|
||||||
|
assert _bbox_etag_suffix(-90, -180, 90, 180) == ""
|
||||||
|
|
||||||
|
def test_bbox_etag_suffix_partial_is_empty(self):
|
||||||
|
from routers.data import _bbox_etag_suffix
|
||||||
|
assert _bbox_etag_suffix(None, -180, 90, 180) == ""
|
||||||
|
|
||||||
|
def test_apply_bbox_preserves_non_list_values(self):
|
||||||
|
from routers.data import _apply_bbox_to_payload, _FAST_BBOX_HEAVY_KEYS
|
||||||
|
payload = {
|
||||||
|
"commercial_flights": [{"lat": 35, "lng": -75, "id": "x"}],
|
||||||
|
"satellite_source": "tle", # not a list, must pass through
|
||||||
|
"sigint_totals": {"total": 1}, # dict — must pass through
|
||||||
|
}
|
||||||
|
out = _apply_bbox_to_payload(dict(payload), _FAST_BBOX_HEAVY_KEYS, 30, -80, 40, -70)
|
||||||
|
assert out["satellite_source"] == "tle"
|
||||||
|
assert out["sigint_totals"] == {"total": 1}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user