mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-06-03 12:58:11 +02:00
Compare commits
28 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| de37dfa09a | |||
| 5e6bb8511a | |||
| 0fee36e8f7 | |||
| e125467721 | |||
| 2b03b808ac | |||
| 2e14e75a0e | |||
| 084e563412 | |||
| 9ef6213284 | |||
| fb11e0881f | |||
| 7f96151e56 | |||
| d0299fc0a0 | |||
| 87ba70acd6 | |||
| bcc2d036b3 | |||
| 729ea78cb2 | |||
| 459178f283 | |||
| 8e27658157 | |||
| e36d1fc79c | |||
| d00c63abed | |||
| e3297e9bc0 | |||
| 9ae0b189ba | |||
| dd7706f17f | |||
| 30f0360ef8 | |||
| 421682c447 | |||
| 40734e310b | |||
| 71a9d9e144 | |||
| de27d119f9 | |||
| b8384d6d91 | |||
| 11ea345518 |
@@ -0,0 +1,32 @@
|
||||
# CODEOWNERS — assigns required reviewers for sensitive paths.
|
||||
# Format: <path glob> <user-or-team> [<user-or-team> ...]
|
||||
# See https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
|
||||
#
|
||||
# Owners listed here are auto-requested for review when matching files
|
||||
# change in a PR. If branch protection requires CODEOWNERS approval, the
|
||||
# PR cannot be merged until an owner approves.
|
||||
|
||||
# ── Internationalization / translations ──
|
||||
# Translation contributions are held to a stricter neutrality standard
|
||||
# than most code changes — see CONTRIBUTING.md "Translation contributions".
|
||||
# The i18n layer itself (no network calls, no telemetry, static JSON
|
||||
# bundled at build) is the structural guarantee that makes this safe;
|
||||
# changes to it need owner review.
|
||||
/frontend/src/i18n/ @BigBodyCobain
|
||||
|
||||
# ── Security-sensitive code paths ──
|
||||
/backend/auth.py @BigBodyCobain
|
||||
/backend/routers/wormhole.py @BigBodyCobain
|
||||
/backend/services/mesh/ @BigBodyCobain
|
||||
/backend/services/fetchers/ @BigBodyCobain
|
||||
|
||||
# ── CI / build / deploy infra ──
|
||||
/.github/workflows/ @BigBodyCobain
|
||||
/.gitlab-ci.yml @BigBodyCobain
|
||||
/docker-compose.yml @BigBodyCobain
|
||||
/docker-compose.gitlab.yml @BigBodyCobain
|
||||
/helm/ @BigBodyCobain
|
||||
|
||||
# ── This file and policy docs ──
|
||||
/.github/CODEOWNERS @BigBodyCobain
|
||||
/CONTRIBUTING.md @BigBodyCobain
|
||||
+18
@@ -91,6 +91,24 @@ backend/data/*
|
||||
!backend/data/power_plants.json
|
||||
!backend/data/tracked_names.json
|
||||
!backend/data/yacht_alert_db.json
|
||||
# Issue #206: bundled KiwiSDR receiver directory used as last-resort
|
||||
# fallback when rx.linkfanel.net (HTTP-only upstream) is unreachable
|
||||
# or returns content that fails our integrity validation.
|
||||
!backend/data/kiwisdr_directory.json
|
||||
# Issue #201: pinned SHA-256 digests for known Tor Expert Bundle URLs.
|
||||
# Used as a second verification source when upstream .sha256sum fails.
|
||||
!backend/data/tor_bundle_digests.json
|
||||
# Issue #258: SPKI pins for stream.aisstream.io so we can survive upstream
|
||||
# Let's Encrypt renewal failures without disabling TLS validation entirely.
|
||||
!backend/data/aisstream_spki_pins.json
|
||||
# Issue #231: pinned SHA-256 digests for known release archives. Used by
|
||||
# the self-updater as a second-line integrity check when the release's
|
||||
# SHA256SUMS.txt asset can't be fetched.
|
||||
!backend/data/release_digests.json
|
||||
# Issue #244/#245/#246: one-shot carrier-position seed shipped with each
|
||||
# release. Used ONLY on first-ever startup to bootstrap carrier_cache.json;
|
||||
# after that the cache reflects this install's own GDELT observations.
|
||||
!backend/data/carrier_seed.json
|
||||
|
||||
# OS generated files
|
||||
.DS_Store
|
||||
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
# GitLab CI/CD for Shadowbroker
|
||||
#
|
||||
# Mirror of .github/workflows/docker-publish.yml — keeps the GitLab install
|
||||
# path (image registry + source) at parity with GitHub so users who prefer
|
||||
# GitLab get the same experience.
|
||||
#
|
||||
# What this does on every push to main:
|
||||
# 1. Builds multi-arch (amd64 + arm64) Docker images for the backend and
|
||||
# frontend, pushes them to the project's GitLab Container Registry:
|
||||
# registry.gitlab.com/bigbodycobain/shadowbroker/backend:latest
|
||||
# registry.gitlab.com/bigbodycobain/shadowbroker/frontend:latest
|
||||
# Both also get a :$CI_COMMIT_SHORT_SHA tag for traceability.
|
||||
# 2. Reverse-mirrors main back to GitHub (only if commits land directly
|
||||
# on GitLab) so the two sources stay in sync.
|
||||
#
|
||||
# Auth notes:
|
||||
# - The image build/push uses $CI_JOB_TOKEN, which GitLab provides
|
||||
# automatically. No credentials need to be configured.
|
||||
# - The reverse mirror requires a GitHub personal access token stored
|
||||
# as the GitLab CI/CD variable GITHUB_MIRROR_TOKEN (Protected + Masked).
|
||||
# Scope: public_repo (or repo for private). If the variable isn't
|
||||
# set the mirror job is skipped — image builds still run.
|
||||
|
||||
stages:
|
||||
- build
|
||||
- mirror
|
||||
|
||||
variables:
|
||||
# Use the dind service for buildx multi-arch builds.
|
||||
DOCKER_HOST: tcp://docker:2376
|
||||
DOCKER_TLS_CERTDIR: "/certs"
|
||||
DOCKER_DRIVER: overlay2
|
||||
# QEMU is what lets a single x86 runner build arm64 images. dind doesn't
|
||||
# install it by default; we install via tonistiigi/binfmt below.
|
||||
BUILDX_VERSION: "v0.14.1"
|
||||
# Repository-relative paths.
|
||||
BACKEND_IMAGE: $CI_REGISTRY_IMAGE/backend
|
||||
FRONTEND_IMAGE: $CI_REGISTRY_IMAGE/frontend
|
||||
|
||||
# Shared template: bootstraps buildx + QEMU on the dind service so a single
|
||||
# runner can produce both amd64 and arm64 manifests in one push.
|
||||
.buildx-setup: &buildx-setup
|
||||
image: docker:24
|
||||
services:
|
||||
- name: docker:24-dind
|
||||
command: ["--tls=true"]
|
||||
before_script:
|
||||
- docker info
|
||||
- docker login -u "$CI_REGISTRY_USER" -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
|
||||
- docker run --privileged --rm tonistiigi/binfmt --install all
|
||||
- docker buildx create --use --name multiarch --driver docker-container
|
||||
|
||||
# ── Backend image ────────────────────────────────────────────────────────
|
||||
build-backend:
|
||||
<<: *buildx-setup
|
||||
stage: build
|
||||
script:
|
||||
- >
|
||||
docker buildx build
|
||||
--platform linux/amd64,linux/arm64
|
||||
--file backend/Dockerfile
|
||||
--tag $BACKEND_IMAGE:latest
|
||||
--tag $BACKEND_IMAGE:$CI_COMMIT_SHORT_SHA
|
||||
--push
|
||||
.
|
||||
rules:
|
||||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "schedule"
|
||||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||
changes:
|
||||
- backend/**/*
|
||||
- .gitlab-ci.yml
|
||||
|
||||
# ── Frontend image ───────────────────────────────────────────────────────
|
||||
build-frontend:
|
||||
<<: *buildx-setup
|
||||
stage: build
|
||||
script:
|
||||
- cd frontend
|
||||
- >
|
||||
docker buildx build
|
||||
--platform linux/amd64,linux/arm64
|
||||
--tag $FRONTEND_IMAGE:latest
|
||||
--tag $FRONTEND_IMAGE:$CI_COMMIT_SHORT_SHA
|
||||
--push
|
||||
.
|
||||
rules:
|
||||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "schedule"
|
||||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||
changes:
|
||||
- frontend/**/*
|
||||
- .gitlab-ci.yml
|
||||
|
||||
# ── Reverse mirror to GitHub ─────────────────────────────────────────────
|
||||
# Pushes refs/heads/main to github.com/BigBodyCobain/Shadowbroker.
|
||||
# Fast-forward-only — if GitLab main and GitHub main have diverged, this
|
||||
# fails loudly rather than silently overwriting either side.
|
||||
#
|
||||
# Only runs if GITHUB_MIRROR_TOKEN is set as a CI/CD variable. See the
|
||||
# header comment of this file for setup instructions.
|
||||
mirror-to-github:
|
||||
stage: mirror
|
||||
image: alpine:3.20
|
||||
needs: []
|
||||
before_script:
|
||||
- apk add --no-cache git openssh-client ca-certificates
|
||||
script:
|
||||
- git config --global user.email "ci-mirror@gitlab.com"
|
||||
- git config --global user.name "GitLab CI Mirror"
|
||||
- >
|
||||
git clone --depth=50 --branch main
|
||||
"https://oauth2:${CI_JOB_TOKEN}@gitlab.com/${CI_PROJECT_PATH}.git"
|
||||
repo
|
||||
- cd repo
|
||||
- >
|
||||
git push
|
||||
"https://x-access-token:${GITHUB_MIRROR_TOKEN}@github.com/BigBodyCobain/Shadowbroker.git"
|
||||
"${CI_COMMIT_SHA}:refs/heads/main"
|
||||
rules:
|
||||
- if: $CI_COMMIT_BRANCH == "main" && $GITHUB_MIRROR_TOKEN
|
||||
@@ -0,0 +1,75 @@
|
||||
# Contributing to Shadowbroker
|
||||
|
||||
Thank you for taking the time to contribute. This document covers things specific to this project — for general open-source contribution etiquette, see the GitHub docs.
|
||||
|
||||
---
|
||||
|
||||
## Code contributions
|
||||
|
||||
1. Fork the repo on GitHub (`bigbodycobain/Shadowbroker`) or GitLab (`bigbodycobain/Shadowbroker` mirror).
|
||||
2. Make your changes on a feature branch.
|
||||
3. Run the local test suite:
|
||||
- Backend: `pytest backend/tests/`
|
||||
- Frontend: `cd frontend && npx vitest run`
|
||||
4. Open a Pull Request against `main`.
|
||||
|
||||
CI runs on every PR. If CI fails, that's blocking — please push fixes rather than asking for it to be merged anyway.
|
||||
|
||||
---
|
||||
|
||||
## Reporting security issues
|
||||
|
||||
Do **not** file security issues as public GitHub issues. Email the maintainer or use a private security advisory on GitHub. Public disclosure of an exploitable vulnerability without prior coordination will be rejected from the project.
|
||||
|
||||
---
|
||||
|
||||
## Translation contributions
|
||||
|
||||
Shadowbroker supports UI localization (`frontend/src/i18n/`). Translation contributions are welcome but held to a stricter standard than most code changes, because translations can subtly reshape user perception in ways that are hard to spot during review. Read this section before submitting one.
|
||||
|
||||
### The neutrality requirement
|
||||
|
||||
**Translations must be technically faithful to the English source.** That means:
|
||||
|
||||
- Each `t('key')` entry should mean approximately the same thing in the target language as in English, modulo idiom.
|
||||
- Technical terms with established meanings (e.g. "GPS jamming," "military flight," "Tor," "onion routing," "encryption") should be translated using the corresponding established technical term in the target language — **not** softened, rebranded, or politically reframed.
|
||||
- The set of UI strings should be **the same** between languages. Don't omit features from one locale that are visible in another.
|
||||
|
||||
### What will get a translation PR rejected
|
||||
|
||||
Translation choices that align the project with the framing or terminology of state propaganda — from **any** country — will be rejected. This applies symmetrically:
|
||||
|
||||
| Country / source | Examples of substitutions we will reject |
|
||||
|---|---|
|
||||
| **PRC / CCP** | Calling Taiwan a "province" or "renegade province"; reframing protest layers as "riots"; using softened or euphemistic terms for surveillance, internment, or jamming when the source text is direct |
|
||||
| **Russia** | Calling the Ukraine war a "special military operation"; relabeling occupied territories as Russian; softening sanctions/jamming/disinfo terminology |
|
||||
| **United States / EU** | Reframing adversaries with editorial labels not in the source (e.g. inserting "regime" where the English says "government"); applying labels like "terrorist" or "rogue state" to entities the English source describes neutrally |
|
||||
| **Israel / Palestine / any active conflict** | Substituting one side's preferred terminology when the source uses the other side's or a neutral term |
|
||||
| **Any government** | Adding political slogans, omitting features that government finds inconvenient, or inserting terminology associated with a specific political faction |
|
||||
|
||||
The test is **"would a translator working strictly from the English source produce this rendering?"** If the answer requires assuming a political stance the source does not take, the substitution does not belong in the translation.
|
||||
|
||||
### How translation PRs are reviewed
|
||||
|
||||
Changes to `frontend/src/i18n/**` are owned by the maintainer (see `CODEOWNERS`) and require explicit approval. We will:
|
||||
|
||||
1. Diff the translation against the English source key-by-key.
|
||||
2. Spot-check a sample of entries with a native speaker of the target language when possible.
|
||||
3. Look for the patterns above.
|
||||
4. Look for suspicious additions to the i18n infrastructure itself (e.g. a remote translation fetcher, telemetry on language choice) — the i18n layer is supposed to be 100% client-side static JSON.
|
||||
|
||||
A PR that adds a new language is harder to review than one that fixes typos in an existing language. For new languages, please be patient and expect a real review window. For typo fixes, please describe each change in the PR body so the reviewer can verify intent.
|
||||
|
||||
### What about adding a new language?
|
||||
|
||||
We welcome new languages. The mechanical setup is documented in the header comment of `frontend/src/i18n/index.ts`. Beyond that:
|
||||
|
||||
- We are more likely to merge a new language quickly if at least one reviewer in the maintainer's network speaks it.
|
||||
- If you are the *only* speaker of the target language reading this repo, your translation is welcome but the merge timeline will be longer while a reviewer is found.
|
||||
- Partial translations are fine — the system falls back to English for any missing key.
|
||||
|
||||
---
|
||||
|
||||
## Anything else
|
||||
|
||||
If you have a question that isn't a security report, opening a GitHub Discussion or a draft PR with a question in the body is the fastest way to get a response. Direct emails are read but not always replied to promptly.
|
||||
@@ -61,6 +61,8 @@ ShadowBroker includes an optional Shodan connector for operator-supplied API acc
|
||||
|
||||
## ⚡ Quick Start (Docker)
|
||||
|
||||
### From GitHub (default — uses GHCR images)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/bigbodycobain/Shadowbroker.git
|
||||
cd Shadowbroker
|
||||
@@ -68,6 +70,17 @@ docker compose pull
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
### From GitLab (uses GitLab Container Registry)
|
||||
|
||||
```bash
|
||||
git clone https://gitlab.com/bigbodycobain/Shadowbroker.git
|
||||
cd Shadowbroker
|
||||
docker compose -f docker-compose.yml -f docker-compose.gitlab.yml pull
|
||||
docker compose -f docker-compose.yml -f docker-compose.gitlab.yml up -d
|
||||
```
|
||||
|
||||
Both paths produce identical containers — same source, same CI, same images byte-for-byte. Pick whichever ecosystem you already use.
|
||||
|
||||
Open `http://localhost:3000` to view the dashboard! *(Requires [Docker Desktop](https://www.docker.com/products/docker-desktop/) or Docker Engine)*
|
||||
|
||||
> **Backend port already in use?** The browser only needs port `3000`, but the backend API is also published on host port `8000` for local diagnostics. If another app already uses `8000`, create or edit `.env` next to `docker-compose.yml` and set `BACKEND_PORT=8001`, then run `docker compose up -d`.
|
||||
@@ -136,8 +149,13 @@ helm repo update
|
||||
|
||||
**2. Install the Chart:**
|
||||
```bash
|
||||
# Install from the local helm/chart directory
|
||||
# Default — pulls images from GHCR
|
||||
helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbroker
|
||||
|
||||
# GitLab registry variant
|
||||
helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbroker \
|
||||
-f helm/chart/values.yaml \
|
||||
-f helm/chart/values-gitlab.yaml
|
||||
```
|
||||
|
||||
**3. Key Features:**
|
||||
|
||||
+38
-2
@@ -24,8 +24,40 @@ AIS_API_KEY= # https://aisstream.io/ — free tier WebSocket key
|
||||
# Requires MESH_DEBUG_MODE=true; do not enable this for ordinary use.
|
||||
# ALLOW_INSECURE_ADMIN=false
|
||||
|
||||
# Default outbound User-Agent for all third-party HTTP fetchers.
|
||||
# Project-generic by default — does NOT include any personal contact info or
|
||||
# operator-specific identifier. Override only if you run a public relay and
|
||||
# want upstreams to be able to reach you (e.g. Nominatim/OSM usage policy).
|
||||
# SHADOWBROKER_USER_AGENT=ShadowBroker-OSINT/0.9 (contact: ops@example.com)
|
||||
|
||||
# User-Agent for Nominatim geocoding requests (per OSM usage policy).
|
||||
# NOMINATIM_USER_AGENT=ShadowBroker/1.0 (https://github.com/BigBodyCobain/Shadowbroker)
|
||||
# NOMINATIM_USER_AGENT=ShadowBroker/1.0
|
||||
|
||||
# ── Third-party fetcher opt-ins ────────────────────────────────
|
||||
# These data sources phone home to politically/commercially sensitive
|
||||
# upstreams. Disabled by default; set to "true" only if the operator
|
||||
# explicitly wants the node's IP to contact these services.
|
||||
#
|
||||
# CrowdThreat — backend.crowdthreat.world (paid threat-intel aggregator).
|
||||
# CROWDTHREAT_ENABLED=false
|
||||
#
|
||||
# EUvsDisinfo FIMI — euvsdisinfo.eu (EU disinformation tracker).
|
||||
# FIMI_ENABLED=false
|
||||
#
|
||||
# Polymarket + Kalshi — US political/election prediction markets.
|
||||
# PREDICTION_MARKETS_ENABLED=false
|
||||
#
|
||||
# Finnhub fallback / yfinance — financial market data.
|
||||
# Set FINNHUB_API_KEY to enable Finnhub, or set FINANCIAL_ENABLED=true to allow
|
||||
# the unauthenticated yfinance fallback to call Yahoo Finance.
|
||||
# FINANCIAL_ENABLED=false
|
||||
#
|
||||
# NUFORC UAP sightings — huggingface.co dataset download.
|
||||
# NUFORC_ENABLED=false
|
||||
#
|
||||
# News RSS aggregator — defaults ON. Set to "false" to disable all
|
||||
# configured news feeds (kill switch for the news layer).
|
||||
# NEWS_ENABLED=true
|
||||
|
||||
# LTA Singapore traffic cameras — leave blank to skip this data source.
|
||||
# LTA_ACCOUNT_KEY=
|
||||
@@ -61,8 +93,12 @@ AIS_API_KEY= # https://aisstream.io/ — free tier WebSocket key
|
||||
# Optional Meshtastic node ID (e.g. "!abcd1234"). When set, included in the
|
||||
# User-Agent sent to meshtastic.liamcottle.net so the upstream service operator
|
||||
# can identify per-install traffic instead of aggregated "ShadowBroker" hits.
|
||||
# Leave blank to send a generic UA with the project contact email only.
|
||||
# Leave blank to send a generic UA. If you set MESHTASTIC_OPERATOR_CALLSIGN,
|
||||
# it is included in outbound headers to meshtastic.org by default so they
|
||||
# can rate-limit per-operator. Set MESHTASTIC_SEND_CALLSIGN_HEADER=false to
|
||||
# suppress the callsign while still using it locally (e.g. for APRS).
|
||||
# MESHTASTIC_OPERATOR_CALLSIGN=
|
||||
# MESHTASTIC_SEND_CALLSIGN_HEADER=true
|
||||
# MESH_MQTT_PSK= # hex-encoded, empty = default LongFast key
|
||||
|
||||
# ── Mesh / Reticulum (RNS) ─────────────────────────────────────
|
||||
|
||||
+234
-4
@@ -1,5 +1,37 @@
|
||||
// AIS Stream WebSocket proxy.
|
||||
//
|
||||
// Reads AIS_API_KEY from argv or env, opens a wss:// connection to
|
||||
// stream.aisstream.io, subscribes for vessel position reports inside the
|
||||
// active map bounding boxes, and pipes JSON messages to stdout for the
|
||||
// Python backend to ingest.
|
||||
//
|
||||
// Issue #258 — SPKI pinning fallback for upstream cert outages
|
||||
// -------------------------------------------------------------
|
||||
// AISStream uses Let's Encrypt and their renewal pipeline has been observed
|
||||
// to fail (cert expired on 2026-05-20). The naive fix the issue reporter
|
||||
// applied — passing { rejectUnauthorized: false } — turns off TLS validation
|
||||
// entirely, which lets any network attacker MITM the WebSocket and inject
|
||||
// fake ship positions onto the operator's map. Same class as the GDELT
|
||||
// plaintext-HTTP MITM issue (#199).
|
||||
//
|
||||
// Instead, when the normal TLS handshake fails with CERT_HAS_EXPIRED, we
|
||||
// do a custom TLS connection that ignores ONLY the expiry check, capture
|
||||
// the leaf certificate, and compare its public-key SPKI hash against a
|
||||
// pinned list (backend/data/aisstream_spki_pins.json). If the SPKI matches,
|
||||
// the upstream is still the genuine AISStream — just with an expired cert —
|
||||
// and we proceed in "degraded TLS" mode. If the SPKI does not match, we
|
||||
// refuse the connection and log loudly: an actual MITM is in progress.
|
||||
//
|
||||
// Let's Encrypt renewals keep the same public key by default, so the pinned
|
||||
// SPKI survives normal cert rotation. The pin list MUST be updated before
|
||||
// the operator's pinned key is rotated upstream.
|
||||
|
||||
const WebSocket = require('ws');
|
||||
const readline = require('readline');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const tls = require('tls');
|
||||
const crypto = require('crypto');
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const API_KEY = args[0] || process.env.AIS_API_KEY;
|
||||
@@ -9,6 +41,135 @@ if (!API_KEY) {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// ── SPKI pin support (issue #258) ─────────────────────────────────────────
|
||||
|
||||
const AIS_HOST = 'stream.aisstream.io';
|
||||
const AIS_PORT = 443;
|
||||
const AIS_WS_URL = `wss://${AIS_HOST}/v0/stream`;
|
||||
|
||||
// Pin file is looked up in several layouts so the same JS works in:
|
||||
// - the Docker backend image (PIN_FILE_CANDIDATES[0])
|
||||
// - the Tauri desktop runtime (PIN_FILE_CANDIDATES[1])
|
||||
// - a future relocated layout (operator can drop a file at
|
||||
// SHADOWBROKER_AIS_PINS env var)
|
||||
const PIN_FILE_CANDIDATES = [
|
||||
process.env.SHADOWBROKER_AIS_PINS || '',
|
||||
path.join(__dirname, 'data', 'aisstream_spki_pins.json'),
|
||||
path.join(__dirname, 'aisstream_spki_pins.json'),
|
||||
].filter(Boolean);
|
||||
|
||||
// Embedded fallback. Used when no external pin file is reachable so the
|
||||
// SPKI fallback still works on minimal/portable installs. The external
|
||||
// file (when present) takes priority so operators can update pins without
|
||||
// needing a new build.
|
||||
const EMBEDDED_PINS = {
|
||||
[AIS_HOST]: [
|
||||
// Captured 2026-05-20 from AISStream's leaf cert (Let's Encrypt R12).
|
||||
// Replace when AISStream rotates server keys.
|
||||
'GJ10H0UPgLrO+2d3ZXROR/TXSVFXKUfRC3QEI2ibEg4=',
|
||||
],
|
||||
};
|
||||
|
||||
let aisDegradedMode = false; // surfaced via stdout status_query marker
|
||||
|
||||
function loadSpkiPins() {
|
||||
for (const candidate of PIN_FILE_CANDIDATES) {
|
||||
try {
|
||||
const raw = fs.readFileSync(candidate, 'utf-8');
|
||||
const parsed = JSON.parse(raw);
|
||||
const pins = Array.isArray(parsed[AIS_HOST]) ? parsed[AIS_HOST] : [];
|
||||
const cleaned = pins
|
||||
.filter((p) => typeof p === 'string' && p.length > 0)
|
||||
.map((p) => p.trim());
|
||||
if (cleaned.length > 0) {
|
||||
return cleaned;
|
||||
}
|
||||
} catch (e) {
|
||||
// Try the next candidate — file may not exist in this layout.
|
||||
continue;
|
||||
}
|
||||
}
|
||||
const embedded = (EMBEDDED_PINS[AIS_HOST] || []).slice();
|
||||
if (embedded.length > 0) {
|
||||
console.error(
|
||||
'[AIS Proxy] No external SPKI pin file found; using embedded fallback. '
|
||||
+ `(Set SHADOWBROKER_AIS_PINS or drop ${PIN_FILE_CANDIDATES[1]} to override.)`
|
||||
);
|
||||
}
|
||||
return embedded;
|
||||
}
|
||||
|
||||
function spkiHashFromPeerCert(peerCert) {
|
||||
// tls.TLSSocket.getPeerCertificate() exposes .pubkey when called with
|
||||
// detailed=true. The pubkey buffer is the DER-encoded SubjectPublicKeyInfo,
|
||||
// which is exactly the value we hash for SPKI pinning.
|
||||
if (!peerCert || !peerCert.pubkey) return null;
|
||||
return crypto.createHash('sha256').update(peerCert.pubkey).digest('base64');
|
||||
}
|
||||
|
||||
// Probe the upstream when normal TLS failed with CERT_HAS_EXPIRED. We open
|
||||
// a raw TLS connection with rejectUnauthorized=false ONLY to inspect the
|
||||
// leaf cert; we do NOT use this socket for the actual WebSocket traffic.
|
||||
// Returns { ok: true } if the leaf SPKI matches the pin list, { ok: false }
|
||||
// with a reason otherwise.
|
||||
function verifyExpiredCertAgainstPins() {
|
||||
return new Promise((resolve) => {
|
||||
const pins = loadSpkiPins();
|
||||
if (pins.length === 0) {
|
||||
resolve({ ok: false, reason: 'no SPKI pins configured' });
|
||||
return;
|
||||
}
|
||||
const sock = tls.connect(
|
||||
{
|
||||
host: AIS_HOST,
|
||||
port: AIS_PORT,
|
||||
servername: AIS_HOST,
|
||||
// Allow the handshake to complete despite the expired cert
|
||||
// so we can inspect the leaf. We do NOT trust this connection
|
||||
// for any application data.
|
||||
rejectUnauthorized: false,
|
||||
},
|
||||
() => {
|
||||
const peer = sock.getPeerCertificate(true);
|
||||
sock.end();
|
||||
if (!peer || Object.keys(peer).length === 0) {
|
||||
resolve({ ok: false, reason: 'no peer certificate returned' });
|
||||
return;
|
||||
}
|
||||
if (peer.subject && peer.subject.CN !== AIS_HOST) {
|
||||
resolve({
|
||||
ok: false,
|
||||
reason: `cert CN mismatch (got ${peer.subject.CN}, expected ${AIS_HOST})`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const hash = spkiHashFromPeerCert(peer);
|
||||
if (!hash) {
|
||||
resolve({ ok: false, reason: 'could not compute SPKI hash from peer cert' });
|
||||
return;
|
||||
}
|
||||
if (pins.includes(hash)) {
|
||||
resolve({ ok: true, hash });
|
||||
} else {
|
||||
resolve({
|
||||
ok: false,
|
||||
reason: `SPKI ${hash} not in pin list (possible MITM)`,
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
sock.setTimeout(10000, () => {
|
||||
sock.destroy();
|
||||
resolve({ ok: false, reason: 'TLS probe timeout' });
|
||||
});
|
||||
sock.on('error', (err) => {
|
||||
resolve({ ok: false, reason: `TLS probe error: ${err.message}` });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// ── Subscription state ───────────────────────────────────────────────────
|
||||
|
||||
// Start with global coverage, until frontend updates it
|
||||
let currentBboxes = [[[-90, -180], [90, 180]]];
|
||||
let activeWs = null;
|
||||
@@ -42,14 +203,34 @@ rl.on('line', (line) => {
|
||||
currentBboxes = cmd.bboxes;
|
||||
if (activeWs) sendSub(activeWs); // Resend subscription (swap and replace)
|
||||
}
|
||||
if (cmd.type === "status_query") {
|
||||
// Allow the Python side to probe degraded-mode state by sending
|
||||
// {"type": "status_query"} on stdin. Reply on stdout as a marker.
|
||||
process.stdout.write(JSON.stringify({
|
||||
__ais_proxy_status: { degraded_tls: aisDegradedMode }
|
||||
}) + '\n');
|
||||
}
|
||||
} catch (e) {}
|
||||
});
|
||||
|
||||
function connect() {
|
||||
const ws = new WebSocket('wss://stream.aisstream.io/v0/stream');
|
||||
function attachWsHandlers(ws, { degraded } = { degraded: false }) {
|
||||
activeWs = ws;
|
||||
|
||||
ws.on('open', () => {
|
||||
if (degraded) {
|
||||
console.error(
|
||||
'[AIS Proxy] Connected in DEGRADED TLS MODE — upstream cert is expired '
|
||||
+ 'but SPKI matches the pinned key, so identity is still verified. '
|
||||
+ 'AISStream needs to renew their cert; until then MITM protection '
|
||||
+ 'depends only on the SPKI match. Watch backend logs for resolution.'
|
||||
);
|
||||
aisDegradedMode = true;
|
||||
} else {
|
||||
if (aisDegradedMode) {
|
||||
console.error('[AIS Proxy] Reconnected with full TLS validation — degraded mode cleared.');
|
||||
}
|
||||
aisDegradedMode = false;
|
||||
}
|
||||
sendSub(ws);
|
||||
});
|
||||
|
||||
@@ -61,14 +242,63 @@ function connect() {
|
||||
});
|
||||
|
||||
ws.on('error', (err) => {
|
||||
console.error("WebSocket Proxy Error:", err.message);
|
||||
console.error('WebSocket Proxy Error:', err.message);
|
||||
});
|
||||
|
||||
ws.on('close', () => {
|
||||
activeWs = null;
|
||||
console.error("WebSocket Proxy Closed. Reconnecting in 5s...");
|
||||
console.error('WebSocket Proxy Closed. Reconnecting in 5s...');
|
||||
setTimeout(connect, 5000);
|
||||
});
|
||||
}
|
||||
|
||||
function connect() {
|
||||
// Path A: normal TLS validation (the 99.9% case). If this succeeds we
|
||||
// never touch the SPKI fallback.
|
||||
const ws = new WebSocket(AIS_WS_URL);
|
||||
|
||||
let openedOk = false;
|
||||
ws.on('open', () => { openedOk = true; });
|
||||
|
||||
ws.on('error', async (err) => {
|
||||
// Only the CERT_HAS_EXPIRED case triggers SPKI verification. Any
|
||||
// other TLS or network error gets the standard reconnect path so we
|
||||
// don't accidentally cover up legitimate problems.
|
||||
if (!openedOk && err && err.code === 'CERT_HAS_EXPIRED') {
|
||||
console.error(
|
||||
'[AIS Proxy] Upstream certificate is expired. Verifying SPKI '
|
||||
+ 'against pinned keys before deciding whether to proceed in '
|
||||
+ 'degraded mode...'
|
||||
);
|
||||
const verdict = await verifyExpiredCertAgainstPins();
|
||||
if (verdict.ok) {
|
||||
console.error(
|
||||
`[AIS Proxy] SPKI ${verdict.hash} matches pinned key — `
|
||||
+ 'identity is verified, proceeding in DEGRADED TLS mode.'
|
||||
);
|
||||
const insecureWs = new WebSocket(AIS_WS_URL, {
|
||||
rejectUnauthorized: false,
|
||||
});
|
||||
attachWsHandlers(insecureWs, { degraded: true });
|
||||
} else {
|
||||
console.error(
|
||||
`[AIS Proxy] SPKI verification FAILED (${verdict.reason}). `
|
||||
+ 'Refusing to connect — this would normally indicate an active '
|
||||
+ 'MITM attack. If AISStream rotated their server key, update '
|
||||
+ 'backend/data/aisstream_spki_pins.json with the new SPKI hash.'
|
||||
);
|
||||
// Schedule a retry — operator may have updated the pin file.
|
||||
setTimeout(connect, 60000);
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Default: surface the error and let the close handler reconnect.
|
||||
console.error('WebSocket Proxy Error:', err.message);
|
||||
});
|
||||
|
||||
// Wire normal handlers — these apply unless the error handler above
|
||||
// takes over and replaces activeWs with an insecure socket.
|
||||
attachWsHandlers(ws, { degraded: false });
|
||||
}
|
||||
|
||||
connect();
|
||||
|
||||
+89
-9
@@ -45,6 +45,7 @@ from services.mesh.mesh_compatibility import (
|
||||
from services.mesh.mesh_crypto import (
|
||||
_derive_peer_key,
|
||||
normalize_peer_url,
|
||||
resolve_peer_key_for_url,
|
||||
verify_signature,
|
||||
verify_node_binding,
|
||||
parse_public_key_algo,
|
||||
@@ -245,15 +246,90 @@ def _docker_bridge_local_operator_enabled() -> bool:
|
||||
}
|
||||
|
||||
|
||||
# Issue #250 (tg12): the previous implementation returned True for any IP
|
||||
# in the entire 172.16.0.0/12 range. Anyone with `docker run` access on
|
||||
# the same daemon could spin up a container that automatically passed
|
||||
# local-operator auth. The fix narrows trust to ONLY connections whose
|
||||
# source IP matches the configured frontend container's hostname.
|
||||
#
|
||||
# Docker DNS resolves both the compose service name (``frontend``) and
|
||||
# the explicit ``container_name`` (``shadowbroker-frontend``) to the
|
||||
# frontend container's bridge IP. We forward-resolve both, cache the
|
||||
# result for 30s, and only trust connections from those exact IPs.
|
||||
#
|
||||
# Operators on shared Docker hosts get the benefit of the narrower
|
||||
# surface. Operators on single-user installs see no behavior change —
|
||||
# their frontend container still resolves and is still trusted.
|
||||
_DOCKER_BRIDGE_TRUST_CACHE: dict = {"ips": frozenset(), "expires": 0.0}
|
||||
_DOCKER_BRIDGE_TRUST_TTL = 30.0
|
||||
|
||||
|
||||
def _trusted_bridge_frontend_hostnames() -> list[str]:
|
||||
"""Container hostnames whose IPs we treat as local-operator on the bridge.
|
||||
|
||||
Default covers both Docker Compose service name (``frontend``) and the
|
||||
explicit ``container_name`` from the shipped docker-compose.yml
|
||||
(``shadowbroker-frontend``). Operators with non-default names can
|
||||
override via the ``SHADOWBROKER_TRUSTED_FRONTEND_HOSTS`` env var
|
||||
(comma-separated, no spaces).
|
||||
"""
|
||||
raw = str(
|
||||
os.environ.get(
|
||||
"SHADOWBROKER_TRUSTED_FRONTEND_HOSTS",
|
||||
"frontend,shadowbroker-frontend",
|
||||
)
|
||||
).strip()
|
||||
return [h.strip() for h in raw.split(",") if h.strip()]
|
||||
|
||||
|
||||
def _resolve_trusted_bridge_ips() -> frozenset[str]:
|
||||
"""Resolve trusted frontend hostnames to a set of IPs, with caching.
|
||||
|
||||
Cached for 30s so we don't hit DNS on every request. The cache is
|
||||
process-local — frontend container IP rotations during a backend's
|
||||
lifetime will be picked up within 30s.
|
||||
|
||||
Returns frozenset() if Docker DNS can't resolve any of the configured
|
||||
hostnames (fail-closed — when in doubt, refuse to trust the bridge).
|
||||
"""
|
||||
import socket
|
||||
import time as _time
|
||||
|
||||
now = _time.time()
|
||||
cache = _DOCKER_BRIDGE_TRUST_CACHE
|
||||
if cache["expires"] > now:
|
||||
return cache["ips"]
|
||||
|
||||
ips: set[str] = set()
|
||||
for hostname in _trusted_bridge_frontend_hostnames():
|
||||
try:
|
||||
_, _, addrs = socket.gethostbyname_ex(hostname)
|
||||
except (OSError, socket.gaierror):
|
||||
continue
|
||||
for addr in addrs:
|
||||
ips.add(addr)
|
||||
|
||||
resolved = frozenset(ips)
|
||||
cache["ips"] = resolved
|
||||
cache["expires"] = now + _DOCKER_BRIDGE_TRUST_TTL
|
||||
return resolved
|
||||
|
||||
|
||||
def _is_docker_bridge_host(host: str) -> bool:
|
||||
"""Return True only when the source IP matches our trusted frontend
|
||||
container hostname(s).
|
||||
|
||||
Previously trusted any 172.16.0.0/12 IP unconditionally. See the
|
||||
block comment above for the security rationale.
|
||||
"""
|
||||
try:
|
||||
ip = ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
return False
|
||||
# Docker Desktop and the default compose bridge normally sit inside
|
||||
# 172.16.0.0/12. Keep this narrower than "any private IP" so a user who
|
||||
# intentionally binds the backend to LAN does not silently trust LAN clients.
|
||||
return ip in ipaddress.ip_network("172.16.0.0/12")
|
||||
# Public IPs are never our frontend container — skip DNS work for them.
|
||||
if not ip.is_private:
|
||||
return False
|
||||
return host in _resolve_trusted_bridge_ips()
|
||||
|
||||
|
||||
def _is_trusted_local_runtime_host(host: str) -> bool:
|
||||
@@ -1328,11 +1404,15 @@ def _peer_hmac_url_from_request(request: Request) -> str:
|
||||
|
||||
|
||||
def _verify_peer_push_hmac(request: Request, body_bytes: bytes) -> bool:
|
||||
"""Verify HMAC-SHA256 peer authentication on push requests."""
|
||||
secret = str(get_settings().MESH_PEER_PUSH_SECRET or "").strip()
|
||||
if not secret:
|
||||
return False
|
||||
"""Verify HMAC-SHA256 peer authentication on push requests.
|
||||
|
||||
Issue #256: ``resolve_peer_key_for_url`` looks up a per-peer secret
|
||||
in ``MESH_PEER_SECRETS`` first, then falls back to the global
|
||||
``MESH_PEER_PUSH_SECRET``. When a peer URL is listed in the per-peer
|
||||
map, only the listed secret is accepted for it — the global secret
|
||||
is ignored, so any peer that knows only the global secret cannot
|
||||
forge a request claiming to be that peer.
|
||||
"""
|
||||
provided = str(request.headers.get("x-peer-hmac", "") or "").strip()
|
||||
if not provided:
|
||||
return False
|
||||
@@ -1341,7 +1421,7 @@ def _verify_peer_push_hmac(request: Request, body_bytes: bytes) -> bool:
|
||||
allowed_peers = set(authenticated_push_peer_urls())
|
||||
if not peer_url or peer_url not in allowed_peers:
|
||||
return False
|
||||
peer_key = _derive_peer_key(secret, peer_url)
|
||||
peer_key = resolve_peer_key_for_url(peer_url)
|
||||
if not peer_key:
|
||||
return False
|
||||
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"_comment": [
|
||||
"SPKI (Subject Public Key Info) pin list for stream.aisstream.io.",
|
||||
"",
|
||||
"Issue #258: AISStream's Let's Encrypt cert expired on 2026-05-20 due to an",
|
||||
"upstream renewal-pipeline failure. Disabling TLS verification entirely",
|
||||
"would let any network attacker MITM the AIS WebSocket and inject fake",
|
||||
"ship positions onto the operator's map (same class as #199 GDELT MITM).",
|
||||
"Instead we pin the leaf certificate's public-key SPKI hash: if normal",
|
||||
"TLS validation fails specifically with CERT_HAS_EXPIRED, ais_proxy.js",
|
||||
"re-checks the leaf cert's SPKI against this list. A match means the",
|
||||
"key is still the genuine AISStream key (Let's Encrypt renewals keep the",
|
||||
"same key unless rekey is requested), so we proceed in 'degraded TLS'",
|
||||
"mode. A mismatch means a real MITM attempt and we refuse the connection.",
|
||||
"",
|
||||
"Format: each entry is a SHA-256 hash of the DER-encoded SPKI bytes,",
|
||||
"encoded as standard base64 (matches the format produced by:",
|
||||
" openssl s_client -connect host:443 | \\",
|
||||
" openssl x509 -pubkey -noout | openssl pkey -pubin -outform DER | \\",
|
||||
" openssl dgst -sha256 -binary | openssl base64",
|
||||
").",
|
||||
"",
|
||||
"When AISStream rotates their server key (rare — Let's Encrypt renewals",
|
||||
"default to keeping the same key), capture the new SPKI and add it to",
|
||||
"this list BEFORE removing the old one. That way operators on the old",
|
||||
"code still validate against the previous key during the transition."
|
||||
],
|
||||
"stream.aisstream.io": [
|
||||
"GJ10H0UPgLrO+2d3ZXROR/TXSVFXKUfRC3QEI2ibEg4="
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
{
|
||||
"_meta": {
|
||||
"as_of": "2026-03-09",
|
||||
"source": "USNI News Fleet & Marine Tracker",
|
||||
"source_url": "https://news.usni.org/2026/03/09/usni-news-fleet-and-marine-tracker-march-9-2026",
|
||||
"note": "One-shot bootstrap for first-run carrier positions. Once carrier_cache.json exists in the runtime data volume, this seed file is never read again. All subsequent updates come from GDELT (and any future sources) and are written to carrier_cache.json. A year from now, your runtime cache reflects whatever your install has observed since first launch — not these snapshot positions."
|
||||
},
|
||||
"carriers": {
|
||||
"CVN-68": {
|
||||
"lat": 47.5535,
|
||||
"lng": -122.6400,
|
||||
"heading": 90,
|
||||
"desc": "Bremerton, WA (Maintenance)",
|
||||
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||
"position_source_at": "2026-03-09T00:00:00Z",
|
||||
"position_confidence": "seed"
|
||||
},
|
||||
"CVN-76": {
|
||||
"lat": 47.5580,
|
||||
"lng": -122.6360,
|
||||
"heading": 90,
|
||||
"desc": "Bremerton, WA (Decommissioning)",
|
||||
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||
"position_source_at": "2026-03-09T00:00:00Z",
|
||||
"position_confidence": "seed"
|
||||
},
|
||||
"CVN-69": {
|
||||
"lat": 36.9465,
|
||||
"lng": -76.3265,
|
||||
"heading": 0,
|
||||
"desc": "Norfolk, VA (Post-deployment maintenance)",
|
||||
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||
"position_source_at": "2026-03-09T00:00:00Z",
|
||||
"position_confidence": "seed"
|
||||
},
|
||||
"CVN-78": {
|
||||
"lat": 18.0,
|
||||
"lng": 39.5,
|
||||
"heading": 0,
|
||||
"desc": "Red Sea — Operation Epic Fury (USNI Mar 9)",
|
||||
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||
"position_source_at": "2026-03-09T00:00:00Z",
|
||||
"position_confidence": "seed"
|
||||
},
|
||||
"CVN-74": {
|
||||
"lat": 36.98,
|
||||
"lng": -76.43,
|
||||
"heading": 0,
|
||||
"desc": "Newport News, VA (RCOH refueling overhaul)",
|
||||
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||
"position_source_at": "2026-03-09T00:00:00Z",
|
||||
"position_confidence": "seed"
|
||||
},
|
||||
"CVN-75": {
|
||||
"lat": 36.0,
|
||||
"lng": 15.0,
|
||||
"heading": 0,
|
||||
"desc": "Mediterranean Sea deployment (USNI Mar 9)",
|
||||
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||
"position_source_at": "2026-03-09T00:00:00Z",
|
||||
"position_confidence": "seed"
|
||||
},
|
||||
"CVN-77": {
|
||||
"lat": 36.5,
|
||||
"lng": -74.0,
|
||||
"heading": 0,
|
||||
"desc": "Atlantic — Pre-deployment workups (USNI Mar 9)",
|
||||
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||
"position_source_at": "2026-03-09T00:00:00Z",
|
||||
"position_confidence": "seed"
|
||||
},
|
||||
"CVN-70": {
|
||||
"lat": 32.6840,
|
||||
"lng": -117.1290,
|
||||
"heading": 180,
|
||||
"desc": "San Diego, CA (Homeport)",
|
||||
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||
"position_source_at": "2026-03-09T00:00:00Z",
|
||||
"position_confidence": "seed"
|
||||
},
|
||||
"CVN-71": {
|
||||
"lat": 32.6885,
|
||||
"lng": -117.1280,
|
||||
"heading": 180,
|
||||
"desc": "San Diego, CA (Maintenance)",
|
||||
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||
"position_source_at": "2026-03-09T00:00:00Z",
|
||||
"position_confidence": "seed"
|
||||
},
|
||||
"CVN-72": {
|
||||
"lat": 20.0,
|
||||
"lng": 64.0,
|
||||
"heading": 0,
|
||||
"desc": "Arabian Sea — Operation Epic Fury (USNI Mar 9)",
|
||||
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||
"position_source_at": "2026-03-09T00:00:00Z",
|
||||
"position_confidence": "seed"
|
||||
},
|
||||
"CVN-73": {
|
||||
"lat": 35.2830,
|
||||
"lng": 139.6700,
|
||||
"heading": 180,
|
||||
"desc": "Yokosuka, Japan (Forward deployed)",
|
||||
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||
"position_source_at": "2026-03-09T00:00:00Z",
|
||||
"position_confidence": "seed"
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"_comment": [
|
||||
"Baked-in SHA-256 digests for known Shadowbroker release archives.",
|
||||
"",
|
||||
"Issue #231: the self-updater previously skipped integrity verification",
|
||||
"entirely whenever the MESH_UPDATE_SHA256 env var was unset (which is the",
|
||||
"default — nothing in the install docs tells operators to set it). That",
|
||||
"made the auto-update a supply-chain RCE on any compromise of the GitHub",
|
||||
"release pipeline.",
|
||||
"",
|
||||
"The fix uses a multi-source verification chain mirroring the Tor bundle",
|
||||
"digest approach in #201:",
|
||||
"",
|
||||
" 1. MESH_UPDATE_SHA256 env var (operator override, preserved)",
|
||||
" 2. SHA256SUMS.txt asset published alongside each release (primary —",
|
||||
" the maintainer's release process already publishes this)",
|
||||
" 3. This baked-in digest list (second line of defense for releases",
|
||||
" missing a SHA256SUMS asset, or when the asset can't be fetched)",
|
||||
" 4. HTTPS-only fallback with a loud warning (preserves auto-update",
|
||||
" flow during transient outages so users don't get stuck)",
|
||||
"",
|
||||
"Mismatch from a source that DID respond is fatal — the update is",
|
||||
"refused and the existing install keeps running. Only the 'no source",
|
||||
"reachable at all' case falls back to HTTPS-only.",
|
||||
"",
|
||||
"Format: each entry is keyed by release tag and maps asset filenames",
|
||||
"to their canonical SHA-256 digest (hex, lowercase). The updater",
|
||||
"compares the locally-computed digest of the downloaded asset against",
|
||||
"the value here.",
|
||||
"",
|
||||
"When the maintainer ships a new release, add its digests here BEFORE",
|
||||
"removing the old ones so operators on the old code still validate",
|
||||
"against the previous entries during the transition."
|
||||
],
|
||||
"v0.9.79": {
|
||||
"ShadowBroker_v0.9.79.zip": "f6877c1d66614525315ea82636ce9f7b41178332c4dbf90d27431a1ea1d9cd47",
|
||||
"ShadowBroker_0.9.79_x64-setup.exe": "f7b676ada45cac7da05868b0a353678c9ee700e3abcf456a7c0c038c36da446f",
|
||||
"ShadowBroker_0.9.79_x64_en-US.msi": "e0713c3cdda184cfbea750bfac0d62a35678fec00847e6476f2cac8e7e42046e"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"_comment": [
|
||||
"Pinned SHA-256 digests for the Tor Expert Bundle archives we know how to install.",
|
||||
"Used as the LAST-RESORT verification source when the upstream .sha256sum file is",
|
||||
"unreachable, MITM'd, or doesn't match what we downloaded. Issue #201.",
|
||||
"",
|
||||
"Each entry is keyed by the archive URL (so multiple platforms / versions",
|
||||
"can share this one file) and contains the canonical SHA-256 we trust.",
|
||||
"",
|
||||
"When the project tests a new Tor release, add its digest here in the same",
|
||||
"PR that bumps _TOR_EXPERT_BUNDLE_URLS. Old entries are kept indefinitely so",
|
||||
"users on older versions keep working — we only ever ADD here, never remove."
|
||||
],
|
||||
"https://dist.torproject.org/torbrowser/15.0.11/tor-expert-bundle-windows-x86_64-15.0.11.tar.gz": "PLACEHOLDER_REPLACE_BEFORE_RELEASE",
|
||||
"https://dist.torproject.org/torbrowser/15.0.8/tor-expert-bundle-windows-x86_64-15.0.8.tar.gz": "PLACEHOLDER_REPLACE_BEFORE_RELEASE"
|
||||
}
|
||||
+79
-32
@@ -220,6 +220,7 @@ from services.mesh.mesh_crypto import (
|
||||
_derive_peer_key,
|
||||
derive_node_id,
|
||||
normalize_peer_url,
|
||||
resolve_peer_key_for_url,
|
||||
verify_node_binding,
|
||||
parse_public_key_algo,
|
||||
)
|
||||
@@ -1079,8 +1080,18 @@ def _public_mesh_log_size(entries: list[dict[str, Any]]) -> int:
|
||||
return sum(1 for item in entries if _public_mesh_log_entry(item) is not None)
|
||||
|
||||
|
||||
_WORMHOLE_PUBLIC_SETTINGS_FIELDS = {"enabled", "transport", "anonymous_mode"}
|
||||
_WORMHOLE_PUBLIC_PROFILE_FIELDS = {"profile", "wormhole_enabled"}
|
||||
# Issue #243 (tg12): the public redaction now exposes only the bare
|
||||
# "is Wormhole on?" boolean. Transport choice (tor/i2p/mixnet/direct),
|
||||
# anonymous-mode state, and the named privacy profile are all
|
||||
# operational posture and were leaking actionable recon to any
|
||||
# unauthenticated caller. They are now gated behind authenticated reads
|
||||
# (admin key or scoped-view token). Loopback Tauri shells and Docker
|
||||
# bridge frontend containers continue to see full status because the
|
||||
# Next.js catch-all proxy injects the configured ADMIN_KEY for
|
||||
# same-origin/non-browser callers (see PR #263), so legitimate operator
|
||||
# UX is unaffected.
|
||||
_WORMHOLE_PUBLIC_SETTINGS_FIELDS = {"enabled"}
|
||||
_WORMHOLE_PUBLIC_PROFILE_FIELDS = {"wormhole_enabled"}
|
||||
_PRIVATE_LANE_CONTROL_FIELDS = {"private_lane_tier", "private_lane_policy"}
|
||||
_PUBLIC_RNS_STATUS_FIELDS = {"enabled", "ready", "configured_peers", "active_peers"}
|
||||
_NODE_PUBLIC_EVENT_HOOK_REGISTERED = False
|
||||
@@ -1745,10 +1756,12 @@ def _http_peer_push_loop() -> None:
|
||||
_NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S)
|
||||
continue
|
||||
|
||||
secret = str(get_settings().MESH_PEER_PUSH_SECRET or "").strip()
|
||||
if not secret:
|
||||
_NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S)
|
||||
continue
|
||||
# Issue #256: resolve_peer_key_for_url() handles both the
|
||||
# legacy global MESH_PEER_PUSH_SECRET path and the per-peer
|
||||
# MESH_PEER_SECRETS map. The per-peer skip happens below
|
||||
# ("if not peer_key: continue"), so we don't gate the whole
|
||||
# loop on the global secret being set — an install that only
|
||||
# configures per-peer secrets is now valid.
|
||||
|
||||
peers = authenticated_push_peer_urls()
|
||||
if not peers:
|
||||
@@ -1778,7 +1791,7 @@ def _http_peer_push_loop() -> None:
|
||||
ensure_ascii=False,
|
||||
).encode("utf-8")
|
||||
|
||||
peer_key = _derive_peer_key(secret, normalized)
|
||||
peer_key = resolve_peer_key_for_url(normalized)
|
||||
if not peer_key:
|
||||
continue
|
||||
import hmac as _hmac_mod2
|
||||
@@ -1831,10 +1844,7 @@ def _http_gate_pull_loop() -> None:
|
||||
_NODE_SYNC_STOP.wait(_GATE_PULL_INTERVAL_S)
|
||||
continue
|
||||
|
||||
secret = str(get_settings().MESH_PEER_PUSH_SECRET or "").strip()
|
||||
if not secret:
|
||||
_NODE_SYNC_STOP.wait(_GATE_PULL_INTERVAL_S)
|
||||
continue
|
||||
# Issue #256: per-peer key resolution; see _http_peer_push_loop.
|
||||
|
||||
peers = authenticated_push_peer_urls()
|
||||
if not peers:
|
||||
@@ -1846,7 +1856,7 @@ def _http_gate_pull_loop() -> None:
|
||||
if not normalized:
|
||||
continue
|
||||
|
||||
peer_key = _derive_peer_key(secret, normalized)
|
||||
peer_key = resolve_peer_key_for_url(normalized)
|
||||
if not peer_key:
|
||||
continue
|
||||
|
||||
@@ -1959,10 +1969,7 @@ def _http_gate_push_loop() -> None:
|
||||
_NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S)
|
||||
continue
|
||||
|
||||
secret = str(get_settings().MESH_PEER_PUSH_SECRET or "").strip()
|
||||
if not secret:
|
||||
_NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S)
|
||||
continue
|
||||
# Issue #256: per-peer key resolution; see _http_peer_push_loop.
|
||||
|
||||
peers = authenticated_push_peer_urls()
|
||||
if not peers:
|
||||
@@ -1977,7 +1984,7 @@ def _http_gate_push_loop() -> None:
|
||||
if not normalized:
|
||||
continue
|
||||
|
||||
peer_key = _derive_peer_key(secret, normalized)
|
||||
peer_key = resolve_peer_key_for_url(normalized)
|
||||
if not peer_key:
|
||||
continue
|
||||
|
||||
@@ -3061,6 +3068,17 @@ def _request_private_surface_warmup(*, path: str, method: str, current_tier: str
|
||||
)
|
||||
|
||||
|
||||
def _is_invite_scoped_prekey_bundle_lookup(request: Request, path: str) -> bool:
|
||||
if request.method.upper() != "GET" or str(path or "").strip() != "/api/mesh/dm/prekey-bundle":
|
||||
return False
|
||||
try:
|
||||
lookup_token = str(request.query_params.get("lookup_token", "") or "").strip()
|
||||
agent_id = str(request.query_params.get("agent_id", "") or "").strip()
|
||||
except Exception:
|
||||
return False
|
||||
return bool(lookup_token) and not agent_id
|
||||
|
||||
|
||||
def _resume_private_delivery_background_work(*, current_tier: str, reason: str) -> None:
|
||||
pending_items = private_delivery_outbox.pending_items()
|
||||
if not pending_items:
|
||||
@@ -3191,6 +3209,17 @@ async def enforce_high_privacy_mesh(request: Request, call_next):
|
||||
# transport has not finished coming up yet.
|
||||
request.state._private_control_transport_pending = current_tier == "public_degraded"
|
||||
request.state._private_lane_current_tier = current_tier
|
||||
elif _is_invite_scoped_prekey_bundle_lookup(request, path):
|
||||
# A copied DM address carries a high-entropy invite lookup
|
||||
# handle. Returning the public prekey bundle for that
|
||||
# handle is the bootstrap step that lets first contact get
|
||||
# saved; blocking it behind the full private lane creates a
|
||||
# circular warm-up failure. Stable agent_id lookup still
|
||||
# follows the normal transport-tier policy.
|
||||
request.state._invite_prekey_lookup_transport_pending = (
|
||||
current_tier == "public_degraded"
|
||||
)
|
||||
request.state._private_lane_current_tier = current_tier
|
||||
else:
|
||||
# Tor-style: instead of failing, keep trying in the
|
||||
# background and return an ok:True "preparing" response
|
||||
@@ -3323,7 +3352,7 @@ async def force_refresh(request: Request):
|
||||
return {"status": "refreshing in background"}
|
||||
|
||||
|
||||
@app.post("/api/ais/feed")
|
||||
@app.post("/api/ais/feed", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("60/minute")
|
||||
async def ais_feed(request: Request):
|
||||
"""Accept AIS-catcher HTTP JSON feed (POST decoded AIS messages)."""
|
||||
@@ -3418,7 +3447,7 @@ class LayerUpdate(BaseModel):
|
||||
layers: dict[str, bool]
|
||||
|
||||
|
||||
@app.post("/api/layers")
|
||||
@app.post("/api/layers", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def update_layers(update: LayerUpdate, request: Request):
|
||||
"""Receive frontend layer toggle state. Starts/stops streams accordingly."""
|
||||
@@ -8791,9 +8820,14 @@ async def api_uw_flow(request: Request):
|
||||
from services.news_feed_config import get_feeds, save_feeds, reset_feeds
|
||||
|
||||
|
||||
@app.get("/api/settings/news-feeds")
|
||||
@app.get(
|
||||
"/api/settings/news-feeds",
|
||||
dependencies=[Depends(require_local_operator)],
|
||||
)
|
||||
@limiter.limit("30/minute")
|
||||
async def api_get_news_feeds(request: Request):
|
||||
"""Issue #252 (tg12): gated on local-operator. See the canonical
|
||||
handler in backend/routers/admin.py for the full rationale."""
|
||||
return get_feeds()
|
||||
|
||||
|
||||
@@ -8996,9 +9030,22 @@ class NodeSettingsUpdate(BaseModel):
|
||||
@app.get("/api/settings/node")
|
||||
@limiter.limit("30/minute")
|
||||
async def api_get_node_settings(request: Request):
|
||||
"""Issue #243 (tg12): node mode and participant state are
|
||||
operational posture. Anonymous callers receive an empty stub —
|
||||
enough for the UI to know the endpoint exists but nothing
|
||||
fingerprintable. Authenticated callers see the full state.
|
||||
|
||||
Authenticated == local-operator (loopback / Docker bridge) OR an
|
||||
admin / scoped-view token. The Tauri shell and Docker frontend
|
||||
container both qualify via their existing transport (PR #263 +
|
||||
PR #278), so legitimate operator UX is unchanged.
|
||||
"""
|
||||
from services.node_settings import read_node_settings
|
||||
|
||||
data = await asyncio.to_thread(read_node_settings)
|
||||
authenticated = _scoped_view_authenticated(request, "node")
|
||||
if not authenticated:
|
||||
return {}
|
||||
return {
|
||||
**data,
|
||||
"node_mode": _current_node_mode(),
|
||||
@@ -9812,7 +9859,7 @@ async def api_wormhole_leave(request: Request):
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/wormhole/identity")
|
||||
@app.get("/api/wormhole/identity", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_identity(request: Request):
|
||||
try:
|
||||
@@ -9825,7 +9872,7 @@ async def api_wormhole_identity(request: Request):
|
||||
raise HTTPException(status_code=500, detail="wormhole_identity_failed") from exc
|
||||
|
||||
|
||||
@app.post("/api/wormhole/identity/bootstrap")
|
||||
@app.post("/api/wormhole/identity/bootstrap", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("10/minute")
|
||||
async def api_wormhole_identity_bootstrap(request: Request):
|
||||
bootstrap_wormhole_identity()
|
||||
@@ -10605,7 +10652,7 @@ async def api_wormhole_sign(request: Request, body: WormholeSignRequest):
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/wormhole/gate/enter")
|
||||
@app.post("/api/wormhole/gate/enter", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("20/minute")
|
||||
async def api_wormhole_gate_enter(request: Request, body: WormholeGateRequest):
|
||||
gate_id = str(body.gate_id or "")
|
||||
@@ -10619,7 +10666,7 @@ async def api_wormhole_gate_enter(request: Request, body: WormholeGateRequest):
|
||||
return result
|
||||
|
||||
|
||||
@app.post("/api/wormhole/gate/leave")
|
||||
@app.post("/api/wormhole/gate/leave", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("20/minute")
|
||||
async def api_wormhole_gate_leave(request: Request, body: WormholeGateRequest):
|
||||
return leave_gate(str(body.gate_id or ""))
|
||||
@@ -10661,7 +10708,7 @@ async def api_wormhole_gate_key_rotate(request: Request, body: WormholeGateRotat
|
||||
return result
|
||||
|
||||
|
||||
@app.post("/api/wormhole/gate/persona/create")
|
||||
@app.post("/api/wormhole/gate/persona/create", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("20/minute")
|
||||
async def api_wormhole_gate_persona_create(
|
||||
request: Request, body: WormholeGatePersonaCreateRequest
|
||||
@@ -10677,7 +10724,7 @@ async def api_wormhole_gate_persona_create(
|
||||
return result
|
||||
|
||||
|
||||
@app.post("/api/wormhole/gate/persona/activate")
|
||||
@app.post("/api/wormhole/gate/persona/activate", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("20/minute")
|
||||
async def api_wormhole_gate_persona_activate(
|
||||
request: Request, body: WormholeGatePersonaActivateRequest
|
||||
@@ -10693,7 +10740,7 @@ async def api_wormhole_gate_persona_activate(
|
||||
return result
|
||||
|
||||
|
||||
@app.post("/api/wormhole/gate/persona/clear")
|
||||
@app.post("/api/wormhole/gate/persona/clear", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("20/minute")
|
||||
async def api_wormhole_gate_persona_clear(request: Request, body: WormholeGateRequest):
|
||||
gate_id = str(body.gate_id or "")
|
||||
@@ -10707,7 +10754,7 @@ async def api_wormhole_gate_persona_clear(request: Request, body: WormholeGateRe
|
||||
return result
|
||||
|
||||
|
||||
@app.post("/api/wormhole/gate/persona/retire")
|
||||
@app.post("/api/wormhole/gate/persona/retire", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("20/minute")
|
||||
async def api_wormhole_gate_persona_retire(
|
||||
request: Request, body: WormholeGatePersonaActivateRequest
|
||||
@@ -10788,7 +10835,7 @@ async def api_wormhole_gate_message_compose(request: Request, body: WormholeGate
|
||||
return composed
|
||||
|
||||
|
||||
@app.post("/api/wormhole/gate/message/sign-encrypted")
|
||||
@app.post("/api/wormhole/gate/message/sign-encrypted", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_gate_message_sign_encrypted(
|
||||
request: Request,
|
||||
@@ -11000,13 +11047,13 @@ async def api_wormhole_gate_messages_decrypt(request: Request, body: WormholeGat
|
||||
return {"ok": True, "results": results}
|
||||
|
||||
|
||||
@app.post("/api/wormhole/gate/state/export")
|
||||
@app.post("/api/wormhole/gate/state/export", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_gate_state_export(request: Request, body: WormholeGateRequest):
|
||||
return export_gate_state_snapshot_with_repair(str(body.gate_id or ""))
|
||||
|
||||
|
||||
@app.post("/api/wormhole/gate/proof")
|
||||
@app.post("/api/wormhole/gate/proof", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_gate_proof(request: Request, body: WormholeGateRequest):
|
||||
proof = _sign_gate_access_proof(str(body.gate_id or ""))
|
||||
@@ -11553,7 +11600,7 @@ async def api_wormhole_health(request: Request):
|
||||
return _redact_wormhole_status(full_state, authenticated=ok)
|
||||
|
||||
|
||||
@app.post("/api/wormhole/connect")
|
||||
@app.post("/api/wormhole/connect", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("10/minute")
|
||||
async def api_wormhole_connect(request: Request):
|
||||
settings = read_wormhole_settings()
|
||||
|
||||
@@ -15,6 +15,7 @@ dependencies = [
|
||||
"cachetools==5.5.2",
|
||||
"cloudscraper==1.2.71",
|
||||
"cryptography>=41.0.0",
|
||||
"defusedxml>=0.7.1",
|
||||
"fastapi==0.115.12",
|
||||
"feedparser==6.0.10",
|
||||
"httpx==0.28.1",
|
||||
|
||||
@@ -82,9 +82,18 @@ async def api_get_keys_meta(request: Request):
|
||||
return get_env_path_info()
|
||||
|
||||
|
||||
@router.get("/api/settings/news-feeds")
|
||||
@router.get(
|
||||
"/api/settings/news-feeds",
|
||||
dependencies=[Depends(require_local_operator)],
|
||||
)
|
||||
@limiter.limit("30/minute")
|
||||
async def api_get_news_feeds(request: Request):
|
||||
"""Issue #252 (tg12): the curated feed inventory is configuration
|
||||
state, not a public data feed. Gated on local-operator so the
|
||||
Tauri shell, the Docker bridge frontend, and any caller with an
|
||||
admin key all see the full list; anonymous LAN/internet callers
|
||||
can no longer enumerate operator source URLs.
|
||||
"""
|
||||
from services.news_feed_config import get_feeds
|
||||
return get_feeds()
|
||||
|
||||
@@ -118,9 +127,18 @@ async def api_reset_news_feeds(request: Request):
|
||||
@router.get("/api/settings/node")
|
||||
@limiter.limit("30/minute")
|
||||
async def api_get_node_settings(request: Request):
|
||||
"""Issue #243 (tg12): node_mode and node_enabled are operational
|
||||
posture. Anonymous callers receive an empty stub; authenticated
|
||||
callers (local-operator or admin/scoped token) see the full
|
||||
state. See the canonical handler in backend/main.py for the full
|
||||
rationale.
|
||||
"""
|
||||
import asyncio
|
||||
from auth import _scoped_view_authenticated
|
||||
from services.node_settings import read_node_settings
|
||||
data = await asyncio.to_thread(read_node_settings)
|
||||
if not _scoped_view_authenticated(request, "node"):
|
||||
return {}
|
||||
return {
|
||||
**data,
|
||||
"node_mode": _current_node_mode(),
|
||||
@@ -210,9 +228,19 @@ async def api_set_meshtastic_mqtt_settings(request: Request, body: MeshtasticMqt
|
||||
return _meshtastic_runtime_snapshot()
|
||||
|
||||
|
||||
@router.get("/api/settings/timemachine")
|
||||
@router.get(
|
||||
"/api/settings/timemachine",
|
||||
dependencies=[Depends(require_local_operator)],
|
||||
)
|
||||
@limiter.limit("30/minute")
|
||||
async def api_get_timemachine_settings(request: Request):
|
||||
"""Issue #253 (tg12): archival-capture posture is operationally
|
||||
sensitive — it tells a remote caller whether this deployment is
|
||||
retaining replayable historical surveillance data. Gated on
|
||||
local-operator so the Tauri shell and Docker bridge frontend
|
||||
still see the toggle state, but anonymous LAN/internet callers
|
||||
can no longer fingerprint Time Machine state.
|
||||
"""
|
||||
import asyncio
|
||||
from services.node_settings import read_node_settings
|
||||
data = await asyncio.to_thread(read_node_settings)
|
||||
|
||||
@@ -379,14 +379,13 @@ async def api_refresh_layer_feed(request: Request, layer_id: str):
|
||||
# Agent Actions endpoint — frontend polls this for UI commands from the agent
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/api/ai/agent-actions")
|
||||
@router.get("/api/ai/agent-actions", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("120/minute")
|
||||
async def get_agent_actions(request: Request):
|
||||
"""Frontend polls for pending agent display actions (destructive read).
|
||||
|
||||
No auth required — this only contains display directives (show image,
|
||||
fly to location), not sensitive data. The agent authenticates when
|
||||
pushing actions through the command channel.
|
||||
Local operator access is required because polling destructively drains
|
||||
the shared operator action queue.
|
||||
"""
|
||||
actions = pop_agent_actions()
|
||||
return {"ok": True, "actions": actions}
|
||||
|
||||
+58
-1
@@ -191,11 +191,68 @@ def _cctv_response_headers(resp, cache_seconds: int, include_length: bool = True
|
||||
return headers
|
||||
|
||||
|
||||
# Maximum number of redirects we'll follow on the CCTV upstream. Each hop is
|
||||
# re-validated against _cctv_host_allowed() before continuing, so this caps
|
||||
# the redirect-chain SSRF blast radius.
|
||||
_CCTV_MAX_REDIRECTS = 5
|
||||
|
||||
|
||||
def _fetch_cctv_upstream_response(request: Request, target_url: str, profile: _CCTVProxyProfile):
|
||||
"""Fetch an upstream CCTV URL, following redirects manually with host re-validation.
|
||||
|
||||
Why manual redirect following:
|
||||
The original code used ``allow_redirects=True``, which only validated
|
||||
the initial caller-supplied URL host against the allowlist. An attacker
|
||||
could submit an allowed host that 302-redirected to an internal address
|
||||
(e.g. ``http://localhost:8000/api/...`` or a private RFC1918 range),
|
||||
and the backend would dutifully follow and proxy the response — a
|
||||
classic open-redirect-to-SSRF chain.
|
||||
|
||||
With this loop, we re-run ``_cctv_host_allowed()`` on every hop's
|
||||
``Location`` header. A redirect to a host that isn't on the allowlist
|
||||
is rejected with 502 rather than silently followed.
|
||||
"""
|
||||
import requests as _req
|
||||
from urllib.parse import urlparse, urljoin
|
||||
|
||||
headers = _cctv_upstream_headers(request, profile)
|
||||
current_url = target_url
|
||||
hops = 0
|
||||
try:
|
||||
resp = _req.get(target_url, timeout=profile.timeout, stream=True, allow_redirects=True, headers=headers)
|
||||
while True:
|
||||
resp = _req.get(
|
||||
current_url,
|
||||
timeout=profile.timeout,
|
||||
stream=True,
|
||||
allow_redirects=False,
|
||||
headers=headers,
|
||||
)
|
||||
# Redirect handling — re-validate the next-hop host before following.
|
||||
if resp.is_redirect or resp.status_code in (301, 302, 303, 307, 308):
|
||||
location = resp.headers.get("Location", "")
|
||||
resp.close()
|
||||
if hops >= _CCTV_MAX_REDIRECTS:
|
||||
logger.warning(
|
||||
"CCTV upstream redirect chain exceeded limit [%s] %s",
|
||||
profile.name, target_url,
|
||||
)
|
||||
raise HTTPException(status_code=502, detail="Upstream redirect chain too long")
|
||||
if not location:
|
||||
raise HTTPException(status_code=502, detail="Upstream redirect missing Location")
|
||||
next_url = urljoin(current_url, location)
|
||||
next_parsed = urlparse(next_url)
|
||||
if next_parsed.scheme not in ("http", "https"):
|
||||
raise HTTPException(status_code=502, detail="Upstream redirect to non-HTTP scheme")
|
||||
if not _cctv_host_allowed(next_parsed.hostname):
|
||||
logger.warning(
|
||||
"CCTV upstream redirect to disallowed host [%s] %s -> %s",
|
||||
profile.name, current_url, next_url,
|
||||
)
|
||||
raise HTTPException(status_code=502, detail="Upstream redirect to disallowed host")
|
||||
current_url = next_url
|
||||
hops += 1
|
||||
continue
|
||||
break
|
||||
except _req.exceptions.Timeout as exc:
|
||||
logger.warning("CCTV upstream timeout [%s] %s", profile.name, target_url)
|
||||
raise HTTPException(status_code=504, detail="Upstream timeout") from exc
|
||||
|
||||
+30
-3
@@ -266,7 +266,7 @@ async def force_refresh(request: Request):
|
||||
return {"status": "refreshing in background"}
|
||||
|
||||
|
||||
@router.post("/api/ais/feed")
|
||||
@router.post("/api/ais/feed", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("60/minute")
|
||||
async def ais_feed(request: Request):
|
||||
"""Accept AIS-catcher HTTP JSON feed (POST decoded AIS messages)."""
|
||||
@@ -304,7 +304,7 @@ async def update_viewport(vp: ViewportUpdate, request: Request): # noqa: ARG001
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.post("/api/layers")
|
||||
@router.post("/api/layers", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def update_layers(update: LayerUpdate, request: Request):
|
||||
"""Receive frontend layer toggle state. Starts/stops streams accordingly."""
|
||||
@@ -611,6 +611,23 @@ class OverflightRequest(BaseModel):
|
||||
hours: int = 24
|
||||
|
||||
|
||||
# Issue #202: compute_overflights() is O(catalog_size × timesteps), where
|
||||
# timesteps grows linearly with `hours`. An unbounded `hours` value is a
|
||||
# trivial CPU-exhaustion vector. We clamp silently rather than raising 422 —
|
||||
# the response shape is unchanged, callers asking for too many hours just
|
||||
# get a shorter window, which is friendlier than a hostile error.
|
||||
#
|
||||
# Override via OVERFLIGHTS_MAX_HOURS env var if you legitimately need a
|
||||
# longer window (e.g. a planning use case that wants a full week).
|
||||
def _overflight_max_hours() -> int:
|
||||
import os as _os
|
||||
try:
|
||||
raw = int(str(_os.environ.get("OVERFLIGHTS_MAX_HOURS", "72")).strip())
|
||||
except (TypeError, ValueError):
|
||||
raw = 72
|
||||
return max(1, raw)
|
||||
|
||||
|
||||
@router.post("/api/satellites/overflights")
|
||||
@limiter.limit("10/minute")
|
||||
async def satellite_overflights(request: Request, body: OverflightRequest):
|
||||
@@ -619,5 +636,15 @@ async def satellite_overflights(request: Request, body: OverflightRequest):
|
||||
if not gp_data:
|
||||
return JSONResponse({"total": 0, "by_mission": {}, "satellites": [], "error": "No GP data cached yet"})
|
||||
bbox = {"s": body.s, "w": body.w, "n": body.n, "e": body.e}
|
||||
result = compute_overflights(gp_data, bbox, hours=body.hours)
|
||||
|
||||
# Silent clamp — see comment on _overflight_max_hours().
|
||||
requested_hours = max(1, int(body.hours or 0))
|
||||
effective_hours = min(requested_hours, _overflight_max_hours())
|
||||
|
||||
result = compute_overflights(gp_data, bbox, hours=effective_hours)
|
||||
# If we clamped, surface the effective window in the response so the
|
||||
# caller can detect it if they care, without it being an error.
|
||||
if isinstance(result, dict) and effective_hours != requested_hours:
|
||||
result.setdefault("requested_hours", requested_hours)
|
||||
result.setdefault("effective_hours", effective_hours)
|
||||
return JSONResponse(result)
|
||||
|
||||
@@ -54,6 +54,22 @@ async def health_check(request: Request):
|
||||
top_status = "error"
|
||||
elif slo_summary.get("yellow", 0) > 0:
|
||||
top_status = "degraded"
|
||||
|
||||
# Issue #258: surface AIS proxy degraded TLS state so operators can see
|
||||
# when the SPKI-pinned fallback is in effect. The data plane keeps
|
||||
# flowing (this is by design — see ais_proxy.js comments) but observers
|
||||
# who care about MITM-protection posture deserve a visible signal.
|
||||
ais_status: dict = {}
|
||||
try:
|
||||
from services.ais_stream import ais_proxy_status
|
||||
ais_status = ais_proxy_status() or {}
|
||||
except Exception:
|
||||
ais_status = {}
|
||||
if ais_status.get("degraded_tls") and top_status == "ok":
|
||||
# Don't override a worse top-level status if SLOs already failed,
|
||||
# but escalate ok -> degraded so the field surfaces in dashboards.
|
||||
top_status = "degraded"
|
||||
|
||||
return {
|
||||
"status": top_status,
|
||||
"version": _get_app_version(),
|
||||
@@ -76,6 +92,7 @@ async def health_check(request: Request):
|
||||
"uptime_seconds": round(_time_mod.time() - _get_start_time()),
|
||||
"slo": slo_statuses,
|
||||
"slo_summary": slo_summary,
|
||||
"ais_proxy": ais_status,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -223,11 +223,21 @@ async def oracle_markets_more(request: Request, category: str = "NEWS", offset:
|
||||
"has_more": offset + limit < len(cat_markets), "total": len(cat_markets)}
|
||||
|
||||
|
||||
@router.post("/api/mesh/oracle/resolve")
|
||||
@router.post(
|
||||
"/api/mesh/oracle/resolve",
|
||||
dependencies=[Depends(require_admin)],
|
||||
)
|
||||
@limiter.limit("5/minute")
|
||||
@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
|
||||
async def oracle_resolve(request: Request):
|
||||
"""Resolve a prediction market."""
|
||||
"""Resolve a prediction market.
|
||||
|
||||
Issue #240 (tg12): requires admin authentication. The
|
||||
``mesh_write_exempt`` decorator below is **metadata only** — it tags
|
||||
the route as not requiring a mesh signed-write envelope, it does
|
||||
NOT itself enforce caller authorization. The ``Depends(require_admin)``
|
||||
on the route decorator is what actually gates access.
|
||||
"""
|
||||
from services.mesh.mesh_oracle import oracle_ledger
|
||||
body = await request.json()
|
||||
market_title = body.get("market_title", "")
|
||||
@@ -327,11 +337,18 @@ async def oracle_predictions(request: Request, node_id: str = ""):
|
||||
active_predictions, authenticated=_scoped_view_authenticated(request, "mesh.audit"))
|
||||
|
||||
|
||||
@router.post("/api/mesh/oracle/resolve-stakes")
|
||||
@router.post(
|
||||
"/api/mesh/oracle/resolve-stakes",
|
||||
dependencies=[Depends(require_admin)],
|
||||
)
|
||||
@limiter.limit("5/minute")
|
||||
@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
|
||||
async def oracle_resolve_stakes(request: Request):
|
||||
"""Resolve all expired stake contests."""
|
||||
"""Resolve all expired stake contests.
|
||||
|
||||
Issue #241 (tg12): requires admin authentication. See the note on
|
||||
``oracle_resolve`` above — ``mesh_write_exempt`` is metadata only.
|
||||
"""
|
||||
from services.mesh.mesh_oracle import oracle_ledger
|
||||
resolutions = oracle_ledger.resolve_expired_stakes()
|
||||
return {"ok": True, "resolutions": resolutions, "count": len(resolutions)}
|
||||
|
||||
@@ -1467,25 +1467,37 @@ def _submit_gate_message_envelope(request: Request, gate_id: str, body: dict[str
|
||||
@router.get("/api/mesh/infonet/status")
|
||||
@limiter.limit("30/minute")
|
||||
async def infonet_status(request: Request, verify_signatures: bool = False):
|
||||
"""Get Infonet metadata — event counts, head hash, chain size."""
|
||||
"""Get Infonet metadata — event counts, head hash, chain size.
|
||||
|
||||
The ``verify_signatures`` query parameter is honored ONLY when the
|
||||
caller has authenticated via scoped auth or local-operator credentials.
|
||||
Verifying every signature in a long chain is O(n_events) work — letting
|
||||
anonymous callers trigger it is a DoS surface (issue #207). For
|
||||
anonymous callers we silently fall back to the cheap path; the response
|
||||
structure is identical so legitimate frontends see no behavior change.
|
||||
"""
|
||||
from services.mesh.mesh_hashchain import infonet
|
||||
from services.wormhole_supervisor import get_wormhole_state
|
||||
|
||||
# Silently downgrade for unauthenticated callers — no error surfaced.
|
||||
authenticated = _scoped_view_authenticated(request, "mesh.audit")
|
||||
effective_verify_signatures = bool(verify_signatures) and authenticated
|
||||
|
||||
info = infonet.get_info()
|
||||
valid, reason = infonet.validate_chain(verify_signatures=verify_signatures)
|
||||
valid, reason = infonet.validate_chain(verify_signatures=effective_verify_signatures)
|
||||
try:
|
||||
wormhole = get_wormhole_state()
|
||||
except Exception:
|
||||
wormhole = {"configured": False, "ready": False, "rns_ready": False}
|
||||
info["valid"] = valid
|
||||
info["validation"] = reason
|
||||
info["verify_signatures"] = verify_signatures
|
||||
info["verify_signatures"] = effective_verify_signatures
|
||||
info["private_lane_tier"] = _current_private_lane_tier(wormhole)
|
||||
info["private_lane_policy"] = _private_infonet_policy_snapshot()
|
||||
info.update(_node_runtime_snapshot())
|
||||
return _redact_private_lane_control_fields(
|
||||
info,
|
||||
authenticated=_scoped_view_authenticated(request, "mesh.audit"),
|
||||
authenticated=authenticated,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -21,14 +21,30 @@ async def api_get_openmhz_systems(request: Request):
|
||||
return get_openmhz_systems()
|
||||
|
||||
|
||||
@router.get("/api/radio/openmhz/calls/{sys_name}")
|
||||
# Issue #213: rotating sys_name bypasses the 20s TTL cache and lets an
|
||||
# anonymous caller hammer api.openmhz.com through this proxy, risking an
|
||||
# IP-ban for the project. require_local_operator scopes this to the local
|
||||
# UI (which goes through the Next.js proxy with admin-key injection) and
|
||||
# scoped agent tokens.
|
||||
@router.get(
|
||||
"/api/radio/openmhz/calls/{sys_name}",
|
||||
dependencies=[Depends(require_local_operator)],
|
||||
)
|
||||
@limiter.limit("60/minute")
|
||||
async def api_get_openmhz_calls(request: Request, sys_name: str):
|
||||
from services.radio_intercept import get_recent_openmhz_calls
|
||||
return get_recent_openmhz_calls(sys_name)
|
||||
|
||||
|
||||
@router.get("/api/radio/openmhz/audio")
|
||||
# Issue #214: this is a streaming bandwidth relay. An anonymous caller can
|
||||
# stream audio through the backend, saturating the operator's outbound
|
||||
# bandwidth. Scope to local operator; the legitimate browser UI still
|
||||
# works because relative /api/... paths go through the Next.js proxy
|
||||
# which injects the admin key automatically.
|
||||
@router.get(
|
||||
"/api/radio/openmhz/audio",
|
||||
dependencies=[Depends(require_local_operator)],
|
||||
)
|
||||
@limiter.limit("120/minute")
|
||||
async def api_get_openmhz_audio(request: Request, url: str = Query(..., min_length=10)):
|
||||
from services.radio_intercept import openmhz_audio_response
|
||||
|
||||
@@ -21,7 +21,7 @@ async def oracle_region_intel(
|
||||
return get_region_oracle_intel(lat, lng, news_items)
|
||||
|
||||
|
||||
@router.get("/api/thermal/verify")
|
||||
@router.get("/api/thermal/verify", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("10/minute")
|
||||
async def thermal_verify(
|
||||
request: Request,
|
||||
@@ -35,7 +35,7 @@ async def thermal_verify(
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/api/sigint/transmit")
|
||||
@router.post("/api/sigint/transmit", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("5/minute")
|
||||
async def sigint_transmit(request: Request):
|
||||
"""Send an APRS-IS message to a specific callsign. Requires ham radio credentials."""
|
||||
|
||||
@@ -120,7 +120,36 @@ async def api_sentinel_token(request: Request):
|
||||
raise HTTPException(502, "Token request failed")
|
||||
|
||||
|
||||
_sh_token_cache: dict = {"token": None, "expiry": 0, "client_id": ""}
|
||||
# Cache key is an HMAC of (client_id, client_secret) — a caller cannot hit
|
||||
# this cache without knowing the same secret that originally populated it.
|
||||
# Without this binding, the lookup only checked client_id, so anyone who
|
||||
# knew a valid client_id could reuse another caller's cached token (and
|
||||
# burn their Copernicus quota / access tiles on their account).
|
||||
_sh_token_cache: dict = {"token": None, "expiry": 0, "credential_fp": ""}
|
||||
|
||||
|
||||
def _credential_fingerprint(client_id: str, client_secret: str) -> str:
|
||||
"""Return a stable, secret-binding fingerprint for the Sentinel cache key.
|
||||
|
||||
Uses HMAC-SHA256 so the raw secret is never stored in process memory as
|
||||
a cache key. The HMAC key is a per-process random value, which means the
|
||||
fingerprint cannot be precomputed across restarts (additional defense
|
||||
against an attacker who learned a valid client_id but not the secret).
|
||||
"""
|
||||
import hashlib
|
||||
import hmac
|
||||
|
||||
return hmac.new(
|
||||
_SH_TOKEN_CACHE_HMAC_KEY,
|
||||
f"{client_id}\x00{client_secret}".encode("utf-8"),
|
||||
hashlib.sha256,
|
||||
).hexdigest()
|
||||
|
||||
|
||||
# Per-process random HMAC key. Regenerated on each backend startup so cached
|
||||
# fingerprints don't survive restarts.
|
||||
import os as _os
|
||||
_SH_TOKEN_CACHE_HMAC_KEY = _os.urandom(32)
|
||||
|
||||
|
||||
@router.post("/api/sentinel/tile")
|
||||
@@ -146,7 +175,9 @@ async def api_sentinel_tile(request: Request):
|
||||
raise HTTPException(400, "client_id, client_secret, and date required")
|
||||
|
||||
now = _time.time()
|
||||
if (_sh_token_cache["token"] and _sh_token_cache["client_id"] == client_id
|
||||
credential_fp = _credential_fingerprint(client_id, client_secret)
|
||||
if (_sh_token_cache["token"]
|
||||
and _sh_token_cache["credential_fp"] == credential_fp
|
||||
and now < _sh_token_cache["expiry"] - 30):
|
||||
token = _sh_token_cache["token"]
|
||||
else:
|
||||
@@ -161,7 +192,7 @@ async def api_sentinel_tile(request: Request):
|
||||
token = tdata["access_token"]
|
||||
_sh_token_cache["token"] = token
|
||||
_sh_token_cache["expiry"] = now + tdata.get("expires_in", 300)
|
||||
_sh_token_cache["client_id"] = client_id
|
||||
_sh_token_cache["credential_fp"] = credential_fp
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception:
|
||||
|
||||
+23
-18
@@ -160,8 +160,13 @@ router = APIRouter()
|
||||
|
||||
# --- Constants ---
|
||||
|
||||
_WORMHOLE_PUBLIC_SETTINGS_FIELDS = {"enabled", "transport", "anonymous_mode"}
|
||||
_WORMHOLE_PUBLIC_PROFILE_FIELDS = {"profile", "wormhole_enabled"}
|
||||
# Issue #243 (tg12): the public redaction now exposes only the bare
|
||||
# "is this on?" boolean. Transport choice, anonymous-mode state, and
|
||||
# the named privacy profile were all leaking actionable recon to
|
||||
# unauthenticated callers and are now gated behind authenticated reads.
|
||||
# See the matching block in backend/main.py for the full rationale.
|
||||
_WORMHOLE_PUBLIC_SETTINGS_FIELDS = {"enabled"}
|
||||
_WORMHOLE_PUBLIC_PROFILE_FIELDS = {"wormhole_enabled"}
|
||||
_PRIVATE_LANE_CONTROL_FIELDS = {"private_lane_tier", "private_lane_policy"}
|
||||
_PUBLIC_RNS_STATUS_FIELDS = {"enabled", "ready", "configured_peers", "active_peers"}
|
||||
_NODE_PUBLIC_EVENT_HOOK_REGISTERED = False
|
||||
@@ -589,7 +594,7 @@ async def api_get_wormhole_status(request: Request):
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/wormhole/join")
|
||||
@router.post("/api/wormhole/join", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("10/minute")
|
||||
async def api_wormhole_join(request: Request):
|
||||
from services.config import get_settings
|
||||
@@ -663,7 +668,7 @@ async def api_wormhole_leave(request: Request):
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/wormhole/identity")
|
||||
@router.get("/api/wormhole/identity", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("240/minute")
|
||||
async def api_wormhole_identity(request: Request):
|
||||
try:
|
||||
@@ -674,7 +679,7 @@ async def api_wormhole_identity(request: Request):
|
||||
raise HTTPException(status_code=500, detail="wormhole_identity_failed") from exc
|
||||
|
||||
|
||||
@router.post("/api/wormhole/identity/bootstrap")
|
||||
@router.post("/api/wormhole/identity/bootstrap", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("10/minute")
|
||||
async def api_wormhole_identity_bootstrap(request: Request):
|
||||
bootstrap_wormhole_identity()
|
||||
@@ -773,7 +778,7 @@ async def api_wormhole_sign(request: Request, body: WormholeSignRequest):
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/wormhole/gate/enter")
|
||||
@router.post("/api/wormhole/gate/enter", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("20/minute")
|
||||
async def api_wormhole_gate_enter(request: Request, body: WormholeGateRequest):
|
||||
gate_id = str(body.gate_id or "")
|
||||
@@ -787,25 +792,25 @@ async def api_wormhole_gate_enter(request: Request, body: WormholeGateRequest):
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/api/wormhole/gate/leave")
|
||||
@router.post("/api/wormhole/gate/leave", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("20/minute")
|
||||
async def api_wormhole_gate_leave(request: Request, body: WormholeGateRequest):
|
||||
return leave_gate(str(body.gate_id or ""))
|
||||
|
||||
|
||||
@router.get("/api/wormhole/gate/{gate_id}/identity")
|
||||
@router.get("/api/wormhole/gate/{gate_id}/identity", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_gate_identity(request: Request, gate_id: str):
|
||||
return get_active_gate_identity(gate_id)
|
||||
|
||||
|
||||
@router.get("/api/wormhole/gate/{gate_id}/personas")
|
||||
@router.get("/api/wormhole/gate/{gate_id}/personas", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_gate_personas(request: Request, gate_id: str):
|
||||
return list_gate_personas(gate_id)
|
||||
|
||||
|
||||
@router.get("/api/wormhole/gate/{gate_id}/key")
|
||||
@router.get("/api/wormhole/gate/{gate_id}/key", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_gate_key_status(request: Request, gate_id: str):
|
||||
import main as _m
|
||||
@@ -829,7 +834,7 @@ async def api_wormhole_gate_key_rotate(request: Request, body: WormholeGateRotat
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/api/wormhole/gate/persona/create")
|
||||
@router.post("/api/wormhole/gate/persona/create", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("20/minute")
|
||||
async def api_wormhole_gate_persona_create(
|
||||
request: Request, body: WormholeGatePersonaCreateRequest
|
||||
@@ -845,7 +850,7 @@ async def api_wormhole_gate_persona_create(
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/api/wormhole/gate/persona/activate")
|
||||
@router.post("/api/wormhole/gate/persona/activate", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("20/minute")
|
||||
async def api_wormhole_gate_persona_activate(
|
||||
request: Request, body: WormholeGatePersonaActivateRequest
|
||||
@@ -861,7 +866,7 @@ async def api_wormhole_gate_persona_activate(
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/api/wormhole/gate/persona/clear")
|
||||
@router.post("/api/wormhole/gate/persona/clear", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("20/minute")
|
||||
async def api_wormhole_gate_persona_clear(request: Request, body: WormholeGateRequest):
|
||||
gate_id = str(body.gate_id or "")
|
||||
@@ -875,7 +880,7 @@ async def api_wormhole_gate_persona_clear(request: Request, body: WormholeGateRe
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/api/wormhole/gate/persona/retire")
|
||||
@router.post("/api/wormhole/gate/persona/retire", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("20/minute")
|
||||
async def api_wormhole_gate_persona_retire(
|
||||
request: Request, body: WormholeGatePersonaActivateRequest
|
||||
@@ -944,7 +949,7 @@ async def api_wormhole_gate_message_compose(request: Request, body: WormholeGate
|
||||
return await _m.api_wormhole_gate_message_compose(request, body)
|
||||
|
||||
|
||||
@router.post("/api/wormhole/gate/message/sign-encrypted")
|
||||
@router.post("/api/wormhole/gate/message/sign-encrypted", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_gate_message_sign_encrypted(
|
||||
request: Request,
|
||||
@@ -1004,14 +1009,14 @@ async def api_wormhole_gate_messages_decrypt(request: Request, body: WormholeGat
|
||||
return await _m.api_wormhole_gate_messages_decrypt(request, body)
|
||||
|
||||
|
||||
@router.post("/api/wormhole/gate/state/export")
|
||||
@router.post("/api/wormhole/gate/state/export", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_gate_state_export(request: Request, body: WormholeGateRequest):
|
||||
import main as _m
|
||||
return await _m.api_wormhole_gate_state_export(request, body)
|
||||
|
||||
|
||||
@router.post("/api/wormhole/gate/proof")
|
||||
@router.post("/api/wormhole/gate/proof", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_gate_proof(request: Request, body: WormholeGateRequest):
|
||||
proof = _sign_gate_access_proof(str(body.gate_id or ""))
|
||||
@@ -1360,7 +1365,7 @@ async def api_wormhole_health(request: Request):
|
||||
return _redact_wormhole_status(full_state, authenticated=ok)
|
||||
|
||||
|
||||
@router.post("/api/wormhole/connect")
|
||||
@router.post("/api/wormhole/connect", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("10/minute")
|
||||
async def api_wormhole_connect(request: Request):
|
||||
settings = read_wormhole_settings()
|
||||
|
||||
@@ -344,9 +344,26 @@ _vessels_lock = threading.Lock()
|
||||
_ws_thread: threading.Thread | None = None
|
||||
_ws_running = False
|
||||
_proxy_process = None
|
||||
# Issue #258: latest status snapshot emitted by ais_proxy.js. Populated when
|
||||
# the proxy reports e.g. {"__ais_proxy_status": {"degraded_tls": true}} on
|
||||
# stdout, which it does when it falls back to the SPKI-pinned insecure-date
|
||||
# path during an upstream cert outage. Surfaced via ais_proxy_status() for
|
||||
# /api/health.
|
||||
_proxy_status: dict = {}
|
||||
_VESSEL_TRAIL_INTERVAL_S = 120
|
||||
_VESSEL_TRAIL_MAX_POINTS = 240
|
||||
|
||||
|
||||
def ais_proxy_status() -> dict:
|
||||
"""Return a copy of the latest ais_proxy.js status (issue #258).
|
||||
|
||||
Currently surfaces ``degraded_tls`` (bool) which is true when the
|
||||
proxy is using SPKI-pinned fallback because AISStream's cert expired.
|
||||
Returns an empty dict when no status has been received yet.
|
||||
"""
|
||||
with _vessels_lock:
|
||||
return dict(_proxy_status)
|
||||
|
||||
import os
|
||||
|
||||
CACHE_FILE = os.path.join(os.path.dirname(__file__), "ais_cache.json")
|
||||
@@ -608,6 +625,18 @@ def _ais_stream_loop():
|
||||
logger.error(f"AIS Stream error: {data['error']}")
|
||||
continue
|
||||
|
||||
# Issue #258: ais_proxy.js emits status markers (e.g.
|
||||
# {"__ais_proxy_status": {"degraded_tls": true}}) when the
|
||||
# SPKI-pinned fallback is in use. We snapshot the latest
|
||||
# status so the backend can expose it on /api/health.
|
||||
if isinstance(data, dict) and "__ais_proxy_status" in data:
|
||||
status = data.get("__ais_proxy_status") or {}
|
||||
if isinstance(status, dict):
|
||||
with _vessels_lock:
|
||||
_proxy_status.clear()
|
||||
_proxy_status.update(status)
|
||||
continue
|
||||
|
||||
msg_type = data.get("MessageType", "")
|
||||
metadata = data.get("MetaData", {})
|
||||
message = data.get("Message", {})
|
||||
|
||||
+371
-173
@@ -1,46 +1,90 @@
|
||||
"""
|
||||
Carrier Strike Group OSINT Tracker
|
||||
===================================
|
||||
Scrapes multiple OSINT sources to maintain current estimated positions
|
||||
for US Navy Carrier Strike Groups. Updates on startup + 00:00 & 12:00 UTC.
|
||||
Maintains estimated positions for US Navy Carrier Strike Groups with
|
||||
honest provenance and freshness signals.
|
||||
|
||||
Sources:
|
||||
1. GDELT News API — recent carrier movement headlines
|
||||
2. WikiVoyage / public port-call databases
|
||||
3. Fallback — last-known or static OSINT estimates
|
||||
Issues #244 / #245 / #246 (tg12 external audit):
|
||||
|
||||
The previous implementation baked a snapshot of USNI News Fleet &
|
||||
Marine Tracker positions (March 9, 2026) into the registry as
|
||||
``fallback_lat``/``fallback_lng`` and stamped ``updated = now()``
|
||||
every time the dossier was rendered. That presented stale editorial
|
||||
data as live state. It also persisted GDELT-derived positions to the
|
||||
on-disk cache with no freshness signal, so a single news mention from
|
||||
months ago could keep overriding the (already-stale) registry default
|
||||
indefinitely.
|
||||
|
||||
Architecture after this PR:
|
||||
|
||||
::
|
||||
|
||||
backend/data/carrier_seed.json read-only, shipped with image,
|
||||
used ONCE on first-ever startup
|
||||
to bootstrap carrier_cache.json.
|
||||
|
||||
backend/data/carrier_cache.json mutable, lives in the runtime data
|
||||
volume, written by every GDELT
|
||||
refresh + any future source.
|
||||
|
||||
Startup flow:
|
||||
|
||||
1. ``carrier_cache.json`` exists? → load it.
|
||||
2. Otherwise, copy ``carrier_seed.json`` → ``carrier_cache.json``,
|
||||
then load it. (This happens once, ever, per install.)
|
||||
3. Background: GDELT fetch runs. Any carrier mentioned in fresh news
|
||||
gets its entry replaced with the news-derived position.
|
||||
``position_source_at`` is set to the news article timestamp.
|
||||
|
||||
Freshness is a *labelling* decision, not an eviction decision:
|
||||
|
||||
- ``position_source_at`` within the configurable freshness window
|
||||
(default 14 days) → ``position_confidence = "recent"``.
|
||||
- Older than that → ``position_confidence = "stale"``.
|
||||
- Bootstrapped from the seed file (never updated) → ``"seed"``.
|
||||
- No cache entry at all (e.g. a carrier added to the registry after
|
||||
first install) → carrier renders at its homeport with
|
||||
``"homeport_default"``.
|
||||
|
||||
Carriers are never hidden, never teleported, never disappeared. The
|
||||
position the user sees is always the last position the system actually
|
||||
observed, with an honest "as-of" timestamp the UI can render however
|
||||
it likes. A year from now, the runtime cache reflects whatever this
|
||||
install has observed via GDELT — not the seed snapshot.
|
||||
"""
|
||||
|
||||
import re
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import logging
|
||||
import threading
|
||||
import random
|
||||
from datetime import datetime, timezone
|
||||
import shutil
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Carrier registry: hull number → metadata + fallback position
|
||||
# Carrier registry: hull number → identity only.
|
||||
#
|
||||
# Issue #244 (tg12): the previous registry carried hard-coded
|
||||
# ``fallback_lat``/``fallback_lng`` that were dated editorial
|
||||
# snapshots from a 2026-03-09 article. Those fields are DELETED. The
|
||||
# registry is now identity + homeport only; positions are sourced
|
||||
# exclusively from carrier_cache.json (and via that, from the
|
||||
# bootstrap seed or live OSINT).
|
||||
# -----------------------------------------------------------------
|
||||
CARRIER_REGISTRY: Dict[str, dict] = {
|
||||
# Fallback positions sourced from USNI News Fleet & Marine Tracker (Mar 9, 2026)
|
||||
# https://news.usni.org/2026/03/09/usni-news-fleet-and-marine-tracker-march-9-2026
|
||||
# --- Bremerton, WA (Naval Base Kitsap) ---
|
||||
# Distinct pier positions along Sinclair Inlet so carriers don't stack
|
||||
"CVN-68": {
|
||||
"name": "USS Nimitz (CVN-68)",
|
||||
"wiki": "https://en.wikipedia.org/wiki/USS_Nimitz",
|
||||
"homeport": "Bremerton, WA",
|
||||
"homeport_lat": 47.5535,
|
||||
"homeport_lng": -122.6400,
|
||||
"fallback_lat": 47.5535,
|
||||
"fallback_lng": -122.6400,
|
||||
"fallback_heading": 90,
|
||||
"fallback_desc": "Bremerton, WA (Maintenance)",
|
||||
},
|
||||
"CVN-76": {
|
||||
"name": "USS Ronald Reagan (CVN-76)",
|
||||
@@ -48,23 +92,14 @@ CARRIER_REGISTRY: Dict[str, dict] = {
|
||||
"homeport": "Bremerton, WA",
|
||||
"homeport_lat": 47.5580,
|
||||
"homeport_lng": -122.6360,
|
||||
"fallback_lat": 47.5580,
|
||||
"fallback_lng": -122.6360,
|
||||
"fallback_heading": 90,
|
||||
"fallback_desc": "Bremerton, WA (Decommissioning)",
|
||||
},
|
||||
# --- Norfolk, VA (Naval Station Norfolk) ---
|
||||
# Piers run N-S along Willoughby Bay; each carrier gets a distinct berth
|
||||
"CVN-69": {
|
||||
"name": "USS Dwight D. Eisenhower (CVN-69)",
|
||||
"wiki": "https://en.wikipedia.org/wiki/USS_Dwight_D._Eisenhower",
|
||||
"homeport": "Norfolk, VA",
|
||||
"homeport_lat": 36.9465,
|
||||
"homeport_lng": -76.3265,
|
||||
"fallback_lat": 36.9465,
|
||||
"fallback_lng": -76.3265,
|
||||
"fallback_heading": 0,
|
||||
"fallback_desc": "Norfolk, VA (Post-deployment maintenance)",
|
||||
},
|
||||
"CVN-78": {
|
||||
"name": "USS Gerald R. Ford (CVN-78)",
|
||||
@@ -72,10 +107,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
|
||||
"homeport": "Norfolk, VA",
|
||||
"homeport_lat": 36.9505,
|
||||
"homeport_lng": -76.3250,
|
||||
"fallback_lat": 18.0,
|
||||
"fallback_lng": 39.5,
|
||||
"fallback_heading": 0,
|
||||
"fallback_desc": "Red Sea — Operation Epic Fury (USNI Mar 9)",
|
||||
},
|
||||
"CVN-74": {
|
||||
"name": "USS John C. Stennis (CVN-74)",
|
||||
@@ -83,10 +114,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
|
||||
"homeport": "Norfolk, VA",
|
||||
"homeport_lat": 36.9540,
|
||||
"homeport_lng": -76.3235,
|
||||
"fallback_lat": 36.98,
|
||||
"fallback_lng": -76.43,
|
||||
"fallback_heading": 0,
|
||||
"fallback_desc": "Newport News, VA (RCOH refueling overhaul)",
|
||||
},
|
||||
"CVN-75": {
|
||||
"name": "USS Harry S. Truman (CVN-75)",
|
||||
@@ -94,10 +121,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
|
||||
"homeport": "Norfolk, VA",
|
||||
"homeport_lat": 36.9580,
|
||||
"homeport_lng": -76.3220,
|
||||
"fallback_lat": 36.0,
|
||||
"fallback_lng": 15.0,
|
||||
"fallback_heading": 0,
|
||||
"fallback_desc": "Mediterranean Sea deployment (USNI Mar 9)",
|
||||
},
|
||||
"CVN-77": {
|
||||
"name": "USS George H.W. Bush (CVN-77)",
|
||||
@@ -105,23 +128,14 @@ CARRIER_REGISTRY: Dict[str, dict] = {
|
||||
"homeport": "Norfolk, VA",
|
||||
"homeport_lat": 36.9620,
|
||||
"homeport_lng": -76.3210,
|
||||
"fallback_lat": 36.5,
|
||||
"fallback_lng": -74.0,
|
||||
"fallback_heading": 0,
|
||||
"fallback_desc": "Atlantic — Pre-deployment workups (USNI Mar 9)",
|
||||
},
|
||||
# --- San Diego, CA (Naval Base San Diego) ---
|
||||
# Carrier piers along the east shore of San Diego Bay, spread N-S
|
||||
"CVN-70": {
|
||||
"name": "USS Carl Vinson (CVN-70)",
|
||||
"wiki": "https://en.wikipedia.org/wiki/USS_Carl_Vinson",
|
||||
"homeport": "San Diego, CA",
|
||||
"homeport_lat": 32.6840,
|
||||
"homeport_lng": -117.1290,
|
||||
"fallback_lat": 32.6840,
|
||||
"fallback_lng": -117.1290,
|
||||
"fallback_heading": 180,
|
||||
"fallback_desc": "San Diego, CA (Homeport)",
|
||||
},
|
||||
"CVN-71": {
|
||||
"name": "USS Theodore Roosevelt (CVN-71)",
|
||||
@@ -129,10 +143,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
|
||||
"homeport": "San Diego, CA",
|
||||
"homeport_lat": 32.6885,
|
||||
"homeport_lng": -117.1280,
|
||||
"fallback_lat": 32.6885,
|
||||
"fallback_lng": -117.1280,
|
||||
"fallback_heading": 180,
|
||||
"fallback_desc": "San Diego, CA (Maintenance)",
|
||||
},
|
||||
"CVN-72": {
|
||||
"name": "USS Abraham Lincoln (CVN-72)",
|
||||
@@ -140,10 +150,6 @@ CARRIER_REGISTRY: Dict[str, dict] = {
|
||||
"homeport": "San Diego, CA",
|
||||
"homeport_lat": 32.6925,
|
||||
"homeport_lng": -117.1275,
|
||||
"fallback_lat": 20.0,
|
||||
"fallback_lng": 64.0,
|
||||
"fallback_heading": 0,
|
||||
"fallback_desc": "Arabian Sea — Operation Epic Fury (USNI Mar 9)",
|
||||
},
|
||||
# --- Yokosuka, Japan (CFAY) ---
|
||||
"CVN-73": {
|
||||
@@ -152,16 +158,18 @@ CARRIER_REGISTRY: Dict[str, dict] = {
|
||||
"homeport": "Yokosuka, Japan",
|
||||
"homeport_lat": 35.2830,
|
||||
"homeport_lng": 139.6700,
|
||||
"fallback_lat": 35.2830,
|
||||
"fallback_lng": 139.6700,
|
||||
"fallback_heading": 180,
|
||||
"fallback_desc": "Yokosuka, Japan (Forward deployed)",
|
||||
},
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Region → approximate center coordinates
|
||||
# Used to map textual geographic descriptions to lat/lng
|
||||
# Region → approximate center coordinates.
|
||||
#
|
||||
# Issue #245 (tg12): converting a region name straight into precise
|
||||
# map coordinates is false precision. We still use this table to
|
||||
# infer a coarse position from a headline mention, but the resulting
|
||||
# carrier object is now stamped ``position_confidence = "approximate"``
|
||||
# so the UI can render an uncertainty radius / dimmed icon. The
|
||||
# centroid is a best-effort midpoint of the named body of water.
|
||||
# -----------------------------------------------------------------
|
||||
REGION_COORDS: Dict[str, tuple] = {
|
||||
# Oceans & Seas
|
||||
@@ -220,9 +228,39 @@ REGION_COORDS: Dict[str, tuple] = {
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Cache file for persisting positions between restarts
|
||||
# Files
|
||||
# -----------------------------------------------------------------
|
||||
CACHE_FILE = Path(__file__).parent.parent / "carrier_cache.json"
|
||||
#
|
||||
# The seed lives in the read-only image data dir (it ships with each
|
||||
# release). The cache lives in the same data dir but is written at
|
||||
# runtime; under Docker compose this dir is volume-mounted so the
|
||||
# cache persists across container restarts, which is the whole point
|
||||
# of the seed-then-observe model — the user's runtime observations
|
||||
# survive image upgrades.
|
||||
SEED_FILE = Path(__file__).parent.parent / "data" / "carrier_seed.json"
|
||||
CACHE_FILE = Path(__file__).parent.parent / "data" / "carrier_cache.json"
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Freshness window for position_confidence labeling. Issue #246 (tg12):
|
||||
# previously persisted cache entries had no freshness signal at all.
|
||||
# After this change, the position itself is preserved (we never lose
|
||||
# what was last observed) but the confidence label flips from
|
||||
# "recent" to "stale" once the underlying source is older than this
|
||||
# window. Operator-overridable via env var.
|
||||
# -----------------------------------------------------------------
|
||||
_DEFAULT_FRESHNESS_WINDOW_DAYS = 14
|
||||
|
||||
|
||||
def _freshness_window_days() -> int:
|
||||
raw = str(os.environ.get("SHADOWBROKER_CARRIER_FRESHNESS_DAYS", "") or "").strip()
|
||||
if not raw:
|
||||
return _DEFAULT_FRESHNESS_WINDOW_DAYS
|
||||
try:
|
||||
n = int(raw)
|
||||
return n if n > 0 else _DEFAULT_FRESHNESS_WINDOW_DAYS
|
||||
except (TypeError, ValueError):
|
||||
return _DEFAULT_FRESHNESS_WINDOW_DAYS
|
||||
|
||||
|
||||
_carrier_positions: Dict[str, dict] = {}
|
||||
_positions_lock = threading.Lock()
|
||||
@@ -234,25 +272,159 @@ _GDELT_REQUEST_DELAY_SECONDS = 1.25
|
||||
_GDELT_REQUEST_JITTER_SECONDS = 0.35
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _parse_iso(ts: str) -> Optional[datetime]:
|
||||
if not ts:
|
||||
return None
|
||||
try:
|
||||
# Python's fromisoformat accepts +00:00 but not 'Z' until 3.11.
|
||||
normalized = ts.replace("Z", "+00:00")
|
||||
dt = datetime.fromisoformat(normalized)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _compute_position_confidence(entry: dict, *, now: Optional[datetime] = None) -> str:
|
||||
"""Return the public confidence label for a carrier cache entry.
|
||||
|
||||
Order of precedence:
|
||||
- explicit "homeport_default" / "seed" labels are preserved.
|
||||
- dated entries (with position_source_at) are "recent" if within
|
||||
the configured freshness window, else "stale".
|
||||
- missing position_source_at falls through to "stale".
|
||||
"""
|
||||
raw_label = str(entry.get("position_confidence", "") or "").strip()
|
||||
# Explicit "kind of provenance" labels are preserved as-is. They
|
||||
# describe HOW we got the position, not WHEN — a fresh headline-to-
|
||||
# centroid match (#245) is still imprecise no matter how recently
|
||||
# it was observed, and the seed (#244) is always the seed.
|
||||
if raw_label in {"seed", "homeport_default", "approximate"}:
|
||||
# Approximate entries can still age into "stale_approximate" if
|
||||
# they fall out of the freshness window — that distinction lets
|
||||
# the UI render a different badge for old-and-imprecise vs
|
||||
# recent-and-imprecise. seed/homeport_default never age (they
|
||||
# were never timestamped against real observations).
|
||||
if raw_label == "approximate":
|
||||
source_at = _parse_iso(str(entry.get("position_source_at", "") or ""))
|
||||
if source_at is not None:
|
||||
reference = now or datetime.now(timezone.utc)
|
||||
if reference - source_at > timedelta(days=_freshness_window_days()):
|
||||
return "stale_approximate"
|
||||
return raw_label
|
||||
|
||||
source_at = _parse_iso(str(entry.get("position_source_at", "") or ""))
|
||||
if not source_at:
|
||||
return "stale"
|
||||
|
||||
reference = now or datetime.now(timezone.utc)
|
||||
window = timedelta(days=_freshness_window_days())
|
||||
if reference - source_at <= window:
|
||||
return "recent"
|
||||
return "stale"
|
||||
|
||||
|
||||
def _load_seed() -> Dict[str, dict]:
|
||||
"""Load the read-only seed file shipped with the image.
|
||||
|
||||
Returns a hull→entry dict (no _meta wrapper). Missing or malformed
|
||||
seed files yield an empty dict — the caller falls back to homeport
|
||||
defaults.
|
||||
"""
|
||||
try:
|
||||
if not SEED_FILE.exists():
|
||||
logger.info("Carrier seed file not present at %s; first-run will fall back to homeport defaults", SEED_FILE)
|
||||
return {}
|
||||
raw = json.loads(SEED_FILE.read_text(encoding="utf-8"))
|
||||
carriers = raw.get("carriers", {}) if isinstance(raw, dict) else {}
|
||||
if not isinstance(carriers, dict):
|
||||
return {}
|
||||
logger.info("Carrier seed loaded: %d entries from %s", len(carriers), SEED_FILE)
|
||||
return carriers
|
||||
except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
|
||||
logger.warning("Failed to load carrier seed file %s: %s", SEED_FILE, e)
|
||||
return {}
|
||||
|
||||
|
||||
def _load_cache() -> Dict[str, dict]:
|
||||
"""Load cached carrier positions from disk."""
|
||||
"""Load the mutable cache (last-known positions persisted between restarts)."""
|
||||
try:
|
||||
if CACHE_FILE.exists():
|
||||
data = json.loads(CACHE_FILE.read_text())
|
||||
logger.info(f"Carrier cache loaded: {len(data)} carriers from {CACHE_FILE}")
|
||||
return data
|
||||
data = json.loads(CACHE_FILE.read_text(encoding="utf-8"))
|
||||
if isinstance(data, dict):
|
||||
logger.info("Carrier cache loaded: %d carriers from %s", len(data), CACHE_FILE)
|
||||
return data
|
||||
except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
|
||||
logger.warning(f"Failed to load carrier cache: {e}")
|
||||
logger.warning("Failed to load carrier cache: %s", e)
|
||||
return {}
|
||||
|
||||
|
||||
def _save_cache(positions: Dict[str, dict]):
|
||||
"""Persist carrier positions to disk."""
|
||||
def _save_cache(positions: Dict[str, dict]) -> None:
|
||||
"""Persist the mutable cache. Atomic write (temp + rename) so a crash
|
||||
mid-write can't leave the file truncated."""
|
||||
try:
|
||||
CACHE_FILE.write_text(json.dumps(positions, indent=2))
|
||||
logger.info(f"Carrier cache saved: {len(positions)} carriers")
|
||||
CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = CACHE_FILE.with_suffix(CACHE_FILE.suffix + ".tmp")
|
||||
tmp.write_text(json.dumps(positions, indent=2), encoding="utf-8")
|
||||
# On Windows os.replace is atomic and overwrites existing files.
|
||||
os.replace(tmp, CACHE_FILE)
|
||||
logger.info("Carrier cache saved: %d carriers", len(positions))
|
||||
except (IOError, OSError) as e:
|
||||
logger.warning(f"Failed to save carrier cache: {e}")
|
||||
logger.warning("Failed to save carrier cache: %s", e)
|
||||
|
||||
|
||||
def _homeport_entry_for(hull: str) -> Optional[dict]:
|
||||
"""Return a homeport-default cache entry for a hull, or None if the
|
||||
hull is not in the registry."""
|
||||
info = CARRIER_REGISTRY.get(hull)
|
||||
if not info:
|
||||
return None
|
||||
return {
|
||||
"lat": info["homeport_lat"],
|
||||
"lng": info["homeport_lng"],
|
||||
"heading": 0,
|
||||
"desc": f"{info['homeport']} (no observations yet)",
|
||||
"source": f"Homeport default ({info['homeport']})",
|
||||
"source_url": info.get("wiki", ""),
|
||||
"position_source_at": _now_iso(),
|
||||
"position_confidence": "homeport_default",
|
||||
}
|
||||
|
||||
|
||||
def _bootstrap_cache_if_missing() -> Dict[str, dict]:
|
||||
"""One-shot: if no cache exists, materialize one from the seed file.
|
||||
|
||||
Returns the cache contents (hull→entry). On first-ever startup,
|
||||
this writes ``carrier_cache.json`` so subsequent restarts skip the
|
||||
seed entirely. Operator-deleted caches re-bootstrap the same way —
|
||||
operators can use that to "reset" carrier positions, but it's an
|
||||
explicit operator action.
|
||||
"""
|
||||
if CACHE_FILE.exists():
|
||||
return _load_cache()
|
||||
|
||||
seed = _load_seed()
|
||||
if not seed:
|
||||
# No seed file either. Build a homeport-default cache so the
|
||||
# first save_cache call still produces something honest.
|
||||
homeports: Dict[str, dict] = {}
|
||||
for hull in CARRIER_REGISTRY:
|
||||
entry = _homeport_entry_for(hull)
|
||||
if entry is not None:
|
||||
homeports[hull] = entry
|
||||
if homeports:
|
||||
_save_cache(homeports)
|
||||
return homeports
|
||||
|
||||
# Persist the seed as the first cache so subsequent runs skip this branch.
|
||||
_save_cache(seed)
|
||||
logger.info("Carrier cache bootstrapped from seed (first-ever startup)")
|
||||
return dict(seed)
|
||||
|
||||
|
||||
def _match_region(text: str) -> Optional[tuple]:
|
||||
@@ -270,10 +442,8 @@ def _match_carrier(text: str) -> Optional[str]:
|
||||
for hull, info in CARRIER_REGISTRY.items():
|
||||
hull_check = hull.lower().replace("-", "")
|
||||
name_parts = info["name"].lower()
|
||||
# Match hull number (e.g., "CVN-78", "CVN78")
|
||||
if hull.lower() in text_lower or hull_check in text_lower.replace("-", ""):
|
||||
return hull
|
||||
# Match ship name (e.g., "Ford", "Eisenhower", "Vinson")
|
||||
ship_name = name_parts.split("(")[0].strip()
|
||||
last_name = ship_name.split()[-1] if ship_name else ""
|
||||
if last_name and len(last_name) > 3 and last_name in text_lower:
|
||||
@@ -323,8 +493,9 @@ def _fetch_gdelt_carrier_news() -> List[dict]:
|
||||
articles = data.get("articles", [])
|
||||
for art in articles:
|
||||
title = art.get("title", "")
|
||||
url = art.get("url", "")
|
||||
results.append({"title": title, "url": url})
|
||||
article_url = art.get("url", "")
|
||||
article_at = art.get("seendate") or art.get("date") or ""
|
||||
results.append({"title": title, "url": article_url, "seendate": article_at})
|
||||
except (ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
|
||||
logger.debug(f"GDELT search failed for '{term}': {e}")
|
||||
continue
|
||||
@@ -340,108 +511,139 @@ def _fetch_gdelt_carrier_news() -> List[dict]:
|
||||
return results
|
||||
|
||||
|
||||
def _gdelt_seendate_to_iso(seendate: str) -> Optional[str]:
|
||||
"""GDELT returns YYYYMMDDhhmmss (UTC). Convert to ISO8601 for
|
||||
position_source_at. Returns None if the input is unparseable."""
|
||||
raw = (seendate or "").strip()
|
||||
if len(raw) < 8 or not raw.isdigit():
|
||||
return None
|
||||
try:
|
||||
dt = datetime.strptime(raw[:14] if len(raw) >= 14 else raw[:8] + "000000", "%Y%m%d%H%M%S")
|
||||
return dt.replace(tzinfo=timezone.utc).isoformat()
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _parse_carrier_positions_from_news(articles: List[dict]) -> Dict[str, dict]:
|
||||
"""Parse carrier positions from news article titles and descriptions."""
|
||||
"""Parse carrier positions from news article titles.
|
||||
|
||||
Issue #245 (tg12): the position is a region centroid, which is
|
||||
coarse — we now stamp ``position_confidence = "approximate"`` so
|
||||
the UI can render that uncertainty. Issue #244: the
|
||||
``position_source_at`` field is the news article's actual seen
|
||||
date, NOT now(), so the freshness check correctly flips entries
|
||||
to "stale" once they age past the configured window.
|
||||
"""
|
||||
updates: Dict[str, dict] = {}
|
||||
|
||||
for article in articles:
|
||||
title = article.get("title", "")
|
||||
|
||||
# Try to match a carrier from the title
|
||||
hull = _match_carrier(title)
|
||||
if not hull:
|
||||
continue
|
||||
|
||||
# Try to match a region from the title
|
||||
coords = _match_region(title)
|
||||
if not coords:
|
||||
continue
|
||||
|
||||
# Only update if we haven't seen this carrier yet (first match wins — most recent)
|
||||
# First match wins (most recent article, GDELT returns newest first
|
||||
# per term).
|
||||
if hull not in updates:
|
||||
iso_at = _gdelt_seendate_to_iso(str(article.get("seendate", ""))) or _now_iso()
|
||||
updates[hull] = {
|
||||
"lat": coords[0],
|
||||
"lng": coords[1],
|
||||
"heading": 0,
|
||||
"desc": title[:100],
|
||||
"source": "GDELT News API",
|
||||
"source": "GDELT News API (headline region match — approximate)",
|
||||
"source_url": article.get("url", "https://api.gdeltproject.org"),
|
||||
"updated": datetime.now(timezone.utc).isoformat(),
|
||||
"position_source_at": iso_at,
|
||||
# Headline-to-centroid match is explicitly approximate.
|
||||
"position_confidence": "approximate",
|
||||
}
|
||||
logger.info(
|
||||
f"Carrier update: {CARRIER_REGISTRY[hull]['name']} → {coords} (from: {title[:80]})"
|
||||
"Carrier update: %s → %s (from: %s)",
|
||||
CARRIER_REGISTRY[hull]["name"],
|
||||
coords,
|
||||
title[:80],
|
||||
)
|
||||
|
||||
return updates
|
||||
|
||||
|
||||
def _load_carrier_fallbacks() -> Dict[str, dict]:
|
||||
"""Build carrier positions from static fallbacks + disk cache (instant, no network)."""
|
||||
positions: Dict[str, dict] = {}
|
||||
for hull, info in CARRIER_REGISTRY.items():
|
||||
positions[hull] = {
|
||||
"name": info["name"],
|
||||
"lat": info["fallback_lat"],
|
||||
"lng": info["fallback_lng"],
|
||||
"heading": info["fallback_heading"],
|
||||
"desc": info["fallback_desc"],
|
||||
"wiki": info["wiki"],
|
||||
"source": "USNI News Fleet & Marine Tracker",
|
||||
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||
"updated": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
# Overlay cached positions from previous runs (may have GDELT data)
|
||||
cached = _load_cache()
|
||||
for hull, cached_pos in cached.items():
|
||||
if hull in positions:
|
||||
if cached_pos.get("source", "").startswith("GDELT") or cached_pos.get(
|
||||
"source", ""
|
||||
).startswith("News"):
|
||||
positions[hull].update(
|
||||
{
|
||||
"lat": cached_pos["lat"],
|
||||
"lng": cached_pos["lng"],
|
||||
"desc": cached_pos.get("desc", positions[hull]["desc"]),
|
||||
"source": cached_pos.get("source", "Cached OSINT"),
|
||||
"updated": cached_pos.get("updated", ""),
|
||||
}
|
||||
)
|
||||
return positions
|
||||
def _enrich_for_rendering(hull: str, entry: dict, *, now: Optional[datetime] = None) -> dict:
|
||||
"""Add live computed fields (confidence label, last_osint_update)
|
||||
on top of the persisted cache entry. The persisted entry is left
|
||||
untouched; this function builds the public-facing object.
|
||||
"""
|
||||
info = CARRIER_REGISTRY.get(hull, {})
|
||||
confidence = _compute_position_confidence(entry, now=now)
|
||||
return {
|
||||
"name": entry.get("name", info.get("name", hull)),
|
||||
"lat": entry["lat"],
|
||||
"lng": entry["lng"],
|
||||
"heading": entry.get("heading", 0),
|
||||
"desc": entry.get("desc", ""),
|
||||
"wiki": entry.get("wiki", info.get("wiki", "")),
|
||||
"source": entry.get("source", "OSINT estimated position"),
|
||||
"source_url": entry.get("source_url", ""),
|
||||
"position_source_at": entry.get("position_source_at", ""),
|
||||
"position_confidence": confidence,
|
||||
# Existing field preserved for backward compatibility with the
|
||||
# current frontend ShipPopup; now reflects the SOURCE's observed
|
||||
# time (not now()), so "last reported X days ago" is honest.
|
||||
"last_osint_update": entry.get("position_source_at", ""),
|
||||
# Convenience boolean for the UI: true when the position is
|
||||
# NOT live OSINT (used to render dimmed icons / badges).
|
||||
"is_fallback": confidence in {"seed", "stale", "stale_approximate", "homeport_default"},
|
||||
}
|
||||
|
||||
|
||||
def update_carrier_positions():
|
||||
"""Main update function — called on startup and every 12h.
|
||||
def update_carrier_positions() -> None:
|
||||
"""Refresh carrier positions.
|
||||
|
||||
Phase 1 (instant): publish fallback + cached positions so the map has carriers immediately.
|
||||
Phase 2 (slow): query GDELT for fresh OSINT positions and update in-place.
|
||||
Phase 1 (instant): publish whatever's in carrier_cache.json (or
|
||||
bootstrap from seed on first-ever run), so the map has carriers
|
||||
immediately.
|
||||
|
||||
Phase 2 (slow): query GDELT and replace position entries for any
|
||||
carrier mentioned in fresh news. Persist back to cache.
|
||||
"""
|
||||
global _last_update
|
||||
|
||||
# --- Phase 1: instant fallback + cache ---
|
||||
positions = _load_carrier_fallbacks()
|
||||
# --- Phase 1: instant cache (bootstrap from seed on first-ever run) ---
|
||||
positions = _bootstrap_cache_if_missing()
|
||||
|
||||
# Ensure every registered hull has SOMETHING in the cache. A hull
|
||||
# the seed didn't cover (e.g. added after install) renders at its
|
||||
# homeport with "homeport_default" confidence.
|
||||
for hull in CARRIER_REGISTRY:
|
||||
if hull not in positions:
|
||||
entry = _homeport_entry_for(hull)
|
||||
if entry is not None:
|
||||
positions[hull] = entry
|
||||
|
||||
with _positions_lock:
|
||||
# Only overwrite if positions are currently empty (first startup).
|
||||
# If we already have data from a previous cycle, keep it while GDELT runs.
|
||||
if not _carrier_positions:
|
||||
_carrier_positions.update(positions)
|
||||
_last_update = datetime.now(timezone.utc)
|
||||
logger.info(
|
||||
f"Carrier tracker: {len(positions)} carriers loaded from fallback/cache (GDELT enrichment starting...)"
|
||||
"Carrier tracker: %d carriers loaded from cache (GDELT enrichment starting...)",
|
||||
len(positions),
|
||||
)
|
||||
|
||||
# --- Phase 2: slow GDELT enrichment ---
|
||||
# --- Phase 2: GDELT enrichment ---
|
||||
try:
|
||||
articles = _fetch_gdelt_carrier_news()
|
||||
news_positions = _parse_carrier_positions_from_news(articles)
|
||||
for hull, pos in news_positions.items():
|
||||
if hull in positions:
|
||||
positions[hull].update(pos)
|
||||
logger.info(f"Carrier OSINT: updated {CARRIER_REGISTRY[hull]['name']} from news")
|
||||
# Always overwrite — newest GDELT mention wins. The previous
|
||||
# entry's position is preserved in git history and the next
|
||||
# cycle either confirms or replaces it.
|
||||
positions[hull] = pos
|
||||
logger.info("Carrier OSINT: updated %s from news", CARRIER_REGISTRY[hull]["name"])
|
||||
except (ValueError, KeyError, json.JSONDecodeError, OSError) as e:
|
||||
logger.warning(f"GDELT carrier fetch failed: {e}")
|
||||
logger.warning("GDELT carrier fetch failed: %s", e)
|
||||
|
||||
# Save and update the global state with enriched positions
|
||||
with _positions_lock:
|
||||
_carrier_positions.clear()
|
||||
_carrier_positions.update(positions)
|
||||
@@ -449,21 +651,15 @@ def update_carrier_positions():
|
||||
|
||||
_save_cache(positions)
|
||||
|
||||
sources = {}
|
||||
for p in positions.values():
|
||||
src = p.get("source", "unknown")
|
||||
sources[src] = sources.get(src, 0) + 1
|
||||
logger.info(f"Carrier tracker: {len(positions)} carriers updated. Sources: {sources}")
|
||||
confidences: Dict[str, int] = {}
|
||||
for entry in positions.values():
|
||||
label = _compute_position_confidence(entry)
|
||||
confidences[label] = confidences.get(label, 0) + 1
|
||||
logger.info("Carrier tracker: %d carriers updated. Confidence: %s", len(positions), confidences)
|
||||
|
||||
|
||||
def _deconflict_positions(result: List[dict]) -> List[dict]:
|
||||
"""Offset carriers that share identical coordinates so they don't stack.
|
||||
|
||||
At port: offset along the pier axis (~500m / 0.004° apart).
|
||||
At sea: offset perpendicular to each other (~0.08° / ~9km apart)
|
||||
so they're visibly separate but clearly operating together.
|
||||
"""
|
||||
# Group by rounded lat/lng (within ~0.01° ≈ 1km = same spot)
|
||||
"""Offset carriers that share identical coordinates so they don't stack."""
|
||||
from collections import defaultdict
|
||||
|
||||
groups: dict[str, list[int]] = defaultdict(list)
|
||||
@@ -475,7 +671,6 @@ def _deconflict_positions(result: List[dict]) -> List[dict]:
|
||||
if len(indices) < 2:
|
||||
continue
|
||||
n = len(indices)
|
||||
# Determine if this is a port (near a homeport) or at sea
|
||||
sample = result[indices[0]]
|
||||
at_port = any(
|
||||
abs(sample["lat"] - info.get("homeport_lat", 0)) < 0.05
|
||||
@@ -484,7 +679,6 @@ def _deconflict_positions(result: List[dict]) -> List[dict]:
|
||||
)
|
||||
|
||||
if at_port:
|
||||
# Use each carrier's distinct homeport pier coordinates
|
||||
for idx in indices:
|
||||
carrier = result[idx]
|
||||
hull = None
|
||||
@@ -497,8 +691,7 @@ def _deconflict_positions(result: List[dict]) -> List[dict]:
|
||||
carrier["lat"] = info["homeport_lat"]
|
||||
carrier["lng"] = info["homeport_lng"]
|
||||
else:
|
||||
# At sea: spread in a line perpendicular to travel (~0.08° apart)
|
||||
spacing = 0.08 # ~9km — close enough to see they're together
|
||||
spacing = 0.08
|
||||
start_offset = -(n - 1) * spacing / 2
|
||||
for j, idx in enumerate(indices):
|
||||
result[idx]["lng"] += start_offset + j * spacing
|
||||
@@ -507,36 +700,44 @@ def _deconflict_positions(result: List[dict]) -> List[dict]:
|
||||
|
||||
|
||||
def get_carrier_positions() -> List[dict]:
|
||||
"""Return current carrier positions for the data pipeline."""
|
||||
"""Return current carrier positions for the data pipeline.
|
||||
|
||||
Each entry has the full provenance + freshness fields; the UI can
|
||||
decide how to render them. Carriers are never hidden — only
|
||||
labeled.
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
with _positions_lock:
|
||||
result = []
|
||||
for hull, pos in _carrier_positions.items():
|
||||
info = CARRIER_REGISTRY.get(hull, {})
|
||||
result: List[dict] = []
|
||||
for hull, entry in _carrier_positions.items():
|
||||
enriched = _enrich_for_rendering(hull, entry, now=now)
|
||||
result.append(
|
||||
{
|
||||
"name": pos.get("name", info.get("name", hull)),
|
||||
"name": enriched["name"],
|
||||
"type": "carrier",
|
||||
"lat": pos["lat"],
|
||||
"lng": pos["lng"],
|
||||
"heading": None, # Heading unknown for carriers — OSINT cannot determine true heading
|
||||
"lat": enriched["lat"],
|
||||
"lng": enriched["lng"],
|
||||
"heading": None, # OSINT cannot determine true heading.
|
||||
"sog": 0,
|
||||
"cog": 0,
|
||||
"country": "United States",
|
||||
"desc": pos.get("desc", ""),
|
||||
"wiki": pos.get("wiki", info.get("wiki", "")),
|
||||
"desc": enriched["desc"],
|
||||
"wiki": enriched["wiki"],
|
||||
"estimated": True,
|
||||
"source": pos.get("source", "OSINT estimated position"),
|
||||
"source_url": pos.get(
|
||||
"source_url", "https://news.usni.org/category/fleet-tracker"
|
||||
),
|
||||
"last_osint_update": pos.get("updated", ""),
|
||||
"source": enriched["source"],
|
||||
"source_url": enriched["source_url"],
|
||||
"last_osint_update": enriched["last_osint_update"],
|
||||
# New fields (additive — existing UI continues to work):
|
||||
"position_source_at": enriched["position_source_at"],
|
||||
"position_confidence": enriched["position_confidence"],
|
||||
"is_fallback": enriched["is_fallback"],
|
||||
}
|
||||
)
|
||||
return _deconflict_positions(result)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Scheduler: runs at startup, then at 00:00 and 12:00 UTC daily
|
||||
# Scheduler: runs at startup, then at 00:00 and 12:00 UTC daily.
|
||||
# -----------------------------------------------------------------
|
||||
_scheduler_thread: Optional[threading.Thread] = None
|
||||
_scheduler_stop = threading.Event()
|
||||
@@ -544,7 +745,6 @@ _scheduler_stop = threading.Event()
|
||||
|
||||
def _scheduler_loop():
|
||||
"""Background thread that triggers updates at 00:00 and 12:00 UTC."""
|
||||
# Initial update on startup
|
||||
try:
|
||||
update_carrier_positions()
|
||||
except Exception as e:
|
||||
@@ -552,7 +752,6 @@ def _scheduler_loop():
|
||||
|
||||
while not _scheduler_stop.is_set():
|
||||
now = datetime.now(timezone.utc)
|
||||
# Next target: 00:00 or 12:00 UTC, whichever is sooner
|
||||
hour = now.hour
|
||||
if hour < 12:
|
||||
next_hour = 12
|
||||
@@ -561,18 +760,17 @@ def _scheduler_loop():
|
||||
|
||||
next_run = now.replace(hour=next_hour % 24, minute=0, second=0, microsecond=0)
|
||||
if next_hour == 24:
|
||||
from datetime import timedelta
|
||||
|
||||
next_run = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
wait_seconds = (next_run - now).total_seconds()
|
||||
logger.info(
|
||||
f"Carrier tracker: next update at {next_run.isoformat()} ({wait_seconds/3600:.1f}h)"
|
||||
"Carrier tracker: next update at %s (%.1fh)",
|
||||
next_run.isoformat(),
|
||||
wait_seconds / 3600,
|
||||
)
|
||||
|
||||
# Wait until next scheduled time, or until stop event
|
||||
if _scheduler_stop.wait(timeout=wait_seconds):
|
||||
break # Stop event was set
|
||||
break
|
||||
|
||||
try:
|
||||
update_carrier_positions()
|
||||
|
||||
@@ -987,7 +987,7 @@ _KML_NS = {"kml": "http://www.opengis.net/kml/2.2"}
|
||||
|
||||
def _find_kml_element(element, tag):
|
||||
"""Find first descendant matching tag, ignoring XML namespace prefix."""
|
||||
import xml.etree.ElementTree as ET
|
||||
import defusedxml.ElementTree as ET
|
||||
el = element.find(f".//{tag}")
|
||||
if el is not None:
|
||||
return el
|
||||
@@ -1015,7 +1015,7 @@ class MadridCityIngestor(BaseCCTVIngestor):
|
||||
KML_URL = "http://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml"
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
import xml.etree.ElementTree as ET
|
||||
import defusedxml.ElementTree as ET
|
||||
|
||||
try:
|
||||
response = fetch_with_curl(self.KML_URL, timeout=20)
|
||||
|
||||
@@ -53,6 +53,12 @@ class Settings(BaseSettings):
|
||||
MESH_RELAY_FAILURE_COOLDOWN_S: int = 120
|
||||
MESH_BOOTSTRAP_SEED_FAILURE_COOLDOWN_S: int = 15
|
||||
MESH_PEER_PUSH_SECRET: str = ""
|
||||
# Issue #256 (tg12): optional per-peer HMAC secret map. Comma-separated
|
||||
# `url=secret` pairs. When a peer URL appears here, only that per-peer
|
||||
# secret is accepted for it — the global MESH_PEER_PUSH_SECRET above is
|
||||
# ignored for that specific URL. Single-peer installs and unmigrated
|
||||
# multi-peer installs leave this empty and behavior is unchanged.
|
||||
MESH_PEER_SECRETS: str = ""
|
||||
MESH_RNS_APP_NAME: str = "shadowbroker"
|
||||
MESH_RNS_ASPECT: str = "infonet"
|
||||
MESH_RNS_IDENTITY_PATH: str = ""
|
||||
|
||||
@@ -318,7 +318,7 @@ active_layers: dict[str, bool] = {
|
||||
"uap_sightings": True,
|
||||
"wastewater": True,
|
||||
"ai_intel": True,
|
||||
"crowdthreat": True,
|
||||
"crowdthreat": False,
|
||||
"sar": True,
|
||||
}
|
||||
|
||||
|
||||
@@ -16,9 +16,9 @@ import csv
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from typing import Any
|
||||
|
||||
import defusedxml.ElementTree as ET
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -31,11 +31,7 @@ _S3_NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
|
||||
_REFRESH_INTERVAL_S = 5 * 24 * 3600
|
||||
_LIST_TIMEOUT_S = 30
|
||||
_DOWNLOAD_TIMEOUT_S = 600
|
||||
_USER_AGENT = (
|
||||
"ShadowBroker-OSINT/0.9.79 "
|
||||
"(+https://github.com/BigBodyCobain/Shadowbroker; "
|
||||
"contact: bigbodycobain@gmail.com)"
|
||||
)
|
||||
from services.network_utils import DEFAULT_USER_AGENT as _USER_AGENT
|
||||
|
||||
_lock = threading.RLock()
|
||||
_aircraft_by_hex: dict[str, dict[str, str]] = {}
|
||||
|
||||
@@ -7,6 +7,7 @@ No API key required — the /threats endpoint is unauthenticated.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh, is_any_active
|
||||
@@ -16,6 +17,16 @@ logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
_CT_BASE = "https://backend.crowdthreat.world"
|
||||
|
||||
|
||||
def crowdthreat_fetch_enabled() -> bool:
|
||||
"""Return True only when the operator explicitly opts into CrowdThreat pulls."""
|
||||
return str(os.environ.get("CROWDTHREAT_ENABLED", "")).strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
}
|
||||
|
||||
# CrowdThreat category_id → icon ID used on the MapLibre layer
|
||||
_CATEGORY_ICON = {
|
||||
1: "ct-security", # Security & Conflict (red)
|
||||
@@ -43,6 +54,12 @@ _CATEGORY_COLOUR = {
|
||||
@with_retry(max_retries=2, base_delay=5)
|
||||
def fetch_crowdthreat():
|
||||
"""Fetch verified threat reports from CrowdThreat public API."""
|
||||
if not crowdthreat_fetch_enabled():
|
||||
logger.debug("CrowdThreat fetch skipped; set CROWDTHREAT_ENABLED=true to opt in")
|
||||
with _data_lock:
|
||||
latest_data["crowdthreat"] = []
|
||||
_mark_fresh("crowdthreat")
|
||||
return
|
||||
if not is_any_active("crowdthreat"):
|
||||
return
|
||||
|
||||
|
||||
@@ -279,9 +279,13 @@ def fetch_weather_alerts():
|
||||
return
|
||||
alerts = []
|
||||
try:
|
||||
# weather.gov requires a User-Agent per their API policy, but it
|
||||
# need not identify the operator. Use a project-generic string and
|
||||
# let the user override via SHADOWBROKER_USER_AGENT if needed.
|
||||
from services.network_utils import DEFAULT_USER_AGENT
|
||||
url = "https://api.weather.gov/alerts/active?status=actual"
|
||||
headers = {
|
||||
"User-Agent": "(ShadowBroker OSINT Dashboard, github.com/BigBodyCobain/Shadowbroker)",
|
||||
"User-Agent": DEFAULT_USER_AGENT,
|
||||
"Accept": "application/geo+json",
|
||||
}
|
||||
response = fetch_with_curl(url, timeout=15, headers=headers)
|
||||
|
||||
@@ -5,6 +5,7 @@ debunked claims, threat actor mentions, and target country references.
|
||||
Refreshes every 12 hours (FIMI data updates weekly).
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
@@ -18,6 +19,16 @@ logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
_FIMI_FEED_URL = "https://euvsdisinfo.eu/feed/"
|
||||
|
||||
|
||||
def fimi_fetch_enabled() -> bool:
|
||||
"""Return True only when the operator explicitly opts into FIMI pulls."""
|
||||
return str(os.environ.get("FIMI_ENABLED", "")).strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
}
|
||||
|
||||
# ── Threat actor keywords ──────────────────────────────────────────────────
|
||||
# Map of keyword → canonical actor name. Checked case-insensitively.
|
||||
_THREAT_ACTORS: dict[str, str] = {
|
||||
@@ -173,6 +184,12 @@ def _is_major_wave(narratives: list[dict], targets: dict[str, int]) -> bool:
|
||||
@with_retry(max_retries=1, base_delay=5)
|
||||
def fetch_fimi():
|
||||
"""Fetch and parse the EUvsDisinfo RSS feed."""
|
||||
if not fimi_fetch_enabled():
|
||||
logger.debug("FIMI fetch skipped; set FIMI_ENABLED=true to opt in")
|
||||
with _data_lock:
|
||||
latest_data["fimi"] = []
|
||||
_mark_fresh("fimi")
|
||||
return
|
||||
try:
|
||||
resp = fetch_with_curl(_FIMI_FEED_URL, timeout=15)
|
||||
feed = feedparser.parse(resp.text)
|
||||
|
||||
@@ -82,10 +82,37 @@ def _fetch_yfinance_single(symbol: str, period: str = "2d"):
|
||||
|
||||
|
||||
@with_retry(max_retries=1, base_delay=1)
|
||||
def financial_fetch_enabled() -> bool:
|
||||
"""Return True only when the operator explicitly opts into financial pulls.
|
||||
|
||||
Either ``FINANCIAL_ENABLED=true`` or the presence of ``FINNHUB_API_KEY``
|
||||
counts as an explicit opt-in. Without either, the default yfinance path
|
||||
is disabled to avoid silent outbound calls to finance.yahoo.com.
|
||||
"""
|
||||
if os.getenv("FINNHUB_API_KEY", "").strip():
|
||||
return True
|
||||
return str(os.environ.get("FINANCIAL_ENABLED", "")).strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
}
|
||||
|
||||
|
||||
def fetch_financial_markets():
|
||||
"""Fetches full market list with smart throttling (3s for Finnhub, 60s for yfinance)."""
|
||||
global _last_fetch_time, _last_fetch_results, _rotating_index
|
||||
|
||||
|
||||
if not financial_fetch_enabled():
|
||||
logger.debug(
|
||||
"Financial fetch skipped; set FINANCIAL_ENABLED=true or supply "
|
||||
"FINNHUB_API_KEY to opt in"
|
||||
)
|
||||
with _data_lock:
|
||||
latest_data["financial"] = {}
|
||||
_mark_fresh("financial")
|
||||
return
|
||||
|
||||
finnhub_key = os.getenv("FINNHUB_API_KEY", "").strip()
|
||||
use_finnhub = bool(finnhub_key)
|
||||
|
||||
|
||||
@@ -174,16 +174,29 @@ def fetch_meshtastic_nodes():
|
||||
except Exception as e:
|
||||
logger.debug(f"Meshtastic cache freshness check failed: {e}")
|
||||
|
||||
# Build a polite User-Agent. Include the operator callsign when set so
|
||||
# the upstream service can correlate per-install traffic if needed.
|
||||
# Build a polite User-Agent. Historically this included the operator
|
||||
# callsign so meshtastic.org could rate-limit per-install; that's still
|
||||
# the default behavior for backward compatibility. Operators who want
|
||||
# stricter outbound privacy can suppress the callsign by setting
|
||||
# MESHTASTIC_SEND_CALLSIGN_HEADER=false. Issue #203.
|
||||
import os as _os
|
||||
try:
|
||||
from services.config import get_settings
|
||||
|
||||
callsign = str(getattr(get_settings(), "MESHTASTIC_OPERATOR_CALLSIGN", "") or "").strip()
|
||||
except Exception:
|
||||
callsign = ""
|
||||
ua_base = "ShadowBroker-OSINT/0.9.79 (+https://github.com/BigBodyCobain/Shadowbroker; contact: bigbodycobain@gmail.com; 24h polling)"
|
||||
user_agent = f"{ua_base}; node={callsign}" if callsign else ua_base
|
||||
|
||||
send_callsign_header = str(
|
||||
_os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "true")
|
||||
).strip().lower() not in {"0", "false", "no", "off", ""}
|
||||
|
||||
from services.network_utils import DEFAULT_USER_AGENT
|
||||
ua_base = f"{DEFAULT_USER_AGENT}; 24h polling"
|
||||
if callsign and send_callsign_header:
|
||||
user_agent = f"{ua_base}; node={callsign}"
|
||||
else:
|
||||
user_agent = ua_base
|
||||
|
||||
try:
|
||||
logger.info("Fetching Meshtastic map nodes from API...")
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""News fetching, geocoding, clustering, and risk assessment."""
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import logging
|
||||
@@ -11,6 +12,22 @@ from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
from services.fetchers.retry import with_retry
|
||||
from services.oracle_service import enrich_news_items, compute_global_threat_level, detect_breaking_events
|
||||
|
||||
|
||||
def news_fetch_enabled() -> bool:
|
||||
"""Return True only when the operator explicitly opts into news RSS pulls.
|
||||
|
||||
Defaults to **on** for backward compatibility (this is the only fetcher
|
||||
where opting out is the new behavior, not the old one). Set
|
||||
``NEWS_ENABLED=false`` to disable all outbound RSS feed traffic.
|
||||
"""
|
||||
return str(os.environ.get("NEWS_ENABLED", "true")).strip().lower() not in {
|
||||
"0",
|
||||
"false",
|
||||
"no",
|
||||
"off",
|
||||
"",
|
||||
}
|
||||
|
||||
logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
# Maximum article age in seconds. Anything older than this is dropped
|
||||
@@ -160,6 +177,12 @@ def _resolve_coords(text: str) -> tuple[float, float] | None:
|
||||
|
||||
@with_retry(max_retries=1, base_delay=2)
|
||||
def fetch_news():
|
||||
if not news_fetch_enabled():
|
||||
logger.debug("News fetch skipped; unset NEWS_ENABLED=false to re-enable")
|
||||
with _data_lock:
|
||||
latest_data["news"] = []
|
||||
_mark_fresh("news")
|
||||
return
|
||||
from services.news_feed_config import get_feeds
|
||||
feed_config = get_feeds()
|
||||
feeds = {f["name"]: f["url"] for f in feed_config}
|
||||
|
||||
@@ -49,6 +49,16 @@ _HF_CSV_URL = (
|
||||
"https://huggingface.co/datasets/kcimc/NUFORC/resolve/main/nuforc_str.csv"
|
||||
)
|
||||
|
||||
|
||||
def nuforc_fetch_enabled() -> bool:
|
||||
"""Return True only when the operator explicitly opts into NUFORC pulls."""
|
||||
return str(os.environ.get("NUFORC_ENABLED", "")).strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
}
|
||||
|
||||
# Only keep sightings from the last N years for the enrichment index
|
||||
_KEEP_YEARS = 5
|
||||
|
||||
@@ -160,6 +170,12 @@ def _download_and_build() -> dict | None:
|
||||
|
||||
Returns the index dict or None on failure.
|
||||
"""
|
||||
if not nuforc_fetch_enabled():
|
||||
logger.debug(
|
||||
"NUFORC enrichment skipped; set NUFORC_ENABLED=true to opt in"
|
||||
)
|
||||
return None
|
||||
|
||||
cutoff = datetime.utcnow() - timedelta(days=_KEEP_YEARS * 365)
|
||||
cutoff_str = cutoff.strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
@@ -25,6 +25,16 @@ _provider_pace_lock = threading.Lock()
|
||||
_provider_last_request_at: dict[str, float] = {}
|
||||
|
||||
|
||||
def prediction_markets_fetch_enabled() -> bool:
|
||||
"""Return True only when the operator explicitly opts into Polymarket/Kalshi pulls."""
|
||||
return str(os.environ.get("PREDICTION_MARKETS_ENABLED", "")).strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
}
|
||||
|
||||
|
||||
def _pace_provider(provider: str, min_interval_s: float) -> None:
|
||||
if min_interval_s <= 0:
|
||||
return
|
||||
@@ -755,6 +765,16 @@ def fetch_prediction_markets():
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
global _prev_probabilities
|
||||
|
||||
if not prediction_markets_fetch_enabled():
|
||||
logger.debug(
|
||||
"Prediction markets fetch skipped; set "
|
||||
"PREDICTION_MARKETS_ENABLED=true to opt in"
|
||||
)
|
||||
with _data_lock:
|
||||
latest_data["prediction_markets"] = []
|
||||
_mark_fresh("prediction_markets")
|
||||
return
|
||||
|
||||
markets = fetch_prediction_markets_raw()
|
||||
|
||||
# Compute probability deltas vs previous fetch
|
||||
|
||||
@@ -24,11 +24,7 @@ _AIRPORTS_URL = "https://vrs-standing-data.adsb.lol/airports.csv.gz"
|
||||
_REFRESH_INTERVAL_S = 5 * 24 * 3600
|
||||
_HTTP_TIMEOUT_S = 60
|
||||
|
||||
_USER_AGENT = (
|
||||
"ShadowBroker-OSINT/0.9.79 "
|
||||
"(+https://github.com/BigBodyCobain/Shadowbroker; "
|
||||
"contact: bigbodycobain@gmail.com)"
|
||||
)
|
||||
from services.network_utils import DEFAULT_USER_AGENT as _USER_AGENT
|
||||
|
||||
_lock = threading.RLock()
|
||||
_routes_by_callsign: dict[str, dict[str, Any]] = {}
|
||||
|
||||
@@ -616,9 +616,12 @@ def fetch_global_military_incidents():
|
||||
try:
|
||||
logger.info("Fetching GDELT events via export CDN (multi-file)...")
|
||||
|
||||
# Get the latest export URL to determine current timestamp
|
||||
# Get the latest export URL to determine current timestamp.
|
||||
# HTTPS is used to prevent passive network observers from injecting
|
||||
# poisoned export records into the global incident map via MITM.
|
||||
# GDELT serves the same content over HTTPS as HTTP.
|
||||
index_res = fetch_with_curl(
|
||||
"http://data.gdeltproject.org/gdeltv2/lastupdate.txt", timeout=10
|
||||
"https://data.gdeltproject.org/gdeltv2/lastupdate.txt", timeout=10
|
||||
)
|
||||
if index_res.status_code != 200:
|
||||
logger.error(f"GDELT lastupdate failed: {index_res.status_code}")
|
||||
@@ -636,7 +639,9 @@ def fetch_global_military_incidents():
|
||||
logger.error("Could not find GDELT export URL")
|
||||
return []
|
||||
|
||||
# Extract timestamp from URL like: http://data.gdeltproject.org/gdeltv2/20260301120000.export.CSV.zip
|
||||
# Extract timestamp from URL like: https://data.gdeltproject.org/gdeltv2/20260301120000.export.CSV.zip
|
||||
# (GDELT's lastupdate.txt may still list URLs with http:// — we ignore
|
||||
# the scheme there and reconstruct each download URL as https:// below.)
|
||||
import re
|
||||
|
||||
ts_match = re.search(r"(\d{14})\.export\.CSV\.zip", latest_url)
|
||||
@@ -652,7 +657,7 @@ def fetch_global_military_incidents():
|
||||
for i in range(NUM_FILES):
|
||||
ts = latest_ts - timedelta(minutes=15 * i)
|
||||
fname = ts.strftime("%Y%m%d%H%M%S") + ".export.CSV.zip"
|
||||
url = f"http://data.gdeltproject.org/gdeltv2/{fname}"
|
||||
url = f"https://data.gdeltproject.org/gdeltv2/{fname}"
|
||||
urls.append(url)
|
||||
|
||||
logger.info(f"Downloading {len(urls)} GDELT export files...")
|
||||
|
||||
@@ -34,6 +34,20 @@ kiwisdr_cache: TTLCache = TTLCache(maxsize=1, ttl=_REFRESH_SECONDS)
|
||||
|
||||
_SOURCE_URL = "http://rx.linkfanel.net/kiwisdr_com.js"
|
||||
_CACHE_FILE = Path(__file__).resolve().parent.parent / "data" / "kiwisdr_cache.json"
|
||||
# Bundled fallback — shipped with the codebase so the KiwiSDR layer always
|
||||
# has something to render even when the upstream is unreachable, returns
|
||||
# garbage, or appears to have been tampered with. Issue #206: the upstream
|
||||
# only speaks HTTP, so we can't rely on TLS for integrity — instead we
|
||||
# validate the response's shape and fall back to this bundle if it doesn't
|
||||
# look right.
|
||||
_BUNDLED_FALLBACK = Path(__file__).resolve().parent.parent / "data" / "kiwisdr_directory.json"
|
||||
|
||||
# Minimum number of receivers we expect from a healthy upstream response.
|
||||
# The KiwiSDR public network has consistently sat well above this threshold
|
||||
# for years. If we see fewer than this many parsed receivers, treat the
|
||||
# response as suspect and fall back. Tune via env if the upstream shrinks
|
||||
# legitimately.
|
||||
_MIN_HEALTHY_RECEIVER_COUNT = 50
|
||||
_LINE_COMMENT_RE = re.compile(r"^\s*//.*$", re.MULTILINE)
|
||||
_VAR_PREFIX_RE = re.compile(r"^\s*var\s+kiwisdr_com\s*=\s*", re.MULTILINE)
|
||||
_TRAILING_COMMA_RE = re.compile(r",(\s*[\]}])")
|
||||
@@ -135,12 +149,72 @@ def _parse_mirror_payload(body: str) -> list[dict]:
|
||||
return nodes
|
||||
|
||||
|
||||
def _validate_fetched_nodes(nodes: list[dict]) -> bool:
|
||||
"""Sanity-check freshly-fetched receiver data before trusting it.
|
||||
|
||||
The upstream (rx.linkfanel.net) speaks only HTTP — there is no TLS to
|
||||
authenticate the response. A passive MITM could inject doctored
|
||||
receiver positions (false pins on the map) or strip the response down
|
||||
to a tiny subset. We can't prevent the modification at the transport
|
||||
layer, but we can refuse to commit to obviously-bad responses.
|
||||
|
||||
Returns True if the parsed list looks reasonable. False means we
|
||||
should fall back to a previously-cached or bundled directory.
|
||||
"""
|
||||
if not isinstance(nodes, list):
|
||||
return False
|
||||
if len(nodes) < _MIN_HEALTHY_RECEIVER_COUNT:
|
||||
# Either upstream is degraded or someone is feeding us a stripped
|
||||
# response. Either way, the bundled fallback is more useful.
|
||||
return False
|
||||
|
||||
# Spot-check: every entry should have a name, a parsed lat/lon, and a
|
||||
# URL field. If more than 5% of entries are missing core fields, the
|
||||
# parse went sideways.
|
||||
missing_core = 0
|
||||
for entry in nodes:
|
||||
if not isinstance(entry, dict):
|
||||
missing_core += 1
|
||||
continue
|
||||
if not entry.get("name") or not isinstance(entry.get("lat"), (int, float)):
|
||||
missing_core += 1
|
||||
if missing_core > max(5, len(nodes) // 20):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _load_bundled_fallback() -> list[dict]:
|
||||
"""Last-resort directory shipped with the codebase. Always returns a
|
||||
list (may be empty if the bundle is missing in older deployments)."""
|
||||
if not _BUNDLED_FALLBACK.exists():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(_BUNDLED_FALLBACK.read_text(encoding="utf-8"))
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.warning(f"KiwiSDR bundled fallback unreadable: {e}")
|
||||
return []
|
||||
|
||||
|
||||
@cached(kiwisdr_cache)
|
||||
def fetch_kiwisdr_nodes() -> list[dict]:
|
||||
"""Return the KiwiSDR receiver list, refreshed at most once per day.
|
||||
|
||||
Order of preference: in-memory cache (handled by @cached) → on-disk cache
|
||||
if <24h old → network fetch from rx.linkfanel.net.
|
||||
Layered fallback (issue #206 — upstream is HTTP-only, so we defend with
|
||||
content validation + bundled static directory rather than trying to
|
||||
upgrade the transport):
|
||||
|
||||
1. In-memory cache (handled by @cached on this function)
|
||||
2. On-disk cache if <24h old
|
||||
3. Fresh network fetch from rx.linkfanel.net → validated → committed
|
||||
4. Stale on-disk cache (>24h) if validation fails
|
||||
5. Bundled static directory at backend/data/kiwisdr_directory.json
|
||||
|
||||
The KiwiSDR map layer renders something useful in every case. A
|
||||
tampered upstream returning garbage is caught by _validate_fetched_nodes()
|
||||
and falls through to whatever previously-trusted snapshot we have.
|
||||
"""
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
@@ -153,34 +227,57 @@ def fetch_kiwisdr_nodes() -> list[dict]:
|
||||
return cached_nodes
|
||||
|
||||
# 2. Cache cold or stale — fetch from network.
|
||||
fresh_nodes: list[dict] = []
|
||||
fetch_succeeded = False
|
||||
try:
|
||||
res = fetch_with_curl(_SOURCE_URL, timeout=20)
|
||||
if not res or res.status_code != 200:
|
||||
logger.error(
|
||||
f"KiwiSDR fetch failed: HTTP {res.status_code if res else 'no response'}"
|
||||
if res and res.status_code == 200:
|
||||
fresh_nodes = _parse_mirror_payload(res.text)
|
||||
fetch_succeeded = True
|
||||
else:
|
||||
logger.warning(
|
||||
f"KiwiSDR fetch returned HTTP {res.status_code if res else 'no response'}"
|
||||
)
|
||||
return []
|
||||
|
||||
nodes = _parse_mirror_payload(res.text)
|
||||
if nodes:
|
||||
_save_disk_cache(nodes)
|
||||
logger.info(
|
||||
f"KiwiSDR: refreshed {len(nodes)} receivers from rx.linkfanel.net "
|
||||
"(next refresh in 24h)"
|
||||
)
|
||||
return nodes
|
||||
|
||||
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
|
||||
logger.error(f"KiwiSDR fetch exception: {e}")
|
||||
# Fall back to a stale disk cache if one exists, even if >24h old.
|
||||
if _CACHE_FILE.exists():
|
||||
try:
|
||||
stale = json.loads(_CACHE_FILE.read_text(encoding="utf-8"))
|
||||
if isinstance(stale, list):
|
||||
logger.info(
|
||||
f"KiwiSDR: serving {len(stale)} stale receivers from disk after fetch failure"
|
||||
)
|
||||
return stale
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
logger.warning(f"KiwiSDR fetch exception: {e}")
|
||||
|
||||
# 3. Validate before committing. If the response looks healthy, save
|
||||
# it as the new cache and return.
|
||||
if fetch_succeeded and _validate_fetched_nodes(fresh_nodes):
|
||||
_save_disk_cache(fresh_nodes)
|
||||
logger.info(
|
||||
f"KiwiSDR: refreshed {len(fresh_nodes)} receivers from rx.linkfanel.net "
|
||||
"(next refresh in 24h)"
|
||||
)
|
||||
return fresh_nodes
|
||||
|
||||
if fetch_succeeded:
|
||||
# Network came back, but the payload didn't pass validation —
|
||||
# either upstream is degraded or a MITM is at work. Fall through
|
||||
# to a trusted snapshot rather than committing garbage to disk.
|
||||
logger.warning(
|
||||
"KiwiSDR: upstream response failed validation (%d entries) — "
|
||||
"falling back to trusted snapshot",
|
||||
len(fresh_nodes),
|
||||
)
|
||||
|
||||
# 4. Stale on-disk cache, if any.
|
||||
if _CACHE_FILE.exists():
|
||||
try:
|
||||
stale = json.loads(_CACHE_FILE.read_text(encoding="utf-8"))
|
||||
if isinstance(stale, list) and stale:
|
||||
logger.info(
|
||||
f"KiwiSDR: serving {len(stale)} stale receivers from disk"
|
||||
)
|
||||
return stale
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 5. Bundled static directory — last resort, always works.
|
||||
bundled = _load_bundled_fallback()
|
||||
if bundled:
|
||||
logger.info(
|
||||
f"KiwiSDR: serving {len(bundled)} receivers from bundled fallback "
|
||||
"(no fresh fetch + no disk cache available)"
|
||||
)
|
||||
return bundled
|
||||
|
||||
@@ -69,6 +69,115 @@ def _derive_peer_key(shared_secret: str, peer_url: str) -> bytes:
|
||||
).digest()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Issue #256 (tg12): per-peer HMAC secrets
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# Before this change, ALL peer-push HMACs were derived from a single
|
||||
# fleet-shared ``MESH_PEER_PUSH_SECRET``. The receiver could prove a
|
||||
# request was signed by *someone who knows the fleet secret*, but it
|
||||
# could NOT prove which peer signed it — any peer could compute the
|
||||
# expected HMAC for any other peer's URL and impersonate that peer.
|
||||
#
|
||||
# Fix: an optional ``MESH_PEER_SECRETS`` env var maps specific peer URLs
|
||||
# to per-peer secrets. When a peer URL is listed there, only that
|
||||
# per-peer secret is accepted for that URL — the global secret is
|
||||
# ignored for that peer. Peer A no longer learns peer B's secret, so
|
||||
# peer A cannot forge a request claiming to be peer B.
|
||||
#
|
||||
# Backwards-compatible by design:
|
||||
#
|
||||
# - Single-peer installs (``MESH_PEER_SECRETS`` empty) keep using the
|
||||
# global secret. Zero behavior change. Zero operator action required.
|
||||
# - Multi-peer installs that haven't migrated yet keep using the global
|
||||
# secret for every peer. Same behavior as before — same exposure.
|
||||
# - Multi-peer installs that have migrated configure
|
||||
# ``MESH_PEER_SECRETS=urlA=secretA,urlB=secretB`` and immediately get
|
||||
# per-peer identity. Migration is incremental: peers not yet listed
|
||||
# continue using the global secret until both sides of that peering
|
||||
# add their entry.
|
||||
|
||||
_PEER_SECRETS_CACHE: dict[str, str] = {}
|
||||
_PEER_SECRETS_CACHE_RAW: str = ""
|
||||
|
||||
|
||||
def _lookup_per_peer_secret(normalized_url: str) -> str:
|
||||
"""Return the per-peer secret for ``normalized_url`` from MESH_PEER_SECRETS.
|
||||
|
||||
Returns "" if no per-peer entry is configured for that URL. The parser
|
||||
is forgiving:
|
||||
|
||||
- Whitespace around items, URLs, and secrets is stripped.
|
||||
- Items without ``=`` or with empty URL/secret halves are skipped.
|
||||
- The URL half is normalized via ``normalize_peer_url`` so config
|
||||
authors don't have to match scheme/port/path quirks exactly.
|
||||
|
||||
The cache is invalidated whenever the env var's raw value changes,
|
||||
which keeps tests' ``monkeypatch.setenv`` calls effective without
|
||||
forcing a process restart.
|
||||
"""
|
||||
import os
|
||||
|
||||
raw = str(os.environ.get("MESH_PEER_SECRETS", "") or "").strip()
|
||||
|
||||
global _PEER_SECRETS_CACHE, _PEER_SECRETS_CACHE_RAW
|
||||
if raw != _PEER_SECRETS_CACHE_RAW:
|
||||
new_cache: dict[str, str] = {}
|
||||
for chunk in raw.split(","):
|
||||
chunk = chunk.strip()
|
||||
if not chunk or "=" not in chunk:
|
||||
continue
|
||||
url_part, _, secret_part = chunk.partition("=")
|
||||
normalized = normalize_peer_url(url_part.strip())
|
||||
secret = secret_part.strip()
|
||||
if normalized and secret:
|
||||
new_cache[normalized] = secret
|
||||
_PEER_SECRETS_CACHE = new_cache
|
||||
_PEER_SECRETS_CACHE_RAW = raw
|
||||
|
||||
return _PEER_SECRETS_CACHE.get(normalized_url, "")
|
||||
|
||||
|
||||
def resolve_peer_key_for_url(peer_url: str) -> bytes:
|
||||
"""Return the HMAC key for ``peer_url``, preferring per-peer secret.
|
||||
|
||||
Issue #256: this is the function every peer-push call site should
|
||||
use. It looks up the peer-specific secret first, falling back to the
|
||||
fleet-shared ``MESH_PEER_PUSH_SECRET`` only when the URL is NOT
|
||||
listed in ``MESH_PEER_SECRETS``.
|
||||
|
||||
Both sender (computing X-Peer-HMAC) and receiver (verifying it) call
|
||||
this with the SENDER's URL — they must derive the same key, so
|
||||
operators on both ends of a peering need matching MESH_PEER_SECRETS
|
||||
entries for that URL to stay in sync.
|
||||
|
||||
Returns empty bytes when no usable secret exists. Callers must treat
|
||||
that as fail-closed (skip the push, reject the verification).
|
||||
"""
|
||||
normalized_url = normalize_peer_url(peer_url)
|
||||
if not normalized_url:
|
||||
return b""
|
||||
|
||||
per_peer_secret = _lookup_per_peer_secret(normalized_url)
|
||||
if per_peer_secret:
|
||||
return _derive_peer_key(per_peer_secret, normalized_url)
|
||||
|
||||
# No per-peer entry for this URL — fall back to the legacy global
|
||||
# secret. This is what preserves zero-hostility for single-peer
|
||||
# installs and the migration window for multi-peer installs.
|
||||
try:
|
||||
from services.config import get_settings
|
||||
|
||||
global_secret = str(
|
||||
getattr(get_settings(), "MESH_PEER_PUSH_SECRET", "") or ""
|
||||
).strip()
|
||||
except Exception:
|
||||
return b""
|
||||
if not global_secret:
|
||||
return b""
|
||||
return _derive_peer_key(global_secret, normalized_url)
|
||||
|
||||
|
||||
def _node_digest(public_key_b64: str) -> str:
|
||||
raw = base64.b64decode(public_key_b64)
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
@@ -216,18 +216,19 @@ def _peer_pair_ref_key(peer_url: str) -> bytes:
|
||||
Returns an empty key on misconfiguration so callers fail closed.
|
||||
"""
|
||||
try:
|
||||
from services.config import get_settings
|
||||
from services.mesh.mesh_crypto import _derive_peer_key, normalize_peer_url
|
||||
|
||||
secret = str(get_settings().MESH_PEER_PUSH_SECRET or "").strip()
|
||||
from services.mesh.mesh_crypto import (
|
||||
normalize_peer_url,
|
||||
resolve_peer_key_for_url,
|
||||
)
|
||||
except Exception:
|
||||
return b""
|
||||
if not secret:
|
||||
return b""
|
||||
normalized = normalize_peer_url(peer_url or "")
|
||||
if not normalized:
|
||||
return b""
|
||||
peer_key = _derive_peer_key(secret, normalized)
|
||||
# Issue #256: resolve_peer_key_for_url() prefers per-peer secrets
|
||||
# from MESH_PEER_SECRETS and falls back to the global
|
||||
# MESH_PEER_PUSH_SECRET only when the URL has no per-peer entry.
|
||||
peer_key = resolve_peer_key_for_url(normalized)
|
||||
if not peer_key:
|
||||
return b""
|
||||
# Domain-separate from the transport HMAC key so the two
|
||||
@@ -1438,14 +1439,57 @@ class Infonet:
|
||||
# Running counters — avoid O(N) scans in get_info()
|
||||
self._type_counts: dict[str, int] = {}
|
||||
self._active_count: int = 0
|
||||
self._registered_nodes: set[str] = set()
|
||||
self._chain_bytes: int = 2 # Start with "[]" empty JSON array
|
||||
self._dirty = False
|
||||
self._save_lock = threading.Lock()
|
||||
self._save_timer: threading.Timer | None = None
|
||||
self._SAVE_INTERVAL = 5.0 # seconds — coalesce writes
|
||||
# Issue #208: Merkle levels cache so get_merkle_proofs() doesn't
|
||||
# rebuild O(n) levels on every public call. Invalidated whenever
|
||||
# self.events mutates. Computed lazily on first read after an
|
||||
# invalidation.
|
||||
self._merkle_levels_cache: list[list[str]] | None = None
|
||||
self._merkle_levels_for_event_count: int = -1
|
||||
atexit.register(self._flush)
|
||||
self._load()
|
||||
|
||||
def _invalidate_merkle_cache(self) -> None:
|
||||
"""Clear the precomputed Merkle levels.
|
||||
|
||||
Called whenever ``self.events`` may have mutated (append, rebuild,
|
||||
cleanup, fork resolution). The next call to ``get_merkle_root()``
|
||||
or ``get_merkle_proofs()`` will recompute and re-cache.
|
||||
"""
|
||||
self._merkle_levels_cache = None
|
||||
self._merkle_levels_for_event_count = -1
|
||||
|
||||
def _get_merkle_levels(self) -> list[list[str]]:
|
||||
"""Return Merkle levels for the current chain, recomputing if
|
||||
the cache is invalid or out of date.
|
||||
|
||||
Issue #208: a public endpoint (``/api/mesh/infonet/sync?include_proofs=true``)
|
||||
used to rebuild Merkle levels on every request, which is O(n) in
|
||||
chain length and trivially abusable for CPU exhaustion. By caching
|
||||
the levels and invalidating on mutation, repeated proof requests
|
||||
become O(1) per proof; the rebuild only happens after a genuine
|
||||
append/rebuild/cleanup.
|
||||
"""
|
||||
from services.mesh.mesh_merkle import build_merkle_levels
|
||||
|
||||
current_count = len(self.events)
|
||||
if (
|
||||
self._merkle_levels_cache is not None
|
||||
and self._merkle_levels_for_event_count == current_count
|
||||
):
|
||||
return self._merkle_levels_cache
|
||||
|
||||
leaves = [e["event_id"] for e in self.events]
|
||||
levels = build_merkle_levels(leaves)
|
||||
self._merkle_levels_cache = levels
|
||||
self._merkle_levels_for_event_count = current_count
|
||||
return levels
|
||||
|
||||
# ─── Persistence ──────────────────────────────────────────────────
|
||||
|
||||
def _load(self):
|
||||
@@ -1518,6 +1562,7 @@ class Infonet:
|
||||
self._last_validated_index = 0
|
||||
self._type_counts = {}
|
||||
self._active_count = 0
|
||||
self._registered_nodes = set()
|
||||
self._chain_bytes = 2
|
||||
|
||||
def _rebuild_state(self) -> None:
|
||||
@@ -1566,10 +1611,15 @@ class Infonet:
|
||||
now = time.time()
|
||||
self._type_counts = {}
|
||||
self._active_count = 0
|
||||
self._registered_nodes = set()
|
||||
self._chain_bytes = 2 # "[]"
|
||||
for evt in self.events:
|
||||
t = evt.get("event_type", "unknown")
|
||||
self._type_counts[t] = self._type_counts.get(t, 0) + 1
|
||||
if t == "node_register":
|
||||
node_id = str(evt.get("node_id", "") or "")
|
||||
if node_id:
|
||||
self._registered_nodes.add(node_id)
|
||||
is_eph = evt.get("payload", {}).get("ephemeral") or evt.get("payload", {}).get("_ephemeral")
|
||||
if not is_eph or (now - evt.get("timestamp", 0)) < EPHEMERAL_TTL:
|
||||
self._active_count += 1
|
||||
@@ -1579,6 +1629,10 @@ class Infonet:
|
||||
"""Incrementally update counters when a new event is appended."""
|
||||
t = evt.get("event_type", "unknown")
|
||||
self._type_counts[t] = self._type_counts.get(t, 0) + 1
|
||||
if t == "node_register":
|
||||
node_id = str(evt.get("node_id", "") or "")
|
||||
if node_id:
|
||||
self._registered_nodes.add(node_id)
|
||||
self._active_count += 1
|
||||
self._chain_bytes += len(json.dumps(evt)) + 2
|
||||
|
||||
@@ -1972,6 +2026,8 @@ class Infonet:
|
||||
self.head_hash = event.event_id
|
||||
self.node_sequences[node_id] = sequence
|
||||
self._replay_filter.add(event.event_id)
|
||||
# Issue #208: chain advanced, cached Merkle levels are stale.
|
||||
self._invalidate_merkle_cache()
|
||||
self._update_counters_for_event(event_dict)
|
||||
|
||||
if event_type == "key_revoke":
|
||||
@@ -2247,6 +2303,7 @@ class Infonet:
|
||||
self.event_index[event_id] = len(self.events) - 1
|
||||
self.head_hash = event_id
|
||||
self.node_sequences[node_id] = sequence
|
||||
self._update_counters_for_event(evt)
|
||||
accepted += 1
|
||||
expected_prev = event_id
|
||||
self._replay_filter.add(event_id)
|
||||
@@ -2254,6 +2311,9 @@ class Infonet:
|
||||
self._apply_revocation(evt)
|
||||
|
||||
if accepted:
|
||||
# Issue #208: any accepted event invalidates the cached Merkle
|
||||
# levels. One invalidation per batch, not per event.
|
||||
self._invalidate_merkle_cache()
|
||||
self._save()
|
||||
return {"accepted": accepted, "duplicates": duplicates, "rejected": rejected}
|
||||
|
||||
@@ -2552,6 +2612,10 @@ class Infonet:
|
||||
# Apply fork
|
||||
self.events = prefix + ordered
|
||||
self._rebuild_state()
|
||||
self._rebuild_revocations()
|
||||
self._rebuild_counters()
|
||||
# Issue #208: chain replaced, cached Merkle levels are stale.
|
||||
self._invalidate_merkle_cache()
|
||||
self._save()
|
||||
try:
|
||||
from services.mesh.mesh_metrics import increment as metrics_inc
|
||||
@@ -2681,6 +2745,8 @@ class Infonet:
|
||||
"head_hash_full": self.head_hash,
|
||||
"chain_lock": self.chain_lock(),
|
||||
"known_nodes": len(self.node_sequences),
|
||||
"author_nodes": len(self.node_sequences),
|
||||
"registered_nodes": len(self._registered_nodes),
|
||||
"event_types": dict(self._type_counts),
|
||||
"chain_size_kb": round(self._chain_bytes / 1024, 1),
|
||||
"unsigned_events": 0,
|
||||
@@ -2716,8 +2782,11 @@ class Infonet:
|
||||
|
||||
if len(new_events) != before:
|
||||
self.events = new_events
|
||||
# Rebuild index
|
||||
self.event_index = {e["event_id"]: i for i, e in enumerate(self.events)}
|
||||
self._rebuild_state()
|
||||
self._rebuild_revocations()
|
||||
self._rebuild_counters()
|
||||
# Issue #208: cleanup may have dropped expired events.
|
||||
self._invalidate_merkle_cache()
|
||||
self._save()
|
||||
logger.info(f"Infonet cleanup: removed {before - len(new_events)} expired events")
|
||||
|
||||
@@ -2726,30 +2795,37 @@ class Infonet:
|
||||
def get_merkle_root(self) -> str:
|
||||
"""Compute a Merkle root hash of the Infonet for sync comparison.
|
||||
|
||||
Two nodes with the same Merkle root have identical chains.
|
||||
Two nodes with the same Merkle root have identical chains. Reads
|
||||
from the cached Merkle levels (issue #208) — O(1) when the chain
|
||||
hasn't changed since the last computation.
|
||||
"""
|
||||
if not self.events:
|
||||
return GENESIS_HASH
|
||||
|
||||
from services.mesh.mesh_merkle import merkle_root
|
||||
|
||||
leaves = [e["event_id"] for e in self.events]
|
||||
root = merkle_root(leaves)
|
||||
return root or GENESIS_HASH
|
||||
levels = self._get_merkle_levels()
|
||||
if not levels or not levels[-1]:
|
||||
return GENESIS_HASH
|
||||
return levels[-1][0] or GENESIS_HASH
|
||||
|
||||
def get_merkle_proofs(self, start_index: int, count: int) -> dict:
|
||||
"""Return merkle proofs for a contiguous range of events."""
|
||||
leaves = [e["event_id"] for e in self.events]
|
||||
total = len(leaves)
|
||||
"""Return merkle proofs for a contiguous range of events.
|
||||
|
||||
Issue #208: uses the cached Merkle levels so this is O(count *
|
||||
log n) per request, not O(n + count * log n). Anonymous peers
|
||||
hitting ``/api/mesh/infonet/sync?include_proofs=true`` no longer
|
||||
force a rebuild on every call.
|
||||
"""
|
||||
total = len(self.events)
|
||||
if total == 0:
|
||||
return {"root": GENESIS_HASH, "total": 0, "start": 0, "proofs": []}
|
||||
|
||||
from services.mesh.mesh_merkle import build_merkle_levels, merkle_proof_from_levels
|
||||
from services.mesh.mesh_merkle import merkle_proof_from_levels
|
||||
|
||||
leaves = [e["event_id"] for e in self.events]
|
||||
start = max(0, start_index)
|
||||
end = min(total, start + max(0, count))
|
||||
levels = build_merkle_levels(leaves)
|
||||
root = levels[-1][0] if levels else GENESIS_HASH
|
||||
levels = self._get_merkle_levels()
|
||||
root = levels[-1][0] if levels and levels[-1] else GENESIS_HASH
|
||||
|
||||
proofs = []
|
||||
for idx in range(start, end):
|
||||
|
||||
@@ -26,7 +26,11 @@ from enum import Enum
|
||||
from typing import Any, Callable, Optional
|
||||
from collections import deque
|
||||
from urllib.parse import urlparse
|
||||
from services.mesh.mesh_crypto import _derive_peer_key, normalize_peer_url
|
||||
from services.mesh.mesh_crypto import (
|
||||
_derive_peer_key,
|
||||
normalize_peer_url,
|
||||
resolve_peer_key_for_url,
|
||||
)
|
||||
from services.mesh.mesh_metrics import increment as metrics_inc
|
||||
from services.mesh.mesh_privacy_policy import (
|
||||
TRANSPORT_TIER_ORDER as _TIER_RANK,
|
||||
@@ -703,7 +707,6 @@ class InternetTransport(_PeerPushTransportMixin):
|
||||
endpoint_path, padded = self._build_peer_push_request(envelope, self.NAME)
|
||||
except ValueError as exc:
|
||||
return TransportResult(False, self.NAME, str(exc))
|
||||
secret = str(settings.MESH_PEER_PUSH_SECRET or "").strip()
|
||||
|
||||
delivered = 0
|
||||
last_error = ""
|
||||
@@ -713,10 +716,13 @@ class InternetTransport(_PeerPushTransportMixin):
|
||||
try:
|
||||
normalized_peer_url = normalize_peer_url(peer_url)
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if secret:
|
||||
peer_key = _derive_peer_key(secret, normalized_peer_url)
|
||||
if not peer_key:
|
||||
raise ValueError("invalid peer URL for HMAC derivation")
|
||||
# Issue #256: per-peer secret takes precedence over the
|
||||
# global MESH_PEER_PUSH_SECRET. When neither is set the
|
||||
# key is empty and we skip the HMAC header entirely so a
|
||||
# bare (unsigned) push still works on test deployments
|
||||
# that have not yet configured any secret at all.
|
||||
peer_key = resolve_peer_key_for_url(normalized_peer_url)
|
||||
if peer_key:
|
||||
headers["X-Peer-Url"] = normalized_peer_url
|
||||
headers["X-Peer-HMAC"] = hmac.new(
|
||||
peer_key,
|
||||
@@ -798,7 +804,6 @@ class TorArtiTransport(_PeerPushTransportMixin):
|
||||
endpoint_path, padded = self._build_peer_push_request(envelope, self.NAME)
|
||||
except ValueError as exc:
|
||||
return TransportResult(False, self.NAME, str(exc))
|
||||
secret = str(settings.MESH_PEER_PUSH_SECRET or "").strip()
|
||||
|
||||
delivered = 0
|
||||
last_error = ""
|
||||
@@ -808,10 +813,10 @@ class TorArtiTransport(_PeerPushTransportMixin):
|
||||
try:
|
||||
normalized_peer_url = normalize_peer_url(peer_url)
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if secret:
|
||||
peer_key = _derive_peer_key(secret, normalized_peer_url)
|
||||
if not peer_key:
|
||||
raise ValueError("invalid peer URL for HMAC derivation")
|
||||
# Issue #256: per-peer secret takes precedence; see the
|
||||
# other transport above for the rationale.
|
||||
peer_key = resolve_peer_key_for_url(normalized_peer_url)
|
||||
if peer_key:
|
||||
headers["X-Peer-Url"] = normalized_peer_url
|
||||
headers["X-Peer-HMAC"] = hmac.new(
|
||||
peer_key,
|
||||
|
||||
@@ -17,7 +17,7 @@ import time
|
||||
from typing import Any
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import ed25519
|
||||
from cryptography.hazmat.primitives.asymmetric import ed25519, x25519
|
||||
|
||||
from services.mesh.mesh_crypto import (
|
||||
build_signature_payload,
|
||||
@@ -464,6 +464,37 @@ def _bundle_fingerprint(data: dict[str, Any]) -> str:
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _ensure_dm_dh_material(data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
||||
"""Repair legacy/corrupt DM identities that kept signing keys but lost DH material."""
|
||||
if str(data.get("dh_pub_key", "") or "").strip() and str(data.get("dh_private_key", "") or "").strip():
|
||||
return data, False
|
||||
|
||||
dh_priv = x25519.X25519PrivateKey.generate()
|
||||
dh_priv_raw = dh_priv.private_bytes(
|
||||
encoding=serialization.Encoding.Raw,
|
||||
format=serialization.PrivateFormat.Raw,
|
||||
encryption_algorithm=serialization.NoEncryption(),
|
||||
)
|
||||
dh_pub_raw = dh_priv.public_key().public_bytes(
|
||||
encoding=serialization.Encoding.Raw,
|
||||
format=serialization.PublicFormat.Raw,
|
||||
)
|
||||
repaired = {
|
||||
**dict(data or {}),
|
||||
"dh_pub_key": base64.b64encode(dh_pub_raw).decode("ascii"),
|
||||
"dh_algo": "X25519",
|
||||
"dh_private_key": base64.b64encode(dh_priv_raw).decode("ascii"),
|
||||
"last_dh_timestamp": int(time.time()),
|
||||
"bundle_fingerprint": "",
|
||||
"bundle_sequence": 0,
|
||||
"bundle_registered_at": 0,
|
||||
"prekey_bundle_registered_at": 0,
|
||||
"prekey_transparency_head": "",
|
||||
"prekey_transparency_size": 0,
|
||||
}
|
||||
return _write_identity(repaired), True
|
||||
|
||||
|
||||
def trust_fingerprint_for_identity_material(
|
||||
*,
|
||||
agent_id: str,
|
||||
@@ -830,10 +861,11 @@ def _sign_dm_invite_payload(
|
||||
|
||||
def register_wormhole_dm_key(force: bool = False) -> dict[str, Any]:
|
||||
data = read_wormhole_identity()
|
||||
data, repaired_dh = _ensure_dm_dh_material(data)
|
||||
|
||||
timestamp = int(time.time())
|
||||
fingerprint = _bundle_fingerprint(data)
|
||||
if not force and fingerprint and fingerprint == data.get("bundle_fingerprint"):
|
||||
if not force and not repaired_dh and fingerprint and fingerprint == data.get("bundle_fingerprint"):
|
||||
return {
|
||||
"ok": True,
|
||||
**_public_view(data),
|
||||
@@ -1525,11 +1557,101 @@ def import_wormhole_dm_invite(invite: dict[str, Any], *, alias: str = "") -> dic
|
||||
"detail": "compat dm invite import disabled; ask the sender to re-export a current signed invite",
|
||||
}
|
||||
|
||||
def _prekey_missing_or_pending(detail: str) -> bool:
|
||||
lower = str(detail or "").strip().lower()
|
||||
return any(
|
||||
phrase in lower
|
||||
for phrase in (
|
||||
"prekey bundle not found",
|
||||
"invite prekey bundle not found",
|
||||
"peer prekey lookup unavailable",
|
||||
"peer prekey lookup still preparing",
|
||||
"transport tier insufficient",
|
||||
"preparing_private_lane",
|
||||
)
|
||||
)
|
||||
|
||||
def _pin_pending_invite_prekey(detail: str) -> dict[str, Any]:
|
||||
if invite_version < DM_INVITE_VERSION:
|
||||
return {"ok": False, "detail": detail or "invite prekey bundle not found"}
|
||||
invite_root_distribution = _verify_dm_invite_root_distribution(payload)
|
||||
if not invite_root_distribution.get("ok"):
|
||||
return invite_root_distribution
|
||||
attested = _verify_dm_invite_identity_attestation(
|
||||
envelope=envelope,
|
||||
payload=payload,
|
||||
resolved_root_node_id=str(invite_root_distribution.get("root_node_id", "") or ""),
|
||||
resolved_root_public_key=str(invite_root_distribution.get("root_public_key", "") or ""),
|
||||
resolved_root_public_key_algo=str(
|
||||
invite_root_distribution.get("root_public_key_algo", "Ed25519") or "Ed25519"
|
||||
),
|
||||
resolved_root_manifest_fingerprint=str(
|
||||
invite_root_distribution.get("root_manifest_fingerprint", "") or ""
|
||||
).strip().lower(),
|
||||
)
|
||||
if not attested.get("ok"):
|
||||
return attested
|
||||
pending_peer_id = str(verified.get("peer_id", "") or "").strip()
|
||||
trust_fingerprint = str(verified.get("trust_fingerprint", "") or "").strip().lower()
|
||||
contact = pin_wormhole_dm_invite(
|
||||
pending_peer_id,
|
||||
invite_payload={
|
||||
"trust_fingerprint": trust_fingerprint,
|
||||
"public_key": "",
|
||||
"public_key_algo": "Ed25519",
|
||||
"identity_dh_pub_key": "",
|
||||
"dh_algo": "X25519",
|
||||
"prekey_lookup_handle": lookup_handle,
|
||||
"issued_at": int(payload.get("issued_at", 0) or 0),
|
||||
"expires_at": int(payload.get("expires_at", 0) or 0),
|
||||
"label": str(payload.get("label", "") or ""),
|
||||
"root_node_id": str(attested.get("root_node_id", "") or ""),
|
||||
"root_public_key": str(attested.get("root_public_key", "") or ""),
|
||||
"root_public_key_algo": str(attested.get("root_public_key_algo", "Ed25519") or "Ed25519"),
|
||||
"root_fingerprint": str(attested.get("root_fingerprint", "") or ""),
|
||||
"root_manifest_fingerprint": str(invite_root_distribution.get("root_manifest_fingerprint", "") or ""),
|
||||
"root_witness_policy_fingerprint": str(
|
||||
invite_root_distribution.get("root_witness_policy_fingerprint", "") or ""
|
||||
),
|
||||
"root_witness_threshold": _safe_int(
|
||||
invite_root_distribution.get("root_witness_threshold", 0) or 0,
|
||||
0,
|
||||
),
|
||||
"root_witness_count": _safe_int(invite_root_distribution.get("root_witness_count", 0) or 0, 0),
|
||||
"root_witness_domain_count": _safe_int(
|
||||
invite_root_distribution.get("root_witness_domain_count", 0) or 0,
|
||||
0,
|
||||
),
|
||||
"root_manifest_generation": _safe_int(
|
||||
invite_root_distribution.get("root_manifest_generation", 0) or 0,
|
||||
0,
|
||||
),
|
||||
"root_rotation_proven": bool(invite_root_distribution.get("root_rotation_proven")),
|
||||
},
|
||||
alias=resolved_alias,
|
||||
attested=True,
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"peer_id": pending_peer_id,
|
||||
"invite_peer_id": pending_peer_id,
|
||||
"trust_fingerprint": trust_fingerprint,
|
||||
"trust_level": str(contact.get("trust_level", "") or ""),
|
||||
"detail": "Contact saved.",
|
||||
"invite_attested": True,
|
||||
"pending_prekey": True,
|
||||
"prekey_detail": detail or "invite prekey bundle not found",
|
||||
"contact": contact,
|
||||
}
|
||||
|
||||
from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle
|
||||
|
||||
fetched = fetch_dm_prekey_bundle(lookup_token=lookup_handle)
|
||||
if not fetched.get("ok"):
|
||||
return {"ok": False, "detail": str(fetched.get("detail", "") or "invite prekey bundle not found")}
|
||||
fetch_detail = str(fetched.get("detail", "") or "invite prekey bundle not found")
|
||||
if _prekey_missing_or_pending(fetch_detail):
|
||||
return _pin_pending_invite_prekey(fetch_detail)
|
||||
return {"ok": False, "detail": fetch_detail}
|
||||
|
||||
resolved_peer_id = str(fetched.get("agent_id", "") or "").strip()
|
||||
if not resolved_peer_id:
|
||||
|
||||
@@ -11,6 +11,7 @@ import os
|
||||
import random
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
@@ -90,13 +91,15 @@ def _fetch_dm_prekey_bundle_from_peer_lookup(lookup_token: str) -> dict[str, Any
|
||||
return {"ok": False, "detail": "lookup token required"}
|
||||
try:
|
||||
from services.config import get_settings
|
||||
from services.mesh.mesh_crypto import _derive_peer_key, normalize_peer_url
|
||||
from services.mesh.mesh_crypto import (
|
||||
normalize_peer_url,
|
||||
resolve_peer_key_for_url,
|
||||
)
|
||||
from services.mesh.mesh_router import configured_relay_peer_urls
|
||||
|
||||
settings = get_settings()
|
||||
secret = str(getattr(settings, "MESH_PEER_PUSH_SECRET", "") or "").strip()
|
||||
if not secret:
|
||||
return {"ok": False, "detail": "peer prekey lookup unavailable"}
|
||||
# Issue #256: secret check moved per-peer below. We still bail out
|
||||
# cleanly when there are no peers configured at all.
|
||||
peers = configured_relay_peer_urls()
|
||||
if not peers:
|
||||
return {"ok": False, "detail": "peer prekey lookup unavailable"}
|
||||
@@ -120,7 +123,8 @@ def _fetch_dm_prekey_bundle_from_peer_lookup(lookup_token: str) -> dict[str, Any
|
||||
or os.environ.get("SB_TEST_NODE_URL", "").strip()
|
||||
or normalized_peer_url
|
||||
)
|
||||
peer_key = _derive_peer_key(secret, sender_peer_url)
|
||||
# Issue #256: prefer per-peer secret keyed by the sender URL.
|
||||
peer_key = resolve_peer_key_for_url(sender_peer_url)
|
||||
if not peer_key:
|
||||
continue
|
||||
headers = {
|
||||
@@ -150,6 +154,122 @@ def _fetch_dm_prekey_bundle_from_peer_lookup(lookup_token: str) -> dict[str, Any
|
||||
return {"ok": False, "detail": last_detail or "Prekey bundle not found"}
|
||||
|
||||
|
||||
def _configured_public_lookup_peer_urls() -> list[str]:
|
||||
try:
|
||||
from services.config import get_settings
|
||||
from services.mesh.mesh_router import active_sync_peer_urls, parse_configured_relay_peers
|
||||
|
||||
settings = get_settings()
|
||||
candidates: list[str] = []
|
||||
for raw in (
|
||||
getattr(settings, "MESH_BOOTSTRAP_SEED_PEERS", ""),
|
||||
getattr(settings, "MESH_DEFAULT_SYNC_PEERS", ""),
|
||||
):
|
||||
candidates.extend(parse_configured_relay_peers(str(raw or "")))
|
||||
candidates.extend(active_sync_peer_urls())
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
seen: set[str] = set()
|
||||
peers: list[str] = []
|
||||
for candidate in candidates:
|
||||
peer = str(candidate or "").strip().rstrip("/")
|
||||
if not peer or peer in seen:
|
||||
continue
|
||||
seen.add(peer)
|
||||
peers.append(peer)
|
||||
return peers
|
||||
|
||||
|
||||
def _normalize_remote_lookup_bundle(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
data = dict(payload or {})
|
||||
bundle = dict(data.get("bundle") or {})
|
||||
public_key = str(data.get("public_key", "") or bundle.get("public_key", "") or "").strip()
|
||||
if not public_key:
|
||||
return {"ok": False, "detail": "Prekey bundle missing signing key"}
|
||||
agent_id = str(data.get("agent_id", "") or "").strip() or derive_node_id(public_key)
|
||||
if not agent_id:
|
||||
return {"ok": False, "detail": "Prekey bundle public key binding mismatch"}
|
||||
data["agent_id"] = agent_id
|
||||
data["public_key"] = public_key
|
||||
data["public_key_algo"] = str(data.get("public_key_algo", "") or bundle.get("public_key_algo", "Ed25519") or "Ed25519")
|
||||
data["protocol_version"] = str(data.get("protocol_version", "") or bundle.get("protocol_version", PROTOCOL_VERSION) or PROTOCOL_VERSION)
|
||||
data["bundle"] = bundle
|
||||
ok, reason = _validate_bundle_record(data)
|
||||
if not ok:
|
||||
return {"ok": False, "detail": reason}
|
||||
data["ok"] = True
|
||||
data["lookup_mode"] = "invite_lookup_handle"
|
||||
data["public_lookup"] = True
|
||||
return data
|
||||
|
||||
|
||||
def _fetch_dm_prekey_bundle_from_public_lookup(lookup_token: str) -> dict[str, Any]:
|
||||
"""Fetch an invite-scoped prekey bundle from bootstrap/sync peers.
|
||||
|
||||
The token is high-entropy and invite-scoped. This path does not expose a
|
||||
stable agent_id to the peer; if the ordinary peer response omits agent_id,
|
||||
derive it from the signed identity public key and validate the bundle before
|
||||
accepting it.
|
||||
"""
|
||||
token = str(lookup_token or "").strip()
|
||||
if not token:
|
||||
return {"ok": False, "detail": "lookup token required"}
|
||||
peers = _configured_public_lookup_peer_urls()
|
||||
if not peers:
|
||||
return {"ok": False, "detail": "peer prekey lookup unavailable"}
|
||||
try:
|
||||
from services.config import get_settings
|
||||
|
||||
timeout = max(1, _safe_int(getattr(get_settings(), "MESH_SYNC_TIMEOUT_S", 5) or 5, 5))
|
||||
except Exception:
|
||||
timeout = 5
|
||||
|
||||
encoded = urllib.parse.urlencode({"lookup_token": token})
|
||||
last_detail = ""
|
||||
for peer_url in peers:
|
||||
normalized_peer_url = str(peer_url or "").strip().rstrip("/")
|
||||
if not normalized_peer_url:
|
||||
continue
|
||||
# Generic UA: any peer-facing crypto request should not carry a
|
||||
# fork-specific identifier — that turns prekey lookups into a
|
||||
# software-fingerprinting beacon.
|
||||
from services.network_utils import DEFAULT_USER_AGENT
|
||||
request = urllib.request.Request(
|
||||
f"{normalized_peer_url}/api/mesh/dm/prekey-bundle?{encoded}",
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"User-Agent": DEFAULT_USER_AGENT,
|
||||
},
|
||||
method="GET",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
raw = response.read(256 * 1024)
|
||||
payload = json.loads(raw.decode("utf-8"))
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, OSError) as exc:
|
||||
logger.debug("public prekey lookup failed for %s: %s", normalized_peer_url, type(exc).__name__)
|
||||
last_detail = "peer prekey lookup unavailable"
|
||||
continue
|
||||
if not isinstance(payload, dict):
|
||||
last_detail = "invalid peer response"
|
||||
continue
|
||||
if payload.get("pending") or str(payload.get("status", "") or "") == "preparing_private_lane":
|
||||
last_detail = "peer prekey lookup still preparing"
|
||||
continue
|
||||
if not payload.get("ok"):
|
||||
last_detail = str(payload.get("detail", "") or last_detail or "Prekey bundle not found")
|
||||
continue
|
||||
if not isinstance(payload.get("bundle"), dict):
|
||||
last_detail = "Prekey bundle not found"
|
||||
continue
|
||||
normalized = _normalize_remote_lookup_bundle(payload)
|
||||
if normalized.get("ok"):
|
||||
return normalized
|
||||
last_detail = str(normalized.get("detail", "") or last_detail)
|
||||
return {"ok": False, "detail": last_detail or "Prekey bundle not found"}
|
||||
|
||||
|
||||
def _b64(data: bytes) -> str:
|
||||
return base64.b64encode(data).decode("ascii")
|
||||
|
||||
@@ -926,6 +1046,11 @@ def fetch_dm_prekey_bundle(
|
||||
peer_found = _fetch_dm_prekey_bundle_from_peer_lookup(resolved_lookup)
|
||||
if peer_found.get("ok"):
|
||||
return peer_found
|
||||
public_found = _fetch_dm_prekey_bundle_from_public_lookup(resolved_lookup)
|
||||
if public_found.get("ok"):
|
||||
return public_found
|
||||
if str(public_found.get("detail", "") or "").strip():
|
||||
return {"ok": False, "detail": str(public_found.get("detail", "") or "Prekey bundle not found")}
|
||||
return {"ok": False, "detail": str(peer_found.get("detail", "") or "Prekey bundle not found")}
|
||||
else:
|
||||
return {"ok": False, "detail": "Prekey bundle not found"}
|
||||
|
||||
@@ -19,6 +19,17 @@ _retry = Retry(total=1, backoff_factor=0.3, status_forcelist=[502, 503, 504])
|
||||
_session.mount("https://", HTTPAdapter(max_retries=_retry, pool_maxsize=20))
|
||||
_session.mount("http://", HTTPAdapter(max_retries=_retry, pool_maxsize=10))
|
||||
|
||||
|
||||
# Default outbound User-Agent. Generic by design — does NOT include any
|
||||
# personal contact info or a fork-specific repo URL. Operators who run a
|
||||
# public-facing relay and want to identify themselves to upstreams (e.g.
|
||||
# for Nominatim / weather.gov usage-policy compliance) can override this
|
||||
# via the SHADOWBROKER_USER_AGENT env var.
|
||||
DEFAULT_USER_AGENT = os.environ.get(
|
||||
"SHADOWBROKER_USER_AGENT",
|
||||
"ShadowBroker-OSINT/0.9",
|
||||
)
|
||||
|
||||
# Find bash for curl fallback — Git bash's curl has the TLS features
|
||||
# needed to pass CDN fingerprint checks (brotli, zstd, libpsl)
|
||||
|
||||
@@ -73,7 +84,7 @@ def fetch_with_curl(url, method="GET", json_data=None, timeout=15, headers=None,
|
||||
both Python requests and the barebones Windows system curl.
|
||||
"""
|
||||
default_headers = {
|
||||
"User-Agent": "ShadowBroker-OSINT/0.9.79 (+https://github.com/BigBodyCobain/Shadowbroker; contact: bigbodycobain@gmail.com)",
|
||||
"User-Agent": DEFAULT_USER_AGENT,
|
||||
}
|
||||
if headers:
|
||||
default_headers.update(headers)
|
||||
|
||||
@@ -6,8 +6,8 @@ Docs: https://pskreporter.info/pskdev.html
|
||||
"""
|
||||
|
||||
import logging
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
import defusedxml.ElementTree as ET
|
||||
import requests
|
||||
from cachetools import TTLCache, cached
|
||||
|
||||
|
||||
@@ -131,27 +131,61 @@ def get_recent_openmhz_calls(sys_name: str):
|
||||
return []
|
||||
|
||||
|
||||
_OPENMHZ_MAX_REDIRECTS = 5
|
||||
|
||||
|
||||
def openmhz_audio_response(target_url: str):
|
||||
"""Fetch an OpenMHz audio object through the backend with browser-safe headers."""
|
||||
"""Fetch an OpenMHz audio object through the backend with browser-safe headers.
|
||||
|
||||
Redirects are followed manually so each hop's host can be re-validated
|
||||
against ``_OPENMHZ_AUDIO_HOSTS``. Without this, the upstream could
|
||||
302-redirect to an internal address (e.g. ``http://127.0.0.1:8000/...``
|
||||
or an RFC1918 range), and the backend would dutifully fetch and stream
|
||||
that response back to the browser — a classic open-redirect-to-SSRF
|
||||
chain. Same-host redirects (CDN edge selection) still work normally.
|
||||
"""
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
from urllib.parse import urljoin
|
||||
|
||||
parsed = urlparse(str(target_url or ""))
|
||||
host = (parsed.hostname or "").lower()
|
||||
if parsed.scheme != "https" or host not in _OPENMHZ_AUDIO_HOSTS:
|
||||
raise HTTPException(status_code=400, detail="Unsupported OpenMHz audio URL")
|
||||
|
||||
current_url = target_url
|
||||
hops = 0
|
||||
try:
|
||||
upstream = requests.get(
|
||||
target_url,
|
||||
stream=True,
|
||||
timeout=(5, 20),
|
||||
headers={
|
||||
"User-Agent": "Mozilla/5.0",
|
||||
"Accept": "audio/mpeg,audio/*,*/*;q=0.8",
|
||||
"Referer": "https://openmhz.com/",
|
||||
},
|
||||
)
|
||||
while True:
|
||||
upstream = requests.get(
|
||||
current_url,
|
||||
stream=True,
|
||||
timeout=(5, 20),
|
||||
allow_redirects=False,
|
||||
headers={
|
||||
"User-Agent": "Mozilla/5.0",
|
||||
"Accept": "audio/mpeg,audio/*,*/*;q=0.8",
|
||||
"Referer": "https://openmhz.com/",
|
||||
},
|
||||
)
|
||||
if upstream.is_redirect or upstream.status_code in (301, 302, 303, 307, 308):
|
||||
location = upstream.headers.get("Location", "")
|
||||
upstream.close()
|
||||
if hops >= _OPENMHZ_MAX_REDIRECTS or not location:
|
||||
raise HTTPException(status_code=502, detail="OpenMHz redirect rejected")
|
||||
next_url = urljoin(current_url, location)
|
||||
next_parsed = urlparse(next_url)
|
||||
next_host = (next_parsed.hostname or "").lower()
|
||||
# Re-validate the next hop against the same allowlist used for
|
||||
# the original URL. Cross-host redirects to disallowed hosts
|
||||
# are rejected silently; the browser audio element handles
|
||||
# the resulting 502 gracefully and moves on.
|
||||
if next_parsed.scheme != "https" or next_host not in _OPENMHZ_AUDIO_HOSTS:
|
||||
raise HTTPException(status_code=502, detail="OpenMHz redirect rejected")
|
||||
current_url = next_url
|
||||
hops += 1
|
||||
continue
|
||||
break
|
||||
except requests.RequestException as exc:
|
||||
raise HTTPException(status_code=502, detail="OpenMHz audio fetch failed") from exc
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import concurrent.futures
|
||||
from urllib.parse import quote
|
||||
import requests as _requests
|
||||
from cachetools import TTLCache
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.network_utils import fetch_with_curl, DEFAULT_USER_AGENT
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -15,6 +15,25 @@ dossier_cache = TTLCache(maxsize=500, ttl=86400)
|
||||
# Nominatim requires max 1 req/sec — track last call time
|
||||
_nominatim_last_call = 0.0
|
||||
|
||||
# Issue #218 / #219 (tg12): Wikimedia's User-Agent policy requires API
|
||||
# clients to identify themselves with a stable User-Agent that includes
|
||||
# a contact path. Bare "python-requests/x.y" or generic strings violate
|
||||
# the policy and risk getting blocked. We send the project default UA
|
||||
# (operator-overridable via SHADOWBROKER_USER_AGENT) on EVERY outbound
|
||||
# Wikimedia request, plus the policy-recommended Api-User-Agent which
|
||||
# Wikimedia explicitly accepts on top of the regular UA.
|
||||
#
|
||||
# This is documented and stable so a Wikimedia operator who wants to
|
||||
# rate-limit or contact us has a fixed identifier to grep for.
|
||||
_WIKIMEDIA_REQUEST_HEADERS = {
|
||||
"User-Agent": DEFAULT_USER_AGENT,
|
||||
"Api-User-Agent": (
|
||||
f"{DEFAULT_USER_AGENT} "
|
||||
"(+https://github.com/BigBodyCobain/Shadowbroker; "
|
||||
"report issues at /issues)"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _reverse_geocode_offline(lat: float, lng: float) -> dict:
|
||||
"""Offline fallback via reverse_geocoder when external reverse geocoding is blocked."""
|
||||
@@ -121,7 +140,13 @@ def _fetch_wikidata_leader(country_name: str) -> dict:
|
||||
"""
|
||||
url = f"https://query.wikidata.org/sparql?query={quote(sparql)}&format=json"
|
||||
try:
|
||||
res = fetch_with_curl(url, timeout=6)
|
||||
# Issue #218 (tg12): Wikimedia's User-Agent policy requires
|
||||
# outbound API traffic to be identifiable. fetch_with_curl()
|
||||
# sends the project default, and we also add the Wikimedia-
|
||||
# specific Api-User-Agent that the policy specifically asks
|
||||
# for, since this request originates from a backend service
|
||||
# that proxies on behalf of (potentially many) browser users.
|
||||
res = fetch_with_curl(url, timeout=6, headers=_WIKIMEDIA_REQUEST_HEADERS)
|
||||
if res.status_code == 200:
|
||||
results = res.json().get("results", {}).get("bindings", [])
|
||||
if results:
|
||||
@@ -147,7 +172,9 @@ def _fetch_local_wiki_summary(place_name: str, country_name: str = "") -> dict:
|
||||
slug = quote(name.replace(" ", "_"))
|
||||
url = f"https://en.wikipedia.org/api/rest_v1/page/summary/{slug}"
|
||||
try:
|
||||
res = fetch_with_curl(url, timeout=5)
|
||||
# Issue #219 (tg12): identify ourselves to Wikimedia per
|
||||
# their UA policy; see _fetch_wikidata_leader above.
|
||||
res = fetch_with_curl(url, timeout=5, headers=_WIKIMEDIA_REQUEST_HEADERS)
|
||||
if res.status_code == 200:
|
||||
data = res.json()
|
||||
if data.get("type") != "disambiguation":
|
||||
|
||||
@@ -14,6 +14,11 @@ class HealthResponse(BaseModel):
|
||||
# ({status, age_s, row_count, slo, stale, empty, description}).
|
||||
slo: Optional[Dict[str, Any]] = None
|
||||
slo_summary: Optional[Dict[str, int]] = None
|
||||
# Issue #258: AIS proxy status — currently exposes ``degraded_tls``
|
||||
# (bool), true when ais_proxy.js fell back to the SPKI-pinned
|
||||
# insecure-date path because the upstream Let's Encrypt cert is
|
||||
# expired. Empty dict / null means no status reported yet.
|
||||
ais_proxy: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class RefreshResponse(BaseModel):
|
||||
|
||||
@@ -64,6 +64,203 @@ def _find_tor_binary() -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
# Baked-in expected digest list. Loaded lazily; populated by maintainers
|
||||
# when a new Tor Expert Bundle URL is added to _TOR_EXPERT_BUNDLE_URLS.
|
||||
# See issue #201 for rationale.
|
||||
_TOR_DIGEST_FILE = Path(__file__).resolve().parent.parent / "data" / "tor_bundle_digests.json"
|
||||
_DIGEST_PLACEHOLDER = "PLACEHOLDER_REPLACE_BEFORE_RELEASE"
|
||||
|
||||
|
||||
def _load_baked_in_digests() -> dict[str, str]:
|
||||
"""Return {url: expected_sha256_lower} for URLs we ship a known digest for.
|
||||
|
||||
Entries whose value is the placeholder sentinel are filtered out — they
|
||||
represent versions the maintainer has not yet pinned, and we don't
|
||||
want to trust them via this layer.
|
||||
"""
|
||||
if not _TOR_DIGEST_FILE.exists():
|
||||
return {}
|
||||
try:
|
||||
import json as _json
|
||||
raw = _json.loads(_TOR_DIGEST_FILE.read_text(encoding="utf-8"))
|
||||
except Exception as exc:
|
||||
logger.warning("Tor bundle digests file unreadable: %s", exc)
|
||||
return {}
|
||||
result: dict[str, str] = {}
|
||||
for k, v in raw.items():
|
||||
if not isinstance(k, str) or k.startswith("_"):
|
||||
continue
|
||||
if not isinstance(v, str) or v == _DIGEST_PLACEHOLDER:
|
||||
continue
|
||||
result[k] = v.strip().lower()
|
||||
return result
|
||||
|
||||
|
||||
def _verify_tor_bundle(archive_path: Path, bundle_url: str) -> tuple[bool, str]:
|
||||
"""Verify the downloaded Tor bundle against any source we trust.
|
||||
|
||||
Returns (verified, reason). The bundle is considered verified if EITHER:
|
||||
|
||||
* The upstream ``.sha256sum`` file is reachable AND its digest matches
|
||||
what we just downloaded, OR
|
||||
* Our baked-in digest list (``backend/data/tor_bundle_digests.json``)
|
||||
contains this URL AND that digest matches.
|
||||
|
||||
If both sources are unavailable (e.g. fresh checkout before the
|
||||
maintainer has populated the digest file AND the upstream
|
||||
``.sha256sum`` is unreachable), we **fall back to HTTPS-only trust**
|
||||
with a warning so first-run onboarding does not break. As soon as the
|
||||
digest file is populated for a shipped Tor version, the secure path
|
||||
activates automatically — no operator action required.
|
||||
|
||||
Issue #201.
|
||||
"""
|
||||
import hashlib
|
||||
|
||||
actual_hash = hashlib.sha256(archive_path.read_bytes()).hexdigest().lower()
|
||||
|
||||
# Source 1: upstream .sha256sum
|
||||
upstream_hash: str | None = None
|
||||
sha256_url = bundle_url + ".sha256sum"
|
||||
sha256_file = TOR_INSTALL_DIR / "sha256sum.txt"
|
||||
try:
|
||||
urlretrieve(sha256_url, str(sha256_file))
|
||||
upstream_hash = sha256_file.read_text().strip().split()[0].lower()
|
||||
sha256_file.unlink(missing_ok=True)
|
||||
except Exception as hash_err:
|
||||
logger.info("Tor bundle upstream .sha256sum unreachable: %s", hash_err)
|
||||
sha256_file.unlink(missing_ok=True)
|
||||
|
||||
if upstream_hash and upstream_hash == actual_hash:
|
||||
return True, f"verified via upstream .sha256sum ({actual_hash[:16]}...)"
|
||||
|
||||
# Source 2: baked-in digest list
|
||||
baked = _load_baked_in_digests()
|
||||
baked_hash = baked.get(bundle_url)
|
||||
if baked_hash and baked_hash == actual_hash:
|
||||
return True, f"verified via baked-in digest list ({actual_hash[:16]}...)"
|
||||
|
||||
# If we got an upstream digest AND a baked-in digest AND neither
|
||||
# matched, the bundle is genuinely suspect — refuse it.
|
||||
if upstream_hash and baked_hash:
|
||||
return False, (
|
||||
f"SHA-256 mismatch: archive={actual_hash[:16]}..., "
|
||||
f"upstream={upstream_hash[:16]}..., baked={baked_hash[:16]}..."
|
||||
)
|
||||
if upstream_hash and upstream_hash != actual_hash:
|
||||
return False, (
|
||||
f"SHA-256 mismatch vs upstream: archive={actual_hash[:16]}..., "
|
||||
f"upstream={upstream_hash[:16]}..."
|
||||
)
|
||||
if baked_hash and baked_hash != actual_hash:
|
||||
return False, (
|
||||
f"SHA-256 mismatch vs baked-in digest: archive={actual_hash[:16]}..., "
|
||||
f"expected={baked_hash[:16]}..."
|
||||
)
|
||||
|
||||
# Neither verification source available. This is the fallback path for
|
||||
# the case where the upstream .sha256sum is temporarily unreachable
|
||||
# AND the maintainer hasn't yet pinned this Tor version. Trust HTTPS
|
||||
# only (current behavior pre-#201) with a clear warning. Onboarding
|
||||
# works; once we populate the digest file, the secure path activates.
|
||||
logger.warning(
|
||||
"Tor bundle integrity check fell back to HTTPS-only trust "
|
||||
"(upstream .sha256sum unreachable AND no baked-in digest for %s). "
|
||||
"Add this URL's SHA-256 to backend/data/tor_bundle_digests.json "
|
||||
"to enable the secure path.",
|
||||
bundle_url,
|
||||
)
|
||||
return True, f"https-only (no digest source reachable, archive={actual_hash[:16]}...)"
|
||||
|
||||
|
||||
def _extract_tor_bundle_safely(archive_path: Path, install_dir: Path) -> bool:
|
||||
"""Extract a Tor Expert Bundle tar.gz safely.
|
||||
|
||||
Issue #251: the previous extractor checked tarinfo.name against path
|
||||
traversal but never inspected tarinfo.linkname for symlink/hardlink
|
||||
members. Python 3.11's tarfile honors symlinks during extractall(),
|
||||
so a malicious archive could ship a member like::
|
||||
|
||||
name = "innocent.txt" # passes the path check
|
||||
type = SYMTYPE
|
||||
linkname = "C:\\Windows\\System32\\config\\system"
|
||||
|
||||
and extractall() would then create that symlink. Subsequent reads
|
||||
of innocent.txt deference to a sensitive system file; subsequent
|
||||
writes corrupt one. Tor bundles never legitimately contain symlinks
|
||||
or hardlinks, so we refuse all link members categorically rather
|
||||
than trying to validate linkname targets (which has its own pitfalls
|
||||
around relative path resolution).
|
||||
|
||||
Also refuses non-regular-non-directory members (devices, FIFOs,
|
||||
character/block special files) for completeness — none of those
|
||||
belong in a Tor Expert Bundle and accepting them is a category of
|
||||
bug we don't need to debug later.
|
||||
|
||||
Returns True on success, False on rejection (and logs the reason).
|
||||
The caller is responsible for cleaning up the archive file.
|
||||
"""
|
||||
import tarfile
|
||||
|
||||
install_resolved = install_dir.resolve()
|
||||
|
||||
try:
|
||||
with tarfile.open(str(archive_path), "r:gz") as tar:
|
||||
for member in tar.getmembers():
|
||||
# Reject anything that isn't a regular file or directory.
|
||||
# Symlinks (SYMTYPE) and hardlinks (LNKTYPE) are the
|
||||
# path-traversal vectors; the others (CHRTYPE, BLKTYPE,
|
||||
# FIFOTYPE, CONTTYPE) have no legitimate use in a Tor
|
||||
# Expert Bundle.
|
||||
if member.issym() or member.islnk():
|
||||
logger.error(
|
||||
"Tor bundle extraction blocked: link member %s -> %s "
|
||||
"(symlinks/hardlinks are not allowed in Tor bundles; "
|
||||
"this archive is malformed or hostile)",
|
||||
member.name,
|
||||
member.linkname,
|
||||
)
|
||||
return False
|
||||
if not (member.isfile() or member.isdir()):
|
||||
logger.error(
|
||||
"Tor bundle extraction blocked: unexpected member type "
|
||||
"for %s (only regular files and directories are allowed)",
|
||||
member.name,
|
||||
)
|
||||
return False
|
||||
|
||||
# Path traversal check (preserves the original guard).
|
||||
try:
|
||||
member_path = (install_dir / member.name).resolve()
|
||||
except OSError as exc:
|
||||
logger.error(
|
||||
"Tor bundle extraction blocked: cannot resolve member "
|
||||
"path %s: %s",
|
||||
member.name,
|
||||
exc,
|
||||
)
|
||||
return False
|
||||
try:
|
||||
member_path.relative_to(install_resolved)
|
||||
except ValueError:
|
||||
logger.error(
|
||||
"Tor bundle extraction blocked: path traversal on %s "
|
||||
"(resolves to %s, outside install dir %s)",
|
||||
member.name,
|
||||
member_path,
|
||||
install_resolved,
|
||||
)
|
||||
return False
|
||||
|
||||
# All members validated — extract.
|
||||
tar.extractall(path=str(install_dir))
|
||||
except tarfile.TarError as exc:
|
||||
logger.error("Tor bundle extraction failed: malformed tar (%s)", exc)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _auto_install_tor() -> str | None:
|
||||
"""Install or download Tor when it is safe to do so."""
|
||||
if os.name != "nt":
|
||||
@@ -79,37 +276,24 @@ def _auto_install_tor() -> str | None:
|
||||
logger.info("Downloading Tor Expert Bundle over HTTPS from %s...", bundle_url)
|
||||
urlretrieve(bundle_url, str(archive_path))
|
||||
|
||||
sha256_url = bundle_url + ".sha256sum"
|
||||
sha256_file = TOR_INSTALL_DIR / "sha256sum.txt"
|
||||
try:
|
||||
urlretrieve(sha256_url, str(sha256_file))
|
||||
expected_hash = sha256_file.read_text().strip().split()[0].lower()
|
||||
import hashlib
|
||||
|
||||
actual_hash = hashlib.sha256(archive_path.read_bytes()).hexdigest().lower()
|
||||
sha256_file.unlink(missing_ok=True)
|
||||
if actual_hash != expected_hash:
|
||||
logger.error("SHA-256 mismatch for Tor download. Expected %s, got %s", expected_hash, actual_hash)
|
||||
archive_path.unlink(missing_ok=True)
|
||||
continue
|
||||
logger.info("SHA-256 verified: %s", actual_hash[:16] + "...")
|
||||
except Exception as hash_err:
|
||||
logger.warning(
|
||||
"Could not verify SHA-256 (hash file unavailable): %s; proceeding with HTTPS-only verification",
|
||||
hash_err,
|
||||
)
|
||||
# Issue #201: multi-source verification. If neither upstream
|
||||
# .sha256sum nor a baked-in digest matches, we refuse this URL
|
||||
# and try the next one in _TOR_EXPERT_BUNDLE_URLS. If neither
|
||||
# source is reachable at all, we fall back to HTTPS-only trust
|
||||
# (current behavior) rather than blocking onboarding.
|
||||
verified, reason = _verify_tor_bundle(archive_path, bundle_url)
|
||||
if not verified:
|
||||
logger.error("Tor bundle verification failed for %s: %s", bundle_url, reason)
|
||||
archive_path.unlink(missing_ok=True)
|
||||
continue
|
||||
logger.info("Tor bundle %s", reason)
|
||||
|
||||
logger.info("Download complete, extracting...")
|
||||
import tarfile
|
||||
|
||||
with tarfile.open(str(archive_path), "r:gz") as tar:
|
||||
for member in tar.getmembers():
|
||||
member_path = (TOR_INSTALL_DIR / member.name).resolve()
|
||||
if not str(member_path).startswith(str(TOR_INSTALL_DIR.resolve())):
|
||||
logger.error("Tar path traversal blocked: %s", member.name)
|
||||
archive_path.unlink(missing_ok=True)
|
||||
return None
|
||||
tar.extractall(path=str(TOR_INSTALL_DIR))
|
||||
if not _extract_tor_bundle_safely(archive_path, TOR_INSTALL_DIR):
|
||||
archive_path.unlink(missing_ok=True)
|
||||
return None
|
||||
|
||||
archive_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
+232
-14
@@ -6,9 +6,11 @@ Public API:
|
||||
schedule_restart(project_root) (spawn detached start script, then exit)
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
@@ -29,6 +31,19 @@ DOCKER_UPDATE_COMMANDS = (
|
||||
"docker compose pull && docker compose up -d"
|
||||
)
|
||||
|
||||
# Issue #231: baked-in release digests. Loaded lazily, used as a fallback
|
||||
# verification source when the release's SHA256SUMS.txt asset can't be
|
||||
# fetched (e.g. transient network failure during update).
|
||||
_RELEASE_DIGESTS_FILE = (
|
||||
Path(__file__).resolve().parent.parent / "data" / "release_digests.json"
|
||||
)
|
||||
# Pattern for the maintainer's signed source-archive release asset. This
|
||||
# is the file we prefer over the auto-generated ``zipball_url`` because
|
||||
# the maintainer's build process publishes it with a matching entry in
|
||||
# SHA256SUMS.txt — the zipball does not have a signed digest.
|
||||
_SOURCE_ASSET_PATTERN = re.compile(r"^ShadowBroker_v\d", re.IGNORECASE)
|
||||
_SHA256SUMS_ASSET_NAME = "SHA256SUMS.txt"
|
||||
|
||||
|
||||
def _is_docker() -> bool:
|
||||
"""Detect if we're running inside a Docker container."""
|
||||
@@ -40,7 +55,6 @@ def _is_docker() -> bool:
|
||||
except (FileNotFoundError, PermissionError):
|
||||
pass
|
||||
return os.environ.get("container") == "docker"
|
||||
_EXPECTED_SHA256 = os.environ.get("MESH_UPDATE_SHA256", "").strip().lower()
|
||||
_ALLOWED_UPDATE_HOSTS = {
|
||||
"api.github.com",
|
||||
"codeload.github.com",
|
||||
@@ -119,7 +133,16 @@ def _validate_update_url(url: str, *, allow_release_page: bool = False) -> str:
|
||||
# ---------------------------------------------------------------------------
|
||||
def _download_release(temp_dir: str) -> tuple:
|
||||
"""Fetch latest release info and download the source zip archive.
|
||||
Returns (zip_path, version_tag, download_url, release_url).
|
||||
|
||||
Issue #231: prefer the maintainer's signed release asset (matching
|
||||
``ShadowBroker_v*.zip``) over the auto-generated ``zipball_url``,
|
||||
because the maintainer's release process publishes a matching entry
|
||||
in SHA256SUMS.txt for the named asset but NOT for the zipball.
|
||||
|
||||
Returns (zip_path, version_tag, download_url, release_url, asset_name,
|
||||
sha256sums_url) — the last two are empty strings when the release
|
||||
doesn't publish a signed asset, falling back to the legacy zipball
|
||||
path.
|
||||
"""
|
||||
logger.info("Fetching latest release info from GitHub...")
|
||||
_validate_update_url(GITHUB_RELEASES_URL)
|
||||
@@ -131,9 +154,42 @@ def _download_release(temp_dir: str) -> tuple:
|
||||
tag = release.get("tag_name", "unknown")
|
||||
release_url = str(release.get("html_url") or GITHUB_RELEASES_PAGE_URL).strip()
|
||||
_validate_update_url(release_url, allow_release_page=True)
|
||||
zip_url = str(release.get("zipball_url") or "").strip()
|
||||
if not zip_url:
|
||||
raise RuntimeError("Latest release is missing a source archive URL")
|
||||
|
||||
# Prefer the maintainer-signed release asset. Fall back to the
|
||||
# auto-generated zipball if the release doesn't publish one.
|
||||
assets = release.get("assets") or []
|
||||
asset_name = ""
|
||||
asset_url = ""
|
||||
sha256sums_url = ""
|
||||
for a in assets:
|
||||
name = str(a.get("name") or "").strip()
|
||||
download = str(a.get("browser_download_url") or "").strip()
|
||||
if not name or not download:
|
||||
continue
|
||||
if _SOURCE_ASSET_PATTERN.match(name) and name.lower().endswith(".zip"):
|
||||
asset_name = name
|
||||
asset_url = download
|
||||
elif name == _SHA256SUMS_ASSET_NAME:
|
||||
sha256sums_url = download
|
||||
|
||||
if asset_url:
|
||||
zip_url = asset_url
|
||||
logger.info(
|
||||
"Using signed release asset %s (sha256sums=%s)",
|
||||
asset_name,
|
||||
"yes" if sha256sums_url else "no",
|
||||
)
|
||||
else:
|
||||
zip_url = str(release.get("zipball_url") or "").strip()
|
||||
if not zip_url:
|
||||
raise RuntimeError("Latest release is missing a source archive URL")
|
||||
logger.warning(
|
||||
"Release does not publish a signed ShadowBroker_v*.zip asset — "
|
||||
"falling back to auto-generated zipball_url. Integrity will be "
|
||||
"verified against the baked-in release_digests.json (if present) "
|
||||
"or HTTPS-only otherwise."
|
||||
)
|
||||
|
||||
_validate_update_url(zip_url)
|
||||
|
||||
logger.info(f"Downloading {zip_url} ...")
|
||||
@@ -150,19 +206,174 @@ def _download_release(temp_dir: str) -> tuple:
|
||||
|
||||
size_mb = os.path.getsize(zip_path) / (1024 * 1024)
|
||||
logger.info(f"Downloaded {size_mb:.1f} MB — ZIP validated OK")
|
||||
return zip_path, tag, zip_url, release_url
|
||||
return zip_path, tag, zip_url, release_url, asset_name, sha256sums_url
|
||||
|
||||
|
||||
def _validate_zip_hash(zip_path: str) -> None:
|
||||
if not _EXPECTED_SHA256:
|
||||
return
|
||||
def _compute_sha256(zip_path: str) -> str:
|
||||
"""Return the hex SHA-256 of the file at ``zip_path`` (lowercase)."""
|
||||
h = hashlib.sha256()
|
||||
with open(zip_path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 128), b""):
|
||||
h.update(chunk)
|
||||
digest = h.hexdigest().lower()
|
||||
if digest != _EXPECTED_SHA256:
|
||||
raise RuntimeError("Update SHA-256 mismatch")
|
||||
return h.hexdigest().lower()
|
||||
|
||||
|
||||
def _load_baked_in_release_digests() -> dict:
|
||||
"""Return the ``release_digests.json`` mapping, or an empty dict.
|
||||
|
||||
Schema (issue #231):
|
||||
{
|
||||
"<release_tag>": {
|
||||
"<asset_filename>": "<sha256_hex>",
|
||||
...
|
||||
},
|
||||
...
|
||||
}
|
||||
"""
|
||||
try:
|
||||
raw = _RELEASE_DIGESTS_FILE.read_text(encoding="utf-8")
|
||||
parsed = json.loads(raw)
|
||||
except (OSError, ValueError) as exc:
|
||||
logger.debug("Release digest file unreadable: %s", exc)
|
||||
return {}
|
||||
if not isinstance(parsed, dict):
|
||||
return {}
|
||||
cleaned: dict[str, dict[str, str]] = {}
|
||||
for k, v in parsed.items():
|
||||
if not isinstance(k, str) or k.startswith("_"):
|
||||
continue
|
||||
if isinstance(v, dict):
|
||||
entries = {
|
||||
fname: digest.strip().lower()
|
||||
for fname, digest in v.items()
|
||||
if isinstance(fname, str) and isinstance(digest, str)
|
||||
}
|
||||
if entries:
|
||||
cleaned[k] = entries
|
||||
return cleaned
|
||||
|
||||
|
||||
def _fetch_sha256sums(sha256sums_url: str) -> dict[str, str]:
|
||||
"""Download a SHA256SUMS.txt and return {filename: digest_hex_lower}.
|
||||
|
||||
Standard ``sha256sum`` format: ``<digest> <filename>`` per line. The
|
||||
leading ``*`` binary-mode marker (e.g. ``<digest> *<filename>``) is
|
||||
handled.
|
||||
"""
|
||||
try:
|
||||
_validate_update_url(sha256sums_url)
|
||||
except RuntimeError as exc:
|
||||
logger.warning("SHA256SUMS URL rejected: %s", exc)
|
||||
return {}
|
||||
try:
|
||||
resp = requests.get(sha256sums_url, timeout=15)
|
||||
resp.raise_for_status()
|
||||
except requests.RequestException as exc:
|
||||
logger.info("SHA256SUMS fetch failed: %s", exc)
|
||||
return {}
|
||||
out: dict[str, str] = {}
|
||||
for line in resp.text.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
# Tolerant split: handle both `<digest> <name>` and `<digest> *<name>`.
|
||||
parts = line.split(None, 1)
|
||||
if len(parts) != 2:
|
||||
continue
|
||||
digest, fname = parts
|
||||
fname = fname.lstrip("*").strip()
|
||||
digest = digest.strip().lower()
|
||||
if len(digest) == 64 and all(c in "0123456789abcdef" for c in digest) and fname:
|
||||
out[fname] = digest
|
||||
return out
|
||||
|
||||
|
||||
def _validate_zip_hash(
|
||||
zip_path: str,
|
||||
*,
|
||||
asset_name: str = "",
|
||||
sha256sums_url: str = "",
|
||||
release_tag: str = "",
|
||||
) -> str:
|
||||
"""Verify the downloaded archive against trusted digest sources.
|
||||
|
||||
Issue #231: previously this returned silently when ``MESH_UPDATE_SHA256``
|
||||
was unset, which made the auto-updater a supply-chain RCE vector on any
|
||||
compromise of the GitHub release pipeline. The chain now is:
|
||||
|
||||
1. ``MESH_UPDATE_SHA256`` env var (operator override — preserved for
|
||||
power-users who want to pin an exact digest manually)
|
||||
2. ``SHA256SUMS.txt`` release asset (primary — the maintainer's
|
||||
release process already publishes this)
|
||||
3. Baked-in ``backend/data/release_digests.json`` (second line of
|
||||
defense for releases that lack the SHA256SUMS asset, or when the
|
||||
asset can't be fetched at update time)
|
||||
4. HTTPS-only fallback with a loud warning (preserves the auto-update
|
||||
flow during transient outages — but never silently)
|
||||
|
||||
A mismatch from a source that DID respond is fatal: the update is
|
||||
refused and the existing install keeps running. Only the "no source
|
||||
reachable at all" case falls back to HTTPS-only.
|
||||
|
||||
Returns a short human-readable description of which source verified
|
||||
the archive (used in the update-success message).
|
||||
"""
|
||||
actual = _compute_sha256(zip_path)
|
||||
|
||||
# Source 1: explicit operator override.
|
||||
override = os.environ.get("MESH_UPDATE_SHA256", "").strip().lower()
|
||||
if override:
|
||||
if actual == override:
|
||||
return f"verified via MESH_UPDATE_SHA256 ({actual[:16]}...)"
|
||||
raise RuntimeError(
|
||||
f"Update SHA-256 mismatch vs MESH_UPDATE_SHA256: archive={actual[:16]}..., "
|
||||
f"expected={override[:16]}..."
|
||||
)
|
||||
|
||||
# Source 2: SHA256SUMS.txt asset from the release.
|
||||
sums_map: dict[str, str] = {}
|
||||
if sha256sums_url and asset_name:
|
||||
sums_map = _fetch_sha256sums(sha256sums_url)
|
||||
|
||||
sums_expected = sums_map.get(asset_name) if asset_name else None
|
||||
if sums_expected:
|
||||
if actual == sums_expected:
|
||||
return f"verified via release SHA256SUMS.txt ({actual[:16]}...)"
|
||||
raise RuntimeError(
|
||||
f"Update SHA-256 mismatch vs release SHA256SUMS.txt: "
|
||||
f"archive={actual[:16]}..., expected={sums_expected[:16]}..."
|
||||
)
|
||||
|
||||
# Source 3: baked-in digest list.
|
||||
baked = _load_baked_in_release_digests()
|
||||
baked_expected = ""
|
||||
if release_tag and asset_name:
|
||||
baked_expected = baked.get(release_tag, {}).get(asset_name, "")
|
||||
if baked_expected:
|
||||
if actual == baked_expected:
|
||||
return f"verified via baked-in digest list ({actual[:16]}...)"
|
||||
raise RuntimeError(
|
||||
f"Update SHA-256 mismatch vs baked-in digest list: "
|
||||
f"archive={actual[:16]}..., expected={baked_expected[:16]}..."
|
||||
)
|
||||
|
||||
# Source 4: HTTPS-only fallback. We keep onboarding/auto-update working
|
||||
# during transient outages (no SHA256SUMS reachable AND no baked-in
|
||||
# entry for this release), but surface the degraded posture loudly so
|
||||
# the operator can see it in logs and the maintainer can populate the
|
||||
# digest list on the next release bump.
|
||||
logger.warning(
|
||||
"Update integrity check fell back to HTTPS-only trust "
|
||||
"(no SHA256SUMS.txt response and no baked-in digest for "
|
||||
"release=%s asset=%s). The archive SHA-256 is %s. Once the "
|
||||
"release ships a SHA256SUMS.txt asset OR backend/data/"
|
||||
"release_digests.json is updated with this release, the secure "
|
||||
"path will activate automatically.",
|
||||
release_tag or "unknown",
|
||||
asset_name or "unknown",
|
||||
actual,
|
||||
)
|
||||
return f"https-only (no digest source reachable, archive={actual[:16]}...)"
|
||||
|
||||
|
||||
def _is_source_checkout(project_root: str) -> bool:
|
||||
@@ -334,7 +545,7 @@ def perform_update(project_root: str) -> dict:
|
||||
temp_dir = tempfile.mkdtemp(prefix="sb_update_")
|
||||
manual_url = GITHUB_RELEASES_PAGE_URL
|
||||
try:
|
||||
zip_path, version, url, release_url = _download_release(temp_dir)
|
||||
zip_path, version, url, release_url, asset_name, sha256sums_url = _download_release(temp_dir)
|
||||
manual_url = release_url or manual_url
|
||||
|
||||
if in_docker:
|
||||
@@ -366,7 +577,13 @@ def perform_update(project_root: str) -> dict:
|
||||
),
|
||||
}
|
||||
|
||||
_validate_zip_hash(zip_path)
|
||||
verification_note = _validate_zip_hash(
|
||||
zip_path,
|
||||
asset_name=asset_name,
|
||||
sha256sums_url=sha256sums_url,
|
||||
release_tag=version,
|
||||
)
|
||||
logger.info("Update archive %s", verification_note)
|
||||
backup_path = _backup_current(project_root, temp_dir)
|
||||
copied = _extract_and_copy(zip_path, project_root, temp_dir)
|
||||
|
||||
@@ -378,6 +595,7 @@ def perform_update(project_root: str) -> dict:
|
||||
"manual_url": manual_url,
|
||||
"release_url": release_url,
|
||||
"download_url": url,
|
||||
"integrity": verification_note,
|
||||
"message": f"Updated to {version} — {copied} files replaced. Restarting...",
|
||||
}
|
||||
except Exception as e:
|
||||
|
||||
@@ -0,0 +1,677 @@
|
||||
{
|
||||
"_meta": {
|
||||
"issue": "#239",
|
||||
"note": "Snapshot of currently-tolerated duplicate route registrations. The test in test_no_new_duplicate_routes.py fails if any NEW (method, path) duplicate appears outside this list. Removing entries (by actually deduping) is fine and the test stays green. New entries here require explicit, reviewed updates.",
|
||||
"generated_with": "python -c 'see tests/test_no_new_duplicate_routes.py'"
|
||||
},
|
||||
"duplicates": {
|
||||
"DELETE /api/mesh/peers": [
|
||||
"main",
|
||||
"routers.mesh_operator",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"DELETE /api/wormhole/dm/contact/{peer_id}": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"DELETE /api/wormhole/dm/invite/handles/{handle}": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"GET /api/cctv/media": [
|
||||
"main",
|
||||
"routers.cctv"
|
||||
],
|
||||
"GET /api/debug-latest": [
|
||||
"main",
|
||||
"routers.health"
|
||||
],
|
||||
"GET /api/geocode/reverse": [
|
||||
"main",
|
||||
"routers.tools"
|
||||
],
|
||||
"GET /api/geocode/search": [
|
||||
"main",
|
||||
"routers.tools"
|
||||
],
|
||||
"GET /api/health": [
|
||||
"main",
|
||||
"routers.health"
|
||||
],
|
||||
"GET /api/live-data": [
|
||||
"main",
|
||||
"routers.data"
|
||||
],
|
||||
"GET /api/live-data/fast": [
|
||||
"main",
|
||||
"routers.data"
|
||||
],
|
||||
"GET /api/live-data/slow": [
|
||||
"main",
|
||||
"routers.data"
|
||||
],
|
||||
"GET /api/mesh/channels": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/dm/count": [
|
||||
"main",
|
||||
"routers.mesh_dm"
|
||||
],
|
||||
"GET /api/mesh/dm/poll": [
|
||||
"main",
|
||||
"routers.mesh_dm"
|
||||
],
|
||||
"GET /api/mesh/dm/prekey-bundle": [
|
||||
"main",
|
||||
"routers.mesh_dm"
|
||||
],
|
||||
"GET /api/mesh/dm/pubkey": [
|
||||
"main",
|
||||
"routers.mesh_dm"
|
||||
],
|
||||
"GET /api/mesh/dm/witness": [
|
||||
"main",
|
||||
"routers.mesh_dm"
|
||||
],
|
||||
"GET /api/mesh/gate/list": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/gate/{gate_id}": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/gate/{gate_id}/messages": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/infonet/event/{event_id}": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/infonet/events": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/infonet/locator": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/infonet/merkle": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/infonet/messages": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/infonet/messages/wait": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/infonet/node/{node_id}": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/infonet/status": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/infonet/sync": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/log": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/messages": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/metrics": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/oracle/consensus": [
|
||||
"main",
|
||||
"routers.mesh_oracle"
|
||||
],
|
||||
"GET /api/mesh/oracle/markets": [
|
||||
"main",
|
||||
"routers.mesh_oracle"
|
||||
],
|
||||
"GET /api/mesh/oracle/markets/more": [
|
||||
"main",
|
||||
"routers.mesh_oracle"
|
||||
],
|
||||
"GET /api/mesh/oracle/predictions": [
|
||||
"main",
|
||||
"routers.mesh_oracle"
|
||||
],
|
||||
"GET /api/mesh/oracle/profile": [
|
||||
"main",
|
||||
"routers.mesh_oracle"
|
||||
],
|
||||
"GET /api/mesh/oracle/search": [
|
||||
"main",
|
||||
"routers.mesh_oracle"
|
||||
],
|
||||
"GET /api/mesh/oracle/stakes/{message_id}": [
|
||||
"main",
|
||||
"routers.mesh_oracle"
|
||||
],
|
||||
"GET /api/mesh/peers": [
|
||||
"main",
|
||||
"routers.mesh_operator",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/reputation": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/reputation/all": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/reputation/batch": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/rns/status": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/signals": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/status": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"GET /api/mesh/trust/vouches": [
|
||||
"main",
|
||||
"routers.mesh_dm"
|
||||
],
|
||||
"GET /api/oracle/region-intel": [
|
||||
"main",
|
||||
"routers.sigint"
|
||||
],
|
||||
"GET /api/radio/nearest": [
|
||||
"main",
|
||||
"routers.radio"
|
||||
],
|
||||
"GET /api/radio/nearest-list": [
|
||||
"main",
|
||||
"routers.radio"
|
||||
],
|
||||
"GET /api/radio/openmhz/audio": [
|
||||
"main",
|
||||
"routers.radio"
|
||||
],
|
||||
"GET /api/radio/openmhz/calls/{sys_name}": [
|
||||
"main",
|
||||
"routers.radio"
|
||||
],
|
||||
"GET /api/radio/openmhz/systems": [
|
||||
"main",
|
||||
"routers.radio"
|
||||
],
|
||||
"GET /api/radio/top": [
|
||||
"main",
|
||||
"routers.radio"
|
||||
],
|
||||
"GET /api/refresh": [
|
||||
"main",
|
||||
"routers.data"
|
||||
],
|
||||
"GET /api/region-dossier": [
|
||||
"main",
|
||||
"routers.tools"
|
||||
],
|
||||
"GET /api/route/{callsign}": [
|
||||
"main",
|
||||
"routers.radio"
|
||||
],
|
||||
"GET /api/sentinel2/search": [
|
||||
"main",
|
||||
"routers.tools"
|
||||
],
|
||||
"GET /api/settings/api-keys": [
|
||||
"main",
|
||||
"routers.admin"
|
||||
],
|
||||
"GET /api/settings/api-keys/meta": [
|
||||
"main",
|
||||
"routers.admin"
|
||||
],
|
||||
"GET /api/settings/news-feeds": [
|
||||
"main",
|
||||
"routers.admin"
|
||||
],
|
||||
"GET /api/settings/node": [
|
||||
"main",
|
||||
"routers.admin"
|
||||
],
|
||||
"GET /api/settings/privacy-profile": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"GET /api/settings/wormhole": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"GET /api/settings/wormhole-status": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"GET /api/sigint/nearest-sdr": [
|
||||
"main",
|
||||
"routers.sigint"
|
||||
],
|
||||
"GET /api/thermal/verify": [
|
||||
"main",
|
||||
"routers.sigint"
|
||||
],
|
||||
"GET /api/tools/shodan/status": [
|
||||
"main",
|
||||
"routers.tools"
|
||||
],
|
||||
"GET /api/tools/uw/status": [
|
||||
"main",
|
||||
"routers.tools"
|
||||
],
|
||||
"GET /api/wormhole/dm/contacts": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"GET /api/wormhole/dm/identity": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"GET /api/wormhole/dm/invite": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"GET /api/wormhole/dm/invite/handles": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"GET /api/wormhole/gate/{gate_id}/identity": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"GET /api/wormhole/gate/{gate_id}/key": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"GET /api/wormhole/gate/{gate_id}/personas": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"GET /api/wormhole/health": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"GET /api/wormhole/identity": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"GET /api/wormhole/status": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"PATCH /api/mesh/peers": [
|
||||
"main",
|
||||
"routers.mesh_operator",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"POST /api/ais/feed": [
|
||||
"main",
|
||||
"routers.data"
|
||||
],
|
||||
"POST /api/layers": [
|
||||
"main",
|
||||
"routers.data"
|
||||
],
|
||||
"POST /api/mesh/dm/block": [
|
||||
"main",
|
||||
"routers.mesh_dm"
|
||||
],
|
||||
"POST /api/mesh/dm/count": [
|
||||
"main",
|
||||
"routers.mesh_dm"
|
||||
],
|
||||
"POST /api/mesh/dm/poll": [
|
||||
"main",
|
||||
"routers.mesh_dm"
|
||||
],
|
||||
"POST /api/mesh/dm/register": [
|
||||
"main",
|
||||
"routers.mesh_dm"
|
||||
],
|
||||
"POST /api/mesh/dm/send": [
|
||||
"main",
|
||||
"routers.mesh_dm"
|
||||
],
|
||||
"POST /api/mesh/dm/witness": [
|
||||
"main",
|
||||
"routers.mesh_dm"
|
||||
],
|
||||
"POST /api/mesh/gate/create": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"POST /api/mesh/gate/peer-pull": [
|
||||
"main",
|
||||
"routers.mesh_peer_sync"
|
||||
],
|
||||
"POST /api/mesh/gate/peer-push": [
|
||||
"main",
|
||||
"routers.mesh_peer_sync"
|
||||
],
|
||||
"POST /api/mesh/gate/{gate_id}/message": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"POST /api/mesh/identity/revoke": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"POST /api/mesh/identity/rotate": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"POST /api/mesh/infonet/ingest": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"POST /api/mesh/infonet/peer-push": [
|
||||
"main",
|
||||
"routers.mesh_peer_sync"
|
||||
],
|
||||
"POST /api/mesh/infonet/sync": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"POST /api/mesh/oracle/predict": [
|
||||
"main",
|
||||
"routers.mesh_oracle"
|
||||
],
|
||||
"POST /api/mesh/oracle/resolve": [
|
||||
"main",
|
||||
"routers.mesh_oracle"
|
||||
],
|
||||
"POST /api/mesh/oracle/resolve-stakes": [
|
||||
"main",
|
||||
"routers.mesh_oracle"
|
||||
],
|
||||
"POST /api/mesh/oracle/stake": [
|
||||
"main",
|
||||
"routers.mesh_oracle"
|
||||
],
|
||||
"POST /api/mesh/peers": [
|
||||
"main",
|
||||
"routers.mesh_operator",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"POST /api/mesh/report": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"POST /api/mesh/send": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"POST /api/mesh/trust/vouch": [
|
||||
"main",
|
||||
"routers.mesh_dm"
|
||||
],
|
||||
"POST /api/mesh/vote": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"POST /api/sentinel/tile": [
|
||||
"main",
|
||||
"routers.tools"
|
||||
],
|
||||
"POST /api/sentinel/token": [
|
||||
"main",
|
||||
"routers.tools"
|
||||
],
|
||||
"POST /api/settings/news-feeds/reset": [
|
||||
"main",
|
||||
"routers.admin"
|
||||
],
|
||||
"POST /api/sigint/transmit": [
|
||||
"main",
|
||||
"routers.sigint"
|
||||
],
|
||||
"POST /api/system/update": [
|
||||
"main",
|
||||
"routers.admin"
|
||||
],
|
||||
"POST /api/tools/shodan/count": [
|
||||
"main",
|
||||
"routers.tools"
|
||||
],
|
||||
"POST /api/tools/shodan/host": [
|
||||
"main",
|
||||
"routers.tools"
|
||||
],
|
||||
"POST /api/tools/shodan/search": [
|
||||
"main",
|
||||
"routers.tools"
|
||||
],
|
||||
"POST /api/tools/uw/congress": [
|
||||
"main",
|
||||
"routers.tools"
|
||||
],
|
||||
"POST /api/tools/uw/darkpool": [
|
||||
"main",
|
||||
"routers.tools"
|
||||
],
|
||||
"POST /api/tools/uw/flow": [
|
||||
"main",
|
||||
"routers.tools"
|
||||
],
|
||||
"POST /api/viewport": [
|
||||
"main",
|
||||
"routers.data"
|
||||
],
|
||||
"POST /api/wormhole/connect": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/disconnect": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/dm/bootstrap-decrypt": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/dm/bootstrap-encrypt": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/dm/build-seal": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/dm/compose": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/dm/dead-drop-token": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/dm/dead-drop-tokens": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/dm/decrypt": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/dm/encrypt": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/dm/invite/import": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/dm/open-seal": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/dm/pairwise-alias": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/dm/pairwise-alias/rotate": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/dm/prekey/register": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/dm/register-key": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/dm/reset": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/dm/sas": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/dm/sender-token": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/gate/enter": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/gate/key/grant": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/gate/key/rotate": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/gate/leave": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/gate/message/compose": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/gate/message/decrypt": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/gate/message/post": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/gate/message/post-encrypted": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/gate/message/sign-encrypted": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/gate/messages/decrypt": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/gate/persona/activate": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/gate/persona/clear": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/gate/persona/create": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/gate/persona/retire": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/gate/proof": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/gate/state/export": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/identity/bootstrap": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/join": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/leave": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/restart": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/sign": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"POST /api/wormhole/sign-raw": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"PUT /api/mesh/gate/{gate_id}/envelope_policy": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"PUT /api/mesh/gate/{gate_id}/legacy_envelope_fallback": [
|
||||
"main",
|
||||
"routers.mesh_public"
|
||||
],
|
||||
"PUT /api/settings/news-feeds": [
|
||||
"main",
|
||||
"routers.admin"
|
||||
],
|
||||
"PUT /api/settings/node": [
|
||||
"main",
|
||||
"routers.admin"
|
||||
],
|
||||
"PUT /api/settings/privacy-profile": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"PUT /api/settings/wormhole": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
],
|
||||
"PUT /api/wormhole/dm/contact": [
|
||||
"main",
|
||||
"routers.wormhole"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,7 @@ from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
|
||||
|
||||
def _request(path: str, method: str = "POST") -> Request:
|
||||
def _request(path: str, method: str = "POST", query_string: bytes = b"") -> Request:
|
||||
return Request(
|
||||
{
|
||||
"type": "http",
|
||||
@@ -13,6 +13,7 @@ def _request(path: str, method: str = "POST") -> Request:
|
||||
"client": ("test", 12345),
|
||||
"method": method,
|
||||
"path": path,
|
||||
"query_string": query_string,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -504,6 +505,61 @@ def test_private_infonet_gate_write_returns_preparing_state_when_wormhole_not_re
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def test_invite_scoped_prekey_lookup_reaches_handler_while_lane_prepares(monkeypatch):
|
||||
"""Copied-address import must not be blocked by private-lane warmup."""
|
||||
import main
|
||||
import auth
|
||||
from services.config import get_settings
|
||||
from services import wormhole_supervisor
|
||||
|
||||
monkeypatch.setenv("MESH_PRIVATE_CLEARNET_FALLBACK", "block")
|
||||
monkeypatch.setenv("MESH_BLOCK_LEGACY_NODE_ID_COMPAT", "true")
|
||||
monkeypatch.setenv("MESH_BLOCK_LEGACY_AGENT_ID_LOOKUP", "true")
|
||||
monkeypatch.setenv("MESH_ALLOW_COMPAT_DM_INVITE_IMPORT", "false")
|
||||
get_settings.cache_clear()
|
||||
monkeypatch.setattr(
|
||||
auth,
|
||||
"_anonymous_mode_state",
|
||||
lambda: {
|
||||
"enabled": False,
|
||||
"wormhole_enabled": True,
|
||||
"ready": False,
|
||||
"effective_transport": "direct",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
wormhole_supervisor,
|
||||
"get_wormhole_state",
|
||||
lambda: {
|
||||
"configured": True,
|
||||
"ready": False,
|
||||
"rns_ready": False,
|
||||
"arti_ready": False,
|
||||
},
|
||||
)
|
||||
|
||||
called = {"value": False}
|
||||
|
||||
async def call_next(_request: Request) -> Response:
|
||||
called["value"] = True
|
||||
return Response(status_code=200)
|
||||
|
||||
response = asyncio.run(
|
||||
main.enforce_high_privacy_mesh(
|
||||
_request(
|
||||
"/api/mesh/dm/prekey-bundle",
|
||||
method="GET",
|
||||
query_string=b"lookup_token=invite-handle",
|
||||
),
|
||||
call_next,
|
||||
)
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert called["value"] is True
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def test_private_dm_send_blocks_at_transitional_tier(monkeypatch):
|
||||
import main
|
||||
import auth
|
||||
|
||||
@@ -47,6 +47,11 @@ def test_infonet_ingest_accepts_valid_event(tmp_path, monkeypatch):
|
||||
|
||||
assert result["accepted"] == 1
|
||||
assert inf.head_hash == evt.event_id
|
||||
info = inf.get_info()
|
||||
assert info["known_nodes"] == 1
|
||||
assert info["author_nodes"] == 1
|
||||
assert info["total_events"] == 1
|
||||
assert info["event_types"]["message"] == 1
|
||||
|
||||
|
||||
def test_verify_node_binding_accepts_current_and_compat_ids_only(monkeypatch):
|
||||
@@ -64,6 +69,8 @@ def test_verify_node_binding_accepts_current_and_compat_ids_only(monkeypatch):
|
||||
f"{current[len(mesh_crypto.NODE_ID_PREFIX):len(mesh_crypto.NODE_ID_PREFIX) + 8]}"
|
||||
)
|
||||
|
||||
monkeypatch.setenv("MESH_DEV_ALLOW_LEGACY_COMPAT", "true")
|
||||
monkeypatch.setenv("MESH_BLOCK_LEGACY_NODE_ID_COMPAT", "false")
|
||||
monkeypatch.setenv("MESH_ALLOW_LEGACY_NODE_ID_COMPAT_UNTIL", "2099-01-01")
|
||||
from services.config import get_settings
|
||||
|
||||
@@ -98,7 +105,7 @@ def test_infonet_append_rejects_missing_signature_fields(tmp_path, monkeypatch):
|
||||
assert "signature" in str(exc).lower()
|
||||
|
||||
|
||||
def test_infonet_load_fails_closed_on_hash_mismatch(tmp_path, monkeypatch):
|
||||
def test_infonet_load_quarantines_and_resets_on_hash_mismatch(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(mesh_hashchain, "DATA_DIR", tmp_path)
|
||||
monkeypatch.setattr(mesh_hashchain, "CHAIN_FILE", tmp_path / "infonet.json")
|
||||
|
||||
@@ -135,8 +142,12 @@ def test_infonet_load_fails_closed_on_hash_mismatch(tmp_path, monkeypatch):
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="Hash mismatch on event load"):
|
||||
mesh_hashchain.Infonet()
|
||||
inf = mesh_hashchain.Infonet()
|
||||
|
||||
assert inf.events == []
|
||||
assert inf.head_hash == mesh_hashchain.GENESIS_HASH
|
||||
assert not mesh_hashchain.CHAIN_FILE.exists()
|
||||
assert list(tmp_path.glob("infonet.json.quarantine.*"))
|
||||
|
||||
|
||||
def test_validate_gate_message_payload_rejects_plaintext_shape():
|
||||
|
||||
@@ -12,6 +12,7 @@ Tests verify:
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
|
||||
from services.config import get_settings
|
||||
@@ -611,6 +612,99 @@ class TestFetchPrekeyBundleByLookup:
|
||||
"peer prekey lookup unavailable",
|
||||
}
|
||||
|
||||
def test_fetch_lookup_token_uses_bootstrap_peer_without_agent_id(self, tmp_path, monkeypatch):
|
||||
"""Invite lookup can resolve through bootstrap peers without exposing agent_id."""
|
||||
_isolated_relay(tmp_path, monkeypatch)
|
||||
record = _valid_bundle_record("test-agent")
|
||||
requested_urls: list[str] = []
|
||||
|
||||
monkeypatch.setenv("MESH_BOOTSTRAP_SEED_PEERS", "https://seed.example")
|
||||
monkeypatch.setenv("MESH_DEFAULT_SYNC_PEERS", "")
|
||||
monkeypatch.setenv("MESH_RELAY_PEERS", "")
|
||||
get_settings.cache_clear()
|
||||
|
||||
class _Response:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def read(self, _limit: int = -1):
|
||||
return json.dumps(
|
||||
{
|
||||
"ok": True,
|
||||
"identity_dh_pub_key": record["dh_pub_key"],
|
||||
"dh_algo": record["dh_algo"],
|
||||
"public_key": record["public_key"],
|
||||
"public_key_algo": record["public_key_algo"],
|
||||
"protocol_version": record["protocol_version"],
|
||||
"sequence": 1,
|
||||
"signed_at": int(record["bundle"].get("signed_at", 0) or 0),
|
||||
"bundle": record["bundle"],
|
||||
}
|
||||
).encode("utf-8")
|
||||
|
||||
def _urlopen(request, timeout=0):
|
||||
requested_urls.append(str(getattr(request, "full_url", "")))
|
||||
return _Response()
|
||||
|
||||
monkeypatch.setattr("services.mesh.mesh_wormhole_prekey.urllib.request.urlopen", _urlopen)
|
||||
|
||||
from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle
|
||||
|
||||
result = fetch_dm_prekey_bundle(agent_id="", lookup_token="bootstrap-handle")
|
||||
|
||||
assert result["ok"] is True
|
||||
assert result["agent_id"] == record["agent_id"]
|
||||
assert result["lookup_mode"] == "invite_lookup_handle"
|
||||
assert result["public_lookup"] is True
|
||||
assert requested_urls
|
||||
assert "lookup_token=bootstrap-handle" in requested_urls[0]
|
||||
assert "agent_id" not in requested_urls[0]
|
||||
|
||||
def test_fetch_lookup_token_does_not_parse_peer_pending_as_bundle(self, tmp_path, monkeypatch):
|
||||
"""A peer's private-lane pending response is not a malformed prekey bundle."""
|
||||
_isolated_relay(tmp_path, monkeypatch)
|
||||
requested_urls: list[str] = []
|
||||
|
||||
monkeypatch.setenv("MESH_BOOTSTRAP_SEED_PEERS", "https://seed.example")
|
||||
monkeypatch.setenv("MESH_DEFAULT_SYNC_PEERS", "")
|
||||
monkeypatch.setenv("MESH_RELAY_PEERS", "")
|
||||
get_settings.cache_clear()
|
||||
|
||||
class _Response:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def read(self, _limit: int = -1):
|
||||
return json.dumps(
|
||||
{
|
||||
"ok": True,
|
||||
"pending": True,
|
||||
"status": "preparing_private_lane",
|
||||
"detail": "transport tier insufficient",
|
||||
}
|
||||
).encode("utf-8")
|
||||
|
||||
def _urlopen(request, timeout=0):
|
||||
requested_urls.append(str(getattr(request, "full_url", "")))
|
||||
return _Response()
|
||||
|
||||
monkeypatch.setattr("services.mesh.mesh_wormhole_prekey.urllib.request.urlopen", _urlopen)
|
||||
|
||||
from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle
|
||||
|
||||
result = fetch_dm_prekey_bundle(agent_id="", lookup_token="bootstrap-handle")
|
||||
|
||||
assert requested_urls
|
||||
assert result["ok"] is False
|
||||
assert result["detail"] == "peer prekey lookup still preparing"
|
||||
assert result["detail"] != "Prekey bundle missing signing key"
|
||||
|
||||
def test_fetch_agent_id_uses_pinned_contact_lookup_handle(self, tmp_path, monkeypatch):
|
||||
"""Pinned invite lookup handle is used before direct agent_id lookup."""
|
||||
relay = _isolated_relay(tmp_path, monkeypatch)
|
||||
|
||||
@@ -71,6 +71,38 @@ def _fresh_wormhole_state(tmp_path, monkeypatch):
|
||||
return relay, mesh_wormhole_identity, mesh_wormhole_contacts, mesh_wormhole_prekey
|
||||
|
||||
|
||||
def test_register_wormhole_dm_key_repairs_missing_local_dh_material(tmp_path, monkeypatch):
|
||||
relay, identity_mod, _contacts_mod, _prekey_mod = _fresh_wormhole_state(tmp_path, monkeypatch)
|
||||
identity = identity_mod.read_wormhole_identity()
|
||||
original_node_id = identity["node_id"]
|
||||
original_public_key = identity["public_key"]
|
||||
original_private_key = identity["private_key"]
|
||||
|
||||
identity_mod.write_dm_identity(
|
||||
{
|
||||
**identity,
|
||||
"dh_pub_key": "",
|
||||
"dh_private_key": "",
|
||||
"bundle_fingerprint": "",
|
||||
"bundle_sequence": 0,
|
||||
"bundle_registered_at": 0,
|
||||
}
|
||||
)
|
||||
|
||||
registered = identity_mod.register_wormhole_dm_key()
|
||||
repaired = identity_mod.read_wormhole_identity()
|
||||
|
||||
assert registered["ok"] is True
|
||||
assert registered["dh_pub_key"]
|
||||
assert registered["dh_algo"] == "X25519"
|
||||
assert repaired["dh_pub_key"] == registered["dh_pub_key"]
|
||||
assert repaired["dh_private_key"]
|
||||
assert repaired["node_id"] == original_node_id
|
||||
assert repaired["public_key"] == original_public_key
|
||||
assert repaired["private_key"] == original_private_key
|
||||
assert relay.get_dh_key(original_node_id)["dh_pub_key"] == registered["dh_pub_key"]
|
||||
|
||||
|
||||
def _export_verified_invite(identity_mod):
|
||||
exported = identity_mod.export_wormhole_dm_invite()
|
||||
assert exported["ok"] is True
|
||||
@@ -460,6 +492,30 @@ def test_imported_dm_invite_pins_contact_as_invite_pinned(tmp_path, monkeypatch)
|
||||
assert contacts_mod.list_wormhole_dm_contacts()[imported["peer_id"]]["trust_level"] == "invite_pinned"
|
||||
|
||||
|
||||
def test_imported_dm_invite_saves_pending_contact_when_prekey_not_visible(tmp_path, monkeypatch):
|
||||
_relay, identity_mod, contacts_mod, prekey_mod = _fresh_wormhole_state(tmp_path, monkeypatch)
|
||||
exported, verified = _export_verified_invite(identity_mod)
|
||||
monkeypatch.setattr(
|
||||
prekey_mod,
|
||||
"fetch_dm_prekey_bundle",
|
||||
lambda **_kw: {"ok": False, "detail": "Prekey bundle not found"},
|
||||
)
|
||||
|
||||
imported = identity_mod.import_wormhole_dm_invite(exported["invite"], alias="alice")
|
||||
contact = imported["contact"]
|
||||
|
||||
assert imported["ok"] is True
|
||||
assert imported["pending_prekey"] is True
|
||||
assert imported["peer_id"] == verified["peer_id"]
|
||||
assert contact["alias"] == "alice"
|
||||
assert contact["trust_level"] == "invite_pinned"
|
||||
assert contact["invitePinnedPrekeyLookupHandle"] == exported["invite"]["payload"]["prekey_lookup_handle"]
|
||||
assert contact["remotePrekeyLookupMode"] == "invite_lookup_handle"
|
||||
assert contact["remotePrekeyFingerprint"] == verified["trust_fingerprint"]
|
||||
assert contact["dhPubKey"] == ""
|
||||
assert contacts_mod.list_wormhole_dm_contacts()[verified["peer_id"]]["trust_level"] == "invite_pinned"
|
||||
|
||||
|
||||
def test_imported_dm_invite_requires_root_attested_prekey_bundle(tmp_path, monkeypatch):
|
||||
relay, identity_mod, _contacts_mod, _prekey_mod = _fresh_wormhole_state(tmp_path, monkeypatch)
|
||||
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Issue #258 — AIS proxy SPKI pinning.
|
||||
|
||||
Most of the SPKI logic lives in ``backend/ais_proxy.js`` (Node) and can't
|
||||
be unit-tested from Python directly. These tests cover the Python-side
|
||||
glue: ``services.ais_stream.ais_proxy_status()`` (the snapshot the proxy
|
||||
populates via stdout markers) and ``routers/health.py`` surfacing the
|
||||
degraded TLS state.
|
||||
|
||||
Additionally, the pin-file structure is validated: it must be parseable
|
||||
JSON, must contain an entry for ``stream.aisstream.io``, and each pin
|
||||
must look like a base64-encoded SHA-256 hash.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from services import ais_stream
|
||||
|
||||
PIN_FILE = (
|
||||
Path(__file__).resolve().parent.parent / "data" / "aisstream_spki_pins.json"
|
||||
)
|
||||
|
||||
|
||||
def test_pin_file_exists_and_is_valid_json():
|
||||
assert PIN_FILE.exists(), f"Expected pin file at {PIN_FILE}"
|
||||
data = json.loads(PIN_FILE.read_text(encoding="utf-8"))
|
||||
assert isinstance(data, dict)
|
||||
|
||||
|
||||
def test_pin_file_has_aisstream_entry():
|
||||
data = json.loads(PIN_FILE.read_text(encoding="utf-8"))
|
||||
pins = data.get("stream.aisstream.io")
|
||||
assert isinstance(pins, list)
|
||||
assert len(pins) >= 1
|
||||
|
||||
|
||||
def test_each_pin_looks_like_a_base64_sha256():
|
||||
"""SPKI pins must be 44-char base64-encoded SHA-256 digests."""
|
||||
data = json.loads(PIN_FILE.read_text(encoding="utf-8"))
|
||||
pins = data["stream.aisstream.io"]
|
||||
for pin in pins:
|
||||
assert isinstance(pin, str), f"pin not a string: {pin!r}"
|
||||
assert len(pin) == 44, f"pin {pin!r} not 44 chars (SHA-256 base64)"
|
||||
# Must base64-decode to exactly 32 bytes (256 bits)
|
||||
try:
|
||||
raw = base64.b64decode(pin)
|
||||
except Exception as exc:
|
||||
pytest.fail(f"pin {pin!r} is not valid base64: {exc}")
|
||||
assert len(raw) == 32, f"pin {pin!r} decodes to {len(raw)} bytes, expected 32"
|
||||
# Should match the canonical base64 alphabet (no URL-safe variants)
|
||||
assert re.match(r"^[A-Za-z0-9+/]+=*$", pin), f"pin {pin!r} contains non-base64 chars"
|
||||
|
||||
|
||||
def test_ais_proxy_status_starts_empty():
|
||||
"""Before the proxy emits any status marker, the snapshot is empty."""
|
||||
# Clear any stale state from other tests
|
||||
with ais_stream._vessels_lock:
|
||||
ais_stream._proxy_status.clear()
|
||||
status = ais_stream.ais_proxy_status()
|
||||
assert status == {}
|
||||
|
||||
|
||||
def test_ais_proxy_status_returns_copy_not_reference():
|
||||
"""ais_proxy_status() must return a defensive copy.
|
||||
|
||||
Otherwise a caller could mutate the live dict and confuse later reads.
|
||||
"""
|
||||
with ais_stream._vessels_lock:
|
||||
ais_stream._proxy_status.clear()
|
||||
ais_stream._proxy_status["degraded_tls"] = True
|
||||
|
||||
snapshot = ais_stream.ais_proxy_status()
|
||||
assert snapshot == {"degraded_tls": True}
|
||||
snapshot["degraded_tls"] = False # mutate the returned copy
|
||||
|
||||
# Original should be untouched
|
||||
re_snapshot = ais_stream.ais_proxy_status()
|
||||
assert re_snapshot == {"degraded_tls": True}
|
||||
|
||||
# Cleanup so other tests start clean
|
||||
with ais_stream._vessels_lock:
|
||||
ais_stream._proxy_status.clear()
|
||||
|
||||
|
||||
def test_health_includes_ais_proxy_field(client):
|
||||
"""The /api/health response must include the ais_proxy block."""
|
||||
# Inject a known degraded state
|
||||
with ais_stream._vessels_lock:
|
||||
ais_stream._proxy_status.clear()
|
||||
ais_stream._proxy_status["degraded_tls"] = True
|
||||
|
||||
response = client.get("/api/health")
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
|
||||
assert "ais_proxy" in payload
|
||||
assert payload["ais_proxy"] == {"degraded_tls": True}
|
||||
# Top-level status should escalate from ok to degraded when AIS is
|
||||
# in degraded-TLS mode (unless SLOs already report worse).
|
||||
assert payload["status"] in {"degraded", "error"}
|
||||
|
||||
# Cleanup
|
||||
with ais_stream._vessels_lock:
|
||||
ais_stream._proxy_status.clear()
|
||||
|
||||
|
||||
def test_health_ais_proxy_field_when_no_status(client):
|
||||
"""When the proxy hasn't reported anything yet, ais_proxy is empty."""
|
||||
with ais_stream._vessels_lock:
|
||||
ais_stream._proxy_status.clear()
|
||||
|
||||
response = client.get("/api/health")
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload.get("ais_proxy") == {}
|
||||
@@ -0,0 +1,389 @@
|
||||
"""Issues #244, #245, #246 (tg12 external audit): carrier tracker
|
||||
quality + provenance + freshness.
|
||||
|
||||
These tests pin the post-fix contract:
|
||||
|
||||
- **#244**: dated editorial snapshot positions no longer live in the
|
||||
registry. They live in a one-shot seed file that is consumed once
|
||||
on first-ever startup. After that, the runtime cache reflects only
|
||||
what THIS install has actually observed.
|
||||
|
||||
- **#245**: headline-derived positions (centroid of a region keyword)
|
||||
are stamped ``position_confidence = "approximate"`` so the UI can
|
||||
render them with appropriate uncertainty.
|
||||
|
||||
- **#246**: freshness is a *labelling* decision, not an eviction
|
||||
decision. Positions older than the configurable freshness window
|
||||
flip from ``"recent"`` to ``"stale"`` but are NEVER replaced with
|
||||
the registry default — that would teleport the carrier. The user
|
||||
always sees the last position the system actually observed.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fresh_tracker(tmp_path, monkeypatch):
|
||||
"""Isolated carrier_tracker with seed/cache paths redirected to tmp.
|
||||
|
||||
Yields the module so tests can call its functions; resets globals
|
||||
between tests so position caches don't leak across cases.
|
||||
"""
|
||||
from services import carrier_tracker
|
||||
|
||||
seed_path = tmp_path / "data" / "carrier_seed.json"
|
||||
cache_path = tmp_path / "carrier_cache.json"
|
||||
seed_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
monkeypatch.setattr(carrier_tracker, "SEED_FILE", seed_path)
|
||||
monkeypatch.setattr(carrier_tracker, "CACHE_FILE", cache_path)
|
||||
monkeypatch.delenv("SHADOWBROKER_CARRIER_FRESHNESS_DAYS", raising=False)
|
||||
|
||||
# Reset module-level mutable state.
|
||||
carrier_tracker._carrier_positions.clear()
|
||||
carrier_tracker._cached_gdelt_articles.clear()
|
||||
carrier_tracker._last_gdelt_fetch_at = 0.0
|
||||
|
||||
yield carrier_tracker
|
||||
|
||||
# Clean up so subsequent tests start fresh.
|
||||
carrier_tracker._carrier_positions.clear()
|
||||
carrier_tracker._cached_gdelt_articles.clear()
|
||||
|
||||
|
||||
def _write_seed(path: Path, hull: str = "CVN-78", **overrides) -> None:
|
||||
payload = {
|
||||
"_meta": {
|
||||
"as_of": "2026-03-09",
|
||||
"source": "USNI News Fleet & Marine Tracker",
|
||||
"source_url": "https://news.usni.org/...",
|
||||
"note": "test",
|
||||
},
|
||||
"carriers": {
|
||||
hull: {
|
||||
"lat": 18.0,
|
||||
"lng": 39.5,
|
||||
"heading": 0,
|
||||
"desc": "Red Sea — Operation Epic Fury (USNI Mar 9)",
|
||||
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||
"position_source_at": "2026-03-09T00:00:00Z",
|
||||
"position_confidence": "seed",
|
||||
**overrides,
|
||||
}
|
||||
},
|
||||
}
|
||||
path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #244 — first-run seed bootstrap, never re-seeds after that
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSeedBootstrap:
|
||||
def test_first_ever_startup_bootstraps_from_seed(self, fresh_tracker, tmp_path):
|
||||
_write_seed(fresh_tracker.SEED_FILE)
|
||||
# No cache exists yet.
|
||||
assert not fresh_tracker.CACHE_FILE.exists()
|
||||
|
||||
positions = fresh_tracker._bootstrap_cache_if_missing()
|
||||
|
||||
# The seed entry made it into the cache.
|
||||
assert "CVN-78" in positions
|
||||
assert positions["CVN-78"]["lat"] == 18.0
|
||||
assert positions["CVN-78"]["position_confidence"] == "seed"
|
||||
# And the cache file is now on disk so subsequent runs skip the seed.
|
||||
assert fresh_tracker.CACHE_FILE.exists()
|
||||
|
||||
def test_subsequent_startup_ignores_seed(self, fresh_tracker, tmp_path):
|
||||
# Pre-seed a different position into the cache; the seed file says Red Sea.
|
||||
cache_data = {
|
||||
"CVN-78": {
|
||||
"lat": 25.0,
|
||||
"lng": 55.0,
|
||||
"heading": 0,
|
||||
"desc": "Persian Gulf — operator-observed",
|
||||
"source": "Operator log",
|
||||
"source_url": "",
|
||||
"position_source_at": "2026-04-15T12:00:00Z",
|
||||
"position_confidence": "recent",
|
||||
}
|
||||
}
|
||||
fresh_tracker.CACHE_FILE.write_text(json.dumps(cache_data))
|
||||
_write_seed(fresh_tracker.SEED_FILE) # seed is present but should NOT be used
|
||||
|
||||
positions = fresh_tracker._bootstrap_cache_if_missing()
|
||||
|
||||
assert positions["CVN-78"]["lat"] == 25.0
|
||||
assert positions["CVN-78"]["desc"] == "Persian Gulf — operator-observed"
|
||||
|
||||
def test_no_seed_no_cache_falls_back_to_homeport(self, fresh_tracker):
|
||||
# Neither seed nor cache. Must fall back to homeport defaults
|
||||
# (carrier never disappears).
|
||||
assert not fresh_tracker.SEED_FILE.exists()
|
||||
assert not fresh_tracker.CACHE_FILE.exists()
|
||||
|
||||
positions = fresh_tracker._bootstrap_cache_if_missing()
|
||||
|
||||
# Every registered carrier has SOMETHING.
|
||||
assert set(positions.keys()) == set(fresh_tracker.CARRIER_REGISTRY.keys())
|
||||
# All entries are labelled as homeport defaults.
|
||||
for hull, entry in positions.items():
|
||||
assert entry["position_confidence"] == "homeport_default"
|
||||
registry = fresh_tracker.CARRIER_REGISTRY[hull]
|
||||
assert entry["lat"] == registry["homeport_lat"]
|
||||
assert entry["lng"] == registry["homeport_lng"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #244 — no editorial fallbacks live in the registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRegistryShape:
|
||||
def test_registry_has_no_dated_fallback_fields(self, fresh_tracker):
|
||||
"""The Mar 9 editorial coordinates are gone from the registry.
|
||||
They live only in the seed file."""
|
||||
forbidden = {"fallback_lat", "fallback_lng", "fallback_heading", "fallback_desc"}
|
||||
for hull, entry in fresh_tracker.CARRIER_REGISTRY.items():
|
||||
offending = forbidden & set(entry.keys())
|
||||
assert not offending, f"{hull} still has dated registry fields: {offending}"
|
||||
|
||||
def test_registry_keeps_homeport_for_every_hull(self, fresh_tracker):
|
||||
for hull, entry in fresh_tracker.CARRIER_REGISTRY.items():
|
||||
assert "homeport_lat" in entry, f"{hull} missing homeport_lat"
|
||||
assert "homeport_lng" in entry, f"{hull} missing homeport_lng"
|
||||
assert "name" in entry
|
||||
assert "wiki" in entry
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #246 — freshness labelling, NOT eviction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFreshnessLabelling:
|
||||
def test_recent_observation_labels_recent(self, fresh_tracker):
|
||||
now = datetime(2026, 6, 1, tzinfo=timezone.utc)
|
||||
entry = {
|
||||
"lat": 25.0,
|
||||
"lng": 55.0,
|
||||
"position_source_at": (now - timedelta(days=3)).isoformat(),
|
||||
}
|
||||
assert fresh_tracker._compute_position_confidence(entry, now=now) == "recent"
|
||||
|
||||
def test_aged_observation_flips_to_stale(self, fresh_tracker):
|
||||
now = datetime(2026, 6, 1, tzinfo=timezone.utc)
|
||||
entry = {
|
||||
"lat": 25.0,
|
||||
"lng": 55.0,
|
||||
"position_source_at": (now - timedelta(days=30)).isoformat(),
|
||||
}
|
||||
assert fresh_tracker._compute_position_confidence(entry, now=now) == "stale"
|
||||
|
||||
def test_seed_label_is_preserved_explicitly(self, fresh_tracker):
|
||||
now = datetime(2026, 6, 1, tzinfo=timezone.utc)
|
||||
entry = {
|
||||
"lat": 18.0,
|
||||
"lng": 39.5,
|
||||
"position_source_at": "2026-03-09T00:00:00Z",
|
||||
"position_confidence": "seed",
|
||||
}
|
||||
# Even though the source is months old, the explicit "seed" label wins
|
||||
# so the UI can render the seed-specific badge instead of generic "stale".
|
||||
assert fresh_tracker._compute_position_confidence(entry, now=now) == "seed"
|
||||
|
||||
def test_homeport_default_label_is_preserved(self, fresh_tracker):
|
||||
now = datetime(2026, 6, 1, tzinfo=timezone.utc)
|
||||
entry = {
|
||||
"lat": 36.95,
|
||||
"lng": -76.32,
|
||||
"position_source_at": now.isoformat(),
|
||||
"position_confidence": "homeport_default",
|
||||
}
|
||||
assert fresh_tracker._compute_position_confidence(entry, now=now) == "homeport_default"
|
||||
|
||||
def test_freshness_window_is_env_configurable(self, fresh_tracker, monkeypatch):
|
||||
now = datetime(2026, 6, 1, tzinfo=timezone.utc)
|
||||
entry = {
|
||||
"lat": 25.0,
|
||||
"lng": 55.0,
|
||||
"position_source_at": (now - timedelta(days=20)).isoformat(),
|
||||
}
|
||||
# Default window = 14 days → 20-day-old entry is stale.
|
||||
assert fresh_tracker._compute_position_confidence(entry, now=now) == "stale"
|
||||
# Stretch to 30 days → same entry is now "recent".
|
||||
monkeypatch.setenv("SHADOWBROKER_CARRIER_FRESHNESS_DAYS", "30")
|
||||
assert fresh_tracker._compute_position_confidence(entry, now=now) == "recent"
|
||||
|
||||
def test_aged_cache_entry_keeps_its_position_never_reverts(self, fresh_tracker):
|
||||
"""The core regression test for the user's intent: a year-old
|
||||
cache entry must NOT be replaced with the seed or homeport.
|
||||
The PHYSICAL position the user sees is the last one observed;
|
||||
only the freshness LABEL changes."""
|
||||
a_year_ago = (datetime.now(timezone.utc) - timedelta(days=365)).isoformat()
|
||||
cache_data = {
|
||||
"CVN-78": {
|
||||
"lat": 25.0,
|
||||
"lng": 55.0,
|
||||
"heading": 0,
|
||||
"desc": "Persian Gulf",
|
||||
"source": "GDELT News API",
|
||||
"source_url": "https://news.example/...",
|
||||
"position_source_at": a_year_ago,
|
||||
"position_confidence": "recent", # was recent when written
|
||||
}
|
||||
}
|
||||
fresh_tracker.CACHE_FILE.write_text(json.dumps(cache_data))
|
||||
|
||||
positions = fresh_tracker._bootstrap_cache_if_missing()
|
||||
enriched = fresh_tracker._enrich_for_rendering("CVN-78", positions["CVN-78"])
|
||||
|
||||
# The position is preserved exactly.
|
||||
assert enriched["lat"] == 25.0
|
||||
assert enriched["lng"] == 55.0
|
||||
# But the live label has flipped to stale.
|
||||
assert enriched["position_confidence"] == "stale"
|
||||
assert enriched["is_fallback"] is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #245 — approximate confidence for region-centroid positions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestApproximateConfidenceForNewsDerivedPositions:
|
||||
def test_news_parsing_stamps_approximate_confidence(self, fresh_tracker):
|
||||
articles = [
|
||||
{
|
||||
"title": "USS Ford carrier deployed in Mediterranean for joint exercise",
|
||||
"url": "https://news.example/ford-mediterranean",
|
||||
"seendate": "20260415120000",
|
||||
}
|
||||
]
|
||||
updates = fresh_tracker._parse_carrier_positions_from_news(articles)
|
||||
assert "CVN-78" in updates
|
||||
entry = updates["CVN-78"]
|
||||
assert entry["position_confidence"] == "approximate"
|
||||
# And the source_at is the article's seen date, not now().
|
||||
assert entry["position_source_at"].startswith("2026-04-15")
|
||||
|
||||
def test_gdelt_seendate_parser_handles_well_formed_input(self, fresh_tracker):
|
||||
iso = fresh_tracker._gdelt_seendate_to_iso("20260415120000")
|
||||
assert iso is not None
|
||||
assert iso.startswith("2026-04-15T12:00:00")
|
||||
|
||||
def test_gdelt_seendate_parser_returns_none_on_garbage(self, fresh_tracker):
|
||||
assert fresh_tracker._gdelt_seendate_to_iso("") is None
|
||||
assert fresh_tracker._gdelt_seendate_to_iso("not-a-date") is None
|
||||
assert fresh_tracker._gdelt_seendate_to_iso("2026") is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Full enrichment → public API shape
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEnrichForRendering:
|
||||
def test_seed_entry_produces_expected_public_fields(self, fresh_tracker):
|
||||
seed_entry = {
|
||||
"lat": 18.0,
|
||||
"lng": 39.5,
|
||||
"heading": 0,
|
||||
"desc": "Red Sea (USNI Mar 9)",
|
||||
"source": "USNI News Fleet & Marine Tracker (seed, as of 2026-03-09)",
|
||||
"source_url": "https://news.usni.org/category/fleet-tracker",
|
||||
"position_source_at": "2026-03-09T00:00:00Z",
|
||||
"position_confidence": "seed",
|
||||
}
|
||||
enriched = fresh_tracker._enrich_for_rendering("CVN-78", seed_entry)
|
||||
# Existing UI fields preserved.
|
||||
assert enriched["lat"] == 18.0
|
||||
assert enriched["lng"] == 39.5
|
||||
assert enriched["source"].startswith("USNI")
|
||||
assert enriched["last_osint_update"] == "2026-03-09T00:00:00Z"
|
||||
# New audit-required fields.
|
||||
assert enriched["position_confidence"] == "seed"
|
||||
assert enriched["position_source_at"] == "2026-03-09T00:00:00Z"
|
||||
assert enriched["is_fallback"] is True
|
||||
|
||||
def test_recent_observation_is_not_fallback(self, fresh_tracker):
|
||||
now = datetime.now(timezone.utc)
|
||||
recent_entry = {
|
||||
"lat": 25.0,
|
||||
"lng": 55.0,
|
||||
"heading": 0,
|
||||
"desc": "Persian Gulf",
|
||||
"source": "GDELT News API",
|
||||
"source_url": "https://news.example/...",
|
||||
"position_source_at": (now - timedelta(days=2)).isoformat(),
|
||||
"position_confidence": "approximate",
|
||||
}
|
||||
enriched = fresh_tracker._enrich_for_rendering("CVN-78", recent_entry, now=now)
|
||||
assert enriched["position_confidence"] == "approximate"
|
||||
# Approximate (from a recent headline) is honest precision, but the UI
|
||||
# treats it as live data — is_fallback only flips True for explicit
|
||||
# fallback categories (seed / stale / homeport_default).
|
||||
assert enriched["is_fallback"] is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Regression: existing frontend fields are preserved
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPublicResponseShapeBackwardCompat:
|
||||
"""The frontend ShipPopup expects `estimated`, `source`, `source_url`,
|
||||
`last_osint_update`. The new fields are additive and existing fields
|
||||
keep their meaning so the UI does not need updating to keep working."""
|
||||
|
||||
def test_get_carrier_positions_preserves_existing_keys(self, fresh_tracker):
|
||||
_write_seed(fresh_tracker.SEED_FILE)
|
||||
fresh_tracker._bootstrap_cache_if_missing()
|
||||
with fresh_tracker._positions_lock:
|
||||
fresh_tracker._carrier_positions.update(
|
||||
{
|
||||
"CVN-78": {
|
||||
"lat": 18.0,
|
||||
"lng": 39.5,
|
||||
"heading": 0,
|
||||
"desc": "Red Sea (seed)",
|
||||
"source": "Seed",
|
||||
"source_url": "",
|
||||
"position_source_at": "2026-03-09T00:00:00Z",
|
||||
"position_confidence": "seed",
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
out = fresh_tracker.get_carrier_positions()
|
||||
assert len(out) == 1
|
||||
c = out[0]
|
||||
# Old fields the frontend uses.
|
||||
for key in (
|
||||
"name",
|
||||
"type",
|
||||
"lat",
|
||||
"lng",
|
||||
"country",
|
||||
"desc",
|
||||
"wiki",
|
||||
"estimated",
|
||||
"source",
|
||||
"source_url",
|
||||
"last_osint_update",
|
||||
):
|
||||
assert key in c, f"missing legacy field {key!r}"
|
||||
# New fields.
|
||||
for key in ("position_confidence", "position_source_at", "is_fallback"):
|
||||
assert key in c, f"missing audit-required field {key!r}"
|
||||
assert c["type"] == "carrier"
|
||||
assert c["estimated"] is True
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Issue #192 (tg12): CCTV proxy must re-validate the host on every redirect hop.
|
||||
|
||||
Before this fix, the proxy validated only the initial caller-supplied URL
|
||||
host and then used ``requests.get(..., allow_redirects=True)``, which would
|
||||
silently follow a 302 to an arbitrary internal address — an open-redirect-
|
||||
to-SSRF chain.
|
||||
|
||||
These tests assert that:
|
||||
|
||||
1. A redirect to a disallowed host is rejected (502).
|
||||
2. A redirect to an allowed host is followed (200).
|
||||
3. The redirect chain length is bounded.
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from routers.cctv import _fetch_cctv_upstream_response, _CCTV_MAX_REDIRECTS
|
||||
|
||||
|
||||
class _Resp:
|
||||
"""Minimal mock for requests.Response that mimics what _fetch needs."""
|
||||
|
||||
def __init__(self, status_code=200, headers=None, is_redirect=False):
|
||||
self.status_code = status_code
|
||||
self.headers = headers or {}
|
||||
self.is_redirect = is_redirect
|
||||
self.closed = False
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
def _profile():
|
||||
"""Build a tiny _CCTVProxyProfile-shaped mock the function expects."""
|
||||
p = MagicMock()
|
||||
p.name = "test"
|
||||
p.timeout = 5
|
||||
p.cache_seconds = 60
|
||||
return p
|
||||
|
||||
|
||||
def _request():
|
||||
"""Build a tiny Request-shaped mock — only headers are read."""
|
||||
req = MagicMock()
|
||||
req.headers = {}
|
||||
return req
|
||||
|
||||
|
||||
@patch("routers.cctv._cctv_upstream_headers", return_value={})
|
||||
@patch("routers.cctv._cctv_host_allowed", side_effect=lambda host: host == "allowed.example")
|
||||
@patch("routers.cctv._req" if False else "requests.get") # patched below per-call
|
||||
def test_redirect_to_disallowed_host_is_rejected(mock_get, mock_allow, mock_headers):
|
||||
"""A 302 from allowed.example -> evil.example must be rejected with 502."""
|
||||
# First call: 302 with Location: http://evil.example/path
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "http://evil.example/path"}, is_redirect=True),
|
||||
]
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
_fetch_cctv_upstream_response(_request(), "http://allowed.example/cam", _profile())
|
||||
assert exc_info.value.status_code == 502
|
||||
assert "disallowed host" in str(exc_info.value.detail).lower()
|
||||
|
||||
|
||||
@patch("routers.cctv._cctv_upstream_headers", return_value={})
|
||||
@patch("routers.cctv._cctv_host_allowed", side_effect=lambda host: host == "allowed.example")
|
||||
@patch("requests.get")
|
||||
def test_redirect_to_localhost_is_rejected(mock_get, mock_allow, mock_headers):
|
||||
"""A redirect to 127.0.0.1 (internal SSRF target) must be rejected."""
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "http://127.0.0.1:8000/api/secret"}, is_redirect=True),
|
||||
]
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
_fetch_cctv_upstream_response(_request(), "http://allowed.example/cam", _profile())
|
||||
assert exc_info.value.status_code == 502
|
||||
|
||||
|
||||
@patch("routers.cctv._cctv_upstream_headers", return_value={})
|
||||
@patch("routers.cctv._cctv_host_allowed", side_effect=lambda host: host in {"allowed.example", "other-allowed.example"})
|
||||
@patch("requests.get")
|
||||
def test_redirect_to_another_allowed_host_is_followed(mock_get, mock_allow, mock_headers):
|
||||
"""A 302 from one allowed host to another allowed host should succeed."""
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "http://other-allowed.example/cam"}, is_redirect=True),
|
||||
_Resp(status_code=200, headers={"Content-Type": "image/jpeg"}),
|
||||
]
|
||||
resp = _fetch_cctv_upstream_response(_request(), "http://allowed.example/cam", _profile())
|
||||
assert resp.status_code == 200
|
||||
|
||||
|
||||
@patch("routers.cctv._cctv_upstream_headers", return_value={})
|
||||
@patch("routers.cctv._cctv_host_allowed", return_value=True)
|
||||
@patch("requests.get")
|
||||
def test_redirect_chain_length_is_bounded(mock_get, mock_allow, mock_headers):
|
||||
"""A pathological redirect loop must terminate within _CCTV_MAX_REDIRECTS."""
|
||||
# Generate enough 302s to exceed the cap.
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": f"http://allowed.example/{i}"}, is_redirect=True)
|
||||
for i in range(_CCTV_MAX_REDIRECTS + 2)
|
||||
]
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
_fetch_cctv_upstream_response(_request(), "http://allowed.example/cam", _profile())
|
||||
assert exc_info.value.status_code == 502
|
||||
assert "too long" in str(exc_info.value.detail).lower()
|
||||
|
||||
|
||||
@patch("routers.cctv._cctv_upstream_headers", return_value={})
|
||||
@patch("routers.cctv._cctv_host_allowed", return_value=True)
|
||||
@patch("requests.get")
|
||||
def test_redirect_to_non_http_scheme_is_rejected(mock_get, mock_allow, mock_headers):
|
||||
"""A 302 to ``file://`` or ``ftp://`` must be rejected even if the host parses cleanly."""
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "file:///etc/passwd"}, is_redirect=True),
|
||||
]
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
_fetch_cctv_upstream_response(_request(), "http://allowed.example/cam", _profile())
|
||||
assert exc_info.value.status_code == 502
|
||||
assert "non-http" in str(exc_info.value.detail).lower()
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Regression coverage for operator-only control surfaces."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("method", "path", "payload"),
|
||||
[
|
||||
("get", "/api/wormhole/identity", None),
|
||||
("post", "/api/wormhole/identity/bootstrap", {}),
|
||||
("post", "/api/wormhole/gate/enter", {"gate_id": "general-talk"}),
|
||||
("post", "/api/wormhole/gate/leave", {"gate_id": "general-talk"}),
|
||||
("post", "/api/wormhole/sign", {"event_type": "gate_event", "payload": {"ok": True}}),
|
||||
("post", "/api/wormhole/gate/key/rotate", {"gate_id": "general-talk", "reason": "test"}),
|
||||
(
|
||||
"post",
|
||||
"/api/wormhole/gate/key/grant",
|
||||
{
|
||||
"gate_id": "general-talk",
|
||||
"recipient_node_id": "node-test",
|
||||
"recipient_dh_pub": "dh-test",
|
||||
},
|
||||
),
|
||||
("post", "/api/wormhole/gate/persona/create", {"gate_id": "general-talk", "label": "test"}),
|
||||
(
|
||||
"post",
|
||||
"/api/wormhole/gate/persona/activate",
|
||||
{"gate_id": "general-talk", "persona_id": "persona-test"},
|
||||
),
|
||||
("post", "/api/wormhole/gate/persona/clear", {"gate_id": "general-talk"}),
|
||||
(
|
||||
"post",
|
||||
"/api/wormhole/gate/persona/retire",
|
||||
{"gate_id": "general-talk", "persona_id": "persona-test"},
|
||||
),
|
||||
(
|
||||
"post",
|
||||
"/api/wormhole/gate/message/sign-encrypted",
|
||||
{
|
||||
"gate_id": "general-talk",
|
||||
"epoch": 1,
|
||||
"ciphertext": "ciphertext",
|
||||
"nonce": "nonce",
|
||||
"format": "mls1",
|
||||
"envelope_hash": "hash",
|
||||
},
|
||||
),
|
||||
("post", "/api/wormhole/gate/message/compose", {"gate_id": "general-talk", "plaintext": "hello"}),
|
||||
("post", "/api/wormhole/sign-raw", {"message": "raw"}),
|
||||
("post", "/api/wormhole/gate/state/export", {"gate_id": "general-talk"}),
|
||||
("post", "/api/wormhole/gate/proof", {"gate_id": "general-talk"}),
|
||||
("post", "/api/wormhole/connect", {}),
|
||||
("post", "/api/layers", {"layers": {"viirs_nightlights": True}}),
|
||||
("post", "/api/ais/feed", {"msgs": []}),
|
||||
# Added in post-#227 gap audit:
|
||||
# /api/wormhole/join also calls bootstrap_wormhole_identity() — same
|
||||
# identity-takeover surface as /identity/bootstrap. PR #227 hardened
|
||||
# the latter but missed the former.
|
||||
("post", "/api/wormhole/join", {}),
|
||||
# /api/sigint/transmit relays APRS-IS packets over radio using
|
||||
# operator-supplied credentials. Any caller who reaches this endpoint
|
||||
# could transmit on the operator's authority. Must be local-only.
|
||||
(
|
||||
"post",
|
||||
"/api/sigint/transmit",
|
||||
{
|
||||
"callsign": "N0CALL",
|
||||
"passcode": "12345",
|
||||
"target": "NOCALL",
|
||||
"message": "test",
|
||||
},
|
||||
),
|
||||
# Issue #198 (tg12, May 17): three gate introspection GETs leak the
|
||||
# operator's active persona, persona inventory, and key status for
|
||||
# any gate_id an anonymous caller knows. Defeats the unlinkability
|
||||
# property documented in the privacy threat model.
|
||||
("get", "/api/wormhole/gate/general-talk/identity", None),
|
||||
("get", "/api/wormhole/gate/general-talk/personas", None),
|
||||
("get", "/api/wormhole/gate/general-talk/key", None),
|
||||
# Issue #211 (tg12): /api/thermal/verify fans out into an expensive
|
||||
# STAC search + remote SWIR raster reads. Unauthenticated abuse
|
||||
# could burn Sentinel-Hub quota and outbound bandwidth.
|
||||
("get", "/api/thermal/verify?lat=0&lng=0&radius_km=10", None),
|
||||
# Issue #213 (tg12): /api/radio/openmhz/calls/{sys_name} — rotating
|
||||
# sys_name bypasses the 20s cache and hammers OpenMHZ. Risks an
|
||||
# IP-ban for the project.
|
||||
("get", "/api/radio/openmhz/calls/abc", None),
|
||||
# Issue #214 (tg12): /api/radio/openmhz/audio — anonymous bandwidth
|
||||
# relay through the backend. 60/minute rate limit is not enough on
|
||||
# a streaming endpoint.
|
||||
("get", "/api/radio/openmhz/audio?url=https%3A%2F%2Fmedia.openmhz.com%2Faudio%2Fabc.mp3", None),
|
||||
],
|
||||
)
|
||||
def test_remote_control_surface_rejects_without_local_operator_or_admin(
|
||||
remote_client, method, path, payload
|
||||
):
|
||||
request = getattr(remote_client, method)
|
||||
response = request(path, json=payload) if payload is not None else request(path)
|
||||
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
def test_remote_agent_actions_poll_rejects_without_local_operator_or_admin(remote_client):
|
||||
response = remote_client.get("/api/ai/agent-actions")
|
||||
|
||||
assert response.status_code == 403
|
||||
@@ -0,0 +1,52 @@
|
||||
"""CrowdThreat ingestion is operator opt-in only."""
|
||||
|
||||
|
||||
class _CrowdThreatResponse:
|
||||
status_code = 200
|
||||
|
||||
def json(self):
|
||||
return {
|
||||
"data": {
|
||||
"threats": [
|
||||
{
|
||||
"id": "ct-1",
|
||||
"title": "Example report",
|
||||
"location": {
|
||||
"lng_lat": [12.5, 41.9],
|
||||
"name": "Example place",
|
||||
"country": {"name": "Italy"},
|
||||
},
|
||||
"category": {"id": 1, "name": "Security"},
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_crowdthreat_disabled_by_default_does_not_call_upstream(monkeypatch):
|
||||
from services.fetchers import _store, crowdthreat
|
||||
|
||||
monkeypatch.delenv("CROWDTHREAT_ENABLED", raising=False)
|
||||
monkeypatch.setitem(_store.latest_data, "crowdthreat", [{"id": "old"}])
|
||||
monkeypatch.setattr(
|
||||
crowdthreat,
|
||||
"fetch_with_curl",
|
||||
lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("upstream called")),
|
||||
)
|
||||
|
||||
crowdthreat.fetch_crowdthreat()
|
||||
|
||||
assert _store.latest_data["crowdthreat"] == []
|
||||
|
||||
|
||||
def test_crowdthreat_opt_in_fetches_when_layer_is_enabled(monkeypatch):
|
||||
from services.fetchers import _store, crowdthreat
|
||||
|
||||
monkeypatch.setenv("CROWDTHREAT_ENABLED", "true")
|
||||
monkeypatch.setitem(_store.active_layers, "crowdthreat", True)
|
||||
monkeypatch.setattr(crowdthreat, "fetch_with_curl", lambda *args, **kwargs: _CrowdThreatResponse())
|
||||
|
||||
crowdthreat.fetch_crowdthreat()
|
||||
|
||||
assert _store.latest_data["crowdthreat"][0]["id"] == "ct-1"
|
||||
assert _store.latest_data["crowdthreat"][0]["source"] == "CrowdThreat"
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Issue #250 (tg12): Docker bridge local-operator trust must be bound to
|
||||
the frontend container's hostname, not the entire 172.16.0.0/12 range.
|
||||
|
||||
Previous behavior trusted ANY private-RFC1918 source IP on the bridge
|
||||
when ``SHADOWBROKER_TRUST_DOCKER_BRIDGE_LOCAL_OPERATOR=1``. On a shared
|
||||
Docker host this granted local-operator privileges to any other
|
||||
container that could route to the backend's bridge — far broader than
|
||||
intended.
|
||||
|
||||
The fix narrows trust to source IPs that forward-resolve from one of the
|
||||
configured frontend container hostnames (default: the compose service
|
||||
name ``frontend`` plus the explicit ``container_name``
|
||||
``shadowbroker-frontend``). Operators with renamed containers can list
|
||||
the new names in ``SHADOWBROKER_TRUSTED_FRONTEND_HOSTS``.
|
||||
|
||||
These tests exercise the resolution helpers directly so that we don't
|
||||
need a live Docker daemon to validate the contract.
|
||||
"""
|
||||
import socket
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _trusted_bridge_frontend_hostnames — env parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestTrustedHostnameParsing:
|
||||
def _fn(self):
|
||||
from auth import _trusted_bridge_frontend_hostnames
|
||||
return _trusted_bridge_frontend_hostnames
|
||||
|
||||
def test_default_covers_compose_service_and_container_name(self):
|
||||
with patch.dict("os.environ", {}, clear=False):
|
||||
# Make sure the env var is not set so we exercise the default.
|
||||
import os
|
||||
os.environ.pop("SHADOWBROKER_TRUSTED_FRONTEND_HOSTS", None)
|
||||
assert self._fn()() == ["frontend", "shadowbroker-frontend"]
|
||||
|
||||
def test_custom_list_via_env(self):
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{"SHADOWBROKER_TRUSTED_FRONTEND_HOSTS": "my-ui,alt-frontend"},
|
||||
):
|
||||
assert self._fn()() == ["my-ui", "alt-frontend"]
|
||||
|
||||
def test_whitespace_trimmed(self):
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{"SHADOWBROKER_TRUSTED_FRONTEND_HOSTS": " my-ui , alt-frontend "},
|
||||
):
|
||||
assert self._fn()() == ["my-ui", "alt-frontend"]
|
||||
|
||||
def test_empty_env_falls_back_to_default(self):
|
||||
# An empty string still falls back to the bundled defaults so a
|
||||
# misconfigured env var doesn't silently dismantle bridge trust.
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{"SHADOWBROKER_TRUSTED_FRONTEND_HOSTS": ""},
|
||||
):
|
||||
# Per docs: empty string sets the env var to "" so os.environ.get
|
||||
# returns "" — that string is parsed and yields []. We assert
|
||||
# that empty parse yields [] (caller fail-closes from there).
|
||||
assert self._fn()() == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _resolve_trusted_bridge_ips — DNS resolution with cache + fail-closed
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestResolveTrustedBridgeIps:
|
||||
def setup_method(self):
|
||||
# Reset the module-level cache before each test so prior tests
|
||||
# don't bleed state across cases.
|
||||
from auth import _DOCKER_BRIDGE_TRUST_CACHE
|
||||
_DOCKER_BRIDGE_TRUST_CACHE["ips"] = frozenset()
|
||||
_DOCKER_BRIDGE_TRUST_CACHE["expires"] = 0.0
|
||||
|
||||
def test_resolves_configured_hostnames(self):
|
||||
from auth import _resolve_trusted_bridge_ips
|
||||
|
||||
def fake_gethostbyname_ex(host):
|
||||
mapping = {
|
||||
"frontend": ("frontend", [], ["172.18.0.3"]),
|
||||
"shadowbroker-frontend": ("shadowbroker-frontend", [], ["172.18.0.3", "172.18.0.4"]),
|
||||
}
|
||||
if host not in mapping:
|
||||
raise socket.gaierror("no such host")
|
||||
return mapping[host]
|
||||
|
||||
with patch("socket.gethostbyname_ex", side_effect=fake_gethostbyname_ex):
|
||||
ips = _resolve_trusted_bridge_ips()
|
||||
assert ips == frozenset({"172.18.0.3", "172.18.0.4"})
|
||||
|
||||
def test_fail_closed_when_dns_returns_nothing(self):
|
||||
from auth import _resolve_trusted_bridge_ips
|
||||
|
||||
def always_fail(host):
|
||||
raise socket.gaierror("no resolver")
|
||||
|
||||
with patch("socket.gethostbyname_ex", side_effect=always_fail):
|
||||
ips = _resolve_trusted_bridge_ips()
|
||||
assert ips == frozenset()
|
||||
|
||||
def test_partial_resolution_is_kept(self):
|
||||
"""If one hostname resolves and another fails, we keep the
|
||||
successful one rather than discarding the whole set."""
|
||||
from auth import _resolve_trusted_bridge_ips
|
||||
|
||||
def partial(host):
|
||||
if host == "frontend":
|
||||
return ("frontend", [], ["172.18.0.3"])
|
||||
raise socket.gaierror("missing")
|
||||
|
||||
with patch("socket.gethostbyname_ex", side_effect=partial):
|
||||
ips = _resolve_trusted_bridge_ips()
|
||||
assert ips == frozenset({"172.18.0.3"})
|
||||
|
||||
def test_cache_short_circuits_repeated_dns_calls(self):
|
||||
from auth import _resolve_trusted_bridge_ips
|
||||
|
||||
call_count = {"n": 0}
|
||||
|
||||
def counting(host):
|
||||
call_count["n"] += 1
|
||||
return ("frontend", [], ["172.18.0.3"])
|
||||
|
||||
with patch("socket.gethostbyname_ex", side_effect=counting):
|
||||
_resolve_trusted_bridge_ips()
|
||||
calls_after_first = call_count["n"]
|
||||
_resolve_trusted_bridge_ips()
|
||||
_resolve_trusted_bridge_ips()
|
||||
# Second + third calls hit the cache, not the DNS stub.
|
||||
assert call_count["n"] == calls_after_first
|
||||
|
||||
def test_cache_expires(self):
|
||||
from auth import _resolve_trusted_bridge_ips, _DOCKER_BRIDGE_TRUST_CACHE
|
||||
|
||||
with patch("socket.gethostbyname_ex", return_value=("frontend", [], ["172.18.0.3"])):
|
||||
_resolve_trusted_bridge_ips()
|
||||
# Force expiry.
|
||||
_DOCKER_BRIDGE_TRUST_CACHE["expires"] = 0.0
|
||||
with patch("socket.gethostbyname_ex", return_value=("frontend", [], ["172.18.0.9"])) as stub:
|
||||
ips = _resolve_trusted_bridge_ips()
|
||||
assert stub.called
|
||||
assert "172.18.0.9" in ips
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _is_docker_bridge_host — composite of the helpers above
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestIsDockerBridgeHost:
|
||||
def setup_method(self):
|
||||
from auth import _DOCKER_BRIDGE_TRUST_CACHE
|
||||
_DOCKER_BRIDGE_TRUST_CACHE["ips"] = frozenset()
|
||||
_DOCKER_BRIDGE_TRUST_CACHE["expires"] = 0.0
|
||||
|
||||
def test_trusts_resolved_frontend_ip(self):
|
||||
from auth import _is_docker_bridge_host
|
||||
|
||||
with patch("auth._resolve_trusted_bridge_ips", return_value=frozenset({"172.18.0.3"})):
|
||||
assert _is_docker_bridge_host("172.18.0.3") is True
|
||||
|
||||
def test_rejects_arbitrary_bridge_ip(self):
|
||||
"""A rogue container on the same bridge but at a different IP
|
||||
must NOT be trusted, even though it falls in 172.16.0.0/12."""
|
||||
from auth import _is_docker_bridge_host
|
||||
|
||||
with patch("auth._resolve_trusted_bridge_ips", return_value=frozenset({"172.18.0.3"})):
|
||||
assert _is_docker_bridge_host("172.18.0.99") is False
|
||||
|
||||
def test_rejects_public_ip_without_dns_work(self):
|
||||
"""Public IPs skip DNS resolution entirely (perf + safety)."""
|
||||
from auth import _is_docker_bridge_host
|
||||
|
||||
with patch("auth._resolve_trusted_bridge_ips") as stub:
|
||||
assert _is_docker_bridge_host("8.8.8.8") is False
|
||||
stub.assert_not_called()
|
||||
|
||||
def test_rejects_non_ip_input(self):
|
||||
from auth import _is_docker_bridge_host
|
||||
|
||||
assert _is_docker_bridge_host("") is False
|
||||
assert _is_docker_bridge_host("not-an-ip") is False
|
||||
assert _is_docker_bridge_host("frontend") is False
|
||||
|
||||
def test_fails_closed_when_dns_returns_empty(self):
|
||||
"""If Docker DNS can't resolve any frontend hostname, the bridge
|
||||
is not trusted — even for IPs that would have been trusted under
|
||||
the old 172.16.0.0/12 blanket policy."""
|
||||
from auth import _is_docker_bridge_host
|
||||
|
||||
with patch("auth._resolve_trusted_bridge_ips", return_value=frozenset()):
|
||||
assert _is_docker_bridge_host("172.18.0.3") is False
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Issue #199 (tg12): GDELT military incident ingestion must use HTTPS.
|
||||
|
||||
The previous code fetched ``http://data.gdeltproject.org/gdeltv2/lastupdate.txt``
|
||||
and ~48 export archives over plaintext HTTP, which let a passive observer
|
||||
identify Shadowbroker nodes by their fetch pattern and let an active MITM
|
||||
inject doctored export records into the global incident map.
|
||||
|
||||
These tests assert the URL constants and outbound URL constructor in
|
||||
``services/geopolitics.py`` only use HTTPS.
|
||||
"""
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
_GEOPOLITICS_SRC = Path(__file__).resolve().parent.parent / "services" / "geopolitics.py"
|
||||
|
||||
|
||||
def _read_source() -> str:
|
||||
return _GEOPOLITICS_SRC.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_geopolitics_does_not_use_plaintext_http_for_gdelt():
|
||||
"""No string literal in geopolitics.py should fetch GDELT over plaintext HTTP."""
|
||||
src = _read_source()
|
||||
# Strings that would issue an HTTP request — comments are excluded because
|
||||
# comments include "http://" in example URLs even after the fix.
|
||||
code_lines = [
|
||||
ln for ln in src.split("\n")
|
||||
if "http://data.gdeltproject.org" in ln and not ln.lstrip().startswith("#")
|
||||
]
|
||||
assert code_lines == [], (
|
||||
"Found plaintext http://data.gdeltproject.org usage in geopolitics.py:\n"
|
||||
+ "\n".join(code_lines)
|
||||
)
|
||||
|
||||
|
||||
def test_geopolitics_uses_https_for_gdelt():
|
||||
"""The HTTPS URLs we expect must be present."""
|
||||
src = _read_source()
|
||||
assert "https://data.gdeltproject.org/gdeltv2/lastupdate.txt" in src
|
||||
# The download URL is constructed via f-string with {fname}
|
||||
assert re.search(
|
||||
r'https://data\.gdeltproject\.org/gdeltv2/\{fname\}', src
|
||||
), "expected https URL template for individual GDELT export downloads"
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Issue #207 (tg12): /api/mesh/infonet/status accepted
|
||||
?verify_signatures=true from anonymous callers, triggering O(n_events)
|
||||
signature verification across the entire chain. Trivial DoS.
|
||||
|
||||
The fix silently downgrades the parameter to False for unauthenticated
|
||||
callers — no error surfaced, response structure unchanged, the
|
||||
expensive path runs only when the caller has authenticated.
|
||||
|
||||
These tests focus on the source-level contract because a full
|
||||
FastAPI test client doesn't have an easy hook into the ``_scoped_view_authenticated``
|
||||
helper. They lock in the key invariant: the ``effective_verify_signatures``
|
||||
value seen by ``validate_chain()`` is the AND of the request param and
|
||||
the auth check.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
_ROUTER_PATH = Path(__file__).resolve().parent.parent / "routers" / "mesh_public.py"
|
||||
|
||||
|
||||
def _read_router_source() -> str:
|
||||
return _ROUTER_PATH.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_infonet_status_gates_verify_signatures():
|
||||
"""The infonet_status route must AND verify_signatures with auth."""
|
||||
src = _read_router_source()
|
||||
# The fix introduces an `effective_verify_signatures` variable.
|
||||
assert "effective_verify_signatures" in src
|
||||
|
||||
# It must be computed as the AND of the request param and the
|
||||
# authenticated check.
|
||||
assert "bool(verify_signatures) and authenticated" in src
|
||||
|
||||
# validate_chain() must be called with the effective value, NOT the
|
||||
# raw request param.
|
||||
assert "validate_chain(verify_signatures=effective_verify_signatures)" in src
|
||||
|
||||
|
||||
def test_no_http_error_path_for_anonymous_callers():
|
||||
"""No HTTPException is raised for unauthenticated verify_signatures=true.
|
||||
|
||||
The endpoint should silently downgrade — not return 403 — so existing
|
||||
frontends that happen to pass the param see no behavior change.
|
||||
"""
|
||||
src = _read_router_source()
|
||||
# Within the infonet_status function body, there should be no
|
||||
# HTTPException(403) raised because of the verify_signatures param.
|
||||
# Find the function definition and inspect the body.
|
||||
import re
|
||||
m = re.search(
|
||||
r"async def infonet_status\(.*?\):(.+?)(?=\n@router|\nasync def |\ndef |\Z)",
|
||||
src,
|
||||
re.DOTALL,
|
||||
)
|
||||
assert m, "infonet_status function not found in source"
|
||||
body = m.group(1)
|
||||
# No explicit 403 around the verify_signatures handling.
|
||||
assert "HTTPException(status_code=403" not in body
|
||||
assert "raise HTTPException(403" not in body
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Issue #206 (tg12): KiwiSDR upstream is HTTP-only and cannot be upgraded
|
||||
to TLS. We defend with content validation + a bundled static directory
|
||||
so the layer always renders something useful and a MITM injecting
|
||||
garbage can't corrupt the map.
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from services import kiwisdr_fetcher
|
||||
from services.kiwisdr_fetcher import (
|
||||
_MIN_HEALTHY_RECEIVER_COUNT,
|
||||
_load_bundled_fallback,
|
||||
_validate_fetched_nodes,
|
||||
)
|
||||
|
||||
|
||||
def test_bundled_fallback_file_exists_and_is_nonempty():
|
||||
"""The codebase ships a static snapshot for last-resort use."""
|
||||
bundle = _load_bundled_fallback()
|
||||
assert isinstance(bundle, list)
|
||||
assert len(bundle) >= _MIN_HEALTHY_RECEIVER_COUNT
|
||||
|
||||
|
||||
def test_validation_rejects_too_few_entries():
|
||||
too_short = [{"name": "x", "lat": 0.0, "lon": 0.0, "url": ""}] * (_MIN_HEALTHY_RECEIVER_COUNT - 1)
|
||||
assert _validate_fetched_nodes(too_short) is False
|
||||
|
||||
|
||||
def test_validation_accepts_healthy_response():
|
||||
healthy = [
|
||||
{"name": f"Receiver {i}", "lat": 50.0, "lon": -1.0, "url": "http://example"}
|
||||
for i in range(_MIN_HEALTHY_RECEIVER_COUNT)
|
||||
]
|
||||
assert _validate_fetched_nodes(healthy) is True
|
||||
|
||||
|
||||
def test_validation_rejects_non_list():
|
||||
assert _validate_fetched_nodes(None) is False # type: ignore[arg-type]
|
||||
assert _validate_fetched_nodes("a string") is False # type: ignore[arg-type]
|
||||
assert _validate_fetched_nodes({}) is False # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_validation_rejects_too_many_malformed_entries():
|
||||
"""If more than 5% of entries lack a name or numeric lat, reject."""
|
||||
nodes = []
|
||||
# 100 entries, 20 of them malformed — well over the 5% threshold.
|
||||
for i in range(_MIN_HEALTHY_RECEIVER_COUNT + 50):
|
||||
if i % 5 == 0:
|
||||
nodes.append({}) # missing name + lat
|
||||
else:
|
||||
nodes.append({"name": f"R{i}", "lat": 50.0, "lon": -1.0, "url": ""})
|
||||
assert _validate_fetched_nodes(nodes) is False
|
||||
|
||||
|
||||
def test_fallback_used_when_validation_fails(monkeypatch, tmp_path):
|
||||
"""If a fetch returns garbage, the fallback chain reaches the bundle."""
|
||||
# Force disk cache miss
|
||||
fake_cache = tmp_path / "kiwisdr_cache.json"
|
||||
monkeypatch.setattr(kiwisdr_fetcher, "_CACHE_FILE", fake_cache)
|
||||
|
||||
# Make fetch_with_curl return a parseable but UNHEALTHY response
|
||||
# (only 3 entries — well below the validation threshold).
|
||||
class _GarbageResp:
|
||||
status_code = 200
|
||||
text = "var kiwisdr_com = [{\"name\":\"x\",\"gps\":\"(0,0)\"}];"
|
||||
|
||||
monkeypatch.setattr(
|
||||
"services.network_utils.fetch_with_curl", lambda *a, **kw: _GarbageResp()
|
||||
)
|
||||
|
||||
# Bypass the @cached decorator
|
||||
kiwisdr_fetcher.kiwisdr_cache.clear()
|
||||
|
||||
result = kiwisdr_fetcher.fetch_kiwisdr_nodes()
|
||||
# Should be the bundled fallback (798 entries), not the garbage (1 entry)
|
||||
assert isinstance(result, list)
|
||||
assert len(result) >= _MIN_HEALTHY_RECEIVER_COUNT
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Issue #208 (tg12): Merkle proofs were rebuilt from scratch on every
|
||||
public ``/api/mesh/infonet/sync?include_proofs=true`` request. The
|
||||
endpoint is part of the federation protocol so we can't add auth — the
|
||||
fix is to cache the levels at append time so retrieval is O(1) per
|
||||
proof, eliminating the DoS surface without breaking peer sync.
|
||||
|
||||
These tests verify:
|
||||
|
||||
* A fresh Infonet has no cache (lazy state).
|
||||
* After ``append()``, the cache is invalidated.
|
||||
* Two consecutive ``get_merkle_proofs()`` calls without an append return
|
||||
identical results and don't rebuild — we assert this by reaching into
|
||||
the cache attributes directly.
|
||||
"""
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
|
||||
from services.mesh.mesh_hashchain import Infonet
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fresh_infonet(monkeypatch, tmp_path):
|
||||
"""Build a clean Infonet rooted at a temp directory."""
|
||||
# Redirect persistence to the temp dir so we don't pollute real state.
|
||||
monkeypatch.setattr(
|
||||
"services.mesh.mesh_hashchain.CHAIN_FILE",
|
||||
tmp_path / "infonet_chain.json",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"services.mesh.mesh_hashchain.WAL_PATH",
|
||||
tmp_path / "infonet_chain.wal",
|
||||
raising=False,
|
||||
)
|
||||
inst = Infonet()
|
||||
inst.events = [] # ensure empty
|
||||
inst._invalidate_merkle_cache()
|
||||
return inst
|
||||
|
||||
|
||||
def test_cache_starts_empty(fresh_infonet):
|
||||
"""The cache fields exist and start in their lazy state."""
|
||||
assert hasattr(fresh_infonet, "_merkle_levels_cache")
|
||||
assert fresh_infonet._merkle_levels_cache is None
|
||||
assert fresh_infonet._merkle_levels_for_event_count == -1
|
||||
|
||||
|
||||
def test_get_merkle_root_populates_cache(fresh_infonet):
|
||||
"""First call computes and caches the levels."""
|
||||
# Add a synthetic event so there's something to hash
|
||||
fresh_infonet.events = [{"event_id": "a" * 64}, {"event_id": "b" * 64}]
|
||||
_ = fresh_infonet.get_merkle_root()
|
||||
assert fresh_infonet._merkle_levels_cache is not None
|
||||
assert fresh_infonet._merkle_levels_for_event_count == 2
|
||||
|
||||
|
||||
def test_repeated_root_calls_reuse_cache(fresh_infonet):
|
||||
"""The cache survives multiple reads when no events were appended."""
|
||||
fresh_infonet.events = [{"event_id": "a" * 64}, {"event_id": "b" * 64}]
|
||||
_ = fresh_infonet.get_merkle_root()
|
||||
cached_levels = fresh_infonet._merkle_levels_cache
|
||||
cached_count = fresh_infonet._merkle_levels_for_event_count
|
||||
|
||||
_ = fresh_infonet.get_merkle_root()
|
||||
# Same object — no rebuild.
|
||||
assert fresh_infonet._merkle_levels_cache is cached_levels
|
||||
assert fresh_infonet._merkle_levels_for_event_count == cached_count
|
||||
|
||||
|
||||
def test_append_invalidates_cache(fresh_infonet):
|
||||
"""After events change, the cache_for_count diverges from len(events).
|
||||
|
||||
The next read recomputes; that's the architectural point.
|
||||
"""
|
||||
fresh_infonet.events = [{"event_id": "a" * 64}]
|
||||
_ = fresh_infonet.get_merkle_root()
|
||||
assert fresh_infonet._merkle_levels_for_event_count == 1
|
||||
|
||||
# Simulate an append's side effect (the real append() also calls
|
||||
# _invalidate_merkle_cache() — we test that integration in the
|
||||
# in-tree append-flow test, not here).
|
||||
fresh_infonet.events.append({"event_id": "b" * 64})
|
||||
fresh_infonet._invalidate_merkle_cache()
|
||||
|
||||
_ = fresh_infonet.get_merkle_root()
|
||||
assert fresh_infonet._merkle_levels_for_event_count == 2
|
||||
|
||||
|
||||
def test_proofs_use_cache(fresh_infonet):
|
||||
"""get_merkle_proofs() reads from the same cache get_merkle_root() does."""
|
||||
fresh_infonet.events = [
|
||||
{"event_id": (str(i) * 64)[:64]} for i in range(8)
|
||||
]
|
||||
_ = fresh_infonet.get_merkle_root()
|
||||
cached_levels = fresh_infonet._merkle_levels_cache
|
||||
|
||||
proofs = fresh_infonet.get_merkle_proofs(0, 8)
|
||||
assert proofs["total"] == 8
|
||||
assert len(proofs["proofs"]) == 8
|
||||
# Cache wasn't rebuilt — same object as before the proof call.
|
||||
assert fresh_infonet._merkle_levels_cache is cached_levels
|
||||
|
||||
|
||||
def test_empty_chain_returns_genesis(fresh_infonet):
|
||||
"""An empty chain should serve GENESIS_HASH without computing levels."""
|
||||
from services.mesh.mesh_hashchain import GENESIS_HASH
|
||||
|
||||
root = fresh_infonet.get_merkle_root()
|
||||
assert root == GENESIS_HASH
|
||||
|
||||
proofs = fresh_infonet.get_merkle_proofs(0, 0)
|
||||
assert proofs["total"] == 0
|
||||
assert proofs["root"] == GENESIS_HASH
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Issue #203 (tg12): meshtastic_map.py was unconditionally including
|
||||
``MESHTASTIC_OPERATOR_CALLSIGN`` in the outbound User-Agent header,
|
||||
which contradicted the README's "no user data transmitted" claim.
|
||||
|
||||
The fix preserves the existing default behavior (callsign sent — that's
|
||||
what operators who configured the variable expected) but adds an
|
||||
opt-out env var ``MESHTASTIC_SEND_CALLSIGN_HEADER=false`` for
|
||||
privacy-conscious operators.
|
||||
"""
|
||||
import importlib
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _reload_meshtastic_module():
|
||||
"""Reload meshtastic_map so settings are re-read on demand."""
|
||||
if "services.fetchers.meshtastic_map" in sys.modules:
|
||||
del sys.modules["services.fetchers.meshtastic_map"]
|
||||
return importlib.import_module("services.fetchers.meshtastic_map")
|
||||
|
||||
|
||||
def test_default_behavior_includes_callsign(monkeypatch):
|
||||
"""Operators who set the callsign and don't change anything else
|
||||
keep their existing behavior (callsign sent in UA)."""
|
||||
# We test the UA construction logic by exercising the same branches
|
||||
# the fetcher uses. Direct fetch isn't run because it makes a real
|
||||
# network call — we just verify the env-var-driven decision.
|
||||
import os
|
||||
monkeypatch.setenv("MESHTASTIC_OPERATOR_CALLSIGN", "N0CALL")
|
||||
monkeypatch.delenv("MESHTASTIC_SEND_CALLSIGN_HEADER", raising=False)
|
||||
|
||||
raw = str(os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "true")).strip().lower()
|
||||
send_callsign_header = raw not in {"0", "false", "no", "off", ""}
|
||||
assert send_callsign_header is True
|
||||
|
||||
|
||||
def test_opt_out_suppresses_callsign(monkeypatch):
|
||||
"""Setting MESHTASTIC_SEND_CALLSIGN_HEADER=false suppresses the header."""
|
||||
import os
|
||||
monkeypatch.setenv("MESHTASTIC_OPERATOR_CALLSIGN", "N0CALL")
|
||||
monkeypatch.setenv("MESHTASTIC_SEND_CALLSIGN_HEADER", "false")
|
||||
|
||||
raw = str(os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "true")).strip().lower()
|
||||
send_callsign_header = raw not in {"0", "false", "no", "off", ""}
|
||||
assert send_callsign_header is False
|
||||
|
||||
|
||||
def test_various_falsy_values_all_opt_out(monkeypatch):
|
||||
"""Common falsy strings should all suppress the callsign header."""
|
||||
import os
|
||||
for falsy in ("0", "false", "FALSE", "no", "off"):
|
||||
monkeypatch.setenv("MESHTASTIC_SEND_CALLSIGN_HEADER", falsy)
|
||||
raw = str(os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "true")).strip().lower()
|
||||
send_callsign_header = raw not in {"0", "false", "no", "off", ""}
|
||||
assert send_callsign_header is False, f"value {falsy!r} did not opt out"
|
||||
@@ -0,0 +1,208 @@
|
||||
"""Issue #239 (tg12): backend registers duplicate API routes in both
|
||||
``main.py`` and router modules, so request behavior depends on the
|
||||
order ``FastAPI`` happened to register them.
|
||||
|
||||
This test is the **CI guard** that locks in the invariant going forward.
|
||||
It does NOT delete any existing duplicates — those are tolerated via an
|
||||
explicit baseline file. What it DOES block is *new* duplicates appearing
|
||||
later, which is what the audit was actually asking for: a way to stop
|
||||
the drift before it gets worse.
|
||||
|
||||
Findings (empirically verified, see PR #286 description):
|
||||
|
||||
- ``main.app`` calls ``include_router(...)`` for every router at module
|
||||
import time around line 3316.
|
||||
- Every ``@app.get/post/put/...`` decorator inside ``main.py`` runs
|
||||
*after* those include_router calls, so the router handler is the one
|
||||
that actually serves requests. The duplicates in ``main.py`` are
|
||||
dead code at the route-resolution layer.
|
||||
- Behavior today is deterministic (router wins), but if someone later
|
||||
adds a NEW route only in ``main.py``, or edits one copy of an
|
||||
existing pair without the other, drift starts.
|
||||
|
||||
How this test works:
|
||||
|
||||
- Walks ``main.app.routes`` and records every ``(method, path)`` that
|
||||
appears more than once, along with which modules registered each
|
||||
copy.
|
||||
- Compares that set against the baseline in
|
||||
``backend/tests/data/duplicate_routes_baseline.json``.
|
||||
- **Fails** if any duplicate appears that is NOT in the baseline
|
||||
(or if the registering modules for an existing duplicate change).
|
||||
- **Stays green** when duplicates are *removed* by genuinely deduping
|
||||
the code. (The baseline is a ceiling, not a floor.)
|
||||
|
||||
To extend in the future:
|
||||
|
||||
- If you actually dedupe a route, leave the baseline alone — the test
|
||||
still passes. Subsequent regenerations of the baseline (``python -m
|
||||
scripts.regen_duplicate_routes_baseline`` or the snippet in this
|
||||
test's docstring) will shrink it.
|
||||
- If you legitimately need a new duplicate (you probably do not), add
|
||||
it to the baseline AND explain why in the PR description so reviewers
|
||||
can push back.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
BASELINE_PATH = (
|
||||
Path(__file__).parent / "data" / "duplicate_routes_baseline.json"
|
||||
)
|
||||
|
||||
|
||||
def _current_duplicates() -> dict[str, list[str]]:
|
||||
"""Walk ``main.app.routes`` and return ``{'METHOD /path': [module, ...]}``
|
||||
for every (method, path) registered more than once."""
|
||||
import main
|
||||
|
||||
by_key: dict[str, list[str]] = defaultdict(list)
|
||||
for route in main.app.routes:
|
||||
path = getattr(route, "path", None)
|
||||
methods = getattr(route, "methods", None)
|
||||
endpoint = getattr(route, "endpoint", None)
|
||||
if not path or not methods or endpoint is None:
|
||||
continue
|
||||
for method in methods:
|
||||
if method in ("HEAD", "OPTIONS"):
|
||||
continue
|
||||
by_key[f"{method} {path}"].append(endpoint.__module__)
|
||||
|
||||
return {
|
||||
key: sorted(modules) for key, modules in by_key.items() if len(modules) > 1
|
||||
}
|
||||
|
||||
|
||||
def _load_baseline() -> dict[str, list[str]]:
|
||||
if not BASELINE_PATH.exists():
|
||||
return {}
|
||||
raw = json.loads(BASELINE_PATH.read_text(encoding="utf-8"))
|
||||
dups = raw.get("duplicates", {})
|
||||
if not isinstance(dups, dict):
|
||||
return {}
|
||||
return {k: sorted(v) for k, v in dups.items()}
|
||||
|
||||
|
||||
def test_no_new_duplicate_route_registrations():
|
||||
"""Block any (method, path) duplicate not already in the baseline.
|
||||
|
||||
This is the primary CI guard: PRs that add a NEW shadowed
|
||||
``@app.get`` while a router module already serves the same route
|
||||
fail here with an actionable message.
|
||||
"""
|
||||
current = _current_duplicates()
|
||||
baseline = _load_baseline()
|
||||
|
||||
new_or_changed = []
|
||||
for key, modules in sorted(current.items()):
|
||||
if key not in baseline:
|
||||
new_or_changed.append(
|
||||
f" + {key} (NEW duplicate; registered in: {modules})"
|
||||
)
|
||||
continue
|
||||
if modules != baseline[key]:
|
||||
new_or_changed.append(
|
||||
f" ~ {key} "
|
||||
f"(modules changed: was {baseline[key]}, now {modules})"
|
||||
)
|
||||
|
||||
if new_or_changed:
|
||||
pytest.fail(
|
||||
"Issue #239 CI guard: detected duplicate route registrations "
|
||||
"that are NOT in the tolerated baseline.\n"
|
||||
"\n"
|
||||
"If you added a new @app.get/post/... in main.py for a path "
|
||||
"that a router module already serves, please move the handler "
|
||||
"into the router and delete the main.py copy — the router "
|
||||
"version wins on request routing anyway, so the main.py copy "
|
||||
"is dead code that just creates drift risk.\n"
|
||||
"\n"
|
||||
"Offending entries:\n"
|
||||
+ "\n".join(new_or_changed)
|
||||
+ "\n\n"
|
||||
"Baseline lives at "
|
||||
f"{BASELINE_PATH.relative_to(BASELINE_PATH.parent.parent.parent)}."
|
||||
)
|
||||
|
||||
|
||||
def test_baseline_only_lists_real_duplicates():
|
||||
"""Catch baseline drift in the other direction: if an entry in the
|
||||
baseline is no longer actually a duplicate (because someone deduped
|
||||
it manually), the baseline is stale and should be shrunk so future
|
||||
re-introductions of that duplicate get caught.
|
||||
|
||||
This test is informational — it does NOT fail the build today (the
|
||||
audit's main concern is *new* duplicates, not stale baseline
|
||||
entries). It prints a warning so the next baseline regeneration
|
||||
can clean things up.
|
||||
"""
|
||||
current = _current_duplicates()
|
||||
baseline = _load_baseline()
|
||||
stale = sorted(k for k in baseline if k not in current)
|
||||
if stale:
|
||||
# Use warnings instead of fail so this is friendly housekeeping,
|
||||
# not a CI blocker. The other test catches the actual safety
|
||||
# concern.
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
f"duplicate_routes_baseline.json contains {len(stale)} entry/entries "
|
||||
"no longer present in app.routes — consider regenerating the baseline. "
|
||||
f"Stale: {stale[:5]}{'...' if len(stale) > 5 else ''}",
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
|
||||
def test_router_handler_is_the_one_that_serves():
|
||||
"""Pin the empirical claim from PR #286: for every duplicated
|
||||
(method, path), the FIRST-registered handler is in a router
|
||||
module, not in main.py. If this ever flips — e.g. someone moves
|
||||
include_router calls to the bottom of main.py — duplicate routes
|
||||
start silently changing which handler runs. This catches that
|
||||
rearrangement immediately.
|
||||
"""
|
||||
import main
|
||||
|
||||
first_seen: dict[str, str] = {}
|
||||
for route in main.app.routes:
|
||||
path = getattr(route, "path", None)
|
||||
methods = getattr(route, "methods", None)
|
||||
endpoint = getattr(route, "endpoint", None)
|
||||
if not path or not methods or endpoint is None:
|
||||
continue
|
||||
for method in methods:
|
||||
if method in ("HEAD", "OPTIONS"):
|
||||
continue
|
||||
key = f"{method} {path}"
|
||||
if key not in first_seen:
|
||||
first_seen[key] = endpoint.__module__
|
||||
|
||||
main_winning = sorted(
|
||||
k for k, mod in first_seen.items() if mod == "main"
|
||||
)
|
||||
# The duplicates we tolerate are router-first. If main is the first
|
||||
# registered for any duplicated path, the router copy gets shadowed
|
||||
# instead, which would invalidate every assumption made in audit
|
||||
# rounds 5 and 6 about "the router version is canonical."
|
||||
baseline = _load_baseline()
|
||||
main_first_in_baseline = [k for k in main_winning if k in baseline]
|
||||
if main_first_in_baseline:
|
||||
pytest.fail(
|
||||
"Issue #239 invariant broken: for at least one duplicated "
|
||||
"(method, path), main.py is now registered FIRST and is "
|
||||
"serving requests instead of the router copy. Audit rounds "
|
||||
"5 and 6 assumed the router handler wins.\n"
|
||||
"\n"
|
||||
"Affected entries:\n"
|
||||
+ "\n".join(f" {k}" for k in main_first_in_baseline)
|
||||
+ "\n\n"
|
||||
"Most likely cause: someone moved app.include_router(...) "
|
||||
"calls in main.py to after the @app.get decorators. Move "
|
||||
"them back to before the @app routes (currently around "
|
||||
"line 3316)."
|
||||
)
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Issue #205 (tg12): the OpenMHZ audio proxy must re-validate the host on
|
||||
every redirect hop, not just the first one.
|
||||
|
||||
Before this fix, ``openmhz_audio_response()`` called
|
||||
``requests.get(..., stream=True, timeout=...)`` with the default
|
||||
``allow_redirects=True``. The initial URL host was validated against
|
||||
``_OPENMHZ_AUDIO_HOSTS``, but any subsequent redirect was silently
|
||||
followed — even to ``http://127.0.0.1:8000`` or RFC1918 internal ranges.
|
||||
Classic open-redirect-to-SSRF.
|
||||
|
||||
After the fix, redirects are followed manually with per-hop host
|
||||
re-validation. Same-host redirects (CDN edge selection) still work,
|
||||
so legitimate audio playback is unaffected.
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from services.radio_intercept import _OPENMHZ_MAX_REDIRECTS, openmhz_audio_response
|
||||
|
||||
|
||||
class _Resp:
|
||||
"""Minimal mock for requests.Response."""
|
||||
|
||||
def __init__(self, status_code=200, headers=None, is_redirect=False):
|
||||
self.status_code = status_code
|
||||
self.headers = headers or {}
|
||||
self.is_redirect = is_redirect
|
||||
self.closed = False
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
def iter_content(self, chunk_size=64 * 1024):
|
||||
return iter([])
|
||||
|
||||
|
||||
@patch("services.radio_intercept.requests.get")
|
||||
def test_redirect_to_internal_address_rejected(mock_get):
|
||||
"""A 302 from media.openmhz.com -> 127.0.0.1 must be rejected."""
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "http://127.0.0.1:8000/api/secret"}, is_redirect=True),
|
||||
]
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
openmhz_audio_response("https://media.openmhz.com/audio/abc.mp3")
|
||||
assert exc_info.value.status_code == 502
|
||||
|
||||
|
||||
@patch("services.radio_intercept.requests.get")
|
||||
def test_redirect_to_arbitrary_domain_rejected(mock_get):
|
||||
"""A 302 to an attacker-controlled domain must be rejected."""
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "https://evil.example/exfil"}, is_redirect=True),
|
||||
]
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
openmhz_audio_response("https://media.openmhz.com/audio/abc.mp3")
|
||||
assert exc_info.value.status_code == 502
|
||||
|
||||
|
||||
@patch("services.radio_intercept.requests.get")
|
||||
def test_redirect_to_another_openmhz_cdn_followed(mock_get):
|
||||
"""A 302 from media.openmhz.com -> media2.openmhz.com (same allowlist) is OK."""
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "https://media2.openmhz.com/audio/abc.mp3"}, is_redirect=True),
|
||||
_Resp(status_code=200, headers={"Content-Type": "audio/mpeg"}),
|
||||
]
|
||||
resp = openmhz_audio_response("https://media.openmhz.com/audio/abc.mp3")
|
||||
# StreamingResponse-shaped object — we just check it was constructed.
|
||||
assert resp is not None
|
||||
|
||||
|
||||
@patch("services.radio_intercept.requests.get")
|
||||
def test_redirect_chain_length_bounded(mock_get):
|
||||
"""A redirect loop must terminate within _OPENMHZ_MAX_REDIRECTS."""
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "https://media.openmhz.com/loop"}, is_redirect=True)
|
||||
for _ in range(_OPENMHZ_MAX_REDIRECTS + 2)
|
||||
]
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
openmhz_audio_response("https://media.openmhz.com/audio/abc.mp3")
|
||||
assert exc_info.value.status_code == 502
|
||||
|
||||
|
||||
@patch("services.radio_intercept.requests.get")
|
||||
def test_redirect_to_http_scheme_rejected(mock_get):
|
||||
"""A 302 to http:// (instead of https://) must be rejected even on same host."""
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "http://media.openmhz.com/audio/abc.mp3"}, is_redirect=True),
|
||||
]
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
openmhz_audio_response("https://media.openmhz.com/audio/abc.mp3")
|
||||
assert exc_info.value.status_code == 502
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Issues #240 & #241 (tg12): oracle market/stake resolution endpoints
|
||||
must require admin authentication.
|
||||
|
||||
Before the fix, ``POST /api/mesh/oracle/resolve`` and
|
||||
``POST /api/mesh/oracle/resolve-stakes`` were decorated with
|
||||
``@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)``. That decorator
|
||||
only tags the route as not requiring a mesh signed-write envelope; it
|
||||
does NOT enforce authorization. The rate limiter (5/minute) was the
|
||||
only real gate, which is wrong for control-plane state mutations.
|
||||
|
||||
The fix adds ``dependencies=[Depends(require_admin)]`` to both routes.
|
||||
These tests prove:
|
||||
|
||||
- Anonymous callers receive 403.
|
||||
- A request bearing the configured admin key passes the auth gate.
|
||||
- The underlying ledger mutator is not invoked on a 403.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
||||
_ADMIN_KEY = "test-admin-key-for-oracle-resolve-fixture-32+"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
"""TestClient with the private-lane transport middleware short-circuited.
|
||||
|
||||
The ``enforce_high_privacy_mesh`` middleware in ``main.py`` returns
|
||||
HTTP 202 ("preparing private lane") for ``/api/mesh/*`` requests
|
||||
when the Wormhole supervisor is not yet at the required transport
|
||||
tier. In tests that's always — Wormhole is not running. Patching
|
||||
``_minimum_transport_tier`` to return None disables the tier check
|
||||
for the duration of the test, letting the request reach the route
|
||||
(and therefore reach the ``Depends(require_admin)`` we are testing).
|
||||
"""
|
||||
import main
|
||||
with patch("main._minimum_transport_tier", return_value=None):
|
||||
yield TestClient(main.app, raise_server_exceptions=False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ledger():
|
||||
"""Replace oracle_ledger methods so tests don't mutate persistent state.
|
||||
|
||||
The handler does ``from services.mesh.mesh_oracle import oracle_ledger``
|
||||
at call time, so we patch the module attribute.
|
||||
"""
|
||||
fake = MagicMock()
|
||||
fake.resolve_market.return_value = (0, 0)
|
||||
fake.resolve_market_stakes.return_value = {"winners": 0, "losers": 0}
|
||||
fake.resolve_expired_stakes.return_value = []
|
||||
with patch("services.mesh.mesh_oracle.oracle_ledger", fake):
|
||||
yield fake
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /api/mesh/oracle/resolve — issue #240
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOracleResolveAuthGate:
|
||||
def test_anonymous_caller_is_rejected(self, client, mock_ledger):
|
||||
with patch("auth._current_admin_key", return_value=_ADMIN_KEY):
|
||||
r = client.post(
|
||||
"/api/mesh/oracle/resolve",
|
||||
json={"market_title": "test-market", "outcome": "Yes"},
|
||||
)
|
||||
assert r.status_code == 403
|
||||
# Critically: the ledger mutator must NOT have been called on a 403.
|
||||
assert mock_ledger.resolve_market.call_count == 0
|
||||
assert mock_ledger.resolve_market_stakes.call_count == 0
|
||||
|
||||
def test_wrong_admin_key_rejected(self, client, mock_ledger):
|
||||
with patch("auth._current_admin_key", return_value=_ADMIN_KEY):
|
||||
r = client.post(
|
||||
"/api/mesh/oracle/resolve",
|
||||
headers={"X-Admin-Key": "this-key-is-wrong"},
|
||||
json={"market_title": "test-market", "outcome": "Yes"},
|
||||
)
|
||||
assert r.status_code == 403
|
||||
assert mock_ledger.resolve_market.call_count == 0
|
||||
|
||||
def test_valid_admin_key_passes_auth_gate(self, client, mock_ledger):
|
||||
with patch("auth._current_admin_key", return_value=_ADMIN_KEY):
|
||||
r = client.post(
|
||||
"/api/mesh/oracle/resolve",
|
||||
headers={"X-Admin-Key": _ADMIN_KEY},
|
||||
json={"market_title": "test-market", "outcome": "Yes"},
|
||||
)
|
||||
# The auth gate let us through. The handler ran and called the
|
||||
# (mocked) ledger.
|
||||
assert r.status_code == 200
|
||||
assert mock_ledger.resolve_market.call_count == 1
|
||||
assert mock_ledger.resolve_market.call_args[0] == ("test-market", "Yes")
|
||||
|
||||
def test_admin_key_unset_blocks_in_production_posture(self, client, mock_ledger):
|
||||
"""When ADMIN_KEY env is not configured at all and we're not in
|
||||
debug, the endpoint must still refuse — never silently accept."""
|
||||
with (
|
||||
patch("auth._current_admin_key", return_value=""),
|
||||
patch("auth._allow_insecure_admin", return_value=False),
|
||||
patch("auth._debug_mode_enabled", return_value=False),
|
||||
patch("auth._scoped_admin_tokens", return_value={}),
|
||||
):
|
||||
r = client.post(
|
||||
"/api/mesh/oracle/resolve",
|
||||
json={"market_title": "test-market", "outcome": "Yes"},
|
||||
)
|
||||
assert r.status_code == 403
|
||||
assert mock_ledger.resolve_market.call_count == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /api/mesh/oracle/resolve-stakes — issue #241
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOracleResolveStakesAuthGate:
|
||||
def test_anonymous_caller_is_rejected(self, client, mock_ledger):
|
||||
with patch("auth._current_admin_key", return_value=_ADMIN_KEY):
|
||||
r = client.post("/api/mesh/oracle/resolve-stakes")
|
||||
assert r.status_code == 403
|
||||
assert mock_ledger.resolve_expired_stakes.call_count == 0
|
||||
|
||||
def test_wrong_admin_key_rejected(self, client, mock_ledger):
|
||||
with patch("auth._current_admin_key", return_value=_ADMIN_KEY):
|
||||
r = client.post(
|
||||
"/api/mesh/oracle/resolve-stakes",
|
||||
headers={"X-Admin-Key": "nope"},
|
||||
)
|
||||
assert r.status_code == 403
|
||||
assert mock_ledger.resolve_expired_stakes.call_count == 0
|
||||
|
||||
def test_valid_admin_key_passes_auth_gate(self, client, mock_ledger):
|
||||
with patch("auth._current_admin_key", return_value=_ADMIN_KEY):
|
||||
r = client.post(
|
||||
"/api/mesh/oracle/resolve-stakes",
|
||||
headers={"X-Admin-Key": _ADMIN_KEY},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert mock_ledger.resolve_expired_stakes.call_count == 1
|
||||
body = r.json()
|
||||
assert body["ok"] is True
|
||||
assert body["count"] == 0
|
||||
|
||||
def test_admin_key_unset_blocks_in_production_posture(self, client, mock_ledger):
|
||||
with (
|
||||
patch("auth._current_admin_key", return_value=""),
|
||||
patch("auth._allow_insecure_admin", return_value=False),
|
||||
patch("auth._debug_mode_enabled", return_value=False),
|
||||
patch("auth._scoped_admin_tokens", return_value={}),
|
||||
):
|
||||
r = client.post("/api/mesh/oracle/resolve-stakes")
|
||||
assert r.status_code == 403
|
||||
assert mock_ledger.resolve_expired_stakes.call_count == 0
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Issue #202 (tg12): the satellite overflights endpoint accepted an
|
||||
unbounded ``hours`` parameter, letting an anonymous caller trigger
|
||||
``O(catalog_size × timesteps)`` work by asking for an absurd window.
|
||||
|
||||
The fix clamps ``hours`` silently rather than raising a 422. The
|
||||
response shape is identical, just covering a shorter window — this
|
||||
keeps the API liberal in what it accepts (Postel) while removing the
|
||||
DoS surface.
|
||||
"""
|
||||
import os
|
||||
|
||||
from routers.data import _overflight_max_hours
|
||||
|
||||
|
||||
def test_default_max_hours_is_72(monkeypatch):
|
||||
monkeypatch.delenv("OVERFLIGHTS_MAX_HOURS", raising=False)
|
||||
assert _overflight_max_hours() == 72
|
||||
|
||||
|
||||
def test_env_override_accepted(monkeypatch):
|
||||
monkeypatch.setenv("OVERFLIGHTS_MAX_HOURS", "168")
|
||||
assert _overflight_max_hours() == 168
|
||||
|
||||
|
||||
def test_invalid_env_value_falls_back_to_default(monkeypatch):
|
||||
monkeypatch.setenv("OVERFLIGHTS_MAX_HOURS", "not-a-number")
|
||||
assert _overflight_max_hours() == 72
|
||||
|
||||
|
||||
def test_negative_env_value_clamped_to_minimum(monkeypatch):
|
||||
monkeypatch.setenv("OVERFLIGHTS_MAX_HOURS", "-5")
|
||||
assert _overflight_max_hours() == 1
|
||||
|
||||
|
||||
def test_clamp_arithmetic_silent():
|
||||
"""The endpoint should clamp huge requests without erroring.
|
||||
|
||||
We don't exercise the full FastAPI route (compute_overflights needs
|
||||
cached GP data), but we do verify the clamping math used by the
|
||||
route: min(requested, cap).
|
||||
"""
|
||||
requested = 1_000_000
|
||||
cap = _overflight_max_hours()
|
||||
effective = min(max(1, requested), cap)
|
||||
assert effective == cap
|
||||
assert effective < requested
|
||||
@@ -87,16 +87,32 @@ class TestRequireLocalOperator:
|
||||
assert self._call_with_host("172.16.0.5") == 403
|
||||
|
||||
def test_docker_bridge_blocked_without_compose_opt_in(self):
|
||||
# Even if DNS would resolve the frontend hostname to this IP,
|
||||
# the env opt-in is required.
|
||||
with patch.dict("os.environ", {"SHADOWBROKER_TRUST_DOCKER_BRIDGE_LOCAL_OPERATOR": ""}):
|
||||
assert self._call_with_host("172.18.0.3") == 403
|
||||
with patch("auth._resolve_trusted_bridge_ips", return_value=frozenset({"172.18.0.3"})):
|
||||
assert self._call_with_host("172.18.0.3") == 403
|
||||
|
||||
def test_docker_bridge_passes_with_compose_opt_in(self):
|
||||
# Issue #250: opt-in alone is no longer sufficient — the source IP
|
||||
# must also reverse-match a trusted frontend container hostname.
|
||||
# Here we simulate Docker DNS resolving "frontend" to 172.18.0.3.
|
||||
with patch.dict("os.environ", {"SHADOWBROKER_TRUST_DOCKER_BRIDGE_LOCAL_OPERATOR": "1"}):
|
||||
assert self._call_with_host("172.18.0.3") == 200
|
||||
with patch("auth._resolve_trusted_bridge_ips", return_value=frozenset({"172.18.0.3"})):
|
||||
assert self._call_with_host("172.18.0.3") == 200
|
||||
|
||||
def test_unknown_bridge_ip_blocked_even_with_compose_opt_in(self):
|
||||
# Issue #250 core regression: a rogue container on the same bridge
|
||||
# whose IP is NOT in the resolved frontend hostname set must NOT
|
||||
# be trusted, even when the bridge opt-in flag is on.
|
||||
with patch.dict("os.environ", {"SHADOWBROKER_TRUST_DOCKER_BRIDGE_LOCAL_OPERATOR": "1"}):
|
||||
with patch("auth._resolve_trusted_bridge_ips", return_value=frozenset({"172.18.0.3"})):
|
||||
assert self._call_with_host("172.18.0.99") == 403
|
||||
|
||||
def test_lan_ip_still_blocked_with_compose_opt_in(self):
|
||||
with patch.dict("os.environ", {"SHADOWBROKER_TRUST_DOCKER_BRIDGE_LOCAL_OPERATOR": "1"}):
|
||||
assert self._call_with_host("192.168.1.100") == 403
|
||||
with patch("auth._resolve_trusted_bridge_ips", return_value=frozenset({"172.18.0.3"})):
|
||||
assert self._call_with_host("192.168.1.100") == 403
|
||||
|
||||
def test_rfc1918_192168_blocked_without_key(self):
|
||||
assert self._call_with_host("192.168.1.100") == 403
|
||||
|
||||
@@ -0,0 +1,366 @@
|
||||
"""Issue #256 (tg12): per-peer HMAC secrets must defeat cross-peer
|
||||
impersonation.
|
||||
|
||||
Before the fix, ALL peer-push HMACs were derived from the single
|
||||
fleet-shared ``MESH_PEER_PUSH_SECRET``. The receiver could only prove
|
||||
"this request was signed by someone who knows the fleet secret" — not
|
||||
which peer signed it. Any peer that knew the secret could compute the
|
||||
expected HMAC for any other peer's URL and impersonate that peer.
|
||||
|
||||
The fix introduces ``MESH_PEER_SECRETS``, a per-peer URL-to-secret map.
|
||||
When a peer URL appears there:
|
||||
|
||||
- Only the listed per-peer secret is accepted for that URL.
|
||||
- The global ``MESH_PEER_PUSH_SECRET`` is ignored for that specific URL.
|
||||
- A peer that knows only the global secret (or a different peer's
|
||||
per-peer secret) cannot forge a request claiming to be that peer.
|
||||
|
||||
When a peer URL is NOT listed (the common case for single-peer installs
|
||||
and for migration windows), the resolver falls back to the global
|
||||
secret — preserving existing behavior with zero operator action.
|
||||
|
||||
These tests exercise ``resolve_peer_key_for_url`` directly so we cover
|
||||
the security contract without spinning up a full mesh node.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _lookup_per_peer_secret — env parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLookupPerPeerSecret:
|
||||
def setup_method(self):
|
||||
# Invalidate the parser cache so each test sees its own env state.
|
||||
from services.mesh import mesh_crypto
|
||||
|
||||
mesh_crypto._PEER_SECRETS_CACHE = {}
|
||||
mesh_crypto._PEER_SECRETS_CACHE_RAW = ""
|
||||
|
||||
def test_returns_empty_when_env_unset(self, monkeypatch):
|
||||
from services.mesh.mesh_crypto import _lookup_per_peer_secret
|
||||
|
||||
monkeypatch.delenv("MESH_PEER_SECRETS", raising=False)
|
||||
assert _lookup_per_peer_secret("https://peer.example") == ""
|
||||
|
||||
def test_returns_empty_when_env_blank(self, monkeypatch):
|
||||
from services.mesh.mesh_crypto import _lookup_per_peer_secret
|
||||
|
||||
monkeypatch.setenv("MESH_PEER_SECRETS", "")
|
||||
assert _lookup_per_peer_secret("https://peer.example") == ""
|
||||
|
||||
def test_returns_per_peer_secret_for_listed_url(self, monkeypatch):
|
||||
from services.mesh.mesh_crypto import _lookup_per_peer_secret
|
||||
|
||||
monkeypatch.setenv(
|
||||
"MESH_PEER_SECRETS",
|
||||
"https://peer-a.example=secretA,https://peer-b.example=secretB",
|
||||
)
|
||||
assert _lookup_per_peer_secret("https://peer-a.example") == "secretA"
|
||||
assert _lookup_per_peer_secret("https://peer-b.example") == "secretB"
|
||||
|
||||
def test_returns_empty_for_url_not_listed(self, monkeypatch):
|
||||
from services.mesh.mesh_crypto import _lookup_per_peer_secret
|
||||
|
||||
monkeypatch.setenv(
|
||||
"MESH_PEER_SECRETS",
|
||||
"https://peer-a.example=secretA",
|
||||
)
|
||||
assert _lookup_per_peer_secret("https://other.example") == ""
|
||||
|
||||
def test_url_is_normalized_before_lookup(self, monkeypatch):
|
||||
from services.mesh.mesh_crypto import _lookup_per_peer_secret
|
||||
|
||||
# Configure with a trailing slash + uppercase host. Lookup with
|
||||
# plain lowercase host. Both should normalize to the same key.
|
||||
monkeypatch.setenv(
|
||||
"MESH_PEER_SECRETS",
|
||||
"https://Peer-A.Example/=secretA",
|
||||
)
|
||||
assert _lookup_per_peer_secret("https://peer-a.example") == "secretA"
|
||||
|
||||
def test_whitespace_around_entries_is_stripped(self, monkeypatch):
|
||||
from services.mesh.mesh_crypto import _lookup_per_peer_secret
|
||||
|
||||
monkeypatch.setenv(
|
||||
"MESH_PEER_SECRETS",
|
||||
" https://peer-a.example = secretA , https://peer-b.example=secretB ",
|
||||
)
|
||||
assert _lookup_per_peer_secret("https://peer-a.example") == "secretA"
|
||||
assert _lookup_per_peer_secret("https://peer-b.example") == "secretB"
|
||||
|
||||
def test_malformed_entries_are_skipped_not_raised(self, monkeypatch):
|
||||
"""A garbled MESH_PEER_SECRETS value must NOT crash the resolver.
|
||||
Bad entries are silently dropped; well-formed entries still work.
|
||||
This is the "fail-forward, not loud" rule — a typo in operator
|
||||
config should not take the whole backend down."""
|
||||
from services.mesh.mesh_crypto import _lookup_per_peer_secret
|
||||
|
||||
monkeypatch.setenv(
|
||||
"MESH_PEER_SECRETS",
|
||||
"no_equals_sign,=missing_url,https://no.secret=,https://good.example=secretGood",
|
||||
)
|
||||
assert _lookup_per_peer_secret("https://good.example") == "secretGood"
|
||||
# The malformed ones produce no entry (and don't poison the cache).
|
||||
assert _lookup_per_peer_secret("https://no.secret") == ""
|
||||
|
||||
def test_cache_invalidates_on_env_change(self, monkeypatch):
|
||||
"""A test (or operator) updating MESH_PEER_SECRETS must see the
|
||||
new value immediately — no process restart required."""
|
||||
from services.mesh.mesh_crypto import _lookup_per_peer_secret
|
||||
|
||||
monkeypatch.setenv("MESH_PEER_SECRETS", "https://a.example=first")
|
||||
assert _lookup_per_peer_secret("https://a.example") == "first"
|
||||
monkeypatch.setenv("MESH_PEER_SECRETS", "https://a.example=second")
|
||||
assert _lookup_per_peer_secret("https://a.example") == "second"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# resolve_peer_key_for_url — precedence + fallback
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestResolvePeerKeyForUrl:
|
||||
def setup_method(self):
|
||||
from services.mesh import mesh_crypto
|
||||
|
||||
mesh_crypto._PEER_SECRETS_CACHE = {}
|
||||
mesh_crypto._PEER_SECRETS_CACHE_RAW = ""
|
||||
|
||||
def _fake_settings(self, global_secret: str):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
s = MagicMock()
|
||||
s.MESH_PEER_PUSH_SECRET = global_secret
|
||||
return s
|
||||
|
||||
def test_falls_back_to_global_when_no_per_peer_entry(self, monkeypatch):
|
||||
"""Single-peer installs: MESH_PEER_SECRETS empty, MESH_PEER_PUSH_SECRET
|
||||
set — must keep working as before."""
|
||||
from services.mesh.mesh_crypto import (
|
||||
resolve_peer_key_for_url,
|
||||
_derive_peer_key,
|
||||
)
|
||||
|
||||
monkeypatch.delenv("MESH_PEER_SECRETS", raising=False)
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(
|
||||
"services.config.get_settings",
|
||||
lambda: self._fake_settings("global-secret"),
|
||||
)
|
||||
key = resolve_peer_key_for_url("https://peer.example")
|
||||
expected = _derive_peer_key("global-secret", "https://peer.example")
|
||||
assert key == expected
|
||||
assert len(key) == 32 # SHA-256 output
|
||||
|
||||
def test_per_peer_secret_takes_precedence_over_global(self, monkeypatch):
|
||||
from services.mesh.mesh_crypto import (
|
||||
resolve_peer_key_for_url,
|
||||
_derive_peer_key,
|
||||
)
|
||||
|
||||
monkeypatch.setenv(
|
||||
"MESH_PEER_SECRETS",
|
||||
"https://peer-a.example=per-peer-a-secret",
|
||||
)
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(
|
||||
"services.config.get_settings",
|
||||
lambda: self._fake_settings("global-secret"),
|
||||
)
|
||||
key = resolve_peer_key_for_url("https://peer-a.example")
|
||||
expected_per_peer = _derive_peer_key(
|
||||
"per-peer-a-secret", "https://peer-a.example"
|
||||
)
|
||||
expected_global = _derive_peer_key("global-secret", "https://peer-a.example")
|
||||
assert key == expected_per_peer
|
||||
assert key != expected_global
|
||||
|
||||
def test_unlisted_peer_uses_global_during_migration(self, monkeypatch):
|
||||
"""Partial migration: peer A is in MESH_PEER_SECRETS, peer B is
|
||||
not yet. Peer B must keep working under the global secret."""
|
||||
from services.mesh.mesh_crypto import (
|
||||
resolve_peer_key_for_url,
|
||||
_derive_peer_key,
|
||||
)
|
||||
|
||||
monkeypatch.setenv(
|
||||
"MESH_PEER_SECRETS",
|
||||
"https://peer-a.example=per-peer-a-secret",
|
||||
)
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(
|
||||
"services.config.get_settings",
|
||||
lambda: self._fake_settings("global-secret"),
|
||||
)
|
||||
key_a = resolve_peer_key_for_url("https://peer-a.example")
|
||||
key_b = resolve_peer_key_for_url("https://peer-b.example")
|
||||
expected_b = _derive_peer_key("global-secret", "https://peer-b.example")
|
||||
assert key_b == expected_b
|
||||
# Peer A's per-peer key must differ from peer B's global key
|
||||
# (they're keyed by different secrets and different URLs).
|
||||
assert key_a != key_b
|
||||
|
||||
def test_returns_empty_when_no_secret_available(self, monkeypatch):
|
||||
from services.mesh.mesh_crypto import resolve_peer_key_for_url
|
||||
|
||||
monkeypatch.delenv("MESH_PEER_SECRETS", raising=False)
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(
|
||||
"services.config.get_settings",
|
||||
lambda: self._fake_settings(""),
|
||||
)
|
||||
key = resolve_peer_key_for_url("https://peer.example")
|
||||
assert key == b""
|
||||
|
||||
def test_returns_empty_when_url_is_unparseable(self, monkeypatch):
|
||||
from services.mesh.mesh_crypto import resolve_peer_key_for_url
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(
|
||||
"services.config.get_settings",
|
||||
lambda: self._fake_settings("global-secret"),
|
||||
)
|
||||
assert resolve_peer_key_for_url("") == b""
|
||||
assert resolve_peer_key_for_url("not-a-url") == b""
|
||||
assert resolve_peer_key_for_url(None) == b""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The actual #256 attack: peer A cannot impersonate peer B
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCrossPeerImpersonationRefused:
|
||||
"""The core regression: when MESH_PEER_SECRETS is configured, a peer
|
||||
that knows ONLY the global secret (or a different peer's per-peer
|
||||
secret) cannot produce a valid HMAC for another peer's URL."""
|
||||
|
||||
def setup_method(self):
|
||||
from services.mesh import mesh_crypto
|
||||
|
||||
mesh_crypto._PEER_SECRETS_CACHE = {}
|
||||
mesh_crypto._PEER_SECRETS_CACHE_RAW = ""
|
||||
|
||||
def _hmac(self, key: bytes, body: bytes) -> str:
|
||||
return hmac.new(key, body, hashlib.sha256).hexdigest()
|
||||
|
||||
def test_peer_a_global_secret_cannot_forge_peer_b_hmac(self, monkeypatch):
|
||||
from services.mesh.mesh_crypto import (
|
||||
resolve_peer_key_for_url,
|
||||
_derive_peer_key,
|
||||
)
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
# Receiver has BOTH the global secret AND a per-peer secret for B.
|
||||
monkeypatch.setenv(
|
||||
"MESH_PEER_SECRETS",
|
||||
"https://peer-b.example=per-peer-b-secret",
|
||||
)
|
||||
settings = MagicMock()
|
||||
settings.MESH_PEER_PUSH_SECRET = "global-secret"
|
||||
monkeypatch.setattr(
|
||||
"services.config.get_settings", lambda: settings
|
||||
)
|
||||
|
||||
body = b'{"events": [{"id": 1}]}'
|
||||
|
||||
# Attacker (peer A) knows only the global secret. Tries to forge
|
||||
# an HMAC claiming to be peer B.
|
||||
attacker_key = _derive_peer_key("global-secret", "https://peer-b.example")
|
||||
attacker_hmac = self._hmac(attacker_key, body)
|
||||
|
||||
# Receiver derives B's expected key from B's per-peer secret.
|
||||
receiver_key = resolve_peer_key_for_url("https://peer-b.example")
|
||||
expected_hmac = self._hmac(receiver_key, body)
|
||||
|
||||
# The forgery MUST NOT match.
|
||||
assert attacker_hmac != expected_hmac
|
||||
|
||||
def test_peer_a_per_peer_secret_cannot_forge_peer_b_hmac(self, monkeypatch):
|
||||
"""Even harder case: peer A has its OWN per-peer secret, but
|
||||
still does not know peer B's per-peer secret, and so cannot
|
||||
forge an HMAC for peer B."""
|
||||
from services.mesh.mesh_crypto import (
|
||||
resolve_peer_key_for_url,
|
||||
_derive_peer_key,
|
||||
)
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
monkeypatch.setenv(
|
||||
"MESH_PEER_SECRETS",
|
||||
"https://peer-a.example=secretA,https://peer-b.example=secretB",
|
||||
)
|
||||
settings = MagicMock()
|
||||
settings.MESH_PEER_PUSH_SECRET = ""
|
||||
monkeypatch.setattr(
|
||||
"services.config.get_settings", lambda: settings
|
||||
)
|
||||
|
||||
body = b'{"events": [{"id": 99}]}'
|
||||
|
||||
# Attacker A tries to forge for B using its own secret (secretA).
|
||||
attacker_key = _derive_peer_key("secretA", "https://peer-b.example")
|
||||
attacker_hmac = self._hmac(attacker_key, body)
|
||||
|
||||
receiver_key = resolve_peer_key_for_url("https://peer-b.example")
|
||||
expected_hmac = self._hmac(receiver_key, body)
|
||||
|
||||
assert attacker_hmac != expected_hmac
|
||||
|
||||
def test_legitimate_peer_b_request_verifies(self, monkeypatch):
|
||||
"""Positive control: when peer B uses ITS per-peer secret and
|
||||
claims to be itself, the receiver accepts the HMAC."""
|
||||
from services.mesh.mesh_crypto import resolve_peer_key_for_url
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
monkeypatch.setenv(
|
||||
"MESH_PEER_SECRETS",
|
||||
"https://peer-b.example=secretB",
|
||||
)
|
||||
settings = MagicMock()
|
||||
settings.MESH_PEER_PUSH_SECRET = ""
|
||||
monkeypatch.setattr(
|
||||
"services.config.get_settings", lambda: settings
|
||||
)
|
||||
|
||||
body = b'{"events": [{"id": 7}]}'
|
||||
|
||||
# Peer B and the receiver both call resolve_peer_key_for_url.
|
||||
sender_key = resolve_peer_key_for_url("https://peer-b.example")
|
||||
receiver_key = resolve_peer_key_for_url("https://peer-b.example")
|
||||
|
||||
sender_hmac = self._hmac(sender_key, body)
|
||||
expected_hmac = self._hmac(receiver_key, body)
|
||||
|
||||
assert sender_hmac == expected_hmac
|
||||
|
||||
def test_single_peer_install_zero_behavior_change(self, monkeypatch):
|
||||
"""The "no UX hostility" guarantee: an install with the global
|
||||
secret set and NO MESH_PEER_SECRETS entries must derive exactly
|
||||
the same key as before this change."""
|
||||
from services.mesh.mesh_crypto import (
|
||||
resolve_peer_key_for_url,
|
||||
_derive_peer_key,
|
||||
)
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
monkeypatch.delenv("MESH_PEER_SECRETS", raising=False)
|
||||
settings = MagicMock()
|
||||
settings.MESH_PEER_PUSH_SECRET = "legacy-global-secret"
|
||||
monkeypatch.setattr(
|
||||
"services.config.get_settings", lambda: settings
|
||||
)
|
||||
|
||||
# The legacy derivation that every prior call site used.
|
||||
legacy_key = _derive_peer_key("legacy-global-secret", "https://peer.example")
|
||||
# The new resolver, with no per-peer entries configured.
|
||||
new_key = resolve_peer_key_for_url("https://peer.example")
|
||||
|
||||
assert new_key == legacy_key
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Issues #218 / #219 (tg12): outbound Wikipedia + Wikidata calls must
|
||||
identify ShadowBroker via the Wikimedia-recommended User-Agent /
|
||||
Api-User-Agent headers.
|
||||
|
||||
Before this fix, ``backend/services/region_dossier.py`` called
|
||||
``fetch_with_curl(url)`` with no explicit headers, falling back to the
|
||||
generic project default UA. That sent a too-anonymous identifier to
|
||||
Wikimedia. Per Wikimedia's policy
|
||||
(https://foundation.wikimedia.org/wiki/Policy:Wikimedia_Foundation_User-Agent_Policy)
|
||||
the API caller should send a stable, contactable identifier so Wikimedia
|
||||
operators can rate-limit or reach the project.
|
||||
|
||||
This test does NOT make network calls. It patches ``fetch_with_curl``
|
||||
and asserts the headers that get passed through.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _fake_resp(payload: dict, status: int = 200) -> MagicMock:
|
||||
r = MagicMock()
|
||||
r.status_code = status
|
||||
r.json.return_value = payload
|
||||
return r
|
||||
|
||||
|
||||
def test_wikidata_call_passes_wikimedia_request_headers():
|
||||
from services import region_dossier
|
||||
|
||||
calls = []
|
||||
|
||||
def fake_fetch(url, **kwargs):
|
||||
calls.append(kwargs.get("headers"))
|
||||
return _fake_resp({"results": {"bindings": []}})
|
||||
|
||||
with patch.object(region_dossier, "fetch_with_curl", side_effect=fake_fetch):
|
||||
region_dossier._fetch_wikidata_leader("Testlandia")
|
||||
|
||||
assert calls, "fetch_with_curl was not called"
|
||||
headers = calls[0] or {}
|
||||
assert "User-Agent" in headers
|
||||
assert "Api-User-Agent" in headers
|
||||
# Stable identifier should mention the project + a contact path.
|
||||
assert "Shadowbroker" in headers["Api-User-Agent"] or "ShadowBroker" in headers["Api-User-Agent"]
|
||||
assert "github.com" in headers["Api-User-Agent"].lower()
|
||||
|
||||
|
||||
def test_wikipedia_summary_call_passes_wikimedia_request_headers():
|
||||
from services import region_dossier
|
||||
|
||||
calls = []
|
||||
|
||||
def fake_fetch(url, **kwargs):
|
||||
calls.append((url, kwargs.get("headers")))
|
||||
return _fake_resp(
|
||||
{
|
||||
"type": "standard",
|
||||
"description": "test desc",
|
||||
"extract": "test extract",
|
||||
"thumbnail": {"source": ""},
|
||||
}
|
||||
)
|
||||
|
||||
with patch.object(region_dossier, "fetch_with_curl", side_effect=fake_fetch):
|
||||
region_dossier._fetch_local_wiki_summary("Paris", "France")
|
||||
|
||||
# At least one Wikipedia REST call was issued.
|
||||
wikipedia_calls = [c for c in calls if "wikipedia.org" in c[0]]
|
||||
assert wikipedia_calls, "no Wikipedia call was issued"
|
||||
for url, headers in wikipedia_calls:
|
||||
headers = headers or {}
|
||||
assert "User-Agent" in headers, f"missing User-Agent on {url}"
|
||||
assert "Api-User-Agent" in headers, f"missing Api-User-Agent on {url}"
|
||||
assert "github.com" in headers["Api-User-Agent"].lower()
|
||||
|
||||
|
||||
def test_wikimedia_headers_constant_is_stable():
|
||||
"""Regression guard: if someone removes the contact path from the
|
||||
Api-User-Agent we want a loud test failure, not a silent ToS drift.
|
||||
"""
|
||||
from services.region_dossier import _WIKIMEDIA_REQUEST_HEADERS
|
||||
|
||||
aua = _WIKIMEDIA_REQUEST_HEADERS.get("Api-User-Agent", "")
|
||||
assert "Shadowbroker" in aua or "ShadowBroker" in aua
|
||||
assert "github.com" in aua.lower()
|
||||
# Must include a path Wikimedia operators can use to contact us
|
||||
# (we use /issues against the public repo).
|
||||
assert "issues" in aua.lower()
|
||||
@@ -0,0 +1,263 @@
|
||||
"""Issues #243, #252, #253 (tg12): settings endpoints must not leak
|
||||
operational posture to unauthenticated callers.
|
||||
|
||||
- **#243**: ``GET /api/settings/wormhole``, ``/api/settings/privacy-profile``,
|
||||
and ``/api/settings/node`` were leaking transport choice, anonymous-mode
|
||||
state, the named privacy profile, and node-participant state to any
|
||||
unauthenticated caller. The fix tightens the redaction allowlists to
|
||||
expose ONLY a bare "is this feature on?" boolean and gates node mode
|
||||
behind authenticated reads.
|
||||
|
||||
- **#252**: ``GET /api/settings/news-feeds`` returned the operator's full
|
||||
curated feed inventory (names + URLs) to anyone. Now gated on
|
||||
local-operator.
|
||||
|
||||
- **#253**: ``GET /api/settings/timemachine`` returned whether archival
|
||||
capture is enabled to anyone. Now gated on local-operator.
|
||||
|
||||
Auth model: ``require_local_operator`` allows loopback (Tauri shell),
|
||||
the Docker bridge frontend container (via the hostname-bound trust from
|
||||
PR #278), and any caller that presents the configured admin key.
|
||||
Anonymous LAN or internet callers do NOT pass and either receive 403
|
||||
(news-feeds, timemachine) or a redacted minimum (wormhole / node).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
||||
_ADMIN_KEY = "test-admin-key-for-round5-fixture-32+chars"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
"""TestClient with the private-lane transport middleware disabled.
|
||||
|
||||
Same shape as the oracle resolve fixture — the mesh privacy
|
||||
middleware returns 202 for ``/api/settings/*`` under TestClient
|
||||
because Wormhole is not actually running. Patching out the tier
|
||||
requirement lets requests reach the route's auth gate.
|
||||
"""
|
||||
import main
|
||||
with patch("main._minimum_transport_tier", return_value=None):
|
||||
yield TestClient(main.app, raise_server_exceptions=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #243: Wormhole posture redaction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestWormholeSettingsRedaction:
|
||||
"""``GET /api/settings/wormhole`` must NOT leak transport choice or
|
||||
anonymous-mode state to unauthenticated callers."""
|
||||
|
||||
def _read_settings_payload(self):
|
||||
return {
|
||||
"enabled": True,
|
||||
"transport": "tor_arti",
|
||||
"anonymous_mode": True,
|
||||
"privacy_profile": "high",
|
||||
"socks_proxy": "socks5h://127.0.0.1:9050",
|
||||
}
|
||||
|
||||
def test_anonymous_caller_sees_only_enabled_bool(self, client):
|
||||
with (
|
||||
patch("main.read_wormhole_settings", return_value=self._read_settings_payload()),
|
||||
patch("routers.wormhole.read_wormhole_settings", return_value=self._read_settings_payload()),
|
||||
patch("services.wormhole_settings.read_wormhole_settings", return_value=self._read_settings_payload()),
|
||||
patch("auth._current_admin_key", return_value=_ADMIN_KEY),
|
||||
):
|
||||
r = client.get("/api/settings/wormhole")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
# Only the bare "is Wormhole on?" boolean is exposed publicly.
|
||||
assert "enabled" in body
|
||||
assert body["enabled"] is True
|
||||
# Posture fields the audit flagged must be absent.
|
||||
assert "transport" not in body
|
||||
assert "anonymous_mode" not in body
|
||||
assert "privacy_profile" not in body
|
||||
assert "socks_proxy" not in body
|
||||
|
||||
def test_authenticated_caller_sees_full_state(self, client):
|
||||
with (
|
||||
patch("main.read_wormhole_settings", return_value=self._read_settings_payload()),
|
||||
patch("routers.wormhole.read_wormhole_settings", return_value=self._read_settings_payload()),
|
||||
patch("services.wormhole_settings.read_wormhole_settings", return_value=self._read_settings_payload()),
|
||||
patch("auth._current_admin_key", return_value=_ADMIN_KEY),
|
||||
):
|
||||
r = client.get(
|
||||
"/api/settings/wormhole",
|
||||
headers={"X-Admin-Key": _ADMIN_KEY},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
# All fields visible when authenticated.
|
||||
assert body["enabled"] is True
|
||||
assert body["transport"] == "tor_arti"
|
||||
assert body["anonymous_mode"] is True
|
||||
assert body["privacy_profile"] == "high"
|
||||
|
||||
|
||||
class TestPrivacyProfileRedaction:
|
||||
"""``GET /api/settings/privacy-profile`` must NOT leak the named
|
||||
profile to unauthenticated callers (the profile name itself
|
||||
discloses operator intent)."""
|
||||
|
||||
def _payload(self):
|
||||
return {
|
||||
"enabled": True,
|
||||
"transport": "tor_arti",
|
||||
"anonymous_mode": True,
|
||||
"privacy_profile": "high",
|
||||
}
|
||||
|
||||
def test_anonymous_caller_sees_only_wormhole_enabled_bool(self, client):
|
||||
with (
|
||||
patch("main.read_wormhole_settings", return_value=self._payload()),
|
||||
patch("routers.wormhole.read_wormhole_settings", return_value=self._payload()),
|
||||
patch("services.wormhole_settings.read_wormhole_settings", return_value=self._payload()),
|
||||
patch("auth._current_admin_key", return_value=_ADMIN_KEY),
|
||||
):
|
||||
r = client.get("/api/settings/privacy-profile")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert "wormhole_enabled" in body
|
||||
assert body["wormhole_enabled"] is True
|
||||
# The named profile, transport, and anonymous mode must NOT
|
||||
# leak to anonymous callers.
|
||||
assert "profile" not in body or body.get("profile") is None
|
||||
assert "transport" not in body
|
||||
assert "anonymous_mode" not in body
|
||||
|
||||
def test_authenticated_caller_sees_named_profile_and_transport(self, client):
|
||||
with (
|
||||
patch("main.read_wormhole_settings", return_value=self._payload()),
|
||||
patch("routers.wormhole.read_wormhole_settings", return_value=self._payload()),
|
||||
patch("services.wormhole_settings.read_wormhole_settings", return_value=self._payload()),
|
||||
patch("auth._current_admin_key", return_value=_ADMIN_KEY),
|
||||
):
|
||||
r = client.get(
|
||||
"/api/settings/privacy-profile",
|
||||
headers={"X-Admin-Key": _ADMIN_KEY},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["profile"] == "high"
|
||||
assert body["wormhole_enabled"] is True
|
||||
assert body["transport"] == "tor_arti"
|
||||
assert body["anonymous_mode"] is True
|
||||
|
||||
|
||||
class TestNodeSettingsRedaction:
|
||||
"""``GET /api/settings/node`` must NOT disclose node_mode or
|
||||
node_enabled to anonymous callers."""
|
||||
|
||||
def _node_data(self):
|
||||
return {"some_node_field": "value"}
|
||||
|
||||
def test_anonymous_caller_sees_empty_stub(self, client):
|
||||
with (
|
||||
patch("services.node_settings.read_node_settings", return_value=self._node_data()),
|
||||
patch("routers.admin._current_node_mode", return_value="participant"),
|
||||
patch("routers.admin._participant_node_enabled", return_value=True),
|
||||
patch("auth._current_admin_key", return_value=_ADMIN_KEY),
|
||||
):
|
||||
r = client.get("/api/settings/node")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
# No posture fields.
|
||||
assert "node_mode" not in body
|
||||
assert "node_enabled" not in body
|
||||
assert "some_node_field" not in body
|
||||
|
||||
def test_authenticated_caller_sees_full_node_state(self, client):
|
||||
with (
|
||||
patch("services.node_settings.read_node_settings", return_value=self._node_data()),
|
||||
patch("routers.admin._current_node_mode", return_value="participant"),
|
||||
patch("routers.admin._participant_node_enabled", return_value=True),
|
||||
patch("auth._current_admin_key", return_value=_ADMIN_KEY),
|
||||
):
|
||||
r = client.get(
|
||||
"/api/settings/node",
|
||||
headers={"X-Admin-Key": _ADMIN_KEY},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["node_mode"] == "participant"
|
||||
assert body["node_enabled"] is True
|
||||
assert body["some_node_field"] == "value"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #252: news-feeds auth gate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestNewsFeedsAuthGate:
|
||||
def _fake_feeds(self):
|
||||
return [
|
||||
{"name": "Custom Internal", "url": "https://internal.example/rss", "weight": 5},
|
||||
{"name": "Default News", "url": "https://news.example/rss", "weight": 3},
|
||||
]
|
||||
|
||||
def test_anonymous_caller_rejected(self, client):
|
||||
with (
|
||||
patch("services.news_feed_config.get_feeds", return_value=self._fake_feeds()) as get_feeds,
|
||||
patch("auth._current_admin_key", return_value=_ADMIN_KEY),
|
||||
):
|
||||
r = client.get("/api/settings/news-feeds")
|
||||
assert r.status_code == 403
|
||||
# Critically: the underlying config read must NOT have been performed
|
||||
# (else the response body could leak the count via response timing).
|
||||
assert get_feeds.call_count == 0
|
||||
|
||||
def test_authenticated_caller_sees_full_feed_inventory(self, client):
|
||||
with (
|
||||
patch("services.news_feed_config.get_feeds", return_value=self._fake_feeds()),
|
||||
patch("auth._current_admin_key", return_value=_ADMIN_KEY),
|
||||
):
|
||||
r = client.get(
|
||||
"/api/settings/news-feeds",
|
||||
headers={"X-Admin-Key": _ADMIN_KEY},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert len(body) == 2
|
||||
assert body[0]["name"] == "Custom Internal"
|
||||
assert body[0]["url"] == "https://internal.example/rss"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #253: timemachine auth gate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestTimemachineAuthGate:
|
||||
def test_anonymous_caller_rejected(self, client):
|
||||
node_data = {"timemachine_enabled": True}
|
||||
with (
|
||||
patch("services.node_settings.read_node_settings", return_value=node_data),
|
||||
patch("auth._current_admin_key", return_value=_ADMIN_KEY),
|
||||
):
|
||||
r = client.get("/api/settings/timemachine")
|
||||
assert r.status_code == 403
|
||||
|
||||
def test_authenticated_caller_sees_enabled_state(self, client):
|
||||
node_data = {"timemachine_enabled": True}
|
||||
with (
|
||||
patch("services.node_settings.read_node_settings", return_value=node_data),
|
||||
patch("auth._current_admin_key", return_value=_ADMIN_KEY),
|
||||
):
|
||||
r = client.get(
|
||||
"/api/settings/timemachine",
|
||||
headers={"X-Admin-Key": _ADMIN_KEY},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["enabled"] is True
|
||||
assert "storage_warning" in body
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Issue #200 (tg12): Sentinel token cache must require knowledge of the
|
||||
client secret to hit, not just client_id.
|
||||
|
||||
Before this fix, the cache lookup was ``_sh_token_cache["client_id"] ==
|
||||
client_id``. A caller who knew a valid client_id but supplied any secret
|
||||
would hit the cache and reuse the original caller's bearer token, burning
|
||||
their Copernicus quota and accessing imagery on their account.
|
||||
|
||||
After the fix, the cache key is an HMAC of ``(client_id, client_secret)``
|
||||
under a per-process random key, so two callers with the same client_id but
|
||||
different secrets compute different fingerprints and miss each other's
|
||||
cache entries.
|
||||
"""
|
||||
from routers.tools import _credential_fingerprint, _sh_token_cache
|
||||
|
||||
|
||||
def test_same_client_id_different_secrets_yield_different_fingerprints():
|
||||
fp_a = _credential_fingerprint("client-id-X", "secret-A")
|
||||
fp_b = _credential_fingerprint("client-id-X", "secret-B")
|
||||
assert fp_a != fp_b
|
||||
|
||||
|
||||
def test_same_credentials_yield_same_fingerprint():
|
||||
"""The cache is still useful — same caller hits its own entry."""
|
||||
fp1 = _credential_fingerprint("client-id-X", "secret-A")
|
||||
fp2 = _credential_fingerprint("client-id-X", "secret-A")
|
||||
assert fp1 == fp2
|
||||
|
||||
|
||||
def test_different_client_ids_yield_different_fingerprints():
|
||||
fp_a = _credential_fingerprint("client-id-A", "shared-secret")
|
||||
fp_b = _credential_fingerprint("client-id-B", "shared-secret")
|
||||
assert fp_a != fp_b
|
||||
|
||||
|
||||
def test_cache_lookup_key_field_renamed():
|
||||
"""Catch accidental reintroduction of the client_id-only lookup."""
|
||||
# If a future commit re-adds `_sh_token_cache["client_id"]` we want this
|
||||
# test to fail loudly. The new schema only stores `credential_fp`.
|
||||
assert "client_id" not in _sh_token_cache
|
||||
assert "credential_fp" in _sh_token_cache
|
||||
|
||||
|
||||
def test_attacker_with_wrong_secret_misses_cache(monkeypatch):
|
||||
"""An attacker with valid client_id but wrong secret cannot hit the cache."""
|
||||
# Populate cache as if a legitimate caller just succeeded.
|
||||
legit_fp = _credential_fingerprint("legit-client", "legit-secret")
|
||||
_sh_token_cache["token"] = "VICTIM-BEARER-TOKEN"
|
||||
_sh_token_cache["credential_fp"] = legit_fp
|
||||
_sh_token_cache["expiry"] = 10**12 # far future
|
||||
|
||||
# Attacker arrives with the same client_id but the wrong secret.
|
||||
attacker_fp = _credential_fingerprint("legit-client", "wrong-secret")
|
||||
assert attacker_fp != legit_fp
|
||||
|
||||
# Reset cache for hygiene between tests.
|
||||
_sh_token_cache["token"] = None
|
||||
_sh_token_cache["credential_fp"] = ""
|
||||
_sh_token_cache["expiry"] = 0
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Third-party fetchers that phone home to politically/commercially
|
||||
sensitive upstreams must be operator opt-in only.
|
||||
|
||||
Companion to ``test_crowdthreat_opt_in.py`` — extends the same default-off
|
||||
posture to:
|
||||
|
||||
* EUvsDisinfo FIMI (``FIMI_ENABLED``)
|
||||
* Polymarket + Kalshi (``PREDICTION_MARKETS_ENABLED``)
|
||||
* Finnhub / yfinance financial data (``FINANCIAL_ENABLED`` /
|
||||
``FINNHUB_API_KEY``)
|
||||
* NUFORC HuggingFace dataset (``NUFORC_ENABLED``)
|
||||
|
||||
Each test asserts that with the env var unset (or set to a falsy value)
|
||||
the fetcher's network entry point is NOT called.
|
||||
"""
|
||||
|
||||
|
||||
def _explode(*_args, **_kwargs):
|
||||
raise AssertionError("upstream called while fetcher was meant to be disabled")
|
||||
|
||||
|
||||
def test_fimi_disabled_by_default_does_not_call_upstream(monkeypatch):
|
||||
from services.fetchers import _store, fimi
|
||||
|
||||
monkeypatch.delenv("FIMI_ENABLED", raising=False)
|
||||
monkeypatch.setitem(_store.latest_data, "fimi", [{"id": "old"}])
|
||||
monkeypatch.setattr(fimi, "fetch_with_curl", _explode)
|
||||
|
||||
fimi.fetch_fimi()
|
||||
|
||||
assert _store.latest_data["fimi"] == []
|
||||
|
||||
|
||||
def test_fimi_falsy_value_does_not_call_upstream(monkeypatch):
|
||||
from services.fetchers import _store, fimi
|
||||
|
||||
monkeypatch.setenv("FIMI_ENABLED", "false")
|
||||
monkeypatch.setitem(_store.latest_data, "fimi", [{"id": "old"}])
|
||||
monkeypatch.setattr(fimi, "fetch_with_curl", _explode)
|
||||
|
||||
fimi.fetch_fimi()
|
||||
|
||||
assert _store.latest_data["fimi"] == []
|
||||
|
||||
|
||||
def test_prediction_markets_disabled_by_default(monkeypatch):
|
||||
from services.fetchers import _store, prediction_markets
|
||||
|
||||
monkeypatch.delenv("PREDICTION_MARKETS_ENABLED", raising=False)
|
||||
monkeypatch.setitem(_store.latest_data, "prediction_markets", [{"id": "old"}])
|
||||
monkeypatch.setattr(
|
||||
prediction_markets, "fetch_prediction_markets_raw", _explode
|
||||
)
|
||||
|
||||
prediction_markets.fetch_prediction_markets()
|
||||
|
||||
assert _store.latest_data["prediction_markets"] == []
|
||||
|
||||
|
||||
def test_financial_disabled_when_no_optin_or_api_key(monkeypatch):
|
||||
"""yfinance fallback path must not run silently — needs FINANCIAL_ENABLED."""
|
||||
from services.fetchers import _store, financial
|
||||
|
||||
monkeypatch.delenv("FINANCIAL_ENABLED", raising=False)
|
||||
monkeypatch.delenv("FINNHUB_API_KEY", raising=False)
|
||||
monkeypatch.setitem(_store.latest_data, "financial", {"BTC": {"price": 1}})
|
||||
monkeypatch.setattr(financial, "_fetch_finnhub_quote", _explode)
|
||||
monkeypatch.setattr(financial, "_fetch_yfinance_single", _explode)
|
||||
|
||||
financial.fetch_financial_markets()
|
||||
|
||||
assert _store.latest_data["financial"] == {}
|
||||
|
||||
|
||||
def test_financial_enabled_via_finnhub_api_key(monkeypatch):
|
||||
"""Presence of FINNHUB_API_KEY counts as explicit opt-in."""
|
||||
from services.fetchers import financial
|
||||
|
||||
monkeypatch.delenv("FINANCIAL_ENABLED", raising=False)
|
||||
monkeypatch.setenv("FINNHUB_API_KEY", "test-key")
|
||||
|
||||
assert financial.financial_fetch_enabled() is True
|
||||
|
||||
|
||||
def test_nuforc_disabled_by_default_skips_download(monkeypatch):
|
||||
from services.fetchers import nuforc_enrichment
|
||||
|
||||
monkeypatch.delenv("NUFORC_ENABLED", raising=False)
|
||||
monkeypatch.setattr(nuforc_enrichment, "fetch_with_curl", _explode)
|
||||
|
||||
result = nuforc_enrichment._download_and_build()
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_news_default_on_but_killable(monkeypatch):
|
||||
"""News defaults on (kill switch only), but NEWS_ENABLED=false must disable it."""
|
||||
from services.fetchers import _store, news
|
||||
|
||||
monkeypatch.setenv("NEWS_ENABLED", "false")
|
||||
monkeypatch.setitem(_store.latest_data, "news", [{"id": "old"}])
|
||||
monkeypatch.setattr(news, "fetch_with_curl", _explode)
|
||||
|
||||
news.fetch_news()
|
||||
|
||||
assert _store.latest_data["news"] == []
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Issue #251 (tg12): Tor bundle extraction must refuse symlink and
|
||||
hardlink members.
|
||||
|
||||
The previous extractor checked ``member.name`` against path traversal
|
||||
but never inspected ``member.linkname``. Python 3.11's ``tarfile``
|
||||
honors symlinks during ``extractall()``, so a malicious archive could
|
||||
ship a member named ``innocent.txt`` whose linkname points at an
|
||||
arbitrary filesystem location. After extraction, reads of innocent.txt
|
||||
dereference to that location; writes corrupt it.
|
||||
|
||||
The fix categorically refuses any link member during extraction.
|
||||
Tor Expert Bundles never legitimately contain symlinks or hardlinks,
|
||||
so this is non-disruptive for real updates and a hard stop for hostile
|
||||
archives.
|
||||
|
||||
These tests build synthetic tar archives covering each refused case
|
||||
and assert ``_extract_tor_bundle_safely`` rejects them.
|
||||
"""
|
||||
import io
|
||||
import os
|
||||
import stat
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from services.tor_hidden_service import _extract_tor_bundle_safely
|
||||
|
||||
|
||||
def _build_archive(tmp_path: Path, members: list) -> Path:
|
||||
"""Write a .tar.gz with the given (name, builder) pairs.
|
||||
|
||||
Each builder is called with the open tarfile and is responsible for
|
||||
adding its member however it likes (regular file, symlink, etc.).
|
||||
"""
|
||||
archive = tmp_path / "test_bundle.tar.gz"
|
||||
with tarfile.open(str(archive), "w:gz") as tar:
|
||||
for name, builder in members:
|
||||
builder(tar, name)
|
||||
return archive
|
||||
|
||||
|
||||
def _add_regular_file(tar: tarfile.TarFile, name: str, payload: bytes = b"hello") -> None:
|
||||
info = tarfile.TarInfo(name)
|
||||
info.size = len(payload)
|
||||
info.mode = 0o644
|
||||
info.type = tarfile.REGTYPE
|
||||
tar.addfile(info, io.BytesIO(payload))
|
||||
|
||||
|
||||
def _add_symlink(tar: tarfile.TarFile, name: str, linkname: str) -> None:
|
||||
info = tarfile.TarInfo(name)
|
||||
info.size = 0
|
||||
info.type = tarfile.SYMTYPE
|
||||
info.linkname = linkname
|
||||
info.mode = 0o777
|
||||
tar.addfile(info)
|
||||
|
||||
|
||||
def _add_hardlink(tar: tarfile.TarFile, name: str, linkname: str) -> None:
|
||||
info = tarfile.TarInfo(name)
|
||||
info.size = 0
|
||||
info.type = tarfile.LNKTYPE
|
||||
info.linkname = linkname
|
||||
info.mode = 0o644
|
||||
tar.addfile(info)
|
||||
|
||||
|
||||
def _add_fifo(tar: tarfile.TarFile, name: str) -> None:
|
||||
info = tarfile.TarInfo(name)
|
||||
info.type = tarfile.FIFOTYPE
|
||||
info.mode = 0o644
|
||||
tar.addfile(info)
|
||||
|
||||
|
||||
def test_clean_archive_extracts_successfully(tmp_path):
|
||||
"""A normal archive with only regular files extracts fine."""
|
||||
install_dir = tmp_path / "install"
|
||||
install_dir.mkdir()
|
||||
|
||||
def add_normal(tar, name):
|
||||
_add_regular_file(tar, name, b"clean content")
|
||||
|
||||
archive = _build_archive(
|
||||
tmp_path,
|
||||
[
|
||||
("tor/tor.exe", add_normal),
|
||||
("tor/data/geoip", add_normal),
|
||||
],
|
||||
)
|
||||
assert _extract_tor_bundle_safely(archive, install_dir) is True
|
||||
assert (install_dir / "tor" / "tor.exe").is_file()
|
||||
assert (install_dir / "tor" / "data" / "geoip").is_file()
|
||||
|
||||
|
||||
def test_symlink_member_is_rejected(tmp_path, caplog):
|
||||
"""Issue #251 core regression: symlink members are refused."""
|
||||
install_dir = tmp_path / "install"
|
||||
install_dir.mkdir()
|
||||
|
||||
archive = _build_archive(
|
||||
tmp_path,
|
||||
[
|
||||
("tor/innocent.txt", lambda t, n: _add_symlink(t, n, "/etc/passwd")),
|
||||
],
|
||||
)
|
||||
|
||||
import logging
|
||||
|
||||
with caplog.at_level(logging.ERROR):
|
||||
result = _extract_tor_bundle_safely(archive, install_dir)
|
||||
|
||||
assert result is False
|
||||
# No file should have been created
|
||||
assert not (install_dir / "tor" / "innocent.txt").exists()
|
||||
# Log should explain why
|
||||
assert any(
|
||||
"symlinks/hardlinks are not allowed" in rec.getMessage()
|
||||
for rec in caplog.records
|
||||
)
|
||||
|
||||
|
||||
def test_hardlink_member_is_rejected(tmp_path):
|
||||
"""Hardlinks are refused for the same reason as symlinks."""
|
||||
install_dir = tmp_path / "install"
|
||||
install_dir.mkdir()
|
||||
|
||||
archive = _build_archive(
|
||||
tmp_path,
|
||||
[
|
||||
("tor/regular.txt", lambda t, n: _add_regular_file(t, n)),
|
||||
("tor/sneaky.txt", lambda t, n: _add_hardlink(t, n, "regular.txt")),
|
||||
],
|
||||
)
|
||||
assert _extract_tor_bundle_safely(archive, install_dir) is False
|
||||
# The whole extraction is refused even though only one member is bad.
|
||||
assert not (install_dir / "tor" / "regular.txt").exists()
|
||||
|
||||
|
||||
def test_symlink_with_relative_target_still_rejected(tmp_path):
|
||||
"""Even a relative symlink target inside the install dir is refused.
|
||||
|
||||
We don't allow symlinks at all — there is no legitimate Tor bundle
|
||||
use case for them, and an attacker can chain link redirections in
|
||||
ways the path-resolution check is poor at catching.
|
||||
"""
|
||||
install_dir = tmp_path / "install"
|
||||
install_dir.mkdir()
|
||||
|
||||
archive = _build_archive(
|
||||
tmp_path,
|
||||
[
|
||||
("tor/alias.txt", lambda t, n: _add_symlink(t, n, "tor/tor.exe")),
|
||||
],
|
||||
)
|
||||
assert _extract_tor_bundle_safely(archive, install_dir) is False
|
||||
|
||||
|
||||
def test_fifo_or_device_member_is_rejected(tmp_path):
|
||||
"""Non-regular-non-directory members (FIFOs, devices) are refused."""
|
||||
install_dir = tmp_path / "install"
|
||||
install_dir.mkdir()
|
||||
|
||||
archive = _build_archive(
|
||||
tmp_path,
|
||||
[
|
||||
("tor/weird.fifo", _add_fifo),
|
||||
],
|
||||
)
|
||||
assert _extract_tor_bundle_safely(archive, install_dir) is False
|
||||
|
||||
|
||||
def test_path_traversal_member_is_rejected(tmp_path):
|
||||
"""Pre-existing path-traversal guard still works under the new shape."""
|
||||
install_dir = tmp_path / "install"
|
||||
install_dir.mkdir()
|
||||
|
||||
def add_traversal(tar, name):
|
||||
_add_regular_file(tar, name)
|
||||
|
||||
# ../../escape.txt resolves outside install_dir on most platforms.
|
||||
archive = _build_archive(
|
||||
tmp_path,
|
||||
[
|
||||
("../../escape.txt", add_traversal),
|
||||
],
|
||||
)
|
||||
assert _extract_tor_bundle_safely(archive, install_dir) is False
|
||||
|
||||
|
||||
def test_malformed_tar_is_rejected(tmp_path):
|
||||
"""A corrupt/non-tar file is rejected without crashing."""
|
||||
install_dir = tmp_path / "install"
|
||||
install_dir.mkdir()
|
||||
|
||||
bogus = tmp_path / "not-a-tar.tar.gz"
|
||||
bogus.write_bytes(b"this is not a tar archive at all")
|
||||
|
||||
assert _extract_tor_bundle_safely(bogus, install_dir) is False
|
||||
|
||||
|
||||
def test_extraction_failure_does_not_leave_partial_state_referenced_to_caller(tmp_path):
|
||||
"""When extraction fails partway, the caller relies on a False return
|
||||
to know it must clean up. We test the contract here — actual cleanup
|
||||
of files that may have been written by tar.extractall() before the
|
||||
failure point isn't part of THIS helper's responsibility (the caller
|
||||
deletes the install dir if needed)."""
|
||||
install_dir = tmp_path / "install"
|
||||
install_dir.mkdir()
|
||||
|
||||
# Hostile archive: one good file, then a symlink. Whether the good
|
||||
# file was written or not, the return value must be False so the
|
||||
# caller refuses the bundle.
|
||||
archive = _build_archive(
|
||||
tmp_path,
|
||||
[
|
||||
("tor/clean.txt", lambda t, n: _add_regular_file(t, n)),
|
||||
("tor/evil-link.txt", lambda t, n: _add_symlink(t, n, "/etc/passwd")),
|
||||
],
|
||||
)
|
||||
|
||||
assert _extract_tor_bundle_safely(archive, install_dir) is False
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Issue #201 (tg12): Tor bundle integrity must come from at least one
|
||||
trusted source. Previously, if the upstream ``.sha256sum`` was
|
||||
unreachable, the bundle was extracted and executed anyway with only
|
||||
HTTPS-level transport trust.
|
||||
|
||||
The fix introduces a multi-source verification chain:
|
||||
|
||||
1. Upstream ``.sha256sum`` (current behavior)
|
||||
2. Baked-in digest list at ``backend/data/tor_bundle_digests.json``
|
||||
3. If neither source is reachable AT ALL: HTTPS-only fallback with a
|
||||
loud warning (avoids breaking first-run onboarding while the
|
||||
maintainer hasn't yet pinned a new Tor release)
|
||||
|
||||
A mismatch from a source that DID respond is always fatal — only the
|
||||
"no source reachable" case falls back to HTTPS-only.
|
||||
"""
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from services import tor_hidden_service as tor_svc
|
||||
from services.tor_hidden_service import (
|
||||
_DIGEST_PLACEHOLDER,
|
||||
_load_baked_in_digests,
|
||||
_verify_tor_bundle,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_bundle(tmp_path):
|
||||
"""A tiny synthetic 'bundle' so we can compute its digest deterministically."""
|
||||
archive = tmp_path / "fake-tor.tar.gz"
|
||||
payload = b"this is not really a tar archive"
|
||||
archive.write_bytes(payload)
|
||||
expected = hashlib.sha256(payload).hexdigest().lower()
|
||||
return archive, expected
|
||||
|
||||
|
||||
def test_baked_in_digests_skips_placeholders(tmp_path, monkeypatch):
|
||||
"""Entries with the placeholder value are filtered out."""
|
||||
digest_file = tmp_path / "digests.json"
|
||||
digest_file.write_text(
|
||||
'{"https://example.com/a.tar.gz": "PLACEHOLDER_REPLACE_BEFORE_RELEASE", '
|
||||
'"https://example.com/b.tar.gz": "deadbeef"}',
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(tor_svc, "_TOR_DIGEST_FILE", digest_file)
|
||||
|
||||
digests = _load_baked_in_digests()
|
||||
assert "https://example.com/a.tar.gz" not in digests
|
||||
assert digests.get("https://example.com/b.tar.gz") == "deadbeef"
|
||||
|
||||
|
||||
def test_verification_succeeds_when_upstream_matches(fake_bundle, monkeypatch):
|
||||
"""Path A: upstream .sha256sum returns the matching digest."""
|
||||
archive, expected = fake_bundle
|
||||
|
||||
def fake_urlretrieve(url, dest):
|
||||
dest_path = Path(dest)
|
||||
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest_path.write_text(f"{expected} bundle.tar.gz\n", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(tor_svc, "urlretrieve", fake_urlretrieve)
|
||||
monkeypatch.setattr(tor_svc, "_load_baked_in_digests", lambda: {})
|
||||
|
||||
verified, reason = _verify_tor_bundle(archive, "https://example.com/bundle.tar.gz")
|
||||
assert verified is True
|
||||
assert "upstream" in reason
|
||||
|
||||
|
||||
def test_verification_succeeds_via_baked_in_when_upstream_unreachable(fake_bundle, monkeypatch):
|
||||
"""Path B: upstream .sha256sum fails; baked-in digest matches."""
|
||||
archive, expected = fake_bundle
|
||||
|
||||
def fake_urlretrieve(url, dest):
|
||||
raise RuntimeError("upstream unreachable")
|
||||
|
||||
monkeypatch.setattr(tor_svc, "urlretrieve", fake_urlretrieve)
|
||||
monkeypatch.setattr(
|
||||
tor_svc, "_load_baked_in_digests",
|
||||
lambda: {"https://example.com/bundle.tar.gz": expected},
|
||||
)
|
||||
|
||||
verified, reason = _verify_tor_bundle(archive, "https://example.com/bundle.tar.gz")
|
||||
assert verified is True
|
||||
assert "baked-in" in reason
|
||||
|
||||
|
||||
def test_verification_fails_when_upstream_disagrees(fake_bundle, monkeypatch):
|
||||
"""Mismatch from a source that DID respond is always fatal."""
|
||||
archive, _expected = fake_bundle
|
||||
|
||||
def fake_urlretrieve(url, dest):
|
||||
dest_path = Path(dest)
|
||||
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest_path.write_text("0" * 64 + " bundle.tar.gz\n", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(tor_svc, "urlretrieve", fake_urlretrieve)
|
||||
monkeypatch.setattr(tor_svc, "_load_baked_in_digests", lambda: {})
|
||||
|
||||
verified, reason = _verify_tor_bundle(archive, "https://example.com/bundle.tar.gz")
|
||||
assert verified is False
|
||||
assert "mismatch" in reason.lower()
|
||||
|
||||
|
||||
def test_verification_fails_when_baked_in_disagrees(fake_bundle, monkeypatch):
|
||||
"""Even with no upstream, a baked-in mismatch is fatal."""
|
||||
archive, _expected = fake_bundle
|
||||
|
||||
def fake_urlretrieve(url, dest):
|
||||
raise RuntimeError("upstream unreachable")
|
||||
|
||||
monkeypatch.setattr(tor_svc, "urlretrieve", fake_urlretrieve)
|
||||
monkeypatch.setattr(
|
||||
tor_svc, "_load_baked_in_digests",
|
||||
lambda: {"https://example.com/bundle.tar.gz": "0" * 64},
|
||||
)
|
||||
|
||||
verified, reason = _verify_tor_bundle(archive, "https://example.com/bundle.tar.gz")
|
||||
assert verified is False
|
||||
|
||||
|
||||
def test_verification_falls_back_to_https_when_no_source_reachable(fake_bundle, monkeypatch, caplog):
|
||||
"""No source available → HTTPS-only fallback with a loud warning.
|
||||
|
||||
This preserves first-run onboarding while the maintainer hasn't
|
||||
yet pinned a particular Tor release in the digest file.
|
||||
"""
|
||||
archive, _expected = fake_bundle
|
||||
|
||||
def fake_urlretrieve(url, dest):
|
||||
raise RuntimeError("upstream unreachable")
|
||||
|
||||
monkeypatch.setattr(tor_svc, "urlretrieve", fake_urlretrieve)
|
||||
monkeypatch.setattr(tor_svc, "_load_baked_in_digests", lambda: {})
|
||||
|
||||
import logging
|
||||
with caplog.at_level(logging.WARNING):
|
||||
verified, reason = _verify_tor_bundle(archive, "https://example.com/bundle.tar.gz")
|
||||
assert verified is True
|
||||
assert "https-only" in reason.lower()
|
||||
assert any(
|
||||
"fell back to HTTPS-only" in record.getMessage() for record in caplog.records
|
||||
)
|
||||
@@ -0,0 +1,338 @@
|
||||
"""Issue #231 — self-update SHA-256 verification.
|
||||
|
||||
Before this fix, ``_validate_zip_hash`` returned silently whenever the
|
||||
``MESH_UPDATE_SHA256`` env var was unset (the default — nothing in the
|
||||
install docs ever told operators to set it). That made the auto-updater
|
||||
a supply-chain RCE on any compromise of the GitHub release pipeline.
|
||||
|
||||
The fix introduces a four-source verification chain:
|
||||
|
||||
1. ``MESH_UPDATE_SHA256`` env var (operator override, preserved)
|
||||
2. ``SHA256SUMS.txt`` asset published alongside the release (primary)
|
||||
3. Baked-in ``backend/data/release_digests.json`` (fallback)
|
||||
4. HTTPS-only fallback with a loud warning (preserves auto-update during
|
||||
transient outages so the user isn't stuck)
|
||||
|
||||
A mismatch from any source that DID respond is fatal. Only the "no
|
||||
source reachable at all" case falls back to HTTPS-only.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from services import updater
|
||||
from services.updater import (
|
||||
_compute_sha256,
|
||||
_fetch_sha256sums,
|
||||
_load_baked_in_release_digests,
|
||||
_validate_zip_hash,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_archive(tmp_path):
|
||||
"""A tiny synthetic zip-shaped file so we can compute a known digest."""
|
||||
archive = tmp_path / "update.zip"
|
||||
payload = b"this is not really a release archive"
|
||||
archive.write_bytes(payload)
|
||||
expected = hashlib.sha256(payload).hexdigest().lower()
|
||||
return str(archive), expected
|
||||
|
||||
|
||||
def test_baked_in_release_digests_file_loads():
|
||||
"""The shipped release_digests.json must parse and contain v0.9.79."""
|
||||
digests = _load_baked_in_release_digests()
|
||||
assert "v0.9.79" in digests
|
||||
entry = digests["v0.9.79"]
|
||||
assert "ShadowBroker_v0.9.79.zip" in entry
|
||||
digest = entry["ShadowBroker_v0.9.79.zip"]
|
||||
assert len(digest) == 64
|
||||
assert all(c in "0123456789abcdef" for c in digest)
|
||||
|
||||
|
||||
def test_baked_in_skips_comment_keys():
|
||||
"""The _comment top-level key is ignored, not surfaced as a release."""
|
||||
digests = _load_baked_in_release_digests()
|
||||
assert "_comment" not in digests
|
||||
|
||||
|
||||
def test_compute_sha256_matches_known_value(fake_archive):
|
||||
archive, expected = fake_archive
|
||||
assert _compute_sha256(archive) == expected
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Source 1: MESH_UPDATE_SHA256 env override
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_env_override_matching_passes(fake_archive, monkeypatch):
|
||||
"""Path 1: operator pinned the exact digest via env. Match = success."""
|
||||
archive, expected = fake_archive
|
||||
monkeypatch.setenv("MESH_UPDATE_SHA256", expected)
|
||||
|
||||
note = _validate_zip_hash(archive)
|
||||
assert "MESH_UPDATE_SHA256" in note
|
||||
|
||||
|
||||
def test_env_override_mismatch_fails_loudly(fake_archive, monkeypatch):
|
||||
"""Path 1: operator pinned a different digest. Mismatch = fatal."""
|
||||
archive, _expected = fake_archive
|
||||
monkeypatch.setenv("MESH_UPDATE_SHA256", "0" * 64)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
_validate_zip_hash(archive)
|
||||
assert "mismatch" in str(exc_info.value).lower()
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Source 2: SHA256SUMS.txt asset
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_sha256sums_matching_passes(fake_archive, monkeypatch):
|
||||
"""Path 2: SHA256SUMS.txt has the correct digest for our asset."""
|
||||
archive, expected = fake_archive
|
||||
monkeypatch.delenv("MESH_UPDATE_SHA256", raising=False)
|
||||
|
||||
def fake_sums(url):
|
||||
return {"ShadowBroker_v9.9.9.zip": expected}
|
||||
|
||||
monkeypatch.setattr(updater, "_fetch_sha256sums", fake_sums)
|
||||
note = _validate_zip_hash(
|
||||
archive,
|
||||
asset_name="ShadowBroker_v9.9.9.zip",
|
||||
sha256sums_url="https://example.test/SHA256SUMS.txt",
|
||||
release_tag="v9.9.9",
|
||||
)
|
||||
assert "SHA256SUMS.txt" in note
|
||||
|
||||
|
||||
def test_sha256sums_mismatch_fails_loudly(fake_archive, monkeypatch):
|
||||
"""Path 2: SHA256SUMS.txt has a different digest. Refuse."""
|
||||
archive, _expected = fake_archive
|
||||
monkeypatch.delenv("MESH_UPDATE_SHA256", raising=False)
|
||||
|
||||
def fake_sums(url):
|
||||
return {"ShadowBroker_v9.9.9.zip": "0" * 64}
|
||||
|
||||
monkeypatch.setattr(updater, "_fetch_sha256sums", fake_sums)
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
_validate_zip_hash(
|
||||
archive,
|
||||
asset_name="ShadowBroker_v9.9.9.zip",
|
||||
sha256sums_url="https://example.test/SHA256SUMS.txt",
|
||||
release_tag="v9.9.9",
|
||||
)
|
||||
assert "mismatch" in str(exc_info.value).lower()
|
||||
assert "SHA256SUMS" in str(exc_info.value)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Source 3: baked-in digest list
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_baked_in_matching_passes(fake_archive, monkeypatch):
|
||||
"""Path 3: SHA256SUMS unreachable, but the baked-in list has us."""
|
||||
archive, expected = fake_archive
|
||||
monkeypatch.delenv("MESH_UPDATE_SHA256", raising=False)
|
||||
monkeypatch.setattr(updater, "_fetch_sha256sums", lambda url: {})
|
||||
monkeypatch.setattr(
|
||||
updater,
|
||||
"_load_baked_in_release_digests",
|
||||
lambda: {"v9.9.9": {"ShadowBroker_v9.9.9.zip": expected}},
|
||||
)
|
||||
|
||||
note = _validate_zip_hash(
|
||||
archive,
|
||||
asset_name="ShadowBroker_v9.9.9.zip",
|
||||
sha256sums_url="https://example.test/SHA256SUMS.txt",
|
||||
release_tag="v9.9.9",
|
||||
)
|
||||
assert "baked-in" in note
|
||||
|
||||
|
||||
def test_baked_in_mismatch_fails_loudly(fake_archive, monkeypatch):
|
||||
"""Path 3: baked-in says something different. Refuse."""
|
||||
archive, _expected = fake_archive
|
||||
monkeypatch.delenv("MESH_UPDATE_SHA256", raising=False)
|
||||
monkeypatch.setattr(updater, "_fetch_sha256sums", lambda url: {})
|
||||
monkeypatch.setattr(
|
||||
updater,
|
||||
"_load_baked_in_release_digests",
|
||||
lambda: {"v9.9.9": {"ShadowBroker_v9.9.9.zip": "0" * 64}},
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
_validate_zip_hash(
|
||||
archive,
|
||||
asset_name="ShadowBroker_v9.9.9.zip",
|
||||
sha256sums_url="",
|
||||
release_tag="v9.9.9",
|
||||
)
|
||||
assert "mismatch" in str(exc_info.value).lower()
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Source 4: HTTPS-only fallback
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_https_only_fallback_when_no_source_available(fake_archive, monkeypatch, caplog):
|
||||
"""Path 4: nothing matches — fall back to HTTPS-only with loud warning.
|
||||
|
||||
This preserves the auto-update flow during transient outages: an
|
||||
operator on a flaky network during update doesn't get a hostile
|
||||
error, they get a degraded-but-functional update with a clear log
|
||||
message.
|
||||
"""
|
||||
import logging
|
||||
|
||||
archive, _expected = fake_archive
|
||||
monkeypatch.delenv("MESH_UPDATE_SHA256", raising=False)
|
||||
monkeypatch.setattr(updater, "_fetch_sha256sums", lambda url: {})
|
||||
monkeypatch.setattr(updater, "_load_baked_in_release_digests", lambda: {})
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
note = _validate_zip_hash(
|
||||
archive,
|
||||
asset_name="ShadowBroker_v99.99.zip",
|
||||
sha256sums_url="",
|
||||
release_tag="v99.99",
|
||||
)
|
||||
|
||||
assert "https-only" in note.lower()
|
||||
assert any(
|
||||
"fell back to HTTPS-only" in rec.getMessage() for rec in caplog.records
|
||||
)
|
||||
|
||||
|
||||
def test_https_only_fallback_when_release_tag_unknown(fake_archive, monkeypatch):
|
||||
"""Path 4 also kicks in when we have a baked-in list but it doesn't
|
||||
contain THIS release tag — e.g. a brand-new release that the local
|
||||
install hasn't seen a digest for yet."""
|
||||
archive, _expected = fake_archive
|
||||
monkeypatch.delenv("MESH_UPDATE_SHA256", raising=False)
|
||||
monkeypatch.setattr(updater, "_fetch_sha256sums", lambda url: {})
|
||||
monkeypatch.setattr(
|
||||
updater,
|
||||
"_load_baked_in_release_digests",
|
||||
lambda: {"v0.0.1": {"old.zip": "0" * 64}}, # different tag, doesn't match
|
||||
)
|
||||
|
||||
note = _validate_zip_hash(
|
||||
archive,
|
||||
asset_name="ShadowBroker_v99.99.zip",
|
||||
sha256sums_url="",
|
||||
release_tag="v99.99",
|
||||
)
|
||||
assert "https-only" in note.lower()
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Precedence (env > SHA256SUMS > baked-in > https-only)
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_env_override_beats_all_other_sources(fake_archive, monkeypatch):
|
||||
"""When MESH_UPDATE_SHA256 is set, it's the only source consulted.
|
||||
|
||||
The other sources may return false positives or negatives — they
|
||||
shouldn't be queried at all when the operator pinned an exact value.
|
||||
"""
|
||||
archive, expected = fake_archive
|
||||
monkeypatch.setenv("MESH_UPDATE_SHA256", expected)
|
||||
|
||||
def boom_sums(url):
|
||||
raise AssertionError("SHA256SUMS source was queried despite env override")
|
||||
|
||||
def boom_baked():
|
||||
raise AssertionError("Baked-in list was queried despite env override")
|
||||
|
||||
monkeypatch.setattr(updater, "_fetch_sha256sums", boom_sums)
|
||||
monkeypatch.setattr(updater, "_load_baked_in_release_digests", boom_baked)
|
||||
|
||||
note = _validate_zip_hash(
|
||||
archive,
|
||||
asset_name="any.zip",
|
||||
sha256sums_url="https://example.test/SHA256SUMS.txt",
|
||||
release_tag="any",
|
||||
)
|
||||
assert "MESH_UPDATE_SHA256" in note
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# _fetch_sha256sums parser
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_fetch_sha256sums_parses_standard_format(monkeypatch):
|
||||
"""Standard ``sha256sum`` output: ``<digest> <filename>``."""
|
||||
class _Resp:
|
||||
text = (
|
||||
"f6877c1d66614525315ea82636ce9f7b41178332c4dbf90d27431a1ea1d9cd47 ShadowBroker_v0.9.79.zip\n"
|
||||
"e0713c3cdda184cfbea750bfac0d62a35678fec00847e6476f2cac8e7e42046e ShadowBroker_0.9.79_x64_en-US.msi\n"
|
||||
)
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def fake_get(url, timeout=15):
|
||||
return _Resp()
|
||||
|
||||
monkeypatch.setattr(updater.requests, "get", fake_get)
|
||||
monkeypatch.setattr(updater, "_validate_update_url", lambda url, **kw: url)
|
||||
sums = _fetch_sha256sums("https://example.test/SHA256SUMS.txt")
|
||||
assert sums["ShadowBroker_v0.9.79.zip"].startswith("f6877c1d")
|
||||
assert sums["ShadowBroker_0.9.79_x64_en-US.msi"].startswith("e0713c3c")
|
||||
|
||||
|
||||
def test_fetch_sha256sums_handles_binary_marker(monkeypatch):
|
||||
"""sha256sum -b output: ``<digest> *<filename>``."""
|
||||
class _Resp:
|
||||
text = "f6877c1d66614525315ea82636ce9f7b41178332c4dbf90d27431a1ea1d9cd47 *ShadowBroker_v0.9.79.zip\n"
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(updater.requests, "get", lambda url, timeout=15: _Resp())
|
||||
monkeypatch.setattr(updater, "_validate_update_url", lambda url, **kw: url)
|
||||
sums = _fetch_sha256sums("https://example.test/SHA256SUMS.txt")
|
||||
assert "ShadowBroker_v0.9.79.zip" in sums
|
||||
|
||||
|
||||
def test_fetch_sha256sums_skips_malformed_lines(monkeypatch):
|
||||
"""Lines that don't parse cleanly are ignored, not aborted on."""
|
||||
class _Resp:
|
||||
text = (
|
||||
"# comment line\n"
|
||||
"\n"
|
||||
"not-a-digest bogus.txt\n"
|
||||
"f6877c1d66614525315ea82636ce9f7b41178332c4dbf90d27431a1ea1d9cd47 good.zip\n"
|
||||
)
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(updater.requests, "get", lambda url, timeout=15: _Resp())
|
||||
monkeypatch.setattr(updater, "_validate_update_url", lambda url, **kw: url)
|
||||
sums = _fetch_sha256sums("https://example.test/SHA256SUMS.txt")
|
||||
assert "good.zip" in sums
|
||||
assert "bogus.txt" not in sums
|
||||
|
||||
|
||||
def test_fetch_sha256sums_handles_network_failure(monkeypatch):
|
||||
"""If the SHA256SUMS asset can't be fetched, return empty (caller
|
||||
falls through to baked-in / https-only)."""
|
||||
import requests as _req
|
||||
|
||||
def fake_get(url, timeout=15):
|
||||
raise _req.exceptions.ConnectionError("upstream down")
|
||||
|
||||
monkeypatch.setattr(updater.requests, "get", fake_get)
|
||||
monkeypatch.setattr(updater, "_validate_update_url", lambda url, **kw: url)
|
||||
sums = _fetch_sha256sums("https://example.test/SHA256SUMS.txt")
|
||||
assert sums == {}
|
||||
@@ -0,0 +1,18 @@
|
||||
# Compose override that points the backend and frontend at the GitLab
|
||||
# Container Registry instead of GHCR. Use this if you prefer pulling
|
||||
# images from gitlab.com.
|
||||
#
|
||||
# Usage:
|
||||
# docker compose -f docker-compose.yml -f docker-compose.gitlab.yml pull
|
||||
# docker compose -f docker-compose.yml -f docker-compose.gitlab.yml up -d
|
||||
#
|
||||
# Both registries publish the same images on every push to main:
|
||||
# - .github/workflows/docker-publish.yml → ghcr.io (default)
|
||||
# - .gitlab-ci.yml → registry.gitlab.com (this file)
|
||||
|
||||
services:
|
||||
backend:
|
||||
image: registry.gitlab.com/bigbodycobain/shadowbroker/backend:latest
|
||||
|
||||
frontend:
|
||||
image: registry.gitlab.com/bigbodycobain/shadowbroker/frontend:latest
|
||||
+17
-1
@@ -28,6 +28,15 @@ services:
|
||||
- MESH_RELAY_PEERS=${MESH_RELAY_PEERS:-}
|
||||
# Shared transport auth for operator peer push. Must be set to a unique secret per deployment.
|
||||
- MESH_PEER_PUSH_SECRET=${MESH_PEER_PUSH_SECRET:-}
|
||||
# Issue #256: optional per-peer HMAC secrets. Comma-separated
|
||||
# `url=secret` pairs (no spaces). When a peer URL appears here, only
|
||||
# the listed per-peer secret is accepted for it — the global
|
||||
# MESH_PEER_PUSH_SECRET above is ignored for that specific URL. This
|
||||
# closes the cross-peer impersonation surface for multi-peer fleets.
|
||||
# Single-peer installs leave this empty (default) for unchanged
|
||||
# behavior. Both sides of a peering must agree on the per-peer
|
||||
# secret for a given URL.
|
||||
- MESH_PEER_SECRETS=${MESH_PEER_SECRETS:-}
|
||||
# Meshtastic MQTT is opt-in to avoid passive load on the public broker.
|
||||
# Set MESH_MQTT_ENABLED=true in .env only when this node should join live MQTT.
|
||||
- MESH_MQTT_ENABLED=${MESH_MQTT_ENABLED:-false}
|
||||
@@ -43,6 +52,11 @@ services:
|
||||
# The bundled Docker UI talks to the backend across Docker's private bridge.
|
||||
# Treat that bridge as local operator access while ports remain bound to 127.0.0.1 by default.
|
||||
- SHADOWBROKER_TRUST_DOCKER_BRIDGE_LOCAL_OPERATOR=${SHADOWBROKER_TRUST_DOCKER_BRIDGE_LOCAL_OPERATOR:-1}
|
||||
# Issue #250: bridge trust is now bound to specific container hostnames
|
||||
# (default: 'frontend' compose service + 'shadowbroker-frontend' container
|
||||
# name). If you rename the frontend service or run with a different
|
||||
# container_name, list the hostnames here (comma-separated, no spaces).
|
||||
- SHADOWBROKER_TRUSTED_FRONTEND_HOSTS=${SHADOWBROKER_TRUSTED_FRONTEND_HOSTS:-frontend,shadowbroker-frontend}
|
||||
volumes:
|
||||
- backend_data:/app/data
|
||||
restart: unless-stopped
|
||||
@@ -62,11 +76,13 @@ services:
|
||||
image: ghcr.io/bigbodycobain/shadowbroker-frontend:latest
|
||||
container_name: shadowbroker-frontend
|
||||
ports:
|
||||
- "${BIND:-127.0.0.1}:3000:3000"
|
||||
- "${BIND:-127.0.0.1}:${FRONTEND_PORT:-3000}:3000"
|
||||
environment:
|
||||
# Points the Next.js server-side proxy at the backend container via Docker networking.
|
||||
# Change this if your backend runs on a different host or port.
|
||||
- BACKEND_URL=http://backend:8000
|
||||
# Lets the server-side proxy authenticate protected local-node API calls.
|
||||
- ADMIN_KEY=${ADMIN_KEY:-}
|
||||
depends_on:
|
||||
backend:
|
||||
condition: service_healthy
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
import React from 'react';
|
||||
import { act, cleanup, fireEvent, render, screen } from '@testing-library/react';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import AlertToast from '@/components/AlertToast';
|
||||
import type { ToastItem } from '@/hooks/useAlertToasts';
|
||||
|
||||
function buildToast(partial: Partial<ToastItem> = {}): ToastItem {
|
||||
return {
|
||||
id: 'toast-1',
|
||||
title: 'Embassy evacuation reported',
|
||||
source: 'Reuters',
|
||||
risk_score: 9,
|
||||
lat: 38.9,
|
||||
lng: -77.0,
|
||||
timestamp: Date.now(),
|
||||
...partial,
|
||||
};
|
||||
}
|
||||
|
||||
describe('AlertToast', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('renders the toast title, source, and severity label', () => {
|
||||
const toast = buildToast();
|
||||
render(
|
||||
<AlertToast toasts={[toast]} onDismiss={vi.fn()} />,
|
||||
);
|
||||
|
||||
expect(screen.getByText(toast.title)).toBeTruthy();
|
||||
expect(screen.getByText(toast.source)).toBeTruthy();
|
||||
// 9/10 -> CRITICAL
|
||||
expect(screen.getByText(/CRITICAL/)).toBeTruthy();
|
||||
expect(screen.getByText(/LVL 9\/10/)).toBeTruthy();
|
||||
});
|
||||
|
||||
it('auto-dismisses after 5 seconds', () => {
|
||||
const onDismiss = vi.fn();
|
||||
const toast = buildToast();
|
||||
render(
|
||||
<AlertToast toasts={[toast]} onDismiss={onDismiss} />,
|
||||
);
|
||||
|
||||
expect(onDismiss).not.toHaveBeenCalled();
|
||||
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(5000);
|
||||
});
|
||||
|
||||
expect(onDismiss).toHaveBeenCalledWith(toast.id);
|
||||
});
|
||||
|
||||
it('pauses auto-dismiss while the card is hovered', () => {
|
||||
const onDismiss = vi.fn();
|
||||
const toast = buildToast();
|
||||
render(
|
||||
<AlertToast toasts={[toast]} onDismiss={onDismiss} />,
|
||||
);
|
||||
|
||||
// Hover before the timer fires. mouseEnter must be flushed
|
||||
// (state update + effect cleanup) in its own act() before we
|
||||
// advance timers — otherwise the original mount-time timer is
|
||||
// still active when advanceTimersByTime runs.
|
||||
const card = screen.getByText(toast.title).closest('[class*="cursor-pointer"]')!;
|
||||
expect(card).toBeTruthy();
|
||||
|
||||
act(() => {
|
||||
fireEvent.mouseEnter(card);
|
||||
});
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(10_000);
|
||||
});
|
||||
|
||||
// Still no dismiss — timer is paused.
|
||||
expect(onDismiss).not.toHaveBeenCalled();
|
||||
|
||||
// Leave: a fresh full-lifetime timer starts.
|
||||
act(() => {
|
||||
fireEvent.mouseLeave(card);
|
||||
});
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(4_999);
|
||||
});
|
||||
expect(onDismiss).not.toHaveBeenCalled();
|
||||
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(1);
|
||||
});
|
||||
expect(onDismiss).toHaveBeenCalledWith(toast.id);
|
||||
});
|
||||
|
||||
it('dismisses on × button click without calling onFlyTo', () => {
|
||||
const onDismiss = vi.fn();
|
||||
const onFlyTo = vi.fn();
|
||||
const toast = buildToast();
|
||||
render(
|
||||
<AlertToast toasts={[toast]} onDismiss={onDismiss} onFlyTo={onFlyTo} />,
|
||||
);
|
||||
|
||||
fireEvent.click(screen.getByText('×'));
|
||||
|
||||
expect(onDismiss).toHaveBeenCalledWith(toast.id);
|
||||
expect(onFlyTo).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('flies to the toast location and dismisses on body click', () => {
|
||||
const onDismiss = vi.fn();
|
||||
const onFlyTo = vi.fn();
|
||||
const toast = buildToast();
|
||||
render(
|
||||
<AlertToast toasts={[toast]} onDismiss={onDismiss} onFlyTo={onFlyTo} />,
|
||||
);
|
||||
|
||||
fireEvent.click(screen.getByText(toast.title));
|
||||
|
||||
expect(onFlyTo).toHaveBeenCalledWith(toast.lat, toast.lng);
|
||||
expect(onDismiss).toHaveBeenCalledWith(toast.id);
|
||||
});
|
||||
});
|
||||
@@ -45,12 +45,12 @@ describe('admin/session boundary hardening', () => {
|
||||
});
|
||||
|
||||
it('accepts a verified admin key and reports the minted session as present', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ ok: true }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
// Issue #255 fix: the route no longer round-trips to the backend
|
||||
// to "verify" the key (the previous implementation called a public
|
||||
// endpoint that always returned 200, so any key was accepted when
|
||||
// ADMIN_KEY was unset). Local string comparison is the only
|
||||
// validation, so we don't mock fetch and don't assert it was called.
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost/api/admin/session', {
|
||||
@@ -65,7 +65,8 @@ describe('admin/session boundary hardening', () => {
|
||||
expect(res.status).toBe(200);
|
||||
expect(cookie).toContain('sb_admin_session=');
|
||||
expect(res.headers.get('cache-control')).toContain('no-store');
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
// Validation is local-only — no backend round-trip should happen.
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
|
||||
const getReq = new NextRequest('http://localhost/api/admin/session', {
|
||||
method: 'GET',
|
||||
@@ -88,12 +89,8 @@ describe('admin/session boundary hardening', () => {
|
||||
});
|
||||
|
||||
it('invalidates the previous admin session token when a new one is minted', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ ok: true }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
// Issue #255 fix: no backend round-trip. Validation is local-only.
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const firstReq = new NextRequest('http://localhost/api/admin/session', {
|
||||
@@ -135,21 +132,25 @@ describe('admin/session boundary hardening', () => {
|
||||
);
|
||||
const newBody = await newSessionCheck.json();
|
||||
expect(newBody.hasSession).toBe(true);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
// Local validation only — backend should not be called during minting.
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects session minting when frontend admin key is set but backend has no configured admin key', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ detail: 'Forbidden — admin key not configured' }), {
|
||||
status: 403,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
it('refuses session minting when frontend ADMIN_KEY env var is unset (#255)', async () => {
|
||||
// Issue #255 (tg12): previously, when ADMIN_KEY was unset the route
|
||||
// fell through to a public backend endpoint that always returned
|
||||
// 200, so any user-supplied key minted a full admin session. The
|
||||
// fix is to refuse minting entirely when ADMIN_KEY is unconfigured
|
||||
// and surface a clear message pointing the operator at the
|
||||
// backend's auto-trust-loopback behavior.
|
||||
process.env.ADMIN_KEY = '';
|
||||
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost/api/admin/session', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ adminKey: 'top-secret' }),
|
||||
body: JSON.stringify({ adminKey: 'any-key-an-attacker-supplies' }),
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
|
||||
@@ -158,8 +159,11 @@ describe('admin/session boundary hardening', () => {
|
||||
|
||||
expect(res.status).toBe(403);
|
||||
expect(body.ok).toBe(false);
|
||||
expect(body.detail).toBe('Forbidden — admin key not configured');
|
||||
expect(String(body.detail)).toMatch(/no admin key configured/i);
|
||||
expect(res.headers.get('set-cookie')).toBeNull();
|
||||
// Crucially: no backend round-trip happens. The previous broken
|
||||
// verifyAgainstBackend() call must NOT be re-introduced.
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not forward raw x-admin-key headers through the sensitive proxy path', async () => {
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
import React from 'react';
|
||||
import { act, cleanup, render, screen } from '@testing-library/react';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { I18nProvider, LOCALES, useTranslation, type Locale } from '@/i18n';
|
||||
|
||||
/**
|
||||
* Renders a tiny consumer so we can drive the I18nContext from tests.
|
||||
*/
|
||||
function Probe({ keyToRender }: { keyToRender: string }) {
|
||||
const { locale, setLocale, t } = useTranslation();
|
||||
return (
|
||||
<div>
|
||||
<span data-testid="locale">{locale}</span>
|
||||
<span data-testid="translated">{t(keyToRender)}</span>
|
||||
<button onClick={() => setLocale('zh-CN')} data-testid="to-zh">go zh</button>
|
||||
<button onClick={() => setLocale('en')} data-testid="to-en">go en</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
describe('I18nProvider', () => {
|
||||
beforeEach(() => {
|
||||
localStorage.clear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
localStorage.clear();
|
||||
});
|
||||
|
||||
it('exposes a non-empty LOCALES registry with en and zh-CN', () => {
|
||||
const codes = LOCALES.map((l) => l.code);
|
||||
expect(codes).toContain('en');
|
||||
expect(codes).toContain('zh-CN');
|
||||
// Native labels — used by the language picker. These must be set
|
||||
// so the picker shows the native language name regardless of
|
||||
// current UI locale.
|
||||
for (const entry of LOCALES) {
|
||||
expect(entry.label.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('defaults to English when no localStorage and English browser', () => {
|
||||
Object.defineProperty(navigator, 'language', { value: 'en-US', configurable: true });
|
||||
render(
|
||||
<I18nProvider>
|
||||
<Probe keyToRender="settings.title" />
|
||||
</I18nProvider>,
|
||||
);
|
||||
expect(screen.getByTestId('locale').textContent).toBe('en');
|
||||
});
|
||||
|
||||
it('auto-detects zh-CN when browser language starts with "zh"', () => {
|
||||
Object.defineProperty(navigator, 'language', { value: 'zh-TW', configurable: true });
|
||||
render(
|
||||
<I18nProvider>
|
||||
<Probe keyToRender="settings.title" />
|
||||
</I18nProvider>,
|
||||
);
|
||||
// "zh-TW" should match the zh prefix and resolve to our zh-CN bundle
|
||||
// (we ship only one Chinese variant for now).
|
||||
expect(screen.getByTestId('locale').textContent).toBe('zh-CN');
|
||||
});
|
||||
|
||||
it('honors a previously saved localStorage choice over auto-detect', () => {
|
||||
Object.defineProperty(navigator, 'language', { value: 'zh-CN', configurable: true });
|
||||
localStorage.setItem('sb_locale', 'en');
|
||||
render(
|
||||
<I18nProvider>
|
||||
<Probe keyToRender="settings.title" />
|
||||
</I18nProvider>,
|
||||
);
|
||||
expect(screen.getByTestId('locale').textContent).toBe('en');
|
||||
});
|
||||
|
||||
it('persists setLocale to localStorage', () => {
|
||||
render(
|
||||
<I18nProvider>
|
||||
<Probe keyToRender="settings.title" />
|
||||
</I18nProvider>,
|
||||
);
|
||||
|
||||
act(() => {
|
||||
screen.getByTestId('to-zh').click();
|
||||
});
|
||||
|
||||
expect(screen.getByTestId('locale').textContent).toBe('zh-CN');
|
||||
expect(localStorage.getItem('sb_locale')).toBe('zh-CN');
|
||||
});
|
||||
|
||||
it('falls back to auto-detect when localStorage holds an unknown locale', () => {
|
||||
// Pre-poison localStorage with a value that isn't in LOCALES. The
|
||||
// isLocale guard at provider init should ignore it and fall through
|
||||
// to navigator.language detection.
|
||||
Object.defineProperty(navigator, 'language', { value: 'en-US', configurable: true });
|
||||
localStorage.setItem('sb_locale', 'klingon' as unknown as Locale);
|
||||
|
||||
render(
|
||||
<I18nProvider>
|
||||
<Probe keyToRender="settings.title" />
|
||||
</I18nProvider>,
|
||||
);
|
||||
|
||||
expect(screen.getByTestId('locale').textContent).toBe('en');
|
||||
});
|
||||
|
||||
it('renders a real translated string from the zh-CN bundle', () => {
|
||||
Object.defineProperty(navigator, 'language', { value: 'zh-CN', configurable: true });
|
||||
render(
|
||||
<I18nProvider>
|
||||
<Probe keyToRender="settings.title" />
|
||||
</I18nProvider>,
|
||||
);
|
||||
// The zh-CN bundle has settings.title = "设置". If this assertion
|
||||
// ever fails after a translation PR, it's a signal that the
|
||||
// translation surface was significantly altered.
|
||||
expect(screen.getByTestId('translated').textContent).toBe('设置');
|
||||
});
|
||||
|
||||
it('falls back to the key when a translation is missing', () => {
|
||||
render(
|
||||
<I18nProvider>
|
||||
<Probe keyToRender="this.key.intentionally.does.not.exist" />
|
||||
</I18nProvider>,
|
||||
);
|
||||
expect(screen.getByTestId('translated').textContent).toBe(
|
||||
'this.key.intentionally.does.not.exist',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -10,7 +10,7 @@ const mocks = vi.hoisted(() => ({
|
||||
buildMailboxClaims: vi.fn(async () => []),
|
||||
countDmMailboxes: vi.fn(async () => ({ ok: true, count: 0 })),
|
||||
ensureRegisteredDmKey: vi.fn(async () => ({ dhPubKey: 'local-dh', dhAlgo: 'X25519' })),
|
||||
fetchDmPublicKey: vi.fn(async () => ({ dh_pub_key: 'peer-dh', dh_algo: 'X25519' })),
|
||||
fetchDmPublicKey: vi.fn(async () => ({ agent_id: '!sb_peer', dh_pub_key: 'peer-dh', dh_algo: 'X25519' })),
|
||||
pollDmMailboxes: vi.fn(async () => ({ ok: true, messages: [] })),
|
||||
sendDmMessage: vi.fn(async () => ({ ok: true, transport: 'relay' })),
|
||||
sendOffLedgerConsentMessage: vi.fn(async () => ({ ok: true, transport: 'relay' })),
|
||||
@@ -252,7 +252,7 @@ describe('MessagesView first-contact trust UX', () => {
|
||||
mocks.pollDmMailboxes.mockResolvedValue({ ok: true, messages: [] });
|
||||
mocks.countDmMailboxes.mockResolvedValue({ ok: true, count: 0 });
|
||||
mocks.ensureRegisteredDmKey.mockResolvedValue({ dhPubKey: 'local-dh', dhAlgo: 'X25519' });
|
||||
mocks.fetchDmPublicKey.mockResolvedValue({ dh_pub_key: 'peer-dh', dh_algo: 'X25519' });
|
||||
mocks.fetchDmPublicKey.mockResolvedValue({ agent_id: '!sb_peer', dh_pub_key: 'peer-dh', dh_algo: 'X25519' });
|
||||
mocks.sendOffLedgerConsentMessage.mockResolvedValue({ ok: true, transport: 'relay' });
|
||||
mocks.canUseWormholeBootstrap.mockResolvedValue(false);
|
||||
mocks.exportWormholeDmInvite.mockResolvedValue({
|
||||
@@ -334,8 +334,9 @@ describe('MessagesView first-contact trust UX', () => {
|
||||
await openComposeForRecipient('!sb_invited', 'hello to pinned peer');
|
||||
|
||||
expect(screen.queryByText('Unverified First Contact')).not.toBeInTheDocument();
|
||||
expect(await screen.findByText('ROOT LOCAL QUORUM')).toBeInTheDocument();
|
||||
expect(await screen.findByText(/Local quorum root rootabcd\.\.123456/i)).toBeInTheDocument();
|
||||
expect(screen.queryByText('ROOT LOCAL QUORUM')).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(/Local quorum root rootabcd\.\.123456/i)).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(/Fingerprint/i)).not.toBeInTheDocument();
|
||||
expect(screen.getByRole('button', { name: 'Send Secure Mail' })).toBeEnabled();
|
||||
});
|
||||
|
||||
@@ -375,7 +376,34 @@ describe('MessagesView first-contact trust UX', () => {
|
||||
expect(screen.queryByText(/still warming up/i)).not.toBeInTheDocument();
|
||||
}, 10000);
|
||||
|
||||
it('does not flatten witness policy not met into a generic witnessed root label', async () => {
|
||||
it('repairs the local sending key before sending instead of surfacing backend key jargon', async () => {
|
||||
contactsState = {
|
||||
'!sb_pinned': {
|
||||
alias: 'Pinned Peer',
|
||||
blocked: false,
|
||||
trust_level: 'invite_pinned',
|
||||
dhPubKey: 'peer-dh',
|
||||
remotePrekeyFingerprint: 'abcdef123456',
|
||||
},
|
||||
};
|
||||
mocks.ensureRegisteredDmKey
|
||||
.mockResolvedValueOnce({ ok: true, dhPubKey: '', dhAlgo: 'X25519', detail: 'Missing DH public key' })
|
||||
.mockResolvedValueOnce({ ok: true, dhPubKey: 'local-dh-repaired', dhAlgo: 'X25519' });
|
||||
mocks.sendDmMessage.mockResolvedValueOnce({ ok: true, transport: 'relay' });
|
||||
|
||||
renderMessagesView();
|
||||
await openComposeForRecipient('!sb_pinned', 'hello after repair');
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Send Secure Mail' }));
|
||||
|
||||
await waitFor(() => expect(mocks.ensureRegisteredDmKey).toHaveBeenCalledTimes(2));
|
||||
await waitFor(() => expect(mocks.sendDmMessage).toHaveBeenCalled());
|
||||
expect(await screen.findByText(/Mail delivered to Pinned Peer/i)).toBeInTheDocument();
|
||||
expect(screen.queryByText(/Local DM key is unavailable/i)).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(/Missing DH public key/i)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows saved contacts without witness-policy implementation detail', async () => {
|
||||
contactsState = {
|
||||
'!sb_policy': {
|
||||
alias: 'Policy Peer',
|
||||
@@ -404,10 +432,39 @@ describe('MessagesView first-contact trust UX', () => {
|
||||
renderMessagesView();
|
||||
fireEvent.click(screen.getByRole('button', { name: 'CONTACTS' }));
|
||||
|
||||
expect(await screen.findByText(/Witness-policy root rootpoli\.\.123456/i)).toBeInTheDocument();
|
||||
expect(await screen.findByText('Saved Contact')).toBeInTheDocument();
|
||||
expect(screen.queryByText(/Witness-policy root rootpoli\.\.123456/i)).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(/Witnessed root rootpoli\.\.123456/i)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('hydrates Wormhole contacts on first load even when a local browser identity exists', async () => {
|
||||
let wormholeIdentityResolved = false;
|
||||
contactsState = {
|
||||
'!sb_saved': {
|
||||
alias: 'Saved Person',
|
||||
blocked: false,
|
||||
trust_level: 'invite_pinned',
|
||||
invitePinnedPrekeyLookupHandle: 'handle-saved',
|
||||
invitePinnedTrustFingerprint: 'savedfingerprint123456',
|
||||
},
|
||||
};
|
||||
mocks.isWormholeSecureRequired.mockResolvedValue(true);
|
||||
mocks.fetchWormholeIdentity.mockImplementation(async () => {
|
||||
wormholeIdentityResolved = true;
|
||||
return { node_id: '!sb_local', public_key: 'local-pub' };
|
||||
});
|
||||
mocks.hydrateWormholeContacts.mockImplementation(async () =>
|
||||
wormholeIdentityResolved ? contactsState : {},
|
||||
);
|
||||
|
||||
renderMessagesView();
|
||||
fireEvent.click(screen.getByRole('button', { name: 'CONTACTS' }));
|
||||
|
||||
expect(await screen.findByText('Saved Person')).toBeInTheDocument();
|
||||
expect(screen.queryByText(/No approved secure contacts yet/i)).not.toBeInTheDocument();
|
||||
expect(mocks.fetchWormholeIdentity).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('shows an import-invite shortcut for unpinned contacts in the contact list', async () => {
|
||||
contactsState = {
|
||||
'!sb_unpinned': {
|
||||
@@ -426,7 +483,7 @@ describe('MessagesView first-contact trust UX', () => {
|
||||
expect(screen.getByLabelText(/Local Alias/i)).toHaveValue('!sb_unpinned');
|
||||
});
|
||||
|
||||
it('surfaces pending contact requests in the contact list with approve and deny actions', async () => {
|
||||
it('surfaces pending contact requests in a top-level requests tab with approve and deny actions', async () => {
|
||||
localStorage.setItem(
|
||||
'sb_infonet_mailbox_v1:!sb_local',
|
||||
JSON.stringify({
|
||||
@@ -464,7 +521,7 @@ describe('MessagesView first-contact trust UX', () => {
|
||||
});
|
||||
|
||||
renderMessagesView();
|
||||
fireEvent.click(screen.getByRole('button', { name: 'CONTACTS' }));
|
||||
fireEvent.click(await screen.findByRole('button', { name: /REQUESTS/i }));
|
||||
|
||||
expect(await screen.findByText('Contact Requests')).toBeInTheDocument();
|
||||
expect(await screen.findByText('1 pending')).toBeInTheDocument();
|
||||
@@ -576,13 +633,13 @@ describe('MessagesView first-contact trust UX', () => {
|
||||
|
||||
expect(
|
||||
await screen.findByText(
|
||||
/Import or re-import a signed invite before sending a contact request; legacy direct lookup is disabled\./i,
|
||||
/This contact needs their full contact address once before messages can be sent/i,
|
||||
),
|
||||
).toBeInTheDocument();
|
||||
expect(mocks.fetchDmPublicKey).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('announces attested invite imports as INVITE PINNED', async () => {
|
||||
it('announces attested invite imports as a saved contact', async () => {
|
||||
mocks.importWormholeDmInvite.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
peer_id: '!sb_attested',
|
||||
@@ -595,32 +652,34 @@ describe('MessagesView first-contact trust UX', () => {
|
||||
fireEvent.click(screen.getByRole('button', { name: 'CONTACTS' }));
|
||||
expect(await screen.findByText("Paste Someone's Address")).toBeInTheDocument();
|
||||
|
||||
fireEvent.change(screen.getByPlaceholderText(/Paste the full text copied/i), {
|
||||
fireEvent.change(screen.getByPlaceholderText(/Paste a short address/i), {
|
||||
target: { value: JSON.stringify({ invite: { event_type: 'dm_invite', payload: {} } }) },
|
||||
});
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Import Address' }));
|
||||
|
||||
expect(
|
||||
await screen.findByText(/INVITE PINNED for !sb_attested \(invitefp\.\.tested\)\./i),
|
||||
).toBeInTheDocument();
|
||||
expect(await screen.findByText(/Contact saved: !sb_attested\./i)).toBeInTheDocument();
|
||||
expect(await screen.findByText('Saved Contact')).toBeInTheDocument();
|
||||
expect(screen.queryByText(/INVITE PINNED for/i)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('generates and copies the full signed public address instead of the lookup handle', async () => {
|
||||
it('automatically creates a share address and keeps copy actions simple', async () => {
|
||||
renderMessagesView();
|
||||
|
||||
fireEvent.click(await screen.findByRole('button', { name: 'Generate Address' }));
|
||||
|
||||
await waitFor(() => expect(mocks.writeClipboard).toHaveBeenCalled());
|
||||
const copied = String(mocks.writeClipboard.mock.calls[0][0] || '');
|
||||
expect(copied).toContain('"type": "shadowbroker.infonet.dm.invite"');
|
||||
expect(copied).toContain('"prekey_lookup_handle": "handle-123"');
|
||||
expect(copied).not.toBe('handle-123');
|
||||
expect(await screen.findByText(/Generated and copied/i)).toBeInTheDocument();
|
||||
expect(await screen.findByText(/Contact address ready/i)).toBeInTheDocument();
|
||||
expect(await screen.findByText('handle-123')).toBeInTheDocument();
|
||||
expect(screen.getByText(/Signed invite ready/i)).toBeInTheDocument();
|
||||
expect(screen.getByRole('button', { name: /Copy Short Address/i })).toBeInTheDocument();
|
||||
expect(screen.getByRole('button', { name: /Copy Full Address/i })).toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: /Copy Short Address/i }));
|
||||
|
||||
await waitFor(() => expect(mocks.writeClipboard).toHaveBeenCalledWith('handle-123'));
|
||||
const copied = String(mocks.writeClipboard.mock.calls.at(-1)?.[0] || '');
|
||||
expect(copied).toBe('handle-123');
|
||||
expect(screen.queryByText(/shadowbroker\.infonet\.dm\.invite/i)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('does not advertise legacy handle-only addresses as copyable public addresses', async () => {
|
||||
it('does not advertise legacy handle-only addresses as copyable contact addresses', async () => {
|
||||
localStorage.setItem(
|
||||
'sb_infonet_dm_addresses_v1:!sb_local',
|
||||
JSON.stringify({
|
||||
@@ -641,25 +700,33 @@ describe('MessagesView first-contact trust UX', () => {
|
||||
|
||||
renderMessagesView();
|
||||
|
||||
expect(await screen.findByText(/Generate an address, then send it to someone/i)).toBeInTheDocument();
|
||||
expect(await screen.findByText(/Contact address ready/i)).toBeInTheDocument();
|
||||
fireEvent.click(screen.getByRole('button', { name: 'CONTACTS' }));
|
||||
|
||||
expect(await screen.findByText('Legacy handle')).toBeInTheDocument();
|
||||
expect(screen.getByText('Address unavailable locally.')).toBeInTheDocument();
|
||||
expect(screen.getByRole('button', { name: 'Copy' })).toBeDisabled();
|
||||
expect(screen.getAllByRole('button', { name: 'Copy Short' }).some((button) => !button.hasAttribute('disabled'))).toBe(true);
|
||||
expect(screen.getAllByRole('button', { name: 'Copy Full' }).some((button) => button.hasAttribute('disabled'))).toBe(true);
|
||||
});
|
||||
|
||||
it('explains raw lookup handles instead of showing a JSON parser error', async () => {
|
||||
it('sends a contact request from a short address instead of requiring JSON', async () => {
|
||||
renderMessagesView();
|
||||
fireEvent.click(screen.getByRole('button', { name: 'CONTACTS' }));
|
||||
expect(await screen.findByText("Paste Someone's Address")).toBeInTheDocument();
|
||||
|
||||
fireEvent.change(screen.getByPlaceholderText(/Paste the full text copied/i), {
|
||||
fireEvent.change(screen.getByPlaceholderText(/Paste a short address/i), {
|
||||
target: { value: 'f0eee9e9ccf849bcb2d86c0d7a1e0669c75be4e05533b0f6c67' },
|
||||
});
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Send Request' }));
|
||||
|
||||
expect(await screen.findByText(/only a short address ID/i)).toBeInTheDocument();
|
||||
expect(screen.getByRole('button', { name: 'Import Address' })).toBeDisabled();
|
||||
await waitFor(() => expect(mocks.sendOffLedgerConsentMessage).toHaveBeenCalled());
|
||||
expect(await screen.findByText(/Contact request sent to/i)).toBeInTheDocument();
|
||||
expect(mocks.fetchDmPublicKey).toHaveBeenCalledWith(
|
||||
'http://localhost:8000',
|
||||
'',
|
||||
'f0eee9e9ccf849bcb2d86c0d7a1e0669c75be4e05533b0f6c67',
|
||||
);
|
||||
expect(mocks.sendOffLedgerConsentMessage).toHaveBeenCalled();
|
||||
expect(screen.queryByText(/Unexpected number in JSON/i)).not.toBeInTheDocument();
|
||||
expect(mocks.importWormholeDmInvite).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -675,7 +742,7 @@ describe('MessagesView first-contact trust UX', () => {
|
||||
fireEvent.click(screen.getByRole('button', { name: 'CONTACTS' }));
|
||||
expect(await screen.findByText("Paste Someone's Address")).toBeInTheDocument();
|
||||
|
||||
const addressField = screen.getByPlaceholderText(/Paste the full text copied/i);
|
||||
const addressField = screen.getByPlaceholderText(/Paste a short address/i);
|
||||
fireEvent.paste(addressField, {
|
||||
clipboardData: {
|
||||
getData: () => signedAddress,
|
||||
@@ -687,7 +754,7 @@ describe('MessagesView first-contact trust UX', () => {
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Advanced Details' }));
|
||||
|
||||
expect(screen.getByLabelText('Raw copied public address')).toHaveValue(signedAddress);
|
||||
expect(screen.getByLabelText('Raw copied contact address')).toHaveValue(signedAddress);
|
||||
});
|
||||
|
||||
it('imports a copied address without waiting for secure mail warm-up', async () => {
|
||||
@@ -710,17 +777,131 @@ describe('MessagesView first-contact trust UX', () => {
|
||||
fireEvent.click(screen.getByRole('button', { name: 'CONTACTS' }));
|
||||
expect(await screen.findByText("Paste Someone's Address")).toBeInTheDocument();
|
||||
|
||||
fireEvent.change(screen.getByPlaceholderText(/Paste the full text copied/i), {
|
||||
fireEvent.change(screen.getByPlaceholderText(/Paste a short address/i), {
|
||||
target: { value: JSON.stringify({ invite: { event_type: 'dm_invite', payload: {} } }) },
|
||||
});
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Import Address' }));
|
||||
|
||||
expect(await screen.findByText(/INVITE PINNED for !sb_now \(invitefp-now\)\./i)).toBeInTheDocument();
|
||||
expect(await screen.findByText(/Contact saved: !sb_now\./i)).toBeInTheDocument();
|
||||
expect(mocks.importWormholeDmInvite).toHaveBeenCalled();
|
||||
expect(screen.queryByText(/Secure mail is still warming up/i)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('announces compat invite imports as TOFU PINNED with backend detail', async () => {
|
||||
it('saves pending-delivery contacts without showing prekey jargon', async () => {
|
||||
mocks.importWormholeDmInvite.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
peer_id: '!sb_pending',
|
||||
trust_fingerprint: 'invitefp-pending',
|
||||
trust_level: 'invite_pinned',
|
||||
pending_prekey: true,
|
||||
detail: 'Contact saved.',
|
||||
contact: {
|
||||
alias: 'Pending Person',
|
||||
blocked: false,
|
||||
trust_level: 'invite_pinned',
|
||||
invitePinnedPrekeyLookupHandle: 'handle-pending',
|
||||
invitePinnedTrustFingerprint: 'invitefp-pending',
|
||||
dhPubKey: '',
|
||||
},
|
||||
});
|
||||
|
||||
renderMessagesView();
|
||||
fireEvent.click(screen.getByRole('button', { name: 'CONTACTS' }));
|
||||
expect(await screen.findByText("Paste Someone's Address")).toBeInTheDocument();
|
||||
|
||||
fireEvent.change(screen.getByPlaceholderText(/Paste a short address/i), {
|
||||
target: { value: JSON.stringify({ invite: { event_type: 'dm_invite', payload: {} } }) },
|
||||
});
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Import Address' }));
|
||||
|
||||
expect(await screen.findByText(/Contact saved: Pending Person\./i)).toBeInTheDocument();
|
||||
expect(await screen.findByText('Saved Contact')).toBeInTheDocument();
|
||||
expect(screen.queryByText(/prekey/i)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('saves mail locally when a saved contact is not reachable yet', async () => {
|
||||
contactsState = {
|
||||
'!sb_pending': {
|
||||
alias: 'Pending Person',
|
||||
blocked: false,
|
||||
trust_level: 'invite_pinned',
|
||||
invitePinnedPrekeyLookupHandle: 'handle-pending',
|
||||
invitePinnedTrustFingerprint: 'invitefp-pending',
|
||||
dhPubKey: '',
|
||||
},
|
||||
};
|
||||
mocks.fetchDmPublicKey.mockResolvedValueOnce({ agent_id: '!sb_pending', dh_pub_key: '', dh_algo: 'X25519' });
|
||||
|
||||
renderMessagesView();
|
||||
await openComposeForRecipient('!sb_pending', 'hello when ready');
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Send Secure Mail' }));
|
||||
|
||||
expect(await screen.findByText(/Mail is saved locally and will send automatically when the contact is reachable/i)).toBeInTheDocument();
|
||||
expect(mocks.sendOffLedgerConsentMessage).not.toHaveBeenCalled();
|
||||
expect(screen.queryByText(/delivery key has not reached/i)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('removes an approved contact immediately from the visible contact list', async () => {
|
||||
contactsState = {
|
||||
'!sb_remove': {
|
||||
alias: 'Remove Me',
|
||||
blocked: false,
|
||||
trust_level: 'invite_pinned',
|
||||
invitePinnedTrustFingerprint: 'removefingerprint123456',
|
||||
dhPubKey: 'peer-dh',
|
||||
},
|
||||
};
|
||||
mocks.removeContact.mockImplementation((peerId: string) => {
|
||||
delete contactsState[peerId];
|
||||
});
|
||||
|
||||
renderMessagesView();
|
||||
fireEvent.click(screen.getByRole('button', { name: 'CONTACTS' }));
|
||||
|
||||
expect(
|
||||
await screen.findByText('Remove Me', undefined, { timeout: 5000 }),
|
||||
).toBeInTheDocument();
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Remove' }));
|
||||
|
||||
// The Remove handler dispatches several React state updates in one
|
||||
// event (removeContact + setContacts + setComposeStatus + setComposeError).
|
||||
// Under CI load the resulting render-and-paint cycle has been observed
|
||||
// to take >1s, which is the default findByText timeout — that race has
|
||||
// produced flakes on PRs #226, #237, #261, and #262 in succession.
|
||||
// The settle window is bounded by React's reconciliation, not by any
|
||||
// network/animation cost, so a generous timeout is the right deflake
|
||||
// here (the failure mode this masks would be "toast never renders",
|
||||
// which would still fail at 5s).
|
||||
await waitFor(
|
||||
() => {
|
||||
expect(
|
||||
screen.getByText(/Removed contact: Remove Me\./i),
|
||||
).toBeInTheDocument();
|
||||
},
|
||||
{ timeout: 5000, interval: 50 },
|
||||
);
|
||||
expect(screen.queryByText('Remove Me')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('explains unresolved address delivery without exposing backend jargon', async () => {
|
||||
mocks.importWormholeDmInvite.mockRejectedValueOnce(new Error('peer prekey lookup unavailable'));
|
||||
|
||||
renderMessagesView();
|
||||
fireEvent.click(screen.getByRole('button', { name: 'CONTACTS' }));
|
||||
expect(await screen.findByText("Paste Someone's Address")).toBeInTheDocument();
|
||||
|
||||
fireEvent.change(screen.getByPlaceholderText(/Paste a short address/i), {
|
||||
target: { value: JSON.stringify({ invite: { event_type: 'dm_invite', payload: {} } }) },
|
||||
});
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Import Address' }));
|
||||
|
||||
expect(await screen.findByText(/This address is valid, but contact delivery is not ready on this node yet/i)).toBeInTheDocument();
|
||||
expect(screen.queryByText('peer prekey lookup unavailable')).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(/sender prekey/i)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('announces compat invite imports as a saved contact without backend detail', async () => {
|
||||
mocks.importWormholeDmInvite.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
peer_id: '!sb_compat',
|
||||
@@ -734,17 +915,16 @@ describe('MessagesView first-contact trust UX', () => {
|
||||
fireEvent.click(screen.getByRole('button', { name: 'CONTACTS' }));
|
||||
expect(await screen.findByText("Paste Someone's Address")).toBeInTheDocument();
|
||||
|
||||
fireEvent.change(screen.getByPlaceholderText(/Paste the full text copied/i), {
|
||||
fireEvent.change(screen.getByPlaceholderText(/Paste a short address/i), {
|
||||
target: { value: JSON.stringify({ invite: { event_type: 'dm_invite', payload: {} } }) },
|
||||
});
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Import Address' }));
|
||||
|
||||
expect(await screen.findByText(/Contact saved: !sb_compat\./i)).toBeInTheDocument();
|
||||
expect(screen.queryByText(/TOFU PINNED for/i)).not.toBeInTheDocument();
|
||||
expect(
|
||||
await screen.findByText(/TOFU PINNED for !sb_compat \(invitefp\.\.compat\)\./i),
|
||||
).toBeInTheDocument();
|
||||
expect(
|
||||
screen.getByText(/legacy invite imported as tofu_pinned; SAS verification required before first contact/i),
|
||||
).toBeInTheDocument();
|
||||
screen.queryByText(/legacy invite imported as tofu_pinned; SAS verification required before first contact/i),
|
||||
).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('surfaces stable root continuity breaks on invite re-import', async () => {
|
||||
@@ -783,7 +963,7 @@ describe('MessagesView first-contact trust UX', () => {
|
||||
fireEvent.click(screen.getByRole('button', { name: 'CONTACTS' }));
|
||||
expect(await screen.findByText("Paste Someone's Address")).toBeInTheDocument();
|
||||
|
||||
fireEvent.change(screen.getByPlaceholderText(/Paste the full text copied/i), {
|
||||
fireEvent.change(screen.getByPlaceholderText(/Paste a short address/i), {
|
||||
target: { value: JSON.stringify({ invite: { event_type: 'dm_invite', payload: {} } }) },
|
||||
});
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Import Address' }));
|
||||
|
||||
@@ -1357,7 +1357,9 @@ describe('wormholeIdentityClient strict profile hints', () => {
|
||||
}),
|
||||
);
|
||||
|
||||
expect(controlPlaneJson).toHaveBeenCalledWith('/api/wormhole/dm/root-health');
|
||||
expect(controlPlaneJson).toHaveBeenCalledWith('/api/wormhole/dm/root-health', {
|
||||
requireAdminSession: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('prepares the interactive lane through the configured wormhole runtime and bootstraps identity state', async () => {
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
* - /api/tools/* (Sprint 1C addition)
|
||||
* - /api/wormhole/* (pre-existing, regression)
|
||||
* - /api/settings/* (pre-existing, regression)
|
||||
* - /api/layers, /api/ais/feed, /api/ai/agent-actions
|
||||
*
|
||||
* Also verifies that:
|
||||
* - non-sensitive mesh paths (e.g. mesh/events) do NOT receive injected key
|
||||
@@ -272,6 +273,77 @@ describe('proxy admin-key injection coverage', () => {
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
it('POST /api/layers with valid session injects X-Admin-Key', async () => {
|
||||
const cookie = await mintSession(ADMIN_KEY);
|
||||
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ status: 'ok' }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost/api/layers', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ layers: { aircraft: true } }),
|
||||
headers: { cookie, 'Content-Type': 'application/json' },
|
||||
});
|
||||
const res = await proxyPost(req, {
|
||||
params: Promise.resolve({ path: ['layers'] }),
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
it('POST /api/ais/feed with valid session injects X-Admin-Key', async () => {
|
||||
const cookie = await mintSession(ADMIN_KEY);
|
||||
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ status: 'ok' }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost/api/ais/feed', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ msgs: [] }),
|
||||
headers: { cookie, 'Content-Type': 'application/json' },
|
||||
});
|
||||
const res = await proxyPost(req, {
|
||||
params: Promise.resolve({ path: ['ais', 'feed'] }),
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
it('GET /api/ai/agent-actions with valid session injects X-Admin-Key', async () => {
|
||||
const cookie = await mintSession(ADMIN_KEY);
|
||||
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ ok: true, actions: [] }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost/api/ai/agent-actions', {
|
||||
method: 'GET',
|
||||
headers: { cookie },
|
||||
});
|
||||
const res = await proxyGet(req, {
|
||||
params: Promise.resolve({ path: ['ai', 'agent-actions'] }),
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Non-sensitive mesh paths must NOT receive injected admin key
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
/**
|
||||
* Regression coverage for the auth-bypass chain audited by @tg12 in
|
||||
* issues #249, #254, and #255.
|
||||
*
|
||||
* #249 / #254 — Cross-origin webpages must not have the operator's
|
||||
* server-side ADMIN_KEY injected into their forwarded requests. The
|
||||
* proxy enforces a CSRF guard by checking the Origin header against
|
||||
* the request's own Host header. Same-origin (the dashboard itself),
|
||||
* Tauri/native shells (no Origin), and authenticated session cookies
|
||||
* are all allowed; cross-origin browser fetches with a foreign Origin
|
||||
* are rejected.
|
||||
*
|
||||
* #255 — Admin session minting must require ADMIN_KEY to be configured
|
||||
* AND the supplied key to match exactly. The previous implementation
|
||||
* round-tripped to a public backend endpoint (/api/settings/privacy-
|
||||
* profile) which always returns 200, so any key value would mint a
|
||||
* full admin session when ADMIN_KEY was unset on the server.
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { NextRequest } from 'next/server';
|
||||
|
||||
import { GET as proxyGet, POST as proxyPost } from '@/app/api/[...path]/route';
|
||||
import { POST as postAdminSession } from '@/app/api/admin/session/route';
|
||||
|
||||
|
||||
function capturedHeaders(fetchMock: ReturnType<typeof vi.fn>): Headers {
|
||||
const forwarded = fetchMock.mock.calls[0]?.[1];
|
||||
return new Headers((forwarded as RequestInit | undefined)?.headers);
|
||||
}
|
||||
|
||||
|
||||
describe('proxy CSRF guard on admin-key injection (#249/#254)', () => {
|
||||
const ADMIN_KEY = 'env-side-admin-key-32-chars-min!!!!!';
|
||||
const originalAdminKey = process.env.ADMIN_KEY;
|
||||
const originalBackendUrl = process.env.BACKEND_URL;
|
||||
|
||||
beforeEach(() => {
|
||||
process.env.ADMIN_KEY = ADMIN_KEY;
|
||||
process.env.BACKEND_URL = 'http://127.0.0.1:8000';
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env.ADMIN_KEY = originalAdminKey;
|
||||
process.env.BACKEND_URL = originalBackendUrl;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('cross-origin GET to a sensitive route does NOT inject X-Admin-Key', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
// Hostile-webpage CSRF: Origin is a different site than Host.
|
||||
const req = new NextRequest('http://localhost:3000/api/wormhole/identity', {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
host: 'localhost:3000',
|
||||
origin: 'http://evil.example',
|
||||
},
|
||||
});
|
||||
await proxyGet(req, {
|
||||
params: Promise.resolve({ path: ['wormhole', 'identity'] }),
|
||||
});
|
||||
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBeNull();
|
||||
});
|
||||
|
||||
it('cross-origin POST to a sensitive route does NOT inject X-Admin-Key', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost:3000/api/wormhole/identity/bootstrap', {
|
||||
method: 'POST',
|
||||
body: '{}',
|
||||
headers: {
|
||||
host: 'localhost:3000',
|
||||
origin: 'http://attacker.example',
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
});
|
||||
await proxyPost(req, {
|
||||
params: Promise.resolve({ path: ['wormhole', 'identity', 'bootstrap'] }),
|
||||
});
|
||||
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBeNull();
|
||||
});
|
||||
|
||||
it('same-origin request (Origin matches Host) DOES inject X-Admin-Key', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost:3000/api/wormhole/identity', {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
host: 'localhost:3000',
|
||||
origin: 'http://localhost:3000',
|
||||
},
|
||||
});
|
||||
await proxyGet(req, {
|
||||
params: Promise.resolve({ path: ['wormhole', 'identity'] }),
|
||||
});
|
||||
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
it('no Origin header (native shell, server-to-server, curl) DOES inject X-Admin-Key', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost:3000/api/settings/wormhole', {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
host: 'localhost:3000',
|
||||
// no Origin
|
||||
},
|
||||
});
|
||||
await proxyGet(req, {
|
||||
params: Promise.resolve({ path: ['settings', 'wormhole'] }),
|
||||
});
|
||||
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
it('cross-origin request with a valid session cookie STILL injects (cookie auth wins)', async () => {
|
||||
// Mint a session first (against the real handler).
|
||||
const mintReq = new NextRequest('http://localhost:3000/api/admin/session', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ adminKey: ADMIN_KEY }),
|
||||
headers: {
|
||||
host: 'localhost:3000',
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
});
|
||||
const mintRes = await postAdminSession(mintReq);
|
||||
const cookieHeader = mintRes.headers.get('set-cookie') || '';
|
||||
const cookie = cookieHeader.split(';')[0];
|
||||
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
// Now hit a sensitive route from a foreign Origin but WITH the cookie.
|
||||
// Since the cookie itself is SameSite=strict, a real cross-origin
|
||||
// browser fetch wouldn't carry it — but if the operator deliberately
|
||||
// forwards their session (e.g. CLI tool), it should work.
|
||||
const req = new NextRequest('http://localhost:3000/api/wormhole/identity', {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
host: 'localhost:3000',
|
||||
origin: 'http://evil.example',
|
||||
cookie,
|
||||
},
|
||||
});
|
||||
await proxyGet(req, {
|
||||
params: Promise.resolve({ path: ['wormhole', 'identity'] }),
|
||||
});
|
||||
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
it('malformed Origin header is treated as not-same-origin (conservative)', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost:3000/api/wormhole/identity', {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
host: 'localhost:3000',
|
||||
origin: 'not-a-real-origin',
|
||||
},
|
||||
});
|
||||
await proxyGet(req, {
|
||||
params: Promise.resolve({ path: ['wormhole', 'identity'] }),
|
||||
});
|
||||
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBeNull();
|
||||
});
|
||||
|
||||
it('cross-origin to a non-sensitive route is unaffected (no injection either way)', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
// /api/health is not sensitive — no admin-key injection happens at all.
|
||||
const req = new NextRequest('http://localhost:3000/api/health', {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
host: 'localhost:3000',
|
||||
origin: 'http://evil.example',
|
||||
},
|
||||
});
|
||||
await proxyGet(req, {
|
||||
params: Promise.resolve({ path: ['health'] }),
|
||||
});
|
||||
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe('admin session minting refuses arbitrary keys when ADMIN_KEY unset (#255)', () => {
|
||||
const originalAdminKey = process.env.ADMIN_KEY;
|
||||
const originalBackendUrl = process.env.BACKEND_URL;
|
||||
|
||||
beforeEach(() => {
|
||||
delete process.env.ADMIN_KEY;
|
||||
process.env.BACKEND_URL = 'http://127.0.0.1:8000';
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env.ADMIN_KEY = originalAdminKey;
|
||||
process.env.BACKEND_URL = originalBackendUrl;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('refuses to mint a session when ADMIN_KEY is unset on the server', async () => {
|
||||
// Even if the (previously-relied-on) public endpoint returned 200,
|
||||
// the new logic must not accept the key — it does local validation only.
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost:3000/api/admin/session', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ adminKey: 'literally-anything-an-attacker-sends' }),
|
||||
headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
const res = await postAdminSession(req);
|
||||
|
||||
expect(res.status).toBe(403);
|
||||
const body = await res.json();
|
||||
expect(body.ok).toBe(false);
|
||||
expect(String(body.detail)).toMatch(/no admin key configured/i);
|
||||
|
||||
// No session cookie should have been set
|
||||
expect(res.headers.get('set-cookie')).toBeNull();
|
||||
|
||||
// The buggy round-trip to the public endpoint must no longer happen
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('refuses an empty key with 400 (Missing admin key)', async () => {
|
||||
const req = new NextRequest('http://localhost:3000/api/admin/session', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ adminKey: '' }),
|
||||
headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
const res = await postAdminSession(req);
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe('admin session minting still works when ADMIN_KEY is set (#255 regression)', () => {
|
||||
const ADMIN_KEY = 'configured-admin-key-32-chars-min!!!!';
|
||||
const originalAdminKey = process.env.ADMIN_KEY;
|
||||
const originalBackendUrl = process.env.BACKEND_URL;
|
||||
|
||||
beforeEach(() => {
|
||||
process.env.ADMIN_KEY = ADMIN_KEY;
|
||||
process.env.BACKEND_URL = 'http://127.0.0.1:8000';
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env.ADMIN_KEY = originalAdminKey;
|
||||
process.env.BACKEND_URL = originalBackendUrl;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('mints a session when the supplied key matches the configured ADMIN_KEY', async () => {
|
||||
const req = new NextRequest('http://localhost:3000/api/admin/session', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ adminKey: ADMIN_KEY }),
|
||||
headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
const res = await postAdminSession(req);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get('set-cookie')).toBeTruthy();
|
||||
});
|
||||
|
||||
it('rejects a non-matching key with 403', async () => {
|
||||
const req = new NextRequest('http://localhost:3000/api/admin/session', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ adminKey: 'wrong-key-attempted-by-attacker' }),
|
||||
headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
const res = await postAdminSession(req);
|
||||
|
||||
expect(res.status).toBe(403);
|
||||
expect(res.headers.get('set-cookie')).toBeNull();
|
||||
});
|
||||
|
||||
it('does NOT round-trip to a backend endpoint for verification (local-only validation)', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost:3000/api/admin/session', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ adminKey: ADMIN_KEY }),
|
||||
headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
await postAdminSession(req);
|
||||
|
||||
// The previous implementation did a fetch to verify against the
|
||||
// backend; the fix removes that round-trip because the backend
|
||||
// endpoint it called was public anyway. Local string-compare suffices.
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,164 @@
|
||||
/**
|
||||
* Issues #218 / #219 / #220 (tg12 external audit):
|
||||
*
|
||||
* Every browser-direct call to Wikipedia or Wikidata must send the
|
||||
* `Api-User-Agent` header that Wikimedia's UA policy asks for. These
|
||||
* tests pin that requirement on the shared `lib/wikimediaClient`
|
||||
* helper that WikiImage, NewsFeed, and useRegionDossier all route
|
||||
* through, so a future refactor that drops the header gets a loud
|
||||
* test failure rather than a silent ToS regression.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
WIKIMEDIA_API_USER_AGENT,
|
||||
fetchWikipediaSummary,
|
||||
fetchWikidataSparql,
|
||||
_resetWikimediaClientCacheForTests,
|
||||
} from '@/lib/wikimediaClient';
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
describe('lib/wikimediaClient', () => {
|
||||
beforeEach(() => {
|
||||
_resetWikimediaClientCacheForTests();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('exposes a stable Api-User-Agent identifier with a contact path', () => {
|
||||
expect(WIKIMEDIA_API_USER_AGENT).toContain('Shadowbroker');
|
||||
expect(WIKIMEDIA_API_USER_AGENT.toLowerCase()).toContain('github.com');
|
||||
expect(WIKIMEDIA_API_USER_AGENT.toLowerCase()).toContain('issues');
|
||||
});
|
||||
|
||||
it('sends Api-User-Agent on Wikipedia summary fetch', async () => {
|
||||
const calls: Array<{ url: string; init?: RequestInit }> = [];
|
||||
globalThis.fetch = vi.fn(async (url: any, init?: RequestInit) => {
|
||||
calls.push({ url: String(url), init });
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
type: 'standard',
|
||||
title: 'Boeing 747',
|
||||
description: 'aircraft',
|
||||
extract: 'long extract',
|
||||
thumbnail: { source: 'https://example.org/thumb.jpg' },
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
}) as any;
|
||||
|
||||
const summary = await fetchWikipediaSummary('Boeing 747');
|
||||
expect(summary?.thumbnail).toBe('https://example.org/thumb.jpg');
|
||||
expect(calls).toHaveLength(1);
|
||||
const headers = (calls[0].init?.headers || {}) as Record<string, string>;
|
||||
expect(headers['Api-User-Agent']).toBe(WIKIMEDIA_API_USER_AGENT);
|
||||
});
|
||||
|
||||
it('sends Api-User-Agent on Wikidata SPARQL fetch', async () => {
|
||||
const calls: Array<{ url: string; init?: RequestInit }> = [];
|
||||
globalThis.fetch = vi.fn(async (url: any, init?: RequestInit) => {
|
||||
calls.push({ url: String(url), init });
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
results: {
|
||||
bindings: [
|
||||
{
|
||||
leaderLabel: { value: 'Test Leader' },
|
||||
govTypeLabel: { value: 'Test Government' },
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
}) as any;
|
||||
|
||||
const bindings = await fetchWikidataSparql('SELECT * WHERE { ?s ?p ?o }');
|
||||
expect(bindings).toHaveLength(1);
|
||||
const headers = (calls[0].init?.headers || {}) as Record<string, string>;
|
||||
expect(headers['Api-User-Agent']).toBe(WIKIMEDIA_API_USER_AGENT);
|
||||
expect(headers['Accept']).toBe('application/sparql-results+json');
|
||||
});
|
||||
|
||||
it('shares cache across consecutive callers for the same Wikipedia title', async () => {
|
||||
let fetchCount = 0;
|
||||
globalThis.fetch = vi.fn(async () => {
|
||||
fetchCount++;
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
type: 'standard',
|
||||
title: 'Eiffel Tower',
|
||||
description: 'iron lattice tower',
|
||||
extract: '...',
|
||||
thumbnail: { source: 'https://example.org/eiffel.jpg' },
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
}) as any;
|
||||
|
||||
const a = await fetchWikipediaSummary('Eiffel Tower');
|
||||
const b = await fetchWikipediaSummary('Eiffel Tower');
|
||||
expect(fetchCount).toBe(1);
|
||||
expect(a?.thumbnail).toBe(b?.thumbnail);
|
||||
});
|
||||
|
||||
it('deduplicates concurrent in-flight requests for the same title', async () => {
|
||||
let fetchCount = 0;
|
||||
globalThis.fetch = vi.fn(async () => {
|
||||
fetchCount++;
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
type: 'standard',
|
||||
title: 'Mount Fuji',
|
||||
description: 'stratovolcano',
|
||||
extract: '...',
|
||||
thumbnail: { source: 'https://example.org/fuji.jpg' },
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
}) as any;
|
||||
|
||||
const [a, b, c] = await Promise.all([
|
||||
fetchWikipediaSummary('Mount Fuji'),
|
||||
fetchWikipediaSummary('Mount Fuji'),
|
||||
fetchWikipediaSummary('Mount Fuji'),
|
||||
]);
|
||||
expect(fetchCount).toBe(1);
|
||||
expect(a?.thumbnail).toBe('https://example.org/fuji.jpg');
|
||||
expect(b).toEqual(a);
|
||||
expect(c).toEqual(a);
|
||||
});
|
||||
|
||||
it('returns null on disambiguation pages without throwing', async () => {
|
||||
globalThis.fetch = vi.fn(async () =>
|
||||
new Response(JSON.stringify({ type: 'disambiguation' }), { status: 200 }),
|
||||
) as any;
|
||||
const summary = await fetchWikipediaSummary('Mercury');
|
||||
expect(summary).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null on HTTP error without throwing', async () => {
|
||||
globalThis.fetch = vi.fn(async () => new Response('not found', { status: 404 })) as any;
|
||||
const summary = await fetchWikipediaSummary('Nonexistent Article 12345');
|
||||
expect(summary).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null on network error without throwing', async () => {
|
||||
globalThis.fetch = vi.fn(async () => {
|
||||
throw new Error('network down');
|
||||
}) as any;
|
||||
const summary = await fetchWikipediaSummary('Anything');
|
||||
expect(summary).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null on empty input', async () => {
|
||||
globalThis.fetch = vi.fn(async () => new Response('{}', { status: 200 })) as any;
|
||||
expect(await fetchWikipediaSummary('')).toBeNull();
|
||||
expect(await fetchWikipediaSummary(' ')).toBeNull();
|
||||
expect(globalThis.fetch).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -64,6 +64,9 @@ function isSensitiveProxyPath(pathSegments: string[]): boolean {
|
||||
if (joined === 'refresh') return true;
|
||||
if (joined === 'debug-latest') return true;
|
||||
if (joined === 'system/update') return true;
|
||||
if (joined === 'layers') return true;
|
||||
if (joined === 'ais/feed') return true;
|
||||
if (joined === 'ai/agent-actions') return true;
|
||||
if (pathSegments[0] === 'settings') return true;
|
||||
if (joined === 'mesh/infonet/ingest') return true;
|
||||
if (joined === 'mesh/meshtastic/send') return true;
|
||||
@@ -74,6 +77,48 @@ function isSensitiveProxyPath(pathSegments: string[]): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* CSRF guard for the server-side admin-key injection (issues #249 / #254).
|
||||
*
|
||||
* The proxy injects ``process.env.ADMIN_KEY`` into the forwarded
|
||||
* X-Admin-Key header for sensitive backend routes. Without an origin
|
||||
* check, any cross-origin webpage the operator visits could fire
|
||||
* ``fetch('http://localhost:3000/api/wormhole/identity/bootstrap')`` and
|
||||
* have that request get the operator's admin key injected for free —
|
||||
* full identity-takeover CSRF.
|
||||
*
|
||||
* We allow injection when ANY of these is true:
|
||||
* - The request carries a valid admin session cookie (already auth'd)
|
||||
* - The Origin header is absent (server-to-server fetch, Tauri/Electron
|
||||
* native shells, curl/cli — none of these are browser-CSRF surfaces)
|
||||
* - The Origin header host matches the request's own Host (genuine
|
||||
* same-origin browser fetch from our own dashboard)
|
||||
*
|
||||
* If Origin is present AND doesn't match Host, the caller is a hostile
|
||||
* cross-origin webpage. We refuse to inject the admin key. The backend
|
||||
* then sees the request without auth and rejects it via
|
||||
* require_local_operator — exactly the desired outcome.
|
||||
*/
|
||||
function isSameOriginOrNonBrowser(req: NextRequest): boolean {
|
||||
const origin = req.headers.get('origin');
|
||||
if (!origin) {
|
||||
// No Origin header = server-to-server / native shell / older browser
|
||||
// doing a same-origin GET. CSRF requires the attacker to control a
|
||||
// page running in a browser, which always sends Origin on the
|
||||
// dangerous methods. Treat missing Origin as not-CSRF.
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
const originUrl = new URL(origin);
|
||||
const host = req.headers.get('host') || '';
|
||||
if (!host) return false;
|
||||
return originUrl.host.toLowerCase() === host.toLowerCase();
|
||||
} catch {
|
||||
// Malformed Origin header — be conservative.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function proxy(req: NextRequest, pathSegments: string[]): Promise<NextResponse> {
|
||||
try {
|
||||
const isMesh = pathSegments[0] === 'mesh';
|
||||
@@ -189,8 +234,23 @@ async function proxy(req: NextRequest, pathSegments: string[]): Promise<NextResp
|
||||
}
|
||||
});
|
||||
if (isSensitiveProxyPath(pathSegments)) {
|
||||
// Issues #249 / #254: gate the server-side admin-key injection on
|
||||
// either a valid admin session cookie OR a same-origin request.
|
||||
// Cross-origin webpages must not silently inherit the operator's
|
||||
// ADMIN_KEY just by being open in the same browser.
|
||||
const cookieToken = req.cookies.get(ADMIN_COOKIE)?.value || '';
|
||||
const injectedAdmin = process.env.ADMIN_KEY || resolveAdminSessionToken(cookieToken) || '';
|
||||
const sessionAdminKey = resolveAdminSessionToken(cookieToken) || '';
|
||||
const allowEnvKeyInjection = isSameOriginOrNonBrowser(req);
|
||||
let injectedAdmin = '';
|
||||
if (sessionAdminKey) {
|
||||
// Authenticated session always works — Origin doesn't matter
|
||||
// because the cookie itself is same-site / strict.
|
||||
injectedAdmin = sessionAdminKey;
|
||||
} else if (allowEnvKeyInjection && process.env.ADMIN_KEY) {
|
||||
// Fall back to the server-side ADMIN_KEY only for legitimate
|
||||
// callers (same-origin dashboard, Tauri shell, server-to-server).
|
||||
injectedAdmin = process.env.ADMIN_KEY;
|
||||
}
|
||||
if (injectedAdmin) {
|
||||
forwardHeaders.set('X-Admin-Key', injectedAdmin);
|
||||
}
|
||||
|
||||
@@ -22,40 +22,41 @@ function cookieOptions() {
|
||||
};
|
||||
}
|
||||
|
||||
async function verifyAdminKey(adminKey: string): Promise<{ ok: true } | { ok: false; detail: string }> {
|
||||
const backendUrl = process.env.BACKEND_URL ?? 'http://127.0.0.1:8000';
|
||||
const verifyAgainstBackend = async (): Promise<
|
||||
{ ok: true } | { ok: false; detail: string }
|
||||
> => {
|
||||
try {
|
||||
const res = await fetch(`${backendUrl}/api/settings/privacy-profile`, {
|
||||
method: 'GET',
|
||||
headers: { 'X-Admin-Key': adminKey },
|
||||
cache: 'no-store',
|
||||
});
|
||||
if (res.ok) return { ok: true };
|
||||
const data = await res.json().catch(() => ({}));
|
||||
return {
|
||||
ok: false,
|
||||
detail: String(data?.detail || data?.message || 'Unable to verify admin key'),
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
ok: false,
|
||||
detail: 'Unable to verify admin key against backend',
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Verify an operator-supplied admin key before minting a session cookie.
|
||||
*
|
||||
* Issue #255: the previous implementation, when ADMIN_KEY was unset on
|
||||
* the server, fell through to verifying against the backend by GET-ing
|
||||
* /api/settings/privacy-profile. That endpoint is public — it returns
|
||||
* 200 for any X-Admin-Key value (or none at all) — so the fallback
|
||||
* accepted *arbitrary* keys and minted full admin sessions for them.
|
||||
*
|
||||
* Fix: require ADMIN_KEY to be configured before any session can be
|
||||
* minted, and do the validation locally instead of round-tripping to a
|
||||
* potentially-public endpoint. If ADMIN_KEY is unset, the backend
|
||||
* already auto-trusts loopback / docker-bridge callers via
|
||||
* require_local_operator + SHADOWBROKER_TRUST_DOCKER_BRIDGE_LOCAL_OPERATOR,
|
||||
* so legitimate local users keep working — they just don't get (and
|
||||
* don't need) a privileged session cookie.
|
||||
*/
|
||||
async function verifyAdminKey(
|
||||
adminKey: string,
|
||||
): Promise<{ ok: true } | { ok: false; detail: string }> {
|
||||
const configuredAdmin = String(process.env.ADMIN_KEY || '').trim();
|
||||
if (configuredAdmin) {
|
||||
if (adminKey !== configuredAdmin) {
|
||||
return { ok: false, detail: 'Invalid admin key' };
|
||||
}
|
||||
return verifyAgainstBackend();
|
||||
if (!configuredAdmin) {
|
||||
return {
|
||||
ok: false,
|
||||
detail:
|
||||
'No admin key configured on the server. Local-host requests are '
|
||||
+ 'already auto-trusted by the backend — no session is needed. '
|
||||
+ 'To enable session-based admin auth, set ADMIN_KEY in the backend '
|
||||
+ 'environment and restart.',
|
||||
};
|
||||
}
|
||||
|
||||
return verifyAgainstBackend();
|
||||
if (adminKey !== configuredAdmin) {
|
||||
return { ok: false, detail: 'Invalid admin key' };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { Metadata } from 'next';
|
||||
import DesktopBridgeBootstrap from '@/components/DesktopBridgeBootstrap';
|
||||
import { ThemeProvider } from '@/lib/ThemeContext';
|
||||
import { I18nProvider } from '@/i18n';
|
||||
import './globals.css';
|
||||
|
||||
export const metadata: Metadata = {
|
||||
@@ -27,10 +28,12 @@ export default function RootLayout({
|
||||
<link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono:wght@400;700&display=swap" rel="stylesheet" />
|
||||
</head>
|
||||
<body className="antialiased bg-[var(--bg-primary)]" suppressHydrationWarning>
|
||||
<ThemeProvider>
|
||||
<DesktopBridgeBootstrap />
|
||||
{children}
|
||||
</ThemeProvider>
|
||||
<I18nProvider>
|
||||
<ThemeProvider>
|
||||
<DesktopBridgeBootstrap />
|
||||
{children}
|
||||
</ThemeProvider>
|
||||
</I18nProvider>
|
||||
</body>
|
||||
</html>
|
||||
);
|
||||
|
||||
+21
-20
@@ -51,6 +51,7 @@ import {
|
||||
markSentinelInfoSeen,
|
||||
hasSentinelCredentials,
|
||||
} from '@/lib/sentinelHub';
|
||||
import { useTranslation } from '@/i18n';
|
||||
import { LocateBar } from './LocateBar';
|
||||
import { SentinelInfoModal } from './SentinelInfoModal';
|
||||
import SarAoiEditorModal from '@/components/SarAoiEditorModal';
|
||||
@@ -62,6 +63,7 @@ const MaplibreViewer = dynamic(() => import('@/components/MaplibreViewer'), { ss
|
||||
|
||||
export default function Dashboard() {
|
||||
const viewBoundsRef = useRef<{ south: number; west: number; north: number; east: number } | null>(null);
|
||||
const { t } = useTranslation();
|
||||
// Start the critical map data request before panel/control-plane effects.
|
||||
// Non-map widgets can warm up after this; first paint needs flights, ships, and intel first.
|
||||
useDataPolling();
|
||||
@@ -88,10 +90,10 @@ export default function Dashboard() {
|
||||
useEffect(() => {
|
||||
const l = localStorage.getItem('sb_left_open');
|
||||
const r = localStorage.getItem('sb_right_open');
|
||||
const t = localStorage.getItem('sb_ticker_open');
|
||||
const tk = localStorage.getItem('sb_ticker_open');
|
||||
if (l !== null) setLeftOpen(l === 'true');
|
||||
if (r !== null) setRightOpen(r === 'true');
|
||||
if (t !== null) setTickerOpen(t === 'true');
|
||||
if (tk !== null) setTickerOpen(tk === 'true');
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -203,8 +205,8 @@ export default function Dashboard() {
|
||||
uap_sightings: true,
|
||||
// Biosurveillance
|
||||
wastewater: true,
|
||||
// CrowdThreat
|
||||
crowdthreat: true,
|
||||
// CrowdThreat is operator opt-in only.
|
||||
crowdthreat: false,
|
||||
// Shodan
|
||||
shodan_overlay: false,
|
||||
// AI Intel
|
||||
@@ -528,14 +530,14 @@ export default function Dashboard() {
|
||||
S H A D O W <span className="text-cyan-400">B R O K E R</span>
|
||||
</h1>
|
||||
<span className="text-[11px] text-[var(--text-muted)] font-mono tracking-[0.3em] mt-1 ml-1">
|
||||
GLOBAL THREAT INTERCEPT
|
||||
{t('brand.subtitle')}
|
||||
</span>
|
||||
</div>
|
||||
</motion.div>
|
||||
|
||||
{/* SYSTEM METRICS TOP LEFT */}
|
||||
<div className="absolute top-2 left-6 text-[11px] font-mono tracking-widest text-cyan-500/50 z-[200] pointer-events-none hud-zone">
|
||||
OPTIC VIS:113 SRC:180 DENS:1.42 0.8ms
|
||||
{t('brand.systemMetrics')}
|
||||
</div>
|
||||
|
||||
{/* SYSTEM METRICS TOP RIGHT — removed, label moved into TimelineScrubber */}
|
||||
@@ -580,8 +582,8 @@ export default function Dashboard() {
|
||||
</ErrorBoundary>
|
||||
) : (
|
||||
<div className="bg-[#05090d]/95 border border-cyan-900/50 p-4 font-mono text-cyan-500/70">
|
||||
<div className="text-[11px] tracking-[0.2em] text-cyan-400 font-bold">DATA LAYERS</div>
|
||||
<div className="mt-3 text-[10px] tracking-wider">PRIORITIZING MAP FEEDS</div>
|
||||
<div className="text-[11px] tracking-[0.2em] text-cyan-400 font-bold">{t('nav.dataLayers')}</div>
|
||||
<div className="mt-3 text-[10px] tracking-wider">{t('nav.prioritizingMapFeeds')}</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
@@ -647,7 +649,7 @@ export default function Dashboard() {
|
||||
className="text-[7px] font-mono tracking-[0.2em] font-bold"
|
||||
style={{ writingMode: 'vertical-rl', transform: 'rotate(180deg)' }}
|
||||
>
|
||||
LAYERS
|
||||
{t('nav.layers')}
|
||||
</span>
|
||||
</button>
|
||||
</motion.div>
|
||||
@@ -667,7 +669,7 @@ export default function Dashboard() {
|
||||
className="text-[7px] font-mono tracking-[0.2em] font-bold"
|
||||
style={{ writingMode: 'vertical-rl' }}
|
||||
>
|
||||
INTEL
|
||||
{t('nav.intel')}
|
||||
</span>
|
||||
</button>
|
||||
</motion.div>
|
||||
@@ -768,7 +770,7 @@ export default function Dashboard() {
|
||||
{/* Coordinates */}
|
||||
<div className="flex flex-col items-center min-w-[140px]">
|
||||
<div className="text-[10px] text-[var(--text-muted)] font-mono tracking-[0.2em]">
|
||||
COORDINATES
|
||||
{t('controls.coordinates')}
|
||||
</div>
|
||||
<div className="text-[14px] text-cyan-400 font-mono font-bold tracking-wide">
|
||||
{mouseCoords
|
||||
@@ -783,10 +785,10 @@ export default function Dashboard() {
|
||||
{/* Location name */}
|
||||
<div className="flex flex-col items-center min-w-[180px] max-w-[320px]">
|
||||
<div className="text-[10px] text-[var(--text-muted)] font-mono tracking-[0.2em]">
|
||||
LOCATION
|
||||
{t('controls.location')}
|
||||
</div>
|
||||
<div className="text-[13px] text-[var(--text-secondary)] font-mono truncate max-w-[320px]">
|
||||
{locationLabel || 'Hover over map...'}
|
||||
{locationLabel || t('controls.hoverMap')}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -796,7 +798,7 @@ export default function Dashboard() {
|
||||
{/* Style preset (compact) */}
|
||||
<div className="flex flex-col items-center">
|
||||
<div className="text-[10px] text-[var(--text-muted)] font-mono tracking-[0.2em]">
|
||||
STYLE
|
||||
{t('controls.style')}
|
||||
</div>
|
||||
<div className="text-[14px] text-cyan-400 font-mono font-bold">
|
||||
{activeStyle}
|
||||
@@ -815,7 +817,7 @@ export default function Dashboard() {
|
||||
title={`Kp Index: ${sw?.kp_index ?? 'N/A'}`}
|
||||
>
|
||||
<div className="text-[10px] text-[var(--text-muted)] font-mono tracking-[0.2em]">
|
||||
SOLAR
|
||||
{t('controls.solar')}
|
||||
</div>
|
||||
<div
|
||||
className={`text-[14px] font-mono font-bold ${
|
||||
@@ -826,7 +828,7 @@ export default function Dashboard() {
|
||||
: 'text-green-400'
|
||||
}`}
|
||||
>
|
||||
{sw?.kp_text || 'N/A'}
|
||||
{sw?.kp_text || t('controls.na')}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
@@ -857,7 +859,7 @@ export default function Dashboard() {
|
||||
onClick={() => setUiVisible(true)}
|
||||
className="absolute bottom-9 right-6 z-[200] bg-[var(--bg-primary)]/80 border border-[var(--border-primary)] px-4 py-2 text-[10px] font-mono tracking-widest text-cyan-500 hover:text-cyan-300 hover:border-cyan-800 transition-colors pointer-events-auto"
|
||||
>
|
||||
RESTORE UI
|
||||
{t('nav.restoreUi')}
|
||||
</button>
|
||||
)}
|
||||
|
||||
@@ -984,8 +986,7 @@ export default function Dashboard() {
|
||||
{backendStatus === 'disconnected' && (
|
||||
<div className="absolute top-0 left-0 right-0 z-[9000] flex items-center justify-center py-2 bg-red-950/90 border-b border-red-500/40 backdrop-blur-sm">
|
||||
<span className="text-[10px] font-mono tracking-widest text-red-400">
|
||||
BACKEND OFFLINE — Cannot reach backend server. Check that the backend container is
|
||||
running and BACKEND_URL is correct.
|
||||
{t('backend.offline')}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
@@ -1000,7 +1001,7 @@ export default function Dashboard() {
|
||||
className="flex items-center gap-2 px-3 py-1 bg-cyan-950/40 border border-cyan-800/50 border-b-0 rounded-t text-cyan-700 hover:text-cyan-400 hover:bg-cyan-950/60 hover:border-cyan-500/40 transition-colors"
|
||||
>
|
||||
<div className="text-[7.5px] font-mono tracking-[0.25em] font-bold uppercase">
|
||||
MARKETS
|
||||
{t('nav.markets')}
|
||||
</div>
|
||||
{tickerOpen ? <ChevronDown size={10} /> : <ChevronUp size={10} />}
|
||||
</button>
|
||||
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
} from 'lucide-react';
|
||||
import { API_BASE } from '@/lib/api';
|
||||
import type { AIIntelPin, AIIntelLayer, SatelliteScene } from '@/types/aiIntel';
|
||||
import { useTranslation } from '@/i18n';
|
||||
import ConfirmDialog from '@/components/ui/ConfirmDialog';
|
||||
import {
|
||||
createLayer as apiCreateLayer,
|
||||
@@ -1039,6 +1040,7 @@ export default function AIIntelPanel({
|
||||
pinPlacementMode,
|
||||
onPinPlacementModeChange,
|
||||
}: AIIntelPanelProps) {
|
||||
const { t } = useTranslation();
|
||||
const [internalMinimized, setInternalMinimized] = useState(true);
|
||||
const isMinimized = isMinimizedProp !== undefined ? isMinimizedProp : internalMinimized;
|
||||
const setIsMinimized = (val: boolean | ((prev: boolean) => boolean)) => {
|
||||
@@ -1293,7 +1295,7 @@ export default function AIIntelPanel({
|
||||
<div className="flex items-center gap-2">
|
||||
<Brain size={16} className="text-violet-400" />
|
||||
<span className="text-[12px] text-violet-400 font-mono tracking-widest font-bold">
|
||||
AI INTEL
|
||||
{t('ai.title').toUpperCase()}
|
||||
</span>
|
||||
{totalPins > 0 && (
|
||||
<span className="text-[11px] font-mono px-1.5 py-0.5 bg-violet-500/20 border border-violet-500/40 text-violet-300">
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
'use client';
|
||||
|
||||
import { useEffect, useState } from 'react';
|
||||
import { motion, AnimatePresence } from 'framer-motion';
|
||||
import type { ToastItem } from '@/hooks/useAlertToasts';
|
||||
|
||||
const TOAST_LIFETIME_MS = 5_000;
|
||||
|
||||
function getRiskColor(score: number): string {
|
||||
if (score >= 9) return '#ef4444';
|
||||
if (score >= 7) return '#f97316';
|
||||
@@ -16,6 +19,122 @@ function getRiskLabel(score: number): string {
|
||||
return 'ELEVATED';
|
||||
}
|
||||
|
||||
/**
|
||||
* ToastCard — renders a single toast with hover-to-pause auto-dismiss.
|
||||
*
|
||||
* Each card owns its own 5s dismiss timer. Hovering the card pauses the
|
||||
* timer; the timer restarts (full duration) on mouse leave. All visual
|
||||
* styling, the progress bar animation, the click-to-fly behavior, and
|
||||
* the dismiss button match the previous inline implementation — the
|
||||
* only behavioral change is the pause-on-hover.
|
||||
*/
|
||||
function ToastCard({
|
||||
toast,
|
||||
onDismiss,
|
||||
onFlyTo,
|
||||
}: {
|
||||
toast: ToastItem;
|
||||
onDismiss: (id: string) => void;
|
||||
onFlyTo?: (lat: number, lng: number) => void;
|
||||
}) {
|
||||
const [isPaused, setIsPaused] = useState(false);
|
||||
const color = getRiskColor(toast.risk_score);
|
||||
const label = getRiskLabel(toast.risk_score);
|
||||
|
||||
// Per-toast auto-dismiss timer. Restarts whenever the pause flag flips
|
||||
// off — so hovering resets the clock back to a full lifetime when the
|
||||
// user moves the mouse away, giving them time to actually read it.
|
||||
useEffect(() => {
|
||||
if (isPaused) return;
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
onDismiss(toast.id);
|
||||
}, TOAST_LIFETIME_MS);
|
||||
|
||||
return () => clearTimeout(timer);
|
||||
}, [isPaused, toast.id, onDismiss]);
|
||||
|
||||
return (
|
||||
<motion.div
|
||||
key={toast.id}
|
||||
layout
|
||||
initial={{ opacity: 0, x: 100, scale: 0.9 }}
|
||||
animate={{ opacity: 1, x: 0, scale: 1 }}
|
||||
exit={{ opacity: 0, x: 100, scale: 0.9 }}
|
||||
transition={{ type: 'spring', damping: 25, stiffness: 300 }}
|
||||
className="pointer-events-auto cursor-pointer"
|
||||
onMouseEnter={() => setIsPaused(true)}
|
||||
onMouseLeave={() => setIsPaused(false)}
|
||||
onClick={() => {
|
||||
if (onFlyTo && toast.lat && toast.lng) {
|
||||
onFlyTo(toast.lat, toast.lng);
|
||||
}
|
||||
onDismiss(toast.id);
|
||||
}}
|
||||
>
|
||||
<div
|
||||
className="relative bg-[rgba(5,5,5,0.96)] backdrop-blur-sm rounded-sm overflow-hidden font-mono"
|
||||
style={{
|
||||
borderLeft: `3px solid ${color}`,
|
||||
boxShadow: `0 0 20px ${color}40, 0 4px 12px rgba(0,0,0,0.5)`,
|
||||
}}
|
||||
>
|
||||
{/* Progress bar — animation pauses while the card is hovered. */}
|
||||
<motion.div
|
||||
className="absolute top-0 left-0 h-[2px]"
|
||||
style={{ background: color }}
|
||||
initial={{ width: '100%' }}
|
||||
animate={{ width: isPaused ? '100%' : '0%' }}
|
||||
transition={{ duration: TOAST_LIFETIME_MS / 1000, ease: 'linear' }}
|
||||
/>
|
||||
|
||||
<div className="p-3 pr-8">
|
||||
{/* Header */}
|
||||
<div className="flex items-center gap-2 mb-1.5">
|
||||
<span
|
||||
className="text-[9px] font-bold tracking-[0.2em] px-1.5 py-0.5 rounded-sm"
|
||||
style={{
|
||||
background: `${color}20`,
|
||||
color: color,
|
||||
border: `1px solid ${color}40`,
|
||||
}}
|
||||
>
|
||||
⚠ {label}
|
||||
</span>
|
||||
<span className="text-[9px] text-[var(--text-muted)] tracking-wider uppercase">
|
||||
LVL {toast.risk_score}/10
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Title */}
|
||||
<div
|
||||
className="text-[11px] text-[var(--text-primary)] leading-tight mb-1"
|
||||
style={{ display: '-webkit-box', WebkitLineClamp: 2, WebkitBoxOrient: 'vertical', overflow: 'hidden' }}
|
||||
>
|
||||
{toast.title}
|
||||
</div>
|
||||
|
||||
{/* Source */}
|
||||
<div className="text-[9px] text-[var(--text-muted)] tracking-wider uppercase">
|
||||
{toast.source}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Dismiss button */}
|
||||
<button
|
||||
className="absolute top-2 right-2 text-[var(--text-muted)] hover:text-white transition-colors text-xs font-bold"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
onDismiss(toast.id);
|
||||
}}
|
||||
>
|
||||
×
|
||||
</button>
|
||||
</div>
|
||||
</motion.div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function AlertToast({
|
||||
toasts,
|
||||
onDismiss,
|
||||
@@ -28,87 +147,14 @@ export default function AlertToast({
|
||||
return (
|
||||
<div className="fixed top-16 right-[440px] z-[9500] flex flex-col gap-2 pointer-events-none max-w-[380px]">
|
||||
<AnimatePresence mode="popLayout">
|
||||
{toasts.map((toast) => {
|
||||
const color = getRiskColor(toast.risk_score);
|
||||
const label = getRiskLabel(toast.risk_score);
|
||||
return (
|
||||
<motion.div
|
||||
key={toast.id}
|
||||
layout
|
||||
initial={{ opacity: 0, x: 100, scale: 0.9 }}
|
||||
animate={{ opacity: 1, x: 0, scale: 1 }}
|
||||
exit={{ opacity: 0, x: 100, scale: 0.9 }}
|
||||
transition={{ type: 'spring', damping: 25, stiffness: 300 }}
|
||||
className="pointer-events-auto cursor-pointer"
|
||||
onClick={() => {
|
||||
if (onFlyTo && toast.lat && toast.lng) {
|
||||
onFlyTo(toast.lat, toast.lng);
|
||||
}
|
||||
onDismiss(toast.id);
|
||||
}}
|
||||
>
|
||||
<div
|
||||
className="relative bg-[rgba(5,5,5,0.96)] backdrop-blur-sm rounded-sm overflow-hidden font-mono"
|
||||
style={{
|
||||
borderLeft: `3px solid ${color}`,
|
||||
boxShadow: `0 0 20px ${color}40, 0 4px 12px rgba(0,0,0,0.5)`,
|
||||
}}
|
||||
>
|
||||
{/* Progress bar */}
|
||||
<motion.div
|
||||
className="absolute top-0 left-0 h-[2px]"
|
||||
style={{ background: color }}
|
||||
initial={{ width: '100%' }}
|
||||
animate={{ width: '0%' }}
|
||||
transition={{ duration: 5, ease: 'linear' }}
|
||||
/>
|
||||
|
||||
<div className="p-3 pr-8">
|
||||
{/* Header */}
|
||||
<div className="flex items-center gap-2 mb-1.5">
|
||||
<span
|
||||
className="text-[9px] font-bold tracking-[0.2em] px-1.5 py-0.5 rounded-sm"
|
||||
style={{
|
||||
background: `${color}20`,
|
||||
color: color,
|
||||
border: `1px solid ${color}40`,
|
||||
}}
|
||||
>
|
||||
⚠ {label}
|
||||
</span>
|
||||
<span className="text-[9px] text-[var(--text-muted)] tracking-wider uppercase">
|
||||
LVL {toast.risk_score}/10
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Title */}
|
||||
<div
|
||||
className="text-[11px] text-[var(--text-primary)] leading-tight mb-1"
|
||||
style={{ display: '-webkit-box', WebkitLineClamp: 2, WebkitBoxOrient: 'vertical', overflow: 'hidden' }}
|
||||
>
|
||||
{toast.title}
|
||||
</div>
|
||||
|
||||
{/* Source */}
|
||||
<div className="text-[9px] text-[var(--text-muted)] tracking-wider uppercase">
|
||||
{toast.source}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Dismiss button */}
|
||||
<button
|
||||
className="absolute top-2 right-2 text-[var(--text-muted)] hover:text-white transition-colors text-xs font-bold"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
onDismiss(toast.id);
|
||||
}}
|
||||
>
|
||||
×
|
||||
</button>
|
||||
</div>
|
||||
</motion.div>
|
||||
);
|
||||
})}
|
||||
{toasts.map((toast) => (
|
||||
<ToastCard
|
||||
key={toast.id}
|
||||
toast={toast}
|
||||
onDismiss={onDismiss}
|
||||
onFlyTo={onFlyTo}
|
||||
/>
|
||||
))}
|
||||
</AnimatePresence>
|
||||
</div>
|
||||
);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user