mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-06-05 22:06:40 +02:00
Compare commits
25 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| de7cf3bf10 | |||
| 9ef6213284 | |||
| fb11e0881f | |||
| 7f96151e56 | |||
| d0299fc0a0 | |||
| 87ba70acd6 | |||
| bcc2d036b3 | |||
| 729ea78cb2 | |||
| 459178f283 | |||
| 8e27658157 | |||
| e36d1fc79c | |||
| d00c63abed | |||
| e3297e9bc0 | |||
| 9ae0b189ba | |||
| dd7706f17f | |||
| 30f0360ef8 | |||
| 421682c447 | |||
| 40734e310b | |||
| 71a9d9e144 | |||
| de27d119f9 | |||
| b8384d6d91 | |||
| 11ea345518 | |||
| 25a98a9869 | |||
| 2ce0e43ee5 | |||
| b86a258535 |
@@ -67,6 +67,12 @@ ADMIN_KEY=
|
||||
# SHADOWBROKER_SLOW_FETCH_CONCURRENCY=4
|
||||
# SHADOWBROKER_STARTUP_HEAVY_CONCURRENCY=2
|
||||
|
||||
# Infonet bootstrap/sync responsiveness. Defaults favor fast seed failure
|
||||
# detection so stale onion peers do not make the terminal look hung.
|
||||
# MESH_SYNC_TIMEOUT_S=5
|
||||
# MESH_SYNC_MAX_PEERS_PER_CYCLE=3
|
||||
# MESH_BOOTSTRAP_SEED_FAILURE_COOLDOWN_S=15
|
||||
|
||||
# Google Earth Engine for VIIRS night lights change detection (optional).
|
||||
# pip install earthengine-api
|
||||
# GEE_SERVICE_ACCOUNT_KEY=
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
# CODEOWNERS — assigns required reviewers for sensitive paths.
|
||||
# Format: <path glob> <user-or-team> [<user-or-team> ...]
|
||||
# See https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
|
||||
#
|
||||
# Owners listed here are auto-requested for review when matching files
|
||||
# change in a PR. If branch protection requires CODEOWNERS approval, the
|
||||
# PR cannot be merged until an owner approves.
|
||||
|
||||
# ── Internationalization / translations ──
|
||||
# Translation contributions are held to a stricter neutrality standard
|
||||
# than most code changes — see CONTRIBUTING.md "Translation contributions".
|
||||
# The i18n layer itself (no network calls, no telemetry, static JSON
|
||||
# bundled at build) is the structural guarantee that makes this safe;
|
||||
# changes to it need owner review.
|
||||
/frontend/src/i18n/ @BigBodyCobain
|
||||
|
||||
# ── Security-sensitive code paths ──
|
||||
/backend/auth.py @BigBodyCobain
|
||||
/backend/routers/wormhole.py @BigBodyCobain
|
||||
/backend/services/mesh/ @BigBodyCobain
|
||||
/backend/services/fetchers/ @BigBodyCobain
|
||||
|
||||
# ── CI / build / deploy infra ──
|
||||
/.github/workflows/ @BigBodyCobain
|
||||
/.gitlab-ci.yml @BigBodyCobain
|
||||
/docker-compose.yml @BigBodyCobain
|
||||
/docker-compose.gitlab.yml @BigBodyCobain
|
||||
/helm/ @BigBodyCobain
|
||||
|
||||
# ── This file and policy docs ──
|
||||
/.github/CODEOWNERS @BigBodyCobain
|
||||
/CONTRIBUTING.md @BigBodyCobain
|
||||
+16
@@ -91,6 +91,20 @@ backend/data/*
|
||||
!backend/data/power_plants.json
|
||||
!backend/data/tracked_names.json
|
||||
!backend/data/yacht_alert_db.json
|
||||
# Issue #206: bundled KiwiSDR receiver directory used as last-resort
|
||||
# fallback when rx.linkfanel.net (HTTP-only upstream) is unreachable
|
||||
# or returns content that fails our integrity validation.
|
||||
!backend/data/kiwisdr_directory.json
|
||||
# Issue #201: pinned SHA-256 digests for known Tor Expert Bundle URLs.
|
||||
# Used as a second verification source when upstream .sha256sum fails.
|
||||
!backend/data/tor_bundle_digests.json
|
||||
# Issue #258: SPKI pins for stream.aisstream.io so we can survive upstream
|
||||
# Let's Encrypt renewal failures without disabling TLS validation entirely.
|
||||
!backend/data/aisstream_spki_pins.json
|
||||
# Issue #231: pinned SHA-256 digests for known release archives. Used by
|
||||
# the self-updater as a second-line integrity check when the release's
|
||||
# SHA256SUMS.txt asset can't be fetched.
|
||||
!backend/data/release_digests.json
|
||||
|
||||
# OS generated files
|
||||
.DS_Store
|
||||
@@ -173,6 +187,8 @@ backend/services/test_*.py
|
||||
# Local analysis & dev tools
|
||||
backend/analyze_xlsx.py
|
||||
backend/services/ais_cache.json
|
||||
graphify/
|
||||
graphify-out/
|
||||
|
||||
# ========================
|
||||
# Internal docs & brainstorming (never commit)
|
||||
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
# GitLab CI/CD for Shadowbroker
|
||||
#
|
||||
# Mirror of .github/workflows/docker-publish.yml — keeps the GitLab install
|
||||
# path (image registry + source) at parity with GitHub so users who prefer
|
||||
# GitLab get the same experience.
|
||||
#
|
||||
# What this does on every push to main:
|
||||
# 1. Builds multi-arch (amd64 + arm64) Docker images for the backend and
|
||||
# frontend, pushes them to the project's GitLab Container Registry:
|
||||
# registry.gitlab.com/bigbodycobain/shadowbroker/backend:latest
|
||||
# registry.gitlab.com/bigbodycobain/shadowbroker/frontend:latest
|
||||
# Both also get a :$CI_COMMIT_SHORT_SHA tag for traceability.
|
||||
# 2. Reverse-mirrors main back to GitHub (only if commits land directly
|
||||
# on GitLab) so the two sources stay in sync.
|
||||
#
|
||||
# Auth notes:
|
||||
# - The image build/push uses $CI_JOB_TOKEN, which GitLab provides
|
||||
# automatically. No credentials need to be configured.
|
||||
# - The reverse mirror requires a GitHub personal access token stored
|
||||
# as the GitLab CI/CD variable GITHUB_MIRROR_TOKEN (Protected + Masked).
|
||||
# Scope: public_repo (or repo for private). If the variable isn't
|
||||
# set the mirror job is skipped — image builds still run.
|
||||
|
||||
stages:
|
||||
- build
|
||||
- mirror
|
||||
|
||||
variables:
|
||||
# Use the dind service for buildx multi-arch builds.
|
||||
DOCKER_HOST: tcp://docker:2376
|
||||
DOCKER_TLS_CERTDIR: "/certs"
|
||||
DOCKER_DRIVER: overlay2
|
||||
# QEMU is what lets a single x86 runner build arm64 images. dind doesn't
|
||||
# install it by default; we install via tonistiigi/binfmt below.
|
||||
BUILDX_VERSION: "v0.14.1"
|
||||
# Repository-relative paths.
|
||||
BACKEND_IMAGE: $CI_REGISTRY_IMAGE/backend
|
||||
FRONTEND_IMAGE: $CI_REGISTRY_IMAGE/frontend
|
||||
|
||||
# Shared template: bootstraps buildx + QEMU on the dind service so a single
|
||||
# runner can produce both amd64 and arm64 manifests in one push.
|
||||
.buildx-setup: &buildx-setup
|
||||
image: docker:24
|
||||
services:
|
||||
- name: docker:24-dind
|
||||
command: ["--tls=true"]
|
||||
before_script:
|
||||
- docker info
|
||||
- docker login -u "$CI_REGISTRY_USER" -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
|
||||
- docker run --privileged --rm tonistiigi/binfmt --install all
|
||||
- docker buildx create --use --name multiarch --driver docker-container
|
||||
|
||||
# ── Backend image ────────────────────────────────────────────────────────
|
||||
build-backend:
|
||||
<<: *buildx-setup
|
||||
stage: build
|
||||
script:
|
||||
- >
|
||||
docker buildx build
|
||||
--platform linux/amd64,linux/arm64
|
||||
--file backend/Dockerfile
|
||||
--tag $BACKEND_IMAGE:latest
|
||||
--tag $BACKEND_IMAGE:$CI_COMMIT_SHORT_SHA
|
||||
--push
|
||||
.
|
||||
rules:
|
||||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "schedule"
|
||||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||
changes:
|
||||
- backend/**/*
|
||||
- .gitlab-ci.yml
|
||||
|
||||
# ── Frontend image ───────────────────────────────────────────────────────
|
||||
build-frontend:
|
||||
<<: *buildx-setup
|
||||
stage: build
|
||||
script:
|
||||
- cd frontend
|
||||
- >
|
||||
docker buildx build
|
||||
--platform linux/amd64,linux/arm64
|
||||
--tag $FRONTEND_IMAGE:latest
|
||||
--tag $FRONTEND_IMAGE:$CI_COMMIT_SHORT_SHA
|
||||
--push
|
||||
.
|
||||
rules:
|
||||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "schedule"
|
||||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||
changes:
|
||||
- frontend/**/*
|
||||
- .gitlab-ci.yml
|
||||
|
||||
# ── Reverse mirror to GitHub ─────────────────────────────────────────────
|
||||
# Pushes refs/heads/main to github.com/BigBodyCobain/Shadowbroker.
|
||||
# Fast-forward-only — if GitLab main and GitHub main have diverged, this
|
||||
# fails loudly rather than silently overwriting either side.
|
||||
#
|
||||
# Only runs if GITHUB_MIRROR_TOKEN is set as a CI/CD variable. See the
|
||||
# header comment of this file for setup instructions.
|
||||
mirror-to-github:
|
||||
stage: mirror
|
||||
image: alpine:3.20
|
||||
needs: []
|
||||
before_script:
|
||||
- apk add --no-cache git openssh-client ca-certificates
|
||||
script:
|
||||
- git config --global user.email "ci-mirror@gitlab.com"
|
||||
- git config --global user.name "GitLab CI Mirror"
|
||||
- >
|
||||
git clone --depth=50 --branch main
|
||||
"https://oauth2:${CI_JOB_TOKEN}@gitlab.com/${CI_PROJECT_PATH}.git"
|
||||
repo
|
||||
- cd repo
|
||||
- >
|
||||
git push
|
||||
"https://x-access-token:${GITHUB_MIRROR_TOKEN}@github.com/BigBodyCobain/Shadowbroker.git"
|
||||
"${CI_COMMIT_SHA}:refs/heads/main"
|
||||
rules:
|
||||
- if: $CI_COMMIT_BRANCH == "main" && $GITHUB_MIRROR_TOKEN
|
||||
@@ -0,0 +1,75 @@
|
||||
# Contributing to Shadowbroker
|
||||
|
||||
Thank you for taking the time to contribute. This document covers things specific to this project — for general open-source contribution etiquette, see the GitHub docs.
|
||||
|
||||
---
|
||||
|
||||
## Code contributions
|
||||
|
||||
1. Fork the repo on GitHub (`bigbodycobain/Shadowbroker`) or GitLab (`bigbodycobain/Shadowbroker` mirror).
|
||||
2. Make your changes on a feature branch.
|
||||
3. Run the local test suite:
|
||||
- Backend: `pytest backend/tests/`
|
||||
- Frontend: `cd frontend && npx vitest run`
|
||||
4. Open a Pull Request against `main`.
|
||||
|
||||
CI runs on every PR. If CI fails, that's blocking — please push fixes rather than asking for it to be merged anyway.
|
||||
|
||||
---
|
||||
|
||||
## Reporting security issues
|
||||
|
||||
Do **not** file security issues as public GitHub issues. Email the maintainer or use a private security advisory on GitHub. Public disclosure of an exploitable vulnerability without prior coordination will be rejected from the project.
|
||||
|
||||
---
|
||||
|
||||
## Translation contributions
|
||||
|
||||
Shadowbroker supports UI localization (`frontend/src/i18n/`). Translation contributions are welcome but held to a stricter standard than most code changes, because translations can subtly reshape user perception in ways that are hard to spot during review. Read this section before submitting one.
|
||||
|
||||
### The neutrality requirement
|
||||
|
||||
**Translations must be technically faithful to the English source.** That means:
|
||||
|
||||
- Each `t('key')` entry should mean approximately the same thing in the target language as in English, modulo idiom.
|
||||
- Technical terms with established meanings (e.g. "GPS jamming," "military flight," "Tor," "onion routing," "encryption") should be translated using the corresponding established technical term in the target language — **not** softened, rebranded, or politically reframed.
|
||||
- The set of UI strings should be **the same** between languages. Don't omit features from one locale that are visible in another.
|
||||
|
||||
### What will get a translation PR rejected
|
||||
|
||||
Translation choices that align the project with the framing or terminology of state propaganda — from **any** country — will be rejected. This applies symmetrically:
|
||||
|
||||
| Country / source | Examples of substitutions we will reject |
|
||||
|---|---|
|
||||
| **PRC / CCP** | Calling Taiwan a "province" or "renegade province"; reframing protest layers as "riots"; using softened or euphemistic terms for surveillance, internment, or jamming when the source text is direct |
|
||||
| **Russia** | Calling the Ukraine war a "special military operation"; relabeling occupied territories as Russian; softening sanctions/jamming/disinfo terminology |
|
||||
| **United States / EU** | Reframing adversaries with editorial labels not in the source (e.g. inserting "regime" where the English says "government"); applying labels like "terrorist" or "rogue state" to entities the English source describes neutrally |
|
||||
| **Israel / Palestine / any active conflict** | Substituting one side's preferred terminology when the source uses the other side's or a neutral term |
|
||||
| **Any government** | Adding political slogans, omitting features that government finds inconvenient, or inserting terminology associated with a specific political faction |
|
||||
|
||||
The test is **"would a translator working strictly from the English source produce this rendering?"** If the answer requires assuming a political stance the source does not take, the substitution does not belong in the translation.
|
||||
|
||||
### How translation PRs are reviewed
|
||||
|
||||
Changes to `frontend/src/i18n/**` are owned by the maintainer (see `CODEOWNERS`) and require explicit approval. We will:
|
||||
|
||||
1. Diff the translation against the English source key-by-key.
|
||||
2. Spot-check a sample of entries with a native speaker of the target language when possible.
|
||||
3. Look for the patterns above.
|
||||
4. Look for suspicious additions to the i18n infrastructure itself (e.g. a remote translation fetcher, telemetry on language choice) — the i18n layer is supposed to be 100% client-side static JSON.
|
||||
|
||||
A PR that adds a new language is harder to review than one that fixes typos in an existing language. For new languages, please be patient and expect a real review window. For typo fixes, please describe each change in the PR body so the reviewer can verify intent.
|
||||
|
||||
### What about adding a new language?
|
||||
|
||||
We welcome new languages. The mechanical setup is documented in the header comment of `frontend/src/i18n/index.ts`. Beyond that:
|
||||
|
||||
- We are more likely to merge a new language quickly if at least one reviewer in the maintainer's network speaks it.
|
||||
- If you are the *only* speaker of the target language reading this repo, your translation is welcome but the merge timeline will be longer while a reviewer is found.
|
||||
- Partial translations are fine — the system falls back to English for any missing key.
|
||||
|
||||
---
|
||||
|
||||
## Anything else
|
||||
|
||||
If you have a question that isn't a security report, opening a GitHub Discussion or a draft PR with a question in the body is the fastest way to get a response. Direct emails are read but not always replied to promptly.
|
||||
@@ -61,6 +61,8 @@ ShadowBroker includes an optional Shodan connector for operator-supplied API acc
|
||||
|
||||
## ⚡ Quick Start (Docker)
|
||||
|
||||
### From GitHub (default — uses GHCR images)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/bigbodycobain/Shadowbroker.git
|
||||
cd Shadowbroker
|
||||
@@ -68,6 +70,17 @@ docker compose pull
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
### From GitLab (uses GitLab Container Registry)
|
||||
|
||||
```bash
|
||||
git clone https://gitlab.com/bigbodycobain/Shadowbroker.git
|
||||
cd Shadowbroker
|
||||
docker compose -f docker-compose.yml -f docker-compose.gitlab.yml pull
|
||||
docker compose -f docker-compose.yml -f docker-compose.gitlab.yml up -d
|
||||
```
|
||||
|
||||
Both paths produce identical containers — same source, same CI, same images byte-for-byte. Pick whichever ecosystem you already use.
|
||||
|
||||
Open `http://localhost:3000` to view the dashboard! *(Requires [Docker Desktop](https://www.docker.com/products/docker-desktop/) or Docker Engine)*
|
||||
|
||||
> **Backend port already in use?** The browser only needs port `3000`, but the backend API is also published on host port `8000` for local diagnostics. If another app already uses `8000`, create or edit `.env` next to `docker-compose.yml` and set `BACKEND_PORT=8001`, then run `docker compose up -d`.
|
||||
@@ -136,8 +149,13 @@ helm repo update
|
||||
|
||||
**2. Install the Chart:**
|
||||
```bash
|
||||
# Install from the local helm/chart directory
|
||||
# Default — pulls images from GHCR
|
||||
helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbroker
|
||||
|
||||
# GitLab registry variant
|
||||
helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbroker \
|
||||
-f helm/chart/values.yaml \
|
||||
-f helm/chart/values-gitlab.yaml
|
||||
```
|
||||
|
||||
**3. Key Features:**
|
||||
|
||||
+38
-2
@@ -24,8 +24,40 @@ AIS_API_KEY= # https://aisstream.io/ — free tier WebSocket key
|
||||
# Requires MESH_DEBUG_MODE=true; do not enable this for ordinary use.
|
||||
# ALLOW_INSECURE_ADMIN=false
|
||||
|
||||
# Default outbound User-Agent for all third-party HTTP fetchers.
|
||||
# Project-generic by default — does NOT include any personal contact info or
|
||||
# operator-specific identifier. Override only if you run a public relay and
|
||||
# want upstreams to be able to reach you (e.g. Nominatim/OSM usage policy).
|
||||
# SHADOWBROKER_USER_AGENT=ShadowBroker-OSINT/0.9 (contact: ops@example.com)
|
||||
|
||||
# User-Agent for Nominatim geocoding requests (per OSM usage policy).
|
||||
# NOMINATIM_USER_AGENT=ShadowBroker/1.0 (https://github.com/BigBodyCobain/Shadowbroker)
|
||||
# NOMINATIM_USER_AGENT=ShadowBroker/1.0
|
||||
|
||||
# ── Third-party fetcher opt-ins ────────────────────────────────
|
||||
# These data sources phone home to politically/commercially sensitive
|
||||
# upstreams. Disabled by default; set to "true" only if the operator
|
||||
# explicitly wants the node's IP to contact these services.
|
||||
#
|
||||
# CrowdThreat — backend.crowdthreat.world (paid threat-intel aggregator).
|
||||
# CROWDTHREAT_ENABLED=false
|
||||
#
|
||||
# EUvsDisinfo FIMI — euvsdisinfo.eu (EU disinformation tracker).
|
||||
# FIMI_ENABLED=false
|
||||
#
|
||||
# Polymarket + Kalshi — US political/election prediction markets.
|
||||
# PREDICTION_MARKETS_ENABLED=false
|
||||
#
|
||||
# Finnhub fallback / yfinance — financial market data.
|
||||
# Set FINNHUB_API_KEY to enable Finnhub, or set FINANCIAL_ENABLED=true to allow
|
||||
# the unauthenticated yfinance fallback to call Yahoo Finance.
|
||||
# FINANCIAL_ENABLED=false
|
||||
#
|
||||
# NUFORC UAP sightings — huggingface.co dataset download.
|
||||
# NUFORC_ENABLED=false
|
||||
#
|
||||
# News RSS aggregator — defaults ON. Set to "false" to disable all
|
||||
# configured news feeds (kill switch for the news layer).
|
||||
# NEWS_ENABLED=true
|
||||
|
||||
# LTA Singapore traffic cameras — leave blank to skip this data source.
|
||||
# LTA_ACCOUNT_KEY=
|
||||
@@ -61,8 +93,12 @@ AIS_API_KEY= # https://aisstream.io/ — free tier WebSocket key
|
||||
# Optional Meshtastic node ID (e.g. "!abcd1234"). When set, included in the
|
||||
# User-Agent sent to meshtastic.liamcottle.net so the upstream service operator
|
||||
# can identify per-install traffic instead of aggregated "ShadowBroker" hits.
|
||||
# Leave blank to send a generic UA with the project contact email only.
|
||||
# Leave blank to send a generic UA. If you set MESHTASTIC_OPERATOR_CALLSIGN,
|
||||
# it is included in outbound headers to meshtastic.org by default so they
|
||||
# can rate-limit per-operator. Set MESHTASTIC_SEND_CALLSIGN_HEADER=false to
|
||||
# suppress the callsign while still using it locally (e.g. for APRS).
|
||||
# MESHTASTIC_OPERATOR_CALLSIGN=
|
||||
# MESHTASTIC_SEND_CALLSIGN_HEADER=true
|
||||
# MESH_MQTT_PSK= # hex-encoded, empty = default LongFast key
|
||||
|
||||
# ── Mesh / Reticulum (RNS) ─────────────────────────────────────
|
||||
|
||||
+234
-4
@@ -1,5 +1,37 @@
|
||||
// AIS Stream WebSocket proxy.
|
||||
//
|
||||
// Reads AIS_API_KEY from argv or env, opens a wss:// connection to
|
||||
// stream.aisstream.io, subscribes for vessel position reports inside the
|
||||
// active map bounding boxes, and pipes JSON messages to stdout for the
|
||||
// Python backend to ingest.
|
||||
//
|
||||
// Issue #258 — SPKI pinning fallback for upstream cert outages
|
||||
// -------------------------------------------------------------
|
||||
// AISStream uses Let's Encrypt and their renewal pipeline has been observed
|
||||
// to fail (cert expired on 2026-05-20). The naive fix the issue reporter
|
||||
// applied — passing { rejectUnauthorized: false } — turns off TLS validation
|
||||
// entirely, which lets any network attacker MITM the WebSocket and inject
|
||||
// fake ship positions onto the operator's map. Same class as the GDELT
|
||||
// plaintext-HTTP MITM issue (#199).
|
||||
//
|
||||
// Instead, when the normal TLS handshake fails with CERT_HAS_EXPIRED, we
|
||||
// do a custom TLS connection that ignores ONLY the expiry check, capture
|
||||
// the leaf certificate, and compare its public-key SPKI hash against a
|
||||
// pinned list (backend/data/aisstream_spki_pins.json). If the SPKI matches,
|
||||
// the upstream is still the genuine AISStream — just with an expired cert —
|
||||
// and we proceed in "degraded TLS" mode. If the SPKI does not match, we
|
||||
// refuse the connection and log loudly: an actual MITM is in progress.
|
||||
//
|
||||
// Let's Encrypt renewals keep the same public key by default, so the pinned
|
||||
// SPKI survives normal cert rotation. The pin list MUST be updated before
|
||||
// the operator's pinned key is rotated upstream.
|
||||
|
||||
const WebSocket = require('ws');
|
||||
const readline = require('readline');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const tls = require('tls');
|
||||
const crypto = require('crypto');
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const API_KEY = args[0] || process.env.AIS_API_KEY;
|
||||
@@ -9,6 +41,135 @@ if (!API_KEY) {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// ── SPKI pin support (issue #258) ─────────────────────────────────────────
|
||||
|
||||
const AIS_HOST = 'stream.aisstream.io';
|
||||
const AIS_PORT = 443;
|
||||
const AIS_WS_URL = `wss://${AIS_HOST}/v0/stream`;
|
||||
|
||||
// Pin file is looked up in several layouts so the same JS works in:
|
||||
// - the Docker backend image (PIN_FILE_CANDIDATES[0])
|
||||
// - the Tauri desktop runtime (PIN_FILE_CANDIDATES[1])
|
||||
// - a future relocated layout (operator can drop a file at
|
||||
// SHADOWBROKER_AIS_PINS env var)
|
||||
const PIN_FILE_CANDIDATES = [
|
||||
process.env.SHADOWBROKER_AIS_PINS || '',
|
||||
path.join(__dirname, 'data', 'aisstream_spki_pins.json'),
|
||||
path.join(__dirname, 'aisstream_spki_pins.json'),
|
||||
].filter(Boolean);
|
||||
|
||||
// Embedded fallback. Used when no external pin file is reachable so the
|
||||
// SPKI fallback still works on minimal/portable installs. The external
|
||||
// file (when present) takes priority so operators can update pins without
|
||||
// needing a new build.
|
||||
const EMBEDDED_PINS = {
|
||||
[AIS_HOST]: [
|
||||
// Captured 2026-05-20 from AISStream's leaf cert (Let's Encrypt R12).
|
||||
// Replace when AISStream rotates server keys.
|
||||
'GJ10H0UPgLrO+2d3ZXROR/TXSVFXKUfRC3QEI2ibEg4=',
|
||||
],
|
||||
};
|
||||
|
||||
let aisDegradedMode = false; // surfaced via stdout status_query marker
|
||||
|
||||
function loadSpkiPins() {
|
||||
for (const candidate of PIN_FILE_CANDIDATES) {
|
||||
try {
|
||||
const raw = fs.readFileSync(candidate, 'utf-8');
|
||||
const parsed = JSON.parse(raw);
|
||||
const pins = Array.isArray(parsed[AIS_HOST]) ? parsed[AIS_HOST] : [];
|
||||
const cleaned = pins
|
||||
.filter((p) => typeof p === 'string' && p.length > 0)
|
||||
.map((p) => p.trim());
|
||||
if (cleaned.length > 0) {
|
||||
return cleaned;
|
||||
}
|
||||
} catch (e) {
|
||||
// Try the next candidate — file may not exist in this layout.
|
||||
continue;
|
||||
}
|
||||
}
|
||||
const embedded = (EMBEDDED_PINS[AIS_HOST] || []).slice();
|
||||
if (embedded.length > 0) {
|
||||
console.error(
|
||||
'[AIS Proxy] No external SPKI pin file found; using embedded fallback. '
|
||||
+ `(Set SHADOWBROKER_AIS_PINS or drop ${PIN_FILE_CANDIDATES[1]} to override.)`
|
||||
);
|
||||
}
|
||||
return embedded;
|
||||
}
|
||||
|
||||
function spkiHashFromPeerCert(peerCert) {
|
||||
// tls.TLSSocket.getPeerCertificate() exposes .pubkey when called with
|
||||
// detailed=true. The pubkey buffer is the DER-encoded SubjectPublicKeyInfo,
|
||||
// which is exactly the value we hash for SPKI pinning.
|
||||
if (!peerCert || !peerCert.pubkey) return null;
|
||||
return crypto.createHash('sha256').update(peerCert.pubkey).digest('base64');
|
||||
}
|
||||
|
||||
// Probe the upstream when normal TLS failed with CERT_HAS_EXPIRED. We open
|
||||
// a raw TLS connection with rejectUnauthorized=false ONLY to inspect the
|
||||
// leaf cert; we do NOT use this socket for the actual WebSocket traffic.
|
||||
// Returns { ok: true } if the leaf SPKI matches the pin list, { ok: false }
|
||||
// with a reason otherwise.
|
||||
function verifyExpiredCertAgainstPins() {
|
||||
return new Promise((resolve) => {
|
||||
const pins = loadSpkiPins();
|
||||
if (pins.length === 0) {
|
||||
resolve({ ok: false, reason: 'no SPKI pins configured' });
|
||||
return;
|
||||
}
|
||||
const sock = tls.connect(
|
||||
{
|
||||
host: AIS_HOST,
|
||||
port: AIS_PORT,
|
||||
servername: AIS_HOST,
|
||||
// Allow the handshake to complete despite the expired cert
|
||||
// so we can inspect the leaf. We do NOT trust this connection
|
||||
// for any application data.
|
||||
rejectUnauthorized: false,
|
||||
},
|
||||
() => {
|
||||
const peer = sock.getPeerCertificate(true);
|
||||
sock.end();
|
||||
if (!peer || Object.keys(peer).length === 0) {
|
||||
resolve({ ok: false, reason: 'no peer certificate returned' });
|
||||
return;
|
||||
}
|
||||
if (peer.subject && peer.subject.CN !== AIS_HOST) {
|
||||
resolve({
|
||||
ok: false,
|
||||
reason: `cert CN mismatch (got ${peer.subject.CN}, expected ${AIS_HOST})`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const hash = spkiHashFromPeerCert(peer);
|
||||
if (!hash) {
|
||||
resolve({ ok: false, reason: 'could not compute SPKI hash from peer cert' });
|
||||
return;
|
||||
}
|
||||
if (pins.includes(hash)) {
|
||||
resolve({ ok: true, hash });
|
||||
} else {
|
||||
resolve({
|
||||
ok: false,
|
||||
reason: `SPKI ${hash} not in pin list (possible MITM)`,
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
sock.setTimeout(10000, () => {
|
||||
sock.destroy();
|
||||
resolve({ ok: false, reason: 'TLS probe timeout' });
|
||||
});
|
||||
sock.on('error', (err) => {
|
||||
resolve({ ok: false, reason: `TLS probe error: ${err.message}` });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// ── Subscription state ───────────────────────────────────────────────────
|
||||
|
||||
// Start with global coverage, until frontend updates it
|
||||
let currentBboxes = [[[-90, -180], [90, 180]]];
|
||||
let activeWs = null;
|
||||
@@ -42,14 +203,34 @@ rl.on('line', (line) => {
|
||||
currentBboxes = cmd.bboxes;
|
||||
if (activeWs) sendSub(activeWs); // Resend subscription (swap and replace)
|
||||
}
|
||||
if (cmd.type === "status_query") {
|
||||
// Allow the Python side to probe degraded-mode state by sending
|
||||
// {"type": "status_query"} on stdin. Reply on stdout as a marker.
|
||||
process.stdout.write(JSON.stringify({
|
||||
__ais_proxy_status: { degraded_tls: aisDegradedMode }
|
||||
}) + '\n');
|
||||
}
|
||||
} catch (e) {}
|
||||
});
|
||||
|
||||
function connect() {
|
||||
const ws = new WebSocket('wss://stream.aisstream.io/v0/stream');
|
||||
function attachWsHandlers(ws, { degraded } = { degraded: false }) {
|
||||
activeWs = ws;
|
||||
|
||||
ws.on('open', () => {
|
||||
if (degraded) {
|
||||
console.error(
|
||||
'[AIS Proxy] Connected in DEGRADED TLS MODE — upstream cert is expired '
|
||||
+ 'but SPKI matches the pinned key, so identity is still verified. '
|
||||
+ 'AISStream needs to renew their cert; until then MITM protection '
|
||||
+ 'depends only on the SPKI match. Watch backend logs for resolution.'
|
||||
);
|
||||
aisDegradedMode = true;
|
||||
} else {
|
||||
if (aisDegradedMode) {
|
||||
console.error('[AIS Proxy] Reconnected with full TLS validation — degraded mode cleared.');
|
||||
}
|
||||
aisDegradedMode = false;
|
||||
}
|
||||
sendSub(ws);
|
||||
});
|
||||
|
||||
@@ -61,14 +242,63 @@ function connect() {
|
||||
});
|
||||
|
||||
ws.on('error', (err) => {
|
||||
console.error("WebSocket Proxy Error:", err.message);
|
||||
console.error('WebSocket Proxy Error:', err.message);
|
||||
});
|
||||
|
||||
ws.on('close', () => {
|
||||
activeWs = null;
|
||||
console.error("WebSocket Proxy Closed. Reconnecting in 5s...");
|
||||
console.error('WebSocket Proxy Closed. Reconnecting in 5s...');
|
||||
setTimeout(connect, 5000);
|
||||
});
|
||||
}
|
||||
|
||||
function connect() {
|
||||
// Path A: normal TLS validation (the 99.9% case). If this succeeds we
|
||||
// never touch the SPKI fallback.
|
||||
const ws = new WebSocket(AIS_WS_URL);
|
||||
|
||||
let openedOk = false;
|
||||
ws.on('open', () => { openedOk = true; });
|
||||
|
||||
ws.on('error', async (err) => {
|
||||
// Only the CERT_HAS_EXPIRED case triggers SPKI verification. Any
|
||||
// other TLS or network error gets the standard reconnect path so we
|
||||
// don't accidentally cover up legitimate problems.
|
||||
if (!openedOk && err && err.code === 'CERT_HAS_EXPIRED') {
|
||||
console.error(
|
||||
'[AIS Proxy] Upstream certificate is expired. Verifying SPKI '
|
||||
+ 'against pinned keys before deciding whether to proceed in '
|
||||
+ 'degraded mode...'
|
||||
);
|
||||
const verdict = await verifyExpiredCertAgainstPins();
|
||||
if (verdict.ok) {
|
||||
console.error(
|
||||
`[AIS Proxy] SPKI ${verdict.hash} matches pinned key — `
|
||||
+ 'identity is verified, proceeding in DEGRADED TLS mode.'
|
||||
);
|
||||
const insecureWs = new WebSocket(AIS_WS_URL, {
|
||||
rejectUnauthorized: false,
|
||||
});
|
||||
attachWsHandlers(insecureWs, { degraded: true });
|
||||
} else {
|
||||
console.error(
|
||||
`[AIS Proxy] SPKI verification FAILED (${verdict.reason}). `
|
||||
+ 'Refusing to connect — this would normally indicate an active '
|
||||
+ 'MITM attack. If AISStream rotated their server key, update '
|
||||
+ 'backend/data/aisstream_spki_pins.json with the new SPKI hash.'
|
||||
);
|
||||
// Schedule a retry — operator may have updated the pin file.
|
||||
setTimeout(connect, 60000);
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Default: surface the error and let the close handler reconnect.
|
||||
console.error('WebSocket Proxy Error:', err.message);
|
||||
});
|
||||
|
||||
// Wire normal handlers — these apply unless the error handler above
|
||||
// takes over and replaces activeWs with an insecure socket.
|
||||
attachWsHandlers(ws, { degraded: false });
|
||||
}
|
||||
|
||||
connect();
|
||||
|
||||
+81
-4
@@ -245,15 +245,90 @@ def _docker_bridge_local_operator_enabled() -> bool:
|
||||
}
|
||||
|
||||
|
||||
# Issue #250 (tg12): the previous implementation returned True for any IP
|
||||
# in the entire 172.16.0.0/12 range. Anyone with `docker run` access on
|
||||
# the same daemon could spin up a container that automatically passed
|
||||
# local-operator auth. The fix narrows trust to ONLY connections whose
|
||||
# source IP matches the configured frontend container's hostname.
|
||||
#
|
||||
# Docker DNS resolves both the compose service name (``frontend``) and
|
||||
# the explicit ``container_name`` (``shadowbroker-frontend``) to the
|
||||
# frontend container's bridge IP. We forward-resolve both, cache the
|
||||
# result for 30s, and only trust connections from those exact IPs.
|
||||
#
|
||||
# Operators on shared Docker hosts get the benefit of the narrower
|
||||
# surface. Operators on single-user installs see no behavior change —
|
||||
# their frontend container still resolves and is still trusted.
|
||||
_DOCKER_BRIDGE_TRUST_CACHE: dict = {"ips": frozenset(), "expires": 0.0}
|
||||
_DOCKER_BRIDGE_TRUST_TTL = 30.0
|
||||
|
||||
|
||||
def _trusted_bridge_frontend_hostnames() -> list[str]:
|
||||
"""Container hostnames whose IPs we treat as local-operator on the bridge.
|
||||
|
||||
Default covers both Docker Compose service name (``frontend``) and the
|
||||
explicit ``container_name`` from the shipped docker-compose.yml
|
||||
(``shadowbroker-frontend``). Operators with non-default names can
|
||||
override via the ``SHADOWBROKER_TRUSTED_FRONTEND_HOSTS`` env var
|
||||
(comma-separated, no spaces).
|
||||
"""
|
||||
raw = str(
|
||||
os.environ.get(
|
||||
"SHADOWBROKER_TRUSTED_FRONTEND_HOSTS",
|
||||
"frontend,shadowbroker-frontend",
|
||||
)
|
||||
).strip()
|
||||
return [h.strip() for h in raw.split(",") if h.strip()]
|
||||
|
||||
|
||||
def _resolve_trusted_bridge_ips() -> frozenset[str]:
|
||||
"""Resolve trusted frontend hostnames to a set of IPs, with caching.
|
||||
|
||||
Cached for 30s so we don't hit DNS on every request. The cache is
|
||||
process-local — frontend container IP rotations during a backend's
|
||||
lifetime will be picked up within 30s.
|
||||
|
||||
Returns frozenset() if Docker DNS can't resolve any of the configured
|
||||
hostnames (fail-closed — when in doubt, refuse to trust the bridge).
|
||||
"""
|
||||
import socket
|
||||
import time as _time
|
||||
|
||||
now = _time.time()
|
||||
cache = _DOCKER_BRIDGE_TRUST_CACHE
|
||||
if cache["expires"] > now:
|
||||
return cache["ips"]
|
||||
|
||||
ips: set[str] = set()
|
||||
for hostname in _trusted_bridge_frontend_hostnames():
|
||||
try:
|
||||
_, _, addrs = socket.gethostbyname_ex(hostname)
|
||||
except (OSError, socket.gaierror):
|
||||
continue
|
||||
for addr in addrs:
|
||||
ips.add(addr)
|
||||
|
||||
resolved = frozenset(ips)
|
||||
cache["ips"] = resolved
|
||||
cache["expires"] = now + _DOCKER_BRIDGE_TRUST_TTL
|
||||
return resolved
|
||||
|
||||
|
||||
def _is_docker_bridge_host(host: str) -> bool:
|
||||
"""Return True only when the source IP matches our trusted frontend
|
||||
container hostname(s).
|
||||
|
||||
Previously trusted any 172.16.0.0/12 IP unconditionally. See the
|
||||
block comment above for the security rationale.
|
||||
"""
|
||||
try:
|
||||
ip = ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
return False
|
||||
# Docker Desktop and the default compose bridge normally sit inside
|
||||
# 172.16.0.0/12. Keep this narrower than "any private IP" so a user who
|
||||
# intentionally binds the backend to LAN does not silently trust LAN clients.
|
||||
return ip in ipaddress.ip_network("172.16.0.0/12")
|
||||
# Public IPs are never our frontend container — skip DNS work for them.
|
||||
if not ip.is_private:
|
||||
return False
|
||||
return host in _resolve_trusted_bridge_ips()
|
||||
|
||||
|
||||
def _is_trusted_local_runtime_host(host: str) -> bool:
|
||||
@@ -361,6 +436,8 @@ async def _verify_openclaw_hmac(request: Request) -> bool:
|
||||
# Bind request body: digest the raw bytes so any body tampering
|
||||
# invalidates the signature. Empty/absent bodies hash as sha256(b"").
|
||||
body_bytes = await request.body()
|
||||
# Keep the cached body available for downstream handlers that call request.json().
|
||||
request._body = body_bytes
|
||||
body_digest = _hashlib_mod.sha256(body_bytes).hexdigest()
|
||||
|
||||
# Compute expected signature: HMAC-SHA256(secret, METHOD|path|ts|nonce|body_digest)
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"_comment": [
|
||||
"SPKI (Subject Public Key Info) pin list for stream.aisstream.io.",
|
||||
"",
|
||||
"Issue #258: AISStream's Let's Encrypt cert expired on 2026-05-20 due to an",
|
||||
"upstream renewal-pipeline failure. Disabling TLS verification entirely",
|
||||
"would let any network attacker MITM the AIS WebSocket and inject fake",
|
||||
"ship positions onto the operator's map (same class as #199 GDELT MITM).",
|
||||
"Instead we pin the leaf certificate's public-key SPKI hash: if normal",
|
||||
"TLS validation fails specifically with CERT_HAS_EXPIRED, ais_proxy.js",
|
||||
"re-checks the leaf cert's SPKI against this list. A match means the",
|
||||
"key is still the genuine AISStream key (Let's Encrypt renewals keep the",
|
||||
"same key unless rekey is requested), so we proceed in 'degraded TLS'",
|
||||
"mode. A mismatch means a real MITM attempt and we refuse the connection.",
|
||||
"",
|
||||
"Format: each entry is a SHA-256 hash of the DER-encoded SPKI bytes,",
|
||||
"encoded as standard base64 (matches the format produced by:",
|
||||
" openssl s_client -connect host:443 | \\",
|
||||
" openssl x509 -pubkey -noout | openssl pkey -pubin -outform DER | \\",
|
||||
" openssl dgst -sha256 -binary | openssl base64",
|
||||
").",
|
||||
"",
|
||||
"When AISStream rotates their server key (rare — Let's Encrypt renewals",
|
||||
"default to keeping the same key), capture the new SPKI and add it to",
|
||||
"this list BEFORE removing the old one. That way operators on the old",
|
||||
"code still validate against the previous key during the transition."
|
||||
],
|
||||
"stream.aisstream.io": [
|
||||
"GJ10H0UPgLrO+2d3ZXROR/TXSVFXKUfRC3QEI2ibEg4="
|
||||
]
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"_comment": [
|
||||
"Baked-in SHA-256 digests for known Shadowbroker release archives.",
|
||||
"",
|
||||
"Issue #231: the self-updater previously skipped integrity verification",
|
||||
"entirely whenever the MESH_UPDATE_SHA256 env var was unset (which is the",
|
||||
"default — nothing in the install docs tells operators to set it). That",
|
||||
"made the auto-update a supply-chain RCE on any compromise of the GitHub",
|
||||
"release pipeline.",
|
||||
"",
|
||||
"The fix uses a multi-source verification chain mirroring the Tor bundle",
|
||||
"digest approach in #201:",
|
||||
"",
|
||||
" 1. MESH_UPDATE_SHA256 env var (operator override, preserved)",
|
||||
" 2. SHA256SUMS.txt asset published alongside each release (primary —",
|
||||
" the maintainer's release process already publishes this)",
|
||||
" 3. This baked-in digest list (second line of defense for releases",
|
||||
" missing a SHA256SUMS asset, or when the asset can't be fetched)",
|
||||
" 4. HTTPS-only fallback with a loud warning (preserves auto-update",
|
||||
" flow during transient outages so users don't get stuck)",
|
||||
"",
|
||||
"Mismatch from a source that DID respond is fatal — the update is",
|
||||
"refused and the existing install keeps running. Only the 'no source",
|
||||
"reachable at all' case falls back to HTTPS-only.",
|
||||
"",
|
||||
"Format: each entry is keyed by release tag and maps asset filenames",
|
||||
"to their canonical SHA-256 digest (hex, lowercase). The updater",
|
||||
"compares the locally-computed digest of the downloaded asset against",
|
||||
"the value here.",
|
||||
"",
|
||||
"When the maintainer ships a new release, add its digests here BEFORE",
|
||||
"removing the old ones so operators on the old code still validate",
|
||||
"against the previous entries during the transition."
|
||||
],
|
||||
"v0.9.79": {
|
||||
"ShadowBroker_v0.9.79.zip": "f6877c1d66614525315ea82636ce9f7b41178332c4dbf90d27431a1ea1d9cd47",
|
||||
"ShadowBroker_0.9.79_x64-setup.exe": "f7b676ada45cac7da05868b0a353678c9ee700e3abcf456a7c0c038c36da446f",
|
||||
"ShadowBroker_0.9.79_x64_en-US.msi": "e0713c3cdda184cfbea750bfac0d62a35678fec00847e6476f2cac8e7e42046e"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"_comment": [
|
||||
"Pinned SHA-256 digests for the Tor Expert Bundle archives we know how to install.",
|
||||
"Used as the LAST-RESORT verification source when the upstream .sha256sum file is",
|
||||
"unreachable, MITM'd, or doesn't match what we downloaded. Issue #201.",
|
||||
"",
|
||||
"Each entry is keyed by the archive URL (so multiple platforms / versions",
|
||||
"can share this one file) and contains the canonical SHA-256 we trust.",
|
||||
"",
|
||||
"When the project tests a new Tor release, add its digest here in the same",
|
||||
"PR that bumps _TOR_EXPERT_BUNDLE_URLS. Old entries are kept indefinitely so",
|
||||
"users on older versions keep working — we only ever ADD here, never remove."
|
||||
],
|
||||
"https://dist.torproject.org/torbrowser/15.0.11/tor-expert-bundle-windows-x86_64-15.0.11.tar.gz": "PLACEHOLDER_REPLACE_BEFORE_RELEASE",
|
||||
"https://dist.torproject.org/torbrowser/15.0.8/tor-expert-bundle-windows-x86_64-15.0.8.tar.gz": "PLACEHOLDER_REPLACE_BEFORE_RELEASE"
|
||||
}
|
||||
@@ -14,4 +14,9 @@ if [ -d /app/image-data ]; then
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -z "${PRIVACY_CORE_ALLOWED_SHA256:-}" ] && [ -f /app/libprivacy_core.so ]; then
|
||||
PRIVACY_CORE_ALLOWED_SHA256="$(sha256sum /app/libprivacy_core.so | awk '{print $1}')"
|
||||
export PRIVACY_CORE_ALLOWED_SHA256
|
||||
fi
|
||||
|
||||
exec "$@"
|
||||
|
||||
+145
-25
@@ -14,7 +14,7 @@ from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from json import JSONDecodeError
|
||||
|
||||
APP_VERSION = "0.9.75"
|
||||
APP_VERSION = "0.9.79"
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -1386,7 +1386,12 @@ def _peer_sync_response(peer_url: str, body: dict[str, Any]) -> dict[str, Any]:
|
||||
if _infonet_private_transport_required() and not _is_private_infonet_transport(transport):
|
||||
raise RuntimeError(_infonet_private_transport_error())
|
||||
|
||||
timeout = int(get_settings().MESH_RELAY_PUSH_TIMEOUT_S or 10)
|
||||
settings = get_settings()
|
||||
timeout = int(
|
||||
getattr(settings, "MESH_SYNC_TIMEOUT_S", 0)
|
||||
or getattr(settings, "MESH_RELAY_PUSH_TIMEOUT_S", 0)
|
||||
or 10
|
||||
)
|
||||
kwargs: dict[str, Any] = {
|
||||
"json": body,
|
||||
"timeout": timeout,
|
||||
@@ -1509,6 +1514,8 @@ def _run_public_sync_cycle() -> SyncWorkerState:
|
||||
|
||||
records = _filter_infonet_sync_records(store.records())
|
||||
peers = eligible_sync_peers(records, now=time.time())
|
||||
max_peers = max(1, int(getattr(get_settings(), "MESH_SYNC_MAX_PEERS_PER_CYCLE", 0) or 3))
|
||||
peers = peers[:max_peers]
|
||||
with _NODE_RUNTIME_LOCK:
|
||||
current_state = get_sync_state()
|
||||
if not peers:
|
||||
@@ -1571,14 +1578,25 @@ def _run_public_sync_cycle() -> SyncWorkerState:
|
||||
return updated
|
||||
|
||||
last_error = error
|
||||
settings = get_settings()
|
||||
is_seed_peer = str(getattr(record, "role", "") or "").strip().lower() == "seed"
|
||||
cooldown_s = int(getattr(settings, "MESH_RELAY_FAILURE_COOLDOWN_S", 120) or 120)
|
||||
if is_seed_peer:
|
||||
cooldown_s = int(
|
||||
getattr(settings, "MESH_BOOTSTRAP_SEED_FAILURE_COOLDOWN_S", cooldown_s)
|
||||
or cooldown_s
|
||||
)
|
||||
store.mark_failure(
|
||||
record.peer_url,
|
||||
"sync",
|
||||
error=error,
|
||||
cooldown_s=int(get_settings().MESH_RELAY_FAILURE_COOLDOWN_S or 120),
|
||||
cooldown_s=cooldown_s,
|
||||
now=time.time(),
|
||||
)
|
||||
store.save()
|
||||
failure_backoff_s = int(settings.MESH_SYNC_FAILURE_BACKOFF_S or 60)
|
||||
if is_seed_peer:
|
||||
failure_backoff_s = min(failure_backoff_s, max(1, cooldown_s))
|
||||
updated = finish_sync(
|
||||
started,
|
||||
ok=False,
|
||||
@@ -1588,7 +1606,7 @@ def _run_public_sync_cycle() -> SyncWorkerState:
|
||||
fork_detected=forked,
|
||||
now=time.time(),
|
||||
interval_s=int(get_settings().MESH_SYNC_INTERVAL_S or 300),
|
||||
failure_backoff_s=int(get_settings().MESH_SYNC_FAILURE_BACKOFF_S or 60),
|
||||
failure_backoff_s=failure_backoff_s,
|
||||
)
|
||||
with _NODE_RUNTIME_LOCK:
|
||||
set_sync_state(updated)
|
||||
@@ -3043,6 +3061,17 @@ def _request_private_surface_warmup(*, path: str, method: str, current_tier: str
|
||||
)
|
||||
|
||||
|
||||
def _is_invite_scoped_prekey_bundle_lookup(request: Request, path: str) -> bool:
|
||||
if request.method.upper() != "GET" or str(path or "").strip() != "/api/mesh/dm/prekey-bundle":
|
||||
return False
|
||||
try:
|
||||
lookup_token = str(request.query_params.get("lookup_token", "") or "").strip()
|
||||
agent_id = str(request.query_params.get("agent_id", "") or "").strip()
|
||||
except Exception:
|
||||
return False
|
||||
return bool(lookup_token) and not agent_id
|
||||
|
||||
|
||||
def _resume_private_delivery_background_work(*, current_tier: str, reason: str) -> None:
|
||||
pending_items = private_delivery_outbox.pending_items()
|
||||
if not pending_items:
|
||||
@@ -3061,6 +3090,24 @@ def _resume_private_delivery_background_work(*, current_tier: str, reason: str)
|
||||
)
|
||||
|
||||
|
||||
def _is_public_meshtastic_lane_path(path: str, method: str) -> bool:
|
||||
"""Routes for the public Meshtastic MQTT lane.
|
||||
|
||||
These are intentionally outside the Wormhole/Infonet private transport
|
||||
lifecycle. Polling public MeshChat must not wake or re-enable Wormhole.
|
||||
"""
|
||||
normalized_path = str(path or "").strip()
|
||||
method_name = str(method or "").upper()
|
||||
if method_name == "POST" and normalized_path == "/api/mesh/meshtastic/send":
|
||||
return True
|
||||
if method_name == "GET" and normalized_path in {
|
||||
"/api/mesh/messages",
|
||||
"/api/mesh/channels",
|
||||
}:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _upgrade_invite_scoped_contact_preferences_background() -> dict[str, Any]:
|
||||
try:
|
||||
from services.mesh.mesh_wormhole_contacts import upgrade_invite_scoped_contact_preferences
|
||||
@@ -3092,7 +3139,11 @@ def _refresh_lookup_handle_rotation_background(*, reason: str) -> dict[str, Any]
|
||||
@app.middleware("http")
|
||||
async def enforce_high_privacy_mesh(request: Request, call_next):
|
||||
path = request.url.path
|
||||
if path.startswith("/api/mesh") or path.startswith("/api/wormhole/gate/") or path.startswith("/api/wormhole/dm/"):
|
||||
private_mesh_path = path.startswith("/api/mesh") and not _is_public_meshtastic_lane_path(
|
||||
path,
|
||||
request.method,
|
||||
)
|
||||
if private_mesh_path or path.startswith("/api/wormhole/gate/") or path.startswith("/api/wormhole/dm/"):
|
||||
request.state._private_lane_started_at = time.perf_counter()
|
||||
current_tier = "public_degraded"
|
||||
try:
|
||||
@@ -3151,6 +3202,17 @@ async def enforce_high_privacy_mesh(request: Request, call_next):
|
||||
# transport has not finished coming up yet.
|
||||
request.state._private_control_transport_pending = current_tier == "public_degraded"
|
||||
request.state._private_lane_current_tier = current_tier
|
||||
elif _is_invite_scoped_prekey_bundle_lookup(request, path):
|
||||
# A copied DM address carries a high-entropy invite lookup
|
||||
# handle. Returning the public prekey bundle for that
|
||||
# handle is the bootstrap step that lets first contact get
|
||||
# saved; blocking it behind the full private lane creates a
|
||||
# circular warm-up failure. Stable agent_id lookup still
|
||||
# follows the normal transport-tier policy.
|
||||
request.state._invite_prekey_lookup_transport_pending = (
|
||||
current_tier == "public_degraded"
|
||||
)
|
||||
request.state._private_lane_current_tier = current_tier
|
||||
else:
|
||||
# Tor-style: instead of failing, keep trying in the
|
||||
# background and return an ok:True "preparing" response
|
||||
@@ -3193,7 +3255,7 @@ async def enforce_high_privacy_mesh(request: Request, call_next):
|
||||
# Don't block the request on the upgrade — the transport
|
||||
# manager will converge in the background.
|
||||
if (
|
||||
path.startswith("/api/mesh")
|
||||
private_mesh_path
|
||||
and str(data.get("privacy_profile", "default")).lower() == "high"
|
||||
and not bool(data.get("enabled"))
|
||||
):
|
||||
@@ -3283,7 +3345,7 @@ async def force_refresh(request: Request):
|
||||
return {"status": "refreshing in background"}
|
||||
|
||||
|
||||
@app.post("/api/ais/feed")
|
||||
@app.post("/api/ais/feed", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("60/minute")
|
||||
async def ais_feed(request: Request):
|
||||
"""Accept AIS-catcher HTTP JSON feed (POST decoded AIS messages)."""
|
||||
@@ -3378,7 +3440,7 @@ class LayerUpdate(BaseModel):
|
||||
layers: dict[str, bool]
|
||||
|
||||
|
||||
@app.post("/api/layers")
|
||||
@app.post("/api/layers", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def update_layers(update: LayerUpdate, request: Request):
|
||||
"""Receive frontend layer toggle state. Starts/stops streams accordingly."""
|
||||
@@ -3426,8 +3488,16 @@ async def update_layers(update: LayerUpdate, request: Request):
|
||||
from services.sigint_bridge import sigint_grid
|
||||
|
||||
if old_mesh and not new_mesh:
|
||||
sigint_grid.mesh.stop()
|
||||
logger.info("Meshtastic MQTT bridge stopped (layer disabled)")
|
||||
try:
|
||||
from services.meshtastic_mqtt_settings import mqtt_bridge_enabled
|
||||
keep_chat_running = mqtt_bridge_enabled()
|
||||
except Exception:
|
||||
keep_chat_running = False
|
||||
if keep_chat_running:
|
||||
logger.info("Meshtastic map layer disabled; MQTT bridge kept running for MeshChat")
|
||||
else:
|
||||
sigint_grid.mesh.stop()
|
||||
logger.info("Meshtastic MQTT bridge stopped (layer disabled)")
|
||||
elif not old_mesh and new_mesh:
|
||||
# Respect the global MESH_MQTT_ENABLED gate even when the UI layer is
|
||||
# toggled on. The layer toggle should not bypass the opt-in flag that
|
||||
@@ -4361,9 +4431,11 @@ async def mesh_send(request: Request):
|
||||
any_ok = any(r.ok for r in results)
|
||||
|
||||
# ─── Mirror to Meshtastic bridge feed ────────────────────────
|
||||
# The MQTT broker won't echo our own publishes back to our subscriber,
|
||||
# so inject successfully-sent messages into the bridge's deque directly.
|
||||
if any_ok and envelope.routed_via == "meshtastic":
|
||||
# The MQTT broker won't echo our own publishes back to our subscriber, so
|
||||
# inject successfully-sent channel broadcasts into the bridge directly.
|
||||
# Node-targeted packets must not appear in the public channel feed.
|
||||
is_direct_destination = MeshtasticTransport._parse_node_id(destination) is not None
|
||||
if any_ok and envelope.routed_via == "meshtastic" and not is_direct_destination:
|
||||
try:
|
||||
from services.sigint_bridge import sigint_grid
|
||||
|
||||
@@ -4371,16 +4443,22 @@ async def mesh_send(request: Request):
|
||||
if bridge:
|
||||
from datetime import datetime
|
||||
|
||||
bridge.messages.appendleft(
|
||||
append_text = getattr(bridge, "append_text_message", None)
|
||||
message_record = (
|
||||
{
|
||||
"from": MeshtasticTransport.mesh_address_for_sender(node_id),
|
||||
"to": destination if MeshtasticTransport._parse_node_id(destination) is not None else "broadcast",
|
||||
"to": "broadcast",
|
||||
"text": message,
|
||||
"region": credentials.get("mesh_region", "US"),
|
||||
"root": credentials.get("mesh_region", "US"),
|
||||
"channel": body.get("channel", "LongFast"),
|
||||
"timestamp": datetime.utcnow().isoformat() + "Z",
|
||||
}
|
||||
)
|
||||
if callable(append_text):
|
||||
append_text(message_record)
|
||||
else:
|
||||
bridge.messages.appendleft(message_record)
|
||||
except Exception:
|
||||
pass # Non-critical
|
||||
|
||||
@@ -4390,6 +4468,8 @@ async def mesh_send(request: Request):
|
||||
"event_id": "",
|
||||
"routed_via": envelope.routed_via,
|
||||
"route_reason": envelope.route_reason,
|
||||
"direct": is_direct_destination,
|
||||
"channel_echo": not is_direct_destination,
|
||||
"results": [r.to_dict() for r in results],
|
||||
}
|
||||
|
||||
@@ -4488,6 +4568,7 @@ async def mesh_messages(
|
||||
root: str = "",
|
||||
channel: str = "",
|
||||
limit: int = 30,
|
||||
include_direct: bool = False,
|
||||
):
|
||||
"""Get recent Meshtastic text messages from the MQTT bridge."""
|
||||
from services.sigint_bridge import sigint_grid
|
||||
@@ -4509,6 +4590,12 @@ async def mesh_messages(
|
||||
msgs = [m for m in msgs if m.get("root", "").upper() == root_filter]
|
||||
if channel:
|
||||
msgs = [m for m in msgs if m.get("channel", "").lower() == channel.lower()]
|
||||
if not include_direct:
|
||||
msgs = [
|
||||
m
|
||||
for m in msgs
|
||||
if str(m.get("to") or "broadcast").strip().lower() in {"", "broadcast", "^all"}
|
||||
]
|
||||
return msgs[: min(limit, 100)]
|
||||
|
||||
|
||||
@@ -8789,6 +8876,16 @@ export_wormhole_dm_invite = getattr(
|
||||
"export_wormhole_dm_invite",
|
||||
_wormhole_identity_unavailable,
|
||||
)
|
||||
list_prekey_lookup_handle_records_for_ui = getattr(
|
||||
_mesh_wormhole_identity,
|
||||
"list_prekey_lookup_handle_records_for_ui",
|
||||
_wormhole_identity_unavailable,
|
||||
)
|
||||
revoke_prekey_lookup_handle = getattr(
|
||||
_mesh_wormhole_identity,
|
||||
"revoke_prekey_lookup_handle",
|
||||
_wormhole_identity_unavailable,
|
||||
)
|
||||
import_wormhole_dm_invite = getattr(
|
||||
_mesh_wormhole_identity,
|
||||
"import_wormhole_dm_invite",
|
||||
@@ -8935,6 +9032,13 @@ async def api_get_node_settings(request: Request):
|
||||
@limiter.limit("10/minute")
|
||||
async def api_set_node_settings(request: Request, body: NodeSettingsUpdate):
|
||||
_refresh_node_peer_store()
|
||||
if bool(body.enabled):
|
||||
try:
|
||||
from services.transport_lane_isolation import disable_public_mesh_lane
|
||||
|
||||
disable_public_mesh_lane(reason="private_node_enabled")
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to disable public Mesh while enabling private node: %s", exc)
|
||||
result = _set_participant_node_enabled(bool(body.enabled))
|
||||
if bool(body.enabled):
|
||||
_kick_public_sync_background("operator_enable")
|
||||
@@ -9659,7 +9763,7 @@ async def api_get_wormhole_status(request: Request):
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/wormhole/join", dependencies=[Depends(require_local_operator)])
|
||||
@app.post("/api/wormhole/join")
|
||||
@limiter.limit("10/minute")
|
||||
async def api_wormhole_join(request: Request):
|
||||
existing = read_wormhole_settings()
|
||||
@@ -9713,7 +9817,7 @@ async def api_wormhole_join(request: Request):
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/wormhole/leave", dependencies=[Depends(require_local_operator)])
|
||||
@app.post("/api/wormhole/leave")
|
||||
@limiter.limit("10/minute")
|
||||
async def api_wormhole_leave(request: Request):
|
||||
updated = write_wormhole_settings(enabled=False)
|
||||
@@ -9776,11 +9880,27 @@ async def api_wormhole_dm_identity(request: Request):
|
||||
|
||||
@app.get("/api/wormhole/dm/invite", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_dm_invite(request: Request):
|
||||
return export_wormhole_dm_invite()
|
||||
async def api_wormhole_dm_invite(
|
||||
request: Request,
|
||||
label: str = Query("", max_length=96),
|
||||
expires_in_s: int = Query(0, ge=0, le=2_592_000),
|
||||
):
|
||||
return export_wormhole_dm_invite(label=label, expires_in_s=expires_in_s)
|
||||
|
||||
|
||||
@app.post("/api/wormhole/dm/invite/import", dependencies=[Depends(require_admin)])
|
||||
@app.get("/api/wormhole/dm/invite/handles", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_dm_invite_handles(request: Request):
|
||||
return list_prekey_lookup_handle_records_for_ui()
|
||||
|
||||
|
||||
@app.delete("/api/wormhole/dm/invite/handles/{handle}", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_dm_invite_handle_revoke(request: Request, handle: str):
|
||||
return revoke_prekey_lookup_handle(handle)
|
||||
|
||||
|
||||
@app.post("/api/wormhole/dm/invite/import", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_dm_invite_import(request: Request, body: WormholeDmInviteImportRequest):
|
||||
return import_wormhole_dm_invite(
|
||||
@@ -10527,19 +10647,19 @@ async def api_wormhole_gate_leave(request: Request, body: WormholeGateRequest):
|
||||
return leave_gate(str(body.gate_id or ""))
|
||||
|
||||
|
||||
@app.get("/api/wormhole/gate/{gate_id}/identity", dependencies=[Depends(require_local_operator)])
|
||||
@app.get("/api/wormhole/gate/{gate_id}/identity")
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_gate_identity(request: Request, gate_id: str):
|
||||
return get_active_gate_identity(gate_id)
|
||||
|
||||
|
||||
@app.get("/api/wormhole/gate/{gate_id}/personas", dependencies=[Depends(require_local_operator)])
|
||||
@app.get("/api/wormhole/gate/{gate_id}/personas")
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_gate_personas(request: Request, gate_id: str):
|
||||
return list_gate_personas(gate_id)
|
||||
|
||||
|
||||
@app.get("/api/wormhole/gate/{gate_id}/key", dependencies=[Depends(require_local_operator)])
|
||||
@app.get("/api/wormhole/gate/{gate_id}/key")
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_gate_key_status(request: Request, gate_id: str):
|
||||
exposure = metadata_exposure_for_request(request, authenticated=True)
|
||||
@@ -10722,7 +10842,7 @@ async def api_wormhole_gate_message_sign_encrypted(
|
||||
return signed
|
||||
|
||||
|
||||
@app.post("/api/wormhole/gate/message/post-encrypted", dependencies=[Depends(require_local_operator)])
|
||||
@app.post("/api/wormhole/gate/message/post-encrypted")
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_gate_message_post_encrypted(
|
||||
request: Request,
|
||||
@@ -11455,7 +11575,7 @@ async def api_wormhole_health(request: Request):
|
||||
return _redact_wormhole_status(full_state, authenticated=ok)
|
||||
|
||||
|
||||
@app.post("/api/wormhole/connect", dependencies=[Depends(require_admin)])
|
||||
@app.post("/api/wormhole/connect", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("10/minute")
|
||||
async def api_wormhole_connect(request: Request):
|
||||
settings = read_wormhole_settings()
|
||||
|
||||
@@ -7,7 +7,7 @@ py-modules = []
|
||||
|
||||
[project]
|
||||
name = "backend"
|
||||
version = "0.9.75"
|
||||
version = "0.9.79"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"apscheduler==3.10.3",
|
||||
@@ -15,6 +15,7 @@ dependencies = [
|
||||
"cachetools==5.5.2",
|
||||
"cloudscraper==1.2.71",
|
||||
"cryptography>=41.0.0",
|
||||
"defusedxml>=0.7.1",
|
||||
"fastapi==0.115.12",
|
||||
"feedparser==6.0.10",
|
||||
"httpx==0.28.1",
|
||||
@@ -43,7 +44,7 @@ dev = ["pytest>=8.3.4", "pytest-asyncio==0.25.0", "ruff>=0.9.0", "black>=24.0.0"
|
||||
|
||||
[tool.ruff.lint]
|
||||
# The current backend carries historical style debt in large legacy modules.
|
||||
# Keep CI focused on actionable correctness checks for the v0.9.75 release.
|
||||
# Keep CI focused on actionable correctness checks for the v0.9.79 release.
|
||||
ignore = ["E401", "E402", "E701", "E731", "E741", "F401", "F402", "F541", "F811", "F841"]
|
||||
|
||||
[tool.black]
|
||||
|
||||
@@ -132,6 +132,13 @@ async def api_get_node_settings(request: Request):
|
||||
@limiter.limit("10/minute")
|
||||
async def api_set_node_settings(request: Request, body: NodeSettingsUpdate):
|
||||
_refresh_node_peer_store()
|
||||
if bool(body.enabled):
|
||||
try:
|
||||
from services.transport_lane_isolation import disable_public_mesh_lane
|
||||
|
||||
disable_public_mesh_lane(reason="private_node_enabled")
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to disable public Mesh while enabling private node: %s", exc)
|
||||
result = _set_participant_node_enabled(bool(body.enabled))
|
||||
if bool(body.enabled):
|
||||
try:
|
||||
@@ -174,17 +181,22 @@ async def api_set_meshtastic_mqtt_settings(request: Request, body: MeshtasticMqt
|
||||
|
||||
enabled_requested = updates.get("enabled")
|
||||
settings = write_meshtastic_mqtt_settings(**updates)
|
||||
if isinstance(enabled_requested, bool):
|
||||
logger.info("Meshtastic MQTT settings update: enabled=%s", enabled_requested)
|
||||
|
||||
if enabled_requested is True:
|
||||
# Public MQTT and Wormhole are intentionally mutually exclusive lanes.
|
||||
try:
|
||||
from services.node_settings import write_node_settings
|
||||
from services.wormhole_settings import write_wormhole_settings
|
||||
from services.wormhole_supervisor import disconnect_wormhole
|
||||
|
||||
write_wormhole_settings(enabled=False)
|
||||
disconnect_wormhole(reason="public_mesh_enabled")
|
||||
write_node_settings(enabled=False)
|
||||
_set_participant_node_enabled(False)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to disable Wormhole while enabling public mesh: %s", exc)
|
||||
logger.warning("Failed to disable private mesh lane while enabling public mesh: %s", exc)
|
||||
|
||||
if bool(settings.get("enabled")):
|
||||
if sigint_grid.mesh.is_running():
|
||||
@@ -357,8 +369,8 @@ async def api_reset_all_agent_credentials(request: Request):
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"new_hmac_secret": new_secret,
|
||||
"detail": "All agent credentials have been reset. Reconfigure your agent with the new credentials.",
|
||||
"hmac_regenerated": True,
|
||||
"detail": "All agent credentials have been reset. Use the agent connection screen to generate or reveal replacement credentials.",
|
||||
**results,
|
||||
}
|
||||
|
||||
|
||||
@@ -379,14 +379,13 @@ async def api_refresh_layer_feed(request: Request, layer_id: str):
|
||||
# Agent Actions endpoint — frontend polls this for UI commands from the agent
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/api/ai/agent-actions")
|
||||
@router.get("/api/ai/agent-actions", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("120/minute")
|
||||
async def get_agent_actions(request: Request):
|
||||
"""Frontend polls for pending agent display actions (destructive read).
|
||||
|
||||
No auth required — this only contains display directives (show image,
|
||||
fly to location), not sensitive data. The agent authenticates when
|
||||
pushing actions through the command channel.
|
||||
Local operator access is required because polling destructively drains
|
||||
the shared operator action queue.
|
||||
"""
|
||||
actions = pop_agent_actions()
|
||||
return {"ok": True, "actions": actions}
|
||||
@@ -1585,7 +1584,7 @@ async def agent_tool_manifest(request: Request):
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"version": "0.9.75",
|
||||
"version": "0.9.79",
|
||||
"access_tier": access_tier,
|
||||
"available_commands": available_commands,
|
||||
"transport": {
|
||||
@@ -2221,7 +2220,7 @@ async def api_capabilities(request: Request):
|
||||
access_tier = str(get_settings().OPENCLAW_ACCESS_TIER or "restricted").strip().lower()
|
||||
return {
|
||||
"ok": True,
|
||||
"version": "0.9.75",
|
||||
"version": "0.9.79",
|
||||
"auth": {
|
||||
"method": "HMAC-SHA256",
|
||||
"headers": ["X-SB-Timestamp", "X-SB-Nonce", "X-SB-Signature"],
|
||||
|
||||
+58
-1
@@ -191,11 +191,68 @@ def _cctv_response_headers(resp, cache_seconds: int, include_length: bool = True
|
||||
return headers
|
||||
|
||||
|
||||
# Maximum number of redirects we'll follow on the CCTV upstream. Each hop is
|
||||
# re-validated against _cctv_host_allowed() before continuing, so this caps
|
||||
# the redirect-chain SSRF blast radius.
|
||||
_CCTV_MAX_REDIRECTS = 5
|
||||
|
||||
|
||||
def _fetch_cctv_upstream_response(request: Request, target_url: str, profile: _CCTVProxyProfile):
|
||||
"""Fetch an upstream CCTV URL, following redirects manually with host re-validation.
|
||||
|
||||
Why manual redirect following:
|
||||
The original code used ``allow_redirects=True``, which only validated
|
||||
the initial caller-supplied URL host against the allowlist. An attacker
|
||||
could submit an allowed host that 302-redirected to an internal address
|
||||
(e.g. ``http://localhost:8000/api/...`` or a private RFC1918 range),
|
||||
and the backend would dutifully follow and proxy the response — a
|
||||
classic open-redirect-to-SSRF chain.
|
||||
|
||||
With this loop, we re-run ``_cctv_host_allowed()`` on every hop's
|
||||
``Location`` header. A redirect to a host that isn't on the allowlist
|
||||
is rejected with 502 rather than silently followed.
|
||||
"""
|
||||
import requests as _req
|
||||
from urllib.parse import urlparse, urljoin
|
||||
|
||||
headers = _cctv_upstream_headers(request, profile)
|
||||
current_url = target_url
|
||||
hops = 0
|
||||
try:
|
||||
resp = _req.get(target_url, timeout=profile.timeout, stream=True, allow_redirects=True, headers=headers)
|
||||
while True:
|
||||
resp = _req.get(
|
||||
current_url,
|
||||
timeout=profile.timeout,
|
||||
stream=True,
|
||||
allow_redirects=False,
|
||||
headers=headers,
|
||||
)
|
||||
# Redirect handling — re-validate the next-hop host before following.
|
||||
if resp.is_redirect or resp.status_code in (301, 302, 303, 307, 308):
|
||||
location = resp.headers.get("Location", "")
|
||||
resp.close()
|
||||
if hops >= _CCTV_MAX_REDIRECTS:
|
||||
logger.warning(
|
||||
"CCTV upstream redirect chain exceeded limit [%s] %s",
|
||||
profile.name, target_url,
|
||||
)
|
||||
raise HTTPException(status_code=502, detail="Upstream redirect chain too long")
|
||||
if not location:
|
||||
raise HTTPException(status_code=502, detail="Upstream redirect missing Location")
|
||||
next_url = urljoin(current_url, location)
|
||||
next_parsed = urlparse(next_url)
|
||||
if next_parsed.scheme not in ("http", "https"):
|
||||
raise HTTPException(status_code=502, detail="Upstream redirect to non-HTTP scheme")
|
||||
if not _cctv_host_allowed(next_parsed.hostname):
|
||||
logger.warning(
|
||||
"CCTV upstream redirect to disallowed host [%s] %s -> %s",
|
||||
profile.name, current_url, next_url,
|
||||
)
|
||||
raise HTTPException(status_code=502, detail="Upstream redirect to disallowed host")
|
||||
current_url = next_url
|
||||
hops += 1
|
||||
continue
|
||||
break
|
||||
except _req.exceptions.Timeout as exc:
|
||||
logger.warning("CCTV upstream timeout [%s] %s", profile.name, target_url)
|
||||
raise HTTPException(status_code=504, detail="Upstream timeout") from exc
|
||||
|
||||
+40
-5
@@ -266,7 +266,7 @@ async def force_refresh(request: Request):
|
||||
return {"status": "refreshing in background"}
|
||||
|
||||
|
||||
@router.post("/api/ais/feed")
|
||||
@router.post("/api/ais/feed", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("60/minute")
|
||||
async def ais_feed(request: Request):
|
||||
"""Accept AIS-catcher HTTP JSON feed (POST decoded AIS messages)."""
|
||||
@@ -304,7 +304,7 @@ async def update_viewport(vp: ViewportUpdate, request: Request): # noqa: ARG001
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.post("/api/layers")
|
||||
@router.post("/api/layers", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def update_layers(update: LayerUpdate, request: Request):
|
||||
"""Receive frontend layer toggle state. Starts/stops streams accordingly."""
|
||||
@@ -335,8 +335,16 @@ async def update_layers(update: LayerUpdate, request: Request):
|
||||
logger.info("AIS stream started (ship layer enabled)")
|
||||
from services.sigint_bridge import sigint_grid
|
||||
if old_mesh and not new_mesh:
|
||||
sigint_grid.mesh.stop()
|
||||
logger.info("Meshtastic MQTT bridge stopped (layer disabled)")
|
||||
try:
|
||||
from services.meshtastic_mqtt_settings import mqtt_bridge_enabled
|
||||
keep_chat_running = mqtt_bridge_enabled()
|
||||
except Exception:
|
||||
keep_chat_running = False
|
||||
if keep_chat_running:
|
||||
logger.info("Meshtastic map layer disabled; MQTT bridge kept running for MeshChat")
|
||||
else:
|
||||
sigint_grid.mesh.stop()
|
||||
logger.info("Meshtastic MQTT bridge stopped (layer disabled)")
|
||||
elif not old_mesh and new_mesh:
|
||||
try:
|
||||
from services.meshtastic_mqtt_settings import mqtt_bridge_enabled
|
||||
@@ -603,6 +611,23 @@ class OverflightRequest(BaseModel):
|
||||
hours: int = 24
|
||||
|
||||
|
||||
# Issue #202: compute_overflights() is O(catalog_size × timesteps), where
|
||||
# timesteps grows linearly with `hours`. An unbounded `hours` value is a
|
||||
# trivial CPU-exhaustion vector. We clamp silently rather than raising 422 —
|
||||
# the response shape is unchanged, callers asking for too many hours just
|
||||
# get a shorter window, which is friendlier than a hostile error.
|
||||
#
|
||||
# Override via OVERFLIGHTS_MAX_HOURS env var if you legitimately need a
|
||||
# longer window (e.g. a planning use case that wants a full week).
|
||||
def _overflight_max_hours() -> int:
|
||||
import os as _os
|
||||
try:
|
||||
raw = int(str(_os.environ.get("OVERFLIGHTS_MAX_HOURS", "72")).strip())
|
||||
except (TypeError, ValueError):
|
||||
raw = 72
|
||||
return max(1, raw)
|
||||
|
||||
|
||||
@router.post("/api/satellites/overflights")
|
||||
@limiter.limit("10/minute")
|
||||
async def satellite_overflights(request: Request, body: OverflightRequest):
|
||||
@@ -611,5 +636,15 @@ async def satellite_overflights(request: Request, body: OverflightRequest):
|
||||
if not gp_data:
|
||||
return JSONResponse({"total": 0, "by_mission": {}, "satellites": [], "error": "No GP data cached yet"})
|
||||
bbox = {"s": body.s, "w": body.w, "n": body.n, "e": body.e}
|
||||
result = compute_overflights(gp_data, bbox, hours=body.hours)
|
||||
|
||||
# Silent clamp — see comment on _overflight_max_hours().
|
||||
requested_hours = max(1, int(body.hours or 0))
|
||||
effective_hours = min(requested_hours, _overflight_max_hours())
|
||||
|
||||
result = compute_overflights(gp_data, bbox, hours=effective_hours)
|
||||
# If we clamped, surface the effective window in the response so the
|
||||
# caller can detect it if they care, without it being an error.
|
||||
if isinstance(result, dict) and effective_hours != requested_hours:
|
||||
result.setdefault("requested_hours", requested_hours)
|
||||
result.setdefault("effective_hours", effective_hours)
|
||||
return JSONResponse(result)
|
||||
|
||||
@@ -8,7 +8,7 @@ from services.data_fetcher import get_latest_data
|
||||
from services.schemas import HealthResponse
|
||||
import os
|
||||
|
||||
APP_VERSION = os.environ.get("_HEALTH_APP_VERSION", "0.9.75")
|
||||
APP_VERSION = os.environ.get("_HEALTH_APP_VERSION", "0.9.79")
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -54,6 +54,22 @@ async def health_check(request: Request):
|
||||
top_status = "error"
|
||||
elif slo_summary.get("yellow", 0) > 0:
|
||||
top_status = "degraded"
|
||||
|
||||
# Issue #258: surface AIS proxy degraded TLS state so operators can see
|
||||
# when the SPKI-pinned fallback is in effect. The data plane keeps
|
||||
# flowing (this is by design — see ais_proxy.js comments) but observers
|
||||
# who care about MITM-protection posture deserve a visible signal.
|
||||
ais_status: dict = {}
|
||||
try:
|
||||
from services.ais_stream import ais_proxy_status
|
||||
ais_status = ais_proxy_status() or {}
|
||||
except Exception:
|
||||
ais_status = {}
|
||||
if ais_status.get("degraded_tls") and top_status == "ok":
|
||||
# Don't override a worse top-level status if SLOs already failed,
|
||||
# but escalate ok -> degraded so the field surfaces in dashboards.
|
||||
top_status = "degraded"
|
||||
|
||||
return {
|
||||
"status": top_status,
|
||||
"version": _get_app_version(),
|
||||
@@ -76,6 +92,7 @@ async def health_check(request: Request):
|
||||
"uptime_seconds": round(_time_mod.time() - _get_start_time()),
|
||||
"slo": slo_statuses,
|
||||
"slo_summary": slo_summary,
|
||||
"ais_proxy": ais_status,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -223,11 +223,21 @@ async def oracle_markets_more(request: Request, category: str = "NEWS", offset:
|
||||
"has_more": offset + limit < len(cat_markets), "total": len(cat_markets)}
|
||||
|
||||
|
||||
@router.post("/api/mesh/oracle/resolve")
|
||||
@router.post(
|
||||
"/api/mesh/oracle/resolve",
|
||||
dependencies=[Depends(require_admin)],
|
||||
)
|
||||
@limiter.limit("5/minute")
|
||||
@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
|
||||
async def oracle_resolve(request: Request):
|
||||
"""Resolve a prediction market."""
|
||||
"""Resolve a prediction market.
|
||||
|
||||
Issue #240 (tg12): requires admin authentication. The
|
||||
``mesh_write_exempt`` decorator below is **metadata only** — it tags
|
||||
the route as not requiring a mesh signed-write envelope, it does
|
||||
NOT itself enforce caller authorization. The ``Depends(require_admin)``
|
||||
on the route decorator is what actually gates access.
|
||||
"""
|
||||
from services.mesh.mesh_oracle import oracle_ledger
|
||||
body = await request.json()
|
||||
market_title = body.get("market_title", "")
|
||||
@@ -327,11 +337,18 @@ async def oracle_predictions(request: Request, node_id: str = ""):
|
||||
active_predictions, authenticated=_scoped_view_authenticated(request, "mesh.audit"))
|
||||
|
||||
|
||||
@router.post("/api/mesh/oracle/resolve-stakes")
|
||||
@router.post(
|
||||
"/api/mesh/oracle/resolve-stakes",
|
||||
dependencies=[Depends(require_admin)],
|
||||
)
|
||||
@limiter.limit("5/minute")
|
||||
@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)
|
||||
async def oracle_resolve_stakes(request: Request):
|
||||
"""Resolve all expired stake contests."""
|
||||
"""Resolve all expired stake contests.
|
||||
|
||||
Issue #241 (tg12): requires admin authentication. See the note on
|
||||
``oracle_resolve`` above — ``mesh_write_exempt`` is metadata only.
|
||||
"""
|
||||
from services.mesh.mesh_oracle import oracle_ledger
|
||||
resolutions = oracle_ledger.resolve_expired_stakes()
|
||||
return {"ok": True, "resolutions": resolutions, "count": len(resolutions)}
|
||||
|
||||
@@ -721,9 +721,11 @@ async def mesh_send(request: Request):
|
||||
any_ok = any(r.ok for r in results)
|
||||
|
||||
# ─── Mirror to Meshtastic bridge feed ────────────────────────
|
||||
# The MQTT broker won't echo our own publishes back to our subscriber,
|
||||
# so inject successfully-sent messages into the bridge's deque directly.
|
||||
if any_ok and envelope.routed_via == "meshtastic":
|
||||
# The MQTT broker won't echo our own publishes back to our subscriber, so
|
||||
# inject successfully-sent channel broadcasts into the bridge directly.
|
||||
# Node-targeted packets must not appear in the public channel feed.
|
||||
is_direct_destination = MeshtasticTransport._parse_node_id(destination) is not None
|
||||
if any_ok and envelope.routed_via == "meshtastic" and not is_direct_destination:
|
||||
try:
|
||||
from services.sigint_bridge import sigint_grid
|
||||
|
||||
@@ -734,7 +736,7 @@ async def mesh_send(request: Request):
|
||||
bridge.messages.appendleft(
|
||||
{
|
||||
"from": MeshtasticTransport.mesh_address_for_sender(node_id),
|
||||
"to": destination if MeshtasticTransport._parse_node_id(destination) is not None else "broadcast",
|
||||
"to": "broadcast",
|
||||
"text": message,
|
||||
"region": credentials.get("mesh_region", "US"),
|
||||
"channel": body.get("channel", "LongFast"),
|
||||
@@ -750,6 +752,8 @@ async def mesh_send(request: Request):
|
||||
"event_id": "",
|
||||
"routed_via": envelope.routed_via,
|
||||
"route_reason": envelope.route_reason,
|
||||
"direct": is_direct_destination,
|
||||
"channel_echo": not is_direct_destination,
|
||||
"results": [r.to_dict() for r in results],
|
||||
}
|
||||
|
||||
@@ -818,9 +822,10 @@ async def meshtastic_public_send(request: Request):
|
||||
if not cb_ok:
|
||||
results = [TransportResult(False, "meshtastic", cb_reason)]
|
||||
else:
|
||||
is_direct_destination = MeshtasticTransport._parse_node_id(destination) is not None
|
||||
envelope.route_reason = (
|
||||
"Local public Meshtastic MQTT path"
|
||||
if MeshtasticTransport._parse_node_id(destination) is None
|
||||
if not is_direct_destination
|
||||
else "Local public Meshtastic direct node path"
|
||||
)
|
||||
credentials = {"mesh_region": str(body.get("mesh_region", "US") or "US")}
|
||||
@@ -830,23 +835,28 @@ async def meshtastic_public_send(request: Request):
|
||||
results = [result]
|
||||
|
||||
any_ok = any(r.ok for r in results)
|
||||
if any_ok and envelope.routed_via == "meshtastic":
|
||||
is_direct_destination = MeshtasticTransport._parse_node_id(destination) is not None
|
||||
if any_ok and envelope.routed_via == "meshtastic" and not is_direct_destination:
|
||||
try:
|
||||
from datetime import datetime
|
||||
from services.sigint_bridge import sigint_grid
|
||||
|
||||
bridge = sigint_grid.mesh
|
||||
if bridge:
|
||||
bridge.messages.appendleft(
|
||||
{
|
||||
"from": MeshtasticTransport.mesh_address_for_sender(sender_id),
|
||||
"to": destination if MeshtasticTransport._parse_node_id(destination) is not None else "broadcast",
|
||||
"text": message,
|
||||
"region": str(body.get("mesh_region", "US") or "US"),
|
||||
"channel": str(body.get("channel", "LongFast") or "LongFast"),
|
||||
"timestamp": datetime.utcnow().isoformat() + "Z",
|
||||
}
|
||||
)
|
||||
record = {
|
||||
"from": MeshtasticTransport.mesh_address_for_sender(sender_id),
|
||||
"to": "broadcast",
|
||||
"text": message,
|
||||
"region": str(body.get("mesh_region", "US") or "US"),
|
||||
"root": str(body.get("mesh_region", "US") or "US"),
|
||||
"channel": str(body.get("channel", "LongFast") or "LongFast"),
|
||||
"timestamp": datetime.utcnow().isoformat() + "Z",
|
||||
}
|
||||
append_text = getattr(bridge, "append_text_message", None)
|
||||
if callable(append_text):
|
||||
append_text(record)
|
||||
else:
|
||||
bridge.messages.appendleft(record)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -856,6 +866,8 @@ async def meshtastic_public_send(request: Request):
|
||||
"event_id": "",
|
||||
"routed_via": envelope.routed_via,
|
||||
"route_reason": envelope.route_reason,
|
||||
"direct": is_direct_destination,
|
||||
"channel_echo": not is_direct_destination,
|
||||
"results": [r.to_dict() for r in results],
|
||||
}
|
||||
|
||||
@@ -954,6 +966,7 @@ async def mesh_messages(
|
||||
root: str = "",
|
||||
channel: str = "",
|
||||
limit: int = 30,
|
||||
include_direct: bool = False,
|
||||
):
|
||||
"""Get recent Meshtastic text messages from the MQTT bridge."""
|
||||
from services.sigint_bridge import sigint_grid
|
||||
@@ -975,6 +988,12 @@ async def mesh_messages(
|
||||
msgs = [m for m in msgs if m.get("root", "").upper() == root_filter]
|
||||
if channel:
|
||||
msgs = [m for m in msgs if m.get("channel", "").lower() == channel.lower()]
|
||||
if not include_direct:
|
||||
msgs = [
|
||||
m
|
||||
for m in msgs
|
||||
if str(m.get("to") or "broadcast").strip().lower() in {"", "broadcast", "^all"}
|
||||
]
|
||||
return msgs[: min(limit, 100)]
|
||||
|
||||
|
||||
@@ -1448,25 +1467,37 @@ def _submit_gate_message_envelope(request: Request, gate_id: str, body: dict[str
|
||||
@router.get("/api/mesh/infonet/status")
|
||||
@limiter.limit("30/minute")
|
||||
async def infonet_status(request: Request, verify_signatures: bool = False):
|
||||
"""Get Infonet metadata — event counts, head hash, chain size."""
|
||||
"""Get Infonet metadata — event counts, head hash, chain size.
|
||||
|
||||
The ``verify_signatures`` query parameter is honored ONLY when the
|
||||
caller has authenticated via scoped auth or local-operator credentials.
|
||||
Verifying every signature in a long chain is O(n_events) work — letting
|
||||
anonymous callers trigger it is a DoS surface (issue #207). For
|
||||
anonymous callers we silently fall back to the cheap path; the response
|
||||
structure is identical so legitimate frontends see no behavior change.
|
||||
"""
|
||||
from services.mesh.mesh_hashchain import infonet
|
||||
from services.wormhole_supervisor import get_wormhole_state
|
||||
|
||||
# Silently downgrade for unauthenticated callers — no error surfaced.
|
||||
authenticated = _scoped_view_authenticated(request, "mesh.audit")
|
||||
effective_verify_signatures = bool(verify_signatures) and authenticated
|
||||
|
||||
info = infonet.get_info()
|
||||
valid, reason = infonet.validate_chain(verify_signatures=verify_signatures)
|
||||
valid, reason = infonet.validate_chain(verify_signatures=effective_verify_signatures)
|
||||
try:
|
||||
wormhole = get_wormhole_state()
|
||||
except Exception:
|
||||
wormhole = {"configured": False, "ready": False, "rns_ready": False}
|
||||
info["valid"] = valid
|
||||
info["validation"] = reason
|
||||
info["verify_signatures"] = verify_signatures
|
||||
info["verify_signatures"] = effective_verify_signatures
|
||||
info["private_lane_tier"] = _current_private_lane_tier(wormhole)
|
||||
info["private_lane_policy"] = _private_infonet_policy_snapshot()
|
||||
info.update(_node_runtime_snapshot())
|
||||
return _redact_private_lane_control_fields(
|
||||
info,
|
||||
authenticated=_scoped_view_authenticated(request, "mesh.audit"),
|
||||
authenticated=authenticated,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -21,14 +21,30 @@ async def api_get_openmhz_systems(request: Request):
|
||||
return get_openmhz_systems()
|
||||
|
||||
|
||||
@router.get("/api/radio/openmhz/calls/{sys_name}")
|
||||
# Issue #213: rotating sys_name bypasses the 20s TTL cache and lets an
|
||||
# anonymous caller hammer api.openmhz.com through this proxy, risking an
|
||||
# IP-ban for the project. require_local_operator scopes this to the local
|
||||
# UI (which goes through the Next.js proxy with admin-key injection) and
|
||||
# scoped agent tokens.
|
||||
@router.get(
|
||||
"/api/radio/openmhz/calls/{sys_name}",
|
||||
dependencies=[Depends(require_local_operator)],
|
||||
)
|
||||
@limiter.limit("60/minute")
|
||||
async def api_get_openmhz_calls(request: Request, sys_name: str):
|
||||
from services.radio_intercept import get_recent_openmhz_calls
|
||||
return get_recent_openmhz_calls(sys_name)
|
||||
|
||||
|
||||
@router.get("/api/radio/openmhz/audio")
|
||||
# Issue #214: this is a streaming bandwidth relay. An anonymous caller can
|
||||
# stream audio through the backend, saturating the operator's outbound
|
||||
# bandwidth. Scope to local operator; the legitimate browser UI still
|
||||
# works because relative /api/... paths go through the Next.js proxy
|
||||
# which injects the admin key automatically.
|
||||
@router.get(
|
||||
"/api/radio/openmhz/audio",
|
||||
dependencies=[Depends(require_local_operator)],
|
||||
)
|
||||
@limiter.limit("120/minute")
|
||||
async def api_get_openmhz_audio(request: Request, url: str = Query(..., min_length=10)):
|
||||
from services.radio_intercept import openmhz_audio_response
|
||||
|
||||
@@ -21,7 +21,7 @@ async def oracle_region_intel(
|
||||
return get_region_oracle_intel(lat, lng, news_items)
|
||||
|
||||
|
||||
@router.get("/api/thermal/verify")
|
||||
@router.get("/api/thermal/verify", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("10/minute")
|
||||
async def thermal_verify(
|
||||
request: Request,
|
||||
@@ -35,7 +35,7 @@ async def thermal_verify(
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/api/sigint/transmit")
|
||||
@router.post("/api/sigint/transmit", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("5/minute")
|
||||
async def sigint_transmit(request: Request):
|
||||
"""Send an APRS-IS message to a specific callsign. Requires ham radio credentials."""
|
||||
|
||||
@@ -120,7 +120,36 @@ async def api_sentinel_token(request: Request):
|
||||
raise HTTPException(502, "Token request failed")
|
||||
|
||||
|
||||
_sh_token_cache: dict = {"token": None, "expiry": 0, "client_id": ""}
|
||||
# Cache key is an HMAC of (client_id, client_secret) — a caller cannot hit
|
||||
# this cache without knowing the same secret that originally populated it.
|
||||
# Without this binding, the lookup only checked client_id, so anyone who
|
||||
# knew a valid client_id could reuse another caller's cached token (and
|
||||
# burn their Copernicus quota / access tiles on their account).
|
||||
_sh_token_cache: dict = {"token": None, "expiry": 0, "credential_fp": ""}
|
||||
|
||||
|
||||
def _credential_fingerprint(client_id: str, client_secret: str) -> str:
|
||||
"""Return a stable, secret-binding fingerprint for the Sentinel cache key.
|
||||
|
||||
Uses HMAC-SHA256 so the raw secret is never stored in process memory as
|
||||
a cache key. The HMAC key is a per-process random value, which means the
|
||||
fingerprint cannot be precomputed across restarts (additional defense
|
||||
against an attacker who learned a valid client_id but not the secret).
|
||||
"""
|
||||
import hashlib
|
||||
import hmac
|
||||
|
||||
return hmac.new(
|
||||
_SH_TOKEN_CACHE_HMAC_KEY,
|
||||
f"{client_id}\x00{client_secret}".encode("utf-8"),
|
||||
hashlib.sha256,
|
||||
).hexdigest()
|
||||
|
||||
|
||||
# Per-process random HMAC key. Regenerated on each backend startup so cached
|
||||
# fingerprints don't survive restarts.
|
||||
import os as _os
|
||||
_SH_TOKEN_CACHE_HMAC_KEY = _os.urandom(32)
|
||||
|
||||
|
||||
@router.post("/api/sentinel/tile")
|
||||
@@ -146,7 +175,9 @@ async def api_sentinel_tile(request: Request):
|
||||
raise HTTPException(400, "client_id, client_secret, and date required")
|
||||
|
||||
now = _time.time()
|
||||
if (_sh_token_cache["token"] and _sh_token_cache["client_id"] == client_id
|
||||
credential_fp = _credential_fingerprint(client_id, client_secret)
|
||||
if (_sh_token_cache["token"]
|
||||
and _sh_token_cache["credential_fp"] == credential_fp
|
||||
and now < _sh_token_cache["expiry"] - 30):
|
||||
token = _sh_token_cache["token"]
|
||||
else:
|
||||
@@ -161,7 +192,7 @@ async def api_sentinel_tile(request: Request):
|
||||
token = tdata["access_token"]
|
||||
_sh_token_cache["token"] = token
|
||||
_sh_token_cache["expiry"] = now + tdata.get("expires_in", 300)
|
||||
_sh_token_cache["client_id"] = client_id
|
||||
_sh_token_cache["credential_fp"] = credential_fp
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception:
|
||||
|
||||
+60
-311
@@ -78,6 +78,21 @@ export_wormhole_dm_invite = getattr(
|
||||
"export_wormhole_dm_invite",
|
||||
_wormhole_identity_unavailable,
|
||||
)
|
||||
list_prekey_lookup_handle_records_for_ui = getattr(
|
||||
_mesh_wormhole_identity,
|
||||
"list_prekey_lookup_handle_records_for_ui",
|
||||
_wormhole_identity_unavailable,
|
||||
)
|
||||
rename_prekey_lookup_handle = getattr(
|
||||
_mesh_wormhole_identity,
|
||||
"rename_prekey_lookup_handle",
|
||||
_wormhole_identity_unavailable,
|
||||
)
|
||||
revoke_prekey_lookup_handle = getattr(
|
||||
_mesh_wormhole_identity,
|
||||
"revoke_prekey_lookup_handle",
|
||||
_wormhole_identity_unavailable,
|
||||
)
|
||||
import_wormhole_dm_invite = getattr(
|
||||
_mesh_wormhole_identity,
|
||||
"import_wormhole_dm_invite",
|
||||
@@ -311,6 +326,10 @@ class WormholeDmInviteImportRequest(BaseModel):
|
||||
alias: str = ""
|
||||
|
||||
|
||||
class WormholeDmInviteHandleUpdateRequest(BaseModel):
|
||||
label: str = ""
|
||||
|
||||
|
||||
class WormholeDmSenderTokenRequest(BaseModel):
|
||||
recipient_id: str
|
||||
delivery_class: str
|
||||
@@ -477,6 +496,7 @@ def decrypt_wormhole_dm_envelope(
|
||||
remote_alias: str | None = None,
|
||||
session_welcome: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Delegate to main.py, which owns current MLS/alias/legacy gating behavior."""
|
||||
import main as _m
|
||||
|
||||
return _m.decrypt_wormhole_dm_envelope(
|
||||
@@ -489,71 +509,13 @@ def decrypt_wormhole_dm_envelope(
|
||||
session_welcome=session_welcome,
|
||||
)
|
||||
|
||||
resolved_local, resolved_remote = _resolve_dm_aliases(
|
||||
peer_id=peer_id,
|
||||
local_alias=local_alias,
|
||||
remote_alias=remote_alias,
|
||||
)
|
||||
normalized_format = str(payload_format or "dm1").strip().lower() or "dm1"
|
||||
if normalized_format != "mls1" and is_dm_locked_to_mls(resolved_local, resolved_remote):
|
||||
return {
|
||||
"ok": False,
|
||||
"detail": "DM session is locked to MLS format",
|
||||
"required_format": "mls1",
|
||||
"current_format": normalized_format,
|
||||
}
|
||||
if normalized_format == "mls1":
|
||||
has_session = has_mls_dm_session(resolved_local, resolved_remote)
|
||||
if not has_session.get("ok"):
|
||||
return has_session
|
||||
if not has_session.get("exists"):
|
||||
ensured = ensure_mls_dm_session(resolved_local, resolved_remote, str(session_welcome or ""))
|
||||
if not ensured.get("ok"):
|
||||
return ensured
|
||||
decrypted = decrypt_mls_dm(
|
||||
resolved_local,
|
||||
resolved_remote,
|
||||
str(ciphertext or ""),
|
||||
str(nonce or ""),
|
||||
)
|
||||
if not decrypted.get("ok"):
|
||||
return decrypted
|
||||
return {
|
||||
"ok": True,
|
||||
"peer_id": str(peer_id or "").strip(),
|
||||
"local_alias": resolved_local,
|
||||
"remote_alias": resolved_remote,
|
||||
"plaintext": str(decrypted.get("plaintext", "") or ""),
|
||||
"format": "mls1",
|
||||
}
|
||||
|
||||
from services.wormhole_supervisor import get_transport_tier
|
||||
|
||||
current_tier = get_transport_tier()
|
||||
if str(current_tier or "").startswith("private_"):
|
||||
return {
|
||||
"ok": False,
|
||||
"detail": "MLS format required in private transport mode — legacy DM decrypt blocked",
|
||||
}
|
||||
logger.warning("legacy dm decrypt path used")
|
||||
legacy = decrypt_wormhole_dm(peer_id=str(peer_id or ""), ciphertext=str(ciphertext or ""))
|
||||
if not legacy.get("ok"):
|
||||
return legacy
|
||||
return {
|
||||
"ok": True,
|
||||
"peer_id": str(peer_id or "").strip(),
|
||||
"local_alias": resolved_local,
|
||||
"remote_alias": resolved_remote,
|
||||
"plaintext": str(legacy.get("result", "") or ""),
|
||||
"format": "dm1",
|
||||
}
|
||||
|
||||
|
||||
|
||||
# --- Routes ---
|
||||
|
||||
@router.get("/api/settings/wormhole")
|
||||
@limiter.limit("30/minute")
|
||||
@limiter.limit("240/minute")
|
||||
async def api_get_wormhole_settings(request: Request):
|
||||
settings = await asyncio.to_thread(read_wormhole_settings)
|
||||
return _redact_wormhole_settings(settings, authenticated=_scoped_view_authenticated(request, "wormhole"))
|
||||
@@ -582,248 +544,9 @@ async def api_set_wormhole_settings(request: Request, body: WormholeUpdate):
|
||||
return {**updated, "requires_restart": False, "runtime": state}
|
||||
|
||||
|
||||
class PrivacyProfileUpdate(BaseModel):
|
||||
profile: str
|
||||
|
||||
|
||||
class WormholeSignRequest(BaseModel):
|
||||
event_type: str
|
||||
payload: dict
|
||||
sequence: int | None = None
|
||||
gate_id: str | None = None
|
||||
|
||||
|
||||
class WormholeSignRawRequest(BaseModel):
|
||||
message: str
|
||||
|
||||
|
||||
class WormholeDmEncryptRequest(BaseModel):
|
||||
peer_id: str
|
||||
peer_dh_pub: str = ""
|
||||
plaintext: str
|
||||
local_alias: str | None = None
|
||||
remote_alias: str | None = None
|
||||
remote_prekey_bundle: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class WormholeDmComposeRequest(BaseModel):
|
||||
peer_id: str
|
||||
peer_dh_pub: str = ""
|
||||
plaintext: str
|
||||
local_alias: str | None = None
|
||||
remote_alias: str | None = None
|
||||
remote_prekey_bundle: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class WormholeDmDecryptRequest(BaseModel):
|
||||
peer_id: str
|
||||
ciphertext: str
|
||||
format: str = "dm1"
|
||||
nonce: str = ""
|
||||
local_alias: str | None = None
|
||||
remote_alias: str | None = None
|
||||
session_welcome: str | None = None
|
||||
|
||||
|
||||
class WormholeDmResetRequest(BaseModel):
|
||||
peer_id: str | None = None
|
||||
|
||||
|
||||
class WormholeDmBootstrapEncryptRequest(BaseModel):
|
||||
peer_id: str
|
||||
plaintext: str
|
||||
|
||||
|
||||
class WormholeDmBootstrapDecryptRequest(BaseModel):
|
||||
sender_id: str = ""
|
||||
ciphertext: str
|
||||
|
||||
|
||||
class WormholeDmSenderTokenRequest(BaseModel):
|
||||
recipient_id: str
|
||||
delivery_class: str
|
||||
recipient_token: str = ""
|
||||
count: int = 1
|
||||
|
||||
|
||||
class WormholeOpenSealRequest(BaseModel):
|
||||
sender_seal: str
|
||||
candidate_dh_pub: str = ""
|
||||
recipient_id: str
|
||||
expected_msg_id: str
|
||||
|
||||
|
||||
class WormholeBuildSealRequest(BaseModel):
|
||||
recipient_id: str
|
||||
recipient_dh_pub: str = ""
|
||||
msg_id: str
|
||||
timestamp: int
|
||||
|
||||
|
||||
class WormholeDeadDropTokenRequest(BaseModel):
|
||||
peer_id: str
|
||||
peer_dh_pub: str = ""
|
||||
peer_ref: str = ""
|
||||
|
||||
|
||||
class WormholePairwiseAliasRequest(BaseModel):
|
||||
peer_id: str
|
||||
peer_dh_pub: str = ""
|
||||
|
||||
|
||||
class WormholePairwiseAliasRotateRequest(BaseModel):
|
||||
peer_id: str
|
||||
peer_dh_pub: str = ""
|
||||
grace_ms: int = 45_000
|
||||
|
||||
|
||||
class WormholeDeadDropContactsRequest(BaseModel):
|
||||
contacts: list[dict[str, Any]]
|
||||
limit: int = 24
|
||||
|
||||
|
||||
class WormholeSasRequest(BaseModel):
|
||||
peer_id: str
|
||||
peer_dh_pub: str = ""
|
||||
words: int = 8
|
||||
peer_ref: str = ""
|
||||
|
||||
|
||||
class WormholeGateRequest(BaseModel):
|
||||
gate_id: str
|
||||
rotate: bool = False
|
||||
|
||||
|
||||
class WormholeGatePersonaCreateRequest(BaseModel):
|
||||
gate_id: str
|
||||
label: str = ""
|
||||
|
||||
|
||||
class WormholeGatePersonaActivateRequest(BaseModel):
|
||||
gate_id: str
|
||||
persona_id: str
|
||||
|
||||
|
||||
class WormholeGateKeyGrantRequest(BaseModel):
|
||||
gate_id: str
|
||||
recipient_node_id: str
|
||||
recipient_dh_pub: str
|
||||
recipient_scope: str = "member"
|
||||
|
||||
|
||||
class WormholeGateComposeRequest(BaseModel):
|
||||
gate_id: str
|
||||
plaintext: str
|
||||
reply_to: str = ""
|
||||
compat_plaintext: bool = False
|
||||
|
||||
|
||||
class WormholeGateDecryptRequest(BaseModel):
|
||||
gate_id: str
|
||||
epoch: int = 0
|
||||
ciphertext: str
|
||||
nonce: str = ""
|
||||
sender_ref: str = ""
|
||||
format: str = "mls1"
|
||||
gate_envelope: str = ""
|
||||
envelope_hash: str = ""
|
||||
recovery_envelope: bool = False
|
||||
compat_decrypt: bool = False
|
||||
event_id: str = ""
|
||||
|
||||
|
||||
class WormholeGateDecryptBatchRequest(BaseModel):
|
||||
messages: list[WormholeGateDecryptRequest]
|
||||
|
||||
|
||||
class WormholeGateRotateRequest(BaseModel):
|
||||
gate_id: str
|
||||
reason: str = "manual_rotate"
|
||||
|
||||
def decrypt_wormhole_dm_envelope(
|
||||
*,
|
||||
peer_id: str,
|
||||
ciphertext: str,
|
||||
payload_format: str = "dm1",
|
||||
nonce: str = "",
|
||||
local_alias: str | None = None,
|
||||
remote_alias: str | None = None,
|
||||
session_welcome: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
import main as _m
|
||||
|
||||
return _m.decrypt_wormhole_dm_envelope(
|
||||
peer_id=peer_id,
|
||||
ciphertext=ciphertext,
|
||||
payload_format=payload_format,
|
||||
nonce=nonce,
|
||||
local_alias=local_alias,
|
||||
remote_alias=remote_alias,
|
||||
session_welcome=session_welcome,
|
||||
)
|
||||
|
||||
resolved_local, resolved_remote = _resolve_dm_aliases(
|
||||
peer_id=peer_id,
|
||||
local_alias=local_alias,
|
||||
remote_alias=remote_alias,
|
||||
)
|
||||
normalized_format = str(payload_format or "dm1").strip().lower() or "dm1"
|
||||
if normalized_format != "mls1" and is_dm_locked_to_mls(resolved_local, resolved_remote):
|
||||
return {
|
||||
"ok": False,
|
||||
"detail": "DM session is locked to MLS format",
|
||||
"required_format": "mls1",
|
||||
"current_format": normalized_format,
|
||||
}
|
||||
if normalized_format == "mls1":
|
||||
has_session = has_mls_dm_session(resolved_local, resolved_remote)
|
||||
if not has_session.get("ok"):
|
||||
return has_session
|
||||
if not has_session.get("exists"):
|
||||
ensured = ensure_mls_dm_session(resolved_local, resolved_remote, str(session_welcome or ""))
|
||||
if not ensured.get("ok"):
|
||||
return ensured
|
||||
decrypted = decrypt_mls_dm(
|
||||
resolved_local,
|
||||
resolved_remote,
|
||||
str(ciphertext or ""),
|
||||
str(nonce or ""),
|
||||
)
|
||||
if not decrypted.get("ok"):
|
||||
return decrypted
|
||||
return {
|
||||
"ok": True,
|
||||
"peer_id": str(peer_id or "").strip(),
|
||||
"local_alias": resolved_local,
|
||||
"remote_alias": resolved_remote,
|
||||
"plaintext": str(decrypted.get("plaintext", "") or ""),
|
||||
"format": "mls1",
|
||||
}
|
||||
|
||||
from services.wormhole_supervisor import get_transport_tier
|
||||
|
||||
current_tier = get_transport_tier()
|
||||
if str(current_tier or "").startswith("private_"):
|
||||
return {
|
||||
"ok": False,
|
||||
"detail": "MLS format required in private transport mode — legacy DM decrypt blocked",
|
||||
}
|
||||
logger.warning("legacy dm decrypt path used")
|
||||
legacy = decrypt_wormhole_dm(peer_id=str(peer_id or ""), ciphertext=str(ciphertext or ""))
|
||||
if not legacy.get("ok"):
|
||||
return legacy
|
||||
return {
|
||||
"ok": True,
|
||||
"peer_id": str(peer_id or "").strip(),
|
||||
"local_alias": resolved_local,
|
||||
"remote_alias": resolved_remote,
|
||||
"plaintext": str(legacy.get("result", "") or ""),
|
||||
"format": "dm1",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/settings/privacy-profile")
|
||||
@limiter.limit("30/minute")
|
||||
@limiter.limit("240/minute")
|
||||
async def api_get_privacy_profile(request: Request):
|
||||
data = await asyncio.to_thread(read_wormhole_settings)
|
||||
return _redact_privacy_profile_settings(
|
||||
@@ -833,7 +556,7 @@ async def api_get_privacy_profile(request: Request):
|
||||
|
||||
|
||||
@router.get("/api/settings/wormhole-status")
|
||||
@limiter.limit("30/minute")
|
||||
@limiter.limit("240/minute")
|
||||
async def api_get_wormhole_status(request: Request):
|
||||
state = await asyncio.to_thread(get_wormhole_state)
|
||||
transport_tier = _current_private_lane_tier(state)
|
||||
@@ -907,7 +630,7 @@ async def api_wormhole_join(request: Request):
|
||||
)
|
||||
|
||||
# Enable node participation so the sync/push workers connect to peers.
|
||||
# This is the voluntary opt-in — the node only joins the network when
|
||||
# This is the voluntary opt-in — the node only joins the network when
|
||||
# the user explicitly opens the Wormhole.
|
||||
from services.node_settings import write_node_settings
|
||||
|
||||
@@ -923,7 +646,7 @@ async def api_wormhole_join(request: Request):
|
||||
}
|
||||
|
||||
|
||||
@router.post("/api/wormhole/leave", dependencies=[Depends(require_local_operator)])
|
||||
@router.post("/api/wormhole/leave")
|
||||
@limiter.limit("10/minute")
|
||||
async def api_wormhole_leave(request: Request):
|
||||
updated = write_wormhole_settings(enabled=False)
|
||||
@@ -941,7 +664,7 @@ async def api_wormhole_leave(request: Request):
|
||||
|
||||
|
||||
@router.get("/api/wormhole/identity", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
@limiter.limit("240/minute")
|
||||
async def api_wormhole_identity(request: Request):
|
||||
try:
|
||||
bootstrap_wormhole_persona_state()
|
||||
@@ -970,7 +693,7 @@ async def api_wormhole_identity_bootstrap(request: Request):
|
||||
|
||||
|
||||
@router.get("/api/wormhole/dm/identity", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
@limiter.limit("240/minute")
|
||||
async def api_wormhole_dm_identity(request: Request):
|
||||
try:
|
||||
bootstrap_wormhole_persona_state()
|
||||
@@ -982,11 +705,37 @@ async def api_wormhole_dm_identity(request: Request):
|
||||
|
||||
@router.get("/api/wormhole/dm/invite", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_dm_invite(request: Request):
|
||||
return export_wormhole_dm_invite()
|
||||
async def api_wormhole_dm_invite(
|
||||
request: Request,
|
||||
label: str = Query("", max_length=96),
|
||||
expires_in_s: int = Query(0, ge=0, le=2_592_000),
|
||||
):
|
||||
return export_wormhole_dm_invite(label=label, expires_in_s=expires_in_s)
|
||||
|
||||
|
||||
@router.post("/api/wormhole/dm/invite/import", dependencies=[Depends(require_admin)])
|
||||
@router.get("/api/wormhole/dm/invite/handles", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("240/minute")
|
||||
async def api_wormhole_dm_invite_handles(request: Request):
|
||||
return list_prekey_lookup_handle_records_for_ui()
|
||||
|
||||
|
||||
@router.patch("/api/wormhole/dm/invite/handles/{handle}", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("60/minute")
|
||||
async def api_wormhole_dm_invite_handle_update(
|
||||
request: Request,
|
||||
handle: str,
|
||||
body: WormholeDmInviteHandleUpdateRequest,
|
||||
):
|
||||
return rename_prekey_lookup_handle(handle, str(body.label or "").strip())
|
||||
|
||||
|
||||
@router.delete("/api/wormhole/dm/invite/handles/{handle}", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_dm_invite_handle_revoke(request: Request, handle: str):
|
||||
return revoke_prekey_lookup_handle(handle)
|
||||
|
||||
|
||||
@router.post("/api/wormhole/dm/invite/import", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_dm_invite_import(request: Request, body: WormholeDmInviteImportRequest):
|
||||
return import_wormhole_dm_invite(
|
||||
@@ -1205,7 +954,7 @@ async def api_wormhole_gate_message_sign_encrypted(
|
||||
return await _m.api_wormhole_gate_message_sign_encrypted(request, body)
|
||||
|
||||
|
||||
@router.post("/api/wormhole/gate/message/post-encrypted", dependencies=[Depends(require_local_operator)])
|
||||
@router.post("/api/wormhole/gate/message/post-encrypted")
|
||||
@limiter.limit("30/minute")
|
||||
async def api_wormhole_gate_message_post_encrypted(
|
||||
request: Request,
|
||||
@@ -1547,7 +1296,7 @@ class PrivateDeliveryActionRequest(BaseModel):
|
||||
|
||||
|
||||
@router.get("/api/wormhole/status")
|
||||
@limiter.limit("30/minute")
|
||||
@limiter.limit("240/minute")
|
||||
async def api_wormhole_status(request: Request):
|
||||
import main as _m
|
||||
|
||||
@@ -1590,7 +1339,7 @@ async def api_wormhole_private_delivery_action(
|
||||
|
||||
|
||||
@router.get("/api/wormhole/health")
|
||||
@limiter.limit("30/minute")
|
||||
@limiter.limit("240/minute")
|
||||
async def api_wormhole_health(request: Request):
|
||||
state = get_wormhole_state()
|
||||
transport_tier = _current_private_lane_tier(state)
|
||||
@@ -1611,7 +1360,7 @@ async def api_wormhole_health(request: Request):
|
||||
return _redact_wormhole_status(full_state, authenticated=ok)
|
||||
|
||||
|
||||
@router.post("/api/wormhole/connect", dependencies=[Depends(require_admin)])
|
||||
@router.post("/api/wormhole/connect", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("10/minute")
|
||||
async def api_wormhole_connect(request: Request):
|
||||
settings = read_wormhole_settings()
|
||||
|
||||
@@ -344,9 +344,26 @@ _vessels_lock = threading.Lock()
|
||||
_ws_thread: threading.Thread | None = None
|
||||
_ws_running = False
|
||||
_proxy_process = None
|
||||
# Issue #258: latest status snapshot emitted by ais_proxy.js. Populated when
|
||||
# the proxy reports e.g. {"__ais_proxy_status": {"degraded_tls": true}} on
|
||||
# stdout, which it does when it falls back to the SPKI-pinned insecure-date
|
||||
# path during an upstream cert outage. Surfaced via ais_proxy_status() for
|
||||
# /api/health.
|
||||
_proxy_status: dict = {}
|
||||
_VESSEL_TRAIL_INTERVAL_S = 120
|
||||
_VESSEL_TRAIL_MAX_POINTS = 240
|
||||
|
||||
|
||||
def ais_proxy_status() -> dict:
|
||||
"""Return a copy of the latest ais_proxy.js status (issue #258).
|
||||
|
||||
Currently surfaces ``degraded_tls`` (bool) which is true when the
|
||||
proxy is using SPKI-pinned fallback because AISStream's cert expired.
|
||||
Returns an empty dict when no status has been received yet.
|
||||
"""
|
||||
with _vessels_lock:
|
||||
return dict(_proxy_status)
|
||||
|
||||
import os
|
||||
|
||||
CACHE_FILE = os.path.join(os.path.dirname(__file__), "ais_cache.json")
|
||||
@@ -608,6 +625,18 @@ def _ais_stream_loop():
|
||||
logger.error(f"AIS Stream error: {data['error']}")
|
||||
continue
|
||||
|
||||
# Issue #258: ais_proxy.js emits status markers (e.g.
|
||||
# {"__ais_proxy_status": {"degraded_tls": true}}) when the
|
||||
# SPKI-pinned fallback is in use. We snapshot the latest
|
||||
# status so the backend can expose it on /api/health.
|
||||
if isinstance(data, dict) and "__ais_proxy_status" in data:
|
||||
status = data.get("__ais_proxy_status") or {}
|
||||
if isinstance(status, dict):
|
||||
with _vessels_lock:
|
||||
_proxy_status.clear()
|
||||
_proxy_status.update(status)
|
||||
continue
|
||||
|
||||
msg_type = data.get("MessageType", "")
|
||||
metadata = data.get("MetaData", {})
|
||||
message = data.get("Message", {})
|
||||
|
||||
@@ -987,7 +987,7 @@ _KML_NS = {"kml": "http://www.opengis.net/kml/2.2"}
|
||||
|
||||
def _find_kml_element(element, tag):
|
||||
"""Find first descendant matching tag, ignoring XML namespace prefix."""
|
||||
import xml.etree.ElementTree as ET
|
||||
import defusedxml.ElementTree as ET
|
||||
el = element.find(f".//{tag}")
|
||||
if el is not None:
|
||||
return el
|
||||
@@ -1015,7 +1015,7 @@ class MadridCityIngestor(BaseCCTVIngestor):
|
||||
KML_URL = "http://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml"
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
import xml.etree.ElementTree as ET
|
||||
import defusedxml.ElementTree as ET
|
||||
|
||||
try:
|
||||
response = fetch_with_curl(self.KML_URL, timeout=20)
|
||||
|
||||
@@ -46,9 +46,12 @@ class Settings(BaseSettings):
|
||||
MESH_NODE_MODE: str = "participant"
|
||||
MESH_SYNC_INTERVAL_S: int = 300
|
||||
MESH_SYNC_FAILURE_BACKOFF_S: int = 60
|
||||
MESH_SYNC_TIMEOUT_S: int = 5
|
||||
MESH_SYNC_MAX_PEERS_PER_CYCLE: int = 3
|
||||
MESH_RELAY_PUSH_TIMEOUT_S: int = 10
|
||||
MESH_RELAY_MAX_FAILURES: int = 3
|
||||
MESH_RELAY_FAILURE_COOLDOWN_S: int = 120
|
||||
MESH_BOOTSTRAP_SEED_FAILURE_COOLDOWN_S: int = 15
|
||||
MESH_PEER_PUSH_SECRET: str = ""
|
||||
MESH_RNS_APP_NAME: str = "shadowbroker"
|
||||
MESH_RNS_ASPECT: str = "infonet"
|
||||
|
||||
@@ -318,7 +318,7 @@ active_layers: dict[str, bool] = {
|
||||
"uap_sightings": True,
|
||||
"wastewater": True,
|
||||
"ai_intel": True,
|
||||
"crowdthreat": True,
|
||||
"crowdthreat": False,
|
||||
"sar": True,
|
||||
}
|
||||
|
||||
|
||||
@@ -16,9 +16,9 @@ import csv
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from typing import Any
|
||||
|
||||
import defusedxml.ElementTree as ET
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -31,11 +31,7 @@ _S3_NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
|
||||
_REFRESH_INTERVAL_S = 5 * 24 * 3600
|
||||
_LIST_TIMEOUT_S = 30
|
||||
_DOWNLOAD_TIMEOUT_S = 600
|
||||
_USER_AGENT = (
|
||||
"ShadowBroker-OSINT/0.9.75 "
|
||||
"(+https://github.com/BigBodyCobain/Shadowbroker; "
|
||||
"contact: bigbodycobain@gmail.com)"
|
||||
)
|
||||
from services.network_utils import DEFAULT_USER_AGENT as _USER_AGENT
|
||||
|
||||
_lock = threading.RLock()
|
||||
_aircraft_by_hex: dict[str, dict[str, str]] = {}
|
||||
|
||||
@@ -7,6 +7,7 @@ No API key required — the /threats endpoint is unauthenticated.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh, is_any_active
|
||||
@@ -16,6 +17,16 @@ logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
_CT_BASE = "https://backend.crowdthreat.world"
|
||||
|
||||
|
||||
def crowdthreat_fetch_enabled() -> bool:
|
||||
"""Return True only when the operator explicitly opts into CrowdThreat pulls."""
|
||||
return str(os.environ.get("CROWDTHREAT_ENABLED", "")).strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
}
|
||||
|
||||
# CrowdThreat category_id → icon ID used on the MapLibre layer
|
||||
_CATEGORY_ICON = {
|
||||
1: "ct-security", # Security & Conflict (red)
|
||||
@@ -43,6 +54,12 @@ _CATEGORY_COLOUR = {
|
||||
@with_retry(max_retries=2, base_delay=5)
|
||||
def fetch_crowdthreat():
|
||||
"""Fetch verified threat reports from CrowdThreat public API."""
|
||||
if not crowdthreat_fetch_enabled():
|
||||
logger.debug("CrowdThreat fetch skipped; set CROWDTHREAT_ENABLED=true to opt in")
|
||||
with _data_lock:
|
||||
latest_data["crowdthreat"] = []
|
||||
_mark_fresh("crowdthreat")
|
||||
return
|
||||
if not is_any_active("crowdthreat"):
|
||||
return
|
||||
|
||||
|
||||
@@ -279,9 +279,13 @@ def fetch_weather_alerts():
|
||||
return
|
||||
alerts = []
|
||||
try:
|
||||
# weather.gov requires a User-Agent per their API policy, but it
|
||||
# need not identify the operator. Use a project-generic string and
|
||||
# let the user override via SHADOWBROKER_USER_AGENT if needed.
|
||||
from services.network_utils import DEFAULT_USER_AGENT
|
||||
url = "https://api.weather.gov/alerts/active?status=actual"
|
||||
headers = {
|
||||
"User-Agent": "(ShadowBroker OSINT Dashboard, github.com/BigBodyCobain/Shadowbroker)",
|
||||
"User-Agent": DEFAULT_USER_AGENT,
|
||||
"Accept": "application/geo+json",
|
||||
}
|
||||
response = fetch_with_curl(url, timeout=15, headers=headers)
|
||||
|
||||
@@ -5,6 +5,7 @@ debunked claims, threat actor mentions, and target country references.
|
||||
Refreshes every 12 hours (FIMI data updates weekly).
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
@@ -18,6 +19,16 @@ logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
_FIMI_FEED_URL = "https://euvsdisinfo.eu/feed/"
|
||||
|
||||
|
||||
def fimi_fetch_enabled() -> bool:
|
||||
"""Return True only when the operator explicitly opts into FIMI pulls."""
|
||||
return str(os.environ.get("FIMI_ENABLED", "")).strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
}
|
||||
|
||||
# ── Threat actor keywords ──────────────────────────────────────────────────
|
||||
# Map of keyword → canonical actor name. Checked case-insensitively.
|
||||
_THREAT_ACTORS: dict[str, str] = {
|
||||
@@ -173,6 +184,12 @@ def _is_major_wave(narratives: list[dict], targets: dict[str, int]) -> bool:
|
||||
@with_retry(max_retries=1, base_delay=5)
|
||||
def fetch_fimi():
|
||||
"""Fetch and parse the EUvsDisinfo RSS feed."""
|
||||
if not fimi_fetch_enabled():
|
||||
logger.debug("FIMI fetch skipped; set FIMI_ENABLED=true to opt in")
|
||||
with _data_lock:
|
||||
latest_data["fimi"] = []
|
||||
_mark_fresh("fimi")
|
||||
return
|
||||
try:
|
||||
resp = fetch_with_curl(_FIMI_FEED_URL, timeout=15)
|
||||
feed = feedparser.parse(resp.text)
|
||||
|
||||
@@ -82,10 +82,37 @@ def _fetch_yfinance_single(symbol: str, period: str = "2d"):
|
||||
|
||||
|
||||
@with_retry(max_retries=1, base_delay=1)
|
||||
def financial_fetch_enabled() -> bool:
|
||||
"""Return True only when the operator explicitly opts into financial pulls.
|
||||
|
||||
Either ``FINANCIAL_ENABLED=true`` or the presence of ``FINNHUB_API_KEY``
|
||||
counts as an explicit opt-in. Without either, the default yfinance path
|
||||
is disabled to avoid silent outbound calls to finance.yahoo.com.
|
||||
"""
|
||||
if os.getenv("FINNHUB_API_KEY", "").strip():
|
||||
return True
|
||||
return str(os.environ.get("FINANCIAL_ENABLED", "")).strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
}
|
||||
|
||||
|
||||
def fetch_financial_markets():
|
||||
"""Fetches full market list with smart throttling (3s for Finnhub, 60s for yfinance)."""
|
||||
global _last_fetch_time, _last_fetch_results, _rotating_index
|
||||
|
||||
|
||||
if not financial_fetch_enabled():
|
||||
logger.debug(
|
||||
"Financial fetch skipped; set FINANCIAL_ENABLED=true or supply "
|
||||
"FINNHUB_API_KEY to opt in"
|
||||
)
|
||||
with _data_lock:
|
||||
latest_data["financial"] = {}
|
||||
_mark_fresh("financial")
|
||||
return
|
||||
|
||||
finnhub_key = os.getenv("FINNHUB_API_KEY", "").strip()
|
||||
use_finnhub = bool(finnhub_key)
|
||||
|
||||
|
||||
@@ -174,16 +174,29 @@ def fetch_meshtastic_nodes():
|
||||
except Exception as e:
|
||||
logger.debug(f"Meshtastic cache freshness check failed: {e}")
|
||||
|
||||
# Build a polite User-Agent. Include the operator callsign when set so
|
||||
# the upstream service can correlate per-install traffic if needed.
|
||||
# Build a polite User-Agent. Historically this included the operator
|
||||
# callsign so meshtastic.org could rate-limit per-install; that's still
|
||||
# the default behavior for backward compatibility. Operators who want
|
||||
# stricter outbound privacy can suppress the callsign by setting
|
||||
# MESHTASTIC_SEND_CALLSIGN_HEADER=false. Issue #203.
|
||||
import os as _os
|
||||
try:
|
||||
from services.config import get_settings
|
||||
|
||||
callsign = str(getattr(get_settings(), "MESHTASTIC_OPERATOR_CALLSIGN", "") or "").strip()
|
||||
except Exception:
|
||||
callsign = ""
|
||||
ua_base = "ShadowBroker-OSINT/0.9.75 (+https://github.com/BigBodyCobain/Shadowbroker; contact: bigbodycobain@gmail.com; 24h polling)"
|
||||
user_agent = f"{ua_base}; node={callsign}" if callsign else ua_base
|
||||
|
||||
send_callsign_header = str(
|
||||
_os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "true")
|
||||
).strip().lower() not in {"0", "false", "no", "off", ""}
|
||||
|
||||
from services.network_utils import DEFAULT_USER_AGENT
|
||||
ua_base = f"{DEFAULT_USER_AGENT}; 24h polling"
|
||||
if callsign and send_callsign_header:
|
||||
user_agent = f"{ua_base}; node={callsign}"
|
||||
else:
|
||||
user_agent = ua_base
|
||||
|
||||
try:
|
||||
logger.info("Fetching Meshtastic map nodes from API...")
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""News fetching, geocoding, clustering, and risk assessment."""
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import logging
|
||||
@@ -11,6 +12,22 @@ from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
from services.fetchers.retry import with_retry
|
||||
from services.oracle_service import enrich_news_items, compute_global_threat_level, detect_breaking_events
|
||||
|
||||
|
||||
def news_fetch_enabled() -> bool:
|
||||
"""Return True only when the operator explicitly opts into news RSS pulls.
|
||||
|
||||
Defaults to **on** for backward compatibility (this is the only fetcher
|
||||
where opting out is the new behavior, not the old one). Set
|
||||
``NEWS_ENABLED=false`` to disable all outbound RSS feed traffic.
|
||||
"""
|
||||
return str(os.environ.get("NEWS_ENABLED", "true")).strip().lower() not in {
|
||||
"0",
|
||||
"false",
|
||||
"no",
|
||||
"off",
|
||||
"",
|
||||
}
|
||||
|
||||
logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
# Maximum article age in seconds. Anything older than this is dropped
|
||||
@@ -160,6 +177,12 @@ def _resolve_coords(text: str) -> tuple[float, float] | None:
|
||||
|
||||
@with_retry(max_retries=1, base_delay=2)
|
||||
def fetch_news():
|
||||
if not news_fetch_enabled():
|
||||
logger.debug("News fetch skipped; unset NEWS_ENABLED=false to re-enable")
|
||||
with _data_lock:
|
||||
latest_data["news"] = []
|
||||
_mark_fresh("news")
|
||||
return
|
||||
from services.news_feed_config import get_feeds
|
||||
feed_config = get_feeds()
|
||||
feeds = {f["name"]: f["url"] for f in feed_config}
|
||||
|
||||
@@ -49,6 +49,16 @@ _HF_CSV_URL = (
|
||||
"https://huggingface.co/datasets/kcimc/NUFORC/resolve/main/nuforc_str.csv"
|
||||
)
|
||||
|
||||
|
||||
def nuforc_fetch_enabled() -> bool:
|
||||
"""Return True only when the operator explicitly opts into NUFORC pulls."""
|
||||
return str(os.environ.get("NUFORC_ENABLED", "")).strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
}
|
||||
|
||||
# Only keep sightings from the last N years for the enrichment index
|
||||
_KEEP_YEARS = 5
|
||||
|
||||
@@ -160,6 +170,12 @@ def _download_and_build() -> dict | None:
|
||||
|
||||
Returns the index dict or None on failure.
|
||||
"""
|
||||
if not nuforc_fetch_enabled():
|
||||
logger.debug(
|
||||
"NUFORC enrichment skipped; set NUFORC_ENABLED=true to opt in"
|
||||
)
|
||||
return None
|
||||
|
||||
cutoff = datetime.utcnow() - timedelta(days=_KEEP_YEARS * 365)
|
||||
cutoff_str = cutoff.strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
@@ -25,6 +25,16 @@ _provider_pace_lock = threading.Lock()
|
||||
_provider_last_request_at: dict[str, float] = {}
|
||||
|
||||
|
||||
def prediction_markets_fetch_enabled() -> bool:
|
||||
"""Return True only when the operator explicitly opts into Polymarket/Kalshi pulls."""
|
||||
return str(os.environ.get("PREDICTION_MARKETS_ENABLED", "")).strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
}
|
||||
|
||||
|
||||
def _pace_provider(provider: str, min_interval_s: float) -> None:
|
||||
if min_interval_s <= 0:
|
||||
return
|
||||
@@ -755,6 +765,16 @@ def fetch_prediction_markets():
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
global _prev_probabilities
|
||||
|
||||
if not prediction_markets_fetch_enabled():
|
||||
logger.debug(
|
||||
"Prediction markets fetch skipped; set "
|
||||
"PREDICTION_MARKETS_ENABLED=true to opt in"
|
||||
)
|
||||
with _data_lock:
|
||||
latest_data["prediction_markets"] = []
|
||||
_mark_fresh("prediction_markets")
|
||||
return
|
||||
|
||||
markets = fetch_prediction_markets_raw()
|
||||
|
||||
# Compute probability deltas vs previous fetch
|
||||
|
||||
@@ -24,11 +24,7 @@ _AIRPORTS_URL = "https://vrs-standing-data.adsb.lol/airports.csv.gz"
|
||||
_REFRESH_INTERVAL_S = 5 * 24 * 3600
|
||||
_HTTP_TIMEOUT_S = 60
|
||||
|
||||
_USER_AGENT = (
|
||||
"ShadowBroker-OSINT/0.9.75 "
|
||||
"(+https://github.com/BigBodyCobain/Shadowbroker; "
|
||||
"contact: bigbodycobain@gmail.com)"
|
||||
)
|
||||
from services.network_utils import DEFAULT_USER_AGENT as _USER_AGENT
|
||||
|
||||
_lock = threading.RLock()
|
||||
_routes_by_callsign: dict[str, dict[str, Any]] = {}
|
||||
|
||||
@@ -616,9 +616,12 @@ def fetch_global_military_incidents():
|
||||
try:
|
||||
logger.info("Fetching GDELT events via export CDN (multi-file)...")
|
||||
|
||||
# Get the latest export URL to determine current timestamp
|
||||
# Get the latest export URL to determine current timestamp.
|
||||
# HTTPS is used to prevent passive network observers from injecting
|
||||
# poisoned export records into the global incident map via MITM.
|
||||
# GDELT serves the same content over HTTPS as HTTP.
|
||||
index_res = fetch_with_curl(
|
||||
"http://data.gdeltproject.org/gdeltv2/lastupdate.txt", timeout=10
|
||||
"https://data.gdeltproject.org/gdeltv2/lastupdate.txt", timeout=10
|
||||
)
|
||||
if index_res.status_code != 200:
|
||||
logger.error(f"GDELT lastupdate failed: {index_res.status_code}")
|
||||
@@ -636,7 +639,9 @@ def fetch_global_military_incidents():
|
||||
logger.error("Could not find GDELT export URL")
|
||||
return []
|
||||
|
||||
# Extract timestamp from URL like: http://data.gdeltproject.org/gdeltv2/20260301120000.export.CSV.zip
|
||||
# Extract timestamp from URL like: https://data.gdeltproject.org/gdeltv2/20260301120000.export.CSV.zip
|
||||
# (GDELT's lastupdate.txt may still list URLs with http:// — we ignore
|
||||
# the scheme there and reconstruct each download URL as https:// below.)
|
||||
import re
|
||||
|
||||
ts_match = re.search(r"(\d{14})\.export\.CSV\.zip", latest_url)
|
||||
@@ -652,7 +657,7 @@ def fetch_global_military_incidents():
|
||||
for i in range(NUM_FILES):
|
||||
ts = latest_ts - timedelta(minutes=15 * i)
|
||||
fname = ts.strftime("%Y%m%d%H%M%S") + ".export.CSV.zip"
|
||||
url = f"http://data.gdeltproject.org/gdeltv2/{fname}"
|
||||
url = f"https://data.gdeltproject.org/gdeltv2/{fname}"
|
||||
urls.append(url)
|
||||
|
||||
logger.info(f"Downloading {len(urls)} GDELT export files...")
|
||||
|
||||
@@ -34,6 +34,20 @@ kiwisdr_cache: TTLCache = TTLCache(maxsize=1, ttl=_REFRESH_SECONDS)
|
||||
|
||||
_SOURCE_URL = "http://rx.linkfanel.net/kiwisdr_com.js"
|
||||
_CACHE_FILE = Path(__file__).resolve().parent.parent / "data" / "kiwisdr_cache.json"
|
||||
# Bundled fallback — shipped with the codebase so the KiwiSDR layer always
|
||||
# has something to render even when the upstream is unreachable, returns
|
||||
# garbage, or appears to have been tampered with. Issue #206: the upstream
|
||||
# only speaks HTTP, so we can't rely on TLS for integrity — instead we
|
||||
# validate the response's shape and fall back to this bundle if it doesn't
|
||||
# look right.
|
||||
_BUNDLED_FALLBACK = Path(__file__).resolve().parent.parent / "data" / "kiwisdr_directory.json"
|
||||
|
||||
# Minimum number of receivers we expect from a healthy upstream response.
|
||||
# The KiwiSDR public network has consistently sat well above this threshold
|
||||
# for years. If we see fewer than this many parsed receivers, treat the
|
||||
# response as suspect and fall back. Tune via env if the upstream shrinks
|
||||
# legitimately.
|
||||
_MIN_HEALTHY_RECEIVER_COUNT = 50
|
||||
_LINE_COMMENT_RE = re.compile(r"^\s*//.*$", re.MULTILINE)
|
||||
_VAR_PREFIX_RE = re.compile(r"^\s*var\s+kiwisdr_com\s*=\s*", re.MULTILINE)
|
||||
_TRAILING_COMMA_RE = re.compile(r",(\s*[\]}])")
|
||||
@@ -135,12 +149,72 @@ def _parse_mirror_payload(body: str) -> list[dict]:
|
||||
return nodes
|
||||
|
||||
|
||||
def _validate_fetched_nodes(nodes: list[dict]) -> bool:
|
||||
"""Sanity-check freshly-fetched receiver data before trusting it.
|
||||
|
||||
The upstream (rx.linkfanel.net) speaks only HTTP — there is no TLS to
|
||||
authenticate the response. A passive MITM could inject doctored
|
||||
receiver positions (false pins on the map) or strip the response down
|
||||
to a tiny subset. We can't prevent the modification at the transport
|
||||
layer, but we can refuse to commit to obviously-bad responses.
|
||||
|
||||
Returns True if the parsed list looks reasonable. False means we
|
||||
should fall back to a previously-cached or bundled directory.
|
||||
"""
|
||||
if not isinstance(nodes, list):
|
||||
return False
|
||||
if len(nodes) < _MIN_HEALTHY_RECEIVER_COUNT:
|
||||
# Either upstream is degraded or someone is feeding us a stripped
|
||||
# response. Either way, the bundled fallback is more useful.
|
||||
return False
|
||||
|
||||
# Spot-check: every entry should have a name, a parsed lat/lon, and a
|
||||
# URL field. If more than 5% of entries are missing core fields, the
|
||||
# parse went sideways.
|
||||
missing_core = 0
|
||||
for entry in nodes:
|
||||
if not isinstance(entry, dict):
|
||||
missing_core += 1
|
||||
continue
|
||||
if not entry.get("name") or not isinstance(entry.get("lat"), (int, float)):
|
||||
missing_core += 1
|
||||
if missing_core > max(5, len(nodes) // 20):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _load_bundled_fallback() -> list[dict]:
|
||||
"""Last-resort directory shipped with the codebase. Always returns a
|
||||
list (may be empty if the bundle is missing in older deployments)."""
|
||||
if not _BUNDLED_FALLBACK.exists():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(_BUNDLED_FALLBACK.read_text(encoding="utf-8"))
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.warning(f"KiwiSDR bundled fallback unreadable: {e}")
|
||||
return []
|
||||
|
||||
|
||||
@cached(kiwisdr_cache)
|
||||
def fetch_kiwisdr_nodes() -> list[dict]:
|
||||
"""Return the KiwiSDR receiver list, refreshed at most once per day.
|
||||
|
||||
Order of preference: in-memory cache (handled by @cached) → on-disk cache
|
||||
if <24h old → network fetch from rx.linkfanel.net.
|
||||
Layered fallback (issue #206 — upstream is HTTP-only, so we defend with
|
||||
content validation + bundled static directory rather than trying to
|
||||
upgrade the transport):
|
||||
|
||||
1. In-memory cache (handled by @cached on this function)
|
||||
2. On-disk cache if <24h old
|
||||
3. Fresh network fetch from rx.linkfanel.net → validated → committed
|
||||
4. Stale on-disk cache (>24h) if validation fails
|
||||
5. Bundled static directory at backend/data/kiwisdr_directory.json
|
||||
|
||||
The KiwiSDR map layer renders something useful in every case. A
|
||||
tampered upstream returning garbage is caught by _validate_fetched_nodes()
|
||||
and falls through to whatever previously-trusted snapshot we have.
|
||||
"""
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
@@ -153,34 +227,57 @@ def fetch_kiwisdr_nodes() -> list[dict]:
|
||||
return cached_nodes
|
||||
|
||||
# 2. Cache cold or stale — fetch from network.
|
||||
fresh_nodes: list[dict] = []
|
||||
fetch_succeeded = False
|
||||
try:
|
||||
res = fetch_with_curl(_SOURCE_URL, timeout=20)
|
||||
if not res or res.status_code != 200:
|
||||
logger.error(
|
||||
f"KiwiSDR fetch failed: HTTP {res.status_code if res else 'no response'}"
|
||||
if res and res.status_code == 200:
|
||||
fresh_nodes = _parse_mirror_payload(res.text)
|
||||
fetch_succeeded = True
|
||||
else:
|
||||
logger.warning(
|
||||
f"KiwiSDR fetch returned HTTP {res.status_code if res else 'no response'}"
|
||||
)
|
||||
return []
|
||||
|
||||
nodes = _parse_mirror_payload(res.text)
|
||||
if nodes:
|
||||
_save_disk_cache(nodes)
|
||||
logger.info(
|
||||
f"KiwiSDR: refreshed {len(nodes)} receivers from rx.linkfanel.net "
|
||||
"(next refresh in 24h)"
|
||||
)
|
||||
return nodes
|
||||
|
||||
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
|
||||
logger.error(f"KiwiSDR fetch exception: {e}")
|
||||
# Fall back to a stale disk cache if one exists, even if >24h old.
|
||||
if _CACHE_FILE.exists():
|
||||
try:
|
||||
stale = json.loads(_CACHE_FILE.read_text(encoding="utf-8"))
|
||||
if isinstance(stale, list):
|
||||
logger.info(
|
||||
f"KiwiSDR: serving {len(stale)} stale receivers from disk after fetch failure"
|
||||
)
|
||||
return stale
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
logger.warning(f"KiwiSDR fetch exception: {e}")
|
||||
|
||||
# 3. Validate before committing. If the response looks healthy, save
|
||||
# it as the new cache and return.
|
||||
if fetch_succeeded and _validate_fetched_nodes(fresh_nodes):
|
||||
_save_disk_cache(fresh_nodes)
|
||||
logger.info(
|
||||
f"KiwiSDR: refreshed {len(fresh_nodes)} receivers from rx.linkfanel.net "
|
||||
"(next refresh in 24h)"
|
||||
)
|
||||
return fresh_nodes
|
||||
|
||||
if fetch_succeeded:
|
||||
# Network came back, but the payload didn't pass validation —
|
||||
# either upstream is degraded or a MITM is at work. Fall through
|
||||
# to a trusted snapshot rather than committing garbage to disk.
|
||||
logger.warning(
|
||||
"KiwiSDR: upstream response failed validation (%d entries) — "
|
||||
"falling back to trusted snapshot",
|
||||
len(fresh_nodes),
|
||||
)
|
||||
|
||||
# 4. Stale on-disk cache, if any.
|
||||
if _CACHE_FILE.exists():
|
||||
try:
|
||||
stale = json.loads(_CACHE_FILE.read_text(encoding="utf-8"))
|
||||
if isinstance(stale, list) and stale:
|
||||
logger.info(
|
||||
f"KiwiSDR: serving {len(stale)} stale receivers from disk"
|
||||
)
|
||||
return stale
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 5. Bundled static directory — last resort, always works.
|
||||
bundled = _load_bundled_fallback()
|
||||
if bundled:
|
||||
logger.info(
|
||||
f"KiwiSDR: serving {len(bundled)} receivers from bundled fallback "
|
||||
"(no fresh fetch + no disk cache available)"
|
||||
)
|
||||
return bundled
|
||||
|
||||
@@ -1264,6 +1264,21 @@ class DMRelay:
|
||||
)
|
||||
self._save()
|
||||
|
||||
def unregister_prekey_lookup_alias(self, alias: str) -> bool:
|
||||
"""Remove an invite-scoped lookup alias from the local relay."""
|
||||
handle = str(alias or "").strip()
|
||||
if not handle:
|
||||
return False
|
||||
removed = False
|
||||
with self._lock:
|
||||
self._refresh_from_shared_relay()
|
||||
if handle in self._prekey_lookup_aliases:
|
||||
del self._prekey_lookup_aliases[handle]
|
||||
removed = True
|
||||
if removed:
|
||||
self._save()
|
||||
return removed
|
||||
|
||||
def consume_one_time_prekey(self, agent_id: str) -> dict[str, Any] | None:
|
||||
"""Atomically claim the next published one-time prekey for a peer bundle."""
|
||||
claimed: dict[str, Any] | None = None
|
||||
|
||||
@@ -1438,14 +1438,57 @@ class Infonet:
|
||||
# Running counters — avoid O(N) scans in get_info()
|
||||
self._type_counts: dict[str, int] = {}
|
||||
self._active_count: int = 0
|
||||
self._registered_nodes: set[str] = set()
|
||||
self._chain_bytes: int = 2 # Start with "[]" empty JSON array
|
||||
self._dirty = False
|
||||
self._save_lock = threading.Lock()
|
||||
self._save_timer: threading.Timer | None = None
|
||||
self._SAVE_INTERVAL = 5.0 # seconds — coalesce writes
|
||||
# Issue #208: Merkle levels cache so get_merkle_proofs() doesn't
|
||||
# rebuild O(n) levels on every public call. Invalidated whenever
|
||||
# self.events mutates. Computed lazily on first read after an
|
||||
# invalidation.
|
||||
self._merkle_levels_cache: list[list[str]] | None = None
|
||||
self._merkle_levels_for_event_count: int = -1
|
||||
atexit.register(self._flush)
|
||||
self._load()
|
||||
|
||||
def _invalidate_merkle_cache(self) -> None:
|
||||
"""Clear the precomputed Merkle levels.
|
||||
|
||||
Called whenever ``self.events`` may have mutated (append, rebuild,
|
||||
cleanup, fork resolution). The next call to ``get_merkle_root()``
|
||||
or ``get_merkle_proofs()`` will recompute and re-cache.
|
||||
"""
|
||||
self._merkle_levels_cache = None
|
||||
self._merkle_levels_for_event_count = -1
|
||||
|
||||
def _get_merkle_levels(self) -> list[list[str]]:
|
||||
"""Return Merkle levels for the current chain, recomputing if
|
||||
the cache is invalid or out of date.
|
||||
|
||||
Issue #208: a public endpoint (``/api/mesh/infonet/sync?include_proofs=true``)
|
||||
used to rebuild Merkle levels on every request, which is O(n) in
|
||||
chain length and trivially abusable for CPU exhaustion. By caching
|
||||
the levels and invalidating on mutation, repeated proof requests
|
||||
become O(1) per proof; the rebuild only happens after a genuine
|
||||
append/rebuild/cleanup.
|
||||
"""
|
||||
from services.mesh.mesh_merkle import build_merkle_levels
|
||||
|
||||
current_count = len(self.events)
|
||||
if (
|
||||
self._merkle_levels_cache is not None
|
||||
and self._merkle_levels_for_event_count == current_count
|
||||
):
|
||||
return self._merkle_levels_cache
|
||||
|
||||
leaves = [e["event_id"] for e in self.events]
|
||||
levels = build_merkle_levels(leaves)
|
||||
self._merkle_levels_cache = levels
|
||||
self._merkle_levels_for_event_count = current_count
|
||||
return levels
|
||||
|
||||
# ─── Persistence ──────────────────────────────────────────────────
|
||||
|
||||
def _load(self):
|
||||
@@ -1518,6 +1561,7 @@ class Infonet:
|
||||
self._last_validated_index = 0
|
||||
self._type_counts = {}
|
||||
self._active_count = 0
|
||||
self._registered_nodes = set()
|
||||
self._chain_bytes = 2
|
||||
|
||||
def _rebuild_state(self) -> None:
|
||||
@@ -1566,10 +1610,15 @@ class Infonet:
|
||||
now = time.time()
|
||||
self._type_counts = {}
|
||||
self._active_count = 0
|
||||
self._registered_nodes = set()
|
||||
self._chain_bytes = 2 # "[]"
|
||||
for evt in self.events:
|
||||
t = evt.get("event_type", "unknown")
|
||||
self._type_counts[t] = self._type_counts.get(t, 0) + 1
|
||||
if t == "node_register":
|
||||
node_id = str(evt.get("node_id", "") or "")
|
||||
if node_id:
|
||||
self._registered_nodes.add(node_id)
|
||||
is_eph = evt.get("payload", {}).get("ephemeral") or evt.get("payload", {}).get("_ephemeral")
|
||||
if not is_eph or (now - evt.get("timestamp", 0)) < EPHEMERAL_TTL:
|
||||
self._active_count += 1
|
||||
@@ -1579,6 +1628,10 @@ class Infonet:
|
||||
"""Incrementally update counters when a new event is appended."""
|
||||
t = evt.get("event_type", "unknown")
|
||||
self._type_counts[t] = self._type_counts.get(t, 0) + 1
|
||||
if t == "node_register":
|
||||
node_id = str(evt.get("node_id", "") or "")
|
||||
if node_id:
|
||||
self._registered_nodes.add(node_id)
|
||||
self._active_count += 1
|
||||
self._chain_bytes += len(json.dumps(evt)) + 2
|
||||
|
||||
@@ -1972,6 +2025,8 @@ class Infonet:
|
||||
self.head_hash = event.event_id
|
||||
self.node_sequences[node_id] = sequence
|
||||
self._replay_filter.add(event.event_id)
|
||||
# Issue #208: chain advanced, cached Merkle levels are stale.
|
||||
self._invalidate_merkle_cache()
|
||||
self._update_counters_for_event(event_dict)
|
||||
|
||||
if event_type == "key_revoke":
|
||||
@@ -2247,6 +2302,7 @@ class Infonet:
|
||||
self.event_index[event_id] = len(self.events) - 1
|
||||
self.head_hash = event_id
|
||||
self.node_sequences[node_id] = sequence
|
||||
self._update_counters_for_event(evt)
|
||||
accepted += 1
|
||||
expected_prev = event_id
|
||||
self._replay_filter.add(event_id)
|
||||
@@ -2254,6 +2310,9 @@ class Infonet:
|
||||
self._apply_revocation(evt)
|
||||
|
||||
if accepted:
|
||||
# Issue #208: any accepted event invalidates the cached Merkle
|
||||
# levels. One invalidation per batch, not per event.
|
||||
self._invalidate_merkle_cache()
|
||||
self._save()
|
||||
return {"accepted": accepted, "duplicates": duplicates, "rejected": rejected}
|
||||
|
||||
@@ -2552,6 +2611,10 @@ class Infonet:
|
||||
# Apply fork
|
||||
self.events = prefix + ordered
|
||||
self._rebuild_state()
|
||||
self._rebuild_revocations()
|
||||
self._rebuild_counters()
|
||||
# Issue #208: chain replaced, cached Merkle levels are stale.
|
||||
self._invalidate_merkle_cache()
|
||||
self._save()
|
||||
try:
|
||||
from services.mesh.mesh_metrics import increment as metrics_inc
|
||||
@@ -2681,6 +2744,8 @@ class Infonet:
|
||||
"head_hash_full": self.head_hash,
|
||||
"chain_lock": self.chain_lock(),
|
||||
"known_nodes": len(self.node_sequences),
|
||||
"author_nodes": len(self.node_sequences),
|
||||
"registered_nodes": len(self._registered_nodes),
|
||||
"event_types": dict(self._type_counts),
|
||||
"chain_size_kb": round(self._chain_bytes / 1024, 1),
|
||||
"unsigned_events": 0,
|
||||
@@ -2716,8 +2781,11 @@ class Infonet:
|
||||
|
||||
if len(new_events) != before:
|
||||
self.events = new_events
|
||||
# Rebuild index
|
||||
self.event_index = {e["event_id"]: i for i, e in enumerate(self.events)}
|
||||
self._rebuild_state()
|
||||
self._rebuild_revocations()
|
||||
self._rebuild_counters()
|
||||
# Issue #208: cleanup may have dropped expired events.
|
||||
self._invalidate_merkle_cache()
|
||||
self._save()
|
||||
logger.info(f"Infonet cleanup: removed {before - len(new_events)} expired events")
|
||||
|
||||
@@ -2726,30 +2794,37 @@ class Infonet:
|
||||
def get_merkle_root(self) -> str:
|
||||
"""Compute a Merkle root hash of the Infonet for sync comparison.
|
||||
|
||||
Two nodes with the same Merkle root have identical chains.
|
||||
Two nodes with the same Merkle root have identical chains. Reads
|
||||
from the cached Merkle levels (issue #208) — O(1) when the chain
|
||||
hasn't changed since the last computation.
|
||||
"""
|
||||
if not self.events:
|
||||
return GENESIS_HASH
|
||||
|
||||
from services.mesh.mesh_merkle import merkle_root
|
||||
|
||||
leaves = [e["event_id"] for e in self.events]
|
||||
root = merkle_root(leaves)
|
||||
return root or GENESIS_HASH
|
||||
levels = self._get_merkle_levels()
|
||||
if not levels or not levels[-1]:
|
||||
return GENESIS_HASH
|
||||
return levels[-1][0] or GENESIS_HASH
|
||||
|
||||
def get_merkle_proofs(self, start_index: int, count: int) -> dict:
|
||||
"""Return merkle proofs for a contiguous range of events."""
|
||||
leaves = [e["event_id"] for e in self.events]
|
||||
total = len(leaves)
|
||||
"""Return merkle proofs for a contiguous range of events.
|
||||
|
||||
Issue #208: uses the cached Merkle levels so this is O(count *
|
||||
log n) per request, not O(n + count * log n). Anonymous peers
|
||||
hitting ``/api/mesh/infonet/sync?include_proofs=true`` no longer
|
||||
force a rebuild on every call.
|
||||
"""
|
||||
total = len(self.events)
|
||||
if total == 0:
|
||||
return {"root": GENESIS_HASH, "total": 0, "start": 0, "proofs": []}
|
||||
|
||||
from services.mesh.mesh_merkle import build_merkle_levels, merkle_proof_from_levels
|
||||
from services.mesh.mesh_merkle import merkle_proof_from_levels
|
||||
|
||||
leaves = [e["event_id"] for e in self.events]
|
||||
start = max(0, start_index)
|
||||
end = min(total, start + max(0, count))
|
||||
levels = build_merkle_levels(leaves)
|
||||
root = levels[-1][0] if levels else GENESIS_HASH
|
||||
levels = self._get_merkle_levels()
|
||||
root = levels[-1][0] if levels and levels[-1] else GENESIS_HASH
|
||||
|
||||
proofs = []
|
||||
for idx in range(start, end):
|
||||
|
||||
@@ -30,10 +30,19 @@ def eligible_sync_peers(records: list[PeerRecord], *, now: float | None = None)
|
||||
for record in records
|
||||
if record.bucket == "sync" and record.enabled and int(record.cooldown_until or 0) <= current_time
|
||||
]
|
||||
|
||||
def _seed_priority(record: PeerRecord) -> int:
|
||||
role = str(record.role or "").strip().lower()
|
||||
source = str(record.source or "").strip().lower()
|
||||
if role == "seed" and source in {"bundle", "bootstrap_promoted"}:
|
||||
return 0
|
||||
return 1
|
||||
|
||||
return sorted(
|
||||
candidates,
|
||||
key=lambda record: (
|
||||
-int(record.last_sync_ok_at or 0),
|
||||
_seed_priority(record),
|
||||
int(record.failure_count or 0),
|
||||
int(record.added_at or 0),
|
||||
record.peer_url,
|
||||
|
||||
@@ -258,6 +258,12 @@ class PeerStore:
|
||||
self._records[record.record_key()] = record
|
||||
return record
|
||||
|
||||
explicit_seed_refresh = (
|
||||
record.bucket == "sync"
|
||||
and record.role == "seed"
|
||||
and record.source in {"bundle", "bootstrap_promoted"}
|
||||
)
|
||||
|
||||
merged = PeerRecord(
|
||||
bucket=record.bucket,
|
||||
source=record.source,
|
||||
@@ -272,9 +278,9 @@ class PeerStore:
|
||||
last_seen_at=max(existing.last_seen_at, record.last_seen_at),
|
||||
last_sync_ok_at=max(existing.last_sync_ok_at, record.last_sync_ok_at),
|
||||
last_push_ok_at=max(existing.last_push_ok_at, record.last_push_ok_at),
|
||||
last_error=record.last_error or existing.last_error,
|
||||
failure_count=max(existing.failure_count, record.failure_count),
|
||||
cooldown_until=max(existing.cooldown_until, record.cooldown_until),
|
||||
last_error="" if explicit_seed_refresh else record.last_error or existing.last_error,
|
||||
failure_count=0 if explicit_seed_refresh else max(existing.failure_count, record.failure_count),
|
||||
cooldown_until=0 if explicit_seed_refresh else max(existing.cooldown_until, record.cooldown_until),
|
||||
metadata={**existing.metadata, **record.metadata},
|
||||
)
|
||||
self._records[record.record_key()] = merged
|
||||
|
||||
@@ -520,7 +520,7 @@ class MeshtasticTransport:
|
||||
|
||||
def _on_connect(client, userdata, flags, rc):
|
||||
if rc == 0:
|
||||
info = client.publish(topic, payload, qos=0)
|
||||
info = client.publish(topic, payload, qos=1)
|
||||
info.wait_for_publish(timeout=5)
|
||||
published[0] = True
|
||||
client.disconnect()
|
||||
@@ -550,9 +550,9 @@ class MeshtasticTransport:
|
||||
True,
|
||||
self.NAME,
|
||||
(
|
||||
f"Published direct to !{to_node:08x} via {region}/{channel}"
|
||||
f"Broker accepted direct publish to !{to_node:08x} via {region}/{channel}"
|
||||
if direct_node is not None
|
||||
else f"Published to {region}/{channel} ({len(payload)}B protobuf)"
|
||||
else f"Broker accepted channel publish to {region}/{channel} ({len(payload)}B protobuf)"
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
|
||||
@@ -11,12 +11,13 @@ import base64
|
||||
import hmac
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import secrets
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import ed25519
|
||||
from cryptography.hazmat.primitives.asymmetric import ed25519, x25519
|
||||
|
||||
from services.mesh.mesh_crypto import (
|
||||
build_signature_payload,
|
||||
@@ -51,6 +52,8 @@ PREKEY_LOOKUP_ROTATE_BEFORE_REMAINING_USES = 8
|
||||
PREKEY_LOOKUP_ROTATION_OVERLAP_S = 12 * 60 * 60
|
||||
PREKEY_LOOKUP_ROTATION_ACTIVE_CAP = 4
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _safe_int(val, default=0) -> int:
|
||||
try:
|
||||
@@ -107,6 +110,7 @@ def _default_identity() -> dict[str, Any]:
|
||||
def _prekey_lookup_handle_record(
|
||||
handle: str,
|
||||
*,
|
||||
label: str = "",
|
||||
issued_at: int = 0,
|
||||
expires_at: int = 0,
|
||||
max_uses: int = 0,
|
||||
@@ -125,6 +129,7 @@ def _prekey_lookup_handle_record(
|
||||
bounded_max_uses = max(1, _safe_int(max_uses or PREKEY_LOOKUP_HANDLE_MAX_USES, PREKEY_LOOKUP_HANDLE_MAX_USES))
|
||||
return {
|
||||
"handle": str(handle or "").strip(),
|
||||
"label": str(label or "").strip()[:96],
|
||||
"issued_at": issued,
|
||||
"expires_at": bounded_expires_at,
|
||||
"max_uses": bounded_max_uses,
|
||||
@@ -152,8 +157,10 @@ def _coerce_prekey_lookup_handle_record(
|
||||
max_uses = _safe_int(value.get("max_uses", PREKEY_LOOKUP_HANDLE_MAX_USES) or PREKEY_LOOKUP_HANDLE_MAX_USES)
|
||||
use_count = _safe_int(value.get("use_count", value.get("uses", 0)) or 0, 0)
|
||||
last_used_at = _safe_int(value.get("last_used_at", value.get("last_used", 0)) or 0, 0)
|
||||
label = str(value.get("label", "") or "").strip()
|
||||
return _prekey_lookup_handle_record(
|
||||
handle,
|
||||
label=label,
|
||||
issued_at=issued_at,
|
||||
expires_at=expires_at,
|
||||
max_uses=max_uses,
|
||||
@@ -228,6 +235,23 @@ def _fresh_prekey_lookup_handle_record(*, now: int | None = None) -> dict[str, A
|
||||
)
|
||||
|
||||
|
||||
def _prekey_registration_failure_blocks_dm_invite(detail: str) -> bool:
|
||||
"""Only trust-root failures block address export; transport warm-up can finish later."""
|
||||
lowered = str(detail or "").lower()
|
||||
critical_markers = (
|
||||
"root transparency",
|
||||
"external root witness",
|
||||
"stable root",
|
||||
"witness threshold",
|
||||
"witness finality",
|
||||
"root manifest",
|
||||
"root witness",
|
||||
"manifest_fingerprint",
|
||||
"policy fingerprint",
|
||||
)
|
||||
return any(marker in lowered for marker in critical_markers)
|
||||
|
||||
|
||||
def _bounded_lookup_handle_records(
|
||||
records: list[dict[str, Any]],
|
||||
*,
|
||||
@@ -440,6 +464,37 @@ def _bundle_fingerprint(data: dict[str, Any]) -> str:
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _ensure_dm_dh_material(data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
||||
"""Repair legacy/corrupt DM identities that kept signing keys but lost DH material."""
|
||||
if str(data.get("dh_pub_key", "") or "").strip() and str(data.get("dh_private_key", "") or "").strip():
|
||||
return data, False
|
||||
|
||||
dh_priv = x25519.X25519PrivateKey.generate()
|
||||
dh_priv_raw = dh_priv.private_bytes(
|
||||
encoding=serialization.Encoding.Raw,
|
||||
format=serialization.PrivateFormat.Raw,
|
||||
encryption_algorithm=serialization.NoEncryption(),
|
||||
)
|
||||
dh_pub_raw = dh_priv.public_key().public_bytes(
|
||||
encoding=serialization.Encoding.Raw,
|
||||
format=serialization.PublicFormat.Raw,
|
||||
)
|
||||
repaired = {
|
||||
**dict(data or {}),
|
||||
"dh_pub_key": base64.b64encode(dh_pub_raw).decode("ascii"),
|
||||
"dh_algo": "X25519",
|
||||
"dh_private_key": base64.b64encode(dh_priv_raw).decode("ascii"),
|
||||
"last_dh_timestamp": int(time.time()),
|
||||
"bundle_fingerprint": "",
|
||||
"bundle_sequence": 0,
|
||||
"bundle_registered_at": 0,
|
||||
"prekey_bundle_registered_at": 0,
|
||||
"prekey_transparency_head": "",
|
||||
"prekey_transparency_size": 0,
|
||||
}
|
||||
return _write_identity(repaired), True
|
||||
|
||||
|
||||
def trust_fingerprint_for_identity_material(
|
||||
*,
|
||||
agent_id: str,
|
||||
@@ -806,10 +861,11 @@ def _sign_dm_invite_payload(
|
||||
|
||||
def register_wormhole_dm_key(force: bool = False) -> dict[str, Any]:
|
||||
data = read_wormhole_identity()
|
||||
data, repaired_dh = _ensure_dm_dh_material(data)
|
||||
|
||||
timestamp = int(time.time())
|
||||
fingerprint = _bundle_fingerprint(data)
|
||||
if not force and fingerprint and fingerprint == data.get("bundle_fingerprint"):
|
||||
if not force and not repaired_dh and fingerprint and fingerprint == data.get("bundle_fingerprint"):
|
||||
return {
|
||||
"ok": True,
|
||||
**_public_view(data),
|
||||
@@ -884,6 +940,7 @@ def export_wormhole_dm_invite(*, label: str = "", expires_in_s: int = 0) -> dict
|
||||
existing_handles.append(
|
||||
_prekey_lookup_handle_record(
|
||||
lookup_handle,
|
||||
label=str(label or "").strip(),
|
||||
issued_at=issued_at,
|
||||
expires_at=expires_at,
|
||||
)
|
||||
@@ -920,14 +977,25 @@ def export_wormhole_dm_invite(*, label: str = "", expires_in_s: int = 0) -> dict
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
prekey_registration: dict[str, Any] = {"ok": False, "detail": "prekey bundle publish not attempted"}
|
||||
try:
|
||||
from services.mesh.mesh_wormhole_prekey import register_wormhole_prekey_bundle
|
||||
|
||||
registered = register_wormhole_prekey_bundle()
|
||||
if not registered.get("ok"):
|
||||
return {"ok": False, "detail": str(registered.get("detail", "") or "prekey bundle registration failed")}
|
||||
prekey_registration = register_wormhole_prekey_bundle()
|
||||
if not prekey_registration.get("ok"):
|
||||
detail = str(prekey_registration.get("detail", "") or "prekey bundle registration failed")
|
||||
if _prekey_registration_failure_blocks_dm_invite(detail):
|
||||
return {"ok": False, "detail": detail}
|
||||
logger.warning(
|
||||
"DM invite prekey publish pending: %s",
|
||||
detail,
|
||||
)
|
||||
except Exception as exc:
|
||||
return {"ok": False, "detail": str(exc) or "prekey bundle registration failed"}
|
||||
prekey_registration = {"ok": False, "detail": str(exc) or "prekey bundle registration failed"}
|
||||
detail = str(prekey_registration.get("detail", "") or "")
|
||||
if _prekey_registration_failure_blocks_dm_invite(detail):
|
||||
return {"ok": False, "detail": detail}
|
||||
logger.warning("DM invite prekey publish pending: %s", prekey_registration["detail"])
|
||||
|
||||
invite_node_id, invite_public_key, invite_private_key = _generate_invite_signing_identity()
|
||||
payload = _attach_dm_invite_root_distribution(payload)
|
||||
@@ -958,6 +1026,8 @@ def export_wormhole_dm_invite(*, label: str = "", expires_in_s: int = 0) -> dict
|
||||
"peer_id": str(invite_node_id or ""),
|
||||
"trust_fingerprint": str(payload.get("identity_commitment", "") or ""),
|
||||
"invite": invite,
|
||||
"prekey_publish_pending": not bool(prekey_registration.get("ok")),
|
||||
"prekey_registration": prekey_registration,
|
||||
}
|
||||
|
||||
|
||||
@@ -980,6 +1050,140 @@ def get_prekey_lookup_handle_records() -> list[dict[str, Any]]:
|
||||
]
|
||||
|
||||
|
||||
def list_prekey_lookup_handle_records_for_ui(*, now: int | None = None) -> dict[str, Any]:
|
||||
"""Return shareable DM address records without exposing local identity secrets."""
|
||||
current_time = _safe_int(now or time.time(), int(time.time()))
|
||||
addresses: list[dict[str, Any]] = []
|
||||
for record in get_prekey_lookup_handle_records():
|
||||
handle = str(record.get("handle", "") or "").strip()
|
||||
if not handle:
|
||||
continue
|
||||
expires_at = _effective_prekey_lookup_handle_expires_at(record)
|
||||
max_uses = max(
|
||||
1,
|
||||
_safe_int(
|
||||
record.get("max_uses", PREKEY_LOOKUP_HANDLE_MAX_USES) or PREKEY_LOOKUP_HANDLE_MAX_USES,
|
||||
PREKEY_LOOKUP_HANDLE_MAX_USES,
|
||||
),
|
||||
)
|
||||
use_count = max(0, _safe_int(record.get("use_count", 0) or 0, 0))
|
||||
addresses.append(
|
||||
{
|
||||
"handle": handle,
|
||||
"label": str(record.get("label", "") or "").strip(),
|
||||
"issued_at": _safe_int(record.get("issued_at", 0) or 0, 0),
|
||||
"expires_at": expires_at,
|
||||
"max_uses": max_uses,
|
||||
"use_count": use_count,
|
||||
"remaining_uses": max(0, max_uses - use_count),
|
||||
"last_used_at": _safe_int(record.get("last_used_at", 0) or 0, 0),
|
||||
"expired": bool(expires_at > 0 and current_time >= expires_at),
|
||||
"exhausted": bool(use_count >= max_uses),
|
||||
}
|
||||
)
|
||||
addresses.sort(key=lambda item: _safe_int(item.get("issued_at", 0) or 0, 0), reverse=True)
|
||||
return {"ok": True, "addresses": addresses}
|
||||
|
||||
|
||||
def rename_prekey_lookup_handle(handle: str, label: str) -> dict[str, Any]:
|
||||
"""Rename an active invite-scoped DM lookup handle without changing the handle."""
|
||||
lookup_handle = str(handle or "").strip()
|
||||
next_label = str(label or "").strip()[:96]
|
||||
if not lookup_handle:
|
||||
return {"ok": False, "detail": "missing_lookup_handle"}
|
||||
|
||||
current_time = int(time.time())
|
||||
data = read_wormhole_identity()
|
||||
existing, _ = _normalize_prekey_lookup_handles(
|
||||
data.get("prekey_lookup_handles", []),
|
||||
fallback_issued_at=current_time,
|
||||
now=current_time,
|
||||
)
|
||||
updated = False
|
||||
next_records: list[dict[str, Any]] = []
|
||||
for record in existing:
|
||||
current = dict(record)
|
||||
if str(current.get("handle", "") or "").strip() == lookup_handle:
|
||||
current["label"] = next_label
|
||||
updated = True
|
||||
next_records.append(current)
|
||||
|
||||
if not updated:
|
||||
return {
|
||||
"ok": False,
|
||||
"handle": lookup_handle,
|
||||
"label": next_label,
|
||||
"updated": False,
|
||||
"detail": "lookup_handle_not_found",
|
||||
}
|
||||
|
||||
normalized_records, _ = _normalize_prekey_lookup_handles(
|
||||
next_records,
|
||||
fallback_issued_at=current_time,
|
||||
now=current_time,
|
||||
)
|
||||
_write_identity({"prekey_lookup_handles": normalized_records})
|
||||
return {
|
||||
"ok": True,
|
||||
"handle": lookup_handle,
|
||||
"label": next_label,
|
||||
"updated": True,
|
||||
}
|
||||
|
||||
|
||||
def revoke_prekey_lookup_handle(handle: str) -> dict[str, Any]:
|
||||
"""Revoke an invite-scoped DM lookup handle for future first-contact attempts."""
|
||||
lookup_handle = str(handle or "").strip()
|
||||
if not lookup_handle:
|
||||
return {"ok": False, "detail": "missing_lookup_handle"}
|
||||
current_time = int(time.time())
|
||||
data = read_wormhole_identity()
|
||||
existing, _ = _normalize_prekey_lookup_handles(
|
||||
data.get("prekey_lookup_handles", []),
|
||||
fallback_issued_at=current_time,
|
||||
now=current_time,
|
||||
)
|
||||
next_records = [
|
||||
dict(record)
|
||||
for record in existing
|
||||
if str(record.get("handle", "") or "").strip() != lookup_handle
|
||||
]
|
||||
identity_removed = len(next_records) != len(existing)
|
||||
if identity_removed:
|
||||
_write_identity({"prekey_lookup_handles": next_records})
|
||||
|
||||
relay_removed = False
|
||||
try:
|
||||
from services.mesh.mesh_dm_relay import dm_relay
|
||||
|
||||
relay_removed = bool(dm_relay.unregister_prekey_lookup_alias(lookup_handle))
|
||||
except Exception:
|
||||
relay_removed = False
|
||||
|
||||
republished = False
|
||||
detail = ""
|
||||
if identity_removed:
|
||||
try:
|
||||
from services.mesh.mesh_wormhole_prekey import register_wormhole_prekey_bundle
|
||||
|
||||
registered = register_wormhole_prekey_bundle()
|
||||
republished = bool(registered.get("ok"))
|
||||
if not republished:
|
||||
detail = str(registered.get("detail", "") or "prekey bundle republish failed")
|
||||
except Exception as exc:
|
||||
detail = str(exc) or "prekey bundle republish failed"
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"handle": lookup_handle,
|
||||
"revoked": bool(identity_removed or relay_removed),
|
||||
"identity_removed": identity_removed,
|
||||
"relay_removed": relay_removed,
|
||||
"republished": republished,
|
||||
"detail": detail,
|
||||
}
|
||||
|
||||
|
||||
def record_prekey_lookup_handle_use(handle: str, *, now: int | None = None) -> dict[str, Any] | None:
|
||||
lookup_handle = str(handle or "").strip()
|
||||
if not lookup_handle:
|
||||
@@ -999,6 +1203,7 @@ def record_prekey_lookup_handle_use(handle: str, *, now: int | None = None) -> d
|
||||
if str(current.get("handle", "") or "").strip() == lookup_handle:
|
||||
current = _prekey_lookup_handle_record(
|
||||
lookup_handle,
|
||||
label=str(current.get("label", "") or "").strip(),
|
||||
issued_at=_safe_int(current.get("issued_at", 0) or 0, current_time),
|
||||
expires_at=_safe_int(current.get("expires_at", 0) or 0, 0),
|
||||
max_uses=_safe_int(current.get("max_uses", PREKEY_LOOKUP_HANDLE_MAX_USES) or PREKEY_LOOKUP_HANDLE_MAX_USES),
|
||||
@@ -1129,6 +1334,7 @@ def maybe_rotate_prekey_lookup_handles(*, now: int | None = None) -> dict[str, A
|
||||
candidate_records.append(
|
||||
_prekey_lookup_handle_record(
|
||||
old_handle,
|
||||
label=str(record.get("label", "") or "").strip(),
|
||||
issued_at=_safe_int(record.get("issued_at", 0) or 0, current_time),
|
||||
expires_at=overlap_expires_at,
|
||||
max_uses=_safe_int(record.get("max_uses", PREKEY_LOOKUP_HANDLE_MAX_USES) or PREKEY_LOOKUP_HANDLE_MAX_USES),
|
||||
@@ -1351,11 +1557,101 @@ def import_wormhole_dm_invite(invite: dict[str, Any], *, alias: str = "") -> dic
|
||||
"detail": "compat dm invite import disabled; ask the sender to re-export a current signed invite",
|
||||
}
|
||||
|
||||
def _prekey_missing_or_pending(detail: str) -> bool:
|
||||
lower = str(detail or "").strip().lower()
|
||||
return any(
|
||||
phrase in lower
|
||||
for phrase in (
|
||||
"prekey bundle not found",
|
||||
"invite prekey bundle not found",
|
||||
"peer prekey lookup unavailable",
|
||||
"peer prekey lookup still preparing",
|
||||
"transport tier insufficient",
|
||||
"preparing_private_lane",
|
||||
)
|
||||
)
|
||||
|
||||
def _pin_pending_invite_prekey(detail: str) -> dict[str, Any]:
|
||||
if invite_version < DM_INVITE_VERSION:
|
||||
return {"ok": False, "detail": detail or "invite prekey bundle not found"}
|
||||
invite_root_distribution = _verify_dm_invite_root_distribution(payload)
|
||||
if not invite_root_distribution.get("ok"):
|
||||
return invite_root_distribution
|
||||
attested = _verify_dm_invite_identity_attestation(
|
||||
envelope=envelope,
|
||||
payload=payload,
|
||||
resolved_root_node_id=str(invite_root_distribution.get("root_node_id", "") or ""),
|
||||
resolved_root_public_key=str(invite_root_distribution.get("root_public_key", "") or ""),
|
||||
resolved_root_public_key_algo=str(
|
||||
invite_root_distribution.get("root_public_key_algo", "Ed25519") or "Ed25519"
|
||||
),
|
||||
resolved_root_manifest_fingerprint=str(
|
||||
invite_root_distribution.get("root_manifest_fingerprint", "") or ""
|
||||
).strip().lower(),
|
||||
)
|
||||
if not attested.get("ok"):
|
||||
return attested
|
||||
pending_peer_id = str(verified.get("peer_id", "") or "").strip()
|
||||
trust_fingerprint = str(verified.get("trust_fingerprint", "") or "").strip().lower()
|
||||
contact = pin_wormhole_dm_invite(
|
||||
pending_peer_id,
|
||||
invite_payload={
|
||||
"trust_fingerprint": trust_fingerprint,
|
||||
"public_key": "",
|
||||
"public_key_algo": "Ed25519",
|
||||
"identity_dh_pub_key": "",
|
||||
"dh_algo": "X25519",
|
||||
"prekey_lookup_handle": lookup_handle,
|
||||
"issued_at": int(payload.get("issued_at", 0) or 0),
|
||||
"expires_at": int(payload.get("expires_at", 0) or 0),
|
||||
"label": str(payload.get("label", "") or ""),
|
||||
"root_node_id": str(attested.get("root_node_id", "") or ""),
|
||||
"root_public_key": str(attested.get("root_public_key", "") or ""),
|
||||
"root_public_key_algo": str(attested.get("root_public_key_algo", "Ed25519") or "Ed25519"),
|
||||
"root_fingerprint": str(attested.get("root_fingerprint", "") or ""),
|
||||
"root_manifest_fingerprint": str(invite_root_distribution.get("root_manifest_fingerprint", "") or ""),
|
||||
"root_witness_policy_fingerprint": str(
|
||||
invite_root_distribution.get("root_witness_policy_fingerprint", "") or ""
|
||||
),
|
||||
"root_witness_threshold": _safe_int(
|
||||
invite_root_distribution.get("root_witness_threshold", 0) or 0,
|
||||
0,
|
||||
),
|
||||
"root_witness_count": _safe_int(invite_root_distribution.get("root_witness_count", 0) or 0, 0),
|
||||
"root_witness_domain_count": _safe_int(
|
||||
invite_root_distribution.get("root_witness_domain_count", 0) or 0,
|
||||
0,
|
||||
),
|
||||
"root_manifest_generation": _safe_int(
|
||||
invite_root_distribution.get("root_manifest_generation", 0) or 0,
|
||||
0,
|
||||
),
|
||||
"root_rotation_proven": bool(invite_root_distribution.get("root_rotation_proven")),
|
||||
},
|
||||
alias=resolved_alias,
|
||||
attested=True,
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"peer_id": pending_peer_id,
|
||||
"invite_peer_id": pending_peer_id,
|
||||
"trust_fingerprint": trust_fingerprint,
|
||||
"trust_level": str(contact.get("trust_level", "") or ""),
|
||||
"detail": "Contact saved.",
|
||||
"invite_attested": True,
|
||||
"pending_prekey": True,
|
||||
"prekey_detail": detail or "invite prekey bundle not found",
|
||||
"contact": contact,
|
||||
}
|
||||
|
||||
from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle
|
||||
|
||||
fetched = fetch_dm_prekey_bundle(lookup_token=lookup_handle)
|
||||
if not fetched.get("ok"):
|
||||
return {"ok": False, "detail": str(fetched.get("detail", "") or "invite prekey bundle not found")}
|
||||
fetch_detail = str(fetched.get("detail", "") or "invite prekey bundle not found")
|
||||
if _prekey_missing_or_pending(fetch_detail):
|
||||
return _pin_pending_invite_prekey(fetch_detail)
|
||||
return {"ok": False, "detail": fetch_detail}
|
||||
|
||||
resolved_peer_id = str(fetched.get("agent_id", "") or "").strip()
|
||||
if not resolved_peer_id:
|
||||
|
||||
@@ -11,6 +11,7 @@ import os
|
||||
import random
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
@@ -150,6 +151,122 @@ def _fetch_dm_prekey_bundle_from_peer_lookup(lookup_token: str) -> dict[str, Any
|
||||
return {"ok": False, "detail": last_detail or "Prekey bundle not found"}
|
||||
|
||||
|
||||
def _configured_public_lookup_peer_urls() -> list[str]:
|
||||
try:
|
||||
from services.config import get_settings
|
||||
from services.mesh.mesh_router import active_sync_peer_urls, parse_configured_relay_peers
|
||||
|
||||
settings = get_settings()
|
||||
candidates: list[str] = []
|
||||
for raw in (
|
||||
getattr(settings, "MESH_BOOTSTRAP_SEED_PEERS", ""),
|
||||
getattr(settings, "MESH_DEFAULT_SYNC_PEERS", ""),
|
||||
):
|
||||
candidates.extend(parse_configured_relay_peers(str(raw or "")))
|
||||
candidates.extend(active_sync_peer_urls())
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
seen: set[str] = set()
|
||||
peers: list[str] = []
|
||||
for candidate in candidates:
|
||||
peer = str(candidate or "").strip().rstrip("/")
|
||||
if not peer or peer in seen:
|
||||
continue
|
||||
seen.add(peer)
|
||||
peers.append(peer)
|
||||
return peers
|
||||
|
||||
|
||||
def _normalize_remote_lookup_bundle(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
data = dict(payload or {})
|
||||
bundle = dict(data.get("bundle") or {})
|
||||
public_key = str(data.get("public_key", "") or bundle.get("public_key", "") or "").strip()
|
||||
if not public_key:
|
||||
return {"ok": False, "detail": "Prekey bundle missing signing key"}
|
||||
agent_id = str(data.get("agent_id", "") or "").strip() or derive_node_id(public_key)
|
||||
if not agent_id:
|
||||
return {"ok": False, "detail": "Prekey bundle public key binding mismatch"}
|
||||
data["agent_id"] = agent_id
|
||||
data["public_key"] = public_key
|
||||
data["public_key_algo"] = str(data.get("public_key_algo", "") or bundle.get("public_key_algo", "Ed25519") or "Ed25519")
|
||||
data["protocol_version"] = str(data.get("protocol_version", "") or bundle.get("protocol_version", PROTOCOL_VERSION) or PROTOCOL_VERSION)
|
||||
data["bundle"] = bundle
|
||||
ok, reason = _validate_bundle_record(data)
|
||||
if not ok:
|
||||
return {"ok": False, "detail": reason}
|
||||
data["ok"] = True
|
||||
data["lookup_mode"] = "invite_lookup_handle"
|
||||
data["public_lookup"] = True
|
||||
return data
|
||||
|
||||
|
||||
def _fetch_dm_prekey_bundle_from_public_lookup(lookup_token: str) -> dict[str, Any]:
|
||||
"""Fetch an invite-scoped prekey bundle from bootstrap/sync peers.
|
||||
|
||||
The token is high-entropy and invite-scoped. This path does not expose a
|
||||
stable agent_id to the peer; if the ordinary peer response omits agent_id,
|
||||
derive it from the signed identity public key and validate the bundle before
|
||||
accepting it.
|
||||
"""
|
||||
token = str(lookup_token or "").strip()
|
||||
if not token:
|
||||
return {"ok": False, "detail": "lookup token required"}
|
||||
peers = _configured_public_lookup_peer_urls()
|
||||
if not peers:
|
||||
return {"ok": False, "detail": "peer prekey lookup unavailable"}
|
||||
try:
|
||||
from services.config import get_settings
|
||||
|
||||
timeout = max(1, _safe_int(getattr(get_settings(), "MESH_SYNC_TIMEOUT_S", 5) or 5, 5))
|
||||
except Exception:
|
||||
timeout = 5
|
||||
|
||||
encoded = urllib.parse.urlencode({"lookup_token": token})
|
||||
last_detail = ""
|
||||
for peer_url in peers:
|
||||
normalized_peer_url = str(peer_url or "").strip().rstrip("/")
|
||||
if not normalized_peer_url:
|
||||
continue
|
||||
# Generic UA: any peer-facing crypto request should not carry a
|
||||
# fork-specific identifier — that turns prekey lookups into a
|
||||
# software-fingerprinting beacon.
|
||||
from services.network_utils import DEFAULT_USER_AGENT
|
||||
request = urllib.request.Request(
|
||||
f"{normalized_peer_url}/api/mesh/dm/prekey-bundle?{encoded}",
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"User-Agent": DEFAULT_USER_AGENT,
|
||||
},
|
||||
method="GET",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
raw = response.read(256 * 1024)
|
||||
payload = json.loads(raw.decode("utf-8"))
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, OSError) as exc:
|
||||
logger.debug("public prekey lookup failed for %s: %s", normalized_peer_url, type(exc).__name__)
|
||||
last_detail = "peer prekey lookup unavailable"
|
||||
continue
|
||||
if not isinstance(payload, dict):
|
||||
last_detail = "invalid peer response"
|
||||
continue
|
||||
if payload.get("pending") or str(payload.get("status", "") or "") == "preparing_private_lane":
|
||||
last_detail = "peer prekey lookup still preparing"
|
||||
continue
|
||||
if not payload.get("ok"):
|
||||
last_detail = str(payload.get("detail", "") or last_detail or "Prekey bundle not found")
|
||||
continue
|
||||
if not isinstance(payload.get("bundle"), dict):
|
||||
last_detail = "Prekey bundle not found"
|
||||
continue
|
||||
normalized = _normalize_remote_lookup_bundle(payload)
|
||||
if normalized.get("ok"):
|
||||
return normalized
|
||||
last_detail = str(normalized.get("detail", "") or last_detail)
|
||||
return {"ok": False, "detail": last_detail or "Prekey bundle not found"}
|
||||
|
||||
|
||||
def _b64(data: bytes) -> str:
|
||||
return base64.b64encode(data).decode("ascii")
|
||||
|
||||
@@ -926,6 +1043,11 @@ def fetch_dm_prekey_bundle(
|
||||
peer_found = _fetch_dm_prekey_bundle_from_peer_lookup(resolved_lookup)
|
||||
if peer_found.get("ok"):
|
||||
return peer_found
|
||||
public_found = _fetch_dm_prekey_bundle_from_public_lookup(resolved_lookup)
|
||||
if public_found.get("ok"):
|
||||
return public_found
|
||||
if str(public_found.get("detail", "") or "").strip():
|
||||
return {"ok": False, "detail": str(public_found.get("detail", "") or "Prekey bundle not found")}
|
||||
return {"ok": False, "detail": str(peer_found.get("detail", "") or "Prekey bundle not found")}
|
||||
else:
|
||||
return {"ok": False, "detail": "Prekey bundle not found"}
|
||||
|
||||
@@ -12,6 +12,7 @@ from __future__ import annotations
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -23,7 +24,7 @@ from cryptography.hazmat.primitives.asymmetric import ed25519
|
||||
|
||||
from services.mesh.mesh_crypto import build_signature_payload, derive_node_id, verify_node_binding, verify_signature
|
||||
from services.mesh.mesh_protocol import PROTOCOL_VERSION
|
||||
from services.mesh.mesh_secure_storage import read_domain_json, write_domain_json
|
||||
from services.mesh.mesh_secure_storage import SecureStorageError, read_domain_json, write_domain_json
|
||||
from services.mesh.mesh_wormhole_identity import root_identity_fingerprint_for_material
|
||||
from services.mesh.mesh_wormhole_persona import (
|
||||
bootstrap_wormhole_persona_state,
|
||||
@@ -51,6 +52,7 @@ DEFAULT_ROOT_WITNESS_THRESHOLD = 2
|
||||
DEFAULT_ROOT_WITNESS_MANAGEMENT_SCOPE = "local"
|
||||
DEFAULT_ROOT_WITNESS_INDEPENDENCE_GROUP = "local_system"
|
||||
DEFAULT_ROOT_EXTERNAL_WITNESS_MAX_AGE_S = 3600
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _safe_int(val: Any, default: int = 0) -> int:
|
||||
@@ -461,12 +463,22 @@ def witness_policy_fingerprint(policy: dict[str, Any]) -> str:
|
||||
|
||||
|
||||
def read_root_distribution_state() -> dict[str, Any]:
|
||||
raw = read_domain_json(
|
||||
ROOT_DISTRIBUTION_DOMAIN,
|
||||
ROOT_DISTRIBUTION_FILE,
|
||||
_default_state,
|
||||
base_dir=DATA_DIR,
|
||||
)
|
||||
try:
|
||||
raw = read_domain_json(
|
||||
ROOT_DISTRIBUTION_DOMAIN,
|
||||
ROOT_DISTRIBUTION_FILE,
|
||||
_default_state,
|
||||
base_dir=DATA_DIR,
|
||||
)
|
||||
except SecureStorageError as exc:
|
||||
detail = str(exc)
|
||||
if "Failed to decrypt domain JSON" not in detail:
|
||||
raise
|
||||
logger.warning(
|
||||
"Root distribution state could not decrypt; regenerating local witness distribution: %s",
|
||||
detail,
|
||||
)
|
||||
raw = _default_state()
|
||||
state = {**_default_state(), **dict(raw or {})}
|
||||
state["witness_identity"] = {**_empty_witness_identity(), **dict(state.get("witness_identity") or {})}
|
||||
witness_identities, witness_changed = _normalize_witness_identities(
|
||||
|
||||
@@ -108,8 +108,18 @@ def normalize_topic_filter(value: str) -> str | None:
|
||||
return "/".join(parts)
|
||||
|
||||
|
||||
def _default_topic_for_root(root: str) -> str:
|
||||
return f"msh/{root}/2/e/{DEFAULT_CHANNEL}/#"
|
||||
def _default_topics_for_root(root: str) -> list[str]:
|
||||
"""Return the default LongFast subscriptions for a region root.
|
||||
|
||||
The public broker carries protobuf/encrypted traffic under ``/e/`` and
|
||||
companion decoded JSON traffic under ``/json/``. Positions often arrive on
|
||||
the protobuf path, while public text is commonly easiest to observe on the
|
||||
JSON path.
|
||||
"""
|
||||
return [
|
||||
f"msh/{root}/2/e/{DEFAULT_CHANNEL}/#",
|
||||
f"msh/{root}/2/json/{DEFAULT_CHANNEL}/#",
|
||||
]
|
||||
|
||||
|
||||
def build_subscription_topics(
|
||||
@@ -124,7 +134,11 @@ def build_subscription_topics(
|
||||
# via MESH_MQTT_EXTRA_ROOTS to avoid flooding the public broker.
|
||||
roots.extend(root for root in (normalize_root(item) for item in _split_config_values(extra_roots)) if root)
|
||||
|
||||
topics = [_default_topic_for_root(root) for root in _dedupe(roots)]
|
||||
topics = [
|
||||
topic
|
||||
for root in _dedupe(roots)
|
||||
for topic in _default_topics_for_root(root)
|
||||
]
|
||||
topics.extend(
|
||||
topic
|
||||
for topic in (
|
||||
|
||||
@@ -19,6 +19,17 @@ _retry = Retry(total=1, backoff_factor=0.3, status_forcelist=[502, 503, 504])
|
||||
_session.mount("https://", HTTPAdapter(max_retries=_retry, pool_maxsize=20))
|
||||
_session.mount("http://", HTTPAdapter(max_retries=_retry, pool_maxsize=10))
|
||||
|
||||
|
||||
# Default outbound User-Agent. Generic by design — does NOT include any
|
||||
# personal contact info or a fork-specific repo URL. Operators who run a
|
||||
# public-facing relay and want to identify themselves to upstreams (e.g.
|
||||
# for Nominatim / weather.gov usage-policy compliance) can override this
|
||||
# via the SHADOWBROKER_USER_AGENT env var.
|
||||
DEFAULT_USER_AGENT = os.environ.get(
|
||||
"SHADOWBROKER_USER_AGENT",
|
||||
"ShadowBroker-OSINT/0.9",
|
||||
)
|
||||
|
||||
# Find bash for curl fallback — Git bash's curl has the TLS features
|
||||
# needed to pass CDN fingerprint checks (brotli, zstd, libpsl)
|
||||
|
||||
@@ -73,7 +84,7 @@ def fetch_with_curl(url, method="GET", json_data=None, timeout=15, headers=None,
|
||||
both Python requests and the barebones Windows system curl.
|
||||
"""
|
||||
default_headers = {
|
||||
"User-Agent": "ShadowBroker-OSINT/0.9.75 (+https://github.com/BigBodyCobain/Shadowbroker; contact: bigbodycobain@gmail.com)",
|
||||
"User-Agent": DEFAULT_USER_AGENT,
|
||||
}
|
||||
if headers:
|
||||
default_headers.update(headers)
|
||||
|
||||
@@ -6,8 +6,8 @@ Docs: https://pskreporter.info/pskdev.html
|
||||
"""
|
||||
|
||||
import logging
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
import defusedxml.ElementTree as ET
|
||||
import requests
|
||||
from cachetools import TTLCache, cached
|
||||
|
||||
|
||||
@@ -131,27 +131,61 @@ def get_recent_openmhz_calls(sys_name: str):
|
||||
return []
|
||||
|
||||
|
||||
_OPENMHZ_MAX_REDIRECTS = 5
|
||||
|
||||
|
||||
def openmhz_audio_response(target_url: str):
|
||||
"""Fetch an OpenMHz audio object through the backend with browser-safe headers."""
|
||||
"""Fetch an OpenMHz audio object through the backend with browser-safe headers.
|
||||
|
||||
Redirects are followed manually so each hop's host can be re-validated
|
||||
against ``_OPENMHZ_AUDIO_HOSTS``. Without this, the upstream could
|
||||
302-redirect to an internal address (e.g. ``http://127.0.0.1:8000/...``
|
||||
or an RFC1918 range), and the backend would dutifully fetch and stream
|
||||
that response back to the browser — a classic open-redirect-to-SSRF
|
||||
chain. Same-host redirects (CDN edge selection) still work normally.
|
||||
"""
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
from urllib.parse import urljoin
|
||||
|
||||
parsed = urlparse(str(target_url or ""))
|
||||
host = (parsed.hostname or "").lower()
|
||||
if parsed.scheme != "https" or host not in _OPENMHZ_AUDIO_HOSTS:
|
||||
raise HTTPException(status_code=400, detail="Unsupported OpenMHz audio URL")
|
||||
|
||||
current_url = target_url
|
||||
hops = 0
|
||||
try:
|
||||
upstream = requests.get(
|
||||
target_url,
|
||||
stream=True,
|
||||
timeout=(5, 20),
|
||||
headers={
|
||||
"User-Agent": "Mozilla/5.0",
|
||||
"Accept": "audio/mpeg,audio/*,*/*;q=0.8",
|
||||
"Referer": "https://openmhz.com/",
|
||||
},
|
||||
)
|
||||
while True:
|
||||
upstream = requests.get(
|
||||
current_url,
|
||||
stream=True,
|
||||
timeout=(5, 20),
|
||||
allow_redirects=False,
|
||||
headers={
|
||||
"User-Agent": "Mozilla/5.0",
|
||||
"Accept": "audio/mpeg,audio/*,*/*;q=0.8",
|
||||
"Referer": "https://openmhz.com/",
|
||||
},
|
||||
)
|
||||
if upstream.is_redirect or upstream.status_code in (301, 302, 303, 307, 308):
|
||||
location = upstream.headers.get("Location", "")
|
||||
upstream.close()
|
||||
if hops >= _OPENMHZ_MAX_REDIRECTS or not location:
|
||||
raise HTTPException(status_code=502, detail="OpenMHz redirect rejected")
|
||||
next_url = urljoin(current_url, location)
|
||||
next_parsed = urlparse(next_url)
|
||||
next_host = (next_parsed.hostname or "").lower()
|
||||
# Re-validate the next hop against the same allowlist used for
|
||||
# the original URL. Cross-host redirects to disallowed hosts
|
||||
# are rejected silently; the browser audio element handles
|
||||
# the resulting 502 gracefully and moves on.
|
||||
if next_parsed.scheme != "https" or next_host not in _OPENMHZ_AUDIO_HOSTS:
|
||||
raise HTTPException(status_code=502, detail="OpenMHz redirect rejected")
|
||||
current_url = next_url
|
||||
hops += 1
|
||||
continue
|
||||
break
|
||||
except requests.RequestException as exc:
|
||||
raise HTTPException(status_code=502, detail="OpenMHz audio fetch failed") from exc
|
||||
|
||||
|
||||
@@ -14,6 +14,11 @@ class HealthResponse(BaseModel):
|
||||
# ({status, age_s, row_count, slo, stale, empty, description}).
|
||||
slo: Optional[Dict[str, Any]] = None
|
||||
slo_summary: Optional[Dict[str, int]] = None
|
||||
# Issue #258: AIS proxy status — currently exposes ``degraded_tls``
|
||||
# (bool), true when ais_proxy.js fell back to the SPKI-pinned
|
||||
# insecure-date path because the upstream Let's Encrypt cert is
|
||||
# expired. Empty dict / null means no status reported yet.
|
||||
ais_proxy: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class RefreshResponse(BaseModel):
|
||||
|
||||
@@ -20,7 +20,7 @@ from cachetools import TTLCache
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SHODAN_BASE = "https://api.shodan.io"
|
||||
_USER_AGENT = "ShadowBroker/0.9.75 local Shodan connector"
|
||||
_USER_AGENT = "ShadowBroker/0.9.79 local Shodan connector"
|
||||
_REQUEST_TIMEOUT = 15
|
||||
_MIN_INTERVAL_SECONDS = 1.05 # Shodan docs say API plans are rate limited to ~1 req/sec.
|
||||
_DEFAULT_SEARCH_PAGES = 1
|
||||
|
||||
@@ -545,6 +545,198 @@ class MeshtasticBridge:
|
||||
self._message_dedupe[key] = now
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _message_dedupe_key(message: dict) -> str:
|
||||
sender = str(message.get("from") or "???").strip().lower()
|
||||
recipient = str(message.get("to") or "broadcast").strip().lower()
|
||||
text = str(message.get("text") or "").strip()
|
||||
channel = str(message.get("channel") or "LongFast").strip().lower()
|
||||
root = str(message.get("root") or message.get("region") or "").strip().lower()
|
||||
if root == "us":
|
||||
root = "us"
|
||||
return f"{sender}:{recipient}:{root}:{channel}:{text}"
|
||||
|
||||
def append_text_message(self, message: dict, *, dedupe_window_s: float = 5.0) -> bool:
|
||||
"""Append a Meshtastic text message unless it is a near-immediate echo."""
|
||||
if not str(message.get("text") or "").strip():
|
||||
return False
|
||||
now = time.time()
|
||||
cutoff = now - max(1.0, dedupe_window_s)
|
||||
next_message = dict(message)
|
||||
next_message.setdefault("to", "broadcast")
|
||||
next_message.setdefault("channel", "LongFast")
|
||||
next_message.setdefault("timestamp", datetime.utcnow().isoformat() + "Z")
|
||||
key = self._message_dedupe_key(next_message)
|
||||
for existing in list(self.messages)[:40]:
|
||||
if self._message_dedupe_key(existing) != key:
|
||||
continue
|
||||
try:
|
||||
existing_ts_raw = existing.get("timestamp")
|
||||
existing_ts = (
|
||||
datetime.fromisoformat(str(existing_ts_raw).replace("Z", "+00:00")).timestamp()
|
||||
if existing_ts_raw
|
||||
else now
|
||||
)
|
||||
except Exception:
|
||||
existing_ts = now
|
||||
if existing_ts >= cutoff:
|
||||
if not existing.get("root") and next_message.get("root"):
|
||||
existing["root"] = next_message.get("root")
|
||||
if not existing.get("region") and next_message.get("region"):
|
||||
existing["region"] = next_message.get("region")
|
||||
return False
|
||||
self.messages.appendleft(next_message)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _coerce_node_ref(value) -> str:
|
||||
"""Normalize Meshtastic node identifiers into the public !xxxxxxxx form."""
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, int):
|
||||
return f"!{value & 0xFFFFFFFF:08x}"
|
||||
raw = str(value).strip()
|
||||
if not raw:
|
||||
return ""
|
||||
if raw.startswith("!"):
|
||||
return raw
|
||||
lowered = raw.lower()
|
||||
if lowered.startswith("0x"):
|
||||
try:
|
||||
return f"!{int(lowered, 16) & 0xFFFFFFFF:08x}"
|
||||
except ValueError:
|
||||
return raw
|
||||
if raw.isdigit():
|
||||
try:
|
||||
return f"!{int(raw) & 0xFFFFFFFF:08x}"
|
||||
except ValueError:
|
||||
return raw
|
||||
if len(raw) == 8 and all(ch in "0123456789abcdefABCDEF" for ch in raw):
|
||||
return f"!{raw.lower()}"
|
||||
return raw
|
||||
|
||||
@staticmethod
|
||||
def _first_text_value(*values) -> str:
|
||||
for value in values:
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode("utf-8", errors="replace")
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
if text:
|
||||
return MeshtasticBridge._repair_text_mojibake(text)
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _repair_text_mojibake(text: str) -> str:
|
||||
"""Repair common UTF-8-as-Latin-1 mojibake from MQTT JSON bridges."""
|
||||
if not text or not any(marker in text for marker in ("Ã", "Ð", "Ñ")):
|
||||
return text
|
||||
try:
|
||||
repaired = text.encode("latin-1").decode("utf-8").strip()
|
||||
except UnicodeError:
|
||||
return text
|
||||
if repaired and repaired != text:
|
||||
return repaired
|
||||
return text
|
||||
|
||||
@staticmethod
|
||||
def _first_present(*values):
|
||||
for value in values:
|
||||
if value is not None and value != "":
|
||||
return value
|
||||
return None
|
||||
|
||||
def _extract_json_text_message(self, data: dict, topic: str) -> dict | None:
|
||||
"""Extract a public Meshtastic text event from decoded MQTT JSON.
|
||||
|
||||
Meshtastic JSON brokers are not perfectly uniform. Some packets expose
|
||||
text at the top level, some under ``decoded`` or ``payload``. Keep this
|
||||
permissive for receive, but only return messages with non-empty text.
|
||||
"""
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
topic_meta = parse_topic_metadata(topic)
|
||||
packet = data.get("packet") if isinstance(data.get("packet"), dict) else {}
|
||||
decoded = data.get("decoded") if isinstance(data.get("decoded"), dict) else {}
|
||||
payload_obj = data.get("payload")
|
||||
payload = payload_obj if isinstance(payload_obj, dict) else {}
|
||||
decoded_payload_obj = decoded.get("payload") if decoded else None
|
||||
decoded_payload = decoded_payload_obj if isinstance(decoded_payload_obj, dict) else {}
|
||||
|
||||
text = self._first_text_value(
|
||||
data.get("text"),
|
||||
data.get("message"),
|
||||
data.get("msg"),
|
||||
payload_obj if isinstance(payload_obj, str) else "",
|
||||
payload.get("text"),
|
||||
payload.get("message"),
|
||||
payload.get("msg"),
|
||||
payload.get("payload") if isinstance(payload.get("payload"), str) else "",
|
||||
decoded.get("text"),
|
||||
decoded.get("message"),
|
||||
decoded.get("payload") if isinstance(decoded.get("payload"), str) else "",
|
||||
decoded_payload.get("text"),
|
||||
decoded_payload.get("message"),
|
||||
decoded_payload.get("msg"),
|
||||
)
|
||||
if not text:
|
||||
return None
|
||||
|
||||
sender = self._coerce_node_ref(
|
||||
self._first_present(
|
||||
data.get("from"),
|
||||
data.get("fromId"),
|
||||
data.get("from_id"),
|
||||
data.get("sender"),
|
||||
data.get("senderId"),
|
||||
data.get("sender_id"),
|
||||
packet.get("from"),
|
||||
packet.get("fromId"),
|
||||
packet.get("from_id"),
|
||||
decoded.get("from"),
|
||||
)
|
||||
)
|
||||
recipient = self._coerce_node_ref(
|
||||
self._first_present(
|
||||
data.get("to"),
|
||||
data.get("toId"),
|
||||
data.get("to_id"),
|
||||
data.get("recipient"),
|
||||
data.get("recipientId"),
|
||||
data.get("recipient_id"),
|
||||
packet.get("to"),
|
||||
packet.get("toId"),
|
||||
packet.get("to_id"),
|
||||
decoded.get("to"),
|
||||
)
|
||||
)
|
||||
if not recipient or recipient in {"!ffffffff", "broadcast"}:
|
||||
recipient = "broadcast"
|
||||
|
||||
timestamp = datetime.utcnow().isoformat() + "Z"
|
||||
rx_time = self._first_present(
|
||||
data.get("rxTime"),
|
||||
data.get("rx_time"),
|
||||
data.get("timestamp"),
|
||||
packet.get("rxTime"),
|
||||
packet.get("timestamp"),
|
||||
)
|
||||
if isinstance(rx_time, (int, float)) and rx_time > 0:
|
||||
try:
|
||||
timestamp = datetime.fromtimestamp(float(rx_time), tz=timezone.utc).isoformat()
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
|
||||
return {
|
||||
"from": sender or topic.split("/")[-1],
|
||||
"to": recipient,
|
||||
"text": text[:500],
|
||||
"region": topic_meta["region"],
|
||||
"root": topic_meta["root"],
|
||||
"channel": topic_meta["channel"],
|
||||
"timestamp": timestamp,
|
||||
}
|
||||
|
||||
def start(self):
|
||||
if self._thread and self._thread.is_alive():
|
||||
if not self._stop.is_set():
|
||||
@@ -693,6 +885,9 @@ class MeshtasticBridge:
|
||||
if "/json/" in topic:
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
text_message = self._extract_json_text_message(data, topic)
|
||||
if text_message:
|
||||
self.append_text_message(text_message, dedupe_window_s=30.0)
|
||||
if self._rate_limited():
|
||||
return
|
||||
self._ingest_data(data, topic)
|
||||
@@ -715,7 +910,7 @@ class MeshtasticBridge:
|
||||
topic_meta["root"],
|
||||
):
|
||||
return
|
||||
self.messages.appendleft(
|
||||
self.append_text_message(
|
||||
{
|
||||
"from": data.get("from", "???"),
|
||||
"to": recipient,
|
||||
|
||||
@@ -64,6 +64,203 @@ def _find_tor_binary() -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
# Baked-in expected digest list. Loaded lazily; populated by maintainers
|
||||
# when a new Tor Expert Bundle URL is added to _TOR_EXPERT_BUNDLE_URLS.
|
||||
# See issue #201 for rationale.
|
||||
_TOR_DIGEST_FILE = Path(__file__).resolve().parent.parent / "data" / "tor_bundle_digests.json"
|
||||
_DIGEST_PLACEHOLDER = "PLACEHOLDER_REPLACE_BEFORE_RELEASE"
|
||||
|
||||
|
||||
def _load_baked_in_digests() -> dict[str, str]:
|
||||
"""Return {url: expected_sha256_lower} for URLs we ship a known digest for.
|
||||
|
||||
Entries whose value is the placeholder sentinel are filtered out — they
|
||||
represent versions the maintainer has not yet pinned, and we don't
|
||||
want to trust them via this layer.
|
||||
"""
|
||||
if not _TOR_DIGEST_FILE.exists():
|
||||
return {}
|
||||
try:
|
||||
import json as _json
|
||||
raw = _json.loads(_TOR_DIGEST_FILE.read_text(encoding="utf-8"))
|
||||
except Exception as exc:
|
||||
logger.warning("Tor bundle digests file unreadable: %s", exc)
|
||||
return {}
|
||||
result: dict[str, str] = {}
|
||||
for k, v in raw.items():
|
||||
if not isinstance(k, str) or k.startswith("_"):
|
||||
continue
|
||||
if not isinstance(v, str) or v == _DIGEST_PLACEHOLDER:
|
||||
continue
|
||||
result[k] = v.strip().lower()
|
||||
return result
|
||||
|
||||
|
||||
def _verify_tor_bundle(archive_path: Path, bundle_url: str) -> tuple[bool, str]:
|
||||
"""Verify the downloaded Tor bundle against any source we trust.
|
||||
|
||||
Returns (verified, reason). The bundle is considered verified if EITHER:
|
||||
|
||||
* The upstream ``.sha256sum`` file is reachable AND its digest matches
|
||||
what we just downloaded, OR
|
||||
* Our baked-in digest list (``backend/data/tor_bundle_digests.json``)
|
||||
contains this URL AND that digest matches.
|
||||
|
||||
If both sources are unavailable (e.g. fresh checkout before the
|
||||
maintainer has populated the digest file AND the upstream
|
||||
``.sha256sum`` is unreachable), we **fall back to HTTPS-only trust**
|
||||
with a warning so first-run onboarding does not break. As soon as the
|
||||
digest file is populated for a shipped Tor version, the secure path
|
||||
activates automatically — no operator action required.
|
||||
|
||||
Issue #201.
|
||||
"""
|
||||
import hashlib
|
||||
|
||||
actual_hash = hashlib.sha256(archive_path.read_bytes()).hexdigest().lower()
|
||||
|
||||
# Source 1: upstream .sha256sum
|
||||
upstream_hash: str | None = None
|
||||
sha256_url = bundle_url + ".sha256sum"
|
||||
sha256_file = TOR_INSTALL_DIR / "sha256sum.txt"
|
||||
try:
|
||||
urlretrieve(sha256_url, str(sha256_file))
|
||||
upstream_hash = sha256_file.read_text().strip().split()[0].lower()
|
||||
sha256_file.unlink(missing_ok=True)
|
||||
except Exception as hash_err:
|
||||
logger.info("Tor bundle upstream .sha256sum unreachable: %s", hash_err)
|
||||
sha256_file.unlink(missing_ok=True)
|
||||
|
||||
if upstream_hash and upstream_hash == actual_hash:
|
||||
return True, f"verified via upstream .sha256sum ({actual_hash[:16]}...)"
|
||||
|
||||
# Source 2: baked-in digest list
|
||||
baked = _load_baked_in_digests()
|
||||
baked_hash = baked.get(bundle_url)
|
||||
if baked_hash and baked_hash == actual_hash:
|
||||
return True, f"verified via baked-in digest list ({actual_hash[:16]}...)"
|
||||
|
||||
# If we got an upstream digest AND a baked-in digest AND neither
|
||||
# matched, the bundle is genuinely suspect — refuse it.
|
||||
if upstream_hash and baked_hash:
|
||||
return False, (
|
||||
f"SHA-256 mismatch: archive={actual_hash[:16]}..., "
|
||||
f"upstream={upstream_hash[:16]}..., baked={baked_hash[:16]}..."
|
||||
)
|
||||
if upstream_hash and upstream_hash != actual_hash:
|
||||
return False, (
|
||||
f"SHA-256 mismatch vs upstream: archive={actual_hash[:16]}..., "
|
||||
f"upstream={upstream_hash[:16]}..."
|
||||
)
|
||||
if baked_hash and baked_hash != actual_hash:
|
||||
return False, (
|
||||
f"SHA-256 mismatch vs baked-in digest: archive={actual_hash[:16]}..., "
|
||||
f"expected={baked_hash[:16]}..."
|
||||
)
|
||||
|
||||
# Neither verification source available. This is the fallback path for
|
||||
# the case where the upstream .sha256sum is temporarily unreachable
|
||||
# AND the maintainer hasn't yet pinned this Tor version. Trust HTTPS
|
||||
# only (current behavior pre-#201) with a clear warning. Onboarding
|
||||
# works; once we populate the digest file, the secure path activates.
|
||||
logger.warning(
|
||||
"Tor bundle integrity check fell back to HTTPS-only trust "
|
||||
"(upstream .sha256sum unreachable AND no baked-in digest for %s). "
|
||||
"Add this URL's SHA-256 to backend/data/tor_bundle_digests.json "
|
||||
"to enable the secure path.",
|
||||
bundle_url,
|
||||
)
|
||||
return True, f"https-only (no digest source reachable, archive={actual_hash[:16]}...)"
|
||||
|
||||
|
||||
def _extract_tor_bundle_safely(archive_path: Path, install_dir: Path) -> bool:
|
||||
"""Extract a Tor Expert Bundle tar.gz safely.
|
||||
|
||||
Issue #251: the previous extractor checked tarinfo.name against path
|
||||
traversal but never inspected tarinfo.linkname for symlink/hardlink
|
||||
members. Python 3.11's tarfile honors symlinks during extractall(),
|
||||
so a malicious archive could ship a member like::
|
||||
|
||||
name = "innocent.txt" # passes the path check
|
||||
type = SYMTYPE
|
||||
linkname = "C:\\Windows\\System32\\config\\system"
|
||||
|
||||
and extractall() would then create that symlink. Subsequent reads
|
||||
of innocent.txt deference to a sensitive system file; subsequent
|
||||
writes corrupt one. Tor bundles never legitimately contain symlinks
|
||||
or hardlinks, so we refuse all link members categorically rather
|
||||
than trying to validate linkname targets (which has its own pitfalls
|
||||
around relative path resolution).
|
||||
|
||||
Also refuses non-regular-non-directory members (devices, FIFOs,
|
||||
character/block special files) for completeness — none of those
|
||||
belong in a Tor Expert Bundle and accepting them is a category of
|
||||
bug we don't need to debug later.
|
||||
|
||||
Returns True on success, False on rejection (and logs the reason).
|
||||
The caller is responsible for cleaning up the archive file.
|
||||
"""
|
||||
import tarfile
|
||||
|
||||
install_resolved = install_dir.resolve()
|
||||
|
||||
try:
|
||||
with tarfile.open(str(archive_path), "r:gz") as tar:
|
||||
for member in tar.getmembers():
|
||||
# Reject anything that isn't a regular file or directory.
|
||||
# Symlinks (SYMTYPE) and hardlinks (LNKTYPE) are the
|
||||
# path-traversal vectors; the others (CHRTYPE, BLKTYPE,
|
||||
# FIFOTYPE, CONTTYPE) have no legitimate use in a Tor
|
||||
# Expert Bundle.
|
||||
if member.issym() or member.islnk():
|
||||
logger.error(
|
||||
"Tor bundle extraction blocked: link member %s -> %s "
|
||||
"(symlinks/hardlinks are not allowed in Tor bundles; "
|
||||
"this archive is malformed or hostile)",
|
||||
member.name,
|
||||
member.linkname,
|
||||
)
|
||||
return False
|
||||
if not (member.isfile() or member.isdir()):
|
||||
logger.error(
|
||||
"Tor bundle extraction blocked: unexpected member type "
|
||||
"for %s (only regular files and directories are allowed)",
|
||||
member.name,
|
||||
)
|
||||
return False
|
||||
|
||||
# Path traversal check (preserves the original guard).
|
||||
try:
|
||||
member_path = (install_dir / member.name).resolve()
|
||||
except OSError as exc:
|
||||
logger.error(
|
||||
"Tor bundle extraction blocked: cannot resolve member "
|
||||
"path %s: %s",
|
||||
member.name,
|
||||
exc,
|
||||
)
|
||||
return False
|
||||
try:
|
||||
member_path.relative_to(install_resolved)
|
||||
except ValueError:
|
||||
logger.error(
|
||||
"Tor bundle extraction blocked: path traversal on %s "
|
||||
"(resolves to %s, outside install dir %s)",
|
||||
member.name,
|
||||
member_path,
|
||||
install_resolved,
|
||||
)
|
||||
return False
|
||||
|
||||
# All members validated — extract.
|
||||
tar.extractall(path=str(install_dir))
|
||||
except tarfile.TarError as exc:
|
||||
logger.error("Tor bundle extraction failed: malformed tar (%s)", exc)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _auto_install_tor() -> str | None:
|
||||
"""Install or download Tor when it is safe to do so."""
|
||||
if os.name != "nt":
|
||||
@@ -79,37 +276,24 @@ def _auto_install_tor() -> str | None:
|
||||
logger.info("Downloading Tor Expert Bundle over HTTPS from %s...", bundle_url)
|
||||
urlretrieve(bundle_url, str(archive_path))
|
||||
|
||||
sha256_url = bundle_url + ".sha256sum"
|
||||
sha256_file = TOR_INSTALL_DIR / "sha256sum.txt"
|
||||
try:
|
||||
urlretrieve(sha256_url, str(sha256_file))
|
||||
expected_hash = sha256_file.read_text().strip().split()[0].lower()
|
||||
import hashlib
|
||||
|
||||
actual_hash = hashlib.sha256(archive_path.read_bytes()).hexdigest().lower()
|
||||
sha256_file.unlink(missing_ok=True)
|
||||
if actual_hash != expected_hash:
|
||||
logger.error("SHA-256 mismatch for Tor download. Expected %s, got %s", expected_hash, actual_hash)
|
||||
archive_path.unlink(missing_ok=True)
|
||||
continue
|
||||
logger.info("SHA-256 verified: %s", actual_hash[:16] + "...")
|
||||
except Exception as hash_err:
|
||||
logger.warning(
|
||||
"Could not verify SHA-256 (hash file unavailable): %s; proceeding with HTTPS-only verification",
|
||||
hash_err,
|
||||
)
|
||||
# Issue #201: multi-source verification. If neither upstream
|
||||
# .sha256sum nor a baked-in digest matches, we refuse this URL
|
||||
# and try the next one in _TOR_EXPERT_BUNDLE_URLS. If neither
|
||||
# source is reachable at all, we fall back to HTTPS-only trust
|
||||
# (current behavior) rather than blocking onboarding.
|
||||
verified, reason = _verify_tor_bundle(archive_path, bundle_url)
|
||||
if not verified:
|
||||
logger.error("Tor bundle verification failed for %s: %s", bundle_url, reason)
|
||||
archive_path.unlink(missing_ok=True)
|
||||
continue
|
||||
logger.info("Tor bundle %s", reason)
|
||||
|
||||
logger.info("Download complete, extracting...")
|
||||
import tarfile
|
||||
|
||||
with tarfile.open(str(archive_path), "r:gz") as tar:
|
||||
for member in tar.getmembers():
|
||||
member_path = (TOR_INSTALL_DIR / member.name).resolve()
|
||||
if not str(member_path).startswith(str(TOR_INSTALL_DIR.resolve())):
|
||||
logger.error("Tar path traversal blocked: %s", member.name)
|
||||
archive_path.unlink(missing_ok=True)
|
||||
return None
|
||||
tar.extractall(path=str(TOR_INSTALL_DIR))
|
||||
if not _extract_tor_bundle_safely(archive_path, TOR_INSTALL_DIR):
|
||||
archive_path.unlink(missing_ok=True)
|
||||
return None
|
||||
|
||||
archive_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def disable_public_mesh_lane(*, reason: str = "private_lane_enabled") -> dict[str, Any]:
|
||||
"""Disable public Meshtastic MQTT before private Wormhole/Infonet starts."""
|
||||
result: dict[str, Any] = {
|
||||
"ok": True,
|
||||
"reason": reason,
|
||||
"settings_disabled": False,
|
||||
"runtime_stopped": False,
|
||||
}
|
||||
|
||||
# Scheduled Wormhole prewarm must not mutate the user's explicit public
|
||||
# MeshChat session. Only a deliberate private-lane activation should sever
|
||||
# the public MQTT lane.
|
||||
normalized_reason = str(reason or "").strip().lower()
|
||||
if normalized_reason == "wormhole_scheduled_prewarm" or normalized_reason.endswith(":scheduled_prewarm"):
|
||||
try:
|
||||
from services.meshtastic_mqtt_settings import mqtt_bridge_enabled
|
||||
|
||||
if mqtt_bridge_enabled():
|
||||
logger.info("Keeping public Mesh lane active during Wormhole prewarm: %s", reason)
|
||||
result["skipped"] = True
|
||||
result["skip_reason"] = "public_mesh_user_enabled"
|
||||
return result
|
||||
except Exception as exc:
|
||||
logger.debug("Could not inspect public Mesh state during %s: %s", reason, exc)
|
||||
|
||||
logger.info("Disabling public Mesh lane: %s", reason)
|
||||
|
||||
try:
|
||||
from services.meshtastic_mqtt_settings import write_meshtastic_mqtt_settings
|
||||
|
||||
settings = write_meshtastic_mqtt_settings(enabled=False)
|
||||
result["settings_disabled"] = not bool(settings.get("enabled"))
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to disable public Mesh settings during %s: %s", reason, exc)
|
||||
result["ok"] = False
|
||||
result["settings_error"] = str(exc)
|
||||
|
||||
try:
|
||||
from services.sigint_bridge import sigint_grid
|
||||
|
||||
if sigint_grid.mesh.is_running():
|
||||
sigint_grid.mesh.stop()
|
||||
result["runtime_stopped"] = not sigint_grid.mesh.is_running()
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to stop public Mesh runtime during %s: %s", reason, exc)
|
||||
result["ok"] = False
|
||||
result["runtime_error"] = str(exc)
|
||||
|
||||
return result
|
||||
@@ -24,7 +24,7 @@ from cachetools import TTLCache
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_FINNHUB_BASE = "https://finnhub.io/api/v1"
|
||||
_USER_AGENT = "ShadowBroker/0.9.75 Finnhub connector"
|
||||
_USER_AGENT = "ShadowBroker/0.9.79 Finnhub connector"
|
||||
_REQUEST_TIMEOUT = 12
|
||||
_MIN_INTERVAL_SECONDS = 0.35 # Stay well under 60 calls/min
|
||||
|
||||
|
||||
+232
-14
@@ -6,9 +6,11 @@ Public API:
|
||||
schedule_restart(project_root) (spawn detached start script, then exit)
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
@@ -29,6 +31,19 @@ DOCKER_UPDATE_COMMANDS = (
|
||||
"docker compose pull && docker compose up -d"
|
||||
)
|
||||
|
||||
# Issue #231: baked-in release digests. Loaded lazily, used as a fallback
|
||||
# verification source when the release's SHA256SUMS.txt asset can't be
|
||||
# fetched (e.g. transient network failure during update).
|
||||
_RELEASE_DIGESTS_FILE = (
|
||||
Path(__file__).resolve().parent.parent / "data" / "release_digests.json"
|
||||
)
|
||||
# Pattern for the maintainer's signed source-archive release asset. This
|
||||
# is the file we prefer over the auto-generated ``zipball_url`` because
|
||||
# the maintainer's build process publishes it with a matching entry in
|
||||
# SHA256SUMS.txt — the zipball does not have a signed digest.
|
||||
_SOURCE_ASSET_PATTERN = re.compile(r"^ShadowBroker_v\d", re.IGNORECASE)
|
||||
_SHA256SUMS_ASSET_NAME = "SHA256SUMS.txt"
|
||||
|
||||
|
||||
def _is_docker() -> bool:
|
||||
"""Detect if we're running inside a Docker container."""
|
||||
@@ -40,7 +55,6 @@ def _is_docker() -> bool:
|
||||
except (FileNotFoundError, PermissionError):
|
||||
pass
|
||||
return os.environ.get("container") == "docker"
|
||||
_EXPECTED_SHA256 = os.environ.get("MESH_UPDATE_SHA256", "").strip().lower()
|
||||
_ALLOWED_UPDATE_HOSTS = {
|
||||
"api.github.com",
|
||||
"codeload.github.com",
|
||||
@@ -119,7 +133,16 @@ def _validate_update_url(url: str, *, allow_release_page: bool = False) -> str:
|
||||
# ---------------------------------------------------------------------------
|
||||
def _download_release(temp_dir: str) -> tuple:
|
||||
"""Fetch latest release info and download the source zip archive.
|
||||
Returns (zip_path, version_tag, download_url, release_url).
|
||||
|
||||
Issue #231: prefer the maintainer's signed release asset (matching
|
||||
``ShadowBroker_v*.zip``) over the auto-generated ``zipball_url``,
|
||||
because the maintainer's release process publishes a matching entry
|
||||
in SHA256SUMS.txt for the named asset but NOT for the zipball.
|
||||
|
||||
Returns (zip_path, version_tag, download_url, release_url, asset_name,
|
||||
sha256sums_url) — the last two are empty strings when the release
|
||||
doesn't publish a signed asset, falling back to the legacy zipball
|
||||
path.
|
||||
"""
|
||||
logger.info("Fetching latest release info from GitHub...")
|
||||
_validate_update_url(GITHUB_RELEASES_URL)
|
||||
@@ -131,9 +154,42 @@ def _download_release(temp_dir: str) -> tuple:
|
||||
tag = release.get("tag_name", "unknown")
|
||||
release_url = str(release.get("html_url") or GITHUB_RELEASES_PAGE_URL).strip()
|
||||
_validate_update_url(release_url, allow_release_page=True)
|
||||
zip_url = str(release.get("zipball_url") or "").strip()
|
||||
if not zip_url:
|
||||
raise RuntimeError("Latest release is missing a source archive URL")
|
||||
|
||||
# Prefer the maintainer-signed release asset. Fall back to the
|
||||
# auto-generated zipball if the release doesn't publish one.
|
||||
assets = release.get("assets") or []
|
||||
asset_name = ""
|
||||
asset_url = ""
|
||||
sha256sums_url = ""
|
||||
for a in assets:
|
||||
name = str(a.get("name") or "").strip()
|
||||
download = str(a.get("browser_download_url") or "").strip()
|
||||
if not name or not download:
|
||||
continue
|
||||
if _SOURCE_ASSET_PATTERN.match(name) and name.lower().endswith(".zip"):
|
||||
asset_name = name
|
||||
asset_url = download
|
||||
elif name == _SHA256SUMS_ASSET_NAME:
|
||||
sha256sums_url = download
|
||||
|
||||
if asset_url:
|
||||
zip_url = asset_url
|
||||
logger.info(
|
||||
"Using signed release asset %s (sha256sums=%s)",
|
||||
asset_name,
|
||||
"yes" if sha256sums_url else "no",
|
||||
)
|
||||
else:
|
||||
zip_url = str(release.get("zipball_url") or "").strip()
|
||||
if not zip_url:
|
||||
raise RuntimeError("Latest release is missing a source archive URL")
|
||||
logger.warning(
|
||||
"Release does not publish a signed ShadowBroker_v*.zip asset — "
|
||||
"falling back to auto-generated zipball_url. Integrity will be "
|
||||
"verified against the baked-in release_digests.json (if present) "
|
||||
"or HTTPS-only otherwise."
|
||||
)
|
||||
|
||||
_validate_update_url(zip_url)
|
||||
|
||||
logger.info(f"Downloading {zip_url} ...")
|
||||
@@ -150,19 +206,174 @@ def _download_release(temp_dir: str) -> tuple:
|
||||
|
||||
size_mb = os.path.getsize(zip_path) / (1024 * 1024)
|
||||
logger.info(f"Downloaded {size_mb:.1f} MB — ZIP validated OK")
|
||||
return zip_path, tag, zip_url, release_url
|
||||
return zip_path, tag, zip_url, release_url, asset_name, sha256sums_url
|
||||
|
||||
|
||||
def _validate_zip_hash(zip_path: str) -> None:
|
||||
if not _EXPECTED_SHA256:
|
||||
return
|
||||
def _compute_sha256(zip_path: str) -> str:
|
||||
"""Return the hex SHA-256 of the file at ``zip_path`` (lowercase)."""
|
||||
h = hashlib.sha256()
|
||||
with open(zip_path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 128), b""):
|
||||
h.update(chunk)
|
||||
digest = h.hexdigest().lower()
|
||||
if digest != _EXPECTED_SHA256:
|
||||
raise RuntimeError("Update SHA-256 mismatch")
|
||||
return h.hexdigest().lower()
|
||||
|
||||
|
||||
def _load_baked_in_release_digests() -> dict:
|
||||
"""Return the ``release_digests.json`` mapping, or an empty dict.
|
||||
|
||||
Schema (issue #231):
|
||||
{
|
||||
"<release_tag>": {
|
||||
"<asset_filename>": "<sha256_hex>",
|
||||
...
|
||||
},
|
||||
...
|
||||
}
|
||||
"""
|
||||
try:
|
||||
raw = _RELEASE_DIGESTS_FILE.read_text(encoding="utf-8")
|
||||
parsed = json.loads(raw)
|
||||
except (OSError, ValueError) as exc:
|
||||
logger.debug("Release digest file unreadable: %s", exc)
|
||||
return {}
|
||||
if not isinstance(parsed, dict):
|
||||
return {}
|
||||
cleaned: dict[str, dict[str, str]] = {}
|
||||
for k, v in parsed.items():
|
||||
if not isinstance(k, str) or k.startswith("_"):
|
||||
continue
|
||||
if isinstance(v, dict):
|
||||
entries = {
|
||||
fname: digest.strip().lower()
|
||||
for fname, digest in v.items()
|
||||
if isinstance(fname, str) and isinstance(digest, str)
|
||||
}
|
||||
if entries:
|
||||
cleaned[k] = entries
|
||||
return cleaned
|
||||
|
||||
|
||||
def _fetch_sha256sums(sha256sums_url: str) -> dict[str, str]:
|
||||
"""Download a SHA256SUMS.txt and return {filename: digest_hex_lower}.
|
||||
|
||||
Standard ``sha256sum`` format: ``<digest> <filename>`` per line. The
|
||||
leading ``*`` binary-mode marker (e.g. ``<digest> *<filename>``) is
|
||||
handled.
|
||||
"""
|
||||
try:
|
||||
_validate_update_url(sha256sums_url)
|
||||
except RuntimeError as exc:
|
||||
logger.warning("SHA256SUMS URL rejected: %s", exc)
|
||||
return {}
|
||||
try:
|
||||
resp = requests.get(sha256sums_url, timeout=15)
|
||||
resp.raise_for_status()
|
||||
except requests.RequestException as exc:
|
||||
logger.info("SHA256SUMS fetch failed: %s", exc)
|
||||
return {}
|
||||
out: dict[str, str] = {}
|
||||
for line in resp.text.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
# Tolerant split: handle both `<digest> <name>` and `<digest> *<name>`.
|
||||
parts = line.split(None, 1)
|
||||
if len(parts) != 2:
|
||||
continue
|
||||
digest, fname = parts
|
||||
fname = fname.lstrip("*").strip()
|
||||
digest = digest.strip().lower()
|
||||
if len(digest) == 64 and all(c in "0123456789abcdef" for c in digest) and fname:
|
||||
out[fname] = digest
|
||||
return out
|
||||
|
||||
|
||||
def _validate_zip_hash(
|
||||
zip_path: str,
|
||||
*,
|
||||
asset_name: str = "",
|
||||
sha256sums_url: str = "",
|
||||
release_tag: str = "",
|
||||
) -> str:
|
||||
"""Verify the downloaded archive against trusted digest sources.
|
||||
|
||||
Issue #231: previously this returned silently when ``MESH_UPDATE_SHA256``
|
||||
was unset, which made the auto-updater a supply-chain RCE vector on any
|
||||
compromise of the GitHub release pipeline. The chain now is:
|
||||
|
||||
1. ``MESH_UPDATE_SHA256`` env var (operator override — preserved for
|
||||
power-users who want to pin an exact digest manually)
|
||||
2. ``SHA256SUMS.txt`` release asset (primary — the maintainer's
|
||||
release process already publishes this)
|
||||
3. Baked-in ``backend/data/release_digests.json`` (second line of
|
||||
defense for releases that lack the SHA256SUMS asset, or when the
|
||||
asset can't be fetched at update time)
|
||||
4. HTTPS-only fallback with a loud warning (preserves the auto-update
|
||||
flow during transient outages — but never silently)
|
||||
|
||||
A mismatch from a source that DID respond is fatal: the update is
|
||||
refused and the existing install keeps running. Only the "no source
|
||||
reachable at all" case falls back to HTTPS-only.
|
||||
|
||||
Returns a short human-readable description of which source verified
|
||||
the archive (used in the update-success message).
|
||||
"""
|
||||
actual = _compute_sha256(zip_path)
|
||||
|
||||
# Source 1: explicit operator override.
|
||||
override = os.environ.get("MESH_UPDATE_SHA256", "").strip().lower()
|
||||
if override:
|
||||
if actual == override:
|
||||
return f"verified via MESH_UPDATE_SHA256 ({actual[:16]}...)"
|
||||
raise RuntimeError(
|
||||
f"Update SHA-256 mismatch vs MESH_UPDATE_SHA256: archive={actual[:16]}..., "
|
||||
f"expected={override[:16]}..."
|
||||
)
|
||||
|
||||
# Source 2: SHA256SUMS.txt asset from the release.
|
||||
sums_map: dict[str, str] = {}
|
||||
if sha256sums_url and asset_name:
|
||||
sums_map = _fetch_sha256sums(sha256sums_url)
|
||||
|
||||
sums_expected = sums_map.get(asset_name) if asset_name else None
|
||||
if sums_expected:
|
||||
if actual == sums_expected:
|
||||
return f"verified via release SHA256SUMS.txt ({actual[:16]}...)"
|
||||
raise RuntimeError(
|
||||
f"Update SHA-256 mismatch vs release SHA256SUMS.txt: "
|
||||
f"archive={actual[:16]}..., expected={sums_expected[:16]}..."
|
||||
)
|
||||
|
||||
# Source 3: baked-in digest list.
|
||||
baked = _load_baked_in_release_digests()
|
||||
baked_expected = ""
|
||||
if release_tag and asset_name:
|
||||
baked_expected = baked.get(release_tag, {}).get(asset_name, "")
|
||||
if baked_expected:
|
||||
if actual == baked_expected:
|
||||
return f"verified via baked-in digest list ({actual[:16]}...)"
|
||||
raise RuntimeError(
|
||||
f"Update SHA-256 mismatch vs baked-in digest list: "
|
||||
f"archive={actual[:16]}..., expected={baked_expected[:16]}..."
|
||||
)
|
||||
|
||||
# Source 4: HTTPS-only fallback. We keep onboarding/auto-update working
|
||||
# during transient outages (no SHA256SUMS reachable AND no baked-in
|
||||
# entry for this release), but surface the degraded posture loudly so
|
||||
# the operator can see it in logs and the maintainer can populate the
|
||||
# digest list on the next release bump.
|
||||
logger.warning(
|
||||
"Update integrity check fell back to HTTPS-only trust "
|
||||
"(no SHA256SUMS.txt response and no baked-in digest for "
|
||||
"release=%s asset=%s). The archive SHA-256 is %s. Once the "
|
||||
"release ships a SHA256SUMS.txt asset OR backend/data/"
|
||||
"release_digests.json is updated with this release, the secure "
|
||||
"path will activate automatically.",
|
||||
release_tag or "unknown",
|
||||
asset_name or "unknown",
|
||||
actual,
|
||||
)
|
||||
return f"https-only (no digest source reachable, archive={actual[:16]}...)"
|
||||
|
||||
|
||||
def _is_source_checkout(project_root: str) -> bool:
|
||||
@@ -334,7 +545,7 @@ def perform_update(project_root: str) -> dict:
|
||||
temp_dir = tempfile.mkdtemp(prefix="sb_update_")
|
||||
manual_url = GITHUB_RELEASES_PAGE_URL
|
||||
try:
|
||||
zip_path, version, url, release_url = _download_release(temp_dir)
|
||||
zip_path, version, url, release_url, asset_name, sha256sums_url = _download_release(temp_dir)
|
||||
manual_url = release_url or manual_url
|
||||
|
||||
if in_docker:
|
||||
@@ -366,7 +577,13 @@ def perform_update(project_root: str) -> dict:
|
||||
),
|
||||
}
|
||||
|
||||
_validate_zip_hash(zip_path)
|
||||
verification_note = _validate_zip_hash(
|
||||
zip_path,
|
||||
asset_name=asset_name,
|
||||
sha256sums_url=sha256sums_url,
|
||||
release_tag=version,
|
||||
)
|
||||
logger.info("Update archive %s", verification_note)
|
||||
backup_path = _backup_current(project_root, temp_dir)
|
||||
copied = _extract_and_copy(zip_path, project_root, temp_dir)
|
||||
|
||||
@@ -378,6 +595,7 @@ def perform_update(project_root: str) -> dict:
|
||||
"manual_url": manual_url,
|
||||
"release_url": release_url,
|
||||
"download_url": url,
|
||||
"integrity": verification_note,
|
||||
"message": f"Updated to {version} — {copied} files replaced. Restarting...",
|
||||
}
|
||||
except Exception as e:
|
||||
|
||||
@@ -243,6 +243,48 @@ def _pid_alive(pid: int) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def _find_wormhole_server_pid() -> int:
|
||||
if os.name == "nt":
|
||||
return 0
|
||||
proc_dir = Path("/proc")
|
||||
if not proc_dir.exists():
|
||||
return 0
|
||||
current_pid = os.getpid()
|
||||
script_name = WORMHOLE_SCRIPT.name
|
||||
script_path = str(WORMHOLE_SCRIPT)
|
||||
for entry in proc_dir.iterdir():
|
||||
if not entry.name.isdigit():
|
||||
continue
|
||||
pid = int(entry.name)
|
||||
if pid == current_pid:
|
||||
continue
|
||||
try:
|
||||
raw = (entry / "cmdline").read_bytes()
|
||||
except OSError:
|
||||
continue
|
||||
cmdline = raw.replace(b"\x00", b" ").decode("utf-8", errors="replace")
|
||||
if script_path in cmdline or script_name in cmdline:
|
||||
return pid
|
||||
return 0
|
||||
|
||||
|
||||
def _terminate_pid(pid: int, *, timeout_s: float = 5.0) -> None:
|
||||
if os.name == "nt" or pid <= 0:
|
||||
return
|
||||
try:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
except Exception:
|
||||
return
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while time.monotonic() < deadline and _pid_alive(pid):
|
||||
time.sleep(0.1)
|
||||
if _pid_alive(pid):
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _probe_ready(timeout_s: float = 1.5) -> bool:
|
||||
try:
|
||||
with urlopen(f"http://{WORMHOLE_HOST}:{WORMHOLE_PORT}/api/health", timeout=timeout_s) as resp:
|
||||
@@ -266,17 +308,32 @@ def _probe_json(path: str, timeout_s: float = 1.5) -> dict[str, Any] | None:
|
||||
def _current_runtime_state() -> dict[str, Any]:
|
||||
settings = read_wormhole_settings()
|
||||
status = read_wormhole_status()
|
||||
configured = bool(settings.get("enabled"))
|
||||
running = False
|
||||
ready = False
|
||||
pid = int(status.get("pid", 0) or 0)
|
||||
if _PROCESS and _PROCESS.poll() is None:
|
||||
if not configured:
|
||||
# Disabled private transport must stay disabled even if a stale local
|
||||
# wormhole process is still answering on the health port. Public
|
||||
# MeshChat relies on this state to keep the MQTT and Wormhole lanes
|
||||
# mutually exclusive.
|
||||
pid = 0
|
||||
ready = False
|
||||
elif _PROCESS and _PROCESS.poll() is None:
|
||||
running = True
|
||||
pid = int(_PROCESS.pid or 0)
|
||||
elif _pid_alive(pid):
|
||||
running = True
|
||||
elif _probe_ready(timeout_s=0.35):
|
||||
running = True
|
||||
pid = 0
|
||||
ready = running and _probe_ready()
|
||||
else:
|
||||
if _pid_alive(pid):
|
||||
running = True
|
||||
else:
|
||||
discovered_pid = _find_wormhole_server_pid()
|
||||
if discovered_pid > 0:
|
||||
running = True
|
||||
pid = discovered_pid
|
||||
if not running and _probe_ready(timeout_s=0.35):
|
||||
running = True
|
||||
pid = 0
|
||||
ready = running and _probe_ready()
|
||||
if not running:
|
||||
pid = 0
|
||||
transport_active = status.get("transport_active", "") if ready else ""
|
||||
@@ -319,13 +376,13 @@ def _current_runtime_state() -> dict[str, Any]:
|
||||
anonymous_mode = bool(settings.get("anonymous_mode"))
|
||||
anonymous_mode_ready = bool(
|
||||
anonymous_mode
|
||||
and settings.get("enabled")
|
||||
and configured
|
||||
and ready
|
||||
and effective_transport in {"tor", "tor_arti", "i2p", "mixnet"}
|
||||
)
|
||||
snapshot = {
|
||||
"installed": _installed(),
|
||||
"configured": bool(settings.get("enabled")),
|
||||
"configured": configured,
|
||||
"running": running,
|
||||
"ready": ready,
|
||||
"transport_configured": str(settings.get("transport", "direct") or "direct"),
|
||||
@@ -395,6 +452,12 @@ def get_wormhole_state() -> dict[str, Any]:
|
||||
def connect_wormhole(*, reason: str = "connect") -> dict[str, Any]:
|
||||
with _LOCK:
|
||||
_invalidate_state_cache()
|
||||
try:
|
||||
from services.transport_lane_isolation import disable_public_mesh_lane
|
||||
|
||||
disable_public_mesh_lane(reason=f"wormhole_{reason}")
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to enforce public/private lane isolation during %s: %s", reason, exc)
|
||||
settings = read_wormhole_settings()
|
||||
if not settings.get("enabled"):
|
||||
settings = settings.copy()
|
||||
@@ -487,8 +550,8 @@ def connect_wormhole(*, reason: str = "connect") -> dict[str, Any]:
|
||||
def disconnect_wormhole(*, reason: str = "disconnect") -> dict[str, Any]:
|
||||
with _LOCK:
|
||||
_invalidate_state_cache()
|
||||
current = _current_runtime_state()
|
||||
pid = int(current.get("pid", 0) or 0)
|
||||
status = read_wormhole_status()
|
||||
pid = int(status.get("pid", 0) or 0)
|
||||
global _PROCESS
|
||||
if _PROCESS and _PROCESS.poll() is None:
|
||||
try:
|
||||
@@ -499,14 +562,15 @@ def disconnect_wormhole(*, reason: str = "disconnect") -> dict[str, Any]:
|
||||
_PROCESS.kill()
|
||||
except Exception:
|
||||
pass
|
||||
elif os.name != "nt" and _pid_alive(pid):
|
||||
try:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
except Exception:
|
||||
pass
|
||||
if os.name != "nt":
|
||||
_terminate_pid(pid)
|
||||
discovered_pid = _find_wormhole_server_pid()
|
||||
if discovered_pid > 0 and discovered_pid != pid:
|
||||
_terminate_pid(discovered_pid)
|
||||
_PROCESS = None
|
||||
write_wormhole_status(
|
||||
reason=reason,
|
||||
configured=False,
|
||||
running=False,
|
||||
ready=False,
|
||||
pid=0,
|
||||
|
||||
@@ -5,7 +5,7 @@ from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
|
||||
|
||||
def _request(path: str, method: str = "POST") -> Request:
|
||||
def _request(path: str, method: str = "POST", query_string: bytes = b"") -> Request:
|
||||
return Request(
|
||||
{
|
||||
"type": "http",
|
||||
@@ -13,6 +13,7 @@ def _request(path: str, method: str = "POST") -> Request:
|
||||
"client": ("test", 12345),
|
||||
"method": method,
|
||||
"path": path,
|
||||
"query_string": query_string,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -504,6 +505,61 @@ def test_private_infonet_gate_write_returns_preparing_state_when_wormhole_not_re
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def test_invite_scoped_prekey_lookup_reaches_handler_while_lane_prepares(monkeypatch):
|
||||
"""Copied-address import must not be blocked by private-lane warmup."""
|
||||
import main
|
||||
import auth
|
||||
from services.config import get_settings
|
||||
from services import wormhole_supervisor
|
||||
|
||||
monkeypatch.setenv("MESH_PRIVATE_CLEARNET_FALLBACK", "block")
|
||||
monkeypatch.setenv("MESH_BLOCK_LEGACY_NODE_ID_COMPAT", "true")
|
||||
monkeypatch.setenv("MESH_BLOCK_LEGACY_AGENT_ID_LOOKUP", "true")
|
||||
monkeypatch.setenv("MESH_ALLOW_COMPAT_DM_INVITE_IMPORT", "false")
|
||||
get_settings.cache_clear()
|
||||
monkeypatch.setattr(
|
||||
auth,
|
||||
"_anonymous_mode_state",
|
||||
lambda: {
|
||||
"enabled": False,
|
||||
"wormhole_enabled": True,
|
||||
"ready": False,
|
||||
"effective_transport": "direct",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
wormhole_supervisor,
|
||||
"get_wormhole_state",
|
||||
lambda: {
|
||||
"configured": True,
|
||||
"ready": False,
|
||||
"rns_ready": False,
|
||||
"arti_ready": False,
|
||||
},
|
||||
)
|
||||
|
||||
called = {"value": False}
|
||||
|
||||
async def call_next(_request: Request) -> Response:
|
||||
called["value"] = True
|
||||
return Response(status_code=200)
|
||||
|
||||
response = asyncio.run(
|
||||
main.enforce_high_privacy_mesh(
|
||||
_request(
|
||||
"/api/mesh/dm/prekey-bundle",
|
||||
method="GET",
|
||||
query_string=b"lookup_token=invite-handle",
|
||||
),
|
||||
call_next,
|
||||
)
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert called["value"] is True
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def test_private_dm_send_blocks_at_transitional_tier(monkeypatch):
|
||||
import main
|
||||
import auth
|
||||
|
||||
@@ -47,6 +47,11 @@ def test_infonet_ingest_accepts_valid_event(tmp_path, monkeypatch):
|
||||
|
||||
assert result["accepted"] == 1
|
||||
assert inf.head_hash == evt.event_id
|
||||
info = inf.get_info()
|
||||
assert info["known_nodes"] == 1
|
||||
assert info["author_nodes"] == 1
|
||||
assert info["total_events"] == 1
|
||||
assert info["event_types"]["message"] == 1
|
||||
|
||||
|
||||
def test_verify_node_binding_accepts_current_and_compat_ids_only(monkeypatch):
|
||||
@@ -64,6 +69,8 @@ def test_verify_node_binding_accepts_current_and_compat_ids_only(monkeypatch):
|
||||
f"{current[len(mesh_crypto.NODE_ID_PREFIX):len(mesh_crypto.NODE_ID_PREFIX) + 8]}"
|
||||
)
|
||||
|
||||
monkeypatch.setenv("MESH_DEV_ALLOW_LEGACY_COMPAT", "true")
|
||||
monkeypatch.setenv("MESH_BLOCK_LEGACY_NODE_ID_COMPAT", "false")
|
||||
monkeypatch.setenv("MESH_ALLOW_LEGACY_NODE_ID_COMPAT_UNTIL", "2099-01-01")
|
||||
from services.config import get_settings
|
||||
|
||||
@@ -98,7 +105,7 @@ def test_infonet_append_rejects_missing_signature_fields(tmp_path, monkeypatch):
|
||||
assert "signature" in str(exc).lower()
|
||||
|
||||
|
||||
def test_infonet_load_fails_closed_on_hash_mismatch(tmp_path, monkeypatch):
|
||||
def test_infonet_load_quarantines_and_resets_on_hash_mismatch(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(mesh_hashchain, "DATA_DIR", tmp_path)
|
||||
monkeypatch.setattr(mesh_hashchain, "CHAIN_FILE", tmp_path / "infonet.json")
|
||||
|
||||
@@ -135,8 +142,12 @@ def test_infonet_load_fails_closed_on_hash_mismatch(tmp_path, monkeypatch):
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="Hash mismatch on event load"):
|
||||
mesh_hashchain.Infonet()
|
||||
inf = mesh_hashchain.Infonet()
|
||||
|
||||
assert inf.events == []
|
||||
assert inf.head_hash == mesh_hashchain.GENESIS_HASH
|
||||
assert not mesh_hashchain.CHAIN_FILE.exists()
|
||||
assert list(tmp_path.glob("infonet.json.quarantine.*"))
|
||||
|
||||
|
||||
def test_validate_gate_message_payload_rejects_plaintext_shape():
|
||||
|
||||
@@ -37,6 +37,30 @@ def test_eligible_sync_peers_filters_bucket_and_cooldown():
|
||||
assert [record.peer_url for record in candidates] == ["https://active.example"]
|
||||
|
||||
|
||||
def test_eligible_sync_peers_prioritizes_explicit_bootstrap_seed():
|
||||
old_runtime = make_sync_peer_record(
|
||||
peer_url="https://old-runtime.example",
|
||||
transport="clearnet",
|
||||
role="participant",
|
||||
source="runtime",
|
||||
now=100,
|
||||
)
|
||||
seed = make_sync_peer_record(
|
||||
peer_url="https://node.shadowbroker.info",
|
||||
transport="clearnet",
|
||||
role="seed",
|
||||
source="bundle",
|
||||
now=200,
|
||||
)
|
||||
|
||||
candidates = eligible_sync_peers([old_runtime, seed], now=300)
|
||||
|
||||
assert [record.peer_url for record in candidates] == [
|
||||
"https://node.shadowbroker.info",
|
||||
"https://old-runtime.example",
|
||||
]
|
||||
|
||||
|
||||
def test_finish_sync_success_updates_schedule():
|
||||
state = begin_sync(SyncWorkerState(), peer_url="https://seed.example", now=100)
|
||||
finished = finish_sync(
|
||||
|
||||
@@ -96,3 +96,38 @@ def test_peer_store_failure_and_success_lifecycle(tmp_path):
|
||||
assert recovered.cooldown_until == 0
|
||||
assert recovered.last_error == ""
|
||||
assert recovered.last_sync_ok_at == 250
|
||||
|
||||
|
||||
def test_upsert_explicit_seed_clears_stale_cooldown(tmp_path):
|
||||
store = PeerStore(tmp_path / "peer_store.json")
|
||||
store.upsert(
|
||||
make_sync_peer_record(
|
||||
peer_url="https://node.shadowbroker.info",
|
||||
transport="clearnet",
|
||||
role="seed",
|
||||
source="bundle",
|
||||
now=100,
|
||||
)
|
||||
)
|
||||
failed = store.mark_failure(
|
||||
"https://node.shadowbroker.info",
|
||||
"sync",
|
||||
error="timed out",
|
||||
cooldown_s=120,
|
||||
now=110,
|
||||
)
|
||||
assert failed.cooldown_until == 230
|
||||
|
||||
refreshed = store.upsert(
|
||||
make_sync_peer_record(
|
||||
peer_url="https://node.shadowbroker.info",
|
||||
transport="clearnet",
|
||||
role="seed",
|
||||
source="bundle",
|
||||
now=120,
|
||||
)
|
||||
)
|
||||
|
||||
assert refreshed.failure_count == 0
|
||||
assert refreshed.cooldown_until == 0
|
||||
assert refreshed.last_error == ""
|
||||
|
||||
@@ -2,15 +2,21 @@ from services.mesh.meshtastic_topics import build_subscription_topics, known_roo
|
||||
|
||||
|
||||
def test_default_subscription_is_longfast_only():
|
||||
assert build_subscription_topics() == ["msh/US/2/e/LongFast/#"]
|
||||
assert build_subscription_topics() == [
|
||||
"msh/US/2/e/LongFast/#",
|
||||
"msh/US/2/json/LongFast/#",
|
||||
]
|
||||
assert known_roots() == ["US"]
|
||||
|
||||
|
||||
def test_extra_roots_are_longfast_only():
|
||||
assert build_subscription_topics(extra_roots="EU_868,ANZ") == [
|
||||
"msh/US/2/e/LongFast/#",
|
||||
"msh/US/2/json/LongFast/#",
|
||||
"msh/EU_868/2/e/LongFast/#",
|
||||
"msh/EU_868/2/json/LongFast/#",
|
||||
"msh/ANZ/2/e/LongFast/#",
|
||||
"msh/ANZ/2/json/LongFast/#",
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ Tests verify:
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
|
||||
from services.config import get_settings
|
||||
@@ -611,6 +612,99 @@ class TestFetchPrekeyBundleByLookup:
|
||||
"peer prekey lookup unavailable",
|
||||
}
|
||||
|
||||
def test_fetch_lookup_token_uses_bootstrap_peer_without_agent_id(self, tmp_path, monkeypatch):
|
||||
"""Invite lookup can resolve through bootstrap peers without exposing agent_id."""
|
||||
_isolated_relay(tmp_path, monkeypatch)
|
||||
record = _valid_bundle_record("test-agent")
|
||||
requested_urls: list[str] = []
|
||||
|
||||
monkeypatch.setenv("MESH_BOOTSTRAP_SEED_PEERS", "https://seed.example")
|
||||
monkeypatch.setenv("MESH_DEFAULT_SYNC_PEERS", "")
|
||||
monkeypatch.setenv("MESH_RELAY_PEERS", "")
|
||||
get_settings.cache_clear()
|
||||
|
||||
class _Response:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def read(self, _limit: int = -1):
|
||||
return json.dumps(
|
||||
{
|
||||
"ok": True,
|
||||
"identity_dh_pub_key": record["dh_pub_key"],
|
||||
"dh_algo": record["dh_algo"],
|
||||
"public_key": record["public_key"],
|
||||
"public_key_algo": record["public_key_algo"],
|
||||
"protocol_version": record["protocol_version"],
|
||||
"sequence": 1,
|
||||
"signed_at": int(record["bundle"].get("signed_at", 0) or 0),
|
||||
"bundle": record["bundle"],
|
||||
}
|
||||
).encode("utf-8")
|
||||
|
||||
def _urlopen(request, timeout=0):
|
||||
requested_urls.append(str(getattr(request, "full_url", "")))
|
||||
return _Response()
|
||||
|
||||
monkeypatch.setattr("services.mesh.mesh_wormhole_prekey.urllib.request.urlopen", _urlopen)
|
||||
|
||||
from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle
|
||||
|
||||
result = fetch_dm_prekey_bundle(agent_id="", lookup_token="bootstrap-handle")
|
||||
|
||||
assert result["ok"] is True
|
||||
assert result["agent_id"] == record["agent_id"]
|
||||
assert result["lookup_mode"] == "invite_lookup_handle"
|
||||
assert result["public_lookup"] is True
|
||||
assert requested_urls
|
||||
assert "lookup_token=bootstrap-handle" in requested_urls[0]
|
||||
assert "agent_id" not in requested_urls[0]
|
||||
|
||||
def test_fetch_lookup_token_does_not_parse_peer_pending_as_bundle(self, tmp_path, monkeypatch):
|
||||
"""A peer's private-lane pending response is not a malformed prekey bundle."""
|
||||
_isolated_relay(tmp_path, monkeypatch)
|
||||
requested_urls: list[str] = []
|
||||
|
||||
monkeypatch.setenv("MESH_BOOTSTRAP_SEED_PEERS", "https://seed.example")
|
||||
monkeypatch.setenv("MESH_DEFAULT_SYNC_PEERS", "")
|
||||
monkeypatch.setenv("MESH_RELAY_PEERS", "")
|
||||
get_settings.cache_clear()
|
||||
|
||||
class _Response:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def read(self, _limit: int = -1):
|
||||
return json.dumps(
|
||||
{
|
||||
"ok": True,
|
||||
"pending": True,
|
||||
"status": "preparing_private_lane",
|
||||
"detail": "transport tier insufficient",
|
||||
}
|
||||
).encode("utf-8")
|
||||
|
||||
def _urlopen(request, timeout=0):
|
||||
requested_urls.append(str(getattr(request, "full_url", "")))
|
||||
return _Response()
|
||||
|
||||
monkeypatch.setattr("services.mesh.mesh_wormhole_prekey.urllib.request.urlopen", _urlopen)
|
||||
|
||||
from services.mesh.mesh_wormhole_prekey import fetch_dm_prekey_bundle
|
||||
|
||||
result = fetch_dm_prekey_bundle(agent_id="", lookup_token="bootstrap-handle")
|
||||
|
||||
assert requested_urls
|
||||
assert result["ok"] is False
|
||||
assert result["detail"] == "peer prekey lookup still preparing"
|
||||
assert result["detail"] != "Prekey bundle missing signing key"
|
||||
|
||||
def test_fetch_agent_id_uses_pinned_contact_lookup_handle(self, tmp_path, monkeypatch):
|
||||
"""Pinned invite lookup handle is used before direct agent_id lookup."""
|
||||
relay = _isolated_relay(tmp_path, monkeypatch)
|
||||
|
||||
@@ -71,6 +71,38 @@ def _fresh_wormhole_state(tmp_path, monkeypatch):
|
||||
return relay, mesh_wormhole_identity, mesh_wormhole_contacts, mesh_wormhole_prekey
|
||||
|
||||
|
||||
def test_register_wormhole_dm_key_repairs_missing_local_dh_material(tmp_path, monkeypatch):
|
||||
relay, identity_mod, _contacts_mod, _prekey_mod = _fresh_wormhole_state(tmp_path, monkeypatch)
|
||||
identity = identity_mod.read_wormhole_identity()
|
||||
original_node_id = identity["node_id"]
|
||||
original_public_key = identity["public_key"]
|
||||
original_private_key = identity["private_key"]
|
||||
|
||||
identity_mod.write_dm_identity(
|
||||
{
|
||||
**identity,
|
||||
"dh_pub_key": "",
|
||||
"dh_private_key": "",
|
||||
"bundle_fingerprint": "",
|
||||
"bundle_sequence": 0,
|
||||
"bundle_registered_at": 0,
|
||||
}
|
||||
)
|
||||
|
||||
registered = identity_mod.register_wormhole_dm_key()
|
||||
repaired = identity_mod.read_wormhole_identity()
|
||||
|
||||
assert registered["ok"] is True
|
||||
assert registered["dh_pub_key"]
|
||||
assert registered["dh_algo"] == "X25519"
|
||||
assert repaired["dh_pub_key"] == registered["dh_pub_key"]
|
||||
assert repaired["dh_private_key"]
|
||||
assert repaired["node_id"] == original_node_id
|
||||
assert repaired["public_key"] == original_public_key
|
||||
assert repaired["private_key"] == original_private_key
|
||||
assert relay.get_dh_key(original_node_id)["dh_pub_key"] == registered["dh_pub_key"]
|
||||
|
||||
|
||||
def _export_verified_invite(identity_mod):
|
||||
exported = identity_mod.export_wormhole_dm_invite()
|
||||
assert exported["ok"] is True
|
||||
@@ -460,6 +492,30 @@ def test_imported_dm_invite_pins_contact_as_invite_pinned(tmp_path, monkeypatch)
|
||||
assert contacts_mod.list_wormhole_dm_contacts()[imported["peer_id"]]["trust_level"] == "invite_pinned"
|
||||
|
||||
|
||||
def test_imported_dm_invite_saves_pending_contact_when_prekey_not_visible(tmp_path, monkeypatch):
|
||||
_relay, identity_mod, contacts_mod, prekey_mod = _fresh_wormhole_state(tmp_path, monkeypatch)
|
||||
exported, verified = _export_verified_invite(identity_mod)
|
||||
monkeypatch.setattr(
|
||||
prekey_mod,
|
||||
"fetch_dm_prekey_bundle",
|
||||
lambda **_kw: {"ok": False, "detail": "Prekey bundle not found"},
|
||||
)
|
||||
|
||||
imported = identity_mod.import_wormhole_dm_invite(exported["invite"], alias="alice")
|
||||
contact = imported["contact"]
|
||||
|
||||
assert imported["ok"] is True
|
||||
assert imported["pending_prekey"] is True
|
||||
assert imported["peer_id"] == verified["peer_id"]
|
||||
assert contact["alias"] == "alice"
|
||||
assert contact["trust_level"] == "invite_pinned"
|
||||
assert contact["invitePinnedPrekeyLookupHandle"] == exported["invite"]["payload"]["prekey_lookup_handle"]
|
||||
assert contact["remotePrekeyLookupMode"] == "invite_lookup_handle"
|
||||
assert contact["remotePrekeyFingerprint"] == verified["trust_fingerprint"]
|
||||
assert contact["dhPubKey"] == ""
|
||||
assert contacts_mod.list_wormhole_dm_contacts()[verified["peer_id"]]["trust_level"] == "invite_pinned"
|
||||
|
||||
|
||||
def test_imported_dm_invite_requires_root_attested_prekey_bundle(tmp_path, monkeypatch):
|
||||
relay, identity_mod, _contacts_mod, _prekey_mod = _fresh_wormhole_state(tmp_path, monkeypatch)
|
||||
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Issue #258 — AIS proxy SPKI pinning.
|
||||
|
||||
Most of the SPKI logic lives in ``backend/ais_proxy.js`` (Node) and can't
|
||||
be unit-tested from Python directly. These tests cover the Python-side
|
||||
glue: ``services.ais_stream.ais_proxy_status()`` (the snapshot the proxy
|
||||
populates via stdout markers) and ``routers/health.py`` surfacing the
|
||||
degraded TLS state.
|
||||
|
||||
Additionally, the pin-file structure is validated: it must be parseable
|
||||
JSON, must contain an entry for ``stream.aisstream.io``, and each pin
|
||||
must look like a base64-encoded SHA-256 hash.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from services import ais_stream
|
||||
|
||||
PIN_FILE = (
|
||||
Path(__file__).resolve().parent.parent / "data" / "aisstream_spki_pins.json"
|
||||
)
|
||||
|
||||
|
||||
def test_pin_file_exists_and_is_valid_json():
|
||||
assert PIN_FILE.exists(), f"Expected pin file at {PIN_FILE}"
|
||||
data = json.loads(PIN_FILE.read_text(encoding="utf-8"))
|
||||
assert isinstance(data, dict)
|
||||
|
||||
|
||||
def test_pin_file_has_aisstream_entry():
|
||||
data = json.loads(PIN_FILE.read_text(encoding="utf-8"))
|
||||
pins = data.get("stream.aisstream.io")
|
||||
assert isinstance(pins, list)
|
||||
assert len(pins) >= 1
|
||||
|
||||
|
||||
def test_each_pin_looks_like_a_base64_sha256():
|
||||
"""SPKI pins must be 44-char base64-encoded SHA-256 digests."""
|
||||
data = json.loads(PIN_FILE.read_text(encoding="utf-8"))
|
||||
pins = data["stream.aisstream.io"]
|
||||
for pin in pins:
|
||||
assert isinstance(pin, str), f"pin not a string: {pin!r}"
|
||||
assert len(pin) == 44, f"pin {pin!r} not 44 chars (SHA-256 base64)"
|
||||
# Must base64-decode to exactly 32 bytes (256 bits)
|
||||
try:
|
||||
raw = base64.b64decode(pin)
|
||||
except Exception as exc:
|
||||
pytest.fail(f"pin {pin!r} is not valid base64: {exc}")
|
||||
assert len(raw) == 32, f"pin {pin!r} decodes to {len(raw)} bytes, expected 32"
|
||||
# Should match the canonical base64 alphabet (no URL-safe variants)
|
||||
assert re.match(r"^[A-Za-z0-9+/]+=*$", pin), f"pin {pin!r} contains non-base64 chars"
|
||||
|
||||
|
||||
def test_ais_proxy_status_starts_empty():
|
||||
"""Before the proxy emits any status marker, the snapshot is empty."""
|
||||
# Clear any stale state from other tests
|
||||
with ais_stream._vessels_lock:
|
||||
ais_stream._proxy_status.clear()
|
||||
status = ais_stream.ais_proxy_status()
|
||||
assert status == {}
|
||||
|
||||
|
||||
def test_ais_proxy_status_returns_copy_not_reference():
|
||||
"""ais_proxy_status() must return a defensive copy.
|
||||
|
||||
Otherwise a caller could mutate the live dict and confuse later reads.
|
||||
"""
|
||||
with ais_stream._vessels_lock:
|
||||
ais_stream._proxy_status.clear()
|
||||
ais_stream._proxy_status["degraded_tls"] = True
|
||||
|
||||
snapshot = ais_stream.ais_proxy_status()
|
||||
assert snapshot == {"degraded_tls": True}
|
||||
snapshot["degraded_tls"] = False # mutate the returned copy
|
||||
|
||||
# Original should be untouched
|
||||
re_snapshot = ais_stream.ais_proxy_status()
|
||||
assert re_snapshot == {"degraded_tls": True}
|
||||
|
||||
# Cleanup so other tests start clean
|
||||
with ais_stream._vessels_lock:
|
||||
ais_stream._proxy_status.clear()
|
||||
|
||||
|
||||
def test_health_includes_ais_proxy_field(client):
|
||||
"""The /api/health response must include the ais_proxy block."""
|
||||
# Inject a known degraded state
|
||||
with ais_stream._vessels_lock:
|
||||
ais_stream._proxy_status.clear()
|
||||
ais_stream._proxy_status["degraded_tls"] = True
|
||||
|
||||
response = client.get("/api/health")
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
|
||||
assert "ais_proxy" in payload
|
||||
assert payload["ais_proxy"] == {"degraded_tls": True}
|
||||
# Top-level status should escalate from ok to degraded when AIS is
|
||||
# in degraded-TLS mode (unless SLOs already report worse).
|
||||
assert payload["status"] in {"degraded", "error"}
|
||||
|
||||
# Cleanup
|
||||
with ais_stream._vessels_lock:
|
||||
ais_stream._proxy_status.clear()
|
||||
|
||||
|
||||
def test_health_ais_proxy_field_when_no_status(client):
|
||||
"""When the proxy hasn't reported anything yet, ais_proxy is empty."""
|
||||
with ais_stream._vessels_lock:
|
||||
ais_stream._proxy_status.clear()
|
||||
|
||||
response = client.get("/api/health")
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload.get("ais_proxy") == {}
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Issue #192 (tg12): CCTV proxy must re-validate the host on every redirect hop.
|
||||
|
||||
Before this fix, the proxy validated only the initial caller-supplied URL
|
||||
host and then used ``requests.get(..., allow_redirects=True)``, which would
|
||||
silently follow a 302 to an arbitrary internal address — an open-redirect-
|
||||
to-SSRF chain.
|
||||
|
||||
These tests assert that:
|
||||
|
||||
1. A redirect to a disallowed host is rejected (502).
|
||||
2. A redirect to an allowed host is followed (200).
|
||||
3. The redirect chain length is bounded.
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from routers.cctv import _fetch_cctv_upstream_response, _CCTV_MAX_REDIRECTS
|
||||
|
||||
|
||||
class _Resp:
|
||||
"""Minimal mock for requests.Response that mimics what _fetch needs."""
|
||||
|
||||
def __init__(self, status_code=200, headers=None, is_redirect=False):
|
||||
self.status_code = status_code
|
||||
self.headers = headers or {}
|
||||
self.is_redirect = is_redirect
|
||||
self.closed = False
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
def _profile():
|
||||
"""Build a tiny _CCTVProxyProfile-shaped mock the function expects."""
|
||||
p = MagicMock()
|
||||
p.name = "test"
|
||||
p.timeout = 5
|
||||
p.cache_seconds = 60
|
||||
return p
|
||||
|
||||
|
||||
def _request():
|
||||
"""Build a tiny Request-shaped mock — only headers are read."""
|
||||
req = MagicMock()
|
||||
req.headers = {}
|
||||
return req
|
||||
|
||||
|
||||
@patch("routers.cctv._cctv_upstream_headers", return_value={})
|
||||
@patch("routers.cctv._cctv_host_allowed", side_effect=lambda host: host == "allowed.example")
|
||||
@patch("routers.cctv._req" if False else "requests.get") # patched below per-call
|
||||
def test_redirect_to_disallowed_host_is_rejected(mock_get, mock_allow, mock_headers):
|
||||
"""A 302 from allowed.example -> evil.example must be rejected with 502."""
|
||||
# First call: 302 with Location: http://evil.example/path
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "http://evil.example/path"}, is_redirect=True),
|
||||
]
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
_fetch_cctv_upstream_response(_request(), "http://allowed.example/cam", _profile())
|
||||
assert exc_info.value.status_code == 502
|
||||
assert "disallowed host" in str(exc_info.value.detail).lower()
|
||||
|
||||
|
||||
@patch("routers.cctv._cctv_upstream_headers", return_value={})
|
||||
@patch("routers.cctv._cctv_host_allowed", side_effect=lambda host: host == "allowed.example")
|
||||
@patch("requests.get")
|
||||
def test_redirect_to_localhost_is_rejected(mock_get, mock_allow, mock_headers):
|
||||
"""A redirect to 127.0.0.1 (internal SSRF target) must be rejected."""
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "http://127.0.0.1:8000/api/secret"}, is_redirect=True),
|
||||
]
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
_fetch_cctv_upstream_response(_request(), "http://allowed.example/cam", _profile())
|
||||
assert exc_info.value.status_code == 502
|
||||
|
||||
|
||||
@patch("routers.cctv._cctv_upstream_headers", return_value={})
|
||||
@patch("routers.cctv._cctv_host_allowed", side_effect=lambda host: host in {"allowed.example", "other-allowed.example"})
|
||||
@patch("requests.get")
|
||||
def test_redirect_to_another_allowed_host_is_followed(mock_get, mock_allow, mock_headers):
|
||||
"""A 302 from one allowed host to another allowed host should succeed."""
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "http://other-allowed.example/cam"}, is_redirect=True),
|
||||
_Resp(status_code=200, headers={"Content-Type": "image/jpeg"}),
|
||||
]
|
||||
resp = _fetch_cctv_upstream_response(_request(), "http://allowed.example/cam", _profile())
|
||||
assert resp.status_code == 200
|
||||
|
||||
|
||||
@patch("routers.cctv._cctv_upstream_headers", return_value={})
|
||||
@patch("routers.cctv._cctv_host_allowed", return_value=True)
|
||||
@patch("requests.get")
|
||||
def test_redirect_chain_length_is_bounded(mock_get, mock_allow, mock_headers):
|
||||
"""A pathological redirect loop must terminate within _CCTV_MAX_REDIRECTS."""
|
||||
# Generate enough 302s to exceed the cap.
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": f"http://allowed.example/{i}"}, is_redirect=True)
|
||||
for i in range(_CCTV_MAX_REDIRECTS + 2)
|
||||
]
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
_fetch_cctv_upstream_response(_request(), "http://allowed.example/cam", _profile())
|
||||
assert exc_info.value.status_code == 502
|
||||
assert "too long" in str(exc_info.value.detail).lower()
|
||||
|
||||
|
||||
@patch("routers.cctv._cctv_upstream_headers", return_value={})
|
||||
@patch("routers.cctv._cctv_host_allowed", return_value=True)
|
||||
@patch("requests.get")
|
||||
def test_redirect_to_non_http_scheme_is_rejected(mock_get, mock_allow, mock_headers):
|
||||
"""A 302 to ``file://`` or ``ftp://`` must be rejected even if the host parses cleanly."""
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "file:///etc/passwd"}, is_redirect=True),
|
||||
]
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
_fetch_cctv_upstream_response(_request(), "http://allowed.example/cam", _profile())
|
||||
assert exc_info.value.status_code == 502
|
||||
assert "non-http" in str(exc_info.value.detail).lower()
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Regression coverage for operator-only control surfaces."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("method", "path", "payload"),
|
||||
[
|
||||
("get", "/api/wormhole/identity", None),
|
||||
("post", "/api/wormhole/identity/bootstrap", {}),
|
||||
("post", "/api/wormhole/gate/enter", {"gate_id": "general-talk"}),
|
||||
("post", "/api/wormhole/gate/leave", {"gate_id": "general-talk"}),
|
||||
("post", "/api/wormhole/sign", {"event_type": "gate_event", "payload": {"ok": True}}),
|
||||
("post", "/api/wormhole/gate/key/rotate", {"gate_id": "general-talk", "reason": "test"}),
|
||||
(
|
||||
"post",
|
||||
"/api/wormhole/gate/key/grant",
|
||||
{
|
||||
"gate_id": "general-talk",
|
||||
"recipient_node_id": "node-test",
|
||||
"recipient_dh_pub": "dh-test",
|
||||
},
|
||||
),
|
||||
("post", "/api/wormhole/gate/persona/create", {"gate_id": "general-talk", "label": "test"}),
|
||||
(
|
||||
"post",
|
||||
"/api/wormhole/gate/persona/activate",
|
||||
{"gate_id": "general-talk", "persona_id": "persona-test"},
|
||||
),
|
||||
("post", "/api/wormhole/gate/persona/clear", {"gate_id": "general-talk"}),
|
||||
(
|
||||
"post",
|
||||
"/api/wormhole/gate/persona/retire",
|
||||
{"gate_id": "general-talk", "persona_id": "persona-test"},
|
||||
),
|
||||
(
|
||||
"post",
|
||||
"/api/wormhole/gate/message/sign-encrypted",
|
||||
{
|
||||
"gate_id": "general-talk",
|
||||
"epoch": 1,
|
||||
"ciphertext": "ciphertext",
|
||||
"nonce": "nonce",
|
||||
"format": "mls1",
|
||||
"envelope_hash": "hash",
|
||||
},
|
||||
),
|
||||
("post", "/api/wormhole/gate/message/compose", {"gate_id": "general-talk", "plaintext": "hello"}),
|
||||
("post", "/api/wormhole/sign-raw", {"message": "raw"}),
|
||||
("post", "/api/wormhole/gate/state/export", {"gate_id": "general-talk"}),
|
||||
("post", "/api/wormhole/gate/proof", {"gate_id": "general-talk"}),
|
||||
("post", "/api/wormhole/connect", {}),
|
||||
("post", "/api/layers", {"layers": {"viirs_nightlights": True}}),
|
||||
("post", "/api/ais/feed", {"msgs": []}),
|
||||
# Added in post-#227 gap audit:
|
||||
# /api/wormhole/join also calls bootstrap_wormhole_identity() — same
|
||||
# identity-takeover surface as /identity/bootstrap. PR #227 hardened
|
||||
# the latter but missed the former.
|
||||
("post", "/api/wormhole/join", {}),
|
||||
# /api/sigint/transmit relays APRS-IS packets over radio using
|
||||
# operator-supplied credentials. Any caller who reaches this endpoint
|
||||
# could transmit on the operator's authority. Must be local-only.
|
||||
(
|
||||
"post",
|
||||
"/api/sigint/transmit",
|
||||
{
|
||||
"callsign": "N0CALL",
|
||||
"passcode": "12345",
|
||||
"target": "NOCALL",
|
||||
"message": "test",
|
||||
},
|
||||
),
|
||||
# Issue #198 (tg12, May 17): three gate introspection GETs leak the
|
||||
# operator's active persona, persona inventory, and key status for
|
||||
# any gate_id an anonymous caller knows. Defeats the unlinkability
|
||||
# property documented in the privacy threat model.
|
||||
("get", "/api/wormhole/gate/general-talk/identity", None),
|
||||
("get", "/api/wormhole/gate/general-talk/personas", None),
|
||||
("get", "/api/wormhole/gate/general-talk/key", None),
|
||||
# Issue #211 (tg12): /api/thermal/verify fans out into an expensive
|
||||
# STAC search + remote SWIR raster reads. Unauthenticated abuse
|
||||
# could burn Sentinel-Hub quota and outbound bandwidth.
|
||||
("get", "/api/thermal/verify?lat=0&lng=0&radius_km=10", None),
|
||||
# Issue #213 (tg12): /api/radio/openmhz/calls/{sys_name} — rotating
|
||||
# sys_name bypasses the 20s cache and hammers OpenMHZ. Risks an
|
||||
# IP-ban for the project.
|
||||
("get", "/api/radio/openmhz/calls/abc", None),
|
||||
# Issue #214 (tg12): /api/radio/openmhz/audio — anonymous bandwidth
|
||||
# relay through the backend. 60/minute rate limit is not enough on
|
||||
# a streaming endpoint.
|
||||
("get", "/api/radio/openmhz/audio?url=https%3A%2F%2Fmedia.openmhz.com%2Faudio%2Fabc.mp3", None),
|
||||
],
|
||||
)
|
||||
def test_remote_control_surface_rejects_without_local_operator_or_admin(
|
||||
remote_client, method, path, payload
|
||||
):
|
||||
request = getattr(remote_client, method)
|
||||
response = request(path, json=payload) if payload is not None else request(path)
|
||||
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
def test_remote_agent_actions_poll_rejects_without_local_operator_or_admin(remote_client):
|
||||
response = remote_client.get("/api/ai/agent-actions")
|
||||
|
||||
assert response.status_code == 403
|
||||
@@ -0,0 +1,52 @@
|
||||
"""CrowdThreat ingestion is operator opt-in only."""
|
||||
|
||||
|
||||
class _CrowdThreatResponse:
|
||||
status_code = 200
|
||||
|
||||
def json(self):
|
||||
return {
|
||||
"data": {
|
||||
"threats": [
|
||||
{
|
||||
"id": "ct-1",
|
||||
"title": "Example report",
|
||||
"location": {
|
||||
"lng_lat": [12.5, 41.9],
|
||||
"name": "Example place",
|
||||
"country": {"name": "Italy"},
|
||||
},
|
||||
"category": {"id": 1, "name": "Security"},
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_crowdthreat_disabled_by_default_does_not_call_upstream(monkeypatch):
|
||||
from services.fetchers import _store, crowdthreat
|
||||
|
||||
monkeypatch.delenv("CROWDTHREAT_ENABLED", raising=False)
|
||||
monkeypatch.setitem(_store.latest_data, "crowdthreat", [{"id": "old"}])
|
||||
monkeypatch.setattr(
|
||||
crowdthreat,
|
||||
"fetch_with_curl",
|
||||
lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("upstream called")),
|
||||
)
|
||||
|
||||
crowdthreat.fetch_crowdthreat()
|
||||
|
||||
assert _store.latest_data["crowdthreat"] == []
|
||||
|
||||
|
||||
def test_crowdthreat_opt_in_fetches_when_layer_is_enabled(monkeypatch):
|
||||
from services.fetchers import _store, crowdthreat
|
||||
|
||||
monkeypatch.setenv("CROWDTHREAT_ENABLED", "true")
|
||||
monkeypatch.setitem(_store.active_layers, "crowdthreat", True)
|
||||
monkeypatch.setattr(crowdthreat, "fetch_with_curl", lambda *args, **kwargs: _CrowdThreatResponse())
|
||||
|
||||
crowdthreat.fetch_crowdthreat()
|
||||
|
||||
assert _store.latest_data["crowdthreat"][0]["id"] == "ct-1"
|
||||
assert _store.latest_data["crowdthreat"][0]["source"] == "CrowdThreat"
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Issue #250 (tg12): Docker bridge local-operator trust must be bound to
|
||||
the frontend container's hostname, not the entire 172.16.0.0/12 range.
|
||||
|
||||
Previous behavior trusted ANY private-RFC1918 source IP on the bridge
|
||||
when ``SHADOWBROKER_TRUST_DOCKER_BRIDGE_LOCAL_OPERATOR=1``. On a shared
|
||||
Docker host this granted local-operator privileges to any other
|
||||
container that could route to the backend's bridge — far broader than
|
||||
intended.
|
||||
|
||||
The fix narrows trust to source IPs that forward-resolve from one of the
|
||||
configured frontend container hostnames (default: the compose service
|
||||
name ``frontend`` plus the explicit ``container_name``
|
||||
``shadowbroker-frontend``). Operators with renamed containers can list
|
||||
the new names in ``SHADOWBROKER_TRUSTED_FRONTEND_HOSTS``.
|
||||
|
||||
These tests exercise the resolution helpers directly so that we don't
|
||||
need a live Docker daemon to validate the contract.
|
||||
"""
|
||||
import socket
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _trusted_bridge_frontend_hostnames — env parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestTrustedHostnameParsing:
|
||||
def _fn(self):
|
||||
from auth import _trusted_bridge_frontend_hostnames
|
||||
return _trusted_bridge_frontend_hostnames
|
||||
|
||||
def test_default_covers_compose_service_and_container_name(self):
|
||||
with patch.dict("os.environ", {}, clear=False):
|
||||
# Make sure the env var is not set so we exercise the default.
|
||||
import os
|
||||
os.environ.pop("SHADOWBROKER_TRUSTED_FRONTEND_HOSTS", None)
|
||||
assert self._fn()() == ["frontend", "shadowbroker-frontend"]
|
||||
|
||||
def test_custom_list_via_env(self):
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{"SHADOWBROKER_TRUSTED_FRONTEND_HOSTS": "my-ui,alt-frontend"},
|
||||
):
|
||||
assert self._fn()() == ["my-ui", "alt-frontend"]
|
||||
|
||||
def test_whitespace_trimmed(self):
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{"SHADOWBROKER_TRUSTED_FRONTEND_HOSTS": " my-ui , alt-frontend "},
|
||||
):
|
||||
assert self._fn()() == ["my-ui", "alt-frontend"]
|
||||
|
||||
def test_empty_env_falls_back_to_default(self):
|
||||
# An empty string still falls back to the bundled defaults so a
|
||||
# misconfigured env var doesn't silently dismantle bridge trust.
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{"SHADOWBROKER_TRUSTED_FRONTEND_HOSTS": ""},
|
||||
):
|
||||
# Per docs: empty string sets the env var to "" so os.environ.get
|
||||
# returns "" — that string is parsed and yields []. We assert
|
||||
# that empty parse yields [] (caller fail-closes from there).
|
||||
assert self._fn()() == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _resolve_trusted_bridge_ips — DNS resolution with cache + fail-closed
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestResolveTrustedBridgeIps:
|
||||
def setup_method(self):
|
||||
# Reset the module-level cache before each test so prior tests
|
||||
# don't bleed state across cases.
|
||||
from auth import _DOCKER_BRIDGE_TRUST_CACHE
|
||||
_DOCKER_BRIDGE_TRUST_CACHE["ips"] = frozenset()
|
||||
_DOCKER_BRIDGE_TRUST_CACHE["expires"] = 0.0
|
||||
|
||||
def test_resolves_configured_hostnames(self):
|
||||
from auth import _resolve_trusted_bridge_ips
|
||||
|
||||
def fake_gethostbyname_ex(host):
|
||||
mapping = {
|
||||
"frontend": ("frontend", [], ["172.18.0.3"]),
|
||||
"shadowbroker-frontend": ("shadowbroker-frontend", [], ["172.18.0.3", "172.18.0.4"]),
|
||||
}
|
||||
if host not in mapping:
|
||||
raise socket.gaierror("no such host")
|
||||
return mapping[host]
|
||||
|
||||
with patch("socket.gethostbyname_ex", side_effect=fake_gethostbyname_ex):
|
||||
ips = _resolve_trusted_bridge_ips()
|
||||
assert ips == frozenset({"172.18.0.3", "172.18.0.4"})
|
||||
|
||||
def test_fail_closed_when_dns_returns_nothing(self):
|
||||
from auth import _resolve_trusted_bridge_ips
|
||||
|
||||
def always_fail(host):
|
||||
raise socket.gaierror("no resolver")
|
||||
|
||||
with patch("socket.gethostbyname_ex", side_effect=always_fail):
|
||||
ips = _resolve_trusted_bridge_ips()
|
||||
assert ips == frozenset()
|
||||
|
||||
def test_partial_resolution_is_kept(self):
|
||||
"""If one hostname resolves and another fails, we keep the
|
||||
successful one rather than discarding the whole set."""
|
||||
from auth import _resolve_trusted_bridge_ips
|
||||
|
||||
def partial(host):
|
||||
if host == "frontend":
|
||||
return ("frontend", [], ["172.18.0.3"])
|
||||
raise socket.gaierror("missing")
|
||||
|
||||
with patch("socket.gethostbyname_ex", side_effect=partial):
|
||||
ips = _resolve_trusted_bridge_ips()
|
||||
assert ips == frozenset({"172.18.0.3"})
|
||||
|
||||
def test_cache_short_circuits_repeated_dns_calls(self):
|
||||
from auth import _resolve_trusted_bridge_ips
|
||||
|
||||
call_count = {"n": 0}
|
||||
|
||||
def counting(host):
|
||||
call_count["n"] += 1
|
||||
return ("frontend", [], ["172.18.0.3"])
|
||||
|
||||
with patch("socket.gethostbyname_ex", side_effect=counting):
|
||||
_resolve_trusted_bridge_ips()
|
||||
calls_after_first = call_count["n"]
|
||||
_resolve_trusted_bridge_ips()
|
||||
_resolve_trusted_bridge_ips()
|
||||
# Second + third calls hit the cache, not the DNS stub.
|
||||
assert call_count["n"] == calls_after_first
|
||||
|
||||
def test_cache_expires(self):
|
||||
from auth import _resolve_trusted_bridge_ips, _DOCKER_BRIDGE_TRUST_CACHE
|
||||
|
||||
with patch("socket.gethostbyname_ex", return_value=("frontend", [], ["172.18.0.3"])):
|
||||
_resolve_trusted_bridge_ips()
|
||||
# Force expiry.
|
||||
_DOCKER_BRIDGE_TRUST_CACHE["expires"] = 0.0
|
||||
with patch("socket.gethostbyname_ex", return_value=("frontend", [], ["172.18.0.9"])) as stub:
|
||||
ips = _resolve_trusted_bridge_ips()
|
||||
assert stub.called
|
||||
assert "172.18.0.9" in ips
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _is_docker_bridge_host — composite of the helpers above
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestIsDockerBridgeHost:
|
||||
def setup_method(self):
|
||||
from auth import _DOCKER_BRIDGE_TRUST_CACHE
|
||||
_DOCKER_BRIDGE_TRUST_CACHE["ips"] = frozenset()
|
||||
_DOCKER_BRIDGE_TRUST_CACHE["expires"] = 0.0
|
||||
|
||||
def test_trusts_resolved_frontend_ip(self):
|
||||
from auth import _is_docker_bridge_host
|
||||
|
||||
with patch("auth._resolve_trusted_bridge_ips", return_value=frozenset({"172.18.0.3"})):
|
||||
assert _is_docker_bridge_host("172.18.0.3") is True
|
||||
|
||||
def test_rejects_arbitrary_bridge_ip(self):
|
||||
"""A rogue container on the same bridge but at a different IP
|
||||
must NOT be trusted, even though it falls in 172.16.0.0/12."""
|
||||
from auth import _is_docker_bridge_host
|
||||
|
||||
with patch("auth._resolve_trusted_bridge_ips", return_value=frozenset({"172.18.0.3"})):
|
||||
assert _is_docker_bridge_host("172.18.0.99") is False
|
||||
|
||||
def test_rejects_public_ip_without_dns_work(self):
|
||||
"""Public IPs skip DNS resolution entirely (perf + safety)."""
|
||||
from auth import _is_docker_bridge_host
|
||||
|
||||
with patch("auth._resolve_trusted_bridge_ips") as stub:
|
||||
assert _is_docker_bridge_host("8.8.8.8") is False
|
||||
stub.assert_not_called()
|
||||
|
||||
def test_rejects_non_ip_input(self):
|
||||
from auth import _is_docker_bridge_host
|
||||
|
||||
assert _is_docker_bridge_host("") is False
|
||||
assert _is_docker_bridge_host("not-an-ip") is False
|
||||
assert _is_docker_bridge_host("frontend") is False
|
||||
|
||||
def test_fails_closed_when_dns_returns_empty(self):
|
||||
"""If Docker DNS can't resolve any frontend hostname, the bridge
|
||||
is not trusted — even for IPs that would have been trusted under
|
||||
the old 172.16.0.0/12 blanket policy."""
|
||||
from auth import _is_docker_bridge_host
|
||||
|
||||
with patch("auth._resolve_trusted_bridge_ips", return_value=frozenset()):
|
||||
assert _is_docker_bridge_host("172.18.0.3") is False
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Issue #199 (tg12): GDELT military incident ingestion must use HTTPS.
|
||||
|
||||
The previous code fetched ``http://data.gdeltproject.org/gdeltv2/lastupdate.txt``
|
||||
and ~48 export archives over plaintext HTTP, which let a passive observer
|
||||
identify Shadowbroker nodes by their fetch pattern and let an active MITM
|
||||
inject doctored export records into the global incident map.
|
||||
|
||||
These tests assert the URL constants and outbound URL constructor in
|
||||
``services/geopolitics.py`` only use HTTPS.
|
||||
"""
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
_GEOPOLITICS_SRC = Path(__file__).resolve().parent.parent / "services" / "geopolitics.py"
|
||||
|
||||
|
||||
def _read_source() -> str:
|
||||
return _GEOPOLITICS_SRC.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_geopolitics_does_not_use_plaintext_http_for_gdelt():
|
||||
"""No string literal in geopolitics.py should fetch GDELT over plaintext HTTP."""
|
||||
src = _read_source()
|
||||
# Strings that would issue an HTTP request — comments are excluded because
|
||||
# comments include "http://" in example URLs even after the fix.
|
||||
code_lines = [
|
||||
ln for ln in src.split("\n")
|
||||
if "http://data.gdeltproject.org" in ln and not ln.lstrip().startswith("#")
|
||||
]
|
||||
assert code_lines == [], (
|
||||
"Found plaintext http://data.gdeltproject.org usage in geopolitics.py:\n"
|
||||
+ "\n".join(code_lines)
|
||||
)
|
||||
|
||||
|
||||
def test_geopolitics_uses_https_for_gdelt():
|
||||
"""The HTTPS URLs we expect must be present."""
|
||||
src = _read_source()
|
||||
assert "https://data.gdeltproject.org/gdeltv2/lastupdate.txt" in src
|
||||
# The download URL is constructed via f-string with {fname}
|
||||
assert re.search(
|
||||
r'https://data\.gdeltproject\.org/gdeltv2/\{fname\}', src
|
||||
), "expected https URL template for individual GDELT export downloads"
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Issue #207 (tg12): /api/mesh/infonet/status accepted
|
||||
?verify_signatures=true from anonymous callers, triggering O(n_events)
|
||||
signature verification across the entire chain. Trivial DoS.
|
||||
|
||||
The fix silently downgrades the parameter to False for unauthenticated
|
||||
callers — no error surfaced, response structure unchanged, the
|
||||
expensive path runs only when the caller has authenticated.
|
||||
|
||||
These tests focus on the source-level contract because a full
|
||||
FastAPI test client doesn't have an easy hook into the ``_scoped_view_authenticated``
|
||||
helper. They lock in the key invariant: the ``effective_verify_signatures``
|
||||
value seen by ``validate_chain()`` is the AND of the request param and
|
||||
the auth check.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
_ROUTER_PATH = Path(__file__).resolve().parent.parent / "routers" / "mesh_public.py"
|
||||
|
||||
|
||||
def _read_router_source() -> str:
|
||||
return _ROUTER_PATH.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_infonet_status_gates_verify_signatures():
|
||||
"""The infonet_status route must AND verify_signatures with auth."""
|
||||
src = _read_router_source()
|
||||
# The fix introduces an `effective_verify_signatures` variable.
|
||||
assert "effective_verify_signatures" in src
|
||||
|
||||
# It must be computed as the AND of the request param and the
|
||||
# authenticated check.
|
||||
assert "bool(verify_signatures) and authenticated" in src
|
||||
|
||||
# validate_chain() must be called with the effective value, NOT the
|
||||
# raw request param.
|
||||
assert "validate_chain(verify_signatures=effective_verify_signatures)" in src
|
||||
|
||||
|
||||
def test_no_http_error_path_for_anonymous_callers():
|
||||
"""No HTTPException is raised for unauthenticated verify_signatures=true.
|
||||
|
||||
The endpoint should silently downgrade — not return 403 — so existing
|
||||
frontends that happen to pass the param see no behavior change.
|
||||
"""
|
||||
src = _read_router_source()
|
||||
# Within the infonet_status function body, there should be no
|
||||
# HTTPException(403) raised because of the verify_signatures param.
|
||||
# Find the function definition and inspect the body.
|
||||
import re
|
||||
m = re.search(
|
||||
r"async def infonet_status\(.*?\):(.+?)(?=\n@router|\nasync def |\ndef |\Z)",
|
||||
src,
|
||||
re.DOTALL,
|
||||
)
|
||||
assert m, "infonet_status function not found in source"
|
||||
body = m.group(1)
|
||||
# No explicit 403 around the verify_signatures handling.
|
||||
assert "HTTPException(status_code=403" not in body
|
||||
assert "raise HTTPException(403" not in body
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Issue #206 (tg12): KiwiSDR upstream is HTTP-only and cannot be upgraded
|
||||
to TLS. We defend with content validation + a bundled static directory
|
||||
so the layer always renders something useful and a MITM injecting
|
||||
garbage can't corrupt the map.
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from services import kiwisdr_fetcher
|
||||
from services.kiwisdr_fetcher import (
|
||||
_MIN_HEALTHY_RECEIVER_COUNT,
|
||||
_load_bundled_fallback,
|
||||
_validate_fetched_nodes,
|
||||
)
|
||||
|
||||
|
||||
def test_bundled_fallback_file_exists_and_is_nonempty():
|
||||
"""The codebase ships a static snapshot for last-resort use."""
|
||||
bundle = _load_bundled_fallback()
|
||||
assert isinstance(bundle, list)
|
||||
assert len(bundle) >= _MIN_HEALTHY_RECEIVER_COUNT
|
||||
|
||||
|
||||
def test_validation_rejects_too_few_entries():
|
||||
too_short = [{"name": "x", "lat": 0.0, "lon": 0.0, "url": ""}] * (_MIN_HEALTHY_RECEIVER_COUNT - 1)
|
||||
assert _validate_fetched_nodes(too_short) is False
|
||||
|
||||
|
||||
def test_validation_accepts_healthy_response():
|
||||
healthy = [
|
||||
{"name": f"Receiver {i}", "lat": 50.0, "lon": -1.0, "url": "http://example"}
|
||||
for i in range(_MIN_HEALTHY_RECEIVER_COUNT)
|
||||
]
|
||||
assert _validate_fetched_nodes(healthy) is True
|
||||
|
||||
|
||||
def test_validation_rejects_non_list():
|
||||
assert _validate_fetched_nodes(None) is False # type: ignore[arg-type]
|
||||
assert _validate_fetched_nodes("a string") is False # type: ignore[arg-type]
|
||||
assert _validate_fetched_nodes({}) is False # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_validation_rejects_too_many_malformed_entries():
|
||||
"""If more than 5% of entries lack a name or numeric lat, reject."""
|
||||
nodes = []
|
||||
# 100 entries, 20 of them malformed — well over the 5% threshold.
|
||||
for i in range(_MIN_HEALTHY_RECEIVER_COUNT + 50):
|
||||
if i % 5 == 0:
|
||||
nodes.append({}) # missing name + lat
|
||||
else:
|
||||
nodes.append({"name": f"R{i}", "lat": 50.0, "lon": -1.0, "url": ""})
|
||||
assert _validate_fetched_nodes(nodes) is False
|
||||
|
||||
|
||||
def test_fallback_used_when_validation_fails(monkeypatch, tmp_path):
|
||||
"""If a fetch returns garbage, the fallback chain reaches the bundle."""
|
||||
# Force disk cache miss
|
||||
fake_cache = tmp_path / "kiwisdr_cache.json"
|
||||
monkeypatch.setattr(kiwisdr_fetcher, "_CACHE_FILE", fake_cache)
|
||||
|
||||
# Make fetch_with_curl return a parseable but UNHEALTHY response
|
||||
# (only 3 entries — well below the validation threshold).
|
||||
class _GarbageResp:
|
||||
status_code = 200
|
||||
text = "var kiwisdr_com = [{\"name\":\"x\",\"gps\":\"(0,0)\"}];"
|
||||
|
||||
monkeypatch.setattr(
|
||||
"services.network_utils.fetch_with_curl", lambda *a, **kw: _GarbageResp()
|
||||
)
|
||||
|
||||
# Bypass the @cached decorator
|
||||
kiwisdr_fetcher.kiwisdr_cache.clear()
|
||||
|
||||
result = kiwisdr_fetcher.fetch_kiwisdr_nodes()
|
||||
# Should be the bundled fallback (798 entries), not the garbage (1 entry)
|
||||
assert isinstance(result, list)
|
||||
assert len(result) >= _MIN_HEALTHY_RECEIVER_COUNT
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Issue #208 (tg12): Merkle proofs were rebuilt from scratch on every
|
||||
public ``/api/mesh/infonet/sync?include_proofs=true`` request. The
|
||||
endpoint is part of the federation protocol so we can't add auth — the
|
||||
fix is to cache the levels at append time so retrieval is O(1) per
|
||||
proof, eliminating the DoS surface without breaking peer sync.
|
||||
|
||||
These tests verify:
|
||||
|
||||
* A fresh Infonet has no cache (lazy state).
|
||||
* After ``append()``, the cache is invalidated.
|
||||
* Two consecutive ``get_merkle_proofs()`` calls without an append return
|
||||
identical results and don't rebuild — we assert this by reaching into
|
||||
the cache attributes directly.
|
||||
"""
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
|
||||
from services.mesh.mesh_hashchain import Infonet
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fresh_infonet(monkeypatch, tmp_path):
|
||||
"""Build a clean Infonet rooted at a temp directory."""
|
||||
# Redirect persistence to the temp dir so we don't pollute real state.
|
||||
monkeypatch.setattr(
|
||||
"services.mesh.mesh_hashchain.CHAIN_FILE",
|
||||
tmp_path / "infonet_chain.json",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"services.mesh.mesh_hashchain.WAL_PATH",
|
||||
tmp_path / "infonet_chain.wal",
|
||||
raising=False,
|
||||
)
|
||||
inst = Infonet()
|
||||
inst.events = [] # ensure empty
|
||||
inst._invalidate_merkle_cache()
|
||||
return inst
|
||||
|
||||
|
||||
def test_cache_starts_empty(fresh_infonet):
|
||||
"""The cache fields exist and start in their lazy state."""
|
||||
assert hasattr(fresh_infonet, "_merkle_levels_cache")
|
||||
assert fresh_infonet._merkle_levels_cache is None
|
||||
assert fresh_infonet._merkle_levels_for_event_count == -1
|
||||
|
||||
|
||||
def test_get_merkle_root_populates_cache(fresh_infonet):
|
||||
"""First call computes and caches the levels."""
|
||||
# Add a synthetic event so there's something to hash
|
||||
fresh_infonet.events = [{"event_id": "a" * 64}, {"event_id": "b" * 64}]
|
||||
_ = fresh_infonet.get_merkle_root()
|
||||
assert fresh_infonet._merkle_levels_cache is not None
|
||||
assert fresh_infonet._merkle_levels_for_event_count == 2
|
||||
|
||||
|
||||
def test_repeated_root_calls_reuse_cache(fresh_infonet):
|
||||
"""The cache survives multiple reads when no events were appended."""
|
||||
fresh_infonet.events = [{"event_id": "a" * 64}, {"event_id": "b" * 64}]
|
||||
_ = fresh_infonet.get_merkle_root()
|
||||
cached_levels = fresh_infonet._merkle_levels_cache
|
||||
cached_count = fresh_infonet._merkle_levels_for_event_count
|
||||
|
||||
_ = fresh_infonet.get_merkle_root()
|
||||
# Same object — no rebuild.
|
||||
assert fresh_infonet._merkle_levels_cache is cached_levels
|
||||
assert fresh_infonet._merkle_levels_for_event_count == cached_count
|
||||
|
||||
|
||||
def test_append_invalidates_cache(fresh_infonet):
|
||||
"""After events change, the cache_for_count diverges from len(events).
|
||||
|
||||
The next read recomputes; that's the architectural point.
|
||||
"""
|
||||
fresh_infonet.events = [{"event_id": "a" * 64}]
|
||||
_ = fresh_infonet.get_merkle_root()
|
||||
assert fresh_infonet._merkle_levels_for_event_count == 1
|
||||
|
||||
# Simulate an append's side effect (the real append() also calls
|
||||
# _invalidate_merkle_cache() — we test that integration in the
|
||||
# in-tree append-flow test, not here).
|
||||
fresh_infonet.events.append({"event_id": "b" * 64})
|
||||
fresh_infonet._invalidate_merkle_cache()
|
||||
|
||||
_ = fresh_infonet.get_merkle_root()
|
||||
assert fresh_infonet._merkle_levels_for_event_count == 2
|
||||
|
||||
|
||||
def test_proofs_use_cache(fresh_infonet):
|
||||
"""get_merkle_proofs() reads from the same cache get_merkle_root() does."""
|
||||
fresh_infonet.events = [
|
||||
{"event_id": (str(i) * 64)[:64]} for i in range(8)
|
||||
]
|
||||
_ = fresh_infonet.get_merkle_root()
|
||||
cached_levels = fresh_infonet._merkle_levels_cache
|
||||
|
||||
proofs = fresh_infonet.get_merkle_proofs(0, 8)
|
||||
assert proofs["total"] == 8
|
||||
assert len(proofs["proofs"]) == 8
|
||||
# Cache wasn't rebuilt — same object as before the proof call.
|
||||
assert fresh_infonet._merkle_levels_cache is cached_levels
|
||||
|
||||
|
||||
def test_empty_chain_returns_genesis(fresh_infonet):
|
||||
"""An empty chain should serve GENESIS_HASH without computing levels."""
|
||||
from services.mesh.mesh_hashchain import GENESIS_HASH
|
||||
|
||||
root = fresh_infonet.get_merkle_root()
|
||||
assert root == GENESIS_HASH
|
||||
|
||||
proofs = fresh_infonet.get_merkle_proofs(0, 0)
|
||||
assert proofs["total"] == 0
|
||||
assert proofs["root"] == GENESIS_HASH
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Issue #203 (tg12): meshtastic_map.py was unconditionally including
|
||||
``MESHTASTIC_OPERATOR_CALLSIGN`` in the outbound User-Agent header,
|
||||
which contradicted the README's "no user data transmitted" claim.
|
||||
|
||||
The fix preserves the existing default behavior (callsign sent — that's
|
||||
what operators who configured the variable expected) but adds an
|
||||
opt-out env var ``MESHTASTIC_SEND_CALLSIGN_HEADER=false`` for
|
||||
privacy-conscious operators.
|
||||
"""
|
||||
import importlib
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _reload_meshtastic_module():
|
||||
"""Reload meshtastic_map so settings are re-read on demand."""
|
||||
if "services.fetchers.meshtastic_map" in sys.modules:
|
||||
del sys.modules["services.fetchers.meshtastic_map"]
|
||||
return importlib.import_module("services.fetchers.meshtastic_map")
|
||||
|
||||
|
||||
def test_default_behavior_includes_callsign(monkeypatch):
|
||||
"""Operators who set the callsign and don't change anything else
|
||||
keep their existing behavior (callsign sent in UA)."""
|
||||
# We test the UA construction logic by exercising the same branches
|
||||
# the fetcher uses. Direct fetch isn't run because it makes a real
|
||||
# network call — we just verify the env-var-driven decision.
|
||||
import os
|
||||
monkeypatch.setenv("MESHTASTIC_OPERATOR_CALLSIGN", "N0CALL")
|
||||
monkeypatch.delenv("MESHTASTIC_SEND_CALLSIGN_HEADER", raising=False)
|
||||
|
||||
raw = str(os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "true")).strip().lower()
|
||||
send_callsign_header = raw not in {"0", "false", "no", "off", ""}
|
||||
assert send_callsign_header is True
|
||||
|
||||
|
||||
def test_opt_out_suppresses_callsign(monkeypatch):
|
||||
"""Setting MESHTASTIC_SEND_CALLSIGN_HEADER=false suppresses the header."""
|
||||
import os
|
||||
monkeypatch.setenv("MESHTASTIC_OPERATOR_CALLSIGN", "N0CALL")
|
||||
monkeypatch.setenv("MESHTASTIC_SEND_CALLSIGN_HEADER", "false")
|
||||
|
||||
raw = str(os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "true")).strip().lower()
|
||||
send_callsign_header = raw not in {"0", "false", "no", "off", ""}
|
||||
assert send_callsign_header is False
|
||||
|
||||
|
||||
def test_various_falsy_values_all_opt_out(monkeypatch):
|
||||
"""Common falsy strings should all suppress the callsign header."""
|
||||
import os
|
||||
for falsy in ("0", "false", "FALSE", "no", "off"):
|
||||
monkeypatch.setenv("MESHTASTIC_SEND_CALLSIGN_HEADER", falsy)
|
||||
raw = str(os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "true")).strip().lower()
|
||||
send_callsign_header = raw not in {"0", "false", "no", "off", ""}
|
||||
assert send_callsign_header is False, f"value {falsy!r} did not opt out"
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Issue #205 (tg12): the OpenMHZ audio proxy must re-validate the host on
|
||||
every redirect hop, not just the first one.
|
||||
|
||||
Before this fix, ``openmhz_audio_response()`` called
|
||||
``requests.get(..., stream=True, timeout=...)`` with the default
|
||||
``allow_redirects=True``. The initial URL host was validated against
|
||||
``_OPENMHZ_AUDIO_HOSTS``, but any subsequent redirect was silently
|
||||
followed — even to ``http://127.0.0.1:8000`` or RFC1918 internal ranges.
|
||||
Classic open-redirect-to-SSRF.
|
||||
|
||||
After the fix, redirects are followed manually with per-hop host
|
||||
re-validation. Same-host redirects (CDN edge selection) still work,
|
||||
so legitimate audio playback is unaffected.
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from services.radio_intercept import _OPENMHZ_MAX_REDIRECTS, openmhz_audio_response
|
||||
|
||||
|
||||
class _Resp:
|
||||
"""Minimal mock for requests.Response."""
|
||||
|
||||
def __init__(self, status_code=200, headers=None, is_redirect=False):
|
||||
self.status_code = status_code
|
||||
self.headers = headers or {}
|
||||
self.is_redirect = is_redirect
|
||||
self.closed = False
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
def iter_content(self, chunk_size=64 * 1024):
|
||||
return iter([])
|
||||
|
||||
|
||||
@patch("services.radio_intercept.requests.get")
|
||||
def test_redirect_to_internal_address_rejected(mock_get):
|
||||
"""A 302 from media.openmhz.com -> 127.0.0.1 must be rejected."""
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "http://127.0.0.1:8000/api/secret"}, is_redirect=True),
|
||||
]
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
openmhz_audio_response("https://media.openmhz.com/audio/abc.mp3")
|
||||
assert exc_info.value.status_code == 502
|
||||
|
||||
|
||||
@patch("services.radio_intercept.requests.get")
|
||||
def test_redirect_to_arbitrary_domain_rejected(mock_get):
|
||||
"""A 302 to an attacker-controlled domain must be rejected."""
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "https://evil.example/exfil"}, is_redirect=True),
|
||||
]
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
openmhz_audio_response("https://media.openmhz.com/audio/abc.mp3")
|
||||
assert exc_info.value.status_code == 502
|
||||
|
||||
|
||||
@patch("services.radio_intercept.requests.get")
|
||||
def test_redirect_to_another_openmhz_cdn_followed(mock_get):
|
||||
"""A 302 from media.openmhz.com -> media2.openmhz.com (same allowlist) is OK."""
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "https://media2.openmhz.com/audio/abc.mp3"}, is_redirect=True),
|
||||
_Resp(status_code=200, headers={"Content-Type": "audio/mpeg"}),
|
||||
]
|
||||
resp = openmhz_audio_response("https://media.openmhz.com/audio/abc.mp3")
|
||||
# StreamingResponse-shaped object — we just check it was constructed.
|
||||
assert resp is not None
|
||||
|
||||
|
||||
@patch("services.radio_intercept.requests.get")
|
||||
def test_redirect_chain_length_bounded(mock_get):
|
||||
"""A redirect loop must terminate within _OPENMHZ_MAX_REDIRECTS."""
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "https://media.openmhz.com/loop"}, is_redirect=True)
|
||||
for _ in range(_OPENMHZ_MAX_REDIRECTS + 2)
|
||||
]
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
openmhz_audio_response("https://media.openmhz.com/audio/abc.mp3")
|
||||
assert exc_info.value.status_code == 502
|
||||
|
||||
|
||||
@patch("services.radio_intercept.requests.get")
|
||||
def test_redirect_to_http_scheme_rejected(mock_get):
|
||||
"""A 302 to http:// (instead of https://) must be rejected even on same host."""
|
||||
mock_get.side_effect = [
|
||||
_Resp(status_code=302, headers={"Location": "http://media.openmhz.com/audio/abc.mp3"}, is_redirect=True),
|
||||
]
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
openmhz_audio_response("https://media.openmhz.com/audio/abc.mp3")
|
||||
assert exc_info.value.status_code == 502
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Issues #240 & #241 (tg12): oracle market/stake resolution endpoints
|
||||
must require admin authentication.
|
||||
|
||||
Before the fix, ``POST /api/mesh/oracle/resolve`` and
|
||||
``POST /api/mesh/oracle/resolve-stakes`` were decorated with
|
||||
``@mesh_write_exempt(MeshWriteExemption.ADMIN_CONTROL)``. That decorator
|
||||
only tags the route as not requiring a mesh signed-write envelope; it
|
||||
does NOT enforce authorization. The rate limiter (5/minute) was the
|
||||
only real gate, which is wrong for control-plane state mutations.
|
||||
|
||||
The fix adds ``dependencies=[Depends(require_admin)]`` to both routes.
|
||||
These tests prove:
|
||||
|
||||
- Anonymous callers receive 403.
|
||||
- A request bearing the configured admin key passes the auth gate.
|
||||
- The underlying ledger mutator is not invoked on a 403.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
||||
_ADMIN_KEY = "test-admin-key-for-oracle-resolve-fixture-32+"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
"""TestClient with the private-lane transport middleware short-circuited.
|
||||
|
||||
The ``enforce_high_privacy_mesh`` middleware in ``main.py`` returns
|
||||
HTTP 202 ("preparing private lane") for ``/api/mesh/*`` requests
|
||||
when the Wormhole supervisor is not yet at the required transport
|
||||
tier. In tests that's always — Wormhole is not running. Patching
|
||||
``_minimum_transport_tier`` to return None disables the tier check
|
||||
for the duration of the test, letting the request reach the route
|
||||
(and therefore reach the ``Depends(require_admin)`` we are testing).
|
||||
"""
|
||||
import main
|
||||
with patch("main._minimum_transport_tier", return_value=None):
|
||||
yield TestClient(main.app, raise_server_exceptions=False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ledger():
|
||||
"""Replace oracle_ledger methods so tests don't mutate persistent state.
|
||||
|
||||
The handler does ``from services.mesh.mesh_oracle import oracle_ledger``
|
||||
at call time, so we patch the module attribute.
|
||||
"""
|
||||
fake = MagicMock()
|
||||
fake.resolve_market.return_value = (0, 0)
|
||||
fake.resolve_market_stakes.return_value = {"winners": 0, "losers": 0}
|
||||
fake.resolve_expired_stakes.return_value = []
|
||||
with patch("services.mesh.mesh_oracle.oracle_ledger", fake):
|
||||
yield fake
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /api/mesh/oracle/resolve — issue #240
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOracleResolveAuthGate:
|
||||
def test_anonymous_caller_is_rejected(self, client, mock_ledger):
|
||||
with patch("auth._current_admin_key", return_value=_ADMIN_KEY):
|
||||
r = client.post(
|
||||
"/api/mesh/oracle/resolve",
|
||||
json={"market_title": "test-market", "outcome": "Yes"},
|
||||
)
|
||||
assert r.status_code == 403
|
||||
# Critically: the ledger mutator must NOT have been called on a 403.
|
||||
assert mock_ledger.resolve_market.call_count == 0
|
||||
assert mock_ledger.resolve_market_stakes.call_count == 0
|
||||
|
||||
def test_wrong_admin_key_rejected(self, client, mock_ledger):
|
||||
with patch("auth._current_admin_key", return_value=_ADMIN_KEY):
|
||||
r = client.post(
|
||||
"/api/mesh/oracle/resolve",
|
||||
headers={"X-Admin-Key": "this-key-is-wrong"},
|
||||
json={"market_title": "test-market", "outcome": "Yes"},
|
||||
)
|
||||
assert r.status_code == 403
|
||||
assert mock_ledger.resolve_market.call_count == 0
|
||||
|
||||
def test_valid_admin_key_passes_auth_gate(self, client, mock_ledger):
|
||||
with patch("auth._current_admin_key", return_value=_ADMIN_KEY):
|
||||
r = client.post(
|
||||
"/api/mesh/oracle/resolve",
|
||||
headers={"X-Admin-Key": _ADMIN_KEY},
|
||||
json={"market_title": "test-market", "outcome": "Yes"},
|
||||
)
|
||||
# The auth gate let us through. The handler ran and called the
|
||||
# (mocked) ledger.
|
||||
assert r.status_code == 200
|
||||
assert mock_ledger.resolve_market.call_count == 1
|
||||
assert mock_ledger.resolve_market.call_args[0] == ("test-market", "Yes")
|
||||
|
||||
def test_admin_key_unset_blocks_in_production_posture(self, client, mock_ledger):
|
||||
"""When ADMIN_KEY env is not configured at all and we're not in
|
||||
debug, the endpoint must still refuse — never silently accept."""
|
||||
with (
|
||||
patch("auth._current_admin_key", return_value=""),
|
||||
patch("auth._allow_insecure_admin", return_value=False),
|
||||
patch("auth._debug_mode_enabled", return_value=False),
|
||||
patch("auth._scoped_admin_tokens", return_value={}),
|
||||
):
|
||||
r = client.post(
|
||||
"/api/mesh/oracle/resolve",
|
||||
json={"market_title": "test-market", "outcome": "Yes"},
|
||||
)
|
||||
assert r.status_code == 403
|
||||
assert mock_ledger.resolve_market.call_count == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /api/mesh/oracle/resolve-stakes — issue #241
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOracleResolveStakesAuthGate:
|
||||
def test_anonymous_caller_is_rejected(self, client, mock_ledger):
|
||||
with patch("auth._current_admin_key", return_value=_ADMIN_KEY):
|
||||
r = client.post("/api/mesh/oracle/resolve-stakes")
|
||||
assert r.status_code == 403
|
||||
assert mock_ledger.resolve_expired_stakes.call_count == 0
|
||||
|
||||
def test_wrong_admin_key_rejected(self, client, mock_ledger):
|
||||
with patch("auth._current_admin_key", return_value=_ADMIN_KEY):
|
||||
r = client.post(
|
||||
"/api/mesh/oracle/resolve-stakes",
|
||||
headers={"X-Admin-Key": "nope"},
|
||||
)
|
||||
assert r.status_code == 403
|
||||
assert mock_ledger.resolve_expired_stakes.call_count == 0
|
||||
|
||||
def test_valid_admin_key_passes_auth_gate(self, client, mock_ledger):
|
||||
with patch("auth._current_admin_key", return_value=_ADMIN_KEY):
|
||||
r = client.post(
|
||||
"/api/mesh/oracle/resolve-stakes",
|
||||
headers={"X-Admin-Key": _ADMIN_KEY},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert mock_ledger.resolve_expired_stakes.call_count == 1
|
||||
body = r.json()
|
||||
assert body["ok"] is True
|
||||
assert body["count"] == 0
|
||||
|
||||
def test_admin_key_unset_blocks_in_production_posture(self, client, mock_ledger):
|
||||
with (
|
||||
patch("auth._current_admin_key", return_value=""),
|
||||
patch("auth._allow_insecure_admin", return_value=False),
|
||||
patch("auth._debug_mode_enabled", return_value=False),
|
||||
patch("auth._scoped_admin_tokens", return_value={}),
|
||||
):
|
||||
r = client.post("/api/mesh/oracle/resolve-stakes")
|
||||
assert r.status_code == 403
|
||||
assert mock_ledger.resolve_expired_stakes.call_count == 0
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Issue #202 (tg12): the satellite overflights endpoint accepted an
|
||||
unbounded ``hours`` parameter, letting an anonymous caller trigger
|
||||
``O(catalog_size × timesteps)`` work by asking for an absurd window.
|
||||
|
||||
The fix clamps ``hours`` silently rather than raising a 422. The
|
||||
response shape is identical, just covering a shorter window — this
|
||||
keeps the API liberal in what it accepts (Postel) while removing the
|
||||
DoS surface.
|
||||
"""
|
||||
import os
|
||||
|
||||
from routers.data import _overflight_max_hours
|
||||
|
||||
|
||||
def test_default_max_hours_is_72(monkeypatch):
|
||||
monkeypatch.delenv("OVERFLIGHTS_MAX_HOURS", raising=False)
|
||||
assert _overflight_max_hours() == 72
|
||||
|
||||
|
||||
def test_env_override_accepted(monkeypatch):
|
||||
monkeypatch.setenv("OVERFLIGHTS_MAX_HOURS", "168")
|
||||
assert _overflight_max_hours() == 168
|
||||
|
||||
|
||||
def test_invalid_env_value_falls_back_to_default(monkeypatch):
|
||||
monkeypatch.setenv("OVERFLIGHTS_MAX_HOURS", "not-a-number")
|
||||
assert _overflight_max_hours() == 72
|
||||
|
||||
|
||||
def test_negative_env_value_clamped_to_minimum(monkeypatch):
|
||||
monkeypatch.setenv("OVERFLIGHTS_MAX_HOURS", "-5")
|
||||
assert _overflight_max_hours() == 1
|
||||
|
||||
|
||||
def test_clamp_arithmetic_silent():
|
||||
"""The endpoint should clamp huge requests without erroring.
|
||||
|
||||
We don't exercise the full FastAPI route (compute_overflights needs
|
||||
cached GP data), but we do verify the clamping math used by the
|
||||
route: min(requested, cap).
|
||||
"""
|
||||
requested = 1_000_000
|
||||
cap = _overflight_max_hours()
|
||||
effective = min(max(1, requested), cap)
|
||||
assert effective == cap
|
||||
assert effective < requested
|
||||
@@ -87,16 +87,32 @@ class TestRequireLocalOperator:
|
||||
assert self._call_with_host("172.16.0.5") == 403
|
||||
|
||||
def test_docker_bridge_blocked_without_compose_opt_in(self):
|
||||
# Even if DNS would resolve the frontend hostname to this IP,
|
||||
# the env opt-in is required.
|
||||
with patch.dict("os.environ", {"SHADOWBROKER_TRUST_DOCKER_BRIDGE_LOCAL_OPERATOR": ""}):
|
||||
assert self._call_with_host("172.18.0.3") == 403
|
||||
with patch("auth._resolve_trusted_bridge_ips", return_value=frozenset({"172.18.0.3"})):
|
||||
assert self._call_with_host("172.18.0.3") == 403
|
||||
|
||||
def test_docker_bridge_passes_with_compose_opt_in(self):
|
||||
# Issue #250: opt-in alone is no longer sufficient — the source IP
|
||||
# must also reverse-match a trusted frontend container hostname.
|
||||
# Here we simulate Docker DNS resolving "frontend" to 172.18.0.3.
|
||||
with patch.dict("os.environ", {"SHADOWBROKER_TRUST_DOCKER_BRIDGE_LOCAL_OPERATOR": "1"}):
|
||||
assert self._call_with_host("172.18.0.3") == 200
|
||||
with patch("auth._resolve_trusted_bridge_ips", return_value=frozenset({"172.18.0.3"})):
|
||||
assert self._call_with_host("172.18.0.3") == 200
|
||||
|
||||
def test_unknown_bridge_ip_blocked_even_with_compose_opt_in(self):
|
||||
# Issue #250 core regression: a rogue container on the same bridge
|
||||
# whose IP is NOT in the resolved frontend hostname set must NOT
|
||||
# be trusted, even when the bridge opt-in flag is on.
|
||||
with patch.dict("os.environ", {"SHADOWBROKER_TRUST_DOCKER_BRIDGE_LOCAL_OPERATOR": "1"}):
|
||||
with patch("auth._resolve_trusted_bridge_ips", return_value=frozenset({"172.18.0.3"})):
|
||||
assert self._call_with_host("172.18.0.99") == 403
|
||||
|
||||
def test_lan_ip_still_blocked_with_compose_opt_in(self):
|
||||
with patch.dict("os.environ", {"SHADOWBROKER_TRUST_DOCKER_BRIDGE_LOCAL_OPERATOR": "1"}):
|
||||
assert self._call_with_host("192.168.1.100") == 403
|
||||
with patch("auth._resolve_trusted_bridge_ips", return_value=frozenset({"172.18.0.3"})):
|
||||
assert self._call_with_host("192.168.1.100") == 403
|
||||
|
||||
def test_rfc1918_192168_blocked_without_key(self):
|
||||
assert self._call_with_host("192.168.1.100") == 403
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Issue #200 (tg12): Sentinel token cache must require knowledge of the
|
||||
client secret to hit, not just client_id.
|
||||
|
||||
Before this fix, the cache lookup was ``_sh_token_cache["client_id"] ==
|
||||
client_id``. A caller who knew a valid client_id but supplied any secret
|
||||
would hit the cache and reuse the original caller's bearer token, burning
|
||||
their Copernicus quota and accessing imagery on their account.
|
||||
|
||||
After the fix, the cache key is an HMAC of ``(client_id, client_secret)``
|
||||
under a per-process random key, so two callers with the same client_id but
|
||||
different secrets compute different fingerprints and miss each other's
|
||||
cache entries.
|
||||
"""
|
||||
from routers.tools import _credential_fingerprint, _sh_token_cache
|
||||
|
||||
|
||||
def test_same_client_id_different_secrets_yield_different_fingerprints():
|
||||
fp_a = _credential_fingerprint("client-id-X", "secret-A")
|
||||
fp_b = _credential_fingerprint("client-id-X", "secret-B")
|
||||
assert fp_a != fp_b
|
||||
|
||||
|
||||
def test_same_credentials_yield_same_fingerprint():
|
||||
"""The cache is still useful — same caller hits its own entry."""
|
||||
fp1 = _credential_fingerprint("client-id-X", "secret-A")
|
||||
fp2 = _credential_fingerprint("client-id-X", "secret-A")
|
||||
assert fp1 == fp2
|
||||
|
||||
|
||||
def test_different_client_ids_yield_different_fingerprints():
|
||||
fp_a = _credential_fingerprint("client-id-A", "shared-secret")
|
||||
fp_b = _credential_fingerprint("client-id-B", "shared-secret")
|
||||
assert fp_a != fp_b
|
||||
|
||||
|
||||
def test_cache_lookup_key_field_renamed():
|
||||
"""Catch accidental reintroduction of the client_id-only lookup."""
|
||||
# If a future commit re-adds `_sh_token_cache["client_id"]` we want this
|
||||
# test to fail loudly. The new schema only stores `credential_fp`.
|
||||
assert "client_id" not in _sh_token_cache
|
||||
assert "credential_fp" in _sh_token_cache
|
||||
|
||||
|
||||
def test_attacker_with_wrong_secret_misses_cache(monkeypatch):
|
||||
"""An attacker with valid client_id but wrong secret cannot hit the cache."""
|
||||
# Populate cache as if a legitimate caller just succeeded.
|
||||
legit_fp = _credential_fingerprint("legit-client", "legit-secret")
|
||||
_sh_token_cache["token"] = "VICTIM-BEARER-TOKEN"
|
||||
_sh_token_cache["credential_fp"] = legit_fp
|
||||
_sh_token_cache["expiry"] = 10**12 # far future
|
||||
|
||||
# Attacker arrives with the same client_id but the wrong secret.
|
||||
attacker_fp = _credential_fingerprint("legit-client", "wrong-secret")
|
||||
assert attacker_fp != legit_fp
|
||||
|
||||
# Reset cache for hygiene between tests.
|
||||
_sh_token_cache["token"] = None
|
||||
_sh_token_cache["credential_fp"] = ""
|
||||
_sh_token_cache["expiry"] = 0
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Third-party fetchers that phone home to politically/commercially
|
||||
sensitive upstreams must be operator opt-in only.
|
||||
|
||||
Companion to ``test_crowdthreat_opt_in.py`` — extends the same default-off
|
||||
posture to:
|
||||
|
||||
* EUvsDisinfo FIMI (``FIMI_ENABLED``)
|
||||
* Polymarket + Kalshi (``PREDICTION_MARKETS_ENABLED``)
|
||||
* Finnhub / yfinance financial data (``FINANCIAL_ENABLED`` /
|
||||
``FINNHUB_API_KEY``)
|
||||
* NUFORC HuggingFace dataset (``NUFORC_ENABLED``)
|
||||
|
||||
Each test asserts that with the env var unset (or set to a falsy value)
|
||||
the fetcher's network entry point is NOT called.
|
||||
"""
|
||||
|
||||
|
||||
def _explode(*_args, **_kwargs):
|
||||
raise AssertionError("upstream called while fetcher was meant to be disabled")
|
||||
|
||||
|
||||
def test_fimi_disabled_by_default_does_not_call_upstream(monkeypatch):
|
||||
from services.fetchers import _store, fimi
|
||||
|
||||
monkeypatch.delenv("FIMI_ENABLED", raising=False)
|
||||
monkeypatch.setitem(_store.latest_data, "fimi", [{"id": "old"}])
|
||||
monkeypatch.setattr(fimi, "fetch_with_curl", _explode)
|
||||
|
||||
fimi.fetch_fimi()
|
||||
|
||||
assert _store.latest_data["fimi"] == []
|
||||
|
||||
|
||||
def test_fimi_falsy_value_does_not_call_upstream(monkeypatch):
|
||||
from services.fetchers import _store, fimi
|
||||
|
||||
monkeypatch.setenv("FIMI_ENABLED", "false")
|
||||
monkeypatch.setitem(_store.latest_data, "fimi", [{"id": "old"}])
|
||||
monkeypatch.setattr(fimi, "fetch_with_curl", _explode)
|
||||
|
||||
fimi.fetch_fimi()
|
||||
|
||||
assert _store.latest_data["fimi"] == []
|
||||
|
||||
|
||||
def test_prediction_markets_disabled_by_default(monkeypatch):
|
||||
from services.fetchers import _store, prediction_markets
|
||||
|
||||
monkeypatch.delenv("PREDICTION_MARKETS_ENABLED", raising=False)
|
||||
monkeypatch.setitem(_store.latest_data, "prediction_markets", [{"id": "old"}])
|
||||
monkeypatch.setattr(
|
||||
prediction_markets, "fetch_prediction_markets_raw", _explode
|
||||
)
|
||||
|
||||
prediction_markets.fetch_prediction_markets()
|
||||
|
||||
assert _store.latest_data["prediction_markets"] == []
|
||||
|
||||
|
||||
def test_financial_disabled_when_no_optin_or_api_key(monkeypatch):
|
||||
"""yfinance fallback path must not run silently — needs FINANCIAL_ENABLED."""
|
||||
from services.fetchers import _store, financial
|
||||
|
||||
monkeypatch.delenv("FINANCIAL_ENABLED", raising=False)
|
||||
monkeypatch.delenv("FINNHUB_API_KEY", raising=False)
|
||||
monkeypatch.setitem(_store.latest_data, "financial", {"BTC": {"price": 1}})
|
||||
monkeypatch.setattr(financial, "_fetch_finnhub_quote", _explode)
|
||||
monkeypatch.setattr(financial, "_fetch_yfinance_single", _explode)
|
||||
|
||||
financial.fetch_financial_markets()
|
||||
|
||||
assert _store.latest_data["financial"] == {}
|
||||
|
||||
|
||||
def test_financial_enabled_via_finnhub_api_key(monkeypatch):
|
||||
"""Presence of FINNHUB_API_KEY counts as explicit opt-in."""
|
||||
from services.fetchers import financial
|
||||
|
||||
monkeypatch.delenv("FINANCIAL_ENABLED", raising=False)
|
||||
monkeypatch.setenv("FINNHUB_API_KEY", "test-key")
|
||||
|
||||
assert financial.financial_fetch_enabled() is True
|
||||
|
||||
|
||||
def test_nuforc_disabled_by_default_skips_download(monkeypatch):
|
||||
from services.fetchers import nuforc_enrichment
|
||||
|
||||
monkeypatch.delenv("NUFORC_ENABLED", raising=False)
|
||||
monkeypatch.setattr(nuforc_enrichment, "fetch_with_curl", _explode)
|
||||
|
||||
result = nuforc_enrichment._download_and_build()
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_news_default_on_but_killable(monkeypatch):
|
||||
"""News defaults on (kill switch only), but NEWS_ENABLED=false must disable it."""
|
||||
from services.fetchers import _store, news
|
||||
|
||||
monkeypatch.setenv("NEWS_ENABLED", "false")
|
||||
monkeypatch.setitem(_store.latest_data, "news", [{"id": "old"}])
|
||||
monkeypatch.setattr(news, "fetch_with_curl", _explode)
|
||||
|
||||
news.fetch_news()
|
||||
|
||||
assert _store.latest_data["news"] == []
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Issue #251 (tg12): Tor bundle extraction must refuse symlink and
|
||||
hardlink members.
|
||||
|
||||
The previous extractor checked ``member.name`` against path traversal
|
||||
but never inspected ``member.linkname``. Python 3.11's ``tarfile``
|
||||
honors symlinks during ``extractall()``, so a malicious archive could
|
||||
ship a member named ``innocent.txt`` whose linkname points at an
|
||||
arbitrary filesystem location. After extraction, reads of innocent.txt
|
||||
dereference to that location; writes corrupt it.
|
||||
|
||||
The fix categorically refuses any link member during extraction.
|
||||
Tor Expert Bundles never legitimately contain symlinks or hardlinks,
|
||||
so this is non-disruptive for real updates and a hard stop for hostile
|
||||
archives.
|
||||
|
||||
These tests build synthetic tar archives covering each refused case
|
||||
and assert ``_extract_tor_bundle_safely`` rejects them.
|
||||
"""
|
||||
import io
|
||||
import os
|
||||
import stat
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from services.tor_hidden_service import _extract_tor_bundle_safely
|
||||
|
||||
|
||||
def _build_archive(tmp_path: Path, members: list) -> Path:
|
||||
"""Write a .tar.gz with the given (name, builder) pairs.
|
||||
|
||||
Each builder is called with the open tarfile and is responsible for
|
||||
adding its member however it likes (regular file, symlink, etc.).
|
||||
"""
|
||||
archive = tmp_path / "test_bundle.tar.gz"
|
||||
with tarfile.open(str(archive), "w:gz") as tar:
|
||||
for name, builder in members:
|
||||
builder(tar, name)
|
||||
return archive
|
||||
|
||||
|
||||
def _add_regular_file(tar: tarfile.TarFile, name: str, payload: bytes = b"hello") -> None:
|
||||
info = tarfile.TarInfo(name)
|
||||
info.size = len(payload)
|
||||
info.mode = 0o644
|
||||
info.type = tarfile.REGTYPE
|
||||
tar.addfile(info, io.BytesIO(payload))
|
||||
|
||||
|
||||
def _add_symlink(tar: tarfile.TarFile, name: str, linkname: str) -> None:
|
||||
info = tarfile.TarInfo(name)
|
||||
info.size = 0
|
||||
info.type = tarfile.SYMTYPE
|
||||
info.linkname = linkname
|
||||
info.mode = 0o777
|
||||
tar.addfile(info)
|
||||
|
||||
|
||||
def _add_hardlink(tar: tarfile.TarFile, name: str, linkname: str) -> None:
|
||||
info = tarfile.TarInfo(name)
|
||||
info.size = 0
|
||||
info.type = tarfile.LNKTYPE
|
||||
info.linkname = linkname
|
||||
info.mode = 0o644
|
||||
tar.addfile(info)
|
||||
|
||||
|
||||
def _add_fifo(tar: tarfile.TarFile, name: str) -> None:
|
||||
info = tarfile.TarInfo(name)
|
||||
info.type = tarfile.FIFOTYPE
|
||||
info.mode = 0o644
|
||||
tar.addfile(info)
|
||||
|
||||
|
||||
def test_clean_archive_extracts_successfully(tmp_path):
|
||||
"""A normal archive with only regular files extracts fine."""
|
||||
install_dir = tmp_path / "install"
|
||||
install_dir.mkdir()
|
||||
|
||||
def add_normal(tar, name):
|
||||
_add_regular_file(tar, name, b"clean content")
|
||||
|
||||
archive = _build_archive(
|
||||
tmp_path,
|
||||
[
|
||||
("tor/tor.exe", add_normal),
|
||||
("tor/data/geoip", add_normal),
|
||||
],
|
||||
)
|
||||
assert _extract_tor_bundle_safely(archive, install_dir) is True
|
||||
assert (install_dir / "tor" / "tor.exe").is_file()
|
||||
assert (install_dir / "tor" / "data" / "geoip").is_file()
|
||||
|
||||
|
||||
def test_symlink_member_is_rejected(tmp_path, caplog):
|
||||
"""Issue #251 core regression: symlink members are refused."""
|
||||
install_dir = tmp_path / "install"
|
||||
install_dir.mkdir()
|
||||
|
||||
archive = _build_archive(
|
||||
tmp_path,
|
||||
[
|
||||
("tor/innocent.txt", lambda t, n: _add_symlink(t, n, "/etc/passwd")),
|
||||
],
|
||||
)
|
||||
|
||||
import logging
|
||||
|
||||
with caplog.at_level(logging.ERROR):
|
||||
result = _extract_tor_bundle_safely(archive, install_dir)
|
||||
|
||||
assert result is False
|
||||
# No file should have been created
|
||||
assert not (install_dir / "tor" / "innocent.txt").exists()
|
||||
# Log should explain why
|
||||
assert any(
|
||||
"symlinks/hardlinks are not allowed" in rec.getMessage()
|
||||
for rec in caplog.records
|
||||
)
|
||||
|
||||
|
||||
def test_hardlink_member_is_rejected(tmp_path):
|
||||
"""Hardlinks are refused for the same reason as symlinks."""
|
||||
install_dir = tmp_path / "install"
|
||||
install_dir.mkdir()
|
||||
|
||||
archive = _build_archive(
|
||||
tmp_path,
|
||||
[
|
||||
("tor/regular.txt", lambda t, n: _add_regular_file(t, n)),
|
||||
("tor/sneaky.txt", lambda t, n: _add_hardlink(t, n, "regular.txt")),
|
||||
],
|
||||
)
|
||||
assert _extract_tor_bundle_safely(archive, install_dir) is False
|
||||
# The whole extraction is refused even though only one member is bad.
|
||||
assert not (install_dir / "tor" / "regular.txt").exists()
|
||||
|
||||
|
||||
def test_symlink_with_relative_target_still_rejected(tmp_path):
|
||||
"""Even a relative symlink target inside the install dir is refused.
|
||||
|
||||
We don't allow symlinks at all — there is no legitimate Tor bundle
|
||||
use case for them, and an attacker can chain link redirections in
|
||||
ways the path-resolution check is poor at catching.
|
||||
"""
|
||||
install_dir = tmp_path / "install"
|
||||
install_dir.mkdir()
|
||||
|
||||
archive = _build_archive(
|
||||
tmp_path,
|
||||
[
|
||||
("tor/alias.txt", lambda t, n: _add_symlink(t, n, "tor/tor.exe")),
|
||||
],
|
||||
)
|
||||
assert _extract_tor_bundle_safely(archive, install_dir) is False
|
||||
|
||||
|
||||
def test_fifo_or_device_member_is_rejected(tmp_path):
|
||||
"""Non-regular-non-directory members (FIFOs, devices) are refused."""
|
||||
install_dir = tmp_path / "install"
|
||||
install_dir.mkdir()
|
||||
|
||||
archive = _build_archive(
|
||||
tmp_path,
|
||||
[
|
||||
("tor/weird.fifo", _add_fifo),
|
||||
],
|
||||
)
|
||||
assert _extract_tor_bundle_safely(archive, install_dir) is False
|
||||
|
||||
|
||||
def test_path_traversal_member_is_rejected(tmp_path):
|
||||
"""Pre-existing path-traversal guard still works under the new shape."""
|
||||
install_dir = tmp_path / "install"
|
||||
install_dir.mkdir()
|
||||
|
||||
def add_traversal(tar, name):
|
||||
_add_regular_file(tar, name)
|
||||
|
||||
# ../../escape.txt resolves outside install_dir on most platforms.
|
||||
archive = _build_archive(
|
||||
tmp_path,
|
||||
[
|
||||
("../../escape.txt", add_traversal),
|
||||
],
|
||||
)
|
||||
assert _extract_tor_bundle_safely(archive, install_dir) is False
|
||||
|
||||
|
||||
def test_malformed_tar_is_rejected(tmp_path):
|
||||
"""A corrupt/non-tar file is rejected without crashing."""
|
||||
install_dir = tmp_path / "install"
|
||||
install_dir.mkdir()
|
||||
|
||||
bogus = tmp_path / "not-a-tar.tar.gz"
|
||||
bogus.write_bytes(b"this is not a tar archive at all")
|
||||
|
||||
assert _extract_tor_bundle_safely(bogus, install_dir) is False
|
||||
|
||||
|
||||
def test_extraction_failure_does_not_leave_partial_state_referenced_to_caller(tmp_path):
|
||||
"""When extraction fails partway, the caller relies on a False return
|
||||
to know it must clean up. We test the contract here — actual cleanup
|
||||
of files that may have been written by tar.extractall() before the
|
||||
failure point isn't part of THIS helper's responsibility (the caller
|
||||
deletes the install dir if needed)."""
|
||||
install_dir = tmp_path / "install"
|
||||
install_dir.mkdir()
|
||||
|
||||
# Hostile archive: one good file, then a symlink. Whether the good
|
||||
# file was written or not, the return value must be False so the
|
||||
# caller refuses the bundle.
|
||||
archive = _build_archive(
|
||||
tmp_path,
|
||||
[
|
||||
("tor/clean.txt", lambda t, n: _add_regular_file(t, n)),
|
||||
("tor/evil-link.txt", lambda t, n: _add_symlink(t, n, "/etc/passwd")),
|
||||
],
|
||||
)
|
||||
|
||||
assert _extract_tor_bundle_safely(archive, install_dir) is False
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Issue #201 (tg12): Tor bundle integrity must come from at least one
|
||||
trusted source. Previously, if the upstream ``.sha256sum`` was
|
||||
unreachable, the bundle was extracted and executed anyway with only
|
||||
HTTPS-level transport trust.
|
||||
|
||||
The fix introduces a multi-source verification chain:
|
||||
|
||||
1. Upstream ``.sha256sum`` (current behavior)
|
||||
2. Baked-in digest list at ``backend/data/tor_bundle_digests.json``
|
||||
3. If neither source is reachable AT ALL: HTTPS-only fallback with a
|
||||
loud warning (avoids breaking first-run onboarding while the
|
||||
maintainer hasn't yet pinned a new Tor release)
|
||||
|
||||
A mismatch from a source that DID respond is always fatal — only the
|
||||
"no source reachable" case falls back to HTTPS-only.
|
||||
"""
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from services import tor_hidden_service as tor_svc
|
||||
from services.tor_hidden_service import (
|
||||
_DIGEST_PLACEHOLDER,
|
||||
_load_baked_in_digests,
|
||||
_verify_tor_bundle,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_bundle(tmp_path):
|
||||
"""A tiny synthetic 'bundle' so we can compute its digest deterministically."""
|
||||
archive = tmp_path / "fake-tor.tar.gz"
|
||||
payload = b"this is not really a tar archive"
|
||||
archive.write_bytes(payload)
|
||||
expected = hashlib.sha256(payload).hexdigest().lower()
|
||||
return archive, expected
|
||||
|
||||
|
||||
def test_baked_in_digests_skips_placeholders(tmp_path, monkeypatch):
|
||||
"""Entries with the placeholder value are filtered out."""
|
||||
digest_file = tmp_path / "digests.json"
|
||||
digest_file.write_text(
|
||||
'{"https://example.com/a.tar.gz": "PLACEHOLDER_REPLACE_BEFORE_RELEASE", '
|
||||
'"https://example.com/b.tar.gz": "deadbeef"}',
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(tor_svc, "_TOR_DIGEST_FILE", digest_file)
|
||||
|
||||
digests = _load_baked_in_digests()
|
||||
assert "https://example.com/a.tar.gz" not in digests
|
||||
assert digests.get("https://example.com/b.tar.gz") == "deadbeef"
|
||||
|
||||
|
||||
def test_verification_succeeds_when_upstream_matches(fake_bundle, monkeypatch):
|
||||
"""Path A: upstream .sha256sum returns the matching digest."""
|
||||
archive, expected = fake_bundle
|
||||
|
||||
def fake_urlretrieve(url, dest):
|
||||
dest_path = Path(dest)
|
||||
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest_path.write_text(f"{expected} bundle.tar.gz\n", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(tor_svc, "urlretrieve", fake_urlretrieve)
|
||||
monkeypatch.setattr(tor_svc, "_load_baked_in_digests", lambda: {})
|
||||
|
||||
verified, reason = _verify_tor_bundle(archive, "https://example.com/bundle.tar.gz")
|
||||
assert verified is True
|
||||
assert "upstream" in reason
|
||||
|
||||
|
||||
def test_verification_succeeds_via_baked_in_when_upstream_unreachable(fake_bundle, monkeypatch):
|
||||
"""Path B: upstream .sha256sum fails; baked-in digest matches."""
|
||||
archive, expected = fake_bundle
|
||||
|
||||
def fake_urlretrieve(url, dest):
|
||||
raise RuntimeError("upstream unreachable")
|
||||
|
||||
monkeypatch.setattr(tor_svc, "urlretrieve", fake_urlretrieve)
|
||||
monkeypatch.setattr(
|
||||
tor_svc, "_load_baked_in_digests",
|
||||
lambda: {"https://example.com/bundle.tar.gz": expected},
|
||||
)
|
||||
|
||||
verified, reason = _verify_tor_bundle(archive, "https://example.com/bundle.tar.gz")
|
||||
assert verified is True
|
||||
assert "baked-in" in reason
|
||||
|
||||
|
||||
def test_verification_fails_when_upstream_disagrees(fake_bundle, monkeypatch):
|
||||
"""Mismatch from a source that DID respond is always fatal."""
|
||||
archive, _expected = fake_bundle
|
||||
|
||||
def fake_urlretrieve(url, dest):
|
||||
dest_path = Path(dest)
|
||||
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest_path.write_text("0" * 64 + " bundle.tar.gz\n", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(tor_svc, "urlretrieve", fake_urlretrieve)
|
||||
monkeypatch.setattr(tor_svc, "_load_baked_in_digests", lambda: {})
|
||||
|
||||
verified, reason = _verify_tor_bundle(archive, "https://example.com/bundle.tar.gz")
|
||||
assert verified is False
|
||||
assert "mismatch" in reason.lower()
|
||||
|
||||
|
||||
def test_verification_fails_when_baked_in_disagrees(fake_bundle, monkeypatch):
|
||||
"""Even with no upstream, a baked-in mismatch is fatal."""
|
||||
archive, _expected = fake_bundle
|
||||
|
||||
def fake_urlretrieve(url, dest):
|
||||
raise RuntimeError("upstream unreachable")
|
||||
|
||||
monkeypatch.setattr(tor_svc, "urlretrieve", fake_urlretrieve)
|
||||
monkeypatch.setattr(
|
||||
tor_svc, "_load_baked_in_digests",
|
||||
lambda: {"https://example.com/bundle.tar.gz": "0" * 64},
|
||||
)
|
||||
|
||||
verified, reason = _verify_tor_bundle(archive, "https://example.com/bundle.tar.gz")
|
||||
assert verified is False
|
||||
|
||||
|
||||
def test_verification_falls_back_to_https_when_no_source_reachable(fake_bundle, monkeypatch, caplog):
|
||||
"""No source available → HTTPS-only fallback with a loud warning.
|
||||
|
||||
This preserves first-run onboarding while the maintainer hasn't
|
||||
yet pinned a particular Tor release in the digest file.
|
||||
"""
|
||||
archive, _expected = fake_bundle
|
||||
|
||||
def fake_urlretrieve(url, dest):
|
||||
raise RuntimeError("upstream unreachable")
|
||||
|
||||
monkeypatch.setattr(tor_svc, "urlretrieve", fake_urlretrieve)
|
||||
monkeypatch.setattr(tor_svc, "_load_baked_in_digests", lambda: {})
|
||||
|
||||
import logging
|
||||
with caplog.at_level(logging.WARNING):
|
||||
verified, reason = _verify_tor_bundle(archive, "https://example.com/bundle.tar.gz")
|
||||
assert verified is True
|
||||
assert "https-only" in reason.lower()
|
||||
assert any(
|
||||
"fell back to HTTPS-only" in record.getMessage() for record in caplog.records
|
||||
)
|
||||
@@ -0,0 +1,338 @@
|
||||
"""Issue #231 — self-update SHA-256 verification.
|
||||
|
||||
Before this fix, ``_validate_zip_hash`` returned silently whenever the
|
||||
``MESH_UPDATE_SHA256`` env var was unset (the default — nothing in the
|
||||
install docs ever told operators to set it). That made the auto-updater
|
||||
a supply-chain RCE on any compromise of the GitHub release pipeline.
|
||||
|
||||
The fix introduces a four-source verification chain:
|
||||
|
||||
1. ``MESH_UPDATE_SHA256`` env var (operator override, preserved)
|
||||
2. ``SHA256SUMS.txt`` asset published alongside the release (primary)
|
||||
3. Baked-in ``backend/data/release_digests.json`` (fallback)
|
||||
4. HTTPS-only fallback with a loud warning (preserves auto-update during
|
||||
transient outages so the user isn't stuck)
|
||||
|
||||
A mismatch from any source that DID respond is fatal. Only the "no
|
||||
source reachable at all" case falls back to HTTPS-only.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from services import updater
|
||||
from services.updater import (
|
||||
_compute_sha256,
|
||||
_fetch_sha256sums,
|
||||
_load_baked_in_release_digests,
|
||||
_validate_zip_hash,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_archive(tmp_path):
|
||||
"""A tiny synthetic zip-shaped file so we can compute a known digest."""
|
||||
archive = tmp_path / "update.zip"
|
||||
payload = b"this is not really a release archive"
|
||||
archive.write_bytes(payload)
|
||||
expected = hashlib.sha256(payload).hexdigest().lower()
|
||||
return str(archive), expected
|
||||
|
||||
|
||||
def test_baked_in_release_digests_file_loads():
|
||||
"""The shipped release_digests.json must parse and contain v0.9.79."""
|
||||
digests = _load_baked_in_release_digests()
|
||||
assert "v0.9.79" in digests
|
||||
entry = digests["v0.9.79"]
|
||||
assert "ShadowBroker_v0.9.79.zip" in entry
|
||||
digest = entry["ShadowBroker_v0.9.79.zip"]
|
||||
assert len(digest) == 64
|
||||
assert all(c in "0123456789abcdef" for c in digest)
|
||||
|
||||
|
||||
def test_baked_in_skips_comment_keys():
|
||||
"""The _comment top-level key is ignored, not surfaced as a release."""
|
||||
digests = _load_baked_in_release_digests()
|
||||
assert "_comment" not in digests
|
||||
|
||||
|
||||
def test_compute_sha256_matches_known_value(fake_archive):
|
||||
archive, expected = fake_archive
|
||||
assert _compute_sha256(archive) == expected
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Source 1: MESH_UPDATE_SHA256 env override
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_env_override_matching_passes(fake_archive, monkeypatch):
|
||||
"""Path 1: operator pinned the exact digest via env. Match = success."""
|
||||
archive, expected = fake_archive
|
||||
monkeypatch.setenv("MESH_UPDATE_SHA256", expected)
|
||||
|
||||
note = _validate_zip_hash(archive)
|
||||
assert "MESH_UPDATE_SHA256" in note
|
||||
|
||||
|
||||
def test_env_override_mismatch_fails_loudly(fake_archive, monkeypatch):
|
||||
"""Path 1: operator pinned a different digest. Mismatch = fatal."""
|
||||
archive, _expected = fake_archive
|
||||
monkeypatch.setenv("MESH_UPDATE_SHA256", "0" * 64)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
_validate_zip_hash(archive)
|
||||
assert "mismatch" in str(exc_info.value).lower()
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Source 2: SHA256SUMS.txt asset
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_sha256sums_matching_passes(fake_archive, monkeypatch):
|
||||
"""Path 2: SHA256SUMS.txt has the correct digest for our asset."""
|
||||
archive, expected = fake_archive
|
||||
monkeypatch.delenv("MESH_UPDATE_SHA256", raising=False)
|
||||
|
||||
def fake_sums(url):
|
||||
return {"ShadowBroker_v9.9.9.zip": expected}
|
||||
|
||||
monkeypatch.setattr(updater, "_fetch_sha256sums", fake_sums)
|
||||
note = _validate_zip_hash(
|
||||
archive,
|
||||
asset_name="ShadowBroker_v9.9.9.zip",
|
||||
sha256sums_url="https://example.test/SHA256SUMS.txt",
|
||||
release_tag="v9.9.9",
|
||||
)
|
||||
assert "SHA256SUMS.txt" in note
|
||||
|
||||
|
||||
def test_sha256sums_mismatch_fails_loudly(fake_archive, monkeypatch):
|
||||
"""Path 2: SHA256SUMS.txt has a different digest. Refuse."""
|
||||
archive, _expected = fake_archive
|
||||
monkeypatch.delenv("MESH_UPDATE_SHA256", raising=False)
|
||||
|
||||
def fake_sums(url):
|
||||
return {"ShadowBroker_v9.9.9.zip": "0" * 64}
|
||||
|
||||
monkeypatch.setattr(updater, "_fetch_sha256sums", fake_sums)
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
_validate_zip_hash(
|
||||
archive,
|
||||
asset_name="ShadowBroker_v9.9.9.zip",
|
||||
sha256sums_url="https://example.test/SHA256SUMS.txt",
|
||||
release_tag="v9.9.9",
|
||||
)
|
||||
assert "mismatch" in str(exc_info.value).lower()
|
||||
assert "SHA256SUMS" in str(exc_info.value)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Source 3: baked-in digest list
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_baked_in_matching_passes(fake_archive, monkeypatch):
|
||||
"""Path 3: SHA256SUMS unreachable, but the baked-in list has us."""
|
||||
archive, expected = fake_archive
|
||||
monkeypatch.delenv("MESH_UPDATE_SHA256", raising=False)
|
||||
monkeypatch.setattr(updater, "_fetch_sha256sums", lambda url: {})
|
||||
monkeypatch.setattr(
|
||||
updater,
|
||||
"_load_baked_in_release_digests",
|
||||
lambda: {"v9.9.9": {"ShadowBroker_v9.9.9.zip": expected}},
|
||||
)
|
||||
|
||||
note = _validate_zip_hash(
|
||||
archive,
|
||||
asset_name="ShadowBroker_v9.9.9.zip",
|
||||
sha256sums_url="https://example.test/SHA256SUMS.txt",
|
||||
release_tag="v9.9.9",
|
||||
)
|
||||
assert "baked-in" in note
|
||||
|
||||
|
||||
def test_baked_in_mismatch_fails_loudly(fake_archive, monkeypatch):
|
||||
"""Path 3: baked-in says something different. Refuse."""
|
||||
archive, _expected = fake_archive
|
||||
monkeypatch.delenv("MESH_UPDATE_SHA256", raising=False)
|
||||
monkeypatch.setattr(updater, "_fetch_sha256sums", lambda url: {})
|
||||
monkeypatch.setattr(
|
||||
updater,
|
||||
"_load_baked_in_release_digests",
|
||||
lambda: {"v9.9.9": {"ShadowBroker_v9.9.9.zip": "0" * 64}},
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
_validate_zip_hash(
|
||||
archive,
|
||||
asset_name="ShadowBroker_v9.9.9.zip",
|
||||
sha256sums_url="",
|
||||
release_tag="v9.9.9",
|
||||
)
|
||||
assert "mismatch" in str(exc_info.value).lower()
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Source 4: HTTPS-only fallback
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_https_only_fallback_when_no_source_available(fake_archive, monkeypatch, caplog):
|
||||
"""Path 4: nothing matches — fall back to HTTPS-only with loud warning.
|
||||
|
||||
This preserves the auto-update flow during transient outages: an
|
||||
operator on a flaky network during update doesn't get a hostile
|
||||
error, they get a degraded-but-functional update with a clear log
|
||||
message.
|
||||
"""
|
||||
import logging
|
||||
|
||||
archive, _expected = fake_archive
|
||||
monkeypatch.delenv("MESH_UPDATE_SHA256", raising=False)
|
||||
monkeypatch.setattr(updater, "_fetch_sha256sums", lambda url: {})
|
||||
monkeypatch.setattr(updater, "_load_baked_in_release_digests", lambda: {})
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
note = _validate_zip_hash(
|
||||
archive,
|
||||
asset_name="ShadowBroker_v99.99.zip",
|
||||
sha256sums_url="",
|
||||
release_tag="v99.99",
|
||||
)
|
||||
|
||||
assert "https-only" in note.lower()
|
||||
assert any(
|
||||
"fell back to HTTPS-only" in rec.getMessage() for rec in caplog.records
|
||||
)
|
||||
|
||||
|
||||
def test_https_only_fallback_when_release_tag_unknown(fake_archive, monkeypatch):
|
||||
"""Path 4 also kicks in when we have a baked-in list but it doesn't
|
||||
contain THIS release tag — e.g. a brand-new release that the local
|
||||
install hasn't seen a digest for yet."""
|
||||
archive, _expected = fake_archive
|
||||
monkeypatch.delenv("MESH_UPDATE_SHA256", raising=False)
|
||||
monkeypatch.setattr(updater, "_fetch_sha256sums", lambda url: {})
|
||||
monkeypatch.setattr(
|
||||
updater,
|
||||
"_load_baked_in_release_digests",
|
||||
lambda: {"v0.0.1": {"old.zip": "0" * 64}}, # different tag, doesn't match
|
||||
)
|
||||
|
||||
note = _validate_zip_hash(
|
||||
archive,
|
||||
asset_name="ShadowBroker_v99.99.zip",
|
||||
sha256sums_url="",
|
||||
release_tag="v99.99",
|
||||
)
|
||||
assert "https-only" in note.lower()
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Precedence (env > SHA256SUMS > baked-in > https-only)
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_env_override_beats_all_other_sources(fake_archive, monkeypatch):
|
||||
"""When MESH_UPDATE_SHA256 is set, it's the only source consulted.
|
||||
|
||||
The other sources may return false positives or negatives — they
|
||||
shouldn't be queried at all when the operator pinned an exact value.
|
||||
"""
|
||||
archive, expected = fake_archive
|
||||
monkeypatch.setenv("MESH_UPDATE_SHA256", expected)
|
||||
|
||||
def boom_sums(url):
|
||||
raise AssertionError("SHA256SUMS source was queried despite env override")
|
||||
|
||||
def boom_baked():
|
||||
raise AssertionError("Baked-in list was queried despite env override")
|
||||
|
||||
monkeypatch.setattr(updater, "_fetch_sha256sums", boom_sums)
|
||||
monkeypatch.setattr(updater, "_load_baked_in_release_digests", boom_baked)
|
||||
|
||||
note = _validate_zip_hash(
|
||||
archive,
|
||||
asset_name="any.zip",
|
||||
sha256sums_url="https://example.test/SHA256SUMS.txt",
|
||||
release_tag="any",
|
||||
)
|
||||
assert "MESH_UPDATE_SHA256" in note
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# _fetch_sha256sums parser
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_fetch_sha256sums_parses_standard_format(monkeypatch):
|
||||
"""Standard ``sha256sum`` output: ``<digest> <filename>``."""
|
||||
class _Resp:
|
||||
text = (
|
||||
"f6877c1d66614525315ea82636ce9f7b41178332c4dbf90d27431a1ea1d9cd47 ShadowBroker_v0.9.79.zip\n"
|
||||
"e0713c3cdda184cfbea750bfac0d62a35678fec00847e6476f2cac8e7e42046e ShadowBroker_0.9.79_x64_en-US.msi\n"
|
||||
)
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def fake_get(url, timeout=15):
|
||||
return _Resp()
|
||||
|
||||
monkeypatch.setattr(updater.requests, "get", fake_get)
|
||||
monkeypatch.setattr(updater, "_validate_update_url", lambda url, **kw: url)
|
||||
sums = _fetch_sha256sums("https://example.test/SHA256SUMS.txt")
|
||||
assert sums["ShadowBroker_v0.9.79.zip"].startswith("f6877c1d")
|
||||
assert sums["ShadowBroker_0.9.79_x64_en-US.msi"].startswith("e0713c3c")
|
||||
|
||||
|
||||
def test_fetch_sha256sums_handles_binary_marker(monkeypatch):
|
||||
"""sha256sum -b output: ``<digest> *<filename>``."""
|
||||
class _Resp:
|
||||
text = "f6877c1d66614525315ea82636ce9f7b41178332c4dbf90d27431a1ea1d9cd47 *ShadowBroker_v0.9.79.zip\n"
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(updater.requests, "get", lambda url, timeout=15: _Resp())
|
||||
monkeypatch.setattr(updater, "_validate_update_url", lambda url, **kw: url)
|
||||
sums = _fetch_sha256sums("https://example.test/SHA256SUMS.txt")
|
||||
assert "ShadowBroker_v0.9.79.zip" in sums
|
||||
|
||||
|
||||
def test_fetch_sha256sums_skips_malformed_lines(monkeypatch):
|
||||
"""Lines that don't parse cleanly are ignored, not aborted on."""
|
||||
class _Resp:
|
||||
text = (
|
||||
"# comment line\n"
|
||||
"\n"
|
||||
"not-a-digest bogus.txt\n"
|
||||
"f6877c1d66614525315ea82636ce9f7b41178332c4dbf90d27431a1ea1d9cd47 good.zip\n"
|
||||
)
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(updater.requests, "get", lambda url, timeout=15: _Resp())
|
||||
monkeypatch.setattr(updater, "_validate_update_url", lambda url, **kw: url)
|
||||
sums = _fetch_sha256sums("https://example.test/SHA256SUMS.txt")
|
||||
assert "good.zip" in sums
|
||||
assert "bogus.txt" not in sums
|
||||
|
||||
|
||||
def test_fetch_sha256sums_handles_network_failure(monkeypatch):
|
||||
"""If the SHA256SUMS asset can't be fetched, return empty (caller
|
||||
falls through to baked-in / https-only)."""
|
||||
import requests as _req
|
||||
|
||||
def fake_get(url, timeout=15):
|
||||
raise _req.exceptions.ConnectionError("upstream down")
|
||||
|
||||
monkeypatch.setattr(updater.requests, "get", fake_get)
|
||||
monkeypatch.setattr(updater, "_validate_update_url", lambda url, **kw: url)
|
||||
sums = _fetch_sha256sums("https://example.test/SHA256SUMS.txt")
|
||||
assert sums == {}
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@shadowbroker/desktop-shell",
|
||||
"version": "0.9.75",
|
||||
"version": "0.9.79",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@shadowbroker/desktop-shell",
|
||||
"version": "0.9.75",
|
||||
"version": "0.9.79",
|
||||
"devDependencies": {
|
||||
"typescript": "^5.6.0"
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shadowbroker/desktop-shell",
|
||||
"version": "0.9.75",
|
||||
"version": "0.9.79",
|
||||
"private": true,
|
||||
"description": "ShadowBroker desktop shell packaging, runtime bridge, and release tooling",
|
||||
"scripts": {
|
||||
|
||||
+1
-1
@@ -4201,7 +4201,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "shadowbroker-tauri-shell"
|
||||
version = "0.9.75"
|
||||
version = "0.9.79"
|
||||
dependencies = [
|
||||
"axum",
|
||||
"base64 0.22.1",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "shadowbroker-tauri-shell"
|
||||
version = "0.9.75"
|
||||
version = "0.9.79"
|
||||
edition = "2021"
|
||||
|
||||
[build-dependencies]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$schema": "https://schema.tauri.app/config/2",
|
||||
"productName": "ShadowBroker",
|
||||
"version": "0.9.75",
|
||||
"version": "0.9.79",
|
||||
"identifier": "com.shadowbroker.desktop",
|
||||
"build": {
|
||||
"frontendDist": "../../../frontend/out",
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
# Compose override that points the backend and frontend at the GitLab
|
||||
# Container Registry instead of GHCR. Use this if you prefer pulling
|
||||
# images from gitlab.com.
|
||||
#
|
||||
# Usage:
|
||||
# docker compose -f docker-compose.yml -f docker-compose.gitlab.yml pull
|
||||
# docker compose -f docker-compose.yml -f docker-compose.gitlab.yml up -d
|
||||
#
|
||||
# Both registries publish the same images on every push to main:
|
||||
# - .github/workflows/docker-publish.yml → ghcr.io (default)
|
||||
# - .gitlab-ci.yml → registry.gitlab.com (this file)
|
||||
|
||||
services:
|
||||
backend:
|
||||
image: registry.gitlab.com/bigbodycobain/shadowbroker/backend:latest
|
||||
|
||||
frontend:
|
||||
image: registry.gitlab.com/bigbodycobain/shadowbroker/frontend:latest
|
||||
+8
-1
@@ -43,6 +43,11 @@ services:
|
||||
# The bundled Docker UI talks to the backend across Docker's private bridge.
|
||||
# Treat that bridge as local operator access while ports remain bound to 127.0.0.1 by default.
|
||||
- SHADOWBROKER_TRUST_DOCKER_BRIDGE_LOCAL_OPERATOR=${SHADOWBROKER_TRUST_DOCKER_BRIDGE_LOCAL_OPERATOR:-1}
|
||||
# Issue #250: bridge trust is now bound to specific container hostnames
|
||||
# (default: 'frontend' compose service + 'shadowbroker-frontend' container
|
||||
# name). If you rename the frontend service or run with a different
|
||||
# container_name, list the hostnames here (comma-separated, no spaces).
|
||||
- SHADOWBROKER_TRUSTED_FRONTEND_HOSTS=${SHADOWBROKER_TRUSTED_FRONTEND_HOSTS:-frontend,shadowbroker-frontend}
|
||||
volumes:
|
||||
- backend_data:/app/data
|
||||
restart: unless-stopped
|
||||
@@ -62,11 +67,13 @@ services:
|
||||
image: ghcr.io/bigbodycobain/shadowbroker-frontend:latest
|
||||
container_name: shadowbroker-frontend
|
||||
ports:
|
||||
- "${BIND:-127.0.0.1}:3000:3000"
|
||||
- "${BIND:-127.0.0.1}:${FRONTEND_PORT:-3000}:3000"
|
||||
environment:
|
||||
# Points the Next.js server-side proxy at the backend container via Docker networking.
|
||||
# Change this if your backend runs on a different host or port.
|
||||
- BACKEND_URL=http://backend:8000
|
||||
# Lets the server-side proxy authenticate protected local-node API calls.
|
||||
- ADMIN_KEY=${ADMIN_KEY:-}
|
||||
depends_on:
|
||||
backend:
|
||||
condition: service_healthy
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "frontend",
|
||||
"version": "0.9.75",
|
||||
"version": "0.9.79",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "frontend",
|
||||
"version": "0.9.75",
|
||||
"version": "0.9.79",
|
||||
"dependencies": {
|
||||
"@mapbox/point-geometry": "^1.1.0",
|
||||
"@tauri-apps/plugin-process": "^2.3.1",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "frontend",
|
||||
"version": "0.9.75",
|
||||
"version": "0.9.79",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "node scripts/dev-all.cjs",
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
import React from 'react';
|
||||
import { act, cleanup, fireEvent, render, screen } from '@testing-library/react';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import AlertToast from '@/components/AlertToast';
|
||||
import type { ToastItem } from '@/hooks/useAlertToasts';
|
||||
|
||||
function buildToast(partial: Partial<ToastItem> = {}): ToastItem {
|
||||
return {
|
||||
id: 'toast-1',
|
||||
title: 'Embassy evacuation reported',
|
||||
source: 'Reuters',
|
||||
risk_score: 9,
|
||||
lat: 38.9,
|
||||
lng: -77.0,
|
||||
timestamp: Date.now(),
|
||||
...partial,
|
||||
};
|
||||
}
|
||||
|
||||
describe('AlertToast', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('renders the toast title, source, and severity label', () => {
|
||||
const toast = buildToast();
|
||||
render(
|
||||
<AlertToast toasts={[toast]} onDismiss={vi.fn()} />,
|
||||
);
|
||||
|
||||
expect(screen.getByText(toast.title)).toBeTruthy();
|
||||
expect(screen.getByText(toast.source)).toBeTruthy();
|
||||
// 9/10 -> CRITICAL
|
||||
expect(screen.getByText(/CRITICAL/)).toBeTruthy();
|
||||
expect(screen.getByText(/LVL 9\/10/)).toBeTruthy();
|
||||
});
|
||||
|
||||
it('auto-dismisses after 5 seconds', () => {
|
||||
const onDismiss = vi.fn();
|
||||
const toast = buildToast();
|
||||
render(
|
||||
<AlertToast toasts={[toast]} onDismiss={onDismiss} />,
|
||||
);
|
||||
|
||||
expect(onDismiss).not.toHaveBeenCalled();
|
||||
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(5000);
|
||||
});
|
||||
|
||||
expect(onDismiss).toHaveBeenCalledWith(toast.id);
|
||||
});
|
||||
|
||||
it('pauses auto-dismiss while the card is hovered', () => {
|
||||
const onDismiss = vi.fn();
|
||||
const toast = buildToast();
|
||||
render(
|
||||
<AlertToast toasts={[toast]} onDismiss={onDismiss} />,
|
||||
);
|
||||
|
||||
// Hover before the timer fires. mouseEnter must be flushed
|
||||
// (state update + effect cleanup) in its own act() before we
|
||||
// advance timers — otherwise the original mount-time timer is
|
||||
// still active when advanceTimersByTime runs.
|
||||
const card = screen.getByText(toast.title).closest('[class*="cursor-pointer"]')!;
|
||||
expect(card).toBeTruthy();
|
||||
|
||||
act(() => {
|
||||
fireEvent.mouseEnter(card);
|
||||
});
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(10_000);
|
||||
});
|
||||
|
||||
// Still no dismiss — timer is paused.
|
||||
expect(onDismiss).not.toHaveBeenCalled();
|
||||
|
||||
// Leave: a fresh full-lifetime timer starts.
|
||||
act(() => {
|
||||
fireEvent.mouseLeave(card);
|
||||
});
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(4_999);
|
||||
});
|
||||
expect(onDismiss).not.toHaveBeenCalled();
|
||||
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(1);
|
||||
});
|
||||
expect(onDismiss).toHaveBeenCalledWith(toast.id);
|
||||
});
|
||||
|
||||
it('dismisses on × button click without calling onFlyTo', () => {
|
||||
const onDismiss = vi.fn();
|
||||
const onFlyTo = vi.fn();
|
||||
const toast = buildToast();
|
||||
render(
|
||||
<AlertToast toasts={[toast]} onDismiss={onDismiss} onFlyTo={onFlyTo} />,
|
||||
);
|
||||
|
||||
fireEvent.click(screen.getByText('×'));
|
||||
|
||||
expect(onDismiss).toHaveBeenCalledWith(toast.id);
|
||||
expect(onFlyTo).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('flies to the toast location and dismisses on body click', () => {
|
||||
const onDismiss = vi.fn();
|
||||
const onFlyTo = vi.fn();
|
||||
const toast = buildToast();
|
||||
render(
|
||||
<AlertToast toasts={[toast]} onDismiss={onDismiss} onFlyTo={onFlyTo} />,
|
||||
);
|
||||
|
||||
fireEvent.click(screen.getByText(toast.title));
|
||||
|
||||
expect(onFlyTo).toHaveBeenCalledWith(toast.lat, toast.lng);
|
||||
expect(onDismiss).toHaveBeenCalledWith(toast.id);
|
||||
});
|
||||
});
|
||||
@@ -45,12 +45,12 @@ describe('admin/session boundary hardening', () => {
|
||||
});
|
||||
|
||||
it('accepts a verified admin key and reports the minted session as present', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ ok: true }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
// Issue #255 fix: the route no longer round-trips to the backend
|
||||
// to "verify" the key (the previous implementation called a public
|
||||
// endpoint that always returned 200, so any key was accepted when
|
||||
// ADMIN_KEY was unset). Local string comparison is the only
|
||||
// validation, so we don't mock fetch and don't assert it was called.
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost/api/admin/session', {
|
||||
@@ -65,7 +65,8 @@ describe('admin/session boundary hardening', () => {
|
||||
expect(res.status).toBe(200);
|
||||
expect(cookie).toContain('sb_admin_session=');
|
||||
expect(res.headers.get('cache-control')).toContain('no-store');
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
// Validation is local-only — no backend round-trip should happen.
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
|
||||
const getReq = new NextRequest('http://localhost/api/admin/session', {
|
||||
method: 'GET',
|
||||
@@ -88,12 +89,8 @@ describe('admin/session boundary hardening', () => {
|
||||
});
|
||||
|
||||
it('invalidates the previous admin session token when a new one is minted', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ ok: true }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
// Issue #255 fix: no backend round-trip. Validation is local-only.
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const firstReq = new NextRequest('http://localhost/api/admin/session', {
|
||||
@@ -135,21 +132,25 @@ describe('admin/session boundary hardening', () => {
|
||||
);
|
||||
const newBody = await newSessionCheck.json();
|
||||
expect(newBody.hasSession).toBe(true);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
// Local validation only — backend should not be called during minting.
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects session minting when frontend admin key is set but backend has no configured admin key', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ detail: 'Forbidden — admin key not configured' }), {
|
||||
status: 403,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
it('refuses session minting when frontend ADMIN_KEY env var is unset (#255)', async () => {
|
||||
// Issue #255 (tg12): previously, when ADMIN_KEY was unset the route
|
||||
// fell through to a public backend endpoint that always returned
|
||||
// 200, so any user-supplied key minted a full admin session. The
|
||||
// fix is to refuse minting entirely when ADMIN_KEY is unconfigured
|
||||
// and surface a clear message pointing the operator at the
|
||||
// backend's auto-trust-loopback behavior.
|
||||
process.env.ADMIN_KEY = '';
|
||||
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost/api/admin/session', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ adminKey: 'top-secret' }),
|
||||
body: JSON.stringify({ adminKey: 'any-key-an-attacker-supplies' }),
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
|
||||
@@ -158,8 +159,11 @@ describe('admin/session boundary hardening', () => {
|
||||
|
||||
expect(res.status).toBe(403);
|
||||
expect(body.ok).toBe(false);
|
||||
expect(body.detail).toBe('Forbidden — admin key not configured');
|
||||
expect(String(body.detail)).toMatch(/no admin key configured/i);
|
||||
expect(res.headers.get('set-cookie')).toBeNull();
|
||||
// Crucially: no backend round-trip happens. The previous broken
|
||||
// verifyAgainstBackend() call must NOT be re-introduced.
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not forward raw x-admin-key headers through the sensitive proxy path', async () => {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user