mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-06-06 06:13:54 +02:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 BigBodyCobain
|
6a74105e6a |
+2
-8
@@ -39,8 +39,8 @@ ADMIN_KEY=
|
||||
# NUFORC_MAPBOX_TOKEN=
|
||||
|
||||
# Optional startup-risk controls.
|
||||
# On Windows, external curl fallback is off by default. LiveUAMap uses UI consent
|
||||
# when you enable Global Incidents (or set SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=true).
|
||||
# On Windows, external curl fallback and the Playwright LiveUAMap scraper are
|
||||
# disabled by default so blocked upstream feeds cannot interrupt start.bat.
|
||||
# SHADOWBROKER_ENABLE_WINDOWS_CURL_FALLBACK=false
|
||||
# SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=false
|
||||
# AIS starts by default when AIS_API_KEY is set. Set to 0/false to force-disable.
|
||||
@@ -128,14 +128,8 @@ ADMIN_KEY=
|
||||
# MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI=backend/../ops/root_transparency_ledger.json
|
||||
|
||||
# ── Self Update ────────────────────────────────────────────────
|
||||
# Optional ZIP updater digest pin. The updater checks this first, then
|
||||
# backend/data/release_digests.json, then the release SHA256SUMS.txt asset.
|
||||
# MESH_UPDATE_SHA256=
|
||||
|
||||
# Optional strict nonce-only frontend CSP. Leave unset unless the exact build
|
||||
# has been verified to hydrate cleanly in your deployment.
|
||||
# SHADOWBROKER_STRICT_CSP=1
|
||||
|
||||
# ── Wormhole (Local Agent) ─────────────────────────────────────
|
||||
# WORMHOLE_URL=http://127.0.0.1:8787
|
||||
# WORMHOLE_TRANSPORT=direct
|
||||
|
||||
@@ -198,7 +198,6 @@ graphify-out/
|
||||
# Internal docs & brainstorming (never commit)
|
||||
# ========================
|
||||
docs/*
|
||||
!docs/OUTBOUND_DATA.md
|
||||
!docs/mesh/
|
||||
docs/mesh/*
|
||||
!docs/mesh/threat-model.md
|
||||
|
||||
+12
-42
@@ -13,22 +13,13 @@
|
||||
# 2. Reverse-mirrors main back to GitHub (only if commits land directly
|
||||
# on GitLab) so the two sources stay in sync.
|
||||
#
|
||||
# Pipelines on this repo were instant-failing for free-tier accounts until
|
||||
# identity verification was added — the May 2026 bump in this comment is
|
||||
# the marker commit that confirms runner allocation after verification.
|
||||
#
|
||||
# Auth notes:
|
||||
# - The image build/push uses $CI_JOB_TOKEN, which GitLab provides
|
||||
# automatically. No credentials need to be configured.
|
||||
# - The reverse mirror authenticates to GitHub via a per-repo SSH
|
||||
# deploy key. The private half is stored as the File-type GitLab
|
||||
# CI/CD variable GITHUB_MIRROR_SSH_KEY (Protected). The matching
|
||||
# public key is added to github.com/BigBodyCobain/Shadowbroker/
|
||||
# settings/keys with write access. This is a tighter-scoped
|
||||
# replacement for a personal access token: it can ONLY push to
|
||||
# Shadowbroker, never expires, and rotating it is a one-click
|
||||
# delete on GitHub's deploy-keys page. If the variable isn't set,
|
||||
# the mirror job is skipped — image builds still run.
|
||||
# - The reverse mirror requires a GitHub personal access token stored
|
||||
# as the GitLab CI/CD variable GITHUB_MIRROR_TOKEN (Protected + Masked).
|
||||
# Scope: public_repo (or repo for private). If the variable isn't
|
||||
# set the mirror job is skipped — image builds still run.
|
||||
|
||||
stages:
|
||||
- build
|
||||
@@ -57,11 +48,7 @@ variables:
|
||||
- docker info
|
||||
- docker login -u "$CI_REGISTRY_USER" -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
|
||||
- docker run --privileged --rm tonistiigi/binfmt --install all
|
||||
# buildx --driver docker-container can't read TLS from the env vars
|
||||
# the GitLab dind service exports. Wrap them in a docker context and
|
||||
# bind buildx to it. See https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-buildx
|
||||
- docker context create tls-env
|
||||
- docker buildx create --use --name multiarch --driver docker-container tls-env
|
||||
- docker buildx create --use --name multiarch --driver docker-container
|
||||
|
||||
# ── Backend image ────────────────────────────────────────────────────────
|
||||
build-backend:
|
||||
@@ -106,35 +93,18 @@ build-frontend:
|
||||
- .gitlab-ci.yml
|
||||
|
||||
# ── Reverse mirror to GitHub ─────────────────────────────────────────────
|
||||
# Pushes refs/heads/main to github.com/BigBodyCobain/Shadowbroker via SSH
|
||||
# using a per-repo deploy key. Fast-forward-only by default — if GitLab
|
||||
# main and GitHub main have diverged, the push fails loudly rather than
|
||||
# silently overwriting either side.
|
||||
# Pushes refs/heads/main to github.com/BigBodyCobain/Shadowbroker.
|
||||
# Fast-forward-only — if GitLab main and GitHub main have diverged, this
|
||||
# fails loudly rather than silently overwriting either side.
|
||||
#
|
||||
# Only runs if GITHUB_MIRROR_SSH_KEY is set as a File-type CI/CD variable.
|
||||
# See the header comment of this file for setup instructions.
|
||||
# Only runs if GITHUB_MIRROR_TOKEN is set as a CI/CD variable. See the
|
||||
# header comment of this file for setup instructions.
|
||||
mirror-to-github:
|
||||
stage: mirror
|
||||
image: alpine:3.20
|
||||
needs: []
|
||||
before_script:
|
||||
- apk add --no-cache git openssh-client ca-certificates
|
||||
- mkdir -p ~/.ssh
|
||||
- chmod 700 ~/.ssh
|
||||
# Install the deploy key. File-type CI variable exposes the path; copy
|
||||
# to ~/.ssh/id_ed25519 with restrictive perms so ssh accepts it.
|
||||
- cp "$GITHUB_MIRROR_SSH_KEY" ~/.ssh/id_ed25519
|
||||
- chmod 600 ~/.ssh/id_ed25519
|
||||
# Pin github.com's current host keys so we never trust a man-in-the-
|
||||
# middle. Sourced from https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints
|
||||
# (rotated 2023-03-24 after the previous RSA key leak).
|
||||
- |
|
||||
cat > ~/.ssh/known_hosts <<'EOF'
|
||||
github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
|
||||
github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
|
||||
github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
|
||||
EOF
|
||||
- chmod 644 ~/.ssh/known_hosts
|
||||
script:
|
||||
- git config --global user.email "ci-mirror@gitlab.com"
|
||||
- git config --global user.name "GitLab CI Mirror"
|
||||
@@ -145,7 +115,7 @@ mirror-to-github:
|
||||
- cd repo
|
||||
- >
|
||||
git push
|
||||
"git@github.com:BigBodyCobain/Shadowbroker.git"
|
||||
"https://x-access-token:${GITHUB_MIRROR_TOKEN}@github.com/BigBodyCobain/Shadowbroker.git"
|
||||
"${CI_COMMIT_SHA}:refs/heads/main"
|
||||
rules:
|
||||
- if: $CI_COMMIT_BRANCH == "main" && $GITHUB_MIRROR_SSH_KEY
|
||||
- if: $CI_COMMIT_BRANCH == "main" && $GITHUB_MIRROR_TOKEN
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
|
||||
**ShadowBroker** is a decentralized intelligence platform that aggregates real-time, multi-domain OSINT telemetry from 60+ live intelligence feeds into a single dark-ops map interface. Aircraft, ships, satellites, conflict zones, CCTV networks, GPS jamming, internet-connected devices, police scanners, mesh radio nodes, and breaking geopolitical events — all updating in real time on one screen as well as an obfuscated communications protocol and information exchange infrastructure.
|
||||
|
||||
Built with **Next.js**, **MapLibre GL**, **FastAPI**, and **Python**. 35+ toggleable data layers, including SAR ground-change detection. Multiple visual modes (DEFAULT / SATELLITE / FLIR / NVG / CRT). Right-click any point on Earth for a country dossier, head-of-state lookup, and the latest Sentinel-2 satellite photo. ShadowBroker has no accounts, product telemetry, or analytics; the dashboard talks to your self-hosted backend, while optional live OSINT panels may contact their configured public data providers when you use them.
|
||||
Built with **Next.js**, **MapLibre GL**, **FastAPI**, and **Python**. 35+ toggleable data layers, including SAR ground-change detection. Multiple visual modes (DEFAULT / SATELLITE / FLIR / NVG / CRT). Right-click any point on Earth for a country dossier, head-of-state lookup, and the latest Sentinel-2 satellite photo. No user data is collected or transmitted — the dashboard runs entirely in your browser against a self-hosted backend.
|
||||
|
||||
Designed for analysts, researchers, radio operators, and anyone who wants to see what the world looks like when every public signal is on the same map.
|
||||
|
||||
@@ -28,7 +28,7 @@ Designed for analysts, researchers, radio operators, and anyone who wants to see
|
||||
|
||||
A surprising amount of global telemetry is already public — aircraft ADS-B broadcasts, maritime AIS signals, satellite orbital data, earthquake sensors, mesh radio networks, police scanner feeds, environmental monitoring stations, internet infrastructure telemetry, and more. This data is scattered across dozens of tools and APIs. ShadowBroker combines all of it into a single interface.
|
||||
|
||||
The project does not introduce new surveillance capabilities — it aggregates and visualizes existing public datasets. It is fully open-source so anyone can audit exactly what data is accessed and how. ShadowBroker does not include product telemetry, analytics, or accounts. Operator-supplied keys stay in your local deployment, but live OSINT features necessarily make outbound requests to the public data providers you enable or query.
|
||||
The project does not introduce new surveillance capabilities — it aggregates and visualizes existing public datasets. It is fully open-source so anyone can audit exactly what data is accessed and how. No user data is collected or transmitted — everything runs locally against a self-hosted backend. No telemetry, no analytics, no accounts.
|
||||
|
||||
### Shodan Connector
|
||||
|
||||
@@ -113,20 +113,6 @@ That's it. `pull` grabs the latest images, `up -d` restarts the containers.
|
||||
>
|
||||
> Podman users should run the equivalent provider command, for example `podman-compose pull` and `podman-compose up -d`, or use `./compose.sh --engine podman pull` and `./compose.sh --engine podman up -d` from a bash-compatible shell.
|
||||
|
||||
### Update Integrity
|
||||
|
||||
Docker updates are delivered through signed container registries. The legacy ZIP self-updater verifies release archives through this chain, in order:
|
||||
|
||||
* `MESH_UPDATE_SHA256` when an operator pins a digest explicitly.
|
||||
* `backend/data/release_digests.json` for bundled release pins.
|
||||
* The release `SHA256SUMS.txt` asset on GitHub when a bundled pin is not present.
|
||||
|
||||
Release maintainers should run `python backend/scripts/release_helper.py hash <ShadowBroker_vX.Y.Z.zip>` before publishing, then publish `SHA256SUMS.txt` and update `backend/data/release_digests.json` when shipping a ZIP updater target. The updater keeps the operator override path intact instead of failing closed on missing bundled digests, so existing installs do not get stranded by a release-process mistake.
|
||||
|
||||
### CSP Hardening
|
||||
|
||||
The production frontend ships with a hydration-compatible CSP and a strict nonce-only CSP in `Content-Security-Policy-Report-Only`. Set `SHADOWBROKER_STRICT_CSP=1` only after verifying the exact build hydrates correctly in your deployment. Runtime Google Fonts are not required; the bundled Next font pipeline serves the dashboard font from the app build.
|
||||
|
||||
### ⚠️ **Stuck on the old version?**
|
||||
|
||||
**If `git pull` fails or `docker compose up` keeps building from source instead of pulling images**, your clone predates a March 2026 repository migration that rewrote commit history. A normal `git pull` cannot fix this. Run:
|
||||
@@ -188,7 +174,7 @@ ShadowBroker v0.9.7 ships **InfoNet** (decentralized intelligence mesh + Soverei
|
||||
| Channel | Privacy Status | Details |
|
||||
|---|---|---|
|
||||
| **Meshtastic / APRS** | **PUBLIC** | RF radio transmissions are public and interceptable by design. |
|
||||
| **InfoNet Gate Chat** | **OBFUSCATED** | Messages are obfuscated with gate personas and canonical payload signing, but NOT end-to-end encrypted. Metadata is not hidden despite being designed through Tor and Reticulum (Work in progress). |
|
||||
| **InfoNet Gate Chat** | **OBFUSCATED** | Messages are obfuscated with gate personas and canonical payload signing, but NOT end-to-end encrypted. Metadata is not hidden. |
|
||||
| **Dead Drop DMs** | **STRONGEST CURRENT LANE** | Token-based epoch mailbox with SAS word verification. Strongest lane in this build, but not yet confidently private. |
|
||||
| **Sovereign Shell governance** | **PUBLIC LEDGER** | Petitions, votes, upgrade hashes, and dispute stakes are signed events on a public hashchain. Pseudonymous via gate persona, but governance actions are intentionally observable. |
|
||||
| **Privacy primitives (RingCT / stealth / DEX)** | **NOT YET WIRED** | Locked Protocol contracts are in place, but the cryptographic scheme has not been chosen. The privacy-core Rust crate is the integration target for a future sprint. |
|
||||
@@ -213,7 +199,7 @@ The first decentralized intelligence communication and governance layer built di
|
||||
|
||||
**Communication layer (since v0.9.6):**
|
||||
|
||||
* **InfoNet Experimental Testnet** — A global, obfuscated message relay using Tor and Reticulum. Anyone running ShadowBroker can transmit and receive on the InfoNet. Messages pass through a Wormhole relay layer with gate personas, Ed25519 canonical payload signing, and transport obfuscation.
|
||||
* **InfoNet Experimental Testnet** — A global, obfuscated message relay. Anyone running ShadowBroker can transmit and receive on the InfoNet. Messages pass through a Wormhole relay layer with gate personas, Ed25519 canonical payload signing, and transport obfuscation.
|
||||
* **Mesh Chat Panel** — Three-tab interface: **INFONET** (gate chat with obfuscated transport), **MESH** (Meshtastic radio integration), **DEAD DROP** (peer-to-peer message exchange with token-based epoch mailboxes — strongest current lane).
|
||||
* **Gate Persona System** — Pseudonymous identities with Ed25519 signing keys, prekey bundles, SAS word contact verification, and abuse reporting.
|
||||
* **Mesh Terminal** — Built-in CLI: `send`, `dm`, market commands, gate state inspection. Draggable panel, minimizes to the top bar. Type `help` to see all commands.
|
||||
@@ -233,7 +219,7 @@ The first decentralized intelligence communication and governance layer built di
|
||||
|
||||
**Privacy primitive runway (NEW in v0.9.7):**
|
||||
|
||||
* **Function Keys — Anonymous Credential Scaffolding** — The plumbing is in place for nullifiers, challenge-response, two-phase commit receipts, enumerated denial codes, and batched settlement. Today's challenge-response is an HMAC-based placeholder for integration testing, not a production anonymous or zero-knowledge citizenship proof. True unlinkable issuance still waits on a primitive decision (RSA blind sigs vs BBS+ vs U-Prove vs Idemix).
|
||||
* **Function Keys — Anonymous Citizenship Proof** — A citizen proves "I am an Infonet citizen" without revealing their Infonet identity. 5 of 6 pieces shipped: nullifiers, challenge-response, two-phase commit receipts, enumerated denial codes, batched settlement. Issuance via blind signatures waits on a primitive decision (RSA blind sigs vs BBS+ vs U-Prove vs Idemix).
|
||||
* **Locked Protocol Contracts** — Stable interfaces in `services/infonet/privacy/contracts.py` for ring signatures, stealth addresses, Pedersen commitments, range proofs, and DEX matching. The `privacy-core` Rust crate is the integration target — no caller of the privacy module needs to know which scheme is active.
|
||||
* **Sprint 11+ Path** — When the cryptographic scheme is chosen, primitives wire into the locked Protocols without API churn.
|
||||
|
||||
@@ -577,8 +563,6 @@ ShadowBroker v0.9.7 is composed of three vertically-stacked planes — the **Ope
|
||||
| [OSM Nominatim](https://nominatim.openstreetmap.org) | Place name geocoding (LOCATE bar) | On-demand | No |
|
||||
| [CARTO Basemaps](https://carto.com) | Dark map tiles | Continuous | No |
|
||||
|
||||
**Outbound privacy & audit (#348–#366):** Each self-hosted install uses its own backend IP and per-install User-Agent handle. See [docs/OUTBOUND_DATA.md](docs/OUTBOUND_DATA.md) for what contacts third parties, opt-in/env controls, and accepted tradeoffs (CCTV Referer, basemap CDN, LiveUAMap, etc.).
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Getting Started
|
||||
@@ -600,16 +584,9 @@ Open `http://localhost:3000` to view the dashboard.
|
||||
> **Deploying publicly or on a LAN?** No configuration needed for most setups.
|
||||
> The frontend proxies all API calls through the Next.js server to `BACKEND_URL`,
|
||||
> which defaults to `http://backend:8000` (Docker internal networking).
|
||||
> Host port `8000` is only published for local API/debug access (`127.0.0.1:8000`
|
||||
> in `docker-compose.yml`). If it conflicts with another service, set
|
||||
> `BACKEND_PORT=8001` in `.env`; leave `BACKEND_URL` as `http://backend:8000`
|
||||
> because that is the Docker-internal port.
|
||||
>
|
||||
> **Running the backend outside Docker** (`cd backend && python main.py`):
|
||||
> the dev server binds **loopback only** (`127.0.0.1:8000`) so other machines on
|
||||
> your LAN cannot hit admin/local-trust routes with an empty `ADMIN_KEY`. Set
|
||||
> `SHADOWBROKER_DEV_BIND_ALL=true` in `.env` only when you deliberately need
|
||||
> `0.0.0.0` and use a strong `ADMIN_KEY` for any non-local callers.
|
||||
> Host port `8000` is only published for local API/debug access. If it conflicts
|
||||
> with another service, set `BACKEND_PORT=8001` in `.env`; leave `BACKEND_URL`
|
||||
> as `http://backend:8000` because that is the Docker-internal port.
|
||||
> The backend memory cap is controlled by `BACKEND_MEMORY_LIMIT` and defaults
|
||||
> to `4G`. If Docker reports OOM events, the backend will restart and slow
|
||||
> layers can look empty until they repopulate.
|
||||
|
||||
+13
-43
@@ -18,15 +18,6 @@ AIS_API_KEY= # https://aisstream.io/ — free tier WebSocket key
|
||||
# AISHUB_USERNAME=
|
||||
# AISHUB_POLL_INTERVAL_MINUTES=20
|
||||
|
||||
# `python main.py` (uvicorn reload) binds 127.0.0.1:8000 by default so LAN clients
|
||||
# cannot reach a dev server with empty ADMIN_KEY (#375). Set true only when you
|
||||
# intentionally need 0.0.0.0 and understand the local-trust implications.
|
||||
# SHADOWBROKER_DEV_BIND_ALL=false
|
||||
#
|
||||
# Thread pool for GDELT, LiveUAMap, CCTV ingest, and slow-tier refresh batches.
|
||||
# Keeps heavy jobs from starving fast flight/ship workers (default 2).
|
||||
# SHADOWBROKER_HEAVY_FETCH_WORKERS=2
|
||||
|
||||
# Override allowed CORS origins (comma-separated). Defaults to localhost + LAN auto-detect.
|
||||
# CORS_ORIGINS=http://192.168.1.50:3000,https://my-domain.com
|
||||
|
||||
@@ -40,9 +31,11 @@ AIS_API_KEY= # https://aisstream.io/ — free tier WebSocket key
|
||||
# Requires MESH_DEBUG_MODE=true; do not enable this for ordinary use.
|
||||
# ALLOW_INSECURE_ADMIN=false
|
||||
|
||||
# Per-install operator handle. Round 7a: outbound third-party API calls send
|
||||
# this handle as the User-Agent (e.g. operator-7f3a92), not a shared app name,
|
||||
# so upstreams rate-limit one install instead of blocking every user.
|
||||
# Per-install operator handle. Round 7a: every outbound third-party API
|
||||
# call (Wikipedia, Wikidata, Nominatim, GDELT, OpenMHz, Broadcastify,
|
||||
# weather.gov, NUFORC, etc.) includes this handle in the User-Agent so
|
||||
# upstreams can rate-limit / contact the specific install instead of
|
||||
# treating every Shadowbroker user as one entity.
|
||||
#
|
||||
# Default empty -> a stable pseudonymous handle (e.g. "operator-7f3a92") is
|
||||
# auto-generated on first run and persisted to backend/data/operator_handle.json.
|
||||
@@ -50,8 +43,10 @@ AIS_API_KEY= # https://aisstream.io/ — free tier WebSocket key
|
||||
# set it here. Special characters are sanitized to dashes.
|
||||
# OPERATOR_HANDLE=
|
||||
|
||||
# Full User-Agent override (replaces the operator handle entirely). Rare;
|
||||
# most installs should use OPERATOR_HANDLE only.
|
||||
# Default outbound User-Agent for all third-party HTTP fetchers. Operators
|
||||
# who run a public relay and want a completely custom UA can set this; it
|
||||
# bypasses the per-operator helper entirely. Most installs should leave it
|
||||
# unset and use OPERATOR_HANDLE instead.
|
||||
# SHADOWBROKER_USER_AGENT=
|
||||
|
||||
# Nominatim-specific User-Agent override (OSM usage policy). Leave unset to
|
||||
@@ -71,29 +66,14 @@ AIS_API_KEY= # https://aisstream.io/ — free tier WebSocket key
|
||||
# FIMI_ENABLED=false
|
||||
#
|
||||
# Polymarket + Kalshi — US political/election prediction markets.
|
||||
# Default off; enable from Global Threat Intercept (MKT toggle) or set true here.
|
||||
# PREDICTION_MARKETS_ENABLED=false
|
||||
# When enabled, polls use a jittered schedule (not the fixed 5-minute slow tier):
|
||||
# PREDICTION_MARKETS_INTERVAL_MINUTES=7
|
||||
# PREDICTION_MARKETS_SCHEDULER_JITTER_S=240
|
||||
# PREDICTION_MARKETS_INITIAL_DELAY_MAX_S=180
|
||||
# PREDICTION_MARKETS_PRE_FETCH_JITTER_S=90
|
||||
# PREDICTION_MARKETS_PROVIDER_GAP_JITTER_S=45
|
||||
# MESH_POLYMARKET_PAGE_DELAY_JITTER_S=0.08
|
||||
# MESH_KALSHI_PAGE_DELAY_JITTER_S=0.2
|
||||
#
|
||||
# Finnhub fallback / yfinance — financial market data.
|
||||
# Set FINNHUB_API_KEY to enable Finnhub, or set FINANCIAL_ENABLED=true to allow
|
||||
# the unauthenticated yfinance fallback to call Yahoo Finance.
|
||||
# FINANCIAL_ENABLED=false
|
||||
#
|
||||
# NUFORC UAP map layer — live scrape from nuforc.org (rolling window, default 60 days).
|
||||
# Refreshed weekly (Mon 12:00 UTC); cache reused for up to 7 days between runs.
|
||||
# NUFORC_RECENT_DAYS=60
|
||||
# NUFORC_CACHE_TTL_HOURS=168
|
||||
# On Windows, live scrape uses Python requests by default; optional:
|
||||
# SHADOWBROKER_ENABLE_WINDOWS_CURL_FALLBACK=true
|
||||
# NUFORC enrichment index (HF dataset) is separate — opt-in only:
|
||||
# NUFORC UAP sightings — huggingface.co dataset download.
|
||||
# NUFORC_ENABLED=false
|
||||
#
|
||||
# News RSS aggregator — defaults ON. Set to "false" to disable all
|
||||
@@ -107,12 +87,6 @@ AIS_API_KEY= # https://aisstream.io/ — free tier WebSocket key
|
||||
# Free MAP_KEY from https://firms.modaps.eosdis.nasa.gov/map/#d:24hrs;@0.0,0.0,3.0z
|
||||
# FIRMS_MAP_KEY=
|
||||
|
||||
# Ukraine frontline mirror (GitHub). Default follows cyterat/deepstate-map-data@main.
|
||||
# Pin an immutable commit SHA so ingest cannot silently change if main is force-pushed (#362).
|
||||
# Example (verify on GitHub before use): main @ b479954e94696bc5622c7818fd20a64a699f4fe8
|
||||
# DEEPSTATE_MIRROR_COMMIT=b479954e94696bc5622c7818fd20a64a699f4fe8
|
||||
# DEEPSTATE_MIRROR_REPO=cyterat/deepstate-map-data
|
||||
|
||||
# Ukraine air raid alerts from alerts.in.ua — free token from https://alerts.in.ua/
|
||||
# ALERTS_IN_UA_TOKEN=
|
||||
|
||||
@@ -142,16 +116,12 @@ AIS_API_KEY= # https://aisstream.io/ — free tier WebSocket key
|
||||
# can identify per-install traffic instead of aggregated "ShadowBroker" hits.
|
||||
# Leave blank to send a generic UA. If you set MESHTASTIC_OPERATOR_CALLSIGN,
|
||||
# it is included in outbound headers to meshtastic.org by default so they
|
||||
# can rate-limit per-operator. Callsign is NOT sent upstream unless you opt in.
|
||||
# can rate-limit per-operator. Set MESHTASTIC_SEND_CALLSIGN_HEADER=false to
|
||||
# suppress the callsign while still using it locally (e.g. for APRS).
|
||||
# MESHTASTIC_OPERATOR_CALLSIGN=
|
||||
# MESHTASTIC_SEND_CALLSIGN_HEADER=false
|
||||
# MESHTASTIC_SEND_CALLSIGN_HEADER=true
|
||||
# MESH_MQTT_PSK= # hex-encoded, empty = default LongFast key
|
||||
|
||||
# LiveUAMap Playwright scraper (#348). Linux/macOS: on by default when Global
|
||||
# Incidents layer is active. Windows: off until the operator enables Global
|
||||
# Incidents in the UI (consent dialog) or sets SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=true.
|
||||
# SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=false forces off on all platforms.
|
||||
|
||||
# ── Mesh / Reticulum (RNS) ─────────────────────────────────────
|
||||
# Full-node / participant-node posture for public Infonet sync.
|
||||
# MESH_NODE_MODE=participant # participant | relay | perimeter
|
||||
|
||||
+31
-32
@@ -113,14 +113,8 @@ def _scoped_admin_tokens() -> dict[str, list[str]]:
|
||||
return normalized
|
||||
|
||||
|
||||
def _request_scope_path(request: Request) -> str:
|
||||
"""Return the ASGI request-line path, not the Host-derived URL path."""
|
||||
scope = getattr(request, "scope", {}) or {}
|
||||
return str(scope.get("path") or "")
|
||||
|
||||
|
||||
def _required_scope_for_request(request: Request) -> str:
|
||||
path = _request_scope_path(request)
|
||||
path = str(request.url.path or "")
|
||||
if path.startswith("/api/wormhole/gate/"):
|
||||
return "gate"
|
||||
if path.startswith("/api/wormhole/dm/"):
|
||||
@@ -449,7 +443,7 @@ async def _verify_openclaw_hmac(request: Request) -> bool:
|
||||
|
||||
# Compute expected signature: HMAC-SHA256(secret, METHOD|path|ts|nonce|body_digest)
|
||||
method = str(request.method or "").upper()
|
||||
path = _request_scope_path(request)
|
||||
path = str(request.url.path or "")
|
||||
message = f"{method}|{path}|{ts_str}|{nonce}|{body_digest}"
|
||||
expected = hmac.new(
|
||||
secret.encode("utf-8"),
|
||||
@@ -521,32 +515,33 @@ _KNOWN_COMPROMISED_PEER_PUSH_SECRET_SHA256 = (
|
||||
def _validate_admin_startup() -> None:
|
||||
admin_key = _current_admin_key()
|
||||
|
||||
if not admin_key:
|
||||
logger.warning(
|
||||
"ADMIN_KEY is not set. Local-operator/admin endpoints will reject "
|
||||
"remote callers until ADMIN_KEY is configured."
|
||||
)
|
||||
return
|
||||
if not admin_key or len(admin_key) < 32:
|
||||
import secrets
|
||||
|
||||
if len(admin_key) < 32:
|
||||
reason = f"too short ({len(admin_key)} chars, minimum 32)"
|
||||
reason = "not set" if not admin_key else f"too short ({len(admin_key)} chars, minimum 32)"
|
||||
new_key = secrets.token_hex(32) # 64-char hex string
|
||||
try:
|
||||
debug_mode = bool(getattr(get_settings(), "MESH_DEBUG_MODE", False))
|
||||
except Exception:
|
||||
debug_mode = False
|
||||
if debug_mode:
|
||||
logger.warning(
|
||||
"ADMIN_KEY is %s. Debug mode is enabled, so startup will continue, "
|
||||
"but production deployments must use a 32+ character key.",
|
||||
from routers.ai_intel import _write_env_value
|
||||
|
||||
_write_env_value("ADMIN_KEY", new_key)
|
||||
os.environ["ADMIN_KEY"] = new_key
|
||||
logger.info(
|
||||
"ADMIN_KEY was %s — auto-generated a strong 64-character key and "
|
||||
"saved it to .env. Admin/mesh endpoints are now secured.",
|
||||
reason,
|
||||
)
|
||||
return
|
||||
logger.error(
|
||||
"ADMIN_KEY is %s. Refusing to start because auto-generating a backend-only "
|
||||
"replacement would desynchronize the frontend and backend containers.",
|
||||
reason,
|
||||
)
|
||||
raise SystemExit(1)
|
||||
# Clear settings cache so the rest of startup picks up the new key
|
||||
try:
|
||||
get_settings.cache_clear()
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"ADMIN_KEY is %s and could not auto-generate: %s. "
|
||||
"Admin/mesh endpoints may be unavailable.",
|
||||
reason,
|
||||
exc,
|
||||
)
|
||||
|
||||
|
||||
def _validate_insecure_admin_startup() -> None:
|
||||
@@ -749,7 +744,8 @@ def _is_debug_test_request(request: Request) -> bool:
|
||||
if not _debug_mode_enabled():
|
||||
return False
|
||||
client_host = (request.client.host or "").lower() if request.client else ""
|
||||
return client_host == "test"
|
||||
url_host = (request.url.hostname or "").lower() if request.url else ""
|
||||
return client_host == "test" or url_host == "test"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1401,7 +1397,10 @@ def _peer_hmac_url_from_request(request: Request) -> str:
|
||||
header_url = normalize_peer_url(str(request.headers.get("x-peer-url", "") or ""))
|
||||
if header_url:
|
||||
return header_url
|
||||
return ""
|
||||
if not request.url:
|
||||
return ""
|
||||
base_url = f"{request.url.scheme}://{request.url.netloc}".rstrip("/")
|
||||
return normalize_peer_url(base_url)
|
||||
|
||||
|
||||
def _verify_peer_push_hmac(request: Request, body_bytes: bytes) -> bool:
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
},
|
||||
{
|
||||
"name": "BBC",
|
||||
"url": "https://feeds.bbci.co.uk/news/world/rss.xml",
|
||||
"url": "http://feeds.bbci.co.uk/news/world/rss.xml",
|
||||
"weight": 3
|
||||
},
|
||||
{
|
||||
@@ -47,7 +47,7 @@
|
||||
},
|
||||
{
|
||||
"name": "Xinhua",
|
||||
"url": "https://www.news.cn/english/rss/worldrss.xml",
|
||||
"url": "http://www.news.cn/english/rss/worldrss.xml",
|
||||
"weight": 2
|
||||
},
|
||||
{
|
||||
|
||||
@@ -41,10 +41,5 @@
|
||||
"ShadowBroker_v0.9.8.zip": "183bb5cd62b9b9349d95df5ef7696cb6ca810ab4b991fa9dab6f898af4c7a175",
|
||||
"ShadowBroker_0.9.8_x64-setup.exe": "94a0309862e9c81c92cdcbfea8eec9dbb97eef19ded82b26217b397defbc810c",
|
||||
"ShadowBroker_0.9.8_x64_en-US.msi": "fe22f9d51e4360d74c18a7250c2fbb9ed4fa4c7a884b3ac0d04a21115466386b"
|
||||
},
|
||||
"v0.9.81": {
|
||||
"ShadowBroker_v0.9.81.zip": "f81f454bdc88e9a32c351df38212b8cfa624704d65764b971bb091eef62259c6",
|
||||
"ShadowBroker_0.9.81_x64-setup.exe": "25e9a95d0d8ce959a7d08fe8e7406772ae24b596652793e81d1de5d02510a5a6",
|
||||
"ShadowBroker_0.9.81_x64_en-US.msi": "34e655fc0c0f195ee4ac978f228a4b2b9d5565253b8771aca9ef4693409e9e70"
|
||||
}
|
||||
}
|
||||
|
||||
+106
-451
@@ -1,4 +1,4 @@
|
||||
import os
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
|
||||
@@ -8,24 +8,18 @@ import asyncio
|
||||
import base64
|
||||
import hmac
|
||||
import importlib
|
||||
import ipaddress
|
||||
import secrets
|
||||
import hashlib as _hashlib_mod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from json import JSONDecodeError
|
||||
|
||||
APP_VERSION = "0.9.81"
|
||||
APP_VERSION = "0.9.8"
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
_start_time = time.time()
|
||||
_MESH_ONLY = os.environ.get("MESH_ONLY", "").strip().lower() in ("1", "true", "yes")
|
||||
_HEADLESS_MESH_NODE_RUNTIME = os.environ.get("SHADOWBROKER_MESH_NODE_RUNTIME", "").strip().lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
_WARNED_LEGACY_DM_PUBKEY_LOOKUPS: set[str] = set()
|
||||
|
||||
|
||||
@@ -310,7 +304,6 @@ from auth import (
|
||||
_private_plane_access_denied_payload,
|
||||
_private_infonet_policy_snapshot,
|
||||
_private_plane_refusal_response,
|
||||
_request_scope_path,
|
||||
_scoped_admin_tokens,
|
||||
_scoped_view_authenticated as _scoped_view_authenticated_auth,
|
||||
_security_headers,
|
||||
@@ -1102,7 +1095,6 @@ _WORMHOLE_PUBLIC_PROFILE_FIELDS = {"wormhole_enabled"}
|
||||
_PRIVATE_LANE_CONTROL_FIELDS = {"private_lane_tier", "private_lane_policy"}
|
||||
_PUBLIC_RNS_STATUS_FIELDS = {"enabled", "ready", "configured_peers", "active_peers"}
|
||||
_NODE_PUBLIC_EVENT_HOOK_REGISTERED = False
|
||||
_NODE_RUNTIME_THREADS_STARTED = False
|
||||
_INFONET_PRIVATE_TRANSPORT_LOCK = threading.Lock()
|
||||
|
||||
|
||||
@@ -1192,49 +1184,6 @@ def _filter_infonet_sync_records(records: list[Any]) -> list[Any]:
|
||||
]
|
||||
|
||||
|
||||
def _infonet_peer_url_allowed(peer_url: str) -> bool:
|
||||
if not _infonet_private_transport_required():
|
||||
return True
|
||||
return _is_private_infonet_transport(peer_transport_kind(peer_url))
|
||||
|
||||
|
||||
def _filter_infonet_peer_urls(peer_urls: list[str]) -> list[str]:
|
||||
if not _infonet_private_transport_required():
|
||||
return peer_urls
|
||||
return [peer_url for peer_url in peer_urls if _infonet_peer_url_allowed(peer_url)]
|
||||
|
||||
|
||||
def _infonet_peer_requests_proxies(normalized_peer_url: str) -> dict[str, str] | None:
|
||||
"""Return requests proxy settings for a sync/push peer, enforcing private policy."""
|
||||
transport = peer_transport_kind(normalized_peer_url)
|
||||
if _infonet_private_transport_required() and not _is_private_infonet_transport(transport):
|
||||
raise RuntimeError(_infonet_private_transport_error())
|
||||
if transport != "onion":
|
||||
return None
|
||||
if not bool(get_settings().MESH_ARTI_ENABLED):
|
||||
raise RuntimeError("onion peer requests require Arti to be enabled")
|
||||
from services.wormhole_supervisor import _check_arti_ready
|
||||
|
||||
if not _check_arti_ready():
|
||||
raise RuntimeError("onion peer requests require a ready Arti transport")
|
||||
socks_port = int(get_settings().MESH_ARTI_SOCKS_PORT or 9050)
|
||||
proxy = f"socks5h://127.0.0.1:{socks_port}"
|
||||
return {"http": proxy, "https": proxy}
|
||||
|
||||
|
||||
def _local_infonet_peer_url() -> str:
|
||||
"""Return this node's advertised peer URL for HMAC peer authentication."""
|
||||
configured = normalize_peer_url(str(getattr(get_settings(), "MESH_PUBLIC_PEER_URL", "") or ""))
|
||||
if configured:
|
||||
return configured
|
||||
try:
|
||||
from services.tor_hidden_service import tor_service
|
||||
|
||||
return normalize_peer_url(str(tor_service.onion_address or ""))
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _ensure_infonet_private_transport_ready(reason: str = "") -> bool:
|
||||
"""Warm the local onion transport before private Infonet sync.
|
||||
|
||||
@@ -1308,13 +1257,6 @@ def _refresh_node_peer_store(*, now: float | None = None) -> dict[str, Any]:
|
||||
operator_peers = configured_relay_peer_urls()
|
||||
bootstrap_seed_peers = _configured_bootstrap_seed_peer_urls()
|
||||
skipped_clearnet_peers = 0
|
||||
pruned_clearnet_peers = 0
|
||||
if private_transport_required:
|
||||
for key, record in list(store._records.items()):
|
||||
if _is_private_infonet_transport(str(getattr(record, "transport", "") or "")):
|
||||
continue
|
||||
del store._records[key]
|
||||
pruned_clearnet_peers += 1
|
||||
for peer_url in operator_peers:
|
||||
transport = peer_transport_kind(peer_url)
|
||||
if not transport:
|
||||
@@ -1422,7 +1364,6 @@ def _refresh_node_peer_store(*, now: float | None = None) -> dict[str, Any]:
|
||||
"node_mode": mode,
|
||||
"private_transport_required": private_transport_required,
|
||||
"skipped_clearnet_peer_count": skipped_clearnet_peers,
|
||||
"pruned_clearnet_peer_count": pruned_clearnet_peers,
|
||||
"manifest_loaded": manifest is not None,
|
||||
"manifest_signer_id": manifest.signer_id if manifest is not None else "",
|
||||
"manifest_valid_until": int(manifest.valid_until or 0) if manifest is not None else 0,
|
||||
@@ -1443,28 +1384,6 @@ def _materialize_local_infonet_state() -> None:
|
||||
from services.mesh.mesh_hashchain import infonet
|
||||
|
||||
infonet.ensure_materialized()
|
||||
try:
|
||||
_hydrate_gate_store_from_chain(list(infonet.events))
|
||||
_hydrate_dm_relay_from_chain(list(infonet.events))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class PeerSyncHTTPError(RuntimeError):
|
||||
def __init__(self, status_code: int, detail: str, *, retry_after_s: int = 0):
|
||||
self.status_code = int(status_code or 0)
|
||||
self.retry_after_s = int(retry_after_s or 0)
|
||||
message = str(detail or f"HTTP {self.status_code}").strip()
|
||||
if not message.upper().startswith("HTTP"):
|
||||
message = f"HTTP {self.status_code}: {message}"
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
def _parse_retry_after_seconds(value: str) -> int:
|
||||
try:
|
||||
return max(0, int(float(str(value or "").strip())))
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def _peer_sync_response(peer_url: str, body: dict[str, Any]) -> dict[str, Any]:
|
||||
@@ -1527,8 +1446,7 @@ def _peer_sync_response(peer_url: str, body: dict[str, Any]) -> dict[str, Any]:
|
||||
raise ValueError(f"peer sync returned non-JSON response ({response.status_code})") from exc
|
||||
if response.status_code != 200:
|
||||
detail = str(payload.get("detail", "") or f"HTTP {response.status_code}").strip()
|
||||
retry_after_s = _parse_retry_after_seconds(response.headers.get("Retry-After", ""))
|
||||
raise PeerSyncHTTPError(response.status_code, detail, retry_after_s=retry_after_s)
|
||||
raise ValueError(detail or f"HTTP {response.status_code}")
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("peer sync returned malformed payload")
|
||||
return payload
|
||||
@@ -1567,46 +1485,6 @@ def _hydrate_gate_store_from_chain(events: list[dict]) -> int:
|
||||
return count
|
||||
|
||||
|
||||
def _hydrate_dm_relay_from_chain(events: list[dict]) -> int:
|
||||
"""Copy accepted dm_message chain events into the local encrypted DM relay."""
|
||||
import hashlib
|
||||
|
||||
from services.mesh.mesh_dm_relay import dm_relay
|
||||
from services.mesh.mesh_hashchain import infonet
|
||||
|
||||
count = 0
|
||||
for evt in events:
|
||||
if evt.get("event_type") != "dm_message":
|
||||
continue
|
||||
event_id = str(evt.get("event_id", "") or "").strip()
|
||||
if not event_id or event_id not in infonet.event_index:
|
||||
continue
|
||||
canonical = infonet.events[infonet.event_index[event_id]]
|
||||
payload = canonical.get("payload") if isinstance(canonical.get("payload"), dict) else {}
|
||||
sender_token_hash = hashlib.sha256(
|
||||
f"hashchain-dm-sender|{event_id}|{canonical.get('node_id', '')}".encode("utf-8")
|
||||
).hexdigest()
|
||||
try:
|
||||
result = dm_relay.deposit(
|
||||
sender_id=str(canonical.get("node_id", "") or ""),
|
||||
raw_sender_id=str(canonical.get("node_id", "") or ""),
|
||||
recipient_id=str(payload.get("recipient_id", "") or ""),
|
||||
ciphertext=str(payload.get("ciphertext", "") or ""),
|
||||
msg_id=str(payload.get("msg_id", "") or ""),
|
||||
delivery_class=str(payload.get("delivery_class", "") or ""),
|
||||
recipient_token=str(payload.get("recipient_token", "") or "") or None,
|
||||
sender_seal=str(payload.get("sender_seal", "") or ""),
|
||||
sender_token_hash=sender_token_hash,
|
||||
payload_format=str(payload.get("format", "dm1") or "dm1"),
|
||||
session_welcome=str(payload.get("session_welcome", "") or ""),
|
||||
)
|
||||
if result.get("ok"):
|
||||
count += 1
|
||||
except Exception:
|
||||
pass
|
||||
return count
|
||||
|
||||
|
||||
def _sync_from_peer(
|
||||
peer_url: str,
|
||||
*,
|
||||
@@ -1660,7 +1538,6 @@ def _sync_from_peer(
|
||||
return True, "", False, 0
|
||||
result = infonet.ingest_events(events)
|
||||
_hydrate_gate_store_from_chain(events)
|
||||
_hydrate_dm_relay_from_chain(events)
|
||||
rejected = list(result.get("rejected", []) or [])
|
||||
if rejected:
|
||||
return False, f"sync ingest rejected {len(rejected)} event(s)", False, 0
|
||||
@@ -1723,8 +1600,6 @@ def _run_public_sync_cycle() -> SyncWorkerState:
|
||||
|
||||
last_error = "sync failed"
|
||||
for record in peers:
|
||||
retry_after_s = 0
|
||||
http_status_code = 0
|
||||
started = begin_sync(
|
||||
current_state,
|
||||
peer_url=record.peer_url,
|
||||
@@ -1735,17 +1610,6 @@ def _run_public_sync_cycle() -> SyncWorkerState:
|
||||
set_sync_state(started)
|
||||
try:
|
||||
ok, error, forked, retry_after_s = _sync_from_peer(record.peer_url)
|
||||
except PeerSyncHTTPError as exc:
|
||||
# _sync_from_peer catches PeerSyncRateLimited internally (4-tuple
|
||||
# path for 429 with Retry-After). Other non-200 statuses surface
|
||||
# here as PeerSyncHTTPError — pull retry_after_s + status off it
|
||||
# so the cooldown calculation below can honor server hints even
|
||||
# for non-429 throttling responses.
|
||||
ok = False
|
||||
error = str(exc)
|
||||
forked = False
|
||||
retry_after_s = int(exc.retry_after_s or 0)
|
||||
http_status_code = int(exc.status_code or 0)
|
||||
except Exception as exc:
|
||||
ok = False
|
||||
error = str(exc or type(exc).__name__)
|
||||
@@ -1776,10 +1640,6 @@ def _run_public_sync_cycle() -> SyncWorkerState:
|
||||
getattr(settings, "MESH_BOOTSTRAP_SEED_FAILURE_COOLDOWN_S", cooldown_s)
|
||||
or cooldown_s
|
||||
)
|
||||
if http_status_code == 429:
|
||||
failure_count = max(int(getattr(record, "failure_count", 0) or 0), current_state.consecutive_failures)
|
||||
exponential_429_s = min(900, 60 * (2 ** min(failure_count, 4)))
|
||||
cooldown_s = max(cooldown_s, retry_after_s, exponential_429_s)
|
||||
store.mark_failure(
|
||||
record.peer_url,
|
||||
"sync",
|
||||
@@ -1790,7 +1650,7 @@ def _run_public_sync_cycle() -> SyncWorkerState:
|
||||
store.save()
|
||||
failure_backoff_s = int(settings.MESH_SYNC_FAILURE_BACKOFF_S or 60)
|
||||
if is_seed_peer:
|
||||
failure_backoff_s = max(failure_backoff_s, max(1, cooldown_s))
|
||||
failure_backoff_s = min(failure_backoff_s, max(1, cooldown_s))
|
||||
updated = finish_sync(
|
||||
started,
|
||||
ok=False,
|
||||
@@ -1890,7 +1750,7 @@ def _propagate_public_event_to_peers(event_dict: dict[str, Any]) -> None:
|
||||
|
||||
if not _participant_node_enabled():
|
||||
return
|
||||
if not _filter_infonet_peer_urls(authenticated_push_peer_urls()):
|
||||
if not authenticated_push_peer_urls():
|
||||
return
|
||||
|
||||
envelope = MeshEnvelope(
|
||||
@@ -1924,45 +1784,6 @@ def _schedule_public_event_propagation(event_dict: dict[str, Any]) -> None:
|
||||
).start()
|
||||
|
||||
|
||||
def _infonet_node_runtime_requested() -> bool:
|
||||
return (not _MESH_ONLY) or _HEADLESS_MESH_NODE_RUNTIME
|
||||
|
||||
|
||||
def _start_infonet_node_runtime(reason: str = "startup") -> None:
|
||||
"""Start sync/push/pull workers for participant nodes."""
|
||||
global _NODE_PUBLIC_EVENT_HOOK_REGISTERED, _NODE_RUNTIME_THREADS_STARTED
|
||||
|
||||
if not _infonet_node_runtime_requested():
|
||||
return
|
||||
try:
|
||||
from services.mesh.mesh_hashchain import register_public_event_append_hook
|
||||
|
||||
_materialize_local_infonet_state()
|
||||
_refresh_node_peer_store()
|
||||
if _node_runtime_supported():
|
||||
if not _participant_node_enabled():
|
||||
logger.info("Infonet participant auto-enabled for private seed sync")
|
||||
_set_participant_node_enabled(True)
|
||||
threading.Thread(
|
||||
target=lambda: _ensure_infonet_private_transport_ready(reason),
|
||||
daemon=True,
|
||||
name="infonet-private-transport-warmup",
|
||||
).start()
|
||||
_NODE_SYNC_STOP.clear()
|
||||
if not _NODE_RUNTIME_THREADS_STARTED:
|
||||
threading.Thread(target=_public_infonet_sync_loop, daemon=True).start()
|
||||
threading.Thread(target=_http_peer_push_loop, daemon=True).start()
|
||||
threading.Thread(target=_http_gate_push_loop, daemon=True).start()
|
||||
threading.Thread(target=_http_gate_pull_loop, daemon=True).start()
|
||||
_NODE_RUNTIME_THREADS_STARTED = True
|
||||
_kick_public_sync_background(reason)
|
||||
if not _NODE_PUBLIC_EVENT_HOOK_REGISTERED:
|
||||
register_public_event_append_hook(_schedule_public_event_propagation)
|
||||
_NODE_PUBLIC_EVENT_HOOK_REGISTERED = True
|
||||
except Exception as e:
|
||||
logger.warning(f"Node bootstrap runtime failed to initialize: {e}")
|
||||
|
||||
|
||||
# ─── Background HTTP Peer Push Worker ────────────────────────────────────
|
||||
# Runs alongside the sync loop. Every PUSH_INTERVAL seconds, batches new
|
||||
# Infonet events and sends them via HMAC-authenticated POST to push peers.
|
||||
@@ -1970,7 +1791,6 @@ def _start_infonet_node_runtime(reason: str = "startup") -> None:
|
||||
_PEER_PUSH_INTERVAL_S = 10
|
||||
_PEER_PUSH_BATCH_SIZE = 50
|
||||
_peer_push_last_index: dict[str, int] = {} # peer_url → last pushed event index
|
||||
_INFONET_SYNC_RATE_LIMIT = "600/minute"
|
||||
|
||||
|
||||
def _http_peer_push_loop() -> None:
|
||||
@@ -1992,7 +1812,7 @@ def _http_peer_push_loop() -> None:
|
||||
# loop on the global secret being set — an install that only
|
||||
# configures per-peer secrets is now valid.
|
||||
|
||||
peers = _filter_infonet_peer_urls(authenticated_push_peer_urls())
|
||||
peers = authenticated_push_peer_urls()
|
||||
if not peers:
|
||||
_NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S)
|
||||
continue
|
||||
@@ -2020,8 +1840,7 @@ def _http_peer_push_loop() -> None:
|
||||
ensure_ascii=False,
|
||||
).encode("utf-8")
|
||||
|
||||
sender_url = _local_infonet_peer_url()
|
||||
peer_key = resolve_peer_key_for_url(sender_url)
|
||||
peer_key = resolve_peer_key_for_url(normalized)
|
||||
if not peer_key:
|
||||
continue
|
||||
import hmac as _hmac_mod2
|
||||
@@ -2029,21 +1848,14 @@ def _http_peer_push_loop() -> None:
|
||||
hmac_hex = _hmac_mod2.new(peer_key, body_bytes, _hashlib_mod2.sha256).hexdigest()
|
||||
|
||||
timeout = int(get_settings().MESH_RELAY_PUSH_TIMEOUT_S or 10)
|
||||
proxies = _infonet_peer_requests_proxies(normalized)
|
||||
request_kwargs: dict[str, Any] = {
|
||||
"data": body_bytes,
|
||||
"headers": {
|
||||
"Content-Type": "application/json",
|
||||
"X-Peer-Url": sender_url,
|
||||
"X-Peer-HMAC": hmac_hex,
|
||||
},
|
||||
"timeout": timeout,
|
||||
}
|
||||
if proxies:
|
||||
request_kwargs["proxies"] = proxies
|
||||
resp = _requests.post(
|
||||
f"{normalized}/api/mesh/infonet/peer-push",
|
||||
**request_kwargs,
|
||||
data=body_bytes,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"X-Peer-HMAC": hmac_hex,
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
_peer_push_last_index[normalized] = last_idx + len(batch)
|
||||
@@ -2083,7 +1895,7 @@ def _http_gate_pull_loop() -> None:
|
||||
|
||||
# Issue #256: per-peer key resolution; see _http_peer_push_loop.
|
||||
|
||||
peers = _filter_infonet_peer_urls(authenticated_push_peer_urls())
|
||||
peers = authenticated_push_peer_urls()
|
||||
if not peers:
|
||||
_NODE_SYNC_STOP.wait(_GATE_PULL_INTERVAL_S)
|
||||
continue
|
||||
@@ -2093,8 +1905,7 @@ def _http_gate_pull_loop() -> None:
|
||||
if not normalized:
|
||||
continue
|
||||
|
||||
sender_url = _local_infonet_peer_url()
|
||||
peer_key = resolve_peer_key_for_url(sender_url)
|
||||
peer_key = resolve_peer_key_for_url(normalized)
|
||||
if not peer_key:
|
||||
continue
|
||||
|
||||
@@ -2114,21 +1925,14 @@ def _http_gate_pull_loop() -> None:
|
||||
discovery_hmac = _hmac_pull.new(peer_key, discovery_body, _hashlib_pull.sha256).hexdigest()
|
||||
|
||||
timeout = int(get_settings().MESH_RELAY_PUSH_TIMEOUT_S or 10)
|
||||
proxies = _infonet_peer_requests_proxies(normalized)
|
||||
discovery_kwargs: dict[str, Any] = {
|
||||
"data": discovery_body,
|
||||
"headers": {
|
||||
"Content-Type": "application/json",
|
||||
"X-Peer-Url": sender_url,
|
||||
"X-Peer-HMAC": discovery_hmac,
|
||||
},
|
||||
"timeout": timeout,
|
||||
}
|
||||
if proxies:
|
||||
discovery_kwargs["proxies"] = proxies
|
||||
resp = _requests.post(
|
||||
f"{normalized}/api/mesh/gate/peer-pull",
|
||||
**discovery_kwargs,
|
||||
data=discovery_body,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"X-Peer-HMAC": discovery_hmac,
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
continue
|
||||
@@ -2158,20 +1962,14 @@ def _http_gate_pull_loop() -> None:
|
||||
|
||||
pull_hmac = _hmac_pull.new(peer_key, pull_body, _hashlib_pull.sha256).hexdigest()
|
||||
|
||||
pull_kwargs: dict[str, Any] = {
|
||||
"data": pull_body,
|
||||
"headers": {
|
||||
"Content-Type": "application/json",
|
||||
"X-Peer-Url": sender_url,
|
||||
"X-Peer-HMAC": pull_hmac,
|
||||
},
|
||||
"timeout": timeout,
|
||||
}
|
||||
if proxies:
|
||||
pull_kwargs["proxies"] = proxies
|
||||
pull_resp = _requests.post(
|
||||
f"{normalized}/api/mesh/gate/peer-pull",
|
||||
**pull_kwargs,
|
||||
data=pull_body,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"X-Peer-HMAC": pull_hmac,
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
if pull_resp.status_code != 200:
|
||||
continue
|
||||
@@ -2222,7 +2020,7 @@ def _http_gate_push_loop() -> None:
|
||||
|
||||
# Issue #256: per-peer key resolution; see _http_peer_push_loop.
|
||||
|
||||
peers = _filter_infonet_peer_urls(authenticated_push_peer_urls())
|
||||
peers = authenticated_push_peer_urls()
|
||||
if not peers:
|
||||
_NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S)
|
||||
continue
|
||||
@@ -2235,8 +2033,7 @@ def _http_gate_push_loop() -> None:
|
||||
if not normalized:
|
||||
continue
|
||||
|
||||
sender_url = _local_infonet_peer_url()
|
||||
peer_key = resolve_peer_key_for_url(sender_url)
|
||||
peer_key = resolve_peer_key_for_url(normalized)
|
||||
if not peer_key:
|
||||
continue
|
||||
|
||||
@@ -2267,21 +2064,14 @@ def _http_gate_push_loop() -> None:
|
||||
hmac_hex = _hmac_mod3.new(peer_key, body_bytes, _hashlib_mod3.sha256).hexdigest()
|
||||
|
||||
timeout = int(get_settings().MESH_RELAY_PUSH_TIMEOUT_S or 10)
|
||||
proxies = _infonet_peer_requests_proxies(normalized)
|
||||
request_kwargs: dict[str, Any] = {
|
||||
"data": body_bytes,
|
||||
"headers": {
|
||||
"Content-Type": "application/json",
|
||||
"X-Peer-Url": sender_url,
|
||||
"X-Peer-HMAC": hmac_hex,
|
||||
},
|
||||
"timeout": timeout,
|
||||
}
|
||||
if proxies:
|
||||
request_kwargs["proxies"] = proxies
|
||||
resp = _requests.post(
|
||||
f"{normalized}/api/mesh/gate/peer-push",
|
||||
**request_kwargs,
|
||||
data=body_bytes,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"X-Peer-HMAC": hmac_hex,
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
peer_counts[gate_id] = last + len(batch)
|
||||
@@ -2623,8 +2413,32 @@ async def lifespan(app: FastAPI):
|
||||
daemon=True,
|
||||
name="wormhole-startup-sync",
|
||||
).start()
|
||||
try:
|
||||
from services.mesh.mesh_hashchain import register_public_event_append_hook
|
||||
|
||||
_start_infonet_node_runtime("startup")
|
||||
_materialize_local_infonet_state()
|
||||
_refresh_node_peer_store()
|
||||
if _node_runtime_supported():
|
||||
if not _participant_node_enabled():
|
||||
logger.info("Infonet participant auto-enabled for private seed sync")
|
||||
_set_participant_node_enabled(True)
|
||||
threading.Thread(
|
||||
target=lambda: _ensure_infonet_private_transport_ready("startup"),
|
||||
daemon=True,
|
||||
name="infonet-private-transport-warmup",
|
||||
).start()
|
||||
_NODE_SYNC_STOP.clear()
|
||||
threading.Thread(target=_public_infonet_sync_loop, daemon=True).start()
|
||||
_kick_public_sync_background("startup")
|
||||
threading.Thread(target=_http_peer_push_loop, daemon=True).start()
|
||||
threading.Thread(target=_http_gate_push_loop, daemon=True).start()
|
||||
threading.Thread(target=_http_gate_pull_loop, daemon=True).start()
|
||||
global _NODE_PUBLIC_EVENT_HOOK_REGISTERED
|
||||
if not _NODE_PUBLIC_EVENT_HOOK_REGISTERED:
|
||||
register_public_event_append_hook(_schedule_public_event_propagation)
|
||||
_NODE_PUBLIC_EVENT_HOOK_REGISTERED = True
|
||||
except Exception as e:
|
||||
logger.warning(f"Node bootstrap runtime failed to initialize: {e}")
|
||||
|
||||
if not _MESH_ONLY:
|
||||
# Prime the static route/airport database from vrs-standing-data.adsb.lol
|
||||
@@ -2729,7 +2543,7 @@ async def json_decode_error_handler(_request: Request, _exc: JSONDecodeError):
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def private_plane_http_exception_handler(request: Request, exc: StarletteHTTPException):
|
||||
if exc.status_code == 403 and _is_private_plane_access_path(_request_scope_path(request), request.method):
|
||||
if exc.status_code == 403 and _is_private_plane_access_path(request.url.path, request.method):
|
||||
return await _private_plane_refusal_response(
|
||||
request,
|
||||
status_code=403,
|
||||
@@ -2763,7 +2577,7 @@ async def mesh_security_headers(request: Request, call_next):
|
||||
@app.middleware("http")
|
||||
async def mesh_no_store_headers(request: Request, call_next):
|
||||
response = await call_next(request)
|
||||
if _request_scope_path(request).startswith("/api/mesh/"):
|
||||
if request.url.path.startswith("/api/mesh/"):
|
||||
response.headers["Cache-Control"] = "no-store, max-age=0"
|
||||
response.headers["Pragma"] = "no-cache"
|
||||
return response
|
||||
@@ -2863,83 +2677,6 @@ def _redact_public_event(event: dict) -> dict:
|
||||
return _redact_vote_gate(_redact_key_rotate_payload(_redact_gate_metadata(event)))
|
||||
|
||||
|
||||
def _is_loopback_host(host: str) -> bool:
|
||||
value = str(host or "").strip().lower()
|
||||
if not value:
|
||||
return False
|
||||
if value.startswith("[") and "]" in value:
|
||||
value = value[1 : value.index("]")]
|
||||
if ":" in value and value.count(":") == 1:
|
||||
value = value.rsplit(":", 1)[0]
|
||||
if value in {"localhost", "ip6-localhost"}:
|
||||
return True
|
||||
try:
|
||||
return ipaddress.ip_address(value).is_loopback
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _is_onion_host(host: str) -> bool:
|
||||
value = str(host or "").strip().lower()
|
||||
if not value:
|
||||
return False
|
||||
if ":" in value and value.count(":") == 1:
|
||||
value = value.rsplit(":", 1)[0]
|
||||
return value.endswith(".onion")
|
||||
|
||||
|
||||
def _forwarded_for_hosts(request) -> list[str]:
|
||||
headers = getattr(request, "headers", {}) or {}
|
||||
hosts: list[str] = []
|
||||
x_forwarded_for = str(headers.get("x-forwarded-for", "") or "")
|
||||
hosts.extend(part.strip() for part in x_forwarded_for.split(",") if part.strip())
|
||||
forwarded = str(headers.get("forwarded", "") or "")
|
||||
for section in forwarded.split(","):
|
||||
for item in section.split(";"):
|
||||
key, sep, value = item.strip().partition("=")
|
||||
if sep and key.strip().lower() == "for":
|
||||
hosts.append(value.strip().strip('"').strip("[]"))
|
||||
return hosts
|
||||
|
||||
|
||||
def _request_appears_private_infonet_transport(request) -> bool:
|
||||
"""Return whether a sync request is safe to carry private ledger events.
|
||||
|
||||
This is intentionally fail-closed for the private event surface only. A
|
||||
questionable request still gets public events; gate/DM ciphertext simply
|
||||
stays out of the response.
|
||||
"""
|
||||
if not _infonet_private_transport_required() or request is None:
|
||||
return False
|
||||
|
||||
client = getattr(request, "client", None)
|
||||
client_host = str(getattr(client, "host", "") or "")
|
||||
if not (_is_loopback_host(client_host) or _is_onion_host(client_host)):
|
||||
return False
|
||||
|
||||
forwarded_hosts = _forwarded_for_hosts(request)
|
||||
if forwarded_hosts and any(not (_is_loopback_host(host) or _is_onion_host(host)) for host in forwarded_hosts):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _infonet_sync_response_events(events: list[dict], request=None) -> list[dict]:
|
||||
"""Build the sync event surface for the current transport policy."""
|
||||
include_private = _request_appears_private_infonet_transport(request)
|
||||
response: list[dict] = []
|
||||
for event in events:
|
||||
if not isinstance(event, dict):
|
||||
continue
|
||||
event_type = str(event.get("event_type", "") or "")
|
||||
if event_type in {"gate_message", "dm_message"}:
|
||||
if include_private:
|
||||
response.append(dict(event))
|
||||
continue
|
||||
response.append(_redact_public_event(event))
|
||||
return response
|
||||
|
||||
|
||||
def _trusted_gate_reply_to(event: dict) -> str:
|
||||
if not isinstance(event, dict):
|
||||
return ""
|
||||
@@ -3457,7 +3194,7 @@ def _refresh_lookup_handle_rotation_background(*, reason: str) -> dict[str, Any]
|
||||
|
||||
@app.middleware("http")
|
||||
async def enforce_high_privacy_mesh(request: Request, call_next):
|
||||
path = _request_scope_path(request)
|
||||
path = request.url.path
|
||||
private_mesh_path = path.startswith("/api/mesh") and not _is_public_meshtastic_lane_path(
|
||||
path,
|
||||
request.method,
|
||||
@@ -3617,7 +3354,7 @@ async def enforce_high_privacy_mesh(request: Request, call_next):
|
||||
@app.middleware("http")
|
||||
async def apply_no_store_to_sensitive_paths(request: Request, call_next):
|
||||
response = await call_next(request)
|
||||
if _is_sensitive_no_store_path(_request_scope_path(request)):
|
||||
if _is_sensitive_no_store_path(request.url.path):
|
||||
for key, value in _NO_STORE_HEADERS.items():
|
||||
response.headers[key] = value
|
||||
return response
|
||||
@@ -3851,17 +3588,7 @@ async def update_layers(update: LayerUpdate, request: Request):
|
||||
@app.get("/api/live-data")
|
||||
@limiter.limit("120/minute")
|
||||
async def live_data(request: Request):
|
||||
etag = _current_etag(prefix="live|full|")
|
||||
if request.headers.get("if-none-match") == etag:
|
||||
return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
|
||||
from services.fetchers._store import get_latest_data_deepcopy_snapshot
|
||||
|
||||
payload = get_latest_data_deepcopy_snapshot()
|
||||
return Response(
|
||||
content=orjson.dumps(_sanitize_payload(payload)),
|
||||
media_type="application/json",
|
||||
headers={"ETag": etag, "Cache-Control": "no-cache"},
|
||||
)
|
||||
return get_latest_data()
|
||||
|
||||
|
||||
def _etag_response(request: Request, payload: dict, prefix: str = "", default=None):
|
||||
@@ -5534,15 +5261,32 @@ def _submit_gate_message_envelope(request: Request, gate_id: str, body: dict[str
|
||||
if not cooldown_ok:
|
||||
return {"ok": False, "detail": cooldown_reason}
|
||||
|
||||
# Advance sequence counter (replay protection) without appending to
|
||||
# the public infonet chain — gate messages are private.
|
||||
try:
|
||||
from services.mesh.mesh_hashchain import infonet, gate_store
|
||||
|
||||
seq_ok, seq_reason = _validate_private_signed_sequence(
|
||||
infonet,
|
||||
sender_id,
|
||||
sequence,
|
||||
domain="gate_message",
|
||||
)
|
||||
if not seq_ok:
|
||||
return {"ok": False, "detail": seq_reason}
|
||||
except ValueError as exc:
|
||||
return {"ok": False, "detail": str(exc)}
|
||||
except Exception:
|
||||
logger.exception("Failed to advance sequence for gate message")
|
||||
return {"ok": False, "detail": "Failed to record gate message"}
|
||||
|
||||
gate_manager.record_message(gate_id)
|
||||
_record_gate_post_cooldown(sender_id, gate_id)
|
||||
logger.info("Encrypted gate message accepted on obfuscated gate plane")
|
||||
|
||||
# Build and commit the encrypted gate event to the private Infonet ledger.
|
||||
# The main hashchain is the durable propagation surface; gate_store is the
|
||||
# local materialized view used by the existing decrypt/UI path.
|
||||
# Build gate event and store in gate_store (private — not on public chain).
|
||||
try:
|
||||
from services.mesh.mesh_hashchain import infonet
|
||||
from services.mesh.mesh_hashchain import _private_gate_event_id
|
||||
import time as _time
|
||||
|
||||
store_payload = dict(gate_payload)
|
||||
@@ -5564,24 +5308,19 @@ def _submit_gate_message_envelope(request: Request, gate_id: str, body: dict[str
|
||||
"public_key_algo": public_key_algo,
|
||||
"protocol_version": protocol_version or PROTOCOL_VERSION,
|
||||
}
|
||||
gate_event = infonet.append_private_gate_message(
|
||||
node_id=sender_id,
|
||||
payload=store_payload,
|
||||
signature=signature,
|
||||
sequence=sequence,
|
||||
public_key=public_key,
|
||||
public_key_algo=public_key_algo,
|
||||
protocol_version=protocol_version or PROTOCOL_VERSION,
|
||||
timestamp=float(gate_event.get("timestamp", 0) or 0),
|
||||
)
|
||||
except ValueError as exc:
|
||||
return {"ok": False, "detail": str(exc)}
|
||||
gate_event["event_id"] = _private_gate_event_id(gate_id, sender_id, sequence, gate_event)
|
||||
except Exception:
|
||||
logger.exception("Failed to append gate message to private Infonet ledger")
|
||||
logger.exception("Failed to prepare private gate message for queued release")
|
||||
return {"ok": False, "detail": "Failed to record gate message"}
|
||||
|
||||
# Append to the local gate_store immediately so the author sees the same
|
||||
# materialized gate view that peers will hydrate after private sync.
|
||||
# Append to the local gate_store immediately. The gate_store is a
|
||||
# per-node persistent ciphertext chain; writing to it is a local
|
||||
# operation with no network dependency. Previously this happened only
|
||||
# inside the release worker's attempt_private_release path, which
|
||||
# meant messages sat in the outbox — invisible to the author and the
|
||||
# gate UI — until the transport tier reached the release floor.
|
||||
# Decoupling local visibility from network fan-out: append locally now,
|
||||
# queue the release for network propagation when the lane is ready.
|
||||
try:
|
||||
from services.mesh.mesh_hashchain import gate_store
|
||||
|
||||
@@ -5708,7 +5447,7 @@ async def infonet_locator(request: Request, limit: int = Query(32, ge=4, le=128)
|
||||
|
||||
|
||||
@app.post("/api/mesh/infonet/sync")
|
||||
@limiter.limit(_INFONET_SYNC_RATE_LIMIT)
|
||||
@limiter.limit("30/minute")
|
||||
@mesh_write_exempt(MeshWriteExemption.PEER_GOSSIP)
|
||||
async def infonet_sync_post(
|
||||
request: Request,
|
||||
@@ -5761,7 +5500,8 @@ async def infonet_sync_post(
|
||||
elif matched_hash == GENESIS_HASH and len(locator) > 1:
|
||||
forked = True
|
||||
|
||||
events = _infonet_sync_response_events(events, request=request)
|
||||
# Filter out legacy gate_message events — not part of the public sync surface.
|
||||
events = [_redact_public_event(e) for e in events if e.get("event_type") != "gate_message"]
|
||||
|
||||
response = {
|
||||
"events": events,
|
||||
@@ -5824,7 +5564,7 @@ async def mesh_rns_status(request: Request):
|
||||
|
||||
|
||||
@app.get("/api/mesh/infonet/sync")
|
||||
@limiter.limit(_INFONET_SYNC_RATE_LIMIT)
|
||||
@limiter.limit("30/minute")
|
||||
async def infonet_sync(
|
||||
request: Request,
|
||||
after_hash: str = "",
|
||||
@@ -5862,7 +5602,8 @@ async def infonet_sync(
|
||||
)
|
||||
base = after_hash or GENESIS_HASH
|
||||
events = infonet.get_events_after(base, limit=limit)
|
||||
events = _infonet_sync_response_events(events, request=request)
|
||||
# Filter out legacy gate_message events — not part of the public sync surface.
|
||||
events = [_redact_public_event(e) for e in events if e.get("event_type") != "gate_message"]
|
||||
return {
|
||||
"events": events,
|
||||
"after_hash": base,
|
||||
@@ -5901,7 +5642,6 @@ async def infonet_ingest(request: Request):
|
||||
|
||||
result = infonet.ingest_events(events)
|
||||
_hydrate_gate_store_from_chain(events)
|
||||
_hydrate_dm_relay_from_chain(events)
|
||||
return {"ok": True, **result}
|
||||
|
||||
|
||||
@@ -5942,7 +5682,6 @@ async def infonet_peer_push(request: Request):
|
||||
|
||||
result = infonet.ingest_events(events)
|
||||
_hydrate_gate_store_from_chain(events)
|
||||
_hydrate_dm_relay_from_chain(events)
|
||||
return {"ok": True, **result}
|
||||
|
||||
|
||||
@@ -6502,12 +6241,6 @@ async def infonet_event(request: Request, event_id: str):
|
||||
)
|
||||
return _strip_gate_for_access(evt, access)
|
||||
return {"ok": False, "detail": "Event not found"}
|
||||
if evt.get("event_type") == "dm_message":
|
||||
return await _private_plane_refusal_response(
|
||||
request,
|
||||
status_code=403,
|
||||
payload=_private_plane_access_denied_payload(),
|
||||
)
|
||||
if evt.get("event_type") == "gate_message":
|
||||
gate_id = str(evt.get("payload", {}).get("gate", "") or evt.get("gate", "") or "").strip()
|
||||
access = _verify_gate_access(request, gate_id) if gate_id else ""
|
||||
@@ -6532,7 +6265,7 @@ async def infonet_node_events(
|
||||
from services.mesh.mesh_hashchain import infonet
|
||||
|
||||
events = infonet.get_events_by_node(node_id, limit=limit)
|
||||
events = [e for e in events if e.get("event_type") not in {"gate_message", "dm_message"}]
|
||||
events = [e for e in events if e.get("event_type") != "gate_message"]
|
||||
events = [_redact_public_event(e) for e in infonet.decorate_events(events)]
|
||||
events = _redact_public_node_history(
|
||||
events,
|
||||
@@ -6557,7 +6290,7 @@ async def infonet_events_by_type(
|
||||
else:
|
||||
events = list(reversed(infonet.events))
|
||||
events = events[offset : offset + limit]
|
||||
events = [e for e in events if e.get("event_type") not in {"gate_message", "dm_message"}]
|
||||
events = [e for e in events if e.get("event_type") != "gate_message"]
|
||||
events = [_redact_public_event(e) for e in infonet.decorate_events(events)]
|
||||
return {
|
||||
"events": events,
|
||||
@@ -7295,7 +7028,6 @@ async def _dm_send_from_signed_request(request: Request):
|
||||
relay_salt_hex = str(body.get("relay_salt", "") or "").strip().lower()
|
||||
msg_id = str(body.get("msg_id", "")).strip()
|
||||
timestamp = _safe_int(body.get("timestamp", 0) or 0)
|
||||
sequence = _safe_int(body.get("sequence", 0) or 0)
|
||||
nonce = str(body.get("nonce", "")).strip()
|
||||
|
||||
if not sender_id or not recipient_id or not ciphertext or not msg_id or not timestamp:
|
||||
@@ -7369,7 +7101,7 @@ async def _dm_send_from_signed_request(request: Request):
|
||||
ok_seq, seq_reason = _validate_private_signed_sequence(
|
||||
infonet,
|
||||
sender_id,
|
||||
sequence,
|
||||
int(body.get("sequence", 0) or 0),
|
||||
domain="dm_send",
|
||||
)
|
||||
if not ok_seq:
|
||||
@@ -7403,47 +7135,7 @@ async def _dm_send_from_signed_request(request: Request):
|
||||
"sender_seal": sender_seal,
|
||||
"relay_salt": relay_salt_hex,
|
||||
}
|
||||
hashchain_spool: dict[str, Any] = {"ok": False, "detail": "not attempted"}
|
||||
try:
|
||||
from services.mesh.mesh_hashchain import infonet
|
||||
|
||||
chain_payload = dict(prepared.payload if prepared is not None else {})
|
||||
if not chain_payload:
|
||||
chain_payload = {
|
||||
"recipient_id": recipient_id,
|
||||
"delivery_class": delivery_class,
|
||||
"recipient_token": recipient_token if delivery_class == "shared" else "",
|
||||
"ciphertext": ciphertext,
|
||||
"msg_id": msg_id,
|
||||
"timestamp": timestamp,
|
||||
"format": payload_format,
|
||||
}
|
||||
chain_payload["transport_lock"] = "private_strong"
|
||||
chain_event = infonet.append_private_dm_message(
|
||||
node_id=sender_id,
|
||||
payload=chain_payload,
|
||||
signature=str(prepared.signature if prepared is not None else body.get("signature", "") or ""),
|
||||
sequence=sequence,
|
||||
public_key=str(prepared.public_key if prepared is not None else body.get("public_key", "") or ""),
|
||||
public_key_algo=str(
|
||||
prepared.public_key_algo if prepared is not None else body.get("public_key_algo", "") or ""
|
||||
),
|
||||
protocol_version=str(
|
||||
prepared.protocol_version if prepared is not None else body.get("protocol_version", "") or ""
|
||||
)
|
||||
or PROTOCOL_VERSION,
|
||||
timestamp=float(timestamp or time.time()),
|
||||
)
|
||||
_hydrate_dm_relay_from_chain([chain_event])
|
||||
hashchain_spool = {
|
||||
"ok": True,
|
||||
"event_id": str(chain_event.get("event_id", "") or ""),
|
||||
"limit": 2,
|
||||
}
|
||||
except Exception as exc:
|
||||
hashchain_spool = {"ok": False, "detail": str(exc) or type(exc).__name__}
|
||||
queued_result = _queue_dm_release(current_tier=tier, payload=release_payload)
|
||||
queued_result["hashchain_spool"] = hashchain_spool
|
||||
if transport_upgrade_pending:
|
||||
queued_result["private_transport_pending"] = True
|
||||
return queued_result
|
||||
@@ -9419,11 +9111,6 @@ async def api_get_node_settings(request: Request):
|
||||
async def api_set_node_settings(request: Request, body: NodeSettingsUpdate):
|
||||
_refresh_node_peer_store()
|
||||
if bool(body.enabled):
|
||||
if _infonet_private_transport_required() and not _ensure_infonet_private_transport_ready("operator_enable"):
|
||||
return JSONResponse(
|
||||
{"ok": False, "detail": _infonet_private_transport_error()},
|
||||
status_code=503,
|
||||
)
|
||||
try:
|
||||
from services.transport_lane_isolation import disable_public_mesh_lane
|
||||
|
||||
@@ -9432,7 +9119,6 @@ async def api_set_node_settings(request: Request, body: NodeSettingsUpdate):
|
||||
logger.warning("Failed to disable public Mesh while enabling private node: %s", exc)
|
||||
result = _set_participant_node_enabled(bool(body.enabled))
|
||||
if bool(body.enabled):
|
||||
_start_infonet_node_runtime("operator_enable")
|
||||
_kick_public_sync_background("operator_enable")
|
||||
return result
|
||||
|
||||
@@ -12054,36 +11740,5 @@ async def system_update(request: Request):
|
||||
return result
|
||||
|
||||
|
||||
def _dev_uvicorn_bind_host() -> str:
|
||||
"""Default loopback for `python main.py` so LAN clients cannot reach a dev server (#375).
|
||||
|
||||
Docker compose still publishes 127.0.0.1:8000; the dashboard stays on :3000.
|
||||
Set SHADOWBROKER_DEV_BIND_ALL=true only when you intentionally need LAN access
|
||||
(and use ADMIN_KEY for remote callers).
|
||||
"""
|
||||
if str(os.environ.get("SHADOWBROKER_DEV_BIND_ALL", "")).strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
}:
|
||||
return "0.0.0.0"
|
||||
return "127.0.0.1"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_host = _dev_uvicorn_bind_host()
|
||||
_port = int(os.environ.get("BACKEND_PORT", "8000"))
|
||||
if _host == "127.0.0.1":
|
||||
logger.info(
|
||||
"Dev server binding %s:%s (loopback). Set SHADOWBROKER_DEV_BIND_ALL=true for 0.0.0.0.",
|
||||
_host,
|
||||
_port,
|
||||
)
|
||||
uvicorn.run(
|
||||
"main:app",
|
||||
host=_host,
|
||||
port=_port,
|
||||
reload=True,
|
||||
timeout_keep_alive=120,
|
||||
)
|
||||
uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True, timeout_keep_alive=120)
|
||||
|
||||
@@ -7,15 +7,15 @@ py-modules = []
|
||||
|
||||
[project]
|
||||
name = "backend"
|
||||
version = "0.9.81"
|
||||
version = "0.9.8"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"apscheduler==3.10.3",
|
||||
"beautifulsoup4>=4.9.0",
|
||||
"cachetools==5.5.2",
|
||||
"cryptography>=46.0.7",
|
||||
"cryptography>=41.0.0",
|
||||
"defusedxml>=0.7.1",
|
||||
"fastapi==0.136.3",
|
||||
"fastapi==0.115.12",
|
||||
"feedparser==6.0.10",
|
||||
"httpx==0.28.1",
|
||||
"playwright==1.59.0",
|
||||
@@ -24,7 +24,7 @@ dependencies = [
|
||||
"pydantic-settings==2.8.1",
|
||||
"pystac-client==0.8.6",
|
||||
"python-dotenv==1.2.2",
|
||||
"requests==2.33.0",
|
||||
"requests==2.31.0",
|
||||
"PySocks==1.7.1",
|
||||
"reverse-geocoder==1.5.1",
|
||||
"sgp4==2.25",
|
||||
@@ -33,18 +33,17 @@ dependencies = [
|
||||
"paho-mqtt>=1.6.0,<2.0.0",
|
||||
"PyNaCl>=1.5.0",
|
||||
"slowapi==0.1.9",
|
||||
"starlette==1.0.1",
|
||||
"vaderSentiment>=3.3.0",
|
||||
"uvicorn==0.34.0",
|
||||
"yfinance==1.3.0",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = ["pytest>=9.0.3", "pytest-asyncio>=1.4.0", "ruff>=0.9.0", "black>=24.0.0"]
|
||||
dev = ["pytest>=8.3.4", "pytest-asyncio==0.25.0", "ruff>=0.9.0", "black>=24.0.0"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
# The current backend carries historical style debt in large legacy modules.
|
||||
# Keep CI focused on actionable correctness checks for the v0.9.81 release.
|
||||
# Keep CI focused on actionable correctness checks for the v0.9.8 release.
|
||||
ignore = ["E401", "E402", "E701", "E731", "E741", "F401", "F402", "F541", "F811", "F841"]
|
||||
|
||||
[tool.black]
|
||||
|
||||
@@ -1590,7 +1590,7 @@ async def agent_tool_manifest(request: Request):
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"version": "0.9.81",
|
||||
"version": "0.9.8",
|
||||
"access_tier": access_tier,
|
||||
"available_commands": available_commands,
|
||||
"transport": {
|
||||
@@ -2226,7 +2226,7 @@ async def api_capabilities(request: Request):
|
||||
access_tier = str(get_settings().OPENCLAW_ACCESS_TIER or "restricted").strip().lower()
|
||||
return {
|
||||
"ok": True,
|
||||
"version": "0.9.81",
|
||||
"version": "0.9.8",
|
||||
"auth": {
|
||||
"method": "HMAC-SHA256",
|
||||
"headers": ["X-SB-Timestamp", "X-SB-Nonce", "X-SB-Signature"],
|
||||
|
||||
+2
-110
@@ -1,7 +1,6 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
from typing import Any
|
||||
from fastapi import APIRouter, Request, Response, Query, Depends
|
||||
@@ -9,7 +8,7 @@ from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel
|
||||
from limiter import limiter
|
||||
from auth import require_admin, require_local_operator
|
||||
from services.data_fetcher import update_all_data
|
||||
from services.data_fetcher import get_latest_data, update_all_data
|
||||
import orjson
|
||||
import json as json_mod
|
||||
|
||||
@@ -31,14 +30,6 @@ class LayerUpdate(BaseModel):
|
||||
layers: dict[str, bool]
|
||||
|
||||
|
||||
class LiveUamapOptInUpdate(BaseModel):
|
||||
opted_in: bool
|
||||
|
||||
|
||||
class PredictionMarketsOptInUpdate(BaseModel):
|
||||
opted_in: bool
|
||||
|
||||
|
||||
_LAST_VIEWPORT_UPDATE: tuple | None = None
|
||||
_LAST_VIEWPORT_UPDATE_TS = 0.0
|
||||
_VIEWPORT_UPDATE_LOCK = threading.Lock()
|
||||
@@ -395,95 +386,6 @@ async def update_viewport(vp: ViewportUpdate, request: Request): # noqa: ARG001
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.get("/api/liveuamap/scraper-status", dependencies=[Depends(require_local_operator)])
|
||||
async def api_liveuamap_scraper_status():
|
||||
"""Whether LiveUAMap Playwright may run (Windows needs UI opt-in unless env forces)."""
|
||||
from services.liveuamap_settings import liveuamap_scraper_status
|
||||
|
||||
return liveuamap_scraper_status()
|
||||
|
||||
|
||||
@router.post("/api/liveuamap/scraper-opt-in", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("10/minute")
|
||||
async def api_liveuamap_scraper_opt_in(body: LiveUamapOptInUpdate, request: Request):
|
||||
"""Persist operator consent for LiveUAMap scraper (#348)."""
|
||||
from services.liveuamap_settings import liveuamap_scraper_status, set_liveuamap_ui_opt_in
|
||||
|
||||
set_liveuamap_ui_opt_in(body.opted_in)
|
||||
if body.opted_in:
|
||||
from services.fetchers._store import is_any_active
|
||||
|
||||
if is_any_active("global_incidents"):
|
||||
threading.Thread(target=_run_liveuamap_refresh, daemon=True).start()
|
||||
return liveuamap_scraper_status()
|
||||
|
||||
|
||||
def _run_liveuamap_refresh() -> None:
|
||||
try:
|
||||
from services.fetchers.geo import update_liveuamap
|
||||
|
||||
update_liveuamap()
|
||||
except Exception as e:
|
||||
logger.warning("LiveUAMap refresh after opt-in failed: %s", e)
|
||||
|
||||
|
||||
@router.get("/api/prediction-markets/status", dependencies=[Depends(require_local_operator)])
|
||||
async def api_prediction_markets_status():
|
||||
"""Whether Polymarket/Kalshi fetches and news market correlation are enabled."""
|
||||
from services.prediction_markets_settings import prediction_markets_status
|
||||
|
||||
return prediction_markets_status()
|
||||
|
||||
|
||||
@router.post("/api/prediction-markets/opt-in", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("10/minute")
|
||||
async def api_prediction_markets_opt_in(body: PredictionMarketsOptInUpdate, request: Request):
|
||||
"""Enable or disable prediction market fetches + intercept story correlation."""
|
||||
from services.config import get_settings
|
||||
from services.prediction_markets_settings import (
|
||||
prediction_markets_status,
|
||||
set_prediction_markets_ui_opt_in,
|
||||
)
|
||||
from routers.ai_intel import _write_env_value
|
||||
|
||||
set_prediction_markets_ui_opt_in(body.opted_in)
|
||||
_write_env_value("PREDICTION_MARKETS_ENABLED", "true" if body.opted_in else "false")
|
||||
os.environ["PREDICTION_MARKETS_ENABLED"] = "true" if body.opted_in else "false"
|
||||
get_settings.cache_clear()
|
||||
|
||||
if body.opted_in:
|
||||
threading.Thread(target=_run_prediction_markets_refresh, daemon=True).start()
|
||||
else:
|
||||
threading.Thread(target=_run_prediction_markets_disable, daemon=True).start()
|
||||
|
||||
return prediction_markets_status()
|
||||
|
||||
|
||||
def _run_prediction_markets_refresh() -> None:
|
||||
try:
|
||||
from services.fetchers.prediction_markets import fetch_prediction_markets
|
||||
from services.fetchers.news import fetch_news
|
||||
|
||||
fetch_prediction_markets()
|
||||
fetch_news()
|
||||
except Exception as e:
|
||||
logger.warning("Prediction markets refresh after opt-in failed: %s", e)
|
||||
|
||||
|
||||
def _run_prediction_markets_disable() -> None:
|
||||
try:
|
||||
from services.fetchers._store import _data_lock, _mark_fresh, latest_data
|
||||
from services.fetchers.news import fetch_news
|
||||
|
||||
with _data_lock:
|
||||
latest_data["prediction_markets"] = []
|
||||
latest_data["trending_markets"] = []
|
||||
_mark_fresh("prediction_markets")
|
||||
fetch_news()
|
||||
except Exception as e:
|
||||
logger.warning("Prediction markets disable cleanup failed: %s", e)
|
||||
|
||||
|
||||
@router.post("/api/layers", dependencies=[Depends(require_local_operator)])
|
||||
@limiter.limit("30/minute")
|
||||
async def update_layers(update: LayerUpdate, request: Request):
|
||||
@@ -554,17 +456,7 @@ async def update_layers(update: LayerUpdate, request: Request):
|
||||
@router.get("/api/live-data")
|
||||
@limiter.limit("120/minute")
|
||||
async def live_data(request: Request):
|
||||
etag = _current_etag(prefix="live|full|")
|
||||
if request.headers.get("if-none-match") == etag:
|
||||
return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
|
||||
from services.fetchers._store import get_latest_data_deepcopy_snapshot
|
||||
|
||||
payload = get_latest_data_deepcopy_snapshot()
|
||||
return Response(
|
||||
content=orjson.dumps(_sanitize_payload(payload)),
|
||||
media_type="application/json",
|
||||
headers={"ETag": etag, "Cache-Control": "no-cache"},
|
||||
)
|
||||
return get_latest_data()
|
||||
|
||||
|
||||
@router.get("/api/bootstrap/critical")
|
||||
|
||||
@@ -8,7 +8,7 @@ from services.data_fetcher import get_latest_data
|
||||
from services.schemas import HealthResponse
|
||||
import os
|
||||
|
||||
APP_VERSION = os.environ.get("_HEALTH_APP_VERSION", "0.9.81")
|
||||
APP_VERSION = os.environ.get("_HEALTH_APP_VERSION", "0.9.8")
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@@ -55,12 +55,6 @@ def _hydrate_gate_store_from_chain(events: list) -> int:
|
||||
return count
|
||||
|
||||
|
||||
def _hydrate_dm_relay_from_chain(events: list) -> int:
|
||||
import main as _m
|
||||
|
||||
return int(_m._hydrate_dm_relay_from_chain(events))
|
||||
|
||||
|
||||
@router.post("/api/mesh/infonet/peer-push")
|
||||
@limiter.limit("30/minute")
|
||||
async def infonet_peer_push(request: Request):
|
||||
@@ -88,7 +82,6 @@ async def infonet_peer_push(request: Request):
|
||||
return {"ok": True, "accepted": 0, "duplicates": 0, "rejected": []}
|
||||
result = infonet.ingest_events(events)
|
||||
_hydrate_gate_store_from_chain(events)
|
||||
_hydrate_dm_relay_from_chain(events)
|
||||
return {"ok": True, **result}
|
||||
|
||||
|
||||
|
||||
@@ -65,7 +65,6 @@ from services.mesh.mesh_signed_events import (
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
_INFONET_SYNC_RATE_LIMIT = "600/minute"
|
||||
|
||||
|
||||
def _signed_body(request: Request) -> dict[str, Any]:
|
||||
@@ -264,19 +263,6 @@ def _redact_public_event(event: dict) -> dict:
|
||||
return _redact_vote_gate(_redact_key_rotate_payload(_redact_gate_metadata(event)))
|
||||
|
||||
|
||||
def _infonet_private_transport_required() -> bool:
|
||||
import main as _m
|
||||
|
||||
return bool(_m._infonet_private_transport_required())
|
||||
|
||||
|
||||
def _infonet_sync_response_events(events: list[dict], request=None) -> list[dict]:
|
||||
"""Build the sync event surface for the current transport policy."""
|
||||
import main as _m
|
||||
|
||||
return _m._infonet_sync_response_events(events, request=request)
|
||||
|
||||
|
||||
def _trusted_gate_reply_to(event: dict) -> str:
|
||||
if not isinstance(event, dict):
|
||||
return ""
|
||||
@@ -588,12 +574,6 @@ def _hydrate_gate_store_from_chain(events: list[dict]) -> int:
|
||||
pass
|
||||
return count
|
||||
|
||||
|
||||
def _hydrate_dm_relay_from_chain(events: list[dict]) -> int:
|
||||
import main as _m
|
||||
|
||||
return int(_m._hydrate_dm_relay_from_chain(events))
|
||||
|
||||
# --- Safe type helpers ---
|
||||
|
||||
def _safe_int(val, default=0):
|
||||
@@ -1551,7 +1531,7 @@ async def infonet_locator(request: Request, limit: int = Query(32, ge=4, le=128)
|
||||
|
||||
|
||||
@router.post("/api/mesh/infonet/sync")
|
||||
@limiter.limit(_INFONET_SYNC_RATE_LIMIT)
|
||||
@limiter.limit("30/minute")
|
||||
@mesh_write_exempt(MeshWriteExemption.PEER_GOSSIP)
|
||||
async def infonet_sync_post(
|
||||
request: Request,
|
||||
@@ -1604,7 +1584,8 @@ async def infonet_sync_post(
|
||||
elif matched_hash == GENESIS_HASH and len(locator) > 1:
|
||||
forked = True
|
||||
|
||||
events = _infonet_sync_response_events(events, request=request)
|
||||
# Filter out legacy gate_message events — not part of the public sync surface.
|
||||
events = [_redact_public_event(e) for e in events if e.get("event_type") != "gate_message"]
|
||||
|
||||
response = {
|
||||
"events": events,
|
||||
@@ -1665,7 +1646,7 @@ async def mesh_rns_status(request: Request):
|
||||
|
||||
|
||||
@router.get("/api/mesh/infonet/sync")
|
||||
@limiter.limit(_INFONET_SYNC_RATE_LIMIT)
|
||||
@limiter.limit("30/minute")
|
||||
async def infonet_sync(
|
||||
request: Request,
|
||||
after_hash: str = "",
|
||||
@@ -1703,7 +1684,8 @@ async def infonet_sync(
|
||||
)
|
||||
base = after_hash or GENESIS_HASH
|
||||
events = infonet.get_events_after(base, limit=limit)
|
||||
events = _infonet_sync_response_events(events, request=request)
|
||||
# Filter out legacy gate_message events — not part of the public sync surface.
|
||||
events = [_redact_public_event(e) for e in events if e.get("event_type") != "gate_message"]
|
||||
return {
|
||||
"events": events,
|
||||
"after_hash": base,
|
||||
@@ -1742,7 +1724,6 @@ async def infonet_ingest(request: Request):
|
||||
|
||||
result = infonet.ingest_events(events)
|
||||
_hydrate_gate_store_from_chain(events)
|
||||
_hydrate_dm_relay_from_chain(events)
|
||||
return {"ok": True, **result}
|
||||
|
||||
|
||||
@@ -2298,12 +2279,6 @@ async def infonet_event(request: Request, event_id: str):
|
||||
)
|
||||
return _strip_gate_for_access(evt, access)
|
||||
return {"ok": False, "detail": "Event not found"}
|
||||
if evt.get("event_type") == "dm_message":
|
||||
return await _private_plane_refusal_response(
|
||||
request,
|
||||
status_code=403,
|
||||
payload=_private_plane_access_denied_payload(),
|
||||
)
|
||||
if evt.get("event_type") == "gate_message":
|
||||
gate_id = str(evt.get("payload", {}).get("gate", "") or evt.get("gate", "") or "").strip()
|
||||
access = _verify_gate_access(request, gate_id) if gate_id else ""
|
||||
@@ -2328,7 +2303,7 @@ async def infonet_node_events(
|
||||
from services.mesh.mesh_hashchain import infonet
|
||||
|
||||
events = infonet.get_events_by_node(node_id, limit=limit)
|
||||
events = [e for e in events if e.get("event_type") not in {"gate_message", "dm_message"}]
|
||||
events = [e for e in events if e.get("event_type") != "gate_message"]
|
||||
events = [_redact_public_event(e) for e in infonet.decorate_events(events)]
|
||||
events = _redact_public_node_history(
|
||||
events,
|
||||
@@ -2353,7 +2328,7 @@ async def infonet_events_by_type(
|
||||
else:
|
||||
events = list(reversed(infonet.events))
|
||||
events = events[offset : offset + limit]
|
||||
events = [e for e in events if e.get("event_type") not in {"gate_message", "dm_message"}]
|
||||
events = [e for e in events if e.get("event_type") != "gate_message"]
|
||||
events = [_redact_public_event(e) for e in infonet.decorate_events(events)]
|
||||
return {
|
||||
"events": events,
|
||||
|
||||
@@ -85,39 +85,6 @@ async def api_geocode_reverse(
|
||||
return await asyncio.to_thread(reverse_geocode, lat, lng, local_only)
|
||||
|
||||
|
||||
# ── Wikimedia proxy (#360) — browser calls these instead of wikipedia.org ───
|
||||
@router.get("/api/wikipedia/summary")
|
||||
@limiter.limit("60/minute")
|
||||
def api_wikipedia_summary(
|
||||
request: Request,
|
||||
title: str = Query(..., min_length=1, max_length=256),
|
||||
):
|
||||
"""Proxy Wikipedia REST summaries through the self-hosted backend."""
|
||||
from services.region_dossier import fetch_wikipedia_page_summary
|
||||
|
||||
summary = fetch_wikipedia_page_summary(title)
|
||||
if summary is None:
|
||||
return JSONResponse(status_code=404, content={"detail": "not_found"})
|
||||
return summary
|
||||
|
||||
|
||||
class WikidataSparqlRequest(BaseModel):
|
||||
query: str
|
||||
|
||||
|
||||
@router.post("/api/wikidata/sparql")
|
||||
@limiter.limit("30/minute")
|
||||
def api_wikidata_sparql(request: Request, body: WikidataSparqlRequest):
|
||||
"""Proxy Wikidata SPARQL so the browser never contacts query.wikidata.org."""
|
||||
from services.region_dossier import fetch_wikidata_sparql_bindings
|
||||
|
||||
q = (body.query or "").strip()
|
||||
if len(q) > 12_000:
|
||||
raise HTTPException(400, "SPARQL query too large")
|
||||
bindings = fetch_wikidata_sparql_bindings(q)
|
||||
return {"bindings": bindings}
|
||||
|
||||
|
||||
# ── Sentinel proxy routes (Issue #299/#300/#301, reported by tg12) ──────────
|
||||
# These three endpoints relay external Sentinel / Planetary Computer
|
||||
# requests through the backend to avoid browser CORS blocks. They are
|
||||
|
||||
@@ -29,7 +29,7 @@ def main() -> None:
|
||||
from services.network_utils import outbound_user_agent
|
||||
ua = outbound_user_agent("release-script-power-plants")
|
||||
except Exception:
|
||||
ua = "operator-release-script (purpose: power-plants)"
|
||||
ua = "Shadowbroker/0.9 (release-script-power-plants; +https://github.com/BigBodyCobain/Shadowbroker/issues)"
|
||||
req = urllib.request.Request(CSV_URL, headers={"User-Agent": ua})
|
||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||
raw = resp.read().decode("utf-8")
|
||||
|
||||
@@ -167,11 +167,6 @@ def cmd_hash(args: argparse.Namespace) -> int:
|
||||
print("")
|
||||
print("Updater pin:")
|
||||
print(f"MESH_UPDATE_SHA256={digest}")
|
||||
print("")
|
||||
print("Release checklist:")
|
||||
print(" - add this digest to SHA256SUMS.txt for the GitHub release")
|
||||
print(" - add/update backend/data/release_digests.json for bundled updater verification")
|
||||
print(" - keep MESH_UPDATE_SHA256 available as the operator override path")
|
||||
return 0 if asset_matches else 2
|
||||
|
||||
|
||||
|
||||
@@ -92,37 +92,18 @@ SECRET_REGEX+='pypi-[0-9a-zA-Z-]{50,}' # PyPI token
|
||||
TEXT_FILES=$(grep -ivE '\.(png|jpg|jpeg|gif|ico|svg|woff2?|ttf|eot|pbf|zip|tar|gz|db|sqlite|xlsx|pdf|mp[34]|wav|ogg|webm|webp|avif)$' "$FILELIST" | grep -v 'scan-secrets\.sh$' || true)
|
||||
|
||||
if [[ -n "$TEXT_FILES" ]]; then
|
||||
# Known-public exclusions: lines matching `<host-or-ip> ssh-<algo> <key>`
|
||||
# are SSH known_hosts entries — the host's PUBLIC fingerprint, which is
|
||||
# by definition safe to commit (the whole point of pinning known_hosts
|
||||
# is to publish the fingerprint widely so MITM is detectable). Filter
|
||||
# these out before flagging the file.
|
||||
KNOWN_HOSTS_LINE='^[[:space:]]*[a-zA-Z0-9._:,*-]+([[:space:]]+[a-zA-Z0-9._:,*-]+)?[[:space:]]+(ssh-rsa|ssh-ed25519|ssh-dss|ecdsa-sha2-nistp256|ecdsa-sha2-nistp384|ecdsa-sha2-nistp521)[[:space:]]+AAAA'
|
||||
|
||||
# Use grep with file list, skip missing/binary, limit output
|
||||
CONTENT_HITS=$(echo "$TEXT_FILES" | xargs grep -lE "$SECRET_REGEX" 2>/dev/null || true)
|
||||
if [[ -n "$CONTENT_HITS" ]]; then
|
||||
REAL_HITS=""
|
||||
REAL_REPORT=""
|
||||
while IFS= read -r f; do
|
||||
[[ -z "$f" ]] && continue
|
||||
# Re-grep this file, but filter out known_hosts-style lines.
|
||||
FILE_HITS=$(grep -nE "$SECRET_REGEX" "$f" 2>/dev/null | grep -vE "$KNOWN_HOSTS_LINE" || true)
|
||||
if [[ -n "$FILE_HITS" ]]; then
|
||||
REAL_HITS+="$f"$'\n'
|
||||
REAL_REPORT+=" ${RED}$f${NC}"$'\n'
|
||||
# Show first 2 matching lines for context
|
||||
while IFS= read -r line; do
|
||||
[[ -z "$line" ]] && continue
|
||||
REAL_REPORT+=" ${YELLOW}$line${NC}"$'\n'
|
||||
done < <(echo "$FILE_HITS" | head -2)
|
||||
fi
|
||||
done <<< "$CONTENT_HITS"
|
||||
if [[ -n "$REAL_HITS" ]]; then
|
||||
echo -e "\n${RED}BLOCKED: Embedded secrets/tokens found in:${NC}"
|
||||
echo -en "$REAL_REPORT"
|
||||
FOUND=1
|
||||
fi
|
||||
echo -e "\n${RED}BLOCKED: Embedded secrets/tokens found in:${NC}"
|
||||
echo "$CONTENT_HITS" | while read -r f; do
|
||||
echo -e " ${RED}$f${NC}"
|
||||
# Show first matching line for context
|
||||
grep -nE "$SECRET_REGEX" "$f" 2>/dev/null | head -2 | while read -r line; do
|
||||
echo -e " ${YELLOW}$line${NC}"
|
||||
done
|
||||
done
|
||||
FOUND=1
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
@@ -1012,33 +1012,14 @@ def _extract_img_src(html_fragment: str):
|
||||
class MadridCityIngestor(BaseCCTVIngestor):
|
||||
"""Madrid City Hall traffic cameras from datos.madrid.es KML feed."""
|
||||
|
||||
KML_URL_HTTPS = "https://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml"
|
||||
KML_URL_HTTP = "http://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml"
|
||||
|
||||
def _fetch_kml(self):
|
||||
"""Prefer HTTPS; fall back to legacy HTTP if the catalog is HTTP-only (#363)."""
|
||||
last_error: Exception | None = None
|
||||
for url in (self.KML_URL_HTTPS, self.KML_URL_HTTP):
|
||||
try:
|
||||
response = fetch_with_curl(url, timeout=20)
|
||||
response.raise_for_status()
|
||||
if url == self.KML_URL_HTTP:
|
||||
logger.warning(
|
||||
"MadridCityIngestor: HTTPS KML unavailable, using HTTP catalog feed"
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
logger.debug("MadridCityIngestor: KML fetch failed for %s: %s", url, e)
|
||||
if last_error is not None:
|
||||
raise last_error
|
||||
raise RuntimeError("Madrid KML fetch failed")
|
||||
KML_URL = "http://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml"
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
import defusedxml.ElementTree as ET
|
||||
|
||||
try:
|
||||
response = self._fetch_kml()
|
||||
response = fetch_with_curl(self.KML_URL, timeout=20)
|
||||
response.raise_for_status()
|
||||
except Exception as e:
|
||||
logger.error(f"MadridCityIngestor: failed to fetch KML: {e}")
|
||||
return []
|
||||
|
||||
@@ -32,7 +32,6 @@ class Settings(BaseSettings):
|
||||
MESH_ARTI_ENABLED: bool = False
|
||||
MESH_ARTI_SOCKS_PORT: int = 9050
|
||||
MESH_RELAY_PEERS: str = ""
|
||||
MESH_PUBLIC_PEER_URL: str = ""
|
||||
# Bootstrap seeds are discovery hints, not authoritative network roots.
|
||||
# Nodes promote healthy discovered peers from the store/manifest over time.
|
||||
MESH_BOOTSTRAP_SEED_PEERS: str = "http://gqpbunqbgtkcqilvclm3xrkt3zowjyl3s62kkktvojgvxzizamvbrqid.onion:8000"
|
||||
|
||||
@@ -19,7 +19,6 @@ import concurrent.futures
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
@@ -145,18 +144,13 @@ _STARTUP_HEAVY_REFRESH_DELAY_S = float(os.environ.get("SHADOWBROKER_STARTUP_HEAV
|
||||
_STARTUP_HEAVY_REFRESH_STARTED = False
|
||||
_STARTUP_HEAVY_REFRESH_LOCK = threading.Lock()
|
||||
_FETCH_WORKERS = int(os.environ.get("SHADOWBROKER_FETCH_WORKERS", "8"))
|
||||
_HEAVY_FETCH_WORKERS = int(os.environ.get("SHADOWBROKER_HEAVY_FETCH_WORKERS", "2"))
|
||||
_SLOW_FETCH_CONCURRENCY = int(os.environ.get("SHADOWBROKER_SLOW_FETCH_CONCURRENCY", "4"))
|
||||
_STARTUP_HEAVY_CONCURRENCY = int(os.environ.get("SHADOWBROKER_STARTUP_HEAVY_CONCURRENCY", "2"))
|
||||
|
||||
# Fast-tier pool (flights, ships, sigint, …). Slow / heavy work uses a separate pool
|
||||
# so Playwright, GDELT, CCTV ingest, etc. cannot starve the 60s refresh path (#375).
|
||||
# Shared thread pool — reused across all fetch cycles instead of creating/destroying per tick
|
||||
_SHARED_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=max(2, _FETCH_WORKERS), thread_name_prefix="fetch"
|
||||
)
|
||||
_SLOW_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=max(1, _HEAVY_FETCH_WORKERS), thread_name_prefix="fetch-slow"
|
||||
)
|
||||
|
||||
|
||||
def _cache_json_safe(value):
|
||||
@@ -325,42 +319,10 @@ def seed_startup_caches() -> None:
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scheduler & Orchestration
|
||||
# ---------------------------------------------------------------------------
|
||||
def _executor_for_task_label(label: str) -> concurrent.futures.ThreadPoolExecutor:
|
||||
if label.startswith(("slow-tier", "startup-heavy")):
|
||||
return _SLOW_EXECUTOR
|
||||
return _SHARED_EXECUTOR
|
||||
|
||||
|
||||
def _run_task_with_health_on_executor(
|
||||
executor: concurrent.futures.ThreadPoolExecutor,
|
||||
func,
|
||||
name: str | None = None,
|
||||
) -> None:
|
||||
"""Run a scheduled job on the given pool so it cannot starve fast-tier workers."""
|
||||
task_name = name or getattr(func, "__name__", "task")
|
||||
future = executor.submit(func)
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
future.result(timeout=_TASK_HARD_TIMEOUT_S)
|
||||
duration = time.perf_counter() - start
|
||||
from services.fetch_health import record_success
|
||||
|
||||
record_success(task_name, duration_s=duration)
|
||||
if duration > _SLOW_FETCH_S:
|
||||
logger.warning("task slow: %s took %.2f}s", task_name, duration)
|
||||
except Exception as e:
|
||||
duration = time.perf_counter() - start
|
||||
from services.fetch_health import record_failure
|
||||
|
||||
record_failure(task_name, error=e, duration_s=duration)
|
||||
logger.exception("task failed: %s", task_name)
|
||||
|
||||
|
||||
def _run_tasks(label: str, funcs: list, *, max_concurrency: int | None = None):
|
||||
"""Run tasks concurrently and log any exceptions (do not fail silently)."""
|
||||
if not funcs:
|
||||
return
|
||||
executor = _executor_for_task_label(label)
|
||||
if max_concurrency is None:
|
||||
if label.startswith("slow-tier"):
|
||||
max_concurrency = _SLOW_FETCH_CONCURRENCY
|
||||
@@ -373,7 +335,7 @@ def _run_tasks(label: str, funcs: list, *, max_concurrency: int | None = None):
|
||||
remaining_funcs = list(funcs)
|
||||
while remaining_funcs:
|
||||
batch, remaining_funcs = remaining_funcs[:max_concurrency], remaining_funcs[max_concurrency:]
|
||||
futures = {executor.submit(func): (func.__name__, time.perf_counter()) for func in batch}
|
||||
futures = {_SHARED_EXECUTOR.submit(func): (func.__name__, time.perf_counter()) for func in batch}
|
||||
_drain_task_futures(label, futures)
|
||||
|
||||
|
||||
@@ -443,6 +405,7 @@ def update_slow_data():
|
||||
logger.info("Slow-tier data update starting...")
|
||||
slow_funcs = [
|
||||
fetch_news,
|
||||
fetch_prediction_markets,
|
||||
fetch_earthquakes,
|
||||
fetch_firms_fires,
|
||||
fetch_firms_country_fires,
|
||||
@@ -784,27 +747,6 @@ def start_scheduler():
|
||||
misfire_grace_time=120,
|
||||
)
|
||||
|
||||
# Prediction markets — own jittered cadence (Polymarket/Kalshi clearnet egress).
|
||||
# Kept off the fixed 5-minute slow tier so poll timing is less fingerprintable.
|
||||
from services.fetchers.prediction_markets import fetch_prediction_markets
|
||||
|
||||
_pm_interval_m = max(5, int(os.environ.get("PREDICTION_MARKETS_INTERVAL_MINUTES", "7")))
|
||||
_pm_jitter_s = max(0, int(os.environ.get("PREDICTION_MARKETS_SCHEDULER_JITTER_S", "240")))
|
||||
_pm_initial_max_s = max(0, int(os.environ.get("PREDICTION_MARKETS_INITIAL_DELAY_MAX_S", "180")))
|
||||
_pm_first_run = datetime.utcnow() + timedelta(
|
||||
seconds=random.randint(30, max(30, _pm_initial_max_s))
|
||||
)
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health(fetch_prediction_markets, "fetch_prediction_markets"),
|
||||
"interval",
|
||||
minutes=_pm_interval_m,
|
||||
jitter=_pm_jitter_s,
|
||||
next_run_time=_pm_first_run,
|
||||
id="prediction_markets",
|
||||
max_instances=1,
|
||||
misfire_grace_time=300,
|
||||
)
|
||||
|
||||
# Weather alerts — every 5 minutes (time-critical, separate from slow tier)
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health(fetch_weather_alerts, "fetch_weather_alerts"),
|
||||
@@ -902,7 +844,7 @@ def start_scheduler():
|
||||
|
||||
# GDELT — every 30 minutes (downloads 32 ZIP files per call, avoid rate limits)
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health_on_executor(_SLOW_EXECUTOR, fetch_gdelt, "fetch_gdelt"),
|
||||
lambda: _run_task_with_health(fetch_gdelt, "fetch_gdelt"),
|
||||
"interval",
|
||||
minutes=30,
|
||||
id="gdelt",
|
||||
@@ -910,9 +852,7 @@ def start_scheduler():
|
||||
misfire_grace_time=120,
|
||||
)
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health_on_executor(
|
||||
_SLOW_EXECUTOR, update_liveuamap, "update_liveuamap"
|
||||
),
|
||||
lambda: _run_task_with_health(update_liveuamap, "update_liveuamap"),
|
||||
"interval",
|
||||
minutes=30,
|
||||
id="liveuamap",
|
||||
@@ -973,9 +913,7 @@ def start_scheduler():
|
||||
logger.warning(f"CCTV post-ingest refresh failed: {e}")
|
||||
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health_on_executor(
|
||||
_SLOW_EXECUTOR, _run_cctv_ingest_cycle, "cctv_ingest_cycle"
|
||||
),
|
||||
_run_cctv_ingest_cycle,
|
||||
"interval",
|
||||
minutes=10,
|
||||
id="cctv_ingest",
|
||||
@@ -1055,9 +993,9 @@ def start_scheduler():
|
||||
misfire_grace_time=600,
|
||||
)
|
||||
|
||||
# UAP sightings (NUFORC) — weekly Mondays 12:00 UTC. Rolling ~60-day window;
|
||||
# each self-hosted install pulls live nuforc.org so operators see current
|
||||
# reports (typically ~400–500 mappable pins). Disk cache TTL defaults to 7d.
|
||||
# UAP sightings (NUFORC) — weekly on Mondays at 12:00 UTC. The layer is a
|
||||
# rolling last-60-days digest; refreshing once a week is enough cadence
|
||||
# for human-readable map exploration and keeps load on nuforc.org light.
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health(
|
||||
lambda: fetch_uap_sightings(force_refresh=True),
|
||||
@@ -1192,10 +1130,7 @@ def start_scheduler():
|
||||
def stop_scheduler():
|
||||
if _scheduler:
|
||||
_scheduler.shutdown(wait=False)
|
||||
_SLOW_EXECUTOR.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
|
||||
def get_latest_data():
|
||||
from services.fetchers._store import get_latest_data_deepcopy_snapshot
|
||||
|
||||
return get_latest_data_deepcopy_snapshot()
|
||||
return get_latest_data_subset(*latest_data.keys())
|
||||
|
||||
@@ -241,22 +241,16 @@ def get_active_layers_version() -> int:
|
||||
def get_latest_data_subset(*keys: str) -> DashboardData:
|
||||
"""Return a deep snapshot of only the requested top-level keys.
|
||||
|
||||
Grabs references under the lock, then deep-copies outside it so fetcher
|
||||
writers are not blocked for the duration of a large clone (#375).
|
||||
This avoids cloning the entire dashboard store for endpoints that only need
|
||||
a small tier-specific subset. Deep copy ensures callers cannot mutate
|
||||
nested structures (e.g. individual flight dicts) and affect the live store.
|
||||
"""
|
||||
with _data_lock:
|
||||
items = [(key, latest_data.get(key)) for key in keys]
|
||||
snap: DashboardData = {}
|
||||
for key, value in items:
|
||||
snap[key] = copy.deepcopy(value)
|
||||
return snap
|
||||
|
||||
|
||||
def get_latest_data_deepcopy_snapshot() -> DashboardData:
|
||||
"""Deep-copy the full dashboard for legacy /api/live-data consumers."""
|
||||
with _data_lock:
|
||||
items = list(latest_data.items())
|
||||
return {key: copy.deepcopy(value) for key, value in items}
|
||||
snap: DashboardData = {}
|
||||
for key in keys:
|
||||
value = latest_data.get(key)
|
||||
snap[key] = copy.deepcopy(value)
|
||||
return snap
|
||||
|
||||
|
||||
def get_latest_data_subset_refs(*keys: str) -> DashboardData:
|
||||
|
||||
@@ -38,6 +38,8 @@ _S3_NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
|
||||
_REFRESH_INTERVAL_S = 5 * 24 * 3600
|
||||
_LIST_TIMEOUT_S = 30
|
||||
_DOWNLOAD_TIMEOUT_S = 600
|
||||
from services.network_utils import DEFAULT_USER_AGENT as _USER_AGENT
|
||||
|
||||
_lock = threading.RLock()
|
||||
_aircraft_by_hex: dict[str, dict[str, str]] = {}
|
||||
_last_refresh = 0.0
|
||||
|
||||
@@ -692,8 +692,7 @@ _NUFORC_TILESET = "nuforc.cmm18aqea06bu1mmselhpnano-0ce5v"
|
||||
_NUFORC_TOKEN = os.environ.get("NUFORC_MAPBOX_TOKEN", "").strip()
|
||||
_NUFORC_RADIUS_M = 200_000 # 200 km query radius
|
||||
_NUFORC_LIMIT = 50 # max features per tilequery call
|
||||
# Rolling window shown on the map (~2 calendar months). Override via NUFORC_RECENT_DAYS.
|
||||
_NUFORC_RECENT_DAYS = max(1, int(os.environ.get("NUFORC_RECENT_DAYS", "60")))
|
||||
_NUFORC_RECENT_DAYS = int(os.environ.get("NUFORC_RECENT_DAYS", "60"))
|
||||
_NUFORC_HF_FALLBACK_LIMIT = max(25, int(os.environ.get("NUFORC_HF_FALLBACK_LIMIT", "250")))
|
||||
_NUFORC_HF_GEOCODE_LIMIT = max(25, int(os.environ.get("NUFORC_HF_GEOCODE_LIMIT", "150")))
|
||||
_NUFORC_GEOCODE_WORKERS = max(1, int(os.environ.get("NUFORC_GEOCODE_WORKERS", "1")))
|
||||
@@ -701,12 +700,6 @@ _NUFORC_GEOCODE_WORKERS = max(1, int(os.environ.get("NUFORC_GEOCODE_WORKERS", "1
|
||||
# practice, so a 0.3s spacing keeps us well under any soft throttle while
|
||||
# still rebuilding a full 12-month window in ~10 minutes.
|
||||
_NUFORC_GEOCODE_SPACING_S = float(os.environ.get("NUFORC_GEOCODE_SPACING_S", "0.3"))
|
||||
# Disk cache TTL — match the weekly scheduler so restarts between fetches still
|
||||
# serve the same rolling 60-day snapshot without hammering nuforc.org daily.
|
||||
_NUFORC_CACHE_TTL_S = max(
|
||||
3600,
|
||||
int(os.environ.get("NUFORC_CACHE_TTL_HOURS", "168")) * 3600,
|
||||
)
|
||||
_NUFORC_DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"
|
||||
_NUFORC_SIGHTINGS_CACHE_FILE = _NUFORC_DATA_DIR / "nuforc_recent_sightings.json"
|
||||
_NUFORC_LOCATION_CACHE_FILE = _NUFORC_DATA_DIR / "nuforc_location_cache.json"
|
||||
@@ -773,35 +766,6 @@ def _fetch_nuforc_tilequery(lng: float, lat: float) -> list[dict]:
|
||||
return []
|
||||
|
||||
|
||||
def _uap_cutoff_date_str() -> str:
|
||||
return (datetime.utcnow() - timedelta(days=_NUFORC_RECENT_DAYS)).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def _uap_sighting_date_str(sighting: dict) -> str | None:
|
||||
"""Normalize a sighting row to YYYY-MM-DD for window filtering."""
|
||||
from services.fetchers.nuforc_enrichment import _parse_date
|
||||
|
||||
raw = str(sighting.get("date_time") or sighting.get("occurred") or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
parsed = _parse_date(raw)
|
||||
if parsed:
|
||||
return parsed
|
||||
if len(raw) >= 10 and raw[4] == "-" and raw[7] == "-":
|
||||
return raw[:10]
|
||||
return None
|
||||
|
||||
|
||||
def _filter_uap_sightings_recent(sightings: list[dict]) -> list[dict]:
|
||||
"""Drop anything outside the rolling NUFORC_RECENT_DAYS window."""
|
||||
cutoff = _uap_cutoff_date_str()
|
||||
return [
|
||||
sighting
|
||||
for sighting in sightings
|
||||
if (_uap_sighting_date_str(sighting) or "") >= cutoff
|
||||
]
|
||||
|
||||
|
||||
def _parse_nuforc_tile_date(value: str) -> datetime | None:
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
@@ -838,41 +802,19 @@ def _load_nuforc_sightings_cache(*, force_refresh: bool = False) -> list[dict] |
|
||||
built_dt = datetime.fromisoformat(built) if built else None
|
||||
if built_dt is None:
|
||||
return None
|
||||
if (datetime.utcnow() - built_dt).total_seconds() > _NUFORC_CACHE_TTL_S:
|
||||
return None
|
||||
if raw.get("cutoff_days") != _NUFORC_RECENT_DAYS:
|
||||
logger.info(
|
||||
"UAP sightings: cache cutoff_days mismatch (%s != %s); rebuilding",
|
||||
raw.get("cutoff_days"),
|
||||
_NUFORC_RECENT_DAYS,
|
||||
)
|
||||
if (datetime.utcnow() - built_dt).total_seconds() > 86400:
|
||||
return None
|
||||
sightings = raw.get("sightings")
|
||||
if isinstance(sightings, list):
|
||||
if len(sightings) <= 0:
|
||||
logger.info("UAP sightings: cache is fresh but empty; rebuilding")
|
||||
return None
|
||||
filtered = _filter_uap_sightings_recent(sightings)
|
||||
if not filtered:
|
||||
logger.warning(
|
||||
"UAP sightings: cache had %d rows but none within last %d days; rebuilding",
|
||||
len(sightings),
|
||||
_NUFORC_RECENT_DAYS,
|
||||
)
|
||||
return None
|
||||
if len(filtered) < len(sightings):
|
||||
logger.info(
|
||||
"UAP sightings: dropped %d stale cached rows outside %d-day window",
|
||||
len(sightings) - len(filtered),
|
||||
_NUFORC_RECENT_DAYS,
|
||||
)
|
||||
logger.info(
|
||||
"UAP sightings: loaded %d cached reports from %s (within %d-day window)",
|
||||
len(filtered),
|
||||
"UAP sightings: loaded %d cached reports from %s",
|
||||
len(sightings),
|
||||
built,
|
||||
_NUFORC_RECENT_DAYS,
|
||||
)
|
||||
return filtered
|
||||
return sightings
|
||||
except Exception as e:
|
||||
logger.warning("UAP sightings: cache load error: %s", e)
|
||||
return None
|
||||
@@ -886,7 +828,6 @@ def _save_nuforc_sightings_cache(sightings: list[dict]) -> None:
|
||||
_NUFORC_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"built": datetime.utcnow().isoformat(),
|
||||
"cutoff_days": _NUFORC_RECENT_DAYS,
|
||||
"count": len(sightings),
|
||||
"sightings": sightings,
|
||||
}
|
||||
@@ -1094,129 +1035,28 @@ def _nuforc_months_for_window(days: int) -> list[str]:
|
||||
return months
|
||||
|
||||
|
||||
def _parse_nuforc_live_datatables_rows(raw_rows: list) -> list[dict]:
|
||||
"""Parse wpDataTables ``data`` array into normalized row dicts."""
|
||||
def _nuforc_fetch_month_live(yyyymm: str, cookie_jar: Path) -> list[dict]:
|
||||
"""Pull one month of NUFORC sightings via the live wpDataTables AJAX.
|
||||
|
||||
Returns a list of raw row dicts with the fields we care about:
|
||||
id, occurred (YYYY-MM-DD), posted (YYYY-MM-DD), city, state, country,
|
||||
shape_raw, summary, explanation. Empty list on any failure — caller
|
||||
decides whether a failure is fatal.
|
||||
"""
|
||||
from services.fetchers.nuforc_enrichment import _parse_date
|
||||
|
||||
out: list[dict] = []
|
||||
for raw in raw_rows:
|
||||
if not isinstance(raw, list) or len(raw) < 8:
|
||||
continue
|
||||
link_html = str(raw[0] or "")
|
||||
occurred_raw = str(raw[1] or "")
|
||||
city = str(raw[2] or "").strip()
|
||||
state = str(raw[3] or "").strip()
|
||||
country = str(raw[4] or "").strip()
|
||||
shape_raw = (str(raw[5] or "").strip() or "Unknown")
|
||||
summary = str(raw[6] or "").strip()
|
||||
reported_raw = str(raw[7] or "")
|
||||
explanation = str(raw[9] or "").strip() if len(raw) > 9 and raw[9] else ""
|
||||
|
||||
occurred_ymd = _parse_date(occurred_raw)
|
||||
if not occurred_ymd:
|
||||
continue
|
||||
if not city and not state and not country:
|
||||
continue
|
||||
|
||||
id_match = _NUFORC_LIVE_SIGHTING_ID_RE.search(link_html)
|
||||
if id_match:
|
||||
sighting_id = f"NUFORC-{id_match.group(1)}"
|
||||
else:
|
||||
digest = hashlib.sha1(
|
||||
f"{occurred_ymd}|{city}|{state}|{summary}".encode("utf-8", "ignore")
|
||||
).hexdigest()[:12]
|
||||
sighting_id = f"NUFORC-{digest}"
|
||||
|
||||
if summary and len(summary) > 280:
|
||||
summary = summary[:277] + "..."
|
||||
if not summary:
|
||||
summary = "Sighting reported"
|
||||
|
||||
out.append({
|
||||
"id": sighting_id,
|
||||
"occurred": occurred_ymd,
|
||||
"posted": _parse_date(reported_raw) or occurred_ymd,
|
||||
"city": city,
|
||||
"state": state,
|
||||
"country": country,
|
||||
"shape_raw": shape_raw,
|
||||
"summary": summary,
|
||||
"explanation": explanation,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _nuforc_fetch_month_live_requests(yyyymm: str) -> list[dict]:
|
||||
"""Live NUFORC month fetch via requests (Windows-safe when curl is disabled)."""
|
||||
import requests
|
||||
|
||||
index_url = _NUFORC_LIVE_INDEX_URL.format(yyyymm=yyyymm)
|
||||
ajax_url = _NUFORC_LIVE_AJAX_URL.format(yyyymm=yyyymm)
|
||||
headers = {"User-Agent": _nuforc_live_user_agent()}
|
||||
session = requests.Session()
|
||||
session.headers.update(headers)
|
||||
try:
|
||||
index_res = session.get(index_url, timeout=60)
|
||||
except requests.RequestException as e:
|
||||
logger.warning("NUFORC live (requests): index fetch failed for %s: %s", yyyymm, e)
|
||||
return []
|
||||
if index_res.status_code != 200 or not index_res.text:
|
||||
logger.warning(
|
||||
"NUFORC live (requests): index HTTP %s for %s",
|
||||
index_res.status_code,
|
||||
yyyymm,
|
||||
)
|
||||
return []
|
||||
nonce_match = _NUFORC_LIVE_NONCE_RE.search(index_res.text)
|
||||
if not nonce_match:
|
||||
logger.warning("NUFORC live (requests): wdtNonce not found for %s", yyyymm)
|
||||
return []
|
||||
nonce = nonce_match.group(1)
|
||||
post_data = (
|
||||
"draw=1"
|
||||
"&columns%5B0%5D%5Bdata%5D=0&columns%5B0%5D%5Bsearchable%5D=true&columns%5B0%5D%5Borderable%5D=false"
|
||||
"&columns%5B1%5D%5Bdata%5D=1&columns%5B1%5D%5Bsearchable%5D=true&columns%5B1%5D%5Borderable%5D=true"
|
||||
"&order%5B0%5D%5Bcolumn%5D=1&order%5B0%5D%5Bdir%5D=desc"
|
||||
"&start=0&length=-1"
|
||||
"&search%5Bvalue%5D=&search%5Bregex%5D=false"
|
||||
f"&wdtNonce={nonce}"
|
||||
)
|
||||
try:
|
||||
ajax_res = session.post(
|
||||
ajax_url,
|
||||
data=post_data,
|
||||
headers={
|
||||
**headers,
|
||||
"Referer": index_url,
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
},
|
||||
timeout=120,
|
||||
)
|
||||
except requests.RequestException as e:
|
||||
logger.warning("NUFORC live (requests): ajax failed for %s: %s", yyyymm, e)
|
||||
return []
|
||||
if ajax_res.status_code != 200 or not ajax_res.text:
|
||||
logger.warning(
|
||||
"NUFORC live (requests): ajax HTTP %s for %s",
|
||||
ajax_res.status_code,
|
||||
yyyymm,
|
||||
)
|
||||
return []
|
||||
try:
|
||||
payload = ajax_res.json()
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning("NUFORC live (requests): ajax JSON decode failed for %s: %s", yyyymm, e)
|
||||
return []
|
||||
return _parse_nuforc_live_datatables_rows(payload.get("data") or [])
|
||||
|
||||
|
||||
def _nuforc_fetch_month_live_curl(yyyymm: str, cookie_jar: Path) -> list[dict]:
|
||||
"""Pull one month of NUFORC sightings via curl + wpDataTables AJAX."""
|
||||
curl_bin = shutil.which("curl") or "curl"
|
||||
index_url = _NUFORC_LIVE_INDEX_URL.format(yyyymm=yyyymm)
|
||||
ajax_url = _NUFORC_LIVE_AJAX_URL.format(yyyymm=yyyymm)
|
||||
|
||||
if not external_curl_fallback_enabled():
|
||||
logger.warning(
|
||||
"NUFORC live: external curl disabled on Windows for %s; "
|
||||
"set SHADOWBROKER_ENABLE_WINDOWS_CURL_FALLBACK=1 to opt in.",
|
||||
yyyymm,
|
||||
)
|
||||
return []
|
||||
|
||||
# Step 1: GET the month index to capture session cookies + fresh nonce.
|
||||
try:
|
||||
index_res = subprocess.run(
|
||||
@@ -1285,27 +1125,65 @@ def _nuforc_fetch_month_live_curl(yyyymm: str, cookie_jar: Path) -> list[dict]:
|
||||
logger.warning("NUFORC live: ajax JSON decode failed for %s: %s", yyyymm, e)
|
||||
return []
|
||||
|
||||
return _parse_nuforc_live_datatables_rows(payload.get("data") or [])
|
||||
raw_rows = payload.get("data") or []
|
||||
out: list[dict] = []
|
||||
for raw in raw_rows:
|
||||
if not isinstance(raw, list) or len(raw) < 8:
|
||||
continue
|
||||
link_html = str(raw[0] or "")
|
||||
occurred_raw = str(raw[1] or "")
|
||||
city = str(raw[2] or "").strip()
|
||||
state = str(raw[3] or "").strip()
|
||||
country = str(raw[4] or "").strip()
|
||||
shape_raw = (str(raw[5] or "").strip() or "Unknown")
|
||||
summary = str(raw[6] or "").strip()
|
||||
reported_raw = str(raw[7] or "")
|
||||
explanation = str(raw[9] or "").strip() if len(raw) > 9 and raw[9] else ""
|
||||
|
||||
occurred_ymd = _parse_date(occurred_raw)
|
||||
if not occurred_ymd:
|
||||
continue
|
||||
if not city and not state and not country:
|
||||
continue
|
||||
|
||||
def _nuforc_fetch_month_live(yyyymm: str, cookie_jar: Path) -> list[dict]:
|
||||
"""Pull one month of NUFORC sightings via live wpDataTables AJAX."""
|
||||
if external_curl_fallback_enabled():
|
||||
rows = _nuforc_fetch_month_live_curl(yyyymm, cookie_jar)
|
||||
if rows:
|
||||
return rows
|
||||
return _nuforc_fetch_month_live_requests(yyyymm)
|
||||
id_match = _NUFORC_LIVE_SIGHTING_ID_RE.search(link_html)
|
||||
if id_match:
|
||||
sighting_id = f"NUFORC-{id_match.group(1)}"
|
||||
else:
|
||||
digest = hashlib.sha1(
|
||||
f"{occurred_ymd}|{city}|{state}|{summary}".encode("utf-8", "ignore")
|
||||
).hexdigest()[:12]
|
||||
sighting_id = f"NUFORC-{digest}"
|
||||
|
||||
if summary and len(summary) > 280:
|
||||
summary = summary[:277] + "..."
|
||||
if not summary:
|
||||
summary = "Sighting reported"
|
||||
|
||||
out.append({
|
||||
"id": sighting_id,
|
||||
"occurred": occurred_ymd,
|
||||
"posted": _parse_date(reported_raw) or occurred_ymd,
|
||||
"city": city,
|
||||
"state": state,
|
||||
"country": country,
|
||||
"shape_raw": shape_raw,
|
||||
"summary": summary,
|
||||
"explanation": explanation,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _build_recent_uap_sightings() -> list[dict]:
|
||||
"""Build the rolling UAP sightings layer from live NUFORC data.
|
||||
"""Build the rolling 1-year UAP sightings layer from live NUFORC data.
|
||||
|
||||
Hits nuforc.org's public sub-index once per month in the window, drops
|
||||
anything outside the exact day-precision cutoff, dedupes by sighting id,
|
||||
geocodes city+state via the existing location cache, and returns rows
|
||||
keyed to the same schema the frontend already renders.
|
||||
"""
|
||||
cutoff_str = _uap_cutoff_date_str()
|
||||
cutoff_dt = datetime.utcnow() - timedelta(days=_NUFORC_RECENT_DAYS)
|
||||
cutoff_str = cutoff_dt.strftime("%Y-%m-%d")
|
||||
months = _nuforc_months_for_window(_NUFORC_RECENT_DAYS)
|
||||
|
||||
try:
|
||||
@@ -1652,12 +1530,11 @@ def _build_uap_sightings_from_hf_mirror() -> list[dict]:
|
||||
|
||||
@with_retry(max_retries=1, base_delay=5)
|
||||
def fetch_uap_sightings(*, force_refresh: bool = False):
|
||||
"""Fetch rolling-window UAP sightings from live NUFORC.
|
||||
"""Fetch last-year UAP sightings from NUFORC.
|
||||
|
||||
Startup reads the cached snapshot when still within NUFORC_CACHE_TTL_HOURS
|
||||
(default 168h / one week). The weekly scheduler forces a rebuild so every
|
||||
install refreshes the same ~60-day layer without daily load on nuforc.org.
|
||||
Operators can also POST /api/refresh (admin) to pull immediately.
|
||||
Startup reads the cached daily snapshot when it is still fresh. The daily
|
||||
scheduler forces a rebuild so this layer updates once per day instead of
|
||||
churning continuously.
|
||||
"""
|
||||
from services.fetchers._store import is_any_active
|
||||
|
||||
@@ -1690,16 +1567,12 @@ def fetch_uap_sightings(*, force_refresh: bool = False):
|
||||
live_error,
|
||||
)
|
||||
|
||||
if sightings:
|
||||
sightings = _filter_uap_sightings_recent(sightings)
|
||||
|
||||
with _data_lock:
|
||||
latest_data["uap_sightings"] = sightings or []
|
||||
if sightings:
|
||||
_mark_fresh("uap_sightings")
|
||||
return
|
||||
|
||||
# Unreachable legacy Mapbox tilequery path (kept for reference).
|
||||
cutoff = datetime.utcnow() - timedelta(days=_NUFORC_RECENT_DAYS)
|
||||
|
||||
# Query the grid concurrently (up to 8 threads)
|
||||
|
||||
@@ -20,9 +20,17 @@ def _env_flag(name: str) -> str:
|
||||
|
||||
|
||||
def liveuamap_scraper_enabled() -> bool:
|
||||
from services.liveuamap_settings import liveuamap_scraper_enabled as _enabled
|
||||
"""Return whether the Playwright-based LiveUAMap scraper should run.
|
||||
|
||||
return _enabled()
|
||||
It is useful enrichment, but it starts a browser/Node driver and must not be
|
||||
allowed to destabilize Windows local startup.
|
||||
"""
|
||||
setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
|
||||
if setting in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
if setting in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
return os.name != "nt"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -202,17 +210,10 @@ def update_liveuamap():
|
||||
if not is_any_active("global_incidents"):
|
||||
return
|
||||
if not liveuamap_scraper_enabled():
|
||||
from services.liveuamap_settings import liveuamap_requires_ui_opt_in
|
||||
|
||||
if liveuamap_requires_ui_opt_in():
|
||||
logger.info(
|
||||
"Liveuamap scraper disabled: enable Global Incidents in the UI to "
|
||||
"consent, or set SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=1."
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"Liveuamap scraper disabled; set SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=1 to opt in."
|
||||
)
|
||||
logger.info(
|
||||
"Liveuamap scraper disabled for this runtime; set "
|
||||
"SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=1 to opt in."
|
||||
)
|
||||
return
|
||||
logger.info("Running scheduled Liveuamap scraper...")
|
||||
try:
|
||||
|
||||
@@ -188,8 +188,8 @@ def fetch_meshtastic_nodes():
|
||||
callsign = ""
|
||||
|
||||
send_callsign_header = str(
|
||||
_os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "false")
|
||||
).strip().lower() in {"1", "true", "yes", "on"}
|
||||
_os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "true")
|
||||
).strip().lower() not in {"0", "false", "no", "off", ""}
|
||||
|
||||
# Round 7a: outbound_user_agent already includes the per-install handle.
|
||||
# The optional Meshtastic callsign is appended as additional context so
|
||||
|
||||
@@ -9,7 +9,6 @@ import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import urlencode
|
||||
@@ -22,34 +21,23 @@ _prev_probabilities: dict[str, float] = {}
|
||||
_market_cache = TTLCache(maxsize=1, ttl=300)
|
||||
_POLYMARKET_PAGE_DELAY_S = float(os.environ.get("MESH_POLYMARKET_PAGE_DELAY_S", "0.02"))
|
||||
_KALSHI_PAGE_DELAY_S = float(os.environ.get("MESH_KALSHI_PAGE_DELAY_S", "0.08"))
|
||||
_POLYMARKET_PAGE_DELAY_JITTER_S = float(os.environ.get("MESH_POLYMARKET_PAGE_DELAY_JITTER_S", "0.08"))
|
||||
_KALSHI_PAGE_DELAY_JITTER_S = float(os.environ.get("MESH_KALSHI_PAGE_DELAY_JITTER_S", "0.2"))
|
||||
# Random delay before each full Polymarket+Kalshi cycle (decorrelates from other slow-tier jobs).
|
||||
_PRE_FETCH_JITTER_S = float(os.environ.get("PREDICTION_MARKETS_PRE_FETCH_JITTER_S", "90"))
|
||||
# Random pause between finishing Polymarket pagination and starting Kalshi.
|
||||
_PROVIDER_GAP_JITTER_S = float(os.environ.get("PREDICTION_MARKETS_PROVIDER_GAP_JITTER_S", "45"))
|
||||
_provider_pace_lock = threading.Lock()
|
||||
_provider_last_request_at: dict[str, float] = {}
|
||||
|
||||
|
||||
def prediction_markets_fetch_enabled() -> bool:
|
||||
"""Return True when UI opt-in or PREDICTION_MARKETS_ENABLED enables pulls."""
|
||||
from services.prediction_markets_settings import prediction_markets_fetch_enabled as _enabled
|
||||
|
||||
return _enabled()
|
||||
"""Return True only when the operator explicitly opts into Polymarket/Kalshi pulls."""
|
||||
return str(os.environ.get("PREDICTION_MARKETS_ENABLED", "")).strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
}
|
||||
|
||||
|
||||
def _pace_provider(provider: str, min_interval_s: float) -> None:
|
||||
if min_interval_s <= 0:
|
||||
return
|
||||
jitter_s = (
|
||||
_POLYMARKET_PAGE_DELAY_JITTER_S
|
||||
if provider == "polymarket"
|
||||
else _KALSHI_PAGE_DELAY_JITTER_S
|
||||
if provider == "kalshi"
|
||||
else 0.0
|
||||
)
|
||||
min_interval_s += random.uniform(0.0, jitter_s) if jitter_s > 0 else 0.0
|
||||
with _provider_pace_lock:
|
||||
now = time.monotonic()
|
||||
wait_s = min_interval_s - (now - _provider_last_request_at.get(provider, 0.0))
|
||||
@@ -59,24 +47,6 @@ def _pace_provider(provider: str, min_interval_s: float) -> None:
|
||||
_provider_last_request_at[provider] = now
|
||||
|
||||
|
||||
def _apply_pre_fetch_jitter() -> None:
|
||||
if _PRE_FETCH_JITTER_S <= 0:
|
||||
return
|
||||
delay = random.uniform(0.0, _PRE_FETCH_JITTER_S)
|
||||
if delay >= 1.0:
|
||||
logger.debug("Prediction markets: pre-fetch jitter %.1fs", delay)
|
||||
time.sleep(delay)
|
||||
|
||||
|
||||
def _apply_provider_gap_jitter() -> None:
|
||||
if _PROVIDER_GAP_JITTER_S <= 0:
|
||||
return
|
||||
delay = random.uniform(0.0, _PROVIDER_GAP_JITTER_S)
|
||||
if delay >= 1.0:
|
||||
logger.debug("Prediction markets: provider gap jitter %.1fs", delay)
|
||||
time.sleep(delay)
|
||||
|
||||
|
||||
def _finite_or_none(value):
|
||||
try:
|
||||
n = float(value)
|
||||
@@ -780,9 +750,7 @@ def _merge_markets(poly_events: list[dict], kalshi_events: list[dict]) -> list[d
|
||||
@cached(_market_cache)
|
||||
def fetch_prediction_markets_raw() -> list[dict]:
|
||||
"""Fetch and merge prediction markets from both sources. Cached 5 min."""
|
||||
_apply_pre_fetch_jitter()
|
||||
poly = _fetch_polymarket_events()
|
||||
_apply_provider_gap_jitter()
|
||||
kalshi = _fetch_kalshi_events()
|
||||
merged = _merge_markets(poly, kalshi)
|
||||
logger.info(
|
||||
|
||||
@@ -30,6 +30,8 @@ _AIRPORTS_URL = "https://vrs-standing-data.adsb.lol/airports.csv.gz"
|
||||
_REFRESH_INTERVAL_S = 5 * 24 * 3600
|
||||
_HTTP_TIMEOUT_S = 60
|
||||
|
||||
from services.network_utils import DEFAULT_USER_AGENT as _USER_AGENT
|
||||
|
||||
_lock = threading.RLock()
|
||||
_routes_by_callsign: dict[str, dict[str, Any]] = {}
|
||||
_airports_by_icao: dict[str, dict[str, Any]] = {}
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import os
|
||||
import requests
|
||||
import logging
|
||||
import zipfile
|
||||
@@ -21,50 +20,6 @@ logger = logging.getLogger(__name__)
|
||||
# Cache Frontline data for 30 minutes, it doesn't move that fast
|
||||
frontline_cache = TTLCache(maxsize=1, ttl=1800)
|
||||
|
||||
_DEFAULT_DEEPSTATE_MIRROR_REPO = "cyterat/deepstate-map-data"
|
||||
|
||||
|
||||
def _deepstate_mirror_ref() -> tuple[str, str]:
|
||||
"""Return (github_repo_slug, git_ref) for the DeepState mirror.
|
||||
|
||||
When ``DEEPSTATE_MIRROR_COMMIT`` is set, ingest is pinned to that immutable
|
||||
SHA instead of following the mutable ``main`` branch (#362).
|
||||
"""
|
||||
repo = (os.environ.get("DEEPSTATE_MIRROR_REPO") or _DEFAULT_DEEPSTATE_MIRROR_REPO).strip()
|
||||
if repo.count("/") != 1:
|
||||
repo = _DEFAULT_DEEPSTATE_MIRROR_REPO
|
||||
commit = (os.environ.get("DEEPSTATE_MIRROR_COMMIT") or "").strip()
|
||||
ref = commit if commit else "main"
|
||||
return repo, ref
|
||||
|
||||
|
||||
def _latest_deepstate_geo_path(tree_items: list) -> str | None:
|
||||
geo_files = [
|
||||
item["path"]
|
||||
for item in tree_items
|
||||
if isinstance(item, dict)
|
||||
and str(item.get("path", "")).startswith("data/deepstatemap_data_")
|
||||
and str(item.get("path", "")).endswith(".geojson")
|
||||
]
|
||||
return sorted(geo_files)[-1] if geo_files else None
|
||||
|
||||
|
||||
def _annotate_deepstate_geojson(data: dict) -> dict:
|
||||
name_map = {
|
||||
0: "Russian-occupied areas",
|
||||
1: "Russian advance",
|
||||
2: "Liberated area",
|
||||
3: "Russian-occupied areas", # Crimea / LPR / DPR
|
||||
4: "Directions of UA attacks",
|
||||
}
|
||||
if "features" in data:
|
||||
for idx, feature in enumerate(data["features"]):
|
||||
if "properties" not in feature or feature["properties"] is None:
|
||||
feature["properties"] = {}
|
||||
feature["properties"]["name"] = name_map.get(idx, "Russian-occupied areas")
|
||||
feature["properties"]["zone_id"] = idx
|
||||
return data
|
||||
|
||||
|
||||
@cached(frontline_cache)
|
||||
def fetch_ukraine_frontlines():
|
||||
@@ -72,34 +27,67 @@ def fetch_ukraine_frontlines():
|
||||
Fetches the latest GeoJSON data representing the Ukraine frontline.
|
||||
We use the cyterat/deepstate-map-data github mirror since the public API is locked.
|
||||
"""
|
||||
repo, ref = _deepstate_mirror_ref()
|
||||
try:
|
||||
logger.info("Fetching DeepStateMap from GitHub mirror (%s @ %s)...", repo, ref)
|
||||
logger.info("Fetching DeepStateMap from GitHub mirror...")
|
||||
|
||||
tree_url = f"https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1"
|
||||
# First, query the repo tree to find the latest file name
|
||||
tree_url = (
|
||||
"https://api.github.com/repos/cyterat/deepstate-map-data/git/trees/main?recursive=1"
|
||||
)
|
||||
res_tree = requests.get(tree_url, timeout=10)
|
||||
|
||||
if res_tree.status_code == 200:
|
||||
latest_file = _latest_deepstate_geo_path(res_tree.json().get("tree", []))
|
||||
if latest_file:
|
||||
raw_url = f"https://raw.githubusercontent.com/{repo}/{ref}/{latest_file}"
|
||||
logger.info("Downloading DeepStateMap: %s", raw_url)
|
||||
tree_data = res_tree.json().get("tree", [])
|
||||
# Filter for geojson files in data folder
|
||||
geo_files = [
|
||||
item["path"]
|
||||
for item in tree_data
|
||||
if item["path"].startswith("data/deepstatemap_data_")
|
||||
and item["path"].endswith(".geojson")
|
||||
]
|
||||
|
||||
if geo_files:
|
||||
# Get the alphabetically latest file (since it's named with YYYYMMDD)
|
||||
latest_file = sorted(geo_files)[-1]
|
||||
|
||||
raw_url = f"https://raw.githubusercontent.com/cyterat/deepstate-map-data/main/{latest_file}"
|
||||
logger.info(f"Downloading latest DeepStateMap: {raw_url}")
|
||||
|
||||
res_geo = requests.get(raw_url, timeout=20)
|
||||
if res_geo.status_code == 200:
|
||||
return _annotate_deepstate_geojson(res_geo.json())
|
||||
logger.error(
|
||||
"Failed to fetch parsed Github Raw GeoJSON: %s", res_geo.status_code
|
||||
)
|
||||
else:
|
||||
logger.error("No deepstatemap_data_*.geojson files in mirror tree at %s", ref)
|
||||
data = res_geo.json()
|
||||
|
||||
# The Cyterat GitHub mirror strips all properties and just provides a raw array of Feature polygons.
|
||||
# Based on DeepStateMap's frontend mapping, the array index corresponds to the zone type:
|
||||
# 0: Russian-occupied areas
|
||||
# 1: Russian advance
|
||||
# 2: Liberated area
|
||||
# 3: Uncontested/Crimea (often folded into occupied)
|
||||
name_map = {
|
||||
0: "Russian-occupied areas",
|
||||
1: "Russian advance",
|
||||
2: "Liberated area",
|
||||
3: "Russian-occupied areas", # Crimea / LPR / DPR
|
||||
4: "Directions of UA attacks",
|
||||
}
|
||||
|
||||
if "features" in data:
|
||||
for idx, feature in enumerate(data["features"]):
|
||||
if "properties" not in feature or feature["properties"] is None:
|
||||
feature["properties"] = {}
|
||||
|
||||
feature["properties"]["name"] = name_map.get(
|
||||
idx, "Russian-occupied areas"
|
||||
)
|
||||
feature["properties"]["zone_id"] = idx
|
||||
|
||||
return data
|
||||
else:
|
||||
logger.error(
|
||||
f"Failed to fetch parsed Github Raw GeoJSON: {res_geo.status_code}"
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
"Failed to fetch Github tree for Deepstatemap (%s @ %s): %s",
|
||||
repo,
|
||||
ref,
|
||||
res_tree.status_code,
|
||||
)
|
||||
logger.error(f"Failed to fetch Github Tree for Deepstatemap: {res_tree.status_code}")
|
||||
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
|
||||
logger.error(f"Error fetching DeepStateMap: {e}")
|
||||
return None
|
||||
|
||||
@@ -1,20 +1,14 @@
|
||||
"""Function Keys — anonymous credential scaffolding.
|
||||
"""Function Keys — anonymous citizenship proof.
|
||||
|
||||
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.4,
|
||||
``infonet-economy/BRAINDUMP.md`` §11 item 9.
|
||||
|
||||
A citizen should eventually be able to prove "I am a UBI-eligible
|
||||
Infonet citizen" to a real-world operator (food bank, community
|
||||
service) **without revealing their Infonet identity**. The current
|
||||
Python implementation wires the accounting, nullifier, receipt, and
|
||||
operator flows, but its HMAC challenge-response is a placeholder for
|
||||
integration tests. It is not a production anonymous or zero-knowledge
|
||||
citizenship proof until blind signatures or anonymous credentials are
|
||||
selected and wired.
|
||||
|
||||
The naive approach (scramble a public key, record each redemption on
|
||||
chain) leaks identity through metadata correlation (time, location,
|
||||
operator, frequency).
|
||||
A citizen should be able to prove "I am a UBI-eligible Infonet
|
||||
citizen" to a real-world operator (food bank, community service)
|
||||
**without revealing their Infonet identity**. The naive approach
|
||||
(scramble a public key, record each redemption on chain) leaks
|
||||
identity through metadata correlation (time, location, operator,
|
||||
frequency).
|
||||
|
||||
The full design has six pieces; five are implemented in pure Python
|
||||
here. The remaining piece — issuance via blind signatures or
|
||||
@@ -33,8 +27,7 @@ Pieces:
|
||||
operator: tracked via ``NullifierTracker``.
|
||||
3. **Challenge-response** (`challenge_response.py`) — operator
|
||||
issues a fresh nonce, key-holder signs with the Function Key's
|
||||
secret. This is HMAC placeholder plumbing for screenshot/replay
|
||||
resistance, not the final anonymous credential proof.
|
||||
secret. Prevents screenshot attacks, key sharing, replay.
|
||||
4. **Two-phase commit receipts** (`receipt.py`) — Phase 1
|
||||
verification receipt (operator-signed, day-level date NOT
|
||||
timestamp, no node_id). Phase 2 fulfillment receipt (citizen
|
||||
|
||||
@@ -32,14 +32,14 @@ logger = logging.getLogger(__name__)
|
||||
_REFRESH_SECONDS = 24 * 3600
|
||||
kiwisdr_cache: TTLCache = TTLCache(maxsize=1, ttl=_REFRESH_SECONDS)
|
||||
|
||||
_SOURCE_URL_HTTP = "http://rx.linkfanel.net/kiwisdr_com.js"
|
||||
_SOURCE_URL_HTTPS = "https://rx.linkfanel.net/kiwisdr_com.js"
|
||||
_SOURCE_URL = "http://rx.linkfanel.net/kiwisdr_com.js"
|
||||
_CACHE_FILE = Path(__file__).resolve().parent.parent / "data" / "kiwisdr_cache.json"
|
||||
# Bundled fallback — shipped with the codebase so the KiwiSDR layer always
|
||||
# has something to render even when the upstream is unreachable, returns
|
||||
# garbage, or appears to have been tampered with. Issue #206 / #364: try HTTPS
|
||||
# first, then HTTP; we still validate shape and fall back to this bundle if the
|
||||
# payload does not look right.
|
||||
# garbage, or appears to have been tampered with. Issue #206: the upstream
|
||||
# only speaks HTTP, so we can't rely on TLS for integrity — instead we
|
||||
# validate the response's shape and fall back to this bundle if it doesn't
|
||||
# look right.
|
||||
_BUNDLED_FALLBACK = Path(__file__).resolve().parent.parent / "data" / "kiwisdr_directory.json"
|
||||
|
||||
# Minimum number of receivers we expect from a healthy upstream response.
|
||||
@@ -184,29 +184,6 @@ def _validate_fetched_nodes(nodes: list[dict]) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def _fetch_mirror_payload_text() -> str | None:
|
||||
"""Try HTTPS first, then HTTP. Shape validation still applies (#364)."""
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
last_error: Exception | None = None
|
||||
for url in (_SOURCE_URL_HTTPS, _SOURCE_URL_HTTP):
|
||||
try:
|
||||
res = fetch_with_curl(url, timeout=20)
|
||||
if res and res.status_code == 200:
|
||||
if url == _SOURCE_URL_HTTP:
|
||||
logger.info(
|
||||
"KiwiSDR: HTTPS mirror unavailable; using HTTP with shape validation"
|
||||
)
|
||||
return res.text
|
||||
last_error = RuntimeError(f"HTTP {getattr(res, 'status_code', 'unknown')}")
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
logger.debug("KiwiSDR mirror fetch failed for %s: %s", url, e)
|
||||
if last_error is not None:
|
||||
logger.warning("KiwiSDR mirror fetch failed: %s", last_error)
|
||||
return None
|
||||
|
||||
|
||||
def _load_bundled_fallback() -> list[dict]:
|
||||
"""Last-resort directory shipped with the codebase. Always returns a
|
||||
list (may be empty if the bundle is missing in older deployments)."""
|
||||
@@ -225,8 +202,9 @@ def _load_bundled_fallback() -> list[dict]:
|
||||
def fetch_kiwisdr_nodes() -> list[dict]:
|
||||
"""Return the KiwiSDR receiver list, refreshed at most once per day.
|
||||
|
||||
Layered fallback (issue #206 / #364 — HTTPS first, HTTP fallback, plus
|
||||
content validation + bundled static directory):
|
||||
Layered fallback (issue #206 — upstream is HTTP-only, so we defend with
|
||||
content validation + bundled static directory rather than trying to
|
||||
upgrade the transport):
|
||||
|
||||
1. In-memory cache (handled by @cached on this function)
|
||||
2. On-disk cache if <24h old
|
||||
@@ -238,6 +216,8 @@ def fetch_kiwisdr_nodes() -> list[dict]:
|
||||
tampered upstream returning garbage is caught by _validate_fetched_nodes()
|
||||
and falls through to whatever previously-trusted snapshot we have.
|
||||
"""
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
# 1. Trust on-disk cache if fresh.
|
||||
cached_nodes = _load_disk_cache()
|
||||
if cached_nodes is not None:
|
||||
@@ -250,12 +230,14 @@ def fetch_kiwisdr_nodes() -> list[dict]:
|
||||
fresh_nodes: list[dict] = []
|
||||
fetch_succeeded = False
|
||||
try:
|
||||
body = _fetch_mirror_payload_text()
|
||||
if body:
|
||||
fresh_nodes = _parse_mirror_payload(body)
|
||||
res = fetch_with_curl(_SOURCE_URL, timeout=20)
|
||||
if res and res.status_code == 200:
|
||||
fresh_nodes = _parse_mirror_payload(res.text)
|
||||
fetch_succeeded = True
|
||||
else:
|
||||
logger.warning("KiwiSDR fetch returned no usable mirror payload")
|
||||
logger.warning(
|
||||
f"KiwiSDR fetch returned HTTP {res.status_code if res else 'no response'}"
|
||||
)
|
||||
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
|
||||
logger.warning(f"KiwiSDR fetch exception: {e}")
|
||||
|
||||
|
||||
@@ -27,15 +27,8 @@ def fetch_liveuamap():
|
||||
browser = p.chromium.launch(
|
||||
headless=True, args=["--disable-blink-features=AutomationControlled"]
|
||||
)
|
||||
from services.network_utils import outbound_user_agent
|
||||
|
||||
# Per-install handle (no shared Shadowbroker product token). Stealth remains
|
||||
# for Turnstile; see docs/OUTBOUND_DATA.md #348.
|
||||
playwright_ua = (
|
||||
f"Mozilla/5.0 (compatible; {outbound_user_agent('liveuamap')})"
|
||||
)
|
||||
context = browser.new_context(
|
||||
user_agent=playwright_ua,
|
||||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
viewport={"width": 1920, "height": 1080},
|
||||
color_scheme="dark",
|
||||
)
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
"""LiveUAMap Playwright scraper opt-in (#348) — UI consent on Windows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_OPT_IN_FILE = Path(__file__).resolve().parent.parent / "data" / "liveuamap_scraper_opt_in.json"
|
||||
_OPT_IN_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _env_flag(name: str) -> str:
|
||||
return str(os.getenv(name, "")).strip().lower()
|
||||
|
||||
|
||||
def liveuamap_requires_ui_opt_in() -> bool:
|
||||
"""Windows local installs need explicit consent before Playwright contacts LiveUAMap."""
|
||||
return os.name == "nt"
|
||||
|
||||
|
||||
def get_liveuamap_ui_opt_in() -> bool:
|
||||
if not _OPT_IN_FILE.exists():
|
||||
return False
|
||||
try:
|
||||
payload = json.loads(_OPT_IN_FILE.read_text(encoding="utf-8"))
|
||||
return bool(payload.get("opted_in"))
|
||||
except (OSError, json.JSONDecodeError, TypeError) as e:
|
||||
logger.warning("LiveUAMap opt-in file unreadable: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
def set_liveuamap_ui_opt_in(opted_in: bool) -> None:
|
||||
_OPT_IN_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
with _OPT_IN_LOCK:
|
||||
_OPT_IN_FILE.write_text(
|
||||
json.dumps({"opted_in": bool(opted_in)}, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def liveuamap_scraper_enabled() -> bool:
|
||||
"""Whether the Playwright LiveUAMap scraper may run on this backend."""
|
||||
setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
|
||||
if setting in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
if setting in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
if not liveuamap_requires_ui_opt_in():
|
||||
return True
|
||||
return get_liveuamap_ui_opt_in()
|
||||
|
||||
|
||||
def liveuamap_scraper_status() -> dict[str, Any]:
|
||||
setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
|
||||
env_override = None
|
||||
if setting in {"1", "true", "yes", "on"}:
|
||||
env_override = "on"
|
||||
elif setting in {"0", "false", "no", "off"}:
|
||||
env_override = "off"
|
||||
ui_opted_in = get_liveuamap_ui_opt_in()
|
||||
requires = liveuamap_requires_ui_opt_in()
|
||||
return {
|
||||
"platform_requires_opt_in": requires,
|
||||
"ui_opted_in": ui_opted_in,
|
||||
"scraper_enabled": liveuamap_scraper_enabled(),
|
||||
"env_override": env_override,
|
||||
}
|
||||
@@ -33,9 +33,8 @@ Each event contains:
|
||||
|
||||
Persistence: JSON file at backend/data/infonet.json
|
||||
|
||||
Encrypted gate chat events are private-chain ciphertext records. They are
|
||||
excluded from public read surfaces and replicated only over private Infonet
|
||||
transports.
|
||||
Encrypted gate chat events are intentionally kept off the public chain and
|
||||
persisted separately via GateMessageStore.
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -65,8 +64,6 @@ from services.mesh.mesh_schema import (
|
||||
ACTIVE_PUBLIC_LEDGER_EVENT_TYPES,
|
||||
PUBLIC_LEDGER_EVENT_TYPES,
|
||||
validate_event_payload,
|
||||
validate_private_dm_ledger_payload,
|
||||
validate_private_gate_ledger_payload,
|
||||
validate_protocol_fields,
|
||||
validate_public_ledger_payload,
|
||||
)
|
||||
@@ -130,12 +127,6 @@ GATE_SEGMENT_MAX_COMPRESSED_BYTES = max(
|
||||
int(os.environ.get("MESH_GATE_SEGMENT_MAX_COMPRESSED_BYTES", str(2 * 1024 * 1024)) or str(2 * 1024 * 1024)),
|
||||
)
|
||||
GATE_SEGMENT_STORAGE_VERSION = 1
|
||||
DM_HASHCHAIN_SPOOL_LIMIT = max(1, int(os.environ.get("MESH_DM_HASHCHAIN_SPOOL_LIMIT", "2") or "2"))
|
||||
DM_HASHCHAIN_SPOOL_SENDER_LIMIT = max(
|
||||
1,
|
||||
int(os.environ.get("MESH_DM_HASHCHAIN_SPOOL_SENDER_LIMIT", "1") or "1"),
|
||||
)
|
||||
DM_HASHCHAIN_SPOOL_TTL_S = max(60, int(os.environ.get("MESH_DM_HASHCHAIN_SPOOL_TTL_S", "3600") or "3600"))
|
||||
_PUBLIC_EVENT_APPEND_HOOKS: list[Any] = []
|
||||
_PUBLIC_EVENT_APPEND_HOOKS_LOCK = threading.Lock()
|
||||
|
||||
@@ -349,32 +340,6 @@ def _private_gate_event_id(
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def _private_gate_signature_payload_variants(gate_id: str, event: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
payload = _private_gate_signature_payload(gate_id, event)
|
||||
variants: list[dict[str, Any]] = [payload]
|
||||
event_payload = event.get("payload") if isinstance(event.get("payload"), dict) else {}
|
||||
reply_to = str(event_payload.get("reply_to", "") or "").strip()
|
||||
if reply_to:
|
||||
variants.append(_private_gate_signature_payload(gate_id, event, include_reply_to=False))
|
||||
if "epoch" in payload:
|
||||
no_epoch = dict(payload)
|
||||
no_epoch.pop("epoch", None)
|
||||
variants.append(no_epoch)
|
||||
if reply_to:
|
||||
no_epoch_no_reply = _private_gate_signature_payload(gate_id, event, include_reply_to=False)
|
||||
no_epoch_no_reply.pop("epoch", None)
|
||||
variants.append(no_epoch_no_reply)
|
||||
deduped: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for variant in variants:
|
||||
material = json.dumps(variant, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
||||
if material in seen:
|
||||
continue
|
||||
seen.add(material)
|
||||
deduped.append(variant)
|
||||
return deduped
|
||||
|
||||
|
||||
def _sanitize_private_gate_event(gate_id: str, event: dict[str, Any]) -> dict[str, Any]:
|
||||
payload = event.get("payload") if isinstance(event.get("payload"), dict) else {}
|
||||
sanitized = {
|
||||
@@ -1603,18 +1568,11 @@ class Infonet:
|
||||
def _rebuild_state(self) -> None:
|
||||
self.event_index = {}
|
||||
self.node_sequences = {}
|
||||
# Keep private signed-write replay domains that are not represented
|
||||
# on-chain, but rebuild the gate_message sequence domain from chain
|
||||
# events so reloads/fork application do not mix it with public
|
||||
# per-node message sequences.
|
||||
preserved_domains = {}
|
||||
if isinstance(getattr(self, "sequence_domains", None), dict):
|
||||
preserved_domains = {
|
||||
key: value
|
||||
for key, value in self.sequence_domains.items()
|
||||
if not str(key or "").endswith("|gate_message")
|
||||
}
|
||||
self.sequence_domains = dict(preserved_domains)
|
||||
# Keep private signed-write replay domains across public-chain
|
||||
# rebuilds; these domains protect local side effects that are not
|
||||
# represented as public Infonet events.
|
||||
if not isinstance(getattr(self, "sequence_domains", None), dict):
|
||||
self.sequence_domains = {}
|
||||
self.public_key_bindings = {}
|
||||
self.revocations = {}
|
||||
self._replay_filter = ReplayFilter()
|
||||
@@ -1626,12 +1584,9 @@ class Infonet:
|
||||
node_id = evt.get("node_id", "")
|
||||
sequence = _safe_int(evt.get("sequence", 0) or 0, 0)
|
||||
if node_id and sequence:
|
||||
sequence_table, sequence_key = self._sequence_table_for_event(
|
||||
evt.get("event_type", ""), node_id
|
||||
)
|
||||
last = sequence_table.get(sequence_key, 0)
|
||||
last = self.node_sequences.get(node_id, 0)
|
||||
if sequence > last:
|
||||
sequence_table[sequence_key] = sequence
|
||||
self.node_sequences[node_id] = sequence
|
||||
public_key = str(evt.get("public_key", "") or "")
|
||||
if public_key and node_id:
|
||||
existing = self.public_key_bindings.get(public_key)
|
||||
@@ -1943,295 +1898,6 @@ class Infonet:
|
||||
self._save()
|
||||
return True, "ok"
|
||||
|
||||
def _sequence_table_for_event(self, event_type: str, node_id: str) -> tuple[dict[str, int], str]:
|
||||
normalized = str(event_type or "").strip().lower()
|
||||
if normalized == "gate_message":
|
||||
return self.sequence_domains, f"{node_id}|gate_message"
|
||||
if normalized == "dm_message":
|
||||
return self.sequence_domains, f"{node_id}|dm_message"
|
||||
return self.node_sequences, node_id
|
||||
|
||||
def _dm_spool_target_key(self, payload: dict[str, Any]) -> tuple[str, str]:
|
||||
delivery_class = str(payload.get("delivery_class", "") or "").strip().lower()
|
||||
if delivery_class == "shared":
|
||||
key = str(payload.get("recipient_token", "") or "").strip()
|
||||
else:
|
||||
key = str(payload.get("recipient_id", "") or "").strip()
|
||||
return delivery_class, key
|
||||
|
||||
def _dm_spool_active_counts(
|
||||
self,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
sender_id: str = "",
|
||||
now: float | None = None,
|
||||
) -> tuple[int, int]:
|
||||
delivery_class, key = self._dm_spool_target_key(payload)
|
||||
if not key:
|
||||
return 0, 0
|
||||
sender_id = str(sender_id or "").strip()
|
||||
current = time.time() if now is None else float(now)
|
||||
total_count = 0
|
||||
sender_count = 0
|
||||
for evt in reversed(self.events):
|
||||
if evt.get("event_type") != "dm_message":
|
||||
continue
|
||||
evt_payload = evt.get("payload") if isinstance(evt.get("payload"), dict) else {}
|
||||
evt_delivery_class, evt_key = self._dm_spool_target_key(evt_payload)
|
||||
if evt_delivery_class != delivery_class:
|
||||
continue
|
||||
if evt_key != key:
|
||||
continue
|
||||
evt_ts = float(evt_payload.get("timestamp", evt.get("timestamp", 0)) or 0)
|
||||
if evt_ts > 0 and current - evt_ts > DM_HASHCHAIN_SPOOL_TTL_S:
|
||||
continue
|
||||
total_count += 1
|
||||
if sender_id and str(evt.get("node_id", "") or "").strip() == sender_id:
|
||||
sender_count += 1
|
||||
if total_count >= DM_HASHCHAIN_SPOOL_LIMIT and (
|
||||
not sender_id or sender_count >= DM_HASHCHAIN_SPOOL_SENDER_LIMIT
|
||||
):
|
||||
break
|
||||
return total_count, sender_count
|
||||
|
||||
def _dm_spool_active_count(self, payload: dict[str, Any], *, now: float | None = None) -> int:
|
||||
total_count, _sender_count = self._dm_spool_active_counts(payload, now=now)
|
||||
return total_count
|
||||
|
||||
def append_private_dm_message(
|
||||
self,
|
||||
*,
|
||||
node_id: str,
|
||||
payload: dict,
|
||||
signature: str,
|
||||
sequence: int,
|
||||
public_key: str,
|
||||
public_key_algo: str,
|
||||
protocol_version: str = "",
|
||||
timestamp: float = 0,
|
||||
) -> dict:
|
||||
"""Append an encrypted DM dead-drop message to the private Infonet ledger.
|
||||
|
||||
The event is a small offline spool, capped per mailbox target, so the
|
||||
hashchain can carry a couple of sealed DMs without becoming an
|
||||
unbounded global mailbox.
|
||||
"""
|
||||
event_type = "dm_message"
|
||||
if sequence <= 0:
|
||||
raise ValueError("sequence is required and must be > 0")
|
||||
sequence_table, sequence_key = self._sequence_table_for_event(event_type, node_id)
|
||||
last = sequence_table.get(sequence_key, 0)
|
||||
if sequence <= last:
|
||||
raise ValueError(f"Replay detected: sequence {sequence} <= last {last}")
|
||||
|
||||
raw_payload = dict(payload or {})
|
||||
if "message" in raw_payload or "plaintext" in raw_payload or "_local_plaintext" in raw_payload:
|
||||
raise ValueError("private DM ledger payload must not contain plaintext")
|
||||
if str(raw_payload.get("transport_lock", "") or "").strip().lower() != "private_strong":
|
||||
raise ValueError("DM hashchain spool requires private_strong transport_lock")
|
||||
|
||||
payload = normalize_payload(event_type, raw_payload)
|
||||
ok, reason = validate_private_dm_ledger_payload(payload)
|
||||
if not ok:
|
||||
raise ValueError(reason)
|
||||
total_count, sender_count = self._dm_spool_active_counts(payload, sender_id=node_id)
|
||||
if sender_count >= DM_HASHCHAIN_SPOOL_SENDER_LIMIT:
|
||||
raise ValueError("DM hashchain sender spool full for recipient")
|
||||
if total_count >= DM_HASHCHAIN_SPOOL_LIMIT:
|
||||
raise ValueError("DM hashchain spool full for recipient")
|
||||
|
||||
payload_json = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
||||
if len(payload_json.encode("utf-8")) > MAX_PAYLOAD_BYTES:
|
||||
raise ValueError("payload exceeds max size")
|
||||
|
||||
protocol_version = str(protocol_version or PROTOCOL_VERSION)
|
||||
ok, reason = validate_protocol_fields(protocol_version, NETWORK_ID)
|
||||
if not ok:
|
||||
raise ValueError(reason)
|
||||
|
||||
if not (signature and public_key and public_key_algo):
|
||||
raise ValueError("Missing signature fields")
|
||||
algo = parse_public_key_algo(public_key_algo)
|
||||
if not algo:
|
||||
raise ValueError("Unsupported public_key_algo")
|
||||
if not verify_node_binding(node_id, public_key):
|
||||
raise ValueError("node_id mismatch")
|
||||
bound, bind_reason = self._bind_public_key(public_key, node_id)
|
||||
if not bound:
|
||||
raise ValueError(bind_reason)
|
||||
sig_payload = build_signature_payload(
|
||||
event_type=event_type,
|
||||
node_id=node_id,
|
||||
sequence=sequence,
|
||||
payload=payload,
|
||||
)
|
||||
if not verify_signature(
|
||||
public_key_b64=public_key,
|
||||
public_key_algo=public_key_algo,
|
||||
signature_hex=signature,
|
||||
payload=sig_payload,
|
||||
):
|
||||
raise ValueError("Invalid signature")
|
||||
|
||||
revoked, _info = self._revocation_status(public_key)
|
||||
if revoked:
|
||||
raise ValueError("public key is revoked")
|
||||
|
||||
event = ChainEvent(
|
||||
prev_hash=self.head_hash,
|
||||
event_type=event_type,
|
||||
node_id=node_id,
|
||||
payload=payload,
|
||||
timestamp=float(timestamp or time.time()),
|
||||
sequence=sequence,
|
||||
signature=signature,
|
||||
public_key=public_key,
|
||||
public_key_algo=public_key_algo,
|
||||
protocol_version=protocol_version,
|
||||
)
|
||||
event_dict = event.to_dict()
|
||||
self._write_wal(event_dict)
|
||||
self.events.append(event_dict)
|
||||
self.event_index[event.event_id] = len(self.events) - 1
|
||||
self.head_hash = event.event_id
|
||||
sequence_table[sequence_key] = sequence
|
||||
self._replay_filter.add(event.event_id)
|
||||
self._invalidate_merkle_cache()
|
||||
self._update_counters_for_event(event_dict)
|
||||
self._save()
|
||||
|
||||
try:
|
||||
from services.mesh.mesh_rns import rns_bridge
|
||||
|
||||
rns_bridge.publish_event(event_dict)
|
||||
except Exception:
|
||||
pass
|
||||
_notify_public_event_append_hooks(event_dict)
|
||||
logger.info(
|
||||
f"Infonet append [dm_message] by {_redact_node(node_id)} seq={sequence} "
|
||||
f"id={event.event_id[:16]}..."
|
||||
)
|
||||
return event_dict
|
||||
|
||||
def append_private_gate_message(
|
||||
self,
|
||||
*,
|
||||
node_id: str,
|
||||
payload: dict,
|
||||
signature: str,
|
||||
sequence: int,
|
||||
public_key: str,
|
||||
public_key_algo: str,
|
||||
protocol_version: str = "",
|
||||
timestamp: float = 0,
|
||||
) -> dict:
|
||||
"""Append an encrypted gate message to the private Infonet ledger.
|
||||
|
||||
Gate messages use their own sequence domain so a gate post cannot
|
||||
consume or replay-block the author's public broadcast sequence.
|
||||
"""
|
||||
event_type = "gate_message"
|
||||
if sequence <= 0:
|
||||
raise ValueError("sequence is required and must be > 0")
|
||||
sequence_table, sequence_key = self._sequence_table_for_event(event_type, node_id)
|
||||
last = sequence_table.get(sequence_key, 0)
|
||||
if sequence <= last:
|
||||
raise ValueError(f"Replay detected: sequence {sequence} <= last {last}")
|
||||
|
||||
raw_payload = dict(payload or {})
|
||||
if "message" in raw_payload or "_local_plaintext" in raw_payload or "_local_reply_to" in raw_payload:
|
||||
raise ValueError("private gate ledger payload must not contain plaintext")
|
||||
if str(raw_payload.get("transport_lock", "") or "").strip().lower() != "private_strong":
|
||||
raise ValueError("gate messages require private_strong transport_lock")
|
||||
|
||||
payload = normalize_payload(event_type, raw_payload)
|
||||
ok, reason = validate_private_gate_ledger_payload(payload)
|
||||
if not ok:
|
||||
raise ValueError(reason)
|
||||
|
||||
payload_json = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
||||
if len(payload_json.encode("utf-8")) > MAX_PAYLOAD_BYTES:
|
||||
raise ValueError("payload exceeds max size")
|
||||
|
||||
protocol_version = str(protocol_version or PROTOCOL_VERSION)
|
||||
ok, reason = validate_protocol_fields(protocol_version, NETWORK_ID)
|
||||
if not ok:
|
||||
raise ValueError(reason)
|
||||
|
||||
if not (signature and public_key and public_key_algo):
|
||||
raise ValueError("Missing signature fields")
|
||||
algo = parse_public_key_algo(public_key_algo)
|
||||
if not algo:
|
||||
raise ValueError("Unsupported public_key_algo")
|
||||
if not verify_node_binding(node_id, public_key):
|
||||
raise ValueError("node_id mismatch")
|
||||
bound, bind_reason = self._bind_public_key(public_key, node_id)
|
||||
if not bound:
|
||||
raise ValueError(bind_reason)
|
||||
event_for_signature = {"payload": payload}
|
||||
signature_ok = False
|
||||
for signature_payload in _private_gate_signature_payload_variants(
|
||||
str(payload.get("gate", "") or ""),
|
||||
event_for_signature,
|
||||
):
|
||||
sig_payload = build_signature_payload(
|
||||
event_type=event_type,
|
||||
node_id=node_id,
|
||||
sequence=sequence,
|
||||
payload=signature_payload,
|
||||
)
|
||||
if verify_signature(
|
||||
public_key_b64=public_key,
|
||||
public_key_algo=public_key_algo,
|
||||
signature_hex=signature,
|
||||
payload=sig_payload,
|
||||
):
|
||||
signature_ok = True
|
||||
break
|
||||
if not signature_ok:
|
||||
raise ValueError("Invalid signature")
|
||||
|
||||
revoked, _info = self._revocation_status(public_key)
|
||||
if revoked:
|
||||
raise ValueError("public key is revoked")
|
||||
|
||||
event = ChainEvent(
|
||||
prev_hash=self.head_hash,
|
||||
event_type=event_type,
|
||||
node_id=node_id,
|
||||
payload=payload,
|
||||
timestamp=float(timestamp or time.time()),
|
||||
sequence=sequence,
|
||||
signature=signature,
|
||||
public_key=public_key,
|
||||
public_key_algo=public_key_algo,
|
||||
protocol_version=protocol_version,
|
||||
)
|
||||
event_dict = event.to_dict()
|
||||
self._write_wal(event_dict)
|
||||
self.events.append(event_dict)
|
||||
self.event_index[event.event_id] = len(self.events) - 1
|
||||
self.head_hash = event.event_id
|
||||
sequence_table[sequence_key] = sequence
|
||||
self._replay_filter.add(event.event_id)
|
||||
self._invalidate_merkle_cache()
|
||||
self._update_counters_for_event(event_dict)
|
||||
self._save()
|
||||
|
||||
try:
|
||||
from services.mesh.mesh_rns import rns_bridge
|
||||
|
||||
rns_bridge.publish_event(event_dict)
|
||||
except Exception:
|
||||
pass
|
||||
_notify_public_event_append_hooks(event_dict)
|
||||
|
||||
logger.info(
|
||||
f"Infonet append [gate_message] by {_redact_node(node_id)} seq={sequence} "
|
||||
f"id={event.event_id[:16]}..."
|
||||
)
|
||||
return event_dict
|
||||
|
||||
def append(
|
||||
self,
|
||||
event_type: str,
|
||||
@@ -2412,18 +2078,6 @@ class Infonet:
|
||||
if not event_id or not prev_hash:
|
||||
rejected.append({"index": idx, "reason": "Missing event_id or prev_hash"})
|
||||
continue
|
||||
if event_id in self.event_index:
|
||||
duplicates += 1
|
||||
continue
|
||||
if self._replay_filter.seen(event_id):
|
||||
try:
|
||||
from services.mesh.mesh_metrics import increment as metrics_inc
|
||||
|
||||
metrics_inc("ingest_replay_seen")
|
||||
except Exception:
|
||||
pass
|
||||
duplicates += 1
|
||||
continue
|
||||
if prev_hash != expected_prev:
|
||||
try:
|
||||
from services.mesh.mesh_metrics import increment as metrics_inc
|
||||
@@ -2442,14 +2096,25 @@ class Infonet:
|
||||
pass
|
||||
rejected.append({"index": idx, "reason": "network_id mismatch"})
|
||||
continue
|
||||
if event_id in self.event_index:
|
||||
duplicates += 1
|
||||
continue
|
||||
if self._replay_filter.seen(event_id):
|
||||
try:
|
||||
from services.mesh.mesh_metrics import increment as metrics_inc
|
||||
|
||||
metrics_inc("ingest_replay_seen")
|
||||
except Exception:
|
||||
pass
|
||||
duplicates += 1
|
||||
continue
|
||||
if prev_hash != self.head_hash:
|
||||
rejected.append({"index": idx, "reason": "prev_hash does not match head"})
|
||||
continue
|
||||
if sequence <= 0:
|
||||
rejected.append({"index": idx, "reason": "Invalid sequence"})
|
||||
continue
|
||||
sequence_table, sequence_key = self._sequence_table_for_event(event_type, node_id)
|
||||
last = sequence_table.get(sequence_key, 0)
|
||||
last = self.node_sequences.get(node_id, 0)
|
||||
if sequence <= last:
|
||||
rejected.append({"index": idx, "reason": "Replay detected"})
|
||||
continue
|
||||
@@ -2484,18 +2149,7 @@ class Infonet:
|
||||
if not ok:
|
||||
rejected.append({"index": idx, "reason": reason})
|
||||
continue
|
||||
if event_type == "gate_message":
|
||||
ok, reason = validate_private_gate_ledger_payload(payload)
|
||||
elif event_type == "dm_message":
|
||||
ok, reason = validate_private_dm_ledger_payload(payload)
|
||||
if ok:
|
||||
total_count, sender_count = self._dm_spool_active_counts(payload, sender_id=str(evt.get("node_id", "") or ""))
|
||||
if sender_count >= DM_HASHCHAIN_SPOOL_SENDER_LIMIT:
|
||||
ok, reason = False, "DM hashchain sender spool full for recipient"
|
||||
elif total_count >= DM_HASHCHAIN_SPOOL_LIMIT:
|
||||
ok, reason = False, "DM hashchain spool full for recipient"
|
||||
else:
|
||||
ok, reason = validate_public_ledger_payload(event_type, payload)
|
||||
ok, reason = validate_public_ledger_payload(event_type, payload)
|
||||
if not ok:
|
||||
rejected.append({"index": idx, "reason": reason})
|
||||
continue
|
||||
@@ -2571,7 +2225,7 @@ class Infonet:
|
||||
pass
|
||||
rejected.append({"index": idx, "reason": "public key is revoked"})
|
||||
continue
|
||||
last_seq = sequence_table.get(sequence_key, 0)
|
||||
last_seq = self.node_sequences.get(node_id, 0)
|
||||
if sequence <= last_seq:
|
||||
try:
|
||||
from services.mesh.mesh_metrics import increment as metrics_inc
|
||||
@@ -2607,30 +2261,18 @@ class Infonet:
|
||||
rejected.append({"index": idx, "reason": bind_reason})
|
||||
continue
|
||||
|
||||
if event_type == "gate_message":
|
||||
signature_payloads = _private_gate_signature_payload_variants(
|
||||
str(payload.get("gate", "") or ""),
|
||||
evt,
|
||||
)
|
||||
else:
|
||||
signature_payloads = [payload]
|
||||
signature_ok = False
|
||||
for signature_payload in signature_payloads:
|
||||
sig_payload = build_signature_payload(
|
||||
event_type=event_type,
|
||||
node_id=node_id,
|
||||
sequence=sequence,
|
||||
payload=signature_payload,
|
||||
)
|
||||
if verify_signature(
|
||||
public_key_b64=public_key,
|
||||
public_key_algo=public_key_algo,
|
||||
signature_hex=signature,
|
||||
payload=sig_payload,
|
||||
):
|
||||
signature_ok = True
|
||||
break
|
||||
if not signature_ok:
|
||||
sig_payload = build_signature_payload(
|
||||
event_type=event_type,
|
||||
node_id=node_id,
|
||||
sequence=sequence,
|
||||
payload=payload,
|
||||
)
|
||||
if not verify_signature(
|
||||
public_key_b64=public_key,
|
||||
public_key_algo=public_key_algo,
|
||||
signature_hex=signature,
|
||||
payload=sig_payload,
|
||||
):
|
||||
try:
|
||||
from services.mesh.mesh_metrics import increment as metrics_inc
|
||||
|
||||
@@ -2660,7 +2302,7 @@ class Infonet:
|
||||
self.events.append(evt)
|
||||
self.event_index[event_id] = len(self.events) - 1
|
||||
self.head_hash = event_id
|
||||
sequence_table[sequence_key] = sequence
|
||||
self.node_sequences[node_id] = sequence
|
||||
self._update_counters_for_event(evt)
|
||||
accepted += 1
|
||||
expected_prev = event_id
|
||||
@@ -2723,7 +2365,6 @@ class Infonet:
|
||||
verify_node_binding,
|
||||
)
|
||||
|
||||
event_type = evt_dict.get("event_type", "")
|
||||
node_id = evt_dict.get("node_id", "")
|
||||
if not parse_public_key_algo(public_key_algo):
|
||||
return False, f"Unsupported public_key_algo at index {i}"
|
||||
@@ -2734,41 +2375,21 @@ class Infonet:
|
||||
return False, f"public key binding conflict at index {i}"
|
||||
seen_public_keys[public_key] = node_id
|
||||
|
||||
payload = evt_dict.get("payload", {})
|
||||
if event_type == "gate_message":
|
||||
ok, reason = validate_private_gate_ledger_payload(payload)
|
||||
if not ok:
|
||||
return False, f"Invalid gate_message payload at index {i}: {reason}"
|
||||
signature_payloads = _private_gate_signature_payload_variants(
|
||||
str(payload.get("gate", "") or ""),
|
||||
evt_dict,
|
||||
)
|
||||
elif event_type == "dm_message":
|
||||
ok, reason = validate_private_dm_ledger_payload(payload)
|
||||
if not ok:
|
||||
return False, f"Invalid dm_message payload at index {i}: {reason}"
|
||||
signature_payloads = [normalize_payload(event_type, payload)]
|
||||
else:
|
||||
signature_payloads = [
|
||||
normalize_payload(event_type, payload)
|
||||
]
|
||||
signature_ok = False
|
||||
for signature_payload in signature_payloads:
|
||||
sig_payload = build_signature_payload(
|
||||
event_type=event_type,
|
||||
node_id=node_id,
|
||||
sequence=_safe_int(evt_dict.get("sequence", 0) or 0, 0),
|
||||
payload=signature_payload,
|
||||
)
|
||||
if verify_signature(
|
||||
public_key_b64=public_key,
|
||||
public_key_algo=public_key_algo,
|
||||
signature_hex=signature,
|
||||
payload=sig_payload,
|
||||
):
|
||||
signature_ok = True
|
||||
break
|
||||
if not signature_ok:
|
||||
normalized = normalize_payload(
|
||||
evt_dict.get("event_type", ""), evt_dict.get("payload", {})
|
||||
)
|
||||
sig_payload = build_signature_payload(
|
||||
event_type=evt_dict.get("event_type", ""),
|
||||
node_id=node_id,
|
||||
sequence=_safe_int(evt_dict.get("sequence", 0) or 0, 0),
|
||||
payload=normalized,
|
||||
)
|
||||
if not verify_signature(
|
||||
public_key_b64=public_key,
|
||||
public_key_algo=public_key_algo,
|
||||
signature_hex=signature,
|
||||
payload=sig_payload,
|
||||
):
|
||||
return False, f"Invalid signature at index {i}"
|
||||
|
||||
prev = evt_dict["event_id"]
|
||||
@@ -2833,48 +2454,27 @@ class Infonet:
|
||||
verify_node_binding,
|
||||
)
|
||||
|
||||
event_type = evt_dict.get("event_type", "")
|
||||
node_id = evt_dict.get("node_id", "")
|
||||
if not parse_public_key_algo(public_key_algo):
|
||||
return False, f"Unsupported public_key_algo at index {i}"
|
||||
if not verify_node_binding(node_id, public_key):
|
||||
return False, f"node_id mismatch at index {i}"
|
||||
|
||||
payload = evt_dict.get("payload", {})
|
||||
if event_type == "gate_message":
|
||||
ok, reason = validate_private_gate_ledger_payload(payload)
|
||||
if not ok:
|
||||
return False, f"Invalid gate_message payload at index {i}: {reason}"
|
||||
signature_payloads = _private_gate_signature_payload_variants(
|
||||
str(payload.get("gate", "") or ""),
|
||||
evt_dict,
|
||||
)
|
||||
elif event_type == "dm_message":
|
||||
ok, reason = validate_private_dm_ledger_payload(payload)
|
||||
if not ok:
|
||||
return False, f"Invalid dm_message payload at index {i}: {reason}"
|
||||
signature_payloads = [normalize_payload(event_type, payload)]
|
||||
else:
|
||||
signature_payloads = [
|
||||
normalize_payload(event_type, payload)
|
||||
]
|
||||
signature_ok = False
|
||||
for signature_payload in signature_payloads:
|
||||
sig_payload = build_signature_payload(
|
||||
event_type=event_type,
|
||||
node_id=node_id,
|
||||
sequence=_safe_int(evt_dict.get("sequence", 0) or 0, 0),
|
||||
payload=signature_payload,
|
||||
)
|
||||
if verify_signature(
|
||||
public_key_b64=public_key,
|
||||
public_key_algo=public_key_algo,
|
||||
signature_hex=signature,
|
||||
payload=sig_payload,
|
||||
):
|
||||
signature_ok = True
|
||||
break
|
||||
if not signature_ok:
|
||||
normalized = normalize_payload(
|
||||
evt_dict.get("event_type", ""), evt_dict.get("payload", {})
|
||||
)
|
||||
sig_payload = build_signature_payload(
|
||||
event_type=evt_dict.get("event_type", ""),
|
||||
node_id=node_id,
|
||||
sequence=_safe_int(evt_dict.get("sequence", 0) or 0, 0),
|
||||
payload=normalized,
|
||||
)
|
||||
if not verify_signature(
|
||||
public_key_b64=public_key,
|
||||
public_key_algo=public_key_algo,
|
||||
signature_hex=signature,
|
||||
payload=sig_payload,
|
||||
):
|
||||
return False, f"Invalid signature at index {i}"
|
||||
prev = evt_dict["event_id"]
|
||||
|
||||
@@ -2938,14 +2538,7 @@ class Infonet:
|
||||
node_id = evt.get("node_id", "")
|
||||
sequence = _safe_int(evt.get("sequence", 0) or 0, 0)
|
||||
if node_id and sequence:
|
||||
sequence_key = (
|
||||
f"{node_id}|gate_message"
|
||||
if str(evt.get("event_type", "") or "").strip().lower() == "gate_message"
|
||||
else f"{node_id}|dm_message"
|
||||
if str(evt.get("event_type", "") or "").strip().lower() == "dm_message"
|
||||
else node_id
|
||||
)
|
||||
last_seq[sequence_key] = max(last_seq.get(sequence_key, 0), sequence)
|
||||
last_seq[node_id] = max(last_seq.get(node_id, 0), sequence)
|
||||
public_key = str(evt.get("public_key", "") or "")
|
||||
if public_key and node_id:
|
||||
seen_public_keys.setdefault(public_key, node_id)
|
||||
@@ -2965,21 +2558,8 @@ class Infonet:
|
||||
existing_idx = self.event_index.get(event_id)
|
||||
if existing_idx is not None and existing_idx <= prev_index:
|
||||
return False, "duplicate event_id"
|
||||
if event_type == "gate_message":
|
||||
payload = dict(payload or {})
|
||||
elif event_type == "dm_message":
|
||||
payload = normalize_payload(event_type, dict(payload or {}))
|
||||
else:
|
||||
payload = normalize_payload(event_type, dict(payload or {}))
|
||||
payload = normalize_payload(event_type, dict(payload or {}))
|
||||
ok, reason = validate_event_payload(event_type, payload)
|
||||
if not ok:
|
||||
return False, reason
|
||||
if event_type == "gate_message":
|
||||
ok, reason = validate_private_gate_ledger_payload(payload)
|
||||
elif event_type == "dm_message":
|
||||
ok, reason = validate_private_dm_ledger_payload(payload)
|
||||
else:
|
||||
ok, reason = validate_public_ledger_payload(event_type, payload)
|
||||
if not ok:
|
||||
return False, reason
|
||||
proto = evt.get("protocol_version") or PROTOCOL_VERSION
|
||||
@@ -2993,14 +2573,7 @@ class Infonet:
|
||||
revoked, _info = self._revocation_status(public_key)
|
||||
if revoked and event_type != "key_revoke":
|
||||
return False, "public key revoked"
|
||||
sequence_key = (
|
||||
f"{node_id}|gate_message"
|
||||
if event_type == "gate_message"
|
||||
else f"{node_id}|dm_message"
|
||||
if event_type == "dm_message"
|
||||
else node_id
|
||||
)
|
||||
last = last_seq.get(sequence_key, 0)
|
||||
last = last_seq.get(node_id, 0)
|
||||
if sequence <= last:
|
||||
return False, "sequence replay"
|
||||
from services.mesh.mesh_crypto import (
|
||||
@@ -3018,35 +2591,23 @@ class Infonet:
|
||||
if existing and existing != node_id:
|
||||
return False, "public key binding conflict"
|
||||
seen_public_keys[public_key] = node_id
|
||||
if event_type == "gate_message":
|
||||
signature_payloads = _private_gate_signature_payload_variants(
|
||||
str(payload.get("gate", "") or ""),
|
||||
evt,
|
||||
)
|
||||
else:
|
||||
signature_payloads = [payload]
|
||||
signature_ok = False
|
||||
for signature_payload in signature_payloads:
|
||||
sig_payload = build_signature_payload(
|
||||
event_type=event_type,
|
||||
node_id=node_id,
|
||||
sequence=sequence,
|
||||
payload=signature_payload,
|
||||
)
|
||||
if verify_signature(
|
||||
public_key_b64=public_key,
|
||||
public_key_algo=public_key_algo,
|
||||
signature_hex=signature,
|
||||
payload=sig_payload,
|
||||
):
|
||||
signature_ok = True
|
||||
break
|
||||
if not signature_ok:
|
||||
sig_payload = build_signature_payload(
|
||||
event_type=event_type,
|
||||
node_id=node_id,
|
||||
sequence=sequence,
|
||||
payload=payload,
|
||||
)
|
||||
if not verify_signature(
|
||||
public_key_b64=public_key,
|
||||
public_key_algo=public_key_algo,
|
||||
signature_hex=signature,
|
||||
payload=sig_payload,
|
||||
):
|
||||
return False, "invalid signature"
|
||||
computed = ChainEvent.from_dict(evt).event_id
|
||||
if computed != event_id:
|
||||
return False, "event_id mismatch"
|
||||
last_seq[sequence_key] = sequence
|
||||
last_seq[node_id] = sequence
|
||||
|
||||
# Apply fork
|
||||
self.events = prefix + ordered
|
||||
|
||||
@@ -276,6 +276,5 @@ def should_run_sync(
|
||||
) -> bool:
|
||||
current_time = int(now if now is not None else time.time())
|
||||
if state.last_outcome == "running":
|
||||
started_at = int(state.last_sync_started_at or 0)
|
||||
return started_at <= 0 or current_time - started_at >= 300
|
||||
return False
|
||||
return int(state.next_sync_due_at or 0) <= current_time
|
||||
|
||||
@@ -2,9 +2,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable
|
||||
|
||||
@@ -36,58 +33,6 @@ def _require_fields(payload: dict[str, Any], fields: tuple[str, ...]) -> tuple[b
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _decode_base64ish(value: Any) -> bytes | None:
|
||||
raw = str(value or "").strip()
|
||||
if not raw or any(ch.isspace() for ch in raw):
|
||||
return None
|
||||
padded = raw + ("=" * (-len(raw) % 4))
|
||||
for altchars in (None, b"-_"):
|
||||
try:
|
||||
return base64.b64decode(padded.encode("ascii"), altchars=altchars, validate=True)
|
||||
except (binascii.Error, UnicodeEncodeError, ValueError):
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _byte_entropy(data: bytes) -> float:
|
||||
if not data:
|
||||
return 0.0
|
||||
counts = [0] * 256
|
||||
for byte in data:
|
||||
counts[byte] += 1
|
||||
total = float(len(data))
|
||||
return -sum((count / total) * math.log2(count / total) for count in counts if count)
|
||||
|
||||
|
||||
def _validate_sealed_bytes_field(
|
||||
payload: dict[str, Any],
|
||||
field: str,
|
||||
*,
|
||||
min_bytes: int = 8,
|
||||
entropy_floor: float = 2.5,
|
||||
) -> tuple[bool, str]:
|
||||
data = _decode_base64ish(payload.get(field, ""))
|
||||
if data is None:
|
||||
return False, f"{field} must be base64-encoded sealed bytes"
|
||||
if len(data) < min_bytes:
|
||||
return False, f"{field} is too short"
|
||||
|
||||
# Short test vectors and compact envelopes can be low entropy; only apply
|
||||
# heuristics once there is enough material to distinguish a sealed blob
|
||||
# from accidental base64-encoded plaintext.
|
||||
if len(data) >= 32:
|
||||
printable = sum(1 for byte in data if 32 <= byte <= 126 or byte in (9, 10, 13))
|
||||
if printable / len(data) > 0.9:
|
||||
try:
|
||||
data.decode("utf-8")
|
||||
return False, f"{field} looks like encoded plaintext"
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
if _byte_entropy(data) < entropy_floor:
|
||||
return False, f"{field} entropy is too low for sealed bytes"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_message(payload: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, reason = _require_fields(
|
||||
payload, ("message", "destination", "channel", "priority", "ephemeral")
|
||||
@@ -386,7 +331,6 @@ ACTIVE_PUBLIC_LEDGER_EVENT_TYPES: frozenset[str] = frozenset(
|
||||
LEGACY_PUBLIC_LEDGER_EVENT_TYPES: frozenset[str] = frozenset(
|
||||
{
|
||||
"gate_message",
|
||||
"dm_message",
|
||||
}
|
||||
)
|
||||
"""Event types that exist historically on the public chain and must remain
|
||||
@@ -481,8 +425,6 @@ def validate_event_payload(event_type: str, payload: dict[str, Any]) -> tuple[bo
|
||||
|
||||
|
||||
def validate_public_ledger_payload(event_type: str, payload: dict[str, Any]) -> tuple[bool, str]:
|
||||
if event_type == "gate_message":
|
||||
return validate_private_gate_ledger_payload(payload)
|
||||
if event_type not in PUBLIC_LEDGER_EVENT_TYPES and event_type not in _EXTENSION_VALIDATORS:
|
||||
return False, f"{event_type} is not allowed on the public ledger"
|
||||
forbidden = sorted(
|
||||
@@ -499,92 +441,6 @@ def validate_public_ledger_payload(event_type: str, payload: dict[str, Any]) ->
|
||||
return True, "ok"
|
||||
|
||||
|
||||
_PRIVATE_GATE_LEDGER_ALLOWED_FIELDS: frozenset[str] = frozenset(
|
||||
{
|
||||
"gate",
|
||||
"ciphertext",
|
||||
"nonce",
|
||||
"sender_ref",
|
||||
"format",
|
||||
"epoch",
|
||||
"gate_envelope",
|
||||
"envelope_hash",
|
||||
"reply_to",
|
||||
"transport_lock",
|
||||
"signed_context",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def validate_private_gate_ledger_payload(payload: dict[str, Any]) -> tuple[bool, str]:
|
||||
"""Validate ciphertext-only gate events for private Infonet replication."""
|
||||
ok, reason = validate_event_payload("gate_message", payload)
|
||||
if not ok:
|
||||
return ok, reason
|
||||
unexpected = sorted(
|
||||
key
|
||||
for key in payload.keys()
|
||||
if str(key or "").strip().lower() not in _PRIVATE_GATE_LEDGER_ALLOWED_FIELDS
|
||||
)
|
||||
if unexpected:
|
||||
return False, f"private gate ledger payload contains unsupported fields: {', '.join(unexpected)}"
|
||||
if "message" in payload or "_local_plaintext" in payload or "_local_reply_to" in payload:
|
||||
return False, "private gate ledger payload must not contain plaintext"
|
||||
transport_lock = str(payload.get("transport_lock", "") or "").strip().lower()
|
||||
if transport_lock and transport_lock not in {"private", "private_strong", "rns", "onion"}:
|
||||
return False, "gate messages require private transport_lock"
|
||||
ok, reason = _validate_sealed_bytes_field(payload, "ciphertext")
|
||||
if not ok:
|
||||
return ok, reason
|
||||
ok, reason = _validate_sealed_bytes_field(payload, "nonce")
|
||||
if not ok:
|
||||
return ok, reason
|
||||
return True, "ok"
|
||||
|
||||
|
||||
_PRIVATE_DM_LEDGER_ALLOWED_FIELDS: frozenset[str] = frozenset(
|
||||
{
|
||||
"recipient_id",
|
||||
"delivery_class",
|
||||
"recipient_token",
|
||||
"ciphertext",
|
||||
"msg_id",
|
||||
"timestamp",
|
||||
"format",
|
||||
"session_welcome",
|
||||
"sender_seal",
|
||||
"relay_salt",
|
||||
"transport_lock",
|
||||
"signed_context",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def validate_private_dm_ledger_payload(payload: dict[str, Any]) -> tuple[bool, str]:
|
||||
"""Validate ciphertext-only DM dead-drop events for private Infonet replication."""
|
||||
ok, reason = validate_event_payload("dm_message", payload)
|
||||
if not ok:
|
||||
return ok, reason
|
||||
unexpected = sorted(
|
||||
key
|
||||
for key in payload.keys()
|
||||
if str(key or "").strip().lower() not in _PRIVATE_DM_LEDGER_ALLOWED_FIELDS
|
||||
)
|
||||
if unexpected:
|
||||
return False, f"private DM ledger payload contains unsupported fields: {', '.join(unexpected)}"
|
||||
if "message" in payload or "plaintext" in payload or "_local_plaintext" in payload:
|
||||
return False, "private DM ledger payload must not contain plaintext"
|
||||
transport_lock = str(payload.get("transport_lock", "") or "").strip().lower()
|
||||
if transport_lock != "private_strong":
|
||||
return False, "DM hashchain spool requires private_strong transport_lock"
|
||||
if not str(payload.get("ciphertext", "") or "").strip():
|
||||
return False, "ciphertext cannot be empty"
|
||||
ok, reason = _validate_sealed_bytes_field(payload, "ciphertext")
|
||||
if not ok:
|
||||
return ok, reason
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def validate_protocol_fields(protocol_version: str, network_id: str) -> tuple[bool, str]:
|
||||
if protocol_version != PROTOCOL_VERSION:
|
||||
return False, "Unsupported protocol_version"
|
||||
|
||||
@@ -38,11 +38,6 @@ _REVOCATION_TTL_CACHE: dict[str, dict[str, Any]] = {}
|
||||
_REVOCATION_TTL_LOCK = threading.Lock()
|
||||
_REVOCATION_REFRESH_LOCK = threading.Lock()
|
||||
_REVOCATION_REFRESH_FAIL_FAST_WINDOW_S = 5.0
|
||||
|
||||
|
||||
def _request_scope_path(request: Request) -> str:
|
||||
scope = getattr(request, "scope", {}) or {}
|
||||
return str(scope.get("path") or "")
|
||||
_REVOCATION_REFRESH_RETRY_AFTER_S = 5
|
||||
_REVOCATION_PRECHECK_UNAVAILABLE_DETAIL = "Signed event integrity preflight unavailable"
|
||||
|
||||
@@ -171,7 +166,7 @@ def _canonical_signed_write_retry_payload(
|
||||
signed_context = build_signed_context(
|
||||
event_type=prepared.event_type,
|
||||
kind=prepared.kind.value,
|
||||
endpoint=_request_scope_path(request),
|
||||
endpoint=str(request.url.path or ""),
|
||||
lane_floor=_content_private_required_transport_tier(prepared.kind),
|
||||
sequence_domain=_signed_context_sequence_domain(prepared),
|
||||
node_id=prepared.node_id,
|
||||
@@ -545,7 +540,7 @@ def _apply_signed_context_policy(prepared: "PreparedSignedWrite", request: Reque
|
||||
ok, reason = validate_signed_context(
|
||||
event_type=prepared.event_type,
|
||||
kind=prepared.kind.value,
|
||||
endpoint=_request_scope_path(request),
|
||||
endpoint=str(request.url.path or ""),
|
||||
lane_floor=_content_private_required_transport_tier(prepared.kind),
|
||||
sequence_domain=_signed_context_sequence_domain(prepared),
|
||||
node_id=prepared.node_id,
|
||||
|
||||
@@ -234,12 +234,12 @@ def _fetch_dm_prekey_bundle_from_public_lookup(lookup_token: str) -> dict[str, A
|
||||
# Generic UA: any peer-facing crypto request should not carry a
|
||||
# fork-specific identifier — that turns prekey lookups into a
|
||||
# software-fingerprinting beacon.
|
||||
from services.network_utils import default_user_agent
|
||||
from services.network_utils import DEFAULT_USER_AGENT
|
||||
request = urllib.request.Request(
|
||||
f"{normalized_peer_url}/api/mesh/dm/prekey-bundle?{encoded}",
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"User-Agent": default_user_agent(),
|
||||
"User-Agent": DEFAULT_USER_AGENT,
|
||||
},
|
||||
method="GET",
|
||||
)
|
||||
|
||||
@@ -34,9 +34,9 @@ _session.mount("http://", HTTPAdapter(max_retries=_retry, pool_maxsize=10))
|
||||
# upstream's only recourse was to block "Shadowbroker" as a whole — which
|
||||
# would take out every other install too.
|
||||
#
|
||||
# Fix: give each install a stable pseudonymous handle used as the entire
|
||||
# User-Agent product token (no shared "Shadowbroker" label). Upstreams see
|
||||
# ``operator-7f3a92`` (or ``OPERATOR_HANDLE``), not one monolithic app name.
|
||||
# Fix: give each install a stable pseudonymous handle and include it in
|
||||
# the User-Agent. Now an upstream can rate-limit or block the offending
|
||||
# operator without affecting anyone else.
|
||||
#
|
||||
# The handle:
|
||||
#
|
||||
@@ -51,6 +51,7 @@ _session.mount("http://", HTTPAdapter(max_retries=_retry, pool_maxsize=10))
|
||||
# - Is NEVER mixed into mesh / Wormhole / Infonet identity. This layer is
|
||||
# strictly for public third-party API attribution.
|
||||
|
||||
_SHADOWBROKER_VERSION = "0.9"
|
||||
_OPERATOR_HANDLE_FILE = (
|
||||
Path(__file__).parent.parent / "data" / "operator_handle.json"
|
||||
)
|
||||
@@ -145,12 +146,7 @@ def get_operator_handle() -> str:
|
||||
# 3. On-disk handle from a previous run.
|
||||
persisted = _load_persisted_operator_handle()
|
||||
if persisted:
|
||||
normalized = _normalize_handle(persisted)
|
||||
# Migrate legacy auto-generated handles (pre-Round-7a ``shadow-`` prefix).
|
||||
if normalized.startswith("shadow-"):
|
||||
normalized = f"operator-{normalized[len('shadow-'):]}"
|
||||
_persist_operator_handle(normalized)
|
||||
_OPERATOR_HANDLE_CACHE = normalized
|
||||
_OPERATOR_HANDLE_CACHE = _normalize_handle(persisted)
|
||||
return _OPERATOR_HANDLE_CACHE
|
||||
|
||||
# 4. Generate, persist, return.
|
||||
@@ -174,21 +170,41 @@ def _normalize_handle(raw: str) -> str:
|
||||
return safe[:48] if safe else "anonymous"
|
||||
|
||||
|
||||
_CONTACT_URL = "https://github.com/BigBodyCobain/Shadowbroker/issues"
|
||||
|
||||
|
||||
def outbound_user_agent(purpose: str = "") -> str:
|
||||
"""Build a User-Agent for an outbound third-party HTTP request.
|
||||
|
||||
Returns the per-install handle only, e.g. ``operator-7f3a92`` or
|
||||
``operator-7f3a92 (purpose: wikipedia)``. No shared project name — so
|
||||
upstream abuse teams cannot block every install with one ``Shadowbroker``
|
||||
rule.
|
||||
Returns something like::
|
||||
|
||||
Set ``SHADOWBROKER_USER_AGENT`` to override the entire string if needed.
|
||||
Shadowbroker/0.9 (operator: shadow-7f3a92; purpose: wikipedia;
|
||||
+https://github.com/BigBodyCobain/Shadowbroker/issues)
|
||||
|
||||
The ``purpose`` is optional but recommended — it tells the upstream
|
||||
what feature of ours is making the call (``wikipedia``, ``openmhz``,
|
||||
``nominatim``, etc.), which makes their logs and our complaints
|
||||
actionable.
|
||||
|
||||
Every outbound call in the backend that previously sent a custom
|
||||
User-Agent should call this helper instead. Centralizing here means:
|
||||
- one place to change the contact URL,
|
||||
- one place to bump the version on release,
|
||||
- one place a Wikimedia / OpenMHz operator can reach to ask for
|
||||
the project to back off, with a per-install handle so they can
|
||||
target the specific install instead of the project as a whole.
|
||||
"""
|
||||
handle = get_operator_handle()
|
||||
if purpose:
|
||||
purpose_clean = _normalize_handle(purpose)
|
||||
return f"{handle} (purpose: {purpose_clean})"
|
||||
return handle
|
||||
return (
|
||||
f"Shadowbroker/{_SHADOWBROKER_VERSION} "
|
||||
f"(operator: {handle}; purpose: {purpose_clean}; +{_CONTACT_URL})"
|
||||
)
|
||||
return (
|
||||
f"Shadowbroker/{_SHADOWBROKER_VERSION} "
|
||||
f"(operator: {handle}; +{_CONTACT_URL})"
|
||||
)
|
||||
|
||||
|
||||
def _reset_operator_handle_cache_for_tests() -> None:
|
||||
@@ -199,13 +215,19 @@ def _reset_operator_handle_cache_for_tests() -> None:
|
||||
_OPERATOR_HANDLE_CACHE = ""
|
||||
|
||||
|
||||
def default_user_agent() -> str:
|
||||
"""Default User-Agent for ``fetch_with_curl`` and legacy call sites."""
|
||||
custom = (os.environ.get("SHADOWBROKER_USER_AGENT") or "").strip()
|
||||
if custom:
|
||||
return custom
|
||||
return outbound_user_agent()
|
||||
|
||||
# Default outbound User-Agent. Retained for backwards compatibility with
|
||||
# call sites that haven't been migrated to ``outbound_user_agent()`` yet.
|
||||
# Operators who want full per-install attribution should set the
|
||||
# ``OPERATOR_HANDLE`` setting and migrate call sites incrementally.
|
||||
#
|
||||
# Operators who run a public-facing relay can also override the whole UA
|
||||
# string via the ``SHADOWBROKER_USER_AGENT`` env var. That override
|
||||
# completely bypasses the per-operator helper; only use it if you know
|
||||
# what you're doing.
|
||||
DEFAULT_USER_AGENT = os.environ.get(
|
||||
"SHADOWBROKER_USER_AGENT",
|
||||
f"Shadowbroker/{_SHADOWBROKER_VERSION}",
|
||||
)
|
||||
|
||||
# Find bash for curl fallback — Git bash's curl has the TLS features
|
||||
# needed to pass CDN fingerprint checks (brotli, zstd, libpsl)
|
||||
@@ -261,7 +283,7 @@ def fetch_with_curl(url, method="GET", json_data=None, timeout=15, headers=None,
|
||||
both Python requests and the barebones Windows system curl.
|
||||
"""
|
||||
default_headers = {
|
||||
"User-Agent": default_user_agent(),
|
||||
"User-Agent": DEFAULT_USER_AGENT,
|
||||
}
|
||||
if headers:
|
||||
default_headers.update(headers)
|
||||
|
||||
@@ -12,8 +12,6 @@ logger = logging.getLogger(__name__)
|
||||
CONFIG_PATH = Path(__file__).parent.parent / "config" / "news_feeds.json"
|
||||
MAX_FEEDS = 50
|
||||
_FEED_URL_REPLACEMENTS = {
|
||||
"http://feeds.bbci.co.uk/news/world/rss.xml": "https://feeds.bbci.co.uk/news/world/rss.xml",
|
||||
"http://www.news.cn/english/rss/worldrss.xml": "https://www.news.cn/english/rss/worldrss.xml",
|
||||
"https://www.channelnewsasia.com/rssfeed/8395986": "https://www.channelnewsasia.com/api/v1/rss-outbound-feed?_format=xml",
|
||||
}
|
||||
_DEAD_FEED_URLS = {
|
||||
@@ -29,7 +27,7 @@ _DEAD_FEED_URLS = {
|
||||
|
||||
DEFAULT_FEEDS = [
|
||||
{"name": "NPR", "url": "https://feeds.npr.org/1004/rss.xml", "weight": 4},
|
||||
{"name": "BBC", "url": "https://feeds.bbci.co.uk/news/world/rss.xml", "weight": 3},
|
||||
{"name": "BBC", "url": "http://feeds.bbci.co.uk/news/world/rss.xml", "weight": 3},
|
||||
{"name": "AlJazeera", "url": "https://www.aljazeera.com/xml/rss/all.xml", "weight": 2},
|
||||
{"name": "NYT", "url": "https://rss.nytimes.com/services/xml/rss/nyt/World.xml", "weight": 1},
|
||||
{"name": "GDACS", "url": "https://www.gdacs.org/xml/rss.xml", "weight": 5},
|
||||
@@ -37,7 +35,7 @@ DEFAULT_FEEDS = [
|
||||
{"name": "Bellingcat", "url": "https://www.bellingcat.com/feed/", "weight": 4},
|
||||
{"name": "Guardian", "url": "https://www.theguardian.com/world/rss", "weight": 3},
|
||||
{"name": "TASS", "url": "https://tass.com/rss/v2.xml", "weight": 2},
|
||||
{"name": "Xinhua", "url": "https://www.news.cn/english/rss/worldrss.xml", "weight": 2},
|
||||
{"name": "Xinhua", "url": "http://www.news.cn/english/rss/worldrss.xml", "weight": 2},
|
||||
{"name": "CNA", "url": "https://www.channelnewsasia.com/api/v1/rss-outbound-feed?_format=xml", "weight": 3},
|
||||
{"name": "Mercopress", "url": "https://en.mercopress.com/rss/", "weight": 3},
|
||||
{"name": "SCMP", "url": "https://www.scmp.com/rss/91/feed", "weight": 4},
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
"""Operator opt-in for Polymarket/Kalshi outbound fetches (Global Threat Intercept)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_OPT_IN_FILE = Path(__file__).resolve().parent.parent / "data" / "prediction_markets_opt_in.json"
|
||||
_OPT_IN_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _env_flag(name: str) -> str:
|
||||
return str(os.getenv(name, "")).strip().lower()
|
||||
|
||||
|
||||
def get_prediction_markets_ui_opt_in() -> bool:
|
||||
if not _OPT_IN_FILE.exists():
|
||||
return False
|
||||
try:
|
||||
payload = json.loads(_OPT_IN_FILE.read_text(encoding="utf-8"))
|
||||
return bool(payload.get("opted_in"))
|
||||
except (OSError, json.JSONDecodeError, TypeError) as exc:
|
||||
logger.warning("Prediction markets opt-in file unreadable: %s", exc)
|
||||
return False
|
||||
|
||||
|
||||
def set_prediction_markets_ui_opt_in(opted_in: bool) -> None:
|
||||
_OPT_IN_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
with _OPT_IN_LOCK:
|
||||
_OPT_IN_FILE.write_text(
|
||||
json.dumps({"opted_in": bool(opted_in)}, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def prediction_markets_env_forced_on() -> bool:
|
||||
return _env_flag("PREDICTION_MARKETS_ENABLED") in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def prediction_markets_env_forced_off() -> bool:
|
||||
return _env_flag("PREDICTION_MARKETS_ENABLED") in {"0", "false", "no", "off"}
|
||||
|
||||
|
||||
def prediction_markets_fetch_enabled() -> bool:
|
||||
"""True when UI opt-in or env enables Polymarket/Kalshi pulls."""
|
||||
if get_prediction_markets_ui_opt_in():
|
||||
return True
|
||||
return prediction_markets_env_forced_on()
|
||||
|
||||
|
||||
def prediction_markets_status() -> dict[str, Any]:
|
||||
ui_opted_in = get_prediction_markets_ui_opt_in()
|
||||
env_on = prediction_markets_env_forced_on()
|
||||
env_off = prediction_markets_env_forced_off()
|
||||
env_override = None
|
||||
if env_on:
|
||||
env_override = "on"
|
||||
elif env_off:
|
||||
env_override = "off"
|
||||
return {
|
||||
"enabled": prediction_markets_fetch_enabled(),
|
||||
"ui_opted_in": ui_opted_in,
|
||||
"env_override": env_override,
|
||||
"jitter": {
|
||||
"scheduler_interval_minutes": int(
|
||||
os.environ.get("PREDICTION_MARKETS_INTERVAL_MINUTES", "7")
|
||||
),
|
||||
"scheduler_jitter_seconds": int(
|
||||
os.environ.get("PREDICTION_MARKETS_SCHEDULER_JITTER_S", "240")
|
||||
),
|
||||
"pre_fetch_jitter_seconds": float(
|
||||
os.environ.get("PREDICTION_MARKETS_PRE_FETCH_JITTER_S", "90")
|
||||
),
|
||||
},
|
||||
}
|
||||
@@ -301,36 +301,3 @@ def get_region_dossier(lat: float, lng: float) -> dict:
|
||||
|
||||
dossier_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
|
||||
def fetch_wikipedia_page_summary(title: str) -> dict | None:
|
||||
"""Wikipedia REST summary for a page title (backend-proxied for #360)."""
|
||||
trimmed = (title or "").strip()
|
||||
if not trimmed:
|
||||
return None
|
||||
data = _fetch_local_wiki_summary(trimmed, "")
|
||||
if not data.get("extract") and not data.get("description"):
|
||||
return None
|
||||
return {
|
||||
"title": trimmed,
|
||||
"description": data.get("description", ""),
|
||||
"extract": data.get("extract", ""),
|
||||
"thumbnail": data.get("thumbnail", ""),
|
||||
"type": "standard",
|
||||
}
|
||||
|
||||
|
||||
def fetch_wikidata_sparql_bindings(sparql: str) -> list:
|
||||
"""Run a Wikidata SPARQL query; returns bindings list (empty on failure)."""
|
||||
trimmed = (sparql or "").strip()
|
||||
if not trimmed:
|
||||
return []
|
||||
url = f"https://query.wikidata.org/sparql?query={quote(trimmed)}&format=json"
|
||||
try:
|
||||
res = fetch_with_curl(url, timeout=8, headers=_wikimedia_request_headers())
|
||||
if res.status_code == 200:
|
||||
bindings = res.json().get("results", {}).get("bindings", [])
|
||||
return bindings if isinstance(bindings, list) else []
|
||||
except (ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
|
||||
logger.warning("Wikidata SPARQL failed: %s", e)
|
||||
return []
|
||||
|
||||
@@ -26,46 +26,6 @@ def _planetary_user_agent() -> str:
|
||||
return outbound_user_agent("sentinel2-planetary-computer")
|
||||
|
||||
|
||||
def _sign_planetary_href(href: str) -> str:
|
||||
"""Sign a Planetary Computer blob URL with a short-lived SAS token."""
|
||||
if not href or "blob.core.windows.net" not in href:
|
||||
return href
|
||||
try:
|
||||
account = href.split(".blob.core.windows.net")[0].split("//")[-1]
|
||||
token_resp = requests.get(
|
||||
f"https://planetarycomputer.microsoft.com/api/sas/v1/token/{account}",
|
||||
timeout=5,
|
||||
headers={"User-Agent": _planetary_user_agent()},
|
||||
)
|
||||
token_resp.raise_for_status()
|
||||
token = token_resp.json().get("token", "")
|
||||
if not token:
|
||||
return href
|
||||
sep = "&" if "?" in href else "?"
|
||||
return f"{href}{sep}{token}"
|
||||
except (requests.RequestException, ValueError, KeyError):
|
||||
return href
|
||||
|
||||
|
||||
def _scene_from_stac_feature(item: dict) -> dict:
|
||||
assets = item.get("assets", {}) or {}
|
||||
rendered = assets.get("rendered_preview") or {}
|
||||
thumbnail = assets.get("thumbnail") or {}
|
||||
props = item.get("properties", {}) or {}
|
||||
thumb_href = _sign_planetary_href(thumbnail.get("href") or rendered.get("href") or "")
|
||||
full_href = _sign_planetary_href(rendered.get("href") or thumbnail.get("href") or "")
|
||||
return {
|
||||
"found": True,
|
||||
"scene_id": item.get("id"),
|
||||
"datetime": props.get("datetime"),
|
||||
"cloud_cover": props.get("eo:cloud_cover"),
|
||||
"thumbnail_url": thumb_href or None,
|
||||
"fullres_url": full_href or None,
|
||||
"bbox": list(item.get("bbox", [])) if item.get("bbox") else None,
|
||||
"platform": props.get("platform", "Sentinel-2"),
|
||||
}
|
||||
|
||||
|
||||
def _esri_imagery_fallback(lat: float, lng: float) -> dict:
|
||||
lat_span = 0.18
|
||||
lng_span = 0.24
|
||||
@@ -93,14 +53,14 @@ def _esri_imagery_fallback(lat: float, lng: float) -> dict:
|
||||
|
||||
|
||||
def search_sentinel2_scene(lat: float, lng: float) -> dict:
|
||||
"""Search for up to 3 recent Sentinel-2 L2A scenes covering a point."""
|
||||
"""Search for the latest Sentinel-2 L2A scene covering a point."""
|
||||
cache_key = f"{round(lat, 2)}_{round(lng, 2)}"
|
||||
if cache_key in _sentinel_cache:
|
||||
return _sentinel_cache[cache_key]
|
||||
|
||||
try:
|
||||
end = datetime.utcnow()
|
||||
start = end - timedelta(days=60)
|
||||
start = end - timedelta(days=30)
|
||||
search_payload = {
|
||||
"collections": ["sentinel-2-l2a"],
|
||||
"intersects": {"type": "Point", "coordinates": [lng, lat]},
|
||||
@@ -123,8 +83,26 @@ def search_sentinel2_scene(lat: float, lng: float) -> dict:
|
||||
_sentinel_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
scenes = [_scene_from_stac_feature(item) for item in features[:3]]
|
||||
result = {**scenes[0], "scenes": scenes}
|
||||
item = features[0]
|
||||
assets = item.get("assets", {}) or {}
|
||||
rendered = assets.get("rendered_preview") or {}
|
||||
thumbnail = assets.get("thumbnail") or {}
|
||||
|
||||
# Full-res image URL — what opens when user clicks
|
||||
fullres_url = rendered.get("href") or thumbnail.get("href")
|
||||
# Thumbnail URL — what shows in the popup card
|
||||
thumb_url = thumbnail.get("href") or rendered.get("href")
|
||||
|
||||
result = {
|
||||
"found": True,
|
||||
"scene_id": item.get("id"),
|
||||
"datetime": item.get("properties", {}).get("datetime"),
|
||||
"cloud_cover": item.get("properties", {}).get("eo:cloud_cover"),
|
||||
"thumbnail_url": thumb_url,
|
||||
"fullres_url": fullres_url,
|
||||
"bbox": list(item.get("bbox", [])) if item.get("bbox") else None,
|
||||
"platform": item.get("properties", {}).get("platform", "Sentinel-2"),
|
||||
}
|
||||
_sentinel_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ SLO_REGISTRY: Dict[str, SLO] = {
|
||||
"uap_sightings": SLO(
|
||||
max_age_s=26 * _HOUR,
|
||||
min_rows=50,
|
||||
description="NUFORC rolling 60-day window (weekly refresh)",
|
||||
description="NUFORC rolling 60-day window (daily refresh)",
|
||||
),
|
||||
"wastewater": SLO(
|
||||
max_age_s=30 * _HOUR,
|
||||
|
||||
@@ -87,28 +87,11 @@ def _run_gate_release_once(monkeypatch, *, transport_tier="private_strong"):
|
||||
def _patch_for_successful_post(monkeypatch, module):
|
||||
"""Apply standard monkeypatches so a gate_message post succeeds."""
|
||||
import main
|
||||
from services.mesh import mesh_hashchain
|
||||
|
||||
_setup_gate_outbox(monkeypatch)
|
||||
monkeypatch.setattr(main, "_verify_gate_message_signed_write", lambda **kw: (True, "ok", kw.get("reply_to", "")))
|
||||
monkeypatch.setattr(main, "_resolve_envelope_policy", lambda _gate_id: "envelope_disabled")
|
||||
|
||||
def _fake_private_gate_append(**kwargs):
|
||||
return {
|
||||
"event_id": f"ledger-ev-{kwargs.get('sequence', 0)}",
|
||||
"event_type": "gate_message",
|
||||
"node_id": kwargs["node_id"],
|
||||
"payload": dict(kwargs["payload"]),
|
||||
"timestamp": kwargs.get("timestamp", 0) or 123.0,
|
||||
"sequence": kwargs["sequence"],
|
||||
"signature": kwargs["signature"],
|
||||
"public_key": kwargs["public_key"],
|
||||
"public_key_algo": kwargs["public_key_algo"],
|
||||
"protocol_version": kwargs.get("protocol_version", "infonet/2"),
|
||||
}
|
||||
|
||||
monkeypatch.setattr(mesh_hashchain.infonet, "append_private_gate_message", _fake_private_gate_append)
|
||||
|
||||
from services.mesh.mesh_reputation import gate_manager, reputation_ledger
|
||||
|
||||
monkeypatch.setattr(gate_manager, "can_enter", lambda *a, **kw: (True, "ok"))
|
||||
@@ -272,30 +255,19 @@ def test_gate_post_preserves_gate_envelope_in_store(monkeypatch):
|
||||
|
||||
|
||||
def test_gate_post_advances_sequence(monkeypatch):
|
||||
"""append_private_gate_message must receive the gate sequence."""
|
||||
"""validate_and_set_sequence must be called to advance the counter."""
|
||||
import main
|
||||
from services.mesh import mesh_hashchain
|
||||
|
||||
_patch_for_successful_post(monkeypatch, main)
|
||||
|
||||
append_calls = []
|
||||
seq_calls = []
|
||||
|
||||
def track_private_append(**kwargs):
|
||||
append_calls.append(kwargs)
|
||||
return {
|
||||
"event_id": "ev-seq",
|
||||
"event_type": "gate_message",
|
||||
"node_id": kwargs["node_id"],
|
||||
"payload": dict(kwargs["payload"]),
|
||||
"timestamp": kwargs.get("timestamp", 0) or 123.0,
|
||||
"sequence": kwargs["sequence"],
|
||||
"signature": kwargs["signature"],
|
||||
"public_key": kwargs["public_key"],
|
||||
"public_key_algo": kwargs["public_key_algo"],
|
||||
"protocol_version": kwargs.get("protocol_version", "infonet/2"),
|
||||
}
|
||||
def track_seq(node_id, seq, *, domain=""):
|
||||
seq_calls.append((node_id, seq, domain))
|
||||
return (True, "ok")
|
||||
|
||||
monkeypatch.setattr(mesh_hashchain.infonet, "append_private_gate_message", track_private_append)
|
||||
monkeypatch.setattr(mesh_hashchain.infonet, "validate_and_set_sequence", track_seq)
|
||||
monkeypatch.setattr(
|
||||
mesh_hashchain.gate_store,
|
||||
"append",
|
||||
@@ -308,9 +280,8 @@ def test_gate_post_advances_sequence(monkeypatch):
|
||||
|
||||
assert result["ok"] is True
|
||||
assert result["queued"] is True
|
||||
assert len(append_calls) == 1
|
||||
assert append_calls[0]["node_id"] == "!sb_test1234567890"
|
||||
assert append_calls[0]["sequence"] == 42
|
||||
assert len(seq_calls) == 1
|
||||
assert seq_calls[0] == ("!sb_test1234567890", 42, "gate_message")
|
||||
|
||||
|
||||
def test_gate_post_rejects_replay_via_sequence(monkeypatch):
|
||||
@@ -319,11 +290,11 @@ def test_gate_post_rejects_replay_via_sequence(monkeypatch):
|
||||
from services.mesh import mesh_hashchain
|
||||
|
||||
_patch_for_successful_post(monkeypatch, main)
|
||||
|
||||
def reject_private_append(**_kwargs):
|
||||
raise ValueError("Replay detected: sequence 1 <= last 1")
|
||||
|
||||
monkeypatch.setattr(mesh_hashchain.infonet, "append_private_gate_message", reject_private_append)
|
||||
monkeypatch.setattr(
|
||||
mesh_hashchain.infonet,
|
||||
"validate_and_set_sequence",
|
||||
lambda node_id, seq: (False, "Replay detected: sequence 1 <= last 1"),
|
||||
)
|
||||
|
||||
gate_id = "infonet"
|
||||
body = _build_gate_message_body(gate_id, sequence=1)
|
||||
|
||||
@@ -117,11 +117,3 @@ def test_finish_solo_sync_marks_first_node_ready_without_peer_failure():
|
||||
assert finished.next_sync_due_at == 500
|
||||
assert should_run_sync(finished, now=499) is False
|
||||
assert should_run_sync(finished, now=500) is True
|
||||
|
||||
|
||||
def test_should_run_sync_recovers_stale_running_state():
|
||||
fresh = SyncWorkerState(last_sync_started_at=100, last_outcome="running")
|
||||
stale = SyncWorkerState(last_sync_started_at=100, last_outcome="running")
|
||||
|
||||
assert should_run_sync(fresh, now=399) is False
|
||||
assert should_run_sync(stale, now=400) is True
|
||||
|
||||
@@ -8,53 +8,6 @@ from cryptography.hazmat.primitives.asymmetric import ed25519
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
|
||||
def test_onion_peer_requests_use_arti_socks_proxy(monkeypatch):
|
||||
import main
|
||||
from services import wormhole_supervisor
|
||||
|
||||
monkeypatch.setattr(main, "_infonet_private_transport_required", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
main,
|
||||
"get_settings",
|
||||
lambda: SimpleNamespace(MESH_ARTI_ENABLED=True, MESH_ARTI_SOCKS_PORT=19050),
|
||||
)
|
||||
monkeypatch.setattr(wormhole_supervisor, "_check_arti_ready", lambda: True)
|
||||
|
||||
proxies = main._infonet_peer_requests_proxies("http://exampleabcd.onion:8000")
|
||||
|
||||
assert proxies == {
|
||||
"http": "socks5h://127.0.0.1:19050",
|
||||
"https": "socks5h://127.0.0.1:19050",
|
||||
}
|
||||
|
||||
|
||||
def test_private_peer_requests_reject_clearnet(monkeypatch):
|
||||
import main
|
||||
|
||||
monkeypatch.setattr(main, "_infonet_private_transport_required", lambda: True)
|
||||
|
||||
try:
|
||||
main._infonet_peer_requests_proxies("https://seed.example")
|
||||
except RuntimeError as exc:
|
||||
assert "private Infonet requires onion/RNS transport" in str(exc)
|
||||
else:
|
||||
raise AssertionError("clearnet peer was allowed while private transport is required")
|
||||
|
||||
|
||||
def test_local_peer_url_prefers_configured_public_peer_url(monkeypatch):
|
||||
import main
|
||||
|
||||
monkeypatch.setattr(
|
||||
main,
|
||||
"get_settings",
|
||||
lambda: SimpleNamespace(
|
||||
MESH_PUBLIC_PEER_URL="HTTP://LOCALPEEREXAMPLE.onion:8000/",
|
||||
),
|
||||
)
|
||||
|
||||
assert main._local_infonet_peer_url() == "http://localpeerexample.onion:8000"
|
||||
|
||||
|
||||
def _write_signed_manifest(path, *, private_key):
|
||||
from services.mesh.mesh_bootstrap_manifest import BOOTSTRAP_MANIFEST_VERSION
|
||||
from services.mesh.mesh_crypto import canonical_json
|
||||
@@ -189,134 +142,6 @@ def test_refresh_node_peer_store_suppresses_clearnet_seed_by_default(tmp_path, m
|
||||
assert store.records_for_bucket("sync") == []
|
||||
|
||||
|
||||
def test_refresh_node_peer_store_prunes_persisted_clearnet_records_in_private_mode(tmp_path, monkeypatch):
|
||||
import main
|
||||
from services.config import get_settings
|
||||
from services.mesh import mesh_peer_store as peer_store_mod
|
||||
|
||||
peer_store_path = tmp_path / "peer_store.json"
|
||||
monkeypatch.setattr(peer_store_mod, "DEFAULT_PEER_STORE_PATH", peer_store_path)
|
||||
store = peer_store_mod.PeerStore(peer_store_path)
|
||||
store.upsert(
|
||||
peer_store_mod.make_bootstrap_peer_record(
|
||||
peer_url="https://node.shadowbroker.info",
|
||||
transport="clearnet",
|
||||
role="seed",
|
||||
signer_id="shadowbroker-default",
|
||||
now=1_749_999_900,
|
||||
)
|
||||
)
|
||||
store.upsert(
|
||||
peer_store_mod.make_sync_peer_record(
|
||||
peer_url="https://node.shadowbroker.info",
|
||||
transport="clearnet",
|
||||
role="seed",
|
||||
source="bundle",
|
||||
now=1_749_999_900,
|
||||
)
|
||||
)
|
||||
store.upsert(
|
||||
peer_store_mod.make_push_peer_record(
|
||||
peer_url="https://node.shadowbroker.info",
|
||||
transport="clearnet",
|
||||
role="relay",
|
||||
now=1_749_999_900,
|
||||
)
|
||||
)
|
||||
store.save()
|
||||
|
||||
onion_seed = "http://gqpbunqbgtkcqilvclm3xrkt3zowjyl3s62kkktvojgvxzizamvbrqid.onion:8000"
|
||||
monkeypatch.setenv("MESH_RELAY_PEERS", "")
|
||||
monkeypatch.setenv("MESH_BOOTSTRAP_SEED_PEERS", onion_seed)
|
||||
monkeypatch.setenv("MESH_DEFAULT_SYNC_PEERS", "")
|
||||
monkeypatch.delenv("MESH_INFONET_ALLOW_CLEARNET_SYNC", raising=False)
|
||||
monkeypatch.setenv("MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY", "")
|
||||
get_settings.cache_clear()
|
||||
|
||||
try:
|
||||
snapshot = main._refresh_node_peer_store(now=1_750_000_000)
|
||||
store = peer_store_mod.PeerStore(peer_store_path)
|
||||
store.load()
|
||||
finally:
|
||||
get_settings.cache_clear()
|
||||
|
||||
assert snapshot["private_transport_required"] is True
|
||||
assert snapshot["pruned_clearnet_peer_count"] == 3
|
||||
assert [record.peer_url for record in store.records()] == [onion_seed, onion_seed]
|
||||
assert {record.bucket for record in store.records()} == {"bootstrap", "sync"}
|
||||
assert all(record.transport == "onion" for record in store.records())
|
||||
|
||||
|
||||
def test_infonet_peer_url_filter_excludes_clearnet_in_private_mode(monkeypatch):
|
||||
import main
|
||||
from services.config import get_settings
|
||||
|
||||
monkeypatch.delenv("MESH_INFONET_ALLOW_CLEARNET_SYNC", raising=False)
|
||||
get_settings.cache_clear()
|
||||
|
||||
try:
|
||||
assert main._filter_infonet_peer_urls(
|
||||
[
|
||||
"https://node.shadowbroker.info",
|
||||
"http://gqpbunqbgtkcqilvclm3xrkt3zowjyl3s62kkktvojgvxzizamvbrqid.onion:8000",
|
||||
]
|
||||
) == ["http://gqpbunqbgtkcqilvclm3xrkt3zowjyl3s62kkktvojgvxzizamvbrqid.onion:8000"]
|
||||
finally:
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def test_public_sync_cycle_backs_off_on_429_retry_after(tmp_path, monkeypatch):
|
||||
import time
|
||||
|
||||
import main
|
||||
from services.config import get_settings
|
||||
from services.mesh import mesh_peer_store as peer_store_mod
|
||||
|
||||
peer_store_path = tmp_path / "peer_store.json"
|
||||
monkeypatch.setattr(peer_store_mod, "DEFAULT_PEER_STORE_PATH", peer_store_path)
|
||||
onion_seed = "http://gqpbunqbgtkcqilvclm3xrkt3zowjyl3s62kkktvojgvxzizamvbrqid.onion:8000"
|
||||
store = peer_store_mod.PeerStore(peer_store_path)
|
||||
store.upsert(
|
||||
peer_store_mod.make_sync_peer_record(
|
||||
peer_url=onion_seed,
|
||||
transport="onion",
|
||||
role="seed",
|
||||
source="bundle",
|
||||
now=1_750_000_000,
|
||||
)
|
||||
)
|
||||
store.save()
|
||||
|
||||
monkeypatch.delenv("MESH_INFONET_ALLOW_CLEARNET_SYNC", raising=False)
|
||||
monkeypatch.setenv("MESH_SYNC_FAILURE_BACKOFF_S", "60")
|
||||
monkeypatch.setenv("MESH_BOOTSTRAP_SEED_FAILURE_COOLDOWN_S", "15")
|
||||
get_settings.cache_clear()
|
||||
monkeypatch.setattr(main, "_participant_node_enabled", lambda: True)
|
||||
monkeypatch.setattr(main, "_ensure_infonet_private_transport_ready", lambda reason="": True)
|
||||
monkeypatch.setattr(
|
||||
main,
|
||||
"_sync_from_peer",
|
||||
lambda peer_url: (_ for _ in ()).throw(
|
||||
main.PeerSyncHTTPError(429, "rate limited", retry_after_s=180)
|
||||
),
|
||||
)
|
||||
main.set_sync_state(main.SyncWorkerState())
|
||||
|
||||
try:
|
||||
before = int(time.time())
|
||||
state = main._run_public_sync_cycle()
|
||||
store = peer_store_mod.PeerStore(peer_store_path)
|
||||
store.load()
|
||||
finally:
|
||||
get_settings.cache_clear()
|
||||
main.set_sync_state(main.SyncWorkerState())
|
||||
|
||||
record = store.records_for_bucket("sync")[0]
|
||||
assert state.last_error == "HTTP 429: rate limited"
|
||||
assert state.next_sync_due_at >= before + 180
|
||||
assert record.cooldown_until >= before + 180
|
||||
|
||||
|
||||
def test_verify_peer_push_hmac_requires_allowlisted_peer(monkeypatch):
|
||||
import hashlib
|
||||
import hmac
|
||||
@@ -400,29 +225,3 @@ def test_public_sync_cycle_allows_first_node_without_peers(tmp_path, monkeypatch
|
||||
assert result.last_error == ""
|
||||
assert result.last_peer_url == ""
|
||||
assert result.consecutive_failures == 0
|
||||
|
||||
|
||||
def test_headless_mesh_node_runtime_is_explicit(monkeypatch):
|
||||
import main
|
||||
|
||||
monkeypatch.setattr(main, "_MESH_ONLY", True)
|
||||
monkeypatch.setattr(main, "_HEADLESS_MESH_NODE_RUNTIME", False)
|
||||
assert main._infonet_node_runtime_requested() is False
|
||||
|
||||
monkeypatch.setattr(main, "_HEADLESS_MESH_NODE_RUNTIME", True)
|
||||
assert main._infonet_node_runtime_requested() is True
|
||||
|
||||
|
||||
def test_meshnode_scripts_enable_private_hashchain_runtime():
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(__file__).resolve().parents[3]
|
||||
bat = (root / "meshnode.bat").read_text(encoding="utf-8")
|
||||
sh = (root / "meshnode.sh").read_text(encoding="utf-8")
|
||||
|
||||
for script in (bat, sh):
|
||||
assert "SHADOWBROKER_MESH_NODE_RUNTIME=true" in script
|
||||
assert "MESH_INFONET_ALLOW_CLEARNET_SYNC=false" in script
|
||||
assert "MESH_ARTI_ENABLED=true" in script
|
||||
assert "MESH_DM_HASHCHAIN_SPOOL_LIMIT=2" in script
|
||||
assert "gqpbunqbgtkcqilvclm3xrkt3zowjyl3s62kkktvojgvxzizamvbrqid.onion:8000" in script
|
||||
|
||||
@@ -1,213 +0,0 @@
|
||||
import base64
|
||||
import time
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import ed25519
|
||||
|
||||
from services.config import get_settings
|
||||
from services.mesh import mesh_crypto, mesh_dm_relay, mesh_hashchain, mesh_protocol, mesh_secure_storage
|
||||
|
||||
|
||||
def _keypair():
|
||||
private_key = ed25519.Ed25519PrivateKey.generate()
|
||||
public_raw = private_key.public_key().public_bytes(
|
||||
encoding=serialization.Encoding.Raw,
|
||||
format=serialization.PublicFormat.Raw,
|
||||
)
|
||||
public_key = base64.b64encode(public_raw).decode("utf-8")
|
||||
node_id = mesh_crypto.derive_node_id(public_key)
|
||||
return private_key, public_key, node_id
|
||||
|
||||
|
||||
def _payload(recipient_id: str = "recipient-a", msg_id: str = "dm-1") -> dict:
|
||||
return mesh_protocol.normalize_payload(
|
||||
"dm_message",
|
||||
{
|
||||
"recipient_id": recipient_id,
|
||||
"delivery_class": "request",
|
||||
"recipient_token": "",
|
||||
"ciphertext": base64.b64encode(f"cipher-{msg_id}".encode("utf-8")).decode("ascii"),
|
||||
"msg_id": msg_id,
|
||||
"timestamp": int(time.time()),
|
||||
"format": "mls1",
|
||||
"transport_lock": "private_strong",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _signature(private_key, node_id: str, sequence: int, payload: dict) -> str:
|
||||
signature_payload = mesh_crypto.build_signature_payload(
|
||||
event_type="dm_message",
|
||||
node_id=node_id,
|
||||
sequence=sequence,
|
||||
payload=payload,
|
||||
)
|
||||
return private_key.sign(signature_payload.encode("utf-8")).hex()
|
||||
|
||||
|
||||
def _fresh_infonet(tmp_path, monkeypatch) -> mesh_hashchain.Infonet:
|
||||
monkeypatch.setattr(mesh_hashchain, "DATA_DIR", tmp_path)
|
||||
monkeypatch.setattr(mesh_hashchain, "CHAIN_FILE", tmp_path / "infonet.json")
|
||||
monkeypatch.setattr(mesh_hashchain, "WAL_FILE", tmp_path / "infonet.wal")
|
||||
return mesh_hashchain.Infonet()
|
||||
|
||||
|
||||
def _fresh_relay(tmp_path, monkeypatch) -> mesh_dm_relay.DMRelay:
|
||||
monkeypatch.setattr(mesh_dm_relay, "DATA_DIR", tmp_path)
|
||||
monkeypatch.setattr(mesh_dm_relay, "RELAY_FILE", tmp_path / "dm_relay.json")
|
||||
monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path)
|
||||
monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key")
|
||||
get_settings.cache_clear()
|
||||
return mesh_dm_relay.DMRelay()
|
||||
|
||||
|
||||
def test_private_dm_hashchain_spools_two_ciphertexts_per_recipient_from_distinct_senders(tmp_path, monkeypatch):
|
||||
inf = _fresh_infonet(tmp_path, monkeypatch)
|
||||
senders = [_keypair(), _keypair()]
|
||||
|
||||
for idx, (private_key, public_key, node_id) in enumerate(senders, start=1):
|
||||
payload = _payload(msg_id=f"dm-{idx}")
|
||||
event = inf.append_private_dm_message(
|
||||
node_id=node_id,
|
||||
payload=payload,
|
||||
signature=_signature(private_key, node_id, 1, payload),
|
||||
sequence=1,
|
||||
public_key=public_key,
|
||||
public_key_algo="Ed25519",
|
||||
protocol_version=mesh_protocol.PROTOCOL_VERSION,
|
||||
timestamp=float(payload["timestamp"]),
|
||||
)
|
||||
assert event["event_type"] == "dm_message"
|
||||
|
||||
private_key, public_key, node_id = _keypair()
|
||||
third = _payload(msg_id="dm-3")
|
||||
try:
|
||||
inf.append_private_dm_message(
|
||||
node_id=node_id,
|
||||
payload=third,
|
||||
signature=_signature(private_key, node_id, 1, third),
|
||||
sequence=1,
|
||||
public_key=public_key,
|
||||
public_key_algo="Ed25519",
|
||||
protocol_version=mesh_protocol.PROTOCOL_VERSION,
|
||||
timestamp=float(third["timestamp"]),
|
||||
)
|
||||
except ValueError as exc:
|
||||
assert "spool full" in str(exc)
|
||||
else:
|
||||
raise AssertionError("third DM spool event was accepted")
|
||||
|
||||
for _private_key, _public_key, sender_node_id in senders:
|
||||
assert inf.sequence_domains[f"{sender_node_id}|dm_message"] == 1
|
||||
assert inf.validate_chain(verify_signatures=True)[0] is True
|
||||
|
||||
|
||||
def test_private_dm_hashchain_limits_one_active_spool_per_sender_recipient_pair(tmp_path, monkeypatch):
|
||||
inf = _fresh_infonet(tmp_path, monkeypatch)
|
||||
private_key, public_key, node_id = _keypair()
|
||||
|
||||
first = _payload(msg_id="dm-1")
|
||||
inf.append_private_dm_message(
|
||||
node_id=node_id,
|
||||
payload=first,
|
||||
signature=_signature(private_key, node_id, 1, first),
|
||||
sequence=1,
|
||||
public_key=public_key,
|
||||
public_key_algo="Ed25519",
|
||||
protocol_version=mesh_protocol.PROTOCOL_VERSION,
|
||||
timestamp=float(first["timestamp"]),
|
||||
)
|
||||
|
||||
second = _payload(msg_id="dm-2")
|
||||
try:
|
||||
inf.append_private_dm_message(
|
||||
node_id=node_id,
|
||||
payload=second,
|
||||
signature=_signature(private_key, node_id, 2, second),
|
||||
sequence=2,
|
||||
public_key=public_key,
|
||||
public_key_algo="Ed25519",
|
||||
protocol_version=mesh_protocol.PROTOCOL_VERSION,
|
||||
timestamp=float(second["timestamp"]),
|
||||
)
|
||||
except ValueError as exc:
|
||||
assert "sender spool full" in str(exc)
|
||||
else:
|
||||
raise AssertionError("second DM from same sender to same recipient was accepted")
|
||||
|
||||
|
||||
def test_private_dm_hashchain_rejects_plaintext(tmp_path, monkeypatch):
|
||||
inf = _fresh_infonet(tmp_path, monkeypatch)
|
||||
private_key, public_key, node_id = _keypair()
|
||||
payload = _payload()
|
||||
payload["message"] = "plaintext"
|
||||
|
||||
try:
|
||||
inf.append_private_dm_message(
|
||||
node_id=node_id,
|
||||
payload=payload,
|
||||
signature=_signature(private_key, node_id, 1, _payload()),
|
||||
sequence=1,
|
||||
public_key=public_key,
|
||||
public_key_algo="Ed25519",
|
||||
protocol_version=mesh_protocol.PROTOCOL_VERSION,
|
||||
)
|
||||
except ValueError as exc:
|
||||
assert "plaintext" in str(exc)
|
||||
else:
|
||||
raise AssertionError("private DM append accepted plaintext")
|
||||
|
||||
|
||||
def test_private_dm_hashchain_rejects_non_sealed_ciphertext_shape(tmp_path, monkeypatch):
|
||||
inf = _fresh_infonet(tmp_path, monkeypatch)
|
||||
private_key, public_key, node_id = _keypair()
|
||||
payload = _payload()
|
||||
payload["ciphertext"] = "not sealed plaintext"
|
||||
|
||||
try:
|
||||
inf.append_private_dm_message(
|
||||
node_id=node_id,
|
||||
payload=payload,
|
||||
signature=_signature(private_key, node_id, 1, payload),
|
||||
sequence=1,
|
||||
public_key=public_key,
|
||||
public_key_algo="Ed25519",
|
||||
protocol_version=mesh_protocol.PROTOCOL_VERSION,
|
||||
)
|
||||
except ValueError as exc:
|
||||
assert "sealed bytes" in str(exc)
|
||||
else:
|
||||
raise AssertionError("private DM append accepted non-base64 ciphertext")
|
||||
|
||||
|
||||
def test_hydrate_dm_relay_from_chain_delivers_to_poll_claim(tmp_path, monkeypatch):
|
||||
inf = _fresh_infonet(tmp_path / "chain", monkeypatch)
|
||||
relay = _fresh_relay(tmp_path / "relay", monkeypatch)
|
||||
monkeypatch.setattr(mesh_hashchain, "infonet", inf)
|
||||
monkeypatch.setattr(mesh_dm_relay, "dm_relay", relay)
|
||||
|
||||
private_key, public_key, node_id = _keypair()
|
||||
payload = _payload(recipient_id="recipient-a", msg_id="dm-chain-1")
|
||||
event = inf.append_private_dm_message(
|
||||
node_id=node_id,
|
||||
payload=payload,
|
||||
signature=_signature(private_key, node_id, 1, payload),
|
||||
sequence=1,
|
||||
public_key=public_key,
|
||||
public_key_algo="Ed25519",
|
||||
protocol_version=mesh_protocol.PROTOCOL_VERSION,
|
||||
timestamp=float(payload["timestamp"]),
|
||||
)
|
||||
|
||||
from main import _hydrate_dm_relay_from_chain
|
||||
|
||||
assert _hydrate_dm_relay_from_chain([event]) == 1
|
||||
messages, more = relay.collect_claims(
|
||||
"recipient-a",
|
||||
[{"type": "requests", "token": "recipient-request-token"}],
|
||||
limit=8,
|
||||
)
|
||||
|
||||
assert more is False
|
||||
assert [message["msg_id"] for message in messages] == ["dm-chain-1"]
|
||||
assert messages[0]["ciphertext"] == payload["ciphertext"]
|
||||
@@ -1,269 +0,0 @@
|
||||
import base64
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import ed25519
|
||||
|
||||
from services.mesh import mesh_crypto, mesh_hashchain, mesh_protocol
|
||||
|
||||
|
||||
def _keypair():
|
||||
private_key = ed25519.Ed25519PrivateKey.generate()
|
||||
public_raw = private_key.public_key().public_bytes(
|
||||
encoding=serialization.Encoding.Raw,
|
||||
format=serialization.PublicFormat.Raw,
|
||||
)
|
||||
public_key = base64.b64encode(public_raw).decode("utf-8")
|
||||
node_id = mesh_crypto.derive_node_id(public_key)
|
||||
return private_key, public_key, node_id
|
||||
|
||||
|
||||
def _sign(private_key, *, event_type: str, node_id: str, sequence: int, payload: dict) -> str:
|
||||
signature_payload = mesh_crypto.build_signature_payload(
|
||||
event_type=event_type,
|
||||
node_id=node_id,
|
||||
sequence=sequence,
|
||||
payload=payload,
|
||||
)
|
||||
return private_key.sign(signature_payload.encode("utf-8")).hex()
|
||||
|
||||
|
||||
def _message_payload(text: str) -> dict:
|
||||
return mesh_protocol.normalize_payload(
|
||||
"message",
|
||||
{
|
||||
"message": text,
|
||||
"destination": "broadcast",
|
||||
"channel": "LongFast",
|
||||
"priority": "normal",
|
||||
"ephemeral": False,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _gate_payload(gate_id: str = "ops-gate", *, epoch: int = 2, plaintext: bool = False) -> dict:
|
||||
payload = {
|
||||
"gate": gate_id,
|
||||
"ciphertext": base64.b64encode(b"encrypted-gate-ciphertext").decode("ascii"),
|
||||
"nonce": base64.b64encode(b"nonce-value-1234").decode("ascii"),
|
||||
"sender_ref": "sender-ref-1",
|
||||
"format": "mls1",
|
||||
"transport_lock": "private_strong",
|
||||
}
|
||||
if epoch > 0:
|
||||
payload["epoch"] = epoch
|
||||
if plaintext:
|
||||
payload["message"] = "this must never land on the chain"
|
||||
return mesh_protocol.normalize_payload("gate_message", payload) if not plaintext else payload
|
||||
|
||||
|
||||
def _gate_event(
|
||||
private_key,
|
||||
public_key: str,
|
||||
node_id: str,
|
||||
*,
|
||||
sequence: int,
|
||||
prev_hash: str,
|
||||
payload: dict,
|
||||
signature_payload: dict | None = None,
|
||||
) -> dict:
|
||||
signature = _sign(
|
||||
private_key,
|
||||
event_type="gate_message",
|
||||
node_id=node_id,
|
||||
sequence=sequence,
|
||||
payload=signature_payload or payload,
|
||||
)
|
||||
return mesh_hashchain.ChainEvent(
|
||||
prev_hash=prev_hash,
|
||||
event_type="gate_message",
|
||||
node_id=node_id,
|
||||
payload=payload,
|
||||
timestamp=1234.0 + sequence,
|
||||
sequence=sequence,
|
||||
signature=signature,
|
||||
public_key=public_key,
|
||||
public_key_algo="Ed25519",
|
||||
protocol_version=mesh_protocol.PROTOCOL_VERSION,
|
||||
network_id=mesh_protocol.NETWORK_ID,
|
||||
).to_dict()
|
||||
|
||||
|
||||
def _fresh_infonet(tmp_path, monkeypatch) -> mesh_hashchain.Infonet:
|
||||
monkeypatch.setattr(mesh_hashchain, "DATA_DIR", tmp_path)
|
||||
monkeypatch.setattr(mesh_hashchain, "CHAIN_FILE", tmp_path / "infonet.json")
|
||||
monkeypatch.setattr(mesh_hashchain, "WAL_FILE", tmp_path / "infonet.wal")
|
||||
return mesh_hashchain.Infonet()
|
||||
|
||||
|
||||
def test_private_gate_fork_uses_gate_sequence_domain_and_signature_variants(tmp_path, monkeypatch):
|
||||
inf = _fresh_infonet(tmp_path, monkeypatch)
|
||||
private_key, public_key, node_id = _keypair()
|
||||
|
||||
public_payload = _message_payload("public prefix")
|
||||
public_event = inf.append(
|
||||
event_type="message",
|
||||
node_id=node_id,
|
||||
payload=public_payload,
|
||||
sequence=1,
|
||||
signature=_sign(
|
||||
private_key,
|
||||
event_type="message",
|
||||
node_id=node_id,
|
||||
sequence=1,
|
||||
payload=public_payload,
|
||||
),
|
||||
public_key=public_key,
|
||||
public_key_algo="Ed25519",
|
||||
protocol_version=mesh_protocol.PROTOCOL_VERSION,
|
||||
)
|
||||
|
||||
gate_payload = _gate_payload(epoch=3)
|
||||
signature_payload = dict(gate_payload)
|
||||
signature_payload.pop("epoch", None)
|
||||
gate_event = _gate_event(
|
||||
private_key,
|
||||
public_key,
|
||||
node_id,
|
||||
sequence=1,
|
||||
prev_hash=public_event["event_id"],
|
||||
payload=gate_payload,
|
||||
signature_payload=signature_payload,
|
||||
)
|
||||
|
||||
ok, reason = inf.apply_fork([gate_event], gate_event["event_id"], proof_count=2, quorum=2)
|
||||
|
||||
assert ok is True, reason
|
||||
assert inf.events[-1]["event_type"] == "gate_message"
|
||||
assert inf.node_sequences[node_id] == 1
|
||||
assert inf.sequence_domains[f"{node_id}|gate_message"] == 1
|
||||
assert inf.validate_chain(verify_signatures=True)[0] is True
|
||||
|
||||
|
||||
def test_private_gate_fork_rejects_plaintext_payload(tmp_path, monkeypatch):
|
||||
inf = _fresh_infonet(tmp_path, monkeypatch)
|
||||
private_key, public_key, node_id = _keypair()
|
||||
|
||||
public_payload = _message_payload("public prefix")
|
||||
public_event = inf.append(
|
||||
event_type="message",
|
||||
node_id=node_id,
|
||||
payload=public_payload,
|
||||
sequence=1,
|
||||
signature=_sign(
|
||||
private_key,
|
||||
event_type="message",
|
||||
node_id=node_id,
|
||||
sequence=1,
|
||||
payload=public_payload,
|
||||
),
|
||||
public_key=public_key,
|
||||
public_key_algo="Ed25519",
|
||||
protocol_version=mesh_protocol.PROTOCOL_VERSION,
|
||||
)
|
||||
|
||||
plaintext_payload = _gate_payload(plaintext=True)
|
||||
gate_event = _gate_event(
|
||||
private_key,
|
||||
public_key,
|
||||
node_id,
|
||||
sequence=1,
|
||||
prev_hash=public_event["event_id"],
|
||||
payload=plaintext_payload,
|
||||
)
|
||||
|
||||
ok, reason = inf.apply_fork([gate_event], gate_event["event_id"], proof_count=2, quorum=2)
|
||||
|
||||
assert ok is False
|
||||
assert "normalized" in reason or "plaintext" in reason
|
||||
assert len(inf.events) == 1
|
||||
assert "gate_message" not in inf.get_info()["event_types"]
|
||||
|
||||
|
||||
def test_append_private_gate_message_rejects_plaintext_before_normalizing(tmp_path, monkeypatch):
|
||||
inf = _fresh_infonet(tmp_path, monkeypatch)
|
||||
private_key, public_key, node_id = _keypair()
|
||||
payload = _gate_payload()
|
||||
payload["message"] = "plaintext should not be silently dropped"
|
||||
|
||||
try:
|
||||
inf.append_private_gate_message(
|
||||
node_id=node_id,
|
||||
payload=payload,
|
||||
sequence=1,
|
||||
signature=_sign(
|
||||
private_key,
|
||||
event_type="gate_message",
|
||||
node_id=node_id,
|
||||
sequence=1,
|
||||
payload=_gate_payload(),
|
||||
),
|
||||
public_key=public_key,
|
||||
public_key_algo="Ed25519",
|
||||
protocol_version=mesh_protocol.PROTOCOL_VERSION,
|
||||
)
|
||||
except ValueError as exc:
|
||||
assert "plaintext" in str(exc)
|
||||
else:
|
||||
raise AssertionError("private gate append accepted plaintext")
|
||||
|
||||
assert inf.events == []
|
||||
|
||||
|
||||
def test_append_private_gate_message_requires_private_strong_transport_lock(tmp_path, monkeypatch):
|
||||
inf = _fresh_infonet(tmp_path, monkeypatch)
|
||||
private_key, public_key, node_id = _keypair()
|
||||
payload = _gate_payload()
|
||||
payload.pop("transport_lock", None)
|
||||
|
||||
try:
|
||||
inf.append_private_gate_message(
|
||||
node_id=node_id,
|
||||
payload=payload,
|
||||
sequence=1,
|
||||
signature=_sign(
|
||||
private_key,
|
||||
event_type="gate_message",
|
||||
node_id=node_id,
|
||||
sequence=1,
|
||||
payload=_gate_payload(),
|
||||
),
|
||||
public_key=public_key,
|
||||
public_key_algo="Ed25519",
|
||||
protocol_version=mesh_protocol.PROTOCOL_VERSION,
|
||||
)
|
||||
except ValueError as exc:
|
||||
assert "private_strong" in str(exc)
|
||||
else:
|
||||
raise AssertionError("private gate append accepted missing transport_lock")
|
||||
|
||||
assert inf.events == []
|
||||
|
||||
|
||||
def test_append_private_gate_message_rejects_non_sealed_ciphertext_shape(tmp_path, monkeypatch):
|
||||
inf = _fresh_infonet(tmp_path, monkeypatch)
|
||||
private_key, public_key, node_id = _keypair()
|
||||
payload = _gate_payload()
|
||||
payload["ciphertext"] = "not sealed plaintext"
|
||||
|
||||
try:
|
||||
inf.append_private_gate_message(
|
||||
node_id=node_id,
|
||||
payload=payload,
|
||||
sequence=1,
|
||||
signature=_sign(
|
||||
private_key,
|
||||
event_type="gate_message",
|
||||
node_id=node_id,
|
||||
sequence=1,
|
||||
payload=payload,
|
||||
),
|
||||
public_key=public_key,
|
||||
public_key_algo="Ed25519",
|
||||
protocol_version=mesh_protocol.PROTOCOL_VERSION,
|
||||
)
|
||||
except ValueError as exc:
|
||||
assert "sealed bytes" in str(exc)
|
||||
else:
|
||||
raise AssertionError("private gate append accepted non-base64 ciphertext")
|
||||
|
||||
assert inf.events == []
|
||||
@@ -1,12 +1,14 @@
|
||||
"""S14B private sync gate event policy.
|
||||
"""S14B Public Sync Gate Event Filter.
|
||||
|
||||
Private Infonet sync carries encrypted gate_message ledger events. If a node
|
||||
is configured to allow clearnet-compatible sync, those gate events are filtered
|
||||
out of the sync response.
|
||||
Tests:
|
||||
- GET /api/mesh/infonet/sync excludes gate_message when local infonet contains legacy gate_message plus public events
|
||||
- POST /api/mesh/infonet/sync excludes gate_message under the same condition
|
||||
- Both main app and router-served paths are covered
|
||||
- Non-gate public redactions still hold (vote gate label stripped, key_rotate identity stripped)
|
||||
- Do not overclaim that gate_message is removed from historical infonet storage or ingest
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
|
||||
from starlette.requests import Request
|
||||
@@ -15,6 +17,9 @@ import main
|
||||
from services.mesh import mesh_hashchain
|
||||
|
||||
|
||||
# ── Helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _message_event() -> dict:
|
||||
return {
|
||||
"event_id": "msg-1",
|
||||
@@ -78,7 +83,6 @@ def _gate_message_event() -> dict:
|
||||
"nonce": "nonce-1",
|
||||
"sender_ref": "sender-ref-1",
|
||||
"format": "mls1",
|
||||
"transport_lock": "private_strong",
|
||||
},
|
||||
"timestamp": 103.0,
|
||||
"sequence": 4,
|
||||
@@ -89,31 +93,9 @@ def _gate_message_event() -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _dm_message_event() -> dict:
|
||||
return {
|
||||
"event_id": "dm-1",
|
||||
"event_type": "dm_message",
|
||||
"node_id": "!node-5",
|
||||
"payload": {
|
||||
"recipient_id": "recipient-a",
|
||||
"delivery_class": "request",
|
||||
"recipient_token": "",
|
||||
"ciphertext": base64.b64encode(b"sealed-dm-ciphertext").decode("ascii"),
|
||||
"msg_id": "dm-1",
|
||||
"timestamp": 104,
|
||||
"format": "mls1",
|
||||
"transport_lock": "private_strong",
|
||||
},
|
||||
"timestamp": 104.0,
|
||||
"sequence": 5,
|
||||
"signature": "sig",
|
||||
"public_key": "pub",
|
||||
"public_key_algo": "Ed25519",
|
||||
"protocol_version": "infonet/2",
|
||||
}
|
||||
|
||||
|
||||
class _FakeInfonet:
|
||||
"""Minimal fake infonet with a gate_message among public events."""
|
||||
|
||||
def __init__(self):
|
||||
self.head_hash = "head-1"
|
||||
self.events = [
|
||||
@@ -131,10 +113,12 @@ class _FakeInfonet:
|
||||
return int(getattr(limit, "default", 100) or 100)
|
||||
|
||||
def get_events_after(self, after_hash: str, limit=100):
|
||||
return [dict(e) for e in self.events[: self._limit_value(limit)]]
|
||||
resolved = self._limit_value(limit)
|
||||
return [dict(e) for e in self.events[:resolved]]
|
||||
|
||||
def get_events_after_locator(self, locator: list[str], limit=100):
|
||||
return self.head_hash, 0, [dict(e) for e in self.events[: self._limit_value(limit)]]
|
||||
resolved = self._limit_value(limit)
|
||||
return self.head_hash, 0, [dict(e) for e in self.events[:resolved]]
|
||||
|
||||
def get_merkle_proofs(self, start_index: int, count: int):
|
||||
return {"root": "merkle-root", "total": len(self.events), "start": start_index, "proofs": []}
|
||||
@@ -143,7 +127,7 @@ class _FakeInfonet:
|
||||
return "merkle-root"
|
||||
|
||||
|
||||
def _json_request(path: str, body: dict, *, client_host: str = "127.0.0.1", headers: dict[str, str] | None = None) -> Request:
|
||||
def _json_request(path: str, body: dict) -> Request:
|
||||
payload = json.dumps(body).encode("utf-8")
|
||||
sent = {"value": False}
|
||||
|
||||
@@ -153,14 +137,11 @@ def _json_request(path: str, body: dict, *, client_host: str = "127.0.0.1", head
|
||||
sent["value"] = True
|
||||
return {"type": "http.request", "body": payload, "more_body": False}
|
||||
|
||||
raw_headers = [(b"content-type", b"application/json")]
|
||||
for key, value in dict(headers or {}).items():
|
||||
raw_headers.append((key.lower().encode("ascii"), str(value).encode("ascii")))
|
||||
return Request(
|
||||
{
|
||||
"type": "http",
|
||||
"headers": raw_headers,
|
||||
"client": (client_host, 12345),
|
||||
"headers": [(b"content-type", b"application/json")],
|
||||
"client": ("test", 12345),
|
||||
"method": "POST",
|
||||
"path": path,
|
||||
},
|
||||
@@ -168,15 +149,20 @@ def _json_request(path: str, body: dict, *, client_host: str = "127.0.0.1", head
|
||||
)
|
||||
|
||||
|
||||
def _get_request(path: str, *, client_host: str = "127.0.0.1", headers: dict[str, str] | None = None) -> Request:
|
||||
def _get_request(path: str) -> Request:
|
||||
sent = {"value": False}
|
||||
|
||||
async def receive():
|
||||
if sent["value"]:
|
||||
return {"type": "http.request", "body": b"", "more_body": False}
|
||||
sent["value"] = True
|
||||
return {"type": "http.request", "body": b"", "more_body": False}
|
||||
|
||||
return Request(
|
||||
{
|
||||
"type": "http",
|
||||
"headers": [(key.lower().encode("ascii"), str(value).encode("ascii")) for key, value in dict(headers or {}).items()],
|
||||
"client": (client_host, 12345),
|
||||
"headers": [],
|
||||
"client": ("test", 12345),
|
||||
"method": "GET",
|
||||
"path": path,
|
||||
},
|
||||
@@ -184,166 +170,120 @@ def _get_request(path: str, *, client_host: str = "127.0.0.1", headers: dict[str
|
||||
)
|
||||
|
||||
|
||||
def _force_private_sync(monkeypatch):
|
||||
monkeypatch.setattr(main, "_infonet_private_transport_required", lambda: True)
|
||||
monkeypatch.setattr(main, "_request_appears_private_infonet_transport", lambda request: True)
|
||||
# ── GET sync excludes gate_message (main app) ──────────────────────────
|
||||
|
||||
|
||||
def _force_private_policy_only(monkeypatch):
|
||||
monkeypatch.setattr(main, "_infonet_private_transport_required", lambda: True)
|
||||
|
||||
|
||||
def _force_clearnet_sync(monkeypatch):
|
||||
monkeypatch.setattr(main, "_infonet_private_transport_required", lambda: False)
|
||||
|
||||
|
||||
def _event_types(events: list[dict]) -> list[str]:
|
||||
return [str(e.get("event_type", "")) for e in events]
|
||||
|
||||
|
||||
def test_private_sync_redacts_private_events_from_exposed_clearnet_request(monkeypatch):
|
||||
_force_private_policy_only(monkeypatch)
|
||||
request = _get_request("/api/mesh/infonet/sync", client_host="203.0.113.10")
|
||||
|
||||
events = main._infonet_sync_response_events(
|
||||
[_message_event(), _gate_message_event(), _dm_message_event()],
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert _event_types(events) == ["message"]
|
||||
|
||||
|
||||
def test_private_sync_includes_private_events_for_loopback_request(monkeypatch):
|
||||
_force_private_policy_only(monkeypatch)
|
||||
request = _get_request("/api/mesh/infonet/sync", client_host="127.0.0.1")
|
||||
|
||||
events = main._infonet_sync_response_events(
|
||||
[_message_event(), _gate_message_event(), _dm_message_event()],
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert _event_types(events) == ["message", "gate_message", "dm_message"]
|
||||
|
||||
|
||||
def test_private_sync_redacts_private_events_when_forwarded_for_is_clearnet(monkeypatch):
|
||||
_force_private_policy_only(monkeypatch)
|
||||
request = _get_request(
|
||||
"/api/mesh/infonet/sync",
|
||||
client_host="127.0.0.1",
|
||||
headers={"x-forwarded-for": "198.51.100.44"},
|
||||
)
|
||||
|
||||
events = main._infonet_sync_response_events(
|
||||
[_message_event(), _gate_message_event(), _dm_message_event()],
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert _event_types(events) == ["message"]
|
||||
|
||||
|
||||
def test_get_sync_includes_gate_message_on_private_transport(client, monkeypatch):
|
||||
_force_private_sync(monkeypatch)
|
||||
def test_get_sync_excludes_gate_message(client, monkeypatch):
|
||||
"""GET /api/mesh/infonet/sync must not return gate_message events."""
|
||||
monkeypatch.setattr(mesh_hashchain, "infonet", _FakeInfonet(), raising=False)
|
||||
|
||||
data = client.get("/api/mesh/infonet/sync").json()
|
||||
|
||||
assert "gate_message" in _event_types(data["events"])
|
||||
assert data["count"] == 4
|
||||
resp = client.get("/api/mesh/infonet/sync")
|
||||
data = resp.json()
|
||||
event_types = [e["event_type"] for e in data["events"]]
|
||||
assert "gate_message" not in event_types
|
||||
assert "message" in event_types
|
||||
assert "vote" in event_types
|
||||
assert "key_rotate" in event_types
|
||||
|
||||
|
||||
def test_post_sync_includes_gate_message_on_private_transport(monkeypatch):
|
||||
_force_private_sync(monkeypatch)
|
||||
def test_get_sync_count_excludes_gate_message(client, monkeypatch):
|
||||
"""GET sync count field must reflect filtered events (gate_message excluded)."""
|
||||
monkeypatch.setattr(mesh_hashchain, "infonet", _FakeInfonet(), raising=False)
|
||||
resp = client.get("/api/mesh/infonet/sync")
|
||||
data = resp.json()
|
||||
assert data["count"] == 3 # message, vote, key_rotate — not gate_message
|
||||
|
||||
|
||||
# ── POST sync excludes gate_message (main app) ─────────────────────────
|
||||
|
||||
|
||||
def test_post_sync_excludes_gate_message(monkeypatch):
|
||||
"""POST /api/mesh/infonet/sync must not return gate_message events."""
|
||||
monkeypatch.setattr(mesh_hashchain, "infonet", _FakeInfonet(), raising=False)
|
||||
result = asyncio.run(
|
||||
main.infonet_sync_post(
|
||||
_json_request("/api/mesh/infonet/sync", {"locator": ["head-1"]})
|
||||
)
|
||||
)
|
||||
|
||||
assert "gate_message" in _event_types(result["events"])
|
||||
assert result["count"] == 4
|
||||
event_types = [e["event_type"] for e in result["events"]]
|
||||
assert "gate_message" not in event_types
|
||||
assert "message" in event_types
|
||||
assert "vote" in event_types
|
||||
assert "key_rotate" in event_types
|
||||
|
||||
|
||||
def test_router_get_sync_includes_gate_message_on_private_transport(monkeypatch):
|
||||
def test_post_sync_count_excludes_gate_message(monkeypatch):
|
||||
"""POST sync count field must reflect filtered events."""
|
||||
monkeypatch.setattr(mesh_hashchain, "infonet", _FakeInfonet(), raising=False)
|
||||
result = asyncio.run(
|
||||
main.infonet_sync_post(
|
||||
_json_request("/api/mesh/infonet/sync", {"locator": ["head-1"]})
|
||||
)
|
||||
)
|
||||
assert result["count"] == 3
|
||||
|
||||
|
||||
# ── Router-served paths ────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_router_get_sync_excludes_gate_message(monkeypatch):
|
||||
"""Router GET /api/mesh/infonet/sync must not return gate_message."""
|
||||
from routers.mesh_public import infonet_sync
|
||||
|
||||
_force_private_sync(monkeypatch)
|
||||
monkeypatch.setattr(mesh_hashchain, "infonet", _FakeInfonet(), raising=False)
|
||||
|
||||
result = asyncio.run(infonet_sync(_get_request("/api/mesh/infonet/sync")))
|
||||
|
||||
assert "gate_message" in _event_types(result["events"])
|
||||
assert result["count"] == len(result["events"])
|
||||
event_types = [e["event_type"] for e in result["events"]]
|
||||
assert "gate_message" not in event_types
|
||||
assert "message" in event_types
|
||||
assert data_count_matches(result)
|
||||
|
||||
|
||||
def test_router_post_sync_includes_gate_message_on_private_transport(monkeypatch):
|
||||
def test_router_post_sync_excludes_gate_message(monkeypatch):
|
||||
"""Router POST /api/mesh/infonet/sync must not return gate_message."""
|
||||
from routers.mesh_public import infonet_sync_post
|
||||
|
||||
_force_private_sync(monkeypatch)
|
||||
monkeypatch.setattr(mesh_hashchain, "infonet", _FakeInfonet(), raising=False)
|
||||
|
||||
result = asyncio.run(
|
||||
infonet_sync_post(
|
||||
_json_request("/api/mesh/infonet/sync", {"locator": ["head-1"]})
|
||||
)
|
||||
)
|
||||
|
||||
assert "gate_message" in _event_types(result["events"])
|
||||
assert result["count"] == len(result["events"])
|
||||
event_types = [e["event_type"] for e in result["events"]]
|
||||
assert "gate_message" not in event_types
|
||||
assert "message" in event_types
|
||||
assert data_count_matches(result)
|
||||
|
||||
|
||||
def test_get_sync_excludes_gate_message_when_clearnet_sync_allowed(client, monkeypatch):
|
||||
_force_clearnet_sync(monkeypatch)
|
||||
monkeypatch.setattr(mesh_hashchain, "infonet", _FakeInfonet(), raising=False)
|
||||
|
||||
data = client.get("/api/mesh/infonet/sync").json()
|
||||
|
||||
assert "gate_message" not in _event_types(data["events"])
|
||||
assert data["count"] == 3
|
||||
def data_count_matches(result: dict) -> bool:
|
||||
return result["count"] == len(result["events"])
|
||||
|
||||
|
||||
def test_post_sync_excludes_gate_message_when_clearnet_sync_allowed(monkeypatch):
|
||||
_force_clearnet_sync(monkeypatch)
|
||||
monkeypatch.setattr(mesh_hashchain, "infonet", _FakeInfonet(), raising=False)
|
||||
|
||||
result = asyncio.run(
|
||||
main.infonet_sync_post(
|
||||
_json_request("/api/mesh/infonet/sync", {"locator": ["head-1"]})
|
||||
)
|
||||
)
|
||||
|
||||
assert "gate_message" not in _event_types(result["events"])
|
||||
assert result["count"] == 3
|
||||
# ── Non-gate redactions still hold ─────────────────────────────────────
|
||||
|
||||
|
||||
def test_get_sync_still_redacts_vote_gate_label(client, monkeypatch):
|
||||
_force_private_sync(monkeypatch)
|
||||
"""Public sync must still strip gate label from vote payload."""
|
||||
monkeypatch.setattr(mesh_hashchain, "infonet", _FakeInfonet(), raising=False)
|
||||
|
||||
events = client.get("/api/mesh/infonet/sync").json()["events"]
|
||||
resp = client.get("/api/mesh/infonet/sync")
|
||||
events = resp.json()["events"]
|
||||
vote = next(e for e in events if e["event_type"] == "vote")
|
||||
|
||||
assert "gate" not in vote.get("payload", {})
|
||||
|
||||
|
||||
def test_get_sync_still_redacts_key_rotate_identity(client, monkeypatch):
|
||||
_force_private_sync(monkeypatch)
|
||||
"""Public sync must still strip old identity fields from key_rotate payload."""
|
||||
monkeypatch.setattr(mesh_hashchain, "infonet", _FakeInfonet(), raising=False)
|
||||
|
||||
events = client.get("/api/mesh/infonet/sync").json()["events"]
|
||||
resp = client.get("/api/mesh/infonet/sync")
|
||||
events = resp.json()["events"]
|
||||
rotate = next(e for e in events if e["event_type"] == "key_rotate")
|
||||
payload = rotate.get("payload", {})
|
||||
|
||||
assert "old_node_id" not in payload
|
||||
assert "old_public_key" not in payload
|
||||
assert "old_signature" not in payload
|
||||
|
||||
|
||||
def test_post_sync_still_redacts_vote_and_rotate(monkeypatch):
|
||||
_force_private_sync(monkeypatch)
|
||||
"""POST sync must still apply standard public redactions to non-gate events."""
|
||||
monkeypatch.setattr(mesh_hashchain, "infonet", _FakeInfonet(), raising=False)
|
||||
|
||||
result = asyncio.run(
|
||||
main.infonet_sync_post(
|
||||
_json_request("/api/mesh/infonet/sync", {"locator": ["head-1"]})
|
||||
@@ -351,17 +291,24 @@ def test_post_sync_still_redacts_vote_and_rotate(monkeypatch):
|
||||
)
|
||||
vote = next(e for e in result["events"] if e["event_type"] == "vote")
|
||||
rotate = next(e for e in result["events"] if e["event_type"] == "key_rotate")
|
||||
|
||||
assert "gate" not in vote.get("payload", {})
|
||||
assert "old_node_id" not in rotate.get("payload", {})
|
||||
|
||||
|
||||
# ── No overclaim ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_gate_message_still_in_fake_infonet_storage():
|
||||
"""The filter does NOT remove gate_message from underlying storage.
|
||||
This test documents that the infonet still holds gate_message events;
|
||||
only the public sync response surface filters them out."""
|
||||
fake = _FakeInfonet()
|
||||
assert "gate_message" in _event_types(fake.events)
|
||||
all_types = [e["event_type"] for e in fake.events]
|
||||
assert "gate_message" in all_types
|
||||
|
||||
|
||||
def test_private_sync_with_only_gate_messages_returns_gate_events(client, monkeypatch):
|
||||
def test_sync_with_only_gate_messages_returns_empty(client, monkeypatch):
|
||||
"""If infonet contains only gate_message events, sync returns empty list."""
|
||||
class _GateOnlyInfonet:
|
||||
head_hash = "head-1"
|
||||
events = [_gate_message_event()]
|
||||
@@ -378,10 +325,8 @@ def test_private_sync_with_only_gate_messages_returns_gate_events(client, monkey
|
||||
def get_merkle_root(self):
|
||||
return "r"
|
||||
|
||||
_force_private_sync(monkeypatch)
|
||||
monkeypatch.setattr(mesh_hashchain, "infonet", _GateOnlyInfonet(), raising=False)
|
||||
|
||||
data = client.get("/api/mesh/infonet/sync").json()
|
||||
|
||||
assert _event_types(data["events"]) == ["gate_message"]
|
||||
assert data["count"] == 1
|
||||
resp = client.get("/api/mesh/infonet/sync")
|
||||
data = resp.json()
|
||||
assert data["events"] == []
|
||||
assert data["count"] == 0
|
||||
|
||||
@@ -66,20 +66,6 @@ def _make_gate_message_event(priv, pub_b64, node_id, sequence, prev_hash, gate_i
|
||||
return evt.to_dict()
|
||||
|
||||
|
||||
def _make_gate_payload(gate_id="test-gate") -> dict:
|
||||
return mesh_protocol.normalize_payload(
|
||||
"gate_message",
|
||||
{
|
||||
"gate": gate_id,
|
||||
"ciphertext": base64.b64encode(b"encrypted-data").decode(),
|
||||
"nonce": base64.b64encode(b"nonce-value-1234").decode(),
|
||||
"sender_ref": "sender-abc",
|
||||
"format": "mls1",
|
||||
"transport_lock": "private_strong",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def fresh_env(tmp_path, monkeypatch):
|
||||
"""Set up isolated infonet + gate_store, return (infonet, gate_store)."""
|
||||
@@ -103,74 +89,6 @@ def fresh_env(tmp_path, monkeypatch):
|
||||
# ── Rejected gate_message must NOT hydrate gate_store ─────────────────────
|
||||
|
||||
|
||||
def test_append_private_gate_message_uses_hashchain_gate_sequence(fresh_env):
|
||||
"""Local gate posts become private hashchain events in a gate sequence domain."""
|
||||
inf, _gs = fresh_env
|
||||
priv, pub_b64, node_id = _make_keypair()
|
||||
sequence = 1
|
||||
payload = _make_gate_payload("test-gate")
|
||||
sig_payload = mesh_crypto.build_signature_payload(
|
||||
event_type="gate_message",
|
||||
node_id=node_id,
|
||||
sequence=sequence,
|
||||
payload=payload,
|
||||
)
|
||||
signature = priv.sign(sig_payload.encode("utf-8")).hex()
|
||||
|
||||
event = inf.append_private_gate_message(
|
||||
node_id=node_id,
|
||||
payload=payload,
|
||||
signature=signature,
|
||||
sequence=sequence,
|
||||
public_key=pub_b64,
|
||||
public_key_algo="Ed25519",
|
||||
protocol_version=mesh_protocol.PROTOCOL_VERSION,
|
||||
timestamp=123.0,
|
||||
)
|
||||
|
||||
assert event["event_type"] == "gate_message"
|
||||
assert inf.head_hash == event["event_id"]
|
||||
assert inf.sequence_domains[f"{node_id}|gate_message"] == sequence
|
||||
assert inf.node_sequences.get(node_id, 0) == 0
|
||||
assert event["payload"]["transport_lock"] == "private_strong"
|
||||
|
||||
|
||||
def test_ingest_accepts_new_suffix_after_duplicate_prefix(fresh_env):
|
||||
"""Peer-push batches may include events the receiver already has."""
|
||||
inf, _gs = fresh_env
|
||||
priv, pub_b64, node_id = _make_keypair()
|
||||
evt1 = _make_gate_message_event(
|
||||
priv,
|
||||
pub_b64,
|
||||
node_id,
|
||||
sequence=1,
|
||||
prev_hash=mesh_hashchain.GENESIS_HASH,
|
||||
)
|
||||
assert inf.ingest_events([evt1])["accepted"] == 1
|
||||
evt2 = _make_gate_message_event(
|
||||
priv,
|
||||
pub_b64,
|
||||
node_id,
|
||||
sequence=2,
|
||||
prev_hash=evt1["event_id"],
|
||||
)
|
||||
assert inf.ingest_events([evt2])["accepted"] == 1
|
||||
evt3 = _make_gate_message_event(
|
||||
priv,
|
||||
pub_b64,
|
||||
node_id,
|
||||
sequence=3,
|
||||
prev_hash=evt2["event_id"],
|
||||
)
|
||||
|
||||
result = inf.ingest_events([evt1, evt2, evt3])
|
||||
|
||||
assert result["duplicates"] == 2
|
||||
assert result["accepted"] == 1
|
||||
assert result["rejected"] == []
|
||||
assert inf.head_hash == evt3["event_id"]
|
||||
|
||||
|
||||
def test_rejected_event_does_not_hydrate_gate_store(fresh_env):
|
||||
"""A gate_message rejected by ingest must not appear in gate_store."""
|
||||
inf, gs = fresh_env
|
||||
|
||||
@@ -22,11 +22,9 @@ class TestHealthEndpoint:
|
||||
|
||||
|
||||
class TestLiveDataEndpoints:
|
||||
def test_live_data_returns_200_or_304(self, client):
|
||||
def test_live_data_returns_200(self, client):
|
||||
r = client.get("/api/live-data")
|
||||
assert r.status_code in (200, 304)
|
||||
if r.status_code == 200:
|
||||
assert r.headers.get("etag")
|
||||
assert r.status_code == 200
|
||||
|
||||
def test_live_data_fast_returns_200_or_304(self, client):
|
||||
r = client.get("/api/live-data/fast")
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
from starlette.requests import Request
|
||||
|
||||
import auth
|
||||
|
||||
|
||||
async def _empty_receive():
|
||||
return {"type": "http.request", "body": b"", "more_body": False}
|
||||
|
||||
|
||||
def _request(path: str, *, host: str = "example.com/health?x=", client_host: str = "203.0.113.10") -> Request:
|
||||
return Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": "GET",
|
||||
"scheme": "http",
|
||||
"server": ("127.0.0.1", 8000),
|
||||
"client": (client_host, 12345),
|
||||
"path": path,
|
||||
"raw_path": path.encode("ascii"),
|
||||
"query_string": b"",
|
||||
"headers": [(b"host", host.encode("ascii"))],
|
||||
},
|
||||
receive=_empty_receive,
|
||||
)
|
||||
|
||||
|
||||
def test_scope_auth_uses_asgi_path_not_host_derived_url_path():
|
||||
request = _request("/api/mesh/gate/alpha/message")
|
||||
|
||||
assert auth._request_scope_path(request) == "/api/mesh/gate/alpha/message"
|
||||
assert auth._required_scope_for_request(request) == "mesh"
|
||||
|
||||
|
||||
def test_debug_test_request_does_not_trust_host_header(monkeypatch):
|
||||
monkeypatch.setattr(auth, "_debug_mode_enabled", lambda: True)
|
||||
|
||||
request = _request("/api/admin", host="test/api/public?x=")
|
||||
|
||||
assert auth._is_debug_test_request(request) is False
|
||||
|
||||
|
||||
def test_peer_hmac_identity_requires_explicit_peer_url_header():
|
||||
request = _request("/api/mesh/infonet/push", host="https://peer.example/api/public?x=")
|
||||
|
||||
assert auth._peer_hmac_url_from_request(request) == ""
|
||||
|
||||
request = _request("/api/mesh/infonet/push")
|
||||
request.scope["headers"].append((b"x-peer-url", b"https://peer.example/"))
|
||||
|
||||
assert auth._peer_hmac_url_from_request(request) == "https://peer.example"
|
||||
@@ -1,46 +0,0 @@
|
||||
"""DeepState GitHub mirror pinning (#362)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import services.geopolitics as gp
|
||||
|
||||
|
||||
def test_deepstate_mirror_ref_defaults(monkeypatch):
|
||||
monkeypatch.delenv("DEEPSTATE_MIRROR_COMMIT", raising=False)
|
||||
monkeypatch.delenv("DEEPSTATE_MIRROR_REPO", raising=False)
|
||||
repo, ref = gp._deepstate_mirror_ref()
|
||||
assert repo == "cyterat/deepstate-map-data"
|
||||
assert ref == "main"
|
||||
|
||||
|
||||
def test_deepstate_mirror_ref_pinned_commit(monkeypatch):
|
||||
monkeypatch.setenv("DEEPSTATE_MIRROR_COMMIT", "abc123def456")
|
||||
monkeypatch.setenv("DEEPSTATE_MIRROR_REPO", "cyterat/deepstate-map-data")
|
||||
repo, ref = gp._deepstate_mirror_ref()
|
||||
assert repo == "cyterat/deepstate-map-data"
|
||||
assert ref == "abc123def456"
|
||||
|
||||
|
||||
def test_fetch_ukraine_frontlines_uses_pinned_tree_url(monkeypatch):
|
||||
monkeypatch.setenv("DEEPSTATE_MIRROR_COMMIT", "deadbeef")
|
||||
gp.frontline_cache.clear()
|
||||
|
||||
tree_resp = MagicMock(status_code=200)
|
||||
tree_resp.json.return_value = {
|
||||
"tree": [{"path": "data/deepstatemap_data_20260101.geojson"}]
|
||||
}
|
||||
geo_resp = MagicMock(status_code=200)
|
||||
geo_resp.json.return_value = {"features": []}
|
||||
|
||||
with patch("services.geopolitics.requests.get", side_effect=[tree_resp, geo_resp]) as get:
|
||||
result = gp.fetch_ukraine_frontlines()
|
||||
|
||||
assert result == {"features": []}
|
||||
tree_call = get.call_args_list[0][0][0]
|
||||
raw_call = get.call_args_list[1][0][0]
|
||||
assert "/git/trees/deadbeef" in tree_call
|
||||
assert "raw.githubusercontent.com/cyterat/deepstate-map-data/deadbeef/" in raw_call
|
||||
|
||||
gp.frontline_cache.clear()
|
||||
@@ -1,67 +0,0 @@
|
||||
"""Regression tests for GitHub #375 production-readiness fixes."""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestDevBindHost:
|
||||
def test_defaults_to_loopback(self, monkeypatch):
|
||||
monkeypatch.delenv("SHADOWBROKER_DEV_BIND_ALL", raising=False)
|
||||
from main import _dev_uvicorn_bind_host
|
||||
|
||||
assert _dev_uvicorn_bind_host() == "127.0.0.1"
|
||||
|
||||
@pytest.mark.parametrize("value", ("1", "true", "yes", "on", "TRUE"))
|
||||
def test_bind_all_opt_in(self, monkeypatch, value):
|
||||
monkeypatch.setenv("SHADOWBROKER_DEV_BIND_ALL", value)
|
||||
from main import _dev_uvicorn_bind_host
|
||||
|
||||
assert _dev_uvicorn_bind_host() == "0.0.0.0"
|
||||
|
||||
|
||||
class TestDataStoreSnapshots:
|
||||
def test_deepcopy_snapshot_isolated_from_store(self):
|
||||
from services.fetchers import _store
|
||||
|
||||
original = [{"title": "baseline"}]
|
||||
with _store._data_lock:
|
||||
_store.latest_data["news"] = list(original)
|
||||
snap = _store.get_latest_data_deepcopy_snapshot()
|
||||
snap["news"][0]["title"] = "mutated"
|
||||
with _store._data_lock:
|
||||
assert _store.latest_data["news"][0]["title"] == "baseline"
|
||||
|
||||
def test_subset_deepcopy_isolated(self):
|
||||
from services.fetchers import _store
|
||||
|
||||
with _store._data_lock:
|
||||
_store.latest_data["news"] = [{"title": "subset"}]
|
||||
snap = _store.get_latest_data_subset("news")
|
||||
snap["news"][0]["title"] = "changed"
|
||||
with _store._data_lock:
|
||||
assert _store.latest_data["news"][0]["title"] == "subset"
|
||||
|
||||
|
||||
class TestHeavyFetchExecutorRouting:
|
||||
def test_slow_tier_uses_slow_executor(self):
|
||||
from services.data_fetcher import (
|
||||
_SLOW_EXECUTOR,
|
||||
_SHARED_EXECUTOR,
|
||||
_executor_for_task_label,
|
||||
)
|
||||
|
||||
assert _executor_for_task_label("slow-tier-refresh") is _SLOW_EXECUTOR
|
||||
assert _executor_for_task_label("startup-heavy-warm") is _SLOW_EXECUTOR
|
||||
assert _executor_for_task_label("fast-tier-refresh") is _SHARED_EXECUTOR
|
||||
|
||||
|
||||
class TestLiveDataFullEndpoint:
|
||||
def test_live_data_supports_etag_304(self, client):
|
||||
r1 = client.get("/api/live-data")
|
||||
assert r1.status_code == 200
|
||||
etag = r1.headers.get("etag")
|
||||
assert etag
|
||||
r2 = client.get("/api/live-data", headers={"If-None-Match": etag})
|
||||
assert r2.status_code == 304
|
||||
assert r2.headers.get("etag") == etag
|
||||
@@ -1,29 +0,0 @@
|
||||
"""KiwiSDR mirror prefers HTTPS (#364)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from services.kiwisdr_fetcher import (
|
||||
_SOURCE_URL_HTTP,
|
||||
_SOURCE_URL_HTTPS,
|
||||
_fetch_mirror_payload_text,
|
||||
)
|
||||
|
||||
|
||||
def test_fetch_mirror_tries_https_before_http():
|
||||
calls: list[str] = []
|
||||
|
||||
def fake_fetch(url, **kwargs):
|
||||
calls.append(url)
|
||||
if url == _SOURCE_URL_HTTPS:
|
||||
raise ConnectionError("tls not available")
|
||||
res = MagicMock()
|
||||
res.status_code = 200
|
||||
res.text = "var kiwisdr_com = [];"
|
||||
return res
|
||||
|
||||
with patch("services.network_utils.fetch_with_curl", side_effect=fake_fetch):
|
||||
body = _fetch_mirror_payload_text()
|
||||
|
||||
assert body == "var kiwisdr_com = [];"
|
||||
assert calls == [_SOURCE_URL_HTTPS, _SOURCE_URL_HTTP]
|
||||
@@ -1,45 +0,0 @@
|
||||
"""LiveUAMap scraper UI opt-in on Windows (#348)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from services import liveuamap_settings as settings
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def opt_in_file(tmp_path, monkeypatch):
|
||||
path = tmp_path / "liveuamap_scraper_opt_in.json"
|
||||
monkeypatch.setattr(settings, "_OPT_IN_FILE", path)
|
||||
return path
|
||||
|
||||
|
||||
def test_windows_defaults_off_without_opt_in(monkeypatch, opt_in_file):
|
||||
monkeypatch.setattr(settings.os, "name", "nt")
|
||||
monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
|
||||
assert settings.liveuamap_scraper_enabled() is False
|
||||
assert settings.liveuamap_requires_ui_opt_in() is True
|
||||
|
||||
|
||||
def test_windows_opt_in_enables_scraper(monkeypatch, opt_in_file):
|
||||
monkeypatch.setattr(settings.os, "name", "nt")
|
||||
monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
|
||||
settings.set_liveuamap_ui_opt_in(True)
|
||||
assert settings.liveuamap_scraper_enabled() is True
|
||||
assert json.loads(opt_in_file.read_text())["opted_in"] is True
|
||||
|
||||
|
||||
def test_linux_enabled_without_opt_in(monkeypatch, opt_in_file):
|
||||
monkeypatch.setattr(settings.os, "name", "posix")
|
||||
monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
|
||||
assert settings.liveuamap_requires_ui_opt_in() is False
|
||||
assert settings.liveuamap_scraper_enabled() is True
|
||||
|
||||
|
||||
def test_env_force_off_overrides_ui_opt_in(monkeypatch, opt_in_file):
|
||||
monkeypatch.setattr(settings.os, "name", "nt")
|
||||
monkeypatch.setenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", "false")
|
||||
settings.set_liveuamap_ui_opt_in(True)
|
||||
assert settings.liveuamap_scraper_enabled() is False
|
||||
@@ -1,27 +0,0 @@
|
||||
"""Madrid CCTV KML prefers HTTPS (#363)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from services.cctv_pipeline import MadridCityIngestor
|
||||
|
||||
|
||||
def test_madrid_fetch_kml_tries_https_before_http():
|
||||
ingestor = MadridCityIngestor()
|
||||
calls: list[str] = []
|
||||
|
||||
def fake_fetch(url, **kwargs):
|
||||
calls.append(url)
|
||||
if url == ingestor.KML_URL_HTTPS:
|
||||
raise ConnectionError("tls handshake failed")
|
||||
res = MagicMock()
|
||||
res.status_code = 200
|
||||
res.content = b'<?xml version="1.0"?><kml xmlns="http://www.opengis.net/kml/2.2"></kml>'
|
||||
res.raise_for_status = MagicMock()
|
||||
return res
|
||||
|
||||
with patch("services.cctv_pipeline.fetch_with_curl", side_effect=fake_fetch):
|
||||
response = ingestor._fetch_kml()
|
||||
|
||||
assert response.status_code == 200
|
||||
assert calls == [ingestor.KML_URL_HTTPS, ingestor.KML_URL_HTTP]
|
||||
@@ -1,27 +1,56 @@
|
||||
"""Issue #350: Meshtastic callsign in outbound UA is opt-in, not default."""
|
||||
import os
|
||||
"""Issue #203 (tg12): meshtastic_map.py was unconditionally including
|
||||
``MESHTASTIC_OPERATOR_CALLSIGN`` in the outbound User-Agent header,
|
||||
which contradicted the README's "no user data transmitted" claim.
|
||||
|
||||
The fix preserves the existing default behavior (callsign sent — that's
|
||||
what operators who configured the variable expected) but adds an
|
||||
opt-out env var ``MESHTASTIC_SEND_CALLSIGN_HEADER=false`` for
|
||||
privacy-conscious operators.
|
||||
"""
|
||||
import importlib
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _send_callsign_header_from_env() -> bool:
|
||||
raw = str(os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "false")).strip().lower()
|
||||
return raw in {"1", "true", "yes", "on"}
|
||||
def _reload_meshtastic_module():
|
||||
"""Reload meshtastic_map so settings are re-read on demand."""
|
||||
if "services.fetchers.meshtastic_map" in sys.modules:
|
||||
del sys.modules["services.fetchers.meshtastic_map"]
|
||||
return importlib.import_module("services.fetchers.meshtastic_map")
|
||||
|
||||
|
||||
def test_default_does_not_send_callsign(monkeypatch):
|
||||
def test_default_behavior_includes_callsign(monkeypatch):
|
||||
"""Operators who set the callsign and don't change anything else
|
||||
keep their existing behavior (callsign sent in UA)."""
|
||||
# We test the UA construction logic by exercising the same branches
|
||||
# the fetcher uses. Direct fetch isn't run because it makes a real
|
||||
# network call — we just verify the env-var-driven decision.
|
||||
import os
|
||||
monkeypatch.setenv("MESHTASTIC_OPERATOR_CALLSIGN", "N0CALL")
|
||||
monkeypatch.delenv("MESHTASTIC_SEND_CALLSIGN_HEADER", raising=False)
|
||||
assert _send_callsign_header_from_env() is False
|
||||
|
||||
raw = str(os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "true")).strip().lower()
|
||||
send_callsign_header = raw not in {"0", "false", "no", "off", ""}
|
||||
assert send_callsign_header is True
|
||||
|
||||
|
||||
def test_opt_in_sends_callsign(monkeypatch):
|
||||
def test_opt_out_suppresses_callsign(monkeypatch):
|
||||
"""Setting MESHTASTIC_SEND_CALLSIGN_HEADER=false suppresses the header."""
|
||||
import os
|
||||
monkeypatch.setenv("MESHTASTIC_OPERATOR_CALLSIGN", "N0CALL")
|
||||
monkeypatch.setenv("MESHTASTIC_SEND_CALLSIGN_HEADER", "true")
|
||||
assert _send_callsign_header_from_env() is True
|
||||
monkeypatch.setenv("MESHTASTIC_SEND_CALLSIGN_HEADER", "false")
|
||||
|
||||
raw = str(os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "true")).strip().lower()
|
||||
send_callsign_header = raw not in {"0", "false", "no", "off", ""}
|
||||
assert send_callsign_header is False
|
||||
|
||||
|
||||
def test_various_falsy_values_do_not_opt_in(monkeypatch):
|
||||
for falsy in ("0", "false", "FALSE", "no", "off", ""):
|
||||
def test_various_falsy_values_all_opt_out(monkeypatch):
|
||||
"""Common falsy strings should all suppress the callsign header."""
|
||||
import os
|
||||
for falsy in ("0", "false", "FALSE", "no", "off"):
|
||||
monkeypatch.setenv("MESHTASTIC_SEND_CALLSIGN_HEADER", falsy)
|
||||
assert _send_callsign_header_from_env() is False, f"value {falsy!r} should not opt in"
|
||||
raw = str(os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "true")).strip().lower()
|
||||
send_callsign_header = raw not in {"0", "false", "no", "off", ""}
|
||||
assert send_callsign_header is False, f"value {falsy!r} did not opt out"
|
||||
|
||||
@@ -5,7 +5,7 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
from services.fetchers.news import _resolve_coords
|
||||
from services.news_feed_config import DEFAULT_FEEDS, _normalise_feeds
|
||||
from services.news_feed_config import DEFAULT_FEEDS
|
||||
|
||||
|
||||
CONFIG_PATH = Path(__file__).parent.parent / "config" / "news_feeds.json"
|
||||
@@ -152,14 +152,3 @@ class TestFeedConfig:
|
||||
urls = {f["url"] for f in DEFAULT_FEEDS}
|
||||
assert "https://www.reutersagency.com/feed/?best-topics=world" not in urls
|
||||
assert "https://rsshub.app/apnews/topics/world-news" not in urls
|
||||
|
||||
def test_legacy_http_feeds_are_migrated_to_https(self):
|
||||
feeds = _normalise_feeds(
|
||||
[
|
||||
{"name": "BBC", "url": "http://feeds.bbci.co.uk/news/world/rss.xml", "weight": 3},
|
||||
{"name": "Xinhua", "url": "http://www.news.cn/english/rss/worldrss.xml", "weight": 2},
|
||||
]
|
||||
)
|
||||
urls = {f["url"] for f in feeds}
|
||||
assert "https://feeds.bbci.co.uk/news/world/rss.xml" in urls
|
||||
assert "https://www.news.cn/english/rss/worldrss.xml" in urls
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
"""Regression coverage for OpenClaw skill HMAC environment names."""
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _load_sb_query(monkeypatch):
|
||||
module_path = Path(__file__).resolve().parents[2] / "openclaw-skills" / "shadowbroker" / "sb_query.py"
|
||||
spec = importlib.util.spec_from_file_location("shadowbroker_skill_sb_query_test", module_path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_openclaw_skill_prefers_hmac_secret_env(monkeypatch):
|
||||
monkeypatch.setenv("SHADOWBROKER_HMAC_SECRET", "preferred-hmac-secret")
|
||||
monkeypatch.setenv("SHADOWBROKER_KEY", "legacy-hmac-secret")
|
||||
|
||||
module = _load_sb_query(monkeypatch)
|
||||
|
||||
assert module.ShadowBrokerClient()._hmac_secret == "preferred-hmac-secret"
|
||||
|
||||
|
||||
def test_openclaw_skill_accepts_legacy_key_as_hmac_secret_alias(monkeypatch):
|
||||
monkeypatch.delenv("SHADOWBROKER_HMAC_SECRET", raising=False)
|
||||
monkeypatch.setenv("SHADOWBROKER_KEY", "legacy-hmac-secret")
|
||||
|
||||
module = _load_sb_query(monkeypatch)
|
||||
client = module.ShadowBrokerClient()
|
||||
headers = client._sign_headers("GET", "/api/ai/tools")
|
||||
|
||||
assert client._hmac_secret == "legacy-hmac-secret"
|
||||
assert "X-SB-Timestamp" in headers
|
||||
assert "X-SB-Nonce" in headers
|
||||
assert "X-SB-Signature" in headers
|
||||
assert "Authorization" not in headers
|
||||
assert "X-Admin-Key" not in headers
|
||||
@@ -133,19 +133,23 @@ class TestOperatorHandleGeneration:
|
||||
|
||||
|
||||
class TestOutboundUserAgentString:
|
||||
def test_ua_is_operator_handle(self, isolated_handle):
|
||||
def test_includes_operator_handle(self, isolated_handle):
|
||||
ua = isolated_handle.outbound_user_agent()
|
||||
handle = isolated_handle.get_operator_handle()
|
||||
assert ua == handle
|
||||
assert f"operator: {handle}" in ua
|
||||
|
||||
def test_includes_purpose_when_provided(self, isolated_handle):
|
||||
ua = isolated_handle.outbound_user_agent("wikipedia")
|
||||
handle = isolated_handle.get_operator_handle()
|
||||
assert ua == f"{handle} (purpose: wikipedia)"
|
||||
assert "purpose: wikipedia" in ua
|
||||
|
||||
def test_no_shadowbroker_product_token(self, isolated_handle):
|
||||
ua = isolated_handle.outbound_user_agent("nominatim")
|
||||
assert "shadowbroker" not in ua.lower()
|
||||
def test_includes_contact_path(self, isolated_handle):
|
||||
ua = isolated_handle.outbound_user_agent()
|
||||
assert "github.com" in ua.lower()
|
||||
assert "shadowbroker" in ua.lower()
|
||||
|
||||
def test_version_prefix(self, isolated_handle):
|
||||
ua = isolated_handle.outbound_user_agent()
|
||||
assert ua.startswith("Shadowbroker/")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -177,8 +181,8 @@ class TestWikimediaCallsAreNowPerOperator:
|
||||
assert "Api-User-Agent" in headers
|
||||
handle = isolated_handle.get_operator_handle()
|
||||
for header_value in (headers["User-Agent"], headers["Api-User-Agent"]):
|
||||
assert header_value.startswith(handle), (
|
||||
f"Wikimedia UA must be the per-operator handle; got {header_value!r}"
|
||||
assert f"operator: {handle}" in header_value, (
|
||||
f"Wikimedia UA must include the per-operator handle; got {header_value!r}"
|
||||
)
|
||||
|
||||
def test_wikipedia_summary_uses_per_operator_ua(self, isolated_handle, monkeypatch):
|
||||
@@ -207,8 +211,7 @@ class TestWikimediaCallsAreNowPerOperator:
|
||||
assert wikipedia_hits, "Wikipedia summary fetch was not called"
|
||||
for _url, headers in wikipedia_hits:
|
||||
handle = isolated_handle.get_operator_handle()
|
||||
ua = headers.get("User-Agent", "")
|
||||
assert ua.startswith(handle), f"Wikipedia UA must be the operator handle; got {ua!r}"
|
||||
assert f"operator: {handle}" in headers.get("User-Agent", "")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -230,7 +233,6 @@ class TestNoMonsterUserAgentRemains:
|
||||
"""
|
||||
|
||||
BANNED_LITERALS = (
|
||||
"Shadowbroker/",
|
||||
"ShadowBroker-OSINT/1.0",
|
||||
"ShadowBroker-OSINT/0.9",
|
||||
"ShadowBroker-FeedIngester/1.0",
|
||||
@@ -238,8 +240,6 @@ class TestNoMonsterUserAgentRemains:
|
||||
"ShadowBroker/0.9.79 Finnhub connector",
|
||||
"ShadowBroker/0.9.8 local Shodan connector",
|
||||
"ShadowBroker/0.9.8 Finnhub connector",
|
||||
"ShadowBroker/0.9.81 local Shodan connector",
|
||||
"ShadowBroker/0.9.81 Finnhub connector",
|
||||
"Mozilla/5.0 (compatible; ShadowBroker CCTV proxy)",
|
||||
)
|
||||
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
"""Prediction market fetch timing uses jitter to reduce poll fingerprinting."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from services.fetchers import prediction_markets as pm
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_market_cache():
|
||||
pm._market_cache.clear()
|
||||
yield
|
||||
pm._market_cache.clear()
|
||||
|
||||
|
||||
def test_pre_fetch_jitter_sleeps_when_configured(monkeypatch):
|
||||
monkeypatch.setattr(pm, "_PRE_FETCH_JITTER_S", 10.0)
|
||||
sleeps: list[float] = []
|
||||
monkeypatch.setattr(pm.time, "sleep", lambda s: sleeps.append(s))
|
||||
monkeypatch.setattr(pm.random, "uniform", lambda _a, _b: 4.5)
|
||||
|
||||
pm._apply_pre_fetch_jitter()
|
||||
|
||||
assert sleeps == [4.5]
|
||||
|
||||
|
||||
def test_fetch_raw_applies_provider_gap(monkeypatch):
|
||||
monkeypatch.setenv("PREDICTION_MARKETS_ENABLED", "true")
|
||||
monkeypatch.setattr(pm, "_apply_pre_fetch_jitter", lambda: None)
|
||||
gap_calls: list[int] = []
|
||||
|
||||
def _track_gap():
|
||||
gap_calls.append(1)
|
||||
|
||||
monkeypatch.setattr(pm, "_apply_provider_gap_jitter", _track_gap)
|
||||
monkeypatch.setattr(pm, "_fetch_polymarket_events", lambda: [])
|
||||
monkeypatch.setattr(pm, "_fetch_kalshi_events", lambda: [])
|
||||
monkeypatch.setattr(pm, "_merge_markets", lambda _p, _k: [])
|
||||
|
||||
pm.fetch_prediction_markets_raw()
|
||||
|
||||
assert gap_calls == [1]
|
||||
|
||||
|
||||
def test_pace_provider_adds_per_page_jitter(monkeypatch):
|
||||
monkeypatch.setattr(pm, "_POLYMARKET_PAGE_DELAY_JITTER_S", 1.0)
|
||||
monkeypatch.setattr(pm, "_provider_last_request_at", {"polymarket": pm.time.monotonic()})
|
||||
monkeypatch.setattr(pm.random, "uniform", lambda _a, _b: 0.5)
|
||||
sleeps: list[float] = []
|
||||
monkeypatch.setattr(pm.time, "sleep", lambda s: sleeps.append(s))
|
||||
|
||||
pm._pace_provider("polymarket", 0.02)
|
||||
|
||||
assert sleeps == [pytest.approx(0.52)]
|
||||
@@ -1,24 +0,0 @@
|
||||
"""UI opt-in for prediction markets (Global Threat Intercept)."""
|
||||
|
||||
from services import prediction_markets_settings as pm_settings
|
||||
from services.fetchers import prediction_markets
|
||||
|
||||
|
||||
def test_ui_opt_in_enables_fetch(monkeypatch, tmp_path):
|
||||
opt_file = tmp_path / "prediction_markets_opt_in.json"
|
||||
monkeypatch.setattr(pm_settings, "_OPT_IN_FILE", opt_file)
|
||||
monkeypatch.delenv("PREDICTION_MARKETS_ENABLED", raising=False)
|
||||
|
||||
assert pm_settings.prediction_markets_fetch_enabled() is False
|
||||
|
||||
pm_settings.set_prediction_markets_ui_opt_in(True)
|
||||
assert pm_settings.prediction_markets_fetch_enabled() is True
|
||||
assert prediction_markets.prediction_markets_fetch_enabled() is True
|
||||
|
||||
|
||||
def test_env_force_on_without_ui_file(monkeypatch, tmp_path):
|
||||
opt_file = tmp_path / "prediction_markets_opt_in.json"
|
||||
monkeypatch.setattr(pm_settings, "_OPT_IN_FILE", opt_file)
|
||||
monkeypatch.setenv("PREDICTION_MARKETS_ENABLED", "true")
|
||||
|
||||
assert pm_settings.prediction_markets_fetch_enabled() is True
|
||||
@@ -1,74 +0,0 @@
|
||||
"""Right-click dossier returns up to 3 signed Sentinel-2 scenes."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from services import sentinel_search as ss
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_sentinel_cache():
|
||||
ss._sentinel_cache.clear()
|
||||
yield
|
||||
ss._sentinel_cache.clear()
|
||||
|
||||
|
||||
def _stac_feature(scene_id: str, dt: str, cloud: float) -> dict:
|
||||
href = f"https://sentinel2euwest.blob.core.windows.net/sentinel2-l2a/{scene_id}.tif"
|
||||
return {
|
||||
"id": scene_id,
|
||||
"bbox": [0, 0, 1, 1],
|
||||
"properties": {
|
||||
"datetime": dt,
|
||||
"eo:cloud_cover": cloud,
|
||||
"platform": "Sentinel-2A",
|
||||
},
|
||||
"assets": {
|
||||
"rendered_preview": {"href": href},
|
||||
"thumbnail": {"href": href},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@patch("services.sentinel_search.requests.get")
|
||||
@patch("services.sentinel_search.requests.post")
|
||||
def test_search_returns_three_scenes(mock_post, mock_get):
|
||||
mock_post.return_value = MagicMock(
|
||||
ok=True,
|
||||
raise_for_status=MagicMock(),
|
||||
json=lambda: {
|
||||
"features": [
|
||||
_stac_feature("s1", "2026-05-28T10:00:00Z", 5.0),
|
||||
_stac_feature("s2", "2026-05-20T10:00:00Z", 12.0),
|
||||
_stac_feature("s3", "2026-05-10T10:00:00Z", 18.0),
|
||||
],
|
||||
},
|
||||
)
|
||||
mock_get.return_value = MagicMock(
|
||||
ok=True,
|
||||
raise_for_status=MagicMock(),
|
||||
json=lambda: {"token": "sig=test"},
|
||||
)
|
||||
|
||||
result = ss.search_sentinel2_scene(29.0, 51.0)
|
||||
|
||||
assert result["found"] is True
|
||||
assert result["scene_id"] == "s1"
|
||||
assert len(result["scenes"]) == 3
|
||||
assert result["scenes"][1]["scene_id"] == "s2"
|
||||
assert "sig=test" in (result["scenes"][0]["fullres_url"] or "")
|
||||
|
||||
|
||||
@patch("services.sentinel_search.requests.post")
|
||||
def test_search_esri_fallback_has_no_scenes(mock_post):
|
||||
mock_post.return_value = MagicMock(
|
||||
ok=True,
|
||||
raise_for_status=MagicMock(),
|
||||
json=lambda: {"features": []},
|
||||
)
|
||||
|
||||
result = ss.search_sentinel2_scene(29.0, 51.0)
|
||||
|
||||
assert result["fallback"] is True
|
||||
assert "scenes" not in result
|
||||
@@ -45,10 +45,8 @@ def test_fimi_falsy_value_does_not_call_upstream(monkeypatch):
|
||||
|
||||
def test_prediction_markets_disabled_by_default(monkeypatch):
|
||||
from services.fetchers import _store, prediction_markets
|
||||
from services import prediction_markets_settings as pm_settings
|
||||
|
||||
monkeypatch.delenv("PREDICTION_MARKETS_ENABLED", raising=False)
|
||||
monkeypatch.setattr(pm_settings, "get_prediction_markets_ui_opt_in", lambda: False)
|
||||
monkeypatch.setitem(_store.latest_data, "prediction_markets", [{"id": "old"}])
|
||||
monkeypatch.setattr(
|
||||
prediction_markets, "fetch_prediction_markets_raw", _explode
|
||||
|
||||
@@ -24,7 +24,6 @@ These tests pin the new behavior:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime as real_datetime
|
||||
|
||||
@@ -226,39 +225,28 @@ def test_fetch_uap_sightings_succeeds_when_fallback_returns_data(monkeypatch):
|
||||
assert canary_calls == [], "canary should not trip when fallback supplies data"
|
||||
|
||||
|
||||
def test_uap_scheduler_runs_weekly():
|
||||
"""UAP layer refreshes weekly so each install pulls live NUFORC on a steady cadence."""
|
||||
def test_uap_scheduler_runs_weekly_not_daily():
|
||||
"""The cron job for the UAP layer must be configured for Mondays at
|
||||
12:00 UTC, not daily. Daily was the pre-fix default; weekly matches
|
||||
the layer's stated cadence (a rolling 60-day digest) and keeps load
|
||||
on nuforc.org light."""
|
||||
from services import data_fetcher
|
||||
|
||||
with open(data_fetcher.__file__, "r", encoding="utf-8") as f:
|
||||
src = data_fetcher.__file__
|
||||
with open(src, "r", encoding="utf-8") as f:
|
||||
text = f.read()
|
||||
|
||||
assert "uap_sightings_weekly" in text
|
||||
idx = text.index("uap_sightings_weekly")
|
||||
block = text[max(0, idx - 600) : idx + 120]
|
||||
assert 'day_of_week="mon"' in block
|
||||
|
||||
|
||||
def test_uap_cache_rejects_stale_rows_on_load(tmp_path, monkeypatch):
|
||||
"""Disk cache must not resurrect sightings outside the rolling window."""
|
||||
from services.fetchers import earth_observation as eo
|
||||
|
||||
cache_file = tmp_path / "nuforc_recent_sightings.json"
|
||||
monkeypatch.setattr(eo, "_NUFORC_SIGHTINGS_CACHE_FILE", cache_file)
|
||||
monkeypatch.setattr(eo, "datetime", _FixedDateTime)
|
||||
|
||||
cache_file.write_text(
|
||||
json.dumps({
|
||||
"built": _FixedDateTime.utcnow().isoformat(),
|
||||
"cutoff_days": 60,
|
||||
"sightings": [
|
||||
{"id": "NUFORC-old", "date_time": "2023-06-01", "lat": 39.0, "lng": -105.0},
|
||||
{"id": "NUFORC-new", "date_time": "2026-04-20", "lat": 40.0, "lng": -104.0},
|
||||
],
|
||||
}),
|
||||
encoding="utf-8",
|
||||
# Anchor on the scheduler block by id, then assert the cron triggers.
|
||||
assert "uap_sightings_weekly" in text, (
|
||||
"scheduler id should be uap_sightings_weekly (was uap_sightings_daily pre-fix)"
|
||||
)
|
||||
# The day_of_week directive is the difference between daily and weekly.
|
||||
# If somebody flips it back to daily, this fires.
|
||||
weekly_block = text.split("uap_sightings_weekly", 1)[0]
|
||||
# Walk backwards for the matching add_job call.
|
||||
add_job_idx = weekly_block.rfind("add_job(")
|
||||
assert add_job_idx >= 0, "could not locate add_job block for UAP scheduler"
|
||||
job_block = text[add_job_idx : text.find(")", text.index("uap_sightings_weekly")) + 1]
|
||||
assert 'day_of_week="mon"' in job_block, (
|
||||
f"expected day_of_week='mon' in UAP scheduler block:\n{job_block}"
|
||||
)
|
||||
|
||||
loaded = eo._load_nuforc_sightings_cache()
|
||||
assert loaded is not None
|
||||
assert [s["id"] for s in loaded] == ["NUFORC-new"]
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
"""Backend Wikimedia proxy routes (#360)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def test_wikipedia_summary_route_returns_payload(client):
|
||||
sample = {
|
||||
"title": "Paris",
|
||||
"description": "capital",
|
||||
"extract": "Paris is the capital of France.",
|
||||
"thumbnail": "https://example.org/t.jpg",
|
||||
"type": "standard",
|
||||
}
|
||||
with patch(
|
||||
"services.region_dossier.fetch_wikipedia_page_summary",
|
||||
return_value=sample,
|
||||
):
|
||||
r = client.get("/api/wikipedia/summary", params={"title": "Paris"})
|
||||
assert r.status_code == 200
|
||||
assert r.json()["title"] == "Paris"
|
||||
|
||||
|
||||
def test_wikidata_sparql_route_returns_bindings(client):
|
||||
bindings = [{"x": {"value": "1"}}]
|
||||
with patch(
|
||||
"services.region_dossier.fetch_wikidata_sparql_bindings",
|
||||
return_value=bindings,
|
||||
):
|
||||
r = client.post("/api/wikidata/sparql", json={"query": "SELECT ?x WHERE {}"})
|
||||
assert r.status_code == 200
|
||||
assert r.json()["bindings"] == bindings
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@shadowbroker/desktop-shell",
|
||||
"version": "0.9.81",
|
||||
"version": "0.9.8",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@shadowbroker/desktop-shell",
|
||||
"version": "0.9.81",
|
||||
"version": "0.9.8",
|
||||
"devDependencies": {
|
||||
"typescript": "^5.6.0"
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shadowbroker/desktop-shell",
|
||||
"version": "0.9.81",
|
||||
"version": "0.9.8",
|
||||
"private": true,
|
||||
"description": "ShadowBroker desktop shell packaging, runtime bridge, and release tooling",
|
||||
"scripts": {
|
||||
|
||||
@@ -130,45 +130,6 @@ function stageBackendRuntime() {
|
||||
});
|
||||
stagePrivacyCoreArtifact();
|
||||
stageReleaseAttestation();
|
||||
stageStartScripts();
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy ``start.bat`` and ``start.sh`` from the repo root into the
|
||||
* staged backend-runtime/ so they sit next to ``privacy_core.dll``.
|
||||
*
|
||||
* Why: an MSI/EXE/AppImage user who wants to launch via the dev-style
|
||||
* scripts (because the desktop shell is failing, or they prefer the
|
||||
* browser frontend at localhost:3000) shouldn't have to clone the
|
||||
* source repo just to get the scripts. Having them inside the install
|
||||
* directory also means the bundled ``privacy_core.dll`` fallback in
|
||||
* those scripts resolves to the SAME directory as the script, which
|
||||
* is exactly the layout the v0.9.81 script update is looking for.
|
||||
*
|
||||
* Tracked from issue #319: users who fell back to start.bat from
|
||||
* their MSI install dir had to go fetch it from GitHub, then saw a
|
||||
* scary "install Rust" warning because the script didn't know where
|
||||
* the bundled DLL was. Bundling the script removes both problems.
|
||||
*/
|
||||
function stageStartScripts() {
|
||||
const scripts = ['start.bat', 'start.sh'];
|
||||
for (const name of scripts) {
|
||||
const src = path.join(repoRoot, name);
|
||||
if (!fs.existsSync(src)) {
|
||||
console.warn(`backend-runtime staged without ${name} (not at repo root)`);
|
||||
continue;
|
||||
}
|
||||
const dst = path.join(outputDir, name);
|
||||
fs.copyFileSync(src, dst);
|
||||
// Preserve executable bit on POSIX systems for the .sh script.
|
||||
if (name.endsWith('.sh') && process.platform !== 'win32') {
|
||||
try {
|
||||
fs.chmodSync(dst, 0o755);
|
||||
} catch {
|
||||
/* best-effort; not fatal on filesystems that don't honor chmod */
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function stagePrivacyCoreArtifact() {
|
||||
|
||||
+1
-1
@@ -4201,7 +4201,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "shadowbroker-tauri-shell"
|
||||
version = "0.9.81"
|
||||
version = "0.9.8"
|
||||
dependencies = [
|
||||
"axum",
|
||||
"base64 0.22.1",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "shadowbroker-tauri-shell"
|
||||
version = "0.9.81"
|
||||
version = "0.9.8"
|
||||
edition = "2021"
|
||||
|
||||
[build-dependencies]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$schema": "https://schema.tauri.app/config/2",
|
||||
"productName": "ShadowBroker",
|
||||
"version": "0.9.81",
|
||||
"version": "0.9.8",
|
||||
"identifier": "com.shadowbroker.desktop",
|
||||
"build": {
|
||||
"frontendDist": "../../../frontend/out",
|
||||
@@ -38,7 +38,7 @@
|
||||
},
|
||||
"plugins": {
|
||||
"updater": {
|
||||
"pubkey": "dW50cnVzdGVkIGNvbW1lbnQ6IG1pbmlzaWduIHB1YmxpYyBrZXk6IDVEMTFERDdCNjhBRTk3MDcKUldRSGw2NW9lOTBSWGRjS1ZobFN5TkZsd3NkZ2g2L09WZzU4aytTR2FtN3ZtR0ZKejlNNldTbFUK",
|
||||
"pubkey": "dW50cnVzdGVkIGNvbW1lbnQ6IG1pbmlzaWduIHB1YmxpYyBrZXk6IEUxODExMjQ4MkJBMThFNTgKUldSWWpxRXJTQktCNFF3ZXNQbndUK0pVWUEwNDNuajcrUGI3ZEI4TWtDUDlQdHhudmlHUkNjQUUK",
|
||||
"endpoints": [
|
||||
"https://github.com/BigBodyCobain/Shadowbroker/releases/latest/download/latest.json"
|
||||
],
|
||||
|
||||
@@ -6,7 +6,7 @@ services:
|
||||
dockerfile: ./backend/Dockerfile
|
||||
container_name: shadowbroker-relay
|
||||
ports:
|
||||
- "127.0.0.1:8000:8000"
|
||||
- "0.0.0.0:8000:8000"
|
||||
env_file: .env
|
||||
volumes:
|
||||
- relay_data:/app/data
|
||||
|
||||
@@ -24,16 +24,8 @@ services:
|
||||
# Private Infonet bootstrap seeds. Seeds are discovery hints, not fixed roots.
|
||||
- MESH_BOOTSTRAP_SEED_PEERS=${MESH_BOOTSTRAP_SEED_PEERS:-http://gqpbunqbgtkcqilvclm3xrkt3zowjyl3s62kkktvojgvxzizamvbrqid.onion:8000}
|
||||
- MESH_DEFAULT_SYNC_PEERS=${MESH_DEFAULT_SYNC_PEERS:-}
|
||||
- MESH_SYNC_TIMEOUT_S=${MESH_SYNC_TIMEOUT_S:-5}
|
||||
- MESH_RELAY_PUSH_TIMEOUT_S=${MESH_RELAY_PUSH_TIMEOUT_S:-45}
|
||||
# Explicitly opt into HTTPS/IP-based peer sync. Default remains private transports only.
|
||||
- MESH_INFONET_ALLOW_CLEARNET_SYNC=${MESH_INFONET_ALLOW_CLEARNET_SYNC:-false}
|
||||
# Tor/Arti SOCKS transport for private .onion Infonet sync.
|
||||
- MESH_ARTI_ENABLED=${MESH_ARTI_ENABLED:-false}
|
||||
- MESH_ARTI_SOCKS_PORT=${MESH_ARTI_SOCKS_PORT:-9050}
|
||||
# Operator-trusted sync/push peers. Leave empty unless you control the peer secret on both sides.
|
||||
- MESH_RELAY_PEERS=${MESH_RELAY_PEERS:-}
|
||||
- MESH_PUBLIC_PEER_URL=${MESH_PUBLIC_PEER_URL:-}
|
||||
# Shared transport auth for operator peer push. Must be set to a unique secret per deployment.
|
||||
- MESH_PEER_PUSH_SECRET=${MESH_PEER_PUSH_SECRET:-}
|
||||
# Issue #256: optional per-peer HMAC secrets. Comma-separated
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
# Outbound data and third-party exposure
|
||||
|
||||
Shadowbroker is **self-hosted**: each install uses its own backend egress IP. This document is the operator-facing record for GitHub audit issues **#348–#366** (tg12): what contacts third parties, why, and how to opt out without losing unrelated features.
|
||||
|
||||
## Architecture
|
||||
|
||||
| Path | Who calls third parties |
|
||||
|------|-------------------------|
|
||||
| Map UI → `/api/*` → fetchers | **This install’s backend** |
|
||||
| Basemap tiles / fonts | **Operator’s browser** (CARTO, demotiles.maplibre.org) |
|
||||
| CCTV still/video proxy | **Backend** (Referer/Origin set per agency — see #349) |
|
||||
|
||||
---
|
||||
|
||||
## Issue disposition summary
|
||||
|
||||
| Issue | Status | Approach |
|
||||
|-------|--------|----------|
|
||||
| **#351** | Fixed | Region dossier via backend proxy |
|
||||
| **#352** | Fixed | Geocode via `/api/geocode` only |
|
||||
| **#360** | Fixed | Wikipedia/Wikidata via backend |
|
||||
| **#362** | Fixed | `DEEPSTATE_MIRROR_COMMIT` optional pin |
|
||||
| **#363** | Fixed | Madrid KML HTTPS-first |
|
||||
| **#364** | Fixed | KiwiSDR HTTPS-first + validation |
|
||||
| **#348** | Accepted + gated | Windows UI opt-in; env override; stealth documented |
|
||||
| **#349** | Accepted + documented | Agency-required Referer on backend proxy only |
|
||||
| **#350** | Mitigated | Callsign in UA **off by default**; opt-in `MESHTASTIC_SEND_CALLSIGN_HEADER=true` |
|
||||
| **#354** | Accepted + documented | Default basemap CDN; optional self-hosted tiles |
|
||||
| **#361** | Mitigated | UA is **install handle only** (`operator-…`), not shared `Shadowbroker/` token |
|
||||
| **#366** | Accepted + documented | Honest per-install scrape; feature degrades if blocked |
|
||||
|
||||
---
|
||||
|
||||
## Per-install User-Agent (#361)
|
||||
|
||||
- **Code:** `backend/services/network_utils.py` — `outbound_user_agent()`, `OPERATOR_HANDLE`
|
||||
- **Sent:** `operator-7f3a92` or `your-handle (purpose: nominatim)` — **no** shared app product name
|
||||
- **Why:** Upstreams can rate-limit **one install**; a block on `operator-abc123` does not require blocking every Shadowbroker user
|
||||
- **Override:** `SHADOWBROKER_USER_AGENT` replaces the entire string
|
||||
- **Note:** The same handle across Wikipedia, Broadcastify, etc. still correlates **your** traffic across those sites — that is intentional per-install attribution, not anonymity
|
||||
|
||||
---
|
||||
|
||||
## LiveUAMap scraper (#348)
|
||||
|
||||
- **Layer:** `global_incidents` (LiveUAMap map pins; **GDELT** text still loads without LiveUAMap)
|
||||
- **Code:** `backend/services/liveuamap_scraper.py` (Playwright + stealth for Turnstile)
|
||||
- **Windows:** Scraper **off** until you enable **Global Incidents** and confirm the UI dialog → `backend/data/liveuamap_scraper_opt_in.json`
|
||||
- **Linux/macOS:** Scraper runs when the layer is on (unless env forces off)
|
||||
- **API:** `GET /api/liveuamap/scraper-status`, `POST /api/liveuamap/scraper-opt-in`
|
||||
- **Env:** `SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=true|false` overrides UI on all platforms
|
||||
- **Honesty:** Backend-only; no browser-direct LiveUAMap from end users. Stealth remains a functional tradeoff for Turnstile; disable layer or env if unacceptable
|
||||
|
||||
## UAP sightings (NUFORC map layer)
|
||||
|
||||
- **Window:** last **60 days** (`NUFORC_RECENT_DAYS`, ~2 months) from **live** nuforc.org
|
||||
- **Cadence:** **Weekly** (Monday 12:00 UTC) per install; typical yield **~400–500** geocoded pins
|
||||
- **Between weeks:** `backend/data/nuforc_recent_sightings.json` (7-day TTL) so restarts do not wipe the layer
|
||||
- **Immediate pull:** admin `GET /api/refresh` on that install
|
||||
- **Not used for map pins:** stale Hugging Face mirror (frozen ~2023) unless live is down and mirror happens to have in-window rows
|
||||
|
||||
---
|
||||
|
||||
## CCTV proxy Referer / Origin (#349)
|
||||
|
||||
- **Code:** `backend/routers/cctv.py`, `backend/main.py`
|
||||
- **Behavior:** Backend proxies streams and sets `Referer` / `Origin` each agency expects (e.g. `https://511ga.org/cctv`, `https://informo.madrid.es/`)
|
||||
- **Exposure:** Agency sees **backend IP**, not each viewer’s browser
|
||||
- **Not removed:** Without these headers, most public DOT/city feeds return 403 — this is not end-user browser impersonation, it is the same headers a normal browser session would send to play the feed
|
||||
|
||||
---
|
||||
|
||||
## Meshtastic map callsign (#350)
|
||||
|
||||
- **Layer:** `sigint_meshtastic` must be active for `fetch_meshtastic_nodes()`
|
||||
- **Default:** `MESHTASTIC_SEND_CALLSIGN_HEADER=false` — callsign **not** sent to `meshtastic.liamcottle.net` unless you set `true`
|
||||
- **Optional:** `MESHTASTIC_OPERATOR_CALLSIGN` for local display; header only when explicitly enabled
|
||||
|
||||
---
|
||||
|
||||
## Basemap CDN (#354)
|
||||
|
||||
- **Code:** `frontend/src/components/map/styles/mapStyles.ts`, `frontend/public/map-style.json`
|
||||
- **Hosts:** `*.basemaps.cartocdn.com`, `demotiles.maplibre.org`
|
||||
- **Exposure:** **Browser** loads tiles (client IP + pan/zoom), not the backend
|
||||
- **Mitigation:** Self-host raster tiles and point MapLibre `sources` at your tile server (operator choice; not required for core features)
|
||||
|
||||
---
|
||||
|
||||
## Broadcastify top feeds (#366)
|
||||
|
||||
- **Code:** `backend/services/radio_intercept.py`
|
||||
- **Behavior:** Backend fetches `https://www.broadcastify.com/listen/top` with per-install handle UA; parses public HTML for feed metadata and CDN stream URLs
|
||||
- **Exposure:** Your backend IP; 5-minute cache
|
||||
- **If blocked:** Panel shows empty list — feature not removed from the app
|
||||
- **Not:** Fake Chrome UA or cloudscraper bypass (removed in Round 7a)
|
||||
|
||||
---
|
||||
|
||||
## Ukraine frontline mirror (#362)
|
||||
|
||||
- **Layer:** `ukraine_frontline` / `frontlines`
|
||||
- **Pin:** `DEEPSTATE_MIRROR_COMMIT`, optional `DEEPSTATE_MIRROR_REPO`
|
||||
|
||||
## Madrid CCTV (#363) / KiwiSDR (#364)
|
||||
|
||||
- Madrid: HTTPS-first KML catalog; image URLs unchanged
|
||||
- KiwiSDR: HTTPS-first directory fetch; shape validation + bundled fallback
|
||||
|
||||
---
|
||||
|
||||
## Operator checklist
|
||||
|
||||
1. Set `OPERATOR_HANDLE` if you want a recognizable name on upstream logs.
|
||||
2. Pin `DEEPSTATE_MIRROR_COMMIT` for reproducible frontlines (optional).
|
||||
3. Windows: enable Global Incidents in UI only if you accept LiveUAMap server contact.
|
||||
4. Set `SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=false` to forbid LiveUAMap entirely.
|
||||
5. Set `MESHTASTIC_SEND_CALLSIGN_HEADER=true` only if you want callsign sent upstream.
|
||||
6. Self-host map tiles if basemap CDN exposure matters.
|
||||
Generated
+288
-292
File diff suppressed because it is too large
Load Diff
+6
-10
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "frontend",
|
||||
"version": "0.9.81",
|
||||
"version": "0.9.8",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "node scripts/dev-all.cjs",
|
||||
@@ -27,10 +27,10 @@
|
||||
"hls.js": "^1.6.15",
|
||||
"lucide-react": "^0.575.0",
|
||||
"maplibre-gl": "^4.7.1",
|
||||
"next": "^16.2.7",
|
||||
"next": "16.1.6",
|
||||
"qrcode": "^1.5.4",
|
||||
"react": "^19.2.7",
|
||||
"react-dom": "^19.2.7",
|
||||
"react": "19.2.4",
|
||||
"react-dom": "^19.2.4",
|
||||
"react-map-gl": "^8.1.0",
|
||||
"satellite.js": "^6.0.2",
|
||||
"zod": "^4.3.6"
|
||||
@@ -42,7 +42,7 @@
|
||||
"@types/node": "^20",
|
||||
"@types/react": "^19",
|
||||
"@types/react-dom": "^19",
|
||||
"@vitest/coverage-v8": "^4.1.8",
|
||||
"@vitest/coverage-v8": "^4.1.0",
|
||||
"concurrently": "^9.2.1",
|
||||
"eslint": "^9",
|
||||
"eslint-config-next": "16.2.4",
|
||||
@@ -50,10 +50,6 @@
|
||||
"prettier": "^3.8.3",
|
||||
"tailwindcss": "^4",
|
||||
"typescript": "^5",
|
||||
"vite": "^8.0.16",
|
||||
"vitest": "^4.1.8"
|
||||
},
|
||||
"overrides": {
|
||||
"postcss": "8.5.15"
|
||||
"vitest": "^4.1.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -147,18 +147,18 @@ describe('middleware matcher exclusions', () => {
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Runtime Google Fonts domains are not required in CSP
|
||||
// 5. Google Fonts domains are preserved in CSP
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('local font CSP', () => {
|
||||
it('style-src does not allow https://fonts.googleapis.com', () => {
|
||||
describe('Google Fonts domains in CSP', () => {
|
||||
it('style-src includes https://fonts.googleapis.com', () => {
|
||||
const csp = getCsp();
|
||||
expect(csp).not.toContain('https://fonts.googleapis.com');
|
||||
expect(csp).toContain('https://fonts.googleapis.com');
|
||||
});
|
||||
|
||||
it('font-src does not allow https://fonts.gstatic.com', () => {
|
||||
it('font-src includes https://fonts.gstatic.com', () => {
|
||||
const csp = getCsp();
|
||||
expect(csp).not.toContain('https://fonts.gstatic.com');
|
||||
expect(csp).toContain('https://fonts.gstatic.com');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -178,9 +178,9 @@ describe('production CSP directive completeness', () => {
|
||||
expect(csp).not.toMatch(/script-src [^;]*'nonce-/);
|
||||
});
|
||||
|
||||
it('has style-src with hydration-compatible inline styles only', () => {
|
||||
it('has style-src with unsafe-inline and fonts.googleapis.com', () => {
|
||||
expect(csp).toMatch(/style-src [^;]*'unsafe-inline'/);
|
||||
expect(csp).not.toMatch(/style-src [^;]*https:\/\/fonts\.googleapis\.com/);
|
||||
expect(csp).toMatch(/style-src [^;]*https:\/\/fonts\.googleapis\.com/);
|
||||
});
|
||||
|
||||
it('has worker-src self blob:', () => {
|
||||
|
||||
@@ -130,17 +130,16 @@ describe('unchanged directives in production', () => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it('style-src preserves unsafe-inline without runtime Google Fonts', () => {
|
||||
it('style-src preserves unsafe-inline and Google Fonts', () => {
|
||||
const styleSrc = getDirective('style-src');
|
||||
expect(styleSrc).toContain("'unsafe-inline'");
|
||||
expect(styleSrc).not.toContain('https://fonts.googleapis.com');
|
||||
expect(styleSrc).toContain('https://fonts.googleapis.com');
|
||||
});
|
||||
|
||||
it('font-src preserves self and data without runtime Google Fonts', () => {
|
||||
it('font-src preserves data: and fonts.gstatic.com', () => {
|
||||
const fontSrc = getDirective('font-src');
|
||||
expect(fontSrc).toContain("'self'");
|
||||
expect(fontSrc).toContain('data:');
|
||||
expect(fontSrc).not.toContain('https://fonts.gstatic.com');
|
||||
expect(fontSrc).toContain('https://fonts.gstatic.com');
|
||||
});
|
||||
|
||||
it('worker-src self blob:', () => {
|
||||
|
||||
@@ -9,12 +9,12 @@ import {
|
||||
} from '@/lib/updateRuntime';
|
||||
|
||||
const RELEASE: GitHubLatestRelease = {
|
||||
html_url: 'https://github.com/BigBodyCobain/Shadowbroker/releases/tag/v0.9.81',
|
||||
html_url: 'https://github.com/BigBodyCobain/Shadowbroker/releases/tag/v0.9.8',
|
||||
assets: [
|
||||
{ name: 'ShadowBroker_0.9.81_x64_en-US.msi', browser_download_url: 'https://example.test/windows.msi' },
|
||||
{ name: 'ShadowBroker_0.9.81_x64-setup.exe', browser_download_url: 'https://example.test/windows-setup.exe' },
|
||||
{ name: 'ShadowBroker_0.9.81_aarch64.dmg', browser_download_url: 'https://example.test/macos.dmg' },
|
||||
{ name: 'ShadowBroker_0.9.81_amd64.AppImage', browser_download_url: 'https://example.test/linux.AppImage' },
|
||||
{ name: 'ShadowBroker_0.9.8_x64_en-US.msi', browser_download_url: 'https://example.test/windows.msi' },
|
||||
{ name: 'ShadowBroker_0.9.8_x64-setup.exe', browser_download_url: 'https://example.test/windows-setup.exe' },
|
||||
{ name: 'ShadowBroker_0.9.8_aarch64.dmg', browser_download_url: 'https://example.test/macos.dmg' },
|
||||
{ name: 'ShadowBroker_0.9.8_amd64.AppImage', browser_download_url: 'https://example.test/linux.AppImage' },
|
||||
],
|
||||
};
|
||||
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
import React from 'react';
|
||||
import { act, cleanup, render, screen } from '@testing-library/react';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('@/mesh/meshIdentity', () => ({
|
||||
getNodeIdentity: vi.fn(() => ({ publicKey: 'test-public-key', nodeId: '!sb_test' })),
|
||||
getWormholeIdentityDescriptor: vi.fn(() => ({ publicKey: 'wormhole-public-key' })),
|
||||
}));
|
||||
|
||||
vi.mock('@/mesh/wormholeIdentityClient', () => ({
|
||||
activateWormholeGatePersona: vi.fn(),
|
||||
createWormholeGatePersona: vi.fn(),
|
||||
enterWormholeGate: vi.fn(),
|
||||
fetchWormholeIdentity: vi.fn(),
|
||||
listWormholeGatePersonas: vi.fn(async () => ({ personas: [] })),
|
||||
}));
|
||||
|
||||
vi.mock('@/components/InfonetTerminal/GateView', () => ({ default: () => <div>Gate view</div> }));
|
||||
vi.mock('@/components/InfonetTerminal/MarketView', () => ({ default: () => <div>Market view</div> }));
|
||||
vi.mock('@/components/InfonetTerminal/ProfileView', () => ({ default: () => <div>Profile view</div> }));
|
||||
vi.mock('@/components/InfonetTerminal/MessagesView', () => ({ default: () => <div>Messages view</div> }));
|
||||
vi.mock('@/components/InfonetTerminal/TerminalDashboard', () => ({ default: () => <div>Dashboard</div> }));
|
||||
vi.mock('@/components/InfonetTerminal/WeatherWidget', () => ({ default: () => <div>Weather</div> }));
|
||||
vi.mock('@/components/InfonetTerminal/TrendingPosts', () => ({ default: () => <div>Trending</div> }));
|
||||
vi.mock('@/components/InfonetTerminal/HashchainEvents', () => ({ default: () => <div>Hashchain</div> }));
|
||||
vi.mock('@/components/InfonetTerminal/NetworkStats', () => ({ default: () => <div>Network stats</div> }));
|
||||
vi.mock('@/components/InfonetTerminal/AIQueryView', () => ({ default: () => <div>AI view</div> }));
|
||||
vi.mock('@/components/InfonetTerminal/PetitionsView', () => ({ default: () => <div>Petitions</div> }));
|
||||
vi.mock('@/components/InfonetTerminal/UpgradeView', () => ({ default: () => <div>Upgrades</div> }));
|
||||
vi.mock('@/components/InfonetTerminal/ResolutionView', () => ({ default: () => <div>Resolution</div> }));
|
||||
vi.mock('@/components/InfonetTerminal/GateShutdownView', () => ({ default: () => <div>Gate shutdown</div> }));
|
||||
vi.mock('@/components/InfonetTerminal/BootstrapView', () => ({ default: () => <div>Bootstrap</div> }));
|
||||
vi.mock('@/components/InfonetTerminal/FunctionKeyView', () => ({ default: () => <div>Function keys</div> }));
|
||||
|
||||
describe('InfonetShell gate directory', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
cleanup();
|
||||
});
|
||||
|
||||
it('renders available gates under the landing logo section', async () => {
|
||||
const { default: InfonetShell } = await import('@/components/InfonetTerminal/InfonetShell');
|
||||
|
||||
render(<InfonetShell isOpen onClose={() => {}} />);
|
||||
await act(async () => {
|
||||
vi.advanceTimersByTime(2500);
|
||||
});
|
||||
|
||||
expect(screen.getByText('AVAILABLE OBFUSCATED GATES:')).toBeTruthy();
|
||||
expect(screen.getByRole('button', { name: /\[>\]\s*infonet/i })).toBeTruthy();
|
||||
expect(screen.getByRole('button', { name: /\[>\]\s*gathered-intel/i })).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -203,19 +203,6 @@ describe('page.tsx decomposition — no admin-session/proxy regression', () => {
|
||||
const locateBar = readAppFile('LocateBar.tsx');
|
||||
expect(locateBar).toContain('API_BASE');
|
||||
expect(locateBar).toContain('/api/geocode/search');
|
||||
expect(locateBar).not.toContain('nominatim.openstreetmap.org');
|
||||
});
|
||||
|
||||
it('useRegionDossier uses backend dossier APIs (no browser-direct enrichment)', () => {
|
||||
const hook = fs.readFileSync(
|
||||
path.resolve(__dirname, '../../hooks/useRegionDossier.ts'),
|
||||
'utf-8',
|
||||
);
|
||||
expect(hook).toContain('/api/region-dossier');
|
||||
expect(hook).toContain('/api/sentinel2/search');
|
||||
expect(hook).not.toContain('nominatim.openstreetmap.org');
|
||||
expect(hook).not.toContain('planetarycomputer.microsoft.com');
|
||||
expect(hook).not.toContain('restcountries.com');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
* - /api/tools/* (Sprint 1C addition)
|
||||
* - /api/wormhole/* (pre-existing, regression)
|
||||
* - /api/settings/* (pre-existing, regression)
|
||||
* - /api/layers, /api/ais/feed, /api/ai/*, /api/sar/mode-b/*
|
||||
* - /api/layers, /api/ais/feed, /api/ai/agent-actions
|
||||
*
|
||||
* Also verifies that:
|
||||
* - non-sensitive mesh paths (e.g. mesh/events) do NOT receive injected key
|
||||
@@ -344,99 +344,6 @@ describe('proxy admin-key injection coverage', () => {
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
it('GET /api/ai/pins with valid session injects X-Admin-Key', async () => {
|
||||
const cookie = await mintSession(ADMIN_KEY);
|
||||
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ ok: true, pins: [] }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost/api/ai/pins?limit=500', {
|
||||
method: 'GET',
|
||||
headers: { cookie },
|
||||
});
|
||||
const res = await proxyGet(req, {
|
||||
params: Promise.resolve({ path: ['ai', 'pins'] }),
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
it('GET /api/ai/layers with valid session injects X-Admin-Key', async () => {
|
||||
const cookie = await mintSession(ADMIN_KEY);
|
||||
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ ok: true, layers: [] }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost/api/ai/layers', {
|
||||
method: 'GET',
|
||||
headers: { cookie },
|
||||
});
|
||||
const res = await proxyGet(req, {
|
||||
params: Promise.resolve({ path: ['ai', 'layers'] }),
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
it('GET /api/ai/timemachine/config with valid session injects X-Admin-Key', async () => {
|
||||
const cookie = await mintSession(ADMIN_KEY);
|
||||
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ ok: true, config: {} }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost/api/ai/timemachine/config', {
|
||||
method: 'GET',
|
||||
headers: { cookie },
|
||||
});
|
||||
const res = await proxyGet(req, {
|
||||
params: Promise.resolve({ path: ['ai', 'timemachine', 'config'] }),
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
it('POST /api/sar/mode-b/enable with valid session injects X-Admin-Key', async () => {
|
||||
const cookie = await mintSession(ADMIN_KEY);
|
||||
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response(JSON.stringify({ ok: true }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost/api/sar/mode-b/enable', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ earthdata_user: 'operator', earthdata_token: 'token' }),
|
||||
headers: { cookie, 'Content-Type': 'application/json' },
|
||||
});
|
||||
const res = await proxyPost(req, {
|
||||
params: Promise.resolve({ path: ['sar', 'mode-b', 'enable'] }),
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Non-sensitive mesh paths must NOT receive injected admin key
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
@@ -110,111 +110,6 @@ describe('proxy CSRF guard on admin-key injection (#249/#254)', () => {
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
it('same-origin request behind a reverse proxy uses X-Forwarded-Host for injection', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://frontend:3000/api/settings/api-keys', {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
host: 'frontend:3000',
|
||||
origin: 'https://shadowbroker.example',
|
||||
'x-forwarded-host': 'shadowbroker.example',
|
||||
},
|
||||
});
|
||||
await proxyGet(req, {
|
||||
params: Promise.resolve({ path: ['settings', 'api-keys'] }),
|
||||
});
|
||||
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
it('same-origin request behind a Docker bridge proxy can use a private Host with X-Forwarded-Host', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://172.18.0.3:3000/api/settings/api-keys', {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
host: '172.18.0.3:3000',
|
||||
origin: 'https://shadowbroker.example',
|
||||
'x-forwarded-host': 'shadowbroker.example',
|
||||
},
|
||||
});
|
||||
await proxyGet(req, {
|
||||
params: Promise.resolve({ path: ['settings', 'api-keys'] }),
|
||||
});
|
||||
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
it('same-origin request behind a reverse proxy uses Forwarded host for injection', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://frontend:3000/api/tools/shodan/status', {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
host: 'frontend:3000',
|
||||
origin: 'https://shadowbroker.example',
|
||||
forwarded: 'for=172.18.0.1;proto=https;host="shadowbroker.example"',
|
||||
},
|
||||
});
|
||||
await proxyGet(req, {
|
||||
params: Promise.resolve({ path: ['tools', 'shodan', 'status'] }),
|
||||
});
|
||||
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
it('cross-origin request cannot spoof same-origin with X-Forwarded-Host on a public Host', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('https://shadowbroker.example/api/settings/api-keys', {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
host: 'shadowbroker.example',
|
||||
origin: 'https://evil.example',
|
||||
'x-forwarded-host': 'evil.example',
|
||||
},
|
||||
});
|
||||
await proxyGet(req, {
|
||||
params: Promise.resolve({ path: ['settings', 'api-keys'] }),
|
||||
});
|
||||
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBeNull();
|
||||
});
|
||||
|
||||
it('cross-origin request cannot spoof same-origin with X-Forwarded-Host on localhost', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost:3000/api/settings/api-keys', {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
host: 'localhost:3000',
|
||||
origin: 'https://evil.example',
|
||||
'x-forwarded-host': 'evil.example',
|
||||
},
|
||||
});
|
||||
await proxyGet(req, {
|
||||
params: Promise.resolve({ path: ['settings', 'api-keys'] }),
|
||||
});
|
||||
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBeNull();
|
||||
});
|
||||
|
||||
it('no Origin header (native shell, server-to-server, curl) DOES inject X-Admin-Key', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }),
|
||||
@@ -235,46 +130,6 @@ describe('proxy CSRF guard on admin-key injection (#249/#254)', () => {
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
it('GET /api/refresh without Origin does NOT receive env ADMIN_KEY fallback', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{}', { status: 403, headers: { 'Content-Type': 'application/json' } }),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost:3000/api/refresh', {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
host: 'localhost:3000',
|
||||
// no Origin
|
||||
},
|
||||
});
|
||||
await proxyGet(req, {
|
||||
params: Promise.resolve({ path: ['refresh'] }),
|
||||
});
|
||||
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBeNull();
|
||||
});
|
||||
|
||||
it('GET /api/refresh with same-origin Origin still receives env ADMIN_KEY fallback', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const req = new NextRequest('http://localhost:3000/api/refresh', {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
host: 'localhost:3000',
|
||||
origin: 'http://localhost:3000',
|
||||
},
|
||||
});
|
||||
await proxyGet(req, {
|
||||
params: Promise.resolve({ path: ['refresh'] }),
|
||||
});
|
||||
|
||||
expect(capturedHeaders(fetchMock).get('X-Admin-Key')).toBe(ADMIN_KEY);
|
||||
});
|
||||
|
||||
it('cross-origin request with a valid session cookie STILL injects (cookie auth wins)', async () => {
|
||||
// Mint a session first (against the real handler).
|
||||
const mintReq = new NextRequest('http://localhost:3000/api/admin/session', {
|
||||
|
||||
@@ -1,8 +1,21 @@
|
||||
/**
|
||||
* #360: Wikipedia / Wikidata traffic is proxied via the self-hosted backend.
|
||||
* Issues #218 / #219 / #220 (tg12 external audit) + Round 7a:
|
||||
*
|
||||
* Every browser-direct call to Wikipedia or Wikidata must send the
|
||||
* `Api-User-Agent` header that Wikimedia's UA policy asks for, AND must
|
||||
* embed the per-install operator handle so Wikimedia can rate-limit /
|
||||
* contact the specific operator instead of treating "Shadowbroker" as
|
||||
* one giant entity.
|
||||
*
|
||||
* These tests pin both requirements on the shared `lib/wikimediaClient`
|
||||
* helper that WikiImage, NewsFeed, and useRegionDossier all route
|
||||
* through. A future refactor that drops either the header OR the
|
||||
* per-operator handle gets a loud test failure rather than a silent
|
||||
* ToS / privacy regression.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
buildWikimediaUserAgent,
|
||||
fetchWikipediaSummary,
|
||||
fetchWikidataSparql,
|
||||
_resetWikimediaClientCacheForTests,
|
||||
@@ -10,6 +23,18 @@ import {
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
// Helper: stub fetch so calls to /api/settings/operator-handle return a
|
||||
// known handle, and everything else proxies to whatever the test set up.
|
||||
function withHandle(handle: string, otherFetch: typeof globalThis.fetch) {
|
||||
return vi.fn(async (input: any, init?: RequestInit) => {
|
||||
const url = String(input);
|
||||
if (url.endsWith('/api/settings/operator-handle')) {
|
||||
return new Response(JSON.stringify({ handle }), { status: 200 });
|
||||
}
|
||||
return otherFetch(input, init);
|
||||
});
|
||||
}
|
||||
|
||||
describe('lib/wikimediaClient', () => {
|
||||
beforeEach(() => {
|
||||
_resetWikimediaClientCacheForTests();
|
||||
@@ -20,78 +45,194 @@ describe('lib/wikimediaClient', () => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('fetches Wikipedia summary through backend proxy', async () => {
|
||||
const calls: string[] = [];
|
||||
it('builds a stable per-operator Api-User-Agent with contact path', async () => {
|
||||
globalThis.fetch = withHandle(
|
||||
'operator-abc123',
|
||||
vi.fn(async () => new Response('{}', { status: 200 })) as any,
|
||||
) as any;
|
||||
const ua = await buildWikimediaUserAgent('wikipedia-summary');
|
||||
expect(ua).toContain('Shadowbroker');
|
||||
expect(ua.toLowerCase()).toContain('github.com');
|
||||
expect(ua.toLowerCase()).toContain('issues');
|
||||
expect(ua).toContain('operator: operator-abc123');
|
||||
expect(ua).toContain('purpose: wikipedia-summary');
|
||||
});
|
||||
|
||||
it('falls back to "operator-offline" when handle endpoint is unreachable', async () => {
|
||||
globalThis.fetch = vi.fn(async (input: any) => {
|
||||
calls.push(String(input));
|
||||
const url = String(input);
|
||||
if (url.endsWith('/api/settings/operator-handle')) {
|
||||
return new Response('forbidden', { status: 403 });
|
||||
}
|
||||
return new Response('{}', { status: 200 });
|
||||
}) as any;
|
||||
const ua = await buildWikimediaUserAgent('test');
|
||||
expect(ua).toContain('operator: operator-offline');
|
||||
});
|
||||
|
||||
it('sends per-operator Api-User-Agent on Wikipedia summary fetch', async () => {
|
||||
const wikiCalls: Array<{ url: string; init?: RequestInit }> = [];
|
||||
const baseFetch = vi.fn(async (url: any, init?: RequestInit) => {
|
||||
wikiCalls.push({ url: String(url), init });
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
type: 'standard',
|
||||
title: 'Boeing 747',
|
||||
description: 'aircraft',
|
||||
extract: 'long extract',
|
||||
thumbnail: 'https://example.org/thumb.jpg',
|
||||
type: 'standard',
|
||||
thumbnail: { source: 'https://example.org/thumb.jpg' },
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
}) as any;
|
||||
});
|
||||
globalThis.fetch = withHandle('operator-test01', baseFetch as any) as any;
|
||||
|
||||
const summary = await fetchWikipediaSummary('Boeing 747');
|
||||
expect(summary?.thumbnail).toBe('https://example.org/thumb.jpg');
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0]).toContain('/api/wikipedia/summary');
|
||||
expect(calls[0]).not.toContain('wikipedia.org');
|
||||
// wikiCalls only captures calls to non-handle URLs.
|
||||
expect(wikiCalls).toHaveLength(1);
|
||||
const headers = (wikiCalls[0].init?.headers || {}) as Record<string, string>;
|
||||
expect(headers['Api-User-Agent']).toContain('operator: operator-test01');
|
||||
expect(headers['Api-User-Agent']).toContain('purpose: wikipedia-summary');
|
||||
});
|
||||
|
||||
it('fetches Wikidata SPARQL through backend proxy', async () => {
|
||||
it('sends per-operator Api-User-Agent on Wikidata SPARQL fetch', async () => {
|
||||
const calls: Array<{ url: string; init?: RequestInit }> = [];
|
||||
globalThis.fetch = vi.fn(async (url: any, init?: RequestInit) => {
|
||||
const baseFetch = vi.fn(async (url: any, init?: RequestInit) => {
|
||||
calls.push({ url: String(url), init });
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
bindings: [{ leaderLabel: { value: 'Test Leader' } }],
|
||||
results: { bindings: [{ leaderLabel: { value: 'Test Leader' } }] },
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
}) as any;
|
||||
});
|
||||
globalThis.fetch = withHandle('operator-sparql', baseFetch as any) as any;
|
||||
|
||||
const bindings = await fetchWikidataSparql('SELECT * WHERE { ?s ?p ?o }');
|
||||
expect(bindings).toHaveLength(1);
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0].url).toContain('/api/wikidata/sparql');
|
||||
expect(calls[0].init?.method).toBe('POST');
|
||||
expect(calls[0].url).not.toContain('wikidata.org');
|
||||
const headers = (calls[0].init?.headers || {}) as Record<string, string>;
|
||||
expect(headers['Api-User-Agent']).toContain('operator: operator-sparql');
|
||||
expect(headers['Api-User-Agent']).toContain('purpose: wikidata-sparql');
|
||||
expect(headers['Accept']).toBe('application/sparql-results+json');
|
||||
});
|
||||
|
||||
it('deduplicates concurrent Wikipedia summary requests', async () => {
|
||||
let hits = 0;
|
||||
globalThis.fetch = vi.fn(async () => {
|
||||
hits += 1;
|
||||
it('handle endpoint is queried only ONCE across many wiki fetches', async () => {
|
||||
let handleCalls = 0;
|
||||
let wikiCalls = 0;
|
||||
globalThis.fetch = vi.fn(async (input: any) => {
|
||||
const url = String(input);
|
||||
if (url.endsWith('/api/settings/operator-handle')) {
|
||||
handleCalls++;
|
||||
return new Response(JSON.stringify({ handle: 'operator-cache' }), { status: 200 });
|
||||
}
|
||||
wikiCalls++;
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
title: 'Mount Fuji',
|
||||
description: 'mountain',
|
||||
extract: 'extract',
|
||||
thumbnail: '',
|
||||
type: 'standard',
|
||||
title: 'X',
|
||||
description: '',
|
||||
extract: '',
|
||||
thumbnail: { source: 'https://example.org/x.jpg' },
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
}) as any;
|
||||
|
||||
await fetchWikipediaSummary('Eiffel Tower');
|
||||
await fetchWikipediaSummary('Mount Fuji');
|
||||
await fetchWikipediaSummary('Statue of Liberty');
|
||||
expect(handleCalls).toBe(1);
|
||||
expect(wikiCalls).toBe(3);
|
||||
});
|
||||
|
||||
it('shares cache across consecutive callers for the same Wikipedia title', async () => {
|
||||
let fetchCount = 0;
|
||||
const baseFetch = vi.fn(async () => {
|
||||
fetchCount++;
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
type: 'standard',
|
||||
title: 'Eiffel Tower',
|
||||
description: 'iron lattice tower',
|
||||
extract: '...',
|
||||
thumbnail: { source: 'https://example.org/eiffel.jpg' },
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
});
|
||||
globalThis.fetch = withHandle('operator-cache', baseFetch as any) as any;
|
||||
|
||||
const a = await fetchWikipediaSummary('Eiffel Tower');
|
||||
const b = await fetchWikipediaSummary('Eiffel Tower');
|
||||
expect(fetchCount).toBe(1);
|
||||
expect(a?.thumbnail).toBe(b?.thumbnail);
|
||||
});
|
||||
|
||||
it('deduplicates concurrent in-flight requests for the same title', async () => {
|
||||
let fetchCount = 0;
|
||||
const baseFetch = vi.fn(async () => {
|
||||
fetchCount++;
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
type: 'standard',
|
||||
title: 'Mount Fuji',
|
||||
description: 'stratovolcano',
|
||||
extract: '...',
|
||||
thumbnail: { source: 'https://example.org/fuji.jpg' },
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
});
|
||||
globalThis.fetch = withHandle('operator-cache', baseFetch as any) as any;
|
||||
|
||||
const [a, b, c] = await Promise.all([
|
||||
fetchWikipediaSummary('Mount Fuji'),
|
||||
fetchWikipediaSummary('Mount Fuji'),
|
||||
fetchWikipediaSummary('Mount Fuji'),
|
||||
]);
|
||||
expect(a?.title).toBe('Mount Fuji');
|
||||
expect(fetchCount).toBe(1);
|
||||
expect(a?.thumbnail).toBe('https://example.org/fuji.jpg');
|
||||
expect(b).toEqual(a);
|
||||
expect(c).toEqual(a);
|
||||
expect(hits).toBe(1);
|
||||
});
|
||||
|
||||
it('returns null on Wikipedia 404', async () => {
|
||||
globalThis.fetch = vi.fn(async () => new Response('{}', { status: 404 })) as any;
|
||||
expect(await fetchWikipediaSummary('Nonexistent Article 12345')).toBeNull();
|
||||
it('returns null on disambiguation pages without throwing', async () => {
|
||||
globalThis.fetch = withHandle(
|
||||
'operator-cache',
|
||||
vi.fn(async () =>
|
||||
new Response(JSON.stringify({ type: 'disambiguation' }), { status: 200 }),
|
||||
) as any,
|
||||
) as any;
|
||||
const summary = await fetchWikipediaSummary('Mercury');
|
||||
expect(summary).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null on HTTP error without throwing', async () => {
|
||||
globalThis.fetch = withHandle(
|
||||
'operator-cache',
|
||||
vi.fn(async () => new Response('not found', { status: 404 })) as any,
|
||||
) as any;
|
||||
const summary = await fetchWikipediaSummary('Nonexistent Article 12345');
|
||||
expect(summary).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null on network error without throwing', async () => {
|
||||
globalThis.fetch = withHandle(
|
||||
'operator-cache',
|
||||
vi.fn(async () => {
|
||||
throw new Error('network down');
|
||||
}) as any,
|
||||
) as any;
|
||||
const summary = await fetchWikipediaSummary('Anything');
|
||||
expect(summary).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null on empty input without fetching anything', async () => {
|
||||
globalThis.fetch = vi.fn(async () => new Response('{}', { status: 200 })) as any;
|
||||
expect(await fetchWikipediaSummary('')).toBeNull();
|
||||
expect(await fetchWikipediaSummary(' ')).toBeNull();
|
||||
expect(globalThis.fetch).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -12,7 +12,6 @@ export function LocateBar({ onLocate, onOpenChange }: { onLocate: (lat: number,
|
||||
const [value, setValue] = useState('');
|
||||
const [results, setResults] = useState<{ label: string; lat: number; lng: number }[]>([]);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [searchError, setSearchError] = useState<string | null>(null);
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
const timerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const searchAbortRef = useRef<AbortController | null>(null);
|
||||
@@ -59,15 +58,14 @@ export function LocateBar({ onLocate, onOpenChange }: { onLocate: (lat: number,
|
||||
if (searchAbortRef.current) searchAbortRef.current.abort();
|
||||
if (q.trim().length < 2) {
|
||||
setResults([]);
|
||||
setSearchError(null);
|
||||
return;
|
||||
}
|
||||
timerRef.current = setTimeout(async () => {
|
||||
setLoading(true);
|
||||
setSearchError(null);
|
||||
searchAbortRef.current = new AbortController();
|
||||
const signal = searchAbortRef.current.signal;
|
||||
try {
|
||||
// Try backend proxy first (has caching + rate-limit compliance)
|
||||
const res = await fetch(
|
||||
`${API_BASE}/api/geocode/search?q=${encodeURIComponent(q)}&limit=5`,
|
||||
{ signal },
|
||||
@@ -82,19 +80,43 @@ export function LocateBar({ onLocate, onOpenChange }: { onLocate: (lat: number,
|
||||
}),
|
||||
);
|
||||
setResults(mapped);
|
||||
if (mapped.length === 0) {
|
||||
setSearchError('No places found');
|
||||
}
|
||||
} else {
|
||||
console.warn(`[Locate] Geocode proxy HTTP ${res.status}`);
|
||||
setResults([]);
|
||||
setSearchError('Place search unavailable — check backend connection');
|
||||
// Backend proxy returned an error — fall back to direct Nominatim
|
||||
console.warn(`[Locate] Proxy returned HTTP ${res.status}, falling back to Nominatim`);
|
||||
const directRes = await fetch(
|
||||
`https://nominatim.openstreetmap.org/search?q=${encodeURIComponent(q)}&format=json&limit=5`,
|
||||
{ headers: { 'Accept-Language': 'en' }, signal },
|
||||
);
|
||||
const data = await directRes.json();
|
||||
setResults(
|
||||
data.map((r: { display_name: string; lat: string; lon: string }) => ({
|
||||
label: r.display_name,
|
||||
lat: parseFloat(r.lat),
|
||||
lng: parseFloat(r.lon),
|
||||
})),
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
if ((err as Error)?.name !== 'AbortError') {
|
||||
console.warn('[Locate] Geocode proxy failed:', err);
|
||||
// Proxy completely failed — try direct Nominatim as last resort
|
||||
try {
|
||||
const directRes = await fetch(
|
||||
`https://nominatim.openstreetmap.org/search?q=${encodeURIComponent(q)}&format=json&limit=5`,
|
||||
{ headers: { 'Accept-Language': 'en' } },
|
||||
);
|
||||
const data = await directRes.json();
|
||||
setResults(
|
||||
data.map((r: { display_name: string; lat: string; lon: string }) => ({
|
||||
label: r.display_name,
|
||||
lat: parseFloat(r.lat),
|
||||
lng: parseFloat(r.lon),
|
||||
})),
|
||||
);
|
||||
} catch {
|
||||
setResults([]);
|
||||
}
|
||||
} else {
|
||||
setResults([]);
|
||||
setSearchError('Place search unavailable — check backend connection');
|
||||
}
|
||||
} finally {
|
||||
setLoading(false);
|
||||
@@ -194,11 +216,6 @@ export function LocateBar({ onLocate, onOpenChange }: { onLocate: (lat: number,
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
{searchError && results.length === 0 && !loading && value.trim().length >= 2 && (
|
||||
<div className="absolute bottom-full left-0 right-0 mb-1 bg-[var(--bg-secondary)] border border-amber-800/50 px-3 py-2 text-[10px] font-mono text-amber-200/90">
|
||||
{searchError}
|
||||
</div>
|
||||
)}
|
||||
{results.length > 0 && (
|
||||
<div className="absolute bottom-full left-0 right-0 mb-1 bg-[var(--bg-secondary)] border border-[var(--border-primary)] overflow-hidden shadow-[0_-8px_30px_rgba(0,0,0,0.4)] max-h-[200px] overflow-y-auto styled-scrollbar">
|
||||
{results.map((r, i) => (
|
||||
|
||||
@@ -66,10 +66,7 @@ function isSensitiveProxyPath(pathSegments: string[]): boolean {
|
||||
if (joined === 'system/update') return true;
|
||||
if (joined === 'layers') return true;
|
||||
if (joined === 'ais/feed') return true;
|
||||
if (pathSegments[0] === 'ai') return true;
|
||||
if (pathSegments[0] === 'sar' && (pathSegments[1] === 'mode-b' || pathSegments[1] === 'aois')) {
|
||||
return true;
|
||||
}
|
||||
if (joined === 'ai/agent-actions') return true;
|
||||
if (pathSegments[0] === 'settings') return true;
|
||||
if (joined === 'mesh/infonet/ingest') return true;
|
||||
if (joined === 'mesh/meshtastic/send') return true;
|
||||
@@ -80,72 +77,6 @@ function isSensitiveProxyPath(pathSegments: string[]): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
function normalizeHeaderHost(host: string | null): string {
|
||||
return (host || '').trim().replace(/^"|"$/g, '').toLowerCase();
|
||||
}
|
||||
|
||||
function hostnameFromHeaderHost(host: string): string {
|
||||
const normalized = normalizeHeaderHost(host);
|
||||
if (!normalized) return '';
|
||||
try {
|
||||
return new URL(`http://${normalized}`).hostname.toLowerCase();
|
||||
} catch {
|
||||
return normalized.replace(/:\d+$/, '').toLowerCase();
|
||||
}
|
||||
}
|
||||
|
||||
function isPrivateIpv4(hostname: string): boolean {
|
||||
const parts = hostname.split('.').map((part) => Number(part));
|
||||
if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) {
|
||||
return false;
|
||||
}
|
||||
const [first, second] = parts;
|
||||
return first === 10 || (first === 172 && second >= 16 && second <= 31) || (first === 192 && second === 168);
|
||||
}
|
||||
|
||||
function isInternalProxyHost(host: string): boolean {
|
||||
const hostname = hostnameFromHeaderHost(host);
|
||||
if (!hostname || hostname === 'localhost' || hostname === '127.0.0.1' || hostname === '::1') {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
!hostname.includes('.') ||
|
||||
isPrivateIpv4(hostname) ||
|
||||
hostname.endsWith('.internal') ||
|
||||
hostname.endsWith('.docker')
|
||||
);
|
||||
}
|
||||
|
||||
function forwardedHostCandidates(req: NextRequest): string[] {
|
||||
const hosts = new Set<string>();
|
||||
const directHost = normalizeHeaderHost(req.headers.get('host'));
|
||||
if (directHost) hosts.add(directHost);
|
||||
|
||||
if (!isInternalProxyHost(directHost)) {
|
||||
return [...hosts];
|
||||
}
|
||||
|
||||
const forwardedHost = req.headers.get('x-forwarded-host');
|
||||
if (forwardedHost) {
|
||||
for (const value of forwardedHost.split(',')) {
|
||||
const host = normalizeHeaderHost(value);
|
||||
if (host) hosts.add(host);
|
||||
}
|
||||
}
|
||||
|
||||
const forwarded = req.headers.get('forwarded');
|
||||
if (forwarded) {
|
||||
const hostPattern = /(?:^|[;,])\s*host=(?:"([^"]+)"|([^;,]+))/gi;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = hostPattern.exec(forwarded)) !== null) {
|
||||
const host = normalizeHeaderHost(match[1] || match[2] || '');
|
||||
if (host) hosts.add(host);
|
||||
}
|
||||
}
|
||||
|
||||
return [...hosts];
|
||||
}
|
||||
|
||||
/**
|
||||
* CSRF guard for the server-side admin-key injection (issues #249 / #254).
|
||||
*
|
||||
@@ -160,10 +91,8 @@ function forwardedHostCandidates(req: NextRequest): string[] {
|
||||
* - The request carries a valid admin session cookie (already auth'd)
|
||||
* - The Origin header is absent (server-to-server fetch, Tauri/Electron
|
||||
* native shells, curl/cli — none of these are browser-CSRF surfaces)
|
||||
* - The Origin header host matches the request's own Host or, when the
|
||||
* direct Host is an internal service name, a reverse proxy's forwarded
|
||||
* host (genuine same-origin browser fetch from our own dashboard,
|
||||
* including Docker/Traefik deployments where Host is internal)
|
||||
* - The Origin header host matches the request's own Host (genuine
|
||||
* same-origin browser fetch from our own dashboard)
|
||||
*
|
||||
* If Origin is present AND doesn't match Host, the caller is a hostile
|
||||
* cross-origin webpage. We refuse to inject the admin key. The backend
|
||||
@@ -181,30 +110,15 @@ function isSameOriginOrNonBrowser(req: NextRequest): boolean {
|
||||
}
|
||||
try {
|
||||
const originUrl = new URL(origin);
|
||||
const originHost = normalizeHeaderHost(originUrl.host);
|
||||
if (!originHost) return false;
|
||||
return forwardedHostCandidates(req).includes(originHost);
|
||||
const host = req.headers.get('host') || '';
|
||||
if (!host) return false;
|
||||
return originUrl.host.toLowerCase() === host.toLowerCase();
|
||||
} catch {
|
||||
// Malformed Origin header — be conservative.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function canUseEnvAdminKey(req: NextRequest, pathSegments: string[]): boolean {
|
||||
const joined = pathSegments.join('/');
|
||||
const origin = req.headers.get('origin');
|
||||
|
||||
// /api/refresh is a state-changing GET. A browser can trigger a no-Origin
|
||||
// GET through ambient mechanisms such as image/script navigation, so do not
|
||||
// give that shape the server-side ADMIN_KEY fallback. Valid admin-session
|
||||
// cookies are still handled separately below.
|
||||
if (joined === 'refresh' && req.method.toUpperCase() === 'GET' && !origin) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return isSameOriginOrNonBrowser(req);
|
||||
}
|
||||
|
||||
async function proxy(req: NextRequest, pathSegments: string[]): Promise<NextResponse> {
|
||||
try {
|
||||
const isMesh = pathSegments[0] === 'mesh';
|
||||
@@ -326,7 +240,7 @@ async function proxy(req: NextRequest, pathSegments: string[]): Promise<NextResp
|
||||
// ADMIN_KEY just by being open in the same browser.
|
||||
const cookieToken = req.cookies.get(ADMIN_COOKIE)?.value || '';
|
||||
const sessionAdminKey = resolveAdminSessionToken(cookieToken) || '';
|
||||
const allowEnvKeyInjection = canUseEnvAdminKey(req, pathSegments);
|
||||
const allowEnvKeyInjection = isSameOriginOrNonBrowser(req);
|
||||
let injectedAdmin = '';
|
||||
if (sessionAdminKey) {
|
||||
// Authenticated session always works — Origin doesn't matter
|
||||
|
||||
@@ -55,7 +55,7 @@
|
||||
body {
|
||||
background: var(--background);
|
||||
color: var(--foreground);
|
||||
font-family: var(--font-jetbrains-mono), var(--font-roboto-mono), 'Roboto Mono', monospace;
|
||||
font-family: 'JetBrains Mono', var(--font-roboto-mono), 'Roboto Mono', monospace;
|
||||
}
|
||||
|
||||
/* Global interactive cursor hints */
|
||||
@@ -139,7 +139,7 @@ textarea:disabled {
|
||||
padding: 12px 16px;
|
||||
color: #d1d5db;
|
||||
font-family:
|
||||
var(--font-jetbrains-mono), var(--font-roboto-mono), 'Roboto Mono', monospace, 'Microsoft YaHei', 'PingFang SC',
|
||||
'JetBrains Mono', var(--font-roboto-mono), 'Roboto Mono', monospace, 'Microsoft YaHei', 'PingFang SC',
|
||||
'Noto Sans SC', 'Noto Sans JP', 'Noto Sans KR', sans-serif;
|
||||
font-size: 13px;
|
||||
min-width: 240px;
|
||||
@@ -377,7 +377,7 @@ textarea:disabled {
|
||||
/* ── INFONET CRT TERMINAL EFFECTS ── */
|
||||
|
||||
.infonet-font {
|
||||
font-family: var(--font-jetbrains-mono), ui-monospace, SFMono-Regular, monospace;
|
||||
font-family: 'JetBrains Mono', ui-monospace, SFMono-Regular, monospace;
|
||||
}
|
||||
|
||||
/* CRT scanline overlay — scoped to .crt containers only */
|
||||
|
||||
@@ -1,17 +1,9 @@
|
||||
import type { Metadata } from 'next';
|
||||
import { JetBrains_Mono } from 'next/font/google';
|
||||
import DesktopBridgeBootstrap from '@/components/DesktopBridgeBootstrap';
|
||||
import { ThemeProvider } from '@/lib/ThemeContext';
|
||||
import { I18nProvider } from '@/i18n';
|
||||
import './globals.css';
|
||||
|
||||
const jetBrainsMono = JetBrains_Mono({
|
||||
subsets: ['latin'],
|
||||
weight: ['400', '700'],
|
||||
display: 'swap',
|
||||
variable: '--font-jetbrains-mono',
|
||||
});
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: 'WORLDVIEW // ORBITAL TRACKING',
|
||||
description: 'Advanced Geopolitical Risk Dashboard',
|
||||
@@ -30,7 +22,12 @@ export default function RootLayout({
|
||||
}>) {
|
||||
return (
|
||||
<html lang="en">
|
||||
<body className={`${jetBrainsMono.variable} antialiased bg-[var(--bg-primary)]`} suppressHydrationWarning>
|
||||
<head>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossOrigin="anonymous" />
|
||||
<link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono:wght@400;700&display=swap" rel="stylesheet" />
|
||||
</head>
|
||||
<body className="antialiased bg-[var(--bg-primary)]" suppressHydrationWarning>
|
||||
<I18nProvider>
|
||||
<ThemeProvider>
|
||||
<DesktopBridgeBootstrap />
|
||||
|
||||
@@ -632,18 +632,10 @@ function ConnectModalBody({ apiEndpoint, handleCopy, copied }: ConnectModalBodyP
|
||||
const buildSnippet = (key: string) => {
|
||||
const lines: string[] = [];
|
||||
if (resolvedUrl) lines.push(`SHADOWBROKER_URL=${resolvedUrl}`);
|
||||
lines.push(`SHADOWBROKER_HMAC_SECRET=${key}`);
|
||||
lines.push(`SHADOWBROKER_KEY=${key}`);
|
||||
lines.push(`SHADOWBROKER_ACCESS=${accessTier}`);
|
||||
if (connectionMode === 'remote' && nodeId) lines.push(`SHADOWBROKER_NODE_ID=${nodeId}`);
|
||||
lines.push('');
|
||||
lines.push('# AUTH DIRECTIVE:');
|
||||
lines.push('# SHADOWBROKER_HMAC_SECRET is a shared HMAC signing secret, not a raw API key.');
|
||||
lines.push('# Never send it as X-Admin-Key, Authorization: Bearer, a query parameter, or a plain request header.');
|
||||
lines.push('# Every direct ShadowBroker API request must be HMAC-SHA256 signed with:');
|
||||
lines.push('# X-SB-Timestamp, X-SB-Nonce, X-SB-Signature');
|
||||
lines.push('# Signature input: METHOD|path|timestamp|nonce|sha256(body)');
|
||||
lines.push('# Use the ShadowBrokerClient/openclaw skill helper so requests are signed automatically.');
|
||||
lines.push('');
|
||||
lines.push('# OPERATING DIRECTIVE:');
|
||||
lines.push('# You are a remote OpenClaw agent connected to ShadowBroker.');
|
||||
lines.push('# ShadowBroker is a live intelligence and telemetry platform, not a narrow single-purpose API.');
|
||||
|
||||
@@ -20,22 +20,22 @@ import {
|
||||
Heart,
|
||||
} from 'lucide-react';
|
||||
|
||||
const CURRENT_VERSION = '0.9.81';
|
||||
const CURRENT_VERSION = '0.9.8';
|
||||
const STORAGE_KEY = `shadowbroker_changelog_v${CURRENT_VERSION}`;
|
||||
const RELEASE_TITLE = 'Signed Auto-Update + Update Button Race Fix';
|
||||
const RELEASE_TITLE = 'Cumulative Fuel/CO2, AIS Resilience, and Data-Layer Repair';
|
||||
|
||||
const HEADLINE_FEATURES = [
|
||||
{
|
||||
icon: <KeyRound size={20} className="text-purple-400" />,
|
||||
icon: <Plane size={20} className="text-orange-400" />,
|
||||
accent: 'purple' as const,
|
||||
title: 'Signed Auto-Update Going Forward (one manual hop)',
|
||||
subtitle: 'After installing v0.9.81, the in-app Update button finally works end-to-end. This release establishes a fresh signing key — every release from here is a one-click upgrade.',
|
||||
title: 'Cumulative Fuel & CO2 per Flight',
|
||||
subtitle: 'The aircraft tooltip now shows how much fuel each plane has actually burned in the air, not just the per-hour rate.',
|
||||
details: [
|
||||
'tauri.conf.json now carries a fresh minisign pubkey (the previous keypair was generated before v0.9.79 shipped but the matching private key was lost before any release was actually signed, so no release before v0.9.81 has working auto-update).',
|
||||
'The v0.9.81 release artifacts ship with a signed latest.json + .sig files so every install on v0.9.81 or later can verify and apply the next release automatically via the Tauri updater plugin.',
|
||||
'One-time cost: if you are upgrading from v0.9.79 or v0.9.8, the click-Update path falls back to a manual download because the new pubkey does not match the one baked into your install. Click the MANUAL DOWNLOAD button in the update dialog → grab the .msi from the release page → run it → from then on auto-update works in-app.',
|
||||
'New flight_observations module tracks first-seen-at per ICAO24 hex. Multiplies the model-based rate by elapsed observation time to produce running totals — FUEL BURNED (gal) and CO2 EMITTED (kg) — in the EMISSIONS ESTIMATE block.',
|
||||
'15-minute gap between sightings resets the session (treated as a new flight: landed and took off again, or transited a dead zone). The cumulative counter survives trail pruning so map-rendering lifecycle and emission tracking are independent.',
|
||||
'24-hour clamp defends against clock-skew bugs; per-icao prune every 5 minutes keeps memory bounded. The per-hour rate is still shown as smaller context underneath each cumulative figure.',
|
||||
],
|
||||
callToAction: 'CLICK UPDATE → DOWNLOAD MSI ONCE → AUTO-UPDATE FOREVER',
|
||||
callToAction: 'OPEN A FLIGHT TOOLTIP → EMISSIONS ESTIMATE',
|
||||
},
|
||||
{
|
||||
icon: <Network size={20} className="text-amber-400" />,
|
||||
@@ -64,11 +64,6 @@ const HEADLINE_FEATURES = [
|
||||
];
|
||||
|
||||
const NEW_FEATURES = [
|
||||
{
|
||||
icon: <Plane size={18} className="text-orange-400" />,
|
||||
title: 'Cumulative Fuel & CO2 per Flight',
|
||||
desc: 'Aircraft tooltip now shows how much fuel each plane has actually burned in the air since first observation, not just the per-hour rate. 15-minute gap between sightings resets the session; 24-hour clamp protects against clock skew; per-icao prune every 5 minutes keeps memory bounded.',
|
||||
},
|
||||
{
|
||||
icon: <Plane size={18} className="text-cyan-400" />,
|
||||
title: 'Per-Flight Source Attribution',
|
||||
@@ -87,8 +82,6 @@ const NEW_FEATURES = [
|
||||
];
|
||||
|
||||
const BUG_FIXES = [
|
||||
'Update button no longer throws "admin_session_required" on desktop installs. The initial updateAction now syncs to Tauri detection at React-init time (window.__TAURI__ is injected before mount), so a click before the async runtime probe completes opens the GitHub release page in a browser instead of POSTing to /api/system/update.',
|
||||
'Desktop installer now bundles defusedxml + PySocks (declared in pyproject.toml but missing from the venv shipped with v0.9.79 and the initial v0.9.8 publish). Fixes the bundled-backend launch crash reported in #319 and #296 (managed_backend_exited_early:exit code: 103).',
|
||||
'UAP layer no longer serves 3-year-old NUFORC sightings via the Hugging Face static-mirror fallback (60-day cutoff now applied to the fallback path too).',
|
||||
'GPS jamming detection now counts nac_p == 0 (the actual GPS-lost signal) instead of filtering it out as an old-transponder artifact.',
|
||||
'GPS jamming thresholds lowered (MIN_AIRCRAFT 5 → 3, MIN_RATIO 0.30 → 0.20) so sparser hotspots clear the bar without losing the 1-aircraft noise cushion.',
|
||||
|
||||
@@ -38,17 +38,19 @@ export default function FunctionKeyView({ onBack }: FunctionKeyViewProps) {
|
||||
<ChevronLeft size={14} className="mr-1" /> BACK
|
||||
</button>
|
||||
<div className="text-sm text-purple-400 font-bold uppercase tracking-widest flex items-center gap-2">
|
||||
<KeyRound size={16} /> FUNCTION KEYS — Credential Scaffolding
|
||||
<KeyRound size={16} /> FUNCTION KEYS — Anonymous Citizenship Proof
|
||||
</div>
|
||||
<div />
|
||||
</div>
|
||||
|
||||
<div className="flex-1 overflow-y-auto pr-3 space-y-4">
|
||||
<div className="text-xs text-gray-400 leading-relaxed">
|
||||
Function Keys wire the nullifier, receipt, and settlement plumbing for future
|
||||
anonymous credential proofs. The current challenge-response is an HMAC placeholder,
|
||||
not production zero-knowledge citizenship. True unlinkable issuance still waits on
|
||||
blind signatures or anonymous credentials.
|
||||
A citizen proves "I am an Infonet citizen" to a real-world
|
||||
operator <span className="text-purple-400">without revealing their Infonet identity</span>.
|
||||
The naive approach (scramble a public key, record each redemption on chain) leaks
|
||||
identity through metadata correlation. The Function Keys design is six pieces;
|
||||
five are implemented; one (issuance via blind signatures / anonymous credentials)
|
||||
waits on a cryptographic primitive decision.
|
||||
</div>
|
||||
|
||||
{status && (
|
||||
|
||||
@@ -298,33 +298,6 @@ export default function InfonetShell({
|
||||
setCurrentView(view);
|
||||
};
|
||||
|
||||
const renderGateDirectory = (variant: 'landing' | 'command' = 'command') => (
|
||||
<div
|
||||
className={
|
||||
variant === 'landing'
|
||||
? 'w-full max-w-3xl border border-cyan-950/50 bg-black/20 px-4 py-3 text-left shadow-[0_0_18px_rgba(6,182,212,0.06)]'
|
||||
: 'text-gray-400'
|
||||
}
|
||||
>
|
||||
<p className={`${variant === 'landing' ? 'text-[11px]' : ''} text-gray-400 uppercase tracking-[0.18em]`}>
|
||||
AVAILABLE OBFUSCATED GATES:
|
||||
</p>
|
||||
<div className={`grid grid-cols-1 sm:grid-cols-2 xl:grid-cols-3 ${variant === 'landing' ? 'gap-x-8 gap-y-1.5 mt-2' : 'gap-2 mt-2'}`}>
|
||||
{GATES.map(gate => (
|
||||
<button
|
||||
key={gate}
|
||||
type="button"
|
||||
className="group flex min-h-[24px] items-center text-left text-gray-300 hover:text-white transition-colors"
|
||||
onClick={() => handleNavigate('gate', gate)}
|
||||
>
|
||||
<span className="text-gray-500 mr-2 group-hover:text-cyan-400 transition-colors">[{'>'}]</span>
|
||||
<span className="truncate group-hover:drop-shadow-[0_0_5px_rgba(6,182,212,0.8)]">{gate}</span>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
const openGateWhenReady = async (
|
||||
gateTarget: string,
|
||||
operation: () => Promise<void>,
|
||||
@@ -498,7 +471,19 @@ export default function InfonetShell({
|
||||
setHistory([]);
|
||||
return;
|
||||
} else if (trimmedCmd === 'gates') {
|
||||
output = renderGateDirectory('command');
|
||||
output = (
|
||||
<div className="text-gray-400">
|
||||
<p>AVAILABLE OBFUSCATED GATES:</p>
|
||||
<div className="grid grid-cols-2 md:grid-cols-3 gap-2 mt-2">
|
||||
{GATES.map(gate => (
|
||||
<div key={gate} className="flex items-center cursor-pointer hover:text-gray-300 group" onClick={() => handleNavigate('gate', gate)}>
|
||||
<span className="text-gray-500 mr-2 group-hover:text-cyan-400 transition-colors">[{'>'}]</span>
|
||||
<span className="text-gray-300 group-hover:text-white transition-colors group-hover:drop-shadow-[0_0_5px_rgba(6,182,212,0.8)]">{gate}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
} else if (trimmedCmd.startsWith('join ') || trimmedCmd.startsWith('g/')) {
|
||||
const target = trimmedCmd.startsWith('g/') ? trimmedCmd.slice(2) : trimmedCmd.split(' ')[1];
|
||||
if (GATES.includes(target)) {
|
||||
@@ -676,9 +661,6 @@ export default function InfonetShell({
|
||||
<p>Type <span className="text-green-400 font-bold">'gates'</span> or <span className="text-green-400 font-bold">g/</span> to view available chatrooms.</p>
|
||||
</div>
|
||||
<NetworkStats />
|
||||
<div className="mt-5 w-full flex justify-center">
|
||||
{renderGateDirectory('landing')}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<HashchainEvents />
|
||||
|
||||
@@ -2,9 +2,7 @@
|
||||
|
||||
import { useState, useMemo } from 'react';
|
||||
import { motion, AnimatePresence } from 'framer-motion';
|
||||
import { AlertTriangle, Clock, Minus, Plus, ExternalLink, Brain, Loader2, TrendingUp } from 'lucide-react';
|
||||
import ConfirmDialog from '@/components/ui/ConfirmDialog';
|
||||
import { usePredictionMarketsOptIn } from '@/hooks/usePredictionMarketsOptIn';
|
||||
import { AlertTriangle, Clock, Minus, Plus, ExternalLink, Brain, Loader2 } from 'lucide-react';
|
||||
import React, { useEffect, useRef, useCallback } from 'react';
|
||||
import WikiImage from '@/components/WikiImage';
|
||||
import { fetchWikipediaSummary } from '@/lib/wikimediaClient';
|
||||
@@ -334,9 +332,6 @@ function NewsFeedInner({ selectedEntity, regionDossier, regionDossierLoading, on
|
||||
const [aiSummaryOpen, setAiSummaryOpen] = useState(false);
|
||||
const [aiSummary, setAiSummary] = useState<any>(null);
|
||||
const [aiSummaryLoading, setAiSummaryLoading] = useState(false);
|
||||
const [pmConsentOpen, setPmConsentOpen] = useState(false);
|
||||
const { status: pmStatus, setOptIn: setPmOptIn } = usePredictionMarketsOptIn();
|
||||
const marketsCorrelationEnabled = pmStatus?.enabled ?? false;
|
||||
const itemRefs = useRef<(HTMLDivElement | null)[]>([]);
|
||||
|
||||
// Intentionally omitting map click triggers for expanding
|
||||
@@ -1362,7 +1357,7 @@ function NewsFeedInner({ selectedEntity, regionDossier, regionDossierLoading, on
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
{marketsCorrelationEnabled && item.prediction_odds && item.prediction_odds.consensus_pct != null && (
|
||||
{item.prediction_odds && item.prediction_odds.consensus_pct != null && (
|
||||
<div className="border-b border-[var(--border-primary)] pb-2">
|
||||
<span className="text-[var(--text-muted)] text-[10px] block mb-1.5">MARKET CORRELATION</span>
|
||||
<div className="p-2 bg-purple-950/30 border border-purple-500/30 rounded-sm">
|
||||
@@ -1435,37 +1430,7 @@ function NewsFeedInner({ selectedEntity, regionDossier, regionDossierLoading, on
|
||||
/* CCTV is now handled by the fullscreen OPTIC INTERCEPT modal in MaplibreViewer */
|
||||
if (selectedEntity?.type === 'cctv') return null;
|
||||
|
||||
const pmJitter = pmStatus?.jitter;
|
||||
const pmConsentMessage =
|
||||
'Enabling prediction markets lets this node contact Polymarket and Kalshi over clearnet from your server IP (not through the wormhole). ' +
|
||||
'Matching headlines may show a purple MKT strip with consensus odds. ' +
|
||||
(pmJitter
|
||||
? `Poll timing is jittered (~${pmJitter.scheduler_interval_minutes} min base + up to ${pmJitter.scheduler_jitter_seconds}s) to reduce obvious patterns. `
|
||||
: 'Poll timing is jittered to reduce obvious patterns. ') +
|
||||
'Wormhole/Tor still only covers private mesh traffic. Turn off anytime with MKT OFF.';
|
||||
|
||||
return (
|
||||
<>
|
||||
<ConfirmDialog
|
||||
open={pmConsentOpen}
|
||||
title="Enable prediction market correlation?"
|
||||
message={pmConsentMessage}
|
||||
confirmLabel="Enable MKT"
|
||||
cancelLabel="Cancel"
|
||||
danger={false}
|
||||
onCancel={() => setPmConsentOpen(false)}
|
||||
onConfirm={() => {
|
||||
void (async () => {
|
||||
try {
|
||||
await setPmOptIn(true);
|
||||
} catch (e) {
|
||||
console.warn('Prediction markets opt-in failed:', e);
|
||||
} finally {
|
||||
setPmConsentOpen(false);
|
||||
}
|
||||
})();
|
||||
}}
|
||||
/>
|
||||
<motion.div
|
||||
initial={{ y: 50, opacity: 0 }}
|
||||
animate={{ y: 0, opacity: 1 }}
|
||||
@@ -1520,38 +1485,10 @@ function NewsFeedInner({ selectedEntity, regionDossier, regionDossierLoading, on
|
||||
initial={{ height: 0, opacity: 0 }}
|
||||
animate={{ height: "auto", opacity: 1 }}
|
||||
exit={{ height: 0, opacity: 0 }}
|
||||
className="text-[10px] text-cyan-500/80 mt-1 flex items-center justify-between font-bold relative z-10 gap-2"
|
||||
className="text-[10px] text-cyan-500/80 mt-1 flex items-center justify-between font-bold relative z-10"
|
||||
>
|
||||
<div className="flex items-center gap-1.5 min-w-0">
|
||||
<span className="px-1 border border-cyan-500/30 shrink-0">SYS.STATUS: MONITORING</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
if (marketsCorrelationEnabled) {
|
||||
void setPmOptIn(false).catch((err) => {
|
||||
console.warn('Prediction markets opt-out failed:', err);
|
||||
});
|
||||
} else {
|
||||
setPmConsentOpen(true);
|
||||
}
|
||||
}}
|
||||
className={`shrink-0 flex items-center gap-1 px-1.5 py-0.5 border rounded-sm transition-colors ${
|
||||
marketsCorrelationEnabled
|
||||
? 'border-purple-500/50 bg-purple-950/40 text-purple-300'
|
||||
: 'border-cyan-800/40 bg-black/40 text-cyan-700 hover:text-purple-300 hover:border-purple-600/40'
|
||||
}`}
|
||||
title={
|
||||
marketsCorrelationEnabled
|
||||
? 'Prediction market correlation on intercept stories (clearnet Polymarket/Kalshi)'
|
||||
: 'Enable prediction market correlation on intercept stories'
|
||||
}
|
||||
>
|
||||
<TrendingUp size={10} />
|
||||
MKT {marketsCorrelationEnabled ? 'ON' : 'OFF'}
|
||||
</button>
|
||||
</div>
|
||||
<span className="flex items-center gap-1 shrink-0"><Clock size={10} /> {data?.last_updated ? formatTime(data.last_updated) : "SCANNING"}</span>
|
||||
<span className="px-1 border border-cyan-500/30">SYS.STATUS: MONITORING</span>
|
||||
<span className="flex items-center gap-1"><Clock size={10} /> {data?.last_updated ? formatTime(data.last_updated) : "SCANNING"}</span>
|
||||
</motion.div>
|
||||
)}
|
||||
</AnimatePresence>
|
||||
@@ -1904,7 +1841,7 @@ function NewsFeedInner({ selectedEntity, regionDossier, regionDossierLoading, on
|
||||
<span className="text-cyan-300 opacity-90">{item.machine_assessment}</span>
|
||||
</div>
|
||||
)}
|
||||
{marketsCorrelationEnabled && item.prediction_odds && item.prediction_odds.consensus_pct != null && (
|
||||
{item.prediction_odds && item.prediction_odds.consensus_pct != null && (
|
||||
<div className="mt-1 px-1.5 py-1 bg-purple-950/30 border border-purple-500/30 rounded-sm text-[11px] font-mono flex items-center gap-1.5">
|
||||
<span className="text-purple-400 font-bold">MKT</span>
|
||||
<span className="text-purple-300 truncate flex-1" title={item.prediction_odds.title}>{item.prediction_odds.title}</span>
|
||||
@@ -1996,7 +1933,6 @@ function NewsFeedInner({ selectedEntity, regionDossier, regionDossierLoading, on
|
||||
|
||||
|
||||
</motion.div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import React, { useState, useEffect } from 'react';
|
||||
import { motion, AnimatePresence } from 'framer-motion';
|
||||
import { X, ExternalLink, Key, Shield, Radar, Globe, Satellite, Ship, Radio, Bot, Copy, Check, Network } from 'lucide-react';
|
||||
|
||||
const CURRENT_ONBOARDING_VERSION = '0.9.81-agentic-onboarding-1';
|
||||
const CURRENT_ONBOARDING_VERSION = '0.9.8-agentic-onboarding-1';
|
||||
const STORAGE_KEY = `shadowbroker_onboarding_complete_v${CURRENT_ONBOARDING_VERSION}`;
|
||||
const LEGACY_STORAGE_KEY = 'shadowbroker_onboarding_complete';
|
||||
|
||||
@@ -129,16 +129,13 @@ const OnboardingModal = React.memo(function OnboardingModal({
|
||||
|
||||
const agentSnippet = [
|
||||
`SHADOWBROKER_URL=${agentEndpoint}`,
|
||||
agentSecret ? `SHADOWBROKER_HMAC_SECRET=${agentSecret}` : 'SHADOWBROKER_HMAC_SECRET=<generate in ShadowBroker>',
|
||||
agentSecret ? `SHADOWBROKER_KEY=${agentSecret}` : 'SHADOWBROKER_KEY=<generate in ShadowBroker>',
|
||||
`SHADOWBROKER_ACCESS=${agentTier}`,
|
||||
'',
|
||||
'# FIRST: load available tools',
|
||||
`GET ${agentEndpoint}/api/ai/tools`,
|
||||
'',
|
||||
'# Auth: SHADOWBROKER_HMAC_SECRET is not a raw API key.',
|
||||
'# Sign every direct request with X-SB-Timestamp, X-SB-Nonce, and X-SB-Signature.',
|
||||
'# Signature input: METHOD|path|timestamp|nonce|sha256(body).',
|
||||
'# Do not send the secret as X-Admin-Key, Authorization, or a query parameter.',
|
||||
'# Auth: HMAC-SHA256 signed requests.',
|
||||
'# Restricted = read-only telemetry. Full = can write when asked.',
|
||||
].join('\n');
|
||||
const remoteAgentNeedsTor = agentMode === 'remote' && !torAddress;
|
||||
|
||||
@@ -1325,7 +1325,7 @@ const SettingsPanel = React.memo(function SettingsPanel({
|
||||
className={`flex-1 px-4 py-2.5 text-sm font-mono tracking-widest font-bold transition-colors flex items-center justify-center gap-1.5 ${activeTab === 'sentinel' ? 'text-purple-400 border-b-2 border-purple-500 bg-purple-950/10' : 'text-[var(--text-muted)] hover:text-[var(--text-secondary)]'}`}
|
||||
>
|
||||
<Satellite size={10} />
|
||||
SENTINEL
|
||||
{t('settings.shodan').toUpperCase()}
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setActiveTab('sar')}
|
||||
|
||||
@@ -576,7 +576,7 @@ export default function ShodanPanel({
|
||||
fetch(`${API_BASE}/api/settings/api-keys`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ SHODAN_API_KEY: shodanApiKey.trim() }),
|
||||
body: JSON.stringify({ env_key: 'SHODAN_API_KEY', value: shodanApiKey.trim() }),
|
||||
})
|
||||
.then(() => refreshStatus())
|
||||
.finally(() => setKeySaving(false));
|
||||
@@ -599,7 +599,7 @@ export default function ShodanPanel({
|
||||
fetch(`${API_BASE}/api/settings/api-keys`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ SHODAN_API_KEY: shodanApiKey.trim() }),
|
||||
body: JSON.stringify({ env_key: 'SHODAN_API_KEY', value: shodanApiKey.trim() }),
|
||||
})
|
||||
.then(() => refreshStatus())
|
||||
.finally(() => setKeySaving(false));
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user