release: prepare v0.9.7

This commit is contained in:
BigBodyCobain
2026-05-01 22:56:50 -06:00
parent ea457f27da
commit 28b3bd5ebf
670 changed files with 187059 additions and 14005 deletions
+178
View File
@@ -0,0 +1,178 @@
"""ai_intel_store — compatibility wrapper around ai_pin_store + layer injection.
openclaw_channel.py and routers/ai_intel.py import from this module name.
All pin/layer logic lives in ai_pin_store.py; this module re-exports with the
expected function signatures and adds the layer injection helper.
"""
import logging
import time
from typing import Any
from services.ai_pin_store import (
create_pin,
create_pins_batch,
get_pins,
delete_pin,
clear_pins,
pin_count,
pins_as_geojson,
purge_expired,
# Layer CRUD
create_layer,
get_layers,
update_layer,
delete_layer,
# Feed layers
get_feed_layers,
replace_layer_pins,
)
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Re-exports expected by openclaw_channel._dispatch_command
# ---------------------------------------------------------------------------
def get_all_intel_pins() -> list[dict[str, Any]]:
"""Return all active pins (no filter, generous limit)."""
return get_pins(limit=2000)
def add_intel_pin(args: dict[str, Any]) -> dict[str, Any]:
"""Create a single pin from a command-channel args dict."""
ea = args.get("entity_attachment")
return create_pin(
lat=float(args.get("lat", 0)),
lng=float(args.get("lng", 0)),
label=str(args.get("label", ""))[:200],
category=str(args.get("category", "custom")),
layer_id=str(args.get("layer_id", "")),
color=str(args.get("color", "")),
description=str(args.get("description", "")),
source=str(args.get("source", "openclaw")),
source_url=str(args.get("source_url", "")),
confidence=float(args.get("confidence", 1.0)),
ttl_hours=float(args.get("ttl_hours", 0)),
metadata=args.get("metadata") or {},
entity_attachment=ea if isinstance(ea, dict) else None,
)
def delete_intel_pin(pin_id: str) -> bool:
"""Delete a pin by ID."""
return delete_pin(pin_id)
# Layer helpers for OpenClaw
def create_intel_layer(args: dict[str, Any]) -> dict[str, Any]:
"""Create a layer from a command-channel args dict."""
return create_layer(
name=str(args.get("name", "Untitled"))[:100],
description=str(args.get("description", ""))[:500],
source=str(args.get("source", "openclaw"))[:50],
color=str(args.get("color", "")),
feed_url=str(args.get("feed_url", "")),
feed_interval=int(args.get("feed_interval", 300)),
)
def get_intel_layers() -> list[dict[str, Any]]:
"""Return all layers with pin counts."""
return get_layers()
def update_intel_layer(layer_id: str, args: dict[str, Any]) -> dict[str, Any] | None:
"""Update a layer from a command-channel args dict."""
return update_layer(layer_id, **{
k: v for k, v in args.items()
if k in ("name", "description", "visible", "color", "feed_url", "feed_interval")
})
def delete_intel_layer(layer_id: str) -> int:
"""Delete a layer and its pins. Returns pin count removed."""
return delete_layer(layer_id)
# ---------------------------------------------------------------------------
# Layer injection — inserts agent data into native telemetry layers
# ---------------------------------------------------------------------------
# Layers that agents are allowed to inject into.
_INJECTABLE_LAYERS = frozenset({
"cctv", "ships", "sigint", "kiwisdr", "military_bases",
"datacenters", "power_plants", "satnogs_stations",
"volcanoes", "earthquakes", "news", "viirs_change_nodes",
"air_quality",
})
def inject_layer_data(
layer: str,
items: list[dict[str, Any]],
mode: str = "append",
) -> dict[str, Any]:
"""Inject agent data into a native telemetry layer."""
from services.fetchers._store import latest_data, _data_lock, bump_data_version
layer = str(layer or "").strip()
if layer not in _INJECTABLE_LAYERS:
return {"ok": False, "detail": f"layer '{layer}' not injectable"}
items = list(items or [])[:200]
if not items:
return {"ok": False, "detail": "no items provided"}
now = time.time()
tagged = []
for item in items:
if not isinstance(item, dict):
continue
entry = dict(item)
entry["_injected"] = True
entry["_source"] = "user:openclaw"
entry["_injected_at"] = now
tagged.append(entry)
with _data_lock:
existing = latest_data.get(layer)
if not isinstance(existing, list):
existing = []
if mode == "replace":
existing = [e for e in existing if not e.get("_injected")]
existing.extend(tagged)
latest_data[layer] = existing
bump_data_version()
return {
"ok": True,
"layer": layer,
"injected": len(tagged),
"mode": mode,
}
def clear_injected_data(layer: str = "") -> dict[str, Any]:
"""Remove all injected items from a layer (or all layers)."""
from services.fetchers._store import latest_data, _data_lock, bump_data_version
removed = 0
with _data_lock:
targets = [layer] if layer else list(_INJECTABLE_LAYERS)
for lyr in targets:
existing = latest_data.get(lyr)
if not isinstance(existing, list):
continue
before = len(existing)
latest_data[lyr] = [e for e in existing if not e.get("_injected")]
removed += before - len(latest_data[lyr])
if removed:
bump_data_version()
return {"ok": True, "removed": removed}
+633
View File
@@ -0,0 +1,633 @@
"""AI Intel pin storage — layered pin system with JSON file persistence.
Supports:
- Named pin layers (created by user or AI)
- Pins with optional entity attachment (track moving objects)
- Pin source tracking (user vs openclaw)
- Layer visibility toggles
- External feed URL per layer (for Phase 5)
- GeoJSON export per layer or all layers
"""
import json
import logging
import os
import threading
import time
import uuid
from datetime import datetime
from typing import Any, Optional
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Pin schema
# ---------------------------------------------------------------------------
PIN_CATEGORIES = {
"threat", "news", "geolocation", "custom", "anomaly",
"military", "maritime", "flight", "infrastructure", "weather",
"sigint", "prediction", "research",
}
PIN_COLORS = {
"threat": "#ef4444", # red
"news": "#f59e0b", # amber
"geolocation": "#8b5cf6", # violet
"custom": "#3b82f6", # blue
"anomaly": "#f97316", # orange
"military": "#dc2626", # dark red
"maritime": "#0ea5e9", # sky
"flight": "#6366f1", # indigo
"infrastructure": "#64748b", # slate
"weather": "#22d3ee", # cyan
"sigint": "#a855f7", # purple
"prediction": "#eab308", # yellow
"research": "#10b981", # emerald
}
LAYER_COLORS = [
"#3b82f6", "#ef4444", "#22d3ee", "#f59e0b", "#8b5cf6",
"#10b981", "#f97316", "#6366f1", "#ec4899", "#14b8a6",
]
# ---------------------------------------------------------------------------
# In-memory store
# ---------------------------------------------------------------------------
_layers: list[dict[str, Any]] = []
_pins: list[dict[str, Any]] = []
_lock = threading.Lock()
# Persistence file path
_PERSIST_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data")
_PERSIST_FILE = os.path.join(_PERSIST_DIR, "pin_layers.json")
_OLD_PERSIST_FILE = os.path.join(_PERSIST_DIR, "ai_pins.json")
def _ensure_persist_dir():
try:
os.makedirs(_PERSIST_DIR, exist_ok=True)
except OSError:
pass
def _save_to_disk():
"""Persist layers and pins to JSON file. Called under lock."""
try:
_ensure_persist_dir()
with open(_PERSIST_FILE, "w", encoding="utf-8") as f:
json.dump({"layers": _layers, "pins": _pins}, f, indent=2, default=str)
except (OSError, IOError) as e:
logger.warning(f"Failed to persist pin layers: {e}")
def _load_from_disk():
"""Load layers and pins from disk on startup."""
global _layers, _pins
try:
if os.path.exists(_PERSIST_FILE):
with open(_PERSIST_FILE, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, dict):
_layers = data.get("layers", [])
_pins = data.get("pins", [])
logger.info(f"Loaded {len(_layers)} layers, {len(_pins)} pins from disk")
return
# Migrate from old flat pin file
if os.path.exists(_OLD_PERSIST_FILE):
with open(_OLD_PERSIST_FILE, "r", encoding="utf-8") as f:
old_pins = json.load(f)
if isinstance(old_pins, list) and old_pins:
legacy_layer = _make_layer("Legacy", "Migrated pins", source="system")
_layers.append(legacy_layer)
for p in old_pins:
if isinstance(p, dict):
p["layer_id"] = legacy_layer["id"]
_pins.append(p)
logger.info(f"Migrated {len(_pins)} pins from ai_pins.json into Legacy layer")
_save_to_disk()
except (OSError, IOError, json.JSONDecodeError) as e:
logger.warning(f"Failed to load pin layers from disk: {e}")
def _make_layer(
name: str,
description: str = "",
source: str = "user",
color: str = "",
feed_url: str = "",
feed_interval: int = 300,
) -> dict[str, Any]:
"""Create a layer dict."""
layer_id = str(uuid.uuid4())[:12]
now = time.time()
return {
"id": layer_id,
"name": name[:100],
"description": description[:500],
"source": source[:50],
"visible": True,
"color": color or LAYER_COLORS[len(_layers) % len(LAYER_COLORS)],
"created_at": now,
"created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
"feed_url": feed_url[:1000] if feed_url else "",
"feed_interval": max(60, min(86400, feed_interval)),
"pin_count": 0,
}
# Load on import
_load_from_disk()
# One-time cleanup: remove correlation_engine auto-pins (no longer generated)
_corr_before = len(_pins)
_pins[:] = [p for p in _pins if p.get("source") != "correlation_engine"]
if len(_pins) < _corr_before:
logger.info("Cleaned up %d legacy correlation_engine pins", _corr_before - len(_pins))
_save_to_disk()
# ---------------------------------------------------------------------------
# Layer CRUD
# ---------------------------------------------------------------------------
def create_layer(
name: str,
description: str = "",
source: str = "user",
color: str = "",
feed_url: str = "",
feed_interval: int = 300,
) -> dict[str, Any]:
"""Create a new pin layer."""
with _lock:
layer = _make_layer(name, description, source, color, feed_url, feed_interval)
_layers.append(layer)
_save_to_disk()
return layer
def get_layers() -> list[dict[str, Any]]:
"""Return all layers with current pin counts."""
now = time.time()
with _lock:
result = []
for layer in _layers:
count = sum(
1 for p in _pins
if p.get("layer_id") == layer["id"]
and not (p.get("expires_at") and p["expires_at"] < now)
)
result.append({**layer, "pin_count": count})
return result
def update_layer(layer_id: str, **updates) -> Optional[dict[str, Any]]:
"""Update layer fields. Returns updated layer or None if not found."""
allowed = {"name", "description", "visible", "color", "feed_url", "feed_interval", "feed_last_fetched"}
with _lock:
for layer in _layers:
if layer["id"] == layer_id:
for k, v in updates.items():
if k in allowed and v is not None:
if k == "name":
layer[k] = str(v)[:100]
elif k == "description":
layer[k] = str(v)[:500]
elif k == "visible":
layer[k] = bool(v)
elif k == "color":
layer[k] = str(v)[:20]
elif k == "feed_url":
layer[k] = str(v)[:1000]
elif k == "feed_interval":
layer[k] = max(60, min(86400, int(v)))
elif k == "feed_last_fetched":
layer[k] = float(v)
_save_to_disk()
return dict(layer)
return None
def delete_layer(layer_id: str) -> int:
"""Delete a layer and all its pins. Returns count of pins removed."""
with _lock:
before_layers = len(_layers)
_layers[:] = [l for l in _layers if l["id"] != layer_id]
if len(_layers) == before_layers:
return 0 # not found
before_pins = len(_pins)
_pins[:] = [p for p in _pins if p.get("layer_id") != layer_id]
removed = before_pins - len(_pins)
_save_to_disk()
return removed
# ---------------------------------------------------------------------------
# Pin CRUD
# ---------------------------------------------------------------------------
def create_pin(
lat: float,
lng: float,
label: str,
category: str = "custom",
*,
layer_id: str = "",
color: str = "",
description: str = "",
source: str = "openclaw",
source_url: str = "",
confidence: float = 1.0,
ttl_hours: float = 0,
metadata: Optional[dict] = None,
entity_attachment: Optional[dict] = None,
) -> dict[str, Any]:
"""Create a single pin and return it."""
pin_id = str(uuid.uuid4())[:12]
now = time.time()
cat = category if category in PIN_CATEGORIES else "custom"
pin_color = color or PIN_COLORS.get(cat, "#3b82f6")
# Validate entity_attachment if provided
attachment = None
if entity_attachment and isinstance(entity_attachment, dict):
etype = str(entity_attachment.get("entity_type", "")).strip()
eid = str(entity_attachment.get("entity_id", "")).strip()
if etype and eid:
attachment = {
"entity_type": etype[:50],
"entity_id": eid[:100],
"entity_label": str(entity_attachment.get("entity_label", ""))[:200],
}
pin = {
"id": pin_id,
"layer_id": layer_id or "",
"lat": lat,
"lng": lng,
"label": label[:200],
"category": cat,
"color": pin_color,
"description": description[:2000],
"source": source[:100],
"source_url": source_url[:500],
"confidence": max(0.0, min(1.0, confidence)),
"created_at": now,
"created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
"expires_at": now + (ttl_hours * 3600) if ttl_hours > 0 else None,
"metadata": metadata or {},
"entity_attachment": attachment,
"comments": [],
}
with _lock:
_pins.append(pin)
_save_to_disk()
return pin
def create_pins_batch(items: list[dict], default_layer_id: str = "") -> list[dict[str, Any]]:
"""Create multiple pins at once."""
created = []
now = time.time()
with _lock:
for item in items[:200]: # max 200 per batch
pin_id = str(uuid.uuid4())[:12]
cat = item.get("category", "custom")
if cat not in PIN_CATEGORIES:
cat = "custom"
pin_color = item.get("color", "") or PIN_COLORS.get(cat, "#3b82f6")
ttl = float(item.get("ttl_hours", 0) or 0)
attachment = None
ea = item.get("entity_attachment")
if ea and isinstance(ea, dict):
etype = str(ea.get("entity_type", "")).strip()
eid = str(ea.get("entity_id", "")).strip()
if etype and eid:
attachment = {
"entity_type": etype[:50],
"entity_id": eid[:100],
"entity_label": str(ea.get("entity_label", ""))[:200],
}
pin = {
"id": pin_id,
"layer_id": item.get("layer_id", default_layer_id) or "",
"lat": float(item.get("lat", 0)),
"lng": float(item.get("lng", 0)),
"label": str(item.get("label", ""))[:200],
"category": cat,
"color": pin_color,
"description": str(item.get("description", ""))[:2000],
"source": str(item.get("source", "openclaw"))[:100],
"source_url": str(item.get("source_url", ""))[:500],
"confidence": max(0.0, min(1.0, float(item.get("confidence", 1.0)))),
"created_at": now,
"created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
"expires_at": now + (ttl * 3600) if ttl > 0 else None,
"metadata": item.get("metadata", {}),
"entity_attachment": attachment,
"comments": [],
}
_pins.append(pin)
created.append(pin)
_save_to_disk()
return created
def get_pins(
category: str = "",
source: str = "",
layer_id: str = "",
limit: int = 500,
include_expired: bool = False,
) -> list[dict[str, Any]]:
"""Get pins with optional filters."""
now = time.time()
with _lock:
results = []
for pin in _pins:
if not include_expired and pin.get("expires_at") and pin["expires_at"] < now:
continue
if category and pin.get("category") != category:
continue
if source and pin.get("source") != source:
continue
if layer_id and pin.get("layer_id") != layer_id:
continue
results.append(pin)
if len(results) >= limit:
break
return results
def get_pin(pin_id: str) -> Optional[dict[str, Any]]:
"""Return a single pin by ID (including comments), or None."""
with _lock:
for pin in _pins:
if pin.get("id") == pin_id:
# Ensure comments key exists for legacy pins
if "comments" not in pin:
pin["comments"] = []
return dict(pin)
return None
def update_pin(pin_id: str, **updates) -> Optional[dict[str, Any]]:
"""Update a pin's editable fields (label, description, category, color)."""
allowed = {"label", "description", "category", "color"}
with _lock:
for pin in _pins:
if pin.get("id") != pin_id:
continue
for k, v in updates.items():
if k not in allowed or v is None:
continue
if k == "label":
pin[k] = str(v)[:200]
elif k == "description":
pin[k] = str(v)[:2000]
elif k == "category":
cat = str(v)
if cat in PIN_CATEGORIES:
pin[k] = cat
# Refresh color if it was the category default
if not updates.get("color"):
pin["color"] = PIN_COLORS.get(cat, pin.get("color", "#3b82f6"))
elif k == "color":
pin[k] = str(v)[:20]
pin["updated_at"] = time.time()
_save_to_disk()
return dict(pin)
return None
def add_pin_comment(
pin_id: str,
text: str,
author: str = "user",
author_label: str = "",
reply_to: str = "",
) -> Optional[dict[str, Any]]:
"""Append a comment to a pin. Returns the updated pin (with all comments)."""
text = (text or "").strip()
if not text:
return None
with _lock:
for pin in _pins:
if pin.get("id") != pin_id:
continue
if "comments" not in pin or not isinstance(pin["comments"], list):
pin["comments"] = []
comment = {
"id": str(uuid.uuid4())[:12],
"text": text[:4000],
"author": (author or "user")[:50],
"author_label": (author_label or "")[:100],
"reply_to": (reply_to or "")[:12],
"created_at": time.time(),
"created_at_iso": datetime.utcnow().isoformat() + "Z",
}
pin["comments"].append(comment)
_save_to_disk()
return dict(pin)
return None
def delete_pin_comment(pin_id: str, comment_id: str) -> bool:
"""Remove a single comment from a pin."""
with _lock:
for pin in _pins:
if pin.get("id") != pin_id:
continue
comments = pin.get("comments") or []
before = len(comments)
pin["comments"] = [c for c in comments if c.get("id") != comment_id]
if len(pin["comments"]) < before:
_save_to_disk()
return True
return False
return False
def delete_pin(pin_id: str) -> bool:
"""Delete a single pin by ID."""
with _lock:
before = len(_pins)
_pins[:] = [p for p in _pins if p.get("id") != pin_id]
if len(_pins) < before:
_save_to_disk()
return True
return False
def clear_pins(category: str = "", source: str = "", layer_id: str = "") -> int:
"""Clear pins, optionally filtered. Returns count removed."""
with _lock:
before = len(_pins)
def keep(p):
if layer_id and p.get("layer_id") != layer_id:
return True # different layer, keep
if category and source:
return not (p.get("category") == category and p.get("source") == source)
if category:
return p.get("category") != category
if source:
return p.get("source") != source
if layer_id:
return p.get("layer_id") != layer_id
return False
if not category and not source and not layer_id:
_pins.clear()
else:
_pins[:] = [p for p in _pins if keep(p)]
removed = before - len(_pins)
if removed:
_save_to_disk()
return removed
def get_feed_layers() -> list[dict[str, Any]]:
"""Return layers that have a non-empty feed_url."""
with _lock:
return [dict(l) for l in _layers if l.get("feed_url")]
def replace_layer_pins(layer_id: str, new_pins: list[dict[str, Any]]) -> int:
"""Atomically replace all pins in a layer with new_pins. Returns count added."""
now = time.time()
with _lock:
# Remove old pins for this layer
_pins[:] = [p for p in _pins if p.get("layer_id") != layer_id]
# Add new pins
added = 0
for item in new_pins[:500]: # cap at 500 per feed
pin_id = str(uuid.uuid4())[:12]
cat = item.get("category", "custom")
if cat not in PIN_CATEGORIES:
cat = "custom"
pin_color = item.get("color", "") or PIN_COLORS.get(cat, "#3b82f6")
attachment = None
ea = item.get("entity_attachment")
if ea and isinstance(ea, dict):
etype = str(ea.get("entity_type", "")).strip()
eid = str(ea.get("entity_id", "")).strip()
if etype and eid:
attachment = {
"entity_type": etype[:50],
"entity_id": eid[:100],
"entity_label": str(ea.get("entity_label", ""))[:200],
}
pin = {
"id": pin_id,
"layer_id": layer_id,
"lat": float(item.get("lat", 0)),
"lng": float(item.get("lng", 0)),
"label": str(item.get("label", item.get("name", "")))[:200],
"category": cat,
"color": pin_color,
"description": str(item.get("description", ""))[:2000],
"source": str(item.get("source", "feed"))[:100],
"source_url": str(item.get("source_url", ""))[:500],
"confidence": max(0.0, min(1.0, float(item.get("confidence", 1.0)))),
"created_at": now,
"created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
"expires_at": None,
"metadata": item.get("metadata", {}),
"entity_attachment": attachment,
"comments": [],
}
_pins.append(pin)
added += 1
_save_to_disk()
return added
def purge_expired() -> int:
"""Remove expired pins. Called periodically."""
now = time.time()
with _lock:
before = len(_pins)
_pins[:] = [p for p in _pins if not (p.get("expires_at") and p["expires_at"] < now)]
removed = before - len(_pins)
if removed:
_save_to_disk()
return removed
def pin_count() -> dict[str, int]:
"""Return counts by category."""
now = time.time()
counts: dict[str, int] = {}
with _lock:
for pin in _pins:
if pin.get("expires_at") and pin["expires_at"] < now:
continue
cat = pin.get("category", "custom")
counts[cat] = counts.get(cat, 0) + 1
return counts
def pins_as_geojson(layer_id: str = "") -> dict[str, Any]:
"""Convert active pins to GeoJSON FeatureCollection for the map layer."""
now = time.time()
features = []
with _lock:
# Build set of visible layer IDs
visible_layers = {l["id"] for l in _layers if l.get("visible", True)}
for pin in _pins:
if pin.get("expires_at") and pin["expires_at"] < now:
continue
# Layer filter
pid_layer = pin.get("layer_id", "")
if layer_id and pid_layer != layer_id:
continue
# Skip pins in hidden layers
if pid_layer and pid_layer not in visible_layers:
continue
props = {
"id": pin["id"],
"layer_id": pid_layer,
"label": pin["label"],
"category": pin["category"],
"color": pin["color"],
"description": pin.get("description", ""),
"source": pin["source"],
"source_url": pin.get("source_url", ""),
"confidence": pin.get("confidence", 1.0),
"created_at": pin.get("created_at_iso", ""),
"comment_count": len(pin.get("comments") or []),
}
# Entity attachment info (frontend resolves position)
ea = pin.get("entity_attachment")
if ea:
props["entity_attachment"] = ea
features.append({
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [pin["lng"], pin["lat"]],
},
"properties": props,
})
return {
"type": "FeatureCollection",
"features": features,
}
+189
View File
@@ -0,0 +1,189 @@
"""Analysis Zone store — OpenClaw-placed map overlays with analyst notes.
These render as the dashed-border squares on the correlations layer.
Unlike automated correlations (which are recomputed every cycle), analysis
zones persist until the agent or user deletes them, or their TTL expires.
Shape matches the correlation alert schema so the frontend renders them
identically — the ``source`` field marks them as agent-placed and enables
the delete button in the popup.
"""
import json
import logging
import os
import threading
import time
import uuid
from typing import Any
logger = logging.getLogger(__name__)
_zones: list[dict[str, Any]] = []
_lock = threading.Lock()
_PERSIST_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data")
_PERSIST_FILE = os.path.join(_PERSIST_DIR, "analysis_zones.json")
ZONE_CATEGORIES = {
"contradiction", # narrative vs telemetry mismatch
"analysis", # general analyst note / assessment
"warning", # potential threat or risk area
"observation", # neutral observation worth marking
"hypothesis", # unverified theory to investigate
}
# Map categories to correlation type colors on the frontend
CATEGORY_COLORS = {
"contradiction": "amber",
"analysis": "cyan",
"warning": "red",
"observation": "blue",
"hypothesis": "purple",
}
def _ensure_dir():
try:
os.makedirs(_PERSIST_DIR, exist_ok=True)
except OSError:
pass
def _save():
"""Persist to disk. Called under lock."""
try:
_ensure_dir()
with open(_PERSIST_FILE, "w", encoding="utf-8") as f:
json.dump(_zones, f, indent=2, default=str)
except Exception as e:
logger.warning("Failed to save analysis zones: %s", e)
def _load():
"""Load from disk on startup."""
global _zones
try:
if os.path.exists(_PERSIST_FILE):
with open(_PERSIST_FILE, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, list):
_zones = data
logger.info("Loaded %d analysis zones from disk", len(_zones))
except Exception as e:
logger.warning("Failed to load analysis zones: %s", e)
# Load on import
_load()
def _expire():
"""Remove zones past their TTL. Called under lock."""
now = time.time()
before = len(_zones)
_zones[:] = [
z for z in _zones
if z.get("ttl_hours", 0) <= 0
or (now - z.get("created_at", now)) < z["ttl_hours"] * 3600
]
removed = before - len(_zones)
if removed:
logger.info("Expired %d analysis zones", removed)
def create_zone(
*,
lat: float,
lng: float,
title: str,
body: str,
category: str = "analysis",
severity: str = "medium",
cell_size_deg: float = 1.0,
ttl_hours: float = 0,
source: str = "openclaw",
drivers: list[str] | None = None,
) -> dict[str, Any]:
"""Create an analysis zone. Returns the created zone dict."""
category = category if category in ZONE_CATEGORIES else "analysis"
if severity not in ("high", "medium", "low"):
severity = "medium"
cell_size_deg = max(0.1, min(cell_size_deg, 10.0))
zone: dict[str, Any] = {
"id": str(uuid.uuid4())[:12],
"lat": lat,
"lng": lng,
"type": "analysis_zone",
"category": category,
"severity": severity,
"score": {"high": 90, "medium": 60, "low": 30}.get(severity, 60),
"title": title[:200],
"body": body[:2000],
"drivers": (drivers or [title])[:5],
"cell_size": cell_size_deg,
"source": source,
"created_at": time.time(),
"ttl_hours": ttl_hours,
}
with _lock:
_expire()
_zones.append(zone)
_save()
logger.info("Analysis zone created: %s at (%.2f, %.2f)", title[:40], lat, lng)
return zone
def list_zones() -> list[dict[str, Any]]:
"""Return all live (non-expired) zones."""
with _lock:
_expire()
return list(_zones)
def get_zone(zone_id: str) -> dict[str, Any] | None:
"""Get a single zone by ID."""
with _lock:
for z in _zones:
if z["id"] == zone_id:
return dict(z)
return None
def delete_zone(zone_id: str) -> bool:
"""Delete a zone by ID. Returns True if found and removed."""
with _lock:
before = len(_zones)
_zones[:] = [z for z in _zones if z["id"] != zone_id]
if len(_zones) < before:
_save()
return True
return False
def clear_zones(*, source: str | None = None) -> int:
"""Clear all zones, optionally filtered by source. Returns count removed."""
with _lock:
before = len(_zones)
if source:
_zones[:] = [z for z in _zones if z.get("source") != source]
else:
_zones.clear()
removed = before - len(_zones)
if removed:
_save()
return removed
def get_live_zones() -> list[dict[str, Any]]:
"""Return zones formatted for the correlation engine merge.
This is called by compute_correlations() to inject agent-placed zones
into the correlations list that the frontend renders as map squares.
"""
with _lock:
_expire()
return [dict(z) for z in _zones]
+26 -38
View File
@@ -4,11 +4,12 @@ Keys are stored in the backend .env file and loaded via python-dotenv.
"""
import os
import re
from pathlib import Path
# Path to the backend .env file
ENV_PATH = Path(__file__).parent.parent / ".env"
# Path to the example template that ships with the repo
ENV_EXAMPLE_PATH = Path(__file__).parent.parent.parent / ".env.example"
# ---------------------------------------------------------------------------
# API Registry — every external service the dashboard depends on
@@ -143,15 +144,33 @@ API_REGISTRY = [
]
def _obfuscate(value: str) -> str:
"""Show first 4 chars, mask the rest with bullets."""
if not value or len(value) <= 4:
return "••••••••"
return value[:4] + "•" * (len(value) - 4)
def get_env_path_info() -> dict:
"""Return absolute paths for the backend .env and .env.example template.
Surfaced to the frontend so the API Keys settings panel can tell users
exactly where to put their keys when in-app editing fails (admin-not-set,
file permissions, read-only filesystem, etc.).
"""
env_path = ENV_PATH.resolve()
example_path = ENV_EXAMPLE_PATH.resolve()
return {
"env_path": str(env_path),
"env_path_exists": env_path.exists(),
"env_path_writable": os.access(env_path.parent, os.W_OK)
and (not env_path.exists() or os.access(env_path, os.W_OK)),
"env_example_path": str(example_path),
"env_example_path_exists": example_path.exists(),
}
def get_api_keys():
"""Return the full API registry with obfuscated key values."""
"""Return the API registry with a binary set/unset flag per key.
Key values themselves are NEVER returned to the client — not even an
obfuscated prefix. Users edit the .env file directly; the panel uses
`is_set` to render a CONFIGURED / NOT CONFIGURED badge and the path
info from `get_env_path_info()` to tell them where to put each key.
"""
result = []
for api in API_REGISTRY:
entry = {
@@ -163,41 +182,10 @@ def get_api_keys():
"required": api["required"],
"has_key": api["env_key"] is not None,
"env_key": api["env_key"],
"value_obfuscated": None,
"is_set": False,
}
if api["env_key"]:
raw = os.environ.get(api["env_key"], "")
entry["value_obfuscated"] = _obfuscate(raw)
entry["is_set"] = bool(raw)
result.append(entry)
return result
def update_api_key(env_key: str, new_value: str) -> bool:
"""Update a single key in the .env file and in the current process env."""
valid_keys = {api["env_key"] for api in API_REGISTRY if api.get("env_key")}
if env_key not in valid_keys:
return False
if not isinstance(new_value, str):
return False
if "\n" in new_value or "\r" in new_value:
return False
if not ENV_PATH.exists():
ENV_PATH.write_text("", encoding="utf-8")
# Update os.environ immediately
os.environ[env_key] = new_value
# Update the .env file on disk
content = ENV_PATH.read_text(encoding="utf-8")
pattern = re.compile(rf"^{re.escape(env_key)}=.*$", re.MULTILINE)
if pattern.search(content):
content = pattern.sub(f"{env_key}={new_value}", content)
else:
content = content.rstrip("\n") + f"\n{env_key}={new_value}\n"
ENV_PATH.write_text(content, encoding="utf-8")
return True
+99
View File
@@ -818,6 +818,105 @@ out body;
return cameras
# ---------------------------------------------------------------------------
# ALPR / Surveillance Camera Locations (OSM Overpass)
# ---------------------------------------------------------------------------
# Queries OpenStreetMap for ALPR/LPR tagged surveillance cameras.
# These cameras rarely have public media URLs — this ingestor captures
# their LOCATIONS for situational awareness (density heatmap, blind-spot
# analysis). No plate-read data is fetched — only publicly-mapped positions.
class OSMALPRCameraIngestor(BaseCCTVIngestor):
"""ALPR / license-plate reader camera locations from OpenStreetMap.
Searches for nodes tagged with surveillance:type=ALPR or
man_made=surveillance + camera:type values indicating plate readers.
Only geolocations are ingested — no live feeds or detection data.
"""
URL = "https://overpass-api.de/api/interpreter"
QUERY = """
[out:json][timeout:45];
(
node["surveillance:type"="ALPR"];
node["surveillance:type"="alpr"];
node["surveillance:type"="LPR"];
node["surveillance:type"="lpr"];
node["man_made"="surveillance"]["camera:type"="ALPR"];
node["man_made"="surveillance"]["camera:type"="alpr"];
node["man_made"="surveillance"]["camera:type"="LPR"];
node["man_made"="surveillance"]["camera:type"="lpr"];
node["man_made"="surveillance"]["description"~"[Ll]icense [Pp]late"];
node["man_made"="surveillance"]["description"~"ALPR"];
node["man_made"="surveillance"]["description"~"Flock"];
);
out body;
""".strip()
def fetch_data(self) -> List[Dict[str, Any]]:
query = quote(self.QUERY, safe="")
resp = fetch_with_curl(
f"{self.URL}?data={query}",
timeout=50,
headers={"Accept": "application/json"},
)
if not resp or resp.status_code != 200:
logger.warning(
"OSM ALPR camera fetch failed: HTTP %s",
resp.status_code if resp else "no response",
)
return []
data = resp.json()
cameras = []
for item in data.get("elements", []) if isinstance(data, dict) else []:
lat = item.get("lat")
lon = item.get("lon")
if lat is None or lon is None:
continue
try:
lat, lon = float(lat), float(lon)
except (ValueError, TypeError):
continue
tags = item.get("tags", {}) if isinstance(item.get("tags"), dict) else {}
# Extract what we can from tags
operator = (
tags.get("operator")
or tags.get("brand")
or tags.get("network")
or "Unknown"
)
description = (
tags.get("description")
or tags.get("name")
or tags.get("surveillance:type", "ALPR")
)
direction = (
tags.get("camera:direction")
or tags.get("direction")
or tags.get("surveillance:direction")
or "Unknown"
)
# ALPR cameras typically have no public media URL — use a
# placeholder so the pin renders but no proxy attempt is made.
cameras.append(
{
"id": f"ALPR-{item.get('id')}",
"source_agency": str(operator)[:60],
"lat": lat,
"lon": lon,
"direction_facing": f"ALPR: {str(description)[:100]} ({str(direction)[:30]})",
"media_url": "",
"media_type": "none",
"refresh_rate_seconds": 0,
}
)
logger.info("OSM ALPR ingestor found %d cameras", len(cameras))
return cameras
# ---------------------------------------------------------------------------
# DGT Spain — National Road Cameras
+260 -7
View File
@@ -10,6 +10,10 @@ class Settings(BaseSettings):
ALLOW_INSECURE_ADMIN: bool = False
PUBLIC_API_KEY: str = ""
# OpenClaw agent connectivity
OPENCLAW_HMAC_SECRET: str = "" # HMAC shared secret for direct mode (auto-generated if empty)
OPENCLAW_ACCESS_TIER: str = "restricted" # "full" or "restricted"
# Data sources
AIS_API_KEY: str = ""
OPENSKY_CLIENT_ID: str = ""
@@ -27,7 +31,8 @@ class Settings(BaseSettings):
MESH_RNS_ENABLED: bool = False
MESH_ARTI_ENABLED: bool = False
MESH_ARTI_SOCKS_PORT: int = 9050
MESH_RELAY_PEERS: str = "http://cipher0.shadowbroker.info:8000"
MESH_RELAY_PEERS: str = ""
MESH_DEFAULT_SYNC_PEERS: str = "https://node.shadowbroker.info"
MESH_BOOTSTRAP_DISABLED: bool = False
MESH_BOOTSTRAP_MANIFEST_PATH: str = "data/bootstrap_peers.json"
MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY: str = ""
@@ -37,7 +42,7 @@ class Settings(BaseSettings):
MESH_RELAY_PUSH_TIMEOUT_S: int = 10
MESH_RELAY_MAX_FAILURES: int = 3
MESH_RELAY_FAILURE_COOLDOWN_S: int = 120
MESH_PEER_PUSH_SECRET: str = "Mv63UvLfwqOEVWeRBXjA8MtFl2nEkkhUlLYVHiX1Zzo"
MESH_PEER_PUSH_SECRET: str = ""
MESH_RNS_APP_NAME: str = "shadowbroker"
MESH_RNS_ASPECT: str = "infonet"
MESH_RNS_IDENTITY_PATH: str = ""
@@ -60,7 +65,8 @@ class Settings(BaseSettings):
# Keep a low background cadence on private RNS links so quiet nodes are less
# trivially fingerprintable by silence alone. Set to 0 to disable explicitly.
MESH_RNS_COVER_INTERVAL_S: int = 30
MESH_RNS_COVER_SIZE: int = 64
MESH_RNS_COVER_SIZE: int = 512
MESH_DM_MAILBOX_TTL_S: int = 900
MESH_RNS_IBF_WINDOW: int = 256
MESH_RNS_IBF_TABLE_SIZE: int = 64
MESH_RNS_IBF_MINHASH_SIZE: int = 16
@@ -75,44 +81,221 @@ class Settings(BaseSettings):
MESH_RNS_IBF_FAIL_THRESHOLD: int = 3
MESH_RNS_IBF_COOLDOWN_S: int = 120
MESH_VERIFY_INTERVAL_S: int = 600
MESH_VERIFY_SIGNATURES: bool = True
# MESH_VERIFY_SIGNATURES is intentionally removed — the audit loop in main.py
# always calls validate_chain_incremental(verify_signatures=True). Any value
# set in the environment is ignored.
MESH_DM_SECURE_MODE: bool = True
MESH_DM_TOKEN_PEPPER: str = ""
MESH_DM_ALLOW_LEGACY_GET: bool = False
MESH_ALLOW_LEGACY_DM1_UNTIL: str = ""
MESH_ALLOW_LEGACY_DM_GET_UNTIL: str = ""
MESH_ALLOW_LEGACY_DM_SIGNATURE_COMPAT_UNTIL: str = ""
MESH_DM_PERSIST_SPOOL: bool = False
MESH_DM_RELAY_FILE_PATH: str = ""
MESH_DM_RELAY_AUTO_RELOAD: bool = False
MESH_DM_REQUIRE_SENDER_SEAL_SHARED: bool = True
MESH_DM_NONCE_TTL_S: int = 300
MESH_DM_NONCE_CACHE_MAX: int = 4096
MESH_DM_NONCE_PER_AGENT_MAX: int = 256
MESH_DM_REQUEST_MAX_AGE_S: int = 300
MESH_DM_REQUEST_MAILBOX_LIMIT: int = 12
MESH_DM_SHARED_MAILBOX_LIMIT: int = 48
MESH_DM_SELF_MAILBOX_LIMIT: int = 12
MESH_BLOCK_LEGACY_AGENT_ID_LOOKUP: bool = True
MESH_ALLOW_COMPAT_DM_INVITE_IMPORT: bool = False
MESH_ALLOW_COMPAT_DM_INVITE_IMPORT_UNTIL: str = ""
MESH_ALLOW_LEGACY_NODE_ID_COMPAT_UNTIL: str = ""
# Rotate voter-blinding salts on a rolling cadence so new reputation
# events do not reuse one forever-stable blinded identity.
MESH_VOTER_BLIND_SALT_ROTATE_DAYS: int = 30
# Keep historical salts long enough to cover live vote records, so
# duplicate-vote detection and wallet-cost accounting survive rotation.
MESH_VOTER_BLIND_SALT_GRACE_DAYS: int = 30
MESH_DM_MAX_MSG_BYTES: int = 8192
MESH_DM_ALLOW_SENDER_SEAL: bool = False
# TTL for DH key and prekey bundle registrations — stale entries are pruned.
MESH_DM_KEY_TTL_DAYS: int = 30
# TTL for invite-scoped prekey lookup aliases; shorter windows reduce
# long-lived relay linkage between opaque lookup handles and agent IDs.
MESH_DM_PREKEY_LOOKUP_ALIAS_TTL_DAYS: int = 14
# TTL for relay witness history; keep continuity metadata bounded instead
# of relying on a hidden hardcoded retention window.
MESH_DM_WITNESS_TTL_DAYS: int = 14
# TTL for mailbox binding metadata — shorter = smaller metadata footprint on disk.
MESH_DM_BINDING_TTL_DAYS: int = 7
MESH_DM_BINDING_TTL_DAYS: int = 3
# When False, mailbox bindings are memory-only (agents re-register on restart).
MESH_DM_METADATA_PERSIST: bool = True
# Enable explicitly only if restart continuity is worth persisting DM graph metadata.
MESH_DM_METADATA_PERSIST: bool = False
# Second explicit opt-in for at-rest DM metadata persistence. This keeps a
# single boolean flip from silently writing mailbox graph metadata to disk.
MESH_DM_METADATA_PERSIST_ACKNOWLEDGE: bool = False
# Optional import path for externally managed root witness material packages.
# Relative paths resolve from the backend directory.
MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH: str = ""
# Optional URI for externally managed root witness material packages.
# Supports file:// and http(s):// sources; when set it overrides the local path.
MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_URI: str = ""
# Maximum acceptable age for externally sourced root witness packages.
# Strong DM trust fails closed when the imported package exported_at is older than this.
MESH_DM_ROOT_EXTERNAL_WITNESS_MAX_AGE_S: int = 3600
# Warning threshold for externally sourced root witness packages.
# When current external witness material reaches this age, operator health degrades to warning
# before the strong path eventually fails closed at MAX_AGE.
MESH_DM_ROOT_EXTERNAL_WITNESS_WARN_AGE_S: int = 2700
# Optional export path for the append-only stable-root transparency ledger.
# Relative paths resolve from the backend directory.
MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH: str = ""
# Optional URI used to read back and verify published transparency ledgers.
# Supports file:// and http(s):// sources.
MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI: str = ""
# Maximum acceptable age for externally read transparency ledgers.
# Strong DM trust fails closed when exported_at is older than this.
MESH_DM_ROOT_TRANSPARENCY_LEDGER_MAX_AGE_S: int = 3600
# Warning threshold for externally read transparency ledgers.
# When current external transparency readback reaches this age, operator health degrades to warning
# before the strong path eventually fails closed at MAX_AGE.
MESH_DM_ROOT_TRANSPARENCY_LEDGER_WARN_AGE_S: int = 2700
MESH_SCOPED_TOKENS: str = ""
# Deprecated legacy env vars kept for backward config compatibility only.
# Ordinary shipped gate flows keep MLS decrypt local; backend decrypt is
# reserved for explicit recovery reads.
MESH_GATE_BACKEND_DECRYPT_COMPAT: bool = False
MESH_GATE_BACKEND_DECRYPT_COMPAT_ACKNOWLEDGE: bool = False
MESH_BACKEND_GATE_DECRYPT_COMPAT: bool = False
# Deprecated legacy env vars kept for backward config compatibility only.
# Ordinary shipped gate flows keep compose/post local and submit encrypted
# payloads to the backend for sign/post only.
MESH_GATE_BACKEND_PLAINTEXT_COMPAT: bool = False
MESH_GATE_BACKEND_PLAINTEXT_COMPAT_ACKNOWLEDGE: bool = False
MESH_BACKEND_GATE_PLAINTEXT_COMPAT: bool = False
# Runtime gate for recovery envelopes. When off, per-gate
# envelope_recovery / envelope_always policies fail closed to
# envelope_disabled. Default True so the Reddit-like durable history
# model works out of the box: any member with the gate_secret can
# decrypt every envelope encrypted from the moment they had that key.
# Set MESH_GATE_RECOVERY_ENVELOPE_ENABLE=false to revert to MLS-only
# forward-secret behavior (your own history becomes unreadable after
# the sending ratchet advances).
MESH_GATE_RECOVERY_ENVELOPE_ENABLE: bool = True
MESH_GATE_RECOVERY_ENVELOPE_ENABLE_ACKNOWLEDGE: bool = True
# Durable gate plaintext retention is disabled by default. Enable only
# when the operator explicitly accepts the at-rest privacy tradeoff.
MESH_GATE_PLAINTEXT_PERSIST: bool = False
MESH_GATE_PLAINTEXT_PERSIST_ACKNOWLEDGE: bool = False
MESH_GATE_SESSION_ROTATE_MSGS: int = 50
MESH_GATE_SESSION_ROTATE_S: int = 3600
MESH_GATE_LEGACY_ENVELOPE_FALLBACK_MAX_DAYS: int = 30
# Add a randomized grace window before anonymous gate-session auto-rotation
# so threshold-triggered identity swaps are less trivially correlated.
MESH_GATE_SESSION_ROTATE_JITTER_S: int = 180
# Gate persona (named identity) rotation thresholds. Rotating the signing
# key limits the linkability window. Zero = disabled.
MESH_GATE_PERSONA_ROTATE_MSGS: int = 200
MESH_GATE_PERSONA_ROTATE_S: int = 604800 # 7 days
MESH_GATE_PERSONA_ROTATE_JITTER_S: int = 600
# Feature-flagged session stream for multiplexed gate room updates.
# Disabled by default so rollout stays explicit while stream-first rooms bake.
MESH_GATE_SESSION_STREAM_ENABLED: bool = False
MESH_GATE_SESSION_STREAM_HEARTBEAT_S: int = 20
MESH_GATE_SESSION_STREAM_BATCH_MS: int = 1500
MESH_GATE_SESSION_STREAM_MAX_GATES: int = 16
# Private gate APIs expose a backward-jittered timestamp view so observers
# cannot trivially align exact send times from response metadata alone.
MESH_GATE_TIMESTAMP_JITTER_S: int = 60
# Ban/kick gate-secret rotation is on by default (hardening Rec #10): the
# invariant has baked and a ban that does not rotate is effectively a
# display-only removal. Set MESH_GATE_BAN_KICK_ROTATION_ENABLE=false to
# revert to observe-only during incident triage.
MESH_GATE_BAN_KICK_ROTATION_ENABLE: bool = True
MESH_BLOCK_LEGACY_NODE_ID_COMPAT: bool = True
MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK: bool = False
MESH_ACK_RAW_FALLBACK_AT_OWN_RISK: bool = False
MESH_SECURE_STORAGE_SECRET: str = ""
MESH_PRIVATE_LOG_TTL_S: int = 900
# Sprint 1 rollout: restored DM boot probes stay disabled by default until
# the architect reviews false positives from the observe-only path.
MESH_DM_RESTORED_SESSION_BOOT_PROBE_ENABLE: bool = False
# Queued DM release requires explicit per-item approval before any weaker
# relay fallback. Silent fallback is not a safe private-mode default.
MESH_PRIVATE_RELEASE_APPROVAL_ENABLE: bool = True
# Expiry for user-approved scoped private relay fallback policy. The policy
# is still bounded by hidden-transport checks before it can auto-release.
MESH_PRIVATE_RELAY_POLICY_TTL_S: int = 3600
# Background privacy prewarm prepares keys/aliases/transport readiness
# before send-time. Anonymous mode uses a cadence gate so user clicks do
# not directly create hidden-transport activity.
MESH_PRIVACY_PREWARM_ENABLE: bool = True
MESH_PRIVACY_PREWARM_INTERVAL_S: int = 300
MESH_PRIVACY_PREWARM_ANON_CADENCE_S: int = 300
# Sprint 4 rollout: authenticated RNS cover markers remain disabled until
# the observer-equivalence and receive-path DoS tests are green.
MESH_RNS_COVER_AUTH_MARKER_ENABLE: bool = False
# Signed-write revocation lookups use a short local TTL; stale entries force
# a local rebuild before honor. Offline/local-refresh failures remain
# observe-only until the later enforcement sprint.
MESH_SIGNED_REVOCATION_CACHE_TTL_S: int = 300
MESH_SIGNED_REVOCATION_CACHE_ENFORCE: bool = True
MESH_SIGNED_WRITE_CONTEXT_REQUIRED: bool = True
# Sprint 5 rollout: when enabled, root witness finality requires
# independent quorum for threshold>1 witnessed roots before they count as
# verified first-contact provenance.
WORMHOLE_ROOT_WITNESS_FINALITY_ENFORCE: bool = False
# Optional JSON artifact generated by CI/release workflow for the Sprint 8
# release gate. Relative paths resolve from the backend directory.
# dev = permissive local/dev behavior; testnet-private = strict private
# defaults; release-candidate = no compatibility/debug escape hatches.
MESH_RELEASE_PROFILE: str = "dev"
MESH_RELEASE_ATTESTATION_PATH: str = ""
# Operator release attestation for the Sprint 8 release gate. This does
# not change runtime behavior; it only records that the DM relay security
# suite was run and passed for the release candidate.
MESH_RELEASE_DM_RELAY_SECURITY_SUITE_GREEN: bool = False
PRIVACY_CORE_MIN_VERSION: str = "0.1.0"
PRIVACY_CORE_ALLOWED_SHA256: str = ""
PRIVACY_CORE_DEV_OVERRIDE: bool = False
# Sprint 4 rollout: fail fast when the loaded privacy-core artifact is
# missing required FFI symbols expected by the current Python bridge.
PRIVACY_CORE_EXPORT_SET_AUDIT_ENABLE: bool = True
# Clearnet fallback policy for private-tier messages.
# "block" (default) = refuse to send private messages over clearnet.
# "allow" = fall back to clearnet when Tor/RNS is unavailable (weaker privacy).
MESH_PRIVATE_CLEARNET_FALLBACK: str = "block"
# Second explicit opt-in for private-tier clearnet fallback. Without this
# acknowledgement, "allow" remains requested but not effective.
MESH_PRIVATE_CLEARNET_FALLBACK_ACKNOWLEDGE: bool = False
# Meshtastic MQTT bridge — disabled by default to avoid hammering the
# public broker. Users opt in explicitly.
MESH_MQTT_ENABLED: bool = False
# Meshtastic MQTT broker credentials (defaults match public firmware).
MESH_MQTT_BROKER: str = "mqtt.meshtastic.org"
MESH_MQTT_PORT: int = 1883
MESH_MQTT_USER: str = "meshdev"
MESH_MQTT_PASS: str = "large4cats"
# Hex-encoded PSK — empty string means use the default LongFast key.
# Must decode to exactly 16 or 32 bytes when set.
MESH_MQTT_PSK: str = ""
# Optional operator-provided Meshtastic node ID (e.g. "!abcd1234") included
# in the User-Agent when fetching from meshtastic.liamcottle.net so the
# service operator can identify per-install traffic instead of a generic
# "ShadowBroker" aggregate.
MESHTASTIC_OPERATOR_CALLSIGN: str = ""
# SAR (Synthetic Aperture Radar) data layer
# Mode A — free catalog metadata, no account, default-on
MESH_SAR_CATALOG_ENABLED: bool = True
# Mode B — free pre-processed anomalies (OPERA / EGMS / GFM / EMS / UNOSAT)
# Two-step opt-in: must be "allow" AND _ACKNOWLEDGE must be true
MESH_SAR_PRODUCTS_FETCH: str = "block"
MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE: bool = False
# NASA Earthdata Login (free) — required for OPERA products
MESH_SAR_EARTHDATA_USER: str = ""
MESH_SAR_EARTHDATA_TOKEN: str = ""
# Copernicus Data Space (free) — required for EGMS / EMS products
MESH_SAR_COPERNICUS_USER: str = ""
MESH_SAR_COPERNICUS_TOKEN: str = ""
# Whether OpenClaw agents may read/act on the SAR layer
MESH_SAR_OPENCLAW_ENABLED: bool = True
# Require private-tier transport before signing/broadcasting SAR anomalies
MESH_SAR_REQUIRE_PRIVATE_TIER: bool = True
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
@@ -120,3 +303,73 @@ class Settings(BaseSettings):
@lru_cache
def get_settings() -> Settings:
return Settings()
def private_clearnet_fallback_requested(settings: Settings | None = None) -> str:
snapshot = settings or get_settings()
policy = str(getattr(snapshot, "MESH_PRIVATE_CLEARNET_FALLBACK", "block") or "block").strip().lower()
return "allow" if policy == "allow" else "block"
def private_clearnet_fallback_effective(settings: Settings | None = None) -> str:
snapshot = settings or get_settings()
requested = private_clearnet_fallback_requested(snapshot)
acknowledged = bool(getattr(snapshot, "MESH_PRIVATE_CLEARNET_FALLBACK_ACKNOWLEDGE", False))
if requested == "allow" and acknowledged:
return "allow"
return "block"
def backend_gate_decrypt_compat_effective(settings: Settings | None = None) -> bool:
snapshot = settings or get_settings()
return bool(
getattr(snapshot, "MESH_BACKEND_GATE_DECRYPT_COMPAT", False)
or getattr(snapshot, "MESH_GATE_BACKEND_DECRYPT_COMPAT", False)
)
def backend_gate_plaintext_compat_effective(settings: Settings | None = None) -> bool:
snapshot = settings or get_settings()
return bool(
getattr(snapshot, "MESH_BACKEND_GATE_PLAINTEXT_COMPAT", False)
or getattr(snapshot, "MESH_GATE_BACKEND_PLAINTEXT_COMPAT", False)
)
def gate_recovery_envelope_effective(settings: Settings | None = None) -> bool:
snapshot = settings or get_settings()
requested = bool(getattr(snapshot, "MESH_GATE_RECOVERY_ENVELOPE_ENABLE", False))
acknowledged = bool(getattr(snapshot, "MESH_GATE_RECOVERY_ENVELOPE_ENABLE_ACKNOWLEDGE", False))
return requested and acknowledged
def gate_plaintext_persist_effective(settings: Settings | None = None) -> bool:
snapshot = settings or get_settings()
requested = bool(getattr(snapshot, "MESH_GATE_PLAINTEXT_PERSIST", False))
acknowledged = bool(getattr(snapshot, "MESH_GATE_PLAINTEXT_PERSIST_ACKNOWLEDGE", False))
return requested and acknowledged
def gate_ban_kick_rotation_enabled(settings: Settings | None = None) -> bool:
snapshot = settings or get_settings()
return bool(getattr(snapshot, "MESH_GATE_BAN_KICK_ROTATION_ENABLE", False))
def dm_restored_session_boot_probe_enabled(settings: Settings | None = None) -> bool:
snapshot = settings or get_settings()
return bool(getattr(snapshot, "MESH_DM_RESTORED_SESSION_BOOT_PROBE_ENABLE", False))
def signed_revocation_cache_ttl_s(settings: Settings | None = None) -> int:
snapshot = settings or get_settings()
return max(0, int(getattr(snapshot, "MESH_SIGNED_REVOCATION_CACHE_TTL_S", 300) or 0))
def signed_revocation_cache_enforce(settings: Settings | None = None) -> bool:
snapshot = settings or get_settings()
return bool(getattr(snapshot, "MESH_SIGNED_REVOCATION_CACHE_ENFORCE", False))
def wormhole_root_witness_finality_enforce(settings: Settings | None = None) -> bool:
snapshot = settings or get_settings()
return bool(getattr(snapshot, "WORMHOLE_ROOT_WITNESS_FINALITY_ENFORCE", False))
+443 -2
View File
@@ -8,9 +8,13 @@ Correlation types:
- RF Anomaly: GPS jamming + internet outage (both required)
- Military Buildup: Military flights + naval vessels + GDELT conflict events
- Infrastructure Cascade: Internet outage + KiwiSDR offline in same zone
- Possible Contradiction: Official denial/statement + infrastructure disruption
in same region — hypothesis generator, NOT verdict
"""
import logging
import math
import re
from collections import defaultdict
logger = logging.getLogger(__name__)
@@ -306,6 +310,427 @@ def _detect_infra_cascades(data: dict) -> list[dict]:
return alerts
# ---------------------------------------------------------------------------
# Possible Contradiction: official denial/statement + infra disruption
#
# This is a HYPOTHESIS GENERATOR, not a verdict engine. It says "LOOK HERE"
# when an official statement (denial, clarification, refusal) co-locates with
# infrastructure disruption (internet outage, sigint change). The human or
# higher-order reasoning decides what actually happened.
#
# Context ratings:
# STRONG — denial + outage + prediction market movement in same region
# MODERATE — denial + outage (no market signal)
# WEAK — denial + minor outage or distant co-location
# DETECTION_GAP — denial found but NO telemetry to verify (equally valuable)
# ---------------------------------------------------------------------------
# Denial / official-statement patterns in headlines and URL slugs
_DENIAL_PATTERNS = [
re.compile(p, re.IGNORECASE) for p in [
r"\bden(?:y|ies|ied|ial)\b",
r"\brefut(?:e[ds]?|ing)\b",
r"\breject(?:s|ed|ing)?\b",
r"\bclarif(?:y|ies|ied|ication)\b",
r"\bdismiss(?:es|ed|ing)?\b",
r"\bno\s+attack\b",
r"\bdid\s+not\s+(?:attack|strike|bomb|target|order|invade|kill)\b",
r"\bnever\s+(?:attack|strike|bomb|target|order|invade|happen)\b",
r"\bfalse\s+(?:report|claim|allegation|rumor|narrative)\b",
r"\bmisinformation\b",
r"\bdisinformation\b",
r"\bpropaganda\b",
r"\b(?:army|military|government|ministry|official)\s+(?:says|clarifies|denies|refutes)\b",
r"\brumor[s]?\b.*\buntrue\b",
r"\bcategorically\b",
r"\bbaseless\b",
]
]
# Broader cell radius for sparse telemetry regions (Africa, Central Asia, etc.)
# These regions have fewer IODA/RIPE probes so outage data is sparser
_SPARSE_REGIONS_LAT_RANGES = [
(-35, 37), # Africa roughly
(25, 50), # Central Asia band (when lng 40-90)
]
def _is_sparse_region(lat: float, lng: float) -> bool:
"""Check if coordinates fall in a region with sparse telemetry coverage."""
# Africa
if -35 <= lat <= 37 and -20 <= lng <= 55:
return True
# Central Asia
if 25 <= lat <= 50 and 40 <= lng <= 90:
return True
# South America interior
if -55 <= lat <= 12 and -80 <= lng <= -35:
return True
return False
def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
"""Great-circle distance in km."""
R = 6371.0
dlat = math.radians(lat2 - lat1)
dlon = math.radians(lon2 - lon1)
a = (math.sin(dlat / 2) ** 2 +
math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) *
math.sin(dlon / 2) ** 2)
return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
def _matches_denial(text: str) -> bool:
"""Check if text matches any denial/official-statement pattern."""
return any(p.search(text) for p in _DENIAL_PATTERNS)
def _detect_contradictions(data: dict) -> list[dict]:
"""Detect possible contradictions between official statements and telemetry.
Scans GDELT headlines for denial language, then checks whether internet
outages or other infrastructure disruptions exist in the same geographic
region. Scores confidence and lists alternative explanations.
"""
gdelt = data.get("gdelt") or []
internet_outages = data.get("internet_outages") or []
news = data.get("news") or []
prediction_markets = data.get("prediction_markets") or []
# ── Step 1: Find GDELT events with denial/official-statement language ──
denial_events: list[dict] = []
# GDELT comes as GeoJSON features
gdelt_features = gdelt
if isinstance(gdelt, dict):
gdelt_features = gdelt.get("features", [])
for feature in gdelt_features:
# Handle both GeoJSON features and flat dicts
if "properties" in feature and "geometry" in feature:
props = feature.get("properties", {})
geom = feature.get("geometry", {})
coords = geom.get("coordinates", [])
if len(coords) >= 2:
lng, lat = float(coords[0]), float(coords[1])
else:
continue
headlines = props.get("_headlines_list", [])
urls = props.get("_urls_list", [])
name = props.get("name", "")
count = props.get("count", 1)
else:
lat = feature.get("lat") or feature.get("actionGeo_Lat")
lng = feature.get("lng") or feature.get("lon") or feature.get("actionGeo_Long")
if lat is None or lng is None:
continue
lat, lng = float(lat), float(lng)
headlines = [feature.get("title", "")]
urls = [feature.get("sourceurl", "")]
name = feature.get("name", "")
count = 1
# Check all headlines + URL slugs for denial patterns
all_text = " ".join(str(h) for h in headlines if h)
all_text += " " + " ".join(str(u) for u in urls if u)
if _matches_denial(all_text):
denial_events.append({
"lat": lat,
"lng": lng,
"headlines": [h for h in headlines if h][:5],
"urls": [u for u in urls if u][:3],
"location_name": name,
"event_count": count,
})
# Also scan news articles for denial language
for article in news:
title = str(article.get("title", "") or "")
desc = str(article.get("description", "") or article.get("summary", "") or "")
if not _matches_denial(title + " " + desc):
continue
# News articles often lack coordinates — try to match to GDELT locations
# For now, only include if we have coordinates
lat = article.get("lat") or article.get("latitude")
lng = article.get("lng") or article.get("lon") or article.get("longitude")
if lat is not None and lng is not None:
denial_events.append({
"lat": float(lat),
"lng": float(lng),
"headlines": [title],
"urls": [article.get("url") or article.get("link") or ""],
"location_name": "",
"event_count": 1,
})
if not denial_events:
return []
# ── Step 2: Cross-reference with internet outages ──
alerts: list[dict] = []
for denial in denial_events:
d_lat, d_lng = denial["lat"], denial["lng"]
sparse = _is_sparse_region(d_lat, d_lng)
search_radius_km = 1500.0 if sparse else 500.0
# Find nearby outages
nearby_outages: list[dict] = []
for outage in internet_outages:
o_lat = outage.get("lat") or outage.get("latitude")
o_lng = outage.get("lng") or outage.get("lon") or outage.get("longitude")
if o_lat is None or o_lng is None:
continue
try:
dist = _haversine_km(d_lat, d_lng, float(o_lat), float(o_lng))
except (ValueError, TypeError):
continue
if dist <= search_radius_km:
nearby_outages.append({
"region": outage.get("region_name") or outage.get("country_name", ""),
"severity": _outage_pct(outage),
"distance_km": round(dist, 0),
"level": outage.get("level", ""),
})
# ── Step 3: Check prediction markets for related movements ──
denial_text = " ".join(denial["headlines"]).lower()
related_markets: list[dict] = []
for market in prediction_markets:
m_title = str(market.get("title", "") or market.get("question", "") or "").lower()
# Look for keyword overlap between denial and market
denial_words = set(re.findall(r"[a-z]{4,}", denial_text))
market_words = set(re.findall(r"[a-z]{4,}", m_title))
overlap = denial_words & market_words - {"that", "this", "with", "from", "have", "been", "were", "will", "says", "said"}
if len(overlap) >= 2:
prob = market.get("probability") or market.get("lastTradePrice") or market.get("yes_price")
if prob is not None:
related_markets.append({
"title": market.get("title") or market.get("question"),
"probability": float(prob),
})
# ── Step 4: Score confidence and assign context rating ──
indicators = 1 # denial itself
drivers: list[str] = []
# Primary driver: the denial headline
headline_display = denial["headlines"][0] if denial["headlines"] else "Official statement"
if len(headline_display) > 80:
headline_display = headline_display[:77] + "..."
drivers.append(f'"{headline_display}"')
# Outage co-location
has_outage = False
if nearby_outages:
best_outage = max(nearby_outages, key=lambda o: o["severity"])
if best_outage["severity"] >= 10:
indicators += 1
has_outage = True
drivers.append(
f"Internet outage {best_outage['severity']:.0f}% "
f"({best_outage['region']}, {best_outage['distance_km']:.0f}km away)"
)
elif best_outage["severity"] > 0:
indicators += 0.5 # minor outage, partial indicator
has_outage = True
drivers.append(
f"Minor outage ({best_outage['region']}, "
f"{best_outage['distance_km']:.0f}km away)"
)
# Prediction market signal
has_market = False
if related_markets:
indicators += 1
has_market = True
top_market = related_markets[0]
drivers.append(
f"Market: \"{top_market['title'][:50]}\" "
f"at {top_market['probability']:.0%}"
)
# Multiple denial sources strengthen the signal
if denial["event_count"] > 1:
indicators += 0.5
drivers.append(f"{denial['event_count']} sources reporting")
# Context rating
if has_outage and has_market:
context = "STRONG"
elif has_outage:
context = "MODERATE"
elif has_market:
context = "WEAK" # market signal without infra disruption
else:
context = "DETECTION_GAP"
# Severity mapping
if context == "STRONG":
sev = "high"
elif context == "MODERATE":
sev = "medium"
else:
sev = "low"
# Alternative explanations (always present — this is a hypothesis generator)
alternatives: list[str] = []
if has_outage:
alternatives.append("Routine infrastructure maintenance or cable damage")
alternatives.append("Weather-related outage coinciding with news cycle")
if not has_outage and context == "DETECTION_GAP":
alternatives.append("Statement may be truthful — no contradicting telemetry found")
alternatives.append("Telemetry coverage gap in this region")
alternatives.append("Denial may be responding to social media rumors, not real events")
lat_c, lng_c = _cell_center(_cell_key(d_lat, d_lng))
alerts.append({
"lat": lat_c,
"lng": lng_c,
"type": "contradiction",
"severity": sev,
"score": _severity_score(sev),
"drivers": drivers[:4],
"cell_size": _CELL_SIZE,
"context": context,
"alternatives": alternatives[:3],
"location_name": denial.get("location_name", ""),
"headlines": denial["headlines"][:3],
"related_markets": related_markets[:3],
"nearby_outages": nearby_outages[:5],
})
# Deduplicate: keep highest-scored alert per cell
seen_cells: dict[str, dict] = {}
for alert in alerts:
key = _cell_key(alert["lat"], alert["lng"])
if key not in seen_cells or alert["score"] > seen_cells[key]["score"]:
seen_cells[key] = alert
result = list(seen_cells.values())
if result:
by_context = defaultdict(int)
for a in result:
by_context[a["context"]] += 1
logger.info(
"Contradictions: %d possible (%s)",
len(result),
", ".join(f"{v} {k}" for k, v in sorted(by_context.items())),
)
return result
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Correlation → Pin bridge
# ---------------------------------------------------------------------------
# Types and their pin categories
_CORR_PIN_CATEGORIES = {
"rf_anomaly": "anomaly",
"military_buildup": "military",
"infra_cascade": "infrastructure",
"contradiction": "research",
}
# Deduplicate: don't re-pin the same cell within this window (seconds).
_CORR_PIN_DEDUP_WINDOW = 600 # 10 minutes
_recent_corr_pins: dict[str, float] = {}
def _auto_pin_correlations(alerts: list[dict]) -> int:
"""Create AI Intel pins for high-severity correlation alerts.
Only pins alerts with severity >= medium. Uses cell-key dedup so the
same grid cell doesn't get re-pinned every fetch cycle.
Returns the number of pins created this cycle.
"""
import time as _time
now = _time.time()
# Evict stale dedup entries
expired = [k for k, ts in _recent_corr_pins.items() if now - ts > _CORR_PIN_DEDUP_WINDOW]
for k in expired:
_recent_corr_pins.pop(k, None)
created = 0
for alert in alerts:
sev = alert.get("severity", "low")
if sev == "low":
continue # Don't pin low-severity noise
lat = alert.get("lat")
lng = alert.get("lng")
if lat is None or lng is None:
continue
# Dedup key: type + cell
dedup_key = f"{alert['type']}:{_cell_key(lat, lng)}"
if dedup_key in _recent_corr_pins:
continue
category = _CORR_PIN_CATEGORIES.get(alert["type"], "anomaly")
drivers = alert.get("drivers", [])
atype = alert["type"]
if atype == "contradiction":
ctx = alert.get("context", "")
label = f"[{ctx}] Possible Contradiction"
parts = list(drivers)
if alert.get("alternatives"):
parts.append("Alternatives: " + "; ".join(alert["alternatives"][:2]))
description = " | ".join(parts) if parts else "Narrative contradiction detected"
else:
label = f"[{sev.upper()}] {atype.replace('_', ' ').title()}"
description = "; ".join(drivers) if drivers else "Multi-layer correlation alert"
try:
from services.ai_pin_store import create_pin
meta = {
"correlation_type": atype,
"severity": sev,
"drivers": drivers,
"cell_size": alert.get("cell_size", _CELL_SIZE),
}
# Add contradiction-specific metadata
if atype == "contradiction":
meta["context_rating"] = alert.get("context", "")
meta["alternatives"] = alert.get("alternatives", [])
meta["headlines"] = alert.get("headlines", [])
meta["location_name"] = alert.get("location_name", "")
if alert.get("related_markets"):
meta["related_markets"] = alert["related_markets"]
create_pin(
lat=lat,
lng=lng,
label=label,
category=category,
description=description,
source="correlation_engine",
confidence=alert.get("score", 60) / 100.0,
ttl_hours=2.0, # Auto-expire correlation pins after 2 hours
metadata=meta,
)
_recent_corr_pins[dedup_key] = now
created += 1
except Exception as exc:
logger.warning("Failed to auto-pin correlation: %s", exc)
if created:
logger.info("Correlation engine auto-pinned %d alerts", created)
return created
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
@@ -330,13 +755,29 @@ def compute_correlations(data: dict) -> list[dict]:
except Exception as e:
logger.error("Correlation engine infra cascade error: %s", e)
# Contradiction detection removed from automated engine — too many false
# positives from regex headline matching. Contradiction/analysis alerts are
# now placed by OpenClaw agents via place_analysis_zone, which lets an LLM
# reason about the evidence rather than pattern-matching keywords.
try:
from services.analysis_zone_store import get_live_zones
alerts.extend(get_live_zones())
except Exception as e:
logger.error("Analysis zone merge error: %s", e)
rf = sum(1 for a in alerts if a["type"] == "rf_anomaly")
mil = sum(1 for a in alerts if a["type"] == "military_buildup")
infra = sum(1 for a in alerts if a["type"] == "infra_cascade")
contra = sum(1 for a in alerts if a["type"] == "contradiction")
if alerts:
logger.info(
"Correlations: %d alerts (%d rf, %d mil, %d infra)",
len(alerts), rf, mil, infra,
"Correlations: %d alerts (%d rf, %d mil, %d infra, %d contra)",
len(alerts), rf, mil, infra, contra,
)
# Correlation alerts are returned in the correlations data feed only.
# They are NOT auto-pinned to AI Intel — that layer is reserved for
# user / OpenClaw pins. Correlations are visualised via the dedicated
# correlations overlay on the map.
return alerts
+322 -9
View File
@@ -16,9 +16,12 @@ Heavy logic has been extracted into services/fetchers/:
import logging
import concurrent.futures
import json
import math
import os
import time
from datetime import datetime, timedelta
from pathlib import Path
from dotenv import load_dotenv
load_dotenv()
@@ -56,6 +59,7 @@ from services.fetchers.earth_observation import ( # noqa: F401
fetch_air_quality,
fetch_volcanoes,
fetch_viirs_change_nodes,
fetch_uap_sightings,
)
from services.fetchers.infrastructure import ( # noqa: F401
fetch_internet_outages,
@@ -90,10 +94,35 @@ from services.fetchers.meshtastic_map import (
load_meshtastic_cache_if_available,
) # noqa: F401
from services.fetchers.fimi import fetch_fimi # noqa: F401
from services.fetchers.crowdthreat import fetch_crowdthreat # noqa: F401
from services.fetchers.wastewater import fetch_wastewater # noqa: F401
from services.fetchers.sar_catalog import fetch_sar_catalog # noqa: F401
from services.fetchers.sar_products import fetch_sar_products # noqa: F401
from services.ais_stream import prune_stale_vessels # noqa: F401
logger = logging.getLogger(__name__)
_SLOW_FETCH_S = float(os.environ.get("FETCH_SLOW_THRESHOLD_S", "5"))
# Hard wall-clock limit per individual fetch task. A task that exceeds this
# is treated as a failure so it cannot block an entire fetch tier indefinitely.
_TASK_HARD_TIMEOUT_S = float(os.environ.get("FETCH_TASK_TIMEOUT_S", "120"))
_FAST_STARTUP_CACHE_MAX_AGE_S = float(os.environ.get("FAST_STARTUP_CACHE_MAX_AGE_S", "300"))
_FAST_STARTUP_CACHE_PATH = Path(__file__).resolve().parents[1] / "data" / "fast_startup_cache.json"
_FAST_STARTUP_CACHE_KEYS = (
"commercial_flights",
"military_flights",
"private_flights",
"private_jets",
"tracked_flights",
"ships",
"uavs",
"gps_jamming",
"satellites",
"satellite_source",
"satellite_analysis",
"sigint",
"sigint_totals",
"trains",
)
# Shared thread pool — reused across all fetch cycles instead of creating/destroying per tick
_SHARED_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
@@ -101,6 +130,80 @@ _SHARED_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
)
def _cache_json_safe(value):
if isinstance(value, float):
return value if math.isfinite(value) else None
if isinstance(value, dict):
return {str(k): _cache_json_safe(v) for k, v in value.items()}
if isinstance(value, (list, tuple)):
return [_cache_json_safe(v) for v in value]
return value
def _load_fast_startup_cache_if_available() -> bool:
"""Seed moving layers from a recent disk cache while live fetches warm up."""
if _FAST_STARTUP_CACHE_MAX_AGE_S <= 0 or not _FAST_STARTUP_CACHE_PATH.exists():
return False
try:
with _FAST_STARTUP_CACHE_PATH.open("r", encoding="utf-8") as fh:
payload = json.load(fh)
cached_at = float(payload.get("cached_at") or 0)
age_s = time.time() - cached_at
if cached_at <= 0 or age_s > _FAST_STARTUP_CACHE_MAX_AGE_S:
logger.info("Skipping stale fast startup cache (age %.1fs)", age_s)
return False
layers = payload.get("layers") or {}
freshness = payload.get("freshness") or {}
loaded: list[str] = []
with _data_lock:
for key in _FAST_STARTUP_CACHE_KEYS:
if key in layers:
latest_data[key] = layers[key]
loaded.append(key)
for key, ts in freshness.items():
source_timestamps[str(key)] = ts
if payload.get("last_updated"):
latest_data["last_updated"] = payload.get("last_updated")
if not loaded:
return False
from services.fetchers._store import bump_data_version
bump_data_version()
logger.info(
"Loaded fast startup cache for %d layers (age %.1fs) so the map can paint before remote feeds finish",
len(loaded),
age_s,
)
return True
except Exception as e:
logger.warning("Fast startup cache load failed (non-fatal): %s", e)
return False
def _save_fast_startup_cache() -> None:
"""Persist recent moving layers for the next cold start."""
try:
with _data_lock:
payload = {
"cached_at": time.time(),
"last_updated": latest_data.get("last_updated"),
"layers": {key: latest_data.get(key) for key in _FAST_STARTUP_CACHE_KEYS},
"freshness": {
key: source_timestamps.get(key)
for key in _FAST_STARTUP_CACHE_KEYS
if source_timestamps.get(key)
},
}
safe_payload = _cache_json_safe(payload)
_FAST_STARTUP_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
tmp_path = _FAST_STARTUP_CACHE_PATH.with_suffix(".tmp")
with tmp_path.open("w", encoding="utf-8") as fh:
json.dump(safe_payload, fh, separators=(",", ":"))
tmp_path.replace(_FAST_STARTUP_CACHE_PATH)
except Exception as e:
logger.debug("Fast startup cache save skipped: %s", e)
# ---------------------------------------------------------------------------
# Scheduler & Orchestration
# ---------------------------------------------------------------------------
@@ -109,10 +212,12 @@ def _run_tasks(label: str, funcs: list):
if not funcs:
return
futures = {_SHARED_EXECUTOR.submit(func): (func.__name__, time.perf_counter()) for func in funcs}
for future in concurrent.futures.as_completed(futures):
name, start = futures[future]
# Iterate directly so future.result(timeout=...) is the blocking call.
# as_completed() blocks inside __next__() waiting for completion — the timeout
# on result() would never be reached for a hanging task under that pattern.
for future, (name, start) in futures.items():
try:
future.result()
future.result(timeout=_TASK_HARD_TIMEOUT_S)
duration = time.perf_counter() - start
from services.fetch_health import record_success
@@ -164,6 +269,7 @@ def update_fast_data():
latest_data["last_updated"] = datetime.utcnow().isoformat()
from services.fetchers._store import bump_data_version
bump_data_version()
_save_fast_startup_cache()
logger.info("Fast-tier update complete.")
@@ -219,6 +325,7 @@ def update_all_data(*, startup_mode: bool = False):
logger.info("Full data update starting (parallel)...")
# Preload Meshtastic map cache immediately (instant, from disk)
load_meshtastic_cache_if_available()
_load_fast_startup_cache_if_available()
with _data_lock:
meshtastic_seeded = bool(latest_data.get("meshtastic_map_nodes"))
futures = {
@@ -231,6 +338,11 @@ def update_all_data(*, startup_mode: bool = False):
_SHARED_EXECUTOR.submit(fetch_fimi): ("fetch_fimi", time.perf_counter()),
_SHARED_EXECUTOR.submit(fetch_gdelt): ("fetch_gdelt", time.perf_counter()),
_SHARED_EXECUTOR.submit(update_liveuamap): ("update_liveuamap", time.perf_counter()),
_SHARED_EXECUTOR.submit(fetch_uap_sightings): ("fetch_uap_sightings", time.perf_counter()),
_SHARED_EXECUTOR.submit(fetch_wastewater): ("fetch_wastewater", time.perf_counter()),
_SHARED_EXECUTOR.submit(fetch_crowdthreat): ("fetch_crowdthreat", time.perf_counter()),
_SHARED_EXECUTOR.submit(fetch_sar_catalog): ("fetch_sar_catalog", time.perf_counter()),
_SHARED_EXECUTOR.submit(fetch_sar_products): ("fetch_sar_products", time.perf_counter()),
}
if not startup_mode or not meshtastic_seeded:
futures[_SHARED_EXECUTOR.submit(fetch_meshtastic_nodes)] = (
@@ -241,10 +353,9 @@ def update_all_data(*, startup_mode: bool = False):
logger.info(
"Startup preload: Meshtastic cache already loaded, deferring remote map refresh to scheduled cadence"
)
for future in concurrent.futures.as_completed(futures):
name, start = futures[future]
for future, (name, start) in futures.items():
try:
future.result()
future.result(timeout=_TASK_HARD_TIMEOUT_S)
duration = time.perf_counter() - start
from services.fetch_health import record_success
@@ -257,6 +368,42 @@ def update_all_data(*, startup_mode: bool = False):
record_failure(name, error=e, duration_s=duration)
logger.exception(f"full-refresh task failed: {name}")
# Run CCTV ingest immediately so cameras are available on first request
# (the scheduled job also runs every 10 min for ongoing refresh).
if startup_mode:
try:
from services.cctv_pipeline import (
TFLJamCamIngestor, LTASingaporeIngestor, AustinTXIngestor,
NYCDOTIngestor, CaltransIngestor, ColoradoDOTIngestor,
WSDOTIngestor, GeorgiaDOTIngestor, IllinoisDOTIngestor,
MichiganDOTIngestor, WindyWebcamsIngestor, DGTNationalIngestor,
MadridCityIngestor, OSMTrafficCameraIngestor, get_all_cameras,
)
from services.cctv_pipeline import OSMALPRCameraIngestor
_startup_ingestors = [
TFLJamCamIngestor(), LTASingaporeIngestor(), AustinTXIngestor(),
NYCDOTIngestor(), CaltransIngestor(), ColoradoDOTIngestor(),
WSDOTIngestor(), GeorgiaDOTIngestor(), IllinoisDOTIngestor(),
MichiganDOTIngestor(), WindyWebcamsIngestor(), DGTNationalIngestor(),
MadridCityIngestor(), OSMTrafficCameraIngestor(),
OSMALPRCameraIngestor(),
]
logger.info("Running CCTV ingest at startup (%d ingestors)...", len(_startup_ingestors))
ingest_futures = {
_SHARED_EXECUTOR.submit(ing.ingest): ing.__class__.__name__
for ing in _startup_ingestors
}
for fut in concurrent.futures.as_completed(ingest_futures, timeout=90):
name = ingest_futures[fut]
try:
fut.result()
except Exception as e:
logger.warning("CCTV startup ingest %s failed: %s", name, e)
fetch_cctv()
logger.info("CCTV startup ingest complete — %d cameras in DB", len(get_all_cameras()))
except Exception as e:
logger.warning("CCTV startup ingest failed (non-fatal): %s", e)
logger.info("Full data update complete.")
@@ -406,6 +553,38 @@ def start_scheduler():
misfire_grace_time=60,
)
# Route database — bulk refresh from vrs-standing-data.adsb.lol every 5
# days. Replaces the legacy /api/0/routeset POST (blocked under our UA,
# and broken upstream). Airline schedules change on a quarterly cycle,
# so 5 days is well within the staleness budget; new flight numbers
# added within the window simply fall back to UNKNOWN until refresh.
from services.fetchers.route_database import refresh_route_database
_scheduler.add_job(
lambda: _run_task_with_health(refresh_route_database, "refresh_route_database"),
"interval",
days=5,
id="route_database",
max_instances=1,
misfire_grace_time=3600,
)
# Aircraft metadata database — bulk refresh from OpenSky's public S3
# bucket every 5 days. Provides hex24 -> ICAO type so OpenSky-sourced
# flights (which lack 't' in /states/all) get aircraft category and
# fuel/CO2 emissions populated. Snapshots are monthly; 5 days catches
# newer drops without hammering the bucket.
from services.fetchers.aircraft_database import refresh_aircraft_database
_scheduler.add_job(
lambda: _run_task_with_health(refresh_aircraft_database, "refresh_aircraft_database"),
"interval",
days=5,
id="aircraft_database",
max_instances=1,
misfire_grace_time=3600,
)
# GDELT — every 30 minutes (downloads 32 ZIP files per call, avoid rate limits)
_scheduler.add_job(
lambda: _run_task_with_health(fetch_gdelt, "fetch_gdelt"),
@@ -510,14 +689,21 @@ def start_scheduler():
misfire_grace_time=120,
)
# Meshtastic map API — every 4 hours, fetch global node positions
# Meshtastic map API — once per day with a per-install random offset to
# avoid thundering the one-person hobby service at the top of the hour.
# The fetcher also short-circuits on a fresh on-disk cache, so the
# practical network cadence is closer to "once per day per install".
import random as _random_jitter
_meshtastic_jitter_minutes = _random_jitter.randint(0, 180)
_scheduler.add_job(
lambda: _run_task_with_health(fetch_meshtastic_nodes, "fetch_meshtastic_nodes"),
"interval",
hours=4,
hours=24,
minutes=_meshtastic_jitter_minutes,
id="meshtastic_map",
max_instances=1,
misfire_grace_time=600,
misfire_grace_time=3600,
)
# Oracle resolution sweep — every hour, check if any markets with predictions have concluded
@@ -550,9 +736,136 @@ def start_scheduler():
misfire_grace_time=600,
)
# UAP sightings (NUFORC) — daily at 12:00 UTC
_scheduler.add_job(
lambda: _run_task_with_health(
lambda: fetch_uap_sightings(force_refresh=True),
"fetch_uap_sightings",
),
"cron",
hour=12,
minute=0,
id="uap_sightings_daily",
max_instances=1,
misfire_grace_time=3600,
)
# WastewaterSCAN pathogen surveillance — daily at 12:00 UTC (samples update ~daily)
_scheduler.add_job(
lambda: _run_task_with_health(fetch_wastewater, "fetch_wastewater"),
"cron",
hour=12,
minute=0,
id="wastewater_daily",
max_instances=1,
misfire_grace_time=3600,
)
# CrowdThreat verified threat intelligence — daily at 12:00 UTC
_scheduler.add_job(
lambda: _run_task_with_health(fetch_crowdthreat, "fetch_crowdthreat"),
"cron",
hour=12,
minute=0,
id="crowdthreat_daily",
max_instances=1,
misfire_grace_time=3600,
)
# SAR catalog (Mode A) — every hour, free metadata from ASF Search.
# No account, no downloads, no DSP. Pure scene catalog + coverage hints.
_scheduler.add_job(
lambda: _run_task_with_health(fetch_sar_catalog, "fetch_sar_catalog"),
"interval",
hours=1,
id="sar_catalog",
max_instances=1,
misfire_grace_time=600,
next_run_time=datetime.utcnow() + timedelta(minutes=3),
)
# SAR products (Mode B) — every 30 minutes, opt-in only.
# Pre-processed deformation/flood/damage anomalies from OPERA, EGMS, GFM,
# EMS, UNOSAT. Disabled until both MESH_SAR_PRODUCTS_FETCH=allow and
# MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE=true are set.
_scheduler.add_job(
lambda: _run_task_with_health(fetch_sar_products, "fetch_sar_products"),
"interval",
minutes=30,
id="sar_products",
max_instances=1,
misfire_grace_time=600,
next_run_time=datetime.utcnow() + timedelta(minutes=5),
)
# ── Time Machine auto-snapshots ─────────────────────────────────────
# Compressed snapshots taken on two profiles (high_freq + standard).
# Intervals are read from _timemachine_config at each invocation so
# config changes via the API take effect without restarting.
def _auto_snapshot_high_freq():
"""Auto-snapshot fast-moving layers (flights, ships, satellites)."""
try:
from services.node_settings import read_node_settings
if not read_node_settings().get("timemachine_enabled", False):
return # Time Machine is off — skip
from routers.ai_intel import _timemachine_config, _take_snapshot_internal
cfg = _timemachine_config["profiles"]["high_freq"]
if cfg["interval_minutes"] <= 0:
return # disabled
layers = cfg["layers"]
result = _take_snapshot_internal(layers=layers, profile="auto_high_freq", compress=True)
logger.info("Time Machine auto-snapshot (high_freq): %s — %s layers",
result.get("snapshot_id"), len(result.get("layers", [])))
except Exception as e:
logger.warning("Time Machine auto-snapshot (high_freq) failed: %s", e)
def _auto_snapshot_standard():
"""Auto-snapshot contextual layers (news, earthquakes, weather, etc.)."""
try:
from services.node_settings import read_node_settings
if not read_node_settings().get("timemachine_enabled", False):
return # Time Machine is off — skip
from routers.ai_intel import _timemachine_config, _take_snapshot_internal
cfg = _timemachine_config["profiles"]["standard"]
if cfg["interval_minutes"] <= 0:
return # disabled
layers = cfg["layers"]
result = _take_snapshot_internal(layers=layers, profile="auto_standard", compress=True)
logger.info("Time Machine auto-snapshot (standard): %s — %s layers",
result.get("snapshot_id"), len(result.get("layers", [])))
except Exception as e:
logger.warning("Time Machine auto-snapshot (standard) failed: %s", e)
_scheduler.add_job(
_auto_snapshot_high_freq,
"interval",
minutes=15,
id="timemachine_high_freq",
max_instances=1,
misfire_grace_time=60,
next_run_time=datetime.utcnow() + timedelta(minutes=2), # first snapshot 2m after startup
)
_scheduler.add_job(
_auto_snapshot_standard,
"interval",
minutes=120,
id="timemachine_standard",
max_instances=1,
misfire_grace_time=300,
next_run_time=datetime.utcnow() + timedelta(minutes=5), # first snapshot 5m after startup
)
_scheduler.start()
logger.info("Scheduler started.")
# Start the feed ingester daemon (refreshes feed-backed pin layers)
try:
from services.feed_ingester import start_feed_ingester
start_feed_ingester()
except Exception as e:
logger.warning("Failed to start feed ingester: %s", e)
def stop_scheduler():
if _scheduler:
File diff suppressed because it is too large Load Diff
+238
View File
@@ -0,0 +1,238 @@
"""Feed Ingester — background daemon that refreshes feed-backed pin layers.
Layers with a non-empty `feed_url` are polled at their `feed_interval`
(seconds, minimum 60). The feed is expected to return either:
1. GeoJSON FeatureCollection — features are converted to pins
2. JSON array of pin objects — used directly
Each refresh atomically replaces the layer's pins with the new data.
"""
import logging
import threading
import time
from typing import Any
import requests
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# State
# ---------------------------------------------------------------------------
_running = False
_thread: threading.Thread | None = None
_CHECK_INTERVAL = 30 # seconds between scanning for layers that need refresh
_last_fetched: dict[str, float] = {} # layer_id → last fetch timestamp
_FETCH_TIMEOUT = 20 # seconds
# ---------------------------------------------------------------------------
# GeoJSON → pin conversion
# ---------------------------------------------------------------------------
def _geojson_features_to_pins(features: list[dict]) -> list[dict[str, Any]]:
"""Convert GeoJSON Feature objects to pin dicts."""
pins: list[dict[str, Any]] = []
for feat in features:
if not isinstance(feat, dict):
continue
geom = feat.get("geometry") or {}
props = feat.get("properties") or {}
# Extract coordinates
coords = geom.get("coordinates")
if geom.get("type") != "Point" or not coords or len(coords) < 2:
continue
lng, lat = float(coords[0]), float(coords[1])
if not (-90 <= lat <= 90 and -180 <= lng <= 180):
continue
pin: dict[str, Any] = {
"lat": lat,
"lng": lng,
"label": str(props.get("label", props.get("name", props.get("title", ""))))[:200],
"category": str(props.get("category", "custom"))[:50],
"color": str(props.get("color", ""))[:20],
"description": str(props.get("description", props.get("summary", "")))[:2000],
"source": "feed",
"source_url": str(props.get("source_url", props.get("url", props.get("link", ""))))[:500],
"confidence": float(props.get("confidence", 1.0)),
}
# Entity attachment if present
entity_type = props.get("entity_type", "")
entity_id = props.get("entity_id", "")
if entity_type and entity_id:
pin["entity_attachment"] = {
"entity_type": str(entity_type),
"entity_id": str(entity_id),
"entity_label": str(props.get("entity_label", "")),
}
pins.append(pin)
return pins
def _parse_feed_response(data: Any) -> list[dict[str, Any]]:
"""Parse a feed response into a list of pin dicts."""
if isinstance(data, dict):
# GeoJSON FeatureCollection
if data.get("type") == "FeatureCollection" and isinstance(data.get("features"), list):
return _geojson_features_to_pins(data["features"])
# Single Feature
if data.get("type") == "Feature":
return _geojson_features_to_pins([data])
# Wrapped response like {"ok": true, "data": [...]}
inner = data.get("data") or data.get("results") or data.get("pins") or data.get("items")
if isinstance(inner, list):
return _normalize_pin_list(inner)
if isinstance(data, list):
# Check if first item looks like a GeoJSON Feature
if data and isinstance(data[0], dict) and data[0].get("type") == "Feature":
return _geojson_features_to_pins(data)
return _normalize_pin_list(data)
return []
def _normalize_pin_list(items: list) -> list[dict[str, Any]]:
"""Normalize a list of raw pin objects, ensuring lat/lng are present."""
pins: list[dict[str, Any]] = []
for item in items:
if not isinstance(item, dict):
continue
lat = item.get("lat") or item.get("latitude")
lng = item.get("lng") or item.get("lon") or item.get("longitude")
if lat is None or lng is None:
continue
try:
lat, lng = float(lat), float(lng)
except (ValueError, TypeError):
continue
if not (-90 <= lat <= 90 and -180 <= lng <= 180):
continue
pin: dict[str, Any] = {
"lat": lat,
"lng": lng,
"label": str(item.get("label", item.get("name", item.get("title", ""))))[:200],
"category": str(item.get("category", "custom"))[:50],
"color": str(item.get("color", ""))[:20],
"description": str(item.get("description", item.get("summary", "")))[:2000],
"source": "feed",
"source_url": str(item.get("source_url", item.get("url", item.get("link", ""))))[:500],
"confidence": float(item.get("confidence", 1.0)),
}
entity_type = item.get("entity_type", "")
entity_id = item.get("entity_id", "")
if entity_type and entity_id:
pin["entity_attachment"] = {
"entity_type": str(entity_type),
"entity_id": str(entity_id),
"entity_label": str(item.get("entity_label", "")),
}
pins.append(pin)
return pins
# ---------------------------------------------------------------------------
# Fetch a single layer
# ---------------------------------------------------------------------------
def _fetch_layer_feed(layer: dict[str, Any]) -> None:
"""Fetch a feed URL and replace the layer's pins."""
layer_id = layer["id"]
feed_url = layer["feed_url"]
layer_name = layer.get("name", layer_id)
try:
resp = requests.get(
feed_url,
timeout=_FETCH_TIMEOUT,
headers={"User-Agent": "ShadowBroker-FeedIngester/1.0"},
)
resp.raise_for_status()
data = resp.json()
except requests.RequestException as e:
logger.warning("Feed fetch failed for layer '%s' (%s): %s", layer_name, feed_url, e)
return
except (ValueError, TypeError) as e:
logger.warning("Feed parse failed for layer '%s' (%s): %s", layer_name, feed_url, e)
return
pins = _parse_feed_response(data)
from services.ai_pin_store import replace_layer_pins, update_layer
count = replace_layer_pins(layer_id, pins)
# Update layer metadata with last_fetched timestamp
update_layer(layer_id, feed_last_fetched=time.time())
_last_fetched[layer_id] = time.time()
logger.info("Feed refresh for layer '%s': %d pins from %s", layer_name, count, feed_url)
# ---------------------------------------------------------------------------
# Main loop
# ---------------------------------------------------------------------------
def _ingest_loop() -> None:
"""Daemon loop: scan for feed layers and refresh those that are due."""
while _running:
try:
from services.ai_pin_store import get_feed_layers
layers = get_feed_layers()
now = time.time()
for layer in layers:
layer_id = layer["id"]
interval = max(60, layer.get("feed_interval", 300))
last = _last_fetched.get(layer_id, 0)
if now - last >= interval:
try:
_fetch_layer_feed(layer)
except Exception as e:
logger.warning("Feed ingestion error for layer %s: %s",
layer.get("name", layer_id), e)
except Exception as e:
logger.error("Feed ingester loop error: %s", e)
# Sleep in short increments so we can stop cleanly
for _ in range(int(_CHECK_INTERVAL)):
if not _running:
break
time.sleep(1)
# ---------------------------------------------------------------------------
# Start / stop
# ---------------------------------------------------------------------------
def start_feed_ingester() -> None:
"""Start the feed ingester daemon thread."""
global _running, _thread
if _thread and _thread.is_alive():
return
_running = True
_thread = threading.Thread(target=_ingest_loop, daemon=True, name="feed-ingester")
_thread.start()
logger.info("Feed ingester daemon started (check interval=%ds)", _CHECK_INTERVAL)
def stop_feed_ingester() -> None:
"""Stop the feed ingester daemon."""
global _running
_running = False
+96 -11
View File
@@ -4,6 +4,7 @@ Central location for latest_data, source_timestamps, and the data lock.
Every fetcher imports from here instead of maintaining its own copy.
"""
import copy
import threading
import logging
from datetime import datetime
@@ -42,6 +43,7 @@ class DashboardData(TypedDict, total=False):
gps_jamming: List[Dict[str, Any]]
satellites: List[Dict[str, Any]]
satellite_source: str
satellite_analysis: Dict[str, Any]
prediction_markets: List[Dict[str, Any]]
sigint: List[Dict[str, Any]]
sigint_totals: Dict[str, Any]
@@ -61,6 +63,12 @@ class DashboardData(TypedDict, total=False):
fimi: Dict[str, Any]
psk_reporter: List[Dict[str, Any]]
correlations: List[Dict[str, Any]]
uap_sightings: List[Dict[str, Any]]
wastewater: List[Dict[str, Any]]
crowdthreat: List[Dict[str, Any]]
sar_scenes: List[Dict[str, Any]]
sar_anomalies: List[Dict[str, Any]]
sar_aoi_coverage: List[Dict[str, Any]]
# In-memory store
@@ -105,6 +113,12 @@ latest_data: DashboardData = {
"fimi": {},
"psk_reporter": [],
"correlations": [],
"uap_sightings": [],
"wastewater": [],
"crowdthreat": [],
"sar_scenes": [],
"sar_anomalies": [],
"sar_aoi_coverage": [],
}
# Per-source freshness timestamps
@@ -117,9 +131,21 @@ source_freshness: dict[str, dict] = {}
def _mark_fresh(*keys):
"""Record the current UTC time for one or more data source keys."""
now = datetime.utcnow().isoformat()
global _data_version
changed: list[tuple[str, int, int]] = [] # (layer, version, count)
with _data_lock:
for k in keys:
source_timestamps[k] = now
_layer_versions[k] = _layer_versions.get(k, 0) + 1
# Grab entity count while we hold the lock (cheap len())
val = latest_data.get(k)
count = len(val) if isinstance(val, list) else (1 if val is not None else 0)
changed.append((k, _layer_versions[k], count))
# Publish partial fetch progress immediately so the frontend can
# observe newly available data without waiting for the entire tier.
_data_version += 1
# Notify SSE listeners outside the lock to avoid deadlocks
_notify_layer_change(changed)
# Thread lock for safe reads/writes to latest_data
@@ -129,16 +155,73 @@ _data_lock = threading.Lock()
# Used for cheap ETag generation instead of MD5-hashing the full response.
_data_version: int = 0
# Per-layer version counters — incremented only when that specific layer
# refreshes. Used by get_layer_slice for per-layer incremental updates
# and by the SSE stream to push targeted layer_changed notifications.
_layer_versions: dict[str, int] = {}
# ---------------------------------------------------------------------------
# Layer-change notification callbacks (thread → async SSE bridge)
# ---------------------------------------------------------------------------
_layer_change_callbacks: list = []
_layer_change_callbacks_lock = threading.Lock()
def register_layer_change_callback(callback) -> None:
"""Register a callback invoked on every _mark_fresh().
Signature: callback(layer: str, version: int, count: int)
Called from fetcher threads — must be thread-safe.
"""
with _layer_change_callbacks_lock:
_layer_change_callbacks.append(callback)
def unregister_layer_change_callback(callback) -> None:
"""Remove a previously registered callback."""
with _layer_change_callbacks_lock:
try:
_layer_change_callbacks.remove(callback)
except ValueError:
pass
def _notify_layer_change(changed: list[tuple[str, int, int]]) -> None:
"""Fire all registered callbacks for each changed layer."""
with _layer_change_callbacks_lock:
cbs = list(_layer_change_callbacks)
for cb in cbs:
for layer, version, count in changed:
try:
cb(layer, version, count)
except Exception:
pass
def get_layer_versions() -> dict[str, int]:
"""Return a snapshot of all per-layer version counters."""
with _data_lock:
return dict(_layer_versions)
def get_layer_version(layer: str) -> int:
"""Return the version counter for a single layer (0 if never refreshed)."""
with _data_lock:
return _layer_versions.get(layer, 0)
def bump_data_version() -> None:
"""Increment the data version counter after a fetch cycle completes."""
global _data_version
_data_version += 1
with _data_lock:
_data_version += 1
def get_data_version() -> int:
"""Return the current data version (for ETag generation)."""
return _data_version
with _data_lock:
return _data_version
_active_layers_version: int = 0
@@ -156,21 +239,17 @@ def get_active_layers_version() -> int:
def get_latest_data_subset(*keys: str) -> DashboardData:
"""Return a shallow snapshot of only the requested top-level keys.
"""Return a deep snapshot of only the requested top-level keys.
This avoids cloning the entire dashboard store for endpoints that only need
a small tier-specific subset.
a small tier-specific subset. Deep copy ensures callers cannot mutate
nested structures (e.g. individual flight dicts) and affect the live store.
"""
with _data_lock:
snap: DashboardData = {}
for key in keys:
value = latest_data.get(key)
if isinstance(value, list):
snap[key] = list(value)
elif isinstance(value, dict):
snap[key] = dict(value)
else:
snap[key] = value
snap[key] = copy.deepcopy(value)
return snap
@@ -231,10 +310,16 @@ active_layers: dict[str, bool] = {
"satnogs": True,
"tinygs": True,
"ukraine_alerts": True,
"power_plants": False,
"power_plants": True,
"viirs_nightlights": False,
"psk_reporter": True,
"correlations": True,
"contradictions": True,
"uap_sightings": True,
"wastewater": True,
"ai_intel": True,
"crowdthreat": True,
"sar": True,
}
@@ -0,0 +1,177 @@
"""OpenSky aircraft metadata: ICAO24 hex -> ICAO type code + friendly model.
OpenSky's /states/all does not include aircraft type, so OpenSky-sourced
flights arrive with ``t`` field empty. This module bulk-loads the public
OpenSky aircraft database (one snapshot CSV per month, ~108 MB uncompressed,
~600k aircraft) once every 5 days and exposes a fast in-memory hex lookup.
The data is also useful when adsb.lol's live API is degraded: even the
adsb.lol /v2 feed sometimes returns aircraft with empty ``t`` for newly seen
transponders, and the lookup gracefully fills those in too.
"""
from __future__ import annotations
import csv
import logging
import threading
import time
import xml.etree.ElementTree as ET
from typing import Any
import requests
logger = logging.getLogger(__name__)
_BUCKET_LIST_URL = (
"https://s3.opensky-network.org/data-samples?prefix=metadata/&list-type=2"
)
_BUCKET_BASE = "https://s3.opensky-network.org/data-samples/"
_S3_NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
_REFRESH_INTERVAL_S = 5 * 24 * 3600
_LIST_TIMEOUT_S = 30
_DOWNLOAD_TIMEOUT_S = 600
_USER_AGENT = (
"ShadowBroker-OSINT/0.9.7 "
"(+https://github.com/BigBodyCobain/Shadowbroker; "
"contact: bigbodycobain@gmail.com)"
)
_lock = threading.RLock()
_aircraft_by_hex: dict[str, dict[str, str]] = {}
_last_refresh = 0.0
_in_progress = False
def _latest_snapshot_key() -> str:
"""Discover the most recent aircraft-database-complete snapshot key."""
response = requests.get(
_BUCKET_LIST_URL,
timeout=_LIST_TIMEOUT_S,
headers={"User-Agent": _USER_AGENT},
)
response.raise_for_status()
root = ET.fromstring(response.text)
keys: list[str] = []
for content in root.iter(f"{_S3_NS}Contents"):
key_el = content.find(f"{_S3_NS}Key")
if key_el is None or not key_el.text:
continue
if "aircraft-database-complete-" in key_el.text and key_el.text.endswith(".csv"):
keys.append(key_el.text)
if not keys:
raise RuntimeError("no aircraft-database-complete snapshot found in bucket listing")
return sorted(keys)[-1]
def _stream_csv_index(url: str) -> dict[str, dict[str, str]]:
"""Stream-parse the OpenSky aircraft CSV into a hex-keyed index.
The CSV uses single-quote quoting, so csv.DictReader is configured with
``quotechar="'"``. Rows are processed line-by-line via iter_lines() to
keep memory bounded even though the file is ~108 MB.
"""
with requests.get(
url,
timeout=_DOWNLOAD_TIMEOUT_S,
stream=True,
headers={"User-Agent": _USER_AGENT},
) as response:
response.raise_for_status()
line_iter = (
line.decode("utf-8", errors="replace")
for line in response.iter_lines(decode_unicode=False)
if line
)
reader = csv.DictReader(line_iter, quotechar="'")
index: dict[str, dict[str, str]] = {}
for row in reader:
hex_code = (row.get("icao24") or "").strip().lower()
if not hex_code or hex_code == "000000":
continue
typecode = (row.get("typecode") or "").strip().upper()
model = (row.get("model") or "").strip()
mfr = (row.get("manufacturerName") or "").strip()
registration = (row.get("registration") or "").strip().upper()
operator = (row.get("operator") or "").strip()
if not (typecode or model):
continue
entry: dict[str, str] = {}
if typecode:
entry["typecode"] = typecode
if model:
entry["model"] = model
if mfr:
entry["manufacturer"] = mfr
if registration:
entry["registration"] = registration
if operator:
entry["operator"] = operator
index[hex_code] = entry
return index
def refresh_aircraft_database(force: bool = False) -> bool:
"""Download the latest OpenSky aircraft snapshot and rebuild the index.
Returns True if a refresh was performed (success or attempted), False if
skipped because the cache is still fresh or another refresh is in flight.
"""
global _last_refresh, _in_progress
now = time.time()
with _lock:
if _in_progress:
return False
if not force and (now - _last_refresh) < _REFRESH_INTERVAL_S and _aircraft_by_hex:
return False
_in_progress = True
try:
started = time.time()
key = _latest_snapshot_key()
index = _stream_csv_index(_BUCKET_BASE + key)
with _lock:
_aircraft_by_hex.clear()
_aircraft_by_hex.update(index)
_last_refresh = time.time()
logger.info(
"aircraft database refreshed in %.1fs from %s: %d aircraft",
time.time() - started,
key,
len(index),
)
return True
except (requests.RequestException, OSError, ValueError, ET.ParseError) as exc:
logger.warning("aircraft database refresh failed: %s", exc)
return True
finally:
with _lock:
_in_progress = False
def lookup_aircraft(icao24: str) -> dict[str, str] | None:
"""Return the metadata record for an ICAO24 hex code, or None."""
key = (icao24 or "").strip().lower()
if not key:
return None
with _lock:
entry = _aircraft_by_hex.get(key)
return dict(entry) if entry else None
def lookup_aircraft_type(icao24: str) -> str:
"""Return the ICAO type code (e.g. 'B738', 'GLF4') or '' if unknown."""
entry = lookup_aircraft(icao24)
if not entry:
return ""
return entry.get("typecode", "")
def aircraft_database_status() -> dict[str, Any]:
with _lock:
return {
"last_refresh": _last_refresh,
"aircraft": len(_aircraft_by_hex),
"in_progress": _in_progress,
}
+129
View File
@@ -0,0 +1,129 @@
"""CrowdThreat fetcher — crowdsourced global threat intelligence.
Polls verified threat reports from CrowdThreat's public API and normalises
them into map-ready records with category-based icon IDs.
No API key required — the /threats endpoint is unauthenticated.
"""
import logging
from services.network_utils import fetch_with_curl
from services.fetchers._store import latest_data, _data_lock, _mark_fresh, is_any_active
from services.fetchers.retry import with_retry
logger = logging.getLogger("services.data_fetcher")
_CT_BASE = "https://backend.crowdthreat.world"
# CrowdThreat category_id → icon ID used on the MapLibre layer
_CATEGORY_ICON = {
1: "ct-security", # Security & Conflict (red)
2: "ct-crime", # Crime & Safety (blue)
3: "ct-aviation", # Aviation (green)
4: "ct-maritime", # Maritime (teal)
5: "ct-infrastructure", # Industrial & Infra (orange)
6: "ct-special", # Special Threats (purple)
7: "ct-social", # Social & Political (pink)
8: "ct-other", # Other (gray)
}
_CATEGORY_COLOUR = {
1: "#ef4444", # red
2: "#3b82f6", # blue
3: "#22c55e", # green
4: "#14b8a6", # teal
5: "#f97316", # orange
6: "#a855f7", # purple
7: "#ec4899", # pink
8: "#6b7280", # gray
}
@with_retry(max_retries=2, base_delay=5)
def fetch_crowdthreat():
"""Fetch verified threat reports from CrowdThreat public API."""
if not is_any_active("crowdthreat"):
return
try:
resp = fetch_with_curl(f"{_CT_BASE}/threats", timeout=20)
if not resp or resp.status_code != 200:
logger.warning("CrowdThreat API returned %s", getattr(resp, "status_code", "None"))
return
payload = resp.json()
raw_threats = payload.get("data", {}).get("threats", [])
if not raw_threats:
logger.debug("CrowdThreat returned 0 threats")
return
except Exception as e:
logger.error("CrowdThreat fetch error: %s", e)
return
processed = []
for t in raw_threats:
loc = t.get("location") or {}
lng_lat = loc.get("lng_lat")
if not lng_lat or len(lng_lat) < 2:
continue
try:
lng = float(lng_lat[0])
lat = float(lng_lat[1])
except (TypeError, ValueError):
continue
cat = t.get("category") or {}
cat_id = cat.get("id", 8)
subcat = t.get("subcategory") or {}
threat_type = t.get("type") or {}
dates = t.get("dates") or {}
occurred = dates.get("occurred") or {}
reported = dates.get("reported") or {}
# Extract all available detail from the API response
summary = (t.get("summary") or t.get("description") or "").strip()
verification = (t.get("verification_status") or t.get("status") or "").strip()
country_obj = loc.get("country") or {}
country = country_obj.get("name", "") if isinstance(country_obj, dict) else str(country_obj or "")
media = t.get("media") or t.get("images") or t.get("attachments") or []
source_url = t.get("source_url") or t.get("url") or t.get("link") or ""
severity = t.get("severity") or t.get("severity_level") or t.get("risk_level") or ""
votes = t.get("votes") or t.get("upvotes") or 0
reporter = t.get("user") or t.get("reporter") or {}
reporter_name = reporter.get("name", "") if isinstance(reporter, dict) else ""
processed.append({
"id": t.get("id"),
"title": t.get("title", ""),
"summary": summary[:500] if summary else "",
"lat": lat,
"lng": lng,
"address": loc.get("name", ""),
"city": loc.get("city", ""),
"country": country,
"category": cat.get("name", "Other"),
"category_id": cat_id,
"category_colour": _CATEGORY_COLOUR.get(cat_id, "#6b7280"),
"subcategory": subcat.get("name", ""),
"threat_type": threat_type.get("name", ""),
"icon_id": _CATEGORY_ICON.get(cat_id, "ct-other"),
"occurred": occurred.get("raw", ""),
"occurred_iso": occurred.get("iso", ""),
"timeago": occurred.get("timeago", ""),
"reported": reported.get("raw", ""),
"verification": verification,
"severity": str(severity),
"source_url": source_url,
"media_urls": [m.get("url") or m for m in media[:3]] if isinstance(media, list) else [],
"votes": int(votes) if votes else 0,
"reporter": reporter_name,
"source": "CrowdThreat",
})
logger.info("CrowdThreat: fetched %d verified threats", len(processed))
with _data_lock:
latest_data["crowdthreat"] = processed
_mark_fresh("crowdthreat")
+855 -1
View File
@@ -1,14 +1,19 @@
"""Earth-observation fetchers — earthquakes, FIRMS fires, space weather, weather radar,
severe weather alerts, air quality, volcanoes."""
import concurrent.futures
import csv
import hashlib
import io
import json
import logging
import os
import re
import shutil
import subprocess
import time
import heapq
from datetime import datetime
from datetime import datetime, timedelta
from pathlib import Path
from services.network_utils import fetch_with_curl
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
@@ -596,3 +601,852 @@ def fetch_viirs_change_nodes():
if nodes:
_mark_fresh("viirs_change_nodes")
logger.info(f"VIIRS change nodes: {len(nodes)} nodes from {len(_VIIRS_AOIS)} AOIs")
# ---------------------------------------------------------------------------
# UAP Sightings (NUFORC — National UAP Reporting Center)
# ---------------------------------------------------------------------------
# Shape → canonical category mapping for consistent frontend filtering
_UAP_SHAPE_MAP = {
"light": "light", "fireball": "fireball", "orb": "orb",
"sphere": "orb", "circle": "orb", "oval": "orb", "egg": "orb",
"triangle": "triangle", "delta": "triangle", "chevron": "triangle",
"boomerang": "triangle",
"cigar": "cigar", "cylinder": "cigar", "tube": "cigar",
"disk": "disk", "disc": "disk", "saucer": "disk",
"diamond": "diamond", "cone": "diamond", "cross": "diamond",
"rectangle": "rectangle", "square": "rectangle",
"formation": "formation", "cluster": "formation",
"changing": "changing", "flash": "flash", "star": "light",
"tic-tac": "tic-tac", "tic tac": "tic-tac",
}
# US state → approximate centroid for coarse geocoding when city lookup fails
_US_STATE_COORDS: dict[str, tuple[float, float]] = {
"AL": (32.8, -86.8), "AK": (64.2, -152.5), "AZ": (34.0, -111.1),
"AR": (35.2, -91.8), "CA": (36.8, -119.4), "CO": (39.6, -105.3),
"CT": (41.6, -72.7), "DE": (39.3, -75.5), "FL": (27.8, -81.8),
"GA": (32.7, -83.5), "HI": (19.9, -155.6), "ID": (44.1, -114.7),
"IL": (40.3, -89.0), "IN": (40.3, -86.1), "IA": (42.0, -93.2),
"KS": (39.0, -98.5), "KY": (37.8, -84.3), "LA": (31.2, -92.5),
"ME": (45.3, -69.4), "MD": (39.0, -76.6), "MA": (42.4, -71.4),
"MI": (44.3, -85.6), "MN": (46.7, -94.7), "MS": (32.7, -89.5),
"MO": (38.6, -91.8), "MT": (46.8, -110.4), "NE": (41.5, -99.9),
"NV": (38.8, -116.4), "NH": (43.2, -71.6), "NJ": (40.1, -74.4),
"NM": (34.5, -106.0), "NY": (43.0, -75.0), "NC": (35.6, -79.8),
"ND": (47.5, -100.5), "OH": (40.4, -82.9), "OK": (35.0, -97.1),
"OR": (43.8, -120.6), "PA": (41.2, -77.2), "RI": (41.6, -71.5),
"SC": (33.8, -81.2), "SD": (43.9, -99.4), "TN": (35.5, -86.6),
"TX": (31.0, -97.6), "UT": (39.3, -111.1), "VT": (44.6, -72.6),
"VA": (37.4, -78.7), "WA": (47.4, -120.7), "WV": (38.6, -80.6),
"WI": (43.8, -88.8), "WY": (43.1, -107.6), "DC": (38.9, -77.0),
}
def _normalize_uap_shape(raw: str) -> str:
"""Normalize a raw NUFORC shape string to a canonical category."""
key = raw.strip().lower()
return _UAP_SHAPE_MAP.get(key, "unknown")
def _reverse_geocode_state(lat: float, lng: float) -> tuple[str, str]:
"""Best-effort reverse-geocode a lat/lng to (state_abbr, country).
Uses the _US_STATE_COORDS centroid table for fast approximate matching.
Returns ('', 'Unknown') if no close match is found.
"""
best_state = ""
best_dist = 999.0
for st, (slat, slng) in _US_STATE_COORDS.items():
d = ((lat - slat) ** 2 + (lng - slng) ** 2) ** 0.5
if d < best_dist:
best_dist = d
best_state = st
if best_dist < 5.0: # ~5 degrees tolerance
return best_state, "US"
return "", "Unknown"
# ── NUFORC Mapbox Tilequery API ─────────────────────────────────────────
# NUFORC's website switched to a JS-rendered Mapbox GL map. The old HTML
# table scraper is defunct. We now query the Mapbox Tilequery API against
# NUFORC's public tileset to get precise sighting coordinates.
#
# Tileset: nuforc.cmm18aqea06bu1mmselhpnano-0ce5v
# Layer: Sightings Fields: Count, From, To, LinkLat, LinkLon
#
# We sample a grid of points across the US/world with a 100 km radius and
# filter to sightings within the last 60 days.
_NUFORC_TILESET = "nuforc.cmm18aqea06bu1mmselhpnano-0ce5v"
_NUFORC_TOKEN = os.environ.get("NUFORC_MAPBOX_TOKEN", "").strip()
_NUFORC_RADIUS_M = 200_000 # 200 km query radius
_NUFORC_LIMIT = 50 # max features per tilequery call
_NUFORC_RECENT_DAYS = int(os.environ.get("NUFORC_RECENT_DAYS", "60"))
_NUFORC_GEOCODE_WORKERS = max(1, int(os.environ.get("NUFORC_GEOCODE_WORKERS", "1")))
# Photon (Komoot) is more lenient than Nominatim — ~200ms per query in
# practice, so a 0.3s spacing keeps us well under any soft throttle while
# still rebuilding a full 12-month window in ~10 minutes.
_NUFORC_GEOCODE_SPACING_S = float(os.environ.get("NUFORC_GEOCODE_SPACING_S", "0.3"))
_NUFORC_DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"
_NUFORC_SIGHTINGS_CACHE_FILE = _NUFORC_DATA_DIR / "nuforc_recent_sightings.json"
_NUFORC_LOCATION_CACHE_FILE = _NUFORC_DATA_DIR / "nuforc_location_cache.json"
# Live NUFORC databank scraping (wpDataTables server-side AJAX).
# The HuggingFace mirror froze at 2023-12-20, so we pull directly from
# nuforc.org's monthly sub-index. Each month page embeds a wdtNonce we
# must extract, then POST to admin-ajax.php to get the DataTables JSON.
_NUFORC_LIVE_INDEX_URL = "https://nuforc.org/subndx/?id=e{yyyymm}"
_NUFORC_LIVE_AJAX_URL = (
"https://nuforc.org/wp-admin/admin-ajax.php"
"?action=get_wdtable&table_id=1&wdt_var1=YearMonth&wdt_var2={yyyymm}"
)
_NUFORC_LIVE_NONCE_RE = re.compile(
r'id=["\']wdtNonceFrontendServerSide_1["\'][^>]*value=["\']([a-f0-9]+)["\']'
)
_NUFORC_LIVE_SIGHTING_ID_RE = re.compile(r"id=(\d+)")
_NUFORC_LIVE_USER_AGENT = "Mozilla/5.0 (ShadowBroker-OSINT NUFORC-fetcher)"
_NUFORC_LIVE_SESSION_COOKIES = _NUFORC_DATA_DIR / "nuforc_session.cookies"
# Sample grid covering continental US, Alaska, Hawaii, Canada, UK, Australia
_TILEQUERY_GRID: list[tuple[float, float]] = [
# Continental US — ~4° spacing (lon, lat)
(-122.4, 37.8), (-118.2, 34.1), (-112.1, 33.4), (-104.9, 39.7),
(-95.4, 29.8), (-96.8, 32.8), (-87.6, 41.9), (-84.4, 33.7),
(-81.7, 41.5), (-80.2, 25.8), (-77.0, 38.9), (-74.0, 40.7),
(-71.1, 42.4), (-90.2, 38.6), (-93.3, 44.9), (-111.9, 40.8),
(-122.7, 45.5), (-86.2, 39.8), (-106.6, 35.1), (-73.9, 43.2),
(-76.6, 39.3), (-97.5, 35.5), (-83.0, 42.3), (-117.2, 32.7),
(-82.5, 28.0), (-78.6, 35.8), (-90.1, 30.0), (-71.4, 41.8),
# Alaska, Hawaii
(-149.9, 61.2), (-155.5, 19.9),
# Canada
(-79.4, 43.7), (-123.1, 49.3), (-73.6, 45.5),
# UK & Europe
(-0.1, 51.5), (-3.2, 55.9),
# Australia
(151.2, -33.9), (144.9, -37.8),
]
def _fetch_nuforc_tilequery(lng: float, lat: float) -> list[dict]:
"""Query NUFORC Mapbox tileset around a single point, return raw features."""
if not _NUFORC_TOKEN:
return []
url = (
f"https://api.mapbox.com/v4/{_NUFORC_TILESET}/tilequery/"
f"{lng},{lat}.json"
f"?radius={_NUFORC_RADIUS_M}&limit={_NUFORC_LIMIT}"
f"&access_token={_NUFORC_TOKEN}"
)
try:
resp = fetch_with_curl(url, timeout=12)
if resp.status_code == 200:
data = resp.json()
return data.get("features", [])
except Exception:
pass
return []
def _parse_nuforc_tile_date(value: str) -> datetime | None:
raw = str(value or "").strip()
if not raw:
return None
raw = raw.replace("T", " ")
raw = re.sub(r"\s+local$", "", raw, flags=re.IGNORECASE)
raw = re.sub(r"\s+utc$", "", raw, flags=re.IGNORECASE)
for fmt in (
"%Y-%m-%d %H:%M:%S",
"%Y-%m-%d %H:%M",
"%Y-%m-%d",
"%m/%d/%Y %H:%M",
"%m/%d/%Y",
):
try:
return datetime.strptime(raw, fmt)
except (TypeError, ValueError):
continue
match = re.match(r"^(\d{4}-\d{2}-\d{2})", raw)
if match:
try:
return datetime.strptime(match.group(1), "%Y-%m-%d")
except ValueError:
return None
return None
def _load_nuforc_sightings_cache(*, force_refresh: bool = False) -> list[dict] | None:
if force_refresh or not _NUFORC_SIGHTINGS_CACHE_FILE.exists():
return None
try:
raw = json.loads(_NUFORC_SIGHTINGS_CACHE_FILE.read_text(encoding="utf-8"))
built = raw.get("built", "")
built_dt = datetime.fromisoformat(built) if built else None
if built_dt is None:
return None
if (datetime.utcnow() - built_dt).total_seconds() > 86400:
return None
sightings = raw.get("sightings")
if isinstance(sightings, list):
if len(sightings) <= 0:
logger.info("UAP sightings: cache is fresh but empty; rebuilding")
return None
logger.info(
"UAP sightings: loaded %d cached reports from %s",
len(sightings),
built,
)
return sightings
except Exception as e:
logger.warning("UAP sightings: cache load error: %s", e)
return None
def _save_nuforc_sightings_cache(sightings: list[dict]) -> None:
if not sightings:
logger.warning("UAP sightings: refusing to save empty daily cache")
return
try:
_NUFORC_DATA_DIR.mkdir(parents=True, exist_ok=True)
payload = {
"built": datetime.utcnow().isoformat(),
"count": len(sightings),
"sightings": sightings,
}
_NUFORC_SIGHTINGS_CACHE_FILE.write_text(
json.dumps(payload, separators=(",", ":")),
encoding="utf-8",
)
except Exception as e:
logger.warning("UAP sightings: cache save error: %s", e)
def _load_nuforc_location_cache() -> dict[str, list[float] | None]:
if not _NUFORC_LOCATION_CACHE_FILE.exists():
return {}
try:
raw = json.loads(_NUFORC_LOCATION_CACHE_FILE.read_text(encoding="utf-8"))
if not isinstance(raw, dict):
return {}
cache: dict[str, list[float] | None] = {}
for key, value in raw.items():
if not isinstance(key, str):
continue
if (
isinstance(value, list)
and len(value) == 2
and all(isinstance(v, (int, float)) for v in value)
):
cache[key] = [float(value[0]), float(value[1])]
elif value is None:
cache[key] = None
return cache
except Exception as e:
logger.warning("UAP sightings: location cache load error: %s", e)
return {}
def _save_nuforc_location_cache(cache: dict[str, list[float] | None]) -> None:
try:
_NUFORC_DATA_DIR.mkdir(parents=True, exist_ok=True)
_NUFORC_LOCATION_CACHE_FILE.write_text(
json.dumps(cache, separators=(",", ":")),
encoding="utf-8",
)
except Exception as e:
logger.warning("UAP sightings: location cache save error: %s", e)
def _normalize_uap_location(raw: str) -> str:
return re.sub(r"\s+", " ", str(raw or "").strip())
def _uap_country_from_location(location: str, state: str) -> str:
if state:
return "US"
upper = location.upper()
if "USA" in upper or "UNITED STATES" in upper:
return "US"
parts = [part.strip() for part in location.split(",") if part.strip()]
if not parts:
return "Unknown"
country = parts[-1]
return country.upper() if len(country) == 2 else country
_US_COUNTRY_ALIASES = {
"", "USA", "US", "U.S.", "U.S.A.",
"UNITED STATES", "UNITED STATES OF AMERICA",
}
def _uap_geocode_candidates(
location: str, city: str, state: str, country: str = ""
) -> list[str]:
"""Build geocode query candidates in priority order.
NUFORC's live databank is international, so we must query with the
actual country first. Only when the country is empty or explicitly US
do we fall back to the legacy USA-assumption behavior.
"""
candidates: list[str] = []
c = (country or "").strip()
c_upper = c.upper()
is_us = c_upper in _US_COUNTRY_ALIASES
if not is_us:
# Non-US: try country-qualified queries first to prevent the
# geocoder from fuzzy-matching to a same-named US city.
if city and state:
candidates.append(f"{city}, {state}, {c}")
if city:
candidates.append(f"{city}, {c}")
if city and state:
candidates.append(f"{city}, {state}")
if city:
candidates.append(city)
else:
if city and state:
candidates.append(f"{city}, {state}, USA")
candidates.append(f"{city}, {state}")
if city:
candidates.append(city)
normalized = _normalize_uap_location(location)
if normalized:
candidates.append(normalized)
parts = [part.strip() for part in normalized.split(",") if part.strip()]
if len(parts) >= 2:
candidates.append(", ".join(parts[:2]))
if parts:
candidates.append(parts[0])
deduped: list[str] = []
seen: set[str] = set()
for candidate in candidates:
key = candidate.lower()
if key in seen:
continue
seen.add(key)
deduped.append(candidate)
return deduped
def _photon_lookup(query: str) -> list[float] | None:
"""Query Komoot's public Photon instance (OSM-based, no API key).
Returns [lat, lng] on success, None on any failure. We bypass the
shared search_geocode() helper on purpose: it falls back to an
airport-name token matcher on failure that confidently returns
completely wrong coordinates, which poisoned the cache for years.
"""
from urllib.parse import urlencode
params = urlencode({"q": query, "limit": 1})
url = f"https://photon.komoot.io/api?{params}"
try:
res = fetch_with_curl(
url,
headers={
"User-Agent": "ShadowBroker-OSINT/1.0 (NUFORC-UAP-layer)",
"Accept-Language": "en",
},
timeout=10,
)
except Exception:
return None
if not res or res.status_code != 200:
return None
try:
payload = res.json()
except Exception:
return None
features = (payload or {}).get("features") or []
if not features:
return None
try:
# GeoJSON order is [lng, lat] — flip to our [lat, lng] convention.
coords = features[0]["geometry"]["coordinates"]
return [float(coords[1]), float(coords[0])]
except (KeyError, IndexError, TypeError, ValueError):
return None
def _geocode_uap_location(
location: str, city: str, state: str, country: str = ""
) -> list[float] | None:
"""Resolve a NUFORC sighting location to [lat, lng] via Photon.
Returns None on failure. The caller caches None alongside real hits
so we don't retry unresolvable queries every run.
"""
for query in _uap_geocode_candidates(location, city, state, country):
coords = _photon_lookup(query)
if coords:
return coords
return None
def _build_uap_sighting_id(row: dict, occurred: str, location: str) -> str:
raw_id = str(row.get("Sighting", "") or row.get("sighting", "")).strip()
if raw_id:
return raw_id
digest = hashlib.sha1(
f"{occurred}|{location}|{row.get('Summary', '')}|{row.get('Text', '')}".encode("utf-8", "ignore")
).hexdigest()[:12]
return f"NUFORC-{digest}"
def _nuforc_months_for_window(days: int) -> list[str]:
"""Enumerate YYYYMM strings covering the rolling `days`-day window.
Returned newest first. Always includes the current month even if the
window technically starts later, because new reports land there.
"""
today = datetime.utcnow().date()
start = today - timedelta(days=days)
months: list[str] = []
cur = today.replace(day=1)
start_floor = start.replace(day=1)
while cur >= start_floor:
months.append(cur.strftime("%Y%m"))
if cur.month == 1:
cur = cur.replace(year=cur.year - 1, month=12)
else:
cur = cur.replace(month=cur.month - 1)
return months
def _nuforc_fetch_month_live(yyyymm: str, cookie_jar: Path) -> list[dict]:
"""Pull one month of NUFORC sightings via the live wpDataTables AJAX.
Returns a list of raw row dicts with the fields we care about:
id, occurred (YYYY-MM-DD), posted (YYYY-MM-DD), city, state, country,
shape_raw, summary, explanation. Empty list on any failure — caller
decides whether a failure is fatal.
"""
from services.fetchers.nuforc_enrichment import _parse_date
curl_bin = shutil.which("curl") or "curl"
index_url = _NUFORC_LIVE_INDEX_URL.format(yyyymm=yyyymm)
ajax_url = _NUFORC_LIVE_AJAX_URL.format(yyyymm=yyyymm)
# Step 1: GET the month index to capture session cookies + fresh nonce.
try:
index_res = subprocess.run(
[
curl_bin, "-sL",
"-A", _NUFORC_LIVE_USER_AGENT,
"-c", str(cookie_jar),
"-b", str(cookie_jar),
index_url,
],
capture_output=True, text=True, timeout=60,
encoding="utf-8", errors="replace",
)
except (subprocess.SubprocessError, OSError) as e:
logger.warning("NUFORC live: index fetch failed for %s: %s", yyyymm, e)
return []
if index_res.returncode != 0 or not index_res.stdout:
logger.warning(
"NUFORC live: index fetch exit=%s for %s", index_res.returncode, yyyymm,
)
return []
nonce_match = _NUFORC_LIVE_NONCE_RE.search(index_res.stdout)
if not nonce_match:
logger.warning("NUFORC live: wdtNonce not found on index page for %s", yyyymm)
return []
nonce = nonce_match.group(1)
# Step 2: POST to admin-ajax.php with length=-1 to pull the whole month.
post_data = (
"draw=1"
"&columns%5B0%5D%5Bdata%5D=0&columns%5B0%5D%5Bsearchable%5D=true&columns%5B0%5D%5Borderable%5D=false"
"&columns%5B1%5D%5Bdata%5D=1&columns%5B1%5D%5Bsearchable%5D=true&columns%5B1%5D%5Borderable%5D=true"
"&order%5B0%5D%5Bcolumn%5D=1&order%5B0%5D%5Bdir%5D=desc"
"&start=0&length=-1"
"&search%5Bvalue%5D=&search%5Bregex%5D=false"
f"&wdtNonce={nonce}"
)
try:
ajax_res = subprocess.run(
[
curl_bin, "-sL",
"-A", _NUFORC_LIVE_USER_AGENT,
"-c", str(cookie_jar),
"-b", str(cookie_jar),
"-X", "POST",
"-H", f"Referer: {index_url}",
"-H", "X-Requested-With: XMLHttpRequest",
"-H", "Content-Type: application/x-www-form-urlencoded",
"--data", post_data,
ajax_url,
],
capture_output=True, text=True, timeout=120,
encoding="utf-8", errors="replace",
)
except (subprocess.SubprocessError, OSError) as e:
logger.warning("NUFORC live: ajax fetch failed for %s: %s", yyyymm, e)
return []
if ajax_res.returncode != 0 or not ajax_res.stdout:
logger.warning(
"NUFORC live: ajax fetch exit=%s for %s", ajax_res.returncode, yyyymm,
)
return []
try:
payload = json.loads(ajax_res.stdout)
except json.JSONDecodeError as e:
logger.warning("NUFORC live: ajax JSON decode failed for %s: %s", yyyymm, e)
return []
raw_rows = payload.get("data") or []
out: list[dict] = []
for raw in raw_rows:
if not isinstance(raw, list) or len(raw) < 8:
continue
link_html = str(raw[0] or "")
occurred_raw = str(raw[1] or "")
city = str(raw[2] or "").strip()
state = str(raw[3] or "").strip()
country = str(raw[4] or "").strip()
shape_raw = (str(raw[5] or "").strip() or "Unknown")
summary = str(raw[6] or "").strip()
reported_raw = str(raw[7] or "")
explanation = str(raw[9] or "").strip() if len(raw) > 9 and raw[9] else ""
occurred_ymd = _parse_date(occurred_raw)
if not occurred_ymd:
continue
if not city and not state and not country:
continue
id_match = _NUFORC_LIVE_SIGHTING_ID_RE.search(link_html)
if id_match:
sighting_id = f"NUFORC-{id_match.group(1)}"
else:
digest = hashlib.sha1(
f"{occurred_ymd}|{city}|{state}|{summary}".encode("utf-8", "ignore")
).hexdigest()[:12]
sighting_id = f"NUFORC-{digest}"
if summary and len(summary) > 280:
summary = summary[:277] + "..."
if not summary:
summary = "Sighting reported"
out.append({
"id": sighting_id,
"occurred": occurred_ymd,
"posted": _parse_date(reported_raw) or occurred_ymd,
"city": city,
"state": state,
"country": country,
"shape_raw": shape_raw,
"summary": summary,
"explanation": explanation,
})
return out
def _build_recent_uap_sightings() -> list[dict]:
"""Build the rolling 1-year UAP sightings layer from live NUFORC data.
Hits nuforc.org's public sub-index once per month in the window, drops
anything outside the exact day-precision cutoff, dedupes by sighting id,
geocodes city+state via the existing location cache, and returns rows
keyed to the same schema the frontend already renders.
"""
cutoff_dt = datetime.utcnow() - timedelta(days=_NUFORC_RECENT_DAYS)
cutoff_str = cutoff_dt.strftime("%Y-%m-%d")
months = _nuforc_months_for_window(_NUFORC_RECENT_DAYS)
try:
_NUFORC_DATA_DIR.mkdir(parents=True, exist_ok=True)
except Exception:
pass
rows: list[dict] = []
locations: dict[str, tuple[str, str]] = {}
seen_ids: set[str] = set()
total_pulled = 0
months_with_data = 0
for yyyymm in months:
month_rows = _nuforc_fetch_month_live(yyyymm, _NUFORC_LIVE_SESSION_COOKIES)
if month_rows:
months_with_data += 1
total_pulled += len(month_rows)
for row in month_rows:
if row["occurred"] < cutoff_str:
continue
if row["id"] in seen_ids:
continue
seen_ids.add(row["id"])
# Build the geocode key as "City, State, Country" to match the
# existing 3,000+ entry location cache (format: "Toronto, ON, Canada").
parts = [row["city"], row["state"], row["country"]]
location = _normalize_uap_location(
", ".join(p for p in parts if p) if any(parts) else ""
)
if not location:
continue
row["location"] = location
locations.setdefault(location, (row["city"], row["state"], row["country"]))
row["shape"] = (
_normalize_uap_shape(row["shape_raw"])
if row["shape_raw"] != "Unknown"
else "unknown"
)
if not row["country"]:
row["country"] = _uap_country_from_location(location, row["state"])
rows.append(row)
# Clean up the cookie jar — we don't reuse it across runs.
try:
if _NUFORC_LIVE_SESSION_COOKIES.exists():
_NUFORC_LIVE_SESSION_COOKIES.unlink()
except Exception:
pass
# Source-integrity canary: if the upstream plugin changed its
# DataTables schema or the wdtNonce regex is stale, total_pulled
# collapses to ~0 without any HTTP error. assert_canary logs a loud
# ERROR so the failure is visible in the health registry and the
# daily refresh log, instead of silently serving a stale cache.
from services.slo import assert_canary
assert_canary("uap_sightings", total_pulled)
if not rows:
raise RuntimeError(
f"NUFORC live: zero rows pulled across {len(months)} months "
f"(months_with_data={months_with_data})"
)
from services.geocode_validate import coord_in_country
location_cache = _load_nuforc_location_cache()
missing_locations = [location for location in locations if location not in location_cache]
if missing_locations:
logger.info(
"UAP sightings: geocoding %d new locations (throttled at %.1fs spacing)",
len(missing_locations),
_NUFORC_GEOCODE_SPACING_S,
)
# Sequential with spacing — Photon is fast and lenient but we
# stay sub-second to be polite. Incremental cache saves every 50
# hits keep long runs resumable.
resolved = 0
bbox_rejected = 0
save_every = 50
for idx, location in enumerate(missing_locations):
city, state, country = locations[location]
coords = None
try:
coords = _geocode_uap_location(location, city, state, country)
except Exception:
coords = None
# Country-bbox post-filter: reject namesake collisions like
# "Milan, WI" landing in Milan, Italy. Unknown countries
# (bbox not registered) are passed through unchanged.
if coords and country:
inside = coord_in_country(coords[0], coords[1], country)
if inside is False:
logger.warning(
"UAP sightings: bbox reject %r -> (%.3f, %.3f) not in %s",
location, coords[0], coords[1], country,
)
coords = None
bbox_rejected += 1
location_cache[location] = coords
if coords:
resolved += 1
if (idx + 1) % save_every == 0:
_save_nuforc_location_cache(location_cache)
logger.info(
"UAP sightings: geocoded %d/%d (%d resolved, %d bbox-rejected)",
idx + 1, len(missing_locations), resolved, bbox_rejected,
)
if idx + 1 < len(missing_locations):
time.sleep(_NUFORC_GEOCODE_SPACING_S)
_save_nuforc_location_cache(location_cache)
logger.info(
"UAP sightings: geocoding complete — %d/%d resolved, %d bbox-rejected",
resolved, len(missing_locations), bbox_rejected,
)
sightings: list[dict] = []
skipped_unmapped = 0
skipped_bbox = 0
for row in rows:
coords = location_cache.get(row["location"])
if not coords:
skipped_unmapped += 1
continue
# Apply bbox filter to pre-existing cache entries too — this
# cleans up the ~1-2% of cached coords that pre-dated the bbox
# check without requiring a full cache rebuild.
if row.get("country"):
inside = coord_in_country(coords[0], coords[1], row["country"])
if inside is False:
skipped_bbox += 1
continue
sightings.append(
{
"id": row["id"],
"date_time": row["occurred"],
"city": row["city"],
"state": row["state"],
"country": row["country"],
"shape": row["shape"],
"shape_raw": row["shape_raw"],
"duration": row.get("duration", ""),
"summary": row["summary"],
"posted": row["posted"],
"lat": float(coords[0]),
"lng": float(coords[1]),
"count": 1,
"source": "NUFORC",
}
)
if row.get("explanation"):
sightings[-1]["explanation"] = row["explanation"]
sightings.sort(
key=lambda sighting: (
sighting.get("date_time", ""),
sighting.get("posted", ""),
str(sighting.get("id", "")),
),
reverse=True,
)
logger.info(
"UAP sightings: %d mapped reports from %d rows across %d months "
"(cutoff %s, %d unmapped, %d bbox-rejected)",
len(sightings),
total_pulled,
len(months),
cutoff_str,
skipped_unmapped,
skipped_bbox,
)
return sightings
@with_retry(max_retries=1, base_delay=5)
def fetch_uap_sightings(*, force_refresh: bool = False):
"""Fetch last-year UAP sightings from NUFORC.
Startup reads the cached daily snapshot when it is still fresh. The daily
scheduler forces a rebuild so this layer updates once per day instead of
churning continuously.
"""
from services.fetchers._store import is_any_active
if not is_any_active("uap_sightings"):
return
sightings = _load_nuforc_sightings_cache(force_refresh=force_refresh)
if sightings is None:
sightings = _build_recent_uap_sightings()
_save_nuforc_sightings_cache(sightings)
with _data_lock:
latest_data["uap_sightings"] = sightings
_mark_fresh("uap_sightings")
return
cutoff = datetime.utcnow() - timedelta(days=_NUFORC_RECENT_DAYS)
# Query the grid concurrently (up to 8 threads)
all_features: list[dict] = []
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
futures = {
pool.submit(_fetch_nuforc_tilequery, lng, lat): (lng, lat)
for lng, lat in _TILEQUERY_GRID
}
for fut in concurrent.futures.as_completed(futures, timeout=60):
try:
all_features.extend(fut.result())
except Exception:
pass
# Deduplicate by (LinkLat, LinkLon) and filter to recent sightings
seen: set[tuple[str, str]] = set()
sightings: list[dict] = []
enriched_count = 0
for feat in all_features:
props = feat.get("properties", {})
link_lat = props.get("LinkLat", "")
link_lon = props.get("LinkLon", "")
if not link_lat or not link_lon:
continue
key = (link_lat, link_lon)
if key in seen:
continue
seen.add(key)
# Filter by date — keep if the latest sighting date >= cutoff
to_date = props.get("To", "")
from_date = props.get("From", "")
latest_date = to_date or from_date
latest_dt = _parse_nuforc_tile_date(latest_date)
if latest_dt is not None and latest_dt < cutoff:
continue
try:
lat = float(link_lat)
lng = float(link_lon)
except (ValueError, TypeError):
continue
count = int(props.get("Count", "1") or "1")
state_abbr, country = _reverse_geocode_state(lat, lng)
# Enrich with HF NUFORC dataset (shape, duration, city, summary)
enrichment = enrich_sighting(state_abbr, from_date, to_date)
city = enrichment.get("city", "")
shape_raw = enrichment.get("shape_raw", "Unknown")
shape = _normalize_uap_shape(shape_raw) if shape_raw != "Unknown" else "unknown"
duration = enrichment.get("duration", "")
summary = enrichment.get("summary", "")
if enrichment:
enriched_count += 1
# Build display summary: prefer enriched text, fall back to count-based
if not summary:
summary = f"{count} sighting(s) reported" if count > 1 else "Sighting reported"
sightings.append({
"id": f"NUFORC-{hash(key) & 0xFFFFFFFF:08x}",
"date_time": from_date if from_date == to_date else f"{from_date} to {to_date}",
"city": city,
"state": state_abbr,
"country": country,
"shape": shape,
"shape_raw": shape_raw,
"duration": duration,
"summary": summary,
"posted": to_date,
"lat": lat,
"lng": lng,
"count": count,
"source": "NUFORC",
})
logger.info(
f"UAP sightings: {len(sightings)} recent from NUFORC tilequery "
f"({len(all_features)} raw, {enriched_count} enriched)"
)
with _data_lock:
latest_data["uap_sightings"] = sightings
if sightings:
_mark_fresh("uap_sightings")
+194 -23
View File
@@ -1,20 +1,24 @@
"""
Fuel burn & CO2 emissions estimator for private jets.
Fuel burn & CO2 emissions estimator.
Based on manufacturer-published cruise fuel burn rates (GPH at long-range cruise).
1 US gallon of Jet-A produces ~21.1 lbs (9.57 kg) of CO2.
Piston entries use 100LL (avgas), which is close enough to Jet-A in CO2 yield
(~8.4 kg/gal vs 9.57 kg/gal); we keep one constant to stay simple — the result
is a slight over-estimate for piston aircraft, which is preferable to under.
"""
JET_A_CO2_KG_PER_GALLON = 9.57
# ICAO type code -> gallons per hour at long-range cruise
FUEL_BURN_GPH: dict[str, int] = {
# Gulfstream
# ── Gulfstream ─────────────────────────────────────────────────────
"GLF6": 430, # G650/G650ER
"G700": 480, # G700
"GLF5": 390, # G550
"GVSP": 400, # GV-SP
"GLF4": 330, # G-IV
# Bombardier
# ── Bombardier business ────────────────────────────────────────────
"GL7T": 490, # Global 7500
"GLEX": 430, # Global Express/6000/6500
"GL5T": 420, # Global 5000/5500
@@ -22,51 +26,208 @@ FUEL_BURN_GPH: dict[str, int] = {
"CL60": 310, # Challenger 604/605
"CL30": 200, # Challenger 300
"CL65": 320, # Challenger 650
# Dassault
# ── Bombardier regional jets ──────────────────────────────────────
"CRJ2": 360, # CRJ-100/200
"CRJ7": 380, # CRJ-700
"CRJ9": 410, # CRJ-900
"CRJX": 440, # CRJ-1000
# ── Dassault ───────────────────────────────────────────────────────
"F7X": 350, # Falcon 7X
"F8X": 370, # Falcon 8X
"F900": 285, # Falcon 900/900EX/900LX
"F2TH": 230, # Falcon 2000
"FA50": 240, # Falcon 50
# Cessna
# ── Cessna Citation ────────────────────────────────────────────────
"CITX": 280, # Citation X
"C750": 280, # Citation X (alt code)
"C68A": 195, # Citation Latitude
"C700": 230, # Citation Longitude
"C680": 220, # Citation Sovereign
"C560": 190, # Citation Excel/XLS
"C56X": 195, # Citation Excel/XLS/XLS+
"C560": 190, # Citation Excel/XLS (legacy)
"C550": 165, # Citation II/Bravo/V
"C525": 80, # Citation CJ1
"C25A": 100, # CJ1+ / 525A
"C25B": 110, # CJ2+ / 525B
"C25C": 130, # CJ4 (some operators)
"C510": 75, # Citation Mustang
"C650": 240, # Citation III/VI/VII
"CJ3": 120, # CJ3
"CJ4": 135, # CJ4
# Boeing
"B737": 850, # BBJ (737)
"B738": 920, # BBJ2 (737-800)
# ── Cessna piston / turboprop singles & twins ─────────────────────
"C172": 9, # Skyhawk
"C152": 6,
"C150": 6,
"C170": 8,
"C177": 11,
"C180": 12,
"C182": 13, # Skylane
"C185": 14,
"C206": 15,
"C208": 50, # Caravan (turboprop)
"C210": 18,
"C310": 32,
"C340": 38,
"C414": 36,
"C421": 40,
# ── Boeing mainline ────────────────────────────────────────────────
"B737": 850, # 737-700 / BBJ
"B738": 920, # 737-800
"B739": 880, # 737-900/900ER
"B38M": 700, # 737-8 MAX
"B39M": 740, # 737-9 MAX
"B752": 1100, # 757-200
"B753": 1200, # 757-300
"B762": 1400, # 767-200
"B763": 1450, # 767-300/300ER
"B764": 1500, # 767-400ER
"B772": 1850, # 777-200
"B77L": 1900, # 777-200LR / 777F
"B77W": 2050, # 777-300ER
"B788": 1200, # 787-8
# Airbus
"A318": 780, # ACJ318
"A319": 850, # ACJ319
"A320": 900, # ACJ320
"B789": 1300, # 787-9
"B78X": 1350, # 787-10
"B744": 3050, # 747-400
"B748": 2900, # 747-8
# ── Airbus mainline ────────────────────────────────────────────────
"A318": 780, # A318
"A319": 850, # A319
"A320": 900, # A320
"A321": 990, # A321
"A19N": 580, # A319neo
"A20N": 580, # A320neo
"A21N": 700, # A321neo
"A332": 1500, # A330-200
"A333": 1550, # A330-300
"A338": 1300, # A330-800neo
"A339": 1350, # A330-900neo
"A343": 1800, # A340-300
"A346": 2100, # A340-600
# Pilatus
"A359": 1450, # A350-900
"A35K": 1600, # A350-1000
"A388": 3200, # A380-800
# ── Embraer regional / business ───────────────────────────────────
"E135": 300, # Legacy 600/650 (regional ERJ-135 base)
"E145": 320, # ERJ-145
"E170": 460, # E170
"E75L": 490, # E175-LR
"E75S": 490, # E175 standard
"E175": 490, # E175 (some)
"E190": 580, # E190
"E195": 600, # E195
"E290": 510, # E190-E2
"E295": 540, # E195-E2
"E50P": 135, # Phenom 300 (also Phenom 100 var)
"E55P": 185, # Praetor 500 / Legacy 500
"E545": 170, # Praetor 500 (alt)
"E500": 80, # Phenom 100
# ── ATR / Bombardier / Saab turboprops ────────────────────────────
"AT43": 230, # ATR 42-300/-320
"AT45": 230, # ATR 42-500
"AT46": 250, # ATR 42-600
"AT72": 300, # ATR 72-200/-210
"AT75": 280, # ATR 72-500
"AT76": 280, # ATR 72-600
"DH8A": 220, # Dash 8 -100
"DH8B": 240, # Dash 8 -200
"DH8C": 280, # Dash 8 -300
"DH8D": 300, # Dash 8 Q400
"SF34": 200, # Saab 340
"SB20": 220, # Saab 2000
# ── Pilatus / Daher single-engine turboprops ──────────────────────
"PC24": 115, # PC-24
"PC12": 60, # PC-12
# Embraer
"E55P": 185, # Legacy 500
"E135": 300, # Legacy 600/650
"E50P": 135, # Phenom 300
"E500": 80, # Phenom 100
# Learjet
"TBM7": 60, # TBM 700/850
"TBM8": 65, # TBM 850 alt
"TBM9": 70, # TBM 900/930/940/960
"M600": 60, # Piper M600
"P46T": 22, # PA-46 Meridian (turboprop variant)
# ── Learjet ────────────────────────────────────────────────────────
"LJ60": 195, # Learjet 60
"LJ75": 185, # Learjet 75
"LJ45": 175, # Learjet 45
# Hawker
"LJ31": 165, # Learjet 31
"LJ40": 175, # Learjet 40
"LJ55": 195, # Learjet 55
# ── Hawker / Beechjet ─────────────────────────────────────────────
"H25B": 210, # Hawker 800/800XP
"H25C": 215, # Hawker 900XP
# Beechcraft
"BE40": 150, # Beechjet 400 / Hawker 400XP
"PRM1": 130, # Premier I
# ── Beechcraft King Air ───────────────────────────────────────────
"B350": 100, # King Air 350
"B200": 80, # King Air 200/250
"BE20": 80, # K-Air 200 (alt)
"BE9L": 60, # K-Air 90
"BE9T": 70, # K-Air F90
"BE10": 100, # K-Air 100
"BE30": 90, # K-Air 300
# ── Beechcraft / Cirrus / Piper / Mooney pistons ──────────────────
"BE23": 9, # Sundowner
"BE33": 13, # Bonanza 33
"BE35": 14, # Bonanza V-tail
"BE36": 16, # A36 Bonanza
"BE55": 24, # Baron 55
"BE58": 28, # Baron 58
"BE76": 17, # Duchess
"BE95": 20, # Travel Air
"P28A": 10, # PA-28 Warrior/Archer
"P28B": 11, # PA-28 Cherokee
"P28R": 12, # PA-28R Arrow
"P32R": 14, # PA-32R Lance/Saratoga
"PA11": 5, # Cub Special
"PA12": 6, # Super Cruiser
"PA18": 6, # Super Cub
"PA22": 8, # Tri-Pacer
"PA23": 18, # Apache / Aztec
"PA24": 12, # Comanche
"PA25": 12, # Pawnee
"PA28": 10, # PA-28 generic
"PA30": 16, # Twin Comanche
"PA31": 30, # Navajo
"PA32": 14, # Cherokee Six / Saratoga
"PA34": 18, # Seneca
"PA38": 5, # Tomahawk
"PA44": 17, # Seminole
"PA46": 18, # Malibu / Mirage / Matrix
"M20P": 12, # Mooney M20 (generic)
"SR20": 11, # Cirrus SR20
"SR22": 16, # Cirrus SR22
"S22T": 19, # SR22T (turbo)
"DA40": 9, # Diamond DA40
"DA42": 14, # Diamond DA42 TwinStar
"DA62": 17, # Diamond DA62
"DV20": 6, # Diamond Katana
# ── Helicopters (civilian) ────────────────────────────────────────
"A109": 60, # AW109
"A119": 50, # AW119
"A139": 130, # AW139
"A169": 90, # AW169
"A189": 145, # AW189
"AS35": 55, # AS350 AStar
"AS50": 55, # AStar (alt)
"AS65": 110, # Dauphin
"B06": 35, # Bell 206 JetRanger
"B407": 50, # Bell 407
"B412": 145, # Bell 412
"B429": 80, # Bell 429
"B505": 35, # Bell 505
"EC30": 50, # H125 / EC130
"EC35": 70, # EC135
"EC45": 85, # EC145
"EC75": 130, # EC175
"H125": 55,
"H130": 50,
"H135": 70,
"H145": 85,
"H155": 110,
"H160": 95,
"H175": 130,
"R22": 9, # Robinson R22 (piston)
"R44": 16, # Robinson R44 (piston)
"R66": 30, # Robinson R66 (turbine)
"S76": 140, # Sikorsky S-76
"S92": 220, # Sikorsky S-92
}
# Common string names -> ICAO type code
@@ -108,13 +269,23 @@ def get_emissions_info(model: str) -> dict | None:
if not model:
return None
model_clean = model.strip()
model_upper = model_clean.upper()
# Try direct ICAO code match first
gph = FUEL_BURN_GPH.get(model_clean.upper())
gph = FUEL_BURN_GPH.get(model_upper)
if gph is None:
# Try alias lookup
code = _ALIASES.get(model_clean)
if code:
gph = FUEL_BURN_GPH.get(code)
if gph is None:
# Friendly names from the Plane-Alert DB often lead with the ICAO type
# code as the first token (e.g. "B200 Super King Air"). Probe each
# token against FUEL_BURN_GPH directly.
for token in model_upper.replace("-", " ").replace(",", " ").split():
candidate = FUEL_BURN_GPH.get(token)
if candidate is not None:
gph = candidate
break
if gph is None:
# Fuzzy: check if any alias is a substring
model_lower = model_clean.lower()
+164 -170
View File
@@ -13,12 +13,13 @@ import concurrent.futures
import random
import requests
from datetime import datetime
from cachetools import TTLCache
from services.network_utils import fetch_with_curl
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
from services.fetchers.plane_alert import enrich_with_plane_alert, enrich_with_tracked_names
from services.fetchers.emissions import get_emissions_info
from services.fetchers.retry import with_retry
from services.fetchers.route_database import lookup_route
from services.fetchers.aircraft_database import lookup_aircraft_type
from services.constants import GPS_JAMMING_NACP_THRESHOLD, GPS_JAMMING_MIN_RATIO, GPS_JAMMING_MIN_AIRCRAFT
logger = logging.getLogger("services.data_fetcher")
@@ -76,6 +77,7 @@ opensky_client = OpenSkyClient(
# Throttling and caching for OpenSky (400 req/day limit)
last_opensky_fetch = 0
cached_opensky_flights = []
_opensky_cache_lock = threading.Lock()
# ---------------------------------------------------------------------------
# Supplemental ADS-B sources for blind-spot gap-filling
@@ -98,6 +100,7 @@ _AIRPLANES_LIVE_DELAY_SECONDS = 1.2
_AIRPLANES_LIVE_DELAY_JITTER_SECONDS = 0.4
last_supplemental_fetch = 0
cached_supplemental_flights = []
_supplemental_cache_lock = threading.Lock()
# Helicopter type codes (backend classification)
_HELI_TYPES_BACKEND = {
@@ -255,10 +258,11 @@ flight_trails = {} # {icao_hex: {points: [[lat, lng, alt, ts], ...], last_seen:
_trails_lock = threading.Lock()
_MAX_TRACKED_TRAILS = 2000
# Routes cache
dynamic_routes_cache = TTLCache(maxsize=5000, ttl=7200)
routes_fetch_in_progress = False
_routes_lock = threading.Lock()
# Route enrichment is now served from services.fetchers.route_database, which
# bulk-loads vrs-standing-data.adsb.lol/routes.csv.gz once per day and looks up
# callsigns from an in-memory index. Replaces the legacy /api/0/routeset POST,
# which was both blocked under the ShadowBroker UA (HTTP 451) and broken
# upstream (returning 201 with empty body even for unblocked clients).
def _fetch_supplemental_sources(seen_hex: set) -> list:
@@ -266,12 +270,13 @@ def _fetch_supplemental_sources(seen_hex: set) -> list:
global last_supplemental_fetch, cached_supplemental_flights
now = time.time()
if now - last_supplemental_fetch < _SUPPLEMENTAL_FETCH_INTERVAL:
return [
f
for f in cached_supplemental_flights
if f.get("hex", "").lower().strip() not in seen_hex
]
with _supplemental_cache_lock:
if now - last_supplemental_fetch < _SUPPLEMENTAL_FETCH_INTERVAL:
return [
f
for f in cached_supplemental_flights
if f.get("hex", "").lower().strip() not in seen_hex
]
new_supplemental = []
supplemental_hex = set()
@@ -363,8 +368,9 @@ def _fetch_supplemental_sources(seen_hex: set) -> list:
fi_count = len(new_supplemental) - ap_count
cached_supplemental_flights = new_supplemental
last_supplemental_fetch = now
with _supplemental_cache_lock:
cached_supplemental_flights = new_supplemental
last_supplemental_fetch = now
if new_supplemental:
_mark_fresh("supplemental_flights")
@@ -375,73 +381,6 @@ def _fetch_supplemental_sources(seen_hex: set) -> list:
return new_supplemental
def fetch_routes_background(sampled):
global routes_fetch_in_progress
with _routes_lock:
if routes_fetch_in_progress:
return
routes_fetch_in_progress = True
try:
callsigns_to_query = []
for f in sampled:
c_sign = str(f.get("flight", "")).strip()
if c_sign and c_sign != "UNKNOWN":
callsigns_to_query.append(
{"callsign": c_sign, "lat": f.get("lat", 0), "lng": f.get("lon", 0)}
)
batch_size = 100
batches = [
callsigns_to_query[i : i + batch_size]
for i in range(0, len(callsigns_to_query), batch_size)
]
for batch in batches:
try:
r = fetch_with_curl(
"https://api.adsb.lol/api/0/routeset",
method="POST",
json_data={"planes": batch},
timeout=15,
)
if r.status_code == 200:
route_data = r.json()
route_list = []
if isinstance(route_data, dict):
route_list = route_data.get("value", [])
elif isinstance(route_data, list):
route_list = route_data
for route in route_list:
callsign = route.get("callsign", "")
airports = route.get("_airports", [])
if airports and len(airports) >= 2:
orig_apt = airports[0]
dest_apt = airports[-1]
with _routes_lock:
dynamic_routes_cache[callsign] = {
"orig_name": f"{orig_apt.get('iata', '')}: {orig_apt.get('name', 'Unknown')}",
"dest_name": f"{dest_apt.get('iata', '')}: {dest_apt.get('name', 'Unknown')}",
"orig_loc": [orig_apt.get("lon", 0), orig_apt.get("lat", 0)],
"dest_loc": [dest_apt.get("lon", 0), dest_apt.get("lat", 0)],
}
time.sleep(0.25)
except (
requests.RequestException,
ConnectionError,
TimeoutError,
ValueError,
KeyError,
json.JSONDecodeError,
OSError,
) as e:
logger.debug(f"Route batch request failed: {e}")
finally:
with _routes_lock:
routes_fetch_in_progress = False
def _classify_and_publish(all_adsb_flights):
"""Shared pipeline: normalize raw ADS-B data → classify → merge → publish to latest_data.
@@ -453,13 +392,6 @@ def _classify_and_publish(all_adsb_flights):
if not all_adsb_flights:
return
with _routes_lock:
already_running = routes_fetch_in_progress
if not already_running:
threading.Thread(
target=fetch_routes_background, args=(all_adsb_flights,), daemon=True
).start()
for f in all_adsb_flights:
try:
lat = f.get("lat")
@@ -478,8 +410,7 @@ def _classify_and_publish(all_adsb_flights):
origin_name = "UNKNOWN"
dest_name = "UNKNOWN"
with _routes_lock:
cached_route = dynamic_routes_cache.get(flight_str)
cached_route = lookup_route(flight_str)
if cached_route:
origin_name = cached_route["orig_name"]
dest_name = cached_route["dest_name"]
@@ -501,7 +432,18 @@ def _classify_and_publish(all_adsb_flights):
gs_knots = f.get("gs")
speed_knots = round(gs_knots, 1) if isinstance(gs_knots, (int, float)) else None
model_upper = f.get("t", "").upper()
# OpenSky's /states/all doesn't carry the aircraft type, so its
# records arrive with t="Unknown". Backfill from the OpenSky
# aircraft metadata DB by ICAO24 hex so heli classification and
# downstream emissions enrichment both see a real type code.
raw_type = str(f.get("t") or "").strip()
if not raw_type or raw_type.lower() == "unknown":
looked_up_type = lookup_aircraft_type(f.get("hex", ""))
if looked_up_type:
f["t"] = looked_up_type
raw_type = looked_up_type
model_upper = raw_type.upper()
if model_upper == "TWR":
continue
@@ -543,8 +485,14 @@ def _classify_and_publish(all_adsb_flights):
for f in flights:
enrich_with_plane_alert(f)
enrich_with_tracked_names(f)
# Attach fuel-burn / CO2 emissions estimate when model is known
# Attach fuel-burn / CO2 emissions estimate when model is known.
# OpenSky's /states/all doesn't carry aircraft type, so OpenSky-sourced
# flights arrive with model="Unknown". For tracked planes, the
# Plane-Alert DB has the friendly type name in alert_type, and the
# emissions aliases table already maps those names to ICAO codes.
model = f.get("model")
if not model or model.strip().lower() in {"", "unknown"}:
model = f.get("alert_type") or ""
if model:
emi = get_emissions_info(model)
if emi:
@@ -618,6 +566,10 @@ def _classify_and_publish(all_adsb_flights):
latest_data["flights"] = flights
# Merge tracked civilian flights with tracked military flights
# Stale tracked flights (not seen in any ADS-B source for >5 min) are dropped.
_TRACKED_STALE_S = 300 # 5 minutes
_merge_ts = time.time()
with _data_lock:
existing_tracked = copy.deepcopy(latest_data.get("tracked_flights", []))
@@ -625,10 +577,12 @@ def _classify_and_publish(all_adsb_flights):
for t in tracked:
icao = t.get("icao24", "").upper()
if icao:
t["_seen_at"] = _merge_ts
fresh_tracked_map[icao] = t
merged_tracked = []
seen_icaos = set()
stale_dropped = 0
for old_t in existing_tracked:
icao = old_t.get("icao24", "").upper()
if icao in fresh_tracked_map:
@@ -639,8 +593,13 @@ def _classify_and_publish(all_adsb_flights):
merged_tracked.append(fresh)
seen_icaos.add(icao)
else:
merged_tracked.append(old_t)
seen_icaos.add(icao)
# Keep stale entry only if it was seen recently
age = _merge_ts - old_t.get("_seen_at", 0)
if age < _TRACKED_STALE_S:
merged_tracked.append(old_t)
seen_icaos.add(icao)
else:
stale_dropped += 1
for icao, t in fresh_tracked_map.items():
if icao not in seen_icaos:
@@ -649,10 +608,12 @@ def _classify_and_publish(all_adsb_flights):
with _data_lock:
latest_data["tracked_flights"] = merged_tracked
logger.info(
f"Tracked flights: {len(merged_tracked)} total ({len(fresh_tracked_map)} fresh from civilian)"
f"Tracked flights: {len(merged_tracked)} total ({len(fresh_tracked_map)} fresh from civilian, {stale_dropped} stale dropped)"
)
# --- Trail Accumulation ---
_TRAIL_INTERVAL_S = 600 # only record a new trail point every 10 minutes
def _accumulate_trail(f, now_ts, check_route=True):
hex_id = f.get("icao24", "").lower()
if not hex_id:
@@ -668,7 +629,11 @@ def _classify_and_publish(all_adsb_flights):
if hex_id not in flight_trails:
flight_trails[hex_id] = {"points": [], "last_seen": now_ts}
trail_data = flight_trails[hex_id]
if (
# Only append a new point if 10 minutes have passed since the last one
last_point_ts = trail_data["points"][-1][3] if trail_data["points"] else 0
if now_ts - last_point_ts < _TRAIL_INTERVAL_S:
trail_data["last_seen"] = now_ts
elif (
trail_data["points"]
and trail_data["points"][-1][0] == point[0]
and trail_data["points"][-1][1] == point[1]
@@ -688,22 +653,26 @@ def _classify_and_publish(all_adsb_flights):
tracked_snapshot = copy.deepcopy(latest_data.get("tracked_flights", []))
raw_flights_snapshot = list(latest_data.get("flights", []))
all_lists = [commercial, private_jets, private_ga, existing_tracked]
# Commercial/private: skip trail if route is known (route line replaces trail)
route_check_lists = [commercial, private_jets, private_ga]
# Tracked + military: ALWAYS accumulate trails (high-interest flights)
always_trail_lists = [existing_tracked, military_snapshot]
seen_hexes = set()
trail_count = 0
with _trails_lock:
for flist in all_lists:
for flist in route_check_lists:
for f in flist:
count, hex_id = _accumulate_trail(f, now_ts, check_route=True)
trail_count += count
if hex_id:
seen_hexes.add(hex_id)
for mf in military_snapshot:
count, hex_id = _accumulate_trail(mf, now_ts, check_route=False)
trail_count += count
if hex_id:
seen_hexes.add(hex_id)
for flist in always_trail_lists:
for f in flist:
count, hex_id = _accumulate_trail(f, now_ts, check_route=False)
trail_count += count
if hex_id:
seen_hexes.add(hex_id)
tracked_hexes = {t.get("icao24", "").lower() for t in tracked_snapshot}
stale_keys = []
@@ -889,79 +858,100 @@ def _enrich_with_opensky_and_supplemental(adsb_flights):
now = time.time()
global last_opensky_fetch, cached_opensky_flights
if now - last_opensky_fetch > 300:
with _opensky_cache_lock:
_need_opensky = now - last_opensky_fetch > 300
if not _need_opensky:
opensky_snapshot = list(cached_opensky_flights)
if _need_opensky:
token = opensky_client.get_token()
if token:
opensky_regions = [
{
"name": "Africa",
"bbox": {"lamin": -35.0, "lomin": -20.0, "lamax": 38.0, "lomax": 55.0},
},
{
"name": "Asia",
"bbox": {"lamin": 0.0, "lomin": 30.0, "lamax": 75.0, "lomax": 150.0},
},
{
"name": "South America",
"bbox": {"lamin": -60.0, "lomin": -95.0, "lamax": 15.0, "lomax": -30.0},
},
]
# One global /states/all query = 4 credits flat per OpenSky
# docs (https://openskynetwork.github.io/opensky-api/rest.html).
# At the current 5-minute cadence that's 4 × 288 = 1152
# credits/day, ~29% of the 4000-credit standard daily quota,
# and returns every aircraft worldwide in a single call.
# The previous 3-regional-bbox approach cost 12 credits/cycle
# AND missed North America, Europe, and Oceania entirely.
new_opensky_flights = []
for os_reg in opensky_regions:
try:
bb = os_reg["bbox"]
os_url = f"https://opensky-network.org/api/states/all?lamin={bb['lamin']}&lomin={bb['lomin']}&lamax={bb['lamax']}&lomax={bb['lomax']}"
headers = {"Authorization": f"Bearer {token}"}
os_res = requests.get(os_url, headers=headers, timeout=15)
try:
os_url = "https://opensky-network.org/api/states/all"
headers = {"Authorization": f"Bearer {token}"}
os_res = requests.get(os_url, headers=headers, timeout=30)
if os_res.status_code == 200:
os_data = os_res.json()
states = os_data.get("states") or []
logger.info(
f"OpenSky: Fetched {len(states)} states for {os_reg['name']}"
if os_res.status_code == 200:
os_data = os_res.json()
states = os_data.get("states") or []
remaining = os_res.headers.get("X-Rate-Limit-Remaining", "?")
logger.info(
f"OpenSky: fetched {len(states)} global states "
f"(credits remaining: {remaining})"
)
for s in states:
if s[5] is None or s[6] is None:
continue
new_opensky_flights.append(
{
"hex": s[0],
"flight": s[1].strip() if s[1] else "UNKNOWN",
"r": s[2],
"lon": s[5],
"lat": s[6],
"alt_baro": (s[7] * 3.28084) if s[7] else 0,
"track": s[10] or 0,
"gs": (s[9] * 1.94384) if s[9] else 0,
"t": "Unknown",
"is_opensky": True,
}
)
elif os_res.status_code == 429:
retry_after = os_res.headers.get("X-Rate-Limit-Retry-After-Seconds", "?")
logger.warning(
f"OpenSky daily quota exhausted (4000 credits). "
f"Retry after {retry_after}s. Serving stale data until reset."
)
else:
logger.warning(
f"OpenSky /states/all failed: HTTP {os_res.status_code}"
)
except (
requests.RequestException,
ConnectionError,
TimeoutError,
ValueError,
KeyError,
json.JSONDecodeError,
OSError,
) as ex:
logger.error(f"OpenSky global fetch error: {ex}")
for s in states:
new_opensky_flights.append(
{
"hex": s[0],
"flight": s[1].strip() if s[1] else "UNKNOWN",
"r": s[2],
"lon": s[5],
"lat": s[6],
"alt_baro": (s[7] * 3.28084) if s[7] else 0,
"track": s[10] or 0,
"gs": (s[9] * 1.94384) if s[9] else 0,
"t": "Unknown",
"is_opensky": True,
}
)
else:
logger.warning(
f"OpenSky API {os_reg['name']} failed: {os_res.status_code}"
)
except (
requests.RequestException,
ConnectionError,
TimeoutError,
ValueError,
KeyError,
json.JSONDecodeError,
OSError,
) as ex:
logger.error(f"OpenSky fetching error for {os_reg['name']}: {ex}")
cached_opensky_flights = new_opensky_flights
last_opensky_fetch = now
with _opensky_cache_lock:
if new_opensky_flights:
cached_opensky_flights = new_opensky_flights
last_opensky_fetch = now
opensky_snapshot = new_opensky_flights or list(cached_opensky_flights)
else:
# Token refresh failed — fall back to existing cached data
with _opensky_cache_lock:
opensky_snapshot = list(cached_opensky_flights)
# Merge OpenSky (dedup by hex)
for osf in cached_opensky_flights:
for osf in opensky_snapshot:
h = osf.get("hex")
if h and h.lower().strip() not in seen_hex:
all_flights.append(osf)
seen_hex.add(h.lower().strip())
# Publish OpenSky-merged data immediately so users see flights even if
# supplemental gap-fill is slow or rate-limited (airplanes.live can take
# 100+ seconds when its regional endpoints are throttled).
if len(all_flights) > len(adsb_flights):
logger.info(
f"OpenSky merge: {len(all_flights) - len(adsb_flights)} additional aircraft, "
"publishing before supplemental gap-fill"
)
_classify_and_publish(all_flights)
# Supplemental gap-fill
try:
gap_fill = _fetch_supplemental_sources(seen_hex)
@@ -1008,14 +998,18 @@ def fetch_flights():
if adsb_flights:
logger.info(f"adsb.lol: {len(adsb_flights)} aircraft — publishing immediately")
_classify_and_publish(adsb_flights)
# Phase 2: kick off slow enrichment in background
threading.Thread(
target=_enrich_with_opensky_and_supplemental,
args=(adsb_flights,),
daemon=True,
).start()
else:
logger.warning("adsb.lol returned 0 aircraft")
logger.warning(
"adsb.lol returned 0 aircraft — relying on OpenSky/supplemental sources"
)
# Phase 2: always run — OpenSky is the fallback when adsb.lol blocks us
# (it has been known to 451 the bulk regional endpoint), and supplemental
# gap-fill should always run regardless of Phase 1 success.
threading.Thread(
target=_enrich_with_opensky_and_supplemental,
args=(adsb_flights,),
daemon=True,
).start()
except Exception as e:
logger.error(f"Error fetching flights: {e}")
+158 -26
View File
@@ -1,10 +1,13 @@
"""Ship and geopolitics fetchers — AIS vessels, carriers, frontlines, GDELT, LiveUAmap, fishing."""
import csv
import concurrent.futures
import io
import math
import os
import logging
import time
from urllib.parse import urlencode
from services.network_utils import fetch_with_curl
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
from services.fetchers.retry import with_retry
@@ -27,20 +30,24 @@ def fetch_ships():
from services.ais_stream import get_ais_vessels
from services.carrier_tracker import get_carrier_positions
ships = []
try:
carriers = get_carrier_positions()
ships.extend(carriers)
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
logger.error(f"Carrier tracker error (non-fatal): {e}")
carriers = []
with concurrent.futures.ThreadPoolExecutor(max_workers=2, thread_name_prefix="ship_fetch") as executor:
carrier_future = executor.submit(get_carrier_positions)
ais_future = executor.submit(get_ais_vessels)
try:
ais_vessels = get_ais_vessels()
ships.extend(ais_vessels)
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
logger.error(f"AIS stream error (non-fatal): {e}")
ais_vessels = []
try:
carriers = carrier_future.result()
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
logger.error(f"Carrier tracker error (non-fatal): {e}")
carriers = []
try:
ais_vessels = ais_future.result()
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
logger.error(f"AIS stream error (non-fatal): {e}")
ais_vessels = []
ships = list(carriers or [])
ships.extend(ais_vessels or [])
# Enrich ships with yacht alert data (tracked superyachts)
from services.fetchers.yacht_alert import enrich_with_yacht_alert
@@ -200,52 +207,177 @@ def update_liveuamap():
# ---------------------------------------------------------------------------
# Fishing Activity (Global Fishing Watch)
# ---------------------------------------------------------------------------
def _fishing_vessel_key(event: dict) -> str:
vessel_ssvid = str(event.get("vessel_ssvid", "") or "").strip()
if vessel_ssvid:
return f"ssvid:{vessel_ssvid}"
vessel_id = str(event.get("vessel_id", "") or "").strip()
if vessel_id:
return f"vid:{vessel_id}"
vessel_name = str(event.get("vessel_name", "") or "").strip().upper()
vessel_flag = str(event.get("vessel_flag", "") or "").strip().upper()
if vessel_name:
return f"name:{vessel_name}|flag:{vessel_flag}"
return f"event:{event.get('id', '')}"
def _fishing_event_rank(event: dict) -> tuple[str, str, float, str]:
return (
str(event.get("end", "") or ""),
str(event.get("start", "") or ""),
float(event.get("duration_hrs", 0) or 0),
str(event.get("id", "") or ""),
)
def _dedupe_fishing_events(events: list[dict]) -> list[dict]:
latest_by_vessel: dict[str, dict] = {}
counts_by_vessel: dict[str, int] = {}
for event in events:
vessel_key = _fishing_vessel_key(event)
counts_by_vessel[vessel_key] = counts_by_vessel.get(vessel_key, 0) + 1
current = latest_by_vessel.get(vessel_key)
if current is None or _fishing_event_rank(event) > _fishing_event_rank(current):
latest_by_vessel[vessel_key] = event
deduped: list[dict] = []
for vessel_key, event in latest_by_vessel.items():
event_copy = dict(event)
event_copy["event_count"] = counts_by_vessel.get(vessel_key, 1)
deduped.append(event_copy)
deduped.sort(key=_fishing_event_rank, reverse=True)
return deduped
_FISHING_FETCH_INTERVAL_S = 3600 # once per hour — GFW data has ~5 day lag
_last_fishing_fetch_ts: float = 0.0
@with_retry(max_retries=1, base_delay=5)
def fetch_fishing_activity():
"""Fetch recent fishing events from Global Fishing Watch (~5 day lag)."""
from services.fetchers._store import is_any_active
global _last_fishing_fetch_ts
from services.fetchers._store import is_any_active, latest_data
if not is_any_active("fishing_activity"):
return
# Skip if we already have data and fetched less than an hour ago
now = time.time()
if latest_data.get("fishing_activity") and (now - _last_fishing_fetch_ts) < _FISHING_FETCH_INTERVAL_S:
return
token = os.environ.get("GFW_API_TOKEN", "")
if not token:
logger.debug("GFW_API_TOKEN not set, skipping fishing activity fetch")
return
events = []
try:
url = (
"https://gateway.api.globalfishingwatch.org/v3/events"
"?datasets[0]=public-global-fishing-events:latest"
"&limit=500&sort=start&sort-direction=DESC"
)
import datetime as _dt
_end = _dt.date.today().isoformat()
_start = (_dt.date.today() - _dt.timedelta(days=7)).isoformat()
page_size = max(1, int(os.environ.get("GFW_EVENTS_PAGE_SIZE", "500") or "500"))
offset = 0
seen_offsets: set[int] = set()
seen_ids: set[str] = set()
headers = {"Authorization": f"Bearer {token}"}
response = fetch_with_curl(url, timeout=30, headers=headers)
if response.status_code == 200:
entries = response.json().get("entries", [])
while True:
if offset in seen_offsets:
logger.warning("Fishing activity pagination repeated offset=%s; stopping fetch", offset)
break
seen_offsets.add(offset)
query = urlencode(
{
"datasets[0]": "public-global-fishing-events:latest",
"start-date": _start,
"end-date": _end,
"limit": page_size,
"offset": offset,
}
)
url = f"https://gateway.api.globalfishingwatch.org/v3/events?{query}"
response = fetch_with_curl(url, timeout=30, headers=headers)
if response.status_code != 200:
logger.warning(
"Fishing activity fetch failed at offset=%s: HTTP %s",
offset,
response.status_code,
)
break
payload = response.json() or {}
entries = payload.get("entries", [])
if not entries:
break
added_this_page = 0
for e in entries:
pos = e.get("position", {})
vessel = e.get("vessel") or {}
lat = pos.get("lat")
lng = pos.get("lon")
if lat is None or lng is None:
continue
event_id = str(e.get("id", "") or "")
if event_id and event_id in seen_ids:
continue
if event_id:
seen_ids.add(event_id)
dur = e.get("event", {}).get("duration", 0) or 0
events.append(
{
"id": e.get("id", ""),
"id": event_id,
"type": e.get("type", "fishing"),
"lat": lat,
"lng": lng,
"start": e.get("start", ""),
"end": e.get("end", ""),
"vessel_name": (e.get("vessel") or {}).get("name", "Unknown"),
"vessel_flag": (e.get("vessel") or {}).get("flag", ""),
"vessel_id": str(vessel.get("id", "") or ""),
"vessel_ssvid": str(vessel.get("ssvid", "") or ""),
"vessel_name": vessel.get("name", "Unknown"),
"vessel_flag": vessel.get("flag", ""),
"duration_hrs": round(dur / 3600, 1),
}
)
logger.info(f"Fishing activity: {len(events)} events")
added_this_page += 1
if len(entries) < page_size:
break
next_offset = payload.get("nextOffset")
if next_offset is None:
next_offset = (payload.get("pagination") or {}).get("nextOffset")
if next_offset is None:
next_offset = offset + page_size
try:
next_offset = int(next_offset)
except (TypeError, ValueError):
next_offset = offset + page_size
if next_offset <= offset:
logger.warning(
"Fishing activity pagination produced non-increasing next offset=%s; stopping fetch",
next_offset,
)
break
if added_this_page == 0:
logger.warning(
"Fishing activity page at offset=%s added no new events; stopping fetch",
offset,
)
break
offset = next_offset
raw_event_count = len(events)
events = _dedupe_fishing_events(events)
logger.info("Fishing activity: %s raw events -> %s deduped vessels", raw_event_count, len(events))
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
logger.error(f"Error fetching fishing activity: {e}")
with _data_lock:
latest_data["fishing_activity"] = events
if events:
_mark_fresh("fishing_activity")
_last_fishing_fetch_ts = time.time()
+46 -2
View File
@@ -25,7 +25,10 @@ logger = logging.getLogger("services.data_fetcher")
_API_URL = "https://meshtastic.liamcottle.net/api/v1/nodes"
_CACHE_FILE = Path(__file__).resolve().parent.parent.parent / "data" / "meshtastic_nodes_cache.json"
_FETCH_TIMEOUT = 90 # seconds — response is ~37MB, needs time on slow connections
_MAX_AGE_HOURS = 4 # discard nodes not seen within this window (matches refresh interval)
_MAX_AGE_HOURS = 24 # discard nodes not seen within this window
# Skip network fetch if cached data is fresher than this — the API is a
# one-person hobby service, so we prefer stale data over hammering it.
_CACHE_TRUST_HOURS = 20
# Track when we last fetched so the frontend can show staleness
_last_fetch_ts: float = 0.0
@@ -141,13 +144,54 @@ def fetch_meshtastic_nodes():
return
global _last_fetch_ts
# Trust a recent cache on disk — avoids hammering the upstream HTTP API
# when every install polls on roughly the same cadence.
try:
if _CACHE_FILE.exists():
mtime = _CACHE_FILE.stat().st_mtime
if time.time() - mtime < _CACHE_TRUST_HOURS * 3600:
# If memory is empty (cold start), hydrate from cache and skip fetch.
with _data_lock:
has_memory = bool(latest_data.get("meshtastic_map_nodes"))
if not has_memory:
cached = _load_cache()
if cached:
with _data_lock:
latest_data["meshtastic_map_nodes"] = cached
latest_data["meshtastic_map_fetched_at"] = mtime
_mark_fresh("meshtastic_map")
logger.info(
"Meshtastic map: cache fresh (<%.0fh), skipping network fetch",
_CACHE_TRUST_HOURS,
)
return
else:
logger.info(
"Meshtastic map: cache fresh (<%.0fh), skipping network fetch",
_CACHE_TRUST_HOURS,
)
return
except Exception as e:
logger.debug(f"Meshtastic cache freshness check failed: {e}")
# Build a polite User-Agent. Include the operator callsign when set so
# the upstream service can correlate per-install traffic if needed.
try:
from services.config import get_settings
callsign = str(getattr(get_settings(), "MESHTASTIC_OPERATOR_CALLSIGN", "") or "").strip()
except Exception:
callsign = ""
ua_base = "ShadowBroker-OSINT/0.9.7 (+https://github.com/BigBodyCobain/Shadowbroker; contact: bigbodycobain@gmail.com; 24h polling)"
user_agent = f"{ua_base}; node={callsign}" if callsign else ua_base
try:
logger.info("Fetching Meshtastic map nodes from API...")
resp = requests.get(
_API_URL,
timeout=_FETCH_TIMEOUT,
headers={
"User-Agent": "ShadowBroker/1.0 (OSINT dashboard, 4h polling)",
"User-Agent": user_agent,
"Accept": "application/json",
},
)
+16 -4
View File
@@ -2,6 +2,7 @@
import json
import logging
import time
import requests
from services.network_utils import fetch_with_curl
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
@@ -296,17 +297,23 @@ def fetch_military_flights():
with _data_lock:
latest_data["military_flights"] = remaining_mil
# Store tracked military flights — update positions for existing entries
# Store tracked military flights — update positions for existing entries.
# Drop stale entries not refreshed by ANY source (civilian or military) within 5 min.
_TRACKED_STALE_S = 300 # 5 minutes
_merge_ts = time.time()
with _data_lock:
existing_tracked = list(latest_data.get("tracked_flights", []))
fresh_mil_map = {}
for t in tracked_mil:
icao = t.get("icao24", "").upper()
if icao:
t["_seen_at"] = _merge_ts
fresh_mil_map[icao] = t
updated_tracked = []
seen_icaos = set()
stale_dropped = 0
for old_t in existing_tracked:
icao = old_t.get("icao24", "").upper()
if icao in fresh_mil_map:
@@ -317,11 +324,16 @@ def fetch_military_flights():
updated_tracked.append(fresh)
seen_icaos.add(icao)
else:
updated_tracked.append(old_t)
seen_icaos.add(icao)
# Keep stale entry only if it was seen recently
age = _merge_ts - old_t.get("_seen_at", 0)
if age < _TRACKED_STALE_S:
updated_tracked.append(old_t)
seen_icaos.add(icao)
else:
stale_dropped += 1
for icao, t in fresh_mil_map.items():
if icao not in seen_icaos:
updated_tracked.append(t)
with _data_lock:
latest_data["tracked_flights"] = updated_tracked
logger.info(f"Tracked flights: {len(updated_tracked)} total ({len(tracked_mil)} from military)")
logger.info(f"Tracked flights: {len(updated_tracked)} total ({len(tracked_mil)} from military, {stale_dropped} stale dropped)")
+17
View File
@@ -1,6 +1,8 @@
"""News fetching, geocoding, clustering, and risk assessment."""
import re
import time
import logging
import calendar
import concurrent.futures
import requests
import feedparser
@@ -11,6 +13,10 @@ from services.oracle_service import enrich_news_items, compute_global_threat_lev
logger = logging.getLogger("services.data_fetcher")
# Maximum article age in seconds. Anything older than this is dropped
# during each fetch cycle so the threat feed stays current.
_MAX_ARTICLE_AGE_SECS = 48 * 3600 # 48 hours
# Keyword -> coordinate mapping for geocoding news articles
_KEYWORD_COORDS = {
@@ -178,6 +184,17 @@ def fetch_news():
if not feed:
continue
for entry in feed.entries[:5]:
# Drop articles older than the max-age threshold so the
# threat feed doesn't show stale stories across cycles.
pp = entry.get("published_parsed")
if pp:
try:
entry_epoch = calendar.timegm(pp)
if time.time() - entry_epoch > _MAX_ARTICLE_AGE_SECS:
continue
except (TypeError, ValueError, OverflowError):
pass # unparseable date — keep the article
title = entry.get('title', '')
summary = entry.get('summary', '')
@@ -0,0 +1,360 @@
"""NUFORC Enrichment — downloads the Hugging Face NUFORC dataset and builds
a compact spatial+temporal index for enriching tilequery hits with shape,
duration, city, and summary text.
The full CSV (~170 MB) is streamed once and processed into a lightweight JSON
cache (~1-3 MB) stored at ``backend/data/nuforc_enrichment.json``. Subsequent
startups load from cache until it expires (30 days).
Index structure::
{
"built": "2026-04-08T12:00:00",
"count": 12345,
"by_state": {
"AZ": [
{"d": "2024-01-15", "city": "Tucson", "shape": "triangle",
"dur": "5 minutes", "summary": "Bright triangular object..."},
...
],
...
}
}
Entries within each state are sorted by date descending (newest first).
"""
import csv
import gzip
import io
import json
import logging
import os
import re
import threading
import time
from datetime import datetime, timedelta
from pathlib import Path
from services.network_utils import fetch_with_curl
logger = logging.getLogger(__name__)
_DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"
_CACHE_FILE = _DATA_DIR / "nuforc_enrichment.json"
_CACHE_TTL_DAYS = 1 # Rebuild daily — fresh data each cycle
# HuggingFace dataset — use the structured string export, not the old flat blob.
_HF_CSV_URL = (
"https://huggingface.co/datasets/kcimc/NUFORC/resolve/main/nuforc_str.csv"
)
# Only keep sightings from the last N years for the enrichment index
_KEEP_YEARS = 5
# ── In-memory index ────────────────────────────────────────────────────────
_index: dict | None = None
_index_lock = threading.Lock()
_building = False
# US state abbreviations for parsing "City, ST" locations
_US_STATES = {
"AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DE", "FL", "GA",
"HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD",
"MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ",
"NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", "RI", "SC",
"SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY",
"DC",
}
def _parse_location(loc: str) -> tuple[str, str]:
"""Parse 'City, ST' or 'City, ST (explanation)' → (city, state_abbr).
Returns ('', '') if unparseable.
"""
if not loc:
return "", ""
loc = re.sub(r"\s*\(.*\)\s*$", "", loc).strip()
parts = [p.strip() for p in loc.split(",") if p.strip()]
if len(parts) < 2:
return "", ""
for idx in range(len(parts) - 1):
candidate = parts[idx + 1].upper().strip()
if candidate in _US_STATES:
city = ", ".join(parts[: idx + 1]).strip()
return city, candidate
candidate = parts[-1].upper().strip()
if candidate in _US_STATES:
return ", ".join(parts[:-1]).strip(), candidate
return parts[0], ""
def _parse_date(date_str: str) -> str:
"""Best-effort parse NUFORC date strings → 'YYYY-MM-DD'.
Returns '' on failure.
"""
if not date_str:
return ""
cleaned = str(date_str).strip()
cleaned = re.sub(r"\s+local$", "", cleaned, flags=re.IGNORECASE)
cleaned = re.sub(r"\s+utc$", "", cleaned, flags=re.IGNORECASE)
cleaned = cleaned.replace("T", " ")
for fmt in (
"%m/%d/%Y %H:%M",
"%m/%d/%Y %I:%M:%S %p",
"%m/%d/%Y",
"%Y-%m-%d %H:%M:%S",
"%Y-%m-%d %H:%M",
"%Y-%m-%d",
):
try:
return datetime.strptime(cleaned, fmt).strftime("%Y-%m-%d")
except (ValueError, TypeError):
continue
match = re.match(r"^(\d{4}-\d{2}-\d{2})", cleaned)
if match:
return match.group(1)
return ""
def _load_cache() -> dict | None:
"""Load the on-disk cache if it exists and is fresh enough."""
if not _CACHE_FILE.exists():
return None
try:
raw = _CACHE_FILE.read_text(encoding="utf-8")
data = json.loads(raw)
built = data.get("built", "")
if built:
built_dt = datetime.fromisoformat(built)
if datetime.utcnow() - built_dt < timedelta(days=_CACHE_TTL_DAYS):
if int(data.get("count", 0) or 0) <= 0:
logger.info("NUFORC enrichment: cache is fresh but empty; rebuilding")
return None
logger.info(
"NUFORC enrichment: loaded cache (%d entries, built %s)",
data.get("count", 0), built,
)
return data
else:
logger.info("NUFORC enrichment: cache expired (built %s)", built)
except Exception as e:
logger.warning("NUFORC enrichment: cache load error: %s", e)
return None
def _save_cache(data: dict):
"""Persist the enrichment index to disk."""
try:
_DATA_DIR.mkdir(parents=True, exist_ok=True)
_CACHE_FILE.write_text(json.dumps(data, separators=(",", ":")), encoding="utf-8")
logger.info("NUFORC enrichment: saved cache (%d entries)", data.get("count", 0))
except Exception as e:
logger.warning("NUFORC enrichment: cache save error: %s", e)
def _download_and_build() -> dict | None:
"""Stream-download the HF CSV and build the enrichment index.
Returns the index dict or None on failure.
"""
cutoff = datetime.utcnow() - timedelta(days=_KEEP_YEARS * 365)
cutoff_str = cutoff.strftime("%Y-%m-%d")
logger.info("NUFORC enrichment: downloading HF dataset (this may take a minute)...")
try:
resp = fetch_with_curl(_HF_CSV_URL, timeout=180, follow_redirects=True)
if not resp or resp.status_code != 200:
logger.warning(
"NUFORC enrichment: download failed HTTP %s",
getattr(resp, "status_code", "None"),
)
return None
except Exception as e:
logger.error("NUFORC enrichment: download error: %s", e)
return None
# Parse CSV from response text
by_state: dict[str, list[dict]] = {}
total = 0
kept = 0
try:
reader = csv.DictReader(io.StringIO(resp.text))
for row in reader:
total += 1
occurred = _parse_date(
row.get("Occurred", "")
or row.get("Date / Time", "")
or row.get("Date", "")
)
if not occurred or occurred < cutoff_str:
continue
city, state = _parse_location(
row.get("Location", "")
or row.get("City", "")
or row.get("location", "")
)
if not state:
continue # can't index without state
shape = (row.get("Shape", "") or row.get("shape", "") or "").strip()
duration = (row.get("Duration", "") or row.get("duration", "") or "").strip()
summary = (
row.get("Summary", "")
or row.get("summary", "")
or row.get("Text", "")
or row.get("text", "")
or ""
).strip()
if summary and len(summary) > 200:
summary = summary[:197] + "..."
entry = {"d": occurred, "city": city, "shape": shape}
if duration:
entry["dur"] = duration
if summary:
entry["sum"] = summary
by_state.setdefault(state, []).append(entry)
kept += 1
except Exception as e:
logger.error("NUFORC enrichment: CSV parse error: %s", e)
return None
# Sort each state's entries by date descending (newest first)
for st in by_state:
by_state[st].sort(key=lambda e: e["d"], reverse=True)
data = {
"built": datetime.utcnow().isoformat(),
"count": kept,
"by_state": by_state,
}
logger.info(
"NUFORC enrichment: built index — %d entries from %d total rows (%d states)",
kept, total, len(by_state),
)
return data
def _ensure_index():
"""Load or build the enrichment index (thread-safe, non-blocking)."""
global _index, _building
with _index_lock:
if _index is not None:
return
if _building:
return # another thread is already building
_building = True
# Try loading from disk first
cached = _load_cache()
if cached:
with _index_lock:
_index = cached
_building = False
return
# Download and build in background so we don't block startup
def _build():
global _index, _building
try:
result = _download_and_build()
if result:
_save_cache(result)
with _index_lock:
_index = result
else:
logger.warning("NUFORC enrichment: build failed, enrichment unavailable")
finally:
with _index_lock:
_building = False
thread = threading.Thread(target=_build, name="nuforc-enrichment", daemon=True)
thread.start()
def refresh_enrichment_index():
"""Force-rebuild the enrichment index. Called by the daily cron job.
Downloads the latest HF CSV, rebuilds the in-memory + disk cache.
Runs synchronously (meant to be called from a background thread).
"""
global _index
logger.info("NUFORC enrichment: daily refresh starting...")
result = _download_and_build()
if result:
_save_cache(result)
with _index_lock:
_index = result
logger.info("NUFORC enrichment: daily refresh complete (%d entries)", result.get("count", 0))
else:
logger.warning("NUFORC enrichment: daily refresh failed, keeping stale index")
def enrich_sighting(state: str, from_date: str, to_date: str) -> dict:
"""Look up enrichment data for a tilequery hit.
Args:
state: 2-letter US state code (from reverse geocode)
from_date: earliest sighting date (YYYY-MM-DD)
to_date: latest sighting date (YYYY-MM-DD)
Returns:
Dict with optional keys: city, shape, duration, summary.
Empty dict if no match found.
"""
_ensure_index()
with _index_lock:
idx = _index
if not idx or not state:
return {}
entries = idx.get("by_state", {}).get(state, [])
if not entries:
return {}
# Find the best match by date proximity
target = to_date or from_date
if not target:
# No date filter — just return the most recent entry for this state
e = entries[0]
else:
best = None
best_dist = 999999
for e in entries:
# Simple string distance on dates (YYYY-MM-DD sorts lexicographically)
try:
t = datetime.strptime(target, "%Y-%m-%d")
d = datetime.strptime(e["d"], "%Y-%m-%d")
dist = abs((t - d).days)
except (ValueError, TypeError):
continue
if dist < best_dist:
best_dist = dist
best = e
if dist == 0:
break # exact date match
if best is None or best_dist > 90:
return {} # no match within 3 months
e = best
result = {}
if e.get("city"):
result["city"] = e["city"]
if e.get("shape"):
result["shape"] = e["shape"]
result["shape_raw"] = e["shape"]
if e.get("dur"):
result["duration"] = e["dur"]
if e.get("sum"):
result["summary"] = e["sum"]
return result
+464 -16
View File
@@ -8,14 +8,33 @@ full metadata (volume, end dates, descriptions, source badges).
import json
import logging
import math
import os
import threading
import time
from urllib.parse import urlencode
from cachetools import TTLCache, cached
logger = logging.getLogger("services.data_fetcher")
_market_cache = TTLCache(maxsize=1, ttl=60) # 60-second TTL — markets change fast
# Delta tracking: {market_title: previous_consensus_pct}
_prev_probabilities: dict[str, float] = {}
_market_cache = TTLCache(maxsize=1, ttl=300)
_POLYMARKET_PAGE_DELAY_S = float(os.environ.get("MESH_POLYMARKET_PAGE_DELAY_S", "0.02"))
_KALSHI_PAGE_DELAY_S = float(os.environ.get("MESH_KALSHI_PAGE_DELAY_S", "0.08"))
_provider_pace_lock = threading.Lock()
_provider_last_request_at: dict[str, float] = {}
def _pace_provider(provider: str, min_interval_s: float) -> None:
if min_interval_s <= 0:
return
with _provider_pace_lock:
now = time.monotonic()
wait_s = min_interval_s - (now - _provider_last_request_at.get(provider, 0.0))
if wait_s > 0:
time.sleep(wait_s)
now = time.monotonic()
_provider_last_request_at[provider] = now
def _finite_or_none(value):
@@ -28,7 +47,7 @@ def _finite_or_none(value):
# ---------------------------------------------------------------------------
# Category classification
# ---------------------------------------------------------------------------
CATEGORIES = ["POLITICS", "CONFLICT", "NEWS", "FINANCE", "CRYPTO"]
CATEGORIES = ["POLITICS", "CONFLICT", "NEWS", "FINANCE", "CRYPTO", "SPORTS"]
_KALSHI_CATEGORY_MAP = {
"Politics": "POLITICS",
@@ -38,7 +57,7 @@ _KALSHI_CATEGORY_MAP = {
"Tech": "FINANCE",
"Science": "NEWS",
"Climate and Weather": "NEWS",
"Sports": "NEWS",
"Sports": "SPORTS",
"Culture": "NEWS",
}
@@ -62,7 +81,14 @@ _TAG_CATEGORY_MAP = {
"Ethereum": "CRYPTO",
"AI": "NEWS",
"Science": "NEWS",
"Sports": "NEWS",
"Sports": "SPORTS",
"NBA": "SPORTS",
"NFL": "SPORTS",
"MLB": "SPORTS",
"NHL": "SPORTS",
"Soccer": "SPORTS",
"Tennis": "SPORTS",
"Golf": "SPORTS",
"Culture": "NEWS",
"Entertainment": "NEWS",
"Tech": "FINANCE",
@@ -152,6 +178,26 @@ _KEYWORD_CATEGORIES = {
"market cap",
"revenue",
],
"SPORTS": [
"nba",
"nfl",
"mlb",
"nhl",
"wnba",
"soccer",
"football",
"basketball",
"baseball",
"hockey",
"ufc",
"mma",
"tennis",
"golf",
"championship",
"playoffs",
"world cup",
"super bowl",
],
}
@@ -177,21 +223,186 @@ def _classify_category(title: str, poly_tags: list[str], kalshi_category: str) -
return "NEWS"
def _polymarket_event_to_entry(ev: dict) -> dict | None:
title = ev.get("title", "")
if not title:
return None
markets = ev.get("markets", [])
best_pct = None
total_volume = 0
outcomes = []
for m in markets:
raw_op = m.get("outcomePrices")
price = None
try:
op = json.loads(raw_op) if isinstance(raw_op, str) else raw_op
if isinstance(op, list) and len(op) >= 1:
price = _finite_or_none(op[0])
except (json.JSONDecodeError, ValueError, TypeError):
pass
if price is None:
price = _finite_or_none(m.get("lastTradePrice") or m.get("bestBid"))
pct = None
if price is not None:
try:
pct = round(price * 100, 1)
if best_pct is None or pct > best_pct:
best_pct = pct
except (ValueError, TypeError):
pass
volume = _finite_or_none(m.get("volume", 0) or 0)
if volume is not None:
total_volume += volume
oname = m.get("groupItemTitle") or ""
if oname and pct is not None:
outcomes.append({"name": oname, "pct": pct})
if len(outcomes) > 2:
outcomes.sort(key=lambda x: x["pct"], reverse=True)
else:
outcomes = []
tag_labels = [t.get("label", "") for t in ev.get("tags", []) if t.get("label")]
return {
"title": title,
"source": "polymarket",
"pct": best_pct,
"slug": ev.get("slug", ""),
"description": ev.get("description") or "",
"end_date": ev.get("endDate"),
"volume": round(total_volume, 2),
"volume_24h": round(_finite_or_none(ev.get("volume24hr", 0) or 0) or 0, 2),
"tags": tag_labels,
"outcomes": outcomes,
}
def _kalshi_market_pct(m: dict) -> float | None:
bid = _finite_or_none(m.get("yes_bid_dollars"))
ask = _finite_or_none(m.get("yes_ask_dollars"))
last = _finite_or_none(m.get("last_price_dollars"))
if bid is not None and ask is not None and ask >= bid:
return round(((bid + ask) / 2) * 100, 1)
if last is not None:
return round(last * 100, 1)
cents = _finite_or_none(m.get("yes_price") or m.get("last_price"))
if cents is None:
return None
return round(cents * 100, 1) if cents <= 1 else round(cents, 1)
def _kalshi_market_volume(m: dict) -> float:
for key in ("volume_24h_fp", "volume_fp", "dollar_volume", "volume"):
value = _finite_or_none(m.get(key))
if value is not None:
return value
return 0
def _kalshi_market_category(m: dict) -> str:
text = " ".join(
str(m.get(k, "") or "")
for k in ("ticker", "event_ticker", "mve_collection_ticker", "title", "yes_sub_title", "no_sub_title")
).lower()
if any(token in text for token in ("sports", "xnba", "xnfl", "xmlb", "xnhl", "soccer", "tennis", "golf")):
return "Sports"
return str(m.get("category", "") or "")
def _kalshi_event_to_entry(ev: dict, markets: list[dict] | None = None) -> dict | None:
title = ev.get("title", "")
if not title:
return None
markets = markets or ev.get("markets", []) or []
best_pct = None
total_volume = 0.0
close_dates = []
outcomes = []
first_ticker = ""
descriptions = []
for m in markets:
first_ticker = first_ticker or m.get("ticker", "")
pct = _kalshi_market_pct(m)
if pct is not None:
if best_pct is None or pct > best_pct:
best_pct = pct
oname = m.get("yes_sub_title") or m.get("sub_title") or m.get("title") or ""
if oname and oname != title:
outcomes.append({"name": oname, "pct": pct})
total_volume += _kalshi_market_volume(m)
cd = m.get("close_time") or m.get("close_date") or m.get("expiration_time")
if cd:
close_dates.append(cd)
desc = (m.get("rules_primary") or m.get("rules_secondary") or "").strip()
if desc:
descriptions.append(desc)
if len(outcomes) > 2:
outcomes.sort(key=lambda x: x["pct"], reverse=True)
else:
outcomes = []
desc = (ev.get("settle_details") or ev.get("underlying") or "").strip()
if not desc and descriptions:
desc = descriptions[0]
return {
"title": title,
"source": "kalshi",
"pct": best_pct,
"ticker": first_ticker or ev.get("event_ticker", "") or ev.get("ticker", ""),
"description": desc,
"sub_title": ev.get("sub_title", ""),
"end_date": max(close_dates) if close_dates else None,
"volume": round(total_volume, 2),
"category": ev.get("category", ""),
"outcomes": outcomes,
}
def _kalshi_market_to_entry(m: dict) -> dict | None:
title = m.get("title") or m.get("yes_sub_title") or ""
if not title:
return None
pct = _kalshi_market_pct(m)
volume = _kalshi_market_volume(m)
desc = (m.get("rules_primary") or m.get("rules_secondary") or "").strip()
end_date = m.get("close_time") or m.get("expiration_time") or m.get("expected_expiration_time")
return {
"title": title,
"source": "kalshi",
"pct": pct,
"ticker": m.get("ticker", "") or m.get("event_ticker", ""),
"description": desc,
"sub_title": m.get("subtitle", ""),
"end_date": end_date,
"volume": round(volume, 2),
"category": _kalshi_market_category(m),
"outcomes": [],
}
# ---------------------------------------------------------------------------
# Polymarket
# ---------------------------------------------------------------------------
def _fetch_polymarket_events() -> list[dict]:
"""Fetch active events from Polymarket Gamma API (no auth required).
Fetches up to 500 events (multiple pages) for better search coverage.
Fetches paginated active events, bounded by MESH_POLYMARKET_MAX_EVENTS
so boot-time refresh does not become unbounded.
"""
from services.network_utils import fetch_with_curl
all_events = []
for offset in range(0, 500, 100):
page_size = 250
max_events = int(os.environ.get("MESH_POLYMARKET_MAX_EVENTS", "5000"))
for offset in range(0, max_events, page_size):
try:
_pace_provider("polymarket", _POLYMARKET_PAGE_DELAY_S)
resp = fetch_with_curl(
f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=100&offset={offset}",
f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit={page_size}&offset={offset}",
timeout=15,
)
if not resp or resp.status_code != 200:
@@ -200,6 +411,8 @@ def _fetch_polymarket_events() -> list[dict]:
if not isinstance(page, list) or not page:
break
all_events.extend(page)
if len(page) < page_size:
break
except Exception as e:
logger.warning(f"Polymarket page offset={offset} error: {e}")
break
@@ -286,6 +499,42 @@ def _fetch_kalshi_events() -> list[dict]:
"""Fetch active events from Kalshi public API (no auth required)."""
from services.network_utils import fetch_with_curl
try:
max_events = int(os.environ.get("MESH_KALSHI_MAX_EVENTS", "2000"))
page_size = 200
markets = []
cursor = ""
while len(markets) < max_events:
params = {"status": "open", "limit": str(page_size)}
if cursor:
params["cursor"] = cursor
_pace_provider("kalshi", _KALSHI_PAGE_DELAY_S)
resp = fetch_with_curl(
f"https://api.elections.kalshi.com/trade-api/v2/markets?{urlencode(params)}",
timeout=15,
)
if not resp or resp.status_code != 200:
break
data = resp.json()
page = data.get("markets", []) if isinstance(data, dict) else []
if not page:
break
markets.extend(page)
cursor = data.get("cursor") or ""
if not cursor or len(page) < page_size:
break
results = []
for market in markets:
entry = _kalshi_market_to_entry(market)
if entry:
results.append(entry)
if results:
logger.info(f"Kalshi: fetched {len(results)} active events from v2")
return results
except Exception as e:
logger.warning(f"Kalshi v2 fetch error, falling back to legacy v1: {e}")
try:
resp = fetch_with_curl(
"https://api.elections.kalshi.com/v1/events?status=open&limit=100",
@@ -540,11 +789,11 @@ def fetch_prediction_markets():
# ---------------------------------------------------------------------------
# Direct API search (not limited to cached data)
# ---------------------------------------------------------------------------
def search_polymarket_direct(query: str, limit: int = 20) -> list[dict]:
def search_polymarket_direct(query: str, limit: int = 20, offset: int = 0) -> list[dict]:
"""Search Polymarket by scanning API pages for title matches.
The Gamma API has no text search parameter, so we scan cached events
plus additional pages until we find enough matches or exhaust the scan.
Prefer Polymarket's public search endpoint, then fall back to scanning
Gamma event pages if search is unavailable.
"""
from services.network_utils import fetch_with_curl
@@ -552,11 +801,53 @@ def search_polymarket_direct(query: str, limit: int = 20) -> list[dict]:
q_words = set(q_lower.split())
results = []
try:
params = urlencode({"q": query, "limit": str(limit), "offset": str(max(0, offset))})
_pace_provider("polymarket", _POLYMARKET_PAGE_DELAY_S)
resp = fetch_with_curl(
f"https://gamma-api.polymarket.com/public-search?{params}",
timeout=15,
)
if resp and resp.status_code == 200:
data = resp.json()
events = data.get("events", []) if isinstance(data, dict) else []
for ev in events:
if ev.get("closed") or ev.get("active") is False:
continue
entry = _polymarket_event_to_entry(ev)
if not entry:
continue
category = _classify_category(entry["title"], entry.get("tags", []), "")
pct = _finite_or_none(entry.get("pct"))
sources = [{"name": "POLY", "pct": pct}] if pct is not None else []
results.append(
{
"title": entry["title"],
"polymarket_pct": pct,
"kalshi_pct": None,
"consensus_pct": pct,
"description": entry.get("description", ""),
"end_date": entry.get("end_date"),
"volume": entry.get("volume", 0),
"volume_24h": entry.get("volume_24h", 0),
"kalshi_volume": 0,
"category": category,
"sources": sources,
"slug": entry.get("slug", ""),
"outcomes": entry.get("outcomes", []),
}
)
logger.info(f"Polymarket search '{query}': {len(results)} results via public-search")
return results[:limit]
except Exception as e:
logger.warning(f"Polymarket public-search '{query}' error: {e}")
# Scan up to 2000 events (10 pages of 200) looking for title matches
for offset in range(0, 2000, 200):
for scan_offset in range(0, 3000, 200):
try:
_pace_provider("polymarket", _POLYMARKET_PAGE_DELAY_S)
resp = fetch_with_curl(
f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=200&offset={offset}",
f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=200&offset={scan_offset}",
timeout=15,
)
if not resp or resp.status_code != 200:
@@ -637,11 +928,168 @@ def search_polymarket_direct(query: str, limit: int = 20) -> list[dict]:
}
)
# Stop scanning if we have enough results
if len(results) >= limit:
if len(results) >= offset + limit:
break
except Exception as e:
logger.warning(f"Polymarket search scan offset={offset} error: {e}")
logger.warning(f"Polymarket search scan offset={scan_offset} error: {e}")
break
logger.info(f"Polymarket search '{query}': {len(results)} results (scanned API)")
return results[:limit]
return results[offset : offset + limit]
def search_kalshi_direct(query: str, limit: int = 20, offset: int = 0) -> list[dict]:
"""Search Kalshi events by scanning API pages for title matches."""
from services.network_utils import fetch_with_curl
q_lower = query.lower()
q_words = set(q_lower.split())
results = []
try:
max_scan = int(os.environ.get("MESH_KALSHI_SEARCH_SCAN_EVENTS", "1200"))
page_size = 200
cursor = ""
scanned = 0
while scanned < max_scan and len(results) < offset + limit:
params = {"status": "open", "limit": str(page_size)}
if cursor:
params["cursor"] = cursor
_pace_provider("kalshi", _KALSHI_PAGE_DELAY_S)
resp = fetch_with_curl(
f"https://api.elections.kalshi.com/trade-api/v2/markets?{urlencode(params)}",
timeout=15,
)
if not resp or resp.status_code != 200:
break
data = resp.json()
markets = data.get("markets", []) if isinstance(data, dict) else []
if not markets:
break
scanned += len(markets)
for market in markets:
haystack = " ".join(
str(market.get(k, "") or "")
for k in ("title", "yes_sub_title", "no_sub_title", "event_ticker", "ticker")
).lower()
if q_lower not in haystack and not any(w in haystack for w in q_words):
continue
entry = _kalshi_market_to_entry(market)
if not entry:
continue
pct = _finite_or_none(entry.get("pct"))
sources = [{"name": "KALSHI", "pct": pct}] if pct is not None else []
category = _classify_category(entry["title"], [], entry.get("category", ""))
results.append({
"title": entry["title"],
"polymarket_pct": None,
"kalshi_pct": pct,
"consensus_pct": pct,
"description": entry.get("description", ""),
"end_date": entry.get("end_date"),
"volume": 0,
"volume_24h": 0,
"kalshi_volume": entry.get("volume", 0),
"category": category,
"sources": sources,
"slug": "",
"kalshi_ticker": entry.get("ticker", ""),
"outcomes": entry.get("outcomes", []),
})
if len(results) >= offset + limit:
break
cursor = data.get("cursor") or ""
if not cursor or len(markets) < page_size:
break
if results:
logger.info(f"Kalshi search '{query}': {len(results)} results via v2 scan")
return results[offset : offset + limit]
except Exception as e:
logger.warning(f"Kalshi v2 search '{query}' error, falling back to legacy v1: {e}")
try:
resp = fetch_with_curl(
"https://api.elections.kalshi.com/v1/events?status=open&limit=200",
timeout=15,
)
if not resp or resp.status_code != 200:
return []
data = resp.json()
events = data.get("events", []) if isinstance(data, dict) else []
for ev in events:
title = ev.get("title", "")
if not title:
continue
title_lower = title.lower()
if q_lower not in title_lower and not any(w in title_lower for w in q_words):
continue
markets = ev.get("markets", [])
best_pct = None
total_volume = 0
close_dates = []
outcomes = []
for m in markets:
price = m.get("yes_price") or m.get("last_price")
pct = None
if price is not None:
try:
price = _finite_or_none(price)
if price is None:
raise ValueError("non-finite")
pct = round(price, 1)
if pct <= 1:
pct = round(pct * 100, 1)
if best_pct is None or pct > best_pct:
best_pct = pct
except (ValueError, TypeError):
pass
try:
volume = _finite_or_none(
m.get("dollar_volume", 0) or m.get("volume", 0) or 0
)
if volume is not None:
total_volume += int(volume)
except (ValueError, TypeError):
pass
cd = m.get("close_date")
if cd:
close_dates.append(cd)
oname = m.get("title") or m.get("subtitle", "")
if oname and pct is not None:
outcomes.append({"name": oname, "pct": pct})
if len(outcomes) > 2:
outcomes.sort(key=lambda x: x["pct"], reverse=True)
else:
outcomes = []
desc = (ev.get("settle_details") or ev.get("underlying") or "").strip()
category = _classify_category(title, [], ev.get("category", ""))
sources = []
if best_pct is not None:
sources.append({"name": "KALSHI", "pct": best_pct})
results.append({
"title": title,
"polymarket_pct": None,
"kalshi_pct": best_pct,
"consensus_pct": best_pct,
"description": desc,
"end_date": max(close_dates) if close_dates else None,
"volume": total_volume,
"volume_24h": 0,
"kalshi_volume": total_volume,
"category": category,
"sources": sources,
"slug": "",
"kalshi_ticker": ev.get("ticker", ""),
"outcomes": outcomes,
})
if len(results) >= offset + limit:
break
except Exception as e:
logger.warning(f"Kalshi search '{query}' error: {e}")
logger.info(f"Kalshi search '{query}': {len(results)} results")
return results[offset : offset + limit]
+166
View File
@@ -0,0 +1,166 @@
"""Static route + airport database loaded from vrs-standing-data.adsb.lol.
Replaces the per-batch /api/0/routeset POST with a single daily bulk download.
Routes change ~weekly when airlines update schedules, so a 24h refresh cadence
is far more than sufficient and removes ~all live-API pressure on adsb.lol.
"""
from __future__ import annotations
import csv
import gzip
import io
import logging
import threading
import time
from typing import Any
import requests
logger = logging.getLogger(__name__)
_ROUTES_URL = "https://vrs-standing-data.adsb.lol/routes.csv.gz"
_AIRPORTS_URL = "https://vrs-standing-data.adsb.lol/airports.csv.gz"
_REFRESH_INTERVAL_S = 5 * 24 * 3600
_HTTP_TIMEOUT_S = 60
_USER_AGENT = (
"ShadowBroker-OSINT/0.9.7 "
"(+https://github.com/BigBodyCobain/Shadowbroker; "
"contact: bigbodycobain@gmail.com)"
)
_lock = threading.RLock()
_routes_by_callsign: dict[str, dict[str, Any]] = {}
_airports_by_icao: dict[str, dict[str, Any]] = {}
_last_refresh = 0.0
_refresh_in_progress = False
def _fetch_csv_gz(url: str) -> list[dict[str, str]]:
response = requests.get(
url,
timeout=_HTTP_TIMEOUT_S,
headers={"User-Agent": _USER_AGENT, "Accept-Encoding": "gzip"},
)
response.raise_for_status()
text = gzip.decompress(response.content).decode("utf-8-sig")
return list(csv.DictReader(io.StringIO(text)))
def _build_route_index(rows: list[dict[str, str]]) -> dict[str, dict[str, Any]]:
index: dict[str, dict[str, Any]] = {}
for row in rows:
callsign = (row.get("Callsign") or "").strip().upper()
airport_codes = (row.get("AirportCodes") or "").strip()
if not callsign or not airport_codes:
continue
icaos = [c.strip() for c in airport_codes.split("-") if c.strip()]
if len(icaos) < 2:
continue
index[callsign] = {
"airline_code": (row.get("AirlineCode") or "").strip(),
"airport_codes": airport_codes,
"airport_icaos": icaos,
}
return index
def _build_airport_index(rows: list[dict[str, str]]) -> dict[str, dict[str, Any]]:
index: dict[str, dict[str, Any]] = {}
for row in rows:
icao = (row.get("ICAO") or "").strip().upper()
if not icao:
continue
try:
lat = float(row.get("Latitude") or 0)
lon = float(row.get("Longitude") or 0)
except (TypeError, ValueError):
continue
index[icao] = {
"name": (row.get("Name") or "").strip(),
"iata": (row.get("IATA") or "").strip(),
"country": (row.get("CountryISO2") or "").strip(),
"lat": lat,
"lon": lon,
}
return index
def refresh_route_database(force: bool = False) -> bool:
"""Pull routes.csv.gz + airports.csv.gz and rebuild the in-memory indexes.
Returns True if a refresh was performed (success or attempted), False if
skipped because the cache is still fresh or another refresh is in flight.
"""
global _last_refresh, _refresh_in_progress
now = time.time()
with _lock:
if _refresh_in_progress:
return False
if not force and (now - _last_refresh) < _REFRESH_INTERVAL_S and _routes_by_callsign:
return False
_refresh_in_progress = True
try:
started = time.time()
airport_rows = _fetch_csv_gz(_AIRPORTS_URL)
route_rows = _fetch_csv_gz(_ROUTES_URL)
airports = _build_airport_index(airport_rows)
routes = _build_route_index(route_rows)
with _lock:
_airports_by_icao.clear()
_airports_by_icao.update(airports)
_routes_by_callsign.clear()
_routes_by_callsign.update(routes)
_last_refresh = time.time()
logger.info(
"route database refreshed in %.1fs: %d routes, %d airports",
time.time() - started,
len(routes),
len(airports),
)
return True
except (requests.RequestException, OSError, ValueError) as exc:
logger.warning("route database refresh failed: %s", exc)
return True
finally:
with _lock:
_refresh_in_progress = False
def lookup_route(callsign: str) -> dict[str, Any] | None:
"""Resolve a callsign to {orig_name, dest_name, orig_loc, dest_loc} or None.
Matches the shape produced by the legacy fetch_routes_background cache so
the caller in flights.py can be a drop-in replacement.
"""
key = (callsign or "").strip().upper()
if not key:
return None
with _lock:
route = _routes_by_callsign.get(key)
if not route:
return None
icaos = route["airport_icaos"]
orig = _airports_by_icao.get(icaos[0].upper())
dest = _airports_by_icao.get(icaos[-1].upper())
if not orig or not dest:
return None
return {
"orig_name": f"{orig['iata']}: {orig['name']}" if orig["iata"] else orig["name"],
"dest_name": f"{dest['iata']}: {dest['name']}" if dest["iata"] else dest["name"],
"orig_loc": [orig["lon"], orig["lat"]],
"dest_loc": [dest["lon"], dest["lat"]],
}
def route_database_status() -> dict[str, Any]:
with _lock:
return {
"last_refresh": _last_refresh,
"routes": len(_routes_by_callsign),
"airports": len(_airports_by_icao),
"in_progress": _refresh_in_progress,
}
+74
View File
@@ -0,0 +1,74 @@
"""SAR catalog fetcher (Mode A — default-on, free, no account).
Hits ASF Search every hour for Sentinel-1 scenes that touched any of
the operator-defined AOIs in the last ~36h. Pure metadata, no
downloads.
Result is written to ``latest_data["sar_scenes"]`` and a per-AOI
coverage summary to ``latest_data["sar_aoi_coverage"]``.
"""
from __future__ import annotations
import logging
from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
from services.fetchers.retry import with_retry
from services.sar.sar_aoi import load_aois
from services.sar.sar_catalog_client import estimate_next_pass, search_scenes_for_aoi
from services.sar.sar_config import catalog_enabled
logger = logging.getLogger(__name__)
@with_retry(max_retries=1, base_delay=2)
def fetch_sar_catalog() -> None:
"""Refresh the SAR scene catalog for all configured AOIs."""
if not catalog_enabled():
return
if not is_any_active("sar"):
return
aois = load_aois()
if not aois:
logger.debug("SAR catalog: no AOIs configured")
return
all_scenes: list[dict] = []
coverage: list[dict] = []
for aoi in aois:
try:
scenes = search_scenes_for_aoi(aoi)
except (ConnectionError, TimeoutError, OSError, ValueError) as exc:
logger.debug("SAR catalog %s: %s", aoi.id, exc)
scenes = []
scene_dicts = [s.to_dict() for s in scenes]
all_scenes.extend(scene_dicts)
next_pass = estimate_next_pass(scenes)
coverage.append(
{
"aoi_id": aoi.id,
"aoi_name": aoi.name,
"category": aoi.category,
"center_lat": aoi.center_lat,
"center_lon": aoi.center_lon,
"radius_km": aoi.radius_km,
"recent_scene_count": len(scene_dicts),
"latest_scene_time": (
max((s["time"] for s in scene_dicts), default="")
if scene_dicts
else ""
),
**next_pass,
}
)
with _data_lock:
latest_data["sar_scenes"] = all_scenes
latest_data["sar_aoi_coverage"] = coverage
if all_scenes or coverage:
_mark_fresh("sar_scenes", "sar_aoi_coverage")
logger.info(
"SAR catalog: %d scenes across %d AOIs",
len(all_scenes),
len(aois),
)
+103
View File
@@ -0,0 +1,103 @@
"""SAR pre-processed product fetcher (Mode B — opt-in, free, account needed).
Pulls already-computed deformation, flood, water, and damage products
from NASA OPERA, Copernicus EGMS, GFM, EMS, and UNOSAT. No local DSP.
Two-step opt-in: ``MESH_SAR_PRODUCTS_FETCH=allow`` AND
``MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE=true``. When either flag is
unset, this fetcher logs a single startup hint and returns.
"""
from __future__ import annotations
import logging
from typing import Any
from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
from services.fetchers.retry import with_retry
from services.sar.sar_aoi import load_aois
from services.sar.sar_config import products_fetch_enabled, products_fetch_status
from services.sar.sar_normalize import SarAnomaly
from services.sar.sar_products_client import (
fetch_egms_for_aoi,
fetch_ems_for_aoi,
fetch_gfm_for_aoi,
fetch_opera_for_aoi,
fetch_unosat_for_aoi,
)
from services.sar.sar_signing import emit_signed_anomaly
logger = logging.getLogger(__name__)
_LOGGED_DISABLED_HINT = False
def _hint_disabled_once() -> None:
global _LOGGED_DISABLED_HINT
if _LOGGED_DISABLED_HINT:
return
_LOGGED_DISABLED_HINT = True
status = products_fetch_status()
missing = ", ".join(status.get("missing", [])) or "nothing"
logger.info(
"SAR Mode B (ground-change alerts) is disabled. Missing: %s. "
"Enable in Settings → SAR or set the env vars listed in .env.example. "
"Free signup: https://urs.earthdata.nasa.gov/users/new",
missing,
)
@with_retry(max_retries=1, base_delay=3)
def fetch_sar_products() -> None:
"""Refresh pre-processed SAR anomalies for all configured AOIs."""
if not products_fetch_enabled():
_hint_disabled_once()
return
if not is_any_active("sar"):
return
aois = load_aois()
if not aois:
logger.debug("SAR products: no AOIs configured")
return
seen_ids: set[str] = set()
all_anomalies: list[dict[str, Any]] = []
publish_summary = {"signed": 0, "skipped": 0, "reasons": {}}
for aoi in aois:
for fetcher in (
fetch_opera_for_aoi,
fetch_egms_for_aoi,
fetch_gfm_for_aoi,
fetch_ems_for_aoi,
fetch_unosat_for_aoi,
):
try:
anomalies: list[SarAnomaly] = fetcher(aoi) or []
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as exc:
logger.debug("SAR %s for %s failed: %s", fetcher.__name__, aoi.id, exc)
anomalies = []
for a in anomalies:
if a.anomaly_id in seen_ids:
continue
seen_ids.add(a.anomaly_id)
all_anomalies.append(a.to_dict())
status = emit_signed_anomaly(a)
if status.get("signed"):
publish_summary["signed"] += 1
else:
publish_summary["skipped"] += 1
reason = status.get("reason", "unknown")
publish_summary["reasons"][reason] = (
publish_summary["reasons"].get(reason, 0) + 1
)
with _data_lock:
latest_data["sar_anomalies"] = all_anomalies
if all_anomalies:
_mark_fresh("sar_anomalies")
logger.info(
"SAR products: %d anomalies (%d signed, %d skipped)",
len(all_anomalies),
publish_summary["signed"],
publish_summary["skipped"],
)
+485 -352
View File
@@ -5,6 +5,11 @@ CelesTrak Fair Use Policy (https://celestrak.org/NORAD/elements/):
- Use If-Modified-Since headers for conditional requests
- No parallel/concurrent connections — one request at a time
- Set a descriptive User-Agent
Analysis features (derived from cached TLEs — no extra network requests):
- Maneuver detection: TLE-to-TLE comparison per satellite
- Decay anomaly: mean-motion change rate monitoring
- Overflight counting: 24h ground-track sampling for a bounding box
"""
import math
@@ -41,6 +46,67 @@ _sat_classified_cache = {"data": None, "gp_fetch_ts": 0}
_SAT_CACHE_PATH = Path(__file__).parent.parent.parent / "data" / "sat_gp_cache.json"
_SAT_CACHE_META_PATH = Path(__file__).parent.parent.parent / "data" / "sat_gp_cache_meta.json"
# ── Historical TLE storage for maneuver & decay detection ───────────────────
# Stores the previous TLE snapshot keyed by NORAD_CAT_ID.
# Populated when a fresh CelesTrak fetch replaces cached data.
# Persisted to disk so analysis survives restarts.
_SAT_HISTORY_PATH = Path(__file__).parent.parent.parent / "data" / "sat_tle_history.json"
_tle_history: dict[int, dict] = {} # {norad_id: {elements + "epoch_ts"}}
def _load_tle_history():
"""Load previous TLE snapshot from disk."""
global _tle_history
try:
if _SAT_HISTORY_PATH.exists():
with open(_SAT_HISTORY_PATH, "r") as f:
raw = json.load(f)
_tle_history = {int(k): v for k, v in raw.items()}
logger.info(f"Satellites: Loaded TLE history for {len(_tle_history)} objects")
except (IOError, OSError, json.JSONDecodeError, ValueError, KeyError) as e:
logger.warning(f"Satellites: Failed to load TLE history: {e}")
_tle_history = {}
def _save_tle_history():
"""Persist current TLE snapshot as history for next comparison."""
try:
_SAT_HISTORY_PATH.parent.mkdir(parents=True, exist_ok=True)
with open(_SAT_HISTORY_PATH, "w") as f:
json.dump(_tle_history, f)
except (IOError, OSError) as e:
logger.warning(f"Satellites: Failed to save TLE history: {e}")
def _snapshot_current_tles(gp_data):
"""Capture orbital elements from current GP data as the new 'previous' snapshot.
Called once per CelesTrak fetch (every 24h). The old snapshot becomes
the comparison baseline for maneuver/decay detection.
"""
global _tle_history
new_snapshot = {}
for sat in gp_data:
norad_id = sat.get("NORAD_CAT_ID")
if norad_id is None:
continue
epoch_str = sat.get("EPOCH", "")
try:
epoch_dt = datetime.strptime(epoch_str[:19], "%Y-%m-%dT%H:%M:%S")
epoch_ts = epoch_dt.timestamp()
except (ValueError, TypeError):
epoch_ts = 0
new_snapshot[int(norad_id)] = {
"MEAN_MOTION": sat.get("MEAN_MOTION"),
"ECCENTRICITY": sat.get("ECCENTRICITY"),
"INCLINATION": sat.get("INCLINATION"),
"RA_OF_ASC_NODE": sat.get("RA_OF_ASC_NODE"),
"BSTAR": sat.get("BSTAR"),
"epoch_ts": epoch_ts,
}
_tle_history = new_snapshot
_save_tle_history()
def _load_sat_cache():
"""Load satellite GP data from local disk cache."""
@@ -99,360 +165,368 @@ def _save_cache_meta():
# Satellite intelligence classification database
# Matched by substring against OBJECT_NAME (case-insensitive).
# Order matters — first match wins, so specific names go before generic prefixes.
_SAT_INTEL_DB = [
(
"USA 224",
{
"country": "USA",
"mission": "military_recon",
"sat_type": "KH-11 Reconnaissance",
"wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
},
),
(
"USA 245",
{
"country": "USA",
"mission": "military_recon",
"sat_type": "KH-11 Reconnaissance",
"wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
},
),
(
"USA 290",
{
"country": "USA",
"mission": "military_recon",
"sat_type": "KH-11 Reconnaissance",
"wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
},
),
(
"USA 314",
{
"country": "USA",
"mission": "military_recon",
"sat_type": "KH-11 Reconnaissance",
"wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
},
),
(
"USA 338",
{
"country": "USA",
"mission": "military_recon",
"sat_type": "Keyhole Successor",
"wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
},
),
(
"TOPAZ",
{
"country": "Russia",
"mission": "military_recon",
"sat_type": "Optical Reconnaissance",
"wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)",
},
),
(
"PERSONA",
{
"country": "Russia",
"mission": "military_recon",
"sat_type": "Optical Reconnaissance",
"wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)",
},
),
(
"KONDOR",
{
"country": "Russia",
"mission": "military_sar",
"sat_type": "SAR Reconnaissance",
"wiki": "https://en.wikipedia.org/wiki/Kondor_(satellite)",
},
),
(
"BARS-M",
{
"country": "Russia",
"mission": "military_recon",
"sat_type": "Mapping Reconnaissance",
"wiki": "https://en.wikipedia.org/wiki/Bars-M",
},
),
(
"YAOGAN",
{
"country": "China",
"mission": "military_recon",
"sat_type": "Remote Sensing / ELINT",
"wiki": "https://en.wikipedia.org/wiki/Yaogan",
},
),
(
"GAOFEN",
{
"country": "China",
"mission": "military_recon",
"sat_type": "High-Res Imaging",
"wiki": "https://en.wikipedia.org/wiki/Gaofen",
},
),
(
"JILIN",
{
"country": "China",
"mission": "commercial_imaging",
"sat_type": "Video / Imaging",
"wiki": "https://en.wikipedia.org/wiki/Jilin-1",
},
),
(
"OFEK",
{
"country": "Israel",
"mission": "military_recon",
"sat_type": "Reconnaissance",
"wiki": "https://en.wikipedia.org/wiki/Ofeq",
},
),
(
"CSO",
{
"country": "France",
"mission": "military_recon",
"sat_type": "Optical Reconnaissance",
"wiki": "https://en.wikipedia.org/wiki/CSO_(satellite)",
},
),
(
"IGS",
{
"country": "Japan",
"mission": "military_recon",
"sat_type": "Intelligence Gathering",
"wiki": "https://en.wikipedia.org/wiki/Information_Gathering_Satellite",
},
),
(
"CAPELLA",
{
"country": "USA",
"mission": "sar",
"sat_type": "SAR Imaging",
"wiki": "https://en.wikipedia.org/wiki/Capella_Space",
},
),
(
"ICEYE",
{
"country": "Finland",
"mission": "sar",
"sat_type": "SAR Microsatellite",
"wiki": "https://en.wikipedia.org/wiki/ICEYE",
},
),
(
"COSMO-SKYMED",
{
"country": "Italy",
"mission": "sar",
"sat_type": "SAR Constellation",
"wiki": "https://en.wikipedia.org/wiki/COSMO-SkyMed",
},
),
(
"TANDEM",
{
"country": "Germany",
"mission": "sar",
"sat_type": "SAR Interferometry",
"wiki": "https://en.wikipedia.org/wiki/TanDEM-X",
},
),
(
"PAZ",
{
"country": "Spain",
"mission": "sar",
"sat_type": "SAR Imaging",
"wiki": "https://en.wikipedia.org/wiki/PAZ_(satellite)",
},
),
(
"WORLDVIEW",
{
"country": "USA",
"mission": "commercial_imaging",
"sat_type": "Maxar High-Res",
"wiki": "https://en.wikipedia.org/wiki/WorldView-3",
},
),
(
"GEOEYE",
{
"country": "USA",
"mission": "commercial_imaging",
"sat_type": "Maxar Imaging",
"wiki": "https://en.wikipedia.org/wiki/GeoEye-1",
},
),
(
"PLEIADES",
{
"country": "France",
"mission": "commercial_imaging",
"sat_type": "Airbus Imaging",
"wiki": "https://en.wikipedia.org/wiki/Pl%C3%A9iades_(satellite)",
},
),
(
"SPOT",
{
"country": "France",
"mission": "commercial_imaging",
"sat_type": "Airbus Medium-Res",
"wiki": "https://en.wikipedia.org/wiki/SPOT_(satellite)",
},
),
(
"PLANET",
{
"country": "USA",
"mission": "commercial_imaging",
"sat_type": "PlanetScope",
"wiki": "https://en.wikipedia.org/wiki/Planet_Labs",
},
),
(
"SKYSAT",
{
"country": "USA",
"mission": "commercial_imaging",
"sat_type": "Planet Video",
"wiki": "https://en.wikipedia.org/wiki/SkySat",
},
),
(
"BLACKSKY",
{
"country": "USA",
"mission": "commercial_imaging",
"sat_type": "BlackSky Imaging",
"wiki": "https://en.wikipedia.org/wiki/BlackSky",
},
),
(
"NROL",
{
"country": "USA",
"mission": "sigint",
"sat_type": "Classified NRO",
"wiki": "https://en.wikipedia.org/wiki/National_Reconnaissance_Office",
},
),
(
"MENTOR",
{
"country": "USA",
"mission": "sigint",
"sat_type": "SIGINT / ELINT",
"wiki": "https://en.wikipedia.org/wiki/Mentor_(satellite)",
},
),
(
"LUCH",
{
"country": "Russia",
"mission": "sigint",
"sat_type": "Relay / SIGINT",
"wiki": "https://en.wikipedia.org/wiki/Luch_(satellite)",
},
),
(
"SHIJIAN",
{
"country": "China",
"mission": "sigint",
"sat_type": "ELINT / Tech Demo",
"wiki": "https://en.wikipedia.org/wiki/Shijian",
},
),
(
"NAVSTAR",
{
"country": "USA",
"mission": "navigation",
"sat_type": "GPS",
"wiki": "https://en.wikipedia.org/wiki/GPS_satellite_blocks",
},
),
(
"GLONASS",
{
"country": "Russia",
"mission": "navigation",
"sat_type": "GLONASS",
"wiki": "https://en.wikipedia.org/wiki/GLONASS",
},
),
(
"BEIDOU",
{
"country": "China",
"mission": "navigation",
"sat_type": "BeiDou",
"wiki": "https://en.wikipedia.org/wiki/BeiDou",
},
),
(
"GALILEO",
{
"country": "EU",
"mission": "navigation",
"sat_type": "Galileo",
"wiki": "https://en.wikipedia.org/wiki/Galileo_(satellite_navigation)",
},
),
(
"SBIRS",
{
"country": "USA",
"mission": "early_warning",
"sat_type": "Missile Warning",
"wiki": "https://en.wikipedia.org/wiki/Space-Based_Infrared_System",
},
),
(
"TUNDRA",
{
"country": "Russia",
"mission": "early_warning",
"sat_type": "Missile Warning",
"wiki": "https://en.wikipedia.org/wiki/Tundra_(satellite)",
},
),
(
"ISS",
{
"country": "Intl",
"mission": "space_station",
"sat_type": "Space Station",
"wiki": "https://en.wikipedia.org/wiki/International_Space_Station",
},
),
(
"TIANGONG",
{
"country": "China",
"mission": "space_station",
"sat_type": "Space Station",
"wiki": "https://en.wikipedia.org/wiki/Tiangong_space_station",
},
),
# ── USA Keyhole / Reconnaissance ────────────────────────────────────────
("USA 224", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
("USA 245", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
("USA 290", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
("USA 314", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
("USA 338", {"country": "USA", "mission": "military_recon", "sat_type": "Keyhole Successor", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
# ── USA SIGINT / NRO ────────────────────────────────────────────────────
("NROL", {"country": "USA", "mission": "sigint", "sat_type": "Classified NRO", "wiki": "https://en.wikipedia.org/wiki/National_Reconnaissance_Office"}),
("MENTOR", {"country": "USA", "mission": "sigint", "sat_type": "SIGINT / ELINT (Orion)", "wiki": "https://en.wikipedia.org/wiki/Mentor_(satellite)"}),
("TRUMPET", {"country": "USA", "mission": "sigint", "sat_type": "SIGINT (HEO)", "wiki": "https://en.wikipedia.org/wiki/Trumpet_(satellite)"}),
("INTRUDER", {"country": "USA", "mission": "sigint", "sat_type": "Naval SIGINT (NOSS)", "wiki": "https://en.wikipedia.org/wiki/Naval_Ocean_Surveillance_System"}),
# ── USA Early Warning / Missile Defense ─────────────────────────────────
("SBIRS", {"country": "USA", "mission": "early_warning", "sat_type": "Missile Warning", "wiki": "https://en.wikipedia.org/wiki/Space-Based_Infrared_System"}),
("DSP", {"country": "USA", "mission": "early_warning", "sat_type": "Defense Support Program", "wiki": "https://en.wikipedia.org/wiki/Defense_Support_Program"}),
# ── USA Communications (Military) ───────────────────────────────────────
("MUOS", {"country": "USA", "mission": "military_comms", "sat_type": "Mobile User Objective System", "wiki": "https://en.wikipedia.org/wiki/Mobile_User_Objective_System"}),
("AEHF", {"country": "USA", "mission": "military_comms", "sat_type": "Advanced EHF", "wiki": "https://en.wikipedia.org/wiki/Advanced_Extremely_High_Frequency"}),
("WGS", {"country": "USA", "mission": "military_comms", "sat_type": "Wideband Global SATCOM", "wiki": "https://en.wikipedia.org/wiki/Wideband_Global_SATCOM"}),
("MILSTAR", {"country": "USA", "mission": "military_comms", "sat_type": "Milstar Secure Comms", "wiki": "https://en.wikipedia.org/wiki/Milstar"}),
# ── USA Navigation ──────────────────────────────────────────────────────
("NAVSTAR", {"country": "USA", "mission": "navigation", "sat_type": "GPS", "wiki": "https://en.wikipedia.org/wiki/GPS_satellite_blocks"}),
# ── Russia Reconnaissance ───────────────────────────────────────────────
("TOPAZ", {"country": "Russia", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)"}),
("PERSONA", {"country": "Russia", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)"}),
("KONDOR", {"country": "Russia", "mission": "military_sar", "sat_type": "SAR Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Kondor_(satellite)"}),
("BARS-M", {"country": "Russia", "mission": "military_recon", "sat_type": "Mapping Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Bars-M"}),
("RAZDAN", {"country": "Russia", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Razdan_(satellite)"}),
("LOTOS", {"country": "Russia", "mission": "sigint", "sat_type": "ELINT (Lotos-S)", "wiki": "https://en.wikipedia.org/wiki/Lotos-S"}),
("PION", {"country": "Russia", "mission": "sigint", "sat_type": "Naval SIGINT/Radar", "wiki": "https://en.wikipedia.org/wiki/Pion-NKS"}),
("LUCH", {"country": "Russia", "mission": "sigint", "sat_type": "Relay / SIGINT", "wiki": "https://en.wikipedia.org/wiki/Luch_(satellite)"}),
# ── Russia Early Warning & Navigation ───────────────────────────────────
("TUNDRA", {"country": "Russia", "mission": "early_warning", "sat_type": "Missile Warning (EKS)", "wiki": "https://en.wikipedia.org/wiki/Tundra_(satellite)"}),
("GLONASS", {"country": "Russia", "mission": "navigation", "sat_type": "GLONASS", "wiki": "https://en.wikipedia.org/wiki/GLONASS"}),
# ── China Military / Intel ──────────────────────────────────────────────
("YAOGAN", {"country": "China", "mission": "military_recon", "sat_type": "Remote Sensing / ELINT", "wiki": "https://en.wikipedia.org/wiki/Yaogan"}),
("GAOFEN", {"country": "China", "mission": "military_recon", "sat_type": "High-Res Imaging", "wiki": "https://en.wikipedia.org/wiki/Gaofen"}),
("JILIN", {"country": "China", "mission": "commercial_imaging", "sat_type": "Video / Imaging", "wiki": "https://en.wikipedia.org/wiki/Jilin-1"}),
("SHIJIAN", {"country": "China", "mission": "sigint", "sat_type": "ELINT / Tech Demo", "wiki": "https://en.wikipedia.org/wiki/Shijian"}),
("TONGXIN JISHU SHIYAN", {"country": "China", "mission": "military_comms", "sat_type": "Military Comms Test", "wiki": "https://en.wikipedia.org/wiki/Tongxin_Jishu_Shiyan"}),
("BEIDOU", {"country": "China", "mission": "navigation", "sat_type": "BeiDou", "wiki": "https://en.wikipedia.org/wiki/BeiDou"}),
("TIANGONG", {"country": "China", "mission": "space_station", "sat_type": "Space Station", "wiki": "https://en.wikipedia.org/wiki/Tiangong_space_station"}),
# ── Allied Military / Intel ─────────────────────────────────────────────
("OFEK", {"country": "Israel", "mission": "military_recon", "sat_type": "Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Ofeq"}),
("EROS", {"country": "Israel", "mission": "commercial_imaging", "sat_type": "High-Res Imaging", "wiki": "https://en.wikipedia.org/wiki/EROS_(satellite)"}),
("CSO", {"country": "France", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/CSO_(satellite)"}),
("HELIOS", {"country": "France", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Helios_(satellite)"}),
("CERES", {"country": "France", "mission": "sigint", "sat_type": "ELINT Constellation", "wiki": "https://en.wikipedia.org/wiki/CERES_(satellite)"}),
("IGS", {"country": "Japan", "mission": "military_recon", "sat_type": "Intelligence Gathering", "wiki": "https://en.wikipedia.org/wiki/Information_Gathering_Satellite"}),
("KOMPSAT", {"country": "South Korea", "mission": "military_recon", "sat_type": "Multi-Purpose Satellite", "wiki": "https://en.wikipedia.org/wiki/KOMPSAT"}),
("SAR-LUPE", {"country": "Germany", "mission": "military_sar", "sat_type": "SAR Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/SAR-Lupe"}),
("SARAH", {"country": "Germany", "mission": "military_sar", "sat_type": "SAR Successor (SARah)", "wiki": "https://en.wikipedia.org/wiki/SARah"}),
# ── Commercial SAR ──────────────────────────────────────────────────────
("CAPELLA", {"country": "USA", "mission": "sar", "sat_type": "SAR Imaging", "wiki": "https://en.wikipedia.org/wiki/Capella_Space"}),
("ICEYE", {"country": "Finland", "mission": "sar", "sat_type": "SAR Microsatellite", "wiki": "https://en.wikipedia.org/wiki/ICEYE"}),
("COSMO-SKYMED", {"country": "Italy", "mission": "sar", "sat_type": "SAR Constellation", "wiki": "https://en.wikipedia.org/wiki/COSMO-SkyMed"}),
("TANDEM", {"country": "Germany", "mission": "sar", "sat_type": "SAR Interferometry", "wiki": "https://en.wikipedia.org/wiki/TanDEM-X"}),
("PAZ", {"country": "Spain", "mission": "sar", "sat_type": "SAR Imaging", "wiki": "https://en.wikipedia.org/wiki/PAZ_(satellite)"}),
("UMBRA", {"country": "USA", "mission": "sar", "sat_type": "SAR Microsatellite", "wiki": "https://en.wikipedia.org/wiki/Umbra_(company)"}),
# ── Commercial Optical Imaging ──────────────────────────────────────────
("WORLDVIEW", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Maxar High-Res", "wiki": "https://en.wikipedia.org/wiki/WorldView-3"}),
("GEOEYE", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Maxar Imaging", "wiki": "https://en.wikipedia.org/wiki/GeoEye-1"}),
("LEGION", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Maxar Legion", "wiki": "https://en.wikipedia.org/wiki/WorldView_Legion"}),
("PLEIADES", {"country": "France", "mission": "commercial_imaging", "sat_type": "Airbus Imaging", "wiki": "https://en.wikipedia.org/wiki/Pl%C3%A9iades_(satellite)"}),
("SPOT", {"country": "France", "mission": "commercial_imaging", "sat_type": "Airbus Medium-Res", "wiki": "https://en.wikipedia.org/wiki/SPOT_(satellite)"}),
("SKYSAT", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Planet Video", "wiki": "https://en.wikipedia.org/wiki/SkySat"}),
("BLACKSKY", {"country": "USA", "mission": "commercial_imaging", "sat_type": "BlackSky Imaging", "wiki": "https://en.wikipedia.org/wiki/BlackSky"}),
# ── Starlink (separate category) ────────────────────────────────────────
("STARLINK", {"country": "USA", "mission": "starlink", "sat_type": "Starlink Mega-Constellation", "wiki": "https://en.wikipedia.org/wiki/Starlink"}),
# ── Other Constellations ────────────────────────────────────────────────
("ONEWEB", {"country": "UK", "mission": "constellation", "sat_type": "OneWeb LEO Broadband", "wiki": "https://en.wikipedia.org/wiki/OneWeb"}),
("GALILEO", {"country": "EU", "mission": "navigation", "sat_type": "Galileo", "wiki": "https://en.wikipedia.org/wiki/Galileo_(satellite_navigation)"}),
# ── Space Stations ──────────────────────────────────────────────────────
("ISS", {"country": "Intl", "mission": "space_station", "sat_type": "Space Station", "wiki": "https://en.wikipedia.org/wiki/International_Space_Station"}),
# ── Generic fallback patterns (last resort) ─────────────────────────────
("PLANET", {"country": "USA", "mission": "commercial_imaging", "sat_type": "PlanetScope", "wiki": "https://en.wikipedia.org/wiki/Planet_Labs"}),
]
# CelesTrak SATCAT owner codes → country mapping for satellites not matched by name.
# Used as a secondary classifier alongside name-pattern matching.
_OWNER_CODE_MAP = {
"US": "USA", "CIS": "Russia", "PRC": "China", "ISS": "Intl",
"FR": "France", "UK": "UK", "GER": "Germany", "JPN": "Japan",
"IND": "India", "ISRA": "Israel", "IT": "Italy", "KOR": "South Korea",
"ESA": "EU", "NATO": "NATO", "TURK": "Turkey", "UAE": "UAE",
"AUS": "Australia", "CA": "Canada", "SPN": "Spain", "FIN": "Finland",
"BRAZ": "Brazil", "IRAN": "Iran", "NKOR": "North Korea",
}
# ── Maneuver detection thresholds (per Lemmens & Krag 2014, Kim et al. 2021) ─
# These are above TLE fitting noise but low enough to catch real maneuvers.
_MANEUVER_THRESHOLDS = {
"period_min": 0.1, # minutes — above TLE noise (~0.01–0.05 min)
"inclination_deg": 0.05, # degrees — above J2 secular drift (~0.001°/day)
"eccentricity": 0.005, # above TLE fitting noise (~0.0001–0.001)
"raan_residual_deg": 0.5, # degrees — only after J2 correction (Vallado §9.4)
}
# ── Decay anomaly threshold ─────────────────────────────────────────────────
# Flag if mean motion change rate exceeds this (rev/day per day).
# Normal drag-induced decay is ~0.001 rev/day/day for LEO.
_DECAY_MM_RATE_THRESHOLD = 0.01 # rev/day per day
def _j2_raan_rate(inclination_deg, mean_motion_revday):
"""Expected RAAN precession rate due to J2 (Vallado §9.4).
Returns degrees/day. Negative for prograde orbits.
"""
J2 = 1.08263e-3
Re = 6378.137 # km
mu = 398600.4418 # km^3/s^2
n_rad_s = mean_motion_revday * 2 * math.pi / 86400.0
if n_rad_s <= 0:
return 0.0
a = (mu / (n_rad_s ** 2)) ** (1.0 / 3.0) # semi-major axis in km
if a <= Re:
return 0.0
cos_i = math.cos(math.radians(inclination_deg))
raan_rate = -1.5 * n_rad_s * J2 * (Re / a) ** 2 * cos_i
return math.degrees(raan_rate) * 86400.0 / (2 * math.pi) # deg/day
def detect_maneuvers(current_gp_data):
"""Compare current TLEs against stored history to detect orbital maneuvers.
Returns list of maneuver alert dicts. Only runs when _tle_history is populated
(i.e., after the second CelesTrak fetch or from persisted history).
Thresholds from Lemmens & Krag (2014), Kim et al. (2021).
"""
if not _tle_history:
return []
alerts = []
for sat in current_gp_data:
norad_id = sat.get("NORAD_CAT_ID")
if norad_id is None:
continue
norad_id = int(norad_id)
prev = _tle_history.get(norad_id)
if prev is None:
continue
cur_mm = sat.get("MEAN_MOTION")
cur_inc = sat.get("INCLINATION")
cur_ecc = sat.get("ECCENTRICITY")
cur_raan = sat.get("RA_OF_ASC_NODE")
prev_mm = prev.get("MEAN_MOTION")
prev_inc = prev.get("INCLINATION")
prev_ecc = prev.get("ECCENTRICITY")
prev_raan = prev.get("RA_OF_ASC_NODE")
if any(v is None for v in (cur_mm, cur_inc, cur_ecc, cur_raan,
prev_mm, prev_inc, prev_ecc, prev_raan)):
continue
# Convert mean motion (rev/day) to period (minutes)
cur_period = 1440.0 / cur_mm if cur_mm > 0 else 0
prev_period = 1440.0 / prev_mm if prev_mm > 0 else 0
reasons = []
t = _MANEUVER_THRESHOLDS
delta_period = abs(cur_period - prev_period)
if delta_period > t["period_min"]:
reasons.append(f"period Δ{delta_period:+.3f} min")
delta_inc = abs(cur_inc - prev_inc)
if delta_inc > t["inclination_deg"]:
reasons.append(f"inclination Δ{delta_inc:+.4f}°")
delta_ecc = abs(cur_ecc - prev_ecc)
if delta_ecc > t["eccentricity"]:
reasons.append(f"eccentricity Δ{delta_ecc:+.6f}")
# RAAN with J2 correction — only flag residual beyond expected precession
epoch_str = sat.get("EPOCH", "")
try:
epoch_dt = datetime.strptime(epoch_str[:19], "%Y-%m-%dT%H:%M:%S")
epoch_ts = epoch_dt.timestamp()
except (ValueError, TypeError):
epoch_ts = 0
prev_epoch_ts = prev.get("epoch_ts", 0)
dt_days = (epoch_ts - prev_epoch_ts) / 86400.0 if (epoch_ts and prev_epoch_ts) else 1.0
if dt_days > 0:
expected_raan_drift = _j2_raan_rate(cur_inc, cur_mm) * dt_days
actual_raan_change = cur_raan - prev_raan
# Normalize to [-180, 180]
actual_raan_change = (actual_raan_change + 180) % 360 - 180
raan_residual = abs(actual_raan_change - expected_raan_drift)
if raan_residual > t["raan_residual_deg"]:
reasons.append(f"RAAN residual {raan_residual:.3f}° (J2-corrected)")
if reasons:
alerts.append({
"norad_id": norad_id,
"name": sat.get("OBJECT_NAME", "UNKNOWN"),
"type": "maneuver",
"reasons": reasons,
"epoch": sat.get("EPOCH", ""),
"delta_period_min": round(delta_period, 4),
"delta_inclination_deg": round(delta_inc, 5),
"delta_eccentricity": round(delta_ecc, 7),
})
logger.info(f"Satellites: Maneuver scan — {len(alerts)} detections from {len(current_gp_data)} objects")
return alerts
def detect_decay_anomalies(current_gp_data):
"""Flag satellites with abnormal mean-motion change rates (possible decay).
A rapidly increasing mean motion indicates orbital decay — the satellite
is losing altitude. Normal LEO drag is ~0.001 rev/day/day.
"""
if not _tle_history:
return []
alerts = []
for sat in current_gp_data:
norad_id = sat.get("NORAD_CAT_ID")
if norad_id is None:
continue
norad_id = int(norad_id)
prev = _tle_history.get(norad_id)
if prev is None:
continue
cur_mm = sat.get("MEAN_MOTION")
prev_mm = prev.get("MEAN_MOTION")
if cur_mm is None or prev_mm is None:
continue
epoch_str = sat.get("EPOCH", "")
try:
epoch_dt = datetime.strptime(epoch_str[:19], "%Y-%m-%dT%H:%M:%S")
epoch_ts = epoch_dt.timestamp()
except (ValueError, TypeError):
continue
prev_epoch_ts = prev.get("epoch_ts", 0)
dt_days = (epoch_ts - prev_epoch_ts) / 86400.0 if (epoch_ts and prev_epoch_ts) else 0
if dt_days < 0.5:
continue # Need at least 12h between TLEs for meaningful comparison
mm_rate = (cur_mm - prev_mm) / dt_days # rev/day per day
if abs(mm_rate) > _DECAY_MM_RATE_THRESHOLD:
cur_alt_km = (8681663.7 / (cur_mm ** (2.0 / 3.0))) - 6371.0 if cur_mm > 0 else 0
alerts.append({
"norad_id": norad_id,
"name": sat.get("OBJECT_NAME", "UNKNOWN"),
"type": "decay_anomaly",
"mm_rate": round(mm_rate, 6),
"current_mm": round(cur_mm, 4),
"approx_alt_km": round(cur_alt_km, 1),
"epoch": sat.get("EPOCH", ""),
"dt_days": round(dt_days, 2),
})
logger.info(f"Satellites: Decay scan — {len(alerts)} anomalies detected")
return alerts
def compute_overflights(gp_data, bbox, hours=24, step_minutes=10):
"""Count unique satellites whose ground track enters a bounding box.
Args:
gp_data: Full GP catalog (list of dicts with orbital elements).
bbox: Dict with keys 's', 'w', 'n', 'e' (degrees).
hours: Look-back window (default 24h).
step_minutes: Sampling interval (default 10 min).
Returns dict with total count and per-mission breakdown.
Uses SGP4 propagation — CPU cost is ~O(catalog_size × timesteps).
Only propagates satellites that could plausibly overfly the bbox latitude range.
"""
if not gp_data or not bbox:
return {"total": 0, "by_mission": {}, "satellites": []}
south, west = bbox["s"], bbox["w"]
north, east = bbox["n"], bbox["e"]
now = datetime.utcnow()
steps = int(hours * 60 / step_minutes)
# Pre-filter: only propagate sats whose inclination allows them to reach bbox latitude
max_lat = max(abs(south), abs(north))
candidates = [s for s in gp_data if s.get("INCLINATION") is not None
and s.get("INCLINATION") >= max_lat * 0.8] # 20% margin
seen_ids = set()
results = []
by_mission = {}
for s in candidates:
norad_id = s.get("NORAD_CAT_ID")
mean_motion = s.get("MEAN_MOTION")
ecc = s.get("ECCENTRICITY")
incl = s.get("INCLINATION")
raan = s.get("RA_OF_ASC_NODE")
argp = s.get("ARG_OF_PERICENTER")
ma = s.get("MEAN_ANOMALY")
bstar = s.get("BSTAR", 0)
epoch_str = s.get("EPOCH", "")
if any(v is None for v in (mean_motion, ecc, incl, raan, argp, ma, epoch_str)):
continue
try:
epoch_dt = datetime.strptime(epoch_str[:19], "%Y-%m-%dT%H:%M:%S")
epoch_jd, epoch_fr = jday(
epoch_dt.year, epoch_dt.month, epoch_dt.day,
epoch_dt.hour, epoch_dt.minute, epoch_dt.second,
)
sat_obj = Satrec()
sat_obj.sgp4init(
WGS72, "i", norad_id or 0,
(epoch_jd + epoch_fr) - 2433281.5,
bstar, 0.0, 0.0, ecc,
math.radians(argp), math.radians(incl), math.radians(ma),
mean_motion * 2 * math.pi / 1440.0, math.radians(raan),
)
except (ValueError, TypeError):
continue
for step in range(steps):
t = now - timedelta(minutes=step * step_minutes)
jd_t, fr_t = jday(t.year, t.month, t.day, t.hour, t.minute, t.second)
e, r, _ = sat_obj.sgp4(jd_t, fr_t)
if e != 0:
continue
x, y, z = r
gmst = _gmst(jd_t + fr_t)
lng_rad = math.atan2(y, x) - gmst
lat_deg = math.degrees(math.atan2(z, math.sqrt(x * x + y * y)))
lng_deg = math.degrees(lng_rad) % 360
if lng_deg > 180:
lng_deg -= 360
# Check bounding box (handles antimeridian crossing)
lat_in = south <= lat_deg <= north
if west <= east:
lng_in = west <= lng_deg <= east
else:
lng_in = lng_deg >= west or lng_deg <= east
if lat_in and lng_in and norad_id not in seen_ids:
seen_ids.add(norad_id)
name = s.get("OBJECT_NAME", "UNKNOWN")
# Classify for mission breakdown
mission = "unknown"
for key, meta in _SAT_INTEL_DB:
if key.upper() in name.upper():
mission = meta.get("mission", "unknown")
break
by_mission[mission] = by_mission.get(mission, 0) + 1
results.append({"norad_id": norad_id, "name": name, "mission": mission})
break # Already counted this sat, move to next
return {"total": len(results), "by_mission": by_mission, "satellites": results}
def _parse_tle_to_gp(name, norad_id, line1, line2):
"""Convert TLE two-line element to CelesTrak GP-style dict."""
@@ -539,9 +613,18 @@ def fetch_satellites():
if not is_any_active("satellites"):
return
sats = []
maneuver_alerts = []
decay_alerts = []
starlink_summary = {}
data = None
classified = None
try:
now_ts = time.time()
# On first call, load TLE history from disk for maneuver detection
if not _tle_history:
_load_tle_history()
# On first call, try disk cache before hitting CelesTrak
if _sat_gp_cache["data"] is None:
disk_data = _load_sat_cache()
@@ -594,6 +677,9 @@ def fetch_satellites():
if lm:
_sat_gp_cache["last_modified"] = lm
_save_sat_cache(gp_data)
# Snapshot current TLEs as history before overwriting
# (the old _tle_history becomes the comparison baseline)
_snapshot_current_tles(gp_data)
logger.info(
f"Satellites: Downloaded {len(gp_data)} GP records from CelesTrak"
)
@@ -651,11 +737,14 @@ def fetch_satellites():
and _sat_classified_cache["data"]
):
classified = _sat_classified_cache["data"]
starlink_summary = _sat_classified_cache.get("starlink_summary", {})
logger.info(
f"Satellites: Using cached classification ({len(classified)} sats, TLEs unchanged)"
)
else:
classified = []
starlink_count = 0
starlink_shells = {} # inclination shell → count
for sat in data:
name = sat.get("OBJECT_NAME", "UNKNOWN").upper()
intel = None
@@ -663,8 +752,24 @@ def fetch_satellites():
if key.upper() in name:
intel = dict(meta)
break
if not intel:
# Secondary classification via SATCAT owner code
owner = sat.get("OWNER", sat.get("OBJECT_OWNER", ""))
if owner in _OWNER_CODE_MAP:
intel = {"country": _OWNER_CODE_MAP[owner], "mission": "general", "sat_type": "Unclassified"}
if not intel:
continue
# Starlink: count and summarize but don't propagate individually
# (6000+ sats would be too expensive to position every 60s)
if intel.get("mission") == "starlink":
starlink_count += 1
inc = sat.get("INCLINATION")
if inc is not None:
shell_key = f"{round(inc, 0):.0f}°"
starlink_shells[shell_key] = starlink_shells.get(shell_key, 0) + 1
continue # Skip individual propagation
entry = {
"id": sat.get("NORAD_CAT_ID"),
"name": sat.get("OBJECT_NAME", "UNKNOWN"),
@@ -679,14 +784,35 @@ def fetch_satellites():
}
entry.update(intel)
classified.append(entry)
starlink_summary = {
"total": starlink_count,
"shells": starlink_shells,
}
_sat_classified_cache["data"] = classified
_sat_classified_cache["starlink_summary"] = starlink_summary
_sat_classified_cache["gp_fetch_ts"] = _sat_gp_cache["last_fetch"]
logger.info(
f"Satellites: {len(classified)} intel-classified out of {len(data)} total in catalog"
f"Satellites: {len(classified)} intel-classified, "
f"{starlink_count} Starlink (summarized), "
f"out of {len(data)} total in catalog"
)
all_sats = classified
# ── Run analysis detectors against the full GP catalog ──────────────
# These use cached TLEs only — no extra network requests.
maneuver_alerts = []
decay_alerts = []
try:
maneuver_alerts = detect_maneuvers(data)
except (ValueError, TypeError, KeyError, ZeroDivisionError) as e:
logger.error(f"Satellites: Maneuver detection error: {e}")
try:
decay_alerts = detect_decay_anomalies(data)
except (ValueError, TypeError, KeyError, ZeroDivisionError) as e:
logger.error(f"Satellites: Decay detection error: {e}")
now = datetime.utcnow()
jd, fr = jday(
now.year, now.month, now.day, now.hour, now.minute, now.second + now.microsecond / 1e6
@@ -800,6 +926,13 @@ def fetch_satellites():
with _data_lock:
latest_data["satellites"] = sats
latest_data["satellite_source"] = _sat_gp_cache.get("source", "none")
latest_data["satellite_analysis"] = {
"maneuvers": maneuver_alerts,
"decay_anomalies": decay_alerts,
"starlink": starlink_summary,
"catalog_size": len(data) if data else 0,
"classified_count": len(classified) if classified else 0,
}
_mark_fresh("satellites")
else:
with _data_lock:
+216
View File
@@ -0,0 +1,216 @@
"""WastewaterSCAN fetcher — pathogen surveillance via wastewater monitoring.
Data source: Stanford/Emory WastewaterSCAN project
- Plant locations: https://storage.googleapis.com/wastewater-dev-data/json/plants.json
- Time series: https://storage.googleapis.com/wastewater-dev-data/json/{uuid}.json
All data is public, no authentication required. ~192 treatment plants across
the US with daily sampling for COVID (N Gene), Influenza A/B, RSV, Norovirus,
MPXV, Measles, H5N1, and others.
"""
import logging
import time
import concurrent.futures
from datetime import datetime, timedelta
from services.network_utils import fetch_with_curl
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
from services.fetchers.retry import with_retry
logger = logging.getLogger(__name__)
_GCS_BASE = "https://storage.googleapis.com/wastewater-dev-data/json"
# Cache the plants list for 24 hours (it rarely changes)
_plants_cache: list[dict] = []
_plants_cache_ts: float = 0
_PLANTS_CACHE_TTL = 86400 # 24 hours
# Key pathogen targets to extract — maps internal target name to display label
_TARGET_DISPLAY: dict[str, str] = {
"N Gene": "COVID-19",
"Influenza A F1R1": "Influenza A",
"Influenza B": "Influenza B",
"RSV": "RSV",
"Noro_G2": "Norovirus",
"MPXV_G2R_WA": "Mpox",
"InfA_H5": "H5N1 (Bird Flu)",
"HMPV_4": "HMPV",
"Rota": "Rotavirus",
"HAV": "Hepatitis A",
"C_auris": "Candida auris",
"EVD68": "Enterovirus D68",
}
# Activity categories that represent elevated/alert levels
_ALERT_CATEGORIES = {"high", "very high", "above normal"}
def _fetch_plants() -> list[dict]:
"""Fetch the full plants list from GCS, with 24h caching."""
global _plants_cache, _plants_cache_ts
if _plants_cache and (time.time() - _plants_cache_ts) < _PLANTS_CACHE_TTL:
return _plants_cache
url = f"{_GCS_BASE}/plants.json"
resp = fetch_with_curl(url, timeout=30)
if resp.status_code != 200:
logger.warning(f"WastewaterSCAN plants fetch failed: HTTP {resp.status_code}")
return _plants_cache # return stale cache on failure
data = resp.json()
plants = data.get("plants", [])
_plants_cache = plants
_plants_cache_ts = time.time()
logger.info(f"WastewaterSCAN: cached {len(plants)} plant locations")
return plants
def _fetch_plant_latest(plant_id: str) -> dict | None:
"""Fetch the most recent sample for a single plant.
Returns a dict with pathogen levels or None on failure.
"""
url = f"{_GCS_BASE}/{plant_id}.json"
try:
resp = fetch_with_curl(url, timeout=12)
if resp.status_code != 200:
return None
data = resp.json()
samples = data.get("samples", [])
if not samples:
return None
# Find the most recent sample (last element, sorted by date)
latest = samples[-1]
collection_date = latest.get("collection_date", "")
# Skip samples older than 30 days
try:
sample_dt = datetime.strptime(collection_date, "%Y-%m-%d")
if sample_dt < datetime.utcnow() - timedelta(days=30):
return None
except (ValueError, TypeError):
pass
# Extract key pathogen levels
targets = latest.get("targets", {})
pathogens: list[dict] = []
alert_count = 0
for target_key, display_name in _TARGET_DISPLAY.items():
target_data = targets.get(target_key)
if not target_data:
continue
concentration = target_data.get("gc_g_dry_weight", 0) or 0
activity = target_data.get("activity_category", "not calculated")
normalized = target_data.get("gc_g_dry_weight_pmmov", 0) or 0
if concentration <= 0 and normalized <= 0:
continue # no detection
is_alert = activity.lower() in _ALERT_CATEGORIES
if is_alert:
alert_count += 1
pathogens.append({
"name": display_name,
"target_key": target_key,
"concentration": round(concentration, 1),
"normalized": round(normalized, 6),
"activity": activity,
"alert": is_alert,
})
if not pathogens:
return None
return {
"collection_date": collection_date,
"pathogens": pathogens,
"alert_count": alert_count,
}
except Exception as e:
logger.debug(f"WastewaterSCAN: failed to fetch plant {plant_id}: {e}")
return None
@with_retry(max_retries=1, base_delay=5)
def fetch_wastewater():
"""Fetch WastewaterSCAN plant locations and latest pathogen levels.
1. Fetches the plant list (cached 24h) for locations.
2. Concurrently fetches time series for all plants, extracting only
the most recent sample's pathogen data.
3. Merges into a flat list suitable for map rendering.
"""
from services.fetchers._store import is_any_active
if not is_any_active("wastewater"):
return
plants = _fetch_plants()
if not plants:
logger.warning("WastewaterSCAN: no plant data available")
return
# Build base records from plant metadata
plant_map: dict[str, dict] = {}
for p in plants:
point = p.get("point") or {}
coords = point.get("coordinates") or []
if len(coords) < 2:
continue
pid = p.get("id") or p.get("uuid", "")
if not pid:
continue
plant_map[pid] = {
"id": pid,
"name": p.get("name", ""),
"site_name": p.get("site_name", ""),
"city": p.get("city", ""),
"state": p.get("state", ""),
"country": p.get("country", "US"),
"population": p.get("sewershed_pop"),
"lat": coords[1],
"lng": coords[0],
"pathogens": [],
"alert_count": 0,
"collection_date": "",
"source": "WastewaterSCAN",
}
# Fetch latest samples concurrently (up to 12 threads)
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
futures = {
pool.submit(_fetch_plant_latest, pid): pid
for pid in plant_map
}
for fut in concurrent.futures.as_completed(futures, timeout=120):
pid = futures[fut]
try:
result = fut.result()
if result:
plant_map[pid]["pathogens"] = result["pathogens"]
plant_map[pid]["alert_count"] = result["alert_count"]
plant_map[pid]["collection_date"] = result["collection_date"]
except Exception:
pass
nodes = list(plant_map.values())
active_nodes = [n for n in nodes if n["pathogens"]]
logger.info(
f"WastewaterSCAN: {len(nodes)} plants, "
f"{len(active_nodes)} with recent pathogen data, "
f"{sum(n['alert_count'] for n in nodes)} total alerts"
)
with _data_lock:
latest_data["wastewater"] = nodes
if nodes:
_mark_fresh("wastewater")
+57 -31
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import json
import os
import re
import time
import threading
from typing import Any, Dict, List
@@ -81,43 +82,63 @@ def _load_local_search_cache() -> List[Dict[str, Any]]:
def _search_local_fallback(query: str, limit: int) -> List[Dict[str, Any]]:
"""Strict local lookup used only when ``local_only=True`` is set.
Historical behaviour (substring-token-in-haystack matching) produced
catastrophically wrong results: any query containing a common word
would match the first airport with that word anywhere in its name,
which silently poisoned every cache downstream. Fixed to require
whole-word matches against airport name/IATA/id and cached-geocode
labels.
"""
q = query.strip().lower()
if not q:
return []
q_tokens = set(re.findall(r"[a-z0-9]+", q))
if not q_tokens:
return []
matches: List[Dict[str, Any]] = []
seen: set[tuple[float, float, str]] = set()
def _whole_word_tokens(text: str) -> set[str]:
return set(re.findall(r"[a-z0-9]+", (text or "").lower()))
for item in cached_airports:
haystacks = [
str(item.get("name", "")).lower(),
str(item.get("iata", "")).lower(),
str(item.get("id", "")).lower(),
]
if any(q in h for h in haystacks):
label = f'{item.get("name", "Airport")} ({item.get("iata", "")})'
key = (float(item["lat"]), float(item["lng"]), label)
if key not in seen:
seen.add(key)
matches.append(
{
"label": label,
"lat": float(item["lat"]),
"lng": float(item["lng"]),
}
)
if len(matches) >= limit:
return matches
name_tokens = _whole_word_tokens(item.get("name", ""))
iata = str(item.get("iata", "")).lower().strip()
icao = str(item.get("id", "")).lower().strip()
# IATA/ICAO must match exactly; name must share ALL query tokens
# with the airport name (not "any token in haystack").
exact_code = bool(iata and iata in q_tokens) or bool(icao and icao in q_tokens)
name_match = bool(q_tokens) and q_tokens.issubset(name_tokens)
if not (exact_code or name_match):
continue
label = f'{item.get("name", "Airport")} ({item.get("iata", "")})'
key = (float(item["lat"]), float(item["lng"]), label)
if key not in seen:
seen.add(key)
matches.append(
{
"label": label,
"lat": float(item["lat"]),
"lng": float(item["lng"]),
}
)
if len(matches) >= limit:
return matches
for item in _load_local_search_cache():
label = str(item.get("label", ""))
if q in label.lower():
key = (float(item["lat"]), float(item["lng"]), label)
if key not in seen:
seen.add(key)
matches.append(item)
if len(matches) >= limit:
break
label_tokens = _whole_word_tokens(label)
if not q_tokens.issubset(label_tokens):
continue
key = (float(item["lat"]), float(item["lng"]), label)
if key not in seen:
seen.add(key)
matches.append(item)
if len(matches) >= limit:
break
return matches
@@ -163,9 +184,14 @@ def search_geocode(query: str, limit: int = 5, local_only: bool = False) -> List
timeout=6,
)
except Exception:
results = _search_local_fallback(q, limit)
_set_cache(key, results)
return results
# Intentionally no silent airport-name fallback. Callers that
# want offline results should pass ``local_only=True``; anything
# else means we return an empty list so the caller can decide
# whether to retry or propagate the failure. The old behaviour
# of falling through to _search_local_fallback silently poisoned
# every downstream cache with airport coordinates for any query.
_set_cache(key, [])
return []
results: List[Dict[str, Any]] = []
if res and res.status_code == 200:
@@ -184,9 +210,9 @@ def search_geocode(query: str, limit: int = 5, local_only: bool = False) -> List
continue
except Exception:
results = []
if not results:
results = _search_local_fallback(q, limit)
# No silent airport-name fallback on empty results either — same
# reason as above. Empty means empty.
_set_cache(key, results)
return results
+246
View File
@@ -0,0 +1,246 @@
"""Country-bbox post-filter for geocoded results.
Any fetcher that turns a country-tagged row into a lat/lng should call
``coord_in_country()`` after the geocoder returns. If the coordinate
falls outside the country's bounding box, the result is almost
certainly a namesake collision (e.g. "Milan, WI" landing in Milan,
Italy) and the caller should reject or retry with a stronger query.
This is a cheap sanity gate that catches geocoder mistakes no human
operator will ever spot by eye across thousands of points.
Bounding boxes are deliberately generous — they include territories,
overseas islands, and a small buffer — so that legitimate coastal or
border cities are never false-rejected. Goal is to catch "wrong
continent", not "off by a few km".
"""
from __future__ import annotations
from typing import Optional, Tuple
# (min_lat, min_lng, max_lat, max_lng)
_COUNTRY_BBOX: dict[str, Tuple[float, float, float, float]] = {
# North America
"USA": (18.0, -180.0, 72.0, -65.0), # inc. Alaska + Hawaii
"Canada": (41.0, -142.0, 84.0, -52.0),
"Mexico": (14.0, -120.0, 33.0, -86.0),
# South & Central America
"Brazil": (-35.0, -74.5, 6.0, -34.0),
"Argentina": (-56.0, -74.0, -21.5, -53.0),
"Chile": (-56.0, -76.0, -17.0, -66.0),
"Colombia": (-5.0, -82.0, 13.5, -66.5),
"Peru": (-19.0, -82.0, 0.5, -68.5),
"Venezuela": (0.5, -73.5, 12.5, -59.5),
"Ecuador": (-5.5, -92.5, 2.0, -75.0), # inc. Galápagos
"Bolivia": (-23.0, -69.5, -9.5, -57.5),
"Uruguay": (-35.0, -58.5, -30.0, -53.0),
"Paraguay": (-28.0, -63.0, -19.0, -54.0),
"Guatemala": (13.5, -92.5, 18.0, -88.0),
"Honduras": (12.5, -89.5, 16.5, -83.0),
"Nicaragua": (10.5, -88.0, 15.5, -83.0),
"Costa Rica": (8.0, -86.0, 11.5, -82.5),
"Panama": (7.0, -83.5, 9.7, -77.0),
"El Salvador": (13.0, -90.5, 14.5, -87.5),
"Cuba": (19.5, -85.0, 23.5, -74.0),
"Dominican Republic": (17.5, -72.5, 20.0, -68.0),
"Haiti": (17.5, -74.5, 20.5, -71.5),
"Jamaica": (17.5, -78.5, 18.7, -76.0),
"Puerto Rico": (17.5, -68.0, 19.0, -65.0),
# Europe
"United Kingdom": (49.0, -9.0, 61.0, 2.5),
"Ireland": (51.0, -11.0, 56.0, -5.0),
"France": (41.0, -5.5, 51.5, 9.8),
"Germany": (47.0, 5.5, 56.0, 15.5),
"Spain": (27.0, -18.5, 44.0, 4.5), # inc. Canary Islands
"Portugal": (32.0, -32.0, 42.5, -6.0), # inc. Azores + Madeira
"Italy": (36.0, 6.5, 47.5, 19.0),
"Netherlands": (50.5, 3.0, 53.8, 7.3),
"Belgium": (49.4, 2.5, 51.6, 6.5),
"Switzerland": (45.7, 5.8, 48.0, 10.6),
"Austria": (46.3, 9.5, 49.1, 17.2),
"Poland": (49.0, 14.0, 55.0, 24.2),
"Czech Republic": (48.5, 12.0, 51.2, 18.9),
"Slovakia": (47.7, 16.8, 49.7, 22.6),
"Hungary": (45.7, 16.1, 48.6, 22.9),
"Romania": (43.6, 20.2, 48.3, 29.7),
"Bulgaria": (41.2, 22.3, 44.3, 28.7),
"Greece": (34.7, 19.3, 41.8, 29.7),
"Turkey": (35.8, 25.6, 42.2, 44.8),
"Ukraine": (44.3, 22.1, 52.4, 40.3),
"Belarus": (51.2, 23.1, 56.2, 32.8),
"Russia": (41.0, 19.0, 82.0, 180.0),
"Sweden": (55.0, 10.5, 69.1, 24.2),
"Norway": (57.9, 4.5, 71.2, 31.1),
"Finland": (59.7, 20.5, 70.1, 31.6),
"Denmark": (54.5, 8.0, 57.9, 15.3),
"Iceland": (63.3, -24.6, 66.6, -13.4),
"Serbia": (42.2, 18.8, 46.2, 23.0),
"Croatia": (42.3, 13.4, 46.6, 19.5),
"Slovenia": (45.4, 13.3, 46.9, 16.7),
"Bosnia and Herzegovina": (42.5, 15.7, 45.3, 19.7),
"North Macedonia": (40.8, 20.4, 42.4, 23.1),
"Albania": (39.6, 19.2, 42.7, 21.1),
"Kosovo": (41.8, 20.0, 43.3, 21.8),
"Moldova": (45.4, 26.6, 48.5, 30.2),
"Lithuania": (53.8, 20.9, 56.5, 26.9),
"Latvia": (55.6, 20.9, 58.1, 28.3),
"Estonia": (57.5, 21.7, 59.8, 28.3),
"Luxembourg": (49.4, 5.7, 50.2, 6.6),
"Malta": (35.7, 14.1, 36.1, 14.7),
"Cyprus": (34.5, 32.2, 35.8, 34.7),
# Middle East
"Israel": (29.4, 34.2, 33.4, 35.9),
"Lebanon": (33.0, 35.1, 34.7, 36.7),
"Jordan": (29.1, 34.9, 33.4, 39.4),
"Syria": (32.3, 35.7, 37.4, 42.4),
"Iraq": (29.0, 38.8, 37.4, 48.8),
"Iran": (25.0, 44.0, 40.0, 63.4),
"Saudi Arabia": (16.3, 34.5, 32.2, 55.7),
"Yemen": (12.0, 42.5, 19.0, 54.5),
"United Arab Emirates": (22.6, 51.5, 26.1, 56.4),
"Oman": (16.6, 52.0, 26.4, 59.9),
"Qatar": (24.4, 50.7, 26.2, 51.7),
"Bahrain": (25.8, 50.4, 26.4, 50.8),
"Kuwait": (28.5, 46.5, 30.1, 48.4),
"Afghanistan": (29.4, 60.5, 38.5, 74.9),
# Asia
"India": (6.0, 68.0, 36.0, 98.0),
"Pakistan": (23.7, 60.9, 37.1, 77.8),
"Bangladesh": (20.6, 88.0, 26.6, 92.7),
"Sri Lanka": (5.9, 79.5, 9.9, 82.0),
"Nepal": (26.3, 80.0, 30.5, 88.2),
"China": (18.0, 73.0, 54.0, 135.5),
"Mongolia": (41.6, 87.7, 52.2, 119.9),
"Japan": (24.0, 122.0, 46.0, 146.0),
"South Korea": (33.1, 125.1, 38.6, 131.9),
"North Korea": (37.7, 124.2, 43.0, 130.7),
"Taiwan": (21.8, 119.3, 25.4, 122.1),
"Hong Kong": (22.1, 113.8, 22.6, 114.5),
"Vietnam": (8.2, 102.1, 23.4, 109.5),
"Thailand": (5.6, 97.3, 20.5, 105.7),
"Cambodia": (10.4, 102.3, 14.7, 107.7),
"Laos": (13.9, 100.0, 22.5, 107.7),
"Myanmar": (9.5, 92.1, 28.6, 101.2),
"Malaysia": (0.8, 99.5, 7.5, 119.3),
"Singapore": (1.1, 103.5, 1.5, 104.1),
"Indonesia": (-11.1, 94.8, 6.1, 141.1),
"Philippines": (4.5, 116.0, 21.5, 127.0),
"Brunei": (4.0, 114.0, 5.1, 115.4),
"Kazakhstan": (40.5, 46.4, 55.5, 87.4),
"Uzbekistan": (37.1, 55.9, 45.6, 73.2),
"Kyrgyzstan": (39.1, 69.2, 43.3, 80.3),
"Tajikistan": (36.6, 67.3, 41.1, 75.2),
"Turkmenistan": (35.1, 52.4, 42.8, 66.7),
"Azerbaijan": (38.3, 44.7, 41.9, 50.6),
"Armenia": (38.8, 43.4, 41.3, 46.6),
"Georgia": (41.0, 40.0, 43.6, 46.8),
# Oceania
"Australia": (-44.0, 112.0, -9.0, 155.0),
"New Zealand": (-48.0, 165.0, -33.0, 179.5),
"Papua New Guinea": (-11.7, 140.8, -1.0, 156.0),
"Fiji": (-21.0, 176.8, -12.4, -178.3), # crosses antimeridian; see handling
# Africa (selected — most common NUFORC reporters)
"South Africa": (-35.0, 16.0, -22.0, 33.0),
"Egypt": (21.7, 24.7, 31.7, 36.9),
"Morocco": (27.6, -13.2, 35.9, -1.0),
"Algeria": (18.9, -8.7, 37.1, 12.0),
"Tunisia": (30.2, 7.5, 37.5, 11.6),
"Libya": (19.5, 9.3, 33.2, 25.2),
"Sudan": (8.6, 21.8, 22.2, 38.6),
"Ethiopia": (3.4, 32.9, 14.9, 48.0),
"Kenya": (-4.7, 33.9, 5.5, 41.9),
"Tanzania": (-11.8, 29.3, -0.9, 40.4),
"Uganda": (-1.5, 29.5, 4.2, 35.0),
"Nigeria": (4.2, 2.6, 13.9, 14.7),
"Ghana": (4.7, -3.3, 11.2, 1.2),
"Senegal": (12.3, -17.6, 16.7, -11.3),
"Ivory Coast": (4.3, -8.6, 10.7, -2.5),
"Cameroon": (1.6, 8.5, 13.1, 16.2),
"Angola": (-18.1, 11.7, -4.4, 24.1),
"Zimbabwe": (-22.5, 25.2, -15.6, 33.1),
"Zambia": (-18.1, 21.9, -8.2, 33.7),
"Mozambique": (-26.9, 30.2, -10.5, 40.9),
"Madagascar": (-25.7, 43.2, -11.9, 50.5),
"Democratic Republic of the Congo": (-13.5, 12.2, 5.4, 31.4),
"Rwanda": (-2.9, 28.8, -1.0, 30.9),
}
# Common aliases used in NUFORC / other data sources.
_COUNTRY_ALIASES: dict[str, str] = {
"US": "USA",
"U.S.": "USA",
"U.S.A.": "USA",
"United States": "USA",
"United States of America": "USA",
"America": "USA",
"UK": "United Kingdom",
"U.K.": "United Kingdom",
"Britain": "United Kingdom",
"Great Britain": "United Kingdom",
"England": "United Kingdom",
"Scotland": "United Kingdom",
"Wales": "United Kingdom",
"Northern Ireland": "United Kingdom",
"Czechia": "Czech Republic",
"Czechoslovakia": "Czech Republic",
"South Korea": "South Korea",
"Korea": "South Korea",
"Republic of Korea": "South Korea",
"Democratic People's Republic of Korea": "North Korea",
"DPRK": "North Korea",
"Russian Federation": "Russia",
"Viet Nam": "Vietnam",
"Côte d'Ivoire": "Ivory Coast",
"Cote d'Ivoire": "Ivory Coast",
"DR Congo": "Democratic Republic of the Congo",
"DRC": "Democratic Republic of the Congo",
"Congo-Kinshasa": "Democratic Republic of the Congo",
"Macedonia": "North Macedonia",
"Burma": "Myanmar",
"Holland": "Netherlands",
}
def canonical_country(country: str) -> str:
"""Normalise a country string to its registry key."""
if not country:
return ""
c = country.strip()
return _COUNTRY_ALIASES.get(c, c)
def coord_in_country(lat: float, lng: float, country: str) -> Optional[bool]:
"""Return True if (lat, lng) is inside the country bbox, False if it
is outside, or None if the country is unknown (cannot validate — the
caller should treat unknown as "pass", not "fail").
"""
try:
lat_f = float(lat)
lng_f = float(lng)
except (TypeError, ValueError):
return None
if not (-90.0 <= lat_f <= 90.0 and -180.0 <= lng_f <= 180.0):
return False
c = canonical_country(country)
bbox = _COUNTRY_BBOX.get(c)
if bbox is None:
return None
min_lat, min_lng, max_lat, max_lng = bbox
return min_lat <= lat_f <= max_lat and min_lng <= lng_f <= max_lng
def validate_geocode(
lat: float,
lng: float,
country: str,
) -> bool:
"""Higher-level gate used in fetcher geocoding loops.
Returns True if the coordinate is acceptable for the given country,
False if it's clearly a namesake collision that should be rejected.
Unknown countries are treated as "accept" so we don't throw away
otherwise-good data for uncovered regions.
"""
result = coord_in_country(lat, lng, country)
return result is not False
+92 -10
View File
@@ -201,10 +201,12 @@ def _is_gibberish(text):
# Persistent cache for article titles — survives across GDELT cache refreshes
# Bounded to 5000 entries with 24hr TTL to prevent unbounded memory growth
_article_title_cache = TTLCache(maxsize=5000, ttl=86400)
_article_snippet_cache: dict[str, str | None] = {}
_article_url_safety_cache = TTLCache(maxsize=5000, ttl=3600)
_TITLE_FETCH_MAX_REDIRECTS = 3
_TITLE_FETCH_READ_BYTES = 32768
_ALLOWED_ARTICLE_PORTS = {80, 443, 8080, 8443}
_MAX_SNIPPET_LEN = 200
def _hostname_resolves_public(hostname: str, port: int) -> bool:
@@ -269,6 +271,30 @@ def _is_safe_public_article_url(url: str) -> tuple[bool, str]:
return result
def _extract_snippet(url: str, chunk: str) -> None:
"""Extract og:description or meta description from an already-fetched HTML chunk."""
import re
import html as html_mod
if url in _article_snippet_cache:
return
snippet = None
# Try og:description first
for pattern in (
r'<meta[^>]+property=["\']og:description["\'][^>]+content=["\']([^"\'>]+)["\']',
r'<meta[^>]+content=["\']([^"\'>]+)["\'][^>]+property=["\']og:description["\']',
r'<meta[^>]+name=["\']description["\'][^>]+content=["\']([^"\'>]+)["\']',
r'<meta[^>]+content=["\']([^"\'>]+)["\'][^>]+name=["\']description["\']',
):
m = re.search(pattern, chunk, re.I)
if m:
snippet = html_mod.unescape(m.group(1)).strip()
break
if snippet and len(snippet) > _MAX_SNIPPET_LEN:
snippet = snippet[:_MAX_SNIPPET_LEN - 3].rsplit(" ", 1)[0] + "..."
_article_snippet_cache[url] = snippet if snippet and len(snippet) > 15 else None
def _fetch_article_title(url):
"""Fetch the real headline from an article's HTML <title> or og:title tag.
Returns the title string, or None if it can't be fetched.
@@ -343,6 +369,8 @@ def _fetch_article_title(url):
title = title[:117] + "..."
if len(title) > 10:
_article_title_cache[url] = title
# Also extract og:description / meta description for snippet
_extract_snippet(url, chunk)
return title
_article_title_cache[url] = None
@@ -405,21 +433,49 @@ def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_
actor1 = row[6].strip() if len(row) > 6 else ""
actor2 = row[16].strip() if len(row) > 16 else ""
# Extract enrichment fields from GDELT CSV
event_date = row[1].strip() if len(row) > 1 else ""
full_event_code = row[26].strip() if len(row) > 26 else ""
quad_class = int(row[29]) if len(row) > 29 and row[29].strip().isdigit() else 0
goldstein = float(row[30]) if len(row) > 30 and row[30].strip() else 0.0
num_mentions = int(row[31]) if len(row) > 31 and row[31].strip().isdigit() else 0
num_sources = int(row[32]) if len(row) > 32 and row[32].strip().isdigit() else 0
num_articles = int(row[33]) if len(row) > 33 and row[33].strip().isdigit() else 0
avg_tone = float(row[34]) if len(row) > 34 and row[34].strip() else 0.0
loc_key = f"{round(lat, 1)}_{round(lng, 1)}"
if loc_key in seen_locs:
# Merge: increment count and add source URL if new (dedup by domain)
# Merge: increment count, accumulate intensity, add source URL
idx = loc_index[loc_key]
feat = features[idx]
feat["properties"]["count"] = feat["properties"].get("count", 1) + 1
urls = feat["properties"].get("_urls", [])
seen_domains = feat["properties"].get("_domains", set())
props = feat["properties"]
props["count"] = props.get("count", 1) + 1
# Track worst Goldstein score (most negative = most intense)
if goldstein < props.get("goldstein", 0):
props["goldstein"] = round(goldstein, 1)
# Accumulate mentions/sources for importance ranking
props["num_mentions"] = props.get("num_mentions", 0) + num_mentions
props["num_sources"] = props.get("num_sources", 0) + num_sources
props["num_articles"] = props.get("num_articles", 0) + num_articles
# Track latest date
if event_date and event_date > props.get("event_date", ""):
props["event_date"] = event_date
# Collect actors
actors = props.get("_actors_set", set())
if actor1:
actors.add(actor1)
if actor2:
actors.add(actor2)
props["_actors_set"] = actors
urls = props.get("_urls", [])
seen_domains = props.get("_domains", set())
if source_url:
domain = _extract_domain(source_url)
if domain not in seen_domains and len(urls) < 10:
urls.append(source_url)
seen_domains.add(domain)
feat["properties"]["_urls"] = urls
feat["properties"]["_domains"] = seen_domains
props["_urls"] = urls
props["_domains"] = seen_domains
continue
seen_locs.add(loc_key)
@@ -429,6 +485,11 @@ def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_
or "Unknown Incident"
)
domain = _extract_domain(source_url) if source_url else ""
actors_set = set()
if actor1:
actors_set.add(actor1)
if actor2:
actors_set.add(actor2)
loc_index[loc_key] = len(features)
features.append(
{
@@ -436,6 +497,17 @@ def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_
"properties": {
"name": name,
"count": 1,
"event_date": event_date,
"event_code": full_event_code,
"quad_class": quad_class,
"goldstein": round(goldstein, 1),
"num_mentions": num_mentions,
"num_sources": num_sources,
"num_articles": num_articles,
"avg_tone": round(avg_tone, 1),
"actor1": actor1,
"actor2": actor2,
"_actors_set": actors_set,
"_urls": [source_url] if source_url else [],
"_domains": {domain} if domain else set(),
},
@@ -468,12 +540,19 @@ def _build_feature_html(features, fetched_titles=None):
for f in features:
urls = f["properties"].pop("_urls", [])
f["properties"].pop("_domains", None)
# Convert actors set to sorted list for JSON serialization
actors_set = f["properties"].pop("_actors_set", set())
if actors_set:
f["properties"]["actors"] = sorted(actors_set)[:6]
headlines = []
snippets = []
for u in urls:
real_title = fetched_titles.get(u) if fetched_titles else None
headlines.append(real_title if real_title else _url_to_headline(u))
snippets.append(_article_snippet_cache.get(u) or "")
f["properties"]["_urls_list"] = urls
f["properties"]["_headlines_list"] = headlines
f["properties"]["_snippets_list"] = snippets
if urls:
links = []
for u, h in zip(urls, headlines):
@@ -498,16 +577,19 @@ def _enrich_gdelt_titles_background(features, all_article_urls):
fetched_count = sum(1 for v in fetched_titles.values() if v)
logger.info(f"[BG] Resolved {fetched_count}/{len(all_article_urls)} article titles")
# Update features in-place with real titles
# Update features in-place with real titles and snippets
for f in features:
urls = f["properties"].get("_urls_list", [])
if not urls:
continue
headlines = []
snippets = []
for u in urls:
real_title = fetched_titles.get(u)
headlines.append(real_title if real_title else _url_to_headline(u))
snippets.append(_article_snippet_cache.get(u) or "")
f["properties"]["_headlines_list"] = headlines
f["properties"]["_snippets_list"] = snippets
links = []
for u, h in zip(urls, headlines):
safe_url = u if u.startswith(("http://", "https://")) else "about:blank"
@@ -564,8 +646,8 @@ def fetch_global_military_incidents():
latest_ts = datetime.strptime(ts_match.group(1), "%Y%m%d%H%M%S")
# Generate URLs for the last 8 hours (32 files at 15-min intervals)
NUM_FILES = 32
# Generate URLs for the last 12 hours (48 files at 15-min intervals)
NUM_FILES = 48
urls = []
for i in range(NUM_FILES):
ts = latest_ts - timedelta(minutes=15 * i)
@@ -583,7 +665,7 @@ def fetch_global_military_incidents():
logger.info(f"Downloaded {successful}/{len(urls)} GDELT exports")
# Parse all downloaded files
CONFLICT_CODES = {"14", "17", "18", "19", "20"}
CONFLICT_CODES = {"13", "14", "15", "16", "17", "18", "19", "20"}
features = []
seen_locs = set()
loc_index = {} # loc_key -> index in features
+129
View File
@@ -0,0 +1,129 @@
"""Infonet economy & governance layer.
Layered ON TOP OF the existing mesh primitives in ``services/mesh/``.
The chain-write cutover (2026-04-28) registers Infonet event types
with ``mesh_schema`` and ``mesh_hashchain`` so production writes flow
through the legacy chain. The cutover is performed at import time by
``services.infonet._chain_cutover``.
The only legacy file modified by the cutover is ``mesh_schema.py``,
which gained a generic extension hook (``register_extension_validator``).
``mesh_hashchain.py`` is byte-identical to its Sprint 1 baseline; the
cutover mutates its module-level ``ACTIVE_APPEND_EVENT_TYPES`` set
(which is a mutable ``set``, not a frozenset, by design).
See ``infonet-economy/IMPLEMENTATION_PLAN.md`` and ``infonet-economy/BUILD_LOG.md``
in the repository root for the build order, sprint scope, and integration
principles. ``infonet-economy/RULES_SKELETON.md`` is the source of truth
for any formula / value / state machine implemented here.
"""
# Trigger the chain-write cutover at import time. Idempotent — see
# ``_chain_cutover.perform_cutover``. This must happen before any
# adapter or producer uses mesh_schema.validate_event_payload on a
# new event type.
from services.infonet import _chain_cutover as _chain_cutover_module
_chain_cutover_module.perform_cutover()
del _chain_cutover_module
from services.infonet.config import (
CONFIG,
CONFIG_SCHEMA,
CROSS_FIELD_INVARIANTS,
IMMUTABLE_PRINCIPLES,
InvalidPetition,
reset_config_for_tests,
validate_config_schema_completeness,
validate_cross_field_invariants,
validate_petition_value,
)
from services.infonet.identity_rotation import (
RotationBlocker,
RotationDecision,
rotation_descendants,
validate_rotation,
)
from services.infonet.markets import (
EvidenceBundle,
MarketStatus,
ResolutionResult,
build_snapshot,
collect_evidence,
collect_resolution_stakes,
compute_market_status,
compute_snapshot_event_hash,
evidence_content_hash,
excluded_predictor_ids,
find_snapshot,
is_first_for_side,
is_predictor_excluded,
resolve_market,
should_advance_phase,
submission_hash,
)
from services.infonet.reputation import (
OracleRepBreakdown,
compute_common_rep,
compute_oracle_rep,
compute_oracle_rep_active,
compute_oracle_rep_lifetime,
decay_factor_for_age,
last_successful_prediction_ts,
)
from services.infonet.schema import (
INFONET_ECONOMY_EVENT_TYPES,
InfonetEventSchema,
get_infonet_schema,
validate_infonet_event_payload,
)
from services.infonet.time_validity import (
chain_majority_time,
event_meets_phase_window,
is_event_too_future,
)
__all__ = [
"CONFIG",
"CONFIG_SCHEMA",
"CROSS_FIELD_INVARIANTS",
"IMMUTABLE_PRINCIPLES",
"INFONET_ECONOMY_EVENT_TYPES",
"EvidenceBundle",
"InfonetEventSchema",
"InvalidPetition",
"MarketStatus",
"OracleRepBreakdown",
"ResolutionResult",
"RotationBlocker",
"RotationDecision",
"build_snapshot",
"chain_majority_time",
"collect_evidence",
"collect_resolution_stakes",
"compute_common_rep",
"compute_market_status",
"compute_oracle_rep",
"compute_oracle_rep_active",
"compute_oracle_rep_lifetime",
"compute_snapshot_event_hash",
"decay_factor_for_age",
"event_meets_phase_window",
"evidence_content_hash",
"excluded_predictor_ids",
"find_snapshot",
"get_infonet_schema",
"is_event_too_future",
"is_first_for_side",
"is_predictor_excluded",
"last_successful_prediction_ts",
"reset_config_for_tests",
"resolve_market",
"rotation_descendants",
"should_advance_phase",
"submission_hash",
"validate_config_schema_completeness",
"validate_cross_field_invariants",
"validate_infonet_event_payload",
"validate_petition_value",
"validate_rotation",
]
+108
View File
@@ -0,0 +1,108 @@
"""Chain-write cutover — register Infonet economy event types with the
legacy mesh_schema + mesh_hashchain at import time.
Source of truth: ``infonet-economy/BUILD_LOG.md`` Sprint 4 §6.2 cutover
decision (Option C — rename + coexist with new event-type names).
Before this cutover, Sprints 1-7 produced economy events through
``InfonetHashchainAdapter.dry_run_append`` only. None of them landed
on the legacy chain because ``mesh_hashchain.Infonet.append`` rejected
any event_type not in ``ACTIVE_APPEND_EVENT_TYPES``.
This module performs the surgical wiring needed for production writes:
1. Mutates ``mesh_hashchain.ACTIVE_APPEND_EVENT_TYPES`` (a mutable
set, not a frozenset) to include every type in
``INFONET_ECONOMY_EVENT_TYPES``.
2. Registers each economy event type's payload validator with
``mesh_schema._EXTENSION_VALIDATORS`` via the Sprint-8-polish
``register_extension_validator`` hook.
The cutover is **idempotent**: importing this module twice leaves the
state unchanged.
The direction is **one-way**: infonet imports mesh_*; mesh never
imports infonet. mesh_schema's hook is generic — it doesn't know
about infonet specifically.
What is NOT modified by this cutover:
- ``mesh_schema.SCHEMA_REGISTRY`` — legacy validators stay as-is.
Economy types use the parallel ``_EXTENSION_VALIDATORS`` registry.
- ``mesh_schema.ACTIVE_PUBLIC_LEDGER_EVENT_TYPES`` — legacy frozenset
unchanged. The runtime decision in
``mesh_hashchain.Infonet.append`` consults the mutable
``ACTIVE_APPEND_EVENT_TYPES`` set.
- ``mesh_hashchain.py`` — byte-identical to its Sprint 1 baseline.
- The legacy ``normalize_payload`` and "no ephemeral on this type"
checks — extension events skip them. Economy event payloads
already have their own normalization (the schema in
``services/infonet/schema.py``).
"""
from __future__ import annotations
import threading
from services.infonet.schema import (
INFONET_ECONOMY_EVENT_TYPES,
validate_infonet_event_payload,
)
from services.mesh import mesh_hashchain, mesh_schema
_CUTOVER_LOCK = threading.Lock()
_CUTOVER_DONE = False
def perform_cutover() -> None:
"""Idempotent registration of every Infonet economy event type.
Safe to call multiple times. After the first call, repeat calls
are no-ops (the lock + sentinel guard re-entry).
"""
global _CUTOVER_DONE
with _CUTOVER_LOCK:
if _CUTOVER_DONE:
return
# Extend the active-append set so mesh_hashchain.Infonet.append
# accepts these types. The set is mutable by design (legacy
# mesh_hashchain.py line 163 uses set(), not frozenset()).
mesh_hashchain.ACTIVE_APPEND_EVENT_TYPES.update(INFONET_ECONOMY_EVENT_TYPES)
# Register a validator for each. The lambda binds to the loop
# variable via default-arg trick to avoid late-binding bugs.
for event_type in INFONET_ECONOMY_EVENT_TYPES:
mesh_schema.register_extension_validator(
event_type,
lambda payload, _et=event_type: validate_infonet_event_payload(_et, payload),
)
_CUTOVER_DONE = True
def cutover_status() -> dict[str, object]:
"""Diagnostic — used by tests and health endpoints to confirm the
cutover ran and registered every type."""
return {
"done": _CUTOVER_DONE,
"registered_types": sorted(
t for t in INFONET_ECONOMY_EVENT_TYPES
if mesh_schema.is_extension_event_type(t)
),
"missing_types": sorted(
t for t in INFONET_ECONOMY_EVENT_TYPES
if not mesh_schema.is_extension_event_type(t)
),
"active_append_includes_economy": INFONET_ECONOMY_EVENT_TYPES.issubset(
mesh_hashchain.ACTIVE_APPEND_EVENT_TYPES
),
}
# Run automatically when the module is imported. The infonet package
# __init__ imports this module, so any code that uses
# ``services.infonet`` at all triggers the cutover. Production callers
# don't need to do anything explicit.
perform_cutover()
__all__ = ["cutover_status", "perform_cutover"]
@@ -0,0 +1,38 @@
"""Adapter layer between the Infonet economy package and the legacy
``services/mesh/`` primitives.
Rule: **adapters import from mesh, mesh never imports from infonet.**
This keeps the dependency direction one-way and lets us delete the
infonet package without touching mesh.
The legacy mesh files (``mesh_schema.py``, ``mesh_signed_events.py``,
``mesh_hashchain.py``, ``mesh_reputation.py``, ``mesh_oracle.py``) stay
byte-identical through Sprint 3. From Sprint 4 onward, when actual chain
writes for new event types start happening, the hashchain adapter is
the single integration point that decides whether to:
1. Modify ``ACTIVE_APPEND_EVENT_TYPES`` in ``mesh_schema.py`` (one-shot,
minimal mesh change), OR
2. Maintain a parallel append surface in ``hashchain_adapter`` that
shares the on-disk chain file but bypasses the legacy event-type
gate.
The decision is recorded in ``infonet-economy/BUILD_LOG.md`` Sprint 4
when made.
"""
from services.infonet.adapters.hashchain_adapter import (
InfonetHashchainAdapter,
extended_active_event_types,
)
from services.infonet.adapters.signed_write_adapter import (
INFONET_SIGNED_WRITE_KINDS,
InfonetSignedWriteKind,
)
__all__ = [
"INFONET_SIGNED_WRITE_KINDS",
"InfonetHashchainAdapter",
"InfonetSignedWriteKind",
"extended_active_event_types",
]
@@ -0,0 +1,178 @@
"""Gate adapter — Sprint 6 implementation.
Bridges chain history to the gate sacrifice / locking / shutdown
lifecycle. Same ``chain_provider`` pattern as the other adapters.
"""
from __future__ import annotations
import time
from typing import Any, Callable, Iterable
from services.infonet.gates import (
AppealValidation,
EntryDecision,
GateMeta,
LockedGateState,
ShutdownState,
SuspensionState,
can_enter,
compute_member_set,
compute_shutdown_state,
compute_suspension_state,
cumulative_member_oracle_rep,
get_gate_meta,
is_locked,
is_member,
is_ratified,
locked_at,
locked_by,
paused_execution_remaining_sec,
validate_appeal_filing,
validate_lock_request,
validate_shutdown_filing,
validate_suspend_filing,
)
from services.infonet.gates.locking import LockValidation
from services.infonet.gates.shutdown.suspend import FilingValidation
from services.infonet.time_validity import chain_majority_time
_ChainProvider = Callable[[], Iterable[dict[str, Any]]]
def _empty_chain() -> list[dict[str, Any]]:
return []
class InfonetGateAdapter:
"""Project chain state into gate views."""
def __init__(self, chain_provider: _ChainProvider | None = None) -> None:
self._chain_provider: _ChainProvider = chain_provider or _empty_chain
def _events(self) -> list[dict[str, Any]]:
return [e for e in self._chain_provider() if isinstance(e, dict)]
def _now(self, override: float | None) -> float:
if override is not None:
return float(override)
events = self._events()
chain_now = chain_majority_time(events)
return chain_now if chain_now > 0 else float(time.time())
# ── Metadata + membership ────────────────────────────────────────
def gate_meta(self, gate_id: str) -> GateMeta | None:
return get_gate_meta(gate_id, self._events())
def member_set(self, gate_id: str) -> set[str]:
return compute_member_set(gate_id, self._events())
def is_member(self, node_id: str, gate_id: str) -> bool:
return is_member(node_id, gate_id, self._events())
def can_enter(self, node_id: str, gate_id: str) -> EntryDecision:
return can_enter(node_id, gate_id, self._events())
# ── Ratification ─────────────────────────────────────────────────
def is_ratified(self, gate_id: str) -> bool:
return is_ratified(gate_id, self._events())
def cumulative_member_oracle_rep(self, gate_id: str) -> float:
return cumulative_member_oracle_rep(gate_id, self._events())
# ── Locking ──────────────────────────────────────────────────────
def is_locked(self, gate_id: str) -> bool:
return is_locked(gate_id, self._events())
def locked_state(self, gate_id: str) -> LockedGateState:
events = self._events()
return LockedGateState(
locked=is_locked(gate_id, events),
locked_at=locked_at(gate_id, events),
locked_by=locked_by(gate_id, events),
)
def validate_lock_request(
self, node_id: str, gate_id: str, *, lock_cost: int | None = None,
) -> LockValidation:
return validate_lock_request(node_id, gate_id, self._events(), lock_cost=lock_cost)
# ── Suspension ───────────────────────────────────────────────────
def suspension_state(
self, gate_id: str, *, now: float | None = None,
) -> SuspensionState:
return compute_suspension_state(gate_id, self._events(), now=self._now(now))
def validate_suspend_filing(
self,
gate_id: str,
filer_id: str,
*,
reason: str,
evidence_hashes: list[str],
now: float | None = None,
filer_cooldown_until: float | None = None,
) -> FilingValidation:
return validate_suspend_filing(
gate_id, filer_id,
reason=reason, evidence_hashes=evidence_hashes,
chain=self._events(), now=self._now(now),
filer_cooldown_until=filer_cooldown_until,
)
# ── Shutdown ─────────────────────────────────────────────────────
def shutdown_state(
self, gate_id: str, *, now: float | None = None,
) -> ShutdownState:
return compute_shutdown_state(gate_id, self._events(), now=self._now(now))
def validate_shutdown_filing(
self,
gate_id: str,
filer_id: str,
*,
reason: str,
evidence_hashes: list[str],
now: float | None = None,
filer_cooldown_until: float | None = None,
) -> FilingValidation:
return validate_shutdown_filing(
gate_id, filer_id,
reason=reason, evidence_hashes=evidence_hashes,
chain=self._events(), now=self._now(now),
filer_cooldown_until=filer_cooldown_until,
)
# ── Appeal ───────────────────────────────────────────────────────
def validate_appeal_filing(
self,
gate_id: str,
target_petition_id: str,
filer_id: str,
*,
reason: str,
evidence_hashes: list[str],
now: float | None = None,
filer_cooldown_until: float | None = None,
) -> AppealValidation:
return validate_appeal_filing(
gate_id, target_petition_id, filer_id,
reason=reason, evidence_hashes=evidence_hashes,
chain=self._events(), now=self._now(now),
filer_cooldown_until=filer_cooldown_until,
)
def paused_execution_remaining_sec(
self,
target_petition_id: str,
*,
appeal_filed_at: float,
) -> float:
return paused_execution_remaining_sec(
target_petition_id, self._events(),
appeal_filed_at=appeal_filed_at,
)
__all__ = ["InfonetGateAdapter"]
@@ -0,0 +1,125 @@
"""Bridge between Infonet economy events and the legacy ``mesh_hashchain``.
Sprint 1 ships this as a **dry-run-only** surface. We do NOT call the
legacy ``Infonet.append`` for new event types because that method
hard-rejects anything not in ``ACTIVE_APPEND_EVENT_TYPES`` (defined in
``mesh_schema.py``). Modifying that set is a Sprint 4 task — it requires
the rest of the producer code to exist, otherwise a malformed
``prediction_create`` could land on the chain with no resolver to
process it.
What this adapter DOES today:
- ``extended_active_event_types()`` — returns the union of legacy active
types and new economy types, for tooling that needs the full surface
(e.g. RPC layer, frontend type generation).
- ``InfonetHashchainAdapter.dry_run_append`` — validates a payload
against the new schema and returns the event dict the legacy
``Infonet.append`` would have built. Useful for tests and for the
future cutover plan.
What this adapter will do in Sprint 4:
- ``append_infonet_event`` — actually call ``Infonet.append`` once
``ACTIVE_APPEND_EVENT_TYPES`` is unioned with the economy types.
The Sprint 1 contract:
- ``mesh_hashchain.py`` is byte-identical to the pre-Sprint-1 baseline.
- No event reaches the legacy chain via this adapter in Sprint 1.
- Tests cover validation behavior only.
"""
from __future__ import annotations
import hashlib
import json
import time
from typing import Any
from services.mesh.mesh_schema import (
ACTIVE_PUBLIC_LEDGER_EVENT_TYPES as _LEGACY_ACTIVE_TYPES,
)
from services.infonet.schema import (
INFONET_ECONOMY_EVENT_TYPES,
validate_infonet_event_payload,
)
def extended_active_event_types() -> frozenset[str]:
"""Union of legacy active types and new economy types.
Frozen at import time. The legacy set is itself a frozenset so this
is safe to call from any thread.
"""
return _LEGACY_ACTIVE_TYPES | INFONET_ECONOMY_EVENT_TYPES
class InfonetHashchainAdapter:
"""Validation-only adapter for new Infonet economy events.
Real chain integration lives in Sprint 4. Tests should use
``dry_run_append`` to assert that producer code is constructing
correctly-shaped events before the cutover.
"""
def dry_run_append(
self,
event_type: str,
node_id: str,
payload: dict[str, Any],
*,
sequence: int = 1,
timestamp: float | None = None,
) -> dict[str, Any]:
"""Validate and return a synthetic event dict.
Mirrors the shape that ``mesh_hashchain.Infonet.append`` would
produce for legacy types — same field set, same ordering. Does
NOT compute a real signature (Sprint 4 territory) and does NOT
write to disk.
Raises ``ValueError`` on validation failure — the same exception
type the legacy ``append`` raises so callers don't need to
special-case the cutover later.
"""
if event_type not in INFONET_ECONOMY_EVENT_TYPES:
raise ValueError(f"event_type {event_type!r} not in INFONET_ECONOMY_EVENT_TYPES")
if not isinstance(node_id, str) or not node_id:
raise ValueError("node_id is required")
if not isinstance(sequence, int) or isinstance(sequence, bool) or sequence <= 0:
raise ValueError("sequence must be a positive integer")
ok, reason = validate_infonet_event_payload(event_type, payload)
if not ok:
raise ValueError(reason)
ts = float(timestamp) if timestamp is not None else float(time.time())
canonical = {
"event_type": event_type,
"node_id": node_id,
"payload": payload,
"timestamp": ts,
"sequence": sequence,
}
encoded = json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
event_id = hashlib.sha256(encoded.encode("utf-8")).hexdigest()
return {
"event_id": event_id,
"event_type": event_type,
"node_id": node_id,
"timestamp": ts,
"sequence": sequence,
"payload": payload,
# signature / public_key intentionally omitted in Sprint 1.
"is_provisional": True,
}
__all__ = [
"InfonetHashchainAdapter",
"extended_active_event_types",
]
@@ -0,0 +1,124 @@
"""Adapter from chain history to the market lifecycle / resolution view.
Sprint 4: real implementation (replaces the Sprint 1 ``NotImplementedError``
skeleton). Wires the pure functions in ``services/infonet/markets/`` to
the same chain-provider pattern used by ``InfonetReputationAdapter``.
Sprint 5 will extend this with dispute open / dispute_stake / dispute
resolve methods. Sprint 8 will extend the resolution path with
bootstrap-mode handling.
"""
from __future__ import annotations
from typing import Any, Callable, Iterable
from services.infonet.markets import (
DisputeView,
EvidenceBundle,
MarketStatus,
ResolutionResult,
build_snapshot,
collect_disputes,
collect_evidence,
collect_resolution_stakes,
compute_dispute_outcome,
compute_market_status,
compute_snapshot_event_hash,
dispute_settlement_effects,
effective_outcome,
excluded_predictor_ids,
find_snapshot,
is_predictor_excluded,
market_was_reversed,
resolve_market,
should_advance_phase,
)
_ChainProvider = Callable[[], Iterable[dict[str, Any]]]
def _empty_chain() -> list[dict[str, Any]]:
return []
class InfonetOracleAdapter:
"""Project chain state into market lifecycle + resolution views."""
def __init__(self, chain_provider: _ChainProvider | None = None) -> None:
self._chain_provider: _ChainProvider = chain_provider or _empty_chain
def _events(self) -> list[dict[str, Any]]:
return [e for e in self._chain_provider() if isinstance(e, dict)]
# ── Lifecycle ────────────────────────────────────────────────────
def market_status(self, market_id: str, *, now: float) -> MarketStatus:
return compute_market_status(market_id, self._events(), now=now)
def should_advance_phase(
self, market_id: str, *, now: float,
) -> tuple[MarketStatus, MarketStatus] | None:
return should_advance_phase(market_id, self._events(), now=now)
# ── Snapshot ─────────────────────────────────────────────────────
def take_snapshot(self, market_id: str, *, frozen_at: float) -> dict[str, Any]:
return build_snapshot(market_id, self._events(), frozen_at=frozen_at)
def find_snapshot(self, market_id: str) -> dict[str, Any] | None:
return find_snapshot(market_id, self._events())
@staticmethod
def snapshot_event_hash(
snapshot_payload: dict[str, Any],
*,
market_id: str,
creator_node_id: str,
sequence: int,
) -> str:
return compute_snapshot_event_hash(
snapshot_payload,
market_id=market_id,
creator_node_id=creator_node_id,
sequence=sequence,
)
# ── Evidence ─────────────────────────────────────────────────────
def collect_evidence(self, market_id: str) -> list[EvidenceBundle]:
return collect_evidence(market_id, self._events())
# ── Resolution ───────────────────────────────────────────────────
def excluded_predictor_ids(self, market_id: str) -> set[str]:
return excluded_predictor_ids(market_id, self._events())
def is_predictor_excluded(self, node_id: str, market_id: str) -> bool:
return is_predictor_excluded(node_id, market_id, self._events())
def collect_resolution_stakes(self, market_id: str):
return collect_resolution_stakes(market_id, self._events())
def resolve_market(
self, market_id: str, *, is_provisional: bool = False,
) -> ResolutionResult:
return resolve_market(market_id, self._events(), is_provisional=is_provisional)
# ── Disputes (Sprint 5) ──────────────────────────────────────────
def collect_disputes(self, market_id: str) -> list[DisputeView]:
return collect_disputes(market_id, self._events())
@staticmethod
def compute_dispute_outcome(dispute: DisputeView) -> str:
return compute_dispute_outcome(dispute)
@staticmethod
def dispute_settlement_effects(dispute: DisputeView) -> dict:
return dispute_settlement_effects(dispute)
def market_was_reversed(self, market_id: str) -> bool:
return market_was_reversed(market_id, self._events())
def effective_outcome(self, market_id: str, original_outcome: str) -> str:
return effective_outcome(original_outcome, market_id, self._events())
__all__ = ["InfonetOracleAdapter"]
@@ -0,0 +1,107 @@
"""Adapter that projects chain history into the new reputation views.
Sprint 2: real implementation. Replaces the Sprint 1 ``NotImplementedError``
skeleton with the pure functions in ``services/infonet/reputation/``.
Why this exists rather than callers importing the pure functions
directly: the adapter is the single integration boundary that future
sprints will extend (Sprint 3 wraps anti-gaming penalties around the
common-rep view, Sprint 4 extends the oracle-rep balance with
resolution-stake redistribution, Sprint 5 layers in dispute reversal).
By keeping callers on this adapter, the producer code never has to
change as those layers ship.
The adapter takes a ``chain_provider`` callable rather than reaching
into ``mesh_hashchain`` itself. Two reasons:
1. Tests pass a list of synthetic events directly — no hashchain
instance required, no fixture overhead.
2. Sprint 4 cutover decisions (parallel append surface vs unifying
``ACTIVE_APPEND_EVENT_TYPES``) won't ripple into reputation code.
Cross-cutting design rule: reputation reads are background work. They
must NEVER block a user-facing request. The adapter exposes only pure
synchronous functions because they ARE pure — caches at the adapter
layer (Sprint 3+) make repeat reads cheap. Callers that need real-time
freshness should call directly on each request; callers that can
tolerate staleness should poll a cached adapter instance.
"""
from __future__ import annotations
import time
from typing import Any, Callable, Iterable
from services.infonet.reputation import (
OracleRepBreakdown,
compute_common_rep,
compute_oracle_rep,
compute_oracle_rep_active,
compute_oracle_rep_lifetime,
decay_factor_for_age,
last_successful_prediction_ts,
)
from services.infonet.reputation.oracle_rep import compute_oracle_rep_breakdown
from services.infonet.time_validity import chain_majority_time
_ChainProvider = Callable[[], Iterable[dict[str, Any]]]
def _empty_chain() -> list[dict[str, Any]]:
return []
class InfonetReputationAdapter:
"""Project chain state into oracle/common rep views.
``chain_provider`` is a zero-arg callable returning an iterable of
chain events. Pass a closure that reads from
``mesh_hashchain.Infonet.events`` in production, or a literal list
in tests.
"""
def __init__(self, chain_provider: _ChainProvider | None = None) -> None:
self._chain_provider: _ChainProvider = chain_provider or _empty_chain
def _events(self) -> list[dict[str, Any]]:
return [e for e in self._chain_provider() if isinstance(e, dict)]
def oracle_rep(self, node_id: str) -> float:
return compute_oracle_rep(node_id, self._events())
def oracle_rep_breakdown(self, node_id: str) -> OracleRepBreakdown:
return compute_oracle_rep_breakdown(node_id, self._events())
def oracle_rep_lifetime(self, node_id: str) -> float:
return compute_oracle_rep_lifetime(node_id, self._events())
def oracle_rep_active(self, node_id: str, *, now: float | None = None) -> float:
events = self._events()
if now is None:
chain_now = chain_majority_time(events)
# Fall back to local clock only when the chain has no
# distinct-node history yet (genesis / fresh mesh). This is
# the only place a local clock leaks into governance —
# acceptable because there are no oracles to penalize yet.
now = chain_now if chain_now > 0 else time.time()
return compute_oracle_rep_active(node_id, events, now=now)
def common_rep(self, node_id: str) -> float:
return compute_common_rep(node_id, self._events())
def last_successful_prediction_ts(self, node_id: str) -> float | None:
return last_successful_prediction_ts(node_id, self._events())
def decay_factor(self, node_id: str, *, now: float | None = None) -> float:
events = self._events()
if now is None:
now = chain_majority_time(events) or time.time()
last_ts = last_successful_prediction_ts(node_id, events)
if last_ts is None:
return 0.0
days = max(0.0, (float(now) - last_ts) / 86400.0)
return decay_factor_for_age(days)
__all__ = ["InfonetReputationAdapter"]
@@ -0,0 +1,97 @@
"""Parallel ``SignedWriteKind`` enum for Infonet economy events.
Why a parallel enum and not extending the legacy one:
The legacy ``services/mesh/mesh_signed_events.SignedWriteKind`` is
imported in many places and changing it ripples through DM, gate, and
oracle code that we are not modifying in Sprint 1. Instead we publish a
parallel enum here for the new event types and rely on the hashchain
adapter to translate or co-route as needed.
Sprint 7+ may collapse these two enums once the upgrade-hash governance
is shipped and a coordinated cutover is possible.
"""
from __future__ import annotations
from enum import Enum
class InfonetSignedWriteKind(str, Enum):
# Reputation
UPREP = "uprep"
DOWNREP = "downrep"
# Markets / resolution-as-prediction
PREDICTION_CREATE = "prediction_create"
PREDICTION_PLACE = "prediction_place"
TRUTH_STAKE_PLACE = "truth_stake_place"
TRUTH_STAKE_RESOLVE = "truth_stake_resolve"
MARKET_SNAPSHOT = "market_snapshot"
EVIDENCE_SUBMIT = "evidence_submit"
RESOLUTION_STAKE = "resolution_stake"
BOOTSTRAP_RESOLUTION_VOTE = "bootstrap_resolution_vote"
RESOLUTION_FINALIZE = "resolution_finalize"
# Disputes
DISPUTE_OPEN = "dispute_open"
DISPUTE_STAKE = "dispute_stake"
DISPUTE_RESOLVE = "dispute_resolve"
# Gates (extend legacy GATE_CREATE / GATE_MESSAGE)
GATE_ENTER = "gate_enter"
GATE_EXIT = "gate_exit"
GATE_LOCK = "gate_lock"
# Gate shutdown lifecycle
GATE_SUSPEND_FILE = "gate_suspend_file"
GATE_SUSPEND_VOTE = "gate_suspend_vote"
GATE_SUSPEND_EXECUTE = "gate_suspend_execute"
GATE_SHUTDOWN_FILE = "gate_shutdown_file"
GATE_SHUTDOWN_VOTE = "gate_shutdown_vote"
GATE_SHUTDOWN_EXECUTE = "gate_shutdown_execute"
GATE_UNSUSPEND = "gate_unsuspend"
GATE_SHUTDOWN_APPEAL_FILE = "gate_shutdown_appeal_file"
GATE_SHUTDOWN_APPEAL_VOTE = "gate_shutdown_appeal_vote"
GATE_SHUTDOWN_APPEAL_RESOLVE = "gate_shutdown_appeal_resolve"
# Governance
PETITION_FILE = "petition_file"
PETITION_SIGN = "petition_sign"
PETITION_VOTE = "petition_vote"
CHALLENGE_FILE = "challenge_file"
CHALLENGE_VOTE = "challenge_vote"
PETITION_EXECUTE = "petition_execute"
# Upgrade-hash governance
UPGRADE_PROPOSE = "upgrade_propose"
UPGRADE_SIGN = "upgrade_sign"
UPGRADE_VOTE = "upgrade_vote"
UPGRADE_CHALLENGE = "upgrade_challenge"
UPGRADE_CHALLENGE_VOTE = "upgrade_challenge_vote"
UPGRADE_SIGNAL_READY = "upgrade_signal_ready"
UPGRADE_ACTIVATE = "upgrade_activate"
# Identity
NODE_REGISTER = "node_register"
IDENTITY_ROTATE = "identity_rotate"
CITIZENSHIP_CLAIM = "citizenship_claim"
# Economy
COIN_TRANSFER = "coin_transfer"
COIN_MINT = "coin_mint"
BOUNTY_CREATE = "bounty_create"
BOUNTY_CLAIM = "bounty_claim"
# Content
POST_CREATE = "post_create"
POST_REPLY = "post_reply"
INFONET_SIGNED_WRITE_KINDS: frozenset[InfonetSignedWriteKind] = frozenset(InfonetSignedWriteKind)
__all__ = [
"INFONET_SIGNED_WRITE_KINDS",
"InfonetSignedWriteKind",
]
@@ -0,0 +1,78 @@
"""Bootstrap mode — Argon2id PoW + eligibility + one-vote-per-node dedup.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.10 step 0.5.
Bootstrap mode replaces oracle-rep-weighted resolution with
**eligible-node-one-vote** for the first ``bootstrap_market_count``
(default 100) markets. Each eligible Heavy Node submits a
``bootstrap_resolution_vote`` event with an Argon2id PoW solution.
Key Sprint 8 invariants:
- **Argon2id is Heavy-Node-only.** Light Nodes lack the ≥64 MB RAM
required per computation. The PoW verifier does NOT run on Light
Nodes.
- **Salt = raw ``snapshot_event_hash`` bytes.** Hex-encoding or any
reformatting causes a consensus fork. The salt MUST be the exact
byte sequence of the snapshot event hash.
- **Leading-zero check is on RAW output bytes, MSB first.** Different
bit ordering causes a consensus fork.
- **Identity age is measured against ``market.snapshot.frozen_at``,
NOT against ``now``.** This is deterministic — every node computes
the same eligibility from the same chain state. Prevents clock
manipulation.
- **One-vote-per-node tie-break is stateless.** Among multiple votes
from the same node_id for the same market_id, the canonical vote is
the one with the LOWEST LEXICOGRAPHICAL ``event_hash``. Every node
selects the same canonical vote regardless of observation order.
- **Anti-DoS funnel runs cheapest-first.** Schema → signature →
identity age → predictor exclusion → phase + dedup → Argon2id.
Argon2id is last because it's the most expensive.
Sprint 8 ships the eligibility + dedup + ramp pipeline in pure
Python. ``verify_pow`` is a structural verifier that takes the
already-computed hash output as input — it does NOT call Argon2id
itself. Production callers wire this through ``privacy-core`` Rust.
A future sprint will add the Rust binding; until then, tests
synthesize valid hash outputs.
"""
from services.infonet.bootstrap.argon2id import (
canonical_pow_preimage,
has_leading_zero_bits,
verify_pow_structure,
)
from services.infonet.bootstrap.eligibility import (
EligibilityDecision,
is_identity_age_eligible,
validate_bootstrap_eligibility,
)
from services.infonet.bootstrap.filter_funnel import (
FunnelStage,
run_filter_funnel,
)
from services.infonet.bootstrap.one_vote_dedup import (
canonical_event_hash,
deduplicate_votes,
)
from services.infonet.bootstrap.ramp import (
ActiveFeatures,
compute_active_features,
network_node_count,
)
__all__ = [
"ActiveFeatures",
"EligibilityDecision",
"FunnelStage",
"canonical_event_hash",
"canonical_pow_preimage",
"compute_active_features",
"deduplicate_votes",
"has_leading_zero_bits",
"is_identity_age_eligible",
"network_node_count",
"run_filter_funnel",
"validate_bootstrap_eligibility",
"verify_pow_structure",
]
@@ -0,0 +1,146 @@
"""Argon2id canonicalization — preimage construction and leading-zero check.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.10 step 0.5
+ the ``CONFIG['bootstrap_pow_argon2id_*']`` comment block.
Two consensus-critical pieces of canonicalization:
1. **Canonical preimage** — exact byte sequence the Argon2id call
takes as `password`. UTF-8 encoded, "|"-delimited, no trailing
delimiter. Format:
"bootstrap_resolution_vote" || protocol_version || node_id ||
market_id || side || snapshot_event_hash || pow_nonce
The component order MUST match the spec exactly. Any deviation
causes consensus fork.
2. **Leading-zero check** — operates on RAW Argon2id output bytes,
MSB first (big-endian bit numbering). Difficulty N requires the
first N bits of the 32-byte output to be zero. With difficulty=16
that means the first 2 bytes are 0x00 0x00.
Sprint 8 does NOT execute Argon2id itself — the verifier here takes
an already-computed hash bytes object as input. Production callers
wire this through ``privacy-core`` Rust binding. A stub Python
implementation is intentionally absent to avoid accidental drift
between the Sprint 8 pure-Python path and the eventual Rust path.
"""
from __future__ import annotations
from services.infonet.config import CONFIG, IMMUTABLE_PRINCIPLES
def canonical_pow_preimage(
*,
node_id: str,
market_id: str,
side: str,
snapshot_event_hash: str,
pow_nonce: int,
protocol_version: str | None = None,
) -> bytes:
"""Build the canonical preimage for the Argon2id ``password`` input.
Returns UTF-8 bytes of ``"bootstrap_resolution_vote|<version>|<node>|
<market>|<side>|<snapshot_hash>|<nonce>"`` with NO trailing delimiter.
``protocol_version`` defaults to ``IMMUTABLE_PRINCIPLES['protocol_version']``
— it's pulled at call time so a hard-fork upgrade picks up the
new value automatically. Pass an explicit value when computing
against a hypothetical version (test scenarios).
"""
if not isinstance(node_id, str) or not node_id:
raise ValueError("node_id must be a non-empty string")
if not isinstance(market_id, str) or not market_id:
raise ValueError("market_id must be a non-empty string")
if side not in ("yes", "no"):
raise ValueError("side must be 'yes' or 'no'")
if not isinstance(snapshot_event_hash, str) or not snapshot_event_hash:
raise ValueError("snapshot_event_hash must be a non-empty string")
if not isinstance(pow_nonce, int) or isinstance(pow_nonce, bool) or pow_nonce < 0:
raise ValueError("pow_nonce must be a non-negative int")
pv = protocol_version if protocol_version is not None else IMMUTABLE_PRINCIPLES["protocol_version"]
if not isinstance(pv, str) or not pv:
raise ValueError("protocol_version must be a non-empty string")
parts = [
"bootstrap_resolution_vote",
pv,
node_id,
market_id,
side,
snapshot_event_hash,
str(pow_nonce),
]
return "|".join(parts).encode("utf-8")
def has_leading_zero_bits(raw_output: bytes, difficulty: int) -> bool:
"""``True`` if the first ``difficulty`` bits of ``raw_output``
are all zero.
Bit numbering: MSB first (big-endian). Byte order: as-is in the
raw output. With difficulty=16, the first two bytes must be
``\\x00\\x00``. With difficulty=4, the first byte must be in
``\\x00``..``\\x0f``.
"""
if not isinstance(raw_output, (bytes, bytearray)):
raise ValueError("raw_output must be bytes")
if not isinstance(difficulty, int) or difficulty < 0:
raise ValueError("difficulty must be a non-negative int")
if difficulty == 0:
return True
full_bytes, remaining_bits = divmod(difficulty, 8)
if len(raw_output) < full_bytes + (1 if remaining_bits else 0):
return False
for i in range(full_bytes):
if raw_output[i] != 0:
return False
if remaining_bits:
# The next byte's top `remaining_bits` bits must be zero.
next_byte = raw_output[full_bytes]
# Mask of the top `remaining_bits` bits (MSB first).
mask = ((0xFF << (8 - remaining_bits)) & 0xFF)
if (next_byte & mask) != 0:
return False
return True
def verify_pow_structure(
*,
raw_output: bytes,
difficulty: int | None = None,
expected_output_len: int | None = None,
) -> bool:
"""Verify the Argon2id output's structural properties.
- Output length must match ``expected_output_len`` (default
``CONFIG['bootstrap_pow_argon2id_output_len']``, fixed at 32).
- Leading zero check passes for ``difficulty`` (default
``CONFIG['bootstrap_pow_difficulty']``).
Does NOT verify that ``raw_output`` was actually produced by
Argon2id from the canonical preimage — that's the caller's job
via ``privacy-core`` Rust binding (or Python's ``argon2-cffi`` in
test environments). Sprint 8 keeps the cryptographic-call layer
as an external concern.
"""
if not isinstance(raw_output, (bytes, bytearray)):
return False
expected = expected_output_len if expected_output_len is not None else int(
CONFIG["bootstrap_pow_argon2id_output_len"]
)
if len(raw_output) != expected:
return False
diff = difficulty if difficulty is not None else int(CONFIG["bootstrap_pow_difficulty"])
return has_leading_zero_bits(raw_output, diff)
__all__ = [
"canonical_pow_preimage",
"has_leading_zero_bits",
"verify_pow_structure",
]
@@ -0,0 +1,129 @@
"""Bootstrap eligibility — identity age + predictor exclusion.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.10 step 0.5
(``is_bootstrap_eligible``).
Two gates:
1. **Identity age vs ``frozen_at`` (NOT ``now``).** Spec is explicit:
node.created_at + (bootstrap_min_identity_age_days * 86400)
<= market.snapshot.frozen_at
Measuring against the frozen snapshot timestamp keeps eligibility
deterministic — every node computes the same set from the same
chain state. Measuring against ``now`` would make eligibility
depend on local clock, which is a clock-manipulation attack
surface.
2. **Predictor exclusion.** Same as normal resolution:
``frozen_predictor_ids ∪ rotation_descendants(frozen_predictor_ids)``.
Reuses ``services.infonet.markets.resolution.excluded_predictor_ids``
(Sprint 4) — single source of truth.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Iterable
from services.infonet.config import CONFIG
from services.infonet.markets.resolution import excluded_predictor_ids
from services.infonet.markets.snapshot import find_snapshot
_SECONDS_PER_DAY = 86400.0
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
def _node_created_at(node_id: str, chain: Iterable[dict[str, Any]]) -> float | None:
"""First chain appearance of ``node_id`` — used as a proxy for
``node.created_at``. Per RULES §2.1: "Timestamp of first appearance
on chain". A ``node_register`` event is preferred when present;
otherwise the earliest event signed by ``node_id``.
"""
earliest_register: float | None = None
earliest_any: float | None = None
for ev in chain:
if not isinstance(ev, dict):
continue
author = ev.get("node_id")
if author != node_id:
continue
try:
ts = float(ev.get("timestamp") or 0.0)
except (TypeError, ValueError):
continue
if ev.get("event_type") == "node_register":
if earliest_register is None or ts < earliest_register:
earliest_register = ts
if earliest_any is None or ts < earliest_any:
earliest_any = ts
return earliest_register if earliest_register is not None else earliest_any
def is_identity_age_eligible(
node_id: str,
market_id: str,
chain: Iterable[dict[str, Any]],
*,
min_age_days: float | None = None,
) -> bool:
"""``True`` iff
``node.created_at + min_age_days * 86400 <= market.snapshot.frozen_at``.
Returns ``False`` if the snapshot doesn't exist yet, the node has
no chain history, or the timing condition fails.
"""
chain_list = list(chain)
snapshot = find_snapshot(market_id, chain_list)
if snapshot is None:
return False
try:
frozen_at = float(snapshot.get("frozen_at") or 0.0)
except (TypeError, ValueError):
return False
created_at = _node_created_at(node_id, chain_list)
if created_at is None:
return False
age_days = float(min_age_days if min_age_days is not None
else CONFIG["bootstrap_min_identity_age_days"])
threshold_ts = created_at + age_days * _SECONDS_PER_DAY
return threshold_ts <= frozen_at
@dataclass(frozen=True)
class EligibilityDecision:
eligible: bool
reason: str
def validate_bootstrap_eligibility(
node_id: str,
market_id: str,
chain: Iterable[dict[str, Any]],
) -> EligibilityDecision:
"""Combine identity-age + predictor-exclusion checks.
Used by the Sprint 8 anti-DoS funnel and by the bootstrap
resolution path itself.
"""
chain_list = list(chain)
if find_snapshot(market_id, chain_list) is None:
return EligibilityDecision(False, "snapshot_missing")
if not is_identity_age_eligible(node_id, market_id, chain_list):
return EligibilityDecision(False, "identity_age_too_young")
if node_id in excluded_predictor_ids(market_id, chain_list):
return EligibilityDecision(False, "predictor_excluded")
return EligibilityDecision(True, "ok")
__all__ = [
"EligibilityDecision",
"is_identity_age_eligible",
"validate_bootstrap_eligibility",
]
@@ -0,0 +1,76 @@
"""Anti-DoS filter funnel — cheapest-first validator chain.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.10 step 0.5
"Anti-DoS filter funnel (validation order for bootstrap_resolution_vote)".
Validation order (each stage short-circuits to reject):
1. Schema — format / required fields / enum sanity (free)
2. Signature — Ed25519 verify (~µs)
3. Identity age — vs snapshot.frozen_at (chain lookup)
4. Predictor — vs frozen_predictor_ids ∪ rotation_descendants
5. Phase + dedup
6. Argon2id PoW — most expensive (~64MB allocation + hash)
Why ordering matters: an attacker flooding malformed events should
never trigger the Argon2id work. Schema rejection happens first
(microseconds), so the funnel discards cheap-to-reject inputs cheap.
Sprint 8 ships the funnel as a list of ``FunnelStage`` callables.
Production callers compose them in order; each stage returns
``(accepted, reason)``.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Callable
_StageFn = Callable[[dict[str, Any]], tuple[bool, str]]
@dataclass(frozen=True)
class FunnelStage:
name: str
check: _StageFn
cost_tier: int
"""Cost ranking 1=cheapest, 6=most expensive. Used by tests to
confirm the stages are in the spec's ordering."""
def run_filter_funnel(
event: dict[str, Any],
stages: list[FunnelStage],
) -> tuple[bool, str]:
"""Run ``stages`` in order; return on the first failure.
Returns ``(True, "ok")`` if every stage passes, otherwise
``(False, "<stage>: <reason>")`` with the failing stage's name
and reason. The stage's own ``cost_tier`` is included in the
failing diagnostic so monitoring can spot when expensive stages
are doing the work cheap stages should have caught.
"""
if not isinstance(event, dict):
return False, "schema: event must be an object"
seen_tiers: list[int] = []
for stage in stages:
if seen_tiers and stage.cost_tier < max(seen_tiers):
# Sprint 8 invariant: tiers must be monotonically
# non-decreasing. A misordered funnel is a developer
# error, not an attacker input — fail loudly.
raise ValueError(
f"filter funnel out of order: stage {stage.name} "
f"has cost_tier={stage.cost_tier} after a higher tier"
)
seen_tiers.append(stage.cost_tier)
ok, reason = stage.check(event)
if not ok:
return False, f"{stage.name}: {reason}"
return True, "ok"
__all__ = [
"FunnelStage",
"run_filter_funnel",
]
@@ -0,0 +1,85 @@
"""Stateless one-vote-per-node dedup.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.10 step 0.5
("Phase valid + one-vote-per-node (stateless duplicate resolution)").
The protocol allows a node to submit only one
``bootstrap_resolution_vote`` per market_id. If duplicates appear
(retries, network split + heal, malicious flooding), the canonical
choice is **the vote with the lowest lexicographical event_hash**.
Key property: this is **stateless and order-independent**. Every node
computes the same canonical vote regardless of which duplicate they
saw first. No "last-write-wins" or "first-write-wins" — just the
hash comparison.
``event_hash = SHA-256(canonical_serialize(event))`` — must include
signature, payload, and metadata so two events with different
payloads produce different hashes.
"""
from __future__ import annotations
import hashlib
import json
from typing import Any, Iterable
def canonical_event_hash(event: dict[str, Any]) -> str:
"""SHA-256 of the canonically-serialized event.
Canonicalization: sorted keys, compact separators, UTF-8.
Includes every field on the event dict — payload, signature (if
present), node_id, timestamp, sequence, event_type. Different
inputs always produce different hashes.
"""
encoded = json.dumps(event, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
def deduplicate_votes(
market_id: str,
chain: Iterable[dict[str, Any]],
) -> list[dict[str, Any]]:
"""Return the canonical set of ``bootstrap_resolution_vote`` events
for ``market_id`` — at most one per ``node_id``, with the lowest
lexicographical ``canonical_event_hash`` chosen on collision.
The returned list is sorted by ``(node_id, event_hash)`` so the
output is deterministic for any chain ordering.
"""
candidates_per_node: dict[str, list[tuple[str, dict[str, Any]]]] = {}
for ev in chain:
if not isinstance(ev, dict):
continue
if ev.get("event_type") != "bootstrap_resolution_vote":
continue
if _payload(ev).get("market_id") != market_id:
continue
node = ev.get("node_id")
if not isinstance(node, str) or not node:
continue
h = canonical_event_hash(ev)
candidates_per_node.setdefault(node, []).append((h, ev))
canonical: list[dict[str, Any]] = []
for node, candidates in candidates_per_node.items():
# Lowest lexicographical event_hash wins. Stable secondary
# sort by sequence to make the choice deterministic for
# any duplicate hash (which would itself be a SHA-256
# collision — so academically impossible).
candidates.sort(key=lambda c: (c[0], int(c[1].get("sequence") or 0)))
canonical.append(candidates[0][1])
canonical.sort(key=lambda e: (e.get("node_id") or "", canonical_event_hash(e)))
return canonical
__all__ = [
"canonical_event_hash",
"deduplicate_votes",
]
+105
View File
@@ -0,0 +1,105 @@
"""Soft feature activation ramp — node-count milestones.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §1.2
(``CONFIG['bootstrap_threshold']`` comment) + the spec's general
"phase activation by network size" theme.
The protocol activates features in stages as the network grows. The
canonical milestones are 1k / 2k / 5k / 10k node count, but the
specific thresholds and which features they unlock are a Sprint 8+
design choice that's expected to evolve via governance.
Sprint 8 ships:
- ``network_node_count(chain)`` — distinct ``node_register`` events
on the chain.
- ``compute_active_features(chain)`` — returns an ``ActiveFeatures``
flag set indicating which protocol features are currently active.
Today's bindings:
- ``bootstrap_resolution_active`` — True while node count is below
``bootstrap_threshold`` (default 1000). Bootstrap-mode markets use
eligible-node-one-vote resolution.
- ``staked_resolution_active`` — True once node count crosses 1k.
Oracle-rep-weighted resolution staking is the primary mechanism.
- ``governance_petitions_active`` — True at 2k+. Petitions can be
filed.
- ``upgrade_governance_active`` — True at 5k+. Upgrade-hash
governance is unlocked.
- ``commoncoin_active`` — True at 10k+. CommonCoin minting starts.
These bindings are intentionally simple — production wiring will
read them via governance petitions that adjust ``bootstrap_threshold``
and the milestones themselves.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Iterable
from services.infonet.config import CONFIG
def network_node_count(chain: Iterable[dict[str, Any]]) -> int:
"""Distinct nodes that have appeared on the chain.
Counted as: distinct ``node_id`` from ``node_register`` events.
If no ``node_register`` events exist on the chain (e.g. test
chains that only synthesize markets/predictions), falls back to
distinct authoring nodes across all events. Production chains
will have the registers.
"""
registered: set[str] = set()
fallback: set[str] = set()
for ev in chain:
if not isinstance(ev, dict):
continue
node = ev.get("node_id")
if not isinstance(node, str) or not node:
continue
fallback.add(node)
if ev.get("event_type") == "node_register":
registered.add(node)
return len(registered) if registered else len(fallback)
@dataclass(frozen=True)
class ActiveFeatures:
bootstrap_resolution_active: bool
staked_resolution_active: bool
governance_petitions_active: bool
upgrade_governance_active: bool
commoncoin_active: bool
node_count: int
# Milestone thresholds promoted to CONFIG 2026-04-28 (Sprint 8 polish).
# Governance can now tune them via petition; the cross-field invariant
# in config.py enforces strict ascending order across the four tiers.
def compute_active_features(chain: Iterable[dict[str, Any]]) -> ActiveFeatures:
chain_list = [e for e in chain if isinstance(e, dict)]
n = network_node_count(chain_list)
bootstrap_threshold = int(CONFIG["bootstrap_threshold"])
return ActiveFeatures(
# Bootstrap resolution is active until the network crosses the
# bootstrap_threshold. Once crossed, it's still allowed for
# bootstrap-indexed markets, but new markets default to
# staked resolution.
bootstrap_resolution_active=n < bootstrap_threshold,
staked_resolution_active=n >= int(CONFIG["ramp_staked_resolution_threshold"]),
governance_petitions_active=n >= int(CONFIG["ramp_petitions_threshold"]),
upgrade_governance_active=n >= int(CONFIG["ramp_upgrade_threshold"]),
commoncoin_active=n >= int(CONFIG["ramp_commoncoin_threshold"]),
node_count=n,
)
__all__ = [
"ActiveFeatures",
"compute_active_features",
"network_node_count",
]
+519
View File
@@ -0,0 +1,519 @@
"""Constitutional + governable parameters for the Infonet economy.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §1.
- ``IMMUTABLE_PRINCIPLES`` — constitutional, exposed as a ``MappingProxyType``.
Mutation attempts raise ``TypeError`` at the language level. New keys can
only be added through upgrade-hash governance (Sprint 7) which is itself
governed by these principles — i.e. a hard fork.
- ``CONFIG`` — amendable parameters. Live (mutable) dict; all writes go
through ``validate_petition_value`` first. The dict itself is a
module-level singleton — the governance DSL executor (Sprint 7) is the
only intended writer in production. Tests must use
``reset_config_for_tests`` to restore baseline.
- ``CONFIG_SCHEMA`` — per-key bounds and types. Itself an immutable
``MappingProxyType``. New schema entries require a hard fork (same flow
as ``IMMUTABLE_PRINCIPLES``).
- ``CROSS_FIELD_INVARIANTS`` — ordered-pair invariants checked AFTER all
updates in a ``BATCH_UPDATE_PARAMS``. Spec note: supermajority must
always exceed quorum, etc.
This file is read by every subsequent sprint. Adding a CONFIG key without
adding a matching CONFIG_SCHEMA entry is a Sprint 1 invariant violation
and is asserted by the tests.
"""
from __future__ import annotations
from copy import deepcopy
from types import MappingProxyType
from typing import Any
class InvalidPetition(ValueError):
"""Raised by ``validate_petition_value`` and the governance DSL executor.
Signals that a proposed CONFIG mutation is rejected by the schema or by
a cross-field invariant. The DSL executor (Sprint 7) catches this and
rolls back the petition — never partially applies.
"""
# ─── Constitutional principles ───────────────────────────────────────────
# Immutable. Mutation attempts raise TypeError at the language level
# because MappingProxyType is read-only.
#
# RULES_SKELETON.md §1.1 — adding a key here is a hard fork.
IMMUTABLE_PRINCIPLES: MappingProxyType = MappingProxyType({
"oracle_rep_source": "predictions_only",
"hashchain_append_only": True,
"audit_public": True,
"identity_permissionless": True,
"signature_required": True,
"redemption_path_exists": True,
"coin_governance_firewall": True,
"protocol_version": "0.1.0",
})
# ─── Amendable parameters ────────────────────────────────────────────────
# RULES_SKELETON.md §1.2.
# Mutable dict. Production writes only via the Sprint 7 governance DSL
# executor which calls validate_petition_value first.
_BASELINE_CONFIG: dict[str, Any] = {
# ── Reputation ──
"vote_decay_days": 90,
"min_rep_to_vote": 3,
"min_rep_to_create_gate": 10,
"bootstrap_threshold": 1000,
"weekly_vote_base": 5,
"weekly_vote_per_oracle": 10,
"daily_vote_limit_per_target": 1,
# ── Oracle Rep ──
"oracle_min_earned": 0.01,
"farming_soft_threshold": 0.60,
"farming_hard_threshold": 0.80,
"farming_easy_bet_cutoff": 0.80,
"subjective_oracle_rep_mint": False,
# ── Market Liquidity ──
"min_market_participants": 5,
"min_market_total_stake": 10.0,
# ── Resolution Phase ──
"evidence_window_hours": 48,
"resolution_window_hours": 72,
"evidence_bond_cost": 2.0,
"evidence_first_bonus": 0.5,
"resolution_supermajority": 0.75,
"min_resolution_stake_total": 20.0,
"resolution_loser_burn_pct": 0.02,
"data_unavailable_threshold": 0.33,
"resolution_stalemate_burn_pct": 0.02,
# ── Governance Decay ──
"governance_decay_days": 90,
"governance_decay_factor": 0.50,
# ── Time Validity ──
"max_future_event_drift_sec": 300,
"phase_boundary_stale_reject": True,
# ── Identity Rotation ──
"rotation_blocked_during_stakes": True,
# ── Anti-Gaming ──
"vcs_min_weight": 0.10,
"clustering_min_weight": 0.20,
"temporal_burst_window_sec": 300,
"temporal_burst_min_upreps": 5,
"progressive_penalty_base": 1.0,
# Common-rep base formula multiplier (RULES §3.3). Promoted from
# Sprint 2's module-private constant 2026-04-28 so governance can
# tune the default common-rep payout per uprep.
"common_rep_weight_factor": 0.10,
# Progressive-penalty trigger threshold — average correlation
# score above which the whale-deterrence multiplier kicks in
# (Sprint 3 polish 2026-04-28). 0.0 = disabled (Sprint 3 default
# behavior preserved).
"progressive_penalty_threshold": 0.0,
# ── Gates ──
"gate_ratification_rep": 50,
"gate_lock_cost_per_member": 10,
"gate_lock_min_members": 5,
"gate_creation_rate_limit": 5,
# ── Truth Stakes ──
"truth_stake_min_days": 1,
"truth_stake_max_days": 7,
"truth_stake_grace_hours": 24,
"truth_stake_max_extensions": 3,
"truth_stake_tie_burn_pct": 0.20,
"truth_stake_self_stake": False,
# ── Dispute Resolution ──
"dispute_window_days": 7,
"dispute_common_rep_stakeable": True,
# ── CommonCoin ──
"monthly_mint_amount": 100000,
"ubi_share_pct": 0.50,
"oracle_dividend_pct": 0.50,
"citizenship_sacrifice_cost": 10,
"year1_max_coins_per_node": 10000,
# ── Governance ──
"petition_filing_cost": 15,
"petition_signature_threshold": 0.25,
"petition_signature_window_days": 14,
"petition_vote_window_days": 7,
"petition_supermajority": 0.67,
"petition_quorum": 0.30,
"challenge_filing_cost": 25,
"challenge_window_hours": 48,
# ── Upgrade-Hash Governance ──
"upgrade_filing_cost": 25,
"upgrade_signature_threshold": 0.25,
"upgrade_signature_window_days": 14,
"upgrade_vote_window_days": 14,
"upgrade_supermajority": 0.80,
"upgrade_quorum": 0.40,
"upgrade_activation_threshold": 0.67,
"upgrade_activation_window_days": 30,
"upgrade_challenge_window_hours": 48,
# ── Gate Shutdown ──
"gate_suspend_filing_cost": 15,
"gate_shutdown_filing_cost": 25,
"gate_suspend_supermajority": 0.67,
"gate_suspend_locked_supermajority": 0.75,
"gate_shutdown_supermajority": 0.75,
"gate_shutdown_locked_supermajority": 0.80,
"gate_shutdown_quorum": 0.30,
"gate_suspend_duration_days": 30,
"gate_shutdown_execution_delay_days": 7,
"gate_shutdown_cooldown_days": 90,
"gate_shutdown_fail_penalty_days": 30,
"gate_shutdown_appeal_filing_cost": 20,
"gate_shutdown_appeal_window_hours": 48,
"gate_shutdown_appeal_vote_window_days": 7,
"gate_shutdown_appeal_supermajority": 0.67,
"gate_shutdown_appeal_locked_supermajority": 0.75,
"gate_shutdown_appeal_quorum": 0.30,
# ── Market Creation ──
"market_creation_bond": 3,
"market_creation_bond_return_threshold": 5,
# ── Bootstrap ──
"bootstrap_market_count": 100,
"bootstrap_evidence_bond_cost": 0,
"bootstrap_resolution_mode": "eligible_node_one_vote",
"bootstrap_resolution_supermajority": 0.75,
"bootstrap_min_identity_age_days": 3,
"bootstrap_pow_algorithm": "argon2id",
"bootstrap_pow_argon2id_version": 0x13,
"bootstrap_pow_argon2id_m": 65536,
"bootstrap_pow_argon2id_t": 3,
"bootstrap_pow_argon2id_p": 1,
"bootstrap_pow_argon2id_output_len": 32,
"bootstrap_pow_difficulty": 16,
# ── Ramp milestones (Sprint 8 polish 2026-04-28) ──
# Network-size thresholds at which features activate. Promoted
# from Sprint 8 hardcoded constants so governance can tune them.
# Values denote the minimum distinct-node count required.
"ramp_staked_resolution_threshold": 1000,
"ramp_petitions_threshold": 2000,
"ramp_upgrade_threshold": 5000,
"ramp_commoncoin_threshold": 10000,
}
CONFIG: dict[str, Any] = deepcopy(_BASELINE_CONFIG)
def reset_config_for_tests() -> None:
"""Restore CONFIG to the pre-petition baseline. Tests only.
Used by the autouse fixture in ``services/infonet/tests/conftest.py`` so
that one test mutating CONFIG (via a simulated petition execution)
cannot leak state into the next test.
"""
CONFIG.clear()
CONFIG.update(deepcopy(_BASELINE_CONFIG))
# ─── CONFIG schema (per-key bounds) ──────────────────────────────────────
# RULES_SKELETON.md §1.3.
# Itself an immutable structure — new keys require upgrade-hash governance
# (a hard fork). validate_petition_value rejects any key not present here.
_SCHEMA_TYPES = {
"int": (int,),
"float": (int, float),
"bool": (bool,),
"str": (str,),
}
_CONFIG_SCHEMA_BACKING: dict[str, MappingProxyType] = {
# ── Reputation ──
"vote_decay_days": MappingProxyType({"type": "int", "min": 7, "max": 365}),
"min_rep_to_vote": MappingProxyType({"type": "int", "min": 0, "max": 100}),
"min_rep_to_create_gate": MappingProxyType({"type": "int", "min": 1, "max": 1000}),
"bootstrap_threshold": MappingProxyType({"type": "int", "min": 100, "max": 100000}),
"weekly_vote_base": MappingProxyType({"type": "int", "min": 1, "max": 100}),
"weekly_vote_per_oracle": MappingProxyType({"type": "int", "min": 1, "max": 1000}),
"daily_vote_limit_per_target": MappingProxyType({"type": "int", "min": 1, "max": 10}),
# ── Oracle Rep ──
"oracle_min_earned": MappingProxyType({"type": "float", "min": 0.001, "max": 1.0}),
"farming_soft_threshold": MappingProxyType({"type": "float", "min": 0.10, "max": 0.95}),
"farming_hard_threshold": MappingProxyType({"type": "float", "min": 0.20, "max": 0.99}),
"farming_easy_bet_cutoff": MappingProxyType({"type": "float", "min": 0.50, "max": 0.99}),
"subjective_oracle_rep_mint": MappingProxyType({"type": "bool"}),
# ── Market Liquidity ──
"min_market_participants": MappingProxyType({"type": "int", "min": 2, "max": 100}),
"min_market_total_stake": MappingProxyType({"type": "float", "min": 1.0, "max": 1000.0}),
# ── Resolution ──
"evidence_window_hours": MappingProxyType({"type": "int", "min": 12, "max": 168}),
"resolution_window_hours": MappingProxyType({"type": "int", "min": 24, "max": 336}),
"evidence_bond_cost": MappingProxyType({"type": "float", "min": 0.5, "max": 50.0}),
"evidence_first_bonus": MappingProxyType({"type": "float", "min": 0.0, "max": 10.0}),
"resolution_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.95}),
"min_resolution_stake_total": MappingProxyType({"type": "float", "min": 5.0, "max": 500.0}),
"resolution_loser_burn_pct": MappingProxyType({"type": "float", "min": 0.0, "max": 0.10}),
"data_unavailable_threshold": MappingProxyType({"type": "float", "min": 0.10, "max": 0.50}),
"resolution_stalemate_burn_pct": MappingProxyType({"type": "float", "min": 0.0, "max": 0.10}),
# ── Governance Decay ──
"governance_decay_days": MappingProxyType({"type": "int", "min": 7, "max": 365}),
"governance_decay_factor": MappingProxyType({"type": "float", "min": 0.10, "max": 0.99}),
# ── Time Validity ──
"max_future_event_drift_sec": MappingProxyType({"type": "int", "min": 30, "max": 3600}),
"phase_boundary_stale_reject": MappingProxyType({"type": "bool"}),
# ── Identity Rotation ──
"rotation_blocked_during_stakes": MappingProxyType({"type": "bool"}),
# ── Anti-Gaming ──
"vcs_min_weight": MappingProxyType({"type": "float", "min": 0.0, "max": 1.0}),
"clustering_min_weight": MappingProxyType({"type": "float", "min": 0.0, "max": 1.0}),
"temporal_burst_window_sec": MappingProxyType({"type": "int", "min": 30, "max": 3600}),
"temporal_burst_min_upreps": MappingProxyType({"type": "int", "min": 2, "max": 100}),
"progressive_penalty_base": MappingProxyType({"type": "float", "min": 0.1, "max": 100.0}),
"common_rep_weight_factor": MappingProxyType({"type": "float", "min": 0.0, "max": 1.0}),
"progressive_penalty_threshold": MappingProxyType({"type": "float", "min": 0.0, "max": 1.0}),
# ── Gates ──
"gate_ratification_rep": MappingProxyType({"type": "int", "min": 1, "max": 10000}),
"gate_lock_cost_per_member": MappingProxyType({"type": "int", "min": 1, "max": 1000}),
"gate_lock_min_members": MappingProxyType({"type": "int", "min": 2, "max": 1000}),
"gate_creation_rate_limit": MappingProxyType({"type": "int", "min": 1, "max": 100}),
# ── Truth Stakes ──
"truth_stake_min_days": MappingProxyType({"type": "int", "min": 1, "max": 30}),
"truth_stake_max_days": MappingProxyType({"type": "int", "min": 1, "max": 90}),
"truth_stake_grace_hours": MappingProxyType({"type": "int", "min": 1, "max": 168}),
"truth_stake_max_extensions": MappingProxyType({"type": "int", "min": 0, "max": 10}),
"truth_stake_tie_burn_pct": MappingProxyType({"type": "float", "min": 0.0, "max": 0.50}),
"truth_stake_self_stake": MappingProxyType({"type": "bool"}),
# ── Dispute Resolution ──
"dispute_window_days": MappingProxyType({"type": "int", "min": 1, "max": 30}),
"dispute_common_rep_stakeable": MappingProxyType({"type": "bool"}),
# ── CommonCoin ──
"monthly_mint_amount": MappingProxyType({"type": "int", "min": 1, "max": 1_000_000_000}),
"ubi_share_pct": MappingProxyType({"type": "float", "min": 0.0, "max": 1.0}),
"oracle_dividend_pct": MappingProxyType({"type": "float", "min": 0.0, "max": 1.0}),
"citizenship_sacrifice_cost": MappingProxyType({"type": "int", "min": 1, "max": 1000}),
"year1_max_coins_per_node": MappingProxyType({"type": "int", "min": 1, "max": 1_000_000_000}),
# ── Governance ──
"petition_filing_cost": MappingProxyType({"type": "int", "min": 1, "max": 100}),
"petition_signature_threshold": MappingProxyType({"type": "float", "min": 0.05, "max": 0.50}),
"petition_signature_window_days": MappingProxyType({"type": "int", "min": 1, "max": 60}),
"petition_vote_window_days": MappingProxyType({"type": "int", "min": 1, "max": 30}),
"petition_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.95}),
"petition_quorum": MappingProxyType({"type": "float", "min": 0.10, "max": 0.80}),
"challenge_filing_cost": MappingProxyType({"type": "int", "min": 1, "max": 200}),
"challenge_window_hours": MappingProxyType({"type": "int", "min": 12, "max": 168}),
# ── Upgrade-Hash Governance ──
"upgrade_filing_cost": MappingProxyType({"type": "int", "min": 1, "max": 200}),
"upgrade_signature_threshold": MappingProxyType({"type": "float", "min": 0.05, "max": 0.50}),
"upgrade_signature_window_days": MappingProxyType({"type": "int", "min": 1, "max": 60}),
"upgrade_vote_window_days": MappingProxyType({"type": "int", "min": 1, "max": 60}),
"upgrade_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.99}),
"upgrade_quorum": MappingProxyType({"type": "float", "min": 0.10, "max": 0.95}),
"upgrade_activation_threshold": MappingProxyType({"type": "float", "min": 0.51, "max": 0.99}),
"upgrade_activation_window_days": MappingProxyType({"type": "int", "min": 1, "max": 90}),
"upgrade_challenge_window_hours": MappingProxyType({"type": "int", "min": 12, "max": 168}),
# ── Gate Shutdown ──
"gate_suspend_filing_cost": MappingProxyType({"type": "int", "min": 1, "max": 200}),
"gate_shutdown_filing_cost": MappingProxyType({"type": "int", "min": 1, "max": 200}),
"gate_suspend_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.95}),
"gate_suspend_locked_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.95}),
"gate_shutdown_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.99}),
"gate_shutdown_locked_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.99}),
"gate_shutdown_quorum": MappingProxyType({"type": "float", "min": 0.10, "max": 0.80}),
"gate_suspend_duration_days": MappingProxyType({"type": "int", "min": 1, "max": 365}),
"gate_shutdown_execution_delay_days": MappingProxyType({"type": "int", "min": 1, "max": 90}),
"gate_shutdown_cooldown_days": MappingProxyType({"type": "int", "min": 7, "max": 365}),
"gate_shutdown_fail_penalty_days": MappingProxyType({"type": "int", "min": 0, "max": 365}),
"gate_shutdown_appeal_filing_cost": MappingProxyType({"type": "int", "min": 1, "max": 200}),
"gate_shutdown_appeal_window_hours": MappingProxyType({"type": "int", "min": 12, "max": 168}),
"gate_shutdown_appeal_vote_window_days": MappingProxyType({"type": "int", "min": 1, "max": 30}),
"gate_shutdown_appeal_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.95}),
"gate_shutdown_appeal_locked_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.95}),
"gate_shutdown_appeal_quorum": MappingProxyType({"type": "float", "min": 0.10, "max": 0.80}),
# ── Market Creation ──
"market_creation_bond": MappingProxyType({"type": "int", "min": 0, "max": 1000}),
"market_creation_bond_return_threshold": MappingProxyType({"type": "int", "min": 1, "max": 1000}),
# ── Bootstrap ──
"bootstrap_market_count": MappingProxyType({"type": "int", "min": 0, "max": 100000}),
"bootstrap_evidence_bond_cost": MappingProxyType({"type": "float", "min": 0.0, "max": 50.0}),
"bootstrap_resolution_mode": MappingProxyType({"type": "str", "enum": ("eligible_node_one_vote",)}),
"bootstrap_resolution_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.95}),
"bootstrap_min_identity_age_days": MappingProxyType({"type": "int", "min": 0, "max": 365}),
"bootstrap_pow_algorithm": MappingProxyType({"type": "str", "enum": ("argon2id",)}),
"bootstrap_pow_argon2id_version": MappingProxyType({"type": "int", "enum": (0x13,)}),
"bootstrap_pow_argon2id_m": MappingProxyType({"type": "int", "min": 8192, "max": 1_048_576}),
"bootstrap_pow_argon2id_t": MappingProxyType({"type": "int", "min": 1, "max": 100}),
"bootstrap_pow_argon2id_p": MappingProxyType({"type": "int", "min": 1, "max": 16}),
"bootstrap_pow_argon2id_output_len": MappingProxyType({"type": "int", "enum": (32,)}),
"bootstrap_pow_difficulty": MappingProxyType({"type": "int", "min": 1, "max": 64}),
# ── Ramp milestones ──
"ramp_staked_resolution_threshold": MappingProxyType({"type": "int", "min": 1, "max": 10_000_000}),
"ramp_petitions_threshold": MappingProxyType({"type": "int", "min": 1, "max": 10_000_000}),
"ramp_upgrade_threshold": MappingProxyType({"type": "int", "min": 1, "max": 10_000_000}),
"ramp_commoncoin_threshold": MappingProxyType({"type": "int", "min": 1, "max": 10_000_000}),
}
CONFIG_SCHEMA: MappingProxyType = MappingProxyType(_CONFIG_SCHEMA_BACKING)
# ─── Cross-field invariants ──────────────────────────────────────────────
# RULES_SKELETON.md §1.3.
# Each tuple is (left_key, op, right_key). Only ">" supported today —
# extend the dispatch in validate_cross_field_invariants when new ops
# appear in the spec.
CROSS_FIELD_INVARIANTS: tuple[tuple[str, str, str], ...] = (
("petition_supermajority", ">", "petition_quorum"),
("resolution_supermajority", ">", "data_unavailable_threshold"),
("upgrade_supermajority", ">", "upgrade_quorum"),
("gate_shutdown_supermajority", ">", "gate_shutdown_quorum"),
("gate_suspend_supermajority", ">", "gate_shutdown_quorum"),
("farming_hard_threshold", ">", "farming_soft_threshold"),
("truth_stake_max_days", ">", "truth_stake_min_days"),
("upgrade_filing_cost", ">", "petition_filing_cost"),
# Ramp milestones must be in strict ascending order so each tier
# genuinely activates additional capability (Sprint 8 polish
# 2026-04-28).
("ramp_petitions_threshold", ">", "ramp_staked_resolution_threshold"),
("ramp_upgrade_threshold", ">", "ramp_petitions_threshold"),
("ramp_commoncoin_threshold", ">", "ramp_upgrade_threshold"),
)
# ─── Validators (used by the Sprint 7 governance DSL executor) ───────────
def validate_petition_value(
key: str,
value: Any,
current_config: dict[str, Any] | None = None,
) -> None:
"""Validate one (key, value) pair against ``CONFIG_SCHEMA``.
Raises ``InvalidPetition`` on any failure. Returns ``None`` on success.
``current_config`` is accepted for API symmetry with the spec snippet
in RULES §1.3 — current Sprint 1 logic doesn't need it. Future
cross-field-aware updates may consult it.
"""
del current_config # deliberately unused — see docstring
schema = CONFIG_SCHEMA.get(key)
if schema is None:
raise InvalidPetition(f"No schema for key: {key}")
type_name = schema["type"]
expected = _SCHEMA_TYPES.get(type_name)
if expected is None:
raise InvalidPetition(f"Schema for {key} has unknown type: {type_name}")
if type_name == "bool":
if not isinstance(value, bool):
raise InvalidPetition(
f"Type mismatch for {key}: expected bool, got {type(value).__name__}"
)
elif type_name == "int":
if isinstance(value, bool) or not isinstance(value, int):
raise InvalidPetition(
f"Type mismatch for {key}: expected int, got {type(value).__name__}"
)
elif type_name == "float":
if isinstance(value, bool) or not isinstance(value, expected):
raise InvalidPetition(
f"Type mismatch for {key}: expected float, got {type(value).__name__}"
)
else: # str
if not isinstance(value, expected):
raise InvalidPetition(
f"Type mismatch for {key}: expected {type_name}, got {type(value).__name__}"
)
if "min" in schema and value < schema["min"]:
raise InvalidPetition(f"{key}={value} below minimum {schema['min']}")
if "max" in schema and value > schema["max"]:
raise InvalidPetition(f"{key}={value} above maximum {schema['max']}")
if "enum" in schema and value not in schema["enum"]:
raise InvalidPetition(f"{key}={value} not in allowed values {tuple(schema['enum'])}")
def validate_cross_field_invariants(config: dict[str, Any]) -> None:
"""Check every entry of ``CROSS_FIELD_INVARIANTS`` against ``config``.
Called by the DSL executor AFTER all updates from a single petition
payload have been applied to a candidate config dict. Raises
``InvalidPetition`` on the first violation. The candidate config is
discarded by the executor when this raises.
"""
for left_key, op, right_key in CROSS_FIELD_INVARIANTS:
if left_key not in config:
raise InvalidPetition(f"Cross-field invariant references missing key: {left_key}")
if right_key not in config:
raise InvalidPetition(f"Cross-field invariant references missing key: {right_key}")
left_val = config[left_key]
right_val = config[right_key]
if op == ">":
if not (left_val > right_val):
raise InvalidPetition(
f"Cross-field invariant violated: {left_key}={left_val} must be > "
f"{right_key}={right_val}"
)
else:
raise InvalidPetition(f"Unknown cross-field operator: {op}")
def validate_config_schema_completeness() -> None:
"""Sprint 1 invariant: every CONFIG key has a matching CONFIG_SCHEMA entry.
Raises ``InvalidPetition`` listing missing keys. Called both from the
Sprint 1 adversarial test and from the DSL executor on startup.
"""
missing = sorted(set(CONFIG.keys()) - set(CONFIG_SCHEMA.keys()))
extra = sorted(set(CONFIG_SCHEMA.keys()) - set(CONFIG.keys()))
if missing:
raise InvalidPetition(f"CONFIG keys without CONFIG_SCHEMA entry: {missing}")
if extra:
raise InvalidPetition(f"CONFIG_SCHEMA keys without CONFIG entry: {extra}")
__all__ = [
"CONFIG",
"CONFIG_SCHEMA",
"CROSS_FIELD_INVARIANTS",
"IMMUTABLE_PRINCIPLES",
"InvalidPetition",
"reset_config_for_tests",
"validate_config_schema_completeness",
"validate_cross_field_invariants",
"validate_petition_value",
]
+106
View File
@@ -0,0 +1,106 @@
"""Event construction helpers for the Infonet economy.
A thin layer over ``services/infonet/schema.py``: each public function
builds a payload dict for one event type, validates it, and returns it.
The caller is responsible for signing the event and routing it through
``services/infonet/adapters/hashchain_adapter.py`` for actual append.
Sprint 1 scope: payload builders + validation. No chain writes. The
hashchain adapter's ``append_infonet_event`` is the eventual integration
point — see ``adapters/hashchain_adapter.py``.
Why a builder layer and not free-form dicts:
- Centralizes the canonical field set per event_type so callers can't
drift from the schema.
- Allows future sprints to attach deterministic computation (e.g.
``probability_at_bet`` reconstruction in Sprint 4) without changing
callers.
- Matches the "events extend, never replace" rule from the plan §3.1 —
the legacy event constructors in ``mesh_schema.py`` keep working
unchanged; new event types live here.
"""
from __future__ import annotations
from typing import Any
from services.infonet.schema import (
INFONET_ECONOMY_EVENT_TYPES,
validate_infonet_event_payload,
)
class EventConstructionError(ValueError):
"""Raised when a payload fails validation at build time.
Distinct from chain-level errors (signature, replay, sequence) —
those originate in the hashchain adapter, not here.
"""
def build_event(event_type: str, payload: dict[str, Any]) -> dict[str, Any]:
"""Validate and return a payload for ``event_type``.
The returned dict is a shallow copy — callers can attach signature,
sequence, public_key, etc. before passing it to the hashchain
adapter for append.
"""
if event_type not in INFONET_ECONOMY_EVENT_TYPES:
raise EventConstructionError(
f"event_type {event_type!r} is not in INFONET_ECONOMY_EVENT_TYPES"
)
payload = dict(payload or {})
ok, reason = validate_infonet_event_payload(event_type, payload)
if not ok:
raise EventConstructionError(f"{event_type}: {reason}")
return payload
# ─── Convenience builders ────────────────────────────────────────────────
# Sprint 1 ships only a representative slice. Full per-type builders for
# the producing modules (markets/, gates/, governance/, ...) live in
# their respective sprints — they will all funnel through ``build_event``
# so this module stays the single validation choke point.
def build_uprep(target_node_id: str, target_event_id: str) -> dict[str, Any]:
return build_event("uprep", {
"target_node_id": target_node_id,
"target_event_id": target_event_id,
})
def build_citizenship_claim(sacrifice_amount: int) -> dict[str, Any]:
return build_event("citizenship_claim", {"sacrifice_amount": sacrifice_amount})
def build_petition_file(
petition_id: str,
petition_payload: dict[str, Any],
) -> dict[str, Any]:
return build_event("petition_file", {
"petition_id": petition_id,
"petition_payload": petition_payload,
})
def build_petition_vote(petition_id: str, vote: str) -> dict[str, Any]:
return build_event("petition_vote", {"petition_id": petition_id, "vote": vote})
def build_node_register(public_key: str, public_key_algo: str, node_class: str) -> dict[str, Any]:
return build_event("node_register", {
"public_key": public_key,
"public_key_algo": public_key_algo,
"node_class": node_class,
})
__all__ = [
"EventConstructionError",
"build_citizenship_claim",
"build_event",
"build_node_register",
"build_petition_file",
"build_petition_vote",
"build_uprep",
]
@@ -0,0 +1,77 @@
"""Gate sacrifice + locking + shutdown lifecycle (Sprint 6).
Pure-function design: every entry point reads the chain and returns a
deterministic value. State (member set / suspended_until / shutdown
status / appeal status) is derived, never stored.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.16, §5.3,
§5.5.
"""
from services.infonet.gates.locking import (
LockedGateState,
is_locked,
locked_at,
locked_by,
validate_lock_request,
)
from services.infonet.gates.ratification import (
RATIFICATION_THRESHOLD,
cumulative_member_oracle_rep,
is_ratified,
)
from services.infonet.gates.sacrifice import (
EntryDecision,
EntryRefusal,
can_enter,
compute_member_set,
is_member,
)
from services.infonet.gates.shutdown.appeal import (
AppealValidation,
paused_execution_remaining_sec,
validate_appeal_filing,
)
from services.infonet.gates.shutdown.shutdown import (
ShutdownState,
compute_shutdown_state,
validate_shutdown_filing,
)
from services.infonet.gates.shutdown.suspend import (
SuspensionState,
compute_suspension_state,
validate_suspend_filing,
)
from services.infonet.gates.state import (
GateMeta,
events_for_gate,
get_gate_meta,
)
__all__ = [
"AppealValidation",
"EntryDecision",
"EntryRefusal",
"GateMeta",
"LockedGateState",
"RATIFICATION_THRESHOLD",
"ShutdownState",
"SuspensionState",
"can_enter",
"compute_member_set",
"compute_shutdown_state",
"compute_suspension_state",
"cumulative_member_oracle_rep",
"events_for_gate",
"get_gate_meta",
"is_locked",
"is_member",
"is_ratified",
"locked_at",
"locked_by",
"paused_execution_remaining_sec",
"validate_appeal_filing",
"validate_lock_request",
"validate_shutdown_filing",
"validate_suspend_filing",
]
+153
View File
@@ -0,0 +1,153 @@
"""Gate locking — "constitutionalize-a-gate".
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §5.3 step 4 +
``CONFIG['gate_lock_cost_per_member']`` / ``CONFIG['gate_lock_min_members']``.
Locking semantics:
- Each ``gate_lock`` event records one member contributing
``CONFIG['gate_lock_cost_per_member']`` (default 10) common rep.
- A gate is "locked" once ≥ ``CONFIG['gate_lock_min_members']``
(default 5) distinct current members have each emitted a valid
``gate_lock`` event.
- Once locked, the gate's rules become immutable — no governance
petition can modify them. Only an upgrade-hash governance event
(out of scope for Sprint 6) can amend a locked gate's rules.
Validation rules for an incoming ``gate_lock`` event (callers in
production should run these *before* emitting):
- The gate exists.
- The locker is a current member.
- The locker hasn't already locked this gate (one lock per node).
- The locker has paid (the burn happens at emit time; this module
asserts the schematic ``lock_cost`` matches CONFIG).
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Iterable
from services.infonet.config import CONFIG
from services.infonet.gates.sacrifice import compute_member_set
from services.infonet.gates.state import events_for_gate
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
def _lock_cost_per_member() -> int:
return int(CONFIG["gate_lock_cost_per_member"])
def _lock_min_members() -> int:
return int(CONFIG["gate_lock_min_members"])
@dataclass(frozen=True)
class LockedGateState:
locked: bool
locked_at: float | None
locked_by: tuple[str, ...]
def _collect_lock_contributions(
gate_id: str,
chain: Iterable[dict[str, Any]],
) -> list[tuple[str, float]]:
"""Return ``[(node_id, timestamp)]`` for each accepted ``gate_lock``
event in chain order. Subsequent locks from the same node are
ignored (one lock per node)."""
chain_list = list(chain)
members = compute_member_set(gate_id, chain_list)
seen: set[str] = set()
out: list[tuple[str, float]] = []
for ev in events_for_gate(gate_id, chain_list):
if ev.get("event_type") != "gate_lock":
continue
node = ev.get("node_id")
if not isinstance(node, str) or not node:
continue
if node in seen:
continue
if node not in members:
# Non-member lock attempt — ignored. The producer-side
# check should also refuse to emit, but resolver-side
# enforcement is defense-in-depth.
continue
p = _payload(ev)
try:
paid = float(p.get("lock_cost") or 0.0)
except (TypeError, ValueError):
paid = 0.0
if paid < float(_lock_cost_per_member()):
continue
seen.add(node)
out.append((node, float(ev.get("timestamp") or 0.0)))
return out
def _state(gate_id: str, chain: Iterable[dict[str, Any]]) -> LockedGateState:
contributions = _collect_lock_contributions(gate_id, chain)
if len(contributions) < _lock_min_members():
return LockedGateState(locked=False, locked_at=None, locked_by=())
contributions.sort(key=lambda c: c[1])
threshold_ts = contributions[_lock_min_members() - 1][1]
nodes = tuple(c[0] for c in contributions)
return LockedGateState(locked=True, locked_at=threshold_ts, locked_by=nodes)
def is_locked(gate_id: str, chain: Iterable[dict[str, Any]]) -> bool:
return _state(gate_id, chain).locked
def locked_at(gate_id: str, chain: Iterable[dict[str, Any]]) -> float | None:
return _state(gate_id, chain).locked_at
def locked_by(gate_id: str, chain: Iterable[dict[str, Any]]) -> tuple[str, ...]:
return _state(gate_id, chain).locked_by
@dataclass(frozen=True)
class LockValidation:
accepted: bool
reason: str
cost: int
def validate_lock_request(
node_id: str,
gate_id: str,
chain: Iterable[dict[str, Any]],
*,
lock_cost: int | None = None,
) -> LockValidation:
"""Pre-emit check for a ``gate_lock`` event from ``node_id``.
Returns ``accepted=False`` with a structured ``reason`` when
rejected — the UI surfaces these directly so the user knows what
needs to change.
"""
chain_list = list(chain)
cost = int(_lock_cost_per_member() if lock_cost is None else lock_cost)
if cost < _lock_cost_per_member():
return LockValidation(False, "lock_cost_below_min", cost)
if node_id not in compute_member_set(gate_id, chain_list):
return LockValidation(False, "not_a_member", cost)
if node_id in {n for n, _ in _collect_lock_contributions(gate_id, chain_list)}:
return LockValidation(False, "already_locked_by_node", cost)
return LockValidation(True, "ok", cost)
__all__ = [
"LockedGateState",
"LockValidation",
"is_locked",
"locked_at",
"locked_by",
"validate_lock_request",
]
@@ -0,0 +1,53 @@
"""Gate ratification — cumulative oracle rep threshold.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §5.3 step 3.
A gate is "ratified" once the SUM of its members' oracle rep crosses
``CONFIG['gate_ratification_rep']`` (default 50). Ratification is a
recognition signal — it doesn't gate any functionality, but UI may
surface it as "this gate is established / legitimate".
Pure function over the chain. The threshold is governable via petition
(Sprint 7) by changing the CONFIG value.
"""
from __future__ import annotations
from typing import Any, Iterable
from services.infonet.config import CONFIG
from services.infonet.gates.sacrifice import compute_member_set
from services.infonet.reputation import compute_oracle_rep
def _ratification_threshold() -> int:
return int(CONFIG["gate_ratification_rep"])
# Public alias for consumers who don't want to import CONFIG.
RATIFICATION_THRESHOLD = _ratification_threshold()
def cumulative_member_oracle_rep(
gate_id: str,
chain: Iterable[dict[str, Any]],
) -> float:
"""Sum of current members' oracle rep balances."""
chain_list = list(chain)
members = compute_member_set(gate_id, chain_list)
return sum(compute_oracle_rep(m, chain_list) for m in members)
def is_ratified(
gate_id: str,
chain: Iterable[dict[str, Any]],
) -> bool:
"""``True`` once cumulative member oracle rep meets the threshold."""
return cumulative_member_oracle_rep(gate_id, chain) >= float(_ratification_threshold())
__all__ = [
"RATIFICATION_THRESHOLD",
"cumulative_member_oracle_rep",
"is_ratified",
]
+192
View File
@@ -0,0 +1,192 @@
"""Gate sacrifice mechanic — burn-on-entry, not threshold check.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.16, §5.3 step 2.
A node enters a gate by **burning** common rep equal to
``gate.entry_sacrifice``. The burn is permanent and non-refundable
(even on voluntary exit). This is the constitutional difference from
threshold-based access: you can't fake having enough rep — you have
to spend it.
The eligibility checks happen *before* the burn:
- Node's common rep ≥ ``min_overall_rep + entry_sacrifice``.
- Node's per-gate rep meets each ``min_gate_rep[required_gate]``.
If those pass, the entry is accepted, ``entry_sacrifice`` is burned
from the node's common rep, and the node is recorded as a member.
This module exposes pure functions:
- ``can_enter(node_id, gate_id, chain)`` — eligibility check + cost,
returning a structured ``EntryDecision`` so the UI can render
exactly *why* a node can't enter (cross-cutting non-hostile UX rule).
- ``compute_member_set(gate_id, chain)`` — current members from
``gate_enter`` − ``gate_exit`` events.
- ``is_member(node_id, gate_id, chain)`` — convenience.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Iterable
from services.infonet.gates.state import events_for_gate, get_gate_meta
from services.infonet.reputation import compute_common_rep
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
def compute_member_set(
gate_id: str,
chain: Iterable[dict[str, Any]],
) -> set[str]:
"""Current member set: ``gate_enter`` − ``gate_exit`` − members
booted by ``gate_shutdown_execute``. The shutdown case zeroes the
set out — once a gate is shut down, there are no members.
"""
chain_list = list(chain)
events = events_for_gate(gate_id, chain_list)
members: set[str] = set()
shutdown_seen = False
for ev in events:
et = ev.get("event_type")
if et == "gate_shutdown_execute":
shutdown_seen = True
members = set()
continue
node = ev.get("node_id")
if not isinstance(node, str) or not node:
continue
if et == "gate_enter":
if not shutdown_seen:
members.add(node)
elif et == "gate_exit":
members.discard(node)
return members
def is_member(
node_id: str,
gate_id: str,
chain: Iterable[dict[str, Any]],
) -> bool:
return node_id in compute_member_set(gate_id, list(chain))
@dataclass(frozen=True)
class EntryRefusal:
"""Structured "why a node can't enter" diagnostic.
The cross-cutting non-hostile UX rule (BUILD_LOG.md design rules
§1) requires the UI to show the user a path forward — not a
blanket "denied". This dataclass carries enough info for the
frontend to render "you need 5 more common rep" or "you need
more rep in gate X".
"""
kind: str
detail: str
@dataclass(frozen=True)
class EntryDecision:
accepted: bool
cost: int
refusals: tuple[EntryRefusal, ...]
def compute_gate_rep(
node_id: str,
gate_id: str,
chain: Iterable[dict[str, Any]],
) -> float:
"""Per-gate reputation: common rep earned from upreps cast by
members of ``gate_id``.
Sprint 6 ships a simple variant: same formula as
``compute_common_rep`` but only upreps from current members of
``gate_id`` count. Anti-gaming penalties (Sprint 3) still apply
via the underlying ``compute_common_rep`` call when called with
the synthetic chain — but for Sprint 6 we filter at the chain
level and pass the filtered chain to the global function.
A more sophisticated per-gate formula (e.g. using only upreps
that happened *while* the upreper was a member, or weighting by
in-gate activity) is open for governance to specify later.
"""
chain_list = [e for e in chain if isinstance(e, dict)]
members = compute_member_set(gate_id, chain_list)
if not members:
return 0.0
# Filter to upreps authored by current gate members targeting node_id.
# Pass the WHOLE chain to compute_common_rep (it needs full event
# history for oracle_rep computation of the upreper); but limit
# which uprep events count by stripping non-member ones.
filtered: list[dict[str, Any]] = []
for ev in chain_list:
if ev.get("event_type") == "uprep":
author = ev.get("node_id")
if author not in members:
continue
filtered.append(ev)
return compute_common_rep(node_id, filtered)
def can_enter(
node_id: str,
gate_id: str,
chain: Iterable[dict[str, Any]],
) -> EntryDecision:
"""RULES §3.16 — eligibility + cost.
Returns a structured decision. ``accepted=True`` means: burning
``cost`` common rep from ``node_id`` satisfies all entry rules.
``accepted=False`` lists every reason refusal occurred so the UI
can show all of them at once.
"""
chain_list = list(chain)
meta = get_gate_meta(gate_id, chain_list)
if meta is None:
return EntryDecision(
accepted=False, cost=0,
refusals=(EntryRefusal(kind="gate_not_found", detail=gate_id),),
)
if is_member(node_id, gate_id, chain_list):
return EntryDecision(
accepted=False, cost=0,
refusals=(EntryRefusal(kind="already_member", detail=gate_id),),
)
refusals: list[EntryRefusal] = []
common_rep = compute_common_rep(node_id, chain_list)
needed = meta.min_overall_rep + meta.entry_sacrifice
if common_rep < needed:
refusals.append(EntryRefusal(
kind="insufficient_common_rep",
detail=f"have {common_rep:.4f}, need {needed} (min_overall_rep "
f"{meta.min_overall_rep} + entry_sacrifice {meta.entry_sacrifice})",
))
for required_gate, min_rep in meta.min_gate_rep.items():
gate_rep = compute_gate_rep(node_id, required_gate, chain_list)
if gate_rep < min_rep:
refusals.append(EntryRefusal(
kind="insufficient_gate_rep",
detail=f"gate {required_gate}: have {gate_rep:.4f}, need {min_rep}",
))
return EntryDecision(
accepted=not refusals, cost=meta.entry_sacrifice if not refusals else 0,
refusals=tuple(refusals),
)
__all__ = [
"EntryDecision",
"EntryRefusal",
"can_enter",
"compute_gate_rep",
"compute_member_set",
"is_member",
]
@@ -0,0 +1,43 @@
"""Gate shutdown lifecycle — Tier 1 suspend, Tier 2 shutdown, typed appeal.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §5.5.
Three modules with clean separation of concerns:
- ``suspend.py`` — Tier 1: 30-day reversible freeze. Filed via
``gate_suspend_file``, voted on, executed via
``gate_suspend_execute``, auto-unsuspends after 30 days unless a
shutdown petition passes.
- ``shutdown.py`` — Tier 2: 7-day-delayed archive. PREREQUISITE: gate
must currently be suspended.
- ``appeal.py`` — Typed shutdown appeal: pauses the 7-day execution
timer, max one appeal per shutdown, 48h window after vote passage.
"""
from services.infonet.gates.shutdown.appeal import (
AppealValidation,
paused_execution_remaining_sec,
validate_appeal_filing,
)
from services.infonet.gates.shutdown.shutdown import (
ShutdownState,
compute_shutdown_state,
validate_shutdown_filing,
)
from services.infonet.gates.shutdown.suspend import (
SuspensionState,
compute_suspension_state,
validate_suspend_filing,
)
__all__ = [
"AppealValidation",
"ShutdownState",
"SuspensionState",
"compute_shutdown_state",
"compute_suspension_state",
"paused_execution_remaining_sec",
"validate_appeal_filing",
"validate_shutdown_filing",
"validate_suspend_filing",
]
@@ -0,0 +1,189 @@
"""Typed shutdown appeal — pauses execution timer, anti-stall bounded.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §5.5 step 7.
An appeal pauses the 7-day shutdown execution timer. The
"anti-stall" property limits abuse:
- One appeal per shutdown petition (no infinite re-appeals).
- 48-hour filing window after the shutdown vote passes.
- If the appeal fails, the original shutdown's execution timer
resumes from where it was paused — the shutdown still happens,
just delayed by the appeal-vote duration.
This module exposes:
- ``validate_appeal_filing`` — pre-emit checks.
- ``paused_execution_remaining_sec`` — compute how much time was
remaining on the shutdown timer when the appeal was filed (so the
resolver can resume the timer from that point).
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Iterable
from services.infonet.config import CONFIG
from services.infonet.gates.shutdown.shutdown import compute_shutdown_state
from services.infonet.gates.state import get_gate_meta
_SECONDS_PER_HOUR = 3600.0
_SECONDS_PER_DAY = 86400.0
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
@dataclass(frozen=True)
class AppealValidation:
accepted: bool
reason: str
def _shutdown_petition_filed_at(
target_petition_id: str,
chain: Iterable[dict[str, Any]],
) -> float | None:
for ev in chain:
if not isinstance(ev, dict):
continue
if ev.get("event_type") != "gate_shutdown_file":
continue
if _payload(ev).get("petition_id") == target_petition_id:
return float(ev.get("timestamp") or 0.0)
return None
def _shutdown_vote_passed_at(
target_petition_id: str,
chain: Iterable[dict[str, Any]],
) -> float | None:
"""Return the timestamp of the ``gate_shutdown_vote`` event whose
payload says ``vote=="passed"`` for the target petition. The
appeal window starts here."""
for ev in chain:
if not isinstance(ev, dict):
continue
if ev.get("event_type") != "gate_shutdown_vote":
continue
p = _payload(ev)
if p.get("petition_id") != target_petition_id:
continue
if p.get("vote") == "passed":
return float(ev.get("timestamp") or 0.0)
return None
def _has_appeal(
target_petition_id: str,
chain: Iterable[dict[str, Any]],
) -> bool:
for ev in chain:
if not isinstance(ev, dict):
continue
if ev.get("event_type") != "gate_shutdown_appeal_file":
continue
if _payload(ev).get("target_petition_id") == target_petition_id:
return True
return False
def validate_appeal_filing(
gate_id: str,
target_petition_id: str,
filer_id: str,
*,
reason: str,
evidence_hashes: list[str],
chain: Iterable[dict[str, Any]],
now: float,
filer_cooldown_until: float | None = None,
) -> AppealValidation:
"""Pre-emit validation for ``gate_shutdown_appeal_file``.
Rejects if:
- Reason or evidence missing.
- Gate doesn't exist.
- Target shutdown petition doesn't exist.
- Target petition is not currently in "executing" status (i.e.
vote hasn't passed yet, or shutdown already executed).
- 48-hour filing window has elapsed since vote passage.
- Target petition already has an appeal (one per shutdown).
- Filer cooldown active.
"""
chain_list = list(chain)
if not isinstance(reason, str) or not reason.strip():
return AppealValidation(False, "reason_empty")
if not isinstance(evidence_hashes, list) or not evidence_hashes:
return AppealValidation(False, "evidence_required")
if get_gate_meta(gate_id, chain_list) is None:
return AppealValidation(False, "gate_not_found")
if not _shutdown_petition_filed_at(target_petition_id, chain_list):
return AppealValidation(False, "target_petition_not_found")
# The "already-filed" check fires before the status check on
# purpose — once an appeal is filed, the petition status flips
# from "executing" to "appealed", and surfacing that as
# "target_not_in_executing_state" would mislead a second filer
# about *why* their appeal was refused. Spec invariant: one
# appeal per shutdown; surface that directly.
if _has_appeal(target_petition_id, chain_list):
return AppealValidation(False, "appeal_already_filed")
state = compute_shutdown_state(gate_id, chain_list, now=now)
if state.pending_status not in ("executing",):
return AppealValidation(False, "target_not_in_executing_state")
vote_passed = _shutdown_vote_passed_at(target_petition_id, chain_list)
if vote_passed is None:
return AppealValidation(False, "vote_not_passed")
window_s = float(CONFIG["gate_shutdown_appeal_window_hours"]) * _SECONDS_PER_HOUR
if now > vote_passed + window_s:
return AppealValidation(False, "appeal_window_expired")
if filer_cooldown_until is not None and filer_cooldown_until > now:
return AppealValidation(False, "filer_cooldown_active")
# filer_id is consumed by the producer event payload, not by validation here.
del filer_id
return AppealValidation(True, "ok")
def paused_execution_remaining_sec(
target_petition_id: str,
chain: Iterable[dict[str, Any]],
*,
appeal_filed_at: float,
) -> float:
"""Compute how much time was remaining on the shutdown's
execution timer when the appeal was filed.
The original shutdown's ``execution_at`` was
``vote_passed_at + execution_delay_days * 86400``. The remaining
time at appeal-filing time is ``execution_at - appeal_filed_at``,
clamped to ≥ 0.
The producer of the ``gate_shutdown_appeal_resolve`` event with
``outcome="resumed"`` should attach
``resumed_execution_at = now + this_value`` so the timer resumes
from where it paused.
"""
chain_list = list(chain)
vote_passed = _shutdown_vote_passed_at(target_petition_id, chain_list)
if vote_passed is None:
return 0.0
delay_s = float(CONFIG["gate_shutdown_execution_delay_days"]) * _SECONDS_PER_DAY
execution_at = vote_passed + delay_s
remaining = execution_at - float(appeal_filed_at)
return max(0.0, remaining)
__all__ = [
"AppealValidation",
"paused_execution_remaining_sec",
"validate_appeal_filing",
]
@@ -0,0 +1,195 @@
"""Tier 2: 7-day-delayed shutdown.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §5.5 steps 5-8.
PREREQUISITE: gate must currently be suspended. The shutdown petition
itself is a vote among oracle-rep holders. If it passes, a 7-day
execution delay opens (the appeal window). After the delay (and any
appeal resolution), the ``gate_shutdown_execute`` event archives the
gate permanently.
State derivation:
- A shutdown petition can be: ``filed``, ``vote_passed``, ``executing``
(after vote, during 7-day delay), ``appealed`` (timer paused),
``executed``, ``failed``, ``voided_appeal``.
- This module computes the petition status from chain events; it does
NOT execute the petition itself (the producer emits
``gate_shutdown_execute`` based on this status).
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Iterable
from services.infonet.config import CONFIG
from services.infonet.gates.shutdown.suspend import (
FilingValidation,
compute_suspension_state,
)
from services.infonet.gates.state import events_for_gate, get_gate_meta
_SECONDS_PER_DAY = 86400.0
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
@dataclass(frozen=True)
class ShutdownState:
"""Derived snapshot of all shutdown petitions filed against a gate."""
has_pending: bool
pending_petition_id: str | None
pending_status: str | None # "filed" | "vote_passed" | "executing" | "appealed" | "failed"
execution_at: float | None
executed: bool
def compute_shutdown_state(
gate_id: str,
chain: Iterable[dict[str, Any]],
*,
now: float,
) -> ShutdownState:
chain_list = list(chain)
events = events_for_gate(gate_id, chain_list)
petitions: dict[str, dict[str, Any]] = {}
for ev in events:
et = ev.get("event_type")
if et != "gate_shutdown_file":
continue
p = _payload(ev)
pid = p.get("petition_id")
if not isinstance(pid, str) or not pid:
continue
petitions[pid] = {
"filed_at": float(ev.get("timestamp") or 0.0),
"status": "filed",
"execution_at": None,
"appealed": False,
}
# Walk votes/executions/appeals in chain order.
chain_all = [e for e in chain_list if isinstance(e, dict)]
chain_all.sort(key=lambda e: (float(e.get("timestamp") or 0.0), int(e.get("sequence") or 0)))
for ev in chain_all:
et = ev.get("event_type")
if et not in ("gate_shutdown_vote", "gate_shutdown_execute",
"gate_shutdown_appeal_file", "gate_shutdown_appeal_resolve"):
continue
p = _payload(ev)
if et == "gate_shutdown_vote":
pid = p.get("petition_id")
if not isinstance(pid, str) or pid not in petitions:
continue
# Sprint 6 simplification: a vote event with payload
# {"vote": "passed"} is treated as the canonical pass
# signal. Real production may aggregate per-voter votes
# in Sprint 7's governance DSL — Sprint 6 honors whichever
# outcome the spec-side vote tally already reached.
outcome = p.get("vote")
if outcome == "passed":
petitions[pid]["status"] = "executing"
delay_s = float(CONFIG["gate_shutdown_execution_delay_days"]) * _SECONDS_PER_DAY
petitions[pid]["execution_at"] = float(ev.get("timestamp") or 0.0) + delay_s
elif outcome == "failed":
petitions[pid]["status"] = "failed"
elif et == "gate_shutdown_appeal_file":
target = p.get("target_petition_id")
if isinstance(target, str) and target in petitions:
petitions[target]["appealed"] = True
petitions[target]["status"] = "appealed"
petitions[target]["execution_at"] = None # paused
elif et == "gate_shutdown_appeal_resolve":
target = p.get("target_petition_id")
outcome = p.get("outcome")
if isinstance(target, str) and target in petitions:
if outcome == "voided_shutdown":
petitions[target]["status"] = "voided_appeal"
elif outcome == "resumed":
petitions[target]["status"] = "executing"
# execution_at restored by the producer who emitted
# the resolve event with a fresh execution_at field.
new_exec = p.get("resumed_execution_at")
try:
petitions[target]["execution_at"] = float(new_exec)
except (TypeError, ValueError):
petitions[target]["execution_at"] = None
elif et == "gate_shutdown_execute":
pid = p.get("petition_id")
if isinstance(pid, str) and pid in petitions:
petitions[pid]["status"] = "executed"
executed = any(p["status"] == "executed" for p in petitions.values())
pending_pid = None
pending = None
for pid, p in petitions.items():
if p["status"] in ("filed", "executing", "appealed"):
pending_pid = pid
pending = p
break
return ShutdownState(
has_pending=pending is not None,
pending_petition_id=pending_pid,
pending_status=pending["status"] if pending else None,
execution_at=pending["execution_at"] if pending else None,
executed=executed,
)
def validate_shutdown_filing(
gate_id: str,
filer_id: str,
*,
reason: str,
evidence_hashes: list[str],
chain: Iterable[dict[str, Any]],
now: float,
filer_cooldown_until: float | None = None,
) -> FilingValidation:
"""Pre-emit validation for ``gate_shutdown_file``.
Critical Sprint 6 invariant: shutdown filings REQUIRE the gate to
currently be suspended. This is the spec's two-tier escalation
safeguard — a gate cannot be shut down without first surviving a
suspension period.
"""
chain_list = list(chain)
if not isinstance(reason, str) or not reason.strip():
return FilingValidation(False, "reason_empty")
if not isinstance(evidence_hashes, list) or not evidence_hashes:
return FilingValidation(False, "evidence_required")
if get_gate_meta(gate_id, chain_list) is None:
return FilingValidation(False, "gate_not_found")
suspension = compute_suspension_state(gate_id, chain_list, now=now)
if suspension.status == "shutdown":
return FilingValidation(False, "gate_already_shutdown")
if suspension.status != "suspended":
return FilingValidation(False, "gate_not_suspended")
shutdown = compute_shutdown_state(gate_id, chain_list, now=now)
if shutdown.has_pending:
return FilingValidation(False, "shutdown_already_pending")
if filer_cooldown_until is not None and filer_cooldown_until > now:
return FilingValidation(False, "filer_cooldown_active")
_ = filer_id
return FilingValidation(True, "ok")
__all__ = [
"ShutdownState",
"compute_shutdown_state",
"validate_shutdown_filing",
]
@@ -0,0 +1,172 @@
"""Tier 1: 30-day reversible suspend.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §5.5 steps 1-4.
State derivation:
- A gate is "suspended" iff:
- the most recent ``gate_suspend_execute`` event is more recent
than any ``gate_unsuspend`` or ``gate_shutdown_execute`` event,
- AND the suspended_until window has not yet elapsed.
- ``compute_suspension_state`` returns the current suspension status
including the auto-unsuspend timestamp.
- ``validate_suspend_filing`` is the pre-emit check the UI should use
before letting a node sign a ``gate_suspend_file`` event.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Iterable
from services.infonet.config import CONFIG
from services.infonet.gates.state import events_for_gate, get_gate_meta
_SECONDS_PER_DAY = 86400.0
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
@dataclass(frozen=True)
class SuspensionState:
"""``status`` is one of ``"active"``, ``"suspended"``,
``"shutdown"``. ``suspended_until`` is the auto-unsuspend
timestamp or ``None`` when not currently suspended."""
status: str
suspended_at: float | None
suspended_until: float | None
last_shutdown_petition_at: float | None
"""Used for 90-day cooldown checks on subsequent shutdown petitions."""
def compute_suspension_state(
gate_id: str,
chain: Iterable[dict[str, Any]],
*,
now: float,
) -> SuspensionState:
chain_list = list(chain)
events = events_for_gate(gate_id, chain_list)
last_shutdown_filed_ts: float | None = None
last_shutdown_executed_ts: float | None = None
suspended_at: float | None = None
last_unsuspend_ts: float | None = None
for ev in events:
et = ev.get("event_type")
ts = float(ev.get("timestamp") or 0.0)
if et == "gate_suspend_execute":
suspended_at = ts
elif et == "gate_unsuspend":
last_unsuspend_ts = ts
elif et == "gate_shutdown_file":
last_shutdown_filed_ts = ts
elif et == "gate_shutdown_execute":
last_shutdown_executed_ts = ts
if last_shutdown_executed_ts is not None:
return SuspensionState(
status="shutdown",
suspended_at=suspended_at,
suspended_until=None,
last_shutdown_petition_at=last_shutdown_filed_ts,
)
if suspended_at is None:
return SuspensionState(
status="active",
suspended_at=None,
suspended_until=None,
last_shutdown_petition_at=last_shutdown_filed_ts,
)
if last_unsuspend_ts is not None and last_unsuspend_ts > suspended_at:
return SuspensionState(
status="active",
suspended_at=None,
suspended_until=None,
last_shutdown_petition_at=last_shutdown_filed_ts,
)
duration = float(CONFIG["gate_suspend_duration_days"]) * _SECONDS_PER_DAY
suspended_until = suspended_at + duration
if now >= suspended_until:
# Window auto-elapsed; even without an explicit gate_unsuspend
# event, the gate is logically active again.
return SuspensionState(
status="active",
suspended_at=None,
suspended_until=None,
last_shutdown_petition_at=last_shutdown_filed_ts,
)
return SuspensionState(
status="suspended",
suspended_at=suspended_at,
suspended_until=suspended_until,
last_shutdown_petition_at=last_shutdown_filed_ts,
)
@dataclass(frozen=True)
class FilingValidation:
accepted: bool
reason: str
def validate_suspend_filing(
gate_id: str,
filer_id: str,
*,
reason: str,
evidence_hashes: list[str],
chain: Iterable[dict[str, Any]],
now: float,
filer_cooldown_until: float | None = None,
) -> FilingValidation:
"""Pre-emit validation for a ``gate_suspend_file`` event.
Rejects if:
- Reason is empty.
- No evidence hashes.
- Gate doesn't exist.
- Gate is already suspended or shut down.
- Filer's cooldown is still active.
- Gate's 90-day shutdown-petition cooldown is active.
"""
chain_list = list(chain)
if not isinstance(reason, str) or not reason.strip():
return FilingValidation(False, "reason_empty")
if not isinstance(evidence_hashes, list) or not evidence_hashes:
return FilingValidation(False, "evidence_required")
if not all(isinstance(h, str) and h for h in evidence_hashes):
return FilingValidation(False, "evidence_hashes_invalid")
if get_gate_meta(gate_id, chain_list) is None:
return FilingValidation(False, "gate_not_found")
state = compute_suspension_state(gate_id, chain_list, now=now)
if state.status == "shutdown":
return FilingValidation(False, "gate_shutdown")
if state.status == "suspended":
return FilingValidation(False, "already_suspended")
if filer_cooldown_until is not None and filer_cooldown_until > now:
return FilingValidation(False, "filer_cooldown_active")
if state.last_shutdown_petition_at is not None:
cooldown_s = float(CONFIG["gate_shutdown_cooldown_days"]) * _SECONDS_PER_DAY
if now < state.last_shutdown_petition_at + cooldown_s:
return FilingValidation(False, "gate_cooldown_active")
_ = filer_id # producer logs filer separately; not consulted for validation here.
return FilingValidation(True, "ok")
__all__ = [
"FilingValidation",
"SuspensionState",
"compute_suspension_state",
"validate_suspend_filing",
]
+111
View File
@@ -0,0 +1,111 @@
"""Common chain helpers shared across the gates package.
The legacy ``gate_create`` event is owned by mesh_schema (it predates
the economy layer). Sprint 6 reads those events and extracts the
structured fields it needs from the ``rules`` payload, with sensible
defaults when a key is missing — same pattern the rest of the
protocol uses for forward compatibility.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Iterable
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
def _gate_id(event: dict[str, Any]) -> str:
p = _payload(event)
gid = p.get("gate_id") or p.get("gate")
return str(gid) if isinstance(gid, str) else ""
def events_for_gate(
gate_id: str,
chain: Iterable[dict[str, Any]],
) -> list[dict[str, Any]]:
"""All events that reference ``gate_id``, sorted by chain order."""
out: list[dict[str, Any]] = []
for ev in chain:
if not isinstance(ev, dict):
continue
if _gate_id(ev) == gate_id:
out.append(ev)
out.sort(key=lambda e: (float(e.get("timestamp") or 0.0), int(e.get("sequence") or 0)))
return out
@dataclass(frozen=True)
class GateMeta:
"""Static metadata extracted from the original ``gate_create`` event."""
gate_id: str
creator_node_id: str
display_name: str
entry_sacrifice: int
min_overall_rep: int
min_gate_rep: dict[str, int]
created_at: float
raw_rules: dict[str, Any]
def _safe_int(val: Any, default: int = 0) -> int:
try:
if isinstance(val, bool):
return default
return int(val)
except (TypeError, ValueError):
return default
def get_gate_meta(
gate_id: str,
chain: Iterable[dict[str, Any]],
) -> GateMeta | None:
"""Return the gate's static metadata, or ``None`` if no
``gate_create`` event exists for it on the chain.
Multiple ``gate_create`` events with the same gate_id are unusual
but possible at peer-gossip ingestion time; the FIRST one wins
(same first-write-wins pattern as ``find_snapshot``). Subsequent
forgeries are ignored.
"""
for ev in chain:
if not isinstance(ev, dict):
continue
if ev.get("event_type") != "gate_create":
continue
if _gate_id(ev) != gate_id:
continue
p = _payload(ev)
rules = p.get("rules")
if not isinstance(rules, dict):
rules = {}
cross_gate = rules.get("min_gate_rep")
if not isinstance(cross_gate, dict):
cross_gate = {}
return GateMeta(
gate_id=gate_id,
creator_node_id=str(ev.get("node_id") or ""),
display_name=str(p.get("display_name") or ""),
entry_sacrifice=_safe_int(rules.get("entry_sacrifice"), 0),
min_overall_rep=_safe_int(rules.get("min_overall_rep"), 0),
min_gate_rep={
str(k): _safe_int(v, 0)
for k, v in cross_gate.items()
if isinstance(k, str) and k
},
created_at=float(ev.get("timestamp") or 0.0),
raw_rules=dict(rules),
)
return None
__all__ = [
"GateMeta",
"events_for_gate",
"get_gate_meta",
]
@@ -0,0 +1,56 @@
"""Governance — petitions, declarative DSL executor, constitutional
challenge, and upgrade-hash governance (Sprint 7).
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.15, §5.4, §5.6.
The DSL executor is the centerpiece of Sprint 7. It is intentionally
**not a sandbox**: it cannot run arbitrary code, period. The four
allowed payload types (UPDATE_PARAM / BATCH_UPDATE_PARAMS /
ENABLE_FEATURE / DISABLE_FEATURE) are dispatched as plain Python
switch cases. There is NO ``eval``, ``exec``, ``compile``, or
dynamic attribute access anywhere in the executor. The whole class
of code-injection attacks goes away by design.
Protocol upgrades that need new logic use upgrade-hash governance —
nodes vote on a software release hash, not on-chain code.
"""
from services.infonet.governance.challenge import (
ChallengeState,
compute_challenge_state,
validate_challenge_filing,
)
from services.infonet.governance.dsl_executor import (
DSLExecutionResult,
apply_petition_payload,
forbidden_attributes_check,
)
from services.infonet.governance.petition import (
PetitionState,
compute_petition_state,
network_governance_weight,
validate_petition_filing,
)
from services.infonet.governance.upgrade_hash import (
HeavyNodeReadinessState,
UpgradeProposalState,
compute_upgrade_state,
validate_upgrade_proposal,
)
__all__ = [
"ChallengeState",
"DSLExecutionResult",
"HeavyNodeReadinessState",
"PetitionState",
"UpgradeProposalState",
"apply_petition_payload",
"compute_challenge_state",
"compute_petition_state",
"compute_upgrade_state",
"forbidden_attributes_check",
"network_governance_weight",
"validate_challenge_filing",
"validate_petition_filing",
"validate_upgrade_proposal",
]
@@ -0,0 +1,161 @@
"""Constitutional challenge — 48-hour window after a petition passes.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §5.4 step 4.
A challenger sacrifices ``challenge_filing_cost`` (default 25) common
rep to file a challenge against a passed petition. The challenge then
goes to a vote — if it succeeds (``uphold`` wins by majority oracle
rep), the petition is voided. If it fails, the challenger loses the
sacrificed rep and the petition proceeds to execution.
This module exposes:
- ``compute_challenge_state(petition_id, chain, *, now)`` — derives
the challenge outcome from chain events.
- ``validate_challenge_filing(filer_common_rep, ...)`` — pre-emit
check.
Sprint 7 voting tally uses ``oracle_rep_active`` weight, same as
petition voting itself.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Iterable
from services.infonet.config import CONFIG
from services.infonet.reputation import compute_oracle_rep_active
_HOUR_S = 3600.0
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
@dataclass
class ChallengeState:
petition_id: str
filed: bool
filer_id: str | None
filed_at: float | None
deadline: float | None
uphold_weight: float
void_weight: float
outcome: str # "voided" | "rejected" | "pending" | "none"
def compute_challenge_state(
petition_id: str,
chain: Iterable[dict[str, Any]],
*,
now: float,
) -> ChallengeState:
chain_list = [e for e in chain if isinstance(e, dict)]
file_event = None
vote_events: list[dict[str, Any]] = []
for ev in chain_list:
if _payload(ev).get("petition_id") != petition_id:
continue
et = ev.get("event_type")
if et == "challenge_file":
if file_event is None:
file_event = ev
elif et == "challenge_vote":
vote_events.append(ev)
if file_event is None:
return ChallengeState(
petition_id=petition_id, filed=False,
filer_id=None, filed_at=None, deadline=None,
uphold_weight=0.0, void_weight=0.0, outcome="none",
)
filed_at = float(file_event.get("timestamp") or 0.0)
deadline = filed_at + float(CONFIG["challenge_window_hours"]) * _HOUR_S
state = ChallengeState(
petition_id=petition_id, filed=True,
filer_id=str(file_event.get("node_id") or ""),
filed_at=filed_at, deadline=deadline,
uphold_weight=0.0, void_weight=0.0,
outcome="pending",
)
seen: dict[str, str] = {}
cache: dict[str, float] = {}
for ev in sorted(vote_events,
key=lambda e: (float(e.get("timestamp") or 0.0),
int(e.get("sequence") or 0))):
voter = ev.get("node_id")
if not isinstance(voter, str) or not voter or voter in seen:
continue
ts = float(ev.get("timestamp") or 0.0)
if ts < filed_at or ts > deadline:
continue
vote = _payload(ev).get("vote")
if vote not in ("uphold", "void"):
continue
seen[voter] = vote
if voter not in cache:
cache[voter] = compute_oracle_rep_active(voter, chain_list, now=ts)
w = cache[voter]
if vote == "uphold":
# "uphold" means: uphold the constitutional challenge —
# i.e. void the original petition. Per RULES §5.4 step 4:
# "Challenge upheld → 'voided_challenge' (petition killed)".
state.uphold_weight += w
else: # "void" the challenge → original petition stands
state.void_weight += w
if now <= deadline:
return state # still pending
if state.uphold_weight > state.void_weight:
state.outcome = "voided"
else:
state.outcome = "rejected"
return state
@dataclass(frozen=True)
class ChallengeFilingValidation:
accepted: bool
reason: str
def validate_challenge_filing(
filer_common_rep: float,
petition_id: str,
chain: Iterable[dict[str, Any]],
*,
now: float,
) -> ChallengeFilingValidation:
"""Pre-emit check for a ``challenge_file`` event.
Rejects if:
- Filer lacks the ``challenge_filing_cost``.
- A challenge already exists on this petition.
- The challenge window for the petition has elapsed (caller is
expected to have already verified the petition's voting closed
successfully — that timestamp comes from
``compute_petition_state``).
"""
if filer_common_rep < float(CONFIG["challenge_filing_cost"]):
return ChallengeFilingValidation(False, "insufficient_common_rep")
state = compute_challenge_state(petition_id, list(chain), now=now)
if state.filed:
return ChallengeFilingValidation(False, "challenge_already_filed")
return ChallengeFilingValidation(True, "ok")
__all__ = [
"ChallengeFilingValidation",
"ChallengeState",
"compute_challenge_state",
"validate_challenge_filing",
]
@@ -0,0 +1,223 @@
"""Declarative DSL executor — the type-safe, no-eval petition applier.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §1.2 (the
governance section comment block) + §5.4 step 5.
CRITICAL design property: this module **cannot execute arbitrary
code**. It is a switch over four typed payload variants, each with a
fully-validated key/value or feature-flag operation. There is NO use
of ``eval``, ``exec``, ``compile``, ``ast.parse``, ``getattr`` with a
runtime key, ``__import__``, ``subprocess``, ``os.system``, or any
other dynamic-execution primitive.
The whole class of code-injection attacks is eliminated by design —
even if an attacker passes a maliciously crafted petition payload, the
executor either applies a typed value or rejects with
``InvalidPetition``. There is no path to executing the attacker's
input as code.
Sprint 7's adversarial tests assert this invariant by reading this
file's source bytes and confirming none of the forbidden builtins
appear (``forbidden_attributes_check``).
"""
from __future__ import annotations
from copy import deepcopy
from dataclasses import dataclass, field
from typing import Any
from services.infonet.config import (
CONFIG,
CONFIG_SCHEMA,
IMMUTABLE_PRINCIPLES,
InvalidPetition,
validate_cross_field_invariants,
validate_petition_value,
)
_ALLOWED_PAYLOAD_TYPES = frozenset({
"UPDATE_PARAM",
"BATCH_UPDATE_PARAMS",
"ENABLE_FEATURE",
"DISABLE_FEATURE",
})
@dataclass
class DSLExecutionResult:
"""Outcome of applying a petition payload.
``new_config`` is a fresh dict — the caller decides whether to
swap the live ``CONFIG`` with it. Sprint 7's tests apply the
result and verify the swap; production callers wire this through
the ``petition_execute`` event handler.
"""
new_config: dict[str, Any]
changed_keys: tuple[str, ...] = field(default_factory=tuple)
def _check_payload_shape(payload: Any) -> str:
if not isinstance(payload, dict):
raise InvalidPetition("petition_payload must be an object")
payload_type = payload.get("type")
if payload_type not in _ALLOWED_PAYLOAD_TYPES:
raise InvalidPetition(
f"unknown petition_payload type: {payload_type!r}; "
f"allowed: {sorted(_ALLOWED_PAYLOAD_TYPES)}"
)
return str(payload_type)
def _check_key_writeable(key: str) -> None:
"""Reject writes to keys not in CONFIG_SCHEMA. ``IMMUTABLE_PRINCIPLES``
keys never appear in ``CONFIG_SCHEMA``, so this also rejects them.
"""
if not isinstance(key, str) or not key:
raise InvalidPetition("CONFIG key must be a non-empty string")
if key not in CONFIG_SCHEMA:
# Also surface a clearer diagnostic if the user attempted to
# mutate an IMMUTABLE_PRINCIPLES key.
if key in IMMUTABLE_PRINCIPLES:
raise InvalidPetition(
f"key {key!r} is in IMMUTABLE_PRINCIPLES — only an "
f"upgrade-hash governance hard fork can change it"
)
raise InvalidPetition(f"unknown CONFIG key: {key!r}")
def _apply_update_param(
payload: dict[str, Any],
candidate: dict[str, Any],
) -> tuple[dict[str, Any], list[str]]:
if "key" not in payload or "value" not in payload:
raise InvalidPetition("UPDATE_PARAM requires key + value")
key = payload["key"]
value = payload["value"]
_check_key_writeable(key)
validate_petition_value(key, value, candidate)
candidate[key] = value
return candidate, [key]
def _apply_batch_update(
payload: dict[str, Any],
candidate: dict[str, Any],
) -> tuple[dict[str, Any], list[str]]:
updates = payload.get("updates")
if not isinstance(updates, list) or not updates:
raise InvalidPetition("BATCH_UPDATE_PARAMS requires a non-empty 'updates' list")
seen_keys: set[str] = set()
changed: list[str] = []
for u in updates:
if not isinstance(u, dict) or "key" not in u or "value" not in u:
raise InvalidPetition("BATCH_UPDATE_PARAMS entries must be {key, value}")
key = u["key"]
if key in seen_keys:
raise InvalidPetition(f"duplicate key in BATCH_UPDATE_PARAMS: {key!r}")
seen_keys.add(key)
_check_key_writeable(key)
validate_petition_value(key, u["value"], candidate)
candidate[key] = u["value"]
changed.append(key)
return candidate, changed
def _apply_feature_toggle(
payload: dict[str, Any],
candidate: dict[str, Any],
*,
enable: bool,
) -> tuple[dict[str, Any], list[str]]:
feature = payload.get("feature")
if not isinstance(feature, str) or not feature:
raise InvalidPetition("ENABLE_FEATURE / DISABLE_FEATURE requires non-empty 'feature'")
_check_key_writeable(feature)
schema = CONFIG_SCHEMA.get(feature)
if schema is None or schema.get("type") != "bool":
raise InvalidPetition(
f"feature {feature!r} is not a boolean CONFIG key"
)
candidate[feature] = bool(enable)
return candidate, [feature]
def apply_petition_payload(
payload: dict[str, Any],
current_config: dict[str, Any] | None = None,
) -> DSLExecutionResult:
"""Apply a validated petition payload to a CANDIDATE copy of CONFIG.
Transactional: validation runs against the candidate; if any check
fails, the candidate is discarded and ``InvalidPetition`` is
raised. The live ``CONFIG`` is never partially mutated.
Pass ``current_config`` when applying against a hypothetical state
(testing, upgrade-hash dry-runs). Otherwise the live ``CONFIG`` is
deep-copied as the starting point.
"""
payload_type = _check_payload_shape(payload)
candidate = deepcopy(current_config) if current_config is not None else deepcopy(CONFIG)
if payload_type == "UPDATE_PARAM":
candidate, changed = _apply_update_param(payload, candidate)
elif payload_type == "BATCH_UPDATE_PARAMS":
candidate, changed = _apply_batch_update(payload, candidate)
elif payload_type == "ENABLE_FEATURE":
candidate, changed = _apply_feature_toggle(payload, candidate, enable=True)
elif payload_type == "DISABLE_FEATURE":
candidate, changed = _apply_feature_toggle(payload, candidate, enable=False)
else: # pragma: no cover — _check_payload_shape gated this
raise InvalidPetition(f"unhandled payload type: {payload_type}")
# Cross-field invariants validated against the FINAL candidate.
validate_cross_field_invariants(candidate)
return DSLExecutionResult(new_config=candidate, changed_keys=tuple(changed))
# ─── No-eval guard ──────────────────────────────────────────────────────
# Forbidden attribute names whose presence in this module's source
# would violate the "no arbitrary code execution" property. Sprint 7's
# adversarial test reads this file and asserts none of these substrings
# appear (outside of this list and the guard function below — the
# guard's job is to *name* the forbidden surface, not use it).
_FORBIDDEN_ATTRIBUTES: frozenset[str] = frozenset({
# Call-syntax tokens. Scanned against this module's source by the
# Sprint 7 adversarial test. Bare module names (``subprocess``,
# ``os``, etc.) are deliberately NOT in this set — their mere
# mention in prose is harmless; what we forbid is the CALL.
"eval(",
"exec(",
"compile(",
"__import__(",
"ast.parse(",
"subprocess.run(",
"subprocess.Popen(",
"subprocess.call(",
"subprocess.check_output(",
"os.system(",
"os.popen(",
"pickle.loads(",
"marshal.loads(",
})
def forbidden_attributes_check() -> tuple[str, ...]:
"""Return the curated list of forbidden surface names.
Used by the Sprint 7 adversarial test to scan this module's source
for any forbidden token. Exposed as a function so the test stays
decoupled from the module's internal layout.
"""
return tuple(sorted(_FORBIDDEN_ATTRIBUTES))
__all__ = [
"DSLExecutionResult",
"apply_petition_payload",
"forbidden_attributes_check",
]
@@ -0,0 +1,315 @@
"""Petition state machine — pure function over chain history.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.15, §5.4.
State diagram:
petition_file
│
▼ status="signatures"
petition_sign × N (collect signature_governance_weight)
│
▼ if signature_governance_weight ≥ 25% × network → status="voting"
│ if 14 days elapsed and threshold not met → status="failed_signatures"
petition_vote × N (oracle_rep_active weighted)
│
▼ if 7 days elapsed:
│ check quorum (30%) + supermajority (67%)
│ status="challenge" (passed) or "failed_vote"
challenge_file (optional, 48h window) + challenge_vote × N
│
▼ if challenge passes → status="voided_challenge"
│ else → status="passed"
petition_execute → status="executed"
Voting weights use ``oracle_rep_active`` (governance-decayed) per
RULES §3.15. Total network weight is the sum across all nodes
referenced by signature/vote events plus any node with chain
activity (we use the union of acting nodes' weights — same as
``compute_network_governance_weight``).
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Iterable
from services.infonet.config import CONFIG
from services.infonet.reputation import compute_oracle_rep_active
_DAY_S = 86400.0
_HOUR_S = 3600.0
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
@dataclass
class PetitionState:
petition_id: str
status: str # "signatures" | "voting" | "challenge" | "passed" |
# "executed" | "failed_signatures" | "failed_vote" |
# "voided_challenge" | "not_found"
filer_id: str
filed_at: float
petition_payload: dict[str, Any] = field(default_factory=dict)
signature_governance_weight: float = 0.0
signature_threshold_at_filing: float = 0.0
votes_for_weight: float = 0.0
votes_against_weight: float = 0.0
voting_started_at: float | None = None
voting_deadline: float | None = None
challenge_window_until: float | None = None
def _governance_weight_provider(
node_id: str,
chain: list[dict[str, Any]],
*,
at: float,
cache: dict[str, float],
) -> float:
"""Memoize per-call: governance weight for ``node_id`` evaluated at
chain time ``at``. Cached because petitions iterate signatures and
votes from many nodes, and recomputing oracle rep per call is
expensive on long chains."""
key = node_id
if key in cache:
return cache[key]
w = compute_oracle_rep_active(node_id, chain, now=at)
cache[key] = w
return w
def network_governance_weight(
chain: Iterable[dict[str, Any]],
*,
now: float,
) -> float:
"""Total network ``oracle_rep_active`` at chain time ``now``.
Sum across every node that has authored at least one event on the
chain. Matches RULES §3.15: "sum(node.oracle_rep_active for all
nodes)". Newly-created nodes that haven't yet signed any event
have zero weight and contribute nothing — including them is a
no-op.
"""
chain_list = [e for e in chain if isinstance(e, dict)]
nodes: set[str] = set()
for ev in chain_list:
nid = ev.get("node_id")
if isinstance(nid, str) and nid:
nodes.add(nid)
cache: dict[str, float] = {}
return sum(
_governance_weight_provider(n, chain_list, at=now, cache=cache)
for n in nodes
)
def compute_petition_state(
petition_id: str,
chain: Iterable[dict[str, Any]],
*,
now: float,
) -> PetitionState:
"""Derive the current state of ``petition_id`` from chain history.
``now`` is the evaluation timestamp — pass
``time_validity.chain_majority_time(chain)`` in production.
"""
chain_list = [e for e in chain if isinstance(e, dict)]
file_event = None
sign_events: list[dict[str, Any]] = []
vote_events: list[dict[str, Any]] = []
execute_event = None
challenge_filed_event = None
challenge_vote_events: list[dict[str, Any]] = []
for ev in chain_list:
et = ev.get("event_type")
p = _payload(ev)
pid = p.get("petition_id")
if pid != petition_id:
continue
if et == "petition_file":
if file_event is None: # first-write-wins
file_event = ev
elif et == "petition_sign":
sign_events.append(ev)
elif et == "petition_vote":
vote_events.append(ev)
elif et == "petition_execute":
execute_event = ev
elif et == "challenge_file":
if challenge_filed_event is None:
challenge_filed_event = ev
elif et == "challenge_vote":
challenge_vote_events.append(ev)
if file_event is None:
return PetitionState(
petition_id=petition_id, status="not_found",
filer_id="", filed_at=0.0,
)
state = PetitionState(
petition_id=petition_id,
status="signatures",
filer_id=str(file_event.get("node_id") or ""),
filed_at=float(file_event.get("timestamp") or 0.0),
petition_payload=dict(_payload(file_event).get("petition_payload") or {}),
)
cache: dict[str, float] = {}
network_weight = network_governance_weight(chain_list, now=now)
state.signature_threshold_at_filing = (
network_weight * float(CONFIG["petition_signature_threshold"])
)
# ── Signatures phase ──
sign_window_s = float(CONFIG["petition_signature_window_days"]) * _DAY_S
seen_signers: set[str] = set()
for ev in sorted(sign_events,
key=lambda e: (float(e.get("timestamp") or 0.0),
int(e.get("sequence") or 0))):
signer = ev.get("node_id")
if not isinstance(signer, str) or not signer:
continue
if signer in seen_signers:
continue
seen_signers.add(signer)
ts = float(ev.get("timestamp") or 0.0)
# Only count signatures that landed within the window.
if ts > state.filed_at + sign_window_s:
continue
weight = _governance_weight_provider(signer, chain_list, at=ts, cache=cache)
state.signature_governance_weight += weight
if state.signature_governance_weight >= state.signature_threshold_at_filing > 0:
# Find the timestamp the threshold was crossed (= last signature
# that crossed it). Sprint 7 simplification: use the latest
# signature event timestamp as the voting-phase start.
latest_sig_ts = max((float(e.get("timestamp") or 0.0) for e in sign_events
if e.get("node_id") in seen_signers),
default=state.filed_at)
state.status = "voting"
state.voting_started_at = latest_sig_ts
state.voting_deadline = latest_sig_ts + float(CONFIG["petition_vote_window_days"]) * _DAY_S
else:
if now > state.filed_at + sign_window_s:
state.status = "failed_signatures"
return state
# Still collecting signatures.
return state
# ── Voting phase ──
seen_voters: dict[str, str] = {} # node_id → "for"|"against"
for ev in sorted(vote_events,
key=lambda e: (float(e.get("timestamp") or 0.0),
int(e.get("sequence") or 0))):
voter = ev.get("node_id")
if not isinstance(voter, str) or not voter:
continue
if voter in seen_voters: # one vote per node — first wins
continue
ts = float(ev.get("timestamp") or 0.0)
if state.voting_started_at is not None and ts < state.voting_started_at:
continue
if state.voting_deadline is not None and ts > state.voting_deadline:
continue
vote = _payload(ev).get("vote")
if vote not in ("for", "against"):
continue
seen_voters[voter] = vote
weight = _governance_weight_provider(voter, chain_list, at=ts, cache=cache)
if vote == "for":
state.votes_for_weight += weight
else:
state.votes_against_weight += weight
if state.voting_deadline is not None and now <= state.voting_deadline:
# Voting still open.
return state
# Voting closed — tally.
participating = state.votes_for_weight + state.votes_against_weight
quorum_required = network_weight * float(CONFIG["petition_quorum"])
if participating < quorum_required:
state.status = "failed_vote"
return state
if participating == 0:
state.status = "failed_vote"
return state
if state.votes_for_weight / participating < float(CONFIG["petition_supermajority"]):
state.status = "failed_vote"
return state
# Petition passed the vote — enter challenge window.
state.status = "challenge"
state.challenge_window_until = (
(state.voting_deadline or state.filed_at)
+ float(CONFIG["challenge_window_hours"]) * _HOUR_S
)
# ── Challenge phase ──
from services.infonet.governance.challenge import (
compute_challenge_state as _compute_challenge_state,
)
challenge_state = _compute_challenge_state(petition_id, chain_list, now=now)
if challenge_state.outcome == "voided":
state.status = "voided_challenge"
return state
if state.challenge_window_until is not None and now <= state.challenge_window_until:
# Challenge window still open.
return state
# Challenge window closed without voiding the petition.
state.status = "passed"
if execute_event is not None:
state.status = "executed"
return state
@dataclass(frozen=True)
class FilingValidation:
accepted: bool
reason: str
def validate_petition_filing(
filer_common_rep: float,
*,
petition_payload: dict[str, Any],
) -> FilingValidation:
"""Pre-emit check for a ``petition_file`` event.
The producer must verify the filer has at least
``petition_filing_cost`` common rep available to burn. The
payload structure is also validated up-front (cheaper to reject
here than during execution).
"""
if filer_common_rep < float(CONFIG["petition_filing_cost"]):
return FilingValidation(False, "insufficient_common_rep")
if not isinstance(petition_payload, dict):
return FilingValidation(False, "petition_payload_not_object")
if "type" not in petition_payload:
return FilingValidation(False, "petition_payload_missing_type")
return FilingValidation(True, "ok")
__all__ = [
"FilingValidation",
"PetitionState",
"compute_petition_state",
"network_governance_weight",
"validate_petition_filing",
]
@@ -0,0 +1,340 @@
"""Upgrade-hash governance — RULES §3.15 (formalization), §5.6.
Protocol upgrades that need new logic (formulas, event types, state
machines) cannot be expressed as parameter changes — the declarative
DSL has no way to ship new code. The Round 8 formalization replaces
that gap with **upgrade-hash governance**: developers publish a
software release, the network votes on its SHA-256 release hash, and
nodes upgrade their software.
Lifecycle:
1. Filing (``upgrade_propose``) — 25 common rep, includes the
``release_hash``, description, target_protocol_version.
2. Signatures (14 days) — 25% of network ``oracle_rep_active``.
3. Voting (14 days) — **80% supermajority + 40% quorum** (higher
bars than param petitions).
4. Constitutional challenge window (48 hours).
5. Activation (30 days): Heavy Nodes that have downloaded the new
release emit ``upgrade_signal_ready``. Once **67%** of Heavy
Nodes have signaled, the upgrade activates and ``protocol_version``
increments.
6. Failure modes: ``failed_signatures``, ``failed_vote``,
``voided_challenge``, ``failed_activation`` (≥33% of Heavy Nodes
couldn't or wouldn't upgrade — network not ready).
Heavy-Node detection: a node is "Heavy" if its transport tier is
``private_strong`` per IMPLEMENTATION_PLAN §3.5. For Sprint 7's pure
chain-only computation, we rely on the producer to mark
``upgrade_signal_ready`` events with ``release_hash`` matching the
proposal — and only Heavy Nodes can emit that event in production
(producer-side enforcement; this module verifies the chain-derived
state).
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Iterable
from services.infonet.config import CONFIG
from services.infonet.reputation import compute_oracle_rep_active
_DAY_S = 86400.0
_HOUR_S = 3600.0
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
@dataclass
class HeavyNodeReadinessState:
total_heavy_nodes: int
ready_count: int
fraction: float
threshold_met: bool
@dataclass
class UpgradeProposalState:
proposal_id: str
status: str # "signatures" | "voting" | "challenge" | "activation" |
# "activated" | "failed_signatures" | "failed_vote" |
# "voided_challenge" | "failed_activation" | "not_found"
proposer_id: str
filed_at: float
release_hash: str = ""
target_protocol_version: str = ""
signature_governance_weight: float = 0.0
votes_for_weight: float = 0.0
votes_against_weight: float = 0.0
voting_started_at: float | None = None
voting_deadline: float | None = None
challenge_window_until: float | None = None
activation_deadline: float | None = None
readiness: HeavyNodeReadinessState = field(
default_factory=lambda: HeavyNodeReadinessState(0, 0, 0.0, False),
)
def compute_upgrade_state(
proposal_id: str,
chain: Iterable[dict[str, Any]],
*,
now: float,
heavy_node_ids: set[str] | None = None,
) -> UpgradeProposalState:
"""Derive the proposal's current state from chain events.
``heavy_node_ids`` is the set of nodes the caller knows to be
Heavy at chain time ``now``. Production callers compute this from
`wormhole_supervisor.get_transport_tier()` × the chain's known
nodes. Tests pass an explicit set.
"""
chain_list = [e for e in chain if isinstance(e, dict)]
heavy_set = set(heavy_node_ids) if heavy_node_ids is not None else set()
propose_event = None
sign_events: list[dict[str, Any]] = []
vote_events: list[dict[str, Any]] = []
challenge_event = None
challenge_vote_events: list[dict[str, Any]] = []
signal_ready_events: list[dict[str, Any]] = []
activate_event = None
for ev in chain_list:
et = ev.get("event_type")
p = _payload(ev)
pid = p.get("proposal_id")
if pid != proposal_id:
continue
if et == "upgrade_propose":
if propose_event is None:
propose_event = ev
elif et == "upgrade_sign":
sign_events.append(ev)
elif et == "upgrade_vote":
vote_events.append(ev)
elif et == "upgrade_challenge":
if challenge_event is None:
challenge_event = ev
elif et == "upgrade_challenge_vote":
challenge_vote_events.append(ev)
elif et == "upgrade_signal_ready":
signal_ready_events.append(ev)
elif et == "upgrade_activate":
activate_event = ev
if propose_event is None:
return UpgradeProposalState(
proposal_id=proposal_id, status="not_found",
proposer_id="", filed_at=0.0,
)
pp = _payload(propose_event)
state = UpgradeProposalState(
proposal_id=proposal_id,
status="signatures",
proposer_id=str(propose_event.get("node_id") or ""),
filed_at=float(propose_event.get("timestamp") or 0.0),
release_hash=str(pp.get("release_hash") or ""),
target_protocol_version=str(pp.get("target_protocol_version") or ""),
)
cache: dict[str, float] = {}
def _w(node_id: str, at: float) -> float:
if node_id not in cache:
cache[node_id] = compute_oracle_rep_active(node_id, chain_list, now=at)
return cache[node_id]
# Network weight at "now" — used for signature + quorum thresholds.
nodes: set[str] = set()
for ev in chain_list:
nid = ev.get("node_id")
if isinstance(nid, str) and nid:
nodes.add(nid)
network_weight = sum(_w(n, now) for n in nodes)
sig_threshold = network_weight * float(CONFIG["upgrade_signature_threshold"])
# ── Signatures ──
sig_window_s = float(CONFIG["upgrade_signature_window_days"]) * _DAY_S
seen_sig: set[str] = set()
for ev in sorted(sign_events,
key=lambda e: (float(e.get("timestamp") or 0.0),
int(e.get("sequence") or 0))):
signer = ev.get("node_id")
if not isinstance(signer, str) or signer in seen_sig:
continue
ts = float(ev.get("timestamp") or 0.0)
if ts > state.filed_at + sig_window_s:
continue
seen_sig.add(signer)
state.signature_governance_weight += _w(signer, ts)
if state.signature_governance_weight >= sig_threshold > 0:
latest_sig_ts = max((float(e.get("timestamp") or 0.0) for e in sign_events
if e.get("node_id") in seen_sig),
default=state.filed_at)
state.status = "voting"
state.voting_started_at = latest_sig_ts
state.voting_deadline = latest_sig_ts + float(CONFIG["upgrade_vote_window_days"]) * _DAY_S
else:
if now > state.filed_at + sig_window_s:
state.status = "failed_signatures"
return state
return state
# ── Voting ──
seen_voters: dict[str, str] = {}
for ev in sorted(vote_events,
key=lambda e: (float(e.get("timestamp") or 0.0),
int(e.get("sequence") or 0))):
voter = ev.get("node_id")
if not isinstance(voter, str) or voter in seen_voters:
continue
ts = float(ev.get("timestamp") or 0.0)
if state.voting_started_at is None or ts < state.voting_started_at:
continue
if state.voting_deadline is None or ts > state.voting_deadline:
continue
vote = _payload(ev).get("vote")
if vote not in ("for", "against"):
continue
seen_voters[voter] = vote
w = _w(voter, ts)
if vote == "for":
state.votes_for_weight += w
else:
state.votes_against_weight += w
if state.voting_deadline is not None and now <= state.voting_deadline:
return state
participating = state.votes_for_weight + state.votes_against_weight
quorum_required = network_weight * float(CONFIG["upgrade_quorum"])
if participating < quorum_required or participating == 0:
state.status = "failed_vote"
return state
if state.votes_for_weight / participating < float(CONFIG["upgrade_supermajority"]):
state.status = "failed_vote"
return state
# Vote passed — challenge window.
state.status = "challenge"
state.challenge_window_until = (
(state.voting_deadline or state.filed_at)
+ float(CONFIG["upgrade_challenge_window_hours"]) * _HOUR_S
)
# Process upgrade_challenge_vote — uphold-majority voids the proposal.
if challenge_event is not None:
uphold_w = 0.0
void_w = 0.0
seen_cv: dict[str, str] = {}
for ev in sorted(challenge_vote_events,
key=lambda e: (float(e.get("timestamp") or 0.0),
int(e.get("sequence") or 0))):
voter = ev.get("node_id")
if not isinstance(voter, str) or voter in seen_cv:
continue
ts = float(ev.get("timestamp") or 0.0)
challenge_at = float(challenge_event.get("timestamp") or 0.0)
if ts < challenge_at or ts > (state.challenge_window_until or 0.0):
continue
vote = _payload(ev).get("vote")
if vote not in ("uphold", "void"):
continue
seen_cv[voter] = vote
w = _w(voter, ts)
if vote == "uphold":
uphold_w += w
else:
void_w += w
if (state.challenge_window_until is not None
and now > state.challenge_window_until
and uphold_w > void_w):
state.status = "voided_challenge"
return state
if state.challenge_window_until is not None and now <= state.challenge_window_until:
return state
# Challenge cleared → activation phase.
state.status = "activation"
state.activation_deadline = (
(state.challenge_window_until or state.filed_at)
+ float(CONFIG["upgrade_activation_window_days"]) * _DAY_S
)
# ── Heavy-Node readiness ──
seen_ready: set[str] = set()
for ev in signal_ready_events:
node = ev.get("node_id")
if not isinstance(node, str) or node in seen_ready:
continue
if node not in heavy_set:
continue # only Heavy Nodes can signal
if _payload(ev).get("release_hash") != state.release_hash:
continue
seen_ready.add(node)
total_heavy = max(len(heavy_set), 1)
fraction = len(seen_ready) / total_heavy if heavy_set else 0.0
threshold = float(CONFIG["upgrade_activation_threshold"])
state.readiness = HeavyNodeReadinessState(
total_heavy_nodes=len(heavy_set),
ready_count=len(seen_ready),
fraction=fraction,
threshold_met=fraction >= threshold,
)
if activate_event is not None:
state.status = "activated"
return state
if state.readiness.threshold_met:
# Producer can emit upgrade_activate now — until then the
# status is "activation" with threshold_met=True so the UI
# can prompt.
return state
if state.activation_deadline is not None and now > state.activation_deadline:
state.status = "failed_activation"
return state
@dataclass(frozen=True)
class UpgradeFilingValidation:
accepted: bool
reason: str
def validate_upgrade_proposal(
filer_common_rep: float,
*,
release_hash: str,
release_description: str,
target_protocol_version: str,
) -> UpgradeFilingValidation:
"""Pre-emit check for ``upgrade_propose``."""
if filer_common_rep < float(CONFIG["upgrade_filing_cost"]):
return UpgradeFilingValidation(False, "insufficient_common_rep")
if not isinstance(release_hash, str) or not release_hash.strip():
return UpgradeFilingValidation(False, "release_hash_required")
if not isinstance(release_description, str) or len(release_description) > 4000:
return UpgradeFilingValidation(False, "release_description_invalid")
if not isinstance(target_protocol_version, str) or not target_protocol_version.strip():
return UpgradeFilingValidation(False, "target_protocol_version_required")
return UpgradeFilingValidation(True, "ok")
__all__ = [
"HeavyNodeReadinessState",
"UpgradeFilingValidation",
"UpgradeProposalState",
"compute_upgrade_state",
"validate_upgrade_proposal",
]
@@ -0,0 +1,276 @@
"""Identity rotation gates and obligation inheritance.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.13.
Pure functions over the chain. Two sets of responsibilities:
1. **Gating (``validate_rotation``):** reject a rotation if the old
identity holds active resolution stakes, dispute stakes, or truth
stakes. Predictor exclusion + governance decay + rep transfer are
inherited automatically by ``rotation_descendants`` — they are NOT
gates, they are computations downstream resolvers run.
2. **Descendant tracking (``rotation_descendants``):** given a node,
return the full transitive closure of identities it has rotated
into. Used by Sprint 4's predictor-exclusion logic to compute
``frozen_predictor_ids ∪ rotation_descendants(frozen_predictor_ids)``
from the snapshot at resolution time.
Cross-cutting design rule (BUILD_LOG.md): a user attempting to rotate
while holding active stakes must NOT see a hostile UI message. The
caller is expected to:
- Show the user which stakes are blocking rotation.
- Offer to wait for those stakes to settle, or cancel pending
unresolved stakes (where the protocol allows).
- Queue the rotation for retry after settlement.
This module returns structured rejection reasons (a tuple of
``(blocker_kind, count, sample_ids)``) so the UI can render exactly
that. It never returns "rejected" without the diagnostic shape.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Iterable
@dataclass(frozen=True)
class RotationBlocker:
"""One reason the rotation is currently rejected.
``kind`` is one of:
- ``"resolution_stake"`` — open resolution stakes on a market
that has not yet finalized.
- ``"dispute_stake"`` — open dispute stakes that have not yet
resolved.
- ``"truth_stake"`` — truth stakes still inside their
``duration_days`` window without a resolve event.
``count`` is the number of blocking obligations of that kind.
``sample_ids`` is up to 5 string identifiers (market_id /
dispute_id / message_id) so the UI can show "3 markets and 1
dispute are still pending" with deep links.
"""
kind: str
count: int
sample_ids: tuple[str, ...]
@dataclass(frozen=True)
class RotationDecision:
accepted: bool
blockers: tuple[RotationBlocker, ...]
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
def _market_status_lookup(events: list[dict[str, Any]]) -> dict[str, str]:
"""Last-write-wins map of market_id → terminal status.
Sprint 2 only knows two terminal statuses (FINAL, INVALID) — the
full lifecycle is Sprint 4. Markets without a ``resolution_finalize``
are treated as still open (active stakes).
"""
status: dict[str, str] = {}
for ev in events:
if ev.get("event_type") != "resolution_finalize":
continue
p = _payload(ev)
mid = p.get("market_id")
if isinstance(mid, str) and mid:
outcome = p.get("outcome")
status[mid] = "invalid" if outcome == "invalid" else "final"
return status
def _dispute_status_lookup(events: list[dict[str, Any]]) -> dict[str, str]:
status: dict[str, str] = {}
for ev in events:
if ev.get("event_type") != "dispute_resolve":
continue
p = _payload(ev)
did = p.get("dispute_id")
if isinstance(did, str) and did:
status[did] = "resolved"
return status
def _truth_stake_resolved_messages(events: list[dict[str, Any]]) -> set[str]:
resolved: set[str] = set()
for ev in events:
if ev.get("event_type") != "truth_stake_resolve":
continue
p = _payload(ev)
mid = p.get("message_id")
if isinstance(mid, str) and mid:
resolved.add(mid)
return resolved
def _active_resolution_stakes(node_id: str, events: list[dict[str, Any]]) -> list[str]:
market_status = _market_status_lookup(events)
out: list[str] = []
for ev in events:
if ev.get("event_type") != "resolution_stake":
continue
if ev.get("node_id") != node_id:
continue
p = _payload(ev)
mid = p.get("market_id")
if not isinstance(mid, str) or not mid:
continue
if market_status.get(mid) is None:
out.append(mid)
return out
def _active_dispute_stakes(node_id: str, events: list[dict[str, Any]]) -> list[str]:
dispute_status = _dispute_status_lookup(events)
out: list[str] = []
for ev in events:
if ev.get("event_type") != "dispute_stake":
continue
if ev.get("node_id") != node_id:
continue
p = _payload(ev)
did = p.get("dispute_id")
if not isinstance(did, str) or not did:
continue
if dispute_status.get(did) is None:
out.append(did)
return out
def _active_truth_stakes(
node_id: str,
events: list[dict[str, Any]],
*,
now: float,
) -> list[str]:
"""A truth stake is active if its (placed_at + duration_days * 86400)
is in the future relative to ``now`` AND no ``truth_stake_resolve``
has landed for its message.
"""
resolved = _truth_stake_resolved_messages(events)
out: list[str] = []
for ev in events:
if ev.get("event_type") != "truth_stake_place":
continue
if ev.get("node_id") != node_id:
continue
p = _payload(ev)
mid = p.get("message_id")
if not isinstance(mid, str) or not mid:
continue
if mid in resolved:
continue
try:
placed_at = float(ev.get("timestamp") or 0.0)
duration_days = int(p.get("duration_days") or 0)
except (TypeError, ValueError):
continue
expires_at = placed_at + duration_days * 86400.0
if expires_at > now:
out.append(mid)
return out
def validate_rotation(
rotation_event: dict[str, Any],
chain: Iterable[dict[str, Any]],
*,
now: float,
) -> RotationDecision:
"""Decide whether ``rotation_event`` is permitted right now.
Sprint 2 enforces RULES §3.13 Gate 1 only. Gate 2 (obligation
inheritance) is computation, not gating, and is handled by the
Sprint 4 predictor-exclusion logic that consults
``rotation_descendants``.
The returned ``RotationDecision`` includes structured blockers so
the UI can offer a non-hostile retry path (see module docstring).
"""
if rotation_event.get("event_type") != "identity_rotate":
raise ValueError("validate_rotation requires an identity_rotate event")
payload = _payload(rotation_event)
old_node_id = payload.get("old_node_id")
if not isinstance(old_node_id, str) or not old_node_id:
raise ValueError("identity_rotate payload missing old_node_id")
events = [e for e in chain if isinstance(e, dict)]
blockers: list[RotationBlocker] = []
res = _active_resolution_stakes(old_node_id, events)
if res:
blockers.append(RotationBlocker(
kind="resolution_stake",
count=len(res),
sample_ids=tuple(res[:5]),
))
dis = _active_dispute_stakes(old_node_id, events)
if dis:
blockers.append(RotationBlocker(
kind="dispute_stake",
count=len(dis),
sample_ids=tuple(dis[:5]),
))
tru = _active_truth_stakes(old_node_id, events, now=now)
if tru:
blockers.append(RotationBlocker(
kind="truth_stake",
count=len(tru),
sample_ids=tuple(tru[:5]),
))
return RotationDecision(accepted=not blockers, blockers=tuple(blockers))
def rotation_descendants(
node_id: str,
chain: Iterable[dict[str, Any]],
) -> set[str]:
"""All identities that descend from ``node_id`` via ``identity_rotate``.
Excludes ``node_id`` itself. Used by Sprint 4 predictor exclusion.
"""
events = [e for e in chain if isinstance(e, dict)]
# Build a forward map: old_node_id -> {new_node_id, new_node_id, ...}.
# New node_id of an identity_rotate is the event's signer (per spec).
forward: dict[str, set[str]] = {}
for ev in events:
if ev.get("event_type") != "identity_rotate":
continue
p = _payload(ev)
old = p.get("old_node_id")
new = ev.get("node_id")
if not isinstance(old, str) or not isinstance(new, str):
continue
if not old or not new or old == new:
continue
forward.setdefault(old, set()).add(new)
out: set[str] = set()
stack = list(forward.get(node_id, set()))
while stack:
cur = stack.pop()
if cur in out:
continue
out.add(cur)
for nxt in forward.get(cur, ()):
if nxt not in out:
stack.append(nxt)
return out
__all__ = [
"RotationBlocker",
"RotationDecision",
"rotation_descendants",
"validate_rotation",
]
@@ -0,0 +1,79 @@
"""Market lifecycle, snapshot, evidence, and resolution.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.10, §5.2.
Pure-function design (same as Sprint 2/3): every entry point takes
``(market_id, chain, ...)`` and returns a deterministic value or a
structured result. The producer is responsible for emitting the
resulting events to the chain through the adapter layer.
"""
from services.infonet.markets.data_unavailable import (
is_data_unavailable_triggered,
resolve_data_unavailable_effects,
)
from services.infonet.markets.dispute import (
DisputeView,
collect_disputes,
compute_dispute_outcome,
dispute_settlement_effects,
effective_outcome,
market_was_reversed,
)
from services.infonet.markets.evidence import (
EvidenceBundle,
collect_evidence,
evidence_content_hash,
is_first_for_side,
submission_hash,
)
from services.infonet.markets.lifecycle import (
MarketStatus,
compute_market_status,
should_advance_phase,
)
from services.infonet.markets.resolution import (
ResolutionResult,
collect_resolution_stakes,
excluded_predictor_ids,
is_predictor_excluded,
resolve_market,
)
from services.infonet.markets.snapshot import (
build_snapshot,
compute_snapshot_event_hash,
find_snapshot,
)
from services.infonet.markets.stalemate_burn import (
apply_to_stakes as apply_stalemate_burn,
stalemate_burn_pct,
)
__all__ = [
"DisputeView",
"EvidenceBundle",
"MarketStatus",
"ResolutionResult",
"apply_stalemate_burn",
"build_snapshot",
"collect_disputes",
"collect_evidence",
"collect_resolution_stakes",
"compute_dispute_outcome",
"compute_market_status",
"compute_snapshot_event_hash",
"dispute_settlement_effects",
"effective_outcome",
"evidence_content_hash",
"excluded_predictor_ids",
"find_snapshot",
"is_data_unavailable_triggered",
"is_first_for_side",
"is_predictor_excluded",
"market_was_reversed",
"resolve_data_unavailable_effects",
"resolve_market",
"should_advance_phase",
"stalemate_burn_pct",
"submission_hash",
]
@@ -0,0 +1,103 @@
"""DATA_UNAVAILABLE resolution path — Round 8 phantom-evidence defense.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.10 Step 1.5
+ the ``CONFIG['data_unavailable_threshold']`` comment block.
Threshold: when ``oracle_da / oracle_all >= data_unavailable_threshold``
(default 33% of oracle-rep stake), the market is INVALID and:
- ALL evidence-submitter bonds are SLASHED (burned, not returned).
The premise: evidence existed but couldn't be verified — the
submitters are at fault.
- DA voters' resolution stakes are returned in FULL (they acted
correctly).
- yes/no resolution stakes get the stalemate burn applied (they
participated despite bad evidence; small burn makes blind staking
expensive).
This is distinct from the no-supermajority stalemate path: there, all
stakes (including DA) take the burn and bonds are returned in good
faith. Sprint 5 keeps the two paths separate to match the spec.
"""
from __future__ import annotations
from typing import Any
from services.infonet.config import CONFIG
from services.infonet.markets.evidence import EvidenceBundle
from services.infonet.markets.stalemate_burn import (
apply_to_stakes,
split_burn_and_return,
)
def is_data_unavailable_triggered(stakes: list[Any]) -> bool:
"""``True`` if oracle DA stakes meet or exceed the threshold."""
oracle_all = sum(getattr(s, "amount", 0.0) for s in stakes
if getattr(s, "rep_type", None) == "oracle")
if oracle_all <= 0:
return False
oracle_da = sum(getattr(s, "amount", 0.0) for s in stakes
if getattr(s, "side", None) == "data_unavailable"
and getattr(s, "rep_type", None) == "oracle")
return oracle_da / oracle_all >= float(CONFIG["data_unavailable_threshold"])
def resolve_data_unavailable_effects(
stakes: list[Any],
bundles: list[EvidenceBundle],
) -> dict[str, Any]:
"""Compute the rep-transfer effects for a DA-triggered INVALID
resolution. Returns a dict with the same keys ``ResolutionResult``
expects, ready for the caller to fold in.
Side effects layered:
- DA voters get full return.
- yes/no resolution stakers get the stalemate burn.
- Evidence submitters: bonds slashed (forfeit).
"""
out: dict[str, Any] = {
"stake_returns": {},
"bond_forfeits": {},
"bond_returns": {},
"burned": 0.0,
}
da_stakes = [s for s in stakes if getattr(s, "side", None) == "data_unavailable"]
other_stakes = [s for s in stakes if getattr(s, "side", None) in ("yes", "no")]
# DA voters: full return.
for s in da_stakes:
node_id = getattr(s, "node_id", None)
rep_type = getattr(s, "rep_type", None)
amount = float(getattr(s, "amount", 0.0))
if not isinstance(node_id, str) or rep_type not in ("oracle", "common") or amount <= 0:
continue
key = (node_id, rep_type)
out["stake_returns"][key] = out["stake_returns"].get(key, 0.0) + amount
# yes/no stakers: stalemate burn.
burn_returns, burn_total = apply_to_stakes(
({"node_id": s.node_id, "rep_type": s.rep_type, "amount": s.amount} for s in other_stakes),
)
for k, v in burn_returns.items():
out["stake_returns"][k] = out["stake_returns"].get(k, 0.0) + v
out["burned"] += burn_total
# Evidence bonds: slashed (forfeited) — burned.
for b in bundles:
if b.bond > 0:
out["bond_forfeits"][b.node_id] = (
out["bond_forfeits"].get(b.node_id, 0.0) + b.bond
)
out["burned"] += b.bond
return out
__all__ = [
"is_data_unavailable_triggered",
"resolve_data_unavailable_effects",
"split_burn_and_return", # re-exported for callers' convenience
]
+284
View File
@@ -0,0 +1,284 @@
"""Bounded-reversal disputes — RULES §3.12.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.12 + §5.2
step 6.
A dispute is a post-finality challenge. Any node can stake oracle rep
to open one, and other nodes can stake oracle OR common rep on
``confirm`` (uphold the original outcome) or ``reverse`` (flip it).
Oracle-rep simple majority decides — resolution already established
the supermajority, so a simple majority is enough to overturn.
If the dispute reverses, **only this market's oracle rep is
recalculated**. Downstream rep earned in OTHER markets from rep
originally minted here is NOT clawed back. No cascading rewrites.
That's the "bounded" in bounded reversal.
Two effects:
1. ``effective_outcome(market_id, chain)`` returns the flipped
outcome if a reversed dispute exists; the unmodified outcome
otherwise. Used by ``oracle_rep._market_is_mintable`` so
reputation views automatically reflect the reversal.
2. ``compute_dispute_outcome`` returns ``"upheld" | "reversed" |
"tie"`` from accumulated stakes — used when an authoritative
``dispute_resolve`` event has not yet landed.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Iterable
from services.infonet.config import CONFIG
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
@dataclass
class DisputeView:
"""Chain-derived view of a single dispute."""
dispute_id: str
market_id: str
challenger_id: str
challenger_stake: float
opened_at: float
confirm_stakes: list[dict] = field(default_factory=list)
reverse_stakes: list[dict] = field(default_factory=list)
resolved_outcome: str | None = None # "upheld" | "reversed" | "tie"
resolved_at: float | None = None
@property
def is_resolved(self) -> bool:
return self.resolved_outcome is not None
def _dispute_id(event: dict[str, Any]) -> str:
"""Pick the dispute_id from a dispute event payload, falling back
to the event_id for ``dispute_open`` (which is the canonical
identifier the rest of the chain references)."""
p = _payload(event)
did = p.get("dispute_id")
if isinstance(did, str) and did:
return did
# dispute_open events: synthesize from event_id if present, else
# market_id + opener + timestamp. Producers SHOULD attach a
# dispute_id explicitly — Sprint 7+ enforces this in the schema.
eid = event.get("event_id")
if isinstance(eid, str) and eid:
return eid
market_id = p.get("market_id") or ""
return f"dispute:{market_id}:{event.get('node_id','')}:{event.get('timestamp','')}"
def collect_disputes(
market_id: str,
chain: Iterable[dict[str, Any]],
) -> list[DisputeView]:
"""All disputes filed against ``market_id``, sorted by open time."""
chain_list = [e for e in chain if isinstance(e, dict)]
open_events = [e for e in chain_list
if e.get("event_type") == "dispute_open"
and _payload(e).get("market_id") == market_id]
if not open_events:
return []
# Build by dispute_id keyed off the open event.
disputes: dict[str, DisputeView] = {}
open_id_by_market_event: dict[str, str] = {}
for ev in open_events:
p = _payload(ev)
did = _dispute_id(ev)
challenger = ev.get("node_id") or ""
try:
cstake = float(p.get("challenger_stake") or 0.0)
except (TypeError, ValueError):
cstake = 0.0
opened_at = float(ev.get("timestamp") or 0.0)
disputes[did] = DisputeView(
dispute_id=did, market_id=str(market_id),
challenger_id=str(challenger), challenger_stake=cstake,
opened_at=opened_at,
)
open_id_by_market_event[ev.get("event_id") or ""] = did
# Stakes reference dispute_id explicitly (Sprint 1 schema).
for ev in chain_list:
if ev.get("event_type") != "dispute_stake":
continue
p = _payload(ev)
did = p.get("dispute_id")
if not isinstance(did, str) or did not in disputes:
continue
side = p.get("side")
if side not in ("confirm", "reverse"):
continue
rep_type = p.get("rep_type")
if rep_type not in ("oracle", "common"):
continue
try:
amount = float(p.get("amount") or 0.0)
except (TypeError, ValueError):
continue
if amount <= 0:
continue
record = {
"node_id": ev.get("node_id") or "",
"amount": amount,
"rep_type": rep_type,
}
target = disputes[did].confirm_stakes if side == "confirm" else disputes[did].reverse_stakes
target.append(record)
# Resolution events.
for ev in chain_list:
if ev.get("event_type") != "dispute_resolve":
continue
p = _payload(ev)
did = p.get("dispute_id")
if not isinstance(did, str) or did not in disputes:
continue
outcome = p.get("outcome")
if outcome not in ("upheld", "reversed", "tie"):
continue
disputes[did].resolved_outcome = outcome
disputes[did].resolved_at = float(ev.get("timestamp") or 0.0)
return sorted(disputes.values(), key=lambda d: (d.opened_at, d.dispute_id))
def compute_dispute_outcome(dispute: DisputeView) -> str:
"""Apply RULES §3.12 to compute the dispute outcome from its
accumulated stakes (oracle-rep majority).
Returns ``"upheld"`` (default — original outcome stands),
``"reversed"``, or ``"tie"``. ``"tie"`` is treated as upheld for
bookkeeping but is reported separately so callers can log it.
"""
confirm_oracle = sum(
s.get("amount", 0.0) for s in dispute.confirm_stakes
if s.get("rep_type") == "oracle"
)
reverse_oracle = sum(
s.get("amount", 0.0) for s in dispute.reverse_stakes
if s.get("rep_type") == "oracle"
)
if confirm_oracle > reverse_oracle:
return "upheld"
if reverse_oracle > confirm_oracle:
return "reversed"
return "tie"
def market_was_reversed(market_id: str, chain: Iterable[dict[str, Any]]) -> bool:
"""``True`` if any dispute on ``market_id`` resolved as reversed.
Multiple disputes on the same market are unusual but possible —
if any one reverses, the market's effective outcome flips. A
subsequent dispute that re-reverses would flip it back, but
Sprint 5 leaves multi-dispute behavior intentionally simple
(last reversed wins — see ``effective_outcome``).
"""
for d in collect_disputes(market_id, chain):
if d.resolved_outcome == "reversed":
return True
return False
def _flip(outcome: str) -> str:
if outcome == "yes":
return "no"
if outcome == "no":
return "yes"
return outcome
def effective_outcome(
original_outcome: str,
market_id: str,
chain: Iterable[dict[str, Any]],
) -> str:
"""Apply bounded reversal to a market's outcome.
Walks resolved disputes in chain order. Each ``reversed`` outcome
flips the running outcome; ``upheld`` and ``tie`` leave it. The
final value is the **effective** outcome that ``oracle_rep`` and
``last_successful_prediction_ts`` should use.
BOUNDED: this function operates on a single market_id only. It does
NOT cascade into other markets even if oracle rep used to stake in
those other markets came from this one.
"""
if original_outcome not in ("yes", "no"):
return original_outcome
current = original_outcome
for d in collect_disputes(market_id, chain):
if d.resolved_outcome == "reversed":
current = _flip(current)
return current
def dispute_settlement_effects(dispute: DisputeView) -> dict[str, Any]:
"""Compute rep transfers from a *resolved* dispute.
Per RULES §3.12: winning side splits the loser pool, 2% loser tax
burned, oracle and common pools settle independently.
Returns the same shape as ``resolve_data_unavailable_effects`` — a
dict the caller folds into a higher-level result.
"""
out: dict[str, Any] = {
"stake_returns": {},
"stake_winnings": {},
"burned": 0.0,
}
if not dispute.is_resolved:
return out
outcome = dispute.resolved_outcome
if outcome == "tie":
# Return all stakes intact.
for s in dispute.confirm_stakes + dispute.reverse_stakes:
key = (s["node_id"], s["rep_type"])
out["stake_returns"][key] = out["stake_returns"].get(key, 0.0) + s["amount"]
return out
winners = dispute.confirm_stakes if outcome == "upheld" else dispute.reverse_stakes
losers = dispute.reverse_stakes if outcome == "upheld" else dispute.confirm_stakes
burn_pct = float(CONFIG["resolution_loser_burn_pct"])
for rep_type in ("oracle", "common"):
rep_winners = [s for s in winners if s["rep_type"] == rep_type]
rep_losers = [s for s in losers if s["rep_type"] == rep_type]
winner_pool = sum(s["amount"] for s in rep_winners)
loser_pool = sum(s["amount"] for s in rep_losers)
for s in rep_winners:
key = (s["node_id"], rep_type)
out["stake_returns"][key] = out["stake_returns"].get(key, 0.0) + s["amount"]
if winner_pool == 0 or loser_pool == 0:
continue
burn = loser_pool * burn_pct
distributable = loser_pool - burn
out["burned"] += burn
for s in rep_winners:
share = s["amount"] / winner_pool
winnings = share * distributable
key = (s["node_id"], rep_type)
out["stake_winnings"][key] = out["stake_winnings"].get(key, 0.0) + winnings
return out
__all__ = [
"DisputeView",
"collect_disputes",
"compute_dispute_outcome",
"dispute_settlement_effects",
"effective_outcome",
"market_was_reversed",
]
@@ -0,0 +1,200 @@
"""Evidence canonicalization + first-submitter detection.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §2.2 (evidence
bundle fields), §3.10 (Step 4 — bond resolution + first-submitter
bonus).
Two distinct hashes per evidence bundle:
- ``evidence_content_hash`` — SHA-256 of ``(market_id || claimed_outcome
|| sorted(evidence_hashes) || normalized_utf8(source_description))``.
**Excludes node_id**. Two submitters who present the same evidence
produce the same content hash — used for duplicate detection across
authors.
- ``submission_hash`` — SHA-256 of ``(evidence_content_hash || node_id
|| timestamp)``. **Includes node_id**. Used for authorship + chain
ordering + first-submitter detection.
"Same evidence = same evidence_content_hash" — submission ordering
determines who is the FIRST submitter for a side. The first submitter
per outcome side gets ``CONFIG['evidence_first_bonus']`` (capped by the
losing-bond pool, never minted) when their side wins.
"""
from __future__ import annotations
import hashlib
import unicodedata
from dataclasses import dataclass
from typing import Any, Iterable
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
def _normalize_utf8(s: str) -> str:
"""NFC-normalize so visually-identical strings hash identically."""
return unicodedata.normalize("NFC", s)
def evidence_content_hash(
market_id: str,
claimed_outcome: str,
evidence_hashes: list[str],
source_description: str,
) -> str:
"""SHA-256 of the canonical evidence content. Excludes node_id."""
if claimed_outcome not in ("yes", "no"):
raise ValueError("claimed_outcome must be 'yes' or 'no'")
sorted_hashes = sorted(str(h) for h in (evidence_hashes or []))
canonical = "|".join([
"evidence_content",
str(market_id),
claimed_outcome,
",".join(sorted_hashes),
_normalize_utf8(str(source_description or "")),
])
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def submission_hash(
content_hash: str,
node_id: str,
timestamp: float,
) -> str:
"""SHA-256 of ``content_hash || node_id || timestamp``.
Timestamp is rendered with ``repr(float)`` for cross-implementation
determinism — Python's repr gives the shortest round-trippable
decimal, which is stable across CPython versions.
"""
canonical = "|".join([
"evidence_submission",
str(content_hash),
str(node_id),
repr(float(timestamp)),
])
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
@dataclass(frozen=True)
class EvidenceBundle:
"""Chain-derived view of one ``evidence_submit`` event."""
node_id: str
market_id: str
claimed_outcome: str
evidence_hashes: tuple[str, ...]
source_description: str
bond: float
timestamp: float
sequence: int
content_hash: str
submission_hash: str
is_first_for_side: bool
def collect_evidence(
market_id: str,
chain: Iterable[dict[str, Any]],
) -> list[EvidenceBundle]:
"""Return all ``evidence_submit`` events for ``market_id`` as
``EvidenceBundle``s, sorted by chain order, with
``is_first_for_side`` set on the first event per outcome side
whose ``content_hash`` is unique within that side.
"""
events: list[dict[str, Any]] = []
for ev in chain:
if not isinstance(ev, dict):
continue
if ev.get("event_type") != "evidence_submit":
continue
if _payload(ev).get("market_id") != market_id:
continue
events.append(ev)
events.sort(key=lambda e: (float(e.get("timestamp") or 0.0), int(e.get("sequence") or 0)))
seen_content_per_side: dict[str, set[str]] = {"yes": set(), "no": set()}
bundles: list[EvidenceBundle] = []
first_set_per_side: dict[str, bool] = {"yes": False, "no": False}
for ev in events:
p = _payload(ev)
node_id = ev.get("node_id")
outcome = p.get("claimed_outcome")
if not isinstance(node_id, str) or not node_id:
continue
if outcome not in ("yes", "no"):
continue
evhashes = p.get("evidence_hashes") or []
if not isinstance(evhashes, list):
continue
source_desc = p.get("source_description") or ""
bond = p.get("bond")
try:
bond_f = float(bond) if bond is not None else 0.0
except (TypeError, ValueError):
bond_f = 0.0
ts = float(ev.get("timestamp") or 0.0)
seq = int(ev.get("sequence") or 0)
chash = p.get("evidence_content_hash") or evidence_content_hash(
market_id, outcome, [str(h) for h in evhashes], str(source_desc),
)
shash = p.get("submission_hash") or submission_hash(chash, node_id, ts)
# First-for-side: this event is the first occurrence (in chain
# order) of a content hash for this side that we haven't seen
# before. Duplicate submitters of the same content do NOT
# qualify for the bonus.
is_first = False
if chash not in seen_content_per_side[outcome] and not first_set_per_side[outcome]:
is_first = True
first_set_per_side[outcome] = True
seen_content_per_side[outcome].add(chash)
bundles.append(EvidenceBundle(
node_id=node_id,
market_id=str(market_id),
claimed_outcome=outcome,
evidence_hashes=tuple(str(h) for h in evhashes),
source_description=str(source_desc),
bond=bond_f,
timestamp=ts,
sequence=seq,
content_hash=str(chash),
submission_hash=str(shash),
is_first_for_side=is_first,
))
return bundles
def is_first_for_side(
market_id: str,
claimed_outcome: str,
candidate_content_hash: str,
chain: Iterable[dict[str, Any]],
) -> bool:
"""Would a NEW evidence submission with ``candidate_content_hash``
be the first for ``claimed_outcome``?
True if no prior ``evidence_submit`` for ``market_id`` on
``claimed_outcome`` exists (regardless of content hash). The bonus
is for being temporally first per side, not per content hash.
"""
if claimed_outcome not in ("yes", "no"):
return False
for bundle in collect_evidence(market_id, chain):
if bundle.claimed_outcome == claimed_outcome:
return False
return True
__all__ = [
"EvidenceBundle",
"collect_evidence",
"evidence_content_hash",
"is_first_for_side",
"submission_hash",
]
@@ -0,0 +1,150 @@
"""Market lifecycle state machine.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §5.2 + §3.10.
Five logical statuses:
PREDICTING — open for predictions; no snapshot yet.
EVIDENCE — snapshot frozen; evidence window open
(CONFIG['evidence_window_hours']).
RESOLVING — evidence window closed; resolution staking window open
(CONFIG['resolution_window_hours']).
FINAL — resolution_finalize event landed with a real outcome.
INVALID — resolution_finalize event landed with outcome="invalid".
Transitions are decided by ``chain_majority_time`` (per RULES §3.14
Rule 3) — no single node's local clock can unilaterally advance a
market. That rule keeps producers honest even when network partitions
shift local time.
"""
from __future__ import annotations
from enum import Enum
from typing import Any, Iterable
from services.infonet.config import CONFIG
class MarketStatus(str, Enum):
PREDICTING = "predicting"
EVIDENCE = "evidence"
RESOLVING = "resolving"
FINAL = "final"
INVALID = "invalid"
_SECONDS_PER_HOUR = 3600.0
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
def _market_id(event: dict[str, Any]) -> str:
return str(_payload(event).get("market_id") or "")
def _events_for_market(market_id: str, chain: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
out: list[dict[str, Any]] = []
for ev in chain:
if not isinstance(ev, dict):
continue
if _market_id(ev) == market_id:
out.append(ev)
out.sort(key=lambda e: (float(e.get("timestamp") or 0.0), int(e.get("sequence") or 0)))
return out
def compute_market_status(
market_id: str,
chain: Iterable[dict[str, Any]],
*,
now: float,
) -> MarketStatus:
"""Return the current status of ``market_id`` at chain time ``now``.
Status is derived from the chain — it's never stored. The producer
that emits ``market_snapshot`` and ``resolution_finalize`` events
is responsible for using the same ``now`` value (typically
``chain_majority_time(chain)``) so every node converges on the same
status.
"""
events = _events_for_market(market_id, chain)
if not events:
return MarketStatus.PREDICTING # treated as not-yet-existing
create_event = next((e for e in events if e.get("event_type") == "prediction_create"), None)
if create_event is None:
return MarketStatus.PREDICTING
finalize = next((e for e in events if e.get("event_type") == "resolution_finalize"), None)
if finalize is not None:
outcome = _payload(finalize).get("outcome")
return MarketStatus.INVALID if outcome == "invalid" else MarketStatus.FINAL
snapshot = next((e for e in events if e.get("event_type") == "market_snapshot"), None)
if snapshot is None:
return MarketStatus.PREDICTING
snapshot_ts = float(snapshot.get("timestamp") or _payload(snapshot).get("frozen_at") or 0.0)
evidence_close = snapshot_ts + float(CONFIG["evidence_window_hours"]) * _SECONDS_PER_HOUR
if now < evidence_close:
return MarketStatus.EVIDENCE
return MarketStatus.RESOLVING
def should_advance_phase(
market_id: str,
chain: Iterable[dict[str, Any]],
*,
now: float,
) -> tuple[MarketStatus, MarketStatus] | None:
"""If a phase advance is due, return ``(current, next)``. Else ``None``.
The producer should call this on a heartbeat and emit the
appropriate event when a transition is ready:
- PREDICTING → EVIDENCE: emit ``market_snapshot``.
- EVIDENCE → RESOLVING: just a status change (no chain event).
- RESOLVING → FINAL/INVALID: emit ``resolution_finalize``.
"""
events = _events_for_market(market_id, chain)
if not events:
return None
create_event = next((e for e in events if e.get("event_type") == "prediction_create"), None)
if create_event is None:
return None
finalize = next((e for e in events if e.get("event_type") == "resolution_finalize"), None)
if finalize is not None:
return None # already terminal
create_payload = _payload(create_event)
trigger_date = float(create_payload.get("trigger_date") or 0.0)
snapshot = next((e for e in events if e.get("event_type") == "market_snapshot"), None)
if snapshot is None:
# PREDICTING — advance to EVIDENCE iff trigger_date has passed in
# majority chain time.
if now >= trigger_date:
return (MarketStatus.PREDICTING, MarketStatus.EVIDENCE)
return None
snapshot_ts = float(snapshot.get("timestamp") or _payload(snapshot).get("frozen_at") or 0.0)
evidence_close = snapshot_ts + float(CONFIG["evidence_window_hours"]) * _SECONDS_PER_HOUR
resolution_close = evidence_close + float(CONFIG["resolution_window_hours"]) * _SECONDS_PER_HOUR
if now < evidence_close:
return None # still EVIDENCE
if now < resolution_close:
return (MarketStatus.EVIDENCE, MarketStatus.RESOLVING)
return (MarketStatus.RESOLVING, MarketStatus.FINAL)
__all__ = [
"MarketStatus",
"compute_market_status",
"should_advance_phase",
]
@@ -0,0 +1,488 @@
"""Market resolution — the RULES §3.10 decision procedure.
Pure function over the chain. Returns a structured ``ResolutionResult``
with the decided outcome plus all rep-transfer effects (bond returns /
forfeits / first-submitter bonuses / loser-pool burns / stalemate
burns / DA bond slashing).
Sprint 5 layers in the Round 8 defenses on top of Sprint 4's
state-machine scaffolding:
- DATA_UNAVAILABLE phantom-evidence slashing — when DA stakes meet
the threshold, ALL evidence bonds are forfeited (burned), DA voters
get full return, yes/no stakers take the stalemate burn.
- Stalemate burn on supermajority-failed INVALID — when both sides
staked above the min total but no side reached the supermajority,
ALL resolution stakes (yes/no/DA) take the burn. Bonds are returned
in good faith — the market failed, not the submitters.
What Sprint 5 still does NOT handle:
- Bootstrap-mode resolution (Sprint 8 — ``bootstrap_resolution_vote``
events with Argon2id PoW).
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Iterable
from services.infonet.config import CONFIG
from services.infonet.identity_rotation import rotation_descendants
from services.infonet.markets.data_unavailable import (
is_data_unavailable_triggered,
resolve_data_unavailable_effects,
)
from services.infonet.markets.evidence import collect_evidence
from services.infonet.markets.snapshot import find_snapshot
from services.infonet.markets.stalemate_burn import apply_to_stakes
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
def excluded_predictor_ids(
market_id: str,
chain: Iterable[dict[str, Any]],
) -> set[str]:
"""Predictor exclusion set for ``market_id`` resolution.
RULES §3.13 / §3.10 Step 1: ``frozen_predictor_ids ∪
rotation_descendants(frozen_predictor_ids)``. Walks the on-chain
rotation links — never mutates the snapshot.
Returns an empty set if no snapshot exists yet. The caller decides
whether that means "open for everyone" (no exclusion) or "reject
all" (snapshot required) — Sprint 4 ``collect_resolution_stakes``
treats absence-of-snapshot as "no exclusion".
"""
snapshot = find_snapshot(market_id, chain)
if snapshot is None:
return set()
frozen = snapshot.get("frozen_predictor_ids") or []
if not isinstance(frozen, list):
return set()
base: set[str] = {str(x) for x in frozen if isinstance(x, str) and x}
out = set(base)
chain_list = [e for e in chain if isinstance(e, dict)]
for original in base:
for desc in rotation_descendants(original, chain_list):
out.add(desc)
return out
def is_predictor_excluded(
node_id: str,
market_id: str,
chain: Iterable[dict[str, Any]],
) -> bool:
return node_id in excluded_predictor_ids(market_id, chain)
@dataclass
class _ResolutionStake:
node_id: str
side: str
amount: float
rep_type: str
timestamp: float
sequence: int
def collect_resolution_stakes(
market_id: str,
chain: Iterable[dict[str, Any]],
*,
exclude_predictors: bool = True,
) -> list[_ResolutionStake]:
"""All ``resolution_stake`` events for ``market_id`` (sorted),
with predictor exclusion applied by default.
Excluded stakes are silently dropped — they cannot influence the
outcome (RULES §3.10 Step 1). The producer-side check should also
refuse to emit them in the first place, but the resolver MUST
enforce here too because the chain is ingested from peers.
"""
excluded = excluded_predictor_ids(market_id, chain) if exclude_predictors else set()
out: list[_ResolutionStake] = []
for ev in chain:
if not isinstance(ev, dict):
continue
if ev.get("event_type") != "resolution_stake":
continue
p = _payload(ev)
if p.get("market_id") != market_id:
continue
node_id = ev.get("node_id")
if not isinstance(node_id, str) or not node_id:
continue
if node_id in excluded:
continue
side = p.get("side")
if side not in ("yes", "no", "data_unavailable"):
continue
amount = p.get("amount")
try:
amt = float(amount) if amount is not None else 0.0
except (TypeError, ValueError):
continue
if amt <= 0:
continue
rep_type = p.get("rep_type")
if rep_type not in ("oracle", "common"):
continue
out.append(_ResolutionStake(
node_id=node_id,
side=side,
amount=amt,
rep_type=rep_type,
timestamp=float(ev.get("timestamp") or 0.0),
sequence=int(ev.get("sequence") or 0),
))
out.sort(key=lambda s: (s.timestamp, s.sequence))
return out
@dataclass
class ResolutionResult:
"""Outcome + every rep-transfer effect that resolution should apply.
The producer of ``resolution_finalize`` writes the outcome onto the
chain; downstream chain readers (`oracle_rep`, `common_rep`)
recompute their views from the chain alone — they do not consume
this struct. The struct exists for tests and for the UI's
"resolution explainer" view, where users want to see *why* a market
resolved a particular way.
"""
market_id: str
outcome: str # "yes" | "no" | "invalid"
is_provisional: bool
reason: str # short diagnostic — e.g. "no_evidence", "supermajority_yes"
bond_returns: dict[str, float] = field(default_factory=dict)
bond_forfeits: dict[str, float] = field(default_factory=dict)
first_submitter_bonuses: dict[str, float] = field(default_factory=dict)
stake_returns: dict[tuple[str, str], float] = field(default_factory=dict)
"""``{(node_id, rep_type): amount}`` — full or partial returns of
resolution stakes (winners and stalemate-INVALID returns)."""
stake_winnings: dict[tuple[str, str], float] = field(default_factory=dict)
"""``{(node_id, rep_type): amount}`` — extra winnings from the
loser pool (winners only)."""
burned_amount: float = 0.0
def _supermajority_winner(
yes: float,
no: float,
threshold: float,
) -> str | None:
total = yes + no
if total <= 0:
return None
if yes / total >= threshold:
return "yes"
if no / total >= threshold:
return "no"
return None
def resolve_market(
market_id: str,
chain: Iterable[dict[str, Any]],
*,
is_provisional: bool = False,
) -> ResolutionResult:
"""Apply RULES §3.10 to compute the resolution.
Sprint 4 implements:
- Step 0: zero-evidence → INVALID (return all stakes, no penalty).
- Step 1: predictor exclusion via ``collect_resolution_stakes``.
- Step 1.5 (partial): DA threshold detection → INVALID.
*Phantom-evidence slashing is Sprint 5.*
- Step 2: oracle-rep supermajority check.
*Stalemate burn is Sprint 5.*
- Step 2.5: winning-side evidence required.
- Step 3: distribute resolution stakes (oracle + common pools, 2%
loser burn).
- Step 4: evidence bond resolution + first-submitter bonus capped
at losing-bond-pool budget.
Bootstrap-mode markets (``bootstrap_index <=
CONFIG['bootstrap_market_count']``) take a different path that
Sprint 8 will provide. Until then bootstrap markets resolve to
INVALID with reason ``bootstrap_pending``.
"""
chain_list = [e for e in chain if isinstance(e, dict)]
create_event = next(
(e for e in chain_list if e.get("event_type") == "prediction_create"
and _payload(e).get("market_id") == market_id),
None,
)
if create_event is None:
return ResolutionResult(
market_id=market_id, outcome="invalid",
is_provisional=is_provisional, reason="no_market",
)
create_payload = _payload(create_event)
bootstrap_index = create_payload.get("bootstrap_index")
if bootstrap_index is not None:
try:
bootstrap_index = int(bootstrap_index)
except (TypeError, ValueError):
bootstrap_index = None
bundles = collect_evidence(market_id, chain_list)
stakes = collect_resolution_stakes(market_id, chain_list, exclude_predictors=True)
# Step 0: zero-evidence → INVALID, return everything.
if not bundles:
result = ResolutionResult(
market_id=market_id, outcome="invalid",
is_provisional=is_provisional, reason="no_evidence",
)
for s in stakes:
result.stake_returns[(s.node_id, s.rep_type)] = (
result.stake_returns.get((s.node_id, s.rep_type), 0.0) + s.amount
)
return result
# Step 0.5: bootstrap mode (Sprint 8 — eligible-node-one-vote).
if (bootstrap_index is not None
and bootstrap_index <= int(CONFIG["bootstrap_market_count"])):
from services.infonet.bootstrap import (
deduplicate_votes,
validate_bootstrap_eligibility,
)
canonical_votes = deduplicate_votes(market_id, chain_list)
# Filter to eligible voters per RULES §3.10 step 0.5
# is_bootstrap_eligible.
eligible_votes = []
for v in canonical_votes:
node_id = v.get("node_id")
if not isinstance(node_id, str) or not node_id:
continue
if not validate_bootstrap_eligibility(node_id, market_id, chain_list).eligible:
continue
side = _payload(v).get("side")
if side not in ("yes", "no"):
continue
eligible_votes.append((node_id, side))
votes_yes = sum(1 for _, side in eligible_votes if side == "yes")
votes_no = sum(1 for _, side in eligible_votes if side == "no")
votes_total = votes_yes + votes_no
# Min participation gate.
if votes_total < int(CONFIG["min_market_participants"]):
result = ResolutionResult(
market_id=market_id, outcome="invalid",
is_provisional=is_provisional,
reason="bootstrap_below_min_participation",
)
for b in bundles:
result.bond_returns[b.node_id] = (
result.bond_returns.get(b.node_id, 0.0) + b.bond
)
return result
threshold = float(CONFIG["bootstrap_resolution_supermajority"])
if votes_yes / votes_total >= threshold:
winning_side = "yes"
elif votes_no / votes_total >= threshold:
winning_side = "no"
else:
result = ResolutionResult(
market_id=market_id, outcome="invalid",
is_provisional=is_provisional,
reason="bootstrap_no_supermajority",
)
for b in bundles:
result.bond_returns[b.node_id] = (
result.bond_returns.get(b.node_id, 0.0) + b.bond
)
return result
# Step 2.5 (winning-side evidence required) still applies in
# bootstrap mode.
winning_evidence = [b for b in bundles if b.claimed_outcome == winning_side]
if not winning_evidence:
result = ResolutionResult(
market_id=market_id, outcome="invalid",
is_provisional=is_provisional,
reason="no_winning_side_evidence",
)
for b in bundles:
result.bond_returns[b.node_id] = (
result.bond_returns.get(b.node_id, 0.0) + b.bond
)
return result
# Bootstrap markets pass directly to prediction scoring — no
# resolution-stake settlement (no oracle-rep stakes were
# collected). Evidence bonds are returned (they were 0 in
# bootstrap mode by spec, but stated for completeness).
result = ResolutionResult(
market_id=market_id, outcome=winning_side,
is_provisional=is_provisional,
reason=f"bootstrap_supermajority_{winning_side}",
)
for b in bundles:
if b.claimed_outcome == winning_side:
result.bond_returns[b.node_id] = (
result.bond_returns.get(b.node_id, 0.0) + b.bond
)
else:
result.bond_forfeits[b.node_id] = (
result.bond_forfeits.get(b.node_id, 0.0) + b.bond
)
return result
# Step 1.5: DA threshold check (Sprint 5 — phantom-evidence slashing).
if is_data_unavailable_triggered(stakes):
result = ResolutionResult(
market_id=market_id, outcome="invalid",
is_provisional=is_provisional, reason="data_unavailable",
)
effects = resolve_data_unavailable_effects(stakes, bundles)
for k, v in effects["stake_returns"].items():
result.stake_returns[k] = result.stake_returns.get(k, 0.0) + v
for node, amount in effects["bond_forfeits"].items():
result.bond_forfeits[node] = result.bond_forfeits.get(node, 0.0) + amount
result.burned_amount += float(effects["burned"])
return result
# Step 2: oracle-rep supermajority.
yes_oracle = sum(s.amount for s in stakes if s.side == "yes" and s.rep_type == "oracle")
no_oracle = sum(s.amount for s in stakes if s.side == "no" and s.rep_type == "oracle")
if yes_oracle + no_oracle < float(CONFIG["min_resolution_stake_total"]):
result = ResolutionResult(
market_id=market_id, outcome="invalid",
is_provisional=is_provisional, reason="below_min_resolution_stake",
)
for s in stakes:
result.stake_returns[(s.node_id, s.rep_type)] = (
result.stake_returns.get((s.node_id, s.rep_type), 0.0) + s.amount
)
for b in bundles:
result.bond_returns[b.node_id] = result.bond_returns.get(b.node_id, 0.0) + b.bond
return result
threshold = float(CONFIG["resolution_supermajority"])
winning_side = _supermajority_winner(yes_oracle, no_oracle, threshold)
if winning_side is None:
# No supermajority — Sprint 5 stalemate burn applies.
# Per RULES §3.10 step 2 alternate: ALL resolution stakes
# (yes / no / DA) take the burn; bonds are returned in good
# faith because the market failed (not the submitters).
result = ResolutionResult(
market_id=market_id, outcome="invalid",
is_provisional=is_provisional, reason="no_supermajority",
)
burn_returns, burn_total = apply_to_stakes(
({"node_id": s.node_id, "rep_type": s.rep_type, "amount": s.amount} for s in stakes),
)
for k, v in burn_returns.items():
result.stake_returns[k] = result.stake_returns.get(k, 0.0) + v
result.burned_amount += burn_total
for b in bundles:
result.bond_returns[b.node_id] = result.bond_returns.get(b.node_id, 0.0) + b.bond
return result
# Step 2.5: winning-side evidence required.
winning_evidence = [b for b in bundles if b.claimed_outcome == winning_side]
if not winning_evidence:
result = ResolutionResult(
market_id=market_id, outcome="invalid",
is_provisional=is_provisional, reason="no_winning_side_evidence",
)
for s in stakes:
result.stake_returns[(s.node_id, s.rep_type)] = (
result.stake_returns.get((s.node_id, s.rep_type), 0.0) + s.amount
)
for b in bundles:
result.bond_returns[b.node_id] = result.bond_returns.get(b.node_id, 0.0) + b.bond
return result
# Step 3: distribute resolution stakes per rep type.
result = ResolutionResult(
market_id=market_id, outcome=winning_side,
is_provisional=is_provisional,
reason=f"supermajority_{winning_side}",
)
burn_pct = float(CONFIG["resolution_loser_burn_pct"])
for rep_type in ("oracle", "common"):
winners = [s for s in stakes if s.side == winning_side and s.rep_type == rep_type]
# Losers exclude data_unavailable here — they vote on evidence
# quality, not outcome. Their stakes are returned in full.
losers = [s for s in stakes
if s.side != winning_side
and s.side != "data_unavailable"
and s.rep_type == rep_type]
winner_pool = sum(s.amount for s in winners)
loser_pool = sum(s.amount for s in losers)
# Always return the principal of winners and DA voters.
for s in winners:
result.stake_returns[(s.node_id, rep_type)] = (
result.stake_returns.get((s.node_id, rep_type), 0.0) + s.amount
)
for s in stakes:
if s.rep_type != rep_type:
continue
if s.side != "data_unavailable":
continue
result.stake_returns[(s.node_id, rep_type)] = (
result.stake_returns.get((s.node_id, rep_type), 0.0) + s.amount
)
if winner_pool == 0 or loser_pool == 0:
continue
burn_amt = loser_pool * burn_pct
distributable = loser_pool - burn_amt
result.burned_amount += burn_amt
for s in winners:
share = s.amount / winner_pool
winnings = share * distributable
result.stake_winnings[(s.node_id, rep_type)] = (
result.stake_winnings.get((s.node_id, rep_type), 0.0) + winnings
)
# Losing stakes are forfeited — don't return them.
# Step 4: evidence bonds.
losing_bond_pool = sum(b.bond for b in bundles if b.claimed_outcome != winning_side)
bonus_budget = losing_bond_pool
for b in bundles:
if b.claimed_outcome == winning_side:
result.bond_returns[b.node_id] = result.bond_returns.get(b.node_id, 0.0) + b.bond
if b.is_first_for_side and bonus_budget > 0:
bonus_amt = min(float(CONFIG["evidence_first_bonus"]), bonus_budget)
if bonus_amt > 0:
result.first_submitter_bonuses[b.node_id] = (
result.first_submitter_bonuses.get(b.node_id, 0.0) + bonus_amt
)
bonus_budget -= bonus_amt
else:
result.bond_forfeits[b.node_id] = result.bond_forfeits.get(b.node_id, 0.0) + b.bond
# Remaining unspent bonus budget burns (deflationary).
result.burned_amount += bonus_budget
# NOTE: subjective markets are allowed to resolve (they still
# produce a final outcome), but oracle rep is not minted from them
# — that gate lives in ``oracle_rep._market_is_mintable``.
return result
__all__ = [
"ResolutionResult",
"collect_resolution_stakes",
"excluded_predictor_ids",
"is_predictor_excluded",
"resolve_market",
]
@@ -0,0 +1,171 @@
"""Market snapshot — frozen at PREDICTING → EVIDENCE transition.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §2.2 (snapshot
fields), §3.10 (snapshot_event_hash usage), §5.2 (when emitted).
The snapshot is the **commitment boundary** for all downstream
evaluation. Once frozen:
- Liquidity gates (``min_market_participants``,
``min_market_total_stake``) are evaluated against frozen values, not
live state.
- Predictor exclusion is computed from ``frozen_predictor_ids``
(UNION ``rotation_descendants`` at resolution time).
- Bootstrap PoW uses ``snapshot_event_hash`` as its salt so attackers
can't pre-mine before the boundary.
The snapshot itself is **immutable** by spec — the producer emits it
once and never updates it. Sprint 4 enforces immutability by ignoring
any subsequent ``market_snapshot`` events with the same market_id
(``find_snapshot`` returns the FIRST one). Tests assert this invariant.
"""
from __future__ import annotations
import hashlib
import json
from typing import Any, Iterable
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
def _events_for_market(market_id: str, chain: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
out: list[dict[str, Any]] = []
for ev in chain:
if not isinstance(ev, dict):
continue
if _payload(ev).get("market_id") == market_id:
out.append(ev)
out.sort(key=lambda e: (float(e.get("timestamp") or 0.0), int(e.get("sequence") or 0)))
return out
def build_snapshot(
market_id: str,
chain: Iterable[dict[str, Any]],
*,
frozen_at: float,
) -> dict[str, Any]:
"""Compute the snapshot payload deterministically from chain history.
Walks ``prediction_place`` events for ``market_id``, in chain order,
and produces the frozen counts / stake totals / predictor list /
yes-no probability state. The resulting dict is ready to be written
as the payload of a ``market_snapshot`` event.
``frozen_at`` is the canonical commitment timestamp — typically
``chain_majority_time(chain)`` at the moment the producer decides
to advance to EVIDENCE. Pass it explicitly so the function stays
pure and deterministic.
"""
events = _events_for_market(market_id, chain)
predictor_ids: list[str] = []
seen_predictors: set[str] = set()
yes_weight = 0.0
no_weight = 0.0
total_stake = 0.0
for ev in events:
if ev.get("event_type") != "prediction_place":
continue
node = ev.get("node_id")
if not isinstance(node, str) or not node:
continue
p = _payload(ev)
side = p.get("side")
if side not in ("yes", "no"):
continue
if node not in seen_predictors:
seen_predictors.add(node)
predictor_ids.append(node)
stake = p.get("stake_amount")
if stake is not None:
try:
a = float(stake)
except (TypeError, ValueError):
a = 0.0
if a > 0:
total_stake += a
if side == "yes":
yes_weight += a
else:
no_weight += a
else:
# Free pick = 1.0 virtual stake (RULES §5.2).
if side == "yes":
yes_weight += 1.0
else:
no_weight += 1.0
pool = yes_weight + no_weight
if pool > 0:
yes_p = yes_weight / pool
else:
yes_p = 0.5
no_p = 1.0 - yes_p
return {
"market_id": market_id,
"frozen_participant_count": len(predictor_ids),
"frozen_total_stake": total_stake,
"frozen_predictor_ids": predictor_ids,
"frozen_probability_state": {"yes": yes_p, "no": no_p},
"frozen_at": float(frozen_at),
}
def compute_snapshot_event_hash(
snapshot_payload: dict[str, Any],
*,
market_id: str,
creator_node_id: str,
sequence: int,
) -> str:
"""Canonical SHA-256 of the snapshot event.
This hash is what bootstrap PoW uses as its salt (RULES §3.10 step
0.5) — committing this value on-chain prevents pre-mining of
bootstrap votes. The serialization is canonical (sorted keys,
compact separators, UTF-8) so every node arrives at the same hex.
The producer should append this value to the snapshot payload as
``snapshot_event_hash`` before emitting the event.
"""
canonical = {
"event_type": "market_snapshot",
"market_id": market_id,
"node_id": creator_node_id,
"sequence": int(sequence),
"payload": snapshot_payload,
}
encoded = json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
def find_snapshot(
market_id: str,
chain: Iterable[dict[str, Any]],
) -> dict[str, Any] | None:
"""Return the FIRST ``market_snapshot`` payload for ``market_id``.
Subsequent ``market_snapshot`` events with the same market_id are
ignored — snapshots are immutable per RULES §2.2. This is a
structural enforcement, not just a convention; an attacker who
forges a second snapshot cannot influence resolution.
"""
events = _events_for_market(market_id, chain)
for ev in events:
if ev.get("event_type") == "market_snapshot":
return _payload(ev)
return None
__all__ = [
"build_snapshot",
"compute_snapshot_event_hash",
"find_snapshot",
]
@@ -0,0 +1,94 @@
"""Stalemate burn — Round 8 anti-griefing defense.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.10 Step 2
(no-supermajority branch) and the comment block above
``CONFIG['resolution_stalemate_burn_pct']``.
The problem: without a stalemate burn, a >25% cartel can stake the
contrarian side at zero cost to permanently force INVALID and halt
oracle rep minting. Burning a small percentage of every resolution
stake when consensus fails makes that strategy progressively expensive
— the cartel bleeds rep over time.
Critical constraint (RULES §3.10 step 2 comment): the stalemate burn
ONLY applies when:
- both sides staked (total ≥ min threshold), AND
- evidence exists, AND
- supermajority not reached.
It does NOT apply when:
- zero evidence (the market gave no signal at all — not griefing),
- below-minimum participation, OR
- below-minimum stake total (uninformative — not griefing).
That's why the helper here is *non-default* — it's invoked only by the
specific branches in ``resolution.py`` that match the spec's
"genuine disagreement" case.
"""
from __future__ import annotations
from typing import Iterable
from services.infonet.config import CONFIG
def stalemate_burn_pct() -> float:
"""Current burn percentage from CONFIG. Helper so callers don't
need to remember the key name."""
return float(CONFIG["resolution_stalemate_burn_pct"])
def split_burn_and_return(amount: float, burn_pct: float | None = None) -> tuple[float, float]:
"""Compute (burn_amount, returned_amount) for a single stake."""
if amount <= 0:
return 0.0, 0.0
pct = float(stalemate_burn_pct() if burn_pct is None else burn_pct)
if pct <= 0:
return 0.0, float(amount)
if pct >= 1:
return float(amount), 0.0
burn = float(amount) * pct
returned = float(amount) - burn
return burn, returned
def apply_to_stakes(
stakes: Iterable[dict],
*,
burn_pct: float | None = None,
) -> tuple[dict[tuple[str, str], float], float]:
"""Apply the stalemate burn to ``stakes`` (iterable of dicts with
``node_id``, ``rep_type``, ``amount``).
Returns ``(returns_by_(node, rep_type), total_burned)``. The caller
folds these into the larger ``ResolutionResult`` rather than
mutating any state directly.
"""
pct = float(stalemate_burn_pct() if burn_pct is None else burn_pct)
returns: dict[tuple[str, str], float] = {}
total_burned = 0.0
for s in stakes:
node_id = s.get("node_id") if isinstance(s, dict) else getattr(s, "node_id", None)
rep_type = s.get("rep_type") if isinstance(s, dict) else getattr(s, "rep_type", None)
amount = s.get("amount") if isinstance(s, dict) else getattr(s, "amount", None)
try:
amt = float(amount) if amount is not None else 0.0
except (TypeError, ValueError):
amt = 0.0
if amt <= 0 or not isinstance(node_id, str) or rep_type not in ("oracle", "common"):
continue
burn, ret = split_burn_and_return(amt, pct)
if ret > 0:
returns[(node_id, rep_type)] = returns.get((node_id, rep_type), 0.0) + ret
total_burned += burn
return returns, total_burned
__all__ = [
"apply_to_stakes",
"split_burn_and_return",
"stalemate_burn_pct",
]
@@ -0,0 +1,60 @@
"""Two-tier state model + epoch finality (Sprint 10).
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.14 Rule 4,
``infonet-economy/IMPLEMENTATION_PLAN.md`` §3.7.
Splits protocol state into two consistency tiers:
- **Tier 1 — Eventually consistent (CRDT-friendly).** Common rep,
gate activity, content posting, upreps, vote karma. Computed
locally during partitions; merges without conflict on reconnect.
- **Tier 2 — Epoch finality required.** Oracle rep minting,
governance execution, market FINAL status, dispute outcomes,
(eventually) coin minting / dividends. MUST NOT become
economically final until an epoch checkpoint is confirmed by a
threshold of Heavy Nodes across Reticulum bridges.
Sprint 10 ships the Tier-1/Tier-2 classification, the chain-staleness
heuristic that producers consult to set ``is_provisional=True`` on
Tier-2 events, and the structural model for an `EpochCheckpoint`. The
full epoch-checkpoint protocol (BFT / threshold sigs / DAG) is open
engineering work — IMPLEMENTATION_PLAN §6.5 — and is intentionally
NOT specified here. The model + thresholds are in place; the
inter-node agreement protocol slots in later.
Why this matters today: ``oracle_rep._market_is_mintable`` (Sprint 2)
already gates on ``is_provisional == False``. Sprint 10 gives
producers the helper to set that flag correctly.
"""
from services.infonet.partition.epoch_checkpoint import (
EpochCheckpoint,
EpochCheckpointStatus,
canonical_epoch_root,
is_checkpoint_confirmed,
)
from services.infonet.partition.provisional import (
DEFAULT_MAX_CHAIN_LAG_S,
chain_lag_seconds,
is_chain_stale,
should_mark_provisional,
)
from services.infonet.partition.two_tier_state import (
TIER1_EVENT_TYPES,
TIER2_EVENT_TYPES,
classify_event_type,
)
__all__ = [
"DEFAULT_MAX_CHAIN_LAG_S",
"EpochCheckpoint",
"EpochCheckpointStatus",
"TIER1_EVENT_TYPES",
"TIER2_EVENT_TYPES",
"canonical_epoch_root",
"chain_lag_seconds",
"classify_event_type",
"is_chain_stale",
"is_checkpoint_confirmed",
"should_mark_provisional",
]
@@ -0,0 +1,144 @@
"""Epoch checkpoint model.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.14 Rule 4
("Epoch checkpoint: a global Merkle root for that epoch agreed upon
by a threshold of Heavy Nodes across Reticulum bridges. Epoch
duration, threshold, and checkpoint protocol: OPEN ENGINEERING
PROBLEM").
Sprint 10 ships the **structural model** only. The inter-node
agreement protocol (BFT vs threshold sigs vs DAG-style) is open per
IMPLEMENTATION_PLAN §6.5 and is intentionally NOT specified here.
What IS specified:
- A canonical ``EpochCheckpoint`` dataclass: epoch_id + root_hash +
participating_heavy_node_ids + threshold.
- A ``canonical_epoch_root`` helper that computes a deterministic
SHA-256 over a chain segment for a given epoch window. Every
Heavy Node computes the same value from the same chain prefix —
that's the whole point of the structural commitment.
- An ``is_checkpoint_confirmed`` predicate that says "yes, this
epoch's root has Heavy Node agreement at or above the threshold".
When the inter-node protocol lands, it produces ``EpochCheckpoint``
records that ``is_checkpoint_confirmed`` consults. Until then,
producers can hand-construct test scenarios.
"""
from __future__ import annotations
import hashlib
import json
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Iterable
class EpochCheckpointStatus(str, Enum):
PENDING = "pending"
CONFIRMED = "confirmed"
FAILED = "failed" # threshold not met by epoch deadline
@dataclass(frozen=True)
class EpochCheckpoint:
"""One epoch's chain-state commitment.
``root_hash`` is computed over the epoch's chain events using
``canonical_epoch_root``. ``participating_heavy_node_ids``
records which Heavy Nodes have signed off on this root —
confirmation requires ``len(participating) / total_heavy >=
threshold``.
Sprint 10 simplification: ``signatures`` is a dict from
``heavy_node_id`` to a placeholder bytes blob. Production wires
in the chosen threshold-signature scheme (BLS, FROST, etc.) —
those signatures aggregate into a single root signature, but
Sprint 10's structural model just tracks who signed.
"""
epoch_id: int
root_hash: str
epoch_start_ts: float
epoch_end_ts: float
participating_heavy_node_ids: frozenset[str] = frozenset()
signatures: dict[str, bytes] = field(default_factory=dict)
threshold: float = 0.67 # 67% of Heavy Nodes — same as upgrade activation
def participation_fraction(self, *, total_heavy_nodes: int) -> float:
if total_heavy_nodes <= 0:
return 0.0
return len(self.participating_heavy_node_ids) / total_heavy_nodes
def status(self, *, total_heavy_nodes: int, now: float) -> EpochCheckpointStatus:
if self.participation_fraction(total_heavy_nodes=total_heavy_nodes) >= self.threshold:
return EpochCheckpointStatus.CONFIRMED
if now > self.epoch_end_ts:
return EpochCheckpointStatus.FAILED
return EpochCheckpointStatus.PENDING
def canonical_epoch_root(
chain: Iterable[dict[str, Any]],
*,
epoch_start_ts: float,
epoch_end_ts: float,
) -> str:
"""SHA-256 over canonically-serialized events in the epoch window.
Events are filtered by ``epoch_start_ts <= timestamp < epoch_end_ts``
and sorted by ``(timestamp, sequence, event_id-or-hash)`` for
deterministic ordering. Empty epoch returns the SHA-256 of the
empty string (so even an "empty" epoch has a stable root).
Every Heavy Node computing this from the same chain prefix gets
the same hex string. Disagreement on this value is the signal
that a partition has produced divergent histories.
"""
in_window: list[dict[str, Any]] = []
for ev in chain:
if not isinstance(ev, dict):
continue
try:
ts = float(ev.get("timestamp") or 0.0)
except (TypeError, ValueError):
continue
if epoch_start_ts <= ts < epoch_end_ts:
in_window.append(ev)
in_window.sort(key=lambda e: (
float(e.get("timestamp") or 0.0),
int(e.get("sequence") or 0),
str(e.get("event_id") or ""),
))
h = hashlib.sha256()
for ev in in_window:
encoded = json.dumps(
ev, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
)
h.update(encoded.encode("utf-8"))
h.update(b"\n")
return h.hexdigest()
def is_checkpoint_confirmed(
checkpoint: EpochCheckpoint,
*,
total_heavy_nodes: int,
now: float,
) -> bool:
"""Convenience: ``True`` iff the checkpoint has reached the
Heavy Node threshold."""
return (
checkpoint.status(total_heavy_nodes=total_heavy_nodes, now=now)
== EpochCheckpointStatus.CONFIRMED
)
__all__ = [
"EpochCheckpoint",
"EpochCheckpointStatus",
"canonical_epoch_root",
"is_checkpoint_confirmed",
]
@@ -0,0 +1,94 @@
"""Provisional-flag heuristic — chain-staleness detection.
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §3.7
("initial implementation can gate economic events with
``is_provisional=True`` whenever the local chain head's
``chain_majority_time`` is older than X seconds").
Sprint 10 ships the placeholder for full epoch finality. Producers
emitting Tier 2 events consult ``should_mark_provisional`` to decide
whether to set ``is_provisional=True``. Once the formal epoch
checkpoint protocol is shipped (IMPLEMENTATION_PLAN §6.5), this
heuristic gets replaced with a check against the latest confirmed
checkpoint.
Until then, the heuristic is: if local chain time hasn't advanced in
``DEFAULT_MAX_CHAIN_LAG_S`` seconds, the network is partitioned (or
dramatically slow); Tier 2 events emitted now are provisional.
Cross-cutting design rule: a partitioned node must NOT block the
user from emitting actions. Tier 1 actions are always live; Tier 2
actions are accepted but marked provisional. Reconnection promotes
provisional events to final once the checkpoint clears.
"""
from __future__ import annotations
from typing import Any, Iterable
from services.infonet.partition.two_tier_state import classify_event_type
from services.infonet.time_validity import chain_majority_time
# Default: 60 seconds. After 1 minute without a chain advance from a
# distinct node, Tier 2 events get marked provisional. This is a
# conservative default — production deployments will likely tune
# higher (5-10 minutes) once the network is large and partitions
# are rare. Currently NOT in CONFIG_SCHEMA — see Sprint 10 hand-off
# notes for the open governance question.
DEFAULT_MAX_CHAIN_LAG_S: float = 60.0
def chain_lag_seconds(
chain: Iterable[dict[str, Any]],
*,
now: float,
) -> float:
"""Seconds elapsed between ``chain_majority_time(chain)`` and ``now``.
Returns ``0.0`` if ``now`` is at or before chain time (clock skew
or the chain genuinely caught up just now). Always non-negative.
"""
cmt = chain_majority_time(chain)
if cmt <= 0:
# Empty chain — no events from distinct nodes yet. Treat as
# "infinite lag" so Tier 2 emissions are provisional.
return float("inf")
return max(0.0, float(now) - cmt)
def is_chain_stale(
chain: Iterable[dict[str, Any]],
*,
now: float,
max_lag_seconds: float = DEFAULT_MAX_CHAIN_LAG_S,
) -> bool:
"""``True`` iff the chain hasn't advanced in ``max_lag_seconds``."""
return chain_lag_seconds(chain, now=now) > float(max_lag_seconds)
def should_mark_provisional(
event_type: str,
chain: Iterable[dict[str, Any]],
*,
now: float,
max_lag_seconds: float = DEFAULT_MAX_CHAIN_LAG_S,
) -> bool:
"""Should ``event_type`` carry ``is_provisional=True`` if emitted now?
Tier 1 events: always ``False`` (they're CRDT-friendly).
Tier 2 events: ``True`` iff chain is stale.
Infrastructure / unknown: ``False`` (no economic finality at stake).
"""
tier = classify_event_type(event_type)
if tier != "tier2":
return False
return is_chain_stale(chain, now=now, max_lag_seconds=max_lag_seconds)
__all__ = [
"DEFAULT_MAX_CHAIN_LAG_S",
"chain_lag_seconds",
"is_chain_stale",
"should_mark_provisional",
]
@@ -0,0 +1,166 @@
"""Tier 1 / Tier 2 event-type classification.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.14 Rule 4.
Tier 1 events:
- **Eventually consistent.** A node operating in a partition can
produce them locally; on reconnect they merge into the global
view without conflict.
- **CRDT-friendly.** No total ordering required.
- Examples: upreps, gate enter/exit, gate messages (off-chain),
citizenship claim signal, content posts.
Tier 2 events:
- **Epoch finality required.** A node operating in a partition can
*propose* them locally with ``is_provisional=True``, but they MUST
NOT become economically final until an epoch checkpoint confirms
the chain head.
- Examples: oracle rep minting (via ``resolution_finalize``),
governance execution, dispute outcomes.
The classifier returns "tier1", "tier2", or "infrastructure" (for
event types that don't directly affect economic state — e.g.
``node_register``).
"""
from __future__ import annotations
from services.infonet.schema import INFONET_ECONOMY_EVENT_TYPES
# Sprint 10 baseline classification. Future governance can rebalance
# via upgrade-hash governance (the classification is a constitutional
# property — moving an event between tiers changes finality semantics).
TIER1_EVENT_TYPES: frozenset[str] = frozenset({
# Reputation surface — common rep is fully chain-derived and
# CRDT-friendly. Upreps from disjoint partitions add commutatively.
"uprep",
"downrep",
# Gate membership — entering / exiting a gate is local action;
# final view is just the union (modulo exit removals).
"gate_enter",
"gate_exit",
"gate_lock", # locking is a vote — partition-local locks count
# Content / citizenship signals — pure local actions.
"post_create",
"post_reply",
"citizenship_claim",
# Predictions are local; what's NOT Tier 1 is the resolution.
"prediction_create",
"prediction_place",
# Truth stakes — same: placing is Tier 1, resolving is Tier 2.
"truth_stake_place",
# Bounty creation / claim acknowledgements — local action.
"bounty_create",
"bounty_claim",
})
TIER2_EVENT_TYPES: frozenset[str] = frozenset({
# Resolution finality — must be confirmed by epoch checkpoint
# before oracle_rep mints.
"market_snapshot",
"evidence_submit",
"resolution_stake",
"bootstrap_resolution_vote",
"resolution_finalize",
# Truth stake resolution.
"truth_stake_resolve",
# Disputes — the bounded-reversal mechanic depends on a stable
# global view; partition-only dispute resolution would diverge.
"dispute_open",
"dispute_stake",
"dispute_resolve",
# Gate shutdown — irreversible state change; must reach global
# consensus before execute.
"gate_suspend_file",
"gate_suspend_vote",
"gate_suspend_execute",
"gate_shutdown_file",
"gate_shutdown_vote",
"gate_shutdown_execute",
"gate_unsuspend",
"gate_shutdown_appeal_file",
"gate_shutdown_appeal_vote",
"gate_shutdown_appeal_resolve",
# Governance — petitions and upgrades affect protocol params /
# release hash globally. Must not execute provisionally.
"petition_file",
"petition_sign",
"petition_vote",
"challenge_file",
"challenge_vote",
"petition_execute",
"upgrade_propose",
"upgrade_sign",
"upgrade_vote",
"upgrade_challenge",
"upgrade_challenge_vote",
"upgrade_signal_ready",
"upgrade_activate",
# Coin events — when shipped, must not double-mint across
# partitions. (Sprint 9 currently SKIPPED; classification kept
# so it's ready when un-skipped.)
"coin_transfer",
"coin_mint",
# Identity rotation — re-keying must reach global consensus.
"identity_rotate",
})
# Infrastructure events — neither tier (don't directly drive
# economic state). Currently just node_register.
_INFRASTRUCTURE_TYPES: frozenset[str] = frozenset({
"node_register",
})
def classify_event_type(event_type: str) -> str:
"""Return ``"tier1"`` / ``"tier2"`` / ``"infrastructure"`` /
``"unknown"``.
Validates the classification covers the entire
``INFONET_ECONOMY_EVENT_TYPES`` surface — Sprint 10's invariant
test asserts this.
"""
if event_type in TIER1_EVENT_TYPES:
return "tier1"
if event_type in TIER2_EVENT_TYPES:
return "tier2"
if event_type in _INFRASTRUCTURE_TYPES:
return "infrastructure"
return "unknown"
def assert_classification_complete() -> None:
"""Sprint 10 invariant: every economy event type is classified.
Called from the test suite. Raising at import time would be too
aggressive — a future event type added without a tier assignment
should fail loudly in CI, not crash production.
"""
classified = TIER1_EVENT_TYPES | TIER2_EVENT_TYPES | _INFRASTRUCTURE_TYPES
missing = sorted(INFONET_ECONOMY_EVENT_TYPES - classified)
if missing:
raise AssertionError(
f"Tier classification incomplete — these event types have no "
f"tier assignment: {missing}. Add them to TIER1_EVENT_TYPES, "
f"TIER2_EVENT_TYPES, or _INFRASTRUCTURE_TYPES."
)
overlap = TIER1_EVENT_TYPES & TIER2_EVENT_TYPES
if overlap:
raise AssertionError(
f"Tier classification overlapping — these types are in both "
f"Tier 1 and Tier 2: {sorted(overlap)}"
)
__all__ = [
"TIER1_EVENT_TYPES",
"TIER2_EVENT_TYPES",
"assert_classification_complete",
"classify_event_type",
]
@@ -0,0 +1,84 @@
"""Privacy layer scaffolding (Sprint 11+ runway).
The privacy layer protects the protocol's core promise:
**your identity is your reputation, not your legal name**.
Constitutional anchors (IMPLEMENTATION_PLAN.md §4):
- **Reputation chain is fully public.** Every uprep / prediction /
vote / governance action is signed and visible. The privacy layer
does NOT hide reputation actions.
- **Coin ledger is privacy-preserving.** When the coin layer ships,
transfers / balances / DEX trades are shielded. Privacy work in
this folder is what makes that possible.
- **Optional privacy is no privacy.** The default for coin
transactions must be shielded — opt-out cannot exist or it
destroys the anonymity set.
This package is intentionally **scaffolding only** at present. Each
primitive (RingCT, stealth addresses, shielded balance commitments,
DEX) defines its public interface as a typed Protocol so production
code can depend on the *shape* before any specific cryptographic
implementation is committed.
The non-cryptographic pieces of the Function Keys design (nullifier
hashing, challenge-response orchestration, two-phase commit receipts,
batched settlement aggregation) ARE implemented here in pure Python.
The remaining cryptographic primitive (blind signature / anonymous
credential scheme) is the only piece blocking production deployment
of Function Keys; everything around it is ready.
See ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4 and
``infonet-economy/BRAINDUMP.md`` §5.6, §11 item 9 for design
rationale.
"""
from services.infonet.privacy.contracts import (
BalanceCommitment,
DEXOrderBook,
PrivacyPrimitiveStatus,
RingSignatureScheme,
StealthAddressScheme,
)
from services.infonet.privacy.dex import DEXScaffolding
from services.infonet.privacy.function_keys import (
BatchedSettlementBatch,
DenialCode,
FunctionKey,
FunctionKeyChallenge,
FunctionKeyResponse,
NullifierTracker,
Receipt,
ReceiptPair,
derive_nullifier,
issue_challenge,
sign_response,
verify_response,
)
from services.infonet.privacy.ringct import RingCTScaffolding
from services.infonet.privacy.shielded_balance import ShieldedBalanceScaffolding
from services.infonet.privacy.stealth_address import StealthAddressScaffolding
__all__ = [
"BalanceCommitment",
"BatchedSettlementBatch",
"DEXOrderBook",
"DEXScaffolding",
"DenialCode",
"FunctionKey",
"FunctionKeyChallenge",
"FunctionKeyResponse",
"NullifierTracker",
"PrivacyPrimitiveStatus",
"Receipt",
"ReceiptPair",
"RingCTScaffolding",
"RingSignatureScheme",
"ShieldedBalanceScaffolding",
"StealthAddressScaffolding",
"StealthAddressScheme",
"derive_nullifier",
"issue_challenge",
"sign_response",
"verify_response",
]
@@ -0,0 +1,190 @@
"""Typed protocols for the cryptographic primitives.
Production code depends on the **shape** of each privacy primitive
through a ``Protocol`` defined here. Concrete implementations (Rust
binding, Python reference, test mock) all match the same shape, so
swapping them is a one-line import change.
Sprint 11+ ships:
- A reference Python implementation for testing (probably built on
``cryptography`` or ``ecdsa`` packages, narrow scope).
- A production Rust binding via ``privacy-core`` crate.
Today (Sprint 11+ runway), this module ships:
- ``Protocol``s for each primitive.
- ``PrivacyPrimitiveStatus`` enum so callers can introspect which
implementations are wired in.
- A registry of "not yet implemented" statuses with diagnostic
pointers for future implementers.
"""
from __future__ import annotations
from enum import Enum
from typing import Any, Protocol, runtime_checkable
class PrivacyPrimitiveStatus(str, Enum):
"""Lifecycle status for each privacy primitive.
Used by health endpoints / UI to communicate "this feature is
not yet shielded" honestly. The cross-cutting non-hostile UX
rule (BUILD_LOG.md design rules §1) forbids silently pretending
a primitive is ready when it isn't — surface the truth.
"""
NOT_IMPLEMENTED = "not_implemented"
SCAFFOLDING = "scaffolding"
REFERENCE_IMPL = "reference_impl"
PRODUCTION_RUST = "production_rust"
# ─── Ring confidential transactions ─────────────────────────────────────
@runtime_checkable
class RingSignatureScheme(Protocol):
"""Signs a transaction with a ring of public keys, hiding which
member of the ring actually signed.
Implementations must guarantee:
- **Unforgeable.** Without one of the ring members' private keys,
no valid ring signature exists for the transaction.
- **Anonymous within the ring.** Verifiers learn that *some*
ring member signed, not which.
- **Linkable.** Two signatures from the same private key produce
the same ``key image`` (used to detect double-spends).
"""
def sign(
self,
*,
message: bytes,
signer_private_key: bytes,
ring_public_keys: list[bytes],
) -> dict[str, Any]:
"""Return ``{"signature": ..., "key_image": ...}``."""
...
def verify(
self,
*,
message: bytes,
signature: dict[str, Any],
ring_public_keys: list[bytes],
) -> bool: ...
def status(self) -> PrivacyPrimitiveStatus: ...
# ─── Stealth addresses ──────────────────────────────────────────────────
@runtime_checkable
class StealthAddressScheme(Protocol):
"""Derives a one-time recipient address per transaction.
Implementations must guarantee:
- **Unlinkable.** An external observer cannot tell that two
stealth addresses belong to the same recipient.
- **Recipient-recoverable.** Only the recipient (using their
view key) can determine that an output is theirs.
"""
def derive_one_time_address(
self,
*,
recipient_view_key: bytes,
recipient_spend_key: bytes,
sender_random: bytes,
) -> bytes: ...
def is_for_recipient(
self,
*,
one_time_address: bytes,
recipient_view_key: bytes,
recipient_spend_key: bytes,
sender_random: bytes,
) -> bool: ...
def status(self) -> PrivacyPrimitiveStatus: ...
# ─── Shielded balance commitment ────────────────────────────────────────
@runtime_checkable
class BalanceCommitment(Protocol):
"""Pedersen / homomorphic commitment to a balance.
Implementations must allow:
- Commit to a balance ``B`` with blinding factor ``r``.
- Verify a sum-of-commitments equals zero (proving inputs ==
outputs without revealing amounts).
- Range proofs (proving each output is non-negative).
"""
def commit(self, *, amount: int, blinding: bytes) -> bytes: ...
def verify_balance(
self,
*,
input_commitments: list[bytes],
output_commitments: list[bytes],
) -> bool: ...
def range_proof(
self,
*,
amount: int,
blinding: bytes,
max_bits: int = 64,
) -> bytes: ...
def verify_range_proof(
self,
*,
commitment: bytes,
proof: bytes,
max_bits: int = 64,
) -> bool: ...
def status(self) -> PrivacyPrimitiveStatus: ...
# ─── DEX order book ─────────────────────────────────────────────────────
@runtime_checkable
class DEXOrderBook(Protocol):
"""Privacy-preserving decentralized exchange interface.
DEX operates ON TOP of the shielded coin layer — orders reference
shielded inputs/outputs, settlement burns + mints shielded
commitments. The ``DEXOrderBook`` Protocol is intentionally
abstract because the specific scheme (CoW-style batched
settlement, atomic swap, MimbleWimble-flavored aggregation) is
still open per IMPLEMENTATION_PLAN.md §6.4.
"""
def place_order(self, *, order: dict[str, Any]) -> str:
"""Return the on-chain ``order_id``."""
...
def cancel_order(self, *, order_id: str, owner_signature: bytes) -> None: ...
def match_orders(self) -> list[dict[str, Any]]:
"""Return the list of matched trades for atomic settlement."""
...
def status(self) -> PrivacyPrimitiveStatus: ...
__all__ = [
"BalanceCommitment",
"DEXOrderBook",
"PrivacyPrimitiveStatus",
"RingSignatureScheme",
"StealthAddressScheme",
]
+49
View File
@@ -0,0 +1,49 @@
"""Decentralized exchange — Sprint 11+ scaffolding.
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.3,
§6.4 (open: same chain vs side-chain).
The DEX operates on top of the shielded coin layer. Orders reference
shielded inputs and outputs; settlement burns + mints commitments
atomically. The Sprint 11+ scaffolding here defines the order /
settlement shapes without committing to a specific matching scheme
(CoW-style batch auction, atomic swap, etc.).
External exchanges WILL list CommonCoin regardless of protocol
design — the protocol's privacy layer is what prevents external-
exchange listings from de-anonymizing protocol participants. The
on-chain DEX is the *primary* exchange mechanism, not the only one.
"""
from __future__ import annotations
from typing import Any
from services.infonet.privacy.contracts import PrivacyPrimitiveStatus
class DEXScaffolding:
"""Placeholder until the DEX scheme decision (§6.4) is made and a
matching engine is built on top of the shielded coin layer."""
_DIAGNOSTIC = (
"DEX is scaffolding only — see IMPLEMENTATION_PLAN.md §6.4 "
"for the open scheme decision (same chain vs side-chain) and "
"§4.3 for the privacy requirements. Production implementation "
"depends on the shielded coin layer being shipped first."
)
def place_order(self, *, order: dict[str, Any]) -> str:
raise NotImplementedError(self._DIAGNOSTIC)
def cancel_order(self, *, order_id: str, owner_signature: bytes) -> None:
raise NotImplementedError(self._DIAGNOSTIC)
def match_orders(self) -> list[dict[str, Any]]:
raise NotImplementedError(self._DIAGNOSTIC)
def status(self) -> PrivacyPrimitiveStatus:
return PrivacyPrimitiveStatus.NOT_IMPLEMENTED
__all__ = ["DEXScaffolding"]
@@ -0,0 +1,85 @@
"""Function Keys — anonymous citizenship proof.
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.4,
``infonet-economy/BRAINDUMP.md`` §11 item 9.
A citizen should be able to prove "I am a UBI-eligible Infonet
citizen" to a real-world operator (food bank, community service)
**without revealing their Infonet identity**. The naive approach
(scramble a public key, record each redemption on chain) leaks
identity through metadata correlation (time, location, operator,
frequency).
The full design has six pieces; five are implemented in pure Python
here. The remaining piece — issuance via blind signatures or
anonymous credentials — is the only cryptographic primitive that
needs an external library.
Pieces:
1. **Issuance** (NOT IMPLEMENTED — needs blind sig / BBS+ / U-Prove
/ Idemix). The ``FunctionKey`` dataclass models what an issued
key looks like; production wires the issuer through a Protocol
when the scheme is chosen.
2. **Nullifiers** (`nullifier.py`) — SHA-256 of secret + operator_id.
Different operators see different nullifiers for the same key,
so cross-operator linkage is impossible. One-time-use per
operator: tracked via ``NullifierTracker``.
3. **Challenge-response** (`challenge_response.py`) — operator
issues a fresh nonce, key-holder signs with the Function Key's
secret. Prevents screenshot attacks, key sharing, replay.
4. **Two-phase commit receipts** (`receipt.py`) — Phase 1
verification receipt (operator-signed, day-level date NOT
timestamp, no node_id). Phase 2 fulfillment receipt (citizen
counter-signs after service rendered). Receipts NEVER published
on-chain — only surface on dispute.
5. **Enumerated denial codes** (`receipt.py`) — operators can
reject for exactly three reasons: invalid signature, nullifier
already seen, rate limit exceeded. Prevents discrimination via
freeform rejection.
6. **Batched/coarse-grained settlement** (`batched_settlement.py`)
— operators settle in aggregate. Chain sees "Operator X
verified N function keys this period." Per-redemption records
never reach the chain.
Cross-cutting design rule: the user redeeming a Function Key must
not be blocked by privacy/security mechanics. If the cryptographic
primitive is unavailable in the local node, the redemption is
queued for retry once the operator has connectivity, NOT refused.
"""
from services.infonet.privacy.function_keys.batched_settlement import (
BatchedSettlementBatch,
)
from services.infonet.privacy.function_keys.challenge_response import (
FunctionKey,
FunctionKeyChallenge,
FunctionKeyResponse,
issue_challenge,
sign_response,
verify_response,
)
from services.infonet.privacy.function_keys.nullifier import (
NullifierTracker,
derive_nullifier,
)
from services.infonet.privacy.function_keys.receipt import (
DenialCode,
Receipt,
ReceiptPair,
)
__all__ = [
"BatchedSettlementBatch",
"DenialCode",
"FunctionKey",
"FunctionKeyChallenge",
"FunctionKeyResponse",
"NullifierTracker",
"Receipt",
"ReceiptPair",
"derive_nullifier",
"issue_challenge",
"sign_response",
"verify_response",
]
@@ -0,0 +1,112 @@
"""Batched settlement — aggregate counts, no individual records on-chain.
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.4
piece 6.
Per-redemption records on-chain would be a privacy disaster: an
observer could correlate "Operator X verified a Function Key at
14:32" with a citizen's known activities to de-anonymize them.
Instead, operators settle in **aggregate**. The chain sees only
``(operator_id, day_bucket, count)`` — verified N keys this day.
Fraud detection happens via statistical auditing rather than
per-redemption traces:
- Operator's count vs their declared population (food bank that
reports 10,000 daily verifications when their service capacity
is 200).
- Distribution shape vs other operators (significant outliers
prompt review).
- Spot audits via dispute mechanism (citizen + operator surface
receipt pair to adjudicator).
The ``BatchedSettlementBatch`` here is what the operator emits
to chain at the end of a settlement period. Receipts NEVER appear
on-chain — they remain off-chain with both parties.
Sprint 11+ scaffolding ships:
- The aggregate batch dataclass.
- A ``record_redemption`` helper that operators call locally per
successful redemption — increments the batch's counter without
storing the receipt.
- A ``finalize_batch`` step that produces the on-chain payload.
This module is **fully implementable** today — it does no
cryptography, just bookkeeping.
"""
from __future__ import annotations
from dataclasses import dataclass, field
@dataclass
class BatchedSettlementBatch:
"""Operator-side batch counter for one settlement period.
Operators construct one of these per ``(period_id, operator_id)``
pair, increment via ``record_redemption`` per successful
redemption, and emit the finalized batch payload at period end.
The data model is intentionally minimal:
- ``period_id`` — the settlement window identifier (e.g.
``"2026-04"`` for monthly).
- ``operator_id`` — committed publicly on-chain so its
non-forgeability is anchored.
- ``successful_count`` — number of successful redemptions
(verification + fulfillment).
- ``denial_counts`` — counts per enumerated DenialCode for
audit visibility. NO per-receipt detail.
"""
period_id: str
operator_id: str
successful_count: int = 0
denial_counts: dict[str, int] = field(default_factory=dict)
finalized: bool = False
def record_redemption(self) -> None:
"""Increment the success counter. NOT idempotent — call
exactly once per successful (verification, fulfillment)
receipt pair the operator commits to."""
if self.finalized:
raise RuntimeError("batch already finalized; cannot record")
self.successful_count += 1
def record_denial(self, code: str) -> None:
"""Track a denial. Operators MUST use one of the enumerated
``DenialCode`` values — Sprint 11+ scaffolding accepts the
string for convenience but production callers should pass
the enum's ``.value``."""
if self.finalized:
raise RuntimeError("batch already finalized; cannot record")
if not isinstance(code, str) or not code:
raise ValueError("denial code must be a non-empty string")
self.denial_counts[code] = self.denial_counts.get(code, 0) + 1
def finalize(self) -> dict:
"""Produce the on-chain payload for this batch.
After ``finalize()``, ``record_redemption`` and
``record_denial`` raise. The returned dict is the canonical
batched-settlement event payload.
Privacy property: per-receipt detail is NOT in the output.
Only counts. The operator may discard receipts after
finalization (subject to local retention policy for dispute
defense).
"""
if self.finalized:
raise RuntimeError("batch already finalized")
self.finalized = True
return {
"period_id": self.period_id,
"operator_id": self.operator_id,
"successful_count": int(self.successful_count),
"denial_counts": {k: int(v) for k, v in self.denial_counts.items()},
}
__all__ = ["BatchedSettlementBatch"]
@@ -0,0 +1,208 @@
"""Challenge-response — live proof of Function Key possession.
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.4
piece 3.
Operator issues a fresh nonce; key-holder signs (challenge || nonce
|| epoch_window) with the Function Key's secret. Operator verifies
by re-deriving the signature.
This defends against:
- **Screenshot attacks** — a recorded "valid proof" from yesterday
is useless against today's challenge.
- **Key sharing** — without the live secret, no valid response
exists; sharing the secret = sharing the key (which has its own
social cost via public reputation).
- **Replay** — the operator stores recent nonces; replayed
responses are rejected.
Sprint 11+ scaffolding ships:
- The ``FunctionKey`` dataclass (the post-issuance shape).
- The challenge / response message structures.
- A pure-Python ``sign_response`` / ``verify_response`` pair using
HMAC-SHA256 as the placeholder MAC scheme. Production wires this
through the eventual blind-sig / anonymous credential primitive.
The HMAC placeholder is **explicitly NOT secure for unlinkable
issuance** — it leaks issuer identity through the verification key.
But it's correctly-shaped for testing the rest of the pipeline
(nullifier flow, receipt flow, batched settlement) without blocking
on the cryptographic decision in IMPLEMENTATION_PLAN §6.4.
"""
from __future__ import annotations
import hashlib
import hmac
import secrets
import time
from dataclasses import dataclass
from typing import Iterable
# Maximum age (in seconds) for a challenge. Outside this window, the
# response is rejected. Defaults to 5 minutes — short enough to defeat
# screenshot attacks, long enough to survive normal network latency on
# slow operator hardware.
DEFAULT_CHALLENGE_TTL_SECONDS = 300
@dataclass(frozen=True)
class FunctionKey:
"""Post-issuance Function Key.
``secret`` is what the citizen retains; production keys derive
additional fields (like ``epoch`` and ``credential``). The blind-
signature implementation populates ``credential`` with the
issuer's signature on the secret + epoch.
Sprint 11+ scaffolding: ``credential`` is just bytes — the
semantic depends on the chosen scheme. Tests can use any
deterministic value.
"""
secret: bytes
epoch: str
credential: bytes
# The issuer's verification context — production stores the
# public params needed to verify ``credential``. Sprint 11+
# scaffolding accepts any opaque bytes.
issuer_context: bytes = b""
@dataclass(frozen=True)
class FunctionKeyChallenge:
"""An operator-generated fresh challenge.
The ``nonce`` is the entropy source; ``operator_id`` ties the
challenge to a specific operator (so cross-operator response
reuse is impossible); ``issued_at`` is the start of the TTL
window.
"""
nonce: bytes
operator_id: str
issued_at: float
def canonical_bytes(self) -> bytes:
# Pipe-delimited UTF-8 — same canonicalization style as the
# Sprint 8 PoW preimage so the convention is uniform.
return b"|".join([
b"function_key_challenge",
self.nonce,
self.operator_id.encode("utf-8"),
repr(self.issued_at).encode("utf-8"),
])
@dataclass(frozen=True)
class FunctionKeyResponse:
"""Citizen's signed response to a challenge."""
nonce: bytes
operator_id: str
issued_at: float
nullifier: str
mac: bytes # in production: blind-signature proof; here HMAC-SHA256
def issue_challenge(*, operator_id: str, now: float | None = None) -> FunctionKeyChallenge:
"""Generate a fresh ``FunctionKeyChallenge`` for ``operator_id``.
The ``nonce`` is 32 bytes from ``secrets.token_bytes`` — full
256-bit entropy, OS-source. ``issued_at`` defaults to
``time.time()`` and is included in the canonical bytes so a
challenge from yesterday cannot be replayed today.
"""
if not isinstance(operator_id, str) or not operator_id:
raise ValueError("operator_id must be a non-empty string")
return FunctionKeyChallenge(
nonce=secrets.token_bytes(32),
operator_id=operator_id,
issued_at=float(now if now is not None else time.time()),
)
def sign_response(
*,
key: FunctionKey,
challenge: FunctionKeyChallenge,
) -> FunctionKeyResponse:
"""Sign a challenge with the Function Key's secret.
Sprint 11+ placeholder uses HMAC-SHA256 with ``key.secret`` as
the MAC key. Production wires the blind-signature scheme here:
the response includes a zero-knowledge proof that the holder
knows a credential signed by the issuer over the secret +
epoch, without revealing which credential.
"""
from services.infonet.privacy.function_keys.nullifier import derive_nullifier
nullifier = derive_nullifier(secret=key.secret, operator_id=challenge.operator_id)
body = challenge.canonical_bytes() + b"|" + nullifier.encode("utf-8")
mac = hmac.new(key.secret, body, hashlib.sha256).digest()
return FunctionKeyResponse(
nonce=challenge.nonce,
operator_id=challenge.operator_id,
issued_at=challenge.issued_at,
nullifier=nullifier,
mac=mac,
)
def verify_response(
*,
response: FunctionKeyResponse,
key: FunctionKey,
max_age: float = DEFAULT_CHALLENGE_TTL_SECONDS,
now: float | None = None,
seen_nonces: Iterable[bytes] = (),
) -> tuple[bool, str]:
"""Verify a response against the matching key + check freshness.
Returns ``(accepted, reason)``. ``accepted=False`` produces one
of these diagnostic reasons:
- ``"stale_challenge"`` — challenge too old.
- ``"replay_nonce_seen"`` — nonce was used in a prior verified
response.
- ``"invalid_mac"`` — MAC didn't verify against the key.
Operators MUST track recently-seen nonces (for the duration of
the TTL plus a margin) to defeat replay. Pass them in via
``seen_nonces``.
Note on the verifier-knows-the-secret problem: with the HMAC
placeholder, the verifier needs ``key.secret`` to verify. That's
obviously NOT private — it's why this is a placeholder. The
production blind-sig scheme verifies *without* knowing the
secret, only the issuer's public verification context.
"""
seen_set = set(seen_nonces)
if response.nonce in seen_set:
return False, "replay_nonce_seen"
age_s = float(now if now is not None else time.time()) - response.issued_at
if age_s > max_age or age_s < 0:
return False, "stale_challenge"
challenge = FunctionKeyChallenge(
nonce=response.nonce,
operator_id=response.operator_id,
issued_at=response.issued_at,
)
body = challenge.canonical_bytes() + b"|" + response.nullifier.encode("utf-8")
expected = hmac.new(key.secret, body, hashlib.sha256).digest()
if not hmac.compare_digest(expected, response.mac):
return False, "invalid_mac"
return True, "ok"
__all__ = [
"DEFAULT_CHALLENGE_TTL_SECONDS",
"FunctionKey",
"FunctionKeyChallenge",
"FunctionKeyResponse",
"issue_challenge",
"sign_response",
"verify_response",
]
@@ -0,0 +1,92 @@
"""Nullifiers — one-time-use markers per (key, operator) pair.
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.4
piece 2.
For each Function Key + operator combination, the nullifier is
nullifier = SHA-256(secret || operator_id)
Properties this gives us:
- **One-time-use per operator.** The operator records the nullifier
on first use; subsequent attempts with the same nullifier are
rejected (denial code ``NULLIFIER_ALREADY_SEEN``).
- **Cross-operator unlinkability.** Different ``operator_id``s
produce different nullifiers for the same secret. Two operators
comparing notes cannot determine that the same key was used at
both — they see two unrelated 32-byte strings.
- **No identity leakage.** The nullifier is a hash; the secret is
never exposed.
Operators MUST commit ``operator_id`` publicly so its non-forgeability
is anchored on chain. Nullifier derivation depends on a
non-forgeable ``operator_id`` (an attacker who could impersonate an
operator could harvest nullifiers).
"""
from __future__ import annotations
import hashlib
from dataclasses import dataclass, field
def derive_nullifier(*, secret: bytes, operator_id: str) -> str:
"""Return the hex SHA-256 of ``secret || operator_id`` (UTF-8 for
operator_id).
Stable across reboots / sessions / operating systems — the same
inputs always produce the same output. That's the whole property
a nullifier needs: deterministic and unforgeable.
"""
if not isinstance(secret, (bytes, bytearray)):
raise TypeError("secret must be bytes")
if not isinstance(operator_id, str) or not operator_id:
raise ValueError("operator_id must be a non-empty string")
h = hashlib.sha256()
h.update(bytes(secret))
h.update(b"|") # explicit separator so concatenation is unambiguous
h.update(operator_id.encode("utf-8"))
return h.hexdigest()
@dataclass
class NullifierTracker:
"""Operator-side store of seen nullifiers.
Sprint 11+ runway: this is the in-memory reference implementation.
Production operators use a persistent, atomic-write store
(database row + uniqueness constraint) so the "already-seen"
check is robust to crashes between the check and the receipt.
The interface is designed for that: ``check_and_record`` is the
only mutation method, and it's atomic — checks then records as
one operation. Production wraps this in a database transaction.
"""
seen: set[str] = field(default_factory=set)
def has_seen(self, nullifier: str) -> bool:
return nullifier in self.seen
def check_and_record(self, nullifier: str) -> bool:
"""Return ``True`` if the nullifier was unseen (and is now
recorded). Return ``False`` if it was already seen — the
operator MUST then issue a denial with code
``NULLIFIER_ALREADY_SEEN``.
The check + record is atomic by design: a concurrent caller
racing with this method will not produce two ``True`` results
for the same nullifier. (In-memory: trivially atomic. Production:
wrap in DB unique-insert.)
"""
if nullifier in self.seen:
return False
self.seen.add(nullifier)
return True
__all__ = [
"NullifierTracker",
"derive_nullifier",
]
@@ -0,0 +1,221 @@
"""Two-phase commit receipts + enumerated denial codes.
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.4
pieces 5-6.
Two phases per redemption:
1. **Verification receipt** (operator → citizen). Operator signs:
``(receipt_id, operator_id, day_bucket, nullifier_prefix)``.
Note: NO timestamp (day-bucket only), NO node_id, NO nullifier
in full (a prefix that's still distinct enough for fraud auditing
but doesn't leak the full unforgeable nullifier).
2. **Fulfillment receipt** (citizen → operator). Citizen counter-
signs the verification receipt after service is rendered. Both
parties hold a copy.
Receipts are NEVER published on-chain. They surface only in
disputes. Settlement to chain happens through batched aggregation
(``batched_settlement.py``).
Denial codes are an **enumerated** set with exactly three values.
Operators cannot reject for freeform reasons — that would be a
discrimination vector. The three reasons are:
- ``INVALID_SIGNATURE`` — challenge-response verification failed.
- ``NULLIFIER_ALREADY_SEEN`` — the (key, operator) pair has already
redeemed once.
- ``RATE_LIMIT_EXCEEDED`` — operator-defined throttle (per-day,
per-hour, etc.) prevents this redemption.
"""
from __future__ import annotations
import hashlib
import hmac
import secrets
from dataclasses import dataclass, field
from datetime import datetime, timezone
from enum import Enum
class DenialCode(str, Enum):
"""Enumerated rejection reasons. Adding a new code is a hard fork."""
INVALID_SIGNATURE = "invalid_signature"
NULLIFIER_ALREADY_SEEN = "nullifier_already_seen"
RATE_LIMIT_EXCEEDED = "rate_limit_exceeded"
def _day_bucket(timestamp: float) -> str:
"""Return the UTC day in ``YYYY-MM-DD`` form for ``timestamp``.
Day-level granularity prevents fine-grained timestamp metadata
from becoming a de-anonymization vector. An operator that issued
100 receipts on the same day cannot link them by timestamp —
they all carry the same day_bucket.
"""
return datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime("%Y-%m-%d")
@dataclass(frozen=True)
class Receipt:
"""One side of the two-phase commit.
``role`` is either ``"verification"`` (Phase 1, operator-signed)
or ``"fulfillment"`` (Phase 2, citizen counter-signed).
"""
role: str
receipt_id: str
operator_id: str
day_bucket: str
nullifier_prefix: str
signature: bytes
@dataclass(frozen=True)
class ReceiptPair:
"""Both phases of a successful redemption.
Held by both citizen and operator. Surfaces only on dispute —
the chain never sees these.
"""
verification: Receipt
fulfillment: Receipt
def _sign(secret: bytes, body: bytes) -> bytes:
return hmac.new(secret, body, hashlib.sha256).digest()
def _verify(secret: bytes, body: bytes, signature: bytes) -> bool:
expected = _sign(secret, body)
return hmac.compare_digest(expected, signature)
def _receipt_body(*, role: str, receipt_id: str, operator_id: str,
day_bucket: str, nullifier_prefix: str) -> bytes:
return b"|".join([
b"function_key_receipt",
role.encode("utf-8"),
receipt_id.encode("utf-8"),
operator_id.encode("utf-8"),
day_bucket.encode("utf-8"),
nullifier_prefix.encode("utf-8"),
])
def issue_verification_receipt(
*,
operator_id: str,
operator_secret: bytes,
nullifier: str,
timestamp: float,
receipt_id: str | None = None,
nullifier_prefix_len: int = 8,
) -> Receipt:
"""Operator-side: issue a Phase-1 verification receipt.
``nullifier_prefix`` is the first ``nullifier_prefix_len`` hex
chars of the full nullifier — enough for the operator to dispute
later (fraud auditing) but NOT enough to identify the citizen
cross-operator. 8 hex chars = 32 bits = ~4 billion possible
prefixes, statistically unlinkable across operators.
"""
if not isinstance(nullifier, str) or len(nullifier) < nullifier_prefix_len:
raise ValueError("nullifier must be a hex string of sufficient length")
rid = receipt_id or secrets.token_hex(16)
prefix = nullifier[:nullifier_prefix_len]
day = _day_bucket(timestamp)
body = _receipt_body(
role="verification", receipt_id=rid, operator_id=operator_id,
day_bucket=day, nullifier_prefix=prefix,
)
sig = _sign(operator_secret, body)
return Receipt(
role="verification",
receipt_id=rid,
operator_id=operator_id,
day_bucket=day,
nullifier_prefix=prefix,
signature=sig,
)
def counter_sign_fulfillment(
*,
verification: Receipt,
citizen_secret: bytes,
) -> Receipt:
"""Citizen-side: counter-sign a verification receipt to acknowledge
service rendered.
The fulfillment receipt has the same field values as the
verification receipt (linking them to the same redemption) but
is signed with the CITIZEN's secret instead of the operator's.
Together they form a ``ReceiptPair``.
"""
if verification.role != "verification":
raise ValueError("input must be a Phase-1 verification receipt")
body = _receipt_body(
role="fulfillment", receipt_id=verification.receipt_id,
operator_id=verification.operator_id, day_bucket=verification.day_bucket,
nullifier_prefix=verification.nullifier_prefix,
)
sig = _sign(citizen_secret, body)
return Receipt(
role="fulfillment",
receipt_id=verification.receipt_id,
operator_id=verification.operator_id,
day_bucket=verification.day_bucket,
nullifier_prefix=verification.nullifier_prefix,
signature=sig,
)
def verify_receipt_pair(
*,
pair: ReceiptPair,
operator_secret: bytes,
citizen_secret: bytes,
) -> bool:
"""Verify both signatures on a ``ReceiptPair``.
Useful in dispute resolution — both parties can independently
confirm the pair is genuine.
"""
if pair.verification.role != "verification":
return False
if pair.fulfillment.role != "fulfillment":
return False
if pair.verification.receipt_id != pair.fulfillment.receipt_id:
return False
if pair.verification.operator_id != pair.fulfillment.operator_id:
return False
v_body = _receipt_body(
role="verification", receipt_id=pair.verification.receipt_id,
operator_id=pair.verification.operator_id,
day_bucket=pair.verification.day_bucket,
nullifier_prefix=pair.verification.nullifier_prefix,
)
if not _verify(operator_secret, v_body, pair.verification.signature):
return False
f_body = _receipt_body(
role="fulfillment", receipt_id=pair.fulfillment.receipt_id,
operator_id=pair.fulfillment.operator_id,
day_bucket=pair.fulfillment.day_bucket,
nullifier_prefix=pair.fulfillment.nullifier_prefix,
)
if not _verify(citizen_secret, f_body, pair.fulfillment.signature):
return False
return True
__all__ = [
"DenialCode",
"Receipt",
"ReceiptPair",
"counter_sign_fulfillment",
"issue_verification_receipt",
"verify_receipt_pair",
]
@@ -0,0 +1,83 @@
"""Ring Confidential Transactions — Sprint 11+ scaffolding.
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.3,
``infonet-economy/BRAINDUMP.md`` §11 item 9.
RingCT combines:
1. **Ring signatures** — hide *who* signed within an anonymity set.
2. **Confidential transactions** — Pedersen commitments hide
*amounts*. A range proof confirms the amount is non-negative.
3. **Key images** — link two outputs spent by the same key (without
revealing which key). Prevents double-spend without breaking
anonymity.
Implementation scheme is **undecided** — IMPLEMENTATION_PLAN.md §6.4
calls out RingCT vs CONFIDENTIAL_TX vs MimbleWimble vs ZK-SNARK as
options. The scaffolding here is scheme-agnostic; production wires
in whichever scheme the architect chooses through the
``RingSignatureScheme`` and ``BalanceCommitment`` Protocols.
Sprint 11+ runway:
- The interface contract is locked (see ``contracts.py``).
- A ``RingCTScaffolding`` placeholder reports
``status=NOT_IMPLEMENTED`` so callers can introspect honestly.
- When the Rust binding lands, instantiate it via the same Protocol
shape and swap the scaffolding for the production class — no
caller changes needed.
Cross-cutting design rule: privacy primitives MUST report their
status truthfully (cross-cutting design rule #1 — non-hostile UX).
A primitive that's not implemented surfaces clearly via the status
endpoint; calling its operations raises ``NotImplementedError`` with
a pointer back to the open issue.
"""
from __future__ import annotations
from typing import Any
from services.infonet.privacy.contracts import PrivacyPrimitiveStatus
class RingCTScaffolding:
"""Placeholder until the Rust ring-signature binding lands.
Calling ``sign`` / ``verify`` raises with a diagnostic that
points the caller back to the design doc. The status method
truthfully reports ``NOT_IMPLEMENTED`` so health endpoints can
surface this state.
"""
_DIAGNOSTIC = (
"RingCT primitive is scaffolding only — see "
"infonet-economy/IMPLEMENTATION_PLAN.md §6.4 for the open "
"scheme decision (RingCT vs CONFIDENTIAL_TX vs MimbleWimble "
"vs ZK-SNARK). Production implementation lands via "
"privacy-core Rust crate when ready."
)
def sign(
self,
*,
message: bytes,
signer_private_key: bytes,
ring_public_keys: list[bytes],
) -> dict[str, Any]:
raise NotImplementedError(self._DIAGNOSTIC)
def verify(
self,
*,
message: bytes,
signature: dict[str, Any],
ring_public_keys: list[bytes],
) -> bool:
raise NotImplementedError(self._DIAGNOSTIC)
def status(self) -> PrivacyPrimitiveStatus:
return PrivacyPrimitiveStatus.NOT_IMPLEMENTED
__all__ = ["RingCTScaffolding"]
@@ -0,0 +1,66 @@
"""Shielded balance commitments — Sprint 11+ scaffolding.
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.3.
Pedersen commitments hide balance amounts while preserving
homomorphic add/subtract. Range proofs (Bulletproofs or similar)
prove each output is non-negative without revealing it.
A balance is committed as ``C = amount * G + blinding * H`` where
``G, H`` are independent generators. ``sum(inputs) - sum(outputs)
== 0`` proves "no value created or destroyed" without revealing any
of the values.
Production implementation lands through the ``BalanceCommitment``
Protocol when the Rust binding is ready.
"""
from __future__ import annotations
from services.infonet.privacy.contracts import PrivacyPrimitiveStatus
class ShieldedBalanceScaffolding:
"""Placeholder until the Rust balance-commitment binding lands."""
_DIAGNOSTIC = (
"Shielded balance primitive is scaffolding only — production "
"implementation requires a Pedersen commitment + range-proof "
"library (e.g. bulletproofs). See "
"infonet-economy/IMPLEMENTATION_PLAN.md §4.3."
)
def commit(self, *, amount: int, blinding: bytes) -> bytes:
raise NotImplementedError(self._DIAGNOSTIC)
def verify_balance(
self,
*,
input_commitments: list[bytes],
output_commitments: list[bytes],
) -> bool:
raise NotImplementedError(self._DIAGNOSTIC)
def range_proof(
self,
*,
amount: int,
blinding: bytes,
max_bits: int = 64,
) -> bytes:
raise NotImplementedError(self._DIAGNOSTIC)
def verify_range_proof(
self,
*,
commitment: bytes,
proof: bytes,
max_bits: int = 64,
) -> bool:
raise NotImplementedError(self._DIAGNOSTIC)
def status(self) -> PrivacyPrimitiveStatus:
return PrivacyPrimitiveStatus.NOT_IMPLEMENTED
__all__ = ["ShieldedBalanceScaffolding"]
@@ -0,0 +1,62 @@
"""Stealth addresses — Sprint 11+ scaffolding.
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.3.
Each transaction generates a fresh one-time recipient address
unlinkable from the recipient's published key. The recipient uses a
private *view key* to scan the chain and identify outputs intended
for them.
Standard scheme (Monero-style, dual-key):
- Recipient publishes ``(view_pub, spend_pub)``.
- Sender generates random ``r``, computes
``one_time_address = H(r * view_pub) * G + spend_pub``.
- Recipient scans chain by checking if
``H(view_priv * R) * G + spend_pub == one_time_address`` for each
output's ``R = r * G``.
Production implementation lands through the ``StealthAddressScheme``
Protocol when the Rust binding is ready. Today, this module ships a
``StealthAddressScaffolding`` placeholder that reports
``NOT_IMPLEMENTED``.
"""
from __future__ import annotations
from services.infonet.privacy.contracts import PrivacyPrimitiveStatus
class StealthAddressScaffolding:
"""Placeholder until the Rust stealth-address binding lands."""
_DIAGNOSTIC = (
"Stealth address primitive is scaffolding only — see "
"infonet-economy/IMPLEMENTATION_PLAN.md §4.3 for the design. "
"Production implementation lands via privacy-core Rust crate."
)
def derive_one_time_address(
self,
*,
recipient_view_key: bytes,
recipient_spend_key: bytes,
sender_random: bytes,
) -> bytes:
raise NotImplementedError(self._DIAGNOSTIC)
def is_for_recipient(
self,
*,
one_time_address: bytes,
recipient_view_key: bytes,
recipient_spend_key: bytes,
sender_random: bytes,
) -> bool:
raise NotImplementedError(self._DIAGNOSTIC)
def status(self) -> PrivacyPrimitiveStatus:
return PrivacyPrimitiveStatus.NOT_IMPLEMENTED
__all__ = ["StealthAddressScaffolding"]
@@ -0,0 +1,59 @@
"""Reputation views — oracle_rep, oracle_rep_active, oracle_rep_lifetime, common_rep.
These are **pure functions** over the chain. No stored state. See
``infonet-economy/IMPLEMENTATION_PLAN.md`` §3.2 for the rationale.
Sprint 2 ships the base formulas (RULES §3.1, §3.2, §3.3, §3.11) without
the anti-gaming penalties. Sprint 3 layers VCS / clustering / temporal /
progressive penalties on top.
"""
from services.infonet.reputation.anti_gaming import (
apply_progressive_penalty,
clustering_penalty,
compute_clustering_coefficient,
compute_farming_pct,
compute_rep_multiplier,
compute_vcs,
farming_multiplier,
is_in_burst,
temporal_multiplier,
)
from services.infonet.reputation.common_rep import compute_common_rep
from services.infonet.reputation.governance_decay import (
compute_oracle_rep_active,
decay_factor_for_age,
)
from services.infonet.reputation.oracle_rep import (
OracleRepBreakdown,
compute_oracle_rep,
compute_oracle_rep_lifetime,
last_successful_prediction_ts,
)
from services.infonet.reputation.weekly_vote_budget import (
compute_weekly_vote_budget,
count_upreps_in_last_week,
is_budget_exceeded,
)
__all__ = [
"OracleRepBreakdown",
"apply_progressive_penalty",
"clustering_penalty",
"compute_clustering_coefficient",
"compute_common_rep",
"compute_farming_pct",
"compute_oracle_rep",
"compute_oracle_rep_active",
"compute_oracle_rep_lifetime",
"compute_rep_multiplier",
"compute_vcs",
"compute_weekly_vote_budget",
"count_upreps_in_last_week",
"decay_factor_for_age",
"farming_multiplier",
"is_budget_exceeded",
"is_in_burst",
"last_successful_prediction_ts",
"temporal_multiplier",
]
@@ -0,0 +1,52 @@
"""Anti-gaming penalties — Sprint 3.
Five layers:
- ``vcs.py``: Vote Correlation Score — detects coordinated upreping rings.
- ``clustering.py``: clustering coefficient — detects sophisticated farming
where voters also uprep each other.
- ``temporal.py``: burst detection — flags suspicious uprep storms.
- ``farming.py``: easy-bet detection — penalizes "predictors" who only
bet on near-certain outcomes.
- ``progressive_penalty.py``: whale deterrence — gaming penalties scale
with the violator's oracle rep so high-rep nodes can't shrug them off.
All five are pure functions over the chain. They run as deterministic
chain analysis (every node computes the same scores from the same chain
history), matching IMPLEMENTATION_PLAN.md §3.3.
Cross-cutting design rule: anti-gaming reads happen in the background.
A user who is being legitimately upreped does not block the UI on
penalty recomputation; the computed common-rep view simply uses the
last cached value and refreshes asynchronously.
"""
from services.infonet.reputation.anti_gaming.clustering import (
clustering_penalty,
compute_clustering_coefficient,
)
from services.infonet.reputation.anti_gaming.farming import (
compute_farming_pct,
farming_multiplier,
)
from services.infonet.reputation.anti_gaming.progressive_penalty import (
apply_progressive_penalty,
compute_rep_multiplier,
)
from services.infonet.reputation.anti_gaming.temporal import (
is_in_burst,
temporal_multiplier,
)
from services.infonet.reputation.anti_gaming.vcs import compute_vcs
__all__ = [
"apply_progressive_penalty",
"clustering_penalty",
"compute_clustering_coefficient",
"compute_farming_pct",
"compute_rep_multiplier",
"compute_vcs",
"farming_multiplier",
"is_in_burst",
"temporal_multiplier",
]
@@ -0,0 +1,119 @@
"""Clustering coefficient — detects sophisticated farming where the
voters who uprep a target also uprep each other.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.5.
For a target B:
voters = {all nodes that uprepped B in decay window}
n = len(voters)
if n < 2:
return 0.0
possible_edges = n * (n - 1) / 2
actual_edges = count of pairs (V1, V2) where V1 has uprepped V2
OR V2 has uprepped V1
clustering = actual_edges / possible_edges
The penalty per RULES §3.3:
target_penalty = max(clustering_min_weight, 1.0 - clustering)
Why this catches what VCS misses: VCS measures *one* upreper's
similarity to the target's fan set. Clustering measures whether the
*entire* fan set is socially networked — a 10-node cabal that
upreps each other is a cluster coefficient near 1.0 even if no
individual upreper has unusual VCS.
"""
from __future__ import annotations
from typing import Any, Iterable
from services.infonet.config import CONFIG
from services.infonet.reputation.anti_gaming.vcs import _upreps_within_window # noqa: I201
_SECONDS_PER_DAY = 86400.0
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
def _decay_window_seconds(decay_window_days: float | None) -> float:
if decay_window_days is not None:
return float(decay_window_days) * _SECONDS_PER_DAY
return float(CONFIG["vote_decay_days"]) * _SECONDS_PER_DAY
def compute_clustering_coefficient(
target_id: str,
chain: Iterable[dict[str, Any]],
*,
now: float | None = None,
decay_window_days: float | None = None,
) -> float:
"""Coefficient in ``[0.0, 1.0]`` for ``target_id``'s voter graph.
0.0 means voters are strangers; 1.0 means every voter has uprepped
every other voter.
"""
if not isinstance(target_id, str) or not target_id:
return 0.0
events = [e for e in chain if isinstance(e, dict)]
if not events:
return 0.0
if now is None:
now = max(float(ev.get("timestamp") or 0.0) for ev in events)
window_s = _decay_window_seconds(decay_window_days)
window_upreps = _upreps_within_window(events, now=now, window_s=window_s)
voters: set[str] = set()
edges: set[tuple[str, str]] = set()
# Build an adjacency map from author -> {targets}.
by_author: dict[str, set[str]] = {}
for ev in window_upreps:
author = ev.get("node_id")
p = _payload(ev)
tgt = p.get("target_node_id")
if not isinstance(author, str) or not isinstance(tgt, str):
continue
if author == tgt:
continue
by_author.setdefault(author, set()).add(tgt)
if tgt == target_id:
voters.add(author)
n = len(voters)
if n < 2:
return 0.0
voter_list = sorted(voters)
for i, v1 in enumerate(voter_list):
for v2 in voter_list[i + 1:]:
v1_upreps_v2 = v2 in by_author.get(v1, ())
v2_upreps_v1 = v1 in by_author.get(v2, ())
if v1_upreps_v2 or v2_upreps_v1:
edges.add((v1, v2))
possible = n * (n - 1) / 2
return len(edges) / possible
def clustering_penalty(coefficient: float) -> float:
"""Per-uprep multiplier from a clustering coefficient.
Spec formula: ``max(clustering_min_weight, 1.0 - coefficient)``.
"""
floor = float(CONFIG["clustering_min_weight"])
return max(floor, 1.0 - float(coefficient))
__all__ = [
"clustering_penalty",
"compute_clustering_coefficient",
]
@@ -0,0 +1,113 @@
"""Aggregate per-node correlation score — feeds progressive penalty.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.6.
Sprint 3 shipped the progressive-penalty math (whale deterrence
multiplier ``1 + log2(rep)``) but did not wire it into a running
aggregate. This module fills that gap.
Per-node aggregate score:
score(node) = mean(1 - vcs(upreper, node)
for every uprep targeting node in the decay window)
Range: ``[0.0, 1.0]``. ``0.0`` means every uprep was orthogonal —
no correlation evidence. ``1.0`` means every uprep was from a fully
overlapping target set — saturated cabal.
When ``score(node) > CONFIG['progressive_penalty_threshold']`` (default
``0.0`` — disabled), the progressive penalty multiplier is applied to
the node's effective common-rep payouts. The threshold default is
``0.0`` so Sprint 3 behavior is preserved for any chain that doesn't
explicitly opt in via governance.
"""
from __future__ import annotations
from typing import Any, Iterable
from services.infonet.config import CONFIG
from services.infonet.reputation.anti_gaming.progressive_penalty import (
apply_progressive_penalty,
)
from services.infonet.reputation.anti_gaming.vcs import compute_vcs
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
def compute_node_correlation_score(
node_id: str,
chain: Iterable[dict[str, Any]],
*,
now: float | None = None,
) -> float:
"""Average correlation evidence (``1 - VCS``) across upreps
targeting ``node_id``.
Returns ``0.0`` when no upreps target the node — no evidence to
support a penalty.
"""
chain_list = [e for e in chain if isinstance(e, dict)]
if not chain_list:
return 0.0
if now is None:
now = max(float(ev.get("timestamp") or 0.0) for ev in chain_list)
correlations: list[float] = []
for ev in chain_list:
if ev.get("event_type") != "uprep":
continue
if _payload(ev).get("target_node_id") != node_id:
continue
upreper = ev.get("node_id")
if not isinstance(upreper, str) or not upreper or upreper == node_id:
continue
try:
ts = float(ev.get("timestamp") or 0.0)
except (TypeError, ValueError):
ts = float(now)
vcs = compute_vcs(upreper, node_id, chain_list, now=ts)
correlations.append(max(0.0, min(1.0, 1.0 - vcs)))
if not correlations:
return 0.0
return sum(correlations) / len(correlations)
def progressive_penalty_multiplier_for(
node_id: str,
chain: Iterable[dict[str, Any]],
*,
oracle_rep: float,
now: float | None = None,
) -> float:
"""Return the multiplier to apply to a node's common-rep payouts.
Returns ``1.0`` when the aggregate correlation score is at or
below ``CONFIG['progressive_penalty_threshold']`` (no penalty).
Above the threshold, the penalty is computed via
``apply_progressive_penalty(score - threshold, oracle_rep)`` and
*subtracted* from 1.0 (clamped to ``[0.0, 1.0]``).
The threshold defaults to ``0.0`` (disabled). Governance can
raise it via petition once aggregate-correlation history is
well-calibrated against real chain data.
"""
threshold = float(CONFIG["progressive_penalty_threshold"])
if threshold <= 0.0:
# Disabled — preserve Sprint 3 behavior.
return 1.0
score = compute_node_correlation_score(node_id, chain, now=now)
if score <= threshold:
return 1.0
over = score - threshold
docked = apply_progressive_penalty(over, oracle_rep)
return max(0.0, 1.0 - docked)
__all__ = [
"compute_node_correlation_score",
"progressive_penalty_multiplier_for",
]
@@ -0,0 +1,118 @@
"""Easy-bet farming detection and enforcement.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.1.
A node's ``farming_pct`` is:
farming_pct = count_easy_bets / total_predictions
where an "easy bet" is a prediction whose ``probability_at_bet``
exceeds ``farming_easy_bet_cutoff`` (default 0.80, expressed as 80.0
on the 0-100 scale used in payloads).
Penalty multiplier:
farming_pct > farming_hard_threshold → oracle_rep_earned *= 0.10
farming_pct > farming_soft_threshold → oracle_rep_earned *= 0.50
otherwise → oracle_rep_earned *= 1.00
The plan §1.2 calls out that the existing ``mesh_oracle.py`` *tracks*
``farming_pct`` but does NOT enforce the multiplier. Sprint 3 adds the
enforcement here, and Sprint 4's `oracle_rep` integration applies it
to mints. Until that wiring lands, this module is exposed as the
authoritative source of the math.
Note on sides: the spec's "easy bet" is a probability-of-the-PICKED-side
test. A free pick at 90% on yes is easy; a contrarian free pick at 10%
on yes (where the chain says yes is 10% likely) is hard. The
``probability_at_bet`` field stores the probability of the YES side at
the time the prediction is placed; we compute the predicted-side
probability accordingly.
"""
from __future__ import annotations
from typing import Any, Iterable
from services.infonet.config import CONFIG
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
def _picked_side_probability(payload: dict[str, Any]) -> float | None:
"""Translate ``probability_at_bet`` (always P(yes) on 0-100) into the
probability of the side actually picked. Returns ``None`` if the
payload is malformed.
"""
side = payload.get("side")
prob = payload.get("probability_at_bet")
if side not in ("yes", "no"):
return None
if prob is None:
return None
try:
p_yes = float(prob)
except (TypeError, ValueError):
return None
if not (0.0 <= p_yes <= 100.0):
return None
return p_yes if side == "yes" else 100.0 - p_yes
def compute_farming_pct(
node_id: str,
chain: Iterable[dict[str, Any]],
) -> float:
"""Fraction of ``node_id``'s predictions whose picked-side probability
exceeded ``farming_easy_bet_cutoff``.
Returns ``0.0`` if the node has no predictions on chain. The cutoff
is on the 0-1 scale in ``CONFIG``; predictions store probability on
the 0-100 scale, so we scale the cutoff for comparison.
"""
if not isinstance(node_id, str) or not node_id:
return 0.0
cutoff_pct = float(CONFIG["farming_easy_bet_cutoff"]) * 100.0
total = 0
easy = 0
for ev in chain:
if not isinstance(ev, dict):
continue
if ev.get("event_type") != "prediction_place":
continue
if ev.get("node_id") != node_id:
continue
picked_p = _picked_side_probability(_payload(ev))
if picked_p is None:
continue
total += 1
if picked_p > cutoff_pct:
easy += 1
if total == 0:
return 0.0
return easy / total
def farming_multiplier(farming_pct: float) -> float:
"""Spec multiplier for a node's mint earnings.
- ``> farming_hard_threshold`` → 0.10
- ``> farming_soft_threshold`` → 0.50
- otherwise → 1.00
"""
pct = float(farming_pct)
if pct > float(CONFIG["farming_hard_threshold"]):
return 0.10
if pct > float(CONFIG["farming_soft_threshold"]):
return 0.50
return 1.00
__all__ = [
"compute_farming_pct",
"farming_multiplier",
]
@@ -0,0 +1,49 @@
"""Progressive penalty — whale deterrence.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.6.
base_penalty = correlation_score # 0.0 to 1.0
rep_multiplier = 1 + log2(max(oracle_rep, 1))
rep_docked = base_penalty * rep_multiplier
The point: a whale with 1024 oracle rep faces a multiplier of 11×, so
the same correlation score that would dock a small node 0.5 rep docks
a whale 5.5 rep. Coordination becomes more expensive as you accumulate
more rep — the protocol's "you can't simply outscale anti-gaming"
defense.
This module exposes the math. Sprint 3 does NOT yet wire it into a
running aggregate-correlation tracker — that requires per-node
correlation history which is a Sprint 4+ concern. The helpers here are
ready for that integration.
"""
from __future__ import annotations
import math
def compute_rep_multiplier(oracle_rep: float) -> float:
"""``1 + log2(max(oracle_rep, 1))``.
- ``oracle_rep <= 1`` → multiplier 1.0
- ``oracle_rep == 2`` → 2.0
- ``oracle_rep == 1024`` → 11.0
"""
rep = max(1.0, float(oracle_rep))
return 1.0 + math.log2(rep)
def apply_progressive_penalty(base_penalty: float, oracle_rep: float) -> float:
"""``base_penalty * compute_rep_multiplier(oracle_rep)``.
``base_penalty`` is intended to be a non-negative correlation score
in ``[0.0, 1.0]``; the function does not clamp.
"""
return float(base_penalty) * compute_rep_multiplier(oracle_rep)
__all__ = [
"apply_progressive_penalty",
"compute_rep_multiplier",
]
@@ -0,0 +1,80 @@
"""Temporal burst detection — flags suspicious uprep storms.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.3 (the
``rep_after_burst`` step).
Definition: a target B is "in a burst" relative to an uprep at time
``t`` if there are at least ``temporal_burst_min_upreps`` upreps to B
within a ``temporal_burst_window_sec`` window centered on ``t`` (the
window includes the uprep being evaluated).
When in burst: per-uprep weight is multiplied by 0.2 (80% reduction).
Otherwise 1.0.
Why a centered window: bursts can be detected on either side of the
suspect uprep. Sliding-forward-only would let an attacker pre-warm the
counter.
"""
from __future__ import annotations
from typing import Any, Iterable
from services.infonet.config import CONFIG
_BURST_REDUCTION_FACTOR = 0.2
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
def is_in_burst(
target_id: str,
uprep_timestamp: float,
chain: Iterable[dict[str, Any]],
) -> bool:
"""Are there ``temporal_burst_min_upreps`` upreps to ``target_id``
within ``temporal_burst_window_sec`` of ``uprep_timestamp``?
"""
if not isinstance(target_id, str) or not target_id:
return False
try:
ts = float(uprep_timestamp)
except (TypeError, ValueError):
return False
window_s = float(CONFIG["temporal_burst_window_sec"])
half = window_s / 2.0
threshold = int(CONFIG["temporal_burst_min_upreps"])
count = 0
for ev in chain:
if not isinstance(ev, dict):
continue
if ev.get("event_type") != "uprep":
continue
p = _payload(ev)
if p.get("target_node_id") != target_id:
continue
try:
ets = float(ev.get("timestamp") or 0.0)
except (TypeError, ValueError):
continue
if ts - half <= ets <= ts + half:
count += 1
if count >= threshold:
return True
return False
def temporal_multiplier(in_burst: bool) -> float:
"""1.0 if not in burst; ``_BURST_REDUCTION_FACTOR`` (0.2) if in burst."""
return _BURST_REDUCTION_FACTOR if in_burst else 1.0
__all__ = [
"is_in_burst",
"temporal_multiplier",
]
@@ -0,0 +1,127 @@
"""Vote Correlation Score — detects coordinated upreping rings.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.4.
For an uprep from A → B:
A_targets = {all nodes A has uprepped in decay window}
B_fans = {all nodes that uprepped B in decay window, excluding A}
if len(B_fans) == 0:
overlap = 0.0
else:
overlap = |A_targets ∩ B_fans| / |B_fans|
correlation_penalty = max(vcs_min_weight, 1.0 - overlap)
The intent: if A always upreps the same group of nodes that always uprep
B (a circle-jerk), the overlap approaches 1 and the penalty floors at
``vcs_min_weight`` (default 0.10 — 10% effective weight, regardless of
how many nodes participate).
Pure function over the chain. Does NOT depend on order beyond the decay
window cutoff.
"""
from __future__ import annotations
from typing import Any, Iterable
from services.infonet.config import CONFIG
_SECONDS_PER_DAY = 86400.0
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
def _decay_window_seconds(decay_window_days: float | None) -> float:
if decay_window_days is not None:
return float(decay_window_days) * _SECONDS_PER_DAY
return float(CONFIG["vote_decay_days"]) * _SECONDS_PER_DAY
def _upreps_within_window(
chain: Iterable[dict[str, Any]],
*,
now: float,
window_s: float,
) -> list[dict[str, Any]]:
"""All ``uprep`` events whose timestamp is in [now - window, now]."""
cutoff = now - window_s
out: list[dict[str, Any]] = []
for ev in chain:
if not isinstance(ev, dict):
continue
if ev.get("event_type") != "uprep":
continue
try:
ts = float(ev.get("timestamp") or 0.0)
except (TypeError, ValueError):
continue
if cutoff <= ts <= now:
out.append(ev)
return out
def compute_vcs(
upreper_id: str,
target_id: str,
chain: Iterable[dict[str, Any]],
*,
now: float | None = None,
decay_window_days: float | None = None,
) -> float:
"""Return the VCS multiplier for an uprep from ``upreper_id`` to ``target_id``.
Range: ``[vcs_min_weight, 1.0]``. A return of 1.0 means no
correlation detected (full weight). A return of ``vcs_min_weight``
(default 0.10) means maximum correlation — the upreper's targets
completely overlap with the target's fan set.
``now`` defaults to the latest timestamp on the chain. Pass an
explicit value when the caller wants a fixed evaluation point (e.g.
Sprint 4 will pass the market snapshot's ``frozen_at``).
"""
if not isinstance(upreper_id, str) or not upreper_id:
return float(CONFIG["vcs_min_weight"])
if not isinstance(target_id, str) or not target_id:
return float(CONFIG["vcs_min_weight"])
if upreper_id == target_id:
return 1.0 # self-uprep is filtered by common_rep; VCS is a no-op here
events = [e for e in chain if isinstance(e, dict)]
if not events:
return 1.0
if now is None:
now = max(float(ev.get("timestamp") or 0.0) for ev in events)
window_s = _decay_window_seconds(decay_window_days)
window_upreps = _upreps_within_window(events, now=now, window_s=window_s)
a_targets: set[str] = set()
b_fans: set[str] = set()
for ev in window_upreps:
author = ev.get("node_id")
p = _payload(ev)
target = p.get("target_node_id")
if not isinstance(author, str) or not isinstance(target, str):
continue
if author == target:
continue
if author == upreper_id:
a_targets.add(target)
if target == target_id and author != upreper_id:
b_fans.add(author)
floor = float(CONFIG["vcs_min_weight"])
if not b_fans:
return 1.0
overlap = len(a_targets & b_fans) / len(b_fans)
return max(floor, 1.0 - overlap)
__all__ = ["compute_vcs"]
@@ -0,0 +1,128 @@
"""Common rep computation with anti-gaming multipliers (Sprint 3).
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.3.
Per-uprep formula:
base_rep = upreper.oracle_rep * weight_factor
rep_after_vcs = base_rep * compute_vcs(upreper, target)
rep_after_clustering = rep_after_vcs * clustering_penalty(coefficient(target))
rep_after_burst = rep_after_clustering * temporal_multiplier(in_burst)
common_rep_earned = rep_after_burst (per uprep; sum across all upreps)
VCS / clustering use the upreps-within-decay-window helper. Temporal
burst uses a centered window (see ``anti_gaming/temporal.py``).
Sprint 3 caches per-uprep evaluations in-process: a single call to
``compute_common_rep`` walks the chain at most three times (once per
multiplier family). Caching across calls is a Sprint 3+ adapter
concern.
Cross-cutting design rule: this is background work. The UI should call
through ``InfonetReputationAdapter.common_rep`` and treat the result
as eventually-consistent — never block a user-visible action waiting
for it.
"""
from __future__ import annotations
from typing import Any, Iterable
from services.infonet.config import CONFIG
from services.infonet.reputation.anti_gaming.clustering import (
clustering_penalty,
compute_clustering_coefficient,
)
from services.infonet.reputation.anti_gaming.correlation_score import (
progressive_penalty_multiplier_for,
)
from services.infonet.reputation.anti_gaming.temporal import (
is_in_burst,
temporal_multiplier,
)
from services.infonet.reputation.anti_gaming.vcs import compute_vcs
from services.infonet.reputation.oracle_rep import compute_oracle_rep
def _default_weight_factor() -> float:
"""RULES §3.3 weight factor — promoted from Sprint 2 module
constant to ``CONFIG['common_rep_weight_factor']`` 2026-04-28 so
governance can tune it via petition.
Tests pass an explicit value to ``compute_common_rep`` to override.
"""
return float(CONFIG["common_rep_weight_factor"])
def compute_common_rep(
node_id: str,
chain: Iterable[dict[str, Any]],
*,
weight_factor: float | None = None,
apply_anti_gaming: bool = True,
) -> float:
"""Common rep balance for ``node_id``.
``apply_anti_gaming=False`` returns the Sprint 2 base formula —
useful for tests that want to isolate the multiplier layer. Default
in production is ``True``.
"""
factor = float(_default_weight_factor() if weight_factor is None else weight_factor)
events = [e for e in chain if isinstance(e, dict)]
rep = 0.0
# Oracle-rep cache keyed by upreper only — oracle_rep is computed
# over the full chain (no time bound) and doesn't change per-uprep.
upreper_cache: dict[str, float] = {}
# NB: do NOT cache the clustering coefficient by node_id alone — it
# is a function of (target, evaluation timestamp). Caching by target
# only would freeze the first uprep's view (often coefficient 0
# before other voters arrive) and skip the penalty for subsequent
# upreps.
for ev in events:
if ev.get("event_type") != "uprep":
continue
payload = ev.get("payload") or {}
if payload.get("target_node_id") != node_id:
continue
upreper = ev.get("node_id")
if not isinstance(upreper, str) or not upreper:
continue
if upreper == node_id:
continue
if upreper not in upreper_cache:
upreper_cache[upreper] = compute_oracle_rep(upreper, events)
base = upreper_cache[upreper] * factor
if apply_anti_gaming:
try:
ts = float(ev.get("timestamp") or 0.0)
except (TypeError, ValueError):
ts = 0.0
vcs = compute_vcs(upreper, node_id, events, now=ts)
coefficient = compute_clustering_coefficient(node_id, events, now=ts)
cluster_mult = clustering_penalty(coefficient)
burst_mult = temporal_multiplier(is_in_burst(node_id, ts, events))
rep += base * vcs * cluster_mult * burst_mult
else:
rep += base
if apply_anti_gaming and rep > 0:
# Progressive-penalty wiring (Sprint 3 polish 2026-04-28).
# Disabled when CONFIG['progressive_penalty_threshold'] == 0,
# so this preserves Sprint 3 behavior by default. Once
# governance raises the threshold via petition, the whale-
# deterrence multiplier kicks in for nodes whose aggregate
# correlation score crosses it. Oracle-rep input is the
# TARGET's rep (not the upreper's) — bigger oracles bear
# bigger penalties for cabal-shaped uprep patterns.
target_oracle_rep = compute_oracle_rep(node_id, events)
rep *= progressive_penalty_multiplier_for(
node_id, events, oracle_rep=target_oracle_rep,
)
return rep
__all__ = ["compute_common_rep"]
@@ -0,0 +1,82 @@
"""Governance weight decay — oracle_rep → oracle_rep_active.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.11.
``oracle_rep_active`` is **only** used for governance weight (petition
signatures, voting, quorum). Resolution staking, dispute staking, and
truth staking continue to use ``oracle_rep`` directly — dormant oracles
can still verify reality even if they aren't governing.
A successful prediction (``last_successful_prediction_ts`` is non-None)
within the decay window keeps a node at full governance weight. Beyond
the window, weight halves (default factor 0.5) per period.
Implemented as a pure function over the chain so every node computes
the same value from the same chain history.
"""
from __future__ import annotations
import math
from typing import Any, Iterable
from services.infonet.config import CONFIG
from services.infonet.reputation.oracle_rep import (
compute_oracle_rep,
last_successful_prediction_ts,
)
_SECONDS_PER_DAY = 86400.0
def decay_factor_for_age(days_since_success: float | None) -> float:
"""Return the multiplier for ``oracle_rep`` → ``oracle_rep_active``.
- ``None``: node has no qualifying success → factor of 0 (no
governance weight; new nodes earn it by predicting correctly in a
mintable market).
- within the decay window (``governance_decay_days``): 1.0.
- beyond: ``governance_decay_factor ** decay_periods``.
"""
if days_since_success is None:
return 0.0
decay_days = float(CONFIG["governance_decay_days"])
factor = float(CONFIG["governance_decay_factor"])
if not (0.0 < factor < 1.0):
# Guard: schema bounds should prevent this, but if a malformed
# config slips through, treat as no-decay.
return 1.0 if days_since_success <= decay_days else 0.0
if days_since_success <= decay_days:
return 1.0
decay_periods = math.floor(days_since_success / decay_days)
return factor ** decay_periods
def compute_oracle_rep_active(
node_id: str,
chain: Iterable[dict[str, Any]],
now: float,
) -> float:
"""Governance-weighted oracle rep at chain time ``now``.
``now`` is passed in (rather than read from ``time.time()``) so the
function stays pure and so tests / replay always produce
deterministic answers. Production callers pass
``time_validity.chain_majority_time(chain)``.
"""
events = list(chain)
balance = compute_oracle_rep(node_id, events)
if balance <= 0:
return 0.0
last_ts = last_successful_prediction_ts(node_id, events)
if last_ts is None:
return 0.0
days = max(0.0, (float(now) - last_ts) / _SECONDS_PER_DAY)
return balance * decay_factor_for_age(days)
__all__ = [
"compute_oracle_rep_active",
"decay_factor_for_age",
]
@@ -0,0 +1,361 @@
"""Oracle rep computation — pure functions over the chain.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.1, §3.2, §3.11.
Constitutional anchor (``IMMUTABLE_PRINCIPLES["oracle_rep_source"] ==
"predictions_only"``): oracle rep may ONLY be minted by verified
predictions against reality. Sprint 2 enforces this by structurally
constraining the mint formula — there is no other code path that
returns a positive contribution to ``compute_oracle_rep``.
A market's prediction mints oracle rep only when ALL of the following
hold:
- The market produced a ``resolution_finalize`` event with
``outcome != "invalid"`` and ``is_provisional == False``.
- The corresponding ``market_snapshot`` shows
``frozen_participant_count >= CONFIG["min_market_participants"]`` AND
``frozen_total_stake >= CONFIG["min_market_total_stake"]``.
- The market is NOT a bootstrap-mode market (Sprint 8 will add the
bootstrap path; until then bootstrap markets contribute zero).
- The market is objective. Subjective markets mint Common Rep only
(RULES §3.1).
- The prediction's ``side`` matches the FINAL outcome.
Lost stakes from incorrect *staked* predictions reduce the running
``oracle_rep`` balance (RULES §3.2 — the staked amount is forfeited to
the winner pool). ``oracle_rep_lifetime`` is monotonically increasing
and ignores losses.
Sprint 2 does NOT yet handle:
- Dispute reversal (Sprint 5 — `dispute_resolve` with `outcome="reversed"`).
- Resolution-stake redistribution (Sprint 4/5 — `resolution_stake`
events and the loser-pool burn).
- Anti-gaming farming multipliers (Sprint 3).
These layers will be added by their owning sprints; the function
signature and return shape are stable.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Iterable
from services.infonet.config import CONFIG
from services.infonet.markets.dispute import effective_outcome as _effective_outcome
from services.infonet.reputation.anti_gaming.farming import (
compute_farming_pct,
farming_multiplier,
)
def _as_event_list(chain: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
"""Accept any iterable, return a stable list ordered by (timestamp, sequence)."""
events = [e for e in chain if isinstance(e, dict)]
events.sort(key=lambda e: (float(e.get("timestamp") or 0.0), int(e.get("sequence") or 0)))
return events
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
@dataclass
class _MarketView:
"""Internal: chain-derived view of a single market.
Populated in one pass over the chain. Holds only what oracle_rep
needs to mint correctly per RULES §3.1/§3.2.
"""
market_id: str
market_type: str = "objective"
bootstrap_index: int | None = None
snapshot: dict[str, Any] | None = None
finalize: dict[str, Any] | None = None
finalize_ts: float = 0.0
predictions: list[dict[str, Any]] = field(default_factory=list)
farming_pct_lookup: dict[str, float] = field(default_factory=dict)
def _index_markets(events: list[dict[str, Any]]) -> dict[str, _MarketView]:
markets: dict[str, _MarketView] = {}
for ev in events:
et = ev.get("event_type")
p = _payload(ev)
mid = p.get("market_id")
if not isinstance(mid, str) or not mid:
continue
m = markets.setdefault(mid, _MarketView(market_id=mid))
if et == "prediction_create":
m.market_type = str(p.get("market_type") or "objective")
if "bootstrap_index" in p and p["bootstrap_index"] is not None:
try:
m.bootstrap_index = int(p["bootstrap_index"])
except (TypeError, ValueError):
m.bootstrap_index = None
elif et == "market_snapshot":
m.snapshot = p
elif et == "resolution_finalize":
m.finalize = p
m.finalize_ts = float(ev.get("timestamp") or 0.0)
elif et == "prediction_place":
m.predictions.append({
"node_id": ev.get("node_id"),
"side": p.get("side"),
"stake_amount": p.get("stake_amount"),
"probability_at_bet": p.get("probability_at_bet"),
"timestamp": ev.get("timestamp"),
})
return markets
def _market_passes_liquidity(market: _MarketView) -> bool:
snap = market.snapshot or {}
try:
participants = int(snap.get("frozen_participant_count") or 0)
total_stake = float(snap.get("frozen_total_stake") or 0.0)
except (TypeError, ValueError):
return False
return (
participants >= int(CONFIG["min_market_participants"])
and total_stake >= float(CONFIG["min_market_total_stake"])
)
def _market_is_mintable(market: _MarketView) -> bool:
"""Return True if the market is final, non-provisional, non-bootstrap,
objective, and passed liquidity. Mintable markets contribute oracle rep
to correct predictors.
"""
finalize = market.finalize
if not finalize:
return False
if finalize.get("is_provisional") is not False:
return False
outcome = finalize.get("outcome")
if outcome not in ("yes", "no"):
return False
if market.market_type != "objective":
return False
# Sprint 8: bootstrap markets that resolved via eligible-node-one-vote
# mint oracle rep from correct predictions, same as normal markets.
# The bootstrap mechanic only changes HOW resolution decides yes/no —
# not whether predictors get rep for being correct. RULES §3.10 step
# 0.5: "Oracle rep minted normally from correct predictions
# (constitutional)".
if not _market_passes_liquidity(market):
return False
return True
def _free_pred_mint(probability_at_bet: float) -> float:
"""RULES §3.1 — mint = max(oracle_min_earned, 1.0 - p/100)."""
if probability_at_bet is None:
return 0.0
try:
prob = float(probability_at_bet)
except (TypeError, ValueError):
return 0.0
if not (0.0 <= prob <= 100.0):
return 0.0
return max(float(CONFIG["oracle_min_earned"]), 1.0 - (prob / 100.0))
def _staked_pred_settlement(
stake_amount: float,
side: str,
outcome: str,
predictions: list[dict[str, Any]],
) -> float:
"""RULES §3.2 — pool settlement for staked predictions.
Returns the *net* change to oracle rep for a single staked
prediction. Positive = winnings (returned stake + share of loser
pool). Negative = forfeited stake.
"""
winning_side = outcome
losing_side = "no" if outcome == "yes" else "yes"
winner_pool = 0.0
loser_pool = 0.0
for pred in predictions:
amt = pred.get("stake_amount")
if amt is None:
continue
try:
a = float(amt)
except (TypeError, ValueError):
continue
if a <= 0:
continue
if pred.get("side") == winning_side:
winner_pool += a
elif pred.get("side") == losing_side:
loser_pool += a
if side == winning_side:
if winner_pool == 0.0:
return float(stake_amount) # degenerate — return stake
if loser_pool == 0.0:
return float(stake_amount) # everyone won — no profit
share = float(stake_amount) / winner_pool
winnings = share * loser_pool
return float(stake_amount) + winnings
elif side == losing_side:
return -float(stake_amount)
return 0.0
@dataclass(frozen=True)
class OracleRepBreakdown:
"""Auditable breakdown of how a node arrived at its oracle_rep balance.
Useful for the UI's reputation-history view and for invariant tests.
Sprint 4+ extensions will add resolution-stake redistribution and
dispute-reversal adjustments to this struct.
"""
free_prediction_mints: float
staked_prediction_returns: float
staked_prediction_losses: float
total: float
def compute_oracle_rep_breakdown(
node_id: str,
chain: Iterable[dict[str, Any]],
) -> OracleRepBreakdown:
"""Per-component breakdown — exposed for tests and audit trails.
Sprint 3 wiring: applies the farming multiplier (RULES §3.1) to
free-pick mints. Staked predictions are NOT farming-penalized — the
farmer is risking actual rep, which is the protocol's deterrent for
that case. Per-spec semantics.
"""
events = _as_event_list(chain)
markets = _index_markets(events)
farming_pct = compute_farming_pct(node_id, events)
farming_mult = farming_multiplier(farming_pct)
free_mint = 0.0
staked_return = 0.0
staked_loss = 0.0
for market in markets.values():
if not _market_is_mintable(market):
continue
original = market.finalize["outcome"] # type: ignore[index]
# Sprint 5 bounded reversal: a resolved dispute can flip the
# effective outcome of THIS market only — no cascade.
outcome = _effective_outcome(original, market.market_id, events)
for pred in market.predictions:
if pred.get("node_id") != node_id:
continue
stake = pred.get("stake_amount")
if stake is None:
if pred.get("side") == outcome:
free_mint += _free_pred_mint(pred.get("probability_at_bet")) * farming_mult
# Wrong free pick: oracle_rep_earned = 0 (RULES §3.1)
else:
delta = _staked_pred_settlement(
stake_amount=stake,
side=pred.get("side", ""),
outcome=outcome,
predictions=market.predictions,
)
if delta >= 0:
staked_return += delta
else:
staked_loss += -delta
total = free_mint + staked_return - staked_loss
if total < 0:
# Oracle rep is non-negative by spec (lost-stake forfeits transfer to
# winners; they never push a balance below zero in isolation, but a
# naive node-only view can underflow if the node never won
# anything). Clamp to zero — the chain analysis on the full network
# always sums to a non-negative total.
total = 0.0
return OracleRepBreakdown(
free_prediction_mints=free_mint,
staked_prediction_returns=staked_return,
staked_prediction_losses=staked_loss,
total=total,
)
def compute_oracle_rep(node_id: str, chain: Iterable[dict[str, Any]]) -> float:
"""Current oracle rep balance for ``node_id``.
Wins (free mint + staked winnings) minus losses (staked forfeits).
Clamped at zero. See ``compute_oracle_rep_breakdown`` for the full
component view.
"""
return compute_oracle_rep_breakdown(node_id, chain).total
def compute_oracle_rep_lifetime(node_id: str, chain: Iterable[dict[str, Any]]) -> float:
"""Cumulative oracle rep ever earned by ``node_id``.
Monotonically increasing (analytics / profiles only — never drives
protocol logic per RULES §2.1). Counts wins; ignores losses.
"""
bd = compute_oracle_rep_breakdown(node_id, chain)
return bd.free_prediction_mints + bd.staked_prediction_returns
def last_successful_prediction_ts(
node_id: str,
chain: Iterable[dict[str, Any]],
) -> float | None:
"""Timestamp of the node's most recent correct prediction in a
market that:
1. Reached FINAL (non-INVALID) status.
2. Was not provisional at finalize time.
3. Passed the frozen liquidity thresholds.
4. Was not later reversed by dispute (Sprint 5 — until then, no
reversal logic; this function only sees the raw outcome).
Returns ``None`` if the node has no qualifying prediction.
Used by ``governance_decay.compute_oracle_rep_active`` to determine
decay age. Per RULES §3.11 INVALID markets do NOT reset the clock —
enforced here by the ``_market_is_mintable`` filter.
"""
events = _as_event_list(chain)
markets = _index_markets(events)
best_ts: float | None = None
for market in markets.values():
if not _market_is_mintable(market):
continue
original = market.finalize["outcome"] # type: ignore[index]
# Sprint 5 bounded reversal: dispute reversal flips the
# effective outcome — predictors who picked the new winning
# side are the ones whose timestamps qualify.
outcome = _effective_outcome(original, market.market_id, events)
finalize_ts = market.finalize_ts
for pred in market.predictions:
if pred.get("node_id") != node_id:
continue
if pred.get("side") != outcome:
continue
ts = float(pred.get("timestamp") or 0.0)
# Use the LATER of prediction timestamp and finalize timestamp —
# the "successful prediction" only crystallizes when finalize lands.
ts = max(ts, finalize_ts)
if best_ts is None or ts > best_ts:
best_ts = ts
return best_ts
__all__ = [
"OracleRepBreakdown",
"compute_oracle_rep",
"compute_oracle_rep_breakdown",
"compute_oracle_rep_lifetime",
"last_successful_prediction_ts",
]
@@ -0,0 +1,96 @@
"""Weekly vote budget — RULES §3.7.
weekly_budget = weekly_vote_base + floor(oracle_rep / weekly_vote_per_oracle)
Notes on placement: the budget is reputation-derived and gates how
many upreps a node can cast in a 7-day window. Anti-gaming penalties
shrink each uprep's *weight*, but the budget is what bounds *count*.
Both layers run together to defeat farming.
Enforcement is upstream of the chain (the producer must check budget
before signing a new ``uprep`` event); this module provides the
computation and a chain-side audit (``count_upreps_in_last_week``) so
verifiers can spot budget violations.
"""
from __future__ import annotations
import math
from typing import Any, Iterable
from services.infonet.config import CONFIG
from services.infonet.reputation.oracle_rep import compute_oracle_rep
_SECONDS_PER_DAY = 86400.0
_WEEK_S = 7 * _SECONDS_PER_DAY
def _payload(event: dict[str, Any]) -> dict[str, Any]:
p = event.get("payload")
return p if isinstance(p, dict) else {}
def compute_weekly_vote_budget(
node_id: str,
chain: Iterable[dict[str, Any]],
) -> int:
"""Per-week uprep budget for ``node_id``."""
base = int(CONFIG["weekly_vote_base"])
per_oracle = int(CONFIG["weekly_vote_per_oracle"])
if per_oracle <= 0:
return base
rep = compute_oracle_rep(node_id, chain)
return base + math.floor(rep / per_oracle)
def count_upreps_in_last_week(
node_id: str,
chain: Iterable[dict[str, Any]],
*,
now: float,
) -> int:
"""Count of ``uprep`` events authored by ``node_id`` in the past 7 days
relative to ``now``. Used by chain-side audits.
"""
cutoff = float(now) - _WEEK_S
count = 0
for ev in chain:
if not isinstance(ev, dict):
continue
if ev.get("event_type") != "uprep":
continue
if ev.get("node_id") != node_id:
continue
try:
ts = float(ev.get("timestamp") or 0.0)
except (TypeError, ValueError):
continue
if cutoff <= ts <= float(now):
count += 1
return count
def is_budget_exceeded(
node_id: str,
chain: Iterable[dict[str, Any]],
*,
now: float,
) -> bool:
"""``True`` if the node has cast more upreps in the past 7 days than
its current weekly budget allows.
Cross-cutting design rule: producers should call this in the
background as a soft-fail check — the user's queued uprep is still
accepted, but flagged for delayed processing rather than refused
outright. Constitutional rejections are reserved for unsigned
writes / replays / rotation-during-active-stakes.
"""
return count_upreps_in_last_week(node_id, chain, now=now) > compute_weekly_vote_budget(node_id, chain)
__all__ = [
"compute_weekly_vote_budget",
"count_upreps_in_last_week",
"is_budget_exceeded",
]
+859
View File
@@ -0,0 +1,859 @@
"""Event-type registry and per-event payload validators for the Infonet
economy layer.
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §4.1.
The legacy ``services/mesh/mesh_schema.py`` ships
``ACTIVE_PUBLIC_LEDGER_EVENT_TYPES`` for the existing mesh / DM / oracle
events. This module ships ``INFONET_ECONOMY_EVENT_TYPES`` — a disjoint
set of 40+ NEW event types added by the economy layer. Sprint 1's
adversarial test asserts the disjointness invariant.
Sprint 1 implements *structural* validators only — they assert payload
shape (required fields, basic types, enum membership). Deep semantic
validation (e.g. that ``probability_at_bet`` was actually computed from
the live chain state, that ``evidence_content_hash`` is canonical) lives
in later sprints alongside the modules that produce those values.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Callable
# ─── Event-type set ──────────────────────────────────────────────────────
# RULES_SKELETON.md §4.1.
# Disjoint from mesh_schema.ACTIVE_PUBLIC_LEDGER_EVENT_TYPES — the union
# is the full public ledger surface once the adapter is wired in.
INFONET_ECONOMY_EVENT_TYPES: frozenset[str] = frozenset({
# Reputation
"uprep",
"downrep", # held off the active set in Sprint 2 — see BRAINDUMP §11
# Markets / resolution-as-prediction
"prediction_create",
"prediction_place",
"truth_stake_place",
"truth_stake_resolve",
"market_snapshot",
"evidence_submit",
"resolution_stake",
"bootstrap_resolution_vote",
"resolution_finalize",
# Disputes
"dispute_open",
"dispute_stake",
"dispute_resolve",
# Gates (extend the existing legacy gate_create)
"gate_enter",
"gate_exit",
"gate_lock",
# Gate shutdown lifecycle
"gate_suspend_file",
"gate_suspend_vote",
"gate_suspend_execute",
"gate_shutdown_file",
"gate_shutdown_vote",
"gate_shutdown_execute",
"gate_unsuspend",
"gate_shutdown_appeal_file",
"gate_shutdown_appeal_vote",
"gate_shutdown_appeal_resolve",
# Governance
"petition_file",
"petition_sign",
"petition_vote",
"challenge_file",
"challenge_vote",
"petition_execute",
# Upgrade-hash governance
"upgrade_propose",
"upgrade_sign",
"upgrade_vote",
"upgrade_challenge",
"upgrade_challenge_vote",
"upgrade_signal_ready",
"upgrade_activate",
# Identity
"node_register",
"identity_rotate",
"citizenship_claim",
# Economy
"coin_transfer",
"coin_mint",
"bounty_create",
"bounty_claim",
# Content
"post_create",
"post_reply",
})
# ─── Validator dataclass + helpers ───────────────────────────────────────
@dataclass(frozen=True)
class InfonetEventSchema:
event_type: str
required_fields: tuple[str, ...]
optional_fields: tuple[str, ...]
validate: Callable[[dict[str, Any]], tuple[bool, str]]
def validate_payload(self, payload: dict[str, Any]) -> tuple[bool, str]:
return self.validate(payload)
def _require(payload: dict[str, Any], fields: tuple[str, ...]) -> tuple[bool, str]:
if not isinstance(payload, dict):
return False, "payload must be an object"
for key in fields:
if key not in payload:
return False, f"Missing field: {key}"
return True, "ok"
def _is_nonempty_str(val: Any) -> bool:
return isinstance(val, str) and bool(val.strip())
def _is_positive_number(val: Any) -> bool:
return isinstance(val, (int, float)) and not isinstance(val, bool) and val > 0
def _is_nonnegative_number(val: Any) -> bool:
return isinstance(val, (int, float)) and not isinstance(val, bool) and val >= 0
# ─── Per-event validators ───────────────────────────────────────────────
# Sprint 1 scope: structural (required fields, type sanity, enum guards).
# Deeper semantic checks (cross-event references, hash canonicalization,
# probability_at_bet reconstruction) ship in the sprint that owns the
# producing module.
def _validate_uprep(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("target_node_id", "target_event_id"))
if not ok:
return ok, why
if not _is_nonempty_str(p["target_node_id"]):
return False, "target_node_id must be non-empty"
if not _is_nonempty_str(p["target_event_id"]):
return False, "target_event_id must be non-empty"
return True, "ok"
def _validate_downrep(p: dict[str, Any]) -> tuple[bool, str]:
return _validate_uprep(p)
def _validate_prediction_create(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("market_id", "market_type", "question", "trigger_date", "creation_bond"))
if not ok:
return ok, why
if not _is_nonempty_str(p["market_id"]):
return False, "market_id must be non-empty"
if p["market_type"] not in ("objective", "subjective"):
return False, "market_type must be 'objective' or 'subjective'"
if not _is_nonempty_str(p["question"]):
return False, "question must be non-empty"
if not _is_positive_number(p["trigger_date"]):
return False, "trigger_date must be a positive timestamp"
if not _is_nonnegative_number(p["creation_bond"]):
return False, "creation_bond must be a non-negative number"
return True, "ok"
def _validate_prediction_place(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("market_id", "side", "probability_at_bet"))
if not ok:
return ok, why
if not _is_nonempty_str(p["market_id"]):
return False, "market_id must be non-empty"
if p["side"] not in ("yes", "no"):
return False, "side must be 'yes' or 'no'"
prob = p["probability_at_bet"]
if not isinstance(prob, (int, float)) or isinstance(prob, bool):
return False, "probability_at_bet must be numeric"
if not (0 <= prob <= 100):
return False, "probability_at_bet must be in [0, 100]"
if "stake_amount" in p:
if p["stake_amount"] is not None and not _is_positive_number(p["stake_amount"]):
return False, "stake_amount must be positive when present"
return True, "ok"
def _validate_truth_stake_place(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("message_id", "poster_id", "side", "amount", "duration_days"))
if not ok:
return ok, why
if not _is_nonempty_str(p["message_id"]):
return False, "message_id must be non-empty"
if not _is_nonempty_str(p["poster_id"]):
return False, "poster_id must be non-empty"
if p["side"] not in ("truth", "false"):
return False, "side must be 'truth' or 'false'"
if not _is_positive_number(p["amount"]):
return False, "amount must be positive"
duration = p["duration_days"]
if not isinstance(duration, int) or isinstance(duration, bool) or duration <= 0:
return False, "duration_days must be a positive integer"
return True, "ok"
def _validate_truth_stake_resolve(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("message_id", "outcome"))
if not ok:
return ok, why
if p["outcome"] not in ("truth", "false", "tie"):
return False, "outcome must be 'truth', 'false', or 'tie'"
return True, "ok"
def _validate_market_snapshot(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(
p,
(
"market_id",
"frozen_participant_count",
"frozen_total_stake",
"frozen_predictor_ids",
"frozen_probability_state",
"frozen_at",
),
)
if not ok:
return ok, why
if not isinstance(p["frozen_participant_count"], int) or isinstance(p["frozen_participant_count"], bool):
return False, "frozen_participant_count must be int"
if p["frozen_participant_count"] < 0:
return False, "frozen_participant_count must be >= 0"
if not _is_nonnegative_number(p["frozen_total_stake"]):
return False, "frozen_total_stake must be a non-negative number"
if not isinstance(p["frozen_predictor_ids"], list):
return False, "frozen_predictor_ids must be a list"
if not all(_is_nonempty_str(x) for x in p["frozen_predictor_ids"]):
return False, "frozen_predictor_ids entries must be non-empty strings"
state = p["frozen_probability_state"]
if not isinstance(state, dict) or "yes" not in state or "no" not in state:
return False, "frozen_probability_state must be {yes, no}"
if not _is_positive_number(p["frozen_at"]):
return False, "frozen_at must be a positive timestamp"
return True, "ok"
def _validate_evidence_submit(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(
p,
(
"market_id",
"claimed_outcome",
"evidence_hashes",
"source_description",
"evidence_content_hash",
"submission_hash",
"bond",
),
)
if not ok:
return ok, why
if p["claimed_outcome"] not in ("yes", "no"):
return False, "claimed_outcome must be 'yes' or 'no'"
if not isinstance(p["evidence_hashes"], list) or not p["evidence_hashes"]:
return False, "evidence_hashes must be a non-empty list"
if not all(_is_nonempty_str(h) for h in p["evidence_hashes"]):
return False, "evidence_hashes entries must be non-empty strings"
if not isinstance(p["source_description"], str):
return False, "source_description must be a string"
if not _is_nonempty_str(p["evidence_content_hash"]):
return False, "evidence_content_hash must be non-empty"
if not _is_nonempty_str(p["submission_hash"]):
return False, "submission_hash must be non-empty"
if not _is_nonnegative_number(p["bond"]):
return False, "bond must be a non-negative number"
return True, "ok"
def _validate_resolution_stake(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("market_id", "side", "amount", "rep_type"))
if not ok:
return ok, why
if p["side"] not in ("yes", "no", "data_unavailable"):
return False, "side must be 'yes' | 'no' | 'data_unavailable'"
if not _is_positive_number(p["amount"]):
return False, "amount must be positive"
if p["rep_type"] not in ("oracle", "common"):
return False, "rep_type must be 'oracle' or 'common'"
return True, "ok"
def _validate_bootstrap_resolution_vote(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("market_id", "side", "pow_nonce"))
if not ok:
return ok, why
if p["side"] not in ("yes", "no"):
return False, "side must be 'yes' or 'no'"
if not isinstance(p["pow_nonce"], int) or isinstance(p["pow_nonce"], bool) or p["pow_nonce"] < 0:
return False, "pow_nonce must be a non-negative integer"
return True, "ok"
def _validate_resolution_finalize(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("market_id", "outcome", "is_provisional", "snapshot_event_hash"))
if not ok:
return ok, why
if p["outcome"] not in ("yes", "no", "invalid"):
return False, "outcome must be 'yes' | 'no' | 'invalid'"
if not isinstance(p["is_provisional"], bool):
return False, "is_provisional must be a boolean"
if not _is_nonempty_str(p["snapshot_event_hash"]):
return False, "snapshot_event_hash must be non-empty"
return True, "ok"
def _validate_dispute_open(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("market_id", "challenger_stake", "reason"))
if not ok:
return ok, why
if not _is_positive_number(p["challenger_stake"]):
return False, "challenger_stake must be positive"
if not _is_nonempty_str(p["reason"]):
return False, "reason must be non-empty"
return True, "ok"
def _validate_dispute_stake(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("dispute_id", "side", "amount", "rep_type"))
if not ok:
return ok, why
if p["side"] not in ("confirm", "reverse"):
return False, "side must be 'confirm' or 'reverse'"
if not _is_positive_number(p["amount"]):
return False, "amount must be positive"
if p["rep_type"] not in ("oracle", "common"):
return False, "rep_type must be 'oracle' or 'common'"
return True, "ok"
def _validate_dispute_resolve(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("dispute_id", "outcome"))
if not ok:
return ok, why
if p["outcome"] not in ("upheld", "reversed", "tie"):
return False, "outcome must be 'upheld' | 'reversed' | 'tie'"
return True, "ok"
def _validate_gate_enter(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("gate_id", "sacrifice_amount"))
if not ok:
return ok, why
if not _is_nonempty_str(p["gate_id"]):
return False, "gate_id must be non-empty"
if not _is_positive_number(p["sacrifice_amount"]):
return False, "sacrifice_amount must be positive"
return True, "ok"
def _validate_gate_exit(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("gate_id",))
if not ok:
return ok, why
if not _is_nonempty_str(p["gate_id"]):
return False, "gate_id must be non-empty"
return True, "ok"
def _validate_gate_lock(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("gate_id", "lock_cost"))
if not ok:
return ok, why
if not _is_nonempty_str(p["gate_id"]):
return False, "gate_id must be non-empty"
if not _is_positive_number(p["lock_cost"]):
return False, "lock_cost must be positive"
return True, "ok"
def _validate_gate_action_petition_file(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("petition_id", "gate_id", "reason", "evidence_hashes"))
if not ok:
return ok, why
if not _is_nonempty_str(p["petition_id"]):
return False, "petition_id must be non-empty"
if not _is_nonempty_str(p["gate_id"]):
return False, "gate_id must be non-empty"
if not isinstance(p["reason"], str) or len(p["reason"]) > 2000:
return False, "reason must be a string up to 2000 chars"
if not isinstance(p["evidence_hashes"], list) or not p["evidence_hashes"]:
return False, "evidence_hashes must be non-empty"
if not all(_is_nonempty_str(h) for h in p["evidence_hashes"]):
return False, "evidence_hashes entries must be non-empty strings"
return True, "ok"
def _validate_gate_action_vote(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("petition_id", "vote"))
if not ok:
return ok, why
if p["vote"] not in ("for", "against"):
return False, "vote must be 'for' or 'against'"
return True, "ok"
def _validate_gate_action_execute(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("petition_id", "gate_id"))
if not ok:
return ok, why
if not _is_nonempty_str(p["petition_id"]):
return False, "petition_id must be non-empty"
if not _is_nonempty_str(p["gate_id"]):
return False, "gate_id must be non-empty"
return True, "ok"
def _validate_gate_unsuspend(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("gate_id",))
if not ok:
return ok, why
if not _is_nonempty_str(p["gate_id"]):
return False, "gate_id must be non-empty"
return True, "ok"
def _validate_gate_shutdown_appeal_file(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("petition_id", "gate_id", "target_petition_id", "reason", "evidence_hashes"))
if not ok:
return ok, why
if not _is_nonempty_str(p["target_petition_id"]):
return False, "target_petition_id must be non-empty"
if not isinstance(p["reason"], str) or len(p["reason"]) > 2000:
return False, "reason must be a string up to 2000 chars"
if not isinstance(p["evidence_hashes"], list) or not p["evidence_hashes"]:
return False, "evidence_hashes must be non-empty"
return True, "ok"
def _validate_gate_shutdown_appeal_resolve(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("petition_id", "outcome"))
if not ok:
return ok, why
if p["outcome"] not in ("voided_shutdown", "resumed"):
return False, "outcome must be 'voided_shutdown' or 'resumed'"
return True, "ok"
_VALID_PETITION_PAYLOAD_TYPES = frozenset({
"UPDATE_PARAM",
"BATCH_UPDATE_PARAMS",
"ENABLE_FEATURE",
"DISABLE_FEATURE",
})
def _validate_petition_file(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("petition_id", "petition_payload"))
if not ok:
return ok, why
if not _is_nonempty_str(p["petition_id"]):
return False, "petition_id must be non-empty"
payload = p["petition_payload"]
if not isinstance(payload, dict) or "type" not in payload:
return False, "petition_payload must be an object with a 'type' field"
if payload["type"] not in _VALID_PETITION_PAYLOAD_TYPES:
return False, f"petition_payload type must be one of {sorted(_VALID_PETITION_PAYLOAD_TYPES)}"
# Structural shape per type. Semantic checks (key existence, bounds)
# happen in the Sprint 7 DSL executor.
t = payload["type"]
if t == "UPDATE_PARAM":
if "key" not in payload or "value" not in payload:
return False, "UPDATE_PARAM requires key + value"
if not _is_nonempty_str(payload["key"]):
return False, "UPDATE_PARAM.key must be non-empty"
elif t == "BATCH_UPDATE_PARAMS":
if "updates" not in payload or not isinstance(payload["updates"], list) or not payload["updates"]:
return False, "BATCH_UPDATE_PARAMS.updates must be a non-empty list"
for u in payload["updates"]:
if not isinstance(u, dict) or "key" not in u or "value" not in u:
return False, "BATCH_UPDATE_PARAMS entries must be {key, value}"
if not _is_nonempty_str(u["key"]):
return False, "BATCH_UPDATE_PARAMS entry key must be non-empty"
elif t in ("ENABLE_FEATURE", "DISABLE_FEATURE"):
if "feature" not in payload or not _is_nonempty_str(payload["feature"]):
return False, f"{t}.feature must be non-empty"
return True, "ok"
def _validate_petition_sign(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("petition_id",))
if not ok:
return ok, why
if not _is_nonempty_str(p["petition_id"]):
return False, "petition_id must be non-empty"
return True, "ok"
def _validate_petition_vote(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("petition_id", "vote"))
if not ok:
return ok, why
if p["vote"] not in ("for", "against"):
return False, "vote must be 'for' or 'against'"
return True, "ok"
def _validate_challenge_file(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("petition_id", "reason"))
if not ok:
return ok, why
if not _is_nonempty_str(p["petition_id"]):
return False, "petition_id must be non-empty"
if not _is_nonempty_str(p["reason"]):
return False, "reason must be non-empty"
return True, "ok"
def _validate_challenge_vote(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("petition_id", "vote"))
if not ok:
return ok, why
if p["vote"] not in ("uphold", "void"):
return False, "vote must be 'uphold' or 'void'"
return True, "ok"
def _validate_petition_execute(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("petition_id",))
if not ok:
return ok, why
if not _is_nonempty_str(p["petition_id"]):
return False, "petition_id must be non-empty"
return True, "ok"
def _validate_upgrade_propose(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(
p,
(
"proposal_id",
"release_hash",
"release_description",
"target_protocol_version",
),
)
if not ok:
return ok, why
if not _is_nonempty_str(p["proposal_id"]):
return False, "proposal_id must be non-empty"
if not _is_nonempty_str(p["release_hash"]):
return False, "release_hash must be non-empty"
if not isinstance(p["release_description"], str) or len(p["release_description"]) > 4000:
return False, "release_description must be a string up to 4000 chars"
if not _is_nonempty_str(p["target_protocol_version"]):
return False, "target_protocol_version must be non-empty"
return True, "ok"
def _validate_upgrade_sign(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("proposal_id",))
if not ok:
return ok, why
if not _is_nonempty_str(p["proposal_id"]):
return False, "proposal_id must be non-empty"
return True, "ok"
def _validate_upgrade_vote(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("proposal_id", "vote"))
if not ok:
return ok, why
if p["vote"] not in ("for", "against"):
return False, "vote must be 'for' or 'against'"
return True, "ok"
def _validate_upgrade_challenge(p: dict[str, Any]) -> tuple[bool, str]:
return _validate_challenge_file({"petition_id": p.get("proposal_id", ""), "reason": p.get("reason", "")})
def _validate_upgrade_challenge_vote(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("proposal_id", "vote"))
if not ok:
return ok, why
if p["vote"] not in ("uphold", "void"):
return False, "vote must be 'uphold' or 'void'"
return True, "ok"
def _validate_upgrade_signal_ready(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("proposal_id", "release_hash"))
if not ok:
return ok, why
if not _is_nonempty_str(p["proposal_id"]):
return False, "proposal_id must be non-empty"
if not _is_nonempty_str(p["release_hash"]):
return False, "release_hash must be non-empty"
return True, "ok"
def _validate_upgrade_activate(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("proposal_id", "new_protocol_version"))
if not ok:
return ok, why
if not _is_nonempty_str(p["proposal_id"]):
return False, "proposal_id must be non-empty"
if not _is_nonempty_str(p["new_protocol_version"]):
return False, "new_protocol_version must be non-empty"
return True, "ok"
def _validate_node_register(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("public_key", "public_key_algo", "node_class"))
if not ok:
return ok, why
if not _is_nonempty_str(p["public_key"]):
return False, "public_key must be non-empty"
if p["public_key_algo"] not in ("ed25519", "ecdsa"):
return False, "public_key_algo must be 'ed25519' or 'ecdsa'"
if p["node_class"] not in ("heavy", "light"):
return False, "node_class must be 'heavy' or 'light'"
return True, "ok"
def _validate_identity_rotate(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(
p,
(
"old_node_id",
"old_public_key",
"old_public_key_algo",
"new_public_key",
"new_public_key_algo",
"old_signature",
),
)
if not ok:
return ok, why
return True, "ok"
def _validate_citizenship_claim(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("sacrifice_amount",))
if not ok:
return ok, why
if not _is_positive_number(p["sacrifice_amount"]):
return False, "sacrifice_amount must be positive"
return True, "ok"
def _validate_coin_transfer(p: dict[str, Any]) -> tuple[bool, str]:
# Sprint 1 logical-only — privacy primitives (RingCT) replace this in
# Sprint 11+. Until then, enforce a simple {to, amount} shape.
ok, why = _require(p, ("to_node_id", "amount"))
if not ok:
return ok, why
if not _is_nonempty_str(p["to_node_id"]):
return False, "to_node_id must be non-empty"
if not _is_positive_number(p["amount"]):
return False, "amount must be positive"
return True, "ok"
def _validate_coin_mint(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("month", "total_minted", "ubi_pool", "dividend_pool"))
if not ok:
return ok, why
if not _is_nonempty_str(p["month"]):
return False, "month must be non-empty (e.g. '2026-04')"
for k in ("total_minted", "ubi_pool", "dividend_pool"):
if not _is_nonnegative_number(p[k]):
return False, f"{k} must be non-negative"
return True, "ok"
def _validate_bounty_create(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("bounty_id", "amount", "description"))
if not ok:
return ok, why
if not _is_nonempty_str(p["bounty_id"]):
return False, "bounty_id must be non-empty"
if not _is_positive_number(p["amount"]):
return False, "amount must be positive"
if not _is_nonempty_str(p["description"]):
return False, "description must be non-empty"
return True, "ok"
def _validate_bounty_claim(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("bounty_id",))
if not ok:
return ok, why
if not _is_nonempty_str(p["bounty_id"]):
return False, "bounty_id must be non-empty"
return True, "ok"
def _validate_post_create(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("post_id", "body"))
if not ok:
return ok, why
if not _is_nonempty_str(p["post_id"]):
return False, "post_id must be non-empty"
if not isinstance(p["body"], str):
return False, "body must be a string"
return True, "ok"
def _validate_post_reply(p: dict[str, Any]) -> tuple[bool, str]:
ok, why = _require(p, ("post_id", "parent_post_id", "body"))
if not ok:
return ok, why
if not _is_nonempty_str(p["post_id"]):
return False, "post_id must be non-empty"
if not _is_nonempty_str(p["parent_post_id"]):
return False, "parent_post_id must be non-empty"
if not isinstance(p["body"], str):
return False, "body must be a string"
return True, "ok"
# ─── Schema registry ─────────────────────────────────────────────────────
_SCHEMA_REGISTRY: dict[str, InfonetEventSchema] = {}
def _reg(event_type: str, required: tuple[str, ...], optional: tuple[str, ...], fn) -> None:
_SCHEMA_REGISTRY[event_type] = InfonetEventSchema(
event_type=event_type,
required_fields=required,
optional_fields=optional,
validate=fn,
)
_reg("uprep", ("target_node_id", "target_event_id"), (), _validate_uprep)
_reg("downrep", ("target_node_id", "target_event_id"), (), _validate_downrep)
_reg("prediction_create",
("market_id", "market_type", "question", "trigger_date", "creation_bond"),
(), _validate_prediction_create)
_reg("prediction_place",
("market_id", "side", "probability_at_bet"),
("stake_amount",), _validate_prediction_place)
_reg("truth_stake_place",
("message_id", "poster_id", "side", "amount", "duration_days"),
(), _validate_truth_stake_place)
_reg("truth_stake_resolve",
("message_id", "outcome"),
(), _validate_truth_stake_resolve)
_reg("market_snapshot",
("market_id", "frozen_participant_count", "frozen_total_stake",
"frozen_predictor_ids", "frozen_probability_state", "frozen_at"),
("snapshot_event_hash",), _validate_market_snapshot)
_reg("evidence_submit",
("market_id", "claimed_outcome", "evidence_hashes", "source_description",
"evidence_content_hash", "submission_hash", "bond"),
(), _validate_evidence_submit)
_reg("resolution_stake",
("market_id", "side", "amount", "rep_type"),
(), _validate_resolution_stake)
_reg("bootstrap_resolution_vote",
("market_id", "side", "pow_nonce"),
(), _validate_bootstrap_resolution_vote)
_reg("resolution_finalize",
("market_id", "outcome", "is_provisional", "snapshot_event_hash"),
(), _validate_resolution_finalize)
_reg("dispute_open", ("market_id", "challenger_stake", "reason"), (), _validate_dispute_open)
_reg("dispute_stake", ("dispute_id", "side", "amount", "rep_type"), (), _validate_dispute_stake)
_reg("dispute_resolve", ("dispute_id", "outcome"), (), _validate_dispute_resolve)
_reg("gate_enter", ("gate_id", "sacrifice_amount"), (), _validate_gate_enter)
_reg("gate_exit", ("gate_id",), (), _validate_gate_exit)
_reg("gate_lock", ("gate_id", "lock_cost"), (), _validate_gate_lock)
_reg("gate_suspend_file",
("petition_id", "gate_id", "reason", "evidence_hashes"), (),
_validate_gate_action_petition_file)
_reg("gate_suspend_vote", ("petition_id", "vote"), (), _validate_gate_action_vote)
_reg("gate_suspend_execute", ("petition_id", "gate_id"), (), _validate_gate_action_execute)
_reg("gate_shutdown_file",
("petition_id", "gate_id", "reason", "evidence_hashes"), (),
_validate_gate_action_petition_file)
_reg("gate_shutdown_vote", ("petition_id", "vote"), (), _validate_gate_action_vote)
_reg("gate_shutdown_execute", ("petition_id", "gate_id"), (), _validate_gate_action_execute)
_reg("gate_unsuspend", ("gate_id",), (), _validate_gate_unsuspend)
_reg("gate_shutdown_appeal_file",
("petition_id", "gate_id", "target_petition_id", "reason", "evidence_hashes"),
(), _validate_gate_shutdown_appeal_file)
_reg("gate_shutdown_appeal_vote", ("petition_id", "vote"), (), _validate_gate_action_vote)
_reg("gate_shutdown_appeal_resolve", ("petition_id", "outcome"), (), _validate_gate_shutdown_appeal_resolve)
_reg("petition_file", ("petition_id", "petition_payload"), (), _validate_petition_file)
_reg("petition_sign", ("petition_id",), (), _validate_petition_sign)
_reg("petition_vote", ("petition_id", "vote"), (), _validate_petition_vote)
_reg("challenge_file", ("petition_id", "reason"), (), _validate_challenge_file)
_reg("challenge_vote", ("petition_id", "vote"), (), _validate_challenge_vote)
_reg("petition_execute", ("petition_id",), (), _validate_petition_execute)
_reg("upgrade_propose",
("proposal_id", "release_hash", "release_description", "target_protocol_version"),
("release_url", "compatibility_notes"),
_validate_upgrade_propose)
_reg("upgrade_sign", ("proposal_id",), (), _validate_upgrade_sign)
_reg("upgrade_vote", ("proposal_id", "vote"), (), _validate_upgrade_vote)
_reg("upgrade_challenge", ("proposal_id", "reason"), (), _validate_upgrade_challenge)
_reg("upgrade_challenge_vote", ("proposal_id", "vote"), (), _validate_upgrade_challenge_vote)
_reg("upgrade_signal_ready", ("proposal_id", "release_hash"), (), _validate_upgrade_signal_ready)
_reg("upgrade_activate", ("proposal_id", "new_protocol_version"), (), _validate_upgrade_activate)
_reg("node_register", ("public_key", "public_key_algo", "node_class"), (), _validate_node_register)
_reg("identity_rotate",
("old_node_id", "old_public_key", "old_public_key_algo",
"new_public_key", "new_public_key_algo", "old_signature"),
(), _validate_identity_rotate)
_reg("citizenship_claim", ("sacrifice_amount",), (), _validate_citizenship_claim)
_reg("coin_transfer", ("to_node_id", "amount"), (), _validate_coin_transfer)
_reg("coin_mint", ("month", "total_minted", "ubi_pool", "dividend_pool"), (), _validate_coin_mint)
_reg("bounty_create", ("bounty_id", "amount", "description"), (), _validate_bounty_create)
_reg("bounty_claim", ("bounty_id",), (), _validate_bounty_claim)
_reg("post_create", ("post_id", "body"), (), _validate_post_create)
_reg("post_reply", ("post_id", "parent_post_id", "body"), (), _validate_post_reply)
def get_infonet_schema(event_type: str) -> InfonetEventSchema | None:
return _SCHEMA_REGISTRY.get(event_type)
def validate_infonet_event_payload(
event_type: str,
payload: dict[str, Any],
) -> tuple[bool, str]:
"""Validate ``payload`` against the schema for ``event_type``.
Sprint 1 contract:
- Event types not in ``INFONET_ECONOMY_EVENT_TYPES`` are rejected.
- Every type in ``INFONET_ECONOMY_EVENT_TYPES`` MUST have a registered
validator (asserted by ``assert_registry_complete``).
"""
if event_type not in INFONET_ECONOMY_EVENT_TYPES:
return False, f"Unknown event_type for infonet economy: {event_type}"
schema = _SCHEMA_REGISTRY.get(event_type)
if schema is None:
return False, f"No validator registered for: {event_type}"
return schema.validate_payload(payload)
def assert_registry_complete() -> None:
"""Sprint 1 invariant: every event type has a validator."""
missing = sorted(INFONET_ECONOMY_EVENT_TYPES - set(_SCHEMA_REGISTRY.keys()))
if missing:
raise AssertionError(f"INFONET_ECONOMY_EVENT_TYPES without validators: {missing}")
__all__ = [
"INFONET_ECONOMY_EVENT_TYPES",
"InfonetEventSchema",
"assert_registry_complete",
"get_infonet_schema",
"validate_infonet_event_payload",
]
@@ -0,0 +1,121 @@
"""Test-only helpers for synthesizing chain events.
Mirrors the dict shape that ``InfonetHashchainAdapter.dry_run_append``
emits, which in turn mirrors the legacy ``mesh_hashchain.Infonet.append``
output. Tests call these helpers to build synthetic chains; production
code is unaffected.
"""
from __future__ import annotations
from typing import Any
def make_event(
event_type: str,
node_id: str,
payload: dict[str, Any],
*,
timestamp: float,
sequence: int = 1,
) -> dict[str, Any]:
return {
"event_type": event_type,
"node_id": node_id,
"timestamp": float(timestamp),
"sequence": int(sequence),
"payload": dict(payload),
}
def make_market_chain(
market_id: str,
creator_id: str,
*,
market_type: str = "objective",
bootstrap_index: int | None = None,
base_ts: float = 1_700_000_000.0,
participants: int = 5,
total_stake: float = 10.0,
outcome: str | None = "yes",
is_provisional: bool = False,
predictions: list[dict[str, Any]] | None = None,
) -> list[dict[str, Any]]:
"""Build a coherent set of events for one market.
Returns events in chain order: prediction_create → prediction_place
(per ``predictions``) → market_snapshot → resolution_finalize (if
``outcome`` is not None). Use this to set up "did the mint rule
fire correctly" tests.
"""
chain: list[dict[str, Any]] = []
seq = 0
def _next_seq() -> int:
nonlocal seq
seq += 1
return seq
chain.append(make_event(
"prediction_create",
creator_id,
{
"market_id": market_id,
"market_type": market_type,
"question": f"Q for {market_id}",
"trigger_date": base_ts + 86400.0,
"creation_bond": 3,
**({"bootstrap_index": bootstrap_index} if bootstrap_index is not None else {}),
},
timestamp=base_ts,
sequence=_next_seq(),
))
predictor_ids: list[str] = []
for i, pred in enumerate(predictions or []):
chain.append(make_event(
"prediction_place",
pred["node_id"],
{
"market_id": market_id,
"side": pred["side"],
"probability_at_bet": pred.get("probability_at_bet", 50.0),
**({"stake_amount": pred["stake_amount"]} if pred.get("stake_amount") is not None else {}),
},
timestamp=base_ts + 60.0 + i,
sequence=_next_seq(),
))
predictor_ids.append(pred["node_id"])
snapshot_ts = base_ts + 3600.0
chain.append(make_event(
"market_snapshot",
creator_id,
{
"market_id": market_id,
"frozen_participant_count": participants,
"frozen_total_stake": float(total_stake),
"frozen_predictor_ids": list(dict.fromkeys(predictor_ids)),
"frozen_probability_state": {"yes": 0.5, "no": 0.5},
"frozen_at": snapshot_ts,
},
timestamp=snapshot_ts,
sequence=_next_seq(),
))
if outcome is not None:
finalize_ts = base_ts + 7200.0
chain.append(make_event(
"resolution_finalize",
creator_id,
{
"market_id": market_id,
"outcome": outcome,
"is_provisional": bool(is_provisional),
"snapshot_event_hash": f"snap-{market_id}",
},
timestamp=finalize_ts,
sequence=_next_seq(),
))
return chain
@@ -0,0 +1,147 @@
"""Test-only chain-builder helpers for gate scenarios."""
from __future__ import annotations
from typing import Any
from services.infonet.tests._chain_factory import make_event
def make_gate_create(
gate_id: str,
creator: str,
*,
ts: float,
seq: int = 1,
entry_sacrifice: int = 5,
min_overall_rep: int = 0,
min_gate_rep: dict[str, int] | None = None,
display_name: str | None = None,
) -> dict[str, Any]:
rules: dict[str, Any] = {
"entry_sacrifice": entry_sacrifice,
"min_overall_rep": min_overall_rep,
}
if min_gate_rep:
rules["min_gate_rep"] = dict(min_gate_rep)
return make_event(
"gate_create", creator,
{"gate_id": gate_id, "display_name": display_name or gate_id, "rules": rules},
timestamp=ts, sequence=seq,
)
def make_gate_enter(gate_id: str, node: str, *, ts: float, seq: int,
sacrifice: int = 5) -> dict[str, Any]:
return make_event(
"gate_enter", node,
{"gate_id": gate_id, "sacrifice_amount": sacrifice},
timestamp=ts, sequence=seq,
)
def make_gate_exit(gate_id: str, node: str, *, ts: float, seq: int) -> dict[str, Any]:
return make_event(
"gate_exit", node,
{"gate_id": gate_id},
timestamp=ts, sequence=seq,
)
def make_gate_lock(gate_id: str, node: str, *, ts: float, seq: int,
lock_cost: int = 10) -> dict[str, Any]:
return make_event(
"gate_lock", node,
{"gate_id": gate_id, "lock_cost": lock_cost},
timestamp=ts, sequence=seq,
)
def make_suspend_file(gate_id: str, filer: str, petition_id: str, *,
ts: float, seq: int,
reason: str = "abuse",
evidence: list[str] | None = None) -> dict[str, Any]:
return make_event(
"gate_suspend_file", filer,
{"petition_id": petition_id, "gate_id": gate_id,
"reason": reason, "evidence_hashes": list(evidence or ["ev1"])},
timestamp=ts, sequence=seq,
)
def make_suspend_execute(gate_id: str, petition_id: str, *,
ts: float, seq: int,
executor: str = "creator") -> dict[str, Any]:
return make_event(
"gate_suspend_execute", executor,
{"petition_id": petition_id, "gate_id": gate_id},
timestamp=ts, sequence=seq,
)
def make_unsuspend(gate_id: str, *, ts: float, seq: int,
executor: str = "creator") -> dict[str, Any]:
return make_event(
"gate_unsuspend", executor,
{"gate_id": gate_id},
timestamp=ts, sequence=seq,
)
def make_shutdown_file(gate_id: str, filer: str, petition_id: str, *,
ts: float, seq: int,
reason: str = "still abusing",
evidence: list[str] | None = None) -> dict[str, Any]:
return make_event(
"gate_shutdown_file", filer,
{"petition_id": petition_id, "gate_id": gate_id,
"reason": reason, "evidence_hashes": list(evidence or ["ev1"])},
timestamp=ts, sequence=seq,
)
def make_shutdown_vote(gate_id: str, petition_id: str, vote: str, *,
ts: float, seq: int,
voter: str = "creator") -> dict[str, Any]:
return make_event(
"gate_shutdown_vote", voter,
{"petition_id": petition_id, "vote": vote, "gate_id": gate_id},
timestamp=ts, sequence=seq,
)
def make_shutdown_execute(gate_id: str, petition_id: str, *,
ts: float, seq: int,
executor: str = "creator") -> dict[str, Any]:
return make_event(
"gate_shutdown_execute", executor,
{"petition_id": petition_id, "gate_id": gate_id},
timestamp=ts, sequence=seq,
)
def make_appeal_file(gate_id: str, target_petition_id: str, filer: str,
petition_id: str, *,
ts: float, seq: int,
reason: str = "appeal",
evidence: list[str] | None = None) -> dict[str, Any]:
return make_event(
"gate_shutdown_appeal_file", filer,
{"petition_id": petition_id, "gate_id": gate_id,
"target_petition_id": target_petition_id,
"reason": reason,
"evidence_hashes": list(evidence or ["ev1"])},
timestamp=ts, sequence=seq,
)
def make_appeal_resolve(gate_id: str, petition_id: str, target_petition_id: str,
outcome: str, *, ts: float, seq: int,
resumed_execution_at: float | None = None,
resolver: str = "creator") -> dict[str, Any]:
payload = {"petition_id": petition_id, "outcome": outcome,
"target_petition_id": target_petition_id, "gate_id": gate_id}
if resumed_execution_at is not None:
payload["resumed_execution_at"] = resumed_execution_at
return make_event("gate_shutdown_appeal_resolve", resolver, payload,
timestamp=ts, sequence=seq)

Some files were not shown because too many files have changed in this diff Show More