mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-09-27 03:01:51 +02:00
release: prepare v0.9.7
This commit is contained in:
@@ -0,0 +1,178 @@
|
||||
"""ai_intel_store — compatibility wrapper around ai_pin_store + layer injection.
|
||||
|
||||
openclaw_channel.py and routers/ai_intel.py import from this module name.
|
||||
All pin/layer logic lives in ai_pin_store.py; this module re-exports with the
|
||||
expected function signatures and adds the layer injection helper.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from services.ai_pin_store import (
|
||||
create_pin,
|
||||
create_pins_batch,
|
||||
get_pins,
|
||||
delete_pin,
|
||||
clear_pins,
|
||||
pin_count,
|
||||
pins_as_geojson,
|
||||
purge_expired,
|
||||
# Layer CRUD
|
||||
create_layer,
|
||||
get_layers,
|
||||
update_layer,
|
||||
delete_layer,
|
||||
# Feed layers
|
||||
get_feed_layers,
|
||||
replace_layer_pins,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Re-exports expected by openclaw_channel._dispatch_command
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_all_intel_pins() -> list[dict[str, Any]]:
|
||||
"""Return all active pins (no filter, generous limit)."""
|
||||
return get_pins(limit=2000)
|
||||
|
||||
|
||||
def add_intel_pin(args: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Create a single pin from a command-channel args dict."""
|
||||
ea = args.get("entity_attachment")
|
||||
return create_pin(
|
||||
lat=float(args.get("lat", 0)),
|
||||
lng=float(args.get("lng", 0)),
|
||||
label=str(args.get("label", ""))[:200],
|
||||
category=str(args.get("category", "custom")),
|
||||
layer_id=str(args.get("layer_id", "")),
|
||||
color=str(args.get("color", "")),
|
||||
description=str(args.get("description", "")),
|
||||
source=str(args.get("source", "openclaw")),
|
||||
source_url=str(args.get("source_url", "")),
|
||||
confidence=float(args.get("confidence", 1.0)),
|
||||
ttl_hours=float(args.get("ttl_hours", 0)),
|
||||
metadata=args.get("metadata") or {},
|
||||
entity_attachment=ea if isinstance(ea, dict) else None,
|
||||
)
|
||||
|
||||
|
||||
def delete_intel_pin(pin_id: str) -> bool:
|
||||
"""Delete a pin by ID."""
|
||||
return delete_pin(pin_id)
|
||||
|
||||
|
||||
# Layer helpers for OpenClaw
|
||||
def create_intel_layer(args: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Create a layer from a command-channel args dict."""
|
||||
return create_layer(
|
||||
name=str(args.get("name", "Untitled"))[:100],
|
||||
description=str(args.get("description", ""))[:500],
|
||||
source=str(args.get("source", "openclaw"))[:50],
|
||||
color=str(args.get("color", "")),
|
||||
feed_url=str(args.get("feed_url", "")),
|
||||
feed_interval=int(args.get("feed_interval", 300)),
|
||||
)
|
||||
|
||||
|
||||
def get_intel_layers() -> list[dict[str, Any]]:
|
||||
"""Return all layers with pin counts."""
|
||||
return get_layers()
|
||||
|
||||
|
||||
def update_intel_layer(layer_id: str, args: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Update a layer from a command-channel args dict."""
|
||||
return update_layer(layer_id, **{
|
||||
k: v for k, v in args.items()
|
||||
if k in ("name", "description", "visible", "color", "feed_url", "feed_interval")
|
||||
})
|
||||
|
||||
|
||||
def delete_intel_layer(layer_id: str) -> int:
|
||||
"""Delete a layer and its pins. Returns pin count removed."""
|
||||
return delete_layer(layer_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Layer injection — inserts agent data into native telemetry layers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Layers that agents are allowed to inject into.
|
||||
_INJECTABLE_LAYERS = frozenset({
|
||||
"cctv", "ships", "sigint", "kiwisdr", "military_bases",
|
||||
"datacenters", "power_plants", "satnogs_stations",
|
||||
"volcanoes", "earthquakes", "news", "viirs_change_nodes",
|
||||
"air_quality",
|
||||
})
|
||||
|
||||
|
||||
def inject_layer_data(
|
||||
layer: str,
|
||||
items: list[dict[str, Any]],
|
||||
mode: str = "append",
|
||||
) -> dict[str, Any]:
|
||||
"""Inject agent data into a native telemetry layer."""
|
||||
from services.fetchers._store import latest_data, _data_lock, bump_data_version
|
||||
|
||||
layer = str(layer or "").strip()
|
||||
if layer not in _INJECTABLE_LAYERS:
|
||||
return {"ok": False, "detail": f"layer '{layer}' not injectable"}
|
||||
|
||||
items = list(items or [])[:200]
|
||||
if not items:
|
||||
return {"ok": False, "detail": "no items provided"}
|
||||
|
||||
now = time.time()
|
||||
tagged = []
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
entry = dict(item)
|
||||
entry["_injected"] = True
|
||||
entry["_source"] = "user:openclaw"
|
||||
entry["_injected_at"] = now
|
||||
tagged.append(entry)
|
||||
|
||||
with _data_lock:
|
||||
existing = latest_data.get(layer)
|
||||
if not isinstance(existing, list):
|
||||
existing = []
|
||||
|
||||
if mode == "replace":
|
||||
existing = [e for e in existing if not e.get("_injected")]
|
||||
|
||||
existing.extend(tagged)
|
||||
latest_data[layer] = existing
|
||||
|
||||
bump_data_version()
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"layer": layer,
|
||||
"injected": len(tagged),
|
||||
"mode": mode,
|
||||
}
|
||||
|
||||
|
||||
def clear_injected_data(layer: str = "") -> dict[str, Any]:
|
||||
"""Remove all injected items from a layer (or all layers)."""
|
||||
from services.fetchers._store import latest_data, _data_lock, bump_data_version
|
||||
|
||||
removed = 0
|
||||
with _data_lock:
|
||||
targets = [layer] if layer else list(_INJECTABLE_LAYERS)
|
||||
for lyr in targets:
|
||||
existing = latest_data.get(lyr)
|
||||
if not isinstance(existing, list):
|
||||
continue
|
||||
before = len(existing)
|
||||
latest_data[lyr] = [e for e in existing if not e.get("_injected")]
|
||||
removed += before - len(latest_data[lyr])
|
||||
|
||||
if removed:
|
||||
bump_data_version()
|
||||
|
||||
return {"ok": True, "removed": removed}
|
||||
@@ -0,0 +1,633 @@
|
||||
"""AI Intel pin storage — layered pin system with JSON file persistence.
|
||||
|
||||
Supports:
|
||||
- Named pin layers (created by user or AI)
|
||||
- Pins with optional entity attachment (track moving objects)
|
||||
- Pin source tracking (user vs openclaw)
|
||||
- Layer visibility toggles
|
||||
- External feed URL per layer (for Phase 5)
|
||||
- GeoJSON export per layer or all layers
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pin schema
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
PIN_CATEGORIES = {
|
||||
"threat", "news", "geolocation", "custom", "anomaly",
|
||||
"military", "maritime", "flight", "infrastructure", "weather",
|
||||
"sigint", "prediction", "research",
|
||||
}
|
||||
|
||||
PIN_COLORS = {
|
||||
"threat": "#ef4444", # red
|
||||
"news": "#f59e0b", # amber
|
||||
"geolocation": "#8b5cf6", # violet
|
||||
"custom": "#3b82f6", # blue
|
||||
"anomaly": "#f97316", # orange
|
||||
"military": "#dc2626", # dark red
|
||||
"maritime": "#0ea5e9", # sky
|
||||
"flight": "#6366f1", # indigo
|
||||
"infrastructure": "#64748b", # slate
|
||||
"weather": "#22d3ee", # cyan
|
||||
"sigint": "#a855f7", # purple
|
||||
"prediction": "#eab308", # yellow
|
||||
"research": "#10b981", # emerald
|
||||
}
|
||||
|
||||
LAYER_COLORS = [
|
||||
"#3b82f6", "#ef4444", "#22d3ee", "#f59e0b", "#8b5cf6",
|
||||
"#10b981", "#f97316", "#6366f1", "#ec4899", "#14b8a6",
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# In-memory store
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_layers: list[dict[str, Any]] = []
|
||||
_pins: list[dict[str, Any]] = []
|
||||
_lock = threading.Lock()
|
||||
|
||||
# Persistence file path
|
||||
_PERSIST_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data")
|
||||
_PERSIST_FILE = os.path.join(_PERSIST_DIR, "pin_layers.json")
|
||||
_OLD_PERSIST_FILE = os.path.join(_PERSIST_DIR, "ai_pins.json")
|
||||
|
||||
|
||||
def _ensure_persist_dir():
|
||||
try:
|
||||
os.makedirs(_PERSIST_DIR, exist_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _save_to_disk():
|
||||
"""Persist layers and pins to JSON file. Called under lock."""
|
||||
try:
|
||||
_ensure_persist_dir()
|
||||
with open(_PERSIST_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump({"layers": _layers, "pins": _pins}, f, indent=2, default=str)
|
||||
except (OSError, IOError) as e:
|
||||
logger.warning(f"Failed to persist pin layers: {e}")
|
||||
|
||||
|
||||
def _load_from_disk():
|
||||
"""Load layers and pins from disk on startup."""
|
||||
global _layers, _pins
|
||||
try:
|
||||
if os.path.exists(_PERSIST_FILE):
|
||||
with open(_PERSIST_FILE, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, dict):
|
||||
_layers = data.get("layers", [])
|
||||
_pins = data.get("pins", [])
|
||||
logger.info(f"Loaded {len(_layers)} layers, {len(_pins)} pins from disk")
|
||||
return
|
||||
|
||||
# Migrate from old flat pin file
|
||||
if os.path.exists(_OLD_PERSIST_FILE):
|
||||
with open(_OLD_PERSIST_FILE, "r", encoding="utf-8") as f:
|
||||
old_pins = json.load(f)
|
||||
if isinstance(old_pins, list) and old_pins:
|
||||
legacy_layer = _make_layer("Legacy", "Migrated pins", source="system")
|
||||
_layers.append(legacy_layer)
|
||||
for p in old_pins:
|
||||
if isinstance(p, dict):
|
||||
p["layer_id"] = legacy_layer["id"]
|
||||
_pins.append(p)
|
||||
logger.info(f"Migrated {len(_pins)} pins from ai_pins.json into Legacy layer")
|
||||
_save_to_disk()
|
||||
except (OSError, IOError, json.JSONDecodeError) as e:
|
||||
logger.warning(f"Failed to load pin layers from disk: {e}")
|
||||
|
||||
|
||||
def _make_layer(
|
||||
name: str,
|
||||
description: str = "",
|
||||
source: str = "user",
|
||||
color: str = "",
|
||||
feed_url: str = "",
|
||||
feed_interval: int = 300,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a layer dict."""
|
||||
layer_id = str(uuid.uuid4())[:12]
|
||||
now = time.time()
|
||||
return {
|
||||
"id": layer_id,
|
||||
"name": name[:100],
|
||||
"description": description[:500],
|
||||
"source": source[:50],
|
||||
"visible": True,
|
||||
"color": color or LAYER_COLORS[len(_layers) % len(LAYER_COLORS)],
|
||||
"created_at": now,
|
||||
"created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
|
||||
"feed_url": feed_url[:1000] if feed_url else "",
|
||||
"feed_interval": max(60, min(86400, feed_interval)),
|
||||
"pin_count": 0,
|
||||
}
|
||||
|
||||
|
||||
# Load on import
|
||||
_load_from_disk()
|
||||
|
||||
# One-time cleanup: remove correlation_engine auto-pins (no longer generated)
|
||||
_corr_before = len(_pins)
|
||||
_pins[:] = [p for p in _pins if p.get("source") != "correlation_engine"]
|
||||
if len(_pins) < _corr_before:
|
||||
logger.info("Cleaned up %d legacy correlation_engine pins", _corr_before - len(_pins))
|
||||
_save_to_disk()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Layer CRUD
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def create_layer(
|
||||
name: str,
|
||||
description: str = "",
|
||||
source: str = "user",
|
||||
color: str = "",
|
||||
feed_url: str = "",
|
||||
feed_interval: int = 300,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a new pin layer."""
|
||||
with _lock:
|
||||
layer = _make_layer(name, description, source, color, feed_url, feed_interval)
|
||||
_layers.append(layer)
|
||||
_save_to_disk()
|
||||
return layer
|
||||
|
||||
|
||||
def get_layers() -> list[dict[str, Any]]:
|
||||
"""Return all layers with current pin counts."""
|
||||
now = time.time()
|
||||
with _lock:
|
||||
result = []
|
||||
for layer in _layers:
|
||||
count = sum(
|
||||
1 for p in _pins
|
||||
if p.get("layer_id") == layer["id"]
|
||||
and not (p.get("expires_at") and p["expires_at"] < now)
|
||||
)
|
||||
result.append({**layer, "pin_count": count})
|
||||
return result
|
||||
|
||||
|
||||
def update_layer(layer_id: str, **updates) -> Optional[dict[str, Any]]:
|
||||
"""Update layer fields. Returns updated layer or None if not found."""
|
||||
allowed = {"name", "description", "visible", "color", "feed_url", "feed_interval", "feed_last_fetched"}
|
||||
with _lock:
|
||||
for layer in _layers:
|
||||
if layer["id"] == layer_id:
|
||||
for k, v in updates.items():
|
||||
if k in allowed and v is not None:
|
||||
if k == "name":
|
||||
layer[k] = str(v)[:100]
|
||||
elif k == "description":
|
||||
layer[k] = str(v)[:500]
|
||||
elif k == "visible":
|
||||
layer[k] = bool(v)
|
||||
elif k == "color":
|
||||
layer[k] = str(v)[:20]
|
||||
elif k == "feed_url":
|
||||
layer[k] = str(v)[:1000]
|
||||
elif k == "feed_interval":
|
||||
layer[k] = max(60, min(86400, int(v)))
|
||||
elif k == "feed_last_fetched":
|
||||
layer[k] = float(v)
|
||||
_save_to_disk()
|
||||
return dict(layer)
|
||||
return None
|
||||
|
||||
|
||||
def delete_layer(layer_id: str) -> int:
|
||||
"""Delete a layer and all its pins. Returns count of pins removed."""
|
||||
with _lock:
|
||||
before_layers = len(_layers)
|
||||
_layers[:] = [l for l in _layers if l["id"] != layer_id]
|
||||
if len(_layers) == before_layers:
|
||||
return 0 # not found
|
||||
before_pins = len(_pins)
|
||||
_pins[:] = [p for p in _pins if p.get("layer_id") != layer_id]
|
||||
removed = before_pins - len(_pins)
|
||||
_save_to_disk()
|
||||
return removed
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pin CRUD
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def create_pin(
|
||||
lat: float,
|
||||
lng: float,
|
||||
label: str,
|
||||
category: str = "custom",
|
||||
*,
|
||||
layer_id: str = "",
|
||||
color: str = "",
|
||||
description: str = "",
|
||||
source: str = "openclaw",
|
||||
source_url: str = "",
|
||||
confidence: float = 1.0,
|
||||
ttl_hours: float = 0,
|
||||
metadata: Optional[dict] = None,
|
||||
entity_attachment: Optional[dict] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a single pin and return it."""
|
||||
pin_id = str(uuid.uuid4())[:12]
|
||||
now = time.time()
|
||||
|
||||
cat = category if category in PIN_CATEGORIES else "custom"
|
||||
pin_color = color or PIN_COLORS.get(cat, "#3b82f6")
|
||||
|
||||
# Validate entity_attachment if provided
|
||||
attachment = None
|
||||
if entity_attachment and isinstance(entity_attachment, dict):
|
||||
etype = str(entity_attachment.get("entity_type", "")).strip()
|
||||
eid = str(entity_attachment.get("entity_id", "")).strip()
|
||||
if etype and eid:
|
||||
attachment = {
|
||||
"entity_type": etype[:50],
|
||||
"entity_id": eid[:100],
|
||||
"entity_label": str(entity_attachment.get("entity_label", ""))[:200],
|
||||
}
|
||||
|
||||
pin = {
|
||||
"id": pin_id,
|
||||
"layer_id": layer_id or "",
|
||||
"lat": lat,
|
||||
"lng": lng,
|
||||
"label": label[:200],
|
||||
"category": cat,
|
||||
"color": pin_color,
|
||||
"description": description[:2000],
|
||||
"source": source[:100],
|
||||
"source_url": source_url[:500],
|
||||
"confidence": max(0.0, min(1.0, confidence)),
|
||||
"created_at": now,
|
||||
"created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
|
||||
"expires_at": now + (ttl_hours * 3600) if ttl_hours > 0 else None,
|
||||
"metadata": metadata or {},
|
||||
"entity_attachment": attachment,
|
||||
"comments": [],
|
||||
}
|
||||
|
||||
with _lock:
|
||||
_pins.append(pin)
|
||||
_save_to_disk()
|
||||
|
||||
return pin
|
||||
|
||||
|
||||
def create_pins_batch(items: list[dict], default_layer_id: str = "") -> list[dict[str, Any]]:
|
||||
"""Create multiple pins at once."""
|
||||
created = []
|
||||
now = time.time()
|
||||
|
||||
with _lock:
|
||||
for item in items[:200]: # max 200 per batch
|
||||
pin_id = str(uuid.uuid4())[:12]
|
||||
cat = item.get("category", "custom")
|
||||
if cat not in PIN_CATEGORIES:
|
||||
cat = "custom"
|
||||
pin_color = item.get("color", "") or PIN_COLORS.get(cat, "#3b82f6")
|
||||
ttl = float(item.get("ttl_hours", 0) or 0)
|
||||
|
||||
attachment = None
|
||||
ea = item.get("entity_attachment")
|
||||
if ea and isinstance(ea, dict):
|
||||
etype = str(ea.get("entity_type", "")).strip()
|
||||
eid = str(ea.get("entity_id", "")).strip()
|
||||
if etype and eid:
|
||||
attachment = {
|
||||
"entity_type": etype[:50],
|
||||
"entity_id": eid[:100],
|
||||
"entity_label": str(ea.get("entity_label", ""))[:200],
|
||||
}
|
||||
|
||||
pin = {
|
||||
"id": pin_id,
|
||||
"layer_id": item.get("layer_id", default_layer_id) or "",
|
||||
"lat": float(item.get("lat", 0)),
|
||||
"lng": float(item.get("lng", 0)),
|
||||
"label": str(item.get("label", ""))[:200],
|
||||
"category": cat,
|
||||
"color": pin_color,
|
||||
"description": str(item.get("description", ""))[:2000],
|
||||
"source": str(item.get("source", "openclaw"))[:100],
|
||||
"source_url": str(item.get("source_url", ""))[:500],
|
||||
"confidence": max(0.0, min(1.0, float(item.get("confidence", 1.0)))),
|
||||
"created_at": now,
|
||||
"created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
|
||||
"expires_at": now + (ttl * 3600) if ttl > 0 else None,
|
||||
"metadata": item.get("metadata", {}),
|
||||
"entity_attachment": attachment,
|
||||
"comments": [],
|
||||
}
|
||||
_pins.append(pin)
|
||||
created.append(pin)
|
||||
|
||||
_save_to_disk()
|
||||
return created
|
||||
|
||||
|
||||
def get_pins(
|
||||
category: str = "",
|
||||
source: str = "",
|
||||
layer_id: str = "",
|
||||
limit: int = 500,
|
||||
include_expired: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Get pins with optional filters."""
|
||||
now = time.time()
|
||||
with _lock:
|
||||
results = []
|
||||
for pin in _pins:
|
||||
if not include_expired and pin.get("expires_at") and pin["expires_at"] < now:
|
||||
continue
|
||||
if category and pin.get("category") != category:
|
||||
continue
|
||||
if source and pin.get("source") != source:
|
||||
continue
|
||||
if layer_id and pin.get("layer_id") != layer_id:
|
||||
continue
|
||||
results.append(pin)
|
||||
if len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
def get_pin(pin_id: str) -> Optional[dict[str, Any]]:
|
||||
"""Return a single pin by ID (including comments), or None."""
|
||||
with _lock:
|
||||
for pin in _pins:
|
||||
if pin.get("id") == pin_id:
|
||||
# Ensure comments key exists for legacy pins
|
||||
if "comments" not in pin:
|
||||
pin["comments"] = []
|
||||
return dict(pin)
|
||||
return None
|
||||
|
||||
|
||||
def update_pin(pin_id: str, **updates) -> Optional[dict[str, Any]]:
|
||||
"""Update a pin's editable fields (label, description, category, color)."""
|
||||
allowed = {"label", "description", "category", "color"}
|
||||
with _lock:
|
||||
for pin in _pins:
|
||||
if pin.get("id") != pin_id:
|
||||
continue
|
||||
for k, v in updates.items():
|
||||
if k not in allowed or v is None:
|
||||
continue
|
||||
if k == "label":
|
||||
pin[k] = str(v)[:200]
|
||||
elif k == "description":
|
||||
pin[k] = str(v)[:2000]
|
||||
elif k == "category":
|
||||
cat = str(v)
|
||||
if cat in PIN_CATEGORIES:
|
||||
pin[k] = cat
|
||||
# Refresh color if it was the category default
|
||||
if not updates.get("color"):
|
||||
pin["color"] = PIN_COLORS.get(cat, pin.get("color", "#3b82f6"))
|
||||
elif k == "color":
|
||||
pin[k] = str(v)[:20]
|
||||
pin["updated_at"] = time.time()
|
||||
_save_to_disk()
|
||||
return dict(pin)
|
||||
return None
|
||||
|
||||
|
||||
def add_pin_comment(
|
||||
pin_id: str,
|
||||
text: str,
|
||||
author: str = "user",
|
||||
author_label: str = "",
|
||||
reply_to: str = "",
|
||||
) -> Optional[dict[str, Any]]:
|
||||
"""Append a comment to a pin. Returns the updated pin (with all comments)."""
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
with _lock:
|
||||
for pin in _pins:
|
||||
if pin.get("id") != pin_id:
|
||||
continue
|
||||
if "comments" not in pin or not isinstance(pin["comments"], list):
|
||||
pin["comments"] = []
|
||||
comment = {
|
||||
"id": str(uuid.uuid4())[:12],
|
||||
"text": text[:4000],
|
||||
"author": (author or "user")[:50],
|
||||
"author_label": (author_label or "")[:100],
|
||||
"reply_to": (reply_to or "")[:12],
|
||||
"created_at": time.time(),
|
||||
"created_at_iso": datetime.utcnow().isoformat() + "Z",
|
||||
}
|
||||
pin["comments"].append(comment)
|
||||
_save_to_disk()
|
||||
return dict(pin)
|
||||
return None
|
||||
|
||||
|
||||
def delete_pin_comment(pin_id: str, comment_id: str) -> bool:
|
||||
"""Remove a single comment from a pin."""
|
||||
with _lock:
|
||||
for pin in _pins:
|
||||
if pin.get("id") != pin_id:
|
||||
continue
|
||||
comments = pin.get("comments") or []
|
||||
before = len(comments)
|
||||
pin["comments"] = [c for c in comments if c.get("id") != comment_id]
|
||||
if len(pin["comments"]) < before:
|
||||
_save_to_disk()
|
||||
return True
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def delete_pin(pin_id: str) -> bool:
|
||||
"""Delete a single pin by ID."""
|
||||
with _lock:
|
||||
before = len(_pins)
|
||||
_pins[:] = [p for p in _pins if p.get("id") != pin_id]
|
||||
if len(_pins) < before:
|
||||
_save_to_disk()
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def clear_pins(category: str = "", source: str = "", layer_id: str = "") -> int:
|
||||
"""Clear pins, optionally filtered. Returns count removed."""
|
||||
with _lock:
|
||||
before = len(_pins)
|
||||
|
||||
def keep(p):
|
||||
if layer_id and p.get("layer_id") != layer_id:
|
||||
return True # different layer, keep
|
||||
if category and source:
|
||||
return not (p.get("category") == category and p.get("source") == source)
|
||||
if category:
|
||||
return p.get("category") != category
|
||||
if source:
|
||||
return p.get("source") != source
|
||||
if layer_id:
|
||||
return p.get("layer_id") != layer_id
|
||||
return False
|
||||
|
||||
if not category and not source and not layer_id:
|
||||
_pins.clear()
|
||||
else:
|
||||
_pins[:] = [p for p in _pins if keep(p)]
|
||||
|
||||
removed = before - len(_pins)
|
||||
if removed:
|
||||
_save_to_disk()
|
||||
return removed
|
||||
|
||||
|
||||
def get_feed_layers() -> list[dict[str, Any]]:
|
||||
"""Return layers that have a non-empty feed_url."""
|
||||
with _lock:
|
||||
return [dict(l) for l in _layers if l.get("feed_url")]
|
||||
|
||||
|
||||
def replace_layer_pins(layer_id: str, new_pins: list[dict[str, Any]]) -> int:
|
||||
"""Atomically replace all pins in a layer with new_pins. Returns count added."""
|
||||
now = time.time()
|
||||
with _lock:
|
||||
# Remove old pins for this layer
|
||||
_pins[:] = [p for p in _pins if p.get("layer_id") != layer_id]
|
||||
# Add new pins
|
||||
added = 0
|
||||
for item in new_pins[:500]: # cap at 500 per feed
|
||||
pin_id = str(uuid.uuid4())[:12]
|
||||
cat = item.get("category", "custom")
|
||||
if cat not in PIN_CATEGORIES:
|
||||
cat = "custom"
|
||||
pin_color = item.get("color", "") or PIN_COLORS.get(cat, "#3b82f6")
|
||||
|
||||
attachment = None
|
||||
ea = item.get("entity_attachment")
|
||||
if ea and isinstance(ea, dict):
|
||||
etype = str(ea.get("entity_type", "")).strip()
|
||||
eid = str(ea.get("entity_id", "")).strip()
|
||||
if etype and eid:
|
||||
attachment = {
|
||||
"entity_type": etype[:50],
|
||||
"entity_id": eid[:100],
|
||||
"entity_label": str(ea.get("entity_label", ""))[:200],
|
||||
}
|
||||
|
||||
pin = {
|
||||
"id": pin_id,
|
||||
"layer_id": layer_id,
|
||||
"lat": float(item.get("lat", 0)),
|
||||
"lng": float(item.get("lng", 0)),
|
||||
"label": str(item.get("label", item.get("name", "")))[:200],
|
||||
"category": cat,
|
||||
"color": pin_color,
|
||||
"description": str(item.get("description", ""))[:2000],
|
||||
"source": str(item.get("source", "feed"))[:100],
|
||||
"source_url": str(item.get("source_url", ""))[:500],
|
||||
"confidence": max(0.0, min(1.0, float(item.get("confidence", 1.0)))),
|
||||
"created_at": now,
|
||||
"created_at_iso": datetime.utcfromtimestamp(now).isoformat() + "Z",
|
||||
"expires_at": None,
|
||||
"metadata": item.get("metadata", {}),
|
||||
"entity_attachment": attachment,
|
||||
"comments": [],
|
||||
}
|
||||
_pins.append(pin)
|
||||
added += 1
|
||||
_save_to_disk()
|
||||
return added
|
||||
|
||||
|
||||
def purge_expired() -> int:
|
||||
"""Remove expired pins. Called periodically."""
|
||||
now = time.time()
|
||||
with _lock:
|
||||
before = len(_pins)
|
||||
_pins[:] = [p for p in _pins if not (p.get("expires_at") and p["expires_at"] < now)]
|
||||
removed = before - len(_pins)
|
||||
if removed:
|
||||
_save_to_disk()
|
||||
return removed
|
||||
|
||||
|
||||
def pin_count() -> dict[str, int]:
|
||||
"""Return counts by category."""
|
||||
now = time.time()
|
||||
counts: dict[str, int] = {}
|
||||
with _lock:
|
||||
for pin in _pins:
|
||||
if pin.get("expires_at") and pin["expires_at"] < now:
|
||||
continue
|
||||
cat = pin.get("category", "custom")
|
||||
counts[cat] = counts.get(cat, 0) + 1
|
||||
return counts
|
||||
|
||||
|
||||
def pins_as_geojson(layer_id: str = "") -> dict[str, Any]:
|
||||
"""Convert active pins to GeoJSON FeatureCollection for the map layer."""
|
||||
now = time.time()
|
||||
features = []
|
||||
with _lock:
|
||||
# Build set of visible layer IDs
|
||||
visible_layers = {l["id"] for l in _layers if l.get("visible", True)}
|
||||
|
||||
for pin in _pins:
|
||||
if pin.get("expires_at") and pin["expires_at"] < now:
|
||||
continue
|
||||
# Layer filter
|
||||
pid_layer = pin.get("layer_id", "")
|
||||
if layer_id and pid_layer != layer_id:
|
||||
continue
|
||||
# Skip pins in hidden layers
|
||||
if pid_layer and pid_layer not in visible_layers:
|
||||
continue
|
||||
|
||||
props = {
|
||||
"id": pin["id"],
|
||||
"layer_id": pid_layer,
|
||||
"label": pin["label"],
|
||||
"category": pin["category"],
|
||||
"color": pin["color"],
|
||||
"description": pin.get("description", ""),
|
||||
"source": pin["source"],
|
||||
"source_url": pin.get("source_url", ""),
|
||||
"confidence": pin.get("confidence", 1.0),
|
||||
"created_at": pin.get("created_at_iso", ""),
|
||||
"comment_count": len(pin.get("comments") or []),
|
||||
}
|
||||
|
||||
# Entity attachment info (frontend resolves position)
|
||||
ea = pin.get("entity_attachment")
|
||||
if ea:
|
||||
props["entity_attachment"] = ea
|
||||
|
||||
features.append({
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [pin["lng"], pin["lat"]],
|
||||
},
|
||||
"properties": props,
|
||||
})
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"features": features,
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
"""Analysis Zone store — OpenClaw-placed map overlays with analyst notes.
|
||||
|
||||
These render as the dashed-border squares on the correlations layer.
|
||||
Unlike automated correlations (which are recomputed every cycle), analysis
|
||||
zones persist until the agent or user deletes them, or their TTL expires.
|
||||
|
||||
Shape matches the correlation alert schema so the frontend renders them
|
||||
identically — the ``source`` field marks them as agent-placed and enables
|
||||
the delete button in the popup.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_zones: list[dict[str, Any]] = []
|
||||
_lock = threading.Lock()
|
||||
|
||||
_PERSIST_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data")
|
||||
_PERSIST_FILE = os.path.join(_PERSIST_DIR, "analysis_zones.json")
|
||||
|
||||
ZONE_CATEGORIES = {
|
||||
"contradiction", # narrative vs telemetry mismatch
|
||||
"analysis", # general analyst note / assessment
|
||||
"warning", # potential threat or risk area
|
||||
"observation", # neutral observation worth marking
|
||||
"hypothesis", # unverified theory to investigate
|
||||
}
|
||||
|
||||
# Map categories to correlation type colors on the frontend
|
||||
CATEGORY_COLORS = {
|
||||
"contradiction": "amber",
|
||||
"analysis": "cyan",
|
||||
"warning": "red",
|
||||
"observation": "blue",
|
||||
"hypothesis": "purple",
|
||||
}
|
||||
|
||||
|
||||
def _ensure_dir():
|
||||
try:
|
||||
os.makedirs(_PERSIST_DIR, exist_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _save():
|
||||
"""Persist to disk. Called under lock."""
|
||||
try:
|
||||
_ensure_dir()
|
||||
with open(_PERSIST_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(_zones, f, indent=2, default=str)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to save analysis zones: %s", e)
|
||||
|
||||
|
||||
def _load():
|
||||
"""Load from disk on startup."""
|
||||
global _zones
|
||||
try:
|
||||
if os.path.exists(_PERSIST_FILE):
|
||||
with open(_PERSIST_FILE, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, list):
|
||||
_zones = data
|
||||
logger.info("Loaded %d analysis zones from disk", len(_zones))
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load analysis zones: %s", e)
|
||||
|
||||
|
||||
# Load on import
|
||||
_load()
|
||||
|
||||
|
||||
def _expire():
|
||||
"""Remove zones past their TTL. Called under lock."""
|
||||
now = time.time()
|
||||
before = len(_zones)
|
||||
_zones[:] = [
|
||||
z for z in _zones
|
||||
if z.get("ttl_hours", 0) <= 0
|
||||
or (now - z.get("created_at", now)) < z["ttl_hours"] * 3600
|
||||
]
|
||||
removed = before - len(_zones)
|
||||
if removed:
|
||||
logger.info("Expired %d analysis zones", removed)
|
||||
|
||||
|
||||
def create_zone(
|
||||
*,
|
||||
lat: float,
|
||||
lng: float,
|
||||
title: str,
|
||||
body: str,
|
||||
category: str = "analysis",
|
||||
severity: str = "medium",
|
||||
cell_size_deg: float = 1.0,
|
||||
ttl_hours: float = 0,
|
||||
source: str = "openclaw",
|
||||
drivers: list[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Create an analysis zone. Returns the created zone dict."""
|
||||
category = category if category in ZONE_CATEGORIES else "analysis"
|
||||
if severity not in ("high", "medium", "low"):
|
||||
severity = "medium"
|
||||
cell_size_deg = max(0.1, min(cell_size_deg, 10.0))
|
||||
|
||||
zone: dict[str, Any] = {
|
||||
"id": str(uuid.uuid4())[:12],
|
||||
"lat": lat,
|
||||
"lng": lng,
|
||||
"type": "analysis_zone",
|
||||
"category": category,
|
||||
"severity": severity,
|
||||
"score": {"high": 90, "medium": 60, "low": 30}.get(severity, 60),
|
||||
"title": title[:200],
|
||||
"body": body[:2000],
|
||||
"drivers": (drivers or [title])[:5],
|
||||
"cell_size": cell_size_deg,
|
||||
"source": source,
|
||||
"created_at": time.time(),
|
||||
"ttl_hours": ttl_hours,
|
||||
}
|
||||
|
||||
with _lock:
|
||||
_expire()
|
||||
_zones.append(zone)
|
||||
_save()
|
||||
|
||||
logger.info("Analysis zone created: %s at (%.2f, %.2f)", title[:40], lat, lng)
|
||||
return zone
|
||||
|
||||
|
||||
def list_zones() -> list[dict[str, Any]]:
|
||||
"""Return all live (non-expired) zones."""
|
||||
with _lock:
|
||||
_expire()
|
||||
return list(_zones)
|
||||
|
||||
|
||||
def get_zone(zone_id: str) -> dict[str, Any] | None:
|
||||
"""Get a single zone by ID."""
|
||||
with _lock:
|
||||
for z in _zones:
|
||||
if z["id"] == zone_id:
|
||||
return dict(z)
|
||||
return None
|
||||
|
||||
|
||||
def delete_zone(zone_id: str) -> bool:
|
||||
"""Delete a zone by ID. Returns True if found and removed."""
|
||||
with _lock:
|
||||
before = len(_zones)
|
||||
_zones[:] = [z for z in _zones if z["id"] != zone_id]
|
||||
if len(_zones) < before:
|
||||
_save()
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def clear_zones(*, source: str | None = None) -> int:
|
||||
"""Clear all zones, optionally filtered by source. Returns count removed."""
|
||||
with _lock:
|
||||
before = len(_zones)
|
||||
if source:
|
||||
_zones[:] = [z for z in _zones if z.get("source") != source]
|
||||
else:
|
||||
_zones.clear()
|
||||
removed = before - len(_zones)
|
||||
if removed:
|
||||
_save()
|
||||
return removed
|
||||
|
||||
|
||||
def get_live_zones() -> list[dict[str, Any]]:
|
||||
"""Return zones formatted for the correlation engine merge.
|
||||
|
||||
This is called by compute_correlations() to inject agent-placed zones
|
||||
into the correlations list that the frontend renders as map squares.
|
||||
"""
|
||||
with _lock:
|
||||
_expire()
|
||||
return [dict(z) for z in _zones]
|
||||
@@ -4,11 +4,12 @@ Keys are stored in the backend .env file and loaded via python-dotenv.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
# Path to the backend .env file
|
||||
ENV_PATH = Path(__file__).parent.parent / ".env"
|
||||
# Path to the example template that ships with the repo
|
||||
ENV_EXAMPLE_PATH = Path(__file__).parent.parent.parent / ".env.example"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API Registry — every external service the dashboard depends on
|
||||
@@ -143,15 +144,33 @@ API_REGISTRY = [
|
||||
]
|
||||
|
||||
|
||||
def _obfuscate(value: str) -> str:
|
||||
"""Show first 4 chars, mask the rest with bullets."""
|
||||
if not value or len(value) <= 4:
|
||||
return "••••••••"
|
||||
return value[:4] + "•" * (len(value) - 4)
|
||||
def get_env_path_info() -> dict:
|
||||
"""Return absolute paths for the backend .env and .env.example template.
|
||||
|
||||
Surfaced to the frontend so the API Keys settings panel can tell users
|
||||
exactly where to put their keys when in-app editing fails (admin-not-set,
|
||||
file permissions, read-only filesystem, etc.).
|
||||
"""
|
||||
env_path = ENV_PATH.resolve()
|
||||
example_path = ENV_EXAMPLE_PATH.resolve()
|
||||
return {
|
||||
"env_path": str(env_path),
|
||||
"env_path_exists": env_path.exists(),
|
||||
"env_path_writable": os.access(env_path.parent, os.W_OK)
|
||||
and (not env_path.exists() or os.access(env_path, os.W_OK)),
|
||||
"env_example_path": str(example_path),
|
||||
"env_example_path_exists": example_path.exists(),
|
||||
}
|
||||
|
||||
|
||||
def get_api_keys():
|
||||
"""Return the full API registry with obfuscated key values."""
|
||||
"""Return the API registry with a binary set/unset flag per key.
|
||||
|
||||
Key values themselves are NEVER returned to the client — not even an
|
||||
obfuscated prefix. Users edit the .env file directly; the panel uses
|
||||
`is_set` to render a CONFIGURED / NOT CONFIGURED badge and the path
|
||||
info from `get_env_path_info()` to tell them where to put each key.
|
||||
"""
|
||||
result = []
|
||||
for api in API_REGISTRY:
|
||||
entry = {
|
||||
@@ -163,41 +182,10 @@ def get_api_keys():
|
||||
"required": api["required"],
|
||||
"has_key": api["env_key"] is not None,
|
||||
"env_key": api["env_key"],
|
||||
"value_obfuscated": None,
|
||||
"is_set": False,
|
||||
}
|
||||
if api["env_key"]:
|
||||
raw = os.environ.get(api["env_key"], "")
|
||||
entry["value_obfuscated"] = _obfuscate(raw)
|
||||
entry["is_set"] = bool(raw)
|
||||
result.append(entry)
|
||||
return result
|
||||
|
||||
|
||||
def update_api_key(env_key: str, new_value: str) -> bool:
|
||||
"""Update a single key in the .env file and in the current process env."""
|
||||
valid_keys = {api["env_key"] for api in API_REGISTRY if api.get("env_key")}
|
||||
if env_key not in valid_keys:
|
||||
return False
|
||||
|
||||
if not isinstance(new_value, str):
|
||||
return False
|
||||
if "\n" in new_value or "\r" in new_value:
|
||||
return False
|
||||
|
||||
if not ENV_PATH.exists():
|
||||
ENV_PATH.write_text("", encoding="utf-8")
|
||||
|
||||
# Update os.environ immediately
|
||||
os.environ[env_key] = new_value
|
||||
|
||||
# Update the .env file on disk
|
||||
content = ENV_PATH.read_text(encoding="utf-8")
|
||||
pattern = re.compile(rf"^{re.escape(env_key)}=.*$", re.MULTILINE)
|
||||
if pattern.search(content):
|
||||
content = pattern.sub(f"{env_key}={new_value}", content)
|
||||
else:
|
||||
content = content.rstrip("\n") + f"\n{env_key}={new_value}\n"
|
||||
|
||||
ENV_PATH.write_text(content, encoding="utf-8")
|
||||
return True
|
||||
|
||||
@@ -818,6 +818,105 @@ out body;
|
||||
return cameras
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ALPR / Surveillance Camera Locations (OSM Overpass)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Queries OpenStreetMap for ALPR/LPR tagged surveillance cameras.
|
||||
# These cameras rarely have public media URLs — this ingestor captures
|
||||
# their LOCATIONS for situational awareness (density heatmap, blind-spot
|
||||
# analysis). No plate-read data is fetched — only publicly-mapped positions.
|
||||
|
||||
|
||||
class OSMALPRCameraIngestor(BaseCCTVIngestor):
|
||||
"""ALPR / license-plate reader camera locations from OpenStreetMap.
|
||||
|
||||
Searches for nodes tagged with surveillance:type=ALPR or
|
||||
man_made=surveillance + camera:type values indicating plate readers.
|
||||
Only geolocations are ingested — no live feeds or detection data.
|
||||
"""
|
||||
|
||||
URL = "https://overpass-api.de/api/interpreter"
|
||||
QUERY = """
|
||||
[out:json][timeout:45];
|
||||
(
|
||||
node["surveillance:type"="ALPR"];
|
||||
node["surveillance:type"="alpr"];
|
||||
node["surveillance:type"="LPR"];
|
||||
node["surveillance:type"="lpr"];
|
||||
node["man_made"="surveillance"]["camera:type"="ALPR"];
|
||||
node["man_made"="surveillance"]["camera:type"="alpr"];
|
||||
node["man_made"="surveillance"]["camera:type"="LPR"];
|
||||
node["man_made"="surveillance"]["camera:type"="lpr"];
|
||||
node["man_made"="surveillance"]["description"~"[Ll]icense [Pp]late"];
|
||||
node["man_made"="surveillance"]["description"~"ALPR"];
|
||||
node["man_made"="surveillance"]["description"~"Flock"];
|
||||
);
|
||||
out body;
|
||||
""".strip()
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
query = quote(self.QUERY, safe="")
|
||||
resp = fetch_with_curl(
|
||||
f"{self.URL}?data={query}",
|
||||
timeout=50,
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
if not resp or resp.status_code != 200:
|
||||
logger.warning(
|
||||
"OSM ALPR camera fetch failed: HTTP %s",
|
||||
resp.status_code if resp else "no response",
|
||||
)
|
||||
return []
|
||||
data = resp.json()
|
||||
cameras = []
|
||||
for item in data.get("elements", []) if isinstance(data, dict) else []:
|
||||
lat = item.get("lat")
|
||||
lon = item.get("lon")
|
||||
if lat is None or lon is None:
|
||||
continue
|
||||
try:
|
||||
lat, lon = float(lat), float(lon)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
tags = item.get("tags", {}) if isinstance(item.get("tags"), dict) else {}
|
||||
|
||||
# Extract what we can from tags
|
||||
operator = (
|
||||
tags.get("operator")
|
||||
or tags.get("brand")
|
||||
or tags.get("network")
|
||||
or "Unknown"
|
||||
)
|
||||
description = (
|
||||
tags.get("description")
|
||||
or tags.get("name")
|
||||
or tags.get("surveillance:type", "ALPR")
|
||||
)
|
||||
direction = (
|
||||
tags.get("camera:direction")
|
||||
or tags.get("direction")
|
||||
or tags.get("surveillance:direction")
|
||||
or "Unknown"
|
||||
)
|
||||
|
||||
# ALPR cameras typically have no public media URL — use a
|
||||
# placeholder so the pin renders but no proxy attempt is made.
|
||||
cameras.append(
|
||||
{
|
||||
"id": f"ALPR-{item.get('id')}",
|
||||
"source_agency": str(operator)[:60],
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"direction_facing": f"ALPR: {str(description)[:100]} ({str(direction)[:30]})",
|
||||
"media_url": "",
|
||||
"media_type": "none",
|
||||
"refresh_rate_seconds": 0,
|
||||
}
|
||||
)
|
||||
logger.info("OSM ALPR ingestor found %d cameras", len(cameras))
|
||||
return cameras
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DGT Spain — National Road Cameras
|
||||
|
||||
+260
-7
@@ -10,6 +10,10 @@ class Settings(BaseSettings):
|
||||
ALLOW_INSECURE_ADMIN: bool = False
|
||||
PUBLIC_API_KEY: str = ""
|
||||
|
||||
# OpenClaw agent connectivity
|
||||
OPENCLAW_HMAC_SECRET: str = "" # HMAC shared secret for direct mode (auto-generated if empty)
|
||||
OPENCLAW_ACCESS_TIER: str = "restricted" # "full" or "restricted"
|
||||
|
||||
# Data sources
|
||||
AIS_API_KEY: str = ""
|
||||
OPENSKY_CLIENT_ID: str = ""
|
||||
@@ -27,7 +31,8 @@ class Settings(BaseSettings):
|
||||
MESH_RNS_ENABLED: bool = False
|
||||
MESH_ARTI_ENABLED: bool = False
|
||||
MESH_ARTI_SOCKS_PORT: int = 9050
|
||||
MESH_RELAY_PEERS: str = "http://cipher0.shadowbroker.info:8000"
|
||||
MESH_RELAY_PEERS: str = ""
|
||||
MESH_DEFAULT_SYNC_PEERS: str = "https://node.shadowbroker.info"
|
||||
MESH_BOOTSTRAP_DISABLED: bool = False
|
||||
MESH_BOOTSTRAP_MANIFEST_PATH: str = "data/bootstrap_peers.json"
|
||||
MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY: str = ""
|
||||
@@ -37,7 +42,7 @@ class Settings(BaseSettings):
|
||||
MESH_RELAY_PUSH_TIMEOUT_S: int = 10
|
||||
MESH_RELAY_MAX_FAILURES: int = 3
|
||||
MESH_RELAY_FAILURE_COOLDOWN_S: int = 120
|
||||
MESH_PEER_PUSH_SECRET: str = "Mv63UvLfwqOEVWeRBXjA8MtFl2nEkkhUlLYVHiX1Zzo"
|
||||
MESH_PEER_PUSH_SECRET: str = ""
|
||||
MESH_RNS_APP_NAME: str = "shadowbroker"
|
||||
MESH_RNS_ASPECT: str = "infonet"
|
||||
MESH_RNS_IDENTITY_PATH: str = ""
|
||||
@@ -60,7 +65,8 @@ class Settings(BaseSettings):
|
||||
# Keep a low background cadence on private RNS links so quiet nodes are less
|
||||
# trivially fingerprintable by silence alone. Set to 0 to disable explicitly.
|
||||
MESH_RNS_COVER_INTERVAL_S: int = 30
|
||||
MESH_RNS_COVER_SIZE: int = 64
|
||||
MESH_RNS_COVER_SIZE: int = 512
|
||||
MESH_DM_MAILBOX_TTL_S: int = 900
|
||||
MESH_RNS_IBF_WINDOW: int = 256
|
||||
MESH_RNS_IBF_TABLE_SIZE: int = 64
|
||||
MESH_RNS_IBF_MINHASH_SIZE: int = 16
|
||||
@@ -75,44 +81,221 @@ class Settings(BaseSettings):
|
||||
MESH_RNS_IBF_FAIL_THRESHOLD: int = 3
|
||||
MESH_RNS_IBF_COOLDOWN_S: int = 120
|
||||
MESH_VERIFY_INTERVAL_S: int = 600
|
||||
MESH_VERIFY_SIGNATURES: bool = True
|
||||
# MESH_VERIFY_SIGNATURES is intentionally removed — the audit loop in main.py
|
||||
# always calls validate_chain_incremental(verify_signatures=True). Any value
|
||||
# set in the environment is ignored.
|
||||
MESH_DM_SECURE_MODE: bool = True
|
||||
MESH_DM_TOKEN_PEPPER: str = ""
|
||||
MESH_DM_ALLOW_LEGACY_GET: bool = False
|
||||
MESH_ALLOW_LEGACY_DM1_UNTIL: str = ""
|
||||
MESH_ALLOW_LEGACY_DM_GET_UNTIL: str = ""
|
||||
MESH_ALLOW_LEGACY_DM_SIGNATURE_COMPAT_UNTIL: str = ""
|
||||
MESH_DM_PERSIST_SPOOL: bool = False
|
||||
MESH_DM_RELAY_FILE_PATH: str = ""
|
||||
MESH_DM_RELAY_AUTO_RELOAD: bool = False
|
||||
MESH_DM_REQUIRE_SENDER_SEAL_SHARED: bool = True
|
||||
MESH_DM_NONCE_TTL_S: int = 300
|
||||
MESH_DM_NONCE_CACHE_MAX: int = 4096
|
||||
MESH_DM_NONCE_PER_AGENT_MAX: int = 256
|
||||
MESH_DM_REQUEST_MAX_AGE_S: int = 300
|
||||
MESH_DM_REQUEST_MAILBOX_LIMIT: int = 12
|
||||
MESH_DM_SHARED_MAILBOX_LIMIT: int = 48
|
||||
MESH_DM_SELF_MAILBOX_LIMIT: int = 12
|
||||
MESH_BLOCK_LEGACY_AGENT_ID_LOOKUP: bool = True
|
||||
MESH_ALLOW_COMPAT_DM_INVITE_IMPORT: bool = False
|
||||
MESH_ALLOW_COMPAT_DM_INVITE_IMPORT_UNTIL: str = ""
|
||||
MESH_ALLOW_LEGACY_NODE_ID_COMPAT_UNTIL: str = ""
|
||||
# Rotate voter-blinding salts on a rolling cadence so new reputation
|
||||
# events do not reuse one forever-stable blinded identity.
|
||||
MESH_VOTER_BLIND_SALT_ROTATE_DAYS: int = 30
|
||||
# Keep historical salts long enough to cover live vote records, so
|
||||
# duplicate-vote detection and wallet-cost accounting survive rotation.
|
||||
MESH_VOTER_BLIND_SALT_GRACE_DAYS: int = 30
|
||||
MESH_DM_MAX_MSG_BYTES: int = 8192
|
||||
MESH_DM_ALLOW_SENDER_SEAL: bool = False
|
||||
# TTL for DH key and prekey bundle registrations — stale entries are pruned.
|
||||
MESH_DM_KEY_TTL_DAYS: int = 30
|
||||
# TTL for invite-scoped prekey lookup aliases; shorter windows reduce
|
||||
# long-lived relay linkage between opaque lookup handles and agent IDs.
|
||||
MESH_DM_PREKEY_LOOKUP_ALIAS_TTL_DAYS: int = 14
|
||||
# TTL for relay witness history; keep continuity metadata bounded instead
|
||||
# of relying on a hidden hardcoded retention window.
|
||||
MESH_DM_WITNESS_TTL_DAYS: int = 14
|
||||
# TTL for mailbox binding metadata — shorter = smaller metadata footprint on disk.
|
||||
MESH_DM_BINDING_TTL_DAYS: int = 7
|
||||
MESH_DM_BINDING_TTL_DAYS: int = 3
|
||||
# When False, mailbox bindings are memory-only (agents re-register on restart).
|
||||
MESH_DM_METADATA_PERSIST: bool = True
|
||||
# Enable explicitly only if restart continuity is worth persisting DM graph metadata.
|
||||
MESH_DM_METADATA_PERSIST: bool = False
|
||||
# Second explicit opt-in for at-rest DM metadata persistence. This keeps a
|
||||
# single boolean flip from silently writing mailbox graph metadata to disk.
|
||||
MESH_DM_METADATA_PERSIST_ACKNOWLEDGE: bool = False
|
||||
# Optional import path for externally managed root witness material packages.
|
||||
# Relative paths resolve from the backend directory.
|
||||
MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_PATH: str = ""
|
||||
# Optional URI for externally managed root witness material packages.
|
||||
# Supports file:// and http(s):// sources; when set it overrides the local path.
|
||||
MESH_DM_ROOT_EXTERNAL_WITNESS_IMPORT_URI: str = ""
|
||||
# Maximum acceptable age for externally sourced root witness packages.
|
||||
# Strong DM trust fails closed when the imported package exported_at is older than this.
|
||||
MESH_DM_ROOT_EXTERNAL_WITNESS_MAX_AGE_S: int = 3600
|
||||
# Warning threshold for externally sourced root witness packages.
|
||||
# When current external witness material reaches this age, operator health degrades to warning
|
||||
# before the strong path eventually fails closed at MAX_AGE.
|
||||
MESH_DM_ROOT_EXTERNAL_WITNESS_WARN_AGE_S: int = 2700
|
||||
# Optional export path for the append-only stable-root transparency ledger.
|
||||
# Relative paths resolve from the backend directory.
|
||||
MESH_DM_ROOT_TRANSPARENCY_LEDGER_EXPORT_PATH: str = ""
|
||||
# Optional URI used to read back and verify published transparency ledgers.
|
||||
# Supports file:// and http(s):// sources.
|
||||
MESH_DM_ROOT_TRANSPARENCY_LEDGER_READBACK_URI: str = ""
|
||||
# Maximum acceptable age for externally read transparency ledgers.
|
||||
# Strong DM trust fails closed when exported_at is older than this.
|
||||
MESH_DM_ROOT_TRANSPARENCY_LEDGER_MAX_AGE_S: int = 3600
|
||||
# Warning threshold for externally read transparency ledgers.
|
||||
# When current external transparency readback reaches this age, operator health degrades to warning
|
||||
# before the strong path eventually fails closed at MAX_AGE.
|
||||
MESH_DM_ROOT_TRANSPARENCY_LEDGER_WARN_AGE_S: int = 2700
|
||||
MESH_SCOPED_TOKENS: str = ""
|
||||
# Deprecated legacy env vars kept for backward config compatibility only.
|
||||
# Ordinary shipped gate flows keep MLS decrypt local; backend decrypt is
|
||||
# reserved for explicit recovery reads.
|
||||
MESH_GATE_BACKEND_DECRYPT_COMPAT: bool = False
|
||||
MESH_GATE_BACKEND_DECRYPT_COMPAT_ACKNOWLEDGE: bool = False
|
||||
MESH_BACKEND_GATE_DECRYPT_COMPAT: bool = False
|
||||
# Deprecated legacy env vars kept for backward config compatibility only.
|
||||
# Ordinary shipped gate flows keep compose/post local and submit encrypted
|
||||
# payloads to the backend for sign/post only.
|
||||
MESH_GATE_BACKEND_PLAINTEXT_COMPAT: bool = False
|
||||
MESH_GATE_BACKEND_PLAINTEXT_COMPAT_ACKNOWLEDGE: bool = False
|
||||
MESH_BACKEND_GATE_PLAINTEXT_COMPAT: bool = False
|
||||
# Runtime gate for recovery envelopes. When off, per-gate
|
||||
# envelope_recovery / envelope_always policies fail closed to
|
||||
# envelope_disabled. Default True so the Reddit-like durable history
|
||||
# model works out of the box: any member with the gate_secret can
|
||||
# decrypt every envelope encrypted from the moment they had that key.
|
||||
# Set MESH_GATE_RECOVERY_ENVELOPE_ENABLE=false to revert to MLS-only
|
||||
# forward-secret behavior (your own history becomes unreadable after
|
||||
# the sending ratchet advances).
|
||||
MESH_GATE_RECOVERY_ENVELOPE_ENABLE: bool = True
|
||||
MESH_GATE_RECOVERY_ENVELOPE_ENABLE_ACKNOWLEDGE: bool = True
|
||||
# Durable gate plaintext retention is disabled by default. Enable only
|
||||
# when the operator explicitly accepts the at-rest privacy tradeoff.
|
||||
MESH_GATE_PLAINTEXT_PERSIST: bool = False
|
||||
MESH_GATE_PLAINTEXT_PERSIST_ACKNOWLEDGE: bool = False
|
||||
MESH_GATE_SESSION_ROTATE_MSGS: int = 50
|
||||
MESH_GATE_SESSION_ROTATE_S: int = 3600
|
||||
MESH_GATE_LEGACY_ENVELOPE_FALLBACK_MAX_DAYS: int = 30
|
||||
# Add a randomized grace window before anonymous gate-session auto-rotation
|
||||
# so threshold-triggered identity swaps are less trivially correlated.
|
||||
MESH_GATE_SESSION_ROTATE_JITTER_S: int = 180
|
||||
# Gate persona (named identity) rotation thresholds. Rotating the signing
|
||||
# key limits the linkability window. Zero = disabled.
|
||||
MESH_GATE_PERSONA_ROTATE_MSGS: int = 200
|
||||
MESH_GATE_PERSONA_ROTATE_S: int = 604800 # 7 days
|
||||
MESH_GATE_PERSONA_ROTATE_JITTER_S: int = 600
|
||||
# Feature-flagged session stream for multiplexed gate room updates.
|
||||
# Disabled by default so rollout stays explicit while stream-first rooms bake.
|
||||
MESH_GATE_SESSION_STREAM_ENABLED: bool = False
|
||||
MESH_GATE_SESSION_STREAM_HEARTBEAT_S: int = 20
|
||||
MESH_GATE_SESSION_STREAM_BATCH_MS: int = 1500
|
||||
MESH_GATE_SESSION_STREAM_MAX_GATES: int = 16
|
||||
# Private gate APIs expose a backward-jittered timestamp view so observers
|
||||
# cannot trivially align exact send times from response metadata alone.
|
||||
MESH_GATE_TIMESTAMP_JITTER_S: int = 60
|
||||
# Ban/kick gate-secret rotation is on by default (hardening Rec #10): the
|
||||
# invariant has baked and a ban that does not rotate is effectively a
|
||||
# display-only removal. Set MESH_GATE_BAN_KICK_ROTATION_ENABLE=false to
|
||||
# revert to observe-only during incident triage.
|
||||
MESH_GATE_BAN_KICK_ROTATION_ENABLE: bool = True
|
||||
MESH_BLOCK_LEGACY_NODE_ID_COMPAT: bool = True
|
||||
MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK: bool = False
|
||||
MESH_ACK_RAW_FALLBACK_AT_OWN_RISK: bool = False
|
||||
MESH_SECURE_STORAGE_SECRET: str = ""
|
||||
MESH_PRIVATE_LOG_TTL_S: int = 900
|
||||
# Sprint 1 rollout: restored DM boot probes stay disabled by default until
|
||||
# the architect reviews false positives from the observe-only path.
|
||||
MESH_DM_RESTORED_SESSION_BOOT_PROBE_ENABLE: bool = False
|
||||
# Queued DM release requires explicit per-item approval before any weaker
|
||||
# relay fallback. Silent fallback is not a safe private-mode default.
|
||||
MESH_PRIVATE_RELEASE_APPROVAL_ENABLE: bool = True
|
||||
# Expiry for user-approved scoped private relay fallback policy. The policy
|
||||
# is still bounded by hidden-transport checks before it can auto-release.
|
||||
MESH_PRIVATE_RELAY_POLICY_TTL_S: int = 3600
|
||||
# Background privacy prewarm prepares keys/aliases/transport readiness
|
||||
# before send-time. Anonymous mode uses a cadence gate so user clicks do
|
||||
# not directly create hidden-transport activity.
|
||||
MESH_PRIVACY_PREWARM_ENABLE: bool = True
|
||||
MESH_PRIVACY_PREWARM_INTERVAL_S: int = 300
|
||||
MESH_PRIVACY_PREWARM_ANON_CADENCE_S: int = 300
|
||||
# Sprint 4 rollout: authenticated RNS cover markers remain disabled until
|
||||
# the observer-equivalence and receive-path DoS tests are green.
|
||||
MESH_RNS_COVER_AUTH_MARKER_ENABLE: bool = False
|
||||
# Signed-write revocation lookups use a short local TTL; stale entries force
|
||||
# a local rebuild before honor. Offline/local-refresh failures remain
|
||||
# observe-only until the later enforcement sprint.
|
||||
MESH_SIGNED_REVOCATION_CACHE_TTL_S: int = 300
|
||||
MESH_SIGNED_REVOCATION_CACHE_ENFORCE: bool = True
|
||||
MESH_SIGNED_WRITE_CONTEXT_REQUIRED: bool = True
|
||||
# Sprint 5 rollout: when enabled, root witness finality requires
|
||||
# independent quorum for threshold>1 witnessed roots before they count as
|
||||
# verified first-contact provenance.
|
||||
WORMHOLE_ROOT_WITNESS_FINALITY_ENFORCE: bool = False
|
||||
# Optional JSON artifact generated by CI/release workflow for the Sprint 8
|
||||
# release gate. Relative paths resolve from the backend directory.
|
||||
# dev = permissive local/dev behavior; testnet-private = strict private
|
||||
# defaults; release-candidate = no compatibility/debug escape hatches.
|
||||
MESH_RELEASE_PROFILE: str = "dev"
|
||||
MESH_RELEASE_ATTESTATION_PATH: str = ""
|
||||
# Operator release attestation for the Sprint 8 release gate. This does
|
||||
# not change runtime behavior; it only records that the DM relay security
|
||||
# suite was run and passed for the release candidate.
|
||||
MESH_RELEASE_DM_RELAY_SECURITY_SUITE_GREEN: bool = False
|
||||
PRIVACY_CORE_MIN_VERSION: str = "0.1.0"
|
||||
PRIVACY_CORE_ALLOWED_SHA256: str = ""
|
||||
PRIVACY_CORE_DEV_OVERRIDE: bool = False
|
||||
# Sprint 4 rollout: fail fast when the loaded privacy-core artifact is
|
||||
# missing required FFI symbols expected by the current Python bridge.
|
||||
PRIVACY_CORE_EXPORT_SET_AUDIT_ENABLE: bool = True
|
||||
# Clearnet fallback policy for private-tier messages.
|
||||
# "block" (default) = refuse to send private messages over clearnet.
|
||||
# "allow" = fall back to clearnet when Tor/RNS is unavailable (weaker privacy).
|
||||
MESH_PRIVATE_CLEARNET_FALLBACK: str = "block"
|
||||
# Second explicit opt-in for private-tier clearnet fallback. Without this
|
||||
# acknowledgement, "allow" remains requested but not effective.
|
||||
MESH_PRIVATE_CLEARNET_FALLBACK_ACKNOWLEDGE: bool = False
|
||||
# Meshtastic MQTT bridge — disabled by default to avoid hammering the
|
||||
# public broker. Users opt in explicitly.
|
||||
MESH_MQTT_ENABLED: bool = False
|
||||
# Meshtastic MQTT broker credentials (defaults match public firmware).
|
||||
MESH_MQTT_BROKER: str = "mqtt.meshtastic.org"
|
||||
MESH_MQTT_PORT: int = 1883
|
||||
MESH_MQTT_USER: str = "meshdev"
|
||||
MESH_MQTT_PASS: str = "large4cats"
|
||||
# Hex-encoded PSK — empty string means use the default LongFast key.
|
||||
# Must decode to exactly 16 or 32 bytes when set.
|
||||
MESH_MQTT_PSK: str = ""
|
||||
# Optional operator-provided Meshtastic node ID (e.g. "!abcd1234") included
|
||||
# in the User-Agent when fetching from meshtastic.liamcottle.net so the
|
||||
# service operator can identify per-install traffic instead of a generic
|
||||
# "ShadowBroker" aggregate.
|
||||
MESHTASTIC_OPERATOR_CALLSIGN: str = ""
|
||||
|
||||
# SAR (Synthetic Aperture Radar) data layer
|
||||
# Mode A — free catalog metadata, no account, default-on
|
||||
MESH_SAR_CATALOG_ENABLED: bool = True
|
||||
# Mode B — free pre-processed anomalies (OPERA / EGMS / GFM / EMS / UNOSAT)
|
||||
# Two-step opt-in: must be "allow" AND _ACKNOWLEDGE must be true
|
||||
MESH_SAR_PRODUCTS_FETCH: str = "block"
|
||||
MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE: bool = False
|
||||
# NASA Earthdata Login (free) — required for OPERA products
|
||||
MESH_SAR_EARTHDATA_USER: str = ""
|
||||
MESH_SAR_EARTHDATA_TOKEN: str = ""
|
||||
# Copernicus Data Space (free) — required for EGMS / EMS products
|
||||
MESH_SAR_COPERNICUS_USER: str = ""
|
||||
MESH_SAR_COPERNICUS_TOKEN: str = ""
|
||||
# Whether OpenClaw agents may read/act on the SAR layer
|
||||
MESH_SAR_OPENCLAW_ENABLED: bool = True
|
||||
# Require private-tier transport before signing/broadcasting SAR anomalies
|
||||
MESH_SAR_REQUIRE_PRIVATE_TIER: bool = True
|
||||
|
||||
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
||||
|
||||
@@ -120,3 +303,73 @@ class Settings(BaseSettings):
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
return Settings()
|
||||
|
||||
|
||||
def private_clearnet_fallback_requested(settings: Settings | None = None) -> str:
|
||||
snapshot = settings or get_settings()
|
||||
policy = str(getattr(snapshot, "MESH_PRIVATE_CLEARNET_FALLBACK", "block") or "block").strip().lower()
|
||||
return "allow" if policy == "allow" else "block"
|
||||
|
||||
|
||||
def private_clearnet_fallback_effective(settings: Settings | None = None) -> str:
|
||||
snapshot = settings or get_settings()
|
||||
requested = private_clearnet_fallback_requested(snapshot)
|
||||
acknowledged = bool(getattr(snapshot, "MESH_PRIVATE_CLEARNET_FALLBACK_ACKNOWLEDGE", False))
|
||||
if requested == "allow" and acknowledged:
|
||||
return "allow"
|
||||
return "block"
|
||||
|
||||
|
||||
def backend_gate_decrypt_compat_effective(settings: Settings | None = None) -> bool:
|
||||
snapshot = settings or get_settings()
|
||||
return bool(
|
||||
getattr(snapshot, "MESH_BACKEND_GATE_DECRYPT_COMPAT", False)
|
||||
or getattr(snapshot, "MESH_GATE_BACKEND_DECRYPT_COMPAT", False)
|
||||
)
|
||||
|
||||
|
||||
def backend_gate_plaintext_compat_effective(settings: Settings | None = None) -> bool:
|
||||
snapshot = settings or get_settings()
|
||||
return bool(
|
||||
getattr(snapshot, "MESH_BACKEND_GATE_PLAINTEXT_COMPAT", False)
|
||||
or getattr(snapshot, "MESH_GATE_BACKEND_PLAINTEXT_COMPAT", False)
|
||||
)
|
||||
|
||||
|
||||
def gate_recovery_envelope_effective(settings: Settings | None = None) -> bool:
|
||||
snapshot = settings or get_settings()
|
||||
requested = bool(getattr(snapshot, "MESH_GATE_RECOVERY_ENVELOPE_ENABLE", False))
|
||||
acknowledged = bool(getattr(snapshot, "MESH_GATE_RECOVERY_ENVELOPE_ENABLE_ACKNOWLEDGE", False))
|
||||
return requested and acknowledged
|
||||
|
||||
|
||||
def gate_plaintext_persist_effective(settings: Settings | None = None) -> bool:
|
||||
snapshot = settings or get_settings()
|
||||
requested = bool(getattr(snapshot, "MESH_GATE_PLAINTEXT_PERSIST", False))
|
||||
acknowledged = bool(getattr(snapshot, "MESH_GATE_PLAINTEXT_PERSIST_ACKNOWLEDGE", False))
|
||||
return requested and acknowledged
|
||||
|
||||
|
||||
def gate_ban_kick_rotation_enabled(settings: Settings | None = None) -> bool:
|
||||
snapshot = settings or get_settings()
|
||||
return bool(getattr(snapshot, "MESH_GATE_BAN_KICK_ROTATION_ENABLE", False))
|
||||
|
||||
|
||||
def dm_restored_session_boot_probe_enabled(settings: Settings | None = None) -> bool:
|
||||
snapshot = settings or get_settings()
|
||||
return bool(getattr(snapshot, "MESH_DM_RESTORED_SESSION_BOOT_PROBE_ENABLE", False))
|
||||
|
||||
|
||||
def signed_revocation_cache_ttl_s(settings: Settings | None = None) -> int:
|
||||
snapshot = settings or get_settings()
|
||||
return max(0, int(getattr(snapshot, "MESH_SIGNED_REVOCATION_CACHE_TTL_S", 300) or 0))
|
||||
|
||||
|
||||
def signed_revocation_cache_enforce(settings: Settings | None = None) -> bool:
|
||||
snapshot = settings or get_settings()
|
||||
return bool(getattr(snapshot, "MESH_SIGNED_REVOCATION_CACHE_ENFORCE", False))
|
||||
|
||||
|
||||
def wormhole_root_witness_finality_enforce(settings: Settings | None = None) -> bool:
|
||||
snapshot = settings or get_settings()
|
||||
return bool(getattr(snapshot, "WORMHOLE_ROOT_WITNESS_FINALITY_ENFORCE", False))
|
||||
|
||||
@@ -8,9 +8,13 @@ Correlation types:
|
||||
- RF Anomaly: GPS jamming + internet outage (both required)
|
||||
- Military Buildup: Military flights + naval vessels + GDELT conflict events
|
||||
- Infrastructure Cascade: Internet outage + KiwiSDR offline in same zone
|
||||
- Possible Contradiction: Official denial/statement + infrastructure disruption
|
||||
in same region — hypothesis generator, NOT verdict
|
||||
"""
|
||||
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
from collections import defaultdict
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -306,6 +310,427 @@ def _detect_infra_cascades(data: dict) -> list[dict]:
|
||||
return alerts
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Possible Contradiction: official denial/statement + infra disruption
|
||||
#
|
||||
# This is a HYPOTHESIS GENERATOR, not a verdict engine. It says "LOOK HERE"
|
||||
# when an official statement (denial, clarification, refusal) co-locates with
|
||||
# infrastructure disruption (internet outage, sigint change). The human or
|
||||
# higher-order reasoning decides what actually happened.
|
||||
#
|
||||
# Context ratings:
|
||||
# STRONG — denial + outage + prediction market movement in same region
|
||||
# MODERATE — denial + outage (no market signal)
|
||||
# WEAK — denial + minor outage or distant co-location
|
||||
# DETECTION_GAP — denial found but NO telemetry to verify (equally valuable)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Denial / official-statement patterns in headlines and URL slugs
|
||||
_DENIAL_PATTERNS = [
|
||||
re.compile(p, re.IGNORECASE) for p in [
|
||||
r"\bden(?:y|ies|ied|ial)\b",
|
||||
r"\brefut(?:e[ds]?|ing)\b",
|
||||
r"\breject(?:s|ed|ing)?\b",
|
||||
r"\bclarif(?:y|ies|ied|ication)\b",
|
||||
r"\bdismiss(?:es|ed|ing)?\b",
|
||||
r"\bno\s+attack\b",
|
||||
r"\bdid\s+not\s+(?:attack|strike|bomb|target|order|invade|kill)\b",
|
||||
r"\bnever\s+(?:attack|strike|bomb|target|order|invade|happen)\b",
|
||||
r"\bfalse\s+(?:report|claim|allegation|rumor|narrative)\b",
|
||||
r"\bmisinformation\b",
|
||||
r"\bdisinformation\b",
|
||||
r"\bpropaganda\b",
|
||||
r"\b(?:army|military|government|ministry|official)\s+(?:says|clarifies|denies|refutes)\b",
|
||||
r"\brumor[s]?\b.*\buntrue\b",
|
||||
r"\bcategorically\b",
|
||||
r"\bbaseless\b",
|
||||
]
|
||||
]
|
||||
|
||||
# Broader cell radius for sparse telemetry regions (Africa, Central Asia, etc.)
|
||||
# These regions have fewer IODA/RIPE probes so outage data is sparser
|
||||
_SPARSE_REGIONS_LAT_RANGES = [
|
||||
(-35, 37), # Africa roughly
|
||||
(25, 50), # Central Asia band (when lng 40-90)
|
||||
]
|
||||
|
||||
|
||||
def _is_sparse_region(lat: float, lng: float) -> bool:
|
||||
"""Check if coordinates fall in a region with sparse telemetry coverage."""
|
||||
# Africa
|
||||
if -35 <= lat <= 37 and -20 <= lng <= 55:
|
||||
return True
|
||||
# Central Asia
|
||||
if 25 <= lat <= 50 and 40 <= lng <= 90:
|
||||
return True
|
||||
# South America interior
|
||||
if -55 <= lat <= 12 and -80 <= lng <= -35:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
|
||||
"""Great-circle distance in km."""
|
||||
R = 6371.0
|
||||
dlat = math.radians(lat2 - lat1)
|
||||
dlon = math.radians(lon2 - lon1)
|
||||
a = (math.sin(dlat / 2) ** 2 +
|
||||
math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) *
|
||||
math.sin(dlon / 2) ** 2)
|
||||
return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
|
||||
|
||||
|
||||
def _matches_denial(text: str) -> bool:
|
||||
"""Check if text matches any denial/official-statement pattern."""
|
||||
return any(p.search(text) for p in _DENIAL_PATTERNS)
|
||||
|
||||
|
||||
def _detect_contradictions(data: dict) -> list[dict]:
|
||||
"""Detect possible contradictions between official statements and telemetry.
|
||||
|
||||
Scans GDELT headlines for denial language, then checks whether internet
|
||||
outages or other infrastructure disruptions exist in the same geographic
|
||||
region. Scores confidence and lists alternative explanations.
|
||||
"""
|
||||
gdelt = data.get("gdelt") or []
|
||||
internet_outages = data.get("internet_outages") or []
|
||||
news = data.get("news") or []
|
||||
prediction_markets = data.get("prediction_markets") or []
|
||||
|
||||
# ── Step 1: Find GDELT events with denial/official-statement language ──
|
||||
denial_events: list[dict] = []
|
||||
|
||||
# GDELT comes as GeoJSON features
|
||||
gdelt_features = gdelt
|
||||
if isinstance(gdelt, dict):
|
||||
gdelt_features = gdelt.get("features", [])
|
||||
|
||||
for feature in gdelt_features:
|
||||
# Handle both GeoJSON features and flat dicts
|
||||
if "properties" in feature and "geometry" in feature:
|
||||
props = feature.get("properties", {})
|
||||
geom = feature.get("geometry", {})
|
||||
coords = geom.get("coordinates", [])
|
||||
if len(coords) >= 2:
|
||||
lng, lat = float(coords[0]), float(coords[1])
|
||||
else:
|
||||
continue
|
||||
headlines = props.get("_headlines_list", [])
|
||||
urls = props.get("_urls_list", [])
|
||||
name = props.get("name", "")
|
||||
count = props.get("count", 1)
|
||||
else:
|
||||
lat = feature.get("lat") or feature.get("actionGeo_Lat")
|
||||
lng = feature.get("lng") or feature.get("lon") or feature.get("actionGeo_Long")
|
||||
if lat is None or lng is None:
|
||||
continue
|
||||
lat, lng = float(lat), float(lng)
|
||||
headlines = [feature.get("title", "")]
|
||||
urls = [feature.get("sourceurl", "")]
|
||||
name = feature.get("name", "")
|
||||
count = 1
|
||||
|
||||
# Check all headlines + URL slugs for denial patterns
|
||||
all_text = " ".join(str(h) for h in headlines if h)
|
||||
all_text += " " + " ".join(str(u) for u in urls if u)
|
||||
|
||||
if _matches_denial(all_text):
|
||||
denial_events.append({
|
||||
"lat": lat,
|
||||
"lng": lng,
|
||||
"headlines": [h for h in headlines if h][:5],
|
||||
"urls": [u for u in urls if u][:3],
|
||||
"location_name": name,
|
||||
"event_count": count,
|
||||
})
|
||||
|
||||
# Also scan news articles for denial language
|
||||
for article in news:
|
||||
title = str(article.get("title", "") or "")
|
||||
desc = str(article.get("description", "") or article.get("summary", "") or "")
|
||||
if not _matches_denial(title + " " + desc):
|
||||
continue
|
||||
# News articles often lack coordinates — try to match to GDELT locations
|
||||
# For now, only include if we have coordinates
|
||||
lat = article.get("lat") or article.get("latitude")
|
||||
lng = article.get("lng") or article.get("lon") or article.get("longitude")
|
||||
if lat is not None and lng is not None:
|
||||
denial_events.append({
|
||||
"lat": float(lat),
|
||||
"lng": float(lng),
|
||||
"headlines": [title],
|
||||
"urls": [article.get("url") or article.get("link") or ""],
|
||||
"location_name": "",
|
||||
"event_count": 1,
|
||||
})
|
||||
|
||||
if not denial_events:
|
||||
return []
|
||||
|
||||
# ── Step 2: Cross-reference with internet outages ──
|
||||
alerts: list[dict] = []
|
||||
|
||||
for denial in denial_events:
|
||||
d_lat, d_lng = denial["lat"], denial["lng"]
|
||||
sparse = _is_sparse_region(d_lat, d_lng)
|
||||
search_radius_km = 1500.0 if sparse else 500.0
|
||||
|
||||
# Find nearby outages
|
||||
nearby_outages: list[dict] = []
|
||||
for outage in internet_outages:
|
||||
o_lat = outage.get("lat") or outage.get("latitude")
|
||||
o_lng = outage.get("lng") or outage.get("lon") or outage.get("longitude")
|
||||
if o_lat is None or o_lng is None:
|
||||
continue
|
||||
try:
|
||||
dist = _haversine_km(d_lat, d_lng, float(o_lat), float(o_lng))
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if dist <= search_radius_km:
|
||||
nearby_outages.append({
|
||||
"region": outage.get("region_name") or outage.get("country_name", ""),
|
||||
"severity": _outage_pct(outage),
|
||||
"distance_km": round(dist, 0),
|
||||
"level": outage.get("level", ""),
|
||||
})
|
||||
|
||||
# ── Step 3: Check prediction markets for related movements ──
|
||||
denial_text = " ".join(denial["headlines"]).lower()
|
||||
related_markets: list[dict] = []
|
||||
for market in prediction_markets:
|
||||
m_title = str(market.get("title", "") or market.get("question", "") or "").lower()
|
||||
# Look for keyword overlap between denial and market
|
||||
denial_words = set(re.findall(r"[a-z]{4,}", denial_text))
|
||||
market_words = set(re.findall(r"[a-z]{4,}", m_title))
|
||||
overlap = denial_words & market_words - {"that", "this", "with", "from", "have", "been", "were", "will", "says", "said"}
|
||||
if len(overlap) >= 2:
|
||||
prob = market.get("probability") or market.get("lastTradePrice") or market.get("yes_price")
|
||||
if prob is not None:
|
||||
related_markets.append({
|
||||
"title": market.get("title") or market.get("question"),
|
||||
"probability": float(prob),
|
||||
})
|
||||
|
||||
# ── Step 4: Score confidence and assign context rating ──
|
||||
indicators = 1 # denial itself
|
||||
drivers: list[str] = []
|
||||
|
||||
# Primary driver: the denial headline
|
||||
headline_display = denial["headlines"][0] if denial["headlines"] else "Official statement"
|
||||
if len(headline_display) > 80:
|
||||
headline_display = headline_display[:77] + "..."
|
||||
drivers.append(f'"{headline_display}"')
|
||||
|
||||
# Outage co-location
|
||||
has_outage = False
|
||||
if nearby_outages:
|
||||
best_outage = max(nearby_outages, key=lambda o: o["severity"])
|
||||
if best_outage["severity"] >= 10:
|
||||
indicators += 1
|
||||
has_outage = True
|
||||
drivers.append(
|
||||
f"Internet outage {best_outage['severity']:.0f}% "
|
||||
f"({best_outage['region']}, {best_outage['distance_km']:.0f}km away)"
|
||||
)
|
||||
elif best_outage["severity"] > 0:
|
||||
indicators += 0.5 # minor outage, partial indicator
|
||||
has_outage = True
|
||||
drivers.append(
|
||||
f"Minor outage ({best_outage['region']}, "
|
||||
f"{best_outage['distance_km']:.0f}km away)"
|
||||
)
|
||||
|
||||
# Prediction market signal
|
||||
has_market = False
|
||||
if related_markets:
|
||||
indicators += 1
|
||||
has_market = True
|
||||
top_market = related_markets[0]
|
||||
drivers.append(
|
||||
f"Market: \"{top_market['title'][:50]}\" "
|
||||
f"at {top_market['probability']:.0%}"
|
||||
)
|
||||
|
||||
# Multiple denial sources strengthen the signal
|
||||
if denial["event_count"] > 1:
|
||||
indicators += 0.5
|
||||
drivers.append(f"{denial['event_count']} sources reporting")
|
||||
|
||||
# Context rating
|
||||
if has_outage and has_market:
|
||||
context = "STRONG"
|
||||
elif has_outage:
|
||||
context = "MODERATE"
|
||||
elif has_market:
|
||||
context = "WEAK" # market signal without infra disruption
|
||||
else:
|
||||
context = "DETECTION_GAP"
|
||||
|
||||
# Severity mapping
|
||||
if context == "STRONG":
|
||||
sev = "high"
|
||||
elif context == "MODERATE":
|
||||
sev = "medium"
|
||||
else:
|
||||
sev = "low"
|
||||
|
||||
# Alternative explanations (always present — this is a hypothesis generator)
|
||||
alternatives: list[str] = []
|
||||
if has_outage:
|
||||
alternatives.append("Routine infrastructure maintenance or cable damage")
|
||||
alternatives.append("Weather-related outage coinciding with news cycle")
|
||||
if not has_outage and context == "DETECTION_GAP":
|
||||
alternatives.append("Statement may be truthful — no contradicting telemetry found")
|
||||
alternatives.append("Telemetry coverage gap in this region")
|
||||
alternatives.append("Denial may be responding to social media rumors, not real events")
|
||||
|
||||
lat_c, lng_c = _cell_center(_cell_key(d_lat, d_lng))
|
||||
alerts.append({
|
||||
"lat": lat_c,
|
||||
"lng": lng_c,
|
||||
"type": "contradiction",
|
||||
"severity": sev,
|
||||
"score": _severity_score(sev),
|
||||
"drivers": drivers[:4],
|
||||
"cell_size": _CELL_SIZE,
|
||||
"context": context,
|
||||
"alternatives": alternatives[:3],
|
||||
"location_name": denial.get("location_name", ""),
|
||||
"headlines": denial["headlines"][:3],
|
||||
"related_markets": related_markets[:3],
|
||||
"nearby_outages": nearby_outages[:5],
|
||||
})
|
||||
|
||||
# Deduplicate: keep highest-scored alert per cell
|
||||
seen_cells: dict[str, dict] = {}
|
||||
for alert in alerts:
|
||||
key = _cell_key(alert["lat"], alert["lng"])
|
||||
if key not in seen_cells or alert["score"] > seen_cells[key]["score"]:
|
||||
seen_cells[key] = alert
|
||||
|
||||
result = list(seen_cells.values())
|
||||
if result:
|
||||
by_context = defaultdict(int)
|
||||
for a in result:
|
||||
by_context[a["context"]] += 1
|
||||
logger.info(
|
||||
"Contradictions: %d possible (%s)",
|
||||
len(result),
|
||||
", ".join(f"{v} {k}" for k, v in sorted(by_context.items())),
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Correlation → Pin bridge
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Types and their pin categories
|
||||
_CORR_PIN_CATEGORIES = {
|
||||
"rf_anomaly": "anomaly",
|
||||
"military_buildup": "military",
|
||||
"infra_cascade": "infrastructure",
|
||||
"contradiction": "research",
|
||||
}
|
||||
|
||||
# Deduplicate: don't re-pin the same cell within this window (seconds).
|
||||
_CORR_PIN_DEDUP_WINDOW = 600 # 10 minutes
|
||||
_recent_corr_pins: dict[str, float] = {}
|
||||
|
||||
|
||||
def _auto_pin_correlations(alerts: list[dict]) -> int:
|
||||
"""Create AI Intel pins for high-severity correlation alerts.
|
||||
|
||||
Only pins alerts with severity >= medium. Uses cell-key dedup so the
|
||||
same grid cell doesn't get re-pinned every fetch cycle.
|
||||
|
||||
Returns the number of pins created this cycle.
|
||||
"""
|
||||
import time as _time
|
||||
|
||||
now = _time.time()
|
||||
|
||||
# Evict stale dedup entries
|
||||
expired = [k for k, ts in _recent_corr_pins.items() if now - ts > _CORR_PIN_DEDUP_WINDOW]
|
||||
for k in expired:
|
||||
_recent_corr_pins.pop(k, None)
|
||||
|
||||
created = 0
|
||||
for alert in alerts:
|
||||
sev = alert.get("severity", "low")
|
||||
if sev == "low":
|
||||
continue # Don't pin low-severity noise
|
||||
|
||||
lat = alert.get("lat")
|
||||
lng = alert.get("lng")
|
||||
if lat is None or lng is None:
|
||||
continue
|
||||
|
||||
# Dedup key: type + cell
|
||||
dedup_key = f"{alert['type']}:{_cell_key(lat, lng)}"
|
||||
if dedup_key in _recent_corr_pins:
|
||||
continue
|
||||
|
||||
category = _CORR_PIN_CATEGORIES.get(alert["type"], "anomaly")
|
||||
drivers = alert.get("drivers", [])
|
||||
atype = alert["type"]
|
||||
|
||||
if atype == "contradiction":
|
||||
ctx = alert.get("context", "")
|
||||
label = f"[{ctx}] Possible Contradiction"
|
||||
parts = list(drivers)
|
||||
if alert.get("alternatives"):
|
||||
parts.append("Alternatives: " + "; ".join(alert["alternatives"][:2]))
|
||||
description = " | ".join(parts) if parts else "Narrative contradiction detected"
|
||||
else:
|
||||
label = f"[{sev.upper()}] {atype.replace('_', ' ').title()}"
|
||||
description = "; ".join(drivers) if drivers else "Multi-layer correlation alert"
|
||||
|
||||
try:
|
||||
from services.ai_pin_store import create_pin
|
||||
|
||||
meta = {
|
||||
"correlation_type": atype,
|
||||
"severity": sev,
|
||||
"drivers": drivers,
|
||||
"cell_size": alert.get("cell_size", _CELL_SIZE),
|
||||
}
|
||||
# Add contradiction-specific metadata
|
||||
if atype == "contradiction":
|
||||
meta["context_rating"] = alert.get("context", "")
|
||||
meta["alternatives"] = alert.get("alternatives", [])
|
||||
meta["headlines"] = alert.get("headlines", [])
|
||||
meta["location_name"] = alert.get("location_name", "")
|
||||
if alert.get("related_markets"):
|
||||
meta["related_markets"] = alert["related_markets"]
|
||||
|
||||
create_pin(
|
||||
lat=lat,
|
||||
lng=lng,
|
||||
label=label,
|
||||
category=category,
|
||||
description=description,
|
||||
source="correlation_engine",
|
||||
confidence=alert.get("score", 60) / 100.0,
|
||||
ttl_hours=2.0, # Auto-expire correlation pins after 2 hours
|
||||
metadata=meta,
|
||||
)
|
||||
_recent_corr_pins[dedup_key] = now
|
||||
created += 1
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to auto-pin correlation: %s", exc)
|
||||
|
||||
if created:
|
||||
logger.info("Correlation engine auto-pinned %d alerts", created)
|
||||
return created
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -330,13 +755,29 @@ def compute_correlations(data: dict) -> list[dict]:
|
||||
except Exception as e:
|
||||
logger.error("Correlation engine infra cascade error: %s", e)
|
||||
|
||||
# Contradiction detection removed from automated engine — too many false
|
||||
# positives from regex headline matching. Contradiction/analysis alerts are
|
||||
# now placed by OpenClaw agents via place_analysis_zone, which lets an LLM
|
||||
# reason about the evidence rather than pattern-matching keywords.
|
||||
try:
|
||||
from services.analysis_zone_store import get_live_zones
|
||||
alerts.extend(get_live_zones())
|
||||
except Exception as e:
|
||||
logger.error("Analysis zone merge error: %s", e)
|
||||
|
||||
rf = sum(1 for a in alerts if a["type"] == "rf_anomaly")
|
||||
mil = sum(1 for a in alerts if a["type"] == "military_buildup")
|
||||
infra = sum(1 for a in alerts if a["type"] == "infra_cascade")
|
||||
contra = sum(1 for a in alerts if a["type"] == "contradiction")
|
||||
if alerts:
|
||||
logger.info(
|
||||
"Correlations: %d alerts (%d rf, %d mil, %d infra)",
|
||||
len(alerts), rf, mil, infra,
|
||||
"Correlations: %d alerts (%d rf, %d mil, %d infra, %d contra)",
|
||||
len(alerts), rf, mil, infra, contra,
|
||||
)
|
||||
|
||||
# Correlation alerts are returned in the correlations data feed only.
|
||||
# They are NOT auto-pinned to AI Intel — that layer is reserved for
|
||||
# user / OpenClaw pins. Correlations are visualised via the dedicated
|
||||
# correlations overlay on the map.
|
||||
|
||||
return alerts
|
||||
|
||||
@@ -16,9 +16,12 @@ Heavy logic has been extracted into services/fetchers/:
|
||||
|
||||
import logging
|
||||
import concurrent.futures
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
@@ -56,6 +59,7 @@ from services.fetchers.earth_observation import ( # noqa: F401
|
||||
fetch_air_quality,
|
||||
fetch_volcanoes,
|
||||
fetch_viirs_change_nodes,
|
||||
fetch_uap_sightings,
|
||||
)
|
||||
from services.fetchers.infrastructure import ( # noqa: F401
|
||||
fetch_internet_outages,
|
||||
@@ -90,10 +94,35 @@ from services.fetchers.meshtastic_map import (
|
||||
load_meshtastic_cache_if_available,
|
||||
) # noqa: F401
|
||||
from services.fetchers.fimi import fetch_fimi # noqa: F401
|
||||
from services.fetchers.crowdthreat import fetch_crowdthreat # noqa: F401
|
||||
from services.fetchers.wastewater import fetch_wastewater # noqa: F401
|
||||
from services.fetchers.sar_catalog import fetch_sar_catalog # noqa: F401
|
||||
from services.fetchers.sar_products import fetch_sar_products # noqa: F401
|
||||
from services.ais_stream import prune_stale_vessels # noqa: F401
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_SLOW_FETCH_S = float(os.environ.get("FETCH_SLOW_THRESHOLD_S", "5"))
|
||||
# Hard wall-clock limit per individual fetch task. A task that exceeds this
|
||||
# is treated as a failure so it cannot block an entire fetch tier indefinitely.
|
||||
_TASK_HARD_TIMEOUT_S = float(os.environ.get("FETCH_TASK_TIMEOUT_S", "120"))
|
||||
_FAST_STARTUP_CACHE_MAX_AGE_S = float(os.environ.get("FAST_STARTUP_CACHE_MAX_AGE_S", "300"))
|
||||
_FAST_STARTUP_CACHE_PATH = Path(__file__).resolve().parents[1] / "data" / "fast_startup_cache.json"
|
||||
_FAST_STARTUP_CACHE_KEYS = (
|
||||
"commercial_flights",
|
||||
"military_flights",
|
||||
"private_flights",
|
||||
"private_jets",
|
||||
"tracked_flights",
|
||||
"ships",
|
||||
"uavs",
|
||||
"gps_jamming",
|
||||
"satellites",
|
||||
"satellite_source",
|
||||
"satellite_analysis",
|
||||
"sigint",
|
||||
"sigint_totals",
|
||||
"trains",
|
||||
)
|
||||
|
||||
# Shared thread pool — reused across all fetch cycles instead of creating/destroying per tick
|
||||
_SHARED_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
|
||||
@@ -101,6 +130,80 @@ _SHARED_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
|
||||
)
|
||||
|
||||
|
||||
def _cache_json_safe(value):
|
||||
if isinstance(value, float):
|
||||
return value if math.isfinite(value) else None
|
||||
if isinstance(value, dict):
|
||||
return {str(k): _cache_json_safe(v) for k, v in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_cache_json_safe(v) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
def _load_fast_startup_cache_if_available() -> bool:
|
||||
"""Seed moving layers from a recent disk cache while live fetches warm up."""
|
||||
if _FAST_STARTUP_CACHE_MAX_AGE_S <= 0 or not _FAST_STARTUP_CACHE_PATH.exists():
|
||||
return False
|
||||
try:
|
||||
with _FAST_STARTUP_CACHE_PATH.open("r", encoding="utf-8") as fh:
|
||||
payload = json.load(fh)
|
||||
cached_at = float(payload.get("cached_at") or 0)
|
||||
age_s = time.time() - cached_at
|
||||
if cached_at <= 0 or age_s > _FAST_STARTUP_CACHE_MAX_AGE_S:
|
||||
logger.info("Skipping stale fast startup cache (age %.1fs)", age_s)
|
||||
return False
|
||||
layers = payload.get("layers") or {}
|
||||
freshness = payload.get("freshness") or {}
|
||||
loaded: list[str] = []
|
||||
with _data_lock:
|
||||
for key in _FAST_STARTUP_CACHE_KEYS:
|
||||
if key in layers:
|
||||
latest_data[key] = layers[key]
|
||||
loaded.append(key)
|
||||
for key, ts in freshness.items():
|
||||
source_timestamps[str(key)] = ts
|
||||
if payload.get("last_updated"):
|
||||
latest_data["last_updated"] = payload.get("last_updated")
|
||||
if not loaded:
|
||||
return False
|
||||
from services.fetchers._store import bump_data_version
|
||||
|
||||
bump_data_version()
|
||||
logger.info(
|
||||
"Loaded fast startup cache for %d layers (age %.1fs) so the map can paint before remote feeds finish",
|
||||
len(loaded),
|
||||
age_s,
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning("Fast startup cache load failed (non-fatal): %s", e)
|
||||
return False
|
||||
|
||||
|
||||
def _save_fast_startup_cache() -> None:
|
||||
"""Persist recent moving layers for the next cold start."""
|
||||
try:
|
||||
with _data_lock:
|
||||
payload = {
|
||||
"cached_at": time.time(),
|
||||
"last_updated": latest_data.get("last_updated"),
|
||||
"layers": {key: latest_data.get(key) for key in _FAST_STARTUP_CACHE_KEYS},
|
||||
"freshness": {
|
||||
key: source_timestamps.get(key)
|
||||
for key in _FAST_STARTUP_CACHE_KEYS
|
||||
if source_timestamps.get(key)
|
||||
},
|
||||
}
|
||||
safe_payload = _cache_json_safe(payload)
|
||||
_FAST_STARTUP_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp_path = _FAST_STARTUP_CACHE_PATH.with_suffix(".tmp")
|
||||
with tmp_path.open("w", encoding="utf-8") as fh:
|
||||
json.dump(safe_payload, fh, separators=(",", ":"))
|
||||
tmp_path.replace(_FAST_STARTUP_CACHE_PATH)
|
||||
except Exception as e:
|
||||
logger.debug("Fast startup cache save skipped: %s", e)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scheduler & Orchestration
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -109,10 +212,12 @@ def _run_tasks(label: str, funcs: list):
|
||||
if not funcs:
|
||||
return
|
||||
futures = {_SHARED_EXECUTOR.submit(func): (func.__name__, time.perf_counter()) for func in funcs}
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
name, start = futures[future]
|
||||
# Iterate directly so future.result(timeout=...) is the blocking call.
|
||||
# as_completed() blocks inside __next__() waiting for completion — the timeout
|
||||
# on result() would never be reached for a hanging task under that pattern.
|
||||
for future, (name, start) in futures.items():
|
||||
try:
|
||||
future.result()
|
||||
future.result(timeout=_TASK_HARD_TIMEOUT_S)
|
||||
duration = time.perf_counter() - start
|
||||
from services.fetch_health import record_success
|
||||
|
||||
@@ -164,6 +269,7 @@ def update_fast_data():
|
||||
latest_data["last_updated"] = datetime.utcnow().isoformat()
|
||||
from services.fetchers._store import bump_data_version
|
||||
bump_data_version()
|
||||
_save_fast_startup_cache()
|
||||
logger.info("Fast-tier update complete.")
|
||||
|
||||
|
||||
@@ -219,6 +325,7 @@ def update_all_data(*, startup_mode: bool = False):
|
||||
logger.info("Full data update starting (parallel)...")
|
||||
# Preload Meshtastic map cache immediately (instant, from disk)
|
||||
load_meshtastic_cache_if_available()
|
||||
_load_fast_startup_cache_if_available()
|
||||
with _data_lock:
|
||||
meshtastic_seeded = bool(latest_data.get("meshtastic_map_nodes"))
|
||||
futures = {
|
||||
@@ -231,6 +338,11 @@ def update_all_data(*, startup_mode: bool = False):
|
||||
_SHARED_EXECUTOR.submit(fetch_fimi): ("fetch_fimi", time.perf_counter()),
|
||||
_SHARED_EXECUTOR.submit(fetch_gdelt): ("fetch_gdelt", time.perf_counter()),
|
||||
_SHARED_EXECUTOR.submit(update_liveuamap): ("update_liveuamap", time.perf_counter()),
|
||||
_SHARED_EXECUTOR.submit(fetch_uap_sightings): ("fetch_uap_sightings", time.perf_counter()),
|
||||
_SHARED_EXECUTOR.submit(fetch_wastewater): ("fetch_wastewater", time.perf_counter()),
|
||||
_SHARED_EXECUTOR.submit(fetch_crowdthreat): ("fetch_crowdthreat", time.perf_counter()),
|
||||
_SHARED_EXECUTOR.submit(fetch_sar_catalog): ("fetch_sar_catalog", time.perf_counter()),
|
||||
_SHARED_EXECUTOR.submit(fetch_sar_products): ("fetch_sar_products", time.perf_counter()),
|
||||
}
|
||||
if not startup_mode or not meshtastic_seeded:
|
||||
futures[_SHARED_EXECUTOR.submit(fetch_meshtastic_nodes)] = (
|
||||
@@ -241,10 +353,9 @@ def update_all_data(*, startup_mode: bool = False):
|
||||
logger.info(
|
||||
"Startup preload: Meshtastic cache already loaded, deferring remote map refresh to scheduled cadence"
|
||||
)
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
name, start = futures[future]
|
||||
for future, (name, start) in futures.items():
|
||||
try:
|
||||
future.result()
|
||||
future.result(timeout=_TASK_HARD_TIMEOUT_S)
|
||||
duration = time.perf_counter() - start
|
||||
from services.fetch_health import record_success
|
||||
|
||||
@@ -257,6 +368,42 @@ def update_all_data(*, startup_mode: bool = False):
|
||||
|
||||
record_failure(name, error=e, duration_s=duration)
|
||||
logger.exception(f"full-refresh task failed: {name}")
|
||||
# Run CCTV ingest immediately so cameras are available on first request
|
||||
# (the scheduled job also runs every 10 min for ongoing refresh).
|
||||
if startup_mode:
|
||||
try:
|
||||
from services.cctv_pipeline import (
|
||||
TFLJamCamIngestor, LTASingaporeIngestor, AustinTXIngestor,
|
||||
NYCDOTIngestor, CaltransIngestor, ColoradoDOTIngestor,
|
||||
WSDOTIngestor, GeorgiaDOTIngestor, IllinoisDOTIngestor,
|
||||
MichiganDOTIngestor, WindyWebcamsIngestor, DGTNationalIngestor,
|
||||
MadridCityIngestor, OSMTrafficCameraIngestor, get_all_cameras,
|
||||
)
|
||||
from services.cctv_pipeline import OSMALPRCameraIngestor
|
||||
_startup_ingestors = [
|
||||
TFLJamCamIngestor(), LTASingaporeIngestor(), AustinTXIngestor(),
|
||||
NYCDOTIngestor(), CaltransIngestor(), ColoradoDOTIngestor(),
|
||||
WSDOTIngestor(), GeorgiaDOTIngestor(), IllinoisDOTIngestor(),
|
||||
MichiganDOTIngestor(), WindyWebcamsIngestor(), DGTNationalIngestor(),
|
||||
MadridCityIngestor(), OSMTrafficCameraIngestor(),
|
||||
OSMALPRCameraIngestor(),
|
||||
]
|
||||
logger.info("Running CCTV ingest at startup (%d ingestors)...", len(_startup_ingestors))
|
||||
ingest_futures = {
|
||||
_SHARED_EXECUTOR.submit(ing.ingest): ing.__class__.__name__
|
||||
for ing in _startup_ingestors
|
||||
}
|
||||
for fut in concurrent.futures.as_completed(ingest_futures, timeout=90):
|
||||
name = ingest_futures[fut]
|
||||
try:
|
||||
fut.result()
|
||||
except Exception as e:
|
||||
logger.warning("CCTV startup ingest %s failed: %s", name, e)
|
||||
fetch_cctv()
|
||||
logger.info("CCTV startup ingest complete — %d cameras in DB", len(get_all_cameras()))
|
||||
except Exception as e:
|
||||
logger.warning("CCTV startup ingest failed (non-fatal): %s", e)
|
||||
|
||||
logger.info("Full data update complete.")
|
||||
|
||||
|
||||
@@ -406,6 +553,38 @@ def start_scheduler():
|
||||
misfire_grace_time=60,
|
||||
)
|
||||
|
||||
# Route database — bulk refresh from vrs-standing-data.adsb.lol every 5
|
||||
# days. Replaces the legacy /api/0/routeset POST (blocked under our UA,
|
||||
# and broken upstream). Airline schedules change on a quarterly cycle,
|
||||
# so 5 days is well within the staleness budget; new flight numbers
|
||||
# added within the window simply fall back to UNKNOWN until refresh.
|
||||
from services.fetchers.route_database import refresh_route_database
|
||||
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health(refresh_route_database, "refresh_route_database"),
|
||||
"interval",
|
||||
days=5,
|
||||
id="route_database",
|
||||
max_instances=1,
|
||||
misfire_grace_time=3600,
|
||||
)
|
||||
|
||||
# Aircraft metadata database — bulk refresh from OpenSky's public S3
|
||||
# bucket every 5 days. Provides hex24 -> ICAO type so OpenSky-sourced
|
||||
# flights (which lack 't' in /states/all) get aircraft category and
|
||||
# fuel/CO2 emissions populated. Snapshots are monthly; 5 days catches
|
||||
# newer drops without hammering the bucket.
|
||||
from services.fetchers.aircraft_database import refresh_aircraft_database
|
||||
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health(refresh_aircraft_database, "refresh_aircraft_database"),
|
||||
"interval",
|
||||
days=5,
|
||||
id="aircraft_database",
|
||||
max_instances=1,
|
||||
misfire_grace_time=3600,
|
||||
)
|
||||
|
||||
# GDELT — every 30 minutes (downloads 32 ZIP files per call, avoid rate limits)
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health(fetch_gdelt, "fetch_gdelt"),
|
||||
@@ -510,14 +689,21 @@ def start_scheduler():
|
||||
misfire_grace_time=120,
|
||||
)
|
||||
|
||||
# Meshtastic map API — every 4 hours, fetch global node positions
|
||||
# Meshtastic map API — once per day with a per-install random offset to
|
||||
# avoid thundering the one-person hobby service at the top of the hour.
|
||||
# The fetcher also short-circuits on a fresh on-disk cache, so the
|
||||
# practical network cadence is closer to "once per day per install".
|
||||
import random as _random_jitter
|
||||
|
||||
_meshtastic_jitter_minutes = _random_jitter.randint(0, 180)
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health(fetch_meshtastic_nodes, "fetch_meshtastic_nodes"),
|
||||
"interval",
|
||||
hours=4,
|
||||
hours=24,
|
||||
minutes=_meshtastic_jitter_minutes,
|
||||
id="meshtastic_map",
|
||||
max_instances=1,
|
||||
misfire_grace_time=600,
|
||||
misfire_grace_time=3600,
|
||||
)
|
||||
|
||||
# Oracle resolution sweep — every hour, check if any markets with predictions have concluded
|
||||
@@ -550,9 +736,136 @@ def start_scheduler():
|
||||
misfire_grace_time=600,
|
||||
)
|
||||
|
||||
# UAP sightings (NUFORC) — daily at 12:00 UTC
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health(
|
||||
lambda: fetch_uap_sightings(force_refresh=True),
|
||||
"fetch_uap_sightings",
|
||||
),
|
||||
"cron",
|
||||
hour=12,
|
||||
minute=0,
|
||||
id="uap_sightings_daily",
|
||||
max_instances=1,
|
||||
misfire_grace_time=3600,
|
||||
)
|
||||
|
||||
# WastewaterSCAN pathogen surveillance — daily at 12:00 UTC (samples update ~daily)
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health(fetch_wastewater, "fetch_wastewater"),
|
||||
"cron",
|
||||
hour=12,
|
||||
minute=0,
|
||||
id="wastewater_daily",
|
||||
max_instances=1,
|
||||
misfire_grace_time=3600,
|
||||
)
|
||||
|
||||
# CrowdThreat verified threat intelligence — daily at 12:00 UTC
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health(fetch_crowdthreat, "fetch_crowdthreat"),
|
||||
"cron",
|
||||
hour=12,
|
||||
minute=0,
|
||||
id="crowdthreat_daily",
|
||||
max_instances=1,
|
||||
misfire_grace_time=3600,
|
||||
)
|
||||
|
||||
# SAR catalog (Mode A) — every hour, free metadata from ASF Search.
|
||||
# No account, no downloads, no DSP. Pure scene catalog + coverage hints.
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health(fetch_sar_catalog, "fetch_sar_catalog"),
|
||||
"interval",
|
||||
hours=1,
|
||||
id="sar_catalog",
|
||||
max_instances=1,
|
||||
misfire_grace_time=600,
|
||||
next_run_time=datetime.utcnow() + timedelta(minutes=3),
|
||||
)
|
||||
|
||||
# SAR products (Mode B) — every 30 minutes, opt-in only.
|
||||
# Pre-processed deformation/flood/damage anomalies from OPERA, EGMS, GFM,
|
||||
# EMS, UNOSAT. Disabled until both MESH_SAR_PRODUCTS_FETCH=allow and
|
||||
# MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE=true are set.
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health(fetch_sar_products, "fetch_sar_products"),
|
||||
"interval",
|
||||
minutes=30,
|
||||
id="sar_products",
|
||||
max_instances=1,
|
||||
misfire_grace_time=600,
|
||||
next_run_time=datetime.utcnow() + timedelta(minutes=5),
|
||||
)
|
||||
|
||||
# ── Time Machine auto-snapshots ─────────────────────────────────────
|
||||
# Compressed snapshots taken on two profiles (high_freq + standard).
|
||||
# Intervals are read from _timemachine_config at each invocation so
|
||||
# config changes via the API take effect without restarting.
|
||||
|
||||
def _auto_snapshot_high_freq():
|
||||
"""Auto-snapshot fast-moving layers (flights, ships, satellites)."""
|
||||
try:
|
||||
from services.node_settings import read_node_settings
|
||||
if not read_node_settings().get("timemachine_enabled", False):
|
||||
return # Time Machine is off — skip
|
||||
from routers.ai_intel import _timemachine_config, _take_snapshot_internal
|
||||
cfg = _timemachine_config["profiles"]["high_freq"]
|
||||
if cfg["interval_minutes"] <= 0:
|
||||
return # disabled
|
||||
layers = cfg["layers"]
|
||||
result = _take_snapshot_internal(layers=layers, profile="auto_high_freq", compress=True)
|
||||
logger.info("Time Machine auto-snapshot (high_freq): %s — %s layers",
|
||||
result.get("snapshot_id"), len(result.get("layers", [])))
|
||||
except Exception as e:
|
||||
logger.warning("Time Machine auto-snapshot (high_freq) failed: %s", e)
|
||||
|
||||
def _auto_snapshot_standard():
|
||||
"""Auto-snapshot contextual layers (news, earthquakes, weather, etc.)."""
|
||||
try:
|
||||
from services.node_settings import read_node_settings
|
||||
if not read_node_settings().get("timemachine_enabled", False):
|
||||
return # Time Machine is off — skip
|
||||
from routers.ai_intel import _timemachine_config, _take_snapshot_internal
|
||||
cfg = _timemachine_config["profiles"]["standard"]
|
||||
if cfg["interval_minutes"] <= 0:
|
||||
return # disabled
|
||||
layers = cfg["layers"]
|
||||
result = _take_snapshot_internal(layers=layers, profile="auto_standard", compress=True)
|
||||
logger.info("Time Machine auto-snapshot (standard): %s — %s layers",
|
||||
result.get("snapshot_id"), len(result.get("layers", [])))
|
||||
except Exception as e:
|
||||
logger.warning("Time Machine auto-snapshot (standard) failed: %s", e)
|
||||
|
||||
_scheduler.add_job(
|
||||
_auto_snapshot_high_freq,
|
||||
"interval",
|
||||
minutes=15,
|
||||
id="timemachine_high_freq",
|
||||
max_instances=1,
|
||||
misfire_grace_time=60,
|
||||
next_run_time=datetime.utcnow() + timedelta(minutes=2), # first snapshot 2m after startup
|
||||
)
|
||||
_scheduler.add_job(
|
||||
_auto_snapshot_standard,
|
||||
"interval",
|
||||
minutes=120,
|
||||
id="timemachine_standard",
|
||||
max_instances=1,
|
||||
misfire_grace_time=300,
|
||||
next_run_time=datetime.utcnow() + timedelta(minutes=5), # first snapshot 5m after startup
|
||||
)
|
||||
|
||||
_scheduler.start()
|
||||
logger.info("Scheduler started.")
|
||||
|
||||
# Start the feed ingester daemon (refreshes feed-backed pin layers)
|
||||
try:
|
||||
from services.feed_ingester import start_feed_ingester
|
||||
start_feed_ingester()
|
||||
except Exception as e:
|
||||
logger.warning("Failed to start feed ingester: %s", e)
|
||||
|
||||
|
||||
def stop_scheduler():
|
||||
if _scheduler:
|
||||
|
||||
+895
-28
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,238 @@
|
||||
"""Feed Ingester — background daemon that refreshes feed-backed pin layers.
|
||||
|
||||
Layers with a non-empty `feed_url` are polled at their `feed_interval`
|
||||
(seconds, minimum 60). The feed is expected to return either:
|
||||
|
||||
1. GeoJSON FeatureCollection — features are converted to pins
|
||||
2. JSON array of pin objects — used directly
|
||||
|
||||
Each refresh atomically replaces the layer's pins with the new data.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# State
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_running = False
|
||||
_thread: threading.Thread | None = None
|
||||
_CHECK_INTERVAL = 30 # seconds between scanning for layers that need refresh
|
||||
_last_fetched: dict[str, float] = {} # layer_id → last fetch timestamp
|
||||
_FETCH_TIMEOUT = 20 # seconds
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GeoJSON → pin conversion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _geojson_features_to_pins(features: list[dict]) -> list[dict[str, Any]]:
|
||||
"""Convert GeoJSON Feature objects to pin dicts."""
|
||||
pins: list[dict[str, Any]] = []
|
||||
for feat in features:
|
||||
if not isinstance(feat, dict):
|
||||
continue
|
||||
geom = feat.get("geometry") or {}
|
||||
props = feat.get("properties") or {}
|
||||
|
||||
# Extract coordinates
|
||||
coords = geom.get("coordinates")
|
||||
if geom.get("type") != "Point" or not coords or len(coords) < 2:
|
||||
continue
|
||||
|
||||
lng, lat = float(coords[0]), float(coords[1])
|
||||
if not (-90 <= lat <= 90 and -180 <= lng <= 180):
|
||||
continue
|
||||
|
||||
pin: dict[str, Any] = {
|
||||
"lat": lat,
|
||||
"lng": lng,
|
||||
"label": str(props.get("label", props.get("name", props.get("title", ""))))[:200],
|
||||
"category": str(props.get("category", "custom"))[:50],
|
||||
"color": str(props.get("color", ""))[:20],
|
||||
"description": str(props.get("description", props.get("summary", "")))[:2000],
|
||||
"source": "feed",
|
||||
"source_url": str(props.get("source_url", props.get("url", props.get("link", ""))))[:500],
|
||||
"confidence": float(props.get("confidence", 1.0)),
|
||||
}
|
||||
|
||||
# Entity attachment if present
|
||||
entity_type = props.get("entity_type", "")
|
||||
entity_id = props.get("entity_id", "")
|
||||
if entity_type and entity_id:
|
||||
pin["entity_attachment"] = {
|
||||
"entity_type": str(entity_type),
|
||||
"entity_id": str(entity_id),
|
||||
"entity_label": str(props.get("entity_label", "")),
|
||||
}
|
||||
|
||||
pins.append(pin)
|
||||
return pins
|
||||
|
||||
|
||||
def _parse_feed_response(data: Any) -> list[dict[str, Any]]:
|
||||
"""Parse a feed response into a list of pin dicts."""
|
||||
if isinstance(data, dict):
|
||||
# GeoJSON FeatureCollection
|
||||
if data.get("type") == "FeatureCollection" and isinstance(data.get("features"), list):
|
||||
return _geojson_features_to_pins(data["features"])
|
||||
# Single Feature
|
||||
if data.get("type") == "Feature":
|
||||
return _geojson_features_to_pins([data])
|
||||
# Wrapped response like {"ok": true, "data": [...]}
|
||||
inner = data.get("data") or data.get("results") or data.get("pins") or data.get("items")
|
||||
if isinstance(inner, list):
|
||||
return _normalize_pin_list(inner)
|
||||
|
||||
if isinstance(data, list):
|
||||
# Check if first item looks like a GeoJSON Feature
|
||||
if data and isinstance(data[0], dict) and data[0].get("type") == "Feature":
|
||||
return _geojson_features_to_pins(data)
|
||||
return _normalize_pin_list(data)
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def _normalize_pin_list(items: list) -> list[dict[str, Any]]:
|
||||
"""Normalize a list of raw pin objects, ensuring lat/lng are present."""
|
||||
pins: list[dict[str, Any]] = []
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
lat = item.get("lat") or item.get("latitude")
|
||||
lng = item.get("lng") or item.get("lon") or item.get("longitude")
|
||||
if lat is None or lng is None:
|
||||
continue
|
||||
try:
|
||||
lat, lng = float(lat), float(lng)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if not (-90 <= lat <= 90 and -180 <= lng <= 180):
|
||||
continue
|
||||
|
||||
pin: dict[str, Any] = {
|
||||
"lat": lat,
|
||||
"lng": lng,
|
||||
"label": str(item.get("label", item.get("name", item.get("title", ""))))[:200],
|
||||
"category": str(item.get("category", "custom"))[:50],
|
||||
"color": str(item.get("color", ""))[:20],
|
||||
"description": str(item.get("description", item.get("summary", "")))[:2000],
|
||||
"source": "feed",
|
||||
"source_url": str(item.get("source_url", item.get("url", item.get("link", ""))))[:500],
|
||||
"confidence": float(item.get("confidence", 1.0)),
|
||||
}
|
||||
|
||||
entity_type = item.get("entity_type", "")
|
||||
entity_id = item.get("entity_id", "")
|
||||
if entity_type and entity_id:
|
||||
pin["entity_attachment"] = {
|
||||
"entity_type": str(entity_type),
|
||||
"entity_id": str(entity_id),
|
||||
"entity_label": str(item.get("entity_label", "")),
|
||||
}
|
||||
|
||||
pins.append(pin)
|
||||
return pins
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fetch a single layer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _fetch_layer_feed(layer: dict[str, Any]) -> None:
|
||||
"""Fetch a feed URL and replace the layer's pins."""
|
||||
layer_id = layer["id"]
|
||||
feed_url = layer["feed_url"]
|
||||
layer_name = layer.get("name", layer_id)
|
||||
|
||||
try:
|
||||
resp = requests.get(
|
||||
feed_url,
|
||||
timeout=_FETCH_TIMEOUT,
|
||||
headers={"User-Agent": "ShadowBroker-FeedIngester/1.0"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except requests.RequestException as e:
|
||||
logger.warning("Feed fetch failed for layer '%s' (%s): %s", layer_name, feed_url, e)
|
||||
return
|
||||
except (ValueError, TypeError) as e:
|
||||
logger.warning("Feed parse failed for layer '%s' (%s): %s", layer_name, feed_url, e)
|
||||
return
|
||||
|
||||
pins = _parse_feed_response(data)
|
||||
|
||||
from services.ai_pin_store import replace_layer_pins, update_layer
|
||||
count = replace_layer_pins(layer_id, pins)
|
||||
|
||||
# Update layer metadata with last_fetched timestamp
|
||||
update_layer(layer_id, feed_last_fetched=time.time())
|
||||
|
||||
_last_fetched[layer_id] = time.time()
|
||||
logger.info("Feed refresh for layer '%s': %d pins from %s", layer_name, count, feed_url)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main loop
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _ingest_loop() -> None:
|
||||
"""Daemon loop: scan for feed layers and refresh those that are due."""
|
||||
while _running:
|
||||
try:
|
||||
from services.ai_pin_store import get_feed_layers
|
||||
|
||||
layers = get_feed_layers()
|
||||
now = time.time()
|
||||
|
||||
for layer in layers:
|
||||
layer_id = layer["id"]
|
||||
interval = max(60, layer.get("feed_interval", 300))
|
||||
last = _last_fetched.get(layer_id, 0)
|
||||
|
||||
if now - last >= interval:
|
||||
try:
|
||||
_fetch_layer_feed(layer)
|
||||
except Exception as e:
|
||||
logger.warning("Feed ingestion error for layer %s: %s",
|
||||
layer.get("name", layer_id), e)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Feed ingester loop error: %s", e)
|
||||
|
||||
# Sleep in short increments so we can stop cleanly
|
||||
for _ in range(int(_CHECK_INTERVAL)):
|
||||
if not _running:
|
||||
break
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Start / stop
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def start_feed_ingester() -> None:
|
||||
"""Start the feed ingester daemon thread."""
|
||||
global _running, _thread
|
||||
if _thread and _thread.is_alive():
|
||||
return
|
||||
_running = True
|
||||
_thread = threading.Thread(target=_ingest_loop, daemon=True, name="feed-ingester")
|
||||
_thread.start()
|
||||
logger.info("Feed ingester daemon started (check interval=%ds)", _CHECK_INTERVAL)
|
||||
|
||||
|
||||
def stop_feed_ingester() -> None:
|
||||
"""Stop the feed ingester daemon."""
|
||||
global _running
|
||||
_running = False
|
||||
@@ -4,6 +4,7 @@ Central location for latest_data, source_timestamps, and the data lock.
|
||||
Every fetcher imports from here instead of maintaining its own copy.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import threading
|
||||
import logging
|
||||
from datetime import datetime
|
||||
@@ -42,6 +43,7 @@ class DashboardData(TypedDict, total=False):
|
||||
gps_jamming: List[Dict[str, Any]]
|
||||
satellites: List[Dict[str, Any]]
|
||||
satellite_source: str
|
||||
satellite_analysis: Dict[str, Any]
|
||||
prediction_markets: List[Dict[str, Any]]
|
||||
sigint: List[Dict[str, Any]]
|
||||
sigint_totals: Dict[str, Any]
|
||||
@@ -61,6 +63,12 @@ class DashboardData(TypedDict, total=False):
|
||||
fimi: Dict[str, Any]
|
||||
psk_reporter: List[Dict[str, Any]]
|
||||
correlations: List[Dict[str, Any]]
|
||||
uap_sightings: List[Dict[str, Any]]
|
||||
wastewater: List[Dict[str, Any]]
|
||||
crowdthreat: List[Dict[str, Any]]
|
||||
sar_scenes: List[Dict[str, Any]]
|
||||
sar_anomalies: List[Dict[str, Any]]
|
||||
sar_aoi_coverage: List[Dict[str, Any]]
|
||||
|
||||
|
||||
# In-memory store
|
||||
@@ -105,6 +113,12 @@ latest_data: DashboardData = {
|
||||
"fimi": {},
|
||||
"psk_reporter": [],
|
||||
"correlations": [],
|
||||
"uap_sightings": [],
|
||||
"wastewater": [],
|
||||
"crowdthreat": [],
|
||||
"sar_scenes": [],
|
||||
"sar_anomalies": [],
|
||||
"sar_aoi_coverage": [],
|
||||
}
|
||||
|
||||
# Per-source freshness timestamps
|
||||
@@ -117,9 +131,21 @@ source_freshness: dict[str, dict] = {}
|
||||
def _mark_fresh(*keys):
|
||||
"""Record the current UTC time for one or more data source keys."""
|
||||
now = datetime.utcnow().isoformat()
|
||||
global _data_version
|
||||
changed: list[tuple[str, int, int]] = [] # (layer, version, count)
|
||||
with _data_lock:
|
||||
for k in keys:
|
||||
source_timestamps[k] = now
|
||||
_layer_versions[k] = _layer_versions.get(k, 0) + 1
|
||||
# Grab entity count while we hold the lock (cheap len())
|
||||
val = latest_data.get(k)
|
||||
count = len(val) if isinstance(val, list) else (1 if val is not None else 0)
|
||||
changed.append((k, _layer_versions[k], count))
|
||||
# Publish partial fetch progress immediately so the frontend can
|
||||
# observe newly available data without waiting for the entire tier.
|
||||
_data_version += 1
|
||||
# Notify SSE listeners outside the lock to avoid deadlocks
|
||||
_notify_layer_change(changed)
|
||||
|
||||
|
||||
# Thread lock for safe reads/writes to latest_data
|
||||
@@ -129,16 +155,73 @@ _data_lock = threading.Lock()
|
||||
# Used for cheap ETag generation instead of MD5-hashing the full response.
|
||||
_data_version: int = 0
|
||||
|
||||
# Per-layer version counters — incremented only when that specific layer
|
||||
# refreshes. Used by get_layer_slice for per-layer incremental updates
|
||||
# and by the SSE stream to push targeted layer_changed notifications.
|
||||
_layer_versions: dict[str, int] = {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Layer-change notification callbacks (thread → async SSE bridge)
|
||||
# ---------------------------------------------------------------------------
|
||||
_layer_change_callbacks: list = []
|
||||
_layer_change_callbacks_lock = threading.Lock()
|
||||
|
||||
|
||||
def register_layer_change_callback(callback) -> None:
|
||||
"""Register a callback invoked on every _mark_fresh().
|
||||
|
||||
Signature: callback(layer: str, version: int, count: int)
|
||||
Called from fetcher threads — must be thread-safe.
|
||||
"""
|
||||
with _layer_change_callbacks_lock:
|
||||
_layer_change_callbacks.append(callback)
|
||||
|
||||
|
||||
def unregister_layer_change_callback(callback) -> None:
|
||||
"""Remove a previously registered callback."""
|
||||
with _layer_change_callbacks_lock:
|
||||
try:
|
||||
_layer_change_callbacks.remove(callback)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
|
||||
def _notify_layer_change(changed: list[tuple[str, int, int]]) -> None:
|
||||
"""Fire all registered callbacks for each changed layer."""
|
||||
with _layer_change_callbacks_lock:
|
||||
cbs = list(_layer_change_callbacks)
|
||||
for cb in cbs:
|
||||
for layer, version, count in changed:
|
||||
try:
|
||||
cb(layer, version, count)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def get_layer_versions() -> dict[str, int]:
|
||||
"""Return a snapshot of all per-layer version counters."""
|
||||
with _data_lock:
|
||||
return dict(_layer_versions)
|
||||
|
||||
|
||||
def get_layer_version(layer: str) -> int:
|
||||
"""Return the version counter for a single layer (0 if never refreshed)."""
|
||||
with _data_lock:
|
||||
return _layer_versions.get(layer, 0)
|
||||
|
||||
|
||||
def bump_data_version() -> None:
|
||||
"""Increment the data version counter after a fetch cycle completes."""
|
||||
global _data_version
|
||||
_data_version += 1
|
||||
with _data_lock:
|
||||
_data_version += 1
|
||||
|
||||
|
||||
def get_data_version() -> int:
|
||||
"""Return the current data version (for ETag generation)."""
|
||||
return _data_version
|
||||
with _data_lock:
|
||||
return _data_version
|
||||
|
||||
|
||||
_active_layers_version: int = 0
|
||||
@@ -156,21 +239,17 @@ def get_active_layers_version() -> int:
|
||||
|
||||
|
||||
def get_latest_data_subset(*keys: str) -> DashboardData:
|
||||
"""Return a shallow snapshot of only the requested top-level keys.
|
||||
"""Return a deep snapshot of only the requested top-level keys.
|
||||
|
||||
This avoids cloning the entire dashboard store for endpoints that only need
|
||||
a small tier-specific subset.
|
||||
a small tier-specific subset. Deep copy ensures callers cannot mutate
|
||||
nested structures (e.g. individual flight dicts) and affect the live store.
|
||||
"""
|
||||
with _data_lock:
|
||||
snap: DashboardData = {}
|
||||
for key in keys:
|
||||
value = latest_data.get(key)
|
||||
if isinstance(value, list):
|
||||
snap[key] = list(value)
|
||||
elif isinstance(value, dict):
|
||||
snap[key] = dict(value)
|
||||
else:
|
||||
snap[key] = value
|
||||
snap[key] = copy.deepcopy(value)
|
||||
return snap
|
||||
|
||||
|
||||
@@ -231,10 +310,16 @@ active_layers: dict[str, bool] = {
|
||||
"satnogs": True,
|
||||
"tinygs": True,
|
||||
"ukraine_alerts": True,
|
||||
"power_plants": False,
|
||||
"power_plants": True,
|
||||
"viirs_nightlights": False,
|
||||
"psk_reporter": True,
|
||||
"correlations": True,
|
||||
"contradictions": True,
|
||||
"uap_sightings": True,
|
||||
"wastewater": True,
|
||||
"ai_intel": True,
|
||||
"crowdthreat": True,
|
||||
"sar": True,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
"""OpenSky aircraft metadata: ICAO24 hex -> ICAO type code + friendly model.
|
||||
|
||||
OpenSky's /states/all does not include aircraft type, so OpenSky-sourced
|
||||
flights arrive with ``t`` field empty. This module bulk-loads the public
|
||||
OpenSky aircraft database (one snapshot CSV per month, ~108 MB uncompressed,
|
||||
~600k aircraft) once every 5 days and exposes a fast in-memory hex lookup.
|
||||
|
||||
The data is also useful when adsb.lol's live API is degraded: even the
|
||||
adsb.lol /v2 feed sometimes returns aircraft with empty ``t`` for newly seen
|
||||
transponders, and the lookup gracefully fills those in too.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_BUCKET_LIST_URL = (
|
||||
"https://s3.opensky-network.org/data-samples?prefix=metadata/&list-type=2"
|
||||
)
|
||||
_BUCKET_BASE = "https://s3.opensky-network.org/data-samples/"
|
||||
_S3_NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
|
||||
_REFRESH_INTERVAL_S = 5 * 24 * 3600
|
||||
_LIST_TIMEOUT_S = 30
|
||||
_DOWNLOAD_TIMEOUT_S = 600
|
||||
_USER_AGENT = (
|
||||
"ShadowBroker-OSINT/0.9.7 "
|
||||
"(+https://github.com/BigBodyCobain/Shadowbroker; "
|
||||
"contact: bigbodycobain@gmail.com)"
|
||||
)
|
||||
|
||||
_lock = threading.RLock()
|
||||
_aircraft_by_hex: dict[str, dict[str, str]] = {}
|
||||
_last_refresh = 0.0
|
||||
_in_progress = False
|
||||
|
||||
|
||||
def _latest_snapshot_key() -> str:
|
||||
"""Discover the most recent aircraft-database-complete snapshot key."""
|
||||
response = requests.get(
|
||||
_BUCKET_LIST_URL,
|
||||
timeout=_LIST_TIMEOUT_S,
|
||||
headers={"User-Agent": _USER_AGENT},
|
||||
)
|
||||
response.raise_for_status()
|
||||
root = ET.fromstring(response.text)
|
||||
keys: list[str] = []
|
||||
for content in root.iter(f"{_S3_NS}Contents"):
|
||||
key_el = content.find(f"{_S3_NS}Key")
|
||||
if key_el is None or not key_el.text:
|
||||
continue
|
||||
if "aircraft-database-complete-" in key_el.text and key_el.text.endswith(".csv"):
|
||||
keys.append(key_el.text)
|
||||
if not keys:
|
||||
raise RuntimeError("no aircraft-database-complete snapshot found in bucket listing")
|
||||
return sorted(keys)[-1]
|
||||
|
||||
|
||||
def _stream_csv_index(url: str) -> dict[str, dict[str, str]]:
|
||||
"""Stream-parse the OpenSky aircraft CSV into a hex-keyed index.
|
||||
|
||||
The CSV uses single-quote quoting, so csv.DictReader is configured with
|
||||
``quotechar="'"``. Rows are processed line-by-line via iter_lines() to
|
||||
keep memory bounded even though the file is ~108 MB.
|
||||
"""
|
||||
with requests.get(
|
||||
url,
|
||||
timeout=_DOWNLOAD_TIMEOUT_S,
|
||||
stream=True,
|
||||
headers={"User-Agent": _USER_AGENT},
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
line_iter = (
|
||||
line.decode("utf-8", errors="replace")
|
||||
for line in response.iter_lines(decode_unicode=False)
|
||||
if line
|
||||
)
|
||||
reader = csv.DictReader(line_iter, quotechar="'")
|
||||
index: dict[str, dict[str, str]] = {}
|
||||
for row in reader:
|
||||
hex_code = (row.get("icao24") or "").strip().lower()
|
||||
if not hex_code or hex_code == "000000":
|
||||
continue
|
||||
typecode = (row.get("typecode") or "").strip().upper()
|
||||
model = (row.get("model") or "").strip()
|
||||
mfr = (row.get("manufacturerName") or "").strip()
|
||||
registration = (row.get("registration") or "").strip().upper()
|
||||
operator = (row.get("operator") or "").strip()
|
||||
if not (typecode or model):
|
||||
continue
|
||||
entry: dict[str, str] = {}
|
||||
if typecode:
|
||||
entry["typecode"] = typecode
|
||||
if model:
|
||||
entry["model"] = model
|
||||
if mfr:
|
||||
entry["manufacturer"] = mfr
|
||||
if registration:
|
||||
entry["registration"] = registration
|
||||
if operator:
|
||||
entry["operator"] = operator
|
||||
index[hex_code] = entry
|
||||
return index
|
||||
|
||||
|
||||
def refresh_aircraft_database(force: bool = False) -> bool:
|
||||
"""Download the latest OpenSky aircraft snapshot and rebuild the index.
|
||||
|
||||
Returns True if a refresh was performed (success or attempted), False if
|
||||
skipped because the cache is still fresh or another refresh is in flight.
|
||||
"""
|
||||
global _last_refresh, _in_progress
|
||||
|
||||
now = time.time()
|
||||
with _lock:
|
||||
if _in_progress:
|
||||
return False
|
||||
if not force and (now - _last_refresh) < _REFRESH_INTERVAL_S and _aircraft_by_hex:
|
||||
return False
|
||||
_in_progress = True
|
||||
|
||||
try:
|
||||
started = time.time()
|
||||
key = _latest_snapshot_key()
|
||||
index = _stream_csv_index(_BUCKET_BASE + key)
|
||||
with _lock:
|
||||
_aircraft_by_hex.clear()
|
||||
_aircraft_by_hex.update(index)
|
||||
_last_refresh = time.time()
|
||||
logger.info(
|
||||
"aircraft database refreshed in %.1fs from %s: %d aircraft",
|
||||
time.time() - started,
|
||||
key,
|
||||
len(index),
|
||||
)
|
||||
return True
|
||||
except (requests.RequestException, OSError, ValueError, ET.ParseError) as exc:
|
||||
logger.warning("aircraft database refresh failed: %s", exc)
|
||||
return True
|
||||
finally:
|
||||
with _lock:
|
||||
_in_progress = False
|
||||
|
||||
|
||||
def lookup_aircraft(icao24: str) -> dict[str, str] | None:
|
||||
"""Return the metadata record for an ICAO24 hex code, or None."""
|
||||
key = (icao24 or "").strip().lower()
|
||||
if not key:
|
||||
return None
|
||||
with _lock:
|
||||
entry = _aircraft_by_hex.get(key)
|
||||
return dict(entry) if entry else None
|
||||
|
||||
|
||||
def lookup_aircraft_type(icao24: str) -> str:
|
||||
"""Return the ICAO type code (e.g. 'B738', 'GLF4') or '' if unknown."""
|
||||
entry = lookup_aircraft(icao24)
|
||||
if not entry:
|
||||
return ""
|
||||
return entry.get("typecode", "")
|
||||
|
||||
|
||||
def aircraft_database_status() -> dict[str, Any]:
|
||||
with _lock:
|
||||
return {
|
||||
"last_refresh": _last_refresh,
|
||||
"aircraft": len(_aircraft_by_hex),
|
||||
"in_progress": _in_progress,
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
"""CrowdThreat fetcher — crowdsourced global threat intelligence.
|
||||
|
||||
Polls verified threat reports from CrowdThreat's public API and normalises
|
||||
them into map-ready records with category-based icon IDs.
|
||||
|
||||
No API key required — the /threats endpoint is unauthenticated.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh, is_any_active
|
||||
from services.fetchers.retry import with_retry
|
||||
|
||||
logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
_CT_BASE = "https://backend.crowdthreat.world"
|
||||
|
||||
# CrowdThreat category_id → icon ID used on the MapLibre layer
|
||||
_CATEGORY_ICON = {
|
||||
1: "ct-security", # Security & Conflict (red)
|
||||
2: "ct-crime", # Crime & Safety (blue)
|
||||
3: "ct-aviation", # Aviation (green)
|
||||
4: "ct-maritime", # Maritime (teal)
|
||||
5: "ct-infrastructure", # Industrial & Infra (orange)
|
||||
6: "ct-special", # Special Threats (purple)
|
||||
7: "ct-social", # Social & Political (pink)
|
||||
8: "ct-other", # Other (gray)
|
||||
}
|
||||
|
||||
_CATEGORY_COLOUR = {
|
||||
1: "#ef4444", # red
|
||||
2: "#3b82f6", # blue
|
||||
3: "#22c55e", # green
|
||||
4: "#14b8a6", # teal
|
||||
5: "#f97316", # orange
|
||||
6: "#a855f7", # purple
|
||||
7: "#ec4899", # pink
|
||||
8: "#6b7280", # gray
|
||||
}
|
||||
|
||||
|
||||
@with_retry(max_retries=2, base_delay=5)
|
||||
def fetch_crowdthreat():
|
||||
"""Fetch verified threat reports from CrowdThreat public API."""
|
||||
if not is_any_active("crowdthreat"):
|
||||
return
|
||||
|
||||
try:
|
||||
resp = fetch_with_curl(f"{_CT_BASE}/threats", timeout=20)
|
||||
if not resp or resp.status_code != 200:
|
||||
logger.warning("CrowdThreat API returned %s", getattr(resp, "status_code", "None"))
|
||||
return
|
||||
|
||||
payload = resp.json()
|
||||
raw_threats = payload.get("data", {}).get("threats", [])
|
||||
if not raw_threats:
|
||||
logger.debug("CrowdThreat returned 0 threats")
|
||||
return
|
||||
|
||||
except Exception as e:
|
||||
logger.error("CrowdThreat fetch error: %s", e)
|
||||
return
|
||||
|
||||
processed = []
|
||||
for t in raw_threats:
|
||||
loc = t.get("location") or {}
|
||||
lng_lat = loc.get("lng_lat")
|
||||
if not lng_lat or len(lng_lat) < 2:
|
||||
continue
|
||||
try:
|
||||
lng = float(lng_lat[0])
|
||||
lat = float(lng_lat[1])
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
|
||||
cat = t.get("category") or {}
|
||||
cat_id = cat.get("id", 8)
|
||||
subcat = t.get("subcategory") or {}
|
||||
threat_type = t.get("type") or {}
|
||||
dates = t.get("dates") or {}
|
||||
occurred = dates.get("occurred") or {}
|
||||
reported = dates.get("reported") or {}
|
||||
|
||||
# Extract all available detail from the API response
|
||||
summary = (t.get("summary") or t.get("description") or "").strip()
|
||||
verification = (t.get("verification_status") or t.get("status") or "").strip()
|
||||
country_obj = loc.get("country") or {}
|
||||
country = country_obj.get("name", "") if isinstance(country_obj, dict) else str(country_obj or "")
|
||||
media = t.get("media") or t.get("images") or t.get("attachments") or []
|
||||
source_url = t.get("source_url") or t.get("url") or t.get("link") or ""
|
||||
severity = t.get("severity") or t.get("severity_level") or t.get("risk_level") or ""
|
||||
votes = t.get("votes") or t.get("upvotes") or 0
|
||||
reporter = t.get("user") or t.get("reporter") or {}
|
||||
reporter_name = reporter.get("name", "") if isinstance(reporter, dict) else ""
|
||||
|
||||
processed.append({
|
||||
"id": t.get("id"),
|
||||
"title": t.get("title", ""),
|
||||
"summary": summary[:500] if summary else "",
|
||||
"lat": lat,
|
||||
"lng": lng,
|
||||
"address": loc.get("name", ""),
|
||||
"city": loc.get("city", ""),
|
||||
"country": country,
|
||||
"category": cat.get("name", "Other"),
|
||||
"category_id": cat_id,
|
||||
"category_colour": _CATEGORY_COLOUR.get(cat_id, "#6b7280"),
|
||||
"subcategory": subcat.get("name", ""),
|
||||
"threat_type": threat_type.get("name", ""),
|
||||
"icon_id": _CATEGORY_ICON.get(cat_id, "ct-other"),
|
||||
"occurred": occurred.get("raw", ""),
|
||||
"occurred_iso": occurred.get("iso", ""),
|
||||
"timeago": occurred.get("timeago", ""),
|
||||
"reported": reported.get("raw", ""),
|
||||
"verification": verification,
|
||||
"severity": str(severity),
|
||||
"source_url": source_url,
|
||||
"media_urls": [m.get("url") or m for m in media[:3]] if isinstance(media, list) else [],
|
||||
"votes": int(votes) if votes else 0,
|
||||
"reporter": reporter_name,
|
||||
"source": "CrowdThreat",
|
||||
})
|
||||
|
||||
logger.info("CrowdThreat: fetched %d verified threats", len(processed))
|
||||
|
||||
with _data_lock:
|
||||
latest_data["crowdthreat"] = processed
|
||||
_mark_fresh("crowdthreat")
|
||||
@@ -1,14 +1,19 @@
|
||||
"""Earth-observation fetchers — earthquakes, FIRMS fires, space weather, weather radar,
|
||||
severe weather alerts, air quality, volcanoes."""
|
||||
|
||||
import concurrent.futures
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
import heapq
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
@@ -596,3 +601,852 @@ def fetch_viirs_change_nodes():
|
||||
if nodes:
|
||||
_mark_fresh("viirs_change_nodes")
|
||||
logger.info(f"VIIRS change nodes: {len(nodes)} nodes from {len(_VIIRS_AOIS)} AOIs")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# UAP Sightings (NUFORC — National UAP Reporting Center)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Shape → canonical category mapping for consistent frontend filtering
|
||||
_UAP_SHAPE_MAP = {
|
||||
"light": "light", "fireball": "fireball", "orb": "orb",
|
||||
"sphere": "orb", "circle": "orb", "oval": "orb", "egg": "orb",
|
||||
"triangle": "triangle", "delta": "triangle", "chevron": "triangle",
|
||||
"boomerang": "triangle",
|
||||
"cigar": "cigar", "cylinder": "cigar", "tube": "cigar",
|
||||
"disk": "disk", "disc": "disk", "saucer": "disk",
|
||||
"diamond": "diamond", "cone": "diamond", "cross": "diamond",
|
||||
"rectangle": "rectangle", "square": "rectangle",
|
||||
"formation": "formation", "cluster": "formation",
|
||||
"changing": "changing", "flash": "flash", "star": "light",
|
||||
"tic-tac": "tic-tac", "tic tac": "tic-tac",
|
||||
}
|
||||
|
||||
# US state → approximate centroid for coarse geocoding when city lookup fails
|
||||
_US_STATE_COORDS: dict[str, tuple[float, float]] = {
|
||||
"AL": (32.8, -86.8), "AK": (64.2, -152.5), "AZ": (34.0, -111.1),
|
||||
"AR": (35.2, -91.8), "CA": (36.8, -119.4), "CO": (39.6, -105.3),
|
||||
"CT": (41.6, -72.7), "DE": (39.3, -75.5), "FL": (27.8, -81.8),
|
||||
"GA": (32.7, -83.5), "HI": (19.9, -155.6), "ID": (44.1, -114.7),
|
||||
"IL": (40.3, -89.0), "IN": (40.3, -86.1), "IA": (42.0, -93.2),
|
||||
"KS": (39.0, -98.5), "KY": (37.8, -84.3), "LA": (31.2, -92.5),
|
||||
"ME": (45.3, -69.4), "MD": (39.0, -76.6), "MA": (42.4, -71.4),
|
||||
"MI": (44.3, -85.6), "MN": (46.7, -94.7), "MS": (32.7, -89.5),
|
||||
"MO": (38.6, -91.8), "MT": (46.8, -110.4), "NE": (41.5, -99.9),
|
||||
"NV": (38.8, -116.4), "NH": (43.2, -71.6), "NJ": (40.1, -74.4),
|
||||
"NM": (34.5, -106.0), "NY": (43.0, -75.0), "NC": (35.6, -79.8),
|
||||
"ND": (47.5, -100.5), "OH": (40.4, -82.9), "OK": (35.0, -97.1),
|
||||
"OR": (43.8, -120.6), "PA": (41.2, -77.2), "RI": (41.6, -71.5),
|
||||
"SC": (33.8, -81.2), "SD": (43.9, -99.4), "TN": (35.5, -86.6),
|
||||
"TX": (31.0, -97.6), "UT": (39.3, -111.1), "VT": (44.6, -72.6),
|
||||
"VA": (37.4, -78.7), "WA": (47.4, -120.7), "WV": (38.6, -80.6),
|
||||
"WI": (43.8, -88.8), "WY": (43.1, -107.6), "DC": (38.9, -77.0),
|
||||
}
|
||||
|
||||
|
||||
def _normalize_uap_shape(raw: str) -> str:
|
||||
"""Normalize a raw NUFORC shape string to a canonical category."""
|
||||
key = raw.strip().lower()
|
||||
return _UAP_SHAPE_MAP.get(key, "unknown")
|
||||
|
||||
|
||||
def _reverse_geocode_state(lat: float, lng: float) -> tuple[str, str]:
|
||||
"""Best-effort reverse-geocode a lat/lng to (state_abbr, country).
|
||||
|
||||
Uses the _US_STATE_COORDS centroid table for fast approximate matching.
|
||||
Returns ('', 'Unknown') if no close match is found.
|
||||
"""
|
||||
best_state = ""
|
||||
best_dist = 999.0
|
||||
for st, (slat, slng) in _US_STATE_COORDS.items():
|
||||
d = ((lat - slat) ** 2 + (lng - slng) ** 2) ** 0.5
|
||||
if d < best_dist:
|
||||
best_dist = d
|
||||
best_state = st
|
||||
if best_dist < 5.0: # ~5 degrees tolerance
|
||||
return best_state, "US"
|
||||
return "", "Unknown"
|
||||
|
||||
|
||||
# ── NUFORC Mapbox Tilequery API ─────────────────────────────────────────
|
||||
# NUFORC's website switched to a JS-rendered Mapbox GL map. The old HTML
|
||||
# table scraper is defunct. We now query the Mapbox Tilequery API against
|
||||
# NUFORC's public tileset to get precise sighting coordinates.
|
||||
#
|
||||
# Tileset: nuforc.cmm18aqea06bu1mmselhpnano-0ce5v
|
||||
# Layer: Sightings Fields: Count, From, To, LinkLat, LinkLon
|
||||
#
|
||||
# We sample a grid of points across the US/world with a 100 km radius and
|
||||
# filter to sightings within the last 60 days.
|
||||
|
||||
_NUFORC_TILESET = "nuforc.cmm18aqea06bu1mmselhpnano-0ce5v"
|
||||
_NUFORC_TOKEN = os.environ.get("NUFORC_MAPBOX_TOKEN", "").strip()
|
||||
_NUFORC_RADIUS_M = 200_000 # 200 km query radius
|
||||
_NUFORC_LIMIT = 50 # max features per tilequery call
|
||||
_NUFORC_RECENT_DAYS = int(os.environ.get("NUFORC_RECENT_DAYS", "60"))
|
||||
_NUFORC_GEOCODE_WORKERS = max(1, int(os.environ.get("NUFORC_GEOCODE_WORKERS", "1")))
|
||||
# Photon (Komoot) is more lenient than Nominatim — ~200ms per query in
|
||||
# practice, so a 0.3s spacing keeps us well under any soft throttle while
|
||||
# still rebuilding a full 12-month window in ~10 minutes.
|
||||
_NUFORC_GEOCODE_SPACING_S = float(os.environ.get("NUFORC_GEOCODE_SPACING_S", "0.3"))
|
||||
_NUFORC_DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"
|
||||
_NUFORC_SIGHTINGS_CACHE_FILE = _NUFORC_DATA_DIR / "nuforc_recent_sightings.json"
|
||||
_NUFORC_LOCATION_CACHE_FILE = _NUFORC_DATA_DIR / "nuforc_location_cache.json"
|
||||
|
||||
# Live NUFORC databank scraping (wpDataTables server-side AJAX).
|
||||
# The HuggingFace mirror froze at 2023-12-20, so we pull directly from
|
||||
# nuforc.org's monthly sub-index. Each month page embeds a wdtNonce we
|
||||
# must extract, then POST to admin-ajax.php to get the DataTables JSON.
|
||||
_NUFORC_LIVE_INDEX_URL = "https://nuforc.org/subndx/?id=e{yyyymm}"
|
||||
_NUFORC_LIVE_AJAX_URL = (
|
||||
"https://nuforc.org/wp-admin/admin-ajax.php"
|
||||
"?action=get_wdtable&table_id=1&wdt_var1=YearMonth&wdt_var2={yyyymm}"
|
||||
)
|
||||
_NUFORC_LIVE_NONCE_RE = re.compile(
|
||||
r'id=["\']wdtNonceFrontendServerSide_1["\'][^>]*value=["\']([a-f0-9]+)["\']'
|
||||
)
|
||||
_NUFORC_LIVE_SIGHTING_ID_RE = re.compile(r"id=(\d+)")
|
||||
_NUFORC_LIVE_USER_AGENT = "Mozilla/5.0 (ShadowBroker-OSINT NUFORC-fetcher)"
|
||||
_NUFORC_LIVE_SESSION_COOKIES = _NUFORC_DATA_DIR / "nuforc_session.cookies"
|
||||
|
||||
# Sample grid covering continental US, Alaska, Hawaii, Canada, UK, Australia
|
||||
_TILEQUERY_GRID: list[tuple[float, float]] = [
|
||||
# Continental US — ~4° spacing (lon, lat)
|
||||
(-122.4, 37.8), (-118.2, 34.1), (-112.1, 33.4), (-104.9, 39.7),
|
||||
(-95.4, 29.8), (-96.8, 32.8), (-87.6, 41.9), (-84.4, 33.7),
|
||||
(-81.7, 41.5), (-80.2, 25.8), (-77.0, 38.9), (-74.0, 40.7),
|
||||
(-71.1, 42.4), (-90.2, 38.6), (-93.3, 44.9), (-111.9, 40.8),
|
||||
(-122.7, 45.5), (-86.2, 39.8), (-106.6, 35.1), (-73.9, 43.2),
|
||||
(-76.6, 39.3), (-97.5, 35.5), (-83.0, 42.3), (-117.2, 32.7),
|
||||
(-82.5, 28.0), (-78.6, 35.8), (-90.1, 30.0), (-71.4, 41.8),
|
||||
# Alaska, Hawaii
|
||||
(-149.9, 61.2), (-155.5, 19.9),
|
||||
# Canada
|
||||
(-79.4, 43.7), (-123.1, 49.3), (-73.6, 45.5),
|
||||
# UK & Europe
|
||||
(-0.1, 51.5), (-3.2, 55.9),
|
||||
# Australia
|
||||
(151.2, -33.9), (144.9, -37.8),
|
||||
]
|
||||
|
||||
|
||||
def _fetch_nuforc_tilequery(lng: float, lat: float) -> list[dict]:
|
||||
"""Query NUFORC Mapbox tileset around a single point, return raw features."""
|
||||
if not _NUFORC_TOKEN:
|
||||
return []
|
||||
url = (
|
||||
f"https://api.mapbox.com/v4/{_NUFORC_TILESET}/tilequery/"
|
||||
f"{lng},{lat}.json"
|
||||
f"?radius={_NUFORC_RADIUS_M}&limit={_NUFORC_LIMIT}"
|
||||
f"&access_token={_NUFORC_TOKEN}"
|
||||
)
|
||||
try:
|
||||
resp = fetch_with_curl(url, timeout=12)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
return data.get("features", [])
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def _parse_nuforc_tile_date(value: str) -> datetime | None:
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
raw = raw.replace("T", " ")
|
||||
raw = re.sub(r"\s+local$", "", raw, flags=re.IGNORECASE)
|
||||
raw = re.sub(r"\s+utc$", "", raw, flags=re.IGNORECASE)
|
||||
for fmt in (
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%Y-%m-%d %H:%M",
|
||||
"%Y-%m-%d",
|
||||
"%m/%d/%Y %H:%M",
|
||||
"%m/%d/%Y",
|
||||
):
|
||||
try:
|
||||
return datetime.strptime(raw, fmt)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
match = re.match(r"^(\d{4}-\d{2}-\d{2})", raw)
|
||||
if match:
|
||||
try:
|
||||
return datetime.strptime(match.group(1), "%Y-%m-%d")
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _load_nuforc_sightings_cache(*, force_refresh: bool = False) -> list[dict] | None:
|
||||
if force_refresh or not _NUFORC_SIGHTINGS_CACHE_FILE.exists():
|
||||
return None
|
||||
try:
|
||||
raw = json.loads(_NUFORC_SIGHTINGS_CACHE_FILE.read_text(encoding="utf-8"))
|
||||
built = raw.get("built", "")
|
||||
built_dt = datetime.fromisoformat(built) if built else None
|
||||
if built_dt is None:
|
||||
return None
|
||||
if (datetime.utcnow() - built_dt).total_seconds() > 86400:
|
||||
return None
|
||||
sightings = raw.get("sightings")
|
||||
if isinstance(sightings, list):
|
||||
if len(sightings) <= 0:
|
||||
logger.info("UAP sightings: cache is fresh but empty; rebuilding")
|
||||
return None
|
||||
logger.info(
|
||||
"UAP sightings: loaded %d cached reports from %s",
|
||||
len(sightings),
|
||||
built,
|
||||
)
|
||||
return sightings
|
||||
except Exception as e:
|
||||
logger.warning("UAP sightings: cache load error: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def _save_nuforc_sightings_cache(sightings: list[dict]) -> None:
|
||||
if not sightings:
|
||||
logger.warning("UAP sightings: refusing to save empty daily cache")
|
||||
return
|
||||
try:
|
||||
_NUFORC_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"built": datetime.utcnow().isoformat(),
|
||||
"count": len(sightings),
|
||||
"sightings": sightings,
|
||||
}
|
||||
_NUFORC_SIGHTINGS_CACHE_FILE.write_text(
|
||||
json.dumps(payload, separators=(",", ":")),
|
||||
encoding="utf-8",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("UAP sightings: cache save error: %s", e)
|
||||
|
||||
|
||||
def _load_nuforc_location_cache() -> dict[str, list[float] | None]:
|
||||
if not _NUFORC_LOCATION_CACHE_FILE.exists():
|
||||
return {}
|
||||
try:
|
||||
raw = json.loads(_NUFORC_LOCATION_CACHE_FILE.read_text(encoding="utf-8"))
|
||||
if not isinstance(raw, dict):
|
||||
return {}
|
||||
cache: dict[str, list[float] | None] = {}
|
||||
for key, value in raw.items():
|
||||
if not isinstance(key, str):
|
||||
continue
|
||||
if (
|
||||
isinstance(value, list)
|
||||
and len(value) == 2
|
||||
and all(isinstance(v, (int, float)) for v in value)
|
||||
):
|
||||
cache[key] = [float(value[0]), float(value[1])]
|
||||
elif value is None:
|
||||
cache[key] = None
|
||||
return cache
|
||||
except Exception as e:
|
||||
logger.warning("UAP sightings: location cache load error: %s", e)
|
||||
return {}
|
||||
|
||||
|
||||
def _save_nuforc_location_cache(cache: dict[str, list[float] | None]) -> None:
|
||||
try:
|
||||
_NUFORC_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
_NUFORC_LOCATION_CACHE_FILE.write_text(
|
||||
json.dumps(cache, separators=(",", ":")),
|
||||
encoding="utf-8",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("UAP sightings: location cache save error: %s", e)
|
||||
|
||||
|
||||
def _normalize_uap_location(raw: str) -> str:
|
||||
return re.sub(r"\s+", " ", str(raw or "").strip())
|
||||
|
||||
|
||||
def _uap_country_from_location(location: str, state: str) -> str:
|
||||
if state:
|
||||
return "US"
|
||||
upper = location.upper()
|
||||
if "USA" in upper or "UNITED STATES" in upper:
|
||||
return "US"
|
||||
parts = [part.strip() for part in location.split(",") if part.strip()]
|
||||
if not parts:
|
||||
return "Unknown"
|
||||
country = parts[-1]
|
||||
return country.upper() if len(country) == 2 else country
|
||||
|
||||
|
||||
_US_COUNTRY_ALIASES = {
|
||||
"", "USA", "US", "U.S.", "U.S.A.",
|
||||
"UNITED STATES", "UNITED STATES OF AMERICA",
|
||||
}
|
||||
|
||||
|
||||
def _uap_geocode_candidates(
|
||||
location: str, city: str, state: str, country: str = ""
|
||||
) -> list[str]:
|
||||
"""Build geocode query candidates in priority order.
|
||||
|
||||
NUFORC's live databank is international, so we must query with the
|
||||
actual country first. Only when the country is empty or explicitly US
|
||||
do we fall back to the legacy USA-assumption behavior.
|
||||
"""
|
||||
candidates: list[str] = []
|
||||
c = (country or "").strip()
|
||||
c_upper = c.upper()
|
||||
is_us = c_upper in _US_COUNTRY_ALIASES
|
||||
|
||||
if not is_us:
|
||||
# Non-US: try country-qualified queries first to prevent the
|
||||
# geocoder from fuzzy-matching to a same-named US city.
|
||||
if city and state:
|
||||
candidates.append(f"{city}, {state}, {c}")
|
||||
if city:
|
||||
candidates.append(f"{city}, {c}")
|
||||
if city and state:
|
||||
candidates.append(f"{city}, {state}")
|
||||
if city:
|
||||
candidates.append(city)
|
||||
else:
|
||||
if city and state:
|
||||
candidates.append(f"{city}, {state}, USA")
|
||||
candidates.append(f"{city}, {state}")
|
||||
if city:
|
||||
candidates.append(city)
|
||||
|
||||
normalized = _normalize_uap_location(location)
|
||||
if normalized:
|
||||
candidates.append(normalized)
|
||||
parts = [part.strip() for part in normalized.split(",") if part.strip()]
|
||||
if len(parts) >= 2:
|
||||
candidates.append(", ".join(parts[:2]))
|
||||
if parts:
|
||||
candidates.append(parts[0])
|
||||
|
||||
deduped: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for candidate in candidates:
|
||||
key = candidate.lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
deduped.append(candidate)
|
||||
return deduped
|
||||
|
||||
|
||||
def _photon_lookup(query: str) -> list[float] | None:
|
||||
"""Query Komoot's public Photon instance (OSM-based, no API key).
|
||||
|
||||
Returns [lat, lng] on success, None on any failure. We bypass the
|
||||
shared search_geocode() helper on purpose: it falls back to an
|
||||
airport-name token matcher on failure that confidently returns
|
||||
completely wrong coordinates, which poisoned the cache for years.
|
||||
"""
|
||||
from urllib.parse import urlencode
|
||||
|
||||
params = urlencode({"q": query, "limit": 1})
|
||||
url = f"https://photon.komoot.io/api?{params}"
|
||||
try:
|
||||
res = fetch_with_curl(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": "ShadowBroker-OSINT/1.0 (NUFORC-UAP-layer)",
|
||||
"Accept-Language": "en",
|
||||
},
|
||||
timeout=10,
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
if not res or res.status_code != 200:
|
||||
return None
|
||||
try:
|
||||
payload = res.json()
|
||||
except Exception:
|
||||
return None
|
||||
features = (payload or {}).get("features") or []
|
||||
if not features:
|
||||
return None
|
||||
try:
|
||||
# GeoJSON order is [lng, lat] — flip to our [lat, lng] convention.
|
||||
coords = features[0]["geometry"]["coordinates"]
|
||||
return [float(coords[1]), float(coords[0])]
|
||||
except (KeyError, IndexError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _geocode_uap_location(
|
||||
location: str, city: str, state: str, country: str = ""
|
||||
) -> list[float] | None:
|
||||
"""Resolve a NUFORC sighting location to [lat, lng] via Photon.
|
||||
|
||||
Returns None on failure. The caller caches None alongside real hits
|
||||
so we don't retry unresolvable queries every run.
|
||||
"""
|
||||
for query in _uap_geocode_candidates(location, city, state, country):
|
||||
coords = _photon_lookup(query)
|
||||
if coords:
|
||||
return coords
|
||||
return None
|
||||
|
||||
|
||||
def _build_uap_sighting_id(row: dict, occurred: str, location: str) -> str:
|
||||
raw_id = str(row.get("Sighting", "") or row.get("sighting", "")).strip()
|
||||
if raw_id:
|
||||
return raw_id
|
||||
digest = hashlib.sha1(
|
||||
f"{occurred}|{location}|{row.get('Summary', '')}|{row.get('Text', '')}".encode("utf-8", "ignore")
|
||||
).hexdigest()[:12]
|
||||
return f"NUFORC-{digest}"
|
||||
|
||||
|
||||
def _nuforc_months_for_window(days: int) -> list[str]:
|
||||
"""Enumerate YYYYMM strings covering the rolling `days`-day window.
|
||||
|
||||
Returned newest first. Always includes the current month even if the
|
||||
window technically starts later, because new reports land there.
|
||||
"""
|
||||
today = datetime.utcnow().date()
|
||||
start = today - timedelta(days=days)
|
||||
months: list[str] = []
|
||||
cur = today.replace(day=1)
|
||||
start_floor = start.replace(day=1)
|
||||
while cur >= start_floor:
|
||||
months.append(cur.strftime("%Y%m"))
|
||||
if cur.month == 1:
|
||||
cur = cur.replace(year=cur.year - 1, month=12)
|
||||
else:
|
||||
cur = cur.replace(month=cur.month - 1)
|
||||
return months
|
||||
|
||||
|
||||
def _nuforc_fetch_month_live(yyyymm: str, cookie_jar: Path) -> list[dict]:
|
||||
"""Pull one month of NUFORC sightings via the live wpDataTables AJAX.
|
||||
|
||||
Returns a list of raw row dicts with the fields we care about:
|
||||
id, occurred (YYYY-MM-DD), posted (YYYY-MM-DD), city, state, country,
|
||||
shape_raw, summary, explanation. Empty list on any failure — caller
|
||||
decides whether a failure is fatal.
|
||||
"""
|
||||
from services.fetchers.nuforc_enrichment import _parse_date
|
||||
|
||||
curl_bin = shutil.which("curl") or "curl"
|
||||
index_url = _NUFORC_LIVE_INDEX_URL.format(yyyymm=yyyymm)
|
||||
ajax_url = _NUFORC_LIVE_AJAX_URL.format(yyyymm=yyyymm)
|
||||
|
||||
# Step 1: GET the month index to capture session cookies + fresh nonce.
|
||||
try:
|
||||
index_res = subprocess.run(
|
||||
[
|
||||
curl_bin, "-sL",
|
||||
"-A", _NUFORC_LIVE_USER_AGENT,
|
||||
"-c", str(cookie_jar),
|
||||
"-b", str(cookie_jar),
|
||||
index_url,
|
||||
],
|
||||
capture_output=True, text=True, timeout=60,
|
||||
encoding="utf-8", errors="replace",
|
||||
)
|
||||
except (subprocess.SubprocessError, OSError) as e:
|
||||
logger.warning("NUFORC live: index fetch failed for %s: %s", yyyymm, e)
|
||||
return []
|
||||
if index_res.returncode != 0 or not index_res.stdout:
|
||||
logger.warning(
|
||||
"NUFORC live: index fetch exit=%s for %s", index_res.returncode, yyyymm,
|
||||
)
|
||||
return []
|
||||
nonce_match = _NUFORC_LIVE_NONCE_RE.search(index_res.stdout)
|
||||
if not nonce_match:
|
||||
logger.warning("NUFORC live: wdtNonce not found on index page for %s", yyyymm)
|
||||
return []
|
||||
nonce = nonce_match.group(1)
|
||||
|
||||
# Step 2: POST to admin-ajax.php with length=-1 to pull the whole month.
|
||||
post_data = (
|
||||
"draw=1"
|
||||
"&columns%5B0%5D%5Bdata%5D=0&columns%5B0%5D%5Bsearchable%5D=true&columns%5B0%5D%5Borderable%5D=false"
|
||||
"&columns%5B1%5D%5Bdata%5D=1&columns%5B1%5D%5Bsearchable%5D=true&columns%5B1%5D%5Borderable%5D=true"
|
||||
"&order%5B0%5D%5Bcolumn%5D=1&order%5B0%5D%5Bdir%5D=desc"
|
||||
"&start=0&length=-1"
|
||||
"&search%5Bvalue%5D=&search%5Bregex%5D=false"
|
||||
f"&wdtNonce={nonce}"
|
||||
)
|
||||
try:
|
||||
ajax_res = subprocess.run(
|
||||
[
|
||||
curl_bin, "-sL",
|
||||
"-A", _NUFORC_LIVE_USER_AGENT,
|
||||
"-c", str(cookie_jar),
|
||||
"-b", str(cookie_jar),
|
||||
"-X", "POST",
|
||||
"-H", f"Referer: {index_url}",
|
||||
"-H", "X-Requested-With: XMLHttpRequest",
|
||||
"-H", "Content-Type: application/x-www-form-urlencoded",
|
||||
"--data", post_data,
|
||||
ajax_url,
|
||||
],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
encoding="utf-8", errors="replace",
|
||||
)
|
||||
except (subprocess.SubprocessError, OSError) as e:
|
||||
logger.warning("NUFORC live: ajax fetch failed for %s: %s", yyyymm, e)
|
||||
return []
|
||||
if ajax_res.returncode != 0 or not ajax_res.stdout:
|
||||
logger.warning(
|
||||
"NUFORC live: ajax fetch exit=%s for %s", ajax_res.returncode, yyyymm,
|
||||
)
|
||||
return []
|
||||
try:
|
||||
payload = json.loads(ajax_res.stdout)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning("NUFORC live: ajax JSON decode failed for %s: %s", yyyymm, e)
|
||||
return []
|
||||
|
||||
raw_rows = payload.get("data") or []
|
||||
out: list[dict] = []
|
||||
for raw in raw_rows:
|
||||
if not isinstance(raw, list) or len(raw) < 8:
|
||||
continue
|
||||
link_html = str(raw[0] or "")
|
||||
occurred_raw = str(raw[1] or "")
|
||||
city = str(raw[2] or "").strip()
|
||||
state = str(raw[3] or "").strip()
|
||||
country = str(raw[4] or "").strip()
|
||||
shape_raw = (str(raw[5] or "").strip() or "Unknown")
|
||||
summary = str(raw[6] or "").strip()
|
||||
reported_raw = str(raw[7] or "")
|
||||
explanation = str(raw[9] or "").strip() if len(raw) > 9 and raw[9] else ""
|
||||
|
||||
occurred_ymd = _parse_date(occurred_raw)
|
||||
if not occurred_ymd:
|
||||
continue
|
||||
if not city and not state and not country:
|
||||
continue
|
||||
|
||||
id_match = _NUFORC_LIVE_SIGHTING_ID_RE.search(link_html)
|
||||
if id_match:
|
||||
sighting_id = f"NUFORC-{id_match.group(1)}"
|
||||
else:
|
||||
digest = hashlib.sha1(
|
||||
f"{occurred_ymd}|{city}|{state}|{summary}".encode("utf-8", "ignore")
|
||||
).hexdigest()[:12]
|
||||
sighting_id = f"NUFORC-{digest}"
|
||||
|
||||
if summary and len(summary) > 280:
|
||||
summary = summary[:277] + "..."
|
||||
if not summary:
|
||||
summary = "Sighting reported"
|
||||
|
||||
out.append({
|
||||
"id": sighting_id,
|
||||
"occurred": occurred_ymd,
|
||||
"posted": _parse_date(reported_raw) or occurred_ymd,
|
||||
"city": city,
|
||||
"state": state,
|
||||
"country": country,
|
||||
"shape_raw": shape_raw,
|
||||
"summary": summary,
|
||||
"explanation": explanation,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _build_recent_uap_sightings() -> list[dict]:
|
||||
"""Build the rolling 1-year UAP sightings layer from live NUFORC data.
|
||||
|
||||
Hits nuforc.org's public sub-index once per month in the window, drops
|
||||
anything outside the exact day-precision cutoff, dedupes by sighting id,
|
||||
geocodes city+state via the existing location cache, and returns rows
|
||||
keyed to the same schema the frontend already renders.
|
||||
"""
|
||||
cutoff_dt = datetime.utcnow() - timedelta(days=_NUFORC_RECENT_DAYS)
|
||||
cutoff_str = cutoff_dt.strftime("%Y-%m-%d")
|
||||
months = _nuforc_months_for_window(_NUFORC_RECENT_DAYS)
|
||||
|
||||
try:
|
||||
_NUFORC_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
rows: list[dict] = []
|
||||
locations: dict[str, tuple[str, str]] = {}
|
||||
seen_ids: set[str] = set()
|
||||
total_pulled = 0
|
||||
months_with_data = 0
|
||||
|
||||
for yyyymm in months:
|
||||
month_rows = _nuforc_fetch_month_live(yyyymm, _NUFORC_LIVE_SESSION_COOKIES)
|
||||
if month_rows:
|
||||
months_with_data += 1
|
||||
total_pulled += len(month_rows)
|
||||
for row in month_rows:
|
||||
if row["occurred"] < cutoff_str:
|
||||
continue
|
||||
if row["id"] in seen_ids:
|
||||
continue
|
||||
seen_ids.add(row["id"])
|
||||
|
||||
# Build the geocode key as "City, State, Country" to match the
|
||||
# existing 3,000+ entry location cache (format: "Toronto, ON, Canada").
|
||||
parts = [row["city"], row["state"], row["country"]]
|
||||
location = _normalize_uap_location(
|
||||
", ".join(p for p in parts if p) if any(parts) else ""
|
||||
)
|
||||
if not location:
|
||||
continue
|
||||
|
||||
row["location"] = location
|
||||
locations.setdefault(location, (row["city"], row["state"], row["country"]))
|
||||
row["shape"] = (
|
||||
_normalize_uap_shape(row["shape_raw"])
|
||||
if row["shape_raw"] != "Unknown"
|
||||
else "unknown"
|
||||
)
|
||||
if not row["country"]:
|
||||
row["country"] = _uap_country_from_location(location, row["state"])
|
||||
rows.append(row)
|
||||
|
||||
# Clean up the cookie jar — we don't reuse it across runs.
|
||||
try:
|
||||
if _NUFORC_LIVE_SESSION_COOKIES.exists():
|
||||
_NUFORC_LIVE_SESSION_COOKIES.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Source-integrity canary: if the upstream plugin changed its
|
||||
# DataTables schema or the wdtNonce regex is stale, total_pulled
|
||||
# collapses to ~0 without any HTTP error. assert_canary logs a loud
|
||||
# ERROR so the failure is visible in the health registry and the
|
||||
# daily refresh log, instead of silently serving a stale cache.
|
||||
from services.slo import assert_canary
|
||||
assert_canary("uap_sightings", total_pulled)
|
||||
|
||||
if not rows:
|
||||
raise RuntimeError(
|
||||
f"NUFORC live: zero rows pulled across {len(months)} months "
|
||||
f"(months_with_data={months_with_data})"
|
||||
)
|
||||
|
||||
from services.geocode_validate import coord_in_country
|
||||
|
||||
location_cache = _load_nuforc_location_cache()
|
||||
missing_locations = [location for location in locations if location not in location_cache]
|
||||
if missing_locations:
|
||||
logger.info(
|
||||
"UAP sightings: geocoding %d new locations (throttled at %.1fs spacing)",
|
||||
len(missing_locations),
|
||||
_NUFORC_GEOCODE_SPACING_S,
|
||||
)
|
||||
# Sequential with spacing — Photon is fast and lenient but we
|
||||
# stay sub-second to be polite. Incremental cache saves every 50
|
||||
# hits keep long runs resumable.
|
||||
resolved = 0
|
||||
bbox_rejected = 0
|
||||
save_every = 50
|
||||
for idx, location in enumerate(missing_locations):
|
||||
city, state, country = locations[location]
|
||||
coords = None
|
||||
try:
|
||||
coords = _geocode_uap_location(location, city, state, country)
|
||||
except Exception:
|
||||
coords = None
|
||||
|
||||
# Country-bbox post-filter: reject namesake collisions like
|
||||
# "Milan, WI" landing in Milan, Italy. Unknown countries
|
||||
# (bbox not registered) are passed through unchanged.
|
||||
if coords and country:
|
||||
inside = coord_in_country(coords[0], coords[1], country)
|
||||
if inside is False:
|
||||
logger.warning(
|
||||
"UAP sightings: bbox reject %r -> (%.3f, %.3f) not in %s",
|
||||
location, coords[0], coords[1], country,
|
||||
)
|
||||
coords = None
|
||||
bbox_rejected += 1
|
||||
|
||||
location_cache[location] = coords
|
||||
if coords:
|
||||
resolved += 1
|
||||
|
||||
if (idx + 1) % save_every == 0:
|
||||
_save_nuforc_location_cache(location_cache)
|
||||
logger.info(
|
||||
"UAP sightings: geocoded %d/%d (%d resolved, %d bbox-rejected)",
|
||||
idx + 1, len(missing_locations), resolved, bbox_rejected,
|
||||
)
|
||||
if idx + 1 < len(missing_locations):
|
||||
time.sleep(_NUFORC_GEOCODE_SPACING_S)
|
||||
_save_nuforc_location_cache(location_cache)
|
||||
logger.info(
|
||||
"UAP sightings: geocoding complete — %d/%d resolved, %d bbox-rejected",
|
||||
resolved, len(missing_locations), bbox_rejected,
|
||||
)
|
||||
|
||||
sightings: list[dict] = []
|
||||
skipped_unmapped = 0
|
||||
skipped_bbox = 0
|
||||
for row in rows:
|
||||
coords = location_cache.get(row["location"])
|
||||
if not coords:
|
||||
skipped_unmapped += 1
|
||||
continue
|
||||
# Apply bbox filter to pre-existing cache entries too — this
|
||||
# cleans up the ~1-2% of cached coords that pre-dated the bbox
|
||||
# check without requiring a full cache rebuild.
|
||||
if row.get("country"):
|
||||
inside = coord_in_country(coords[0], coords[1], row["country"])
|
||||
if inside is False:
|
||||
skipped_bbox += 1
|
||||
continue
|
||||
sightings.append(
|
||||
{
|
||||
"id": row["id"],
|
||||
"date_time": row["occurred"],
|
||||
"city": row["city"],
|
||||
"state": row["state"],
|
||||
"country": row["country"],
|
||||
"shape": row["shape"],
|
||||
"shape_raw": row["shape_raw"],
|
||||
"duration": row.get("duration", ""),
|
||||
"summary": row["summary"],
|
||||
"posted": row["posted"],
|
||||
"lat": float(coords[0]),
|
||||
"lng": float(coords[1]),
|
||||
"count": 1,
|
||||
"source": "NUFORC",
|
||||
}
|
||||
)
|
||||
if row.get("explanation"):
|
||||
sightings[-1]["explanation"] = row["explanation"]
|
||||
|
||||
sightings.sort(
|
||||
key=lambda sighting: (
|
||||
sighting.get("date_time", ""),
|
||||
sighting.get("posted", ""),
|
||||
str(sighting.get("id", "")),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
logger.info(
|
||||
"UAP sightings: %d mapped reports from %d rows across %d months "
|
||||
"(cutoff %s, %d unmapped, %d bbox-rejected)",
|
||||
len(sightings),
|
||||
total_pulled,
|
||||
len(months),
|
||||
cutoff_str,
|
||||
skipped_unmapped,
|
||||
skipped_bbox,
|
||||
)
|
||||
return sightings
|
||||
|
||||
|
||||
@with_retry(max_retries=1, base_delay=5)
|
||||
def fetch_uap_sightings(*, force_refresh: bool = False):
|
||||
"""Fetch last-year UAP sightings from NUFORC.
|
||||
|
||||
Startup reads the cached daily snapshot when it is still fresh. The daily
|
||||
scheduler forces a rebuild so this layer updates once per day instead of
|
||||
churning continuously.
|
||||
"""
|
||||
from services.fetchers._store import is_any_active
|
||||
|
||||
if not is_any_active("uap_sightings"):
|
||||
return
|
||||
|
||||
sightings = _load_nuforc_sightings_cache(force_refresh=force_refresh)
|
||||
if sightings is None:
|
||||
sightings = _build_recent_uap_sightings()
|
||||
_save_nuforc_sightings_cache(sightings)
|
||||
|
||||
with _data_lock:
|
||||
latest_data["uap_sightings"] = sightings
|
||||
_mark_fresh("uap_sightings")
|
||||
return
|
||||
|
||||
cutoff = datetime.utcnow() - timedelta(days=_NUFORC_RECENT_DAYS)
|
||||
|
||||
# Query the grid concurrently (up to 8 threads)
|
||||
all_features: list[dict] = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
futures = {
|
||||
pool.submit(_fetch_nuforc_tilequery, lng, lat): (lng, lat)
|
||||
for lng, lat in _TILEQUERY_GRID
|
||||
}
|
||||
for fut in concurrent.futures.as_completed(futures, timeout=60):
|
||||
try:
|
||||
all_features.extend(fut.result())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Deduplicate by (LinkLat, LinkLon) and filter to recent sightings
|
||||
seen: set[tuple[str, str]] = set()
|
||||
sightings: list[dict] = []
|
||||
enriched_count = 0
|
||||
for feat in all_features:
|
||||
props = feat.get("properties", {})
|
||||
link_lat = props.get("LinkLat", "")
|
||||
link_lon = props.get("LinkLon", "")
|
||||
if not link_lat or not link_lon:
|
||||
continue
|
||||
|
||||
key = (link_lat, link_lon)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
|
||||
# Filter by date — keep if the latest sighting date >= cutoff
|
||||
to_date = props.get("To", "")
|
||||
from_date = props.get("From", "")
|
||||
latest_date = to_date or from_date
|
||||
latest_dt = _parse_nuforc_tile_date(latest_date)
|
||||
if latest_dt is not None and latest_dt < cutoff:
|
||||
continue
|
||||
|
||||
try:
|
||||
lat = float(link_lat)
|
||||
lng = float(link_lon)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
count = int(props.get("Count", "1") or "1")
|
||||
state_abbr, country = _reverse_geocode_state(lat, lng)
|
||||
|
||||
# Enrich with HF NUFORC dataset (shape, duration, city, summary)
|
||||
enrichment = enrich_sighting(state_abbr, from_date, to_date)
|
||||
city = enrichment.get("city", "")
|
||||
shape_raw = enrichment.get("shape_raw", "Unknown")
|
||||
shape = _normalize_uap_shape(shape_raw) if shape_raw != "Unknown" else "unknown"
|
||||
duration = enrichment.get("duration", "")
|
||||
summary = enrichment.get("summary", "")
|
||||
if enrichment:
|
||||
enriched_count += 1
|
||||
|
||||
# Build display summary: prefer enriched text, fall back to count-based
|
||||
if not summary:
|
||||
summary = f"{count} sighting(s) reported" if count > 1 else "Sighting reported"
|
||||
|
||||
sightings.append({
|
||||
"id": f"NUFORC-{hash(key) & 0xFFFFFFFF:08x}",
|
||||
"date_time": from_date if from_date == to_date else f"{from_date} to {to_date}",
|
||||
"city": city,
|
||||
"state": state_abbr,
|
||||
"country": country,
|
||||
"shape": shape,
|
||||
"shape_raw": shape_raw,
|
||||
"duration": duration,
|
||||
"summary": summary,
|
||||
"posted": to_date,
|
||||
"lat": lat,
|
||||
"lng": lng,
|
||||
"count": count,
|
||||
"source": "NUFORC",
|
||||
})
|
||||
|
||||
logger.info(
|
||||
f"UAP sightings: {len(sightings)} recent from NUFORC tilequery "
|
||||
f"({len(all_features)} raw, {enriched_count} enriched)"
|
||||
)
|
||||
|
||||
with _data_lock:
|
||||
latest_data["uap_sightings"] = sightings
|
||||
if sightings:
|
||||
_mark_fresh("uap_sightings")
|
||||
|
||||
@@ -1,20 +1,24 @@
|
||||
"""
|
||||
Fuel burn & CO2 emissions estimator for private jets.
|
||||
Fuel burn & CO2 emissions estimator.
|
||||
Based on manufacturer-published cruise fuel burn rates (GPH at long-range cruise).
|
||||
1 US gallon of Jet-A produces ~21.1 lbs (9.57 kg) of CO2.
|
||||
|
||||
Piston entries use 100LL (avgas), which is close enough to Jet-A in CO2 yield
|
||||
(~8.4 kg/gal vs 9.57 kg/gal); we keep one constant to stay simple — the result
|
||||
is a slight over-estimate for piston aircraft, which is preferable to under.
|
||||
"""
|
||||
|
||||
JET_A_CO2_KG_PER_GALLON = 9.57
|
||||
|
||||
# ICAO type code -> gallons per hour at long-range cruise
|
||||
FUEL_BURN_GPH: dict[str, int] = {
|
||||
# Gulfstream
|
||||
# ── Gulfstream ─────────────────────────────────────────────────────
|
||||
"GLF6": 430, # G650/G650ER
|
||||
"G700": 480, # G700
|
||||
"GLF5": 390, # G550
|
||||
"GVSP": 400, # GV-SP
|
||||
"GLF4": 330, # G-IV
|
||||
# Bombardier
|
||||
# ── Bombardier business ────────────────────────────────────────────
|
||||
"GL7T": 490, # Global 7500
|
||||
"GLEX": 430, # Global Express/6000/6500
|
||||
"GL5T": 420, # Global 5000/5500
|
||||
@@ -22,51 +26,208 @@ FUEL_BURN_GPH: dict[str, int] = {
|
||||
"CL60": 310, # Challenger 604/605
|
||||
"CL30": 200, # Challenger 300
|
||||
"CL65": 320, # Challenger 650
|
||||
# Dassault
|
||||
# ── Bombardier regional jets ──────────────────────────────────────
|
||||
"CRJ2": 360, # CRJ-100/200
|
||||
"CRJ7": 380, # CRJ-700
|
||||
"CRJ9": 410, # CRJ-900
|
||||
"CRJX": 440, # CRJ-1000
|
||||
# ── Dassault ───────────────────────────────────────────────────────
|
||||
"F7X": 350, # Falcon 7X
|
||||
"F8X": 370, # Falcon 8X
|
||||
"F900": 285, # Falcon 900/900EX/900LX
|
||||
"F2TH": 230, # Falcon 2000
|
||||
"FA50": 240, # Falcon 50
|
||||
# Cessna
|
||||
# ── Cessna Citation ────────────────────────────────────────────────
|
||||
"CITX": 280, # Citation X
|
||||
"C750": 280, # Citation X (alt code)
|
||||
"C68A": 195, # Citation Latitude
|
||||
"C700": 230, # Citation Longitude
|
||||
"C680": 220, # Citation Sovereign
|
||||
"C560": 190, # Citation Excel/XLS
|
||||
"C56X": 195, # Citation Excel/XLS/XLS+
|
||||
"C560": 190, # Citation Excel/XLS (legacy)
|
||||
"C550": 165, # Citation II/Bravo/V
|
||||
"C525": 80, # Citation CJ1
|
||||
"C25A": 100, # CJ1+ / 525A
|
||||
"C25B": 110, # CJ2+ / 525B
|
||||
"C25C": 130, # CJ4 (some operators)
|
||||
"C510": 75, # Citation Mustang
|
||||
"C650": 240, # Citation III/VI/VII
|
||||
"CJ3": 120, # CJ3
|
||||
"CJ4": 135, # CJ4
|
||||
# Boeing
|
||||
"B737": 850, # BBJ (737)
|
||||
"B738": 920, # BBJ2 (737-800)
|
||||
# ── Cessna piston / turboprop singles & twins ─────────────────────
|
||||
"C172": 9, # Skyhawk
|
||||
"C152": 6,
|
||||
"C150": 6,
|
||||
"C170": 8,
|
||||
"C177": 11,
|
||||
"C180": 12,
|
||||
"C182": 13, # Skylane
|
||||
"C185": 14,
|
||||
"C206": 15,
|
||||
"C208": 50, # Caravan (turboprop)
|
||||
"C210": 18,
|
||||
"C310": 32,
|
||||
"C340": 38,
|
||||
"C414": 36,
|
||||
"C421": 40,
|
||||
# ── Boeing mainline ────────────────────────────────────────────────
|
||||
"B737": 850, # 737-700 / BBJ
|
||||
"B738": 920, # 737-800
|
||||
"B739": 880, # 737-900/900ER
|
||||
"B38M": 700, # 737-8 MAX
|
||||
"B39M": 740, # 737-9 MAX
|
||||
"B752": 1100, # 757-200
|
||||
"B753": 1200, # 757-300
|
||||
"B762": 1400, # 767-200
|
||||
"B763": 1450, # 767-300/300ER
|
||||
"B764": 1500, # 767-400ER
|
||||
"B772": 1850, # 777-200
|
||||
"B77L": 1900, # 777-200LR / 777F
|
||||
"B77W": 2050, # 777-300ER
|
||||
"B788": 1200, # 787-8
|
||||
# Airbus
|
||||
"A318": 780, # ACJ318
|
||||
"A319": 850, # ACJ319
|
||||
"A320": 900, # ACJ320
|
||||
"B789": 1300, # 787-9
|
||||
"B78X": 1350, # 787-10
|
||||
"B744": 3050, # 747-400
|
||||
"B748": 2900, # 747-8
|
||||
# ── Airbus mainline ────────────────────────────────────────────────
|
||||
"A318": 780, # A318
|
||||
"A319": 850, # A319
|
||||
"A320": 900, # A320
|
||||
"A321": 990, # A321
|
||||
"A19N": 580, # A319neo
|
||||
"A20N": 580, # A320neo
|
||||
"A21N": 700, # A321neo
|
||||
"A332": 1500, # A330-200
|
||||
"A333": 1550, # A330-300
|
||||
"A338": 1300, # A330-800neo
|
||||
"A339": 1350, # A330-900neo
|
||||
"A343": 1800, # A340-300
|
||||
"A346": 2100, # A340-600
|
||||
# Pilatus
|
||||
"A359": 1450, # A350-900
|
||||
"A35K": 1600, # A350-1000
|
||||
"A388": 3200, # A380-800
|
||||
# ── Embraer regional / business ───────────────────────────────────
|
||||
"E135": 300, # Legacy 600/650 (regional ERJ-135 base)
|
||||
"E145": 320, # ERJ-145
|
||||
"E170": 460, # E170
|
||||
"E75L": 490, # E175-LR
|
||||
"E75S": 490, # E175 standard
|
||||
"E175": 490, # E175 (some)
|
||||
"E190": 580, # E190
|
||||
"E195": 600, # E195
|
||||
"E290": 510, # E190-E2
|
||||
"E295": 540, # E195-E2
|
||||
"E50P": 135, # Phenom 300 (also Phenom 100 var)
|
||||
"E55P": 185, # Praetor 500 / Legacy 500
|
||||
"E545": 170, # Praetor 500 (alt)
|
||||
"E500": 80, # Phenom 100
|
||||
# ── ATR / Bombardier / Saab turboprops ────────────────────────────
|
||||
"AT43": 230, # ATR 42-300/-320
|
||||
"AT45": 230, # ATR 42-500
|
||||
"AT46": 250, # ATR 42-600
|
||||
"AT72": 300, # ATR 72-200/-210
|
||||
"AT75": 280, # ATR 72-500
|
||||
"AT76": 280, # ATR 72-600
|
||||
"DH8A": 220, # Dash 8 -100
|
||||
"DH8B": 240, # Dash 8 -200
|
||||
"DH8C": 280, # Dash 8 -300
|
||||
"DH8D": 300, # Dash 8 Q400
|
||||
"SF34": 200, # Saab 340
|
||||
"SB20": 220, # Saab 2000
|
||||
# ── Pilatus / Daher single-engine turboprops ──────────────────────
|
||||
"PC24": 115, # PC-24
|
||||
"PC12": 60, # PC-12
|
||||
# Embraer
|
||||
"E55P": 185, # Legacy 500
|
||||
"E135": 300, # Legacy 600/650
|
||||
"E50P": 135, # Phenom 300
|
||||
"E500": 80, # Phenom 100
|
||||
# Learjet
|
||||
"TBM7": 60, # TBM 700/850
|
||||
"TBM8": 65, # TBM 850 alt
|
||||
"TBM9": 70, # TBM 900/930/940/960
|
||||
"M600": 60, # Piper M600
|
||||
"P46T": 22, # PA-46 Meridian (turboprop variant)
|
||||
# ── Learjet ────────────────────────────────────────────────────────
|
||||
"LJ60": 195, # Learjet 60
|
||||
"LJ75": 185, # Learjet 75
|
||||
"LJ45": 175, # Learjet 45
|
||||
# Hawker
|
||||
"LJ31": 165, # Learjet 31
|
||||
"LJ40": 175, # Learjet 40
|
||||
"LJ55": 195, # Learjet 55
|
||||
# ── Hawker / Beechjet ─────────────────────────────────────────────
|
||||
"H25B": 210, # Hawker 800/800XP
|
||||
"H25C": 215, # Hawker 900XP
|
||||
# Beechcraft
|
||||
"BE40": 150, # Beechjet 400 / Hawker 400XP
|
||||
"PRM1": 130, # Premier I
|
||||
# ── Beechcraft King Air ───────────────────────────────────────────
|
||||
"B350": 100, # King Air 350
|
||||
"B200": 80, # King Air 200/250
|
||||
"BE20": 80, # K-Air 200 (alt)
|
||||
"BE9L": 60, # K-Air 90
|
||||
"BE9T": 70, # K-Air F90
|
||||
"BE10": 100, # K-Air 100
|
||||
"BE30": 90, # K-Air 300
|
||||
# ── Beechcraft / Cirrus / Piper / Mooney pistons ──────────────────
|
||||
"BE23": 9, # Sundowner
|
||||
"BE33": 13, # Bonanza 33
|
||||
"BE35": 14, # Bonanza V-tail
|
||||
"BE36": 16, # A36 Bonanza
|
||||
"BE55": 24, # Baron 55
|
||||
"BE58": 28, # Baron 58
|
||||
"BE76": 17, # Duchess
|
||||
"BE95": 20, # Travel Air
|
||||
"P28A": 10, # PA-28 Warrior/Archer
|
||||
"P28B": 11, # PA-28 Cherokee
|
||||
"P28R": 12, # PA-28R Arrow
|
||||
"P32R": 14, # PA-32R Lance/Saratoga
|
||||
"PA11": 5, # Cub Special
|
||||
"PA12": 6, # Super Cruiser
|
||||
"PA18": 6, # Super Cub
|
||||
"PA22": 8, # Tri-Pacer
|
||||
"PA23": 18, # Apache / Aztec
|
||||
"PA24": 12, # Comanche
|
||||
"PA25": 12, # Pawnee
|
||||
"PA28": 10, # PA-28 generic
|
||||
"PA30": 16, # Twin Comanche
|
||||
"PA31": 30, # Navajo
|
||||
"PA32": 14, # Cherokee Six / Saratoga
|
||||
"PA34": 18, # Seneca
|
||||
"PA38": 5, # Tomahawk
|
||||
"PA44": 17, # Seminole
|
||||
"PA46": 18, # Malibu / Mirage / Matrix
|
||||
"M20P": 12, # Mooney M20 (generic)
|
||||
"SR20": 11, # Cirrus SR20
|
||||
"SR22": 16, # Cirrus SR22
|
||||
"S22T": 19, # SR22T (turbo)
|
||||
"DA40": 9, # Diamond DA40
|
||||
"DA42": 14, # Diamond DA42 TwinStar
|
||||
"DA62": 17, # Diamond DA62
|
||||
"DV20": 6, # Diamond Katana
|
||||
# ── Helicopters (civilian) ────────────────────────────────────────
|
||||
"A109": 60, # AW109
|
||||
"A119": 50, # AW119
|
||||
"A139": 130, # AW139
|
||||
"A169": 90, # AW169
|
||||
"A189": 145, # AW189
|
||||
"AS35": 55, # AS350 AStar
|
||||
"AS50": 55, # AStar (alt)
|
||||
"AS65": 110, # Dauphin
|
||||
"B06": 35, # Bell 206 JetRanger
|
||||
"B407": 50, # Bell 407
|
||||
"B412": 145, # Bell 412
|
||||
"B429": 80, # Bell 429
|
||||
"B505": 35, # Bell 505
|
||||
"EC30": 50, # H125 / EC130
|
||||
"EC35": 70, # EC135
|
||||
"EC45": 85, # EC145
|
||||
"EC75": 130, # EC175
|
||||
"H125": 55,
|
||||
"H130": 50,
|
||||
"H135": 70,
|
||||
"H145": 85,
|
||||
"H155": 110,
|
||||
"H160": 95,
|
||||
"H175": 130,
|
||||
"R22": 9, # Robinson R22 (piston)
|
||||
"R44": 16, # Robinson R44 (piston)
|
||||
"R66": 30, # Robinson R66 (turbine)
|
||||
"S76": 140, # Sikorsky S-76
|
||||
"S92": 220, # Sikorsky S-92
|
||||
}
|
||||
|
||||
# Common string names -> ICAO type code
|
||||
@@ -108,13 +269,23 @@ def get_emissions_info(model: str) -> dict | None:
|
||||
if not model:
|
||||
return None
|
||||
model_clean = model.strip()
|
||||
model_upper = model_clean.upper()
|
||||
# Try direct ICAO code match first
|
||||
gph = FUEL_BURN_GPH.get(model_clean.upper())
|
||||
gph = FUEL_BURN_GPH.get(model_upper)
|
||||
if gph is None:
|
||||
# Try alias lookup
|
||||
code = _ALIASES.get(model_clean)
|
||||
if code:
|
||||
gph = FUEL_BURN_GPH.get(code)
|
||||
if gph is None:
|
||||
# Friendly names from the Plane-Alert DB often lead with the ICAO type
|
||||
# code as the first token (e.g. "B200 Super King Air"). Probe each
|
||||
# token against FUEL_BURN_GPH directly.
|
||||
for token in model_upper.replace("-", " ").replace(",", " ").split():
|
||||
candidate = FUEL_BURN_GPH.get(token)
|
||||
if candidate is not None:
|
||||
gph = candidate
|
||||
break
|
||||
if gph is None:
|
||||
# Fuzzy: check if any alias is a substring
|
||||
model_lower = model_clean.lower()
|
||||
|
||||
@@ -13,12 +13,13 @@ import concurrent.futures
|
||||
import random
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from cachetools import TTLCache
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
from services.fetchers.plane_alert import enrich_with_plane_alert, enrich_with_tracked_names
|
||||
from services.fetchers.emissions import get_emissions_info
|
||||
from services.fetchers.retry import with_retry
|
||||
from services.fetchers.route_database import lookup_route
|
||||
from services.fetchers.aircraft_database import lookup_aircraft_type
|
||||
from services.constants import GPS_JAMMING_NACP_THRESHOLD, GPS_JAMMING_MIN_RATIO, GPS_JAMMING_MIN_AIRCRAFT
|
||||
|
||||
logger = logging.getLogger("services.data_fetcher")
|
||||
@@ -76,6 +77,7 @@ opensky_client = OpenSkyClient(
|
||||
# Throttling and caching for OpenSky (400 req/day limit)
|
||||
last_opensky_fetch = 0
|
||||
cached_opensky_flights = []
|
||||
_opensky_cache_lock = threading.Lock()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Supplemental ADS-B sources for blind-spot gap-filling
|
||||
@@ -98,6 +100,7 @@ _AIRPLANES_LIVE_DELAY_SECONDS = 1.2
|
||||
_AIRPLANES_LIVE_DELAY_JITTER_SECONDS = 0.4
|
||||
last_supplemental_fetch = 0
|
||||
cached_supplemental_flights = []
|
||||
_supplemental_cache_lock = threading.Lock()
|
||||
|
||||
# Helicopter type codes (backend classification)
|
||||
_HELI_TYPES_BACKEND = {
|
||||
@@ -255,10 +258,11 @@ flight_trails = {} # {icao_hex: {points: [[lat, lng, alt, ts], ...], last_seen:
|
||||
_trails_lock = threading.Lock()
|
||||
_MAX_TRACKED_TRAILS = 2000
|
||||
|
||||
# Routes cache
|
||||
dynamic_routes_cache = TTLCache(maxsize=5000, ttl=7200)
|
||||
routes_fetch_in_progress = False
|
||||
_routes_lock = threading.Lock()
|
||||
# Route enrichment is now served from services.fetchers.route_database, which
|
||||
# bulk-loads vrs-standing-data.adsb.lol/routes.csv.gz once per day and looks up
|
||||
# callsigns from an in-memory index. Replaces the legacy /api/0/routeset POST,
|
||||
# which was both blocked under the ShadowBroker UA (HTTP 451) and broken
|
||||
# upstream (returning 201 with empty body even for unblocked clients).
|
||||
|
||||
|
||||
def _fetch_supplemental_sources(seen_hex: set) -> list:
|
||||
@@ -266,12 +270,13 @@ def _fetch_supplemental_sources(seen_hex: set) -> list:
|
||||
global last_supplemental_fetch, cached_supplemental_flights
|
||||
|
||||
now = time.time()
|
||||
if now - last_supplemental_fetch < _SUPPLEMENTAL_FETCH_INTERVAL:
|
||||
return [
|
||||
f
|
||||
for f in cached_supplemental_flights
|
||||
if f.get("hex", "").lower().strip() not in seen_hex
|
||||
]
|
||||
with _supplemental_cache_lock:
|
||||
if now - last_supplemental_fetch < _SUPPLEMENTAL_FETCH_INTERVAL:
|
||||
return [
|
||||
f
|
||||
for f in cached_supplemental_flights
|
||||
if f.get("hex", "").lower().strip() not in seen_hex
|
||||
]
|
||||
|
||||
new_supplemental = []
|
||||
supplemental_hex = set()
|
||||
@@ -363,8 +368,9 @@ def _fetch_supplemental_sources(seen_hex: set) -> list:
|
||||
|
||||
fi_count = len(new_supplemental) - ap_count
|
||||
|
||||
cached_supplemental_flights = new_supplemental
|
||||
last_supplemental_fetch = now
|
||||
with _supplemental_cache_lock:
|
||||
cached_supplemental_flights = new_supplemental
|
||||
last_supplemental_fetch = now
|
||||
if new_supplemental:
|
||||
_mark_fresh("supplemental_flights")
|
||||
|
||||
@@ -375,73 +381,6 @@ def _fetch_supplemental_sources(seen_hex: set) -> list:
|
||||
return new_supplemental
|
||||
|
||||
|
||||
def fetch_routes_background(sampled):
|
||||
global routes_fetch_in_progress
|
||||
with _routes_lock:
|
||||
if routes_fetch_in_progress:
|
||||
return
|
||||
routes_fetch_in_progress = True
|
||||
|
||||
try:
|
||||
callsigns_to_query = []
|
||||
for f in sampled:
|
||||
c_sign = str(f.get("flight", "")).strip()
|
||||
if c_sign and c_sign != "UNKNOWN":
|
||||
callsigns_to_query.append(
|
||||
{"callsign": c_sign, "lat": f.get("lat", 0), "lng": f.get("lon", 0)}
|
||||
)
|
||||
|
||||
batch_size = 100
|
||||
batches = [
|
||||
callsigns_to_query[i : i + batch_size]
|
||||
for i in range(0, len(callsigns_to_query), batch_size)
|
||||
]
|
||||
|
||||
for batch in batches:
|
||||
try:
|
||||
r = fetch_with_curl(
|
||||
"https://api.adsb.lol/api/0/routeset",
|
||||
method="POST",
|
||||
json_data={"planes": batch},
|
||||
timeout=15,
|
||||
)
|
||||
if r.status_code == 200:
|
||||
route_data = r.json()
|
||||
route_list = []
|
||||
if isinstance(route_data, dict):
|
||||
route_list = route_data.get("value", [])
|
||||
elif isinstance(route_data, list):
|
||||
route_list = route_data
|
||||
|
||||
for route in route_list:
|
||||
callsign = route.get("callsign", "")
|
||||
airports = route.get("_airports", [])
|
||||
if airports and len(airports) >= 2:
|
||||
orig_apt = airports[0]
|
||||
dest_apt = airports[-1]
|
||||
with _routes_lock:
|
||||
dynamic_routes_cache[callsign] = {
|
||||
"orig_name": f"{orig_apt.get('iata', '')}: {orig_apt.get('name', 'Unknown')}",
|
||||
"dest_name": f"{dest_apt.get('iata', '')}: {dest_apt.get('name', 'Unknown')}",
|
||||
"orig_loc": [orig_apt.get("lon", 0), orig_apt.get("lat", 0)],
|
||||
"dest_loc": [dest_apt.get("lon", 0), dest_apt.get("lat", 0)],
|
||||
}
|
||||
time.sleep(0.25)
|
||||
except (
|
||||
requests.RequestException,
|
||||
ConnectionError,
|
||||
TimeoutError,
|
||||
ValueError,
|
||||
KeyError,
|
||||
json.JSONDecodeError,
|
||||
OSError,
|
||||
) as e:
|
||||
logger.debug(f"Route batch request failed: {e}")
|
||||
finally:
|
||||
with _routes_lock:
|
||||
routes_fetch_in_progress = False
|
||||
|
||||
|
||||
def _classify_and_publish(all_adsb_flights):
|
||||
"""Shared pipeline: normalize raw ADS-B data → classify → merge → publish to latest_data.
|
||||
|
||||
@@ -453,13 +392,6 @@ def _classify_and_publish(all_adsb_flights):
|
||||
if not all_adsb_flights:
|
||||
return
|
||||
|
||||
with _routes_lock:
|
||||
already_running = routes_fetch_in_progress
|
||||
if not already_running:
|
||||
threading.Thread(
|
||||
target=fetch_routes_background, args=(all_adsb_flights,), daemon=True
|
||||
).start()
|
||||
|
||||
for f in all_adsb_flights:
|
||||
try:
|
||||
lat = f.get("lat")
|
||||
@@ -478,8 +410,7 @@ def _classify_and_publish(all_adsb_flights):
|
||||
origin_name = "UNKNOWN"
|
||||
dest_name = "UNKNOWN"
|
||||
|
||||
with _routes_lock:
|
||||
cached_route = dynamic_routes_cache.get(flight_str)
|
||||
cached_route = lookup_route(flight_str)
|
||||
if cached_route:
|
||||
origin_name = cached_route["orig_name"]
|
||||
dest_name = cached_route["dest_name"]
|
||||
@@ -501,7 +432,18 @@ def _classify_and_publish(all_adsb_flights):
|
||||
gs_knots = f.get("gs")
|
||||
speed_knots = round(gs_knots, 1) if isinstance(gs_knots, (int, float)) else None
|
||||
|
||||
model_upper = f.get("t", "").upper()
|
||||
# OpenSky's /states/all doesn't carry the aircraft type, so its
|
||||
# records arrive with t="Unknown". Backfill from the OpenSky
|
||||
# aircraft metadata DB by ICAO24 hex so heli classification and
|
||||
# downstream emissions enrichment both see a real type code.
|
||||
raw_type = str(f.get("t") or "").strip()
|
||||
if not raw_type or raw_type.lower() == "unknown":
|
||||
looked_up_type = lookup_aircraft_type(f.get("hex", ""))
|
||||
if looked_up_type:
|
||||
f["t"] = looked_up_type
|
||||
raw_type = looked_up_type
|
||||
|
||||
model_upper = raw_type.upper()
|
||||
if model_upper == "TWR":
|
||||
continue
|
||||
|
||||
@@ -543,8 +485,14 @@ def _classify_and_publish(all_adsb_flights):
|
||||
for f in flights:
|
||||
enrich_with_plane_alert(f)
|
||||
enrich_with_tracked_names(f)
|
||||
# Attach fuel-burn / CO2 emissions estimate when model is known
|
||||
# Attach fuel-burn / CO2 emissions estimate when model is known.
|
||||
# OpenSky's /states/all doesn't carry aircraft type, so OpenSky-sourced
|
||||
# flights arrive with model="Unknown". For tracked planes, the
|
||||
# Plane-Alert DB has the friendly type name in alert_type, and the
|
||||
# emissions aliases table already maps those names to ICAO codes.
|
||||
model = f.get("model")
|
||||
if not model or model.strip().lower() in {"", "unknown"}:
|
||||
model = f.get("alert_type") or ""
|
||||
if model:
|
||||
emi = get_emissions_info(model)
|
||||
if emi:
|
||||
@@ -618,6 +566,10 @@ def _classify_and_publish(all_adsb_flights):
|
||||
latest_data["flights"] = flights
|
||||
|
||||
# Merge tracked civilian flights with tracked military flights
|
||||
# Stale tracked flights (not seen in any ADS-B source for >5 min) are dropped.
|
||||
_TRACKED_STALE_S = 300 # 5 minutes
|
||||
_merge_ts = time.time()
|
||||
|
||||
with _data_lock:
|
||||
existing_tracked = copy.deepcopy(latest_data.get("tracked_flights", []))
|
||||
|
||||
@@ -625,10 +577,12 @@ def _classify_and_publish(all_adsb_flights):
|
||||
for t in tracked:
|
||||
icao = t.get("icao24", "").upper()
|
||||
if icao:
|
||||
t["_seen_at"] = _merge_ts
|
||||
fresh_tracked_map[icao] = t
|
||||
|
||||
merged_tracked = []
|
||||
seen_icaos = set()
|
||||
stale_dropped = 0
|
||||
for old_t in existing_tracked:
|
||||
icao = old_t.get("icao24", "").upper()
|
||||
if icao in fresh_tracked_map:
|
||||
@@ -639,8 +593,13 @@ def _classify_and_publish(all_adsb_flights):
|
||||
merged_tracked.append(fresh)
|
||||
seen_icaos.add(icao)
|
||||
else:
|
||||
merged_tracked.append(old_t)
|
||||
seen_icaos.add(icao)
|
||||
# Keep stale entry only if it was seen recently
|
||||
age = _merge_ts - old_t.get("_seen_at", 0)
|
||||
if age < _TRACKED_STALE_S:
|
||||
merged_tracked.append(old_t)
|
||||
seen_icaos.add(icao)
|
||||
else:
|
||||
stale_dropped += 1
|
||||
|
||||
for icao, t in fresh_tracked_map.items():
|
||||
if icao not in seen_icaos:
|
||||
@@ -649,10 +608,12 @@ def _classify_and_publish(all_adsb_flights):
|
||||
with _data_lock:
|
||||
latest_data["tracked_flights"] = merged_tracked
|
||||
logger.info(
|
||||
f"Tracked flights: {len(merged_tracked)} total ({len(fresh_tracked_map)} fresh from civilian)"
|
||||
f"Tracked flights: {len(merged_tracked)} total ({len(fresh_tracked_map)} fresh from civilian, {stale_dropped} stale dropped)"
|
||||
)
|
||||
|
||||
# --- Trail Accumulation ---
|
||||
_TRAIL_INTERVAL_S = 600 # only record a new trail point every 10 minutes
|
||||
|
||||
def _accumulate_trail(f, now_ts, check_route=True):
|
||||
hex_id = f.get("icao24", "").lower()
|
||||
if not hex_id:
|
||||
@@ -668,7 +629,11 @@ def _classify_and_publish(all_adsb_flights):
|
||||
if hex_id not in flight_trails:
|
||||
flight_trails[hex_id] = {"points": [], "last_seen": now_ts}
|
||||
trail_data = flight_trails[hex_id]
|
||||
if (
|
||||
# Only append a new point if 10 minutes have passed since the last one
|
||||
last_point_ts = trail_data["points"][-1][3] if trail_data["points"] else 0
|
||||
if now_ts - last_point_ts < _TRAIL_INTERVAL_S:
|
||||
trail_data["last_seen"] = now_ts
|
||||
elif (
|
||||
trail_data["points"]
|
||||
and trail_data["points"][-1][0] == point[0]
|
||||
and trail_data["points"][-1][1] == point[1]
|
||||
@@ -688,22 +653,26 @@ def _classify_and_publish(all_adsb_flights):
|
||||
tracked_snapshot = copy.deepcopy(latest_data.get("tracked_flights", []))
|
||||
raw_flights_snapshot = list(latest_data.get("flights", []))
|
||||
|
||||
all_lists = [commercial, private_jets, private_ga, existing_tracked]
|
||||
# Commercial/private: skip trail if route is known (route line replaces trail)
|
||||
route_check_lists = [commercial, private_jets, private_ga]
|
||||
# Tracked + military: ALWAYS accumulate trails (high-interest flights)
|
||||
always_trail_lists = [existing_tracked, military_snapshot]
|
||||
seen_hexes = set()
|
||||
trail_count = 0
|
||||
with _trails_lock:
|
||||
for flist in all_lists:
|
||||
for flist in route_check_lists:
|
||||
for f in flist:
|
||||
count, hex_id = _accumulate_trail(f, now_ts, check_route=True)
|
||||
trail_count += count
|
||||
if hex_id:
|
||||
seen_hexes.add(hex_id)
|
||||
|
||||
for mf in military_snapshot:
|
||||
count, hex_id = _accumulate_trail(mf, now_ts, check_route=False)
|
||||
trail_count += count
|
||||
if hex_id:
|
||||
seen_hexes.add(hex_id)
|
||||
for flist in always_trail_lists:
|
||||
for f in flist:
|
||||
count, hex_id = _accumulate_trail(f, now_ts, check_route=False)
|
||||
trail_count += count
|
||||
if hex_id:
|
||||
seen_hexes.add(hex_id)
|
||||
|
||||
tracked_hexes = {t.get("icao24", "").lower() for t in tracked_snapshot}
|
||||
stale_keys = []
|
||||
@@ -889,79 +858,100 @@ def _enrich_with_opensky_and_supplemental(adsb_flights):
|
||||
now = time.time()
|
||||
global last_opensky_fetch, cached_opensky_flights
|
||||
|
||||
if now - last_opensky_fetch > 300:
|
||||
with _opensky_cache_lock:
|
||||
_need_opensky = now - last_opensky_fetch > 300
|
||||
if not _need_opensky:
|
||||
opensky_snapshot = list(cached_opensky_flights)
|
||||
|
||||
if _need_opensky:
|
||||
token = opensky_client.get_token()
|
||||
if token:
|
||||
opensky_regions = [
|
||||
{
|
||||
"name": "Africa",
|
||||
"bbox": {"lamin": -35.0, "lomin": -20.0, "lamax": 38.0, "lomax": 55.0},
|
||||
},
|
||||
{
|
||||
"name": "Asia",
|
||||
"bbox": {"lamin": 0.0, "lomin": 30.0, "lamax": 75.0, "lomax": 150.0},
|
||||
},
|
||||
{
|
||||
"name": "South America",
|
||||
"bbox": {"lamin": -60.0, "lomin": -95.0, "lamax": 15.0, "lomax": -30.0},
|
||||
},
|
||||
]
|
||||
|
||||
# One global /states/all query = 4 credits flat per OpenSky
|
||||
# docs (https://openskynetwork.github.io/opensky-api/rest.html).
|
||||
# At the current 5-minute cadence that's 4 × 288 = 1152
|
||||
# credits/day, ~29% of the 4000-credit standard daily quota,
|
||||
# and returns every aircraft worldwide in a single call.
|
||||
# The previous 3-regional-bbox approach cost 12 credits/cycle
|
||||
# AND missed North America, Europe, and Oceania entirely.
|
||||
new_opensky_flights = []
|
||||
for os_reg in opensky_regions:
|
||||
try:
|
||||
bb = os_reg["bbox"]
|
||||
os_url = f"https://opensky-network.org/api/states/all?lamin={bb['lamin']}&lomin={bb['lomin']}&lamax={bb['lamax']}&lomax={bb['lomax']}"
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
os_res = requests.get(os_url, headers=headers, timeout=15)
|
||||
try:
|
||||
os_url = "https://opensky-network.org/api/states/all"
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
os_res = requests.get(os_url, headers=headers, timeout=30)
|
||||
|
||||
if os_res.status_code == 200:
|
||||
os_data = os_res.json()
|
||||
states = os_data.get("states") or []
|
||||
logger.info(
|
||||
f"OpenSky: Fetched {len(states)} states for {os_reg['name']}"
|
||||
if os_res.status_code == 200:
|
||||
os_data = os_res.json()
|
||||
states = os_data.get("states") or []
|
||||
remaining = os_res.headers.get("X-Rate-Limit-Remaining", "?")
|
||||
logger.info(
|
||||
f"OpenSky: fetched {len(states)} global states "
|
||||
f"(credits remaining: {remaining})"
|
||||
)
|
||||
for s in states:
|
||||
if s[5] is None or s[6] is None:
|
||||
continue
|
||||
new_opensky_flights.append(
|
||||
{
|
||||
"hex": s[0],
|
||||
"flight": s[1].strip() if s[1] else "UNKNOWN",
|
||||
"r": s[2],
|
||||
"lon": s[5],
|
||||
"lat": s[6],
|
||||
"alt_baro": (s[7] * 3.28084) if s[7] else 0,
|
||||
"track": s[10] or 0,
|
||||
"gs": (s[9] * 1.94384) if s[9] else 0,
|
||||
"t": "Unknown",
|
||||
"is_opensky": True,
|
||||
}
|
||||
)
|
||||
elif os_res.status_code == 429:
|
||||
retry_after = os_res.headers.get("X-Rate-Limit-Retry-After-Seconds", "?")
|
||||
logger.warning(
|
||||
f"OpenSky daily quota exhausted (4000 credits). "
|
||||
f"Retry after {retry_after}s. Serving stale data until reset."
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"OpenSky /states/all failed: HTTP {os_res.status_code}"
|
||||
)
|
||||
except (
|
||||
requests.RequestException,
|
||||
ConnectionError,
|
||||
TimeoutError,
|
||||
ValueError,
|
||||
KeyError,
|
||||
json.JSONDecodeError,
|
||||
OSError,
|
||||
) as ex:
|
||||
logger.error(f"OpenSky global fetch error: {ex}")
|
||||
|
||||
for s in states:
|
||||
new_opensky_flights.append(
|
||||
{
|
||||
"hex": s[0],
|
||||
"flight": s[1].strip() if s[1] else "UNKNOWN",
|
||||
"r": s[2],
|
||||
"lon": s[5],
|
||||
"lat": s[6],
|
||||
"alt_baro": (s[7] * 3.28084) if s[7] else 0,
|
||||
"track": s[10] or 0,
|
||||
"gs": (s[9] * 1.94384) if s[9] else 0,
|
||||
"t": "Unknown",
|
||||
"is_opensky": True,
|
||||
}
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"OpenSky API {os_reg['name']} failed: {os_res.status_code}"
|
||||
)
|
||||
except (
|
||||
requests.RequestException,
|
||||
ConnectionError,
|
||||
TimeoutError,
|
||||
ValueError,
|
||||
KeyError,
|
||||
json.JSONDecodeError,
|
||||
OSError,
|
||||
) as ex:
|
||||
logger.error(f"OpenSky fetching error for {os_reg['name']}: {ex}")
|
||||
|
||||
cached_opensky_flights = new_opensky_flights
|
||||
last_opensky_fetch = now
|
||||
with _opensky_cache_lock:
|
||||
if new_opensky_flights:
|
||||
cached_opensky_flights = new_opensky_flights
|
||||
last_opensky_fetch = now
|
||||
opensky_snapshot = new_opensky_flights or list(cached_opensky_flights)
|
||||
else:
|
||||
# Token refresh failed — fall back to existing cached data
|
||||
with _opensky_cache_lock:
|
||||
opensky_snapshot = list(cached_opensky_flights)
|
||||
|
||||
# Merge OpenSky (dedup by hex)
|
||||
for osf in cached_opensky_flights:
|
||||
for osf in opensky_snapshot:
|
||||
h = osf.get("hex")
|
||||
if h and h.lower().strip() not in seen_hex:
|
||||
all_flights.append(osf)
|
||||
seen_hex.add(h.lower().strip())
|
||||
|
||||
# Publish OpenSky-merged data immediately so users see flights even if
|
||||
# supplemental gap-fill is slow or rate-limited (airplanes.live can take
|
||||
# 100+ seconds when its regional endpoints are throttled).
|
||||
if len(all_flights) > len(adsb_flights):
|
||||
logger.info(
|
||||
f"OpenSky merge: {len(all_flights) - len(adsb_flights)} additional aircraft, "
|
||||
"publishing before supplemental gap-fill"
|
||||
)
|
||||
_classify_and_publish(all_flights)
|
||||
|
||||
# Supplemental gap-fill
|
||||
try:
|
||||
gap_fill = _fetch_supplemental_sources(seen_hex)
|
||||
@@ -1008,14 +998,18 @@ def fetch_flights():
|
||||
if adsb_flights:
|
||||
logger.info(f"adsb.lol: {len(adsb_flights)} aircraft — publishing immediately")
|
||||
_classify_and_publish(adsb_flights)
|
||||
|
||||
# Phase 2: kick off slow enrichment in background
|
||||
threading.Thread(
|
||||
target=_enrich_with_opensky_and_supplemental,
|
||||
args=(adsb_flights,),
|
||||
daemon=True,
|
||||
).start()
|
||||
else:
|
||||
logger.warning("adsb.lol returned 0 aircraft")
|
||||
logger.warning(
|
||||
"adsb.lol returned 0 aircraft — relying on OpenSky/supplemental sources"
|
||||
)
|
||||
|
||||
# Phase 2: always run — OpenSky is the fallback when adsb.lol blocks us
|
||||
# (it has been known to 451 the bulk regional endpoint), and supplemental
|
||||
# gap-fill should always run regardless of Phase 1 success.
|
||||
threading.Thread(
|
||||
target=_enrich_with_opensky_and_supplemental,
|
||||
args=(adsb_flights,),
|
||||
daemon=True,
|
||||
).start()
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching flights: {e}")
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
"""Ship and geopolitics fetchers — AIS vessels, carriers, frontlines, GDELT, LiveUAmap, fishing."""
|
||||
|
||||
import csv
|
||||
import concurrent.futures
|
||||
import io
|
||||
import math
|
||||
import os
|
||||
import logging
|
||||
import time
|
||||
from urllib.parse import urlencode
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
from services.fetchers.retry import with_retry
|
||||
@@ -27,20 +30,24 @@ def fetch_ships():
|
||||
from services.ais_stream import get_ais_vessels
|
||||
from services.carrier_tracker import get_carrier_positions
|
||||
|
||||
ships = []
|
||||
try:
|
||||
carriers = get_carrier_positions()
|
||||
ships.extend(carriers)
|
||||
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
|
||||
logger.error(f"Carrier tracker error (non-fatal): {e}")
|
||||
carriers = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2, thread_name_prefix="ship_fetch") as executor:
|
||||
carrier_future = executor.submit(get_carrier_positions)
|
||||
ais_future = executor.submit(get_ais_vessels)
|
||||
|
||||
try:
|
||||
ais_vessels = get_ais_vessels()
|
||||
ships.extend(ais_vessels)
|
||||
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
|
||||
logger.error(f"AIS stream error (non-fatal): {e}")
|
||||
ais_vessels = []
|
||||
try:
|
||||
carriers = carrier_future.result()
|
||||
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
|
||||
logger.error(f"Carrier tracker error (non-fatal): {e}")
|
||||
carriers = []
|
||||
|
||||
try:
|
||||
ais_vessels = ais_future.result()
|
||||
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
|
||||
logger.error(f"AIS stream error (non-fatal): {e}")
|
||||
ais_vessels = []
|
||||
|
||||
ships = list(carriers or [])
|
||||
ships.extend(ais_vessels or [])
|
||||
|
||||
# Enrich ships with yacht alert data (tracked superyachts)
|
||||
from services.fetchers.yacht_alert import enrich_with_yacht_alert
|
||||
@@ -200,52 +207,177 @@ def update_liveuamap():
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fishing Activity (Global Fishing Watch)
|
||||
# ---------------------------------------------------------------------------
|
||||
def _fishing_vessel_key(event: dict) -> str:
|
||||
vessel_ssvid = str(event.get("vessel_ssvid", "") or "").strip()
|
||||
if vessel_ssvid:
|
||||
return f"ssvid:{vessel_ssvid}"
|
||||
vessel_id = str(event.get("vessel_id", "") or "").strip()
|
||||
if vessel_id:
|
||||
return f"vid:{vessel_id}"
|
||||
vessel_name = str(event.get("vessel_name", "") or "").strip().upper()
|
||||
vessel_flag = str(event.get("vessel_flag", "") or "").strip().upper()
|
||||
if vessel_name:
|
||||
return f"name:{vessel_name}|flag:{vessel_flag}"
|
||||
return f"event:{event.get('id', '')}"
|
||||
|
||||
|
||||
def _fishing_event_rank(event: dict) -> tuple[str, str, float, str]:
|
||||
return (
|
||||
str(event.get("end", "") or ""),
|
||||
str(event.get("start", "") or ""),
|
||||
float(event.get("duration_hrs", 0) or 0),
|
||||
str(event.get("id", "") or ""),
|
||||
)
|
||||
|
||||
|
||||
def _dedupe_fishing_events(events: list[dict]) -> list[dict]:
|
||||
latest_by_vessel: dict[str, dict] = {}
|
||||
counts_by_vessel: dict[str, int] = {}
|
||||
|
||||
for event in events:
|
||||
vessel_key = _fishing_vessel_key(event)
|
||||
counts_by_vessel[vessel_key] = counts_by_vessel.get(vessel_key, 0) + 1
|
||||
current = latest_by_vessel.get(vessel_key)
|
||||
if current is None or _fishing_event_rank(event) > _fishing_event_rank(current):
|
||||
latest_by_vessel[vessel_key] = event
|
||||
|
||||
deduped: list[dict] = []
|
||||
for vessel_key, event in latest_by_vessel.items():
|
||||
event_copy = dict(event)
|
||||
event_copy["event_count"] = counts_by_vessel.get(vessel_key, 1)
|
||||
deduped.append(event_copy)
|
||||
|
||||
deduped.sort(key=_fishing_event_rank, reverse=True)
|
||||
return deduped
|
||||
|
||||
|
||||
_FISHING_FETCH_INTERVAL_S = 3600 # once per hour — GFW data has ~5 day lag
|
||||
_last_fishing_fetch_ts: float = 0.0
|
||||
|
||||
|
||||
@with_retry(max_retries=1, base_delay=5)
|
||||
def fetch_fishing_activity():
|
||||
"""Fetch recent fishing events from Global Fishing Watch (~5 day lag)."""
|
||||
from services.fetchers._store import is_any_active
|
||||
global _last_fishing_fetch_ts
|
||||
from services.fetchers._store import is_any_active, latest_data
|
||||
|
||||
if not is_any_active("fishing_activity"):
|
||||
return
|
||||
|
||||
# Skip if we already have data and fetched less than an hour ago
|
||||
now = time.time()
|
||||
if latest_data.get("fishing_activity") and (now - _last_fishing_fetch_ts) < _FISHING_FETCH_INTERVAL_S:
|
||||
return
|
||||
|
||||
token = os.environ.get("GFW_API_TOKEN", "")
|
||||
if not token:
|
||||
logger.debug("GFW_API_TOKEN not set, skipping fishing activity fetch")
|
||||
return
|
||||
events = []
|
||||
try:
|
||||
url = (
|
||||
"https://gateway.api.globalfishingwatch.org/v3/events"
|
||||
"?datasets[0]=public-global-fishing-events:latest"
|
||||
"&limit=500&sort=start&sort-direction=DESC"
|
||||
)
|
||||
import datetime as _dt
|
||||
|
||||
_end = _dt.date.today().isoformat()
|
||||
_start = (_dt.date.today() - _dt.timedelta(days=7)).isoformat()
|
||||
page_size = max(1, int(os.environ.get("GFW_EVENTS_PAGE_SIZE", "500") or "500"))
|
||||
offset = 0
|
||||
seen_offsets: set[int] = set()
|
||||
seen_ids: set[str] = set()
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
response = fetch_with_curl(url, timeout=30, headers=headers)
|
||||
if response.status_code == 200:
|
||||
entries = response.json().get("entries", [])
|
||||
|
||||
while True:
|
||||
if offset in seen_offsets:
|
||||
logger.warning("Fishing activity pagination repeated offset=%s; stopping fetch", offset)
|
||||
break
|
||||
seen_offsets.add(offset)
|
||||
|
||||
query = urlencode(
|
||||
{
|
||||
"datasets[0]": "public-global-fishing-events:latest",
|
||||
"start-date": _start,
|
||||
"end-date": _end,
|
||||
"limit": page_size,
|
||||
"offset": offset,
|
||||
}
|
||||
)
|
||||
url = f"https://gateway.api.globalfishingwatch.org/v3/events?{query}"
|
||||
response = fetch_with_curl(url, timeout=30, headers=headers)
|
||||
if response.status_code != 200:
|
||||
logger.warning(
|
||||
"Fishing activity fetch failed at offset=%s: HTTP %s",
|
||||
offset,
|
||||
response.status_code,
|
||||
)
|
||||
break
|
||||
|
||||
payload = response.json() or {}
|
||||
entries = payload.get("entries", [])
|
||||
if not entries:
|
||||
break
|
||||
|
||||
added_this_page = 0
|
||||
for e in entries:
|
||||
pos = e.get("position", {})
|
||||
vessel = e.get("vessel") or {}
|
||||
lat = pos.get("lat")
|
||||
lng = pos.get("lon")
|
||||
if lat is None or lng is None:
|
||||
continue
|
||||
event_id = str(e.get("id", "") or "")
|
||||
if event_id and event_id in seen_ids:
|
||||
continue
|
||||
if event_id:
|
||||
seen_ids.add(event_id)
|
||||
dur = e.get("event", {}).get("duration", 0) or 0
|
||||
events.append(
|
||||
{
|
||||
"id": e.get("id", ""),
|
||||
"id": event_id,
|
||||
"type": e.get("type", "fishing"),
|
||||
"lat": lat,
|
||||
"lng": lng,
|
||||
"start": e.get("start", ""),
|
||||
"end": e.get("end", ""),
|
||||
"vessel_name": (e.get("vessel") or {}).get("name", "Unknown"),
|
||||
"vessel_flag": (e.get("vessel") or {}).get("flag", ""),
|
||||
"vessel_id": str(vessel.get("id", "") or ""),
|
||||
"vessel_ssvid": str(vessel.get("ssvid", "") or ""),
|
||||
"vessel_name": vessel.get("name", "Unknown"),
|
||||
"vessel_flag": vessel.get("flag", ""),
|
||||
"duration_hrs": round(dur / 3600, 1),
|
||||
}
|
||||
)
|
||||
logger.info(f"Fishing activity: {len(events)} events")
|
||||
added_this_page += 1
|
||||
|
||||
if len(entries) < page_size:
|
||||
break
|
||||
|
||||
next_offset = payload.get("nextOffset")
|
||||
if next_offset is None:
|
||||
next_offset = (payload.get("pagination") or {}).get("nextOffset")
|
||||
if next_offset is None:
|
||||
next_offset = offset + page_size
|
||||
try:
|
||||
next_offset = int(next_offset)
|
||||
except (TypeError, ValueError):
|
||||
next_offset = offset + page_size
|
||||
if next_offset <= offset:
|
||||
logger.warning(
|
||||
"Fishing activity pagination produced non-increasing next offset=%s; stopping fetch",
|
||||
next_offset,
|
||||
)
|
||||
break
|
||||
if added_this_page == 0:
|
||||
logger.warning(
|
||||
"Fishing activity page at offset=%s added no new events; stopping fetch",
|
||||
offset,
|
||||
)
|
||||
break
|
||||
offset = next_offset
|
||||
raw_event_count = len(events)
|
||||
events = _dedupe_fishing_events(events)
|
||||
logger.info("Fishing activity: %s raw events -> %s deduped vessels", raw_event_count, len(events))
|
||||
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
|
||||
logger.error(f"Error fetching fishing activity: {e}")
|
||||
with _data_lock:
|
||||
latest_data["fishing_activity"] = events
|
||||
if events:
|
||||
_mark_fresh("fishing_activity")
|
||||
_last_fishing_fetch_ts = time.time()
|
||||
|
||||
@@ -25,7 +25,10 @@ logger = logging.getLogger("services.data_fetcher")
|
||||
_API_URL = "https://meshtastic.liamcottle.net/api/v1/nodes"
|
||||
_CACHE_FILE = Path(__file__).resolve().parent.parent.parent / "data" / "meshtastic_nodes_cache.json"
|
||||
_FETCH_TIMEOUT = 90 # seconds — response is ~37MB, needs time on slow connections
|
||||
_MAX_AGE_HOURS = 4 # discard nodes not seen within this window (matches refresh interval)
|
||||
_MAX_AGE_HOURS = 24 # discard nodes not seen within this window
|
||||
# Skip network fetch if cached data is fresher than this — the API is a
|
||||
# one-person hobby service, so we prefer stale data over hammering it.
|
||||
_CACHE_TRUST_HOURS = 20
|
||||
|
||||
# Track when we last fetched so the frontend can show staleness
|
||||
_last_fetch_ts: float = 0.0
|
||||
@@ -141,13 +144,54 @@ def fetch_meshtastic_nodes():
|
||||
return
|
||||
global _last_fetch_ts
|
||||
|
||||
# Trust a recent cache on disk — avoids hammering the upstream HTTP API
|
||||
# when every install polls on roughly the same cadence.
|
||||
try:
|
||||
if _CACHE_FILE.exists():
|
||||
mtime = _CACHE_FILE.stat().st_mtime
|
||||
if time.time() - mtime < _CACHE_TRUST_HOURS * 3600:
|
||||
# If memory is empty (cold start), hydrate from cache and skip fetch.
|
||||
with _data_lock:
|
||||
has_memory = bool(latest_data.get("meshtastic_map_nodes"))
|
||||
if not has_memory:
|
||||
cached = _load_cache()
|
||||
if cached:
|
||||
with _data_lock:
|
||||
latest_data["meshtastic_map_nodes"] = cached
|
||||
latest_data["meshtastic_map_fetched_at"] = mtime
|
||||
_mark_fresh("meshtastic_map")
|
||||
logger.info(
|
||||
"Meshtastic map: cache fresh (<%.0fh), skipping network fetch",
|
||||
_CACHE_TRUST_HOURS,
|
||||
)
|
||||
return
|
||||
else:
|
||||
logger.info(
|
||||
"Meshtastic map: cache fresh (<%.0fh), skipping network fetch",
|
||||
_CACHE_TRUST_HOURS,
|
||||
)
|
||||
return
|
||||
except Exception as e:
|
||||
logger.debug(f"Meshtastic cache freshness check failed: {e}")
|
||||
|
||||
# Build a polite User-Agent. Include the operator callsign when set so
|
||||
# the upstream service can correlate per-install traffic if needed.
|
||||
try:
|
||||
from services.config import get_settings
|
||||
|
||||
callsign = str(getattr(get_settings(), "MESHTASTIC_OPERATOR_CALLSIGN", "") or "").strip()
|
||||
except Exception:
|
||||
callsign = ""
|
||||
ua_base = "ShadowBroker-OSINT/0.9.7 (+https://github.com/BigBodyCobain/Shadowbroker; contact: bigbodycobain@gmail.com; 24h polling)"
|
||||
user_agent = f"{ua_base}; node={callsign}" if callsign else ua_base
|
||||
|
||||
try:
|
||||
logger.info("Fetching Meshtastic map nodes from API...")
|
||||
resp = requests.get(
|
||||
_API_URL,
|
||||
timeout=_FETCH_TIMEOUT,
|
||||
headers={
|
||||
"User-Agent": "ShadowBroker/1.0 (OSINT dashboard, 4h polling)",
|
||||
"User-Agent": user_agent,
|
||||
"Accept": "application/json",
|
||||
},
|
||||
)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import requests
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
@@ -296,17 +297,23 @@ def fetch_military_flights():
|
||||
with _data_lock:
|
||||
latest_data["military_flights"] = remaining_mil
|
||||
|
||||
# Store tracked military flights — update positions for existing entries
|
||||
# Store tracked military flights — update positions for existing entries.
|
||||
# Drop stale entries not refreshed by ANY source (civilian or military) within 5 min.
|
||||
_TRACKED_STALE_S = 300 # 5 minutes
|
||||
_merge_ts = time.time()
|
||||
|
||||
with _data_lock:
|
||||
existing_tracked = list(latest_data.get("tracked_flights", []))
|
||||
fresh_mil_map = {}
|
||||
for t in tracked_mil:
|
||||
icao = t.get("icao24", "").upper()
|
||||
if icao:
|
||||
t["_seen_at"] = _merge_ts
|
||||
fresh_mil_map[icao] = t
|
||||
|
||||
updated_tracked = []
|
||||
seen_icaos = set()
|
||||
stale_dropped = 0
|
||||
for old_t in existing_tracked:
|
||||
icao = old_t.get("icao24", "").upper()
|
||||
if icao in fresh_mil_map:
|
||||
@@ -317,11 +324,16 @@ def fetch_military_flights():
|
||||
updated_tracked.append(fresh)
|
||||
seen_icaos.add(icao)
|
||||
else:
|
||||
updated_tracked.append(old_t)
|
||||
seen_icaos.add(icao)
|
||||
# Keep stale entry only if it was seen recently
|
||||
age = _merge_ts - old_t.get("_seen_at", 0)
|
||||
if age < _TRACKED_STALE_S:
|
||||
updated_tracked.append(old_t)
|
||||
seen_icaos.add(icao)
|
||||
else:
|
||||
stale_dropped += 1
|
||||
for icao, t in fresh_mil_map.items():
|
||||
if icao not in seen_icaos:
|
||||
updated_tracked.append(t)
|
||||
with _data_lock:
|
||||
latest_data["tracked_flights"] = updated_tracked
|
||||
logger.info(f"Tracked flights: {len(updated_tracked)} total ({len(tracked_mil)} from military)")
|
||||
logger.info(f"Tracked flights: {len(updated_tracked)} total ({len(tracked_mil)} from military, {stale_dropped} stale dropped)")
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
"""News fetching, geocoding, clustering, and risk assessment."""
|
||||
import re
|
||||
import time
|
||||
import logging
|
||||
import calendar
|
||||
import concurrent.futures
|
||||
import requests
|
||||
import feedparser
|
||||
@@ -11,6 +13,10 @@ from services.oracle_service import enrich_news_items, compute_global_threat_lev
|
||||
|
||||
logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
# Maximum article age in seconds. Anything older than this is dropped
|
||||
# during each fetch cycle so the threat feed stays current.
|
||||
_MAX_ARTICLE_AGE_SECS = 48 * 3600 # 48 hours
|
||||
|
||||
|
||||
# Keyword -> coordinate mapping for geocoding news articles
|
||||
_KEYWORD_COORDS = {
|
||||
@@ -178,6 +184,17 @@ def fetch_news():
|
||||
if not feed:
|
||||
continue
|
||||
for entry in feed.entries[:5]:
|
||||
# Drop articles older than the max-age threshold so the
|
||||
# threat feed doesn't show stale stories across cycles.
|
||||
pp = entry.get("published_parsed")
|
||||
if pp:
|
||||
try:
|
||||
entry_epoch = calendar.timegm(pp)
|
||||
if time.time() - entry_epoch > _MAX_ARTICLE_AGE_SECS:
|
||||
continue
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
pass # unparseable date — keep the article
|
||||
|
||||
title = entry.get('title', '')
|
||||
summary = entry.get('summary', '')
|
||||
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
"""NUFORC Enrichment — downloads the Hugging Face NUFORC dataset and builds
|
||||
a compact spatial+temporal index for enriching tilequery hits with shape,
|
||||
duration, city, and summary text.
|
||||
|
||||
The full CSV (~170 MB) is streamed once and processed into a lightweight JSON
|
||||
cache (~1-3 MB) stored at ``backend/data/nuforc_enrichment.json``. Subsequent
|
||||
startups load from cache until it expires (30 days).
|
||||
|
||||
Index structure::
|
||||
|
||||
{
|
||||
"built": "2026-04-08T12:00:00",
|
||||
"count": 12345,
|
||||
"by_state": {
|
||||
"AZ": [
|
||||
{"d": "2024-01-15", "city": "Tucson", "shape": "triangle",
|
||||
"dur": "5 minutes", "summary": "Bright triangular object..."},
|
||||
...
|
||||
],
|
||||
...
|
||||
}
|
||||
}
|
||||
|
||||
Entries within each state are sorted by date descending (newest first).
|
||||
"""
|
||||
|
||||
import csv
|
||||
import gzip
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"
|
||||
_CACHE_FILE = _DATA_DIR / "nuforc_enrichment.json"
|
||||
_CACHE_TTL_DAYS = 1 # Rebuild daily — fresh data each cycle
|
||||
|
||||
# HuggingFace dataset — use the structured string export, not the old flat blob.
|
||||
_HF_CSV_URL = (
|
||||
"https://huggingface.co/datasets/kcimc/NUFORC/resolve/main/nuforc_str.csv"
|
||||
)
|
||||
|
||||
# Only keep sightings from the last N years for the enrichment index
|
||||
_KEEP_YEARS = 5
|
||||
|
||||
# ── In-memory index ────────────────────────────────────────────────────────
|
||||
_index: dict | None = None
|
||||
_index_lock = threading.Lock()
|
||||
_building = False
|
||||
|
||||
# US state abbreviations for parsing "City, ST" locations
|
||||
_US_STATES = {
|
||||
"AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DE", "FL", "GA",
|
||||
"HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD",
|
||||
"MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ",
|
||||
"NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", "RI", "SC",
|
||||
"SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY",
|
||||
"DC",
|
||||
}
|
||||
|
||||
|
||||
def _parse_location(loc: str) -> tuple[str, str]:
|
||||
"""Parse 'City, ST' or 'City, ST (explanation)' → (city, state_abbr).
|
||||
|
||||
Returns ('', '') if unparseable.
|
||||
"""
|
||||
if not loc:
|
||||
return "", ""
|
||||
loc = re.sub(r"\s*\(.*\)\s*$", "", loc).strip()
|
||||
parts = [p.strip() for p in loc.split(",") if p.strip()]
|
||||
if len(parts) < 2:
|
||||
return "", ""
|
||||
for idx in range(len(parts) - 1):
|
||||
candidate = parts[idx + 1].upper().strip()
|
||||
if candidate in _US_STATES:
|
||||
city = ", ".join(parts[: idx + 1]).strip()
|
||||
return city, candidate
|
||||
candidate = parts[-1].upper().strip()
|
||||
if candidate in _US_STATES:
|
||||
return ", ".join(parts[:-1]).strip(), candidate
|
||||
return parts[0], ""
|
||||
|
||||
|
||||
def _parse_date(date_str: str) -> str:
|
||||
"""Best-effort parse NUFORC date strings → 'YYYY-MM-DD'.
|
||||
|
||||
Returns '' on failure.
|
||||
"""
|
||||
if not date_str:
|
||||
return ""
|
||||
cleaned = str(date_str).strip()
|
||||
cleaned = re.sub(r"\s+local$", "", cleaned, flags=re.IGNORECASE)
|
||||
cleaned = re.sub(r"\s+utc$", "", cleaned, flags=re.IGNORECASE)
|
||||
cleaned = cleaned.replace("T", " ")
|
||||
for fmt in (
|
||||
"%m/%d/%Y %H:%M",
|
||||
"%m/%d/%Y %I:%M:%S %p",
|
||||
"%m/%d/%Y",
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%Y-%m-%d %H:%M",
|
||||
"%Y-%m-%d",
|
||||
):
|
||||
try:
|
||||
return datetime.strptime(cleaned, fmt).strftime("%Y-%m-%d")
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
match = re.match(r"^(\d{4}-\d{2}-\d{2})", cleaned)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return ""
|
||||
|
||||
|
||||
def _load_cache() -> dict | None:
|
||||
"""Load the on-disk cache if it exists and is fresh enough."""
|
||||
if not _CACHE_FILE.exists():
|
||||
return None
|
||||
try:
|
||||
raw = _CACHE_FILE.read_text(encoding="utf-8")
|
||||
data = json.loads(raw)
|
||||
built = data.get("built", "")
|
||||
if built:
|
||||
built_dt = datetime.fromisoformat(built)
|
||||
if datetime.utcnow() - built_dt < timedelta(days=_CACHE_TTL_DAYS):
|
||||
if int(data.get("count", 0) or 0) <= 0:
|
||||
logger.info("NUFORC enrichment: cache is fresh but empty; rebuilding")
|
||||
return None
|
||||
logger.info(
|
||||
"NUFORC enrichment: loaded cache (%d entries, built %s)",
|
||||
data.get("count", 0), built,
|
||||
)
|
||||
return data
|
||||
else:
|
||||
logger.info("NUFORC enrichment: cache expired (built %s)", built)
|
||||
except Exception as e:
|
||||
logger.warning("NUFORC enrichment: cache load error: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def _save_cache(data: dict):
|
||||
"""Persist the enrichment index to disk."""
|
||||
try:
|
||||
_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
_CACHE_FILE.write_text(json.dumps(data, separators=(",", ":")), encoding="utf-8")
|
||||
logger.info("NUFORC enrichment: saved cache (%d entries)", data.get("count", 0))
|
||||
except Exception as e:
|
||||
logger.warning("NUFORC enrichment: cache save error: %s", e)
|
||||
|
||||
|
||||
def _download_and_build() -> dict | None:
|
||||
"""Stream-download the HF CSV and build the enrichment index.
|
||||
|
||||
Returns the index dict or None on failure.
|
||||
"""
|
||||
cutoff = datetime.utcnow() - timedelta(days=_KEEP_YEARS * 365)
|
||||
cutoff_str = cutoff.strftime("%Y-%m-%d")
|
||||
|
||||
logger.info("NUFORC enrichment: downloading HF dataset (this may take a minute)...")
|
||||
try:
|
||||
resp = fetch_with_curl(_HF_CSV_URL, timeout=180, follow_redirects=True)
|
||||
if not resp or resp.status_code != 200:
|
||||
logger.warning(
|
||||
"NUFORC enrichment: download failed HTTP %s",
|
||||
getattr(resp, "status_code", "None"),
|
||||
)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error("NUFORC enrichment: download error: %s", e)
|
||||
return None
|
||||
|
||||
# Parse CSV from response text
|
||||
by_state: dict[str, list[dict]] = {}
|
||||
total = 0
|
||||
kept = 0
|
||||
|
||||
try:
|
||||
reader = csv.DictReader(io.StringIO(resp.text))
|
||||
for row in reader:
|
||||
total += 1
|
||||
occurred = _parse_date(
|
||||
row.get("Occurred", "")
|
||||
or row.get("Date / Time", "")
|
||||
or row.get("Date", "")
|
||||
)
|
||||
if not occurred or occurred < cutoff_str:
|
||||
continue
|
||||
|
||||
city, state = _parse_location(
|
||||
row.get("Location", "")
|
||||
or row.get("City", "")
|
||||
or row.get("location", "")
|
||||
)
|
||||
if not state:
|
||||
continue # can't index without state
|
||||
|
||||
shape = (row.get("Shape", "") or row.get("shape", "") or "").strip()
|
||||
duration = (row.get("Duration", "") or row.get("duration", "") or "").strip()
|
||||
summary = (
|
||||
row.get("Summary", "")
|
||||
or row.get("summary", "")
|
||||
or row.get("Text", "")
|
||||
or row.get("text", "")
|
||||
or ""
|
||||
).strip()
|
||||
if summary and len(summary) > 200:
|
||||
summary = summary[:197] + "..."
|
||||
|
||||
entry = {"d": occurred, "city": city, "shape": shape}
|
||||
if duration:
|
||||
entry["dur"] = duration
|
||||
if summary:
|
||||
entry["sum"] = summary
|
||||
|
||||
by_state.setdefault(state, []).append(entry)
|
||||
kept += 1
|
||||
except Exception as e:
|
||||
logger.error("NUFORC enrichment: CSV parse error: %s", e)
|
||||
return None
|
||||
|
||||
# Sort each state's entries by date descending (newest first)
|
||||
for st in by_state:
|
||||
by_state[st].sort(key=lambda e: e["d"], reverse=True)
|
||||
|
||||
data = {
|
||||
"built": datetime.utcnow().isoformat(),
|
||||
"count": kept,
|
||||
"by_state": by_state,
|
||||
}
|
||||
logger.info(
|
||||
"NUFORC enrichment: built index — %d entries from %d total rows (%d states)",
|
||||
kept, total, len(by_state),
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
def _ensure_index():
|
||||
"""Load or build the enrichment index (thread-safe, non-blocking)."""
|
||||
global _index, _building
|
||||
|
||||
with _index_lock:
|
||||
if _index is not None:
|
||||
return
|
||||
if _building:
|
||||
return # another thread is already building
|
||||
_building = True
|
||||
|
||||
# Try loading from disk first
|
||||
cached = _load_cache()
|
||||
if cached:
|
||||
with _index_lock:
|
||||
_index = cached
|
||||
_building = False
|
||||
return
|
||||
|
||||
# Download and build in background so we don't block startup
|
||||
def _build():
|
||||
global _index, _building
|
||||
try:
|
||||
result = _download_and_build()
|
||||
if result:
|
||||
_save_cache(result)
|
||||
with _index_lock:
|
||||
_index = result
|
||||
else:
|
||||
logger.warning("NUFORC enrichment: build failed, enrichment unavailable")
|
||||
finally:
|
||||
with _index_lock:
|
||||
_building = False
|
||||
|
||||
thread = threading.Thread(target=_build, name="nuforc-enrichment", daemon=True)
|
||||
thread.start()
|
||||
|
||||
|
||||
def refresh_enrichment_index():
|
||||
"""Force-rebuild the enrichment index. Called by the daily cron job.
|
||||
|
||||
Downloads the latest HF CSV, rebuilds the in-memory + disk cache.
|
||||
Runs synchronously (meant to be called from a background thread).
|
||||
"""
|
||||
global _index
|
||||
logger.info("NUFORC enrichment: daily refresh starting...")
|
||||
result = _download_and_build()
|
||||
if result:
|
||||
_save_cache(result)
|
||||
with _index_lock:
|
||||
_index = result
|
||||
logger.info("NUFORC enrichment: daily refresh complete (%d entries)", result.get("count", 0))
|
||||
else:
|
||||
logger.warning("NUFORC enrichment: daily refresh failed, keeping stale index")
|
||||
|
||||
|
||||
def enrich_sighting(state: str, from_date: str, to_date: str) -> dict:
|
||||
"""Look up enrichment data for a tilequery hit.
|
||||
|
||||
Args:
|
||||
state: 2-letter US state code (from reverse geocode)
|
||||
from_date: earliest sighting date (YYYY-MM-DD)
|
||||
to_date: latest sighting date (YYYY-MM-DD)
|
||||
|
||||
Returns:
|
||||
Dict with optional keys: city, shape, duration, summary.
|
||||
Empty dict if no match found.
|
||||
"""
|
||||
_ensure_index()
|
||||
|
||||
with _index_lock:
|
||||
idx = _index
|
||||
|
||||
if not idx or not state:
|
||||
return {}
|
||||
|
||||
entries = idx.get("by_state", {}).get(state, [])
|
||||
if not entries:
|
||||
return {}
|
||||
|
||||
# Find the best match by date proximity
|
||||
target = to_date or from_date
|
||||
if not target:
|
||||
# No date filter — just return the most recent entry for this state
|
||||
e = entries[0]
|
||||
else:
|
||||
best = None
|
||||
best_dist = 999999
|
||||
for e in entries:
|
||||
# Simple string distance on dates (YYYY-MM-DD sorts lexicographically)
|
||||
try:
|
||||
t = datetime.strptime(target, "%Y-%m-%d")
|
||||
d = datetime.strptime(e["d"], "%Y-%m-%d")
|
||||
dist = abs((t - d).days)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if dist < best_dist:
|
||||
best_dist = dist
|
||||
best = e
|
||||
if dist == 0:
|
||||
break # exact date match
|
||||
|
||||
if best is None or best_dist > 90:
|
||||
return {} # no match within 3 months
|
||||
e = best
|
||||
|
||||
result = {}
|
||||
if e.get("city"):
|
||||
result["city"] = e["city"]
|
||||
if e.get("shape"):
|
||||
result["shape"] = e["shape"]
|
||||
result["shape_raw"] = e["shape"]
|
||||
if e.get("dur"):
|
||||
result["duration"] = e["dur"]
|
||||
if e.get("sum"):
|
||||
result["summary"] = e["sum"]
|
||||
return result
|
||||
@@ -8,14 +8,33 @@ full metadata (volume, end dates, descriptions, source badges).
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import urlencode
|
||||
from cachetools import TTLCache, cached
|
||||
|
||||
logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
_market_cache = TTLCache(maxsize=1, ttl=60) # 60-second TTL — markets change fast
|
||||
|
||||
# Delta tracking: {market_title: previous_consensus_pct}
|
||||
_prev_probabilities: dict[str, float] = {}
|
||||
_market_cache = TTLCache(maxsize=1, ttl=300)
|
||||
_POLYMARKET_PAGE_DELAY_S = float(os.environ.get("MESH_POLYMARKET_PAGE_DELAY_S", "0.02"))
|
||||
_KALSHI_PAGE_DELAY_S = float(os.environ.get("MESH_KALSHI_PAGE_DELAY_S", "0.08"))
|
||||
_provider_pace_lock = threading.Lock()
|
||||
_provider_last_request_at: dict[str, float] = {}
|
||||
|
||||
|
||||
def _pace_provider(provider: str, min_interval_s: float) -> None:
|
||||
if min_interval_s <= 0:
|
||||
return
|
||||
with _provider_pace_lock:
|
||||
now = time.monotonic()
|
||||
wait_s = min_interval_s - (now - _provider_last_request_at.get(provider, 0.0))
|
||||
if wait_s > 0:
|
||||
time.sleep(wait_s)
|
||||
now = time.monotonic()
|
||||
_provider_last_request_at[provider] = now
|
||||
|
||||
|
||||
def _finite_or_none(value):
|
||||
@@ -28,7 +47,7 @@ def _finite_or_none(value):
|
||||
# ---------------------------------------------------------------------------
|
||||
# Category classification
|
||||
# ---------------------------------------------------------------------------
|
||||
CATEGORIES = ["POLITICS", "CONFLICT", "NEWS", "FINANCE", "CRYPTO"]
|
||||
CATEGORIES = ["POLITICS", "CONFLICT", "NEWS", "FINANCE", "CRYPTO", "SPORTS"]
|
||||
|
||||
_KALSHI_CATEGORY_MAP = {
|
||||
"Politics": "POLITICS",
|
||||
@@ -38,7 +57,7 @@ _KALSHI_CATEGORY_MAP = {
|
||||
"Tech": "FINANCE",
|
||||
"Science": "NEWS",
|
||||
"Climate and Weather": "NEWS",
|
||||
"Sports": "NEWS",
|
||||
"Sports": "SPORTS",
|
||||
"Culture": "NEWS",
|
||||
}
|
||||
|
||||
@@ -62,7 +81,14 @@ _TAG_CATEGORY_MAP = {
|
||||
"Ethereum": "CRYPTO",
|
||||
"AI": "NEWS",
|
||||
"Science": "NEWS",
|
||||
"Sports": "NEWS",
|
||||
"Sports": "SPORTS",
|
||||
"NBA": "SPORTS",
|
||||
"NFL": "SPORTS",
|
||||
"MLB": "SPORTS",
|
||||
"NHL": "SPORTS",
|
||||
"Soccer": "SPORTS",
|
||||
"Tennis": "SPORTS",
|
||||
"Golf": "SPORTS",
|
||||
"Culture": "NEWS",
|
||||
"Entertainment": "NEWS",
|
||||
"Tech": "FINANCE",
|
||||
@@ -152,6 +178,26 @@ _KEYWORD_CATEGORIES = {
|
||||
"market cap",
|
||||
"revenue",
|
||||
],
|
||||
"SPORTS": [
|
||||
"nba",
|
||||
"nfl",
|
||||
"mlb",
|
||||
"nhl",
|
||||
"wnba",
|
||||
"soccer",
|
||||
"football",
|
||||
"basketball",
|
||||
"baseball",
|
||||
"hockey",
|
||||
"ufc",
|
||||
"mma",
|
||||
"tennis",
|
||||
"golf",
|
||||
"championship",
|
||||
"playoffs",
|
||||
"world cup",
|
||||
"super bowl",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@@ -177,21 +223,186 @@ def _classify_category(title: str, poly_tags: list[str], kalshi_category: str) -
|
||||
return "NEWS"
|
||||
|
||||
|
||||
def _polymarket_event_to_entry(ev: dict) -> dict | None:
|
||||
title = ev.get("title", "")
|
||||
if not title:
|
||||
return None
|
||||
|
||||
markets = ev.get("markets", [])
|
||||
best_pct = None
|
||||
total_volume = 0
|
||||
outcomes = []
|
||||
for m in markets:
|
||||
raw_op = m.get("outcomePrices")
|
||||
price = None
|
||||
try:
|
||||
op = json.loads(raw_op) if isinstance(raw_op, str) else raw_op
|
||||
if isinstance(op, list) and len(op) >= 1:
|
||||
price = _finite_or_none(op[0])
|
||||
except (json.JSONDecodeError, ValueError, TypeError):
|
||||
pass
|
||||
if price is None:
|
||||
price = _finite_or_none(m.get("lastTradePrice") or m.get("bestBid"))
|
||||
pct = None
|
||||
if price is not None:
|
||||
try:
|
||||
pct = round(price * 100, 1)
|
||||
if best_pct is None or pct > best_pct:
|
||||
best_pct = pct
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
volume = _finite_or_none(m.get("volume", 0) or 0)
|
||||
if volume is not None:
|
||||
total_volume += volume
|
||||
oname = m.get("groupItemTitle") or ""
|
||||
if oname and pct is not None:
|
||||
outcomes.append({"name": oname, "pct": pct})
|
||||
|
||||
if len(outcomes) > 2:
|
||||
outcomes.sort(key=lambda x: x["pct"], reverse=True)
|
||||
else:
|
||||
outcomes = []
|
||||
|
||||
tag_labels = [t.get("label", "") for t in ev.get("tags", []) if t.get("label")]
|
||||
return {
|
||||
"title": title,
|
||||
"source": "polymarket",
|
||||
"pct": best_pct,
|
||||
"slug": ev.get("slug", ""),
|
||||
"description": ev.get("description") or "",
|
||||
"end_date": ev.get("endDate"),
|
||||
"volume": round(total_volume, 2),
|
||||
"volume_24h": round(_finite_or_none(ev.get("volume24hr", 0) or 0) or 0, 2),
|
||||
"tags": tag_labels,
|
||||
"outcomes": outcomes,
|
||||
}
|
||||
|
||||
|
||||
def _kalshi_market_pct(m: dict) -> float | None:
|
||||
bid = _finite_or_none(m.get("yes_bid_dollars"))
|
||||
ask = _finite_or_none(m.get("yes_ask_dollars"))
|
||||
last = _finite_or_none(m.get("last_price_dollars"))
|
||||
if bid is not None and ask is not None and ask >= bid:
|
||||
return round(((bid + ask) / 2) * 100, 1)
|
||||
if last is not None:
|
||||
return round(last * 100, 1)
|
||||
cents = _finite_or_none(m.get("yes_price") or m.get("last_price"))
|
||||
if cents is None:
|
||||
return None
|
||||
return round(cents * 100, 1) if cents <= 1 else round(cents, 1)
|
||||
|
||||
|
||||
def _kalshi_market_volume(m: dict) -> float:
|
||||
for key in ("volume_24h_fp", "volume_fp", "dollar_volume", "volume"):
|
||||
value = _finite_or_none(m.get(key))
|
||||
if value is not None:
|
||||
return value
|
||||
return 0
|
||||
|
||||
|
||||
def _kalshi_market_category(m: dict) -> str:
|
||||
text = " ".join(
|
||||
str(m.get(k, "") or "")
|
||||
for k in ("ticker", "event_ticker", "mve_collection_ticker", "title", "yes_sub_title", "no_sub_title")
|
||||
).lower()
|
||||
if any(token in text for token in ("sports", "xnba", "xnfl", "xmlb", "xnhl", "soccer", "tennis", "golf")):
|
||||
return "Sports"
|
||||
return str(m.get("category", "") or "")
|
||||
|
||||
|
||||
def _kalshi_event_to_entry(ev: dict, markets: list[dict] | None = None) -> dict | None:
|
||||
title = ev.get("title", "")
|
||||
if not title:
|
||||
return None
|
||||
|
||||
markets = markets or ev.get("markets", []) or []
|
||||
best_pct = None
|
||||
total_volume = 0.0
|
||||
close_dates = []
|
||||
outcomes = []
|
||||
first_ticker = ""
|
||||
descriptions = []
|
||||
for m in markets:
|
||||
first_ticker = first_ticker or m.get("ticker", "")
|
||||
pct = _kalshi_market_pct(m)
|
||||
if pct is not None:
|
||||
if best_pct is None or pct > best_pct:
|
||||
best_pct = pct
|
||||
oname = m.get("yes_sub_title") or m.get("sub_title") or m.get("title") or ""
|
||||
if oname and oname != title:
|
||||
outcomes.append({"name": oname, "pct": pct})
|
||||
total_volume += _kalshi_market_volume(m)
|
||||
cd = m.get("close_time") or m.get("close_date") or m.get("expiration_time")
|
||||
if cd:
|
||||
close_dates.append(cd)
|
||||
desc = (m.get("rules_primary") or m.get("rules_secondary") or "").strip()
|
||||
if desc:
|
||||
descriptions.append(desc)
|
||||
|
||||
if len(outcomes) > 2:
|
||||
outcomes.sort(key=lambda x: x["pct"], reverse=True)
|
||||
else:
|
||||
outcomes = []
|
||||
|
||||
desc = (ev.get("settle_details") or ev.get("underlying") or "").strip()
|
||||
if not desc and descriptions:
|
||||
desc = descriptions[0]
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"source": "kalshi",
|
||||
"pct": best_pct,
|
||||
"ticker": first_ticker or ev.get("event_ticker", "") or ev.get("ticker", ""),
|
||||
"description": desc,
|
||||
"sub_title": ev.get("sub_title", ""),
|
||||
"end_date": max(close_dates) if close_dates else None,
|
||||
"volume": round(total_volume, 2),
|
||||
"category": ev.get("category", ""),
|
||||
"outcomes": outcomes,
|
||||
}
|
||||
|
||||
|
||||
def _kalshi_market_to_entry(m: dict) -> dict | None:
|
||||
title = m.get("title") or m.get("yes_sub_title") or ""
|
||||
if not title:
|
||||
return None
|
||||
pct = _kalshi_market_pct(m)
|
||||
volume = _kalshi_market_volume(m)
|
||||
desc = (m.get("rules_primary") or m.get("rules_secondary") or "").strip()
|
||||
end_date = m.get("close_time") or m.get("expiration_time") or m.get("expected_expiration_time")
|
||||
return {
|
||||
"title": title,
|
||||
"source": "kalshi",
|
||||
"pct": pct,
|
||||
"ticker": m.get("ticker", "") or m.get("event_ticker", ""),
|
||||
"description": desc,
|
||||
"sub_title": m.get("subtitle", ""),
|
||||
"end_date": end_date,
|
||||
"volume": round(volume, 2),
|
||||
"category": _kalshi_market_category(m),
|
||||
"outcomes": [],
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Polymarket
|
||||
# ---------------------------------------------------------------------------
|
||||
def _fetch_polymarket_events() -> list[dict]:
|
||||
"""Fetch active events from Polymarket Gamma API (no auth required).
|
||||
|
||||
Fetches up to 500 events (multiple pages) for better search coverage.
|
||||
Fetches paginated active events, bounded by MESH_POLYMARKET_MAX_EVENTS
|
||||
so boot-time refresh does not become unbounded.
|
||||
"""
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
all_events = []
|
||||
for offset in range(0, 500, 100):
|
||||
page_size = 250
|
||||
max_events = int(os.environ.get("MESH_POLYMARKET_MAX_EVENTS", "5000"))
|
||||
for offset in range(0, max_events, page_size):
|
||||
try:
|
||||
_pace_provider("polymarket", _POLYMARKET_PAGE_DELAY_S)
|
||||
resp = fetch_with_curl(
|
||||
f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=100&offset={offset}",
|
||||
f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit={page_size}&offset={offset}",
|
||||
timeout=15,
|
||||
)
|
||||
if not resp or resp.status_code != 200:
|
||||
@@ -200,6 +411,8 @@ def _fetch_polymarket_events() -> list[dict]:
|
||||
if not isinstance(page, list) or not page:
|
||||
break
|
||||
all_events.extend(page)
|
||||
if len(page) < page_size:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"Polymarket page offset={offset} error: {e}")
|
||||
break
|
||||
@@ -286,6 +499,42 @@ def _fetch_kalshi_events() -> list[dict]:
|
||||
"""Fetch active events from Kalshi public API (no auth required)."""
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
try:
|
||||
max_events = int(os.environ.get("MESH_KALSHI_MAX_EVENTS", "2000"))
|
||||
page_size = 200
|
||||
markets = []
|
||||
cursor = ""
|
||||
while len(markets) < max_events:
|
||||
params = {"status": "open", "limit": str(page_size)}
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
_pace_provider("kalshi", _KALSHI_PAGE_DELAY_S)
|
||||
resp = fetch_with_curl(
|
||||
f"https://api.elections.kalshi.com/trade-api/v2/markets?{urlencode(params)}",
|
||||
timeout=15,
|
||||
)
|
||||
if not resp or resp.status_code != 200:
|
||||
break
|
||||
data = resp.json()
|
||||
page = data.get("markets", []) if isinstance(data, dict) else []
|
||||
if not page:
|
||||
break
|
||||
markets.extend(page)
|
||||
cursor = data.get("cursor") or ""
|
||||
if not cursor or len(page) < page_size:
|
||||
break
|
||||
|
||||
results = []
|
||||
for market in markets:
|
||||
entry = _kalshi_market_to_entry(market)
|
||||
if entry:
|
||||
results.append(entry)
|
||||
if results:
|
||||
logger.info(f"Kalshi: fetched {len(results)} active events from v2")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"Kalshi v2 fetch error, falling back to legacy v1: {e}")
|
||||
|
||||
try:
|
||||
resp = fetch_with_curl(
|
||||
"https://api.elections.kalshi.com/v1/events?status=open&limit=100",
|
||||
@@ -540,11 +789,11 @@ def fetch_prediction_markets():
|
||||
# ---------------------------------------------------------------------------
|
||||
# Direct API search (not limited to cached data)
|
||||
# ---------------------------------------------------------------------------
|
||||
def search_polymarket_direct(query: str, limit: int = 20) -> list[dict]:
|
||||
def search_polymarket_direct(query: str, limit: int = 20, offset: int = 0) -> list[dict]:
|
||||
"""Search Polymarket by scanning API pages for title matches.
|
||||
|
||||
The Gamma API has no text search parameter, so we scan cached events
|
||||
plus additional pages until we find enough matches or exhaust the scan.
|
||||
Prefer Polymarket's public search endpoint, then fall back to scanning
|
||||
Gamma event pages if search is unavailable.
|
||||
"""
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
@@ -552,11 +801,53 @@ def search_polymarket_direct(query: str, limit: int = 20) -> list[dict]:
|
||||
q_words = set(q_lower.split())
|
||||
results = []
|
||||
|
||||
try:
|
||||
params = urlencode({"q": query, "limit": str(limit), "offset": str(max(0, offset))})
|
||||
_pace_provider("polymarket", _POLYMARKET_PAGE_DELAY_S)
|
||||
resp = fetch_with_curl(
|
||||
f"https://gamma-api.polymarket.com/public-search?{params}",
|
||||
timeout=15,
|
||||
)
|
||||
if resp and resp.status_code == 200:
|
||||
data = resp.json()
|
||||
events = data.get("events", []) if isinstance(data, dict) else []
|
||||
for ev in events:
|
||||
if ev.get("closed") or ev.get("active") is False:
|
||||
continue
|
||||
entry = _polymarket_event_to_entry(ev)
|
||||
if not entry:
|
||||
continue
|
||||
category = _classify_category(entry["title"], entry.get("tags", []), "")
|
||||
pct = _finite_or_none(entry.get("pct"))
|
||||
sources = [{"name": "POLY", "pct": pct}] if pct is not None else []
|
||||
results.append(
|
||||
{
|
||||
"title": entry["title"],
|
||||
"polymarket_pct": pct,
|
||||
"kalshi_pct": None,
|
||||
"consensus_pct": pct,
|
||||
"description": entry.get("description", ""),
|
||||
"end_date": entry.get("end_date"),
|
||||
"volume": entry.get("volume", 0),
|
||||
"volume_24h": entry.get("volume_24h", 0),
|
||||
"kalshi_volume": 0,
|
||||
"category": category,
|
||||
"sources": sources,
|
||||
"slug": entry.get("slug", ""),
|
||||
"outcomes": entry.get("outcomes", []),
|
||||
}
|
||||
)
|
||||
logger.info(f"Polymarket search '{query}': {len(results)} results via public-search")
|
||||
return results[:limit]
|
||||
except Exception as e:
|
||||
logger.warning(f"Polymarket public-search '{query}' error: {e}")
|
||||
|
||||
# Scan up to 2000 events (10 pages of 200) looking for title matches
|
||||
for offset in range(0, 2000, 200):
|
||||
for scan_offset in range(0, 3000, 200):
|
||||
try:
|
||||
_pace_provider("polymarket", _POLYMARKET_PAGE_DELAY_S)
|
||||
resp = fetch_with_curl(
|
||||
f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=200&offset={offset}",
|
||||
f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=200&offset={scan_offset}",
|
||||
timeout=15,
|
||||
)
|
||||
if not resp or resp.status_code != 200:
|
||||
@@ -637,11 +928,168 @@ def search_polymarket_direct(query: str, limit: int = 20) -> list[dict]:
|
||||
}
|
||||
)
|
||||
# Stop scanning if we have enough results
|
||||
if len(results) >= limit:
|
||||
if len(results) >= offset + limit:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"Polymarket search scan offset={offset} error: {e}")
|
||||
logger.warning(f"Polymarket search scan offset={scan_offset} error: {e}")
|
||||
break
|
||||
|
||||
logger.info(f"Polymarket search '{query}': {len(results)} results (scanned API)")
|
||||
return results[:limit]
|
||||
return results[offset : offset + limit]
|
||||
|
||||
|
||||
def search_kalshi_direct(query: str, limit: int = 20, offset: int = 0) -> list[dict]:
|
||||
"""Search Kalshi events by scanning API pages for title matches."""
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
q_lower = query.lower()
|
||||
q_words = set(q_lower.split())
|
||||
results = []
|
||||
|
||||
try:
|
||||
max_scan = int(os.environ.get("MESH_KALSHI_SEARCH_SCAN_EVENTS", "1200"))
|
||||
page_size = 200
|
||||
cursor = ""
|
||||
scanned = 0
|
||||
while scanned < max_scan and len(results) < offset + limit:
|
||||
params = {"status": "open", "limit": str(page_size)}
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
_pace_provider("kalshi", _KALSHI_PAGE_DELAY_S)
|
||||
resp = fetch_with_curl(
|
||||
f"https://api.elections.kalshi.com/trade-api/v2/markets?{urlencode(params)}",
|
||||
timeout=15,
|
||||
)
|
||||
if not resp or resp.status_code != 200:
|
||||
break
|
||||
data = resp.json()
|
||||
markets = data.get("markets", []) if isinstance(data, dict) else []
|
||||
if not markets:
|
||||
break
|
||||
scanned += len(markets)
|
||||
for market in markets:
|
||||
haystack = " ".join(
|
||||
str(market.get(k, "") or "")
|
||||
for k in ("title", "yes_sub_title", "no_sub_title", "event_ticker", "ticker")
|
||||
).lower()
|
||||
if q_lower not in haystack and not any(w in haystack for w in q_words):
|
||||
continue
|
||||
entry = _kalshi_market_to_entry(market)
|
||||
if not entry:
|
||||
continue
|
||||
pct = _finite_or_none(entry.get("pct"))
|
||||
sources = [{"name": "KALSHI", "pct": pct}] if pct is not None else []
|
||||
category = _classify_category(entry["title"], [], entry.get("category", ""))
|
||||
results.append({
|
||||
"title": entry["title"],
|
||||
"polymarket_pct": None,
|
||||
"kalshi_pct": pct,
|
||||
"consensus_pct": pct,
|
||||
"description": entry.get("description", ""),
|
||||
"end_date": entry.get("end_date"),
|
||||
"volume": 0,
|
||||
"volume_24h": 0,
|
||||
"kalshi_volume": entry.get("volume", 0),
|
||||
"category": category,
|
||||
"sources": sources,
|
||||
"slug": "",
|
||||
"kalshi_ticker": entry.get("ticker", ""),
|
||||
"outcomes": entry.get("outcomes", []),
|
||||
})
|
||||
if len(results) >= offset + limit:
|
||||
break
|
||||
cursor = data.get("cursor") or ""
|
||||
if not cursor or len(markets) < page_size:
|
||||
break
|
||||
if results:
|
||||
logger.info(f"Kalshi search '{query}': {len(results)} results via v2 scan")
|
||||
return results[offset : offset + limit]
|
||||
except Exception as e:
|
||||
logger.warning(f"Kalshi v2 search '{query}' error, falling back to legacy v1: {e}")
|
||||
|
||||
try:
|
||||
resp = fetch_with_curl(
|
||||
"https://api.elections.kalshi.com/v1/events?status=open&limit=200",
|
||||
timeout=15,
|
||||
)
|
||||
if not resp or resp.status_code != 200:
|
||||
return []
|
||||
data = resp.json()
|
||||
events = data.get("events", []) if isinstance(data, dict) else []
|
||||
|
||||
for ev in events:
|
||||
title = ev.get("title", "")
|
||||
if not title:
|
||||
continue
|
||||
title_lower = title.lower()
|
||||
if q_lower not in title_lower and not any(w in title_lower for w in q_words):
|
||||
continue
|
||||
|
||||
markets = ev.get("markets", [])
|
||||
best_pct = None
|
||||
total_volume = 0
|
||||
close_dates = []
|
||||
outcomes = []
|
||||
for m in markets:
|
||||
price = m.get("yes_price") or m.get("last_price")
|
||||
pct = None
|
||||
if price is not None:
|
||||
try:
|
||||
price = _finite_or_none(price)
|
||||
if price is None:
|
||||
raise ValueError("non-finite")
|
||||
pct = round(price, 1)
|
||||
if pct <= 1:
|
||||
pct = round(pct * 100, 1)
|
||||
if best_pct is None or pct > best_pct:
|
||||
best_pct = pct
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
try:
|
||||
volume = _finite_or_none(
|
||||
m.get("dollar_volume", 0) or m.get("volume", 0) or 0
|
||||
)
|
||||
if volume is not None:
|
||||
total_volume += int(volume)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
cd = m.get("close_date")
|
||||
if cd:
|
||||
close_dates.append(cd)
|
||||
oname = m.get("title") or m.get("subtitle", "")
|
||||
if oname and pct is not None:
|
||||
outcomes.append({"name": oname, "pct": pct})
|
||||
if len(outcomes) > 2:
|
||||
outcomes.sort(key=lambda x: x["pct"], reverse=True)
|
||||
else:
|
||||
outcomes = []
|
||||
|
||||
desc = (ev.get("settle_details") or ev.get("underlying") or "").strip()
|
||||
category = _classify_category(title, [], ev.get("category", ""))
|
||||
sources = []
|
||||
if best_pct is not None:
|
||||
sources.append({"name": "KALSHI", "pct": best_pct})
|
||||
|
||||
results.append({
|
||||
"title": title,
|
||||
"polymarket_pct": None,
|
||||
"kalshi_pct": best_pct,
|
||||
"consensus_pct": best_pct,
|
||||
"description": desc,
|
||||
"end_date": max(close_dates) if close_dates else None,
|
||||
"volume": total_volume,
|
||||
"volume_24h": 0,
|
||||
"kalshi_volume": total_volume,
|
||||
"category": category,
|
||||
"sources": sources,
|
||||
"slug": "",
|
||||
"kalshi_ticker": ev.get("ticker", ""),
|
||||
"outcomes": outcomes,
|
||||
})
|
||||
if len(results) >= offset + limit:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"Kalshi search '{query}' error: {e}")
|
||||
|
||||
logger.info(f"Kalshi search '{query}': {len(results)} results")
|
||||
return results[offset : offset + limit]
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Static route + airport database loaded from vrs-standing-data.adsb.lol.
|
||||
|
||||
Replaces the per-batch /api/0/routeset POST with a single daily bulk download.
|
||||
Routes change ~weekly when airlines update schedules, so a 24h refresh cadence
|
||||
is far more than sufficient and removes ~all live-API pressure on adsb.lol.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import gzip
|
||||
import io
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_ROUTES_URL = "https://vrs-standing-data.adsb.lol/routes.csv.gz"
|
||||
_AIRPORTS_URL = "https://vrs-standing-data.adsb.lol/airports.csv.gz"
|
||||
_REFRESH_INTERVAL_S = 5 * 24 * 3600
|
||||
_HTTP_TIMEOUT_S = 60
|
||||
|
||||
_USER_AGENT = (
|
||||
"ShadowBroker-OSINT/0.9.7 "
|
||||
"(+https://github.com/BigBodyCobain/Shadowbroker; "
|
||||
"contact: bigbodycobain@gmail.com)"
|
||||
)
|
||||
|
||||
_lock = threading.RLock()
|
||||
_routes_by_callsign: dict[str, dict[str, Any]] = {}
|
||||
_airports_by_icao: dict[str, dict[str, Any]] = {}
|
||||
_last_refresh = 0.0
|
||||
_refresh_in_progress = False
|
||||
|
||||
|
||||
def _fetch_csv_gz(url: str) -> list[dict[str, str]]:
|
||||
response = requests.get(
|
||||
url,
|
||||
timeout=_HTTP_TIMEOUT_S,
|
||||
headers={"User-Agent": _USER_AGENT, "Accept-Encoding": "gzip"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
text = gzip.decompress(response.content).decode("utf-8-sig")
|
||||
return list(csv.DictReader(io.StringIO(text)))
|
||||
|
||||
|
||||
def _build_route_index(rows: list[dict[str, str]]) -> dict[str, dict[str, Any]]:
|
||||
index: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
callsign = (row.get("Callsign") or "").strip().upper()
|
||||
airport_codes = (row.get("AirportCodes") or "").strip()
|
||||
if not callsign or not airport_codes:
|
||||
continue
|
||||
icaos = [c.strip() for c in airport_codes.split("-") if c.strip()]
|
||||
if len(icaos) < 2:
|
||||
continue
|
||||
index[callsign] = {
|
||||
"airline_code": (row.get("AirlineCode") or "").strip(),
|
||||
"airport_codes": airport_codes,
|
||||
"airport_icaos": icaos,
|
||||
}
|
||||
return index
|
||||
|
||||
|
||||
def _build_airport_index(rows: list[dict[str, str]]) -> dict[str, dict[str, Any]]:
|
||||
index: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
icao = (row.get("ICAO") or "").strip().upper()
|
||||
if not icao:
|
||||
continue
|
||||
try:
|
||||
lat = float(row.get("Latitude") or 0)
|
||||
lon = float(row.get("Longitude") or 0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
index[icao] = {
|
||||
"name": (row.get("Name") or "").strip(),
|
||||
"iata": (row.get("IATA") or "").strip(),
|
||||
"country": (row.get("CountryISO2") or "").strip(),
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
}
|
||||
return index
|
||||
|
||||
|
||||
def refresh_route_database(force: bool = False) -> bool:
|
||||
"""Pull routes.csv.gz + airports.csv.gz and rebuild the in-memory indexes.
|
||||
|
||||
Returns True if a refresh was performed (success or attempted), False if
|
||||
skipped because the cache is still fresh or another refresh is in flight.
|
||||
"""
|
||||
global _last_refresh, _refresh_in_progress
|
||||
|
||||
now = time.time()
|
||||
with _lock:
|
||||
if _refresh_in_progress:
|
||||
return False
|
||||
if not force and (now - _last_refresh) < _REFRESH_INTERVAL_S and _routes_by_callsign:
|
||||
return False
|
||||
_refresh_in_progress = True
|
||||
|
||||
try:
|
||||
started = time.time()
|
||||
airport_rows = _fetch_csv_gz(_AIRPORTS_URL)
|
||||
route_rows = _fetch_csv_gz(_ROUTES_URL)
|
||||
airports = _build_airport_index(airport_rows)
|
||||
routes = _build_route_index(route_rows)
|
||||
with _lock:
|
||||
_airports_by_icao.clear()
|
||||
_airports_by_icao.update(airports)
|
||||
_routes_by_callsign.clear()
|
||||
_routes_by_callsign.update(routes)
|
||||
_last_refresh = time.time()
|
||||
logger.info(
|
||||
"route database refreshed in %.1fs: %d routes, %d airports",
|
||||
time.time() - started,
|
||||
len(routes),
|
||||
len(airports),
|
||||
)
|
||||
return True
|
||||
except (requests.RequestException, OSError, ValueError) as exc:
|
||||
logger.warning("route database refresh failed: %s", exc)
|
||||
return True
|
||||
finally:
|
||||
with _lock:
|
||||
_refresh_in_progress = False
|
||||
|
||||
|
||||
def lookup_route(callsign: str) -> dict[str, Any] | None:
|
||||
"""Resolve a callsign to {orig_name, dest_name, orig_loc, dest_loc} or None.
|
||||
|
||||
Matches the shape produced by the legacy fetch_routes_background cache so
|
||||
the caller in flights.py can be a drop-in replacement.
|
||||
"""
|
||||
key = (callsign or "").strip().upper()
|
||||
if not key:
|
||||
return None
|
||||
with _lock:
|
||||
route = _routes_by_callsign.get(key)
|
||||
if not route:
|
||||
return None
|
||||
icaos = route["airport_icaos"]
|
||||
orig = _airports_by_icao.get(icaos[0].upper())
|
||||
dest = _airports_by_icao.get(icaos[-1].upper())
|
||||
if not orig or not dest:
|
||||
return None
|
||||
return {
|
||||
"orig_name": f"{orig['iata']}: {orig['name']}" if orig["iata"] else orig["name"],
|
||||
"dest_name": f"{dest['iata']}: {dest['name']}" if dest["iata"] else dest["name"],
|
||||
"orig_loc": [orig["lon"], orig["lat"]],
|
||||
"dest_loc": [dest["lon"], dest["lat"]],
|
||||
}
|
||||
|
||||
|
||||
def route_database_status() -> dict[str, Any]:
|
||||
with _lock:
|
||||
return {
|
||||
"last_refresh": _last_refresh,
|
||||
"routes": len(_routes_by_callsign),
|
||||
"airports": len(_airports_by_icao),
|
||||
"in_progress": _refresh_in_progress,
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
"""SAR catalog fetcher (Mode A — default-on, free, no account).
|
||||
|
||||
Hits ASF Search every hour for Sentinel-1 scenes that touched any of
|
||||
the operator-defined AOIs in the last ~36h. Pure metadata, no
|
||||
downloads.
|
||||
|
||||
Result is written to ``latest_data["sar_scenes"]`` and a per-AOI
|
||||
coverage summary to ``latest_data["sar_aoi_coverage"]``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
|
||||
from services.fetchers.retry import with_retry
|
||||
from services.sar.sar_aoi import load_aois
|
||||
from services.sar.sar_catalog_client import estimate_next_pass, search_scenes_for_aoi
|
||||
from services.sar.sar_config import catalog_enabled
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@with_retry(max_retries=1, base_delay=2)
|
||||
def fetch_sar_catalog() -> None:
|
||||
"""Refresh the SAR scene catalog for all configured AOIs."""
|
||||
if not catalog_enabled():
|
||||
return
|
||||
if not is_any_active("sar"):
|
||||
return
|
||||
aois = load_aois()
|
||||
if not aois:
|
||||
logger.debug("SAR catalog: no AOIs configured")
|
||||
return
|
||||
|
||||
all_scenes: list[dict] = []
|
||||
coverage: list[dict] = []
|
||||
for aoi in aois:
|
||||
try:
|
||||
scenes = search_scenes_for_aoi(aoi)
|
||||
except (ConnectionError, TimeoutError, OSError, ValueError) as exc:
|
||||
logger.debug("SAR catalog %s: %s", aoi.id, exc)
|
||||
scenes = []
|
||||
scene_dicts = [s.to_dict() for s in scenes]
|
||||
all_scenes.extend(scene_dicts)
|
||||
next_pass = estimate_next_pass(scenes)
|
||||
coverage.append(
|
||||
{
|
||||
"aoi_id": aoi.id,
|
||||
"aoi_name": aoi.name,
|
||||
"category": aoi.category,
|
||||
"center_lat": aoi.center_lat,
|
||||
"center_lon": aoi.center_lon,
|
||||
"radius_km": aoi.radius_km,
|
||||
"recent_scene_count": len(scene_dicts),
|
||||
"latest_scene_time": (
|
||||
max((s["time"] for s in scene_dicts), default="")
|
||||
if scene_dicts
|
||||
else ""
|
||||
),
|
||||
**next_pass,
|
||||
}
|
||||
)
|
||||
|
||||
with _data_lock:
|
||||
latest_data["sar_scenes"] = all_scenes
|
||||
latest_data["sar_aoi_coverage"] = coverage
|
||||
if all_scenes or coverage:
|
||||
_mark_fresh("sar_scenes", "sar_aoi_coverage")
|
||||
logger.info(
|
||||
"SAR catalog: %d scenes across %d AOIs",
|
||||
len(all_scenes),
|
||||
len(aois),
|
||||
)
|
||||
@@ -0,0 +1,103 @@
|
||||
"""SAR pre-processed product fetcher (Mode B — opt-in, free, account needed).
|
||||
|
||||
Pulls already-computed deformation, flood, water, and damage products
|
||||
from NASA OPERA, Copernicus EGMS, GFM, EMS, and UNOSAT. No local DSP.
|
||||
|
||||
Two-step opt-in: ``MESH_SAR_PRODUCTS_FETCH=allow`` AND
|
||||
``MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE=true``. When either flag is
|
||||
unset, this fetcher logs a single startup hint and returns.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
|
||||
from services.fetchers.retry import with_retry
|
||||
from services.sar.sar_aoi import load_aois
|
||||
from services.sar.sar_config import products_fetch_enabled, products_fetch_status
|
||||
from services.sar.sar_normalize import SarAnomaly
|
||||
from services.sar.sar_products_client import (
|
||||
fetch_egms_for_aoi,
|
||||
fetch_ems_for_aoi,
|
||||
fetch_gfm_for_aoi,
|
||||
fetch_opera_for_aoi,
|
||||
fetch_unosat_for_aoi,
|
||||
)
|
||||
from services.sar.sar_signing import emit_signed_anomaly
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_LOGGED_DISABLED_HINT = False
|
||||
|
||||
|
||||
def _hint_disabled_once() -> None:
|
||||
global _LOGGED_DISABLED_HINT
|
||||
if _LOGGED_DISABLED_HINT:
|
||||
return
|
||||
_LOGGED_DISABLED_HINT = True
|
||||
status = products_fetch_status()
|
||||
missing = ", ".join(status.get("missing", [])) or "nothing"
|
||||
logger.info(
|
||||
"SAR Mode B (ground-change alerts) is disabled. Missing: %s. "
|
||||
"Enable in Settings → SAR or set the env vars listed in .env.example. "
|
||||
"Free signup: https://urs.earthdata.nasa.gov/users/new",
|
||||
missing,
|
||||
)
|
||||
|
||||
|
||||
@with_retry(max_retries=1, base_delay=3)
|
||||
def fetch_sar_products() -> None:
|
||||
"""Refresh pre-processed SAR anomalies for all configured AOIs."""
|
||||
if not products_fetch_enabled():
|
||||
_hint_disabled_once()
|
||||
return
|
||||
if not is_any_active("sar"):
|
||||
return
|
||||
aois = load_aois()
|
||||
if not aois:
|
||||
logger.debug("SAR products: no AOIs configured")
|
||||
return
|
||||
|
||||
seen_ids: set[str] = set()
|
||||
all_anomalies: list[dict[str, Any]] = []
|
||||
publish_summary = {"signed": 0, "skipped": 0, "reasons": {}}
|
||||
|
||||
for aoi in aois:
|
||||
for fetcher in (
|
||||
fetch_opera_for_aoi,
|
||||
fetch_egms_for_aoi,
|
||||
fetch_gfm_for_aoi,
|
||||
fetch_ems_for_aoi,
|
||||
fetch_unosat_for_aoi,
|
||||
):
|
||||
try:
|
||||
anomalies: list[SarAnomaly] = fetcher(aoi) or []
|
||||
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as exc:
|
||||
logger.debug("SAR %s for %s failed: %s", fetcher.__name__, aoi.id, exc)
|
||||
anomalies = []
|
||||
for a in anomalies:
|
||||
if a.anomaly_id in seen_ids:
|
||||
continue
|
||||
seen_ids.add(a.anomaly_id)
|
||||
all_anomalies.append(a.to_dict())
|
||||
status = emit_signed_anomaly(a)
|
||||
if status.get("signed"):
|
||||
publish_summary["signed"] += 1
|
||||
else:
|
||||
publish_summary["skipped"] += 1
|
||||
reason = status.get("reason", "unknown")
|
||||
publish_summary["reasons"][reason] = (
|
||||
publish_summary["reasons"].get(reason, 0) + 1
|
||||
)
|
||||
|
||||
with _data_lock:
|
||||
latest_data["sar_anomalies"] = all_anomalies
|
||||
if all_anomalies:
|
||||
_mark_fresh("sar_anomalies")
|
||||
logger.info(
|
||||
"SAR products: %d anomalies (%d signed, %d skipped)",
|
||||
len(all_anomalies),
|
||||
publish_summary["signed"],
|
||||
publish_summary["skipped"],
|
||||
)
|
||||
@@ -5,6 +5,11 @@ CelesTrak Fair Use Policy (https://celestrak.org/NORAD/elements/):
|
||||
- Use If-Modified-Since headers for conditional requests
|
||||
- No parallel/concurrent connections — one request at a time
|
||||
- Set a descriptive User-Agent
|
||||
|
||||
Analysis features (derived from cached TLEs — no extra network requests):
|
||||
- Maneuver detection: TLE-to-TLE comparison per satellite
|
||||
- Decay anomaly: mean-motion change rate monitoring
|
||||
- Overflight counting: 24h ground-track sampling for a bounding box
|
||||
"""
|
||||
|
||||
import math
|
||||
@@ -41,6 +46,67 @@ _sat_classified_cache = {"data": None, "gp_fetch_ts": 0}
|
||||
_SAT_CACHE_PATH = Path(__file__).parent.parent.parent / "data" / "sat_gp_cache.json"
|
||||
_SAT_CACHE_META_PATH = Path(__file__).parent.parent.parent / "data" / "sat_gp_cache_meta.json"
|
||||
|
||||
# ── Historical TLE storage for maneuver & decay detection ───────────────────
|
||||
# Stores the previous TLE snapshot keyed by NORAD_CAT_ID.
|
||||
# Populated when a fresh CelesTrak fetch replaces cached data.
|
||||
# Persisted to disk so analysis survives restarts.
|
||||
_SAT_HISTORY_PATH = Path(__file__).parent.parent.parent / "data" / "sat_tle_history.json"
|
||||
_tle_history: dict[int, dict] = {} # {norad_id: {elements + "epoch_ts"}}
|
||||
|
||||
|
||||
def _load_tle_history():
|
||||
"""Load previous TLE snapshot from disk."""
|
||||
global _tle_history
|
||||
try:
|
||||
if _SAT_HISTORY_PATH.exists():
|
||||
with open(_SAT_HISTORY_PATH, "r") as f:
|
||||
raw = json.load(f)
|
||||
_tle_history = {int(k): v for k, v in raw.items()}
|
||||
logger.info(f"Satellites: Loaded TLE history for {len(_tle_history)} objects")
|
||||
except (IOError, OSError, json.JSONDecodeError, ValueError, KeyError) as e:
|
||||
logger.warning(f"Satellites: Failed to load TLE history: {e}")
|
||||
_tle_history = {}
|
||||
|
||||
|
||||
def _save_tle_history():
|
||||
"""Persist current TLE snapshot as history for next comparison."""
|
||||
try:
|
||||
_SAT_HISTORY_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(_SAT_HISTORY_PATH, "w") as f:
|
||||
json.dump(_tle_history, f)
|
||||
except (IOError, OSError) as e:
|
||||
logger.warning(f"Satellites: Failed to save TLE history: {e}")
|
||||
|
||||
|
||||
def _snapshot_current_tles(gp_data):
|
||||
"""Capture orbital elements from current GP data as the new 'previous' snapshot.
|
||||
|
||||
Called once per CelesTrak fetch (every 24h). The old snapshot becomes
|
||||
the comparison baseline for maneuver/decay detection.
|
||||
"""
|
||||
global _tle_history
|
||||
new_snapshot = {}
|
||||
for sat in gp_data:
|
||||
norad_id = sat.get("NORAD_CAT_ID")
|
||||
if norad_id is None:
|
||||
continue
|
||||
epoch_str = sat.get("EPOCH", "")
|
||||
try:
|
||||
epoch_dt = datetime.strptime(epoch_str[:19], "%Y-%m-%dT%H:%M:%S")
|
||||
epoch_ts = epoch_dt.timestamp()
|
||||
except (ValueError, TypeError):
|
||||
epoch_ts = 0
|
||||
new_snapshot[int(norad_id)] = {
|
||||
"MEAN_MOTION": sat.get("MEAN_MOTION"),
|
||||
"ECCENTRICITY": sat.get("ECCENTRICITY"),
|
||||
"INCLINATION": sat.get("INCLINATION"),
|
||||
"RA_OF_ASC_NODE": sat.get("RA_OF_ASC_NODE"),
|
||||
"BSTAR": sat.get("BSTAR"),
|
||||
"epoch_ts": epoch_ts,
|
||||
}
|
||||
_tle_history = new_snapshot
|
||||
_save_tle_history()
|
||||
|
||||
|
||||
def _load_sat_cache():
|
||||
"""Load satellite GP data from local disk cache."""
|
||||
@@ -99,360 +165,368 @@ def _save_cache_meta():
|
||||
|
||||
|
||||
# Satellite intelligence classification database
|
||||
# Matched by substring against OBJECT_NAME (case-insensitive).
|
||||
# Order matters — first match wins, so specific names go before generic prefixes.
|
||||
_SAT_INTEL_DB = [
|
||||
(
|
||||
"USA 224",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "KH-11 Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
|
||||
},
|
||||
),
|
||||
(
|
||||
"USA 245",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "KH-11 Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
|
||||
},
|
||||
),
|
||||
(
|
||||
"USA 290",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "KH-11 Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
|
||||
},
|
||||
),
|
||||
(
|
||||
"USA 314",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "KH-11 Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
|
||||
},
|
||||
),
|
||||
(
|
||||
"USA 338",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "Keyhole Successor",
|
||||
"wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
|
||||
},
|
||||
),
|
||||
(
|
||||
"TOPAZ",
|
||||
{
|
||||
"country": "Russia",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "Optical Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"PERSONA",
|
||||
{
|
||||
"country": "Russia",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "Optical Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"KONDOR",
|
||||
{
|
||||
"country": "Russia",
|
||||
"mission": "military_sar",
|
||||
"sat_type": "SAR Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Kondor_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"BARS-M",
|
||||
{
|
||||
"country": "Russia",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "Mapping Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Bars-M",
|
||||
},
|
||||
),
|
||||
(
|
||||
"YAOGAN",
|
||||
{
|
||||
"country": "China",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "Remote Sensing / ELINT",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Yaogan",
|
||||
},
|
||||
),
|
||||
(
|
||||
"GAOFEN",
|
||||
{
|
||||
"country": "China",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "High-Res Imaging",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Gaofen",
|
||||
},
|
||||
),
|
||||
(
|
||||
"JILIN",
|
||||
{
|
||||
"country": "China",
|
||||
"mission": "commercial_imaging",
|
||||
"sat_type": "Video / Imaging",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Jilin-1",
|
||||
},
|
||||
),
|
||||
(
|
||||
"OFEK",
|
||||
{
|
||||
"country": "Israel",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Ofeq",
|
||||
},
|
||||
),
|
||||
(
|
||||
"CSO",
|
||||
{
|
||||
"country": "France",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "Optical Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/CSO_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"IGS",
|
||||
{
|
||||
"country": "Japan",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "Intelligence Gathering",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Information_Gathering_Satellite",
|
||||
},
|
||||
),
|
||||
(
|
||||
"CAPELLA",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "sar",
|
||||
"sat_type": "SAR Imaging",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Capella_Space",
|
||||
},
|
||||
),
|
||||
(
|
||||
"ICEYE",
|
||||
{
|
||||
"country": "Finland",
|
||||
"mission": "sar",
|
||||
"sat_type": "SAR Microsatellite",
|
||||
"wiki": "https://en.wikipedia.org/wiki/ICEYE",
|
||||
},
|
||||
),
|
||||
(
|
||||
"COSMO-SKYMED",
|
||||
{
|
||||
"country": "Italy",
|
||||
"mission": "sar",
|
||||
"sat_type": "SAR Constellation",
|
||||
"wiki": "https://en.wikipedia.org/wiki/COSMO-SkyMed",
|
||||
},
|
||||
),
|
||||
(
|
||||
"TANDEM",
|
||||
{
|
||||
"country": "Germany",
|
||||
"mission": "sar",
|
||||
"sat_type": "SAR Interferometry",
|
||||
"wiki": "https://en.wikipedia.org/wiki/TanDEM-X",
|
||||
},
|
||||
),
|
||||
(
|
||||
"PAZ",
|
||||
{
|
||||
"country": "Spain",
|
||||
"mission": "sar",
|
||||
"sat_type": "SAR Imaging",
|
||||
"wiki": "https://en.wikipedia.org/wiki/PAZ_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"WORLDVIEW",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "commercial_imaging",
|
||||
"sat_type": "Maxar High-Res",
|
||||
"wiki": "https://en.wikipedia.org/wiki/WorldView-3",
|
||||
},
|
||||
),
|
||||
(
|
||||
"GEOEYE",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "commercial_imaging",
|
||||
"sat_type": "Maxar Imaging",
|
||||
"wiki": "https://en.wikipedia.org/wiki/GeoEye-1",
|
||||
},
|
||||
),
|
||||
(
|
||||
"PLEIADES",
|
||||
{
|
||||
"country": "France",
|
||||
"mission": "commercial_imaging",
|
||||
"sat_type": "Airbus Imaging",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Pl%C3%A9iades_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"SPOT",
|
||||
{
|
||||
"country": "France",
|
||||
"mission": "commercial_imaging",
|
||||
"sat_type": "Airbus Medium-Res",
|
||||
"wiki": "https://en.wikipedia.org/wiki/SPOT_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"PLANET",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "commercial_imaging",
|
||||
"sat_type": "PlanetScope",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Planet_Labs",
|
||||
},
|
||||
),
|
||||
(
|
||||
"SKYSAT",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "commercial_imaging",
|
||||
"sat_type": "Planet Video",
|
||||
"wiki": "https://en.wikipedia.org/wiki/SkySat",
|
||||
},
|
||||
),
|
||||
(
|
||||
"BLACKSKY",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "commercial_imaging",
|
||||
"sat_type": "BlackSky Imaging",
|
||||
"wiki": "https://en.wikipedia.org/wiki/BlackSky",
|
||||
},
|
||||
),
|
||||
(
|
||||
"NROL",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "sigint",
|
||||
"sat_type": "Classified NRO",
|
||||
"wiki": "https://en.wikipedia.org/wiki/National_Reconnaissance_Office",
|
||||
},
|
||||
),
|
||||
(
|
||||
"MENTOR",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "sigint",
|
||||
"sat_type": "SIGINT / ELINT",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Mentor_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"LUCH",
|
||||
{
|
||||
"country": "Russia",
|
||||
"mission": "sigint",
|
||||
"sat_type": "Relay / SIGINT",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Luch_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"SHIJIAN",
|
||||
{
|
||||
"country": "China",
|
||||
"mission": "sigint",
|
||||
"sat_type": "ELINT / Tech Demo",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Shijian",
|
||||
},
|
||||
),
|
||||
(
|
||||
"NAVSTAR",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "navigation",
|
||||
"sat_type": "GPS",
|
||||
"wiki": "https://en.wikipedia.org/wiki/GPS_satellite_blocks",
|
||||
},
|
||||
),
|
||||
(
|
||||
"GLONASS",
|
||||
{
|
||||
"country": "Russia",
|
||||
"mission": "navigation",
|
||||
"sat_type": "GLONASS",
|
||||
"wiki": "https://en.wikipedia.org/wiki/GLONASS",
|
||||
},
|
||||
),
|
||||
(
|
||||
"BEIDOU",
|
||||
{
|
||||
"country": "China",
|
||||
"mission": "navigation",
|
||||
"sat_type": "BeiDou",
|
||||
"wiki": "https://en.wikipedia.org/wiki/BeiDou",
|
||||
},
|
||||
),
|
||||
(
|
||||
"GALILEO",
|
||||
{
|
||||
"country": "EU",
|
||||
"mission": "navigation",
|
||||
"sat_type": "Galileo",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Galileo_(satellite_navigation)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"SBIRS",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "early_warning",
|
||||
"sat_type": "Missile Warning",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Space-Based_Infrared_System",
|
||||
},
|
||||
),
|
||||
(
|
||||
"TUNDRA",
|
||||
{
|
||||
"country": "Russia",
|
||||
"mission": "early_warning",
|
||||
"sat_type": "Missile Warning",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Tundra_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"ISS",
|
||||
{
|
||||
"country": "Intl",
|
||||
"mission": "space_station",
|
||||
"sat_type": "Space Station",
|
||||
"wiki": "https://en.wikipedia.org/wiki/International_Space_Station",
|
||||
},
|
||||
),
|
||||
(
|
||||
"TIANGONG",
|
||||
{
|
||||
"country": "China",
|
||||
"mission": "space_station",
|
||||
"sat_type": "Space Station",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Tiangong_space_station",
|
||||
},
|
||||
),
|
||||
# ── USA Keyhole / Reconnaissance ────────────────────────────────────────
|
||||
("USA 224", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
|
||||
("USA 245", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
|
||||
("USA 290", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
|
||||
("USA 314", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
|
||||
("USA 338", {"country": "USA", "mission": "military_recon", "sat_type": "Keyhole Successor", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
|
||||
# ── USA SIGINT / NRO ────────────────────────────────────────────────────
|
||||
("NROL", {"country": "USA", "mission": "sigint", "sat_type": "Classified NRO", "wiki": "https://en.wikipedia.org/wiki/National_Reconnaissance_Office"}),
|
||||
("MENTOR", {"country": "USA", "mission": "sigint", "sat_type": "SIGINT / ELINT (Orion)", "wiki": "https://en.wikipedia.org/wiki/Mentor_(satellite)"}),
|
||||
("TRUMPET", {"country": "USA", "mission": "sigint", "sat_type": "SIGINT (HEO)", "wiki": "https://en.wikipedia.org/wiki/Trumpet_(satellite)"}),
|
||||
("INTRUDER", {"country": "USA", "mission": "sigint", "sat_type": "Naval SIGINT (NOSS)", "wiki": "https://en.wikipedia.org/wiki/Naval_Ocean_Surveillance_System"}),
|
||||
# ── USA Early Warning / Missile Defense ─────────────────────────────────
|
||||
("SBIRS", {"country": "USA", "mission": "early_warning", "sat_type": "Missile Warning", "wiki": "https://en.wikipedia.org/wiki/Space-Based_Infrared_System"}),
|
||||
("DSP", {"country": "USA", "mission": "early_warning", "sat_type": "Defense Support Program", "wiki": "https://en.wikipedia.org/wiki/Defense_Support_Program"}),
|
||||
# ── USA Communications (Military) ───────────────────────────────────────
|
||||
("MUOS", {"country": "USA", "mission": "military_comms", "sat_type": "Mobile User Objective System", "wiki": "https://en.wikipedia.org/wiki/Mobile_User_Objective_System"}),
|
||||
("AEHF", {"country": "USA", "mission": "military_comms", "sat_type": "Advanced EHF", "wiki": "https://en.wikipedia.org/wiki/Advanced_Extremely_High_Frequency"}),
|
||||
("WGS", {"country": "USA", "mission": "military_comms", "sat_type": "Wideband Global SATCOM", "wiki": "https://en.wikipedia.org/wiki/Wideband_Global_SATCOM"}),
|
||||
("MILSTAR", {"country": "USA", "mission": "military_comms", "sat_type": "Milstar Secure Comms", "wiki": "https://en.wikipedia.org/wiki/Milstar"}),
|
||||
# ── USA Navigation ──────────────────────────────────────────────────────
|
||||
("NAVSTAR", {"country": "USA", "mission": "navigation", "sat_type": "GPS", "wiki": "https://en.wikipedia.org/wiki/GPS_satellite_blocks"}),
|
||||
# ── Russia Reconnaissance ───────────────────────────────────────────────
|
||||
("TOPAZ", {"country": "Russia", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)"}),
|
||||
("PERSONA", {"country": "Russia", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)"}),
|
||||
("KONDOR", {"country": "Russia", "mission": "military_sar", "sat_type": "SAR Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Kondor_(satellite)"}),
|
||||
("BARS-M", {"country": "Russia", "mission": "military_recon", "sat_type": "Mapping Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Bars-M"}),
|
||||
("RAZDAN", {"country": "Russia", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Razdan_(satellite)"}),
|
||||
("LOTOS", {"country": "Russia", "mission": "sigint", "sat_type": "ELINT (Lotos-S)", "wiki": "https://en.wikipedia.org/wiki/Lotos-S"}),
|
||||
("PION", {"country": "Russia", "mission": "sigint", "sat_type": "Naval SIGINT/Radar", "wiki": "https://en.wikipedia.org/wiki/Pion-NKS"}),
|
||||
("LUCH", {"country": "Russia", "mission": "sigint", "sat_type": "Relay / SIGINT", "wiki": "https://en.wikipedia.org/wiki/Luch_(satellite)"}),
|
||||
# ── Russia Early Warning & Navigation ───────────────────────────────────
|
||||
("TUNDRA", {"country": "Russia", "mission": "early_warning", "sat_type": "Missile Warning (EKS)", "wiki": "https://en.wikipedia.org/wiki/Tundra_(satellite)"}),
|
||||
("GLONASS", {"country": "Russia", "mission": "navigation", "sat_type": "GLONASS", "wiki": "https://en.wikipedia.org/wiki/GLONASS"}),
|
||||
# ── China Military / Intel ──────────────────────────────────────────────
|
||||
("YAOGAN", {"country": "China", "mission": "military_recon", "sat_type": "Remote Sensing / ELINT", "wiki": "https://en.wikipedia.org/wiki/Yaogan"}),
|
||||
("GAOFEN", {"country": "China", "mission": "military_recon", "sat_type": "High-Res Imaging", "wiki": "https://en.wikipedia.org/wiki/Gaofen"}),
|
||||
("JILIN", {"country": "China", "mission": "commercial_imaging", "sat_type": "Video / Imaging", "wiki": "https://en.wikipedia.org/wiki/Jilin-1"}),
|
||||
("SHIJIAN", {"country": "China", "mission": "sigint", "sat_type": "ELINT / Tech Demo", "wiki": "https://en.wikipedia.org/wiki/Shijian"}),
|
||||
("TONGXIN JISHU SHIYAN", {"country": "China", "mission": "military_comms", "sat_type": "Military Comms Test", "wiki": "https://en.wikipedia.org/wiki/Tongxin_Jishu_Shiyan"}),
|
||||
("BEIDOU", {"country": "China", "mission": "navigation", "sat_type": "BeiDou", "wiki": "https://en.wikipedia.org/wiki/BeiDou"}),
|
||||
("TIANGONG", {"country": "China", "mission": "space_station", "sat_type": "Space Station", "wiki": "https://en.wikipedia.org/wiki/Tiangong_space_station"}),
|
||||
# ── Allied Military / Intel ─────────────────────────────────────────────
|
||||
("OFEK", {"country": "Israel", "mission": "military_recon", "sat_type": "Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Ofeq"}),
|
||||
("EROS", {"country": "Israel", "mission": "commercial_imaging", "sat_type": "High-Res Imaging", "wiki": "https://en.wikipedia.org/wiki/EROS_(satellite)"}),
|
||||
("CSO", {"country": "France", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/CSO_(satellite)"}),
|
||||
("HELIOS", {"country": "France", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Helios_(satellite)"}),
|
||||
("CERES", {"country": "France", "mission": "sigint", "sat_type": "ELINT Constellation", "wiki": "https://en.wikipedia.org/wiki/CERES_(satellite)"}),
|
||||
("IGS", {"country": "Japan", "mission": "military_recon", "sat_type": "Intelligence Gathering", "wiki": "https://en.wikipedia.org/wiki/Information_Gathering_Satellite"}),
|
||||
("KOMPSAT", {"country": "South Korea", "mission": "military_recon", "sat_type": "Multi-Purpose Satellite", "wiki": "https://en.wikipedia.org/wiki/KOMPSAT"}),
|
||||
("SAR-LUPE", {"country": "Germany", "mission": "military_sar", "sat_type": "SAR Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/SAR-Lupe"}),
|
||||
("SARAH", {"country": "Germany", "mission": "military_sar", "sat_type": "SAR Successor (SARah)", "wiki": "https://en.wikipedia.org/wiki/SARah"}),
|
||||
# ── Commercial SAR ──────────────────────────────────────────────────────
|
||||
("CAPELLA", {"country": "USA", "mission": "sar", "sat_type": "SAR Imaging", "wiki": "https://en.wikipedia.org/wiki/Capella_Space"}),
|
||||
("ICEYE", {"country": "Finland", "mission": "sar", "sat_type": "SAR Microsatellite", "wiki": "https://en.wikipedia.org/wiki/ICEYE"}),
|
||||
("COSMO-SKYMED", {"country": "Italy", "mission": "sar", "sat_type": "SAR Constellation", "wiki": "https://en.wikipedia.org/wiki/COSMO-SkyMed"}),
|
||||
("TANDEM", {"country": "Germany", "mission": "sar", "sat_type": "SAR Interferometry", "wiki": "https://en.wikipedia.org/wiki/TanDEM-X"}),
|
||||
("PAZ", {"country": "Spain", "mission": "sar", "sat_type": "SAR Imaging", "wiki": "https://en.wikipedia.org/wiki/PAZ_(satellite)"}),
|
||||
("UMBRA", {"country": "USA", "mission": "sar", "sat_type": "SAR Microsatellite", "wiki": "https://en.wikipedia.org/wiki/Umbra_(company)"}),
|
||||
# ── Commercial Optical Imaging ──────────────────────────────────────────
|
||||
("WORLDVIEW", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Maxar High-Res", "wiki": "https://en.wikipedia.org/wiki/WorldView-3"}),
|
||||
("GEOEYE", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Maxar Imaging", "wiki": "https://en.wikipedia.org/wiki/GeoEye-1"}),
|
||||
("LEGION", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Maxar Legion", "wiki": "https://en.wikipedia.org/wiki/WorldView_Legion"}),
|
||||
("PLEIADES", {"country": "France", "mission": "commercial_imaging", "sat_type": "Airbus Imaging", "wiki": "https://en.wikipedia.org/wiki/Pl%C3%A9iades_(satellite)"}),
|
||||
("SPOT", {"country": "France", "mission": "commercial_imaging", "sat_type": "Airbus Medium-Res", "wiki": "https://en.wikipedia.org/wiki/SPOT_(satellite)"}),
|
||||
("SKYSAT", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Planet Video", "wiki": "https://en.wikipedia.org/wiki/SkySat"}),
|
||||
("BLACKSKY", {"country": "USA", "mission": "commercial_imaging", "sat_type": "BlackSky Imaging", "wiki": "https://en.wikipedia.org/wiki/BlackSky"}),
|
||||
# ── Starlink (separate category) ────────────────────────────────────────
|
||||
("STARLINK", {"country": "USA", "mission": "starlink", "sat_type": "Starlink Mega-Constellation", "wiki": "https://en.wikipedia.org/wiki/Starlink"}),
|
||||
# ── Other Constellations ────────────────────────────────────────────────
|
||||
("ONEWEB", {"country": "UK", "mission": "constellation", "sat_type": "OneWeb LEO Broadband", "wiki": "https://en.wikipedia.org/wiki/OneWeb"}),
|
||||
("GALILEO", {"country": "EU", "mission": "navigation", "sat_type": "Galileo", "wiki": "https://en.wikipedia.org/wiki/Galileo_(satellite_navigation)"}),
|
||||
# ── Space Stations ──────────────────────────────────────────────────────
|
||||
("ISS", {"country": "Intl", "mission": "space_station", "sat_type": "Space Station", "wiki": "https://en.wikipedia.org/wiki/International_Space_Station"}),
|
||||
# ── Generic fallback patterns (last resort) ─────────────────────────────
|
||||
("PLANET", {"country": "USA", "mission": "commercial_imaging", "sat_type": "PlanetScope", "wiki": "https://en.wikipedia.org/wiki/Planet_Labs"}),
|
||||
]
|
||||
|
||||
# CelesTrak SATCAT owner codes → country mapping for satellites not matched by name.
|
||||
# Used as a secondary classifier alongside name-pattern matching.
|
||||
_OWNER_CODE_MAP = {
|
||||
"US": "USA", "CIS": "Russia", "PRC": "China", "ISS": "Intl",
|
||||
"FR": "France", "UK": "UK", "GER": "Germany", "JPN": "Japan",
|
||||
"IND": "India", "ISRA": "Israel", "IT": "Italy", "KOR": "South Korea",
|
||||
"ESA": "EU", "NATO": "NATO", "TURK": "Turkey", "UAE": "UAE",
|
||||
"AUS": "Australia", "CA": "Canada", "SPN": "Spain", "FIN": "Finland",
|
||||
"BRAZ": "Brazil", "IRAN": "Iran", "NKOR": "North Korea",
|
||||
}
|
||||
|
||||
# ── Maneuver detection thresholds (per Lemmens & Krag 2014, Kim et al. 2021) ─
|
||||
# These are above TLE fitting noise but low enough to catch real maneuvers.
|
||||
_MANEUVER_THRESHOLDS = {
|
||||
"period_min": 0.1, # minutes — above TLE noise (~0.01–0.05 min)
|
||||
"inclination_deg": 0.05, # degrees — above J2 secular drift (~0.001°/day)
|
||||
"eccentricity": 0.005, # above TLE fitting noise (~0.0001–0.001)
|
||||
"raan_residual_deg": 0.5, # degrees — only after J2 correction (Vallado §9.4)
|
||||
}
|
||||
|
||||
# ── Decay anomaly threshold ─────────────────────────────────────────────────
|
||||
# Flag if mean motion change rate exceeds this (rev/day per day).
|
||||
# Normal drag-induced decay is ~0.001 rev/day/day for LEO.
|
||||
_DECAY_MM_RATE_THRESHOLD = 0.01 # rev/day per day
|
||||
|
||||
|
||||
def _j2_raan_rate(inclination_deg, mean_motion_revday):
|
||||
"""Expected RAAN precession rate due to J2 (Vallado §9.4).
|
||||
|
||||
Returns degrees/day. Negative for prograde orbits.
|
||||
"""
|
||||
J2 = 1.08263e-3
|
||||
Re = 6378.137 # km
|
||||
mu = 398600.4418 # km^3/s^2
|
||||
n_rad_s = mean_motion_revday * 2 * math.pi / 86400.0
|
||||
if n_rad_s <= 0:
|
||||
return 0.0
|
||||
a = (mu / (n_rad_s ** 2)) ** (1.0 / 3.0) # semi-major axis in km
|
||||
if a <= Re:
|
||||
return 0.0
|
||||
cos_i = math.cos(math.radians(inclination_deg))
|
||||
raan_rate = -1.5 * n_rad_s * J2 * (Re / a) ** 2 * cos_i
|
||||
return math.degrees(raan_rate) * 86400.0 / (2 * math.pi) # deg/day
|
||||
|
||||
|
||||
def detect_maneuvers(current_gp_data):
|
||||
"""Compare current TLEs against stored history to detect orbital maneuvers.
|
||||
|
||||
Returns list of maneuver alert dicts. Only runs when _tle_history is populated
|
||||
(i.e., after the second CelesTrak fetch or from persisted history).
|
||||
|
||||
Thresholds from Lemmens & Krag (2014), Kim et al. (2021).
|
||||
"""
|
||||
if not _tle_history:
|
||||
return []
|
||||
|
||||
alerts = []
|
||||
for sat in current_gp_data:
|
||||
norad_id = sat.get("NORAD_CAT_ID")
|
||||
if norad_id is None:
|
||||
continue
|
||||
norad_id = int(norad_id)
|
||||
prev = _tle_history.get(norad_id)
|
||||
if prev is None:
|
||||
continue
|
||||
|
||||
cur_mm = sat.get("MEAN_MOTION")
|
||||
cur_inc = sat.get("INCLINATION")
|
||||
cur_ecc = sat.get("ECCENTRICITY")
|
||||
cur_raan = sat.get("RA_OF_ASC_NODE")
|
||||
prev_mm = prev.get("MEAN_MOTION")
|
||||
prev_inc = prev.get("INCLINATION")
|
||||
prev_ecc = prev.get("ECCENTRICITY")
|
||||
prev_raan = prev.get("RA_OF_ASC_NODE")
|
||||
|
||||
if any(v is None for v in (cur_mm, cur_inc, cur_ecc, cur_raan,
|
||||
prev_mm, prev_inc, prev_ecc, prev_raan)):
|
||||
continue
|
||||
|
||||
# Convert mean motion (rev/day) to period (minutes)
|
||||
cur_period = 1440.0 / cur_mm if cur_mm > 0 else 0
|
||||
prev_period = 1440.0 / prev_mm if prev_mm > 0 else 0
|
||||
|
||||
reasons = []
|
||||
t = _MANEUVER_THRESHOLDS
|
||||
|
||||
delta_period = abs(cur_period - prev_period)
|
||||
if delta_period > t["period_min"]:
|
||||
reasons.append(f"period Δ{delta_period:+.3f} min")
|
||||
|
||||
delta_inc = abs(cur_inc - prev_inc)
|
||||
if delta_inc > t["inclination_deg"]:
|
||||
reasons.append(f"inclination Δ{delta_inc:+.4f}°")
|
||||
|
||||
delta_ecc = abs(cur_ecc - prev_ecc)
|
||||
if delta_ecc > t["eccentricity"]:
|
||||
reasons.append(f"eccentricity Δ{delta_ecc:+.6f}")
|
||||
|
||||
# RAAN with J2 correction — only flag residual beyond expected precession
|
||||
epoch_str = sat.get("EPOCH", "")
|
||||
try:
|
||||
epoch_dt = datetime.strptime(epoch_str[:19], "%Y-%m-%dT%H:%M:%S")
|
||||
epoch_ts = epoch_dt.timestamp()
|
||||
except (ValueError, TypeError):
|
||||
epoch_ts = 0
|
||||
prev_epoch_ts = prev.get("epoch_ts", 0)
|
||||
dt_days = (epoch_ts - prev_epoch_ts) / 86400.0 if (epoch_ts and prev_epoch_ts) else 1.0
|
||||
if dt_days > 0:
|
||||
expected_raan_drift = _j2_raan_rate(cur_inc, cur_mm) * dt_days
|
||||
actual_raan_change = cur_raan - prev_raan
|
||||
# Normalize to [-180, 180]
|
||||
actual_raan_change = (actual_raan_change + 180) % 360 - 180
|
||||
raan_residual = abs(actual_raan_change - expected_raan_drift)
|
||||
if raan_residual > t["raan_residual_deg"]:
|
||||
reasons.append(f"RAAN residual {raan_residual:.3f}° (J2-corrected)")
|
||||
|
||||
if reasons:
|
||||
alerts.append({
|
||||
"norad_id": norad_id,
|
||||
"name": sat.get("OBJECT_NAME", "UNKNOWN"),
|
||||
"type": "maneuver",
|
||||
"reasons": reasons,
|
||||
"epoch": sat.get("EPOCH", ""),
|
||||
"delta_period_min": round(delta_period, 4),
|
||||
"delta_inclination_deg": round(delta_inc, 5),
|
||||
"delta_eccentricity": round(delta_ecc, 7),
|
||||
})
|
||||
|
||||
logger.info(f"Satellites: Maneuver scan — {len(alerts)} detections from {len(current_gp_data)} objects")
|
||||
return alerts
|
||||
|
||||
|
||||
def detect_decay_anomalies(current_gp_data):
|
||||
"""Flag satellites with abnormal mean-motion change rates (possible decay).
|
||||
|
||||
A rapidly increasing mean motion indicates orbital decay — the satellite
|
||||
is losing altitude. Normal LEO drag is ~0.001 rev/day/day.
|
||||
"""
|
||||
if not _tle_history:
|
||||
return []
|
||||
|
||||
alerts = []
|
||||
for sat in current_gp_data:
|
||||
norad_id = sat.get("NORAD_CAT_ID")
|
||||
if norad_id is None:
|
||||
continue
|
||||
norad_id = int(norad_id)
|
||||
prev = _tle_history.get(norad_id)
|
||||
if prev is None:
|
||||
continue
|
||||
|
||||
cur_mm = sat.get("MEAN_MOTION")
|
||||
prev_mm = prev.get("MEAN_MOTION")
|
||||
if cur_mm is None or prev_mm is None:
|
||||
continue
|
||||
|
||||
epoch_str = sat.get("EPOCH", "")
|
||||
try:
|
||||
epoch_dt = datetime.strptime(epoch_str[:19], "%Y-%m-%dT%H:%M:%S")
|
||||
epoch_ts = epoch_dt.timestamp()
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
prev_epoch_ts = prev.get("epoch_ts", 0)
|
||||
dt_days = (epoch_ts - prev_epoch_ts) / 86400.0 if (epoch_ts and prev_epoch_ts) else 0
|
||||
if dt_days < 0.5:
|
||||
continue # Need at least 12h between TLEs for meaningful comparison
|
||||
|
||||
mm_rate = (cur_mm - prev_mm) / dt_days # rev/day per day
|
||||
if abs(mm_rate) > _DECAY_MM_RATE_THRESHOLD:
|
||||
cur_alt_km = (8681663.7 / (cur_mm ** (2.0 / 3.0))) - 6371.0 if cur_mm > 0 else 0
|
||||
alerts.append({
|
||||
"norad_id": norad_id,
|
||||
"name": sat.get("OBJECT_NAME", "UNKNOWN"),
|
||||
"type": "decay_anomaly",
|
||||
"mm_rate": round(mm_rate, 6),
|
||||
"current_mm": round(cur_mm, 4),
|
||||
"approx_alt_km": round(cur_alt_km, 1),
|
||||
"epoch": sat.get("EPOCH", ""),
|
||||
"dt_days": round(dt_days, 2),
|
||||
})
|
||||
|
||||
logger.info(f"Satellites: Decay scan — {len(alerts)} anomalies detected")
|
||||
return alerts
|
||||
|
||||
|
||||
def compute_overflights(gp_data, bbox, hours=24, step_minutes=10):
|
||||
"""Count unique satellites whose ground track enters a bounding box.
|
||||
|
||||
Args:
|
||||
gp_data: Full GP catalog (list of dicts with orbital elements).
|
||||
bbox: Dict with keys 's', 'w', 'n', 'e' (degrees).
|
||||
hours: Look-back window (default 24h).
|
||||
step_minutes: Sampling interval (default 10 min).
|
||||
|
||||
Returns dict with total count and per-mission breakdown.
|
||||
Uses SGP4 propagation — CPU cost is ~O(catalog_size × timesteps).
|
||||
Only propagates satellites that could plausibly overfly the bbox latitude range.
|
||||
"""
|
||||
if not gp_data or not bbox:
|
||||
return {"total": 0, "by_mission": {}, "satellites": []}
|
||||
|
||||
south, west = bbox["s"], bbox["w"]
|
||||
north, east = bbox["n"], bbox["e"]
|
||||
now = datetime.utcnow()
|
||||
steps = int(hours * 60 / step_minutes)
|
||||
|
||||
# Pre-filter: only propagate sats whose inclination allows them to reach bbox latitude
|
||||
max_lat = max(abs(south), abs(north))
|
||||
candidates = [s for s in gp_data if s.get("INCLINATION") is not None
|
||||
and s.get("INCLINATION") >= max_lat * 0.8] # 20% margin
|
||||
|
||||
seen_ids = set()
|
||||
results = []
|
||||
by_mission = {}
|
||||
|
||||
for s in candidates:
|
||||
norad_id = s.get("NORAD_CAT_ID")
|
||||
mean_motion = s.get("MEAN_MOTION")
|
||||
ecc = s.get("ECCENTRICITY")
|
||||
incl = s.get("INCLINATION")
|
||||
raan = s.get("RA_OF_ASC_NODE")
|
||||
argp = s.get("ARG_OF_PERICENTER")
|
||||
ma = s.get("MEAN_ANOMALY")
|
||||
bstar = s.get("BSTAR", 0)
|
||||
epoch_str = s.get("EPOCH", "")
|
||||
|
||||
if any(v is None for v in (mean_motion, ecc, incl, raan, argp, ma, epoch_str)):
|
||||
continue
|
||||
|
||||
try:
|
||||
epoch_dt = datetime.strptime(epoch_str[:19], "%Y-%m-%dT%H:%M:%S")
|
||||
epoch_jd, epoch_fr = jday(
|
||||
epoch_dt.year, epoch_dt.month, epoch_dt.day,
|
||||
epoch_dt.hour, epoch_dt.minute, epoch_dt.second,
|
||||
)
|
||||
sat_obj = Satrec()
|
||||
sat_obj.sgp4init(
|
||||
WGS72, "i", norad_id or 0,
|
||||
(epoch_jd + epoch_fr) - 2433281.5,
|
||||
bstar, 0.0, 0.0, ecc,
|
||||
math.radians(argp), math.radians(incl), math.radians(ma),
|
||||
mean_motion * 2 * math.pi / 1440.0, math.radians(raan),
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
for step in range(steps):
|
||||
t = now - timedelta(minutes=step * step_minutes)
|
||||
jd_t, fr_t = jday(t.year, t.month, t.day, t.hour, t.minute, t.second)
|
||||
e, r, _ = sat_obj.sgp4(jd_t, fr_t)
|
||||
if e != 0:
|
||||
continue
|
||||
x, y, z = r
|
||||
gmst = _gmst(jd_t + fr_t)
|
||||
lng_rad = math.atan2(y, x) - gmst
|
||||
lat_deg = math.degrees(math.atan2(z, math.sqrt(x * x + y * y)))
|
||||
lng_deg = math.degrees(lng_rad) % 360
|
||||
if lng_deg > 180:
|
||||
lng_deg -= 360
|
||||
|
||||
# Check bounding box (handles antimeridian crossing)
|
||||
lat_in = south <= lat_deg <= north
|
||||
if west <= east:
|
||||
lng_in = west <= lng_deg <= east
|
||||
else:
|
||||
lng_in = lng_deg >= west or lng_deg <= east
|
||||
|
||||
if lat_in and lng_in and norad_id not in seen_ids:
|
||||
seen_ids.add(norad_id)
|
||||
name = s.get("OBJECT_NAME", "UNKNOWN")
|
||||
# Classify for mission breakdown
|
||||
mission = "unknown"
|
||||
for key, meta in _SAT_INTEL_DB:
|
||||
if key.upper() in name.upper():
|
||||
mission = meta.get("mission", "unknown")
|
||||
break
|
||||
by_mission[mission] = by_mission.get(mission, 0) + 1
|
||||
results.append({"norad_id": norad_id, "name": name, "mission": mission})
|
||||
break # Already counted this sat, move to next
|
||||
|
||||
return {"total": len(results), "by_mission": by_mission, "satellites": results}
|
||||
|
||||
|
||||
def _parse_tle_to_gp(name, norad_id, line1, line2):
|
||||
"""Convert TLE two-line element to CelesTrak GP-style dict."""
|
||||
@@ -539,9 +613,18 @@ def fetch_satellites():
|
||||
if not is_any_active("satellites"):
|
||||
return
|
||||
sats = []
|
||||
maneuver_alerts = []
|
||||
decay_alerts = []
|
||||
starlink_summary = {}
|
||||
data = None
|
||||
classified = None
|
||||
try:
|
||||
now_ts = time.time()
|
||||
|
||||
# On first call, load TLE history from disk for maneuver detection
|
||||
if not _tle_history:
|
||||
_load_tle_history()
|
||||
|
||||
# On first call, try disk cache before hitting CelesTrak
|
||||
if _sat_gp_cache["data"] is None:
|
||||
disk_data = _load_sat_cache()
|
||||
@@ -594,6 +677,9 @@ def fetch_satellites():
|
||||
if lm:
|
||||
_sat_gp_cache["last_modified"] = lm
|
||||
_save_sat_cache(gp_data)
|
||||
# Snapshot current TLEs as history before overwriting
|
||||
# (the old _tle_history becomes the comparison baseline)
|
||||
_snapshot_current_tles(gp_data)
|
||||
logger.info(
|
||||
f"Satellites: Downloaded {len(gp_data)} GP records from CelesTrak"
|
||||
)
|
||||
@@ -651,11 +737,14 @@ def fetch_satellites():
|
||||
and _sat_classified_cache["data"]
|
||||
):
|
||||
classified = _sat_classified_cache["data"]
|
||||
starlink_summary = _sat_classified_cache.get("starlink_summary", {})
|
||||
logger.info(
|
||||
f"Satellites: Using cached classification ({len(classified)} sats, TLEs unchanged)"
|
||||
)
|
||||
else:
|
||||
classified = []
|
||||
starlink_count = 0
|
||||
starlink_shells = {} # inclination shell → count
|
||||
for sat in data:
|
||||
name = sat.get("OBJECT_NAME", "UNKNOWN").upper()
|
||||
intel = None
|
||||
@@ -663,8 +752,24 @@ def fetch_satellites():
|
||||
if key.upper() in name:
|
||||
intel = dict(meta)
|
||||
break
|
||||
if not intel:
|
||||
# Secondary classification via SATCAT owner code
|
||||
owner = sat.get("OWNER", sat.get("OBJECT_OWNER", ""))
|
||||
if owner in _OWNER_CODE_MAP:
|
||||
intel = {"country": _OWNER_CODE_MAP[owner], "mission": "general", "sat_type": "Unclassified"}
|
||||
if not intel:
|
||||
continue
|
||||
|
||||
# Starlink: count and summarize but don't propagate individually
|
||||
# (6000+ sats would be too expensive to position every 60s)
|
||||
if intel.get("mission") == "starlink":
|
||||
starlink_count += 1
|
||||
inc = sat.get("INCLINATION")
|
||||
if inc is not None:
|
||||
shell_key = f"{round(inc, 0):.0f}°"
|
||||
starlink_shells[shell_key] = starlink_shells.get(shell_key, 0) + 1
|
||||
continue # Skip individual propagation
|
||||
|
||||
entry = {
|
||||
"id": sat.get("NORAD_CAT_ID"),
|
||||
"name": sat.get("OBJECT_NAME", "UNKNOWN"),
|
||||
@@ -679,14 +784,35 @@ def fetch_satellites():
|
||||
}
|
||||
entry.update(intel)
|
||||
classified.append(entry)
|
||||
|
||||
starlink_summary = {
|
||||
"total": starlink_count,
|
||||
"shells": starlink_shells,
|
||||
}
|
||||
_sat_classified_cache["data"] = classified
|
||||
_sat_classified_cache["starlink_summary"] = starlink_summary
|
||||
_sat_classified_cache["gp_fetch_ts"] = _sat_gp_cache["last_fetch"]
|
||||
logger.info(
|
||||
f"Satellites: {len(classified)} intel-classified out of {len(data)} total in catalog"
|
||||
f"Satellites: {len(classified)} intel-classified, "
|
||||
f"{starlink_count} Starlink (summarized), "
|
||||
f"out of {len(data)} total in catalog"
|
||||
)
|
||||
|
||||
all_sats = classified
|
||||
|
||||
# ── Run analysis detectors against the full GP catalog ──────────────
|
||||
# These use cached TLEs only — no extra network requests.
|
||||
maneuver_alerts = []
|
||||
decay_alerts = []
|
||||
try:
|
||||
maneuver_alerts = detect_maneuvers(data)
|
||||
except (ValueError, TypeError, KeyError, ZeroDivisionError) as e:
|
||||
logger.error(f"Satellites: Maneuver detection error: {e}")
|
||||
try:
|
||||
decay_alerts = detect_decay_anomalies(data)
|
||||
except (ValueError, TypeError, KeyError, ZeroDivisionError) as e:
|
||||
logger.error(f"Satellites: Decay detection error: {e}")
|
||||
|
||||
now = datetime.utcnow()
|
||||
jd, fr = jday(
|
||||
now.year, now.month, now.day, now.hour, now.minute, now.second + now.microsecond / 1e6
|
||||
@@ -800,6 +926,13 @@ def fetch_satellites():
|
||||
with _data_lock:
|
||||
latest_data["satellites"] = sats
|
||||
latest_data["satellite_source"] = _sat_gp_cache.get("source", "none")
|
||||
latest_data["satellite_analysis"] = {
|
||||
"maneuvers": maneuver_alerts,
|
||||
"decay_anomalies": decay_alerts,
|
||||
"starlink": starlink_summary,
|
||||
"catalog_size": len(data) if data else 0,
|
||||
"classified_count": len(classified) if classified else 0,
|
||||
}
|
||||
_mark_fresh("satellites")
|
||||
else:
|
||||
with _data_lock:
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
"""WastewaterSCAN fetcher — pathogen surveillance via wastewater monitoring.
|
||||
|
||||
Data source: Stanford/Emory WastewaterSCAN project
|
||||
- Plant locations: https://storage.googleapis.com/wastewater-dev-data/json/plants.json
|
||||
- Time series: https://storage.googleapis.com/wastewater-dev-data/json/{uuid}.json
|
||||
|
||||
All data is public, no authentication required. ~192 treatment plants across
|
||||
the US with daily sampling for COVID (N Gene), Influenza A/B, RSV, Norovirus,
|
||||
MPXV, Measles, H5N1, and others.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
import concurrent.futures
|
||||
from datetime import datetime, timedelta
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
from services.fetchers.retry import with_retry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_GCS_BASE = "https://storage.googleapis.com/wastewater-dev-data/json"
|
||||
|
||||
# Cache the plants list for 24 hours (it rarely changes)
|
||||
_plants_cache: list[dict] = []
|
||||
_plants_cache_ts: float = 0
|
||||
_PLANTS_CACHE_TTL = 86400 # 24 hours
|
||||
|
||||
# Key pathogen targets to extract — maps internal target name to display label
|
||||
_TARGET_DISPLAY: dict[str, str] = {
|
||||
"N Gene": "COVID-19",
|
||||
"Influenza A F1R1": "Influenza A",
|
||||
"Influenza B": "Influenza B",
|
||||
"RSV": "RSV",
|
||||
"Noro_G2": "Norovirus",
|
||||
"MPXV_G2R_WA": "Mpox",
|
||||
"InfA_H5": "H5N1 (Bird Flu)",
|
||||
"HMPV_4": "HMPV",
|
||||
"Rota": "Rotavirus",
|
||||
"HAV": "Hepatitis A",
|
||||
"C_auris": "Candida auris",
|
||||
"EVD68": "Enterovirus D68",
|
||||
}
|
||||
|
||||
# Activity categories that represent elevated/alert levels
|
||||
_ALERT_CATEGORIES = {"high", "very high", "above normal"}
|
||||
|
||||
|
||||
def _fetch_plants() -> list[dict]:
|
||||
"""Fetch the full plants list from GCS, with 24h caching."""
|
||||
global _plants_cache, _plants_cache_ts
|
||||
|
||||
if _plants_cache and (time.time() - _plants_cache_ts) < _PLANTS_CACHE_TTL:
|
||||
return _plants_cache
|
||||
|
||||
url = f"{_GCS_BASE}/plants.json"
|
||||
resp = fetch_with_curl(url, timeout=30)
|
||||
if resp.status_code != 200:
|
||||
logger.warning(f"WastewaterSCAN plants fetch failed: HTTP {resp.status_code}")
|
||||
return _plants_cache # return stale cache on failure
|
||||
|
||||
data = resp.json()
|
||||
plants = data.get("plants", [])
|
||||
_plants_cache = plants
|
||||
_plants_cache_ts = time.time()
|
||||
logger.info(f"WastewaterSCAN: cached {len(plants)} plant locations")
|
||||
return plants
|
||||
|
||||
|
||||
def _fetch_plant_latest(plant_id: str) -> dict | None:
|
||||
"""Fetch the most recent sample for a single plant.
|
||||
|
||||
Returns a dict with pathogen levels or None on failure.
|
||||
"""
|
||||
url = f"{_GCS_BASE}/{plant_id}.json"
|
||||
try:
|
||||
resp = fetch_with_curl(url, timeout=12)
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
data = resp.json()
|
||||
samples = data.get("samples", [])
|
||||
if not samples:
|
||||
return None
|
||||
|
||||
# Find the most recent sample (last element, sorted by date)
|
||||
latest = samples[-1]
|
||||
collection_date = latest.get("collection_date", "")
|
||||
|
||||
# Skip samples older than 30 days
|
||||
try:
|
||||
sample_dt = datetime.strptime(collection_date, "%Y-%m-%d")
|
||||
if sample_dt < datetime.utcnow() - timedelta(days=30):
|
||||
return None
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Extract key pathogen levels
|
||||
targets = latest.get("targets", {})
|
||||
pathogens: list[dict] = []
|
||||
alert_count = 0
|
||||
|
||||
for target_key, display_name in _TARGET_DISPLAY.items():
|
||||
target_data = targets.get(target_key)
|
||||
if not target_data:
|
||||
continue
|
||||
|
||||
concentration = target_data.get("gc_g_dry_weight", 0) or 0
|
||||
activity = target_data.get("activity_category", "not calculated")
|
||||
normalized = target_data.get("gc_g_dry_weight_pmmov", 0) or 0
|
||||
|
||||
if concentration <= 0 and normalized <= 0:
|
||||
continue # no detection
|
||||
|
||||
is_alert = activity.lower() in _ALERT_CATEGORIES
|
||||
if is_alert:
|
||||
alert_count += 1
|
||||
|
||||
pathogens.append({
|
||||
"name": display_name,
|
||||
"target_key": target_key,
|
||||
"concentration": round(concentration, 1),
|
||||
"normalized": round(normalized, 6),
|
||||
"activity": activity,
|
||||
"alert": is_alert,
|
||||
})
|
||||
|
||||
if not pathogens:
|
||||
return None
|
||||
|
||||
return {
|
||||
"collection_date": collection_date,
|
||||
"pathogens": pathogens,
|
||||
"alert_count": alert_count,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.debug(f"WastewaterSCAN: failed to fetch plant {plant_id}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
@with_retry(max_retries=1, base_delay=5)
|
||||
def fetch_wastewater():
|
||||
"""Fetch WastewaterSCAN plant locations and latest pathogen levels.
|
||||
|
||||
1. Fetches the plant list (cached 24h) for locations.
|
||||
2. Concurrently fetches time series for all plants, extracting only
|
||||
the most recent sample's pathogen data.
|
||||
3. Merges into a flat list suitable for map rendering.
|
||||
"""
|
||||
from services.fetchers._store import is_any_active
|
||||
|
||||
if not is_any_active("wastewater"):
|
||||
return
|
||||
|
||||
plants = _fetch_plants()
|
||||
if not plants:
|
||||
logger.warning("WastewaterSCAN: no plant data available")
|
||||
return
|
||||
|
||||
# Build base records from plant metadata
|
||||
plant_map: dict[str, dict] = {}
|
||||
for p in plants:
|
||||
point = p.get("point") or {}
|
||||
coords = point.get("coordinates") or []
|
||||
if len(coords) < 2:
|
||||
continue
|
||||
|
||||
pid = p.get("id") or p.get("uuid", "")
|
||||
if not pid:
|
||||
continue
|
||||
|
||||
plant_map[pid] = {
|
||||
"id": pid,
|
||||
"name": p.get("name", ""),
|
||||
"site_name": p.get("site_name", ""),
|
||||
"city": p.get("city", ""),
|
||||
"state": p.get("state", ""),
|
||||
"country": p.get("country", "US"),
|
||||
"population": p.get("sewershed_pop"),
|
||||
"lat": coords[1],
|
||||
"lng": coords[0],
|
||||
"pathogens": [],
|
||||
"alert_count": 0,
|
||||
"collection_date": "",
|
||||
"source": "WastewaterSCAN",
|
||||
}
|
||||
|
||||
# Fetch latest samples concurrently (up to 12 threads)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
|
||||
futures = {
|
||||
pool.submit(_fetch_plant_latest, pid): pid
|
||||
for pid in plant_map
|
||||
}
|
||||
for fut in concurrent.futures.as_completed(futures, timeout=120):
|
||||
pid = futures[fut]
|
||||
try:
|
||||
result = fut.result()
|
||||
if result:
|
||||
plant_map[pid]["pathogens"] = result["pathogens"]
|
||||
plant_map[pid]["alert_count"] = result["alert_count"]
|
||||
plant_map[pid]["collection_date"] = result["collection_date"]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
nodes = list(plant_map.values())
|
||||
active_nodes = [n for n in nodes if n["pathogens"]]
|
||||
|
||||
logger.info(
|
||||
f"WastewaterSCAN: {len(nodes)} plants, "
|
||||
f"{len(active_nodes)} with recent pathogen data, "
|
||||
f"{sum(n['alert_count'] for n in nodes)} total alerts"
|
||||
)
|
||||
|
||||
with _data_lock:
|
||||
latest_data["wastewater"] = nodes
|
||||
if nodes:
|
||||
_mark_fresh("wastewater")
|
||||
+57
-31
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import threading
|
||||
from typing import Any, Dict, List
|
||||
@@ -81,43 +82,63 @@ def _load_local_search_cache() -> List[Dict[str, Any]]:
|
||||
|
||||
|
||||
def _search_local_fallback(query: str, limit: int) -> List[Dict[str, Any]]:
|
||||
"""Strict local lookup used only when ``local_only=True`` is set.
|
||||
|
||||
Historical behaviour (substring-token-in-haystack matching) produced
|
||||
catastrophically wrong results: any query containing a common word
|
||||
would match the first airport with that word anywhere in its name,
|
||||
which silently poisoned every cache downstream. Fixed to require
|
||||
whole-word matches against airport name/IATA/id and cached-geocode
|
||||
labels.
|
||||
"""
|
||||
q = query.strip().lower()
|
||||
if not q:
|
||||
return []
|
||||
q_tokens = set(re.findall(r"[a-z0-9]+", q))
|
||||
if not q_tokens:
|
||||
return []
|
||||
|
||||
matches: List[Dict[str, Any]] = []
|
||||
seen: set[tuple[float, float, str]] = set()
|
||||
|
||||
def _whole_word_tokens(text: str) -> set[str]:
|
||||
return set(re.findall(r"[a-z0-9]+", (text or "").lower()))
|
||||
|
||||
for item in cached_airports:
|
||||
haystacks = [
|
||||
str(item.get("name", "")).lower(),
|
||||
str(item.get("iata", "")).lower(),
|
||||
str(item.get("id", "")).lower(),
|
||||
]
|
||||
if any(q in h for h in haystacks):
|
||||
label = f'{item.get("name", "Airport")} ({item.get("iata", "")})'
|
||||
key = (float(item["lat"]), float(item["lng"]), label)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
matches.append(
|
||||
{
|
||||
"label": label,
|
||||
"lat": float(item["lat"]),
|
||||
"lng": float(item["lng"]),
|
||||
}
|
||||
)
|
||||
if len(matches) >= limit:
|
||||
return matches
|
||||
name_tokens = _whole_word_tokens(item.get("name", ""))
|
||||
iata = str(item.get("iata", "")).lower().strip()
|
||||
icao = str(item.get("id", "")).lower().strip()
|
||||
# IATA/ICAO must match exactly; name must share ALL query tokens
|
||||
# with the airport name (not "any token in haystack").
|
||||
exact_code = bool(iata and iata in q_tokens) or bool(icao and icao in q_tokens)
|
||||
name_match = bool(q_tokens) and q_tokens.issubset(name_tokens)
|
||||
if not (exact_code or name_match):
|
||||
continue
|
||||
label = f'{item.get("name", "Airport")} ({item.get("iata", "")})'
|
||||
key = (float(item["lat"]), float(item["lng"]), label)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
matches.append(
|
||||
{
|
||||
"label": label,
|
||||
"lat": float(item["lat"]),
|
||||
"lng": float(item["lng"]),
|
||||
}
|
||||
)
|
||||
if len(matches) >= limit:
|
||||
return matches
|
||||
|
||||
for item in _load_local_search_cache():
|
||||
label = str(item.get("label", ""))
|
||||
if q in label.lower():
|
||||
key = (float(item["lat"]), float(item["lng"]), label)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
matches.append(item)
|
||||
if len(matches) >= limit:
|
||||
break
|
||||
label_tokens = _whole_word_tokens(label)
|
||||
if not q_tokens.issubset(label_tokens):
|
||||
continue
|
||||
key = (float(item["lat"]), float(item["lng"]), label)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
matches.append(item)
|
||||
if len(matches) >= limit:
|
||||
break
|
||||
|
||||
return matches
|
||||
|
||||
@@ -163,9 +184,14 @@ def search_geocode(query: str, limit: int = 5, local_only: bool = False) -> List
|
||||
timeout=6,
|
||||
)
|
||||
except Exception:
|
||||
results = _search_local_fallback(q, limit)
|
||||
_set_cache(key, results)
|
||||
return results
|
||||
# Intentionally no silent airport-name fallback. Callers that
|
||||
# want offline results should pass ``local_only=True``; anything
|
||||
# else means we return an empty list so the caller can decide
|
||||
# whether to retry or propagate the failure. The old behaviour
|
||||
# of falling through to _search_local_fallback silently poisoned
|
||||
# every downstream cache with airport coordinates for any query.
|
||||
_set_cache(key, [])
|
||||
return []
|
||||
|
||||
results: List[Dict[str, Any]] = []
|
||||
if res and res.status_code == 200:
|
||||
@@ -184,9 +210,9 @@ def search_geocode(query: str, limit: int = 5, local_only: bool = False) -> List
|
||||
continue
|
||||
except Exception:
|
||||
results = []
|
||||
if not results:
|
||||
results = _search_local_fallback(q, limit)
|
||||
|
||||
# No silent airport-name fallback on empty results either — same
|
||||
# reason as above. Empty means empty.
|
||||
_set_cache(key, results)
|
||||
return results
|
||||
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
"""Country-bbox post-filter for geocoded results.
|
||||
|
||||
Any fetcher that turns a country-tagged row into a lat/lng should call
|
||||
``coord_in_country()`` after the geocoder returns. If the coordinate
|
||||
falls outside the country's bounding box, the result is almost
|
||||
certainly a namesake collision (e.g. "Milan, WI" landing in Milan,
|
||||
Italy) and the caller should reject or retry with a stronger query.
|
||||
|
||||
This is a cheap sanity gate that catches geocoder mistakes no human
|
||||
operator will ever spot by eye across thousands of points.
|
||||
|
||||
Bounding boxes are deliberately generous — they include territories,
|
||||
overseas islands, and a small buffer — so that legitimate coastal or
|
||||
border cities are never false-rejected. Goal is to catch "wrong
|
||||
continent", not "off by a few km".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
# (min_lat, min_lng, max_lat, max_lng)
|
||||
_COUNTRY_BBOX: dict[str, Tuple[float, float, float, float]] = {
|
||||
# North America
|
||||
"USA": (18.0, -180.0, 72.0, -65.0), # inc. Alaska + Hawaii
|
||||
"Canada": (41.0, -142.0, 84.0, -52.0),
|
||||
"Mexico": (14.0, -120.0, 33.0, -86.0),
|
||||
# South & Central America
|
||||
"Brazil": (-35.0, -74.5, 6.0, -34.0),
|
||||
"Argentina": (-56.0, -74.0, -21.5, -53.0),
|
||||
"Chile": (-56.0, -76.0, -17.0, -66.0),
|
||||
"Colombia": (-5.0, -82.0, 13.5, -66.5),
|
||||
"Peru": (-19.0, -82.0, 0.5, -68.5),
|
||||
"Venezuela": (0.5, -73.5, 12.5, -59.5),
|
||||
"Ecuador": (-5.5, -92.5, 2.0, -75.0), # inc. Galápagos
|
||||
"Bolivia": (-23.0, -69.5, -9.5, -57.5),
|
||||
"Uruguay": (-35.0, -58.5, -30.0, -53.0),
|
||||
"Paraguay": (-28.0, -63.0, -19.0, -54.0),
|
||||
"Guatemala": (13.5, -92.5, 18.0, -88.0),
|
||||
"Honduras": (12.5, -89.5, 16.5, -83.0),
|
||||
"Nicaragua": (10.5, -88.0, 15.5, -83.0),
|
||||
"Costa Rica": (8.0, -86.0, 11.5, -82.5),
|
||||
"Panama": (7.0, -83.5, 9.7, -77.0),
|
||||
"El Salvador": (13.0, -90.5, 14.5, -87.5),
|
||||
"Cuba": (19.5, -85.0, 23.5, -74.0),
|
||||
"Dominican Republic": (17.5, -72.5, 20.0, -68.0),
|
||||
"Haiti": (17.5, -74.5, 20.5, -71.5),
|
||||
"Jamaica": (17.5, -78.5, 18.7, -76.0),
|
||||
"Puerto Rico": (17.5, -68.0, 19.0, -65.0),
|
||||
# Europe
|
||||
"United Kingdom": (49.0, -9.0, 61.0, 2.5),
|
||||
"Ireland": (51.0, -11.0, 56.0, -5.0),
|
||||
"France": (41.0, -5.5, 51.5, 9.8),
|
||||
"Germany": (47.0, 5.5, 56.0, 15.5),
|
||||
"Spain": (27.0, -18.5, 44.0, 4.5), # inc. Canary Islands
|
||||
"Portugal": (32.0, -32.0, 42.5, -6.0), # inc. Azores + Madeira
|
||||
"Italy": (36.0, 6.5, 47.5, 19.0),
|
||||
"Netherlands": (50.5, 3.0, 53.8, 7.3),
|
||||
"Belgium": (49.4, 2.5, 51.6, 6.5),
|
||||
"Switzerland": (45.7, 5.8, 48.0, 10.6),
|
||||
"Austria": (46.3, 9.5, 49.1, 17.2),
|
||||
"Poland": (49.0, 14.0, 55.0, 24.2),
|
||||
"Czech Republic": (48.5, 12.0, 51.2, 18.9),
|
||||
"Slovakia": (47.7, 16.8, 49.7, 22.6),
|
||||
"Hungary": (45.7, 16.1, 48.6, 22.9),
|
||||
"Romania": (43.6, 20.2, 48.3, 29.7),
|
||||
"Bulgaria": (41.2, 22.3, 44.3, 28.7),
|
||||
"Greece": (34.7, 19.3, 41.8, 29.7),
|
||||
"Turkey": (35.8, 25.6, 42.2, 44.8),
|
||||
"Ukraine": (44.3, 22.1, 52.4, 40.3),
|
||||
"Belarus": (51.2, 23.1, 56.2, 32.8),
|
||||
"Russia": (41.0, 19.0, 82.0, 180.0),
|
||||
"Sweden": (55.0, 10.5, 69.1, 24.2),
|
||||
"Norway": (57.9, 4.5, 71.2, 31.1),
|
||||
"Finland": (59.7, 20.5, 70.1, 31.6),
|
||||
"Denmark": (54.5, 8.0, 57.9, 15.3),
|
||||
"Iceland": (63.3, -24.6, 66.6, -13.4),
|
||||
"Serbia": (42.2, 18.8, 46.2, 23.0),
|
||||
"Croatia": (42.3, 13.4, 46.6, 19.5),
|
||||
"Slovenia": (45.4, 13.3, 46.9, 16.7),
|
||||
"Bosnia and Herzegovina": (42.5, 15.7, 45.3, 19.7),
|
||||
"North Macedonia": (40.8, 20.4, 42.4, 23.1),
|
||||
"Albania": (39.6, 19.2, 42.7, 21.1),
|
||||
"Kosovo": (41.8, 20.0, 43.3, 21.8),
|
||||
"Moldova": (45.4, 26.6, 48.5, 30.2),
|
||||
"Lithuania": (53.8, 20.9, 56.5, 26.9),
|
||||
"Latvia": (55.6, 20.9, 58.1, 28.3),
|
||||
"Estonia": (57.5, 21.7, 59.8, 28.3),
|
||||
"Luxembourg": (49.4, 5.7, 50.2, 6.6),
|
||||
"Malta": (35.7, 14.1, 36.1, 14.7),
|
||||
"Cyprus": (34.5, 32.2, 35.8, 34.7),
|
||||
# Middle East
|
||||
"Israel": (29.4, 34.2, 33.4, 35.9),
|
||||
"Lebanon": (33.0, 35.1, 34.7, 36.7),
|
||||
"Jordan": (29.1, 34.9, 33.4, 39.4),
|
||||
"Syria": (32.3, 35.7, 37.4, 42.4),
|
||||
"Iraq": (29.0, 38.8, 37.4, 48.8),
|
||||
"Iran": (25.0, 44.0, 40.0, 63.4),
|
||||
"Saudi Arabia": (16.3, 34.5, 32.2, 55.7),
|
||||
"Yemen": (12.0, 42.5, 19.0, 54.5),
|
||||
"United Arab Emirates": (22.6, 51.5, 26.1, 56.4),
|
||||
"Oman": (16.6, 52.0, 26.4, 59.9),
|
||||
"Qatar": (24.4, 50.7, 26.2, 51.7),
|
||||
"Bahrain": (25.8, 50.4, 26.4, 50.8),
|
||||
"Kuwait": (28.5, 46.5, 30.1, 48.4),
|
||||
"Afghanistan": (29.4, 60.5, 38.5, 74.9),
|
||||
# Asia
|
||||
"India": (6.0, 68.0, 36.0, 98.0),
|
||||
"Pakistan": (23.7, 60.9, 37.1, 77.8),
|
||||
"Bangladesh": (20.6, 88.0, 26.6, 92.7),
|
||||
"Sri Lanka": (5.9, 79.5, 9.9, 82.0),
|
||||
"Nepal": (26.3, 80.0, 30.5, 88.2),
|
||||
"China": (18.0, 73.0, 54.0, 135.5),
|
||||
"Mongolia": (41.6, 87.7, 52.2, 119.9),
|
||||
"Japan": (24.0, 122.0, 46.0, 146.0),
|
||||
"South Korea": (33.1, 125.1, 38.6, 131.9),
|
||||
"North Korea": (37.7, 124.2, 43.0, 130.7),
|
||||
"Taiwan": (21.8, 119.3, 25.4, 122.1),
|
||||
"Hong Kong": (22.1, 113.8, 22.6, 114.5),
|
||||
"Vietnam": (8.2, 102.1, 23.4, 109.5),
|
||||
"Thailand": (5.6, 97.3, 20.5, 105.7),
|
||||
"Cambodia": (10.4, 102.3, 14.7, 107.7),
|
||||
"Laos": (13.9, 100.0, 22.5, 107.7),
|
||||
"Myanmar": (9.5, 92.1, 28.6, 101.2),
|
||||
"Malaysia": (0.8, 99.5, 7.5, 119.3),
|
||||
"Singapore": (1.1, 103.5, 1.5, 104.1),
|
||||
"Indonesia": (-11.1, 94.8, 6.1, 141.1),
|
||||
"Philippines": (4.5, 116.0, 21.5, 127.0),
|
||||
"Brunei": (4.0, 114.0, 5.1, 115.4),
|
||||
"Kazakhstan": (40.5, 46.4, 55.5, 87.4),
|
||||
"Uzbekistan": (37.1, 55.9, 45.6, 73.2),
|
||||
"Kyrgyzstan": (39.1, 69.2, 43.3, 80.3),
|
||||
"Tajikistan": (36.6, 67.3, 41.1, 75.2),
|
||||
"Turkmenistan": (35.1, 52.4, 42.8, 66.7),
|
||||
"Azerbaijan": (38.3, 44.7, 41.9, 50.6),
|
||||
"Armenia": (38.8, 43.4, 41.3, 46.6),
|
||||
"Georgia": (41.0, 40.0, 43.6, 46.8),
|
||||
# Oceania
|
||||
"Australia": (-44.0, 112.0, -9.0, 155.0),
|
||||
"New Zealand": (-48.0, 165.0, -33.0, 179.5),
|
||||
"Papua New Guinea": (-11.7, 140.8, -1.0, 156.0),
|
||||
"Fiji": (-21.0, 176.8, -12.4, -178.3), # crosses antimeridian; see handling
|
||||
# Africa (selected — most common NUFORC reporters)
|
||||
"South Africa": (-35.0, 16.0, -22.0, 33.0),
|
||||
"Egypt": (21.7, 24.7, 31.7, 36.9),
|
||||
"Morocco": (27.6, -13.2, 35.9, -1.0),
|
||||
"Algeria": (18.9, -8.7, 37.1, 12.0),
|
||||
"Tunisia": (30.2, 7.5, 37.5, 11.6),
|
||||
"Libya": (19.5, 9.3, 33.2, 25.2),
|
||||
"Sudan": (8.6, 21.8, 22.2, 38.6),
|
||||
"Ethiopia": (3.4, 32.9, 14.9, 48.0),
|
||||
"Kenya": (-4.7, 33.9, 5.5, 41.9),
|
||||
"Tanzania": (-11.8, 29.3, -0.9, 40.4),
|
||||
"Uganda": (-1.5, 29.5, 4.2, 35.0),
|
||||
"Nigeria": (4.2, 2.6, 13.9, 14.7),
|
||||
"Ghana": (4.7, -3.3, 11.2, 1.2),
|
||||
"Senegal": (12.3, -17.6, 16.7, -11.3),
|
||||
"Ivory Coast": (4.3, -8.6, 10.7, -2.5),
|
||||
"Cameroon": (1.6, 8.5, 13.1, 16.2),
|
||||
"Angola": (-18.1, 11.7, -4.4, 24.1),
|
||||
"Zimbabwe": (-22.5, 25.2, -15.6, 33.1),
|
||||
"Zambia": (-18.1, 21.9, -8.2, 33.7),
|
||||
"Mozambique": (-26.9, 30.2, -10.5, 40.9),
|
||||
"Madagascar": (-25.7, 43.2, -11.9, 50.5),
|
||||
"Democratic Republic of the Congo": (-13.5, 12.2, 5.4, 31.4),
|
||||
"Rwanda": (-2.9, 28.8, -1.0, 30.9),
|
||||
}
|
||||
|
||||
# Common aliases used in NUFORC / other data sources.
|
||||
_COUNTRY_ALIASES: dict[str, str] = {
|
||||
"US": "USA",
|
||||
"U.S.": "USA",
|
||||
"U.S.A.": "USA",
|
||||
"United States": "USA",
|
||||
"United States of America": "USA",
|
||||
"America": "USA",
|
||||
"UK": "United Kingdom",
|
||||
"U.K.": "United Kingdom",
|
||||
"Britain": "United Kingdom",
|
||||
"Great Britain": "United Kingdom",
|
||||
"England": "United Kingdom",
|
||||
"Scotland": "United Kingdom",
|
||||
"Wales": "United Kingdom",
|
||||
"Northern Ireland": "United Kingdom",
|
||||
"Czechia": "Czech Republic",
|
||||
"Czechoslovakia": "Czech Republic",
|
||||
"South Korea": "South Korea",
|
||||
"Korea": "South Korea",
|
||||
"Republic of Korea": "South Korea",
|
||||
"Democratic People's Republic of Korea": "North Korea",
|
||||
"DPRK": "North Korea",
|
||||
"Russian Federation": "Russia",
|
||||
"Viet Nam": "Vietnam",
|
||||
"Côte d'Ivoire": "Ivory Coast",
|
||||
"Cote d'Ivoire": "Ivory Coast",
|
||||
"DR Congo": "Democratic Republic of the Congo",
|
||||
"DRC": "Democratic Republic of the Congo",
|
||||
"Congo-Kinshasa": "Democratic Republic of the Congo",
|
||||
"Macedonia": "North Macedonia",
|
||||
"Burma": "Myanmar",
|
||||
"Holland": "Netherlands",
|
||||
}
|
||||
|
||||
|
||||
def canonical_country(country: str) -> str:
|
||||
"""Normalise a country string to its registry key."""
|
||||
if not country:
|
||||
return ""
|
||||
c = country.strip()
|
||||
return _COUNTRY_ALIASES.get(c, c)
|
||||
|
||||
|
||||
def coord_in_country(lat: float, lng: float, country: str) -> Optional[bool]:
|
||||
"""Return True if (lat, lng) is inside the country bbox, False if it
|
||||
is outside, or None if the country is unknown (cannot validate — the
|
||||
caller should treat unknown as "pass", not "fail").
|
||||
"""
|
||||
try:
|
||||
lat_f = float(lat)
|
||||
lng_f = float(lng)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if not (-90.0 <= lat_f <= 90.0 and -180.0 <= lng_f <= 180.0):
|
||||
return False
|
||||
c = canonical_country(country)
|
||||
bbox = _COUNTRY_BBOX.get(c)
|
||||
if bbox is None:
|
||||
return None
|
||||
min_lat, min_lng, max_lat, max_lng = bbox
|
||||
return min_lat <= lat_f <= max_lat and min_lng <= lng_f <= max_lng
|
||||
|
||||
|
||||
def validate_geocode(
|
||||
lat: float,
|
||||
lng: float,
|
||||
country: str,
|
||||
) -> bool:
|
||||
"""Higher-level gate used in fetcher geocoding loops.
|
||||
|
||||
Returns True if the coordinate is acceptable for the given country,
|
||||
False if it's clearly a namesake collision that should be rejected.
|
||||
Unknown countries are treated as "accept" so we don't throw away
|
||||
otherwise-good data for uncovered regions.
|
||||
"""
|
||||
result = coord_in_country(lat, lng, country)
|
||||
return result is not False
|
||||
@@ -201,10 +201,12 @@ def _is_gibberish(text):
|
||||
# Persistent cache for article titles — survives across GDELT cache refreshes
|
||||
# Bounded to 5000 entries with 24hr TTL to prevent unbounded memory growth
|
||||
_article_title_cache = TTLCache(maxsize=5000, ttl=86400)
|
||||
_article_snippet_cache: dict[str, str | None] = {}
|
||||
_article_url_safety_cache = TTLCache(maxsize=5000, ttl=3600)
|
||||
_TITLE_FETCH_MAX_REDIRECTS = 3
|
||||
_TITLE_FETCH_READ_BYTES = 32768
|
||||
_ALLOWED_ARTICLE_PORTS = {80, 443, 8080, 8443}
|
||||
_MAX_SNIPPET_LEN = 200
|
||||
|
||||
|
||||
def _hostname_resolves_public(hostname: str, port: int) -> bool:
|
||||
@@ -269,6 +271,30 @@ def _is_safe_public_article_url(url: str) -> tuple[bool, str]:
|
||||
return result
|
||||
|
||||
|
||||
def _extract_snippet(url: str, chunk: str) -> None:
|
||||
"""Extract og:description or meta description from an already-fetched HTML chunk."""
|
||||
import re
|
||||
import html as html_mod
|
||||
|
||||
if url in _article_snippet_cache:
|
||||
return
|
||||
snippet = None
|
||||
# Try og:description first
|
||||
for pattern in (
|
||||
r'<meta[^>]+property=["\']og:description["\'][^>]+content=["\']([^"\'>]+)["\']',
|
||||
r'<meta[^>]+content=["\']([^"\'>]+)["\'][^>]+property=["\']og:description["\']',
|
||||
r'<meta[^>]+name=["\']description["\'][^>]+content=["\']([^"\'>]+)["\']',
|
||||
r'<meta[^>]+content=["\']([^"\'>]+)["\'][^>]+name=["\']description["\']',
|
||||
):
|
||||
m = re.search(pattern, chunk, re.I)
|
||||
if m:
|
||||
snippet = html_mod.unescape(m.group(1)).strip()
|
||||
break
|
||||
if snippet and len(snippet) > _MAX_SNIPPET_LEN:
|
||||
snippet = snippet[:_MAX_SNIPPET_LEN - 3].rsplit(" ", 1)[0] + "..."
|
||||
_article_snippet_cache[url] = snippet if snippet and len(snippet) > 15 else None
|
||||
|
||||
|
||||
def _fetch_article_title(url):
|
||||
"""Fetch the real headline from an article's HTML <title> or og:title tag.
|
||||
Returns the title string, or None if it can't be fetched.
|
||||
@@ -343,6 +369,8 @@ def _fetch_article_title(url):
|
||||
title = title[:117] + "..."
|
||||
if len(title) > 10:
|
||||
_article_title_cache[url] = title
|
||||
# Also extract og:description / meta description for snippet
|
||||
_extract_snippet(url, chunk)
|
||||
return title
|
||||
|
||||
_article_title_cache[url] = None
|
||||
@@ -405,21 +433,49 @@ def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_
|
||||
actor1 = row[6].strip() if len(row) > 6 else ""
|
||||
actor2 = row[16].strip() if len(row) > 16 else ""
|
||||
|
||||
# Extract enrichment fields from GDELT CSV
|
||||
event_date = row[1].strip() if len(row) > 1 else ""
|
||||
full_event_code = row[26].strip() if len(row) > 26 else ""
|
||||
quad_class = int(row[29]) if len(row) > 29 and row[29].strip().isdigit() else 0
|
||||
goldstein = float(row[30]) if len(row) > 30 and row[30].strip() else 0.0
|
||||
num_mentions = int(row[31]) if len(row) > 31 and row[31].strip().isdigit() else 0
|
||||
num_sources = int(row[32]) if len(row) > 32 and row[32].strip().isdigit() else 0
|
||||
num_articles = int(row[33]) if len(row) > 33 and row[33].strip().isdigit() else 0
|
||||
avg_tone = float(row[34]) if len(row) > 34 and row[34].strip() else 0.0
|
||||
|
||||
loc_key = f"{round(lat, 1)}_{round(lng, 1)}"
|
||||
if loc_key in seen_locs:
|
||||
# Merge: increment count and add source URL if new (dedup by domain)
|
||||
# Merge: increment count, accumulate intensity, add source URL
|
||||
idx = loc_index[loc_key]
|
||||
feat = features[idx]
|
||||
feat["properties"]["count"] = feat["properties"].get("count", 1) + 1
|
||||
urls = feat["properties"].get("_urls", [])
|
||||
seen_domains = feat["properties"].get("_domains", set())
|
||||
props = feat["properties"]
|
||||
props["count"] = props.get("count", 1) + 1
|
||||
# Track worst Goldstein score (most negative = most intense)
|
||||
if goldstein < props.get("goldstein", 0):
|
||||
props["goldstein"] = round(goldstein, 1)
|
||||
# Accumulate mentions/sources for importance ranking
|
||||
props["num_mentions"] = props.get("num_mentions", 0) + num_mentions
|
||||
props["num_sources"] = props.get("num_sources", 0) + num_sources
|
||||
props["num_articles"] = props.get("num_articles", 0) + num_articles
|
||||
# Track latest date
|
||||
if event_date and event_date > props.get("event_date", ""):
|
||||
props["event_date"] = event_date
|
||||
# Collect actors
|
||||
actors = props.get("_actors_set", set())
|
||||
if actor1:
|
||||
actors.add(actor1)
|
||||
if actor2:
|
||||
actors.add(actor2)
|
||||
props["_actors_set"] = actors
|
||||
urls = props.get("_urls", [])
|
||||
seen_domains = props.get("_domains", set())
|
||||
if source_url:
|
||||
domain = _extract_domain(source_url)
|
||||
if domain not in seen_domains and len(urls) < 10:
|
||||
urls.append(source_url)
|
||||
seen_domains.add(domain)
|
||||
feat["properties"]["_urls"] = urls
|
||||
feat["properties"]["_domains"] = seen_domains
|
||||
props["_urls"] = urls
|
||||
props["_domains"] = seen_domains
|
||||
continue
|
||||
seen_locs.add(loc_key)
|
||||
|
||||
@@ -429,6 +485,11 @@ def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_
|
||||
or "Unknown Incident"
|
||||
)
|
||||
domain = _extract_domain(source_url) if source_url else ""
|
||||
actors_set = set()
|
||||
if actor1:
|
||||
actors_set.add(actor1)
|
||||
if actor2:
|
||||
actors_set.add(actor2)
|
||||
loc_index[loc_key] = len(features)
|
||||
features.append(
|
||||
{
|
||||
@@ -436,6 +497,17 @@ def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_
|
||||
"properties": {
|
||||
"name": name,
|
||||
"count": 1,
|
||||
"event_date": event_date,
|
||||
"event_code": full_event_code,
|
||||
"quad_class": quad_class,
|
||||
"goldstein": round(goldstein, 1),
|
||||
"num_mentions": num_mentions,
|
||||
"num_sources": num_sources,
|
||||
"num_articles": num_articles,
|
||||
"avg_tone": round(avg_tone, 1),
|
||||
"actor1": actor1,
|
||||
"actor2": actor2,
|
||||
"_actors_set": actors_set,
|
||||
"_urls": [source_url] if source_url else [],
|
||||
"_domains": {domain} if domain else set(),
|
||||
},
|
||||
@@ -468,12 +540,19 @@ def _build_feature_html(features, fetched_titles=None):
|
||||
for f in features:
|
||||
urls = f["properties"].pop("_urls", [])
|
||||
f["properties"].pop("_domains", None)
|
||||
# Convert actors set to sorted list for JSON serialization
|
||||
actors_set = f["properties"].pop("_actors_set", set())
|
||||
if actors_set:
|
||||
f["properties"]["actors"] = sorted(actors_set)[:6]
|
||||
headlines = []
|
||||
snippets = []
|
||||
for u in urls:
|
||||
real_title = fetched_titles.get(u) if fetched_titles else None
|
||||
headlines.append(real_title if real_title else _url_to_headline(u))
|
||||
snippets.append(_article_snippet_cache.get(u) or "")
|
||||
f["properties"]["_urls_list"] = urls
|
||||
f["properties"]["_headlines_list"] = headlines
|
||||
f["properties"]["_snippets_list"] = snippets
|
||||
if urls:
|
||||
links = []
|
||||
for u, h in zip(urls, headlines):
|
||||
@@ -498,16 +577,19 @@ def _enrich_gdelt_titles_background(features, all_article_urls):
|
||||
fetched_count = sum(1 for v in fetched_titles.values() if v)
|
||||
logger.info(f"[BG] Resolved {fetched_count}/{len(all_article_urls)} article titles")
|
||||
|
||||
# Update features in-place with real titles
|
||||
# Update features in-place with real titles and snippets
|
||||
for f in features:
|
||||
urls = f["properties"].get("_urls_list", [])
|
||||
if not urls:
|
||||
continue
|
||||
headlines = []
|
||||
snippets = []
|
||||
for u in urls:
|
||||
real_title = fetched_titles.get(u)
|
||||
headlines.append(real_title if real_title else _url_to_headline(u))
|
||||
snippets.append(_article_snippet_cache.get(u) or "")
|
||||
f["properties"]["_headlines_list"] = headlines
|
||||
f["properties"]["_snippets_list"] = snippets
|
||||
links = []
|
||||
for u, h in zip(urls, headlines):
|
||||
safe_url = u if u.startswith(("http://", "https://")) else "about:blank"
|
||||
@@ -564,8 +646,8 @@ def fetch_global_military_incidents():
|
||||
|
||||
latest_ts = datetime.strptime(ts_match.group(1), "%Y%m%d%H%M%S")
|
||||
|
||||
# Generate URLs for the last 8 hours (32 files at 15-min intervals)
|
||||
NUM_FILES = 32
|
||||
# Generate URLs for the last 12 hours (48 files at 15-min intervals)
|
||||
NUM_FILES = 48
|
||||
urls = []
|
||||
for i in range(NUM_FILES):
|
||||
ts = latest_ts - timedelta(minutes=15 * i)
|
||||
@@ -583,7 +665,7 @@ def fetch_global_military_incidents():
|
||||
logger.info(f"Downloaded {successful}/{len(urls)} GDELT exports")
|
||||
|
||||
# Parse all downloaded files
|
||||
CONFLICT_CODES = {"14", "17", "18", "19", "20"}
|
||||
CONFLICT_CODES = {"13", "14", "15", "16", "17", "18", "19", "20"}
|
||||
features = []
|
||||
seen_locs = set()
|
||||
loc_index = {} # loc_key -> index in features
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Infonet economy & governance layer.
|
||||
|
||||
Layered ON TOP OF the existing mesh primitives in ``services/mesh/``.
|
||||
The chain-write cutover (2026-04-28) registers Infonet event types
|
||||
with ``mesh_schema`` and ``mesh_hashchain`` so production writes flow
|
||||
through the legacy chain. The cutover is performed at import time by
|
||||
``services.infonet._chain_cutover``.
|
||||
|
||||
The only legacy file modified by the cutover is ``mesh_schema.py``,
|
||||
which gained a generic extension hook (``register_extension_validator``).
|
||||
``mesh_hashchain.py`` is byte-identical to its Sprint 1 baseline; the
|
||||
cutover mutates its module-level ``ACTIVE_APPEND_EVENT_TYPES`` set
|
||||
(which is a mutable ``set``, not a frozenset, by design).
|
||||
|
||||
See ``infonet-economy/IMPLEMENTATION_PLAN.md`` and ``infonet-economy/BUILD_LOG.md``
|
||||
in the repository root for the build order, sprint scope, and integration
|
||||
principles. ``infonet-economy/RULES_SKELETON.md`` is the source of truth
|
||||
for any formula / value / state machine implemented here.
|
||||
"""
|
||||
|
||||
# Trigger the chain-write cutover at import time. Idempotent — see
|
||||
# ``_chain_cutover.perform_cutover``. This must happen before any
|
||||
# adapter or producer uses mesh_schema.validate_event_payload on a
|
||||
# new event type.
|
||||
from services.infonet import _chain_cutover as _chain_cutover_module
|
||||
_chain_cutover_module.perform_cutover()
|
||||
del _chain_cutover_module
|
||||
|
||||
from services.infonet.config import (
|
||||
CONFIG,
|
||||
CONFIG_SCHEMA,
|
||||
CROSS_FIELD_INVARIANTS,
|
||||
IMMUTABLE_PRINCIPLES,
|
||||
InvalidPetition,
|
||||
reset_config_for_tests,
|
||||
validate_config_schema_completeness,
|
||||
validate_cross_field_invariants,
|
||||
validate_petition_value,
|
||||
)
|
||||
from services.infonet.identity_rotation import (
|
||||
RotationBlocker,
|
||||
RotationDecision,
|
||||
rotation_descendants,
|
||||
validate_rotation,
|
||||
)
|
||||
from services.infonet.markets import (
|
||||
EvidenceBundle,
|
||||
MarketStatus,
|
||||
ResolutionResult,
|
||||
build_snapshot,
|
||||
collect_evidence,
|
||||
collect_resolution_stakes,
|
||||
compute_market_status,
|
||||
compute_snapshot_event_hash,
|
||||
evidence_content_hash,
|
||||
excluded_predictor_ids,
|
||||
find_snapshot,
|
||||
is_first_for_side,
|
||||
is_predictor_excluded,
|
||||
resolve_market,
|
||||
should_advance_phase,
|
||||
submission_hash,
|
||||
)
|
||||
from services.infonet.reputation import (
|
||||
OracleRepBreakdown,
|
||||
compute_common_rep,
|
||||
compute_oracle_rep,
|
||||
compute_oracle_rep_active,
|
||||
compute_oracle_rep_lifetime,
|
||||
decay_factor_for_age,
|
||||
last_successful_prediction_ts,
|
||||
)
|
||||
from services.infonet.schema import (
|
||||
INFONET_ECONOMY_EVENT_TYPES,
|
||||
InfonetEventSchema,
|
||||
get_infonet_schema,
|
||||
validate_infonet_event_payload,
|
||||
)
|
||||
from services.infonet.time_validity import (
|
||||
chain_majority_time,
|
||||
event_meets_phase_window,
|
||||
is_event_too_future,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"CONFIG",
|
||||
"CONFIG_SCHEMA",
|
||||
"CROSS_FIELD_INVARIANTS",
|
||||
"IMMUTABLE_PRINCIPLES",
|
||||
"INFONET_ECONOMY_EVENT_TYPES",
|
||||
"EvidenceBundle",
|
||||
"InfonetEventSchema",
|
||||
"InvalidPetition",
|
||||
"MarketStatus",
|
||||
"OracleRepBreakdown",
|
||||
"ResolutionResult",
|
||||
"RotationBlocker",
|
||||
"RotationDecision",
|
||||
"build_snapshot",
|
||||
"chain_majority_time",
|
||||
"collect_evidence",
|
||||
"collect_resolution_stakes",
|
||||
"compute_common_rep",
|
||||
"compute_market_status",
|
||||
"compute_oracle_rep",
|
||||
"compute_oracle_rep_active",
|
||||
"compute_oracle_rep_lifetime",
|
||||
"compute_snapshot_event_hash",
|
||||
"decay_factor_for_age",
|
||||
"event_meets_phase_window",
|
||||
"evidence_content_hash",
|
||||
"excluded_predictor_ids",
|
||||
"find_snapshot",
|
||||
"get_infonet_schema",
|
||||
"is_event_too_future",
|
||||
"is_first_for_side",
|
||||
"is_predictor_excluded",
|
||||
"last_successful_prediction_ts",
|
||||
"reset_config_for_tests",
|
||||
"resolve_market",
|
||||
"rotation_descendants",
|
||||
"should_advance_phase",
|
||||
"submission_hash",
|
||||
"validate_config_schema_completeness",
|
||||
"validate_cross_field_invariants",
|
||||
"validate_infonet_event_payload",
|
||||
"validate_petition_value",
|
||||
"validate_rotation",
|
||||
]
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Chain-write cutover — register Infonet economy event types with the
|
||||
legacy mesh_schema + mesh_hashchain at import time.
|
||||
|
||||
Source of truth: ``infonet-economy/BUILD_LOG.md`` Sprint 4 §6.2 cutover
|
||||
decision (Option C — rename + coexist with new event-type names).
|
||||
|
||||
Before this cutover, Sprints 1-7 produced economy events through
|
||||
``InfonetHashchainAdapter.dry_run_append`` only. None of them landed
|
||||
on the legacy chain because ``mesh_hashchain.Infonet.append`` rejected
|
||||
any event_type not in ``ACTIVE_APPEND_EVENT_TYPES``.
|
||||
|
||||
This module performs the surgical wiring needed for production writes:
|
||||
|
||||
1. Mutates ``mesh_hashchain.ACTIVE_APPEND_EVENT_TYPES`` (a mutable
|
||||
set, not a frozenset) to include every type in
|
||||
``INFONET_ECONOMY_EVENT_TYPES``.
|
||||
2. Registers each economy event type's payload validator with
|
||||
``mesh_schema._EXTENSION_VALIDATORS`` via the Sprint-8-polish
|
||||
``register_extension_validator`` hook.
|
||||
|
||||
The cutover is **idempotent**: importing this module twice leaves the
|
||||
state unchanged.
|
||||
|
||||
The direction is **one-way**: infonet imports mesh_*; mesh never
|
||||
imports infonet. mesh_schema's hook is generic — it doesn't know
|
||||
about infonet specifically.
|
||||
|
||||
What is NOT modified by this cutover:
|
||||
|
||||
- ``mesh_schema.SCHEMA_REGISTRY`` — legacy validators stay as-is.
|
||||
Economy types use the parallel ``_EXTENSION_VALIDATORS`` registry.
|
||||
- ``mesh_schema.ACTIVE_PUBLIC_LEDGER_EVENT_TYPES`` — legacy frozenset
|
||||
unchanged. The runtime decision in
|
||||
``mesh_hashchain.Infonet.append`` consults the mutable
|
||||
``ACTIVE_APPEND_EVENT_TYPES`` set.
|
||||
- ``mesh_hashchain.py`` — byte-identical to its Sprint 1 baseline.
|
||||
- The legacy ``normalize_payload`` and "no ephemeral on this type"
|
||||
checks — extension events skip them. Economy event payloads
|
||||
already have their own normalization (the schema in
|
||||
``services/infonet/schema.py``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
|
||||
from services.infonet.schema import (
|
||||
INFONET_ECONOMY_EVENT_TYPES,
|
||||
validate_infonet_event_payload,
|
||||
)
|
||||
from services.mesh import mesh_hashchain, mesh_schema
|
||||
|
||||
|
||||
_CUTOVER_LOCK = threading.Lock()
|
||||
_CUTOVER_DONE = False
|
||||
|
||||
|
||||
def perform_cutover() -> None:
|
||||
"""Idempotent registration of every Infonet economy event type.
|
||||
|
||||
Safe to call multiple times. After the first call, repeat calls
|
||||
are no-ops (the lock + sentinel guard re-entry).
|
||||
"""
|
||||
global _CUTOVER_DONE
|
||||
with _CUTOVER_LOCK:
|
||||
if _CUTOVER_DONE:
|
||||
return
|
||||
# Extend the active-append set so mesh_hashchain.Infonet.append
|
||||
# accepts these types. The set is mutable by design (legacy
|
||||
# mesh_hashchain.py line 163 uses set(), not frozenset()).
|
||||
mesh_hashchain.ACTIVE_APPEND_EVENT_TYPES.update(INFONET_ECONOMY_EVENT_TYPES)
|
||||
# Register a validator for each. The lambda binds to the loop
|
||||
# variable via default-arg trick to avoid late-binding bugs.
|
||||
for event_type in INFONET_ECONOMY_EVENT_TYPES:
|
||||
mesh_schema.register_extension_validator(
|
||||
event_type,
|
||||
lambda payload, _et=event_type: validate_infonet_event_payload(_et, payload),
|
||||
)
|
||||
_CUTOVER_DONE = True
|
||||
|
||||
|
||||
def cutover_status() -> dict[str, object]:
|
||||
"""Diagnostic — used by tests and health endpoints to confirm the
|
||||
cutover ran and registered every type."""
|
||||
return {
|
||||
"done": _CUTOVER_DONE,
|
||||
"registered_types": sorted(
|
||||
t for t in INFONET_ECONOMY_EVENT_TYPES
|
||||
if mesh_schema.is_extension_event_type(t)
|
||||
),
|
||||
"missing_types": sorted(
|
||||
t for t in INFONET_ECONOMY_EVENT_TYPES
|
||||
if not mesh_schema.is_extension_event_type(t)
|
||||
),
|
||||
"active_append_includes_economy": INFONET_ECONOMY_EVENT_TYPES.issubset(
|
||||
mesh_hashchain.ACTIVE_APPEND_EVENT_TYPES
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# Run automatically when the module is imported. The infonet package
|
||||
# __init__ imports this module, so any code that uses
|
||||
# ``services.infonet`` at all triggers the cutover. Production callers
|
||||
# don't need to do anything explicit.
|
||||
perform_cutover()
|
||||
|
||||
|
||||
__all__ = ["cutover_status", "perform_cutover"]
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Adapter layer between the Infonet economy package and the legacy
|
||||
``services/mesh/`` primitives.
|
||||
|
||||
Rule: **adapters import from mesh, mesh never imports from infonet.**
|
||||
This keeps the dependency direction one-way and lets us delete the
|
||||
infonet package without touching mesh.
|
||||
|
||||
The legacy mesh files (``mesh_schema.py``, ``mesh_signed_events.py``,
|
||||
``mesh_hashchain.py``, ``mesh_reputation.py``, ``mesh_oracle.py``) stay
|
||||
byte-identical through Sprint 3. From Sprint 4 onward, when actual chain
|
||||
writes for new event types start happening, the hashchain adapter is
|
||||
the single integration point that decides whether to:
|
||||
|
||||
1. Modify ``ACTIVE_APPEND_EVENT_TYPES`` in ``mesh_schema.py`` (one-shot,
|
||||
minimal mesh change), OR
|
||||
2. Maintain a parallel append surface in ``hashchain_adapter`` that
|
||||
shares the on-disk chain file but bypasses the legacy event-type
|
||||
gate.
|
||||
|
||||
The decision is recorded in ``infonet-economy/BUILD_LOG.md`` Sprint 4
|
||||
when made.
|
||||
"""
|
||||
|
||||
from services.infonet.adapters.hashchain_adapter import (
|
||||
InfonetHashchainAdapter,
|
||||
extended_active_event_types,
|
||||
)
|
||||
from services.infonet.adapters.signed_write_adapter import (
|
||||
INFONET_SIGNED_WRITE_KINDS,
|
||||
InfonetSignedWriteKind,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"INFONET_SIGNED_WRITE_KINDS",
|
||||
"InfonetHashchainAdapter",
|
||||
"InfonetSignedWriteKind",
|
||||
"extended_active_event_types",
|
||||
]
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Gate adapter — Sprint 6 implementation.
|
||||
|
||||
Bridges chain history to the gate sacrifice / locking / shutdown
|
||||
lifecycle. Same ``chain_provider`` pattern as the other adapters.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any, Callable, Iterable
|
||||
|
||||
from services.infonet.gates import (
|
||||
AppealValidation,
|
||||
EntryDecision,
|
||||
GateMeta,
|
||||
LockedGateState,
|
||||
ShutdownState,
|
||||
SuspensionState,
|
||||
can_enter,
|
||||
compute_member_set,
|
||||
compute_shutdown_state,
|
||||
compute_suspension_state,
|
||||
cumulative_member_oracle_rep,
|
||||
get_gate_meta,
|
||||
is_locked,
|
||||
is_member,
|
||||
is_ratified,
|
||||
locked_at,
|
||||
locked_by,
|
||||
paused_execution_remaining_sec,
|
||||
validate_appeal_filing,
|
||||
validate_lock_request,
|
||||
validate_shutdown_filing,
|
||||
validate_suspend_filing,
|
||||
)
|
||||
from services.infonet.gates.locking import LockValidation
|
||||
from services.infonet.gates.shutdown.suspend import FilingValidation
|
||||
from services.infonet.time_validity import chain_majority_time
|
||||
|
||||
|
||||
_ChainProvider = Callable[[], Iterable[dict[str, Any]]]
|
||||
|
||||
|
||||
def _empty_chain() -> list[dict[str, Any]]:
|
||||
return []
|
||||
|
||||
|
||||
class InfonetGateAdapter:
|
||||
"""Project chain state into gate views."""
|
||||
|
||||
def __init__(self, chain_provider: _ChainProvider | None = None) -> None:
|
||||
self._chain_provider: _ChainProvider = chain_provider or _empty_chain
|
||||
|
||||
def _events(self) -> list[dict[str, Any]]:
|
||||
return [e for e in self._chain_provider() if isinstance(e, dict)]
|
||||
|
||||
def _now(self, override: float | None) -> float:
|
||||
if override is not None:
|
||||
return float(override)
|
||||
events = self._events()
|
||||
chain_now = chain_majority_time(events)
|
||||
return chain_now if chain_now > 0 else float(time.time())
|
||||
|
||||
# ── Metadata + membership ────────────────────────────────────────
|
||||
def gate_meta(self, gate_id: str) -> GateMeta | None:
|
||||
return get_gate_meta(gate_id, self._events())
|
||||
|
||||
def member_set(self, gate_id: str) -> set[str]:
|
||||
return compute_member_set(gate_id, self._events())
|
||||
|
||||
def is_member(self, node_id: str, gate_id: str) -> bool:
|
||||
return is_member(node_id, gate_id, self._events())
|
||||
|
||||
def can_enter(self, node_id: str, gate_id: str) -> EntryDecision:
|
||||
return can_enter(node_id, gate_id, self._events())
|
||||
|
||||
# ── Ratification ─────────────────────────────────────────────────
|
||||
def is_ratified(self, gate_id: str) -> bool:
|
||||
return is_ratified(gate_id, self._events())
|
||||
|
||||
def cumulative_member_oracle_rep(self, gate_id: str) -> float:
|
||||
return cumulative_member_oracle_rep(gate_id, self._events())
|
||||
|
||||
# ── Locking ──────────────────────────────────────────────────────
|
||||
def is_locked(self, gate_id: str) -> bool:
|
||||
return is_locked(gate_id, self._events())
|
||||
|
||||
def locked_state(self, gate_id: str) -> LockedGateState:
|
||||
events = self._events()
|
||||
return LockedGateState(
|
||||
locked=is_locked(gate_id, events),
|
||||
locked_at=locked_at(gate_id, events),
|
||||
locked_by=locked_by(gate_id, events),
|
||||
)
|
||||
|
||||
def validate_lock_request(
|
||||
self, node_id: str, gate_id: str, *, lock_cost: int | None = None,
|
||||
) -> LockValidation:
|
||||
return validate_lock_request(node_id, gate_id, self._events(), lock_cost=lock_cost)
|
||||
|
||||
# ── Suspension ───────────────────────────────────────────────────
|
||||
def suspension_state(
|
||||
self, gate_id: str, *, now: float | None = None,
|
||||
) -> SuspensionState:
|
||||
return compute_suspension_state(gate_id, self._events(), now=self._now(now))
|
||||
|
||||
def validate_suspend_filing(
|
||||
self,
|
||||
gate_id: str,
|
||||
filer_id: str,
|
||||
*,
|
||||
reason: str,
|
||||
evidence_hashes: list[str],
|
||||
now: float | None = None,
|
||||
filer_cooldown_until: float | None = None,
|
||||
) -> FilingValidation:
|
||||
return validate_suspend_filing(
|
||||
gate_id, filer_id,
|
||||
reason=reason, evidence_hashes=evidence_hashes,
|
||||
chain=self._events(), now=self._now(now),
|
||||
filer_cooldown_until=filer_cooldown_until,
|
||||
)
|
||||
|
||||
# ── Shutdown ─────────────────────────────────────────────────────
|
||||
def shutdown_state(
|
||||
self, gate_id: str, *, now: float | None = None,
|
||||
) -> ShutdownState:
|
||||
return compute_shutdown_state(gate_id, self._events(), now=self._now(now))
|
||||
|
||||
def validate_shutdown_filing(
|
||||
self,
|
||||
gate_id: str,
|
||||
filer_id: str,
|
||||
*,
|
||||
reason: str,
|
||||
evidence_hashes: list[str],
|
||||
now: float | None = None,
|
||||
filer_cooldown_until: float | None = None,
|
||||
) -> FilingValidation:
|
||||
return validate_shutdown_filing(
|
||||
gate_id, filer_id,
|
||||
reason=reason, evidence_hashes=evidence_hashes,
|
||||
chain=self._events(), now=self._now(now),
|
||||
filer_cooldown_until=filer_cooldown_until,
|
||||
)
|
||||
|
||||
# ── Appeal ───────────────────────────────────────────────────────
|
||||
def validate_appeal_filing(
|
||||
self,
|
||||
gate_id: str,
|
||||
target_petition_id: str,
|
||||
filer_id: str,
|
||||
*,
|
||||
reason: str,
|
||||
evidence_hashes: list[str],
|
||||
now: float | None = None,
|
||||
filer_cooldown_until: float | None = None,
|
||||
) -> AppealValidation:
|
||||
return validate_appeal_filing(
|
||||
gate_id, target_petition_id, filer_id,
|
||||
reason=reason, evidence_hashes=evidence_hashes,
|
||||
chain=self._events(), now=self._now(now),
|
||||
filer_cooldown_until=filer_cooldown_until,
|
||||
)
|
||||
|
||||
def paused_execution_remaining_sec(
|
||||
self,
|
||||
target_petition_id: str,
|
||||
*,
|
||||
appeal_filed_at: float,
|
||||
) -> float:
|
||||
return paused_execution_remaining_sec(
|
||||
target_petition_id, self._events(),
|
||||
appeal_filed_at=appeal_filed_at,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["InfonetGateAdapter"]
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Bridge between Infonet economy events and the legacy ``mesh_hashchain``.
|
||||
|
||||
Sprint 1 ships this as a **dry-run-only** surface. We do NOT call the
|
||||
legacy ``Infonet.append`` for new event types because that method
|
||||
hard-rejects anything not in ``ACTIVE_APPEND_EVENT_TYPES`` (defined in
|
||||
``mesh_schema.py``). Modifying that set is a Sprint 4 task — it requires
|
||||
the rest of the producer code to exist, otherwise a malformed
|
||||
``prediction_create`` could land on the chain with no resolver to
|
||||
process it.
|
||||
|
||||
What this adapter DOES today:
|
||||
|
||||
- ``extended_active_event_types()`` — returns the union of legacy active
|
||||
types and new economy types, for tooling that needs the full surface
|
||||
(e.g. RPC layer, frontend type generation).
|
||||
- ``InfonetHashchainAdapter.dry_run_append`` — validates a payload
|
||||
against the new schema and returns the event dict the legacy
|
||||
``Infonet.append`` would have built. Useful for tests and for the
|
||||
future cutover plan.
|
||||
|
||||
What this adapter will do in Sprint 4:
|
||||
|
||||
- ``append_infonet_event`` — actually call ``Infonet.append`` once
|
||||
``ACTIVE_APPEND_EVENT_TYPES`` is unioned with the economy types.
|
||||
|
||||
The Sprint 1 contract:
|
||||
|
||||
- ``mesh_hashchain.py`` is byte-identical to the pre-Sprint-1 baseline.
|
||||
- No event reaches the legacy chain via this adapter in Sprint 1.
|
||||
- Tests cover validation behavior only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from services.mesh.mesh_schema import (
|
||||
ACTIVE_PUBLIC_LEDGER_EVENT_TYPES as _LEGACY_ACTIVE_TYPES,
|
||||
)
|
||||
|
||||
from services.infonet.schema import (
|
||||
INFONET_ECONOMY_EVENT_TYPES,
|
||||
validate_infonet_event_payload,
|
||||
)
|
||||
|
||||
|
||||
def extended_active_event_types() -> frozenset[str]:
|
||||
"""Union of legacy active types and new economy types.
|
||||
|
||||
Frozen at import time. The legacy set is itself a frozenset so this
|
||||
is safe to call from any thread.
|
||||
"""
|
||||
return _LEGACY_ACTIVE_TYPES | INFONET_ECONOMY_EVENT_TYPES
|
||||
|
||||
|
||||
class InfonetHashchainAdapter:
|
||||
"""Validation-only adapter for new Infonet economy events.
|
||||
|
||||
Real chain integration lives in Sprint 4. Tests should use
|
||||
``dry_run_append`` to assert that producer code is constructing
|
||||
correctly-shaped events before the cutover.
|
||||
"""
|
||||
|
||||
def dry_run_append(
|
||||
self,
|
||||
event_type: str,
|
||||
node_id: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
sequence: int = 1,
|
||||
timestamp: float | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Validate and return a synthetic event dict.
|
||||
|
||||
Mirrors the shape that ``mesh_hashchain.Infonet.append`` would
|
||||
produce for legacy types — same field set, same ordering. Does
|
||||
NOT compute a real signature (Sprint 4 territory) and does NOT
|
||||
write to disk.
|
||||
|
||||
Raises ``ValueError`` on validation failure — the same exception
|
||||
type the legacy ``append`` raises so callers don't need to
|
||||
special-case the cutover later.
|
||||
"""
|
||||
if event_type not in INFONET_ECONOMY_EVENT_TYPES:
|
||||
raise ValueError(f"event_type {event_type!r} not in INFONET_ECONOMY_EVENT_TYPES")
|
||||
if not isinstance(node_id, str) or not node_id:
|
||||
raise ValueError("node_id is required")
|
||||
if not isinstance(sequence, int) or isinstance(sequence, bool) or sequence <= 0:
|
||||
raise ValueError("sequence must be a positive integer")
|
||||
|
||||
ok, reason = validate_infonet_event_payload(event_type, payload)
|
||||
if not ok:
|
||||
raise ValueError(reason)
|
||||
|
||||
ts = float(timestamp) if timestamp is not None else float(time.time())
|
||||
|
||||
canonical = {
|
||||
"event_type": event_type,
|
||||
"node_id": node_id,
|
||||
"payload": payload,
|
||||
"timestamp": ts,
|
||||
"sequence": sequence,
|
||||
}
|
||||
encoded = json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
||||
event_id = hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
|
||||
return {
|
||||
"event_id": event_id,
|
||||
"event_type": event_type,
|
||||
"node_id": node_id,
|
||||
"timestamp": ts,
|
||||
"sequence": sequence,
|
||||
"payload": payload,
|
||||
# signature / public_key intentionally omitted in Sprint 1.
|
||||
"is_provisional": True,
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"InfonetHashchainAdapter",
|
||||
"extended_active_event_types",
|
||||
]
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Adapter from chain history to the market lifecycle / resolution view.
|
||||
|
||||
Sprint 4: real implementation (replaces the Sprint 1 ``NotImplementedError``
|
||||
skeleton). Wires the pure functions in ``services/infonet/markets/`` to
|
||||
the same chain-provider pattern used by ``InfonetReputationAdapter``.
|
||||
|
||||
Sprint 5 will extend this with dispute open / dispute_stake / dispute
|
||||
resolve methods. Sprint 8 will extend the resolution path with
|
||||
bootstrap-mode handling.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Iterable
|
||||
|
||||
from services.infonet.markets import (
|
||||
DisputeView,
|
||||
EvidenceBundle,
|
||||
MarketStatus,
|
||||
ResolutionResult,
|
||||
build_snapshot,
|
||||
collect_disputes,
|
||||
collect_evidence,
|
||||
collect_resolution_stakes,
|
||||
compute_dispute_outcome,
|
||||
compute_market_status,
|
||||
compute_snapshot_event_hash,
|
||||
dispute_settlement_effects,
|
||||
effective_outcome,
|
||||
excluded_predictor_ids,
|
||||
find_snapshot,
|
||||
is_predictor_excluded,
|
||||
market_was_reversed,
|
||||
resolve_market,
|
||||
should_advance_phase,
|
||||
)
|
||||
|
||||
|
||||
_ChainProvider = Callable[[], Iterable[dict[str, Any]]]
|
||||
|
||||
|
||||
def _empty_chain() -> list[dict[str, Any]]:
|
||||
return []
|
||||
|
||||
|
||||
class InfonetOracleAdapter:
|
||||
"""Project chain state into market lifecycle + resolution views."""
|
||||
|
||||
def __init__(self, chain_provider: _ChainProvider | None = None) -> None:
|
||||
self._chain_provider: _ChainProvider = chain_provider or _empty_chain
|
||||
|
||||
def _events(self) -> list[dict[str, Any]]:
|
||||
return [e for e in self._chain_provider() if isinstance(e, dict)]
|
||||
|
||||
# ── Lifecycle ────────────────────────────────────────────────────
|
||||
def market_status(self, market_id: str, *, now: float) -> MarketStatus:
|
||||
return compute_market_status(market_id, self._events(), now=now)
|
||||
|
||||
def should_advance_phase(
|
||||
self, market_id: str, *, now: float,
|
||||
) -> tuple[MarketStatus, MarketStatus] | None:
|
||||
return should_advance_phase(market_id, self._events(), now=now)
|
||||
|
||||
# ── Snapshot ─────────────────────────────────────────────────────
|
||||
def take_snapshot(self, market_id: str, *, frozen_at: float) -> dict[str, Any]:
|
||||
return build_snapshot(market_id, self._events(), frozen_at=frozen_at)
|
||||
|
||||
def find_snapshot(self, market_id: str) -> dict[str, Any] | None:
|
||||
return find_snapshot(market_id, self._events())
|
||||
|
||||
@staticmethod
|
||||
def snapshot_event_hash(
|
||||
snapshot_payload: dict[str, Any],
|
||||
*,
|
||||
market_id: str,
|
||||
creator_node_id: str,
|
||||
sequence: int,
|
||||
) -> str:
|
||||
return compute_snapshot_event_hash(
|
||||
snapshot_payload,
|
||||
market_id=market_id,
|
||||
creator_node_id=creator_node_id,
|
||||
sequence=sequence,
|
||||
)
|
||||
|
||||
# ── Evidence ─────────────────────────────────────────────────────
|
||||
def collect_evidence(self, market_id: str) -> list[EvidenceBundle]:
|
||||
return collect_evidence(market_id, self._events())
|
||||
|
||||
# ── Resolution ───────────────────────────────────────────────────
|
||||
def excluded_predictor_ids(self, market_id: str) -> set[str]:
|
||||
return excluded_predictor_ids(market_id, self._events())
|
||||
|
||||
def is_predictor_excluded(self, node_id: str, market_id: str) -> bool:
|
||||
return is_predictor_excluded(node_id, market_id, self._events())
|
||||
|
||||
def collect_resolution_stakes(self, market_id: str):
|
||||
return collect_resolution_stakes(market_id, self._events())
|
||||
|
||||
def resolve_market(
|
||||
self, market_id: str, *, is_provisional: bool = False,
|
||||
) -> ResolutionResult:
|
||||
return resolve_market(market_id, self._events(), is_provisional=is_provisional)
|
||||
|
||||
# ── Disputes (Sprint 5) ──────────────────────────────────────────
|
||||
def collect_disputes(self, market_id: str) -> list[DisputeView]:
|
||||
return collect_disputes(market_id, self._events())
|
||||
|
||||
@staticmethod
|
||||
def compute_dispute_outcome(dispute: DisputeView) -> str:
|
||||
return compute_dispute_outcome(dispute)
|
||||
|
||||
@staticmethod
|
||||
def dispute_settlement_effects(dispute: DisputeView) -> dict:
|
||||
return dispute_settlement_effects(dispute)
|
||||
|
||||
def market_was_reversed(self, market_id: str) -> bool:
|
||||
return market_was_reversed(market_id, self._events())
|
||||
|
||||
def effective_outcome(self, market_id: str, original_outcome: str) -> str:
|
||||
return effective_outcome(original_outcome, market_id, self._events())
|
||||
|
||||
|
||||
__all__ = ["InfonetOracleAdapter"]
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Adapter that projects chain history into the new reputation views.
|
||||
|
||||
Sprint 2: real implementation. Replaces the Sprint 1 ``NotImplementedError``
|
||||
skeleton with the pure functions in ``services/infonet/reputation/``.
|
||||
|
||||
Why this exists rather than callers importing the pure functions
|
||||
directly: the adapter is the single integration boundary that future
|
||||
sprints will extend (Sprint 3 wraps anti-gaming penalties around the
|
||||
common-rep view, Sprint 4 extends the oracle-rep balance with
|
||||
resolution-stake redistribution, Sprint 5 layers in dispute reversal).
|
||||
By keeping callers on this adapter, the producer code never has to
|
||||
change as those layers ship.
|
||||
|
||||
The adapter takes a ``chain_provider`` callable rather than reaching
|
||||
into ``mesh_hashchain`` itself. Two reasons:
|
||||
|
||||
1. Tests pass a list of synthetic events directly — no hashchain
|
||||
instance required, no fixture overhead.
|
||||
2. Sprint 4 cutover decisions (parallel append surface vs unifying
|
||||
``ACTIVE_APPEND_EVENT_TYPES``) won't ripple into reputation code.
|
||||
|
||||
Cross-cutting design rule: reputation reads are background work. They
|
||||
must NEVER block a user-facing request. The adapter exposes only pure
|
||||
synchronous functions because they ARE pure — caches at the adapter
|
||||
layer (Sprint 3+) make repeat reads cheap. Callers that need real-time
|
||||
freshness should call directly on each request; callers that can
|
||||
tolerate staleness should poll a cached adapter instance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any, Callable, Iterable
|
||||
|
||||
from services.infonet.reputation import (
|
||||
OracleRepBreakdown,
|
||||
compute_common_rep,
|
||||
compute_oracle_rep,
|
||||
compute_oracle_rep_active,
|
||||
compute_oracle_rep_lifetime,
|
||||
decay_factor_for_age,
|
||||
last_successful_prediction_ts,
|
||||
)
|
||||
from services.infonet.reputation.oracle_rep import compute_oracle_rep_breakdown
|
||||
from services.infonet.time_validity import chain_majority_time
|
||||
|
||||
|
||||
_ChainProvider = Callable[[], Iterable[dict[str, Any]]]
|
||||
|
||||
|
||||
def _empty_chain() -> list[dict[str, Any]]:
|
||||
return []
|
||||
|
||||
|
||||
class InfonetReputationAdapter:
|
||||
"""Project chain state into oracle/common rep views.
|
||||
|
||||
``chain_provider`` is a zero-arg callable returning an iterable of
|
||||
chain events. Pass a closure that reads from
|
||||
``mesh_hashchain.Infonet.events`` in production, or a literal list
|
||||
in tests.
|
||||
"""
|
||||
|
||||
def __init__(self, chain_provider: _ChainProvider | None = None) -> None:
|
||||
self._chain_provider: _ChainProvider = chain_provider or _empty_chain
|
||||
|
||||
def _events(self) -> list[dict[str, Any]]:
|
||||
return [e for e in self._chain_provider() if isinstance(e, dict)]
|
||||
|
||||
def oracle_rep(self, node_id: str) -> float:
|
||||
return compute_oracle_rep(node_id, self._events())
|
||||
|
||||
def oracle_rep_breakdown(self, node_id: str) -> OracleRepBreakdown:
|
||||
return compute_oracle_rep_breakdown(node_id, self._events())
|
||||
|
||||
def oracle_rep_lifetime(self, node_id: str) -> float:
|
||||
return compute_oracle_rep_lifetime(node_id, self._events())
|
||||
|
||||
def oracle_rep_active(self, node_id: str, *, now: float | None = None) -> float:
|
||||
events = self._events()
|
||||
if now is None:
|
||||
chain_now = chain_majority_time(events)
|
||||
# Fall back to local clock only when the chain has no
|
||||
# distinct-node history yet (genesis / fresh mesh). This is
|
||||
# the only place a local clock leaks into governance —
|
||||
# acceptable because there are no oracles to penalize yet.
|
||||
now = chain_now if chain_now > 0 else time.time()
|
||||
return compute_oracle_rep_active(node_id, events, now=now)
|
||||
|
||||
def common_rep(self, node_id: str) -> float:
|
||||
return compute_common_rep(node_id, self._events())
|
||||
|
||||
def last_successful_prediction_ts(self, node_id: str) -> float | None:
|
||||
return last_successful_prediction_ts(node_id, self._events())
|
||||
|
||||
def decay_factor(self, node_id: str, *, now: float | None = None) -> float:
|
||||
events = self._events()
|
||||
if now is None:
|
||||
now = chain_majority_time(events) or time.time()
|
||||
last_ts = last_successful_prediction_ts(node_id, events)
|
||||
if last_ts is None:
|
||||
return 0.0
|
||||
days = max(0.0, (float(now) - last_ts) / 86400.0)
|
||||
return decay_factor_for_age(days)
|
||||
|
||||
|
||||
__all__ = ["InfonetReputationAdapter"]
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Parallel ``SignedWriteKind`` enum for Infonet economy events.
|
||||
|
||||
Why a parallel enum and not extending the legacy one:
|
||||
|
||||
The legacy ``services/mesh/mesh_signed_events.SignedWriteKind`` is
|
||||
imported in many places and changing it ripples through DM, gate, and
|
||||
oracle code that we are not modifying in Sprint 1. Instead we publish a
|
||||
parallel enum here for the new event types and rely on the hashchain
|
||||
adapter to translate or co-route as needed.
|
||||
|
||||
Sprint 7+ may collapse these two enums once the upgrade-hash governance
|
||||
is shipped and a coordinated cutover is possible.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class InfonetSignedWriteKind(str, Enum):
|
||||
# Reputation
|
||||
UPREP = "uprep"
|
||||
DOWNREP = "downrep"
|
||||
|
||||
# Markets / resolution-as-prediction
|
||||
PREDICTION_CREATE = "prediction_create"
|
||||
PREDICTION_PLACE = "prediction_place"
|
||||
TRUTH_STAKE_PLACE = "truth_stake_place"
|
||||
TRUTH_STAKE_RESOLVE = "truth_stake_resolve"
|
||||
MARKET_SNAPSHOT = "market_snapshot"
|
||||
EVIDENCE_SUBMIT = "evidence_submit"
|
||||
RESOLUTION_STAKE = "resolution_stake"
|
||||
BOOTSTRAP_RESOLUTION_VOTE = "bootstrap_resolution_vote"
|
||||
RESOLUTION_FINALIZE = "resolution_finalize"
|
||||
|
||||
# Disputes
|
||||
DISPUTE_OPEN = "dispute_open"
|
||||
DISPUTE_STAKE = "dispute_stake"
|
||||
DISPUTE_RESOLVE = "dispute_resolve"
|
||||
|
||||
# Gates (extend legacy GATE_CREATE / GATE_MESSAGE)
|
||||
GATE_ENTER = "gate_enter"
|
||||
GATE_EXIT = "gate_exit"
|
||||
GATE_LOCK = "gate_lock"
|
||||
|
||||
# Gate shutdown lifecycle
|
||||
GATE_SUSPEND_FILE = "gate_suspend_file"
|
||||
GATE_SUSPEND_VOTE = "gate_suspend_vote"
|
||||
GATE_SUSPEND_EXECUTE = "gate_suspend_execute"
|
||||
GATE_SHUTDOWN_FILE = "gate_shutdown_file"
|
||||
GATE_SHUTDOWN_VOTE = "gate_shutdown_vote"
|
||||
GATE_SHUTDOWN_EXECUTE = "gate_shutdown_execute"
|
||||
GATE_UNSUSPEND = "gate_unsuspend"
|
||||
GATE_SHUTDOWN_APPEAL_FILE = "gate_shutdown_appeal_file"
|
||||
GATE_SHUTDOWN_APPEAL_VOTE = "gate_shutdown_appeal_vote"
|
||||
GATE_SHUTDOWN_APPEAL_RESOLVE = "gate_shutdown_appeal_resolve"
|
||||
|
||||
# Governance
|
||||
PETITION_FILE = "petition_file"
|
||||
PETITION_SIGN = "petition_sign"
|
||||
PETITION_VOTE = "petition_vote"
|
||||
CHALLENGE_FILE = "challenge_file"
|
||||
CHALLENGE_VOTE = "challenge_vote"
|
||||
PETITION_EXECUTE = "petition_execute"
|
||||
|
||||
# Upgrade-hash governance
|
||||
UPGRADE_PROPOSE = "upgrade_propose"
|
||||
UPGRADE_SIGN = "upgrade_sign"
|
||||
UPGRADE_VOTE = "upgrade_vote"
|
||||
UPGRADE_CHALLENGE = "upgrade_challenge"
|
||||
UPGRADE_CHALLENGE_VOTE = "upgrade_challenge_vote"
|
||||
UPGRADE_SIGNAL_READY = "upgrade_signal_ready"
|
||||
UPGRADE_ACTIVATE = "upgrade_activate"
|
||||
|
||||
# Identity
|
||||
NODE_REGISTER = "node_register"
|
||||
IDENTITY_ROTATE = "identity_rotate"
|
||||
CITIZENSHIP_CLAIM = "citizenship_claim"
|
||||
|
||||
# Economy
|
||||
COIN_TRANSFER = "coin_transfer"
|
||||
COIN_MINT = "coin_mint"
|
||||
BOUNTY_CREATE = "bounty_create"
|
||||
BOUNTY_CLAIM = "bounty_claim"
|
||||
|
||||
# Content
|
||||
POST_CREATE = "post_create"
|
||||
POST_REPLY = "post_reply"
|
||||
|
||||
|
||||
INFONET_SIGNED_WRITE_KINDS: frozenset[InfonetSignedWriteKind] = frozenset(InfonetSignedWriteKind)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"INFONET_SIGNED_WRITE_KINDS",
|
||||
"InfonetSignedWriteKind",
|
||||
]
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Bootstrap mode — Argon2id PoW + eligibility + one-vote-per-node dedup.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.10 step 0.5.
|
||||
|
||||
Bootstrap mode replaces oracle-rep-weighted resolution with
|
||||
**eligible-node-one-vote** for the first ``bootstrap_market_count``
|
||||
(default 100) markets. Each eligible Heavy Node submits a
|
||||
``bootstrap_resolution_vote`` event with an Argon2id PoW solution.
|
||||
|
||||
Key Sprint 8 invariants:
|
||||
|
||||
- **Argon2id is Heavy-Node-only.** Light Nodes lack the ≥64 MB RAM
|
||||
required per computation. The PoW verifier does NOT run on Light
|
||||
Nodes.
|
||||
- **Salt = raw ``snapshot_event_hash`` bytes.** Hex-encoding or any
|
||||
reformatting causes a consensus fork. The salt MUST be the exact
|
||||
byte sequence of the snapshot event hash.
|
||||
- **Leading-zero check is on RAW output bytes, MSB first.** Different
|
||||
bit ordering causes a consensus fork.
|
||||
- **Identity age is measured against ``market.snapshot.frozen_at``,
|
||||
NOT against ``now``.** This is deterministic — every node computes
|
||||
the same eligibility from the same chain state. Prevents clock
|
||||
manipulation.
|
||||
- **One-vote-per-node tie-break is stateless.** Among multiple votes
|
||||
from the same node_id for the same market_id, the canonical vote is
|
||||
the one with the LOWEST LEXICOGRAPHICAL ``event_hash``. Every node
|
||||
selects the same canonical vote regardless of observation order.
|
||||
- **Anti-DoS funnel runs cheapest-first.** Schema → signature →
|
||||
identity age → predictor exclusion → phase + dedup → Argon2id.
|
||||
Argon2id is last because it's the most expensive.
|
||||
|
||||
Sprint 8 ships the eligibility + dedup + ramp pipeline in pure
|
||||
Python. ``verify_pow`` is a structural verifier that takes the
|
||||
already-computed hash output as input — it does NOT call Argon2id
|
||||
itself. Production callers wire this through ``privacy-core`` Rust.
|
||||
A future sprint will add the Rust binding; until then, tests
|
||||
synthesize valid hash outputs.
|
||||
"""
|
||||
|
||||
from services.infonet.bootstrap.argon2id import (
|
||||
canonical_pow_preimage,
|
||||
has_leading_zero_bits,
|
||||
verify_pow_structure,
|
||||
)
|
||||
from services.infonet.bootstrap.eligibility import (
|
||||
EligibilityDecision,
|
||||
is_identity_age_eligible,
|
||||
validate_bootstrap_eligibility,
|
||||
)
|
||||
from services.infonet.bootstrap.filter_funnel import (
|
||||
FunnelStage,
|
||||
run_filter_funnel,
|
||||
)
|
||||
from services.infonet.bootstrap.one_vote_dedup import (
|
||||
canonical_event_hash,
|
||||
deduplicate_votes,
|
||||
)
|
||||
from services.infonet.bootstrap.ramp import (
|
||||
ActiveFeatures,
|
||||
compute_active_features,
|
||||
network_node_count,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ActiveFeatures",
|
||||
"EligibilityDecision",
|
||||
"FunnelStage",
|
||||
"canonical_event_hash",
|
||||
"canonical_pow_preimage",
|
||||
"compute_active_features",
|
||||
"deduplicate_votes",
|
||||
"has_leading_zero_bits",
|
||||
"is_identity_age_eligible",
|
||||
"network_node_count",
|
||||
"run_filter_funnel",
|
||||
"validate_bootstrap_eligibility",
|
||||
"verify_pow_structure",
|
||||
]
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Argon2id canonicalization — preimage construction and leading-zero check.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.10 step 0.5
|
||||
+ the ``CONFIG['bootstrap_pow_argon2id_*']`` comment block.
|
||||
|
||||
Two consensus-critical pieces of canonicalization:
|
||||
|
||||
1. **Canonical preimage** — exact byte sequence the Argon2id call
|
||||
takes as `password`. UTF-8 encoded, "|"-delimited, no trailing
|
||||
delimiter. Format:
|
||||
|
||||
"bootstrap_resolution_vote" || protocol_version || node_id ||
|
||||
market_id || side || snapshot_event_hash || pow_nonce
|
||||
|
||||
The component order MUST match the spec exactly. Any deviation
|
||||
causes consensus fork.
|
||||
|
||||
2. **Leading-zero check** — operates on RAW Argon2id output bytes,
|
||||
MSB first (big-endian bit numbering). Difficulty N requires the
|
||||
first N bits of the 32-byte output to be zero. With difficulty=16
|
||||
that means the first 2 bytes are 0x00 0x00.
|
||||
|
||||
Sprint 8 does NOT execute Argon2id itself — the verifier here takes
|
||||
an already-computed hash bytes object as input. Production callers
|
||||
wire this through ``privacy-core`` Rust binding. A stub Python
|
||||
implementation is intentionally absent to avoid accidental drift
|
||||
between the Sprint 8 pure-Python path and the eventual Rust path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from services.infonet.config import CONFIG, IMMUTABLE_PRINCIPLES
|
||||
|
||||
|
||||
def canonical_pow_preimage(
|
||||
*,
|
||||
node_id: str,
|
||||
market_id: str,
|
||||
side: str,
|
||||
snapshot_event_hash: str,
|
||||
pow_nonce: int,
|
||||
protocol_version: str | None = None,
|
||||
) -> bytes:
|
||||
"""Build the canonical preimage for the Argon2id ``password`` input.
|
||||
|
||||
Returns UTF-8 bytes of ``"bootstrap_resolution_vote|<version>|<node>|
|
||||
<market>|<side>|<snapshot_hash>|<nonce>"`` with NO trailing delimiter.
|
||||
|
||||
``protocol_version`` defaults to ``IMMUTABLE_PRINCIPLES['protocol_version']``
|
||||
— it's pulled at call time so a hard-fork upgrade picks up the
|
||||
new value automatically. Pass an explicit value when computing
|
||||
against a hypothetical version (test scenarios).
|
||||
"""
|
||||
if not isinstance(node_id, str) or not node_id:
|
||||
raise ValueError("node_id must be a non-empty string")
|
||||
if not isinstance(market_id, str) or not market_id:
|
||||
raise ValueError("market_id must be a non-empty string")
|
||||
if side not in ("yes", "no"):
|
||||
raise ValueError("side must be 'yes' or 'no'")
|
||||
if not isinstance(snapshot_event_hash, str) or not snapshot_event_hash:
|
||||
raise ValueError("snapshot_event_hash must be a non-empty string")
|
||||
if not isinstance(pow_nonce, int) or isinstance(pow_nonce, bool) or pow_nonce < 0:
|
||||
raise ValueError("pow_nonce must be a non-negative int")
|
||||
pv = protocol_version if protocol_version is not None else IMMUTABLE_PRINCIPLES["protocol_version"]
|
||||
if not isinstance(pv, str) or not pv:
|
||||
raise ValueError("protocol_version must be a non-empty string")
|
||||
|
||||
parts = [
|
||||
"bootstrap_resolution_vote",
|
||||
pv,
|
||||
node_id,
|
||||
market_id,
|
||||
side,
|
||||
snapshot_event_hash,
|
||||
str(pow_nonce),
|
||||
]
|
||||
return "|".join(parts).encode("utf-8")
|
||||
|
||||
|
||||
def has_leading_zero_bits(raw_output: bytes, difficulty: int) -> bool:
|
||||
"""``True`` if the first ``difficulty`` bits of ``raw_output``
|
||||
are all zero.
|
||||
|
||||
Bit numbering: MSB first (big-endian). Byte order: as-is in the
|
||||
raw output. With difficulty=16, the first two bytes must be
|
||||
``\\x00\\x00``. With difficulty=4, the first byte must be in
|
||||
``\\x00``..``\\x0f``.
|
||||
"""
|
||||
if not isinstance(raw_output, (bytes, bytearray)):
|
||||
raise ValueError("raw_output must be bytes")
|
||||
if not isinstance(difficulty, int) or difficulty < 0:
|
||||
raise ValueError("difficulty must be a non-negative int")
|
||||
if difficulty == 0:
|
||||
return True
|
||||
|
||||
full_bytes, remaining_bits = divmod(difficulty, 8)
|
||||
if len(raw_output) < full_bytes + (1 if remaining_bits else 0):
|
||||
return False
|
||||
for i in range(full_bytes):
|
||||
if raw_output[i] != 0:
|
||||
return False
|
||||
if remaining_bits:
|
||||
# The next byte's top `remaining_bits` bits must be zero.
|
||||
next_byte = raw_output[full_bytes]
|
||||
# Mask of the top `remaining_bits` bits (MSB first).
|
||||
mask = ((0xFF << (8 - remaining_bits)) & 0xFF)
|
||||
if (next_byte & mask) != 0:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def verify_pow_structure(
|
||||
*,
|
||||
raw_output: bytes,
|
||||
difficulty: int | None = None,
|
||||
expected_output_len: int | None = None,
|
||||
) -> bool:
|
||||
"""Verify the Argon2id output's structural properties.
|
||||
|
||||
- Output length must match ``expected_output_len`` (default
|
||||
``CONFIG['bootstrap_pow_argon2id_output_len']``, fixed at 32).
|
||||
- Leading zero check passes for ``difficulty`` (default
|
||||
``CONFIG['bootstrap_pow_difficulty']``).
|
||||
|
||||
Does NOT verify that ``raw_output`` was actually produced by
|
||||
Argon2id from the canonical preimage — that's the caller's job
|
||||
via ``privacy-core`` Rust binding (or Python's ``argon2-cffi`` in
|
||||
test environments). Sprint 8 keeps the cryptographic-call layer
|
||||
as an external concern.
|
||||
"""
|
||||
if not isinstance(raw_output, (bytes, bytearray)):
|
||||
return False
|
||||
expected = expected_output_len if expected_output_len is not None else int(
|
||||
CONFIG["bootstrap_pow_argon2id_output_len"]
|
||||
)
|
||||
if len(raw_output) != expected:
|
||||
return False
|
||||
diff = difficulty if difficulty is not None else int(CONFIG["bootstrap_pow_difficulty"])
|
||||
return has_leading_zero_bits(raw_output, diff)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"canonical_pow_preimage",
|
||||
"has_leading_zero_bits",
|
||||
"verify_pow_structure",
|
||||
]
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Bootstrap eligibility — identity age + predictor exclusion.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.10 step 0.5
|
||||
(``is_bootstrap_eligible``).
|
||||
|
||||
Two gates:
|
||||
|
||||
1. **Identity age vs ``frozen_at`` (NOT ``now``).** Spec is explicit:
|
||||
|
||||
node.created_at + (bootstrap_min_identity_age_days * 86400)
|
||||
<= market.snapshot.frozen_at
|
||||
|
||||
Measuring against the frozen snapshot timestamp keeps eligibility
|
||||
deterministic — every node computes the same set from the same
|
||||
chain state. Measuring against ``now`` would make eligibility
|
||||
depend on local clock, which is a clock-manipulation attack
|
||||
surface.
|
||||
|
||||
2. **Predictor exclusion.** Same as normal resolution:
|
||||
``frozen_predictor_ids ∪ rotation_descendants(frozen_predictor_ids)``.
|
||||
Reuses ``services.infonet.markets.resolution.excluded_predictor_ids``
|
||||
(Sprint 4) — single source of truth.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
from services.infonet.markets.resolution import excluded_predictor_ids
|
||||
from services.infonet.markets.snapshot import find_snapshot
|
||||
|
||||
|
||||
_SECONDS_PER_DAY = 86400.0
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
def _node_created_at(node_id: str, chain: Iterable[dict[str, Any]]) -> float | None:
|
||||
"""First chain appearance of ``node_id`` — used as a proxy for
|
||||
``node.created_at``. Per RULES §2.1: "Timestamp of first appearance
|
||||
on chain". A ``node_register`` event is preferred when present;
|
||||
otherwise the earliest event signed by ``node_id``.
|
||||
"""
|
||||
earliest_register: float | None = None
|
||||
earliest_any: float | None = None
|
||||
for ev in chain:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
author = ev.get("node_id")
|
||||
if author != node_id:
|
||||
continue
|
||||
try:
|
||||
ts = float(ev.get("timestamp") or 0.0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if ev.get("event_type") == "node_register":
|
||||
if earliest_register is None or ts < earliest_register:
|
||||
earliest_register = ts
|
||||
if earliest_any is None or ts < earliest_any:
|
||||
earliest_any = ts
|
||||
return earliest_register if earliest_register is not None else earliest_any
|
||||
|
||||
|
||||
def is_identity_age_eligible(
|
||||
node_id: str,
|
||||
market_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
min_age_days: float | None = None,
|
||||
) -> bool:
|
||||
"""``True`` iff
|
||||
``node.created_at + min_age_days * 86400 <= market.snapshot.frozen_at``.
|
||||
|
||||
Returns ``False`` if the snapshot doesn't exist yet, the node has
|
||||
no chain history, or the timing condition fails.
|
||||
"""
|
||||
chain_list = list(chain)
|
||||
snapshot = find_snapshot(market_id, chain_list)
|
||||
if snapshot is None:
|
||||
return False
|
||||
try:
|
||||
frozen_at = float(snapshot.get("frozen_at") or 0.0)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
created_at = _node_created_at(node_id, chain_list)
|
||||
if created_at is None:
|
||||
return False
|
||||
age_days = float(min_age_days if min_age_days is not None
|
||||
else CONFIG["bootstrap_min_identity_age_days"])
|
||||
threshold_ts = created_at + age_days * _SECONDS_PER_DAY
|
||||
return threshold_ts <= frozen_at
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EligibilityDecision:
|
||||
eligible: bool
|
||||
reason: str
|
||||
|
||||
|
||||
def validate_bootstrap_eligibility(
|
||||
node_id: str,
|
||||
market_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> EligibilityDecision:
|
||||
"""Combine identity-age + predictor-exclusion checks.
|
||||
|
||||
Used by the Sprint 8 anti-DoS funnel and by the bootstrap
|
||||
resolution path itself.
|
||||
"""
|
||||
chain_list = list(chain)
|
||||
if find_snapshot(market_id, chain_list) is None:
|
||||
return EligibilityDecision(False, "snapshot_missing")
|
||||
if not is_identity_age_eligible(node_id, market_id, chain_list):
|
||||
return EligibilityDecision(False, "identity_age_too_young")
|
||||
if node_id in excluded_predictor_ids(market_id, chain_list):
|
||||
return EligibilityDecision(False, "predictor_excluded")
|
||||
return EligibilityDecision(True, "ok")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EligibilityDecision",
|
||||
"is_identity_age_eligible",
|
||||
"validate_bootstrap_eligibility",
|
||||
]
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Anti-DoS filter funnel — cheapest-first validator chain.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.10 step 0.5
|
||||
"Anti-DoS filter funnel (validation order for bootstrap_resolution_vote)".
|
||||
|
||||
Validation order (each stage short-circuits to reject):
|
||||
|
||||
1. Schema — format / required fields / enum sanity (free)
|
||||
2. Signature — Ed25519 verify (~µs)
|
||||
3. Identity age — vs snapshot.frozen_at (chain lookup)
|
||||
4. Predictor — vs frozen_predictor_ids ∪ rotation_descendants
|
||||
5. Phase + dedup
|
||||
6. Argon2id PoW — most expensive (~64MB allocation + hash)
|
||||
|
||||
Why ordering matters: an attacker flooding malformed events should
|
||||
never trigger the Argon2id work. Schema rejection happens first
|
||||
(microseconds), so the funnel discards cheap-to-reject inputs cheap.
|
||||
|
||||
Sprint 8 ships the funnel as a list of ``FunnelStage`` callables.
|
||||
Production callers compose them in order; each stage returns
|
||||
``(accepted, reason)``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
_StageFn = Callable[[dict[str, Any]], tuple[bool, str]]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FunnelStage:
|
||||
name: str
|
||||
check: _StageFn
|
||||
cost_tier: int
|
||||
"""Cost ranking 1=cheapest, 6=most expensive. Used by tests to
|
||||
confirm the stages are in the spec's ordering."""
|
||||
|
||||
|
||||
def run_filter_funnel(
|
||||
event: dict[str, Any],
|
||||
stages: list[FunnelStage],
|
||||
) -> tuple[bool, str]:
|
||||
"""Run ``stages`` in order; return on the first failure.
|
||||
|
||||
Returns ``(True, "ok")`` if every stage passes, otherwise
|
||||
``(False, "<stage>: <reason>")`` with the failing stage's name
|
||||
and reason. The stage's own ``cost_tier`` is included in the
|
||||
failing diagnostic so monitoring can spot when expensive stages
|
||||
are doing the work cheap stages should have caught.
|
||||
"""
|
||||
if not isinstance(event, dict):
|
||||
return False, "schema: event must be an object"
|
||||
seen_tiers: list[int] = []
|
||||
for stage in stages:
|
||||
if seen_tiers and stage.cost_tier < max(seen_tiers):
|
||||
# Sprint 8 invariant: tiers must be monotonically
|
||||
# non-decreasing. A misordered funnel is a developer
|
||||
# error, not an attacker input — fail loudly.
|
||||
raise ValueError(
|
||||
f"filter funnel out of order: stage {stage.name} "
|
||||
f"has cost_tier={stage.cost_tier} after a higher tier"
|
||||
)
|
||||
seen_tiers.append(stage.cost_tier)
|
||||
ok, reason = stage.check(event)
|
||||
if not ok:
|
||||
return False, f"{stage.name}: {reason}"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FunnelStage",
|
||||
"run_filter_funnel",
|
||||
]
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Stateless one-vote-per-node dedup.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.10 step 0.5
|
||||
("Phase valid + one-vote-per-node (stateless duplicate resolution)").
|
||||
|
||||
The protocol allows a node to submit only one
|
||||
``bootstrap_resolution_vote`` per market_id. If duplicates appear
|
||||
(retries, network split + heal, malicious flooding), the canonical
|
||||
choice is **the vote with the lowest lexicographical event_hash**.
|
||||
|
||||
Key property: this is **stateless and order-independent**. Every node
|
||||
computes the same canonical vote regardless of which duplicate they
|
||||
saw first. No "last-write-wins" or "first-write-wins" — just the
|
||||
hash comparison.
|
||||
|
||||
``event_hash = SHA-256(canonical_serialize(event))`` — must include
|
||||
signature, payload, and metadata so two events with different
|
||||
payloads produce different hashes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
def canonical_event_hash(event: dict[str, Any]) -> str:
|
||||
"""SHA-256 of the canonically-serialized event.
|
||||
|
||||
Canonicalization: sorted keys, compact separators, UTF-8.
|
||||
Includes every field on the event dict — payload, signature (if
|
||||
present), node_id, timestamp, sequence, event_type. Different
|
||||
inputs always produce different hashes.
|
||||
"""
|
||||
encoded = json.dumps(event, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
||||
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
def deduplicate_votes(
|
||||
market_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return the canonical set of ``bootstrap_resolution_vote`` events
|
||||
for ``market_id`` — at most one per ``node_id``, with the lowest
|
||||
lexicographical ``canonical_event_hash`` chosen on collision.
|
||||
|
||||
The returned list is sorted by ``(node_id, event_hash)`` so the
|
||||
output is deterministic for any chain ordering.
|
||||
"""
|
||||
candidates_per_node: dict[str, list[tuple[str, dict[str, Any]]]] = {}
|
||||
for ev in chain:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
if ev.get("event_type") != "bootstrap_resolution_vote":
|
||||
continue
|
||||
if _payload(ev).get("market_id") != market_id:
|
||||
continue
|
||||
node = ev.get("node_id")
|
||||
if not isinstance(node, str) or not node:
|
||||
continue
|
||||
h = canonical_event_hash(ev)
|
||||
candidates_per_node.setdefault(node, []).append((h, ev))
|
||||
|
||||
canonical: list[dict[str, Any]] = []
|
||||
for node, candidates in candidates_per_node.items():
|
||||
# Lowest lexicographical event_hash wins. Stable secondary
|
||||
# sort by sequence to make the choice deterministic for
|
||||
# any duplicate hash (which would itself be a SHA-256
|
||||
# collision — so academically impossible).
|
||||
candidates.sort(key=lambda c: (c[0], int(c[1].get("sequence") or 0)))
|
||||
canonical.append(candidates[0][1])
|
||||
canonical.sort(key=lambda e: (e.get("node_id") or "", canonical_event_hash(e)))
|
||||
return canonical
|
||||
|
||||
|
||||
__all__ = [
|
||||
"canonical_event_hash",
|
||||
"deduplicate_votes",
|
||||
]
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Soft feature activation ramp — node-count milestones.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §1.2
|
||||
(``CONFIG['bootstrap_threshold']`` comment) + the spec's general
|
||||
"phase activation by network size" theme.
|
||||
|
||||
The protocol activates features in stages as the network grows. The
|
||||
canonical milestones are 1k / 2k / 5k / 10k node count, but the
|
||||
specific thresholds and which features they unlock are a Sprint 8+
|
||||
design choice that's expected to evolve via governance.
|
||||
|
||||
Sprint 8 ships:
|
||||
|
||||
- ``network_node_count(chain)`` — distinct ``node_register`` events
|
||||
on the chain.
|
||||
- ``compute_active_features(chain)`` — returns an ``ActiveFeatures``
|
||||
flag set indicating which protocol features are currently active.
|
||||
|
||||
Today's bindings:
|
||||
|
||||
- ``bootstrap_resolution_active`` — True while node count is below
|
||||
``bootstrap_threshold`` (default 1000). Bootstrap-mode markets use
|
||||
eligible-node-one-vote resolution.
|
||||
- ``staked_resolution_active`` — True once node count crosses 1k.
|
||||
Oracle-rep-weighted resolution staking is the primary mechanism.
|
||||
- ``governance_petitions_active`` — True at 2k+. Petitions can be
|
||||
filed.
|
||||
- ``upgrade_governance_active`` — True at 5k+. Upgrade-hash
|
||||
governance is unlocked.
|
||||
- ``commoncoin_active`` — True at 10k+. CommonCoin minting starts.
|
||||
|
||||
These bindings are intentionally simple — production wiring will
|
||||
read them via governance petitions that adjust ``bootstrap_threshold``
|
||||
and the milestones themselves.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
|
||||
|
||||
def network_node_count(chain: Iterable[dict[str, Any]]) -> int:
|
||||
"""Distinct nodes that have appeared on the chain.
|
||||
|
||||
Counted as: distinct ``node_id`` from ``node_register`` events.
|
||||
If no ``node_register`` events exist on the chain (e.g. test
|
||||
chains that only synthesize markets/predictions), falls back to
|
||||
distinct authoring nodes across all events. Production chains
|
||||
will have the registers.
|
||||
"""
|
||||
registered: set[str] = set()
|
||||
fallback: set[str] = set()
|
||||
for ev in chain:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
node = ev.get("node_id")
|
||||
if not isinstance(node, str) or not node:
|
||||
continue
|
||||
fallback.add(node)
|
||||
if ev.get("event_type") == "node_register":
|
||||
registered.add(node)
|
||||
return len(registered) if registered else len(fallback)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ActiveFeatures:
|
||||
bootstrap_resolution_active: bool
|
||||
staked_resolution_active: bool
|
||||
governance_petitions_active: bool
|
||||
upgrade_governance_active: bool
|
||||
commoncoin_active: bool
|
||||
node_count: int
|
||||
|
||||
|
||||
# Milestone thresholds promoted to CONFIG 2026-04-28 (Sprint 8 polish).
|
||||
# Governance can now tune them via petition; the cross-field invariant
|
||||
# in config.py enforces strict ascending order across the four tiers.
|
||||
|
||||
|
||||
def compute_active_features(chain: Iterable[dict[str, Any]]) -> ActiveFeatures:
|
||||
chain_list = [e for e in chain if isinstance(e, dict)]
|
||||
n = network_node_count(chain_list)
|
||||
bootstrap_threshold = int(CONFIG["bootstrap_threshold"])
|
||||
return ActiveFeatures(
|
||||
# Bootstrap resolution is active until the network crosses the
|
||||
# bootstrap_threshold. Once crossed, it's still allowed for
|
||||
# bootstrap-indexed markets, but new markets default to
|
||||
# staked resolution.
|
||||
bootstrap_resolution_active=n < bootstrap_threshold,
|
||||
staked_resolution_active=n >= int(CONFIG["ramp_staked_resolution_threshold"]),
|
||||
governance_petitions_active=n >= int(CONFIG["ramp_petitions_threshold"]),
|
||||
upgrade_governance_active=n >= int(CONFIG["ramp_upgrade_threshold"]),
|
||||
commoncoin_active=n >= int(CONFIG["ramp_commoncoin_threshold"]),
|
||||
node_count=n,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ActiveFeatures",
|
||||
"compute_active_features",
|
||||
"network_node_count",
|
||||
]
|
||||
@@ -0,0 +1,519 @@
|
||||
"""Constitutional + governable parameters for the Infonet economy.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §1.
|
||||
|
||||
- ``IMMUTABLE_PRINCIPLES`` — constitutional, exposed as a ``MappingProxyType``.
|
||||
Mutation attempts raise ``TypeError`` at the language level. New keys can
|
||||
only be added through upgrade-hash governance (Sprint 7) which is itself
|
||||
governed by these principles — i.e. a hard fork.
|
||||
|
||||
- ``CONFIG`` — amendable parameters. Live (mutable) dict; all writes go
|
||||
through ``validate_petition_value`` first. The dict itself is a
|
||||
module-level singleton — the governance DSL executor (Sprint 7) is the
|
||||
only intended writer in production. Tests must use
|
||||
``reset_config_for_tests`` to restore baseline.
|
||||
|
||||
- ``CONFIG_SCHEMA`` — per-key bounds and types. Itself an immutable
|
||||
``MappingProxyType``. New schema entries require a hard fork (same flow
|
||||
as ``IMMUTABLE_PRINCIPLES``).
|
||||
|
||||
- ``CROSS_FIELD_INVARIANTS`` — ordered-pair invariants checked AFTER all
|
||||
updates in a ``BATCH_UPDATE_PARAMS``. Spec note: supermajority must
|
||||
always exceed quorum, etc.
|
||||
|
||||
This file is read by every subsequent sprint. Adding a CONFIG key without
|
||||
adding a matching CONFIG_SCHEMA entry is a Sprint 1 invariant violation
|
||||
and is asserted by the tests.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from types import MappingProxyType
|
||||
from typing import Any
|
||||
|
||||
|
||||
class InvalidPetition(ValueError):
|
||||
"""Raised by ``validate_petition_value`` and the governance DSL executor.
|
||||
|
||||
Signals that a proposed CONFIG mutation is rejected by the schema or by
|
||||
a cross-field invariant. The DSL executor (Sprint 7) catches this and
|
||||
rolls back the petition — never partially applies.
|
||||
"""
|
||||
|
||||
|
||||
# ─── Constitutional principles ───────────────────────────────────────────
|
||||
# Immutable. Mutation attempts raise TypeError at the language level
|
||||
# because MappingProxyType is read-only.
|
||||
#
|
||||
# RULES_SKELETON.md §1.1 — adding a key here is a hard fork.
|
||||
|
||||
IMMUTABLE_PRINCIPLES: MappingProxyType = MappingProxyType({
|
||||
"oracle_rep_source": "predictions_only",
|
||||
"hashchain_append_only": True,
|
||||
"audit_public": True,
|
||||
"identity_permissionless": True,
|
||||
"signature_required": True,
|
||||
"redemption_path_exists": True,
|
||||
"coin_governance_firewall": True,
|
||||
"protocol_version": "0.1.0",
|
||||
})
|
||||
|
||||
|
||||
# ─── Amendable parameters ────────────────────────────────────────────────
|
||||
# RULES_SKELETON.md §1.2.
|
||||
# Mutable dict. Production writes only via the Sprint 7 governance DSL
|
||||
# executor which calls validate_petition_value first.
|
||||
|
||||
_BASELINE_CONFIG: dict[str, Any] = {
|
||||
# ── Reputation ──
|
||||
"vote_decay_days": 90,
|
||||
"min_rep_to_vote": 3,
|
||||
"min_rep_to_create_gate": 10,
|
||||
"bootstrap_threshold": 1000,
|
||||
"weekly_vote_base": 5,
|
||||
"weekly_vote_per_oracle": 10,
|
||||
"daily_vote_limit_per_target": 1,
|
||||
|
||||
# ── Oracle Rep ──
|
||||
"oracle_min_earned": 0.01,
|
||||
"farming_soft_threshold": 0.60,
|
||||
"farming_hard_threshold": 0.80,
|
||||
"farming_easy_bet_cutoff": 0.80,
|
||||
"subjective_oracle_rep_mint": False,
|
||||
|
||||
# ── Market Liquidity ──
|
||||
"min_market_participants": 5,
|
||||
"min_market_total_stake": 10.0,
|
||||
|
||||
# ── Resolution Phase ──
|
||||
"evidence_window_hours": 48,
|
||||
"resolution_window_hours": 72,
|
||||
"evidence_bond_cost": 2.0,
|
||||
"evidence_first_bonus": 0.5,
|
||||
"resolution_supermajority": 0.75,
|
||||
"min_resolution_stake_total": 20.0,
|
||||
"resolution_loser_burn_pct": 0.02,
|
||||
"data_unavailable_threshold": 0.33,
|
||||
"resolution_stalemate_burn_pct": 0.02,
|
||||
|
||||
# ── Governance Decay ──
|
||||
"governance_decay_days": 90,
|
||||
"governance_decay_factor": 0.50,
|
||||
|
||||
# ── Time Validity ──
|
||||
"max_future_event_drift_sec": 300,
|
||||
"phase_boundary_stale_reject": True,
|
||||
|
||||
# ── Identity Rotation ──
|
||||
"rotation_blocked_during_stakes": True,
|
||||
|
||||
# ── Anti-Gaming ──
|
||||
"vcs_min_weight": 0.10,
|
||||
"clustering_min_weight": 0.20,
|
||||
"temporal_burst_window_sec": 300,
|
||||
"temporal_burst_min_upreps": 5,
|
||||
"progressive_penalty_base": 1.0,
|
||||
# Common-rep base formula multiplier (RULES §3.3). Promoted from
|
||||
# Sprint 2's module-private constant 2026-04-28 so governance can
|
||||
# tune the default common-rep payout per uprep.
|
||||
"common_rep_weight_factor": 0.10,
|
||||
# Progressive-penalty trigger threshold — average correlation
|
||||
# score above which the whale-deterrence multiplier kicks in
|
||||
# (Sprint 3 polish 2026-04-28). 0.0 = disabled (Sprint 3 default
|
||||
# behavior preserved).
|
||||
"progressive_penalty_threshold": 0.0,
|
||||
|
||||
# ── Gates ──
|
||||
"gate_ratification_rep": 50,
|
||||
"gate_lock_cost_per_member": 10,
|
||||
"gate_lock_min_members": 5,
|
||||
"gate_creation_rate_limit": 5,
|
||||
|
||||
# ── Truth Stakes ──
|
||||
"truth_stake_min_days": 1,
|
||||
"truth_stake_max_days": 7,
|
||||
"truth_stake_grace_hours": 24,
|
||||
"truth_stake_max_extensions": 3,
|
||||
"truth_stake_tie_burn_pct": 0.20,
|
||||
"truth_stake_self_stake": False,
|
||||
|
||||
# ── Dispute Resolution ──
|
||||
"dispute_window_days": 7,
|
||||
"dispute_common_rep_stakeable": True,
|
||||
|
||||
# ── CommonCoin ──
|
||||
"monthly_mint_amount": 100000,
|
||||
"ubi_share_pct": 0.50,
|
||||
"oracle_dividend_pct": 0.50,
|
||||
"citizenship_sacrifice_cost": 10,
|
||||
"year1_max_coins_per_node": 10000,
|
||||
|
||||
# ── Governance ──
|
||||
"petition_filing_cost": 15,
|
||||
"petition_signature_threshold": 0.25,
|
||||
"petition_signature_window_days": 14,
|
||||
"petition_vote_window_days": 7,
|
||||
"petition_supermajority": 0.67,
|
||||
"petition_quorum": 0.30,
|
||||
"challenge_filing_cost": 25,
|
||||
"challenge_window_hours": 48,
|
||||
|
||||
# ── Upgrade-Hash Governance ──
|
||||
"upgrade_filing_cost": 25,
|
||||
"upgrade_signature_threshold": 0.25,
|
||||
"upgrade_signature_window_days": 14,
|
||||
"upgrade_vote_window_days": 14,
|
||||
"upgrade_supermajority": 0.80,
|
||||
"upgrade_quorum": 0.40,
|
||||
"upgrade_activation_threshold": 0.67,
|
||||
"upgrade_activation_window_days": 30,
|
||||
"upgrade_challenge_window_hours": 48,
|
||||
|
||||
# ── Gate Shutdown ──
|
||||
"gate_suspend_filing_cost": 15,
|
||||
"gate_shutdown_filing_cost": 25,
|
||||
"gate_suspend_supermajority": 0.67,
|
||||
"gate_suspend_locked_supermajority": 0.75,
|
||||
"gate_shutdown_supermajority": 0.75,
|
||||
"gate_shutdown_locked_supermajority": 0.80,
|
||||
"gate_shutdown_quorum": 0.30,
|
||||
"gate_suspend_duration_days": 30,
|
||||
"gate_shutdown_execution_delay_days": 7,
|
||||
"gate_shutdown_cooldown_days": 90,
|
||||
"gate_shutdown_fail_penalty_days": 30,
|
||||
"gate_shutdown_appeal_filing_cost": 20,
|
||||
"gate_shutdown_appeal_window_hours": 48,
|
||||
"gate_shutdown_appeal_vote_window_days": 7,
|
||||
"gate_shutdown_appeal_supermajority": 0.67,
|
||||
"gate_shutdown_appeal_locked_supermajority": 0.75,
|
||||
"gate_shutdown_appeal_quorum": 0.30,
|
||||
|
||||
# ── Market Creation ──
|
||||
"market_creation_bond": 3,
|
||||
"market_creation_bond_return_threshold": 5,
|
||||
|
||||
# ── Bootstrap ──
|
||||
"bootstrap_market_count": 100,
|
||||
"bootstrap_evidence_bond_cost": 0,
|
||||
"bootstrap_resolution_mode": "eligible_node_one_vote",
|
||||
"bootstrap_resolution_supermajority": 0.75,
|
||||
"bootstrap_min_identity_age_days": 3,
|
||||
"bootstrap_pow_algorithm": "argon2id",
|
||||
"bootstrap_pow_argon2id_version": 0x13,
|
||||
"bootstrap_pow_argon2id_m": 65536,
|
||||
"bootstrap_pow_argon2id_t": 3,
|
||||
"bootstrap_pow_argon2id_p": 1,
|
||||
"bootstrap_pow_argon2id_output_len": 32,
|
||||
"bootstrap_pow_difficulty": 16,
|
||||
|
||||
# ── Ramp milestones (Sprint 8 polish 2026-04-28) ──
|
||||
# Network-size thresholds at which features activate. Promoted
|
||||
# from Sprint 8 hardcoded constants so governance can tune them.
|
||||
# Values denote the minimum distinct-node count required.
|
||||
"ramp_staked_resolution_threshold": 1000,
|
||||
"ramp_petitions_threshold": 2000,
|
||||
"ramp_upgrade_threshold": 5000,
|
||||
"ramp_commoncoin_threshold": 10000,
|
||||
}
|
||||
|
||||
|
||||
CONFIG: dict[str, Any] = deepcopy(_BASELINE_CONFIG)
|
||||
|
||||
|
||||
def reset_config_for_tests() -> None:
|
||||
"""Restore CONFIG to the pre-petition baseline. Tests only.
|
||||
|
||||
Used by the autouse fixture in ``services/infonet/tests/conftest.py`` so
|
||||
that one test mutating CONFIG (via a simulated petition execution)
|
||||
cannot leak state into the next test.
|
||||
"""
|
||||
CONFIG.clear()
|
||||
CONFIG.update(deepcopy(_BASELINE_CONFIG))
|
||||
|
||||
|
||||
# ─── CONFIG schema (per-key bounds) ──────────────────────────────────────
|
||||
# RULES_SKELETON.md §1.3.
|
||||
# Itself an immutable structure — new keys require upgrade-hash governance
|
||||
# (a hard fork). validate_petition_value rejects any key not present here.
|
||||
|
||||
_SCHEMA_TYPES = {
|
||||
"int": (int,),
|
||||
"float": (int, float),
|
||||
"bool": (bool,),
|
||||
"str": (str,),
|
||||
}
|
||||
|
||||
_CONFIG_SCHEMA_BACKING: dict[str, MappingProxyType] = {
|
||||
# ── Reputation ──
|
||||
"vote_decay_days": MappingProxyType({"type": "int", "min": 7, "max": 365}),
|
||||
"min_rep_to_vote": MappingProxyType({"type": "int", "min": 0, "max": 100}),
|
||||
"min_rep_to_create_gate": MappingProxyType({"type": "int", "min": 1, "max": 1000}),
|
||||
"bootstrap_threshold": MappingProxyType({"type": "int", "min": 100, "max": 100000}),
|
||||
"weekly_vote_base": MappingProxyType({"type": "int", "min": 1, "max": 100}),
|
||||
"weekly_vote_per_oracle": MappingProxyType({"type": "int", "min": 1, "max": 1000}),
|
||||
"daily_vote_limit_per_target": MappingProxyType({"type": "int", "min": 1, "max": 10}),
|
||||
|
||||
# ── Oracle Rep ──
|
||||
"oracle_min_earned": MappingProxyType({"type": "float", "min": 0.001, "max": 1.0}),
|
||||
"farming_soft_threshold": MappingProxyType({"type": "float", "min": 0.10, "max": 0.95}),
|
||||
"farming_hard_threshold": MappingProxyType({"type": "float", "min": 0.20, "max": 0.99}),
|
||||
"farming_easy_bet_cutoff": MappingProxyType({"type": "float", "min": 0.50, "max": 0.99}),
|
||||
"subjective_oracle_rep_mint": MappingProxyType({"type": "bool"}),
|
||||
|
||||
# ── Market Liquidity ──
|
||||
"min_market_participants": MappingProxyType({"type": "int", "min": 2, "max": 100}),
|
||||
"min_market_total_stake": MappingProxyType({"type": "float", "min": 1.0, "max": 1000.0}),
|
||||
|
||||
# ── Resolution ──
|
||||
"evidence_window_hours": MappingProxyType({"type": "int", "min": 12, "max": 168}),
|
||||
"resolution_window_hours": MappingProxyType({"type": "int", "min": 24, "max": 336}),
|
||||
"evidence_bond_cost": MappingProxyType({"type": "float", "min": 0.5, "max": 50.0}),
|
||||
"evidence_first_bonus": MappingProxyType({"type": "float", "min": 0.0, "max": 10.0}),
|
||||
"resolution_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.95}),
|
||||
"min_resolution_stake_total": MappingProxyType({"type": "float", "min": 5.0, "max": 500.0}),
|
||||
"resolution_loser_burn_pct": MappingProxyType({"type": "float", "min": 0.0, "max": 0.10}),
|
||||
"data_unavailable_threshold": MappingProxyType({"type": "float", "min": 0.10, "max": 0.50}),
|
||||
"resolution_stalemate_burn_pct": MappingProxyType({"type": "float", "min": 0.0, "max": 0.10}),
|
||||
|
||||
# ── Governance Decay ──
|
||||
"governance_decay_days": MappingProxyType({"type": "int", "min": 7, "max": 365}),
|
||||
"governance_decay_factor": MappingProxyType({"type": "float", "min": 0.10, "max": 0.99}),
|
||||
|
||||
# ── Time Validity ──
|
||||
"max_future_event_drift_sec": MappingProxyType({"type": "int", "min": 30, "max": 3600}),
|
||||
"phase_boundary_stale_reject": MappingProxyType({"type": "bool"}),
|
||||
|
||||
# ── Identity Rotation ──
|
||||
"rotation_blocked_during_stakes": MappingProxyType({"type": "bool"}),
|
||||
|
||||
# ── Anti-Gaming ──
|
||||
"vcs_min_weight": MappingProxyType({"type": "float", "min": 0.0, "max": 1.0}),
|
||||
"clustering_min_weight": MappingProxyType({"type": "float", "min": 0.0, "max": 1.0}),
|
||||
"temporal_burst_window_sec": MappingProxyType({"type": "int", "min": 30, "max": 3600}),
|
||||
"temporal_burst_min_upreps": MappingProxyType({"type": "int", "min": 2, "max": 100}),
|
||||
"progressive_penalty_base": MappingProxyType({"type": "float", "min": 0.1, "max": 100.0}),
|
||||
"common_rep_weight_factor": MappingProxyType({"type": "float", "min": 0.0, "max": 1.0}),
|
||||
"progressive_penalty_threshold": MappingProxyType({"type": "float", "min": 0.0, "max": 1.0}),
|
||||
|
||||
# ── Gates ──
|
||||
"gate_ratification_rep": MappingProxyType({"type": "int", "min": 1, "max": 10000}),
|
||||
"gate_lock_cost_per_member": MappingProxyType({"type": "int", "min": 1, "max": 1000}),
|
||||
"gate_lock_min_members": MappingProxyType({"type": "int", "min": 2, "max": 1000}),
|
||||
"gate_creation_rate_limit": MappingProxyType({"type": "int", "min": 1, "max": 100}),
|
||||
|
||||
# ── Truth Stakes ──
|
||||
"truth_stake_min_days": MappingProxyType({"type": "int", "min": 1, "max": 30}),
|
||||
"truth_stake_max_days": MappingProxyType({"type": "int", "min": 1, "max": 90}),
|
||||
"truth_stake_grace_hours": MappingProxyType({"type": "int", "min": 1, "max": 168}),
|
||||
"truth_stake_max_extensions": MappingProxyType({"type": "int", "min": 0, "max": 10}),
|
||||
"truth_stake_tie_burn_pct": MappingProxyType({"type": "float", "min": 0.0, "max": 0.50}),
|
||||
"truth_stake_self_stake": MappingProxyType({"type": "bool"}),
|
||||
|
||||
# ── Dispute Resolution ──
|
||||
"dispute_window_days": MappingProxyType({"type": "int", "min": 1, "max": 30}),
|
||||
"dispute_common_rep_stakeable": MappingProxyType({"type": "bool"}),
|
||||
|
||||
# ── CommonCoin ──
|
||||
"monthly_mint_amount": MappingProxyType({"type": "int", "min": 1, "max": 1_000_000_000}),
|
||||
"ubi_share_pct": MappingProxyType({"type": "float", "min": 0.0, "max": 1.0}),
|
||||
"oracle_dividend_pct": MappingProxyType({"type": "float", "min": 0.0, "max": 1.0}),
|
||||
"citizenship_sacrifice_cost": MappingProxyType({"type": "int", "min": 1, "max": 1000}),
|
||||
"year1_max_coins_per_node": MappingProxyType({"type": "int", "min": 1, "max": 1_000_000_000}),
|
||||
|
||||
# ── Governance ──
|
||||
"petition_filing_cost": MappingProxyType({"type": "int", "min": 1, "max": 100}),
|
||||
"petition_signature_threshold": MappingProxyType({"type": "float", "min": 0.05, "max": 0.50}),
|
||||
"petition_signature_window_days": MappingProxyType({"type": "int", "min": 1, "max": 60}),
|
||||
"petition_vote_window_days": MappingProxyType({"type": "int", "min": 1, "max": 30}),
|
||||
"petition_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.95}),
|
||||
"petition_quorum": MappingProxyType({"type": "float", "min": 0.10, "max": 0.80}),
|
||||
"challenge_filing_cost": MappingProxyType({"type": "int", "min": 1, "max": 200}),
|
||||
"challenge_window_hours": MappingProxyType({"type": "int", "min": 12, "max": 168}),
|
||||
|
||||
# ── Upgrade-Hash Governance ──
|
||||
"upgrade_filing_cost": MappingProxyType({"type": "int", "min": 1, "max": 200}),
|
||||
"upgrade_signature_threshold": MappingProxyType({"type": "float", "min": 0.05, "max": 0.50}),
|
||||
"upgrade_signature_window_days": MappingProxyType({"type": "int", "min": 1, "max": 60}),
|
||||
"upgrade_vote_window_days": MappingProxyType({"type": "int", "min": 1, "max": 60}),
|
||||
"upgrade_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.99}),
|
||||
"upgrade_quorum": MappingProxyType({"type": "float", "min": 0.10, "max": 0.95}),
|
||||
"upgrade_activation_threshold": MappingProxyType({"type": "float", "min": 0.51, "max": 0.99}),
|
||||
"upgrade_activation_window_days": MappingProxyType({"type": "int", "min": 1, "max": 90}),
|
||||
"upgrade_challenge_window_hours": MappingProxyType({"type": "int", "min": 12, "max": 168}),
|
||||
|
||||
# ── Gate Shutdown ──
|
||||
"gate_suspend_filing_cost": MappingProxyType({"type": "int", "min": 1, "max": 200}),
|
||||
"gate_shutdown_filing_cost": MappingProxyType({"type": "int", "min": 1, "max": 200}),
|
||||
"gate_suspend_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.95}),
|
||||
"gate_suspend_locked_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.95}),
|
||||
"gate_shutdown_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.99}),
|
||||
"gate_shutdown_locked_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.99}),
|
||||
"gate_shutdown_quorum": MappingProxyType({"type": "float", "min": 0.10, "max": 0.80}),
|
||||
"gate_suspend_duration_days": MappingProxyType({"type": "int", "min": 1, "max": 365}),
|
||||
"gate_shutdown_execution_delay_days": MappingProxyType({"type": "int", "min": 1, "max": 90}),
|
||||
"gate_shutdown_cooldown_days": MappingProxyType({"type": "int", "min": 7, "max": 365}),
|
||||
"gate_shutdown_fail_penalty_days": MappingProxyType({"type": "int", "min": 0, "max": 365}),
|
||||
"gate_shutdown_appeal_filing_cost": MappingProxyType({"type": "int", "min": 1, "max": 200}),
|
||||
"gate_shutdown_appeal_window_hours": MappingProxyType({"type": "int", "min": 12, "max": 168}),
|
||||
"gate_shutdown_appeal_vote_window_days": MappingProxyType({"type": "int", "min": 1, "max": 30}),
|
||||
"gate_shutdown_appeal_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.95}),
|
||||
"gate_shutdown_appeal_locked_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.95}),
|
||||
"gate_shutdown_appeal_quorum": MappingProxyType({"type": "float", "min": 0.10, "max": 0.80}),
|
||||
|
||||
# ── Market Creation ──
|
||||
"market_creation_bond": MappingProxyType({"type": "int", "min": 0, "max": 1000}),
|
||||
"market_creation_bond_return_threshold": MappingProxyType({"type": "int", "min": 1, "max": 1000}),
|
||||
|
||||
# ── Bootstrap ──
|
||||
"bootstrap_market_count": MappingProxyType({"type": "int", "min": 0, "max": 100000}),
|
||||
"bootstrap_evidence_bond_cost": MappingProxyType({"type": "float", "min": 0.0, "max": 50.0}),
|
||||
"bootstrap_resolution_mode": MappingProxyType({"type": "str", "enum": ("eligible_node_one_vote",)}),
|
||||
"bootstrap_resolution_supermajority": MappingProxyType({"type": "float", "min": 0.51, "max": 0.95}),
|
||||
"bootstrap_min_identity_age_days": MappingProxyType({"type": "int", "min": 0, "max": 365}),
|
||||
"bootstrap_pow_algorithm": MappingProxyType({"type": "str", "enum": ("argon2id",)}),
|
||||
"bootstrap_pow_argon2id_version": MappingProxyType({"type": "int", "enum": (0x13,)}),
|
||||
"bootstrap_pow_argon2id_m": MappingProxyType({"type": "int", "min": 8192, "max": 1_048_576}),
|
||||
"bootstrap_pow_argon2id_t": MappingProxyType({"type": "int", "min": 1, "max": 100}),
|
||||
"bootstrap_pow_argon2id_p": MappingProxyType({"type": "int", "min": 1, "max": 16}),
|
||||
"bootstrap_pow_argon2id_output_len": MappingProxyType({"type": "int", "enum": (32,)}),
|
||||
"bootstrap_pow_difficulty": MappingProxyType({"type": "int", "min": 1, "max": 64}),
|
||||
|
||||
# ── Ramp milestones ──
|
||||
"ramp_staked_resolution_threshold": MappingProxyType({"type": "int", "min": 1, "max": 10_000_000}),
|
||||
"ramp_petitions_threshold": MappingProxyType({"type": "int", "min": 1, "max": 10_000_000}),
|
||||
"ramp_upgrade_threshold": MappingProxyType({"type": "int", "min": 1, "max": 10_000_000}),
|
||||
"ramp_commoncoin_threshold": MappingProxyType({"type": "int", "min": 1, "max": 10_000_000}),
|
||||
}
|
||||
|
||||
CONFIG_SCHEMA: MappingProxyType = MappingProxyType(_CONFIG_SCHEMA_BACKING)
|
||||
|
||||
|
||||
# ─── Cross-field invariants ──────────────────────────────────────────────
|
||||
# RULES_SKELETON.md §1.3.
|
||||
# Each tuple is (left_key, op, right_key). Only ">" supported today —
|
||||
# extend the dispatch in validate_cross_field_invariants when new ops
|
||||
# appear in the spec.
|
||||
|
||||
CROSS_FIELD_INVARIANTS: tuple[tuple[str, str, str], ...] = (
|
||||
("petition_supermajority", ">", "petition_quorum"),
|
||||
("resolution_supermajority", ">", "data_unavailable_threshold"),
|
||||
("upgrade_supermajority", ">", "upgrade_quorum"),
|
||||
("gate_shutdown_supermajority", ">", "gate_shutdown_quorum"),
|
||||
("gate_suspend_supermajority", ">", "gate_shutdown_quorum"),
|
||||
("farming_hard_threshold", ">", "farming_soft_threshold"),
|
||||
("truth_stake_max_days", ">", "truth_stake_min_days"),
|
||||
("upgrade_filing_cost", ">", "petition_filing_cost"),
|
||||
# Ramp milestones must be in strict ascending order so each tier
|
||||
# genuinely activates additional capability (Sprint 8 polish
|
||||
# 2026-04-28).
|
||||
("ramp_petitions_threshold", ">", "ramp_staked_resolution_threshold"),
|
||||
("ramp_upgrade_threshold", ">", "ramp_petitions_threshold"),
|
||||
("ramp_commoncoin_threshold", ">", "ramp_upgrade_threshold"),
|
||||
)
|
||||
|
||||
|
||||
# ─── Validators (used by the Sprint 7 governance DSL executor) ───────────
|
||||
|
||||
def validate_petition_value(
|
||||
key: str,
|
||||
value: Any,
|
||||
current_config: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""Validate one (key, value) pair against ``CONFIG_SCHEMA``.
|
||||
|
||||
Raises ``InvalidPetition`` on any failure. Returns ``None`` on success.
|
||||
|
||||
``current_config`` is accepted for API symmetry with the spec snippet
|
||||
in RULES §1.3 — current Sprint 1 logic doesn't need it. Future
|
||||
cross-field-aware updates may consult it.
|
||||
"""
|
||||
del current_config # deliberately unused — see docstring
|
||||
schema = CONFIG_SCHEMA.get(key)
|
||||
if schema is None:
|
||||
raise InvalidPetition(f"No schema for key: {key}")
|
||||
|
||||
type_name = schema["type"]
|
||||
expected = _SCHEMA_TYPES.get(type_name)
|
||||
if expected is None:
|
||||
raise InvalidPetition(f"Schema for {key} has unknown type: {type_name}")
|
||||
|
||||
if type_name == "bool":
|
||||
if not isinstance(value, bool):
|
||||
raise InvalidPetition(
|
||||
f"Type mismatch for {key}: expected bool, got {type(value).__name__}"
|
||||
)
|
||||
elif type_name == "int":
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise InvalidPetition(
|
||||
f"Type mismatch for {key}: expected int, got {type(value).__name__}"
|
||||
)
|
||||
elif type_name == "float":
|
||||
if isinstance(value, bool) or not isinstance(value, expected):
|
||||
raise InvalidPetition(
|
||||
f"Type mismatch for {key}: expected float, got {type(value).__name__}"
|
||||
)
|
||||
else: # str
|
||||
if not isinstance(value, expected):
|
||||
raise InvalidPetition(
|
||||
f"Type mismatch for {key}: expected {type_name}, got {type(value).__name__}"
|
||||
)
|
||||
|
||||
if "min" in schema and value < schema["min"]:
|
||||
raise InvalidPetition(f"{key}={value} below minimum {schema['min']}")
|
||||
if "max" in schema and value > schema["max"]:
|
||||
raise InvalidPetition(f"{key}={value} above maximum {schema['max']}")
|
||||
if "enum" in schema and value not in schema["enum"]:
|
||||
raise InvalidPetition(f"{key}={value} not in allowed values {tuple(schema['enum'])}")
|
||||
|
||||
|
||||
def validate_cross_field_invariants(config: dict[str, Any]) -> None:
|
||||
"""Check every entry of ``CROSS_FIELD_INVARIANTS`` against ``config``.
|
||||
|
||||
Called by the DSL executor AFTER all updates from a single petition
|
||||
payload have been applied to a candidate config dict. Raises
|
||||
``InvalidPetition`` on the first violation. The candidate config is
|
||||
discarded by the executor when this raises.
|
||||
"""
|
||||
for left_key, op, right_key in CROSS_FIELD_INVARIANTS:
|
||||
if left_key not in config:
|
||||
raise InvalidPetition(f"Cross-field invariant references missing key: {left_key}")
|
||||
if right_key not in config:
|
||||
raise InvalidPetition(f"Cross-field invariant references missing key: {right_key}")
|
||||
left_val = config[left_key]
|
||||
right_val = config[right_key]
|
||||
if op == ">":
|
||||
if not (left_val > right_val):
|
||||
raise InvalidPetition(
|
||||
f"Cross-field invariant violated: {left_key}={left_val} must be > "
|
||||
f"{right_key}={right_val}"
|
||||
)
|
||||
else:
|
||||
raise InvalidPetition(f"Unknown cross-field operator: {op}")
|
||||
|
||||
|
||||
def validate_config_schema_completeness() -> None:
|
||||
"""Sprint 1 invariant: every CONFIG key has a matching CONFIG_SCHEMA entry.
|
||||
|
||||
Raises ``InvalidPetition`` listing missing keys. Called both from the
|
||||
Sprint 1 adversarial test and from the DSL executor on startup.
|
||||
"""
|
||||
missing = sorted(set(CONFIG.keys()) - set(CONFIG_SCHEMA.keys()))
|
||||
extra = sorted(set(CONFIG_SCHEMA.keys()) - set(CONFIG.keys()))
|
||||
if missing:
|
||||
raise InvalidPetition(f"CONFIG keys without CONFIG_SCHEMA entry: {missing}")
|
||||
if extra:
|
||||
raise InvalidPetition(f"CONFIG_SCHEMA keys without CONFIG entry: {extra}")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CONFIG",
|
||||
"CONFIG_SCHEMA",
|
||||
"CROSS_FIELD_INVARIANTS",
|
||||
"IMMUTABLE_PRINCIPLES",
|
||||
"InvalidPetition",
|
||||
"reset_config_for_tests",
|
||||
"validate_config_schema_completeness",
|
||||
"validate_cross_field_invariants",
|
||||
"validate_petition_value",
|
||||
]
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Event construction helpers for the Infonet economy.
|
||||
|
||||
A thin layer over ``services/infonet/schema.py``: each public function
|
||||
builds a payload dict for one event type, validates it, and returns it.
|
||||
The caller is responsible for signing the event and routing it through
|
||||
``services/infonet/adapters/hashchain_adapter.py`` for actual append.
|
||||
|
||||
Sprint 1 scope: payload builders + validation. No chain writes. The
|
||||
hashchain adapter's ``append_infonet_event`` is the eventual integration
|
||||
point — see ``adapters/hashchain_adapter.py``.
|
||||
|
||||
Why a builder layer and not free-form dicts:
|
||||
- Centralizes the canonical field set per event_type so callers can't
|
||||
drift from the schema.
|
||||
- Allows future sprints to attach deterministic computation (e.g.
|
||||
``probability_at_bet`` reconstruction in Sprint 4) without changing
|
||||
callers.
|
||||
- Matches the "events extend, never replace" rule from the plan §3.1 —
|
||||
the legacy event constructors in ``mesh_schema.py`` keep working
|
||||
unchanged; new event types live here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from services.infonet.schema import (
|
||||
INFONET_ECONOMY_EVENT_TYPES,
|
||||
validate_infonet_event_payload,
|
||||
)
|
||||
|
||||
|
||||
class EventConstructionError(ValueError):
|
||||
"""Raised when a payload fails validation at build time.
|
||||
|
||||
Distinct from chain-level errors (signature, replay, sequence) —
|
||||
those originate in the hashchain adapter, not here.
|
||||
"""
|
||||
|
||||
|
||||
def build_event(event_type: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Validate and return a payload for ``event_type``.
|
||||
|
||||
The returned dict is a shallow copy — callers can attach signature,
|
||||
sequence, public_key, etc. before passing it to the hashchain
|
||||
adapter for append.
|
||||
"""
|
||||
if event_type not in INFONET_ECONOMY_EVENT_TYPES:
|
||||
raise EventConstructionError(
|
||||
f"event_type {event_type!r} is not in INFONET_ECONOMY_EVENT_TYPES"
|
||||
)
|
||||
payload = dict(payload or {})
|
||||
ok, reason = validate_infonet_event_payload(event_type, payload)
|
||||
if not ok:
|
||||
raise EventConstructionError(f"{event_type}: {reason}")
|
||||
return payload
|
||||
|
||||
|
||||
# ─── Convenience builders ────────────────────────────────────────────────
|
||||
# Sprint 1 ships only a representative slice. Full per-type builders for
|
||||
# the producing modules (markets/, gates/, governance/, ...) live in
|
||||
# their respective sprints — they will all funnel through ``build_event``
|
||||
# so this module stays the single validation choke point.
|
||||
|
||||
def build_uprep(target_node_id: str, target_event_id: str) -> dict[str, Any]:
|
||||
return build_event("uprep", {
|
||||
"target_node_id": target_node_id,
|
||||
"target_event_id": target_event_id,
|
||||
})
|
||||
|
||||
|
||||
def build_citizenship_claim(sacrifice_amount: int) -> dict[str, Any]:
|
||||
return build_event("citizenship_claim", {"sacrifice_amount": sacrifice_amount})
|
||||
|
||||
|
||||
def build_petition_file(
|
||||
petition_id: str,
|
||||
petition_payload: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
return build_event("petition_file", {
|
||||
"petition_id": petition_id,
|
||||
"petition_payload": petition_payload,
|
||||
})
|
||||
|
||||
|
||||
def build_petition_vote(petition_id: str, vote: str) -> dict[str, Any]:
|
||||
return build_event("petition_vote", {"petition_id": petition_id, "vote": vote})
|
||||
|
||||
|
||||
def build_node_register(public_key: str, public_key_algo: str, node_class: str) -> dict[str, Any]:
|
||||
return build_event("node_register", {
|
||||
"public_key": public_key,
|
||||
"public_key_algo": public_key_algo,
|
||||
"node_class": node_class,
|
||||
})
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EventConstructionError",
|
||||
"build_citizenship_claim",
|
||||
"build_event",
|
||||
"build_node_register",
|
||||
"build_petition_file",
|
||||
"build_petition_vote",
|
||||
"build_uprep",
|
||||
]
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Gate sacrifice + locking + shutdown lifecycle (Sprint 6).
|
||||
|
||||
Pure-function design: every entry point reads the chain and returns a
|
||||
deterministic value. State (member set / suspended_until / shutdown
|
||||
status / appeal status) is derived, never stored.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.16, §5.3,
|
||||
§5.5.
|
||||
"""
|
||||
|
||||
from services.infonet.gates.locking import (
|
||||
LockedGateState,
|
||||
is_locked,
|
||||
locked_at,
|
||||
locked_by,
|
||||
validate_lock_request,
|
||||
)
|
||||
from services.infonet.gates.ratification import (
|
||||
RATIFICATION_THRESHOLD,
|
||||
cumulative_member_oracle_rep,
|
||||
is_ratified,
|
||||
)
|
||||
from services.infonet.gates.sacrifice import (
|
||||
EntryDecision,
|
||||
EntryRefusal,
|
||||
can_enter,
|
||||
compute_member_set,
|
||||
is_member,
|
||||
)
|
||||
from services.infonet.gates.shutdown.appeal import (
|
||||
AppealValidation,
|
||||
paused_execution_remaining_sec,
|
||||
validate_appeal_filing,
|
||||
)
|
||||
from services.infonet.gates.shutdown.shutdown import (
|
||||
ShutdownState,
|
||||
compute_shutdown_state,
|
||||
validate_shutdown_filing,
|
||||
)
|
||||
from services.infonet.gates.shutdown.suspend import (
|
||||
SuspensionState,
|
||||
compute_suspension_state,
|
||||
validate_suspend_filing,
|
||||
)
|
||||
from services.infonet.gates.state import (
|
||||
GateMeta,
|
||||
events_for_gate,
|
||||
get_gate_meta,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AppealValidation",
|
||||
"EntryDecision",
|
||||
"EntryRefusal",
|
||||
"GateMeta",
|
||||
"LockedGateState",
|
||||
"RATIFICATION_THRESHOLD",
|
||||
"ShutdownState",
|
||||
"SuspensionState",
|
||||
"can_enter",
|
||||
"compute_member_set",
|
||||
"compute_shutdown_state",
|
||||
"compute_suspension_state",
|
||||
"cumulative_member_oracle_rep",
|
||||
"events_for_gate",
|
||||
"get_gate_meta",
|
||||
"is_locked",
|
||||
"is_member",
|
||||
"is_ratified",
|
||||
"locked_at",
|
||||
"locked_by",
|
||||
"paused_execution_remaining_sec",
|
||||
"validate_appeal_filing",
|
||||
"validate_lock_request",
|
||||
"validate_shutdown_filing",
|
||||
"validate_suspend_filing",
|
||||
]
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Gate locking — "constitutionalize-a-gate".
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §5.3 step 4 +
|
||||
``CONFIG['gate_lock_cost_per_member']`` / ``CONFIG['gate_lock_min_members']``.
|
||||
|
||||
Locking semantics:
|
||||
|
||||
- Each ``gate_lock`` event records one member contributing
|
||||
``CONFIG['gate_lock_cost_per_member']`` (default 10) common rep.
|
||||
- A gate is "locked" once ≥ ``CONFIG['gate_lock_min_members']``
|
||||
(default 5) distinct current members have each emitted a valid
|
||||
``gate_lock`` event.
|
||||
- Once locked, the gate's rules become immutable — no governance
|
||||
petition can modify them. Only an upgrade-hash governance event
|
||||
(out of scope for Sprint 6) can amend a locked gate's rules.
|
||||
|
||||
Validation rules for an incoming ``gate_lock`` event (callers in
|
||||
production should run these *before* emitting):
|
||||
|
||||
- The gate exists.
|
||||
- The locker is a current member.
|
||||
- The locker hasn't already locked this gate (one lock per node).
|
||||
- The locker has paid (the burn happens at emit time; this module
|
||||
asserts the schematic ``lock_cost`` matches CONFIG).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
from services.infonet.gates.sacrifice import compute_member_set
|
||||
from services.infonet.gates.state import events_for_gate
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
def _lock_cost_per_member() -> int:
|
||||
return int(CONFIG["gate_lock_cost_per_member"])
|
||||
|
||||
|
||||
def _lock_min_members() -> int:
|
||||
return int(CONFIG["gate_lock_min_members"])
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LockedGateState:
|
||||
locked: bool
|
||||
locked_at: float | None
|
||||
locked_by: tuple[str, ...]
|
||||
|
||||
|
||||
def _collect_lock_contributions(
|
||||
gate_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> list[tuple[str, float]]:
|
||||
"""Return ``[(node_id, timestamp)]`` for each accepted ``gate_lock``
|
||||
event in chain order. Subsequent locks from the same node are
|
||||
ignored (one lock per node)."""
|
||||
chain_list = list(chain)
|
||||
members = compute_member_set(gate_id, chain_list)
|
||||
seen: set[str] = set()
|
||||
out: list[tuple[str, float]] = []
|
||||
for ev in events_for_gate(gate_id, chain_list):
|
||||
if ev.get("event_type") != "gate_lock":
|
||||
continue
|
||||
node = ev.get("node_id")
|
||||
if not isinstance(node, str) or not node:
|
||||
continue
|
||||
if node in seen:
|
||||
continue
|
||||
if node not in members:
|
||||
# Non-member lock attempt — ignored. The producer-side
|
||||
# check should also refuse to emit, but resolver-side
|
||||
# enforcement is defense-in-depth.
|
||||
continue
|
||||
p = _payload(ev)
|
||||
try:
|
||||
paid = float(p.get("lock_cost") or 0.0)
|
||||
except (TypeError, ValueError):
|
||||
paid = 0.0
|
||||
if paid < float(_lock_cost_per_member()):
|
||||
continue
|
||||
seen.add(node)
|
||||
out.append((node, float(ev.get("timestamp") or 0.0)))
|
||||
return out
|
||||
|
||||
|
||||
def _state(gate_id: str, chain: Iterable[dict[str, Any]]) -> LockedGateState:
|
||||
contributions = _collect_lock_contributions(gate_id, chain)
|
||||
if len(contributions) < _lock_min_members():
|
||||
return LockedGateState(locked=False, locked_at=None, locked_by=())
|
||||
contributions.sort(key=lambda c: c[1])
|
||||
threshold_ts = contributions[_lock_min_members() - 1][1]
|
||||
nodes = tuple(c[0] for c in contributions)
|
||||
return LockedGateState(locked=True, locked_at=threshold_ts, locked_by=nodes)
|
||||
|
||||
|
||||
def is_locked(gate_id: str, chain: Iterable[dict[str, Any]]) -> bool:
|
||||
return _state(gate_id, chain).locked
|
||||
|
||||
|
||||
def locked_at(gate_id: str, chain: Iterable[dict[str, Any]]) -> float | None:
|
||||
return _state(gate_id, chain).locked_at
|
||||
|
||||
|
||||
def locked_by(gate_id: str, chain: Iterable[dict[str, Any]]) -> tuple[str, ...]:
|
||||
return _state(gate_id, chain).locked_by
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LockValidation:
|
||||
accepted: bool
|
||||
reason: str
|
||||
cost: int
|
||||
|
||||
|
||||
def validate_lock_request(
|
||||
node_id: str,
|
||||
gate_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
lock_cost: int | None = None,
|
||||
) -> LockValidation:
|
||||
"""Pre-emit check for a ``gate_lock`` event from ``node_id``.
|
||||
|
||||
Returns ``accepted=False`` with a structured ``reason`` when
|
||||
rejected — the UI surfaces these directly so the user knows what
|
||||
needs to change.
|
||||
"""
|
||||
chain_list = list(chain)
|
||||
cost = int(_lock_cost_per_member() if lock_cost is None else lock_cost)
|
||||
if cost < _lock_cost_per_member():
|
||||
return LockValidation(False, "lock_cost_below_min", cost)
|
||||
if node_id not in compute_member_set(gate_id, chain_list):
|
||||
return LockValidation(False, "not_a_member", cost)
|
||||
if node_id in {n for n, _ in _collect_lock_contributions(gate_id, chain_list)}:
|
||||
return LockValidation(False, "already_locked_by_node", cost)
|
||||
return LockValidation(True, "ok", cost)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LockedGateState",
|
||||
"LockValidation",
|
||||
"is_locked",
|
||||
"locked_at",
|
||||
"locked_by",
|
||||
"validate_lock_request",
|
||||
]
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Gate ratification — cumulative oracle rep threshold.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §5.3 step 3.
|
||||
|
||||
A gate is "ratified" once the SUM of its members' oracle rep crosses
|
||||
``CONFIG['gate_ratification_rep']`` (default 50). Ratification is a
|
||||
recognition signal — it doesn't gate any functionality, but UI may
|
||||
surface it as "this gate is established / legitimate".
|
||||
|
||||
Pure function over the chain. The threshold is governable via petition
|
||||
(Sprint 7) by changing the CONFIG value.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
from services.infonet.gates.sacrifice import compute_member_set
|
||||
from services.infonet.reputation import compute_oracle_rep
|
||||
|
||||
|
||||
def _ratification_threshold() -> int:
|
||||
return int(CONFIG["gate_ratification_rep"])
|
||||
|
||||
|
||||
# Public alias for consumers who don't want to import CONFIG.
|
||||
RATIFICATION_THRESHOLD = _ratification_threshold()
|
||||
|
||||
|
||||
def cumulative_member_oracle_rep(
|
||||
gate_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> float:
|
||||
"""Sum of current members' oracle rep balances."""
|
||||
chain_list = list(chain)
|
||||
members = compute_member_set(gate_id, chain_list)
|
||||
return sum(compute_oracle_rep(m, chain_list) for m in members)
|
||||
|
||||
|
||||
def is_ratified(
|
||||
gate_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> bool:
|
||||
"""``True`` once cumulative member oracle rep meets the threshold."""
|
||||
return cumulative_member_oracle_rep(gate_id, chain) >= float(_ratification_threshold())
|
||||
|
||||
|
||||
__all__ = [
|
||||
"RATIFICATION_THRESHOLD",
|
||||
"cumulative_member_oracle_rep",
|
||||
"is_ratified",
|
||||
]
|
||||
@@ -0,0 +1,192 @@
|
||||
"""Gate sacrifice mechanic — burn-on-entry, not threshold check.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.16, §5.3 step 2.
|
||||
|
||||
A node enters a gate by **burning** common rep equal to
|
||||
``gate.entry_sacrifice``. The burn is permanent and non-refundable
|
||||
(even on voluntary exit). This is the constitutional difference from
|
||||
threshold-based access: you can't fake having enough rep — you have
|
||||
to spend it.
|
||||
|
||||
The eligibility checks happen *before* the burn:
|
||||
|
||||
- Node's common rep ≥ ``min_overall_rep + entry_sacrifice``.
|
||||
- Node's per-gate rep meets each ``min_gate_rep[required_gate]``.
|
||||
|
||||
If those pass, the entry is accepted, ``entry_sacrifice`` is burned
|
||||
from the node's common rep, and the node is recorded as a member.
|
||||
|
||||
This module exposes pure functions:
|
||||
|
||||
- ``can_enter(node_id, gate_id, chain)`` — eligibility check + cost,
|
||||
returning a structured ``EntryDecision`` so the UI can render
|
||||
exactly *why* a node can't enter (cross-cutting non-hostile UX rule).
|
||||
- ``compute_member_set(gate_id, chain)`` — current members from
|
||||
``gate_enter`` − ``gate_exit`` events.
|
||||
- ``is_member(node_id, gate_id, chain)`` — convenience.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.gates.state import events_for_gate, get_gate_meta
|
||||
from services.infonet.reputation import compute_common_rep
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
def compute_member_set(
|
||||
gate_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> set[str]:
|
||||
"""Current member set: ``gate_enter`` − ``gate_exit`` − members
|
||||
booted by ``gate_shutdown_execute``. The shutdown case zeroes the
|
||||
set out — once a gate is shut down, there are no members.
|
||||
"""
|
||||
chain_list = list(chain)
|
||||
events = events_for_gate(gate_id, chain_list)
|
||||
members: set[str] = set()
|
||||
shutdown_seen = False
|
||||
for ev in events:
|
||||
et = ev.get("event_type")
|
||||
if et == "gate_shutdown_execute":
|
||||
shutdown_seen = True
|
||||
members = set()
|
||||
continue
|
||||
node = ev.get("node_id")
|
||||
if not isinstance(node, str) or not node:
|
||||
continue
|
||||
if et == "gate_enter":
|
||||
if not shutdown_seen:
|
||||
members.add(node)
|
||||
elif et == "gate_exit":
|
||||
members.discard(node)
|
||||
return members
|
||||
|
||||
|
||||
def is_member(
|
||||
node_id: str,
|
||||
gate_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> bool:
|
||||
return node_id in compute_member_set(gate_id, list(chain))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EntryRefusal:
|
||||
"""Structured "why a node can't enter" diagnostic.
|
||||
|
||||
The cross-cutting non-hostile UX rule (BUILD_LOG.md design rules
|
||||
§1) requires the UI to show the user a path forward — not a
|
||||
blanket "denied". This dataclass carries enough info for the
|
||||
frontend to render "you need 5 more common rep" or "you need
|
||||
more rep in gate X".
|
||||
"""
|
||||
kind: str
|
||||
detail: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EntryDecision:
|
||||
accepted: bool
|
||||
cost: int
|
||||
refusals: tuple[EntryRefusal, ...]
|
||||
|
||||
|
||||
def compute_gate_rep(
|
||||
node_id: str,
|
||||
gate_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> float:
|
||||
"""Per-gate reputation: common rep earned from upreps cast by
|
||||
members of ``gate_id``.
|
||||
|
||||
Sprint 6 ships a simple variant: same formula as
|
||||
``compute_common_rep`` but only upreps from current members of
|
||||
``gate_id`` count. Anti-gaming penalties (Sprint 3) still apply
|
||||
via the underlying ``compute_common_rep`` call when called with
|
||||
the synthetic chain — but for Sprint 6 we filter at the chain
|
||||
level and pass the filtered chain to the global function.
|
||||
|
||||
A more sophisticated per-gate formula (e.g. using only upreps
|
||||
that happened *while* the upreper was a member, or weighting by
|
||||
in-gate activity) is open for governance to specify later.
|
||||
"""
|
||||
chain_list = [e for e in chain if isinstance(e, dict)]
|
||||
members = compute_member_set(gate_id, chain_list)
|
||||
if not members:
|
||||
return 0.0
|
||||
# Filter to upreps authored by current gate members targeting node_id.
|
||||
# Pass the WHOLE chain to compute_common_rep (it needs full event
|
||||
# history for oracle_rep computation of the upreper); but limit
|
||||
# which uprep events count by stripping non-member ones.
|
||||
filtered: list[dict[str, Any]] = []
|
||||
for ev in chain_list:
|
||||
if ev.get("event_type") == "uprep":
|
||||
author = ev.get("node_id")
|
||||
if author not in members:
|
||||
continue
|
||||
filtered.append(ev)
|
||||
return compute_common_rep(node_id, filtered)
|
||||
|
||||
|
||||
def can_enter(
|
||||
node_id: str,
|
||||
gate_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> EntryDecision:
|
||||
"""RULES §3.16 — eligibility + cost.
|
||||
|
||||
Returns a structured decision. ``accepted=True`` means: burning
|
||||
``cost`` common rep from ``node_id`` satisfies all entry rules.
|
||||
``accepted=False`` lists every reason refusal occurred so the UI
|
||||
can show all of them at once.
|
||||
"""
|
||||
chain_list = list(chain)
|
||||
meta = get_gate_meta(gate_id, chain_list)
|
||||
if meta is None:
|
||||
return EntryDecision(
|
||||
accepted=False, cost=0,
|
||||
refusals=(EntryRefusal(kind="gate_not_found", detail=gate_id),),
|
||||
)
|
||||
if is_member(node_id, gate_id, chain_list):
|
||||
return EntryDecision(
|
||||
accepted=False, cost=0,
|
||||
refusals=(EntryRefusal(kind="already_member", detail=gate_id),),
|
||||
)
|
||||
|
||||
refusals: list[EntryRefusal] = []
|
||||
common_rep = compute_common_rep(node_id, chain_list)
|
||||
needed = meta.min_overall_rep + meta.entry_sacrifice
|
||||
if common_rep < needed:
|
||||
refusals.append(EntryRefusal(
|
||||
kind="insufficient_common_rep",
|
||||
detail=f"have {common_rep:.4f}, need {needed} (min_overall_rep "
|
||||
f"{meta.min_overall_rep} + entry_sacrifice {meta.entry_sacrifice})",
|
||||
))
|
||||
for required_gate, min_rep in meta.min_gate_rep.items():
|
||||
gate_rep = compute_gate_rep(node_id, required_gate, chain_list)
|
||||
if gate_rep < min_rep:
|
||||
refusals.append(EntryRefusal(
|
||||
kind="insufficient_gate_rep",
|
||||
detail=f"gate {required_gate}: have {gate_rep:.4f}, need {min_rep}",
|
||||
))
|
||||
return EntryDecision(
|
||||
accepted=not refusals, cost=meta.entry_sacrifice if not refusals else 0,
|
||||
refusals=tuple(refusals),
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EntryDecision",
|
||||
"EntryRefusal",
|
||||
"can_enter",
|
||||
"compute_gate_rep",
|
||||
"compute_member_set",
|
||||
"is_member",
|
||||
]
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Gate shutdown lifecycle — Tier 1 suspend, Tier 2 shutdown, typed appeal.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §5.5.
|
||||
|
||||
Three modules with clean separation of concerns:
|
||||
|
||||
- ``suspend.py`` — Tier 1: 30-day reversible freeze. Filed via
|
||||
``gate_suspend_file``, voted on, executed via
|
||||
``gate_suspend_execute``, auto-unsuspends after 30 days unless a
|
||||
shutdown petition passes.
|
||||
- ``shutdown.py`` — Tier 2: 7-day-delayed archive. PREREQUISITE: gate
|
||||
must currently be suspended.
|
||||
- ``appeal.py`` — Typed shutdown appeal: pauses the 7-day execution
|
||||
timer, max one appeal per shutdown, 48h window after vote passage.
|
||||
"""
|
||||
|
||||
from services.infonet.gates.shutdown.appeal import (
|
||||
AppealValidation,
|
||||
paused_execution_remaining_sec,
|
||||
validate_appeal_filing,
|
||||
)
|
||||
from services.infonet.gates.shutdown.shutdown import (
|
||||
ShutdownState,
|
||||
compute_shutdown_state,
|
||||
validate_shutdown_filing,
|
||||
)
|
||||
from services.infonet.gates.shutdown.suspend import (
|
||||
SuspensionState,
|
||||
compute_suspension_state,
|
||||
validate_suspend_filing,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AppealValidation",
|
||||
"ShutdownState",
|
||||
"SuspensionState",
|
||||
"compute_shutdown_state",
|
||||
"compute_suspension_state",
|
||||
"paused_execution_remaining_sec",
|
||||
"validate_appeal_filing",
|
||||
"validate_shutdown_filing",
|
||||
"validate_suspend_filing",
|
||||
]
|
||||
@@ -0,0 +1,189 @@
|
||||
"""Typed shutdown appeal — pauses execution timer, anti-stall bounded.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §5.5 step 7.
|
||||
|
||||
An appeal pauses the 7-day shutdown execution timer. The
|
||||
"anti-stall" property limits abuse:
|
||||
|
||||
- One appeal per shutdown petition (no infinite re-appeals).
|
||||
- 48-hour filing window after the shutdown vote passes.
|
||||
- If the appeal fails, the original shutdown's execution timer
|
||||
resumes from where it was paused — the shutdown still happens,
|
||||
just delayed by the appeal-vote duration.
|
||||
|
||||
This module exposes:
|
||||
|
||||
- ``validate_appeal_filing`` — pre-emit checks.
|
||||
- ``paused_execution_remaining_sec`` — compute how much time was
|
||||
remaining on the shutdown timer when the appeal was filed (so the
|
||||
resolver can resume the timer from that point).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
from services.infonet.gates.shutdown.shutdown import compute_shutdown_state
|
||||
from services.infonet.gates.state import get_gate_meta
|
||||
|
||||
|
||||
_SECONDS_PER_HOUR = 3600.0
|
||||
_SECONDS_PER_DAY = 86400.0
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AppealValidation:
|
||||
accepted: bool
|
||||
reason: str
|
||||
|
||||
|
||||
def _shutdown_petition_filed_at(
|
||||
target_petition_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> float | None:
|
||||
for ev in chain:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
if ev.get("event_type") != "gate_shutdown_file":
|
||||
continue
|
||||
if _payload(ev).get("petition_id") == target_petition_id:
|
||||
return float(ev.get("timestamp") or 0.0)
|
||||
return None
|
||||
|
||||
|
||||
def _shutdown_vote_passed_at(
|
||||
target_petition_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> float | None:
|
||||
"""Return the timestamp of the ``gate_shutdown_vote`` event whose
|
||||
payload says ``vote=="passed"`` for the target petition. The
|
||||
appeal window starts here."""
|
||||
for ev in chain:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
if ev.get("event_type") != "gate_shutdown_vote":
|
||||
continue
|
||||
p = _payload(ev)
|
||||
if p.get("petition_id") != target_petition_id:
|
||||
continue
|
||||
if p.get("vote") == "passed":
|
||||
return float(ev.get("timestamp") or 0.0)
|
||||
return None
|
||||
|
||||
|
||||
def _has_appeal(
|
||||
target_petition_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> bool:
|
||||
for ev in chain:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
if ev.get("event_type") != "gate_shutdown_appeal_file":
|
||||
continue
|
||||
if _payload(ev).get("target_petition_id") == target_petition_id:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def validate_appeal_filing(
|
||||
gate_id: str,
|
||||
target_petition_id: str,
|
||||
filer_id: str,
|
||||
*,
|
||||
reason: str,
|
||||
evidence_hashes: list[str],
|
||||
chain: Iterable[dict[str, Any]],
|
||||
now: float,
|
||||
filer_cooldown_until: float | None = None,
|
||||
) -> AppealValidation:
|
||||
"""Pre-emit validation for ``gate_shutdown_appeal_file``.
|
||||
|
||||
Rejects if:
|
||||
- Reason or evidence missing.
|
||||
- Gate doesn't exist.
|
||||
- Target shutdown petition doesn't exist.
|
||||
- Target petition is not currently in "executing" status (i.e.
|
||||
vote hasn't passed yet, or shutdown already executed).
|
||||
- 48-hour filing window has elapsed since vote passage.
|
||||
- Target petition already has an appeal (one per shutdown).
|
||||
- Filer cooldown active.
|
||||
"""
|
||||
chain_list = list(chain)
|
||||
if not isinstance(reason, str) or not reason.strip():
|
||||
return AppealValidation(False, "reason_empty")
|
||||
if not isinstance(evidence_hashes, list) or not evidence_hashes:
|
||||
return AppealValidation(False, "evidence_required")
|
||||
if get_gate_meta(gate_id, chain_list) is None:
|
||||
return AppealValidation(False, "gate_not_found")
|
||||
|
||||
if not _shutdown_petition_filed_at(target_petition_id, chain_list):
|
||||
return AppealValidation(False, "target_petition_not_found")
|
||||
|
||||
# The "already-filed" check fires before the status check on
|
||||
# purpose — once an appeal is filed, the petition status flips
|
||||
# from "executing" to "appealed", and surfacing that as
|
||||
# "target_not_in_executing_state" would mislead a second filer
|
||||
# about *why* their appeal was refused. Spec invariant: one
|
||||
# appeal per shutdown; surface that directly.
|
||||
if _has_appeal(target_petition_id, chain_list):
|
||||
return AppealValidation(False, "appeal_already_filed")
|
||||
|
||||
state = compute_shutdown_state(gate_id, chain_list, now=now)
|
||||
if state.pending_status not in ("executing",):
|
||||
return AppealValidation(False, "target_not_in_executing_state")
|
||||
|
||||
vote_passed = _shutdown_vote_passed_at(target_petition_id, chain_list)
|
||||
if vote_passed is None:
|
||||
return AppealValidation(False, "vote_not_passed")
|
||||
window_s = float(CONFIG["gate_shutdown_appeal_window_hours"]) * _SECONDS_PER_HOUR
|
||||
if now > vote_passed + window_s:
|
||||
return AppealValidation(False, "appeal_window_expired")
|
||||
|
||||
if filer_cooldown_until is not None and filer_cooldown_until > now:
|
||||
return AppealValidation(False, "filer_cooldown_active")
|
||||
# filer_id is consumed by the producer event payload, not by validation here.
|
||||
del filer_id
|
||||
return AppealValidation(True, "ok")
|
||||
|
||||
|
||||
def paused_execution_remaining_sec(
|
||||
target_petition_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
appeal_filed_at: float,
|
||||
) -> float:
|
||||
"""Compute how much time was remaining on the shutdown's
|
||||
execution timer when the appeal was filed.
|
||||
|
||||
The original shutdown's ``execution_at`` was
|
||||
``vote_passed_at + execution_delay_days * 86400``. The remaining
|
||||
time at appeal-filing time is ``execution_at - appeal_filed_at``,
|
||||
clamped to ≥ 0.
|
||||
|
||||
The producer of the ``gate_shutdown_appeal_resolve`` event with
|
||||
``outcome="resumed"`` should attach
|
||||
``resumed_execution_at = now + this_value`` so the timer resumes
|
||||
from where it paused.
|
||||
"""
|
||||
chain_list = list(chain)
|
||||
vote_passed = _shutdown_vote_passed_at(target_petition_id, chain_list)
|
||||
if vote_passed is None:
|
||||
return 0.0
|
||||
delay_s = float(CONFIG["gate_shutdown_execution_delay_days"]) * _SECONDS_PER_DAY
|
||||
execution_at = vote_passed + delay_s
|
||||
remaining = execution_at - float(appeal_filed_at)
|
||||
return max(0.0, remaining)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AppealValidation",
|
||||
"paused_execution_remaining_sec",
|
||||
"validate_appeal_filing",
|
||||
]
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Tier 2: 7-day-delayed shutdown.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §5.5 steps 5-8.
|
||||
|
||||
PREREQUISITE: gate must currently be suspended. The shutdown petition
|
||||
itself is a vote among oracle-rep holders. If it passes, a 7-day
|
||||
execution delay opens (the appeal window). After the delay (and any
|
||||
appeal resolution), the ``gate_shutdown_execute`` event archives the
|
||||
gate permanently.
|
||||
|
||||
State derivation:
|
||||
|
||||
- A shutdown petition can be: ``filed``, ``vote_passed``, ``executing``
|
||||
(after vote, during 7-day delay), ``appealed`` (timer paused),
|
||||
``executed``, ``failed``, ``voided_appeal``.
|
||||
- This module computes the petition status from chain events; it does
|
||||
NOT execute the petition itself (the producer emits
|
||||
``gate_shutdown_execute`` based on this status).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
from services.infonet.gates.shutdown.suspend import (
|
||||
FilingValidation,
|
||||
compute_suspension_state,
|
||||
)
|
||||
from services.infonet.gates.state import events_for_gate, get_gate_meta
|
||||
|
||||
|
||||
_SECONDS_PER_DAY = 86400.0
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ShutdownState:
|
||||
"""Derived snapshot of all shutdown petitions filed against a gate."""
|
||||
has_pending: bool
|
||||
pending_petition_id: str | None
|
||||
pending_status: str | None # "filed" | "vote_passed" | "executing" | "appealed" | "failed"
|
||||
execution_at: float | None
|
||||
executed: bool
|
||||
|
||||
|
||||
def compute_shutdown_state(
|
||||
gate_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float,
|
||||
) -> ShutdownState:
|
||||
chain_list = list(chain)
|
||||
events = events_for_gate(gate_id, chain_list)
|
||||
|
||||
petitions: dict[str, dict[str, Any]] = {}
|
||||
for ev in events:
|
||||
et = ev.get("event_type")
|
||||
if et != "gate_shutdown_file":
|
||||
continue
|
||||
p = _payload(ev)
|
||||
pid = p.get("petition_id")
|
||||
if not isinstance(pid, str) or not pid:
|
||||
continue
|
||||
petitions[pid] = {
|
||||
"filed_at": float(ev.get("timestamp") or 0.0),
|
||||
"status": "filed",
|
||||
"execution_at": None,
|
||||
"appealed": False,
|
||||
}
|
||||
|
||||
# Walk votes/executions/appeals in chain order.
|
||||
chain_all = [e for e in chain_list if isinstance(e, dict)]
|
||||
chain_all.sort(key=lambda e: (float(e.get("timestamp") or 0.0), int(e.get("sequence") or 0)))
|
||||
|
||||
for ev in chain_all:
|
||||
et = ev.get("event_type")
|
||||
if et not in ("gate_shutdown_vote", "gate_shutdown_execute",
|
||||
"gate_shutdown_appeal_file", "gate_shutdown_appeal_resolve"):
|
||||
continue
|
||||
p = _payload(ev)
|
||||
|
||||
if et == "gate_shutdown_vote":
|
||||
pid = p.get("petition_id")
|
||||
if not isinstance(pid, str) or pid not in petitions:
|
||||
continue
|
||||
# Sprint 6 simplification: a vote event with payload
|
||||
# {"vote": "passed"} is treated as the canonical pass
|
||||
# signal. Real production may aggregate per-voter votes
|
||||
# in Sprint 7's governance DSL — Sprint 6 honors whichever
|
||||
# outcome the spec-side vote tally already reached.
|
||||
outcome = p.get("vote")
|
||||
if outcome == "passed":
|
||||
petitions[pid]["status"] = "executing"
|
||||
delay_s = float(CONFIG["gate_shutdown_execution_delay_days"]) * _SECONDS_PER_DAY
|
||||
petitions[pid]["execution_at"] = float(ev.get("timestamp") or 0.0) + delay_s
|
||||
elif outcome == "failed":
|
||||
petitions[pid]["status"] = "failed"
|
||||
|
||||
elif et == "gate_shutdown_appeal_file":
|
||||
target = p.get("target_petition_id")
|
||||
if isinstance(target, str) and target in petitions:
|
||||
petitions[target]["appealed"] = True
|
||||
petitions[target]["status"] = "appealed"
|
||||
petitions[target]["execution_at"] = None # paused
|
||||
|
||||
elif et == "gate_shutdown_appeal_resolve":
|
||||
target = p.get("target_petition_id")
|
||||
outcome = p.get("outcome")
|
||||
if isinstance(target, str) and target in petitions:
|
||||
if outcome == "voided_shutdown":
|
||||
petitions[target]["status"] = "voided_appeal"
|
||||
elif outcome == "resumed":
|
||||
petitions[target]["status"] = "executing"
|
||||
# execution_at restored by the producer who emitted
|
||||
# the resolve event with a fresh execution_at field.
|
||||
new_exec = p.get("resumed_execution_at")
|
||||
try:
|
||||
petitions[target]["execution_at"] = float(new_exec)
|
||||
except (TypeError, ValueError):
|
||||
petitions[target]["execution_at"] = None
|
||||
|
||||
elif et == "gate_shutdown_execute":
|
||||
pid = p.get("petition_id")
|
||||
if isinstance(pid, str) and pid in petitions:
|
||||
petitions[pid]["status"] = "executed"
|
||||
|
||||
executed = any(p["status"] == "executed" for p in petitions.values())
|
||||
pending_pid = None
|
||||
pending = None
|
||||
for pid, p in petitions.items():
|
||||
if p["status"] in ("filed", "executing", "appealed"):
|
||||
pending_pid = pid
|
||||
pending = p
|
||||
break
|
||||
|
||||
return ShutdownState(
|
||||
has_pending=pending is not None,
|
||||
pending_petition_id=pending_pid,
|
||||
pending_status=pending["status"] if pending else None,
|
||||
execution_at=pending["execution_at"] if pending else None,
|
||||
executed=executed,
|
||||
)
|
||||
|
||||
|
||||
def validate_shutdown_filing(
|
||||
gate_id: str,
|
||||
filer_id: str,
|
||||
*,
|
||||
reason: str,
|
||||
evidence_hashes: list[str],
|
||||
chain: Iterable[dict[str, Any]],
|
||||
now: float,
|
||||
filer_cooldown_until: float | None = None,
|
||||
) -> FilingValidation:
|
||||
"""Pre-emit validation for ``gate_shutdown_file``.
|
||||
|
||||
Critical Sprint 6 invariant: shutdown filings REQUIRE the gate to
|
||||
currently be suspended. This is the spec's two-tier escalation
|
||||
safeguard — a gate cannot be shut down without first surviving a
|
||||
suspension period.
|
||||
"""
|
||||
chain_list = list(chain)
|
||||
if not isinstance(reason, str) or not reason.strip():
|
||||
return FilingValidation(False, "reason_empty")
|
||||
if not isinstance(evidence_hashes, list) or not evidence_hashes:
|
||||
return FilingValidation(False, "evidence_required")
|
||||
if get_gate_meta(gate_id, chain_list) is None:
|
||||
return FilingValidation(False, "gate_not_found")
|
||||
|
||||
suspension = compute_suspension_state(gate_id, chain_list, now=now)
|
||||
if suspension.status == "shutdown":
|
||||
return FilingValidation(False, "gate_already_shutdown")
|
||||
if suspension.status != "suspended":
|
||||
return FilingValidation(False, "gate_not_suspended")
|
||||
|
||||
shutdown = compute_shutdown_state(gate_id, chain_list, now=now)
|
||||
if shutdown.has_pending:
|
||||
return FilingValidation(False, "shutdown_already_pending")
|
||||
if filer_cooldown_until is not None and filer_cooldown_until > now:
|
||||
return FilingValidation(False, "filer_cooldown_active")
|
||||
_ = filer_id
|
||||
return FilingValidation(True, "ok")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ShutdownState",
|
||||
"compute_shutdown_state",
|
||||
"validate_shutdown_filing",
|
||||
]
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Tier 1: 30-day reversible suspend.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §5.5 steps 1-4.
|
||||
|
||||
State derivation:
|
||||
|
||||
- A gate is "suspended" iff:
|
||||
- the most recent ``gate_suspend_execute`` event is more recent
|
||||
than any ``gate_unsuspend`` or ``gate_shutdown_execute`` event,
|
||||
- AND the suspended_until window has not yet elapsed.
|
||||
- ``compute_suspension_state`` returns the current suspension status
|
||||
including the auto-unsuspend timestamp.
|
||||
- ``validate_suspend_filing`` is the pre-emit check the UI should use
|
||||
before letting a node sign a ``gate_suspend_file`` event.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
from services.infonet.gates.state import events_for_gate, get_gate_meta
|
||||
|
||||
|
||||
_SECONDS_PER_DAY = 86400.0
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SuspensionState:
|
||||
"""``status`` is one of ``"active"``, ``"suspended"``,
|
||||
``"shutdown"``. ``suspended_until`` is the auto-unsuspend
|
||||
timestamp or ``None`` when not currently suspended."""
|
||||
status: str
|
||||
suspended_at: float | None
|
||||
suspended_until: float | None
|
||||
last_shutdown_petition_at: float | None
|
||||
"""Used for 90-day cooldown checks on subsequent shutdown petitions."""
|
||||
|
||||
|
||||
def compute_suspension_state(
|
||||
gate_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float,
|
||||
) -> SuspensionState:
|
||||
chain_list = list(chain)
|
||||
events = events_for_gate(gate_id, chain_list)
|
||||
|
||||
last_shutdown_filed_ts: float | None = None
|
||||
last_shutdown_executed_ts: float | None = None
|
||||
suspended_at: float | None = None
|
||||
last_unsuspend_ts: float | None = None
|
||||
|
||||
for ev in events:
|
||||
et = ev.get("event_type")
|
||||
ts = float(ev.get("timestamp") or 0.0)
|
||||
if et == "gate_suspend_execute":
|
||||
suspended_at = ts
|
||||
elif et == "gate_unsuspend":
|
||||
last_unsuspend_ts = ts
|
||||
elif et == "gate_shutdown_file":
|
||||
last_shutdown_filed_ts = ts
|
||||
elif et == "gate_shutdown_execute":
|
||||
last_shutdown_executed_ts = ts
|
||||
|
||||
if last_shutdown_executed_ts is not None:
|
||||
return SuspensionState(
|
||||
status="shutdown",
|
||||
suspended_at=suspended_at,
|
||||
suspended_until=None,
|
||||
last_shutdown_petition_at=last_shutdown_filed_ts,
|
||||
)
|
||||
|
||||
if suspended_at is None:
|
||||
return SuspensionState(
|
||||
status="active",
|
||||
suspended_at=None,
|
||||
suspended_until=None,
|
||||
last_shutdown_petition_at=last_shutdown_filed_ts,
|
||||
)
|
||||
|
||||
if last_unsuspend_ts is not None and last_unsuspend_ts > suspended_at:
|
||||
return SuspensionState(
|
||||
status="active",
|
||||
suspended_at=None,
|
||||
suspended_until=None,
|
||||
last_shutdown_petition_at=last_shutdown_filed_ts,
|
||||
)
|
||||
|
||||
duration = float(CONFIG["gate_suspend_duration_days"]) * _SECONDS_PER_DAY
|
||||
suspended_until = suspended_at + duration
|
||||
|
||||
if now >= suspended_until:
|
||||
# Window auto-elapsed; even without an explicit gate_unsuspend
|
||||
# event, the gate is logically active again.
|
||||
return SuspensionState(
|
||||
status="active",
|
||||
suspended_at=None,
|
||||
suspended_until=None,
|
||||
last_shutdown_petition_at=last_shutdown_filed_ts,
|
||||
)
|
||||
|
||||
return SuspensionState(
|
||||
status="suspended",
|
||||
suspended_at=suspended_at,
|
||||
suspended_until=suspended_until,
|
||||
last_shutdown_petition_at=last_shutdown_filed_ts,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FilingValidation:
|
||||
accepted: bool
|
||||
reason: str
|
||||
|
||||
|
||||
def validate_suspend_filing(
|
||||
gate_id: str,
|
||||
filer_id: str,
|
||||
*,
|
||||
reason: str,
|
||||
evidence_hashes: list[str],
|
||||
chain: Iterable[dict[str, Any]],
|
||||
now: float,
|
||||
filer_cooldown_until: float | None = None,
|
||||
) -> FilingValidation:
|
||||
"""Pre-emit validation for a ``gate_suspend_file`` event.
|
||||
|
||||
Rejects if:
|
||||
- Reason is empty.
|
||||
- No evidence hashes.
|
||||
- Gate doesn't exist.
|
||||
- Gate is already suspended or shut down.
|
||||
- Filer's cooldown is still active.
|
||||
- Gate's 90-day shutdown-petition cooldown is active.
|
||||
"""
|
||||
chain_list = list(chain)
|
||||
if not isinstance(reason, str) or not reason.strip():
|
||||
return FilingValidation(False, "reason_empty")
|
||||
if not isinstance(evidence_hashes, list) or not evidence_hashes:
|
||||
return FilingValidation(False, "evidence_required")
|
||||
if not all(isinstance(h, str) and h for h in evidence_hashes):
|
||||
return FilingValidation(False, "evidence_hashes_invalid")
|
||||
if get_gate_meta(gate_id, chain_list) is None:
|
||||
return FilingValidation(False, "gate_not_found")
|
||||
state = compute_suspension_state(gate_id, chain_list, now=now)
|
||||
if state.status == "shutdown":
|
||||
return FilingValidation(False, "gate_shutdown")
|
||||
if state.status == "suspended":
|
||||
return FilingValidation(False, "already_suspended")
|
||||
if filer_cooldown_until is not None and filer_cooldown_until > now:
|
||||
return FilingValidation(False, "filer_cooldown_active")
|
||||
if state.last_shutdown_petition_at is not None:
|
||||
cooldown_s = float(CONFIG["gate_shutdown_cooldown_days"]) * _SECONDS_PER_DAY
|
||||
if now < state.last_shutdown_petition_at + cooldown_s:
|
||||
return FilingValidation(False, "gate_cooldown_active")
|
||||
_ = filer_id # producer logs filer separately; not consulted for validation here.
|
||||
return FilingValidation(True, "ok")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FilingValidation",
|
||||
"SuspensionState",
|
||||
"compute_suspension_state",
|
||||
"validate_suspend_filing",
|
||||
]
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Common chain helpers shared across the gates package.
|
||||
|
||||
The legacy ``gate_create`` event is owned by mesh_schema (it predates
|
||||
the economy layer). Sprint 6 reads those events and extracts the
|
||||
structured fields it needs from the ``rules`` payload, with sensible
|
||||
defaults when a key is missing — same pattern the rest of the
|
||||
protocol uses for forward compatibility.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
def _gate_id(event: dict[str, Any]) -> str:
|
||||
p = _payload(event)
|
||||
gid = p.get("gate_id") or p.get("gate")
|
||||
return str(gid) if isinstance(gid, str) else ""
|
||||
|
||||
|
||||
def events_for_gate(
|
||||
gate_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""All events that reference ``gate_id``, sorted by chain order."""
|
||||
out: list[dict[str, Any]] = []
|
||||
for ev in chain:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
if _gate_id(ev) == gate_id:
|
||||
out.append(ev)
|
||||
out.sort(key=lambda e: (float(e.get("timestamp") or 0.0), int(e.get("sequence") or 0)))
|
||||
return out
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GateMeta:
|
||||
"""Static metadata extracted from the original ``gate_create`` event."""
|
||||
gate_id: str
|
||||
creator_node_id: str
|
||||
display_name: str
|
||||
entry_sacrifice: int
|
||||
min_overall_rep: int
|
||||
min_gate_rep: dict[str, int]
|
||||
created_at: float
|
||||
raw_rules: dict[str, Any]
|
||||
|
||||
|
||||
def _safe_int(val: Any, default: int = 0) -> int:
|
||||
try:
|
||||
if isinstance(val, bool):
|
||||
return default
|
||||
return int(val)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def get_gate_meta(
|
||||
gate_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> GateMeta | None:
|
||||
"""Return the gate's static metadata, or ``None`` if no
|
||||
``gate_create`` event exists for it on the chain.
|
||||
|
||||
Multiple ``gate_create`` events with the same gate_id are unusual
|
||||
but possible at peer-gossip ingestion time; the FIRST one wins
|
||||
(same first-write-wins pattern as ``find_snapshot``). Subsequent
|
||||
forgeries are ignored.
|
||||
"""
|
||||
for ev in chain:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
if ev.get("event_type") != "gate_create":
|
||||
continue
|
||||
if _gate_id(ev) != gate_id:
|
||||
continue
|
||||
p = _payload(ev)
|
||||
rules = p.get("rules")
|
||||
if not isinstance(rules, dict):
|
||||
rules = {}
|
||||
cross_gate = rules.get("min_gate_rep")
|
||||
if not isinstance(cross_gate, dict):
|
||||
cross_gate = {}
|
||||
return GateMeta(
|
||||
gate_id=gate_id,
|
||||
creator_node_id=str(ev.get("node_id") or ""),
|
||||
display_name=str(p.get("display_name") or ""),
|
||||
entry_sacrifice=_safe_int(rules.get("entry_sacrifice"), 0),
|
||||
min_overall_rep=_safe_int(rules.get("min_overall_rep"), 0),
|
||||
min_gate_rep={
|
||||
str(k): _safe_int(v, 0)
|
||||
for k, v in cross_gate.items()
|
||||
if isinstance(k, str) and k
|
||||
},
|
||||
created_at=float(ev.get("timestamp") or 0.0),
|
||||
raw_rules=dict(rules),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GateMeta",
|
||||
"events_for_gate",
|
||||
"get_gate_meta",
|
||||
]
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Governance — petitions, declarative DSL executor, constitutional
|
||||
challenge, and upgrade-hash governance (Sprint 7).
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.15, §5.4, §5.6.
|
||||
|
||||
The DSL executor is the centerpiece of Sprint 7. It is intentionally
|
||||
**not a sandbox**: it cannot run arbitrary code, period. The four
|
||||
allowed payload types (UPDATE_PARAM / BATCH_UPDATE_PARAMS /
|
||||
ENABLE_FEATURE / DISABLE_FEATURE) are dispatched as plain Python
|
||||
switch cases. There is NO ``eval``, ``exec``, ``compile``, or
|
||||
dynamic attribute access anywhere in the executor. The whole class
|
||||
of code-injection attacks goes away by design.
|
||||
|
||||
Protocol upgrades that need new logic use upgrade-hash governance —
|
||||
nodes vote on a software release hash, not on-chain code.
|
||||
"""
|
||||
|
||||
from services.infonet.governance.challenge import (
|
||||
ChallengeState,
|
||||
compute_challenge_state,
|
||||
validate_challenge_filing,
|
||||
)
|
||||
from services.infonet.governance.dsl_executor import (
|
||||
DSLExecutionResult,
|
||||
apply_petition_payload,
|
||||
forbidden_attributes_check,
|
||||
)
|
||||
from services.infonet.governance.petition import (
|
||||
PetitionState,
|
||||
compute_petition_state,
|
||||
network_governance_weight,
|
||||
validate_petition_filing,
|
||||
)
|
||||
from services.infonet.governance.upgrade_hash import (
|
||||
HeavyNodeReadinessState,
|
||||
UpgradeProposalState,
|
||||
compute_upgrade_state,
|
||||
validate_upgrade_proposal,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ChallengeState",
|
||||
"DSLExecutionResult",
|
||||
"HeavyNodeReadinessState",
|
||||
"PetitionState",
|
||||
"UpgradeProposalState",
|
||||
"apply_petition_payload",
|
||||
"compute_challenge_state",
|
||||
"compute_petition_state",
|
||||
"compute_upgrade_state",
|
||||
"forbidden_attributes_check",
|
||||
"network_governance_weight",
|
||||
"validate_challenge_filing",
|
||||
"validate_petition_filing",
|
||||
"validate_upgrade_proposal",
|
||||
]
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Constitutional challenge — 48-hour window after a petition passes.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §5.4 step 4.
|
||||
|
||||
A challenger sacrifices ``challenge_filing_cost`` (default 25) common
|
||||
rep to file a challenge against a passed petition. The challenge then
|
||||
goes to a vote — if it succeeds (``uphold`` wins by majority oracle
|
||||
rep), the petition is voided. If it fails, the challenger loses the
|
||||
sacrificed rep and the petition proceeds to execution.
|
||||
|
||||
This module exposes:
|
||||
|
||||
- ``compute_challenge_state(petition_id, chain, *, now)`` — derives
|
||||
the challenge outcome from chain events.
|
||||
- ``validate_challenge_filing(filer_common_rep, ...)`` — pre-emit
|
||||
check.
|
||||
|
||||
Sprint 7 voting tally uses ``oracle_rep_active`` weight, same as
|
||||
petition voting itself.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
from services.infonet.reputation import compute_oracle_rep_active
|
||||
|
||||
|
||||
_HOUR_S = 3600.0
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChallengeState:
|
||||
petition_id: str
|
||||
filed: bool
|
||||
filer_id: str | None
|
||||
filed_at: float | None
|
||||
deadline: float | None
|
||||
uphold_weight: float
|
||||
void_weight: float
|
||||
outcome: str # "voided" | "rejected" | "pending" | "none"
|
||||
|
||||
|
||||
def compute_challenge_state(
|
||||
petition_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float,
|
||||
) -> ChallengeState:
|
||||
chain_list = [e for e in chain if isinstance(e, dict)]
|
||||
|
||||
file_event = None
|
||||
vote_events: list[dict[str, Any]] = []
|
||||
for ev in chain_list:
|
||||
if _payload(ev).get("petition_id") != petition_id:
|
||||
continue
|
||||
et = ev.get("event_type")
|
||||
if et == "challenge_file":
|
||||
if file_event is None:
|
||||
file_event = ev
|
||||
elif et == "challenge_vote":
|
||||
vote_events.append(ev)
|
||||
|
||||
if file_event is None:
|
||||
return ChallengeState(
|
||||
petition_id=petition_id, filed=False,
|
||||
filer_id=None, filed_at=None, deadline=None,
|
||||
uphold_weight=0.0, void_weight=0.0, outcome="none",
|
||||
)
|
||||
|
||||
filed_at = float(file_event.get("timestamp") or 0.0)
|
||||
deadline = filed_at + float(CONFIG["challenge_window_hours"]) * _HOUR_S
|
||||
|
||||
state = ChallengeState(
|
||||
petition_id=petition_id, filed=True,
|
||||
filer_id=str(file_event.get("node_id") or ""),
|
||||
filed_at=filed_at, deadline=deadline,
|
||||
uphold_weight=0.0, void_weight=0.0,
|
||||
outcome="pending",
|
||||
)
|
||||
|
||||
seen: dict[str, str] = {}
|
||||
cache: dict[str, float] = {}
|
||||
for ev in sorted(vote_events,
|
||||
key=lambda e: (float(e.get("timestamp") or 0.0),
|
||||
int(e.get("sequence") or 0))):
|
||||
voter = ev.get("node_id")
|
||||
if not isinstance(voter, str) or not voter or voter in seen:
|
||||
continue
|
||||
ts = float(ev.get("timestamp") or 0.0)
|
||||
if ts < filed_at or ts > deadline:
|
||||
continue
|
||||
vote = _payload(ev).get("vote")
|
||||
if vote not in ("uphold", "void"):
|
||||
continue
|
||||
seen[voter] = vote
|
||||
if voter not in cache:
|
||||
cache[voter] = compute_oracle_rep_active(voter, chain_list, now=ts)
|
||||
w = cache[voter]
|
||||
if vote == "uphold":
|
||||
# "uphold" means: uphold the constitutional challenge —
|
||||
# i.e. void the original petition. Per RULES §5.4 step 4:
|
||||
# "Challenge upheld → 'voided_challenge' (petition killed)".
|
||||
state.uphold_weight += w
|
||||
else: # "void" the challenge → original petition stands
|
||||
state.void_weight += w
|
||||
|
||||
if now <= deadline:
|
||||
return state # still pending
|
||||
|
||||
if state.uphold_weight > state.void_weight:
|
||||
state.outcome = "voided"
|
||||
else:
|
||||
state.outcome = "rejected"
|
||||
return state
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChallengeFilingValidation:
|
||||
accepted: bool
|
||||
reason: str
|
||||
|
||||
|
||||
def validate_challenge_filing(
|
||||
filer_common_rep: float,
|
||||
petition_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float,
|
||||
) -> ChallengeFilingValidation:
|
||||
"""Pre-emit check for a ``challenge_file`` event.
|
||||
|
||||
Rejects if:
|
||||
- Filer lacks the ``challenge_filing_cost``.
|
||||
- A challenge already exists on this petition.
|
||||
- The challenge window for the petition has elapsed (caller is
|
||||
expected to have already verified the petition's voting closed
|
||||
successfully — that timestamp comes from
|
||||
``compute_petition_state``).
|
||||
"""
|
||||
if filer_common_rep < float(CONFIG["challenge_filing_cost"]):
|
||||
return ChallengeFilingValidation(False, "insufficient_common_rep")
|
||||
state = compute_challenge_state(petition_id, list(chain), now=now)
|
||||
if state.filed:
|
||||
return ChallengeFilingValidation(False, "challenge_already_filed")
|
||||
return ChallengeFilingValidation(True, "ok")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ChallengeFilingValidation",
|
||||
"ChallengeState",
|
||||
"compute_challenge_state",
|
||||
"validate_challenge_filing",
|
||||
]
|
||||
@@ -0,0 +1,223 @@
|
||||
"""Declarative DSL executor — the type-safe, no-eval petition applier.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §1.2 (the
|
||||
governance section comment block) + §5.4 step 5.
|
||||
|
||||
CRITICAL design property: this module **cannot execute arbitrary
|
||||
code**. It is a switch over four typed payload variants, each with a
|
||||
fully-validated key/value or feature-flag operation. There is NO use
|
||||
of ``eval``, ``exec``, ``compile``, ``ast.parse``, ``getattr`` with a
|
||||
runtime key, ``__import__``, ``subprocess``, ``os.system``, or any
|
||||
other dynamic-execution primitive.
|
||||
|
||||
The whole class of code-injection attacks is eliminated by design —
|
||||
even if an attacker passes a maliciously crafted petition payload, the
|
||||
executor either applies a typed value or rejects with
|
||||
``InvalidPetition``. There is no path to executing the attacker's
|
||||
input as code.
|
||||
|
||||
Sprint 7's adversarial tests assert this invariant by reading this
|
||||
file's source bytes and confirming none of the forbidden builtins
|
||||
appear (``forbidden_attributes_check``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from services.infonet.config import (
|
||||
CONFIG,
|
||||
CONFIG_SCHEMA,
|
||||
IMMUTABLE_PRINCIPLES,
|
||||
InvalidPetition,
|
||||
validate_cross_field_invariants,
|
||||
validate_petition_value,
|
||||
)
|
||||
|
||||
|
||||
_ALLOWED_PAYLOAD_TYPES = frozenset({
|
||||
"UPDATE_PARAM",
|
||||
"BATCH_UPDATE_PARAMS",
|
||||
"ENABLE_FEATURE",
|
||||
"DISABLE_FEATURE",
|
||||
})
|
||||
|
||||
|
||||
@dataclass
|
||||
class DSLExecutionResult:
|
||||
"""Outcome of applying a petition payload.
|
||||
|
||||
``new_config`` is a fresh dict — the caller decides whether to
|
||||
swap the live ``CONFIG`` with it. Sprint 7's tests apply the
|
||||
result and verify the swap; production callers wire this through
|
||||
the ``petition_execute`` event handler.
|
||||
"""
|
||||
new_config: dict[str, Any]
|
||||
changed_keys: tuple[str, ...] = field(default_factory=tuple)
|
||||
|
||||
|
||||
def _check_payload_shape(payload: Any) -> str:
|
||||
if not isinstance(payload, dict):
|
||||
raise InvalidPetition("petition_payload must be an object")
|
||||
payload_type = payload.get("type")
|
||||
if payload_type not in _ALLOWED_PAYLOAD_TYPES:
|
||||
raise InvalidPetition(
|
||||
f"unknown petition_payload type: {payload_type!r}; "
|
||||
f"allowed: {sorted(_ALLOWED_PAYLOAD_TYPES)}"
|
||||
)
|
||||
return str(payload_type)
|
||||
|
||||
|
||||
def _check_key_writeable(key: str) -> None:
|
||||
"""Reject writes to keys not in CONFIG_SCHEMA. ``IMMUTABLE_PRINCIPLES``
|
||||
keys never appear in ``CONFIG_SCHEMA``, so this also rejects them.
|
||||
"""
|
||||
if not isinstance(key, str) or not key:
|
||||
raise InvalidPetition("CONFIG key must be a non-empty string")
|
||||
if key not in CONFIG_SCHEMA:
|
||||
# Also surface a clearer diagnostic if the user attempted to
|
||||
# mutate an IMMUTABLE_PRINCIPLES key.
|
||||
if key in IMMUTABLE_PRINCIPLES:
|
||||
raise InvalidPetition(
|
||||
f"key {key!r} is in IMMUTABLE_PRINCIPLES — only an "
|
||||
f"upgrade-hash governance hard fork can change it"
|
||||
)
|
||||
raise InvalidPetition(f"unknown CONFIG key: {key!r}")
|
||||
|
||||
|
||||
def _apply_update_param(
|
||||
payload: dict[str, Any],
|
||||
candidate: dict[str, Any],
|
||||
) -> tuple[dict[str, Any], list[str]]:
|
||||
if "key" not in payload or "value" not in payload:
|
||||
raise InvalidPetition("UPDATE_PARAM requires key + value")
|
||||
key = payload["key"]
|
||||
value = payload["value"]
|
||||
_check_key_writeable(key)
|
||||
validate_petition_value(key, value, candidate)
|
||||
candidate[key] = value
|
||||
return candidate, [key]
|
||||
|
||||
|
||||
def _apply_batch_update(
|
||||
payload: dict[str, Any],
|
||||
candidate: dict[str, Any],
|
||||
) -> tuple[dict[str, Any], list[str]]:
|
||||
updates = payload.get("updates")
|
||||
if not isinstance(updates, list) or not updates:
|
||||
raise InvalidPetition("BATCH_UPDATE_PARAMS requires a non-empty 'updates' list")
|
||||
seen_keys: set[str] = set()
|
||||
changed: list[str] = []
|
||||
for u in updates:
|
||||
if not isinstance(u, dict) or "key" not in u or "value" not in u:
|
||||
raise InvalidPetition("BATCH_UPDATE_PARAMS entries must be {key, value}")
|
||||
key = u["key"]
|
||||
if key in seen_keys:
|
||||
raise InvalidPetition(f"duplicate key in BATCH_UPDATE_PARAMS: {key!r}")
|
||||
seen_keys.add(key)
|
||||
_check_key_writeable(key)
|
||||
validate_petition_value(key, u["value"], candidate)
|
||||
candidate[key] = u["value"]
|
||||
changed.append(key)
|
||||
return candidate, changed
|
||||
|
||||
|
||||
def _apply_feature_toggle(
|
||||
payload: dict[str, Any],
|
||||
candidate: dict[str, Any],
|
||||
*,
|
||||
enable: bool,
|
||||
) -> tuple[dict[str, Any], list[str]]:
|
||||
feature = payload.get("feature")
|
||||
if not isinstance(feature, str) or not feature:
|
||||
raise InvalidPetition("ENABLE_FEATURE / DISABLE_FEATURE requires non-empty 'feature'")
|
||||
_check_key_writeable(feature)
|
||||
schema = CONFIG_SCHEMA.get(feature)
|
||||
if schema is None or schema.get("type") != "bool":
|
||||
raise InvalidPetition(
|
||||
f"feature {feature!r} is not a boolean CONFIG key"
|
||||
)
|
||||
candidate[feature] = bool(enable)
|
||||
return candidate, [feature]
|
||||
|
||||
|
||||
def apply_petition_payload(
|
||||
payload: dict[str, Any],
|
||||
current_config: dict[str, Any] | None = None,
|
||||
) -> DSLExecutionResult:
|
||||
"""Apply a validated petition payload to a CANDIDATE copy of CONFIG.
|
||||
|
||||
Transactional: validation runs against the candidate; if any check
|
||||
fails, the candidate is discarded and ``InvalidPetition`` is
|
||||
raised. The live ``CONFIG`` is never partially mutated.
|
||||
|
||||
Pass ``current_config`` when applying against a hypothetical state
|
||||
(testing, upgrade-hash dry-runs). Otherwise the live ``CONFIG`` is
|
||||
deep-copied as the starting point.
|
||||
"""
|
||||
payload_type = _check_payload_shape(payload)
|
||||
candidate = deepcopy(current_config) if current_config is not None else deepcopy(CONFIG)
|
||||
|
||||
if payload_type == "UPDATE_PARAM":
|
||||
candidate, changed = _apply_update_param(payload, candidate)
|
||||
elif payload_type == "BATCH_UPDATE_PARAMS":
|
||||
candidate, changed = _apply_batch_update(payload, candidate)
|
||||
elif payload_type == "ENABLE_FEATURE":
|
||||
candidate, changed = _apply_feature_toggle(payload, candidate, enable=True)
|
||||
elif payload_type == "DISABLE_FEATURE":
|
||||
candidate, changed = _apply_feature_toggle(payload, candidate, enable=False)
|
||||
else: # pragma: no cover — _check_payload_shape gated this
|
||||
raise InvalidPetition(f"unhandled payload type: {payload_type}")
|
||||
|
||||
# Cross-field invariants validated against the FINAL candidate.
|
||||
validate_cross_field_invariants(candidate)
|
||||
|
||||
return DSLExecutionResult(new_config=candidate, changed_keys=tuple(changed))
|
||||
|
||||
|
||||
# ─── No-eval guard ──────────────────────────────────────────────────────
|
||||
|
||||
# Forbidden attribute names whose presence in this module's source
|
||||
# would violate the "no arbitrary code execution" property. Sprint 7's
|
||||
# adversarial test reads this file and asserts none of these substrings
|
||||
# appear (outside of this list and the guard function below — the
|
||||
# guard's job is to *name* the forbidden surface, not use it).
|
||||
|
||||
_FORBIDDEN_ATTRIBUTES: frozenset[str] = frozenset({
|
||||
# Call-syntax tokens. Scanned against this module's source by the
|
||||
# Sprint 7 adversarial test. Bare module names (``subprocess``,
|
||||
# ``os``, etc.) are deliberately NOT in this set — their mere
|
||||
# mention in prose is harmless; what we forbid is the CALL.
|
||||
"eval(",
|
||||
"exec(",
|
||||
"compile(",
|
||||
"__import__(",
|
||||
"ast.parse(",
|
||||
"subprocess.run(",
|
||||
"subprocess.Popen(",
|
||||
"subprocess.call(",
|
||||
"subprocess.check_output(",
|
||||
"os.system(",
|
||||
"os.popen(",
|
||||
"pickle.loads(",
|
||||
"marshal.loads(",
|
||||
})
|
||||
|
||||
|
||||
def forbidden_attributes_check() -> tuple[str, ...]:
|
||||
"""Return the curated list of forbidden surface names.
|
||||
|
||||
Used by the Sprint 7 adversarial test to scan this module's source
|
||||
for any forbidden token. Exposed as a function so the test stays
|
||||
decoupled from the module's internal layout.
|
||||
"""
|
||||
return tuple(sorted(_FORBIDDEN_ATTRIBUTES))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DSLExecutionResult",
|
||||
"apply_petition_payload",
|
||||
"forbidden_attributes_check",
|
||||
]
|
||||
@@ -0,0 +1,315 @@
|
||||
"""Petition state machine — pure function over chain history.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.15, §5.4.
|
||||
|
||||
State diagram:
|
||||
|
||||
petition_file
|
||||
│
|
||||
▼ status="signatures"
|
||||
petition_sign × N (collect signature_governance_weight)
|
||||
│
|
||||
▼ if signature_governance_weight ≥ 25% × network → status="voting"
|
||||
│ if 14 days elapsed and threshold not met → status="failed_signatures"
|
||||
petition_vote × N (oracle_rep_active weighted)
|
||||
│
|
||||
▼ if 7 days elapsed:
|
||||
│ check quorum (30%) + supermajority (67%)
|
||||
│ status="challenge" (passed) or "failed_vote"
|
||||
challenge_file (optional, 48h window) + challenge_vote × N
|
||||
│
|
||||
▼ if challenge passes → status="voided_challenge"
|
||||
│ else → status="passed"
|
||||
petition_execute → status="executed"
|
||||
|
||||
Voting weights use ``oracle_rep_active`` (governance-decayed) per
|
||||
RULES §3.15. Total network weight is the sum across all nodes
|
||||
referenced by signature/vote events plus any node with chain
|
||||
activity (we use the union of acting nodes' weights — same as
|
||||
``compute_network_governance_weight``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
from services.infonet.reputation import compute_oracle_rep_active
|
||||
|
||||
|
||||
_DAY_S = 86400.0
|
||||
_HOUR_S = 3600.0
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class PetitionState:
|
||||
petition_id: str
|
||||
status: str # "signatures" | "voting" | "challenge" | "passed" |
|
||||
# "executed" | "failed_signatures" | "failed_vote" |
|
||||
# "voided_challenge" | "not_found"
|
||||
filer_id: str
|
||||
filed_at: float
|
||||
petition_payload: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
signature_governance_weight: float = 0.0
|
||||
signature_threshold_at_filing: float = 0.0
|
||||
|
||||
votes_for_weight: float = 0.0
|
||||
votes_against_weight: float = 0.0
|
||||
|
||||
voting_started_at: float | None = None
|
||||
voting_deadline: float | None = None
|
||||
challenge_window_until: float | None = None
|
||||
|
||||
|
||||
def _governance_weight_provider(
|
||||
node_id: str,
|
||||
chain: list[dict[str, Any]],
|
||||
*,
|
||||
at: float,
|
||||
cache: dict[str, float],
|
||||
) -> float:
|
||||
"""Memoize per-call: governance weight for ``node_id`` evaluated at
|
||||
chain time ``at``. Cached because petitions iterate signatures and
|
||||
votes from many nodes, and recomputing oracle rep per call is
|
||||
expensive on long chains."""
|
||||
key = node_id
|
||||
if key in cache:
|
||||
return cache[key]
|
||||
w = compute_oracle_rep_active(node_id, chain, now=at)
|
||||
cache[key] = w
|
||||
return w
|
||||
|
||||
|
||||
def network_governance_weight(
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float,
|
||||
) -> float:
|
||||
"""Total network ``oracle_rep_active`` at chain time ``now``.
|
||||
|
||||
Sum across every node that has authored at least one event on the
|
||||
chain. Matches RULES §3.15: "sum(node.oracle_rep_active for all
|
||||
nodes)". Newly-created nodes that haven't yet signed any event
|
||||
have zero weight and contribute nothing — including them is a
|
||||
no-op.
|
||||
"""
|
||||
chain_list = [e for e in chain if isinstance(e, dict)]
|
||||
nodes: set[str] = set()
|
||||
for ev in chain_list:
|
||||
nid = ev.get("node_id")
|
||||
if isinstance(nid, str) and nid:
|
||||
nodes.add(nid)
|
||||
cache: dict[str, float] = {}
|
||||
return sum(
|
||||
_governance_weight_provider(n, chain_list, at=now, cache=cache)
|
||||
for n in nodes
|
||||
)
|
||||
|
||||
|
||||
def compute_petition_state(
|
||||
petition_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float,
|
||||
) -> PetitionState:
|
||||
"""Derive the current state of ``petition_id`` from chain history.
|
||||
|
||||
``now`` is the evaluation timestamp — pass
|
||||
``time_validity.chain_majority_time(chain)`` in production.
|
||||
"""
|
||||
chain_list = [e for e in chain if isinstance(e, dict)]
|
||||
|
||||
file_event = None
|
||||
sign_events: list[dict[str, Any]] = []
|
||||
vote_events: list[dict[str, Any]] = []
|
||||
execute_event = None
|
||||
challenge_filed_event = None
|
||||
challenge_vote_events: list[dict[str, Any]] = []
|
||||
|
||||
for ev in chain_list:
|
||||
et = ev.get("event_type")
|
||||
p = _payload(ev)
|
||||
pid = p.get("petition_id")
|
||||
if pid != petition_id:
|
||||
continue
|
||||
if et == "petition_file":
|
||||
if file_event is None: # first-write-wins
|
||||
file_event = ev
|
||||
elif et == "petition_sign":
|
||||
sign_events.append(ev)
|
||||
elif et == "petition_vote":
|
||||
vote_events.append(ev)
|
||||
elif et == "petition_execute":
|
||||
execute_event = ev
|
||||
elif et == "challenge_file":
|
||||
if challenge_filed_event is None:
|
||||
challenge_filed_event = ev
|
||||
elif et == "challenge_vote":
|
||||
challenge_vote_events.append(ev)
|
||||
|
||||
if file_event is None:
|
||||
return PetitionState(
|
||||
petition_id=petition_id, status="not_found",
|
||||
filer_id="", filed_at=0.0,
|
||||
)
|
||||
|
||||
state = PetitionState(
|
||||
petition_id=petition_id,
|
||||
status="signatures",
|
||||
filer_id=str(file_event.get("node_id") or ""),
|
||||
filed_at=float(file_event.get("timestamp") or 0.0),
|
||||
petition_payload=dict(_payload(file_event).get("petition_payload") or {}),
|
||||
)
|
||||
|
||||
cache: dict[str, float] = {}
|
||||
network_weight = network_governance_weight(chain_list, now=now)
|
||||
state.signature_threshold_at_filing = (
|
||||
network_weight * float(CONFIG["petition_signature_threshold"])
|
||||
)
|
||||
|
||||
# ── Signatures phase ──
|
||||
sign_window_s = float(CONFIG["petition_signature_window_days"]) * _DAY_S
|
||||
seen_signers: set[str] = set()
|
||||
for ev in sorted(sign_events,
|
||||
key=lambda e: (float(e.get("timestamp") or 0.0),
|
||||
int(e.get("sequence") or 0))):
|
||||
signer = ev.get("node_id")
|
||||
if not isinstance(signer, str) or not signer:
|
||||
continue
|
||||
if signer in seen_signers:
|
||||
continue
|
||||
seen_signers.add(signer)
|
||||
ts = float(ev.get("timestamp") or 0.0)
|
||||
# Only count signatures that landed within the window.
|
||||
if ts > state.filed_at + sign_window_s:
|
||||
continue
|
||||
weight = _governance_weight_provider(signer, chain_list, at=ts, cache=cache)
|
||||
state.signature_governance_weight += weight
|
||||
|
||||
if state.signature_governance_weight >= state.signature_threshold_at_filing > 0:
|
||||
# Find the timestamp the threshold was crossed (= last signature
|
||||
# that crossed it). Sprint 7 simplification: use the latest
|
||||
# signature event timestamp as the voting-phase start.
|
||||
latest_sig_ts = max((float(e.get("timestamp") or 0.0) for e in sign_events
|
||||
if e.get("node_id") in seen_signers),
|
||||
default=state.filed_at)
|
||||
state.status = "voting"
|
||||
state.voting_started_at = latest_sig_ts
|
||||
state.voting_deadline = latest_sig_ts + float(CONFIG["petition_vote_window_days"]) * _DAY_S
|
||||
else:
|
||||
if now > state.filed_at + sign_window_s:
|
||||
state.status = "failed_signatures"
|
||||
return state
|
||||
# Still collecting signatures.
|
||||
return state
|
||||
|
||||
# ── Voting phase ──
|
||||
seen_voters: dict[str, str] = {} # node_id → "for"|"against"
|
||||
for ev in sorted(vote_events,
|
||||
key=lambda e: (float(e.get("timestamp") or 0.0),
|
||||
int(e.get("sequence") or 0))):
|
||||
voter = ev.get("node_id")
|
||||
if not isinstance(voter, str) or not voter:
|
||||
continue
|
||||
if voter in seen_voters: # one vote per node — first wins
|
||||
continue
|
||||
ts = float(ev.get("timestamp") or 0.0)
|
||||
if state.voting_started_at is not None and ts < state.voting_started_at:
|
||||
continue
|
||||
if state.voting_deadline is not None and ts > state.voting_deadline:
|
||||
continue
|
||||
vote = _payload(ev).get("vote")
|
||||
if vote not in ("for", "against"):
|
||||
continue
|
||||
seen_voters[voter] = vote
|
||||
weight = _governance_weight_provider(voter, chain_list, at=ts, cache=cache)
|
||||
if vote == "for":
|
||||
state.votes_for_weight += weight
|
||||
else:
|
||||
state.votes_against_weight += weight
|
||||
|
||||
if state.voting_deadline is not None and now <= state.voting_deadline:
|
||||
# Voting still open.
|
||||
return state
|
||||
|
||||
# Voting closed — tally.
|
||||
participating = state.votes_for_weight + state.votes_against_weight
|
||||
quorum_required = network_weight * float(CONFIG["petition_quorum"])
|
||||
if participating < quorum_required:
|
||||
state.status = "failed_vote"
|
||||
return state
|
||||
if participating == 0:
|
||||
state.status = "failed_vote"
|
||||
return state
|
||||
if state.votes_for_weight / participating < float(CONFIG["petition_supermajority"]):
|
||||
state.status = "failed_vote"
|
||||
return state
|
||||
|
||||
# Petition passed the vote — enter challenge window.
|
||||
state.status = "challenge"
|
||||
state.challenge_window_until = (
|
||||
(state.voting_deadline or state.filed_at)
|
||||
+ float(CONFIG["challenge_window_hours"]) * _HOUR_S
|
||||
)
|
||||
|
||||
# ── Challenge phase ──
|
||||
from services.infonet.governance.challenge import (
|
||||
compute_challenge_state as _compute_challenge_state,
|
||||
)
|
||||
challenge_state = _compute_challenge_state(petition_id, chain_list, now=now)
|
||||
if challenge_state.outcome == "voided":
|
||||
state.status = "voided_challenge"
|
||||
return state
|
||||
if state.challenge_window_until is not None and now <= state.challenge_window_until:
|
||||
# Challenge window still open.
|
||||
return state
|
||||
|
||||
# Challenge window closed without voiding the petition.
|
||||
state.status = "passed"
|
||||
|
||||
if execute_event is not None:
|
||||
state.status = "executed"
|
||||
return state
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FilingValidation:
|
||||
accepted: bool
|
||||
reason: str
|
||||
|
||||
|
||||
def validate_petition_filing(
|
||||
filer_common_rep: float,
|
||||
*,
|
||||
petition_payload: dict[str, Any],
|
||||
) -> FilingValidation:
|
||||
"""Pre-emit check for a ``petition_file`` event.
|
||||
|
||||
The producer must verify the filer has at least
|
||||
``petition_filing_cost`` common rep available to burn. The
|
||||
payload structure is also validated up-front (cheaper to reject
|
||||
here than during execution).
|
||||
"""
|
||||
if filer_common_rep < float(CONFIG["petition_filing_cost"]):
|
||||
return FilingValidation(False, "insufficient_common_rep")
|
||||
if not isinstance(petition_payload, dict):
|
||||
return FilingValidation(False, "petition_payload_not_object")
|
||||
if "type" not in petition_payload:
|
||||
return FilingValidation(False, "petition_payload_missing_type")
|
||||
return FilingValidation(True, "ok")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FilingValidation",
|
||||
"PetitionState",
|
||||
"compute_petition_state",
|
||||
"network_governance_weight",
|
||||
"validate_petition_filing",
|
||||
]
|
||||
@@ -0,0 +1,340 @@
|
||||
"""Upgrade-hash governance — RULES §3.15 (formalization), §5.6.
|
||||
|
||||
Protocol upgrades that need new logic (formulas, event types, state
|
||||
machines) cannot be expressed as parameter changes — the declarative
|
||||
DSL has no way to ship new code. The Round 8 formalization replaces
|
||||
that gap with **upgrade-hash governance**: developers publish a
|
||||
software release, the network votes on its SHA-256 release hash, and
|
||||
nodes upgrade their software.
|
||||
|
||||
Lifecycle:
|
||||
|
||||
1. Filing (``upgrade_propose``) — 25 common rep, includes the
|
||||
``release_hash``, description, target_protocol_version.
|
||||
2. Signatures (14 days) — 25% of network ``oracle_rep_active``.
|
||||
3. Voting (14 days) — **80% supermajority + 40% quorum** (higher
|
||||
bars than param petitions).
|
||||
4. Constitutional challenge window (48 hours).
|
||||
5. Activation (30 days): Heavy Nodes that have downloaded the new
|
||||
release emit ``upgrade_signal_ready``. Once **67%** of Heavy
|
||||
Nodes have signaled, the upgrade activates and ``protocol_version``
|
||||
increments.
|
||||
6. Failure modes: ``failed_signatures``, ``failed_vote``,
|
||||
``voided_challenge``, ``failed_activation`` (≥33% of Heavy Nodes
|
||||
couldn't or wouldn't upgrade — network not ready).
|
||||
|
||||
Heavy-Node detection: a node is "Heavy" if its transport tier is
|
||||
``private_strong`` per IMPLEMENTATION_PLAN §3.5. For Sprint 7's pure
|
||||
chain-only computation, we rely on the producer to mark
|
||||
``upgrade_signal_ready`` events with ``release_hash`` matching the
|
||||
proposal — and only Heavy Nodes can emit that event in production
|
||||
(producer-side enforcement; this module verifies the chain-derived
|
||||
state).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
from services.infonet.reputation import compute_oracle_rep_active
|
||||
|
||||
|
||||
_DAY_S = 86400.0
|
||||
_HOUR_S = 3600.0
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class HeavyNodeReadinessState:
|
||||
total_heavy_nodes: int
|
||||
ready_count: int
|
||||
fraction: float
|
||||
threshold_met: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class UpgradeProposalState:
|
||||
proposal_id: str
|
||||
status: str # "signatures" | "voting" | "challenge" | "activation" |
|
||||
# "activated" | "failed_signatures" | "failed_vote" |
|
||||
# "voided_challenge" | "failed_activation" | "not_found"
|
||||
proposer_id: str
|
||||
filed_at: float
|
||||
release_hash: str = ""
|
||||
target_protocol_version: str = ""
|
||||
signature_governance_weight: float = 0.0
|
||||
votes_for_weight: float = 0.0
|
||||
votes_against_weight: float = 0.0
|
||||
voting_started_at: float | None = None
|
||||
voting_deadline: float | None = None
|
||||
challenge_window_until: float | None = None
|
||||
activation_deadline: float | None = None
|
||||
readiness: HeavyNodeReadinessState = field(
|
||||
default_factory=lambda: HeavyNodeReadinessState(0, 0, 0.0, False),
|
||||
)
|
||||
|
||||
|
||||
def compute_upgrade_state(
|
||||
proposal_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float,
|
||||
heavy_node_ids: set[str] | None = None,
|
||||
) -> UpgradeProposalState:
|
||||
"""Derive the proposal's current state from chain events.
|
||||
|
||||
``heavy_node_ids`` is the set of nodes the caller knows to be
|
||||
Heavy at chain time ``now``. Production callers compute this from
|
||||
`wormhole_supervisor.get_transport_tier()` × the chain's known
|
||||
nodes. Tests pass an explicit set.
|
||||
"""
|
||||
chain_list = [e for e in chain if isinstance(e, dict)]
|
||||
heavy_set = set(heavy_node_ids) if heavy_node_ids is not None else set()
|
||||
|
||||
propose_event = None
|
||||
sign_events: list[dict[str, Any]] = []
|
||||
vote_events: list[dict[str, Any]] = []
|
||||
challenge_event = None
|
||||
challenge_vote_events: list[dict[str, Any]] = []
|
||||
signal_ready_events: list[dict[str, Any]] = []
|
||||
activate_event = None
|
||||
|
||||
for ev in chain_list:
|
||||
et = ev.get("event_type")
|
||||
p = _payload(ev)
|
||||
pid = p.get("proposal_id")
|
||||
if pid != proposal_id:
|
||||
continue
|
||||
if et == "upgrade_propose":
|
||||
if propose_event is None:
|
||||
propose_event = ev
|
||||
elif et == "upgrade_sign":
|
||||
sign_events.append(ev)
|
||||
elif et == "upgrade_vote":
|
||||
vote_events.append(ev)
|
||||
elif et == "upgrade_challenge":
|
||||
if challenge_event is None:
|
||||
challenge_event = ev
|
||||
elif et == "upgrade_challenge_vote":
|
||||
challenge_vote_events.append(ev)
|
||||
elif et == "upgrade_signal_ready":
|
||||
signal_ready_events.append(ev)
|
||||
elif et == "upgrade_activate":
|
||||
activate_event = ev
|
||||
|
||||
if propose_event is None:
|
||||
return UpgradeProposalState(
|
||||
proposal_id=proposal_id, status="not_found",
|
||||
proposer_id="", filed_at=0.0,
|
||||
)
|
||||
|
||||
pp = _payload(propose_event)
|
||||
state = UpgradeProposalState(
|
||||
proposal_id=proposal_id,
|
||||
status="signatures",
|
||||
proposer_id=str(propose_event.get("node_id") or ""),
|
||||
filed_at=float(propose_event.get("timestamp") or 0.0),
|
||||
release_hash=str(pp.get("release_hash") or ""),
|
||||
target_protocol_version=str(pp.get("target_protocol_version") or ""),
|
||||
)
|
||||
|
||||
cache: dict[str, float] = {}
|
||||
|
||||
def _w(node_id: str, at: float) -> float:
|
||||
if node_id not in cache:
|
||||
cache[node_id] = compute_oracle_rep_active(node_id, chain_list, now=at)
|
||||
return cache[node_id]
|
||||
|
||||
# Network weight at "now" — used for signature + quorum thresholds.
|
||||
nodes: set[str] = set()
|
||||
for ev in chain_list:
|
||||
nid = ev.get("node_id")
|
||||
if isinstance(nid, str) and nid:
|
||||
nodes.add(nid)
|
||||
network_weight = sum(_w(n, now) for n in nodes)
|
||||
sig_threshold = network_weight * float(CONFIG["upgrade_signature_threshold"])
|
||||
|
||||
# ── Signatures ──
|
||||
sig_window_s = float(CONFIG["upgrade_signature_window_days"]) * _DAY_S
|
||||
seen_sig: set[str] = set()
|
||||
for ev in sorted(sign_events,
|
||||
key=lambda e: (float(e.get("timestamp") or 0.0),
|
||||
int(e.get("sequence") or 0))):
|
||||
signer = ev.get("node_id")
|
||||
if not isinstance(signer, str) or signer in seen_sig:
|
||||
continue
|
||||
ts = float(ev.get("timestamp") or 0.0)
|
||||
if ts > state.filed_at + sig_window_s:
|
||||
continue
|
||||
seen_sig.add(signer)
|
||||
state.signature_governance_weight += _w(signer, ts)
|
||||
|
||||
if state.signature_governance_weight >= sig_threshold > 0:
|
||||
latest_sig_ts = max((float(e.get("timestamp") or 0.0) for e in sign_events
|
||||
if e.get("node_id") in seen_sig),
|
||||
default=state.filed_at)
|
||||
state.status = "voting"
|
||||
state.voting_started_at = latest_sig_ts
|
||||
state.voting_deadline = latest_sig_ts + float(CONFIG["upgrade_vote_window_days"]) * _DAY_S
|
||||
else:
|
||||
if now > state.filed_at + sig_window_s:
|
||||
state.status = "failed_signatures"
|
||||
return state
|
||||
return state
|
||||
|
||||
# ── Voting ──
|
||||
seen_voters: dict[str, str] = {}
|
||||
for ev in sorted(vote_events,
|
||||
key=lambda e: (float(e.get("timestamp") or 0.0),
|
||||
int(e.get("sequence") or 0))):
|
||||
voter = ev.get("node_id")
|
||||
if not isinstance(voter, str) or voter in seen_voters:
|
||||
continue
|
||||
ts = float(ev.get("timestamp") or 0.0)
|
||||
if state.voting_started_at is None or ts < state.voting_started_at:
|
||||
continue
|
||||
if state.voting_deadline is None or ts > state.voting_deadline:
|
||||
continue
|
||||
vote = _payload(ev).get("vote")
|
||||
if vote not in ("for", "against"):
|
||||
continue
|
||||
seen_voters[voter] = vote
|
||||
w = _w(voter, ts)
|
||||
if vote == "for":
|
||||
state.votes_for_weight += w
|
||||
else:
|
||||
state.votes_against_weight += w
|
||||
|
||||
if state.voting_deadline is not None and now <= state.voting_deadline:
|
||||
return state
|
||||
|
||||
participating = state.votes_for_weight + state.votes_against_weight
|
||||
quorum_required = network_weight * float(CONFIG["upgrade_quorum"])
|
||||
if participating < quorum_required or participating == 0:
|
||||
state.status = "failed_vote"
|
||||
return state
|
||||
if state.votes_for_weight / participating < float(CONFIG["upgrade_supermajority"]):
|
||||
state.status = "failed_vote"
|
||||
return state
|
||||
|
||||
# Vote passed — challenge window.
|
||||
state.status = "challenge"
|
||||
state.challenge_window_until = (
|
||||
(state.voting_deadline or state.filed_at)
|
||||
+ float(CONFIG["upgrade_challenge_window_hours"]) * _HOUR_S
|
||||
)
|
||||
|
||||
# Process upgrade_challenge_vote — uphold-majority voids the proposal.
|
||||
if challenge_event is not None:
|
||||
uphold_w = 0.0
|
||||
void_w = 0.0
|
||||
seen_cv: dict[str, str] = {}
|
||||
for ev in sorted(challenge_vote_events,
|
||||
key=lambda e: (float(e.get("timestamp") or 0.0),
|
||||
int(e.get("sequence") or 0))):
|
||||
voter = ev.get("node_id")
|
||||
if not isinstance(voter, str) or voter in seen_cv:
|
||||
continue
|
||||
ts = float(ev.get("timestamp") or 0.0)
|
||||
challenge_at = float(challenge_event.get("timestamp") or 0.0)
|
||||
if ts < challenge_at or ts > (state.challenge_window_until or 0.0):
|
||||
continue
|
||||
vote = _payload(ev).get("vote")
|
||||
if vote not in ("uphold", "void"):
|
||||
continue
|
||||
seen_cv[voter] = vote
|
||||
w = _w(voter, ts)
|
||||
if vote == "uphold":
|
||||
uphold_w += w
|
||||
else:
|
||||
void_w += w
|
||||
if (state.challenge_window_until is not None
|
||||
and now > state.challenge_window_until
|
||||
and uphold_w > void_w):
|
||||
state.status = "voided_challenge"
|
||||
return state
|
||||
|
||||
if state.challenge_window_until is not None and now <= state.challenge_window_until:
|
||||
return state
|
||||
|
||||
# Challenge cleared → activation phase.
|
||||
state.status = "activation"
|
||||
state.activation_deadline = (
|
||||
(state.challenge_window_until or state.filed_at)
|
||||
+ float(CONFIG["upgrade_activation_window_days"]) * _DAY_S
|
||||
)
|
||||
|
||||
# ── Heavy-Node readiness ──
|
||||
seen_ready: set[str] = set()
|
||||
for ev in signal_ready_events:
|
||||
node = ev.get("node_id")
|
||||
if not isinstance(node, str) or node in seen_ready:
|
||||
continue
|
||||
if node not in heavy_set:
|
||||
continue # only Heavy Nodes can signal
|
||||
if _payload(ev).get("release_hash") != state.release_hash:
|
||||
continue
|
||||
seen_ready.add(node)
|
||||
total_heavy = max(len(heavy_set), 1)
|
||||
fraction = len(seen_ready) / total_heavy if heavy_set else 0.0
|
||||
threshold = float(CONFIG["upgrade_activation_threshold"])
|
||||
state.readiness = HeavyNodeReadinessState(
|
||||
total_heavy_nodes=len(heavy_set),
|
||||
ready_count=len(seen_ready),
|
||||
fraction=fraction,
|
||||
threshold_met=fraction >= threshold,
|
||||
)
|
||||
|
||||
if activate_event is not None:
|
||||
state.status = "activated"
|
||||
return state
|
||||
|
||||
if state.readiness.threshold_met:
|
||||
# Producer can emit upgrade_activate now — until then the
|
||||
# status is "activation" with threshold_met=True so the UI
|
||||
# can prompt.
|
||||
return state
|
||||
|
||||
if state.activation_deadline is not None and now > state.activation_deadline:
|
||||
state.status = "failed_activation"
|
||||
return state
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UpgradeFilingValidation:
|
||||
accepted: bool
|
||||
reason: str
|
||||
|
||||
|
||||
def validate_upgrade_proposal(
|
||||
filer_common_rep: float,
|
||||
*,
|
||||
release_hash: str,
|
||||
release_description: str,
|
||||
target_protocol_version: str,
|
||||
) -> UpgradeFilingValidation:
|
||||
"""Pre-emit check for ``upgrade_propose``."""
|
||||
if filer_common_rep < float(CONFIG["upgrade_filing_cost"]):
|
||||
return UpgradeFilingValidation(False, "insufficient_common_rep")
|
||||
if not isinstance(release_hash, str) or not release_hash.strip():
|
||||
return UpgradeFilingValidation(False, "release_hash_required")
|
||||
if not isinstance(release_description, str) or len(release_description) > 4000:
|
||||
return UpgradeFilingValidation(False, "release_description_invalid")
|
||||
if not isinstance(target_protocol_version, str) or not target_protocol_version.strip():
|
||||
return UpgradeFilingValidation(False, "target_protocol_version_required")
|
||||
return UpgradeFilingValidation(True, "ok")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"HeavyNodeReadinessState",
|
||||
"UpgradeFilingValidation",
|
||||
"UpgradeProposalState",
|
||||
"compute_upgrade_state",
|
||||
"validate_upgrade_proposal",
|
||||
]
|
||||
@@ -0,0 +1,276 @@
|
||||
"""Identity rotation gates and obligation inheritance.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.13.
|
||||
|
||||
Pure functions over the chain. Two sets of responsibilities:
|
||||
|
||||
1. **Gating (``validate_rotation``):** reject a rotation if the old
|
||||
identity holds active resolution stakes, dispute stakes, or truth
|
||||
stakes. Predictor exclusion + governance decay + rep transfer are
|
||||
inherited automatically by ``rotation_descendants`` — they are NOT
|
||||
gates, they are computations downstream resolvers run.
|
||||
|
||||
2. **Descendant tracking (``rotation_descendants``):** given a node,
|
||||
return the full transitive closure of identities it has rotated
|
||||
into. Used by Sprint 4's predictor-exclusion logic to compute
|
||||
``frozen_predictor_ids ∪ rotation_descendants(frozen_predictor_ids)``
|
||||
from the snapshot at resolution time.
|
||||
|
||||
Cross-cutting design rule (BUILD_LOG.md): a user attempting to rotate
|
||||
while holding active stakes must NOT see a hostile UI message. The
|
||||
caller is expected to:
|
||||
|
||||
- Show the user which stakes are blocking rotation.
|
||||
- Offer to wait for those stakes to settle, or cancel pending
|
||||
unresolved stakes (where the protocol allows).
|
||||
- Queue the rotation for retry after settlement.
|
||||
|
||||
This module returns structured rejection reasons (a tuple of
|
||||
``(blocker_kind, count, sample_ids)``) so the UI can render exactly
|
||||
that. It never returns "rejected" without the diagnostic shape.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RotationBlocker:
|
||||
"""One reason the rotation is currently rejected.
|
||||
|
||||
``kind`` is one of:
|
||||
- ``"resolution_stake"`` — open resolution stakes on a market
|
||||
that has not yet finalized.
|
||||
- ``"dispute_stake"`` — open dispute stakes that have not yet
|
||||
resolved.
|
||||
- ``"truth_stake"`` — truth stakes still inside their
|
||||
``duration_days`` window without a resolve event.
|
||||
|
||||
``count`` is the number of blocking obligations of that kind.
|
||||
``sample_ids`` is up to 5 string identifiers (market_id /
|
||||
dispute_id / message_id) so the UI can show "3 markets and 1
|
||||
dispute are still pending" with deep links.
|
||||
"""
|
||||
kind: str
|
||||
count: int
|
||||
sample_ids: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RotationDecision:
|
||||
accepted: bool
|
||||
blockers: tuple[RotationBlocker, ...]
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
def _market_status_lookup(events: list[dict[str, Any]]) -> dict[str, str]:
|
||||
"""Last-write-wins map of market_id → terminal status.
|
||||
|
||||
Sprint 2 only knows two terminal statuses (FINAL, INVALID) — the
|
||||
full lifecycle is Sprint 4. Markets without a ``resolution_finalize``
|
||||
are treated as still open (active stakes).
|
||||
"""
|
||||
status: dict[str, str] = {}
|
||||
for ev in events:
|
||||
if ev.get("event_type") != "resolution_finalize":
|
||||
continue
|
||||
p = _payload(ev)
|
||||
mid = p.get("market_id")
|
||||
if isinstance(mid, str) and mid:
|
||||
outcome = p.get("outcome")
|
||||
status[mid] = "invalid" if outcome == "invalid" else "final"
|
||||
return status
|
||||
|
||||
|
||||
def _dispute_status_lookup(events: list[dict[str, Any]]) -> dict[str, str]:
|
||||
status: dict[str, str] = {}
|
||||
for ev in events:
|
||||
if ev.get("event_type") != "dispute_resolve":
|
||||
continue
|
||||
p = _payload(ev)
|
||||
did = p.get("dispute_id")
|
||||
if isinstance(did, str) and did:
|
||||
status[did] = "resolved"
|
||||
return status
|
||||
|
||||
|
||||
def _truth_stake_resolved_messages(events: list[dict[str, Any]]) -> set[str]:
|
||||
resolved: set[str] = set()
|
||||
for ev in events:
|
||||
if ev.get("event_type") != "truth_stake_resolve":
|
||||
continue
|
||||
p = _payload(ev)
|
||||
mid = p.get("message_id")
|
||||
if isinstance(mid, str) and mid:
|
||||
resolved.add(mid)
|
||||
return resolved
|
||||
|
||||
|
||||
def _active_resolution_stakes(node_id: str, events: list[dict[str, Any]]) -> list[str]:
|
||||
market_status = _market_status_lookup(events)
|
||||
out: list[str] = []
|
||||
for ev in events:
|
||||
if ev.get("event_type") != "resolution_stake":
|
||||
continue
|
||||
if ev.get("node_id") != node_id:
|
||||
continue
|
||||
p = _payload(ev)
|
||||
mid = p.get("market_id")
|
||||
if not isinstance(mid, str) or not mid:
|
||||
continue
|
||||
if market_status.get(mid) is None:
|
||||
out.append(mid)
|
||||
return out
|
||||
|
||||
|
||||
def _active_dispute_stakes(node_id: str, events: list[dict[str, Any]]) -> list[str]:
|
||||
dispute_status = _dispute_status_lookup(events)
|
||||
out: list[str] = []
|
||||
for ev in events:
|
||||
if ev.get("event_type") != "dispute_stake":
|
||||
continue
|
||||
if ev.get("node_id") != node_id:
|
||||
continue
|
||||
p = _payload(ev)
|
||||
did = p.get("dispute_id")
|
||||
if not isinstance(did, str) or not did:
|
||||
continue
|
||||
if dispute_status.get(did) is None:
|
||||
out.append(did)
|
||||
return out
|
||||
|
||||
|
||||
def _active_truth_stakes(
|
||||
node_id: str,
|
||||
events: list[dict[str, Any]],
|
||||
*,
|
||||
now: float,
|
||||
) -> list[str]:
|
||||
"""A truth stake is active if its (placed_at + duration_days * 86400)
|
||||
is in the future relative to ``now`` AND no ``truth_stake_resolve``
|
||||
has landed for its message.
|
||||
"""
|
||||
resolved = _truth_stake_resolved_messages(events)
|
||||
out: list[str] = []
|
||||
for ev in events:
|
||||
if ev.get("event_type") != "truth_stake_place":
|
||||
continue
|
||||
if ev.get("node_id") != node_id:
|
||||
continue
|
||||
p = _payload(ev)
|
||||
mid = p.get("message_id")
|
||||
if not isinstance(mid, str) or not mid:
|
||||
continue
|
||||
if mid in resolved:
|
||||
continue
|
||||
try:
|
||||
placed_at = float(ev.get("timestamp") or 0.0)
|
||||
duration_days = int(p.get("duration_days") or 0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
expires_at = placed_at + duration_days * 86400.0
|
||||
if expires_at > now:
|
||||
out.append(mid)
|
||||
return out
|
||||
|
||||
|
||||
def validate_rotation(
|
||||
rotation_event: dict[str, Any],
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float,
|
||||
) -> RotationDecision:
|
||||
"""Decide whether ``rotation_event`` is permitted right now.
|
||||
|
||||
Sprint 2 enforces RULES §3.13 Gate 1 only. Gate 2 (obligation
|
||||
inheritance) is computation, not gating, and is handled by the
|
||||
Sprint 4 predictor-exclusion logic that consults
|
||||
``rotation_descendants``.
|
||||
|
||||
The returned ``RotationDecision`` includes structured blockers so
|
||||
the UI can offer a non-hostile retry path (see module docstring).
|
||||
"""
|
||||
if rotation_event.get("event_type") != "identity_rotate":
|
||||
raise ValueError("validate_rotation requires an identity_rotate event")
|
||||
payload = _payload(rotation_event)
|
||||
old_node_id = payload.get("old_node_id")
|
||||
if not isinstance(old_node_id, str) or not old_node_id:
|
||||
raise ValueError("identity_rotate payload missing old_node_id")
|
||||
|
||||
events = [e for e in chain if isinstance(e, dict)]
|
||||
|
||||
blockers: list[RotationBlocker] = []
|
||||
res = _active_resolution_stakes(old_node_id, events)
|
||||
if res:
|
||||
blockers.append(RotationBlocker(
|
||||
kind="resolution_stake",
|
||||
count=len(res),
|
||||
sample_ids=tuple(res[:5]),
|
||||
))
|
||||
dis = _active_dispute_stakes(old_node_id, events)
|
||||
if dis:
|
||||
blockers.append(RotationBlocker(
|
||||
kind="dispute_stake",
|
||||
count=len(dis),
|
||||
sample_ids=tuple(dis[:5]),
|
||||
))
|
||||
tru = _active_truth_stakes(old_node_id, events, now=now)
|
||||
if tru:
|
||||
blockers.append(RotationBlocker(
|
||||
kind="truth_stake",
|
||||
count=len(tru),
|
||||
sample_ids=tuple(tru[:5]),
|
||||
))
|
||||
|
||||
return RotationDecision(accepted=not blockers, blockers=tuple(blockers))
|
||||
|
||||
|
||||
def rotation_descendants(
|
||||
node_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> set[str]:
|
||||
"""All identities that descend from ``node_id`` via ``identity_rotate``.
|
||||
|
||||
Excludes ``node_id`` itself. Used by Sprint 4 predictor exclusion.
|
||||
"""
|
||||
events = [e for e in chain if isinstance(e, dict)]
|
||||
# Build a forward map: old_node_id -> {new_node_id, new_node_id, ...}.
|
||||
# New node_id of an identity_rotate is the event's signer (per spec).
|
||||
forward: dict[str, set[str]] = {}
|
||||
for ev in events:
|
||||
if ev.get("event_type") != "identity_rotate":
|
||||
continue
|
||||
p = _payload(ev)
|
||||
old = p.get("old_node_id")
|
||||
new = ev.get("node_id")
|
||||
if not isinstance(old, str) or not isinstance(new, str):
|
||||
continue
|
||||
if not old or not new or old == new:
|
||||
continue
|
||||
forward.setdefault(old, set()).add(new)
|
||||
|
||||
out: set[str] = set()
|
||||
stack = list(forward.get(node_id, set()))
|
||||
while stack:
|
||||
cur = stack.pop()
|
||||
if cur in out:
|
||||
continue
|
||||
out.add(cur)
|
||||
for nxt in forward.get(cur, ()):
|
||||
if nxt not in out:
|
||||
stack.append(nxt)
|
||||
return out
|
||||
|
||||
|
||||
__all__ = [
|
||||
"RotationBlocker",
|
||||
"RotationDecision",
|
||||
"rotation_descendants",
|
||||
"validate_rotation",
|
||||
]
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Market lifecycle, snapshot, evidence, and resolution.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.10, §5.2.
|
||||
|
||||
Pure-function design (same as Sprint 2/3): every entry point takes
|
||||
``(market_id, chain, ...)`` and returns a deterministic value or a
|
||||
structured result. The producer is responsible for emitting the
|
||||
resulting events to the chain through the adapter layer.
|
||||
"""
|
||||
|
||||
from services.infonet.markets.data_unavailable import (
|
||||
is_data_unavailable_triggered,
|
||||
resolve_data_unavailable_effects,
|
||||
)
|
||||
from services.infonet.markets.dispute import (
|
||||
DisputeView,
|
||||
collect_disputes,
|
||||
compute_dispute_outcome,
|
||||
dispute_settlement_effects,
|
||||
effective_outcome,
|
||||
market_was_reversed,
|
||||
)
|
||||
from services.infonet.markets.evidence import (
|
||||
EvidenceBundle,
|
||||
collect_evidence,
|
||||
evidence_content_hash,
|
||||
is_first_for_side,
|
||||
submission_hash,
|
||||
)
|
||||
from services.infonet.markets.lifecycle import (
|
||||
MarketStatus,
|
||||
compute_market_status,
|
||||
should_advance_phase,
|
||||
)
|
||||
from services.infonet.markets.resolution import (
|
||||
ResolutionResult,
|
||||
collect_resolution_stakes,
|
||||
excluded_predictor_ids,
|
||||
is_predictor_excluded,
|
||||
resolve_market,
|
||||
)
|
||||
from services.infonet.markets.snapshot import (
|
||||
build_snapshot,
|
||||
compute_snapshot_event_hash,
|
||||
find_snapshot,
|
||||
)
|
||||
from services.infonet.markets.stalemate_burn import (
|
||||
apply_to_stakes as apply_stalemate_burn,
|
||||
stalemate_burn_pct,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"DisputeView",
|
||||
"EvidenceBundle",
|
||||
"MarketStatus",
|
||||
"ResolutionResult",
|
||||
"apply_stalemate_burn",
|
||||
"build_snapshot",
|
||||
"collect_disputes",
|
||||
"collect_evidence",
|
||||
"collect_resolution_stakes",
|
||||
"compute_dispute_outcome",
|
||||
"compute_market_status",
|
||||
"compute_snapshot_event_hash",
|
||||
"dispute_settlement_effects",
|
||||
"effective_outcome",
|
||||
"evidence_content_hash",
|
||||
"excluded_predictor_ids",
|
||||
"find_snapshot",
|
||||
"is_data_unavailable_triggered",
|
||||
"is_first_for_side",
|
||||
"is_predictor_excluded",
|
||||
"market_was_reversed",
|
||||
"resolve_data_unavailable_effects",
|
||||
"resolve_market",
|
||||
"should_advance_phase",
|
||||
"stalemate_burn_pct",
|
||||
"submission_hash",
|
||||
]
|
||||
@@ -0,0 +1,103 @@
|
||||
"""DATA_UNAVAILABLE resolution path — Round 8 phantom-evidence defense.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.10 Step 1.5
|
||||
+ the ``CONFIG['data_unavailable_threshold']`` comment block.
|
||||
|
||||
Threshold: when ``oracle_da / oracle_all >= data_unavailable_threshold``
|
||||
(default 33% of oracle-rep stake), the market is INVALID and:
|
||||
|
||||
- ALL evidence-submitter bonds are SLASHED (burned, not returned).
|
||||
The premise: evidence existed but couldn't be verified — the
|
||||
submitters are at fault.
|
||||
- DA voters' resolution stakes are returned in FULL (they acted
|
||||
correctly).
|
||||
- yes/no resolution stakes get the stalemate burn applied (they
|
||||
participated despite bad evidence; small burn makes blind staking
|
||||
expensive).
|
||||
|
||||
This is distinct from the no-supermajority stalemate path: there, all
|
||||
stakes (including DA) take the burn and bonds are returned in good
|
||||
faith. Sprint 5 keeps the two paths separate to match the spec.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
from services.infonet.markets.evidence import EvidenceBundle
|
||||
from services.infonet.markets.stalemate_burn import (
|
||||
apply_to_stakes,
|
||||
split_burn_and_return,
|
||||
)
|
||||
|
||||
|
||||
def is_data_unavailable_triggered(stakes: list[Any]) -> bool:
|
||||
"""``True`` if oracle DA stakes meet or exceed the threshold."""
|
||||
oracle_all = sum(getattr(s, "amount", 0.0) for s in stakes
|
||||
if getattr(s, "rep_type", None) == "oracle")
|
||||
if oracle_all <= 0:
|
||||
return False
|
||||
oracle_da = sum(getattr(s, "amount", 0.0) for s in stakes
|
||||
if getattr(s, "side", None) == "data_unavailable"
|
||||
and getattr(s, "rep_type", None) == "oracle")
|
||||
return oracle_da / oracle_all >= float(CONFIG["data_unavailable_threshold"])
|
||||
|
||||
|
||||
def resolve_data_unavailable_effects(
|
||||
stakes: list[Any],
|
||||
bundles: list[EvidenceBundle],
|
||||
) -> dict[str, Any]:
|
||||
"""Compute the rep-transfer effects for a DA-triggered INVALID
|
||||
resolution. Returns a dict with the same keys ``ResolutionResult``
|
||||
expects, ready for the caller to fold in.
|
||||
|
||||
Side effects layered:
|
||||
|
||||
- DA voters get full return.
|
||||
- yes/no resolution stakers get the stalemate burn.
|
||||
- Evidence submitters: bonds slashed (forfeit).
|
||||
"""
|
||||
out: dict[str, Any] = {
|
||||
"stake_returns": {},
|
||||
"bond_forfeits": {},
|
||||
"bond_returns": {},
|
||||
"burned": 0.0,
|
||||
}
|
||||
|
||||
da_stakes = [s for s in stakes if getattr(s, "side", None) == "data_unavailable"]
|
||||
other_stakes = [s for s in stakes if getattr(s, "side", None) in ("yes", "no")]
|
||||
|
||||
# DA voters: full return.
|
||||
for s in da_stakes:
|
||||
node_id = getattr(s, "node_id", None)
|
||||
rep_type = getattr(s, "rep_type", None)
|
||||
amount = float(getattr(s, "amount", 0.0))
|
||||
if not isinstance(node_id, str) or rep_type not in ("oracle", "common") or amount <= 0:
|
||||
continue
|
||||
key = (node_id, rep_type)
|
||||
out["stake_returns"][key] = out["stake_returns"].get(key, 0.0) + amount
|
||||
|
||||
# yes/no stakers: stalemate burn.
|
||||
burn_returns, burn_total = apply_to_stakes(
|
||||
({"node_id": s.node_id, "rep_type": s.rep_type, "amount": s.amount} for s in other_stakes),
|
||||
)
|
||||
for k, v in burn_returns.items():
|
||||
out["stake_returns"][k] = out["stake_returns"].get(k, 0.0) + v
|
||||
out["burned"] += burn_total
|
||||
|
||||
# Evidence bonds: slashed (forfeited) — burned.
|
||||
for b in bundles:
|
||||
if b.bond > 0:
|
||||
out["bond_forfeits"][b.node_id] = (
|
||||
out["bond_forfeits"].get(b.node_id, 0.0) + b.bond
|
||||
)
|
||||
out["burned"] += b.bond
|
||||
return out
|
||||
|
||||
|
||||
__all__ = [
|
||||
"is_data_unavailable_triggered",
|
||||
"resolve_data_unavailable_effects",
|
||||
"split_burn_and_return", # re-exported for callers' convenience
|
||||
]
|
||||
@@ -0,0 +1,284 @@
|
||||
"""Bounded-reversal disputes — RULES §3.12.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.12 + §5.2
|
||||
step 6.
|
||||
|
||||
A dispute is a post-finality challenge. Any node can stake oracle rep
|
||||
to open one, and other nodes can stake oracle OR common rep on
|
||||
``confirm`` (uphold the original outcome) or ``reverse`` (flip it).
|
||||
Oracle-rep simple majority decides — resolution already established
|
||||
the supermajority, so a simple majority is enough to overturn.
|
||||
|
||||
If the dispute reverses, **only this market's oracle rep is
|
||||
recalculated**. Downstream rep earned in OTHER markets from rep
|
||||
originally minted here is NOT clawed back. No cascading rewrites.
|
||||
That's the "bounded" in bounded reversal.
|
||||
|
||||
Two effects:
|
||||
|
||||
1. ``effective_outcome(market_id, chain)`` returns the flipped
|
||||
outcome if a reversed dispute exists; the unmodified outcome
|
||||
otherwise. Used by ``oracle_rep._market_is_mintable`` so
|
||||
reputation views automatically reflect the reversal.
|
||||
2. ``compute_dispute_outcome`` returns ``"upheld" | "reversed" |
|
||||
"tie"`` from accumulated stakes — used when an authoritative
|
||||
``dispute_resolve`` event has not yet landed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class DisputeView:
|
||||
"""Chain-derived view of a single dispute."""
|
||||
dispute_id: str
|
||||
market_id: str
|
||||
challenger_id: str
|
||||
challenger_stake: float
|
||||
opened_at: float
|
||||
confirm_stakes: list[dict] = field(default_factory=list)
|
||||
reverse_stakes: list[dict] = field(default_factory=list)
|
||||
resolved_outcome: str | None = None # "upheld" | "reversed" | "tie"
|
||||
resolved_at: float | None = None
|
||||
|
||||
@property
|
||||
def is_resolved(self) -> bool:
|
||||
return self.resolved_outcome is not None
|
||||
|
||||
|
||||
def _dispute_id(event: dict[str, Any]) -> str:
|
||||
"""Pick the dispute_id from a dispute event payload, falling back
|
||||
to the event_id for ``dispute_open`` (which is the canonical
|
||||
identifier the rest of the chain references)."""
|
||||
p = _payload(event)
|
||||
did = p.get("dispute_id")
|
||||
if isinstance(did, str) and did:
|
||||
return did
|
||||
# dispute_open events: synthesize from event_id if present, else
|
||||
# market_id + opener + timestamp. Producers SHOULD attach a
|
||||
# dispute_id explicitly — Sprint 7+ enforces this in the schema.
|
||||
eid = event.get("event_id")
|
||||
if isinstance(eid, str) and eid:
|
||||
return eid
|
||||
market_id = p.get("market_id") or ""
|
||||
return f"dispute:{market_id}:{event.get('node_id','')}:{event.get('timestamp','')}"
|
||||
|
||||
|
||||
def collect_disputes(
|
||||
market_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> list[DisputeView]:
|
||||
"""All disputes filed against ``market_id``, sorted by open time."""
|
||||
chain_list = [e for e in chain if isinstance(e, dict)]
|
||||
|
||||
open_events = [e for e in chain_list
|
||||
if e.get("event_type") == "dispute_open"
|
||||
and _payload(e).get("market_id") == market_id]
|
||||
if not open_events:
|
||||
return []
|
||||
|
||||
# Build by dispute_id keyed off the open event.
|
||||
disputes: dict[str, DisputeView] = {}
|
||||
open_id_by_market_event: dict[str, str] = {}
|
||||
for ev in open_events:
|
||||
p = _payload(ev)
|
||||
did = _dispute_id(ev)
|
||||
challenger = ev.get("node_id") or ""
|
||||
try:
|
||||
cstake = float(p.get("challenger_stake") or 0.0)
|
||||
except (TypeError, ValueError):
|
||||
cstake = 0.0
|
||||
opened_at = float(ev.get("timestamp") or 0.0)
|
||||
disputes[did] = DisputeView(
|
||||
dispute_id=did, market_id=str(market_id),
|
||||
challenger_id=str(challenger), challenger_stake=cstake,
|
||||
opened_at=opened_at,
|
||||
)
|
||||
open_id_by_market_event[ev.get("event_id") or ""] = did
|
||||
|
||||
# Stakes reference dispute_id explicitly (Sprint 1 schema).
|
||||
for ev in chain_list:
|
||||
if ev.get("event_type") != "dispute_stake":
|
||||
continue
|
||||
p = _payload(ev)
|
||||
did = p.get("dispute_id")
|
||||
if not isinstance(did, str) or did not in disputes:
|
||||
continue
|
||||
side = p.get("side")
|
||||
if side not in ("confirm", "reverse"):
|
||||
continue
|
||||
rep_type = p.get("rep_type")
|
||||
if rep_type not in ("oracle", "common"):
|
||||
continue
|
||||
try:
|
||||
amount = float(p.get("amount") or 0.0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if amount <= 0:
|
||||
continue
|
||||
record = {
|
||||
"node_id": ev.get("node_id") or "",
|
||||
"amount": amount,
|
||||
"rep_type": rep_type,
|
||||
}
|
||||
target = disputes[did].confirm_stakes if side == "confirm" else disputes[did].reverse_stakes
|
||||
target.append(record)
|
||||
|
||||
# Resolution events.
|
||||
for ev in chain_list:
|
||||
if ev.get("event_type") != "dispute_resolve":
|
||||
continue
|
||||
p = _payload(ev)
|
||||
did = p.get("dispute_id")
|
||||
if not isinstance(did, str) or did not in disputes:
|
||||
continue
|
||||
outcome = p.get("outcome")
|
||||
if outcome not in ("upheld", "reversed", "tie"):
|
||||
continue
|
||||
disputes[did].resolved_outcome = outcome
|
||||
disputes[did].resolved_at = float(ev.get("timestamp") or 0.0)
|
||||
|
||||
return sorted(disputes.values(), key=lambda d: (d.opened_at, d.dispute_id))
|
||||
|
||||
|
||||
def compute_dispute_outcome(dispute: DisputeView) -> str:
|
||||
"""Apply RULES §3.12 to compute the dispute outcome from its
|
||||
accumulated stakes (oracle-rep majority).
|
||||
|
||||
Returns ``"upheld"`` (default — original outcome stands),
|
||||
``"reversed"``, or ``"tie"``. ``"tie"`` is treated as upheld for
|
||||
bookkeeping but is reported separately so callers can log it.
|
||||
"""
|
||||
confirm_oracle = sum(
|
||||
s.get("amount", 0.0) for s in dispute.confirm_stakes
|
||||
if s.get("rep_type") == "oracle"
|
||||
)
|
||||
reverse_oracle = sum(
|
||||
s.get("amount", 0.0) for s in dispute.reverse_stakes
|
||||
if s.get("rep_type") == "oracle"
|
||||
)
|
||||
if confirm_oracle > reverse_oracle:
|
||||
return "upheld"
|
||||
if reverse_oracle > confirm_oracle:
|
||||
return "reversed"
|
||||
return "tie"
|
||||
|
||||
|
||||
def market_was_reversed(market_id: str, chain: Iterable[dict[str, Any]]) -> bool:
|
||||
"""``True`` if any dispute on ``market_id`` resolved as reversed.
|
||||
|
||||
Multiple disputes on the same market are unusual but possible —
|
||||
if any one reverses, the market's effective outcome flips. A
|
||||
subsequent dispute that re-reverses would flip it back, but
|
||||
Sprint 5 leaves multi-dispute behavior intentionally simple
|
||||
(last reversed wins — see ``effective_outcome``).
|
||||
"""
|
||||
for d in collect_disputes(market_id, chain):
|
||||
if d.resolved_outcome == "reversed":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _flip(outcome: str) -> str:
|
||||
if outcome == "yes":
|
||||
return "no"
|
||||
if outcome == "no":
|
||||
return "yes"
|
||||
return outcome
|
||||
|
||||
|
||||
def effective_outcome(
|
||||
original_outcome: str,
|
||||
market_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> str:
|
||||
"""Apply bounded reversal to a market's outcome.
|
||||
|
||||
Walks resolved disputes in chain order. Each ``reversed`` outcome
|
||||
flips the running outcome; ``upheld`` and ``tie`` leave it. The
|
||||
final value is the **effective** outcome that ``oracle_rep`` and
|
||||
``last_successful_prediction_ts`` should use.
|
||||
|
||||
BOUNDED: this function operates on a single market_id only. It does
|
||||
NOT cascade into other markets even if oracle rep used to stake in
|
||||
those other markets came from this one.
|
||||
"""
|
||||
if original_outcome not in ("yes", "no"):
|
||||
return original_outcome
|
||||
current = original_outcome
|
||||
for d in collect_disputes(market_id, chain):
|
||||
if d.resolved_outcome == "reversed":
|
||||
current = _flip(current)
|
||||
return current
|
||||
|
||||
|
||||
def dispute_settlement_effects(dispute: DisputeView) -> dict[str, Any]:
|
||||
"""Compute rep transfers from a *resolved* dispute.
|
||||
|
||||
Per RULES §3.12: winning side splits the loser pool, 2% loser tax
|
||||
burned, oracle and common pools settle independently.
|
||||
|
||||
Returns the same shape as ``resolve_data_unavailable_effects`` — a
|
||||
dict the caller folds into a higher-level result.
|
||||
"""
|
||||
out: dict[str, Any] = {
|
||||
"stake_returns": {},
|
||||
"stake_winnings": {},
|
||||
"burned": 0.0,
|
||||
}
|
||||
if not dispute.is_resolved:
|
||||
return out
|
||||
outcome = dispute.resolved_outcome
|
||||
if outcome == "tie":
|
||||
# Return all stakes intact.
|
||||
for s in dispute.confirm_stakes + dispute.reverse_stakes:
|
||||
key = (s["node_id"], s["rep_type"])
|
||||
out["stake_returns"][key] = out["stake_returns"].get(key, 0.0) + s["amount"]
|
||||
return out
|
||||
|
||||
winners = dispute.confirm_stakes if outcome == "upheld" else dispute.reverse_stakes
|
||||
losers = dispute.reverse_stakes if outcome == "upheld" else dispute.confirm_stakes
|
||||
burn_pct = float(CONFIG["resolution_loser_burn_pct"])
|
||||
|
||||
for rep_type in ("oracle", "common"):
|
||||
rep_winners = [s for s in winners if s["rep_type"] == rep_type]
|
||||
rep_losers = [s for s in losers if s["rep_type"] == rep_type]
|
||||
winner_pool = sum(s["amount"] for s in rep_winners)
|
||||
loser_pool = sum(s["amount"] for s in rep_losers)
|
||||
|
||||
for s in rep_winners:
|
||||
key = (s["node_id"], rep_type)
|
||||
out["stake_returns"][key] = out["stake_returns"].get(key, 0.0) + s["amount"]
|
||||
|
||||
if winner_pool == 0 or loser_pool == 0:
|
||||
continue
|
||||
burn = loser_pool * burn_pct
|
||||
distributable = loser_pool - burn
|
||||
out["burned"] += burn
|
||||
for s in rep_winners:
|
||||
share = s["amount"] / winner_pool
|
||||
winnings = share * distributable
|
||||
key = (s["node_id"], rep_type)
|
||||
out["stake_winnings"][key] = out["stake_winnings"].get(key, 0.0) + winnings
|
||||
|
||||
return out
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DisputeView",
|
||||
"collect_disputes",
|
||||
"compute_dispute_outcome",
|
||||
"dispute_settlement_effects",
|
||||
"effective_outcome",
|
||||
"market_was_reversed",
|
||||
]
|
||||
@@ -0,0 +1,200 @@
|
||||
"""Evidence canonicalization + first-submitter detection.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §2.2 (evidence
|
||||
bundle fields), §3.10 (Step 4 — bond resolution + first-submitter
|
||||
bonus).
|
||||
|
||||
Two distinct hashes per evidence bundle:
|
||||
|
||||
- ``evidence_content_hash`` — SHA-256 of ``(market_id || claimed_outcome
|
||||
|| sorted(evidence_hashes) || normalized_utf8(source_description))``.
|
||||
**Excludes node_id**. Two submitters who present the same evidence
|
||||
produce the same content hash — used for duplicate detection across
|
||||
authors.
|
||||
- ``submission_hash`` — SHA-256 of ``(evidence_content_hash || node_id
|
||||
|| timestamp)``. **Includes node_id**. Used for authorship + chain
|
||||
ordering + first-submitter detection.
|
||||
|
||||
"Same evidence = same evidence_content_hash" — submission ordering
|
||||
determines who is the FIRST submitter for a side. The first submitter
|
||||
per outcome side gets ``CONFIG['evidence_first_bonus']`` (capped by the
|
||||
losing-bond pool, never minted) when their side wins.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import unicodedata
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
def _normalize_utf8(s: str) -> str:
|
||||
"""NFC-normalize so visually-identical strings hash identically."""
|
||||
return unicodedata.normalize("NFC", s)
|
||||
|
||||
|
||||
def evidence_content_hash(
|
||||
market_id: str,
|
||||
claimed_outcome: str,
|
||||
evidence_hashes: list[str],
|
||||
source_description: str,
|
||||
) -> str:
|
||||
"""SHA-256 of the canonical evidence content. Excludes node_id."""
|
||||
if claimed_outcome not in ("yes", "no"):
|
||||
raise ValueError("claimed_outcome must be 'yes' or 'no'")
|
||||
sorted_hashes = sorted(str(h) for h in (evidence_hashes or []))
|
||||
canonical = "|".join([
|
||||
"evidence_content",
|
||||
str(market_id),
|
||||
claimed_outcome,
|
||||
",".join(sorted_hashes),
|
||||
_normalize_utf8(str(source_description or "")),
|
||||
])
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def submission_hash(
|
||||
content_hash: str,
|
||||
node_id: str,
|
||||
timestamp: float,
|
||||
) -> str:
|
||||
"""SHA-256 of ``content_hash || node_id || timestamp``.
|
||||
|
||||
Timestamp is rendered with ``repr(float)`` for cross-implementation
|
||||
determinism — Python's repr gives the shortest round-trippable
|
||||
decimal, which is stable across CPython versions.
|
||||
"""
|
||||
canonical = "|".join([
|
||||
"evidence_submission",
|
||||
str(content_hash),
|
||||
str(node_id),
|
||||
repr(float(timestamp)),
|
||||
])
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EvidenceBundle:
|
||||
"""Chain-derived view of one ``evidence_submit`` event."""
|
||||
node_id: str
|
||||
market_id: str
|
||||
claimed_outcome: str
|
||||
evidence_hashes: tuple[str, ...]
|
||||
source_description: str
|
||||
bond: float
|
||||
timestamp: float
|
||||
sequence: int
|
||||
content_hash: str
|
||||
submission_hash: str
|
||||
is_first_for_side: bool
|
||||
|
||||
|
||||
def collect_evidence(
|
||||
market_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> list[EvidenceBundle]:
|
||||
"""Return all ``evidence_submit`` events for ``market_id`` as
|
||||
``EvidenceBundle``s, sorted by chain order, with
|
||||
``is_first_for_side`` set on the first event per outcome side
|
||||
whose ``content_hash`` is unique within that side.
|
||||
"""
|
||||
events: list[dict[str, Any]] = []
|
||||
for ev in chain:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
if ev.get("event_type") != "evidence_submit":
|
||||
continue
|
||||
if _payload(ev).get("market_id") != market_id:
|
||||
continue
|
||||
events.append(ev)
|
||||
events.sort(key=lambda e: (float(e.get("timestamp") or 0.0), int(e.get("sequence") or 0)))
|
||||
|
||||
seen_content_per_side: dict[str, set[str]] = {"yes": set(), "no": set()}
|
||||
bundles: list[EvidenceBundle] = []
|
||||
first_set_per_side: dict[str, bool] = {"yes": False, "no": False}
|
||||
|
||||
for ev in events:
|
||||
p = _payload(ev)
|
||||
node_id = ev.get("node_id")
|
||||
outcome = p.get("claimed_outcome")
|
||||
if not isinstance(node_id, str) or not node_id:
|
||||
continue
|
||||
if outcome not in ("yes", "no"):
|
||||
continue
|
||||
evhashes = p.get("evidence_hashes") or []
|
||||
if not isinstance(evhashes, list):
|
||||
continue
|
||||
source_desc = p.get("source_description") or ""
|
||||
bond = p.get("bond")
|
||||
try:
|
||||
bond_f = float(bond) if bond is not None else 0.0
|
||||
except (TypeError, ValueError):
|
||||
bond_f = 0.0
|
||||
ts = float(ev.get("timestamp") or 0.0)
|
||||
seq = int(ev.get("sequence") or 0)
|
||||
|
||||
chash = p.get("evidence_content_hash") or evidence_content_hash(
|
||||
market_id, outcome, [str(h) for h in evhashes], str(source_desc),
|
||||
)
|
||||
shash = p.get("submission_hash") or submission_hash(chash, node_id, ts)
|
||||
|
||||
# First-for-side: this event is the first occurrence (in chain
|
||||
# order) of a content hash for this side that we haven't seen
|
||||
# before. Duplicate submitters of the same content do NOT
|
||||
# qualify for the bonus.
|
||||
is_first = False
|
||||
if chash not in seen_content_per_side[outcome] and not first_set_per_side[outcome]:
|
||||
is_first = True
|
||||
first_set_per_side[outcome] = True
|
||||
seen_content_per_side[outcome].add(chash)
|
||||
|
||||
bundles.append(EvidenceBundle(
|
||||
node_id=node_id,
|
||||
market_id=str(market_id),
|
||||
claimed_outcome=outcome,
|
||||
evidence_hashes=tuple(str(h) for h in evhashes),
|
||||
source_description=str(source_desc),
|
||||
bond=bond_f,
|
||||
timestamp=ts,
|
||||
sequence=seq,
|
||||
content_hash=str(chash),
|
||||
submission_hash=str(shash),
|
||||
is_first_for_side=is_first,
|
||||
))
|
||||
return bundles
|
||||
|
||||
|
||||
def is_first_for_side(
|
||||
market_id: str,
|
||||
claimed_outcome: str,
|
||||
candidate_content_hash: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> bool:
|
||||
"""Would a NEW evidence submission with ``candidate_content_hash``
|
||||
be the first for ``claimed_outcome``?
|
||||
|
||||
True if no prior ``evidence_submit`` for ``market_id`` on
|
||||
``claimed_outcome`` exists (regardless of content hash). The bonus
|
||||
is for being temporally first per side, not per content hash.
|
||||
"""
|
||||
if claimed_outcome not in ("yes", "no"):
|
||||
return False
|
||||
for bundle in collect_evidence(market_id, chain):
|
||||
if bundle.claimed_outcome == claimed_outcome:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EvidenceBundle",
|
||||
"collect_evidence",
|
||||
"evidence_content_hash",
|
||||
"is_first_for_side",
|
||||
"submission_hash",
|
||||
]
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Market lifecycle state machine.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §5.2 + §3.10.
|
||||
|
||||
Five logical statuses:
|
||||
|
||||
PREDICTING — open for predictions; no snapshot yet.
|
||||
EVIDENCE — snapshot frozen; evidence window open
|
||||
(CONFIG['evidence_window_hours']).
|
||||
RESOLVING — evidence window closed; resolution staking window open
|
||||
(CONFIG['resolution_window_hours']).
|
||||
FINAL — resolution_finalize event landed with a real outcome.
|
||||
INVALID — resolution_finalize event landed with outcome="invalid".
|
||||
|
||||
Transitions are decided by ``chain_majority_time`` (per RULES §3.14
|
||||
Rule 3) — no single node's local clock can unilaterally advance a
|
||||
market. That rule keeps producers honest even when network partitions
|
||||
shift local time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
|
||||
|
||||
class MarketStatus(str, Enum):
|
||||
PREDICTING = "predicting"
|
||||
EVIDENCE = "evidence"
|
||||
RESOLVING = "resolving"
|
||||
FINAL = "final"
|
||||
INVALID = "invalid"
|
||||
|
||||
|
||||
_SECONDS_PER_HOUR = 3600.0
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
def _market_id(event: dict[str, Any]) -> str:
|
||||
return str(_payload(event).get("market_id") or "")
|
||||
|
||||
|
||||
def _events_for_market(market_id: str, chain: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
out: list[dict[str, Any]] = []
|
||||
for ev in chain:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
if _market_id(ev) == market_id:
|
||||
out.append(ev)
|
||||
out.sort(key=lambda e: (float(e.get("timestamp") or 0.0), int(e.get("sequence") or 0)))
|
||||
return out
|
||||
|
||||
|
||||
def compute_market_status(
|
||||
market_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float,
|
||||
) -> MarketStatus:
|
||||
"""Return the current status of ``market_id`` at chain time ``now``.
|
||||
|
||||
Status is derived from the chain — it's never stored. The producer
|
||||
that emits ``market_snapshot`` and ``resolution_finalize`` events
|
||||
is responsible for using the same ``now`` value (typically
|
||||
``chain_majority_time(chain)``) so every node converges on the same
|
||||
status.
|
||||
"""
|
||||
events = _events_for_market(market_id, chain)
|
||||
if not events:
|
||||
return MarketStatus.PREDICTING # treated as not-yet-existing
|
||||
|
||||
create_event = next((e for e in events if e.get("event_type") == "prediction_create"), None)
|
||||
if create_event is None:
|
||||
return MarketStatus.PREDICTING
|
||||
|
||||
finalize = next((e for e in events if e.get("event_type") == "resolution_finalize"), None)
|
||||
if finalize is not None:
|
||||
outcome = _payload(finalize).get("outcome")
|
||||
return MarketStatus.INVALID if outcome == "invalid" else MarketStatus.FINAL
|
||||
|
||||
snapshot = next((e for e in events if e.get("event_type") == "market_snapshot"), None)
|
||||
if snapshot is None:
|
||||
return MarketStatus.PREDICTING
|
||||
|
||||
snapshot_ts = float(snapshot.get("timestamp") or _payload(snapshot).get("frozen_at") or 0.0)
|
||||
evidence_close = snapshot_ts + float(CONFIG["evidence_window_hours"]) * _SECONDS_PER_HOUR
|
||||
if now < evidence_close:
|
||||
return MarketStatus.EVIDENCE
|
||||
return MarketStatus.RESOLVING
|
||||
|
||||
|
||||
def should_advance_phase(
|
||||
market_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float,
|
||||
) -> tuple[MarketStatus, MarketStatus] | None:
|
||||
"""If a phase advance is due, return ``(current, next)``. Else ``None``.
|
||||
|
||||
The producer should call this on a heartbeat and emit the
|
||||
appropriate event when a transition is ready:
|
||||
|
||||
- PREDICTING → EVIDENCE: emit ``market_snapshot``.
|
||||
- EVIDENCE → RESOLVING: just a status change (no chain event).
|
||||
- RESOLVING → FINAL/INVALID: emit ``resolution_finalize``.
|
||||
"""
|
||||
events = _events_for_market(market_id, chain)
|
||||
if not events:
|
||||
return None
|
||||
|
||||
create_event = next((e for e in events if e.get("event_type") == "prediction_create"), None)
|
||||
if create_event is None:
|
||||
return None
|
||||
finalize = next((e for e in events if e.get("event_type") == "resolution_finalize"), None)
|
||||
if finalize is not None:
|
||||
return None # already terminal
|
||||
|
||||
create_payload = _payload(create_event)
|
||||
trigger_date = float(create_payload.get("trigger_date") or 0.0)
|
||||
snapshot = next((e for e in events if e.get("event_type") == "market_snapshot"), None)
|
||||
|
||||
if snapshot is None:
|
||||
# PREDICTING — advance to EVIDENCE iff trigger_date has passed in
|
||||
# majority chain time.
|
||||
if now >= trigger_date:
|
||||
return (MarketStatus.PREDICTING, MarketStatus.EVIDENCE)
|
||||
return None
|
||||
|
||||
snapshot_ts = float(snapshot.get("timestamp") or _payload(snapshot).get("frozen_at") or 0.0)
|
||||
evidence_close = snapshot_ts + float(CONFIG["evidence_window_hours"]) * _SECONDS_PER_HOUR
|
||||
resolution_close = evidence_close + float(CONFIG["resolution_window_hours"]) * _SECONDS_PER_HOUR
|
||||
|
||||
if now < evidence_close:
|
||||
return None # still EVIDENCE
|
||||
if now < resolution_close:
|
||||
return (MarketStatus.EVIDENCE, MarketStatus.RESOLVING)
|
||||
return (MarketStatus.RESOLVING, MarketStatus.FINAL)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MarketStatus",
|
||||
"compute_market_status",
|
||||
"should_advance_phase",
|
||||
]
|
||||
@@ -0,0 +1,488 @@
|
||||
"""Market resolution — the RULES §3.10 decision procedure.
|
||||
|
||||
Pure function over the chain. Returns a structured ``ResolutionResult``
|
||||
with the decided outcome plus all rep-transfer effects (bond returns /
|
||||
forfeits / first-submitter bonuses / loser-pool burns / stalemate
|
||||
burns / DA bond slashing).
|
||||
|
||||
Sprint 5 layers in the Round 8 defenses on top of Sprint 4's
|
||||
state-machine scaffolding:
|
||||
|
||||
- DATA_UNAVAILABLE phantom-evidence slashing — when DA stakes meet
|
||||
the threshold, ALL evidence bonds are forfeited (burned), DA voters
|
||||
get full return, yes/no stakers take the stalemate burn.
|
||||
- Stalemate burn on supermajority-failed INVALID — when both sides
|
||||
staked above the min total but no side reached the supermajority,
|
||||
ALL resolution stakes (yes/no/DA) take the burn. Bonds are returned
|
||||
in good faith — the market failed, not the submitters.
|
||||
|
||||
What Sprint 5 still does NOT handle:
|
||||
|
||||
- Bootstrap-mode resolution (Sprint 8 — ``bootstrap_resolution_vote``
|
||||
events with Argon2id PoW).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
from services.infonet.identity_rotation import rotation_descendants
|
||||
from services.infonet.markets.data_unavailable import (
|
||||
is_data_unavailable_triggered,
|
||||
resolve_data_unavailable_effects,
|
||||
)
|
||||
from services.infonet.markets.evidence import collect_evidence
|
||||
from services.infonet.markets.snapshot import find_snapshot
|
||||
from services.infonet.markets.stalemate_burn import apply_to_stakes
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
def excluded_predictor_ids(
|
||||
market_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> set[str]:
|
||||
"""Predictor exclusion set for ``market_id`` resolution.
|
||||
|
||||
RULES §3.13 / §3.10 Step 1: ``frozen_predictor_ids ∪
|
||||
rotation_descendants(frozen_predictor_ids)``. Walks the on-chain
|
||||
rotation links — never mutates the snapshot.
|
||||
|
||||
Returns an empty set if no snapshot exists yet. The caller decides
|
||||
whether that means "open for everyone" (no exclusion) or "reject
|
||||
all" (snapshot required) — Sprint 4 ``collect_resolution_stakes``
|
||||
treats absence-of-snapshot as "no exclusion".
|
||||
"""
|
||||
snapshot = find_snapshot(market_id, chain)
|
||||
if snapshot is None:
|
||||
return set()
|
||||
frozen = snapshot.get("frozen_predictor_ids") or []
|
||||
if not isinstance(frozen, list):
|
||||
return set()
|
||||
base: set[str] = {str(x) for x in frozen if isinstance(x, str) and x}
|
||||
out = set(base)
|
||||
chain_list = [e for e in chain if isinstance(e, dict)]
|
||||
for original in base:
|
||||
for desc in rotation_descendants(original, chain_list):
|
||||
out.add(desc)
|
||||
return out
|
||||
|
||||
|
||||
def is_predictor_excluded(
|
||||
node_id: str,
|
||||
market_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> bool:
|
||||
return node_id in excluded_predictor_ids(market_id, chain)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ResolutionStake:
|
||||
node_id: str
|
||||
side: str
|
||||
amount: float
|
||||
rep_type: str
|
||||
timestamp: float
|
||||
sequence: int
|
||||
|
||||
|
||||
def collect_resolution_stakes(
|
||||
market_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
exclude_predictors: bool = True,
|
||||
) -> list[_ResolutionStake]:
|
||||
"""All ``resolution_stake`` events for ``market_id`` (sorted),
|
||||
with predictor exclusion applied by default.
|
||||
|
||||
Excluded stakes are silently dropped — they cannot influence the
|
||||
outcome (RULES §3.10 Step 1). The producer-side check should also
|
||||
refuse to emit them in the first place, but the resolver MUST
|
||||
enforce here too because the chain is ingested from peers.
|
||||
"""
|
||||
excluded = excluded_predictor_ids(market_id, chain) if exclude_predictors else set()
|
||||
out: list[_ResolutionStake] = []
|
||||
for ev in chain:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
if ev.get("event_type") != "resolution_stake":
|
||||
continue
|
||||
p = _payload(ev)
|
||||
if p.get("market_id") != market_id:
|
||||
continue
|
||||
node_id = ev.get("node_id")
|
||||
if not isinstance(node_id, str) or not node_id:
|
||||
continue
|
||||
if node_id in excluded:
|
||||
continue
|
||||
side = p.get("side")
|
||||
if side not in ("yes", "no", "data_unavailable"):
|
||||
continue
|
||||
amount = p.get("amount")
|
||||
try:
|
||||
amt = float(amount) if amount is not None else 0.0
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if amt <= 0:
|
||||
continue
|
||||
rep_type = p.get("rep_type")
|
||||
if rep_type not in ("oracle", "common"):
|
||||
continue
|
||||
out.append(_ResolutionStake(
|
||||
node_id=node_id,
|
||||
side=side,
|
||||
amount=amt,
|
||||
rep_type=rep_type,
|
||||
timestamp=float(ev.get("timestamp") or 0.0),
|
||||
sequence=int(ev.get("sequence") or 0),
|
||||
))
|
||||
out.sort(key=lambda s: (s.timestamp, s.sequence))
|
||||
return out
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResolutionResult:
|
||||
"""Outcome + every rep-transfer effect that resolution should apply.
|
||||
|
||||
The producer of ``resolution_finalize`` writes the outcome onto the
|
||||
chain; downstream chain readers (`oracle_rep`, `common_rep`)
|
||||
recompute their views from the chain alone — they do not consume
|
||||
this struct. The struct exists for tests and for the UI's
|
||||
"resolution explainer" view, where users want to see *why* a market
|
||||
resolved a particular way.
|
||||
"""
|
||||
market_id: str
|
||||
outcome: str # "yes" | "no" | "invalid"
|
||||
is_provisional: bool
|
||||
reason: str # short diagnostic — e.g. "no_evidence", "supermajority_yes"
|
||||
bond_returns: dict[str, float] = field(default_factory=dict)
|
||||
bond_forfeits: dict[str, float] = field(default_factory=dict)
|
||||
first_submitter_bonuses: dict[str, float] = field(default_factory=dict)
|
||||
stake_returns: dict[tuple[str, str], float] = field(default_factory=dict)
|
||||
"""``{(node_id, rep_type): amount}`` — full or partial returns of
|
||||
resolution stakes (winners and stalemate-INVALID returns)."""
|
||||
stake_winnings: dict[tuple[str, str], float] = field(default_factory=dict)
|
||||
"""``{(node_id, rep_type): amount}`` — extra winnings from the
|
||||
loser pool (winners only)."""
|
||||
burned_amount: float = 0.0
|
||||
|
||||
|
||||
def _supermajority_winner(
|
||||
yes: float,
|
||||
no: float,
|
||||
threshold: float,
|
||||
) -> str | None:
|
||||
total = yes + no
|
||||
if total <= 0:
|
||||
return None
|
||||
if yes / total >= threshold:
|
||||
return "yes"
|
||||
if no / total >= threshold:
|
||||
return "no"
|
||||
return None
|
||||
|
||||
|
||||
def resolve_market(
|
||||
market_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
is_provisional: bool = False,
|
||||
) -> ResolutionResult:
|
||||
"""Apply RULES §3.10 to compute the resolution.
|
||||
|
||||
Sprint 4 implements:
|
||||
|
||||
- Step 0: zero-evidence → INVALID (return all stakes, no penalty).
|
||||
- Step 1: predictor exclusion via ``collect_resolution_stakes``.
|
||||
- Step 1.5 (partial): DA threshold detection → INVALID.
|
||||
*Phantom-evidence slashing is Sprint 5.*
|
||||
- Step 2: oracle-rep supermajority check.
|
||||
*Stalemate burn is Sprint 5.*
|
||||
- Step 2.5: winning-side evidence required.
|
||||
- Step 3: distribute resolution stakes (oracle + common pools, 2%
|
||||
loser burn).
|
||||
- Step 4: evidence bond resolution + first-submitter bonus capped
|
||||
at losing-bond-pool budget.
|
||||
|
||||
Bootstrap-mode markets (``bootstrap_index <=
|
||||
CONFIG['bootstrap_market_count']``) take a different path that
|
||||
Sprint 8 will provide. Until then bootstrap markets resolve to
|
||||
INVALID with reason ``bootstrap_pending``.
|
||||
"""
|
||||
chain_list = [e for e in chain if isinstance(e, dict)]
|
||||
create_event = next(
|
||||
(e for e in chain_list if e.get("event_type") == "prediction_create"
|
||||
and _payload(e).get("market_id") == market_id),
|
||||
None,
|
||||
)
|
||||
if create_event is None:
|
||||
return ResolutionResult(
|
||||
market_id=market_id, outcome="invalid",
|
||||
is_provisional=is_provisional, reason="no_market",
|
||||
)
|
||||
|
||||
create_payload = _payload(create_event)
|
||||
bootstrap_index = create_payload.get("bootstrap_index")
|
||||
if bootstrap_index is not None:
|
||||
try:
|
||||
bootstrap_index = int(bootstrap_index)
|
||||
except (TypeError, ValueError):
|
||||
bootstrap_index = None
|
||||
|
||||
bundles = collect_evidence(market_id, chain_list)
|
||||
stakes = collect_resolution_stakes(market_id, chain_list, exclude_predictors=True)
|
||||
|
||||
# Step 0: zero-evidence → INVALID, return everything.
|
||||
if not bundles:
|
||||
result = ResolutionResult(
|
||||
market_id=market_id, outcome="invalid",
|
||||
is_provisional=is_provisional, reason="no_evidence",
|
||||
)
|
||||
for s in stakes:
|
||||
result.stake_returns[(s.node_id, s.rep_type)] = (
|
||||
result.stake_returns.get((s.node_id, s.rep_type), 0.0) + s.amount
|
||||
)
|
||||
return result
|
||||
|
||||
# Step 0.5: bootstrap mode (Sprint 8 — eligible-node-one-vote).
|
||||
if (bootstrap_index is not None
|
||||
and bootstrap_index <= int(CONFIG["bootstrap_market_count"])):
|
||||
from services.infonet.bootstrap import (
|
||||
deduplicate_votes,
|
||||
validate_bootstrap_eligibility,
|
||||
)
|
||||
|
||||
canonical_votes = deduplicate_votes(market_id, chain_list)
|
||||
# Filter to eligible voters per RULES §3.10 step 0.5
|
||||
# is_bootstrap_eligible.
|
||||
eligible_votes = []
|
||||
for v in canonical_votes:
|
||||
node_id = v.get("node_id")
|
||||
if not isinstance(node_id, str) or not node_id:
|
||||
continue
|
||||
if not validate_bootstrap_eligibility(node_id, market_id, chain_list).eligible:
|
||||
continue
|
||||
side = _payload(v).get("side")
|
||||
if side not in ("yes", "no"):
|
||||
continue
|
||||
eligible_votes.append((node_id, side))
|
||||
|
||||
votes_yes = sum(1 for _, side in eligible_votes if side == "yes")
|
||||
votes_no = sum(1 for _, side in eligible_votes if side == "no")
|
||||
votes_total = votes_yes + votes_no
|
||||
|
||||
# Min participation gate.
|
||||
if votes_total < int(CONFIG["min_market_participants"]):
|
||||
result = ResolutionResult(
|
||||
market_id=market_id, outcome="invalid",
|
||||
is_provisional=is_provisional,
|
||||
reason="bootstrap_below_min_participation",
|
||||
)
|
||||
for b in bundles:
|
||||
result.bond_returns[b.node_id] = (
|
||||
result.bond_returns.get(b.node_id, 0.0) + b.bond
|
||||
)
|
||||
return result
|
||||
|
||||
threshold = float(CONFIG["bootstrap_resolution_supermajority"])
|
||||
if votes_yes / votes_total >= threshold:
|
||||
winning_side = "yes"
|
||||
elif votes_no / votes_total >= threshold:
|
||||
winning_side = "no"
|
||||
else:
|
||||
result = ResolutionResult(
|
||||
market_id=market_id, outcome="invalid",
|
||||
is_provisional=is_provisional,
|
||||
reason="bootstrap_no_supermajority",
|
||||
)
|
||||
for b in bundles:
|
||||
result.bond_returns[b.node_id] = (
|
||||
result.bond_returns.get(b.node_id, 0.0) + b.bond
|
||||
)
|
||||
return result
|
||||
|
||||
# Step 2.5 (winning-side evidence required) still applies in
|
||||
# bootstrap mode.
|
||||
winning_evidence = [b for b in bundles if b.claimed_outcome == winning_side]
|
||||
if not winning_evidence:
|
||||
result = ResolutionResult(
|
||||
market_id=market_id, outcome="invalid",
|
||||
is_provisional=is_provisional,
|
||||
reason="no_winning_side_evidence",
|
||||
)
|
||||
for b in bundles:
|
||||
result.bond_returns[b.node_id] = (
|
||||
result.bond_returns.get(b.node_id, 0.0) + b.bond
|
||||
)
|
||||
return result
|
||||
|
||||
# Bootstrap markets pass directly to prediction scoring — no
|
||||
# resolution-stake settlement (no oracle-rep stakes were
|
||||
# collected). Evidence bonds are returned (they were 0 in
|
||||
# bootstrap mode by spec, but stated for completeness).
|
||||
result = ResolutionResult(
|
||||
market_id=market_id, outcome=winning_side,
|
||||
is_provisional=is_provisional,
|
||||
reason=f"bootstrap_supermajority_{winning_side}",
|
||||
)
|
||||
for b in bundles:
|
||||
if b.claimed_outcome == winning_side:
|
||||
result.bond_returns[b.node_id] = (
|
||||
result.bond_returns.get(b.node_id, 0.0) + b.bond
|
||||
)
|
||||
else:
|
||||
result.bond_forfeits[b.node_id] = (
|
||||
result.bond_forfeits.get(b.node_id, 0.0) + b.bond
|
||||
)
|
||||
return result
|
||||
|
||||
# Step 1.5: DA threshold check (Sprint 5 — phantom-evidence slashing).
|
||||
if is_data_unavailable_triggered(stakes):
|
||||
result = ResolutionResult(
|
||||
market_id=market_id, outcome="invalid",
|
||||
is_provisional=is_provisional, reason="data_unavailable",
|
||||
)
|
||||
effects = resolve_data_unavailable_effects(stakes, bundles)
|
||||
for k, v in effects["stake_returns"].items():
|
||||
result.stake_returns[k] = result.stake_returns.get(k, 0.0) + v
|
||||
for node, amount in effects["bond_forfeits"].items():
|
||||
result.bond_forfeits[node] = result.bond_forfeits.get(node, 0.0) + amount
|
||||
result.burned_amount += float(effects["burned"])
|
||||
return result
|
||||
|
||||
# Step 2: oracle-rep supermajority.
|
||||
yes_oracle = sum(s.amount for s in stakes if s.side == "yes" and s.rep_type == "oracle")
|
||||
no_oracle = sum(s.amount for s in stakes if s.side == "no" and s.rep_type == "oracle")
|
||||
if yes_oracle + no_oracle < float(CONFIG["min_resolution_stake_total"]):
|
||||
result = ResolutionResult(
|
||||
market_id=market_id, outcome="invalid",
|
||||
is_provisional=is_provisional, reason="below_min_resolution_stake",
|
||||
)
|
||||
for s in stakes:
|
||||
result.stake_returns[(s.node_id, s.rep_type)] = (
|
||||
result.stake_returns.get((s.node_id, s.rep_type), 0.0) + s.amount
|
||||
)
|
||||
for b in bundles:
|
||||
result.bond_returns[b.node_id] = result.bond_returns.get(b.node_id, 0.0) + b.bond
|
||||
return result
|
||||
|
||||
threshold = float(CONFIG["resolution_supermajority"])
|
||||
winning_side = _supermajority_winner(yes_oracle, no_oracle, threshold)
|
||||
if winning_side is None:
|
||||
# No supermajority — Sprint 5 stalemate burn applies.
|
||||
# Per RULES §3.10 step 2 alternate: ALL resolution stakes
|
||||
# (yes / no / DA) take the burn; bonds are returned in good
|
||||
# faith because the market failed (not the submitters).
|
||||
result = ResolutionResult(
|
||||
market_id=market_id, outcome="invalid",
|
||||
is_provisional=is_provisional, reason="no_supermajority",
|
||||
)
|
||||
burn_returns, burn_total = apply_to_stakes(
|
||||
({"node_id": s.node_id, "rep_type": s.rep_type, "amount": s.amount} for s in stakes),
|
||||
)
|
||||
for k, v in burn_returns.items():
|
||||
result.stake_returns[k] = result.stake_returns.get(k, 0.0) + v
|
||||
result.burned_amount += burn_total
|
||||
for b in bundles:
|
||||
result.bond_returns[b.node_id] = result.bond_returns.get(b.node_id, 0.0) + b.bond
|
||||
return result
|
||||
|
||||
# Step 2.5: winning-side evidence required.
|
||||
winning_evidence = [b for b in bundles if b.claimed_outcome == winning_side]
|
||||
if not winning_evidence:
|
||||
result = ResolutionResult(
|
||||
market_id=market_id, outcome="invalid",
|
||||
is_provisional=is_provisional, reason="no_winning_side_evidence",
|
||||
)
|
||||
for s in stakes:
|
||||
result.stake_returns[(s.node_id, s.rep_type)] = (
|
||||
result.stake_returns.get((s.node_id, s.rep_type), 0.0) + s.amount
|
||||
)
|
||||
for b in bundles:
|
||||
result.bond_returns[b.node_id] = result.bond_returns.get(b.node_id, 0.0) + b.bond
|
||||
return result
|
||||
|
||||
# Step 3: distribute resolution stakes per rep type.
|
||||
result = ResolutionResult(
|
||||
market_id=market_id, outcome=winning_side,
|
||||
is_provisional=is_provisional,
|
||||
reason=f"supermajority_{winning_side}",
|
||||
)
|
||||
burn_pct = float(CONFIG["resolution_loser_burn_pct"])
|
||||
|
||||
for rep_type in ("oracle", "common"):
|
||||
winners = [s for s in stakes if s.side == winning_side and s.rep_type == rep_type]
|
||||
# Losers exclude data_unavailable here — they vote on evidence
|
||||
# quality, not outcome. Their stakes are returned in full.
|
||||
losers = [s for s in stakes
|
||||
if s.side != winning_side
|
||||
and s.side != "data_unavailable"
|
||||
and s.rep_type == rep_type]
|
||||
winner_pool = sum(s.amount for s in winners)
|
||||
loser_pool = sum(s.amount for s in losers)
|
||||
|
||||
# Always return the principal of winners and DA voters.
|
||||
for s in winners:
|
||||
result.stake_returns[(s.node_id, rep_type)] = (
|
||||
result.stake_returns.get((s.node_id, rep_type), 0.0) + s.amount
|
||||
)
|
||||
for s in stakes:
|
||||
if s.rep_type != rep_type:
|
||||
continue
|
||||
if s.side != "data_unavailable":
|
||||
continue
|
||||
result.stake_returns[(s.node_id, rep_type)] = (
|
||||
result.stake_returns.get((s.node_id, rep_type), 0.0) + s.amount
|
||||
)
|
||||
|
||||
if winner_pool == 0 or loser_pool == 0:
|
||||
continue
|
||||
burn_amt = loser_pool * burn_pct
|
||||
distributable = loser_pool - burn_amt
|
||||
result.burned_amount += burn_amt
|
||||
for s in winners:
|
||||
share = s.amount / winner_pool
|
||||
winnings = share * distributable
|
||||
result.stake_winnings[(s.node_id, rep_type)] = (
|
||||
result.stake_winnings.get((s.node_id, rep_type), 0.0) + winnings
|
||||
)
|
||||
# Losing stakes are forfeited — don't return them.
|
||||
|
||||
# Step 4: evidence bonds.
|
||||
losing_bond_pool = sum(b.bond for b in bundles if b.claimed_outcome != winning_side)
|
||||
bonus_budget = losing_bond_pool
|
||||
|
||||
for b in bundles:
|
||||
if b.claimed_outcome == winning_side:
|
||||
result.bond_returns[b.node_id] = result.bond_returns.get(b.node_id, 0.0) + b.bond
|
||||
if b.is_first_for_side and bonus_budget > 0:
|
||||
bonus_amt = min(float(CONFIG["evidence_first_bonus"]), bonus_budget)
|
||||
if bonus_amt > 0:
|
||||
result.first_submitter_bonuses[b.node_id] = (
|
||||
result.first_submitter_bonuses.get(b.node_id, 0.0) + bonus_amt
|
||||
)
|
||||
bonus_budget -= bonus_amt
|
||||
else:
|
||||
result.bond_forfeits[b.node_id] = result.bond_forfeits.get(b.node_id, 0.0) + b.bond
|
||||
|
||||
# Remaining unspent bonus budget burns (deflationary).
|
||||
result.burned_amount += bonus_budget
|
||||
|
||||
# NOTE: subjective markets are allowed to resolve (they still
|
||||
# produce a final outcome), but oracle rep is not minted from them
|
||||
# — that gate lives in ``oracle_rep._market_is_mintable``.
|
||||
return result
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ResolutionResult",
|
||||
"collect_resolution_stakes",
|
||||
"excluded_predictor_ids",
|
||||
"is_predictor_excluded",
|
||||
"resolve_market",
|
||||
]
|
||||
@@ -0,0 +1,171 @@
|
||||
"""Market snapshot — frozen at PREDICTING → EVIDENCE transition.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §2.2 (snapshot
|
||||
fields), §3.10 (snapshot_event_hash usage), §5.2 (when emitted).
|
||||
|
||||
The snapshot is the **commitment boundary** for all downstream
|
||||
evaluation. Once frozen:
|
||||
|
||||
- Liquidity gates (``min_market_participants``,
|
||||
``min_market_total_stake``) are evaluated against frozen values, not
|
||||
live state.
|
||||
- Predictor exclusion is computed from ``frozen_predictor_ids``
|
||||
(UNION ``rotation_descendants`` at resolution time).
|
||||
- Bootstrap PoW uses ``snapshot_event_hash`` as its salt so attackers
|
||||
can't pre-mine before the boundary.
|
||||
|
||||
The snapshot itself is **immutable** by spec — the producer emits it
|
||||
once and never updates it. Sprint 4 enforces immutability by ignoring
|
||||
any subsequent ``market_snapshot`` events with the same market_id
|
||||
(``find_snapshot`` returns the FIRST one). Tests assert this invariant.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
def _events_for_market(market_id: str, chain: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
out: list[dict[str, Any]] = []
|
||||
for ev in chain:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
if _payload(ev).get("market_id") == market_id:
|
||||
out.append(ev)
|
||||
out.sort(key=lambda e: (float(e.get("timestamp") or 0.0), int(e.get("sequence") or 0)))
|
||||
return out
|
||||
|
||||
|
||||
def build_snapshot(
|
||||
market_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
frozen_at: float,
|
||||
) -> dict[str, Any]:
|
||||
"""Compute the snapshot payload deterministically from chain history.
|
||||
|
||||
Walks ``prediction_place`` events for ``market_id``, in chain order,
|
||||
and produces the frozen counts / stake totals / predictor list /
|
||||
yes-no probability state. The resulting dict is ready to be written
|
||||
as the payload of a ``market_snapshot`` event.
|
||||
|
||||
``frozen_at`` is the canonical commitment timestamp — typically
|
||||
``chain_majority_time(chain)`` at the moment the producer decides
|
||||
to advance to EVIDENCE. Pass it explicitly so the function stays
|
||||
pure and deterministic.
|
||||
"""
|
||||
events = _events_for_market(market_id, chain)
|
||||
|
||||
predictor_ids: list[str] = []
|
||||
seen_predictors: set[str] = set()
|
||||
yes_weight = 0.0
|
||||
no_weight = 0.0
|
||||
total_stake = 0.0
|
||||
|
||||
for ev in events:
|
||||
if ev.get("event_type") != "prediction_place":
|
||||
continue
|
||||
node = ev.get("node_id")
|
||||
if not isinstance(node, str) or not node:
|
||||
continue
|
||||
p = _payload(ev)
|
||||
side = p.get("side")
|
||||
if side not in ("yes", "no"):
|
||||
continue
|
||||
if node not in seen_predictors:
|
||||
seen_predictors.add(node)
|
||||
predictor_ids.append(node)
|
||||
stake = p.get("stake_amount")
|
||||
if stake is not None:
|
||||
try:
|
||||
a = float(stake)
|
||||
except (TypeError, ValueError):
|
||||
a = 0.0
|
||||
if a > 0:
|
||||
total_stake += a
|
||||
if side == "yes":
|
||||
yes_weight += a
|
||||
else:
|
||||
no_weight += a
|
||||
else:
|
||||
# Free pick = 1.0 virtual stake (RULES §5.2).
|
||||
if side == "yes":
|
||||
yes_weight += 1.0
|
||||
else:
|
||||
no_weight += 1.0
|
||||
|
||||
pool = yes_weight + no_weight
|
||||
if pool > 0:
|
||||
yes_p = yes_weight / pool
|
||||
else:
|
||||
yes_p = 0.5
|
||||
no_p = 1.0 - yes_p
|
||||
|
||||
return {
|
||||
"market_id": market_id,
|
||||
"frozen_participant_count": len(predictor_ids),
|
||||
"frozen_total_stake": total_stake,
|
||||
"frozen_predictor_ids": predictor_ids,
|
||||
"frozen_probability_state": {"yes": yes_p, "no": no_p},
|
||||
"frozen_at": float(frozen_at),
|
||||
}
|
||||
|
||||
|
||||
def compute_snapshot_event_hash(
|
||||
snapshot_payload: dict[str, Any],
|
||||
*,
|
||||
market_id: str,
|
||||
creator_node_id: str,
|
||||
sequence: int,
|
||||
) -> str:
|
||||
"""Canonical SHA-256 of the snapshot event.
|
||||
|
||||
This hash is what bootstrap PoW uses as its salt (RULES §3.10 step
|
||||
0.5) — committing this value on-chain prevents pre-mining of
|
||||
bootstrap votes. The serialization is canonical (sorted keys,
|
||||
compact separators, UTF-8) so every node arrives at the same hex.
|
||||
|
||||
The producer should append this value to the snapshot payload as
|
||||
``snapshot_event_hash`` before emitting the event.
|
||||
"""
|
||||
canonical = {
|
||||
"event_type": "market_snapshot",
|
||||
"market_id": market_id,
|
||||
"node_id": creator_node_id,
|
||||
"sequence": int(sequence),
|
||||
"payload": snapshot_payload,
|
||||
}
|
||||
encoded = json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
||||
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def find_snapshot(
|
||||
market_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> dict[str, Any] | None:
|
||||
"""Return the FIRST ``market_snapshot`` payload for ``market_id``.
|
||||
|
||||
Subsequent ``market_snapshot`` events with the same market_id are
|
||||
ignored — snapshots are immutable per RULES §2.2. This is a
|
||||
structural enforcement, not just a convention; an attacker who
|
||||
forges a second snapshot cannot influence resolution.
|
||||
"""
|
||||
events = _events_for_market(market_id, chain)
|
||||
for ev in events:
|
||||
if ev.get("event_type") == "market_snapshot":
|
||||
return _payload(ev)
|
||||
return None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"build_snapshot",
|
||||
"compute_snapshot_event_hash",
|
||||
"find_snapshot",
|
||||
]
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Stalemate burn — Round 8 anti-griefing defense.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.10 Step 2
|
||||
(no-supermajority branch) and the comment block above
|
||||
``CONFIG['resolution_stalemate_burn_pct']``.
|
||||
|
||||
The problem: without a stalemate burn, a >25% cartel can stake the
|
||||
contrarian side at zero cost to permanently force INVALID and halt
|
||||
oracle rep minting. Burning a small percentage of every resolution
|
||||
stake when consensus fails makes that strategy progressively expensive
|
||||
— the cartel bleeds rep over time.
|
||||
|
||||
Critical constraint (RULES §3.10 step 2 comment): the stalemate burn
|
||||
ONLY applies when:
|
||||
|
||||
- both sides staked (total ≥ min threshold), AND
|
||||
- evidence exists, AND
|
||||
- supermajority not reached.
|
||||
|
||||
It does NOT apply when:
|
||||
|
||||
- zero evidence (the market gave no signal at all — not griefing),
|
||||
- below-minimum participation, OR
|
||||
- below-minimum stake total (uninformative — not griefing).
|
||||
|
||||
That's why the helper here is *non-default* — it's invoked only by the
|
||||
specific branches in ``resolution.py`` that match the spec's
|
||||
"genuine disagreement" case.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
|
||||
|
||||
def stalemate_burn_pct() -> float:
|
||||
"""Current burn percentage from CONFIG. Helper so callers don't
|
||||
need to remember the key name."""
|
||||
return float(CONFIG["resolution_stalemate_burn_pct"])
|
||||
|
||||
|
||||
def split_burn_and_return(amount: float, burn_pct: float | None = None) -> tuple[float, float]:
|
||||
"""Compute (burn_amount, returned_amount) for a single stake."""
|
||||
if amount <= 0:
|
||||
return 0.0, 0.0
|
||||
pct = float(stalemate_burn_pct() if burn_pct is None else burn_pct)
|
||||
if pct <= 0:
|
||||
return 0.0, float(amount)
|
||||
if pct >= 1:
|
||||
return float(amount), 0.0
|
||||
burn = float(amount) * pct
|
||||
returned = float(amount) - burn
|
||||
return burn, returned
|
||||
|
||||
|
||||
def apply_to_stakes(
|
||||
stakes: Iterable[dict],
|
||||
*,
|
||||
burn_pct: float | None = None,
|
||||
) -> tuple[dict[tuple[str, str], float], float]:
|
||||
"""Apply the stalemate burn to ``stakes`` (iterable of dicts with
|
||||
``node_id``, ``rep_type``, ``amount``).
|
||||
|
||||
Returns ``(returns_by_(node, rep_type), total_burned)``. The caller
|
||||
folds these into the larger ``ResolutionResult`` rather than
|
||||
mutating any state directly.
|
||||
"""
|
||||
pct = float(stalemate_burn_pct() if burn_pct is None else burn_pct)
|
||||
returns: dict[tuple[str, str], float] = {}
|
||||
total_burned = 0.0
|
||||
for s in stakes:
|
||||
node_id = s.get("node_id") if isinstance(s, dict) else getattr(s, "node_id", None)
|
||||
rep_type = s.get("rep_type") if isinstance(s, dict) else getattr(s, "rep_type", None)
|
||||
amount = s.get("amount") if isinstance(s, dict) else getattr(s, "amount", None)
|
||||
try:
|
||||
amt = float(amount) if amount is not None else 0.0
|
||||
except (TypeError, ValueError):
|
||||
amt = 0.0
|
||||
if amt <= 0 or not isinstance(node_id, str) or rep_type not in ("oracle", "common"):
|
||||
continue
|
||||
burn, ret = split_burn_and_return(amt, pct)
|
||||
if ret > 0:
|
||||
returns[(node_id, rep_type)] = returns.get((node_id, rep_type), 0.0) + ret
|
||||
total_burned += burn
|
||||
return returns, total_burned
|
||||
|
||||
|
||||
__all__ = [
|
||||
"apply_to_stakes",
|
||||
"split_burn_and_return",
|
||||
"stalemate_burn_pct",
|
||||
]
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Two-tier state model + epoch finality (Sprint 10).
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.14 Rule 4,
|
||||
``infonet-economy/IMPLEMENTATION_PLAN.md`` §3.7.
|
||||
|
||||
Splits protocol state into two consistency tiers:
|
||||
|
||||
- **Tier 1 — Eventually consistent (CRDT-friendly).** Common rep,
|
||||
gate activity, content posting, upreps, vote karma. Computed
|
||||
locally during partitions; merges without conflict on reconnect.
|
||||
- **Tier 2 — Epoch finality required.** Oracle rep minting,
|
||||
governance execution, market FINAL status, dispute outcomes,
|
||||
(eventually) coin minting / dividends. MUST NOT become
|
||||
economically final until an epoch checkpoint is confirmed by a
|
||||
threshold of Heavy Nodes across Reticulum bridges.
|
||||
|
||||
Sprint 10 ships the Tier-1/Tier-2 classification, the chain-staleness
|
||||
heuristic that producers consult to set ``is_provisional=True`` on
|
||||
Tier-2 events, and the structural model for an `EpochCheckpoint`. The
|
||||
full epoch-checkpoint protocol (BFT / threshold sigs / DAG) is open
|
||||
engineering work — IMPLEMENTATION_PLAN §6.5 — and is intentionally
|
||||
NOT specified here. The model + thresholds are in place; the
|
||||
inter-node agreement protocol slots in later.
|
||||
|
||||
Why this matters today: ``oracle_rep._market_is_mintable`` (Sprint 2)
|
||||
already gates on ``is_provisional == False``. Sprint 10 gives
|
||||
producers the helper to set that flag correctly.
|
||||
"""
|
||||
|
||||
from services.infonet.partition.epoch_checkpoint import (
|
||||
EpochCheckpoint,
|
||||
EpochCheckpointStatus,
|
||||
canonical_epoch_root,
|
||||
is_checkpoint_confirmed,
|
||||
)
|
||||
from services.infonet.partition.provisional import (
|
||||
DEFAULT_MAX_CHAIN_LAG_S,
|
||||
chain_lag_seconds,
|
||||
is_chain_stale,
|
||||
should_mark_provisional,
|
||||
)
|
||||
from services.infonet.partition.two_tier_state import (
|
||||
TIER1_EVENT_TYPES,
|
||||
TIER2_EVENT_TYPES,
|
||||
classify_event_type,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_MAX_CHAIN_LAG_S",
|
||||
"EpochCheckpoint",
|
||||
"EpochCheckpointStatus",
|
||||
"TIER1_EVENT_TYPES",
|
||||
"TIER2_EVENT_TYPES",
|
||||
"canonical_epoch_root",
|
||||
"chain_lag_seconds",
|
||||
"classify_event_type",
|
||||
"is_chain_stale",
|
||||
"is_checkpoint_confirmed",
|
||||
"should_mark_provisional",
|
||||
]
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Epoch checkpoint model.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.14 Rule 4
|
||||
("Epoch checkpoint: a global Merkle root for that epoch agreed upon
|
||||
by a threshold of Heavy Nodes across Reticulum bridges. Epoch
|
||||
duration, threshold, and checkpoint protocol: OPEN ENGINEERING
|
||||
PROBLEM").
|
||||
|
||||
Sprint 10 ships the **structural model** only. The inter-node
|
||||
agreement protocol (BFT vs threshold sigs vs DAG-style) is open per
|
||||
IMPLEMENTATION_PLAN §6.5 and is intentionally NOT specified here.
|
||||
|
||||
What IS specified:
|
||||
|
||||
- A canonical ``EpochCheckpoint`` dataclass: epoch_id + root_hash +
|
||||
participating_heavy_node_ids + threshold.
|
||||
- A ``canonical_epoch_root`` helper that computes a deterministic
|
||||
SHA-256 over a chain segment for a given epoch window. Every
|
||||
Heavy Node computes the same value from the same chain prefix —
|
||||
that's the whole point of the structural commitment.
|
||||
- An ``is_checkpoint_confirmed`` predicate that says "yes, this
|
||||
epoch's root has Heavy Node agreement at or above the threshold".
|
||||
|
||||
When the inter-node protocol lands, it produces ``EpochCheckpoint``
|
||||
records that ``is_checkpoint_confirmed`` consults. Until then,
|
||||
producers can hand-construct test scenarios.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
class EpochCheckpointStatus(str, Enum):
|
||||
PENDING = "pending"
|
||||
CONFIRMED = "confirmed"
|
||||
FAILED = "failed" # threshold not met by epoch deadline
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EpochCheckpoint:
|
||||
"""One epoch's chain-state commitment.
|
||||
|
||||
``root_hash`` is computed over the epoch's chain events using
|
||||
``canonical_epoch_root``. ``participating_heavy_node_ids``
|
||||
records which Heavy Nodes have signed off on this root —
|
||||
confirmation requires ``len(participating) / total_heavy >=
|
||||
threshold``.
|
||||
|
||||
Sprint 10 simplification: ``signatures`` is a dict from
|
||||
``heavy_node_id`` to a placeholder bytes blob. Production wires
|
||||
in the chosen threshold-signature scheme (BLS, FROST, etc.) —
|
||||
those signatures aggregate into a single root signature, but
|
||||
Sprint 10's structural model just tracks who signed.
|
||||
"""
|
||||
epoch_id: int
|
||||
root_hash: str
|
||||
epoch_start_ts: float
|
||||
epoch_end_ts: float
|
||||
participating_heavy_node_ids: frozenset[str] = frozenset()
|
||||
signatures: dict[str, bytes] = field(default_factory=dict)
|
||||
threshold: float = 0.67 # 67% of Heavy Nodes — same as upgrade activation
|
||||
|
||||
def participation_fraction(self, *, total_heavy_nodes: int) -> float:
|
||||
if total_heavy_nodes <= 0:
|
||||
return 0.0
|
||||
return len(self.participating_heavy_node_ids) / total_heavy_nodes
|
||||
|
||||
def status(self, *, total_heavy_nodes: int, now: float) -> EpochCheckpointStatus:
|
||||
if self.participation_fraction(total_heavy_nodes=total_heavy_nodes) >= self.threshold:
|
||||
return EpochCheckpointStatus.CONFIRMED
|
||||
if now > self.epoch_end_ts:
|
||||
return EpochCheckpointStatus.FAILED
|
||||
return EpochCheckpointStatus.PENDING
|
||||
|
||||
|
||||
def canonical_epoch_root(
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
epoch_start_ts: float,
|
||||
epoch_end_ts: float,
|
||||
) -> str:
|
||||
"""SHA-256 over canonically-serialized events in the epoch window.
|
||||
|
||||
Events are filtered by ``epoch_start_ts <= timestamp < epoch_end_ts``
|
||||
and sorted by ``(timestamp, sequence, event_id-or-hash)`` for
|
||||
deterministic ordering. Empty epoch returns the SHA-256 of the
|
||||
empty string (so even an "empty" epoch has a stable root).
|
||||
|
||||
Every Heavy Node computing this from the same chain prefix gets
|
||||
the same hex string. Disagreement on this value is the signal
|
||||
that a partition has produced divergent histories.
|
||||
"""
|
||||
in_window: list[dict[str, Any]] = []
|
||||
for ev in chain:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
try:
|
||||
ts = float(ev.get("timestamp") or 0.0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if epoch_start_ts <= ts < epoch_end_ts:
|
||||
in_window.append(ev)
|
||||
|
||||
in_window.sort(key=lambda e: (
|
||||
float(e.get("timestamp") or 0.0),
|
||||
int(e.get("sequence") or 0),
|
||||
str(e.get("event_id") or ""),
|
||||
))
|
||||
|
||||
h = hashlib.sha256()
|
||||
for ev in in_window:
|
||||
encoded = json.dumps(
|
||||
ev, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
|
||||
)
|
||||
h.update(encoded.encode("utf-8"))
|
||||
h.update(b"\n")
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def is_checkpoint_confirmed(
|
||||
checkpoint: EpochCheckpoint,
|
||||
*,
|
||||
total_heavy_nodes: int,
|
||||
now: float,
|
||||
) -> bool:
|
||||
"""Convenience: ``True`` iff the checkpoint has reached the
|
||||
Heavy Node threshold."""
|
||||
return (
|
||||
checkpoint.status(total_heavy_nodes=total_heavy_nodes, now=now)
|
||||
== EpochCheckpointStatus.CONFIRMED
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EpochCheckpoint",
|
||||
"EpochCheckpointStatus",
|
||||
"canonical_epoch_root",
|
||||
"is_checkpoint_confirmed",
|
||||
]
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Provisional-flag heuristic — chain-staleness detection.
|
||||
|
||||
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §3.7
|
||||
("initial implementation can gate economic events with
|
||||
``is_provisional=True`` whenever the local chain head's
|
||||
``chain_majority_time`` is older than X seconds").
|
||||
|
||||
Sprint 10 ships the placeholder for full epoch finality. Producers
|
||||
emitting Tier 2 events consult ``should_mark_provisional`` to decide
|
||||
whether to set ``is_provisional=True``. Once the formal epoch
|
||||
checkpoint protocol is shipped (IMPLEMENTATION_PLAN §6.5), this
|
||||
heuristic gets replaced with a check against the latest confirmed
|
||||
checkpoint.
|
||||
|
||||
Until then, the heuristic is: if local chain time hasn't advanced in
|
||||
``DEFAULT_MAX_CHAIN_LAG_S`` seconds, the network is partitioned (or
|
||||
dramatically slow); Tier 2 events emitted now are provisional.
|
||||
|
||||
Cross-cutting design rule: a partitioned node must NOT block the
|
||||
user from emitting actions. Tier 1 actions are always live; Tier 2
|
||||
actions are accepted but marked provisional. Reconnection promotes
|
||||
provisional events to final once the checkpoint clears.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.partition.two_tier_state import classify_event_type
|
||||
from services.infonet.time_validity import chain_majority_time
|
||||
|
||||
|
||||
# Default: 60 seconds. After 1 minute without a chain advance from a
|
||||
# distinct node, Tier 2 events get marked provisional. This is a
|
||||
# conservative default — production deployments will likely tune
|
||||
# higher (5-10 minutes) once the network is large and partitions
|
||||
# are rare. Currently NOT in CONFIG_SCHEMA — see Sprint 10 hand-off
|
||||
# notes for the open governance question.
|
||||
DEFAULT_MAX_CHAIN_LAG_S: float = 60.0
|
||||
|
||||
|
||||
def chain_lag_seconds(
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float,
|
||||
) -> float:
|
||||
"""Seconds elapsed between ``chain_majority_time(chain)`` and ``now``.
|
||||
|
||||
Returns ``0.0`` if ``now`` is at or before chain time (clock skew
|
||||
or the chain genuinely caught up just now). Always non-negative.
|
||||
"""
|
||||
cmt = chain_majority_time(chain)
|
||||
if cmt <= 0:
|
||||
# Empty chain — no events from distinct nodes yet. Treat as
|
||||
# "infinite lag" so Tier 2 emissions are provisional.
|
||||
return float("inf")
|
||||
return max(0.0, float(now) - cmt)
|
||||
|
||||
|
||||
def is_chain_stale(
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float,
|
||||
max_lag_seconds: float = DEFAULT_MAX_CHAIN_LAG_S,
|
||||
) -> bool:
|
||||
"""``True`` iff the chain hasn't advanced in ``max_lag_seconds``."""
|
||||
return chain_lag_seconds(chain, now=now) > float(max_lag_seconds)
|
||||
|
||||
|
||||
def should_mark_provisional(
|
||||
event_type: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float,
|
||||
max_lag_seconds: float = DEFAULT_MAX_CHAIN_LAG_S,
|
||||
) -> bool:
|
||||
"""Should ``event_type`` carry ``is_provisional=True`` if emitted now?
|
||||
|
||||
Tier 1 events: always ``False`` (they're CRDT-friendly).
|
||||
Tier 2 events: ``True`` iff chain is stale.
|
||||
Infrastructure / unknown: ``False`` (no economic finality at stake).
|
||||
"""
|
||||
tier = classify_event_type(event_type)
|
||||
if tier != "tier2":
|
||||
return False
|
||||
return is_chain_stale(chain, now=now, max_lag_seconds=max_lag_seconds)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_MAX_CHAIN_LAG_S",
|
||||
"chain_lag_seconds",
|
||||
"is_chain_stale",
|
||||
"should_mark_provisional",
|
||||
]
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Tier 1 / Tier 2 event-type classification.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.14 Rule 4.
|
||||
|
||||
Tier 1 events:
|
||||
|
||||
- **Eventually consistent.** A node operating in a partition can
|
||||
produce them locally; on reconnect they merge into the global
|
||||
view without conflict.
|
||||
- **CRDT-friendly.** No total ordering required.
|
||||
- Examples: upreps, gate enter/exit, gate messages (off-chain),
|
||||
citizenship claim signal, content posts.
|
||||
|
||||
Tier 2 events:
|
||||
|
||||
- **Epoch finality required.** A node operating in a partition can
|
||||
*propose* them locally with ``is_provisional=True``, but they MUST
|
||||
NOT become economically final until an epoch checkpoint confirms
|
||||
the chain head.
|
||||
- Examples: oracle rep minting (via ``resolution_finalize``),
|
||||
governance execution, dispute outcomes.
|
||||
|
||||
The classifier returns "tier1", "tier2", or "infrastructure" (for
|
||||
event types that don't directly affect economic state — e.g.
|
||||
``node_register``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from services.infonet.schema import INFONET_ECONOMY_EVENT_TYPES
|
||||
|
||||
|
||||
# Sprint 10 baseline classification. Future governance can rebalance
|
||||
# via upgrade-hash governance (the classification is a constitutional
|
||||
# property — moving an event between tiers changes finality semantics).
|
||||
|
||||
TIER1_EVENT_TYPES: frozenset[str] = frozenset({
|
||||
# Reputation surface — common rep is fully chain-derived and
|
||||
# CRDT-friendly. Upreps from disjoint partitions add commutatively.
|
||||
"uprep",
|
||||
"downrep",
|
||||
# Gate membership — entering / exiting a gate is local action;
|
||||
# final view is just the union (modulo exit removals).
|
||||
"gate_enter",
|
||||
"gate_exit",
|
||||
"gate_lock", # locking is a vote — partition-local locks count
|
||||
# Content / citizenship signals — pure local actions.
|
||||
"post_create",
|
||||
"post_reply",
|
||||
"citizenship_claim",
|
||||
# Predictions are local; what's NOT Tier 1 is the resolution.
|
||||
"prediction_create",
|
||||
"prediction_place",
|
||||
# Truth stakes — same: placing is Tier 1, resolving is Tier 2.
|
||||
"truth_stake_place",
|
||||
# Bounty creation / claim acknowledgements — local action.
|
||||
"bounty_create",
|
||||
"bounty_claim",
|
||||
})
|
||||
|
||||
|
||||
TIER2_EVENT_TYPES: frozenset[str] = frozenset({
|
||||
# Resolution finality — must be confirmed by epoch checkpoint
|
||||
# before oracle_rep mints.
|
||||
"market_snapshot",
|
||||
"evidence_submit",
|
||||
"resolution_stake",
|
||||
"bootstrap_resolution_vote",
|
||||
"resolution_finalize",
|
||||
# Truth stake resolution.
|
||||
"truth_stake_resolve",
|
||||
# Disputes — the bounded-reversal mechanic depends on a stable
|
||||
# global view; partition-only dispute resolution would diverge.
|
||||
"dispute_open",
|
||||
"dispute_stake",
|
||||
"dispute_resolve",
|
||||
# Gate shutdown — irreversible state change; must reach global
|
||||
# consensus before execute.
|
||||
"gate_suspend_file",
|
||||
"gate_suspend_vote",
|
||||
"gate_suspend_execute",
|
||||
"gate_shutdown_file",
|
||||
"gate_shutdown_vote",
|
||||
"gate_shutdown_execute",
|
||||
"gate_unsuspend",
|
||||
"gate_shutdown_appeal_file",
|
||||
"gate_shutdown_appeal_vote",
|
||||
"gate_shutdown_appeal_resolve",
|
||||
# Governance — petitions and upgrades affect protocol params /
|
||||
# release hash globally. Must not execute provisionally.
|
||||
"petition_file",
|
||||
"petition_sign",
|
||||
"petition_vote",
|
||||
"challenge_file",
|
||||
"challenge_vote",
|
||||
"petition_execute",
|
||||
"upgrade_propose",
|
||||
"upgrade_sign",
|
||||
"upgrade_vote",
|
||||
"upgrade_challenge",
|
||||
"upgrade_challenge_vote",
|
||||
"upgrade_signal_ready",
|
||||
"upgrade_activate",
|
||||
# Coin events — when shipped, must not double-mint across
|
||||
# partitions. (Sprint 9 currently SKIPPED; classification kept
|
||||
# so it's ready when un-skipped.)
|
||||
"coin_transfer",
|
||||
"coin_mint",
|
||||
# Identity rotation — re-keying must reach global consensus.
|
||||
"identity_rotate",
|
||||
})
|
||||
|
||||
|
||||
# Infrastructure events — neither tier (don't directly drive
|
||||
# economic state). Currently just node_register.
|
||||
_INFRASTRUCTURE_TYPES: frozenset[str] = frozenset({
|
||||
"node_register",
|
||||
})
|
||||
|
||||
|
||||
def classify_event_type(event_type: str) -> str:
|
||||
"""Return ``"tier1"`` / ``"tier2"`` / ``"infrastructure"`` /
|
||||
``"unknown"``.
|
||||
|
||||
Validates the classification covers the entire
|
||||
``INFONET_ECONOMY_EVENT_TYPES`` surface — Sprint 10's invariant
|
||||
test asserts this.
|
||||
"""
|
||||
if event_type in TIER1_EVENT_TYPES:
|
||||
return "tier1"
|
||||
if event_type in TIER2_EVENT_TYPES:
|
||||
return "tier2"
|
||||
if event_type in _INFRASTRUCTURE_TYPES:
|
||||
return "infrastructure"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def assert_classification_complete() -> None:
|
||||
"""Sprint 10 invariant: every economy event type is classified.
|
||||
|
||||
Called from the test suite. Raising at import time would be too
|
||||
aggressive — a future event type added without a tier assignment
|
||||
should fail loudly in CI, not crash production.
|
||||
"""
|
||||
classified = TIER1_EVENT_TYPES | TIER2_EVENT_TYPES | _INFRASTRUCTURE_TYPES
|
||||
missing = sorted(INFONET_ECONOMY_EVENT_TYPES - classified)
|
||||
if missing:
|
||||
raise AssertionError(
|
||||
f"Tier classification incomplete — these event types have no "
|
||||
f"tier assignment: {missing}. Add them to TIER1_EVENT_TYPES, "
|
||||
f"TIER2_EVENT_TYPES, or _INFRASTRUCTURE_TYPES."
|
||||
)
|
||||
overlap = TIER1_EVENT_TYPES & TIER2_EVENT_TYPES
|
||||
if overlap:
|
||||
raise AssertionError(
|
||||
f"Tier classification overlapping — these types are in both "
|
||||
f"Tier 1 and Tier 2: {sorted(overlap)}"
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"TIER1_EVENT_TYPES",
|
||||
"TIER2_EVENT_TYPES",
|
||||
"assert_classification_complete",
|
||||
"classify_event_type",
|
||||
]
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Privacy layer scaffolding (Sprint 11+ runway).
|
||||
|
||||
The privacy layer protects the protocol's core promise:
|
||||
**your identity is your reputation, not your legal name**.
|
||||
|
||||
Constitutional anchors (IMPLEMENTATION_PLAN.md §4):
|
||||
|
||||
- **Reputation chain is fully public.** Every uprep / prediction /
|
||||
vote / governance action is signed and visible. The privacy layer
|
||||
does NOT hide reputation actions.
|
||||
- **Coin ledger is privacy-preserving.** When the coin layer ships,
|
||||
transfers / balances / DEX trades are shielded. Privacy work in
|
||||
this folder is what makes that possible.
|
||||
- **Optional privacy is no privacy.** The default for coin
|
||||
transactions must be shielded — opt-out cannot exist or it
|
||||
destroys the anonymity set.
|
||||
|
||||
This package is intentionally **scaffolding only** at present. Each
|
||||
primitive (RingCT, stealth addresses, shielded balance commitments,
|
||||
DEX) defines its public interface as a typed Protocol so production
|
||||
code can depend on the *shape* before any specific cryptographic
|
||||
implementation is committed.
|
||||
|
||||
The non-cryptographic pieces of the Function Keys design (nullifier
|
||||
hashing, challenge-response orchestration, two-phase commit receipts,
|
||||
batched settlement aggregation) ARE implemented here in pure Python.
|
||||
The remaining cryptographic primitive (blind signature / anonymous
|
||||
credential scheme) is the only piece blocking production deployment
|
||||
of Function Keys; everything around it is ready.
|
||||
|
||||
See ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4 and
|
||||
``infonet-economy/BRAINDUMP.md`` §5.6, §11 item 9 for design
|
||||
rationale.
|
||||
"""
|
||||
|
||||
from services.infonet.privacy.contracts import (
|
||||
BalanceCommitment,
|
||||
DEXOrderBook,
|
||||
PrivacyPrimitiveStatus,
|
||||
RingSignatureScheme,
|
||||
StealthAddressScheme,
|
||||
)
|
||||
from services.infonet.privacy.dex import DEXScaffolding
|
||||
from services.infonet.privacy.function_keys import (
|
||||
BatchedSettlementBatch,
|
||||
DenialCode,
|
||||
FunctionKey,
|
||||
FunctionKeyChallenge,
|
||||
FunctionKeyResponse,
|
||||
NullifierTracker,
|
||||
Receipt,
|
||||
ReceiptPair,
|
||||
derive_nullifier,
|
||||
issue_challenge,
|
||||
sign_response,
|
||||
verify_response,
|
||||
)
|
||||
from services.infonet.privacy.ringct import RingCTScaffolding
|
||||
from services.infonet.privacy.shielded_balance import ShieldedBalanceScaffolding
|
||||
from services.infonet.privacy.stealth_address import StealthAddressScaffolding
|
||||
|
||||
__all__ = [
|
||||
"BalanceCommitment",
|
||||
"BatchedSettlementBatch",
|
||||
"DEXOrderBook",
|
||||
"DEXScaffolding",
|
||||
"DenialCode",
|
||||
"FunctionKey",
|
||||
"FunctionKeyChallenge",
|
||||
"FunctionKeyResponse",
|
||||
"NullifierTracker",
|
||||
"PrivacyPrimitiveStatus",
|
||||
"Receipt",
|
||||
"ReceiptPair",
|
||||
"RingCTScaffolding",
|
||||
"RingSignatureScheme",
|
||||
"ShieldedBalanceScaffolding",
|
||||
"StealthAddressScaffolding",
|
||||
"StealthAddressScheme",
|
||||
"derive_nullifier",
|
||||
"issue_challenge",
|
||||
"sign_response",
|
||||
"verify_response",
|
||||
]
|
||||
@@ -0,0 +1,190 @@
|
||||
"""Typed protocols for the cryptographic primitives.
|
||||
|
||||
Production code depends on the **shape** of each privacy primitive
|
||||
through a ``Protocol`` defined here. Concrete implementations (Rust
|
||||
binding, Python reference, test mock) all match the same shape, so
|
||||
swapping them is a one-line import change.
|
||||
|
||||
Sprint 11+ ships:
|
||||
|
||||
- A reference Python implementation for testing (probably built on
|
||||
``cryptography`` or ``ecdsa`` packages, narrow scope).
|
||||
- A production Rust binding via ``privacy-core`` crate.
|
||||
|
||||
Today (Sprint 11+ runway), this module ships:
|
||||
|
||||
- ``Protocol``s for each primitive.
|
||||
- ``PrivacyPrimitiveStatus`` enum so callers can introspect which
|
||||
implementations are wired in.
|
||||
- A registry of "not yet implemented" statuses with diagnostic
|
||||
pointers for future implementers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
|
||||
class PrivacyPrimitiveStatus(str, Enum):
|
||||
"""Lifecycle status for each privacy primitive.
|
||||
|
||||
Used by health endpoints / UI to communicate "this feature is
|
||||
not yet shielded" honestly. The cross-cutting non-hostile UX
|
||||
rule (BUILD_LOG.md design rules §1) forbids silently pretending
|
||||
a primitive is ready when it isn't — surface the truth.
|
||||
"""
|
||||
NOT_IMPLEMENTED = "not_implemented"
|
||||
SCAFFOLDING = "scaffolding"
|
||||
REFERENCE_IMPL = "reference_impl"
|
||||
PRODUCTION_RUST = "production_rust"
|
||||
|
||||
|
||||
# ─── Ring confidential transactions ─────────────────────────────────────
|
||||
|
||||
@runtime_checkable
|
||||
class RingSignatureScheme(Protocol):
|
||||
"""Signs a transaction with a ring of public keys, hiding which
|
||||
member of the ring actually signed.
|
||||
|
||||
Implementations must guarantee:
|
||||
|
||||
- **Unforgeable.** Without one of the ring members' private keys,
|
||||
no valid ring signature exists for the transaction.
|
||||
- **Anonymous within the ring.** Verifiers learn that *some*
|
||||
ring member signed, not which.
|
||||
- **Linkable.** Two signatures from the same private key produce
|
||||
the same ``key image`` (used to detect double-spends).
|
||||
"""
|
||||
|
||||
def sign(
|
||||
self,
|
||||
*,
|
||||
message: bytes,
|
||||
signer_private_key: bytes,
|
||||
ring_public_keys: list[bytes],
|
||||
) -> dict[str, Any]:
|
||||
"""Return ``{"signature": ..., "key_image": ...}``."""
|
||||
...
|
||||
|
||||
def verify(
|
||||
self,
|
||||
*,
|
||||
message: bytes,
|
||||
signature: dict[str, Any],
|
||||
ring_public_keys: list[bytes],
|
||||
) -> bool: ...
|
||||
|
||||
def status(self) -> PrivacyPrimitiveStatus: ...
|
||||
|
||||
|
||||
# ─── Stealth addresses ──────────────────────────────────────────────────
|
||||
|
||||
@runtime_checkable
|
||||
class StealthAddressScheme(Protocol):
|
||||
"""Derives a one-time recipient address per transaction.
|
||||
|
||||
Implementations must guarantee:
|
||||
|
||||
- **Unlinkable.** An external observer cannot tell that two
|
||||
stealth addresses belong to the same recipient.
|
||||
- **Recipient-recoverable.** Only the recipient (using their
|
||||
view key) can determine that an output is theirs.
|
||||
"""
|
||||
|
||||
def derive_one_time_address(
|
||||
self,
|
||||
*,
|
||||
recipient_view_key: bytes,
|
||||
recipient_spend_key: bytes,
|
||||
sender_random: bytes,
|
||||
) -> bytes: ...
|
||||
|
||||
def is_for_recipient(
|
||||
self,
|
||||
*,
|
||||
one_time_address: bytes,
|
||||
recipient_view_key: bytes,
|
||||
recipient_spend_key: bytes,
|
||||
sender_random: bytes,
|
||||
) -> bool: ...
|
||||
|
||||
def status(self) -> PrivacyPrimitiveStatus: ...
|
||||
|
||||
|
||||
# ─── Shielded balance commitment ────────────────────────────────────────
|
||||
|
||||
@runtime_checkable
|
||||
class BalanceCommitment(Protocol):
|
||||
"""Pedersen / homomorphic commitment to a balance.
|
||||
|
||||
Implementations must allow:
|
||||
|
||||
- Commit to a balance ``B`` with blinding factor ``r``.
|
||||
- Verify a sum-of-commitments equals zero (proving inputs ==
|
||||
outputs without revealing amounts).
|
||||
- Range proofs (proving each output is non-negative).
|
||||
"""
|
||||
|
||||
def commit(self, *, amount: int, blinding: bytes) -> bytes: ...
|
||||
|
||||
def verify_balance(
|
||||
self,
|
||||
*,
|
||||
input_commitments: list[bytes],
|
||||
output_commitments: list[bytes],
|
||||
) -> bool: ...
|
||||
|
||||
def range_proof(
|
||||
self,
|
||||
*,
|
||||
amount: int,
|
||||
blinding: bytes,
|
||||
max_bits: int = 64,
|
||||
) -> bytes: ...
|
||||
|
||||
def verify_range_proof(
|
||||
self,
|
||||
*,
|
||||
commitment: bytes,
|
||||
proof: bytes,
|
||||
max_bits: int = 64,
|
||||
) -> bool: ...
|
||||
|
||||
def status(self) -> PrivacyPrimitiveStatus: ...
|
||||
|
||||
|
||||
# ─── DEX order book ─────────────────────────────────────────────────────
|
||||
|
||||
@runtime_checkable
|
||||
class DEXOrderBook(Protocol):
|
||||
"""Privacy-preserving decentralized exchange interface.
|
||||
|
||||
DEX operates ON TOP of the shielded coin layer — orders reference
|
||||
shielded inputs/outputs, settlement burns + mints shielded
|
||||
commitments. The ``DEXOrderBook`` Protocol is intentionally
|
||||
abstract because the specific scheme (CoW-style batched
|
||||
settlement, atomic swap, MimbleWimble-flavored aggregation) is
|
||||
still open per IMPLEMENTATION_PLAN.md §6.4.
|
||||
"""
|
||||
|
||||
def place_order(self, *, order: dict[str, Any]) -> str:
|
||||
"""Return the on-chain ``order_id``."""
|
||||
...
|
||||
|
||||
def cancel_order(self, *, order_id: str, owner_signature: bytes) -> None: ...
|
||||
|
||||
def match_orders(self) -> list[dict[str, Any]]:
|
||||
"""Return the list of matched trades for atomic settlement."""
|
||||
...
|
||||
|
||||
def status(self) -> PrivacyPrimitiveStatus: ...
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BalanceCommitment",
|
||||
"DEXOrderBook",
|
||||
"PrivacyPrimitiveStatus",
|
||||
"RingSignatureScheme",
|
||||
"StealthAddressScheme",
|
||||
]
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Decentralized exchange — Sprint 11+ scaffolding.
|
||||
|
||||
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.3,
|
||||
§6.4 (open: same chain vs side-chain).
|
||||
|
||||
The DEX operates on top of the shielded coin layer. Orders reference
|
||||
shielded inputs and outputs; settlement burns + mints commitments
|
||||
atomically. The Sprint 11+ scaffolding here defines the order /
|
||||
settlement shapes without committing to a specific matching scheme
|
||||
(CoW-style batch auction, atomic swap, etc.).
|
||||
|
||||
External exchanges WILL list CommonCoin regardless of protocol
|
||||
design — the protocol's privacy layer is what prevents external-
|
||||
exchange listings from de-anonymizing protocol participants. The
|
||||
on-chain DEX is the *primary* exchange mechanism, not the only one.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from services.infonet.privacy.contracts import PrivacyPrimitiveStatus
|
||||
|
||||
|
||||
class DEXScaffolding:
|
||||
"""Placeholder until the DEX scheme decision (§6.4) is made and a
|
||||
matching engine is built on top of the shielded coin layer."""
|
||||
|
||||
_DIAGNOSTIC = (
|
||||
"DEX is scaffolding only — see IMPLEMENTATION_PLAN.md §6.4 "
|
||||
"for the open scheme decision (same chain vs side-chain) and "
|
||||
"§4.3 for the privacy requirements. Production implementation "
|
||||
"depends on the shielded coin layer being shipped first."
|
||||
)
|
||||
|
||||
def place_order(self, *, order: dict[str, Any]) -> str:
|
||||
raise NotImplementedError(self._DIAGNOSTIC)
|
||||
|
||||
def cancel_order(self, *, order_id: str, owner_signature: bytes) -> None:
|
||||
raise NotImplementedError(self._DIAGNOSTIC)
|
||||
|
||||
def match_orders(self) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError(self._DIAGNOSTIC)
|
||||
|
||||
def status(self) -> PrivacyPrimitiveStatus:
|
||||
return PrivacyPrimitiveStatus.NOT_IMPLEMENTED
|
||||
|
||||
|
||||
__all__ = ["DEXScaffolding"]
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Function Keys — anonymous citizenship proof.
|
||||
|
||||
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.4,
|
||||
``infonet-economy/BRAINDUMP.md`` §11 item 9.
|
||||
|
||||
A citizen should be able to prove "I am a UBI-eligible Infonet
|
||||
citizen" to a real-world operator (food bank, community service)
|
||||
**without revealing their Infonet identity**. The naive approach
|
||||
(scramble a public key, record each redemption on chain) leaks
|
||||
identity through metadata correlation (time, location, operator,
|
||||
frequency).
|
||||
|
||||
The full design has six pieces; five are implemented in pure Python
|
||||
here. The remaining piece — issuance via blind signatures or
|
||||
anonymous credentials — is the only cryptographic primitive that
|
||||
needs an external library.
|
||||
|
||||
Pieces:
|
||||
|
||||
1. **Issuance** (NOT IMPLEMENTED — needs blind sig / BBS+ / U-Prove
|
||||
/ Idemix). The ``FunctionKey`` dataclass models what an issued
|
||||
key looks like; production wires the issuer through a Protocol
|
||||
when the scheme is chosen.
|
||||
2. **Nullifiers** (`nullifier.py`) — SHA-256 of secret + operator_id.
|
||||
Different operators see different nullifiers for the same key,
|
||||
so cross-operator linkage is impossible. One-time-use per
|
||||
operator: tracked via ``NullifierTracker``.
|
||||
3. **Challenge-response** (`challenge_response.py`) — operator
|
||||
issues a fresh nonce, key-holder signs with the Function Key's
|
||||
secret. Prevents screenshot attacks, key sharing, replay.
|
||||
4. **Two-phase commit receipts** (`receipt.py`) — Phase 1
|
||||
verification receipt (operator-signed, day-level date NOT
|
||||
timestamp, no node_id). Phase 2 fulfillment receipt (citizen
|
||||
counter-signs after service rendered). Receipts NEVER published
|
||||
on-chain — only surface on dispute.
|
||||
5. **Enumerated denial codes** (`receipt.py`) — operators can
|
||||
reject for exactly three reasons: invalid signature, nullifier
|
||||
already seen, rate limit exceeded. Prevents discrimination via
|
||||
freeform rejection.
|
||||
6. **Batched/coarse-grained settlement** (`batched_settlement.py`)
|
||||
— operators settle in aggregate. Chain sees "Operator X
|
||||
verified N function keys this period." Per-redemption records
|
||||
never reach the chain.
|
||||
|
||||
Cross-cutting design rule: the user redeeming a Function Key must
|
||||
not be blocked by privacy/security mechanics. If the cryptographic
|
||||
primitive is unavailable in the local node, the redemption is
|
||||
queued for retry once the operator has connectivity, NOT refused.
|
||||
"""
|
||||
|
||||
from services.infonet.privacy.function_keys.batched_settlement import (
|
||||
BatchedSettlementBatch,
|
||||
)
|
||||
from services.infonet.privacy.function_keys.challenge_response import (
|
||||
FunctionKey,
|
||||
FunctionKeyChallenge,
|
||||
FunctionKeyResponse,
|
||||
issue_challenge,
|
||||
sign_response,
|
||||
verify_response,
|
||||
)
|
||||
from services.infonet.privacy.function_keys.nullifier import (
|
||||
NullifierTracker,
|
||||
derive_nullifier,
|
||||
)
|
||||
from services.infonet.privacy.function_keys.receipt import (
|
||||
DenialCode,
|
||||
Receipt,
|
||||
ReceiptPair,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"BatchedSettlementBatch",
|
||||
"DenialCode",
|
||||
"FunctionKey",
|
||||
"FunctionKeyChallenge",
|
||||
"FunctionKeyResponse",
|
||||
"NullifierTracker",
|
||||
"Receipt",
|
||||
"ReceiptPair",
|
||||
"derive_nullifier",
|
||||
"issue_challenge",
|
||||
"sign_response",
|
||||
"verify_response",
|
||||
]
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Batched settlement — aggregate counts, no individual records on-chain.
|
||||
|
||||
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.4
|
||||
piece 6.
|
||||
|
||||
Per-redemption records on-chain would be a privacy disaster: an
|
||||
observer could correlate "Operator X verified a Function Key at
|
||||
14:32" with a citizen's known activities to de-anonymize them.
|
||||
|
||||
Instead, operators settle in **aggregate**. The chain sees only
|
||||
``(operator_id, day_bucket, count)`` — verified N keys this day.
|
||||
Fraud detection happens via statistical auditing rather than
|
||||
per-redemption traces:
|
||||
|
||||
- Operator's count vs their declared population (food bank that
|
||||
reports 10,000 daily verifications when their service capacity
|
||||
is 200).
|
||||
- Distribution shape vs other operators (significant outliers
|
||||
prompt review).
|
||||
- Spot audits via dispute mechanism (citizen + operator surface
|
||||
receipt pair to adjudicator).
|
||||
|
||||
The ``BatchedSettlementBatch`` here is what the operator emits
|
||||
to chain at the end of a settlement period. Receipts NEVER appear
|
||||
on-chain — they remain off-chain with both parties.
|
||||
|
||||
Sprint 11+ scaffolding ships:
|
||||
|
||||
- The aggregate batch dataclass.
|
||||
- A ``record_redemption`` helper that operators call locally per
|
||||
successful redemption — increments the batch's counter without
|
||||
storing the receipt.
|
||||
- A ``finalize_batch`` step that produces the on-chain payload.
|
||||
|
||||
This module is **fully implementable** today — it does no
|
||||
cryptography, just bookkeeping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchedSettlementBatch:
|
||||
"""Operator-side batch counter for one settlement period.
|
||||
|
||||
Operators construct one of these per ``(period_id, operator_id)``
|
||||
pair, increment via ``record_redemption`` per successful
|
||||
redemption, and emit the finalized batch payload at period end.
|
||||
|
||||
The data model is intentionally minimal:
|
||||
|
||||
- ``period_id`` — the settlement window identifier (e.g.
|
||||
``"2026-04"`` for monthly).
|
||||
- ``operator_id`` — committed publicly on-chain so its
|
||||
non-forgeability is anchored.
|
||||
- ``successful_count`` — number of successful redemptions
|
||||
(verification + fulfillment).
|
||||
- ``denial_counts`` — counts per enumerated DenialCode for
|
||||
audit visibility. NO per-receipt detail.
|
||||
"""
|
||||
|
||||
period_id: str
|
||||
operator_id: str
|
||||
successful_count: int = 0
|
||||
denial_counts: dict[str, int] = field(default_factory=dict)
|
||||
finalized: bool = False
|
||||
|
||||
def record_redemption(self) -> None:
|
||||
"""Increment the success counter. NOT idempotent — call
|
||||
exactly once per successful (verification, fulfillment)
|
||||
receipt pair the operator commits to."""
|
||||
if self.finalized:
|
||||
raise RuntimeError("batch already finalized; cannot record")
|
||||
self.successful_count += 1
|
||||
|
||||
def record_denial(self, code: str) -> None:
|
||||
"""Track a denial. Operators MUST use one of the enumerated
|
||||
``DenialCode`` values — Sprint 11+ scaffolding accepts the
|
||||
string for convenience but production callers should pass
|
||||
the enum's ``.value``."""
|
||||
if self.finalized:
|
||||
raise RuntimeError("batch already finalized; cannot record")
|
||||
if not isinstance(code, str) or not code:
|
||||
raise ValueError("denial code must be a non-empty string")
|
||||
self.denial_counts[code] = self.denial_counts.get(code, 0) + 1
|
||||
|
||||
def finalize(self) -> dict:
|
||||
"""Produce the on-chain payload for this batch.
|
||||
|
||||
After ``finalize()``, ``record_redemption`` and
|
||||
``record_denial`` raise. The returned dict is the canonical
|
||||
batched-settlement event payload.
|
||||
|
||||
Privacy property: per-receipt detail is NOT in the output.
|
||||
Only counts. The operator may discard receipts after
|
||||
finalization (subject to local retention policy for dispute
|
||||
defense).
|
||||
"""
|
||||
if self.finalized:
|
||||
raise RuntimeError("batch already finalized")
|
||||
self.finalized = True
|
||||
return {
|
||||
"period_id": self.period_id,
|
||||
"operator_id": self.operator_id,
|
||||
"successful_count": int(self.successful_count),
|
||||
"denial_counts": {k: int(v) for k, v in self.denial_counts.items()},
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["BatchedSettlementBatch"]
|
||||
@@ -0,0 +1,208 @@
|
||||
"""Challenge-response — live proof of Function Key possession.
|
||||
|
||||
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.4
|
||||
piece 3.
|
||||
|
||||
Operator issues a fresh nonce; key-holder signs (challenge || nonce
|
||||
|| epoch_window) with the Function Key's secret. Operator verifies
|
||||
by re-deriving the signature.
|
||||
|
||||
This defends against:
|
||||
|
||||
- **Screenshot attacks** — a recorded "valid proof" from yesterday
|
||||
is useless against today's challenge.
|
||||
- **Key sharing** — without the live secret, no valid response
|
||||
exists; sharing the secret = sharing the key (which has its own
|
||||
social cost via public reputation).
|
||||
- **Replay** — the operator stores recent nonces; replayed
|
||||
responses are rejected.
|
||||
|
||||
Sprint 11+ scaffolding ships:
|
||||
|
||||
- The ``FunctionKey`` dataclass (the post-issuance shape).
|
||||
- The challenge / response message structures.
|
||||
- A pure-Python ``sign_response`` / ``verify_response`` pair using
|
||||
HMAC-SHA256 as the placeholder MAC scheme. Production wires this
|
||||
through the eventual blind-sig / anonymous credential primitive.
|
||||
|
||||
The HMAC placeholder is **explicitly NOT secure for unlinkable
|
||||
issuance** — it leaks issuer identity through the verification key.
|
||||
But it's correctly-shaped for testing the rest of the pipeline
|
||||
(nullifier flow, receipt flow, batched settlement) without blocking
|
||||
on the cryptographic decision in IMPLEMENTATION_PLAN §6.4.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
# Maximum age (in seconds) for a challenge. Outside this window, the
|
||||
# response is rejected. Defaults to 5 minutes — short enough to defeat
|
||||
# screenshot attacks, long enough to survive normal network latency on
|
||||
# slow operator hardware.
|
||||
DEFAULT_CHALLENGE_TTL_SECONDS = 300
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FunctionKey:
|
||||
"""Post-issuance Function Key.
|
||||
|
||||
``secret`` is what the citizen retains; production keys derive
|
||||
additional fields (like ``epoch`` and ``credential``). The blind-
|
||||
signature implementation populates ``credential`` with the
|
||||
issuer's signature on the secret + epoch.
|
||||
|
||||
Sprint 11+ scaffolding: ``credential`` is just bytes — the
|
||||
semantic depends on the chosen scheme. Tests can use any
|
||||
deterministic value.
|
||||
"""
|
||||
secret: bytes
|
||||
epoch: str
|
||||
credential: bytes
|
||||
# The issuer's verification context — production stores the
|
||||
# public params needed to verify ``credential``. Sprint 11+
|
||||
# scaffolding accepts any opaque bytes.
|
||||
issuer_context: bytes = b""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FunctionKeyChallenge:
|
||||
"""An operator-generated fresh challenge.
|
||||
|
||||
The ``nonce`` is the entropy source; ``operator_id`` ties the
|
||||
challenge to a specific operator (so cross-operator response
|
||||
reuse is impossible); ``issued_at`` is the start of the TTL
|
||||
window.
|
||||
"""
|
||||
nonce: bytes
|
||||
operator_id: str
|
||||
issued_at: float
|
||||
|
||||
def canonical_bytes(self) -> bytes:
|
||||
# Pipe-delimited UTF-8 — same canonicalization style as the
|
||||
# Sprint 8 PoW preimage so the convention is uniform.
|
||||
return b"|".join([
|
||||
b"function_key_challenge",
|
||||
self.nonce,
|
||||
self.operator_id.encode("utf-8"),
|
||||
repr(self.issued_at).encode("utf-8"),
|
||||
])
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FunctionKeyResponse:
|
||||
"""Citizen's signed response to a challenge."""
|
||||
nonce: bytes
|
||||
operator_id: str
|
||||
issued_at: float
|
||||
nullifier: str
|
||||
mac: bytes # in production: blind-signature proof; here HMAC-SHA256
|
||||
|
||||
|
||||
def issue_challenge(*, operator_id: str, now: float | None = None) -> FunctionKeyChallenge:
|
||||
"""Generate a fresh ``FunctionKeyChallenge`` for ``operator_id``.
|
||||
|
||||
The ``nonce`` is 32 bytes from ``secrets.token_bytes`` — full
|
||||
256-bit entropy, OS-source. ``issued_at`` defaults to
|
||||
``time.time()`` and is included in the canonical bytes so a
|
||||
challenge from yesterday cannot be replayed today.
|
||||
"""
|
||||
if not isinstance(operator_id, str) or not operator_id:
|
||||
raise ValueError("operator_id must be a non-empty string")
|
||||
return FunctionKeyChallenge(
|
||||
nonce=secrets.token_bytes(32),
|
||||
operator_id=operator_id,
|
||||
issued_at=float(now if now is not None else time.time()),
|
||||
)
|
||||
|
||||
|
||||
def sign_response(
|
||||
*,
|
||||
key: FunctionKey,
|
||||
challenge: FunctionKeyChallenge,
|
||||
) -> FunctionKeyResponse:
|
||||
"""Sign a challenge with the Function Key's secret.
|
||||
|
||||
Sprint 11+ placeholder uses HMAC-SHA256 with ``key.secret`` as
|
||||
the MAC key. Production wires the blind-signature scheme here:
|
||||
the response includes a zero-knowledge proof that the holder
|
||||
knows a credential signed by the issuer over the secret +
|
||||
epoch, without revealing which credential.
|
||||
"""
|
||||
from services.infonet.privacy.function_keys.nullifier import derive_nullifier
|
||||
|
||||
nullifier = derive_nullifier(secret=key.secret, operator_id=challenge.operator_id)
|
||||
body = challenge.canonical_bytes() + b"|" + nullifier.encode("utf-8")
|
||||
mac = hmac.new(key.secret, body, hashlib.sha256).digest()
|
||||
return FunctionKeyResponse(
|
||||
nonce=challenge.nonce,
|
||||
operator_id=challenge.operator_id,
|
||||
issued_at=challenge.issued_at,
|
||||
nullifier=nullifier,
|
||||
mac=mac,
|
||||
)
|
||||
|
||||
|
||||
def verify_response(
|
||||
*,
|
||||
response: FunctionKeyResponse,
|
||||
key: FunctionKey,
|
||||
max_age: float = DEFAULT_CHALLENGE_TTL_SECONDS,
|
||||
now: float | None = None,
|
||||
seen_nonces: Iterable[bytes] = (),
|
||||
) -> tuple[bool, str]:
|
||||
"""Verify a response against the matching key + check freshness.
|
||||
|
||||
Returns ``(accepted, reason)``. ``accepted=False`` produces one
|
||||
of these diagnostic reasons:
|
||||
|
||||
- ``"stale_challenge"`` — challenge too old.
|
||||
- ``"replay_nonce_seen"`` — nonce was used in a prior verified
|
||||
response.
|
||||
- ``"invalid_mac"`` — MAC didn't verify against the key.
|
||||
|
||||
Operators MUST track recently-seen nonces (for the duration of
|
||||
the TTL plus a margin) to defeat replay. Pass them in via
|
||||
``seen_nonces``.
|
||||
|
||||
Note on the verifier-knows-the-secret problem: with the HMAC
|
||||
placeholder, the verifier needs ``key.secret`` to verify. That's
|
||||
obviously NOT private — it's why this is a placeholder. The
|
||||
production blind-sig scheme verifies *without* knowing the
|
||||
secret, only the issuer's public verification context.
|
||||
"""
|
||||
seen_set = set(seen_nonces)
|
||||
if response.nonce in seen_set:
|
||||
return False, "replay_nonce_seen"
|
||||
|
||||
age_s = float(now if now is not None else time.time()) - response.issued_at
|
||||
if age_s > max_age or age_s < 0:
|
||||
return False, "stale_challenge"
|
||||
|
||||
challenge = FunctionKeyChallenge(
|
||||
nonce=response.nonce,
|
||||
operator_id=response.operator_id,
|
||||
issued_at=response.issued_at,
|
||||
)
|
||||
body = challenge.canonical_bytes() + b"|" + response.nullifier.encode("utf-8")
|
||||
expected = hmac.new(key.secret, body, hashlib.sha256).digest()
|
||||
if not hmac.compare_digest(expected, response.mac):
|
||||
return False, "invalid_mac"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_CHALLENGE_TTL_SECONDS",
|
||||
"FunctionKey",
|
||||
"FunctionKeyChallenge",
|
||||
"FunctionKeyResponse",
|
||||
"issue_challenge",
|
||||
"sign_response",
|
||||
"verify_response",
|
||||
]
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Nullifiers — one-time-use markers per (key, operator) pair.
|
||||
|
||||
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.4
|
||||
piece 2.
|
||||
|
||||
For each Function Key + operator combination, the nullifier is
|
||||
|
||||
nullifier = SHA-256(secret || operator_id)
|
||||
|
||||
Properties this gives us:
|
||||
|
||||
- **One-time-use per operator.** The operator records the nullifier
|
||||
on first use; subsequent attempts with the same nullifier are
|
||||
rejected (denial code ``NULLIFIER_ALREADY_SEEN``).
|
||||
- **Cross-operator unlinkability.** Different ``operator_id``s
|
||||
produce different nullifiers for the same secret. Two operators
|
||||
comparing notes cannot determine that the same key was used at
|
||||
both — they see two unrelated 32-byte strings.
|
||||
- **No identity leakage.** The nullifier is a hash; the secret is
|
||||
never exposed.
|
||||
|
||||
Operators MUST commit ``operator_id`` publicly so its non-forgeability
|
||||
is anchored on chain. Nullifier derivation depends on a
|
||||
non-forgeable ``operator_id`` (an attacker who could impersonate an
|
||||
operator could harvest nullifiers).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
def derive_nullifier(*, secret: bytes, operator_id: str) -> str:
|
||||
"""Return the hex SHA-256 of ``secret || operator_id`` (UTF-8 for
|
||||
operator_id).
|
||||
|
||||
Stable across reboots / sessions / operating systems — the same
|
||||
inputs always produce the same output. That's the whole property
|
||||
a nullifier needs: deterministic and unforgeable.
|
||||
"""
|
||||
if not isinstance(secret, (bytes, bytearray)):
|
||||
raise TypeError("secret must be bytes")
|
||||
if not isinstance(operator_id, str) or not operator_id:
|
||||
raise ValueError("operator_id must be a non-empty string")
|
||||
h = hashlib.sha256()
|
||||
h.update(bytes(secret))
|
||||
h.update(b"|") # explicit separator so concatenation is unambiguous
|
||||
h.update(operator_id.encode("utf-8"))
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
@dataclass
|
||||
class NullifierTracker:
|
||||
"""Operator-side store of seen nullifiers.
|
||||
|
||||
Sprint 11+ runway: this is the in-memory reference implementation.
|
||||
Production operators use a persistent, atomic-write store
|
||||
(database row + uniqueness constraint) so the "already-seen"
|
||||
check is robust to crashes between the check and the receipt.
|
||||
|
||||
The interface is designed for that: ``check_and_record`` is the
|
||||
only mutation method, and it's atomic — checks then records as
|
||||
one operation. Production wraps this in a database transaction.
|
||||
"""
|
||||
|
||||
seen: set[str] = field(default_factory=set)
|
||||
|
||||
def has_seen(self, nullifier: str) -> bool:
|
||||
return nullifier in self.seen
|
||||
|
||||
def check_and_record(self, nullifier: str) -> bool:
|
||||
"""Return ``True`` if the nullifier was unseen (and is now
|
||||
recorded). Return ``False`` if it was already seen — the
|
||||
operator MUST then issue a denial with code
|
||||
``NULLIFIER_ALREADY_SEEN``.
|
||||
|
||||
The check + record is atomic by design: a concurrent caller
|
||||
racing with this method will not produce two ``True`` results
|
||||
for the same nullifier. (In-memory: trivially atomic. Production:
|
||||
wrap in DB unique-insert.)
|
||||
"""
|
||||
if nullifier in self.seen:
|
||||
return False
|
||||
self.seen.add(nullifier)
|
||||
return True
|
||||
|
||||
|
||||
__all__ = [
|
||||
"NullifierTracker",
|
||||
"derive_nullifier",
|
||||
]
|
||||
@@ -0,0 +1,221 @@
|
||||
"""Two-phase commit receipts + enumerated denial codes.
|
||||
|
||||
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.4
|
||||
pieces 5-6.
|
||||
|
||||
Two phases per redemption:
|
||||
|
||||
1. **Verification receipt** (operator → citizen). Operator signs:
|
||||
``(receipt_id, operator_id, day_bucket, nullifier_prefix)``.
|
||||
Note: NO timestamp (day-bucket only), NO node_id, NO nullifier
|
||||
in full (a prefix that's still distinct enough for fraud auditing
|
||||
but doesn't leak the full unforgeable nullifier).
|
||||
|
||||
2. **Fulfillment receipt** (citizen → operator). Citizen counter-
|
||||
signs the verification receipt after service is rendered. Both
|
||||
parties hold a copy.
|
||||
|
||||
Receipts are NEVER published on-chain. They surface only in
|
||||
disputes. Settlement to chain happens through batched aggregation
|
||||
(``batched_settlement.py``).
|
||||
|
||||
Denial codes are an **enumerated** set with exactly three values.
|
||||
Operators cannot reject for freeform reasons — that would be a
|
||||
discrimination vector. The three reasons are:
|
||||
|
||||
- ``INVALID_SIGNATURE`` — challenge-response verification failed.
|
||||
- ``NULLIFIER_ALREADY_SEEN`` — the (key, operator) pair has already
|
||||
redeemed once.
|
||||
- ``RATE_LIMIT_EXCEEDED`` — operator-defined throttle (per-day,
|
||||
per-hour, etc.) prevents this redemption.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class DenialCode(str, Enum):
|
||||
"""Enumerated rejection reasons. Adding a new code is a hard fork."""
|
||||
INVALID_SIGNATURE = "invalid_signature"
|
||||
NULLIFIER_ALREADY_SEEN = "nullifier_already_seen"
|
||||
RATE_LIMIT_EXCEEDED = "rate_limit_exceeded"
|
||||
|
||||
|
||||
def _day_bucket(timestamp: float) -> str:
|
||||
"""Return the UTC day in ``YYYY-MM-DD`` form for ``timestamp``.
|
||||
|
||||
Day-level granularity prevents fine-grained timestamp metadata
|
||||
from becoming a de-anonymization vector. An operator that issued
|
||||
100 receipts on the same day cannot link them by timestamp —
|
||||
they all carry the same day_bucket.
|
||||
"""
|
||||
return datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Receipt:
|
||||
"""One side of the two-phase commit.
|
||||
|
||||
``role`` is either ``"verification"`` (Phase 1, operator-signed)
|
||||
or ``"fulfillment"`` (Phase 2, citizen counter-signed).
|
||||
"""
|
||||
role: str
|
||||
receipt_id: str
|
||||
operator_id: str
|
||||
day_bucket: str
|
||||
nullifier_prefix: str
|
||||
signature: bytes
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReceiptPair:
|
||||
"""Both phases of a successful redemption.
|
||||
|
||||
Held by both citizen and operator. Surfaces only on dispute —
|
||||
the chain never sees these.
|
||||
"""
|
||||
verification: Receipt
|
||||
fulfillment: Receipt
|
||||
|
||||
|
||||
def _sign(secret: bytes, body: bytes) -> bytes:
|
||||
return hmac.new(secret, body, hashlib.sha256).digest()
|
||||
|
||||
|
||||
def _verify(secret: bytes, body: bytes, signature: bytes) -> bool:
|
||||
expected = _sign(secret, body)
|
||||
return hmac.compare_digest(expected, signature)
|
||||
|
||||
|
||||
def _receipt_body(*, role: str, receipt_id: str, operator_id: str,
|
||||
day_bucket: str, nullifier_prefix: str) -> bytes:
|
||||
return b"|".join([
|
||||
b"function_key_receipt",
|
||||
role.encode("utf-8"),
|
||||
receipt_id.encode("utf-8"),
|
||||
operator_id.encode("utf-8"),
|
||||
day_bucket.encode("utf-8"),
|
||||
nullifier_prefix.encode("utf-8"),
|
||||
])
|
||||
|
||||
|
||||
def issue_verification_receipt(
|
||||
*,
|
||||
operator_id: str,
|
||||
operator_secret: bytes,
|
||||
nullifier: str,
|
||||
timestamp: float,
|
||||
receipt_id: str | None = None,
|
||||
nullifier_prefix_len: int = 8,
|
||||
) -> Receipt:
|
||||
"""Operator-side: issue a Phase-1 verification receipt.
|
||||
|
||||
``nullifier_prefix`` is the first ``nullifier_prefix_len`` hex
|
||||
chars of the full nullifier — enough for the operator to dispute
|
||||
later (fraud auditing) but NOT enough to identify the citizen
|
||||
cross-operator. 8 hex chars = 32 bits = ~4 billion possible
|
||||
prefixes, statistically unlinkable across operators.
|
||||
"""
|
||||
if not isinstance(nullifier, str) or len(nullifier) < nullifier_prefix_len:
|
||||
raise ValueError("nullifier must be a hex string of sufficient length")
|
||||
rid = receipt_id or secrets.token_hex(16)
|
||||
prefix = nullifier[:nullifier_prefix_len]
|
||||
day = _day_bucket(timestamp)
|
||||
body = _receipt_body(
|
||||
role="verification", receipt_id=rid, operator_id=operator_id,
|
||||
day_bucket=day, nullifier_prefix=prefix,
|
||||
)
|
||||
sig = _sign(operator_secret, body)
|
||||
return Receipt(
|
||||
role="verification",
|
||||
receipt_id=rid,
|
||||
operator_id=operator_id,
|
||||
day_bucket=day,
|
||||
nullifier_prefix=prefix,
|
||||
signature=sig,
|
||||
)
|
||||
|
||||
|
||||
def counter_sign_fulfillment(
|
||||
*,
|
||||
verification: Receipt,
|
||||
citizen_secret: bytes,
|
||||
) -> Receipt:
|
||||
"""Citizen-side: counter-sign a verification receipt to acknowledge
|
||||
service rendered.
|
||||
|
||||
The fulfillment receipt has the same field values as the
|
||||
verification receipt (linking them to the same redemption) but
|
||||
is signed with the CITIZEN's secret instead of the operator's.
|
||||
Together they form a ``ReceiptPair``.
|
||||
"""
|
||||
if verification.role != "verification":
|
||||
raise ValueError("input must be a Phase-1 verification receipt")
|
||||
body = _receipt_body(
|
||||
role="fulfillment", receipt_id=verification.receipt_id,
|
||||
operator_id=verification.operator_id, day_bucket=verification.day_bucket,
|
||||
nullifier_prefix=verification.nullifier_prefix,
|
||||
)
|
||||
sig = _sign(citizen_secret, body)
|
||||
return Receipt(
|
||||
role="fulfillment",
|
||||
receipt_id=verification.receipt_id,
|
||||
operator_id=verification.operator_id,
|
||||
day_bucket=verification.day_bucket,
|
||||
nullifier_prefix=verification.nullifier_prefix,
|
||||
signature=sig,
|
||||
)
|
||||
|
||||
|
||||
def verify_receipt_pair(
|
||||
*,
|
||||
pair: ReceiptPair,
|
||||
operator_secret: bytes,
|
||||
citizen_secret: bytes,
|
||||
) -> bool:
|
||||
"""Verify both signatures on a ``ReceiptPair``.
|
||||
|
||||
Useful in dispute resolution — both parties can independently
|
||||
confirm the pair is genuine.
|
||||
"""
|
||||
if pair.verification.role != "verification":
|
||||
return False
|
||||
if pair.fulfillment.role != "fulfillment":
|
||||
return False
|
||||
if pair.verification.receipt_id != pair.fulfillment.receipt_id:
|
||||
return False
|
||||
if pair.verification.operator_id != pair.fulfillment.operator_id:
|
||||
return False
|
||||
v_body = _receipt_body(
|
||||
role="verification", receipt_id=pair.verification.receipt_id,
|
||||
operator_id=pair.verification.operator_id,
|
||||
day_bucket=pair.verification.day_bucket,
|
||||
nullifier_prefix=pair.verification.nullifier_prefix,
|
||||
)
|
||||
if not _verify(operator_secret, v_body, pair.verification.signature):
|
||||
return False
|
||||
f_body = _receipt_body(
|
||||
role="fulfillment", receipt_id=pair.fulfillment.receipt_id,
|
||||
operator_id=pair.fulfillment.operator_id,
|
||||
day_bucket=pair.fulfillment.day_bucket,
|
||||
nullifier_prefix=pair.fulfillment.nullifier_prefix,
|
||||
)
|
||||
if not _verify(citizen_secret, f_body, pair.fulfillment.signature):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DenialCode",
|
||||
"Receipt",
|
||||
"ReceiptPair",
|
||||
"counter_sign_fulfillment",
|
||||
"issue_verification_receipt",
|
||||
"verify_receipt_pair",
|
||||
]
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Ring Confidential Transactions — Sprint 11+ scaffolding.
|
||||
|
||||
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.3,
|
||||
``infonet-economy/BRAINDUMP.md`` §11 item 9.
|
||||
|
||||
RingCT combines:
|
||||
|
||||
1. **Ring signatures** — hide *who* signed within an anonymity set.
|
||||
2. **Confidential transactions** — Pedersen commitments hide
|
||||
*amounts*. A range proof confirms the amount is non-negative.
|
||||
3. **Key images** — link two outputs spent by the same key (without
|
||||
revealing which key). Prevents double-spend without breaking
|
||||
anonymity.
|
||||
|
||||
Implementation scheme is **undecided** — IMPLEMENTATION_PLAN.md §6.4
|
||||
calls out RingCT vs CONFIDENTIAL_TX vs MimbleWimble vs ZK-SNARK as
|
||||
options. The scaffolding here is scheme-agnostic; production wires
|
||||
in whichever scheme the architect chooses through the
|
||||
``RingSignatureScheme`` and ``BalanceCommitment`` Protocols.
|
||||
|
||||
Sprint 11+ runway:
|
||||
|
||||
- The interface contract is locked (see ``contracts.py``).
|
||||
- A ``RingCTScaffolding`` placeholder reports
|
||||
``status=NOT_IMPLEMENTED`` so callers can introspect honestly.
|
||||
- When the Rust binding lands, instantiate it via the same Protocol
|
||||
shape and swap the scaffolding for the production class — no
|
||||
caller changes needed.
|
||||
|
||||
Cross-cutting design rule: privacy primitives MUST report their
|
||||
status truthfully (cross-cutting design rule #1 — non-hostile UX).
|
||||
A primitive that's not implemented surfaces clearly via the status
|
||||
endpoint; calling its operations raises ``NotImplementedError`` with
|
||||
a pointer back to the open issue.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from services.infonet.privacy.contracts import PrivacyPrimitiveStatus
|
||||
|
||||
|
||||
class RingCTScaffolding:
|
||||
"""Placeholder until the Rust ring-signature binding lands.
|
||||
|
||||
Calling ``sign`` / ``verify`` raises with a diagnostic that
|
||||
points the caller back to the design doc. The status method
|
||||
truthfully reports ``NOT_IMPLEMENTED`` so health endpoints can
|
||||
surface this state.
|
||||
"""
|
||||
|
||||
_DIAGNOSTIC = (
|
||||
"RingCT primitive is scaffolding only — see "
|
||||
"infonet-economy/IMPLEMENTATION_PLAN.md §6.4 for the open "
|
||||
"scheme decision (RingCT vs CONFIDENTIAL_TX vs MimbleWimble "
|
||||
"vs ZK-SNARK). Production implementation lands via "
|
||||
"privacy-core Rust crate when ready."
|
||||
)
|
||||
|
||||
def sign(
|
||||
self,
|
||||
*,
|
||||
message: bytes,
|
||||
signer_private_key: bytes,
|
||||
ring_public_keys: list[bytes],
|
||||
) -> dict[str, Any]:
|
||||
raise NotImplementedError(self._DIAGNOSTIC)
|
||||
|
||||
def verify(
|
||||
self,
|
||||
*,
|
||||
message: bytes,
|
||||
signature: dict[str, Any],
|
||||
ring_public_keys: list[bytes],
|
||||
) -> bool:
|
||||
raise NotImplementedError(self._DIAGNOSTIC)
|
||||
|
||||
def status(self) -> PrivacyPrimitiveStatus:
|
||||
return PrivacyPrimitiveStatus.NOT_IMPLEMENTED
|
||||
|
||||
|
||||
__all__ = ["RingCTScaffolding"]
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Shielded balance commitments — Sprint 11+ scaffolding.
|
||||
|
||||
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.3.
|
||||
|
||||
Pedersen commitments hide balance amounts while preserving
|
||||
homomorphic add/subtract. Range proofs (Bulletproofs or similar)
|
||||
prove each output is non-negative without revealing it.
|
||||
|
||||
A balance is committed as ``C = amount * G + blinding * H`` where
|
||||
``G, H`` are independent generators. ``sum(inputs) - sum(outputs)
|
||||
== 0`` proves "no value created or destroyed" without revealing any
|
||||
of the values.
|
||||
|
||||
Production implementation lands through the ``BalanceCommitment``
|
||||
Protocol when the Rust binding is ready.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from services.infonet.privacy.contracts import PrivacyPrimitiveStatus
|
||||
|
||||
|
||||
class ShieldedBalanceScaffolding:
|
||||
"""Placeholder until the Rust balance-commitment binding lands."""
|
||||
|
||||
_DIAGNOSTIC = (
|
||||
"Shielded balance primitive is scaffolding only — production "
|
||||
"implementation requires a Pedersen commitment + range-proof "
|
||||
"library (e.g. bulletproofs). See "
|
||||
"infonet-economy/IMPLEMENTATION_PLAN.md §4.3."
|
||||
)
|
||||
|
||||
def commit(self, *, amount: int, blinding: bytes) -> bytes:
|
||||
raise NotImplementedError(self._DIAGNOSTIC)
|
||||
|
||||
def verify_balance(
|
||||
self,
|
||||
*,
|
||||
input_commitments: list[bytes],
|
||||
output_commitments: list[bytes],
|
||||
) -> bool:
|
||||
raise NotImplementedError(self._DIAGNOSTIC)
|
||||
|
||||
def range_proof(
|
||||
self,
|
||||
*,
|
||||
amount: int,
|
||||
blinding: bytes,
|
||||
max_bits: int = 64,
|
||||
) -> bytes:
|
||||
raise NotImplementedError(self._DIAGNOSTIC)
|
||||
|
||||
def verify_range_proof(
|
||||
self,
|
||||
*,
|
||||
commitment: bytes,
|
||||
proof: bytes,
|
||||
max_bits: int = 64,
|
||||
) -> bool:
|
||||
raise NotImplementedError(self._DIAGNOSTIC)
|
||||
|
||||
def status(self) -> PrivacyPrimitiveStatus:
|
||||
return PrivacyPrimitiveStatus.NOT_IMPLEMENTED
|
||||
|
||||
|
||||
__all__ = ["ShieldedBalanceScaffolding"]
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Stealth addresses — Sprint 11+ scaffolding.
|
||||
|
||||
Source of truth: ``infonet-economy/IMPLEMENTATION_PLAN.md`` §4.3.
|
||||
|
||||
Each transaction generates a fresh one-time recipient address
|
||||
unlinkable from the recipient's published key. The recipient uses a
|
||||
private *view key* to scan the chain and identify outputs intended
|
||||
for them.
|
||||
|
||||
Standard scheme (Monero-style, dual-key):
|
||||
|
||||
- Recipient publishes ``(view_pub, spend_pub)``.
|
||||
- Sender generates random ``r``, computes
|
||||
``one_time_address = H(r * view_pub) * G + spend_pub``.
|
||||
- Recipient scans chain by checking if
|
||||
``H(view_priv * R) * G + spend_pub == one_time_address`` for each
|
||||
output's ``R = r * G``.
|
||||
|
||||
Production implementation lands through the ``StealthAddressScheme``
|
||||
Protocol when the Rust binding is ready. Today, this module ships a
|
||||
``StealthAddressScaffolding`` placeholder that reports
|
||||
``NOT_IMPLEMENTED``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from services.infonet.privacy.contracts import PrivacyPrimitiveStatus
|
||||
|
||||
|
||||
class StealthAddressScaffolding:
|
||||
"""Placeholder until the Rust stealth-address binding lands."""
|
||||
|
||||
_DIAGNOSTIC = (
|
||||
"Stealth address primitive is scaffolding only — see "
|
||||
"infonet-economy/IMPLEMENTATION_PLAN.md §4.3 for the design. "
|
||||
"Production implementation lands via privacy-core Rust crate."
|
||||
)
|
||||
|
||||
def derive_one_time_address(
|
||||
self,
|
||||
*,
|
||||
recipient_view_key: bytes,
|
||||
recipient_spend_key: bytes,
|
||||
sender_random: bytes,
|
||||
) -> bytes:
|
||||
raise NotImplementedError(self._DIAGNOSTIC)
|
||||
|
||||
def is_for_recipient(
|
||||
self,
|
||||
*,
|
||||
one_time_address: bytes,
|
||||
recipient_view_key: bytes,
|
||||
recipient_spend_key: bytes,
|
||||
sender_random: bytes,
|
||||
) -> bool:
|
||||
raise NotImplementedError(self._DIAGNOSTIC)
|
||||
|
||||
def status(self) -> PrivacyPrimitiveStatus:
|
||||
return PrivacyPrimitiveStatus.NOT_IMPLEMENTED
|
||||
|
||||
|
||||
__all__ = ["StealthAddressScaffolding"]
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Reputation views — oracle_rep, oracle_rep_active, oracle_rep_lifetime, common_rep.
|
||||
|
||||
These are **pure functions** over the chain. No stored state. See
|
||||
``infonet-economy/IMPLEMENTATION_PLAN.md`` §3.2 for the rationale.
|
||||
|
||||
Sprint 2 ships the base formulas (RULES §3.1, §3.2, §3.3, §3.11) without
|
||||
the anti-gaming penalties. Sprint 3 layers VCS / clustering / temporal /
|
||||
progressive penalties on top.
|
||||
"""
|
||||
|
||||
from services.infonet.reputation.anti_gaming import (
|
||||
apply_progressive_penalty,
|
||||
clustering_penalty,
|
||||
compute_clustering_coefficient,
|
||||
compute_farming_pct,
|
||||
compute_rep_multiplier,
|
||||
compute_vcs,
|
||||
farming_multiplier,
|
||||
is_in_burst,
|
||||
temporal_multiplier,
|
||||
)
|
||||
from services.infonet.reputation.common_rep import compute_common_rep
|
||||
from services.infonet.reputation.governance_decay import (
|
||||
compute_oracle_rep_active,
|
||||
decay_factor_for_age,
|
||||
)
|
||||
from services.infonet.reputation.oracle_rep import (
|
||||
OracleRepBreakdown,
|
||||
compute_oracle_rep,
|
||||
compute_oracle_rep_lifetime,
|
||||
last_successful_prediction_ts,
|
||||
)
|
||||
from services.infonet.reputation.weekly_vote_budget import (
|
||||
compute_weekly_vote_budget,
|
||||
count_upreps_in_last_week,
|
||||
is_budget_exceeded,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"OracleRepBreakdown",
|
||||
"apply_progressive_penalty",
|
||||
"clustering_penalty",
|
||||
"compute_clustering_coefficient",
|
||||
"compute_common_rep",
|
||||
"compute_farming_pct",
|
||||
"compute_oracle_rep",
|
||||
"compute_oracle_rep_active",
|
||||
"compute_oracle_rep_lifetime",
|
||||
"compute_rep_multiplier",
|
||||
"compute_vcs",
|
||||
"compute_weekly_vote_budget",
|
||||
"count_upreps_in_last_week",
|
||||
"decay_factor_for_age",
|
||||
"farming_multiplier",
|
||||
"is_budget_exceeded",
|
||||
"is_in_burst",
|
||||
"last_successful_prediction_ts",
|
||||
"temporal_multiplier",
|
||||
]
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Anti-gaming penalties — Sprint 3.
|
||||
|
||||
Five layers:
|
||||
|
||||
- ``vcs.py``: Vote Correlation Score — detects coordinated upreping rings.
|
||||
- ``clustering.py``: clustering coefficient — detects sophisticated farming
|
||||
where voters also uprep each other.
|
||||
- ``temporal.py``: burst detection — flags suspicious uprep storms.
|
||||
- ``farming.py``: easy-bet detection — penalizes "predictors" who only
|
||||
bet on near-certain outcomes.
|
||||
- ``progressive_penalty.py``: whale deterrence — gaming penalties scale
|
||||
with the violator's oracle rep so high-rep nodes can't shrug them off.
|
||||
|
||||
All five are pure functions over the chain. They run as deterministic
|
||||
chain analysis (every node computes the same scores from the same chain
|
||||
history), matching IMPLEMENTATION_PLAN.md §3.3.
|
||||
|
||||
Cross-cutting design rule: anti-gaming reads happen in the background.
|
||||
A user who is being legitimately upreped does not block the UI on
|
||||
penalty recomputation; the computed common-rep view simply uses the
|
||||
last cached value and refreshes asynchronously.
|
||||
"""
|
||||
|
||||
from services.infonet.reputation.anti_gaming.clustering import (
|
||||
clustering_penalty,
|
||||
compute_clustering_coefficient,
|
||||
)
|
||||
from services.infonet.reputation.anti_gaming.farming import (
|
||||
compute_farming_pct,
|
||||
farming_multiplier,
|
||||
)
|
||||
from services.infonet.reputation.anti_gaming.progressive_penalty import (
|
||||
apply_progressive_penalty,
|
||||
compute_rep_multiplier,
|
||||
)
|
||||
from services.infonet.reputation.anti_gaming.temporal import (
|
||||
is_in_burst,
|
||||
temporal_multiplier,
|
||||
)
|
||||
from services.infonet.reputation.anti_gaming.vcs import compute_vcs
|
||||
|
||||
__all__ = [
|
||||
"apply_progressive_penalty",
|
||||
"clustering_penalty",
|
||||
"compute_clustering_coefficient",
|
||||
"compute_farming_pct",
|
||||
"compute_rep_multiplier",
|
||||
"compute_vcs",
|
||||
"farming_multiplier",
|
||||
"is_in_burst",
|
||||
"temporal_multiplier",
|
||||
]
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Clustering coefficient — detects sophisticated farming where the
|
||||
voters who uprep a target also uprep each other.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.5.
|
||||
|
||||
For a target B:
|
||||
|
||||
voters = {all nodes that uprepped B in decay window}
|
||||
n = len(voters)
|
||||
|
||||
if n < 2:
|
||||
return 0.0
|
||||
|
||||
possible_edges = n * (n - 1) / 2
|
||||
actual_edges = count of pairs (V1, V2) where V1 has uprepped V2
|
||||
OR V2 has uprepped V1
|
||||
clustering = actual_edges / possible_edges
|
||||
|
||||
The penalty per RULES §3.3:
|
||||
|
||||
target_penalty = max(clustering_min_weight, 1.0 - clustering)
|
||||
|
||||
Why this catches what VCS misses: VCS measures *one* upreper's
|
||||
similarity to the target's fan set. Clustering measures whether the
|
||||
*entire* fan set is socially networked — a 10-node cabal that
|
||||
upreps each other is a cluster coefficient near 1.0 even if no
|
||||
individual upreper has unusual VCS.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
from services.infonet.reputation.anti_gaming.vcs import _upreps_within_window # noqa: I201
|
||||
|
||||
|
||||
_SECONDS_PER_DAY = 86400.0
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
def _decay_window_seconds(decay_window_days: float | None) -> float:
|
||||
if decay_window_days is not None:
|
||||
return float(decay_window_days) * _SECONDS_PER_DAY
|
||||
return float(CONFIG["vote_decay_days"]) * _SECONDS_PER_DAY
|
||||
|
||||
|
||||
def compute_clustering_coefficient(
|
||||
target_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float | None = None,
|
||||
decay_window_days: float | None = None,
|
||||
) -> float:
|
||||
"""Coefficient in ``[0.0, 1.0]`` for ``target_id``'s voter graph.
|
||||
|
||||
0.0 means voters are strangers; 1.0 means every voter has uprepped
|
||||
every other voter.
|
||||
"""
|
||||
if not isinstance(target_id, str) or not target_id:
|
||||
return 0.0
|
||||
events = [e for e in chain if isinstance(e, dict)]
|
||||
if not events:
|
||||
return 0.0
|
||||
|
||||
if now is None:
|
||||
now = max(float(ev.get("timestamp") or 0.0) for ev in events)
|
||||
window_s = _decay_window_seconds(decay_window_days)
|
||||
window_upreps = _upreps_within_window(events, now=now, window_s=window_s)
|
||||
|
||||
voters: set[str] = set()
|
||||
edges: set[tuple[str, str]] = set()
|
||||
# Build an adjacency map from author -> {targets}.
|
||||
by_author: dict[str, set[str]] = {}
|
||||
for ev in window_upreps:
|
||||
author = ev.get("node_id")
|
||||
p = _payload(ev)
|
||||
tgt = p.get("target_node_id")
|
||||
if not isinstance(author, str) or not isinstance(tgt, str):
|
||||
continue
|
||||
if author == tgt:
|
||||
continue
|
||||
by_author.setdefault(author, set()).add(tgt)
|
||||
if tgt == target_id:
|
||||
voters.add(author)
|
||||
|
||||
n = len(voters)
|
||||
if n < 2:
|
||||
return 0.0
|
||||
|
||||
voter_list = sorted(voters)
|
||||
for i, v1 in enumerate(voter_list):
|
||||
for v2 in voter_list[i + 1:]:
|
||||
v1_upreps_v2 = v2 in by_author.get(v1, ())
|
||||
v2_upreps_v1 = v1 in by_author.get(v2, ())
|
||||
if v1_upreps_v2 or v2_upreps_v1:
|
||||
edges.add((v1, v2))
|
||||
|
||||
possible = n * (n - 1) / 2
|
||||
return len(edges) / possible
|
||||
|
||||
|
||||
def clustering_penalty(coefficient: float) -> float:
|
||||
"""Per-uprep multiplier from a clustering coefficient.
|
||||
|
||||
Spec formula: ``max(clustering_min_weight, 1.0 - coefficient)``.
|
||||
"""
|
||||
floor = float(CONFIG["clustering_min_weight"])
|
||||
return max(floor, 1.0 - float(coefficient))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"clustering_penalty",
|
||||
"compute_clustering_coefficient",
|
||||
]
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Aggregate per-node correlation score — feeds progressive penalty.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.6.
|
||||
|
||||
Sprint 3 shipped the progressive-penalty math (whale deterrence
|
||||
multiplier ``1 + log2(rep)``) but did not wire it into a running
|
||||
aggregate. This module fills that gap.
|
||||
|
||||
Per-node aggregate score:
|
||||
|
||||
score(node) = mean(1 - vcs(upreper, node)
|
||||
for every uprep targeting node in the decay window)
|
||||
|
||||
Range: ``[0.0, 1.0]``. ``0.0`` means every uprep was orthogonal —
|
||||
no correlation evidence. ``1.0`` means every uprep was from a fully
|
||||
overlapping target set — saturated cabal.
|
||||
|
||||
When ``score(node) > CONFIG['progressive_penalty_threshold']`` (default
|
||||
``0.0`` — disabled), the progressive penalty multiplier is applied to
|
||||
the node's effective common-rep payouts. The threshold default is
|
||||
``0.0`` so Sprint 3 behavior is preserved for any chain that doesn't
|
||||
explicitly opt in via governance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
from services.infonet.reputation.anti_gaming.progressive_penalty import (
|
||||
apply_progressive_penalty,
|
||||
)
|
||||
from services.infonet.reputation.anti_gaming.vcs import compute_vcs
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
def compute_node_correlation_score(
|
||||
node_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float | None = None,
|
||||
) -> float:
|
||||
"""Average correlation evidence (``1 - VCS``) across upreps
|
||||
targeting ``node_id``.
|
||||
|
||||
Returns ``0.0`` when no upreps target the node — no evidence to
|
||||
support a penalty.
|
||||
"""
|
||||
chain_list = [e for e in chain if isinstance(e, dict)]
|
||||
if not chain_list:
|
||||
return 0.0
|
||||
if now is None:
|
||||
now = max(float(ev.get("timestamp") or 0.0) for ev in chain_list)
|
||||
|
||||
correlations: list[float] = []
|
||||
for ev in chain_list:
|
||||
if ev.get("event_type") != "uprep":
|
||||
continue
|
||||
if _payload(ev).get("target_node_id") != node_id:
|
||||
continue
|
||||
upreper = ev.get("node_id")
|
||||
if not isinstance(upreper, str) or not upreper or upreper == node_id:
|
||||
continue
|
||||
try:
|
||||
ts = float(ev.get("timestamp") or 0.0)
|
||||
except (TypeError, ValueError):
|
||||
ts = float(now)
|
||||
vcs = compute_vcs(upreper, node_id, chain_list, now=ts)
|
||||
correlations.append(max(0.0, min(1.0, 1.0 - vcs)))
|
||||
if not correlations:
|
||||
return 0.0
|
||||
return sum(correlations) / len(correlations)
|
||||
|
||||
|
||||
def progressive_penalty_multiplier_for(
|
||||
node_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
oracle_rep: float,
|
||||
now: float | None = None,
|
||||
) -> float:
|
||||
"""Return the multiplier to apply to a node's common-rep payouts.
|
||||
|
||||
Returns ``1.0`` when the aggregate correlation score is at or
|
||||
below ``CONFIG['progressive_penalty_threshold']`` (no penalty).
|
||||
Above the threshold, the penalty is computed via
|
||||
``apply_progressive_penalty(score - threshold, oracle_rep)`` and
|
||||
*subtracted* from 1.0 (clamped to ``[0.0, 1.0]``).
|
||||
|
||||
The threshold defaults to ``0.0`` (disabled). Governance can
|
||||
raise it via petition once aggregate-correlation history is
|
||||
well-calibrated against real chain data.
|
||||
"""
|
||||
threshold = float(CONFIG["progressive_penalty_threshold"])
|
||||
if threshold <= 0.0:
|
||||
# Disabled — preserve Sprint 3 behavior.
|
||||
return 1.0
|
||||
score = compute_node_correlation_score(node_id, chain, now=now)
|
||||
if score <= threshold:
|
||||
return 1.0
|
||||
over = score - threshold
|
||||
docked = apply_progressive_penalty(over, oracle_rep)
|
||||
return max(0.0, 1.0 - docked)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"compute_node_correlation_score",
|
||||
"progressive_penalty_multiplier_for",
|
||||
]
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Easy-bet farming detection and enforcement.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.1.
|
||||
|
||||
A node's ``farming_pct`` is:
|
||||
|
||||
farming_pct = count_easy_bets / total_predictions
|
||||
|
||||
where an "easy bet" is a prediction whose ``probability_at_bet``
|
||||
exceeds ``farming_easy_bet_cutoff`` (default 0.80, expressed as 80.0
|
||||
on the 0-100 scale used in payloads).
|
||||
|
||||
Penalty multiplier:
|
||||
|
||||
farming_pct > farming_hard_threshold → oracle_rep_earned *= 0.10
|
||||
farming_pct > farming_soft_threshold → oracle_rep_earned *= 0.50
|
||||
otherwise → oracle_rep_earned *= 1.00
|
||||
|
||||
The plan §1.2 calls out that the existing ``mesh_oracle.py`` *tracks*
|
||||
``farming_pct`` but does NOT enforce the multiplier. Sprint 3 adds the
|
||||
enforcement here, and Sprint 4's `oracle_rep` integration applies it
|
||||
to mints. Until that wiring lands, this module is exposed as the
|
||||
authoritative source of the math.
|
||||
|
||||
Note on sides: the spec's "easy bet" is a probability-of-the-PICKED-side
|
||||
test. A free pick at 90% on yes is easy; a contrarian free pick at 10%
|
||||
on yes (where the chain says yes is 10% likely) is hard. The
|
||||
``probability_at_bet`` field stores the probability of the YES side at
|
||||
the time the prediction is placed; we compute the predicted-side
|
||||
probability accordingly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
def _picked_side_probability(payload: dict[str, Any]) -> float | None:
|
||||
"""Translate ``probability_at_bet`` (always P(yes) on 0-100) into the
|
||||
probability of the side actually picked. Returns ``None`` if the
|
||||
payload is malformed.
|
||||
"""
|
||||
side = payload.get("side")
|
||||
prob = payload.get("probability_at_bet")
|
||||
if side not in ("yes", "no"):
|
||||
return None
|
||||
if prob is None:
|
||||
return None
|
||||
try:
|
||||
p_yes = float(prob)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if not (0.0 <= p_yes <= 100.0):
|
||||
return None
|
||||
return p_yes if side == "yes" else 100.0 - p_yes
|
||||
|
||||
|
||||
def compute_farming_pct(
|
||||
node_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> float:
|
||||
"""Fraction of ``node_id``'s predictions whose picked-side probability
|
||||
exceeded ``farming_easy_bet_cutoff``.
|
||||
|
||||
Returns ``0.0`` if the node has no predictions on chain. The cutoff
|
||||
is on the 0-1 scale in ``CONFIG``; predictions store probability on
|
||||
the 0-100 scale, so we scale the cutoff for comparison.
|
||||
"""
|
||||
if not isinstance(node_id, str) or not node_id:
|
||||
return 0.0
|
||||
cutoff_pct = float(CONFIG["farming_easy_bet_cutoff"]) * 100.0
|
||||
|
||||
total = 0
|
||||
easy = 0
|
||||
for ev in chain:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
if ev.get("event_type") != "prediction_place":
|
||||
continue
|
||||
if ev.get("node_id") != node_id:
|
||||
continue
|
||||
picked_p = _picked_side_probability(_payload(ev))
|
||||
if picked_p is None:
|
||||
continue
|
||||
total += 1
|
||||
if picked_p > cutoff_pct:
|
||||
easy += 1
|
||||
if total == 0:
|
||||
return 0.0
|
||||
return easy / total
|
||||
|
||||
|
||||
def farming_multiplier(farming_pct: float) -> float:
|
||||
"""Spec multiplier for a node's mint earnings.
|
||||
|
||||
- ``> farming_hard_threshold`` → 0.10
|
||||
- ``> farming_soft_threshold`` → 0.50
|
||||
- otherwise → 1.00
|
||||
"""
|
||||
pct = float(farming_pct)
|
||||
if pct > float(CONFIG["farming_hard_threshold"]):
|
||||
return 0.10
|
||||
if pct > float(CONFIG["farming_soft_threshold"]):
|
||||
return 0.50
|
||||
return 1.00
|
||||
|
||||
|
||||
__all__ = [
|
||||
"compute_farming_pct",
|
||||
"farming_multiplier",
|
||||
]
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Progressive penalty — whale deterrence.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.6.
|
||||
|
||||
base_penalty = correlation_score # 0.0 to 1.0
|
||||
rep_multiplier = 1 + log2(max(oracle_rep, 1))
|
||||
rep_docked = base_penalty * rep_multiplier
|
||||
|
||||
The point: a whale with 1024 oracle rep faces a multiplier of 11×, so
|
||||
the same correlation score that would dock a small node 0.5 rep docks
|
||||
a whale 5.5 rep. Coordination becomes more expensive as you accumulate
|
||||
more rep — the protocol's "you can't simply outscale anti-gaming"
|
||||
defense.
|
||||
|
||||
This module exposes the math. Sprint 3 does NOT yet wire it into a
|
||||
running aggregate-correlation tracker — that requires per-node
|
||||
correlation history which is a Sprint 4+ concern. The helpers here are
|
||||
ready for that integration.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
|
||||
def compute_rep_multiplier(oracle_rep: float) -> float:
|
||||
"""``1 + log2(max(oracle_rep, 1))``.
|
||||
|
||||
- ``oracle_rep <= 1`` → multiplier 1.0
|
||||
- ``oracle_rep == 2`` → 2.0
|
||||
- ``oracle_rep == 1024`` → 11.0
|
||||
"""
|
||||
rep = max(1.0, float(oracle_rep))
|
||||
return 1.0 + math.log2(rep)
|
||||
|
||||
|
||||
def apply_progressive_penalty(base_penalty: float, oracle_rep: float) -> float:
|
||||
"""``base_penalty * compute_rep_multiplier(oracle_rep)``.
|
||||
|
||||
``base_penalty`` is intended to be a non-negative correlation score
|
||||
in ``[0.0, 1.0]``; the function does not clamp.
|
||||
"""
|
||||
return float(base_penalty) * compute_rep_multiplier(oracle_rep)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"apply_progressive_penalty",
|
||||
"compute_rep_multiplier",
|
||||
]
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Temporal burst detection — flags suspicious uprep storms.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.3 (the
|
||||
``rep_after_burst`` step).
|
||||
|
||||
Definition: a target B is "in a burst" relative to an uprep at time
|
||||
``t`` if there are at least ``temporal_burst_min_upreps`` upreps to B
|
||||
within a ``temporal_burst_window_sec`` window centered on ``t`` (the
|
||||
window includes the uprep being evaluated).
|
||||
|
||||
When in burst: per-uprep weight is multiplied by 0.2 (80% reduction).
|
||||
Otherwise 1.0.
|
||||
|
||||
Why a centered window: bursts can be detected on either side of the
|
||||
suspect uprep. Sliding-forward-only would let an attacker pre-warm the
|
||||
counter.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
|
||||
|
||||
_BURST_REDUCTION_FACTOR = 0.2
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
def is_in_burst(
|
||||
target_id: str,
|
||||
uprep_timestamp: float,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> bool:
|
||||
"""Are there ``temporal_burst_min_upreps`` upreps to ``target_id``
|
||||
within ``temporal_burst_window_sec`` of ``uprep_timestamp``?
|
||||
"""
|
||||
if not isinstance(target_id, str) or not target_id:
|
||||
return False
|
||||
try:
|
||||
ts = float(uprep_timestamp)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
window_s = float(CONFIG["temporal_burst_window_sec"])
|
||||
half = window_s / 2.0
|
||||
threshold = int(CONFIG["temporal_burst_min_upreps"])
|
||||
|
||||
count = 0
|
||||
for ev in chain:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
if ev.get("event_type") != "uprep":
|
||||
continue
|
||||
p = _payload(ev)
|
||||
if p.get("target_node_id") != target_id:
|
||||
continue
|
||||
try:
|
||||
ets = float(ev.get("timestamp") or 0.0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if ts - half <= ets <= ts + half:
|
||||
count += 1
|
||||
if count >= threshold:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def temporal_multiplier(in_burst: bool) -> float:
|
||||
"""1.0 if not in burst; ``_BURST_REDUCTION_FACTOR`` (0.2) if in burst."""
|
||||
return _BURST_REDUCTION_FACTOR if in_burst else 1.0
|
||||
|
||||
|
||||
__all__ = [
|
||||
"is_in_burst",
|
||||
"temporal_multiplier",
|
||||
]
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Vote Correlation Score — detects coordinated upreping rings.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.4.
|
||||
|
||||
For an uprep from A → B:
|
||||
|
||||
A_targets = {all nodes A has uprepped in decay window}
|
||||
B_fans = {all nodes that uprepped B in decay window, excluding A}
|
||||
|
||||
if len(B_fans) == 0:
|
||||
overlap = 0.0
|
||||
else:
|
||||
overlap = |A_targets ∩ B_fans| / |B_fans|
|
||||
|
||||
correlation_penalty = max(vcs_min_weight, 1.0 - overlap)
|
||||
|
||||
The intent: if A always upreps the same group of nodes that always uprep
|
||||
B (a circle-jerk), the overlap approaches 1 and the penalty floors at
|
||||
``vcs_min_weight`` (default 0.10 — 10% effective weight, regardless of
|
||||
how many nodes participate).
|
||||
|
||||
Pure function over the chain. Does NOT depend on order beyond the decay
|
||||
window cutoff.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
|
||||
|
||||
_SECONDS_PER_DAY = 86400.0
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
def _decay_window_seconds(decay_window_days: float | None) -> float:
|
||||
if decay_window_days is not None:
|
||||
return float(decay_window_days) * _SECONDS_PER_DAY
|
||||
return float(CONFIG["vote_decay_days"]) * _SECONDS_PER_DAY
|
||||
|
||||
|
||||
def _upreps_within_window(
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float,
|
||||
window_s: float,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""All ``uprep`` events whose timestamp is in [now - window, now]."""
|
||||
cutoff = now - window_s
|
||||
out: list[dict[str, Any]] = []
|
||||
for ev in chain:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
if ev.get("event_type") != "uprep":
|
||||
continue
|
||||
try:
|
||||
ts = float(ev.get("timestamp") or 0.0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if cutoff <= ts <= now:
|
||||
out.append(ev)
|
||||
return out
|
||||
|
||||
|
||||
def compute_vcs(
|
||||
upreper_id: str,
|
||||
target_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float | None = None,
|
||||
decay_window_days: float | None = None,
|
||||
) -> float:
|
||||
"""Return the VCS multiplier for an uprep from ``upreper_id`` to ``target_id``.
|
||||
|
||||
Range: ``[vcs_min_weight, 1.0]``. A return of 1.0 means no
|
||||
correlation detected (full weight). A return of ``vcs_min_weight``
|
||||
(default 0.10) means maximum correlation — the upreper's targets
|
||||
completely overlap with the target's fan set.
|
||||
|
||||
``now`` defaults to the latest timestamp on the chain. Pass an
|
||||
explicit value when the caller wants a fixed evaluation point (e.g.
|
||||
Sprint 4 will pass the market snapshot's ``frozen_at``).
|
||||
"""
|
||||
if not isinstance(upreper_id, str) or not upreper_id:
|
||||
return float(CONFIG["vcs_min_weight"])
|
||||
if not isinstance(target_id, str) or not target_id:
|
||||
return float(CONFIG["vcs_min_weight"])
|
||||
if upreper_id == target_id:
|
||||
return 1.0 # self-uprep is filtered by common_rep; VCS is a no-op here
|
||||
|
||||
events = [e for e in chain if isinstance(e, dict)]
|
||||
if not events:
|
||||
return 1.0
|
||||
|
||||
if now is None:
|
||||
now = max(float(ev.get("timestamp") or 0.0) for ev in events)
|
||||
window_s = _decay_window_seconds(decay_window_days)
|
||||
window_upreps = _upreps_within_window(events, now=now, window_s=window_s)
|
||||
|
||||
a_targets: set[str] = set()
|
||||
b_fans: set[str] = set()
|
||||
for ev in window_upreps:
|
||||
author = ev.get("node_id")
|
||||
p = _payload(ev)
|
||||
target = p.get("target_node_id")
|
||||
if not isinstance(author, str) or not isinstance(target, str):
|
||||
continue
|
||||
if author == target:
|
||||
continue
|
||||
if author == upreper_id:
|
||||
a_targets.add(target)
|
||||
if target == target_id and author != upreper_id:
|
||||
b_fans.add(author)
|
||||
|
||||
floor = float(CONFIG["vcs_min_weight"])
|
||||
if not b_fans:
|
||||
return 1.0
|
||||
overlap = len(a_targets & b_fans) / len(b_fans)
|
||||
return max(floor, 1.0 - overlap)
|
||||
|
||||
|
||||
__all__ = ["compute_vcs"]
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Common rep computation with anti-gaming multipliers (Sprint 3).
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.3.
|
||||
|
||||
Per-uprep formula:
|
||||
|
||||
base_rep = upreper.oracle_rep * weight_factor
|
||||
rep_after_vcs = base_rep * compute_vcs(upreper, target)
|
||||
rep_after_clustering = rep_after_vcs * clustering_penalty(coefficient(target))
|
||||
rep_after_burst = rep_after_clustering * temporal_multiplier(in_burst)
|
||||
|
||||
common_rep_earned = rep_after_burst (per uprep; sum across all upreps)
|
||||
|
||||
VCS / clustering use the upreps-within-decay-window helper. Temporal
|
||||
burst uses a centered window (see ``anti_gaming/temporal.py``).
|
||||
|
||||
Sprint 3 caches per-uprep evaluations in-process: a single call to
|
||||
``compute_common_rep`` walks the chain at most three times (once per
|
||||
multiplier family). Caching across calls is a Sprint 3+ adapter
|
||||
concern.
|
||||
|
||||
Cross-cutting design rule: this is background work. The UI should call
|
||||
through ``InfonetReputationAdapter.common_rep`` and treat the result
|
||||
as eventually-consistent — never block a user-visible action waiting
|
||||
for it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
from services.infonet.reputation.anti_gaming.clustering import (
|
||||
clustering_penalty,
|
||||
compute_clustering_coefficient,
|
||||
)
|
||||
from services.infonet.reputation.anti_gaming.correlation_score import (
|
||||
progressive_penalty_multiplier_for,
|
||||
)
|
||||
from services.infonet.reputation.anti_gaming.temporal import (
|
||||
is_in_burst,
|
||||
temporal_multiplier,
|
||||
)
|
||||
from services.infonet.reputation.anti_gaming.vcs import compute_vcs
|
||||
from services.infonet.reputation.oracle_rep import compute_oracle_rep
|
||||
|
||||
|
||||
def _default_weight_factor() -> float:
|
||||
"""RULES §3.3 weight factor — promoted from Sprint 2 module
|
||||
constant to ``CONFIG['common_rep_weight_factor']`` 2026-04-28 so
|
||||
governance can tune it via petition.
|
||||
|
||||
Tests pass an explicit value to ``compute_common_rep`` to override.
|
||||
"""
|
||||
return float(CONFIG["common_rep_weight_factor"])
|
||||
|
||||
|
||||
def compute_common_rep(
|
||||
node_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
weight_factor: float | None = None,
|
||||
apply_anti_gaming: bool = True,
|
||||
) -> float:
|
||||
"""Common rep balance for ``node_id``.
|
||||
|
||||
``apply_anti_gaming=False`` returns the Sprint 2 base formula —
|
||||
useful for tests that want to isolate the multiplier layer. Default
|
||||
in production is ``True``.
|
||||
"""
|
||||
factor = float(_default_weight_factor() if weight_factor is None else weight_factor)
|
||||
events = [e for e in chain if isinstance(e, dict)]
|
||||
rep = 0.0
|
||||
# Oracle-rep cache keyed by upreper only — oracle_rep is computed
|
||||
# over the full chain (no time bound) and doesn't change per-uprep.
|
||||
upreper_cache: dict[str, float] = {}
|
||||
# NB: do NOT cache the clustering coefficient by node_id alone — it
|
||||
# is a function of (target, evaluation timestamp). Caching by target
|
||||
# only would freeze the first uprep's view (often coefficient 0
|
||||
# before other voters arrive) and skip the penalty for subsequent
|
||||
# upreps.
|
||||
|
||||
for ev in events:
|
||||
if ev.get("event_type") != "uprep":
|
||||
continue
|
||||
payload = ev.get("payload") or {}
|
||||
if payload.get("target_node_id") != node_id:
|
||||
continue
|
||||
upreper = ev.get("node_id")
|
||||
if not isinstance(upreper, str) or not upreper:
|
||||
continue
|
||||
if upreper == node_id:
|
||||
continue
|
||||
|
||||
if upreper not in upreper_cache:
|
||||
upreper_cache[upreper] = compute_oracle_rep(upreper, events)
|
||||
base = upreper_cache[upreper] * factor
|
||||
|
||||
if apply_anti_gaming:
|
||||
try:
|
||||
ts = float(ev.get("timestamp") or 0.0)
|
||||
except (TypeError, ValueError):
|
||||
ts = 0.0
|
||||
vcs = compute_vcs(upreper, node_id, events, now=ts)
|
||||
coefficient = compute_clustering_coefficient(node_id, events, now=ts)
|
||||
cluster_mult = clustering_penalty(coefficient)
|
||||
burst_mult = temporal_multiplier(is_in_burst(node_id, ts, events))
|
||||
rep += base * vcs * cluster_mult * burst_mult
|
||||
else:
|
||||
rep += base
|
||||
|
||||
if apply_anti_gaming and rep > 0:
|
||||
# Progressive-penalty wiring (Sprint 3 polish 2026-04-28).
|
||||
# Disabled when CONFIG['progressive_penalty_threshold'] == 0,
|
||||
# so this preserves Sprint 3 behavior by default. Once
|
||||
# governance raises the threshold via petition, the whale-
|
||||
# deterrence multiplier kicks in for nodes whose aggregate
|
||||
# correlation score crosses it. Oracle-rep input is the
|
||||
# TARGET's rep (not the upreper's) — bigger oracles bear
|
||||
# bigger penalties for cabal-shaped uprep patterns.
|
||||
target_oracle_rep = compute_oracle_rep(node_id, events)
|
||||
rep *= progressive_penalty_multiplier_for(
|
||||
node_id, events, oracle_rep=target_oracle_rep,
|
||||
)
|
||||
return rep
|
||||
|
||||
|
||||
__all__ = ["compute_common_rep"]
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Governance weight decay — oracle_rep → oracle_rep_active.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.11.
|
||||
|
||||
``oracle_rep_active`` is **only** used for governance weight (petition
|
||||
signatures, voting, quorum). Resolution staking, dispute staking, and
|
||||
truth staking continue to use ``oracle_rep`` directly — dormant oracles
|
||||
can still verify reality even if they aren't governing.
|
||||
|
||||
A successful prediction (``last_successful_prediction_ts`` is non-None)
|
||||
within the decay window keeps a node at full governance weight. Beyond
|
||||
the window, weight halves (default factor 0.5) per period.
|
||||
|
||||
Implemented as a pure function over the chain so every node computes
|
||||
the same value from the same chain history.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
from services.infonet.reputation.oracle_rep import (
|
||||
compute_oracle_rep,
|
||||
last_successful_prediction_ts,
|
||||
)
|
||||
|
||||
|
||||
_SECONDS_PER_DAY = 86400.0
|
||||
|
||||
|
||||
def decay_factor_for_age(days_since_success: float | None) -> float:
|
||||
"""Return the multiplier for ``oracle_rep`` → ``oracle_rep_active``.
|
||||
|
||||
- ``None``: node has no qualifying success → factor of 0 (no
|
||||
governance weight; new nodes earn it by predicting correctly in a
|
||||
mintable market).
|
||||
- within the decay window (``governance_decay_days``): 1.0.
|
||||
- beyond: ``governance_decay_factor ** decay_periods``.
|
||||
"""
|
||||
if days_since_success is None:
|
||||
return 0.0
|
||||
decay_days = float(CONFIG["governance_decay_days"])
|
||||
factor = float(CONFIG["governance_decay_factor"])
|
||||
if not (0.0 < factor < 1.0):
|
||||
# Guard: schema bounds should prevent this, but if a malformed
|
||||
# config slips through, treat as no-decay.
|
||||
return 1.0 if days_since_success <= decay_days else 0.0
|
||||
if days_since_success <= decay_days:
|
||||
return 1.0
|
||||
decay_periods = math.floor(days_since_success / decay_days)
|
||||
return factor ** decay_periods
|
||||
|
||||
|
||||
def compute_oracle_rep_active(
|
||||
node_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
now: float,
|
||||
) -> float:
|
||||
"""Governance-weighted oracle rep at chain time ``now``.
|
||||
|
||||
``now`` is passed in (rather than read from ``time.time()``) so the
|
||||
function stays pure and so tests / replay always produce
|
||||
deterministic answers. Production callers pass
|
||||
``time_validity.chain_majority_time(chain)``.
|
||||
"""
|
||||
events = list(chain)
|
||||
balance = compute_oracle_rep(node_id, events)
|
||||
if balance <= 0:
|
||||
return 0.0
|
||||
last_ts = last_successful_prediction_ts(node_id, events)
|
||||
if last_ts is None:
|
||||
return 0.0
|
||||
days = max(0.0, (float(now) - last_ts) / _SECONDS_PER_DAY)
|
||||
return balance * decay_factor_for_age(days)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"compute_oracle_rep_active",
|
||||
"decay_factor_for_age",
|
||||
]
|
||||
@@ -0,0 +1,361 @@
|
||||
"""Oracle rep computation — pure functions over the chain.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §3.1, §3.2, §3.11.
|
||||
|
||||
Constitutional anchor (``IMMUTABLE_PRINCIPLES["oracle_rep_source"] ==
|
||||
"predictions_only"``): oracle rep may ONLY be minted by verified
|
||||
predictions against reality. Sprint 2 enforces this by structurally
|
||||
constraining the mint formula — there is no other code path that
|
||||
returns a positive contribution to ``compute_oracle_rep``.
|
||||
|
||||
A market's prediction mints oracle rep only when ALL of the following
|
||||
hold:
|
||||
|
||||
- The market produced a ``resolution_finalize`` event with
|
||||
``outcome != "invalid"`` and ``is_provisional == False``.
|
||||
- The corresponding ``market_snapshot`` shows
|
||||
``frozen_participant_count >= CONFIG["min_market_participants"]`` AND
|
||||
``frozen_total_stake >= CONFIG["min_market_total_stake"]``.
|
||||
- The market is NOT a bootstrap-mode market (Sprint 8 will add the
|
||||
bootstrap path; until then bootstrap markets contribute zero).
|
||||
- The market is objective. Subjective markets mint Common Rep only
|
||||
(RULES §3.1).
|
||||
- The prediction's ``side`` matches the FINAL outcome.
|
||||
|
||||
Lost stakes from incorrect *staked* predictions reduce the running
|
||||
``oracle_rep`` balance (RULES §3.2 — the staked amount is forfeited to
|
||||
the winner pool). ``oracle_rep_lifetime`` is monotonically increasing
|
||||
and ignores losses.
|
||||
|
||||
Sprint 2 does NOT yet handle:
|
||||
|
||||
- Dispute reversal (Sprint 5 — `dispute_resolve` with `outcome="reversed"`).
|
||||
- Resolution-stake redistribution (Sprint 4/5 — `resolution_stake`
|
||||
events and the loser-pool burn).
|
||||
- Anti-gaming farming multipliers (Sprint 3).
|
||||
|
||||
These layers will be added by their owning sprints; the function
|
||||
signature and return shape are stable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
from services.infonet.markets.dispute import effective_outcome as _effective_outcome
|
||||
from services.infonet.reputation.anti_gaming.farming import (
|
||||
compute_farming_pct,
|
||||
farming_multiplier,
|
||||
)
|
||||
|
||||
|
||||
def _as_event_list(chain: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Accept any iterable, return a stable list ordered by (timestamp, sequence)."""
|
||||
events = [e for e in chain if isinstance(e, dict)]
|
||||
events.sort(key=lambda e: (float(e.get("timestamp") or 0.0), int(e.get("sequence") or 0)))
|
||||
return events
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class _MarketView:
|
||||
"""Internal: chain-derived view of a single market.
|
||||
|
||||
Populated in one pass over the chain. Holds only what oracle_rep
|
||||
needs to mint correctly per RULES §3.1/§3.2.
|
||||
"""
|
||||
market_id: str
|
||||
market_type: str = "objective"
|
||||
bootstrap_index: int | None = None
|
||||
snapshot: dict[str, Any] | None = None
|
||||
finalize: dict[str, Any] | None = None
|
||||
finalize_ts: float = 0.0
|
||||
predictions: list[dict[str, Any]] = field(default_factory=list)
|
||||
farming_pct_lookup: dict[str, float] = field(default_factory=dict)
|
||||
|
||||
|
||||
def _index_markets(events: list[dict[str, Any]]) -> dict[str, _MarketView]:
|
||||
markets: dict[str, _MarketView] = {}
|
||||
for ev in events:
|
||||
et = ev.get("event_type")
|
||||
p = _payload(ev)
|
||||
mid = p.get("market_id")
|
||||
if not isinstance(mid, str) or not mid:
|
||||
continue
|
||||
m = markets.setdefault(mid, _MarketView(market_id=mid))
|
||||
if et == "prediction_create":
|
||||
m.market_type = str(p.get("market_type") or "objective")
|
||||
if "bootstrap_index" in p and p["bootstrap_index"] is not None:
|
||||
try:
|
||||
m.bootstrap_index = int(p["bootstrap_index"])
|
||||
except (TypeError, ValueError):
|
||||
m.bootstrap_index = None
|
||||
elif et == "market_snapshot":
|
||||
m.snapshot = p
|
||||
elif et == "resolution_finalize":
|
||||
m.finalize = p
|
||||
m.finalize_ts = float(ev.get("timestamp") or 0.0)
|
||||
elif et == "prediction_place":
|
||||
m.predictions.append({
|
||||
"node_id": ev.get("node_id"),
|
||||
"side": p.get("side"),
|
||||
"stake_amount": p.get("stake_amount"),
|
||||
"probability_at_bet": p.get("probability_at_bet"),
|
||||
"timestamp": ev.get("timestamp"),
|
||||
})
|
||||
return markets
|
||||
|
||||
|
||||
def _market_passes_liquidity(market: _MarketView) -> bool:
|
||||
snap = market.snapshot or {}
|
||||
try:
|
||||
participants = int(snap.get("frozen_participant_count") or 0)
|
||||
total_stake = float(snap.get("frozen_total_stake") or 0.0)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return (
|
||||
participants >= int(CONFIG["min_market_participants"])
|
||||
and total_stake >= float(CONFIG["min_market_total_stake"])
|
||||
)
|
||||
|
||||
|
||||
def _market_is_mintable(market: _MarketView) -> bool:
|
||||
"""Return True if the market is final, non-provisional, non-bootstrap,
|
||||
objective, and passed liquidity. Mintable markets contribute oracle rep
|
||||
to correct predictors.
|
||||
"""
|
||||
finalize = market.finalize
|
||||
if not finalize:
|
||||
return False
|
||||
if finalize.get("is_provisional") is not False:
|
||||
return False
|
||||
outcome = finalize.get("outcome")
|
||||
if outcome not in ("yes", "no"):
|
||||
return False
|
||||
if market.market_type != "objective":
|
||||
return False
|
||||
# Sprint 8: bootstrap markets that resolved via eligible-node-one-vote
|
||||
# mint oracle rep from correct predictions, same as normal markets.
|
||||
# The bootstrap mechanic only changes HOW resolution decides yes/no —
|
||||
# not whether predictors get rep for being correct. RULES §3.10 step
|
||||
# 0.5: "Oracle rep minted normally from correct predictions
|
||||
# (constitutional)".
|
||||
if not _market_passes_liquidity(market):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _free_pred_mint(probability_at_bet: float) -> float:
|
||||
"""RULES §3.1 — mint = max(oracle_min_earned, 1.0 - p/100)."""
|
||||
if probability_at_bet is None:
|
||||
return 0.0
|
||||
try:
|
||||
prob = float(probability_at_bet)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
if not (0.0 <= prob <= 100.0):
|
||||
return 0.0
|
||||
return max(float(CONFIG["oracle_min_earned"]), 1.0 - (prob / 100.0))
|
||||
|
||||
|
||||
def _staked_pred_settlement(
|
||||
stake_amount: float,
|
||||
side: str,
|
||||
outcome: str,
|
||||
predictions: list[dict[str, Any]],
|
||||
) -> float:
|
||||
"""RULES §3.2 — pool settlement for staked predictions.
|
||||
|
||||
Returns the *net* change to oracle rep for a single staked
|
||||
prediction. Positive = winnings (returned stake + share of loser
|
||||
pool). Negative = forfeited stake.
|
||||
"""
|
||||
winning_side = outcome
|
||||
losing_side = "no" if outcome == "yes" else "yes"
|
||||
winner_pool = 0.0
|
||||
loser_pool = 0.0
|
||||
for pred in predictions:
|
||||
amt = pred.get("stake_amount")
|
||||
if amt is None:
|
||||
continue
|
||||
try:
|
||||
a = float(amt)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if a <= 0:
|
||||
continue
|
||||
if pred.get("side") == winning_side:
|
||||
winner_pool += a
|
||||
elif pred.get("side") == losing_side:
|
||||
loser_pool += a
|
||||
|
||||
if side == winning_side:
|
||||
if winner_pool == 0.0:
|
||||
return float(stake_amount) # degenerate — return stake
|
||||
if loser_pool == 0.0:
|
||||
return float(stake_amount) # everyone won — no profit
|
||||
share = float(stake_amount) / winner_pool
|
||||
winnings = share * loser_pool
|
||||
return float(stake_amount) + winnings
|
||||
elif side == losing_side:
|
||||
return -float(stake_amount)
|
||||
return 0.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OracleRepBreakdown:
|
||||
"""Auditable breakdown of how a node arrived at its oracle_rep balance.
|
||||
|
||||
Useful for the UI's reputation-history view and for invariant tests.
|
||||
Sprint 4+ extensions will add resolution-stake redistribution and
|
||||
dispute-reversal adjustments to this struct.
|
||||
"""
|
||||
free_prediction_mints: float
|
||||
staked_prediction_returns: float
|
||||
staked_prediction_losses: float
|
||||
total: float
|
||||
|
||||
|
||||
def compute_oracle_rep_breakdown(
|
||||
node_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> OracleRepBreakdown:
|
||||
"""Per-component breakdown — exposed for tests and audit trails.
|
||||
|
||||
Sprint 3 wiring: applies the farming multiplier (RULES §3.1) to
|
||||
free-pick mints. Staked predictions are NOT farming-penalized — the
|
||||
farmer is risking actual rep, which is the protocol's deterrent for
|
||||
that case. Per-spec semantics.
|
||||
"""
|
||||
events = _as_event_list(chain)
|
||||
markets = _index_markets(events)
|
||||
|
||||
farming_pct = compute_farming_pct(node_id, events)
|
||||
farming_mult = farming_multiplier(farming_pct)
|
||||
|
||||
free_mint = 0.0
|
||||
staked_return = 0.0
|
||||
staked_loss = 0.0
|
||||
|
||||
for market in markets.values():
|
||||
if not _market_is_mintable(market):
|
||||
continue
|
||||
original = market.finalize["outcome"] # type: ignore[index]
|
||||
# Sprint 5 bounded reversal: a resolved dispute can flip the
|
||||
# effective outcome of THIS market only — no cascade.
|
||||
outcome = _effective_outcome(original, market.market_id, events)
|
||||
for pred in market.predictions:
|
||||
if pred.get("node_id") != node_id:
|
||||
continue
|
||||
stake = pred.get("stake_amount")
|
||||
if stake is None:
|
||||
if pred.get("side") == outcome:
|
||||
free_mint += _free_pred_mint(pred.get("probability_at_bet")) * farming_mult
|
||||
# Wrong free pick: oracle_rep_earned = 0 (RULES §3.1)
|
||||
else:
|
||||
delta = _staked_pred_settlement(
|
||||
stake_amount=stake,
|
||||
side=pred.get("side", ""),
|
||||
outcome=outcome,
|
||||
predictions=market.predictions,
|
||||
)
|
||||
if delta >= 0:
|
||||
staked_return += delta
|
||||
else:
|
||||
staked_loss += -delta
|
||||
|
||||
total = free_mint + staked_return - staked_loss
|
||||
if total < 0:
|
||||
# Oracle rep is non-negative by spec (lost-stake forfeits transfer to
|
||||
# winners; they never push a balance below zero in isolation, but a
|
||||
# naive node-only view can underflow if the node never won
|
||||
# anything). Clamp to zero — the chain analysis on the full network
|
||||
# always sums to a non-negative total.
|
||||
total = 0.0
|
||||
return OracleRepBreakdown(
|
||||
free_prediction_mints=free_mint,
|
||||
staked_prediction_returns=staked_return,
|
||||
staked_prediction_losses=staked_loss,
|
||||
total=total,
|
||||
)
|
||||
|
||||
|
||||
def compute_oracle_rep(node_id: str, chain: Iterable[dict[str, Any]]) -> float:
|
||||
"""Current oracle rep balance for ``node_id``.
|
||||
|
||||
Wins (free mint + staked winnings) minus losses (staked forfeits).
|
||||
Clamped at zero. See ``compute_oracle_rep_breakdown`` for the full
|
||||
component view.
|
||||
"""
|
||||
return compute_oracle_rep_breakdown(node_id, chain).total
|
||||
|
||||
|
||||
def compute_oracle_rep_lifetime(node_id: str, chain: Iterable[dict[str, Any]]) -> float:
|
||||
"""Cumulative oracle rep ever earned by ``node_id``.
|
||||
|
||||
Monotonically increasing (analytics / profiles only — never drives
|
||||
protocol logic per RULES §2.1). Counts wins; ignores losses.
|
||||
"""
|
||||
bd = compute_oracle_rep_breakdown(node_id, chain)
|
||||
return bd.free_prediction_mints + bd.staked_prediction_returns
|
||||
|
||||
|
||||
def last_successful_prediction_ts(
|
||||
node_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> float | None:
|
||||
"""Timestamp of the node's most recent correct prediction in a
|
||||
market that:
|
||||
|
||||
1. Reached FINAL (non-INVALID) status.
|
||||
2. Was not provisional at finalize time.
|
||||
3. Passed the frozen liquidity thresholds.
|
||||
4. Was not later reversed by dispute (Sprint 5 — until then, no
|
||||
reversal logic; this function only sees the raw outcome).
|
||||
|
||||
Returns ``None`` if the node has no qualifying prediction.
|
||||
|
||||
Used by ``governance_decay.compute_oracle_rep_active`` to determine
|
||||
decay age. Per RULES §3.11 INVALID markets do NOT reset the clock —
|
||||
enforced here by the ``_market_is_mintable`` filter.
|
||||
"""
|
||||
events = _as_event_list(chain)
|
||||
markets = _index_markets(events)
|
||||
|
||||
best_ts: float | None = None
|
||||
for market in markets.values():
|
||||
if not _market_is_mintable(market):
|
||||
continue
|
||||
original = market.finalize["outcome"] # type: ignore[index]
|
||||
# Sprint 5 bounded reversal: dispute reversal flips the
|
||||
# effective outcome — predictors who picked the new winning
|
||||
# side are the ones whose timestamps qualify.
|
||||
outcome = _effective_outcome(original, market.market_id, events)
|
||||
finalize_ts = market.finalize_ts
|
||||
for pred in market.predictions:
|
||||
if pred.get("node_id") != node_id:
|
||||
continue
|
||||
if pred.get("side") != outcome:
|
||||
continue
|
||||
ts = float(pred.get("timestamp") or 0.0)
|
||||
# Use the LATER of prediction timestamp and finalize timestamp —
|
||||
# the "successful prediction" only crystallizes when finalize lands.
|
||||
ts = max(ts, finalize_ts)
|
||||
if best_ts is None or ts > best_ts:
|
||||
best_ts = ts
|
||||
return best_ts
|
||||
|
||||
|
||||
__all__ = [
|
||||
"OracleRepBreakdown",
|
||||
"compute_oracle_rep",
|
||||
"compute_oracle_rep_breakdown",
|
||||
"compute_oracle_rep_lifetime",
|
||||
"last_successful_prediction_ts",
|
||||
]
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Weekly vote budget — RULES §3.7.
|
||||
|
||||
weekly_budget = weekly_vote_base + floor(oracle_rep / weekly_vote_per_oracle)
|
||||
|
||||
Notes on placement: the budget is reputation-derived and gates how
|
||||
many upreps a node can cast in a 7-day window. Anti-gaming penalties
|
||||
shrink each uprep's *weight*, but the budget is what bounds *count*.
|
||||
Both layers run together to defeat farming.
|
||||
|
||||
Enforcement is upstream of the chain (the producer must check budget
|
||||
before signing a new ``uprep`` event); this module provides the
|
||||
computation and a chain-side audit (``count_upreps_in_last_week``) so
|
||||
verifiers can spot budget violations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Any, Iterable
|
||||
|
||||
from services.infonet.config import CONFIG
|
||||
from services.infonet.reputation.oracle_rep import compute_oracle_rep
|
||||
|
||||
|
||||
_SECONDS_PER_DAY = 86400.0
|
||||
_WEEK_S = 7 * _SECONDS_PER_DAY
|
||||
|
||||
|
||||
def _payload(event: dict[str, Any]) -> dict[str, Any]:
|
||||
p = event.get("payload")
|
||||
return p if isinstance(p, dict) else {}
|
||||
|
||||
|
||||
def compute_weekly_vote_budget(
|
||||
node_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
) -> int:
|
||||
"""Per-week uprep budget for ``node_id``."""
|
||||
base = int(CONFIG["weekly_vote_base"])
|
||||
per_oracle = int(CONFIG["weekly_vote_per_oracle"])
|
||||
if per_oracle <= 0:
|
||||
return base
|
||||
rep = compute_oracle_rep(node_id, chain)
|
||||
return base + math.floor(rep / per_oracle)
|
||||
|
||||
|
||||
def count_upreps_in_last_week(
|
||||
node_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float,
|
||||
) -> int:
|
||||
"""Count of ``uprep`` events authored by ``node_id`` in the past 7 days
|
||||
relative to ``now``. Used by chain-side audits.
|
||||
"""
|
||||
cutoff = float(now) - _WEEK_S
|
||||
count = 0
|
||||
for ev in chain:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
if ev.get("event_type") != "uprep":
|
||||
continue
|
||||
if ev.get("node_id") != node_id:
|
||||
continue
|
||||
try:
|
||||
ts = float(ev.get("timestamp") or 0.0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if cutoff <= ts <= float(now):
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def is_budget_exceeded(
|
||||
node_id: str,
|
||||
chain: Iterable[dict[str, Any]],
|
||||
*,
|
||||
now: float,
|
||||
) -> bool:
|
||||
"""``True`` if the node has cast more upreps in the past 7 days than
|
||||
its current weekly budget allows.
|
||||
|
||||
Cross-cutting design rule: producers should call this in the
|
||||
background as a soft-fail check — the user's queued uprep is still
|
||||
accepted, but flagged for delayed processing rather than refused
|
||||
outright. Constitutional rejections are reserved for unsigned
|
||||
writes / replays / rotation-during-active-stakes.
|
||||
"""
|
||||
return count_upreps_in_last_week(node_id, chain, now=now) > compute_weekly_vote_budget(node_id, chain)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"compute_weekly_vote_budget",
|
||||
"count_upreps_in_last_week",
|
||||
"is_budget_exceeded",
|
||||
]
|
||||
@@ -0,0 +1,859 @@
|
||||
"""Event-type registry and per-event payload validators for the Infonet
|
||||
economy layer.
|
||||
|
||||
Source of truth: ``infonet-economy/RULES_SKELETON.md`` §4.1.
|
||||
|
||||
The legacy ``services/mesh/mesh_schema.py`` ships
|
||||
``ACTIVE_PUBLIC_LEDGER_EVENT_TYPES`` for the existing mesh / DM / oracle
|
||||
events. This module ships ``INFONET_ECONOMY_EVENT_TYPES`` — a disjoint
|
||||
set of 40+ NEW event types added by the economy layer. Sprint 1's
|
||||
adversarial test asserts the disjointness invariant.
|
||||
|
||||
Sprint 1 implements *structural* validators only — they assert payload
|
||||
shape (required fields, basic types, enum membership). Deep semantic
|
||||
validation (e.g. that ``probability_at_bet`` was actually computed from
|
||||
the live chain state, that ``evidence_content_hash`` is canonical) lives
|
||||
in later sprints alongside the modules that produce those values.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable
|
||||
|
||||
# ─── Event-type set ──────────────────────────────────────────────────────
|
||||
# RULES_SKELETON.md §4.1.
|
||||
# Disjoint from mesh_schema.ACTIVE_PUBLIC_LEDGER_EVENT_TYPES — the union
|
||||
# is the full public ledger surface once the adapter is wired in.
|
||||
|
||||
INFONET_ECONOMY_EVENT_TYPES: frozenset[str] = frozenset({
|
||||
# Reputation
|
||||
"uprep",
|
||||
"downrep", # held off the active set in Sprint 2 — see BRAINDUMP §11
|
||||
# Markets / resolution-as-prediction
|
||||
"prediction_create",
|
||||
"prediction_place",
|
||||
"truth_stake_place",
|
||||
"truth_stake_resolve",
|
||||
"market_snapshot",
|
||||
"evidence_submit",
|
||||
"resolution_stake",
|
||||
"bootstrap_resolution_vote",
|
||||
"resolution_finalize",
|
||||
# Disputes
|
||||
"dispute_open",
|
||||
"dispute_stake",
|
||||
"dispute_resolve",
|
||||
# Gates (extend the existing legacy gate_create)
|
||||
"gate_enter",
|
||||
"gate_exit",
|
||||
"gate_lock",
|
||||
# Gate shutdown lifecycle
|
||||
"gate_suspend_file",
|
||||
"gate_suspend_vote",
|
||||
"gate_suspend_execute",
|
||||
"gate_shutdown_file",
|
||||
"gate_shutdown_vote",
|
||||
"gate_shutdown_execute",
|
||||
"gate_unsuspend",
|
||||
"gate_shutdown_appeal_file",
|
||||
"gate_shutdown_appeal_vote",
|
||||
"gate_shutdown_appeal_resolve",
|
||||
# Governance
|
||||
"petition_file",
|
||||
"petition_sign",
|
||||
"petition_vote",
|
||||
"challenge_file",
|
||||
"challenge_vote",
|
||||
"petition_execute",
|
||||
# Upgrade-hash governance
|
||||
"upgrade_propose",
|
||||
"upgrade_sign",
|
||||
"upgrade_vote",
|
||||
"upgrade_challenge",
|
||||
"upgrade_challenge_vote",
|
||||
"upgrade_signal_ready",
|
||||
"upgrade_activate",
|
||||
# Identity
|
||||
"node_register",
|
||||
"identity_rotate",
|
||||
"citizenship_claim",
|
||||
# Economy
|
||||
"coin_transfer",
|
||||
"coin_mint",
|
||||
"bounty_create",
|
||||
"bounty_claim",
|
||||
# Content
|
||||
"post_create",
|
||||
"post_reply",
|
||||
})
|
||||
|
||||
|
||||
# ─── Validator dataclass + helpers ───────────────────────────────────────
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InfonetEventSchema:
|
||||
event_type: str
|
||||
required_fields: tuple[str, ...]
|
||||
optional_fields: tuple[str, ...]
|
||||
validate: Callable[[dict[str, Any]], tuple[bool, str]]
|
||||
|
||||
def validate_payload(self, payload: dict[str, Any]) -> tuple[bool, str]:
|
||||
return self.validate(payload)
|
||||
|
||||
|
||||
def _require(payload: dict[str, Any], fields: tuple[str, ...]) -> tuple[bool, str]:
|
||||
if not isinstance(payload, dict):
|
||||
return False, "payload must be an object"
|
||||
for key in fields:
|
||||
if key not in payload:
|
||||
return False, f"Missing field: {key}"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _is_nonempty_str(val: Any) -> bool:
|
||||
return isinstance(val, str) and bool(val.strip())
|
||||
|
||||
|
||||
def _is_positive_number(val: Any) -> bool:
|
||||
return isinstance(val, (int, float)) and not isinstance(val, bool) and val > 0
|
||||
|
||||
|
||||
def _is_nonnegative_number(val: Any) -> bool:
|
||||
return isinstance(val, (int, float)) and not isinstance(val, bool) and val >= 0
|
||||
|
||||
|
||||
# ─── Per-event validators ───────────────────────────────────────────────
|
||||
# Sprint 1 scope: structural (required fields, type sanity, enum guards).
|
||||
# Deeper semantic checks (cross-event references, hash canonicalization,
|
||||
# probability_at_bet reconstruction) ship in the sprint that owns the
|
||||
# producing module.
|
||||
|
||||
def _validate_uprep(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("target_node_id", "target_event_id"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["target_node_id"]):
|
||||
return False, "target_node_id must be non-empty"
|
||||
if not _is_nonempty_str(p["target_event_id"]):
|
||||
return False, "target_event_id must be non-empty"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_downrep(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
return _validate_uprep(p)
|
||||
|
||||
|
||||
def _validate_prediction_create(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("market_id", "market_type", "question", "trigger_date", "creation_bond"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["market_id"]):
|
||||
return False, "market_id must be non-empty"
|
||||
if p["market_type"] not in ("objective", "subjective"):
|
||||
return False, "market_type must be 'objective' or 'subjective'"
|
||||
if not _is_nonempty_str(p["question"]):
|
||||
return False, "question must be non-empty"
|
||||
if not _is_positive_number(p["trigger_date"]):
|
||||
return False, "trigger_date must be a positive timestamp"
|
||||
if not _is_nonnegative_number(p["creation_bond"]):
|
||||
return False, "creation_bond must be a non-negative number"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_prediction_place(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("market_id", "side", "probability_at_bet"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["market_id"]):
|
||||
return False, "market_id must be non-empty"
|
||||
if p["side"] not in ("yes", "no"):
|
||||
return False, "side must be 'yes' or 'no'"
|
||||
prob = p["probability_at_bet"]
|
||||
if not isinstance(prob, (int, float)) or isinstance(prob, bool):
|
||||
return False, "probability_at_bet must be numeric"
|
||||
if not (0 <= prob <= 100):
|
||||
return False, "probability_at_bet must be in [0, 100]"
|
||||
if "stake_amount" in p:
|
||||
if p["stake_amount"] is not None and not _is_positive_number(p["stake_amount"]):
|
||||
return False, "stake_amount must be positive when present"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_truth_stake_place(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("message_id", "poster_id", "side", "amount", "duration_days"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["message_id"]):
|
||||
return False, "message_id must be non-empty"
|
||||
if not _is_nonempty_str(p["poster_id"]):
|
||||
return False, "poster_id must be non-empty"
|
||||
if p["side"] not in ("truth", "false"):
|
||||
return False, "side must be 'truth' or 'false'"
|
||||
if not _is_positive_number(p["amount"]):
|
||||
return False, "amount must be positive"
|
||||
duration = p["duration_days"]
|
||||
if not isinstance(duration, int) or isinstance(duration, bool) or duration <= 0:
|
||||
return False, "duration_days must be a positive integer"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_truth_stake_resolve(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("message_id", "outcome"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if p["outcome"] not in ("truth", "false", "tie"):
|
||||
return False, "outcome must be 'truth', 'false', or 'tie'"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_market_snapshot(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(
|
||||
p,
|
||||
(
|
||||
"market_id",
|
||||
"frozen_participant_count",
|
||||
"frozen_total_stake",
|
||||
"frozen_predictor_ids",
|
||||
"frozen_probability_state",
|
||||
"frozen_at",
|
||||
),
|
||||
)
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not isinstance(p["frozen_participant_count"], int) or isinstance(p["frozen_participant_count"], bool):
|
||||
return False, "frozen_participant_count must be int"
|
||||
if p["frozen_participant_count"] < 0:
|
||||
return False, "frozen_participant_count must be >= 0"
|
||||
if not _is_nonnegative_number(p["frozen_total_stake"]):
|
||||
return False, "frozen_total_stake must be a non-negative number"
|
||||
if not isinstance(p["frozen_predictor_ids"], list):
|
||||
return False, "frozen_predictor_ids must be a list"
|
||||
if not all(_is_nonempty_str(x) for x in p["frozen_predictor_ids"]):
|
||||
return False, "frozen_predictor_ids entries must be non-empty strings"
|
||||
state = p["frozen_probability_state"]
|
||||
if not isinstance(state, dict) or "yes" not in state or "no" not in state:
|
||||
return False, "frozen_probability_state must be {yes, no}"
|
||||
if not _is_positive_number(p["frozen_at"]):
|
||||
return False, "frozen_at must be a positive timestamp"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_evidence_submit(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(
|
||||
p,
|
||||
(
|
||||
"market_id",
|
||||
"claimed_outcome",
|
||||
"evidence_hashes",
|
||||
"source_description",
|
||||
"evidence_content_hash",
|
||||
"submission_hash",
|
||||
"bond",
|
||||
),
|
||||
)
|
||||
if not ok:
|
||||
return ok, why
|
||||
if p["claimed_outcome"] not in ("yes", "no"):
|
||||
return False, "claimed_outcome must be 'yes' or 'no'"
|
||||
if not isinstance(p["evidence_hashes"], list) or not p["evidence_hashes"]:
|
||||
return False, "evidence_hashes must be a non-empty list"
|
||||
if not all(_is_nonempty_str(h) for h in p["evidence_hashes"]):
|
||||
return False, "evidence_hashes entries must be non-empty strings"
|
||||
if not isinstance(p["source_description"], str):
|
||||
return False, "source_description must be a string"
|
||||
if not _is_nonempty_str(p["evidence_content_hash"]):
|
||||
return False, "evidence_content_hash must be non-empty"
|
||||
if not _is_nonempty_str(p["submission_hash"]):
|
||||
return False, "submission_hash must be non-empty"
|
||||
if not _is_nonnegative_number(p["bond"]):
|
||||
return False, "bond must be a non-negative number"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_resolution_stake(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("market_id", "side", "amount", "rep_type"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if p["side"] not in ("yes", "no", "data_unavailable"):
|
||||
return False, "side must be 'yes' | 'no' | 'data_unavailable'"
|
||||
if not _is_positive_number(p["amount"]):
|
||||
return False, "amount must be positive"
|
||||
if p["rep_type"] not in ("oracle", "common"):
|
||||
return False, "rep_type must be 'oracle' or 'common'"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_bootstrap_resolution_vote(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("market_id", "side", "pow_nonce"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if p["side"] not in ("yes", "no"):
|
||||
return False, "side must be 'yes' or 'no'"
|
||||
if not isinstance(p["pow_nonce"], int) or isinstance(p["pow_nonce"], bool) or p["pow_nonce"] < 0:
|
||||
return False, "pow_nonce must be a non-negative integer"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_resolution_finalize(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("market_id", "outcome", "is_provisional", "snapshot_event_hash"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if p["outcome"] not in ("yes", "no", "invalid"):
|
||||
return False, "outcome must be 'yes' | 'no' | 'invalid'"
|
||||
if not isinstance(p["is_provisional"], bool):
|
||||
return False, "is_provisional must be a boolean"
|
||||
if not _is_nonempty_str(p["snapshot_event_hash"]):
|
||||
return False, "snapshot_event_hash must be non-empty"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_dispute_open(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("market_id", "challenger_stake", "reason"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_positive_number(p["challenger_stake"]):
|
||||
return False, "challenger_stake must be positive"
|
||||
if not _is_nonempty_str(p["reason"]):
|
||||
return False, "reason must be non-empty"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_dispute_stake(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("dispute_id", "side", "amount", "rep_type"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if p["side"] not in ("confirm", "reverse"):
|
||||
return False, "side must be 'confirm' or 'reverse'"
|
||||
if not _is_positive_number(p["amount"]):
|
||||
return False, "amount must be positive"
|
||||
if p["rep_type"] not in ("oracle", "common"):
|
||||
return False, "rep_type must be 'oracle' or 'common'"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_dispute_resolve(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("dispute_id", "outcome"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if p["outcome"] not in ("upheld", "reversed", "tie"):
|
||||
return False, "outcome must be 'upheld' | 'reversed' | 'tie'"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_gate_enter(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("gate_id", "sacrifice_amount"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["gate_id"]):
|
||||
return False, "gate_id must be non-empty"
|
||||
if not _is_positive_number(p["sacrifice_amount"]):
|
||||
return False, "sacrifice_amount must be positive"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_gate_exit(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("gate_id",))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["gate_id"]):
|
||||
return False, "gate_id must be non-empty"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_gate_lock(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("gate_id", "lock_cost"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["gate_id"]):
|
||||
return False, "gate_id must be non-empty"
|
||||
if not _is_positive_number(p["lock_cost"]):
|
||||
return False, "lock_cost must be positive"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_gate_action_petition_file(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("petition_id", "gate_id", "reason", "evidence_hashes"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["petition_id"]):
|
||||
return False, "petition_id must be non-empty"
|
||||
if not _is_nonempty_str(p["gate_id"]):
|
||||
return False, "gate_id must be non-empty"
|
||||
if not isinstance(p["reason"], str) or len(p["reason"]) > 2000:
|
||||
return False, "reason must be a string up to 2000 chars"
|
||||
if not isinstance(p["evidence_hashes"], list) or not p["evidence_hashes"]:
|
||||
return False, "evidence_hashes must be non-empty"
|
||||
if not all(_is_nonempty_str(h) for h in p["evidence_hashes"]):
|
||||
return False, "evidence_hashes entries must be non-empty strings"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_gate_action_vote(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("petition_id", "vote"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if p["vote"] not in ("for", "against"):
|
||||
return False, "vote must be 'for' or 'against'"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_gate_action_execute(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("petition_id", "gate_id"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["petition_id"]):
|
||||
return False, "petition_id must be non-empty"
|
||||
if not _is_nonempty_str(p["gate_id"]):
|
||||
return False, "gate_id must be non-empty"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_gate_unsuspend(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("gate_id",))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["gate_id"]):
|
||||
return False, "gate_id must be non-empty"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_gate_shutdown_appeal_file(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("petition_id", "gate_id", "target_petition_id", "reason", "evidence_hashes"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["target_petition_id"]):
|
||||
return False, "target_petition_id must be non-empty"
|
||||
if not isinstance(p["reason"], str) or len(p["reason"]) > 2000:
|
||||
return False, "reason must be a string up to 2000 chars"
|
||||
if not isinstance(p["evidence_hashes"], list) or not p["evidence_hashes"]:
|
||||
return False, "evidence_hashes must be non-empty"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_gate_shutdown_appeal_resolve(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("petition_id", "outcome"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if p["outcome"] not in ("voided_shutdown", "resumed"):
|
||||
return False, "outcome must be 'voided_shutdown' or 'resumed'"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
_VALID_PETITION_PAYLOAD_TYPES = frozenset({
|
||||
"UPDATE_PARAM",
|
||||
"BATCH_UPDATE_PARAMS",
|
||||
"ENABLE_FEATURE",
|
||||
"DISABLE_FEATURE",
|
||||
})
|
||||
|
||||
|
||||
def _validate_petition_file(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("petition_id", "petition_payload"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["petition_id"]):
|
||||
return False, "petition_id must be non-empty"
|
||||
payload = p["petition_payload"]
|
||||
if not isinstance(payload, dict) or "type" not in payload:
|
||||
return False, "petition_payload must be an object with a 'type' field"
|
||||
if payload["type"] not in _VALID_PETITION_PAYLOAD_TYPES:
|
||||
return False, f"petition_payload type must be one of {sorted(_VALID_PETITION_PAYLOAD_TYPES)}"
|
||||
# Structural shape per type. Semantic checks (key existence, bounds)
|
||||
# happen in the Sprint 7 DSL executor.
|
||||
t = payload["type"]
|
||||
if t == "UPDATE_PARAM":
|
||||
if "key" not in payload or "value" not in payload:
|
||||
return False, "UPDATE_PARAM requires key + value"
|
||||
if not _is_nonempty_str(payload["key"]):
|
||||
return False, "UPDATE_PARAM.key must be non-empty"
|
||||
elif t == "BATCH_UPDATE_PARAMS":
|
||||
if "updates" not in payload or not isinstance(payload["updates"], list) or not payload["updates"]:
|
||||
return False, "BATCH_UPDATE_PARAMS.updates must be a non-empty list"
|
||||
for u in payload["updates"]:
|
||||
if not isinstance(u, dict) or "key" not in u or "value" not in u:
|
||||
return False, "BATCH_UPDATE_PARAMS entries must be {key, value}"
|
||||
if not _is_nonempty_str(u["key"]):
|
||||
return False, "BATCH_UPDATE_PARAMS entry key must be non-empty"
|
||||
elif t in ("ENABLE_FEATURE", "DISABLE_FEATURE"):
|
||||
if "feature" not in payload or not _is_nonempty_str(payload["feature"]):
|
||||
return False, f"{t}.feature must be non-empty"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_petition_sign(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("petition_id",))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["petition_id"]):
|
||||
return False, "petition_id must be non-empty"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_petition_vote(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("petition_id", "vote"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if p["vote"] not in ("for", "against"):
|
||||
return False, "vote must be 'for' or 'against'"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_challenge_file(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("petition_id", "reason"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["petition_id"]):
|
||||
return False, "petition_id must be non-empty"
|
||||
if not _is_nonempty_str(p["reason"]):
|
||||
return False, "reason must be non-empty"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_challenge_vote(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("petition_id", "vote"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if p["vote"] not in ("uphold", "void"):
|
||||
return False, "vote must be 'uphold' or 'void'"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_petition_execute(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("petition_id",))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["petition_id"]):
|
||||
return False, "petition_id must be non-empty"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_upgrade_propose(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(
|
||||
p,
|
||||
(
|
||||
"proposal_id",
|
||||
"release_hash",
|
||||
"release_description",
|
||||
"target_protocol_version",
|
||||
),
|
||||
)
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["proposal_id"]):
|
||||
return False, "proposal_id must be non-empty"
|
||||
if not _is_nonempty_str(p["release_hash"]):
|
||||
return False, "release_hash must be non-empty"
|
||||
if not isinstance(p["release_description"], str) or len(p["release_description"]) > 4000:
|
||||
return False, "release_description must be a string up to 4000 chars"
|
||||
if not _is_nonempty_str(p["target_protocol_version"]):
|
||||
return False, "target_protocol_version must be non-empty"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_upgrade_sign(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("proposal_id",))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["proposal_id"]):
|
||||
return False, "proposal_id must be non-empty"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_upgrade_vote(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("proposal_id", "vote"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if p["vote"] not in ("for", "against"):
|
||||
return False, "vote must be 'for' or 'against'"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_upgrade_challenge(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
return _validate_challenge_file({"petition_id": p.get("proposal_id", ""), "reason": p.get("reason", "")})
|
||||
|
||||
|
||||
def _validate_upgrade_challenge_vote(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("proposal_id", "vote"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if p["vote"] not in ("uphold", "void"):
|
||||
return False, "vote must be 'uphold' or 'void'"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_upgrade_signal_ready(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("proposal_id", "release_hash"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["proposal_id"]):
|
||||
return False, "proposal_id must be non-empty"
|
||||
if not _is_nonempty_str(p["release_hash"]):
|
||||
return False, "release_hash must be non-empty"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_upgrade_activate(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("proposal_id", "new_protocol_version"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["proposal_id"]):
|
||||
return False, "proposal_id must be non-empty"
|
||||
if not _is_nonempty_str(p["new_protocol_version"]):
|
||||
return False, "new_protocol_version must be non-empty"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_node_register(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("public_key", "public_key_algo", "node_class"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["public_key"]):
|
||||
return False, "public_key must be non-empty"
|
||||
if p["public_key_algo"] not in ("ed25519", "ecdsa"):
|
||||
return False, "public_key_algo must be 'ed25519' or 'ecdsa'"
|
||||
if p["node_class"] not in ("heavy", "light"):
|
||||
return False, "node_class must be 'heavy' or 'light'"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_identity_rotate(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(
|
||||
p,
|
||||
(
|
||||
"old_node_id",
|
||||
"old_public_key",
|
||||
"old_public_key_algo",
|
||||
"new_public_key",
|
||||
"new_public_key_algo",
|
||||
"old_signature",
|
||||
),
|
||||
)
|
||||
if not ok:
|
||||
return ok, why
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_citizenship_claim(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("sacrifice_amount",))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_positive_number(p["sacrifice_amount"]):
|
||||
return False, "sacrifice_amount must be positive"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_coin_transfer(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
# Sprint 1 logical-only — privacy primitives (RingCT) replace this in
|
||||
# Sprint 11+. Until then, enforce a simple {to, amount} shape.
|
||||
ok, why = _require(p, ("to_node_id", "amount"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["to_node_id"]):
|
||||
return False, "to_node_id must be non-empty"
|
||||
if not _is_positive_number(p["amount"]):
|
||||
return False, "amount must be positive"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_coin_mint(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("month", "total_minted", "ubi_pool", "dividend_pool"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["month"]):
|
||||
return False, "month must be non-empty (e.g. '2026-04')"
|
||||
for k in ("total_minted", "ubi_pool", "dividend_pool"):
|
||||
if not _is_nonnegative_number(p[k]):
|
||||
return False, f"{k} must be non-negative"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_bounty_create(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("bounty_id", "amount", "description"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["bounty_id"]):
|
||||
return False, "bounty_id must be non-empty"
|
||||
if not _is_positive_number(p["amount"]):
|
||||
return False, "amount must be positive"
|
||||
if not _is_nonempty_str(p["description"]):
|
||||
return False, "description must be non-empty"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_bounty_claim(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("bounty_id",))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["bounty_id"]):
|
||||
return False, "bounty_id must be non-empty"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_post_create(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("post_id", "body"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["post_id"]):
|
||||
return False, "post_id must be non-empty"
|
||||
if not isinstance(p["body"], str):
|
||||
return False, "body must be a string"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _validate_post_reply(p: dict[str, Any]) -> tuple[bool, str]:
|
||||
ok, why = _require(p, ("post_id", "parent_post_id", "body"))
|
||||
if not ok:
|
||||
return ok, why
|
||||
if not _is_nonempty_str(p["post_id"]):
|
||||
return False, "post_id must be non-empty"
|
||||
if not _is_nonempty_str(p["parent_post_id"]):
|
||||
return False, "parent_post_id must be non-empty"
|
||||
if not isinstance(p["body"], str):
|
||||
return False, "body must be a string"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
# ─── Schema registry ─────────────────────────────────────────────────────
|
||||
|
||||
_SCHEMA_REGISTRY: dict[str, InfonetEventSchema] = {}
|
||||
|
||||
|
||||
def _reg(event_type: str, required: tuple[str, ...], optional: tuple[str, ...], fn) -> None:
|
||||
_SCHEMA_REGISTRY[event_type] = InfonetEventSchema(
|
||||
event_type=event_type,
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
validate=fn,
|
||||
)
|
||||
|
||||
|
||||
_reg("uprep", ("target_node_id", "target_event_id"), (), _validate_uprep)
|
||||
_reg("downrep", ("target_node_id", "target_event_id"), (), _validate_downrep)
|
||||
|
||||
_reg("prediction_create",
|
||||
("market_id", "market_type", "question", "trigger_date", "creation_bond"),
|
||||
(), _validate_prediction_create)
|
||||
_reg("prediction_place",
|
||||
("market_id", "side", "probability_at_bet"),
|
||||
("stake_amount",), _validate_prediction_place)
|
||||
_reg("truth_stake_place",
|
||||
("message_id", "poster_id", "side", "amount", "duration_days"),
|
||||
(), _validate_truth_stake_place)
|
||||
_reg("truth_stake_resolve",
|
||||
("message_id", "outcome"),
|
||||
(), _validate_truth_stake_resolve)
|
||||
_reg("market_snapshot",
|
||||
("market_id", "frozen_participant_count", "frozen_total_stake",
|
||||
"frozen_predictor_ids", "frozen_probability_state", "frozen_at"),
|
||||
("snapshot_event_hash",), _validate_market_snapshot)
|
||||
_reg("evidence_submit",
|
||||
("market_id", "claimed_outcome", "evidence_hashes", "source_description",
|
||||
"evidence_content_hash", "submission_hash", "bond"),
|
||||
(), _validate_evidence_submit)
|
||||
_reg("resolution_stake",
|
||||
("market_id", "side", "amount", "rep_type"),
|
||||
(), _validate_resolution_stake)
|
||||
_reg("bootstrap_resolution_vote",
|
||||
("market_id", "side", "pow_nonce"),
|
||||
(), _validate_bootstrap_resolution_vote)
|
||||
_reg("resolution_finalize",
|
||||
("market_id", "outcome", "is_provisional", "snapshot_event_hash"),
|
||||
(), _validate_resolution_finalize)
|
||||
|
||||
_reg("dispute_open", ("market_id", "challenger_stake", "reason"), (), _validate_dispute_open)
|
||||
_reg("dispute_stake", ("dispute_id", "side", "amount", "rep_type"), (), _validate_dispute_stake)
|
||||
_reg("dispute_resolve", ("dispute_id", "outcome"), (), _validate_dispute_resolve)
|
||||
|
||||
_reg("gate_enter", ("gate_id", "sacrifice_amount"), (), _validate_gate_enter)
|
||||
_reg("gate_exit", ("gate_id",), (), _validate_gate_exit)
|
||||
_reg("gate_lock", ("gate_id", "lock_cost"), (), _validate_gate_lock)
|
||||
|
||||
_reg("gate_suspend_file",
|
||||
("petition_id", "gate_id", "reason", "evidence_hashes"), (),
|
||||
_validate_gate_action_petition_file)
|
||||
_reg("gate_suspend_vote", ("petition_id", "vote"), (), _validate_gate_action_vote)
|
||||
_reg("gate_suspend_execute", ("petition_id", "gate_id"), (), _validate_gate_action_execute)
|
||||
_reg("gate_shutdown_file",
|
||||
("petition_id", "gate_id", "reason", "evidence_hashes"), (),
|
||||
_validate_gate_action_petition_file)
|
||||
_reg("gate_shutdown_vote", ("petition_id", "vote"), (), _validate_gate_action_vote)
|
||||
_reg("gate_shutdown_execute", ("petition_id", "gate_id"), (), _validate_gate_action_execute)
|
||||
_reg("gate_unsuspend", ("gate_id",), (), _validate_gate_unsuspend)
|
||||
_reg("gate_shutdown_appeal_file",
|
||||
("petition_id", "gate_id", "target_petition_id", "reason", "evidence_hashes"),
|
||||
(), _validate_gate_shutdown_appeal_file)
|
||||
_reg("gate_shutdown_appeal_vote", ("petition_id", "vote"), (), _validate_gate_action_vote)
|
||||
_reg("gate_shutdown_appeal_resolve", ("petition_id", "outcome"), (), _validate_gate_shutdown_appeal_resolve)
|
||||
|
||||
_reg("petition_file", ("petition_id", "petition_payload"), (), _validate_petition_file)
|
||||
_reg("petition_sign", ("petition_id",), (), _validate_petition_sign)
|
||||
_reg("petition_vote", ("petition_id", "vote"), (), _validate_petition_vote)
|
||||
_reg("challenge_file", ("petition_id", "reason"), (), _validate_challenge_file)
|
||||
_reg("challenge_vote", ("petition_id", "vote"), (), _validate_challenge_vote)
|
||||
_reg("petition_execute", ("petition_id",), (), _validate_petition_execute)
|
||||
|
||||
_reg("upgrade_propose",
|
||||
("proposal_id", "release_hash", "release_description", "target_protocol_version"),
|
||||
("release_url", "compatibility_notes"),
|
||||
_validate_upgrade_propose)
|
||||
_reg("upgrade_sign", ("proposal_id",), (), _validate_upgrade_sign)
|
||||
_reg("upgrade_vote", ("proposal_id", "vote"), (), _validate_upgrade_vote)
|
||||
_reg("upgrade_challenge", ("proposal_id", "reason"), (), _validate_upgrade_challenge)
|
||||
_reg("upgrade_challenge_vote", ("proposal_id", "vote"), (), _validate_upgrade_challenge_vote)
|
||||
_reg("upgrade_signal_ready", ("proposal_id", "release_hash"), (), _validate_upgrade_signal_ready)
|
||||
_reg("upgrade_activate", ("proposal_id", "new_protocol_version"), (), _validate_upgrade_activate)
|
||||
|
||||
_reg("node_register", ("public_key", "public_key_algo", "node_class"), (), _validate_node_register)
|
||||
_reg("identity_rotate",
|
||||
("old_node_id", "old_public_key", "old_public_key_algo",
|
||||
"new_public_key", "new_public_key_algo", "old_signature"),
|
||||
(), _validate_identity_rotate)
|
||||
_reg("citizenship_claim", ("sacrifice_amount",), (), _validate_citizenship_claim)
|
||||
|
||||
_reg("coin_transfer", ("to_node_id", "amount"), (), _validate_coin_transfer)
|
||||
_reg("coin_mint", ("month", "total_minted", "ubi_pool", "dividend_pool"), (), _validate_coin_mint)
|
||||
_reg("bounty_create", ("bounty_id", "amount", "description"), (), _validate_bounty_create)
|
||||
_reg("bounty_claim", ("bounty_id",), (), _validate_bounty_claim)
|
||||
|
||||
_reg("post_create", ("post_id", "body"), (), _validate_post_create)
|
||||
_reg("post_reply", ("post_id", "parent_post_id", "body"), (), _validate_post_reply)
|
||||
|
||||
|
||||
def get_infonet_schema(event_type: str) -> InfonetEventSchema | None:
|
||||
return _SCHEMA_REGISTRY.get(event_type)
|
||||
|
||||
|
||||
def validate_infonet_event_payload(
|
||||
event_type: str,
|
||||
payload: dict[str, Any],
|
||||
) -> tuple[bool, str]:
|
||||
"""Validate ``payload`` against the schema for ``event_type``.
|
||||
|
||||
Sprint 1 contract:
|
||||
- Event types not in ``INFONET_ECONOMY_EVENT_TYPES`` are rejected.
|
||||
- Every type in ``INFONET_ECONOMY_EVENT_TYPES`` MUST have a registered
|
||||
validator (asserted by ``assert_registry_complete``).
|
||||
"""
|
||||
if event_type not in INFONET_ECONOMY_EVENT_TYPES:
|
||||
return False, f"Unknown event_type for infonet economy: {event_type}"
|
||||
schema = _SCHEMA_REGISTRY.get(event_type)
|
||||
if schema is None:
|
||||
return False, f"No validator registered for: {event_type}"
|
||||
return schema.validate_payload(payload)
|
||||
|
||||
|
||||
def assert_registry_complete() -> None:
|
||||
"""Sprint 1 invariant: every event type has a validator."""
|
||||
missing = sorted(INFONET_ECONOMY_EVENT_TYPES - set(_SCHEMA_REGISTRY.keys()))
|
||||
if missing:
|
||||
raise AssertionError(f"INFONET_ECONOMY_EVENT_TYPES without validators: {missing}")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"INFONET_ECONOMY_EVENT_TYPES",
|
||||
"InfonetEventSchema",
|
||||
"assert_registry_complete",
|
||||
"get_infonet_schema",
|
||||
"validate_infonet_event_payload",
|
||||
]
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Test-only helpers for synthesizing chain events.
|
||||
|
||||
Mirrors the dict shape that ``InfonetHashchainAdapter.dry_run_append``
|
||||
emits, which in turn mirrors the legacy ``mesh_hashchain.Infonet.append``
|
||||
output. Tests call these helpers to build synthetic chains; production
|
||||
code is unaffected.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
def make_event(
|
||||
event_type: str,
|
||||
node_id: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
timestamp: float,
|
||||
sequence: int = 1,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"event_type": event_type,
|
||||
"node_id": node_id,
|
||||
"timestamp": float(timestamp),
|
||||
"sequence": int(sequence),
|
||||
"payload": dict(payload),
|
||||
}
|
||||
|
||||
|
||||
def make_market_chain(
|
||||
market_id: str,
|
||||
creator_id: str,
|
||||
*,
|
||||
market_type: str = "objective",
|
||||
bootstrap_index: int | None = None,
|
||||
base_ts: float = 1_700_000_000.0,
|
||||
participants: int = 5,
|
||||
total_stake: float = 10.0,
|
||||
outcome: str | None = "yes",
|
||||
is_provisional: bool = False,
|
||||
predictions: list[dict[str, Any]] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Build a coherent set of events for one market.
|
||||
|
||||
Returns events in chain order: prediction_create → prediction_place
|
||||
(per ``predictions``) → market_snapshot → resolution_finalize (if
|
||||
``outcome`` is not None). Use this to set up "did the mint rule
|
||||
fire correctly" tests.
|
||||
"""
|
||||
chain: list[dict[str, Any]] = []
|
||||
seq = 0
|
||||
|
||||
def _next_seq() -> int:
|
||||
nonlocal seq
|
||||
seq += 1
|
||||
return seq
|
||||
|
||||
chain.append(make_event(
|
||||
"prediction_create",
|
||||
creator_id,
|
||||
{
|
||||
"market_id": market_id,
|
||||
"market_type": market_type,
|
||||
"question": f"Q for {market_id}",
|
||||
"trigger_date": base_ts + 86400.0,
|
||||
"creation_bond": 3,
|
||||
**({"bootstrap_index": bootstrap_index} if bootstrap_index is not None else {}),
|
||||
},
|
||||
timestamp=base_ts,
|
||||
sequence=_next_seq(),
|
||||
))
|
||||
|
||||
predictor_ids: list[str] = []
|
||||
for i, pred in enumerate(predictions or []):
|
||||
chain.append(make_event(
|
||||
"prediction_place",
|
||||
pred["node_id"],
|
||||
{
|
||||
"market_id": market_id,
|
||||
"side": pred["side"],
|
||||
"probability_at_bet": pred.get("probability_at_bet", 50.0),
|
||||
**({"stake_amount": pred["stake_amount"]} if pred.get("stake_amount") is not None else {}),
|
||||
},
|
||||
timestamp=base_ts + 60.0 + i,
|
||||
sequence=_next_seq(),
|
||||
))
|
||||
predictor_ids.append(pred["node_id"])
|
||||
|
||||
snapshot_ts = base_ts + 3600.0
|
||||
chain.append(make_event(
|
||||
"market_snapshot",
|
||||
creator_id,
|
||||
{
|
||||
"market_id": market_id,
|
||||
"frozen_participant_count": participants,
|
||||
"frozen_total_stake": float(total_stake),
|
||||
"frozen_predictor_ids": list(dict.fromkeys(predictor_ids)),
|
||||
"frozen_probability_state": {"yes": 0.5, "no": 0.5},
|
||||
"frozen_at": snapshot_ts,
|
||||
},
|
||||
timestamp=snapshot_ts,
|
||||
sequence=_next_seq(),
|
||||
))
|
||||
|
||||
if outcome is not None:
|
||||
finalize_ts = base_ts + 7200.0
|
||||
chain.append(make_event(
|
||||
"resolution_finalize",
|
||||
creator_id,
|
||||
{
|
||||
"market_id": market_id,
|
||||
"outcome": outcome,
|
||||
"is_provisional": bool(is_provisional),
|
||||
"snapshot_event_hash": f"snap-{market_id}",
|
||||
},
|
||||
timestamp=finalize_ts,
|
||||
sequence=_next_seq(),
|
||||
))
|
||||
|
||||
return chain
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Test-only chain-builder helpers for gate scenarios."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from services.infonet.tests._chain_factory import make_event
|
||||
|
||||
|
||||
def make_gate_create(
|
||||
gate_id: str,
|
||||
creator: str,
|
||||
*,
|
||||
ts: float,
|
||||
seq: int = 1,
|
||||
entry_sacrifice: int = 5,
|
||||
min_overall_rep: int = 0,
|
||||
min_gate_rep: dict[str, int] | None = None,
|
||||
display_name: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
rules: dict[str, Any] = {
|
||||
"entry_sacrifice": entry_sacrifice,
|
||||
"min_overall_rep": min_overall_rep,
|
||||
}
|
||||
if min_gate_rep:
|
||||
rules["min_gate_rep"] = dict(min_gate_rep)
|
||||
return make_event(
|
||||
"gate_create", creator,
|
||||
{"gate_id": gate_id, "display_name": display_name or gate_id, "rules": rules},
|
||||
timestamp=ts, sequence=seq,
|
||||
)
|
||||
|
||||
|
||||
def make_gate_enter(gate_id: str, node: str, *, ts: float, seq: int,
|
||||
sacrifice: int = 5) -> dict[str, Any]:
|
||||
return make_event(
|
||||
"gate_enter", node,
|
||||
{"gate_id": gate_id, "sacrifice_amount": sacrifice},
|
||||
timestamp=ts, sequence=seq,
|
||||
)
|
||||
|
||||
|
||||
def make_gate_exit(gate_id: str, node: str, *, ts: float, seq: int) -> dict[str, Any]:
|
||||
return make_event(
|
||||
"gate_exit", node,
|
||||
{"gate_id": gate_id},
|
||||
timestamp=ts, sequence=seq,
|
||||
)
|
||||
|
||||
|
||||
def make_gate_lock(gate_id: str, node: str, *, ts: float, seq: int,
|
||||
lock_cost: int = 10) -> dict[str, Any]:
|
||||
return make_event(
|
||||
"gate_lock", node,
|
||||
{"gate_id": gate_id, "lock_cost": lock_cost},
|
||||
timestamp=ts, sequence=seq,
|
||||
)
|
||||
|
||||
|
||||
def make_suspend_file(gate_id: str, filer: str, petition_id: str, *,
|
||||
ts: float, seq: int,
|
||||
reason: str = "abuse",
|
||||
evidence: list[str] | None = None) -> dict[str, Any]:
|
||||
return make_event(
|
||||
"gate_suspend_file", filer,
|
||||
{"petition_id": petition_id, "gate_id": gate_id,
|
||||
"reason": reason, "evidence_hashes": list(evidence or ["ev1"])},
|
||||
timestamp=ts, sequence=seq,
|
||||
)
|
||||
|
||||
|
||||
def make_suspend_execute(gate_id: str, petition_id: str, *,
|
||||
ts: float, seq: int,
|
||||
executor: str = "creator") -> dict[str, Any]:
|
||||
return make_event(
|
||||
"gate_suspend_execute", executor,
|
||||
{"petition_id": petition_id, "gate_id": gate_id},
|
||||
timestamp=ts, sequence=seq,
|
||||
)
|
||||
|
||||
|
||||
def make_unsuspend(gate_id: str, *, ts: float, seq: int,
|
||||
executor: str = "creator") -> dict[str, Any]:
|
||||
return make_event(
|
||||
"gate_unsuspend", executor,
|
||||
{"gate_id": gate_id},
|
||||
timestamp=ts, sequence=seq,
|
||||
)
|
||||
|
||||
|
||||
def make_shutdown_file(gate_id: str, filer: str, petition_id: str, *,
|
||||
ts: float, seq: int,
|
||||
reason: str = "still abusing",
|
||||
evidence: list[str] | None = None) -> dict[str, Any]:
|
||||
return make_event(
|
||||
"gate_shutdown_file", filer,
|
||||
{"petition_id": petition_id, "gate_id": gate_id,
|
||||
"reason": reason, "evidence_hashes": list(evidence or ["ev1"])},
|
||||
timestamp=ts, sequence=seq,
|
||||
)
|
||||
|
||||
|
||||
def make_shutdown_vote(gate_id: str, petition_id: str, vote: str, *,
|
||||
ts: float, seq: int,
|
||||
voter: str = "creator") -> dict[str, Any]:
|
||||
return make_event(
|
||||
"gate_shutdown_vote", voter,
|
||||
{"petition_id": petition_id, "vote": vote, "gate_id": gate_id},
|
||||
timestamp=ts, sequence=seq,
|
||||
)
|
||||
|
||||
|
||||
def make_shutdown_execute(gate_id: str, petition_id: str, *,
|
||||
ts: float, seq: int,
|
||||
executor: str = "creator") -> dict[str, Any]:
|
||||
return make_event(
|
||||
"gate_shutdown_execute", executor,
|
||||
{"petition_id": petition_id, "gate_id": gate_id},
|
||||
timestamp=ts, sequence=seq,
|
||||
)
|
||||
|
||||
|
||||
def make_appeal_file(gate_id: str, target_petition_id: str, filer: str,
|
||||
petition_id: str, *,
|
||||
ts: float, seq: int,
|
||||
reason: str = "appeal",
|
||||
evidence: list[str] | None = None) -> dict[str, Any]:
|
||||
return make_event(
|
||||
"gate_shutdown_appeal_file", filer,
|
||||
{"petition_id": petition_id, "gate_id": gate_id,
|
||||
"target_petition_id": target_petition_id,
|
||||
"reason": reason,
|
||||
"evidence_hashes": list(evidence or ["ev1"])},
|
||||
timestamp=ts, sequence=seq,
|
||||
)
|
||||
|
||||
|
||||
def make_appeal_resolve(gate_id: str, petition_id: str, target_petition_id: str,
|
||||
outcome: str, *, ts: float, seq: int,
|
||||
resumed_execution_at: float | None = None,
|
||||
resolver: str = "creator") -> dict[str, Any]:
|
||||
payload = {"petition_id": petition_id, "outcome": outcome,
|
||||
"target_petition_id": target_petition_id, "gate_id": gate_id}
|
||||
if resumed_execution_at is not None:
|
||||
payload["resumed_execution_at"] = resumed_execution_at
|
||||
return make_event("gate_shutdown_appeal_resolve", resolver, payload,
|
||||
timestamp=ts, sequence=seq)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user