CalvinBackup/Shadowbroker
1
0
Fork 0
mirror of https://github.com/BigBodyCobain/Shadowbroker.git synced 2026-07-09 21:58:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

Feat/gt analytics openclaw (#392)

Browse Source 
* feat(telegram): auto-translate OSINT channel posts to English

Cherry-picked from @Bobpick PR #391 (telegram-only slice): server-side translation during fetch, SHOW ORIGINAL toggle in TelegramOsintPopup, and on-demand /api/telegram-feed?lang=.

Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

* feat(gt): experimental Derived OSINT analytics with lean-node safeguards

Cherry-picked from @Bobpick PR #391 (GT + OpenClaw slice): Bayesian strategic-risk engine, map overlay, OpenClaw commands, and telegram_rhetoric watchdog. Off by default (GT_ANALYTICS_ENABLED=false, gt_risk layer false). 1 vCPU nodes get cgroup detection, UI warning on layer toggle, and lean profile that skips scheduled ingest/Louvain unless GT_ANALYTICS_ACK_LOW_CPU=true. Backtest HUD removed from dashboard (OpenClaw/API regression only).

Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

---------

Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Shadowbroker
2026-06-16 17:05:46 -06:00
committed by GitHub
parent 9c5a4054f6
commit cfbeabda1e
69 changed files with 8102 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/analytics/__init__.py
+21
View File
@@ -0,0 +1,21 @@
"""Strategic Risk Analytics — game-theoretic early warning layer."""
from analytics.backtest import (
DEFAULT_BACKTEST_ALERT_THRESHOLD,
BacktestReport,
run_historical_backtest,
tune_alert_threshold,
)
from analytics.gt_early_warning import GT_EarlyWarning
from analytics.integration import get_gt_engine, process_feed_item, refresh_from_latest_data
__all__ = [
"BacktestReport",
"DEFAULT_BACKTEST_ALERT_THRESHOLD",
"GT_EarlyWarning",
"get_gt_engine",
"process_feed_item",
"refresh_from_latest_data",
"run_historical_backtest",
"tune_alert_threshold",
]
backend/analytics/backtest.py
+287
View File
@@ -0,0 +1,287 @@
"""Historical backtesting for Strategic Risk Analytics.
This is **benchmark validation**, not forward-weeks prediction on live feeds.
The suite scores whether costly-signal patterns + Bayesian updating correctly
classify curated pre-crisis text snippets (positive cases) vs cheap-talk
controls (negative cases) at a tuned alert threshold. A high accuracy on this
labeled corpus does **not** imply the engine will score 100% on messy,
adversarial, or weeks-ahead production telemetry — opponents adapt, labels are
easier here than in the wild, and the window is retrospective.
Reports accuracy and a conservative Wilson 95% confidence lower bound on the
benchmark only. Treat 100% here as "classifier fits the benchmark," not "ship
it for multi-week forecasting." For live week-over-week scoring with delayed
labels, see ``rolling_backtest.py``.
"""
from __future__ import annotations
import math
from dataclasses import dataclass, field
from typing import Any, Literal
from analytics.gt_early_warning import GT_EarlyWarning
from analytics.historical_events import (
HistoricalCase,
default_historical_cases,
expanded_historical_cases,
)
from analytics.settings import GTAnalyticsSettings
DomainName = Literal["financial", "unrest", "conflict"]
# Validated on expanded suite (82 cases, Wilson lower >= 0.95 at 100% accuracy).
DEFAULT_BACKTEST_ALERT_THRESHOLD = 0.26
MAX_BACKTEST_ALERT_THRESHOLD = 0.39
@dataclass(frozen=True)
class CaseResult:
case_id: str
name: str
kind: str
region: str
domain: str
expected_alert: bool
alerted: bool
correct: bool
peak_domain_risk: float
peak_composite_risk: float
costly_signals: list[str]
tags: tuple[str, ...] = field(default_factory=tuple)
@dataclass(frozen=True)
class BacktestReport:
total_cases: int
correct: int
accuracy: float
confidence_rate: float
wilson_lower_95: float
wilson_upper_95: float
true_positives: int
true_negatives: int
false_positives: int
false_negatives: int
sensitivity: float
specificity: float
alert_threshold: float
target_confidence: float
meets_target: bool
case_results: tuple[CaseResult, ...]
def to_dict(self) -> dict[str, Any]:
return {
"total_cases": self.total_cases,
"correct": self.correct,
"accuracy": round(self.accuracy, 4),
"confidence_rate": round(self.confidence_rate, 4),
"wilson_lower_95": round(self.wilson_lower_95, 4),
"wilson_upper_95": round(self.wilson_upper_95, 4),
"true_positives": self.true_positives,
"true_negatives": self.true_negatives,
"false_positives": self.false_positives,
"false_negatives": self.false_negatives,
"sensitivity": round(self.sensitivity, 4),
"specificity": round(self.specificity, 4),
"alert_threshold": self.alert_threshold,
"target_confidence": self.target_confidence,
"meets_target": self.meets_target,
"cases": [
{
"case_id": row.case_id,
"name": row.name,
"kind": row.kind,
"correct": row.correct,
"alerted": row.alerted,
"peak_domain_risk": round(row.peak_domain_risk, 4),
"peak_composite_risk": round(row.peak_composite_risk, 4),
"costly_signals": row.costly_signals,
}
for row in self.case_results
],
}
def wilson_interval(
successes: int,
total: int,
z: float = 1.96,
) -> tuple[float, float]:
"""Wilson score interval for a binomial proportion (95% default)."""
if total <= 0:
return 0.0, 0.0
phat = successes / total
z2 = z * z
denom = 1.0 + z2 / total
center = (phat + z2 / (2.0 * total)) / denom
margin = (
z
* math.sqrt((phat * (1.0 - phat) + z2 / (4.0 * total)) / total)
/ denom
)
return max(0.0, center - margin), min(1.0, center + margin)
def _domain_risk(engine: GT_EarlyWarning, region: str, domain: str) -> float:
if domain in ("financial", "unrest", "conflict"):
return engine.get_prior(region, domain)
return engine.composite_risk(region)
def _evaluate_case(
case: HistoricalCase,
*,
settings: GTAnalyticsSettings,
alert_threshold: float,
) -> CaseResult:
engine = GT_EarlyWarning(settings)
peak_domain = float(settings.base_prior)
peak_composite = float(settings.base_prior)
detected_signals: set[str] = set()
for item in case.to_feed_dicts():
result = engine.process_feed_item(item)
for sig in (result or {}).get("signals") or {}:
detected_signals.add(str(sig))
domain_risk = _domain_risk(engine, case.region, case.domain)
composite = engine.composite_risk(case.region)
peak_domain = max(peak_domain, domain_risk)
peak_composite = max(peak_composite, composite)
# Domain-specific score for labeled events; composite as secondary for conflict.
score = peak_domain
if case.domain == "conflict":
score = max(peak_domain, peak_composite * 0.95)
alerted = score >= alert_threshold
expected_alert = case.kind == "positive"
return CaseResult(
case_id=case.case_id,
name=case.name,
kind=case.kind,
region=case.region,
domain=case.domain,
expected_alert=expected_alert,
alerted=alerted,
correct=alerted == expected_alert,
peak_domain_risk=peak_domain,
peak_composite_risk=peak_composite,
costly_signals=sorted(detected_signals),
tags=case.tags,
)
def run_historical_backtest(
cases: tuple[HistoricalCase, ...] | None = None,
*,
settings: GTAnalyticsSettings | None = None,
alert_threshold: float | None = None,
target_confidence: float = 0.80,
use_expanded_suite: bool = True,
) -> BacktestReport:
"""
Run labeled historical cases and compute accuracy + Wilson 95% CI.
``confidence_rate`` is the conservative Wilson lower bound — the metric
used for pass/fail against ``target_confidence``.
"""
cfg = settings or GTAnalyticsSettings(enabled=True)
threshold = float(
alert_threshold
if alert_threshold is not None
else DEFAULT_BACKTEST_ALERT_THRESHOLD
)
if cases is not None:
suite = cases
elif use_expanded_suite:
suite = expanded_historical_cases()
else:
suite = default_historical_cases()
results = tuple(
_evaluate_case(case, settings=cfg, alert_threshold=threshold) for case in suite
)
tp = sum(1 for r in results if r.expected_alert and r.alerted)
tn = sum(1 for r in results if not r.expected_alert and not r.alerted)
fp = sum(1 for r in results if not r.expected_alert and r.alerted)
fn = sum(1 for r in results if r.expected_alert and not r.alerted)
correct = tp + tn
total = len(results)
accuracy = correct / total if total else 0.0
lower, upper = wilson_interval(correct, total)
pos_total = sum(1 for r in results if r.expected_alert)
neg_total = total - pos_total
sensitivity = tp / pos_total if pos_total else 0.0
specificity = tn / neg_total if neg_total else 0.0
return BacktestReport(
total_cases=total,
correct=correct,
accuracy=accuracy,
confidence_rate=lower,
wilson_lower_95=lower,
wilson_upper_95=upper,
true_positives=tp,
true_negatives=tn,
false_positives=fp,
false_negatives=fn,
sensitivity=sensitivity,
specificity=specificity,
alert_threshold=threshold,
target_confidence=target_confidence,
meets_target=lower >= target_confidence,
case_results=results,
)
def tune_alert_threshold(
cases: tuple[HistoricalCase, ...] | None = None,
*,
settings: GTAnalyticsSettings | None = None,
min_threshold: float = 0.20,
max_threshold: float = 0.65,
step: float = 0.01,
target_confidence: float = 0.95,
) -> tuple[float, BacktestReport]:
"""Grid-search alert threshold to maximize Wilson lower bound."""
if cases is not None:
suite = cases
else:
suite = expanded_historical_cases()
best_threshold = min_threshold
best_report = run_historical_backtest(
suite,
settings=settings,
alert_threshold=min_threshold,
target_confidence=target_confidence,
)
steps = int(round((max_threshold - min_threshold) / step))
for i in range(steps + 1):
threshold = min_threshold + i * step
report = run_historical_backtest(
suite,
settings=settings,
alert_threshold=threshold,
target_confidence=target_confidence,
)
better_confidence = report.confidence_rate > best_report.confidence_rate
tied_confidence = math.isclose(
report.confidence_rate, best_report.confidence_rate, rel_tol=0.0, abs_tol=1e-9
)
better_accuracy = report.accuracy > best_report.accuracy
tied_accuracy = math.isclose(
report.accuracy, best_report.accuracy, rel_tol=0.0, abs_tol=1e-9
)
prefer_higher_threshold = (
tied_confidence and tied_accuracy and threshold > best_threshold
)
if better_confidence or (tied_confidence and better_accuracy) or prefer_higher_threshold:
best_threshold = threshold
best_report = report
return best_threshold, best_report
backend/analytics/daily_store.py
+140
View File
@@ -0,0 +1,140 @@
"""Daily GT risk readings for micro rolling averages."""
from __future__ import annotations
import json
import logging
import os
import threading
from dataclasses import asdict, dataclass, field
from datetime import date, datetime, timezone
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
_DAILY_DIR = Path(__file__).parent.parent / "data" / "gt_rolling" / "daily"
_store_lock = threading.Lock()
def daily_store_dir() -> Path:
override = str(os.environ.get("GT_DAILY_STORE_DIR", "")).strip()
if override:
return Path(override)
return _DAILY_DIR
def utc_today() -> date:
return datetime.now(timezone.utc).date()
def date_id(when: date | datetime | None = None) -> str:
if when is None:
when = utc_today()
if isinstance(when, datetime):
when = when.date()
return when.isoformat()
@dataclass
class DailyRegionReading:
region: str
composite_risk: float
financial: float
unrest: float
conflict: float
peak_score: float
readings: int = 1
last_captured_at: str = ""
def to_dict(self) -> dict[str, Any]:
return asdict(self)
@classmethod
def from_dict(cls, raw: dict[str, Any]) -> DailyRegionReading:
return cls(
region=str(raw.get("region") or "").strip().lower(),
composite_risk=float(raw.get("composite_risk") or 0.0),
financial=float(raw.get("financial") or 0.0),
unrest=float(raw.get("unrest") or 0.0),
conflict=float(raw.get("conflict") or 0.0),
peak_score=float(raw.get("peak_score") or 0.0),
readings=int(raw.get("readings") or 1),
last_captured_at=str(raw.get("last_captured_at") or ""),
)
@dataclass
class DailySnapshot:
date: str
regions: dict[str, DailyRegionReading] = field(default_factory=dict)
last_updated_at: str = ""
def to_dict(self) -> dict[str, Any]:
return {
"date": self.date,
"last_updated_at": self.last_updated_at,
"regions": {key: row.to_dict() for key, row in self.regions.items()},
}
@classmethod
def from_dict(cls, raw: dict[str, Any]) -> DailySnapshot:
regions: dict[str, DailyRegionReading] = {}
for key, row in (raw.get("regions") or {}).items():
if isinstance(row, dict):
reading = DailyRegionReading.from_dict(row)
regions[str(key).strip().lower()] = reading
return cls(
date=str(raw.get("date") or ""),
regions=regions,
last_updated_at=str(raw.get("last_updated_at") or ""),
)
def _daily_path(day_id: str) -> Path:
safe = day_id.replace("/", "-").replace("..", "")
return daily_store_dir() / f"{safe}.json"
def _ensure_dir() -> None:
daily_store_dir().mkdir(parents=True, exist_ok=True)
def list_daily_ids(*, newest_first: bool = True, limit: int | None = None) -> list[str]:
_ensure_dir()
ids = sorted(
(path.stem for path in daily_store_dir().glob("*.json")),
reverse=newest_first,
)
if limit is not None:
return ids[:limit]
return ids
def load_daily(day: date | str | None = None) -> DailySnapshot | None:
day_id = date_id(day) if day is not None else date_id()
path = _daily_path(day_id)
if not path.is_file():
return None
try:
raw = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(raw, dict):
return None
return DailySnapshot.from_dict(raw)
except (OSError, json.JSONDecodeError, TypeError, ValueError):
logger.exception("Failed to load GT daily reading %s", day_id)
return None
def save_daily(snapshot: DailySnapshot) -> None:
_ensure_dir()
path = _daily_path(snapshot.date)
tmp = path.with_suffix(".json.tmp")
payload = json.dumps(snapshot.to_dict(), indent=2, sort_keys=True)
with _store_lock:
tmp.write_text(payload, encoding="utf-8")
tmp.replace(path)
def utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
backend/analytics/feed_adapter.py
+206
View File
@@ -0,0 +1,206 @@
"""Normalize Shadowbroker feed records into GT analytics feed items."""
from __future__ import annotations
import re
from typing import Any, Iterable
_DOMAIN_CONFLICT = "conflict"
_DOMAIN_UNREST = "unrest"
_DOMAIN_FINANCIAL = "financial"
_CONFLICT_HINTS = re.compile(
r"\b(war|missile|strike|attack|military|invasion|troop|shelling|drone|bomb|nuclear)\b",
re.I,
)
_UNREST_HINTS = re.compile(
r"\b(protest|rally|strike|riot|unrest|mobiliz|demonstrat|curfew|purge|coup)\b",
re.I,
)
_FINANCIAL_HINTS = re.compile(
r"\b(payroll|loan|default|bankruptcy|liquidity|sanction|supply\s+chain|delay|shortage)\b",
re.I,
)
def _clean_region(value: Any) -> str:
region = str(value or "").strip().lower()
return region or "global"
def _infer_domain(text: str, explicit: str | None = None) -> str:
if explicit in {_DOMAIN_CONFLICT, _DOMAIN_UNREST, _DOMAIN_FINANCIAL}:
return explicit
if _CONFLICT_HINTS.search(text):
return _DOMAIN_CONFLICT
if _UNREST_HINTS.search(text):
return _DOMAIN_UNREST
if _FINANCIAL_HINTS.search(text):
return _DOMAIN_FINANCIAL
return _DOMAIN_FINANCIAL
def _text_from_record(
record: dict[str, Any],
*,
prefer_translation: bool = False,
) -> str:
"""Build ingest text; prefer English translations for Telegram OSINT when set."""
if prefer_translation:
translated_parts = [
record.get("title_translated"),
record.get("description_translated"),
]
translated = "\n".join(
str(p).strip() for p in translated_parts if p and str(p).strip()
)
if translated:
return translated
parts = [
record.get("title"),
record.get("description"),
record.get("text"),
record.get("summary"),
]
return "\n".join(str(p).strip() for p in parts if p and str(p).strip())
_HASHTAG_REGION = re.compile(r"#([a-z][a-z0-9_-]{2,})", re.I)
def _region_from_hashtags(text: str) -> str | None:
"""Map common theater hashtags (#Ukraine) to dossier/heatmap region keys."""
for match in _HASHTAG_REGION.finditer(text or ""):
tag = match.group(1).lower()
if tag in {
"ukraine",
"russia",
"israel",
"iran",
"gaza",
"syria",
"taiwan",
"china",
"belfast",
"uk",
"usa",
}:
return tag
return None
def _region_from_record(record: dict[str, Any], *, text: str = "") -> str:
for key in ("geotag", "region", "country", "location"):
if record.get(key):
return _clean_region(record[key])
hashtag_region = _region_from_hashtags(text)
if hashtag_region:
return hashtag_region
coords = record.get("coords")
if isinstance(coords, (list, tuple)) and len(coords) >= 2:
try:
lat = float(coords[0])
lng = float(coords[1])
return f"{lat:.2f},{lng:.2f}"
except (TypeError, ValueError):
pass
return "global"
def _entities_from_record(record: dict[str, Any]) -> list[str]:
entities: list[str] = []
for key in ("entities", "tags", "keywords"):
raw = record.get(key)
if isinstance(raw, list):
entities.extend(str(v).strip() for v in raw if str(v).strip())
elif isinstance(raw, str) and raw.strip():
entities.extend(part.strip() for part in raw.split(",") if part.strip())
channel = str(record.get("channel") or "").strip()
if channel:
entities.append(f"channel:{channel}")
source = str(record.get("source") or "").strip()
if source:
entities.append(f"source:{source}")
return entities
def normalize_feed_item(record: dict[str, Any], *, source_type: str = "generic") -> dict[str, Any]:
"""Map a news/Telegram/GDELT record into the GT engine schema."""
prefer_translation = source_type == "telegram_osint"
text = _text_from_record(record, prefer_translation=prefer_translation)
if prefer_translation and not text.strip():
text = _text_from_record(record, prefer_translation=False)
region = _region_from_record(record, text=text)
domain = _infer_domain(text, record.get("domain"))
coords = record.get("coords")
lat = lng = None
if isinstance(coords, (list, tuple)) and len(coords) >= 2:
try:
lat = float(coords[0])
lng = float(coords[1])
except (TypeError, ValueError):
lat = lng = None
return {
"id": record.get("id") or record.get("link"),
"text": text,
"source": str(record.get("source") or source_type),
"source_type": source_type,
"region": region,
"domain": domain,
"entities": _entities_from_record(record),
"coords": [lat, lng] if lat is not None and lng is not None else None,
"published": record.get("published"),
"risk_score": record.get("risk_score"),
}
def iter_telegram_posts(payload: dict[str, Any] | None) -> Iterable[dict[str, Any]]:
from services.telegram_translate import apply_post_translation, telegram_translate_enabled
posts = list((payload or {}).get("posts") or [])
for post in posts:
if not isinstance(post, dict):
continue
if not (post.get("description") or post.get("title")):
continue
enriched = (
apply_post_translation(post)
if telegram_translate_enabled()
else post
)
yield normalize_feed_item(enriched, source_type="telegram_osint")
def iter_news_items(payload: list[dict[str, Any]] | None) -> Iterable[dict[str, Any]]:
for item in list(payload or []):
if not isinstance(item, dict):
continue
yield normalize_feed_item(item, source_type="news")
for article in list(item.get("articles") or []):
if isinstance(article, dict):
yield normalize_feed_item(article, source_type="news_cluster")
def iter_gdelt_features(payload: list[dict[str, Any]] | None) -> Iterable[dict[str, Any]]:
for feature in list(payload or []):
if not isinstance(feature, dict):
continue
props = dict(feature.get("properties") or {})
geometry = dict(feature.get("geometry") or {})
coords = None
if geometry.get("type") == "Point":
raw = geometry.get("coordinates")
if isinstance(raw, (list, tuple)) and len(raw) >= 2:
coords = [float(raw[1]), float(raw[0])]
record = {
"title": props.get("name") or props.get("title"),
"description": props.get("snippet") or props.get("description"),
"source": props.get("source") or "gdelt",
"coords": coords,
"published": props.get("date") or props.get("published"),
"region": props.get("location") or props.get("country"),
}
if record["title"] or record["description"]:
yield normalize_feed_item(record, source_type="gdelt")
backend/analytics/gt_alerts.py
+128
View File
@@ -0,0 +1,128 @@
"""Top strategic-risk alerts — ranked regions with map coordinates."""
from __future__ import annotations
from typing import Any
from analytics.integration import get_gt_engine
from analytics.settings import get_gt_settings
def _peak_score(props: dict[str, Any]) -> float:
composite = float(props.get("risk") or 0.0)
financial = float(props.get("financial") or 0.0)
unrest = float(props.get("unrest") or 0.0)
conflict = float(props.get("conflict") or 0.0)
return max(composite, financial, unrest, conflict)
def _valid_coords(coords: Any) -> tuple[float, float] | None:
if not isinstance(coords, (list, tuple)) or len(coords) < 2:
return None
try:
lng = float(coords[0])
lat = float(coords[1])
except (TypeError, ValueError):
return None
if not (-90.0 <= lat <= 90.0 and -180.0 <= lng <= 180.0):
return None
if abs(lat) < 0.001 and abs(lng) < 0.001:
return None
return lat, lng
def _region_label(region: str) -> str:
text = str(region or "").strip()
if not text:
return "unknown"
if "," in text:
parts = [piece.strip() for piece in text.split(",") if piece.strip()]
if len(parts) >= 2:
try:
lat = float(parts[0])
lng = float(parts[-1])
return f"{lat:.2f}°, {lng:.2f}°"
except ValueError:
pass
return text.replace("_", " ")
def parse_heatmap_alerts(
heatmap: dict[str, Any] | None,
*,
limit: int = 8,
) -> tuple[list[dict[str, Any]], int]:
"""Return ranked alerts and count of regions plottable on the map."""
features = (heatmap or {}).get("features") or []
rows: list[dict[str, Any]] = []
for feature in features:
if not isinstance(feature, dict):
continue
geometry = feature.get("geometry") or {}
coords = _valid_coords(geometry.get("coordinates"))
if coords is None:
continue
lat, lng = coords
props = feature.get("properties") or {}
region = str(props.get("region") or "").strip().lower()
if not region:
continue
score = _peak_score(props)
rows.append(
{
"region": region,
"region_label": _region_label(region),
"risk": round(float(props.get("risk") or 0.0), 4),
"financial": round(float(props.get("financial") or 0.0), 4),
"unrest": round(float(props.get("unrest") or 0.0), 4),
"conflict": round(float(props.get("conflict") or 0.0), 4),
"contagion": round(float(props.get("contagion") or 0.0), 4),
"score": round(score, 4),
"lat": lat,
"lng": lng,
"ignition": bool(props.get("micro_ignition")),
"risk_3d_avg": props.get("risk_3d_avg"),
"risk_delta": props.get("risk_delta"),
"updates": int(props.get("updates") or 0),
}
)
rows.sort(
key=lambda row: (
bool(row.get("ignition")),
float(row.get("risk_delta") or 0.0),
float(row.get("score") or 0.0),
),
reverse=True,
)
return rows[: max(1, limit)], len(rows)
def top_gt_alerts(*, limit: int = 8) -> dict[str, Any]:
"""Ranked top regions for API / OpenClaw."""
settings = get_gt_settings()
engine = get_gt_engine()
heatmap: dict[str, Any] = {"type": "FeatureCollection", "features": []}
engine_regions = 0
if engine is not None:
heatmap = engine.get_risk_heatmap()
with engine._lock: # noqa: SLF001 — intentional meta read
engine_regions = len(engine._regions)
alerts, plotted = parse_heatmap_alerts(heatmap, limit=limit)
tracked = len(heatmap.get("features") or [])
return {
"alerts": alerts,
"tracked_regions": tracked,
"engine_regions": engine_regions,
"plotted_regions": plotted,
"max_regions": settings.max_heatmap_features,
"note": (
"Layer count is tracked GT regions (cap "
f"{settings.max_heatmap_features}), not raw feed events. "
"Only regions with valid coordinates appear on the map."
),
}
backend/analytics/gt_early_warning.py
+593
View File
@@ -0,0 +1,593 @@
"""Game-theoretic early warning analytics with Bayesian updating and contagion graph."""
from __future__ import annotations
import logging
import re
import threading
from collections import defaultdict
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any, DefaultDict
import networkx as nx
import numpy as np
from analytics.settings import GTAnalyticsSettings, get_gt_settings
logger = logging.getLogger(__name__)
DomainName = str # financial | unrest | conflict
_DOMAINS: tuple[DomainName, ...] = ("financial", "unrest", "conflict")
_DEFAULT_LIKELIHOODS: dict[DomainName, dict[str, float]] = {
"financial": {"distress": 0.75, "normal": 0.25},
"unrest": {"distress": 0.82, "normal": 0.22},
"conflict": {"distress": 0.78, "normal": 0.18},
}
_DEFAULT_SIGNAL_WEIGHTS: dict[str, float] = {
"payroll_loan": 3.0,
"supply_delay": 2.2,
"elite_relocation": 2.8,
"purge": 3.5,
"protest_mobilize": 2.5,
"gps_jamming": 2.7,
"troop_movement": 3.0,
"bank_run": 3.2,
"sanctions_escalation": 2.4,
"ceasefire_break": 2.6,
}
# Costly-signal regex patterns (cheap talk filtered by absence of match).
_SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
"payroll_loan": [
re.compile(r"payroll\s+loan", re.I),
re.compile(r"merchant\s+cash\s+advance", re.I),
re.compile(r"working\s+capital\s+loan", re.I),
],
"supply_delay": [
re.compile(r"supply\s+(chain\s+)?delay", re.I),
re.compile(r"shipping\s+delay", re.I),
re.compile(r"logistics\s+backlog", re.I),
re.compile(r"port\s+congestion", re.I),
],
"elite_relocation": [
re.compile(r"elite\s+(asset\s+)?relocation", re.I),
re.compile(r"oligarch\s+jet", re.I),
re.compile(r"private\s+jet\s+exodus", re.I),
re.compile(r"capital\s+flight", re.I),
],
"purge": [
re.compile(r"\bpurge\b", re.I),
re.compile(r"political\s+purge", re.I),
re.compile(r"security\s+apparatus\s+reshuffle", re.I),
],
"protest_mobilize": [
re.compile(r"protest\s+mobil", re.I),
re.compile(r"mass\s+rally", re.I),
re.compile(r"general\s+strike", re.I),
re.compile(r"\bstrike\b", re.I),
re.compile(r"\brally\b", re.I),
],
"gps_jamming": [
re.compile(r"gps\s+jam", re.I),
re.compile(r"gnss\s+interference", re.I),
re.compile(r"spoofing\s+spike", re.I),
],
"troop_movement": [
re.compile(r"troop\s+movement", re.I),
re.compile(r"military\s+mobil", re.I),
re.compile(r"armored\s+convoy", re.I),
re.compile(r"troop\s+buildup", re.I),
],
"bank_run": [
re.compile(r"bank\s+run", re.I),
re.compile(r"deposit\s+flight", re.I),
re.compile(r"liquidity\s+crunch", re.I),
],
"sanctions_escalation": [
re.compile(r"sanctions?\s+escalat", re.I),
re.compile(r"new\s+sanctions?", re.I),
re.compile(r"export\s+controls?\s+tighten", re.I),
],
"ceasefire_break": [
re.compile(r"ceasefire\s+(broken|violated|collapse)", re.I),
re.compile(r"truce\s+end", re.I),
],
}
_SIGNAL_DOMAINS: dict[str, DomainName] = {
"payroll_loan": "financial",
"supply_delay": "financial",
"bank_run": "financial",
"sanctions_escalation": "financial",
"protest_mobilize": "unrest",
"purge": "unrest",
"elite_relocation": "financial",
"gps_jamming": "conflict",
"troop_movement": "conflict",
"ceasefire_break": "conflict",
}
@dataclass
class RegionState:
"""Per-region Bayesian beliefs and metadata."""
priors: dict[DomainName, float] = field(default_factory=lambda: defaultdict(float))
coords: list[float] | None = None
signal_volume: DefaultDict[str, float] = field(default_factory=lambda: defaultdict(float))
update_count: int = 0
@dataclass
class HistoryEntry:
timestamp: str
domain: DomainName
signals: dict[str, float]
strength: float
prior: float
posterior: float
source: str
deviation_score: float
class GT_EarlyWarning:
"""
Game-Theoretic Early Warning System with Bayesian updating.
Tracks distress probabilities per region/domain, classifies costly signals vs
cheap talk, and propagates risk through an entity interaction graph.
"""
def __init__(self, settings: GTAnalyticsSettings | None = None) -> None:
self.settings = settings or get_gt_settings()
self.G: nx.Graph = nx.Graph()
self._regions: dict[str, RegionState] = {}
self._history: dict[str, list[HistoryEntry]] = defaultdict(list)
self._seen_item_ids: set[str] = set()
self._lock = threading.RLock()
self.likelihoods = dict(_DEFAULT_LIKELIHOODS)
self.signal_weights = dict(_DEFAULT_SIGNAL_WEIGHTS)
self.signal_weights.update(self.settings.signal_weight_overrides)
self._base_prior = float(self.settings.base_prior)
def _utcnow(self) -> str:
return datetime.now(timezone.utc).isoformat()
def _region_state(self, region: str) -> RegionState:
key = str(region or "global").strip().lower() or "global"
if key not in self._regions:
state = RegionState()
for domain in _DOMAINS:
state.priors[domain] = self._base_prior
self._regions[key] = state
return self._regions[key]
def get_prior(self, region: str, domain: DomainName) -> float:
with self._lock:
return float(self._region_state(region).priors.get(domain, self._base_prior))
def set_prior(self, region: str, domain: DomainName, value: float) -> None:
with self._lock:
state = self._region_state(region)
state.priors[domain] = float(
np.clip(value, self.settings.min_prob, self.settings.max_prob)
)
def composite_risk(self, region: str) -> float:
"""Weighted composite across domains (conflict weighted highest)."""
weights = {"financial": 0.25, "unrest": 0.35, "conflict": 0.40}
with self._lock:
state = self._region_state(region)
total = 0.0
weight_sum = 0.0
for domain, weight in weights.items():
total += float(state.priors.get(domain, self._base_prior)) * weight
weight_sum += weight
return float(total / weight_sum) if weight_sum else self._base_prior
def classify_signals(self, text: str, source: str = "") -> dict[str, float]:
"""Return weighted costly-signal strengths detected in text."""
text_lower = (text or "").lower()
signals: dict[str, float] = {}
for signal_name, patterns in _SIGNAL_PATTERNS.items():
weight = float(self.signal_weights.get(signal_name, 1.0))
if any(pattern.search(text_lower) for pattern in patterns):
signals[signal_name] = weight
rally_strike_count = text_lower.count("rally") + text_lower.count("strike")
if rally_strike_count > 3:
signals["protest_mobilize"] = signals.get("protest_mobilize", 0.0) + 1.5
# Source credibility nudge (Telegram OSINT channels treated as moderate-cost signals).
if source and "t.me/" in source.lower() and signals:
for key in list(signals):
signals[key] = round(signals[key] * 1.05, 3)
return signals
def _deviation_score(self, region: str, domain: DomainName, strength: float) -> float:
"""Deviation from rolling regional norm — herding/coordination detector input."""
with self._lock:
state = self._region_state(region)
baseline = max(state.signal_volume[domain], 1.0)
state.signal_volume[domain] += strength
state.update_count += 1
return float(strength / baseline)
def bayesian_update(
self,
region: str,
domain: DomainName,
evidence_strength: float = 1.0,
) -> float:
"""
Bayesian update: P(distress|evidence) from likelihood table and prior.
evidence_strength scales how far belief moves toward the likelihood posterior.
"""
domain = domain if domain in _DOMAINS else "financial"
lik = self.likelihoods.get(domain, self.likelihoods["financial"])
with self._lock:
state = self._region_state(region)
prior = float(state.priors.get(domain, self._base_prior))
p_e_given_d = lik["distress"]
p_e_given_not_d = lik["normal"]
p_e = (p_e_given_d * prior) + (p_e_given_not_d * (1.0 - prior))
if p_e <= 0:
posterior = prior
else:
posterior = (p_e_given_d * prior) / p_e
scaled = prior + (posterior - prior) * float(evidence_strength)
clipped = float(np.clip(scaled, self.settings.min_prob, self.settings.max_prob))
state.priors[domain] = clipped
return clipped
def _update_graph(
self,
region: str,
entities: list[str],
strength: float,
coords: list[float] | None,
) -> None:
region_key = str(region or "global").strip().lower() or "global"
self.G.add_node(region_key, node_type="region", region=region_key)
if coords and len(coords) >= 2:
self.G.nodes[region_key]["coords"] = coords
for entity in entities:
entity_key = str(entity).strip()
if not entity_key:
continue
self.G.add_node(entity_key, node_type="entity", region=region_key)
self.G.add_edge(
region_key,
entity_key,
weight=float(strength),
timestamp=self._utcnow(),
)
for i, e1 in enumerate(entities):
for e2 in entities[i + 1 :]:
k1, k2 = str(e1).strip(), str(e2).strip()
if not k1 or not k2:
continue
self.G.add_edge(
k1,
k2,
weight=float(strength),
timestamp=self._utcnow(),
)
def process_feed_item(self, item: dict[str, Any]) -> dict[str, Any]:
"""Process one normalized feed item and update beliefs + contagion graph."""
region = str(item.get("region") or item.get("geotag") or "global").strip().lower()
text = str(item.get("text") or "")
source = str(item.get("source") or "unknown")
explicit_domain = str(item.get("domain") or "").strip().lower()
entities = list(item.get("entities") or [])
coords = item.get("coords")
item_id = str(item.get("id") or f"{source}|{hash(text)}")
if self.settings.watched_channels:
channel = ""
for entity in entities:
if str(entity).startswith("channel:"):
channel = str(entity).split(":", 1)[-1].lower()
break
if channel and channel not in {c.lower() for c in self.settings.watched_channels}:
return {
"region": region,
"skipped": True,
"reason": "channel_not_watched",
"risk_score": self.composite_risk(region),
"signals": {},
}
with self._lock:
if item_id and item_id in self._seen_item_ids:
return {
"region": region,
"skipped": True,
"reason": "duplicate",
"risk_score": self.composite_risk(region),
"signals": {},
}
if item_id:
self._seen_item_ids.add(item_id)
signals = self.classify_signals(text, source)
total_strength = float(sum(signals.values()))
if total_strength <= 0:
return {
"region": region,
"risk_score": self.composite_risk(region),
"signals": {},
"contagion_potential": self._get_contagion_score(region),
}
domains_touched: set[DomainName] = set()
if explicit_domain in _DOMAINS:
domains_touched.add(explicit_domain)
for signal_name in signals:
domains_touched.add(_SIGNAL_DOMAINS.get(signal_name, explicit_domain or "financial"))
if not domains_touched:
domains_touched.add("financial")
evidence_strength = min(
total_strength / max(self.settings.evidence_scale, 0.1),
self.settings.evidence_cap,
)
posteriors: dict[str, float] = {}
deviation = 0.0
for domain in domains_touched:
prior = self.get_prior(region, domain)
deviation = max(deviation, self._deviation_score(region, domain, total_strength))
posterior = self.bayesian_update(
region=region,
domain=domain,
evidence_strength=evidence_strength * (1.0 + 0.15 * deviation),
)
posteriors[domain] = posterior
if isinstance(coords, (list, tuple)) and len(coords) >= 2:
with self._lock:
state = self._region_state(region)
try:
state.coords = [float(coords[0]), float(coords[1])]
except (TypeError, ValueError):
pass
self._update_graph(region, entities, total_strength, coords if isinstance(coords, list) else None)
composite = self.composite_risk(region)
entry = HistoryEntry(
timestamp=self._utcnow(),
domain=explicit_domain if explicit_domain in _DOMAINS else next(iter(domains_touched)),
signals=signals,
strength=total_strength,
prior=self._base_prior,
posterior=composite,
source=source,
deviation_score=deviation,
)
with self._lock:
history = self._history[region]
history.append(entry)
max_hist = max(10, int(self.settings.max_history_per_region))
if len(history) > max_hist:
self._history[region] = history[-max_hist:]
logger.info(
"GT update region=%s domains=%s composite=%.3f signals=%d deviation=%.2f",
region,
",".join(sorted(domains_touched)),
composite,
len(signals),
deviation,
)
return {
"region": region,
"domains": sorted(domains_touched),
"domain_posteriors": posteriors,
"risk_score": composite,
"signals": signals,
"deviation_score": deviation,
"contagion_potential": self._get_contagion_score(region),
"interpretation": self._interpret_risk(composite),
}
def _interpret_risk(self, risk: float) -> str:
threshold = float(self.settings.high_risk_threshold)
if risk >= threshold:
return (
f"Elevated strategic risk ({risk:.2f} ≥ {threshold:.2f}). "
"Watch for costly-signal clustering and cross-region contagion."
)
if risk >= threshold * 0.7:
return "Moderate risk — monitor for herding and repeated costly signals."
return "Baseline risk — no strong costly-signal cluster detected."
def _get_contagion_score(self, region: str) -> float:
"""Graph-based contagion: mean composite risk of graph neighbors."""
region_key = str(region or "global").strip().lower() or "global"
with self._lock:
if region_key not in self.G:
return 0.0
try:
neighbors = list(self.G.neighbors(region_key))
except nx.NetworkXError:
return 0.0
if not neighbors:
return 0.0
neighbor_risks = [self.composite_risk(str(n)) for n in neighbors]
return float(np.mean(neighbor_risks))
def compute_herding_clusters(self) -> list[dict[str, Any]]:
"""Louvain community detection on entity graph (coordination/herding proxy)."""
with self._lock:
if self.G.number_of_edges() == 0:
return []
weighted = nx.Graph()
for u, v, data in self.G.edges(data=True):
weight = float(data.get("weight") or 0.0)
if weight < self.settings.louvain_min_weight:
continue
if weighted.has_edge(u, v):
weighted[u][v]["weight"] = weighted[u][v].get("weight", 0.0) + weight
else:
weighted.add_edge(u, v, weight=weight)
if weighted.number_of_edges() == 0:
return []
try:
communities = list(nx.community.louvain_communities(weighted, weight="weight", seed=42))
except Exception as exc:
logger.warning("Louvain clustering failed: %s", exc)
return []
clusters: list[dict[str, Any]] = []
for idx, community in enumerate(communities):
members = sorted(str(node) for node in community)
region_members = [m for m in members if m in self._regions]
risks = [self.composite_risk(r) for r in region_members]
clusters.append(
{
"cluster_id": idx,
"size": len(members),
"members": members[:50],
"mean_risk": float(np.mean(risks)) if risks else self._base_prior,
"regions": region_members,
}
)
clusters.sort(key=lambda row: row["mean_risk"], reverse=True)
return clusters
def get_risk_heatmap(self) -> dict[str, Any]:
"""GeoJSON FeatureCollection for frontend risk overlay."""
features: list[dict[str, Any]] = []
with self._lock:
items = list(self._regions.items())[: max(1, self.settings.max_heatmap_features)]
for region, state in items:
coords = state.coords
geometry: dict[str, Any]
if coords and len(coords) >= 2:
geometry = {"type": "Point", "coordinates": [float(coords[1]), float(coords[0])]}
else:
geometry = {"type": "Point", "coordinates": [0.0, 0.0]}
composite = self.composite_risk(region)
features.append(
{
"type": "Feature",
"properties": {
"region": region,
"risk": round(composite, 4),
"financial": round(float(state.priors.get("financial", self._base_prior)), 4),
"unrest": round(float(state.priors.get("unrest", self._base_prior)), 4),
"conflict": round(float(state.priors.get("conflict", self._base_prior)), 4),
"contagion": round(self._get_contagion_score(region), 4),
"updates": state.update_count,
},
"geometry": geometry,
}
)
return {"type": "FeatureCollection", "features": features}
def get_dossier(self, region: str) -> dict[str, Any]:
"""Explainable GT rationale and recent signal history for a region."""
region_key = str(region or "global").strip().lower() or "global"
with self._lock:
state = self._region_state(region_key)
recent = list(self._history.get(region_key, [])[-10:])
composite = self.composite_risk(region_key)
return {
"region": region_key,
"current_risk": round(composite, 4),
"domain_risks": {
domain: round(float(state.priors.get(domain, self._base_prior)), 4)
for domain in _DOMAINS
},
"recent_signals": [
{
"timestamp": entry.timestamp,
"domain": entry.domain,
"signals": entry.signals,
"strength": entry.strength,
"posterior": round(entry.posterior, 4),
"source": entry.source,
"deviation_score": round(entry.deviation_score, 3),
}
for entry in recent
],
"contagion_risk": round(self._get_contagion_score(region_key), 4),
"herding_clusters": self.compute_herding_clusters()[:5],
"interpretation": self._interpret_risk(composite),
"scenarios": self._build_scenarios(region_key, composite),
}
def _build_scenarios(self, region: str, composite: float) -> list[dict[str, str]]:
threshold = float(self.settings.high_risk_threshold)
if composite < threshold * 0.7:
return [
{
"name": "Status quo",
"summary": "Signals remain diffuse; no coordinated costly-signal cascade.",
}
]
if composite < threshold:
return [
{
"name": "Escalation watch",
"summary": "Rising costly-signal density — coordination risk within 4-8 weeks.",
},
{
"name": "False alarm",
"summary": "Cheap-talk amplification without follow-on costly signals.",
},
]
return [
{
"name": "Contagion spread",
"summary": "High posterior + graph coupling — adjacent regions likely to update upward.",
},
{
"name": "Localized shock",
"summary": "Region-specific distress; contagion limited if graph neighbors stay quiet.",
},
]
def snapshot(self) -> dict[str, Any]:
"""Serialize engine state for debugging or persistence."""
with self._lock:
return {
"regions": {
region: {
"priors": dict(state.priors),
"coords": state.coords,
"updates": state.update_count,
}
for region, state in self._regions.items()
},
"graph_nodes": self.G.number_of_nodes(),
"graph_edges": self.G.number_of_edges(),
"processed_items": len(self._seen_item_ids),
}
backend/analytics/historical_events.py
+649
View File
@@ -0,0 +1,649 @@
"""Curated historical early-warning cases for GT backtesting.
Each positive case bundles pre-crisis costly-signal snippets drawn from documented
precursors (financial, unrest, conflict). Negative cases are cheap-talk controls.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Literal
CaseKind = Literal["positive", "negative"]
@dataclass(frozen=True)
class BacktestFeed:
text: str
source: str = "backtest"
domain: str = "financial"
days_before_event: int = 30
@dataclass(frozen=True)
class HistoricalCase:
"""Single labeled backtest scenario."""
case_id: str
name: str
region: str
domain: str
kind: CaseKind
event_date: str
description: str
feeds: tuple[BacktestFeed, ...] = field(default_factory=tuple)
tags: tuple[str, ...] = field(default_factory=tuple)
def to_feed_dicts(self) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
for idx, feed in enumerate(self.feeds):
items.append(
{
"id": f"{self.case_id}-{idx}",
"text": feed.text,
"source": feed.source,
"region": self.region,
"domain": feed.domain or self.domain,
"published": feed.days_before_event,
}
)
return items
def _variant_case(case: HistoricalCase, suffix: str, feeds: tuple[BacktestFeed, ...]) -> HistoricalCase:
return HistoricalCase(
case_id=f"{case.case_id}__{suffix}",
name=f"{case.name} ({suffix})",
region=case.region,
domain=case.domain,
kind=case.kind,
event_date=case.event_date,
description=case.description,
feeds=feeds,
tags=case.tags + (f"variant:{suffix}",),
)
def expanded_historical_cases() -> tuple[HistoricalCase, ...]:
"""Base suite plus paraphrase variants for statistical confidence."""
base = list(default_historical_cases())
extras: list[HistoricalCase] = []
variant_feeds: dict[str, tuple[tuple[BacktestFeed, ...], ...]] = {
"fin_2008_us": (
(
BacktestFeed(
"Small businesses turn to payroll loan products as credit lines freeze.",
domain="financial",
days_before_event=100,
),
BacktestFeed(
"FDIC monitors liquidity crunch; interbank spreads widen sharply.",
domain="financial",
days_before_event=60,
),
),
(
BacktestFeed(
"Merchant cash advance volumes spike; payroll loan demand at record highs.",
domain="financial",
days_before_event=80,
),
BacktestFeed(
"Money market funds see inflows as deposit flight from regional banks continues.",
domain="financial",
days_before_event=40,
),
),
),
"fin_2020_supply": (
(
BacktestFeed(
"Electronics firms report shipping delay and port congestion across Pearl River Delta.",
domain="financial",
days_before_event=45,
),
BacktestFeed(
"Supply chain delay widens; logistics backlog hits automotive suppliers.",
domain="financial",
days_before_event=20,
),
),
(
BacktestFeed(
"Container shortage fuels shipping delay; supply chain delay indices jump.",
domain="financial",
days_before_event=35,
),
BacktestFeed(
"Electronics assemblers warn of logistics backlog as port congestion spreads.",
domain="financial",
days_before_event=20,
),
BacktestFeed(
"Automotive suppliers flag supply chain delay after factory shutdowns in Hubei.",
domain="financial",
days_before_event=10,
),
),
),
"fin_2022_sanctions": (
(
BacktestFeed(
"Treasury drafts new sanctions escalation package on energy and finance sectors.",
domain="financial",
days_before_event=30,
),
BacktestFeed(
"Capital flight accelerates; elite relocation flights depart Moscow airports.",
domain="financial",
days_before_event=14,
),
),
),
"unrest_arab_spring_egypt": (
(
BacktestFeed(
"Cairo activists schedule mass rally; protest mobilization leaflets distributed.",
domain="unrest",
days_before_event=18,
),
BacktestFeed(
"Labor federations call general strike; strike posters cover downtown.",
domain="unrest",
days_before_event=8,
),
),
),
"conflict_2022_ukraine": (
(
BacktestFeed(
"Convoy of armored vehicles confirms troop movement near Sumy Oblast.",
source="t.me/war_monitor",
domain="conflict",
days_before_event=20,
),
BacktestFeed(
"GNSS interference warnings follow GPS jamming spike along Belarus border.",
source="t.me/osintdefender",
domain="conflict",
days_before_event=10,
),
),
(
BacktestFeed(
"Military mobilization notices circulate; troop buildup confirmed by satellite firms.",
domain="conflict",
days_before_event=12,
),
),
),
"neg_weather_us": (
(
BacktestFeed("Autumn foliage peaks in Vermont; pleasant hiking weather continues."),
BacktestFeed("County fair announces pie contest and livestock exhibitions."),
),
(
BacktestFeed("Meteorologists predict mild hurricane season remainder for Gulf Coast."),
),
),
"neg_sports_uk": (
(
BacktestFeed("Rugby Six Nations standings update after weekend fixtures."),
BacktestFeed("Local marathon registration opens for charity runners."),
),
),
"neg_tech_global": (
(
BacktestFeed("Chipmaker announces efficiency gains in next-generation processor."),
BacktestFeed("Cloud provider opens new green datacenter in Nordic region."),
),
),
}
for case in base:
variants = variant_feeds.get(case.case_id, ())
for idx, feeds in enumerate(variants):
extras.append(_variant_case(case, f"v{idx+1}", feeds))
# Additional cheap-talk controls to widen negative sample
cheap_talk_regions = (
("australia", "Museum opens contemporary art exhibit to strong attendance."),
("spain", "Tomato harvest festival scheduled; regional trains add weekend service."),
("south_korea", "K-pop group announces world tour dates for autumn."),
("mexico", "Coastal cleanup volunteers restore beach habitats before holiday season."),
("sweden", "City council approves bike lane expansion along waterfront."),
("norway", "Salmon exports remain stable; fishing fleets report normal catch volumes."),
("italy", "Truffle festival returns; restaurants publish seasonal tasting menus."),
("poland", "University researchers release open-source astronomy software."),
("thailand", "Monsoon rains ease; rice planting proceeds on normal schedule."),
("vietnam", "Electronics assembly plants report steady export order books."),
("south_africa", "Wildlife reserve reports rising ecotourism bookings."),
("argentina", "Wine harvest festival opens; export cooperatives meet volume targets."),
("netherlands", "Cycling championship draws international teams to canal district."),
("belgium", "Chocolate exporters report stable holiday shipment schedules."),
("portugal", "Offshore wind auction attracts multiple renewable bidders."),
("greece", "Island ferry operators add routes ahead of summer travel season."),
("turkey", "Cotton harvest forecast unchanged; textile orders stable."),
("indonesia", "Volcano monitoring reports routine activity; tourism continues."),
("philippines", "Coconut processors report normal logistics to export markets."),
("malaysia", "Palm oil shipments on schedule; port throughput normal."),
("new_zealand", "Sheep shearing competition draws rural crowds."),
("ireland", "Tech conference highlights open-source database tooling."),
("finland", "Sauna culture festival celebrates heritage with local artisans."),
("denmark", "Wind turbine maintenance contracts renewed on prior terms."),
("austria", "Ski resorts prepare slopes after early snowfall."),
("switzerland", "Watchmakers unveil mechanical movement prototypes at trade fair."),
("czech_republic", "Glassmakers export decorative pieces ahead of holiday season."),
("romania", "Carpathian hiking trails reopen after spring maintenance."),
("hungary", "Thermal bath tourism bookings rise for winter wellness season."),
("peru", "Coffee cooperatives report stable harvest and export schedules."),
("colombia", "Flower exporters prepare Valentine's shipments on normal cadence."),
("morocco", "Citrus harvest meets forecasts; agricultural credit unchanged."),
("kenya", "Tea auction volumes steady; freight routes operate normally."),
("nigeria", "Nollywood studio announces family comedy release dates."),
("ethiopia", "Coffee ceremony festival highlights regional bean varieties."),
("saudi_arabia", "Desert conservation project plants drought-resistant shrubs."),
("uae", "Airport duty-free operators expand luxury retail concourse."),
("qatar", "Stadium operators prepare hospitality packages for sporting events."),
("singapore", "Port authority reports container throughput on seasonal trend."),
("hong_kong", "Art auction previews draw collectors to harborfront gallery."),
("chile", "Vineyard tours report strong bookings ahead of harvest festival weekend."),
("uruguay", "Beef exporters maintain steady shipment schedules to European buyers."),
("iceland", "Geothermal spa resorts report normal winter visitor volumes."),
("luxembourg", "Fund administrators publish routine quarterly disclosure filings."),
("slovakia", "Mountain lodges prepare ski season openings after early snowfall."),
("croatia", "Adriatic ferry operators add summer routes on prior timetable."),
("bulgaria", "Rose oil cooperatives report stable export volumes to fragrance buyers."),
("serbia", "Danube barge traffic proceeds on normal freight schedules."),
("latvia", "Timber mills export lumber on unchanged contract terms."),
("lithuania", "Baltic wind farms complete scheduled turbine maintenance rotations."),
("estonia", "Digital residency applications processed at routine monthly pace."),
("panama", "Canal transit volumes remain on seasonal trend; shipping fees unchanged."),
)
for idx, (region, text) in enumerate(cheap_talk_regions):
extras.append(
HistoricalCase(
case_id=f"neg_extra_{idx:02d}",
name=f"Benign regional news ({region})",
region=region,
domain="financial",
kind="negative",
event_date="2020-01-01",
description="Expanded cheap-talk control.",
feeds=(BacktestFeed(text),),
tags=("control", "expanded"),
)
)
return tuple(base + extras)
def default_historical_cases() -> tuple[HistoricalCase, ...]:
"""Benchmark suite — expand as new validated precursors are added."""
return (
# ── Financial distress ─────────────────────────────────────────────
HistoricalCase(
case_id="fin_2008_us",
name="2008 US financial crisis",
region="united_states",
domain="financial",
kind="positive",
event_date="2008-09-15",
description="Payroll-loan distress, liquidity crunch, and deposit flight precursors.",
tags=("2008", "financial", "lehman"),
feeds=(
BacktestFeed(
"Franchise operators increasingly rely on payroll loan facilities as working capital tightens.",
domain="financial",
days_before_event=120,
),
BacktestFeed(
"Regional banks report liquidity crunch; CFOs warn of merchant cash advance reliance.",
domain="financial",
days_before_event=90,
),
BacktestFeed(
"Deposit flight accelerates at mid-size lenders; analysts flag bank run risk.",
domain="financial",
days_before_event=45,
),
),
),
HistoricalCase(
case_id="fin_2020_supply",
name="COVID supply-chain shock",
region="china",
domain="financial",
kind="positive",
event_date="2020-02-01",
description="Port congestion and logistics backlog ahead of global supply shock.",
tags=("covid", "supply_chain", "financial"),
feeds=(
BacktestFeed(
"Major port congestion reported; shipping delay spreads to electronics suppliers.",
domain="financial",
days_before_event=60,
),
BacktestFeed(
"Automakers warn of supply chain delay and logistics backlog across Wuhan corridor.",
domain="financial",
days_before_event=30,
),
BacktestFeed(
"Factory restarts slip as supply delay and port congestion persist into Q1.",
domain="financial",
days_before_event=14,
),
),
),
HistoricalCase(
case_id="fin_2022_sanctions",
name="Russia sanctions escalation",
region="russia",
domain="financial",
kind="positive",
event_date="2022-02-24",
description="Sanctions escalation and capital flight ahead of invasion.",
tags=("sanctions", "ukraine", "financial"),
feeds=(
BacktestFeed(
"Western allies prepare new sanctions escalation on major Russian banks.",
domain="financial",
days_before_event=45,
),
BacktestFeed(
"Oligarch jet movements suggest elite relocation and capital flight from Moscow.",
domain="financial",
days_before_event=21,
),
BacktestFeed(
"Central bank intervenes as new sanctions tighten export controls on finance sector.",
domain="financial",
days_before_event=10,
),
),
),
# ── Civil unrest ─────────────────────────────────────────────────
HistoricalCase(
case_id="unrest_arab_spring_tunisia",
name="Arab Spring — Tunisia",
region="tunisia",
domain="unrest",
kind="positive",
event_date="2010-12-17",
description="Protest mobilization and strike waves before Jasmine Revolution.",
tags=("arab_spring", "unrest"),
feeds=(
BacktestFeed(
"Student groups announce protest mobilization after vendor self-immolation.",
domain="unrest",
days_before_event=14,
),
BacktestFeed(
"Mass rally planned in Tunis; general strike called by labor unions.",
domain="unrest",
days_before_event=7,
),
),
),
HistoricalCase(
case_id="unrest_arab_spring_egypt",
name="Arab Spring — Egypt",
region="egypt",
domain="unrest",
kind="positive",
event_date="2011-01-25",
description="Mobilization spikes and security reshuffles before Tahrir.",
tags=("arab_spring", "unrest"),
feeds=(
BacktestFeed(
"Opposition calls protest mobilization in Cairo; strike notices circulate online.",
domain="unrest",
days_before_event=21,
),
BacktestFeed(
"Reports of political purge within interior ministry security apparatus reshuffle.",
domain="unrest",
days_before_event=10,
),
BacktestFeed(
"Mass rally and strike coordination spreads; rally posters appear in Alexandria.",
domain="unrest",
days_before_event=5,
),
),
),
HistoricalCase(
case_id="unrest_2019_chile",
name="Chile 2019 metro protests",
region="chile",
domain="unrest",
kind="positive",
event_date="2019-10-18",
description="Transit fare protests escalate to general strike.",
tags=("unrest", "latam"),
feeds=(
BacktestFeed(
"Students organize mass rally after metro fare hike; protest mobilization trending.",
domain="unrest",
days_before_event=10,
),
BacktestFeed(
"Unions announce general strike; rally and strike hashtags spike nationwide.",
domain="unrest",
days_before_event=3,
),
),
),
# ── Conflict / war ───────────────────────────────────────────────
HistoricalCase(
case_id="conflict_2022_ukraine",
name="2022 Ukraine invasion buildup",
region="ukraine",
domain="conflict",
kind="positive",
event_date="2022-02-24",
description="Troop movement and GPS jamming precursors on northern border.",
tags=("ukraine", "conflict"),
feeds=(
BacktestFeed(
"OSINT reports troop movement and armored convoy near Belarus border.",
source="t.me/war_monitor",
domain="conflict",
days_before_event=30,
),
BacktestFeed(
"GPS jamming spike reported along northern corridor; GNSS interference warnings issued.",
source="t.me/osintdefender",
domain="conflict",
days_before_event=14,
),
BacktestFeed(
"Satellite imagery shows troop buildup; military mobilization near Kharkiv axis.",
domain="conflict",
days_before_event=7,
),
),
),
HistoricalCase(
case_id="conflict_2023_gaza",
name="2023 Gaza conflict escalation",
region="israel",
domain="conflict",
kind="positive",
event_date="2023-10-07",
description="Ceasefire breakdown and troop movement signals.",
tags=("gaza", "conflict"),
feeds=(
BacktestFeed(
"Border units report troop movement near Gaza envelope; ceasefire broken overnight.",
domain="conflict",
days_before_event=14,
),
BacktestFeed(
"Truce end announced; armored convoy repositioning reported by local observers.",
domain="conflict",
days_before_event=5,
),
),
),
HistoricalCase(
case_id="conflict_2020_nagorno",
name="2020 Nagorno-Karabakh renewal",
region="armenia",
domain="conflict",
kind="positive",
event_date="2020-09-27",
description="Artillery and troop buildup precursors.",
tags=("caucasus", "conflict"),
feeds=(
BacktestFeed(
"Drone strikes reported on line of contact; troop movement on Armenian-Azeri border.",
domain="conflict",
days_before_event=21,
),
BacktestFeed(
"GPS jamming spike reported in conflict zone; military mobilization notices leaked.",
domain="conflict",
days_before_event=7,
),
),
),
# ── Recent financial / corporate distress pattern ────────────────
HistoricalCase(
case_id="fin_2023_banking",
name="2023 regional banking stress",
region="united_states",
domain="financial",
kind="positive",
event_date="2023-03-10",
description="Deposit flight and liquidity stress (SVB precursor pattern).",
tags=("svb", "financial", "2023"),
feeds=(
BacktestFeed(
"Tech lenders face deposit flight; VC portfolio companies move payroll to money market funds.",
domain="financial",
days_before_event=21,
),
BacktestFeed(
"Analysts warn liquidity crunch at regional banks holding long-duration bonds.",
domain="financial",
days_before_event=7,
),
),
),
# ── Negative controls (cheap talk / benign) ─────────────────────
HistoricalCase(
case_id="neg_weather_us",
name="Benign weather coverage",
region="united_states",
domain="financial",
kind="negative",
event_date="2019-06-01",
description="No costly signals — should remain near baseline.",
tags=("control",),
feeds=(
BacktestFeed("Sunny weekend expected across the Midwest with mild temperatures."),
BacktestFeed("Local festival draws crowds; farmers market expands summer hours."),
),
),
HistoricalCase(
case_id="neg_sports_uk",
name="Benign sports coverage",
region="uk",
domain="unrest",
kind="negative",
event_date="2018-07-01",
description="Sports chatter without mobilization costly signals.",
tags=("control",),
feeds=(
BacktestFeed("Premier league season review: top scorers and transfer rumors."),
BacktestFeed("Cricket test match ends early due to rain delay at Lord's."),
),
),
HistoricalCase(
case_id="neg_tech_global",
name="Benign tech product launch",
region="global",
domain="financial",
kind="negative",
event_date="2021-09-01",
description="Corporate product news without distress markers.",
tags=("control",),
feeds=(
BacktestFeed("Smartphone maker unveils new camera features at annual keynote."),
BacktestFeed("Quarterly earnings beat expectations; dividend unchanged."),
),
),
HistoricalCase(
case_id="neg_tourism_france",
name="Benign tourism recovery",
region="france",
domain="unrest",
kind="negative",
event_date="2022-08-01",
description="Travel sector recovery without unrest signals.",
tags=("control",),
feeds=(
BacktestFeed("Paris hotels report record summer bookings as tourism rebounds."),
BacktestFeed("Airline adds routes to Nice and Marseille for holiday travelers."),
),
),
HistoricalCase(
case_id="neg_science_japan",
name="Benign science news",
region="japan",
domain="conflict",
kind="negative",
event_date="2020-11-01",
description="Research coverage without conflict markers.",
tags=("control",),
feeds=(
BacktestFeed("Astronomy team publishes comet observations from Mount Fuji observatory."),
BacktestFeed("Robotics lab demonstrates warehouse automation prototype."),
),
),
HistoricalCase(
case_id="neg_agriculture_brazil",
name="Benign agriculture report",
region="brazil",
domain="financial",
kind="negative",
event_date="2017-03-01",
description="Commodity harvest update without supply distress.",
tags=("control",),
feeds=(
BacktestFeed("Soybean harvest forecast revised upward; export volumes steady."),
BacktestFeed("Coffee cooperative reports normal shipping schedules to European buyers."),
),
),
HistoricalCase(
case_id="neg_culture_india",
name="Benign culture coverage",
region="india",
domain="unrest",
kind="negative",
event_date="2016-11-01",
description="Festival coverage without mobilization.",
tags=("control",),
feeds=(
BacktestFeed("Diwali celebrations begin; cities decorate markets with lights."),
BacktestFeed("Film festival opens in Mumbai with premiere screenings."),
),
),
HistoricalCase(
case_id="neg_infrastructure_canada",
name="Benign infrastructure ribbon-cutting",
region="canada",
domain="financial",
kind="negative",
event_date="2015-05-01",
description="Municipal news without financial stress.",
tags=("control",),
feeds=(
BacktestFeed("New light-rail segment opens on schedule; commute times improve."),
BacktestFeed("Municipal bond issuance funds library renovation at prior rates."),
),
),
)
backend/analytics/integration.py
+198
View File
@@ -0,0 +1,198 @@
"""Singleton GT engine and feed-batch integration hooks."""
from __future__ import annotations
import logging
import threading
from datetime import datetime, timezone
from typing import Any
from analytics.feed_adapter import iter_gdelt_features, iter_news_items, iter_telegram_posts
from analytics.gt_early_warning import GT_EarlyWarning
from analytics.settings import gt_analytics_enabled, get_gt_settings, gt_engine_operational, gt_louvain_enabled, gt_scheduled_ingest_enabled
from services.fetchers._store import _data_lock, _mark_fresh, latest_data
logger = logging.getLogger(__name__)
_engine: GT_EarlyWarning | None = None
_engine_lock = threading.Lock()
def get_gt_engine() -> GT_EarlyWarning | None:
"""Return the shared engine when analytics are enabled and runtime allows it."""
global _engine
if not gt_engine_operational():
return None
with _engine_lock:
if _engine is None:
_engine = GT_EarlyWarning(get_gt_settings())
logger.info("Strategic Risk Analytics engine initialized")
return _engine
def reset_gt_engine() -> None:
"""Reset singleton — intended for tests."""
global _engine
get_gt_settings.cache_clear()
with _engine_lock:
_engine = None
def process_feed_item(item: dict[str, Any]) -> dict[str, Any] | None:
"""Process a normalized feed item if analytics are enabled."""
engine = get_gt_engine()
if engine is None:
return None
try:
return engine.process_feed_item(item)
except Exception:
logger.exception("GT process_feed_item failed")
return None
def _persist_gt_snapshot(
engine: GT_EarlyWarning,
*,
processed: int,
sample: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
timestamp = datetime.now(timezone.utc).isoformat()
heatmap = engine.get_risk_heatmap()
micro_summary: dict[str, Any] = {}
try:
from analytics.micro_rolling import capture_daily_readings, enrich_heatmap_features
micro_summary = capture_daily_readings(engine)
heatmap = enrich_heatmap_features(heatmap)
except Exception:
logger.exception("GT micro rolling capture failed")
clusters = engine.compute_herding_clusters()
from analytics.gt_alerts import parse_heatmap_alerts
_, plotted_regions = parse_heatmap_alerts(heatmap)
with engine._lock: # noqa: SLF001 — snapshot meta
engine_regions = len(engine._regions)
settings = get_gt_settings()
payload = {
"enabled": True,
"timestamp": timestamp,
"processed": processed,
"heatmap": heatmap,
"clusters": clusters,
"sample": list(sample or [])[:5],
"regions": len(heatmap.get("features") or []),
"micro": micro_summary,
"meta": {
"tracked_regions": len(heatmap.get("features") or []),
"engine_regions": engine_regions,
"plotted_regions": plotted_regions,
"max_regions": settings.max_heatmap_features,
},
}
with _data_lock:
latest_data["gt_risk"] = payload
_mark_fresh("gt_risk")
return payload
def refresh_from_latest_data(
data_snapshot: dict[str, Any],
*,
persist: bool = True,
) -> dict[str, Any]:
"""
Batch-ingest recent intel layers from the shared data store.
Intended to run after telegram/news/gdelt fetch cycles (near-real-time).
"""
engine = get_gt_engine()
if engine is None:
return {"enabled": False, "processed": 0}
processed = 0
results: list[dict[str, Any]] = []
for item in iter_telegram_posts(data_snapshot.get("telegram_osint")):
result = engine.process_feed_item(item)
if result and not result.get("skipped"):
processed += 1
results.append(result)
for item in iter_news_items(data_snapshot.get("news")):
result = engine.process_feed_item(item)
if result and not result.get("skipped"):
processed += 1
if len(results) < 5:
results.append(result)
for item in iter_gdelt_features(data_snapshot.get("gdelt")):
result = engine.process_feed_item(item)
if result and not result.get("skipped"):
processed += 1
logger.info("GT refresh processed %d items", processed)
summary = {
"enabled": True,
"processed": processed,
"sample": results[:5],
"heatmap_features": len(engine.get_risk_heatmap().get("features") or []),
}
if persist:
snapshot = _persist_gt_snapshot(engine, processed=processed, sample=results)
summary["timestamp"] = snapshot.get("timestamp")
summary["clusters"] = len(snapshot.get("clusters") or [])
return summary
def recompute_gt_herding_clusters() -> dict[str, Any]:
"""Louvain community pass — run on a schedule independent of feed ingest."""
if not gt_louvain_enabled():
return {"enabled": False, "clusters": 0, "reason": "louvain_disabled_on_lean_profile"}
engine = get_gt_engine()
if engine is None:
return {"enabled": False, "clusters": 0}
clusters = engine.compute_herding_clusters()
timestamp = datetime.now(timezone.utc).isoformat()
with _data_lock:
current = dict(latest_data.get("gt_risk") or {})
current["clusters"] = clusters
current["clusters_updated"] = timestamp
current["enabled"] = True
latest_data["gt_risk"] = current
_mark_fresh("gt_risk")
logger.info("GT Louvain recompute: %d clusters", len(clusters))
return {"enabled": True, "clusters": len(clusters), "timestamp": timestamp}
def maybe_refresh_gt_analytics() -> None:
"""Hook for data_fetcher — no-op when analytics are disabled or lean-gated."""
if not gt_scheduled_ingest_enabled():
return
try:
with _data_lock:
snapshot = dict(latest_data)
refresh_from_latest_data(snapshot, persist=True)
except Exception:
logger.exception("GT analytics refresh failed")
def maybe_freeze_gt_weekly_snapshot() -> None:
"""Hook for weekly scheduler — freeze operational backtest snapshot."""
if not gt_engine_operational():
return
try:
from analytics.rolling_backtest import freeze_weekly_snapshot
result = freeze_weekly_snapshot(frozen_by="scheduler")
if result.get("created"):
logger.info(
"GT rolling freeze: week=%s regions=%s alerts=%s",
result.get("week_id"),
result.get("region_count"),
result.get("alert_count"),
)
except Exception:
logger.exception("GT rolling weekly freeze failed")
backend/analytics/micro_rolling.py
+361
View File
@@ -0,0 +1,361 @@
"""Micro rolling 3-day average — fast ignition signal alongside weekly macro."""
from __future__ import annotations
import os
from dataclasses import dataclass
from datetime import date, datetime, timedelta, timezone
from typing import Any
from analytics.daily_store import (
DailyRegionReading,
DailySnapshot,
date_id,
list_daily_ids,
load_daily,
save_daily,
utc_now_iso,
utc_today,
)
from analytics.gt_early_warning import GT_EarlyWarning
from analytics.rolling_backtest import rolling_alert_threshold
DEFAULT_WINDOW_DAYS = 3
DEFAULT_IGNITION_DELTA = 0.10
def _env_int(name: str, default: int) -> int:
raw = str(os.environ.get(name, "")).strip()
if not raw:
return default
try:
return max(1, int(raw))
except ValueError:
return default
def _env_float(name: str, default: float) -> float:
raw = str(os.environ.get(name, "")).strip()
if not raw:
return default
try:
return float(raw)
except ValueError:
return default
def micro_window_days() -> int:
return _env_int("GT_MICRO_ROLLING_DAYS", DEFAULT_WINDOW_DAYS)
def ignition_delta() -> float:
return _env_float("GT_MICRO_IGNITION_DELTA", DEFAULT_IGNITION_DELTA)
def _peak_score(
*,
composite: float,
financial: float,
unrest: float,
conflict: float,
) -> float:
return max(composite, financial, unrest, conflict)
def _region_reading_from_feature(
feature: dict[str, Any],
*,
captured_at: str,
) -> DailyRegionReading | None:
props = feature.get("properties") or {}
region = str(props.get("region") or "").strip().lower()
if not region:
return None
composite = float(props.get("risk") or props.get("composite_risk") or 0.0)
financial = float(props.get("financial") or 0.0)
unrest = float(props.get("unrest") or 0.0)
conflict = float(props.get("conflict") or 0.0)
peak = _peak_score(
composite=composite,
financial=financial,
unrest=unrest,
conflict=conflict,
)
return DailyRegionReading(
region=region,
composite_risk=composite,
financial=financial,
unrest=unrest,
conflict=conflict,
peak_score=peak,
readings=1,
last_captured_at=captured_at,
)
def capture_daily_readings(
engine: GT_EarlyWarning,
*,
when: date | None = None,
) -> dict[str, Any]:
"""
Upsert today's regional readings from the live heatmap.
Each GT refresh updates the current day's latest scores (rolling window
uses one value per calendar day).
"""
day = when or utc_today()
day_key = date_id(day)
captured_at = utc_now_iso()
heatmap = engine.get_risk_heatmap()
existing = load_daily(day) or DailySnapshot(date=day_key, regions={})
updated = 0
for feature in heatmap.get("features") or []:
if not isinstance(feature, dict):
continue
reading = _region_reading_from_feature(feature, captured_at=captured_at)
if reading is None:
continue
prior = existing.regions.get(reading.region)
if prior is None:
existing.regions[reading.region] = reading
updated += 1
continue
prior.composite_risk = reading.composite_risk
prior.financial = reading.financial
prior.unrest = reading.unrest
prior.conflict = reading.conflict
prior.peak_score = max(prior.peak_score, reading.peak_score)
prior.readings += 1
prior.last_captured_at = captured_at
updated += 1
existing.last_updated_at = captured_at
save_daily(existing)
return {
"date": day_key,
"regions": len(existing.regions),
"updated": updated,
"captured_at": captured_at,
}
@dataclass(frozen=True)
class MicroRegionView:
region: str
spot_risk: float
risk_3d_avg: float
risk_delta: float
days_in_window: int
day_scores: tuple[float, ...]
alerted_spot: bool
alerted_3d: bool
ignition: bool
financial: float
unrest: float
conflict: float
def to_dict(self) -> dict[str, Any]:
return {
"region": self.region,
"spot_risk": round(self.spot_risk, 4),
"risk_3d_avg": round(self.risk_3d_avg, 4),
"risk_delta": round(self.risk_delta, 4),
"days_in_window": self.days_in_window,
"day_scores": [round(score, 4) for score in self.day_scores],
"alerted_spot": self.alerted_spot,
"alerted_3d": self.alerted_3d,
"ignition": self.ignition,
"financial": round(self.financial, 4),
"unrest": round(self.unrest, 4),
"conflict": round(self.conflict, 4),
}
def _day_offsets(window_days: int) -> list[int]:
# Today + prior (window_days - 1) days.
return list(range(window_days - 1, -1, -1))
def _historical_dates(as_of: date, window_days: int) -> list[date]:
return [as_of - timedelta(days=offset) for offset in _day_offsets(window_days)]
def compute_micro_view(
region: str,
*,
as_of: date | None = None,
window_days: int | None = None,
alert_threshold: float | None = None,
spot_reading: DailyRegionReading | None = None,
) -> MicroRegionView | None:
"""Compute rolling N-day average and ignition vs spot for one region."""
region_key = str(region or "").strip().lower()
if not region_key:
return None
today = as_of or utc_today()
window = window_days or micro_window_days()
threshold = float(alert_threshold if alert_threshold is not None else rolling_alert_threshold())
delta_min = ignition_delta()
day_scores: list[float] = []
latest: DailyRegionReading | None = spot_reading
for day in _historical_dates(today, window):
snap = load_daily(day)
if snap is None:
continue
row = snap.regions.get(region_key)
if row is None:
continue
day_scores.append(row.peak_score)
if day == today:
latest = row
if latest is None and day_scores:
# Spot may come from yesterday if today not captured yet.
snap = load_daily(today)
if snap:
latest = snap.regions.get(region_key)
if latest is None and not day_scores:
return None
spot = float(latest.peak_score if latest else (day_scores[-1] if day_scores else 0.0))
avg = sum(day_scores) / len(day_scores) if day_scores else spot
risk_delta = spot - avg
ignition = risk_delta >= delta_min and spot >= threshold * 0.75
return MicroRegionView(
region=region_key,
spot_risk=spot,
risk_3d_avg=avg,
risk_delta=risk_delta,
days_in_window=len(day_scores),
day_scores=tuple(day_scores),
alerted_spot=spot >= threshold,
alerted_3d=avg >= threshold,
ignition=ignition,
financial=float(latest.financial if latest else 0.0),
unrest=float(latest.unrest if latest else 0.0),
conflict=float(latest.conflict if latest else 0.0),
)
def compute_all_micro_views(
*,
as_of: date | None = None,
window_days: int | None = None,
alert_threshold: float | None = None,
) -> list[MicroRegionView]:
"""Build micro views for all regions seen in the rolling window."""
today = as_of or utc_today()
window = window_days or micro_window_days()
regions: set[str] = set()
for day in _historical_dates(today, window):
snap = load_daily(day)
if snap is None:
continue
regions.update(snap.regions.keys())
views: list[MicroRegionView] = []
for region in regions:
view = compute_micro_view(
region,
as_of=today,
window_days=window,
alert_threshold=alert_threshold,
)
if view is not None:
views.append(view)
views.sort(key=lambda row: (row.ignition, row.risk_delta, row.spot_risk), reverse=True)
return views
def enrich_heatmap_features(
heatmap: dict[str, Any],
*,
as_of: date | None = None,
window_days: int | None = None,
alert_threshold: float | None = None,
) -> dict[str, Any]:
"""Attach micro rolling fields to heatmap GeoJSON features."""
threshold = float(alert_threshold if alert_threshold is not None else rolling_alert_threshold())
window = window_days or micro_window_days()
features = heatmap.get("features") or []
enriched: list[dict[str, Any]] = []
for feature in features:
if not isinstance(feature, dict):
continue
props = dict(feature.get("properties") or {})
region = str(props.get("region") or "").strip().lower()
view = compute_micro_view(
region,
as_of=as_of,
window_days=window,
alert_threshold=threshold,
) if region else None
if view is not None:
props["risk_spot"] = view.spot_risk
props["risk_3d_avg"] = view.risk_3d_avg
props["risk_delta"] = view.risk_delta
props["micro_days"] = view.days_in_window
props["micro_ignition"] = view.ignition
props["alerted_3d"] = view.alerted_3d
props["day_scores"] = list(view.day_scores)
enriched.append({**feature, "properties": props})
return {
**heatmap,
"features": enriched,
"micro_window_days": window,
"micro_alert_threshold": threshold,
}
def micro_rolling_report(
*,
as_of: date | None = None,
window_days: int | None = None,
limit: int = 15,
) -> dict[str, Any]:
"""API/OpenClaw payload for micro rolling 3-day context."""
today = as_of or utc_today()
window = window_days or micro_window_days()
threshold = rolling_alert_threshold()
views = compute_all_micro_views(
as_of=today,
window_days=window,
alert_threshold=threshold,
)
ignitions = [row for row in views if row.ignition]
alerted_3d = [row for row in views if row.alerted_3d]
top = views[: max(1, limit)]
stored_days = list_daily_ids(newest_first=True, limit=window)
return {
"mode": "micro_rolling",
"window_days": window,
"alert_threshold": threshold,
"ignition_delta": ignition_delta(),
"as_of": date_id(today),
"days_stored": len(stored_days),
"stored_dates": stored_days,
"regions_tracked": len(views),
"ignition_count": len(ignitions),
"alerted_3d_count": len(alerted_3d),
"ignitions": [row.to_dict() for row in ignitions[:limit]],
"top_regions": [row.to_dict() for row in top],
"note": (
f"Micro view: {window}-day rolling average vs spot risk. "
"Ignition = spot jumped above the rolling baseline (events that flare fast). "
"Macro week-over-week validation remains on /api/analytics/rolling."
),
}
backend/analytics/rolling_backtest.py
+382
View File
@@ -0,0 +1,382 @@
"""Rolling weekly operational validation for Strategic Risk Analytics.
Freezes live GT scores each ISO week, accepts delayed outcome labels, and
scores prior-week predictions with accuracy + Wilson 95% CI. Unlike the
static historical benchmark, this measures forward operational usefulness.
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from datetime import date, datetime, timezone
from typing import Any, Literal
from analytics.backtest import DEFAULT_BACKTEST_ALERT_THRESHOLD, wilson_interval
from analytics.gt_early_warning import GT_EarlyWarning
from analytics.integration import get_gt_engine
from analytics.weekly_store import (
VALID_LABELS,
LabelName,
RegionSnapshot,
WeeklySnapshot,
list_week_ids,
load_week,
save_week,
utc_now_iso,
)
MIN_LABELED_FOR_TREND = 5
def _env_float(name: str, default: float) -> float:
raw = str(os.environ.get(name, "")).strip()
if not raw:
return default
try:
return float(raw)
except ValueError:
return default
def rolling_alert_threshold() -> float:
"""Fixed operational alert cutoff — not retroactively tuned per week."""
return _env_float("GT_ROLLING_ALERT_THRESHOLD", DEFAULT_BACKTEST_ALERT_THRESHOLD)
def iso_week_id(when: datetime | date | None = None) -> str:
"""Return ISO week id, e.g. ``2026-W24``."""
if when is None:
when = datetime.now(timezone.utc)
if isinstance(when, datetime):
when = when.date()
year, week, _ = when.isocalendar()
return f"{year}-W{week:02d}"
def _region_rows_from_engine(
engine: GT_EarlyWarning,
*,
alert_threshold: float,
) -> list[RegionSnapshot]:
heatmap = engine.get_risk_heatmap()
rows: list[RegionSnapshot] = []
for feature in heatmap.get("features") or []:
if not isinstance(feature, dict):
continue
props = feature.get("properties") or {}
region = str(props.get("region") or "").strip().lower()
if not region:
continue
composite = float(props.get("risk") or 0.0)
financial = float(props.get("financial") or 0.0)
unrest = float(props.get("unrest") or 0.0)
conflict = float(props.get("conflict") or 0.0)
peak_score = max(composite, financial, unrest, conflict)
rows.append(
RegionSnapshot(
region=region,
composite_risk=composite,
financial=financial,
unrest=unrest,
conflict=conflict,
alerted=peak_score >= alert_threshold,
label="pending",
)
)
rows.sort(key=lambda row: row.composite_risk, reverse=True)
return rows
@dataclass(frozen=True)
class WeekScore:
week_id: str
frozen_at: str
alert_threshold: float
total_regions: int
labeled: int
pending: int
alerted: int
correct: int
accuracy: float
confidence_rate: float
wilson_lower_95: float
wilson_upper_95: float
true_positives: int
true_negatives: int
false_positives: int
false_negatives: int
sensitivity: float
specificity: float
scorable: bool
def to_dict(self) -> dict[str, Any]:
return {
"week_id": self.week_id,
"frozen_at": self.frozen_at,
"alert_threshold": round(self.alert_threshold, 4),
"total_regions": self.total_regions,
"labeled": self.labeled,
"pending": self.pending,
"alerted": self.alerted,
"correct": self.correct,
"accuracy": round(self.accuracy, 4),
"confidence_rate": round(self.confidence_rate, 4),
"wilson_lower_95": round(self.wilson_lower_95, 4),
"wilson_upper_95": round(self.wilson_upper_95, 4),
"true_positives": self.true_positives,
"true_negatives": self.true_negatives,
"false_positives": self.false_positives,
"false_negatives": self.false_negatives,
"sensitivity": round(self.sensitivity, 4),
"specificity": round(self.specificity, 4),
"scorable": self.scorable,
}
def _predicted_positive(row: RegionSnapshot) -> bool:
return row.alerted
def _actual_positive(label: LabelName) -> bool:
return label == "true_escalation"
def _is_correct(row: RegionSnapshot) -> bool:
if row.label == "pending":
return False
predicted = _predicted_positive(row)
if row.label == "true_escalation":
return predicted
if row.label in ("false_alarm", "benign"):
return not predicted
return False
def score_week(snapshot: WeeklySnapshot) -> WeekScore:
"""Score a frozen week against delayed labels (pending rows excluded)."""
labeled_rows = [row for row in snapshot.regions if row.label != "pending"]
pending = len(snapshot.regions) - len(labeled_rows)
tp = sum(
1
for row in labeled_rows
if row.alerted and row.label == "true_escalation"
)
tn = sum(
1
for row in labeled_rows
if not row.alerted and row.label in ("benign", "false_alarm")
)
fp = sum(
1
for row in labeled_rows
if row.alerted and row.label in ("false_alarm", "benign")
)
fn = sum(
1
for row in labeled_rows
if not row.alerted and row.label == "true_escalation"
)
correct = tp + tn
total = len(labeled_rows)
accuracy = correct / total if total else 0.0
lower, upper = wilson_interval(correct, total)
pos_total = sum(1 for row in labeled_rows if _actual_positive(row.label)) # type: ignore[arg-type]
neg_total = total - pos_total
pred_pos = sum(1 for row in labeled_rows if row.alerted)
pred_neg = total - pred_pos
sensitivity = tp / pos_total if pos_total else 0.0
specificity = tn / pred_neg if pred_neg else (1.0 if tn == total and total else 0.0)
return WeekScore(
week_id=snapshot.week_id,
frozen_at=snapshot.frozen_at,
alert_threshold=snapshot.alert_threshold,
total_regions=len(snapshot.regions),
labeled=total,
pending=pending,
alerted=sum(1 for row in snapshot.regions if row.alerted),
correct=correct,
accuracy=accuracy,
confidence_rate=lower,
wilson_lower_95=lower,
wilson_upper_95=upper,
true_positives=tp,
true_negatives=tn,
false_positives=fp,
false_negatives=fn,
sensitivity=sensitivity,
specificity=specificity,
scorable=total >= MIN_LABELED_FOR_TREND,
)
def freeze_weekly_snapshot(
*,
week_id: str | None = None,
alert_threshold: float | None = None,
force: bool = False,
frozen_by: str = "system",
engine: GT_EarlyWarning | None = None,
) -> dict[str, Any]:
"""
Capture current GT heatmap as an immutable weekly operational snapshot.
Idempotent per week unless ``force=True``.
"""
resolved_engine = engine or get_gt_engine()
if resolved_engine is None:
return {"ok": False, "detail": "GT analytics engine unavailable"}
resolved_week = week_id or iso_week_id()
threshold = float(
alert_threshold if alert_threshold is not None else rolling_alert_threshold()
)
existing = load_week(resolved_week)
if existing and existing.regions and not force:
score = score_week(existing)
return {
"ok": True,
"created": False,
"week_id": resolved_week,
"snapshot": existing.to_dict(),
"score": score.to_dict(),
}
regions = _region_rows_from_engine(resolved_engine, alert_threshold=threshold)
snapshot = WeeklySnapshot(
week_id=resolved_week,
frozen_at=utc_now_iso(),
alert_threshold=threshold,
regions=regions,
frozen_by=frozen_by,
)
save_week(snapshot)
score = score_week(snapshot)
return {
"ok": True,
"created": True,
"week_id": resolved_week,
"snapshot": snapshot.to_dict(),
"score": score.to_dict(),
"alert_count": sum(1 for row in regions if row.alerted),
"region_count": len(regions),
}
def label_regions(
week_id: str,
labels: list[dict[str, Any]],
*,
labeled_by: str = "operator",
) -> dict[str, Any]:
"""Apply delayed outcome labels to a frozen week."""
snapshot = load_week(week_id)
if snapshot is None:
return {"ok": False, "detail": f"Week {week_id} not found"}
by_region = {row.region: row for row in snapshot.regions}
updated = 0
skipped: list[str] = []
now = utc_now_iso()
for entry in labels:
if not isinstance(entry, dict):
continue
region = str(entry.get("region") or "").strip().lower()
label = str(entry.get("label") or "").strip().lower()
if not region or label not in VALID_LABELS or label == "pending":
if region:
skipped.append(region)
continue
row = by_region.get(region)
if row is None:
skipped.append(region)
continue
row.label = label # type: ignore[assignment]
row.labeled_at = now
notes = entry.get("notes")
if notes is not None:
row.notes = str(notes)
updated += 1
save_week(snapshot)
score = score_week(snapshot)
return {
"ok": True,
"week_id": week_id,
"updated": updated,
"skipped": skipped,
"labeled_by": labeled_by,
"score": score.to_dict(),
}
def label_region(
week_id: str,
region: str,
label: LabelName,
*,
notes: str = "",
labeled_by: str = "operator",
) -> dict[str, Any]:
return label_regions(
week_id,
[{"region": region, "label": label, "notes": notes}],
labeled_by=labeled_by,
)
def rolling_trend(*, weeks: int = 8) -> list[WeekScore]:
"""Return scored weeks newest-first (only weeks with stored snapshots)."""
ids = list_week_ids(newest_first=True)[: max(1, weeks)]
scores: list[WeekScore] = []
for week_id in ids:
snapshot = load_week(week_id)
if snapshot is None:
continue
scores.append(score_week(snapshot))
return scores
def rolling_report(*, weeks: int = 8, target_confidence: float = 0.80) -> dict[str, Any]:
"""Aggregate operational validation trend for API / OpenClaw."""
threshold = rolling_alert_threshold()
trend = rolling_trend(weeks=weeks)
scorable = [row for row in trend if row.scorable]
latest = scorable[0] if scorable else (trend[0] if trend else None)
accuracy_series = [
{"week_id": row.week_id, "accuracy": round(row.accuracy, 4), "labeled": row.labeled}
for row in reversed(scorable)
]
improving = False
if len(scorable) >= 2:
improving = scorable[0].accuracy >= scorable[1].accuracy
return {
"mode": "rolling_operational",
"alert_threshold": threshold,
"target_confidence": target_confidence,
"weeks_requested": weeks,
"weeks_stored": len(trend),
"weeks_scorable": len(scorable),
"min_labeled_per_week": MIN_LABELED_FOR_TREND,
"latest": latest.to_dict() if latest else None,
"trend": [row.to_dict() for row in trend],
"accuracy_series": accuracy_series,
"improving_vs_prior": improving,
"meets_target": bool(
latest and latest.scorable and latest.confidence_rate >= target_confidence
),
"note": (
"Operational metric: scores frozen weekly predictions against delayed "
"labels. Unlike the static benchmark, this measures live forward utility."
),
}
backend/analytics/settings.py
+158
View File
@@ -0,0 +1,158 @@
"""Configuration for Strategic Risk Analytics (feature-flagged)."""
from __future__ import annotations
import json
import os
from dataclasses import dataclass, field
from functools import lru_cache
from typing import Any
def _env_bool(name: str, default: bool = False) -> bool:
raw = str(os.environ.get(name, "")).strip().lower()
if not raw:
return default
return raw not in {"0", "false", "no", "off"}
def _env_float(name: str, default: float) -> float:
raw = str(os.environ.get(name, "")).strip()
if not raw:
return default
try:
return float(raw)
except ValueError:
return default
def _env_int(name: str, default: int) -> int:
raw = str(os.environ.get(name, "")).strip()
if not raw:
return default
try:
return int(raw)
except ValueError:
return default
def _parse_signal_weights(raw: str) -> dict[str, float]:
if not raw.strip():
return {}
try:
parsed = json.loads(raw)
if isinstance(parsed, dict):
return {str(k): float(v) for k, v in parsed.items()}
except (json.JSONDecodeError, TypeError, ValueError):
pass
weights: dict[str, float] = {}
for part in raw.split(","):
piece = part.strip()
if not piece or "=" not in piece:
continue
key, value = piece.split("=", 1)
try:
weights[key.strip()] = float(value.strip())
except ValueError:
continue
return weights
def resolve_gt_profile() -> str:
from services.runtime_profile import resolve_profile_name
return resolve_profile_name()
def gt_analytics_ack_low_cpu() -> bool:
return _env_bool("GT_ANALYTICS_ACK_LOW_CPU", default=False)
def gt_engine_operational() -> bool:
"""Full GT engine (scheduled ingest, heatmap, Louvain) — not watchdog-only."""
if not get_gt_settings().enabled:
return False
if resolve_gt_profile() == "lean" and not gt_analytics_ack_low_cpu():
return False
return True
def gt_scheduled_ingest_enabled() -> bool:
return gt_engine_operational()
def gt_louvain_enabled() -> bool:
return gt_engine_operational()
@dataclass(frozen=True)
class GTAnalyticsSettings:
enabled: bool = False
profile: str = "standard"
base_prior: float = 0.15
evidence_cap: float = 3.0
evidence_scale: float = 5.0
min_prob: float = 0.01
max_prob: float = 0.99
high_risk_threshold: float = 0.6
max_history_per_region: int = 200
max_heatmap_features: int = 500
louvain_min_weight: float = 0.5
louvain_interval_minutes: int = 30
signal_weight_overrides: dict[str, float] = field(default_factory=dict)
watched_channels: tuple[str, ...] = ()
@lru_cache(maxsize=1)
def get_gt_settings() -> GTAnalyticsSettings:
channels_raw = str(os.environ.get("GT_ANALYTICS_WATCHED_CHANNELS", "")).strip()
channels = tuple(
part.strip().lstrip("@")
for part in channels_raw.split(",")
if part.strip()
)
profile = resolve_gt_profile()
lean = profile == "lean"
return GTAnalyticsSettings(
enabled=_env_bool("GT_ANALYTICS_ENABLED", default=False),
profile=profile,
base_prior=_env_float("GT_ANALYTICS_BASE_PRIOR", 0.15),
evidence_cap=_env_float("GT_ANALYTICS_EVIDENCE_CAP", 3.0),
evidence_scale=_env_float("GT_ANALYTICS_EVIDENCE_SCALE", 5.0),
min_prob=_env_float("GT_ANALYTICS_MIN_PROB", 0.01),
max_prob=_env_float("GT_ANALYTICS_MAX_PROB", 0.99),
high_risk_threshold=_env_float("GT_ANALYTICS_HIGH_RISK_THRESHOLD", 0.6),
max_history_per_region=_env_int("GT_ANALYTICS_MAX_HISTORY", 200),
max_heatmap_features=_env_int(
"GT_ANALYTICS_MAX_HEATMAP_FEATURES",
50 if lean else 500,
),
louvain_min_weight=_env_float("GT_ANALYTICS_LOUVAIN_MIN_WEIGHT", 0.5),
louvain_interval_minutes=max(5, _env_int("GT_ANALYTICS_LOUVAIN_INTERVAL_MINUTES", 30)),
signal_weight_overrides=_parse_signal_weights(
str(os.environ.get("GT_ANALYTICS_SIGNAL_WEIGHTS", ""))
),
watched_channels=channels,
)
def gt_analytics_enabled() -> bool:
return get_gt_settings().enabled
def gt_analytics_status() -> dict[str, Any]:
settings = get_gt_settings()
from services.runtime_profile import get_runtime_profile
runtime = get_runtime_profile()
operational = gt_engine_operational()
return {
"enabled": settings.enabled,
"operational": operational,
"profile": settings.profile,
"ack_low_cpu": gt_analytics_ack_low_cpu(),
"recommended": bool(runtime.get("gt_analytics", {}).get("recommended")),
"lean_node": bool(runtime.get("gt_analytics", {}).get("lean_node")),
"warning": runtime.get("gt_analytics", {}).get("warning"),
"experimental": True,
}
backend/analytics/weekly_store.py
+154
View File
@@ -0,0 +1,154 @@
"""Persistent JSON store for rolling GT operational backtest weeks."""
from __future__ import annotations
import json
import logging
import os
import threading
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Literal
logger = logging.getLogger(__name__)
LabelName = Literal["pending", "true_escalation", "false_alarm", "benign"]
VALID_LABELS: frozenset[str] = frozenset(
{"pending", "true_escalation", "false_alarm", "benign"}
)
_STORE_DIR = Path(__file__).parent.parent / "data" / "gt_rolling"
_store_lock = threading.Lock()
def rolling_store_dir() -> Path:
"""Return the rolling-backtest data directory (override via env in tests)."""
override = str(os.environ.get("GT_ROLLING_STORE_DIR", "")).strip()
if override:
return Path(override)
return _STORE_DIR
@dataclass
class RegionSnapshot:
region: str
composite_risk: float
financial: float
unrest: float
conflict: float
alerted: bool
label: LabelName = "pending"
labeled_at: str | None = None
notes: str = ""
def to_dict(self) -> dict[str, Any]:
return asdict(self)
@classmethod
def from_dict(cls, raw: dict[str, Any]) -> RegionSnapshot:
label = str(raw.get("label") or "pending")
if label not in VALID_LABELS:
label = "pending"
return cls(
region=str(raw.get("region") or "").strip().lower(),
composite_risk=float(raw.get("composite_risk") or 0.0),
financial=float(raw.get("financial") or 0.0),
unrest=float(raw.get("unrest") or 0.0),
conflict=float(raw.get("conflict") or 0.0),
alerted=bool(raw.get("alerted")),
label=label, # type: ignore[arg-type]
labeled_at=raw.get("labeled_at"),
notes=str(raw.get("notes") or ""),
)
@dataclass
class WeeklySnapshot:
week_id: str
frozen_at: str
alert_threshold: float
regions: list[RegionSnapshot] = field(default_factory=list)
frozen_by: str = "system"
def to_dict(self) -> dict[str, Any]:
return {
"week_id": self.week_id,
"frozen_at": self.frozen_at,
"alert_threshold": self.alert_threshold,
"frozen_by": self.frozen_by,
"regions": [row.to_dict() for row in self.regions],
}
@classmethod
def from_dict(cls, raw: dict[str, Any]) -> WeeklySnapshot:
regions = [
RegionSnapshot.from_dict(row)
for row in (raw.get("regions") or [])
if isinstance(row, dict)
]
return cls(
week_id=str(raw.get("week_id") or ""),
frozen_at=str(raw.get("frozen_at") or ""),
alert_threshold=float(raw.get("alert_threshold") or 0.0),
regions=regions,
frozen_by=str(raw.get("frozen_by") or "system"),
)
def _week_path(week_id: str) -> Path:
safe = week_id.replace("/", "-").replace("..", "")
return rolling_store_dir() / f"{safe}.json"
def _ensure_dir() -> None:
rolling_store_dir().mkdir(parents=True, exist_ok=True)
def list_week_ids(*, newest_first: bool = True) -> list[str]:
"""Return stored ISO week ids."""
_ensure_dir()
ids = [
path.stem
for path in rolling_store_dir().glob("*.json")
if path.stem and path.stem != "index"
]
ids.sort(reverse=newest_first)
return ids
def load_week(week_id: str) -> WeeklySnapshot | None:
path = _week_path(week_id)
if not path.is_file():
return None
try:
raw = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(raw, dict):
return None
return WeeklySnapshot.from_dict(raw)
except (OSError, json.JSONDecodeError, TypeError, ValueError):
logger.exception("Failed to load GT rolling week %s", week_id)
return None
def save_week(snapshot: WeeklySnapshot) -> None:
_ensure_dir()
path = _week_path(snapshot.week_id)
tmp = path.with_suffix(".json.tmp")
payload = json.dumps(snapshot.to_dict(), indent=2, sort_keys=True)
with _store_lock:
tmp.write_text(payload, encoding="utf-8")
tmp.replace(path)
def delete_week(week_id: str) -> bool:
path = _week_path(week_id)
if not path.is_file():
return False
with _store_lock:
path.unlink()
return True
def utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat()