Compare commits

...

1 Commits

Author SHA1 Message Date
Shadowbroker cfbeabda1e Feat/gt analytics openclaw (#392)
* feat(telegram): auto-translate OSINT channel posts to English

Cherry-picked from @Bobpick PR #391 (telegram-only slice): server-side translation during fetch, SHOW ORIGINAL toggle in TelegramOsintPopup, and on-demand /api/telegram-feed?lang=.

Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

* feat(gt): experimental Derived OSINT analytics with lean-node safeguards

Cherry-picked from @Bobpick PR #391 (GT + OpenClaw slice): Bayesian strategic-risk engine, map overlay, OpenClaw commands, and telegram_rhetoric watchdog. Off by default (GT_ANALYTICS_ENABLED=false, gt_risk layer false). 1 vCPU nodes get cgroup detection, UI warning on layer toggle, and lean profile that skips scheduled ingest/Louvain unless GT_ANALYTICS_ACK_LOW_CPU=true. Backtest HUD removed from dashboard (OpenClaw/API regression only).

Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

---------

Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-06-16 17:05:46 -06:00
69 changed files with 8102 additions and 78 deletions
+14
View File
@@ -26,6 +26,20 @@ AIS_API_KEY=
# Telegram OSINT map layer — scrapes public t.me/s channel previews (no bot token).
# TELEGRAM_OSINT_ENABLED=true
# TELEGRAM_OSINT_CHANNELS=osintdefender,insiderpaper,aljazeeraenglish,nexta_live,war_monitor
# TELEGRAM_OSINT_TRANSLATE=true
# TELEGRAM_OSINT_TRANSLATE_TO=en
# Strategic Risk Analytics (experimental derived OSINT — off by default)
# GT_ANALYTICS_ENABLED=false
# GT_ANALYTICS_PROFILE=lean
# On 1 vCPU nodes (fleet VPS), leave disabled or set profile=lean. Scheduled ingest
# and Louvain clustering stay off until GT_ANALYTICS_ACK_LOW_CPU=true.
# GT_ANALYTICS_ACK_LOW_CPU=false
# GT_ANALYTICS_BASE_PRIOR=0.15
# GT_ANALYTICS_HIGH_RISK_THRESHOLD=0.6
# GT_ANALYTICS_SIGNAL_WEIGHTS=payroll_loan=3.0,purge=3.5,troop_movement=3.0
# GT_ANALYTICS_WATCHED_CHANNELS=osintdefender,war_monitor,nexta_live
# GT_ANALYTICS_LOUVAIN_INTERVAL_MINUTES=30
# Admin key to protect sensitive endpoints (settings, updates).
# If blank, loopback/localhost requests still work for local single-host dev.
+21
View File
@@ -0,0 +1,21 @@
"""Strategic Risk Analytics — game-theoretic early warning layer."""
from analytics.backtest import (
DEFAULT_BACKTEST_ALERT_THRESHOLD,
BacktestReport,
run_historical_backtest,
tune_alert_threshold,
)
from analytics.gt_early_warning import GT_EarlyWarning
from analytics.integration import get_gt_engine, process_feed_item, refresh_from_latest_data
__all__ = [
"BacktestReport",
"DEFAULT_BACKTEST_ALERT_THRESHOLD",
"GT_EarlyWarning",
"get_gt_engine",
"process_feed_item",
"refresh_from_latest_data",
"run_historical_backtest",
"tune_alert_threshold",
]
+287
View File
@@ -0,0 +1,287 @@
"""Historical backtesting for Strategic Risk Analytics.
This is **benchmark validation**, not forward-weeks prediction on live feeds.
The suite scores whether costly-signal patterns + Bayesian updating correctly
classify curated pre-crisis text snippets (positive cases) vs cheap-talk
controls (negative cases) at a tuned alert threshold. A high accuracy on this
labeled corpus does **not** imply the engine will score 100% on messy,
adversarial, or weeks-ahead production telemetry — opponents adapt, labels are
easier here than in the wild, and the window is retrospective.
Reports accuracy and a conservative Wilson 95% confidence lower bound on the
benchmark only. Treat 100% here as "classifier fits the benchmark," not "ship
it for multi-week forecasting." For live week-over-week scoring with delayed
labels, see ``rolling_backtest.py``.
"""
from __future__ import annotations
import math
from dataclasses import dataclass, field
from typing import Any, Literal
from analytics.gt_early_warning import GT_EarlyWarning
from analytics.historical_events import (
HistoricalCase,
default_historical_cases,
expanded_historical_cases,
)
from analytics.settings import GTAnalyticsSettings
DomainName = Literal["financial", "unrest", "conflict"]
# Validated on expanded suite (82 cases, Wilson lower >= 0.95 at 100% accuracy).
DEFAULT_BACKTEST_ALERT_THRESHOLD = 0.26
MAX_BACKTEST_ALERT_THRESHOLD = 0.39
@dataclass(frozen=True)
class CaseResult:
case_id: str
name: str
kind: str
region: str
domain: str
expected_alert: bool
alerted: bool
correct: bool
peak_domain_risk: float
peak_composite_risk: float
costly_signals: list[str]
tags: tuple[str, ...] = field(default_factory=tuple)
@dataclass(frozen=True)
class BacktestReport:
total_cases: int
correct: int
accuracy: float
confidence_rate: float
wilson_lower_95: float
wilson_upper_95: float
true_positives: int
true_negatives: int
false_positives: int
false_negatives: int
sensitivity: float
specificity: float
alert_threshold: float
target_confidence: float
meets_target: bool
case_results: tuple[CaseResult, ...]
def to_dict(self) -> dict[str, Any]:
return {
"total_cases": self.total_cases,
"correct": self.correct,
"accuracy": round(self.accuracy, 4),
"confidence_rate": round(self.confidence_rate, 4),
"wilson_lower_95": round(self.wilson_lower_95, 4),
"wilson_upper_95": round(self.wilson_upper_95, 4),
"true_positives": self.true_positives,
"true_negatives": self.true_negatives,
"false_positives": self.false_positives,
"false_negatives": self.false_negatives,
"sensitivity": round(self.sensitivity, 4),
"specificity": round(self.specificity, 4),
"alert_threshold": self.alert_threshold,
"target_confidence": self.target_confidence,
"meets_target": self.meets_target,
"cases": [
{
"case_id": row.case_id,
"name": row.name,
"kind": row.kind,
"correct": row.correct,
"alerted": row.alerted,
"peak_domain_risk": round(row.peak_domain_risk, 4),
"peak_composite_risk": round(row.peak_composite_risk, 4),
"costly_signals": row.costly_signals,
}
for row in self.case_results
],
}
def wilson_interval(
successes: int,
total: int,
z: float = 1.96,
) -> tuple[float, float]:
"""Wilson score interval for a binomial proportion (95% default)."""
if total <= 0:
return 0.0, 0.0
phat = successes / total
z2 = z * z
denom = 1.0 + z2 / total
center = (phat + z2 / (2.0 * total)) / denom
margin = (
z
* math.sqrt((phat * (1.0 - phat) + z2 / (4.0 * total)) / total)
/ denom
)
return max(0.0, center - margin), min(1.0, center + margin)
def _domain_risk(engine: GT_EarlyWarning, region: str, domain: str) -> float:
if domain in ("financial", "unrest", "conflict"):
return engine.get_prior(region, domain)
return engine.composite_risk(region)
def _evaluate_case(
case: HistoricalCase,
*,
settings: GTAnalyticsSettings,
alert_threshold: float,
) -> CaseResult:
engine = GT_EarlyWarning(settings)
peak_domain = float(settings.base_prior)
peak_composite = float(settings.base_prior)
detected_signals: set[str] = set()
for item in case.to_feed_dicts():
result = engine.process_feed_item(item)
for sig in (result or {}).get("signals") or {}:
detected_signals.add(str(sig))
domain_risk = _domain_risk(engine, case.region, case.domain)
composite = engine.composite_risk(case.region)
peak_domain = max(peak_domain, domain_risk)
peak_composite = max(peak_composite, composite)
# Domain-specific score for labeled events; composite as secondary for conflict.
score = peak_domain
if case.domain == "conflict":
score = max(peak_domain, peak_composite * 0.95)
alerted = score >= alert_threshold
expected_alert = case.kind == "positive"
return CaseResult(
case_id=case.case_id,
name=case.name,
kind=case.kind,
region=case.region,
domain=case.domain,
expected_alert=expected_alert,
alerted=alerted,
correct=alerted == expected_alert,
peak_domain_risk=peak_domain,
peak_composite_risk=peak_composite,
costly_signals=sorted(detected_signals),
tags=case.tags,
)
def run_historical_backtest(
cases: tuple[HistoricalCase, ...] | None = None,
*,
settings: GTAnalyticsSettings | None = None,
alert_threshold: float | None = None,
target_confidence: float = 0.80,
use_expanded_suite: bool = True,
) -> BacktestReport:
"""
Run labeled historical cases and compute accuracy + Wilson 95% CI.
``confidence_rate`` is the conservative Wilson lower bound — the metric
used for pass/fail against ``target_confidence``.
"""
cfg = settings or GTAnalyticsSettings(enabled=True)
threshold = float(
alert_threshold
if alert_threshold is not None
else DEFAULT_BACKTEST_ALERT_THRESHOLD
)
if cases is not None:
suite = cases
elif use_expanded_suite:
suite = expanded_historical_cases()
else:
suite = default_historical_cases()
results = tuple(
_evaluate_case(case, settings=cfg, alert_threshold=threshold) for case in suite
)
tp = sum(1 for r in results if r.expected_alert and r.alerted)
tn = sum(1 for r in results if not r.expected_alert and not r.alerted)
fp = sum(1 for r in results if not r.expected_alert and r.alerted)
fn = sum(1 for r in results if r.expected_alert and not r.alerted)
correct = tp + tn
total = len(results)
accuracy = correct / total if total else 0.0
lower, upper = wilson_interval(correct, total)
pos_total = sum(1 for r in results if r.expected_alert)
neg_total = total - pos_total
sensitivity = tp / pos_total if pos_total else 0.0
specificity = tn / neg_total if neg_total else 0.0
return BacktestReport(
total_cases=total,
correct=correct,
accuracy=accuracy,
confidence_rate=lower,
wilson_lower_95=lower,
wilson_upper_95=upper,
true_positives=tp,
true_negatives=tn,
false_positives=fp,
false_negatives=fn,
sensitivity=sensitivity,
specificity=specificity,
alert_threshold=threshold,
target_confidence=target_confidence,
meets_target=lower >= target_confidence,
case_results=results,
)
def tune_alert_threshold(
cases: tuple[HistoricalCase, ...] | None = None,
*,
settings: GTAnalyticsSettings | None = None,
min_threshold: float = 0.20,
max_threshold: float = 0.65,
step: float = 0.01,
target_confidence: float = 0.95,
) -> tuple[float, BacktestReport]:
"""Grid-search alert threshold to maximize Wilson lower bound."""
if cases is not None:
suite = cases
else:
suite = expanded_historical_cases()
best_threshold = min_threshold
best_report = run_historical_backtest(
suite,
settings=settings,
alert_threshold=min_threshold,
target_confidence=target_confidence,
)
steps = int(round((max_threshold - min_threshold) / step))
for i in range(steps + 1):
threshold = min_threshold + i * step
report = run_historical_backtest(
suite,
settings=settings,
alert_threshold=threshold,
target_confidence=target_confidence,
)
better_confidence = report.confidence_rate > best_report.confidence_rate
tied_confidence = math.isclose(
report.confidence_rate, best_report.confidence_rate, rel_tol=0.0, abs_tol=1e-9
)
better_accuracy = report.accuracy > best_report.accuracy
tied_accuracy = math.isclose(
report.accuracy, best_report.accuracy, rel_tol=0.0, abs_tol=1e-9
)
prefer_higher_threshold = (
tied_confidence and tied_accuracy and threshold > best_threshold
)
if better_confidence or (tied_confidence and better_accuracy) or prefer_higher_threshold:
best_threshold = threshold
best_report = report
return best_threshold, best_report
+140
View File
@@ -0,0 +1,140 @@
"""Daily GT risk readings for micro rolling averages."""
from __future__ import annotations
import json
import logging
import os
import threading
from dataclasses import asdict, dataclass, field
from datetime import date, datetime, timezone
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
_DAILY_DIR = Path(__file__).parent.parent / "data" / "gt_rolling" / "daily"
_store_lock = threading.Lock()
def daily_store_dir() -> Path:
override = str(os.environ.get("GT_DAILY_STORE_DIR", "")).strip()
if override:
return Path(override)
return _DAILY_DIR
def utc_today() -> date:
return datetime.now(timezone.utc).date()
def date_id(when: date | datetime | None = None) -> str:
if when is None:
when = utc_today()
if isinstance(when, datetime):
when = when.date()
return when.isoformat()
@dataclass
class DailyRegionReading:
region: str
composite_risk: float
financial: float
unrest: float
conflict: float
peak_score: float
readings: int = 1
last_captured_at: str = ""
def to_dict(self) -> dict[str, Any]:
return asdict(self)
@classmethod
def from_dict(cls, raw: dict[str, Any]) -> DailyRegionReading:
return cls(
region=str(raw.get("region") or "").strip().lower(),
composite_risk=float(raw.get("composite_risk") or 0.0),
financial=float(raw.get("financial") or 0.0),
unrest=float(raw.get("unrest") or 0.0),
conflict=float(raw.get("conflict") or 0.0),
peak_score=float(raw.get("peak_score") or 0.0),
readings=int(raw.get("readings") or 1),
last_captured_at=str(raw.get("last_captured_at") or ""),
)
@dataclass
class DailySnapshot:
date: str
regions: dict[str, DailyRegionReading] = field(default_factory=dict)
last_updated_at: str = ""
def to_dict(self) -> dict[str, Any]:
return {
"date": self.date,
"last_updated_at": self.last_updated_at,
"regions": {key: row.to_dict() for key, row in self.regions.items()},
}
@classmethod
def from_dict(cls, raw: dict[str, Any]) -> DailySnapshot:
regions: dict[str, DailyRegionReading] = {}
for key, row in (raw.get("regions") or {}).items():
if isinstance(row, dict):
reading = DailyRegionReading.from_dict(row)
regions[str(key).strip().lower()] = reading
return cls(
date=str(raw.get("date") or ""),
regions=regions,
last_updated_at=str(raw.get("last_updated_at") or ""),
)
def _daily_path(day_id: str) -> Path:
safe = day_id.replace("/", "-").replace("..", "")
return daily_store_dir() / f"{safe}.json"
def _ensure_dir() -> None:
daily_store_dir().mkdir(parents=True, exist_ok=True)
def list_daily_ids(*, newest_first: bool = True, limit: int | None = None) -> list[str]:
_ensure_dir()
ids = sorted(
(path.stem for path in daily_store_dir().glob("*.json")),
reverse=newest_first,
)
if limit is not None:
return ids[:limit]
return ids
def load_daily(day: date | str | None = None) -> DailySnapshot | None:
day_id = date_id(day) if day is not None else date_id()
path = _daily_path(day_id)
if not path.is_file():
return None
try:
raw = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(raw, dict):
return None
return DailySnapshot.from_dict(raw)
except (OSError, json.JSONDecodeError, TypeError, ValueError):
logger.exception("Failed to load GT daily reading %s", day_id)
return None
def save_daily(snapshot: DailySnapshot) -> None:
_ensure_dir()
path = _daily_path(snapshot.date)
tmp = path.with_suffix(".json.tmp")
payload = json.dumps(snapshot.to_dict(), indent=2, sort_keys=True)
with _store_lock:
tmp.write_text(payload, encoding="utf-8")
tmp.replace(path)
def utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
+206
View File
@@ -0,0 +1,206 @@
"""Normalize Shadowbroker feed records into GT analytics feed items."""
from __future__ import annotations
import re
from typing import Any, Iterable
_DOMAIN_CONFLICT = "conflict"
_DOMAIN_UNREST = "unrest"
_DOMAIN_FINANCIAL = "financial"
_CONFLICT_HINTS = re.compile(
r"\b(war|missile|strike|attack|military|invasion|troop|shelling|drone|bomb|nuclear)\b",
re.I,
)
_UNREST_HINTS = re.compile(
r"\b(protest|rally|strike|riot|unrest|mobiliz|demonstrat|curfew|purge|coup)\b",
re.I,
)
_FINANCIAL_HINTS = re.compile(
r"\b(payroll|loan|default|bankruptcy|liquidity|sanction|supply\s+chain|delay|shortage)\b",
re.I,
)
def _clean_region(value: Any) -> str:
region = str(value or "").strip().lower()
return region or "global"
def _infer_domain(text: str, explicit: str | None = None) -> str:
if explicit in {_DOMAIN_CONFLICT, _DOMAIN_UNREST, _DOMAIN_FINANCIAL}:
return explicit
if _CONFLICT_HINTS.search(text):
return _DOMAIN_CONFLICT
if _UNREST_HINTS.search(text):
return _DOMAIN_UNREST
if _FINANCIAL_HINTS.search(text):
return _DOMAIN_FINANCIAL
return _DOMAIN_FINANCIAL
def _text_from_record(
record: dict[str, Any],
*,
prefer_translation: bool = False,
) -> str:
"""Build ingest text; prefer English translations for Telegram OSINT when set."""
if prefer_translation:
translated_parts = [
record.get("title_translated"),
record.get("description_translated"),
]
translated = "\n".join(
str(p).strip() for p in translated_parts if p and str(p).strip()
)
if translated:
return translated
parts = [
record.get("title"),
record.get("description"),
record.get("text"),
record.get("summary"),
]
return "\n".join(str(p).strip() for p in parts if p and str(p).strip())
_HASHTAG_REGION = re.compile(r"#([a-z][a-z0-9_-]{2,})", re.I)
def _region_from_hashtags(text: str) -> str | None:
"""Map common theater hashtags (#Ukraine) to dossier/heatmap region keys."""
for match in _HASHTAG_REGION.finditer(text or ""):
tag = match.group(1).lower()
if tag in {
"ukraine",
"russia",
"israel",
"iran",
"gaza",
"syria",
"taiwan",
"china",
"belfast",
"uk",
"usa",
}:
return tag
return None
def _region_from_record(record: dict[str, Any], *, text: str = "") -> str:
for key in ("geotag", "region", "country", "location"):
if record.get(key):
return _clean_region(record[key])
hashtag_region = _region_from_hashtags(text)
if hashtag_region:
return hashtag_region
coords = record.get("coords")
if isinstance(coords, (list, tuple)) and len(coords) >= 2:
try:
lat = float(coords[0])
lng = float(coords[1])
return f"{lat:.2f},{lng:.2f}"
except (TypeError, ValueError):
pass
return "global"
def _entities_from_record(record: dict[str, Any]) -> list[str]:
entities: list[str] = []
for key in ("entities", "tags", "keywords"):
raw = record.get(key)
if isinstance(raw, list):
entities.extend(str(v).strip() for v in raw if str(v).strip())
elif isinstance(raw, str) and raw.strip():
entities.extend(part.strip() for part in raw.split(",") if part.strip())
channel = str(record.get("channel") or "").strip()
if channel:
entities.append(f"channel:{channel}")
source = str(record.get("source") or "").strip()
if source:
entities.append(f"source:{source}")
return entities
def normalize_feed_item(record: dict[str, Any], *, source_type: str = "generic") -> dict[str, Any]:
"""Map a news/Telegram/GDELT record into the GT engine schema."""
prefer_translation = source_type == "telegram_osint"
text = _text_from_record(record, prefer_translation=prefer_translation)
if prefer_translation and not text.strip():
text = _text_from_record(record, prefer_translation=False)
region = _region_from_record(record, text=text)
domain = _infer_domain(text, record.get("domain"))
coords = record.get("coords")
lat = lng = None
if isinstance(coords, (list, tuple)) and len(coords) >= 2:
try:
lat = float(coords[0])
lng = float(coords[1])
except (TypeError, ValueError):
lat = lng = None
return {
"id": record.get("id") or record.get("link"),
"text": text,
"source": str(record.get("source") or source_type),
"source_type": source_type,
"region": region,
"domain": domain,
"entities": _entities_from_record(record),
"coords": [lat, lng] if lat is not None and lng is not None else None,
"published": record.get("published"),
"risk_score": record.get("risk_score"),
}
def iter_telegram_posts(payload: dict[str, Any] | None) -> Iterable[dict[str, Any]]:
from services.telegram_translate import apply_post_translation, telegram_translate_enabled
posts = list((payload or {}).get("posts") or [])
for post in posts:
if not isinstance(post, dict):
continue
if not (post.get("description") or post.get("title")):
continue
enriched = (
apply_post_translation(post)
if telegram_translate_enabled()
else post
)
yield normalize_feed_item(enriched, source_type="telegram_osint")
def iter_news_items(payload: list[dict[str, Any]] | None) -> Iterable[dict[str, Any]]:
for item in list(payload or []):
if not isinstance(item, dict):
continue
yield normalize_feed_item(item, source_type="news")
for article in list(item.get("articles") or []):
if isinstance(article, dict):
yield normalize_feed_item(article, source_type="news_cluster")
def iter_gdelt_features(payload: list[dict[str, Any]] | None) -> Iterable[dict[str, Any]]:
for feature in list(payload or []):
if not isinstance(feature, dict):
continue
props = dict(feature.get("properties") or {})
geometry = dict(feature.get("geometry") or {})
coords = None
if geometry.get("type") == "Point":
raw = geometry.get("coordinates")
if isinstance(raw, (list, tuple)) and len(raw) >= 2:
coords = [float(raw[1]), float(raw[0])]
record = {
"title": props.get("name") or props.get("title"),
"description": props.get("snippet") or props.get("description"),
"source": props.get("source") or "gdelt",
"coords": coords,
"published": props.get("date") or props.get("published"),
"region": props.get("location") or props.get("country"),
}
if record["title"] or record["description"]:
yield normalize_feed_item(record, source_type="gdelt")
+128
View File
@@ -0,0 +1,128 @@
"""Top strategic-risk alerts — ranked regions with map coordinates."""
from __future__ import annotations
from typing import Any
from analytics.integration import get_gt_engine
from analytics.settings import get_gt_settings
def _peak_score(props: dict[str, Any]) -> float:
composite = float(props.get("risk") or 0.0)
financial = float(props.get("financial") or 0.0)
unrest = float(props.get("unrest") or 0.0)
conflict = float(props.get("conflict") or 0.0)
return max(composite, financial, unrest, conflict)
def _valid_coords(coords: Any) -> tuple[float, float] | None:
if not isinstance(coords, (list, tuple)) or len(coords) < 2:
return None
try:
lng = float(coords[0])
lat = float(coords[1])
except (TypeError, ValueError):
return None
if not (-90.0 <= lat <= 90.0 and -180.0 <= lng <= 180.0):
return None
if abs(lat) < 0.001 and abs(lng) < 0.001:
return None
return lat, lng
def _region_label(region: str) -> str:
text = str(region or "").strip()
if not text:
return "unknown"
if "," in text:
parts = [piece.strip() for piece in text.split(",") if piece.strip()]
if len(parts) >= 2:
try:
lat = float(parts[0])
lng = float(parts[-1])
return f"{lat:.2f}°, {lng:.2f}°"
except ValueError:
pass
return text.replace("_", " ")
def parse_heatmap_alerts(
heatmap: dict[str, Any] | None,
*,
limit: int = 8,
) -> tuple[list[dict[str, Any]], int]:
"""Return ranked alerts and count of regions plottable on the map."""
features = (heatmap or {}).get("features") or []
rows: list[dict[str, Any]] = []
for feature in features:
if not isinstance(feature, dict):
continue
geometry = feature.get("geometry") or {}
coords = _valid_coords(geometry.get("coordinates"))
if coords is None:
continue
lat, lng = coords
props = feature.get("properties") or {}
region = str(props.get("region") or "").strip().lower()
if not region:
continue
score = _peak_score(props)
rows.append(
{
"region": region,
"region_label": _region_label(region),
"risk": round(float(props.get("risk") or 0.0), 4),
"financial": round(float(props.get("financial") or 0.0), 4),
"unrest": round(float(props.get("unrest") or 0.0), 4),
"conflict": round(float(props.get("conflict") or 0.0), 4),
"contagion": round(float(props.get("contagion") or 0.0), 4),
"score": round(score, 4),
"lat": lat,
"lng": lng,
"ignition": bool(props.get("micro_ignition")),
"risk_3d_avg": props.get("risk_3d_avg"),
"risk_delta": props.get("risk_delta"),
"updates": int(props.get("updates") or 0),
}
)
rows.sort(
key=lambda row: (
bool(row.get("ignition")),
float(row.get("risk_delta") or 0.0),
float(row.get("score") or 0.0),
),
reverse=True,
)
return rows[: max(1, limit)], len(rows)
def top_gt_alerts(*, limit: int = 8) -> dict[str, Any]:
"""Ranked top regions for API / OpenClaw."""
settings = get_gt_settings()
engine = get_gt_engine()
heatmap: dict[str, Any] = {"type": "FeatureCollection", "features": []}
engine_regions = 0
if engine is not None:
heatmap = engine.get_risk_heatmap()
with engine._lock: # noqa: SLF001 — intentional meta read
engine_regions = len(engine._regions)
alerts, plotted = parse_heatmap_alerts(heatmap, limit=limit)
tracked = len(heatmap.get("features") or [])
return {
"alerts": alerts,
"tracked_regions": tracked,
"engine_regions": engine_regions,
"plotted_regions": plotted,
"max_regions": settings.max_heatmap_features,
"note": (
"Layer count is tracked GT regions (cap "
f"{settings.max_heatmap_features}), not raw feed events. "
"Only regions with valid coordinates appear on the map."
),
}
+593
View File
@@ -0,0 +1,593 @@
"""Game-theoretic early warning analytics with Bayesian updating and contagion graph."""
from __future__ import annotations
import logging
import re
import threading
from collections import defaultdict
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any, DefaultDict
import networkx as nx
import numpy as np
from analytics.settings import GTAnalyticsSettings, get_gt_settings
logger = logging.getLogger(__name__)
DomainName = str # financial | unrest | conflict
_DOMAINS: tuple[DomainName, ...] = ("financial", "unrest", "conflict")
_DEFAULT_LIKELIHOODS: dict[DomainName, dict[str, float]] = {
"financial": {"distress": 0.75, "normal": 0.25},
"unrest": {"distress": 0.82, "normal": 0.22},
"conflict": {"distress": 0.78, "normal": 0.18},
}
_DEFAULT_SIGNAL_WEIGHTS: dict[str, float] = {
"payroll_loan": 3.0,
"supply_delay": 2.2,
"elite_relocation": 2.8,
"purge": 3.5,
"protest_mobilize": 2.5,
"gps_jamming": 2.7,
"troop_movement": 3.0,
"bank_run": 3.2,
"sanctions_escalation": 2.4,
"ceasefire_break": 2.6,
}
# Costly-signal regex patterns (cheap talk filtered by absence of match).
_SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
"payroll_loan": [
re.compile(r"payroll\s+loan", re.I),
re.compile(r"merchant\s+cash\s+advance", re.I),
re.compile(r"working\s+capital\s+loan", re.I),
],
"supply_delay": [
re.compile(r"supply\s+(chain\s+)?delay", re.I),
re.compile(r"shipping\s+delay", re.I),
re.compile(r"logistics\s+backlog", re.I),
re.compile(r"port\s+congestion", re.I),
],
"elite_relocation": [
re.compile(r"elite\s+(asset\s+)?relocation", re.I),
re.compile(r"oligarch\s+jet", re.I),
re.compile(r"private\s+jet\s+exodus", re.I),
re.compile(r"capital\s+flight", re.I),
],
"purge": [
re.compile(r"\bpurge\b", re.I),
re.compile(r"political\s+purge", re.I),
re.compile(r"security\s+apparatus\s+reshuffle", re.I),
],
"protest_mobilize": [
re.compile(r"protest\s+mobil", re.I),
re.compile(r"mass\s+rally", re.I),
re.compile(r"general\s+strike", re.I),
re.compile(r"\bstrike\b", re.I),
re.compile(r"\brally\b", re.I),
],
"gps_jamming": [
re.compile(r"gps\s+jam", re.I),
re.compile(r"gnss\s+interference", re.I),
re.compile(r"spoofing\s+spike", re.I),
],
"troop_movement": [
re.compile(r"troop\s+movement", re.I),
re.compile(r"military\s+mobil", re.I),
re.compile(r"armored\s+convoy", re.I),
re.compile(r"troop\s+buildup", re.I),
],
"bank_run": [
re.compile(r"bank\s+run", re.I),
re.compile(r"deposit\s+flight", re.I),
re.compile(r"liquidity\s+crunch", re.I),
],
"sanctions_escalation": [
re.compile(r"sanctions?\s+escalat", re.I),
re.compile(r"new\s+sanctions?", re.I),
re.compile(r"export\s+controls?\s+tighten", re.I),
],
"ceasefire_break": [
re.compile(r"ceasefire\s+(broken|violated|collapse)", re.I),
re.compile(r"truce\s+end", re.I),
],
}
_SIGNAL_DOMAINS: dict[str, DomainName] = {
"payroll_loan": "financial",
"supply_delay": "financial",
"bank_run": "financial",
"sanctions_escalation": "financial",
"protest_mobilize": "unrest",
"purge": "unrest",
"elite_relocation": "financial",
"gps_jamming": "conflict",
"troop_movement": "conflict",
"ceasefire_break": "conflict",
}
@dataclass
class RegionState:
"""Per-region Bayesian beliefs and metadata."""
priors: dict[DomainName, float] = field(default_factory=lambda: defaultdict(float))
coords: list[float] | None = None
signal_volume: DefaultDict[str, float] = field(default_factory=lambda: defaultdict(float))
update_count: int = 0
@dataclass
class HistoryEntry:
timestamp: str
domain: DomainName
signals: dict[str, float]
strength: float
prior: float
posterior: float
source: str
deviation_score: float
class GT_EarlyWarning:
"""
Game-Theoretic Early Warning System with Bayesian updating.
Tracks distress probabilities per region/domain, classifies costly signals vs
cheap talk, and propagates risk through an entity interaction graph.
"""
def __init__(self, settings: GTAnalyticsSettings | None = None) -> None:
self.settings = settings or get_gt_settings()
self.G: nx.Graph = nx.Graph()
self._regions: dict[str, RegionState] = {}
self._history: dict[str, list[HistoryEntry]] = defaultdict(list)
self._seen_item_ids: set[str] = set()
self._lock = threading.RLock()
self.likelihoods = dict(_DEFAULT_LIKELIHOODS)
self.signal_weights = dict(_DEFAULT_SIGNAL_WEIGHTS)
self.signal_weights.update(self.settings.signal_weight_overrides)
self._base_prior = float(self.settings.base_prior)
def _utcnow(self) -> str:
return datetime.now(timezone.utc).isoformat()
def _region_state(self, region: str) -> RegionState:
key = str(region or "global").strip().lower() or "global"
if key not in self._regions:
state = RegionState()
for domain in _DOMAINS:
state.priors[domain] = self._base_prior
self._regions[key] = state
return self._regions[key]
def get_prior(self, region: str, domain: DomainName) -> float:
with self._lock:
return float(self._region_state(region).priors.get(domain, self._base_prior))
def set_prior(self, region: str, domain: DomainName, value: float) -> None:
with self._lock:
state = self._region_state(region)
state.priors[domain] = float(
np.clip(value, self.settings.min_prob, self.settings.max_prob)
)
def composite_risk(self, region: str) -> float:
"""Weighted composite across domains (conflict weighted highest)."""
weights = {"financial": 0.25, "unrest": 0.35, "conflict": 0.40}
with self._lock:
state = self._region_state(region)
total = 0.0
weight_sum = 0.0
for domain, weight in weights.items():
total += float(state.priors.get(domain, self._base_prior)) * weight
weight_sum += weight
return float(total / weight_sum) if weight_sum else self._base_prior
def classify_signals(self, text: str, source: str = "") -> dict[str, float]:
"""Return weighted costly-signal strengths detected in text."""
text_lower = (text or "").lower()
signals: dict[str, float] = {}
for signal_name, patterns in _SIGNAL_PATTERNS.items():
weight = float(self.signal_weights.get(signal_name, 1.0))
if any(pattern.search(text_lower) for pattern in patterns):
signals[signal_name] = weight
rally_strike_count = text_lower.count("rally") + text_lower.count("strike")
if rally_strike_count > 3:
signals["protest_mobilize"] = signals.get("protest_mobilize", 0.0) + 1.5
# Source credibility nudge (Telegram OSINT channels treated as moderate-cost signals).
if source and "t.me/" in source.lower() and signals:
for key in list(signals):
signals[key] = round(signals[key] * 1.05, 3)
return signals
def _deviation_score(self, region: str, domain: DomainName, strength: float) -> float:
"""Deviation from rolling regional norm — herding/coordination detector input."""
with self._lock:
state = self._region_state(region)
baseline = max(state.signal_volume[domain], 1.0)
state.signal_volume[domain] += strength
state.update_count += 1
return float(strength / baseline)
def bayesian_update(
self,
region: str,
domain: DomainName,
evidence_strength: float = 1.0,
) -> float:
"""
Bayesian update: P(distress|evidence) from likelihood table and prior.
evidence_strength scales how far belief moves toward the likelihood posterior.
"""
domain = domain if domain in _DOMAINS else "financial"
lik = self.likelihoods.get(domain, self.likelihoods["financial"])
with self._lock:
state = self._region_state(region)
prior = float(state.priors.get(domain, self._base_prior))
p_e_given_d = lik["distress"]
p_e_given_not_d = lik["normal"]
p_e = (p_e_given_d * prior) + (p_e_given_not_d * (1.0 - prior))
if p_e <= 0:
posterior = prior
else:
posterior = (p_e_given_d * prior) / p_e
scaled = prior + (posterior - prior) * float(evidence_strength)
clipped = float(np.clip(scaled, self.settings.min_prob, self.settings.max_prob))
state.priors[domain] = clipped
return clipped
def _update_graph(
self,
region: str,
entities: list[str],
strength: float,
coords: list[float] | None,
) -> None:
region_key = str(region or "global").strip().lower() or "global"
self.G.add_node(region_key, node_type="region", region=region_key)
if coords and len(coords) >= 2:
self.G.nodes[region_key]["coords"] = coords
for entity in entities:
entity_key = str(entity).strip()
if not entity_key:
continue
self.G.add_node(entity_key, node_type="entity", region=region_key)
self.G.add_edge(
region_key,
entity_key,
weight=float(strength),
timestamp=self._utcnow(),
)
for i, e1 in enumerate(entities):
for e2 in entities[i + 1 :]:
k1, k2 = str(e1).strip(), str(e2).strip()
if not k1 or not k2:
continue
self.G.add_edge(
k1,
k2,
weight=float(strength),
timestamp=self._utcnow(),
)
def process_feed_item(self, item: dict[str, Any]) -> dict[str, Any]:
"""Process one normalized feed item and update beliefs + contagion graph."""
region = str(item.get("region") or item.get("geotag") or "global").strip().lower()
text = str(item.get("text") or "")
source = str(item.get("source") or "unknown")
explicit_domain = str(item.get("domain") or "").strip().lower()
entities = list(item.get("entities") or [])
coords = item.get("coords")
item_id = str(item.get("id") or f"{source}|{hash(text)}")
if self.settings.watched_channels:
channel = ""
for entity in entities:
if str(entity).startswith("channel:"):
channel = str(entity).split(":", 1)[-1].lower()
break
if channel and channel not in {c.lower() for c in self.settings.watched_channels}:
return {
"region": region,
"skipped": True,
"reason": "channel_not_watched",
"risk_score": self.composite_risk(region),
"signals": {},
}
with self._lock:
if item_id and item_id in self._seen_item_ids:
return {
"region": region,
"skipped": True,
"reason": "duplicate",
"risk_score": self.composite_risk(region),
"signals": {},
}
if item_id:
self._seen_item_ids.add(item_id)
signals = self.classify_signals(text, source)
total_strength = float(sum(signals.values()))
if total_strength <= 0:
return {
"region": region,
"risk_score": self.composite_risk(region),
"signals": {},
"contagion_potential": self._get_contagion_score(region),
}
domains_touched: set[DomainName] = set()
if explicit_domain in _DOMAINS:
domains_touched.add(explicit_domain)
for signal_name in signals:
domains_touched.add(_SIGNAL_DOMAINS.get(signal_name, explicit_domain or "financial"))
if not domains_touched:
domains_touched.add("financial")
evidence_strength = min(
total_strength / max(self.settings.evidence_scale, 0.1),
self.settings.evidence_cap,
)
posteriors: dict[str, float] = {}
deviation = 0.0
for domain in domains_touched:
prior = self.get_prior(region, domain)
deviation = max(deviation, self._deviation_score(region, domain, total_strength))
posterior = self.bayesian_update(
region=region,
domain=domain,
evidence_strength=evidence_strength * (1.0 + 0.15 * deviation),
)
posteriors[domain] = posterior
if isinstance(coords, (list, tuple)) and len(coords) >= 2:
with self._lock:
state = self._region_state(region)
try:
state.coords = [float(coords[0]), float(coords[1])]
except (TypeError, ValueError):
pass
self._update_graph(region, entities, total_strength, coords if isinstance(coords, list) else None)
composite = self.composite_risk(region)
entry = HistoryEntry(
timestamp=self._utcnow(),
domain=explicit_domain if explicit_domain in _DOMAINS else next(iter(domains_touched)),
signals=signals,
strength=total_strength,
prior=self._base_prior,
posterior=composite,
source=source,
deviation_score=deviation,
)
with self._lock:
history = self._history[region]
history.append(entry)
max_hist = max(10, int(self.settings.max_history_per_region))
if len(history) > max_hist:
self._history[region] = history[-max_hist:]
logger.info(
"GT update region=%s domains=%s composite=%.3f signals=%d deviation=%.2f",
region,
",".join(sorted(domains_touched)),
composite,
len(signals),
deviation,
)
return {
"region": region,
"domains": sorted(domains_touched),
"domain_posteriors": posteriors,
"risk_score": composite,
"signals": signals,
"deviation_score": deviation,
"contagion_potential": self._get_contagion_score(region),
"interpretation": self._interpret_risk(composite),
}
def _interpret_risk(self, risk: float) -> str:
threshold = float(self.settings.high_risk_threshold)
if risk >= threshold:
return (
f"Elevated strategic risk ({risk:.2f}{threshold:.2f}). "
"Watch for costly-signal clustering and cross-region contagion."
)
if risk >= threshold * 0.7:
return "Moderate risk — monitor for herding and repeated costly signals."
return "Baseline risk — no strong costly-signal cluster detected."
def _get_contagion_score(self, region: str) -> float:
"""Graph-based contagion: mean composite risk of graph neighbors."""
region_key = str(region or "global").strip().lower() or "global"
with self._lock:
if region_key not in self.G:
return 0.0
try:
neighbors = list(self.G.neighbors(region_key))
except nx.NetworkXError:
return 0.0
if not neighbors:
return 0.0
neighbor_risks = [self.composite_risk(str(n)) for n in neighbors]
return float(np.mean(neighbor_risks))
def compute_herding_clusters(self) -> list[dict[str, Any]]:
"""Louvain community detection on entity graph (coordination/herding proxy)."""
with self._lock:
if self.G.number_of_edges() == 0:
return []
weighted = nx.Graph()
for u, v, data in self.G.edges(data=True):
weight = float(data.get("weight") or 0.0)
if weight < self.settings.louvain_min_weight:
continue
if weighted.has_edge(u, v):
weighted[u][v]["weight"] = weighted[u][v].get("weight", 0.0) + weight
else:
weighted.add_edge(u, v, weight=weight)
if weighted.number_of_edges() == 0:
return []
try:
communities = list(nx.community.louvain_communities(weighted, weight="weight", seed=42))
except Exception as exc:
logger.warning("Louvain clustering failed: %s", exc)
return []
clusters: list[dict[str, Any]] = []
for idx, community in enumerate(communities):
members = sorted(str(node) for node in community)
region_members = [m for m in members if m in self._regions]
risks = [self.composite_risk(r) for r in region_members]
clusters.append(
{
"cluster_id": idx,
"size": len(members),
"members": members[:50],
"mean_risk": float(np.mean(risks)) if risks else self._base_prior,
"regions": region_members,
}
)
clusters.sort(key=lambda row: row["mean_risk"], reverse=True)
return clusters
def get_risk_heatmap(self) -> dict[str, Any]:
"""GeoJSON FeatureCollection for frontend risk overlay."""
features: list[dict[str, Any]] = []
with self._lock:
items = list(self._regions.items())[: max(1, self.settings.max_heatmap_features)]
for region, state in items:
coords = state.coords
geometry: dict[str, Any]
if coords and len(coords) >= 2:
geometry = {"type": "Point", "coordinates": [float(coords[1]), float(coords[0])]}
else:
geometry = {"type": "Point", "coordinates": [0.0, 0.0]}
composite = self.composite_risk(region)
features.append(
{
"type": "Feature",
"properties": {
"region": region,
"risk": round(composite, 4),
"financial": round(float(state.priors.get("financial", self._base_prior)), 4),
"unrest": round(float(state.priors.get("unrest", self._base_prior)), 4),
"conflict": round(float(state.priors.get("conflict", self._base_prior)), 4),
"contagion": round(self._get_contagion_score(region), 4),
"updates": state.update_count,
},
"geometry": geometry,
}
)
return {"type": "FeatureCollection", "features": features}
def get_dossier(self, region: str) -> dict[str, Any]:
"""Explainable GT rationale and recent signal history for a region."""
region_key = str(region or "global").strip().lower() or "global"
with self._lock:
state = self._region_state(region_key)
recent = list(self._history.get(region_key, [])[-10:])
composite = self.composite_risk(region_key)
return {
"region": region_key,
"current_risk": round(composite, 4),
"domain_risks": {
domain: round(float(state.priors.get(domain, self._base_prior)), 4)
for domain in _DOMAINS
},
"recent_signals": [
{
"timestamp": entry.timestamp,
"domain": entry.domain,
"signals": entry.signals,
"strength": entry.strength,
"posterior": round(entry.posterior, 4),
"source": entry.source,
"deviation_score": round(entry.deviation_score, 3),
}
for entry in recent
],
"contagion_risk": round(self._get_contagion_score(region_key), 4),
"herding_clusters": self.compute_herding_clusters()[:5],
"interpretation": self._interpret_risk(composite),
"scenarios": self._build_scenarios(region_key, composite),
}
def _build_scenarios(self, region: str, composite: float) -> list[dict[str, str]]:
threshold = float(self.settings.high_risk_threshold)
if composite < threshold * 0.7:
return [
{
"name": "Status quo",
"summary": "Signals remain diffuse; no coordinated costly-signal cascade.",
}
]
if composite < threshold:
return [
{
"name": "Escalation watch",
"summary": "Rising costly-signal density — coordination risk within 4-8 weeks.",
},
{
"name": "False alarm",
"summary": "Cheap-talk amplification without follow-on costly signals.",
},
]
return [
{
"name": "Contagion spread",
"summary": "High posterior + graph coupling — adjacent regions likely to update upward.",
},
{
"name": "Localized shock",
"summary": "Region-specific distress; contagion limited if graph neighbors stay quiet.",
},
]
def snapshot(self) -> dict[str, Any]:
"""Serialize engine state for debugging or persistence."""
with self._lock:
return {
"regions": {
region: {
"priors": dict(state.priors),
"coords": state.coords,
"updates": state.update_count,
}
for region, state in self._regions.items()
},
"graph_nodes": self.G.number_of_nodes(),
"graph_edges": self.G.number_of_edges(),
"processed_items": len(self._seen_item_ids),
}
+649
View File
@@ -0,0 +1,649 @@
"""Curated historical early-warning cases for GT backtesting.
Each positive case bundles pre-crisis costly-signal snippets drawn from documented
precursors (financial, unrest, conflict). Negative cases are cheap-talk controls.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Literal
CaseKind = Literal["positive", "negative"]
@dataclass(frozen=True)
class BacktestFeed:
text: str
source: str = "backtest"
domain: str = "financial"
days_before_event: int = 30
@dataclass(frozen=True)
class HistoricalCase:
"""Single labeled backtest scenario."""
case_id: str
name: str
region: str
domain: str
kind: CaseKind
event_date: str
description: str
feeds: tuple[BacktestFeed, ...] = field(default_factory=tuple)
tags: tuple[str, ...] = field(default_factory=tuple)
def to_feed_dicts(self) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
for idx, feed in enumerate(self.feeds):
items.append(
{
"id": f"{self.case_id}-{idx}",
"text": feed.text,
"source": feed.source,
"region": self.region,
"domain": feed.domain or self.domain,
"published": feed.days_before_event,
}
)
return items
def _variant_case(case: HistoricalCase, suffix: str, feeds: tuple[BacktestFeed, ...]) -> HistoricalCase:
return HistoricalCase(
case_id=f"{case.case_id}__{suffix}",
name=f"{case.name} ({suffix})",
region=case.region,
domain=case.domain,
kind=case.kind,
event_date=case.event_date,
description=case.description,
feeds=feeds,
tags=case.tags + (f"variant:{suffix}",),
)
def expanded_historical_cases() -> tuple[HistoricalCase, ...]:
"""Base suite plus paraphrase variants for statistical confidence."""
base = list(default_historical_cases())
extras: list[HistoricalCase] = []
variant_feeds: dict[str, tuple[tuple[BacktestFeed, ...], ...]] = {
"fin_2008_us": (
(
BacktestFeed(
"Small businesses turn to payroll loan products as credit lines freeze.",
domain="financial",
days_before_event=100,
),
BacktestFeed(
"FDIC monitors liquidity crunch; interbank spreads widen sharply.",
domain="financial",
days_before_event=60,
),
),
(
BacktestFeed(
"Merchant cash advance volumes spike; payroll loan demand at record highs.",
domain="financial",
days_before_event=80,
),
BacktestFeed(
"Money market funds see inflows as deposit flight from regional banks continues.",
domain="financial",
days_before_event=40,
),
),
),
"fin_2020_supply": (
(
BacktestFeed(
"Electronics firms report shipping delay and port congestion across Pearl River Delta.",
domain="financial",
days_before_event=45,
),
BacktestFeed(
"Supply chain delay widens; logistics backlog hits automotive suppliers.",
domain="financial",
days_before_event=20,
),
),
(
BacktestFeed(
"Container shortage fuels shipping delay; supply chain delay indices jump.",
domain="financial",
days_before_event=35,
),
BacktestFeed(
"Electronics assemblers warn of logistics backlog as port congestion spreads.",
domain="financial",
days_before_event=20,
),
BacktestFeed(
"Automotive suppliers flag supply chain delay after factory shutdowns in Hubei.",
domain="financial",
days_before_event=10,
),
),
),
"fin_2022_sanctions": (
(
BacktestFeed(
"Treasury drafts new sanctions escalation package on energy and finance sectors.",
domain="financial",
days_before_event=30,
),
BacktestFeed(
"Capital flight accelerates; elite relocation flights depart Moscow airports.",
domain="financial",
days_before_event=14,
),
),
),
"unrest_arab_spring_egypt": (
(
BacktestFeed(
"Cairo activists schedule mass rally; protest mobilization leaflets distributed.",
domain="unrest",
days_before_event=18,
),
BacktestFeed(
"Labor federations call general strike; strike posters cover downtown.",
domain="unrest",
days_before_event=8,
),
),
),
"conflict_2022_ukraine": (
(
BacktestFeed(
"Convoy of armored vehicles confirms troop movement near Sumy Oblast.",
source="t.me/war_monitor",
domain="conflict",
days_before_event=20,
),
BacktestFeed(
"GNSS interference warnings follow GPS jamming spike along Belarus border.",
source="t.me/osintdefender",
domain="conflict",
days_before_event=10,
),
),
(
BacktestFeed(
"Military mobilization notices circulate; troop buildup confirmed by satellite firms.",
domain="conflict",
days_before_event=12,
),
),
),
"neg_weather_us": (
(
BacktestFeed("Autumn foliage peaks in Vermont; pleasant hiking weather continues."),
BacktestFeed("County fair announces pie contest and livestock exhibitions."),
),
(
BacktestFeed("Meteorologists predict mild hurricane season remainder for Gulf Coast."),
),
),
"neg_sports_uk": (
(
BacktestFeed("Rugby Six Nations standings update after weekend fixtures."),
BacktestFeed("Local marathon registration opens for charity runners."),
),
),
"neg_tech_global": (
(
BacktestFeed("Chipmaker announces efficiency gains in next-generation processor."),
BacktestFeed("Cloud provider opens new green datacenter in Nordic region."),
),
),
}
for case in base:
variants = variant_feeds.get(case.case_id, ())
for idx, feeds in enumerate(variants):
extras.append(_variant_case(case, f"v{idx+1}", feeds))
# Additional cheap-talk controls to widen negative sample
cheap_talk_regions = (
("australia", "Museum opens contemporary art exhibit to strong attendance."),
("spain", "Tomato harvest festival scheduled; regional trains add weekend service."),
("south_korea", "K-pop group announces world tour dates for autumn."),
("mexico", "Coastal cleanup volunteers restore beach habitats before holiday season."),
("sweden", "City council approves bike lane expansion along waterfront."),
("norway", "Salmon exports remain stable; fishing fleets report normal catch volumes."),
("italy", "Truffle festival returns; restaurants publish seasonal tasting menus."),
("poland", "University researchers release open-source astronomy software."),
("thailand", "Monsoon rains ease; rice planting proceeds on normal schedule."),
("vietnam", "Electronics assembly plants report steady export order books."),
("south_africa", "Wildlife reserve reports rising ecotourism bookings."),
("argentina", "Wine harvest festival opens; export cooperatives meet volume targets."),
("netherlands", "Cycling championship draws international teams to canal district."),
("belgium", "Chocolate exporters report stable holiday shipment schedules."),
("portugal", "Offshore wind auction attracts multiple renewable bidders."),
("greece", "Island ferry operators add routes ahead of summer travel season."),
("turkey", "Cotton harvest forecast unchanged; textile orders stable."),
("indonesia", "Volcano monitoring reports routine activity; tourism continues."),
("philippines", "Coconut processors report normal logistics to export markets."),
("malaysia", "Palm oil shipments on schedule; port throughput normal."),
("new_zealand", "Sheep shearing competition draws rural crowds."),
("ireland", "Tech conference highlights open-source database tooling."),
("finland", "Sauna culture festival celebrates heritage with local artisans."),
("denmark", "Wind turbine maintenance contracts renewed on prior terms."),
("austria", "Ski resorts prepare slopes after early snowfall."),
("switzerland", "Watchmakers unveil mechanical movement prototypes at trade fair."),
("czech_republic", "Glassmakers export decorative pieces ahead of holiday season."),
("romania", "Carpathian hiking trails reopen after spring maintenance."),
("hungary", "Thermal bath tourism bookings rise for winter wellness season."),
("peru", "Coffee cooperatives report stable harvest and export schedules."),
("colombia", "Flower exporters prepare Valentine's shipments on normal cadence."),
("morocco", "Citrus harvest meets forecasts; agricultural credit unchanged."),
("kenya", "Tea auction volumes steady; freight routes operate normally."),
("nigeria", "Nollywood studio announces family comedy release dates."),
("ethiopia", "Coffee ceremony festival highlights regional bean varieties."),
("saudi_arabia", "Desert conservation project plants drought-resistant shrubs."),
("uae", "Airport duty-free operators expand luxury retail concourse."),
("qatar", "Stadium operators prepare hospitality packages for sporting events."),
("singapore", "Port authority reports container throughput on seasonal trend."),
("hong_kong", "Art auction previews draw collectors to harborfront gallery."),
("chile", "Vineyard tours report strong bookings ahead of harvest festival weekend."),
("uruguay", "Beef exporters maintain steady shipment schedules to European buyers."),
("iceland", "Geothermal spa resorts report normal winter visitor volumes."),
("luxembourg", "Fund administrators publish routine quarterly disclosure filings."),
("slovakia", "Mountain lodges prepare ski season openings after early snowfall."),
("croatia", "Adriatic ferry operators add summer routes on prior timetable."),
("bulgaria", "Rose oil cooperatives report stable export volumes to fragrance buyers."),
("serbia", "Danube barge traffic proceeds on normal freight schedules."),
("latvia", "Timber mills export lumber on unchanged contract terms."),
("lithuania", "Baltic wind farms complete scheduled turbine maintenance rotations."),
("estonia", "Digital residency applications processed at routine monthly pace."),
("panama", "Canal transit volumes remain on seasonal trend; shipping fees unchanged."),
)
for idx, (region, text) in enumerate(cheap_talk_regions):
extras.append(
HistoricalCase(
case_id=f"neg_extra_{idx:02d}",
name=f"Benign regional news ({region})",
region=region,
domain="financial",
kind="negative",
event_date="2020-01-01",
description="Expanded cheap-talk control.",
feeds=(BacktestFeed(text),),
tags=("control", "expanded"),
)
)
return tuple(base + extras)
def default_historical_cases() -> tuple[HistoricalCase, ...]:
"""Benchmark suite — expand as new validated precursors are added."""
return (
# ── Financial distress ─────────────────────────────────────────────
HistoricalCase(
case_id="fin_2008_us",
name="2008 US financial crisis",
region="united_states",
domain="financial",
kind="positive",
event_date="2008-09-15",
description="Payroll-loan distress, liquidity crunch, and deposit flight precursors.",
tags=("2008", "financial", "lehman"),
feeds=(
BacktestFeed(
"Franchise operators increasingly rely on payroll loan facilities as working capital tightens.",
domain="financial",
days_before_event=120,
),
BacktestFeed(
"Regional banks report liquidity crunch; CFOs warn of merchant cash advance reliance.",
domain="financial",
days_before_event=90,
),
BacktestFeed(
"Deposit flight accelerates at mid-size lenders; analysts flag bank run risk.",
domain="financial",
days_before_event=45,
),
),
),
HistoricalCase(
case_id="fin_2020_supply",
name="COVID supply-chain shock",
region="china",
domain="financial",
kind="positive",
event_date="2020-02-01",
description="Port congestion and logistics backlog ahead of global supply shock.",
tags=("covid", "supply_chain", "financial"),
feeds=(
BacktestFeed(
"Major port congestion reported; shipping delay spreads to electronics suppliers.",
domain="financial",
days_before_event=60,
),
BacktestFeed(
"Automakers warn of supply chain delay and logistics backlog across Wuhan corridor.",
domain="financial",
days_before_event=30,
),
BacktestFeed(
"Factory restarts slip as supply delay and port congestion persist into Q1.",
domain="financial",
days_before_event=14,
),
),
),
HistoricalCase(
case_id="fin_2022_sanctions",
name="Russia sanctions escalation",
region="russia",
domain="financial",
kind="positive",
event_date="2022-02-24",
description="Sanctions escalation and capital flight ahead of invasion.",
tags=("sanctions", "ukraine", "financial"),
feeds=(
BacktestFeed(
"Western allies prepare new sanctions escalation on major Russian banks.",
domain="financial",
days_before_event=45,
),
BacktestFeed(
"Oligarch jet movements suggest elite relocation and capital flight from Moscow.",
domain="financial",
days_before_event=21,
),
BacktestFeed(
"Central bank intervenes as new sanctions tighten export controls on finance sector.",
domain="financial",
days_before_event=10,
),
),
),
# ── Civil unrest ─────────────────────────────────────────────────
HistoricalCase(
case_id="unrest_arab_spring_tunisia",
name="Arab Spring — Tunisia",
region="tunisia",
domain="unrest",
kind="positive",
event_date="2010-12-17",
description="Protest mobilization and strike waves before Jasmine Revolution.",
tags=("arab_spring", "unrest"),
feeds=(
BacktestFeed(
"Student groups announce protest mobilization after vendor self-immolation.",
domain="unrest",
days_before_event=14,
),
BacktestFeed(
"Mass rally planned in Tunis; general strike called by labor unions.",
domain="unrest",
days_before_event=7,
),
),
),
HistoricalCase(
case_id="unrest_arab_spring_egypt",
name="Arab Spring — Egypt",
region="egypt",
domain="unrest",
kind="positive",
event_date="2011-01-25",
description="Mobilization spikes and security reshuffles before Tahrir.",
tags=("arab_spring", "unrest"),
feeds=(
BacktestFeed(
"Opposition calls protest mobilization in Cairo; strike notices circulate online.",
domain="unrest",
days_before_event=21,
),
BacktestFeed(
"Reports of political purge within interior ministry security apparatus reshuffle.",
domain="unrest",
days_before_event=10,
),
BacktestFeed(
"Mass rally and strike coordination spreads; rally posters appear in Alexandria.",
domain="unrest",
days_before_event=5,
),
),
),
HistoricalCase(
case_id="unrest_2019_chile",
name="Chile 2019 metro protests",
region="chile",
domain="unrest",
kind="positive",
event_date="2019-10-18",
description="Transit fare protests escalate to general strike.",
tags=("unrest", "latam"),
feeds=(
BacktestFeed(
"Students organize mass rally after metro fare hike; protest mobilization trending.",
domain="unrest",
days_before_event=10,
),
BacktestFeed(
"Unions announce general strike; rally and strike hashtags spike nationwide.",
domain="unrest",
days_before_event=3,
),
),
),
# ── Conflict / war ───────────────────────────────────────────────
HistoricalCase(
case_id="conflict_2022_ukraine",
name="2022 Ukraine invasion buildup",
region="ukraine",
domain="conflict",
kind="positive",
event_date="2022-02-24",
description="Troop movement and GPS jamming precursors on northern border.",
tags=("ukraine", "conflict"),
feeds=(
BacktestFeed(
"OSINT reports troop movement and armored convoy near Belarus border.",
source="t.me/war_monitor",
domain="conflict",
days_before_event=30,
),
BacktestFeed(
"GPS jamming spike reported along northern corridor; GNSS interference warnings issued.",
source="t.me/osintdefender",
domain="conflict",
days_before_event=14,
),
BacktestFeed(
"Satellite imagery shows troop buildup; military mobilization near Kharkiv axis.",
domain="conflict",
days_before_event=7,
),
),
),
HistoricalCase(
case_id="conflict_2023_gaza",
name="2023 Gaza conflict escalation",
region="israel",
domain="conflict",
kind="positive",
event_date="2023-10-07",
description="Ceasefire breakdown and troop movement signals.",
tags=("gaza", "conflict"),
feeds=(
BacktestFeed(
"Border units report troop movement near Gaza envelope; ceasefire broken overnight.",
domain="conflict",
days_before_event=14,
),
BacktestFeed(
"Truce end announced; armored convoy repositioning reported by local observers.",
domain="conflict",
days_before_event=5,
),
),
),
HistoricalCase(
case_id="conflict_2020_nagorno",
name="2020 Nagorno-Karabakh renewal",
region="armenia",
domain="conflict",
kind="positive",
event_date="2020-09-27",
description="Artillery and troop buildup precursors.",
tags=("caucasus", "conflict"),
feeds=(
BacktestFeed(
"Drone strikes reported on line of contact; troop movement on Armenian-Azeri border.",
domain="conflict",
days_before_event=21,
),
BacktestFeed(
"GPS jamming spike reported in conflict zone; military mobilization notices leaked.",
domain="conflict",
days_before_event=7,
),
),
),
# ── Recent financial / corporate distress pattern ────────────────
HistoricalCase(
case_id="fin_2023_banking",
name="2023 regional banking stress",
region="united_states",
domain="financial",
kind="positive",
event_date="2023-03-10",
description="Deposit flight and liquidity stress (SVB precursor pattern).",
tags=("svb", "financial", "2023"),
feeds=(
BacktestFeed(
"Tech lenders face deposit flight; VC portfolio companies move payroll to money market funds.",
domain="financial",
days_before_event=21,
),
BacktestFeed(
"Analysts warn liquidity crunch at regional banks holding long-duration bonds.",
domain="financial",
days_before_event=7,
),
),
),
# ── Negative controls (cheap talk / benign) ─────────────────────
HistoricalCase(
case_id="neg_weather_us",
name="Benign weather coverage",
region="united_states",
domain="financial",
kind="negative",
event_date="2019-06-01",
description="No costly signals — should remain near baseline.",
tags=("control",),
feeds=(
BacktestFeed("Sunny weekend expected across the Midwest with mild temperatures."),
BacktestFeed("Local festival draws crowds; farmers market expands summer hours."),
),
),
HistoricalCase(
case_id="neg_sports_uk",
name="Benign sports coverage",
region="uk",
domain="unrest",
kind="negative",
event_date="2018-07-01",
description="Sports chatter without mobilization costly signals.",
tags=("control",),
feeds=(
BacktestFeed("Premier league season review: top scorers and transfer rumors."),
BacktestFeed("Cricket test match ends early due to rain delay at Lord's."),
),
),
HistoricalCase(
case_id="neg_tech_global",
name="Benign tech product launch",
region="global",
domain="financial",
kind="negative",
event_date="2021-09-01",
description="Corporate product news without distress markers.",
tags=("control",),
feeds=(
BacktestFeed("Smartphone maker unveils new camera features at annual keynote."),
BacktestFeed("Quarterly earnings beat expectations; dividend unchanged."),
),
),
HistoricalCase(
case_id="neg_tourism_france",
name="Benign tourism recovery",
region="france",
domain="unrest",
kind="negative",
event_date="2022-08-01",
description="Travel sector recovery without unrest signals.",
tags=("control",),
feeds=(
BacktestFeed("Paris hotels report record summer bookings as tourism rebounds."),
BacktestFeed("Airline adds routes to Nice and Marseille for holiday travelers."),
),
),
HistoricalCase(
case_id="neg_science_japan",
name="Benign science news",
region="japan",
domain="conflict",
kind="negative",
event_date="2020-11-01",
description="Research coverage without conflict markers.",
tags=("control",),
feeds=(
BacktestFeed("Astronomy team publishes comet observations from Mount Fuji observatory."),
BacktestFeed("Robotics lab demonstrates warehouse automation prototype."),
),
),
HistoricalCase(
case_id="neg_agriculture_brazil",
name="Benign agriculture report",
region="brazil",
domain="financial",
kind="negative",
event_date="2017-03-01",
description="Commodity harvest update without supply distress.",
tags=("control",),
feeds=(
BacktestFeed("Soybean harvest forecast revised upward; export volumes steady."),
BacktestFeed("Coffee cooperative reports normal shipping schedules to European buyers."),
),
),
HistoricalCase(
case_id="neg_culture_india",
name="Benign culture coverage",
region="india",
domain="unrest",
kind="negative",
event_date="2016-11-01",
description="Festival coverage without mobilization.",
tags=("control",),
feeds=(
BacktestFeed("Diwali celebrations begin; cities decorate markets with lights."),
BacktestFeed("Film festival opens in Mumbai with premiere screenings."),
),
),
HistoricalCase(
case_id="neg_infrastructure_canada",
name="Benign infrastructure ribbon-cutting",
region="canada",
domain="financial",
kind="negative",
event_date="2015-05-01",
description="Municipal news without financial stress.",
tags=("control",),
feeds=(
BacktestFeed("New light-rail segment opens on schedule; commute times improve."),
BacktestFeed("Municipal bond issuance funds library renovation at prior rates."),
),
),
)
+198
View File
@@ -0,0 +1,198 @@
"""Singleton GT engine and feed-batch integration hooks."""
from __future__ import annotations
import logging
import threading
from datetime import datetime, timezone
from typing import Any
from analytics.feed_adapter import iter_gdelt_features, iter_news_items, iter_telegram_posts
from analytics.gt_early_warning import GT_EarlyWarning
from analytics.settings import gt_analytics_enabled, get_gt_settings, gt_engine_operational, gt_louvain_enabled, gt_scheduled_ingest_enabled
from services.fetchers._store import _data_lock, _mark_fresh, latest_data
logger = logging.getLogger(__name__)
_engine: GT_EarlyWarning | None = None
_engine_lock = threading.Lock()
def get_gt_engine() -> GT_EarlyWarning | None:
"""Return the shared engine when analytics are enabled and runtime allows it."""
global _engine
if not gt_engine_operational():
return None
with _engine_lock:
if _engine is None:
_engine = GT_EarlyWarning(get_gt_settings())
logger.info("Strategic Risk Analytics engine initialized")
return _engine
def reset_gt_engine() -> None:
"""Reset singleton — intended for tests."""
global _engine
get_gt_settings.cache_clear()
with _engine_lock:
_engine = None
def process_feed_item(item: dict[str, Any]) -> dict[str, Any] | None:
"""Process a normalized feed item if analytics are enabled."""
engine = get_gt_engine()
if engine is None:
return None
try:
return engine.process_feed_item(item)
except Exception:
logger.exception("GT process_feed_item failed")
return None
def _persist_gt_snapshot(
engine: GT_EarlyWarning,
*,
processed: int,
sample: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
timestamp = datetime.now(timezone.utc).isoformat()
heatmap = engine.get_risk_heatmap()
micro_summary: dict[str, Any] = {}
try:
from analytics.micro_rolling import capture_daily_readings, enrich_heatmap_features
micro_summary = capture_daily_readings(engine)
heatmap = enrich_heatmap_features(heatmap)
except Exception:
logger.exception("GT micro rolling capture failed")
clusters = engine.compute_herding_clusters()
from analytics.gt_alerts import parse_heatmap_alerts
_, plotted_regions = parse_heatmap_alerts(heatmap)
with engine._lock: # noqa: SLF001 — snapshot meta
engine_regions = len(engine._regions)
settings = get_gt_settings()
payload = {
"enabled": True,
"timestamp": timestamp,
"processed": processed,
"heatmap": heatmap,
"clusters": clusters,
"sample": list(sample or [])[:5],
"regions": len(heatmap.get("features") or []),
"micro": micro_summary,
"meta": {
"tracked_regions": len(heatmap.get("features") or []),
"engine_regions": engine_regions,
"plotted_regions": plotted_regions,
"max_regions": settings.max_heatmap_features,
},
}
with _data_lock:
latest_data["gt_risk"] = payload
_mark_fresh("gt_risk")
return payload
def refresh_from_latest_data(
data_snapshot: dict[str, Any],
*,
persist: bool = True,
) -> dict[str, Any]:
"""
Batch-ingest recent intel layers from the shared data store.
Intended to run after telegram/news/gdelt fetch cycles (near-real-time).
"""
engine = get_gt_engine()
if engine is None:
return {"enabled": False, "processed": 0}
processed = 0
results: list[dict[str, Any]] = []
for item in iter_telegram_posts(data_snapshot.get("telegram_osint")):
result = engine.process_feed_item(item)
if result and not result.get("skipped"):
processed += 1
results.append(result)
for item in iter_news_items(data_snapshot.get("news")):
result = engine.process_feed_item(item)
if result and not result.get("skipped"):
processed += 1
if len(results) < 5:
results.append(result)
for item in iter_gdelt_features(data_snapshot.get("gdelt")):
result = engine.process_feed_item(item)
if result and not result.get("skipped"):
processed += 1
logger.info("GT refresh processed %d items", processed)
summary = {
"enabled": True,
"processed": processed,
"sample": results[:5],
"heatmap_features": len(engine.get_risk_heatmap().get("features") or []),
}
if persist:
snapshot = _persist_gt_snapshot(engine, processed=processed, sample=results)
summary["timestamp"] = snapshot.get("timestamp")
summary["clusters"] = len(snapshot.get("clusters") or [])
return summary
def recompute_gt_herding_clusters() -> dict[str, Any]:
"""Louvain community pass — run on a schedule independent of feed ingest."""
if not gt_louvain_enabled():
return {"enabled": False, "clusters": 0, "reason": "louvain_disabled_on_lean_profile"}
engine = get_gt_engine()
if engine is None:
return {"enabled": False, "clusters": 0}
clusters = engine.compute_herding_clusters()
timestamp = datetime.now(timezone.utc).isoformat()
with _data_lock:
current = dict(latest_data.get("gt_risk") or {})
current["clusters"] = clusters
current["clusters_updated"] = timestamp
current["enabled"] = True
latest_data["gt_risk"] = current
_mark_fresh("gt_risk")
logger.info("GT Louvain recompute: %d clusters", len(clusters))
return {"enabled": True, "clusters": len(clusters), "timestamp": timestamp}
def maybe_refresh_gt_analytics() -> None:
"""Hook for data_fetcher — no-op when analytics are disabled or lean-gated."""
if not gt_scheduled_ingest_enabled():
return
try:
with _data_lock:
snapshot = dict(latest_data)
refresh_from_latest_data(snapshot, persist=True)
except Exception:
logger.exception("GT analytics refresh failed")
def maybe_freeze_gt_weekly_snapshot() -> None:
"""Hook for weekly scheduler — freeze operational backtest snapshot."""
if not gt_engine_operational():
return
try:
from analytics.rolling_backtest import freeze_weekly_snapshot
result = freeze_weekly_snapshot(frozen_by="scheduler")
if result.get("created"):
logger.info(
"GT rolling freeze: week=%s regions=%s alerts=%s",
result.get("week_id"),
result.get("region_count"),
result.get("alert_count"),
)
except Exception:
logger.exception("GT rolling weekly freeze failed")
+361
View File
@@ -0,0 +1,361 @@
"""Micro rolling 3-day average — fast ignition signal alongside weekly macro."""
from __future__ import annotations
import os
from dataclasses import dataclass
from datetime import date, datetime, timedelta, timezone
from typing import Any
from analytics.daily_store import (
DailyRegionReading,
DailySnapshot,
date_id,
list_daily_ids,
load_daily,
save_daily,
utc_now_iso,
utc_today,
)
from analytics.gt_early_warning import GT_EarlyWarning
from analytics.rolling_backtest import rolling_alert_threshold
DEFAULT_WINDOW_DAYS = 3
DEFAULT_IGNITION_DELTA = 0.10
def _env_int(name: str, default: int) -> int:
raw = str(os.environ.get(name, "")).strip()
if not raw:
return default
try:
return max(1, int(raw))
except ValueError:
return default
def _env_float(name: str, default: float) -> float:
raw = str(os.environ.get(name, "")).strip()
if not raw:
return default
try:
return float(raw)
except ValueError:
return default
def micro_window_days() -> int:
return _env_int("GT_MICRO_ROLLING_DAYS", DEFAULT_WINDOW_DAYS)
def ignition_delta() -> float:
return _env_float("GT_MICRO_IGNITION_DELTA", DEFAULT_IGNITION_DELTA)
def _peak_score(
*,
composite: float,
financial: float,
unrest: float,
conflict: float,
) -> float:
return max(composite, financial, unrest, conflict)
def _region_reading_from_feature(
feature: dict[str, Any],
*,
captured_at: str,
) -> DailyRegionReading | None:
props = feature.get("properties") or {}
region = str(props.get("region") or "").strip().lower()
if not region:
return None
composite = float(props.get("risk") or props.get("composite_risk") or 0.0)
financial = float(props.get("financial") or 0.0)
unrest = float(props.get("unrest") or 0.0)
conflict = float(props.get("conflict") or 0.0)
peak = _peak_score(
composite=composite,
financial=financial,
unrest=unrest,
conflict=conflict,
)
return DailyRegionReading(
region=region,
composite_risk=composite,
financial=financial,
unrest=unrest,
conflict=conflict,
peak_score=peak,
readings=1,
last_captured_at=captured_at,
)
def capture_daily_readings(
engine: GT_EarlyWarning,
*,
when: date | None = None,
) -> dict[str, Any]:
"""
Upsert today's regional readings from the live heatmap.
Each GT refresh updates the current day's latest scores (rolling window
uses one value per calendar day).
"""
day = when or utc_today()
day_key = date_id(day)
captured_at = utc_now_iso()
heatmap = engine.get_risk_heatmap()
existing = load_daily(day) or DailySnapshot(date=day_key, regions={})
updated = 0
for feature in heatmap.get("features") or []:
if not isinstance(feature, dict):
continue
reading = _region_reading_from_feature(feature, captured_at=captured_at)
if reading is None:
continue
prior = existing.regions.get(reading.region)
if prior is None:
existing.regions[reading.region] = reading
updated += 1
continue
prior.composite_risk = reading.composite_risk
prior.financial = reading.financial
prior.unrest = reading.unrest
prior.conflict = reading.conflict
prior.peak_score = max(prior.peak_score, reading.peak_score)
prior.readings += 1
prior.last_captured_at = captured_at
updated += 1
existing.last_updated_at = captured_at
save_daily(existing)
return {
"date": day_key,
"regions": len(existing.regions),
"updated": updated,
"captured_at": captured_at,
}
@dataclass(frozen=True)
class MicroRegionView:
region: str
spot_risk: float
risk_3d_avg: float
risk_delta: float
days_in_window: int
day_scores: tuple[float, ...]
alerted_spot: bool
alerted_3d: bool
ignition: bool
financial: float
unrest: float
conflict: float
def to_dict(self) -> dict[str, Any]:
return {
"region": self.region,
"spot_risk": round(self.spot_risk, 4),
"risk_3d_avg": round(self.risk_3d_avg, 4),
"risk_delta": round(self.risk_delta, 4),
"days_in_window": self.days_in_window,
"day_scores": [round(score, 4) for score in self.day_scores],
"alerted_spot": self.alerted_spot,
"alerted_3d": self.alerted_3d,
"ignition": self.ignition,
"financial": round(self.financial, 4),
"unrest": round(self.unrest, 4),
"conflict": round(self.conflict, 4),
}
def _day_offsets(window_days: int) -> list[int]:
# Today + prior (window_days - 1) days.
return list(range(window_days - 1, -1, -1))
def _historical_dates(as_of: date, window_days: int) -> list[date]:
return [as_of - timedelta(days=offset) for offset in _day_offsets(window_days)]
def compute_micro_view(
region: str,
*,
as_of: date | None = None,
window_days: int | None = None,
alert_threshold: float | None = None,
spot_reading: DailyRegionReading | None = None,
) -> MicroRegionView | None:
"""Compute rolling N-day average and ignition vs spot for one region."""
region_key = str(region or "").strip().lower()
if not region_key:
return None
today = as_of or utc_today()
window = window_days or micro_window_days()
threshold = float(alert_threshold if alert_threshold is not None else rolling_alert_threshold())
delta_min = ignition_delta()
day_scores: list[float] = []
latest: DailyRegionReading | None = spot_reading
for day in _historical_dates(today, window):
snap = load_daily(day)
if snap is None:
continue
row = snap.regions.get(region_key)
if row is None:
continue
day_scores.append(row.peak_score)
if day == today:
latest = row
if latest is None and day_scores:
# Spot may come from yesterday if today not captured yet.
snap = load_daily(today)
if snap:
latest = snap.regions.get(region_key)
if latest is None and not day_scores:
return None
spot = float(latest.peak_score if latest else (day_scores[-1] if day_scores else 0.0))
avg = sum(day_scores) / len(day_scores) if day_scores else spot
risk_delta = spot - avg
ignition = risk_delta >= delta_min and spot >= threshold * 0.75
return MicroRegionView(
region=region_key,
spot_risk=spot,
risk_3d_avg=avg,
risk_delta=risk_delta,
days_in_window=len(day_scores),
day_scores=tuple(day_scores),
alerted_spot=spot >= threshold,
alerted_3d=avg >= threshold,
ignition=ignition,
financial=float(latest.financial if latest else 0.0),
unrest=float(latest.unrest if latest else 0.0),
conflict=float(latest.conflict if latest else 0.0),
)
def compute_all_micro_views(
*,
as_of: date | None = None,
window_days: int | None = None,
alert_threshold: float | None = None,
) -> list[MicroRegionView]:
"""Build micro views for all regions seen in the rolling window."""
today = as_of or utc_today()
window = window_days or micro_window_days()
regions: set[str] = set()
for day in _historical_dates(today, window):
snap = load_daily(day)
if snap is None:
continue
regions.update(snap.regions.keys())
views: list[MicroRegionView] = []
for region in regions:
view = compute_micro_view(
region,
as_of=today,
window_days=window,
alert_threshold=alert_threshold,
)
if view is not None:
views.append(view)
views.sort(key=lambda row: (row.ignition, row.risk_delta, row.spot_risk), reverse=True)
return views
def enrich_heatmap_features(
heatmap: dict[str, Any],
*,
as_of: date | None = None,
window_days: int | None = None,
alert_threshold: float | None = None,
) -> dict[str, Any]:
"""Attach micro rolling fields to heatmap GeoJSON features."""
threshold = float(alert_threshold if alert_threshold is not None else rolling_alert_threshold())
window = window_days or micro_window_days()
features = heatmap.get("features") or []
enriched: list[dict[str, Any]] = []
for feature in features:
if not isinstance(feature, dict):
continue
props = dict(feature.get("properties") or {})
region = str(props.get("region") or "").strip().lower()
view = compute_micro_view(
region,
as_of=as_of,
window_days=window,
alert_threshold=threshold,
) if region else None
if view is not None:
props["risk_spot"] = view.spot_risk
props["risk_3d_avg"] = view.risk_3d_avg
props["risk_delta"] = view.risk_delta
props["micro_days"] = view.days_in_window
props["micro_ignition"] = view.ignition
props["alerted_3d"] = view.alerted_3d
props["day_scores"] = list(view.day_scores)
enriched.append({**feature, "properties": props})
return {
**heatmap,
"features": enriched,
"micro_window_days": window,
"micro_alert_threshold": threshold,
}
def micro_rolling_report(
*,
as_of: date | None = None,
window_days: int | None = None,
limit: int = 15,
) -> dict[str, Any]:
"""API/OpenClaw payload for micro rolling 3-day context."""
today = as_of or utc_today()
window = window_days or micro_window_days()
threshold = rolling_alert_threshold()
views = compute_all_micro_views(
as_of=today,
window_days=window,
alert_threshold=threshold,
)
ignitions = [row for row in views if row.ignition]
alerted_3d = [row for row in views if row.alerted_3d]
top = views[: max(1, limit)]
stored_days = list_daily_ids(newest_first=True, limit=window)
return {
"mode": "micro_rolling",
"window_days": window,
"alert_threshold": threshold,
"ignition_delta": ignition_delta(),
"as_of": date_id(today),
"days_stored": len(stored_days),
"stored_dates": stored_days,
"regions_tracked": len(views),
"ignition_count": len(ignitions),
"alerted_3d_count": len(alerted_3d),
"ignitions": [row.to_dict() for row in ignitions[:limit]],
"top_regions": [row.to_dict() for row in top],
"note": (
f"Micro view: {window}-day rolling average vs spot risk. "
"Ignition = spot jumped above the rolling baseline (events that flare fast). "
"Macro week-over-week validation remains on /api/analytics/rolling."
),
}
+382
View File
@@ -0,0 +1,382 @@
"""Rolling weekly operational validation for Strategic Risk Analytics.
Freezes live GT scores each ISO week, accepts delayed outcome labels, and
scores prior-week predictions with accuracy + Wilson 95% CI. Unlike the
static historical benchmark, this measures forward operational usefulness.
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from datetime import date, datetime, timezone
from typing import Any, Literal
from analytics.backtest import DEFAULT_BACKTEST_ALERT_THRESHOLD, wilson_interval
from analytics.gt_early_warning import GT_EarlyWarning
from analytics.integration import get_gt_engine
from analytics.weekly_store import (
VALID_LABELS,
LabelName,
RegionSnapshot,
WeeklySnapshot,
list_week_ids,
load_week,
save_week,
utc_now_iso,
)
MIN_LABELED_FOR_TREND = 5
def _env_float(name: str, default: float) -> float:
raw = str(os.environ.get(name, "")).strip()
if not raw:
return default
try:
return float(raw)
except ValueError:
return default
def rolling_alert_threshold() -> float:
"""Fixed operational alert cutoff — not retroactively tuned per week."""
return _env_float("GT_ROLLING_ALERT_THRESHOLD", DEFAULT_BACKTEST_ALERT_THRESHOLD)
def iso_week_id(when: datetime | date | None = None) -> str:
"""Return ISO week id, e.g. ``2026-W24``."""
if when is None:
when = datetime.now(timezone.utc)
if isinstance(when, datetime):
when = when.date()
year, week, _ = when.isocalendar()
return f"{year}-W{week:02d}"
def _region_rows_from_engine(
engine: GT_EarlyWarning,
*,
alert_threshold: float,
) -> list[RegionSnapshot]:
heatmap = engine.get_risk_heatmap()
rows: list[RegionSnapshot] = []
for feature in heatmap.get("features") or []:
if not isinstance(feature, dict):
continue
props = feature.get("properties") or {}
region = str(props.get("region") or "").strip().lower()
if not region:
continue
composite = float(props.get("risk") or 0.0)
financial = float(props.get("financial") or 0.0)
unrest = float(props.get("unrest") or 0.0)
conflict = float(props.get("conflict") or 0.0)
peak_score = max(composite, financial, unrest, conflict)
rows.append(
RegionSnapshot(
region=region,
composite_risk=composite,
financial=financial,
unrest=unrest,
conflict=conflict,
alerted=peak_score >= alert_threshold,
label="pending",
)
)
rows.sort(key=lambda row: row.composite_risk, reverse=True)
return rows
@dataclass(frozen=True)
class WeekScore:
week_id: str
frozen_at: str
alert_threshold: float
total_regions: int
labeled: int
pending: int
alerted: int
correct: int
accuracy: float
confidence_rate: float
wilson_lower_95: float
wilson_upper_95: float
true_positives: int
true_negatives: int
false_positives: int
false_negatives: int
sensitivity: float
specificity: float
scorable: bool
def to_dict(self) -> dict[str, Any]:
return {
"week_id": self.week_id,
"frozen_at": self.frozen_at,
"alert_threshold": round(self.alert_threshold, 4),
"total_regions": self.total_regions,
"labeled": self.labeled,
"pending": self.pending,
"alerted": self.alerted,
"correct": self.correct,
"accuracy": round(self.accuracy, 4),
"confidence_rate": round(self.confidence_rate, 4),
"wilson_lower_95": round(self.wilson_lower_95, 4),
"wilson_upper_95": round(self.wilson_upper_95, 4),
"true_positives": self.true_positives,
"true_negatives": self.true_negatives,
"false_positives": self.false_positives,
"false_negatives": self.false_negatives,
"sensitivity": round(self.sensitivity, 4),
"specificity": round(self.specificity, 4),
"scorable": self.scorable,
}
def _predicted_positive(row: RegionSnapshot) -> bool:
return row.alerted
def _actual_positive(label: LabelName) -> bool:
return label == "true_escalation"
def _is_correct(row: RegionSnapshot) -> bool:
if row.label == "pending":
return False
predicted = _predicted_positive(row)
if row.label == "true_escalation":
return predicted
if row.label in ("false_alarm", "benign"):
return not predicted
return False
def score_week(snapshot: WeeklySnapshot) -> WeekScore:
"""Score a frozen week against delayed labels (pending rows excluded)."""
labeled_rows = [row for row in snapshot.regions if row.label != "pending"]
pending = len(snapshot.regions) - len(labeled_rows)
tp = sum(
1
for row in labeled_rows
if row.alerted and row.label == "true_escalation"
)
tn = sum(
1
for row in labeled_rows
if not row.alerted and row.label in ("benign", "false_alarm")
)
fp = sum(
1
for row in labeled_rows
if row.alerted and row.label in ("false_alarm", "benign")
)
fn = sum(
1
for row in labeled_rows
if not row.alerted and row.label == "true_escalation"
)
correct = tp + tn
total = len(labeled_rows)
accuracy = correct / total if total else 0.0
lower, upper = wilson_interval(correct, total)
pos_total = sum(1 for row in labeled_rows if _actual_positive(row.label)) # type: ignore[arg-type]
neg_total = total - pos_total
pred_pos = sum(1 for row in labeled_rows if row.alerted)
pred_neg = total - pred_pos
sensitivity = tp / pos_total if pos_total else 0.0
specificity = tn / pred_neg if pred_neg else (1.0 if tn == total and total else 0.0)
return WeekScore(
week_id=snapshot.week_id,
frozen_at=snapshot.frozen_at,
alert_threshold=snapshot.alert_threshold,
total_regions=len(snapshot.regions),
labeled=total,
pending=pending,
alerted=sum(1 for row in snapshot.regions if row.alerted),
correct=correct,
accuracy=accuracy,
confidence_rate=lower,
wilson_lower_95=lower,
wilson_upper_95=upper,
true_positives=tp,
true_negatives=tn,
false_positives=fp,
false_negatives=fn,
sensitivity=sensitivity,
specificity=specificity,
scorable=total >= MIN_LABELED_FOR_TREND,
)
def freeze_weekly_snapshot(
*,
week_id: str | None = None,
alert_threshold: float | None = None,
force: bool = False,
frozen_by: str = "system",
engine: GT_EarlyWarning | None = None,
) -> dict[str, Any]:
"""
Capture current GT heatmap as an immutable weekly operational snapshot.
Idempotent per week unless ``force=True``.
"""
resolved_engine = engine or get_gt_engine()
if resolved_engine is None:
return {"ok": False, "detail": "GT analytics engine unavailable"}
resolved_week = week_id or iso_week_id()
threshold = float(
alert_threshold if alert_threshold is not None else rolling_alert_threshold()
)
existing = load_week(resolved_week)
if existing and existing.regions and not force:
score = score_week(existing)
return {
"ok": True,
"created": False,
"week_id": resolved_week,
"snapshot": existing.to_dict(),
"score": score.to_dict(),
}
regions = _region_rows_from_engine(resolved_engine, alert_threshold=threshold)
snapshot = WeeklySnapshot(
week_id=resolved_week,
frozen_at=utc_now_iso(),
alert_threshold=threshold,
regions=regions,
frozen_by=frozen_by,
)
save_week(snapshot)
score = score_week(snapshot)
return {
"ok": True,
"created": True,
"week_id": resolved_week,
"snapshot": snapshot.to_dict(),
"score": score.to_dict(),
"alert_count": sum(1 for row in regions if row.alerted),
"region_count": len(regions),
}
def label_regions(
week_id: str,
labels: list[dict[str, Any]],
*,
labeled_by: str = "operator",
) -> dict[str, Any]:
"""Apply delayed outcome labels to a frozen week."""
snapshot = load_week(week_id)
if snapshot is None:
return {"ok": False, "detail": f"Week {week_id} not found"}
by_region = {row.region: row for row in snapshot.regions}
updated = 0
skipped: list[str] = []
now = utc_now_iso()
for entry in labels:
if not isinstance(entry, dict):
continue
region = str(entry.get("region") or "").strip().lower()
label = str(entry.get("label") or "").strip().lower()
if not region or label not in VALID_LABELS or label == "pending":
if region:
skipped.append(region)
continue
row = by_region.get(region)
if row is None:
skipped.append(region)
continue
row.label = label # type: ignore[assignment]
row.labeled_at = now
notes = entry.get("notes")
if notes is not None:
row.notes = str(notes)
updated += 1
save_week(snapshot)
score = score_week(snapshot)
return {
"ok": True,
"week_id": week_id,
"updated": updated,
"skipped": skipped,
"labeled_by": labeled_by,
"score": score.to_dict(),
}
def label_region(
week_id: str,
region: str,
label: LabelName,
*,
notes: str = "",
labeled_by: str = "operator",
) -> dict[str, Any]:
return label_regions(
week_id,
[{"region": region, "label": label, "notes": notes}],
labeled_by=labeled_by,
)
def rolling_trend(*, weeks: int = 8) -> list[WeekScore]:
"""Return scored weeks newest-first (only weeks with stored snapshots)."""
ids = list_week_ids(newest_first=True)[: max(1, weeks)]
scores: list[WeekScore] = []
for week_id in ids:
snapshot = load_week(week_id)
if snapshot is None:
continue
scores.append(score_week(snapshot))
return scores
def rolling_report(*, weeks: int = 8, target_confidence: float = 0.80) -> dict[str, Any]:
"""Aggregate operational validation trend for API / OpenClaw."""
threshold = rolling_alert_threshold()
trend = rolling_trend(weeks=weeks)
scorable = [row for row in trend if row.scorable]
latest = scorable[0] if scorable else (trend[0] if trend else None)
accuracy_series = [
{"week_id": row.week_id, "accuracy": round(row.accuracy, 4), "labeled": row.labeled}
for row in reversed(scorable)
]
improving = False
if len(scorable) >= 2:
improving = scorable[0].accuracy >= scorable[1].accuracy
return {
"mode": "rolling_operational",
"alert_threshold": threshold,
"target_confidence": target_confidence,
"weeks_requested": weeks,
"weeks_stored": len(trend),
"weeks_scorable": len(scorable),
"min_labeled_per_week": MIN_LABELED_FOR_TREND,
"latest": latest.to_dict() if latest else None,
"trend": [row.to_dict() for row in trend],
"accuracy_series": accuracy_series,
"improving_vs_prior": improving,
"meets_target": bool(
latest and latest.scorable and latest.confidence_rate >= target_confidence
),
"note": (
"Operational metric: scores frozen weekly predictions against delayed "
"labels. Unlike the static benchmark, this measures live forward utility."
),
}
+158
View File
@@ -0,0 +1,158 @@
"""Configuration for Strategic Risk Analytics (feature-flagged)."""
from __future__ import annotations
import json
import os
from dataclasses import dataclass, field
from functools import lru_cache
from typing import Any
def _env_bool(name: str, default: bool = False) -> bool:
raw = str(os.environ.get(name, "")).strip().lower()
if not raw:
return default
return raw not in {"0", "false", "no", "off"}
def _env_float(name: str, default: float) -> float:
raw = str(os.environ.get(name, "")).strip()
if not raw:
return default
try:
return float(raw)
except ValueError:
return default
def _env_int(name: str, default: int) -> int:
raw = str(os.environ.get(name, "")).strip()
if not raw:
return default
try:
return int(raw)
except ValueError:
return default
def _parse_signal_weights(raw: str) -> dict[str, float]:
if not raw.strip():
return {}
try:
parsed = json.loads(raw)
if isinstance(parsed, dict):
return {str(k): float(v) for k, v in parsed.items()}
except (json.JSONDecodeError, TypeError, ValueError):
pass
weights: dict[str, float] = {}
for part in raw.split(","):
piece = part.strip()
if not piece or "=" not in piece:
continue
key, value = piece.split("=", 1)
try:
weights[key.strip()] = float(value.strip())
except ValueError:
continue
return weights
def resolve_gt_profile() -> str:
from services.runtime_profile import resolve_profile_name
return resolve_profile_name()
def gt_analytics_ack_low_cpu() -> bool:
return _env_bool("GT_ANALYTICS_ACK_LOW_CPU", default=False)
def gt_engine_operational() -> bool:
"""Full GT engine (scheduled ingest, heatmap, Louvain) — not watchdog-only."""
if not get_gt_settings().enabled:
return False
if resolve_gt_profile() == "lean" and not gt_analytics_ack_low_cpu():
return False
return True
def gt_scheduled_ingest_enabled() -> bool:
return gt_engine_operational()
def gt_louvain_enabled() -> bool:
return gt_engine_operational()
@dataclass(frozen=True)
class GTAnalyticsSettings:
enabled: bool = False
profile: str = "standard"
base_prior: float = 0.15
evidence_cap: float = 3.0
evidence_scale: float = 5.0
min_prob: float = 0.01
max_prob: float = 0.99
high_risk_threshold: float = 0.6
max_history_per_region: int = 200
max_heatmap_features: int = 500
louvain_min_weight: float = 0.5
louvain_interval_minutes: int = 30
signal_weight_overrides: dict[str, float] = field(default_factory=dict)
watched_channels: tuple[str, ...] = ()
@lru_cache(maxsize=1)
def get_gt_settings() -> GTAnalyticsSettings:
channels_raw = str(os.environ.get("GT_ANALYTICS_WATCHED_CHANNELS", "")).strip()
channels = tuple(
part.strip().lstrip("@")
for part in channels_raw.split(",")
if part.strip()
)
profile = resolve_gt_profile()
lean = profile == "lean"
return GTAnalyticsSettings(
enabled=_env_bool("GT_ANALYTICS_ENABLED", default=False),
profile=profile,
base_prior=_env_float("GT_ANALYTICS_BASE_PRIOR", 0.15),
evidence_cap=_env_float("GT_ANALYTICS_EVIDENCE_CAP", 3.0),
evidence_scale=_env_float("GT_ANALYTICS_EVIDENCE_SCALE", 5.0),
min_prob=_env_float("GT_ANALYTICS_MIN_PROB", 0.01),
max_prob=_env_float("GT_ANALYTICS_MAX_PROB", 0.99),
high_risk_threshold=_env_float("GT_ANALYTICS_HIGH_RISK_THRESHOLD", 0.6),
max_history_per_region=_env_int("GT_ANALYTICS_MAX_HISTORY", 200),
max_heatmap_features=_env_int(
"GT_ANALYTICS_MAX_HEATMAP_FEATURES",
50 if lean else 500,
),
louvain_min_weight=_env_float("GT_ANALYTICS_LOUVAIN_MIN_WEIGHT", 0.5),
louvain_interval_minutes=max(5, _env_int("GT_ANALYTICS_LOUVAIN_INTERVAL_MINUTES", 30)),
signal_weight_overrides=_parse_signal_weights(
str(os.environ.get("GT_ANALYTICS_SIGNAL_WEIGHTS", ""))
),
watched_channels=channels,
)
def gt_analytics_enabled() -> bool:
return get_gt_settings().enabled
def gt_analytics_status() -> dict[str, Any]:
settings = get_gt_settings()
from services.runtime_profile import get_runtime_profile
runtime = get_runtime_profile()
operational = gt_engine_operational()
return {
"enabled": settings.enabled,
"operational": operational,
"profile": settings.profile,
"ack_low_cpu": gt_analytics_ack_low_cpu(),
"recommended": bool(runtime.get("gt_analytics", {}).get("recommended")),
"lean_node": bool(runtime.get("gt_analytics", {}).get("lean_node")),
"warning": runtime.get("gt_analytics", {}).get("warning"),
"experimental": True,
}
+154
View File
@@ -0,0 +1,154 @@
"""Persistent JSON store for rolling GT operational backtest weeks."""
from __future__ import annotations
import json
import logging
import os
import threading
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Literal
logger = logging.getLogger(__name__)
LabelName = Literal["pending", "true_escalation", "false_alarm", "benign"]
VALID_LABELS: frozenset[str] = frozenset(
{"pending", "true_escalation", "false_alarm", "benign"}
)
_STORE_DIR = Path(__file__).parent.parent / "data" / "gt_rolling"
_store_lock = threading.Lock()
def rolling_store_dir() -> Path:
"""Return the rolling-backtest data directory (override via env in tests)."""
override = str(os.environ.get("GT_ROLLING_STORE_DIR", "")).strip()
if override:
return Path(override)
return _STORE_DIR
@dataclass
class RegionSnapshot:
region: str
composite_risk: float
financial: float
unrest: float
conflict: float
alerted: bool
label: LabelName = "pending"
labeled_at: str | None = None
notes: str = ""
def to_dict(self) -> dict[str, Any]:
return asdict(self)
@classmethod
def from_dict(cls, raw: dict[str, Any]) -> RegionSnapshot:
label = str(raw.get("label") or "pending")
if label not in VALID_LABELS:
label = "pending"
return cls(
region=str(raw.get("region") or "").strip().lower(),
composite_risk=float(raw.get("composite_risk") or 0.0),
financial=float(raw.get("financial") or 0.0),
unrest=float(raw.get("unrest") or 0.0),
conflict=float(raw.get("conflict") or 0.0),
alerted=bool(raw.get("alerted")),
label=label, # type: ignore[arg-type]
labeled_at=raw.get("labeled_at"),
notes=str(raw.get("notes") or ""),
)
@dataclass
class WeeklySnapshot:
week_id: str
frozen_at: str
alert_threshold: float
regions: list[RegionSnapshot] = field(default_factory=list)
frozen_by: str = "system"
def to_dict(self) -> dict[str, Any]:
return {
"week_id": self.week_id,
"frozen_at": self.frozen_at,
"alert_threshold": self.alert_threshold,
"frozen_by": self.frozen_by,
"regions": [row.to_dict() for row in self.regions],
}
@classmethod
def from_dict(cls, raw: dict[str, Any]) -> WeeklySnapshot:
regions = [
RegionSnapshot.from_dict(row)
for row in (raw.get("regions") or [])
if isinstance(row, dict)
]
return cls(
week_id=str(raw.get("week_id") or ""),
frozen_at=str(raw.get("frozen_at") or ""),
alert_threshold=float(raw.get("alert_threshold") or 0.0),
regions=regions,
frozen_by=str(raw.get("frozen_by") or "system"),
)
def _week_path(week_id: str) -> Path:
safe = week_id.replace("/", "-").replace("..", "")
return rolling_store_dir() / f"{safe}.json"
def _ensure_dir() -> None:
rolling_store_dir().mkdir(parents=True, exist_ok=True)
def list_week_ids(*, newest_first: bool = True) -> list[str]:
"""Return stored ISO week ids."""
_ensure_dir()
ids = [
path.stem
for path in rolling_store_dir().glob("*.json")
if path.stem and path.stem != "index"
]
ids.sort(reverse=newest_first)
return ids
def load_week(week_id: str) -> WeeklySnapshot | None:
path = _week_path(week_id)
if not path.is_file():
return None
try:
raw = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(raw, dict):
return None
return WeeklySnapshot.from_dict(raw)
except (OSError, json.JSONDecodeError, TypeError, ValueError):
logger.exception("Failed to load GT rolling week %s", week_id)
return None
def save_week(snapshot: WeeklySnapshot) -> None:
_ensure_dir()
path = _week_path(snapshot.week_id)
tmp = path.with_suffix(".json.tmp")
payload = json.dumps(snapshot.to_dict(), indent=2, sort_keys=True)
with _store_lock:
tmp.write_text(payload, encoding="utf-8")
tmp.replace(path)
def delete_week(week_id: str) -> bool:
path = _week_path(week_id)
if not path.is_file():
return False
with _store_lock:
path.unlink()
return True
def utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
+2
View File
@@ -372,6 +372,7 @@ osint_router = _load_optional_router("routers.osint")
scm_router = _load_optional_router("routers.scm")
entity_graph_router = _load_optional_router("routers.entity_graph")
intel_feeds_router = _load_optional_router("routers.intel_feeds")
analytics_router = _load_optional_router("routers.analytics")
agent_shell_router = _load_optional_router("routers.agent_shell")
@@ -3801,6 +3802,7 @@ app.include_router(osint_router)
app.include_router(scm_router)
app.include_router(entity_graph_router)
app.include_router(intel_feeds_router)
app.include_router(analytics_router)
app.include_router(agent_shell_router)
from services.data_fetcher import update_all_data
+2
View File
@@ -29,6 +29,8 @@ dependencies = [
"reverse-geocoder==1.5.1",
"sgp4==2.25",
"meshtastic>=2.5.0",
"networkx>=3.4.0",
"numpy>=2.2.0",
"orjson>=3.10.0",
"paho-mqtt>=1.6.0,<2.0.0",
"PyNaCl>=1.5.0",
+5 -3
View File
@@ -2051,7 +2051,7 @@ async def agent_tool_manifest(request: Request):
"description": "Set up a watchdog alert. When triggered, alerts push instantly via SSE stream. Debounced: same watch won't re-fire within 60 seconds.",
"parameters": {
"type": {"type": "string", "required": True, "description": "Watch type",
"enum": ["track_aircraft", "track_callsign", "track_registration", "track_ship", "track_entity", "geofence", "keyword", "prediction_market"]},
"enum": ["track_aircraft", "track_callsign", "track_registration", "track_ship", "track_entity", "geofence", "keyword", "telegram_rhetoric", "prediction_market"]},
"params": {"type": "object", "required": True, "description": "Type-specific parameters (see subtypes)"},
},
"subtypes": {
@@ -2061,7 +2061,8 @@ async def agent_tool_manifest(request: Request):
"track_ship": {"params": {"mmsi": "string (optional)", "imo": "string (optional)", "name": "string (optional)", "owner": "string (optional)", "callsign": "string (optional)"}, "description": "Alert when ship appears by MMSI, IMO, name, owner, or callsign"},
"track_entity": {"params": {"query": "string", "entity_type": "string (optional)", "layers": "list (optional)"}, "description": "Generic exact-first entity tracker when aircraft/ship fields are not known yet"},
"geofence": {"params": {"lat": "float", "lng": "float", "radius_km": "float (default 50)", "entity_types": "list (default ['flights','ships'])"}, "description": "Alert when any entity enters a geographic zone"},
"keyword": {"params": {"keyword": "string"}, "description": "Alert when keyword appears in news/GDELT headlines"},
"keyword": {"params": {"keyword": "string", "include_telegram": "boolean (default true)"}, "description": "Alert when keyword appears in news, GDELT, or Telegram OSINT (searches translated + original text)"},
"telegram_rhetoric": {"params": {"min_risk_score": "int 1-10 (default 7)", "keywords": "list or comma-separated string (optional)", "channels": "list or comma-separated string (optional)"}, "description": "Alert on new high-risk Telegram OSINT posts — rhetoric/escalation monitor"},
"prediction_market": {"params": {"query": "string", "threshold": "float 0-1 (optional)"}, "description": "Alert on prediction market movements matching query"},
},
"example": {"cmd": "add_watch", "args": {"type": "track_registration", "params": {"registration": "N3880"}}},
@@ -2564,7 +2565,8 @@ async def api_capabilities(request: Request):
"track_ship": {"params": {"mmsi": "str (optional)", "imo": "str (optional)", "name": "str (optional)", "owner": "str (optional)", "callsign": "str (optional)"}, "description": "Alert when ship appears by MMSI, IMO, name, owner, or callsign"},
"track_entity": {"params": {"query": "str", "entity_type": "str (optional)", "layers": "list[str] (optional)"}, "description": "Generic exact-first entity watch"},
"geofence": {"params": {"lat": "float", "lng": "float", "radius_km": "float (default 50)", "entity_types": "list (default ['flights','ships'])"}, "description": "Alert when any entity enters a geographic zone"},
"keyword": {"params": {"keyword": "str"}, "description": "Alert when keyword appears in news/GDELT"},
"keyword": {"params": {"keyword": "str", "include_telegram": "bool (default true)"}, "description": "Alert when keyword appears in news, GDELT, or Telegram OSINT"},
"telegram_rhetoric": {"params": {"min_risk_score": "int 1-10 (default 7)", "keywords": "list[str] or comma string (optional)", "channels": "list[str] or comma string (optional)"}, "description": "Alert on new high-risk Telegram OSINT posts"},
"prediction_market": {"params": {"query": "str", "threshold": "float 0-1 (optional)"}, "description": "Alert on prediction market movements"},
},
},
+339
View File
@@ -0,0 +1,339 @@
"""Strategic Risk Analytics API — game-theoretic early warning overlays."""
from __future__ import annotations
import logging
from typing import Any
from fastapi import APIRouter, Depends, HTTPException, Request
from pydantic import BaseModel, Field
from auth import require_local_operator
from limiter import limiter
from analytics.backtest import (
DEFAULT_BACKTEST_ALERT_THRESHOLD,
run_historical_backtest,
tune_alert_threshold,
)
from analytics.feed_adapter import normalize_feed_item
from analytics.integration import get_gt_engine, refresh_from_latest_data
from analytics.gt_alerts import top_gt_alerts
from analytics.micro_rolling import micro_rolling_report
from analytics.rolling_backtest import (
freeze_weekly_snapshot,
label_region,
label_regions,
rolling_alert_threshold,
rolling_report,
score_week,
)
from analytics.weekly_store import load_week
from analytics.settings import gt_analytics_enabled
from services.fetchers._store import _data_lock, get_latest_data_subset_refs, latest_data
logger = logging.getLogger(__name__)
router = APIRouter()
class RiskHeatmapRequest(BaseModel):
"""Optional batch ingest + refresh controls for POST /api/analytics/risk_heatmap."""
refresh: bool = True
items: list[dict[str, Any]] = Field(default_factory=list)
class RollingFreezeRequest(BaseModel):
week_id: str | None = None
force: bool = False
class RollingLabelEntry(BaseModel):
region: str
label: str
notes: str = ""
class RollingLabelRequest(BaseModel):
week_id: str
labels: list[RollingLabelEntry] = Field(default_factory=list)
def _empty_heatmap() -> dict[str, Any]:
return {
"enabled": False,
"type": "FeatureCollection",
"features": [],
"clusters": [],
"processed": 0,
"timestamp": None,
}
def _gt_risk_payload() -> dict[str, Any]:
snap = get_latest_data_subset_refs("gt_risk")
payload = snap.get("gt_risk")
if not isinstance(payload, dict):
return _empty_heatmap()
heatmap = payload.get("heatmap") or {"type": "FeatureCollection", "features": []}
return {
"enabled": bool(payload.get("enabled")),
"type": heatmap.get("type", "FeatureCollection"),
"features": list(heatmap.get("features") or []),
"clusters": list(payload.get("clusters") or []),
"processed": int(payload.get("processed") or 0),
"timestamp": payload.get("timestamp"),
}
@router.get("/api/analytics/risk_heatmap")
@limiter.limit("60/minute")
async def risk_heatmap_get(request: Request) -> dict[str, Any]:
"""Return cached GeoJSON risk overlay (posterior scores per region)."""
if not gt_analytics_enabled():
return _empty_heatmap()
return _gt_risk_payload()
@router.post("/api/analytics/risk_heatmap")
@limiter.limit("12/minute")
async def risk_heatmap_post(
request: Request,
body: RiskHeatmapRequest,
_: None = Depends(require_local_operator),
) -> dict[str, Any]:
"""
Ingest optional feed items and/or refresh beliefs from latest intel layers.
Requires local operator auth — intended for OpenClaw agents and admin tooling.
"""
if not gt_analytics_enabled():
raise HTTPException(status_code=503, detail="Strategic Risk Analytics is disabled")
engine = get_gt_engine()
if engine is None:
raise HTTPException(status_code=503, detail="Strategic Risk Analytics engine unavailable")
ingested = 0
for raw in body.items:
if not isinstance(raw, dict):
continue
source_type = str(raw.get("source_type") or "manual")
item = normalize_feed_item(raw, source_type=source_type)
result = engine.process_feed_item(item)
if result and not result.get("skipped"):
ingested += 1
summary: dict[str, Any] = {"ingested": ingested}
if body.refresh:
with _data_lock:
snapshot = dict(latest_data)
summary.update(refresh_from_latest_data(snapshot, persist=True))
payload = _gt_risk_payload()
payload["ingested"] = ingested
payload["refresh"] = bool(body.refresh)
return payload
@router.get("/api/analytics/dossier/{region}")
@limiter.limit("30/minute")
async def analytics_dossier(request: Request, region: str) -> dict[str, Any]:
"""Game-theoretic rationale, recent costly signals, and scenario sketches."""
region_key = str(region or "").strip().lower()
if not region_key or len(region_key) > 120:
raise HTTPException(status_code=400, detail="Invalid region identifier")
if not gt_analytics_enabled():
return {
"enabled": False,
"region": region_key,
"current_risk": 0.0,
"interpretation": "Strategic Risk Analytics is disabled.",
"recent_signals": [],
"scenarios": [],
}
engine = get_gt_engine()
if engine is None:
raise HTTPException(status_code=503, detail="Strategic Risk Analytics engine unavailable")
dossier = engine.get_dossier(region_key)
dossier["enabled"] = True
return dossier
@router.get("/api/analytics/backtest")
@limiter.limit("6/minute")
async def analytics_backtest(
request: Request,
expanded: bool = True,
tune: bool = False,
target_confidence: float = 0.95,
) -> dict[str, Any]:
"""
Run labeled historical backtest and return accuracy + Wilson 95% CI.
``confidence_rate`` is the Wilson lower bound (conservative pass metric).
"""
if not gt_analytics_enabled():
return {
"enabled": False,
"message": "Strategic Risk Analytics is disabled.",
}
if tune:
threshold, report = tune_alert_threshold(target_confidence=target_confidence)
else:
threshold = DEFAULT_BACKTEST_ALERT_THRESHOLD
report = run_historical_backtest(
use_expanded_suite=expanded,
alert_threshold=threshold,
target_confidence=target_confidence,
)
payload = report.to_dict()
payload["enabled"] = True
payload["expanded_suite"] = expanded
payload["tuned"] = tune
payload["recommended_alert_threshold"] = threshold
return payload
@router.get("/api/analytics/rolling")
@limiter.limit("12/minute")
async def analytics_rolling(
request: Request,
weeks: int = 8,
target_confidence: float = 0.80,
) -> dict[str, Any]:
"""Rolling weekly operational validation — accuracy trend with delayed labels."""
if not gt_analytics_enabled():
return {
"enabled": False,
"message": "Strategic Risk Analytics is disabled.",
}
report = rolling_report(weeks=max(1, min(weeks, 52)), target_confidence=target_confidence)
report["enabled"] = True
return report
@router.get("/api/analytics/alerts")
@limiter.limit("30/minute")
async def analytics_top_alerts(
request: Request,
limit: int = 8,
) -> dict[str, Any]:
"""Top GT risk regions ranked by score — fly-to targets for the map."""
if not gt_analytics_enabled():
return {
"enabled": False,
"message": "Strategic Risk Analytics is disabled.",
}
report = top_gt_alerts(limit=max(1, min(limit, 25)))
report["enabled"] = True
return report
@router.get("/api/analytics/rolling/micro")
@limiter.limit("30/minute")
async def analytics_rolling_micro(
request: Request,
window_days: int = 3,
limit: int = 15,
) -> dict[str, Any]:
"""Rolling 3-day micro average — spot vs baseline, ignition detection."""
if not gt_analytics_enabled():
return {
"enabled": False,
"message": "Strategic Risk Analytics is disabled.",
}
report = micro_rolling_report(
window_days=max(2, min(window_days, 7)),
limit=max(1, min(limit, 50)),
)
report["enabled"] = True
return report
@router.get("/api/analytics/rolling/{week_id}")
@limiter.limit("12/minute")
async def analytics_rolling_week(request: Request, week_id: str) -> dict[str, Any]:
"""Return a single frozen week snapshot and its score."""
if not gt_analytics_enabled():
return {"enabled": False, "message": "Strategic Risk Analytics is disabled."}
snapshot = load_week(str(week_id).strip())
if snapshot is None:
raise HTTPException(status_code=404, detail=f"Week {week_id} not found")
score = score_week(snapshot)
return {
"enabled": True,
"week_id": snapshot.week_id,
"snapshot": snapshot.to_dict(),
"score": score.to_dict(),
"alert_threshold": rolling_alert_threshold(),
}
@router.post("/api/analytics/rolling/freeze")
@limiter.limit("6/minute")
async def analytics_rolling_freeze(
request: Request,
body: RollingFreezeRequest,
_: None = Depends(require_local_operator),
) -> dict[str, Any]:
"""Freeze current GT scores for the ISO week (idempotent unless force=true)."""
if not gt_analytics_enabled():
raise HTTPException(status_code=503, detail="Strategic Risk Analytics is disabled")
result = freeze_weekly_snapshot(
week_id=body.week_id,
force=body.force,
frozen_by="api",
)
if not result.get("ok"):
raise HTTPException(status_code=503, detail=result.get("detail", "Freeze failed"))
result["enabled"] = True
return result
@router.post("/api/analytics/rolling/label")
@limiter.limit("12/minute")
async def analytics_rolling_label(
request: Request,
body: RollingLabelRequest,
_: None = Depends(require_local_operator),
) -> dict[str, Any]:
"""Apply delayed outcome labels to a frozen week."""
if not gt_analytics_enabled():
raise HTTPException(status_code=503, detail="Strategic Risk Analytics is disabled")
week_id = str(body.week_id or "").strip()
if not week_id:
raise HTTPException(status_code=400, detail="week_id required")
if len(body.labels) == 1:
entry = body.labels[0]
result = label_region(
week_id,
entry.region,
entry.label, # type: ignore[arg-type]
notes=entry.notes,
labeled_by="api",
)
else:
result = label_regions(
week_id,
[row.model_dump() for row in body.labels],
labeled_by="api",
)
if not result.get("ok"):
raise HTTPException(status_code=404, detail=result.get("detail", "Label failed"))
result["enabled"] = True
return result
+7 -1
View File
@@ -773,7 +773,7 @@ async def live_data_slow(
"scanners", "weather_alerts", "ukraine_alerts", "air_quality", "volcanoes",
"fishing_activity", "psk_reporter", "correlations", "uap_sightings", "wastewater",
"crowdthreat", "threat_level", "trending_markets", "road_corridor_trends",
"malware_threats", "cyber_threats", "scm_suppliers", "telegram_osint",
"malware_threats", "cyber_threats", "scm_suppliers", "telegram_osint", "gt_risk",
)
freshness = get_source_timestamps_snapshot()
payload = {
@@ -839,6 +839,12 @@ async def live_data_slow(
)
if active_layers.get("telegram_osint", True)
else {"posts": [], "total": 0, "geolocated": 0},
"gt_risk": (
d.get("gt_risk")
or {"enabled": False, "heatmap": {"type": "FeatureCollection", "features": []}, "clusters": []}
)
if active_layers.get("gt_risk", False)
else {"enabled": False, "heatmap": {"type": "FeatureCollection", "features": []}, "clusters": []},
"freshness": freshness,
}
# Issue #288: bbox filter heavy/dense layers only when all four bounds
+13
View File
@@ -85,6 +85,18 @@ async def health_check(request: Request):
):
top_status = "degraded"
runtime: dict = {}
try:
from services.runtime_profile import get_runtime_profile
from analytics.settings import gt_analytics_status
runtime = {
**get_runtime_profile(),
"gt_analytics": gt_analytics_status(),
}
except Exception:
runtime = {}
return {
"status": top_status,
"version": _get_app_version(),
@@ -108,6 +120,7 @@ async def health_check(request: Request):
"slo": slo_statuses,
"slo_summary": slo_summary,
"ais_proxy": ais_status,
"runtime": runtime or None,
}
+12 -4
View File
@@ -14,6 +14,7 @@ from services.fetchers._store import get_latest_data_subset_refs
from services.fetchers.telegram_osint import telegram_media_host_allowed
from services.intel_feeds.country_risk import build_country_risk_payload
from services.network_utils import outbound_user_agent
from services.telegram_translate import apply_posts_translations, normalize_translate_target
logger = logging.getLogger(__name__)
@@ -45,12 +46,19 @@ async def country_risk(request: Request) -> dict:
@router.get("/api/telegram-feed")
@limiter.limit("30/minute")
async def telegram_feed(request: Request) -> dict:
async def telegram_feed(request: Request, lang: str | None = Query(default=None)) -> dict:
snap = get_latest_data_subset_refs("telegram_osint")
payload = snap.get("telegram_osint")
if isinstance(payload, dict) and payload.get("posts") is not None:
return payload
return {"posts": [], "total": 0, "geolocated": 0, "timestamp": None}
if not isinstance(payload, dict) or payload.get("posts") is None:
return {"posts": [], "total": 0, "geolocated": 0, "timestamp": None}
if lang:
target = normalize_translate_target(lang)
localized = dict(payload)
localized["posts"] = apply_posts_translations(list(payload.get("posts") or []), target)
localized["translate_locale"] = target
return localized
return payload
def _infer_telegram_media_type(target_url: str, content_type: str) -> str:
+71 -2
View File
@@ -499,6 +499,12 @@ def update_slow_data():
latest_data["correlations"] = correlations
except Exception as e:
logger.error("Correlation engine failed: %s", e)
try:
from analytics.integration import maybe_refresh_gt_analytics
maybe_refresh_gt_analytics()
except Exception as e:
logger.error("GT analytics refresh failed: %s", e)
from services.fetchers._store import bump_data_version
bump_data_version()
_save_intel_startup_cache()
@@ -807,8 +813,18 @@ def start_scheduler():
# Telegram OSINT — hourly t.me/s channel scrape (kept off the 5-minute slow tier).
_telegram_interval_m = max(15, int(os.environ.get("TELEGRAM_OSINT_INTERVAL_MINUTES", "60")))
def _fetch_telegram_osint_with_gt():
fetch_telegram_osint()
try:
from analytics.integration import maybe_refresh_gt_analytics
maybe_refresh_gt_analytics()
except Exception as exc:
logger.error("GT analytics refresh after telegram failed: %s", exc)
_scheduler.add_job(
lambda: _run_task_with_health(fetch_telegram_osint, "fetch_telegram_osint"),
lambda: _run_task_with_health(_fetch_telegram_osint_with_gt, "fetch_telegram_osint"),
"interval",
minutes=_telegram_interval_m,
next_run_time=datetime.utcnow() + timedelta(seconds=45),
@@ -934,14 +950,67 @@ def start_scheduler():
)
# GDELT — every 30 minutes (downloads 32 ZIP files per call, avoid rate limits)
def _fetch_gdelt_with_gt():
fetch_gdelt()
try:
from analytics.integration import maybe_refresh_gt_analytics
maybe_refresh_gt_analytics()
except Exception as exc:
logger.error("GT analytics refresh after gdelt failed: %s", exc)
_scheduler.add_job(
lambda: _run_task_with_health_on_executor(_SLOW_EXECUTOR, fetch_gdelt, "fetch_gdelt"),
lambda: _run_task_with_health_on_executor(_SLOW_EXECUTOR, _fetch_gdelt_with_gt, "fetch_gdelt"),
"interval",
minutes=30,
id="gdelt",
max_instances=1,
misfire_grace_time=120,
)
# GT analytics — Louvain herding/coordination clusters (feature-flagged).
def _recompute_gt_clusters():
try:
from analytics.integration import recompute_gt_herding_clusters
recompute_gt_herding_clusters()
except Exception as exc:
logger.error("GT Louvain recompute failed: %s", exc)
def _freeze_gt_weekly_snapshot():
try:
from analytics.integration import maybe_freeze_gt_weekly_snapshot
maybe_freeze_gt_weekly_snapshot()
except Exception as exc:
logger.error("GT rolling weekly freeze failed: %s", exc)
try:
from analytics.settings import get_gt_settings, gt_engine_operational
_gt_settings = get_gt_settings()
if gt_engine_operational():
_scheduler.add_job(
_recompute_gt_clusters,
"interval",
minutes=_gt_settings.louvain_interval_minutes,
id="gt_analytics_louvain",
max_instances=1,
misfire_grace_time=300,
next_run_time=datetime.utcnow() + timedelta(minutes=3),
)
_scheduler.add_job(
_freeze_gt_weekly_snapshot,
"cron",
day_of_week="mon",
hour=0,
minute=5,
id="gt_rolling_weekly_freeze",
max_instances=1,
misfire_grace_time=3600,
)
except Exception as exc:
logger.warning("GT Louvain scheduler not registered: %s", exc)
_scheduler.add_job(
lambda: _run_task_with_health_on_executor(
_SLOW_EXECUTOR, update_liveuamap, "update_liveuamap"
+9
View File
@@ -74,6 +74,7 @@ class DashboardData(TypedDict, total=False):
cyber_threats: Dict[str, Any]
scm_suppliers: Dict[str, Any]
telegram_osint: Dict[str, Any]
gt_risk: Dict[str, Any]
# In-memory store
@@ -129,6 +130,13 @@ latest_data: DashboardData = {
"cyber_threats": {"threats": [], "stats": {}},
"scm_suppliers": {"suppliers": [], "total": 0, "critical_count": 0},
"telegram_osint": {"posts": [], "total": 0, "geolocated": 0, "timestamp": None},
"gt_risk": {
"enabled": False,
"heatmap": {"type": "FeatureCollection", "features": []},
"clusters": [],
"processed": 0,
"timestamp": None,
},
}
# Per-source freshness timestamps
@@ -361,6 +369,7 @@ active_layers: dict[str, bool] = {
"scm_suppliers": False,
"cyber_threats": False,
"telegram_osint": True,
"gt_risk": False,
}
+17 -21
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import hashlib
import html
import logging
import os
import re
@@ -11,6 +12,7 @@ from typing import Any
from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
from services.fetchers.news import resolve_coords_match
from services.network_utils import fetch_with_curl, outbound_user_agent
from services.telegram_translate import apply_post_translation, apply_posts_translations
logger = logging.getLogger(__name__)
@@ -174,13 +176,7 @@ def _extract_media(block: str, link: str) -> dict[str, Any]:
def _strip_html(text: str) -> str:
cleaned = re.sub(r"<br\s*/?>", "\n", text, flags=re.IGNORECASE)
cleaned = re.sub(r"<[^>]+>", "", cleaned)
return (
cleaned.replace("&quot;", '"')
.replace("&amp;", "&")
.replace("&lt;", "<")
.replace("&gt;", ">")
.strip()
)
return html.unescape(cleaned).strip()
def _score_risk(text: str) -> int:
@@ -293,20 +289,19 @@ def parse_telegram_channel_html(html: str, channel: str) -> list[dict[str, Any]]
post_id = hashlib.sha1(f"{link}|{published}".encode("utf-8")).hexdigest()[:16]
media = _extract_media(block, link)
posts.append(
{
"id": post_id,
"title": title,
"description": text[:1200],
"link": link,
"published": published,
"source": f"t.me/{channel}",
"channel": channel,
"risk_score": risk_score,
"coords": [coords[0], coords[1]] if coords else None,
**media,
}
)
post = {
"id": post_id,
"title": title,
"description": text[:1200],
"link": link,
"published": published,
"source": f"t.me/{channel}",
"channel": channel,
"risk_score": risk_score,
"coords": [coords[0], coords[1]] if coords else None,
**media,
}
posts.append(apply_post_translation(post))
return posts
@@ -358,6 +353,7 @@ def fetch_telegram_osint() -> dict[str, Any]:
merged_posts, added = _merge_telegram_posts(existing_posts, incoming)
merged_posts = [_refresh_post_coords(post) for post in merged_posts]
merged_posts = apply_posts_translations(merged_posts)
geolocated = sum(1 for p in merged_posts if p.get("coords"))
payload = {
+288 -1
View File
@@ -90,6 +90,15 @@ READ_COMMANDS = frozenset({
# Agent routing helpers
"route_query",
"run_playbook",
"gt_risk_heatmap",
"gt_dossier",
"gt_analyze",
"gt_backtest",
"gt_rolling_freeze",
"gt_rolling_label",
"gt_rolling_backtest",
"gt_micro_rolling",
"gt_top_alerts",
# Private Infonet reads (operator-delegated)
"infonet_status",
"list_gates",
@@ -857,6 +866,284 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
return {"ok": True, "data": _compact_query_result(result), "format": "compressed_v1"}
return {"ok": True, "data": result}
if cmd == "gt_risk_heatmap":
from analytics.settings import gt_analytics_enabled
from analytics.integration import get_gt_engine
from services.fetchers._store import get_latest_data_subset_refs
if not gt_analytics_enabled():
return {"ok": True, "data": {"enabled": False, "features": [], "clusters": []}}
snap = get_latest_data_subset_refs("gt_risk")
payload = dict(snap.get("gt_risk") or {})
engine = get_gt_engine()
if engine is not None and not payload.get("heatmap"):
payload["heatmap"] = engine.get_risk_heatmap()
return {"ok": True, "data": payload}
if cmd == "gt_dossier":
from analytics.settings import gt_analytics_enabled
from analytics.integration import get_gt_engine
region = str(args.get("region", "") or args.get("area", "") or "").strip().lower()
if not region:
return {"ok": False, "detail": "region required (e.g. ukraine, uk, europe)"}
if not gt_analytics_enabled():
return {
"ok": True,
"data": {
"enabled": False,
"region": region,
"interpretation": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
},
}
engine = get_gt_engine()
if engine is None:
return {"ok": False, "detail": "GT analytics engine unavailable"}
return {"ok": True, "data": engine.get_dossier(region)}
if cmd == "gt_analyze":
from analytics.settings import gt_analytics_enabled
from analytics.integration import get_gt_engine, refresh_from_latest_data
from services.fetchers._store import _data_lock, latest_data
if not gt_analytics_enabled():
return {"ok": False, "detail": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED)"}
engine = get_gt_engine()
if engine is None:
return {"ok": False, "detail": "GT analytics engine unavailable"}
feeds = args.get("feeds") if isinstance(args.get("feeds"), (list, tuple)) else None
if feeds:
from analytics.feed_adapter import normalize_feed_item
ingested = 0
for raw in feeds:
if not isinstance(raw, dict):
continue
item = normalize_feed_item(raw, source_type=str(raw.get("source_type") or "openclaw"))
result = engine.process_feed_item(item)
if result and not result.get("skipped"):
ingested += 1
summary = {"ingested": ingested, "enabled": True}
else:
with _data_lock:
snapshot = dict(latest_data)
summary = refresh_from_latest_data(snapshot, persist=True)
region = str(args.get("region", "") or "").strip().lower()
data = {
"refresh": summary,
"heatmap_features": len((summary.get("sample") or [])),
}
if region:
data["dossier"] = engine.get_dossier(region)
else:
data["heatmap"] = engine.get_risk_heatmap()
data["clusters"] = engine.compute_herding_clusters()[:5]
return {"ok": True, "data": data}
if cmd == "gt_backtest":
from analytics.backtest import (
DEFAULT_BACKTEST_ALERT_THRESHOLD,
run_historical_backtest,
tune_alert_threshold,
)
from analytics.historical_events import default_historical_cases, expanded_historical_cases
from analytics.settings import gt_analytics_enabled
if not gt_analytics_enabled():
return {
"ok": True,
"data": {
"enabled": False,
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
},
}
expanded = bool(args.get("expanded", True))
tune = bool(args.get("tune", False))
include_cases = bool(args.get("include_cases", False))
try:
target_confidence = float(args.get("target_confidence", 0.95))
except (TypeError, ValueError):
target_confidence = 0.95
if tune:
suite = expanded_historical_cases() if expanded else default_historical_cases()
threshold, report = tune_alert_threshold(
suite,
target_confidence=target_confidence,
)
else:
raw_threshold = args.get("alert_threshold")
threshold = (
float(raw_threshold)
if raw_threshold is not None
else DEFAULT_BACKTEST_ALERT_THRESHOLD
)
report = run_historical_backtest(
use_expanded_suite=expanded,
alert_threshold=threshold,
target_confidence=target_confidence,
)
data = report.to_dict()
data["enabled"] = True
data["expanded_suite"] = expanded
data["tuned"] = tune
data["recommended_alert_threshold"] = threshold
if _wants_compact(args) or not include_cases:
data.pop("cases", None)
return {"ok": True, "data": data}
if cmd == "gt_rolling_freeze":
from analytics.rolling_backtest import freeze_weekly_snapshot
from analytics.settings import gt_analytics_enabled
if not gt_analytics_enabled():
return {
"ok": True,
"data": {
"enabled": False,
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
},
}
week_id = str(args.get("week_id", "") or "").strip() or None
force = bool(args.get("force", False))
result = freeze_weekly_snapshot(
week_id=week_id,
force=force,
frozen_by="openclaw",
)
if not result.get("ok"):
return {"ok": False, "detail": result.get("detail", "Freeze failed")}
data = dict(result)
data["enabled"] = True
if _wants_compact(args):
data.pop("snapshot", None)
return {"ok": True, "data": data}
if cmd == "gt_rolling_label":
from analytics.rolling_backtest import label_region, label_regions
from analytics.settings import gt_analytics_enabled
if not gt_analytics_enabled():
return {
"ok": True,
"data": {
"enabled": False,
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
},
}
week_id = str(args.get("week_id", "") or "").strip()
if not week_id:
return {"ok": False, "detail": "week_id required"}
labels = args.get("labels")
if isinstance(labels, list) and labels:
result = label_regions(week_id, labels, labeled_by="openclaw")
else:
region = str(args.get("region", "") or "").strip().lower()
label = str(args.get("label", "") or "").strip().lower()
if not region or not label:
return {"ok": False, "detail": "region and label required (or labels batch)"}
result = label_region(
week_id,
region,
label, # type: ignore[arg-type]
notes=str(args.get("notes", "") or ""),
labeled_by="openclaw",
)
if not result.get("ok"):
return {"ok": False, "detail": result.get("detail", "Label failed")}
data = dict(result)
data["enabled"] = True
return {"ok": True, "data": data}
if cmd == "gt_rolling_backtest":
from analytics.rolling_backtest import rolling_report
from analytics.settings import gt_analytics_enabled
if not gt_analytics_enabled():
return {
"ok": True,
"data": {
"enabled": False,
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
},
}
try:
weeks = int(args.get("weeks", 8))
except (TypeError, ValueError):
weeks = 8
try:
target_confidence = float(args.get("target_confidence", 0.80))
except (TypeError, ValueError):
target_confidence = 0.80
data = rolling_report(weeks=weeks, target_confidence=target_confidence)
data["enabled"] = True
if _wants_compact(args):
for row in data.get("trend") or []:
if isinstance(row, dict):
row.pop("frozen_at", None)
return {"ok": True, "data": data}
if cmd == "gt_top_alerts":
from analytics.gt_alerts import top_gt_alerts
from analytics.settings import gt_analytics_enabled
if not gt_analytics_enabled():
return {
"ok": True,
"data": {
"enabled": False,
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
},
}
try:
limit = int(args.get("limit", 8))
except (TypeError, ValueError):
limit = 8
data = top_gt_alerts(limit=limit)
data["enabled"] = True
return {"ok": True, "data": data}
if cmd == "gt_micro_rolling":
from analytics.micro_rolling import micro_rolling_report
from analytics.settings import gt_analytics_enabled
if not gt_analytics_enabled():
return {
"ok": True,
"data": {
"enabled": False,
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
},
}
try:
window_days = int(args.get("window_days", 3))
except (TypeError, ValueError):
window_days = 3
try:
limit = int(args.get("limit", 15))
except (TypeError, ValueError):
limit = 15
data = micro_rolling_report(window_days=window_days, limit=limit)
data["enabled"] = True
if _wants_compact(args):
data.pop("top_regions", None)
data["ignitions"] = (data.get("ignitions") or [])[:5]
return {"ok": True, "data": data}
if cmd == "brief_area":
from services.telemetry import entities_near, search_news, get_layer_slice
lat = args.get("lat")
@@ -1131,7 +1418,7 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
from services.openclaw_watchdog import add_watch
watch_type = str(args.get("type", "")).strip()
if not watch_type:
return {"ok": False, "detail": "watch type required (track_aircraft, track_callsign, track_registration, track_ship, track_entity, geofence, keyword, prediction_market)"}
return {"ok": False, "detail": "watch type required (track_aircraft, track_callsign, track_registration, track_ship, track_entity, geofence, keyword, telegram_rhetoric, prediction_market)"}
watch_params = args.get("params", {})
if not watch_params:
# Allow flat args (e.g. {type: "track_callsign", callsign: "N189AM"})
+174
View File
@@ -36,6 +36,14 @@ LATENCY_TIER_MS: dict[str, int] = {
"entity_expand": 40,
"osint_lookup": 200,
"run_playbook": 120,
"gt_risk_heatmap": 20,
"gt_dossier": 25,
"gt_analyze": 80,
"gt_backtest": 120,
"gt_rolling_freeze": 30,
"gt_rolling_label": 20,
"gt_rolling_backtest": 30,
"gt_micro_rolling": 20,
"infonet_status": 20,
"list_gates": 15,
"read_gate_messages": 40,
@@ -255,6 +263,32 @@ def _news_query(text: str) -> str:
return cleaned.strip(" ?.")
def _gt_region_hint(text: str) -> str:
lowered = str(text or "").lower()
hints = (
"ukraine",
"middle east",
"eastern europe",
"baltics",
"israel",
"iran",
"russia",
"china",
"europe",
"united kingdom",
"uk",
"usa",
"united states",
)
for hint in hints:
if hint in lowered:
return "uk" if hint == "united kingdom" else hint
match = re.search(r"\bon\s+([a-z][a-z\s]{2,30})\b", lowered)
if match:
return match.group(1).strip()
return ""
def route_query(
text: str = "",
*,
@@ -370,6 +404,146 @@ def route_query(
})
return _route_result("news_search", recommended, avoid, alternates)
if any(
k in lowered
for k in (
"gt backtest",
"backtest gt",
"historical backtest",
"wilson confidence",
"confidence rate",
"gt benchmark",
"validate gt",
)
):
tune = any(k in lowered for k in ("tune", "grid search", "optimize threshold"))
expanded = "base" not in lowered
recommended = {
"cmd": "gt_backtest",
"args": _compact_args(
{
"expanded": expanded,
"tune": tune,
"target_confidence": 0.95,
},
compact=compact,
),
}
alternates.append({"cmd": "gt_risk_heatmap", "args": {}})
return _route_result("gt_backtest", recommended, avoid, alternates)
if any(
k in lowered
for k in (
"rolling backtest",
"rolling validation",
"weekly validation",
"operational validation",
"operational backtest",
"week over week",
"week-over-week",
"gt rolling",
"rolling gt",
"weekly gt",
"weekly gt score",
"gt weekly",
"gt snapshot",
"freeze weekly gt",
)
):
micro = any(
k in lowered
for k in (
"3 day",
"3-day",
"three day",
"micro rolling",
"rolling average",
"ignition",
"micro gt",
)
)
freeze = any(
k in lowered
for k in ("freeze", "gt snapshot", "weekly snapshot", "capture week")
)
label = any(k in lowered for k in ("label", "outcome", "escalation"))
if micro and not freeze and not label:
recommended = {
"cmd": "gt_micro_rolling",
"args": _compact_args({"window_days": 3}, compact=compact),
}
intent = "gt_micro_rolling"
elif freeze:
recommended = {
"cmd": "gt_rolling_freeze",
"args": _compact_args({"force": "force" in lowered}, compact=compact),
}
intent = "gt_rolling_freeze"
elif label:
recommended = {
"cmd": "gt_rolling_label",
"args": _compact_args({}, compact=compact),
}
intent = "gt_rolling_label"
else:
recommended = {
"cmd": "gt_rolling_backtest",
"args": _compact_args({"weeks": 8, "target_confidence": 0.80}, compact=compact),
}
intent = "gt_rolling_backtest"
alternates.append({"cmd": "gt_micro_rolling", "args": {"window_days": 3}})
alternates.append({"cmd": "gt_backtest", "args": {"expanded": True, "compact": True}})
return _route_result(intent, recommended, avoid, alternates)
if any(
k in lowered
for k in (
"3 day average",
"3-day average",
"rolling 3 day",
"micro risk",
"risk ignition",
)
):
recommended = {
"cmd": "gt_micro_rolling",
"args": _compact_args({"window_days": 3}, compact=compact),
}
alternates.append({"cmd": "gt_rolling_backtest", "args": {"weeks": 8}})
return _route_result("gt_micro_rolling", recommended, avoid, alternates)
if any(
k in lowered
for k in (
"gt analysis",
"game theoretic",
"game-theoretic",
"strategic risk",
"early warning",
"risk heatmap",
"costly signal",
"gt rationale",
)
):
region_hint = _gt_region_hint(raw)
if region_hint and any(k in lowered for k in ("dossier", "rationale", "scenario")):
recommended = {
"cmd": "gt_dossier",
"args": _compact_args({"region": region_hint}, compact=compact),
}
alternates.append({"cmd": "gt_risk_heatmap", "args": {}})
return _route_result("gt_dossier", recommended, avoid, alternates)
recommended = {
"cmd": "gt_analyze",
"args": _compact_args(
{"refresh": True, "region": region_hint} if region_hint else {"refresh": True},
compact=compact,
),
}
alternates.append({"cmd": "gt_risk_heatmap", "args": {}})
return _route_result("gt_analyze", recommended, avoid, alternates)
if lat is not None and lng is not None and any(
k in lowered for k in ("near", "around", "within", "radius", "brief", "aoi")
):
+128 -7
View File
@@ -22,9 +22,12 @@ logger = logging.getLogger(__name__)
_lock = threading.Lock()
_watches: dict[str, dict[str, Any]] = {} # watch_id -> watch definition
_fired: dict[str, float] = {} # watch_id -> last fire timestamp (debounce)
_seen_posts: dict[str, set[str]] = {} # watch_id -> seen Telegram post ids/links
_running = False
_stop_event = threading.Event()
_TELEGRAM_SEEN_MAX = 500
# Minimum seconds between re-firing the same watch
DEBOUNCE_S = 60.0
# How often the watchdog checks telemetry
@@ -73,6 +76,7 @@ def remove_watch(watch_id: str) -> dict[str, Any]:
with _lock:
removed = _watches.pop(watch_id, None)
_fired.pop(watch_id, None)
_seen_posts.pop(watch_id, None)
if removed:
return {"ok": True, "removed": removed}
return {"ok": False, "detail": f"watch '{watch_id}' not found"}
@@ -90,6 +94,7 @@ def clear_watches() -> dict[str, Any]:
count = len(_watches)
_watches.clear()
_fired.clear()
_seen_posts.clear()
return {"ok": True, "cleared": count}
@@ -157,7 +162,9 @@ def _check_watch(watch: dict, fast: dict, slow: dict) -> dict[str, Any] | None:
if wtype == "geofence":
return _check_geofence(params, fast)
if wtype == "keyword":
return _check_keyword(params, fast, slow)
return _check_keyword(watch["id"], params, fast, slow)
if wtype == "telegram_rhetoric":
return _check_telegram_rhetoric(watch["id"], params, slow)
if wtype == "prediction_market":
return _check_prediction_market(params, slow)
@@ -390,15 +397,41 @@ def _check_geofence(params: dict, fast: dict) -> dict | None:
return None
def _check_keyword(params: dict, fast: dict, slow: dict) -> dict | None:
"""Alert when a keyword appears in news/GDELT."""
def _telegram_post_id(post: dict[str, Any]) -> str:
return str(post.get("id") or post.get("link") or "").strip()
def _mark_seen_posts(watch_id: str, post_ids: list[str]) -> None:
clean = [pid for pid in post_ids if pid]
if not clean:
return
with _lock:
seen = _seen_posts.setdefault(watch_id, set())
seen.update(clean)
if len(seen) > _TELEGRAM_SEEN_MAX:
_seen_posts[watch_id] = set(list(seen)[-_TELEGRAM_SEEN_MAX:])
def _is_seen_post(watch_id: str, post_id: str) -> bool:
if not post_id:
return False
with _lock:
return post_id in _seen_posts.get(watch_id, set())
def _check_keyword(watch_id: str, params: dict, fast: dict, slow: dict) -> dict | None:
"""Alert when a keyword appears in news, GDELT, or Telegram OSINT."""
keyword = str(params.get("keyword", "")).lower().strip()
if not keyword:
return None
matches = []
include_telegram = params.get("include_telegram", True)
if isinstance(include_telegram, str):
include_telegram = include_telegram.strip().lower() not in {"0", "false", "no", "off"}
matches = []
new_telegram_ids: list[str] = []
# Check news articles
for article in slow.get("news", []):
title = str(article.get("title", "") or "").lower()
desc = str(article.get("description", "") or article.get("summary", "") or "").lower()
@@ -409,7 +442,6 @@ def _check_keyword(params: dict, fast: dict, slow: dict) -> dict | None:
"url": article.get("url") or article.get("link"),
})
# Check GDELT
for event in slow.get("gdelt", []):
text = str(event.get("title", "") or event.get("sourceurl", "") or "").lower()
if keyword in text:
@@ -419,14 +451,103 @@ def _check_keyword(params: dict, fast: dict, slow: dict) -> dict | None:
"url": event.get("sourceurl"),
})
if include_telegram:
from services.telegram_osint_text import (
iter_telegram_posts,
keyword_matches_telegram_post,
telegram_post_match_entry,
)
for post in iter_telegram_posts(slow.get("telegram_osint")):
if not keyword_matches_telegram_post(post, keyword):
continue
post_id = _telegram_post_id(post)
if _is_seen_post(watch_id, post_id):
continue
entry = telegram_post_match_entry(post)
matches.append(entry)
if post_id:
new_telegram_ids.append(post_id)
if matches:
if new_telegram_ids:
_mark_seen_posts(watch_id, new_telegram_ids)
sources = sorted({str(match.get("source") or "unknown") for match in matches})
return {
"alert": f"Keyword '{keyword}' found in {len(matches)} articles",
"alert": f"Keyword '{keyword}' found in {len(matches)} items ({', '.join(sources)})",
"data": {"keyword": keyword, "matches": matches[:10]},
}
return None
def _check_telegram_rhetoric(watch_id: str, params: dict, slow: dict) -> dict | None:
"""Alert on new high-risk Telegram OSINT posts (optionally keyword/channel filtered)."""
min_risk = int(params.get("min_risk_score", 7) or 7)
min_risk = max(1, min(min_risk, 10))
raw_keywords = params.get("keywords") or params.get("keyword") or []
if isinstance(raw_keywords, str):
raw_keywords = [part.strip() for part in raw_keywords.split(",") if part.strip()]
keywords = [str(item).lower().strip() for item in raw_keywords if str(item).strip()]
raw_channels = params.get("channels") or params.get("channel") or []
if isinstance(raw_channels, str):
raw_channels = [part.strip() for part in raw_channels.split(",") if part.strip()]
channels = [str(item).lower().strip().lstrip("@") for item in raw_channels if str(item).strip()]
from services.telegram_osint_text import (
iter_telegram_posts,
keyword_matches_telegram_post,
telegram_post_match_entry,
)
matches = []
new_post_ids: list[str] = []
for post in iter_telegram_posts(slow.get("telegram_osint")):
try:
risk = int(post.get("risk_score") or 0)
except (TypeError, ValueError):
risk = 0
if risk < min_risk:
continue
channel = str(post.get("channel") or "").lower().strip()
source = str(post.get("source") or "").lower().strip()
if channels and channel not in channels and not any(ch in source for ch in channels):
continue
if keywords and not any(keyword_matches_telegram_post(post, kw) for kw in keywords):
continue
post_id = _telegram_post_id(post)
if _is_seen_post(watch_id, post_id):
continue
entry = telegram_post_match_entry(post)
matches.append(entry)
if post_id:
new_post_ids.append(post_id)
if not matches:
return None
_mark_seen_posts(watch_id, new_post_ids)
top = max(int(match.get("risk_score") or 0) for match in matches)
return {
"alert": (
f"Telegram rhetoric alert: {len(matches)} new post(s) at LVL {top}/10"
+ (f" (min {min_risk})" if min_risk > 1 else "")
),
"data": {
"min_risk_score": min_risk,
"keywords": keywords,
"channels": channels,
"matches": matches[:10],
},
}
def _check_prediction_market(params: dict, slow: dict) -> dict | None:
"""Alert on prediction market movements."""
query = str(params.get("query", "")).lower().strip()
+106
View File
@@ -0,0 +1,106 @@
"""Container-aware runtime limits for fleet vs desktop deployments."""
from __future__ import annotations
import os
from functools import lru_cache
from pathlib import Path
from typing import Any
def _read_first_int(path: Path) -> int | None:
try:
raw = path.read_text(encoding="utf-8").strip().split()[0]
return int(raw)
except (OSError, ValueError, IndexError):
return None
def detect_cpu_limit() -> float | None:
"""Effective CPU cores from cgroup quota (Docker ``cpus:``), else host count."""
cgroup_v2 = Path("/sys/fs/cgroup/cpu.max")
if cgroup_v2.is_file():
try:
parts = cgroup_v2.read_text(encoding="utf-8").strip().split()
if len(parts) >= 2 and parts[0] != "max":
quota = int(parts[0])
period = int(parts[1])
if quota > 0 and period > 0:
return round(quota / period, 3)
except (OSError, ValueError):
pass
cgroup_v1_quota = Path("/sys/fs/cgroup/cpu/cpu.cfs_quota_us")
cgroup_v1_period = Path("/sys/fs/cgroup/cpu/cpu.cfs_period_us")
if cgroup_v1_quota.is_file() and cgroup_v1_period.is_file():
quota = _read_first_int(cgroup_v1_quota)
period = _read_first_int(cgroup_v1_period)
if quota is not None and period and quota > 0:
return round(quota / period, 3)
try:
import os as _os
count = _os.cpu_count()
return float(count) if count else None
except Exception:
return None
def detect_memory_limit_mb() -> int | None:
cgroup_v2 = Path("/sys/fs/cgroup/memory.max")
if cgroup_v2.is_file():
try:
raw = cgroup_v2.read_text(encoding="utf-8").strip()
if raw and raw != "max":
return int(int(raw) / (1024 * 1024))
except (OSError, ValueError):
pass
cgroup_v1 = Path("/sys/fs/cgroup/memory/memory.limit_in_bytes")
if cgroup_v1.is_file():
try:
raw = _read_first_int(cgroup_v1)
if raw is not None and raw < (1 << 62):
return int(raw / (1024 * 1024))
except (OSError, ValueError):
pass
return None
def resolve_profile_name() -> str:
explicit = str(os.environ.get("GT_ANALYTICS_PROFILE", "")).strip().lower()
if explicit in {"lean", "standard"}:
return explicit
cpu = detect_cpu_limit()
if cpu is not None and cpu <= 1.0:
return "lean"
return "standard"
@lru_cache(maxsize=1)
def get_runtime_profile() -> dict[str, Any]:
cpu_limit = detect_cpu_limit()
memory_mb = detect_memory_limit_mb()
profile = resolve_profile_name()
lean = profile == "lean"
return {
"profile": profile,
"cpu_limit": cpu_limit,
"memory_limit_mb": memory_mb,
"gt_analytics": {
"recommended": not lean,
"lean_node": lean,
"warning": (
"This node is capped at 1 vCPU. Enabling Strategic Risk (Derived OSINT) "
"may slow Telegram, GDELT, and other OSINT fetches. Set "
"GT_ANALYTICS_ACK_LOW_CPU=true after enabling GT_ANALYTICS_ENABLED to run "
"the full engine on lean hardware."
if lean
else None
),
},
}
def clear_runtime_profile_cache() -> None:
get_runtime_profile.cache_clear()
+1
View File
@@ -19,6 +19,7 @@ class HealthResponse(BaseModel):
# insecure-date path because the upstream Let's Encrypt cert is
# expired. Empty dict / null means no status reported yet.
ais_proxy: Optional[Dict[str, Any]] = None
runtime: Optional[Dict[str, Any]] = None
class RefreshResponse(BaseModel):
+66
View File
@@ -0,0 +1,66 @@
"""Shared Telegram OSINT post text helpers for search and watchdog matching."""
from __future__ import annotations
from typing import Any
from services.telegram_translate import source_lang_label
def iter_telegram_posts(layer_payload: Any) -> list[dict[str, Any]]:
"""Normalize telegram_osint layer payloads into a list of post dicts."""
if isinstance(layer_payload, list):
return [post for post in layer_payload if isinstance(post, dict)]
if isinstance(layer_payload, dict):
posts = layer_payload.get("posts")
if isinstance(posts, list):
return [post for post in posts if isinstance(post, dict)]
return []
def telegram_post_search_text(post: dict[str, Any]) -> str:
"""Build a lowercase haystack for keyword matching (translated + original)."""
parts = (
post.get("title_translated"),
post.get("description_translated"),
post.get("title"),
post.get("description"),
post.get("source"),
post.get("channel"),
)
return " ".join(str(part).strip() for part in parts if str(part or "").strip()).lower()
def telegram_post_display_title(post: dict[str, Any]) -> str:
"""Prefer translated headline for alerts and agent-facing summaries."""
translated = str(post.get("title_translated") or post.get("description_translated") or "").strip()
if translated:
return translated.split("\n", 1)[0][:200]
return str(post.get("title") or post.get("description") or "").strip()[:200]
def telegram_post_match_entry(post: dict[str, Any]) -> dict[str, Any]:
"""Compact match record for watchdog alerts and search results."""
lat, lng = None, None
coords = post.get("coords")
if isinstance(coords, (list, tuple)) and len(coords) >= 2:
lat, lng = coords[0], coords[1]
return {
"source": "telegram_osint",
"title": telegram_post_display_title(post),
"original_title": str(post.get("title") or "").strip(),
"url": post.get("link") or "",
"channel": post.get("channel") or post.get("source") or "",
"risk_score": post.get("risk_score"),
"source_lang": post.get("source_lang"),
"source_lang_label": post.get("source_lang_label") or source_lang_label(post.get("source_lang")),
"lat": lat,
"lng": lng,
"id": post.get("id") or post.get("link") or "",
}
def keyword_matches_telegram_post(post: dict[str, Any], keyword: str) -> bool:
needle = str(keyword or "").strip().lower()
if not needle:
return False
return needle in telegram_post_search_text(post)
+243
View File
@@ -0,0 +1,243 @@
"""Auto-translation for Telegram OSINT post text (server-side, cached)."""
from __future__ import annotations
import hashlib
import logging
import os
import re
import urllib.parse
from threading import Lock
from typing import Any
import requests
logger = logging.getLogger(__name__)
_CYRILLIC_RE = re.compile(r"[\u0400-\u04FF]")
_UKRAINIAN_MARKERS_RE = re.compile(r"[іїєґІЇЄҐ]")
_ARABIC_RE = re.compile(r"[\u0600-\u06FF]")
_HEBREW_RE = re.compile(r"[\u0590-\u05FF]")
_CJK_RE = re.compile(r"[\u4e00-\u9fff]")
# Common war-reporting shorthand that machine translation often transliterates.
_POST_TRANSLATION_GLOSSARY: tuple[tuple[re.Pattern[str], str], ...] = (
(re.compile(r"\bBpLa\b", re.IGNORECASE), "UAV"),
(re.compile(r"\bБпЛА\b", re.IGNORECASE), "UAV"),
(re.compile(r"\bбпла\b"), "UAV"),
(re.compile(r"\bБПЛА\b"), "UAV"),
(re.compile(r"\bрсзв\b", re.IGNORECASE), "MLRS"),
(re.compile(r"\bРСЗВ\b"), "MLRS"),
)
_SOURCE_LANG_LABELS = {
"uk": "Ukrainian",
"ru": "Russian",
"en": "English",
"ar": "Arabic",
"he": "Hebrew",
"zh-cn": "Chinese",
"fr": "French",
"de": "German",
"pl": "Polish",
}
_CACHE: dict[str, tuple[str, str]] = {}
_CACHE_LOCK = Lock()
_CACHE_MAX = 512
_LOCALE_TO_GOOGLE = {
"en": "en",
"fr": "fr",
"zh-cn": "zh-CN",
"zh": "zh-CN",
}
def telegram_translate_enabled() -> bool:
return str(os.environ.get("TELEGRAM_OSINT_TRANSLATE", "true")).strip().lower() not in {
"0",
"false",
"no",
"off",
"",
}
def telegram_translate_target() -> str:
raw = str(os.environ.get("TELEGRAM_OSINT_TRANSLATE_TO", "en")).strip().lower()
return _LOCALE_TO_GOOGLE.get(raw, raw or "en")
def normalize_translate_target(locale: str | None) -> str:
raw = str(locale or telegram_translate_target()).strip().lower().replace("_", "-")
return _LOCALE_TO_GOOGLE.get(raw, raw or "en")
def _looks_english(text: str) -> bool:
letters = [char for char in text if char.isalpha()]
if not letters:
return True
ascii_letters = sum(1 for char in letters if ord(char) < 128)
return ascii_letters / len(letters) > 0.9
def contains_cyrillic(text: str) -> bool:
return bool(_CYRILLIC_RE.search(str(text or "")))
def source_lang_label(code: str | None) -> str:
raw = str(code or "").strip().lower().replace("_", "-")
return _SOURCE_LANG_LABELS.get(raw, raw.upper() if raw else "Unknown")
def polish_translation(text: str) -> str:
polished = str(text or "")
for pattern, replacement in _POST_TRANSLATION_GLOSSARY:
polished = pattern.sub(replacement, polished)
return polished.strip()
def guess_source_lang(text: str) -> str:
if _UKRAINIAN_MARKERS_RE.search(text):
return "uk"
if _CYRILLIC_RE.search(text):
return "ru"
if _ARABIC_RE.search(text):
return "ar"
if _HEBREW_RE.search(text):
return "he"
if _CJK_RE.search(text):
return "zh-CN"
if _looks_english(text):
return "en"
return "auto"
def _cache_key(text: str, target_lang: str) -> str:
digest = hashlib.sha1(f"{target_lang}|{text}".encode("utf-8")).hexdigest()
return digest
def _cache_get(text: str, target_lang: str) -> tuple[str, str] | None:
key = _cache_key(text, target_lang)
with _CACHE_LOCK:
return _CACHE.get(key)
def _cache_put(text: str, target_lang: str, translated: str, source_lang: str) -> None:
key = _cache_key(text, target_lang)
with _CACHE_LOCK:
if len(_CACHE) >= _CACHE_MAX:
_CACHE.pop(next(iter(_CACHE)))
_CACHE[key] = (translated, source_lang)
def _google_translate(clean: str, target: str, source: str | None = None) -> tuple[str, str]:
params = {
"client": "gtx",
"sl": source or "auto",
"tl": target,
"dt": "t",
"q": clean[:4500],
}
url = "https://translate.googleapis.com/translate_a/single?" + urllib.parse.urlencode(params)
resp = requests.get(
url,
timeout=8,
headers={"User-Agent": "Mozilla/5.0 (compatible; Shadowbroker-Telegram-Translate/1.0)"},
)
resp.raise_for_status()
data = resp.json()
detected = str(data[2] or guess_source_lang(clean)).strip().lower()
if detected in {"zh-cn", "zh-tw"}:
detected = "zh-CN"
parts: list[str] = []
for chunk in data[0] or []:
if chunk and chunk[0]:
parts.append(str(chunk[0]))
translated = polish_translation("".join(parts).strip() or clean)
return translated, detected
def translate_text(text: str, target_lang: str | None = None) -> tuple[str, str]:
"""Translate text via Google Translate (unofficial client endpoint).
Returns ``(translated_text, detected_source_lang)``.
"""
clean = str(text or "").strip()
if not clean:
return "", "en"
target = normalize_translate_target(target_lang)
if _looks_english(clean) and target == "en":
return clean, "en"
cached = _cache_get(clean, target)
if cached:
return cached
try:
translated, detected = _google_translate(clean, target)
if detected == target or (detected == "en" and target == "en"):
result = (clean, detected)
_cache_put(clean, target, clean, detected)
return result
if contains_cyrillic(translated) and contains_cyrillic(clean):
hinted = guess_source_lang(clean)
if hinted not in {"auto", target}:
retry_translated, retry_detected = _google_translate(clean, target, hinted)
if not contains_cyrillic(retry_translated) or len(retry_translated) > len(translated):
translated, detected = retry_translated, retry_detected
result = (translated, detected)
_cache_put(clean, target, translated, detected)
return result
except Exception as exc:
logger.warning("Telegram translation failed: %s", exc)
fallback_lang = guess_source_lang(clean)
return clean, fallback_lang
def apply_post_translation(post: dict[str, Any], target_lang: str | None = None) -> dict[str, Any]:
"""Add translation fields to a Telegram OSINT post dict."""
if not telegram_translate_enabled():
return post
target = normalize_translate_target(target_lang)
description = str(post.get("description") or "").strip()
title = str(post.get("title") or "").strip()
full_text = description or title
if not full_text:
return post
existing_translated = str(post.get("description_translated") or post.get("title_translated") or "").strip()
if post.get("translate_to") == target and existing_translated:
updated = dict(post)
polished = polish_translation(existing_translated)
if polished != existing_translated:
lines = polished.split("\n", 1)
updated["title_translated"] = lines[0][:160]
updated["description_translated"] = polished[:1200]
updated["source_lang_label"] = source_lang_label(str(post.get("source_lang") or ""))
return updated
translated_full, source_lang = translate_text(full_text, target)
updated = dict(post)
updated["source_lang"] = source_lang
updated["translate_to"] = target
updated["source_lang_label"] = source_lang_label(source_lang)
if translated_full != full_text and source_lang != target:
lines = translated_full.split("\n", 1)
updated["title_translated"] = lines[0][:160]
updated["description_translated"] = translated_full[:1200]
return updated
def apply_posts_translations(
posts: list[dict[str, Any]],
target_lang: str | None = None,
) -> list[dict[str, Any]]:
if not telegram_translate_enabled():
return posts
return [apply_post_translation(post, target_lang) for post in posts]
+15 -14
View File
@@ -97,6 +97,7 @@ _SLOW_KEYS = (
"cyber_threats",
"scm_suppliers",
"telegram_osint",
"gt_risk",
)
@@ -210,6 +211,9 @@ _LAYER_ALIASES = {
"telegram": "telegram_osint",
"telegram_osint": "telegram_osint",
"osint_feed": "telegram_osint",
"gt_risk": "gt_risk",
"strategic_risk": "gt_risk",
"gt_analytics": "gt_risk",
"malware": "malware_threats",
"malware_threats": "malware_threats",
"malware_c2": "malware_threats",
@@ -710,10 +714,10 @@ _UNIVERSAL_SEARCH_SPECS: dict[str, dict[str, Any]] = {
"time_fields": ("updated_at", "timestamp"),
},
"telegram_osint": {
"fields": ("title", "description", "source", "channel", "link"),
"primary_fields": ("title", "description", "channel"),
"label_fields": ("title", "channel"),
"summary_fields": ("description", "source"),
"fields": ("title", "description", "title_translated", "description_translated", "source", "channel", "link"),
"primary_fields": ("title_translated", "title", "description_translated", "description", "channel"),
"label_fields": ("title_translated", "title", "channel"),
"summary_fields": ("description_translated", "description", "source"),
"type_fields": ("channel", "source"),
"id_fields": ("id", "link"),
"time_fields": ("published", "timestamp"),
@@ -2089,30 +2093,27 @@ def search_news(
return {"results": out, "version": get_data_version(), "truncated": True}
if include_telegram:
from services.telegram_osint_text import telegram_post_display_title, telegram_post_search_text
for post in _unwrap_layer_items(snap.get("telegram_osint"), "telegram_osint"):
if not isinstance(post, dict):
continue
text = " ".join(
(
_norm_text(post.get("title")),
_norm_text(post.get("description")),
_norm_text(post.get("source")),
_norm_text(post.get("channel")),
)
)
text = telegram_post_search_text(post)
if not _text_matches_query(query_norm, text):
continue
lat, lng = _extract_coords(post)
out.append(
{
"source_layer": "telegram_osint",
"title": post.get("title") or "",
"summary": post.get("description") or "",
"title": telegram_post_display_title(post),
"summary": post.get("description_translated") or post.get("description") or "",
"source": post.get("source") or post.get("channel") or "Telegram",
"link": post.get("link") or "",
"lat": lat,
"lng": lng,
"risk_score": post.get("risk_score"),
"source_lang": post.get("source_lang"),
"source_lang_label": post.get("source_lang_label"),
}
)
if len(out) >= limit:
+14
View File
@@ -1,9 +1,23 @@
import os
import asyncio
import pytest
from unittest.mock import patch, MagicMock
@pytest.fixture(autouse=True)
def _gt_analytics_standard_profile(monkeypatch: pytest.MonkeyPatch) -> None:
"""Tests assume a standard (non-lean) runtime unless they override profile."""
monkeypatch.setenv("GT_ANALYTICS_PROFILE", os.environ.get("GT_ANALYTICS_PROFILE", "standard"))
try:
from analytics.integration import reset_gt_engine
reset_gt_engine()
except Exception:
pass
@pytest.fixture(autouse=True)
def _suppress_background_services():
"""Prevent real scheduler/stream/tracker from starting during tests."""
+141
View File
@@ -0,0 +1,141 @@
"""API tests for Strategic Risk Analytics routes."""
from __future__ import annotations
import pytest
from analytics.integration import reset_gt_engine
from services.fetchers import _store
@pytest.fixture(autouse=True)
def _reset_gt(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("GT_ANALYTICS_ENABLED", raising=False)
reset_gt_engine()
def test_risk_heatmap_disabled(client) -> None:
response = client.get("/api/analytics/risk_heatmap")
assert response.status_code == 200
payload = response.json()
assert payload["enabled"] is False
assert payload["type"] == "FeatureCollection"
assert payload["features"] == []
def test_dossier_disabled(client) -> None:
response = client.get("/api/analytics/dossier/ukraine")
assert response.status_code == 200
payload = response.json()
assert payload["enabled"] is False
assert payload["region"] == "ukraine"
def test_risk_heatmap_enabled_after_refresh(client, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
reset_gt_engine()
_store.latest_data["telegram_osint"] = {
"posts": [
{
"id": "api-tg-1",
"title": "Troop buildup",
"description": "Troop movement and armored convoy reported near border.",
"source": "t.me/war_monitor",
"channel": "war_monitor",
"coords": [48.5, 37.5],
}
],
"total": 1,
"geolocated": 1,
}
_store.latest_data["news"] = []
_store.latest_data["gdelt"] = []
from analytics.integration import refresh_from_latest_data
refresh_from_latest_data(dict(_store.latest_data), persist=True)
response = client.get("/api/analytics/risk_heatmap")
assert response.status_code == 200
payload = response.json()
assert payload["enabled"] is True
assert len(payload["features"]) >= 1
assert payload["timestamp"] is not None
def test_dossier_enabled(client, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
reset_gt_engine()
_store.latest_data["telegram_osint"] = {
"posts": [
{
"id": "api-tg-2",
"title": "Strike",
"description": "General strike and protest mobilization in capital.",
"source": "t.me/nexta_live",
"channel": "nexta_live",
"coords": [50.45, 30.52],
}
]
}
_store.latest_data["news"] = []
_store.latest_data["gdelt"] = []
from analytics.integration import refresh_from_latest_data
refresh_from_latest_data(dict(_store.latest_data), persist=True)
response = client.get("/api/analytics/dossier/50.45,30.52")
assert response.status_code == 200
payload = response.json()
assert payload["enabled"] is True
assert payload["recent_signals"]
assert "interpretation" in payload
def test_post_risk_heatmap_ingest(client, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
reset_gt_engine()
response = client.post(
"/api/analytics/risk_heatmap",
json={
"refresh": False,
"items": [
{
"title": "GPS interference",
"description": "GPS jamming spike along northern corridor.",
"source": "manual",
"region": "baltics",
"domain": "conflict",
}
],
},
)
assert response.status_code == 200
payload = response.json()
assert payload["enabled"] is True
assert payload["ingested"] == 1
def test_backtest_disabled(client) -> None:
response = client.get("/api/analytics/backtest")
assert response.status_code == 200
payload = response.json()
assert payload["enabled"] is False
def test_backtest_enabled(client, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
reset_gt_engine()
response = client.get("/api/analytics/backtest?expanded=true&tune=false")
assert response.status_code == 200
payload = response.json()
assert payload["enabled"] is True
assert payload["accuracy"] == 1.0
assert payload["confidence_rate"] >= 0.95
assert payload["meets_target"] is True
assert payload["total_cases"] >= 80
+56
View File
@@ -0,0 +1,56 @@
"""Top GT alerts ranking and coordinate filtering."""
from __future__ import annotations
from analytics.gt_alerts import parse_heatmap_alerts, top_gt_alerts
def test_parse_heatmap_filters_invalid_coords() -> None:
heatmap = {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {
"region": "ukraine",
"risk": 0.55,
"conflict": 0.62,
"financial": 0.15,
"unrest": 0.2,
},
"geometry": {"type": "Point", "coordinates": [31.0, 48.0]},
},
{
"type": "Feature",
"properties": {"region": "no_coords", "risk": 0.9},
"geometry": {"type": "Point", "coordinates": [0.0, 0.0]},
},
{
"type": "Feature",
"properties": {"region": "global", "risk": 0.99},
"geometry": {"type": "Point", "coordinates": [0.0, 0.0]},
},
],
}
alerts, plotted = parse_heatmap_alerts(heatmap, limit=5)
assert plotted == 1
assert len(alerts) == 1
assert alerts[0]["region"] == "ukraine"
assert alerts[0]["lat"] == 48.0
assert alerts[0]["lng"] == 31.0
def test_region_label_formats_coordinates() -> None:
from analytics.gt_alerts import _region_label
assert "48.00" in _region_label("48.00,31.17")
assert _region_label("ukraine") == "ukraine"
def test_top_gt_alerts_disabled(monkeypatch) -> None:
monkeypatch.delenv("GT_ANALYTICS_ENABLED", raising=False)
from analytics.integration import reset_gt_engine
reset_gt_engine()
report = top_gt_alerts(limit=3)
assert report["alerts"] == []
+52
View File
@@ -0,0 +1,52 @@
"""Historical backtest validation for Strategic Risk Analytics."""
from __future__ import annotations
from analytics.backtest import (
DEFAULT_BACKTEST_ALERT_THRESHOLD,
MAX_BACKTEST_ALERT_THRESHOLD,
run_historical_backtest,
tune_alert_threshold,
wilson_interval,
)
from analytics.historical_events import default_historical_cases, expanded_historical_cases
def test_wilson_interval_perfect_run() -> None:
lower, upper = wilson_interval(18, 18)
assert lower >= 0.80
assert upper == 1.0
def test_base_suite_meets_eighty_percent_confidence() -> None:
report = run_historical_backtest(
default_historical_cases(),
use_expanded_suite=False,
target_confidence=0.80,
)
assert report.accuracy >= 0.95
assert report.confidence_rate >= 0.80
assert report.meets_target
assert report.false_positives == 0
assert report.false_negatives == 0
def test_expanded_suite_meets_ninety_five_percent_confidence() -> None:
threshold, report = tune_alert_threshold(target_confidence=0.95)
assert len(expanded_historical_cases()) >= 80
assert report.accuracy == 1.0
assert report.confidence_rate >= 0.95
assert report.meets_target
assert report.false_positives == 0
assert report.false_negatives == 0
assert DEFAULT_BACKTEST_ALERT_THRESHOLD <= threshold <= MAX_BACKTEST_ALERT_THRESHOLD
def test_default_backtest_threshold_on_expanded_suite() -> None:
report = run_historical_backtest(
use_expanded_suite=True,
target_confidence=0.95,
)
assert report.alert_threshold == DEFAULT_BACKTEST_ALERT_THRESHOLD
assert report.accuracy == 1.0
assert report.confidence_rate >= 0.95
+150
View File
@@ -0,0 +1,150 @@
"""Tests for Strategic Risk Analytics core scoring."""
from __future__ import annotations
import pytest
from analytics.feed_adapter import normalize_feed_item
from analytics.gt_early_warning import GT_EarlyWarning
from analytics.integration import process_feed_item, refresh_from_latest_data, reset_gt_engine
from analytics.settings import GTAnalyticsSettings
@pytest.fixture
def engine() -> GT_EarlyWarning:
return GT_EarlyWarning(
GTAnalyticsSettings(
enabled=True,
base_prior=0.15,
evidence_cap=3.0,
evidence_scale=5.0,
high_risk_threshold=0.6,
)
)
def test_classify_payroll_loan_signal(engine: GT_EarlyWarning) -> None:
signals = engine.classify_signals("Franchise owners increasingly rely on payroll loan facilities.")
assert "payroll_loan" in signals
assert signals["payroll_loan"] >= 3.0
def test_classify_no_signal_on_generic_text(engine: GT_EarlyWarning) -> None:
signals = engine.classify_signals("Sunny weather expected across the region this weekend.")
assert signals == {}
def test_bayesian_update_increases_risk(engine: GT_EarlyWarning) -> None:
prior = engine.get_prior("uk", "financial")
posterior = engine.bayesian_update("uk", "financial", evidence_strength=2.0)
assert posterior > prior
def test_process_feed_item_updates_region(engine: GT_EarlyWarning) -> None:
item = {
"id": "test-1",
"text": "Mass rally and general strike announced; protest mobilization spreads.",
"source": "t.me/osintdefender",
"region": "ukraine",
"domain": "unrest",
"entities": ["channel:osintdefender"],
"coords": [50.45, 30.52],
}
result = engine.process_feed_item(item)
assert result["signals"]
assert result["risk_score"] > engine.settings.base_prior
assert result["contagion_potential"] >= 0.0
def test_duplicate_items_are_skipped(engine: GT_EarlyWarning) -> None:
item = {
"id": "dup-1",
"text": "GPS jamming spike reported near border corridor.",
"source": "gdelt",
"region": "baltics",
"domain": "conflict",
}
first = engine.process_feed_item(item)
second = engine.process_feed_item(item)
assert not first.get("skipped")
assert second.get("skipped") is True
def test_heatmap_returns_geojson_features(engine: GT_EarlyWarning) -> None:
engine.process_feed_item(
{
"id": "heat-1",
"text": "Troop movement and armored convoy observed overnight.",
"source": "news",
"region": "eastern_europe",
"coords": [48.0, 37.0],
}
)
heatmap = engine.get_risk_heatmap()
assert heatmap["type"] == "FeatureCollection"
assert len(heatmap["features"]) >= 1
feature = heatmap["features"][0]
assert "risk" in feature["properties"]
assert feature["geometry"]["type"] == "Point"
def test_dossier_includes_recent_signals(engine: GT_EarlyWarning) -> None:
engine.process_feed_item(
{
"id": "dos-1",
"text": "Supply chain delay at major port; logistics backlog worsens.",
"source": "news",
"region": "china",
"domain": "financial",
}
)
dossier = engine.get_dossier("china")
assert dossier["region"] == "china"
assert dossier["recent_signals"]
assert "interpretation" in dossier
def test_feed_adapter_normalizes_telegram_post() -> None:
normalized = normalize_feed_item(
{
"title": "Strike expands",
"description": "General strike and rally planned in capital.",
"source": "t.me/nexta_live",
"channel": "nexta_live",
"coords": [53.9, 27.56],
},
source_type="telegram_osint",
)
assert normalized["region"] != "global"
assert normalized["domain"] in {"unrest", "financial", "conflict"}
assert normalized["text"]
def test_integration_disabled_by_default(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("GT_ANALYTICS_ENABLED", raising=False)
reset_gt_engine()
assert process_feed_item({"text": "test", "region": "global"}) is None
def test_refresh_from_latest_data_processes_telegram(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
reset_gt_engine()
latest = {
"telegram_osint": {
"posts": [
{
"id": "tg-1",
"title": "GPS jamming",
"description": "GPS jamming spike reported along northern border.",
"source": "t.me/osintdefender",
"channel": "osintdefender",
"coords": [59.93, 30.33],
}
]
},
"news": [],
"gdelt": [],
}
summary = refresh_from_latest_data(latest, persist=False)
assert summary["enabled"] is True
assert summary["processed"] >= 1
@@ -0,0 +1,29 @@
"""GT feed adapter uses Telegram English translations for costly-signal matching."""
from __future__ import annotations
from analytics.feed_adapter import normalize_feed_item
def test_telegram_prefers_translated_text_for_gt() -> None:
post = {
"title": "Київ 1х БпЛА",
"description": "Обстріл біля Харкова",
"title_translated": "Kyiv 1x UAV",
"description_translated": "Shelling near Kharkiv with troop movement reported",
"source": "t.me/osintdefender",
"coords": [49.99, 36.23],
}
item = normalize_feed_item(post, source_type="telegram_osint")
assert "troop movement" in item["text"].lower()
assert item["domain"] == "conflict"
def test_hashtag_region_maps_ukraine_dossier_key() -> None:
post = {
"title": "Update",
"description_translated": "#Ukraine #USA aircraft spotted on runway",
"source": "t.me/osintdefender",
}
item = normalize_feed_item(post, source_type="telegram_osint")
assert item["region"] == "ukraine"
+35
View File
@@ -0,0 +1,35 @@
"""Lean-profile gating for Strategic Risk Analytics."""
from __future__ import annotations
import pytest
from analytics.integration import get_gt_engine, maybe_refresh_gt_analytics, reset_gt_engine
from analytics.settings import gt_engine_operational, gt_scheduled_ingest_enabled
def test_gt_engine_blocked_on_lean_without_ack(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
monkeypatch.setenv("GT_ANALYTICS_PROFILE", "lean")
monkeypatch.delenv("GT_ANALYTICS_ACK_LOW_CPU", raising=False)
reset_gt_engine()
assert gt_engine_operational() is False
assert get_gt_engine() is None
def test_gt_engine_allowed_on_lean_with_ack(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
monkeypatch.setenv("GT_ANALYTICS_PROFILE", "lean")
monkeypatch.setenv("GT_ANALYTICS_ACK_LOW_CPU", "true")
reset_gt_engine()
assert gt_engine_operational() is True
assert get_gt_engine() is not None
def test_scheduled_ingest_skipped_on_lean(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
monkeypatch.setenv("GT_ANALYTICS_PROFILE", "lean")
monkeypatch.delenv("GT_ANALYTICS_ACK_LOW_CPU", raising=False)
reset_gt_engine()
assert gt_scheduled_ingest_enabled() is False
maybe_refresh_gt_analytics()
+121
View File
@@ -0,0 +1,121 @@
"""Micro rolling 3-day average for Strategic Risk Analytics."""
from __future__ import annotations
from datetime import date, timedelta
from pathlib import Path
import pytest
from analytics.daily_store import DailyRegionReading, DailySnapshot, date_id, save_daily
from analytics.gt_early_warning import GT_EarlyWarning
from analytics.micro_rolling import (
capture_daily_readings,
compute_micro_view,
enrich_heatmap_features,
micro_rolling_report,
)
from analytics.settings import GTAnalyticsSettings
@pytest.fixture()
def daily_store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
store = tmp_path / "daily"
monkeypatch.setenv("GT_DAILY_STORE_DIR", str(store))
return store
def _seed_engine() -> GT_EarlyWarning:
engine = GT_EarlyWarning(GTAnalyticsSettings(enabled=True, base_prior=0.15))
engine.process_feed_item(
{
"text": "Troop movement and military mobilization near border",
"region": "ukraine",
"source": "test",
"source_type": "manual",
}
)
return engine
def _save_day(day: date, region: str, peak: float) -> None:
day_key = date_id(day)
snap = DailySnapshot(date=day_key, regions={})
snap.regions[region] = DailyRegionReading(
region=region,
composite_risk=peak * 0.9,
financial=0.15,
unrest=0.15,
conflict=peak,
peak_score=peak,
readings=1,
last_captured_at=f"{day_key}T12:00:00+00:00",
)
save_daily(snap)
def test_capture_daily_readings(daily_store: Path) -> None:
engine = _seed_engine()
result = capture_daily_readings(engine, when=date(2026, 6, 16))
assert result["regions"] >= 1
again = capture_daily_readings(engine, when=date(2026, 6, 16))
assert again["regions"] >= 1
def test_3day_rolling_average_and_ignition(daily_store: Path) -> None:
region = "ukraine"
today = date(2026, 6, 16)
_save_day(today - timedelta(days=2), region, 0.20)
_save_day(today - timedelta(days=1), region, 0.22)
_save_day(today, region, 0.45)
view = compute_micro_view(region, as_of=today, window_days=3)
assert view is not None
assert view.days_in_window == 3
assert view.risk_3d_avg == pytest.approx(0.29, abs=0.01)
assert view.spot_risk == 0.45
assert view.risk_delta == pytest.approx(0.16, abs=0.01)
assert view.ignition is True
def test_enrich_heatmap_features(daily_store: Path) -> None:
engine = _seed_engine()
today = date(2026, 6, 16)
capture_daily_readings(engine, when=today)
heatmap = engine.get_risk_heatmap()
enriched = enrich_heatmap_features(heatmap, as_of=today, window_days=3)
feature = enriched["features"][0]
props = feature["properties"]
assert "risk_3d_avg" in props
assert "risk_spot" in props
assert "micro_ignition" in props
def test_micro_rolling_report(daily_store: Path) -> None:
region = "ukraine"
today = date(2026, 6, 16)
_save_day(today - timedelta(days=1), region, 0.21)
_save_day(today, region, 0.40)
report = micro_rolling_report(as_of=today, window_days=3, limit=5)
assert report["mode"] == "micro_rolling"
assert report["window_days"] == 3
assert report["regions_tracked"] >= 1
def test_openclaw_micro_command(daily_store: Path, monkeypatch: pytest.MonkeyPatch) -> None:
from analytics.integration import reset_gt_engine
from services.openclaw_channel import _dispatch_command
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
reset_gt_engine()
result = _dispatch_command("gt_micro_rolling", {"window_days": 3, "compact": True})
assert result["ok"] is True
assert result["data"]["mode"] == "micro_rolling"
def test_route_query_micro_intent() -> None:
from services.openclaw_routing import route_query
plan = route_query("Show GT rolling 3 day average and ignition regions")
assert plan["recommended"]["cmd"] == "gt_micro_rolling"
+170
View File
@@ -0,0 +1,170 @@
"""Rolling weekly operational validation for Strategic Risk Analytics."""
from __future__ import annotations
import json
from datetime import date
from pathlib import Path
import pytest
from analytics.backtest import DEFAULT_BACKTEST_ALERT_THRESHOLD
from analytics.gt_early_warning import GT_EarlyWarning
from analytics.integration import reset_gt_engine
from analytics.rolling_backtest import (
freeze_weekly_snapshot,
iso_week_id,
label_regions,
rolling_report,
score_week,
)
from analytics.settings import GTAnalyticsSettings
from analytics.weekly_store import RegionSnapshot, WeeklySnapshot, load_week
@pytest.fixture()
def rolling_store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
store = tmp_path / "gt_rolling"
monkeypatch.setenv("GT_ROLLING_STORE_DIR", str(store))
return store
def _seed_engine() -> GT_EarlyWarning:
engine = GT_EarlyWarning(GTAnalyticsSettings(enabled=True, base_prior=0.15))
engine.process_feed_item(
{
"text": "Troop movement and military mobilization near border",
"region": "ukraine",
"source": "test",
"source_type": "manual",
}
)
engine.process_feed_item(
{
"text": "Routine diplomatic statement about trade",
"region": "canada",
"source": "test",
"source_type": "manual",
}
)
return engine
def test_iso_week_id_format() -> None:
assert iso_week_id(date(2026, 6, 16)) == "2026-W25"
def test_freeze_and_score_week(rolling_store: Path) -> None:
engine = _seed_engine()
result = freeze_weekly_snapshot(
week_id="2026-W10",
engine=engine,
frozen_by="test",
)
assert result["ok"] is True
assert result["created"] is True
assert result["region_count"] >= 2
snapshot = load_week("2026-W10")
assert snapshot is not None
ukraine = next(row for row in snapshot.regions if row.region == "ukraine")
assert ukraine.alerted is True
pending_score = score_week(snapshot)
assert pending_score.labeled == 0
assert pending_score.scorable is False
label_regions(
"2026-W10",
[
{"region": "ukraine", "label": "true_escalation"},
{"region": "canada", "label": "benign"},
],
)
labeled = load_week("2026-W10")
assert labeled is not None
scored = score_week(labeled)
assert scored.labeled == 2
assert scored.true_positives == 1
assert scored.true_negatives == 1
assert scored.accuracy == 1.0
assert scored.confidence_rate >= 0.0
def test_freeze_is_idempotent(rolling_store: Path) -> None:
engine = _seed_engine()
first = freeze_weekly_snapshot(week_id="2026-W11", engine=engine)
second = freeze_weekly_snapshot(week_id="2026-W11", engine=engine)
assert first["created"] is True
assert second["created"] is False
def test_rolling_report_trend(rolling_store: Path) -> None:
engine = _seed_engine()
freeze_weekly_snapshot(week_id="2026-W20", engine=engine)
freeze_weekly_snapshot(week_id="2026-W21", engine=engine)
label_regions("2026-W20", [{"region": "ukraine", "label": "true_escalation"}])
label_regions(
"2026-W21",
[
{"region": "ukraine", "label": "true_escalation"},
{"region": "canada", "label": "benign"},
],
)
report = rolling_report(weeks=4)
assert report["mode"] == "rolling_operational"
assert report["alert_threshold"] == DEFAULT_BACKTEST_ALERT_THRESHOLD
assert len(report["trend"]) == 2
assert report["latest"] is not None
def test_openclaw_rolling_commands(
rolling_store: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
from analytics.integration import get_gt_engine
from services.openclaw_channel import _dispatch_command
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
reset_gt_engine()
engine = get_gt_engine()
assert engine is not None
engine.process_feed_item(
{
"text": "Troop movement and military mobilization near border",
"region": "ukraine",
"source": "test",
"source_type": "manual",
}
)
freeze = _dispatch_command("gt_rolling_freeze", {"week_id": "2026-W30", "compact": True})
assert freeze["ok"] is True
assert freeze["data"]["enabled"] is True
label = _dispatch_command(
"gt_rolling_label",
{
"week_id": "2026-W30",
"region": "ukraine",
"label": "false_alarm",
},
)
assert label["ok"] is True
assert label["data"]["updated"] == 1
trend = _dispatch_command("gt_rolling_backtest", {"weeks": 4, "compact": True})
assert trend["ok"] is True
assert trend["data"]["mode"] == "rolling_operational"
def test_route_query_rolling_intent() -> None:
from services.openclaw_routing import route_query
plan = route_query("Show GT rolling operational backtest week over week")
assert plan["recommended"]["cmd"] == "gt_rolling_backtest"
freeze_plan = route_query("Freeze weekly GT snapshot for operational validation")
assert freeze_plan["recommended"]["cmd"] == "gt_rolling_freeze"
@@ -0,0 +1,60 @@
"""OpenClaw routing and commands for Strategic Risk Analytics."""
from __future__ import annotations
import pytest
from analytics.integration import reset_gt_engine
from services.openclaw_routing import route_query
def test_route_query_gt_analyze_intent() -> None:
plan = route_query("Run GT analysis on UK and Europe feeds")
assert plan["intent"] == "gt_analyze"
assert plan["recommended"]["cmd"] == "gt_analyze"
def test_route_query_gt_dossier_intent() -> None:
plan = route_query("GT rationale dossier for ukraine strategic risk")
assert plan["recommended"]["cmd"] in {"gt_dossier", "gt_analyze"}
def test_gt_analyze_command_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
from services.openclaw_channel import _dispatch_command
monkeypatch.delenv("GT_ANALYTICS_ENABLED", raising=False)
reset_gt_engine()
result = _dispatch_command("gt_analyze", {})
assert result["ok"] is False
def test_route_query_gt_backtest_intent() -> None:
plan = route_query("Run GT historical backtest with Wilson confidence")
assert plan["intent"] == "gt_backtest"
assert plan["recommended"]["cmd"] == "gt_backtest"
assert plan["recommended"]["args"]["expanded"] is True
def test_gt_backtest_command_enabled(monkeypatch: pytest.MonkeyPatch) -> None:
from services.openclaw_channel import _dispatch_command
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
reset_gt_engine()
result = _dispatch_command("gt_backtest", {"expanded": True, "compact": True})
assert result["ok"] is True
data = result["data"]
assert data["enabled"] is True
assert data["accuracy"] == 1.0
assert data["confidence_rate"] >= 0.95
assert data["meets_target"] is True
assert "cases" not in data
def test_gt_backtest_command_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
from services.openclaw_channel import _dispatch_command
monkeypatch.delenv("GT_ANALYTICS_ENABLED", raising=False)
reset_gt_engine()
result = _dispatch_command("gt_backtest", {})
assert result["ok"] is True
assert result["data"]["enabled"] is False
+28
View File
@@ -0,0 +1,28 @@
"""Runtime profile detection for lean fleet nodes."""
from services import runtime_profile
def test_resolve_profile_name_env_override(monkeypatch):
monkeypatch.setenv("GT_ANALYTICS_PROFILE", "standard")
monkeypatch.setattr(runtime_profile, "detect_cpu_limit", lambda: 1.0)
assert runtime_profile.resolve_profile_name() == "standard"
def test_resolve_profile_name_auto_lean_on_one_cpu(monkeypatch):
monkeypatch.delenv("GT_ANALYTICS_PROFILE", raising=False)
monkeypatch.setattr(runtime_profile, "detect_cpu_limit", lambda: 1.0)
assert runtime_profile.resolve_profile_name() == "lean"
def test_runtime_profile_payload(monkeypatch):
monkeypatch.delenv("GT_ANALYTICS_PROFILE", raising=False)
monkeypatch.setattr(runtime_profile, "detect_cpu_limit", lambda: 1.0)
monkeypatch.setattr(runtime_profile, "detect_memory_limit_mb", lambda: 4096)
runtime_profile.clear_runtime_profile_cache()
payload = runtime_profile.get_runtime_profile()
assert payload["profile"] == "lean"
assert payload["cpu_limit"] == 1.0
assert payload["gt_analytics"]["recommended"] is False
assert payload["gt_analytics"]["lean_node"] is True
assert "1 vCPU" in (payload["gt_analytics"]["warning"] or "")
+56
View File
@@ -0,0 +1,56 @@
"""Telegram OSINT auto-translation."""
from services import telegram_translate
def test_guess_source_lang_detects_cyrillic():
assert telegram_translate.guess_source_lang("В Крым поедем несмотря ни на что") == "ru"
def test_apply_post_translation_skips_english(monkeypatch):
monkeypatch.setattr(telegram_translate, "telegram_translate_enabled", lambda: True)
post = {
"title": "Missile strike reported near Kyiv overnight.",
"description": "Missile strike reported near Kyiv overnight.",
}
enriched = telegram_translate.apply_post_translation(post, "en")
assert enriched["source_lang"] == "en"
assert "title_translated" not in enriched
def test_apply_post_translation_adds_fields(monkeypatch):
monkeypatch.setattr(telegram_translate, "telegram_translate_enabled", lambda: True)
monkeypatch.setattr(
telegram_translate,
"translate_text",
lambda text, target_lang=None: (
"We will go to Crimea no matter what. This is our homeland!",
"ru",
),
)
post = {
"title": "«В Крым поедем несмотря ни на что. Это наша родина!»",
"description": "«В Крым поедем несмотря ни на что. Это наша родина!»",
}
enriched = telegram_translate.apply_post_translation(post, "en")
assert enriched["source_lang"] == "ru"
assert enriched["translate_to"] == "en"
assert "Crimea" in enriched["title_translated"]
def test_normalize_translate_target_maps_ui_locales():
assert telegram_translate.normalize_translate_target("zh-CN") == "zh-CN"
assert telegram_translate.normalize_translate_target("fr") == "fr"
def test_source_lang_label_avoids_uk_country_confusion():
assert telegram_translate.source_lang_label("uk") == "Ukrainian"
assert telegram_translate.source_lang_label("ru") == "Russian"
def test_polish_translation_expands_bpla_shorthand():
assert "UAV" in telegram_translate.polish_translation("Kyiv 1x BpLa on Rembazu.")
def test_guess_source_lang_prefers_ukrainian_markers():
assert telegram_translate.guess_source_lang("Київ 1х БпЛА") == "uk"
+100
View File
@@ -0,0 +1,100 @@
"""Telegram OSINT watchdog and search helpers."""
from services import openclaw_watchdog
from services.telegram_osint_text import keyword_matches_telegram_post, telegram_post_search_text
def _telegram_slow_fixture() -> dict:
return {
"telegram_osint": {
"posts": [
{
"id": "tg-uk-1",
"title": "Київ 1х БпЛА на Рембазу.",
"description": "Київ 1х БпЛА на Рембазу.",
"title_translated": "Kyiv 1x UAV on Rembazu.",
"description_translated": "Kyiv 1x UAV on Rembazu.",
"channel": "war_monitor",
"source": "t.me/war_monitor",
"link": "https://t.me/war_monitor/101",
"risk_score": 3,
"source_lang": "uk",
},
{
"id": "tg-ru-1",
"title": "«В Крым поедем несмотря ни на что. Это наша родина!»",
"description": "«В Крым поедем несмотря ни на что. Это наша родина!»",
"title_translated": "We will go to Crimea no matter what. This is our homeland!",
"description_translated": "We will go to Crimea no matter what. This is our homeland!",
"channel": "nexta_live",
"source": "t.me/nexta_live",
"link": "https://t.me/nexta_live/202",
"risk_score": 9,
"source_lang": "ru",
},
],
"total": 2,
}
}
def test_telegram_post_search_text_includes_translated_fields():
post = _telegram_slow_fixture()["telegram_osint"]["posts"][0]
haystack = telegram_post_search_text(post)
assert "kyiv 1x uav on rembazu" in haystack
assert "бпла" in haystack
def test_keyword_matches_telegram_post_searches_translated_and_original():
post = _telegram_slow_fixture()["telegram_osint"]["posts"][1]
assert keyword_matches_telegram_post(post, "crimea")
assert keyword_matches_telegram_post(post, "крым")
def test_watchdog_keyword_matches_telegram_translation(monkeypatch):
monkeypatch.setattr(openclaw_watchdog, "_ensure_running", lambda: None)
openclaw_watchdog.clear_watches()
try:
watch = openclaw_watchdog.add_watch("keyword", {"keyword": "crimea"})
alert = openclaw_watchdog._check_keyword(watch["id"], {"keyword": "crimea"}, {}, _telegram_slow_fixture())
assert alert is not None
assert any(match["source"] == "telegram_osint" for match in alert["data"]["matches"])
assert alert["data"]["matches"][0]["title"].startswith("We will go to Crimea")
# Same Telegram post should not re-alert once seen.
assert openclaw_watchdog._check_keyword(watch["id"], {"keyword": "crimea"}, {}, _telegram_slow_fixture()) is None
finally:
openclaw_watchdog.clear_watches()
def test_watchdog_telegram_rhetoric_alerts_on_high_risk_posts(monkeypatch):
monkeypatch.setattr(openclaw_watchdog, "_ensure_running", lambda: None)
openclaw_watchdog.clear_watches()
try:
watch = openclaw_watchdog.add_watch("telegram_rhetoric", {"min_risk_score": 8})
alert = openclaw_watchdog._check_telegram_rhetoric(watch["id"], {"min_risk_score": 8}, _telegram_slow_fixture())
assert alert is not None
assert "Telegram rhetoric alert" in alert["alert"]
assert len(alert["data"]["matches"]) == 1
assert alert["data"]["matches"][0]["channel"] == "nexta_live"
assert alert["data"]["matches"][0]["risk_score"] == 9
assert openclaw_watchdog._check_telegram_rhetoric(watch["id"], {"min_risk_score": 8}, _telegram_slow_fixture()) is None
finally:
openclaw_watchdog.clear_watches()
def test_watchdog_telegram_rhetoric_supports_channel_filter(monkeypatch):
monkeypatch.setattr(openclaw_watchdog, "_ensure_running", lambda: None)
openclaw_watchdog.clear_watches()
try:
watch = openclaw_watchdog.add_watch(
"telegram_rhetoric",
{"min_risk_score": 7, "channels": ["war_monitor"]},
)
alert = openclaw_watchdog._check_telegram_rhetoric(
watch["id"],
{"min_risk_score": 7, "channels": ["war_monitor"]},
_telegram_slow_fixture(),
)
assert alert is None # war_monitor post is only risk 3
finally:
openclaw_watchdog.clear_watches()
+2
View File
@@ -15,6 +15,8 @@ services:
MESH_DM_PENDING_PER_SENDER_LIMIT: "8"
MESH_DM_PERSIST_SPOOL: "true"
WORMHOLE_STARTUP_DEADLINE_S: "90"
GT_ANALYTICS_ENABLED: "false"
GT_ANALYTICS_PROFILE: "lean"
deploy:
resources:
limits:
+7
View File
@@ -93,6 +93,13 @@ services:
- TELEGRAM_OSINT_ENABLED=${TELEGRAM_OSINT_ENABLED:-true}
- TELEGRAM_OSINT_CHANNELS=${TELEGRAM_OSINT_CHANNELS:-}
- TELEGRAM_OSINT_INTERVAL_MINUTES=${TELEGRAM_OSINT_INTERVAL_MINUTES:-60}
- TELEGRAM_OSINT_TRANSLATE=${TELEGRAM_OSINT_TRANSLATE:-true}
- TELEGRAM_OSINT_TRANSLATE_TO=${TELEGRAM_OSINT_TRANSLATE_TO:-en}
- GT_ANALYTICS_ENABLED=${GT_ANALYTICS_ENABLED:-false}
- GT_ANALYTICS_HIGH_RISK_THRESHOLD=${GT_ANALYTICS_HIGH_RISK_THRESHOLD:-0.6}
- GT_ANALYTICS_BASE_PRIOR=${GT_ANALYTICS_BASE_PRIOR:-0.15}
- GT_ANALYTICS_WATCHED_CHANNELS=${GT_ANALYTICS_WATCHED_CHANNELS:-}
- OPENCLAW_HMAC_SECRET=${OPENCLAW_HMAC_SECRET:-}
volumes:
- backend_data:/app/data
restart: unless-stopped
+14
View File
@@ -37,6 +37,7 @@ import { useDataPolling, LAYER_TOGGLE_EVENT } from '@/hooks/useDataPolling';
import { useBackendStatus, useDataKey, useDataKeys } from '@/hooks/useDataStore';
import { useReverseGeocode } from '@/hooks/useReverseGeocode';
import { useRegionDossier } from '@/hooks/useRegionDossier';
import { useGtDossier } from '@/hooks/useGtDossier';
import { useAgentActions } from '@/hooks/useAgentActions';
import { useFeedHealth } from '@/hooks/useFeedHealth';
import { useKeyboardShortcuts } from '@/hooks/useKeyboardShortcuts';
@@ -237,6 +238,7 @@ export default function Dashboard() {
wastewater: true,
// CrowdThreat is operator opt-in only.
crowdthreat: false,
gt_risk: false,
// Shodan
shodan_overlay: false,
// AI Intel
@@ -244,6 +246,16 @@ export default function Dashboard() {
// SAR (Synthetic Aperture Radar)
sar: true,
});
const regionLat =
selectedEntity?.type === 'region_dossier' ? selectedEntity.extra?.lat : undefined;
const regionLng =
selectedEntity?.type === 'region_dossier' ? selectedEntity.extra?.lng : undefined;
const { gtDossier, gtDossierLoading } = useGtDossier(
typeof regionLat === 'number' ? regionLat : undefined,
typeof regionLng === 'number' ? regionLng : undefined,
regionDossier?.country?.name,
activeLayers.gt_risk,
);
const [shodanResults, setShodanResults] = useState<ShodanSearchMatch[]>([]);
const [, setShodanQueryLabel] = useState('');
const [shodanStyle, setShodanStyle] = useState<import('@/types/shodan').ShodanStyleConfig>({ shape: 'circle', color: '#16a34a', size: 'md' });
@@ -776,6 +788,8 @@ export default function Dashboard() {
selectedEntity={selectedEntity}
regionDossier={regionDossier}
regionDossierLoading={regionDossierLoading}
gtDossier={gtDossier}
gtDossierLoading={gtDossierLoading}
onExpandEntityGraph={() => {
if (isEntityGraphEligible(selectedEntity)) setShowEntityGraph(true);
}}
@@ -0,0 +1,77 @@
'use client';
import React from 'react';
import { GripVertical, Minus, Plus } from 'lucide-react';
import { useTranslation } from '@/i18n';
import { useFloatingPanel } from '@/hooks/useFloatingPanel';
import GtBacktestPanel from '@/components/GtBacktestPanel';
import GtTopAlertsStrip from '@/components/GtTopAlertsStrip';
import type { SelectedEntity } from '@/types/dashboard';
interface Props {
layerEnabled?: boolean;
onFlyTo?: (lat: number, lng: number) => void;
onSelectEntity?: (entity: SelectedEntity | null) => void;
}
export default function GtAnalyticsHud({
layerEnabled = false,
onFlyTo,
onSelectEntity,
}: Props) {
const { t } = useTranslation();
const { position, isMinimized, setIsMinimized, isDragging, onDragStart } = useFloatingPanel(
'sb-gt-analytics-hud-v1',
{ defaultPosition: { x: 24, y: 380 } },
);
if (!layerEnabled) return null;
return (
<div
className={`pointer-events-auto fixed z-[201] flex flex-col border border-amber-700/45 bg-black/80 shadow-[0_0_16px_rgba(245,158,11,0.12)] backdrop-blur-sm ${
isMinimized ? 'w-fit' : 'w-[min(92vw,28rem)]'
} ${isDragging ? 'cursor-grabbing select-none' : ''}`}
style={{ left: position.x, top: position.y }}
>
<div
className={`flex items-center gap-2 bg-amber-950/30 px-2 py-1.5 cursor-grab active:cursor-grabbing ${
isMinimized ? '' : 'border-b border-amber-800/35'
}`}
onMouseDown={onDragStart}
title={t('gtHud.dragHint')}
>
<GripVertical size={12} className="shrink-0 text-amber-600/80" />
<span className="whitespace-nowrap text-[10px] font-mono font-bold tracking-widest text-amber-300">
{t('gtHud.title')}
</span>
{!isMinimized && (
<span className="text-[9px] font-mono tracking-wider text-amber-600/70">
{t('gtHud.dragHint')}
</span>
)}
<button
type="button"
onMouseDown={(event) => event.stopPropagation()}
onClick={() => setIsMinimized((prev) => !prev)}
className="ml-auto p-0.5 text-amber-500 transition-colors hover:text-amber-300"
title={isMinimized ? t('gtHud.expand') : t('gtHud.collapse')}
>
{isMinimized ? <Plus size={14} /> : <Minus size={14} />}
</button>
</div>
{!isMinimized && (
<div className="flex max-h-[min(70vh,28rem)] flex-col overflow-y-auto styled-scrollbar">
<GtBacktestPanel layerEnabled={layerEnabled} embedded />
<GtTopAlertsStrip
layerEnabled={layerEnabled}
onFlyTo={onFlyTo}
onSelectEntity={onSelectEntity}
embedded
/>
</div>
)}
</div>
);
}
+453
View File
@@ -0,0 +1,453 @@
'use client';
import React, { useCallback, useEffect, useState } from 'react';
import { CheckCircle2, Minus, Plus, Radar, RefreshCw, XCircle } from 'lucide-react';
import { API_BASE } from '@/lib/api';
import { useTranslation } from '@/i18n';
import type { GtBacktestReport, GtMicroRollingReport, GtRollingReport } from '@/types/dashboard';
interface Props {
layerEnabled?: boolean;
embedded?: boolean;
}
type TabId = 'benchmark' | 'operational';
function pct(value: number | undefined): string {
if (value == null || Number.isNaN(value)) return '—';
return `${(value * 100).toFixed(1)}%`;
}
export default function GtBacktestPanel({ layerEnabled = false, embedded = false }: Props) {
const { t } = useTranslation();
const [isMinimized, setIsMinimized] = useState(false);
const [activeTab, setActiveTab] = useState<TabId>('operational');
const [benchmark, setBenchmark] = useState<GtBacktestReport | null>(null);
const [rolling, setRolling] = useState<GtRollingReport | null>(null);
const [micro, setMicro] = useState<GtMicroRollingReport | null>(null);
const [loadingBenchmark, setLoadingBenchmark] = useState(false);
const [loadingRolling, setLoadingRolling] = useState(false);
const [loadingMicro, setLoadingMicro] = useState(false);
const [showFailures, setShowFailures] = useState(false);
const refreshBenchmark = useCallback(async () => {
if (!layerEnabled) {
setBenchmark(null);
return;
}
setLoadingBenchmark(true);
try {
const res = await fetch(`${API_BASE}/api/analytics/backtest?expanded=true&tune=false`);
if (res.ok) setBenchmark(await res.json());
} catch {
/* non-fatal */
} finally {
setLoadingBenchmark(false);
}
}, [layerEnabled]);
const refreshRolling = useCallback(async () => {
if (!layerEnabled) {
setRolling(null);
return;
}
setLoadingRolling(true);
try {
const res = await fetch(`${API_BASE}/api/analytics/rolling?weeks=8`);
if (res.ok) setRolling(await res.json());
} catch {
/* non-fatal */
} finally {
setLoadingRolling(false);
}
}, [layerEnabled]);
const refreshMicro = useCallback(async () => {
if (!layerEnabled) {
setMicro(null);
return;
}
setLoadingMicro(true);
try {
const res = await fetch(`${API_BASE}/api/analytics/rolling/micro?window_days=3&limit=6`);
if (res.ok) setMicro(await res.json());
} catch {
/* non-fatal */
} finally {
setLoadingMicro(false);
}
}, [layerEnabled]);
const refresh = useCallback(async () => {
await Promise.all([refreshBenchmark(), refreshRolling(), refreshMicro()]);
}, [refreshBenchmark, refreshRolling, refreshMicro]);
useEffect(() => {
refresh();
if (!layerEnabled) return undefined;
const id = setInterval(refresh, 15 * 60_000);
return () => clearInterval(id);
}, [refresh, layerEnabled]);
const failures = (benchmark?.cases || []).filter((row) => !row.correct);
const operationalScorable = Boolean(
rolling && ((rolling.weeks_scorable ?? 0) > 0 || rolling.latest?.scorable),
);
const benchmarkPass = benchmark?.meets_target;
const rollingPass = rolling?.meets_target;
const passBadge =
activeTab === 'benchmark'
? benchmarkPass
: operationalScorable
? rollingPass
: undefined;
const showCollectingBadge =
activeTab === 'operational' && layerEnabled && rolling?.enabled && !operationalScorable;
const loading =
activeTab === 'benchmark'
? loadingBenchmark
: loadingRolling || loadingMicro;
const latest = rolling?.latest;
const microRegions = micro?.ignitions?.length
? micro.ignitions
: (micro?.top_regions || []).slice(0, 4);
const shellClass = embedded
? 'pointer-events-auto flex-shrink-0 border-b border-amber-800/30 bg-black/70'
: 'pointer-events-auto flex-shrink-0 border border-amber-700/40 bg-black/75 backdrop-blur-sm shadow-[0_0_18px_rgba(245,158,11,0.10)]';
return (
<div className={shellClass}>
<div
className="flex items-center justify-between border-b border-amber-700/30 bg-amber-950/20 px-3 py-2.5 cursor-pointer hover:bg-amber-950/40 transition-colors"
onClick={() => setIsMinimized((prev) => !prev)}
>
<div className="flex items-center gap-2">
<Radar size={16} className="text-amber-400" />
<span className="text-[12px] font-mono font-bold tracking-widest text-amber-400">
{t('gtBacktest.title').toUpperCase()}
</span>
{showCollectingBadge && (
<span className="text-[11px] font-mono px-1.5 py-0.5 tracking-wider border bg-amber-900/25 border-amber-700/40 text-amber-300">
{t('gtBacktest.collecting')}
</span>
)}
{layerEnabled && passBadge != null && (
<span
className={`text-[11px] font-mono px-1.5 py-0.5 tracking-wider border ${
passBadge
? 'bg-emerald-900/30 border-emerald-700/40 text-emerald-300'
: 'bg-red-900/30 border-red-700/40 text-red-300'
}`}
>
{passBadge ? t('gtBacktest.pass') : t('gtBacktest.fail')}
</span>
)}
</div>
<div className="flex items-center gap-2">
<button
type="button"
onClick={(e) => {
e.stopPropagation();
refresh();
}}
title={t('gtBacktest.refresh')}
className="text-amber-600 transition-colors hover:text-amber-400 p-0.5"
>
<RefreshCw size={11} className={loading ? 'animate-spin' : ''} />
</button>
{isMinimized ? (
<Plus size={16} className="text-amber-400" />
) : (
<Minus size={16} className="text-amber-400" />
)}
</div>
</div>
{!isMinimized && (
<div className="px-3 py-2 max-h-60 overflow-y-auto styled-scrollbar space-y-2">
{!layerEnabled ? (
<div className="text-[11px] font-mono tracking-wider text-amber-600/70 py-1">
{t('gtBacktest.layerOff')}
</div>
) : (
<>
<div className="flex gap-1">
{(['operational', 'benchmark'] as TabId[]).map((tab) => (
<button
key={tab}
type="button"
onClick={() => setActiveTab(tab)}
className={`text-[10px] font-mono tracking-widest px-2 py-0.5 border transition-colors ${
activeTab === tab
? 'border-amber-500/60 bg-amber-900/30 text-amber-200'
: 'border-amber-800/30 text-amber-600/80 hover:text-amber-400'
}`}
>
{tab === 'benchmark'
? t('gtBacktest.tabBenchmark')
: t('gtBacktest.tabOperational')}
</button>
))}
</div>
{activeTab === 'benchmark' ? (
!benchmark?.enabled ? (
<div className="text-[11px] font-mono tracking-wider text-amber-600/70 py-1">
{t('gtBacktest.disabled')}
</div>
) : loadingBenchmark && !benchmark.accuracy ? (
<div className="text-[11px] font-mono tracking-wider text-amber-500/80 py-1">
{t('gtBacktest.loading')}
</div>
) : (
<>
<div className="text-[10px] font-mono tracking-wider text-amber-600/60">
{t('gtBacktest.benchmarkNote')}
</div>
<div className="grid grid-cols-2 gap-2">
<div className="border border-amber-800/30 bg-amber-950/15 px-2 py-1.5">
<div className="text-[10px] font-mono tracking-widest text-amber-600/80">
{t('gtBacktest.accuracy')}
</div>
<div className="text-[13px] font-mono font-bold text-amber-200">
{pct(benchmark.accuracy)}
</div>
</div>
<div className="border border-amber-800/30 bg-amber-950/15 px-2 py-1.5">
<div className="text-[10px] font-mono tracking-widest text-amber-600/80">
{t('gtBacktest.confidence')}
</div>
<div className="text-[13px] font-mono font-bold text-amber-200">
{pct(benchmark.confidence_rate)}
</div>
</div>
</div>
<div className="text-[10px] font-mono tracking-wider text-amber-600/70 leading-relaxed">
{t('gtBacktest.cases').replace('{count}', String(benchmark.total_cases))} ·{' '}
{t('gtBacktest.threshold').replace('{value}', benchmark.alert_threshold.toFixed(2))} ·{' '}
{t('gtBacktest.target').replace('{value}', pct(benchmark.target_confidence))}
</div>
<div className="flex flex-wrap gap-2 text-[10px] font-mono tracking-wider">
<span className="text-emerald-400">TP {benchmark.true_positives}</span>
<span className="text-emerald-400">TN {benchmark.true_negatives}</span>
<span className="text-red-400">FP {benchmark.false_positives}</span>
<span className="text-red-400">FN {benchmark.false_negatives}</span>
</div>
<div className="flex items-center gap-1.5 text-[10px] font-mono tracking-wider text-amber-500/90">
{benchmark.meets_target ? (
<CheckCircle2 size={12} className="text-emerald-400 shrink-0" />
) : (
<XCircle size={12} className="text-red-400 shrink-0" />
)}
<span>
{benchmark.meets_target
? t('gtBacktest.meetsTarget')
: t('gtBacktest.belowTarget')}
</span>
</div>
{failures.length > 0 && (
<div>
<button
type="button"
onClick={() => setShowFailures((prev) => !prev)}
className="text-[10px] font-mono tracking-widest text-red-400 hover:text-red-300"
>
{showFailures ? '' : '+'} {t('gtBacktest.misclassified').replace('{count}', String(failures.length))}
</button>
{showFailures && (
<div className="mt-1 space-y-1">
{failures.map((row) => (
<div
key={row.case_id}
className="border border-red-800/30 bg-red-950/15 px-2 py-1 text-[10px] font-mono text-red-200/90"
>
{row.name} ({row.kind})
</div>
))}
</div>
)}
</div>
)}
</>
)
) : !rolling?.enabled && !micro?.enabled ? (
<div className="text-[11px] font-mono tracking-wider text-amber-600/70 py-1">
{t('gtBacktest.disabled')}
</div>
) : (loadingRolling || loadingMicro) && !rolling?.latest && !micro?.regions_tracked ? (
<div className="text-[11px] font-mono tracking-wider text-amber-500/80 py-1">
{t('gtBacktest.operationalLoading')}
</div>
) : (
<>
<div className="border border-amber-800/25 bg-amber-950/10 px-2 py-1.5 space-y-1">
<div className="text-[10px] font-mono tracking-widest text-amber-500/90">
{t('gtBacktest.microTitle').toUpperCase()}
</div>
{micro?.enabled ? (
<>
<div className="text-[10px] font-mono tracking-wider text-amber-600/75">
{t('gtBacktest.microWindow')
.replace('{days}', String(micro.window_days))
.replace('{delta}', micro.ignition_delta.toFixed(2))}
</div>
<div className="flex flex-wrap gap-2 text-[10px] font-mono tracking-wider">
<span className="text-orange-300">
{t('gtBacktest.microIgnitions').replace(
'{count}',
String(micro.ignition_count)
)}
</span>
<span className="text-amber-300/90">
{t('gtBacktest.microAlerted3d').replace(
'{count}',
String(micro.alerted_3d_count)
)}
</span>
</div>
{microRegions.length > 0 ? (
<div className="space-y-0.5">
{microRegions.map((row) => (
<div
key={row.region}
className="text-[10px] font-mono text-amber-200/85 flex items-center gap-1.5"
>
{row.ignition && (
<span className="text-orange-400 border border-orange-700/40 px-1 text-[9px]">
{t('gtBacktest.microIgnitionBadge')}
</span>
)}
<span>
{t('gtBacktest.microRegionLine')
.replace('{region}', row.region)
.replace('{spot}', pct(row.spot_risk))
.replace('{avg}', pct(row.risk_3d_avg))
.replace('{delta}', pct(row.risk_delta))}
</span>
</div>
))}
</div>
) : (
<div className="text-[10px] font-mono tracking-wider text-amber-600/65">
{t('gtBacktest.microEmpty')}
</div>
)}
</>
) : (
<div className="text-[10px] font-mono tracking-wider text-amber-600/65">
{t('gtBacktest.microEmpty')}
</div>
)}
</div>
<div className="text-[10px] font-mono tracking-widest text-amber-600/80 pt-1">
{t('gtBacktest.tabOperational').toUpperCase()} {t('gtBacktest.operationalTrend')}
</div>
{!rolling || rolling.weeks_stored === 0 ? (
<div className="text-[10px] font-mono tracking-wider text-amber-600/70 py-1">
{t('gtBacktest.operationalEmpty')}
</div>
) : (
<>
<div className="grid grid-cols-2 gap-2">
<div className="border border-amber-800/30 bg-amber-950/15 px-2 py-1.5">
<div className="text-[10px] font-mono tracking-widest text-amber-600/80">
{t('gtBacktest.accuracy')}
</div>
<div className="text-[13px] font-mono font-bold text-amber-200">
{latest?.scorable ? pct(latest.accuracy) : '—'}
</div>
</div>
<div className="border border-amber-800/30 bg-amber-950/15 px-2 py-1.5">
<div className="text-[10px] font-mono tracking-widest text-amber-600/80">
{t('gtBacktest.confidence')}
</div>
<div className="text-[13px] font-mono font-bold text-amber-200">
{latest?.scorable ? pct(latest.confidence_rate) : '—'}
</div>
</div>
</div>
<div className="text-[10px] font-mono tracking-wider text-amber-600/70 leading-relaxed">
{t('gtBacktest.operationalWeeks')
.replace('{stored}', String(rolling.weeks_stored))
.replace('{scorable}', String(rolling.weeks_scorable))}
{latest
? ` · ${t('gtBacktest.operationalLabeled')
.replace('{labeled}', String(latest.labeled))
.replace('{pending}', String(latest.pending))}`
: ''}
</div>
{latest && !latest.scorable && (
<div className="text-[10px] font-mono tracking-wider text-amber-500/80">
{t('gtBacktest.operationalMinLabels').replace(
'{count}',
String(rolling.min_labeled_per_week)
)}
</div>
)}
{latest?.scorable && (
<div className="flex flex-wrap gap-2 text-[10px] font-mono tracking-wider">
<span className="text-emerald-400">TP {latest.true_positives}</span>
<span className="text-emerald-400">TN {latest.true_negatives}</span>
<span className="text-red-400">FP {latest.false_positives}</span>
<span className="text-red-400">FN {latest.false_negatives}</span>
</div>
)}
{(rolling.accuracy_series?.length ?? 0) > 0 && (
<div>
<div className="text-[10px] font-mono tracking-widest text-amber-600/80 mb-1">
{t('gtBacktest.operationalTrend')}
</div>
<div className="flex flex-wrap gap-1.5">
{rolling.accuracy_series.map((point) => (
<span
key={point.week_id}
className="text-[10px] font-mono border border-amber-800/30 bg-amber-950/20 px-1.5 py-0.5 text-amber-200/90"
title={`${point.labeled} labeled`}
>
{point.week_id.replace('-W', 'w')}: {pct(point.accuracy)}
</span>
))}
</div>
</div>
)}
{latest?.scorable && (
<div className="flex items-center gap-1.5 text-[10px] font-mono tracking-wider text-amber-500/90">
{rolling.meets_target ? (
<CheckCircle2 size={12} className="text-emerald-400 shrink-0" />
) : (
<XCircle size={12} className="text-red-400 shrink-0" />
)}
<span>
{rolling.improving_vs_prior
? t('gtBacktest.operationalImproving')
: t('gtBacktest.operationalFlat')}
{' · '}
{rolling.meets_target
? t('gtBacktest.meetsTarget')
: t('gtBacktest.belowTarget')}
</span>
</div>
)}
</>
)}
</>
)}
</>
)}
</div>
)}
</div>
);
}
@@ -0,0 +1,121 @@
'use client';
import React, { useMemo } from 'react';
import { ChevronRight, Radar } from 'lucide-react';
import { useTranslation } from '@/i18n';
import { useDataKey } from '@/hooks/useDataStore';
import { extractGtAlerts } from '@/lib/gtAlerts';
import type { SelectedEntity } from '@/types/dashboard';
interface Props {
layerEnabled?: boolean;
onFlyTo?: (lat: number, lng: number) => void;
onSelectEntity?: (entity: SelectedEntity | null) => void;
embedded?: boolean;
}
function pct(value: number): string {
return `${(value * 100).toFixed(0)}%`;
}
export default function GtTopAlertsStrip({
layerEnabled = false,
onFlyTo,
onSelectEntity,
embedded = false,
}: Props) {
const { t } = useTranslation();
const gtRisk = useDataKey('gt_risk');
const { alerts, trackedRegions, plottedRegions, maxRegions } = useMemo(
() => extractGtAlerts(gtRisk, 8),
[gtRisk],
);
if (!layerEnabled || !gtRisk?.enabled) return null;
const handleSelect = (alert: (typeof alerts)[number]) => {
onFlyTo?.(alert.lat, alert.lng);
onSelectEntity?.({
id: alert.region,
type: 'gt_risk',
name: alert.regionLabel,
extra: {
region: alert.region,
risk: alert.risk,
financial: alert.financial,
unrest: alert.unrest,
conflict: alert.conflict,
contagion: alert.contagion,
lat: alert.lat,
lng: alert.lng,
risk_spot: alert.risk,
risk_3d_avg: alert.risk3d,
risk_delta: alert.riskDelta,
micro_ignition: alert.ignition,
},
});
};
const shellClass = embedded
? 'pointer-events-auto border-t border-amber-800/30 bg-black/70'
: 'pointer-events-auto max-w-[min(92vw,52rem)] border border-amber-700/45 bg-black/80 backdrop-blur-sm shadow-[0_0_16px_rgba(245,158,11,0.12)]';
return (
<div className={shellClass}>
<div className="flex items-center gap-2 border-b border-amber-800/35 bg-amber-950/25 px-2.5 py-1.5">
<Radar size={12} className="text-amber-400 shrink-0" />
<span className="text-[10px] font-mono font-bold tracking-widest text-amber-300">
{t('gtAlerts.title')}
</span>
<span className="text-[9px] font-mono tracking-wider text-amber-600/80">
{t('gtAlerts.counts')
.replace('{plotted}', String(plottedRegions))
.replace('{tracked}', String(trackedRegions))
.replace('{max}', String(maxRegions))}
</span>
</div>
{alerts.length === 0 ? (
<div className="px-2.5 py-2 text-[10px] font-mono tracking-wider text-amber-600/70">
{t('gtAlerts.empty')}
</div>
) : (
<div className="flex items-stretch gap-1 overflow-x-auto styled-scrollbar px-2 py-1.5">
{alerts.map((alert) => (
<button
key={alert.region}
type="button"
onClick={() => handleSelect(alert)}
className="group flex min-w-[9.5rem] shrink-0 flex-col gap-0.5 border border-amber-800/35 bg-amber-950/20 px-2 py-1 text-left transition-colors hover:border-amber-600/50 hover:bg-amber-900/25"
>
<div className="flex items-center gap-1">
<span className="truncate text-[10px] font-mono font-bold uppercase text-amber-100">
{alert.regionLabel}
</span>
{alert.ignition && (
<span className="shrink-0 border border-orange-700/50 px-1 text-[8px] font-mono text-orange-300">
{t('gtAlerts.ignition')}
</span>
)}
<ChevronRight
size={10}
className="ml-auto shrink-0 text-amber-600/60 group-hover:text-amber-400"
/>
</div>
<div className="text-[9px] font-mono tracking-wider text-amber-500/90">
{t('gtAlerts.line')
.replace('{risk}', pct(alert.risk))
.replace('{conflict}', pct(alert.conflict))}
</div>
</button>
))}
</div>
)}
<div className="border-t border-amber-900/30 px-2.5 py-1 text-[9px] font-mono leading-relaxed text-amber-600/65">
{t('gtAlerts.hint')}
</div>
</div>
);
}
+81 -1
View File
@@ -185,6 +185,7 @@ import { CorrelationPopup } from '@/components/MaplibreViewer/popups/Correlation
import { WastewaterPopup } from '@/components/MaplibreViewer/popups/WastewaterPopup';
import { MilitaryBasePopup } from '@/components/MaplibreViewer/popups/MilitaryBasePopup';
import { RegionDossierPanel } from '@/components/MaplibreViewer/popups/RegionDossierPanel';
import { GtRiskPopup } from '@/components/MaplibreViewer/popups/GtRiskPopup';
import { TelegramOsintPopup } from '@/components/MaplibreViewer/popups/TelegramOsintPopup';
import {
buildSentinelTileUrl,
@@ -196,6 +197,7 @@ import {
buildEarthquakesGeoJSON,
buildJammingGeoJSON,
buildCorrelationsGeoJSON,
buildGtRiskGeoJSON,
buildTinygsGeoJSON,
buildShodanGeoJSON,
buildAIIntelGeoJSON,
@@ -306,6 +308,7 @@ const MAP_EXTRA_DATA_KEYS = [
'crowdthreat',
'malware_threats',
'telegram_osint',
'gt_risk',
'datacenters',
'firms_fires',
'fishing_activity',
@@ -778,6 +781,11 @@ const MaplibreViewer = ({
[activeLayers.correlations, activeLayers.contradictions, data?.correlations],
);
const gtRiskGeoJSON = useMemo(
() => (activeLayers.gt_risk ? buildGtRiskGeoJSON(data?.gt_risk) : null),
[activeLayers.gt_risk, data?.gt_risk],
);
const tinygsGeoJSON = useMemo(
() => {
void interpTick;
@@ -1724,6 +1732,7 @@ const MaplibreViewer = ({
correlationsGeoJSON && 'corr-infra-fill',
correlationsGeoJSON && 'corr-contra-fill',
correlationsGeoJSON && 'corr-analysis-fill',
gtRiskGeoJSON && 'gt-risk-heatmap',
].filter(Boolean) as string[];
useEffect(() => {
@@ -1820,7 +1829,7 @@ const MaplibreViewer = ({
return (
<div
className={`relative h-full w-full z-0 isolate ${selectedEntity && ['region_dossier', 'gdelt', 'liveuamap', 'news', 'telegram_osint'].includes(selectedEntity.type) ? 'map-focus-active' : ''}`}
className={`relative h-full w-full z-0 isolate ${selectedEntity && ['region_dossier', 'gdelt', 'liveuamap', 'news', 'telegram_osint', 'gt_risk'].includes(selectedEntity.type) ? 'map-focus-active' : ''}`}
style={pinPlacementMode || sarAoiDropMode ? { cursor: 'crosshair' } : undefined}
>
<Map
@@ -2226,6 +2235,55 @@ const MaplibreViewer = ({
/>
</Source>
{/* Strategic Risk Heatmap — Bayesian posterior scores */}
<Source id="gt-risk-source" type="geojson" data={(gtRiskGeoJSON ?? EMPTY_FC)}>
<Layer
id="gt-risk-heatmap"
type="circle"
minzoom={2}
paint={{
'circle-radius': [
'interpolate',
['linear'],
['zoom'],
2,
['+', 6, ['*', 14, ['get', 'risk']]],
6,
['+', 10, ['*', 28, ['get', 'risk']]],
10,
['+', 14, ['*', 40, ['get', 'risk']]],
],
'circle-color': [
'interpolate',
['linear'],
['get', 'risk'],
0.15,
'#22c55e',
0.35,
'#84cc16',
0.5,
'#eab308',
0.65,
'#f97316',
0.8,
'#ef4444',
],
'circle-opacity': [
'interpolate',
['linear'],
['get', 'risk'],
0.15,
0.22,
0.8,
0.72,
],
'circle-stroke-width': 1,
'circle-stroke-color': '#fbbf24',
'circle-stroke-opacity': 0.35,
}}
/>
</Source>
{/* Correlation Alerts — Emergent Intelligence grid squares */}
<Source id="correlations" type="geojson" data={(correlationsGeoJSON ?? EMPTY_FC)}>
{/* RF Anomaly — grey */}
@@ -5712,6 +5770,28 @@ const MaplibreViewer = ({
return <FishingDestinationRoute vesselLat={event.lat} vesselLng={event.lng} destination={dest} />;
})()}
{(() => {
if (selectedEntity?.type !== 'gt_risk' || !selectedEntity.extra) return null;
const props = selectedEntity.extra as Record<string, unknown>;
const lat = Number(props.lat);
const lng = Number(props.lng);
if (!Number.isFinite(lat) || !Number.isFinite(lng)) return null;
return (
<GtRiskPopup
region={String(props.region || props.name || selectedEntity.id)}
risk={Number(props.risk ?? 0)}
financial={Number(props.financial ?? 0)}
unrest={Number(props.unrest ?? 0)}
conflict={Number(props.conflict ?? 0)}
contagion={Number(props.contagion ?? 0)}
interpretation={String(props.interpretation || '')}
lat={lat}
lng={lng}
onClose={() => onEntityClick?.(null)}
/>
);
})()}
{(() => {
if (selectedEntity?.type !== 'telegram_osint' || !data?.telegram_osint?.posts) return null;
const allPosts = data.telegram_osint.posts;
@@ -0,0 +1,188 @@
'use client';
import React, { useEffect, useState } from 'react';
import { Popup } from 'react-map-gl/maplibre';
import { Radar } from 'lucide-react';
import { useTranslation } from '@/i18n';
import { API_BASE } from '@/lib/api';
import { formatGtRegionLabel } from '@/lib/gtAlerts';
import type { GtDossier } from '@/types/dashboard';
export interface GtRiskPopupProps {
region: string;
risk: number;
financial?: number;
unrest?: number;
conflict?: number;
contagion?: number;
interpretation?: string;
lat: number;
lng: number;
onClose: () => void;
}
function riskColor(score: number): string {
if (score >= 0.6) return '#ef4444';
if (score >= 0.4) return '#f97316';
if (score >= 0.25) return '#eab308';
return '#22c55e';
}
function formatSignalName(name: string): string {
return name.replace(/_/g, ' ');
}
async function fetchDossier(region: string, lat: number, lng: number): Promise<GtDossier | null> {
const candidates = [
region.trim().toLowerCase(),
`${lat.toFixed(2)},${lng.toFixed(2)}`,
].filter((value, index, list) => value && list.indexOf(value) === index);
let best: GtDossier | null = null;
for (const key of candidates) {
try {
const response = await fetch(`${API_BASE}/api/analytics/dossier/${encodeURIComponent(key)}`);
if (!response.ok) continue;
const payload = (await response.json()) as GtDossier;
if (!payload.enabled) continue;
if (!best || (payload.current_risk ?? 0) >= (best.current_risk ?? 0)) {
best = payload;
}
} catch {
/* optional analytics */
}
}
return best;
}
export function GtRiskPopup({
region,
risk,
financial,
unrest,
conflict,
contagion,
interpretation,
lat,
lng,
onClose,
}: GtRiskPopupProps) {
const { t } = useTranslation();
const color = riskColor(risk);
const [dossier, setDossier] = useState<GtDossier | null>(null);
const [loadingSignals, setLoadingSignals] = useState(true);
useEffect(() => {
let cancelled = false;
setLoadingSignals(true);
void fetchDossier(region, lat, lng).then((result) => {
if (!cancelled) {
setDossier(result);
setLoadingSignals(false);
}
});
return () => {
cancelled = true;
};
}, [region, lat, lng]);
const resolvedInterpretation = interpretation || dossier?.interpretation || '';
const signals = dossier?.recent_signals || [];
return (
<Popup
longitude={lng}
latitude={lat}
closeButton={false}
closeOnClick={false}
onClose={onClose}
className="threat-popup"
maxWidth="360px"
>
<div className="bg-black/95 border border-amber-700/50 rounded-lg overflow-hidden font-mono text-[11px]">
<div className="px-3 py-2 border-b border-amber-800/40 bg-amber-950/40 flex items-center gap-2">
<Radar size={14} className="text-amber-400" />
<span className="text-amber-300 font-bold tracking-widest text-[10px]">
{t('gtRisk.popupTitle')}
</span>
<button
type="button"
onClick={onClose}
className="ml-auto text-[var(--text-muted)] hover:text-white"
>
</button>
</div>
<div className="p-3 flex flex-col gap-2 max-h-72 overflow-y-auto styled-scrollbar">
<div className="flex justify-between items-center">
<span className="text-[var(--text-muted)]">{t('gtRisk.region')}</span>
<span className="text-white font-bold uppercase">{formatGtRegionLabel(region)}</span>
</div>
<div className="flex justify-between items-center">
<span className="text-[var(--text-muted)]">{t('gtRisk.composite')}</span>
<span className="font-bold" style={{ color }}>
{(risk * 100).toFixed(1)}%
</span>
</div>
<div className="grid grid-cols-3 gap-2 text-[10px]">
<div>
<div className="text-[var(--text-muted)]">{t('gtRisk.financial')}</div>
<div className="text-cyan-300">{((financial ?? 0) * 100).toFixed(0)}%</div>
</div>
<div>
<div className="text-[var(--text-muted)]">{t('gtRisk.unrest')}</div>
<div className="text-orange-300">{((unrest ?? 0) * 100).toFixed(0)}%</div>
</div>
<div>
<div className="text-[var(--text-muted)]">{t('gtRisk.conflict')}</div>
<div className="text-red-300">{((conflict ?? 0) * 100).toFixed(0)}%</div>
</div>
</div>
{contagion != null && contagion > 0 && (
<div className="flex justify-between">
<span className="text-[var(--text-muted)]">{t('gtRisk.contagion')}</span>
<span className="text-purple-300">{(contagion * 100).toFixed(1)}%</span>
</div>
)}
{resolvedInterpretation && (
<p className="text-[var(--text-secondary)] leading-relaxed border-t border-amber-900/40 pt-2">
<span className="text-amber-400 font-bold">&gt;_ </span>
{resolvedInterpretation}
</p>
)}
<div className="border-t border-amber-900/40 pt-2">
<div className="text-[10px] tracking-widest text-amber-500/90 font-bold mb-1.5">
{t('gtRisk.costlySignals')}
</div>
{loadingSignals ? (
<div className="text-[10px] text-amber-600/80">{t('gtRisk.loadingSignals')}</div>
) : signals.length > 0 ? (
<div className="space-y-1.5">
{signals.slice(-4).reverse().map((entry, idx) => (
<div
key={`${entry.timestamp}-${idx}`}
className="border-l-2 border-amber-700/60 pl-2 text-[10px] text-[var(--text-secondary)]"
>
<div className="text-amber-300 uppercase">
{Object.keys(entry.signals || {})
.map(formatSignalName)
.join(', ') || entry.domain}
</div>
<div className="text-[var(--text-muted)] truncate" title={entry.source}>
{entry.source || t('gtRisk.unknownSource')}
</div>
</div>
))}
</div>
) : (
<div className="text-[10px] text-amber-600/75 leading-relaxed">
{t('gtRisk.noSignals')}
</div>
)}
</div>
</div>
</div>
</Popup>
);
}
@@ -1,6 +1,6 @@
'use client';
import React, { useMemo } from 'react';
import React, { useEffect, useMemo, useState } from 'react';
import { Popup } from 'react-map-gl/maplibre';
import { Radio } from 'lucide-react';
import { useTranslation } from '@/i18n';
@@ -69,11 +69,58 @@ function riskTheme(rs: number) {
};
}
function postHeadline(post: TelegramOsintPost): string {
return String(post.title || post.description || 'Telegram intercept').trim();
const CYRILLIC_RE = /[\u0400-\u04FF]/;
function containsCyrillic(text: string): boolean {
return CYRILLIC_RE.test(text);
}
function postDetail(post: TelegramOsintPost): string | null {
function sourceLangLabel(post: TelegramOsintPost): string {
if (post.source_lang_label) return post.source_lang_label;
const code = String(post.source_lang || '').trim().toLowerCase();
const labels: Record<string, string> = {
uk: 'Ukrainian',
ru: 'Russian',
en: 'English',
ar: 'Arabic',
he: 'Hebrew',
'zh-cn': 'Chinese',
fr: 'French',
de: 'German',
pl: 'Polish',
};
return labels[code] || code.toUpperCase();
}
function hasTranslation(post: TelegramOsintPost): boolean {
const translated = String(post.title_translated || post.description_translated || '').trim();
const original = String(post.title || post.description || '').trim();
return Boolean(translated && translated !== original);
}
function postHeadline(post: TelegramOsintPost, showOriginal: boolean): string {
const original = String(post.title || post.description || 'Telegram intercept').trim();
const translated = String(post.title_translated || post.description_translated || '').trim();
if (!showOriginal && translated) {
return translated.split('\n', 1)[0].trim();
}
if (!showOriginal && containsCyrillic(original) && translated) {
return translated.split('\n', 1)[0].trim();
}
return original;
}
function postDetail(post: TelegramOsintPost, showOriginal: boolean): string | null {
if (!showOriginal && post.description_translated) {
const translatedTitle = String(post.title_translated || '').trim();
const translatedBody = String(post.description_translated || '').trim();
if (!translatedBody || translatedBody === translatedTitle) return null;
const extra = translatedBody.startsWith(translatedTitle)
? translatedBody.slice(translatedTitle.length).trim()
: translatedBody;
return extra || null;
}
const title = String(post.title || '').trim();
const description = String(post.description || '').trim();
if (!description || description === title || description.startsWith(title)) return null;
@@ -126,10 +173,12 @@ function TelegramPostMedia({ post }: { post: TelegramOsintPost }) {
function TelegramPostCard({ post }: { post: TelegramOsintPost }) {
const { t } = useTranslation();
const [showOriginal, setShowOriginal] = useState(false);
const rs = post.risk_score ?? 1;
const theme = riskTheme(rs);
const headline = postHeadline(post);
const detail = postDetail(post);
const translated = hasTranslation(post);
const headline = postHeadline(post, showOriginal);
const detail = postDetail(post, showOriginal);
const isHigh = rs >= 8;
return (
@@ -150,12 +199,29 @@ function TelegramPostCard({ post }: { post: TelegramOsintPost }) {
<p className="text-[11px] text-[var(--text-muted)] leading-relaxed whitespace-pre-wrap">{detail}</p>
) : null}
{translated && !showOriginal && post.source_lang ? (
<p className="text-[10px] text-cyan-700/80 uppercase tracking-wider">
{t('telegram.translatedFrom').replace('{lang}', sourceLangLabel(post))}
</p>
) : null}
<TelegramPostMedia post={post} />
<div className="flex items-center gap-1.5 mt-1 flex-wrap">
<span className={`text-[11px] font-bold font-mono px-1.5 py-0.5 rounded-sm border ${theme.badgeClass}`}>
{isHigh ? 'BREAKING' : `LVL: ${rs}/10`}
</span>
{translated ? (
<button
type="button"
onClick={() => setShowOriginal((prev) => !prev)}
className="text-[11px] font-mono text-cyan-600 hover:text-cyan-300 transition-colors"
>
{showOriginal
? t('telegram.showTranslation')
: t('telegram.showOriginal').replace('{lang}', sourceLangLabel(post))}
</button>
) : null}
{post.link ? (
<a
href={post.link}
@@ -172,15 +238,49 @@ function TelegramPostCard({ post }: { post: TelegramOsintPost }) {
}
export function TelegramOsintPopup({ posts, lat, lng, onClose }: TelegramOsintPopupProps) {
const { t } = useTranslation();
const { t, locale } = useTranslation();
const [localizedPosts, setLocalizedPosts] = useState(posts);
useEffect(() => {
setLocalizedPosts(posts);
}, [posts]);
useEffect(() => {
const needsLocalizedFeed = posts.some((post) => !hasTranslation(post));
if (!needsLocalizedFeed) {
return;
}
let cancelled = false;
const controller = new AbortController();
fetch(`/api/telegram-feed?lang=${encodeURIComponent(locale)}`, { signal: controller.signal })
.then((response) => (response.ok ? response.json() : null))
.then((payload) => {
if (cancelled || !payload?.posts) return;
const byId = new Map(
(payload.posts as TelegramOsintPost[]).map((post) => [post.id, post]),
);
setLocalizedPosts(posts.map((post) => byId.get(post.id) || post));
})
.catch(() => {
/* keep feed posts when locale translation fetch fails */
});
return () => {
cancelled = true;
controller.abort();
};
}, [locale, posts]);
const sortedPosts = useMemo(
() =>
[...posts].sort(
[...localizedPosts].sort(
(a, b) =>
(b.risk_score ?? 0) - (a.risk_score ?? 0) ||
String(b.published || '').localeCompare(String(a.published || '')),
),
[posts],
[localizedPosts],
);
const maxRisk = sortedPosts[0]?.risk_score ?? 1;
@@ -252,4 +352,4 @@ export function TelegramOsintPopup({ posts, lat, lng, onClose }: TelegramOsintPo
</div>
</Popup>
);
}
}
+79 -1
View File
@@ -321,7 +321,7 @@ function EmissionsEstimateBlock({ flight }: { flight: any }) {
);
}
function NewsFeedInner({ selectedEntity, regionDossier, regionDossierLoading, onArticleClick, onExpandEntityGraph }: { selectedEntity?: SelectedEntity | null, regionDossier?: RegionDossier | null, regionDossierLoading?: boolean, onArticleClick?: (idx: number, lat?: number, lng?: number, title?: string) => void, onExpandEntityGraph?: () => void }) {
function NewsFeedInner({ selectedEntity, regionDossier, regionDossierLoading, gtDossier, gtDossierLoading, onArticleClick, onExpandEntityGraph }: { selectedEntity?: SelectedEntity | null, regionDossier?: RegionDossier | null, regionDossierLoading?: boolean, gtDossier?: import('@/types/dashboard').GtDossier | null, gtDossierLoading?: boolean, onArticleClick?: (idx: number, lat?: number, lng?: number, title?: string) => void, onExpandEntityGraph?: () => void }) {
const data = useDataKeys([
'news', 'fimi', 'commercial_flights', 'private_flights', 'private_jets',
'military_flights', 'tracked_flights', 'ships', 'gdelt', 'liveuamap',
@@ -535,6 +535,84 @@ function NewsFeedInner({ selectedEntity, regionDossier, regionDossierLoading, on
)}
{/* Sentinel-2 imagery now shown as map popup — see MaplibreViewer */}
{(gtDossierLoading || gtDossier?.enabled) && (
<>
<div className="text-[11px] text-amber-500 tracking-widest font-bold border-b border-amber-900/50 pb-1 mt-2">
STRATEGIC RISK (GT)
</div>
{gtDossierLoading ? (
<div className="text-amber-400/80 text-[11px]">Running game-theoretic analysis...</div>
) : gtDossier ? (
<div className="flex flex-col gap-2">
<div className="flex justify-between">
<span className="text-[var(--text-muted)]">POSTERIOR RISK</span>
<span className="text-amber-300 font-bold">
{((gtDossier.current_risk ?? 0) * 100).toFixed(1)}%
</span>
</div>
{gtDossier.domain_risks && (
<div className="grid grid-cols-3 gap-2 text-[10px]">
<div>
<div className="text-[var(--text-muted)]">FIN</div>
<div className="text-cyan-300">
{((gtDossier.domain_risks.financial ?? 0) * 100).toFixed(0)}%
</div>
</div>
<div>
<div className="text-[var(--text-muted)]">UNREST</div>
<div className="text-orange-300">
{((gtDossier.domain_risks.unrest ?? 0) * 100).toFixed(0)}%
</div>
</div>
<div>
<div className="text-[var(--text-muted)]">CONFLICT</div>
<div className="text-red-300">
{((gtDossier.domain_risks.conflict ?? 0) * 100).toFixed(0)}%
</div>
</div>
</div>
)}
{gtDossier.interpretation && (
<div className="p-2 bg-black/60 border border-amber-800/50 text-[11px] text-amber-200/90 leading-relaxed">
<span className="text-amber-400 font-bold">&gt;_ GT: </span>
{gtDossier.interpretation}
</div>
)}
{gtDossier.recent_signals && gtDossier.recent_signals.length > 0 && (
<div className="flex flex-col gap-1">
<div className="text-[10px] text-[var(--text-muted)] tracking-widest">
COSTLY SIGNALS
</div>
{gtDossier.recent_signals.slice(-3).map((entry, idx) => (
<div
key={`${entry.timestamp}-${idx}`}
className="text-[10px] border-l-2 border-amber-700/60 pl-2 text-[var(--text-secondary)]"
>
<span className="text-amber-300 uppercase">
{Object.keys(entry.signals || {}).join(', ') || entry.domain}
</span>
{' · '}
<span className="text-[var(--text-muted)]">{entry.source}</span>
</div>
))}
</div>
)}
{gtDossier.scenarios && gtDossier.scenarios.length > 0 && (
<div className="flex flex-col gap-1">
<div className="text-[10px] text-[var(--text-muted)] tracking-widest">SCENARIOS</div>
{gtDossier.scenarios.map((scenario) => (
<div key={scenario.name} className="text-[10px] text-[var(--text-secondary)]">
<span className="text-amber-400 font-bold">{scenario.name}: </span>
{scenario.summary}
</div>
))}
</div>
)}
</div>
) : null}
</>
)}
</div>
) : d?.error ? (
<div className="p-4 text-[var(--text-secondary)] text-[12px]">{d.error}</div>
+53 -5
View File
@@ -55,6 +55,8 @@ import { useTheme } from '@/lib/ThemeContext';
import { useTranslation } from '@/i18n';
import SarModeChooserModal from './SarModeChooserModal';
import KiwiSdrConsentDialog from './ui/KiwiSdrConsentDialog';
import { extractGtAlerts } from '@/lib/gtAlerts';
import { gtLeanLayerWarning, useRuntimeProfile } from '@/hooks/useRuntimeProfile';
function relativeTime(iso: string | undefined): string {
if (!iso) return '';
@@ -115,6 +117,7 @@ const FRESHNESS_MAP: Record<string, string> = {
scm_suppliers: 'scm_suppliers',
cyber_threats: 'cyber_threats',
telegram_osint: 'telegram_osint',
gt_risk: 'gt_risk',
};
// POTUS fleet ICAO hex codes for client-side filtering
@@ -726,7 +729,11 @@ const WorldviewLeftPanel = React.memo(function WorldviewLeftPanel({
const [liveuamapModalOpen, setLiveuamapModalOpen] = useState(false);
const [liveuamapPendingEnable, setLiveuamapPendingEnable] = useState<(() => void) | null>(null);
const [gtLeanModalOpen, setGtLeanModalOpen] = useState(false);
const [gtLeanPendingEnable, setGtLeanPendingEnable] = useState<(() => void) | null>(null);
const { needsConsentBeforeEnable, confirmOptIn } = useLiveUamapScraperOptIn();
const runtimeProfile = useRuntimeProfile();
const gtLeanWarning = gtLeanLayerWarning(runtimeProfile);
const withGlobalIncidentsConsent = useCallback(
(layerId: string, turningOn: boolean, apply: () => void) => {
@@ -740,6 +747,18 @@ const WorldviewLeftPanel = React.memo(function WorldviewLeftPanel({
[needsConsentBeforeEnable],
);
const withGtRiskLeanWarning = useCallback(
(layerId: string, turningOn: boolean, apply: () => void) => {
if (layerId === 'gt_risk' && turningOn && gtLeanWarning) {
setGtLeanPendingEnable(() => apply);
setGtLeanModalOpen(true);
return;
}
apply();
},
[gtLeanWarning],
);
const isAllToggleableLayersOn = useMemo(
() =>
Object.entries(activeLayers)
@@ -1371,6 +1390,16 @@ const WorldviewLeftPanel = React.memo(function WorldviewLeftPanel({
count: data?.correlations?.filter((c: { type: string }) => c.type === 'contradiction').length || 0,
icon: Zap,
},
{
id: 'gt_risk',
name: t('layers.derivedOsint'),
source: t('layers.derivedOsintSource'),
count:
extractGtAlerts(data?.gt_risk).plottedRegions ||
data?.gt_risk?.meta?.plotted_regions ||
0,
icon: Radar,
},
{
id: 'day_night',
name: t('layers.dayNight'),
@@ -1394,7 +1423,7 @@ const WorldviewLeftPanel = React.memo(function WorldviewLeftPanel({
sections.forEach((s) => {
// Keep high-traffic intel overlays visible on first paint (GDELT, Telegram, etc.)
initial[s.label] = s.layers.some((l) =>
['global_incidents', 'telegram_osint', 'ukraine_frontline'].includes(l.id),
['global_incidents', 'telegram_osint', 'ukraine_frontline', 'gt_risk'].includes(l.id),
);
});
return initial;
@@ -1746,10 +1775,12 @@ const WorldviewLeftPanel = React.memo(function WorldviewLeftPanel({
return;
}
withGlobalIncidentsConsent(layer.id, !active, () => {
setActiveLayers((prev: ActiveLayers) => ({
...prev,
[layer.id]: !active,
}));
withGtRiskLeanWarning(layer.id, !active, () => {
setActiveLayers((prev: ActiveLayers) => ({
...prev,
[layer.id]: !active,
}));
});
});
}}
>
@@ -2081,6 +2112,23 @@ const WorldviewLeftPanel = React.memo(function WorldviewLeftPanel({
})();
}}
/>
<ConfirmDialog
open={gtLeanModalOpen}
title={t('gtLean.title')}
message={gtLeanWarning || t('gtLean.message')}
confirmLabel={t('gtLean.confirm')}
cancelLabel={t('gtLean.cancel')}
danger={false}
onCancel={() => {
setGtLeanModalOpen(false);
setGtLeanPendingEnable(null);
}}
onConfirm={() => {
gtLeanPendingEnable?.();
setGtLeanModalOpen(false);
setGtLeanPendingEnable(null);
}}
/>
</>
);
});
@@ -1956,3 +1956,64 @@ export function buildSarAoisGeoJSON(aois?: SarAoi[]): FC {
if (features.length === 0) return null;
return { type: 'FeatureCollection' as const, features };
}
// ─── Strategic Risk Analytics (GT early warning) ────────────────────────────
export function buildGtRiskGeoJSON(
payload?: {
enabled?: boolean;
heatmap?: { features?: Array<GTRiskHeatmapFeatureLike> };
} | null,
): FC {
const features = payload?.heatmap?.features;
if (!features?.length) return null;
const normalized = features
.map((feature, index) => {
const coords = feature.geometry?.coordinates;
if (!coords || coords.length < 2) return null;
const [lng, lat] = coords;
if (!Number.isFinite(lat) || !Number.isFinite(lng)) return null;
if (Math.abs(lat) < 0.001 && Math.abs(lng) < 0.001) return null;
const props = feature.properties || {};
const region = String(props.region || `region-${index}`);
return {
type: 'Feature' as const,
properties: {
...props,
type: 'gt_risk',
id: region,
name: region,
lat,
lng,
risk: Number(props.risk ?? 0),
financial: Number(props.financial ?? 0),
unrest: Number(props.unrest ?? 0),
conflict: Number(props.conflict ?? 0),
contagion: Number(props.contagion ?? 0),
},
geometry: {
type: 'Point' as const,
coordinates: [lng, lat] as [number, number],
},
};
})
.filter(Boolean) as GeoJSON.Feature[];
if (!normalized.length) return null;
return { type: 'FeatureCollection' as const, features: normalized };
}
type GTRiskHeatmapFeatureLike = {
properties?: {
region?: string;
risk?: number;
financial?: number;
unrest?: number;
conflict?: number;
contagion?: number;
};
geometry?: {
coordinates?: [number, number];
};
};
+121
View File
@@ -0,0 +1,121 @@
'use client';
import { useCallback, useEffect, useRef, useState } from 'react';
export interface FloatingPanelPosition {
x: number;
y: number;
}
interface StoredFloatingPanelState {
position?: FloatingPanelPosition;
isMinimized?: boolean;
}
interface UseFloatingPanelOptions {
defaultPosition?: FloatingPanelPosition;
minVisible?: number;
}
export function useFloatingPanel(
storageKey: string,
{ defaultPosition = { x: 24, y: 380 }, minVisible = 48 }: UseFloatingPanelOptions = {},
) {
const [position, setPosition] = useState<FloatingPanelPosition>(defaultPosition);
const [isMinimized, setIsMinimized] = useState(false);
const [isDragging, setIsDragging] = useState(false);
const dragStartRef = useRef({ x: 0, y: 0, posX: 0, posY: 0 });
const hydratedRef = useRef(false);
useEffect(() => {
try {
const raw = localStorage.getItem(storageKey);
if (!raw) return;
const parsed = JSON.parse(raw) as StoredFloatingPanelState;
if (
parsed.position &&
Number.isFinite(parsed.position.x) &&
Number.isFinite(parsed.position.y)
) {
setPosition(parsed.position);
}
if (typeof parsed.isMinimized === 'boolean') {
setIsMinimized(parsed.isMinimized);
}
} catch {
/* non-fatal */
} finally {
hydratedRef.current = true;
}
}, [storageKey]);
useEffect(() => {
if (!hydratedRef.current) return;
try {
localStorage.setItem(
storageKey,
JSON.stringify({ position, isMinimized } satisfies StoredFloatingPanelState),
);
} catch {
/* non-fatal */
}
}, [storageKey, position, isMinimized]);
const clampPosition = useCallback(
(next: FloatingPanelPosition): FloatingPanelPosition => {
const maxX = Math.max(0, window.innerWidth - minVisible);
const maxY = Math.max(0, window.innerHeight - minVisible);
return {
x: Math.min(Math.max(0, next.x), maxX),
y: Math.min(Math.max(0, next.y), maxY),
};
},
[minVisible],
);
const onDragStart = useCallback(
(event: React.MouseEvent) => {
event.preventDefault();
setIsDragging(true);
dragStartRef.current = {
x: event.clientX,
y: event.clientY,
posX: position.x,
posY: position.y,
};
},
[position.x, position.y],
);
useEffect(() => {
if (!isDragging) return undefined;
const handleMove = (event: MouseEvent) => {
const dx = event.clientX - dragStartRef.current.x;
const dy = event.clientY - dragStartRef.current.y;
setPosition(
clampPosition({
x: dragStartRef.current.posX + dx,
y: dragStartRef.current.posY + dy,
}),
);
};
const handleUp = () => setIsDragging(false);
window.addEventListener('mousemove', handleMove);
window.addEventListener('mouseup', handleUp);
return () => {
window.removeEventListener('mousemove', handleMove);
window.removeEventListener('mouseup', handleUp);
};
}, [isDragging, clampPosition]);
return {
position,
isMinimized,
setIsMinimized,
isDragging,
onDragStart,
};
}
+58
View File
@@ -0,0 +1,58 @@
import { useEffect, useState } from 'react';
import type { GtDossier } from '@/types/dashboard';
import { API_BASE } from '@/lib/api';
export function useGtDossier(
lat: number | undefined,
lng: number | undefined,
countryName?: string,
enabled = true,
) {
const [gtDossier, setGtDossier] = useState<GtDossier | null>(null);
const [gtDossierLoading, setGtDossierLoading] = useState(false);
useEffect(() => {
if (!enabled || lat == null || lng == null) {
setGtDossier(null);
setGtDossierLoading(false);
return;
}
let cancelled = false;
const regions = [
`${lat.toFixed(2)},${lng.toFixed(2)}`,
countryName?.trim().toLowerCase(),
].filter((value): value is string => Boolean(value));
const load = async () => {
setGtDossierLoading(true);
let best: GtDossier | null = null;
for (const region of regions) {
try {
const response = await fetch(
`${API_BASE}/api/analytics/dossier/${encodeURIComponent(region)}`,
);
if (!response.ok) continue;
const payload = (await response.json()) as GtDossier;
if (!payload.enabled) continue;
if (!best || (payload.current_risk ?? 0) > (best.current_risk ?? 0)) {
best = { ...payload, region };
}
} catch {
// GT analytics optional — ignore fetch errors
}
}
if (!cancelled) {
setGtDossier(best);
setGtDossierLoading(false);
}
};
void load();
return () => {
cancelled = true;
};
}, [lat, lng, countryName, enabled]);
return { gtDossier, gtDossierLoading };
}
+60
View File
@@ -0,0 +1,60 @@
'use client';
import { useEffect, useState } from 'react';
import { API_BASE } from '@/lib/api';
export interface RuntimeGtAnalytics {
enabled?: boolean;
operational?: boolean;
profile?: string;
lean_node?: boolean;
recommended?: boolean;
warning?: string | null;
experimental?: boolean;
}
export interface RuntimeProfile {
profile?: string;
cpu_limit?: number | null;
memory_limit_mb?: number | null;
gt_analytics?: RuntimeGtAnalytics;
}
export function useRuntimeProfile(): RuntimeProfile | null {
const [runtime, setRuntime] = useState<RuntimeProfile | null>(null);
useEffect(() => {
let cancelled = false;
const load = async () => {
try {
const res = await fetch(`${API_BASE}/api/health`, { cache: 'no-store' });
if (!res.ok || cancelled) return;
const body = await res.json();
if (!cancelled && body?.runtime) {
setRuntime(body.runtime as RuntimeProfile);
}
} catch {
/* health unavailable during boot */
}
};
void load();
const timer = window.setInterval(load, 60_000);
return () => {
cancelled = true;
window.clearInterval(timer);
};
}, []);
return runtime;
}
export function gtLeanLayerWarning(runtime: RuntimeProfile | null): string | null {
const gt = runtime?.gt_analytics;
if (!gt?.lean_node) return null;
return (
gt.warning ||
'This node is capped at 1 vCPU. Enabling Strategic Risk (Derived OSINT) may slow OSINT fetches.'
);
}
+77 -2
View File
@@ -208,7 +208,79 @@
"malwareC2": "Malware C2",
"scmSuppliers": "SCM Suppliers",
"cyberThreats": "Cyber Threats",
"telegramOsint": "Telegram OSINT"
"telegramOsint": "Telegram OSINT",
"strategicRisk": "Strategic Risk Heatmap",
"derivedOsint": "Derived OSINT (Strategic Risk)",
"derivedOsintSource": "Experimental · off by default"
},
"gtLean": {
"title": "Enable Derived OSINT on a lean node?",
"message": "Shadowbroker detected a 1 vCPU cap on this node. Turning on the Strategic Risk map layer is safe for display, but enabling the backend engine (GT_ANALYTICS_ENABLED) may slow Telegram, GDELT, and other OSINT fetches. Use OpenClaw watchdog alerts without the full engine on fleet nodes.",
"confirm": "Turn on layer anyway",
"cancel": "Cancel"
},
"gtHud": {
"title": "GT ANALYTICS",
"dragHint": "drag to move",
"collapse": "Collapse panel",
"expand": "Expand panel"
},
"gtAlerts": {
"title": "TOP ALERTS",
"counts": "{plotted} on map · {tracked} tracked (max {max})",
"empty": "No plottable regions yet — need geotagged intel (Telegram/GDELT/news).",
"ignition": "IGNITE",
"line": "risk {risk} · conflict {conflict}",
"hint": "500 = max tracked regions, not individual events. Click to fly there."
},
"gtRisk": {
"popupTitle": "STRATEGIC RISK",
"region": "REGION",
"composite": "POSTERIOR RISK",
"financial": "FIN",
"unrest": "UNREST",
"conflict": "CONFLICT",
"contagion": "CONTAGION",
"costlySignals": "COSTLY SIGNALS",
"loadingSignals": "Loading feed matches…",
"noSignals": "No costly-signal text matched in recent Telegram/GDELT/news for this region. Scores can rise from domain priors or nearby contagion.",
"unknownSource": "unknown source"
},
"gtBacktest": {
"title": "GT Backtest",
"layerOff": "Off — enable Strategic Risk Heatmap in Data Layers.",
"disabled": "GT analytics disabled (set GT_ANALYTICS_ENABLED=true).",
"loading": "Running historical validation…",
"refresh": "Refresh backtest",
"accuracy": "ACCURACY",
"confidence": "WILSON 95% LB",
"cases": "{count} labeled cases",
"threshold": "alert ≥ {value}",
"target": "target {value}",
"pass": "PASS",
"fail": "FAIL",
"collecting": "COLLECTING",
"meetsTarget": "Meets confidence target",
"belowTarget": "Below confidence target",
"misclassified": "{count} misclassified",
"tabBenchmark": "Benchmark",
"tabOperational": "Operational",
"benchmarkNote": "Static labeled corpus — regression test, not live forecasting.",
"operationalLoading": "Loading rolling operational trend…",
"operationalEmpty": "No weekly snapshots yet — freeze runs Mondays 00:05 UTC or via OpenClaw.",
"operationalWeeks": "{stored} weeks stored · {scorable} scorable",
"operationalLabeled": "{labeled} labeled · {pending} pending",
"operationalTrend": "Week-over-week accuracy",
"operationalImproving": "Improving vs prior scorable week",
"operationalFlat": "Flat or down vs prior scorable week",
"operationalMinLabels": "Need ≥{count} labels/week to score",
"microTitle": "3-day micro",
"microWindow": "{days}-day rolling avg · ignition Δ ≥ {delta}",
"microIgnitions": "{count} ignition(s)",
"microAlerted3d": "{count} above threshold on 3d avg",
"microEmpty": "Collecting daily readings — refreshes with GT ingest.",
"microRegionLine": "{region}: spot {spot} · 3d {avg} · Δ {delta}",
"microIgnitionBadge": "IGNITION"
},
"roadCorridor": {
"analyzeHere": "ANALYZE HERE",
@@ -273,6 +345,9 @@
"loadMedia": "VIEW MEDIA (TELEGRAM)",
"openOriginal": "OPEN ON TELEGRAM →",
"embedTitle": "Telegram post embed",
"postsAtLocation": "{count} posts at this location — scroll for more"
"postsAtLocation": "{count} posts at this location — scroll for more",
"translatedFrom": "Translated from {lang}",
"showOriginal": "SHOW ORIGINAL ({lang})",
"showTranslation": "SHOW TRANSLATION"
}
}
+77 -2
View File
@@ -208,7 +208,79 @@
"malwareC2": "Malware C2",
"scmSuppliers": "Fournisseurs SCM",
"cyberThreats": "Cybermenaces",
"telegramOsint": "OSINT Telegram"
"telegramOsint": "OSINT Telegram",
"strategicRisk": "Carte de risque stratégique",
"derivedOsint": "OSINT dérivé (risque stratégique)",
"derivedOsintSource": "Expérimental · désactivé par défaut"
},
"gtLean": {
"title": "Activer l'OSINT dérivé sur un nœud limité ?",
"message": "Shadowbroker a détecté une limite de 1 vCPU. La couche carte peut s'afficher, mais activer le moteur backend peut ralentir les flux OSINT.",
"confirm": "Activer la couche",
"cancel": "Annuler"
},
"gtHud": {
"title": "ANALYTIQUE GT",
"dragHint": "glisser pour déplacer",
"collapse": "Réduire le panneau",
"expand": "Développer le panneau"
},
"gtAlerts": {
"title": "ALERTES TOP",
"counts": "{plotted} sur carte · {tracked} suivies (max {max})",
"empty": "Aucune région plottable — intel géolocalisée requise.",
"ignition": "IGNITE",
"line": "risque {risk} · conflit {conflict}",
"hint": "500 = régions max suivies, pas des événements. Cliquer pour voler."
},
"gtRisk": {
"popupTitle": "RISQUE STRATÉGIQUE",
"region": "RÉGION",
"composite": "RISQUE POSTÉRIEUR",
"financial": "FIN",
"unrest": "TROUBLES",
"conflict": "CONFLIT",
"contagion": "CONTAGION",
"costlySignals": "SIGNAUX COÛTEUX",
"loadingSignals": "Chargement des correspondances…",
"noSignals": "Aucun signal coûteux récent pour cette région dans Telegram/GDELT/news.",
"unknownSource": "source inconnue"
},
"gtBacktest": {
"title": "Backtest GT",
"layerOff": "Désactivé — activez la carte de risque stratégique dans Couches.",
"disabled": "Analytique GT désactivée (GT_ANALYTICS_ENABLED=true).",
"loading": "Validation historique en cours…",
"refresh": "Actualiser le backtest",
"accuracy": "PRÉCISION",
"confidence": "BORNE INF. WILSON 95%",
"cases": "{count} cas étiquetés",
"threshold": "alerte ≥ {value}",
"target": "cible {value}",
"pass": "OK",
"fail": "ÉCHEC",
"collecting": "COLLECTE",
"meetsTarget": "Objectif de confiance atteint",
"belowTarget": "Sous l'objectif de confiance",
"misclassified": "{count} mal classés",
"tabBenchmark": "Référence",
"tabOperational": "Opérationnel",
"benchmarkNote": "Corpus historique étiqueté — test de régression, pas prévision live.",
"operationalLoading": "Chargement de la tendance opérationnelle…",
"operationalEmpty": "Aucun instantané hebdomadaire — gel chaque lundi 00:05 UTC ou via OpenClaw.",
"operationalWeeks": "{stored} semaines · {scorable} exploitables",
"operationalLabeled": "{labeled} étiquetés · {pending} en attente",
"operationalTrend": "Précision semaine après semaine",
"operationalImproving": "En hausse vs semaine précédente",
"operationalFlat": "Stable ou en baisse vs semaine précédente",
"operationalMinLabels": "≥{count} étiquettes/semaine requis",
"microTitle": "Micro 3 jours",
"microWindow": "Moy. glissante {days} j · ignition Δ ≥ {delta}",
"microIgnitions": "{count} ignition(s)",
"microAlerted3d": "{count} au-dessus du seuil (moy. 3j)",
"microEmpty": "Lecture quotidienne en cours — mis à jour à chaque ingest GT.",
"microRegionLine": "{region}: spot {spot} · 3j {avg} · Δ {delta}",
"microIgnitionBadge": "IGNITION"
},
"roadCorridor": {
"analyzeHere": "ANALYSER ICI",
@@ -273,6 +345,9 @@
"loadMedia": "AFFICHER LE MÉDIA (TELEGRAM)",
"openOriginal": "OUVRIR SUR TELEGRAM →",
"embedTitle": "Intégration Telegram",
"postsAtLocation": "{count} posts à cet endroit — faites défiler"
"postsAtLocation": "{count} posts à cet endroit — faites défiler",
"translatedFrom": "Traduit depuis {lang}",
"showOriginal": "AFFICHER L'ORIGINAL ({lang})",
"showTranslation": "AFFICHER LA TRADUCTION"
}
}
+77 -2
View File
@@ -208,7 +208,79 @@
"malwareC2": "恶意软件 C2",
"scmSuppliers": "供应链供应商",
"cyberThreats": "网络威胁",
"telegramOsint": "Telegram OSINT"
"telegramOsint": "Telegram OSINT",
"strategicRisk": "战略风险热力图",
"derivedOsint": "衍生 OSINT(战略风险)",
"derivedOsintSource": "实验功能 · 默认关闭"
},
"gtLean": {
"title": "在低配节点上启用衍生 OSINT",
"message": "Shadowbroker 检测到该节点 CPU 上限为 1 vCPU。开启地图图层通常安全,但启用后端引擎可能会拖慢 Telegram、GDELT 等 OSINT 抓取。",
"confirm": "仍要开启图层",
"cancel": "取消"
},
"gtHud": {
"title": "GT 分析",
"dragHint": "拖动移动",
"collapse": "收起面板",
"expand": "展开面板"
},
"gtAlerts": {
"title": "重点警报",
"counts": "地图 {plotted} · 跟踪 {tracked}(上限 {max}",
"empty": "尚无可绘制区域 — 需要带地理标签的情报。",
"ignition": "点火",
"line": "风险 {risk} · 冲突 {conflict}",
"hint": "500 = 最大跟踪区域数,非事件数。点击飞往。"
},
"gtRisk": {
"popupTitle": "战略风险",
"region": "区域",
"composite": "后验风险",
"financial": "金融",
"unrest": "动荡",
"conflict": "冲突",
"contagion": "传染",
"costlySignals": "成本信号",
"loadingSignals": "正在加载情报匹配…",
"noSignals": "该区域最近在 Telegram/GDELT/新闻中未匹配到成本信号文本。",
"unknownSource": "未知来源"
},
"gtBacktest": {
"title": "GT 回测",
"layerOff": "已关闭 — 请在数据图层中启用战略风险热力图。",
"disabled": "GT 分析未启用(需设置 GT_ANALYTICS_ENABLED=true)。",
"loading": "正在运行历史验证…",
"refresh": "刷新回测",
"accuracy": "准确率",
"confidence": "Wilson 95% 下界",
"cases": "{count} 个标注案例",
"threshold": "警报阈值 ≥ {value}",
"target": "目标 {value}",
"pass": "通过",
"fail": "未通过",
"collecting": "采集中",
"meetsTarget": "达到置信目标",
"belowTarget": "低于置信目标",
"misclassified": "{count} 个误分类",
"tabBenchmark": "基准测试",
"tabOperational": "运营验证",
"benchmarkNote": "静态标注语料 — 回归测试,非实时预测。",
"operationalLoading": "正在加载滚动运营趋势…",
"operationalEmpty": "尚无周快照 — 每周一 00:05 UTC 自动冻结,或通过 OpenClaw。",
"operationalWeeks": "已存 {stored} 周 · {scorable} 周可评分",
"operationalLabeled": "已标注 {labeled} · 待标注 {pending}",
"operationalTrend": "逐周准确率",
"operationalImproving": "较上一可评分周有所提升",
"operationalFlat": "较上一可评分周持平或下降",
"operationalMinLabels": "每周需 ≥{count} 条标注才可评分",
"microTitle": "3日微观",
"microWindow": "{days}日滚动均值 · 点火 Δ ≥ {delta}",
"microIgnitions": "{count} 个点火",
"microAlerted3d": "{count} 个区域 3日均值超阈值",
"microEmpty": "正在采集日读数 — 随 GT 摄入更新。",
"microRegionLine": "{region}:即时 {spot} · 3日 {avg} · Δ {delta}",
"microIgnitionBadge": "点火"
},
"roadCorridor": {
"analyzeHere": "分析此处",
@@ -273,6 +345,9 @@
"loadMedia": "查看媒体(Telegram",
"openOriginal": "在 Telegram 打开 →",
"embedTitle": "Telegram 帖子嵌入",
"postsAtLocation": "此位置 {count} 条帖子 — 向下滚动查看更多"
"postsAtLocation": "此位置 {count} 条帖子 — 向下滚动查看更多",
"translatedFrom": "由 {lang} 翻译",
"showOriginal": "显示原文({lang}",
"showTranslation": "显示译文"
}
}
+104
View File
@@ -0,0 +1,104 @@
import type { GTRiskPayload } from '@/types/dashboard';
export interface GtAlertRow {
region: string;
regionLabel: string;
risk: number;
conflict: number;
unrest: number;
financial: number;
contagion: number;
lat: number;
lng: number;
score: number;
ignition: boolean;
risk3d?: number;
riskDelta?: number;
}
export function formatGtRegionLabel(region: string): string {
const text = String(region || '').trim();
if (!text) return 'unknown';
const coord = text.match(/^(-?\d+(?:\.\d+)?),\s*(-?\d+(?:\.\d+)?)$/);
if (coord) {
return `${Number(coord[1]).toFixed(2)}°, ${Number(coord[2]).toFixed(2)}°`;
}
const parts = text.split(',').map((piece) => piece.trim()).filter(Boolean);
if (parts.length >= 2) {
const lat = Number(parts[0]);
const lng = Number(parts[parts.length - 1]);
if (Number.isFinite(lat) && Number.isFinite(lng)) {
return `${lat.toFixed(2)}°, ${lng.toFixed(2)}°`;
}
}
return text.replace(/_/g, ' ');
}
function validCoords(coords: unknown): { lat: number; lng: number } | null {
if (!Array.isArray(coords) || coords.length < 2) return null;
const lng = Number(coords[0]);
const lat = Number(coords[1]);
if (!Number.isFinite(lat) || !Number.isFinite(lng)) return null;
if (Math.abs(lat) < 0.001 && Math.abs(lng) < 0.001) return null;
return { lat, lng };
}
function peakScore(props: Record<string, unknown>): number {
const composite = Number(props.risk ?? 0);
const financial = Number(props.financial ?? 0);
const unrest = Number(props.unrest ?? 0);
const conflict = Number(props.conflict ?? 0);
return Math.max(composite, financial, unrest, conflict);
}
export function extractGtAlerts(
payload?: GTRiskPayload | null,
limit = 8,
): {
alerts: GtAlertRow[];
trackedRegions: number;
plottedRegions: number;
maxRegions: number;
} {
const features = payload?.heatmap?.features || [];
const meta = payload?.meta;
const rows: GtAlertRow[] = [];
for (const feature of features) {
const coords = validCoords(feature.geometry?.coordinates);
if (!coords) continue;
const props = (feature.properties || {}) as Record<string, unknown>;
const region = String(props.region || '').trim().toLowerCase();
if (!region) continue;
rows.push({
region,
regionLabel: formatGtRegionLabel(region),
risk: Number(props.risk ?? 0),
financial: Number(props.financial ?? 0),
unrest: Number(props.unrest ?? 0),
conflict: Number(props.conflict ?? 0),
contagion: Number(props.contagion ?? 0),
lat: coords.lat,
lng: coords.lng,
score: peakScore(props),
ignition: Boolean(props.micro_ignition),
risk3d: props.risk_3d_avg != null ? Number(props.risk_3d_avg) : undefined,
riskDelta: props.risk_delta != null ? Number(props.risk_delta) : undefined,
});
}
rows.sort((a, b) => {
if (a.ignition !== b.ignition) return a.ignition ? -1 : 1;
const deltaA = a.riskDelta ?? 0;
const deltaB = b.riskDelta ?? 0;
if (deltaA !== deltaB) return deltaB - deltaA;
return b.score - a.score;
});
return {
alerts: rows.slice(0, limit),
trackedRegions: meta?.tracked_regions ?? features.length,
plottedRegions: meta?.plotted_regions ?? rows.length,
maxRegions: meta?.max_regions ?? 500,
};
}
+182
View File
@@ -966,12 +966,193 @@ export interface DashboardData {
timestamp?: string | null;
channels?: string[];
};
gt_risk?: GTRiskPayload;
}
export interface GTRiskHeatmapFeature {
type: 'Feature';
properties: {
region: string;
risk: number;
financial?: number;
unrest?: number;
conflict?: number;
contagion?: number;
updates?: number;
risk_spot?: number;
risk_3d_avg?: number;
risk_delta?: number;
micro_ignition?: boolean;
};
geometry: {
type: 'Point';
coordinates: [number, number];
};
}
export interface GTRiskPayload {
enabled?: boolean;
timestamp?: string | null;
processed?: number;
meta?: {
tracked_regions?: number;
engine_regions?: number;
plotted_regions?: number;
max_regions?: number;
};
heatmap?: {
type: 'FeatureCollection';
features: GTRiskHeatmapFeature[];
};
clusters?: Array<{
cluster_id: number;
size: number;
mean_risk: number;
regions?: string[];
members?: string[];
}>;
}
export interface GtDossierSignalEntry {
timestamp: string;
domain: string;
signals: Record<string, number>;
strength: number;
posterior: number;
source: string;
deviation_score?: number;
}
export interface GtBacktestCaseResult {
case_id: string;
name: string;
kind: string;
correct: boolean;
alerted: boolean;
peak_domain_risk: number;
peak_composite_risk: number;
costly_signals: string[];
}
export interface GtBacktestReport {
enabled?: boolean;
total_cases: number;
correct: number;
accuracy: number;
confidence_rate: number;
wilson_lower_95: number;
wilson_upper_95: number;
true_positives: number;
true_negatives: number;
false_positives: number;
false_negatives: number;
sensitivity: number;
specificity: number;
alert_threshold: number;
target_confidence: number;
meets_target: boolean;
expanded_suite?: boolean;
tuned?: boolean;
recommended_alert_threshold?: number;
cases?: GtBacktestCaseResult[];
}
export interface GtRollingWeekScore {
week_id: string;
frozen_at?: string;
alert_threshold: number;
total_regions: number;
labeled: number;
pending: number;
alerted: number;
correct: number;
accuracy: number;
confidence_rate: number;
wilson_lower_95: number;
wilson_upper_95: number;
true_positives: number;
true_negatives: number;
false_positives: number;
false_negatives: number;
sensitivity: number;
specificity: number;
scorable: boolean;
}
export interface GtMicroRegionView {
region: string;
spot_risk: number;
risk_3d_avg: number;
risk_delta: number;
days_in_window: number;
day_scores: number[];
alerted_spot: boolean;
alerted_3d: boolean;
ignition: boolean;
financial: number;
unrest: number;
conflict: number;
}
export interface GtMicroRollingReport {
enabled?: boolean;
mode?: string;
window_days: number;
alert_threshold: number;
ignition_delta: number;
as_of: string;
days_stored: number;
regions_tracked: number;
ignition_count: number;
alerted_3d_count: number;
ignitions: GtMicroRegionView[];
top_regions: GtMicroRegionView[];
note?: string;
message?: string;
}
export interface GtRollingReport {
enabled?: boolean;
mode?: string;
alert_threshold: number;
target_confidence: number;
weeks_requested: number;
weeks_stored: number;
weeks_scorable: number;
min_labeled_per_week: number;
latest: GtRollingWeekScore | null;
trend: GtRollingWeekScore[];
accuracy_series: { week_id: string; accuracy: number; labeled: number }[];
improving_vs_prior: boolean;
meets_target: boolean;
note?: string;
message?: string;
}
export interface GtDossier {
enabled?: boolean;
region: string;
current_risk: number;
domain_risks?: {
financial?: number;
unrest?: number;
conflict?: number;
};
recent_signals?: GtDossierSignalEntry[];
contagion_risk?: number;
interpretation?: string;
scenarios?: Array<{ name: string; summary: string }>;
}
export interface TelegramOsintPost {
id: string;
title?: string;
description?: string;
title_translated?: string;
description_translated?: string;
source_lang?: string;
source_lang_label?: string;
translate_to?: string;
link?: string;
published?: string;
source?: string;
@@ -1120,6 +1301,7 @@ export interface ActiveLayers {
scm_suppliers: boolean;
cyber_threats: boolean;
telegram_osint: boolean;
gt_risk: boolean;
}
export interface SelectedEntity {
+58
View File
@@ -170,6 +170,36 @@ The channel operates over HMAC-authenticated HTTP with body-integrity binding:
| `await sb.get_slow_telemetry()` | Slow-tier: GDELT, news, earthquakes, markets, correlations, Telegram OSINT, malware/cyber threats, SCM suppliers |
| `await sb.get_report()` | Full structured intelligence report |
### Strategic Risk Analytics (GT early warning)
Requires `GT_ANALYTICS_ENABLED=true` on the ShadowBroker backend.
| Method / command | What It Returns |
|------------------|----------------|
| `await sb.ask("Run GT analysis on UK/Europe feeds")` | Routes to `gt_analyze` |
| `await sb.gt_analyze(region="ukraine")` | Refresh beliefs from Telegram/news/GDELT + dossier |
| `await sb.gt_risk_heatmap()` | GeoJSON posterior risk overlay + Louvain clusters |
| `await sb.gt_dossier("ukraine")` | Costly signals, domain risks, scenarios |
| `await sb.gt_backtest()` | **Static benchmark** — labeled historical cases (regression test) |
| `await sb.gt_backtest(tune=True)` | Grid-search alert threshold for target confidence |
| `await sb.gt_rolling_backtest()` | **Macro operational** — week-over-week accuracy on frozen weekly alerts |
| `await sb.gt_micro_rolling()` | **Micro 3-day rolling avg** — spot vs baseline, ignition detection |
| `await sb.gt_rolling_freeze()` | Freeze this ISO week's GT scores before outcomes are known |
| `await sb.gt_rolling_label(week_id, region=..., label=...)` | Label prior-week outcomes (`true_escalation`, `false_alarm`, `benign`) |
| `await sb.gt_top_alerts()` | Ranked top GT regions with map coordinates |
| `await sb.ask("Run GT historical backtest")` | Routes to `gt_backtest` (benchmark, not operational) |
| `await sb.ask("GT rolling operational backtest trend")` | Routes to `gt_rolling_backtest` |
| `python sb_gt_report.py` | Local helper — backtest + heatmap (+ optional `--region`) |
| `await sb.send_command("gt_analyze", {"region": "europe"})` | Same as `gt_analyze()` |
**Benchmark vs rolling:** Static `gt_backtest` checks the classifier on known textbook
cases. `gt_rolling_backtest` scores **frozen weekly live predictions** against delayed
operator labels — that week-over-week trend (e.g. 54% → 62% → 71%) is the macro
real-world metric. `gt_micro_rolling` adds a **3-day rolling average** per region:
spot risk vs the trailing baseline catches fast ignitions the weekly roll can miss.
Threshold is fixed (`GT_ROLLING_ALERT_THRESHOLD`, default 0.26); ignition when
spot 3d avg ≥ `GT_MICRO_IGNITION_DELTA` (default 0.10).
**When to use**: Use `get_summary()` first. Use `get_layer_slice()` for the layers
you actually need. Reserve full `get_telemetry()` / `get_slow_telemetry()` for rare
cases where you genuinely need every field across every layer.
@@ -692,6 +722,34 @@ When the user asks a question, follow this decision tree:
- YES if the user has configured alert channels
- Use the `AlertDispatcher` with the correct signature
### Telegram rhetoric monitoring (watchdog)
Use watchdog watches for push alerts over SSE — no polling required. Keyword
watches now scan Telegram OSINT too (translated **and** original text).
```python
# Alert when "nuclear" appears in news, GDELT, or Telegram OSINT
await sb.send_command("add_watch", {
"type": "keyword",
"params": {"keyword": "nuclear", "include_telegram": True},
})
# Alert on new high-risk Telegram posts (LVL >= 7) — rhetoric/escalation monitor
await sb.send_command("add_watch", {
"type": "telegram_rhetoric",
"params": {"min_risk_score": 7, "channels": ["nexta_live", "war_monitor"]},
})
# Combine risk threshold + topic filter
await sb.send_command("add_watch", {
"type": "telegram_rhetoric",
"params": {"min_risk_score": 8, "keywords": ["crimea", "escalation", "missile"]},
})
```
When a watch fires, you receive an SSE `alert` event. Forward it with
`sb_alerts.send_intel()` if the user has Discord/Telegram notification channels.
---
## Important Rules
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env python3
"""One-shot get_summary for OpenClaw exec — loads .env.shadowbroker automatically."""
from __future__ import annotations
import asyncio
import json
import os
from pathlib import Path
def _load_env() -> None:
candidates = [
Path.home() / ".openclaw" / "workspace" / ".env.shadowbroker",
Path(__file__).resolve().parent.parent.parent / ".env.shadowbroker",
]
for path in candidates:
if not path.is_file():
continue
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
os.environ.setdefault(key.strip(), value.strip())
break
async def main() -> None:
_load_env()
from sb_query import ShadowBrokerClient
sb = ShadowBrokerClient()
try:
resp = await sb.send_command("get_summary", {"compact": True})
print(json.dumps(resp, indent=2))
finally:
await sb.close()
if __name__ == "__main__":
asyncio.run(main())
+64
View File
@@ -0,0 +1,64 @@
#!/usr/bin/env python3
"""GT Strategic Risk report — backtest + heatmap + optional region dossier.
Backtest scores are benchmark validation on labeled historical snippets, not
forward-weeks prediction on live adversarial telemetry. See SKILL.md.
"""
from __future__ import annotations
import argparse
import asyncio
import json
import os
from pathlib import Path
def _load_env() -> None:
for path in (
Path.home() / ".openclaw" / "workspace" / ".env.shadowbroker",
Path(__file__).resolve().parent.parent.parent / ".env.shadowbroker",
):
if not path.is_file():
continue
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
os.environ.setdefault(key.strip(), value.strip())
break
async def main() -> None:
parser = argparse.ArgumentParser(description="ShadowBroker GT analytics report")
parser.add_argument("--region", default="", help="Optional region for gt_analyze dossier")
parser.add_argument("--tune", action="store_true", help="Grid-search backtest threshold")
args = parser.parse_args()
_load_env()
from sb_query import ShadowBrokerClient
sb = ShadowBrokerClient()
report: dict[str, object] = {
"benchmark_note": (
"Backtest accuracy is on curated pre-crisis snippets vs cheap-talk controls. "
"It does not claim multi-week forward prediction on live feeds."
),
}
try:
report["backtest"] = await sb.gt_backtest(expanded=True, tune=args.tune)
heatmap = await sb.gt_risk_heatmap()
report["heatmap"] = {
"feature_count": len(heatmap.get("features") or []),
"clusters": heatmap.get("clusters") or [],
}
if args.region:
report["analyze"] = await sb.gt_analyze(region=args.region, refresh=True)
finally:
await sb.close()
print(json.dumps(report, indent=2))
if __name__ == "__main__":
asyncio.run(main())
+128
View File
@@ -267,6 +267,17 @@ class ShadowBrokerClient:
Returns:
{ok, tier, reason, transport, pending_commands, pending_tasks, stats}
"""
# /api/ai/channel/status is local-operator only. HMAC-signed remote
# agents must probe via the command channel instead.
if self._hmac_secret:
resp = await self.send_command("get_summary", {"compact": True})
return {
"ok": bool(resp.get("ok")),
"tier": resp.get("tier"),
"status": resp.get("status"),
"transport": "http+hmac",
"reason": "remote_hmac_probe",
}
r = await self._get("/api/ai/channel/status")
return r.json()
@@ -551,6 +562,123 @@ class ShadowBrokerClient:
r.raise_for_status()
return r.json()
# ── Strategic Risk Analytics (game-theoretic early warning) ───────
async def gt_risk_heatmap(self) -> dict:
"""Cached Bayesian risk heatmap (GeoJSON features + Louvain clusters)."""
return self.unwrap_channel_result(await self.send_command("gt_risk_heatmap", {}))
async def gt_dossier(self, region: str) -> dict:
"""GT rationale, costly signals, and scenarios for a region."""
return self.unwrap_channel_result(
await self.send_command("gt_dossier", {"region": region}),
)
async def gt_analyze(
self,
*,
region: str = "",
refresh: bool = True,
feeds: list[dict] | None = None,
) -> dict:
"""Refresh GT beliefs from intel feeds and return heatmap/dossier."""
args: dict[str, Any] = {"refresh": refresh}
if region:
args["region"] = region
if feeds:
args["feeds"] = feeds
return self.unwrap_channel_result(await self.send_command("gt_analyze", args))
async def gt_backtest(
self,
*,
expanded: bool = True,
tune: bool = False,
target_confidence: float = 0.95,
alert_threshold: float | None = None,
include_cases: bool = False,
) -> dict:
"""Run labeled historical backtest; returns accuracy + Wilson 95% CI."""
args: dict[str, Any] = {
"expanded": expanded,
"tune": tune,
"target_confidence": target_confidence,
"include_cases": include_cases,
"compact": True,
}
if alert_threshold is not None:
args["alert_threshold"] = alert_threshold
return self.unwrap_channel_result(await self.send_command("gt_backtest", args))
async def gt_rolling_freeze(
self,
*,
week_id: str | None = None,
force: bool = False,
) -> dict:
"""Freeze current GT scores for the ISO week (operational validation)."""
args: dict[str, Any] = {"compact": True, "force": force}
if week_id:
args["week_id"] = week_id
return self.unwrap_channel_result(await self.send_command("gt_rolling_freeze", args))
async def gt_rolling_label(
self,
week_id: str,
*,
region: str = "",
label: str = "",
notes: str = "",
labels: list[dict] | None = None,
) -> dict:
"""Apply delayed outcome labels to a frozen operational week."""
args: dict[str, Any] = {"week_id": week_id}
if labels:
args["labels"] = labels
else:
args["region"] = region
args["label"] = label
args["notes"] = notes
return self.unwrap_channel_result(await self.send_command("gt_rolling_label", args))
async def gt_rolling_backtest(
self,
*,
weeks: int = 8,
target_confidence: float = 0.80,
) -> dict:
"""Rolling weekly operational accuracy trend (delayed labels)."""
return self.unwrap_channel_result(
await self.send_command(
"gt_rolling_backtest",
{
"weeks": weeks,
"target_confidence": target_confidence,
"compact": True,
},
)
)
async def gt_top_alerts(self, *, limit: int = 8) -> dict:
"""Ranked top GT risk regions with map coordinates."""
return self.unwrap_channel_result(
await self.send_command("gt_top_alerts", {"limit": limit, "compact": True})
)
async def gt_micro_rolling(
self,
*,
window_days: int = 3,
limit: int = 15,
) -> dict:
"""3-day rolling micro average — spot vs baseline, ignition regions."""
return self.unwrap_channel_result(
await self.send_command(
"gt_micro_rolling",
{"window_days": window_days, "limit": limit, "compact": True},
)
)
# ── Geocoding ─────────────────────────────────────────────────────
async def geocode(self, query: str) -> list[dict]:
Generated
+8 -2
View File
@@ -104,7 +104,7 @@ wheels = [
[[package]]
name = "backend"
version = "0.9.82"
version = "0.9.83"
source = { editable = "backend" }
dependencies = [
{ name = "apscheduler" },
@@ -116,6 +116,10 @@ dependencies = [
{ name = "feedparser" },
{ name = "httpx" },
{ name = "meshtastic" },
{ name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "numpy", version = "2.4.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
{ name = "orjson" },
{ name = "paho-mqtt" },
{ name = "playwright" },
@@ -171,6 +175,8 @@ requires-dist = [
{ name = "httpx", specifier = "==0.28.1" },
{ name = "imageio", marker = "extra == 'road-corridor'", specifier = ">=2.34.0" },
{ name = "meshtastic", specifier = ">=2.5.0" },
{ name = "networkx", specifier = ">=3.4.0" },
{ name = "numpy", specifier = ">=2.2.0" },
{ name = "orjson", specifier = ">=3.10.0" },
{ name = "osmnx", marker = "extra == 'road-corridor'", specifier = ">=2.0.0" },
{ name = "paho-mqtt", specifier = ">=1.6.0,<2.0.0" },
@@ -3052,7 +3058,7 @@ wheels = [
[[package]]
name = "shadowbroker"
version = "0.9.82"
version = "0.9.83"
source = { virtual = "." }
[package.metadata]