Feat/gt analytics openclaw (#392)

* feat(telegram): auto-translate OSINT channel posts to English Cherry-picked from @Bobpick PR #391 (telegram-only slice): server-side translation during fetch, SHOW ORIGINAL toggle in TelegramOsintPopup, and on-demand /api/telegram-feed?lang=. Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com> Co-authored-by: Cursor <cursoragent@cursor.com> * feat(gt): experimental Derived OSINT analytics with lean-node safeguards Cherry-picked from @Bobpick PR #391 (GT + OpenClaw slice): Bayesian strategic-risk engine, map overlay, OpenClaw commands, and telegram_rhetoric watchdog. Off by default (GT_ANALYTICS_ENABLED=false, gt_risk layer false). 1 vCPU nodes get cgroup detection, UI warning on layer toggle, and lean profile that skips scheduled ingest/Louvain unless GT_ANALYTICS_ACK_LOW_CPU=true. Backtest HUD removed from dashboard (OpenClaw/API regression only). Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com> Co-authored-by: Cursor <cursoragent@cursor.com> --------- Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com> Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-17 16:07:24 +02:00 · 2026-06-16 17:05:46 -06:00
parent 9c5a4054f6
commit cfbeabda1e
69 changed files with 8101 additions and 77 deletions
@@ -2051,7 +2051,7 @@ async def agent_tool_manifest(request: Request):
                "description": "Set up a watchdog alert. When triggered, alerts push instantly via SSE stream. Debounced: same watch won't re-fire within 60 seconds.",
                "parameters": {
                    "type": {"type": "string", "required": True, "description": "Watch type",
-                             "enum": ["track_aircraft", "track_callsign", "track_registration", "track_ship", "track_entity", "geofence", "keyword", "prediction_market"]},
+                             "enum": ["track_aircraft", "track_callsign", "track_registration", "track_ship", "track_entity", "geofence", "keyword", "telegram_rhetoric", "prediction_market"]},
                    "params": {"type": "object", "required": True, "description": "Type-specific parameters (see subtypes)"},
                },
                "subtypes": {
@@ -2061,7 +2061,8 @@ async def agent_tool_manifest(request: Request):
                    "track_ship": {"params": {"mmsi": "string (optional)", "imo": "string (optional)", "name": "string (optional)", "owner": "string (optional)", "callsign": "string (optional)"}, "description": "Alert when ship appears by MMSI, IMO, name, owner, or callsign"},
                    "track_entity": {"params": {"query": "string", "entity_type": "string (optional)", "layers": "list (optional)"}, "description": "Generic exact-first entity tracker when aircraft/ship fields are not known yet"},
                    "geofence": {"params": {"lat": "float", "lng": "float", "radius_km": "float (default 50)", "entity_types": "list (default ['flights','ships'])"}, "description": "Alert when any entity enters a geographic zone"},
-                    "keyword": {"params": {"keyword": "string"}, "description": "Alert when keyword appears in news/GDELT headlines"},
+                    "keyword": {"params": {"keyword": "string", "include_telegram": "boolean (default true)"}, "description": "Alert when keyword appears in news, GDELT, or Telegram OSINT (searches translated + original text)"},
+                    "telegram_rhetoric": {"params": {"min_risk_score": "int 1-10 (default 7)", "keywords": "list or comma-separated string (optional)", "channels": "list or comma-separated string (optional)"}, "description": "Alert on new high-risk Telegram OSINT posts — rhetoric/escalation monitor"},
                    "prediction_market": {"params": {"query": "string", "threshold": "float 0-1 (optional)"}, "description": "Alert on prediction market movements matching query"},
                },
                "example": {"cmd": "add_watch", "args": {"type": "track_registration", "params": {"registration": "N3880"}}},
@@ -2564,7 +2565,8 @@ async def api_capabilities(request: Request):
                        "track_ship": {"params": {"mmsi": "str (optional)", "imo": "str (optional)", "name": "str (optional)", "owner": "str (optional)", "callsign": "str (optional)"}, "description": "Alert when ship appears by MMSI, IMO, name, owner, or callsign"},
                        "track_entity": {"params": {"query": "str", "entity_type": "str (optional)", "layers": "list[str] (optional)"}, "description": "Generic exact-first entity watch"},
                        "geofence": {"params": {"lat": "float", "lng": "float", "radius_km": "float (default 50)", "entity_types": "list (default ['flights','ships'])"}, "description": "Alert when any entity enters a geographic zone"},
-                        "keyword": {"params": {"keyword": "str"}, "description": "Alert when keyword appears in news/GDELT"},
+                        "keyword": {"params": {"keyword": "str", "include_telegram": "bool (default true)"}, "description": "Alert when keyword appears in news, GDELT, or Telegram OSINT"},
+                        "telegram_rhetoric": {"params": {"min_risk_score": "int 1-10 (default 7)", "keywords": "list[str] or comma string (optional)", "channels": "list[str] or comma string (optional)"}, "description": "Alert on new high-risk Telegram OSINT posts"},
                        "prediction_market": {"params": {"query": "str", "threshold": "float 0-1 (optional)"}, "description": "Alert on prediction market movements"},
                    },
                },
@@ -0,0 +1,339 @@
+"""Strategic Risk Analytics API — game-theoretic early warning overlays."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from pydantic import BaseModel, Field
+
+from auth import require_local_operator
+from limiter import limiter
+from analytics.backtest import (
+    DEFAULT_BACKTEST_ALERT_THRESHOLD,
+    run_historical_backtest,
+    tune_alert_threshold,
+)
+from analytics.feed_adapter import normalize_feed_item
+from analytics.integration import get_gt_engine, refresh_from_latest_data
+from analytics.gt_alerts import top_gt_alerts
+from analytics.micro_rolling import micro_rolling_report
+from analytics.rolling_backtest import (
+    freeze_weekly_snapshot,
+    label_region,
+    label_regions,
+    rolling_alert_threshold,
+    rolling_report,
+    score_week,
+)
+from analytics.weekly_store import load_week
+from analytics.settings import gt_analytics_enabled
+from services.fetchers._store import _data_lock, get_latest_data_subset_refs, latest_data
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+class RiskHeatmapRequest(BaseModel):
+    """Optional batch ingest + refresh controls for POST /api/analytics/risk_heatmap."""
+
+    refresh: bool = True
+    items: list[dict[str, Any]] = Field(default_factory=list)
+
+
+class RollingFreezeRequest(BaseModel):
+    week_id: str | None = None
+    force: bool = False
+
+
+class RollingLabelEntry(BaseModel):
+    region: str
+    label: str
+    notes: str = ""
+
+
+class RollingLabelRequest(BaseModel):
+    week_id: str
+    labels: list[RollingLabelEntry] = Field(default_factory=list)
+
+
+def _empty_heatmap() -> dict[str, Any]:
+    return {
+        "enabled": False,
+        "type": "FeatureCollection",
+        "features": [],
+        "clusters": [],
+        "processed": 0,
+        "timestamp": None,
+    }
+
+
+def _gt_risk_payload() -> dict[str, Any]:
+    snap = get_latest_data_subset_refs("gt_risk")
+    payload = snap.get("gt_risk")
+    if not isinstance(payload, dict):
+        return _empty_heatmap()
+    heatmap = payload.get("heatmap") or {"type": "FeatureCollection", "features": []}
+    return {
+        "enabled": bool(payload.get("enabled")),
+        "type": heatmap.get("type", "FeatureCollection"),
+        "features": list(heatmap.get("features") or []),
+        "clusters": list(payload.get("clusters") or []),
+        "processed": int(payload.get("processed") or 0),
+        "timestamp": payload.get("timestamp"),
+    }
+
+
+@router.get("/api/analytics/risk_heatmap")
+@limiter.limit("60/minute")
+async def risk_heatmap_get(request: Request) -> dict[str, Any]:
+    """Return cached GeoJSON risk overlay (posterior scores per region)."""
+    if not gt_analytics_enabled():
+        return _empty_heatmap()
+    return _gt_risk_payload()
+
+
+@router.post("/api/analytics/risk_heatmap")
+@limiter.limit("12/minute")
+async def risk_heatmap_post(
+    request: Request,
+    body: RiskHeatmapRequest,
+    _: None = Depends(require_local_operator),
+) -> dict[str, Any]:
+    """
+    Ingest optional feed items and/or refresh beliefs from latest intel layers.
+
+    Requires local operator auth — intended for OpenClaw agents and admin tooling.
+    """
+    if not gt_analytics_enabled():
+        raise HTTPException(status_code=503, detail="Strategic Risk Analytics is disabled")
+
+    engine = get_gt_engine()
+    if engine is None:
+        raise HTTPException(status_code=503, detail="Strategic Risk Analytics engine unavailable")
+
+    ingested = 0
+    for raw in body.items:
+        if not isinstance(raw, dict):
+            continue
+        source_type = str(raw.get("source_type") or "manual")
+        item = normalize_feed_item(raw, source_type=source_type)
+        result = engine.process_feed_item(item)
+        if result and not result.get("skipped"):
+            ingested += 1
+
+    summary: dict[str, Any] = {"ingested": ingested}
+    if body.refresh:
+        with _data_lock:
+            snapshot = dict(latest_data)
+        summary.update(refresh_from_latest_data(snapshot, persist=True))
+
+    payload = _gt_risk_payload()
+    payload["ingested"] = ingested
+    payload["refresh"] = bool(body.refresh)
+    return payload
+
+
+@router.get("/api/analytics/dossier/{region}")
+@limiter.limit("30/minute")
+async def analytics_dossier(request: Request, region: str) -> dict[str, Any]:
+    """Game-theoretic rationale, recent costly signals, and scenario sketches."""
+    region_key = str(region or "").strip().lower()
+    if not region_key or len(region_key) > 120:
+        raise HTTPException(status_code=400, detail="Invalid region identifier")
+
+    if not gt_analytics_enabled():
+        return {
+            "enabled": False,
+            "region": region_key,
+            "current_risk": 0.0,
+            "interpretation": "Strategic Risk Analytics is disabled.",
+            "recent_signals": [],
+            "scenarios": [],
+        }
+
+    engine = get_gt_engine()
+    if engine is None:
+        raise HTTPException(status_code=503, detail="Strategic Risk Analytics engine unavailable")
+
+    dossier = engine.get_dossier(region_key)
+    dossier["enabled"] = True
+    return dossier
+
+
+@router.get("/api/analytics/backtest")
+@limiter.limit("6/minute")
+async def analytics_backtest(
+    request: Request,
+    expanded: bool = True,
+    tune: bool = False,
+    target_confidence: float = 0.95,
+) -> dict[str, Any]:
+    """
+    Run labeled historical backtest and return accuracy + Wilson 95% CI.
+
+    ``confidence_rate`` is the Wilson lower bound (conservative pass metric).
+    """
+    if not gt_analytics_enabled():
+        return {
+            "enabled": False,
+            "message": "Strategic Risk Analytics is disabled.",
+        }
+
+    if tune:
+        threshold, report = tune_alert_threshold(target_confidence=target_confidence)
+    else:
+        threshold = DEFAULT_BACKTEST_ALERT_THRESHOLD
+        report = run_historical_backtest(
+            use_expanded_suite=expanded,
+            alert_threshold=threshold,
+            target_confidence=target_confidence,
+        )
+
+    payload = report.to_dict()
+    payload["enabled"] = True
+    payload["expanded_suite"] = expanded
+    payload["tuned"] = tune
+    payload["recommended_alert_threshold"] = threshold
+    return payload
+
+
+@router.get("/api/analytics/rolling")
+@limiter.limit("12/minute")
+async def analytics_rolling(
+    request: Request,
+    weeks: int = 8,
+    target_confidence: float = 0.80,
+) -> dict[str, Any]:
+    """Rolling weekly operational validation — accuracy trend with delayed labels."""
+    if not gt_analytics_enabled():
+        return {
+            "enabled": False,
+            "message": "Strategic Risk Analytics is disabled.",
+        }
+
+    report = rolling_report(weeks=max(1, min(weeks, 52)), target_confidence=target_confidence)
+    report["enabled"] = True
+    return report
+
+
+@router.get("/api/analytics/alerts")
+@limiter.limit("30/minute")
+async def analytics_top_alerts(
+    request: Request,
+    limit: int = 8,
+) -> dict[str, Any]:
+    """Top GT risk regions ranked by score — fly-to targets for the map."""
+    if not gt_analytics_enabled():
+        return {
+            "enabled": False,
+            "message": "Strategic Risk Analytics is disabled.",
+        }
+
+    report = top_gt_alerts(limit=max(1, min(limit, 25)))
+    report["enabled"] = True
+    return report
+
+
+@router.get("/api/analytics/rolling/micro")
+@limiter.limit("30/minute")
+async def analytics_rolling_micro(
+    request: Request,
+    window_days: int = 3,
+    limit: int = 15,
+) -> dict[str, Any]:
+    """Rolling 3-day micro average — spot vs baseline, ignition detection."""
+    if not gt_analytics_enabled():
+        return {
+            "enabled": False,
+            "message": "Strategic Risk Analytics is disabled.",
+        }
+
+    report = micro_rolling_report(
+        window_days=max(2, min(window_days, 7)),
+        limit=max(1, min(limit, 50)),
+    )
+    report["enabled"] = True
+    return report
+
+
+@router.get("/api/analytics/rolling/{week_id}")
+@limiter.limit("12/minute")
+async def analytics_rolling_week(request: Request, week_id: str) -> dict[str, Any]:
+    """Return a single frozen week snapshot and its score."""
+    if not gt_analytics_enabled():
+        return {"enabled": False, "message": "Strategic Risk Analytics is disabled."}
+
+    snapshot = load_week(str(week_id).strip())
+    if snapshot is None:
+        raise HTTPException(status_code=404, detail=f"Week {week_id} not found")
+
+    score = score_week(snapshot)
+    return {
+        "enabled": True,
+        "week_id": snapshot.week_id,
+        "snapshot": snapshot.to_dict(),
+        "score": score.to_dict(),
+        "alert_threshold": rolling_alert_threshold(),
+    }
+
+
+@router.post("/api/analytics/rolling/freeze")
+@limiter.limit("6/minute")
+async def analytics_rolling_freeze(
+    request: Request,
+    body: RollingFreezeRequest,
+    _: None = Depends(require_local_operator),
+) -> dict[str, Any]:
+    """Freeze current GT scores for the ISO week (idempotent unless force=true)."""
+    if not gt_analytics_enabled():
+        raise HTTPException(status_code=503, detail="Strategic Risk Analytics is disabled")
+
+    result = freeze_weekly_snapshot(
+        week_id=body.week_id,
+        force=body.force,
+        frozen_by="api",
+    )
+    if not result.get("ok"):
+        raise HTTPException(status_code=503, detail=result.get("detail", "Freeze failed"))
+    result["enabled"] = True
+    return result
+
+
+@router.post("/api/analytics/rolling/label")
+@limiter.limit("12/minute")
+async def analytics_rolling_label(
+    request: Request,
+    body: RollingLabelRequest,
+    _: None = Depends(require_local_operator),
+) -> dict[str, Any]:
+    """Apply delayed outcome labels to a frozen week."""
+    if not gt_analytics_enabled():
+        raise HTTPException(status_code=503, detail="Strategic Risk Analytics is disabled")
+
+    week_id = str(body.week_id or "").strip()
+    if not week_id:
+        raise HTTPException(status_code=400, detail="week_id required")
+
+    if len(body.labels) == 1:
+        entry = body.labels[0]
+        result = label_region(
+            week_id,
+            entry.region,
+            entry.label,  # type: ignore[arg-type]
+            notes=entry.notes,
+            labeled_by="api",
+        )
+    else:
+        result = label_regions(
+            week_id,
+            [row.model_dump() for row in body.labels],
+            labeled_by="api",
+        )
+
+    if not result.get("ok"):
+        raise HTTPException(status_code=404, detail=result.get("detail", "Label failed"))
+    result["enabled"] = True
+    return result
@@ -773,7 +773,7 @@ async def live_data_slow(
        "scanners", "weather_alerts", "ukraine_alerts", "air_quality", "volcanoes",
        "fishing_activity", "psk_reporter", "correlations", "uap_sightings", "wastewater",
        "crowdthreat", "threat_level", "trending_markets", "road_corridor_trends",
-        "malware_threats", "cyber_threats", "scm_suppliers", "telegram_osint",
+        "malware_threats", "cyber_threats", "scm_suppliers", "telegram_osint", "gt_risk",
    )
    freshness = get_source_timestamps_snapshot()
    payload = {
@@ -839,6 +839,12 @@ async def live_data_slow(
        )
        if active_layers.get("telegram_osint", True)
        else {"posts": [], "total": 0, "geolocated": 0},
+        "gt_risk": (
+            d.get("gt_risk")
+            or {"enabled": False, "heatmap": {"type": "FeatureCollection", "features": []}, "clusters": []}
+        )
+        if active_layers.get("gt_risk", False)
+        else {"enabled": False, "heatmap": {"type": "FeatureCollection", "features": []}, "clusters": []},
        "freshness": freshness,
    }
    # Issue #288: bbox filter heavy/dense layers only when all four bounds
@@ -85,6 +85,18 @@ async def health_check(request: Request):
    ):
        top_status = "degraded"

+    runtime: dict = {}
+    try:
+        from services.runtime_profile import get_runtime_profile
+        from analytics.settings import gt_analytics_status
+
+        runtime = {
+            **get_runtime_profile(),
+            "gt_analytics": gt_analytics_status(),
+        }
+    except Exception:
+        runtime = {}
+
    return {
        "status": top_status,
        "version": _get_app_version(),
@@ -108,6 +120,7 @@ async def health_check(request: Request):
        "slo": slo_statuses,
        "slo_summary": slo_summary,
        "ais_proxy": ais_status,
+        "runtime": runtime or None,
    }


@@ -14,6 +14,7 @@ from services.fetchers._store import get_latest_data_subset_refs
 from services.fetchers.telegram_osint import telegram_media_host_allowed
 from services.intel_feeds.country_risk import build_country_risk_payload
 from services.network_utils import outbound_user_agent
+from services.telegram_translate import apply_posts_translations, normalize_translate_target

 logger = logging.getLogger(__name__)

@@ -45,12 +46,19 @@ async def country_risk(request: Request) -> dict:

@router.get("/api/telegram-feed")
@limiter.limit("30/minute")
-async def telegram_feed(request: Request) -> dict:
+async def telegram_feed(request: Request, lang: str | None = Query(default=None)) -> dict:
    snap = get_latest_data_subset_refs("telegram_osint")
    payload = snap.get("telegram_osint")
-    if isinstance(payload, dict) and payload.get("posts") is not None:
-        return payload
-    return {"posts": [], "total": 0, "geolocated": 0, "timestamp": None}
+    if not isinstance(payload, dict) or payload.get("posts") is None:
+        return {"posts": [], "total": 0, "geolocated": 0, "timestamp": None}
+
+    if lang:
+        target = normalize_translate_target(lang)
+        localized = dict(payload)
+        localized["posts"] = apply_posts_translations(list(payload.get("posts") or []), target)
+        localized["translate_locale"] = target
+        return localized
+    return payload


 def _infer_telegram_media_type(target_url: str, content_type: str) -> str: