Feat/gt analytics openclaw (#392)

* feat(telegram): auto-translate OSINT channel posts to English

Cherry-picked from @Bobpick PR #391 (telegram-only slice): server-side translation during fetch, SHOW ORIGINAL toggle in TelegramOsintPopup, and on-demand /api/telegram-feed?lang=.

Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

* feat(gt): experimental Derived OSINT analytics with lean-node safeguards

Cherry-picked from @Bobpick PR #391 (GT + OpenClaw slice): Bayesian strategic-risk engine, map overlay, OpenClaw commands, and telegram_rhetoric watchdog. Off by default (GT_ANALYTICS_ENABLED=false, gt_risk layer false). 1 vCPU nodes get cgroup detection, UI warning on layer toggle, and lean profile that skips scheduled ingest/Louvain unless GT_ANALYTICS_ACK_LOW_CPU=true. Backtest HUD removed from dashboard (OpenClaw/API regression only).

Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

---------

Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Shadowbroker
2026-06-16 17:05:46 -06:00
committed by GitHub
parent 9c5a4054f6
commit cfbeabda1e
69 changed files with 8102 additions and 78 deletions
+71 -2
View File
@@ -499,6 +499,12 @@ def update_slow_data():
latest_data["correlations"] = correlations
except Exception as e:
logger.error("Correlation engine failed: %s", e)
try:
from analytics.integration import maybe_refresh_gt_analytics
maybe_refresh_gt_analytics()
except Exception as e:
logger.error("GT analytics refresh failed: %s", e)
from services.fetchers._store import bump_data_version
bump_data_version()
_save_intel_startup_cache()
@@ -807,8 +813,18 @@ def start_scheduler():
# Telegram OSINT — hourly t.me/s channel scrape (kept off the 5-minute slow tier).
_telegram_interval_m = max(15, int(os.environ.get("TELEGRAM_OSINT_INTERVAL_MINUTES", "60")))
def _fetch_telegram_osint_with_gt():
fetch_telegram_osint()
try:
from analytics.integration import maybe_refresh_gt_analytics
maybe_refresh_gt_analytics()
except Exception as exc:
logger.error("GT analytics refresh after telegram failed: %s", exc)
_scheduler.add_job(
lambda: _run_task_with_health(fetch_telegram_osint, "fetch_telegram_osint"),
lambda: _run_task_with_health(_fetch_telegram_osint_with_gt, "fetch_telegram_osint"),
"interval",
minutes=_telegram_interval_m,
next_run_time=datetime.utcnow() + timedelta(seconds=45),
@@ -934,14 +950,67 @@ def start_scheduler():
)
# GDELT — every 30 minutes (downloads 32 ZIP files per call, avoid rate limits)
def _fetch_gdelt_with_gt():
fetch_gdelt()
try:
from analytics.integration import maybe_refresh_gt_analytics
maybe_refresh_gt_analytics()
except Exception as exc:
logger.error("GT analytics refresh after gdelt failed: %s", exc)
_scheduler.add_job(
lambda: _run_task_with_health_on_executor(_SLOW_EXECUTOR, fetch_gdelt, "fetch_gdelt"),
lambda: _run_task_with_health_on_executor(_SLOW_EXECUTOR, _fetch_gdelt_with_gt, "fetch_gdelt"),
"interval",
minutes=30,
id="gdelt",
max_instances=1,
misfire_grace_time=120,
)
# GT analytics — Louvain herding/coordination clusters (feature-flagged).
def _recompute_gt_clusters():
try:
from analytics.integration import recompute_gt_herding_clusters
recompute_gt_herding_clusters()
except Exception as exc:
logger.error("GT Louvain recompute failed: %s", exc)
def _freeze_gt_weekly_snapshot():
try:
from analytics.integration import maybe_freeze_gt_weekly_snapshot
maybe_freeze_gt_weekly_snapshot()
except Exception as exc:
logger.error("GT rolling weekly freeze failed: %s", exc)
try:
from analytics.settings import get_gt_settings, gt_engine_operational
_gt_settings = get_gt_settings()
if gt_engine_operational():
_scheduler.add_job(
_recompute_gt_clusters,
"interval",
minutes=_gt_settings.louvain_interval_minutes,
id="gt_analytics_louvain",
max_instances=1,
misfire_grace_time=300,
next_run_time=datetime.utcnow() + timedelta(minutes=3),
)
_scheduler.add_job(
_freeze_gt_weekly_snapshot,
"cron",
day_of_week="mon",
hour=0,
minute=5,
id="gt_rolling_weekly_freeze",
max_instances=1,
misfire_grace_time=3600,
)
except Exception as exc:
logger.warning("GT Louvain scheduler not registered: %s", exc)
_scheduler.add_job(
lambda: _run_task_with_health_on_executor(
_SLOW_EXECUTOR, update_liveuamap, "update_liveuamap"
+9
View File
@@ -74,6 +74,7 @@ class DashboardData(TypedDict, total=False):
cyber_threats: Dict[str, Any]
scm_suppliers: Dict[str, Any]
telegram_osint: Dict[str, Any]
gt_risk: Dict[str, Any]
# In-memory store
@@ -129,6 +130,13 @@ latest_data: DashboardData = {
"cyber_threats": {"threats": [], "stats": {}},
"scm_suppliers": {"suppliers": [], "total": 0, "critical_count": 0},
"telegram_osint": {"posts": [], "total": 0, "geolocated": 0, "timestamp": None},
"gt_risk": {
"enabled": False,
"heatmap": {"type": "FeatureCollection", "features": []},
"clusters": [],
"processed": 0,
"timestamp": None,
},
}
# Per-source freshness timestamps
@@ -361,6 +369,7 @@ active_layers: dict[str, bool] = {
"scm_suppliers": False,
"cyber_threats": False,
"telegram_osint": True,
"gt_risk": False,
}
+17 -21
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import hashlib
import html
import logging
import os
import re
@@ -11,6 +12,7 @@ from typing import Any
from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
from services.fetchers.news import resolve_coords_match
from services.network_utils import fetch_with_curl, outbound_user_agent
from services.telegram_translate import apply_post_translation, apply_posts_translations
logger = logging.getLogger(__name__)
@@ -174,13 +176,7 @@ def _extract_media(block: str, link: str) -> dict[str, Any]:
def _strip_html(text: str) -> str:
cleaned = re.sub(r"<br\s*/?>", "\n", text, flags=re.IGNORECASE)
cleaned = re.sub(r"<[^>]+>", "", cleaned)
return (
cleaned.replace("&quot;", '"')
.replace("&amp;", "&")
.replace("&lt;", "<")
.replace("&gt;", ">")
.strip()
)
return html.unescape(cleaned).strip()
def _score_risk(text: str) -> int:
@@ -293,20 +289,19 @@ def parse_telegram_channel_html(html: str, channel: str) -> list[dict[str, Any]]
post_id = hashlib.sha1(f"{link}|{published}".encode("utf-8")).hexdigest()[:16]
media = _extract_media(block, link)
posts.append(
{
"id": post_id,
"title": title,
"description": text[:1200],
"link": link,
"published": published,
"source": f"t.me/{channel}",
"channel": channel,
"risk_score": risk_score,
"coords": [coords[0], coords[1]] if coords else None,
**media,
}
)
post = {
"id": post_id,
"title": title,
"description": text[:1200],
"link": link,
"published": published,
"source": f"t.me/{channel}",
"channel": channel,
"risk_score": risk_score,
"coords": [coords[0], coords[1]] if coords else None,
**media,
}
posts.append(apply_post_translation(post))
return posts
@@ -358,6 +353,7 @@ def fetch_telegram_osint() -> dict[str, Any]:
merged_posts, added = _merge_telegram_posts(existing_posts, incoming)
merged_posts = [_refresh_post_coords(post) for post in merged_posts]
merged_posts = apply_posts_translations(merged_posts)
geolocated = sum(1 for p in merged_posts if p.get("coords"))
payload = {
+288 -1
View File
@@ -90,6 +90,15 @@ READ_COMMANDS = frozenset({
# Agent routing helpers
"route_query",
"run_playbook",
"gt_risk_heatmap",
"gt_dossier",
"gt_analyze",
"gt_backtest",
"gt_rolling_freeze",
"gt_rolling_label",
"gt_rolling_backtest",
"gt_micro_rolling",
"gt_top_alerts",
# Private Infonet reads (operator-delegated)
"infonet_status",
"list_gates",
@@ -857,6 +866,284 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
return {"ok": True, "data": _compact_query_result(result), "format": "compressed_v1"}
return {"ok": True, "data": result}
if cmd == "gt_risk_heatmap":
from analytics.settings import gt_analytics_enabled
from analytics.integration import get_gt_engine
from services.fetchers._store import get_latest_data_subset_refs
if not gt_analytics_enabled():
return {"ok": True, "data": {"enabled": False, "features": [], "clusters": []}}
snap = get_latest_data_subset_refs("gt_risk")
payload = dict(snap.get("gt_risk") or {})
engine = get_gt_engine()
if engine is not None and not payload.get("heatmap"):
payload["heatmap"] = engine.get_risk_heatmap()
return {"ok": True, "data": payload}
if cmd == "gt_dossier":
from analytics.settings import gt_analytics_enabled
from analytics.integration import get_gt_engine
region = str(args.get("region", "") or args.get("area", "") or "").strip().lower()
if not region:
return {"ok": False, "detail": "region required (e.g. ukraine, uk, europe)"}
if not gt_analytics_enabled():
return {
"ok": True,
"data": {
"enabled": False,
"region": region,
"interpretation": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
},
}
engine = get_gt_engine()
if engine is None:
return {"ok": False, "detail": "GT analytics engine unavailable"}
return {"ok": True, "data": engine.get_dossier(region)}
if cmd == "gt_analyze":
from analytics.settings import gt_analytics_enabled
from analytics.integration import get_gt_engine, refresh_from_latest_data
from services.fetchers._store import _data_lock, latest_data
if not gt_analytics_enabled():
return {"ok": False, "detail": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED)"}
engine = get_gt_engine()
if engine is None:
return {"ok": False, "detail": "GT analytics engine unavailable"}
feeds = args.get("feeds") if isinstance(args.get("feeds"), (list, tuple)) else None
if feeds:
from analytics.feed_adapter import normalize_feed_item
ingested = 0
for raw in feeds:
if not isinstance(raw, dict):
continue
item = normalize_feed_item(raw, source_type=str(raw.get("source_type") or "openclaw"))
result = engine.process_feed_item(item)
if result and not result.get("skipped"):
ingested += 1
summary = {"ingested": ingested, "enabled": True}
else:
with _data_lock:
snapshot = dict(latest_data)
summary = refresh_from_latest_data(snapshot, persist=True)
region = str(args.get("region", "") or "").strip().lower()
data = {
"refresh": summary,
"heatmap_features": len((summary.get("sample") or [])),
}
if region:
data["dossier"] = engine.get_dossier(region)
else:
data["heatmap"] = engine.get_risk_heatmap()
data["clusters"] = engine.compute_herding_clusters()[:5]
return {"ok": True, "data": data}
if cmd == "gt_backtest":
from analytics.backtest import (
DEFAULT_BACKTEST_ALERT_THRESHOLD,
run_historical_backtest,
tune_alert_threshold,
)
from analytics.historical_events import default_historical_cases, expanded_historical_cases
from analytics.settings import gt_analytics_enabled
if not gt_analytics_enabled():
return {
"ok": True,
"data": {
"enabled": False,
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
},
}
expanded = bool(args.get("expanded", True))
tune = bool(args.get("tune", False))
include_cases = bool(args.get("include_cases", False))
try:
target_confidence = float(args.get("target_confidence", 0.95))
except (TypeError, ValueError):
target_confidence = 0.95
if tune:
suite = expanded_historical_cases() if expanded else default_historical_cases()
threshold, report = tune_alert_threshold(
suite,
target_confidence=target_confidence,
)
else:
raw_threshold = args.get("alert_threshold")
threshold = (
float(raw_threshold)
if raw_threshold is not None
else DEFAULT_BACKTEST_ALERT_THRESHOLD
)
report = run_historical_backtest(
use_expanded_suite=expanded,
alert_threshold=threshold,
target_confidence=target_confidence,
)
data = report.to_dict()
data["enabled"] = True
data["expanded_suite"] = expanded
data["tuned"] = tune
data["recommended_alert_threshold"] = threshold
if _wants_compact(args) or not include_cases:
data.pop("cases", None)
return {"ok": True, "data": data}
if cmd == "gt_rolling_freeze":
from analytics.rolling_backtest import freeze_weekly_snapshot
from analytics.settings import gt_analytics_enabled
if not gt_analytics_enabled():
return {
"ok": True,
"data": {
"enabled": False,
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
},
}
week_id = str(args.get("week_id", "") or "").strip() or None
force = bool(args.get("force", False))
result = freeze_weekly_snapshot(
week_id=week_id,
force=force,
frozen_by="openclaw",
)
if not result.get("ok"):
return {"ok": False, "detail": result.get("detail", "Freeze failed")}
data = dict(result)
data["enabled"] = True
if _wants_compact(args):
data.pop("snapshot", None)
return {"ok": True, "data": data}
if cmd == "gt_rolling_label":
from analytics.rolling_backtest import label_region, label_regions
from analytics.settings import gt_analytics_enabled
if not gt_analytics_enabled():
return {
"ok": True,
"data": {
"enabled": False,
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
},
}
week_id = str(args.get("week_id", "") or "").strip()
if not week_id:
return {"ok": False, "detail": "week_id required"}
labels = args.get("labels")
if isinstance(labels, list) and labels:
result = label_regions(week_id, labels, labeled_by="openclaw")
else:
region = str(args.get("region", "") or "").strip().lower()
label = str(args.get("label", "") or "").strip().lower()
if not region or not label:
return {"ok": False, "detail": "region and label required (or labels batch)"}
result = label_region(
week_id,
region,
label, # type: ignore[arg-type]
notes=str(args.get("notes", "") or ""),
labeled_by="openclaw",
)
if not result.get("ok"):
return {"ok": False, "detail": result.get("detail", "Label failed")}
data = dict(result)
data["enabled"] = True
return {"ok": True, "data": data}
if cmd == "gt_rolling_backtest":
from analytics.rolling_backtest import rolling_report
from analytics.settings import gt_analytics_enabled
if not gt_analytics_enabled():
return {
"ok": True,
"data": {
"enabled": False,
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
},
}
try:
weeks = int(args.get("weeks", 8))
except (TypeError, ValueError):
weeks = 8
try:
target_confidence = float(args.get("target_confidence", 0.80))
except (TypeError, ValueError):
target_confidence = 0.80
data = rolling_report(weeks=weeks, target_confidence=target_confidence)
data["enabled"] = True
if _wants_compact(args):
for row in data.get("trend") or []:
if isinstance(row, dict):
row.pop("frozen_at", None)
return {"ok": True, "data": data}
if cmd == "gt_top_alerts":
from analytics.gt_alerts import top_gt_alerts
from analytics.settings import gt_analytics_enabled
if not gt_analytics_enabled():
return {
"ok": True,
"data": {
"enabled": False,
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
},
}
try:
limit = int(args.get("limit", 8))
except (TypeError, ValueError):
limit = 8
data = top_gt_alerts(limit=limit)
data["enabled"] = True
return {"ok": True, "data": data}
if cmd == "gt_micro_rolling":
from analytics.micro_rolling import micro_rolling_report
from analytics.settings import gt_analytics_enabled
if not gt_analytics_enabled():
return {
"ok": True,
"data": {
"enabled": False,
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
},
}
try:
window_days = int(args.get("window_days", 3))
except (TypeError, ValueError):
window_days = 3
try:
limit = int(args.get("limit", 15))
except (TypeError, ValueError):
limit = 15
data = micro_rolling_report(window_days=window_days, limit=limit)
data["enabled"] = True
if _wants_compact(args):
data.pop("top_regions", None)
data["ignitions"] = (data.get("ignitions") or [])[:5]
return {"ok": True, "data": data}
if cmd == "brief_area":
from services.telemetry import entities_near, search_news, get_layer_slice
lat = args.get("lat")
@@ -1131,7 +1418,7 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
from services.openclaw_watchdog import add_watch
watch_type = str(args.get("type", "")).strip()
if not watch_type:
return {"ok": False, "detail": "watch type required (track_aircraft, track_callsign, track_registration, track_ship, track_entity, geofence, keyword, prediction_market)"}
return {"ok": False, "detail": "watch type required (track_aircraft, track_callsign, track_registration, track_ship, track_entity, geofence, keyword, telegram_rhetoric, prediction_market)"}
watch_params = args.get("params", {})
if not watch_params:
# Allow flat args (e.g. {type: "track_callsign", callsign: "N189AM"})
+174
View File
@@ -36,6 +36,14 @@ LATENCY_TIER_MS: dict[str, int] = {
"entity_expand": 40,
"osint_lookup": 200,
"run_playbook": 120,
"gt_risk_heatmap": 20,
"gt_dossier": 25,
"gt_analyze": 80,
"gt_backtest": 120,
"gt_rolling_freeze": 30,
"gt_rolling_label": 20,
"gt_rolling_backtest": 30,
"gt_micro_rolling": 20,
"infonet_status": 20,
"list_gates": 15,
"read_gate_messages": 40,
@@ -255,6 +263,32 @@ def _news_query(text: str) -> str:
return cleaned.strip(" ?.")
def _gt_region_hint(text: str) -> str:
lowered = str(text or "").lower()
hints = (
"ukraine",
"middle east",
"eastern europe",
"baltics",
"israel",
"iran",
"russia",
"china",
"europe",
"united kingdom",
"uk",
"usa",
"united states",
)
for hint in hints:
if hint in lowered:
return "uk" if hint == "united kingdom" else hint
match = re.search(r"\bon\s+([a-z][a-z\s]{2,30})\b", lowered)
if match:
return match.group(1).strip()
return ""
def route_query(
text: str = "",
*,
@@ -370,6 +404,146 @@ def route_query(
})
return _route_result("news_search", recommended, avoid, alternates)
if any(
k in lowered
for k in (
"gt backtest",
"backtest gt",
"historical backtest",
"wilson confidence",
"confidence rate",
"gt benchmark",
"validate gt",
)
):
tune = any(k in lowered for k in ("tune", "grid search", "optimize threshold"))
expanded = "base" not in lowered
recommended = {
"cmd": "gt_backtest",
"args": _compact_args(
{
"expanded": expanded,
"tune": tune,
"target_confidence": 0.95,
},
compact=compact,
),
}
alternates.append({"cmd": "gt_risk_heatmap", "args": {}})
return _route_result("gt_backtest", recommended, avoid, alternates)
if any(
k in lowered
for k in (
"rolling backtest",
"rolling validation",
"weekly validation",
"operational validation",
"operational backtest",
"week over week",
"week-over-week",
"gt rolling",
"rolling gt",
"weekly gt",
"weekly gt score",
"gt weekly",
"gt snapshot",
"freeze weekly gt",
)
):
micro = any(
k in lowered
for k in (
"3 day",
"3-day",
"three day",
"micro rolling",
"rolling average",
"ignition",
"micro gt",
)
)
freeze = any(
k in lowered
for k in ("freeze", "gt snapshot", "weekly snapshot", "capture week")
)
label = any(k in lowered for k in ("label", "outcome", "escalation"))
if micro and not freeze and not label:
recommended = {
"cmd": "gt_micro_rolling",
"args": _compact_args({"window_days": 3}, compact=compact),
}
intent = "gt_micro_rolling"
elif freeze:
recommended = {
"cmd": "gt_rolling_freeze",
"args": _compact_args({"force": "force" in lowered}, compact=compact),
}
intent = "gt_rolling_freeze"
elif label:
recommended = {
"cmd": "gt_rolling_label",
"args": _compact_args({}, compact=compact),
}
intent = "gt_rolling_label"
else:
recommended = {
"cmd": "gt_rolling_backtest",
"args": _compact_args({"weeks": 8, "target_confidence": 0.80}, compact=compact),
}
intent = "gt_rolling_backtest"
alternates.append({"cmd": "gt_micro_rolling", "args": {"window_days": 3}})
alternates.append({"cmd": "gt_backtest", "args": {"expanded": True, "compact": True}})
return _route_result(intent, recommended, avoid, alternates)
if any(
k in lowered
for k in (
"3 day average",
"3-day average",
"rolling 3 day",
"micro risk",
"risk ignition",
)
):
recommended = {
"cmd": "gt_micro_rolling",
"args": _compact_args({"window_days": 3}, compact=compact),
}
alternates.append({"cmd": "gt_rolling_backtest", "args": {"weeks": 8}})
return _route_result("gt_micro_rolling", recommended, avoid, alternates)
if any(
k in lowered
for k in (
"gt analysis",
"game theoretic",
"game-theoretic",
"strategic risk",
"early warning",
"risk heatmap",
"costly signal",
"gt rationale",
)
):
region_hint = _gt_region_hint(raw)
if region_hint and any(k in lowered for k in ("dossier", "rationale", "scenario")):
recommended = {
"cmd": "gt_dossier",
"args": _compact_args({"region": region_hint}, compact=compact),
}
alternates.append({"cmd": "gt_risk_heatmap", "args": {}})
return _route_result("gt_dossier", recommended, avoid, alternates)
recommended = {
"cmd": "gt_analyze",
"args": _compact_args(
{"refresh": True, "region": region_hint} if region_hint else {"refresh": True},
compact=compact,
),
}
alternates.append({"cmd": "gt_risk_heatmap", "args": {}})
return _route_result("gt_analyze", recommended, avoid, alternates)
if lat is not None and lng is not None and any(
k in lowered for k in ("near", "around", "within", "radius", "brief", "aoi")
):
+128 -7
View File
@@ -22,9 +22,12 @@ logger = logging.getLogger(__name__)
_lock = threading.Lock()
_watches: dict[str, dict[str, Any]] = {} # watch_id -> watch definition
_fired: dict[str, float] = {} # watch_id -> last fire timestamp (debounce)
_seen_posts: dict[str, set[str]] = {} # watch_id -> seen Telegram post ids/links
_running = False
_stop_event = threading.Event()
_TELEGRAM_SEEN_MAX = 500
# Minimum seconds between re-firing the same watch
DEBOUNCE_S = 60.0
# How often the watchdog checks telemetry
@@ -73,6 +76,7 @@ def remove_watch(watch_id: str) -> dict[str, Any]:
with _lock:
removed = _watches.pop(watch_id, None)
_fired.pop(watch_id, None)
_seen_posts.pop(watch_id, None)
if removed:
return {"ok": True, "removed": removed}
return {"ok": False, "detail": f"watch '{watch_id}' not found"}
@@ -90,6 +94,7 @@ def clear_watches() -> dict[str, Any]:
count = len(_watches)
_watches.clear()
_fired.clear()
_seen_posts.clear()
return {"ok": True, "cleared": count}
@@ -157,7 +162,9 @@ def _check_watch(watch: dict, fast: dict, slow: dict) -> dict[str, Any] | None:
if wtype == "geofence":
return _check_geofence(params, fast)
if wtype == "keyword":
return _check_keyword(params, fast, slow)
return _check_keyword(watch["id"], params, fast, slow)
if wtype == "telegram_rhetoric":
return _check_telegram_rhetoric(watch["id"], params, slow)
if wtype == "prediction_market":
return _check_prediction_market(params, slow)
@@ -390,15 +397,41 @@ def _check_geofence(params: dict, fast: dict) -> dict | None:
return None
def _check_keyword(params: dict, fast: dict, slow: dict) -> dict | None:
"""Alert when a keyword appears in news/GDELT."""
def _telegram_post_id(post: dict[str, Any]) -> str:
return str(post.get("id") or post.get("link") or "").strip()
def _mark_seen_posts(watch_id: str, post_ids: list[str]) -> None:
clean = [pid for pid in post_ids if pid]
if not clean:
return
with _lock:
seen = _seen_posts.setdefault(watch_id, set())
seen.update(clean)
if len(seen) > _TELEGRAM_SEEN_MAX:
_seen_posts[watch_id] = set(list(seen)[-_TELEGRAM_SEEN_MAX:])
def _is_seen_post(watch_id: str, post_id: str) -> bool:
if not post_id:
return False
with _lock:
return post_id in _seen_posts.get(watch_id, set())
def _check_keyword(watch_id: str, params: dict, fast: dict, slow: dict) -> dict | None:
"""Alert when a keyword appears in news, GDELT, or Telegram OSINT."""
keyword = str(params.get("keyword", "")).lower().strip()
if not keyword:
return None
matches = []
include_telegram = params.get("include_telegram", True)
if isinstance(include_telegram, str):
include_telegram = include_telegram.strip().lower() not in {"0", "false", "no", "off"}
matches = []
new_telegram_ids: list[str] = []
# Check news articles
for article in slow.get("news", []):
title = str(article.get("title", "") or "").lower()
desc = str(article.get("description", "") or article.get("summary", "") or "").lower()
@@ -409,7 +442,6 @@ def _check_keyword(params: dict, fast: dict, slow: dict) -> dict | None:
"url": article.get("url") or article.get("link"),
})
# Check GDELT
for event in slow.get("gdelt", []):
text = str(event.get("title", "") or event.get("sourceurl", "") or "").lower()
if keyword in text:
@@ -419,14 +451,103 @@ def _check_keyword(params: dict, fast: dict, slow: dict) -> dict | None:
"url": event.get("sourceurl"),
})
if include_telegram:
from services.telegram_osint_text import (
iter_telegram_posts,
keyword_matches_telegram_post,
telegram_post_match_entry,
)
for post in iter_telegram_posts(slow.get("telegram_osint")):
if not keyword_matches_telegram_post(post, keyword):
continue
post_id = _telegram_post_id(post)
if _is_seen_post(watch_id, post_id):
continue
entry = telegram_post_match_entry(post)
matches.append(entry)
if post_id:
new_telegram_ids.append(post_id)
if matches:
if new_telegram_ids:
_mark_seen_posts(watch_id, new_telegram_ids)
sources = sorted({str(match.get("source") or "unknown") for match in matches})
return {
"alert": f"Keyword '{keyword}' found in {len(matches)} articles",
"alert": f"Keyword '{keyword}' found in {len(matches)} items ({', '.join(sources)})",
"data": {"keyword": keyword, "matches": matches[:10]},
}
return None
def _check_telegram_rhetoric(watch_id: str, params: dict, slow: dict) -> dict | None:
"""Alert on new high-risk Telegram OSINT posts (optionally keyword/channel filtered)."""
min_risk = int(params.get("min_risk_score", 7) or 7)
min_risk = max(1, min(min_risk, 10))
raw_keywords = params.get("keywords") or params.get("keyword") or []
if isinstance(raw_keywords, str):
raw_keywords = [part.strip() for part in raw_keywords.split(",") if part.strip()]
keywords = [str(item).lower().strip() for item in raw_keywords if str(item).strip()]
raw_channels = params.get("channels") or params.get("channel") or []
if isinstance(raw_channels, str):
raw_channels = [part.strip() for part in raw_channels.split(",") if part.strip()]
channels = [str(item).lower().strip().lstrip("@") for item in raw_channels if str(item).strip()]
from services.telegram_osint_text import (
iter_telegram_posts,
keyword_matches_telegram_post,
telegram_post_match_entry,
)
matches = []
new_post_ids: list[str] = []
for post in iter_telegram_posts(slow.get("telegram_osint")):
try:
risk = int(post.get("risk_score") or 0)
except (TypeError, ValueError):
risk = 0
if risk < min_risk:
continue
channel = str(post.get("channel") or "").lower().strip()
source = str(post.get("source") or "").lower().strip()
if channels and channel not in channels and not any(ch in source for ch in channels):
continue
if keywords and not any(keyword_matches_telegram_post(post, kw) for kw in keywords):
continue
post_id = _telegram_post_id(post)
if _is_seen_post(watch_id, post_id):
continue
entry = telegram_post_match_entry(post)
matches.append(entry)
if post_id:
new_post_ids.append(post_id)
if not matches:
return None
_mark_seen_posts(watch_id, new_post_ids)
top = max(int(match.get("risk_score") or 0) for match in matches)
return {
"alert": (
f"Telegram rhetoric alert: {len(matches)} new post(s) at LVL {top}/10"
+ (f" (min {min_risk})" if min_risk > 1 else "")
),
"data": {
"min_risk_score": min_risk,
"keywords": keywords,
"channels": channels,
"matches": matches[:10],
},
}
def _check_prediction_market(params: dict, slow: dict) -> dict | None:
"""Alert on prediction market movements."""
query = str(params.get("query", "")).lower().strip()
+106
View File
@@ -0,0 +1,106 @@
"""Container-aware runtime limits for fleet vs desktop deployments."""
from __future__ import annotations
import os
from functools import lru_cache
from pathlib import Path
from typing import Any
def _read_first_int(path: Path) -> int | None:
try:
raw = path.read_text(encoding="utf-8").strip().split()[0]
return int(raw)
except (OSError, ValueError, IndexError):
return None
def detect_cpu_limit() -> float | None:
"""Effective CPU cores from cgroup quota (Docker ``cpus:``), else host count."""
cgroup_v2 = Path("/sys/fs/cgroup/cpu.max")
if cgroup_v2.is_file():
try:
parts = cgroup_v2.read_text(encoding="utf-8").strip().split()
if len(parts) >= 2 and parts[0] != "max":
quota = int(parts[0])
period = int(parts[1])
if quota > 0 and period > 0:
return round(quota / period, 3)
except (OSError, ValueError):
pass
cgroup_v1_quota = Path("/sys/fs/cgroup/cpu/cpu.cfs_quota_us")
cgroup_v1_period = Path("/sys/fs/cgroup/cpu/cpu.cfs_period_us")
if cgroup_v1_quota.is_file() and cgroup_v1_period.is_file():
quota = _read_first_int(cgroup_v1_quota)
period = _read_first_int(cgroup_v1_period)
if quota is not None and period and quota > 0:
return round(quota / period, 3)
try:
import os as _os
count = _os.cpu_count()
return float(count) if count else None
except Exception:
return None
def detect_memory_limit_mb() -> int | None:
cgroup_v2 = Path("/sys/fs/cgroup/memory.max")
if cgroup_v2.is_file():
try:
raw = cgroup_v2.read_text(encoding="utf-8").strip()
if raw and raw != "max":
return int(int(raw) / (1024 * 1024))
except (OSError, ValueError):
pass
cgroup_v1 = Path("/sys/fs/cgroup/memory/memory.limit_in_bytes")
if cgroup_v1.is_file():
try:
raw = _read_first_int(cgroup_v1)
if raw is not None and raw < (1 << 62):
return int(raw / (1024 * 1024))
except (OSError, ValueError):
pass
return None
def resolve_profile_name() -> str:
explicit = str(os.environ.get("GT_ANALYTICS_PROFILE", "")).strip().lower()
if explicit in {"lean", "standard"}:
return explicit
cpu = detect_cpu_limit()
if cpu is not None and cpu <= 1.0:
return "lean"
return "standard"
@lru_cache(maxsize=1)
def get_runtime_profile() -> dict[str, Any]:
cpu_limit = detect_cpu_limit()
memory_mb = detect_memory_limit_mb()
profile = resolve_profile_name()
lean = profile == "lean"
return {
"profile": profile,
"cpu_limit": cpu_limit,
"memory_limit_mb": memory_mb,
"gt_analytics": {
"recommended": not lean,
"lean_node": lean,
"warning": (
"This node is capped at 1 vCPU. Enabling Strategic Risk (Derived OSINT) "
"may slow Telegram, GDELT, and other OSINT fetches. Set "
"GT_ANALYTICS_ACK_LOW_CPU=true after enabling GT_ANALYTICS_ENABLED to run "
"the full engine on lean hardware."
if lean
else None
),
},
}
def clear_runtime_profile_cache() -> None:
get_runtime_profile.cache_clear()
+1
View File
@@ -19,6 +19,7 @@ class HealthResponse(BaseModel):
# insecure-date path because the upstream Let's Encrypt cert is
# expired. Empty dict / null means no status reported yet.
ais_proxy: Optional[Dict[str, Any]] = None
runtime: Optional[Dict[str, Any]] = None
class RefreshResponse(BaseModel):
+66
View File
@@ -0,0 +1,66 @@
"""Shared Telegram OSINT post text helpers for search and watchdog matching."""
from __future__ import annotations
from typing import Any
from services.telegram_translate import source_lang_label
def iter_telegram_posts(layer_payload: Any) -> list[dict[str, Any]]:
"""Normalize telegram_osint layer payloads into a list of post dicts."""
if isinstance(layer_payload, list):
return [post for post in layer_payload if isinstance(post, dict)]
if isinstance(layer_payload, dict):
posts = layer_payload.get("posts")
if isinstance(posts, list):
return [post for post in posts if isinstance(post, dict)]
return []
def telegram_post_search_text(post: dict[str, Any]) -> str:
"""Build a lowercase haystack for keyword matching (translated + original)."""
parts = (
post.get("title_translated"),
post.get("description_translated"),
post.get("title"),
post.get("description"),
post.get("source"),
post.get("channel"),
)
return " ".join(str(part).strip() for part in parts if str(part or "").strip()).lower()
def telegram_post_display_title(post: dict[str, Any]) -> str:
"""Prefer translated headline for alerts and agent-facing summaries."""
translated = str(post.get("title_translated") or post.get("description_translated") or "").strip()
if translated:
return translated.split("\n", 1)[0][:200]
return str(post.get("title") or post.get("description") or "").strip()[:200]
def telegram_post_match_entry(post: dict[str, Any]) -> dict[str, Any]:
"""Compact match record for watchdog alerts and search results."""
lat, lng = None, None
coords = post.get("coords")
if isinstance(coords, (list, tuple)) and len(coords) >= 2:
lat, lng = coords[0], coords[1]
return {
"source": "telegram_osint",
"title": telegram_post_display_title(post),
"original_title": str(post.get("title") or "").strip(),
"url": post.get("link") or "",
"channel": post.get("channel") or post.get("source") or "",
"risk_score": post.get("risk_score"),
"source_lang": post.get("source_lang"),
"source_lang_label": post.get("source_lang_label") or source_lang_label(post.get("source_lang")),
"lat": lat,
"lng": lng,
"id": post.get("id") or post.get("link") or "",
}
def keyword_matches_telegram_post(post: dict[str, Any], keyword: str) -> bool:
needle = str(keyword or "").strip().lower()
if not needle:
return False
return needle in telegram_post_search_text(post)
+243
View File
@@ -0,0 +1,243 @@
"""Auto-translation for Telegram OSINT post text (server-side, cached)."""
from __future__ import annotations
import hashlib
import logging
import os
import re
import urllib.parse
from threading import Lock
from typing import Any
import requests
logger = logging.getLogger(__name__)
_CYRILLIC_RE = re.compile(r"[\u0400-\u04FF]")
_UKRAINIAN_MARKERS_RE = re.compile(r"[іїєґІЇЄҐ]")
_ARABIC_RE = re.compile(r"[\u0600-\u06FF]")
_HEBREW_RE = re.compile(r"[\u0590-\u05FF]")
_CJK_RE = re.compile(r"[\u4e00-\u9fff]")
# Common war-reporting shorthand that machine translation often transliterates.
_POST_TRANSLATION_GLOSSARY: tuple[tuple[re.Pattern[str], str], ...] = (
(re.compile(r"\bBpLa\b", re.IGNORECASE), "UAV"),
(re.compile(r"\bБпЛА\b", re.IGNORECASE), "UAV"),
(re.compile(r"\bбпла\b"), "UAV"),
(re.compile(r"\bБПЛА\b"), "UAV"),
(re.compile(r"\bрсзв\b", re.IGNORECASE), "MLRS"),
(re.compile(r"\bРСЗВ\b"), "MLRS"),
)
_SOURCE_LANG_LABELS = {
"uk": "Ukrainian",
"ru": "Russian",
"en": "English",
"ar": "Arabic",
"he": "Hebrew",
"zh-cn": "Chinese",
"fr": "French",
"de": "German",
"pl": "Polish",
}
_CACHE: dict[str, tuple[str, str]] = {}
_CACHE_LOCK = Lock()
_CACHE_MAX = 512
_LOCALE_TO_GOOGLE = {
"en": "en",
"fr": "fr",
"zh-cn": "zh-CN",
"zh": "zh-CN",
}
def telegram_translate_enabled() -> bool:
return str(os.environ.get("TELEGRAM_OSINT_TRANSLATE", "true")).strip().lower() not in {
"0",
"false",
"no",
"off",
"",
}
def telegram_translate_target() -> str:
raw = str(os.environ.get("TELEGRAM_OSINT_TRANSLATE_TO", "en")).strip().lower()
return _LOCALE_TO_GOOGLE.get(raw, raw or "en")
def normalize_translate_target(locale: str | None) -> str:
raw = str(locale or telegram_translate_target()).strip().lower().replace("_", "-")
return _LOCALE_TO_GOOGLE.get(raw, raw or "en")
def _looks_english(text: str) -> bool:
letters = [char for char in text if char.isalpha()]
if not letters:
return True
ascii_letters = sum(1 for char in letters if ord(char) < 128)
return ascii_letters / len(letters) > 0.9
def contains_cyrillic(text: str) -> bool:
return bool(_CYRILLIC_RE.search(str(text or "")))
def source_lang_label(code: str | None) -> str:
raw = str(code or "").strip().lower().replace("_", "-")
return _SOURCE_LANG_LABELS.get(raw, raw.upper() if raw else "Unknown")
def polish_translation(text: str) -> str:
polished = str(text or "")
for pattern, replacement in _POST_TRANSLATION_GLOSSARY:
polished = pattern.sub(replacement, polished)
return polished.strip()
def guess_source_lang(text: str) -> str:
if _UKRAINIAN_MARKERS_RE.search(text):
return "uk"
if _CYRILLIC_RE.search(text):
return "ru"
if _ARABIC_RE.search(text):
return "ar"
if _HEBREW_RE.search(text):
return "he"
if _CJK_RE.search(text):
return "zh-CN"
if _looks_english(text):
return "en"
return "auto"
def _cache_key(text: str, target_lang: str) -> str:
digest = hashlib.sha1(f"{target_lang}|{text}".encode("utf-8")).hexdigest()
return digest
def _cache_get(text: str, target_lang: str) -> tuple[str, str] | None:
key = _cache_key(text, target_lang)
with _CACHE_LOCK:
return _CACHE.get(key)
def _cache_put(text: str, target_lang: str, translated: str, source_lang: str) -> None:
key = _cache_key(text, target_lang)
with _CACHE_LOCK:
if len(_CACHE) >= _CACHE_MAX:
_CACHE.pop(next(iter(_CACHE)))
_CACHE[key] = (translated, source_lang)
def _google_translate(clean: str, target: str, source: str | None = None) -> tuple[str, str]:
params = {
"client": "gtx",
"sl": source or "auto",
"tl": target,
"dt": "t",
"q": clean[:4500],
}
url = "https://translate.googleapis.com/translate_a/single?" + urllib.parse.urlencode(params)
resp = requests.get(
url,
timeout=8,
headers={"User-Agent": "Mozilla/5.0 (compatible; Shadowbroker-Telegram-Translate/1.0)"},
)
resp.raise_for_status()
data = resp.json()
detected = str(data[2] or guess_source_lang(clean)).strip().lower()
if detected in {"zh-cn", "zh-tw"}:
detected = "zh-CN"
parts: list[str] = []
for chunk in data[0] or []:
if chunk and chunk[0]:
parts.append(str(chunk[0]))
translated = polish_translation("".join(parts).strip() or clean)
return translated, detected
def translate_text(text: str, target_lang: str | None = None) -> tuple[str, str]:
"""Translate text via Google Translate (unofficial client endpoint).
Returns ``(translated_text, detected_source_lang)``.
"""
clean = str(text or "").strip()
if not clean:
return "", "en"
target = normalize_translate_target(target_lang)
if _looks_english(clean) and target == "en":
return clean, "en"
cached = _cache_get(clean, target)
if cached:
return cached
try:
translated, detected = _google_translate(clean, target)
if detected == target or (detected == "en" and target == "en"):
result = (clean, detected)
_cache_put(clean, target, clean, detected)
return result
if contains_cyrillic(translated) and contains_cyrillic(clean):
hinted = guess_source_lang(clean)
if hinted not in {"auto", target}:
retry_translated, retry_detected = _google_translate(clean, target, hinted)
if not contains_cyrillic(retry_translated) or len(retry_translated) > len(translated):
translated, detected = retry_translated, retry_detected
result = (translated, detected)
_cache_put(clean, target, translated, detected)
return result
except Exception as exc:
logger.warning("Telegram translation failed: %s", exc)
fallback_lang = guess_source_lang(clean)
return clean, fallback_lang
def apply_post_translation(post: dict[str, Any], target_lang: str | None = None) -> dict[str, Any]:
"""Add translation fields to a Telegram OSINT post dict."""
if not telegram_translate_enabled():
return post
target = normalize_translate_target(target_lang)
description = str(post.get("description") or "").strip()
title = str(post.get("title") or "").strip()
full_text = description or title
if not full_text:
return post
existing_translated = str(post.get("description_translated") or post.get("title_translated") or "").strip()
if post.get("translate_to") == target and existing_translated:
updated = dict(post)
polished = polish_translation(existing_translated)
if polished != existing_translated:
lines = polished.split("\n", 1)
updated["title_translated"] = lines[0][:160]
updated["description_translated"] = polished[:1200]
updated["source_lang_label"] = source_lang_label(str(post.get("source_lang") or ""))
return updated
translated_full, source_lang = translate_text(full_text, target)
updated = dict(post)
updated["source_lang"] = source_lang
updated["translate_to"] = target
updated["source_lang_label"] = source_lang_label(source_lang)
if translated_full != full_text and source_lang != target:
lines = translated_full.split("\n", 1)
updated["title_translated"] = lines[0][:160]
updated["description_translated"] = translated_full[:1200]
return updated
def apply_posts_translations(
posts: list[dict[str, Any]],
target_lang: str | None = None,
) -> list[dict[str, Any]]:
if not telegram_translate_enabled():
return posts
return [apply_post_translation(post, target_lang) for post in posts]
+15 -14
View File
@@ -97,6 +97,7 @@ _SLOW_KEYS = (
"cyber_threats",
"scm_suppliers",
"telegram_osint",
"gt_risk",
)
@@ -210,6 +211,9 @@ _LAYER_ALIASES = {
"telegram": "telegram_osint",
"telegram_osint": "telegram_osint",
"osint_feed": "telegram_osint",
"gt_risk": "gt_risk",
"strategic_risk": "gt_risk",
"gt_analytics": "gt_risk",
"malware": "malware_threats",
"malware_threats": "malware_threats",
"malware_c2": "malware_threats",
@@ -710,10 +714,10 @@ _UNIVERSAL_SEARCH_SPECS: dict[str, dict[str, Any]] = {
"time_fields": ("updated_at", "timestamp"),
},
"telegram_osint": {
"fields": ("title", "description", "source", "channel", "link"),
"primary_fields": ("title", "description", "channel"),
"label_fields": ("title", "channel"),
"summary_fields": ("description", "source"),
"fields": ("title", "description", "title_translated", "description_translated", "source", "channel", "link"),
"primary_fields": ("title_translated", "title", "description_translated", "description", "channel"),
"label_fields": ("title_translated", "title", "channel"),
"summary_fields": ("description_translated", "description", "source"),
"type_fields": ("channel", "source"),
"id_fields": ("id", "link"),
"time_fields": ("published", "timestamp"),
@@ -2089,30 +2093,27 @@ def search_news(
return {"results": out, "version": get_data_version(), "truncated": True}
if include_telegram:
from services.telegram_osint_text import telegram_post_display_title, telegram_post_search_text
for post in _unwrap_layer_items(snap.get("telegram_osint"), "telegram_osint"):
if not isinstance(post, dict):
continue
text = " ".join(
(
_norm_text(post.get("title")),
_norm_text(post.get("description")),
_norm_text(post.get("source")),
_norm_text(post.get("channel")),
)
)
text = telegram_post_search_text(post)
if not _text_matches_query(query_norm, text):
continue
lat, lng = _extract_coords(post)
out.append(
{
"source_layer": "telegram_osint",
"title": post.get("title") or "",
"summary": post.get("description") or "",
"title": telegram_post_display_title(post),
"summary": post.get("description_translated") or post.get("description") or "",
"source": post.get("source") or post.get("channel") or "Telegram",
"link": post.get("link") or "",
"lat": lat,
"lng": lng,
"risk_score": post.get("risk_score"),
"source_lang": post.get("source_lang"),
"source_lang_label": post.get("source_lang_label"),
}
)
if len(out) >= limit: