mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-07-07 04:47:58 +02:00
Feat/gt analytics openclaw (#392)
* feat(telegram): auto-translate OSINT channel posts to English Cherry-picked from @Bobpick PR #391 (telegram-only slice): server-side translation during fetch, SHOW ORIGINAL toggle in TelegramOsintPopup, and on-demand /api/telegram-feed?lang=. Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com> Co-authored-by: Cursor <cursoragent@cursor.com> * feat(gt): experimental Derived OSINT analytics with lean-node safeguards Cherry-picked from @Bobpick PR #391 (GT + OpenClaw slice): Bayesian strategic-risk engine, map overlay, OpenClaw commands, and telegram_rhetoric watchdog. Off by default (GT_ANALYTICS_ENABLED=false, gt_risk layer false). 1 vCPU nodes get cgroup detection, UI warning on layer toggle, and lean profile that skips scheduled ingest/Louvain unless GT_ANALYTICS_ACK_LOW_CPU=true. Backtest HUD removed from dashboard (OpenClaw/API regression only). Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com> Co-authored-by: Cursor <cursoragent@cursor.com> --------- Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -499,6 +499,12 @@ def update_slow_data():
|
||||
latest_data["correlations"] = correlations
|
||||
except Exception as e:
|
||||
logger.error("Correlation engine failed: %s", e)
|
||||
try:
|
||||
from analytics.integration import maybe_refresh_gt_analytics
|
||||
|
||||
maybe_refresh_gt_analytics()
|
||||
except Exception as e:
|
||||
logger.error("GT analytics refresh failed: %s", e)
|
||||
from services.fetchers._store import bump_data_version
|
||||
bump_data_version()
|
||||
_save_intel_startup_cache()
|
||||
@@ -807,8 +813,18 @@ def start_scheduler():
|
||||
|
||||
# Telegram OSINT — hourly t.me/s channel scrape (kept off the 5-minute slow tier).
|
||||
_telegram_interval_m = max(15, int(os.environ.get("TELEGRAM_OSINT_INTERVAL_MINUTES", "60")))
|
||||
|
||||
def _fetch_telegram_osint_with_gt():
|
||||
fetch_telegram_osint()
|
||||
try:
|
||||
from analytics.integration import maybe_refresh_gt_analytics
|
||||
|
||||
maybe_refresh_gt_analytics()
|
||||
except Exception as exc:
|
||||
logger.error("GT analytics refresh after telegram failed: %s", exc)
|
||||
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health(fetch_telegram_osint, "fetch_telegram_osint"),
|
||||
lambda: _run_task_with_health(_fetch_telegram_osint_with_gt, "fetch_telegram_osint"),
|
||||
"interval",
|
||||
minutes=_telegram_interval_m,
|
||||
next_run_time=datetime.utcnow() + timedelta(seconds=45),
|
||||
@@ -934,14 +950,67 @@ def start_scheduler():
|
||||
)
|
||||
|
||||
# GDELT — every 30 minutes (downloads 32 ZIP files per call, avoid rate limits)
|
||||
def _fetch_gdelt_with_gt():
|
||||
fetch_gdelt()
|
||||
try:
|
||||
from analytics.integration import maybe_refresh_gt_analytics
|
||||
|
||||
maybe_refresh_gt_analytics()
|
||||
except Exception as exc:
|
||||
logger.error("GT analytics refresh after gdelt failed: %s", exc)
|
||||
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health_on_executor(_SLOW_EXECUTOR, fetch_gdelt, "fetch_gdelt"),
|
||||
lambda: _run_task_with_health_on_executor(_SLOW_EXECUTOR, _fetch_gdelt_with_gt, "fetch_gdelt"),
|
||||
"interval",
|
||||
minutes=30,
|
||||
id="gdelt",
|
||||
max_instances=1,
|
||||
misfire_grace_time=120,
|
||||
)
|
||||
|
||||
# GT analytics — Louvain herding/coordination clusters (feature-flagged).
|
||||
def _recompute_gt_clusters():
|
||||
try:
|
||||
from analytics.integration import recompute_gt_herding_clusters
|
||||
|
||||
recompute_gt_herding_clusters()
|
||||
except Exception as exc:
|
||||
logger.error("GT Louvain recompute failed: %s", exc)
|
||||
|
||||
def _freeze_gt_weekly_snapshot():
|
||||
try:
|
||||
from analytics.integration import maybe_freeze_gt_weekly_snapshot
|
||||
|
||||
maybe_freeze_gt_weekly_snapshot()
|
||||
except Exception as exc:
|
||||
logger.error("GT rolling weekly freeze failed: %s", exc)
|
||||
|
||||
try:
|
||||
from analytics.settings import get_gt_settings, gt_engine_operational
|
||||
|
||||
_gt_settings = get_gt_settings()
|
||||
if gt_engine_operational():
|
||||
_scheduler.add_job(
|
||||
_recompute_gt_clusters,
|
||||
"interval",
|
||||
minutes=_gt_settings.louvain_interval_minutes,
|
||||
id="gt_analytics_louvain",
|
||||
max_instances=1,
|
||||
misfire_grace_time=300,
|
||||
next_run_time=datetime.utcnow() + timedelta(minutes=3),
|
||||
)
|
||||
_scheduler.add_job(
|
||||
_freeze_gt_weekly_snapshot,
|
||||
"cron",
|
||||
day_of_week="mon",
|
||||
hour=0,
|
||||
minute=5,
|
||||
id="gt_rolling_weekly_freeze",
|
||||
max_instances=1,
|
||||
misfire_grace_time=3600,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("GT Louvain scheduler not registered: %s", exc)
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health_on_executor(
|
||||
_SLOW_EXECUTOR, update_liveuamap, "update_liveuamap"
|
||||
|
||||
@@ -74,6 +74,7 @@ class DashboardData(TypedDict, total=False):
|
||||
cyber_threats: Dict[str, Any]
|
||||
scm_suppliers: Dict[str, Any]
|
||||
telegram_osint: Dict[str, Any]
|
||||
gt_risk: Dict[str, Any]
|
||||
|
||||
|
||||
# In-memory store
|
||||
@@ -129,6 +130,13 @@ latest_data: DashboardData = {
|
||||
"cyber_threats": {"threats": [], "stats": {}},
|
||||
"scm_suppliers": {"suppliers": [], "total": 0, "critical_count": 0},
|
||||
"telegram_osint": {"posts": [], "total": 0, "geolocated": 0, "timestamp": None},
|
||||
"gt_risk": {
|
||||
"enabled": False,
|
||||
"heatmap": {"type": "FeatureCollection", "features": []},
|
||||
"clusters": [],
|
||||
"processed": 0,
|
||||
"timestamp": None,
|
||||
},
|
||||
}
|
||||
|
||||
# Per-source freshness timestamps
|
||||
@@ -361,6 +369,7 @@ active_layers: dict[str, bool] = {
|
||||
"scm_suppliers": False,
|
||||
"cyber_threats": False,
|
||||
"telegram_osint": True,
|
||||
"gt_risk": False,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import html
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
@@ -11,6 +12,7 @@ from typing import Any
|
||||
from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
|
||||
from services.fetchers.news import resolve_coords_match
|
||||
from services.network_utils import fetch_with_curl, outbound_user_agent
|
||||
from services.telegram_translate import apply_post_translation, apply_posts_translations
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -174,13 +176,7 @@ def _extract_media(block: str, link: str) -> dict[str, Any]:
|
||||
def _strip_html(text: str) -> str:
|
||||
cleaned = re.sub(r"<br\s*/?>", "\n", text, flags=re.IGNORECASE)
|
||||
cleaned = re.sub(r"<[^>]+>", "", cleaned)
|
||||
return (
|
||||
cleaned.replace(""", '"')
|
||||
.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.strip()
|
||||
)
|
||||
return html.unescape(cleaned).strip()
|
||||
|
||||
|
||||
def _score_risk(text: str) -> int:
|
||||
@@ -293,20 +289,19 @@ def parse_telegram_channel_html(html: str, channel: str) -> list[dict[str, Any]]
|
||||
post_id = hashlib.sha1(f"{link}|{published}".encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
media = _extract_media(block, link)
|
||||
posts.append(
|
||||
{
|
||||
"id": post_id,
|
||||
"title": title,
|
||||
"description": text[:1200],
|
||||
"link": link,
|
||||
"published": published,
|
||||
"source": f"t.me/{channel}",
|
||||
"channel": channel,
|
||||
"risk_score": risk_score,
|
||||
"coords": [coords[0], coords[1]] if coords else None,
|
||||
**media,
|
||||
}
|
||||
)
|
||||
post = {
|
||||
"id": post_id,
|
||||
"title": title,
|
||||
"description": text[:1200],
|
||||
"link": link,
|
||||
"published": published,
|
||||
"source": f"t.me/{channel}",
|
||||
"channel": channel,
|
||||
"risk_score": risk_score,
|
||||
"coords": [coords[0], coords[1]] if coords else None,
|
||||
**media,
|
||||
}
|
||||
posts.append(apply_post_translation(post))
|
||||
return posts
|
||||
|
||||
|
||||
@@ -358,6 +353,7 @@ def fetch_telegram_osint() -> dict[str, Any]:
|
||||
|
||||
merged_posts, added = _merge_telegram_posts(existing_posts, incoming)
|
||||
merged_posts = [_refresh_post_coords(post) for post in merged_posts]
|
||||
merged_posts = apply_posts_translations(merged_posts)
|
||||
geolocated = sum(1 for p in merged_posts if p.get("coords"))
|
||||
|
||||
payload = {
|
||||
|
||||
@@ -90,6 +90,15 @@ READ_COMMANDS = frozenset({
|
||||
# Agent routing helpers
|
||||
"route_query",
|
||||
"run_playbook",
|
||||
"gt_risk_heatmap",
|
||||
"gt_dossier",
|
||||
"gt_analyze",
|
||||
"gt_backtest",
|
||||
"gt_rolling_freeze",
|
||||
"gt_rolling_label",
|
||||
"gt_rolling_backtest",
|
||||
"gt_micro_rolling",
|
||||
"gt_top_alerts",
|
||||
# Private Infonet reads (operator-delegated)
|
||||
"infonet_status",
|
||||
"list_gates",
|
||||
@@ -857,6 +866,284 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
|
||||
return {"ok": True, "data": _compact_query_result(result), "format": "compressed_v1"}
|
||||
return {"ok": True, "data": result}
|
||||
|
||||
if cmd == "gt_risk_heatmap":
|
||||
from analytics.settings import gt_analytics_enabled
|
||||
from analytics.integration import get_gt_engine
|
||||
from services.fetchers._store import get_latest_data_subset_refs
|
||||
|
||||
if not gt_analytics_enabled():
|
||||
return {"ok": True, "data": {"enabled": False, "features": [], "clusters": []}}
|
||||
snap = get_latest_data_subset_refs("gt_risk")
|
||||
payload = dict(snap.get("gt_risk") or {})
|
||||
engine = get_gt_engine()
|
||||
if engine is not None and not payload.get("heatmap"):
|
||||
payload["heatmap"] = engine.get_risk_heatmap()
|
||||
return {"ok": True, "data": payload}
|
||||
|
||||
if cmd == "gt_dossier":
|
||||
from analytics.settings import gt_analytics_enabled
|
||||
from analytics.integration import get_gt_engine
|
||||
|
||||
region = str(args.get("region", "") or args.get("area", "") or "").strip().lower()
|
||||
if not region:
|
||||
return {"ok": False, "detail": "region required (e.g. ukraine, uk, europe)"}
|
||||
if not gt_analytics_enabled():
|
||||
return {
|
||||
"ok": True,
|
||||
"data": {
|
||||
"enabled": False,
|
||||
"region": region,
|
||||
"interpretation": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
|
||||
},
|
||||
}
|
||||
engine = get_gt_engine()
|
||||
if engine is None:
|
||||
return {"ok": False, "detail": "GT analytics engine unavailable"}
|
||||
return {"ok": True, "data": engine.get_dossier(region)}
|
||||
|
||||
if cmd == "gt_analyze":
|
||||
from analytics.settings import gt_analytics_enabled
|
||||
from analytics.integration import get_gt_engine, refresh_from_latest_data
|
||||
from services.fetchers._store import _data_lock, latest_data
|
||||
|
||||
if not gt_analytics_enabled():
|
||||
return {"ok": False, "detail": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED)"}
|
||||
engine = get_gt_engine()
|
||||
if engine is None:
|
||||
return {"ok": False, "detail": "GT analytics engine unavailable"}
|
||||
|
||||
feeds = args.get("feeds") if isinstance(args.get("feeds"), (list, tuple)) else None
|
||||
if feeds:
|
||||
from analytics.feed_adapter import normalize_feed_item
|
||||
|
||||
ingested = 0
|
||||
for raw in feeds:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
item = normalize_feed_item(raw, source_type=str(raw.get("source_type") or "openclaw"))
|
||||
result = engine.process_feed_item(item)
|
||||
if result and not result.get("skipped"):
|
||||
ingested += 1
|
||||
summary = {"ingested": ingested, "enabled": True}
|
||||
else:
|
||||
with _data_lock:
|
||||
snapshot = dict(latest_data)
|
||||
summary = refresh_from_latest_data(snapshot, persist=True)
|
||||
|
||||
region = str(args.get("region", "") or "").strip().lower()
|
||||
data = {
|
||||
"refresh": summary,
|
||||
"heatmap_features": len((summary.get("sample") or [])),
|
||||
}
|
||||
if region:
|
||||
data["dossier"] = engine.get_dossier(region)
|
||||
else:
|
||||
data["heatmap"] = engine.get_risk_heatmap()
|
||||
data["clusters"] = engine.compute_herding_clusters()[:5]
|
||||
return {"ok": True, "data": data}
|
||||
|
||||
if cmd == "gt_backtest":
|
||||
from analytics.backtest import (
|
||||
DEFAULT_BACKTEST_ALERT_THRESHOLD,
|
||||
run_historical_backtest,
|
||||
tune_alert_threshold,
|
||||
)
|
||||
from analytics.historical_events import default_historical_cases, expanded_historical_cases
|
||||
from analytics.settings import gt_analytics_enabled
|
||||
|
||||
if not gt_analytics_enabled():
|
||||
return {
|
||||
"ok": True,
|
||||
"data": {
|
||||
"enabled": False,
|
||||
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
|
||||
},
|
||||
}
|
||||
|
||||
expanded = bool(args.get("expanded", True))
|
||||
tune = bool(args.get("tune", False))
|
||||
include_cases = bool(args.get("include_cases", False))
|
||||
try:
|
||||
target_confidence = float(args.get("target_confidence", 0.95))
|
||||
except (TypeError, ValueError):
|
||||
target_confidence = 0.95
|
||||
|
||||
if tune:
|
||||
suite = expanded_historical_cases() if expanded else default_historical_cases()
|
||||
threshold, report = tune_alert_threshold(
|
||||
suite,
|
||||
target_confidence=target_confidence,
|
||||
)
|
||||
else:
|
||||
raw_threshold = args.get("alert_threshold")
|
||||
threshold = (
|
||||
float(raw_threshold)
|
||||
if raw_threshold is not None
|
||||
else DEFAULT_BACKTEST_ALERT_THRESHOLD
|
||||
)
|
||||
report = run_historical_backtest(
|
||||
use_expanded_suite=expanded,
|
||||
alert_threshold=threshold,
|
||||
target_confidence=target_confidence,
|
||||
)
|
||||
|
||||
data = report.to_dict()
|
||||
data["enabled"] = True
|
||||
data["expanded_suite"] = expanded
|
||||
data["tuned"] = tune
|
||||
data["recommended_alert_threshold"] = threshold
|
||||
if _wants_compact(args) or not include_cases:
|
||||
data.pop("cases", None)
|
||||
return {"ok": True, "data": data}
|
||||
|
||||
if cmd == "gt_rolling_freeze":
|
||||
from analytics.rolling_backtest import freeze_weekly_snapshot
|
||||
from analytics.settings import gt_analytics_enabled
|
||||
|
||||
if not gt_analytics_enabled():
|
||||
return {
|
||||
"ok": True,
|
||||
"data": {
|
||||
"enabled": False,
|
||||
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
|
||||
},
|
||||
}
|
||||
|
||||
week_id = str(args.get("week_id", "") or "").strip() or None
|
||||
force = bool(args.get("force", False))
|
||||
result = freeze_weekly_snapshot(
|
||||
week_id=week_id,
|
||||
force=force,
|
||||
frozen_by="openclaw",
|
||||
)
|
||||
if not result.get("ok"):
|
||||
return {"ok": False, "detail": result.get("detail", "Freeze failed")}
|
||||
data = dict(result)
|
||||
data["enabled"] = True
|
||||
if _wants_compact(args):
|
||||
data.pop("snapshot", None)
|
||||
return {"ok": True, "data": data}
|
||||
|
||||
if cmd == "gt_rolling_label":
|
||||
from analytics.rolling_backtest import label_region, label_regions
|
||||
from analytics.settings import gt_analytics_enabled
|
||||
|
||||
if not gt_analytics_enabled():
|
||||
return {
|
||||
"ok": True,
|
||||
"data": {
|
||||
"enabled": False,
|
||||
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
|
||||
},
|
||||
}
|
||||
|
||||
week_id = str(args.get("week_id", "") or "").strip()
|
||||
if not week_id:
|
||||
return {"ok": False, "detail": "week_id required"}
|
||||
|
||||
labels = args.get("labels")
|
||||
if isinstance(labels, list) and labels:
|
||||
result = label_regions(week_id, labels, labeled_by="openclaw")
|
||||
else:
|
||||
region = str(args.get("region", "") or "").strip().lower()
|
||||
label = str(args.get("label", "") or "").strip().lower()
|
||||
if not region or not label:
|
||||
return {"ok": False, "detail": "region and label required (or labels batch)"}
|
||||
result = label_region(
|
||||
week_id,
|
||||
region,
|
||||
label, # type: ignore[arg-type]
|
||||
notes=str(args.get("notes", "") or ""),
|
||||
labeled_by="openclaw",
|
||||
)
|
||||
|
||||
if not result.get("ok"):
|
||||
return {"ok": False, "detail": result.get("detail", "Label failed")}
|
||||
data = dict(result)
|
||||
data["enabled"] = True
|
||||
return {"ok": True, "data": data}
|
||||
|
||||
if cmd == "gt_rolling_backtest":
|
||||
from analytics.rolling_backtest import rolling_report
|
||||
from analytics.settings import gt_analytics_enabled
|
||||
|
||||
if not gt_analytics_enabled():
|
||||
return {
|
||||
"ok": True,
|
||||
"data": {
|
||||
"enabled": False,
|
||||
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
|
||||
},
|
||||
}
|
||||
|
||||
try:
|
||||
weeks = int(args.get("weeks", 8))
|
||||
except (TypeError, ValueError):
|
||||
weeks = 8
|
||||
try:
|
||||
target_confidence = float(args.get("target_confidence", 0.80))
|
||||
except (TypeError, ValueError):
|
||||
target_confidence = 0.80
|
||||
|
||||
data = rolling_report(weeks=weeks, target_confidence=target_confidence)
|
||||
data["enabled"] = True
|
||||
if _wants_compact(args):
|
||||
for row in data.get("trend") or []:
|
||||
if isinstance(row, dict):
|
||||
row.pop("frozen_at", None)
|
||||
return {"ok": True, "data": data}
|
||||
|
||||
if cmd == "gt_top_alerts":
|
||||
from analytics.gt_alerts import top_gt_alerts
|
||||
from analytics.settings import gt_analytics_enabled
|
||||
|
||||
if not gt_analytics_enabled():
|
||||
return {
|
||||
"ok": True,
|
||||
"data": {
|
||||
"enabled": False,
|
||||
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
|
||||
},
|
||||
}
|
||||
|
||||
try:
|
||||
limit = int(args.get("limit", 8))
|
||||
except (TypeError, ValueError):
|
||||
limit = 8
|
||||
|
||||
data = top_gt_alerts(limit=limit)
|
||||
data["enabled"] = True
|
||||
return {"ok": True, "data": data}
|
||||
|
||||
if cmd == "gt_micro_rolling":
|
||||
from analytics.micro_rolling import micro_rolling_report
|
||||
from analytics.settings import gt_analytics_enabled
|
||||
|
||||
if not gt_analytics_enabled():
|
||||
return {
|
||||
"ok": True,
|
||||
"data": {
|
||||
"enabled": False,
|
||||
"message": "Strategic Risk Analytics is disabled (GT_ANALYTICS_ENABLED).",
|
||||
},
|
||||
}
|
||||
|
||||
try:
|
||||
window_days = int(args.get("window_days", 3))
|
||||
except (TypeError, ValueError):
|
||||
window_days = 3
|
||||
try:
|
||||
limit = int(args.get("limit", 15))
|
||||
except (TypeError, ValueError):
|
||||
limit = 15
|
||||
|
||||
data = micro_rolling_report(window_days=window_days, limit=limit)
|
||||
data["enabled"] = True
|
||||
if _wants_compact(args):
|
||||
data.pop("top_regions", None)
|
||||
data["ignitions"] = (data.get("ignitions") or [])[:5]
|
||||
return {"ok": True, "data": data}
|
||||
|
||||
if cmd == "brief_area":
|
||||
from services.telemetry import entities_near, search_news, get_layer_slice
|
||||
lat = args.get("lat")
|
||||
@@ -1131,7 +1418,7 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
|
||||
from services.openclaw_watchdog import add_watch
|
||||
watch_type = str(args.get("type", "")).strip()
|
||||
if not watch_type:
|
||||
return {"ok": False, "detail": "watch type required (track_aircraft, track_callsign, track_registration, track_ship, track_entity, geofence, keyword, prediction_market)"}
|
||||
return {"ok": False, "detail": "watch type required (track_aircraft, track_callsign, track_registration, track_ship, track_entity, geofence, keyword, telegram_rhetoric, prediction_market)"}
|
||||
watch_params = args.get("params", {})
|
||||
if not watch_params:
|
||||
# Allow flat args (e.g. {type: "track_callsign", callsign: "N189AM"})
|
||||
|
||||
@@ -36,6 +36,14 @@ LATENCY_TIER_MS: dict[str, int] = {
|
||||
"entity_expand": 40,
|
||||
"osint_lookup": 200,
|
||||
"run_playbook": 120,
|
||||
"gt_risk_heatmap": 20,
|
||||
"gt_dossier": 25,
|
||||
"gt_analyze": 80,
|
||||
"gt_backtest": 120,
|
||||
"gt_rolling_freeze": 30,
|
||||
"gt_rolling_label": 20,
|
||||
"gt_rolling_backtest": 30,
|
||||
"gt_micro_rolling": 20,
|
||||
"infonet_status": 20,
|
||||
"list_gates": 15,
|
||||
"read_gate_messages": 40,
|
||||
@@ -255,6 +263,32 @@ def _news_query(text: str) -> str:
|
||||
return cleaned.strip(" ?.")
|
||||
|
||||
|
||||
def _gt_region_hint(text: str) -> str:
|
||||
lowered = str(text or "").lower()
|
||||
hints = (
|
||||
"ukraine",
|
||||
"middle east",
|
||||
"eastern europe",
|
||||
"baltics",
|
||||
"israel",
|
||||
"iran",
|
||||
"russia",
|
||||
"china",
|
||||
"europe",
|
||||
"united kingdom",
|
||||
"uk",
|
||||
"usa",
|
||||
"united states",
|
||||
)
|
||||
for hint in hints:
|
||||
if hint in lowered:
|
||||
return "uk" if hint == "united kingdom" else hint
|
||||
match = re.search(r"\bon\s+([a-z][a-z\s]{2,30})\b", lowered)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def route_query(
|
||||
text: str = "",
|
||||
*,
|
||||
@@ -370,6 +404,146 @@ def route_query(
|
||||
})
|
||||
return _route_result("news_search", recommended, avoid, alternates)
|
||||
|
||||
if any(
|
||||
k in lowered
|
||||
for k in (
|
||||
"gt backtest",
|
||||
"backtest gt",
|
||||
"historical backtest",
|
||||
"wilson confidence",
|
||||
"confidence rate",
|
||||
"gt benchmark",
|
||||
"validate gt",
|
||||
)
|
||||
):
|
||||
tune = any(k in lowered for k in ("tune", "grid search", "optimize threshold"))
|
||||
expanded = "base" not in lowered
|
||||
recommended = {
|
||||
"cmd": "gt_backtest",
|
||||
"args": _compact_args(
|
||||
{
|
||||
"expanded": expanded,
|
||||
"tune": tune,
|
||||
"target_confidence": 0.95,
|
||||
},
|
||||
compact=compact,
|
||||
),
|
||||
}
|
||||
alternates.append({"cmd": "gt_risk_heatmap", "args": {}})
|
||||
return _route_result("gt_backtest", recommended, avoid, alternates)
|
||||
|
||||
if any(
|
||||
k in lowered
|
||||
for k in (
|
||||
"rolling backtest",
|
||||
"rolling validation",
|
||||
"weekly validation",
|
||||
"operational validation",
|
||||
"operational backtest",
|
||||
"week over week",
|
||||
"week-over-week",
|
||||
"gt rolling",
|
||||
"rolling gt",
|
||||
"weekly gt",
|
||||
"weekly gt score",
|
||||
"gt weekly",
|
||||
"gt snapshot",
|
||||
"freeze weekly gt",
|
||||
)
|
||||
):
|
||||
micro = any(
|
||||
k in lowered
|
||||
for k in (
|
||||
"3 day",
|
||||
"3-day",
|
||||
"three day",
|
||||
"micro rolling",
|
||||
"rolling average",
|
||||
"ignition",
|
||||
"micro gt",
|
||||
)
|
||||
)
|
||||
freeze = any(
|
||||
k in lowered
|
||||
for k in ("freeze", "gt snapshot", "weekly snapshot", "capture week")
|
||||
)
|
||||
label = any(k in lowered for k in ("label", "outcome", "escalation"))
|
||||
if micro and not freeze and not label:
|
||||
recommended = {
|
||||
"cmd": "gt_micro_rolling",
|
||||
"args": _compact_args({"window_days": 3}, compact=compact),
|
||||
}
|
||||
intent = "gt_micro_rolling"
|
||||
elif freeze:
|
||||
recommended = {
|
||||
"cmd": "gt_rolling_freeze",
|
||||
"args": _compact_args({"force": "force" in lowered}, compact=compact),
|
||||
}
|
||||
intent = "gt_rolling_freeze"
|
||||
elif label:
|
||||
recommended = {
|
||||
"cmd": "gt_rolling_label",
|
||||
"args": _compact_args({}, compact=compact),
|
||||
}
|
||||
intent = "gt_rolling_label"
|
||||
else:
|
||||
recommended = {
|
||||
"cmd": "gt_rolling_backtest",
|
||||
"args": _compact_args({"weeks": 8, "target_confidence": 0.80}, compact=compact),
|
||||
}
|
||||
intent = "gt_rolling_backtest"
|
||||
alternates.append({"cmd": "gt_micro_rolling", "args": {"window_days": 3}})
|
||||
alternates.append({"cmd": "gt_backtest", "args": {"expanded": True, "compact": True}})
|
||||
return _route_result(intent, recommended, avoid, alternates)
|
||||
|
||||
if any(
|
||||
k in lowered
|
||||
for k in (
|
||||
"3 day average",
|
||||
"3-day average",
|
||||
"rolling 3 day",
|
||||
"micro risk",
|
||||
"risk ignition",
|
||||
)
|
||||
):
|
||||
recommended = {
|
||||
"cmd": "gt_micro_rolling",
|
||||
"args": _compact_args({"window_days": 3}, compact=compact),
|
||||
}
|
||||
alternates.append({"cmd": "gt_rolling_backtest", "args": {"weeks": 8}})
|
||||
return _route_result("gt_micro_rolling", recommended, avoid, alternates)
|
||||
|
||||
if any(
|
||||
k in lowered
|
||||
for k in (
|
||||
"gt analysis",
|
||||
"game theoretic",
|
||||
"game-theoretic",
|
||||
"strategic risk",
|
||||
"early warning",
|
||||
"risk heatmap",
|
||||
"costly signal",
|
||||
"gt rationale",
|
||||
)
|
||||
):
|
||||
region_hint = _gt_region_hint(raw)
|
||||
if region_hint and any(k in lowered for k in ("dossier", "rationale", "scenario")):
|
||||
recommended = {
|
||||
"cmd": "gt_dossier",
|
||||
"args": _compact_args({"region": region_hint}, compact=compact),
|
||||
}
|
||||
alternates.append({"cmd": "gt_risk_heatmap", "args": {}})
|
||||
return _route_result("gt_dossier", recommended, avoid, alternates)
|
||||
recommended = {
|
||||
"cmd": "gt_analyze",
|
||||
"args": _compact_args(
|
||||
{"refresh": True, "region": region_hint} if region_hint else {"refresh": True},
|
||||
compact=compact,
|
||||
),
|
||||
}
|
||||
alternates.append({"cmd": "gt_risk_heatmap", "args": {}})
|
||||
return _route_result("gt_analyze", recommended, avoid, alternates)
|
||||
|
||||
if lat is not None and lng is not None and any(
|
||||
k in lowered for k in ("near", "around", "within", "radius", "brief", "aoi")
|
||||
):
|
||||
|
||||
@@ -22,9 +22,12 @@ logger = logging.getLogger(__name__)
|
||||
_lock = threading.Lock()
|
||||
_watches: dict[str, dict[str, Any]] = {} # watch_id -> watch definition
|
||||
_fired: dict[str, float] = {} # watch_id -> last fire timestamp (debounce)
|
||||
_seen_posts: dict[str, set[str]] = {} # watch_id -> seen Telegram post ids/links
|
||||
_running = False
|
||||
_stop_event = threading.Event()
|
||||
|
||||
_TELEGRAM_SEEN_MAX = 500
|
||||
|
||||
# Minimum seconds between re-firing the same watch
|
||||
DEBOUNCE_S = 60.0
|
||||
# How often the watchdog checks telemetry
|
||||
@@ -73,6 +76,7 @@ def remove_watch(watch_id: str) -> dict[str, Any]:
|
||||
with _lock:
|
||||
removed = _watches.pop(watch_id, None)
|
||||
_fired.pop(watch_id, None)
|
||||
_seen_posts.pop(watch_id, None)
|
||||
if removed:
|
||||
return {"ok": True, "removed": removed}
|
||||
return {"ok": False, "detail": f"watch '{watch_id}' not found"}
|
||||
@@ -90,6 +94,7 @@ def clear_watches() -> dict[str, Any]:
|
||||
count = len(_watches)
|
||||
_watches.clear()
|
||||
_fired.clear()
|
||||
_seen_posts.clear()
|
||||
return {"ok": True, "cleared": count}
|
||||
|
||||
|
||||
@@ -157,7 +162,9 @@ def _check_watch(watch: dict, fast: dict, slow: dict) -> dict[str, Any] | None:
|
||||
if wtype == "geofence":
|
||||
return _check_geofence(params, fast)
|
||||
if wtype == "keyword":
|
||||
return _check_keyword(params, fast, slow)
|
||||
return _check_keyword(watch["id"], params, fast, slow)
|
||||
if wtype == "telegram_rhetoric":
|
||||
return _check_telegram_rhetoric(watch["id"], params, slow)
|
||||
if wtype == "prediction_market":
|
||||
return _check_prediction_market(params, slow)
|
||||
|
||||
@@ -390,15 +397,41 @@ def _check_geofence(params: dict, fast: dict) -> dict | None:
|
||||
return None
|
||||
|
||||
|
||||
def _check_keyword(params: dict, fast: dict, slow: dict) -> dict | None:
|
||||
"""Alert when a keyword appears in news/GDELT."""
|
||||
def _telegram_post_id(post: dict[str, Any]) -> str:
|
||||
return str(post.get("id") or post.get("link") or "").strip()
|
||||
|
||||
|
||||
def _mark_seen_posts(watch_id: str, post_ids: list[str]) -> None:
|
||||
clean = [pid for pid in post_ids if pid]
|
||||
if not clean:
|
||||
return
|
||||
with _lock:
|
||||
seen = _seen_posts.setdefault(watch_id, set())
|
||||
seen.update(clean)
|
||||
if len(seen) > _TELEGRAM_SEEN_MAX:
|
||||
_seen_posts[watch_id] = set(list(seen)[-_TELEGRAM_SEEN_MAX:])
|
||||
|
||||
|
||||
def _is_seen_post(watch_id: str, post_id: str) -> bool:
|
||||
if not post_id:
|
||||
return False
|
||||
with _lock:
|
||||
return post_id in _seen_posts.get(watch_id, set())
|
||||
|
||||
|
||||
def _check_keyword(watch_id: str, params: dict, fast: dict, slow: dict) -> dict | None:
|
||||
"""Alert when a keyword appears in news, GDELT, or Telegram OSINT."""
|
||||
keyword = str(params.get("keyword", "")).lower().strip()
|
||||
if not keyword:
|
||||
return None
|
||||
|
||||
matches = []
|
||||
include_telegram = params.get("include_telegram", True)
|
||||
if isinstance(include_telegram, str):
|
||||
include_telegram = include_telegram.strip().lower() not in {"0", "false", "no", "off"}
|
||||
|
||||
matches = []
|
||||
new_telegram_ids: list[str] = []
|
||||
|
||||
# Check news articles
|
||||
for article in slow.get("news", []):
|
||||
title = str(article.get("title", "") or "").lower()
|
||||
desc = str(article.get("description", "") or article.get("summary", "") or "").lower()
|
||||
@@ -409,7 +442,6 @@ def _check_keyword(params: dict, fast: dict, slow: dict) -> dict | None:
|
||||
"url": article.get("url") or article.get("link"),
|
||||
})
|
||||
|
||||
# Check GDELT
|
||||
for event in slow.get("gdelt", []):
|
||||
text = str(event.get("title", "") or event.get("sourceurl", "") or "").lower()
|
||||
if keyword in text:
|
||||
@@ -419,14 +451,103 @@ def _check_keyword(params: dict, fast: dict, slow: dict) -> dict | None:
|
||||
"url": event.get("sourceurl"),
|
||||
})
|
||||
|
||||
if include_telegram:
|
||||
from services.telegram_osint_text import (
|
||||
iter_telegram_posts,
|
||||
keyword_matches_telegram_post,
|
||||
telegram_post_match_entry,
|
||||
)
|
||||
|
||||
for post in iter_telegram_posts(slow.get("telegram_osint")):
|
||||
if not keyword_matches_telegram_post(post, keyword):
|
||||
continue
|
||||
post_id = _telegram_post_id(post)
|
||||
if _is_seen_post(watch_id, post_id):
|
||||
continue
|
||||
entry = telegram_post_match_entry(post)
|
||||
matches.append(entry)
|
||||
if post_id:
|
||||
new_telegram_ids.append(post_id)
|
||||
|
||||
if matches:
|
||||
if new_telegram_ids:
|
||||
_mark_seen_posts(watch_id, new_telegram_ids)
|
||||
sources = sorted({str(match.get("source") or "unknown") for match in matches})
|
||||
return {
|
||||
"alert": f"Keyword '{keyword}' found in {len(matches)} articles",
|
||||
"alert": f"Keyword '{keyword}' found in {len(matches)} items ({', '.join(sources)})",
|
||||
"data": {"keyword": keyword, "matches": matches[:10]},
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def _check_telegram_rhetoric(watch_id: str, params: dict, slow: dict) -> dict | None:
|
||||
"""Alert on new high-risk Telegram OSINT posts (optionally keyword/channel filtered)."""
|
||||
min_risk = int(params.get("min_risk_score", 7) or 7)
|
||||
min_risk = max(1, min(min_risk, 10))
|
||||
|
||||
raw_keywords = params.get("keywords") or params.get("keyword") or []
|
||||
if isinstance(raw_keywords, str):
|
||||
raw_keywords = [part.strip() for part in raw_keywords.split(",") if part.strip()]
|
||||
keywords = [str(item).lower().strip() for item in raw_keywords if str(item).strip()]
|
||||
|
||||
raw_channels = params.get("channels") or params.get("channel") or []
|
||||
if isinstance(raw_channels, str):
|
||||
raw_channels = [part.strip() for part in raw_channels.split(",") if part.strip()]
|
||||
channels = [str(item).lower().strip().lstrip("@") for item in raw_channels if str(item).strip()]
|
||||
|
||||
from services.telegram_osint_text import (
|
||||
iter_telegram_posts,
|
||||
keyword_matches_telegram_post,
|
||||
telegram_post_match_entry,
|
||||
)
|
||||
|
||||
matches = []
|
||||
new_post_ids: list[str] = []
|
||||
|
||||
for post in iter_telegram_posts(slow.get("telegram_osint")):
|
||||
try:
|
||||
risk = int(post.get("risk_score") or 0)
|
||||
except (TypeError, ValueError):
|
||||
risk = 0
|
||||
if risk < min_risk:
|
||||
continue
|
||||
|
||||
channel = str(post.get("channel") or "").lower().strip()
|
||||
source = str(post.get("source") or "").lower().strip()
|
||||
if channels and channel not in channels and not any(ch in source for ch in channels):
|
||||
continue
|
||||
|
||||
if keywords and not any(keyword_matches_telegram_post(post, kw) for kw in keywords):
|
||||
continue
|
||||
|
||||
post_id = _telegram_post_id(post)
|
||||
if _is_seen_post(watch_id, post_id):
|
||||
continue
|
||||
|
||||
entry = telegram_post_match_entry(post)
|
||||
matches.append(entry)
|
||||
if post_id:
|
||||
new_post_ids.append(post_id)
|
||||
|
||||
if not matches:
|
||||
return None
|
||||
|
||||
_mark_seen_posts(watch_id, new_post_ids)
|
||||
top = max(int(match.get("risk_score") or 0) for match in matches)
|
||||
return {
|
||||
"alert": (
|
||||
f"Telegram rhetoric alert: {len(matches)} new post(s) at LVL {top}/10"
|
||||
+ (f" (min {min_risk})" if min_risk > 1 else "")
|
||||
),
|
||||
"data": {
|
||||
"min_risk_score": min_risk,
|
||||
"keywords": keywords,
|
||||
"channels": channels,
|
||||
"matches": matches[:10],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _check_prediction_market(params: dict, slow: dict) -> dict | None:
|
||||
"""Alert on prediction market movements."""
|
||||
query = str(params.get("query", "")).lower().strip()
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Container-aware runtime limits for fleet vs desktop deployments."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _read_first_int(path: Path) -> int | None:
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8").strip().split()[0]
|
||||
return int(raw)
|
||||
except (OSError, ValueError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def detect_cpu_limit() -> float | None:
|
||||
"""Effective CPU cores from cgroup quota (Docker ``cpus:``), else host count."""
|
||||
cgroup_v2 = Path("/sys/fs/cgroup/cpu.max")
|
||||
if cgroup_v2.is_file():
|
||||
try:
|
||||
parts = cgroup_v2.read_text(encoding="utf-8").strip().split()
|
||||
if len(parts) >= 2 and parts[0] != "max":
|
||||
quota = int(parts[0])
|
||||
period = int(parts[1])
|
||||
if quota > 0 and period > 0:
|
||||
return round(quota / period, 3)
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
|
||||
cgroup_v1_quota = Path("/sys/fs/cgroup/cpu/cpu.cfs_quota_us")
|
||||
cgroup_v1_period = Path("/sys/fs/cgroup/cpu/cpu.cfs_period_us")
|
||||
if cgroup_v1_quota.is_file() and cgroup_v1_period.is_file():
|
||||
quota = _read_first_int(cgroup_v1_quota)
|
||||
period = _read_first_int(cgroup_v1_period)
|
||||
if quota is not None and period and quota > 0:
|
||||
return round(quota / period, 3)
|
||||
|
||||
try:
|
||||
import os as _os
|
||||
|
||||
count = _os.cpu_count()
|
||||
return float(count) if count else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def detect_memory_limit_mb() -> int | None:
|
||||
cgroup_v2 = Path("/sys/fs/cgroup/memory.max")
|
||||
if cgroup_v2.is_file():
|
||||
try:
|
||||
raw = cgroup_v2.read_text(encoding="utf-8").strip()
|
||||
if raw and raw != "max":
|
||||
return int(int(raw) / (1024 * 1024))
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
|
||||
cgroup_v1 = Path("/sys/fs/cgroup/memory/memory.limit_in_bytes")
|
||||
if cgroup_v1.is_file():
|
||||
try:
|
||||
raw = _read_first_int(cgroup_v1)
|
||||
if raw is not None and raw < (1 << 62):
|
||||
return int(raw / (1024 * 1024))
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def resolve_profile_name() -> str:
|
||||
explicit = str(os.environ.get("GT_ANALYTICS_PROFILE", "")).strip().lower()
|
||||
if explicit in {"lean", "standard"}:
|
||||
return explicit
|
||||
cpu = detect_cpu_limit()
|
||||
if cpu is not None and cpu <= 1.0:
|
||||
return "lean"
|
||||
return "standard"
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_runtime_profile() -> dict[str, Any]:
|
||||
cpu_limit = detect_cpu_limit()
|
||||
memory_mb = detect_memory_limit_mb()
|
||||
profile = resolve_profile_name()
|
||||
lean = profile == "lean"
|
||||
return {
|
||||
"profile": profile,
|
||||
"cpu_limit": cpu_limit,
|
||||
"memory_limit_mb": memory_mb,
|
||||
"gt_analytics": {
|
||||
"recommended": not lean,
|
||||
"lean_node": lean,
|
||||
"warning": (
|
||||
"This node is capped at 1 vCPU. Enabling Strategic Risk (Derived OSINT) "
|
||||
"may slow Telegram, GDELT, and other OSINT fetches. Set "
|
||||
"GT_ANALYTICS_ACK_LOW_CPU=true after enabling GT_ANALYTICS_ENABLED to run "
|
||||
"the full engine on lean hardware."
|
||||
if lean
|
||||
else None
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def clear_runtime_profile_cache() -> None:
|
||||
get_runtime_profile.cache_clear()
|
||||
@@ -19,6 +19,7 @@ class HealthResponse(BaseModel):
|
||||
# insecure-date path because the upstream Let's Encrypt cert is
|
||||
# expired. Empty dict / null means no status reported yet.
|
||||
ais_proxy: Optional[Dict[str, Any]] = None
|
||||
runtime: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class RefreshResponse(BaseModel):
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Shared Telegram OSINT post text helpers for search and watchdog matching."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from services.telegram_translate import source_lang_label
|
||||
|
||||
|
||||
def iter_telegram_posts(layer_payload: Any) -> list[dict[str, Any]]:
|
||||
"""Normalize telegram_osint layer payloads into a list of post dicts."""
|
||||
if isinstance(layer_payload, list):
|
||||
return [post for post in layer_payload if isinstance(post, dict)]
|
||||
if isinstance(layer_payload, dict):
|
||||
posts = layer_payload.get("posts")
|
||||
if isinstance(posts, list):
|
||||
return [post for post in posts if isinstance(post, dict)]
|
||||
return []
|
||||
|
||||
|
||||
def telegram_post_search_text(post: dict[str, Any]) -> str:
|
||||
"""Build a lowercase haystack for keyword matching (translated + original)."""
|
||||
parts = (
|
||||
post.get("title_translated"),
|
||||
post.get("description_translated"),
|
||||
post.get("title"),
|
||||
post.get("description"),
|
||||
post.get("source"),
|
||||
post.get("channel"),
|
||||
)
|
||||
return " ".join(str(part).strip() for part in parts if str(part or "").strip()).lower()
|
||||
|
||||
|
||||
def telegram_post_display_title(post: dict[str, Any]) -> str:
|
||||
"""Prefer translated headline for alerts and agent-facing summaries."""
|
||||
translated = str(post.get("title_translated") or post.get("description_translated") or "").strip()
|
||||
if translated:
|
||||
return translated.split("\n", 1)[0][:200]
|
||||
return str(post.get("title") or post.get("description") or "").strip()[:200]
|
||||
|
||||
|
||||
def telegram_post_match_entry(post: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Compact match record for watchdog alerts and search results."""
|
||||
lat, lng = None, None
|
||||
coords = post.get("coords")
|
||||
if isinstance(coords, (list, tuple)) and len(coords) >= 2:
|
||||
lat, lng = coords[0], coords[1]
|
||||
return {
|
||||
"source": "telegram_osint",
|
||||
"title": telegram_post_display_title(post),
|
||||
"original_title": str(post.get("title") or "").strip(),
|
||||
"url": post.get("link") or "",
|
||||
"channel": post.get("channel") or post.get("source") or "",
|
||||
"risk_score": post.get("risk_score"),
|
||||
"source_lang": post.get("source_lang"),
|
||||
"source_lang_label": post.get("source_lang_label") or source_lang_label(post.get("source_lang")),
|
||||
"lat": lat,
|
||||
"lng": lng,
|
||||
"id": post.get("id") or post.get("link") or "",
|
||||
}
|
||||
|
||||
|
||||
def keyword_matches_telegram_post(post: dict[str, Any], keyword: str) -> bool:
|
||||
needle = str(keyword or "").strip().lower()
|
||||
if not needle:
|
||||
return False
|
||||
return needle in telegram_post_search_text(post)
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Auto-translation for Telegram OSINT post text (server-side, cached)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import urllib.parse
|
||||
from threading import Lock
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CYRILLIC_RE = re.compile(r"[\u0400-\u04FF]")
|
||||
_UKRAINIAN_MARKERS_RE = re.compile(r"[іїєґІЇЄҐ]")
|
||||
_ARABIC_RE = re.compile(r"[\u0600-\u06FF]")
|
||||
_HEBREW_RE = re.compile(r"[\u0590-\u05FF]")
|
||||
_CJK_RE = re.compile(r"[\u4e00-\u9fff]")
|
||||
|
||||
# Common war-reporting shorthand that machine translation often transliterates.
|
||||
_POST_TRANSLATION_GLOSSARY: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(re.compile(r"\bBpLa\b", re.IGNORECASE), "UAV"),
|
||||
(re.compile(r"\bБпЛА\b", re.IGNORECASE), "UAV"),
|
||||
(re.compile(r"\bбпла\b"), "UAV"),
|
||||
(re.compile(r"\bБПЛА\b"), "UAV"),
|
||||
(re.compile(r"\bрсзв\b", re.IGNORECASE), "MLRS"),
|
||||
(re.compile(r"\bРСЗВ\b"), "MLRS"),
|
||||
)
|
||||
|
||||
_SOURCE_LANG_LABELS = {
|
||||
"uk": "Ukrainian",
|
||||
"ru": "Russian",
|
||||
"en": "English",
|
||||
"ar": "Arabic",
|
||||
"he": "Hebrew",
|
||||
"zh-cn": "Chinese",
|
||||
"fr": "French",
|
||||
"de": "German",
|
||||
"pl": "Polish",
|
||||
}
|
||||
|
||||
_CACHE: dict[str, tuple[str, str]] = {}
|
||||
_CACHE_LOCK = Lock()
|
||||
_CACHE_MAX = 512
|
||||
|
||||
_LOCALE_TO_GOOGLE = {
|
||||
"en": "en",
|
||||
"fr": "fr",
|
||||
"zh-cn": "zh-CN",
|
||||
"zh": "zh-CN",
|
||||
}
|
||||
|
||||
|
||||
def telegram_translate_enabled() -> bool:
|
||||
return str(os.environ.get("TELEGRAM_OSINT_TRANSLATE", "true")).strip().lower() not in {
|
||||
"0",
|
||||
"false",
|
||||
"no",
|
||||
"off",
|
||||
"",
|
||||
}
|
||||
|
||||
|
||||
def telegram_translate_target() -> str:
|
||||
raw = str(os.environ.get("TELEGRAM_OSINT_TRANSLATE_TO", "en")).strip().lower()
|
||||
return _LOCALE_TO_GOOGLE.get(raw, raw or "en")
|
||||
|
||||
|
||||
def normalize_translate_target(locale: str | None) -> str:
|
||||
raw = str(locale or telegram_translate_target()).strip().lower().replace("_", "-")
|
||||
return _LOCALE_TO_GOOGLE.get(raw, raw or "en")
|
||||
|
||||
|
||||
def _looks_english(text: str) -> bool:
|
||||
letters = [char for char in text if char.isalpha()]
|
||||
if not letters:
|
||||
return True
|
||||
ascii_letters = sum(1 for char in letters if ord(char) < 128)
|
||||
return ascii_letters / len(letters) > 0.9
|
||||
|
||||
|
||||
def contains_cyrillic(text: str) -> bool:
|
||||
return bool(_CYRILLIC_RE.search(str(text or "")))
|
||||
|
||||
|
||||
def source_lang_label(code: str | None) -> str:
|
||||
raw = str(code or "").strip().lower().replace("_", "-")
|
||||
return _SOURCE_LANG_LABELS.get(raw, raw.upper() if raw else "Unknown")
|
||||
|
||||
|
||||
def polish_translation(text: str) -> str:
|
||||
polished = str(text or "")
|
||||
for pattern, replacement in _POST_TRANSLATION_GLOSSARY:
|
||||
polished = pattern.sub(replacement, polished)
|
||||
return polished.strip()
|
||||
|
||||
|
||||
def guess_source_lang(text: str) -> str:
|
||||
if _UKRAINIAN_MARKERS_RE.search(text):
|
||||
return "uk"
|
||||
if _CYRILLIC_RE.search(text):
|
||||
return "ru"
|
||||
if _ARABIC_RE.search(text):
|
||||
return "ar"
|
||||
if _HEBREW_RE.search(text):
|
||||
return "he"
|
||||
if _CJK_RE.search(text):
|
||||
return "zh-CN"
|
||||
if _looks_english(text):
|
||||
return "en"
|
||||
return "auto"
|
||||
|
||||
|
||||
def _cache_key(text: str, target_lang: str) -> str:
|
||||
digest = hashlib.sha1(f"{target_lang}|{text}".encode("utf-8")).hexdigest()
|
||||
return digest
|
||||
|
||||
|
||||
def _cache_get(text: str, target_lang: str) -> tuple[str, str] | None:
|
||||
key = _cache_key(text, target_lang)
|
||||
with _CACHE_LOCK:
|
||||
return _CACHE.get(key)
|
||||
|
||||
|
||||
def _cache_put(text: str, target_lang: str, translated: str, source_lang: str) -> None:
|
||||
key = _cache_key(text, target_lang)
|
||||
with _CACHE_LOCK:
|
||||
if len(_CACHE) >= _CACHE_MAX:
|
||||
_CACHE.pop(next(iter(_CACHE)))
|
||||
_CACHE[key] = (translated, source_lang)
|
||||
|
||||
|
||||
def _google_translate(clean: str, target: str, source: str | None = None) -> tuple[str, str]:
|
||||
params = {
|
||||
"client": "gtx",
|
||||
"sl": source or "auto",
|
||||
"tl": target,
|
||||
"dt": "t",
|
||||
"q": clean[:4500],
|
||||
}
|
||||
url = "https://translate.googleapis.com/translate_a/single?" + urllib.parse.urlencode(params)
|
||||
resp = requests.get(
|
||||
url,
|
||||
timeout=8,
|
||||
headers={"User-Agent": "Mozilla/5.0 (compatible; Shadowbroker-Telegram-Translate/1.0)"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
detected = str(data[2] or guess_source_lang(clean)).strip().lower()
|
||||
if detected in {"zh-cn", "zh-tw"}:
|
||||
detected = "zh-CN"
|
||||
parts: list[str] = []
|
||||
for chunk in data[0] or []:
|
||||
if chunk and chunk[0]:
|
||||
parts.append(str(chunk[0]))
|
||||
translated = polish_translation("".join(parts).strip() or clean)
|
||||
return translated, detected
|
||||
|
||||
|
||||
def translate_text(text: str, target_lang: str | None = None) -> tuple[str, str]:
|
||||
"""Translate text via Google Translate (unofficial client endpoint).
|
||||
|
||||
Returns ``(translated_text, detected_source_lang)``.
|
||||
"""
|
||||
clean = str(text or "").strip()
|
||||
if not clean:
|
||||
return "", "en"
|
||||
|
||||
target = normalize_translate_target(target_lang)
|
||||
if _looks_english(clean) and target == "en":
|
||||
return clean, "en"
|
||||
|
||||
cached = _cache_get(clean, target)
|
||||
if cached:
|
||||
return cached
|
||||
|
||||
try:
|
||||
translated, detected = _google_translate(clean, target)
|
||||
if detected == target or (detected == "en" and target == "en"):
|
||||
result = (clean, detected)
|
||||
_cache_put(clean, target, clean, detected)
|
||||
return result
|
||||
if contains_cyrillic(translated) and contains_cyrillic(clean):
|
||||
hinted = guess_source_lang(clean)
|
||||
if hinted not in {"auto", target}:
|
||||
retry_translated, retry_detected = _google_translate(clean, target, hinted)
|
||||
if not contains_cyrillic(retry_translated) or len(retry_translated) > len(translated):
|
||||
translated, detected = retry_translated, retry_detected
|
||||
result = (translated, detected)
|
||||
_cache_put(clean, target, translated, detected)
|
||||
return result
|
||||
except Exception as exc:
|
||||
logger.warning("Telegram translation failed: %s", exc)
|
||||
fallback_lang = guess_source_lang(clean)
|
||||
return clean, fallback_lang
|
||||
|
||||
|
||||
def apply_post_translation(post: dict[str, Any], target_lang: str | None = None) -> dict[str, Any]:
|
||||
"""Add translation fields to a Telegram OSINT post dict."""
|
||||
if not telegram_translate_enabled():
|
||||
return post
|
||||
|
||||
target = normalize_translate_target(target_lang)
|
||||
description = str(post.get("description") or "").strip()
|
||||
title = str(post.get("title") or "").strip()
|
||||
full_text = description or title
|
||||
if not full_text:
|
||||
return post
|
||||
|
||||
existing_translated = str(post.get("description_translated") or post.get("title_translated") or "").strip()
|
||||
if post.get("translate_to") == target and existing_translated:
|
||||
updated = dict(post)
|
||||
polished = polish_translation(existing_translated)
|
||||
if polished != existing_translated:
|
||||
lines = polished.split("\n", 1)
|
||||
updated["title_translated"] = lines[0][:160]
|
||||
updated["description_translated"] = polished[:1200]
|
||||
updated["source_lang_label"] = source_lang_label(str(post.get("source_lang") or ""))
|
||||
return updated
|
||||
|
||||
translated_full, source_lang = translate_text(full_text, target)
|
||||
updated = dict(post)
|
||||
updated["source_lang"] = source_lang
|
||||
updated["translate_to"] = target
|
||||
updated["source_lang_label"] = source_lang_label(source_lang)
|
||||
|
||||
if translated_full != full_text and source_lang != target:
|
||||
lines = translated_full.split("\n", 1)
|
||||
updated["title_translated"] = lines[0][:160]
|
||||
updated["description_translated"] = translated_full[:1200]
|
||||
|
||||
return updated
|
||||
|
||||
|
||||
def apply_posts_translations(
|
||||
posts: list[dict[str, Any]],
|
||||
target_lang: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
if not telegram_translate_enabled():
|
||||
return posts
|
||||
return [apply_post_translation(post, target_lang) for post in posts]
|
||||
@@ -97,6 +97,7 @@ _SLOW_KEYS = (
|
||||
"cyber_threats",
|
||||
"scm_suppliers",
|
||||
"telegram_osint",
|
||||
"gt_risk",
|
||||
)
|
||||
|
||||
|
||||
@@ -210,6 +211,9 @@ _LAYER_ALIASES = {
|
||||
"telegram": "telegram_osint",
|
||||
"telegram_osint": "telegram_osint",
|
||||
"osint_feed": "telegram_osint",
|
||||
"gt_risk": "gt_risk",
|
||||
"strategic_risk": "gt_risk",
|
||||
"gt_analytics": "gt_risk",
|
||||
"malware": "malware_threats",
|
||||
"malware_threats": "malware_threats",
|
||||
"malware_c2": "malware_threats",
|
||||
@@ -710,10 +714,10 @@ _UNIVERSAL_SEARCH_SPECS: dict[str, dict[str, Any]] = {
|
||||
"time_fields": ("updated_at", "timestamp"),
|
||||
},
|
||||
"telegram_osint": {
|
||||
"fields": ("title", "description", "source", "channel", "link"),
|
||||
"primary_fields": ("title", "description", "channel"),
|
||||
"label_fields": ("title", "channel"),
|
||||
"summary_fields": ("description", "source"),
|
||||
"fields": ("title", "description", "title_translated", "description_translated", "source", "channel", "link"),
|
||||
"primary_fields": ("title_translated", "title", "description_translated", "description", "channel"),
|
||||
"label_fields": ("title_translated", "title", "channel"),
|
||||
"summary_fields": ("description_translated", "description", "source"),
|
||||
"type_fields": ("channel", "source"),
|
||||
"id_fields": ("id", "link"),
|
||||
"time_fields": ("published", "timestamp"),
|
||||
@@ -2089,30 +2093,27 @@ def search_news(
|
||||
return {"results": out, "version": get_data_version(), "truncated": True}
|
||||
|
||||
if include_telegram:
|
||||
from services.telegram_osint_text import telegram_post_display_title, telegram_post_search_text
|
||||
|
||||
for post in _unwrap_layer_items(snap.get("telegram_osint"), "telegram_osint"):
|
||||
if not isinstance(post, dict):
|
||||
continue
|
||||
text = " ".join(
|
||||
(
|
||||
_norm_text(post.get("title")),
|
||||
_norm_text(post.get("description")),
|
||||
_norm_text(post.get("source")),
|
||||
_norm_text(post.get("channel")),
|
||||
)
|
||||
)
|
||||
text = telegram_post_search_text(post)
|
||||
if not _text_matches_query(query_norm, text):
|
||||
continue
|
||||
lat, lng = _extract_coords(post)
|
||||
out.append(
|
||||
{
|
||||
"source_layer": "telegram_osint",
|
||||
"title": post.get("title") or "",
|
||||
"summary": post.get("description") or "",
|
||||
"title": telegram_post_display_title(post),
|
||||
"summary": post.get("description_translated") or post.get("description") or "",
|
||||
"source": post.get("source") or post.get("channel") or "Telegram",
|
||||
"link": post.get("link") or "",
|
||||
"lat": lat,
|
||||
"lng": lng,
|
||||
"risk_score": post.get("risk_score"),
|
||||
"source_lang": post.get("source_lang"),
|
||||
"source_lang_label": post.get("source_lang_label"),
|
||||
}
|
||||
)
|
||||
if len(out) >= limit:
|
||||
|
||||
Reference in New Issue
Block a user