mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-07-06 12:28:05 +02:00
Feat/gt analytics openclaw (#392)
* feat(telegram): auto-translate OSINT channel posts to English Cherry-picked from @Bobpick PR #391 (telegram-only slice): server-side translation during fetch, SHOW ORIGINAL toggle in TelegramOsintPopup, and on-demand /api/telegram-feed?lang=. Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com> Co-authored-by: Cursor <cursoragent@cursor.com> * feat(gt): experimental Derived OSINT analytics with lean-node safeguards Cherry-picked from @Bobpick PR #391 (GT + OpenClaw slice): Bayesian strategic-risk engine, map overlay, OpenClaw commands, and telegram_rhetoric watchdog. Off by default (GT_ANALYTICS_ENABLED=false, gt_risk layer false). 1 vCPU nodes get cgroup detection, UI warning on layer toggle, and lean profile that skips scheduled ingest/Louvain unless GT_ANALYTICS_ACK_LOW_CPU=true. Backtest HUD removed from dashboard (OpenClaw/API regression only). Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com> Co-authored-by: Cursor <cursoragent@cursor.com> --------- Co-authored-by: Robert Pickett <bobpickettsr@yahoo.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -1,9 +1,23 @@
|
||||
import os
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _gt_analytics_standard_profile(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Tests assume a standard (non-lean) runtime unless they override profile."""
|
||||
monkeypatch.setenv("GT_ANALYTICS_PROFILE", os.environ.get("GT_ANALYTICS_PROFILE", "standard"))
|
||||
try:
|
||||
from analytics.integration import reset_gt_engine
|
||||
|
||||
reset_gt_engine()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _suppress_background_services():
|
||||
"""Prevent real scheduler/stream/tracker from starting during tests."""
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
"""API tests for Strategic Risk Analytics routes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from analytics.integration import reset_gt_engine
|
||||
from services.fetchers import _store
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_gt(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("GT_ANALYTICS_ENABLED", raising=False)
|
||||
reset_gt_engine()
|
||||
|
||||
|
||||
def test_risk_heatmap_disabled(client) -> None:
|
||||
response = client.get("/api/analytics/risk_heatmap")
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["enabled"] is False
|
||||
assert payload["type"] == "FeatureCollection"
|
||||
assert payload["features"] == []
|
||||
|
||||
|
||||
def test_dossier_disabled(client) -> None:
|
||||
response = client.get("/api/analytics/dossier/ukraine")
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["enabled"] is False
|
||||
assert payload["region"] == "ukraine"
|
||||
|
||||
|
||||
def test_risk_heatmap_enabled_after_refresh(client, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
|
||||
reset_gt_engine()
|
||||
|
||||
_store.latest_data["telegram_osint"] = {
|
||||
"posts": [
|
||||
{
|
||||
"id": "api-tg-1",
|
||||
"title": "Troop buildup",
|
||||
"description": "Troop movement and armored convoy reported near border.",
|
||||
"source": "t.me/war_monitor",
|
||||
"channel": "war_monitor",
|
||||
"coords": [48.5, 37.5],
|
||||
}
|
||||
],
|
||||
"total": 1,
|
||||
"geolocated": 1,
|
||||
}
|
||||
_store.latest_data["news"] = []
|
||||
_store.latest_data["gdelt"] = []
|
||||
|
||||
from analytics.integration import refresh_from_latest_data
|
||||
|
||||
refresh_from_latest_data(dict(_store.latest_data), persist=True)
|
||||
|
||||
response = client.get("/api/analytics/risk_heatmap")
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["enabled"] is True
|
||||
assert len(payload["features"]) >= 1
|
||||
assert payload["timestamp"] is not None
|
||||
|
||||
|
||||
def test_dossier_enabled(client, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
|
||||
reset_gt_engine()
|
||||
|
||||
_store.latest_data["telegram_osint"] = {
|
||||
"posts": [
|
||||
{
|
||||
"id": "api-tg-2",
|
||||
"title": "Strike",
|
||||
"description": "General strike and protest mobilization in capital.",
|
||||
"source": "t.me/nexta_live",
|
||||
"channel": "nexta_live",
|
||||
"coords": [50.45, 30.52],
|
||||
}
|
||||
]
|
||||
}
|
||||
_store.latest_data["news"] = []
|
||||
_store.latest_data["gdelt"] = []
|
||||
|
||||
from analytics.integration import refresh_from_latest_data
|
||||
|
||||
refresh_from_latest_data(dict(_store.latest_data), persist=True)
|
||||
|
||||
response = client.get("/api/analytics/dossier/50.45,30.52")
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["enabled"] is True
|
||||
assert payload["recent_signals"]
|
||||
assert "interpretation" in payload
|
||||
|
||||
|
||||
def test_post_risk_heatmap_ingest(client, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
|
||||
reset_gt_engine()
|
||||
|
||||
response = client.post(
|
||||
"/api/analytics/risk_heatmap",
|
||||
json={
|
||||
"refresh": False,
|
||||
"items": [
|
||||
{
|
||||
"title": "GPS interference",
|
||||
"description": "GPS jamming spike along northern corridor.",
|
||||
"source": "manual",
|
||||
"region": "baltics",
|
||||
"domain": "conflict",
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["enabled"] is True
|
||||
assert payload["ingested"] == 1
|
||||
|
||||
|
||||
def test_backtest_disabled(client) -> None:
|
||||
response = client.get("/api/analytics/backtest")
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["enabled"] is False
|
||||
|
||||
|
||||
def test_backtest_enabled(client, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
|
||||
reset_gt_engine()
|
||||
|
||||
response = client.get("/api/analytics/backtest?expanded=true&tune=false")
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["enabled"] is True
|
||||
assert payload["accuracy"] == 1.0
|
||||
assert payload["confidence_rate"] >= 0.95
|
||||
assert payload["meets_target"] is True
|
||||
assert payload["total_cases"] >= 80
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Top GT alerts ranking and coordinate filtering."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from analytics.gt_alerts import parse_heatmap_alerts, top_gt_alerts
|
||||
|
||||
|
||||
def test_parse_heatmap_filters_invalid_coords() -> None:
|
||||
heatmap = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"region": "ukraine",
|
||||
"risk": 0.55,
|
||||
"conflict": 0.62,
|
||||
"financial": 0.15,
|
||||
"unrest": 0.2,
|
||||
},
|
||||
"geometry": {"type": "Point", "coordinates": [31.0, 48.0]},
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {"region": "no_coords", "risk": 0.9},
|
||||
"geometry": {"type": "Point", "coordinates": [0.0, 0.0]},
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {"region": "global", "risk": 0.99},
|
||||
"geometry": {"type": "Point", "coordinates": [0.0, 0.0]},
|
||||
},
|
||||
],
|
||||
}
|
||||
alerts, plotted = parse_heatmap_alerts(heatmap, limit=5)
|
||||
assert plotted == 1
|
||||
assert len(alerts) == 1
|
||||
assert alerts[0]["region"] == "ukraine"
|
||||
assert alerts[0]["lat"] == 48.0
|
||||
assert alerts[0]["lng"] == 31.0
|
||||
|
||||
|
||||
def test_region_label_formats_coordinates() -> None:
|
||||
from analytics.gt_alerts import _region_label
|
||||
|
||||
assert "48.00" in _region_label("48.00,31.17")
|
||||
assert _region_label("ukraine") == "ukraine"
|
||||
|
||||
|
||||
def test_top_gt_alerts_disabled(monkeypatch) -> None:
|
||||
monkeypatch.delenv("GT_ANALYTICS_ENABLED", raising=False)
|
||||
from analytics.integration import reset_gt_engine
|
||||
|
||||
reset_gt_engine()
|
||||
report = top_gt_alerts(limit=3)
|
||||
assert report["alerts"] == []
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Historical backtest validation for Strategic Risk Analytics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from analytics.backtest import (
|
||||
DEFAULT_BACKTEST_ALERT_THRESHOLD,
|
||||
MAX_BACKTEST_ALERT_THRESHOLD,
|
||||
run_historical_backtest,
|
||||
tune_alert_threshold,
|
||||
wilson_interval,
|
||||
)
|
||||
from analytics.historical_events import default_historical_cases, expanded_historical_cases
|
||||
|
||||
|
||||
def test_wilson_interval_perfect_run() -> None:
|
||||
lower, upper = wilson_interval(18, 18)
|
||||
assert lower >= 0.80
|
||||
assert upper == 1.0
|
||||
|
||||
|
||||
def test_base_suite_meets_eighty_percent_confidence() -> None:
|
||||
report = run_historical_backtest(
|
||||
default_historical_cases(),
|
||||
use_expanded_suite=False,
|
||||
target_confidence=0.80,
|
||||
)
|
||||
assert report.accuracy >= 0.95
|
||||
assert report.confidence_rate >= 0.80
|
||||
assert report.meets_target
|
||||
assert report.false_positives == 0
|
||||
assert report.false_negatives == 0
|
||||
|
||||
|
||||
def test_expanded_suite_meets_ninety_five_percent_confidence() -> None:
|
||||
threshold, report = tune_alert_threshold(target_confidence=0.95)
|
||||
assert len(expanded_historical_cases()) >= 80
|
||||
assert report.accuracy == 1.0
|
||||
assert report.confidence_rate >= 0.95
|
||||
assert report.meets_target
|
||||
assert report.false_positives == 0
|
||||
assert report.false_negatives == 0
|
||||
assert DEFAULT_BACKTEST_ALERT_THRESHOLD <= threshold <= MAX_BACKTEST_ALERT_THRESHOLD
|
||||
|
||||
|
||||
def test_default_backtest_threshold_on_expanded_suite() -> None:
|
||||
report = run_historical_backtest(
|
||||
use_expanded_suite=True,
|
||||
target_confidence=0.95,
|
||||
)
|
||||
assert report.alert_threshold == DEFAULT_BACKTEST_ALERT_THRESHOLD
|
||||
assert report.accuracy == 1.0
|
||||
assert report.confidence_rate >= 0.95
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Tests for Strategic Risk Analytics core scoring."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from analytics.feed_adapter import normalize_feed_item
|
||||
from analytics.gt_early_warning import GT_EarlyWarning
|
||||
from analytics.integration import process_feed_item, refresh_from_latest_data, reset_gt_engine
|
||||
from analytics.settings import GTAnalyticsSettings
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def engine() -> GT_EarlyWarning:
|
||||
return GT_EarlyWarning(
|
||||
GTAnalyticsSettings(
|
||||
enabled=True,
|
||||
base_prior=0.15,
|
||||
evidence_cap=3.0,
|
||||
evidence_scale=5.0,
|
||||
high_risk_threshold=0.6,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_classify_payroll_loan_signal(engine: GT_EarlyWarning) -> None:
|
||||
signals = engine.classify_signals("Franchise owners increasingly rely on payroll loan facilities.")
|
||||
assert "payroll_loan" in signals
|
||||
assert signals["payroll_loan"] >= 3.0
|
||||
|
||||
|
||||
def test_classify_no_signal_on_generic_text(engine: GT_EarlyWarning) -> None:
|
||||
signals = engine.classify_signals("Sunny weather expected across the region this weekend.")
|
||||
assert signals == {}
|
||||
|
||||
|
||||
def test_bayesian_update_increases_risk(engine: GT_EarlyWarning) -> None:
|
||||
prior = engine.get_prior("uk", "financial")
|
||||
posterior = engine.bayesian_update("uk", "financial", evidence_strength=2.0)
|
||||
assert posterior > prior
|
||||
|
||||
|
||||
def test_process_feed_item_updates_region(engine: GT_EarlyWarning) -> None:
|
||||
item = {
|
||||
"id": "test-1",
|
||||
"text": "Mass rally and general strike announced; protest mobilization spreads.",
|
||||
"source": "t.me/osintdefender",
|
||||
"region": "ukraine",
|
||||
"domain": "unrest",
|
||||
"entities": ["channel:osintdefender"],
|
||||
"coords": [50.45, 30.52],
|
||||
}
|
||||
result = engine.process_feed_item(item)
|
||||
assert result["signals"]
|
||||
assert result["risk_score"] > engine.settings.base_prior
|
||||
assert result["contagion_potential"] >= 0.0
|
||||
|
||||
|
||||
def test_duplicate_items_are_skipped(engine: GT_EarlyWarning) -> None:
|
||||
item = {
|
||||
"id": "dup-1",
|
||||
"text": "GPS jamming spike reported near border corridor.",
|
||||
"source": "gdelt",
|
||||
"region": "baltics",
|
||||
"domain": "conflict",
|
||||
}
|
||||
first = engine.process_feed_item(item)
|
||||
second = engine.process_feed_item(item)
|
||||
assert not first.get("skipped")
|
||||
assert second.get("skipped") is True
|
||||
|
||||
|
||||
def test_heatmap_returns_geojson_features(engine: GT_EarlyWarning) -> None:
|
||||
engine.process_feed_item(
|
||||
{
|
||||
"id": "heat-1",
|
||||
"text": "Troop movement and armored convoy observed overnight.",
|
||||
"source": "news",
|
||||
"region": "eastern_europe",
|
||||
"coords": [48.0, 37.0],
|
||||
}
|
||||
)
|
||||
heatmap = engine.get_risk_heatmap()
|
||||
assert heatmap["type"] == "FeatureCollection"
|
||||
assert len(heatmap["features"]) >= 1
|
||||
feature = heatmap["features"][0]
|
||||
assert "risk" in feature["properties"]
|
||||
assert feature["geometry"]["type"] == "Point"
|
||||
|
||||
|
||||
def test_dossier_includes_recent_signals(engine: GT_EarlyWarning) -> None:
|
||||
engine.process_feed_item(
|
||||
{
|
||||
"id": "dos-1",
|
||||
"text": "Supply chain delay at major port; logistics backlog worsens.",
|
||||
"source": "news",
|
||||
"region": "china",
|
||||
"domain": "financial",
|
||||
}
|
||||
)
|
||||
dossier = engine.get_dossier("china")
|
||||
assert dossier["region"] == "china"
|
||||
assert dossier["recent_signals"]
|
||||
assert "interpretation" in dossier
|
||||
|
||||
|
||||
def test_feed_adapter_normalizes_telegram_post() -> None:
|
||||
normalized = normalize_feed_item(
|
||||
{
|
||||
"title": "Strike expands",
|
||||
"description": "General strike and rally planned in capital.",
|
||||
"source": "t.me/nexta_live",
|
||||
"channel": "nexta_live",
|
||||
"coords": [53.9, 27.56],
|
||||
},
|
||||
source_type="telegram_osint",
|
||||
)
|
||||
assert normalized["region"] != "global"
|
||||
assert normalized["domain"] in {"unrest", "financial", "conflict"}
|
||||
assert normalized["text"]
|
||||
|
||||
|
||||
def test_integration_disabled_by_default(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("GT_ANALYTICS_ENABLED", raising=False)
|
||||
reset_gt_engine()
|
||||
assert process_feed_item({"text": "test", "region": "global"}) is None
|
||||
|
||||
|
||||
def test_refresh_from_latest_data_processes_telegram(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
|
||||
reset_gt_engine()
|
||||
latest = {
|
||||
"telegram_osint": {
|
||||
"posts": [
|
||||
{
|
||||
"id": "tg-1",
|
||||
"title": "GPS jamming",
|
||||
"description": "GPS jamming spike reported along northern border.",
|
||||
"source": "t.me/osintdefender",
|
||||
"channel": "osintdefender",
|
||||
"coords": [59.93, 30.33],
|
||||
}
|
||||
]
|
||||
},
|
||||
"news": [],
|
||||
"gdelt": [],
|
||||
}
|
||||
summary = refresh_from_latest_data(latest, persist=False)
|
||||
assert summary["enabled"] is True
|
||||
assert summary["processed"] >= 1
|
||||
@@ -0,0 +1,29 @@
|
||||
"""GT feed adapter uses Telegram English translations for costly-signal matching."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from analytics.feed_adapter import normalize_feed_item
|
||||
|
||||
|
||||
def test_telegram_prefers_translated_text_for_gt() -> None:
|
||||
post = {
|
||||
"title": "Київ 1х БпЛА",
|
||||
"description": "Обстріл біля Харкова",
|
||||
"title_translated": "Kyiv 1x UAV",
|
||||
"description_translated": "Shelling near Kharkiv with troop movement reported",
|
||||
"source": "t.me/osintdefender",
|
||||
"coords": [49.99, 36.23],
|
||||
}
|
||||
item = normalize_feed_item(post, source_type="telegram_osint")
|
||||
assert "troop movement" in item["text"].lower()
|
||||
assert item["domain"] == "conflict"
|
||||
|
||||
|
||||
def test_hashtag_region_maps_ukraine_dossier_key() -> None:
|
||||
post = {
|
||||
"title": "Update",
|
||||
"description_translated": "#Ukraine #USA aircraft spotted on runway",
|
||||
"source": "t.me/osintdefender",
|
||||
}
|
||||
item = normalize_feed_item(post, source_type="telegram_osint")
|
||||
assert item["region"] == "ukraine"
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Lean-profile gating for Strategic Risk Analytics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from analytics.integration import get_gt_engine, maybe_refresh_gt_analytics, reset_gt_engine
|
||||
from analytics.settings import gt_engine_operational, gt_scheduled_ingest_enabled
|
||||
|
||||
|
||||
def test_gt_engine_blocked_on_lean_without_ack(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
|
||||
monkeypatch.setenv("GT_ANALYTICS_PROFILE", "lean")
|
||||
monkeypatch.delenv("GT_ANALYTICS_ACK_LOW_CPU", raising=False)
|
||||
reset_gt_engine()
|
||||
assert gt_engine_operational() is False
|
||||
assert get_gt_engine() is None
|
||||
|
||||
|
||||
def test_gt_engine_allowed_on_lean_with_ack(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
|
||||
monkeypatch.setenv("GT_ANALYTICS_PROFILE", "lean")
|
||||
monkeypatch.setenv("GT_ANALYTICS_ACK_LOW_CPU", "true")
|
||||
reset_gt_engine()
|
||||
assert gt_engine_operational() is True
|
||||
assert get_gt_engine() is not None
|
||||
|
||||
|
||||
def test_scheduled_ingest_skipped_on_lean(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
|
||||
monkeypatch.setenv("GT_ANALYTICS_PROFILE", "lean")
|
||||
monkeypatch.delenv("GT_ANALYTICS_ACK_LOW_CPU", raising=False)
|
||||
reset_gt_engine()
|
||||
assert gt_scheduled_ingest_enabled() is False
|
||||
maybe_refresh_gt_analytics()
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Micro rolling 3-day average for Strategic Risk Analytics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from analytics.daily_store import DailyRegionReading, DailySnapshot, date_id, save_daily
|
||||
from analytics.gt_early_warning import GT_EarlyWarning
|
||||
from analytics.micro_rolling import (
|
||||
capture_daily_readings,
|
||||
compute_micro_view,
|
||||
enrich_heatmap_features,
|
||||
micro_rolling_report,
|
||||
)
|
||||
from analytics.settings import GTAnalyticsSettings
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def daily_store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
||||
store = tmp_path / "daily"
|
||||
monkeypatch.setenv("GT_DAILY_STORE_DIR", str(store))
|
||||
return store
|
||||
|
||||
|
||||
def _seed_engine() -> GT_EarlyWarning:
|
||||
engine = GT_EarlyWarning(GTAnalyticsSettings(enabled=True, base_prior=0.15))
|
||||
engine.process_feed_item(
|
||||
{
|
||||
"text": "Troop movement and military mobilization near border",
|
||||
"region": "ukraine",
|
||||
"source": "test",
|
||||
"source_type": "manual",
|
||||
}
|
||||
)
|
||||
return engine
|
||||
|
||||
|
||||
def _save_day(day: date, region: str, peak: float) -> None:
|
||||
day_key = date_id(day)
|
||||
snap = DailySnapshot(date=day_key, regions={})
|
||||
snap.regions[region] = DailyRegionReading(
|
||||
region=region,
|
||||
composite_risk=peak * 0.9,
|
||||
financial=0.15,
|
||||
unrest=0.15,
|
||||
conflict=peak,
|
||||
peak_score=peak,
|
||||
readings=1,
|
||||
last_captured_at=f"{day_key}T12:00:00+00:00",
|
||||
)
|
||||
save_daily(snap)
|
||||
|
||||
|
||||
def test_capture_daily_readings(daily_store: Path) -> None:
|
||||
engine = _seed_engine()
|
||||
result = capture_daily_readings(engine, when=date(2026, 6, 16))
|
||||
assert result["regions"] >= 1
|
||||
again = capture_daily_readings(engine, when=date(2026, 6, 16))
|
||||
assert again["regions"] >= 1
|
||||
|
||||
|
||||
def test_3day_rolling_average_and_ignition(daily_store: Path) -> None:
|
||||
region = "ukraine"
|
||||
today = date(2026, 6, 16)
|
||||
_save_day(today - timedelta(days=2), region, 0.20)
|
||||
_save_day(today - timedelta(days=1), region, 0.22)
|
||||
_save_day(today, region, 0.45)
|
||||
|
||||
view = compute_micro_view(region, as_of=today, window_days=3)
|
||||
assert view is not None
|
||||
assert view.days_in_window == 3
|
||||
assert view.risk_3d_avg == pytest.approx(0.29, abs=0.01)
|
||||
assert view.spot_risk == 0.45
|
||||
assert view.risk_delta == pytest.approx(0.16, abs=0.01)
|
||||
assert view.ignition is True
|
||||
|
||||
|
||||
def test_enrich_heatmap_features(daily_store: Path) -> None:
|
||||
engine = _seed_engine()
|
||||
today = date(2026, 6, 16)
|
||||
capture_daily_readings(engine, when=today)
|
||||
heatmap = engine.get_risk_heatmap()
|
||||
enriched = enrich_heatmap_features(heatmap, as_of=today, window_days=3)
|
||||
feature = enriched["features"][0]
|
||||
props = feature["properties"]
|
||||
assert "risk_3d_avg" in props
|
||||
assert "risk_spot" in props
|
||||
assert "micro_ignition" in props
|
||||
|
||||
|
||||
def test_micro_rolling_report(daily_store: Path) -> None:
|
||||
region = "ukraine"
|
||||
today = date(2026, 6, 16)
|
||||
_save_day(today - timedelta(days=1), region, 0.21)
|
||||
_save_day(today, region, 0.40)
|
||||
|
||||
report = micro_rolling_report(as_of=today, window_days=3, limit=5)
|
||||
assert report["mode"] == "micro_rolling"
|
||||
assert report["window_days"] == 3
|
||||
assert report["regions_tracked"] >= 1
|
||||
|
||||
|
||||
def test_openclaw_micro_command(daily_store: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from analytics.integration import reset_gt_engine
|
||||
from services.openclaw_channel import _dispatch_command
|
||||
|
||||
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
|
||||
reset_gt_engine()
|
||||
result = _dispatch_command("gt_micro_rolling", {"window_days": 3, "compact": True})
|
||||
assert result["ok"] is True
|
||||
assert result["data"]["mode"] == "micro_rolling"
|
||||
|
||||
|
||||
def test_route_query_micro_intent() -> None:
|
||||
from services.openclaw_routing import route_query
|
||||
|
||||
plan = route_query("Show GT rolling 3 day average and ignition regions")
|
||||
assert plan["recommended"]["cmd"] == "gt_micro_rolling"
|
||||
@@ -0,0 +1,170 @@
|
||||
"""Rolling weekly operational validation for Strategic Risk Analytics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from analytics.backtest import DEFAULT_BACKTEST_ALERT_THRESHOLD
|
||||
from analytics.gt_early_warning import GT_EarlyWarning
|
||||
from analytics.integration import reset_gt_engine
|
||||
from analytics.rolling_backtest import (
|
||||
freeze_weekly_snapshot,
|
||||
iso_week_id,
|
||||
label_regions,
|
||||
rolling_report,
|
||||
score_week,
|
||||
)
|
||||
from analytics.settings import GTAnalyticsSettings
|
||||
from analytics.weekly_store import RegionSnapshot, WeeklySnapshot, load_week
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def rolling_store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
||||
store = tmp_path / "gt_rolling"
|
||||
monkeypatch.setenv("GT_ROLLING_STORE_DIR", str(store))
|
||||
return store
|
||||
|
||||
|
||||
def _seed_engine() -> GT_EarlyWarning:
|
||||
engine = GT_EarlyWarning(GTAnalyticsSettings(enabled=True, base_prior=0.15))
|
||||
engine.process_feed_item(
|
||||
{
|
||||
"text": "Troop movement and military mobilization near border",
|
||||
"region": "ukraine",
|
||||
"source": "test",
|
||||
"source_type": "manual",
|
||||
}
|
||||
)
|
||||
engine.process_feed_item(
|
||||
{
|
||||
"text": "Routine diplomatic statement about trade",
|
||||
"region": "canada",
|
||||
"source": "test",
|
||||
"source_type": "manual",
|
||||
}
|
||||
)
|
||||
return engine
|
||||
|
||||
|
||||
def test_iso_week_id_format() -> None:
|
||||
assert iso_week_id(date(2026, 6, 16)) == "2026-W25"
|
||||
|
||||
|
||||
def test_freeze_and_score_week(rolling_store: Path) -> None:
|
||||
engine = _seed_engine()
|
||||
result = freeze_weekly_snapshot(
|
||||
week_id="2026-W10",
|
||||
engine=engine,
|
||||
frozen_by="test",
|
||||
)
|
||||
assert result["ok"] is True
|
||||
assert result["created"] is True
|
||||
assert result["region_count"] >= 2
|
||||
|
||||
snapshot = load_week("2026-W10")
|
||||
assert snapshot is not None
|
||||
ukraine = next(row for row in snapshot.regions if row.region == "ukraine")
|
||||
assert ukraine.alerted is True
|
||||
|
||||
pending_score = score_week(snapshot)
|
||||
assert pending_score.labeled == 0
|
||||
assert pending_score.scorable is False
|
||||
|
||||
label_regions(
|
||||
"2026-W10",
|
||||
[
|
||||
{"region": "ukraine", "label": "true_escalation"},
|
||||
{"region": "canada", "label": "benign"},
|
||||
],
|
||||
)
|
||||
labeled = load_week("2026-W10")
|
||||
assert labeled is not None
|
||||
scored = score_week(labeled)
|
||||
assert scored.labeled == 2
|
||||
assert scored.true_positives == 1
|
||||
assert scored.true_negatives == 1
|
||||
assert scored.accuracy == 1.0
|
||||
assert scored.confidence_rate >= 0.0
|
||||
|
||||
|
||||
def test_freeze_is_idempotent(rolling_store: Path) -> None:
|
||||
engine = _seed_engine()
|
||||
first = freeze_weekly_snapshot(week_id="2026-W11", engine=engine)
|
||||
second = freeze_weekly_snapshot(week_id="2026-W11", engine=engine)
|
||||
assert first["created"] is True
|
||||
assert second["created"] is False
|
||||
|
||||
|
||||
def test_rolling_report_trend(rolling_store: Path) -> None:
|
||||
engine = _seed_engine()
|
||||
freeze_weekly_snapshot(week_id="2026-W20", engine=engine)
|
||||
freeze_weekly_snapshot(week_id="2026-W21", engine=engine)
|
||||
|
||||
label_regions("2026-W20", [{"region": "ukraine", "label": "true_escalation"}])
|
||||
label_regions(
|
||||
"2026-W21",
|
||||
[
|
||||
{"region": "ukraine", "label": "true_escalation"},
|
||||
{"region": "canada", "label": "benign"},
|
||||
],
|
||||
)
|
||||
|
||||
report = rolling_report(weeks=4)
|
||||
assert report["mode"] == "rolling_operational"
|
||||
assert report["alert_threshold"] == DEFAULT_BACKTEST_ALERT_THRESHOLD
|
||||
assert len(report["trend"]) == 2
|
||||
assert report["latest"] is not None
|
||||
|
||||
|
||||
def test_openclaw_rolling_commands(
|
||||
rolling_store: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from analytics.integration import get_gt_engine
|
||||
from services.openclaw_channel import _dispatch_command
|
||||
|
||||
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
|
||||
reset_gt_engine()
|
||||
engine = get_gt_engine()
|
||||
assert engine is not None
|
||||
engine.process_feed_item(
|
||||
{
|
||||
"text": "Troop movement and military mobilization near border",
|
||||
"region": "ukraine",
|
||||
"source": "test",
|
||||
"source_type": "manual",
|
||||
}
|
||||
)
|
||||
|
||||
freeze = _dispatch_command("gt_rolling_freeze", {"week_id": "2026-W30", "compact": True})
|
||||
assert freeze["ok"] is True
|
||||
assert freeze["data"]["enabled"] is True
|
||||
|
||||
label = _dispatch_command(
|
||||
"gt_rolling_label",
|
||||
{
|
||||
"week_id": "2026-W30",
|
||||
"region": "ukraine",
|
||||
"label": "false_alarm",
|
||||
},
|
||||
)
|
||||
assert label["ok"] is True
|
||||
assert label["data"]["updated"] == 1
|
||||
|
||||
trend = _dispatch_command("gt_rolling_backtest", {"weeks": 4, "compact": True})
|
||||
assert trend["ok"] is True
|
||||
assert trend["data"]["mode"] == "rolling_operational"
|
||||
|
||||
|
||||
def test_route_query_rolling_intent() -> None:
|
||||
from services.openclaw_routing import route_query
|
||||
|
||||
plan = route_query("Show GT rolling operational backtest week over week")
|
||||
assert plan["recommended"]["cmd"] == "gt_rolling_backtest"
|
||||
|
||||
freeze_plan = route_query("Freeze weekly GT snapshot for operational validation")
|
||||
assert freeze_plan["recommended"]["cmd"] == "gt_rolling_freeze"
|
||||
@@ -0,0 +1,60 @@
|
||||
"""OpenClaw routing and commands for Strategic Risk Analytics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from analytics.integration import reset_gt_engine
|
||||
from services.openclaw_routing import route_query
|
||||
|
||||
|
||||
def test_route_query_gt_analyze_intent() -> None:
|
||||
plan = route_query("Run GT analysis on UK and Europe feeds")
|
||||
assert plan["intent"] == "gt_analyze"
|
||||
assert plan["recommended"]["cmd"] == "gt_analyze"
|
||||
|
||||
|
||||
def test_route_query_gt_dossier_intent() -> None:
|
||||
plan = route_query("GT rationale dossier for ukraine strategic risk")
|
||||
assert plan["recommended"]["cmd"] in {"gt_dossier", "gt_analyze"}
|
||||
|
||||
|
||||
def test_gt_analyze_command_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from services.openclaw_channel import _dispatch_command
|
||||
|
||||
monkeypatch.delenv("GT_ANALYTICS_ENABLED", raising=False)
|
||||
reset_gt_engine()
|
||||
result = _dispatch_command("gt_analyze", {})
|
||||
assert result["ok"] is False
|
||||
|
||||
|
||||
def test_route_query_gt_backtest_intent() -> None:
|
||||
plan = route_query("Run GT historical backtest with Wilson confidence")
|
||||
assert plan["intent"] == "gt_backtest"
|
||||
assert plan["recommended"]["cmd"] == "gt_backtest"
|
||||
assert plan["recommended"]["args"]["expanded"] is True
|
||||
|
||||
|
||||
def test_gt_backtest_command_enabled(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from services.openclaw_channel import _dispatch_command
|
||||
|
||||
monkeypatch.setenv("GT_ANALYTICS_ENABLED", "true")
|
||||
reset_gt_engine()
|
||||
result = _dispatch_command("gt_backtest", {"expanded": True, "compact": True})
|
||||
assert result["ok"] is True
|
||||
data = result["data"]
|
||||
assert data["enabled"] is True
|
||||
assert data["accuracy"] == 1.0
|
||||
assert data["confidence_rate"] >= 0.95
|
||||
assert data["meets_target"] is True
|
||||
assert "cases" not in data
|
||||
|
||||
|
||||
def test_gt_backtest_command_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from services.openclaw_channel import _dispatch_command
|
||||
|
||||
monkeypatch.delenv("GT_ANALYTICS_ENABLED", raising=False)
|
||||
reset_gt_engine()
|
||||
result = _dispatch_command("gt_backtest", {})
|
||||
assert result["ok"] is True
|
||||
assert result["data"]["enabled"] is False
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Runtime profile detection for lean fleet nodes."""
|
||||
|
||||
from services import runtime_profile
|
||||
|
||||
|
||||
def test_resolve_profile_name_env_override(monkeypatch):
|
||||
monkeypatch.setenv("GT_ANALYTICS_PROFILE", "standard")
|
||||
monkeypatch.setattr(runtime_profile, "detect_cpu_limit", lambda: 1.0)
|
||||
assert runtime_profile.resolve_profile_name() == "standard"
|
||||
|
||||
|
||||
def test_resolve_profile_name_auto_lean_on_one_cpu(monkeypatch):
|
||||
monkeypatch.delenv("GT_ANALYTICS_PROFILE", raising=False)
|
||||
monkeypatch.setattr(runtime_profile, "detect_cpu_limit", lambda: 1.0)
|
||||
assert runtime_profile.resolve_profile_name() == "lean"
|
||||
|
||||
|
||||
def test_runtime_profile_payload(monkeypatch):
|
||||
monkeypatch.delenv("GT_ANALYTICS_PROFILE", raising=False)
|
||||
monkeypatch.setattr(runtime_profile, "detect_cpu_limit", lambda: 1.0)
|
||||
monkeypatch.setattr(runtime_profile, "detect_memory_limit_mb", lambda: 4096)
|
||||
runtime_profile.clear_runtime_profile_cache()
|
||||
payload = runtime_profile.get_runtime_profile()
|
||||
assert payload["profile"] == "lean"
|
||||
assert payload["cpu_limit"] == 1.0
|
||||
assert payload["gt_analytics"]["recommended"] is False
|
||||
assert payload["gt_analytics"]["lean_node"] is True
|
||||
assert "1 vCPU" in (payload["gt_analytics"]["warning"] or "")
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Telegram OSINT auto-translation."""
|
||||
|
||||
from services import telegram_translate
|
||||
|
||||
|
||||
def test_guess_source_lang_detects_cyrillic():
|
||||
assert telegram_translate.guess_source_lang("В Крым поедем несмотря ни на что") == "ru"
|
||||
|
||||
|
||||
def test_apply_post_translation_skips_english(monkeypatch):
|
||||
monkeypatch.setattr(telegram_translate, "telegram_translate_enabled", lambda: True)
|
||||
post = {
|
||||
"title": "Missile strike reported near Kyiv overnight.",
|
||||
"description": "Missile strike reported near Kyiv overnight.",
|
||||
}
|
||||
enriched = telegram_translate.apply_post_translation(post, "en")
|
||||
assert enriched["source_lang"] == "en"
|
||||
assert "title_translated" not in enriched
|
||||
|
||||
|
||||
def test_apply_post_translation_adds_fields(monkeypatch):
|
||||
monkeypatch.setattr(telegram_translate, "telegram_translate_enabled", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
telegram_translate,
|
||||
"translate_text",
|
||||
lambda text, target_lang=None: (
|
||||
"We will go to Crimea no matter what. This is our homeland!",
|
||||
"ru",
|
||||
),
|
||||
)
|
||||
post = {
|
||||
"title": "«В Крым поедем несмотря ни на что. Это наша родина!»",
|
||||
"description": "«В Крым поедем несмотря ни на что. Это наша родина!»",
|
||||
}
|
||||
enriched = telegram_translate.apply_post_translation(post, "en")
|
||||
assert enriched["source_lang"] == "ru"
|
||||
assert enriched["translate_to"] == "en"
|
||||
assert "Crimea" in enriched["title_translated"]
|
||||
|
||||
|
||||
def test_normalize_translate_target_maps_ui_locales():
|
||||
assert telegram_translate.normalize_translate_target("zh-CN") == "zh-CN"
|
||||
assert telegram_translate.normalize_translate_target("fr") == "fr"
|
||||
|
||||
|
||||
def test_source_lang_label_avoids_uk_country_confusion():
|
||||
assert telegram_translate.source_lang_label("uk") == "Ukrainian"
|
||||
assert telegram_translate.source_lang_label("ru") == "Russian"
|
||||
|
||||
|
||||
def test_polish_translation_expands_bpla_shorthand():
|
||||
assert "UAV" in telegram_translate.polish_translation("Kyiv 1x BpLa on Rembazu.")
|
||||
|
||||
|
||||
def test_guess_source_lang_prefers_ukrainian_markers():
|
||||
assert telegram_translate.guess_source_lang("Київ 1х БпЛА") == "uk"
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Telegram OSINT watchdog and search helpers."""
|
||||
|
||||
from services import openclaw_watchdog
|
||||
from services.telegram_osint_text import keyword_matches_telegram_post, telegram_post_search_text
|
||||
|
||||
|
||||
def _telegram_slow_fixture() -> dict:
|
||||
return {
|
||||
"telegram_osint": {
|
||||
"posts": [
|
||||
{
|
||||
"id": "tg-uk-1",
|
||||
"title": "Київ 1х БпЛА на Рембазу.",
|
||||
"description": "Київ 1х БпЛА на Рембазу.",
|
||||
"title_translated": "Kyiv 1x UAV on Rembazu.",
|
||||
"description_translated": "Kyiv 1x UAV on Rembazu.",
|
||||
"channel": "war_monitor",
|
||||
"source": "t.me/war_monitor",
|
||||
"link": "https://t.me/war_monitor/101",
|
||||
"risk_score": 3,
|
||||
"source_lang": "uk",
|
||||
},
|
||||
{
|
||||
"id": "tg-ru-1",
|
||||
"title": "«В Крым поедем несмотря ни на что. Это наша родина!»",
|
||||
"description": "«В Крым поедем несмотря ни на что. Это наша родина!»",
|
||||
"title_translated": "We will go to Crimea no matter what. This is our homeland!",
|
||||
"description_translated": "We will go to Crimea no matter what. This is our homeland!",
|
||||
"channel": "nexta_live",
|
||||
"source": "t.me/nexta_live",
|
||||
"link": "https://t.me/nexta_live/202",
|
||||
"risk_score": 9,
|
||||
"source_lang": "ru",
|
||||
},
|
||||
],
|
||||
"total": 2,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_telegram_post_search_text_includes_translated_fields():
|
||||
post = _telegram_slow_fixture()["telegram_osint"]["posts"][0]
|
||||
haystack = telegram_post_search_text(post)
|
||||
assert "kyiv 1x uav on rembazu" in haystack
|
||||
assert "бпла" in haystack
|
||||
|
||||
|
||||
def test_keyword_matches_telegram_post_searches_translated_and_original():
|
||||
post = _telegram_slow_fixture()["telegram_osint"]["posts"][1]
|
||||
assert keyword_matches_telegram_post(post, "crimea")
|
||||
assert keyword_matches_telegram_post(post, "крым")
|
||||
|
||||
|
||||
def test_watchdog_keyword_matches_telegram_translation(monkeypatch):
|
||||
monkeypatch.setattr(openclaw_watchdog, "_ensure_running", lambda: None)
|
||||
openclaw_watchdog.clear_watches()
|
||||
try:
|
||||
watch = openclaw_watchdog.add_watch("keyword", {"keyword": "crimea"})
|
||||
alert = openclaw_watchdog._check_keyword(watch["id"], {"keyword": "crimea"}, {}, _telegram_slow_fixture())
|
||||
assert alert is not None
|
||||
assert any(match["source"] == "telegram_osint" for match in alert["data"]["matches"])
|
||||
assert alert["data"]["matches"][0]["title"].startswith("We will go to Crimea")
|
||||
# Same Telegram post should not re-alert once seen.
|
||||
assert openclaw_watchdog._check_keyword(watch["id"], {"keyword": "crimea"}, {}, _telegram_slow_fixture()) is None
|
||||
finally:
|
||||
openclaw_watchdog.clear_watches()
|
||||
|
||||
|
||||
def test_watchdog_telegram_rhetoric_alerts_on_high_risk_posts(monkeypatch):
|
||||
monkeypatch.setattr(openclaw_watchdog, "_ensure_running", lambda: None)
|
||||
openclaw_watchdog.clear_watches()
|
||||
try:
|
||||
watch = openclaw_watchdog.add_watch("telegram_rhetoric", {"min_risk_score": 8})
|
||||
alert = openclaw_watchdog._check_telegram_rhetoric(watch["id"], {"min_risk_score": 8}, _telegram_slow_fixture())
|
||||
assert alert is not None
|
||||
assert "Telegram rhetoric alert" in alert["alert"]
|
||||
assert len(alert["data"]["matches"]) == 1
|
||||
assert alert["data"]["matches"][0]["channel"] == "nexta_live"
|
||||
assert alert["data"]["matches"][0]["risk_score"] == 9
|
||||
assert openclaw_watchdog._check_telegram_rhetoric(watch["id"], {"min_risk_score": 8}, _telegram_slow_fixture()) is None
|
||||
finally:
|
||||
openclaw_watchdog.clear_watches()
|
||||
|
||||
|
||||
def test_watchdog_telegram_rhetoric_supports_channel_filter(monkeypatch):
|
||||
monkeypatch.setattr(openclaw_watchdog, "_ensure_running", lambda: None)
|
||||
openclaw_watchdog.clear_watches()
|
||||
try:
|
||||
watch = openclaw_watchdog.add_watch(
|
||||
"telegram_rhetoric",
|
||||
{"min_risk_score": 7, "channels": ["war_monitor"]},
|
||||
)
|
||||
alert = openclaw_watchdog._check_telegram_rhetoric(
|
||||
watch["id"],
|
||||
{"min_risk_score": 7, "channels": ["war_monitor"]},
|
||||
_telegram_slow_fixture(),
|
||||
)
|
||||
assert alert is None # war_monitor post is only risk 3
|
||||
finally:
|
||||
openclaw_watchdog.clear_watches()
|
||||
Reference in New Issue
Block a user