Expose new telemetry and recon toolkit to OpenClaw agents.

Wire telegram_osint, malware, cyber, and SCM into search/slow-tier helpers; add osint_lookup, entity_expand, and osint_sweep commands; update README and skill docs.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
BigBodyCobain
2026-06-08 22:44:16 -06:00
parent af9b3d08cc
commit 1583fd5715
9 changed files with 793 additions and 44 deletions
+74 -3
View File
@@ -1705,11 +1705,12 @@ async def agent_tool_manifest(request: Request):
{
"name": "search_news",
"type": "read",
"description": "Search news and event layers server-side by keyword. Includes news, GDELT, CrowdThreat, and major incident/event feeds without pulling the full slow telemetry feed.",
"description": "Search news and event layers server-side by keyword. Includes news, GDELT, CrowdThreat, Telegram OSINT, and major incident/event feeds without pulling the full slow telemetry feed.",
"parameters": {
"query": {"type": "string", "required": True, "description": "Keyword or phrase to search for"},
"limit": {"type": "integer", "required": False, "description": "Max results (default 10, max 50)"},
"include_gdelt": {"type": "boolean", "required": False, "description": "Include GDELT matches (default true)"},
"include_telegram": {"type": "boolean", "required": False, "description": "Include Telegram OSINT channel posts (default true)"},
"compact": {"type": "boolean", "required": False, "description": "If true, strips empty/None fields from each result and rounds lat/lng to 3 decimals. Response includes format: 'compressed_v1'."},
},
"returns": "{results: [{source_layer, title, summary, source, link, lat, lng, risk_score}], version: int, truncated: bool}",
@@ -1743,6 +1744,55 @@ async def agent_tool_manifest(request: Request):
},
"returns": "{center, radius_km, nearby, topic_news, context_layers}",
},
{
"name": "osint_lookup",
"type": "read",
"description": "Run a passive OSINT recon lookup server-side (same backends as the Recon panel). SSRF-guarded outbound proxies for IP geolocation, DNS, WHOIS, certs, BGP/ASN, sanctions, CVE, MAC vendor, GitHub profile, breach checks, and threat feeds.",
"parameters": {
"tool": {"type": "string", "required": True, "description": "Lookup type: ip, dns, whois, certs, threats, bgp, sanctions, cve, mac, github, leaks, sweep_init"},
"ip": {"type": "string", "required": False, "description": "IPv4/IPv6 for ip or sweep_init"},
"domain": {"type": "string", "required": False, "description": "Domain for dns, whois, certs"},
"query": {"type": "string", "required": False, "description": "Generic query (BGP ASN, sanctions name, optional threats filter)"},
"cve": {"type": "string", "required": False, "description": "CVE id for cve lookup"},
"mac": {"type": "string", "required": False, "description": "MAC address for mac lookup"},
"username": {"type": "string", "required": False, "description": "GitHub username"},
"email": {"type": "string", "required": False, "description": "Email for breach/leak lookup"},
"schema": {"type": "string", "required": False, "description": "Sanctions schema filter: Person, Organization, Company, Vessel, Airplane, LegalEntity"},
"limit": {"type": "integer", "required": False, "description": "Sanctions result cap (default 25, max 100)"},
"cidr": {"type": "integer", "required": False, "description": "CIDR mask for sweep_init (24-32, default 24)"},
},
"returns": "Tool-specific JSON (geo, DNS records, WHOIS, sanctions hits, CVE details, etc.)",
},
{
"name": "osint_tools",
"type": "read",
"description": "List available OSINT recon tools, entity-expand types, and sanctions schemas.",
"parameters": {},
"returns": "{tools: [...], entity_types: [...], sanctions_schemas: [...], notes: {...}}",
},
{
"name": "entity_expand",
"type": "read",
"description": "Expand an entity relationship graph around an aircraft, vessel, IP, company, person, or country. Same backend as /api/entity/expand.",
"parameters": {
"type": {"type": "string", "required": True, "description": "Entity type: aircraft, vessel, company, person, ip, country"},
"id": {"type": "string", "required": True, "description": "Entity identifier (tail number, MMSI, IP, company name, etc.)"},
"registration": {"type": "string", "required": False, "description": "Aircraft registration hint"},
"model": {"type": "string", "required": False, "description": "Aircraft model hint"},
"icao24": {"type": "string", "required": False, "description": "ICAO24 hex for aircraft"},
},
"returns": "{nodes: [...], links: [...]}",
},
{
"name": "osint_sweep",
"type": "write",
"description": "Active subnet device discovery via Shodan InternetDB (ports, vulns, hostnames). Requires full OpenClaw access tier. Private/reserved IPs blocked.",
"parameters": {
"ip": {"type": "string", "required": True, "description": "Public IPv4 anchor for the sweep"},
"cidr": {"type": "integer", "required": False, "description": "Subnet size /24-/32 (default 24)"},
},
"returns": "{center, target_ip, cidr, subnet, devices, summary, sweep_time_ms}",
},
{
"name": "what_changed",
"type": "read",
@@ -2194,6 +2244,11 @@ async def agent_tool_manifest(request: Request):
"Prefer compact lookups first: search_telemetry, find_flights, find_ships, search_news, entities_near, get_layer_slice. Use get_telemetry/get_slow_telemetry/get_report only when focused commands are insufficient.",
"ShadowBroker does expose UAP sightings, wastewater, and tracked_flights/VIP aircraft when those layers are populated. Verify with get_summary or get_layer_slice before claiming a layer is absent.",
"ShadowBroker also exposes fishing_activity, which is the fishing-vessel activity layer backed by Global Fishing Watch data when GFW_API_TOKEN is configured. Do not confuse it with the AIS ships layer.",
"telegram_osint, malware_threats, cyber_threats, and scm_suppliers are live map layers. Use get_summary or get_layer_slice(['telegram_osint']) before claiming they are absent. Aliases: telegram, malware/botnet, cyber/cisa/kev, scm/suppliers.",
"search_telemetry and search_news both index Telegram OSINT posts. For malware C2, botnet IPs, CISA KEV CVEs, or semiconductor suppliers, use search_telemetry or get_layer_slice on the matching layer.",
"The Recon toolkit is available via osint_lookup: IP geolocation, DNS, WHOIS, certs, BGP, sanctions, CVE, MAC vendor, GitHub, breach checks, threat feeds. Call osint_tools first to list supported tools.",
"entity_expand builds relationship graphs for aircraft, vessels, IPs, companies, people, and countries — use after resolving an entity from telemetry or osint_lookup.",
"osint_sweep runs active subnet discovery (Shodan InternetDB) and requires full OpenClaw access tier. Use osint_lookup tool=sweep_init for passive geolocation context only.",
"Use search_telemetry as the Google-style entry point whenever the user gives you a person, place, company, topic, owner, nickname, or natural-language phrase and you do not already know the source layer.",
"Example: for 'Where is Jerry Jones yacht?' search 'Jerry Jones' across all telemetry first, identify the ship match, then refine with find_ships or raw layer context only if needed.",
"For fuzzy natural-language lookups like 'Patriots jet' or 'Jerry Jones yacht', use search_telemetry first and inspect the ranked candidate list before making a hard claim.",
@@ -2354,13 +2409,29 @@ async def api_capabilities(request: Request):
"description": "Universal compact search across telemetry when the entity type or source layer is not obvious.",
},
"search_news": {
"args": {"query": "str", "limit": "int (default 10)", "include_gdelt": "bool (default true)"},
"description": "Search news and event layers by keyword without pulling the whole slow feed.",
"args": {"query": "str", "limit": "int (default 10)", "include_gdelt": "bool (default true)", "include_telegram": "bool (default true)"},
"description": "Search news and event layers by keyword without pulling the whole slow feed. Includes Telegram OSINT when include_telegram is true.",
},
"entities_near": {
"args": {"lat": "float", "lng": "float", "radius_km": "float (default 50)", "entity_types": "list[str] (optional)", "limit": "int (default 25)"},
"description": "Compact proximity search around a point across selected layers.",
},
"osint_lookup": {
"args": {"tool": "str (ip|dns|whois|certs|threats|bgp|sanctions|cve|mac|github|leaks|sweep_init)", "...": "tool-specific params"},
"description": "Passive OSINT recon lookup — same backends as the Recon panel.",
},
"osint_tools": {
"args": {},
"description": "List available recon tools and entity-expand types.",
},
"entity_expand": {
"args": {"type": "str", "id": "str", "registration": "str (optional)", "icao24": "str (optional)"},
"description": "Entity relationship graph expansion.",
},
"osint_sweep": {
"args": {"ip": "str", "cidr": "int (default 24)"},
"description": "Active subnet scan — requires full access tier.",
},
"brief_area": {
"args": {"lat": "float", "lng": "float", "radius_km": "float (default 50)", "entity_types": "list[str] (optional)", "query": "str (optional)", "limit": "int (default 25)", "context_limit": "int (default 10)"},
"description": "One compact area brief: nearby aircraft/ships/entities, optional topic news, and selected context layers.",
+27
View File
@@ -83,6 +83,10 @@ READ_COMMANDS = frozenset({
"sar_pin_click",
# Analysis zones (OpenClaw map overlays)
"list_analysis_zones",
# Recon / OSINT toolkit (server-side proxies, SSRF guarded)
"osint_lookup",
"osint_tools",
"entity_expand",
})
WRITE_COMMANDS = frozenset({
@@ -112,6 +116,8 @@ WRITE_COMMANDS = frozenset({
"place_analysis_zone",
"delete_analysis_zone",
"clear_analysis_zones",
# Active recon (subnet device discovery)
"osint_sweep",
})
@@ -780,6 +786,7 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
query=str(args.get("query", "") or ""),
limit=args.get("limit", 10),
include_gdelt=bool(args.get("include_gdelt", True)),
include_telegram=bool(args.get("include_telegram", True)),
)
if _wants_compact(args):
return {"ok": True, "data": _compact_query_result(result), "format": "compressed_v1"}
@@ -846,6 +853,26 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
return {"ok": True, "data": _compact_query_result(result), "format": "compressed_v1"}
return {"ok": True, "data": result}
if cmd == "osint_lookup":
from services.osint.openclaw_recon import run_osint_lookup
tool = str(args.get("tool", "") or args.get("lookup", "") or args.get("type", "") or "")
result = run_osint_lookup(tool, args)
return {"ok": True, "data": result, "tool": tool.strip().lower()}
if cmd == "osint_tools":
from services.osint.openclaw_recon import osint_tool_help
return {"ok": True, "data": osint_tool_help()}
if cmd == "osint_sweep":
from services.osint.openclaw_recon import run_osint_sweep
result = run_osint_sweep(args)
return {"ok": True, "data": result}
if cmd == "entity_expand":
from services.osint.openclaw_recon import run_entity_expand
result = run_entity_expand(args)
return {"ok": True, "data": result}
if cmd == "get_report":
from services.telemetry import get_cached_telemetry_refs, get_cached_slow_telemetry_refs
fast = get_cached_telemetry_refs()
+135
View File
@@ -0,0 +1,135 @@
"""OpenClaw dispatch for the operator recon / OSINT lookup toolkit."""
from __future__ import annotations
from typing import Any
from services.osint import lookups
from services.osint_intel.resolve import ALLOWED_TYPES, resolve_entity
_OSINT_TOOLS: dict[str, str] = {
"ip": "ip",
"dns": "domain",
"whois": "domain",
"certs": "domain",
"threats": "query",
"bgp": "query",
"sanctions": "query",
"cve": "cve",
"mac": "mac",
"github": "username",
"leaks": "email",
"sweep_init": "ip",
}
_ENTITY_SCHEMAS = frozenset({
"Person",
"Organization",
"Company",
"Vessel",
"Airplane",
"LegalEntity",
})
def _require_str(args: dict[str, Any], *keys: str) -> str:
for key in keys:
value = str(args.get(key, "") or "").strip()
if value:
return value
joined = "/".join(keys)
raise ValueError(f"Missing required argument: {joined}")
def run_osint_lookup(tool: str, args: dict[str, Any]) -> dict[str, Any]:
"""Run a passive OSINT lookup (same backends as /api/osint/*)."""
name = str(tool or "").strip().lower().replace("-", "_")
if name not in _OSINT_TOOLS:
allowed = ", ".join(sorted(_OSINT_TOOLS))
raise ValueError(f"Unknown OSINT tool '{tool}'. Allowed: {allowed}")
if name == "ip":
return lookups.lookup_ip(_require_str(args, "ip", "query", "value"))
if name == "dns":
return lookups.lookup_dns(_require_str(args, "domain", "query", "value"))
if name == "whois":
return lookups.lookup_whois(_require_str(args, "domain", "query", "value"))
if name == "certs":
return lookups.lookup_certs(_require_str(args, "domain", "query", "value"))
if name == "threats":
query = str(args.get("query", "") or args.get("value", "") or "").strip() or None
return lookups.lookup_threats(query)
if name == "bgp":
return lookups.lookup_bgp(_require_str(args, "query", "asn", "value"))
if name == "sanctions":
query = _require_str(args, "query", "name", "value")
schema = str(args.get("schema", "") or "").strip() or None
if schema and schema not in _ENTITY_SCHEMAS:
allowed = ", ".join(sorted(_ENTITY_SCHEMAS))
raise ValueError(f"Invalid schema. Allowed: {allowed}")
limit = args.get("limit", 25)
try:
limit = int(limit)
except (TypeError, ValueError):
limit = 25
limit = max(1, min(100, limit))
return lookups.lookup_sanctions(query, schema=schema, limit=limit)
if name == "cve":
return lookups.lookup_cve(_require_str(args, "cve", "query", "value"))
if name == "mac":
return lookups.lookup_mac(_require_str(args, "mac", "query", "value"))
if name == "github":
return lookups.lookup_github(_require_str(args, "username", "user", "query", "value"))
if name == "leaks":
return lookups.lookup_leaks(_require_str(args, "email", "query", "value"))
if name == "sweep_init":
ip = _require_str(args, "ip", "query", "value")
cidr = args.get("cidr", 24)
try:
cidr = int(cidr)
except (TypeError, ValueError):
cidr = 24
return lookups.sweep_init(ip, cidr)
raise ValueError(f"Unhandled OSINT tool: {name}")
def run_osint_sweep(args: dict[str, Any]) -> dict[str, Any]:
"""Run subnet device discovery (Shodan InternetDB proxy). Requires full access tier."""
ip = _require_str(args, "ip", "query", "value")
cidr = args.get("cidr", 24)
try:
cidr = int(cidr)
except (TypeError, ValueError):
cidr = 24
subnet = lookups.subnet_start_for(ip, cidr)
scan = lookups.sweep_scan(subnet, cidr)
init = lookups.sweep_init(ip, cidr)
return {**init, **scan, "subnet": f"{subnet}/{cidr}"}
def run_entity_expand(args: dict[str, Any]) -> dict[str, Any]:
"""Expand an entity graph node (aircraft, vessel, IP, company, person, country)."""
entity_type = _require_str(args, "type", "entity_type")
entity_id = _require_str(args, "id", "entity_id", "query", "value")
props = {
"label": entity_id,
"registration": str(args.get("registration", "") or "").strip() or None,
"model": str(args.get("model", "") or "").strip() or None,
"icao24": str(args.get("icao24", "") or "").strip() or None,
}
props = {key: value for key, value in props.items() if value is not None}
return resolve_entity(entity_type, entity_id, props)
def osint_tool_help() -> dict[str, Any]:
"""Discovery metadata for agents."""
return {
"tools": sorted(_OSINT_TOOLS),
"entity_types": sorted(ALLOWED_TYPES),
"sanctions_schemas": sorted(_ENTITY_SCHEMAS),
"notes": {
"osint_lookup": "Passive lookups — same data as the Recon panel /api/osint/* routes.",
"osint_sweep": "Active subnet scan via Shodan InternetDB — requires full OpenClaw access tier.",
"entity_expand": "Build a relationship graph around aircraft, vessels, IPs, companies, people, or countries.",
},
}
+200 -30
View File
@@ -93,6 +93,10 @@ _SLOW_KEYS = (
"sar_scenes",
"sar_anomalies",
"sar_aoi_coverage",
"malware_threats",
"cyber_threats",
"scm_suppliers",
"telegram_osint",
)
@@ -158,6 +162,20 @@ _ENTITY_LAYER_ALIASES = {
"uap_sightings": "uap_sightings",
"wastewater": "wastewater",
"pins": "pins",
"telegram": "telegram_osint",
"telegram_osint": "telegram_osint",
"osint_feed": "telegram_osint",
"malware": "malware_threats",
"malware_threats": "malware_threats",
"malware_c2": "malware_threats",
"botnet": "malware_threats",
"cyber": "cyber_threats",
"cyber_threats": "cyber_threats",
"cisa": "cyber_threats",
"kev": "cyber_threats",
"scm": "scm_suppliers",
"scm_suppliers": "scm_suppliers",
"suppliers": "scm_suppliers",
}
_SLICEABLE_LAYERS = tuple(dict.fromkeys(_FAST_KEYS + _SLOW_KEYS))
@@ -188,6 +206,21 @@ _LAYER_ALIASES = {
"sar_coverage": "sar_aoi_coverage",
# Satellite analysis (maneuvers, decay, Starlink)
"satellite_analysis": "satellite_analysis",
# OSINT / cyber / supply-chain overlays
"telegram": "telegram_osint",
"telegram_osint": "telegram_osint",
"osint_feed": "telegram_osint",
"malware": "malware_threats",
"malware_threats": "malware_threats",
"malware_c2": "malware_threats",
"botnet": "malware_threats",
"cyber": "cyber_threats",
"cyber_threats": "cyber_threats",
"cisa": "cyber_threats",
"kev": "cyber_threats",
"scm": "scm_suppliers",
"scm_suppliers": "scm_suppliers",
"suppliers": "scm_suppliers",
}
_UNIVERSAL_SEARCH_DEFAULT_LAYERS = (
@@ -225,6 +258,10 @@ _UNIVERSAL_SEARCH_DEFAULT_LAYERS = (
"tinygs_satellites",
"psk_reporter",
"ukraine_alerts",
"telegram_osint",
"malware_threats",
"cyber_threats",
"scm_suppliers",
)
_GENERIC_QUERY_STOPWORDS = {
@@ -269,7 +306,19 @@ _GENERIC_LAYER_HINTS: dict[str, tuple[str, ...]] = {
"protest": ("crowdthreat", "gdelt", "news", "frontlines", "liveuamap"),
"riot": ("crowdthreat", "gdelt", "news", "frontlines", "liveuamap"),
"event": ("crowdthreat", "gdelt", "news", "frontlines", "liveuamap"),
"news": ("news", "gdelt", "crowdthreat", "frontlines", "liveuamap"),
"news": ("news", "gdelt", "crowdthreat", "frontlines", "liveuamap", "telegram_osint"),
"telegram": ("telegram_osint",),
"osint": ("telegram_osint", "gdelt", "news", "crowdthreat"),
"channel": ("telegram_osint",),
"malware": ("malware_threats",),
"botnet": ("malware_threats",),
"c2": ("malware_threats",),
"cve": ("cyber_threats",),
"cisa": ("cyber_threats",),
"cyber": ("cyber_threats", "malware_threats"),
"supplier": ("scm_suppliers",),
"scm": ("scm_suppliers",),
"semiconductor": ("scm_suppliers",),
"plant": ("power_plants", "wastewater"),
"datacenter": ("datacenters",),
"data": ("datacenters",),
@@ -314,6 +363,10 @@ _SEARCH_GROUP_BY_LAYER = {
"kiwisdr": "signals",
"psk_reporter": "signals",
"ukraine_alerts": "events",
"telegram_osint": "events",
"malware_threats": "cyber",
"cyber_threats": "cyber",
"scm_suppliers": "infrastructure",
}
_SEARCH_QUERY_SYNONYMS: dict[str, tuple[str, ...]] = {
@@ -328,6 +381,9 @@ _SEARCH_QUERY_SYNONYMS: dict[str, tuple[str, ...]] = {
"plants": ("plant",),
"cameras": ("camera",),
"radios": ("radio",),
"telegrams": ("telegram",),
"channels": ("channel",),
"suppliers": ("supplier",),
}
_SEARCH_INDEX_LOCK = threading.Lock()
@@ -653,6 +709,42 @@ _UNIVERSAL_SEARCH_SPECS: dict[str, dict[str, Any]] = {
"id_fields": ("id",),
"time_fields": ("updated_at", "timestamp"),
},
"telegram_osint": {
"fields": ("title", "description", "source", "channel", "link"),
"primary_fields": ("title", "description", "channel"),
"label_fields": ("title", "channel"),
"summary_fields": ("description", "source"),
"type_fields": ("channel", "source"),
"id_fields": ("id", "link"),
"time_fields": ("published", "timestamp"),
},
"malware_threats": {
"fields": ("ip", "malware", "status", "country", "threat_type"),
"primary_fields": ("ip", "malware", "country"),
"label_fields": ("ip", "malware"),
"summary_fields": ("status", "country", "threat_type"),
"type_fields": ("threat_type", "malware"),
"id_fields": ("id", "ip"),
"time_fields": ("first_seen", "last_online", "timestamp"),
},
"cyber_threats": {
"fields": ("id", "name", "vendor", "product", "severity", "source"),
"primary_fields": ("id", "name", "vendor", "product"),
"label_fields": ("id", "name"),
"summary_fields": ("vendor", "product", "severity", "source"),
"type_fields": ("severity", "source"),
"id_fields": ("id",),
"time_fields": ("date", "due", "timestamp"),
},
"scm_suppliers": {
"fields": ("name", "city", "country", "category", "risk_level"),
"primary_fields": ("name", "city", "country", "category"),
"label_fields": ("name", "city"),
"summary_fields": ("country", "category", "risk_level"),
"type_fields": ("category", "risk_level"),
"id_fields": ("id",),
"time_fields": ("timestamp",),
},
}
@@ -734,6 +826,11 @@ def _extract_coords(candidate: dict[str, Any]) -> tuple[float | None, float | No
if isinstance(coords, (list, tuple)) and len(coords) >= 2:
lng = lng if lng is not None else _coerce_float(coords[0])
lat = lat if lat is not None else _coerce_float(coords[1])
if lat is None or lng is None:
coords = candidate.get("coords")
if isinstance(coords, (list, tuple)) and len(coords) >= 2:
lat = lat if lat is not None else _coerce_float(coords[0])
lng = lng if lng is not None else _coerce_float(coords[1])
return lat, lng
@@ -832,6 +929,53 @@ def _layer_group(layer: str) -> str:
return _SEARCH_GROUP_BY_LAYER.get(layer, "other")
_LAYER_NESTED_LIST_KEYS: dict[str, tuple[str, ...]] = {
"telegram_osint": ("posts",),
"malware_threats": ("threats",),
"cyber_threats": ("threats",),
"scm_suppliers": ("suppliers",),
}
_DEFAULT_NESTED_LIST_KEYS = (
"items",
"results",
"vessels",
"features",
"posts",
"threats",
"suppliers",
)
def _unwrap_layer_items(value: Any, layer: str = "") -> list[Any]:
"""Return the searchable/geospatial item list inside a layer value."""
if isinstance(value, list):
return value
if not isinstance(value, dict):
return []
keys = _LAYER_NESTED_LIST_KEYS.get(layer, _DEFAULT_NESTED_LIST_KEYS)
for key in keys:
nested = value.get(key)
if isinstance(nested, list):
return nested
return []
def _layer_record_count(value: Any, layer: str = "") -> int:
if isinstance(value, list):
return len(value)
if isinstance(value, dict):
total = value.get("total")
if isinstance(total, (int, float)):
return int(total)
items = _unwrap_layer_items(value, layer)
if items:
return len(items)
return len(value) if value else 0
if value is None:
return 0
return 1
def _build_search_document(doc_id: int, layer: str, candidate: dict[str, Any], spec: dict[str, Any]) -> dict[str, Any]:
fields = tuple(spec.get("fields", ()))
text = _document_text(candidate, fields)
@@ -880,9 +1024,7 @@ def _get_search_index() -> dict[str, Any]:
for layer in layers:
spec = _UNIVERSAL_SEARCH_SPECS[layer]
items = snap.get(layer) or []
if isinstance(items, dict):
items = items.get("items", []) or items.get("results", []) or items.get("vessels", [])
items = _unwrap_layer_items(snap.get(layer), layer)
if not isinstance(items, list):
continue
for item in items:
@@ -1144,18 +1286,9 @@ def get_telemetry_summary() -> dict[str, Any]:
for layer in layer_names:
value = snap.get(layer)
if isinstance(value, list):
counts[layer] = len(value)
if value:
non_empty_layers.append(layer)
elif isinstance(value, dict):
counts[layer] = len(value)
if value:
non_empty_layers.append(layer)
elif value is None:
counts[layer] = 0
else:
counts[layer] = 1
count = _layer_record_count(value, layer)
counts[layer] = count
if count > 0:
non_empty_layers.append(layer)
alias_examples = {
@@ -1167,6 +1300,16 @@ def get_telemetry_summary() -> dict[str, Any]:
"tracked": "tracked_flights",
"military": "military_flights",
"jets": "private_jets",
"telegram": "telegram_osint",
"osint_feed": "telegram_osint",
"malware": "malware_threats",
"malware_c2": "malware_threats",
"botnet": "malware_threats",
"cyber": "cyber_threats",
"cisa": "cyber_threats",
"kev": "cyber_threats",
"scm": "scm_suppliers",
"suppliers": "scm_suppliers",
}
return {
@@ -1577,14 +1720,7 @@ def _nearby_items_from_layers(
snap = get_latest_data_subset_refs(*layers)
out: dict[str, list[dict[str, Any]]] = {}
for layer in layers:
value = snap.get(layer) or []
if isinstance(value, dict):
if layer == "gdelt" and isinstance(value.get("features"), list):
items = value.get("features") or []
else:
items = value.get("items") or value.get("features") or value.get("vessels") or []
else:
items = value
items = _unwrap_layer_items(snap.get(layer), layer)
if not isinstance(items, list):
continue
matches: list[dict[str, Any]] = []
@@ -1728,6 +1864,9 @@ def correlate_entity(
"crowdthreat",
"frontlines",
"liveuamap",
"telegram_osint",
"malware_threats",
"scm_suppliers",
"military_bases",
"datacenters",
"power_plants",
@@ -1809,13 +1948,17 @@ def search_news(
query: str,
limit: int = 10,
include_gdelt: bool = True,
include_telegram: bool = True,
) -> dict[str, Any]:
"""Search news and event layers server-side and return a compact result set."""
query_norm = _norm_text(query)
if not query_norm:
return {"results": [], "version": get_data_version(), "truncated": False}
snap = get_latest_data_subset_refs("news", "gdelt", "crowdthreat", "liveuamap", "frontlines")
layer_keys = ["news", "gdelt", "crowdthreat", "liveuamap", "frontlines"]
if include_telegram:
layer_keys.append("telegram_osint")
snap = get_latest_data_subset_refs(*layer_keys)
out: list[dict[str, Any]] = []
limit = _coerce_limit(limit, default=10, maximum=50)
@@ -1941,6 +2084,36 @@ def search_news(
if len(out) >= limit:
return {"results": out, "version": get_data_version(), "truncated": True}
if include_telegram:
for post in _unwrap_layer_items(snap.get("telegram_osint"), "telegram_osint"):
if not isinstance(post, dict):
continue
text = " ".join(
(
_norm_text(post.get("title")),
_norm_text(post.get("description")),
_norm_text(post.get("source")),
_norm_text(post.get("channel")),
)
)
if not _text_matches_query(query_norm, text):
continue
lat, lng = _extract_coords(post)
out.append(
{
"source_layer": "telegram_osint",
"title": post.get("title") or "",
"summary": post.get("description") or "",
"source": post.get("source") or post.get("channel") or "Telegram",
"link": post.get("link") or "",
"lat": lat,
"lng": lng,
"risk_score": post.get("risk_score"),
}
)
if len(out) >= limit:
return {"results": out, "version": get_data_version(), "truncated": True}
return {"results": out, "version": get_data_version(), "truncated": False}
@@ -2205,16 +2378,13 @@ def entities_near(
out: list[dict[str, Any]] = []
for layer in layers:
items = snap.get(layer) or []
if isinstance(items, dict):
items = items.get("vessels", []) or items.get("items", [])
items = _unwrap_layer_items(snap.get(layer), layer)
if not isinstance(items, list):
continue
for item in items:
if not isinstance(item, dict):
continue
item_lat = _coerce_float(item.get("lat") or item.get("latitude"))
item_lng = _coerce_float(item.get("lng") or item.get("lon") or item.get("longitude"))
item_lat, item_lng = _extract_coords(item)
if item_lat is None or item_lng is None:
continue
distance = _haversine_km(center_lat, center_lng, item_lat, item_lng)
@@ -28,6 +28,10 @@ def sample_store():
"weather_alerts": list(latest_data.get("weather_alerts") or []),
"gps_jamming": list(latest_data.get("gps_jamming") or []),
"military_bases": list(latest_data.get("military_bases") or []),
"telegram_osint": dict(latest_data.get("telegram_osint") or {}),
"malware_threats": dict(latest_data.get("malware_threats") or {}),
"cyber_threats": dict(latest_data.get("cyber_threats") or {}),
"scm_suppliers": dict(latest_data.get("scm_suppliers") or {}),
}
latest_data["tracked_flights"] = [
{
@@ -188,6 +192,66 @@ def sample_store():
"lng": -76.87,
}
]
latest_data["telegram_osint"] = {
"posts": [
{
"id": "tg-1",
"title": "Missile strike reported near Kyiv overnight",
"description": "OSINT channel reports explosions near Kyiv",
"channel": "osintdefender",
"source": "t.me/osintdefender",
"link": "https://t.me/osintdefender/123",
"published": "2026-06-02T12:00:00+00:00",
"risk_score": 0.8,
"coords": [50.45, 30.52],
}
],
"total": 1,
"geolocated": 1,
}
latest_data["malware_threats"] = {
"threats": [
{
"id": "feodo-1",
"ip": "203.0.113.10",
"malware": "Emotet",
"country": "US",
"threat_type": "botnet_c2",
"lat": 38.95,
"lng": -77.45,
}
],
"total": 1,
}
latest_data["cyber_threats"] = {
"threats": [
{
"id": "CVE-2026-1234",
"name": "Example Vendor RCE",
"vendor": "Example Vendor",
"product": "Example Product",
"severity": "CRITICAL",
"source": "CISA KEV",
}
],
"stats": {"active_cves": 1},
}
latest_data["scm_suppliers"] = {
"suppliers": [
{
"id": "sup-tsmc-hsinchu",
"name": "TSMC Fab 12 (Tier 1)",
"city": "Hsinchu",
"country": "Taiwan",
"category": "Semiconductor",
"risk_level": "NORMAL",
"lat": 24.774,
"lng": 120.992,
}
],
"total": 1,
"critical_count": 0,
}
try:
yield
@@ -475,6 +539,89 @@ def test_correlate_entity_returns_evidence_pack_near_aircraft(sample_store, monk
assert result["recommended_next"]
def test_get_slow_telemetry_includes_new_osint_layers(sample_store, monkeypatch):
import services.telemetry as telemetry
monkeypatch.setattr(telemetry, "get_data_version", lambda: 210)
result = telemetry.get_cached_slow_telemetry()
assert "telegram_osint" in result
assert result["telegram_osint"]["total"] == 1
assert "malware_threats" in result
assert result["malware_threats"]["total"] == 1
assert "scm_suppliers" in result
assert result["scm_suppliers"]["total"] == 1
def test_get_layer_slice_accepts_telegram_alias(sample_store, monkeypatch):
import services.telemetry as telemetry
monkeypatch.setattr(telemetry, "get_data_version", lambda: 211)
result = telemetry.get_layer_slice(layers=["telegram"], limit_per_layer=10)
assert result["requested_layers"] == ["telegram_osint"]
assert result["layers"]["telegram_osint"]["posts"][0]["channel"] == "osintdefender"
def test_get_telemetry_summary_counts_nested_layer_items(sample_store, monkeypatch):
import services.telemetry as telemetry
monkeypatch.setattr(telemetry, "get_data_version", lambda: 212)
result = telemetry.get_telemetry_summary()
assert result["counts"]["telegram_osint"] == 1
assert result["counts"]["malware_threats"] == 1
assert result["counts"]["scm_suppliers"] == 1
assert "telegram_osint" in result["non_empty_layers"]
assert result["layer_aliases"]["telegram"] == "telegram_osint"
assert result["layer_aliases"]["scm"] == "scm_suppliers"
def test_search_news_matches_telegram_osint(sample_store, monkeypatch):
import services.telemetry as telemetry
monkeypatch.setattr(telemetry, "get_data_version", lambda: 213)
result = telemetry.search_news(query="kyiv missile", limit=10, include_telegram=True)
assert result["results"]
assert result["results"][0]["source_layer"] == "telegram_osint"
assert result["results"][0]["lat"] == 50.45
def test_search_telemetry_finds_telegram_malware_and_scm(sample_store, monkeypatch):
import services.telemetry as telemetry
monkeypatch.setattr(telemetry, "get_data_version", lambda: 214)
telegram = telemetry.search_telemetry(query="osintdefender kyiv", limit=10)
assert any(item["source_layer"] == "telegram_osint" for item in telegram["results"])
malware = telemetry.search_telemetry(query="emotet", limit=10)
assert any(item["source_layer"] == "malware_threats" for item in malware["results"])
scm = telemetry.search_telemetry(query="tsmc hsinchu", limit=10)
assert any(item["source_layer"] == "scm_suppliers" for item in scm["results"])
cve = telemetry.search_telemetry(query="CVE-2026-1234", limit=10)
assert any(item["source_layer"] == "cyber_threats" for item in cve["results"])
def test_entities_near_finds_telegram_and_malware(sample_store, monkeypatch):
import services.telemetry as telemetry
monkeypatch.setattr(telemetry, "get_data_version", lambda: 215)
result = telemetry.entities_near(
lat=38.95,
lng=-77.45,
radius_km=50,
entity_types=["telegram", "malware"],
limit=10,
)
layers = {item["source_layer"] for item in result["results"]}
assert "malware_threats" in layers
def test_openclaw_correlate_entity_command(sample_store, monkeypatch):
import services.telemetry as telemetry
from services.openclaw_channel import _dispatch_command
+98
View File
@@ -0,0 +1,98 @@
"""Tests for OpenClaw recon / OSINT command dispatch."""
import pytest
def test_osint_tools_lists_supported_lookups():
from services.osint.openclaw_recon import osint_tool_help
help_data = osint_tool_help()
assert "ip" in help_data["tools"]
assert "sanctions" in help_data["tools"]
assert "aircraft" in help_data["entity_types"]
def test_osint_lookup_ip(monkeypatch):
from services.osint import openclaw_recon
monkeypatch.setattr(
openclaw_recon.lookups,
"lookup_ip",
lambda ip: {"ip": ip, "geo": {"country": "US"}},
)
result = openclaw_recon.run_osint_lookup("ip", {"ip": "8.8.8.8"})
assert result["ip"] == "8.8.8.8"
assert result["geo"]["country"] == "US"
def test_osint_lookup_sanctions_passes_schema(monkeypatch):
from services.osint import openclaw_recon
captured = {}
def fake_sanctions(query, *, schema=None, limit=25):
captured["query"] = query
captured["schema"] = schema
captured["limit"] = limit
return {"query": query, "results": []}
monkeypatch.setattr(openclaw_recon.lookups, "lookup_sanctions", fake_sanctions)
openclaw_recon.run_osint_lookup(
"sanctions",
{"query": "Example Corp", "schema": "Company", "limit": 10},
)
assert captured["query"] == "Example Corp"
assert captured["schema"] == "Company"
assert captured["limit"] == 10
def test_osint_lookup_rejects_unknown_tool():
from services.osint.openclaw_recon import run_osint_lookup
with pytest.raises(ValueError, match="Unknown OSINT tool"):
run_osint_lookup("not_a_tool", {})
def test_openclaw_osint_lookup_command(monkeypatch):
from services import openclaw_channel
monkeypatch.setattr(
"services.osint.openclaw_recon.run_osint_lookup",
lambda tool, args: {"ip": args["ip"], "tool": tool},
)
result = openclaw_channel._dispatch_command(
"osint_lookup",
{"tool": "ip", "ip": "1.1.1.1"},
)
assert result["ok"] is True
assert result["data"]["ip"] == "1.1.1.1"
def test_openclaw_entity_expand_command(monkeypatch):
from services import openclaw_channel
monkeypatch.setattr(
"services.osint.openclaw_recon.run_entity_expand",
lambda args: {"nodes": [{"id": "ip:1.1.1.1"}], "links": []},
)
result = openclaw_channel._dispatch_command(
"entity_expand",
{"type": "ip", "id": "1.1.1.1"},
)
assert result["ok"] is True
assert result["data"]["nodes"][0]["id"] == "ip:1.1.1.1"
def test_osint_sweep_requires_full_tier_for_restricted():
from services.openclaw_channel import WRITE_COMMANDS, allowed_commands
assert "osint_sweep" in WRITE_COMMANDS
assert "osint_sweep" not in allowed_commands("restricted")
assert "osint_sweep" in allowed_commands("full")
def test_osint_lookup_available_on_restricted_tier():
from services.openclaw_channel import allowed_commands
assert "osint_lookup" in allowed_commands("restricted")
assert "entity_expand" in allowed_commands("restricted")