Expose new telemetry and recon toolkit to OpenClaw agents.

Wire telegram_osint, malware, cyber, and SCM into search/slow-tier helpers; add osint_lookup, entity_expand, and osint_sweep commands; update README and skill docs. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-25 21:20:55 +02:00 · 2026-06-08 22:44:16 -06:00
parent af9b3d08cc
commit 1583fd5715
9 changed files with 793 additions and 44 deletions
@@ -83,6 +83,10 @@ READ_COMMANDS = frozenset({
    "sar_pin_click",
    # Analysis zones (OpenClaw map overlays)
    "list_analysis_zones",
+    # Recon / OSINT toolkit (server-side proxies, SSRF guarded)
+    "osint_lookup",
+    "osint_tools",
+    "entity_expand",
 })

 WRITE_COMMANDS = frozenset({
@@ -112,6 +116,8 @@ WRITE_COMMANDS = frozenset({
    "place_analysis_zone",
    "delete_analysis_zone",
    "clear_analysis_zones",
+    # Active recon (subnet device discovery)
+    "osint_sweep",
 })


@@ -780,6 +786,7 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
            query=str(args.get("query", "") or ""),
            limit=args.get("limit", 10),
            include_gdelt=bool(args.get("include_gdelt", True)),
+            include_telegram=bool(args.get("include_telegram", True)),
        )
        if _wants_compact(args):
            return {"ok": True, "data": _compact_query_result(result), "format": "compressed_v1"}
@@ -846,6 +853,26 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]:
            return {"ok": True, "data": _compact_query_result(result), "format": "compressed_v1"}
        return {"ok": True, "data": result}

+    if cmd == "osint_lookup":
+        from services.osint.openclaw_recon import run_osint_lookup
+        tool = str(args.get("tool", "") or args.get("lookup", "") or args.get("type", "") or "")
+        result = run_osint_lookup(tool, args)
+        return {"ok": True, "data": result, "tool": tool.strip().lower()}
+
+    if cmd == "osint_tools":
+        from services.osint.openclaw_recon import osint_tool_help
+        return {"ok": True, "data": osint_tool_help()}
+
+    if cmd == "osint_sweep":
+        from services.osint.openclaw_recon import run_osint_sweep
+        result = run_osint_sweep(args)
+        return {"ok": True, "data": result}
+
+    if cmd == "entity_expand":
+        from services.osint.openclaw_recon import run_entity_expand
+        result = run_entity_expand(args)
+        return {"ok": True, "data": result}
+
    if cmd == "get_report":
        from services.telemetry import get_cached_telemetry_refs, get_cached_slow_telemetry_refs
        fast = get_cached_telemetry_refs()
@@ -0,0 +1,135 @@
+"""OpenClaw dispatch for the operator recon / OSINT lookup toolkit."""
+from __future__ import annotations
+
+from typing import Any
+
+from services.osint import lookups
+from services.osint_intel.resolve import ALLOWED_TYPES, resolve_entity
+
+_OSINT_TOOLS: dict[str, str] = {
+    "ip": "ip",
+    "dns": "domain",
+    "whois": "domain",
+    "certs": "domain",
+    "threats": "query",
+    "bgp": "query",
+    "sanctions": "query",
+    "cve": "cve",
+    "mac": "mac",
+    "github": "username",
+    "leaks": "email",
+    "sweep_init": "ip",
+}
+
+_ENTITY_SCHEMAS = frozenset({
+    "Person",
+    "Organization",
+    "Company",
+    "Vessel",
+    "Airplane",
+    "LegalEntity",
+})
+
+
+def _require_str(args: dict[str, Any], *keys: str) -> str:
+    for key in keys:
+        value = str(args.get(key, "") or "").strip()
+        if value:
+            return value
+    joined = "/".join(keys)
+    raise ValueError(f"Missing required argument: {joined}")
+
+
+def run_osint_lookup(tool: str, args: dict[str, Any]) -> dict[str, Any]:
+    """Run a passive OSINT lookup (same backends as /api/osint/*)."""
+    name = str(tool or "").strip().lower().replace("-", "_")
+    if name not in _OSINT_TOOLS:
+        allowed = ", ".join(sorted(_OSINT_TOOLS))
+        raise ValueError(f"Unknown OSINT tool '{tool}'. Allowed: {allowed}")
+
+    if name == "ip":
+        return lookups.lookup_ip(_require_str(args, "ip", "query", "value"))
+    if name == "dns":
+        return lookups.lookup_dns(_require_str(args, "domain", "query", "value"))
+    if name == "whois":
+        return lookups.lookup_whois(_require_str(args, "domain", "query", "value"))
+    if name == "certs":
+        return lookups.lookup_certs(_require_str(args, "domain", "query", "value"))
+    if name == "threats":
+        query = str(args.get("query", "") or args.get("value", "") or "").strip() or None
+        return lookups.lookup_threats(query)
+    if name == "bgp":
+        return lookups.lookup_bgp(_require_str(args, "query", "asn", "value"))
+    if name == "sanctions":
+        query = _require_str(args, "query", "name", "value")
+        schema = str(args.get("schema", "") or "").strip() or None
+        if schema and schema not in _ENTITY_SCHEMAS:
+            allowed = ", ".join(sorted(_ENTITY_SCHEMAS))
+            raise ValueError(f"Invalid schema. Allowed: {allowed}")
+        limit = args.get("limit", 25)
+        try:
+            limit = int(limit)
+        except (TypeError, ValueError):
+            limit = 25
+        limit = max(1, min(100, limit))
+        return lookups.lookup_sanctions(query, schema=schema, limit=limit)
+    if name == "cve":
+        return lookups.lookup_cve(_require_str(args, "cve", "query", "value"))
+    if name == "mac":
+        return lookups.lookup_mac(_require_str(args, "mac", "query", "value"))
+    if name == "github":
+        return lookups.lookup_github(_require_str(args, "username", "user", "query", "value"))
+    if name == "leaks":
+        return lookups.lookup_leaks(_require_str(args, "email", "query", "value"))
+    if name == "sweep_init":
+        ip = _require_str(args, "ip", "query", "value")
+        cidr = args.get("cidr", 24)
+        try:
+            cidr = int(cidr)
+        except (TypeError, ValueError):
+            cidr = 24
+        return lookups.sweep_init(ip, cidr)
+
+    raise ValueError(f"Unhandled OSINT tool: {name}")
+
+
+def run_osint_sweep(args: dict[str, Any]) -> dict[str, Any]:
+    """Run subnet device discovery (Shodan InternetDB proxy). Requires full access tier."""
+    ip = _require_str(args, "ip", "query", "value")
+    cidr = args.get("cidr", 24)
+    try:
+        cidr = int(cidr)
+    except (TypeError, ValueError):
+        cidr = 24
+    subnet = lookups.subnet_start_for(ip, cidr)
+    scan = lookups.sweep_scan(subnet, cidr)
+    init = lookups.sweep_init(ip, cidr)
+    return {**init, **scan, "subnet": f"{subnet}/{cidr}"}
+
+
+def run_entity_expand(args: dict[str, Any]) -> dict[str, Any]:
+    """Expand an entity graph node (aircraft, vessel, IP, company, person, country)."""
+    entity_type = _require_str(args, "type", "entity_type")
+    entity_id = _require_str(args, "id", "entity_id", "query", "value")
+    props = {
+        "label": entity_id,
+        "registration": str(args.get("registration", "") or "").strip() or None,
+        "model": str(args.get("model", "") or "").strip() or None,
+        "icao24": str(args.get("icao24", "") or "").strip() or None,
+    }
+    props = {key: value for key, value in props.items() if value is not None}
+    return resolve_entity(entity_type, entity_id, props)
+
+
+def osint_tool_help() -> dict[str, Any]:
+    """Discovery metadata for agents."""
+    return {
+        "tools": sorted(_OSINT_TOOLS),
+        "entity_types": sorted(ALLOWED_TYPES),
+        "sanctions_schemas": sorted(_ENTITY_SCHEMAS),
+        "notes": {
+            "osint_lookup": "Passive lookups — same data as the Recon panel /api/osint/* routes.",
+            "osint_sweep": "Active subnet scan via Shodan InternetDB — requires full OpenClaw access tier.",
+            "entity_expand": "Build a relationship graph around aircraft, vessels, IPs, companies, people, or countries.",
+        },
+    }
@@ -93,6 +93,10 @@ _SLOW_KEYS = (
    "sar_scenes",
    "sar_anomalies",
    "sar_aoi_coverage",
+    "malware_threats",
+    "cyber_threats",
+    "scm_suppliers",
+    "telegram_osint",
 )


@@ -158,6 +162,20 @@ _ENTITY_LAYER_ALIASES = {
    "uap_sightings": "uap_sightings",
    "wastewater": "wastewater",
    "pins": "pins",
+    "telegram": "telegram_osint",
+    "telegram_osint": "telegram_osint",
+    "osint_feed": "telegram_osint",
+    "malware": "malware_threats",
+    "malware_threats": "malware_threats",
+    "malware_c2": "malware_threats",
+    "botnet": "malware_threats",
+    "cyber": "cyber_threats",
+    "cyber_threats": "cyber_threats",
+    "cisa": "cyber_threats",
+    "kev": "cyber_threats",
+    "scm": "scm_suppliers",
+    "scm_suppliers": "scm_suppliers",
+    "suppliers": "scm_suppliers",
 }

 _SLICEABLE_LAYERS = tuple(dict.fromkeys(_FAST_KEYS + _SLOW_KEYS))
@@ -188,6 +206,21 @@ _LAYER_ALIASES = {
    "sar_coverage": "sar_aoi_coverage",
    # Satellite analysis (maneuvers, decay, Starlink)
    "satellite_analysis": "satellite_analysis",
+    # OSINT / cyber / supply-chain overlays
+    "telegram": "telegram_osint",
+    "telegram_osint": "telegram_osint",
+    "osint_feed": "telegram_osint",
+    "malware": "malware_threats",
+    "malware_threats": "malware_threats",
+    "malware_c2": "malware_threats",
+    "botnet": "malware_threats",
+    "cyber": "cyber_threats",
+    "cyber_threats": "cyber_threats",
+    "cisa": "cyber_threats",
+    "kev": "cyber_threats",
+    "scm": "scm_suppliers",
+    "scm_suppliers": "scm_suppliers",
+    "suppliers": "scm_suppliers",
 }

 _UNIVERSAL_SEARCH_DEFAULT_LAYERS = (
@@ -225,6 +258,10 @@ _UNIVERSAL_SEARCH_DEFAULT_LAYERS = (
    "tinygs_satellites",
    "psk_reporter",
    "ukraine_alerts",
+    "telegram_osint",
+    "malware_threats",
+    "cyber_threats",
+    "scm_suppliers",
 )

 _GENERIC_QUERY_STOPWORDS = {
@@ -269,7 +306,19 @@ _GENERIC_LAYER_HINTS: dict[str, tuple[str, ...]] = {
    "protest": ("crowdthreat", "gdelt", "news", "frontlines", "liveuamap"),
    "riot": ("crowdthreat", "gdelt", "news", "frontlines", "liveuamap"),
    "event": ("crowdthreat", "gdelt", "news", "frontlines", "liveuamap"),
-    "news": ("news", "gdelt", "crowdthreat", "frontlines", "liveuamap"),
+    "news": ("news", "gdelt", "crowdthreat", "frontlines", "liveuamap", "telegram_osint"),
+    "telegram": ("telegram_osint",),
+    "osint": ("telegram_osint", "gdelt", "news", "crowdthreat"),
+    "channel": ("telegram_osint",),
+    "malware": ("malware_threats",),
+    "botnet": ("malware_threats",),
+    "c2": ("malware_threats",),
+    "cve": ("cyber_threats",),
+    "cisa": ("cyber_threats",),
+    "cyber": ("cyber_threats", "malware_threats"),
+    "supplier": ("scm_suppliers",),
+    "scm": ("scm_suppliers",),
+    "semiconductor": ("scm_suppliers",),
    "plant": ("power_plants", "wastewater"),
    "datacenter": ("datacenters",),
    "data": ("datacenters",),
@@ -314,6 +363,10 @@ _SEARCH_GROUP_BY_LAYER = {
    "kiwisdr": "signals",
    "psk_reporter": "signals",
    "ukraine_alerts": "events",
+    "telegram_osint": "events",
+    "malware_threats": "cyber",
+    "cyber_threats": "cyber",
+    "scm_suppliers": "infrastructure",
 }

 _SEARCH_QUERY_SYNONYMS: dict[str, tuple[str, ...]] = {
@@ -328,6 +381,9 @@ _SEARCH_QUERY_SYNONYMS: dict[str, tuple[str, ...]] = {
    "plants": ("plant",),
    "cameras": ("camera",),
    "radios": ("radio",),
+    "telegrams": ("telegram",),
+    "channels": ("channel",),
+    "suppliers": ("supplier",),
 }

 _SEARCH_INDEX_LOCK = threading.Lock()
@@ -653,6 +709,42 @@ _UNIVERSAL_SEARCH_SPECS: dict[str, dict[str, Any]] = {
        "id_fields": ("id",),
        "time_fields": ("updated_at", "timestamp"),
    },
+    "telegram_osint": {
+        "fields": ("title", "description", "source", "channel", "link"),
+        "primary_fields": ("title", "description", "channel"),
+        "label_fields": ("title", "channel"),
+        "summary_fields": ("description", "source"),
+        "type_fields": ("channel", "source"),
+        "id_fields": ("id", "link"),
+        "time_fields": ("published", "timestamp"),
+    },
+    "malware_threats": {
+        "fields": ("ip", "malware", "status", "country", "threat_type"),
+        "primary_fields": ("ip", "malware", "country"),
+        "label_fields": ("ip", "malware"),
+        "summary_fields": ("status", "country", "threat_type"),
+        "type_fields": ("threat_type", "malware"),
+        "id_fields": ("id", "ip"),
+        "time_fields": ("first_seen", "last_online", "timestamp"),
+    },
+    "cyber_threats": {
+        "fields": ("id", "name", "vendor", "product", "severity", "source"),
+        "primary_fields": ("id", "name", "vendor", "product"),
+        "label_fields": ("id", "name"),
+        "summary_fields": ("vendor", "product", "severity", "source"),
+        "type_fields": ("severity", "source"),
+        "id_fields": ("id",),
+        "time_fields": ("date", "due", "timestamp"),
+    },
+    "scm_suppliers": {
+        "fields": ("name", "city", "country", "category", "risk_level"),
+        "primary_fields": ("name", "city", "country", "category"),
+        "label_fields": ("name", "city"),
+        "summary_fields": ("country", "category", "risk_level"),
+        "type_fields": ("category", "risk_level"),
+        "id_fields": ("id",),
+        "time_fields": ("timestamp",),
+    },
 }


@@ -734,6 +826,11 @@ def _extract_coords(candidate: dict[str, Any]) -> tuple[float | None, float | No
        if isinstance(coords, (list, tuple)) and len(coords) >= 2:
            lng = lng if lng is not None else _coerce_float(coords[0])
            lat = lat if lat is not None else _coerce_float(coords[1])
+    if lat is None or lng is None:
+        coords = candidate.get("coords")
+        if isinstance(coords, (list, tuple)) and len(coords) >= 2:
+            lat = lat if lat is not None else _coerce_float(coords[0])
+            lng = lng if lng is not None else _coerce_float(coords[1])
    return lat, lng


@@ -832,6 +929,53 @@ def _layer_group(layer: str) -> str:
    return _SEARCH_GROUP_BY_LAYER.get(layer, "other")


+_LAYER_NESTED_LIST_KEYS: dict[str, tuple[str, ...]] = {
+    "telegram_osint": ("posts",),
+    "malware_threats": ("threats",),
+    "cyber_threats": ("threats",),
+    "scm_suppliers": ("suppliers",),
+}
+_DEFAULT_NESTED_LIST_KEYS = (
+    "items",
+    "results",
+    "vessels",
+    "features",
+    "posts",
+    "threats",
+    "suppliers",
+)
+
+
+def _unwrap_layer_items(value: Any, layer: str = "") -> list[Any]:
+    """Return the searchable/geospatial item list inside a layer value."""
+    if isinstance(value, list):
+        return value
+    if not isinstance(value, dict):
+        return []
+    keys = _LAYER_NESTED_LIST_KEYS.get(layer, _DEFAULT_NESTED_LIST_KEYS)
+    for key in keys:
+        nested = value.get(key)
+        if isinstance(nested, list):
+            return nested
+    return []
+
+
+def _layer_record_count(value: Any, layer: str = "") -> int:
+    if isinstance(value, list):
+        return len(value)
+    if isinstance(value, dict):
+        total = value.get("total")
+        if isinstance(total, (int, float)):
+            return int(total)
+        items = _unwrap_layer_items(value, layer)
+        if items:
+            return len(items)
+        return len(value) if value else 0
+    if value is None:
+        return 0
+    return 1
+
+
 def _build_search_document(doc_id: int, layer: str, candidate: dict[str, Any], spec: dict[str, Any]) -> dict[str, Any]:
    fields = tuple(spec.get("fields", ()))
    text = _document_text(candidate, fields)
@@ -880,9 +1024,7 @@ def _get_search_index() -> dict[str, Any]:

        for layer in layers:
            spec = _UNIVERSAL_SEARCH_SPECS[layer]
-            items = snap.get(layer) or []
-            if isinstance(items, dict):
-                items = items.get("items", []) or items.get("results", []) or items.get("vessels", [])
+            items = _unwrap_layer_items(snap.get(layer), layer)
            if not isinstance(items, list):
                continue
            for item in items:
@@ -1144,18 +1286,9 @@ def get_telemetry_summary() -> dict[str, Any]:

    for layer in layer_names:
        value = snap.get(layer)
-        if isinstance(value, list):
-            counts[layer] = len(value)
-            if value:
-                non_empty_layers.append(layer)
-        elif isinstance(value, dict):
-            counts[layer] = len(value)
-            if value:
-                non_empty_layers.append(layer)
-        elif value is None:
-            counts[layer] = 0
-        else:
-            counts[layer] = 1
+        count = _layer_record_count(value, layer)
+        counts[layer] = count
+        if count > 0:
            non_empty_layers.append(layer)

    alias_examples = {
@@ -1167,6 +1300,16 @@ def get_telemetry_summary() -> dict[str, Any]:
        "tracked": "tracked_flights",
        "military": "military_flights",
        "jets": "private_jets",
+        "telegram": "telegram_osint",
+        "osint_feed": "telegram_osint",
+        "malware": "malware_threats",
+        "malware_c2": "malware_threats",
+        "botnet": "malware_threats",
+        "cyber": "cyber_threats",
+        "cisa": "cyber_threats",
+        "kev": "cyber_threats",
+        "scm": "scm_suppliers",
+        "suppliers": "scm_suppliers",
    }

    return {
@@ -1577,14 +1720,7 @@ def _nearby_items_from_layers(
    snap = get_latest_data_subset_refs(*layers)
    out: dict[str, list[dict[str, Any]]] = {}
    for layer in layers:
-        value = snap.get(layer) or []
-        if isinstance(value, dict):
-            if layer == "gdelt" and isinstance(value.get("features"), list):
-                items = value.get("features") or []
-            else:
-                items = value.get("items") or value.get("features") or value.get("vessels") or []
-        else:
-            items = value
+        items = _unwrap_layer_items(snap.get(layer), layer)
        if not isinstance(items, list):
            continue
        matches: list[dict[str, Any]] = []
@@ -1728,6 +1864,9 @@ def correlate_entity(
            "crowdthreat",
            "frontlines",
            "liveuamap",
+            "telegram_osint",
+            "malware_threats",
+            "scm_suppliers",
            "military_bases",
            "datacenters",
            "power_plants",
@@ -1809,13 +1948,17 @@ def search_news(
    query: str,
    limit: int = 10,
    include_gdelt: bool = True,
+    include_telegram: bool = True,
 ) -> dict[str, Any]:
    """Search news and event layers server-side and return a compact result set."""
    query_norm = _norm_text(query)
    if not query_norm:
        return {"results": [], "version": get_data_version(), "truncated": False}

-    snap = get_latest_data_subset_refs("news", "gdelt", "crowdthreat", "liveuamap", "frontlines")
+    layer_keys = ["news", "gdelt", "crowdthreat", "liveuamap", "frontlines"]
+    if include_telegram:
+        layer_keys.append("telegram_osint")
+    snap = get_latest_data_subset_refs(*layer_keys)
    out: list[dict[str, Any]] = []
    limit = _coerce_limit(limit, default=10, maximum=50)

@@ -1941,6 +2084,36 @@ def search_news(
            if len(out) >= limit:
                return {"results": out, "version": get_data_version(), "truncated": True}

+    if include_telegram:
+        for post in _unwrap_layer_items(snap.get("telegram_osint"), "telegram_osint"):
+            if not isinstance(post, dict):
+                continue
+            text = " ".join(
+                (
+                    _norm_text(post.get("title")),
+                    _norm_text(post.get("description")),
+                    _norm_text(post.get("source")),
+                    _norm_text(post.get("channel")),
+                )
+            )
+            if not _text_matches_query(query_norm, text):
+                continue
+            lat, lng = _extract_coords(post)
+            out.append(
+                {
+                    "source_layer": "telegram_osint",
+                    "title": post.get("title") or "",
+                    "summary": post.get("description") or "",
+                    "source": post.get("source") or post.get("channel") or "Telegram",
+                    "link": post.get("link") or "",
+                    "lat": lat,
+                    "lng": lng,
+                    "risk_score": post.get("risk_score"),
+                }
+            )
+            if len(out) >= limit:
+                return {"results": out, "version": get_data_version(), "truncated": True}
+
    return {"results": out, "version": get_data_version(), "truncated": False}


@@ -2205,16 +2378,13 @@ def entities_near(
    out: list[dict[str, Any]] = []

    for layer in layers:
-        items = snap.get(layer) or []
-        if isinstance(items, dict):
-            items = items.get("vessels", []) or items.get("items", [])
+        items = _unwrap_layer_items(snap.get(layer), layer)
        if not isinstance(items, list):
            continue
        for item in items:
            if not isinstance(item, dict):
                continue
-            item_lat = _coerce_float(item.get("lat") or item.get("latitude"))
-            item_lng = _coerce_float(item.get("lng") or item.get("lon") or item.get("longitude"))
+            item_lat, item_lng = _extract_coords(item)
            if item_lat is None or item_lng is None:
                continue
            distance = _haversine_km(center_lat, center_lng, item_lat, item_lng)