From 1583fd57155f2b3c331fdd1cc8314c5a641a8647 Mon Sep 17 00:00:00 2001 From: BigBodyCobain <43977454+BigBodyCobain@users.noreply.github.com> Date: Mon, 8 Jun 2026 22:44:16 -0600 Subject: [PATCH] Expose new telemetry and recon toolkit to OpenClaw agents. Wire telegram_osint, malware, cyber, and SCM into search/slow-tier helpers; add osint_lookup, entity_expand, and osint_sweep commands; update README and skill docs. Co-authored-by: Cursor --- README.md | 38 ++- backend/routers/ai_intel.py | 77 ++++++- backend/services/openclaw_channel.py | 27 +++ backend/services/osint/openclaw_recon.py | 135 +++++++++++ backend/services/telemetry.py | 230 ++++++++++++++++--- backend/tests/test_openclaw_query_helpers.py | 147 ++++++++++++ backend/tests/test_openclaw_recon.py | 98 ++++++++ openclaw-skills/shadowbroker/SKILL.md | 55 ++++- openclaw-skills/shadowbroker/sb_query.py | 30 +++ 9 files changed, 793 insertions(+), 44 deletions(-) create mode 100644 backend/services/osint/openclaw_recon.py create mode 100644 backend/tests/test_openclaw_recon.py diff --git a/README.md b/README.md index d9844ec..1225cf6 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ **ShadowBroker** is a decentralized intelligence platform that aggregates real-time, multi-domain OSINT telemetry from 60+ live intelligence feeds into a single dark-ops map interface. Aircraft, ships, satellites, conflict zones, CCTV networks, GPS jamming, internet-connected devices, police scanners, mesh radio nodes, and breaking geopolitical events — all updating in real time on one screen as well as an obfuscated communications protocol and information exchange infrastructure. -Built with **Next.js**, **MapLibre GL**, **FastAPI**, and **Python**. 40+ toggleable data layers, including SAR ground-change detection, a **server-side recon toolkit** (DNS, WHOIS, sanctions, BGP, IP sweep, and more), supply-chain risk overlays, and malware/C2 threat feeds. Multiple visual modes (DEFAULT / SATELLITE / FLIR / NVG / CRT). Right-click any point on Earth for a country dossier, head-of-state lookup, entity-graph expansion, and the latest Sentinel-2 satellite photo. ShadowBroker has no accounts, product telemetry, or analytics; the dashboard talks to your self-hosted backend. Sensitive recon and Shodan queries never hit third-party APIs from the browser — they are proxied through the backend with SSRF guards and local-operator auth. +Built with **Next.js**, **MapLibre GL**, **FastAPI**, and **Python**. 40+ toggleable data layers, including SAR ground-change detection, **Telegram OSINT** (public channel previews geoparsed onto the map), a **server-side recon toolkit** (DNS, WHOIS, sanctions, BGP, IP sweep, and more), supply-chain risk overlays, and malware/C2 + CISA KEV cyber threat feeds. Multiple visual modes (DEFAULT / SATELLITE / FLIR / NVG / CRT). Right-click any point on Earth for a country dossier, head-of-state lookup, entity-graph expansion, and the latest Sentinel-2 satellite photo. ShadowBroker has no accounts, product telemetry, or analytics; the dashboard talks to your self-hosted backend. Sensitive recon and Shodan queries never hit third-party APIs from the browser — they are proxied through the backend with SSRF guards and local-operator auth. The **OpenClaw / agent command channel** exposes the same recon backends plus full telemetry search — no separate API integration required. Designed for analysts, researchers, radio operators, and anyone who wants to see what the world looks like when every public signal is on the same map. @@ -41,7 +41,7 @@ ShadowBroker includes an optional **Shodan connector** for operator-supplied API ## Interesting Use Cases * **Track Air Force One**, the private jets of billionaires and dictators, and every military tanker, ISR, and fighter broadcasting ADS-B. Air Force One and all of the accompanying Presidential/Vice Presidential planes are highlighted and monitored from the moment they leave the ground. -* **Connect an AI agent as a co-analyst** through ShadowBroker's HMAC-signed agentic command channel — supports OpenClaw and any other agent that speaks the protocol (Claude, GPT, LangChain, custom). The agent gets full read/write access to all 40+ data layers, pin placement, map control, SAR ground-change, mesh networking, and alert delivery. It sees everything the operator sees and can take actions on the map in real time. +* **Connect an AI agent as a co-analyst** through ShadowBroker's HMAC-signed agentic command channel — supports OpenClaw and any other agent that speaks the protocol (Claude, GPT, LangChain, custom). The agent gets full read/write access to all 40+ data layers, compact cross-layer search (`search_telemetry`, `search_news`), the full recon toolkit (`osint_lookup` for IP/DNS/WHOIS/sanctions/CVE/etc.), entity-graph expansion, pin placement, map control, SAR ground-change, mesh networking, and alert delivery. It sees everything the operator sees and can take actions on the map in real time. * **Communicate on the InfoNet testnet** — The first decentralized intelligence mesh built into an OSINT tool. Obfuscated messaging with gate personas, Dead Drop peer-to-peer exchange, and a built-in terminal CLI. No accounts, no signup. Privacy is not guaranteed yet — this is an experimental testnet — but the protocol is live and being hardened. * **Right-click anywhere on Earth** for a country dossier (head of state, population, languages), Wikipedia summary, and the latest Sentinel-2 satellite photo at 10m resolution * **Click a KiwiSDR node** and tune into live shortwave radio directly in the dashboard. Click a police scanner feed and eavesdrop in one click. @@ -61,6 +61,7 @@ ShadowBroker includes an optional **Shodan connector** for operator-supplied API * **Expand an entity graph** when you select an aircraft, vessel, company, or IP — Wikidata + OFAC + live store cross-links rendered in the Entity Graph panel * **Monitor supply-chain risk** — Tier 1/2 semiconductor and battery fabs scored against nearby earthquakes, wildfires, and conflict events (SCM panel) * **Toggle malware C2 hotspots** — abuse.ch Feodo Tracker + URLhaus feeds mapped by country (opt-in layer) +* **Monitor Telegram OSINT channels** — public `t.me/s` war/conflict feeds (OSINTdefender, NEXTA, etc.) scraped hourly, risk-scored, geoparsed to metro anchors, and plotted as clickable map pins with inline media * **Overlay global submarine cables** — static TeleGeography-derived cable routes (opt-in layer) @@ -262,6 +263,8 @@ Adapted from the [OSIRIS](https://github.com/simplifaisoul/osiris) recon stack ( **Entity graph** — Select any map entity to open the Entity Graph panel (`GET /api/entity/expand`). Resolves aircraft, vessels, companies, persons, IPs, and countries into a node/link graph (Wikidata SPARQL + OFAC + in-memory flight/ship store). +**OpenClaw / agent access** — The same recon backends are available on the HMAC command channel (no browser local-operator gate): `osint_lookup` (passive IP/DNS/WHOIS/certs/BGP/sanctions/CVE/MAC/GitHub/leaks/threats), `entity_expand` (relationship graph), and `osint_sweep` (active subnet scan — **full** access tier only). Call `osint_tools` to list supported lookup types. Skill package: `openclaw-skills/shadowbroker/` (`SKILL.md` + `sb_query.py`). + **Shodan overlay** (unchanged): * **Internet Device Search** — Query Shodan with your own API key; results plotted as a live overlay @@ -384,6 +387,7 @@ Adapted from the [OSIRIS](https://github.com/simplifaisoul/osiris) recon stack ( * **SCM Supplier Risk** — Tier 1/2 fabs and battery plants (TSMC, Samsung, CATL, etc.) cross-referenced against earthquakes, FIRMS fires, and GDELT conflict proximity. Alerts in the SCM panel; optional map layer. * **Cyber Threats Feed** — Recent CISA Known Exploited Vulnerabilities (KEV) entries exposed via `/api/cyber-threats` and the layer toggle. * **Country Risk Index** — Static geopolitical risk scores with USGS earthquake enrichment via `/api/country-risk`. +* **Telegram OSINT** — Public channel web previews (`t.me/s/*`) from configurable war/OSINT feeds. Hourly incremental merge (no redundant re-scrape), keyword risk scoring, Cyrillic/Arabic place aliases, metro-anchor geocoding (separate from news centroids), inline photo/video via `/api/telegram/media` proxy. Layer key: `telegram_osint`. ### 🌐 Additional Layers & Tools @@ -409,7 +413,9 @@ v0.9.7 turns ShadowBroker from a dashboard a human watches into an intelligence **Capabilities:** -* **Full Telemetry Access** — The agent queries all 40+ data layers: flights, ships, satellites, SIGINT, conflict events, earthquakes, fires, wastewater, malware/C2, SCM overlays, prediction markets, and more. Fast and slow tier endpoints return enriched data with geographic coordinates, timestamps, and source attribution. +* **Full Telemetry Access** — The agent queries all 40+ data layers: flights, ships, satellites, SIGINT, conflict events, earthquakes, fires, wastewater, **Telegram OSINT**, malware/C2, **CISA KEV cyber threats**, SCM overlays, fishing activity (GFW), prediction markets, and more. Fast and slow tier endpoints return enriched data with geographic coordinates, timestamps, and source attribution. +* **Compact Search (preferred over full dumps)** — `get_summary` → `get_layer_slice` with per-layer `since_layer_versions` (SSE `layer_changed` push tells the agent exactly which layers updated). `search_telemetry` is the Google-style cross-layer keyword index. `search_news` covers news, GDELT, CrowdThreat, LiveUAMap, frontlines, and Telegram posts. `entities_near`, `brief_area`, `find_flights`/`find_ships`/`find_entity`, and `correlate_entity` answer targeted questions without multi-megabyte pulls. +* **Recon Toolkit on the Channel** — `osint_lookup` runs the same SSRF-guarded backends as the Recon panel (`ip`, `dns`, `whois`, `certs`, `bgp`, `sanctions`, `cve`, `mac`, `github`, `leaks`, `threats`, `sweep_init`). `entity_expand` builds Wikidata + OFAC relationship graphs. `osint_sweep` runs Shodan InternetDB subnet discovery (**full** tier). Layer aliases: `telegram`, `malware`/`botnet`, `cyber`/`cisa`/`kev`, `scm`/`suppliers`, `gfw`/`fishing`. * **AI Intel Pins** — Place color-coded investigation markers directly on the operator's map. 14 pin categories (threat, anomaly, military, maritime, aviation, SIGINT, infrastructure, etc.) with confidence scores, TTL expiry, source URLs, and batch placement up to 100 pins at once. * **Map Control** — Fly the operator's map view to any coordinate, trigger satellite imagery lookups, and open region dossiers. The agent can direct the operator's attention to specific locations in real time. * **SAR Ground-Change** — Query SAR anomaly feeds, inspect pin details, manage AOIs, and fly the map to watch areas. The agent can monitor for ground deformation, flood extent, or damage and promote anomalies to pins. @@ -422,7 +428,7 @@ v0.9.7 turns ShadowBroker from a dashboard a human watches into an intelligence * **Intelligence Reports** — Generate structured reports with summary stats, top military flights, correlations, earthquake activity, SIGINT counts, and pin inventories. * **Auditable** — Every channel call is logged; the operator can introspect what the agent has done. -**Connect an agent:** Open the AI Intel panel in the left sidebar, click **Connect Agent**, and copy the HMAC secret. From there, point any compatible agent at the channel — for OpenClaw, import `ShadowBrokerClient` from the OpenClaw skill package; for any other agent, use the same HMAC contract documented above (timestamp + nonce + body digest, tier-gated). The channel is the protocol, not the agent. +**Connect an agent:** Open the AI Intel panel in the left sidebar, click **Connect Agent**, and copy the HMAC secret. From there, point any compatible agent at the channel — for OpenClaw, import `ShadowBrokerClient` from `openclaw-skills/shadowbroker/sb_query.py` (see `SKILL.md` for examples); for any other agent, use the same HMAC contract documented above (timestamp + nonce + body digest, tier-gated). Discovery: `GET /api/ai/tools` and `GET /api/ai/capabilities`. The channel is the protocol, not the agent. ### ⏱️ Time Machine — Snapshot Playback (NEW in v0.9.7) @@ -584,6 +590,7 @@ ShadowBroker v0.9.7 is composed of three vertically-stacked planes — the **Ope | [Amtrak](https://www.amtrak.com) | US train positions | ~60s | No | | [DigiTraffic](https://www.digitraffic.fi) | European rail positions | ~60s | No | | [Global Fishing Watch](https://globalfishingwatch.org) | Fishing vessel activity events | ~1hr | **Yes** (`GFW_API_TOKEN`) | +| [Telegram public previews](https://t.me/s) | War/OSINT channel posts (`telegram_osint`) | ~1hr | No (optional `TELEGRAM_OSINT_CHANNELS`) | | Transport for London, NYC DOT, TxDOT | CCTV cameras (UK, US) | ~10min | No | | Caltrans, WSDOT, GDOT, IDOT, MDOT | CCTV cameras (5 US states) | ~10min | No | | Spain DGT, Madrid City | CCTV cameras (Spain) | ~10min | No | @@ -906,15 +913,19 @@ All 41 layers are independently toggleable from the left panel: | Malware C2 | ❌ OFF | abuse.ch Feodo + URLhaus threat points | | SCM Suppliers | ❌ OFF | Tier 1/2 supply-chain risk markers + panel alerts | | Cyber Threats | ❌ OFF | Recent CISA KEV entries (stats in slow-tier payload) | +| Telegram OSINT | ✅ ON | Public war/OSINT Telegram channels — hourly scrape, geoparsed pins | | SAR | ✅ ON | Synthetic aperture radar catalog + anomaly alerts | **Recon & entity tools** (not map layers — left sidebar / selection): -| Tool | Access | Description | -|---|---|---| -| Recon Toolkit | Local operator | DNS, WHOIS, sanctions, BGP, CVE, sweep, etc. via `/api/osint/*` | -| SCM Risk panel | Local operator | Live supplier threat rollup via `/api/scm-suppliers` | -| Entity Graph | Local operator | Graph expansion on selected entities via `/api/entity/expand` | +| Tool | Dashboard access | OpenClaw command | Description | +|---|---|---|---| +| Recon Toolkit | Local operator (`/api/osint/*`) | `osint_lookup`, `osint_sweep`† | IP, DNS, WHOIS, certs, BGP, sanctions, CVE, MAC, GitHub, leaks, threats, subnet sweep | +| Entity Graph | Local operator (`/api/entity/expand`) | `entity_expand` | Wikidata + OFAC + live-store relationship graph | +| SCM Risk panel | Local operator (`/api/scm-suppliers`) | `get_layer_slice(["scm_suppliers"])` | Supplier threat rollup + map markers | +| Tool discovery | — | `osint_tools` | Lists recon lookup types and entity-expand schemas | + +† `osint_sweep` (active InternetDB scan) requires `OPENCLAW_ACCESS_TIER=full`. --- @@ -938,6 +949,7 @@ The platform is optimized for handling massive real-time datasets: ``` Shadowbroker/ +├── openclaw-skills/shadowbroker/ # OpenClaw skill — SKILL.md, sb_query.py client, alerts/monitor helpers ├── backend/ │ ├── main.py # FastAPI app, middleware, API routes (~4,000 lines) │ ├── cctv.db # SQLite CCTV camera database (auto-generated) @@ -951,11 +963,13 @@ Shadowbroker/ │ │ ├── ssrf_guard.py # SSRF validation for operator recon fetches │ │ ├── sanctions/ofac.py # OpenSanctions OFAC SDN index │ │ ├── osint/lookups.py # Server-side recon lookups (Osiris port) +│ │ ├── osint/openclaw_recon.py # OpenClaw dispatch for recon + entity_expand │ │ ├── osint_intel/resolve.py # Entity graph resolver (Wikidata + OFAC) │ │ ├── scm/suppliers.py # Supply-chain risk overlay │ │ ├── intel_feeds/ # Country risk index helpers │ │ ├── fetchers/malware.py # abuse.ch Feodo + URLhaus │ │ ├── fetchers/cyber_status.py # CISA KEV feed +│ │ ├── fetchers/telegram_osint.py # Public Telegram channel scrape + geoparse │ │ ├── third_party/osiris/ # MIT attribution for Osiris-derived code │ │ ├── geopolitics.py # GDELT + Ukraine frontline + air alerts │ │ ├── region_dossier.py # Right-click country/city intelligence @@ -998,7 +1012,7 @@ Shadowbroker/ │ │ ├── osint.py # /api/osint/* recon routes (local operator) │ │ ├── entity_graph.py # /api/entity/expand │ │ ├── scm.py # /api/scm-suppliers -│ │ └── intel_feeds.py # /api/malware, /api/cyber-threats, /api/country-risk +│ │ └── intel_feeds.py # /api/malware, /api/cyber-threats, /api/telegram-feed, /api/country-risk ├── frontend/ │ ├── public/data/ │ │ └── submarine-cables.json # Static undersea cable GeoJSON @@ -1015,6 +1029,7 @@ Shadowbroker/ │ │ ├── ReconPanel.tsx # Server-side OSINT recon toolkit │ │ ├── ScmPanel.tsx # Supply-chain risk command panel │ │ ├── EntityGraphPanel.tsx # Entity graph on map selection +│ │ ├── MaplibreViewer/popups/TelegramOsintPopup.tsx # Threat-intercept styled Telegram pin popups │ │ ├── WorldviewRightPanel.tsx # Search + filter sidebar │ │ ├── AdvancedFilterModal.tsx # Airport/country/owner filtering │ │ ├── MapLegend.tsx # Dynamic legend with all icons @@ -1051,6 +1066,9 @@ MESH_SAR_EARTHDATA_TOKEN= # NASA Earthdata token (paired wit MESH_SAR_COPERNICUS_USER= # Copernicus Data Space user (SAR Mode B — EGMS / EMS) MESH_SAR_COPERNICUS_TOKEN= # Copernicus token (paired with user above) OPENCLAW_ACCESS_TIER=restricted # OpenClaw agent tier: "restricted" (read-only) or "full" +GFW_API_TOKEN=your_gfw_token # Global Fishing Watch — fishing_activity layer (Settings → Maritime) +TELEGRAM_OSINT_ENABLED=true # Telegram OSINT layer (default on) +TELEGRAM_OSINT_CHANNELS=osintdefender,... # Comma-separated public channel slugs (see .env.example) # Private-lane privacy-core pinning (required when Arti or RNS is enabled) PRIVACY_CORE_MIN_VERSION=0.1.0 diff --git a/backend/routers/ai_intel.py b/backend/routers/ai_intel.py index 2f9ce2a..c98087d 100644 --- a/backend/routers/ai_intel.py +++ b/backend/routers/ai_intel.py @@ -1705,11 +1705,12 @@ async def agent_tool_manifest(request: Request): { "name": "search_news", "type": "read", - "description": "Search news and event layers server-side by keyword. Includes news, GDELT, CrowdThreat, and major incident/event feeds without pulling the full slow telemetry feed.", + "description": "Search news and event layers server-side by keyword. Includes news, GDELT, CrowdThreat, Telegram OSINT, and major incident/event feeds without pulling the full slow telemetry feed.", "parameters": { "query": {"type": "string", "required": True, "description": "Keyword or phrase to search for"}, "limit": {"type": "integer", "required": False, "description": "Max results (default 10, max 50)"}, "include_gdelt": {"type": "boolean", "required": False, "description": "Include GDELT matches (default true)"}, + "include_telegram": {"type": "boolean", "required": False, "description": "Include Telegram OSINT channel posts (default true)"}, "compact": {"type": "boolean", "required": False, "description": "If true, strips empty/None fields from each result and rounds lat/lng to 3 decimals. Response includes format: 'compressed_v1'."}, }, "returns": "{results: [{source_layer, title, summary, source, link, lat, lng, risk_score}], version: int, truncated: bool}", @@ -1743,6 +1744,55 @@ async def agent_tool_manifest(request: Request): }, "returns": "{center, radius_km, nearby, topic_news, context_layers}", }, + { + "name": "osint_lookup", + "type": "read", + "description": "Run a passive OSINT recon lookup server-side (same backends as the Recon panel). SSRF-guarded outbound proxies for IP geolocation, DNS, WHOIS, certs, BGP/ASN, sanctions, CVE, MAC vendor, GitHub profile, breach checks, and threat feeds.", + "parameters": { + "tool": {"type": "string", "required": True, "description": "Lookup type: ip, dns, whois, certs, threats, bgp, sanctions, cve, mac, github, leaks, sweep_init"}, + "ip": {"type": "string", "required": False, "description": "IPv4/IPv6 for ip or sweep_init"}, + "domain": {"type": "string", "required": False, "description": "Domain for dns, whois, certs"}, + "query": {"type": "string", "required": False, "description": "Generic query (BGP ASN, sanctions name, optional threats filter)"}, + "cve": {"type": "string", "required": False, "description": "CVE id for cve lookup"}, + "mac": {"type": "string", "required": False, "description": "MAC address for mac lookup"}, + "username": {"type": "string", "required": False, "description": "GitHub username"}, + "email": {"type": "string", "required": False, "description": "Email for breach/leak lookup"}, + "schema": {"type": "string", "required": False, "description": "Sanctions schema filter: Person, Organization, Company, Vessel, Airplane, LegalEntity"}, + "limit": {"type": "integer", "required": False, "description": "Sanctions result cap (default 25, max 100)"}, + "cidr": {"type": "integer", "required": False, "description": "CIDR mask for sweep_init (24-32, default 24)"}, + }, + "returns": "Tool-specific JSON (geo, DNS records, WHOIS, sanctions hits, CVE details, etc.)", + }, + { + "name": "osint_tools", + "type": "read", + "description": "List available OSINT recon tools, entity-expand types, and sanctions schemas.", + "parameters": {}, + "returns": "{tools: [...], entity_types: [...], sanctions_schemas: [...], notes: {...}}", + }, + { + "name": "entity_expand", + "type": "read", + "description": "Expand an entity relationship graph around an aircraft, vessel, IP, company, person, or country. Same backend as /api/entity/expand.", + "parameters": { + "type": {"type": "string", "required": True, "description": "Entity type: aircraft, vessel, company, person, ip, country"}, + "id": {"type": "string", "required": True, "description": "Entity identifier (tail number, MMSI, IP, company name, etc.)"}, + "registration": {"type": "string", "required": False, "description": "Aircraft registration hint"}, + "model": {"type": "string", "required": False, "description": "Aircraft model hint"}, + "icao24": {"type": "string", "required": False, "description": "ICAO24 hex for aircraft"}, + }, + "returns": "{nodes: [...], links: [...]}", + }, + { + "name": "osint_sweep", + "type": "write", + "description": "Active subnet device discovery via Shodan InternetDB (ports, vulns, hostnames). Requires full OpenClaw access tier. Private/reserved IPs blocked.", + "parameters": { + "ip": {"type": "string", "required": True, "description": "Public IPv4 anchor for the sweep"}, + "cidr": {"type": "integer", "required": False, "description": "Subnet size /24-/32 (default 24)"}, + }, + "returns": "{center, target_ip, cidr, subnet, devices, summary, sweep_time_ms}", + }, { "name": "what_changed", "type": "read", @@ -2194,6 +2244,11 @@ async def agent_tool_manifest(request: Request): "Prefer compact lookups first: search_telemetry, find_flights, find_ships, search_news, entities_near, get_layer_slice. Use get_telemetry/get_slow_telemetry/get_report only when focused commands are insufficient.", "ShadowBroker does expose UAP sightings, wastewater, and tracked_flights/VIP aircraft when those layers are populated. Verify with get_summary or get_layer_slice before claiming a layer is absent.", "ShadowBroker also exposes fishing_activity, which is the fishing-vessel activity layer backed by Global Fishing Watch data when GFW_API_TOKEN is configured. Do not confuse it with the AIS ships layer.", + "telegram_osint, malware_threats, cyber_threats, and scm_suppliers are live map layers. Use get_summary or get_layer_slice(['telegram_osint']) before claiming they are absent. Aliases: telegram, malware/botnet, cyber/cisa/kev, scm/suppliers.", + "search_telemetry and search_news both index Telegram OSINT posts. For malware C2, botnet IPs, CISA KEV CVEs, or semiconductor suppliers, use search_telemetry or get_layer_slice on the matching layer.", + "The Recon toolkit is available via osint_lookup: IP geolocation, DNS, WHOIS, certs, BGP, sanctions, CVE, MAC vendor, GitHub, breach checks, threat feeds. Call osint_tools first to list supported tools.", + "entity_expand builds relationship graphs for aircraft, vessels, IPs, companies, people, and countries — use after resolving an entity from telemetry or osint_lookup.", + "osint_sweep runs active subnet discovery (Shodan InternetDB) and requires full OpenClaw access tier. Use osint_lookup tool=sweep_init for passive geolocation context only.", "Use search_telemetry as the Google-style entry point whenever the user gives you a person, place, company, topic, owner, nickname, or natural-language phrase and you do not already know the source layer.", "Example: for 'Where is Jerry Jones yacht?' search 'Jerry Jones' across all telemetry first, identify the ship match, then refine with find_ships or raw layer context only if needed.", "For fuzzy natural-language lookups like 'Patriots jet' or 'Jerry Jones yacht', use search_telemetry first and inspect the ranked candidate list before making a hard claim.", @@ -2354,13 +2409,29 @@ async def api_capabilities(request: Request): "description": "Universal compact search across telemetry when the entity type or source layer is not obvious.", }, "search_news": { - "args": {"query": "str", "limit": "int (default 10)", "include_gdelt": "bool (default true)"}, - "description": "Search news and event layers by keyword without pulling the whole slow feed.", + "args": {"query": "str", "limit": "int (default 10)", "include_gdelt": "bool (default true)", "include_telegram": "bool (default true)"}, + "description": "Search news and event layers by keyword without pulling the whole slow feed. Includes Telegram OSINT when include_telegram is true.", }, "entities_near": { "args": {"lat": "float", "lng": "float", "radius_km": "float (default 50)", "entity_types": "list[str] (optional)", "limit": "int (default 25)"}, "description": "Compact proximity search around a point across selected layers.", }, + "osint_lookup": { + "args": {"tool": "str (ip|dns|whois|certs|threats|bgp|sanctions|cve|mac|github|leaks|sweep_init)", "...": "tool-specific params"}, + "description": "Passive OSINT recon lookup — same backends as the Recon panel.", + }, + "osint_tools": { + "args": {}, + "description": "List available recon tools and entity-expand types.", + }, + "entity_expand": { + "args": {"type": "str", "id": "str", "registration": "str (optional)", "icao24": "str (optional)"}, + "description": "Entity relationship graph expansion.", + }, + "osint_sweep": { + "args": {"ip": "str", "cidr": "int (default 24)"}, + "description": "Active subnet scan — requires full access tier.", + }, "brief_area": { "args": {"lat": "float", "lng": "float", "radius_km": "float (default 50)", "entity_types": "list[str] (optional)", "query": "str (optional)", "limit": "int (default 25)", "context_limit": "int (default 10)"}, "description": "One compact area brief: nearby aircraft/ships/entities, optional topic news, and selected context layers.", diff --git a/backend/services/openclaw_channel.py b/backend/services/openclaw_channel.py index ca5fed1..1d91c3f 100644 --- a/backend/services/openclaw_channel.py +++ b/backend/services/openclaw_channel.py @@ -83,6 +83,10 @@ READ_COMMANDS = frozenset({ "sar_pin_click", # Analysis zones (OpenClaw map overlays) "list_analysis_zones", + # Recon / OSINT toolkit (server-side proxies, SSRF guarded) + "osint_lookup", + "osint_tools", + "entity_expand", }) WRITE_COMMANDS = frozenset({ @@ -112,6 +116,8 @@ WRITE_COMMANDS = frozenset({ "place_analysis_zone", "delete_analysis_zone", "clear_analysis_zones", + # Active recon (subnet device discovery) + "osint_sweep", }) @@ -780,6 +786,7 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]: query=str(args.get("query", "") or ""), limit=args.get("limit", 10), include_gdelt=bool(args.get("include_gdelt", True)), + include_telegram=bool(args.get("include_telegram", True)), ) if _wants_compact(args): return {"ok": True, "data": _compact_query_result(result), "format": "compressed_v1"} @@ -846,6 +853,26 @@ def _dispatch_command(cmd: str, args: dict[str, Any]) -> dict[str, Any]: return {"ok": True, "data": _compact_query_result(result), "format": "compressed_v1"} return {"ok": True, "data": result} + if cmd == "osint_lookup": + from services.osint.openclaw_recon import run_osint_lookup + tool = str(args.get("tool", "") or args.get("lookup", "") or args.get("type", "") or "") + result = run_osint_lookup(tool, args) + return {"ok": True, "data": result, "tool": tool.strip().lower()} + + if cmd == "osint_tools": + from services.osint.openclaw_recon import osint_tool_help + return {"ok": True, "data": osint_tool_help()} + + if cmd == "osint_sweep": + from services.osint.openclaw_recon import run_osint_sweep + result = run_osint_sweep(args) + return {"ok": True, "data": result} + + if cmd == "entity_expand": + from services.osint.openclaw_recon import run_entity_expand + result = run_entity_expand(args) + return {"ok": True, "data": result} + if cmd == "get_report": from services.telemetry import get_cached_telemetry_refs, get_cached_slow_telemetry_refs fast = get_cached_telemetry_refs() diff --git a/backend/services/osint/openclaw_recon.py b/backend/services/osint/openclaw_recon.py new file mode 100644 index 0000000..64b6590 --- /dev/null +++ b/backend/services/osint/openclaw_recon.py @@ -0,0 +1,135 @@ +"""OpenClaw dispatch for the operator recon / OSINT lookup toolkit.""" +from __future__ import annotations + +from typing import Any + +from services.osint import lookups +from services.osint_intel.resolve import ALLOWED_TYPES, resolve_entity + +_OSINT_TOOLS: dict[str, str] = { + "ip": "ip", + "dns": "domain", + "whois": "domain", + "certs": "domain", + "threats": "query", + "bgp": "query", + "sanctions": "query", + "cve": "cve", + "mac": "mac", + "github": "username", + "leaks": "email", + "sweep_init": "ip", +} + +_ENTITY_SCHEMAS = frozenset({ + "Person", + "Organization", + "Company", + "Vessel", + "Airplane", + "LegalEntity", +}) + + +def _require_str(args: dict[str, Any], *keys: str) -> str: + for key in keys: + value = str(args.get(key, "") or "").strip() + if value: + return value + joined = "/".join(keys) + raise ValueError(f"Missing required argument: {joined}") + + +def run_osint_lookup(tool: str, args: dict[str, Any]) -> dict[str, Any]: + """Run a passive OSINT lookup (same backends as /api/osint/*).""" + name = str(tool or "").strip().lower().replace("-", "_") + if name not in _OSINT_TOOLS: + allowed = ", ".join(sorted(_OSINT_TOOLS)) + raise ValueError(f"Unknown OSINT tool '{tool}'. Allowed: {allowed}") + + if name == "ip": + return lookups.lookup_ip(_require_str(args, "ip", "query", "value")) + if name == "dns": + return lookups.lookup_dns(_require_str(args, "domain", "query", "value")) + if name == "whois": + return lookups.lookup_whois(_require_str(args, "domain", "query", "value")) + if name == "certs": + return lookups.lookup_certs(_require_str(args, "domain", "query", "value")) + if name == "threats": + query = str(args.get("query", "") or args.get("value", "") or "").strip() or None + return lookups.lookup_threats(query) + if name == "bgp": + return lookups.lookup_bgp(_require_str(args, "query", "asn", "value")) + if name == "sanctions": + query = _require_str(args, "query", "name", "value") + schema = str(args.get("schema", "") or "").strip() or None + if schema and schema not in _ENTITY_SCHEMAS: + allowed = ", ".join(sorted(_ENTITY_SCHEMAS)) + raise ValueError(f"Invalid schema. Allowed: {allowed}") + limit = args.get("limit", 25) + try: + limit = int(limit) + except (TypeError, ValueError): + limit = 25 + limit = max(1, min(100, limit)) + return lookups.lookup_sanctions(query, schema=schema, limit=limit) + if name == "cve": + return lookups.lookup_cve(_require_str(args, "cve", "query", "value")) + if name == "mac": + return lookups.lookup_mac(_require_str(args, "mac", "query", "value")) + if name == "github": + return lookups.lookup_github(_require_str(args, "username", "user", "query", "value")) + if name == "leaks": + return lookups.lookup_leaks(_require_str(args, "email", "query", "value")) + if name == "sweep_init": + ip = _require_str(args, "ip", "query", "value") + cidr = args.get("cidr", 24) + try: + cidr = int(cidr) + except (TypeError, ValueError): + cidr = 24 + return lookups.sweep_init(ip, cidr) + + raise ValueError(f"Unhandled OSINT tool: {name}") + + +def run_osint_sweep(args: dict[str, Any]) -> dict[str, Any]: + """Run subnet device discovery (Shodan InternetDB proxy). Requires full access tier.""" + ip = _require_str(args, "ip", "query", "value") + cidr = args.get("cidr", 24) + try: + cidr = int(cidr) + except (TypeError, ValueError): + cidr = 24 + subnet = lookups.subnet_start_for(ip, cidr) + scan = lookups.sweep_scan(subnet, cidr) + init = lookups.sweep_init(ip, cidr) + return {**init, **scan, "subnet": f"{subnet}/{cidr}"} + + +def run_entity_expand(args: dict[str, Any]) -> dict[str, Any]: + """Expand an entity graph node (aircraft, vessel, IP, company, person, country).""" + entity_type = _require_str(args, "type", "entity_type") + entity_id = _require_str(args, "id", "entity_id", "query", "value") + props = { + "label": entity_id, + "registration": str(args.get("registration", "") or "").strip() or None, + "model": str(args.get("model", "") or "").strip() or None, + "icao24": str(args.get("icao24", "") or "").strip() or None, + } + props = {key: value for key, value in props.items() if value is not None} + return resolve_entity(entity_type, entity_id, props) + + +def osint_tool_help() -> dict[str, Any]: + """Discovery metadata for agents.""" + return { + "tools": sorted(_OSINT_TOOLS), + "entity_types": sorted(ALLOWED_TYPES), + "sanctions_schemas": sorted(_ENTITY_SCHEMAS), + "notes": { + "osint_lookup": "Passive lookups — same data as the Recon panel /api/osint/* routes.", + "osint_sweep": "Active subnet scan via Shodan InternetDB — requires full OpenClaw access tier.", + "entity_expand": "Build a relationship graph around aircraft, vessels, IPs, companies, people, or countries.", + }, + } diff --git a/backend/services/telemetry.py b/backend/services/telemetry.py index bfcc67d..effcded 100644 --- a/backend/services/telemetry.py +++ b/backend/services/telemetry.py @@ -93,6 +93,10 @@ _SLOW_KEYS = ( "sar_scenes", "sar_anomalies", "sar_aoi_coverage", + "malware_threats", + "cyber_threats", + "scm_suppliers", + "telegram_osint", ) @@ -158,6 +162,20 @@ _ENTITY_LAYER_ALIASES = { "uap_sightings": "uap_sightings", "wastewater": "wastewater", "pins": "pins", + "telegram": "telegram_osint", + "telegram_osint": "telegram_osint", + "osint_feed": "telegram_osint", + "malware": "malware_threats", + "malware_threats": "malware_threats", + "malware_c2": "malware_threats", + "botnet": "malware_threats", + "cyber": "cyber_threats", + "cyber_threats": "cyber_threats", + "cisa": "cyber_threats", + "kev": "cyber_threats", + "scm": "scm_suppliers", + "scm_suppliers": "scm_suppliers", + "suppliers": "scm_suppliers", } _SLICEABLE_LAYERS = tuple(dict.fromkeys(_FAST_KEYS + _SLOW_KEYS)) @@ -188,6 +206,21 @@ _LAYER_ALIASES = { "sar_coverage": "sar_aoi_coverage", # Satellite analysis (maneuvers, decay, Starlink) "satellite_analysis": "satellite_analysis", + # OSINT / cyber / supply-chain overlays + "telegram": "telegram_osint", + "telegram_osint": "telegram_osint", + "osint_feed": "telegram_osint", + "malware": "malware_threats", + "malware_threats": "malware_threats", + "malware_c2": "malware_threats", + "botnet": "malware_threats", + "cyber": "cyber_threats", + "cyber_threats": "cyber_threats", + "cisa": "cyber_threats", + "kev": "cyber_threats", + "scm": "scm_suppliers", + "scm_suppliers": "scm_suppliers", + "suppliers": "scm_suppliers", } _UNIVERSAL_SEARCH_DEFAULT_LAYERS = ( @@ -225,6 +258,10 @@ _UNIVERSAL_SEARCH_DEFAULT_LAYERS = ( "tinygs_satellites", "psk_reporter", "ukraine_alerts", + "telegram_osint", + "malware_threats", + "cyber_threats", + "scm_suppliers", ) _GENERIC_QUERY_STOPWORDS = { @@ -269,7 +306,19 @@ _GENERIC_LAYER_HINTS: dict[str, tuple[str, ...]] = { "protest": ("crowdthreat", "gdelt", "news", "frontlines", "liveuamap"), "riot": ("crowdthreat", "gdelt", "news", "frontlines", "liveuamap"), "event": ("crowdthreat", "gdelt", "news", "frontlines", "liveuamap"), - "news": ("news", "gdelt", "crowdthreat", "frontlines", "liveuamap"), + "news": ("news", "gdelt", "crowdthreat", "frontlines", "liveuamap", "telegram_osint"), + "telegram": ("telegram_osint",), + "osint": ("telegram_osint", "gdelt", "news", "crowdthreat"), + "channel": ("telegram_osint",), + "malware": ("malware_threats",), + "botnet": ("malware_threats",), + "c2": ("malware_threats",), + "cve": ("cyber_threats",), + "cisa": ("cyber_threats",), + "cyber": ("cyber_threats", "malware_threats"), + "supplier": ("scm_suppliers",), + "scm": ("scm_suppliers",), + "semiconductor": ("scm_suppliers",), "plant": ("power_plants", "wastewater"), "datacenter": ("datacenters",), "data": ("datacenters",), @@ -314,6 +363,10 @@ _SEARCH_GROUP_BY_LAYER = { "kiwisdr": "signals", "psk_reporter": "signals", "ukraine_alerts": "events", + "telegram_osint": "events", + "malware_threats": "cyber", + "cyber_threats": "cyber", + "scm_suppliers": "infrastructure", } _SEARCH_QUERY_SYNONYMS: dict[str, tuple[str, ...]] = { @@ -328,6 +381,9 @@ _SEARCH_QUERY_SYNONYMS: dict[str, tuple[str, ...]] = { "plants": ("plant",), "cameras": ("camera",), "radios": ("radio",), + "telegrams": ("telegram",), + "channels": ("channel",), + "suppliers": ("supplier",), } _SEARCH_INDEX_LOCK = threading.Lock() @@ -653,6 +709,42 @@ _UNIVERSAL_SEARCH_SPECS: dict[str, dict[str, Any]] = { "id_fields": ("id",), "time_fields": ("updated_at", "timestamp"), }, + "telegram_osint": { + "fields": ("title", "description", "source", "channel", "link"), + "primary_fields": ("title", "description", "channel"), + "label_fields": ("title", "channel"), + "summary_fields": ("description", "source"), + "type_fields": ("channel", "source"), + "id_fields": ("id", "link"), + "time_fields": ("published", "timestamp"), + }, + "malware_threats": { + "fields": ("ip", "malware", "status", "country", "threat_type"), + "primary_fields": ("ip", "malware", "country"), + "label_fields": ("ip", "malware"), + "summary_fields": ("status", "country", "threat_type"), + "type_fields": ("threat_type", "malware"), + "id_fields": ("id", "ip"), + "time_fields": ("first_seen", "last_online", "timestamp"), + }, + "cyber_threats": { + "fields": ("id", "name", "vendor", "product", "severity", "source"), + "primary_fields": ("id", "name", "vendor", "product"), + "label_fields": ("id", "name"), + "summary_fields": ("vendor", "product", "severity", "source"), + "type_fields": ("severity", "source"), + "id_fields": ("id",), + "time_fields": ("date", "due", "timestamp"), + }, + "scm_suppliers": { + "fields": ("name", "city", "country", "category", "risk_level"), + "primary_fields": ("name", "city", "country", "category"), + "label_fields": ("name", "city"), + "summary_fields": ("country", "category", "risk_level"), + "type_fields": ("category", "risk_level"), + "id_fields": ("id",), + "time_fields": ("timestamp",), + }, } @@ -734,6 +826,11 @@ def _extract_coords(candidate: dict[str, Any]) -> tuple[float | None, float | No if isinstance(coords, (list, tuple)) and len(coords) >= 2: lng = lng if lng is not None else _coerce_float(coords[0]) lat = lat if lat is not None else _coerce_float(coords[1]) + if lat is None or lng is None: + coords = candidate.get("coords") + if isinstance(coords, (list, tuple)) and len(coords) >= 2: + lat = lat if lat is not None else _coerce_float(coords[0]) + lng = lng if lng is not None else _coerce_float(coords[1]) return lat, lng @@ -832,6 +929,53 @@ def _layer_group(layer: str) -> str: return _SEARCH_GROUP_BY_LAYER.get(layer, "other") +_LAYER_NESTED_LIST_KEYS: dict[str, tuple[str, ...]] = { + "telegram_osint": ("posts",), + "malware_threats": ("threats",), + "cyber_threats": ("threats",), + "scm_suppliers": ("suppliers",), +} +_DEFAULT_NESTED_LIST_KEYS = ( + "items", + "results", + "vessels", + "features", + "posts", + "threats", + "suppliers", +) + + +def _unwrap_layer_items(value: Any, layer: str = "") -> list[Any]: + """Return the searchable/geospatial item list inside a layer value.""" + if isinstance(value, list): + return value + if not isinstance(value, dict): + return [] + keys = _LAYER_NESTED_LIST_KEYS.get(layer, _DEFAULT_NESTED_LIST_KEYS) + for key in keys: + nested = value.get(key) + if isinstance(nested, list): + return nested + return [] + + +def _layer_record_count(value: Any, layer: str = "") -> int: + if isinstance(value, list): + return len(value) + if isinstance(value, dict): + total = value.get("total") + if isinstance(total, (int, float)): + return int(total) + items = _unwrap_layer_items(value, layer) + if items: + return len(items) + return len(value) if value else 0 + if value is None: + return 0 + return 1 + + def _build_search_document(doc_id: int, layer: str, candidate: dict[str, Any], spec: dict[str, Any]) -> dict[str, Any]: fields = tuple(spec.get("fields", ())) text = _document_text(candidate, fields) @@ -880,9 +1024,7 @@ def _get_search_index() -> dict[str, Any]: for layer in layers: spec = _UNIVERSAL_SEARCH_SPECS[layer] - items = snap.get(layer) or [] - if isinstance(items, dict): - items = items.get("items", []) or items.get("results", []) or items.get("vessels", []) + items = _unwrap_layer_items(snap.get(layer), layer) if not isinstance(items, list): continue for item in items: @@ -1144,18 +1286,9 @@ def get_telemetry_summary() -> dict[str, Any]: for layer in layer_names: value = snap.get(layer) - if isinstance(value, list): - counts[layer] = len(value) - if value: - non_empty_layers.append(layer) - elif isinstance(value, dict): - counts[layer] = len(value) - if value: - non_empty_layers.append(layer) - elif value is None: - counts[layer] = 0 - else: - counts[layer] = 1 + count = _layer_record_count(value, layer) + counts[layer] = count + if count > 0: non_empty_layers.append(layer) alias_examples = { @@ -1167,6 +1300,16 @@ def get_telemetry_summary() -> dict[str, Any]: "tracked": "tracked_flights", "military": "military_flights", "jets": "private_jets", + "telegram": "telegram_osint", + "osint_feed": "telegram_osint", + "malware": "malware_threats", + "malware_c2": "malware_threats", + "botnet": "malware_threats", + "cyber": "cyber_threats", + "cisa": "cyber_threats", + "kev": "cyber_threats", + "scm": "scm_suppliers", + "suppliers": "scm_suppliers", } return { @@ -1577,14 +1720,7 @@ def _nearby_items_from_layers( snap = get_latest_data_subset_refs(*layers) out: dict[str, list[dict[str, Any]]] = {} for layer in layers: - value = snap.get(layer) or [] - if isinstance(value, dict): - if layer == "gdelt" and isinstance(value.get("features"), list): - items = value.get("features") or [] - else: - items = value.get("items") or value.get("features") or value.get("vessels") or [] - else: - items = value + items = _unwrap_layer_items(snap.get(layer), layer) if not isinstance(items, list): continue matches: list[dict[str, Any]] = [] @@ -1728,6 +1864,9 @@ def correlate_entity( "crowdthreat", "frontlines", "liveuamap", + "telegram_osint", + "malware_threats", + "scm_suppliers", "military_bases", "datacenters", "power_plants", @@ -1809,13 +1948,17 @@ def search_news( query: str, limit: int = 10, include_gdelt: bool = True, + include_telegram: bool = True, ) -> dict[str, Any]: """Search news and event layers server-side and return a compact result set.""" query_norm = _norm_text(query) if not query_norm: return {"results": [], "version": get_data_version(), "truncated": False} - snap = get_latest_data_subset_refs("news", "gdelt", "crowdthreat", "liveuamap", "frontlines") + layer_keys = ["news", "gdelt", "crowdthreat", "liveuamap", "frontlines"] + if include_telegram: + layer_keys.append("telegram_osint") + snap = get_latest_data_subset_refs(*layer_keys) out: list[dict[str, Any]] = [] limit = _coerce_limit(limit, default=10, maximum=50) @@ -1941,6 +2084,36 @@ def search_news( if len(out) >= limit: return {"results": out, "version": get_data_version(), "truncated": True} + if include_telegram: + for post in _unwrap_layer_items(snap.get("telegram_osint"), "telegram_osint"): + if not isinstance(post, dict): + continue + text = " ".join( + ( + _norm_text(post.get("title")), + _norm_text(post.get("description")), + _norm_text(post.get("source")), + _norm_text(post.get("channel")), + ) + ) + if not _text_matches_query(query_norm, text): + continue + lat, lng = _extract_coords(post) + out.append( + { + "source_layer": "telegram_osint", + "title": post.get("title") or "", + "summary": post.get("description") or "", + "source": post.get("source") or post.get("channel") or "Telegram", + "link": post.get("link") or "", + "lat": lat, + "lng": lng, + "risk_score": post.get("risk_score"), + } + ) + if len(out) >= limit: + return {"results": out, "version": get_data_version(), "truncated": True} + return {"results": out, "version": get_data_version(), "truncated": False} @@ -2205,16 +2378,13 @@ def entities_near( out: list[dict[str, Any]] = [] for layer in layers: - items = snap.get(layer) or [] - if isinstance(items, dict): - items = items.get("vessels", []) or items.get("items", []) + items = _unwrap_layer_items(snap.get(layer), layer) if not isinstance(items, list): continue for item in items: if not isinstance(item, dict): continue - item_lat = _coerce_float(item.get("lat") or item.get("latitude")) - item_lng = _coerce_float(item.get("lng") or item.get("lon") or item.get("longitude")) + item_lat, item_lng = _extract_coords(item) if item_lat is None or item_lng is None: continue distance = _haversine_km(center_lat, center_lng, item_lat, item_lng) diff --git a/backend/tests/test_openclaw_query_helpers.py b/backend/tests/test_openclaw_query_helpers.py index bcbc5bb..1093bc8 100644 --- a/backend/tests/test_openclaw_query_helpers.py +++ b/backend/tests/test_openclaw_query_helpers.py @@ -28,6 +28,10 @@ def sample_store(): "weather_alerts": list(latest_data.get("weather_alerts") or []), "gps_jamming": list(latest_data.get("gps_jamming") or []), "military_bases": list(latest_data.get("military_bases") or []), + "telegram_osint": dict(latest_data.get("telegram_osint") or {}), + "malware_threats": dict(latest_data.get("malware_threats") or {}), + "cyber_threats": dict(latest_data.get("cyber_threats") or {}), + "scm_suppliers": dict(latest_data.get("scm_suppliers") or {}), } latest_data["tracked_flights"] = [ { @@ -188,6 +192,66 @@ def sample_store(): "lng": -76.87, } ] + latest_data["telegram_osint"] = { + "posts": [ + { + "id": "tg-1", + "title": "Missile strike reported near Kyiv overnight", + "description": "OSINT channel reports explosions near Kyiv", + "channel": "osintdefender", + "source": "t.me/osintdefender", + "link": "https://t.me/osintdefender/123", + "published": "2026-06-02T12:00:00+00:00", + "risk_score": 0.8, + "coords": [50.45, 30.52], + } + ], + "total": 1, + "geolocated": 1, + } + latest_data["malware_threats"] = { + "threats": [ + { + "id": "feodo-1", + "ip": "203.0.113.10", + "malware": "Emotet", + "country": "US", + "threat_type": "botnet_c2", + "lat": 38.95, + "lng": -77.45, + } + ], + "total": 1, + } + latest_data["cyber_threats"] = { + "threats": [ + { + "id": "CVE-2026-1234", + "name": "Example Vendor RCE", + "vendor": "Example Vendor", + "product": "Example Product", + "severity": "CRITICAL", + "source": "CISA KEV", + } + ], + "stats": {"active_cves": 1}, + } + latest_data["scm_suppliers"] = { + "suppliers": [ + { + "id": "sup-tsmc-hsinchu", + "name": "TSMC Fab 12 (Tier 1)", + "city": "Hsinchu", + "country": "Taiwan", + "category": "Semiconductor", + "risk_level": "NORMAL", + "lat": 24.774, + "lng": 120.992, + } + ], + "total": 1, + "critical_count": 0, + } try: yield @@ -475,6 +539,89 @@ def test_correlate_entity_returns_evidence_pack_near_aircraft(sample_store, monk assert result["recommended_next"] +def test_get_slow_telemetry_includes_new_osint_layers(sample_store, monkeypatch): + import services.telemetry as telemetry + + monkeypatch.setattr(telemetry, "get_data_version", lambda: 210) + result = telemetry.get_cached_slow_telemetry() + + assert "telegram_osint" in result + assert result["telegram_osint"]["total"] == 1 + assert "malware_threats" in result + assert result["malware_threats"]["total"] == 1 + assert "scm_suppliers" in result + assert result["scm_suppliers"]["total"] == 1 + + +def test_get_layer_slice_accepts_telegram_alias(sample_store, monkeypatch): + import services.telemetry as telemetry + + monkeypatch.setattr(telemetry, "get_data_version", lambda: 211) + result = telemetry.get_layer_slice(layers=["telegram"], limit_per_layer=10) + + assert result["requested_layers"] == ["telegram_osint"] + assert result["layers"]["telegram_osint"]["posts"][0]["channel"] == "osintdefender" + + +def test_get_telemetry_summary_counts_nested_layer_items(sample_store, monkeypatch): + import services.telemetry as telemetry + + monkeypatch.setattr(telemetry, "get_data_version", lambda: 212) + result = telemetry.get_telemetry_summary() + + assert result["counts"]["telegram_osint"] == 1 + assert result["counts"]["malware_threats"] == 1 + assert result["counts"]["scm_suppliers"] == 1 + assert "telegram_osint" in result["non_empty_layers"] + assert result["layer_aliases"]["telegram"] == "telegram_osint" + assert result["layer_aliases"]["scm"] == "scm_suppliers" + + +def test_search_news_matches_telegram_osint(sample_store, monkeypatch): + import services.telemetry as telemetry + + monkeypatch.setattr(telemetry, "get_data_version", lambda: 213) + result = telemetry.search_news(query="kyiv missile", limit=10, include_telegram=True) + + assert result["results"] + assert result["results"][0]["source_layer"] == "telegram_osint" + assert result["results"][0]["lat"] == 50.45 + + +def test_search_telemetry_finds_telegram_malware_and_scm(sample_store, monkeypatch): + import services.telemetry as telemetry + + monkeypatch.setattr(telemetry, "get_data_version", lambda: 214) + + telegram = telemetry.search_telemetry(query="osintdefender kyiv", limit=10) + assert any(item["source_layer"] == "telegram_osint" for item in telegram["results"]) + + malware = telemetry.search_telemetry(query="emotet", limit=10) + assert any(item["source_layer"] == "malware_threats" for item in malware["results"]) + + scm = telemetry.search_telemetry(query="tsmc hsinchu", limit=10) + assert any(item["source_layer"] == "scm_suppliers" for item in scm["results"]) + + cve = telemetry.search_telemetry(query="CVE-2026-1234", limit=10) + assert any(item["source_layer"] == "cyber_threats" for item in cve["results"]) + + +def test_entities_near_finds_telegram_and_malware(sample_store, monkeypatch): + import services.telemetry as telemetry + + monkeypatch.setattr(telemetry, "get_data_version", lambda: 215) + result = telemetry.entities_near( + lat=38.95, + lng=-77.45, + radius_km=50, + entity_types=["telegram", "malware"], + limit=10, + ) + + layers = {item["source_layer"] for item in result["results"]} + assert "malware_threats" in layers + + def test_openclaw_correlate_entity_command(sample_store, monkeypatch): import services.telemetry as telemetry from services.openclaw_channel import _dispatch_command diff --git a/backend/tests/test_openclaw_recon.py b/backend/tests/test_openclaw_recon.py new file mode 100644 index 0000000..5464107 --- /dev/null +++ b/backend/tests/test_openclaw_recon.py @@ -0,0 +1,98 @@ +"""Tests for OpenClaw recon / OSINT command dispatch.""" + +import pytest + + +def test_osint_tools_lists_supported_lookups(): + from services.osint.openclaw_recon import osint_tool_help + + help_data = osint_tool_help() + assert "ip" in help_data["tools"] + assert "sanctions" in help_data["tools"] + assert "aircraft" in help_data["entity_types"] + + +def test_osint_lookup_ip(monkeypatch): + from services.osint import openclaw_recon + + monkeypatch.setattr( + openclaw_recon.lookups, + "lookup_ip", + lambda ip: {"ip": ip, "geo": {"country": "US"}}, + ) + result = openclaw_recon.run_osint_lookup("ip", {"ip": "8.8.8.8"}) + assert result["ip"] == "8.8.8.8" + assert result["geo"]["country"] == "US" + + +def test_osint_lookup_sanctions_passes_schema(monkeypatch): + from services.osint import openclaw_recon + + captured = {} + + def fake_sanctions(query, *, schema=None, limit=25): + captured["query"] = query + captured["schema"] = schema + captured["limit"] = limit + return {"query": query, "results": []} + + monkeypatch.setattr(openclaw_recon.lookups, "lookup_sanctions", fake_sanctions) + openclaw_recon.run_osint_lookup( + "sanctions", + {"query": "Example Corp", "schema": "Company", "limit": 10}, + ) + assert captured["query"] == "Example Corp" + assert captured["schema"] == "Company" + assert captured["limit"] == 10 + + +def test_osint_lookup_rejects_unknown_tool(): + from services.osint.openclaw_recon import run_osint_lookup + + with pytest.raises(ValueError, match="Unknown OSINT tool"): + run_osint_lookup("not_a_tool", {}) + + +def test_openclaw_osint_lookup_command(monkeypatch): + from services import openclaw_channel + + monkeypatch.setattr( + "services.osint.openclaw_recon.run_osint_lookup", + lambda tool, args: {"ip": args["ip"], "tool": tool}, + ) + result = openclaw_channel._dispatch_command( + "osint_lookup", + {"tool": "ip", "ip": "1.1.1.1"}, + ) + assert result["ok"] is True + assert result["data"]["ip"] == "1.1.1.1" + + +def test_openclaw_entity_expand_command(monkeypatch): + from services import openclaw_channel + + monkeypatch.setattr( + "services.osint.openclaw_recon.run_entity_expand", + lambda args: {"nodes": [{"id": "ip:1.1.1.1"}], "links": []}, + ) + result = openclaw_channel._dispatch_command( + "entity_expand", + {"type": "ip", "id": "1.1.1.1"}, + ) + assert result["ok"] is True + assert result["data"]["nodes"][0]["id"] == "ip:1.1.1.1" + + +def test_osint_sweep_requires_full_tier_for_restricted(): + from services.openclaw_channel import WRITE_COMMANDS, allowed_commands + + assert "osint_sweep" in WRITE_COMMANDS + assert "osint_sweep" not in allowed_commands("restricted") + assert "osint_sweep" in allowed_commands("full") + + +def test_osint_lookup_available_on_restricted_tier(): + from services.openclaw_channel import allowed_commands + + assert "osint_lookup" in allowed_commands("restricted") + assert "entity_expand" in allowed_commands("restricted") diff --git a/openclaw-skills/shadowbroker/SKILL.md b/openclaw-skills/shadowbroker/SKILL.md index cadd114..e36a23e 100644 --- a/openclaw-skills/shadowbroker/SKILL.md +++ b/openclaw-skills/shadowbroker/SKILL.md @@ -126,7 +126,7 @@ The channel operates over HMAC-authenticated HTTP with body-integrity binding: | Method | What It Returns | |--------|----------------| | `await sb.get_telemetry()` | Fast-tier: flights, ships, satellites, SIGINT, LiveUAMap, CCTV, GPS jamming | -| `await sb.get_slow_telemetry()` | Slow-tier: GDELT, news, earthquakes, markets, correlations | +| `await sb.get_slow_telemetry()` | Slow-tier: GDELT, news, earthquakes, markets, correlations, Telegram OSINT, malware/cyber threats, SCM suppliers | | `await sb.get_report()` | Full structured intelligence report | **When to use**: Use `get_summary()` first. Use `get_layer_slice()` for the layers @@ -148,6 +148,59 @@ Every layer returns maximum telemetry. Key enriched fields: | **GPS Jamming** | `lat`, `lng`, `name`/`region`, `intensity`, `source` | | **Earthquakes** | `lat`, `lng`, `magnitude`, `depth`, `place`, `time` | | **Correlations** | `type`, `severity`, `score`, `lat`, `lng`, `drivers` (triggering layers) | +| **Telegram OSINT** | `title`, `description`, `channel`, `source`, `link`, `published`, `risk_score`, `coords` `[lat, lng]` | +| **Malware Threats** | `ip`, `malware`, `threat_type`, `status`, `country`, `lat`, `lng` (Feodo + URLhaus) | +| **Cyber Threats** | `id` (CVE), `name`, `vendor`, `product`, `severity`, `date` (CISA KEV) | +| **SCM Suppliers** | `name`, `city`, `country`, `category`, `risk_level`, `active_threats`, `lat`, `lng` | + +**Layer aliases for `get_layer_slice` / `search_telemetry`:** `telegram` → `telegram_osint`, `malware`/`botnet` → `malware_threats`, `cyber`/`cisa`/`kev` → `cyber_threats`, `scm`/`suppliers` → `scm_suppliers`. + +### 1b. Recon / OSINT Toolkit + +The Recon panel lookups are available on the OpenClaw command channel — no need to hit `/api/osint/*` directly. + +```python +# List supported tools +await sb.send_command("osint_tools") + +# IP geolocation + threat context +await sb.send_command("osint_lookup", {"tool": "ip", "ip": "8.8.8.8"}) + +# DNS, WHOIS, certificate transparency +await sb.send_command("osint_lookup", {"tool": "dns", "domain": "example.com"}) +await sb.send_command("osint_lookup", {"tool": "whois", "domain": "example.com"}) +await sb.send_command("osint_lookup", {"tool": "certs", "domain": "example.com"}) + +# BGP/ASN, sanctions, CVE, MAC vendor, GitHub, breach check +await sb.send_command("osint_lookup", {"tool": "bgp", "query": "AS15169"}) +await sb.send_command("osint_lookup", {"tool": "sanctions", "query": "Rosneft"}) +await sb.send_command("osint_lookup", {"tool": "cve", "cve": "CVE-2024-1234"}) +await sb.send_command("osint_lookup", {"tool": "mac", "mac": "00:11:22:33:44:55"}) +await sb.send_command("osint_lookup", {"tool": "github", "username": "octocat"}) +await sb.send_command("osint_lookup", {"tool": "leaks", "email": "user@example.com"}) + +# Entity relationship graph (aircraft, vessel, ip, company, person, country) +await sb.send_command("entity_expand", {"type": "ip", "id": "8.8.8.8"}) +await sb.send_command("entity_expand", {"type": "aircraft", "id": "N400QS", "icao24": "a0f011"}) + +# Subnet sweep (full tier only — active Shodan InternetDB scan) +await sb.send_command("osint_sweep", {"ip": "1.2.3.4", "cidr": 24}) +``` + +| `osint_lookup` tool | Args | What you get | +|---------------------|------|--------------| +| `ip` | `ip` | Geo, ISP, ASN, proxy/hosting flags, sanctions cross-check | +| `dns` | `domain` | A/AAAA/MX/NS/TXT records | +| `whois` | `domain` | Registrar, dates, nameservers | +| `certs` | `domain` | Certificate transparency hits | +| `threats` | `query` (optional) | Aggregated threat intel | +| `bgp` | `query` | ASN/prefix routing data | +| `sanctions` | `query`, `schema`, `limit` | OFAC / sanctions index matches | +| `cve` | `cve` | NVD CVE details | +| `mac` | `mac` | Vendor OUI lookup | +| `github` | `username` | Public profile metadata | +| `leaks` | `email` | Breach exposure check | +| `sweep_init` | `ip`, `cidr` | Passive geolocation context for a sweep target | ### 2. Pin Placement (AI Intel Map Layer) diff --git a/openclaw-skills/shadowbroker/sb_query.py b/openclaw-skills/shadowbroker/sb_query.py index 9e37704..9960e5a 100644 --- a/openclaw-skills/shadowbroker/sb_query.py +++ b/openclaw-skills/shadowbroker/sb_query.py @@ -722,6 +722,36 @@ class ShadowBrokerClient: r = await self._get("/api/ai/summary") return r.json() + async def osint_lookup(self, tool: str, **kwargs) -> dict: + """Run a passive OSINT recon lookup (IP, DNS, WHOIS, sanctions, CVE, etc.).""" + args = {"tool": tool, **kwargs} + result = await self.send_command("osint_lookup", args) + if not result.get("ok"): + raise RuntimeError(result.get("detail") or "osint_lookup failed") + return result.get("data") or result + + async def osint_tools(self) -> dict: + """List available OSINT recon tools and entity-expand types.""" + result = await self.send_command("osint_tools") + if not result.get("ok"): + raise RuntimeError(result.get("detail") or "osint_tools failed") + return result.get("data") or result + + async def entity_expand(self, entity_type: str, entity_id: str, **kwargs) -> dict: + """Expand an entity relationship graph.""" + args = {"type": entity_type, "id": entity_id, **kwargs} + result = await self.send_command("entity_expand", args) + if not result.get("ok"): + raise RuntimeError(result.get("detail") or "entity_expand failed") + return result.get("data") or result + + async def osint_sweep(self, ip: str, cidr: int = 24) -> dict: + """Active subnet device discovery (requires full OpenClaw access tier).""" + result = await self.send_command("osint_sweep", {"ip": ip, "cidr": cidr}) + if not result.get("ok"): + raise RuntimeError(result.get("detail") or "osint_sweep failed") + return result.get("data") or result + # ── Encrypted DMs ───────────────────────────────────────────── async def send_encrypted_dm(self, recipient_pubkey: str, message: str) -> dict: