CalvinBackup/Shadowbroker
1
0
Fork 0
mirror of https://github.com/BigBodyCobain/Shadowbroker.git synced 2026-08-29 21:50:37 +02:00
Code Issues Packages Projects Releases Wiki Activity

Expose new telemetry and recon toolkit to OpenClaw agents.

Browse Source 
Wire telegram_osint, malware, cyber, and SCM into search/slow-tier helpers; add osint_lookup, entity_expand, and osint_sweep commands; update README and skill docs.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
BigBodyCobain
2026-06-08 22:44:16 -06:00
co-authored by Cursor
parent af9b3d08cc
commit 1583fd5715
9 changed files with 793 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/telemetry.py
+200 -30
View File
@@ -93,6 +93,10 @@ _SLOW_KEYS = (
"sar_scenes",
"sar_anomalies",
"sar_aoi_coverage",
"malware_threats",
"cyber_threats",
"scm_suppliers",
"telegram_osint",
)
@@ -158,6 +162,20 @@ _ENTITY_LAYER_ALIASES = {
"uap_sightings": "uap_sightings",
"wastewater": "wastewater",
"pins": "pins",
"telegram": "telegram_osint",
"telegram_osint": "telegram_osint",
"osint_feed": "telegram_osint",
"malware": "malware_threats",
"malware_threats": "malware_threats",
"malware_c2": "malware_threats",
"botnet": "malware_threats",
"cyber": "cyber_threats",
"cyber_threats": "cyber_threats",
"cisa": "cyber_threats",
"kev": "cyber_threats",
"scm": "scm_suppliers",
"scm_suppliers": "scm_suppliers",
"suppliers": "scm_suppliers",
}
_SLICEABLE_LAYERS = tuple(dict.fromkeys(_FAST_KEYS + _SLOW_KEYS))
@@ -188,6 +206,21 @@ _LAYER_ALIASES = {
"sar_coverage": "sar_aoi_coverage",
# Satellite analysis (maneuvers, decay, Starlink)
"satellite_analysis": "satellite_analysis",
# OSINT / cyber / supply-chain overlays
"telegram": "telegram_osint",
"telegram_osint": "telegram_osint",
"osint_feed": "telegram_osint",
"malware": "malware_threats",
"malware_threats": "malware_threats",
"malware_c2": "malware_threats",
"botnet": "malware_threats",
"cyber": "cyber_threats",
"cyber_threats": "cyber_threats",
"cisa": "cyber_threats",
"kev": "cyber_threats",
"scm": "scm_suppliers",
"scm_suppliers": "scm_suppliers",
"suppliers": "scm_suppliers",
}
_UNIVERSAL_SEARCH_DEFAULT_LAYERS = (
@@ -225,6 +258,10 @@ _UNIVERSAL_SEARCH_DEFAULT_LAYERS = (
"tinygs_satellites",
"psk_reporter",
"ukraine_alerts",
"telegram_osint",
"malware_threats",
"cyber_threats",
"scm_suppliers",
)
_GENERIC_QUERY_STOPWORDS = {
@@ -269,7 +306,19 @@ _GENERIC_LAYER_HINTS: dict[str, tuple[str, ...]] = {
"protest": ("crowdthreat", "gdelt", "news", "frontlines", "liveuamap"),
"riot": ("crowdthreat", "gdelt", "news", "frontlines", "liveuamap"),
"event": ("crowdthreat", "gdelt", "news", "frontlines", "liveuamap"),
"news": ("news", "gdelt", "crowdthreat", "frontlines", "liveuamap"),
"news": ("news", "gdelt", "crowdthreat", "frontlines", "liveuamap", "telegram_osint"),
"telegram": ("telegram_osint",),
"osint": ("telegram_osint", "gdelt", "news", "crowdthreat"),
"channel": ("telegram_osint",),
"malware": ("malware_threats",),
"botnet": ("malware_threats",),
"c2": ("malware_threats",),
"cve": ("cyber_threats",),
"cisa": ("cyber_threats",),
"cyber": ("cyber_threats", "malware_threats"),
"supplier": ("scm_suppliers",),
"scm": ("scm_suppliers",),
"semiconductor": ("scm_suppliers",),
"plant": ("power_plants", "wastewater"),
"datacenter": ("datacenters",),
"data": ("datacenters",),
@@ -314,6 +363,10 @@ _SEARCH_GROUP_BY_LAYER = {
"kiwisdr": "signals",
"psk_reporter": "signals",
"ukraine_alerts": "events",
"telegram_osint": "events",
"malware_threats": "cyber",
"cyber_threats": "cyber",
"scm_suppliers": "infrastructure",
}
_SEARCH_QUERY_SYNONYMS: dict[str, tuple[str, ...]] = {
@@ -328,6 +381,9 @@ _SEARCH_QUERY_SYNONYMS: dict[str, tuple[str, ...]] = {
"plants": ("plant",),
"cameras": ("camera",),
"radios": ("radio",),
"telegrams": ("telegram",),
"channels": ("channel",),
"suppliers": ("supplier",),
}
_SEARCH_INDEX_LOCK = threading.Lock()
@@ -653,6 +709,42 @@ _UNIVERSAL_SEARCH_SPECS: dict[str, dict[str, Any]] = {
"id_fields": ("id",),
"time_fields": ("updated_at", "timestamp"),
},
"telegram_osint": {
"fields": ("title", "description", "source", "channel", "link"),
"primary_fields": ("title", "description", "channel"),
"label_fields": ("title", "channel"),
"summary_fields": ("description", "source"),
"type_fields": ("channel", "source"),
"id_fields": ("id", "link"),
"time_fields": ("published", "timestamp"),
},
"malware_threats": {
"fields": ("ip", "malware", "status", "country", "threat_type"),
"primary_fields": ("ip", "malware", "country"),
"label_fields": ("ip", "malware"),
"summary_fields": ("status", "country", "threat_type"),
"type_fields": ("threat_type", "malware"),
"id_fields": ("id", "ip"),
"time_fields": ("first_seen", "last_online", "timestamp"),
},
"cyber_threats": {
"fields": ("id", "name", "vendor", "product", "severity", "source"),
"primary_fields": ("id", "name", "vendor", "product"),
"label_fields": ("id", "name"),
"summary_fields": ("vendor", "product", "severity", "source"),
"type_fields": ("severity", "source"),
"id_fields": ("id",),
"time_fields": ("date", "due", "timestamp"),
},
"scm_suppliers": {
"fields": ("name", "city", "country", "category", "risk_level"),
"primary_fields": ("name", "city", "country", "category"),
"label_fields": ("name", "city"),
"summary_fields": ("country", "category", "risk_level"),
"type_fields": ("category", "risk_level"),
"id_fields": ("id",),
"time_fields": ("timestamp",),
},
}
@@ -734,6 +826,11 @@ def _extract_coords(candidate: dict[str, Any]) -> tuple[float | None, float | No
if isinstance(coords, (list, tuple)) and len(coords) >= 2:
lng = lng if lng is not None else _coerce_float(coords[0])
lat = lat if lat is not None else _coerce_float(coords[1])
if lat is None or lng is None:
coords = candidate.get("coords")
if isinstance(coords, (list, tuple)) and len(coords) >= 2:
lat = lat if lat is not None else _coerce_float(coords[0])
lng = lng if lng is not None else _coerce_float(coords[1])
return lat, lng
@@ -832,6 +929,53 @@ def _layer_group(layer: str) -> str:
return _SEARCH_GROUP_BY_LAYER.get(layer, "other")
_LAYER_NESTED_LIST_KEYS: dict[str, tuple[str, ...]] = {
"telegram_osint": ("posts",),
"malware_threats": ("threats",),
"cyber_threats": ("threats",),
"scm_suppliers": ("suppliers",),
}
_DEFAULT_NESTED_LIST_KEYS = (
"items",
"results",
"vessels",
"features",
"posts",
"threats",
"suppliers",
)
def _unwrap_layer_items(value: Any, layer: str = "") -> list[Any]:
"""Return the searchable/geospatial item list inside a layer value."""
if isinstance(value, list):
return value
if not isinstance(value, dict):
return []
keys = _LAYER_NESTED_LIST_KEYS.get(layer, _DEFAULT_NESTED_LIST_KEYS)
for key in keys:
nested = value.get(key)
if isinstance(nested, list):
return nested
return []
def _layer_record_count(value: Any, layer: str = "") -> int:
if isinstance(value, list):
return len(value)
if isinstance(value, dict):
total = value.get("total")
if isinstance(total, (int, float)):
return int(total)
items = _unwrap_layer_items(value, layer)
if items:
return len(items)
return len(value) if value else 0
if value is None:
return 0
return 1
def _build_search_document(doc_id: int, layer: str, candidate: dict[str, Any], spec: dict[str, Any]) -> dict[str, Any]:
fields = tuple(spec.get("fields", ()))
text = _document_text(candidate, fields)
@@ -880,9 +1024,7 @@ def _get_search_index() -> dict[str, Any]:
for layer in layers:
spec = _UNIVERSAL_SEARCH_SPECS[layer]
items = snap.get(layer) or []
if isinstance(items, dict):
items = items.get("items", []) or items.get("results", []) or items.get("vessels", [])
items = _unwrap_layer_items(snap.get(layer), layer)
if not isinstance(items, list):
continue
for item in items:
@@ -1144,18 +1286,9 @@ def get_telemetry_summary() -> dict[str, Any]:
for layer in layer_names:
value = snap.get(layer)
if isinstance(value, list):
counts[layer] = len(value)
if value:
non_empty_layers.append(layer)
elif isinstance(value, dict):
counts[layer] = len(value)
if value:
non_empty_layers.append(layer)
elif value is None:
counts[layer] = 0
else:
counts[layer] = 1
count = _layer_record_count(value, layer)
counts[layer] = count
if count > 0:
non_empty_layers.append(layer)
alias_examples = {
@@ -1167,6 +1300,16 @@ def get_telemetry_summary() -> dict[str, Any]:
"tracked": "tracked_flights",
"military": "military_flights",
"jets": "private_jets",
"telegram": "telegram_osint",
"osint_feed": "telegram_osint",
"malware": "malware_threats",
"malware_c2": "malware_threats",
"botnet": "malware_threats",
"cyber": "cyber_threats",
"cisa": "cyber_threats",
"kev": "cyber_threats",
"scm": "scm_suppliers",
"suppliers": "scm_suppliers",
}
return {
@@ -1577,14 +1720,7 @@ def _nearby_items_from_layers(
snap = get_latest_data_subset_refs(*layers)
out: dict[str, list[dict[str, Any]]] = {}
for layer in layers:
value = snap.get(layer) or []
if isinstance(value, dict):
if layer == "gdelt" and isinstance(value.get("features"), list):
items = value.get("features") or []
else:
items = value.get("items") or value.get("features") or value.get("vessels") or []
else:
items = value
items = _unwrap_layer_items(snap.get(layer), layer)
if not isinstance(items, list):
continue
matches: list[dict[str, Any]] = []
@@ -1728,6 +1864,9 @@ def correlate_entity(
"crowdthreat",
"frontlines",
"liveuamap",
"telegram_osint",
"malware_threats",
"scm_suppliers",
"military_bases",
"datacenters",
"power_plants",
@@ -1809,13 +1948,17 @@ def search_news(
query: str,
limit: int = 10,
include_gdelt: bool = True,
include_telegram: bool = True,
) -> dict[str, Any]:
"""Search news and event layers server-side and return a compact result set."""
query_norm = _norm_text(query)
if not query_norm:
return {"results": [], "version": get_data_version(), "truncated": False}
snap = get_latest_data_subset_refs("news", "gdelt", "crowdthreat", "liveuamap", "frontlines")
layer_keys = ["news", "gdelt", "crowdthreat", "liveuamap", "frontlines"]
if include_telegram:
layer_keys.append("telegram_osint")
snap = get_latest_data_subset_refs(*layer_keys)
out: list[dict[str, Any]] = []
limit = _coerce_limit(limit, default=10, maximum=50)
@@ -1941,6 +2084,36 @@ def search_news(
if len(out) >= limit:
return {"results": out, "version": get_data_version(), "truncated": True}
if include_telegram:
for post in _unwrap_layer_items(snap.get("telegram_osint"), "telegram_osint"):
if not isinstance(post, dict):
continue
text = " ".join(
(
_norm_text(post.get("title")),
_norm_text(post.get("description")),
_norm_text(post.get("source")),
_norm_text(post.get("channel")),
)
)
if not _text_matches_query(query_norm, text):
continue
lat, lng = _extract_coords(post)
out.append(
{
"source_layer": "telegram_osint",
"title": post.get("title") or "",
"summary": post.get("description") or "",
"source": post.get("source") or post.get("channel") or "Telegram",
"link": post.get("link") or "",
"lat": lat,
"lng": lng,
"risk_score": post.get("risk_score"),
}
)
if len(out) >= limit:
return {"results": out, "version": get_data_version(), "truncated": True}
return {"results": out, "version": get_data_version(), "truncated": False}
@@ -2205,16 +2378,13 @@ def entities_near(
out: list[dict[str, Any]] = []
for layer in layers:
items = snap.get(layer) or []
if isinstance(items, dict):
items = items.get("vessels", []) or items.get("items", [])
items = _unwrap_layer_items(snap.get(layer), layer)
if not isinstance(items, list):
continue
for item in items:
if not isinstance(item, dict):
continue
item_lat = _coerce_float(item.get("lat") or item.get("latitude"))
item_lng = _coerce_float(item.get("lng") or item.get("lon") or item.get("longitude"))
item_lat, item_lng = _extract_coords(item)
if item_lat is None or item_lng is None:
continue
distance = _haversine_km(center_lat, center_lng, item_lat, item_lng)