mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-07-07 04:47:58 +02:00
feat: Telegram OSINT map layer, Osiris intel ports, and maritime settings
Add Telegram OSINT with hourly incremental t.me scraping, metro geocoding separate from news centroids, threat-intercept popup UI with inline media, and HTML markers above alert boxes so pins stay clickable. Expose GFW_API_TOKEN in onboarding and Settings Maritime; harden GFW/CCTV/geo fetchers. Port Osiris- derived recon, SCM, entity graph, malware/cyber feeds, sanctions, and submarine cable layers with tests and documentation. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
BigBodyCobain
@@ -70,6 +70,10 @@ class DashboardData(TypedDict, total=False):
|
||||
sar_anomalies: List[Dict[str, Any]]
|
||||
sar_aoi_coverage: List[Dict[str, Any]]
|
||||
road_corridor_trends: Dict[str, Any]
|
||||
malware_threats: Dict[str, Any]
|
||||
cyber_threats: Dict[str, Any]
|
||||
scm_suppliers: Dict[str, Any]
|
||||
telegram_osint: Dict[str, Any]
|
||||
|
||||
|
||||
# In-memory store
|
||||
@@ -121,6 +125,10 @@ latest_data: DashboardData = {
|
||||
"sar_anomalies": [],
|
||||
"sar_aoi_coverage": [],
|
||||
"road_corridor_trends": {"updated_at": None, "corridors": []},
|
||||
"malware_threats": {"threats": [], "total": 0, "timestamp": None},
|
||||
"cyber_threats": {"threats": [], "stats": {}},
|
||||
"scm_suppliers": {"suppliers": [], "total": 0, "critical_count": 0},
|
||||
"telegram_osint": {"posts": [], "total": 0, "geolocated": 0, "timestamp": None},
|
||||
}
|
||||
|
||||
# Per-source freshness timestamps
|
||||
@@ -331,6 +339,11 @@ active_layers: dict[str, bool] = {
|
||||
"crowdthreat": False,
|
||||
"sar": True,
|
||||
"road_corridor_trends": False,
|
||||
"malware_c2": False,
|
||||
"submarine_cables": False,
|
||||
"scm_suppliers": False,
|
||||
"cyber_threats": False,
|
||||
"telegram_osint": True,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""CISA KEV + cyber threat stats (Osiris port)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def fetch_cyber_threats() -> dict[str, Any]:
|
||||
if not is_any_active("cyber_threats"):
|
||||
return latest_data.get("cyber_threats") or {"threats": [], "stats": {}}
|
||||
|
||||
results: dict[str, Any] = {"threats": [], "stats": {}, "timestamp": datetime.now(timezone.utc).isoformat()}
|
||||
try:
|
||||
resp = fetch_with_curl(
|
||||
"https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json",
|
||||
timeout=15,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
vulns = data.get("vulnerabilities") or []
|
||||
results["stats"]["cisa_total"] = len(vulns)
|
||||
now = datetime.now(timezone.utc)
|
||||
recent = []
|
||||
for v in vulns:
|
||||
try:
|
||||
added = datetime.fromisoformat(v.get("dateAdded", "").replace("Z", "+00:00"))
|
||||
days = (now - added).total_seconds() / 86400
|
||||
except Exception:
|
||||
continue
|
||||
if days <= 30:
|
||||
recent.append(v)
|
||||
recent = recent[:10]
|
||||
results["threats"] = [
|
||||
{
|
||||
"id": v.get("cveID"),
|
||||
"name": v.get("vulnerabilityName"),
|
||||
"vendor": v.get("vendorProject"),
|
||||
"product": v.get("product"),
|
||||
"severity": "CRITICAL",
|
||||
"date": v.get("dateAdded"),
|
||||
"due": v.get("dueDate"),
|
||||
"source": "CISA KEV",
|
||||
}
|
||||
for v in recent
|
||||
]
|
||||
except Exception as exc:
|
||||
logger.warning("CISA KEV fetch failed: %s", exc)
|
||||
|
||||
count = len(results["threats"])
|
||||
results["stats"]["active_cves"] = count
|
||||
results["stats"]["threat_level"] = "CRITICAL" if count >= 8 else "HIGH" if count >= 4 else "ELEVATED"
|
||||
|
||||
with _data_lock:
|
||||
latest_data["cyber_threats"] = results
|
||||
_mark_fresh("cyber_threats")
|
||||
return results
|
||||
@@ -278,6 +278,16 @@ _FISHING_FETCH_INTERVAL_S = 3600 # once per hour — GFW data has ~5 day lag
|
||||
_last_fishing_fetch_ts: float = 0.0
|
||||
|
||||
|
||||
def _gfw_int_env(name: str, default: int, *, minimum: int = 1, maximum: int | None = None) -> int:
|
||||
try:
|
||||
value = int(os.environ.get(name, str(default)) or default)
|
||||
except (TypeError, ValueError):
|
||||
value = default
|
||||
if maximum is not None:
|
||||
value = min(maximum, value)
|
||||
return max(minimum, value)
|
||||
|
||||
|
||||
@with_retry(max_retries=1, base_delay=5)
|
||||
def fetch_fishing_activity():
|
||||
"""Fetch recent fishing events from Global Fishing Watch (~5 day lag)."""
|
||||
@@ -300,10 +310,16 @@ def fetch_fishing_activity():
|
||||
try:
|
||||
import datetime as _dt
|
||||
|
||||
# GFW publishes with ~5 day lag; windows shorter than ~7 days often return 0 events.
|
||||
lookback_days = _gfw_int_env("GFW_EVENTS_LOOKBACK_DAYS", 7, minimum=1, maximum=14)
|
||||
max_pages = _gfw_int_env("GFW_EVENTS_MAX_PAGES", 10, minimum=1, maximum=100)
|
||||
timeout_s = _gfw_int_env("GFW_EVENTS_TIMEOUT_S", 90, minimum=30, maximum=180)
|
||||
_end = _dt.date.today().isoformat()
|
||||
_start = (_dt.date.today() - _dt.timedelta(days=7)).isoformat()
|
||||
page_size = max(1, int(os.environ.get("GFW_EVENTS_PAGE_SIZE", "500") or "500"))
|
||||
_start = (_dt.date.today() - _dt.timedelta(days=lookback_days)).isoformat()
|
||||
page_size = _gfw_int_env("GFW_EVENTS_PAGE_SIZE", 500, minimum=1, maximum=1000)
|
||||
offset = 0
|
||||
pages_fetched = 0
|
||||
total_available: int | None = None
|
||||
seen_offsets: set[int] = set()
|
||||
seen_ids: set[str] = set()
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
@@ -324,7 +340,7 @@ def fetch_fishing_activity():
|
||||
}
|
||||
)
|
||||
url = f"https://gateway.api.globalfishingwatch.org/v3/events?{query}"
|
||||
response = fetch_with_curl(url, timeout=30, headers=headers)
|
||||
response = fetch_with_curl(url, timeout=timeout_s, headers=headers)
|
||||
if response.status_code != 200:
|
||||
logger.warning(
|
||||
"Fishing activity fetch failed at offset=%s: HTTP %s",
|
||||
@@ -334,10 +350,16 @@ def fetch_fishing_activity():
|
||||
break
|
||||
|
||||
payload = response.json() or {}
|
||||
if total_available is None:
|
||||
try:
|
||||
total_available = int(payload.get("total")) if payload.get("total") is not None else None
|
||||
except (TypeError, ValueError):
|
||||
total_available = None
|
||||
entries = payload.get("entries", [])
|
||||
if not entries:
|
||||
break
|
||||
|
||||
pages_fetched += 1
|
||||
added_this_page = 0
|
||||
for e in entries:
|
||||
pos = e.get("position", {})
|
||||
@@ -372,6 +394,15 @@ def fetch_fishing_activity():
|
||||
if len(entries) < page_size:
|
||||
break
|
||||
|
||||
if pages_fetched >= max_pages:
|
||||
logger.info(
|
||||
"Fishing activity: capped at %s pages (%s events fetched; GFW total=%s)",
|
||||
max_pages,
|
||||
len(events),
|
||||
total_available if total_available is not None else "unknown",
|
||||
)
|
||||
break
|
||||
|
||||
next_offset = payload.get("nextOffset")
|
||||
if next_offset is None:
|
||||
next_offset = (payload.get("pagination") or {}).get("nextOffset")
|
||||
|
||||
@@ -235,11 +235,11 @@ _DC_GEOCODED_PATH = Path(__file__).parent.parent.parent / "data" / "datacenters_
|
||||
|
||||
|
||||
def fetch_datacenters():
|
||||
"""Load geocoded data centers (5K+ street-level precise locations)."""
|
||||
from services.fetchers._store import is_any_active
|
||||
"""Load geocoded data centers (5K+ street-level precise locations).
|
||||
|
||||
if not is_any_active("datacenters"):
|
||||
return
|
||||
Always loads from disk; /api/live-data/slow gates the payload on the
|
||||
datacenters layer toggle so enabling the layer can render immediately.
|
||||
"""
|
||||
dcs = []
|
||||
try:
|
||||
if not _DC_GEOCODED_PATH.exists():
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Malware C2 / URLhaus feed (abuse.ch, Osiris port)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
COUNTRY_CENTROIDS: dict[str, tuple[float, float]] = {
|
||||
"AF": (65, 33), "AL": (20, 41), "DZ": (3, 28), "AR": (-64, -34), "AU": (134, -25),
|
||||
"AT": (14, 47.5), "BE": (4, 50.8), "BR": (-51, -10), "CA": (-96, 62), "CN": (105, 35),
|
||||
"DE": (10, 51), "FR": (2, 46), "GB": (-2, 54), "IN": (79, 22), "IR": (53, 32),
|
||||
"IT": (12.5, 42.8), "JP": (138, 36), "KR": (128, 36), "MX": (-102, 23.5), "NL": (5.5, 52.5),
|
||||
"PL": (19.5, 52), "RU": (100, 60), "SG": (103.8, 1.35), "TW": (121, 23.7), "UA": (32, 49),
|
||||
"US": (-97, 38), "VN": (106, 16),
|
||||
}
|
||||
|
||||
|
||||
def fetch_malware_threats() -> list[dict[str, Any]]:
|
||||
if not is_any_active("malware_c2"):
|
||||
return latest_data.get("malware_threats") or []
|
||||
|
||||
threats: list[dict[str, Any]] = []
|
||||
threat_id = 0
|
||||
|
||||
try:
|
||||
resp = fetch_with_curl(
|
||||
"https://feodotracker.abuse.ch/downloads/ipblocklist.json",
|
||||
timeout=10,
|
||||
headers={"User-Agent": "Shadowbroker/1.0", "Accept": "application/json"},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
entries = resp.json()
|
||||
if not isinstance(entries, list):
|
||||
entries = []
|
||||
for entry in entries[:200]:
|
||||
cc = entry.get("country")
|
||||
if not cc or cc not in COUNTRY_CENTROIDS:
|
||||
continue
|
||||
lng, lat = COUNTRY_CENTROIDS[cc]
|
||||
j_lng = ((threat_id * 173.7) % 200 - 100) / 100 * 4
|
||||
j_lat = ((threat_id * 293.1) % 200 - 100) / 100 * 4
|
||||
threats.append(
|
||||
{
|
||||
"id": f"feodo-{threat_id}",
|
||||
"lat": lat + j_lat,
|
||||
"lng": lng + j_lng,
|
||||
"ip": entry.get("ip_address") or "unknown",
|
||||
"port": entry.get("dst_port") or 0,
|
||||
"malware": entry.get("malware") or "unknown",
|
||||
"status": entry.get("status") or "active",
|
||||
"first_seen": entry.get("first_seen"),
|
||||
"last_online": entry.get("last_online"),
|
||||
"country": cc,
|
||||
"threat_type": "botnet_c2",
|
||||
}
|
||||
)
|
||||
threat_id += 1
|
||||
except Exception as exc:
|
||||
logger.warning("Feodo fetch failed: %s", exc)
|
||||
|
||||
try:
|
||||
resp = fetch_with_curl(
|
||||
"https://urlhaus-api.abuse.ch/v1/urls/recent/limit/100/",
|
||||
timeout=8,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
urls = (resp.json() or {}).get("urls") or []
|
||||
for u in urls:
|
||||
cc = u.get("country")
|
||||
if not cc or cc not in COUNTRY_CENTROIDS:
|
||||
cc = next(iter(COUNTRY_CENTROIDS))
|
||||
lng, lat = COUNTRY_CENTROIDS[cc]
|
||||
j_lng = ((threat_id * 137.3) % 200 - 100) / 100 * 5
|
||||
j_lat = ((threat_id * 211.7) % 200 - 100) / 100 * 5
|
||||
threats.append(
|
||||
{
|
||||
"id": f"urlhaus-{threat_id}",
|
||||
"lat": lat + j_lat,
|
||||
"lng": lng + j_lng,
|
||||
"ip": u.get("host") or "unknown",
|
||||
"port": 0,
|
||||
"malware": ", ".join(u.get("tags") or []) or u.get("threat") or "malware",
|
||||
"status": u.get("url_status") or "online",
|
||||
"first_seen": u.get("dateadded"),
|
||||
"country": cc,
|
||||
"threat_type": "malware_url",
|
||||
}
|
||||
)
|
||||
threat_id += 1
|
||||
except Exception as exc:
|
||||
logger.debug("URLhaus supplement failed: %s", exc)
|
||||
|
||||
payload = {
|
||||
"threats": threats,
|
||||
"total": len(threats),
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"source": "abuse.ch Feodo Tracker + URLhaus",
|
||||
}
|
||||
with _data_lock:
|
||||
latest_data["malware_threats"] = payload
|
||||
_mark_fresh("malware_threats")
|
||||
return threats
|
||||
@@ -158,21 +158,26 @@ _KEYWORD_COORDS = {
|
||||
_SORTED_KEYWORDS = sorted(_KEYWORD_COORDS.items(), key=lambda x: len(x[0]), reverse=True)
|
||||
|
||||
|
||||
def resolve_coords_match(text: str) -> tuple[tuple[float, float], str] | None:
|
||||
"""Return ((lat, lng), matched_keyword) for the most specific keyword hit."""
|
||||
padded_text = f" {text} "
|
||||
for kw, coords in _SORTED_KEYWORDS:
|
||||
if kw.startswith(" ") or kw.endswith(" "):
|
||||
if kw in padded_text:
|
||||
return coords, kw
|
||||
elif re.search(r"\b" + re.escape(kw) + r"\b", text):
|
||||
return coords, kw
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_coords(text: str) -> tuple[float, float] | None:
|
||||
"""Return (lat, lng) for the most specific keyword match, or None.
|
||||
|
||||
Longer keywords are tried first. Space-padded keywords (" us ", " uk ")
|
||||
use substring matching on padded text; all others use word-boundary regex.
|
||||
"""
|
||||
padded_text = f" {text} "
|
||||
for kw, coords in _SORTED_KEYWORDS:
|
||||
if kw.startswith(" ") or kw.endswith(" "):
|
||||
if kw in padded_text:
|
||||
return coords
|
||||
else:
|
||||
if re.search(r'\b' + re.escape(kw) + r'\b', text):
|
||||
return coords
|
||||
return None
|
||||
match = resolve_coords_match(text)
|
||||
return match[0] if match else None
|
||||
|
||||
|
||||
@with_retry(max_retries=1, base_delay=2)
|
||||
|
||||
@@ -0,0 +1,381 @@
|
||||
"""Telegram OSINT — public channel web previews (t.me/s) with keyword geoparsing."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
|
||||
from services.fetchers.news import resolve_coords_match
|
||||
from services.network_utils import fetch_with_curl, outbound_user_agent
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_CHANNELS = (
|
||||
"osintdefender",
|
||||
"insiderpaper",
|
||||
"aljazeeraenglish",
|
||||
"nexta_live",
|
||||
"war_monitor",
|
||||
"OSINTtechnical",
|
||||
"Liveuamap",
|
||||
)
|
||||
|
||||
_MESSAGE_BLOCK_RE = re.compile(
|
||||
r'<div class="tgme_widget_message_wrap js-widget_message_wrap"[\s\S]*?</div>\s*</div>\s*</div>',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_TEXT_RE = re.compile(
|
||||
r'<div class="tgme_widget_message_text[^>]*>([\s\S]*?)</div>',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_DATE_RE = re.compile(
|
||||
r'<a class="tgme_widget_message_date" href="(https://t\.me/[^"]+)".*?<time datetime="([^"]+)"',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_HAS_VIDEO_RE = re.compile(
|
||||
r'tgme_widget_message_video|js-message_video|<video\s',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_HAS_PHOTO_RE = re.compile(r'tgme_widget_message_photo_wrap', re.IGNORECASE)
|
||||
_VIDEO_SRC_RE = re.compile(r'<video[^>]+src="([^"]+)"', re.IGNORECASE)
|
||||
_BG_IMAGE_RE = re.compile(r"background-image:url\('([^']+)'\)", re.IGNORECASE)
|
||||
|
||||
_TELEGRAM_MEDIA_HOST_SUFFIXES = (".telesco.pe", ".telegram-cdn.org")
|
||||
|
||||
# Cyrillic / Arabic aliases for war-reporting channels (merged after English resolver).
|
||||
_EXTRA_PLACE_KEYWORDS: dict[str, tuple[float, float]] = {
|
||||
"киев": (50.450, 30.523),
|
||||
"київ": (50.450, 30.523),
|
||||
"харьков": (49.993, 36.231),
|
||||
"харків": (49.993, 36.231),
|
||||
"одесса": (46.482, 30.724),
|
||||
"одеса": (46.482, 30.724),
|
||||
"донецк": (48.015, 37.803),
|
||||
"донецьк": (48.015, 37.803),
|
||||
"луганск": (48.574, 39.307),
|
||||
"луганськ": (48.574, 39.307),
|
||||
"москва": (55.755, 37.617),
|
||||
"крым": (45.000, 34.000),
|
||||
"крим": (45.000, 34.000),
|
||||
"бахмут": (48.595, 38.000),
|
||||
"запорожье": (47.838, 35.139),
|
||||
"запоріжжя": (47.838, 35.139),
|
||||
"غزة": (31.416, 34.333),
|
||||
"دمشق": (33.513, 36.276),
|
||||
"بيروت": (33.893, 35.501),
|
||||
"tel aviv": (32.085, 34.781),
|
||||
"תל אביב": (32.085, 34.781),
|
||||
}
|
||||
|
||||
# Country-level news geocodes sit on national centroids that stack with threat alerts.
|
||||
# Telegram uses major metro anchors so pins land on a different map cell than news.
|
||||
_TELEGRAM_ANCHOR_OVERRIDES: dict[str, tuple[float, float]] = {
|
||||
"israel": (32.085, 34.781), # Tel Aviv (news uses central Israel ~Jerusalem corridor)
|
||||
"middle east": (32.085, 34.781),
|
||||
"china": (39.904, 116.407), # Beijing (news uses country centroid)
|
||||
"united states": (40.712, -74.006), # New York (news uses Washington DC)
|
||||
"usa": (40.712, -74.006),
|
||||
"us": (40.712, -74.006),
|
||||
"america": (40.712, -74.006),
|
||||
"uk": (51.507, -0.127), # London
|
||||
"iran": (35.689, 51.389), # Tehran
|
||||
"russia": (55.755, 37.617), # Moscow
|
||||
"ukraine": (50.450, 30.523), # Kyiv
|
||||
"france": (48.856, 2.352), # Paris
|
||||
"germany": (52.520, 13.405), # Berlin
|
||||
"lebanon": (34.433, 35.844), # Tripoli (news uses Beirut corridor)
|
||||
}
|
||||
|
||||
_RISK_KEYWORDS = (
|
||||
"war",
|
||||
"missile",
|
||||
"strike",
|
||||
"attack",
|
||||
"crisis",
|
||||
"tension",
|
||||
"military",
|
||||
"conflict",
|
||||
"defense",
|
||||
"clash",
|
||||
"nuclear",
|
||||
"invasion",
|
||||
"bomb",
|
||||
"drone",
|
||||
"weapon",
|
||||
"sanctions",
|
||||
"ceasefire",
|
||||
"escalation",
|
||||
"killed",
|
||||
"destroyed",
|
||||
"operation",
|
||||
"casualty",
|
||||
"frontline",
|
||||
"threat",
|
||||
"explosion",
|
||||
"shelling",
|
||||
)
|
||||
|
||||
|
||||
def telegram_osint_enabled() -> bool:
|
||||
return str(os.environ.get("TELEGRAM_OSINT_ENABLED", "true")).strip().lower() not in {
|
||||
"0",
|
||||
"false",
|
||||
"no",
|
||||
"off",
|
||||
"",
|
||||
}
|
||||
|
||||
|
||||
def _configured_channels() -> list[str]:
|
||||
raw = str(os.environ.get("TELEGRAM_OSINT_CHANNELS", "")).strip()
|
||||
if raw:
|
||||
return [part.strip().lstrip("@") for part in raw.split(",") if part.strip()]
|
||||
return list(_DEFAULT_CHANNELS)
|
||||
|
||||
|
||||
def telegram_media_host_allowed(hostname: str | None) -> bool:
|
||||
host = str(hostname or "").strip().lower()
|
||||
if not host:
|
||||
return False
|
||||
return any(host.endswith(suffix) for suffix in _TELEGRAM_MEDIA_HOST_SUFFIXES)
|
||||
|
||||
|
||||
def _extract_media(block: str, link: str) -> dict[str, Any]:
|
||||
has_video = bool(_HAS_VIDEO_RE.search(block))
|
||||
has_photo = bool(_HAS_PHOTO_RE.search(block))
|
||||
media_type: str | None = None
|
||||
media_url: str | None = None
|
||||
if has_video:
|
||||
media_type = "video"
|
||||
video_match = _VIDEO_SRC_RE.search(block)
|
||||
if video_match:
|
||||
media_url = video_match.group(1).strip()
|
||||
elif has_photo:
|
||||
media_type = "photo"
|
||||
photo_match = _BG_IMAGE_RE.search(block)
|
||||
if photo_match:
|
||||
media_url = photo_match.group(1).strip()
|
||||
|
||||
embed_url: str | None = None
|
||||
if media_type and link:
|
||||
embed_url = f"{link}?embed=1"
|
||||
|
||||
return {
|
||||
"media_type": media_type,
|
||||
"media_url": media_url,
|
||||
"embed_url": embed_url,
|
||||
}
|
||||
|
||||
|
||||
def _strip_html(text: str) -> str:
|
||||
cleaned = re.sub(r"<br\s*/?>", "\n", text, flags=re.IGNORECASE)
|
||||
cleaned = re.sub(r"<[^>]+>", "", cleaned)
|
||||
return (
|
||||
cleaned.replace(""", '"')
|
||||
.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.strip()
|
||||
)
|
||||
|
||||
|
||||
def _score_risk(text: str) -> int:
|
||||
lower = text.lower()
|
||||
score = 1
|
||||
for kw in _RISK_KEYWORDS:
|
||||
if kw in lower:
|
||||
score += 2
|
||||
return min(10, score)
|
||||
|
||||
|
||||
def _refresh_post_coords(post: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Re-apply geoparsing so stored posts pick up anchor updates."""
|
||||
text = "\n".join(
|
||||
str(part).strip()
|
||||
for part in (post.get("title"), post.get("description"))
|
||||
if part and str(part).strip()
|
||||
)
|
||||
if not text:
|
||||
return post
|
||||
coords = _resolve_telegram_coords(text)
|
||||
if not coords:
|
||||
return post
|
||||
updated = dict(post)
|
||||
updated["coords"] = [coords[0], coords[1]]
|
||||
return updated
|
||||
|
||||
|
||||
def _resolve_telegram_coords(text: str) -> tuple[float, float] | None:
|
||||
lower = text.lower()
|
||||
match = resolve_coords_match(lower)
|
||||
if match:
|
||||
_coords, keyword = match
|
||||
anchor = _TELEGRAM_ANCHOR_OVERRIDES.get(keyword.strip().lower())
|
||||
if anchor:
|
||||
return anchor
|
||||
return _coords
|
||||
for keyword, coords in sorted(_EXTRA_PLACE_KEYWORDS.items(), key=lambda x: len(x[0]), reverse=True):
|
||||
if keyword in lower:
|
||||
return coords
|
||||
return None
|
||||
|
||||
|
||||
def _post_link(post: dict[str, Any]) -> str:
|
||||
return str(post.get("link") or "").strip()
|
||||
|
||||
|
||||
def _extract_new_channel_posts(
|
||||
html: str,
|
||||
channel: str,
|
||||
known_links: set[str],
|
||||
*,
|
||||
bootstrap_limit: int = 12,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return unseen posts from a channel page; stop once we hit a stored link."""
|
||||
parsed = parse_telegram_channel_html(html, channel)
|
||||
if not parsed:
|
||||
return []
|
||||
if not known_links:
|
||||
return parsed[-bootstrap_limit:]
|
||||
|
||||
fresh: list[dict[str, Any]] = []
|
||||
for post in reversed(parsed):
|
||||
link = _post_link(post)
|
||||
if not link:
|
||||
continue
|
||||
if link in known_links:
|
||||
break
|
||||
fresh.append(post)
|
||||
fresh.reverse()
|
||||
return fresh
|
||||
|
||||
|
||||
def _merge_telegram_posts(
|
||||
existing: list[dict[str, Any]],
|
||||
incoming: list[dict[str, Any]],
|
||||
*,
|
||||
max_posts: int = 120,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
known_links = {_post_link(post) for post in existing if _post_link(post)}
|
||||
added = 0
|
||||
for post in incoming:
|
||||
link = _post_link(post)
|
||||
if not link or link in known_links:
|
||||
continue
|
||||
known_links.add(link)
|
||||
existing.append(post)
|
||||
added += 1
|
||||
existing.sort(key=lambda p: str(p.get("published") or ""), reverse=True)
|
||||
return existing[:max_posts], added
|
||||
|
||||
|
||||
def parse_telegram_channel_html(html: str, channel: str) -> list[dict[str, Any]]:
|
||||
"""Parse public t.me/s channel preview HTML into post dicts."""
|
||||
posts: list[dict[str, Any]] = []
|
||||
for block in _MESSAGE_BLOCK_RE.findall(html or ""):
|
||||
text_match = _TEXT_RE.search(block)
|
||||
if not text_match:
|
||||
continue
|
||||
text = _strip_html(text_match.group(1))
|
||||
if len(text) < 10:
|
||||
continue
|
||||
|
||||
date_match = _DATE_RE.search(block)
|
||||
link = date_match.group(1) if date_match else f"https://t.me/{channel}"
|
||||
published = date_match.group(2) if date_match else datetime.now(timezone.utc).isoformat()
|
||||
title = text.split("\n", 1)[0][:160]
|
||||
risk_score = _score_risk(text)
|
||||
coords = _resolve_telegram_coords(text)
|
||||
post_id = hashlib.sha1(f"{link}|{published}".encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
media = _extract_media(block, link)
|
||||
posts.append(
|
||||
{
|
||||
"id": post_id,
|
||||
"title": title,
|
||||
"description": text[:1200],
|
||||
"link": link,
|
||||
"published": published,
|
||||
"source": f"t.me/{channel}",
|
||||
"channel": channel,
|
||||
"risk_score": risk_score,
|
||||
"coords": [coords[0], coords[1]] if coords else None,
|
||||
**media,
|
||||
}
|
||||
)
|
||||
return posts
|
||||
|
||||
|
||||
def fetch_telegram_osint() -> dict[str, Any]:
|
||||
if not is_any_active("telegram_osint"):
|
||||
return latest_data.get("telegram_osint") or {"posts": [], "total": 0, "timestamp": None}
|
||||
|
||||
if not telegram_osint_enabled():
|
||||
with _data_lock:
|
||||
latest_data["telegram_osint"] = {"posts": [], "total": 0, "timestamp": None, "disabled": True}
|
||||
_mark_fresh("telegram_osint")
|
||||
return latest_data["telegram_osint"]
|
||||
|
||||
headers = {
|
||||
"User-Agent": (
|
||||
f"Mozilla/5.0 (compatible; {outbound_user_agent('telegram-osint')}) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "text/html,application/xhtml+xml",
|
||||
}
|
||||
|
||||
with _data_lock:
|
||||
prior = latest_data.get("telegram_osint") or {}
|
||||
existing_posts = list(prior.get("posts") or [])
|
||||
|
||||
known_links = {_post_link(post) for post in existing_posts if _post_link(post)}
|
||||
incoming: list[dict[str, Any]] = []
|
||||
|
||||
for channel in _configured_channels():
|
||||
url = f"https://t.me/s/{channel}"
|
||||
try:
|
||||
resp = fetch_with_curl(url, timeout=15, headers=headers)
|
||||
if not resp or resp.status_code != 200:
|
||||
logger.warning(
|
||||
"Telegram channel %s fetch failed: HTTP %s",
|
||||
channel,
|
||||
resp.status_code if resp else "no response",
|
||||
)
|
||||
continue
|
||||
channel_new = _extract_new_channel_posts(resp.text, channel, known_links)
|
||||
for post in channel_new:
|
||||
link = _post_link(post)
|
||||
if not link or link in known_links:
|
||||
continue
|
||||
known_links.add(link)
|
||||
incoming.append(post)
|
||||
except Exception as exc:
|
||||
logger.warning("Telegram channel %s parse failed: %s", channel, exc)
|
||||
|
||||
merged_posts, added = _merge_telegram_posts(existing_posts, incoming)
|
||||
merged_posts = [_refresh_post_coords(post) for post in merged_posts]
|
||||
geolocated = sum(1 for p in merged_posts if p.get("coords"))
|
||||
|
||||
payload = {
|
||||
"posts": merged_posts,
|
||||
"total": len(merged_posts),
|
||||
"geolocated": geolocated,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"channels": _configured_channels(),
|
||||
"last_fetch_new": added,
|
||||
}
|
||||
|
||||
with _data_lock:
|
||||
latest_data["telegram_osint"] = payload
|
||||
_mark_fresh("telegram_osint")
|
||||
logger.info(
|
||||
"Telegram OSINT: +%s new, %s retained (%s geolocated)",
|
||||
added,
|
||||
len(merged_posts),
|
||||
geolocated,
|
||||
)
|
||||
return payload
|
||||
Reference in New Issue
Block a user