"""CrowdThreat fetcher — crowdsourced global threat intelligence. Polls verified threat reports from CrowdThreat's public API and normalises them into map-ready records with category-based icon IDs. No API key required — the /threats endpoint is unauthenticated. """ import logging import os from services.network_utils import fetch_with_curl from services.fetchers._store import latest_data, _data_lock, _mark_fresh, is_any_active from services.fetchers.retry import with_retry logger = logging.getLogger("services.data_fetcher") _CT_BASE = "https://backend.crowdthreat.world" def crowdthreat_fetch_enabled() -> bool: """Return True only when the operator explicitly opts into CrowdThreat pulls.""" return str(os.environ.get("CROWDTHREAT_ENABLED", "")).strip().lower() in { "1", "true", "yes", "on", } # CrowdThreat category_id → icon ID used on the MapLibre layer _CATEGORY_ICON = { 1: "ct-security", # Security & Conflict (red) 2: "ct-crime", # Crime & Safety (blue) 3: "ct-aviation", # Aviation (green) 4: "ct-maritime", # Maritime (teal) 5: "ct-infrastructure", # Industrial & Infra (orange) 6: "ct-special", # Special Threats (purple) 7: "ct-social", # Social & Political (pink) 8: "ct-other", # Other (gray) } _CATEGORY_COLOUR = { 1: "#ef4444", # red 2: "#3b82f6", # blue 3: "#22c55e", # green 4: "#14b8a6", # teal 5: "#f97316", # orange 6: "#a855f7", # purple 7: "#ec4899", # pink 8: "#6b7280", # gray } @with_retry(max_retries=2, base_delay=5) def fetch_crowdthreat(): """Fetch verified threat reports from CrowdThreat public API.""" if not crowdthreat_fetch_enabled(): logger.debug("CrowdThreat fetch skipped; set CROWDTHREAT_ENABLED=true to opt in") with _data_lock: latest_data["crowdthreat"] = [] _mark_fresh("crowdthreat") return if not is_any_active("crowdthreat"): return try: resp = fetch_with_curl(f"{_CT_BASE}/threats", timeout=20) if not resp or resp.status_code != 200: logger.warning("CrowdThreat API returned %s", getattr(resp, "status_code", "None")) return payload = resp.json() raw_threats = payload.get("data", {}).get("threats", []) if not raw_threats: logger.debug("CrowdThreat returned 0 threats") return except Exception as e: logger.error("CrowdThreat fetch error: %s", e) return processed = [] for t in raw_threats: loc = t.get("location") or {} lng_lat = loc.get("lng_lat") if not lng_lat or len(lng_lat) < 2: continue try: lng = float(lng_lat[0]) lat = float(lng_lat[1]) except (TypeError, ValueError): continue cat = t.get("category") or {} cat_id = cat.get("id", 8) subcat = t.get("subcategory") or {} threat_type = t.get("type") or {} dates = t.get("dates") or {} occurred = dates.get("occurred") or {} reported = dates.get("reported") or {} # Extract all available detail from the API response summary = (t.get("summary") or t.get("description") or "").strip() verification = (t.get("verification_status") or t.get("status") or "").strip() country_obj = loc.get("country") or {} country = country_obj.get("name", "") if isinstance(country_obj, dict) else str(country_obj or "") media = t.get("media") or t.get("images") or t.get("attachments") or [] source_url = t.get("source_url") or t.get("url") or t.get("link") or "" severity = t.get("severity") or t.get("severity_level") or t.get("risk_level") or "" votes = t.get("votes") or t.get("upvotes") or 0 reporter = t.get("user") or t.get("reporter") or {} reporter_name = reporter.get("name", "") if isinstance(reporter, dict) else "" processed.append({ "id": t.get("id"), "title": t.get("title", ""), "summary": summary[:500] if summary else "", "lat": lat, "lng": lng, "address": loc.get("name", ""), "city": loc.get("city", ""), "country": country, "category": cat.get("name", "Other"), "category_id": cat_id, "category_colour": _CATEGORY_COLOUR.get(cat_id, "#6b7280"), "subcategory": subcat.get("name", ""), "threat_type": threat_type.get("name", ""), "icon_id": _CATEGORY_ICON.get(cat_id, "ct-other"), "occurred": occurred.get("raw", ""), "occurred_iso": occurred.get("iso", ""), "timeago": occurred.get("timeago", ""), "reported": reported.get("raw", ""), "verification": verification, "severity": str(severity), "source_url": source_url, "media_urls": [m.get("url") or m for m in media[:3]] if isinstance(media, list) else [], "votes": int(votes) if votes else 0, "reporter": reporter_name, "source": "CrowdThreat", }) logger.info("CrowdThreat: fetched %d verified threats", len(processed)) with _data_lock: latest_data["crowdthreat"] = processed _mark_fresh("crowdthreat")