Shadowbroker/backend/services/fetchers/infrastructure.py

"""Infrastructure fetchers — internet outages (IODA), data centers, CCTV, KiwiSDR."""

import json
import time
import heapq
import logging
from pathlib import Path
from cachetools import TTLCache
from services.network_utils import fetch_with_curl, outbound_user_agent
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
from services.fetchers.retry import with_retry

logger = logging.getLogger(__name__)


# ---------------------------------------------------------------------------
# Internet Outages (IODA — Georgia Tech)
# ---------------------------------------------------------------------------
_region_geocode_cache: TTLCache = TTLCache(maxsize=2000, ttl=86400)


def _geocode_region(region_name: str, country_name: str) -> tuple:
    """Geocode a region using OpenStreetMap Nominatim (cached, respects rate limit)."""
    cache_key = f"{region_name}|{country_name}"
    if cache_key in _region_geocode_cache:
        return _region_geocode_cache[cache_key]
    try:
        import urllib.parse

        query = urllib.parse.quote(f"{region_name}, {country_name}")
        url = f"https://nominatim.openstreetmap.org/search?q={query}&format=json&limit=1"
        response = fetch_with_curl(url, timeout=8, headers={"User-Agent": outbound_user_agent("infrastructure-data")})
        if response.status_code == 200:
            results = response.json()
            if results:
                lat = float(results[0]["lat"])
                lon = float(results[0]["lon"])
                _region_geocode_cache[cache_key] = (lat, lon)
                return (lat, lon)
    except (ConnectionError, TimeoutError, OSError, ValueError, KeyError):
        pass
    _region_geocode_cache[cache_key] = None
    return None


@with_retry(max_retries=1, base_delay=1)
def fetch_internet_outages():
    """Fetch regional internet outage alerts from IODA (Georgia Tech)."""
    from services.fetchers._store import is_any_active

    if not is_any_active("internet_outages"):
        return
    RELIABLE_DATASOURCES = {"bgp", "ping-slash24"}
    outages = []
    try:
        now = int(time.time())
        start = now - 86400
        url = f"https://api.ioda.inetintel.cc.gatech.edu/v2/outages/alerts?from={start}&until={now}&limit=500"
        response = fetch_with_curl(url, timeout=15)
        if response.status_code == 200:
            data = response.json()
            alerts = data.get("data", [])
            region_outages = {}
            for alert in alerts:
                entity = alert.get("entity", {})
                etype = entity.get("type", "")
                level = alert.get("level", "")
                if level == "normal" or etype != "region":
                    continue
                datasource = alert.get("datasource", "")
                if datasource not in RELIABLE_DATASOURCES:
                    continue
                code = entity.get("code", "")
                name = entity.get("name", "")
                attrs = entity.get("attrs", {})
                country_code = attrs.get("country_code", "")
                country_name = attrs.get("country_name", "")
                value = alert.get("value", 0)
                history_value = alert.get("historyValue", 0)
                severity = 0
                if history_value and history_value > 0:
                    severity = round((1 - value / history_value) * 100)
                severity = max(0, min(severity, 100))
                if severity < 10:
                    continue
                if code not in region_outages or severity > region_outages[code]["severity"]:
                    region_outages[code] = {
                        "region_code": code,
                        "region_name": name,
                        "country_code": country_code,
                        "country_name": country_name,
                        "level": level,
                        "datasource": datasource,
                        "severity": severity,
                    }
            geocoded = []
            for rcode, r in region_outages.items():
                coords = _geocode_region(r["region_name"], r["country_name"])
                if coords:
                    r["lat"] = coords[0]
                    r["lng"] = coords[1]
                    geocoded.append(r)
            outages = heapq.nlargest(100, geocoded, key=lambda x: x["severity"])
        logger.info(f"Internet outages: {len(outages)} regions affected")
    except (
        ConnectionError,
        TimeoutError,
        OSError,
        ValueError,
        KeyError,
        TypeError,
        json.JSONDecodeError,
    ) as e:
        logger.error(f"Error fetching internet outages: {e}")
    with _data_lock:
        latest_data["internet_outages"] = outages
    if outages:
        _mark_fresh("internet_outages")


# ---------------------------------------------------------------------------
# RIPE Atlas — complement IODA with probe-level disconnection data
# ---------------------------------------------------------------------------

@with_retry(max_retries=1, base_delay=3)
def fetch_ripe_atlas_probes():
    """Fetch disconnected RIPE Atlas probes and merge into internet_outages (complementing IODA)."""
    from services.fetchers._store import is_any_active

    if not is_any_active("internet_outages"):
        return
    try:
        # 1. Fetch disconnected probes (status=2) — ~2,000 probes, no auth needed
        url_disc = "https://atlas.ripe.net/api/v2/probes/?status=2&page_size=500&format=json"
        resp_disc = fetch_with_curl(url_disc, timeout=20)
        if resp_disc.status_code != 200:
            logger.warning(f"RIPE Atlas probes API returned {resp_disc.status_code}")
            return
        disc_data = resp_disc.json()
        disconnected = disc_data.get("results", [])

        # 2. Fetch connected probe count (page_size=1 — we only need the count)
        url_conn = "https://atlas.ripe.net/api/v2/probes/?status=1&page_size=1&format=json"
        resp_conn = fetch_with_curl(url_conn, timeout=10)
        total_connected = 0
        if resp_conn.status_code == 200:
            total_connected = resp_conn.json().get("count", 0)

        # 3. Group disconnected probes by country
        country_disc: dict = {}
        for p in disconnected:
            cc = p.get("country_code", "")
            if not cc:
                continue
            if cc not in country_disc:
                country_disc[cc] = []
            country_disc[cc].append(p)

        # 4. Get IODA-covered countries to avoid double-reporting
        with _data_lock:
            ioda_outages = list(latest_data.get("internet_outages", []))
        ioda_countries = {
            o.get("country_code", "").upper()
            for o in ioda_outages
            if o.get("datasource") != "ripe-atlas"
        }

        # 5. Build RIPE-only alerts for countries NOT already in IODA
        ripe_alerts = []
        for cc, probes in country_disc.items():
            if cc.upper() in ioda_countries:
                continue  # IODA already covers this country
            if len(probes) < 3:
                continue  # Too few probes to be meaningful

            # Use centroid of disconnected probes as marker location
            lats = [
                p["geometry"]["coordinates"][1]
                for p in probes
                if p.get("geometry") and p["geometry"].get("coordinates")
            ]
            lngs = [
                p["geometry"]["coordinates"][0]
                for p in probes
                if p.get("geometry") and p["geometry"].get("coordinates")
            ]
            if not lats:
                continue

            disc_count = len(probes)
            # Severity: scale 10-80 based on disconnected probe count
            severity = min(80, 10 + disc_count * 2)

            ripe_alerts.append({
                "region_code": f"RIPE-{cc}",
                "region_name": f"{cc} (Atlas probes)",
                "country_code": cc,
                "country_name": cc,
                "level": "critical" if disc_count >= 10 else "warning",
                "datasource": "ripe-atlas",
                "severity": severity,
                "lat": sum(lats) / len(lats),
                "lng": sum(lngs) / len(lngs),
                "probe_count": disc_count,
            })

        # 6. Merge into internet_outages — keep IODA entries, replace old RIPE entries
        with _data_lock:
            current = latest_data.get("internet_outages", [])
            ioda_only = [o for o in current if o.get("datasource") != "ripe-atlas"]
            latest_data["internet_outages"] = ioda_only + ripe_alerts

        if ripe_alerts:
            _mark_fresh("internet_outages")
        logger.info(
            f"RIPE Atlas: {len(ripe_alerts)} countries with probe disconnections "
            f"(from {len(disconnected)} disconnected / ~{total_connected} connected probes)"
        )
    except (
        ConnectionError,
        TimeoutError,
        OSError,
        ValueError,
        KeyError,
        TypeError,
        json.JSONDecodeError,
    ) as e:
        logger.error(f"Error fetching RIPE Atlas probes: {e}")


# ---------------------------------------------------------------------------
# Data Centers (local geocoded JSON)
# ---------------------------------------------------------------------------
_DC_GEOCODED_PATH = Path(__file__).parent.parent.parent / "data" / "datacenters_geocoded.json"


def fetch_datacenters():
    """Load geocoded data centers (5K+ street-level precise locations)."""
    from services.fetchers._store import is_any_active

    if not is_any_active("datacenters"):
        return
    dcs = []
    try:
        if not _DC_GEOCODED_PATH.exists():
            logger.warning(f"Geocoded DC file not found: {_DC_GEOCODED_PATH}")
            return
        raw = json.loads(_DC_GEOCODED_PATH.read_text(encoding="utf-8"))
        for entry in raw:
            lat = entry.get("lat")
            lng = entry.get("lng")
            if lat is None or lng is None:
                continue
            if not (-90 <= lat <= 90 and -180 <= lng <= 180):
                continue
            dcs.append(
                {
                    "name": entry.get("name", "Unknown"),
                    "company": entry.get("company", ""),
                    "street": entry.get("street", ""),
                    "city": entry.get("city", ""),
                    "country": entry.get("country", ""),
                    "zip": entry.get("zip", ""),
                    "lat": lat,
                    "lng": lng,
                }
            )
        logger.info(f"Data centers: {len(dcs)} geocoded locations loaded")
    except (
        ConnectionError,
        TimeoutError,
        OSError,
        ValueError,
        KeyError,
        TypeError,
        json.JSONDecodeError,
    ) as e:
        logger.error(f"Error loading data centers: {e}")
    with _data_lock:
        latest_data["datacenters"] = dcs
    if dcs:
        _mark_fresh("datacenters")


# ---------------------------------------------------------------------------
# Military Bases (static JSON — Western Pacific)
# ---------------------------------------------------------------------------
_MILITARY_BASES_PATH = Path(__file__).parent.parent.parent / "data" / "military_bases.json"


def fetch_military_bases():
    """Load static military base locations (Western Pacific focus)."""
    bases = []
    try:
        if not _MILITARY_BASES_PATH.exists():
            logger.warning(f"Military bases file not found: {_MILITARY_BASES_PATH}")
            return
        raw = json.loads(_MILITARY_BASES_PATH.read_text(encoding="utf-8"))
        for entry in raw:
            lat = entry.get("lat")
            lng = entry.get("lng")
            if lat is None or lng is None:
                continue
            if not (-90 <= lat <= 90 and -180 <= lng <= 180):
                continue
            bases.append({
                "name": entry.get("name", "Unknown"),
                "country": entry.get("country", ""),
                "operator": entry.get("operator", ""),
                "branch": entry.get("branch", ""),
                "lat": lat, "lng": lng,
            })
        logger.info(f"Military bases: {len(bases)} locations loaded")
    except Exception as e:
        logger.error(f"Error loading military bases: {e}")
    with _data_lock:
        latest_data["military_bases"] = bases
    if bases:
        _mark_fresh("military_bases")


# ---------------------------------------------------------------------------
# Power Plants (WRI Global Power Plant Database)
# ---------------------------------------------------------------------------
_POWER_PLANTS_PATH = Path(__file__).parent.parent.parent / "data" / "power_plants.json"


def fetch_power_plants():
    """Load WRI Global Power Plant Database (~35K facilities)."""
    plants = []
    try:
        if not _POWER_PLANTS_PATH.exists():
            logger.warning(f"Power plants file not found: {_POWER_PLANTS_PATH}")
            return
        raw = json.loads(_POWER_PLANTS_PATH.read_text(encoding="utf-8"))
        for entry in raw:
            lat = entry.get("lat")
            lng = entry.get("lng")
            if lat is None or lng is None:
                continue
            if not (-90 <= lat <= 90 and -180 <= lng <= 180):
                continue
            plants.append({
                "name": entry.get("name", "Unknown"),
                "country": entry.get("country", ""),
                "fuel_type": entry.get("fuel_type", "Unknown"),
                "capacity_mw": entry.get("capacity_mw"),
                "owner": entry.get("owner", ""),
                "lat": lat, "lng": lng,
            })
        logger.info(f"Power plants: {len(plants)} facilities loaded")
    except Exception as e:
        logger.error(f"Error loading power plants: {e}")
    with _data_lock:
        latest_data["power_plants"] = plants
    if plants:
        _mark_fresh("power_plants")


# ---------------------------------------------------------------------------
# CCTV Cameras
# ---------------------------------------------------------------------------
def fetch_cctv():
    from services.fetchers._store import is_any_active

    if not is_any_active("cctv"):
        return
    try:
        from services.cctv_pipeline import get_all_cameras

        cameras = get_all_cameras()
        if len(cameras) < 500:
            # Serve the current DB snapshot immediately and let the scheduled
            # ingest cycle populate/refresh cameras asynchronously.
            logger.info(
                "CCTV DB currently has %d cameras — serving cached snapshot and waiting for scheduled ingest",
                len(cameras),
            )
        with _data_lock:
            latest_data["cctv"] = cameras
        _mark_fresh("cctv")
    except (
        ConnectionError,
        TimeoutError,
        OSError,
        ValueError,
        KeyError,
        TypeError,
        json.JSONDecodeError,
    ) as e:
        logger.error(f"Error fetching cctv from DB: {e}")


# ---------------------------------------------------------------------------
# KiwiSDR Receivers
# ---------------------------------------------------------------------------
@with_retry(max_retries=2, base_delay=2)
def fetch_kiwisdr():
    from services.fetchers._store import is_any_active

    if not is_any_active("kiwisdr"):
        return
    try:
        from services.kiwisdr_fetcher import fetch_kiwisdr_nodes

        nodes = fetch_kiwisdr_nodes()
        with _data_lock:
            latest_data["kiwisdr"] = nodes
        _mark_fresh("kiwisdr")
    except (
        ConnectionError,
        TimeoutError,
        OSError,
        ValueError,
        KeyError,
        TypeError,
        json.JSONDecodeError,
    ) as e:
        logger.error(f"Error fetching KiwiSDR nodes: {e}")
        with _data_lock:
            latest_data["kiwisdr"] = []


# ---------------------------------------------------------------------------
# SatNOGS Ground Stations + Observations
# ---------------------------------------------------------------------------
@with_retry(max_retries=2, base_delay=2)
def fetch_satnogs():
    from services.fetchers._store import is_any_active

    if not is_any_active("satnogs"):
        return
    try:
        from services.satnogs_fetcher import fetch_satnogs_stations, fetch_satnogs_observations

        stations = fetch_satnogs_stations()
        obs = fetch_satnogs_observations()
        with _data_lock:
            latest_data["satnogs_stations"] = stations
            latest_data["satnogs_observations"] = obs
        _mark_fresh("satnogs_stations", "satnogs_observations")
    except (
        ConnectionError,
        TimeoutError,
        OSError,
        ValueError,
        KeyError,
        TypeError,
        json.JSONDecodeError,
    ) as e:
        logger.error(f"Error fetching SatNOGS: {e}")


# ---------------------------------------------------------------------------
# PSK Reporter — HF Digital Mode Spots
# ---------------------------------------------------------------------------
@with_retry(max_retries=2, base_delay=2)
def fetch_psk_reporter():
    from services.fetchers._store import is_any_active

    if not is_any_active("psk_reporter"):
        return
    try:
        from services.psk_reporter_fetcher import fetch_psk_reporter_spots

        spots = fetch_psk_reporter_spots()
        with _data_lock:
            latest_data["psk_reporter"] = spots
        _mark_fresh("psk_reporter")
    except (
        ConnectionError,
        TimeoutError,
        OSError,
        ValueError,
        KeyError,
        TypeError,
        json.JSONDecodeError,
    ) as e:
        logger.error(f"Error fetching PSK Reporter: {e}")
        with _data_lock:
            latest_data["psk_reporter"] = []


# ---------------------------------------------------------------------------
# TinyGS LoRa Satellites
# ---------------------------------------------------------------------------
@with_retry(max_retries=2, base_delay=2)
def fetch_tinygs():
    from services.fetchers._store import is_any_active

    if not is_any_active("tinygs"):
        return
    try:
        from services.tinygs_fetcher import fetch_tinygs_satellites

        sats = fetch_tinygs_satellites()
        with _data_lock:
            latest_data["tinygs_satellites"] = sats
        _mark_fresh("tinygs_satellites")
    except (
        ConnectionError,
        TimeoutError,
        OSError,
        ValueError,
        KeyError,
        TypeError,
        json.JSONDecodeError,
    ) as e:
        logger.error(f"Error fetching TinyGS: {e}")


# ---------------------------------------------------------------------------
# Police Scanners (OpenMHZ) — geocode city+state via local GeoNames DB
# ---------------------------------------------------------------------------
_scanner_geo_cache: dict = {}  # city|state -> (lat, lng) — populated once from GeoNames


def _build_scanner_geo_lookup():
    """Build a US city/county→coords lookup from reverse_geocoder's bundled GeoNames CSV."""
    if _scanner_geo_cache:
        return
    try:
        import csv, os, reverse_geocoder as rg

        geo_file = os.path.join(os.path.dirname(rg.__file__), "rg_cities1000.csv")
        # US state abbreviation → admin1 name mapping
        _abbr = {
            "AL": "Alabama",
            "AK": "Alaska",
            "AZ": "Arizona",
            "AR": "Arkansas",
            "CA": "California",
            "CO": "Colorado",
            "CT": "Connecticut",
            "DE": "Delaware",
            "FL": "Florida",
            "GA": "Georgia",
            "HI": "Hawaii",
            "ID": "Idaho",
            "IL": "Illinois",
            "IN": "Indiana",
            "IA": "Iowa",
            "KS": "Kansas",
            "KY": "Kentucky",
            "LA": "Louisiana",
            "ME": "Maine",
            "MD": "Maryland",
            "MA": "Massachusetts",
            "MI": "Michigan",
            "MN": "Minnesota",
            "MS": "Mississippi",
            "MO": "Missouri",
            "MT": "Montana",
            "NE": "Nebraska",
            "NV": "Nevada",
            "NH": "New Hampshire",
            "NJ": "New Jersey",
            "NM": "New Mexico",
            "NY": "New York",
            "NC": "North Carolina",
            "ND": "North Dakota",
            "OH": "Ohio",
            "OK": "Oklahoma",
            "OR": "Oregon",
            "PA": "Pennsylvania",
            "RI": "Rhode Island",
            "SC": "South Carolina",
            "SD": "South Dakota",
            "TN": "Tennessee",
            "TX": "Texas",
            "UT": "Utah",
            "VT": "Vermont",
            "VA": "Virginia",
            "WA": "Washington",
            "WV": "West Virginia",
            "WI": "Wisconsin",
            "WY": "Wyoming",
            "DC": "Washington, D.C.",
        }
        state_full = {v.lower(): k for k, v in _abbr.items()}
        state_full["washington, d.c."] = "DC"

        county_coords = {}  # admin2(county)|state -> (lat, lon) — first city per county
        with open(geo_file, "r", encoding="utf-8") as f:
            reader = csv.reader(f)
            next(reader, None)  # skip header
            for row in reader:
                if len(row) < 6 or row[5] != "US":
                    continue
                lat_s, lon_s, name, admin1, admin2 = row[0], row[1], row[2], row[3], row[4]
                st = state_full.get(admin1.lower(), "")
                if not st:
                    continue
                coords = (float(lat_s), float(lon_s))
                # City name → coords
                _scanner_geo_cache[f"{name.lower()}|{st}"] = coords
                # County name → coords (keep first match per county, usually the largest city)
                if admin2:
                    county_key = f"{admin2.lower()}|{st}"
                    if county_key not in county_coords:
                        county_coords[county_key] = coords
                    # Also strip " County" suffix for matching
                    stripped = admin2.lower().replace(" county", "").strip()
                    stripped_key = f"{stripped}|{st}"
                    if stripped_key not in county_coords:
                        county_coords[stripped_key] = coords

        # Merge county lookups (don't override city entries)
        for k, v in county_coords.items():
            if k not in _scanner_geo_cache:
                _scanner_geo_cache[k] = v
        # Special case: DC
        _scanner_geo_cache["washington|DC"] = (38.89511, -77.03637)
        logger.info(f"Scanner geo lookup: {len(_scanner_geo_cache)} US entries loaded")
    except Exception as e:
        logger.warning(f"Failed to build scanner geo lookup: {e}")


def _geocode_scanner(city: str, state: str):
    """Look up city+state coordinates from local GeoNames cache."""
    _build_scanner_geo_lookup()
    if not city or not state:
        return None
    st = state.upper()
    # Strip trailing state from city (e.g. "Lehigh, PA")
    c = city.strip()
    if ", " in c:
        parts = c.rsplit(", ", 1)
        if len(parts[1]) <= 2:
            c = parts[0]
    name = c.lower()
    # Try exact city match
    result = _scanner_geo_cache.get(f"{name}|{st}")
    if result:
        return result
    # Strip "County" / "Co" suffix
    stripped = name.replace(" county", "").replace(" co", "").strip()
    result = _scanner_geo_cache.get(f"{stripped}|{st}")
    if result:
        return result
    # Normalize "St." / "St" → "Saint"
    import re

    normed = re.sub(r"\bst\.?\s", "saint ", name)
    if normed != name:
        result = _scanner_geo_cache.get(f"{normed}|{st}")
        if result:
            return result
        # Also try with "s" suffix: "St. Marys" → "Saint Marys" and "Saint Mary's"
        for variant in [normed.rstrip("s"), normed.replace("ys", "y's")]:
            result = _scanner_geo_cache.get(f"{variant}|{st}")
            if result:
                return result
    # "Prince Georges" → "Prince George's" (apostrophe variants)
    if "georges" in name:
        key = name.replace("georges", "george's") + "|" + st
        result = _scanner_geo_cache.get(key)
        if result:
            return result
    # Multi-location: "Scott and Carver" → try first part
    if " and " in name:
        first = name.split(" and ")[0].strip()
        result = _scanner_geo_cache.get(f"{first}|{st}")
        if result:
            return result
    # Comma-separated list: "Adams, Jackson, Juneau" → try first
    if ", " in name:
        first = name.split(", ")[0].strip()
        result = _scanner_geo_cache.get(f"{first}|{st}")
        if result:
            return result
    # Drop directional prefix: "North Fulton" → "Fulton"
    for prefix in ("north ", "south ", "east ", "west "):
        if name.startswith(prefix):
            result = _scanner_geo_cache.get(f"{name[len(prefix):]}|{st}")
            if result:
                return result
    return None


@with_retry(max_retries=2, base_delay=2)
def fetch_scanners():
    from services.fetchers._store import is_any_active

    if not is_any_active("scanners"):
        return
    try:
        from services.radio_intercept import get_openmhz_systems

        systems = get_openmhz_systems()
        scanners = []
        for s in systems:
            city = s.get("city", "") or s.get("county", "") or ""
            state = s.get("state", "")
            coords = _geocode_scanner(city, state)
            if not coords:
                continue
            lat, lng = coords
            scanners.append(
                {
                    "shortName": s.get("shortName", ""),
                    "name": s.get("name", "Unknown Scanner"),
                    "lat": round(lat, 5),
                    "lng": round(lng, 5),
                    "city": city,
                    "state": state,
                    "clientCount": s.get("clientCount", 0),
                    "description": s.get("description", ""),
                }
            )
        with _data_lock:
            latest_data["scanners"] = scanners
        if scanners:
            _mark_fresh("scanners")
        logger.info(f"Scanners: {len(scanners)}/{len(systems)} geocoded")
    except (
        ConnectionError,
        TimeoutError,
        OSError,
        ValueError,
        KeyError,
        TypeError,
        json.JSONDecodeError,
    ) as e:
        logger.error(f"Error fetching scanners: {e}")
        with _data_lock:
            latest_data["scanners"] = []