Shadowbroker/backend/services/osint/lookups.py

"""Server-side OSINT lookups (Osiris port, HTTPS outbound only)."""
from __future__ import annotations

import ipaddress
import json
import logging
import re
import socket
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from typing import Any
from urllib.parse import quote

from services.network_utils import fetch_with_curl
from services.sanctions.ofac import match_exact, search_sanctions
from services.ssrf_guard import safe_get, validate_domain, validate_host

logger = logging.getLogger(__name__)

_IPV4_RE = re.compile(r"^(\d{1,3}\.){3}\d{1,3}$")
_IPV6_RE = re.compile(r"^[0-9a-fA-F:]+$")
_CVE_RE = re.compile(r"^CVE-\d{4}-\d{4,}$", re.I)
_ASN_RE = re.compile(r"^(AS)?\d+$", re.I)


def _now_iso() -> str:
    return datetime.now(timezone.utc).isoformat()


def _json_get(url: str, *, timeout: float = 8.0, headers: dict[str, str] | None = None) -> Any:
    resp = fetch_with_curl(url, timeout=timeout, headers=headers or {"Accept": "application/json"})
    if resp.status_code != 200:
        return None
    try:
        return resp.json()
    except Exception:
        return None


def _sanctions_hits(*values: str) -> list[dict[str, Any]] | None:
    hits: list[dict[str, Any]] = []
    seen: set[str] = set()
    for value in values:
        if not value or value in seen:
            continue
        seen.add(value)
        entries = match_exact(value)
        if entries:
            hits.append({"matched_value": value, "entries": entries})
    return hits or None


def lookup_ip(ip: str) -> dict[str, Any]:
    if not _IPV4_RE.match(ip) and not _IPV6_RE.match(ip):
        raise ValueError("Invalid IP format")
    check = validate_host(ip.strip("[]"))
    if not check.get("ok"):
        raise ValueError(check.get("reason", "blocked IP"))

    results: dict[str, Any] = {"ip": ip, "timestamp": _now_iso()}
    fields = (
        "status,message,continent,country,countryCode,region,regionName,city,zip,"
        "lat,lon,timezone,isp,org,as,asname,mobile,proxy,hosting,query"
    )
    geo = _json_get(f"https://ip-api.com/json/{quote(ip)}?fields={fields}", timeout=5)
    if isinstance(geo, dict) and geo.get("status") == "success":
        results["geo"] = {
            "country": geo.get("country"),
            "country_code": geo.get("countryCode"),
            "region": geo.get("regionName"),
            "city": geo.get("city"),
            "lat": geo.get("lat"),
            "lon": geo.get("lon"),
            "timezone": geo.get("timezone"),
            "isp": geo.get("isp"),
            "org": geo.get("org"),
            "as_number": geo.get("as"),
            "as_name": geo.get("asname"),
            "is_mobile": geo.get("mobile"),
            "is_proxy": geo.get("proxy"),
            "is_hosting": geo.get("hosting"),
        }
        results["reputation"] = {
            "is_proxy": bool(geo.get("proxy")),
            "is_hosting": bool(geo.get("hosting")),
            "is_mobile": bool(geo.get("mobile")),
            "risk_level": "HIGH" if geo.get("proxy") else "MEDIUM" if geo.get("hosting") else "LOW",
        }
        sm = _sanctions_hits(geo.get("org") or "", geo.get("isp") or "", geo.get("asname") or "")
        if sm:
            results["sanctions_match"] = {"source": "OFAC SDN", "hits": sm}
    return results


def lookup_dns(domain: str) -> dict[str, Any]:
    if not validate_domain(domain):
        raise ValueError("Invalid domain format")
    results: dict[str, Any] = {"domain": domain, "records": {}, "timestamp": _now_iso()}
    for rtype in ("A", "AAAA", "MX", "NS", "TXT", "CNAME", "SOA"):
        data = _json_get(
            f"https://dns.google/resolve?name={quote(domain)}&type={rtype}",
            timeout=5,
        )
        answers = []
        if isinstance(data, dict):
            for ans in data.get("Answer") or []:
                answers.append(
                    {
                        "name": ans.get("name"),
                        "type": ans.get("type"),
                        "ttl": ans.get("TTL"),
                        "data": ans.get("data"),
                    }
                )
        results["records"][rtype] = answers
    a_records = results["records"].get("A") or []
    mx_records = results["records"].get("MX") or []
    ns_records = results["records"].get("NS") or []
    results["summary"] = {
        "ip_addresses": [r["data"] for r in a_records if r.get("data")],
        "mail_servers": [r["data"] for r in mx_records if r.get("data")],
        "nameservers": [r["data"] for r in ns_records if r.get("data")],
        "total_records": sum(len(v) for v in results["records"].values()),
    }
    return results


def lookup_whois(domain: str) -> dict[str, Any]:
    if not validate_domain(domain):
        raise ValueError("Invalid domain format")
    results: dict[str, Any] = {"domain": domain, "timestamp": _now_iso()}
    rdap = _json_get(f"https://rdap.org/domain/{quote(domain)}", timeout=8)
    if isinstance(rdap, dict):
        entities = []
        for ent in rdap.get("entities") or []:
            vcard = ent.get("vcardArray")
            name = org = None
            if isinstance(vcard, list) and len(vcard) > 1:
                for row in vcard[1]:
                    if row[0] == "fn":
                        name = row[3]
                    if row[0] == "org":
                        org = row[3]
            if name or org:
                entities.append({"handle": ent.get("handle"), "roles": ent.get("roles"), "name": name, "org": org})
        events = [
            {"action": e.get("eventAction"), "date": e.get("eventDate")}
            for e in (rdap.get("events") or [])
        ]
        results["rdap"] = {
            "handle": rdap.get("handle"),
            "name": rdap.get("ldhName"),
            "status": rdap.get("status"),
            "events": events,
            "nameservers": [ns.get("ldhName") for ns in (rdap.get("nameservers") or [])],
            "entities": entities,
        }
        results["registration"] = next((e["date"] for e in events if e["action"] == "registration"), None)
        results["expiration"] = next((e["date"] for e in events if e["action"] == "expiration"), None)
        results["last_changed"] = next((e["date"] for e in events if e["action"] == "last changed"), None)
        sm = _sanctions_hits(*(e.get("name") or "" for e in entities), *(e.get("org") or "" for e in entities))
        if sm:
            results["sanctions_match"] = {"source": "OFAC SDN", "hits": sm}

    try:
        res = safe_get(f"https://{domain}", timeout=5, headers={"User-Agent": "Shadowbroker-OSINT/1.0"})
        headers = {}
        for h in (
            "server",
            "x-powered-by",
            "x-frame-options",
            "strict-transport-security",
            "content-security-policy",
            "x-content-type-options",
            "x-xss-protection",
            "referrer-policy",
            "permissions-policy",
        ):
            val = res.headers.get(h)
            if val:
                headers[h] = val
        score = sum(
            1
            for k in (
                "strict-transport-security",
                "content-security-policy",
                "x-frame-options",
                "x-content-type-options",
                "referrer-policy",
            )
            if k in headers
        ) + (2 if "strict-transport-security" in headers else 0) + (2 if "content-security-policy" in headers else 0)
        results["http"] = {"status": res.status_code, "headers": headers, "final_url": res.url}
        results["security_score"] = {
            "score": score,
            "max": 7,
            "grade": "A" if score >= 5 else "B" if score >= 3 else "C" if score >= 1 else "F",
        }
    except Exception as exc:
        logger.debug("WHOIS header probe failed for %s: %s", domain, exc)
    return results


def lookup_certs(domain: str) -> dict[str, Any]:
    if not validate_domain(domain):
        raise ValueError("Invalid domain format")
    resp = fetch_with_curl(
        f"https://crt.sh/?q=%25.{quote(domain)}&output=json",
        timeout=10,
        headers={"User-Agent": "Shadowbroker-OSINT/1.0"},
    )
    if resp.status_code != 200:
        return {"domain": domain, "certificates": [], "error": "crt.sh unavailable"}
    try:
        certs = resp.json()
    except Exception:
        certs = []
    seen: set[str] = set()
    subdomains: set[str] = set()
    unique: list[dict[str, Any]] = []
    for cert in (certs or [])[:200]:
        key = f"{cert.get('common_name')}-{cert.get('serial_number')}"
        if key in seen:
            continue
        seen.add(key)
        for name in (cert.get("name_value") or "").split("\n"):
            clean = name.strip().replace("*.", "")
            if clean.endswith(domain):
                subdomains.add(clean)
        unique.append(
            {
                "id": cert.get("id"),
                "issuer": cert.get("issuer_name"),
                "common_name": cert.get("common_name"),
                "not_before": cert.get("not_before"),
                "not_after": cert.get("not_after"),
            }
        )
    return {
        "domain": domain,
        "certificates": unique[:50],
        "subdomains": sorted(subdomains)[:100],
        "total_found": len(certs or []),
        "timestamp": _now_iso(),
    }


def lookup_threats(query: str | None = None) -> dict[str, Any]:
    results: dict[str, Any] = {"timestamp": _now_iso()}
    pulses = _json_get("https://otx.alienvault.com/api/v1/pulses/activity?limit=10", timeout=8)
    if isinstance(pulses, dict):
        results["pulses"] = [
            {
                "name": p.get("name"),
                "description": (p.get("description") or "")[:200],
                "created": p.get("created"),
                "tags": (p.get("tags") or [])[:5],
                "adversary": p.get("adversary"),
                "indicators_count": p.get("indicator_count"),
            }
            for p in (pulses.get("results") or [])[:10]
        ]
    if query:
        if _IPV4_RE.match(query):
            try:
                tor_resp = fetch_with_curl("https://check.torproject.org/torbulkexitlist", timeout=5)
                results["tor_exit_node"] = query in (tor_resp.text or "").splitlines() if tor_resp.status_code == 200 else None
            except Exception:
                results["tor_exit_node"] = None
            otx = _json_get(f"https://otx.alienvault.com/api/v1/indicators/IPv4/{quote(query)}/general", timeout=5)
            if isinstance(otx, dict):
                results["otx"] = {
                    "reputation": otx.get("reputation"),
                    "pulse_count": (otx.get("pulse_info") or {}).get("count", 0),
                    "country": otx.get("country_name"),
                    "asn": otx.get("asn"),
                }
        elif validate_domain(query):
            otx = _json_get(f"https://otx.alienvault.com/api/v1/indicators/domain/{quote(query)}/general", timeout=5)
            if isinstance(otx, dict):
                results["otx"] = {"pulse_count": (otx.get("pulse_info") or {}).get("count", 0)}
    pulse_count = (results.get("otx") or {}).get("pulse_count", 0)
    results["threat_level"] = "HIGH" if pulse_count > 5 else "MEDIUM" if pulse_count > 0 else "LOW"
    return results


def lookup_bgp(query: str) -> dict[str, Any]:
    results: dict[str, Any] = {"query": query, "timestamp": _now_iso()}
    if _IPV4_RE.match(query):
        data = _json_get(f"https://api.bgpview.io/ip/{quote(query)}", timeout=8)
        if isinstance(data, dict) and data.get("status") == "ok":
            results["ip"] = data.get("data")
            results["type"] = "ip"
        return results
    if _ASN_RE.match(query):
        asn_num = re.sub(r"^AS", "", query, flags=re.I)
        asn = _json_get(f"https://api.bgpview.io/asn/{asn_num}", timeout=8)
        prefixes = _json_get(f"https://api.bgpview.io/asn/{asn_num}/prefixes", timeout=8)
        peers = _json_get(f"https://api.bgpview.io/asn/{asn_num}/peers", timeout=8)
        if isinstance(asn, dict) and asn.get("status") == "ok":
            results["asn"] = asn.get("data")
        if isinstance(prefixes, dict) and prefixes.get("status") == "ok":
            pdata = prefixes.get("data") or {}
            results["prefixes"] = {
                "ipv4": (pdata.get("ipv4_prefixes") or [])[:20],
                "ipv6": (pdata.get("ipv6_prefixes") or [])[:10],
                "total_v4": len(pdata.get("ipv4_prefixes") or []),
                "total_v6": len(pdata.get("ipv6_prefixes") or []),
            }
        if isinstance(peers, dict) and peers.get("status") == "ok":
            pdata = peers.get("data") or {}
            results["peers"] = {
                "upstream": (pdata.get("ipv4_peers") or [])[:10],
                "total": len(pdata.get("ipv4_peers") or []),
            }
        results["type"] = "asn"
        return results
    raise ValueError("Unrecognized query format. Use IP address or AS number.")


def lookup_sanctions(query: str, *, schema: str | None = None, limit: int = 25) -> dict[str, Any]:
    matches = search_sanctions(query, schema=schema, limit=limit)
    return {
        "query": query,
        "schema": schema,
        "total": len(matches),
        "matches": matches,
        "source": "OpenSanctions / US OFAC SDN",
        "timestamp": _now_iso(),
    }


def lookup_cve(cve: str) -> dict[str, Any]:
    if not _CVE_RE.match(cve):
        raise ValueError("Invalid CVE format")
    cve_id = cve.upper()
    data = _json_get(f"https://cveawg.mitre.org/api/cve/{quote(cve_id)}", timeout=8)
    if isinstance(data, dict) and data.get("cveMetadata"):
        meta = data["cveMetadata"]
        desc = ""
        for block in (data.get("containers") or {}).get("cna", {}).get("descriptions") or []:
            if block.get("lang") == "en":
                desc = block.get("value") or desc
        return {"id": meta.get("cveId", cve_id), "description": desc or "No description.", "timestamp": _now_iso()}
    fallback = _json_get(f"https://cve.circl.lu/api/cve/{quote(cve_id)}", timeout=8)
    if isinstance(fallback, dict):
        return {
            "id": fallback.get("id", cve_id),
            "description": fallback.get("summary") or "No description.",
            "cvss": fallback.get("cvss"),
            "references": (fallback.get("references") or [])[:5],
            "timestamp": _now_iso(),
        }
    raise ValueError("CVE not found")


def lookup_mac(mac: str) -> dict[str, Any]:
    clean = mac.strip().upper()
    clean = re.sub(r"[^A-F0-9:-]", "", clean)
    data = _json_get(f"https://api.macvendors.com/{quote(clean)}", timeout=8)
    if isinstance(data, dict):
        return {"mac": clean, "vendor": data.get("company") or data.get("organization") or "Not Found"}
    if isinstance(data, str) and data:
        return {"mac": clean, "vendor": data}
    return {"mac": clean, "vendor": "Not Found"}


def lookup_github(username: str) -> dict[str, Any]:
    user = _json_get(f"https://api.github.com/users/{quote(username)}", timeout=8)
    if not isinstance(user, dict) or user.get("message") == "Not Found":
        raise ValueError("GitHub user not found")
    repos = _json_get(f"https://api.github.com/users/{quote(username)}/repos?per_page=10&sort=updated", timeout=8)
    return {
        "username": username,
        "profile": {
            "name": user.get("name"),
            "bio": user.get("bio"),
            "company": user.get("company"),
            "location": user.get("location"),
            "public_repos": user.get("public_repos"),
            "followers": user.get("followers"),
            "created_at": user.get("created_at"),
            "html_url": user.get("html_url"),
        },
        "repos": [
            {"name": r.get("name"), "language": r.get("language"), "stars": r.get("stargazers_count")}
            for r in (repos or [])[:10]
            if isinstance(r, dict)
        ],
        "timestamp": _now_iso(),
    }


def lookup_leaks(email: str) -> dict[str, Any]:
    if "@" not in email or len(email) < 5:
        raise ValueError("Invalid email")
    # HIBP requires API key for v3; use public breach directory style via leak-lookup (rate limited)
    data = _json_get(f"https://leakcheck.io/api/public?check={quote(email)}", timeout=8)
    if isinstance(data, dict):
        return {
            "email": email,
            "found": bool(data.get("found")),
            "sources": data.get("sources") or [],
            "timestamp": _now_iso(),
        }
    return {"email": email, "found": False, "sources": [], "timestamp": _now_iso()}


def sweep_init(ip: str, cidr: int = 24) -> dict[str, Any]:
    try:
        addr = ipaddress.IPv4Address(ip)
    except ValueError as exc:
        raise ValueError("Invalid IPv4 address format") from exc
    if addr.is_private or addr.is_loopback or addr.is_link_local or addr.is_reserved:
        raise ValueError("Private and reserved IP ranges are not allowed")
    if cidr < 24 or cidr > 32:
        raise ValueError("CIDR must be between 24 and 32")

    fields = "status,message,country,countryCode,region,regionName,city,lat,lon,isp,org,as,proxy,hosting"
    geo = _json_get(f"https://ip-api.com/json/{quote(ip)}?fields={fields}", timeout=5)
    if not isinstance(geo, dict) or geo.get("status") != "success":
        raise ValueError(f"Geolocation failed: {(geo or {}).get('message', 'unknown')}")
    return {
        "center": {
            "lat": geo.get("lat"),
            "lng": geo.get("lon"),
            "city": geo.get("city"),
            "region": geo.get("regionName"),
            "country": geo.get("country"),
            "countryCode": geo.get("countryCode"),
            "isp": geo.get("isp"),
            "asn": geo.get("as") or "",
            "org": geo.get("org") or "",
        },
        "target_ip": ip,
        "cidr": cidr,
    }


def _internetdb_lookup(ip: str) -> dict[str, Any] | None:
    try:
        resp = fetch_with_curl(
            f"https://internetdb.shodan.io/{quote(ip)}",
            timeout=4,
            headers={"Accept": "application/json"},
        )
        if resp.status_code == 404:
            return None
        if resp.status_code != 200:
            return None
        return resp.json()
    except Exception:
        return None


def sweep_scan(subnet_start: str, cidr: int, *, max_workers: int = 12) -> dict[str, Any]:
    """Scan a /24-/32 via Shodan InternetDB (server-side proxy)."""
    base = int(ipaddress.IPv4Address(subnet_start))
    host_count = 2 ** (32 - cidr)
    if host_count > 256:
        raise ValueError("Subnet too large")
    ips = [str(ipaddress.IPv4Address(base + i)) for i in range(host_count)]
    devices: list[dict[str, Any]] = []
    t0 = time.time()
    with ThreadPoolExecutor(max_workers=max_workers) as pool:
        futures = {pool.submit(_internetdb_lookup, ip): ip for ip in ips}
        for fut in as_completed(futures):
            ip = futures[fut]
            data = fut.result()
            if not data:
                continue
            devices.append(
                {
                    "ip": data.get("ip") or ip,
                    "ports": data.get("ports") or [],
                    "hostnames": data.get("hostnames") or [],
                    "cpes": data.get("cpes") or [],
                    "vulns": data.get("vulns") or [],
                    "tags": data.get("tags") or [],
                }
            )
    return {
        "devices": devices,
        "summary": {"total_hosts": host_count, "total_responsive": len(devices)},
        "sweep_time_ms": int((time.time() - t0) * 1000),
    }


def subnet_start_for(ip: str, cidr: int) -> str:
    net = ipaddress.IPv4Network(f"{ip}/{cidr}", strict=False)
    return str(net.network_address)