god-eye/internal/sources/extra.go

// Additional passive sources added in v2.0 to close the gap with
// subfinder / BBOT. Every source here is:
//   - Free and key-less (no API key required)
//   - Defensive (fail-open — returns an empty slice on any error)
//   - Bounded by the shared HTTP clients
//
// If a source goes offline upstream, the corresponding fetcher keeps
// returning empty — the scan still succeeds.

package sources

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// FetchOmnisint queries the free Omnisint Sonar mirror. It may be offline
// on any given day — fail-open.
func FetchOmnisint(domain string) ([]string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
	defer cancel()

	u := fmt.Sprintf("https://sonar.omnisint.io/subdomains/%s", url.PathEscape(domain))
	req, _ := http.NewRequestWithContext(ctx, "GET", u, nil)
	req.Header.Set("User-Agent", "god-eye-v2")

	resp, err := StandardClient.Do(req)
	if err != nil {
		return []string{}, nil
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return []string{}, nil
	}
	body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024))
	if err != nil {
		return []string{}, nil
	}

	var list []string
	if err := json.Unmarshal(body, &list); err != nil {
		return []string{}, nil
	}

	seen := make(map[string]bool)
	var out []string
	for _, s := range list {
		s = strings.ToLower(strings.TrimSpace(s))
		if s != "" && strings.HasSuffix(s, domain) && !seen[s] {
			seen[s] = true
			out = append(out, s)
		}
	}
	return out, nil
}

// FetchHudsonRock queries the free Cavalier InfoStealer intelligence API.
// Surfaces domain assets referenced in leaked stealer logs; useful for
// discovering shadow internal hostnames.
func FetchHudsonRock(domain string) ([]string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
	defer cancel()

	u := fmt.Sprintf("https://cavalier.hudsonrock.com/api/json/v2/osint-tools/search-by-domain?domain=%s", url.QueryEscape(domain))
	req, _ := http.NewRequestWithContext(ctx, "GET", u, nil)
	req.Header.Set("User-Agent", "god-eye-v2")

	resp, err := StandardClient.Do(req)
	if err != nil {
		return []string{}, nil
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return []string{}, nil
	}
	body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024))
	if err != nil {
		return []string{}, nil
	}

	// HudsonRock returns free-form JSON; we just mine every subdomain-like
	// token from the response body via the shared regex.
	return ExtractSubdomains(string(body), domain), nil
}

// FetchWebArchiveCDX queries the Internet Archive CDX server — a richer
// variant of the existing Wayback source. Pulls URLs with fewer limits
// and extracts hostnames that match the target domain.
func FetchWebArchiveCDX(domain string) ([]string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	u := fmt.Sprintf("https://web.archive.org/cdx/search/cdx?url=*.%s/*&output=json&collapse=urlkey&limit=5000&fl=original", url.QueryEscape(domain))
	req, _ := http.NewRequestWithContext(ctx, "GET", u, nil)
	req.Header.Set("User-Agent", "god-eye-v2")

	resp, err := SlowClient.Do(req)
	if err != nil {
		return []string{}, nil
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return []string{}, nil
	}
	body, err := io.ReadAll(io.LimitReader(resp.Body, 16*1024*1024))
	if err != nil {
		return []string{}, nil
	}

	// Response shape: [["original"], ["url1"], ["url2"], ...] — first row
	// is the header, subsequent rows are single-element arrays with the URL.
	var rows [][]string
	if err := json.Unmarshal(body, &rows); err != nil {
		return []string{}, nil
	}

	seen := make(map[string]bool)
	var out []string
	for i, row := range rows {
		if i == 0 { // skip header
			continue
		}
		if len(row) == 0 {
			continue
		}
		for _, host := range ExtractSubdomains(row[0], domain) {
			if !seen[host] {
				seen[host] = true
				out = append(out, host)
			}
		}
	}
	return out, nil
}

// FetchDigitorus queries the free Digitorus CT log mirror — an alternative
// to crt.sh that sometimes returns fresher data.
func FetchDigitorus(domain string) ([]string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancel()

	u := fmt.Sprintf("https://certificatedetails.com/api/find/%s", url.QueryEscape(domain))
	req, _ := http.NewRequestWithContext(ctx, "GET", u, nil)
	req.Header.Set("User-Agent", "god-eye-v2")

	resp, err := StandardClient.Do(req)
	if err != nil {
		return []string{}, nil
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return []string{}, nil
	}
	body, err := io.ReadAll(io.LimitReader(resp.Body, 4*1024*1024))
	if err != nil {
		return []string{}, nil
	}

	// Free-form JSON; mine hostnames.
	return ExtractSubdomains(string(body), domain), nil
}