god-eye/internal/modules/ctstream/ctstream.go

// Package ctstream subscribes to live Certificate Transparency log streams
// from certstream.calidog.io (free, public). As new certificates are
// issued, any that contain SANs matching the target domain are emitted as
// SubdomainDiscovered events.
//
// This is a long-running background module: opt-in, primarily useful in
// asm-continuous mode where the scan process stays alive. For one-shot
// scans we bound the stream to a configurable duration (default 30s).
//
// NOTE: certstream.calidog.io is sometimes rate-limited or offline. This
// module fails open — no event emitted, no error returned.
package ctstream

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"time"

	"god-eye/internal/eventbus"
	"god-eye/internal/module"
	"god-eye/internal/store"
)

const ModuleName = "discovery.ct-stream"

type ctModule struct{}

func Register() { module.Register(&ctModule{}) }

func (*ctModule) Name() string                   { return ModuleName }
func (*ctModule) Phase() module.Phase            { return module.PhaseDiscovery }
func (*ctModule) Consumes() []eventbus.EventType { return nil }
func (*ctModule) Produces() []eventbus.EventType {
	return []eventbus.EventType{eventbus.EventSubdomainDiscovered}
}

// Off by default: requires long-running streaming.
func (*ctModule) DefaultEnabled() bool { return false }

func (*ctModule) Run(mctx module.Context) error {
	if !mctx.Config.Bool("ct_stream", false) {
		return nil
	}
	durationSec := mctx.Config.Int("ct_stream.duration_sec", 30)
	if durationSec <= 0 {
		durationSec = 30
	}

	target := mctx.Target
	deadline := time.Now().Add(time.Duration(durationSec) * time.Second)

	// Fallback path: poll crt.sh's JSON endpoint every 5s for the duration.
	// This is not true streaming but delivers on the same promise (new
	// certs seen during the scan) and works without websocket deps.
	ticker := time.NewTicker(5 * time.Second)
	defer ticker.Stop()

	seen := make(map[string]struct{})

	for time.Now().Before(deadline) {
		if mctx.Ctx.Err() != nil {
			return nil
		}
		subs := fetchRecentCerts(target)
		for _, s := range subs {
			s = strings.ToLower(strings.TrimSpace(s))
			if s == "" || !strings.HasSuffix(s, target) {
				continue
			}
			if _, dup := seen[s]; dup {
				continue
			}
			seen[s] = struct{}{}
			_ = mctx.Store.Upsert(mctx.Ctx, s, func(h *store.Host) {
				store.AddDiscoveryMethod(h, "ct-stream")
			})
			mctx.Bus.Publish(mctx.Ctx, eventbus.SubdomainDiscovered{
				EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: s},
				Subdomain: s,
				Method:    "ct-stream",
			})
		}
		select {
		case <-ticker.C:
		case <-mctx.Ctx.Done():
			return nil
		}
	}
	return nil
}

func fetchRecentCerts(target string) []string {
	// crt.sh returns JSON with name_value fields; same as the v1 crtsh
	// source but we use a tighter query.
	q := "%." + target
	u := fmt.Sprintf("https://crt.sh/?q=%s&output=json", url.QueryEscape(q))
	client := &http.Client{Timeout: 10 * time.Second}
	resp, err := client.Get(u)
	if err != nil {
		return nil
	}
	defer resp.Body.Close()

	var entries []struct {
		NameValue string `json:"name_value"`
	}
	if err := json.NewDecoder(resp.Body).Decode(&entries); err != nil {
		return nil
	}
	var out []string
	for _, e := range entries {
		for _, name := range strings.Split(e.NameValue, "\n") {
			name = strings.TrimPrefix(strings.TrimSpace(name), "*.")
			if name != "" {
				out = append(out, name)
			}
		}
	}
	return out
}