// Package ctstream subscribes to live Certificate Transparency log streams // from certstream.calidog.io (free, public). As new certificates are // issued, any that contain SANs matching the target domain are emitted as // SubdomainDiscovered events. // // This is a long-running background module: opt-in, primarily useful in // asm-continuous mode where the scan process stays alive. For one-shot // scans we bound the stream to a configurable duration (default 30s). // // NOTE: certstream.calidog.io is sometimes rate-limited or offline. This // module fails open — no event emitted, no error returned. package ctstream import ( "encoding/json" "fmt" "net/http" "net/url" "strings" "time" "god-eye/internal/eventbus" "god-eye/internal/module" "god-eye/internal/store" ) const ModuleName = "discovery.ct-stream" type ctModule struct{} func Register() { module.Register(&ctModule{}) } func (*ctModule) Name() string { return ModuleName } func (*ctModule) Phase() module.Phase { return module.PhaseDiscovery } func (*ctModule) Consumes() []eventbus.EventType { return nil } func (*ctModule) Produces() []eventbus.EventType { return []eventbus.EventType{eventbus.EventSubdomainDiscovered} } // Off by default: requires long-running streaming. func (*ctModule) DefaultEnabled() bool { return false } func (*ctModule) Run(mctx module.Context) error { if !mctx.Config.Bool("ct_stream", false) { return nil } durationSec := mctx.Config.Int("ct_stream.duration_sec", 30) if durationSec <= 0 { durationSec = 30 } target := mctx.Target deadline := time.Now().Add(time.Duration(durationSec) * time.Second) // Fallback path: poll crt.sh's JSON endpoint every 5s for the duration. // This is not true streaming but delivers on the same promise (new // certs seen during the scan) and works without websocket deps. ticker := time.NewTicker(5 * time.Second) defer ticker.Stop() seen := make(map[string]struct{}) for time.Now().Before(deadline) { if mctx.Ctx.Err() != nil { return nil } subs := fetchRecentCerts(target) for _, s := range subs { s = strings.ToLower(strings.TrimSpace(s)) if s == "" || !strings.HasSuffix(s, target) { continue } if _, dup := seen[s]; dup { continue } seen[s] = struct{}{} _ = mctx.Store.Upsert(mctx.Ctx, s, func(h *store.Host) { store.AddDiscoveryMethod(h, "ct-stream") }) mctx.Bus.Publish(mctx.Ctx, eventbus.SubdomainDiscovered{ EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: s}, Subdomain: s, Method: "ct-stream", }) } select { case <-ticker.C: case <-mctx.Ctx.Done(): return nil } } return nil } func fetchRecentCerts(target string) []string { // crt.sh returns JSON with name_value fields; same as the v1 crtsh // source but we use a tighter query. q := "%." + target u := fmt.Sprintf("https://crt.sh/?q=%s&output=json", url.QueryEscape(q)) client := &http.Client{Timeout: 10 * time.Second} resp, err := client.Get(u) if err != nil { return nil } defer resp.Body.Close() var entries []struct { NameValue string `json:"name_value"` } if err := json.NewDecoder(resp.Body).Decode(&entries); err != nil { return nil } var out []string for _, e := range entries { for _, name := range strings.Split(e.NameValue, "\n") { name = strings.TrimPrefix(strings.TrimSpace(name), "*.") if name != "" { out = append(out, name) } } } return out }