mirror of
https://github.com/Vyntral/god-eye.git
synced 2026-08-16 10:00:33 +02:00
feat: v2.0 full rewrite — event-driven pipeline, AI + Nuclei + proxy
Complete architectural overhaul. Replaces the v0.1 monolithic scanner with an event-driven pipeline of auto-registered modules. Foundation (internal/): - eventbus: typed pub/sub, 20 event types, race-safe, drop counter - module: registry with phase-based selection - store: thread-safe host store with per-host locks + deep-copy reads - pipeline: coordinator with phase barriers + panic recovery - config: 5 scan profiles + 3 AI tiers + YAML loader + auto-discovery Modules (26 auto-registered across 6 phases): - Discovery: passive (26 sources), bruteforce, recursive, AXFR, GitHub dorks, CT streaming, permutation, reverse DNS, vhost, ASN, supply chain (npm + PyPI) - Enrichment: HTTP probe + tech fingerprint + TLS appliance ID, ports - Analysis: security checks, takeover (110+ sigs), cloud, JavaScript, GraphQL, JWT, headers (OWASP), HTTP smuggling, AI cascade, Nuclei - Reporting: TXT/JSON/CSV writer + AI scan brief AI layer (internal/ai/ + internal/modules/ai/): - Three profiles: lean (16 GB), balanced (32 GB MoE), heavy (64 GB) - Six event-driven handlers: CVE, JS file, HTTP response, secret filter, multi-agent vuln enrichment, anomaly + executive report - Content-hash cache dedups Ollama calls across hosts - Auto-pull of missing models via /api/pull with streaming progress - End-of-scan AI SCAN BRIEF in terminal with top chains + next actions Nuclei compat layer (internal/nucleitpl/): - Executes ~13k community templates (HTTP subset) - Auto-download of nuclei-templates ZIP to ~/.god-eye/nuclei-templates - Scope filter rejects off-host templates (eliminates OSINT FPs) Operations: - Interactive wizard (internal/wizard/) — zero-flag launch - LivePrinter (internal/tui/) — colorized event stream - Diff engine + scheduler (internal/diff, internal/scheduler) for continuous ASM monitoring with webhook alerts - Proxy support (internal/proxyconf/): http / https / socks5 / socks5h + basic auth Fixes #1 — native SOCKS5 / Tor compatibility via --proxy flag. 185 unit tests across 15 packages, all race-detector clean.
This commit is contained in:
@@ -0,0 +1,151 @@
|
||||
// Package passive is the Fase 0.6 adapter that wraps the v1 passive sources
|
||||
// (internal/sources) as a single Module. It fans out queries to all 20 public
|
||||
// sources in parallel and emits a SubdomainDiscovered event for each result.
|
||||
//
|
||||
// In Fase 1 (Discovery Supremacy) each source will become its own Module with
|
||||
// independent configuration, error reporting, and rate limiting. This
|
||||
// adapter preserves v1 behavior so we reach feature parity immediately.
|
||||
package passive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/sources"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
// ModuleName is the registry identifier.
|
||||
const ModuleName = "passive.v1-aggregate"
|
||||
|
||||
type passiveModule struct{}
|
||||
|
||||
// Register the module in the default registry. Callers import this package
|
||||
// for side effects via the modules meta-package (see internal/modules/all).
|
||||
func Register() { module.Register(&passiveModule{}) }
|
||||
|
||||
func (*passiveModule) Name() string { return ModuleName }
|
||||
func (*passiveModule) Phase() module.Phase { return module.PhaseDiscovery }
|
||||
|
||||
func (*passiveModule) Consumes() []eventbus.EventType { return nil }
|
||||
func (*passiveModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventSubdomainDiscovered, eventbus.EventModuleError}
|
||||
}
|
||||
|
||||
func (*passiveModule) DefaultEnabled() bool { return true }
|
||||
|
||||
// sourceList mirrors the v1 scanner.Run list. Order is preserved for stable
|
||||
// logging.
|
||||
var sourceList = []struct {
|
||||
name string
|
||||
fn func(string) ([]string, error)
|
||||
}{
|
||||
{"crt.sh", sources.FetchCrtsh},
|
||||
{"Certspotter", sources.FetchCertspotter},
|
||||
{"AlienVault", sources.FetchAlienVault},
|
||||
{"HackerTarget", sources.FetchHackerTarget},
|
||||
{"URLScan", sources.FetchURLScan},
|
||||
{"RapidDNS", sources.FetchRapidDNS},
|
||||
{"Anubis", sources.FetchAnubis},
|
||||
{"ThreatMiner", sources.FetchThreatMiner},
|
||||
{"DNSRepo", sources.FetchDNSRepo},
|
||||
{"SubdomainCenter", sources.FetchSubdomainCenter},
|
||||
{"Wayback", sources.FetchWayback},
|
||||
{"CommonCrawl", sources.FetchCommonCrawl},
|
||||
{"Sitedossier", sources.FetchSitedossier},
|
||||
{"Riddler", sources.FetchRiddler},
|
||||
{"Robtex", sources.FetchRobtex},
|
||||
{"DNSHistory", sources.FetchDNSHistory},
|
||||
{"ArchiveToday", sources.FetchArchiveToday},
|
||||
{"JLDC", sources.FetchJLDC},
|
||||
{"SynapsInt", sources.FetchSynapsInt},
|
||||
{"CensysFree", sources.FetchCensysFree},
|
||||
// v2.0 additions — free, no API key, fail-open. Dormant v1 sources
|
||||
// re-activated + 4 net-new endpoints.
|
||||
{"BufferOver", sources.FetchBufferOver}, // dormant v1
|
||||
{"DNSDumpster", sources.FetchDNSDumpster}, // dormant v1
|
||||
{"Omnisint", sources.FetchOmnisint}, // v2 new
|
||||
{"HudsonRock", sources.FetchHudsonRock}, // v2 new
|
||||
{"WebArchiveCDX", sources.FetchWebArchiveCDX}, // v2 new
|
||||
{"Digitorus", sources.FetchDigitorus}, // v2 new
|
||||
}
|
||||
|
||||
func (m *passiveModule) Run(mctx module.Context) error {
|
||||
target := mctx.Target
|
||||
if target == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
// Dedup across sources before emitting — the store will also dedup, but
|
||||
// emitting duplicates just burns bus bandwidth.
|
||||
seen := make(map[string]struct{})
|
||||
var seenMu sync.Mutex
|
||||
|
||||
for _, src := range sourceList {
|
||||
src := src
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
// Respect ctx cancellation between slow sources.
|
||||
if err := mctx.Ctx.Err(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
subs, err := src.fn(target)
|
||||
if err != nil {
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.ModuleError{
|
||||
EventMeta: eventbus.EventMeta{Source: ModuleName + ":" + src.name, Target: target},
|
||||
Module: ModuleName + ":" + src.name,
|
||||
Err: err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
for _, sub := range subs {
|
||||
sub = strings.ToLower(strings.TrimSpace(sub))
|
||||
if sub == "" {
|
||||
continue
|
||||
}
|
||||
if !strings.HasSuffix(sub, target) {
|
||||
continue
|
||||
}
|
||||
seenMu.Lock()
|
||||
if _, dup := seen[sub]; dup {
|
||||
seenMu.Unlock()
|
||||
continue
|
||||
}
|
||||
seen[sub] = struct{}{}
|
||||
seenMu.Unlock()
|
||||
|
||||
// Persist into the store so downstream resolution phases
|
||||
// can find the subdomain even if they subscribed too late
|
||||
// to receive the SubdomainDiscovered event.
|
||||
methodTag := "passive:" + src.name
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, sub, func(h *store.Host) {
|
||||
store.AddDiscoveryMethod(h, methodTag)
|
||||
})
|
||||
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.NewSubdomainDiscovered(
|
||||
ModuleName+":"+src.name,
|
||||
sub,
|
||||
methodTag,
|
||||
))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait for sources OR cancellation.
|
||||
done := make(chan struct{})
|
||||
go func() { wg.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
_ = context.Canceled // keep import
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user