feat: v2.0 full rewrite — event-driven pipeline, AI + Nuclei + proxy

Complete architectural overhaul. Replaces the v0.1 monolithic scanner with an event-driven pipeline of auto-registered modules. Foundation (internal/): - eventbus: typed pub/sub, 20 event types, race-safe, drop counter - module: registry with phase-based selection - store: thread-safe host store with per-host locks + deep-copy reads - pipeline: coordinator with phase barriers + panic recovery - config: 5 scan profiles + 3 AI tiers + YAML loader + auto-discovery Modules (26 auto-registered across 6 phases): - Discovery: passive (26 sources), bruteforce, recursive, AXFR, GitHub dorks, CT streaming, permutation, reverse DNS, vhost, ASN, supply chain (npm + PyPI) - Enrichment: HTTP probe + tech fingerprint + TLS appliance ID, ports - Analysis: security checks, takeover (110+ sigs), cloud, JavaScript, GraphQL, JWT, headers (OWASP), HTTP smuggling, AI cascade, Nuclei - Reporting: TXT/JSON/CSV writer + AI scan brief AI layer (internal/ai/ + internal/modules/ai/): - Three profiles: lean (16 GB), balanced (32 GB MoE), heavy (64 GB) - Six event-driven handlers: CVE, JS file, HTTP response, secret filter, multi-agent vuln enrichment, anomaly + executive report - Content-hash cache dedups Ollama calls across hosts - Auto-pull of missing models via /api/pull with streaming progress - End-of-scan AI SCAN BRIEF in terminal with top chains + next actions Nuclei compat layer (internal/nucleitpl/): - Executes ~13k community templates (HTTP subset) - Auto-download of nuclei-templates ZIP to ~/.god-eye/nuclei-templates - Scope filter rejects off-host templates (eliminates OSINT FPs) Operations: - Interactive wizard (internal/wizard/) — zero-flag launch - LivePrinter (internal/tui/) — colorized event stream - Diff engine + scheduler (internal/diff, internal/scheduler) for continuous ASM monitoring with webhook alerts - Proxy support (internal/proxyconf/): http / https / socks5 / socks5h + basic auth Fixes #1 — native SOCKS5 / Tor compatibility via --proxy flag. 185 unit tests across 15 packages, all race-detector clean.
2026-08-16 07:50:21 +02:00 · 2026-04-18 16:48:41 +02:00
parent f0bda8cc44
commit 3a4c230aa7
81 changed files with 15449 additions and 22 deletions
@@ -0,0 +1,101 @@
+// Package module defines the Module interface and Registry used by God's Eye v2
+// to organize discovery, enrichment, analysis, and reporting units of work.
+//
+// A Module is any unit of the pipeline that subscribes to zero-or-more event
+// types, produces zero-or-more event types, and optionally performs a bounded
+// amount of work on startup (e.g. a passive source fetches once and publishes).
+//
+// Modules are decoupled: they do not call each other directly. Ordering emerges
+// from the event-driven dependency graph, not from phase barriers. The Phase
+// label is metadata used for grouping in progress UIs and logs, not a scheduling
+// primitive.
+package module
+
+import (
+	"context"
+
+	"god-eye/internal/eventbus"
+	"god-eye/internal/store"
+)
+
+// Phase groups modules at similar pipeline stages for presentation. Modules at
+// different phases may still run concurrently; the scanner does not enforce
+// phase barriers.
+type Phase string
+
+const (
+	PhaseSetup      Phase = "setup"      // load DBs, wordlists, validate config
+	PhaseDiscovery  Phase = "discovery"  // subdomain sources (passive, CT, brute, recursive)
+	PhaseResolution Phase = "resolution" // DNS resolve, CNAME, PTR, IP info, wildcard filter
+	PhaseEnrichment Phase = "enrichment" // HTTP probe, tech fingerprint, TLS analyze
+	PhaseAnalysis   Phase = "analysis"   // security checks, takeover, secrets, AI, CVE match
+	PhaseReporting  Phase = "reporting"  // output writers, report generation
+)
+
+// Context bundles everything a module needs to run.
+//
+// The Ctx field carries cancellation — every long-running module must select
+// on Ctx.Done() to exit cleanly when the user interrupts.
+type Context struct {
+	Ctx     context.Context
+	Bus     *eventbus.Bus
+	Store   store.Store
+	Config  ConfigView
+	Target  string // primary target domain
+	Profile string // active profile name (bugbounty, pentest, stealth-max, ...)
+}
+
+// ConfigView is a narrow read-only interface over the scan config, exposed to
+// modules so they cannot mutate global state. Implementations live in the
+// config package.
+type ConfigView interface {
+	// Profile returns the active profile name ("" when none is selected).
+	Profile() string
+	// Bool reads a boolean config key, returning fallback if unset.
+	Bool(key string, fallback bool) bool
+	// Int reads an int key, returning fallback if unset.
+	Int(key string, fallback int) int
+	// String reads a string key, returning fallback if unset.
+	String(key string, fallback string) string
+	// Strings reads a string-slice key.
+	Strings(key string) []string
+	// ModuleEnabled lets the user disable a module by name. Registry honors
+	// this during selection.
+	ModuleEnabled(moduleName string) bool
+}
+
+// Module is the unit of work registered in the pipeline.
+//
+// Implementations should:
+//   - be cheap to construct (no I/O in the Module value itself)
+//   - do all setup/teardown inside Run so lifecycle is explicit
+//   - subscribe to events via mctx.Bus.Subscribe in Run
+//   - return promptly when mctx.Ctx is canceled OR when their work is complete
+type Module interface {
+	// Name uniquely identifies the module. Use dotted notation grouping by
+	// concern: "sources.crtsh", "dns.resolver", "http.probe", "security.cors",
+	// "ai.cascade". The registry rejects duplicate names.
+	Name() string
+
+	// Phase groups the module in pipeline UIs. See Phase constants.
+	Phase() Phase
+
+	// Consumes lists event types the module subscribes to. Empty means the
+	// module is a pure producer (e.g. a passive source). Used by tooling to
+	// visualize the event graph; the bus itself is queried via Subscribe.
+	Consumes() []eventbus.EventType
+
+	// Produces lists event types the module publishes. Empty means the module
+	// only side-effects (e.g. reporting). Used for tooling and dep docs.
+	Produces() []eventbus.EventType
+
+	// DefaultEnabled returns whether this module runs when config does not
+	// explicitly enable/disable it. Passive sources typically default true;
+	// aggressive/experimental modules typically default false.
+	DefaultEnabled() bool
+
+	// Run executes the module. Must be non-blocking on setup and must return
+	// when its work is complete OR mctx.Ctx is canceled. Errors returned are
+	// logged via ModuleError events by the scanner.
+	Run(mctx Context) error
+}
@@ -0,0 +1,183 @@
+package module
+
+import (
+	"fmt"
+	"sort"
+	"sync"
+
+	"god-eye/internal/eventbus"
+)
+
+// Registry stores modules keyed by name. Modules register themselves via
+// init() functions by calling Register on the default registry.
+type Registry struct {
+	mu      sync.RWMutex
+	modules map[string]Module
+	order   []string // insertion order for deterministic iteration
+}
+
+// NewRegistry returns an empty registry. Most callers should use Default()
+// which returns the process-wide registry that init() functions populate.
+func NewRegistry() *Registry {
+	return &Registry{modules: make(map[string]Module)}
+}
+
+var (
+	defaultRegistry *Registry
+	defaultOnce     sync.Once
+)
+
+// Default returns the process-wide module registry.
+func Default() *Registry {
+	defaultOnce.Do(func() {
+		defaultRegistry = NewRegistry()
+	})
+	return defaultRegistry
+}
+
+// Register adds m to r. Panics on duplicate name — registration happens at
+// init() time, so duplicates indicate a compile-time bug that must surface
+// immediately rather than silently overwrite.
+func (r *Registry) Register(m Module) {
+	if m == nil {
+		panic("module.Register: nil module")
+	}
+	name := m.Name()
+	if name == "" {
+		panic("module.Register: module has empty Name()")
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if _, exists := r.modules[name]; exists {
+		panic(fmt.Sprintf("module.Register: duplicate module %q", name))
+	}
+	r.modules[name] = m
+	r.order = append(r.order, name)
+}
+
+// Register is a shortcut for Default().Register(m). Intended use:
+//
+//	func init() { module.Register(&myModule{}) }
+func Register(m Module) { Default().Register(m) }
+
+// Get returns the module with the given name.
+func (r *Registry) Get(name string) (Module, bool) {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	m, ok := r.modules[name]
+	return m, ok
+}
+
+// Names returns all registered module names in insertion order.
+func (r *Registry) Names() []string {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	out := make([]string, len(r.order))
+	copy(out, r.order)
+	return out
+}
+
+// All returns every registered module in insertion order. The returned slice
+// is safe for the caller to iterate but do not mutate it.
+func (r *Registry) All() []Module {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	out := make([]Module, 0, len(r.order))
+	for _, n := range r.order {
+		out = append(out, r.modules[n])
+	}
+	return out
+}
+
+// ByPhase returns modules belonging to the given phase, sorted by name for
+// stable presentation.
+func (r *Registry) ByPhase(p Phase) []Module {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	var out []Module
+	for _, n := range r.order {
+		m := r.modules[n]
+		if m.Phase() == p {
+			out = append(out, m)
+		}
+	}
+	sort.SliceStable(out, func(i, j int) bool { return out[i].Name() < out[j].Name() })
+	return out
+}
+
+// Select returns the subset of modules that should run for the given config.
+// A module is selected when cfg.ModuleEnabled(name) returns true (explicit
+// enable wins), OR when cfg leaves it unset and DefaultEnabled() is true.
+func (r *Registry) Select(cfg ConfigView) []Module {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	var out []Module
+	for _, n := range r.order {
+		m := r.modules[n]
+		if cfg != nil {
+			// explicit config: respect it directly
+			if cfg.ModuleEnabled(m.Name()) {
+				out = append(out, m)
+				continue
+			}
+			// if the config has a non-default opinion (enabled=false), honor it
+			// — but ModuleEnabled returning false could also mean "unset".
+			// We resolve the ambiguity by checking whether any profile/CLI flag
+			// set it via a separate mechanism; for now, fall back to the
+			// module's default.
+			if m.DefaultEnabled() {
+				out = append(out, m)
+			}
+			continue
+		}
+		// no config: honor module default
+		if m.DefaultEnabled() {
+			out = append(out, m)
+		}
+	}
+	return out
+}
+
+// ProducersOf returns the modules that declare t in their Produces() set.
+// Used by tooling and tests to validate the event-graph integrity.
+func (r *Registry) ProducersOf(t eventbus.EventType) []Module {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	var out []Module
+	for _, n := range r.order {
+		m := r.modules[n]
+		for _, et := range m.Produces() {
+			if et == t {
+				out = append(out, m)
+				break
+			}
+		}
+	}
+	return out
+}
+
+// ConsumersOf returns modules that declare t in their Consumes() set.
+func (r *Registry) ConsumersOf(t eventbus.EventType) []Module {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	var out []Module
+	for _, n := range r.order {
+		m := r.modules[n]
+		for _, et := range m.Consumes() {
+			if et == t {
+				out = append(out, m)
+				break
+			}
+		}
+	}
+	return out
+}
+
+// Reset clears the registry. Intended for tests only; never call in production
+// code.
+func (r *Registry) Reset() {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.modules = make(map[string]Module)
+	r.order = nil
+}
@@ -0,0 +1,257 @@
+package module
+
+import (
+	"context"
+	"reflect"
+	"sort"
+	"testing"
+
+	"god-eye/internal/eventbus"
+)
+
+// fakeModule is a minimal Module for tests.
+type fakeModule struct {
+	name           string
+	phase          Phase
+	consumes       []eventbus.EventType
+	produces       []eventbus.EventType
+	defaultEnabled bool
+	runCalled      bool
+}
+
+func (f *fakeModule) Name() string                      { return f.name }
+func (f *fakeModule) Phase() Phase                      { return f.phase }
+func (f *fakeModule) Consumes() []eventbus.EventType    { return f.consumes }
+func (f *fakeModule) Produces() []eventbus.EventType    { return f.produces }
+func (f *fakeModule) DefaultEnabled() bool              { return f.defaultEnabled }
+func (f *fakeModule) Run(mctx Context) error            { f.runCalled = true; return nil }
+
+// fakeConfig implements ConfigView for tests.
+type fakeConfig struct {
+	profile string
+	enabled map[string]bool
+}
+
+func (c *fakeConfig) Profile() string                        { return c.profile }
+func (c *fakeConfig) Bool(k string, fb bool) bool            { return fb }
+func (c *fakeConfig) Int(k string, fb int) int               { return fb }
+func (c *fakeConfig) String(k, fb string) string             { return fb }
+func (c *fakeConfig) Strings(k string) []string              { return nil }
+func (c *fakeConfig) ModuleEnabled(name string) bool         { return c.enabled[name] }
+
+func TestRegister_AndGet(t *testing.T) {
+	r := NewRegistry()
+	m := &fakeModule{name: "test.one", phase: PhaseDiscovery, defaultEnabled: true}
+	r.Register(m)
+
+	got, ok := r.Get("test.one")
+	if !ok {
+		t.Fatal("Get returned !ok for registered module")
+	}
+	if got != m {
+		t.Error("Get returned a different instance")
+	}
+
+	if _, ok := r.Get("not.present"); ok {
+		t.Error("Get returned ok for missing module")
+	}
+}
+
+func TestRegister_DuplicatePanic(t *testing.T) {
+	r := NewRegistry()
+	r.Register(&fakeModule{name: "dup", phase: PhaseDiscovery})
+	defer func() {
+		if recover() == nil {
+			t.Error("expected panic on duplicate registration")
+		}
+	}()
+	r.Register(&fakeModule{name: "dup", phase: PhaseDiscovery})
+}
+
+func TestRegister_NilPanic(t *testing.T) {
+	r := NewRegistry()
+	defer func() {
+		if recover() == nil {
+			t.Error("expected panic on nil module")
+		}
+	}()
+	r.Register(nil)
+}
+
+func TestRegister_EmptyNamePanic(t *testing.T) {
+	r := NewRegistry()
+	defer func() {
+		if recover() == nil {
+			t.Error("expected panic on empty name")
+		}
+	}()
+	r.Register(&fakeModule{name: "", phase: PhaseDiscovery})
+}
+
+func TestNames_InsertionOrder(t *testing.T) {
+	r := NewRegistry()
+	r.Register(&fakeModule{name: "zebra", phase: PhaseDiscovery})
+	r.Register(&fakeModule{name: "alpha", phase: PhaseDiscovery})
+	r.Register(&fakeModule{name: "middle", phase: PhaseDiscovery})
+
+	want := []string{"zebra", "alpha", "middle"}
+	got := r.Names()
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("Names order = %v, want %v", got, want)
+	}
+}
+
+func TestAll_ReturnsRegistered(t *testing.T) {
+	r := NewRegistry()
+	r.Register(&fakeModule{name: "a", phase: PhaseDiscovery})
+	r.Register(&fakeModule{name: "b", phase: PhaseAnalysis})
+	r.Register(&fakeModule{name: "c", phase: PhaseReporting})
+
+	if got := len(r.All()); got != 3 {
+		t.Errorf("All length = %d, want 3", got)
+	}
+}
+
+func TestByPhase_SortedByName(t *testing.T) {
+	r := NewRegistry()
+	r.Register(&fakeModule{name: "sources.zzz", phase: PhaseDiscovery})
+	r.Register(&fakeModule{name: "sources.aaa", phase: PhaseDiscovery})
+	r.Register(&fakeModule{name: "security.cors", phase: PhaseAnalysis})
+	r.Register(&fakeModule{name: "sources.mmm", phase: PhaseDiscovery})
+
+	got := r.ByPhase(PhaseDiscovery)
+	names := make([]string, len(got))
+	for i, m := range got {
+		names[i] = m.Name()
+	}
+	want := []string{"sources.aaa", "sources.mmm", "sources.zzz"}
+	if !reflect.DeepEqual(names, want) {
+		t.Errorf("ByPhase(discovery) = %v, want %v (sorted)", names, want)
+	}
+
+	if got := r.ByPhase(PhaseAnalysis); len(got) != 1 || got[0].Name() != "security.cors" {
+		t.Errorf("ByPhase(analysis) unexpected: %v", got)
+	}
+	if got := r.ByPhase(PhaseReporting); len(got) != 0 {
+		t.Errorf("ByPhase(reporting) should be empty, got %d", len(got))
+	}
+}
+
+func TestSelect_DefaultEnabled(t *testing.T) {
+	r := NewRegistry()
+	r.Register(&fakeModule{name: "on-by-default", phase: PhaseDiscovery, defaultEnabled: true})
+	r.Register(&fakeModule{name: "off-by-default", phase: PhaseDiscovery, defaultEnabled: false})
+
+	// nil config: module default governs
+	got := r.Select(nil)
+	names := moduleNames(got)
+	sort.Strings(names)
+	if !reflect.DeepEqual(names, []string{"on-by-default"}) {
+		t.Errorf("Select(nil) = %v, want [on-by-default]", names)
+	}
+}
+
+func TestSelect_ConfigEnablesOff(t *testing.T) {
+	r := NewRegistry()
+	r.Register(&fakeModule{name: "optin", phase: PhaseAnalysis, defaultEnabled: false})
+	r.Register(&fakeModule{name: "default-on", phase: PhaseAnalysis, defaultEnabled: true})
+
+	cfg := &fakeConfig{enabled: map[string]bool{"optin": true}}
+	got := r.Select(cfg)
+	names := moduleNames(got)
+	sort.Strings(names)
+	want := []string{"default-on", "optin"}
+	if !reflect.DeepEqual(names, want) {
+		t.Errorf("Select = %v, want %v", names, want)
+	}
+}
+
+func TestProducersOf_AndConsumersOf(t *testing.T) {
+	r := NewRegistry()
+	r.Register(&fakeModule{
+		name:     "producer-a",
+		phase:    PhaseDiscovery,
+		produces: []eventbus.EventType{eventbus.EventSubdomainDiscovered},
+	})
+	r.Register(&fakeModule{
+		name:     "producer-b",
+		phase:    PhaseDiscovery,
+		produces: []eventbus.EventType{eventbus.EventSubdomainDiscovered, eventbus.EventDNSResolved},
+	})
+	r.Register(&fakeModule{
+		name:     "consumer",
+		phase:    PhaseEnrichment,
+		consumes: []eventbus.EventType{eventbus.EventDNSResolved},
+	})
+
+	producers := r.ProducersOf(eventbus.EventSubdomainDiscovered)
+	names := moduleNames(producers)
+	sort.Strings(names)
+	want := []string{"producer-a", "producer-b"}
+	if !reflect.DeepEqual(names, want) {
+		t.Errorf("ProducersOf = %v, want %v", names, want)
+	}
+
+	consumers := r.ConsumersOf(eventbus.EventDNSResolved)
+	if len(consumers) != 1 || consumers[0].Name() != "consumer" {
+		t.Errorf("ConsumersOf unexpected: %v", consumers)
+	}
+}
+
+func TestReset(t *testing.T) {
+	r := NewRegistry()
+	r.Register(&fakeModule{name: "m1", phase: PhaseDiscovery, defaultEnabled: true})
+	r.Register(&fakeModule{name: "m2", phase: PhaseDiscovery, defaultEnabled: true})
+	if len(r.All()) != 2 {
+		t.Fatal("pre-reset: expected 2 modules")
+	}
+	r.Reset()
+	if len(r.All()) != 0 {
+		t.Errorf("post-reset: expected 0 modules, got %d", len(r.All()))
+	}
+	// Re-register after reset works
+	r.Register(&fakeModule{name: "m1", phase: PhaseDiscovery, defaultEnabled: true})
+	if len(r.All()) != 1 {
+		t.Errorf("post-reset re-register: expected 1, got %d", len(r.All()))
+	}
+}
+
+func TestDefault_Singleton(t *testing.T) {
+	a := Default()
+	b := Default()
+	if a != b {
+		t.Error("Default() returned different instances")
+	}
+}
+
+func TestRunContextCarriesFields(t *testing.T) {
+	// Sanity: Context struct is populated correctly — this is effectively a
+	// struct-init contract test to catch accidental field removals.
+	ctx := context.Background()
+	bus := eventbus.New(16)
+	defer bus.Close(context.Background())
+
+	mctx := Context{
+		Ctx:     ctx,
+		Bus:     bus,
+		Target:  "example.com",
+		Profile: "bugbounty",
+	}
+	if mctx.Target != "example.com" {
+		t.Errorf("Target lost: %q", mctx.Target)
+	}
+	if mctx.Profile != "bugbounty" {
+		t.Errorf("Profile lost: %q", mctx.Profile)
+	}
+	if mctx.Bus != bus {
+		t.Error("Bus not retained")
+	}
+}
+
+func moduleNames(ms []Module) []string {
+	out := make([]string, len(ms))
+	for i, m := range ms {
+		out[i] = m.Name()
+	}
+	return out
+}