feat: v2.0 full rewrite — event-driven pipeline, AI + Nuclei + proxy

Complete architectural overhaul. Replaces the v0.1 monolithic scanner with an event-driven pipeline of auto-registered modules. Foundation (internal/): - eventbus: typed pub/sub, 20 event types, race-safe, drop counter - module: registry with phase-based selection - store: thread-safe host store with per-host locks + deep-copy reads - pipeline: coordinator with phase barriers + panic recovery - config: 5 scan profiles + 3 AI tiers + YAML loader + auto-discovery Modules (26 auto-registered across 6 phases): - Discovery: passive (26 sources), bruteforce, recursive, AXFR, GitHub dorks, CT streaming, permutation, reverse DNS, vhost, ASN, supply chain (npm + PyPI) - Enrichment: HTTP probe + tech fingerprint + TLS appliance ID, ports - Analysis: security checks, takeover (110+ sigs), cloud, JavaScript, GraphQL, JWT, headers (OWASP), HTTP smuggling, AI cascade, Nuclei - Reporting: TXT/JSON/CSV writer + AI scan brief AI layer (internal/ai/ + internal/modules/ai/): - Three profiles: lean (16 GB), balanced (32 GB MoE), heavy (64 GB) - Six event-driven handlers: CVE, JS file, HTTP response, secret filter, multi-agent vuln enrichment, anomaly + executive report - Content-hash cache dedups Ollama calls across hosts - Auto-pull of missing models via /api/pull with streaming progress - End-of-scan AI SCAN BRIEF in terminal with top chains + next actions Nuclei compat layer (internal/nucleitpl/): - Executes ~13k community templates (HTTP subset) - Auto-download of nuclei-templates ZIP to ~/.god-eye/nuclei-templates - Scope filter rejects off-host templates (eliminates OSINT FPs) Operations: - Interactive wizard (internal/wizard/) — zero-flag launch - LivePrinter (internal/tui/) — colorized event stream - Diff engine + scheduler (internal/diff, internal/scheduler) for continuous ASM monitoring with webhook alerts - Proxy support (internal/proxyconf/): http / https / socks5 / socks5h + basic auth Fixes #1 — native SOCKS5 / Tor compatibility via --proxy flag. 185 unit tests across 15 packages, all race-detector clean.
2026-05-16 13:39:10 +02:00 · 2026-04-18 16:48:41 +02:00
parent f0bda8cc44
commit 3a4c230aa7
81 changed files with 15449 additions and 22 deletions
@@ -0,0 +1,186 @@
+// Package javascript downloads JS files from probed hosts and scans them
+// for secrets with the v1 analyzer. Drains the store at start; also listens
+// for late HTTPProbed events.
+package javascript
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"time"
+
+	"god-eye/internal/eventbus"
+	gohttp "god-eye/internal/http"
+	"god-eye/internal/module"
+	"god-eye/internal/scanner"
+	"god-eye/internal/store"
+)
+
+// publicAPIDenylist covers well-known public/third-party APIs and font
+// services that the v1 regex scanner flags as "API Endpoint" but which
+// are never secrets. Matched case-insensitively as a substring.
+var publicAPIDenylist = []string{
+	"fonts.googleapis.com",
+	"fonts.gstatic.com",
+	"www.googleapis.com",
+	"content.googleapis.com",
+	"api.fastmail.com",
+	"api.forwardemail.net",
+	"cdn.jsdelivr.net",
+	"cdnjs.cloudflare.com",
+	"unpkg.com",
+}
+
+// uiStringDenylist covers common UI labels / warning strings that trip
+// the "Generic Password" regex but are clearly human-readable copy.
+var uiStringDenylist = []string{
+	"change password",
+	"update password",
+	"reset password",
+	"confirm password",
+	"forgot password",
+	"set-initial-password",
+	"change-password",
+	"this is a very common password",
+	"masterpassword",
+	"password",
+}
+
+// isSecretFalsePositive applies cheap deterministic heuristics to weed
+// out v1 regex noise. Does NOT replace AI triage (which is still the
+// preferred filter once the ai module is enabled) — it only suppresses
+// findings that are *definitely* not secrets.
+func isSecretFalsePositive(secret string) bool {
+	low := strings.ToLower(strings.TrimSpace(secret))
+	for _, s := range publicAPIDenylist {
+		if strings.Contains(low, s) {
+			return true
+		}
+	}
+	for _, s := range uiStringDenylist {
+		if strings.Contains(low, s) {
+			return true
+		}
+	}
+	// Very short matches (< 8 chars of unique content) are almost always
+	// labels, not credentials. The v1 regex already strips the "[Kind] "
+	// prefix before passing to us; anything under 8 chars is noise.
+	if len(low) > 0 && len(low) < 8 {
+		return true
+	}
+	return false
+}
+
+const ModuleName = "js.analyzer"
+
+type jsModule struct{}
+
+func Register() { module.Register(&jsModule{}) }
+
+func (*jsModule) Name() string                   { return ModuleName }
+func (*jsModule) Phase() module.Phase            { return module.PhaseAnalysis }
+func (*jsModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventHTTPProbed} }
+func (*jsModule) Produces() []eventbus.EventType {
+	return []eventbus.EventType{eventbus.EventJSFile, eventbus.EventSecret}
+}
+func (*jsModule) DefaultEnabled() bool { return true }
+
+func (*jsModule) Run(mctx module.Context) error {
+	timeout := mctx.Config.Int("timeout", 5)
+	client := gohttp.GetSharedClient(timeout)
+
+	processed := make(map[string]struct{})
+	var mu sync.Mutex
+	shouldProcess := func(host string) bool {
+		mu.Lock()
+		defer mu.Unlock()
+		if _, ok := processed[host]; ok {
+			return false
+		}
+		processed[host] = struct{}{}
+		return true
+	}
+
+	analyze := func(host string) {
+		if mctx.Ctx.Err() != nil {
+			return
+		}
+		jsFiles, secrets := scanner.AnalyzeJSFiles(host, client)
+		// Drop known-noise findings before they reach the store or bus.
+		filtered := secrets[:0]
+		for _, s := range secrets {
+			if isSecretFalsePositive(s) {
+				continue
+			}
+			filtered = append(filtered, s)
+		}
+		secrets = filtered
+		if len(jsFiles) == 0 && len(secrets) == 0 {
+			return
+		}
+		_ = mctx.Store.Upsert(mctx.Ctx, host, func(h *store.Host) {
+			for _, sec := range secrets {
+				h.Secrets = append(h.Secrets, store.Secret{
+					Kind:     "js-regex",
+					Match:    sec,
+					Severity: string(eventbus.SeverityHigh),
+					FoundAt:  time.Now(),
+				})
+			}
+		})
+		for _, jsf := range jsFiles {
+			mctx.Bus.Publish(mctx.Ctx, eventbus.JSFileDiscovered{
+				EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: host},
+				URL:       jsf,
+				Host:      host,
+			})
+		}
+		for _, s := range secrets {
+			mctx.Bus.Publish(mctx.Ctx, eventbus.SecretFound{
+				EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: host},
+				Kind:      "js-regex",
+				Match:     s,
+				Location:  "js-file",
+				Severity:  eventbus.SeverityHigh,
+			})
+		}
+	}
+
+	var wg sync.WaitGroup
+
+	// Drain: every probed host (StatusCode > 0).
+	for _, h := range mctx.Store.All(mctx.Ctx) {
+		if h == nil || h.StatusCode == 0 {
+			continue
+		}
+		if !shouldProcess(h.Subdomain) {
+			continue
+		}
+		host := h.Subdomain
+		wg.Add(1)
+		go func() { defer wg.Done(); analyze(host) }()
+	}
+
+	// Late events.
+	sub := mctx.Bus.Subscribe(eventbus.EventHTTPProbed, func(_ context.Context, e eventbus.Event) {
+		ev, ok := e.(eventbus.HTTPProbed)
+		if !ok || ev.StatusCode == 0 {
+			return
+		}
+		host := ev.Meta().Target
+		if !shouldProcess(host) {
+			return
+		}
+		wg.Add(1)
+		go func() { defer wg.Done(); analyze(host) }()
+	})
+	defer sub.Unsubscribe()
+
+	select {
+	case <-time.After(500 * time.Millisecond):
+	case <-mctx.Ctx.Done():
+	}
+
+	wg.Wait()
+	return nil
+}