feat: v2.0 full rewrite — event-driven pipeline, AI + Nuclei + proxy

Complete architectural overhaul. Replaces the v0.1 monolithic scanner with an event-driven pipeline of auto-registered modules. Foundation (internal/): - eventbus: typed pub/sub, 20 event types, race-safe, drop counter - module: registry with phase-based selection - store: thread-safe host store with per-host locks + deep-copy reads - pipeline: coordinator with phase barriers + panic recovery - config: 5 scan profiles + 3 AI tiers + YAML loader + auto-discovery Modules (26 auto-registered across 6 phases): - Discovery: passive (26 sources), bruteforce, recursive, AXFR, GitHub dorks, CT streaming, permutation, reverse DNS, vhost, ASN, supply chain (npm + PyPI) - Enrichment: HTTP probe + tech fingerprint + TLS appliance ID, ports - Analysis: security checks, takeover (110+ sigs), cloud, JavaScript, GraphQL, JWT, headers (OWASP), HTTP smuggling, AI cascade, Nuclei - Reporting: TXT/JSON/CSV writer + AI scan brief AI layer (internal/ai/ + internal/modules/ai/): - Three profiles: lean (16 GB), balanced (32 GB MoE), heavy (64 GB) - Six event-driven handlers: CVE, JS file, HTTP response, secret filter, multi-agent vuln enrichment, anomaly + executive report - Content-hash cache dedups Ollama calls across hosts - Auto-pull of missing models via /api/pull with streaming progress - End-of-scan AI SCAN BRIEF in terminal with top chains + next actions Nuclei compat layer (internal/nucleitpl/): - Executes ~13k community templates (HTTP subset) - Auto-download of nuclei-templates ZIP to ~/.god-eye/nuclei-templates - Scope filter rejects off-host templates (eliminates OSINT FPs) Operations: - Interactive wizard (internal/wizard/) — zero-flag launch - LivePrinter (internal/tui/) — colorized event stream - Diff engine + scheduler (internal/diff, internal/scheduler) for continuous ASM monitoring with webhook alerts - Proxy support (internal/proxyconf/): http / https / socks5 / socks5h + basic auth Fixes #1 — native SOCKS5 / Tor compatibility via --proxy flag. 185 unit tests across 15 packages, all race-detector clean.
2026-08-18 09:27:15 +02:00 · 2026-04-18 16:48:41 +02:00
parent f0bda8cc44
commit 3a4c230aa7
81 changed files with 15449 additions and 22 deletions
@@ -1,18 +1,39 @@
 package main

 import (
+	"context"
 	"fmt"
 	"os"
+	"os/signal"
+	"syscall"
+	"time"

 	"github.com/spf13/cobra"

 	"god-eye/internal/ai"
 	"god-eye/internal/config"
+	"god-eye/internal/diff"
+	"god-eye/internal/modules/all"
+	"god-eye/internal/nucleitpl"
 	"god-eye/internal/output"
+	"god-eye/internal/pipeline"
+	gohttp "god-eye/internal/http"
+	"god-eye/internal/proxyconf"
 	"god-eye/internal/scanner"
+	"god-eye/internal/scheduler"
+	"god-eye/internal/sources"
+	"god-eye/internal/store"
+	"god-eye/internal/tui"
 	"god-eye/internal/validator"
+	"god-eye/internal/wizard"
 )

+var _ = diff.Compute // ensure diff import is kept in the dependency graph
+
+// rootCmdRef is set by main() so helpers can query which flags cobra saw
+// explicitly on the command line (via Flags().Changed).
+var rootCmdRef *cobra.Command
+
 func main() {
 	var cfg config.Config

@@ -33,6 +54,20 @@ Examples:
  god-eye -d example.com --stealth moderate Moderate stealth (evasion mode)
  god-eye -d example.com --stealth paranoid Maximum stealth (very slow)`,
 		Run: func(cmd *cobra.Command, args []string) {
+			// If no target given and stdin is a TTY, launch the interactive wizard.
+			// Explicit --wizard also triggers it even with a target present (user
+			// wants to review defaults).
+			if (cfg.Domain == "" && wizard.IsInteractive()) || cfg.Wizard {
+				if err := runWizard(&cfg); err != nil {
+					if err == wizard.ErrCancelled {
+						fmt.Println(output.Yellow("cancelled."))
+						os.Exit(130)
+					}
+					fmt.Println(output.Red("[-]"), "wizard:", err)
+					os.Exit(1)
+				}
+			}
+
 			if cfg.Domain == "" {
 				fmt.Println(output.Red("[-]"), "Domain is required. Use -d flag.")
 				cmd.Help()
@@ -58,6 +93,27 @@ Examples:
 				fmt.Println(output.Red("[-]"), "Invalid resolvers:", err.Error())
 				os.Exit(1)
 			}
+			if err := proxyconf.Validate(cfg.Proxy); err != nil {
+				fmt.Println(output.Red("[-]"), "Invalid --proxy:", err.Error())
+				os.Exit(1)
+			}
+			// Propagate proxy config to every HTTP client before anything
+			// else spins up. This must happen after validation and before
+			// the pipeline/scanner starts.
+			if cfg.Proxy != "" {
+				if err := gohttp.SetProxy(cfg.Proxy); err != nil {
+					fmt.Println(output.Red("[-]"), "proxy (http factory):", err.Error())
+					os.Exit(1)
+				}
+				if err := sources.SetProxy(cfg.Proxy); err != nil {
+					fmt.Println(output.Red("[-]"), "proxy (sources):", err.Error())
+					os.Exit(1)
+				}
+				if !cfg.Silent {
+					fmt.Printf("%s Routing HTTP through %s\n",
+						output.BoldCyan("⛓"), output.BoldWhite(proxyconf.Humanize(cfg.Proxy)))
+				}
+			}
 			if err := validator.ValidateConcurrency(cfg.Concurrency); err != nil {
 				fmt.Println(output.Red("[-]"), "Invalid concurrency:", err.Error())
 				os.Exit(1)
@@ -111,6 +167,10 @@ Examples:
 				fmt.Println()
 			}

+			if cfg.UsePipeline {
+				runPipeline(cfg)
+				return
+			}
 			scanner.Run(cfg)
 		},
 	}
@@ -135,8 +195,8 @@ Examples:
 	// AI flags
 	rootCmd.Flags().BoolVar(&cfg.EnableAI, "enable-ai", false, "Enable AI-powered analysis with Ollama (includes CVE search)")
 	rootCmd.Flags().StringVar(&cfg.AIUrl, "ai-url", "http://localhost:11434", "Ollama API URL")
-	rootCmd.Flags().StringVar(&cfg.AIFastModel, "ai-fast-model", "deepseek-r1:1.5b", "Fast triage model")
-	rootCmd.Flags().StringVar(&cfg.AIDeepModel, "ai-deep-model", "qwen2.5-coder:7b", "Deep analysis model (supports function calling)")
+	rootCmd.Flags().StringVar(&cfg.AIFastModel, "ai-fast-model", "qwen3:1.7b", "Fast triage model (Ollama tag)")
+	rootCmd.Flags().StringVar(&cfg.AIDeepModel, "ai-deep-model", "qwen2.5-coder:14b", "Deep analysis model (Ollama tag, supports function calling)")
 	rootCmd.Flags().BoolVar(&cfg.AICascade, "ai-cascade", true, "Use cascade (fast triage + deep analysis)")
 	rootCmd.Flags().BoolVar(&cfg.AIDeepAnalysis, "ai-deep", false, "Enable deep AI analysis on all findings")
 	rootCmd.Flags().BoolVar(&cfg.MultiAgent, "multi-agent", false, "Enable multi-agent orchestration (8 specialized AI agents)")
@@ -144,6 +204,27 @@ Examples:
 	// Stealth flags
 	rootCmd.Flags().StringVar(&cfg.StealthMode, "stealth", "", "Stealth mode: light, moderate, aggressive, paranoid (reduces detection)")

+	// v2 pipeline flags
+	rootCmd.Flags().BoolVar(&cfg.UsePipeline, "pipeline", false, "Use v2 event-driven pipeline (experimental, parity with v1 verified by F0.7)")
+	rootCmd.Flags().BoolVar(&cfg.Wizard, "wizard", false, "Force the interactive setup wizard even when -d is set")
+	rootCmd.Flags().StringVar(&cfg.Profile, "profile", "", "Apply named scan profile (bugbounty, pentest, asm-continuous, stealth-max, quick)")
+	rootCmd.Flags().StringVar(&cfg.ConfigFile, "config", "", "Path to YAML config file (overrides auto-discovery)")
+
+	// Stash the rootCmd in a package var so runPipeline can check which
+	// flags the user set explicitly (cobra is the only thing that knows).
+	rootCmdRef = rootCmd
+	rootCmd.Flags().BoolVar(&cfg.Live, "live", false, "Stream colorized scan events live to the terminal (v2 only)")
+	rootCmd.Flags().IntVar(&cfg.LiveVerbosity, "live-verbosity", 1, "Live view verbosity: 0=findings-only, 1=normal, 2=noisy")
+	rootCmd.Flags().StringVar(&cfg.AIProfile, "ai-profile", "", "AI tier: lean (16GB), balanced (32GB), heavy/max (64GB+). Overrides --ai-fast-model/--ai-deep-model unless those are also set explicitly.")
+	rootCmd.Flags().BoolVar(&cfg.AIVerbose, "ai-verbose", false, "Log every Ollama query (model, prompt/response size, duration) to stderr")
+	rootCmd.Flags().BoolVar(&cfg.AutoPullModels, "ai-auto-pull", true, "Auto-download missing Ollama models before the scan starts")
+	rootCmd.Flags().BoolVar(&cfg.NucleiScan, "nuclei", false, "Run Nuclei-format YAML templates against every probed host")
+	rootCmd.Flags().StringVar(&cfg.NucleiTemplates, "nuclei-templates", "", "Path to Nuclei templates directory (default: $NUCLEI_TEMPLATES, then ~/nuclei-templates, then ~/.god-eye/nuclei-templates)")
+	rootCmd.Flags().BoolVar(&cfg.NucleiAutoDownload, "nuclei-auto-download", true, "Auto-download nuclei-templates ZIP from GitHub when no local dir is found")
+	rootCmd.Flags().StringVar(&cfg.Proxy, "proxy", "", "Route outbound HTTP through a proxy. Supported: http://host:port, https://host:port, socks5://host:port, socks5h://host:port (Tor). Basic auth via http://user:pass@host.")
+	rootCmd.Flags().DurationVar(&cfg.MonitorInterval, "monitor-interval", 0, "Run in continuous monitoring mode, re-scanning every N (e.g. 6h, 24h). Emits diffs.")
+	rootCmd.Flags().StringVar(&cfg.MonitorWebhook, "monitor-webhook", "", "Webhook URL to POST diff reports to in monitoring mode")
+
 	// Recursive discovery flags (enabled by default with --enable-ai)
 	rootCmd.Flags().BoolVar(&cfg.Recursive, "recursive", false, "Enable recursive subdomain discovery with pattern learning")
 	rootCmd.Flags().IntVar(&cfg.RecursiveDepth, "recursive-depth", 3, "Maximum recursion depth (1-5)")
@@ -224,7 +305,288 @@ This data is used for instant, offline CVE lookups during scans.`,
 	}
 	rootCmd.AddCommand(dbInfoCmd)

+	// nuclei-update: force refresh of the auto-downloaded Nuclei template cache
+	nucleiUpdateCmd := &cobra.Command{
+		Use:   "nuclei-update",
+		Short: "Download / refresh Nuclei YAML templates cache",
+		Long: `Fetches the official projectdiscovery/nuclei-templates ZIP archive
+and extracts every .yaml/.yml file into ~/.god-eye/nuclei-templates.
+
+Safe to re-run: existing templates are overwritten in-place. The cache
+is ~40MB on disk and ships thousands of detections that the compat
+layer executes when --nuclei is on.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			home, err := os.UserHomeDir()
+			if err != nil {
+				fmt.Println(output.Red("[-]"), "cannot find home dir:", err)
+				os.Exit(1)
+			}
+			dest := home + "/.god-eye/nuclei-templates"
+
+			fmt.Println(output.BoldCyan("📥 Refreshing Nuclei templates…"))
+			fmt.Printf("   %s %s\n", output.Dim("destination:"), output.BoldWhite(dest))
+
+			// Pull up the downloader. Inline import to keep the subcommand
+			// lightweight when not invoked.
+			dl := nucleitpl.NewDownloader()
+			dl.Verbose = true
+			if err := dl.Refresh(dest); err != nil {
+				fmt.Println(output.Red("[-]"), "refresh failed:", err)
+				os.Exit(1)
+			}
+			fmt.Println(output.Green("✓ Nuclei templates refreshed."))
+		},
+	}
+	rootCmd.AddCommand(nucleiUpdateCmd)
+
 	if err := rootCmd.Execute(); err != nil {
 		os.Exit(1)
 	}
 }
+
+// runPipeline is the v2 entry point. Registers every adapter module, loads
+// optional YAML + profile, and runs the event-driven pipeline under a
+// signal-aware context.
+func runPipeline(cfg config.Config) {
+	// Side-effect registration of all adapter modules (F0.6).
+	all.RegisterAll()
+
+	// Load YAML config if present. --config wins over auto-discovery.
+	path := cfg.ConfigFile
+	if path == "" {
+		path = config.FindConfigFile()
+	}
+	if path != "" {
+		if y, err := config.LoadYAML(path); err != nil {
+			fmt.Println(output.Red("[-]"), "config:", err.Error())
+			os.Exit(1)
+		} else if y != nil {
+			config.ApplyYAML(&cfg, y)
+		}
+	}
+
+	// Apply named scan profile if set.
+	if cfg.Profile != "" {
+		p, ok := config.ProfileByName(cfg.Profile)
+		if !ok {
+			fmt.Println(output.Red("[-]"), "unknown profile:", cfg.Profile)
+			os.Exit(1)
+		}
+		config.ApplyProfile(&cfg, p)
+		if !cfg.Silent {
+			fmt.Printf("%s Profile %s applied: %s\n", output.Green("✓"), output.BoldCyan(p.Name), output.Dim(p.Description))
+		}
+	}
+
+	// Apply AI tier profile (lean/balanced/heavy). Respects explicit
+	// --ai-fast-model / --ai-deep-model overrides.
+	if cfg.AIProfile != "" {
+		p, ok := config.AIProfileByName(cfg.AIProfile)
+		if !ok {
+			fmt.Println(output.Red("[-]"), "unknown AI profile:", cfg.AIProfile,
+				"— valid: lean, balanced, heavy")
+			os.Exit(1)
+		}
+		overrideFast := rootCmdRef != nil && rootCmdRef.Flags().Changed("ai-fast-model")
+		overrideDeep := rootCmdRef != nil && rootCmdRef.Flags().Changed("ai-deep-model")
+		config.ApplyAIProfile(&cfg, p, overrideFast, overrideDeep)
+		if !cfg.Silent {
+			fmt.Printf("%s AI profile %s: %s\n",
+				output.Green("✓"), output.BoldCyan(p.Name), output.Dim(p.Description))
+			fmt.Printf("  %s %s  %s %s\n",
+				output.Dim("triage:"), output.BoldWhite(cfg.AIFastModel),
+				output.Dim("deep:"), output.BoldWhite(cfg.AIDeepModel))
+		}
+	}
+
+	// Handle Ctrl-C gracefully. Set this up BEFORE the model-ensure step
+	// so long downloads can be interrupted cleanly.
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-sigCh
+		fmt.Println()
+		fmt.Println(output.Yellow("⚠  Interrupted — shutting down..."))
+		cancel()
+	}()
+
+	// Ensure Ollama models are present before scan starts.
+	if cfg.EnableAI && cfg.AutoPullModels {
+		if err := ensureAIModels(ctx, &cfg); err != nil {
+			if ctx.Err() == context.Canceled {
+				os.Exit(130)
+			}
+			fmt.Println(output.Red("[-]"), "AI setup:", err)
+			os.Exit(1)
+		}
+	}
+
+	// Continuous monitoring mode: run the scan on an interval, diff and alert.
+	if cfg.MonitorInterval > 0 {
+		runMonitor(ctx, cfg)
+		return
+	}
+
+	p, err := pipeline.New(&cfg, pipeline.Options{})
+	if err != nil {
+		fmt.Println(output.Red("[-]"), err)
+		os.Exit(1)
+	}
+
+	var live *tui.LivePrinter
+	if cfg.Live {
+		live = tui.NewLivePrinter(p.Bus(), cfg.LiveVerbosity)
+	}
+
+	if err := p.Run(ctx); err != nil {
+		if ctx.Err() == context.Canceled {
+			if live != nil {
+				live.Close()
+			}
+			os.Exit(130)
+		}
+		fmt.Println(output.Red("[!]"), "pipeline error:", err)
+		os.Exit(1)
+	}
+
+	if live != nil {
+		live.Close()
+	}
+}
+
+// runMonitor implements the asm-continuous mode: a single pipeline.Run
+// wrapped in scheduler.Scheduler that ticks at MonitorInterval, diffs
+// against the previous snapshot, and alerts on meaningful changes.
+func runMonitor(ctx context.Context, cfg config.Config) {
+	scan := func(scanCtx context.Context) ([]*store.Host, error) {
+		p, err := pipeline.New(&cfg, pipeline.Options{})
+		if err != nil {
+			return nil, err
+		}
+		if err := p.Run(scanCtx); err != nil {
+			return nil, err
+		}
+		return p.Store().All(scanCtx), nil
+	}
+
+	s := scheduler.New(cfg.Domain, cfg.MonitorInterval, scan)
+	s.AddAlerter(scheduler.StdoutAlerter{})
+	if cfg.MonitorWebhook != "" {
+		s.AddAlerter(scheduler.NewWebhookAlerter(cfg.MonitorWebhook))
+	}
+
+	fmt.Printf("%s Monitoring %s every %s — Ctrl-C to stop\n",
+		output.BoldGreen("▣"), output.BoldCyan(cfg.Domain), cfg.MonitorInterval)
+
+	if err := s.Start(ctx); err != nil && !errorIs(err, context.Canceled) {
+		fmt.Println(output.Red("[!]"), "monitor error:", err)
+		os.Exit(1)
+	}
+}
+
+// runWizard starts the interactive setup, then folds the user's choices
+// back into cfg. Forces pipeline mode (wizard is v2-only by design).
+func runWizard(cfg *config.Config) error {
+	choice, err := wizard.Run(context.Background(), wizard.Options{
+		In:        os.Stdin,
+		Out:       os.Stdout,
+		OllamaURL: cfg.AIUrl,
+	})
+	if err != nil {
+		return err
+	}
+
+	cfg.Domain = validator.SanitizeDomain(choice.Target)
+	cfg.UsePipeline = true
+	cfg.Live = choice.Live
+	cfg.LiveVerbosity = choice.LiveVerbosity
+	cfg.Output = choice.Output
+	if choice.Format != "" {
+		cfg.Format = choice.Format
+	}
+
+	// Scan profile name threads through --profile application (later).
+	if choice.ScanProfile != "" {
+		cfg.Profile = choice.ScanProfile
+	}
+
+	// ASM-continuous interval translates into a duration flag.
+	if choice.MonitorInterval != "" {
+		d, parseErr := time.ParseDuration(choice.MonitorInterval)
+		if parseErr != nil {
+			return fmt.Errorf("invalid interval %q: %w", choice.MonitorInterval, parseErr)
+		}
+		cfg.MonitorInterval = d
+	}
+
+	// AI tier.
+	if choice.AIProfile != "" {
+		cfg.EnableAI = true
+		cfg.AIProfile = choice.AIProfile
+		cfg.AIVerbose = choice.AIVerbose
+		cfg.AutoPullModels = choice.AIAutoPull
+	} else {
+		cfg.EnableAI = false
+	}
+
+	return nil
+}
+
+// ensureAIModels checks the Ollama server and downloads any missing models.
+// Prints progress when --ai-verbose is on. Fails open on unreachable
+// Ollama — the AI module itself will no-op gracefully.
+func ensureAIModels(ctx context.Context, cfg *config.Config) error {
+	e := ai.NewModelEnsurer(cfg.AIUrl)
+	e.Verbose = cfg.AIVerbose || cfg.Verbose
+	e.Writer = os.Stderr
+
+	if err := e.Reachable(ctx); err != nil {
+		if !cfg.Silent {
+			fmt.Println(output.Yellow("⚠  "), err.Error())
+			fmt.Println(output.Dim("   AI modules will no-op for this run. Start `ollama serve` to enable."))
+		}
+		return nil
+	}
+
+	models := []string{}
+	if cfg.AIFastModel != "" {
+		models = append(models, cfg.AIFastModel)
+	}
+	if cfg.AIDeepModel != "" && cfg.AIDeepModel != cfg.AIFastModel {
+		models = append(models, cfg.AIDeepModel)
+	}
+	if len(models) == 0 {
+		return nil
+	}
+
+	if !cfg.Silent {
+		fmt.Printf("%s Checking Ollama models: %s\n",
+			output.BoldCyan("⚙"), output.Dim(fmt.Sprintf("%v", models)))
+	}
+	if err := e.EnsureAll(ctx, models); err != nil {
+		return err
+	}
+	if !cfg.Silent {
+		fmt.Printf("%s Models ready\n", output.Green("✓"))
+	}
+	return nil
+}
+
+// errorIs is a thin wrapper for errors.Is that only pulls errors into
+// main when needed.
+func errorIs(err, target error) bool {
+	for err != nil {
+		if err == target {
+			return true
+		}
+		type unwrapper interface{ Unwrap() error }
+		u, ok := err.(unwrapper)
+		if !ok {
+			return false
+		}
+		err = u.Unwrap()
+	}
+	return false
+}