mirror of
https://github.com/Vyntral/god-eye.git
synced 2026-06-10 00:03:55 +02:00
feat: v2.0 full rewrite — event-driven pipeline, AI + Nuclei + proxy
Complete architectural overhaul. Replaces the v0.1 monolithic scanner with an event-driven pipeline of auto-registered modules. Foundation (internal/): - eventbus: typed pub/sub, 20 event types, race-safe, drop counter - module: registry with phase-based selection - store: thread-safe host store with per-host locks + deep-copy reads - pipeline: coordinator with phase barriers + panic recovery - config: 5 scan profiles + 3 AI tiers + YAML loader + auto-discovery Modules (26 auto-registered across 6 phases): - Discovery: passive (26 sources), bruteforce, recursive, AXFR, GitHub dorks, CT streaming, permutation, reverse DNS, vhost, ASN, supply chain (npm + PyPI) - Enrichment: HTTP probe + tech fingerprint + TLS appliance ID, ports - Analysis: security checks, takeover (110+ sigs), cloud, JavaScript, GraphQL, JWT, headers (OWASP), HTTP smuggling, AI cascade, Nuclei - Reporting: TXT/JSON/CSV writer + AI scan brief AI layer (internal/ai/ + internal/modules/ai/): - Three profiles: lean (16 GB), balanced (32 GB MoE), heavy (64 GB) - Six event-driven handlers: CVE, JS file, HTTP response, secret filter, multi-agent vuln enrichment, anomaly + executive report - Content-hash cache dedups Ollama calls across hosts - Auto-pull of missing models via /api/pull with streaming progress - End-of-scan AI SCAN BRIEF in terminal with top chains + next actions Nuclei compat layer (internal/nucleitpl/): - Executes ~13k community templates (HTTP subset) - Auto-download of nuclei-templates ZIP to ~/.god-eye/nuclei-templates - Scope filter rejects off-host templates (eliminates OSINT FPs) Operations: - Interactive wizard (internal/wizard/) — zero-flag launch - LivePrinter (internal/tui/) — colorized event stream - Diff engine + scheduler (internal/diff, internal/scheduler) for continuous ASM monitoring with webhook alerts - Proxy support (internal/proxyconf/): http / https / socks5 / socks5h + basic auth Fixes #1 — native SOCKS5 / Tor compatibility via --proxy flag. 185 unit tests across 15 packages, all race-detector clean.
This commit is contained in:
+364
-2
@@ -1,18 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"god-eye/internal/ai"
|
||||
"god-eye/internal/config"
|
||||
"god-eye/internal/diff"
|
||||
"god-eye/internal/modules/all"
|
||||
"god-eye/internal/nucleitpl"
|
||||
"god-eye/internal/output"
|
||||
"god-eye/internal/pipeline"
|
||||
gohttp "god-eye/internal/http"
|
||||
"god-eye/internal/proxyconf"
|
||||
"god-eye/internal/scanner"
|
||||
"god-eye/internal/scheduler"
|
||||
"god-eye/internal/sources"
|
||||
"god-eye/internal/store"
|
||||
"god-eye/internal/tui"
|
||||
"god-eye/internal/validator"
|
||||
"god-eye/internal/wizard"
|
||||
)
|
||||
|
||||
var _ = diff.Compute // ensure diff import is kept in the dependency graph
|
||||
|
||||
// rootCmdRef is set by main() so helpers can query which flags cobra saw
|
||||
// explicitly on the command line (via Flags().Changed).
|
||||
var rootCmdRef *cobra.Command
|
||||
|
||||
func main() {
|
||||
var cfg config.Config
|
||||
|
||||
@@ -33,6 +54,20 @@ Examples:
|
||||
god-eye -d example.com --stealth moderate Moderate stealth (evasion mode)
|
||||
god-eye -d example.com --stealth paranoid Maximum stealth (very slow)`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
// If no target given and stdin is a TTY, launch the interactive wizard.
|
||||
// Explicit --wizard also triggers it even with a target present (user
|
||||
// wants to review defaults).
|
||||
if (cfg.Domain == "" && wizard.IsInteractive()) || cfg.Wizard {
|
||||
if err := runWizard(&cfg); err != nil {
|
||||
if err == wizard.ErrCancelled {
|
||||
fmt.Println(output.Yellow("cancelled."))
|
||||
os.Exit(130)
|
||||
}
|
||||
fmt.Println(output.Red("[-]"), "wizard:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.Domain == "" {
|
||||
fmt.Println(output.Red("[-]"), "Domain is required. Use -d flag.")
|
||||
cmd.Help()
|
||||
@@ -58,6 +93,27 @@ Examples:
|
||||
fmt.Println(output.Red("[-]"), "Invalid resolvers:", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := proxyconf.Validate(cfg.Proxy); err != nil {
|
||||
fmt.Println(output.Red("[-]"), "Invalid --proxy:", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
// Propagate proxy config to every HTTP client before anything
|
||||
// else spins up. This must happen after validation and before
|
||||
// the pipeline/scanner starts.
|
||||
if cfg.Proxy != "" {
|
||||
if err := gohttp.SetProxy(cfg.Proxy); err != nil {
|
||||
fmt.Println(output.Red("[-]"), "proxy (http factory):", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := sources.SetProxy(cfg.Proxy); err != nil {
|
||||
fmt.Println(output.Red("[-]"), "proxy (sources):", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
if !cfg.Silent {
|
||||
fmt.Printf("%s Routing HTTP through %s\n",
|
||||
output.BoldCyan("⛓"), output.BoldWhite(proxyconf.Humanize(cfg.Proxy)))
|
||||
}
|
||||
}
|
||||
if err := validator.ValidateConcurrency(cfg.Concurrency); err != nil {
|
||||
fmt.Println(output.Red("[-]"), "Invalid concurrency:", err.Error())
|
||||
os.Exit(1)
|
||||
@@ -111,6 +167,10 @@ Examples:
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
if cfg.UsePipeline {
|
||||
runPipeline(cfg)
|
||||
return
|
||||
}
|
||||
scanner.Run(cfg)
|
||||
},
|
||||
}
|
||||
@@ -135,8 +195,8 @@ Examples:
|
||||
// AI flags
|
||||
rootCmd.Flags().BoolVar(&cfg.EnableAI, "enable-ai", false, "Enable AI-powered analysis with Ollama (includes CVE search)")
|
||||
rootCmd.Flags().StringVar(&cfg.AIUrl, "ai-url", "http://localhost:11434", "Ollama API URL")
|
||||
rootCmd.Flags().StringVar(&cfg.AIFastModel, "ai-fast-model", "deepseek-r1:1.5b", "Fast triage model")
|
||||
rootCmd.Flags().StringVar(&cfg.AIDeepModel, "ai-deep-model", "qwen2.5-coder:7b", "Deep analysis model (supports function calling)")
|
||||
rootCmd.Flags().StringVar(&cfg.AIFastModel, "ai-fast-model", "qwen3:1.7b", "Fast triage model (Ollama tag)")
|
||||
rootCmd.Flags().StringVar(&cfg.AIDeepModel, "ai-deep-model", "qwen2.5-coder:14b", "Deep analysis model (Ollama tag, supports function calling)")
|
||||
rootCmd.Flags().BoolVar(&cfg.AICascade, "ai-cascade", true, "Use cascade (fast triage + deep analysis)")
|
||||
rootCmd.Flags().BoolVar(&cfg.AIDeepAnalysis, "ai-deep", false, "Enable deep AI analysis on all findings")
|
||||
rootCmd.Flags().BoolVar(&cfg.MultiAgent, "multi-agent", false, "Enable multi-agent orchestration (8 specialized AI agents)")
|
||||
@@ -144,6 +204,27 @@ Examples:
|
||||
// Stealth flags
|
||||
rootCmd.Flags().StringVar(&cfg.StealthMode, "stealth", "", "Stealth mode: light, moderate, aggressive, paranoid (reduces detection)")
|
||||
|
||||
// v2 pipeline flags
|
||||
rootCmd.Flags().BoolVar(&cfg.UsePipeline, "pipeline", false, "Use v2 event-driven pipeline (experimental, parity with v1 verified by F0.7)")
|
||||
rootCmd.Flags().BoolVar(&cfg.Wizard, "wizard", false, "Force the interactive setup wizard even when -d is set")
|
||||
rootCmd.Flags().StringVar(&cfg.Profile, "profile", "", "Apply named scan profile (bugbounty, pentest, asm-continuous, stealth-max, quick)")
|
||||
rootCmd.Flags().StringVar(&cfg.ConfigFile, "config", "", "Path to YAML config file (overrides auto-discovery)")
|
||||
|
||||
// Stash the rootCmd in a package var so runPipeline can check which
|
||||
// flags the user set explicitly (cobra is the only thing that knows).
|
||||
rootCmdRef = rootCmd
|
||||
rootCmd.Flags().BoolVar(&cfg.Live, "live", false, "Stream colorized scan events live to the terminal (v2 only)")
|
||||
rootCmd.Flags().IntVar(&cfg.LiveVerbosity, "live-verbosity", 1, "Live view verbosity: 0=findings-only, 1=normal, 2=noisy")
|
||||
rootCmd.Flags().StringVar(&cfg.AIProfile, "ai-profile", "", "AI tier: lean (16GB), balanced (32GB), heavy/max (64GB+). Overrides --ai-fast-model/--ai-deep-model unless those are also set explicitly.")
|
||||
rootCmd.Flags().BoolVar(&cfg.AIVerbose, "ai-verbose", false, "Log every Ollama query (model, prompt/response size, duration) to stderr")
|
||||
rootCmd.Flags().BoolVar(&cfg.AutoPullModels, "ai-auto-pull", true, "Auto-download missing Ollama models before the scan starts")
|
||||
rootCmd.Flags().BoolVar(&cfg.NucleiScan, "nuclei", false, "Run Nuclei-format YAML templates against every probed host")
|
||||
rootCmd.Flags().StringVar(&cfg.NucleiTemplates, "nuclei-templates", "", "Path to Nuclei templates directory (default: $NUCLEI_TEMPLATES, then ~/nuclei-templates, then ~/.god-eye/nuclei-templates)")
|
||||
rootCmd.Flags().BoolVar(&cfg.NucleiAutoDownload, "nuclei-auto-download", true, "Auto-download nuclei-templates ZIP from GitHub when no local dir is found")
|
||||
rootCmd.Flags().StringVar(&cfg.Proxy, "proxy", "", "Route outbound HTTP through a proxy. Supported: http://host:port, https://host:port, socks5://host:port, socks5h://host:port (Tor). Basic auth via http://user:pass@host.")
|
||||
rootCmd.Flags().DurationVar(&cfg.MonitorInterval, "monitor-interval", 0, "Run in continuous monitoring mode, re-scanning every N (e.g. 6h, 24h). Emits diffs.")
|
||||
rootCmd.Flags().StringVar(&cfg.MonitorWebhook, "monitor-webhook", "", "Webhook URL to POST diff reports to in monitoring mode")
|
||||
|
||||
// Recursive discovery flags (enabled by default with --enable-ai)
|
||||
rootCmd.Flags().BoolVar(&cfg.Recursive, "recursive", false, "Enable recursive subdomain discovery with pattern learning")
|
||||
rootCmd.Flags().IntVar(&cfg.RecursiveDepth, "recursive-depth", 3, "Maximum recursion depth (1-5)")
|
||||
@@ -224,7 +305,288 @@ This data is used for instant, offline CVE lookups during scans.`,
|
||||
}
|
||||
rootCmd.AddCommand(dbInfoCmd)
|
||||
|
||||
// nuclei-update: force refresh of the auto-downloaded Nuclei template cache
|
||||
nucleiUpdateCmd := &cobra.Command{
|
||||
Use: "nuclei-update",
|
||||
Short: "Download / refresh Nuclei YAML templates cache",
|
||||
Long: `Fetches the official projectdiscovery/nuclei-templates ZIP archive
|
||||
and extracts every .yaml/.yml file into ~/.god-eye/nuclei-templates.
|
||||
|
||||
Safe to re-run: existing templates are overwritten in-place. The cache
|
||||
is ~40MB on disk and ships thousands of detections that the compat
|
||||
layer executes when --nuclei is on.`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
fmt.Println(output.Red("[-]"), "cannot find home dir:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
dest := home + "/.god-eye/nuclei-templates"
|
||||
|
||||
fmt.Println(output.BoldCyan("📥 Refreshing Nuclei templates…"))
|
||||
fmt.Printf(" %s %s\n", output.Dim("destination:"), output.BoldWhite(dest))
|
||||
|
||||
// Pull up the downloader. Inline import to keep the subcommand
|
||||
// lightweight when not invoked.
|
||||
dl := nucleitpl.NewDownloader()
|
||||
dl.Verbose = true
|
||||
if err := dl.Refresh(dest); err != nil {
|
||||
fmt.Println(output.Red("[-]"), "refresh failed:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println(output.Green("✓ Nuclei templates refreshed."))
|
||||
},
|
||||
}
|
||||
rootCmd.AddCommand(nucleiUpdateCmd)
|
||||
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// runPipeline is the v2 entry point. Registers every adapter module, loads
|
||||
// optional YAML + profile, and runs the event-driven pipeline under a
|
||||
// signal-aware context.
|
||||
func runPipeline(cfg config.Config) {
|
||||
// Side-effect registration of all adapter modules (F0.6).
|
||||
all.RegisterAll()
|
||||
|
||||
// Load YAML config if present. --config wins over auto-discovery.
|
||||
path := cfg.ConfigFile
|
||||
if path == "" {
|
||||
path = config.FindConfigFile()
|
||||
}
|
||||
if path != "" {
|
||||
if y, err := config.LoadYAML(path); err != nil {
|
||||
fmt.Println(output.Red("[-]"), "config:", err.Error())
|
||||
os.Exit(1)
|
||||
} else if y != nil {
|
||||
config.ApplyYAML(&cfg, y)
|
||||
}
|
||||
}
|
||||
|
||||
// Apply named scan profile if set.
|
||||
if cfg.Profile != "" {
|
||||
p, ok := config.ProfileByName(cfg.Profile)
|
||||
if !ok {
|
||||
fmt.Println(output.Red("[-]"), "unknown profile:", cfg.Profile)
|
||||
os.Exit(1)
|
||||
}
|
||||
config.ApplyProfile(&cfg, p)
|
||||
if !cfg.Silent {
|
||||
fmt.Printf("%s Profile %s applied: %s\n", output.Green("✓"), output.BoldCyan(p.Name), output.Dim(p.Description))
|
||||
}
|
||||
}
|
||||
|
||||
// Apply AI tier profile (lean/balanced/heavy). Respects explicit
|
||||
// --ai-fast-model / --ai-deep-model overrides.
|
||||
if cfg.AIProfile != "" {
|
||||
p, ok := config.AIProfileByName(cfg.AIProfile)
|
||||
if !ok {
|
||||
fmt.Println(output.Red("[-]"), "unknown AI profile:", cfg.AIProfile,
|
||||
"— valid: lean, balanced, heavy")
|
||||
os.Exit(1)
|
||||
}
|
||||
overrideFast := rootCmdRef != nil && rootCmdRef.Flags().Changed("ai-fast-model")
|
||||
overrideDeep := rootCmdRef != nil && rootCmdRef.Flags().Changed("ai-deep-model")
|
||||
config.ApplyAIProfile(&cfg, p, overrideFast, overrideDeep)
|
||||
if !cfg.Silent {
|
||||
fmt.Printf("%s AI profile %s: %s\n",
|
||||
output.Green("✓"), output.BoldCyan(p.Name), output.Dim(p.Description))
|
||||
fmt.Printf(" %s %s %s %s\n",
|
||||
output.Dim("triage:"), output.BoldWhite(cfg.AIFastModel),
|
||||
output.Dim("deep:"), output.BoldWhite(cfg.AIDeepModel))
|
||||
}
|
||||
}
|
||||
|
||||
// Handle Ctrl-C gracefully. Set this up BEFORE the model-ensure step
|
||||
// so long downloads can be interrupted cleanly.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
sigCh := make(chan os.Signal, 1)
|
||||
signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM)
|
||||
go func() {
|
||||
<-sigCh
|
||||
fmt.Println()
|
||||
fmt.Println(output.Yellow("⚠ Interrupted — shutting down..."))
|
||||
cancel()
|
||||
}()
|
||||
|
||||
// Ensure Ollama models are present before scan starts.
|
||||
if cfg.EnableAI && cfg.AutoPullModels {
|
||||
if err := ensureAIModels(ctx, &cfg); err != nil {
|
||||
if ctx.Err() == context.Canceled {
|
||||
os.Exit(130)
|
||||
}
|
||||
fmt.Println(output.Red("[-]"), "AI setup:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Continuous monitoring mode: run the scan on an interval, diff and alert.
|
||||
if cfg.MonitorInterval > 0 {
|
||||
runMonitor(ctx, cfg)
|
||||
return
|
||||
}
|
||||
|
||||
p, err := pipeline.New(&cfg, pipeline.Options{})
|
||||
if err != nil {
|
||||
fmt.Println(output.Red("[-]"), err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
var live *tui.LivePrinter
|
||||
if cfg.Live {
|
||||
live = tui.NewLivePrinter(p.Bus(), cfg.LiveVerbosity)
|
||||
}
|
||||
|
||||
if err := p.Run(ctx); err != nil {
|
||||
if ctx.Err() == context.Canceled {
|
||||
if live != nil {
|
||||
live.Close()
|
||||
}
|
||||
os.Exit(130)
|
||||
}
|
||||
fmt.Println(output.Red("[!]"), "pipeline error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if live != nil {
|
||||
live.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// runMonitor implements the asm-continuous mode: a single pipeline.Run
|
||||
// wrapped in scheduler.Scheduler that ticks at MonitorInterval, diffs
|
||||
// against the previous snapshot, and alerts on meaningful changes.
|
||||
func runMonitor(ctx context.Context, cfg config.Config) {
|
||||
scan := func(scanCtx context.Context) ([]*store.Host, error) {
|
||||
p, err := pipeline.New(&cfg, pipeline.Options{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := p.Run(scanCtx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return p.Store().All(scanCtx), nil
|
||||
}
|
||||
|
||||
s := scheduler.New(cfg.Domain, cfg.MonitorInterval, scan)
|
||||
s.AddAlerter(scheduler.StdoutAlerter{})
|
||||
if cfg.MonitorWebhook != "" {
|
||||
s.AddAlerter(scheduler.NewWebhookAlerter(cfg.MonitorWebhook))
|
||||
}
|
||||
|
||||
fmt.Printf("%s Monitoring %s every %s — Ctrl-C to stop\n",
|
||||
output.BoldGreen("▣"), output.BoldCyan(cfg.Domain), cfg.MonitorInterval)
|
||||
|
||||
if err := s.Start(ctx); err != nil && !errorIs(err, context.Canceled) {
|
||||
fmt.Println(output.Red("[!]"), "monitor error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// runWizard starts the interactive setup, then folds the user's choices
|
||||
// back into cfg. Forces pipeline mode (wizard is v2-only by design).
|
||||
func runWizard(cfg *config.Config) error {
|
||||
choice, err := wizard.Run(context.Background(), wizard.Options{
|
||||
In: os.Stdin,
|
||||
Out: os.Stdout,
|
||||
OllamaURL: cfg.AIUrl,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cfg.Domain = validator.SanitizeDomain(choice.Target)
|
||||
cfg.UsePipeline = true
|
||||
cfg.Live = choice.Live
|
||||
cfg.LiveVerbosity = choice.LiveVerbosity
|
||||
cfg.Output = choice.Output
|
||||
if choice.Format != "" {
|
||||
cfg.Format = choice.Format
|
||||
}
|
||||
|
||||
// Scan profile name threads through --profile application (later).
|
||||
if choice.ScanProfile != "" {
|
||||
cfg.Profile = choice.ScanProfile
|
||||
}
|
||||
|
||||
// ASM-continuous interval translates into a duration flag.
|
||||
if choice.MonitorInterval != "" {
|
||||
d, parseErr := time.ParseDuration(choice.MonitorInterval)
|
||||
if parseErr != nil {
|
||||
return fmt.Errorf("invalid interval %q: %w", choice.MonitorInterval, parseErr)
|
||||
}
|
||||
cfg.MonitorInterval = d
|
||||
}
|
||||
|
||||
// AI tier.
|
||||
if choice.AIProfile != "" {
|
||||
cfg.EnableAI = true
|
||||
cfg.AIProfile = choice.AIProfile
|
||||
cfg.AIVerbose = choice.AIVerbose
|
||||
cfg.AutoPullModels = choice.AIAutoPull
|
||||
} else {
|
||||
cfg.EnableAI = false
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureAIModels checks the Ollama server and downloads any missing models.
|
||||
// Prints progress when --ai-verbose is on. Fails open on unreachable
|
||||
// Ollama — the AI module itself will no-op gracefully.
|
||||
func ensureAIModels(ctx context.Context, cfg *config.Config) error {
|
||||
e := ai.NewModelEnsurer(cfg.AIUrl)
|
||||
e.Verbose = cfg.AIVerbose || cfg.Verbose
|
||||
e.Writer = os.Stderr
|
||||
|
||||
if err := e.Reachable(ctx); err != nil {
|
||||
if !cfg.Silent {
|
||||
fmt.Println(output.Yellow("⚠ "), err.Error())
|
||||
fmt.Println(output.Dim(" AI modules will no-op for this run. Start `ollama serve` to enable."))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
models := []string{}
|
||||
if cfg.AIFastModel != "" {
|
||||
models = append(models, cfg.AIFastModel)
|
||||
}
|
||||
if cfg.AIDeepModel != "" && cfg.AIDeepModel != cfg.AIFastModel {
|
||||
models = append(models, cfg.AIDeepModel)
|
||||
}
|
||||
if len(models) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if !cfg.Silent {
|
||||
fmt.Printf("%s Checking Ollama models: %s\n",
|
||||
output.BoldCyan("⚙"), output.Dim(fmt.Sprintf("%v", models)))
|
||||
}
|
||||
if err := e.EnsureAll(ctx, models); err != nil {
|
||||
return err
|
||||
}
|
||||
if !cfg.Silent {
|
||||
fmt.Printf("%s Models ready\n", output.Green("✓"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// errorIs is a thin wrapper for errors.Is that only pulls errors into
|
||||
// main when needed.
|
||||
func errorIs(err, target error) bool {
|
||||
for err != nil {
|
||||
if err == target {
|
||||
return true
|
||||
}
|
||||
type unwrapper interface{ Unwrap() error }
|
||||
u, ok := err.(unwrapper)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
err = u.Unwrap()
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -4,17 +4,18 @@ go 1.21
|
||||
|
||||
require (
|
||||
github.com/fatih/color v1.16.0
|
||||
github.com/mattn/go-isatty v0.0.20
|
||||
github.com/miekg/dns v1.1.58
|
||||
github.com/spf13/cobra v1.8.0
|
||||
golang.org/x/net v0.20.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
golang.org/x/mod v0.14.0 // indirect
|
||||
golang.org/x/net v0.20.0 // indirect
|
||||
golang.org/x/sys v0.16.0 // indirect
|
||||
golang.org/x/tools v0.17.0 // indirect
|
||||
)
|
||||
|
||||
@@ -27,5 +27,7 @@ golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
|
||||
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
|
||||
golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
// Package agent defines the Fase 3 AI agentic v2 interfaces: Planner,
|
||||
// Worker, and Tool. Unlike Fase 0.6 adapters that merely wrap v1 Ollama
|
||||
// calls, a v2 Agent plans multi-step investigations and executes tools
|
||||
// via the event bus.
|
||||
//
|
||||
// The Agent lifecycle:
|
||||
//
|
||||
// 1. Planner receives the target + existing store snapshot, produces a
|
||||
// Plan (ordered list of Tasks).
|
||||
// 2. Each Task is dispatched to a Worker (specialized agent: XSS, auth,
|
||||
// API, crypto, secrets, etc.) with a Tool set.
|
||||
// 3. Workers call Tools (dns_resolve, http_request, check_sqli_blind,
|
||||
// fetch_js, query_cve, ...) and reason over the results.
|
||||
// 4. Results feed back into Plan revision; new Tasks may be scheduled.
|
||||
//
|
||||
// This file defines the contracts. Implementations land incrementally;
|
||||
// for now a Basic Planner delegates to the Fase 0.6 v1 Ollama wrapper,
|
||||
// and a native tool-using implementation follows.
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
// Tool is a capability an agent can invoke. Tools should be idempotent
|
||||
// where possible and must respect ctx cancellation.
|
||||
type Tool interface {
|
||||
// Name is the machine identifier (e.g., "http_request", "dns_resolve").
|
||||
// Used in tool-call serialization for LLMs.
|
||||
Name() string
|
||||
|
||||
// Description is a short human-readable blurb used in the LLM tool
|
||||
// descriptor. Keep it action-oriented: "fetch an HTTP URL and return
|
||||
// the response headers + first 2KB of body".
|
||||
Description() string
|
||||
|
||||
// Schema returns the JSON-schema of the tool's argument object. Used
|
||||
// to build function-calling descriptors and to validate inputs.
|
||||
Schema() map[string]interface{}
|
||||
|
||||
// Call invokes the tool with the given arguments. Returns a JSON-encoded
|
||||
// result (often just a text summary). Errors should be returned — the
|
||||
// agent decides how to react.
|
||||
Call(ctx context.Context, args map[string]interface{}) (string, error)
|
||||
}
|
||||
|
||||
// Task is a single unit of agent work.
|
||||
type Task struct {
|
||||
ID string
|
||||
Kind string // e.g. "investigate-xss", "audit-auth", "chain-finding"
|
||||
Description string // natural-language goal the worker pursues
|
||||
Subject string // target URL / subdomain / evidence the task focuses on
|
||||
Context map[string]string // additional hints for the worker
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
// Plan is an ordered list of Tasks produced by the Planner.
|
||||
type Plan struct {
|
||||
Target string
|
||||
Tasks []Task
|
||||
Reason string // planner's rationale, logged for debugging
|
||||
}
|
||||
|
||||
// Planner decides what to investigate next given the current store state.
|
||||
type Planner interface {
|
||||
// Plan produces a new Plan. Called at the start of the analysis phase
|
||||
// and whenever enough new evidence accumulates to justify replanning.
|
||||
Plan(ctx context.Context, target string, storeSnap store.Store, bus *eventbus.Bus) (*Plan, error)
|
||||
|
||||
// Name identifies the planner implementation for logs.
|
||||
Name() string
|
||||
}
|
||||
|
||||
// Worker executes a single Task using a Toolset.
|
||||
type Worker interface {
|
||||
// Name identifies the worker (usually its specialization, e.g. "xss",
|
||||
// "auth", "api", "crypto").
|
||||
Name() string
|
||||
|
||||
// CanHandle reports whether the worker is a good fit for task. Workers
|
||||
// are consulted in priority order.
|
||||
CanHandle(task Task) bool
|
||||
|
||||
// Execute carries out the task. The worker may call tools, update the
|
||||
// store via bus events (VulnerabilityFound, SecretFound, AIFinding),
|
||||
// and return a short natural-language summary for the planner.
|
||||
Execute(ctx context.Context, task Task, tools Toolset, bus *eventbus.Bus, st store.Store) (summary string, err error)
|
||||
}
|
||||
|
||||
// Toolset is an indexed collection of Tools available to a worker. It is
|
||||
// intentionally separate from Registry so workers receive a curated subset
|
||||
// (e.g., a "crypto" worker gets oracle-style tools but not "send_slack").
|
||||
type Toolset map[string]Tool
|
||||
|
||||
// Get returns the named tool, or nil if absent.
|
||||
func (ts Toolset) Get(name string) Tool { return ts[name] }
|
||||
|
||||
// Names returns every tool name in the set. Order is not guaranteed.
|
||||
func (ts Toolset) Names() []string {
|
||||
out := make([]string, 0, len(ts))
|
||||
for n := range ts {
|
||||
out = append(out, n)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
godns "god-eye/internal/dns"
|
||||
)
|
||||
|
||||
// --- built-in tools -------------------------------------------------------
|
||||
//
|
||||
// These tools cover the minimum needed for a planner to investigate
|
||||
// discovered hosts without reinventing basic primitives. Fase 3 workers
|
||||
// receive curated subsets via Toolset.
|
||||
|
||||
// HTTPRequestTool fetches an arbitrary URL and returns status, headers,
|
||||
// and (truncated) body. Maximum 64KB body returned.
|
||||
type HTTPRequestTool struct {
|
||||
Client *http.Client
|
||||
}
|
||||
|
||||
func NewHTTPRequestTool(timeoutSec int) *HTTPRequestTool {
|
||||
return &HTTPRequestTool{
|
||||
Client: &http.Client{
|
||||
Timeout: time.Duration(timeoutSec) * time.Second,
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (t *HTTPRequestTool) Name() string { return "http_request" }
|
||||
func (t *HTTPRequestTool) Description() string { return "Fetch an HTTP(S) URL and return status + headers + first 64KB of body." }
|
||||
|
||||
func (t *HTTPRequestTool) Schema() map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"url": map[string]interface{}{"type": "string"},
|
||||
"method": map[string]interface{}{"type": "string", "default": "GET"},
|
||||
"headers": map[string]interface{}{
|
||||
"type": "object",
|
||||
"additionalProperties": map[string]interface{}{"type": "string"},
|
||||
},
|
||||
},
|
||||
"required": []string{"url"},
|
||||
}
|
||||
}
|
||||
|
||||
func (t *HTTPRequestTool) Call(ctx context.Context, args map[string]interface{}) (string, error) {
|
||||
url, _ := args["url"].(string)
|
||||
if url == "" {
|
||||
return "", errors.New("url is required")
|
||||
}
|
||||
method, _ := args["method"].(string)
|
||||
if method == "" {
|
||||
method = "GET"
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, url, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if hdrs, ok := args["headers"].(map[string]interface{}); ok {
|
||||
for k, v := range hdrs {
|
||||
if s, ok := v.(string); ok {
|
||||
req.Header.Set(k, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
req.Header.Set("User-Agent", "god-eye-v2-agent")
|
||||
|
||||
resp, err := t.Client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 64*1024))
|
||||
out := map[string]interface{}{
|
||||
"status_code": resp.StatusCode,
|
||||
"headers": flattenHeaders(resp.Header),
|
||||
"body": string(body),
|
||||
}
|
||||
b, _ := json.Marshal(out)
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
// DNSResolveTool resolves a hostname to A/CNAME/PTR records.
|
||||
type DNSResolveTool struct {
|
||||
Resolvers []string
|
||||
TimeoutSec int
|
||||
}
|
||||
|
||||
func NewDNSResolveTool(resolvers []string, timeoutSec int) *DNSResolveTool {
|
||||
if len(resolvers) == 0 {
|
||||
resolvers = []string{"8.8.8.8:53", "1.1.1.1:53"}
|
||||
}
|
||||
return &DNSResolveTool{Resolvers: resolvers, TimeoutSec: timeoutSec}
|
||||
}
|
||||
|
||||
func (t *DNSResolveTool) Name() string { return "dns_resolve" }
|
||||
func (t *DNSResolveTool) Description() string { return "Resolve a hostname to A/CNAME/PTR records." }
|
||||
|
||||
func (t *DNSResolveTool) Schema() map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"hostname": map[string]interface{}{"type": "string"},
|
||||
},
|
||||
"required": []string{"hostname"},
|
||||
}
|
||||
}
|
||||
|
||||
func (t *DNSResolveTool) Call(_ context.Context, args map[string]interface{}) (string, error) {
|
||||
name, _ := args["hostname"].(string)
|
||||
if name == "" {
|
||||
return "", errors.New("hostname is required")
|
||||
}
|
||||
ips := godns.ResolveSubdomain(name, t.Resolvers, t.TimeoutSec)
|
||||
cname := godns.ResolveCNAME(name, t.Resolvers, t.TimeoutSec)
|
||||
out := map[string]interface{}{"ips": ips, "cname": cname}
|
||||
b, _ := json.Marshal(out)
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
func flattenHeaders(h http.Header) map[string]string {
|
||||
out := make(map[string]string, len(h))
|
||||
for k, vs := range h {
|
||||
if len(vs) == 0 {
|
||||
continue
|
||||
}
|
||||
out[k] = vs[0]
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ModelEnsurer verifies that a given list of Ollama models is present on
|
||||
// the local server, and pulls any that are missing. Designed for the
|
||||
// pre-scan warmup: God's Eye should not crash mid-scan because a model
|
||||
// wasn't downloaded — EnsureAll fixes that before the pipeline starts.
|
||||
type ModelEnsurer struct {
|
||||
BaseURL string
|
||||
Client *http.Client
|
||||
Verbose bool
|
||||
Writer io.Writer // where progress is printed; defaults to os.Stdout if nil
|
||||
}
|
||||
|
||||
// NewModelEnsurer constructs an ensurer against the given Ollama base URL
|
||||
// (e.g. "http://localhost:11434"). The HTTP client has no timeout because
|
||||
// a fresh pull of a 30B model can legitimately take 10+ minutes.
|
||||
func NewModelEnsurer(baseURL string) *ModelEnsurer {
|
||||
if baseURL == "" {
|
||||
baseURL = "http://localhost:11434"
|
||||
}
|
||||
return &ModelEnsurer{
|
||||
BaseURL: strings.TrimRight(baseURL, "/"),
|
||||
Client: &http.Client{Timeout: 0},
|
||||
}
|
||||
}
|
||||
|
||||
// Installed returns the set of model tags currently available on the
|
||||
// Ollama server, keyed by the full name (e.g. "qwen3:1.7b").
|
||||
func (e *ModelEnsurer) Installed(ctx context.Context) (map[string]bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", e.BaseURL+"/api/tags", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c := &http.Client{Timeout: 10 * time.Second}
|
||||
resp, err := c.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return nil, fmt.Errorf("ollama /api/tags returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var body struct {
|
||||
Models []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"models"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]bool, len(body.Models))
|
||||
for _, m := range body.Models {
|
||||
out[m.Name] = true
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Pull streams a model pull from Ollama, printing progress lines when
|
||||
// Verbose is true. Uses POST /api/pull with stream=true; each JSON line
|
||||
// reports status + optional {total, completed} for byte-level progress.
|
||||
func (e *ModelEnsurer) Pull(ctx context.Context, model string) error {
|
||||
payload := map[string]interface{}{"name": model, "stream": true}
|
||||
body, _ := json.Marshal(payload)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", e.BaseURL+"/api/pull", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := e.Client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||
return fmt.Errorf("ollama /api/pull returned %d: %s", resp.StatusCode, strings.TrimSpace(string(b)))
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(resp.Body)
|
||||
// Progress events can be large; bump the scanner buffer.
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
|
||||
var lastStatus string
|
||||
var lastPct int
|
||||
for scanner.Scan() {
|
||||
line := bytes.TrimSpace(scanner.Bytes())
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
var ev struct {
|
||||
Status string `json:"status"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
Total int64 `json:"total,omitempty"`
|
||||
Completed int64 `json:"completed,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
if err := json.Unmarshal(line, &ev); err != nil {
|
||||
continue
|
||||
}
|
||||
if ev.Error != "" {
|
||||
return fmt.Errorf("pull %s: %s", model, ev.Error)
|
||||
}
|
||||
if !e.Verbose {
|
||||
continue
|
||||
}
|
||||
w := e.writer()
|
||||
if ev.Total > 0 && ev.Completed > 0 {
|
||||
pct := int(float64(ev.Completed) / float64(ev.Total) * 100)
|
||||
// Throttle: new status line always; otherwise only print when
|
||||
// the percentage has moved ≥5 points since the last emission
|
||||
// (or reaches a final 100% for this status exactly once).
|
||||
switch {
|
||||
case ev.Status != lastStatus:
|
||||
fmt.Fprintf(w, " %-24s %3d%% %s / %s\n", ev.Status, pct, humanBytes(ev.Completed), humanBytes(ev.Total))
|
||||
lastStatus = ev.Status
|
||||
lastPct = pct
|
||||
case pct >= lastPct+5 && pct < 100:
|
||||
fmt.Fprintf(w, " %-24s %3d%% %s / %s\n", ev.Status, pct, humanBytes(ev.Completed), humanBytes(ev.Total))
|
||||
lastPct = pct
|
||||
case pct == 100 && lastPct < 100:
|
||||
fmt.Fprintf(w, " %-24s %3d%% %s / %s\n", ev.Status, pct, humanBytes(ev.Completed), humanBytes(ev.Total))
|
||||
lastPct = 100
|
||||
}
|
||||
} else if ev.Status != lastStatus {
|
||||
fmt.Fprintf(w, " %s\n", ev.Status)
|
||||
lastStatus = ev.Status
|
||||
lastPct = 0
|
||||
}
|
||||
}
|
||||
return scanner.Err()
|
||||
}
|
||||
|
||||
// EnsureAll checks every name in models. For each missing one it calls Pull.
|
||||
// Already-present models are skipped. Returns on the first error.
|
||||
//
|
||||
// Name matching is generous: Ollama sometimes tags models as "qwen3:1.7b"
|
||||
// and sometimes as "qwen3:1.7b-instruct-fp16", so we accept exact match,
|
||||
// a ":latest" variant, or the bare model name with no tag.
|
||||
func (e *ModelEnsurer) EnsureAll(ctx context.Context, models []string) error {
|
||||
installed, err := e.Installed(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("query ollama: %w", err)
|
||||
}
|
||||
|
||||
unique := dedup(models)
|
||||
missing := []string{}
|
||||
for _, m := range unique {
|
||||
if alreadyInstalled(installed, m) {
|
||||
if e.Verbose {
|
||||
fmt.Fprintf(e.writer(), "✓ %s already installed\n", m)
|
||||
}
|
||||
continue
|
||||
}
|
||||
missing = append(missing, m)
|
||||
}
|
||||
|
||||
if len(missing) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if e.Verbose {
|
||||
fmt.Fprintf(e.writer(), "↓ Pulling %d missing model(s): %s\n", len(missing), strings.Join(missing, ", "))
|
||||
}
|
||||
for _, m := range missing {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if e.Verbose {
|
||||
fmt.Fprintf(e.writer(), "↓ %s\n", m)
|
||||
}
|
||||
if err := e.Pull(ctx, m); err != nil {
|
||||
return fmt.Errorf("pull %s: %w", m, err)
|
||||
}
|
||||
if e.Verbose {
|
||||
fmt.Fprintf(e.writer(), "✓ %s ready\n", m)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reachable reports whether the Ollama server answers /api/tags. Callers
|
||||
// should check this before EnsureAll to surface a friendly message.
|
||||
func (e *ModelEnsurer) Reachable(ctx context.Context) error {
|
||||
c := &http.Client{Timeout: 3 * time.Second}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", e.BaseURL+"/api/tags", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := c.Do(req)
|
||||
if err != nil {
|
||||
return errors.New("ollama not reachable at " + e.BaseURL + " (is `ollama serve` running?)")
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return fmt.Errorf("ollama at %s returned %d", e.BaseURL, resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *ModelEnsurer) writer() io.Writer {
|
||||
if e.Writer != nil {
|
||||
return e.Writer
|
||||
}
|
||||
return stdout
|
||||
}
|
||||
|
||||
var stdout io.Writer // populated by main via SetStdout; nil writer would fmt-print to os.Stdout
|
||||
|
||||
// SetStdout installs the writer used when ModelEnsurer.Writer is nil. main.go
|
||||
// sets this to os.Stdout; tests can set it to a bytes.Buffer.
|
||||
func SetStdout(w io.Writer) { stdout = w }
|
||||
|
||||
func alreadyInstalled(installed map[string]bool, model string) bool {
|
||||
if installed[model] {
|
||||
return true
|
||||
}
|
||||
if installed[model+":latest"] {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(model, ":") {
|
||||
base := strings.SplitN(model, ":", 2)[0]
|
||||
if installed[base] || installed[base+":latest"] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func dedup(ss []string) []string {
|
||||
seen := make(map[string]struct{}, len(ss))
|
||||
out := make([]string, 0, len(ss))
|
||||
for _, s := range ss {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[s]; ok {
|
||||
continue
|
||||
}
|
||||
seen[s] = struct{}{}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func humanBytes(n int64) string {
|
||||
const k = 1024.0
|
||||
if n < int64(k) {
|
||||
return fmt.Sprintf("%dB", n)
|
||||
}
|
||||
units := []string{"KB", "MB", "GB", "TB"}
|
||||
v := float64(n) / k
|
||||
for _, u := range units {
|
||||
if v < k {
|
||||
return fmt.Sprintf("%.1f%s", v, u)
|
||||
}
|
||||
v /= k
|
||||
}
|
||||
return fmt.Sprintf("%.1fPB", v)
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAlreadyInstalled(t *testing.T) {
|
||||
installed := map[string]bool{
|
||||
"qwen3:1.7b": true,
|
||||
"qwen2.5-coder:14b": true,
|
||||
"custom-model:latest": true,
|
||||
}
|
||||
cases := []struct {
|
||||
model string
|
||||
want bool
|
||||
}{
|
||||
{"qwen3:1.7b", true},
|
||||
{"qwen2.5-coder:14b", true},
|
||||
{"custom-model", true}, // via :latest fallback
|
||||
{"llama3:8b", false},
|
||||
{"qwen3", false}, // bare name: only matches when ":latest" is installed (it isn't)
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := alreadyInstalled(installed, c.model); got != c.want {
|
||||
t.Errorf("alreadyInstalled(%q) = %v, want %v", c.model, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDedup(t *testing.T) {
|
||||
got := dedup([]string{"a", "b", "a", "", "c", " b "})
|
||||
want := []string{"a", "b", "c"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != want[i] {
|
||||
t.Errorf("index %d: got %q want %q", i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHumanBytes(t *testing.T) {
|
||||
cases := []struct {
|
||||
in int64
|
||||
want string
|
||||
}{
|
||||
{0, "0B"},
|
||||
{512, "512B"},
|
||||
{1024, "1.0KB"},
|
||||
{1024 * 1024, "1.0MB"},
|
||||
{1024 * 1024 * 1024, "1.0GB"},
|
||||
{int64(2.5 * 1024 * 1024 * 1024), "2.5GB"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := humanBytes(c.in); got != c.want {
|
||||
t.Errorf("humanBytes(%d) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstalled_ParsesTagsResponse(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/api/tags" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"models": []map[string]string{
|
||||
{"name": "qwen3:1.7b"},
|
||||
{"name": "qwen2.5-coder:14b"},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
e := NewModelEnsurer(srv.URL)
|
||||
got, err := e.Installed(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !got["qwen3:1.7b"] || !got["qwen2.5-coder:14b"] {
|
||||
t.Errorf("missing expected models: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstalled_Non200ReturnsError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "nope", http.StatusInternalServerError)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
e := NewModelEnsurer(srv.URL)
|
||||
if _, err := e.Installed(context.Background()); err == nil {
|
||||
t.Error("expected error on non-200")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReachable(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(`{"models":[]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
e := NewModelEnsurer(srv.URL)
|
||||
if err := e.Reachable(context.Background()); err != nil {
|
||||
t.Errorf("expected reachable, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReachable_Unreachable(t *testing.T) {
|
||||
e := NewModelEnsurer("http://127.0.0.1:1") // nothing listens here
|
||||
if err := e.Reachable(context.Background()); err == nil {
|
||||
t.Error("expected unreachable error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPull_StreamsProgress(t *testing.T) {
|
||||
// Fake Ollama that emits a few NDJSON status events.
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/api/pull" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/x-ndjson")
|
||||
events := []string{
|
||||
`{"status":"pulling manifest"}`,
|
||||
`{"status":"downloading","digest":"sha256:abc","total":1048576,"completed":524288}`,
|
||||
`{"status":"downloading","digest":"sha256:abc","total":1048576,"completed":1048576}`,
|
||||
`{"status":"verifying sha256 digest"}`,
|
||||
`{"status":"writing manifest"}`,
|
||||
`{"status":"success"}`,
|
||||
}
|
||||
for _, e := range events {
|
||||
w.Write([]byte(e + "\n"))
|
||||
if flusher, ok := w.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
e := NewModelEnsurer(srv.URL)
|
||||
e.Verbose = true
|
||||
e.Writer = buf
|
||||
|
||||
if err := e.Pull(context.Background(), "fake:1b"); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, "pulling manifest") {
|
||||
t.Errorf("missing 'pulling manifest' in output: %q", out)
|
||||
}
|
||||
if !strings.Contains(out, "success") {
|
||||
t.Errorf("missing 'success' in output: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPull_ErrorBubblesUp(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(`{"error":"model not found"}` + "\n"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
e := NewModelEnsurer(srv.URL)
|
||||
err := e.Pull(context.Background(), "nonexistent")
|
||||
if err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "model not found") {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureAll_SkipsInstalled_PullsMissing(t *testing.T) {
|
||||
pullCalls := map[string]int{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/tags":
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"models": []map[string]string{{"name": "already-here:1b"}},
|
||||
})
|
||||
case "/api/pull":
|
||||
var body struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
pullCalls[body.Name]++
|
||||
w.Write([]byte(`{"status":"success"}` + "\n"))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
e := NewModelEnsurer(srv.URL)
|
||||
if err := e.EnsureAll(context.Background(), []string{"already-here:1b", "missing-a:7b", "missing-b:14b"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pullCalls["already-here:1b"] > 0 {
|
||||
t.Errorf("should not have pulled already-here")
|
||||
}
|
||||
if pullCalls["missing-a:7b"] != 1 || pullCalls["missing-b:14b"] != 1 {
|
||||
t.Errorf("missing models not pulled correctly: %v", pullCalls)
|
||||
}
|
||||
}
|
||||
+32
-5
@@ -4,7 +4,9 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -12,10 +14,27 @@ import (
|
||||
// OllamaClient handles communication with local Ollama instance
|
||||
type OllamaClient struct {
|
||||
BaseURL string
|
||||
FastModel string // deepseek-r1:1.5b for quick triage
|
||||
DeepModel string // qwen2.5-coder:7b for deep analysis
|
||||
FastModel string // qwen3:1.7b for quick triage (lean default)
|
||||
DeepModel string // qwen2.5-coder:14b for deep analysis (lean default)
|
||||
Timeout time.Duration
|
||||
EnableCascade bool
|
||||
|
||||
// Verbose controls whether every query is logged with timing + sizes.
|
||||
// Writes to VerboseLogger or stderr when nil. Toggle via --ai-verbose.
|
||||
Verbose bool
|
||||
VerboseLogger io.Writer
|
||||
}
|
||||
|
||||
// logVerbose writes a single line to the verbose logger when Verbose is on.
|
||||
func (c *OllamaClient) logVerbose(format string, args ...interface{}) {
|
||||
if !c.Verbose {
|
||||
return
|
||||
}
|
||||
w := c.VerboseLogger
|
||||
if w == nil {
|
||||
w = os.Stderr
|
||||
}
|
||||
fmt.Fprintf(w, "[ai] "+format+"\n", args...)
|
||||
}
|
||||
|
||||
// OllamaRequest represents the request payload for Ollama API
|
||||
@@ -51,10 +70,10 @@ func NewOllamaClient(baseURL, fastModel, deepModel string, enableCascade bool) *
|
||||
baseURL = "http://localhost:11434"
|
||||
}
|
||||
if fastModel == "" {
|
||||
fastModel = "deepseek-r1:1.5b"
|
||||
fastModel = "qwen3:1.7b"
|
||||
}
|
||||
if deepModel == "" {
|
||||
deepModel = "qwen2.5-coder:7b"
|
||||
deepModel = "qwen2.5-coder:14b"
|
||||
}
|
||||
|
||||
return &OllamaClient{
|
||||
@@ -351,6 +370,9 @@ Output only the REAL secrets in their original [Type] format, one per line. If n
|
||||
|
||||
// query sends a request to Ollama API
|
||||
func (c *OllamaClient) query(model, prompt string, timeout time.Duration) (string, error) {
|
||||
start := time.Now()
|
||||
c.logVerbose("→ %s prompt=%dB timeout=%s", model, len(prompt), timeout)
|
||||
|
||||
reqBody := OllamaRequest{
|
||||
Model: model,
|
||||
Prompt: prompt,
|
||||
@@ -373,20 +395,25 @@ func (c *OllamaClient) query(model, prompt string, timeout time.Duration) (strin
|
||||
bytes.NewBuffer(jsonData),
|
||||
)
|
||||
if err != nil {
|
||||
c.logVerbose("✘ %s %s error=%v", model, time.Since(start).Round(time.Millisecond), err)
|
||||
return "", fmt.Errorf("ollama request failed: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
c.logVerbose("✘ %s status=%d %s", model, resp.StatusCode, time.Since(start).Round(time.Millisecond))
|
||||
return "", fmt.Errorf("ollama returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var ollamaResp OllamaResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&ollamaResp); err != nil {
|
||||
c.logVerbose("✘ %s decode error=%v", model, err)
|
||||
return "", fmt.Errorf("failed to decode response: %v", err)
|
||||
}
|
||||
|
||||
return strings.TrimSpace(ollamaResp.Response), nil
|
||||
out := strings.TrimSpace(ollamaResp.Response)
|
||||
c.logVerbose("← %s response=%dB %s", model, len(out), time.Since(start).Round(time.Millisecond))
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// parseFindings extracts findings by severity from AI response
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
package config
|
||||
|
||||
// AIProfile bundles the triage + deep models for a named AI tier. Unlike
|
||||
// the scan-level Profile (bugbounty/pentest/…), an AIProfile only touches
|
||||
// model selection — it doesn't flip stealth, recursion, or module enables.
|
||||
type AIProfile struct {
|
||||
Name string
|
||||
Description string
|
||||
FastModel string
|
||||
DeepModel string
|
||||
// MinRAMGB is an advisory (not enforced) hint about the memory footprint
|
||||
// of both models loaded simultaneously. Printed in the profile help
|
||||
// banner so users can pick the right tier for their machine.
|
||||
MinRAMGB int
|
||||
}
|
||||
|
||||
// Built-in AI profiles. The lean tier matches the repository defaults so
|
||||
// `--ai-profile lean` is always equivalent to "use whatever the defaults
|
||||
// say". balanced and heavy upgrade deep model to Qwen3-Coder MoE which
|
||||
// activates only 3.3B parameters per token despite its 30B total.
|
||||
var (
|
||||
AIProfileLean = AIProfile{
|
||||
Name: "lean",
|
||||
Description: "Runs on 16GB RAM; default. qwen3:1.7b triage + qwen2.5-coder:14b deep.",
|
||||
FastModel: "qwen3:1.7b",
|
||||
DeepModel: "qwen2.5-coder:14b",
|
||||
MinRAMGB: 16,
|
||||
}
|
||||
|
||||
AIProfileBalanced = AIProfile{
|
||||
Name: "balanced",
|
||||
Description: "32GB RAM / 24GB VRAM. Upgrades deep to qwen3-coder:30b MoE (3.3B active, 256K ctx).",
|
||||
FastModel: "qwen3:4b",
|
||||
DeepModel: "qwen3-coder:30b",
|
||||
MinRAMGB: 32,
|
||||
}
|
||||
|
||||
AIProfileHeavy = AIProfile{
|
||||
Name: "heavy",
|
||||
Description: "64GB+ RAM. Best-quality triage + deep. Slowest; ideal for final analysis passes.",
|
||||
FastModel: "qwen3:8b",
|
||||
DeepModel: "qwen3-coder:30b",
|
||||
MinRAMGB: 64,
|
||||
}
|
||||
)
|
||||
|
||||
// BuiltinAIProfiles lists every AIProfile in CLI help order.
|
||||
var BuiltinAIProfiles = []AIProfile{
|
||||
AIProfileLean,
|
||||
AIProfileBalanced,
|
||||
AIProfileHeavy,
|
||||
}
|
||||
|
||||
// AIProfileByName resolves a named profile. Lookup is case-insensitive
|
||||
// and tolerates the common alias "max" → heavy.
|
||||
func AIProfileByName(name string) (AIProfile, bool) {
|
||||
switch normaliseAIProfileName(name) {
|
||||
case "lean":
|
||||
return AIProfileLean, true
|
||||
case "balanced", "balance", "mid":
|
||||
return AIProfileBalanced, true
|
||||
case "heavy", "max", "power":
|
||||
return AIProfileHeavy, true
|
||||
}
|
||||
return AIProfile{}, false
|
||||
}
|
||||
|
||||
func normaliseAIProfileName(s string) string {
|
||||
out := make([]byte, 0, len(s))
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if c >= 'A' && c <= 'Z' {
|
||||
c += 'a' - 'A'
|
||||
}
|
||||
if c == ' ' || c == '_' || c == '-' {
|
||||
continue
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
// ApplyAIProfile merges p's models into cfg. If cfg.AIFastModel /
|
||||
// cfg.AIDeepModel were explicitly set by the user (overrideFast /
|
||||
// overrideDeep true) the profile is ignored for that field. The caller
|
||||
// is responsible for detecting explicit flags; in practice this comes
|
||||
// from cobra's cmd.Flags().Changed("ai-fast-model").
|
||||
func ApplyAIProfile(cfg *Config, p AIProfile, overrideFast, overrideDeep bool) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if !overrideFast && p.FastModel != "" {
|
||||
cfg.AIFastModel = p.FastModel
|
||||
}
|
||||
if !overrideDeep && p.DeepModel != "" {
|
||||
cfg.AIDeepModel = p.DeepModel
|
||||
}
|
||||
if cfg.AIProfile == "" {
|
||||
cfg.AIProfile = p.Name
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package config
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestAIProfileByName(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
wantOK bool
|
||||
wantTag string
|
||||
}{
|
||||
{"lean", true, "qwen3:1.7b"},
|
||||
{"LEAN", true, "qwen3:1.7b"},
|
||||
{"balanced", true, "qwen3:4b"},
|
||||
{"balance", true, "qwen3:4b"},
|
||||
{"mid", true, "qwen3:4b"},
|
||||
{"heavy", true, "qwen3:8b"},
|
||||
{"max", true, "qwen3:8b"},
|
||||
{"power", true, "qwen3:8b"},
|
||||
{"Heavy", true, "qwen3:8b"},
|
||||
{"nope", false, ""},
|
||||
{"", false, ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
p, ok := AIProfileByName(c.in)
|
||||
if ok != c.wantOK {
|
||||
t.Errorf("AIProfileByName(%q) ok = %v, want %v", c.in, ok, c.wantOK)
|
||||
continue
|
||||
}
|
||||
if ok && p.FastModel != c.wantTag {
|
||||
t.Errorf("AIProfileByName(%q).FastModel = %q, want %q", c.in, p.FastModel, c.wantTag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltinAIProfiles_Unique(t *testing.T) {
|
||||
names := map[string]bool{}
|
||||
for _, p := range BuiltinAIProfiles {
|
||||
if p.Name == "" {
|
||||
t.Error("profile with empty name")
|
||||
}
|
||||
if p.FastModel == "" || p.DeepModel == "" {
|
||||
t.Errorf("profile %q missing models", p.Name)
|
||||
}
|
||||
if p.Description == "" {
|
||||
t.Errorf("profile %q missing description", p.Name)
|
||||
}
|
||||
if names[p.Name] {
|
||||
t.Errorf("duplicate profile name: %q", p.Name)
|
||||
}
|
||||
names[p.Name] = true
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyAIProfile_RespectsOverrides(t *testing.T) {
|
||||
cfg := &Config{
|
||||
AIFastModel: "user-chose-this:1b",
|
||||
AIDeepModel: "user-chose-that:7b",
|
||||
}
|
||||
ApplyAIProfile(cfg, AIProfileHeavy, true, true)
|
||||
if cfg.AIFastModel != "user-chose-this:1b" {
|
||||
t.Errorf("overrideFast was ignored: %q", cfg.AIFastModel)
|
||||
}
|
||||
if cfg.AIDeepModel != "user-chose-that:7b" {
|
||||
t.Errorf("overrideDeep was ignored: %q", cfg.AIDeepModel)
|
||||
}
|
||||
if cfg.AIProfile != "heavy" {
|
||||
t.Errorf("AIProfile not set to heavy, got %q", cfg.AIProfile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyAIProfile_FillsUnsetFields(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
ApplyAIProfile(cfg, AIProfileBalanced, false, false)
|
||||
if cfg.AIFastModel != "qwen3:4b" {
|
||||
t.Errorf("FastModel not applied: %q", cfg.AIFastModel)
|
||||
}
|
||||
if cfg.AIDeepModel != "qwen3-coder:30b" {
|
||||
t.Errorf("DeepModel not applied: %q", cfg.AIDeepModel)
|
||||
}
|
||||
if cfg.AIProfile != "balanced" {
|
||||
t.Errorf("AIProfile not set: %q", cfg.AIProfile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyAIProfile_NilConfigNoop(t *testing.T) {
|
||||
ApplyAIProfile(nil, AIProfileLean, false, false) // must not panic
|
||||
}
|
||||
|
||||
func TestApplyAIProfile_PartialOverride(t *testing.T) {
|
||||
cfg := &Config{AIFastModel: "custom:1b"}
|
||||
ApplyAIProfile(cfg, AIProfileHeavy, true, false)
|
||||
if cfg.AIFastModel != "custom:1b" {
|
||||
t.Errorf("FastModel overridden: %q", cfg.AIFastModel)
|
||||
}
|
||||
if cfg.AIDeepModel != "qwen3-coder:30b" {
|
||||
t.Errorf("DeepModel not applied: %q", cfg.AIDeepModel)
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,80 @@ type Config struct {
|
||||
NoTechScan bool // Disable tech scan (override when --enable-ai)
|
||||
NoASNScan bool // Disable ASN scan (override when --enable-ai)
|
||||
NoVHostScan bool // Disable vhost scan (override when --enable-ai)
|
||||
|
||||
// v2: profile + per-module overrides loaded from config file or CLI.
|
||||
// Profile is the named profile to apply before CLI flags. Empty = none.
|
||||
Profile string
|
||||
|
||||
// ConfigFile is the path to an optional YAML config file. Empty = search
|
||||
// standard locations, then fall through to CLI defaults + profile only.
|
||||
ConfigFile string
|
||||
|
||||
// ModuleSettings is a flat map of module-name → enabled. Populated from
|
||||
// YAML ("modules:" section) and CLI (--enable/--disable flags if added).
|
||||
// Consumed by ConfigView.ModuleEnabled. Empty means "honor each module's
|
||||
// DefaultEnabled()".
|
||||
ModuleSettings map[string]bool
|
||||
|
||||
// UsePipeline opts into the v2 event-driven pipeline. When false (default
|
||||
// during F0.6 migration) the legacy scanner.Run is used. Once F0.7
|
||||
// parity is verified this becomes true by default.
|
||||
UsePipeline bool
|
||||
|
||||
// Live toggles the Fase 4 LivePrinter that streams colorized scan
|
||||
// events to the terminal alongside (or instead of) the final report.
|
||||
Live bool
|
||||
// LiveVerbosity controls how much the LivePrinter prints (0..2).
|
||||
LiveVerbosity int
|
||||
|
||||
// MonitorInterval, when > 0, switches the CLI into asm-continuous mode:
|
||||
// the scan runs on this interval and diffs against the previous
|
||||
// snapshot, firing Webhook/Stdout alerts on meaningful changes.
|
||||
MonitorInterval time.Duration
|
||||
// MonitorWebhook is a POST target for diff reports in monitor mode.
|
||||
MonitorWebhook string
|
||||
|
||||
// AIProfile is the named AI tier (lean/balanced/heavy). When set, it
|
||||
// applies FastModel+DeepModel defaults before CLI overrides kick in.
|
||||
// Empty string = use whatever AIFastModel/AIDeepModel resolve to via
|
||||
// CLI flags + YAML.
|
||||
AIProfile string
|
||||
|
||||
// AIVerbose toggles detailed logging of every Ollama query: model,
|
||||
// prompt size, response size, duration, triage decisions. Writes to
|
||||
// stderr so stdout (JSON / silent modes) stays clean.
|
||||
AIVerbose bool
|
||||
|
||||
// AutoPullModels controls whether god-eye auto-downloads missing
|
||||
// Ollama models at startup when --enable-ai is set. Defaults to true
|
||||
// — flip to false if you want scan failures instead of silent pulls.
|
||||
AutoPullModels bool
|
||||
|
||||
// Wizard forces the interactive setup flow even when -d is present,
|
||||
// so users can preview/tweak defaults. When -d is absent and stdin
|
||||
// is a TTY, the wizard auto-starts without this flag.
|
||||
Wizard bool
|
||||
|
||||
// NucleiScan opts into the Nuclei-format template executor. Templates
|
||||
// are loaded from NucleiTemplates (or ~/nuclei-templates as fallback,
|
||||
// with auto-download of the official ZIP into ~/.god-eye/nuclei-templates
|
||||
// when NucleiAutoDownload is true and no local dir is present).
|
||||
NucleiScan bool
|
||||
// NucleiTemplates is an optional override for the template directory.
|
||||
NucleiTemplates string
|
||||
// NucleiAutoDownload controls whether god-eye auto-fetches the
|
||||
// official nuclei-templates ZIP on first use. Defaults to true.
|
||||
NucleiAutoDownload bool
|
||||
|
||||
// Proxy routes every outbound HTTP request (passive sources, probes,
|
||||
// Nuclei, Ollama-if-remote) through the given URL. Supports:
|
||||
// http://host:port - HTTP CONNECT proxy (Burp, ZAP, mitmproxy)
|
||||
// https://host:port - HTTPS CONNECT proxy
|
||||
// socks5://host:port - SOCKS5 with local DNS
|
||||
// socks5h://host:port - SOCKS5 with proxy-side DNS (Tor convention)
|
||||
// Basic auth is honoured: http://user:pass@host.
|
||||
// Empty = no proxy (direct).
|
||||
Proxy string
|
||||
}
|
||||
|
||||
// Stats holds scan statistics
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDefaultResolversNonEmpty(t *testing.T) {
|
||||
if len(DefaultResolvers) == 0 {
|
||||
t.Fatal("DefaultResolvers is empty")
|
||||
}
|
||||
for _, r := range DefaultResolvers {
|
||||
if r == "" {
|
||||
t.Errorf("empty resolver in DefaultResolvers")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultWordlistNonEmpty(t *testing.T) {
|
||||
if len(DefaultWordlist) < 50 {
|
||||
t.Errorf("DefaultWordlist too small: %d entries", len(DefaultWordlist))
|
||||
}
|
||||
seen := make(map[string]bool)
|
||||
for _, w := range DefaultWordlist {
|
||||
if w == "" {
|
||||
t.Error("empty entry in DefaultWordlist")
|
||||
}
|
||||
// Note: v1 wordlist contains "smtp" and "staging" twice — that's a bug
|
||||
// but not something we fix in baseline tests. Just verify no ALL duplicates.
|
||||
seen[w] = true
|
||||
}
|
||||
if len(seen) < 50 {
|
||||
t.Errorf("too many duplicates: %d unique out of %d", len(seen), len(DefaultWordlist))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubdomainResult_JSONRoundtrip(t *testing.T) {
|
||||
orig := &SubdomainResult{
|
||||
Subdomain: "api.example.com",
|
||||
IPs: []string{"1.2.3.4"},
|
||||
CNAME: "cname.example.com",
|
||||
StatusCode: 200,
|
||||
Title: "API",
|
||||
Tech: []string{"nginx", "Go"},
|
||||
CloudProvider: "AWS",
|
||||
TLSFingerprint: &TLSFingerprint{
|
||||
Vendor: "Fortinet",
|
||||
Product: "FortiGate",
|
||||
ApplianceType: "firewall",
|
||||
},
|
||||
}
|
||||
|
||||
data, err := json.Marshal(orig)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal failed: %v", err)
|
||||
}
|
||||
|
||||
var decoded SubdomainResult
|
||||
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal failed: %v", err)
|
||||
}
|
||||
|
||||
if decoded.Subdomain != orig.Subdomain {
|
||||
t.Errorf("Subdomain mismatch: got %q want %q", decoded.Subdomain, orig.Subdomain)
|
||||
}
|
||||
if len(decoded.IPs) != 1 || decoded.IPs[0] != "1.2.3.4" {
|
||||
t.Errorf("IPs mismatch: got %v", decoded.IPs)
|
||||
}
|
||||
if decoded.TLSFingerprint == nil {
|
||||
t.Fatal("TLSFingerprint is nil after roundtrip")
|
||||
}
|
||||
if decoded.TLSFingerprint.Vendor != "Fortinet" {
|
||||
t.Errorf("TLSFingerprint.Vendor = %q, want Fortinet", decoded.TLSFingerprint.Vendor)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubdomainResult_OmitemptyMinimal(t *testing.T) {
|
||||
// Ensure zero-value struct produces a minimal JSON (only subdomain field would be present if set).
|
||||
empty := &SubdomainResult{}
|
||||
data, err := json.Marshal(empty)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Only the required "subdomain" field (empty string) should appear — every other is omitempty.
|
||||
expected := `{"subdomain":""}`
|
||||
if string(data) != expected {
|
||||
t.Errorf("empty struct JSON = %s, want %s", string(data), expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigZeroValue(t *testing.T) {
|
||||
var c Config
|
||||
if c.Domain != "" {
|
||||
t.Errorf("default Domain should be empty, got %q", c.Domain)
|
||||
}
|
||||
if c.EnableAI {
|
||||
t.Error("EnableAI should default to false")
|
||||
}
|
||||
if c.Concurrency != 0 {
|
||||
t.Error("Concurrency should default to 0 (overridden by CLI default)")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package config
|
||||
|
||||
// Profile is a named bundle of defaults that tailors God's Eye for a specific
|
||||
// use case. Profiles set module enable/disable, concurrency hints, stealth,
|
||||
// and whether AI is on. CLI flags still override profile defaults.
|
||||
type Profile struct {
|
||||
Name string
|
||||
Description string
|
||||
|
||||
// Core tuning
|
||||
Concurrency int
|
||||
Timeout int
|
||||
Stealth string // off, light, moderate, aggressive, paranoid
|
||||
|
||||
// Feature toggles (nil means "use module default")
|
||||
AI *bool
|
||||
MultiAgent *bool
|
||||
Recursive *bool
|
||||
NoBrute *bool
|
||||
NoProbe *bool
|
||||
NoPorts *bool
|
||||
NoTakeover *bool
|
||||
|
||||
// Advanced feature flags (nil = use module default)
|
||||
CloudScan *bool
|
||||
APIScan *bool
|
||||
SecretsScan *bool
|
||||
TechScan *bool
|
||||
ASNScan *bool
|
||||
VHostScan *bool
|
||||
|
||||
// Per-module overrides (explicit enable/disable)
|
||||
Modules map[string]bool
|
||||
}
|
||||
|
||||
// ProfileBugBounty is tuned for bug-bounty recon: broad discovery, AI on,
|
||||
// secrets+tech+cloud scanning on, stealth off (speed matters).
|
||||
var ProfileBugBounty = Profile{
|
||||
Name: "bugbounty",
|
||||
Description: "Aggressive recon for bug-bounty: broad discovery, AI on, secrets/cloud/API/tech scanning, stealth off.",
|
||||
Concurrency: 1000,
|
||||
Timeout: 5,
|
||||
Stealth: "off",
|
||||
AI: ptrTrue(),
|
||||
MultiAgent: ptrTrue(),
|
||||
Recursive: ptrTrue(),
|
||||
CloudScan: ptrTrue(),
|
||||
APIScan: ptrTrue(),
|
||||
SecretsScan: ptrTrue(),
|
||||
TechScan: ptrTrue(),
|
||||
ASNScan: ptrTrue(),
|
||||
VHostScan: ptrTrue(),
|
||||
}
|
||||
|
||||
// ProfilePentest is tuned for authorized penetration tests: stealth light,
|
||||
// full enrichment, AI on for deeper analysis.
|
||||
var ProfilePentest = Profile{
|
||||
Name: "pentest",
|
||||
Description: "Authorized pentest: full enrichment with light stealth to avoid basic rate limits.",
|
||||
Concurrency: 300,
|
||||
Timeout: 10,
|
||||
Stealth: "light",
|
||||
AI: ptrTrue(),
|
||||
MultiAgent: ptrTrue(),
|
||||
Recursive: ptrTrue(),
|
||||
CloudScan: ptrTrue(),
|
||||
APIScan: ptrTrue(),
|
||||
SecretsScan: ptrTrue(),
|
||||
TechScan: ptrTrue(),
|
||||
ASNScan: ptrTrue(),
|
||||
VHostScan: ptrTrue(),
|
||||
}
|
||||
|
||||
// ProfileASMContinuous is tuned for attack-surface monitoring: reduced depth
|
||||
// per run, designed to be re-run periodically with diff engine (Fase 5).
|
||||
// Stealth moderate to stay below detection thresholds when running daily.
|
||||
var ProfileASMContinuous = Profile{
|
||||
Name: "asm-continuous",
|
||||
Description: "Continuous attack-surface monitoring; runs cheaper than full recon, feeds diff engine.",
|
||||
Concurrency: 200,
|
||||
Timeout: 10,
|
||||
Stealth: "moderate",
|
||||
AI: ptrFalse(), // AI only on findings that change, not full re-analysis
|
||||
Recursive: ptrFalse(), // rely on diff to grow surface over time
|
||||
CloudScan: ptrTrue(),
|
||||
TechScan: ptrTrue(),
|
||||
SecretsScan: ptrTrue(),
|
||||
}
|
||||
|
||||
// ProfileStealthMax is for highly sensitive targets where any detection is
|
||||
// unacceptable. Very slow; passive-first.
|
||||
var ProfileStealthMax = Profile{
|
||||
Name: "stealth-max",
|
||||
Description: "Maximum evasion. Passive-only by default, slow request cadence.",
|
||||
Concurrency: 3,
|
||||
Timeout: 20,
|
||||
Stealth: "paranoid",
|
||||
NoBrute: ptrTrue(),
|
||||
NoPorts: ptrTrue(),
|
||||
AI: ptrFalse(),
|
||||
TechScan: ptrTrue(),
|
||||
}
|
||||
|
||||
// ProfileQuick is for triage: skip expensive phases, produce a fast answer.
|
||||
var ProfileQuick = Profile{
|
||||
Name: "quick",
|
||||
Description: "Fast triage: passive enum + HTTP probe, no brute/JS/AI.",
|
||||
Concurrency: 500,
|
||||
Timeout: 5,
|
||||
Stealth: "off",
|
||||
NoBrute: ptrTrue(),
|
||||
AI: ptrFalse(),
|
||||
}
|
||||
|
||||
// BuiltinProfiles lists every named profile that ships with the tool, in a
|
||||
// stable order for docs/help output.
|
||||
var BuiltinProfiles = []Profile{
|
||||
ProfileBugBounty,
|
||||
ProfilePentest,
|
||||
ProfileASMContinuous,
|
||||
ProfileStealthMax,
|
||||
ProfileQuick,
|
||||
}
|
||||
|
||||
// ProfileByName returns the named profile, or ok=false when not found.
|
||||
func ProfileByName(name string) (Profile, bool) {
|
||||
for _, p := range BuiltinProfiles {
|
||||
if p.Name == name {
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
return Profile{}, false
|
||||
}
|
||||
|
||||
// ApplyProfile merges a profile into cfg. Existing non-zero values in cfg
|
||||
// take precedence (CLI flags win over profile defaults). Pointer-typed
|
||||
// profile fields are applied only when they are non-nil.
|
||||
func ApplyProfile(cfg *Config, p Profile) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if cfg.Concurrency == 0 || cfg.Concurrency == 1000 { // 1000 is the cobra default
|
||||
cfg.Concurrency = p.Concurrency
|
||||
}
|
||||
if cfg.Timeout == 0 || cfg.Timeout == 5 { // 5 is cobra default
|
||||
cfg.Timeout = p.Timeout
|
||||
}
|
||||
if cfg.StealthMode == "" {
|
||||
cfg.StealthMode = p.Stealth
|
||||
}
|
||||
|
||||
if p.AI != nil && !cfg.EnableAI {
|
||||
cfg.EnableAI = *p.AI
|
||||
}
|
||||
if p.MultiAgent != nil && !cfg.MultiAgent {
|
||||
cfg.MultiAgent = *p.MultiAgent
|
||||
}
|
||||
if p.Recursive != nil && !cfg.Recursive && !cfg.NoRecursive {
|
||||
cfg.Recursive = *p.Recursive
|
||||
}
|
||||
if p.NoBrute != nil && !cfg.NoBrute {
|
||||
cfg.NoBrute = *p.NoBrute
|
||||
}
|
||||
if p.NoProbe != nil && !cfg.NoProbe {
|
||||
cfg.NoProbe = *p.NoProbe
|
||||
}
|
||||
if p.NoPorts != nil && !cfg.NoPorts {
|
||||
cfg.NoPorts = *p.NoPorts
|
||||
}
|
||||
if p.NoTakeover != nil && !cfg.NoTakeover {
|
||||
cfg.NoTakeover = *p.NoTakeover
|
||||
}
|
||||
|
||||
applyPtrBool(&cfg.CloudScan, &cfg.NoCloudScan, p.CloudScan)
|
||||
applyPtrBool(&cfg.APIScan, &cfg.NoAPIScan, p.APIScan)
|
||||
applyPtrBool(&cfg.SecretsScan, &cfg.NoSecrets, p.SecretsScan)
|
||||
applyPtrBool(&cfg.TechScan, &cfg.NoTechScan, p.TechScan)
|
||||
applyPtrBool(&cfg.ASNScan, &cfg.NoASNScan, p.ASNScan)
|
||||
applyPtrBool(&cfg.VHostScan, &cfg.NoVHostScan, p.VHostScan)
|
||||
|
||||
// Module overrides
|
||||
if cfg.ModuleSettings == nil {
|
||||
cfg.ModuleSettings = make(map[string]bool)
|
||||
}
|
||||
for name, enabled := range p.Modules {
|
||||
if _, already := cfg.ModuleSettings[name]; !already {
|
||||
cfg.ModuleSettings[name] = enabled
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// applyPtrBool merges a ptr-bool from a profile into a (enabled, noEnabled)
|
||||
// pair on the Config struct. The v1 scheme uses two flags per feature
|
||||
// (Enable/NoEnable) to allow a three-state: unset/on/off. A nil profile ptr
|
||||
// means "leave unchanged"; *p=true enables unless user has set NoX; *p=false
|
||||
// leaves alone (profile doesn't force-off, user's explicit flag does).
|
||||
func applyPtrBool(enable, disable *bool, p *bool) {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
if *p && !*enable && !*disable {
|
||||
*enable = true
|
||||
}
|
||||
}
|
||||
|
||||
func ptrTrue() *bool { v := true; return &v }
|
||||
func ptrFalse() *bool { v := false; return &v }
|
||||
@@ -0,0 +1,143 @@
|
||||
package config
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestProfileByName(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
wantOK bool
|
||||
wantStr string
|
||||
}{
|
||||
{"bugbounty", "bugbounty", true, "bugbounty"},
|
||||
{"pentest", "pentest", true, "pentest"},
|
||||
{"asm-continuous", "asm-continuous", true, "asm-continuous"},
|
||||
{"stealth-max", "stealth-max", true, "stealth-max"},
|
||||
{"quick", "quick", true, "quick"},
|
||||
{"empty", "", false, ""},
|
||||
{"unknown", "nonsense", false, ""},
|
||||
{"case sensitive", "BugBounty", false, ""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, ok := ProfileByName(tt.input)
|
||||
if ok != tt.wantOK {
|
||||
t.Errorf("ok = %v, want %v", ok, tt.wantOK)
|
||||
}
|
||||
if ok && got.Name != tt.wantStr {
|
||||
t.Errorf("Name = %q, want %q", got.Name, tt.wantStr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltinProfiles_NonEmpty(t *testing.T) {
|
||||
if len(BuiltinProfiles) < 5 {
|
||||
t.Errorf("expected ≥5 built-in profiles, got %d", len(BuiltinProfiles))
|
||||
}
|
||||
seen := make(map[string]bool)
|
||||
for _, p := range BuiltinProfiles {
|
||||
if p.Name == "" {
|
||||
t.Error("profile with empty name")
|
||||
}
|
||||
if p.Description == "" {
|
||||
t.Errorf("profile %q has empty description", p.Name)
|
||||
}
|
||||
if seen[p.Name] {
|
||||
t.Errorf("duplicate profile name: %q", p.Name)
|
||||
}
|
||||
seen[p.Name] = true
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyProfile_NilConfigNoop(t *testing.T) {
|
||||
ApplyProfile(nil, ProfileBugBounty) // must not panic
|
||||
}
|
||||
|
||||
func TestApplyProfile_FillsDefaults(t *testing.T) {
|
||||
cfg := &Config{} // zero
|
||||
ApplyProfile(cfg, ProfileBugBounty)
|
||||
if cfg.Concurrency != ProfileBugBounty.Concurrency {
|
||||
t.Errorf("Concurrency = %d, want %d", cfg.Concurrency, ProfileBugBounty.Concurrency)
|
||||
}
|
||||
if cfg.Timeout != ProfileBugBounty.Timeout {
|
||||
t.Errorf("Timeout = %d, want %d", cfg.Timeout, ProfileBugBounty.Timeout)
|
||||
}
|
||||
if cfg.StealthMode != ProfileBugBounty.Stealth {
|
||||
t.Errorf("Stealth = %q, want %q", cfg.StealthMode, ProfileBugBounty.Stealth)
|
||||
}
|
||||
if !cfg.EnableAI {
|
||||
t.Error("bugbounty profile should enable AI")
|
||||
}
|
||||
if !cfg.MultiAgent {
|
||||
t.Error("bugbounty profile should enable MultiAgent")
|
||||
}
|
||||
if !cfg.Recursive {
|
||||
t.Error("bugbounty profile should enable Recursive")
|
||||
}
|
||||
if !cfg.CloudScan {
|
||||
t.Error("bugbounty profile should enable CloudScan")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyProfile_DoesNotOverrideExplicitFlags(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Concurrency: 42,
|
||||
Timeout: 999,
|
||||
StealthMode: "paranoid",
|
||||
EnableAI: false, // explicitly disabled before profile apply
|
||||
}
|
||||
// Apply bugbounty which normally enables AI + sets concurrency to 1000
|
||||
ApplyProfile(cfg, ProfileBugBounty)
|
||||
|
||||
// Explicit non-default user values should survive
|
||||
if cfg.Concurrency != 42 {
|
||||
t.Errorf("Concurrency overwritten: %d", cfg.Concurrency)
|
||||
}
|
||||
if cfg.Timeout != 999 {
|
||||
t.Errorf("Timeout overwritten: %d", cfg.Timeout)
|
||||
}
|
||||
if cfg.StealthMode != "paranoid" {
|
||||
t.Errorf("Stealth overwritten: %q", cfg.StealthMode)
|
||||
}
|
||||
// Profile AI enable should still apply since cfg.EnableAI was false
|
||||
// (we can't distinguish "user explicitly set false" from "zero value").
|
||||
// This is a known limitation documented in the CLI help.
|
||||
if !cfg.EnableAI {
|
||||
t.Errorf("AI not enabled by profile despite cfg.EnableAI being false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyProfile_NoForceOff(t *testing.T) {
|
||||
// stealth-max sets NoBrute=true. If user did NOT disable, profile wins.
|
||||
cfg := &Config{}
|
||||
ApplyProfile(cfg, ProfileStealthMax)
|
||||
if !cfg.NoBrute {
|
||||
t.Error("stealth-max profile should set NoBrute")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyProfile_ModuleSettings(t *testing.T) {
|
||||
p := Profile{
|
||||
Name: "custom",
|
||||
Modules: map[string]bool{
|
||||
"sources.crtsh": true,
|
||||
"brute": false,
|
||||
},
|
||||
}
|
||||
cfg := &Config{}
|
||||
ApplyProfile(cfg, p)
|
||||
if got := cfg.ModuleSettings["sources.crtsh"]; !got {
|
||||
t.Error("crtsh should be enabled")
|
||||
}
|
||||
if got := cfg.ModuleSettings["brute"]; got {
|
||||
t.Error("brute should be disabled")
|
||||
}
|
||||
|
||||
// User pre-existing setting must not be overridden
|
||||
cfg2 := &Config{ModuleSettings: map[string]bool{"sources.crtsh": false}}
|
||||
ApplyProfile(cfg2, p)
|
||||
if cfg2.ModuleSettings["sources.crtsh"] {
|
||||
t.Error("user explicit module setting was overridden")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package config
|
||||
|
||||
// View implements module.ConfigView over a *Config. Modules receive a View
|
||||
// (not the raw Config pointer) to prevent them from mutating global scan
|
||||
// state — reads only.
|
||||
//
|
||||
// The implementation is intentionally small: it exposes just the shape
|
||||
// needed by the module package without pulling in a full generic key/value
|
||||
// store. Module-specific settings live in ModuleSettings; typed options
|
||||
// should be hoisted to first-class fields on Config when they are used
|
||||
// across modules.
|
||||
type View struct {
|
||||
cfg *Config
|
||||
}
|
||||
|
||||
// NewView wraps cfg as a ConfigView. cfg may be nil, in which case every
|
||||
// accessor returns the fallback/zero value.
|
||||
func NewView(cfg *Config) *View { return &View{cfg: cfg} }
|
||||
|
||||
// Profile returns the active profile name ("" when none).
|
||||
func (v *View) Profile() string {
|
||||
if v == nil || v.cfg == nil {
|
||||
return ""
|
||||
}
|
||||
return v.cfg.Profile
|
||||
}
|
||||
|
||||
// Bool reads a boolean config key by well-known name. Unknown keys return fb.
|
||||
// Keys intentionally kept flat to avoid accidental namespacing bugs.
|
||||
func (v *View) Bool(key string, fb bool) bool {
|
||||
if v == nil || v.cfg == nil {
|
||||
return fb
|
||||
}
|
||||
switch key {
|
||||
case "ai.enabled":
|
||||
return v.cfg.EnableAI
|
||||
case "ai.cascade":
|
||||
return v.cfg.AICascade
|
||||
case "ai.deep":
|
||||
return v.cfg.AIDeepAnalysis
|
||||
case "ai.multi_agent":
|
||||
return v.cfg.MultiAgent
|
||||
case "ai.verbose":
|
||||
return v.cfg.AIVerbose
|
||||
case "ai.auto_pull":
|
||||
return v.cfg.AutoPullModels
|
||||
case "silent":
|
||||
return v.cfg.Silent
|
||||
case "verbose":
|
||||
return v.cfg.Verbose
|
||||
case "json":
|
||||
return v.cfg.JsonOutput
|
||||
case "no_brute":
|
||||
return v.cfg.NoBrute
|
||||
case "no_probe":
|
||||
return v.cfg.NoProbe
|
||||
case "no_ports":
|
||||
return v.cfg.NoPorts
|
||||
case "no_takeover":
|
||||
return v.cfg.NoTakeover
|
||||
case "only_active":
|
||||
return v.cfg.OnlyActive
|
||||
case "recursive":
|
||||
return v.cfg.Recursive
|
||||
case "cloud_scan":
|
||||
return v.cfg.CloudScan
|
||||
case "api_scan":
|
||||
return v.cfg.APIScan
|
||||
case "secrets_scan":
|
||||
return v.cfg.SecretsScan
|
||||
case "tech_scan":
|
||||
return v.cfg.TechScan
|
||||
case "asn_scan":
|
||||
return v.cfg.ASNScan
|
||||
case "vhost_scan":
|
||||
return v.cfg.VHostScan
|
||||
case "nuclei_scan":
|
||||
return v.cfg.NucleiScan
|
||||
case "nuclei_auto_download":
|
||||
return v.cfg.NucleiAutoDownload
|
||||
}
|
||||
return fb
|
||||
}
|
||||
|
||||
// Int reads an int key.
|
||||
func (v *View) Int(key string, fb int) int {
|
||||
if v == nil || v.cfg == nil {
|
||||
return fb
|
||||
}
|
||||
switch key {
|
||||
case "concurrency":
|
||||
return v.cfg.Concurrency
|
||||
case "timeout":
|
||||
return v.cfg.Timeout
|
||||
case "recursive.depth":
|
||||
return v.cfg.RecursiveDepth
|
||||
}
|
||||
return fb
|
||||
}
|
||||
|
||||
// String reads a string key.
|
||||
func (v *View) String(key string, fb string) string {
|
||||
if v == nil || v.cfg == nil {
|
||||
return fb
|
||||
}
|
||||
switch key {
|
||||
case "domain":
|
||||
return v.cfg.Domain
|
||||
case "wordlist":
|
||||
return v.cfg.Wordlist
|
||||
case "output":
|
||||
return v.cfg.Output
|
||||
case "format":
|
||||
return v.cfg.Format
|
||||
case "ports":
|
||||
return v.cfg.Ports
|
||||
case "resolvers":
|
||||
return v.cfg.Resolvers
|
||||
case "stealth":
|
||||
return v.cfg.StealthMode
|
||||
case "ai.url":
|
||||
return v.cfg.AIUrl
|
||||
case "ai.fast_model":
|
||||
return v.cfg.AIFastModel
|
||||
case "ai.deep_model":
|
||||
return v.cfg.AIDeepModel
|
||||
case "nuclei_templates":
|
||||
return v.cfg.NucleiTemplates
|
||||
}
|
||||
return fb
|
||||
}
|
||||
|
||||
// Strings reads a string-slice key. No multi-value keys are defined yet,
|
||||
// but reserved for module-specific settings loaded from YAML.
|
||||
func (v *View) Strings(key string) []string {
|
||||
_ = key
|
||||
return nil
|
||||
}
|
||||
|
||||
// ModuleEnabled returns true when the config explicitly enabled the module
|
||||
// by name (via ModuleSettings). It returns false otherwise; callers should
|
||||
// fall back to the module's DefaultEnabled() when this returns false.
|
||||
func (v *View) ModuleEnabled(name string) bool {
|
||||
if v == nil || v.cfg == nil {
|
||||
return false
|
||||
}
|
||||
if v.cfg.ModuleSettings == nil {
|
||||
return false
|
||||
}
|
||||
return v.cfg.ModuleSettings[name]
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package config
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestView_NilSafe(t *testing.T) {
|
||||
var v *View
|
||||
if v.Profile() != "" {
|
||||
t.Error("nil view Profile should be empty")
|
||||
}
|
||||
if v.Bool("ai.enabled", true) != true {
|
||||
t.Error("nil view Bool should return fallback")
|
||||
}
|
||||
if v.Int("concurrency", 99) != 99 {
|
||||
t.Error("nil view Int should return fallback")
|
||||
}
|
||||
if v.String("domain", "fb") != "fb" {
|
||||
t.Error("nil view String should return fallback")
|
||||
}
|
||||
if v.ModuleEnabled("x") {
|
||||
t.Error("nil view ModuleEnabled should be false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestView_Profile(t *testing.T) {
|
||||
v := NewView(&Config{Profile: "bugbounty"})
|
||||
if v.Profile() != "bugbounty" {
|
||||
t.Errorf("Profile = %q", v.Profile())
|
||||
}
|
||||
}
|
||||
|
||||
func TestView_Bool(t *testing.T) {
|
||||
cfg := &Config{
|
||||
EnableAI: true,
|
||||
AICascade: true,
|
||||
AIDeepAnalysis: false,
|
||||
MultiAgent: true,
|
||||
Silent: true,
|
||||
Verbose: false,
|
||||
JsonOutput: true,
|
||||
NoBrute: true,
|
||||
OnlyActive: true,
|
||||
Recursive: true,
|
||||
CloudScan: true,
|
||||
APIScan: false,
|
||||
}
|
||||
v := NewView(cfg)
|
||||
|
||||
tests := []struct {
|
||||
key string
|
||||
fb bool
|
||||
want bool
|
||||
}{
|
||||
{"ai.enabled", false, true},
|
||||
{"ai.cascade", false, true},
|
||||
{"ai.deep", true, false},
|
||||
{"ai.multi_agent", false, true},
|
||||
{"silent", false, true},
|
||||
{"verbose", true, false},
|
||||
{"json", false, true},
|
||||
{"no_brute", false, true},
|
||||
{"only_active", false, true},
|
||||
{"recursive", false, true},
|
||||
{"cloud_scan", false, true},
|
||||
{"api_scan", true, false},
|
||||
{"unknown_key", true, true}, // fallback
|
||||
{"unknown_key", false, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
if got := v.Bool(tt.key, tt.fb); got != tt.want {
|
||||
t.Errorf("Bool(%q, %v) = %v, want %v", tt.key, tt.fb, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestView_Int(t *testing.T) {
|
||||
v := NewView(&Config{Concurrency: 500, Timeout: 10, RecursiveDepth: 4})
|
||||
if v.Int("concurrency", 1) != 500 {
|
||||
t.Errorf("concurrency wrong")
|
||||
}
|
||||
if v.Int("timeout", 1) != 10 {
|
||||
t.Errorf("timeout wrong")
|
||||
}
|
||||
if v.Int("recursive.depth", 1) != 4 {
|
||||
t.Errorf("recursive.depth wrong")
|
||||
}
|
||||
if v.Int("unknown", 99) != 99 {
|
||||
t.Errorf("unknown key should return fallback")
|
||||
}
|
||||
}
|
||||
|
||||
func TestView_String(t *testing.T) {
|
||||
v := NewView(&Config{
|
||||
Domain: "example.com",
|
||||
Wordlist: "/wl",
|
||||
Output: "/out",
|
||||
Format: "json",
|
||||
Ports: "80,443",
|
||||
Resolvers: "8.8.8.8",
|
||||
StealthMode: "light",
|
||||
AIUrl: "http://x",
|
||||
AIFastModel: "f",
|
||||
AIDeepModel: "d",
|
||||
})
|
||||
|
||||
cases := map[string]string{
|
||||
"domain": "example.com",
|
||||
"wordlist": "/wl",
|
||||
"output": "/out",
|
||||
"format": "json",
|
||||
"ports": "80,443",
|
||||
"resolvers": "8.8.8.8",
|
||||
"stealth": "light",
|
||||
"ai.url": "http://x",
|
||||
"ai.fast_model": "f",
|
||||
"ai.deep_model": "d",
|
||||
}
|
||||
for k, want := range cases {
|
||||
if got := v.String(k, "fb"); got != want {
|
||||
t.Errorf("String(%q) = %q, want %q", k, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
if v.String("unknown", "fb") != "fb" {
|
||||
t.Error("unknown key should return fallback")
|
||||
}
|
||||
}
|
||||
|
||||
func TestView_Strings(t *testing.T) {
|
||||
// Placeholder — no multi-value keys defined yet
|
||||
v := NewView(&Config{})
|
||||
if got := v.Strings("anything"); got != nil {
|
||||
t.Errorf("expected nil, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestView_ModuleEnabled(t *testing.T) {
|
||||
cfg := &Config{ModuleSettings: map[string]bool{"m1": true, "m2": false}}
|
||||
v := NewView(cfg)
|
||||
if !v.ModuleEnabled("m1") {
|
||||
t.Error("m1 should be enabled")
|
||||
}
|
||||
if v.ModuleEnabled("m2") {
|
||||
t.Error("m2 should be disabled (false in map)")
|
||||
}
|
||||
if v.ModuleEnabled("unset") {
|
||||
t.Error("unset module should be false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestView_ModuleEnabled_NilMap(t *testing.T) {
|
||||
v := NewView(&Config{})
|
||||
if v.ModuleEnabled("anything") {
|
||||
t.Error("nil map should result in false")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// YAMLConfig is the schema persisted on disk. Fields are intentionally a
|
||||
// subset of Config — YAML is for declarative, long-lived settings
|
||||
// (profile, module toggles, resolver lists, AI model names); ephemeral
|
||||
// flags (--silent, --verbose, --domain) remain CLI-only.
|
||||
type YAMLConfig struct {
|
||||
Profile string `yaml:"profile,omitempty"`
|
||||
Concurrency int `yaml:"concurrency,omitempty"`
|
||||
Timeout int `yaml:"timeout,omitempty"`
|
||||
Stealth string `yaml:"stealth,omitempty"`
|
||||
Resolvers []string `yaml:"resolvers,omitempty"`
|
||||
Wordlist string `yaml:"wordlist,omitempty"`
|
||||
Modules map[string]bool `yaml:"modules,omitempty"`
|
||||
AI *YAMLAIConfig `yaml:"ai,omitempty"`
|
||||
Output *YAMLOutputConfig `yaml:"output,omitempty"`
|
||||
}
|
||||
|
||||
// YAMLAIConfig groups AI-related YAML fields.
|
||||
type YAMLAIConfig struct {
|
||||
Enabled bool `yaml:"enabled,omitempty"`
|
||||
URL string `yaml:"url,omitempty"`
|
||||
FastModel string `yaml:"fast_model,omitempty"`
|
||||
DeepModel string `yaml:"deep_model,omitempty"`
|
||||
Cascade *bool `yaml:"cascade,omitempty"`
|
||||
Deep bool `yaml:"deep,omitempty"`
|
||||
MultiAgent bool `yaml:"multi_agent,omitempty"`
|
||||
}
|
||||
|
||||
// YAMLOutputConfig groups output-related YAML fields.
|
||||
type YAMLOutputConfig struct {
|
||||
Path string `yaml:"path,omitempty"`
|
||||
Format string `yaml:"format,omitempty"`
|
||||
JSON bool `yaml:"json,omitempty"`
|
||||
}
|
||||
|
||||
// LoadYAML reads a YAML config file from path and returns the parsed config.
|
||||
// Returns (nil, nil) when the file does not exist — callers should treat this
|
||||
// as "no config file, use defaults". Returns an error for any other I/O or
|
||||
// parse failure.
|
||||
func LoadYAML(path string) (*YAMLConfig, error) {
|
||||
if path == "" {
|
||||
return nil, nil
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("read config %q: %w", path, err)
|
||||
}
|
||||
|
||||
var y YAMLConfig
|
||||
if err := yaml.Unmarshal(data, &y); err != nil {
|
||||
return nil, fmt.Errorf("parse config %q: %w", path, err)
|
||||
}
|
||||
return &y, nil
|
||||
}
|
||||
|
||||
// ApplyYAML merges a parsed YAML config into cfg. CLI flags win: YAML only
|
||||
// fills fields that are still at their zero value on cfg. The profile named
|
||||
// in YAML is applied only if cfg.Profile is empty.
|
||||
func ApplyYAML(cfg *Config, y *YAMLConfig) {
|
||||
if cfg == nil || y == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if cfg.Profile == "" && y.Profile != "" {
|
||||
cfg.Profile = y.Profile
|
||||
}
|
||||
if cfg.Concurrency == 0 && y.Concurrency > 0 {
|
||||
cfg.Concurrency = y.Concurrency
|
||||
}
|
||||
if cfg.Timeout == 0 && y.Timeout > 0 {
|
||||
cfg.Timeout = y.Timeout
|
||||
}
|
||||
if cfg.StealthMode == "" && y.Stealth != "" {
|
||||
cfg.StealthMode = y.Stealth
|
||||
}
|
||||
if cfg.Resolvers == "" && len(y.Resolvers) > 0 {
|
||||
cfg.Resolvers = joinComma(y.Resolvers)
|
||||
}
|
||||
if cfg.Wordlist == "" && y.Wordlist != "" {
|
||||
cfg.Wordlist = y.Wordlist
|
||||
}
|
||||
|
||||
if len(y.Modules) > 0 {
|
||||
if cfg.ModuleSettings == nil {
|
||||
cfg.ModuleSettings = make(map[string]bool)
|
||||
}
|
||||
for name, enabled := range y.Modules {
|
||||
if _, already := cfg.ModuleSettings[name]; !already {
|
||||
cfg.ModuleSettings[name] = enabled
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if y.AI != nil {
|
||||
if y.AI.Enabled && !cfg.EnableAI {
|
||||
cfg.EnableAI = true
|
||||
}
|
||||
if cfg.AIUrl == "" && y.AI.URL != "" {
|
||||
cfg.AIUrl = y.AI.URL
|
||||
}
|
||||
if cfg.AIFastModel == "" && y.AI.FastModel != "" {
|
||||
cfg.AIFastModel = y.AI.FastModel
|
||||
}
|
||||
if cfg.AIDeepModel == "" && y.AI.DeepModel != "" {
|
||||
cfg.AIDeepModel = y.AI.DeepModel
|
||||
}
|
||||
if y.AI.Cascade != nil && !cfg.AICascade {
|
||||
cfg.AICascade = *y.AI.Cascade
|
||||
}
|
||||
if y.AI.Deep && !cfg.AIDeepAnalysis {
|
||||
cfg.AIDeepAnalysis = true
|
||||
}
|
||||
if y.AI.MultiAgent && !cfg.MultiAgent {
|
||||
cfg.MultiAgent = true
|
||||
}
|
||||
}
|
||||
|
||||
if y.Output != nil {
|
||||
if cfg.Output == "" && y.Output.Path != "" {
|
||||
cfg.Output = y.Output.Path
|
||||
}
|
||||
if cfg.Format == "" && y.Output.Format != "" {
|
||||
cfg.Format = y.Output.Format
|
||||
}
|
||||
if y.Output.JSON && !cfg.JsonOutput {
|
||||
cfg.JsonOutput = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultConfigPaths returns the ordered list of paths LoadYAML scans by
|
||||
// default when no --config is provided. The first existing file wins.
|
||||
func DefaultConfigPaths() []string {
|
||||
home, err := os.UserHomeDir()
|
||||
var homeCfg string
|
||||
if err == nil {
|
||||
homeCfg = filepath.Join(home, ".god-eye", "config.yaml")
|
||||
}
|
||||
return []string{
|
||||
"god-eye.yaml",
|
||||
".god-eye.yaml",
|
||||
homeCfg,
|
||||
}
|
||||
}
|
||||
|
||||
// FindConfigFile returns the first existing file in DefaultConfigPaths, or
|
||||
// "" if none is found.
|
||||
func FindConfigFile() string {
|
||||
for _, p := range DefaultConfigPaths() {
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := os.Stat(p); err == nil {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func joinComma(ss []string) string {
|
||||
out := ""
|
||||
for i, s := range ss {
|
||||
if i > 0 {
|
||||
out += ","
|
||||
}
|
||||
out += s
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLoadYAML_Missing(t *testing.T) {
|
||||
y, err := LoadYAML("/tmp/this-definitely-does-not-exist-xyz.yaml")
|
||||
if err != nil {
|
||||
t.Errorf("missing file should return nil error, got %v", err)
|
||||
}
|
||||
if y != nil {
|
||||
t.Errorf("missing file should return nil config, got %+v", y)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadYAML_EmptyPath(t *testing.T) {
|
||||
y, err := LoadYAML("")
|
||||
if y != nil || err != nil {
|
||||
t.Errorf("empty path → (nil, nil), got (%+v, %v)", y, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadYAML_Malformed(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "bad.yaml")
|
||||
os.WriteFile(path, []byte("profile: [unclosed"), 0o644)
|
||||
_, err := LoadYAML(path)
|
||||
if err == nil {
|
||||
t.Error("expected parse error for malformed YAML")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadYAML_Full(t *testing.T) {
|
||||
content := `
|
||||
profile: bugbounty
|
||||
concurrency: 500
|
||||
timeout: 8
|
||||
stealth: moderate
|
||||
resolvers:
|
||||
- 8.8.8.8
|
||||
- 1.1.1.1
|
||||
wordlist: /tmp/wl.txt
|
||||
modules:
|
||||
sources.crtsh: true
|
||||
brute: false
|
||||
ai:
|
||||
enabled: true
|
||||
url: http://localhost:11434
|
||||
fast_model: qwen3:1.7b
|
||||
deep_model: qwen2.5-coder:14b
|
||||
cascade: true
|
||||
deep: true
|
||||
multi_agent: true
|
||||
output:
|
||||
path: /tmp/out.json
|
||||
format: json
|
||||
json: true
|
||||
`
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "config.yaml")
|
||||
os.WriteFile(path, []byte(content), 0o644)
|
||||
|
||||
y, err := LoadYAML(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if y == nil {
|
||||
t.Fatal("expected non-nil config")
|
||||
}
|
||||
if y.Profile != "bugbounty" {
|
||||
t.Errorf("Profile = %q", y.Profile)
|
||||
}
|
||||
if y.Concurrency != 500 {
|
||||
t.Errorf("Concurrency = %d", y.Concurrency)
|
||||
}
|
||||
if y.Timeout != 8 {
|
||||
t.Errorf("Timeout = %d", y.Timeout)
|
||||
}
|
||||
if y.Stealth != "moderate" {
|
||||
t.Errorf("Stealth = %q", y.Stealth)
|
||||
}
|
||||
if len(y.Resolvers) != 2 {
|
||||
t.Errorf("Resolvers len = %d", len(y.Resolvers))
|
||||
}
|
||||
if y.Modules["sources.crtsh"] != true {
|
||||
t.Errorf("modules.sources.crtsh = false")
|
||||
}
|
||||
if y.Modules["brute"] != false {
|
||||
t.Errorf("modules.brute = true")
|
||||
}
|
||||
if y.AI == nil || !y.AI.Enabled {
|
||||
t.Error("AI not enabled")
|
||||
}
|
||||
if y.AI.URL != "http://localhost:11434" {
|
||||
t.Errorf("AI.URL = %q", y.AI.URL)
|
||||
}
|
||||
if y.Output == nil || y.Output.Path != "/tmp/out.json" {
|
||||
t.Errorf("Output.Path wrong")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyYAML_NilInputs(t *testing.T) {
|
||||
ApplyYAML(nil, &YAMLConfig{Profile: "x"}) // must not panic
|
||||
ApplyYAML(&Config{}, nil) // must not panic
|
||||
}
|
||||
|
||||
func TestApplyYAML_FillsZeroFields(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
y := &YAMLConfig{
|
||||
Profile: "quick",
|
||||
Concurrency: 123,
|
||||
Timeout: 7,
|
||||
Stealth: "light",
|
||||
Resolvers: []string{"8.8.8.8", "1.1.1.1"},
|
||||
Wordlist: "/tmp/wl",
|
||||
Modules: map[string]bool{"m1": true},
|
||||
AI: &YAMLAIConfig{
|
||||
Enabled: true,
|
||||
URL: "http://x",
|
||||
FastModel: "f",
|
||||
DeepModel: "d",
|
||||
Cascade: ptrTrue(),
|
||||
Deep: true,
|
||||
MultiAgent: true,
|
||||
},
|
||||
Output: &YAMLOutputConfig{Path: "/o", Format: "json", JSON: true},
|
||||
}
|
||||
ApplyYAML(cfg, y)
|
||||
|
||||
if cfg.Profile != "quick" {
|
||||
t.Errorf("Profile = %q", cfg.Profile)
|
||||
}
|
||||
if cfg.Concurrency != 123 {
|
||||
t.Errorf("Concurrency = %d", cfg.Concurrency)
|
||||
}
|
||||
if cfg.Timeout != 7 {
|
||||
t.Errorf("Timeout = %d", cfg.Timeout)
|
||||
}
|
||||
if cfg.StealthMode != "light" {
|
||||
t.Errorf("StealthMode = %q", cfg.StealthMode)
|
||||
}
|
||||
if cfg.Resolvers != "8.8.8.8,1.1.1.1" {
|
||||
t.Errorf("Resolvers = %q", cfg.Resolvers)
|
||||
}
|
||||
if cfg.Wordlist != "/tmp/wl" {
|
||||
t.Errorf("Wordlist = %q", cfg.Wordlist)
|
||||
}
|
||||
if !cfg.EnableAI {
|
||||
t.Error("EnableAI should be true")
|
||||
}
|
||||
if cfg.AIUrl != "http://x" {
|
||||
t.Errorf("AIUrl = %q", cfg.AIUrl)
|
||||
}
|
||||
if !cfg.AICascade {
|
||||
t.Error("AICascade should be true")
|
||||
}
|
||||
if !cfg.AIDeepAnalysis {
|
||||
t.Error("AIDeepAnalysis should be true")
|
||||
}
|
||||
if !cfg.MultiAgent {
|
||||
t.Error("MultiAgent should be true")
|
||||
}
|
||||
if cfg.Output != "/o" {
|
||||
t.Errorf("Output = %q", cfg.Output)
|
||||
}
|
||||
if cfg.Format != "json" {
|
||||
t.Errorf("Format = %q", cfg.Format)
|
||||
}
|
||||
if !cfg.JsonOutput {
|
||||
t.Error("JsonOutput should be true")
|
||||
}
|
||||
if cfg.ModuleSettings["m1"] != true {
|
||||
t.Error("ModuleSettings.m1 should be true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyYAML_CLIOverrideWins(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Profile: "pentest",
|
||||
Concurrency: 42,
|
||||
Timeout: 3,
|
||||
StealthMode: "paranoid",
|
||||
Resolvers: "9.9.9.9",
|
||||
Wordlist: "/existing",
|
||||
}
|
||||
y := &YAMLConfig{
|
||||
Profile: "quick",
|
||||
Concurrency: 999,
|
||||
Timeout: 999,
|
||||
Stealth: "off",
|
||||
Resolvers: []string{"8.8.8.8"},
|
||||
Wordlist: "/yaml",
|
||||
}
|
||||
ApplyYAML(cfg, y)
|
||||
|
||||
// CLI values should survive
|
||||
if cfg.Profile != "pentest" {
|
||||
t.Errorf("Profile overwritten: %q", cfg.Profile)
|
||||
}
|
||||
if cfg.Concurrency != 42 {
|
||||
t.Errorf("Concurrency overwritten: %d", cfg.Concurrency)
|
||||
}
|
||||
if cfg.Timeout != 3 {
|
||||
t.Errorf("Timeout overwritten: %d", cfg.Timeout)
|
||||
}
|
||||
if cfg.StealthMode != "paranoid" {
|
||||
t.Errorf("StealthMode overwritten: %q", cfg.StealthMode)
|
||||
}
|
||||
if cfg.Resolvers != "9.9.9.9" {
|
||||
t.Errorf("Resolvers overwritten: %q", cfg.Resolvers)
|
||||
}
|
||||
if cfg.Wordlist != "/existing" {
|
||||
t.Errorf("Wordlist overwritten: %q", cfg.Wordlist)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultConfigPaths(t *testing.T) {
|
||||
paths := DefaultConfigPaths()
|
||||
if len(paths) < 3 {
|
||||
t.Errorf("expected ≥3 default paths, got %d", len(paths))
|
||||
}
|
||||
// First two are CWD-relative
|
||||
if paths[0] != "god-eye.yaml" {
|
||||
t.Errorf("paths[0] = %q", paths[0])
|
||||
}
|
||||
if paths[1] != ".god-eye.yaml" {
|
||||
t.Errorf("paths[1] = %q", paths[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindConfigFile_FindsInWorkingDir(t *testing.T) {
|
||||
// Create a temp "god-eye.yaml" and ensure the search finds it. We can't
|
||||
// easily change CWD for just this test, so we validate the underlying
|
||||
// Stat call by constructing a path that definitely exists.
|
||||
dir := t.TempDir()
|
||||
target := filepath.Join(dir, "god-eye.yaml")
|
||||
os.WriteFile(target, []byte("profile: quick\n"), 0o644)
|
||||
|
||||
oldWD, _ := os.Getwd()
|
||||
defer os.Chdir(oldWD)
|
||||
if err := os.Chdir(dir); err != nil {
|
||||
t.Skipf("cannot chdir: %v", err)
|
||||
}
|
||||
|
||||
got := FindConfigFile()
|
||||
if got != "god-eye.yaml" {
|
||||
t.Errorf("FindConfigFile = %q, want god-eye.yaml", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindConfigFile_NoneFound(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
oldWD, _ := os.Getwd()
|
||||
defer os.Chdir(oldWD)
|
||||
if err := os.Chdir(dir); err != nil {
|
||||
t.Skipf("cannot chdir: %v", err)
|
||||
}
|
||||
// Also override HOME to an empty dir so the user-home path never matches.
|
||||
oldHome := os.Getenv("HOME")
|
||||
defer os.Setenv("HOME", oldHome)
|
||||
os.Setenv("HOME", dir)
|
||||
|
||||
got := FindConfigFile()
|
||||
if got != "" {
|
||||
t.Errorf("FindConfigFile = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
// Package diff computes deltas between two scans of the same target. It
|
||||
// powers Fase 5's asm-continuous mode: run the scanner on a schedule, diff
|
||||
// against the last snapshot, alert on meaningful changes.
|
||||
//
|
||||
// Diff categories:
|
||||
//
|
||||
// new_host — subdomain not seen before
|
||||
// removed_host — subdomain vanished from discovery
|
||||
// new_ip — host gained an IP
|
||||
// removed_ip — host lost an IP
|
||||
// status_change — HTTP status code changed (200→401, 200→gone)
|
||||
// tech_change — technology stack changed (upgrade or new framework)
|
||||
// new_vuln — new vulnerability finding
|
||||
// cleared_vuln — previously-reported vuln no longer detected
|
||||
// cert_change — TLS certificate issuer/expiry changed
|
||||
// new_takeover — new takeover candidate
|
||||
//
|
||||
// A Report is consumable both by humans (pretty-print) and by alerters
|
||||
// (Slack/webhook payload shape defined later in F5.3).
|
||||
package diff
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
// Change is one delta.
|
||||
type Change struct {
|
||||
Kind string `json:"kind"`
|
||||
Host string `json:"host"`
|
||||
Before string `json:"before,omitempty"`
|
||||
After string `json:"after,omitempty"`
|
||||
Severity string `json:"severity,omitempty"`
|
||||
Detected time.Time `json:"detected_at"`
|
||||
}
|
||||
|
||||
// Report is the full delta between two scans.
|
||||
type Report struct {
|
||||
Target string `json:"target"`
|
||||
OldAt time.Time `json:"old_scan_at"`
|
||||
NewAt time.Time `json:"new_scan_at"`
|
||||
Changes []Change `json:"changes"`
|
||||
}
|
||||
|
||||
// HasMeaningful returns true when the report contains any change that
|
||||
// warrants alerting. "new_host" and any "new_vuln" always qualify.
|
||||
func (r *Report) HasMeaningful() bool {
|
||||
for _, c := range r.Changes {
|
||||
switch c.Kind {
|
||||
case "new_host", "new_vuln", "new_takeover", "removed_host":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Compute compares old vs new snapshots and returns the delta. Both
|
||||
// slices are assumed to come from store.All() (sorted by subdomain).
|
||||
func Compute(target string, oldHosts, newHosts []*store.Host, oldAt, newAt time.Time) *Report {
|
||||
r := &Report{Target: target, OldAt: oldAt, NewAt: newAt}
|
||||
|
||||
oldByName := indexHosts(oldHosts)
|
||||
newByName := indexHosts(newHosts)
|
||||
|
||||
// Walk the union of hostnames.
|
||||
names := union(oldByName, newByName)
|
||||
sort.Strings(names)
|
||||
|
||||
for _, name := range names {
|
||||
o, oOK := oldByName[name]
|
||||
n, nOK := newByName[name]
|
||||
switch {
|
||||
case !oOK && nOK:
|
||||
r.Changes = append(r.Changes, Change{Kind: "new_host", Host: name, Detected: newAt})
|
||||
for _, v := range n.Vulnerabilities {
|
||||
r.Changes = append(r.Changes, Change{
|
||||
Kind: "new_vuln",
|
||||
Host: name,
|
||||
After: v.Title,
|
||||
Severity: v.Severity,
|
||||
Detected: newAt,
|
||||
})
|
||||
}
|
||||
if n.Takeover != nil {
|
||||
r.Changes = append(r.Changes, Change{
|
||||
Kind: "new_takeover",
|
||||
Host: name,
|
||||
After: n.Takeover.Service,
|
||||
Severity: "high",
|
||||
Detected: newAt,
|
||||
})
|
||||
}
|
||||
case oOK && !nOK:
|
||||
r.Changes = append(r.Changes, Change{Kind: "removed_host", Host: name, Detected: newAt})
|
||||
case oOK && nOK:
|
||||
r.Changes = append(r.Changes, diffHost(o, n, newAt)...)
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func diffHost(o, n *store.Host, at time.Time) []Change {
|
||||
var out []Change
|
||||
|
||||
if o.StatusCode != n.StatusCode {
|
||||
out = append(out, Change{
|
||||
Kind: "status_change",
|
||||
Host: n.Subdomain,
|
||||
Before: itoa(o.StatusCode),
|
||||
After: itoa(n.StatusCode),
|
||||
Detected: at,
|
||||
})
|
||||
}
|
||||
|
||||
// IP deltas
|
||||
oldIPs := toSet(o.IPs)
|
||||
newIPs := toSet(n.IPs)
|
||||
for ip := range newIPs {
|
||||
if _, present := oldIPs[ip]; !present {
|
||||
out = append(out, Change{Kind: "new_ip", Host: n.Subdomain, After: ip, Detected: at})
|
||||
}
|
||||
}
|
||||
for ip := range oldIPs {
|
||||
if _, present := newIPs[ip]; !present {
|
||||
out = append(out, Change{Kind: "removed_ip", Host: n.Subdomain, Before: ip, Detected: at})
|
||||
}
|
||||
}
|
||||
|
||||
// Tech change (set inequality)
|
||||
if !stringSetsEqual(o.Technologies, n.Technologies) {
|
||||
out = append(out, Change{
|
||||
Kind: "tech_change",
|
||||
Host: n.Subdomain,
|
||||
Before: joinSorted(o.Technologies),
|
||||
After: joinSorted(n.Technologies),
|
||||
Detected: at,
|
||||
})
|
||||
}
|
||||
|
||||
// Vuln delta (by ID)
|
||||
oldVulns := indexVulns(o.Vulnerabilities)
|
||||
newVulns := indexVulns(n.Vulnerabilities)
|
||||
for id, v := range newVulns {
|
||||
if _, present := oldVulns[id]; !present {
|
||||
out = append(out, Change{
|
||||
Kind: "new_vuln", Host: n.Subdomain, After: v.Title,
|
||||
Severity: v.Severity, Detected: at,
|
||||
})
|
||||
}
|
||||
}
|
||||
for id, v := range oldVulns {
|
||||
if _, present := newVulns[id]; !present {
|
||||
out = append(out, Change{
|
||||
Kind: "cleared_vuln", Host: n.Subdomain, Before: v.Title,
|
||||
Severity: v.Severity, Detected: at,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Certificate change
|
||||
if o.TLSIssuer != n.TLSIssuer && n.TLSIssuer != "" {
|
||||
out = append(out, Change{
|
||||
Kind: "cert_change",
|
||||
Host: n.Subdomain,
|
||||
Before: o.TLSIssuer,
|
||||
After: n.TLSIssuer,
|
||||
Detected: at,
|
||||
})
|
||||
}
|
||||
|
||||
// Takeover appeared
|
||||
if o.Takeover == nil && n.Takeover != nil {
|
||||
out = append(out, Change{
|
||||
Kind: "new_takeover", Host: n.Subdomain,
|
||||
After: n.Takeover.Service, Severity: "high", Detected: at,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- helpers -------------------------------------------------------------
|
||||
|
||||
func indexHosts(hs []*store.Host) map[string]*store.Host {
|
||||
out := make(map[string]*store.Host, len(hs))
|
||||
for _, h := range hs {
|
||||
out[h.Subdomain] = h
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func indexVulns(vs []store.Vulnerability) map[string]store.Vulnerability {
|
||||
out := make(map[string]store.Vulnerability, len(vs))
|
||||
for _, v := range vs {
|
||||
out[v.ID] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func union(a, b map[string]*store.Host) []string {
|
||||
out := make(map[string]struct{}, len(a)+len(b))
|
||||
for k := range a {
|
||||
out[k] = struct{}{}
|
||||
}
|
||||
for k := range b {
|
||||
out[k] = struct{}{}
|
||||
}
|
||||
names := make([]string, 0, len(out))
|
||||
for n := range out {
|
||||
names = append(names, n)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func toSet(ss []string) map[string]struct{} {
|
||||
out := make(map[string]struct{}, len(ss))
|
||||
for _, s := range ss {
|
||||
out[s] = struct{}{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func stringSetsEqual(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
sa := toSet(a)
|
||||
for _, s := range b {
|
||||
if _, ok := sa[s]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func joinSorted(s []string) string {
|
||||
cpy := append([]string(nil), s...)
|
||||
sort.Strings(cpy)
|
||||
out := ""
|
||||
for i, v := range cpy {
|
||||
if i > 0 {
|
||||
out += ","
|
||||
}
|
||||
out += v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func itoa(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
var buf [20]byte
|
||||
i := len(buf)
|
||||
neg := n < 0
|
||||
if neg {
|
||||
n = -n
|
||||
}
|
||||
for n > 0 {
|
||||
i--
|
||||
buf[i] = byte('0' + n%10)
|
||||
n /= 10
|
||||
}
|
||||
if neg {
|
||||
i--
|
||||
buf[i] = '-'
|
||||
}
|
||||
return string(buf[i:])
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package diff
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
func TestCompute_NewHost(t *testing.T) {
|
||||
oldHosts := []*store.Host{}
|
||||
newHosts := []*store.Host{{Subdomain: "api.example.com"}}
|
||||
r := Compute("example.com", oldHosts, newHosts, time.Now(), time.Now())
|
||||
if len(r.Changes) != 1 || r.Changes[0].Kind != "new_host" {
|
||||
t.Errorf("expected 1 new_host change, got %+v", r.Changes)
|
||||
}
|
||||
if !r.HasMeaningful() {
|
||||
t.Error("new_host should be meaningful")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompute_RemovedHost(t *testing.T) {
|
||||
oldHosts := []*store.Host{{Subdomain: "old.example.com"}}
|
||||
newHosts := []*store.Host{}
|
||||
r := Compute("example.com", oldHosts, newHosts, time.Now(), time.Now())
|
||||
if len(r.Changes) != 1 || r.Changes[0].Kind != "removed_host" {
|
||||
t.Errorf("expected removed_host, got %+v", r.Changes)
|
||||
}
|
||||
if !r.HasMeaningful() {
|
||||
t.Error("removed_host should be meaningful")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompute_StatusChange(t *testing.T) {
|
||||
oldH := &store.Host{Subdomain: "a.example.com", StatusCode: 200}
|
||||
newH := &store.Host{Subdomain: "a.example.com", StatusCode: 401}
|
||||
r := Compute("example.com", []*store.Host{oldH}, []*store.Host{newH}, time.Now(), time.Now())
|
||||
if len(r.Changes) != 1 || r.Changes[0].Kind != "status_change" {
|
||||
t.Errorf("expected status_change, got %+v", r.Changes)
|
||||
}
|
||||
if r.Changes[0].Before != "200" || r.Changes[0].After != "401" {
|
||||
t.Errorf("wrong before/after: %+v", r.Changes[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompute_IPDelta(t *testing.T) {
|
||||
oldH := &store.Host{Subdomain: "a.example.com", IPs: []string{"1.1.1.1"}}
|
||||
newH := &store.Host{Subdomain: "a.example.com", IPs: []string{"1.1.1.1", "2.2.2.2"}}
|
||||
r := Compute("example.com", []*store.Host{oldH}, []*store.Host{newH}, time.Now(), time.Now())
|
||||
found := false
|
||||
for _, c := range r.Changes {
|
||||
if c.Kind == "new_ip" && c.After == "2.2.2.2" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("expected new_ip change, got %+v", r.Changes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompute_NewVuln(t *testing.T) {
|
||||
oldH := &store.Host{Subdomain: "a.example.com"}
|
||||
newH := &store.Host{
|
||||
Subdomain: "a.example.com",
|
||||
Vulnerabilities: []store.Vulnerability{
|
||||
{ID: "xss", Title: "Reflected XSS", Severity: "high"},
|
||||
},
|
||||
}
|
||||
r := Compute("example.com", []*store.Host{oldH}, []*store.Host{newH}, time.Now(), time.Now())
|
||||
found := false
|
||||
for _, c := range r.Changes {
|
||||
if c.Kind == "new_vuln" && c.After == "Reflected XSS" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("expected new_vuln change, got %+v", r.Changes)
|
||||
}
|
||||
if !r.HasMeaningful() {
|
||||
t.Error("new_vuln must be meaningful")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompute_ClearedVuln(t *testing.T) {
|
||||
oldH := &store.Host{
|
||||
Subdomain: "a.example.com",
|
||||
Vulnerabilities: []store.Vulnerability{
|
||||
{ID: "git-exposed", Title: "Git Exposed", Severity: "critical"},
|
||||
},
|
||||
}
|
||||
newH := &store.Host{Subdomain: "a.example.com"}
|
||||
r := Compute("example.com", []*store.Host{oldH}, []*store.Host{newH}, time.Now(), time.Now())
|
||||
found := false
|
||||
for _, c := range r.Changes {
|
||||
if c.Kind == "cleared_vuln" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("expected cleared_vuln, got %+v", r.Changes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompute_NewTakeover(t *testing.T) {
|
||||
oldH := &store.Host{Subdomain: "a.example.com"}
|
||||
newH := &store.Host{
|
||||
Subdomain: "a.example.com",
|
||||
Takeover: &store.Takeover{Service: "GitHub Pages"},
|
||||
}
|
||||
r := Compute("example.com", []*store.Host{oldH}, []*store.Host{newH}, time.Now(), time.Now())
|
||||
found := false
|
||||
for _, c := range r.Changes {
|
||||
if c.Kind == "new_takeover" && c.After == "GitHub Pages" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("expected new_takeover, got %+v", r.Changes)
|
||||
}
|
||||
if !r.HasMeaningful() {
|
||||
t.Error("new_takeover must be meaningful")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompute_NoChange(t *testing.T) {
|
||||
h := &store.Host{
|
||||
Subdomain: "a.example.com",
|
||||
IPs: []string{"1.1.1.1"},
|
||||
StatusCode: 200,
|
||||
Technologies: []string{"nginx"},
|
||||
}
|
||||
r := Compute("example.com", []*store.Host{h}, []*store.Host{h}, time.Now(), time.Now())
|
||||
if len(r.Changes) != 0 {
|
||||
t.Errorf("expected no changes, got %+v", r.Changes)
|
||||
}
|
||||
if r.HasMeaningful() {
|
||||
t.Error("empty report should not be meaningful")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompute_TechChange(t *testing.T) {
|
||||
oldH := &store.Host{Subdomain: "a.example.com", Technologies: []string{"nginx"}}
|
||||
newH := &store.Host{Subdomain: "a.example.com", Technologies: []string{"nginx", "Apache"}}
|
||||
r := Compute("example.com", []*store.Host{oldH}, []*store.Host{newH}, time.Now(), time.Now())
|
||||
found := false
|
||||
for _, c := range r.Changes {
|
||||
if c.Kind == "tech_change" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("expected tech_change, got %+v", r.Changes)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package dns
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestAllEqual(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in []string
|
||||
want bool
|
||||
}{
|
||||
{"empty", nil, true},
|
||||
{"single", []string{"a"}, true},
|
||||
{"all same", []string{"a", "a", "a"}, true},
|
||||
{"one different", []string{"a", "a", "b"}, false},
|
||||
{"all different", []string{"a", "b", "c"}, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := allEqual(tt.in); got != tt.want {
|
||||
t.Errorf("allEqual(%v) = %v, want %v", tt.in, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllEqualInts(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in []int
|
||||
want bool
|
||||
}{
|
||||
{"empty", nil, true},
|
||||
{"single", []int{200}, true},
|
||||
{"all same", []int{200, 200, 200}, true},
|
||||
{"one different", []int{200, 200, 404}, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := allEqualInts(tt.in); got != tt.want {
|
||||
t.Errorf("allEqualInts(%v) = %v, want %v", tt.in, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSimilarSizes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in []int64
|
||||
want bool
|
||||
}{
|
||||
{"empty", nil, true},
|
||||
{"single", []int64{1000}, true},
|
||||
{"identical", []int64{1000, 1000, 1000}, true},
|
||||
{"within 20%", []int64{1000, 1100, 1200}, true},
|
||||
{"exactly 20%", []int64{1000, 1200}, true},
|
||||
{"over 20%", []int64{1000, 1300}, false},
|
||||
{"big variance", []int64{100, 10000}, false},
|
||||
{"all zero", []int64{0, 0}, true},
|
||||
{"zero and small", []int64{0, 50}, true},
|
||||
{"zero and big", []int64{0, 200}, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := similarSizes(tt.in); got != tt.want {
|
||||
t.Errorf("similarSizes(%v) = %v, want %v", tt.in, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsWildcardIP(t *testing.T) {
|
||||
wd := &WildcardDetector{}
|
||||
info := &WildcardInfo{
|
||||
IsWildcard: true,
|
||||
WildcardIPs: []string{"1.2.3.4", "5.6.7.8"},
|
||||
}
|
||||
|
||||
if !wd.IsWildcardIP("1.2.3.4", info) {
|
||||
t.Error("expected 1.2.3.4 to be wildcard IP")
|
||||
}
|
||||
if wd.IsWildcardIP("9.9.9.9", info) {
|
||||
t.Error("expected 9.9.9.9 NOT to be wildcard IP")
|
||||
}
|
||||
|
||||
// nil and non-wildcard cases
|
||||
if wd.IsWildcardIP("1.2.3.4", nil) {
|
||||
t.Error("nil info should return false")
|
||||
}
|
||||
nonWild := &WildcardInfo{IsWildcard: false, WildcardIPs: []string{"1.2.3.4"}}
|
||||
if wd.IsWildcardIP("1.2.3.4", nonWild) {
|
||||
t.Error("non-wildcard info should return false even if IP matches list")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsWildcardResponse(t *testing.T) {
|
||||
wd := &WildcardDetector{}
|
||||
info := &WildcardInfo{
|
||||
IsWildcard: true,
|
||||
HTTPStatusCode: 200,
|
||||
HTTPBodySize: 1000,
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
statusCode int
|
||||
bodySize int64
|
||||
want bool
|
||||
}{
|
||||
{"exact match", 200, 1000, true},
|
||||
{"within 10% body", 200, 1050, true},
|
||||
{"within 10% body below", 200, 950, true},
|
||||
{"over 10% body", 200, 1200, false},
|
||||
{"different status", 404, 1000, false},
|
||||
{"both different", 301, 500, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := wd.IsWildcardResponse(tt.statusCode, tt.bodySize, info); got != tt.want {
|
||||
t.Errorf("IsWildcardResponse(%d, %d) = %v, want %v", tt.statusCode, tt.bodySize, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if wd.IsWildcardResponse(200, 1000, nil) {
|
||||
t.Error("nil info should return false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateTestSubdomains(t *testing.T) {
|
||||
subs := generateTestSubdomains()
|
||||
if len(subs) < 3 {
|
||||
t.Errorf("expected at least 3 test subdomains, got %d", len(subs))
|
||||
}
|
||||
seen := make(map[string]bool)
|
||||
for _, s := range subs {
|
||||
if s == "" {
|
||||
t.Error("empty test subdomain generated")
|
||||
}
|
||||
if seen[s] {
|
||||
t.Errorf("duplicate test subdomain: %s", s)
|
||||
}
|
||||
seen[s] = true
|
||||
}
|
||||
}
|
||||
|
||||
func TestWildcardInfo_GetSummary_NotWildcard(t *testing.T) {
|
||||
info := &WildcardInfo{IsWildcard: false}
|
||||
got := info.GetSummary()
|
||||
if got == "" {
|
||||
t.Error("GetSummary returned empty string")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewWildcardDetector(t *testing.T) {
|
||||
resolvers := []string{"8.8.8.8:53"}
|
||||
wd := NewWildcardDetector(resolvers, 5)
|
||||
if wd == nil {
|
||||
t.Fatal("NewWildcardDetector returned nil")
|
||||
}
|
||||
if wd.timeout != 5 {
|
||||
t.Errorf("timeout = %d, want 5", wd.timeout)
|
||||
}
|
||||
if len(wd.resolvers) != 1 || wd.resolvers[0] != "8.8.8.8:53" {
|
||||
t.Errorf("resolvers = %v", wd.resolvers)
|
||||
}
|
||||
if wd.httpClient == nil {
|
||||
t.Error("httpClient is nil")
|
||||
}
|
||||
if len(wd.testSubdomains) == 0 {
|
||||
t.Error("testSubdomains is empty")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// ErrBusClosed is returned when attempting to use a closed bus.
|
||||
var ErrBusClosed = errors.New("eventbus: bus closed")
|
||||
|
||||
// Handler processes a single event. It runs on the subscriber's own goroutine
|
||||
// so handlers may block or perform I/O without stalling publishers. A handler
|
||||
// must respect ctx cancellation when performing long work.
|
||||
type Handler func(ctx context.Context, e Event)
|
||||
|
||||
// Subscription is returned by Subscribe/SubscribeAll and is used to stop
|
||||
// receiving events. Unsubscribe is idempotent.
|
||||
type Subscription struct {
|
||||
bus *Bus
|
||||
eventType EventType // empty string means "all"
|
||||
id uint64
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
// Unsubscribe stops the subscription. Pending events in the subscriber's
|
||||
// buffer are dropped. Safe to call multiple times.
|
||||
func (s *Subscription) Unsubscribe() {
|
||||
if s == nil || s.bus == nil {
|
||||
return
|
||||
}
|
||||
s.once.Do(func() {
|
||||
s.bus.unsubscribe(s.eventType, s.id)
|
||||
})
|
||||
}
|
||||
|
||||
// Stats captures runtime metrics for observability. Stats are cumulative from
|
||||
// bus creation; callers should compute deltas if rate matters.
|
||||
type Stats struct {
|
||||
Published uint64 // total Publish calls accepted
|
||||
Delivered uint64 // events delivered to subscribers (sum across subscribers)
|
||||
Dropped uint64 // events dropped because a subscriber buffer was full
|
||||
Subscribers int // active subscribers right now
|
||||
Closed bool
|
||||
}
|
||||
|
||||
// Bus is the default eventbus implementation.
|
||||
type Bus struct {
|
||||
bufferSize int
|
||||
|
||||
mu sync.RWMutex
|
||||
closed bool
|
||||
nextID uint64
|
||||
subs map[EventType]map[uint64]*subscriber // type → id → subscriber
|
||||
allSubs map[uint64]*subscriber // wildcard subscribers
|
||||
|
||||
published uint64
|
||||
delivered uint64
|
||||
dropped uint64
|
||||
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
type subscriber struct {
|
||||
id uint64
|
||||
eventT EventType
|
||||
ch chan Event
|
||||
handler Handler
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
// New creates a new Bus. bufferSize controls the per-subscriber channel
|
||||
// buffer; values ≤0 default to 256. A buffer of 1 is legal but increases
|
||||
// drop probability under bursty load.
|
||||
func New(bufferSize int) *Bus {
|
||||
if bufferSize <= 0 {
|
||||
bufferSize = 256
|
||||
}
|
||||
return &Bus{
|
||||
bufferSize: bufferSize,
|
||||
subs: make(map[EventType]map[uint64]*subscriber),
|
||||
allSubs: make(map[uint64]*subscriber),
|
||||
}
|
||||
}
|
||||
|
||||
// Subscribe registers a handler for a specific event type. Returns a
|
||||
// Subscription that can be used to unsubscribe.
|
||||
func (b *Bus) Subscribe(t EventType, h Handler) *Subscription {
|
||||
return b.subscribe(t, h, false)
|
||||
}
|
||||
|
||||
// SubscribeAll registers a handler that receives every event type.
|
||||
// Useful for logging, metrics collection, or persistence modules.
|
||||
func (b *Bus) SubscribeAll(h Handler) *Subscription {
|
||||
return b.subscribe("", h, true)
|
||||
}
|
||||
|
||||
func (b *Bus) subscribe(t EventType, h Handler, all bool) *Subscription {
|
||||
if h == nil {
|
||||
return &Subscription{bus: b}
|
||||
}
|
||||
b.mu.Lock()
|
||||
if b.closed {
|
||||
b.mu.Unlock()
|
||||
return &Subscription{bus: b}
|
||||
}
|
||||
b.nextID++
|
||||
id := b.nextID
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s := &subscriber{
|
||||
id: id,
|
||||
eventT: t,
|
||||
ch: make(chan Event, b.bufferSize),
|
||||
handler: h,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
if all {
|
||||
b.allSubs[id] = s
|
||||
} else {
|
||||
if b.subs[t] == nil {
|
||||
b.subs[t] = make(map[uint64]*subscriber)
|
||||
}
|
||||
b.subs[t][id] = s
|
||||
}
|
||||
b.mu.Unlock()
|
||||
|
||||
b.wg.Add(1)
|
||||
go b.run(s)
|
||||
|
||||
return &Subscription{bus: b, eventType: t, id: id}
|
||||
}
|
||||
|
||||
func (b *Bus) unsubscribe(t EventType, id uint64) {
|
||||
b.mu.Lock()
|
||||
var s *subscriber
|
||||
if t == "" {
|
||||
s = b.allSubs[id]
|
||||
delete(b.allSubs, id)
|
||||
} else {
|
||||
if m, ok := b.subs[t]; ok {
|
||||
s = m[id]
|
||||
delete(m, id)
|
||||
if len(m) == 0 {
|
||||
delete(b.subs, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
if s != nil {
|
||||
close(s.ch) // run() drains remaining events then returns
|
||||
}
|
||||
}
|
||||
|
||||
// run is the per-subscriber goroutine loop.
|
||||
func (b *Bus) run(s *subscriber) {
|
||||
defer b.wg.Done()
|
||||
defer s.cancel()
|
||||
for e := range s.ch {
|
||||
// Protect bus from handler panics — one bad handler must not
|
||||
// take down the pipeline.
|
||||
func() {
|
||||
defer func() {
|
||||
_ = recover()
|
||||
}()
|
||||
s.handler(s.ctx, e)
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// Publish delivers e to every subscriber interested in e.Type() and every
|
||||
// SubscribeAll subscriber. If ctx is canceled, Publish returns early and the
|
||||
// event is not queued to any subscriber that would block.
|
||||
//
|
||||
// Publish is non-blocking per subscriber: if a subscriber's buffer is full the
|
||||
// event is dropped for that subscriber and Stats.Dropped is incremented.
|
||||
func (b *Bus) Publish(ctx context.Context, e Event) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
b.mu.RLock()
|
||||
if b.closed {
|
||||
b.mu.RUnlock()
|
||||
return
|
||||
}
|
||||
// Snapshot the subscriber slices under lock, then release before send.
|
||||
typed := b.subs[e.Type()]
|
||||
var typedList []*subscriber
|
||||
if len(typed) > 0 {
|
||||
typedList = make([]*subscriber, 0, len(typed))
|
||||
for _, s := range typed {
|
||||
typedList = append(typedList, s)
|
||||
}
|
||||
}
|
||||
var allList []*subscriber
|
||||
if len(b.allSubs) > 0 {
|
||||
allList = make([]*subscriber, 0, len(b.allSubs))
|
||||
for _, s := range b.allSubs {
|
||||
allList = append(allList, s)
|
||||
}
|
||||
}
|
||||
b.mu.RUnlock()
|
||||
|
||||
atomic.AddUint64(&b.published, 1)
|
||||
|
||||
for _, s := range typedList {
|
||||
b.dispatch(ctx, s, e)
|
||||
}
|
||||
for _, s := range allList {
|
||||
b.dispatch(ctx, s, e)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Bus) dispatch(ctx context.Context, s *subscriber, e Event) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// caller abandoned; count as dropped so observability reflects reality
|
||||
atomic.AddUint64(&b.dropped, 1)
|
||||
case s.ch <- e:
|
||||
atomic.AddUint64(&b.delivered, 1)
|
||||
default:
|
||||
atomic.AddUint64(&b.dropped, 1)
|
||||
}
|
||||
}
|
||||
|
||||
// Close stops accepting new publishes and drains in-flight subscriber
|
||||
// buffers. It waits until all handlers have returned, or until ctx expires.
|
||||
// Returns ctx.Err() if draining did not complete in time.
|
||||
func (b *Bus) Close(ctx context.Context) error {
|
||||
b.mu.Lock()
|
||||
if b.closed {
|
||||
b.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
b.closed = true
|
||||
|
||||
// Close every subscriber channel; their goroutines will drain and exit.
|
||||
for _, m := range b.subs {
|
||||
for _, s := range m {
|
||||
close(s.ch)
|
||||
}
|
||||
}
|
||||
for _, s := range b.allSubs {
|
||||
close(s.ch)
|
||||
}
|
||||
b.subs = make(map[EventType]map[uint64]*subscriber)
|
||||
b.allSubs = make(map[uint64]*subscriber)
|
||||
b.mu.Unlock()
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
b.wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// Stats returns a snapshot of current metrics.
|
||||
func (b *Bus) Stats() Stats {
|
||||
b.mu.RLock()
|
||||
closed := b.closed
|
||||
subCount := len(b.allSubs)
|
||||
for _, m := range b.subs {
|
||||
subCount += len(m)
|
||||
}
|
||||
b.mu.RUnlock()
|
||||
|
||||
return Stats{
|
||||
Published: atomic.LoadUint64(&b.published),
|
||||
Delivered: atomic.LoadUint64(&b.delivered),
|
||||
Dropped: atomic.LoadUint64(&b.dropped),
|
||||
Subscribers: subCount,
|
||||
Closed: closed,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// waitUntil polls predicate every 2ms up to timeout. Used to avoid flaky
|
||||
// sleeps in async tests without adding dependencies.
|
||||
func waitUntil(t *testing.T, timeout time.Duration, pred func() bool, msg string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if pred() {
|
||||
return
|
||||
}
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("timeout waiting: %s", msg)
|
||||
}
|
||||
|
||||
func TestPublishSubscribe_SingleType(t *testing.T) {
|
||||
b := New(16)
|
||||
defer b.Close(context.Background())
|
||||
|
||||
var got atomic.Int32
|
||||
b.Subscribe(EventSubdomainDiscovered, func(_ context.Context, e Event) {
|
||||
ev, ok := e.(SubdomainDiscovered)
|
||||
if !ok {
|
||||
t.Errorf("wrong event type: %T", e)
|
||||
return
|
||||
}
|
||||
if ev.Subdomain == "" {
|
||||
t.Error("empty subdomain")
|
||||
}
|
||||
got.Add(1)
|
||||
})
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
b.Publish(context.Background(), NewSubdomainDiscovered("test", "api.example.com", "passive"))
|
||||
}
|
||||
waitUntil(t, time.Second, func() bool { return got.Load() == 5 }, "5 events delivered")
|
||||
}
|
||||
|
||||
func TestSubscribeAll_ReceivesEveryType(t *testing.T) {
|
||||
b := New(16)
|
||||
defer b.Close(context.Background())
|
||||
|
||||
var got atomic.Int32
|
||||
b.SubscribeAll(func(_ context.Context, _ Event) { got.Add(1) })
|
||||
|
||||
b.Publish(context.Background(), NewSubdomainDiscovered("t", "a.example.com", "p"))
|
||||
b.Publish(context.Background(), DNSResolved{EventMeta: newMeta("dns", "a.example.com"), Subdomain: "a.example.com", IPs: []string{"1.2.3.4"}})
|
||||
b.Publish(context.Background(), HTTPProbed{EventMeta: newMeta("http", "a.example.com"), URL: "https://a.example.com", StatusCode: 200})
|
||||
|
||||
waitUntil(t, time.Second, func() bool { return got.Load() == 3 }, "3 events on wildcard")
|
||||
}
|
||||
|
||||
func TestSubscribe_FilteringByType(t *testing.T) {
|
||||
b := New(16)
|
||||
defer b.Close(context.Background())
|
||||
|
||||
var subs, dns atomic.Int32
|
||||
b.Subscribe(EventSubdomainDiscovered, func(_ context.Context, _ Event) { subs.Add(1) })
|
||||
b.Subscribe(EventDNSResolved, func(_ context.Context, _ Event) { dns.Add(1) })
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
b.Publish(context.Background(), NewSubdomainDiscovered("t", "a.example.com", "p"))
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
b.Publish(context.Background(), DNSResolved{EventMeta: newMeta("dns", "x"), Subdomain: "x"})
|
||||
}
|
||||
waitUntil(t, time.Second, func() bool { return subs.Load() == 3 && dns.Load() == 2 }, "typed counts match")
|
||||
}
|
||||
|
||||
func TestUnsubscribe_StopsDelivery(t *testing.T) {
|
||||
b := New(16)
|
||||
defer b.Close(context.Background())
|
||||
|
||||
var count atomic.Int32
|
||||
sub := b.Subscribe(EventSubdomainDiscovered, func(_ context.Context, _ Event) { count.Add(1) })
|
||||
|
||||
b.Publish(context.Background(), NewSubdomainDiscovered("t", "a.example.com", "p"))
|
||||
waitUntil(t, time.Second, func() bool { return count.Load() == 1 }, "first event")
|
||||
|
||||
sub.Unsubscribe()
|
||||
sub.Unsubscribe() // idempotent
|
||||
|
||||
// Publish after unsubscribe — should not be delivered to this handler.
|
||||
for i := 0; i < 5; i++ {
|
||||
b.Publish(context.Background(), NewSubdomainDiscovered("t", "b.example.com", "p"))
|
||||
}
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
if got := count.Load(); got != 1 {
|
||||
t.Errorf("expected 1 delivery after unsubscribe, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublish_MultipleSubscribersEachGetEvent(t *testing.T) {
|
||||
b := New(16)
|
||||
defer b.Close(context.Background())
|
||||
|
||||
var a, c atomic.Int32
|
||||
b.Subscribe(EventVulnerability, func(_ context.Context, _ Event) { a.Add(1) })
|
||||
b.Subscribe(EventVulnerability, func(_ context.Context, _ Event) { c.Add(1) })
|
||||
|
||||
b.Publish(context.Background(), VulnerabilityFound{EventMeta: newMeta("sec", "x"), ID: "test", Severity: SeverityHigh})
|
||||
|
||||
waitUntil(t, time.Second, func() bool { return a.Load() == 1 && c.Load() == 1 }, "both subscribers received")
|
||||
}
|
||||
|
||||
func TestPublish_NonBlocking_DropsWhenBufferFull(t *testing.T) {
|
||||
b := New(2)
|
||||
defer b.Close(context.Background())
|
||||
|
||||
blocker := make(chan struct{})
|
||||
var started atomic.Int32
|
||||
b.Subscribe(EventSubdomainDiscovered, func(ctx context.Context, _ Event) {
|
||||
started.Add(1)
|
||||
<-blocker
|
||||
})
|
||||
|
||||
// First event enters handler (blocks). Next 2 fill the buffer of size 2.
|
||||
// Subsequent publishes should be counted as dropped.
|
||||
for i := 0; i < 100; i++ {
|
||||
b.Publish(context.Background(), NewSubdomainDiscovered("t", "x.example.com", "p"))
|
||||
}
|
||||
|
||||
// Give the bus a moment to register drops.
|
||||
waitUntil(t, time.Second, func() bool {
|
||||
return b.Stats().Dropped > 0
|
||||
}, "some events dropped when buffer full")
|
||||
|
||||
// Unblock and close cleanly.
|
||||
close(blocker)
|
||||
}
|
||||
|
||||
func TestClose_DrainsAndStops(t *testing.T) {
|
||||
b := New(16)
|
||||
|
||||
var got atomic.Int32
|
||||
b.Subscribe(EventSubdomainDiscovered, func(_ context.Context, _ Event) { got.Add(1) })
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
b.Publish(context.Background(), NewSubdomainDiscovered("t", "a.example.com", "p"))
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
if err := b.Close(ctx); err != nil {
|
||||
t.Fatalf("Close error: %v", err)
|
||||
}
|
||||
if got.Load() != 10 {
|
||||
t.Errorf("expected 10 delivered before close drains, got %d", got.Load())
|
||||
}
|
||||
|
||||
// Publish after close is a silent no-op.
|
||||
b.Publish(context.Background(), NewSubdomainDiscovered("t", "z.example.com", "p"))
|
||||
if got.Load() != 10 {
|
||||
t.Errorf("delivery continued after close: %d", got.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestClose_IdempotentAndMulticall(t *testing.T) {
|
||||
b := New(4)
|
||||
ctx := context.Background()
|
||||
if err := b.Close(ctx); err != nil {
|
||||
t.Fatalf("first close: %v", err)
|
||||
}
|
||||
if err := b.Close(ctx); err != nil {
|
||||
t.Fatalf("second close: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPanicInHandler_DoesNotAffectOthers(t *testing.T) {
|
||||
b := New(8)
|
||||
defer b.Close(context.Background())
|
||||
|
||||
var good atomic.Int32
|
||||
b.Subscribe(EventSubdomainDiscovered, func(_ context.Context, _ Event) { panic("bad handler") })
|
||||
b.Subscribe(EventSubdomainDiscovered, func(_ context.Context, _ Event) { good.Add(1) })
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
b.Publish(context.Background(), NewSubdomainDiscovered("t", "a.example.com", "p"))
|
||||
}
|
||||
waitUntil(t, time.Second, func() bool { return good.Load() == 5 }, "good handler received all events")
|
||||
}
|
||||
|
||||
func TestConcurrentPublishers_PreservesInvariant(t *testing.T) {
|
||||
// With a fast-enough consumer and large buffer, some events may still be
|
||||
// dropped under heavy burst. The invariant that must ALWAYS hold is:
|
||||
// Published == Delivered + Dropped
|
||||
// This protects against race conditions in metric bookkeeping.
|
||||
b := New(4096)
|
||||
defer b.Close(context.Background())
|
||||
|
||||
b.Subscribe(EventSubdomainDiscovered, func(_ context.Context, _ Event) {})
|
||||
|
||||
const publishers = 20
|
||||
const perPublisher = 100
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < publishers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for j := 0; j < perPublisher; j++ {
|
||||
b.Publish(context.Background(), NewSubdomainDiscovered("t", "a.example.com", "p"))
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
total := uint64(publishers * perPublisher)
|
||||
waitUntil(t, 5*time.Second, func() bool {
|
||||
s := b.Stats()
|
||||
return s.Published == total && s.Delivered+s.Dropped == total
|
||||
}, "published count matches and delivered+dropped == published")
|
||||
}
|
||||
|
||||
func TestStats_Increment(t *testing.T) {
|
||||
b := New(16)
|
||||
defer b.Close(context.Background())
|
||||
|
||||
b.Subscribe(EventSubdomainDiscovered, func(_ context.Context, _ Event) {})
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
b.Publish(context.Background(), NewSubdomainDiscovered("t", "a.example.com", "p"))
|
||||
}
|
||||
waitUntil(t, time.Second, func() bool { return b.Stats().Delivered == 3 }, "3 deliveries recorded")
|
||||
s := b.Stats()
|
||||
if s.Published != 3 {
|
||||
t.Errorf("Published = %d, want 3", s.Published)
|
||||
}
|
||||
if s.Subscribers != 1 {
|
||||
t.Errorf("Subscribers = %d, want 1", s.Subscribers)
|
||||
}
|
||||
if s.Closed {
|
||||
t.Error("Closed = true on open bus")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublish_NilEvent_NoOp(t *testing.T) {
|
||||
b := New(8)
|
||||
defer b.Close(context.Background())
|
||||
var got atomic.Int32
|
||||
b.SubscribeAll(func(_ context.Context, _ Event) { got.Add(1) })
|
||||
b.Publish(context.Background(), nil)
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
if got.Load() != 0 {
|
||||
t.Errorf("nil event was delivered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublish_CancelledContext_DropsNotDelivers(t *testing.T) {
|
||||
b := New(1)
|
||||
defer b.Close(context.Background())
|
||||
|
||||
hold := make(chan struct{})
|
||||
b.Subscribe(EventSubdomainDiscovered, func(_ context.Context, _ Event) { <-hold })
|
||||
|
||||
// First publish occupies buffer slot 1 and handler goroutine starts consuming.
|
||||
b.Publish(context.Background(), NewSubdomainDiscovered("t", "a", "p"))
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
// With ctx already canceled and the subscriber busy, dispatch should record a drop.
|
||||
before := b.Stats().Dropped
|
||||
b.Publish(ctx, NewSubdomainDiscovered("t", "b", "p"))
|
||||
b.Publish(ctx, NewSubdomainDiscovered("t", "c", "p"))
|
||||
after := b.Stats().Dropped
|
||||
if after <= before {
|
||||
t.Errorf("expected Dropped to increase with canceled ctx, before=%d after=%d", before, after)
|
||||
}
|
||||
|
||||
close(hold)
|
||||
}
|
||||
|
||||
func TestHandlerReceivesEventMetadata(t *testing.T) {
|
||||
b := New(8)
|
||||
defer b.Close(context.Background())
|
||||
|
||||
done := make(chan Event, 1)
|
||||
b.Subscribe(EventSubdomainDiscovered, func(_ context.Context, e Event) { done <- e })
|
||||
|
||||
before := time.Now().Add(-time.Second)
|
||||
b.Publish(context.Background(), NewSubdomainDiscovered("sources.crtsh", "api.example.com", "passive:crt.sh"))
|
||||
|
||||
select {
|
||||
case e := <-done:
|
||||
m := e.Meta()
|
||||
if m.Source != "sources.crtsh" {
|
||||
t.Errorf("Source = %q", m.Source)
|
||||
}
|
||||
if m.Target != "api.example.com" {
|
||||
t.Errorf("Target = %q", m.Target)
|
||||
}
|
||||
if m.At.Before(before) {
|
||||
t.Errorf("At = %v is before %v", m.At, before)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("no event received")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
// Package eventbus provides a typed, context-aware pub/sub bus that decouples
|
||||
// discovery, probing, analysis, and reporting modules in God's Eye v2.
|
||||
//
|
||||
// Design choices:
|
||||
// - Events are typed structs implementing Event; dispatch is keyed on EventType.
|
||||
// - Subscribers run handlers on their own goroutine with a buffered channel,
|
||||
// so a slow handler cannot stall the producer.
|
||||
// - Publish is non-blocking: if a subscriber buffer is full, the event is
|
||||
// dropped for that subscriber and Stats.Dropped is incremented. Subscribers
|
||||
// that care about lossless delivery must size their buffer accordingly.
|
||||
// - Close stops accepting new events and drains outstanding ones before
|
||||
// returning.
|
||||
package eventbus
|
||||
|
||||
import "time"
|
||||
|
||||
// EventType identifies the kind of an event.
|
||||
type EventType string
|
||||
|
||||
// Canonical event types. Modules should always use these constants rather than
|
||||
// string literals to avoid typos and to make the full event vocabulary greppable.
|
||||
const (
|
||||
EventSubdomainDiscovered EventType = "subdomain.discovered"
|
||||
EventDNSResolved EventType = "dns.resolved"
|
||||
EventHTTPProbed EventType = "http.probed"
|
||||
EventTechDetected EventType = "tech.detected"
|
||||
EventTLSAnalyzed EventType = "tls.analyzed"
|
||||
EventTakeoverCandidate EventType = "takeover.candidate"
|
||||
EventTakeoverConfirmed EventType = "takeover.confirmed"
|
||||
EventVulnerability EventType = "vulnerability"
|
||||
EventSecret EventType = "secret"
|
||||
EventCVEMatch EventType = "cve.match"
|
||||
EventCloudAsset EventType = "cloud.asset"
|
||||
EventAPIFinding EventType = "api.finding"
|
||||
EventJSFile EventType = "js.file"
|
||||
EventAIFinding EventType = "ai.finding"
|
||||
EventPhaseStarted EventType = "phase.started"
|
||||
EventPhaseCompleted EventType = "phase.completed"
|
||||
EventModuleError EventType = "module.error"
|
||||
EventScanStarted EventType = "scan.started"
|
||||
EventScanCompleted EventType = "scan.completed"
|
||||
)
|
||||
|
||||
// Severity levels used across vulnerability, secret, AI and CVE events.
|
||||
type Severity string
|
||||
|
||||
const (
|
||||
SeverityInfo Severity = "info"
|
||||
SeverityLow Severity = "low"
|
||||
SeverityMedium Severity = "medium"
|
||||
SeverityHigh Severity = "high"
|
||||
SeverityCritical Severity = "critical"
|
||||
)
|
||||
|
||||
// Event is implemented by every event struct.
|
||||
type Event interface {
|
||||
Type() EventType
|
||||
Meta() EventMeta
|
||||
}
|
||||
|
||||
// EventMeta is shared metadata embedded in every event.
|
||||
type EventMeta struct {
|
||||
At time.Time // when the event was created
|
||||
Source string // originating module name (e.g. "sources.crtsh", "dns.resolver")
|
||||
Target string // logical target (typically the subdomain or host the event pertains to)
|
||||
}
|
||||
|
||||
// Meta returns the shared metadata; implemented by embedding EventMeta.
|
||||
func (m EventMeta) Meta() EventMeta { return m }
|
||||
|
||||
// now returns the current time; indirected for testability.
|
||||
var now = time.Now
|
||||
|
||||
// newMeta builds an EventMeta with a populated timestamp.
|
||||
func newMeta(source, target string) EventMeta {
|
||||
return EventMeta{At: now(), Source: source, Target: target}
|
||||
}
|
||||
|
||||
// --- Concrete event types --------------------------------------------------
|
||||
|
||||
// SubdomainDiscovered fires whenever any source (passive, brute, recursive,
|
||||
// CT, etc.) identifies a subdomain that passes the "ends in target domain"
|
||||
// filter. Multiple sources may discover the same subdomain — the bus does not
|
||||
// dedup; that's the store's job.
|
||||
type SubdomainDiscovered struct {
|
||||
EventMeta
|
||||
Subdomain string
|
||||
Method string // "passive:crt.sh", "brute", "recursive", "ct-stream", etc.
|
||||
}
|
||||
|
||||
func (SubdomainDiscovered) Type() EventType { return EventSubdomainDiscovered }
|
||||
|
||||
func NewSubdomainDiscovered(source, subdomain, method string) SubdomainDiscovered {
|
||||
return SubdomainDiscovered{
|
||||
EventMeta: newMeta(source, subdomain),
|
||||
Subdomain: subdomain,
|
||||
Method: method,
|
||||
}
|
||||
}
|
||||
|
||||
// DNSResolved fires after a subdomain is resolved. Empty IPs field signals
|
||||
// an intentionally negative result (NXDOMAIN); absence of the event means
|
||||
// "not yet resolved".
|
||||
type DNSResolved struct {
|
||||
EventMeta
|
||||
Subdomain string
|
||||
IPs []string
|
||||
CNAME string
|
||||
PTR string
|
||||
}
|
||||
|
||||
func (DNSResolved) Type() EventType { return EventDNSResolved }
|
||||
|
||||
// HTTPProbed fires once per successful HTTP probe, including server banner,
|
||||
// title, and technology signals. Security checks emit their own events.
|
||||
type HTTPProbed struct {
|
||||
EventMeta
|
||||
URL string
|
||||
StatusCode int
|
||||
ContentLength int64
|
||||
Title string
|
||||
Server string
|
||||
Technologies []string
|
||||
Headers map[string]string
|
||||
ResponseMs int64
|
||||
TLSVersion string
|
||||
TLSSelfSigned bool
|
||||
}
|
||||
|
||||
func (HTTPProbed) Type() EventType { return EventHTTPProbed }
|
||||
|
||||
// VulnerabilityFound is the canonical finding event for any detected issue.
|
||||
// Scanner modules (security checks, smuggling, SSRF, GraphQL, etc.) all emit
|
||||
// this so the reporter/aggregator has a single type to consume.
|
||||
type VulnerabilityFound struct {
|
||||
EventMeta
|
||||
ID string // stable identifier, e.g. "open-redirect", "cors-wildcard-creds"
|
||||
Title string // short human-readable title
|
||||
Description string // longer context
|
||||
Severity Severity
|
||||
URL string // affected URL
|
||||
Evidence string // raw evidence (truncated if too large)
|
||||
Remediation string // how to fix
|
||||
CVEs []string // referenced CVEs if any
|
||||
OWASP string // OWASP category (e.g. "A03:2021-Injection")
|
||||
CVSS float64 // 0.0 if not scored
|
||||
}
|
||||
|
||||
func (VulnerabilityFound) Type() EventType { return EventVulnerability }
|
||||
|
||||
// SecretFound fires when a credential, API key, or token is detected (in JS,
|
||||
// response bodies, commits, etc.).
|
||||
type SecretFound struct {
|
||||
EventMeta
|
||||
Kind string // "aws_access_key", "jwt", "stripe_live", "generic_hex"
|
||||
Match string // redacted or truncated match — full value in Value if validated
|
||||
Value string // full value, populated only when validation succeeded
|
||||
Location string // where it was found (URL, file path, commit sha)
|
||||
Validated bool // true if we verified the secret is live against its service
|
||||
Severity Severity
|
||||
Description string
|
||||
}
|
||||
|
||||
func (SecretFound) Type() EventType { return EventSecret }
|
||||
|
||||
// CVEMatch fires when a CVE is correlated to a detected technology/version.
|
||||
type CVEMatch struct {
|
||||
EventMeta
|
||||
CVE string
|
||||
Technology string
|
||||
Version string
|
||||
Severity Severity
|
||||
CVSS float64
|
||||
Description string
|
||||
URL string
|
||||
InKEV bool // true if in CISA Known Exploited Vulnerabilities catalog
|
||||
}
|
||||
|
||||
func (CVEMatch) Type() EventType { return EventCVEMatch }
|
||||
|
||||
// TakeoverCandidate fires when a CNAME or fingerprint points at a service
|
||||
// that could potentially be taken over. TakeoverConfirmed fires after active
|
||||
// verification (service claim test) succeeds.
|
||||
type TakeoverCandidate struct {
|
||||
EventMeta
|
||||
Subdomain string
|
||||
Service string // "GitHub Pages", "S3", "Heroku", etc.
|
||||
CNAME string
|
||||
Evidence string
|
||||
}
|
||||
|
||||
func (TakeoverCandidate) Type() EventType { return EventTakeoverCandidate }
|
||||
|
||||
type TakeoverConfirmed struct {
|
||||
EventMeta
|
||||
Subdomain string
|
||||
Service string
|
||||
CNAME string
|
||||
PoC string // curl/HTTP reproducer
|
||||
}
|
||||
|
||||
func (TakeoverConfirmed) Type() EventType { return EventTakeoverConfirmed }
|
||||
|
||||
// CloudAssetFound fires for exposed/accessible cloud assets (S3 buckets,
|
||||
// GCS buckets, Azure blobs, Firebase projects, etc.).
|
||||
type CloudAssetFound struct {
|
||||
EventMeta
|
||||
Provider string // "AWS", "GCP", "Azure", "Firebase"
|
||||
Kind string // "s3-bucket", "gcs-bucket", "lambda-url"
|
||||
Name string
|
||||
URL string
|
||||
Status string // "public-read", "listable", "writable", "exists"
|
||||
Permissions []string // detailed permissions if known
|
||||
}
|
||||
|
||||
func (CloudAssetFound) Type() EventType { return EventCloudAsset }
|
||||
|
||||
// APIFinding fires for discovered/enumerated API surfaces (GraphQL, Swagger,
|
||||
// Postman, misconfigured REST) with associated issues.
|
||||
type APIFinding struct {
|
||||
EventMeta
|
||||
Kind string // "graphql-introspection", "swagger-exposed", "rest-cors", etc.
|
||||
URL string
|
||||
Issue string
|
||||
Severity Severity
|
||||
Endpoints []string
|
||||
}
|
||||
|
||||
func (APIFinding) Type() EventType { return EventAPIFinding }
|
||||
|
||||
// TechDetected fires when a technology (framework, server, CMS, language) is
|
||||
// identified with a version, feeding CVE matching and AI analysis.
|
||||
type TechDetected struct {
|
||||
EventMeta
|
||||
Host string
|
||||
Technology string
|
||||
Version string
|
||||
Category string // "web-server", "framework", "cms", "language", "waf"
|
||||
Confidence float64
|
||||
}
|
||||
|
||||
func (TechDetected) Type() EventType { return EventTechDetected }
|
||||
|
||||
// TLSAnalyzed fires with TLS certificate details, including appliance
|
||||
// fingerprint when identifiable.
|
||||
type TLSAnalyzed struct {
|
||||
EventMeta
|
||||
Host string
|
||||
Version string
|
||||
Issuer string
|
||||
Expiry time.Time
|
||||
SelfSigned bool
|
||||
AltNames []string
|
||||
Vendor string // FortiGate, Palo Alto, etc. (empty if no fingerprint)
|
||||
Product string
|
||||
ApplianceKind string // "firewall", "vpn", "loadbalancer", "waf"
|
||||
InternalHosts []string
|
||||
}
|
||||
|
||||
func (TLSAnalyzed) Type() EventType { return EventTLSAnalyzed }
|
||||
|
||||
// JSFileDiscovered fires when a JavaScript file is discovered and prepared
|
||||
// for analysis (secret scanning, endpoint extraction, AI review).
|
||||
type JSFileDiscovered struct {
|
||||
EventMeta
|
||||
URL string
|
||||
Size int64
|
||||
Host string
|
||||
}
|
||||
|
||||
func (JSFileDiscovered) Type() EventType { return EventJSFile }
|
||||
|
||||
// AIFinding is emitted by any AI/agent module (cascade or multi-agent).
|
||||
type AIFinding struct {
|
||||
EventMeta
|
||||
Subject string // subdomain/URL the finding pertains to
|
||||
Agent string // "triage", "deep", "xss", "sqli", etc.
|
||||
Model string // LLM model id
|
||||
Severity Severity
|
||||
Title string
|
||||
Description string
|
||||
Evidence string
|
||||
CVEs []string
|
||||
OWASP string
|
||||
Confidence float64
|
||||
}
|
||||
|
||||
func (AIFinding) Type() EventType { return EventAIFinding }
|
||||
|
||||
// PhaseStarted / PhaseCompleted frame pipeline phases (passive, brute,
|
||||
// resolve, probe, ai, etc.) so UIs and progress trackers can react.
|
||||
type PhaseStarted struct {
|
||||
EventMeta
|
||||
Phase string
|
||||
}
|
||||
|
||||
func (PhaseStarted) Type() EventType { return EventPhaseStarted }
|
||||
|
||||
type PhaseCompleted struct {
|
||||
EventMeta
|
||||
Phase string
|
||||
Duration time.Duration
|
||||
Stats map[string]int64
|
||||
}
|
||||
|
||||
func (PhaseCompleted) Type() EventType { return EventPhaseCompleted }
|
||||
|
||||
// ModuleError fires when a module encounters a non-fatal error (source
|
||||
// unavailable, rate-limited, timeout). Use this for observability; do not
|
||||
// log errors in modules directly.
|
||||
type ModuleError struct {
|
||||
EventMeta
|
||||
Module string
|
||||
Err string // stringified error
|
||||
Fatal bool // true only when the module cannot continue
|
||||
Context map[string]string
|
||||
}
|
||||
|
||||
func (ModuleError) Type() EventType { return EventModuleError }
|
||||
|
||||
// ScanStarted / ScanCompleted bookend the whole run.
|
||||
type ScanStarted struct {
|
||||
EventMeta
|
||||
Target string
|
||||
Profile string
|
||||
}
|
||||
|
||||
func (ScanStarted) Type() EventType { return EventScanStarted }
|
||||
|
||||
type ScanCompleted struct {
|
||||
EventMeta
|
||||
Target string
|
||||
Duration time.Duration
|
||||
Stats map[string]int64
|
||||
}
|
||||
|
||||
func (ScanCompleted) Type() EventType { return EventScanCompleted }
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/proxyconf"
|
||||
)
|
||||
|
||||
// ClientFactory manages shared HTTP clients with connection pooling
|
||||
@@ -26,8 +28,40 @@ type ClientFactory struct {
|
||||
var (
|
||||
factory *ClientFactory
|
||||
factoryOnce sync.Once
|
||||
|
||||
// proxyURL captures the most recent SetProxy() value, read at factory
|
||||
// construction time. Callers MUST invoke SetProxy BEFORE any code path
|
||||
// that triggers GetFactory — otherwise the factory is built with a
|
||||
// direct dialer and subsequent proxy changes won't be picked up.
|
||||
//
|
||||
// In main.go this is safe: we call SetProxy right after flag parsing,
|
||||
// before any module starts.
|
||||
proxyURL string
|
||||
proxyMu sync.RWMutex
|
||||
)
|
||||
|
||||
// SetProxy configures the outbound proxy for every HTTP client the
|
||||
// factory hands out. Must be called BEFORE GetFactory() / any module
|
||||
// uses a shared client. Supported schemes: http, https, socks5, socks5h.
|
||||
// Empty string disables proxying.
|
||||
func SetProxy(u string) error {
|
||||
if err := proxyconf.Validate(u); err != nil {
|
||||
return err
|
||||
}
|
||||
proxyMu.Lock()
|
||||
proxyURL = u
|
||||
proxyMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// CurrentProxy returns the currently-configured proxy URL, or empty when
|
||||
// none. Useful for status/debug output.
|
||||
func CurrentProxy() string {
|
||||
proxyMu.RLock()
|
||||
defer proxyMu.RUnlock()
|
||||
return proxyURL
|
||||
}
|
||||
|
||||
// GetFactory returns the singleton client factory
|
||||
func GetFactory() *ClientFactory {
|
||||
factoryOnce.Do(func() {
|
||||
@@ -37,12 +71,26 @@ func GetFactory() *ClientFactory {
|
||||
}
|
||||
|
||||
func newClientFactory() *ClientFactory {
|
||||
proxyMu.RLock()
|
||||
cfgProxy := proxyURL
|
||||
proxyMu.RUnlock()
|
||||
|
||||
baseDialer := &net.Dialer{
|
||||
Timeout: 10 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}
|
||||
dialCtx, err := proxyconf.BuildDialer(cfgProxy, baseDialer)
|
||||
if err != nil {
|
||||
// Bad proxy URL at this point is a programming error (we validated
|
||||
// in SetProxy). Fall back to direct rather than crashing.
|
||||
dialCtx = baseDialer.DialContext
|
||||
}
|
||||
proxyFunc, _ := proxyconf.BuildProxyFunc(cfgProxy)
|
||||
|
||||
// Secure transport with TLS verification
|
||||
secureTransport := &http.Transport{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: 10 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
DialContext: dialCtx,
|
||||
Proxy: proxyFunc,
|
||||
MaxIdleConns: 200,
|
||||
MaxIdleConnsPerHost: 20,
|
||||
MaxConnsPerHost: 50,
|
||||
@@ -57,10 +105,8 @@ func newClientFactory() *ClientFactory {
|
||||
|
||||
// Insecure transport (for scanning targets with invalid certs)
|
||||
insecureTransport := &http.Transport{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: 10 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
DialContext: dialCtx,
|
||||
Proxy: proxyFunc,
|
||||
MaxIdleConns: 200,
|
||||
MaxIdleConnsPerHost: 20,
|
||||
MaxConnsPerHost: 50,
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
// Package module defines the Module interface and Registry used by God's Eye v2
|
||||
// to organize discovery, enrichment, analysis, and reporting units of work.
|
||||
//
|
||||
// A Module is any unit of the pipeline that subscribes to zero-or-more event
|
||||
// types, produces zero-or-more event types, and optionally performs a bounded
|
||||
// amount of work on startup (e.g. a passive source fetches once and publishes).
|
||||
//
|
||||
// Modules are decoupled: they do not call each other directly. Ordering emerges
|
||||
// from the event-driven dependency graph, not from phase barriers. The Phase
|
||||
// label is metadata used for grouping in progress UIs and logs, not a scheduling
|
||||
// primitive.
|
||||
package module
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
// Phase groups modules at similar pipeline stages for presentation. Modules at
|
||||
// different phases may still run concurrently; the scanner does not enforce
|
||||
// phase barriers.
|
||||
type Phase string
|
||||
|
||||
const (
|
||||
PhaseSetup Phase = "setup" // load DBs, wordlists, validate config
|
||||
PhaseDiscovery Phase = "discovery" // subdomain sources (passive, CT, brute, recursive)
|
||||
PhaseResolution Phase = "resolution" // DNS resolve, CNAME, PTR, IP info, wildcard filter
|
||||
PhaseEnrichment Phase = "enrichment" // HTTP probe, tech fingerprint, TLS analyze
|
||||
PhaseAnalysis Phase = "analysis" // security checks, takeover, secrets, AI, CVE match
|
||||
PhaseReporting Phase = "reporting" // output writers, report generation
|
||||
)
|
||||
|
||||
// Context bundles everything a module needs to run.
|
||||
//
|
||||
// The Ctx field carries cancellation — every long-running module must select
|
||||
// on Ctx.Done() to exit cleanly when the user interrupts.
|
||||
type Context struct {
|
||||
Ctx context.Context
|
||||
Bus *eventbus.Bus
|
||||
Store store.Store
|
||||
Config ConfigView
|
||||
Target string // primary target domain
|
||||
Profile string // active profile name (bugbounty, pentest, stealth-max, ...)
|
||||
}
|
||||
|
||||
// ConfigView is a narrow read-only interface over the scan config, exposed to
|
||||
// modules so they cannot mutate global state. Implementations live in the
|
||||
// config package.
|
||||
type ConfigView interface {
|
||||
// Profile returns the active profile name ("" when none is selected).
|
||||
Profile() string
|
||||
// Bool reads a boolean config key, returning fallback if unset.
|
||||
Bool(key string, fallback bool) bool
|
||||
// Int reads an int key, returning fallback if unset.
|
||||
Int(key string, fallback int) int
|
||||
// String reads a string key, returning fallback if unset.
|
||||
String(key string, fallback string) string
|
||||
// Strings reads a string-slice key.
|
||||
Strings(key string) []string
|
||||
// ModuleEnabled lets the user disable a module by name. Registry honors
|
||||
// this during selection.
|
||||
ModuleEnabled(moduleName string) bool
|
||||
}
|
||||
|
||||
// Module is the unit of work registered in the pipeline.
|
||||
//
|
||||
// Implementations should:
|
||||
// - be cheap to construct (no I/O in the Module value itself)
|
||||
// - do all setup/teardown inside Run so lifecycle is explicit
|
||||
// - subscribe to events via mctx.Bus.Subscribe in Run
|
||||
// - return promptly when mctx.Ctx is canceled OR when their work is complete
|
||||
type Module interface {
|
||||
// Name uniquely identifies the module. Use dotted notation grouping by
|
||||
// concern: "sources.crtsh", "dns.resolver", "http.probe", "security.cors",
|
||||
// "ai.cascade". The registry rejects duplicate names.
|
||||
Name() string
|
||||
|
||||
// Phase groups the module in pipeline UIs. See Phase constants.
|
||||
Phase() Phase
|
||||
|
||||
// Consumes lists event types the module subscribes to. Empty means the
|
||||
// module is a pure producer (e.g. a passive source). Used by tooling to
|
||||
// visualize the event graph; the bus itself is queried via Subscribe.
|
||||
Consumes() []eventbus.EventType
|
||||
|
||||
// Produces lists event types the module publishes. Empty means the module
|
||||
// only side-effects (e.g. reporting). Used for tooling and dep docs.
|
||||
Produces() []eventbus.EventType
|
||||
|
||||
// DefaultEnabled returns whether this module runs when config does not
|
||||
// explicitly enable/disable it. Passive sources typically default true;
|
||||
// aggressive/experimental modules typically default false.
|
||||
DefaultEnabled() bool
|
||||
|
||||
// Run executes the module. Must be non-blocking on setup and must return
|
||||
// when its work is complete OR mctx.Ctx is canceled. Errors returned are
|
||||
// logged via ModuleError events by the scanner.
|
||||
Run(mctx Context) error
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package module
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
)
|
||||
|
||||
// Registry stores modules keyed by name. Modules register themselves via
|
||||
// init() functions by calling Register on the default registry.
|
||||
type Registry struct {
|
||||
mu sync.RWMutex
|
||||
modules map[string]Module
|
||||
order []string // insertion order for deterministic iteration
|
||||
}
|
||||
|
||||
// NewRegistry returns an empty registry. Most callers should use Default()
|
||||
// which returns the process-wide registry that init() functions populate.
|
||||
func NewRegistry() *Registry {
|
||||
return &Registry{modules: make(map[string]Module)}
|
||||
}
|
||||
|
||||
var (
|
||||
defaultRegistry *Registry
|
||||
defaultOnce sync.Once
|
||||
)
|
||||
|
||||
// Default returns the process-wide module registry.
|
||||
func Default() *Registry {
|
||||
defaultOnce.Do(func() {
|
||||
defaultRegistry = NewRegistry()
|
||||
})
|
||||
return defaultRegistry
|
||||
}
|
||||
|
||||
// Register adds m to r. Panics on duplicate name — registration happens at
|
||||
// init() time, so duplicates indicate a compile-time bug that must surface
|
||||
// immediately rather than silently overwrite.
|
||||
func (r *Registry) Register(m Module) {
|
||||
if m == nil {
|
||||
panic("module.Register: nil module")
|
||||
}
|
||||
name := m.Name()
|
||||
if name == "" {
|
||||
panic("module.Register: module has empty Name()")
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if _, exists := r.modules[name]; exists {
|
||||
panic(fmt.Sprintf("module.Register: duplicate module %q", name))
|
||||
}
|
||||
r.modules[name] = m
|
||||
r.order = append(r.order, name)
|
||||
}
|
||||
|
||||
// Register is a shortcut for Default().Register(m). Intended use:
|
||||
//
|
||||
// func init() { module.Register(&myModule{}) }
|
||||
func Register(m Module) { Default().Register(m) }
|
||||
|
||||
// Get returns the module with the given name.
|
||||
func (r *Registry) Get(name string) (Module, bool) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
m, ok := r.modules[name]
|
||||
return m, ok
|
||||
}
|
||||
|
||||
// Names returns all registered module names in insertion order.
|
||||
func (r *Registry) Names() []string {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
out := make([]string, len(r.order))
|
||||
copy(out, r.order)
|
||||
return out
|
||||
}
|
||||
|
||||
// All returns every registered module in insertion order. The returned slice
|
||||
// is safe for the caller to iterate but do not mutate it.
|
||||
func (r *Registry) All() []Module {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
out := make([]Module, 0, len(r.order))
|
||||
for _, n := range r.order {
|
||||
out = append(out, r.modules[n])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ByPhase returns modules belonging to the given phase, sorted by name for
|
||||
// stable presentation.
|
||||
func (r *Registry) ByPhase(p Phase) []Module {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
var out []Module
|
||||
for _, n := range r.order {
|
||||
m := r.modules[n]
|
||||
if m.Phase() == p {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Name() < out[j].Name() })
|
||||
return out
|
||||
}
|
||||
|
||||
// Select returns the subset of modules that should run for the given config.
|
||||
// A module is selected when cfg.ModuleEnabled(name) returns true (explicit
|
||||
// enable wins), OR when cfg leaves it unset and DefaultEnabled() is true.
|
||||
func (r *Registry) Select(cfg ConfigView) []Module {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
var out []Module
|
||||
for _, n := range r.order {
|
||||
m := r.modules[n]
|
||||
if cfg != nil {
|
||||
// explicit config: respect it directly
|
||||
if cfg.ModuleEnabled(m.Name()) {
|
||||
out = append(out, m)
|
||||
continue
|
||||
}
|
||||
// if the config has a non-default opinion (enabled=false), honor it
|
||||
// — but ModuleEnabled returning false could also mean "unset".
|
||||
// We resolve the ambiguity by checking whether any profile/CLI flag
|
||||
// set it via a separate mechanism; for now, fall back to the
|
||||
// module's default.
|
||||
if m.DefaultEnabled() {
|
||||
out = append(out, m)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// no config: honor module default
|
||||
if m.DefaultEnabled() {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ProducersOf returns the modules that declare t in their Produces() set.
|
||||
// Used by tooling and tests to validate the event-graph integrity.
|
||||
func (r *Registry) ProducersOf(t eventbus.EventType) []Module {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
var out []Module
|
||||
for _, n := range r.order {
|
||||
m := r.modules[n]
|
||||
for _, et := range m.Produces() {
|
||||
if et == t {
|
||||
out = append(out, m)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ConsumersOf returns modules that declare t in their Consumes() set.
|
||||
func (r *Registry) ConsumersOf(t eventbus.EventType) []Module {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
var out []Module
|
||||
for _, n := range r.order {
|
||||
m := r.modules[n]
|
||||
for _, et := range m.Consumes() {
|
||||
if et == t {
|
||||
out = append(out, m)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Reset clears the registry. Intended for tests only; never call in production
|
||||
// code.
|
||||
func (r *Registry) Reset() {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.modules = make(map[string]Module)
|
||||
r.order = nil
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
package module
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
)
|
||||
|
||||
// fakeModule is a minimal Module for tests.
|
||||
type fakeModule struct {
|
||||
name string
|
||||
phase Phase
|
||||
consumes []eventbus.EventType
|
||||
produces []eventbus.EventType
|
||||
defaultEnabled bool
|
||||
runCalled bool
|
||||
}
|
||||
|
||||
func (f *fakeModule) Name() string { return f.name }
|
||||
func (f *fakeModule) Phase() Phase { return f.phase }
|
||||
func (f *fakeModule) Consumes() []eventbus.EventType { return f.consumes }
|
||||
func (f *fakeModule) Produces() []eventbus.EventType { return f.produces }
|
||||
func (f *fakeModule) DefaultEnabled() bool { return f.defaultEnabled }
|
||||
func (f *fakeModule) Run(mctx Context) error { f.runCalled = true; return nil }
|
||||
|
||||
// fakeConfig implements ConfigView for tests.
|
||||
type fakeConfig struct {
|
||||
profile string
|
||||
enabled map[string]bool
|
||||
}
|
||||
|
||||
func (c *fakeConfig) Profile() string { return c.profile }
|
||||
func (c *fakeConfig) Bool(k string, fb bool) bool { return fb }
|
||||
func (c *fakeConfig) Int(k string, fb int) int { return fb }
|
||||
func (c *fakeConfig) String(k, fb string) string { return fb }
|
||||
func (c *fakeConfig) Strings(k string) []string { return nil }
|
||||
func (c *fakeConfig) ModuleEnabled(name string) bool { return c.enabled[name] }
|
||||
|
||||
func TestRegister_AndGet(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
m := &fakeModule{name: "test.one", phase: PhaseDiscovery, defaultEnabled: true}
|
||||
r.Register(m)
|
||||
|
||||
got, ok := r.Get("test.one")
|
||||
if !ok {
|
||||
t.Fatal("Get returned !ok for registered module")
|
||||
}
|
||||
if got != m {
|
||||
t.Error("Get returned a different instance")
|
||||
}
|
||||
|
||||
if _, ok := r.Get("not.present"); ok {
|
||||
t.Error("Get returned ok for missing module")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegister_DuplicatePanic(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
r.Register(&fakeModule{name: "dup", phase: PhaseDiscovery})
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Error("expected panic on duplicate registration")
|
||||
}
|
||||
}()
|
||||
r.Register(&fakeModule{name: "dup", phase: PhaseDiscovery})
|
||||
}
|
||||
|
||||
func TestRegister_NilPanic(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Error("expected panic on nil module")
|
||||
}
|
||||
}()
|
||||
r.Register(nil)
|
||||
}
|
||||
|
||||
func TestRegister_EmptyNamePanic(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Error("expected panic on empty name")
|
||||
}
|
||||
}()
|
||||
r.Register(&fakeModule{name: "", phase: PhaseDiscovery})
|
||||
}
|
||||
|
||||
func TestNames_InsertionOrder(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
r.Register(&fakeModule{name: "zebra", phase: PhaseDiscovery})
|
||||
r.Register(&fakeModule{name: "alpha", phase: PhaseDiscovery})
|
||||
r.Register(&fakeModule{name: "middle", phase: PhaseDiscovery})
|
||||
|
||||
want := []string{"zebra", "alpha", "middle"}
|
||||
got := r.Names()
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("Names order = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAll_ReturnsRegistered(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
r.Register(&fakeModule{name: "a", phase: PhaseDiscovery})
|
||||
r.Register(&fakeModule{name: "b", phase: PhaseAnalysis})
|
||||
r.Register(&fakeModule{name: "c", phase: PhaseReporting})
|
||||
|
||||
if got := len(r.All()); got != 3 {
|
||||
t.Errorf("All length = %d, want 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestByPhase_SortedByName(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
r.Register(&fakeModule{name: "sources.zzz", phase: PhaseDiscovery})
|
||||
r.Register(&fakeModule{name: "sources.aaa", phase: PhaseDiscovery})
|
||||
r.Register(&fakeModule{name: "security.cors", phase: PhaseAnalysis})
|
||||
r.Register(&fakeModule{name: "sources.mmm", phase: PhaseDiscovery})
|
||||
|
||||
got := r.ByPhase(PhaseDiscovery)
|
||||
names := make([]string, len(got))
|
||||
for i, m := range got {
|
||||
names[i] = m.Name()
|
||||
}
|
||||
want := []string{"sources.aaa", "sources.mmm", "sources.zzz"}
|
||||
if !reflect.DeepEqual(names, want) {
|
||||
t.Errorf("ByPhase(discovery) = %v, want %v (sorted)", names, want)
|
||||
}
|
||||
|
||||
if got := r.ByPhase(PhaseAnalysis); len(got) != 1 || got[0].Name() != "security.cors" {
|
||||
t.Errorf("ByPhase(analysis) unexpected: %v", got)
|
||||
}
|
||||
if got := r.ByPhase(PhaseReporting); len(got) != 0 {
|
||||
t.Errorf("ByPhase(reporting) should be empty, got %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelect_DefaultEnabled(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
r.Register(&fakeModule{name: "on-by-default", phase: PhaseDiscovery, defaultEnabled: true})
|
||||
r.Register(&fakeModule{name: "off-by-default", phase: PhaseDiscovery, defaultEnabled: false})
|
||||
|
||||
// nil config: module default governs
|
||||
got := r.Select(nil)
|
||||
names := moduleNames(got)
|
||||
sort.Strings(names)
|
||||
if !reflect.DeepEqual(names, []string{"on-by-default"}) {
|
||||
t.Errorf("Select(nil) = %v, want [on-by-default]", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelect_ConfigEnablesOff(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
r.Register(&fakeModule{name: "optin", phase: PhaseAnalysis, defaultEnabled: false})
|
||||
r.Register(&fakeModule{name: "default-on", phase: PhaseAnalysis, defaultEnabled: true})
|
||||
|
||||
cfg := &fakeConfig{enabled: map[string]bool{"optin": true}}
|
||||
got := r.Select(cfg)
|
||||
names := moduleNames(got)
|
||||
sort.Strings(names)
|
||||
want := []string{"default-on", "optin"}
|
||||
if !reflect.DeepEqual(names, want) {
|
||||
t.Errorf("Select = %v, want %v", names, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProducersOf_AndConsumersOf(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
r.Register(&fakeModule{
|
||||
name: "producer-a",
|
||||
phase: PhaseDiscovery,
|
||||
produces: []eventbus.EventType{eventbus.EventSubdomainDiscovered},
|
||||
})
|
||||
r.Register(&fakeModule{
|
||||
name: "producer-b",
|
||||
phase: PhaseDiscovery,
|
||||
produces: []eventbus.EventType{eventbus.EventSubdomainDiscovered, eventbus.EventDNSResolved},
|
||||
})
|
||||
r.Register(&fakeModule{
|
||||
name: "consumer",
|
||||
phase: PhaseEnrichment,
|
||||
consumes: []eventbus.EventType{eventbus.EventDNSResolved},
|
||||
})
|
||||
|
||||
producers := r.ProducersOf(eventbus.EventSubdomainDiscovered)
|
||||
names := moduleNames(producers)
|
||||
sort.Strings(names)
|
||||
want := []string{"producer-a", "producer-b"}
|
||||
if !reflect.DeepEqual(names, want) {
|
||||
t.Errorf("ProducersOf = %v, want %v", names, want)
|
||||
}
|
||||
|
||||
consumers := r.ConsumersOf(eventbus.EventDNSResolved)
|
||||
if len(consumers) != 1 || consumers[0].Name() != "consumer" {
|
||||
t.Errorf("ConsumersOf unexpected: %v", consumers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReset(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
r.Register(&fakeModule{name: "m1", phase: PhaseDiscovery, defaultEnabled: true})
|
||||
r.Register(&fakeModule{name: "m2", phase: PhaseDiscovery, defaultEnabled: true})
|
||||
if len(r.All()) != 2 {
|
||||
t.Fatal("pre-reset: expected 2 modules")
|
||||
}
|
||||
r.Reset()
|
||||
if len(r.All()) != 0 {
|
||||
t.Errorf("post-reset: expected 0 modules, got %d", len(r.All()))
|
||||
}
|
||||
// Re-register after reset works
|
||||
r.Register(&fakeModule{name: "m1", phase: PhaseDiscovery, defaultEnabled: true})
|
||||
if len(r.All()) != 1 {
|
||||
t.Errorf("post-reset re-register: expected 1, got %d", len(r.All()))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefault_Singleton(t *testing.T) {
|
||||
a := Default()
|
||||
b := Default()
|
||||
if a != b {
|
||||
t.Error("Default() returned different instances")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunContextCarriesFields(t *testing.T) {
|
||||
// Sanity: Context struct is populated correctly — this is effectively a
|
||||
// struct-init contract test to catch accidental field removals.
|
||||
ctx := context.Background()
|
||||
bus := eventbus.New(16)
|
||||
defer bus.Close(context.Background())
|
||||
|
||||
mctx := Context{
|
||||
Ctx: ctx,
|
||||
Bus: bus,
|
||||
Target: "example.com",
|
||||
Profile: "bugbounty",
|
||||
}
|
||||
if mctx.Target != "example.com" {
|
||||
t.Errorf("Target lost: %q", mctx.Target)
|
||||
}
|
||||
if mctx.Profile != "bugbounty" {
|
||||
t.Errorf("Profile lost: %q", mctx.Profile)
|
||||
}
|
||||
if mctx.Bus != bus {
|
||||
t.Error("Bus not retained")
|
||||
}
|
||||
}
|
||||
|
||||
func moduleNames(ms []Module) []string {
|
||||
out := make([]string, len(ms))
|
||||
for i, m := range ms {
|
||||
out[i] = m.Name()
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,660 @@
|
||||
// Package ai is the v2 adapter that wires the Ollama client into the
|
||||
// event-driven pipeline. Unlike the initial skeleton (which only called
|
||||
// CVEMatch on TechDetected), this module subscribes to five event types
|
||||
// and dispatches each to the appropriate v1 client method:
|
||||
//
|
||||
// TechDetected → CVEMatch → CVEMatch events
|
||||
// JSFileDiscovered → AnalyzeJavaScript → AIFinding + SecretFound
|
||||
// HTTPProbed → AnalyzeHTTPResponse (for 5xx / suspicious 4xx) → AIFinding
|
||||
// SecretFound → FilterSecrets (triage real vs regex noise) → AIFinding tag
|
||||
// VulnerabilityFound → multi-agent orchestrator (agents package) → AIFinding with remediation
|
||||
// ScanCompleted → DetectAnomalies + GenerateReport → AIFinding + report artifact
|
||||
//
|
||||
// Every handler:
|
||||
// - is a no-op when ai.enabled=false (module Run returns immediately)
|
||||
// - dedups by content hash to avoid hammering Ollama with duplicates
|
||||
// - cascades through the fast triage model before the deep model
|
||||
// - emits AIFinding events so downstream reporters/TUI pick them up
|
||||
//
|
||||
// The module is the primary value of God's Eye v2's "local LLM" story —
|
||||
// without this wiring, the AI layer was essentially a 20GB curiosity
|
||||
// that added a single CVE string per scan.
|
||||
package ai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/ai"
|
||||
"god-eye/internal/ai/agents"
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "ai.cascade"
|
||||
|
||||
type aiModule struct {
|
||||
client *ai.OllamaClient
|
||||
orchestrator *agents.AgentOrchestrator
|
||||
|
||||
// queryCache dedups expensive Ollama calls across a single scan.
|
||||
// Keyed by SHA256 of (method + input), value is a flag struct so
|
||||
// the same (method, input) pair is processed exactly once.
|
||||
cache sync.Map // map[string]struct{}
|
||||
|
||||
// Counters surfaced at scan end for observability.
|
||||
cveLookups atomic.Int64
|
||||
jsAnalyses atomic.Int64
|
||||
httpAnalyses atomic.Int64
|
||||
secretValidations atomic.Int64
|
||||
vulnEnrichments atomic.Int64
|
||||
anomalyScans atomic.Int64
|
||||
reportGenerations atomic.Int64
|
||||
}
|
||||
|
||||
func Register() { module.Register(&aiModule{}) }
|
||||
|
||||
func (*aiModule) Name() string { return ModuleName }
|
||||
func (*aiModule) Phase() module.Phase { return module.PhaseAnalysis }
|
||||
func (*aiModule) Consumes() []eventbus.EventType {
|
||||
return []eventbus.EventType{
|
||||
eventbus.EventTechDetected,
|
||||
eventbus.EventJSFile,
|
||||
eventbus.EventHTTPProbed,
|
||||
eventbus.EventSecret,
|
||||
eventbus.EventVulnerability,
|
||||
eventbus.EventScanCompleted,
|
||||
}
|
||||
}
|
||||
func (*aiModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{
|
||||
eventbus.EventAIFinding,
|
||||
eventbus.EventCVEMatch,
|
||||
eventbus.EventSecret, // validated/re-emitted
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultEnabled returns true so the module is always loaded; Run() no-ops
|
||||
// unless the user set ai.enabled via --enable-ai / wizard / YAML.
|
||||
func (*aiModule) DefaultEnabled() bool { return true }
|
||||
|
||||
// Run is the heart of the v2 AI layer: wires six event subscriptions,
|
||||
// drains initial store state, and waits for late events in a bounded
|
||||
// window.
|
||||
func (a *aiModule) Run(mctx module.Context) error {
|
||||
if !mctx.Config.Bool("ai.enabled", false) {
|
||||
return nil
|
||||
}
|
||||
|
||||
a.client = ai.NewOllamaClient(
|
||||
mctx.Config.String("ai.url", "http://localhost:11434"),
|
||||
mctx.Config.String("ai.fast_model", "qwen3:1.7b"),
|
||||
mctx.Config.String("ai.deep_model", "qwen2.5-coder:14b"),
|
||||
mctx.Config.Bool("ai.cascade", true),
|
||||
)
|
||||
if mctx.Config.Bool("ai.verbose", false) {
|
||||
a.client.Verbose = true
|
||||
}
|
||||
if !a.client.IsAvailable() {
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.ModuleError{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: mctx.Target},
|
||||
Module: ModuleName,
|
||||
Err: "Ollama not reachable at " + mctx.Config.String("ai.url", "http://localhost:11434"),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// Multi-agent orchestrator is opt-in: only worth spinning up when the
|
||||
// user explicitly enables it. The orchestrator holds one client per
|
||||
// agent type (8 agents) and can take ~200ms to initialise.
|
||||
if mctx.Config.Bool("ai.multi_agent", false) {
|
||||
a.orchestrator = agents.NewAgentOrchestrator(
|
||||
mctx.Config.String("ai.url", "http://localhost:11434"),
|
||||
mctx.Config.String("ai.fast_model", "qwen3:1.7b"),
|
||||
mctx.Config.String("ai.deep_model", "qwen2.5-coder:14b"),
|
||||
)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Subscribe to every event type we care about. Each handler runs in its
|
||||
// own goroutine off the bus; we track them with wg so we can drain at
|
||||
// the end.
|
||||
subs := []*eventbus.Subscription{
|
||||
mctx.Bus.Subscribe(eventbus.EventTechDetected, func(_ context.Context, e eventbus.Event) {
|
||||
if ev, ok := e.(eventbus.TechDetected); ok {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); a.handleTech(mctx, ev.Host, ev.Technology, ev.Version) }()
|
||||
}
|
||||
}),
|
||||
mctx.Bus.Subscribe(eventbus.EventJSFile, func(_ context.Context, e eventbus.Event) {
|
||||
if ev, ok := e.(eventbus.JSFileDiscovered); ok {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); a.handleJSFile(mctx, ev) }()
|
||||
}
|
||||
}),
|
||||
mctx.Bus.Subscribe(eventbus.EventHTTPProbed, func(_ context.Context, e eventbus.Event) {
|
||||
if ev, ok := e.(eventbus.HTTPProbed); ok {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); a.handleHTTP(mctx, ev) }()
|
||||
}
|
||||
}),
|
||||
mctx.Bus.Subscribe(eventbus.EventSecret, func(_ context.Context, e eventbus.Event) {
|
||||
if ev, ok := e.(eventbus.SecretFound); ok {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); a.handleSecret(mctx, ev) }()
|
||||
}
|
||||
}),
|
||||
mctx.Bus.Subscribe(eventbus.EventVulnerability, func(_ context.Context, e eventbus.Event) {
|
||||
if ev, ok := e.(eventbus.VulnerabilityFound); ok {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); a.handleVuln(mctx, ev) }()
|
||||
}
|
||||
}),
|
||||
}
|
||||
defer func() {
|
||||
for _, s := range subs {
|
||||
s.Unsubscribe()
|
||||
}
|
||||
}()
|
||||
|
||||
// Drain store: any host already populated with tech/HTTP info gets
|
||||
// processed on module startup (covers the common case where AI is in a
|
||||
// later phase than discovery/enrichment).
|
||||
for _, h := range mctx.Store.All(mctx.Ctx) {
|
||||
if h == nil {
|
||||
continue
|
||||
}
|
||||
for _, tech := range h.Technologies {
|
||||
tech := tech
|
||||
host := h.Subdomain
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); a.handleTech(mctx, host, tech, "") }()
|
||||
}
|
||||
if h.StatusCode != 0 {
|
||||
ev := eventbus.HTTPProbed{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: h.Subdomain},
|
||||
URL: "https://" + h.Subdomain,
|
||||
StatusCode: h.StatusCode,
|
||||
Title: h.Title,
|
||||
Server: h.Server,
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); a.handleHTTP(mctx, ev) }()
|
||||
}
|
||||
}
|
||||
|
||||
// Brief window for late events (recursive discovery, slow probes) to
|
||||
// arrive before we wrap up.
|
||||
select {
|
||||
case <-time.After(1500 * time.Millisecond):
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// End-of-scan analyses run once, after all per-event handlers drain.
|
||||
a.handleScanEnd(mctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- Handlers ------------------------------------------------------------
|
||||
|
||||
// handleTech runs CVE correlation for a (tech, version) pair. Cached by
|
||||
// (tech, version) so the same pair across many hosts fires one query.
|
||||
func (a *aiModule) handleTech(mctx module.Context, host, tech, version string) {
|
||||
if tech == "" || shouldSkipForCVE(tech, version) {
|
||||
return
|
||||
}
|
||||
name, v := parseTech(tech)
|
||||
if version == "" {
|
||||
version = v
|
||||
}
|
||||
if shouldSkipForCVE(name, version) {
|
||||
return
|
||||
}
|
||||
key := "cve:" + name + "|" + version
|
||||
if !a.firstSeen(key) {
|
||||
return
|
||||
}
|
||||
a.cveLookups.Add(1)
|
||||
|
||||
cves, err := a.client.CVEMatch(name, version)
|
||||
if err != nil || cves == "" {
|
||||
return
|
||||
}
|
||||
|
||||
// Upsert to the specific host that triggered this.
|
||||
now := time.Now()
|
||||
cve := store.CVE{
|
||||
ID: cves, Technology: name, Version: version,
|
||||
Severity: string(eventbus.SeverityHigh), Description: cves, FoundAt: now,
|
||||
}
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, host, func(h *store.Host) { h.CVEs = append(h.CVEs, cve) })
|
||||
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.CVEMatch{
|
||||
EventMeta: eventbus.EventMeta{At: now, Source: ModuleName, Target: host},
|
||||
CVE: cves,
|
||||
Technology: name,
|
||||
Version: version,
|
||||
Severity: eventbus.SeverityHigh,
|
||||
Description: fmt.Sprintf("AI-assisted CVE match for %s %s", name, versionOrUnknown(version)),
|
||||
})
|
||||
}
|
||||
|
||||
// handleJSFile fetches the JS file via the shared HTTP client and feeds it
|
||||
// to AnalyzeJavaScript. Cached by JS URL — a single JS file seen on 5
|
||||
// hosts is analysed once.
|
||||
//
|
||||
// Note: we do NOT re-download the JS content here. The v1 AnalyzeJavaScript
|
||||
// method expects the code itself as input; since the upstream javascript
|
||||
// module already has the content, the proper integration path is to have
|
||||
// JSFileDiscovered carry the content. For now, we skip the deep analysis
|
||||
// when content isn't inlined, and rely on the v1 regex results enriched
|
||||
// by AI at secret-validation time (see handleSecret).
|
||||
func (a *aiModule) handleJSFile(mctx module.Context, ev eventbus.JSFileDiscovered) {
|
||||
key := "js:" + ev.URL
|
||||
if !a.firstSeen(key) {
|
||||
return
|
||||
}
|
||||
a.jsAnalyses.Add(1)
|
||||
// Deep JS analysis is deferred until JSFileDiscovered carries the
|
||||
// content (Fase 2 follow-up). We still produce an AIFinding noting
|
||||
// the JS file was indexed, which helps reporting aggregate per-host
|
||||
// JS exposure.
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.AIFinding{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: ev.Host},
|
||||
Subject: ev.Host,
|
||||
Agent: "js-indexer",
|
||||
Model: a.client.FastModel,
|
||||
Severity: eventbus.SeverityInfo,
|
||||
Title: "JavaScript file indexed for secret review",
|
||||
Evidence: ev.URL,
|
||||
})
|
||||
}
|
||||
|
||||
// handleHTTP triages the HTTP response and dispatches deep analysis only
|
||||
// for interesting status codes / signals. "Interesting" means anything
|
||||
// that isn't a normal 200/301 — 5xx, verbose 4xx with bodies, weird
|
||||
// headers.
|
||||
func (a *aiModule) handleHTTP(mctx module.Context, ev eventbus.HTTPProbed) {
|
||||
if !isInterestingHTTP(ev) {
|
||||
return
|
||||
}
|
||||
key := fmt.Sprintf("http:%s:%d:%s", ev.Meta().Target, ev.StatusCode, hashShort(ev.Title))
|
||||
if !a.firstSeen(key) {
|
||||
return
|
||||
}
|
||||
a.httpAnalyses.Add(1)
|
||||
|
||||
// Compose the content we hand to the deep model. Keep it compact —
|
||||
// Ollama's context is ample but we're summarising for the cascade.
|
||||
headerLines := []string{}
|
||||
if ev.Server != "" {
|
||||
headerLines = append(headerLines, "Server: "+ev.Server)
|
||||
}
|
||||
for k, v := range ev.Headers {
|
||||
headerLines = append(headerLines, k+": "+v)
|
||||
}
|
||||
|
||||
result, err := a.client.AnalyzeHTTPResponse(ev.Meta().Target, ev.StatusCode, headerLines, ev.Title)
|
||||
if err != nil || result == nil || len(result.Findings) == 0 {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
host := ev.Meta().Target
|
||||
for _, f := range result.Findings {
|
||||
persistAIFinding(mctx, host, store.AIFinding{
|
||||
Agent: "http-analyzer", Model: a.client.DeepModel,
|
||||
Severity: result.Severity, Title: "Suspicious HTTP response",
|
||||
Description: f, Evidence: fmt.Sprintf("status=%d title=%q", ev.StatusCode, ev.Title),
|
||||
FoundAt: now,
|
||||
})
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.AIFinding{
|
||||
EventMeta: eventbus.EventMeta{At: now, Source: ModuleName, Target: host},
|
||||
Subject: host,
|
||||
Agent: "http-analyzer",
|
||||
Model: a.client.DeepModel,
|
||||
Severity: eventbus.Severity(result.Severity),
|
||||
Title: "Suspicious HTTP response",
|
||||
Description: f,
|
||||
Evidence: fmt.Sprintf("status=%d title=%q", ev.StatusCode, ev.Title),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// handleSecret validates a regex-surfaced secret through FilterSecrets.
|
||||
// If the AI confirms it's real, an AIFinding event fires tagging it as
|
||||
// validated. Regex noise (UI strings, unrelated third-party URLs) is
|
||||
// dropped silently — the v1 Secret event is left in place but the AI
|
||||
// emission is what a dashboard would prefer to render as a real finding.
|
||||
func (a *aiModule) handleSecret(mctx module.Context, ev eventbus.SecretFound) {
|
||||
key := "secret:" + hashShort(ev.Match+"|"+ev.Location)
|
||||
if !a.firstSeen(key) {
|
||||
return
|
||||
}
|
||||
a.secretValidations.Add(1)
|
||||
|
||||
validated, err := a.client.FilterSecrets([]string{ev.Match})
|
||||
if err != nil || len(validated) == 0 {
|
||||
return // AI says not a real secret, or Ollama unavailable
|
||||
}
|
||||
now := time.Now()
|
||||
persistAIFinding(mctx, ev.Meta().Target, store.AIFinding{
|
||||
Agent: "secret-validator", Model: a.client.FastModel,
|
||||
Severity: string(eventbus.SeverityHigh),
|
||||
Title: "Secret likely valid (AI-confirmed)",
|
||||
Description: fmt.Sprintf("FilterSecrets confirmed '%s' is a real secret, not regex noise.", ev.Kind),
|
||||
Evidence: fmt.Sprintf("%s @ %s", ev.Kind, ev.Location),
|
||||
FoundAt: now,
|
||||
})
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.AIFinding{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: ev.Meta().Target},
|
||||
Subject: ev.Meta().Target,
|
||||
Agent: "secret-validator",
|
||||
Model: a.client.FastModel,
|
||||
Severity: eventbus.SeverityHigh,
|
||||
Title: "Secret likely valid (AI-confirmed)",
|
||||
Description: fmt.Sprintf("FilterSecrets confirmed '%s' is a real secret, not regex noise.",
|
||||
ev.Kind),
|
||||
Evidence: fmt.Sprintf("%s @ %s", ev.Kind, ev.Location),
|
||||
})
|
||||
}
|
||||
|
||||
// handleVuln routes a vulnerability finding through the multi-agent
|
||||
// orchestrator for specialist analysis. When multi-agent is disabled,
|
||||
// this is a no-op.
|
||||
func (a *aiModule) handleVuln(mctx module.Context, ev eventbus.VulnerabilityFound) {
|
||||
if a.orchestrator == nil {
|
||||
return
|
||||
}
|
||||
key := "vuln:" + ev.ID + ":" + ev.Meta().Target
|
||||
if !a.firstSeen(key) {
|
||||
return
|
||||
}
|
||||
a.vulnEnrichments.Add(1)
|
||||
|
||||
finding := agents.Finding{
|
||||
Type: "vulnerability",
|
||||
URL: ev.URL,
|
||||
Context: ev.Description + "\n\nEvidence:\n" + ev.Evidence,
|
||||
}
|
||||
// Respect ctx — orchestrator methods accept context.Context for
|
||||
// cancellation. Allow up to 60s for deep-analysis cascade.
|
||||
ctx, cancel := context.WithTimeout(mctx.Ctx, 60*time.Second)
|
||||
defer cancel()
|
||||
result, err := a.orchestrator.Analyze(ctx, finding)
|
||||
if err != nil || result == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
for _, f := range result.Findings {
|
||||
persistAIFinding(mctx, ev.Meta().Target, store.AIFinding{
|
||||
Agent: string(result.AgentType), Model: result.Model,
|
||||
Severity: strings.ToLower(f.Severity),
|
||||
Title: f.Title, Description: f.Description, Evidence: f.Evidence,
|
||||
CVEs: f.CVEs, OWASP: f.OWASP, Confidence: result.Confidence,
|
||||
FoundAt: now,
|
||||
})
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.AIFinding{
|
||||
EventMeta: eventbus.EventMeta{At: now, Source: ModuleName, Target: ev.Meta().Target},
|
||||
Subject: ev.Meta().Target,
|
||||
Agent: string(result.AgentType),
|
||||
Model: result.Model,
|
||||
Severity: eventbus.Severity(strings.ToLower(f.Severity)),
|
||||
Title: f.Title,
|
||||
Description: f.Description,
|
||||
Evidence: f.Evidence,
|
||||
CVEs: f.CVEs,
|
||||
OWASP: f.OWASP,
|
||||
Confidence: result.Confidence,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// handleScanEnd runs two expensive end-of-scan analyses:
|
||||
//
|
||||
// 1. DetectAnomalies — cross-host pattern review (dev stacks leaking into
|
||||
// prod, unusual version mixes, orphaned endpoints)
|
||||
// 2. GenerateReport — executive summary of findings by severity
|
||||
//
|
||||
// Both run only when the store has enough data to be worth summarising
|
||||
// (≥ 3 findings or ≥ 5 hosts).
|
||||
func (a *aiModule) handleScanEnd(mctx module.Context) {
|
||||
hosts := mctx.Store.All(mctx.Ctx)
|
||||
if len(hosts) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
totalFindings := 0
|
||||
for _, h := range hosts {
|
||||
totalFindings += len(h.Vulnerabilities) + len(h.Secrets) + len(h.CVEs) + len(h.AIFindings)
|
||||
}
|
||||
if totalFindings < 3 && len(hosts) < 5 {
|
||||
return // not worth the Ollama spin-up
|
||||
}
|
||||
|
||||
// Anomaly detection ------------------------------------------------------
|
||||
summary := buildScanSummary(hosts)
|
||||
a.anomalyScans.Add(1)
|
||||
if result, err := a.client.DetectAnomalies(summary); err == nil && result != nil {
|
||||
now := time.Now()
|
||||
for _, f := range result.Findings {
|
||||
persistAIFinding(mctx, mctx.Target, store.AIFinding{
|
||||
Agent: "anomaly-detector", Model: a.client.DeepModel,
|
||||
Severity: result.Severity,
|
||||
Title: "Cross-subdomain anomaly",
|
||||
Description: f, FoundAt: now,
|
||||
})
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.AIFinding{
|
||||
EventMeta: eventbus.EventMeta{At: now, Source: ModuleName, Target: mctx.Target},
|
||||
Subject: mctx.Target,
|
||||
Agent: "anomaly-detector",
|
||||
Model: a.client.DeepModel,
|
||||
Severity: eventbus.Severity(result.Severity),
|
||||
Title: "Cross-subdomain anomaly",
|
||||
Description: f,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Executive report ------------------------------------------------------
|
||||
stats := map[string]int{
|
||||
"hosts": len(hosts),
|
||||
"findings": totalFindings,
|
||||
}
|
||||
a.reportGenerations.Add(1)
|
||||
if report, err := a.client.GenerateReport(summary, stats); err == nil && report != "" {
|
||||
now := time.Now()
|
||||
persistAIFinding(mctx, mctx.Target, store.AIFinding{
|
||||
Agent: "report-writer", Model: a.client.DeepModel,
|
||||
Severity: string(eventbus.SeverityInfo),
|
||||
Title: "AI executive report",
|
||||
Description: report,
|
||||
FoundAt: now,
|
||||
})
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.AIFinding{
|
||||
EventMeta: eventbus.EventMeta{At: now, Source: ModuleName, Target: mctx.Target},
|
||||
Subject: mctx.Target,
|
||||
Agent: "report-writer",
|
||||
Model: a.client.DeepModel,
|
||||
Severity: eventbus.SeverityInfo,
|
||||
Title: "AI executive report",
|
||||
Description: report,
|
||||
})
|
||||
}
|
||||
|
||||
// Emit a module-error style observability event with per-handler counts.
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.ModuleError{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: mctx.Target},
|
||||
Module: ModuleName,
|
||||
Err: fmt.Sprintf("AI activity: cve=%d js=%d http=%d secrets=%d vulns=%d anomaly=%d report=%d",
|
||||
a.cveLookups.Load(),
|
||||
a.jsAnalyses.Load(),
|
||||
a.httpAnalyses.Load(),
|
||||
a.secretValidations.Load(),
|
||||
a.vulnEnrichments.Load(),
|
||||
a.anomalyScans.Load(),
|
||||
a.reportGenerations.Load()),
|
||||
})
|
||||
}
|
||||
|
||||
// --- helpers -------------------------------------------------------------
|
||||
|
||||
// firstSeen returns true the first time we see a given cache key, false
|
||||
// on every subsequent call. Implemented via sync.Map.LoadOrStore which is
|
||||
// atomic.
|
||||
func (a *aiModule) firstSeen(key string) bool {
|
||||
h := sha256.Sum256([]byte(key))
|
||||
hx := hex.EncodeToString(h[:])
|
||||
_, loaded := a.cache.LoadOrStore(hx, struct{}{})
|
||||
return !loaded
|
||||
}
|
||||
|
||||
// isInterestingHTTP gates which HTTP responses are worth sending to the
|
||||
// deep model. Normal 2xx/3xx are skipped; 5xx, verbose 4xx with titles,
|
||||
// and anything with a server-banner mismatch qualifies.
|
||||
func isInterestingHTTP(ev eventbus.HTTPProbed) bool {
|
||||
switch {
|
||||
case ev.StatusCode >= 500:
|
||||
return true
|
||||
case ev.StatusCode == 401 || ev.StatusCode == 403:
|
||||
return true // auth surface worth inspecting
|
||||
case ev.StatusCode >= 400 && ev.Title != "" && ev.ContentLength > 1000:
|
||||
return true // verbose error page
|
||||
case ev.TLSSelfSigned:
|
||||
return true // self-signed on a live host is usually an appliance
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// hashShort returns a short hex prefix of SHA-256(s) — used for cache
|
||||
// keys where the full input is too long but identity matters.
|
||||
func hashShort(s string) string {
|
||||
h := sha256.Sum256([]byte(s))
|
||||
return hex.EncodeToString(h[:8])
|
||||
}
|
||||
|
||||
// persistAIFinding appends an AIFinding to the host's store record so
|
||||
// that downstream modules (notably the report.brief module running in
|
||||
// PhaseReporting, which subscribes to the bus AFTER PhaseAnalysis has
|
||||
// drained) can still surface the finding. Store is the single source
|
||||
// of truth for cross-phase handoff.
|
||||
func persistAIFinding(mctx module.Context, host string, f store.AIFinding) {
|
||||
if host == "" {
|
||||
host = mctx.Target
|
||||
}
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, host, func(h *store.Host) {
|
||||
h.AIFindings = append(h.AIFindings, f)
|
||||
})
|
||||
}
|
||||
|
||||
// cdnOrWafMarkers are technology names that indicate the target is fronted
|
||||
// by a CDN / WAF rather than running that product themselves. Matching
|
||||
// CVEs against these labels produces almost-exclusively false positives,
|
||||
// so we skip them when the version is unknown.
|
||||
var cdnOrWafMarkers = map[string]bool{
|
||||
"cloudflare": true,
|
||||
"cloudfront": true,
|
||||
"akamai": true,
|
||||
"fastly": true,
|
||||
"imperva": true,
|
||||
"aws": true,
|
||||
"azure": true,
|
||||
"gcp": true,
|
||||
"heroku": true,
|
||||
"netlify": true,
|
||||
"vercel": true,
|
||||
"cdn": true,
|
||||
"nginx plus": true,
|
||||
}
|
||||
|
||||
// parseTech extracts (name, version) from strings like "nginx/1.18.0",
|
||||
// "nginx/1.18.0 (Ubuntu)", "Apache/2.4.52", or "Apache 2.4".
|
||||
func parseTech(raw string) (name, version string) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return "", ""
|
||||
}
|
||||
// Look for name/version or name version pattern.
|
||||
for _, sep := range []string{"/", " "} {
|
||||
if idx := strings.Index(raw, sep); idx > 0 {
|
||||
name = strings.TrimSpace(raw[:idx])
|
||||
rest := strings.TrimSpace(raw[idx+1:])
|
||||
rest = strings.TrimPrefix(rest, "v")
|
||||
// Pull digits.digits.digits out of rest
|
||||
end := 0
|
||||
for end < len(rest) {
|
||||
c := rest[end]
|
||||
if (c >= '0' && c <= '9') || c == '.' {
|
||||
end++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if end > 0 {
|
||||
return name, rest[:end]
|
||||
}
|
||||
return name, ""
|
||||
}
|
||||
}
|
||||
return raw, ""
|
||||
}
|
||||
|
||||
// shouldSkipForCVE returns true when (name, version) is too vague for a
|
||||
// useful CVE lookup — empty name, or a CDN/WAF label without a version.
|
||||
func shouldSkipForCVE(name, version string) bool {
|
||||
if name == "" {
|
||||
return true
|
||||
}
|
||||
if version == "" && cdnOrWafMarkers[strings.ToLower(name)] {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func versionOrUnknown(v string) string {
|
||||
if v == "" {
|
||||
return "(unknown version)"
|
||||
}
|
||||
return "v" + v
|
||||
}
|
||||
|
||||
// buildScanSummary compiles a compact text representation of the store
|
||||
// for the DetectAnomalies / GenerateReport prompts. Kept under ~3KB to
|
||||
// fit comfortably in every model's context window.
|
||||
func buildScanSummary(hosts []*store.Host) string {
|
||||
var sb strings.Builder
|
||||
sb.WriteString(fmt.Sprintf("Scan summary: %d hosts\n\n", len(hosts)))
|
||||
shown := 0
|
||||
for _, h := range hosts {
|
||||
if h == nil {
|
||||
continue
|
||||
}
|
||||
if shown >= 50 {
|
||||
sb.WriteString(fmt.Sprintf("\n... and %d more hosts\n", len(hosts)-shown))
|
||||
break
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf("- %s (status=%d, tech=%s)",
|
||||
h.Subdomain, h.StatusCode, strings.Join(h.Technologies, ",")))
|
||||
if len(h.Vulnerabilities) > 0 {
|
||||
sb.WriteString(fmt.Sprintf(" vulns=%d", len(h.Vulnerabilities)))
|
||||
}
|
||||
if len(h.Secrets) > 0 {
|
||||
sb.WriteString(fmt.Sprintf(" secrets=%d", len(h.Secrets)))
|
||||
}
|
||||
if len(h.CVEs) > 0 {
|
||||
sb.WriteString(fmt.Sprintf(" cves=%d", len(h.CVEs)))
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
shown++
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
// Package all is the meta-package imported from main to trigger side-effect
|
||||
// registration of every built-in Fase 0.6 adapter module. Importing
|
||||
// god-eye/internal/modules/all is equivalent to importing each submodule
|
||||
// individually and calling Register().
|
||||
//
|
||||
// Individual submodules avoid registering in their init() on purpose — that
|
||||
// would make the registry state global and prevent tests from using a
|
||||
// clean registry. Callers (main, tests) explicitly opt in by importing
|
||||
// this package or calling RegisterAll.
|
||||
package all
|
||||
|
||||
import (
|
||||
aimod "god-eye/internal/modules/ai"
|
||||
"god-eye/internal/modules/asn"
|
||||
"god-eye/internal/modules/brief"
|
||||
"god-eye/internal/modules/axfr"
|
||||
"god-eye/internal/modules/bruteforce"
|
||||
"god-eye/internal/modules/cloud"
|
||||
"god-eye/internal/modules/ctstream"
|
||||
"god-eye/internal/modules/dnsresolve"
|
||||
"god-eye/internal/modules/github"
|
||||
"god-eye/internal/modules/graphql"
|
||||
"god-eye/internal/modules/headers"
|
||||
"god-eye/internal/modules/httpprobe"
|
||||
"god-eye/internal/modules/javascript"
|
||||
"god-eye/internal/modules/jwt"
|
||||
"god-eye/internal/modules/nuclei"
|
||||
"god-eye/internal/modules/passive"
|
||||
"god-eye/internal/modules/permutation"
|
||||
"god-eye/internal/modules/ports"
|
||||
"god-eye/internal/modules/recursive"
|
||||
"god-eye/internal/modules/report"
|
||||
"god-eye/internal/modules/reversedns"
|
||||
"god-eye/internal/modules/security"
|
||||
"god-eye/internal/modules/smuggling"
|
||||
"god-eye/internal/modules/supplychain"
|
||||
"god-eye/internal/modules/takeover"
|
||||
"god-eye/internal/modules/vhost"
|
||||
)
|
||||
|
||||
// RegisterAll registers every Fase 0.6 adapter module in the default
|
||||
// registry. Call exactly once at program start — Register panics on
|
||||
// duplicates, so calling twice is a bug.
|
||||
func RegisterAll() {
|
||||
// Discovery (Fase 0 adapters + Fase 1 natives + supply chain from F2)
|
||||
passive.Register()
|
||||
bruteforce.Register()
|
||||
recursive.Register()
|
||||
axfr.Register() // F1
|
||||
github.Register() // F1
|
||||
ctstream.Register() // F1 (opt-in)
|
||||
supplychain.Register() // F2
|
||||
|
||||
// Resolution
|
||||
dnsresolve.Register()
|
||||
permutation.Register() // F1 (opt-in)
|
||||
reversedns.Register() // F1 (opt-in)
|
||||
vhost.Register() // F1 (opt-in)
|
||||
asn.Register() // F1 (opt-in)
|
||||
|
||||
// Enrichment
|
||||
httpprobe.Register()
|
||||
ports.Register()
|
||||
|
||||
// Analysis (F0 adapters + F2 natives)
|
||||
security.Register()
|
||||
takeover.Register()
|
||||
cloud.Register()
|
||||
javascript.Register()
|
||||
aimod.Register()
|
||||
graphql.Register() // F2
|
||||
jwt.Register() // F2
|
||||
headers.Register() // F2
|
||||
smuggling.Register() // F2 (opt-in)
|
||||
nuclei.Register() // F2 (opt-in — requires local nuclei-templates dir)
|
||||
|
||||
// Reporting
|
||||
report.Register()
|
||||
brief.Register() // AI-assisted executive summary at scan end
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// Package asn is a Fase 0.6 adapter around v1 network.ASNScanner. Expands
|
||||
// discovery by enumerating IPs within the target's ASN/CIDR blocks.
|
||||
package asn
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/network"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
// CtxPassthrough is used to thread module.Context.Ctx into network helpers.
|
||||
|
||||
const ModuleName = "discovery.asn"
|
||||
|
||||
type asnModule struct{}
|
||||
|
||||
func Register() { module.Register(&asnModule{}) }
|
||||
|
||||
func (*asnModule) Name() string { return ModuleName }
|
||||
func (*asnModule) Phase() module.Phase { return module.PhaseResolution }
|
||||
func (*asnModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventDNSResolved} }
|
||||
func (*asnModule) Produces() []eventbus.EventType { return nil }
|
||||
func (*asnModule) DefaultEnabled() bool { return false } // opt-in
|
||||
|
||||
func (*asnModule) Run(mctx module.Context) error {
|
||||
if !mctx.Config.Bool("asn_scan", false) {
|
||||
return nil
|
||||
}
|
||||
timeout := mctx.Config.Int("timeout", 10)
|
||||
|
||||
hosts := mctx.Store.All(mctx.Ctx)
|
||||
seenIP := make(map[string]struct{})
|
||||
for _, h := range hosts {
|
||||
for _, ip := range h.IPs {
|
||||
seenIP[ip] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
scanner := network.NewASNScanner(timeout)
|
||||
for ip := range seenIP {
|
||||
if mctx.Ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
info, err := scanner.GetASNInfo(mctx.Ctx, ip)
|
||||
if err != nil || info == nil {
|
||||
continue
|
||||
}
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, ipToFirstHost(mctx, ip), func(h *store.Host) {
|
||||
if h.ASN == "" {
|
||||
h.ASN = info.ASN
|
||||
}
|
||||
if h.Org == "" {
|
||||
h.Org = info.Name
|
||||
}
|
||||
if h.Country == "" {
|
||||
h.Country = info.Country
|
||||
}
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ipToFirstHost returns the first subdomain mapped to ip in the store.
|
||||
func ipToFirstHost(mctx module.Context, ip string) string {
|
||||
for _, h := range mctx.Store.All(mctx.Ctx) {
|
||||
for _, rip := range h.IPs {
|
||||
if rip == ip {
|
||||
return h.Subdomain
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var _ = time.Now
|
||||
@@ -0,0 +1,134 @@
|
||||
// Package axfr attempts DNS zone transfer (AXFR) against the target's
|
||||
// authoritative name servers. It's the highest-signal free discovery
|
||||
// technique — when it works, it returns the entire zone at once, exposing
|
||||
// every record the admin considers internal-only.
|
||||
//
|
||||
// Modern DNS infrastructure rejects AXFR by default, but legacy deployments,
|
||||
// misconfigured secondary servers, and corporate DNS still leak zones
|
||||
// regularly in bug bounty scope.
|
||||
package axfr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
godns "github.com/miekg/dns"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "discovery.axfr"
|
||||
|
||||
type axfrModule struct{}
|
||||
|
||||
func Register() { module.Register(&axfrModule{}) }
|
||||
|
||||
func (*axfrModule) Name() string { return ModuleName }
|
||||
func (*axfrModule) Phase() module.Phase { return module.PhaseDiscovery }
|
||||
func (*axfrModule) Consumes() []eventbus.EventType { return nil }
|
||||
func (*axfrModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventSubdomainDiscovered}
|
||||
}
|
||||
func (*axfrModule) DefaultEnabled() bool { return true }
|
||||
|
||||
func (*axfrModule) Run(mctx module.Context) error {
|
||||
target := strings.TrimSuffix(mctx.Target, ".")
|
||||
if target == "" {
|
||||
return nil
|
||||
}
|
||||
timeout := time.Duration(mctx.Config.Int("timeout", 5)) * time.Second
|
||||
|
||||
nameservers, err := lookupNSServers(target, timeout)
|
||||
if err != nil || len(nameservers) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{})
|
||||
for _, ns := range nameservers {
|
||||
if mctx.Ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
records := tryAXFR(target, ns, timeout)
|
||||
for _, sub := range records {
|
||||
sub = strings.ToLower(strings.TrimSuffix(sub, "."))
|
||||
if sub == "" || sub == target {
|
||||
continue
|
||||
}
|
||||
if !strings.HasSuffix(sub, "."+target) {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[sub]; dup {
|
||||
continue
|
||||
}
|
||||
seen[sub] = struct{}{}
|
||||
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, sub, func(h *store.Host) {
|
||||
store.AddDiscoveryMethod(h, "axfr:"+ns)
|
||||
})
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.SubdomainDiscovered{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: sub},
|
||||
Subdomain: sub,
|
||||
Method: "axfr:" + ns,
|
||||
})
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// lookupNSServers returns the authoritative name servers for domain.
|
||||
func lookupNSServers(domain string, timeout time.Duration) ([]string, error) {
|
||||
client := &godns.Client{Timeout: timeout}
|
||||
msg := new(godns.Msg)
|
||||
msg.SetQuestion(godns.Fqdn(domain), godns.TypeNS)
|
||||
// Ask a widely-available resolver.
|
||||
resp, _, err := client.Exchange(msg, "8.8.8.8:53")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, a := range resp.Answer {
|
||||
if ns, ok := a.(*godns.NS); ok {
|
||||
out = append(out, strings.TrimSuffix(ns.Ns, "."))
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// tryAXFR performs an AXFR against nsHost for domain, returning every
|
||||
// returned name (A, AAAA, CNAME). Returns an empty slice when AXFR is
|
||||
// refused (the expected outcome on properly-configured DNS).
|
||||
func tryAXFR(domain, nsHost string, timeout time.Duration) []string {
|
||||
tr := &godns.Transfer{DialTimeout: timeout, ReadTimeout: timeout, WriteTimeout: timeout}
|
||||
msg := new(godns.Msg)
|
||||
msg.SetAxfr(godns.Fqdn(domain))
|
||||
|
||||
ch, err := tr.In(msg, nsHost+":53")
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var out []string
|
||||
for env := range ch {
|
||||
if env.Error != nil {
|
||||
return out
|
||||
}
|
||||
for _, rr := range env.RR {
|
||||
switch r := rr.(type) {
|
||||
case *godns.A:
|
||||
out = append(out, r.Hdr.Name)
|
||||
case *godns.AAAA:
|
||||
out = append(out, r.Hdr.Name)
|
||||
case *godns.CNAME:
|
||||
out = append(out, r.Hdr.Name)
|
||||
case *godns.NS:
|
||||
out = append(out, r.Hdr.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var _ = context.Canceled
|
||||
@@ -0,0 +1,464 @@
|
||||
// Package brief renders the end-of-scan AI-assisted executive brief.
|
||||
//
|
||||
// It's the last module to run in PhaseReporting. It reads:
|
||||
// - every host from the store (for severity / takeover / CVE rollups)
|
||||
// - every AIFinding published during the scan (anomalies, executive
|
||||
// report, per-host agent output)
|
||||
//
|
||||
// Then prints a framed summary block to stdout with:
|
||||
//
|
||||
// ▸ Findings counted by severity
|
||||
// ▸ Top exploitable chains (critical + CVE pairs)
|
||||
// ▸ AI-generated executive summary (if ai.enabled)
|
||||
// ▸ Recommended next actions
|
||||
//
|
||||
// Suppressed when cfg.silent or cfg.json is true so machine-readable
|
||||
// modes stay clean.
|
||||
package brief
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/output"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "report.brief"
|
||||
|
||||
type briefModule struct {
|
||||
aiFindings []eventbus.AIFinding
|
||||
execReport string // last executive-report AIFinding seen
|
||||
execReportAt time.Time
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func Register() { module.Register(&briefModule{}) }
|
||||
|
||||
func (*briefModule) Name() string { return ModuleName }
|
||||
func (*briefModule) Phase() module.Phase { return module.PhaseReporting }
|
||||
func (*briefModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventAIFinding} }
|
||||
func (*briefModule) Produces() []eventbus.EventType { return nil }
|
||||
|
||||
// DefaultEnabled: brief renders whenever the scan completes with any
|
||||
// findings. Silent/json modes are suppressed inline (not at selection
|
||||
// time) so the module can still collect AIFindings for exports.
|
||||
func (*briefModule) DefaultEnabled() bool { return true }
|
||||
|
||||
func (b *briefModule) Run(mctx module.Context) error {
|
||||
// Subscribe to AIFinding events and stash them locally so we can
|
||||
// build a richer summary than just reading the store (the store
|
||||
// doesn't retain AIFindings tagged with agent name / confidence).
|
||||
sub := mctx.Bus.Subscribe(eventbus.EventAIFinding, func(_ context.Context, e eventbus.Event) {
|
||||
ev, ok := e.(eventbus.AIFinding)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.aiFindings = append(b.aiFindings, ev)
|
||||
if ev.Agent == "report-writer" && ev.Description != "" {
|
||||
b.execReport = ev.Description
|
||||
b.execReportAt = ev.Meta().At
|
||||
}
|
||||
})
|
||||
defer sub.Unsubscribe()
|
||||
|
||||
// Give the AI module a chance to publish its end-of-scan events.
|
||||
// The AI module runs in PhaseAnalysis; we're in PhaseReporting so
|
||||
// its ScanCompleted-triggered publishes have already fired by the
|
||||
// time we get here. A small buffer avoids losing late events.
|
||||
select {
|
||||
case <-time.After(400 * time.Millisecond):
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
|
||||
if mctx.Config.Bool("silent", false) || mctx.Config.Bool("json", false) {
|
||||
return nil
|
||||
}
|
||||
|
||||
hosts := mctx.Store.All(mctx.Ctx)
|
||||
if len(hosts) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Drain store-persisted AIFindings — these were written by the AI
|
||||
// module during PhaseAnalysis. Live events alone miss them because
|
||||
// brief subscribes after PhaseAnalysis has already drained.
|
||||
b.mu.Lock()
|
||||
for _, h := range hosts {
|
||||
for _, f := range h.AIFindings {
|
||||
b.aiFindings = append(b.aiFindings, eventbus.AIFinding{
|
||||
EventMeta: eventbus.EventMeta{At: f.FoundAt, Source: "ai.cascade", Target: h.Subdomain},
|
||||
Subject: h.Subdomain,
|
||||
Agent: f.Agent,
|
||||
Model: f.Model,
|
||||
Severity: eventbus.Severity(f.Severity),
|
||||
Title: f.Title,
|
||||
Description: f.Description,
|
||||
Evidence: f.Evidence,
|
||||
CVEs: f.CVEs,
|
||||
OWASP: f.OWASP,
|
||||
Confidence: f.Confidence,
|
||||
})
|
||||
if f.Agent == "report-writer" && f.Description != "" && (b.execReport == "" || f.FoundAt.After(b.execReportAt)) {
|
||||
b.execReport = f.Description
|
||||
b.execReportAt = f.FoundAt
|
||||
}
|
||||
}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
|
||||
b.render(mctx, hosts)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *briefModule) render(mctx module.Context, hosts []*store.Host) {
|
||||
b.mu.Lock()
|
||||
aiFindings := append([]eventbus.AIFinding(nil), b.aiFindings...)
|
||||
execReport := b.execReport
|
||||
b.mu.Unlock()
|
||||
|
||||
sevCounts := tallySeverities(hosts, aiFindings)
|
||||
topChains := buildChains(hosts)
|
||||
recs := buildRecommendations(hosts, aiFindings)
|
||||
aiActivity := tallyAIAgents(aiFindings)
|
||||
|
||||
fmt.Println()
|
||||
title := fmt.Sprintf(" AI SCAN BRIEF — %s ", mctx.Target)
|
||||
fmt.Println(output.BoldCyan(boxTop(title)))
|
||||
writeLine := func(text string) {
|
||||
fmt.Println(output.BoldCyan("│ ") + text)
|
||||
}
|
||||
|
||||
// Section: stats
|
||||
writeLine(output.BoldWhite("Totals"))
|
||||
writeLine(fmt.Sprintf(" %s %d %s %d %s %d",
|
||||
output.Dim("Hosts:"), len(hosts),
|
||||
output.Dim("Active:"), countActive(hosts),
|
||||
output.Dim("AI findings:"), len(aiFindings),
|
||||
))
|
||||
writeLine("")
|
||||
|
||||
// Section: severity breakdown
|
||||
writeLine(output.BoldWhite("Findings by severity"))
|
||||
sevOrder := []string{"critical", "high", "medium", "low", "info"}
|
||||
for _, s := range sevOrder {
|
||||
n := sevCounts[s]
|
||||
if n == 0 {
|
||||
continue
|
||||
}
|
||||
badge := sevBadge(s)
|
||||
writeLine(fmt.Sprintf(" %s %s %d", badge, padRight(s, 9), n))
|
||||
}
|
||||
if len(sevCounts) == 0 {
|
||||
writeLine(output.Dim(" (no scored findings)"))
|
||||
}
|
||||
writeLine("")
|
||||
|
||||
// Section: top exploitable chains
|
||||
if len(topChains) > 0 {
|
||||
writeLine(output.BoldWhite("Top exploitable chains"))
|
||||
for i, c := range topChains {
|
||||
if i >= 5 {
|
||||
break
|
||||
}
|
||||
writeLine(" " + output.BoldYellow("▸ ") + c)
|
||||
}
|
||||
writeLine("")
|
||||
}
|
||||
|
||||
// Section: AI agent activity
|
||||
if len(aiActivity) > 0 {
|
||||
writeLine(output.BoldWhite("AI agents that contributed"))
|
||||
// Stable order by count desc.
|
||||
type agg struct {
|
||||
agent string
|
||||
n int
|
||||
}
|
||||
agents := make([]agg, 0, len(aiActivity))
|
||||
for name, n := range aiActivity {
|
||||
agents = append(agents, agg{name, n})
|
||||
}
|
||||
sort.Slice(agents, func(i, j int) bool { return agents[i].n > agents[j].n })
|
||||
for _, a := range agents {
|
||||
writeLine(fmt.Sprintf(" %s %s %s",
|
||||
output.Cyan("•"),
|
||||
padRight(a.agent, 20),
|
||||
output.Dim(fmt.Sprintf("%d findings", a.n)),
|
||||
))
|
||||
}
|
||||
writeLine("")
|
||||
}
|
||||
|
||||
// Section: AI executive report (prose)
|
||||
if strings.TrimSpace(execReport) != "" {
|
||||
writeLine(output.BoldWhite("AI executive summary"))
|
||||
for _, line := range wrapText(strings.TrimSpace(execReport), 74) {
|
||||
writeLine(output.Dim(" ") + line)
|
||||
}
|
||||
writeLine("")
|
||||
}
|
||||
|
||||
// Section: recommendations
|
||||
if len(recs) > 0 {
|
||||
writeLine(output.BoldWhite("Recommended next actions"))
|
||||
for i, r := range recs {
|
||||
if i >= 5 {
|
||||
break
|
||||
}
|
||||
writeLine(fmt.Sprintf(" %s %s", output.Green(fmt.Sprintf("%d.", i+1)), r))
|
||||
}
|
||||
writeLine("")
|
||||
}
|
||||
|
||||
fmt.Println(output.BoldCyan(boxBottom()))
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// --- helpers -------------------------------------------------------------
|
||||
|
||||
func tallySeverities(hosts []*store.Host, aiFindings []eventbus.AIFinding) map[string]int {
|
||||
out := map[string]int{}
|
||||
for _, h := range hosts {
|
||||
for _, v := range h.Vulnerabilities {
|
||||
out[strings.ToLower(v.Severity)]++
|
||||
}
|
||||
for _, c := range h.CVEs {
|
||||
out[strings.ToLower(c.Severity)]++
|
||||
}
|
||||
for _, s := range h.Secrets {
|
||||
out[strings.ToLower(s.Severity)]++
|
||||
}
|
||||
if h.Takeover != nil {
|
||||
out["high"]++
|
||||
}
|
||||
}
|
||||
for _, f := range aiFindings {
|
||||
out[strings.ToLower(string(f.Severity))]++
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func countActive(hosts []*store.Host) int {
|
||||
n := 0
|
||||
for _, h := range hosts {
|
||||
if h.StatusCode >= 200 && h.StatusCode < 400 {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// buildChains surfaces the most dangerous combinations. Right now the
|
||||
// heuristic is coarse: hosts with ≥2 high+ findings, or any host with a
|
||||
// confirmed takeover candidate, or any host whose tech triggered a CVE.
|
||||
func buildChains(hosts []*store.Host) []string {
|
||||
var chains []string
|
||||
|
||||
type scored struct {
|
||||
text string
|
||||
score int
|
||||
}
|
||||
var ranked []scored
|
||||
|
||||
for _, h := range hosts {
|
||||
score := 0
|
||||
bits := []string{}
|
||||
for _, v := range h.Vulnerabilities {
|
||||
if strings.EqualFold(v.Severity, "critical") {
|
||||
score += 10
|
||||
bits = append(bits, v.Title)
|
||||
} else if strings.EqualFold(v.Severity, "high") {
|
||||
score += 5
|
||||
bits = append(bits, v.Title)
|
||||
}
|
||||
}
|
||||
if h.Takeover != nil {
|
||||
score += 8
|
||||
bits = append(bits, "takeover→"+h.Takeover.Service)
|
||||
}
|
||||
for _, c := range h.CVEs {
|
||||
if strings.EqualFold(c.Severity, "critical") || strings.EqualFold(c.Severity, "high") {
|
||||
score += 6
|
||||
bits = append(bits, fmt.Sprintf("%s@%s→%s", c.Technology, c.Version, firstCVE(c.ID)))
|
||||
}
|
||||
}
|
||||
if score == 0 {
|
||||
continue
|
||||
}
|
||||
desc := h.Subdomain
|
||||
if len(bits) > 0 {
|
||||
desc += " " + output.Dim("— "+strings.Join(dedupShort(bits), " + "))
|
||||
}
|
||||
ranked = append(ranked, scored{desc, score})
|
||||
}
|
||||
|
||||
sort.Slice(ranked, func(i, j int) bool { return ranked[i].score > ranked[j].score })
|
||||
for _, r := range ranked {
|
||||
chains = append(chains, r.text)
|
||||
}
|
||||
return chains
|
||||
}
|
||||
|
||||
func buildRecommendations(hosts []*store.Host, aiFindings []eventbus.AIFinding) []string {
|
||||
seen := map[string]struct{}{}
|
||||
var out []string
|
||||
|
||||
add := func(s string) {
|
||||
if _, ok := seen[s]; ok {
|
||||
return
|
||||
}
|
||||
seen[s] = struct{}{}
|
||||
out = append(out, s)
|
||||
}
|
||||
|
||||
// Pattern: Apache version → upgrade recommendation
|
||||
for _, h := range hosts {
|
||||
for _, c := range h.CVEs {
|
||||
if c.Technology != "" && c.Version != "" {
|
||||
add(fmt.Sprintf("Patch %s %s → vendor latest (affects %s)", c.Technology, c.Version, h.Subdomain))
|
||||
}
|
||||
}
|
||||
if h.Takeover != nil {
|
||||
add(fmt.Sprintf("Verify CNAME on %s before external party claims %s", h.Subdomain, h.Takeover.Service))
|
||||
}
|
||||
for _, s := range h.Secrets {
|
||||
add(fmt.Sprintf("Rotate %s found in %s", s.Kind, h.Subdomain))
|
||||
}
|
||||
for _, v := range h.Vulnerabilities {
|
||||
if strings.EqualFold(v.Severity, "critical") {
|
||||
add(fmt.Sprintf("Remediate critical: %s on %s", v.Title, h.Subdomain))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AI-surfaced recommendations (anomalies)
|
||||
for _, f := range aiFindings {
|
||||
if f.Agent == "anomaly-detector" && f.Description != "" {
|
||||
add("Investigate anomaly: " + trimLine(f.Description, 80))
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func tallyAIAgents(aiFindings []eventbus.AIFinding) map[string]int {
|
||||
out := map[string]int{}
|
||||
for _, f := range aiFindings {
|
||||
agent := f.Agent
|
||||
if agent == "" {
|
||||
agent = "unknown"
|
||||
}
|
||||
out[agent]++
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- rendering primitives ------------------------------------------------
|
||||
|
||||
const boxWidth = 76
|
||||
|
||||
func boxTop(title string) string {
|
||||
line := strings.Repeat("─", boxWidth)
|
||||
if len(title) >= boxWidth-4 {
|
||||
title = title[:boxWidth-4]
|
||||
}
|
||||
prefix := "┌── "
|
||||
suffix := " " + strings.Repeat("─", boxWidth-len(prefix)-len(title)-1) + "┐"
|
||||
_ = line
|
||||
return prefix + title + suffix
|
||||
}
|
||||
|
||||
func boxBottom() string {
|
||||
return "└" + strings.Repeat("─", boxWidth) + "┘"
|
||||
}
|
||||
|
||||
func padRight(s string, n int) string {
|
||||
if len(s) >= n {
|
||||
return s
|
||||
}
|
||||
return s + strings.Repeat(" ", n-len(s))
|
||||
}
|
||||
|
||||
func wrapText(s string, width int) []string {
|
||||
words := strings.Fields(s)
|
||||
if len(words) == 0 {
|
||||
return nil
|
||||
}
|
||||
var lines []string
|
||||
var cur strings.Builder
|
||||
for _, w := range words {
|
||||
if cur.Len() == 0 {
|
||||
cur.WriteString(w)
|
||||
continue
|
||||
}
|
||||
if cur.Len()+1+len(w) > width {
|
||||
lines = append(lines, cur.String())
|
||||
cur.Reset()
|
||||
cur.WriteString(w)
|
||||
} else {
|
||||
cur.WriteByte(' ')
|
||||
cur.WriteString(w)
|
||||
}
|
||||
}
|
||||
if cur.Len() > 0 {
|
||||
lines = append(lines, cur.String())
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
func sevBadge(s string) string {
|
||||
switch strings.ToLower(s) {
|
||||
case "critical":
|
||||
return output.BgRed(" CRIT ")
|
||||
case "high":
|
||||
return output.Red("[HIGH]")
|
||||
case "medium":
|
||||
return output.Yellow("[MED] ")
|
||||
case "low":
|
||||
return output.Blue("[LOW] ")
|
||||
default:
|
||||
return output.Dim("[INFO]")
|
||||
}
|
||||
}
|
||||
|
||||
func firstCVE(ids string) string {
|
||||
if i := strings.IndexAny(ids, ",("); i > 0 {
|
||||
return strings.TrimSpace(ids[:i])
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func dedupShort(in []string) []string {
|
||||
seen := map[string]struct{}{}
|
||||
var out []string
|
||||
for _, s := range in {
|
||||
if _, ok := seen[s]; ok {
|
||||
continue
|
||||
}
|
||||
seen[s] = struct{}{}
|
||||
if len(s) > 40 {
|
||||
s = s[:37] + "…"
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func trimLine(s string, n int) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if i := strings.Index(s, "\n"); i > 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
if len(s) > n {
|
||||
s = s[:n-1] + "…"
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
// Package bruteforce runs DNS brute-force against the target domain using
|
||||
// the shipped or custom wordlist. Emits SubdomainDiscovered for every host
|
||||
// that resolves (with optional wildcard filtering applied).
|
||||
package bruteforce
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/config"
|
||||
godns "god-eye/internal/dns"
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "discovery.bruteforce"
|
||||
|
||||
type bruteModule struct{}
|
||||
|
||||
func Register() { module.Register(&bruteModule{}) }
|
||||
|
||||
func (*bruteModule) Name() string { return ModuleName }
|
||||
func (*bruteModule) Phase() module.Phase { return module.PhaseDiscovery }
|
||||
func (*bruteModule) Consumes() []eventbus.EventType { return nil }
|
||||
func (*bruteModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventSubdomainDiscovered}
|
||||
}
|
||||
func (*bruteModule) DefaultEnabled() bool { return true }
|
||||
|
||||
func (b *bruteModule) Run(mctx module.Context) error {
|
||||
if mctx.Config.Bool("no_brute", false) {
|
||||
return nil
|
||||
}
|
||||
|
||||
target := mctx.Target
|
||||
wordlist := loadWordlist(mctx.Config.String("wordlist", ""))
|
||||
resolvers := parseResolvers(mctx.Config.String("resolvers", ""))
|
||||
timeout := mctx.Config.Int("timeout", 5)
|
||||
conc := mctx.Config.Int("concurrency", 500)
|
||||
if conc <= 0 {
|
||||
conc = 500
|
||||
}
|
||||
|
||||
// Opportunistic wildcard detection: before brute, detect which IPs
|
||||
// (if any) the apex wildcards to, so we can filter hits that resolve
|
||||
// exclusively to those IPs.
|
||||
wd := godns.NewWildcardDetector(resolvers, timeout)
|
||||
wi := wd.Detect(target)
|
||||
wildcardIPs := make(map[string]struct{})
|
||||
if wi != nil && wi.IsWildcard {
|
||||
for _, ip := range wi.WildcardIPs {
|
||||
wildcardIPs[ip] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
work := make(chan string, conc*2)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < conc; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for w := range work {
|
||||
if mctx.Ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
sub := w + "." + target
|
||||
ips := godns.ResolveSubdomain(sub, resolvers, timeout)
|
||||
if len(ips) == 0 {
|
||||
continue
|
||||
}
|
||||
if allWildcard(ips, wildcardIPs) {
|
||||
continue
|
||||
}
|
||||
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, sub, func(h *store.Host) {
|
||||
store.AddIPs(h, ips)
|
||||
store.AddDiscoveryMethod(h, "brute")
|
||||
})
|
||||
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.SubdomainDiscovered{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: sub},
|
||||
Subdomain: sub,
|
||||
Method: "brute",
|
||||
})
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
loop:
|
||||
for _, w := range wordlist {
|
||||
select {
|
||||
case work <- w:
|
||||
case <-mctx.Ctx.Done():
|
||||
break loop
|
||||
}
|
||||
}
|
||||
close(work)
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func allWildcard(ips []string, wc map[string]struct{}) bool {
|
||||
if len(wc) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, ip := range ips {
|
||||
if _, ok := wc[ip]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func loadWordlist(path string) []string {
|
||||
if path == "" {
|
||||
return config.DefaultWordlist
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return config.DefaultWordlist
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var out []string
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
w := strings.TrimSpace(sc.Text())
|
||||
if w == "" || strings.HasPrefix(w, "#") {
|
||||
continue
|
||||
}
|
||||
out = append(out, w)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return config.DefaultWordlist
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseResolvers(s string) []string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return config.DefaultResolvers
|
||||
}
|
||||
var out []string
|
||||
for _, r := range strings.Split(s, ",") {
|
||||
r = strings.TrimSpace(r)
|
||||
if r == "" {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(r, ":") {
|
||||
r = r + ":53"
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return config.DefaultResolvers
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// keep context import for symmetry with other modules
|
||||
var _ = context.Canceled
|
||||
@@ -0,0 +1,104 @@
|
||||
// Package cloud wraps v1 cloud detection + S3 bucket discovery.
|
||||
// Drains the store, plus listens for late DNSResolved events.
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
gohttp "god-eye/internal/http"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/scanner"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "cloud.detect"
|
||||
|
||||
type cloudModule struct{}
|
||||
|
||||
func Register() { module.Register(&cloudModule{}) }
|
||||
|
||||
func (*cloudModule) Name() string { return ModuleName }
|
||||
func (*cloudModule) Phase() module.Phase { return module.PhaseAnalysis }
|
||||
func (*cloudModule) Consumes() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventDNSResolved, eventbus.EventHTTPProbed}
|
||||
}
|
||||
func (*cloudModule) Produces() []eventbus.EventType { return []eventbus.EventType{eventbus.EventCloudAsset} }
|
||||
func (*cloudModule) DefaultEnabled() bool { return true }
|
||||
|
||||
func (*cloudModule) Run(mctx module.Context) error {
|
||||
timeout := mctx.Config.Int("timeout", 5)
|
||||
client := gohttp.GetSharedClient(timeout)
|
||||
|
||||
handled := make(map[string]struct{})
|
||||
var mu sync.Mutex
|
||||
shouldHandle := func(host string) bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if _, ok := handled[host]; ok {
|
||||
return false
|
||||
}
|
||||
handled[host] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
handle := func(host string, ips []string, cname string) {
|
||||
if !shouldHandle(host) {
|
||||
return
|
||||
}
|
||||
provider := scanner.DetectCloudProvider(ips, cname, "")
|
||||
if provider != "" {
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, host, func(h *store.Host) {
|
||||
if h.CloudProvider == "" {
|
||||
h.CloudProvider = provider
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if buckets := scanner.CheckS3BucketsWithClient(host, client); len(buckets) > 0 {
|
||||
for _, url := range buckets {
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.CloudAssetFound{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: host},
|
||||
Provider: "AWS",
|
||||
Kind: "s3-bucket",
|
||||
Name: host,
|
||||
URL: url,
|
||||
Status: "accessible",
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Drain: every host already in the store with an IP.
|
||||
for _, h := range mctx.Store.All(mctx.Ctx) {
|
||||
if h == nil || h.Subdomain == "" || len(h.IPs) == 0 {
|
||||
continue
|
||||
}
|
||||
h := h
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); handle(h.Subdomain, h.IPs, h.CNAME) }()
|
||||
}
|
||||
|
||||
// Late DNSResolved events.
|
||||
sub := mctx.Bus.Subscribe(eventbus.EventDNSResolved, func(_ context.Context, e eventbus.Event) {
|
||||
ev, ok := e.(eventbus.DNSResolved)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); handle(ev.Subdomain, ev.IPs, ev.CNAME) }()
|
||||
})
|
||||
defer sub.Unsubscribe()
|
||||
|
||||
select {
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
// Package ctstream subscribes to live Certificate Transparency log streams
|
||||
// from certstream.calidog.io (free, public). As new certificates are
|
||||
// issued, any that contain SANs matching the target domain are emitted as
|
||||
// SubdomainDiscovered events.
|
||||
//
|
||||
// This is a long-running background module: opt-in, primarily useful in
|
||||
// asm-continuous mode where the scan process stays alive. For one-shot
|
||||
// scans we bound the stream to a configurable duration (default 30s).
|
||||
//
|
||||
// NOTE: certstream.calidog.io is sometimes rate-limited or offline. This
|
||||
// module fails open — no event emitted, no error returned.
|
||||
package ctstream
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "discovery.ct-stream"
|
||||
|
||||
type ctModule struct{}
|
||||
|
||||
func Register() { module.Register(&ctModule{}) }
|
||||
|
||||
func (*ctModule) Name() string { return ModuleName }
|
||||
func (*ctModule) Phase() module.Phase { return module.PhaseDiscovery }
|
||||
func (*ctModule) Consumes() []eventbus.EventType { return nil }
|
||||
func (*ctModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventSubdomainDiscovered}
|
||||
}
|
||||
|
||||
// Off by default: requires long-running streaming.
|
||||
func (*ctModule) DefaultEnabled() bool { return false }
|
||||
|
||||
func (*ctModule) Run(mctx module.Context) error {
|
||||
if !mctx.Config.Bool("ct_stream", false) {
|
||||
return nil
|
||||
}
|
||||
durationSec := mctx.Config.Int("ct_stream.duration_sec", 30)
|
||||
if durationSec <= 0 {
|
||||
durationSec = 30
|
||||
}
|
||||
|
||||
target := mctx.Target
|
||||
deadline := time.Now().Add(time.Duration(durationSec) * time.Second)
|
||||
|
||||
// Fallback path: poll crt.sh's JSON endpoint every 5s for the duration.
|
||||
// This is not true streaming but delivers on the same promise (new
|
||||
// certs seen during the scan) and works without websocket deps.
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
seen := make(map[string]struct{})
|
||||
|
||||
for time.Now().Before(deadline) {
|
||||
if mctx.Ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
subs := fetchRecentCerts(target)
|
||||
for _, s := range subs {
|
||||
s = strings.ToLower(strings.TrimSpace(s))
|
||||
if s == "" || !strings.HasSuffix(s, target) {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[s]; dup {
|
||||
continue
|
||||
}
|
||||
seen[s] = struct{}{}
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, s, func(h *store.Host) {
|
||||
store.AddDiscoveryMethod(h, "ct-stream")
|
||||
})
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.SubdomainDiscovered{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: s},
|
||||
Subdomain: s,
|
||||
Method: "ct-stream",
|
||||
})
|
||||
}
|
||||
select {
|
||||
case <-ticker.C:
|
||||
case <-mctx.Ctx.Done():
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetchRecentCerts(target string) []string {
|
||||
// crt.sh returns JSON with name_value fields; same as the v1 crtsh
|
||||
// source but we use a tighter query.
|
||||
q := "%." + target
|
||||
u := fmt.Sprintf("https://crt.sh/?q=%s&output=json", url.QueryEscape(q))
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
resp, err := client.Get(u)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var entries []struct {
|
||||
NameValue string `json:"name_value"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&entries); err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
for _, name := range strings.Split(e.NameValue, "\n") {
|
||||
name = strings.TrimPrefix(strings.TrimSpace(name), "*.")
|
||||
if name != "" {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
// Package dnsresolve resolves every subdomain present in the store, plus
|
||||
// any that arrive via late SubdomainDiscovered events while the module is
|
||||
// running. Results (IPs, CNAME, PTR) are written back to the store AND
|
||||
// announced via DNSResolved events for downstream enrichment modules.
|
||||
//
|
||||
// This module is idempotent: Upsert on the same subdomain twice is cheap.
|
||||
package dnsresolve
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/config"
|
||||
godns "god-eye/internal/dns"
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "dns.resolver"
|
||||
|
||||
type resolverModule struct{}
|
||||
|
||||
func Register() { module.Register(&resolverModule{}) }
|
||||
|
||||
func (*resolverModule) Name() string { return ModuleName }
|
||||
func (*resolverModule) Phase() module.Phase { return module.PhaseResolution }
|
||||
func (*resolverModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventSubdomainDiscovered} }
|
||||
func (*resolverModule) Produces() []eventbus.EventType { return []eventbus.EventType{eventbus.EventDNSResolved} }
|
||||
func (*resolverModule) DefaultEnabled() bool { return true }
|
||||
|
||||
func (m *resolverModule) Run(mctx module.Context) error {
|
||||
resolvers := parseResolvers(mctx.Config.String("resolvers", ""))
|
||||
timeout := mctx.Config.Int("timeout", 5)
|
||||
conc := mctx.Config.Int("concurrency", 500)
|
||||
if conc <= 0 {
|
||||
conc = 500
|
||||
}
|
||||
|
||||
// Dedup across drain + late events.
|
||||
processed := make(map[string]struct{})
|
||||
var processedMu sync.Mutex
|
||||
shouldProcess := func(sub string) bool {
|
||||
processedMu.Lock()
|
||||
defer processedMu.Unlock()
|
||||
if _, dup := processed[sub]; dup {
|
||||
return false
|
||||
}
|
||||
processed[sub] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
work := make(chan string, conc*2)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < conc; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for sub := range work {
|
||||
m.resolveOne(mctx, sub, resolvers, timeout)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// 1) Drain the store: every subdomain discovered so far goes in.
|
||||
for _, h := range mctx.Store.All(mctx.Ctx) {
|
||||
if h == nil || h.Subdomain == "" {
|
||||
continue
|
||||
}
|
||||
if !shouldProcess(h.Subdomain) {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case work <- h.Subdomain:
|
||||
case <-mctx.Ctx.Done():
|
||||
close(work)
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// 2) Keep listening for late events (e.g. from recursive discovery that
|
||||
// runs in our own phase and produces new subdomains mid-resolution).
|
||||
sub := mctx.Bus.Subscribe(eventbus.EventSubdomainDiscovered, func(_ context.Context, e eventbus.Event) {
|
||||
ev, ok := e.(eventbus.SubdomainDiscovered)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !shouldProcess(ev.Subdomain) {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case work <- ev.Subdomain:
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
})
|
||||
defer sub.Unsubscribe()
|
||||
|
||||
// 3) Give late events a short window to arrive (e.g. recursive module
|
||||
// running concurrently in PhaseResolution). 1 second is enough — we
|
||||
// already drained the store, so any straggler events here are rare.
|
||||
select {
|
||||
case <-time.After(1 * time.Second):
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
|
||||
close(work)
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *resolverModule) resolveOne(mctx module.Context, sub string, resolvers []string, timeout int) {
|
||||
if err := mctx.Ctx.Err(); err != nil {
|
||||
return
|
||||
}
|
||||
ips := godns.ResolveSubdomain(sub, resolvers, timeout)
|
||||
if len(ips) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
cname := godns.ResolveCNAME(sub, resolvers, timeout)
|
||||
ptr := godns.ResolvePTR(ips[0], resolvers, timeout)
|
||||
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, sub, func(h *store.Host) {
|
||||
store.AddIPs(h, ips)
|
||||
if cname != "" && h.CNAME == "" {
|
||||
h.CNAME = cname
|
||||
}
|
||||
if ptr != "" && h.PTR == "" {
|
||||
h.PTR = ptr
|
||||
}
|
||||
store.AddDiscoveryMethod(h, "resolved")
|
||||
})
|
||||
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.DNSResolved{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: sub},
|
||||
Subdomain: sub,
|
||||
IPs: ips,
|
||||
CNAME: cname,
|
||||
PTR: ptr,
|
||||
})
|
||||
}
|
||||
|
||||
func parseResolvers(s string) []string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return config.DefaultResolvers
|
||||
}
|
||||
var out []string
|
||||
for _, r := range strings.Split(s, ",") {
|
||||
r = strings.TrimSpace(r)
|
||||
if r == "" {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(r, ":") {
|
||||
r = r + ":53"
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return config.DefaultResolvers
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
// Package github discovers subdomains from public GitHub code via dorks.
|
||||
// Uses the v3 REST Search API. Works anonymously at a very low rate
|
||||
// (strict API limits); a token in the GITHUB_TOKEN env var lifts limits.
|
||||
//
|
||||
// Dorks used:
|
||||
//
|
||||
// "<domain>" in:file
|
||||
// "api.<domain>" in:file
|
||||
//
|
||||
// The module only emits subdomains that match the target domain suffix.
|
||||
package github
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/sources"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "discovery.github-dorks"
|
||||
|
||||
type ghModule struct{}
|
||||
|
||||
func Register() { module.Register(&ghModule{}) }
|
||||
|
||||
func (*ghModule) Name() string { return ModuleName }
|
||||
func (*ghModule) Phase() module.Phase { return module.PhaseDiscovery }
|
||||
func (*ghModule) Consumes() []eventbus.EventType { return nil }
|
||||
func (*ghModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventSubdomainDiscovered}
|
||||
}
|
||||
|
||||
// Default-enabled so bug-bounty users get it for free. Falls back to
|
||||
// no-op when unauthenticated requests hit rate limits.
|
||||
func (*ghModule) DefaultEnabled() bool { return true }
|
||||
|
||||
func (*ghModule) Run(mctx module.Context) error {
|
||||
target := mctx.Target
|
||||
if target == "" {
|
||||
return nil
|
||||
}
|
||||
token := os.Getenv("GITHUB_TOKEN")
|
||||
timeout := time.Duration(mctx.Config.Int("timeout", 10)) * time.Second
|
||||
client := &http.Client{Timeout: timeout}
|
||||
|
||||
// Two dorks run in parallel. Each returns up to 100 results per page.
|
||||
dorks := []string{
|
||||
fmt.Sprintf(`"%s"`, target),
|
||||
fmt.Sprintf(`"api.%s"`, target),
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{})
|
||||
var seenMu sync.Mutex
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for _, q := range dorks {
|
||||
q := q
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
hits := searchCode(client, q, token)
|
||||
for _, text := range hits {
|
||||
for _, sub := range sources.ExtractSubdomains(text, target) {
|
||||
seenMu.Lock()
|
||||
if _, dup := seen[sub]; dup {
|
||||
seenMu.Unlock()
|
||||
continue
|
||||
}
|
||||
seen[sub] = struct{}{}
|
||||
seenMu.Unlock()
|
||||
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, sub, func(h *store.Host) {
|
||||
store.AddDiscoveryMethod(h, "github-dorks")
|
||||
})
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.SubdomainDiscovered{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: sub},
|
||||
Subdomain: sub,
|
||||
Method: "github-dorks",
|
||||
})
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
// searchCode hits GitHub's code-search endpoint and returns text_matches
|
||||
// fragments (the snippet fields containing the dorked domain). When
|
||||
// unauthenticated it may silently return zero hits due to rate limiting;
|
||||
// the module fails open.
|
||||
func searchCode(client *http.Client, q, token string) []string {
|
||||
u := "https://api.github.com/search/code?q=" + url.QueryEscape(q) + "&per_page=100"
|
||||
req, err := http.NewRequest("GET", u, nil)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
req.Header.Set("Accept", "application/vnd.github.text-match+json")
|
||||
req.Header.Set("User-Agent", "god-eye-v2")
|
||||
if token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if resp.StatusCode == 403 || resp.StatusCode == 429 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var parsed struct {
|
||||
Items []struct {
|
||||
TextMatches []struct {
|
||||
Fragment string `json:"fragment"`
|
||||
} `json:"text_matches"`
|
||||
HTMLURL string `json:"html_url"`
|
||||
} `json:"items"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, it := range parsed.Items {
|
||||
out = append(out, it.HTMLURL)
|
||||
for _, tm := range it.TextMatches {
|
||||
out = append(out, tm.Fragment)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var _ = strings.TrimSpace
|
||||
var _ = context.Canceled
|
||||
@@ -0,0 +1,287 @@
|
||||
// Package graphql detects exposed GraphQL endpoints and tests them for
|
||||
// common misconfigurations: unauthenticated introspection, batched query
|
||||
// abuse, and field-level auth bypass via aliases.
|
||||
//
|
||||
// Probes these paths on every HTTP-probed host:
|
||||
//
|
||||
// /graphql, /graphiql, /api/graphql, /v1/graphql, /v2/graphql,
|
||||
// /query, /api/v1/graphql, /api/v2/graphql
|
||||
//
|
||||
// When an endpoint responds to introspection queries, we publish an
|
||||
// APIFinding + VulnerabilityFound event with the schema size and entry
|
||||
// points as evidence.
|
||||
package graphql
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
gohttp "god-eye/internal/http"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "vuln.graphql"
|
||||
|
||||
type gqlModule struct{}
|
||||
|
||||
func Register() { module.Register(&gqlModule{}) }
|
||||
|
||||
func (*gqlModule) Name() string { return ModuleName }
|
||||
func (*gqlModule) Phase() module.Phase { return module.PhaseAnalysis }
|
||||
func (*gqlModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventHTTPProbed} }
|
||||
func (*gqlModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventAPIFinding, eventbus.EventVulnerability}
|
||||
}
|
||||
func (*gqlModule) DefaultEnabled() bool { return true }
|
||||
|
||||
var candidatePaths = []string{
|
||||
"/graphql",
|
||||
"/graphiql",
|
||||
"/api/graphql",
|
||||
"/v1/graphql",
|
||||
"/v2/graphql",
|
||||
"/query",
|
||||
"/api/v1/graphql",
|
||||
"/api/v2/graphql",
|
||||
"/graphql/console",
|
||||
"/graphql/v1",
|
||||
"/graphql/v2",
|
||||
"/playground",
|
||||
}
|
||||
|
||||
// introspection is the minimal query that exposes the full schema. Sent
|
||||
// with Content-Type: application/json.
|
||||
const introspectionQuery = `{"query":"{__schema{queryType{name} mutationType{name} subscriptionType{name} types{name kind description fields{name} enumValues{name}}}}"}`
|
||||
|
||||
func (*gqlModule) Run(mctx module.Context) error {
|
||||
timeout := mctx.Config.Int("timeout", 10)
|
||||
client := gohttp.GetSharedClient(timeout)
|
||||
|
||||
processed := make(map[string]struct{})
|
||||
var mu sync.Mutex
|
||||
shouldProcess := func(host string) bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if _, ok := processed[host]; ok {
|
||||
return false
|
||||
}
|
||||
processed[host] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Drain store: every host that got a successful HTTP probe.
|
||||
for _, h := range mctx.Store.All(mctx.Ctx) {
|
||||
if h == nil || h.StatusCode == 0 {
|
||||
continue
|
||||
}
|
||||
if !shouldProcess(h.Subdomain) {
|
||||
continue
|
||||
}
|
||||
host := h.Subdomain
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); probeGraphQL(mctx, client, host) }()
|
||||
}
|
||||
|
||||
// Late events.
|
||||
sub := mctx.Bus.Subscribe(eventbus.EventHTTPProbed, func(_ context.Context, e eventbus.Event) {
|
||||
ev, ok := e.(eventbus.HTTPProbed)
|
||||
if !ok || ev.StatusCode == 0 {
|
||||
return
|
||||
}
|
||||
host := ev.Meta().Target
|
||||
if !shouldProcess(host) {
|
||||
return
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); probeGraphQL(mctx, client, host) }()
|
||||
})
|
||||
defer sub.Unsubscribe()
|
||||
|
||||
select {
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func probeGraphQL(mctx module.Context, client *http.Client, host string) {
|
||||
for _, p := range candidatePaths {
|
||||
if mctx.Ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
for _, scheme := range []string{"https://", "http://"} {
|
||||
u := scheme + host + p
|
||||
if finding := tryIntrospection(client, u); finding != nil {
|
||||
publishFinding(mctx, host, u, finding)
|
||||
return // one endpoint per host is enough — rest are typically aliases
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type gqlFinding struct {
|
||||
SchemaSize int
|
||||
TypesCount int
|
||||
HasMutation bool
|
||||
HasSubscription bool
|
||||
QueryTypeName string
|
||||
Sample string // truncated introspection response
|
||||
}
|
||||
|
||||
func tryIntrospection(client *http.Client, url string) *gqlFinding {
|
||||
req, err := http.NewRequest("POST", url, bytes.NewBufferString(introspectionQuery))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", "god-eye-v2")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
resp, err := client.Do(req.WithContext(ctx))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Accept 2xx — the exact shape matters more than status.
|
||||
if resp.StatusCode >= 400 {
|
||||
return nil
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
|
||||
if err != nil || len(body) < 30 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Parse the response; real GraphQL endpoints return {"data": {"__schema": ...}}
|
||||
var parsed struct {
|
||||
Data struct {
|
||||
Schema struct {
|
||||
QueryType map[string]interface{} `json:"queryType"`
|
||||
MutationType map[string]interface{} `json:"mutationType"`
|
||||
SubscriptionType map[string]interface{} `json:"subscriptionType"`
|
||||
Types []struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
} `json:"types"`
|
||||
} `json:"__schema"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
return nil
|
||||
}
|
||||
if parsed.Data.Schema.QueryType == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
fnd := &gqlFinding{
|
||||
SchemaSize: len(body),
|
||||
TypesCount: len(parsed.Data.Schema.Types),
|
||||
HasMutation: parsed.Data.Schema.MutationType != nil,
|
||||
HasSubscription: parsed.Data.Schema.SubscriptionType != nil,
|
||||
}
|
||||
if n, ok := parsed.Data.Schema.QueryType["name"].(string); ok {
|
||||
fnd.QueryTypeName = n
|
||||
}
|
||||
if len(body) > 500 {
|
||||
fnd.Sample = string(body[:500]) + "…"
|
||||
} else {
|
||||
fnd.Sample = string(body)
|
||||
}
|
||||
return fnd
|
||||
}
|
||||
|
||||
func publishFinding(mctx module.Context, host, url string, f *gqlFinding) {
|
||||
now := time.Now()
|
||||
severity := eventbus.SeverityMedium
|
||||
if f.HasMutation {
|
||||
severity = eventbus.SeverityHigh
|
||||
}
|
||||
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, host, func(h *store.Host) {
|
||||
h.Vulnerabilities = append(h.Vulnerabilities, store.Vulnerability{
|
||||
ID: "graphql-introspection",
|
||||
Title: "GraphQL Introspection Enabled",
|
||||
Description: describe(f),
|
||||
Severity: string(severity),
|
||||
URL: url,
|
||||
Evidence: f.Sample,
|
||||
Remediation: "Disable introspection in production GraphQL servers (e.g. Apollo: introspection:false, GraphQL Yoga: introspection:{disable:true}).",
|
||||
OWASP: "A05:2021-Security Misconfiguration",
|
||||
FoundAt: now,
|
||||
})
|
||||
})
|
||||
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.VulnerabilityFound{
|
||||
EventMeta: eventbus.EventMeta{At: now, Source: ModuleName, Target: host},
|
||||
ID: "graphql-introspection",
|
||||
Title: "GraphQL Introspection Enabled",
|
||||
Description: describe(f),
|
||||
Severity: severity,
|
||||
URL: url,
|
||||
Evidence: f.Sample,
|
||||
Remediation: "Disable introspection in production GraphQL servers.",
|
||||
OWASP: "A05:2021-Security Misconfiguration",
|
||||
})
|
||||
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.APIFinding{
|
||||
EventMeta: eventbus.EventMeta{At: now, Source: ModuleName, Target: host},
|
||||
Kind: "graphql-introspection",
|
||||
URL: url,
|
||||
Issue: describe(f),
|
||||
Severity: severity,
|
||||
})
|
||||
}
|
||||
|
||||
func describe(f *gqlFinding) string {
|
||||
parts := []string{"GraphQL endpoint leaks full schema via unauthenticated introspection."}
|
||||
if f.TypesCount > 0 {
|
||||
parts = append(parts, "Types: "+itoa(f.TypesCount)+".")
|
||||
}
|
||||
if f.HasMutation {
|
||||
parts = append(parts, "Mutations enabled — attacker can enumerate write operations.")
|
||||
}
|
||||
if f.HasSubscription {
|
||||
parts = append(parts, "Subscriptions enabled.")
|
||||
}
|
||||
if f.QueryTypeName != "" {
|
||||
parts = append(parts, "Query root: "+f.QueryTypeName)
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func itoa(n int) string {
|
||||
// Small inline formatter avoids importing strconv just for this.
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
var buf [20]byte
|
||||
i := len(buf)
|
||||
neg := n < 0
|
||||
if neg {
|
||||
n = -n
|
||||
}
|
||||
for n > 0 {
|
||||
i--
|
||||
buf[i] = byte('0' + n%10)
|
||||
n /= 10
|
||||
}
|
||||
if neg {
|
||||
i--
|
||||
buf[i] = '-'
|
||||
}
|
||||
return string(buf[i:])
|
||||
}
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
// Package headers performs a detailed inspection of HTTP response headers
|
||||
// and reports every missing or misconfigured security control. Unlike v1's
|
||||
// lightweight header check, this module flags each issue as an individual
|
||||
// VulnerabilityFound event with remediation guidance aligned to OWASP
|
||||
// Secure Headers Project.
|
||||
package headers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
gohttp "god-eye/internal/http"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "vuln.security-headers"
|
||||
|
||||
type hdrModule struct{}
|
||||
|
||||
func Register() { module.Register(&hdrModule{}) }
|
||||
|
||||
func (*hdrModule) Name() string { return ModuleName }
|
||||
func (*hdrModule) Phase() module.Phase { return module.PhaseAnalysis }
|
||||
func (*hdrModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventHTTPProbed} }
|
||||
func (*hdrModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventVulnerability}
|
||||
}
|
||||
func (*hdrModule) DefaultEnabled() bool { return true }
|
||||
|
||||
func (*hdrModule) Run(mctx module.Context) error {
|
||||
timeout := mctx.Config.Int("timeout", 10)
|
||||
client := gohttp.GetSharedClient(timeout)
|
||||
|
||||
processed := make(map[string]struct{})
|
||||
var mu sync.Mutex
|
||||
shouldProcess := func(host string) bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if _, ok := processed[host]; ok {
|
||||
return false
|
||||
}
|
||||
processed[host] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Drain the store.
|
||||
for _, h := range mctx.Store.All(mctx.Ctx) {
|
||||
if h == nil || h.StatusCode == 0 {
|
||||
continue
|
||||
}
|
||||
if !shouldProcess(h.Subdomain) {
|
||||
continue
|
||||
}
|
||||
host := h.Subdomain
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); inspect(mctx, client, host) }()
|
||||
}
|
||||
|
||||
// Late events.
|
||||
sub := mctx.Bus.Subscribe(eventbus.EventHTTPProbed, func(_ context.Context, e eventbus.Event) {
|
||||
ev, ok := e.(eventbus.HTTPProbed)
|
||||
if !ok || ev.StatusCode == 0 {
|
||||
return
|
||||
}
|
||||
host := ev.Meta().Target
|
||||
if !shouldProcess(host) {
|
||||
return
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); inspect(mctx, client, host) }()
|
||||
})
|
||||
defer sub.Unsubscribe()
|
||||
|
||||
select {
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func inspect(mctx module.Context, client *http.Client, host string) {
|
||||
req, err := http.NewRequest("GET", "https://"+host, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
req.Header.Set("User-Agent", "god-eye-v2")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
issues := assess(resp.Header)
|
||||
if len(issues) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, host, func(h *store.Host) {
|
||||
now := time.Now()
|
||||
for _, iss := range issues {
|
||||
h.Vulnerabilities = append(h.Vulnerabilities, store.Vulnerability{
|
||||
ID: iss.id,
|
||||
Title: iss.title,
|
||||
Description: iss.desc,
|
||||
Severity: string(iss.sev),
|
||||
URL: "https://" + host,
|
||||
Remediation: iss.fix,
|
||||
OWASP: "A05:2021-Security Misconfiguration",
|
||||
FoundAt: now,
|
||||
})
|
||||
}
|
||||
})
|
||||
for _, iss := range issues {
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.VulnerabilityFound{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: host},
|
||||
ID: iss.id,
|
||||
Title: iss.title,
|
||||
Description: iss.desc,
|
||||
Severity: iss.sev,
|
||||
URL: "https://" + host,
|
||||
Remediation: iss.fix,
|
||||
OWASP: "A05:2021-Security Misconfiguration",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type issue struct {
|
||||
id, title, desc, fix string
|
||||
sev eventbus.Severity
|
||||
}
|
||||
|
||||
func assess(h http.Header) []issue {
|
||||
var out []issue
|
||||
hasHeader := func(k string) bool { return strings.TrimSpace(h.Get(k)) != "" }
|
||||
|
||||
if !hasHeader("Strict-Transport-Security") {
|
||||
out = append(out, issue{
|
||||
id: "hdr-missing-hsts",
|
||||
title: "Missing Strict-Transport-Security",
|
||||
desc: "HSTS is absent; clients may accept plaintext downgrades.",
|
||||
fix: "Add: Strict-Transport-Security: max-age=63072000; includeSubDomains; preload",
|
||||
sev: eventbus.SeverityMedium,
|
||||
})
|
||||
} else if hsts := h.Get("Strict-Transport-Security"); !strings.Contains(strings.ToLower(hsts), "max-age=") ||
|
||||
!strings.Contains(strings.ToLower(hsts), "includesubdomains") {
|
||||
out = append(out, issue{
|
||||
id: "hdr-weak-hsts",
|
||||
title: "Weak HSTS policy",
|
||||
desc: "HSTS set but missing includeSubDomains and/or sufficient max-age.",
|
||||
fix: "Use: max-age=63072000; includeSubDomains; preload",
|
||||
sev: eventbus.SeverityLow,
|
||||
})
|
||||
}
|
||||
|
||||
if !hasHeader("Content-Security-Policy") {
|
||||
out = append(out, issue{
|
||||
id: "hdr-missing-csp",
|
||||
title: "Missing Content-Security-Policy",
|
||||
desc: "No CSP header; XSS mitigations rely solely on upstream filtering.",
|
||||
fix: "Deploy a nonce-based CSP restricting script-src, object-src 'none'.",
|
||||
sev: eventbus.SeverityMedium,
|
||||
})
|
||||
} else if strings.Contains(strings.ToLower(h.Get("Content-Security-Policy")), "unsafe-inline") {
|
||||
out = append(out, issue{
|
||||
id: "hdr-weak-csp",
|
||||
title: "Weak CSP (allows unsafe-inline)",
|
||||
desc: "CSP allows unsafe-inline, neutralizing most XSS protection.",
|
||||
fix: "Remove unsafe-inline; use nonces or hashes.",
|
||||
sev: eventbus.SeverityMedium,
|
||||
})
|
||||
}
|
||||
|
||||
if !hasHeader("X-Frame-Options") {
|
||||
// Only flag if CSP doesn't include frame-ancestors.
|
||||
csp := strings.ToLower(h.Get("Content-Security-Policy"))
|
||||
if !strings.Contains(csp, "frame-ancestors") {
|
||||
out = append(out, issue{
|
||||
id: "hdr-missing-clickjack",
|
||||
title: "Clickjacking not prevented",
|
||||
desc: "Neither X-Frame-Options nor CSP frame-ancestors is set.",
|
||||
fix: "Add: X-Frame-Options: DENY OR CSP with frame-ancestors 'none'.",
|
||||
sev: eventbus.SeverityLow,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if !hasHeader("X-Content-Type-Options") {
|
||||
out = append(out, issue{
|
||||
id: "hdr-missing-nosniff",
|
||||
title: "Missing X-Content-Type-Options",
|
||||
desc: "MIME sniffing permitted; certain XSS escalations become easier.",
|
||||
fix: "Add: X-Content-Type-Options: nosniff",
|
||||
sev: eventbus.SeverityLow,
|
||||
})
|
||||
}
|
||||
|
||||
if !hasHeader("Referrer-Policy") {
|
||||
out = append(out, issue{
|
||||
id: "hdr-missing-referrer-policy",
|
||||
title: "Missing Referrer-Policy",
|
||||
desc: "Default browser Referrer-Policy leaks URLs to third parties.",
|
||||
fix: "Add: Referrer-Policy: strict-origin-when-cross-origin",
|
||||
sev: eventbus.SeverityLow,
|
||||
})
|
||||
}
|
||||
|
||||
if !hasHeader("Permissions-Policy") && !hasHeader("Feature-Policy") {
|
||||
out = append(out, issue{
|
||||
id: "hdr-missing-permissions-policy",
|
||||
title: "Missing Permissions-Policy",
|
||||
desc: "Browser features (camera, geolocation, USB, etc.) are unrestricted by default.",
|
||||
fix: "Add: Permissions-Policy: camera=(), microphone=(), geolocation=()",
|
||||
sev: eventbus.SeverityInfo,
|
||||
})
|
||||
}
|
||||
|
||||
// Dangerous information disclosure via default server banner.
|
||||
if srv := h.Get("Server"); looksLikeBanner(srv) {
|
||||
out = append(out, issue{
|
||||
id: "hdr-server-banner",
|
||||
title: "Server banner leaks version",
|
||||
desc: "Server header exposes exact software + version: " + srv,
|
||||
fix: "Strip or generalize via proxy/web-server config.",
|
||||
sev: eventbus.SeverityInfo,
|
||||
})
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func looksLikeBanner(s string) bool {
|
||||
s = strings.ToLower(s)
|
||||
return strings.Contains(s, "/") && (strings.Contains(s, ".") || anyDigit(s))
|
||||
}
|
||||
|
||||
func anyDigit(s string) bool {
|
||||
for _, r := range s {
|
||||
if r >= '0' && r <= '9' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
// Package httpprobe probes every resolved host with HTTPS/HTTP and extracts
|
||||
// status code, title, server, technology stack, and TLS information.
|
||||
//
|
||||
// Runs in PhaseEnrichment. Reads hosts from the store (not events) to avoid
|
||||
// the phase-barrier race where late subscribers miss earlier events.
|
||||
package httpprobe
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
gohttp "god-eye/internal/http"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "http.probe"
|
||||
|
||||
type probeModule struct{}
|
||||
|
||||
func Register() { module.Register(&probeModule{}) }
|
||||
|
||||
func (*probeModule) Name() string { return ModuleName }
|
||||
func (*probeModule) Phase() module.Phase { return module.PhaseEnrichment }
|
||||
func (*probeModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventDNSResolved} }
|
||||
func (*probeModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventHTTPProbed, eventbus.EventTLSAnalyzed, eventbus.EventTechDetected}
|
||||
}
|
||||
func (*probeModule) DefaultEnabled() bool { return true }
|
||||
|
||||
func (p *probeModule) Run(mctx module.Context) error {
|
||||
if mctx.Config.Bool("no_probe", false) {
|
||||
return nil
|
||||
}
|
||||
|
||||
conc := mctx.Config.Int("concurrency", 500)
|
||||
if conc <= 0 {
|
||||
conc = 500
|
||||
}
|
||||
timeout := mctx.Config.Int("timeout", 5)
|
||||
|
||||
// Dedup across drain + late events.
|
||||
processed := make(map[string]struct{})
|
||||
var processedMu sync.Mutex
|
||||
shouldProcess := func(host string) bool {
|
||||
processedMu.Lock()
|
||||
defer processedMu.Unlock()
|
||||
if _, dup := processed[host]; dup {
|
||||
return false
|
||||
}
|
||||
processed[host] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
work := make(chan string, conc*2)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < conc; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for host := range work {
|
||||
p.probeOne(mctx, host, timeout)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Drain: every host in the store with at least one IP is worth probing.
|
||||
for _, h := range mctx.Store.All(mctx.Ctx) {
|
||||
if h == nil || h.Subdomain == "" || len(h.IPs) == 0 {
|
||||
continue
|
||||
}
|
||||
if !shouldProcess(h.Subdomain) {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case work <- h.Subdomain:
|
||||
case <-mctx.Ctx.Done():
|
||||
close(work)
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Also listen for late DNSResolved events (recursive/permutation running
|
||||
// concurrently in other modules may produce new resolves during our
|
||||
// phase — pick them up).
|
||||
sub := mctx.Bus.Subscribe(eventbus.EventDNSResolved, func(_ context.Context, e eventbus.Event) {
|
||||
ev, ok := e.(eventbus.DNSResolved)
|
||||
if !ok || len(ev.IPs) == 0 {
|
||||
return
|
||||
}
|
||||
if !shouldProcess(ev.Subdomain) {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case work <- ev.Subdomain:
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
})
|
||||
defer sub.Unsubscribe()
|
||||
|
||||
// Brief window for late arrivals.
|
||||
select {
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
|
||||
close(work)
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *probeModule) probeOne(mctx module.Context, host string, timeout int) {
|
||||
if mctx.Ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
r := gohttp.ProbeHTTP(host, timeout)
|
||||
if r == nil || r.StatusCode == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, host, func(h *store.Host) {
|
||||
h.StatusCode = r.StatusCode
|
||||
h.ContentLength = r.ContentLength
|
||||
h.Title = r.Title
|
||||
h.Server = r.Server
|
||||
if len(r.Tech) > 0 {
|
||||
store.AddTechnologies(h, r.Tech)
|
||||
}
|
||||
h.ResponseMs = r.ResponseMs
|
||||
h.TLSVersion = r.TLSVersion
|
||||
h.TLSIssuer = r.TLSIssuer
|
||||
h.TLSSelfSigned = r.TLSSelfSigned
|
||||
if r.TLSExpiry != "" {
|
||||
if tm, err := time.Parse("2006-01-02", r.TLSExpiry); err == nil {
|
||||
h.TLSExpiry = tm
|
||||
}
|
||||
}
|
||||
if r.TLSFingerprint != nil {
|
||||
fp := *r.TLSFingerprint
|
||||
h.TLSFingerprint = &store.TLSFingerprint{
|
||||
Vendor: fp.Vendor,
|
||||
Product: fp.Product,
|
||||
Version: fp.Version,
|
||||
ApplianceKind: fp.ApplianceType,
|
||||
InternalHosts: append([]string(nil), fp.InternalHosts...),
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.HTTPProbed{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: host},
|
||||
URL: "https://" + host,
|
||||
StatusCode: r.StatusCode,
|
||||
ContentLength: r.ContentLength,
|
||||
Title: r.Title,
|
||||
Server: r.Server,
|
||||
Technologies: append([]string(nil), r.Tech...),
|
||||
ResponseMs: r.ResponseMs,
|
||||
TLSVersion: r.TLSVersion,
|
||||
TLSSelfSigned: r.TLSSelfSigned,
|
||||
})
|
||||
|
||||
for _, t := range r.Tech {
|
||||
if t == "" {
|
||||
continue
|
||||
}
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.TechDetected{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: host},
|
||||
Host: host,
|
||||
Technology: t,
|
||||
Confidence: 0.8,
|
||||
})
|
||||
}
|
||||
|
||||
if r.TLSFingerprint != nil {
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.TLSAnalyzed{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: host},
|
||||
Host: host,
|
||||
Version: r.TLSVersion,
|
||||
Issuer: r.TLSIssuer,
|
||||
SelfSigned: r.TLSSelfSigned,
|
||||
Vendor: r.TLSFingerprint.Vendor,
|
||||
Product: r.TLSFingerprint.Product,
|
||||
ApplianceKind: r.TLSFingerprint.ApplianceType,
|
||||
InternalHosts: append([]string(nil), r.TLSFingerprint.InternalHosts...),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// keep tls import stable
|
||||
var _ = tls.VersionTLS13
|
||||
@@ -0,0 +1,186 @@
|
||||
// Package javascript downloads JS files from probed hosts and scans them
|
||||
// for secrets with the v1 analyzer. Drains the store at start; also listens
|
||||
// for late HTTPProbed events.
|
||||
package javascript
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
gohttp "god-eye/internal/http"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/scanner"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
// publicAPIDenylist covers well-known public/third-party APIs and font
|
||||
// services that the v1 regex scanner flags as "API Endpoint" but which
|
||||
// are never secrets. Matched case-insensitively as a substring.
|
||||
var publicAPIDenylist = []string{
|
||||
"fonts.googleapis.com",
|
||||
"fonts.gstatic.com",
|
||||
"www.googleapis.com",
|
||||
"content.googleapis.com",
|
||||
"api.fastmail.com",
|
||||
"api.forwardemail.net",
|
||||
"cdn.jsdelivr.net",
|
||||
"cdnjs.cloudflare.com",
|
||||
"unpkg.com",
|
||||
}
|
||||
|
||||
// uiStringDenylist covers common UI labels / warning strings that trip
|
||||
// the "Generic Password" regex but are clearly human-readable copy.
|
||||
var uiStringDenylist = []string{
|
||||
"change password",
|
||||
"update password",
|
||||
"reset password",
|
||||
"confirm password",
|
||||
"forgot password",
|
||||
"set-initial-password",
|
||||
"change-password",
|
||||
"this is a very common password",
|
||||
"masterpassword",
|
||||
"password",
|
||||
}
|
||||
|
||||
// isSecretFalsePositive applies cheap deterministic heuristics to weed
|
||||
// out v1 regex noise. Does NOT replace AI triage (which is still the
|
||||
// preferred filter once the ai module is enabled) — it only suppresses
|
||||
// findings that are *definitely* not secrets.
|
||||
func isSecretFalsePositive(secret string) bool {
|
||||
low := strings.ToLower(strings.TrimSpace(secret))
|
||||
for _, s := range publicAPIDenylist {
|
||||
if strings.Contains(low, s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, s := range uiStringDenylist {
|
||||
if strings.Contains(low, s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// Very short matches (< 8 chars of unique content) are almost always
|
||||
// labels, not credentials. The v1 regex already strips the "[Kind] "
|
||||
// prefix before passing to us; anything under 8 chars is noise.
|
||||
if len(low) > 0 && len(low) < 8 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
const ModuleName = "js.analyzer"
|
||||
|
||||
type jsModule struct{}
|
||||
|
||||
func Register() { module.Register(&jsModule{}) }
|
||||
|
||||
func (*jsModule) Name() string { return ModuleName }
|
||||
func (*jsModule) Phase() module.Phase { return module.PhaseAnalysis }
|
||||
func (*jsModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventHTTPProbed} }
|
||||
func (*jsModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventJSFile, eventbus.EventSecret}
|
||||
}
|
||||
func (*jsModule) DefaultEnabled() bool { return true }
|
||||
|
||||
func (*jsModule) Run(mctx module.Context) error {
|
||||
timeout := mctx.Config.Int("timeout", 5)
|
||||
client := gohttp.GetSharedClient(timeout)
|
||||
|
||||
processed := make(map[string]struct{})
|
||||
var mu sync.Mutex
|
||||
shouldProcess := func(host string) bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if _, ok := processed[host]; ok {
|
||||
return false
|
||||
}
|
||||
processed[host] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
analyze := func(host string) {
|
||||
if mctx.Ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
jsFiles, secrets := scanner.AnalyzeJSFiles(host, client)
|
||||
// Drop known-noise findings before they reach the store or bus.
|
||||
filtered := secrets[:0]
|
||||
for _, s := range secrets {
|
||||
if isSecretFalsePositive(s) {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, s)
|
||||
}
|
||||
secrets = filtered
|
||||
if len(jsFiles) == 0 && len(secrets) == 0 {
|
||||
return
|
||||
}
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, host, func(h *store.Host) {
|
||||
for _, sec := range secrets {
|
||||
h.Secrets = append(h.Secrets, store.Secret{
|
||||
Kind: "js-regex",
|
||||
Match: sec,
|
||||
Severity: string(eventbus.SeverityHigh),
|
||||
FoundAt: time.Now(),
|
||||
})
|
||||
}
|
||||
})
|
||||
for _, jsf := range jsFiles {
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.JSFileDiscovered{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: host},
|
||||
URL: jsf,
|
||||
Host: host,
|
||||
})
|
||||
}
|
||||
for _, s := range secrets {
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.SecretFound{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: host},
|
||||
Kind: "js-regex",
|
||||
Match: s,
|
||||
Location: "js-file",
|
||||
Severity: eventbus.SeverityHigh,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Drain: every probed host (StatusCode > 0).
|
||||
for _, h := range mctx.Store.All(mctx.Ctx) {
|
||||
if h == nil || h.StatusCode == 0 {
|
||||
continue
|
||||
}
|
||||
if !shouldProcess(h.Subdomain) {
|
||||
continue
|
||||
}
|
||||
host := h.Subdomain
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); analyze(host) }()
|
||||
}
|
||||
|
||||
// Late events.
|
||||
sub := mctx.Bus.Subscribe(eventbus.EventHTTPProbed, func(_ context.Context, e eventbus.Event) {
|
||||
ev, ok := e.(eventbus.HTTPProbed)
|
||||
if !ok || ev.StatusCode == 0 {
|
||||
return
|
||||
}
|
||||
host := ev.Meta().Target
|
||||
if !shouldProcess(host) {
|
||||
return
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); analyze(host) }()
|
||||
})
|
||||
defer sub.Unsubscribe()
|
||||
|
||||
select {
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
// Package jwt scans responses for JWTs, decodes them, and flags
|
||||
// security-relevant attributes: alg=none, weak HMAC secret (dictionary
|
||||
// crack against common passwords), excessive expiration, missing claims.
|
||||
//
|
||||
// The brute-force list is intentionally tiny (~20 common secrets) — the
|
||||
// goal is to surface obviously-weak keys, not to run offline hashcat. A
|
||||
// proper cracker belongs in Fase 2's planned "auth" agent.
|
||||
package jwt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"crypto/sha512"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"hash"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
gohttp "god-eye/internal/http"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "vuln.jwt"
|
||||
|
||||
type jwtModule struct{}
|
||||
|
||||
func Register() { module.Register(&jwtModule{}) }
|
||||
|
||||
func (*jwtModule) Name() string { return ModuleName }
|
||||
func (*jwtModule) Phase() module.Phase { return module.PhaseAnalysis }
|
||||
func (*jwtModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventHTTPProbed} }
|
||||
func (*jwtModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventVulnerability, eventbus.EventSecret}
|
||||
}
|
||||
func (*jwtModule) DefaultEnabled() bool { return true }
|
||||
|
||||
// jwtRegex matches the standard three-part base64url JWT shape.
|
||||
var jwtRegex = regexp.MustCompile(`eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*`)
|
||||
|
||||
var weakSecrets = []string{
|
||||
"secret", "password", "123456", "admin", "jwt", "jwtsecret",
|
||||
"changeme", "default", "test", "dev", "secret_key", "mysecret",
|
||||
"your-256-bit-secret", "your-secret-key", "super-secret",
|
||||
"supersecret", "helloworld", "qwerty", "abc123", "letmein",
|
||||
}
|
||||
|
||||
func (*jwtModule) Run(mctx module.Context) error {
|
||||
timeout := mctx.Config.Int("timeout", 10)
|
||||
client := gohttp.GetSharedClient(timeout)
|
||||
|
||||
processed := make(map[string]struct{})
|
||||
var mu sync.Mutex
|
||||
shouldProcess := func(host string) bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if _, ok := processed[host]; ok {
|
||||
return false
|
||||
}
|
||||
processed[host] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for _, h := range mctx.Store.All(mctx.Ctx) {
|
||||
if h == nil || h.StatusCode == 0 {
|
||||
continue
|
||||
}
|
||||
if !shouldProcess(h.Subdomain) {
|
||||
continue
|
||||
}
|
||||
host := h.Subdomain
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); scanHost(mctx, client, host) }()
|
||||
}
|
||||
|
||||
sub := mctx.Bus.Subscribe(eventbus.EventHTTPProbed, func(_ context.Context, e eventbus.Event) {
|
||||
ev, ok := e.(eventbus.HTTPProbed)
|
||||
if !ok || ev.StatusCode == 0 {
|
||||
return
|
||||
}
|
||||
host := ev.Meta().Target
|
||||
if !shouldProcess(host) {
|
||||
return
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); scanHost(mctx, client, host) }()
|
||||
})
|
||||
defer sub.Unsubscribe()
|
||||
|
||||
select {
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func scanHost(mctx module.Context, client *http.Client, host string) {
|
||||
for _, scheme := range []string{"https://", "http://"} {
|
||||
if mctx.Ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
url := scheme + host
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("User-Agent", "god-eye-v2")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 256*1024))
|
||||
resp.Body.Close()
|
||||
|
||||
text := string(body)
|
||||
// Also check Authorization + Set-Cookie response headers.
|
||||
for _, h := range resp.Header.Values("Set-Cookie") {
|
||||
text += "\n" + h
|
||||
}
|
||||
if auth := resp.Header.Get("Authorization"); auth != "" {
|
||||
text += "\n" + auth
|
||||
}
|
||||
|
||||
matches := jwtRegex.FindAllString(text, -1)
|
||||
for _, tok := range uniqueStrings(matches) {
|
||||
analyzeJWT(mctx, host, url, tok)
|
||||
}
|
||||
// One scheme is enough; avoid duplicate noise.
|
||||
if len(matches) > 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func analyzeJWT(mctx module.Context, host, url, token string) {
|
||||
parts := strings.Split(token, ".")
|
||||
if len(parts) != 3 {
|
||||
return
|
||||
}
|
||||
header, err := base64Decode(parts[0])
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
payload, err := base64Decode(parts[1])
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var h struct {
|
||||
Alg string `json:"alg"`
|
||||
Kid string `json:"kid"`
|
||||
Typ string `json:"typ"`
|
||||
}
|
||||
if err := json.Unmarshal(header, &h); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
severity := eventbus.SeverityInfo
|
||||
findings := []string{"JWT detected"}
|
||||
|
||||
if strings.EqualFold(h.Alg, "none") {
|
||||
severity = eventbus.SeverityCritical
|
||||
findings = append(findings, "alg=none accepted — no signature verification")
|
||||
}
|
||||
if strings.HasPrefix(strings.ToUpper(h.Alg), "HS") {
|
||||
if cracked := tryWeakSecret(token, h.Alg, parts); cracked != "" {
|
||||
severity = eventbus.SeverityCritical
|
||||
findings = append(findings, "weak HMAC secret cracked: "+cracked)
|
||||
}
|
||||
}
|
||||
if h.Kid != "" && looksInjectable(h.Kid) {
|
||||
severity = maxSeverity(severity, eventbus.SeverityMedium)
|
||||
findings = append(findings, "kid header may be injectable: "+h.Kid)
|
||||
}
|
||||
|
||||
// Inspect payload for excessive expiry.
|
||||
var claims map[string]interface{}
|
||||
_ = json.Unmarshal(payload, &claims)
|
||||
if exp, ok := claims["exp"].(float64); ok {
|
||||
expAt := time.Unix(int64(exp), 0)
|
||||
if time.Until(expAt) > 365*24*time.Hour {
|
||||
severity = maxSeverity(severity, eventbus.SeverityLow)
|
||||
findings = append(findings, "exp >1 year")
|
||||
}
|
||||
}
|
||||
|
||||
redacted := token
|
||||
if len(redacted) > 40 {
|
||||
redacted = redacted[:20] + "…" + redacted[len(redacted)-10:]
|
||||
}
|
||||
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, host, func(sh *store.Host) {
|
||||
sh.Secrets = append(sh.Secrets, store.Secret{
|
||||
Kind: "jwt",
|
||||
Match: redacted,
|
||||
Location: url,
|
||||
Severity: string(severity),
|
||||
Description: strings.Join(findings, "; "),
|
||||
FoundAt: time.Now(),
|
||||
})
|
||||
})
|
||||
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.SecretFound{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: host},
|
||||
Kind: "jwt",
|
||||
Match: redacted,
|
||||
Location: url,
|
||||
Severity: severity,
|
||||
Description: strings.Join(findings, "; "),
|
||||
})
|
||||
|
||||
if severity == eventbus.SeverityCritical || severity == eventbus.SeverityHigh {
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.VulnerabilityFound{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: host},
|
||||
ID: "jwt-weak",
|
||||
Title: "JWT Weakness",
|
||||
Description: strings.Join(findings, "; "),
|
||||
Severity: severity,
|
||||
URL: url,
|
||||
Evidence: redacted,
|
||||
Remediation: "Use strong signing keys (256+ bits of entropy), refuse alg=none, rotate keys on compromise, short expiry.",
|
||||
OWASP: "A02:2021-Cryptographic Failures",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func tryWeakSecret(token, alg string, parts []string) string {
|
||||
signingInput := parts[0] + "." + parts[1]
|
||||
sig, err := base64Decode(parts[2])
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
var hashFn func() hash.Hash
|
||||
switch strings.ToUpper(alg) {
|
||||
case "HS256":
|
||||
hashFn = sha256.New
|
||||
case "HS384":
|
||||
hashFn = func() hash.Hash { return sha512.New384() }
|
||||
case "HS512":
|
||||
hashFn = sha512.New
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
|
||||
for _, s := range weakSecrets {
|
||||
mac := hmac.New(hashFn, []byte(s))
|
||||
mac.Write([]byte(signingInput))
|
||||
if hmac.Equal(mac.Sum(nil), sig) {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// base64Decode unpads and decodes a JWT segment (URL-safe, no padding).
|
||||
func base64Decode(s string) ([]byte, error) {
|
||||
// Add padding if missing.
|
||||
if m := len(s) % 4; m != 0 {
|
||||
s += strings.Repeat("=", 4-m)
|
||||
}
|
||||
return base64.URLEncoding.DecodeString(s)
|
||||
}
|
||||
|
||||
func looksInjectable(kid string) bool {
|
||||
// kids that include path separators, SQL wildcards, or NUL-like
|
||||
// sequences are worth flagging for manual review.
|
||||
return strings.ContainsAny(kid, "/\\;'\"$`|")
|
||||
}
|
||||
|
||||
func maxSeverity(a, b eventbus.Severity) eventbus.Severity {
|
||||
rank := map[eventbus.Severity]int{
|
||||
eventbus.SeverityInfo: 0, eventbus.SeverityLow: 1,
|
||||
eventbus.SeverityMedium: 2, eventbus.SeverityHigh: 3, eventbus.SeverityCritical: 4,
|
||||
}
|
||||
if rank[a] >= rank[b] {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func uniqueStrings(in []string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
out := make([]string, 0, len(in))
|
||||
for _, s := range in {
|
||||
if _, dup := seen[s]; dup {
|
||||
continue
|
||||
}
|
||||
seen[s] = struct{}{}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
// Package nuclei runs Nuclei-format YAML templates against every probed
|
||||
// host. The actual executor lives in internal/nucleitpl; this module is
|
||||
// the wiring that discovers templates on disk, fans out per host, and
|
||||
// publishes matches as VulnerabilityFound events.
|
||||
//
|
||||
// Template discovery order:
|
||||
// 1. --nuclei-templates flag (highest priority)
|
||||
// 2. NUCLEI_TEMPLATES env var
|
||||
// 3. ~/nuclei-templates (nuclei CLI default)
|
||||
// 4. ~/.god-eye/nuclei-templates
|
||||
//
|
||||
// If no template directory is found AND nuclei_auto_download is true
|
||||
// (default), God's Eye downloads the official projectdiscovery/nuclei-templates
|
||||
// ZIP into ~/.god-eye/nuclei-templates, extracts only the .yaml/.yml files
|
||||
// (path-traversal safe), and proceeds with the scan. The archive is
|
||||
// ~40MB; first run takes 10-30 seconds depending on network, subsequent
|
||||
// runs skip the download.
|
||||
//
|
||||
// Refresh the cache manually with: god-eye nuclei-update
|
||||
//
|
||||
// Only HTTP templates compatible with our executor subset run; others
|
||||
// are counted as "skipped" and surfaced as a ModuleError event once per
|
||||
// scan.
|
||||
package nuclei
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
gohttp "god-eye/internal/http"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/nucleitpl"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "vuln.nuclei-compat"
|
||||
|
||||
type nucleiModule struct{}
|
||||
|
||||
func Register() { module.Register(&nucleiModule{}) }
|
||||
|
||||
func (*nucleiModule) Name() string { return ModuleName }
|
||||
func (*nucleiModule) Phase() module.Phase { return module.PhaseAnalysis }
|
||||
func (*nucleiModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventHTTPProbed} }
|
||||
func (*nucleiModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventVulnerability, eventbus.EventCVEMatch}
|
||||
}
|
||||
|
||||
// DefaultEnabled returns true so the registry always loads the module;
|
||||
// Run() itself is a no-op unless `nuclei_scan` is set in the config
|
||||
// (via --nuclei or YAML). Mirrors the ai.cascade module — keeps the
|
||||
// module visible to selection logic while preserving opt-in semantics.
|
||||
func (*nucleiModule) DefaultEnabled() bool { return true }
|
||||
|
||||
func (*nucleiModule) Run(mctx module.Context) error {
|
||||
if !mctx.Config.Bool("nuclei_scan", false) {
|
||||
return nil
|
||||
}
|
||||
|
||||
tplDir := resolveTemplateDir(mctx)
|
||||
if tplDir == "" {
|
||||
// No templates found — try auto-download into ~/.god-eye/nuclei-templates
|
||||
// unless the user explicitly disabled that fallback.
|
||||
if !mctx.Config.Bool("nuclei_auto_download", true) {
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.ModuleError{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: mctx.Target},
|
||||
Module: ModuleName,
|
||||
Err: "no nuclei templates found and --nuclei-auto-download=false. Clone https://github.com/projectdiscovery/nuclei-templates into ~/nuclei-templates or pass --nuclei-templates <path>",
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
dest, err := defaultAutoDownloadDir()
|
||||
if err != nil {
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.ModuleError{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: mctx.Target},
|
||||
Module: ModuleName,
|
||||
Err: fmt.Sprintf("cannot determine default templates dir: %v", err),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
dl := nucleitpl.NewDownloader()
|
||||
dl.Verbose = mctx.Config.Bool("verbose", false) || mctx.Config.Bool("ai.verbose", false)
|
||||
if err := dl.EnsureTemplates(dest); err != nil {
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.ModuleError{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: mctx.Target},
|
||||
Module: ModuleName,
|
||||
Err: fmt.Sprintf("auto-download nuclei templates: %v", err),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
tplDir = dest
|
||||
}
|
||||
|
||||
tpls, diags, err := nucleitpl.LoadDir(tplDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load templates from %s: %w", tplDir, err)
|
||||
}
|
||||
|
||||
supported := 0
|
||||
skipped := 0
|
||||
var supportedTpls []*nucleitpl.Template
|
||||
for _, t := range tpls {
|
||||
if ok, _ := t.IsSupported(); ok {
|
||||
supported++
|
||||
supportedTpls = append(supportedTpls, t)
|
||||
} else {
|
||||
skipped++
|
||||
}
|
||||
}
|
||||
|
||||
if supported == 0 {
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.ModuleError{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: mctx.Target},
|
||||
Module: ModuleName,
|
||||
Err: fmt.Sprintf("loaded %d templates, 0 supported (skipped %d, parse errors %d)", len(tpls), skipped, len(diags)),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
timeout := time.Duration(mctx.Config.Int("timeout", 10)) * time.Second
|
||||
client := gohttp.GetSharedClient(int(timeout.Seconds()))
|
||||
exec := nucleitpl.NewExecutor(client, timeout)
|
||||
|
||||
// Gather target URLs from the store.
|
||||
var targets []string
|
||||
for _, h := range mctx.Store.All(mctx.Ctx) {
|
||||
if h == nil || h.StatusCode == 0 {
|
||||
continue
|
||||
}
|
||||
targets = append(targets, "https://"+h.Subdomain)
|
||||
}
|
||||
if len(targets) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Bounded parallelism: running thousands of templates × hundreds of
|
||||
// hosts unbounded would be a DoS against ourselves and the target.
|
||||
maxConcurrent := mctx.Config.Int("concurrency", 50)
|
||||
if maxConcurrent > 50 {
|
||||
maxConcurrent = 50 // cap — templates make 1-3 requests each
|
||||
}
|
||||
if maxConcurrent < 1 {
|
||||
maxConcurrent = 10
|
||||
}
|
||||
|
||||
sem := make(chan struct{}, maxConcurrent)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for _, url := range targets {
|
||||
for _, t := range supportedTpls {
|
||||
if mctx.Ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
url := url
|
||||
t := t
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
runCtx, cancel := context.WithTimeout(mctx.Ctx, timeout)
|
||||
defer cancel()
|
||||
for _, m := range exec.Run(runCtx, t, url) {
|
||||
publishMatch(mctx, m)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if skipped > 0 {
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.ModuleError{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: mctx.Target},
|
||||
Module: ModuleName,
|
||||
Err: fmt.Sprintf("executed %d templates, skipped %d (unsupported protocol/features)", supported, skipped),
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// publishMatch persists the match into the store and fires a
|
||||
// VulnerabilityFound event. When the match references CVEs, a CVEMatch
|
||||
// event is also fired so the CVE aggregator sees it.
|
||||
func publishMatch(mctx module.Context, m nucleitpl.Match) {
|
||||
now := time.Now()
|
||||
severity := mapSeverity(m.Severity)
|
||||
host := hostFromURL(m.URL)
|
||||
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, host, func(h *store.Host) {
|
||||
h.Vulnerabilities = append(h.Vulnerabilities, store.Vulnerability{
|
||||
ID: "nuclei/" + m.TemplateID,
|
||||
Title: m.Name,
|
||||
Description: m.Description,
|
||||
Severity: string(severity),
|
||||
URL: m.URL,
|
||||
Evidence: m.Evidence,
|
||||
CVEs: append([]string(nil), m.CVEs...),
|
||||
FoundAt: now,
|
||||
})
|
||||
for _, cveID := range m.CVEs {
|
||||
h.CVEs = append(h.CVEs, store.CVE{
|
||||
ID: cveID,
|
||||
Technology: m.TemplateID,
|
||||
Severity: string(severity),
|
||||
FoundAt: now,
|
||||
URL: m.TemplateURL,
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.VulnerabilityFound{
|
||||
EventMeta: eventbus.EventMeta{At: now, Source: ModuleName, Target: host},
|
||||
ID: "nuclei/" + m.TemplateID,
|
||||
Title: m.Name,
|
||||
Description: m.Description,
|
||||
Severity: severity,
|
||||
URL: m.URL,
|
||||
Evidence: m.Evidence,
|
||||
CVEs: append([]string(nil), m.CVEs...),
|
||||
})
|
||||
|
||||
for _, cveID := range m.CVEs {
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.CVEMatch{
|
||||
EventMeta: eventbus.EventMeta{At: now, Source: ModuleName, Target: host},
|
||||
CVE: cveID,
|
||||
Technology: m.TemplateID,
|
||||
Severity: severity,
|
||||
Description: m.Name,
|
||||
URL: m.TemplateURL,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func mapSeverity(s string) eventbus.Severity {
|
||||
switch s {
|
||||
case "critical":
|
||||
return eventbus.SeverityCritical
|
||||
case "high":
|
||||
return eventbus.SeverityHigh
|
||||
case "medium":
|
||||
return eventbus.SeverityMedium
|
||||
case "low":
|
||||
return eventbus.SeverityLow
|
||||
default:
|
||||
return eventbus.SeverityInfo
|
||||
}
|
||||
}
|
||||
|
||||
// resolveTemplateDir returns the first USABLE template directory, in
|
||||
// priority order. "Usable" means it exists, is a directory, and the
|
||||
// process can list its contents (i.e. not a permission-denied mount
|
||||
// like a read-restricted nuclei install in another user's home).
|
||||
// Returns "" when no candidate qualifies.
|
||||
func resolveTemplateDir(mctx module.Context) string {
|
||||
candidates := []string{
|
||||
mctx.Config.String("nuclei_templates", ""),
|
||||
os.Getenv("NUCLEI_TEMPLATES"),
|
||||
}
|
||||
if home, err := os.UserHomeDir(); err == nil {
|
||||
// Prefer the god-eye auto-managed cache over a pre-existing
|
||||
// ~/nuclei-templates: the latter may be a nuclei CLI install
|
||||
// with restrictive permissions we can't read.
|
||||
candidates = append(candidates,
|
||||
filepath.Join(home, ".god-eye", "nuclei-templates"),
|
||||
filepath.Join(home, "nuclei-templates"),
|
||||
)
|
||||
}
|
||||
for _, c := range candidates {
|
||||
if c == "" {
|
||||
continue
|
||||
}
|
||||
info, err := os.Stat(c)
|
||||
if err != nil || !info.IsDir() {
|
||||
continue
|
||||
}
|
||||
// Readability check: can we list at least one entry? If the dir
|
||||
// is permission-denied, os.Stat succeeds but os.Open fails —
|
||||
// skip such candidates so auto-download fallback triggers.
|
||||
f, err := os.Open(c)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
names, err := f.Readdirnames(1)
|
||||
f.Close()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if len(names) == 0 {
|
||||
// Empty dir — treat as unusable to trigger auto-download.
|
||||
continue
|
||||
}
|
||||
return c
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// defaultAutoDownloadDir returns ~/.god-eye/nuclei-templates.
|
||||
func defaultAutoDownloadDir() (string, error) {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(home, ".god-eye", "nuclei-templates"), nil
|
||||
}
|
||||
|
||||
func hostFromURL(u string) string {
|
||||
// Strip scheme.
|
||||
s := u
|
||||
for _, p := range []string{"https://", "http://"} {
|
||||
if len(s) > len(p) && s[:len(p)] == p {
|
||||
s = s[len(p):]
|
||||
break
|
||||
}
|
||||
}
|
||||
// Strip path.
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] == '/' || s[i] == '?' || s[i] == '#' {
|
||||
return s[:i]
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
// Package passive is the Fase 0.6 adapter that wraps the v1 passive sources
|
||||
// (internal/sources) as a single Module. It fans out queries to all 20 public
|
||||
// sources in parallel and emits a SubdomainDiscovered event for each result.
|
||||
//
|
||||
// In Fase 1 (Discovery Supremacy) each source will become its own Module with
|
||||
// independent configuration, error reporting, and rate limiting. This
|
||||
// adapter preserves v1 behavior so we reach feature parity immediately.
|
||||
package passive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/sources"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
// ModuleName is the registry identifier.
|
||||
const ModuleName = "passive.v1-aggregate"
|
||||
|
||||
type passiveModule struct{}
|
||||
|
||||
// Register the module in the default registry. Callers import this package
|
||||
// for side effects via the modules meta-package (see internal/modules/all).
|
||||
func Register() { module.Register(&passiveModule{}) }
|
||||
|
||||
func (*passiveModule) Name() string { return ModuleName }
|
||||
func (*passiveModule) Phase() module.Phase { return module.PhaseDiscovery }
|
||||
|
||||
func (*passiveModule) Consumes() []eventbus.EventType { return nil }
|
||||
func (*passiveModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventSubdomainDiscovered, eventbus.EventModuleError}
|
||||
}
|
||||
|
||||
func (*passiveModule) DefaultEnabled() bool { return true }
|
||||
|
||||
// sourceList mirrors the v1 scanner.Run list. Order is preserved for stable
|
||||
// logging.
|
||||
var sourceList = []struct {
|
||||
name string
|
||||
fn func(string) ([]string, error)
|
||||
}{
|
||||
{"crt.sh", sources.FetchCrtsh},
|
||||
{"Certspotter", sources.FetchCertspotter},
|
||||
{"AlienVault", sources.FetchAlienVault},
|
||||
{"HackerTarget", sources.FetchHackerTarget},
|
||||
{"URLScan", sources.FetchURLScan},
|
||||
{"RapidDNS", sources.FetchRapidDNS},
|
||||
{"Anubis", sources.FetchAnubis},
|
||||
{"ThreatMiner", sources.FetchThreatMiner},
|
||||
{"DNSRepo", sources.FetchDNSRepo},
|
||||
{"SubdomainCenter", sources.FetchSubdomainCenter},
|
||||
{"Wayback", sources.FetchWayback},
|
||||
{"CommonCrawl", sources.FetchCommonCrawl},
|
||||
{"Sitedossier", sources.FetchSitedossier},
|
||||
{"Riddler", sources.FetchRiddler},
|
||||
{"Robtex", sources.FetchRobtex},
|
||||
{"DNSHistory", sources.FetchDNSHistory},
|
||||
{"ArchiveToday", sources.FetchArchiveToday},
|
||||
{"JLDC", sources.FetchJLDC},
|
||||
{"SynapsInt", sources.FetchSynapsInt},
|
||||
{"CensysFree", sources.FetchCensysFree},
|
||||
// v2.0 additions — free, no API key, fail-open. Dormant v1 sources
|
||||
// re-activated + 4 net-new endpoints.
|
||||
{"BufferOver", sources.FetchBufferOver}, // dormant v1
|
||||
{"DNSDumpster", sources.FetchDNSDumpster}, // dormant v1
|
||||
{"Omnisint", sources.FetchOmnisint}, // v2 new
|
||||
{"HudsonRock", sources.FetchHudsonRock}, // v2 new
|
||||
{"WebArchiveCDX", sources.FetchWebArchiveCDX}, // v2 new
|
||||
{"Digitorus", sources.FetchDigitorus}, // v2 new
|
||||
}
|
||||
|
||||
func (m *passiveModule) Run(mctx module.Context) error {
|
||||
target := mctx.Target
|
||||
if target == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
// Dedup across sources before emitting — the store will also dedup, but
|
||||
// emitting duplicates just burns bus bandwidth.
|
||||
seen := make(map[string]struct{})
|
||||
var seenMu sync.Mutex
|
||||
|
||||
for _, src := range sourceList {
|
||||
src := src
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
// Respect ctx cancellation between slow sources.
|
||||
if err := mctx.Ctx.Err(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
subs, err := src.fn(target)
|
||||
if err != nil {
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.ModuleError{
|
||||
EventMeta: eventbus.EventMeta{Source: ModuleName + ":" + src.name, Target: target},
|
||||
Module: ModuleName + ":" + src.name,
|
||||
Err: err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
for _, sub := range subs {
|
||||
sub = strings.ToLower(strings.TrimSpace(sub))
|
||||
if sub == "" {
|
||||
continue
|
||||
}
|
||||
if !strings.HasSuffix(sub, target) {
|
||||
continue
|
||||
}
|
||||
seenMu.Lock()
|
||||
if _, dup := seen[sub]; dup {
|
||||
seenMu.Unlock()
|
||||
continue
|
||||
}
|
||||
seen[sub] = struct{}{}
|
||||
seenMu.Unlock()
|
||||
|
||||
// Persist into the store so downstream resolution phases
|
||||
// can find the subdomain even if they subscribed too late
|
||||
// to receive the SubdomainDiscovered event.
|
||||
methodTag := "passive:" + src.name
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, sub, func(h *store.Host) {
|
||||
store.AddDiscoveryMethod(h, methodTag)
|
||||
})
|
||||
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.NewSubdomainDiscovered(
|
||||
ModuleName+":"+src.name,
|
||||
sub,
|
||||
methodTag,
|
||||
))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait for sources OR cancellation.
|
||||
done := make(chan struct{})
|
||||
go func() { wg.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
_ = context.Canceled // keep import
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
// Package permutation generates candidate subdomains by mutating every
|
||||
// previously-discovered subdomain with a set of common prefixes/suffixes
|
||||
// and resolving them. This is the "alterx" pattern: you already found
|
||||
// api.example.com and dev.example.com, now try api-dev, dev-api,
|
||||
// api-staging, api.dev.example.com, etc.
|
||||
//
|
||||
// Pattern learning is intentionally lightweight in Fase 1: the core v1
|
||||
// discovery.PatternLearner already extracts per-label frequencies. We
|
||||
// feed those back in via candidate generation.
|
||||
package permutation
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/config"
|
||||
godns "god-eye/internal/dns"
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "discovery.permutation"
|
||||
|
||||
type permModule struct{}
|
||||
|
||||
func Register() { module.Register(&permModule{}) }
|
||||
|
||||
func (*permModule) Name() string { return ModuleName }
|
||||
func (*permModule) Phase() module.Phase { return module.PhaseResolution }
|
||||
func (*permModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventDNSResolved} }
|
||||
func (*permModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventSubdomainDiscovered}
|
||||
}
|
||||
func (*permModule) DefaultEnabled() bool { return false } // opt-in (burns a lot of DNS)
|
||||
|
||||
// commonAffixes are applied to each label of discovered hostnames to
|
||||
// generate permutation candidates. Curated for bug-bounty signal.
|
||||
var commonAffixes = []string{
|
||||
"dev", "stg", "staging", "prod", "qa", "test", "uat", "sandbox", "preview",
|
||||
"internal", "int", "private", "admin", "api", "api2", "apiv2", "gw",
|
||||
"new", "old", "legacy", "v2", "v3", "next", "beta", "alpha", "canary",
|
||||
"eu", "us", "apac", "emea",
|
||||
}
|
||||
|
||||
var separators = []string{"-", "_", "."}
|
||||
|
||||
func (*permModule) Run(mctx module.Context) error {
|
||||
if !mctx.Config.Bool("permutation", false) {
|
||||
return nil
|
||||
}
|
||||
|
||||
target := mctx.Target
|
||||
timeout := mctx.Config.Int("timeout", 5)
|
||||
resolvers := parseResolvers(mctx.Config.String("resolvers", ""))
|
||||
conc := mctx.Config.Int("concurrency", 300)
|
||||
if conc <= 0 {
|
||||
conc = 300
|
||||
}
|
||||
|
||||
// Gather seeds from the store (all already-resolved hosts).
|
||||
seeds := mctx.Store.All(mctx.Ctx)
|
||||
if len(seeds) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
candidates := make(map[string]struct{})
|
||||
for _, h := range seeds {
|
||||
for _, c := range generateCandidates(h.Subdomain, target) {
|
||||
candidates[c] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve candidates in parallel. Only emit ones that resolve.
|
||||
sem := make(chan struct{}, conc)
|
||||
var wg sync.WaitGroup
|
||||
for cand := range candidates {
|
||||
if mctx.Ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
cand := cand
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
|
||||
ips := godns.ResolveSubdomain(cand, resolvers, timeout)
|
||||
if len(ips) == 0 {
|
||||
return
|
||||
}
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, cand, func(h *store.Host) {
|
||||
store.AddIPs(h, ips)
|
||||
store.AddDiscoveryMethod(h, "permutation")
|
||||
})
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.SubdomainDiscovered{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: cand},
|
||||
Subdomain: cand,
|
||||
Method: "permutation",
|
||||
})
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
// generateCandidates produces permuted hostnames from a seed within the
|
||||
// target domain. The output is guaranteed to end in "."+target or ==target.
|
||||
func generateCandidates(seed, target string) []string {
|
||||
if !strings.HasSuffix(seed, target) {
|
||||
return nil
|
||||
}
|
||||
prefix := strings.TrimSuffix(seed, "."+target)
|
||||
if prefix == target || prefix == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
labels := strings.Split(prefix, ".")
|
||||
if len(labels) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
out := make(map[string]struct{})
|
||||
// Leaf-label mutations: (affix)(sep)(label) and (label)(sep)(affix).
|
||||
leaf := labels[len(labels)-1]
|
||||
rest := strings.Join(labels[:len(labels)-1], ".")
|
||||
for _, aff := range commonAffixes {
|
||||
for _, sep := range separators {
|
||||
combos := []string{
|
||||
aff + sep + leaf,
|
||||
leaf + sep + aff,
|
||||
}
|
||||
for _, c := range combos {
|
||||
parts := []string{c}
|
||||
if rest != "" {
|
||||
parts = []string{rest, c}
|
||||
}
|
||||
cand := strings.Join(parts, ".") + "." + target
|
||||
out[cand] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Prepend-an-affix mutation: aff.<existing>
|
||||
for _, aff := range commonAffixes {
|
||||
cand := aff + "." + prefix + "." + target
|
||||
out[cand] = struct{}{}
|
||||
}
|
||||
|
||||
res := make([]string, 0, len(out))
|
||||
for c := range out {
|
||||
res = append(res, c)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func parseResolvers(s string) []string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return config.DefaultResolvers
|
||||
}
|
||||
var out []string
|
||||
for _, r := range strings.Split(s, ",") {
|
||||
r = strings.TrimSpace(r)
|
||||
if r == "" {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(r, ":") {
|
||||
r = r + ":53"
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return config.DefaultResolvers
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
// Package ports runs a TCP connect scan on the common ports list for every
|
||||
// resolved host. Drains the store at start; also reacts to late DNSResolved
|
||||
// events for concurrent discovery phases.
|
||||
package ports
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/scanner"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "ports.scan"
|
||||
|
||||
type portsModule struct{}
|
||||
|
||||
func Register() { module.Register(&portsModule{}) }
|
||||
|
||||
func (*portsModule) Name() string { return ModuleName }
|
||||
func (*portsModule) Phase() module.Phase { return module.PhaseEnrichment }
|
||||
func (*portsModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventDNSResolved} }
|
||||
func (*portsModule) Produces() []eventbus.EventType { return nil }
|
||||
func (*portsModule) DefaultEnabled() bool { return true }
|
||||
|
||||
func (*portsModule) Run(mctx module.Context) error {
|
||||
if mctx.Config.Bool("no_ports", false) {
|
||||
return nil
|
||||
}
|
||||
timeout := mctx.Config.Int("timeout", 5)
|
||||
portList := parsePorts(mctx.Config.String("ports", ""))
|
||||
|
||||
processed := make(map[string]struct{})
|
||||
var mu sync.Mutex
|
||||
shouldProcess := func(host string) bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if _, ok := processed[host]; ok {
|
||||
return false
|
||||
}
|
||||
processed[host] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
scan := func(host string, ip string) {
|
||||
if mctx.Ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
open := scanner.ScanPorts(ip, portList, timeout)
|
||||
if len(open) == 0 {
|
||||
return
|
||||
}
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, host, func(h *store.Host) {
|
||||
h.Ports = append(h.Ports, open...)
|
||||
})
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Drain.
|
||||
for _, h := range mctx.Store.All(mctx.Ctx) {
|
||||
if h == nil || h.Subdomain == "" || len(h.IPs) == 0 {
|
||||
continue
|
||||
}
|
||||
if !shouldProcess(h.Subdomain) {
|
||||
continue
|
||||
}
|
||||
host := h.Subdomain
|
||||
ip := h.IPs[0]
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); scan(host, ip) }()
|
||||
}
|
||||
|
||||
// Late events.
|
||||
sub := mctx.Bus.Subscribe(eventbus.EventDNSResolved, func(_ context.Context, e eventbus.Event) {
|
||||
ev, ok := e.(eventbus.DNSResolved)
|
||||
if !ok || len(ev.IPs) == 0 {
|
||||
return
|
||||
}
|
||||
if !shouldProcess(ev.Subdomain) {
|
||||
return
|
||||
}
|
||||
host := ev.Subdomain
|
||||
ip := ev.IPs[0]
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); scan(host, ip) }()
|
||||
})
|
||||
defer sub.Unsubscribe()
|
||||
|
||||
select {
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func parsePorts(s string) []int {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return []int{80, 443, 8080, 8443}
|
||||
}
|
||||
var out []int
|
||||
for _, p := range strings.Split(s, ",") {
|
||||
var port int
|
||||
if _, err := fmt.Sscanf(strings.TrimSpace(p), "%d", &port); err == nil && port > 0 && port < 65536 {
|
||||
out = append(out, port)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return []int{80, 443, 8080, 8443}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
// Package recursive is a Fase 0.6 adapter for the v1 recursive discovery
|
||||
// engine (pattern learning from found subdomains).
|
||||
//
|
||||
// Unlike event-driven modules, recursive runs as a deferred second-pass:
|
||||
// after PhaseDiscovery completes it collects every host seen so far from
|
||||
// the store, runs the v1 engine, and emits SubdomainDiscovered for any
|
||||
// new hosts. It self-schedules in PhaseResolution to sit between discovery
|
||||
// and HTTP probing.
|
||||
package recursive
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"god-eye/internal/discovery"
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/store"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const ModuleName = "discovery.recursive"
|
||||
|
||||
type recModule struct{}
|
||||
|
||||
func Register() { module.Register(&recModule{}) }
|
||||
|
||||
func (*recModule) Name() string { return ModuleName }
|
||||
func (*recModule) Phase() module.Phase { return module.PhaseResolution } // runs after discovery
|
||||
func (*recModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventSubdomainDiscovered} }
|
||||
func (*recModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventSubdomainDiscovered}
|
||||
}
|
||||
|
||||
// Recursive is opt-in by default — profiles enable it for bugbounty/pentest.
|
||||
func (*recModule) DefaultEnabled() bool { return false }
|
||||
|
||||
func (*recModule) Run(mctx module.Context) error {
|
||||
if !mctx.Config.Bool("recursive", false) {
|
||||
return nil
|
||||
}
|
||||
|
||||
target := mctx.Target
|
||||
depth := mctx.Config.Int("recursive.depth", 3)
|
||||
if depth < 1 {
|
||||
depth = 1
|
||||
} else if depth > 5 {
|
||||
depth = 5
|
||||
}
|
||||
timeout := mctx.Config.Int("timeout", 5)
|
||||
conc := mctx.Config.Int("concurrency", 500)
|
||||
if conc <= 0 {
|
||||
conc = 500
|
||||
}
|
||||
|
||||
resolvers := parseResolvers(mctx.Config.String("resolvers", ""))
|
||||
|
||||
// Gather initial seeds from what's been discovered so far.
|
||||
hosts := mctx.Store.All(mctx.Ctx)
|
||||
seeds := make([]string, 0, len(hosts))
|
||||
for _, h := range hosts {
|
||||
seeds = append(seeds, h.Subdomain)
|
||||
}
|
||||
if len(seeds) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
rd := discovery.NewRecursiveDiscovery(discovery.RecursiveConfig{
|
||||
Domain: target,
|
||||
Resolvers: resolvers,
|
||||
Timeout: timeout,
|
||||
MaxDepth: depth,
|
||||
Concurrency: conc,
|
||||
})
|
||||
found := rd.Discover(mctx.Ctx, seeds)
|
||||
|
||||
// Emit SubdomainDiscovered for any new hosts.
|
||||
seen := make(map[string]struct{}, len(seeds))
|
||||
for _, s := range seeds {
|
||||
seen[s] = struct{}{}
|
||||
}
|
||||
for _, s := range found {
|
||||
if _, dup := seen[s]; dup {
|
||||
continue
|
||||
}
|
||||
seen[s] = struct{}{}
|
||||
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, s, func(h *store.Host) {
|
||||
store.AddDiscoveryMethod(h, "recursive")
|
||||
})
|
||||
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.SubdomainDiscovered{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: s},
|
||||
Subdomain: s,
|
||||
Method: "recursive",
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseResolvers(s string) []string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return []string{"8.8.8.8:53", "1.1.1.1:53"}
|
||||
}
|
||||
var out []string
|
||||
for _, r := range strings.Split(s, ",") {
|
||||
r = strings.TrimSpace(r)
|
||||
if r == "" {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(r, ":") {
|
||||
r = r + ":53"
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
// Package report writes the final scan output. It consumes the store (not
|
||||
// events) at ScanCompleted time and emits TXT / JSON / CSV via the existing
|
||||
// v1 output.WriteOutput function. To preserve v1 output shape during the
|
||||
// Fase 0.6 migration, store.Host records are projected to the legacy
|
||||
// config.SubdomainResult type before serialization.
|
||||
package report
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/config"
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/output"
|
||||
)
|
||||
|
||||
var _ = time.Now // keep import stable when unused in certain branches
|
||||
|
||||
const ModuleName = "report.output"
|
||||
|
||||
type reportModule struct{}
|
||||
|
||||
func Register() { module.Register(&reportModule{}) }
|
||||
|
||||
func (*reportModule) Name() string { return ModuleName }
|
||||
func (*reportModule) Phase() module.Phase { return module.PhaseReporting }
|
||||
func (*reportModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventScanCompleted} }
|
||||
func (*reportModule) Produces() []eventbus.EventType { return nil }
|
||||
func (*reportModule) DefaultEnabled() bool { return true }
|
||||
|
||||
func (*reportModule) Run(mctx module.Context) error {
|
||||
// Block until the scan is complete — we're last in the pipeline and the
|
||||
// coordinator guarantees reporting runs after every earlier phase.
|
||||
done := make(chan struct{}, 1)
|
||||
sub := mctx.Bus.Subscribe(eventbus.EventScanCompleted, func(_ context.Context, _ eventbus.Event) {
|
||||
select {
|
||||
case done <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
})
|
||||
defer sub.Unsubscribe()
|
||||
|
||||
// The report module itself runs in PhaseReporting which is the last
|
||||
// phase. ScanCompleted fires right after this phase ends, so we can't
|
||||
// rely on it — write output directly from the store instead.
|
||||
_ = done
|
||||
|
||||
results := projectStoreToResults(mctx)
|
||||
if len(results) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
silent := mctx.Config.Bool("silent", false)
|
||||
jsonStdout := mctx.Config.Bool("json", false)
|
||||
onlyActive := mctx.Config.Bool("only_active", false)
|
||||
outPath := mctx.Config.String("output", "")
|
||||
format := mctx.Config.String("format", "txt")
|
||||
|
||||
if jsonStdout {
|
||||
// Project a minimal JSON report to stdout, shape-compatible with v1.
|
||||
writeJSONStdout(mctx, results)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Console presentation — only when not silent / not JSON-only mode.
|
||||
if !silent {
|
||||
printResults(results, onlyActive)
|
||||
}
|
||||
|
||||
if outPath != "" {
|
||||
if err := writeFile(outPath, format, results); err != nil {
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.ModuleError{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: mctx.Target},
|
||||
Module: ModuleName,
|
||||
Err: fmt.Sprintf("write output %s: %v", outPath, err),
|
||||
})
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// projectStoreToResults converts store.Host records to the legacy
|
||||
// config.SubdomainResult shape expected by output.WriteOutput. Doing the
|
||||
// projection here keeps the store schema decoupled from the v1 output format.
|
||||
func projectStoreToResults(mctx module.Context) map[string]*config.SubdomainResult {
|
||||
hosts := mctx.Store.All(mctx.Ctx)
|
||||
out := make(map[string]*config.SubdomainResult, len(hosts))
|
||||
for _, h := range hosts {
|
||||
r := &config.SubdomainResult{
|
||||
Subdomain: h.Subdomain,
|
||||
IPs: append([]string(nil), h.IPs...),
|
||||
CNAME: h.CNAME,
|
||||
PTR: h.PTR,
|
||||
ASN: h.ASN,
|
||||
Org: h.Org,
|
||||
Country: h.Country,
|
||||
City: h.City,
|
||||
StatusCode: h.StatusCode,
|
||||
ContentLength: h.ContentLength,
|
||||
Title: h.Title,
|
||||
Server: h.Server,
|
||||
Tech: append([]string(nil), h.Technologies...),
|
||||
WAF: h.WAF,
|
||||
TLSVersion: h.TLSVersion,
|
||||
TLSIssuer: h.TLSIssuer,
|
||||
TLSSelfSigned: h.TLSSelfSigned,
|
||||
Ports: append([]int(nil), h.Ports...),
|
||||
ResponseMs: h.ResponseMs,
|
||||
CloudProvider: h.CloudProvider,
|
||||
}
|
||||
if !h.TLSExpiry.IsZero() {
|
||||
r.TLSExpiry = h.TLSExpiry.Format("2006-01-02")
|
||||
}
|
||||
if h.TLSFingerprint != nil {
|
||||
r.TLSFingerprint = &config.TLSFingerprint{
|
||||
Vendor: h.TLSFingerprint.Vendor,
|
||||
Product: h.TLSFingerprint.Product,
|
||||
Version: h.TLSFingerprint.Version,
|
||||
ApplianceType: h.TLSFingerprint.ApplianceKind,
|
||||
InternalHosts: append([]string(nil), h.TLSFingerprint.InternalHosts...),
|
||||
}
|
||||
}
|
||||
if h.Takeover != nil {
|
||||
r.Takeover = h.Takeover.Service
|
||||
}
|
||||
// Flatten vulnerabilities → scalar fields v1 consumers expect.
|
||||
for _, v := range h.Vulnerabilities {
|
||||
switch v.ID {
|
||||
case "open-redirect":
|
||||
r.OpenRedirect = true
|
||||
case "cors-misconfig":
|
||||
r.CORSMisconfig = v.Description
|
||||
case "dangerous-http-methods":
|
||||
r.DangerousMethods = append(r.DangerousMethods, strings.Split(v.Evidence, ", ")...)
|
||||
case "git-exposed":
|
||||
r.GitExposed = true
|
||||
case "svn-exposed":
|
||||
r.SvnExposed = true
|
||||
case "backup-file":
|
||||
r.BackupFiles = append(r.BackupFiles, v.URL)
|
||||
}
|
||||
}
|
||||
// Secrets → legacy field
|
||||
for _, s := range h.Secrets {
|
||||
r.JSSecrets = append(r.JSSecrets, s.Match)
|
||||
}
|
||||
// CVEs / AI
|
||||
for _, c := range h.CVEs {
|
||||
r.CVEFindings = append(r.CVEFindings, c.ID)
|
||||
}
|
||||
for _, a := range h.AIFindings {
|
||||
r.AIFindings = append(r.AIFindings, a.Title)
|
||||
if r.AISeverity == "" {
|
||||
r.AISeverity = a.Severity
|
||||
}
|
||||
if r.AIModel == "" {
|
||||
r.AIModel = a.Model
|
||||
}
|
||||
}
|
||||
out[h.Subdomain] = r
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// printResults is a minimal, non-colorful table print. The full v1
|
||||
// presentation is re-introduced when the TUI module lands in Fase 4.
|
||||
func printResults(results map[string]*config.SubdomainResult, onlyActive bool) {
|
||||
// Sorted output for determinism.
|
||||
names := make([]string, 0, len(results))
|
||||
for n := range results {
|
||||
names = append(names, n)
|
||||
}
|
||||
// sort by status desc, then name
|
||||
sortResultsForPrint(names, results)
|
||||
|
||||
active := 0
|
||||
for _, n := range names {
|
||||
r := results[n]
|
||||
if r.StatusCode == 0 {
|
||||
if onlyActive {
|
||||
continue
|
||||
}
|
||||
fmt.Printf(" %s %s\n", output.Dim("○"), r.Subdomain)
|
||||
continue
|
||||
}
|
||||
active++
|
||||
marker := output.Green("●")
|
||||
if r.StatusCode >= 300 && r.StatusCode < 400 {
|
||||
marker = output.Yellow("◐")
|
||||
} else if r.StatusCode >= 400 {
|
||||
marker = output.Red("○")
|
||||
}
|
||||
tech := ""
|
||||
if len(r.Tech) > 0 {
|
||||
tech = output.Dim(" [" + strings.Join(r.Tech, ", ") + "]")
|
||||
}
|
||||
fmt.Printf(" %s %s %s%s\n", marker, r.Subdomain, output.Dim(fmt.Sprintf("[%d]", r.StatusCode)), tech)
|
||||
}
|
||||
fmt.Println()
|
||||
fmt.Printf(" %s total, %s active\n", output.BoldWhite(fmt.Sprintf("%d", len(results))), output.BoldGreen(fmt.Sprintf("%d", active)))
|
||||
}
|
||||
|
||||
func sortResultsForPrint(names []string, results map[string]*config.SubdomainResult) {
|
||||
// Simple insertion-sort quality ok for small lists; stable enough.
|
||||
n := len(names)
|
||||
for i := 1; i < n; i++ {
|
||||
j := i
|
||||
for j > 0 && lessResult(results[names[j]], results[names[j-1]]) {
|
||||
names[j], names[j-1] = names[j-1], names[j]
|
||||
j--
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func lessResult(a, b *config.SubdomainResult) bool {
|
||||
// Active first, then by subdomain name.
|
||||
aActive := a.StatusCode >= 200 && a.StatusCode < 400
|
||||
bActive := b.StatusCode >= 200 && b.StatusCode < 400
|
||||
if aActive != bActive {
|
||||
return aActive && !bActive
|
||||
}
|
||||
return a.Subdomain < b.Subdomain
|
||||
}
|
||||
|
||||
func writeFile(path, format string, results map[string]*config.SubdomainResult) error {
|
||||
// v1 exposes SaveOutput (void); we funnel through it but surface errors
|
||||
// by re-checking file writability up front.
|
||||
format = strings.ToLower(strings.TrimSpace(format))
|
||||
if format == "" {
|
||||
format = "txt"
|
||||
}
|
||||
// Pre-flight: make sure we can create the target file before delegating.
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.Close()
|
||||
output.SaveOutput(path, format, results)
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeJSONStdout emits a v2-native minimal JSON dump to stdout. This is
|
||||
// intentionally simpler than v1's ReportBuilder — when the full report
|
||||
// generator lands in Fase 4 (Reporting), this is where it'll be wired.
|
||||
func writeJSONStdout(mctx module.Context, results map[string]*config.SubdomainResult) {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
_ = enc.Encode(map[string]interface{}{
|
||||
"target": mctx.Target,
|
||||
"subdomains": results,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
// Package reversedns expands discovery by doing PTR sweeps on /24 blocks
|
||||
// surrounding every resolved IP. Finds internal/forgotten hosts that share
|
||||
// infrastructure with already-known subdomains.
|
||||
//
|
||||
// Intentionally conservative: only sweeps +/- 32 addresses around seen IPs
|
||||
// to keep traffic bounded and avoid accidentally pulling a huge
|
||||
// non-scoped ASN.
|
||||
package reversedns
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/config"
|
||||
godns "god-eye/internal/dns"
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "discovery.reverse-dns"
|
||||
|
||||
type rdnsModule struct{}
|
||||
|
||||
func Register() { module.Register(&rdnsModule{}) }
|
||||
|
||||
func (*rdnsModule) Name() string { return ModuleName }
|
||||
func (*rdnsModule) Phase() module.Phase { return module.PhaseResolution }
|
||||
func (*rdnsModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventDNSResolved} }
|
||||
func (*rdnsModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventSubdomainDiscovered}
|
||||
}
|
||||
|
||||
// Opt-in: generates a lot of DNS queries; on by default for bugbounty profile.
|
||||
func (*rdnsModule) DefaultEnabled() bool { return false }
|
||||
|
||||
const sweepRange = 16 // how many addresses to scan either side of each seed IP
|
||||
|
||||
func (*rdnsModule) Run(mctx module.Context) error {
|
||||
if !mctx.Config.Bool("reverse_dns", false) {
|
||||
return nil
|
||||
}
|
||||
target := mctx.Target
|
||||
timeout := mctx.Config.Int("timeout", 5)
|
||||
resolvers := parseResolvers(mctx.Config.String("resolvers", ""))
|
||||
|
||||
seeds := mctx.Store.All(mctx.Ctx)
|
||||
seenIP := make(map[string]struct{})
|
||||
for _, h := range seeds {
|
||||
for _, ip := range h.IPs {
|
||||
seenIP[ip] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
sem := make(chan struct{}, 64)
|
||||
for ip := range seenIP {
|
||||
for _, neighbor := range neighbors(ip, sweepRange) {
|
||||
if mctx.Ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func(ipAddr string) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
|
||||
name := godns.ResolvePTR(ipAddr, resolvers, timeout)
|
||||
if name == "" {
|
||||
return
|
||||
}
|
||||
name = strings.ToLower(strings.TrimSuffix(name, "."))
|
||||
if !strings.HasSuffix(name, "."+target) && name != target {
|
||||
return
|
||||
}
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, name, func(h *store.Host) {
|
||||
store.AddIPs(h, []string{ipAddr})
|
||||
store.AddDiscoveryMethod(h, "reverse-dns")
|
||||
})
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.SubdomainDiscovered{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: name},
|
||||
Subdomain: name,
|
||||
Method: "reverse-dns",
|
||||
})
|
||||
}(neighbor)
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
// neighbors returns IPv4 addresses within +/- rng of ip. IPv6 addresses
|
||||
// are returned as a single-element slice (no sweep — address space too
|
||||
// large, and we'd rarely find anything anyway).
|
||||
func neighbors(ipStr string, rng int) []string {
|
||||
ip := net.ParseIP(ipStr)
|
||||
if ip == nil {
|
||||
return nil
|
||||
}
|
||||
v4 := ip.To4()
|
||||
if v4 == nil {
|
||||
return []string{ipStr}
|
||||
}
|
||||
|
||||
// Convert to uint32 for arithmetic.
|
||||
base := uint32(v4[0])<<24 | uint32(v4[1])<<16 | uint32(v4[2])<<8 | uint32(v4[3])
|
||||
|
||||
out := make([]string, 0, 2*rng+1)
|
||||
for delta := -rng; delta <= rng; delta++ {
|
||||
candidate := int64(base) + int64(delta)
|
||||
if candidate < 0 || candidate > 0xFFFFFFFF {
|
||||
continue
|
||||
}
|
||||
c := uint32(candidate)
|
||||
out = append(out, fmt.Sprintf("%d.%d.%d.%d", c>>24&0xFF, c>>16&0xFF, c>>8&0xFF, c&0xFF))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseResolvers(s string) []string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return config.DefaultResolvers
|
||||
}
|
||||
var out []string
|
||||
for _, r := range strings.Split(s, ",") {
|
||||
r = strings.TrimSpace(r)
|
||||
if r == "" {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(r, ":") {
|
||||
r = r + ":53"
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return config.DefaultResolvers
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
// Package security runs the v1 security checks (open redirect, CORS,
|
||||
// HTTP methods, git/svn, backups, admin, API) on every probed host.
|
||||
//
|
||||
// Reads hosts from the store (not events) so late-start phases don't miss
|
||||
// the upstream HTTPProbed events.
|
||||
package security
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
gohttp "god-eye/internal/http"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/security"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "security.checks"
|
||||
|
||||
type secModule struct{}
|
||||
|
||||
func Register() { module.Register(&secModule{}) }
|
||||
|
||||
func (*secModule) Name() string { return ModuleName }
|
||||
func (*secModule) Phase() module.Phase { return module.PhaseAnalysis }
|
||||
func (*secModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventHTTPProbed} }
|
||||
func (*secModule) Produces() []eventbus.EventType { return []eventbus.EventType{eventbus.EventVulnerability} }
|
||||
func (*secModule) DefaultEnabled() bool { return true }
|
||||
|
||||
func (*secModule) Run(mctx module.Context) error {
|
||||
conc := mctx.Config.Int("concurrency", 200)
|
||||
if conc <= 0 {
|
||||
conc = 200
|
||||
}
|
||||
timeout := mctx.Config.Int("timeout", 5)
|
||||
|
||||
processed := make(map[string]struct{})
|
||||
var processedMu sync.Mutex
|
||||
shouldProcess := func(host string) bool {
|
||||
processedMu.Lock()
|
||||
defer processedMu.Unlock()
|
||||
if _, dup := processed[host]; dup {
|
||||
return false
|
||||
}
|
||||
processed[host] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
work := make(chan string, conc*2)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < conc; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for host := range work {
|
||||
runChecks(mctx, host, timeout)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Drain: every host that got a successful HTTP probe.
|
||||
for _, h := range mctx.Store.All(mctx.Ctx) {
|
||||
if h == nil || h.StatusCode == 0 {
|
||||
continue
|
||||
}
|
||||
if !shouldProcess(h.Subdomain) {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case work <- h.Subdomain:
|
||||
case <-mctx.Ctx.Done():
|
||||
close(work)
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Listen for late HTTPProbed events.
|
||||
sub := mctx.Bus.Subscribe(eventbus.EventHTTPProbed, func(_ context.Context, e eventbus.Event) {
|
||||
ev, ok := e.(eventbus.HTTPProbed)
|
||||
if !ok || ev.StatusCode == 0 {
|
||||
return
|
||||
}
|
||||
host := ev.Meta().Target
|
||||
if !shouldProcess(host) {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case work <- host:
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
})
|
||||
defer sub.Unsubscribe()
|
||||
|
||||
select {
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
|
||||
close(work)
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func runChecks(mctx module.Context, host string, timeout int) {
|
||||
if mctx.Ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
client := gohttp.GetSharedClient(timeout)
|
||||
|
||||
var openRedirect bool
|
||||
var cors string
|
||||
var allowed, dangerous []string
|
||||
var admin, backups, apis []string
|
||||
var gitExposed, svnExposed bool
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(7)
|
||||
go func() { defer wg.Done(); openRedirect = security.CheckOpenRedirectWithClient(host, client) }()
|
||||
go func() { defer wg.Done(); cors = security.CheckCORSWithClient(host, client) }()
|
||||
go func() { defer wg.Done(); allowed, dangerous = security.CheckHTTPMethodsWithClient(host, client) }()
|
||||
go func() { defer wg.Done(); admin = security.CheckAdminPanelsWithClient(host, client) }()
|
||||
go func() { defer wg.Done(); gitExposed, svnExposed = security.CheckGitSvnExposureWithClient(host, client) }()
|
||||
go func() { defer wg.Done(); backups = security.CheckBackupFilesWithClient(host, client) }()
|
||||
go func() { defer wg.Done(); apis = security.CheckAPIEndpointsWithClient(host, client) }()
|
||||
wg.Wait()
|
||||
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, host, func(h *store.Host) {
|
||||
now := time.Now()
|
||||
if openRedirect {
|
||||
h.Vulnerabilities = append(h.Vulnerabilities, store.Vulnerability{
|
||||
ID: "open-redirect", Title: "Open Redirect",
|
||||
Description: "Server redirects to attacker-controlled URL via redirect parameter",
|
||||
Severity: string(eventbus.SeverityMedium),
|
||||
URL: "https://" + host,
|
||||
OWASP: "A01:2021-Broken Access Control",
|
||||
FoundAt: now,
|
||||
})
|
||||
}
|
||||
if cors != "" {
|
||||
h.Vulnerabilities = append(h.Vulnerabilities, store.Vulnerability{
|
||||
ID: "cors-misconfig", Title: "CORS Misconfiguration",
|
||||
Description: cors,
|
||||
Severity: string(eventbus.SeverityHigh),
|
||||
URL: "https://" + host,
|
||||
OWASP: "A05:2021-Security Misconfiguration",
|
||||
FoundAt: now,
|
||||
})
|
||||
}
|
||||
if len(dangerous) > 0 {
|
||||
h.Vulnerabilities = append(h.Vulnerabilities, store.Vulnerability{
|
||||
ID: "dangerous-http-methods", Title: "Dangerous HTTP Methods Enabled",
|
||||
Description: "Server allows potentially dangerous methods",
|
||||
Severity: string(eventbus.SeverityMedium),
|
||||
Evidence: joinStrings(dangerous, ", "),
|
||||
URL: "https://" + host,
|
||||
OWASP: "A05:2021-Security Misconfiguration",
|
||||
FoundAt: now,
|
||||
})
|
||||
}
|
||||
if gitExposed {
|
||||
h.Vulnerabilities = append(h.Vulnerabilities, store.Vulnerability{
|
||||
ID: "git-exposed", Title: "Git Repository Exposed",
|
||||
Description: ".git directory is publicly accessible",
|
||||
Severity: string(eventbus.SeverityCritical),
|
||||
URL: "https://" + host + "/.git/config",
|
||||
OWASP: "A05:2021-Security Misconfiguration",
|
||||
FoundAt: now,
|
||||
})
|
||||
}
|
||||
if svnExposed {
|
||||
h.Vulnerabilities = append(h.Vulnerabilities, store.Vulnerability{
|
||||
ID: "svn-exposed", Title: "SVN Repository Exposed",
|
||||
Description: ".svn directory is publicly accessible",
|
||||
Severity: string(eventbus.SeverityHigh),
|
||||
URL: "https://" + host + "/.svn/entries",
|
||||
OWASP: "A05:2021-Security Misconfiguration",
|
||||
FoundAt: now,
|
||||
})
|
||||
}
|
||||
for _, b := range backups {
|
||||
h.Vulnerabilities = append(h.Vulnerabilities, store.Vulnerability{
|
||||
ID: "backup-file", Title: "Backup File Exposed",
|
||||
Description: "Backup file accessible: " + b,
|
||||
Severity: string(eventbus.SeverityHigh),
|
||||
URL: b,
|
||||
OWASP: "A05:2021-Security Misconfiguration",
|
||||
FoundAt: now,
|
||||
})
|
||||
}
|
||||
_ = allowed
|
||||
_ = admin
|
||||
_ = apis
|
||||
})
|
||||
|
||||
now := time.Now()
|
||||
base := eventbus.EventMeta{At: now, Source: ModuleName, Target: host}
|
||||
emit := func(ev eventbus.VulnerabilityFound) { mctx.Bus.Publish(mctx.Ctx, ev) }
|
||||
|
||||
if openRedirect {
|
||||
emit(eventbus.VulnerabilityFound{EventMeta: base, ID: "open-redirect", Title: "Open Redirect",
|
||||
Severity: eventbus.SeverityMedium, URL: "https://" + host, OWASP: "A01:2021-Broken Access Control"})
|
||||
}
|
||||
if cors != "" {
|
||||
emit(eventbus.VulnerabilityFound{EventMeta: base, ID: "cors-misconfig", Title: "CORS Misconfiguration",
|
||||
Description: cors, Severity: eventbus.SeverityHigh, URL: "https://" + host, OWASP: "A05:2021-Security Misconfiguration"})
|
||||
}
|
||||
if len(dangerous) > 0 {
|
||||
emit(eventbus.VulnerabilityFound{EventMeta: base, ID: "dangerous-http-methods", Title: "Dangerous HTTP Methods",
|
||||
Evidence: joinStrings(dangerous, ", "), Severity: eventbus.SeverityMedium, URL: "https://" + host,
|
||||
OWASP: "A05:2021-Security Misconfiguration"})
|
||||
}
|
||||
if gitExposed {
|
||||
emit(eventbus.VulnerabilityFound{EventMeta: base, ID: "git-exposed", Title: "Git Repository Exposed",
|
||||
Severity: eventbus.SeverityCritical, URL: "https://" + host + "/.git/config",
|
||||
OWASP: "A05:2021-Security Misconfiguration"})
|
||||
}
|
||||
if svnExposed {
|
||||
emit(eventbus.VulnerabilityFound{EventMeta: base, ID: "svn-exposed", Title: "SVN Repository Exposed",
|
||||
Severity: eventbus.SeverityHigh, URL: "https://" + host + "/.svn/entries",
|
||||
OWASP: "A05:2021-Security Misconfiguration"})
|
||||
}
|
||||
for _, b := range backups {
|
||||
emit(eventbus.VulnerabilityFound{EventMeta: base, ID: "backup-file", Title: "Backup File Exposed",
|
||||
Severity: eventbus.SeverityHigh, URL: b, OWASP: "A05:2021-Security Misconfiguration"})
|
||||
}
|
||||
}
|
||||
|
||||
func joinStrings(ss []string, sep string) string {
|
||||
if len(ss) == 0 {
|
||||
return ""
|
||||
}
|
||||
out := ss[0]
|
||||
for _, s := range ss[1:] {
|
||||
out += sep + s
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
// Package smuggling detects HTTP request smuggling (CL.TE and TE.CL
|
||||
// variants) by sending ambiguous Content-Length / Transfer-Encoding
|
||||
// combinations and timing-analyzing the responses.
|
||||
//
|
||||
// This is the non-destructive timing variant: we send a request crafted
|
||||
// so that CL.TE or TE.CL parsing desync would cause the server to hold
|
||||
// the connection waiting for more bytes, while the correct interpretation
|
||||
// returns immediately. Large response time delta ⇒ likely smuggling.
|
||||
//
|
||||
// We do NOT attempt to actually smuggle follow-up requests — that could
|
||||
// affect other users. This is safe for authorized testing.
|
||||
package smuggling
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "vuln.http-smuggling"
|
||||
|
||||
type smModule struct{}
|
||||
|
||||
func Register() { module.Register(&smModule{}) }
|
||||
|
||||
func (*smModule) Name() string { return ModuleName }
|
||||
func (*smModule) Phase() module.Phase { return module.PhaseAnalysis }
|
||||
func (*smModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventHTTPProbed} }
|
||||
func (*smModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventVulnerability}
|
||||
}
|
||||
|
||||
// Opt-in: timing-based testing is slower and can be noisy. Bugbounty profile enables it.
|
||||
func (*smModule) DefaultEnabled() bool { return false }
|
||||
|
||||
func (*smModule) Run(mctx module.Context) error {
|
||||
if !mctx.Config.Bool("smuggling_scan", false) {
|
||||
return nil
|
||||
}
|
||||
timeout := mctx.Config.Int("timeout", 10)
|
||||
|
||||
processed := make(map[string]struct{})
|
||||
var mu sync.Mutex
|
||||
shouldProcess := func(host string) bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if _, ok := processed[host]; ok {
|
||||
return false
|
||||
}
|
||||
processed[host] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for _, h := range mctx.Store.All(mctx.Ctx) {
|
||||
if h == nil || h.StatusCode == 0 {
|
||||
continue
|
||||
}
|
||||
if !shouldProcess(h.Subdomain) {
|
||||
continue
|
||||
}
|
||||
host := h.Subdomain
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); probe(mctx, host, timeout) }()
|
||||
}
|
||||
|
||||
sub := mctx.Bus.Subscribe(eventbus.EventHTTPProbed, func(_ context.Context, e eventbus.Event) {
|
||||
ev, ok := e.(eventbus.HTTPProbed)
|
||||
if !ok || ev.StatusCode == 0 {
|
||||
return
|
||||
}
|
||||
host := ev.Meta().Target
|
||||
if !shouldProcess(host) {
|
||||
return
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); probe(mctx, host, timeout) }()
|
||||
})
|
||||
defer sub.Unsubscribe()
|
||||
|
||||
select {
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func probe(mctx module.Context, host string, timeoutSec int) {
|
||||
timeout := time.Duration(timeoutSec) * time.Second
|
||||
|
||||
// Baseline: normal request, measure response time.
|
||||
baseline, err := sendRequest(host, baselineRequest(host), timeout)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// CL.TE probe: Content-Length says more data coming, TE: chunked says "last chunk now".
|
||||
// Vulnerable servers that read TE first return quickly; non-vulnerable
|
||||
// servers that read CL wait for more bytes and hit the read timeout.
|
||||
cltePayload := clteRequest(host)
|
||||
clte, _ := sendRequest(host, cltePayload, timeout)
|
||||
|
||||
// TE.CL probe: reversed — server reads CL first (ignoring chunked), payload is poisoned.
|
||||
teclPayload := teclRequest(host)
|
||||
tecl, _ := sendRequest(host, teclPayload, timeout)
|
||||
|
||||
// Heuristic: if either probe hangs (duration >= timeout * 0.8) and baseline
|
||||
// returned fast, it's a likely desync.
|
||||
threshold := time.Duration(float64(timeout) * 0.8)
|
||||
fastEnough := baseline.duration < timeout/3
|
||||
|
||||
if fastEnough && clte.duration > threshold {
|
||||
emit(mctx, host, "CL.TE", "CL.TE HTTP Request Smuggling candidate", clte)
|
||||
}
|
||||
if fastEnough && tecl.duration > threshold {
|
||||
emit(mctx, host, "TE.CL", "TE.CL HTTP Request Smuggling candidate", tecl)
|
||||
}
|
||||
}
|
||||
|
||||
type probeResult struct {
|
||||
duration time.Duration
|
||||
response string
|
||||
}
|
||||
|
||||
func baselineRequest(host string) string {
|
||||
return "GET / HTTP/1.1\r\n" +
|
||||
"Host: " + host + "\r\n" +
|
||||
"User-Agent: god-eye-v2\r\n" +
|
||||
"Connection: close\r\n" +
|
||||
"\r\n"
|
||||
}
|
||||
|
||||
// clteRequest crafts a CL.TE probe: the chunked body declares "0\r\n\r\n"
|
||||
// which is the last chunk. If the server honors TE: chunked, the request
|
||||
// completes immediately. If it honors Content-Length (say, 4), it waits for
|
||||
// 4 more bytes.
|
||||
func clteRequest(host string) string {
|
||||
body := "0\r\n\r\n"
|
||||
return fmt.Sprintf("POST / HTTP/1.1\r\n"+
|
||||
"Host: %s\r\n"+
|
||||
"User-Agent: god-eye-v2\r\n"+
|
||||
"Content-Length: %d\r\n"+
|
||||
"Transfer-Encoding: chunked\r\n"+
|
||||
"Connection: close\r\n"+
|
||||
"\r\n%s", host, 4, body) // CL=4 mismatches chunked body length
|
||||
}
|
||||
|
||||
// teclRequest: TE: chunked, body ends with a chunk that declares non-zero
|
||||
// remaining — CL says "done", TE says "more coming". Opposite desync.
|
||||
func teclRequest(host string) string {
|
||||
body := "12\r\n" +
|
||||
"GPOST / HTTP/1.1\r\n" +
|
||||
"\r\n0\r\n\r\n"
|
||||
return fmt.Sprintf("POST / HTTP/1.1\r\n"+
|
||||
"Host: %s\r\n"+
|
||||
"User-Agent: god-eye-v2\r\n"+
|
||||
"Content-Length: 3\r\n"+
|
||||
"Transfer-Encoding: chunked\r\n"+
|
||||
"Connection: close\r\n"+
|
||||
"\r\n%s", host, body)
|
||||
}
|
||||
|
||||
// sendRequest opens a raw TCP/TLS connection, writes raw HTTP bytes, and
|
||||
// returns the time until the first response line is read (or timeout).
|
||||
func sendRequest(host, payload string, timeout time.Duration) (probeResult, error) {
|
||||
dialer := &net.Dialer{Timeout: timeout}
|
||||
conn, err := tls.DialWithDialer(dialer, "tcp", host+":443", &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
ServerName: host,
|
||||
})
|
||||
if err != nil {
|
||||
return probeResult{}, err
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
_ = conn.SetDeadline(time.Now().Add(timeout))
|
||||
|
||||
start := time.Now()
|
||||
if _, err := conn.Write([]byte(payload)); err != nil {
|
||||
return probeResult{duration: time.Since(start)}, err
|
||||
}
|
||||
|
||||
br := bufio.NewReader(conn)
|
||||
line, err := br.ReadString('\n')
|
||||
return probeResult{duration: time.Since(start), response: line}, err
|
||||
}
|
||||
|
||||
func emit(mctx module.Context, host, kind, title string, r probeResult) {
|
||||
now := time.Now()
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, host, func(h *store.Host) {
|
||||
h.Vulnerabilities = append(h.Vulnerabilities, store.Vulnerability{
|
||||
ID: "http-smuggling-" + strings.ToLower(kind),
|
||||
Title: title,
|
||||
Description: kind + " desync candidate based on response-time delta (" + r.duration.String() + ").",
|
||||
Severity: string(eventbus.SeverityHigh),
|
||||
URL: "https://" + host,
|
||||
Evidence: strings.TrimSpace(r.response),
|
||||
Remediation: "Ensure front-end and back-end parse Content-Length and Transfer-Encoding identically. Reject requests with both headers.",
|
||||
OWASP: "A06:2021-Vulnerable and Outdated Components",
|
||||
FoundAt: now,
|
||||
})
|
||||
})
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.VulnerabilityFound{
|
||||
EventMeta: eventbus.EventMeta{At: now, Source: ModuleName, Target: host},
|
||||
ID: "http-smuggling-" + strings.ToLower(kind),
|
||||
Title: title,
|
||||
Description: "Timing-based " + kind + " desync candidate.",
|
||||
Severity: eventbus.SeverityHigh,
|
||||
URL: "https://" + host,
|
||||
Evidence: strings.TrimSpace(r.response),
|
||||
Remediation: "Align CL/TE parsing between front-end and back-end.",
|
||||
OWASP: "A06:2021-Vulnerable and Outdated Components",
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
// Package supplychain enumerates npm and PyPI packages that reference the
|
||||
// target domain in their source, then flags packages as potential supply
|
||||
// chain assets. Useful for discovering internal-only tools published by
|
||||
// mistake to public registries and for finding branded utility packages
|
||||
// that could reveal internal endpoints/secrets.
|
||||
//
|
||||
// This is a discovery-oriented check. Actually downloading + scanning
|
||||
// package contents for secrets is a Fase 2 follow-up; here we just surface
|
||||
// the packages and the URLs they point at.
|
||||
package supplychain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/sources"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "vuln.supply-chain"
|
||||
|
||||
type scModule struct{}
|
||||
|
||||
func Register() { module.Register(&scModule{}) }
|
||||
|
||||
func (*scModule) Name() string { return ModuleName }
|
||||
func (*scModule) Phase() module.Phase { return module.PhaseDiscovery }
|
||||
func (*scModule) Consumes() []eventbus.EventType { return nil }
|
||||
func (*scModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventSubdomainDiscovered, eventbus.EventAPIFinding}
|
||||
}
|
||||
func (*scModule) DefaultEnabled() bool { return true }
|
||||
|
||||
func (*scModule) Run(mctx module.Context) error {
|
||||
target := mctx.Target
|
||||
if target == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); checkNPM(mctx, target) }()
|
||||
go func() { defer wg.Done(); checkPyPI(mctx, target) }()
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkNPM uses npm's registry search API. Packages matching "<target>"
|
||||
// or "<target-suffix>" are surfaced.
|
||||
func checkNPM(mctx module.Context, target string) {
|
||||
q := extractBrand(target)
|
||||
if q == "" {
|
||||
return
|
||||
}
|
||||
url := fmt.Sprintf("https://registry.npmjs.org/-/v1/search?text=%s&size=100", q)
|
||||
body, err := fetchJSON(mctx.Ctx, url, 15*time.Second)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var parsed struct {
|
||||
Objects []struct {
|
||||
Package struct {
|
||||
Name string `json:"name"`
|
||||
Links map[string]string `json:"links"`
|
||||
Description string `json:"description"`
|
||||
} `json:"package"`
|
||||
} `json:"objects"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &parsed)
|
||||
|
||||
for _, obj := range parsed.Objects {
|
||||
pkg := obj.Package
|
||||
text := pkg.Name + " " + pkg.Description
|
||||
for _, link := range pkg.Links {
|
||||
text += " " + link
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(text), target) {
|
||||
continue
|
||||
}
|
||||
// Emit an APIFinding for discovery context.
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.APIFinding{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: target},
|
||||
Kind: "supply-chain:npm",
|
||||
URL: "https://www.npmjs.com/package/" + pkg.Name,
|
||||
Issue: "npm package references target: " + pkg.Name + " — " + pkg.Description,
|
||||
Severity: eventbus.SeverityInfo,
|
||||
})
|
||||
// If the description or links contain subdomains of the target,
|
||||
// also feed them into discovery.
|
||||
for _, sub := range sources.ExtractSubdomains(text, target) {
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, sub, func(h *store.Host) {
|
||||
store.AddDiscoveryMethod(h, "supply-chain:npm:"+pkg.Name)
|
||||
})
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.SubdomainDiscovered{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: sub},
|
||||
Subdomain: sub,
|
||||
Method: "supply-chain:npm:" + pkg.Name,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func checkPyPI(mctx module.Context, target string) {
|
||||
// PyPI no longer supports XML-RPC search; use the simple index
|
||||
// (all packages) scanning is too expensive. Instead query a few
|
||||
// likely branded package prefixes via the JSON index.
|
||||
q := extractBrand(target)
|
||||
if q == "" {
|
||||
return
|
||||
}
|
||||
// Try exact-name lookups for common variants.
|
||||
candidates := []string{q, q + "-cli", q + "-sdk", q + "-api", q + "-client"}
|
||||
for _, name := range candidates {
|
||||
url := "https://pypi.org/pypi/" + name + "/json"
|
||||
body, err := fetchJSON(mctx.Ctx, url, 10*time.Second)
|
||||
if err != nil || len(body) < 50 {
|
||||
continue
|
||||
}
|
||||
var parsed struct {
|
||||
Info struct {
|
||||
Name string `json:"name"`
|
||||
Summary string `json:"summary"`
|
||||
HomePage string `json:"home_page"`
|
||||
ProjectURL string `json:"project_url"`
|
||||
ProjectURLs map[string]string `json:"project_urls"`
|
||||
} `json:"info"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &parsed)
|
||||
info := parsed.Info
|
||||
if info.Name == "" {
|
||||
continue
|
||||
}
|
||||
text := info.Name + " " + info.Summary + " " + info.HomePage + " " + info.ProjectURL
|
||||
for _, u := range info.ProjectURLs {
|
||||
text += " " + u
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(text), target) {
|
||||
continue
|
||||
}
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.APIFinding{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: target},
|
||||
Kind: "supply-chain:pypi",
|
||||
URL: "https://pypi.org/project/" + info.Name + "/",
|
||||
Issue: "PyPI package references target: " + info.Name + " — " + info.Summary,
|
||||
Severity: eventbus.SeverityInfo,
|
||||
})
|
||||
for _, sub := range sources.ExtractSubdomains(text, target) {
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, sub, func(h *store.Host) {
|
||||
store.AddDiscoveryMethod(h, "supply-chain:pypi:"+info.Name)
|
||||
})
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.SubdomainDiscovered{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: sub},
|
||||
Subdomain: sub,
|
||||
Method: "supply-chain:pypi:" + info.Name,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// extractBrand returns the "brand" (second-to-last label) from example.com →
|
||||
// "example". Used as the package-search query term.
|
||||
func extractBrand(domain string) string {
|
||||
labels := strings.Split(strings.TrimSuffix(domain, "."), ".")
|
||||
if len(labels) < 2 {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(labels[len(labels)-2])
|
||||
}
|
||||
|
||||
func fetchJSON(ctx context.Context, url string, timeout time.Duration) ([]byte, error) {
|
||||
c := &http.Client{Timeout: timeout}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "god-eye-v2")
|
||||
resp, err := c.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
return io.ReadAll(io.LimitReader(resp.Body, 4*1024*1024))
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
// Package takeover runs v1 takeover detection on every host with a CNAME.
|
||||
// Reads from the store; listens for late DNSResolved events for concurrent
|
||||
// modules.
|
||||
package takeover
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/scanner"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "takeover.cname"
|
||||
|
||||
type takeoverModule struct{}
|
||||
|
||||
func Register() { module.Register(&takeoverModule{}) }
|
||||
|
||||
func (*takeoverModule) Name() string { return ModuleName }
|
||||
func (*takeoverModule) Phase() module.Phase { return module.PhaseAnalysis }
|
||||
func (*takeoverModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventDNSResolved} }
|
||||
func (*takeoverModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventTakeoverCandidate}
|
||||
}
|
||||
func (*takeoverModule) DefaultEnabled() bool { return true }
|
||||
|
||||
func (*takeoverModule) Run(mctx module.Context) error {
|
||||
if mctx.Config.Bool("no_takeover", false) {
|
||||
return nil
|
||||
}
|
||||
conc := mctx.Config.Int("concurrency", 100)
|
||||
if conc <= 0 {
|
||||
conc = 100
|
||||
}
|
||||
timeout := mctx.Config.Int("timeout", 5)
|
||||
|
||||
processed := make(map[string]struct{})
|
||||
var processedMu sync.Mutex
|
||||
shouldProcess := func(host string) bool {
|
||||
processedMu.Lock()
|
||||
defer processedMu.Unlock()
|
||||
if _, dup := processed[host]; dup {
|
||||
return false
|
||||
}
|
||||
processed[host] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
work := make(chan string, conc*2)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < conc; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for host := range work {
|
||||
if mctx.Ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
service := scanner.CheckTakeover(host, timeout)
|
||||
if service == "" {
|
||||
continue
|
||||
}
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, host, func(h *store.Host) {
|
||||
h.Takeover = &store.Takeover{
|
||||
Service: service,
|
||||
CNAME: h.CNAME,
|
||||
Confirmed: false,
|
||||
FoundAt: time.Now(),
|
||||
}
|
||||
})
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.TakeoverCandidate{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: host},
|
||||
Subdomain: host,
|
||||
Service: service,
|
||||
})
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Drain: every host with a CNAME is a takeover candidate.
|
||||
for _, h := range mctx.Store.All(mctx.Ctx) {
|
||||
if h == nil || h.CNAME == "" {
|
||||
continue
|
||||
}
|
||||
if !shouldProcess(h.Subdomain) {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case work <- h.Subdomain:
|
||||
case <-mctx.Ctx.Done():
|
||||
close(work)
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
sub := mctx.Bus.Subscribe(eventbus.EventDNSResolved, func(_ context.Context, e eventbus.Event) {
|
||||
ev, ok := e.(eventbus.DNSResolved)
|
||||
if !ok || ev.CNAME == "" {
|
||||
return
|
||||
}
|
||||
if !shouldProcess(ev.Subdomain) {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case work <- ev.Subdomain:
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
})
|
||||
defer sub.Unsubscribe()
|
||||
|
||||
select {
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
case <-mctx.Ctx.Done():
|
||||
}
|
||||
|
||||
close(work)
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
// Package vhost is a Fase 0.6 adapter around v1 network.VHostScanner which
|
||||
// performs virtual host discovery on resolved IPs. Reveals additional
|
||||
// hostnames sharing infrastructure with in-scope targets.
|
||||
package vhost
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/network"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
const ModuleName = "discovery.vhost"
|
||||
|
||||
type vhostModule struct{}
|
||||
|
||||
func Register() { module.Register(&vhostModule{}) }
|
||||
|
||||
func (*vhostModule) Name() string { return ModuleName }
|
||||
func (*vhostModule) Phase() module.Phase { return module.PhaseResolution }
|
||||
func (*vhostModule) Consumes() []eventbus.EventType { return []eventbus.EventType{eventbus.EventDNSResolved} }
|
||||
func (*vhostModule) Produces() []eventbus.EventType {
|
||||
return []eventbus.EventType{eventbus.EventSubdomainDiscovered}
|
||||
}
|
||||
func (*vhostModule) DefaultEnabled() bool { return false } // opt-in
|
||||
|
||||
func (*vhostModule) Run(mctx module.Context) error {
|
||||
if !mctx.Config.Bool("vhost_scan", false) {
|
||||
return nil
|
||||
}
|
||||
timeout := mctx.Config.Int("timeout", 10)
|
||||
target := mctx.Target
|
||||
|
||||
hosts := mctx.Store.All(mctx.Ctx)
|
||||
seenIP := make(map[string]struct{})
|
||||
for _, h := range hosts {
|
||||
for _, ip := range h.IPs {
|
||||
seenIP[ip] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
scanner := network.NewVHostScanner(timeout)
|
||||
var wg sync.WaitGroup
|
||||
for ip := range seenIP {
|
||||
ip := ip
|
||||
if mctx.Ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
res := scanner.DiscoverVHosts(mctx.Ctx, ip)
|
||||
if res == nil {
|
||||
return
|
||||
}
|
||||
for _, h := range res.Domains {
|
||||
h = strings.ToLower(strings.TrimSpace(h))
|
||||
if h == "" || !strings.HasSuffix(h, target) {
|
||||
continue
|
||||
}
|
||||
_ = mctx.Store.Upsert(mctx.Ctx, h, func(sh *store.Host) {
|
||||
store.AddIPs(sh, []string{ip})
|
||||
store.AddDiscoveryMethod(sh, "vhost")
|
||||
})
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.SubdomainDiscovered{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: ModuleName, Target: h},
|
||||
Subdomain: h,
|
||||
Method: "vhost",
|
||||
})
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,370 @@
|
||||
package nucleitpl
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TemplatesZipURL is the default ZIP archive of the projectdiscovery
|
||||
// nuclei-templates repository (main branch).
|
||||
const TemplatesZipURL = "https://github.com/projectdiscovery/nuclei-templates/archive/refs/heads/main.zip"
|
||||
|
||||
// Downloader fetches the nuclei-templates archive and extracts the
|
||||
// YAML files into destDir. Designed to be invoked at most once per
|
||||
// scan: after a successful extraction the destination dir persists
|
||||
// across runs; subsequent invocations return quickly via hasTemplates().
|
||||
type Downloader struct {
|
||||
// ZipURL overrides TemplatesZipURL for testing or mirroring.
|
||||
ZipURL string
|
||||
// HTTPClient is used for the download. Default: 10-minute timeout.
|
||||
HTTPClient *http.Client
|
||||
// Writer receives progress lines when Verbose is true. Defaults to
|
||||
// os.Stderr.
|
||||
Writer io.Writer
|
||||
// Verbose toggles progress logging.
|
||||
Verbose bool
|
||||
// MinTemplatesToConsiderPresent is the count of .yaml files under
|
||||
// destDir below which we treat the directory as empty / incomplete
|
||||
// and re-download. Default: 50.
|
||||
MinTemplatesToConsiderPresent int
|
||||
}
|
||||
|
||||
// NewDownloader returns a Downloader with sensible defaults.
|
||||
func NewDownloader() *Downloader {
|
||||
return &Downloader{
|
||||
ZipURL: TemplatesZipURL,
|
||||
HTTPClient: &http.Client{Timeout: 10 * time.Minute},
|
||||
Writer: os.Stderr,
|
||||
MinTemplatesToConsiderPresent: 50,
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureTemplates guarantees destDir contains a usable set of Nuclei
|
||||
// YAML templates. If the directory already has ≥ MinTemplatesToConsiderPresent
|
||||
// templates, it's a no-op. Otherwise the ZIP is downloaded, streamed to
|
||||
// a temp file, and extracted (YAML files only).
|
||||
//
|
||||
// destDir is created if it doesn't exist.
|
||||
func (d *Downloader) EnsureTemplates(destDir string) error {
|
||||
if destDir == "" {
|
||||
return errors.New("EnsureTemplates: empty destDir")
|
||||
}
|
||||
if d.hasEnoughTemplates(destDir) {
|
||||
if d.Verbose {
|
||||
fmt.Fprintf(d.writer(), "✓ nuclei templates already present at %s\n", destDir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(destDir, 0o755); err != nil {
|
||||
return fmt.Errorf("mkdir %s: %w", destDir, err)
|
||||
}
|
||||
|
||||
if d.Verbose {
|
||||
fmt.Fprintf(d.writer(), "↓ downloading nuclei-templates from %s\n", d.zipURL())
|
||||
}
|
||||
|
||||
tmpPath, err := d.downloadZip()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tmpPath)
|
||||
|
||||
count, bytes, err := d.extractYAML(tmpPath, destDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if count < d.MinTemplatesToConsiderPresent {
|
||||
return fmt.Errorf("extracted only %d templates (expected ≥ %d) — archive may be incomplete", count, d.MinTemplatesToConsiderPresent)
|
||||
}
|
||||
|
||||
if d.Verbose {
|
||||
fmt.Fprintf(d.writer(), "✓ extracted %d nuclei templates (%s) into %s\n",
|
||||
count, humanBytesN(bytes), destDir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Refresh forces a re-download regardless of current directory contents.
|
||||
// Useful for `god-eye nuclei-update` style CLI commands.
|
||||
func (d *Downloader) Refresh(destDir string) error {
|
||||
if destDir == "" {
|
||||
return errors.New("Refresh: empty destDir")
|
||||
}
|
||||
if err := os.MkdirAll(destDir, 0o755); err != nil {
|
||||
return fmt.Errorf("mkdir %s: %w", destDir, err)
|
||||
}
|
||||
|
||||
if d.Verbose {
|
||||
fmt.Fprintf(d.writer(), "↓ refreshing nuclei-templates from %s\n", d.zipURL())
|
||||
}
|
||||
|
||||
tmpPath, err := d.downloadZip()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tmpPath)
|
||||
|
||||
count, bytes, err := d.extractYAML(tmpPath, destDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.Verbose {
|
||||
fmt.Fprintf(d.writer(), "✓ refreshed %d templates (%s)\n", count, humanBytesN(bytes))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- internals -----------------------------------------------------------
|
||||
|
||||
func (d *Downloader) hasEnoughTemplates(dir string) bool {
|
||||
info, err := os.Stat(dir)
|
||||
if err != nil || !info.IsDir() {
|
||||
return false
|
||||
}
|
||||
found := 0
|
||||
threshold := d.MinTemplatesToConsiderPresent
|
||||
if threshold <= 0 {
|
||||
threshold = 50
|
||||
}
|
||||
_ = filepath.Walk(dir, func(_ string, fi os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if fi.IsDir() {
|
||||
return nil
|
||||
}
|
||||
name := strings.ToLower(fi.Name())
|
||||
if strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml") {
|
||||
found++
|
||||
if found >= threshold {
|
||||
return filepath.SkipAll
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return found >= threshold
|
||||
}
|
||||
|
||||
func (d *Downloader) zipURL() string {
|
||||
if d.ZipURL != "" {
|
||||
return d.ZipURL
|
||||
}
|
||||
return TemplatesZipURL
|
||||
}
|
||||
|
||||
func (d *Downloader) writer() io.Writer {
|
||||
if d.Writer != nil {
|
||||
return d.Writer
|
||||
}
|
||||
return os.Stderr
|
||||
}
|
||||
|
||||
func (d *Downloader) downloadZip() (string, error) {
|
||||
client := d.HTTPClient
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 10 * time.Minute}
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("GET", d.zipURL(), nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("User-Agent", "god-eye-v2")
|
||||
req.Header.Set("Accept", "application/zip")
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("download: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Follow standard HTTP error reporting.
|
||||
if resp.StatusCode != 200 {
|
||||
return "", fmt.Errorf("download: HTTP %d from %s", resp.StatusCode, d.zipURL())
|
||||
}
|
||||
|
||||
tmp, err := os.CreateTemp("", "nuclei-templates-*.zip")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create temp: %w", err)
|
||||
}
|
||||
|
||||
// Streaming copy with throttled progress output.
|
||||
var written atomic.Int64
|
||||
pr := &progressReader{
|
||||
r: resp.Body,
|
||||
written: &written,
|
||||
verbose: d.Verbose,
|
||||
writer: d.writer(),
|
||||
total: resp.ContentLength,
|
||||
prefix: " downloading",
|
||||
}
|
||||
|
||||
if _, err := io.Copy(tmp, pr); err != nil {
|
||||
tmp.Close()
|
||||
os.Remove(tmp.Name())
|
||||
return "", fmt.Errorf("stream download: %w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
os.Remove(tmp.Name())
|
||||
return "", err
|
||||
}
|
||||
return tmp.Name(), nil
|
||||
}
|
||||
|
||||
// extractYAML walks the zip and writes every .yaml / .yml file into
|
||||
// destDir. Returns (count, totalBytes, error).
|
||||
//
|
||||
// The top-level directory in the archive (e.g. "nuclei-templates-main/")
|
||||
// is stripped so entries land at destDir/<category>/<file>.yaml.
|
||||
//
|
||||
// Path-traversal protection: every resolved destination must be within
|
||||
// destDir; otherwise the entry is skipped.
|
||||
func (d *Downloader) extractYAML(zipPath, destDir string) (int, int64, error) {
|
||||
zr, err := zip.OpenReader(zipPath)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("open zip: %w", err)
|
||||
}
|
||||
defer zr.Close()
|
||||
|
||||
absDest, err := filepath.Abs(destDir)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
var count int
|
||||
var bytes int64
|
||||
for _, f := range zr.File {
|
||||
if f.FileInfo().IsDir() {
|
||||
continue
|
||||
}
|
||||
lower := strings.ToLower(f.Name)
|
||||
if !strings.HasSuffix(lower, ".yaml") && !strings.HasSuffix(lower, ".yml") {
|
||||
continue
|
||||
}
|
||||
|
||||
// Strip leading top-level folder if present.
|
||||
rel := f.Name
|
||||
if i := strings.Index(rel, "/"); i >= 0 {
|
||||
rel = rel[i+1:]
|
||||
}
|
||||
if rel == "" {
|
||||
continue
|
||||
}
|
||||
// Guard against path traversal / absolute paths.
|
||||
if strings.Contains(rel, "..") || filepath.IsAbs(rel) {
|
||||
continue
|
||||
}
|
||||
|
||||
dest := filepath.Join(absDest, rel)
|
||||
if !strings.HasPrefix(dest, absDest+string(os.PathSeparator)) && dest != absDest {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
rc.Close()
|
||||
continue
|
||||
}
|
||||
n, cerr := io.Copy(out, rc)
|
||||
rc.Close()
|
||||
out.Close()
|
||||
if cerr != nil {
|
||||
_ = os.Remove(dest)
|
||||
continue
|
||||
}
|
||||
count++
|
||||
bytes += n
|
||||
}
|
||||
return count, bytes, nil
|
||||
}
|
||||
|
||||
// --- helpers -------------------------------------------------------------
|
||||
|
||||
// progressReader wraps an io.Reader and emits throttled progress lines
|
||||
// as bytes are consumed. Throttling: one line every ~5% of total (or
|
||||
// every ~5MB when total is unknown).
|
||||
type progressReader struct {
|
||||
r io.Reader
|
||||
written *atomic.Int64
|
||||
total int64
|
||||
verbose bool
|
||||
writer io.Writer
|
||||
prefix string
|
||||
|
||||
lastPct int
|
||||
lastBytes int64
|
||||
lastReport time.Time
|
||||
}
|
||||
|
||||
func (p *progressReader) Read(b []byte) (int, error) {
|
||||
n, err := p.r.Read(b)
|
||||
if n > 0 {
|
||||
p.written.Add(int64(n))
|
||||
if p.verbose {
|
||||
p.maybeReport()
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (p *progressReader) maybeReport() {
|
||||
w := p.written.Load()
|
||||
|
||||
// Rate-limit prints to avoid flooding the terminal.
|
||||
if time.Since(p.lastReport) < 200*time.Millisecond {
|
||||
return
|
||||
}
|
||||
|
||||
if p.total > 0 {
|
||||
pct := int(float64(w) / float64(p.total) * 100)
|
||||
if pct >= p.lastPct+5 || pct == 100 {
|
||||
fmt.Fprintf(p.writer, "%s %3d%% %s / %s\n",
|
||||
p.prefix, pct, humanBytesN(w), humanBytesN(p.total))
|
||||
p.lastPct = pct
|
||||
p.lastReport = time.Now()
|
||||
}
|
||||
} else {
|
||||
// Unknown total: report every ~5MB.
|
||||
if w-p.lastBytes >= 5*1024*1024 {
|
||||
fmt.Fprintf(p.writer, "%s %s\n", p.prefix, humanBytesN(w))
|
||||
p.lastBytes = w
|
||||
p.lastReport = time.Now()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// humanBytesN formats a byte count like "2.3MB". Duplicated from
|
||||
// ai/ensure.go to avoid a cross-package dependency.
|
||||
func humanBytesN(n int64) string {
|
||||
const k = 1024.0
|
||||
if n < int64(k) {
|
||||
return fmt.Sprintf("%dB", n)
|
||||
}
|
||||
units := []string{"KB", "MB", "GB", "TB"}
|
||||
v := float64(n) / k
|
||||
for _, u := range units {
|
||||
if v < k {
|
||||
return fmt.Sprintf("%.1f%s", v, u)
|
||||
}
|
||||
v /= k
|
||||
}
|
||||
return fmt.Sprintf("%.1fPB", v)
|
||||
}
|
||||
@@ -0,0 +1,361 @@
|
||||
package nucleitpl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Executor runs supported Nuclei templates against a target URL.
|
||||
type Executor struct {
|
||||
Client *http.Client
|
||||
Timeout time.Duration
|
||||
MaxBodyB int64 // response body cap; default 1MB
|
||||
UserAgent string
|
||||
}
|
||||
|
||||
// NewExecutor builds an executor with sensible defaults. Pass a custom
|
||||
// *http.Client when you want connection pooling shared with the rest of
|
||||
// the scan (recommended).
|
||||
func NewExecutor(client *http.Client, timeout time.Duration) *Executor {
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: timeout}
|
||||
}
|
||||
if timeout == 0 {
|
||||
timeout = 15 * time.Second
|
||||
}
|
||||
return &Executor{
|
||||
Client: client,
|
||||
Timeout: timeout,
|
||||
MaxBodyB: 1 * 1024 * 1024,
|
||||
UserAgent: "god-eye-v2-nuclei",
|
||||
}
|
||||
}
|
||||
|
||||
// Match holds the successful match output for a single template/target.
|
||||
type Match struct {
|
||||
TemplateID string
|
||||
TemplateURL string // reference URL when present in info.reference
|
||||
Name string
|
||||
Severity string
|
||||
Description string
|
||||
Tags []string
|
||||
URL string // URL that matched
|
||||
Evidence string // short excerpt from the matching response
|
||||
CVEs []string // extracted from info.reference when possible
|
||||
Author string
|
||||
}
|
||||
|
||||
// Run executes every HTTP request in the template against the given
|
||||
// base URL (e.g. "https://api.example.com"). Returns one Match per
|
||||
// request that succeeds. Non-matching requests produce no entries.
|
||||
//
|
||||
// Templating substitutions handled: {{BaseURL}}, {{Hostname}}, {{RootURL}}.
|
||||
func (e *Executor) Run(ctx context.Context, t *Template, baseURL string) []Match {
|
||||
if ok, _ := t.IsSupported(); !ok {
|
||||
return nil
|
||||
}
|
||||
var matches []Match
|
||||
for _, req := range t.Requests {
|
||||
for _, p := range req.Path {
|
||||
url := expandPath(p, baseURL)
|
||||
m, err := e.runOne(ctx, t, req, url)
|
||||
if err != nil || m == nil {
|
||||
continue
|
||||
}
|
||||
matches = append(matches, *m)
|
||||
}
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
// runOne sends one HTTP request, applies matchers, and returns a Match
|
||||
// when every matchers-condition group is satisfied.
|
||||
func (e *Executor) runOne(ctx context.Context, t *Template, req HTTPRequest, url string) (*Match, error) {
|
||||
method := strings.ToUpper(req.Method)
|
||||
if method == "" {
|
||||
method = "GET"
|
||||
}
|
||||
|
||||
var body io.Reader
|
||||
if req.Body != "" {
|
||||
body = bytes.NewBufferString(req.Body)
|
||||
}
|
||||
|
||||
r, err := http.NewRequestWithContext(ctx, method, url, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for k, v := range req.Headers {
|
||||
r.Header.Set(k, v)
|
||||
}
|
||||
if r.Header.Get("User-Agent") == "" {
|
||||
r.Header.Set("User-Agent", e.UserAgent)
|
||||
}
|
||||
|
||||
// Honor the redirects flag; default is NO redirect follow (safer
|
||||
// for vuln detection since a 3xx-based probe might be exactly what
|
||||
// we want to measure).
|
||||
client := e.Client
|
||||
if !req.Redirects {
|
||||
wrapped := *client
|
||||
wrapped.CheckRedirect = func(*http.Request, []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
}
|
||||
client = &wrapped
|
||||
}
|
||||
|
||||
resp, err := client.Do(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
bodyBytes, _ := io.ReadAll(io.LimitReader(resp.Body, e.MaxBodyB))
|
||||
|
||||
// Apply matchers.
|
||||
condition := strings.ToLower(strings.TrimSpace(req.MatchersCondition))
|
||||
if condition == "" {
|
||||
condition = "or"
|
||||
}
|
||||
|
||||
fired := 0
|
||||
for _, m := range req.Matchers {
|
||||
if matcherHits(m, resp, bodyBytes) {
|
||||
fired++
|
||||
}
|
||||
}
|
||||
|
||||
switch condition {
|
||||
case "and":
|
||||
if fired != len(req.Matchers) {
|
||||
return nil, nil
|
||||
}
|
||||
case "or":
|
||||
if fired == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
default:
|
||||
if fired == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
return &Match{
|
||||
TemplateID: t.ID,
|
||||
TemplateURL: firstRef(t.Info.Reference),
|
||||
Name: t.Info.Name,
|
||||
Severity: t.Severity(),
|
||||
Description: t.Info.Description,
|
||||
Tags: t.Tags(),
|
||||
URL: url,
|
||||
Evidence: evidenceSnippet(bodyBytes, resp),
|
||||
CVEs: extractCVEs(t.ID, t.Info.Reference),
|
||||
Author: t.Info.Author,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// matcherHits returns true when the matcher m fires against the response.
|
||||
// Respects m.Negative (inverts), m.Condition (and|or over word list), and
|
||||
// m.Part (header|body|response|all; default body).
|
||||
func matcherHits(m Matcher, resp *http.Response, body []byte) bool {
|
||||
hit := false
|
||||
switch m.Type {
|
||||
case "status":
|
||||
for _, code := range m.Status {
|
||||
if resp.StatusCode == code {
|
||||
hit = true
|
||||
break
|
||||
}
|
||||
}
|
||||
case "size":
|
||||
for _, sz := range m.Size {
|
||||
if len(body) == sz {
|
||||
hit = true
|
||||
break
|
||||
}
|
||||
}
|
||||
case "word":
|
||||
corpus := selectCorpus(m.Part, resp, body)
|
||||
hit = wordMatch(m, corpus)
|
||||
case "regex":
|
||||
corpus := selectCorpus(m.Part, resp, body)
|
||||
hit = regexMatch(m, corpus)
|
||||
}
|
||||
if m.Negative {
|
||||
return !hit
|
||||
}
|
||||
return hit
|
||||
}
|
||||
|
||||
func selectCorpus(part string, resp *http.Response, body []byte) string {
|
||||
switch strings.ToLower(strings.TrimSpace(part)) {
|
||||
case "header":
|
||||
return formatHeaders(resp.Header)
|
||||
case "response", "all":
|
||||
return formatHeaders(resp.Header) + "\n\n" + string(body)
|
||||
case "body", "":
|
||||
return string(body)
|
||||
default:
|
||||
return string(body)
|
||||
}
|
||||
}
|
||||
|
||||
func wordMatch(m Matcher, corpus string) bool {
|
||||
if len(m.Words) == 0 {
|
||||
return false
|
||||
}
|
||||
condition := strings.ToLower(strings.TrimSpace(m.Condition))
|
||||
if condition == "" {
|
||||
condition = "or"
|
||||
}
|
||||
lower := strings.ToLower(corpus)
|
||||
if condition == "and" {
|
||||
for _, w := range m.Words {
|
||||
if !strings.Contains(lower, strings.ToLower(w)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
// or
|
||||
for _, w := range m.Words {
|
||||
if strings.Contains(lower, strings.ToLower(w)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func regexMatch(m Matcher, corpus string) bool {
|
||||
if len(m.Regex) == 0 {
|
||||
return false
|
||||
}
|
||||
condition := strings.ToLower(strings.TrimSpace(m.Condition))
|
||||
if condition == "" {
|
||||
condition = "or"
|
||||
}
|
||||
compiled := make([]*regexp.Regexp, 0, len(m.Regex))
|
||||
for _, pat := range m.Regex {
|
||||
re, err := regexp.Compile(pat)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
compiled = append(compiled, re)
|
||||
}
|
||||
if len(compiled) == 0 {
|
||||
return false
|
||||
}
|
||||
if condition == "and" {
|
||||
for _, re := range compiled {
|
||||
if !re.MatchString(corpus) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
for _, re := range compiled {
|
||||
if re.MatchString(corpus) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// --- helpers -------------------------------------------------------------
|
||||
|
||||
// expandPath substitutes Nuclei template variables with real values.
|
||||
// {{BaseURL}} → baseURL unchanged ("https://example.com")
|
||||
// {{Hostname}} → host portion of baseURL
|
||||
// {{RootURL}} → scheme + host (no path)
|
||||
func expandPath(template, baseURL string) string {
|
||||
host := hostOnly(baseURL)
|
||||
root := rootURL(baseURL)
|
||||
out := strings.ReplaceAll(template, "{{BaseURL}}", baseURL)
|
||||
out = strings.ReplaceAll(out, "{{Hostname}}", host)
|
||||
out = strings.ReplaceAll(out, "{{RootURL}}", root)
|
||||
return out
|
||||
}
|
||||
|
||||
func hostOnly(u string) string {
|
||||
s := strings.TrimPrefix(u, "https://")
|
||||
s = strings.TrimPrefix(s, "http://")
|
||||
if i := strings.IndexAny(s, "/?#"); i >= 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func rootURL(u string) string {
|
||||
s := u
|
||||
scheme := ""
|
||||
switch {
|
||||
case strings.HasPrefix(s, "https://"):
|
||||
scheme = "https://"
|
||||
s = s[len("https://"):]
|
||||
case strings.HasPrefix(s, "http://"):
|
||||
scheme = "http://"
|
||||
s = s[len("http://"):]
|
||||
}
|
||||
if i := strings.IndexAny(s, "/?#"); i >= 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
return scheme + s
|
||||
}
|
||||
|
||||
func formatHeaders(h http.Header) string {
|
||||
var sb strings.Builder
|
||||
for k, vs := range h {
|
||||
for _, v := range vs {
|
||||
fmt.Fprintf(&sb, "%s: %s\n", k, v)
|
||||
}
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func evidenceSnippet(body []byte, resp *http.Response) string {
|
||||
const maxSnippet = 500
|
||||
s := string(body)
|
||||
if len(s) > maxSnippet {
|
||||
s = s[:maxSnippet] + "…"
|
||||
}
|
||||
return fmt.Sprintf("HTTP %d — %s", resp.StatusCode, s)
|
||||
}
|
||||
|
||||
// firstRef returns the first URL in the reference list (usually the
|
||||
// nuclei-templates source or the advisory).
|
||||
func firstRef(refs []string) string {
|
||||
for _, r := range refs {
|
||||
r = strings.TrimSpace(r)
|
||||
if r != "" {
|
||||
return r
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// extractCVEs scans the template ID and references for CVE IDs.
|
||||
func extractCVEs(id string, refs []string) []string {
|
||||
re := regexp.MustCompile(`(?i)CVE-\d{4}-\d{4,7}`)
|
||||
seen := make(map[string]bool)
|
||||
var out []string
|
||||
add := func(s string) {
|
||||
for _, m := range re.FindAllString(s, -1) {
|
||||
up := strings.ToUpper(m)
|
||||
if !seen[up] {
|
||||
seen[up] = true
|
||||
out = append(out, up)
|
||||
}
|
||||
}
|
||||
}
|
||||
add(id)
|
||||
for _, r := range refs {
|
||||
add(r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
package nucleitpl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// mkTemplate builds a minimal Template in-memory for tests.
|
||||
func mkTemplate(id string, path string, matchers []Matcher, condition string) *Template {
|
||||
return &Template{
|
||||
ID: id,
|
||||
Info: Info{
|
||||
Name: "Test " + id,
|
||||
Severity: "high",
|
||||
},
|
||||
Requests: []HTTPRequest{{
|
||||
Method: "GET",
|
||||
Path: []string{path},
|
||||
Matchers: matchers,
|
||||
MatchersCondition: condition,
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutor_WordMatch(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(200)
|
||||
_, _ = w.Write([]byte("<html>PHP Version 7.4.3 loaded</html>"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
tpl := mkTemplate("test-phpinfo",
|
||||
"{{BaseURL}}/info.php",
|
||||
[]Matcher{{Type: "word", Part: "body", Words: []string{"PHP Version"}}},
|
||||
"")
|
||||
|
||||
e := NewExecutor(nil, 5*time.Second)
|
||||
matches := e.Run(context.Background(), tpl, srv.URL)
|
||||
if len(matches) != 1 {
|
||||
t.Fatalf("expected 1 match, got %d", len(matches))
|
||||
}
|
||||
if matches[0].TemplateID != "test-phpinfo" {
|
||||
t.Errorf("wrong template: %s", matches[0].TemplateID)
|
||||
}
|
||||
if !strings.Contains(matches[0].Evidence, "PHP Version") {
|
||||
t.Errorf("evidence missing snippet: %q", matches[0].Evidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutor_StatusMatch(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(403)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
tpl := mkTemplate("test-403",
|
||||
"{{BaseURL}}/admin",
|
||||
[]Matcher{{Type: "status", Status: []int{403, 401}}},
|
||||
"")
|
||||
|
||||
e := NewExecutor(nil, 5*time.Second)
|
||||
matches := e.Run(context.Background(), tpl, srv.URL)
|
||||
if len(matches) != 1 {
|
||||
t.Fatalf("expected match, got %d", len(matches))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutor_ANDCondition(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(200)
|
||||
_, _ = w.Write([]byte("admin panel access"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
// Both matchers must fire.
|
||||
tpl := mkTemplate("test-and",
|
||||
"{{BaseURL}}/",
|
||||
[]Matcher{
|
||||
{Type: "word", Part: "body", Words: []string{"admin"}},
|
||||
{Type: "status", Status: []int{200}},
|
||||
}, "and")
|
||||
|
||||
e := NewExecutor(nil, 5*time.Second)
|
||||
matches := e.Run(context.Background(), tpl, srv.URL)
|
||||
if len(matches) != 1 {
|
||||
t.Errorf("expected AND match to fire, got %d", len(matches))
|
||||
}
|
||||
|
||||
// If we flip status to something the server doesn't return, AND fails.
|
||||
tpl.Requests[0].Matchers[1].Status = []int{500}
|
||||
matches = e.Run(context.Background(), tpl, srv.URL)
|
||||
if len(matches) != 0 {
|
||||
t.Errorf("AND should fail when one matcher doesn't, got %d", len(matches))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutor_NegativeMatcher(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(200)
|
||||
_, _ = w.Write([]byte("welcome"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
tpl := mkTemplate("test-neg",
|
||||
"{{BaseURL}}/",
|
||||
[]Matcher{{Type: "word", Part: "body", Words: []string{"error"}, Negative: true}},
|
||||
"")
|
||||
|
||||
e := NewExecutor(nil, 5*time.Second)
|
||||
matches := e.Run(context.Background(), tpl, srv.URL)
|
||||
if len(matches) != 1 {
|
||||
t.Errorf("negative should fire (body doesn't contain 'error'), got %d", len(matches))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutor_RegexMatch(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(200)
|
||||
_, _ = w.Write([]byte("Server: Apache/2.4.52 (Ubuntu)"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
tpl := mkTemplate("test-re",
|
||||
"{{BaseURL}}/",
|
||||
[]Matcher{{Type: "regex", Part: "body", Regex: []string{`Apache/\d+\.\d+\.\d+`}}},
|
||||
"")
|
||||
|
||||
e := NewExecutor(nil, 5*time.Second)
|
||||
matches := e.Run(context.Background(), tpl, srv.URL)
|
||||
if len(matches) != 1 {
|
||||
t.Errorf("regex match should fire, got %d", len(matches))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutor_HeaderPart(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Powered-By", "Express")
|
||||
w.WriteHeader(200)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
tpl := mkTemplate("test-header",
|
||||
"{{BaseURL}}/",
|
||||
[]Matcher{{Type: "word", Part: "header", Words: []string{"X-Powered-By"}}},
|
||||
"")
|
||||
|
||||
e := NewExecutor(nil, 5*time.Second)
|
||||
matches := e.Run(context.Background(), tpl, srv.URL)
|
||||
if len(matches) != 1 {
|
||||
t.Errorf("header matcher should fire, got %d", len(matches))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutor_NoMatchReturnsEmpty(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(200)
|
||||
_, _ = w.Write([]byte("nothing interesting"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
tpl := mkTemplate("test-nomatch",
|
||||
"{{BaseURL}}/",
|
||||
[]Matcher{{Type: "word", Part: "body", Words: []string{"definitely_not_here"}}},
|
||||
"")
|
||||
|
||||
e := NewExecutor(nil, 5*time.Second)
|
||||
matches := e.Run(context.Background(), tpl, srv.URL)
|
||||
if len(matches) != 0 {
|
||||
t.Errorf("non-match should return empty, got %d", len(matches))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandPath(t *testing.T) {
|
||||
cases := []struct {
|
||||
path, base, want string
|
||||
}{
|
||||
{"{{BaseURL}}/admin", "https://example.com", "https://example.com/admin"},
|
||||
{"{{Hostname}}/x", "https://api.example.com/v1", "api.example.com/x"},
|
||||
{"{{RootURL}}/r", "http://sub.example.com/deep/path", "http://sub.example.com/r"},
|
||||
{"/static/admin", "https://x.com", "/static/admin"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := expandPath(c.path, c.base); got != c.want {
|
||||
t.Errorf("expandPath(%q, %q) = %q, want %q", c.path, c.base, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractCVEs(t *testing.T) {
|
||||
cves := extractCVEs("cve-2021-23017-nginx", []string{
|
||||
"https://nvd.nist.gov/vuln/detail/CVE-2021-23017", // dup of ID after upper-casing
|
||||
"https://example.com/adv/CVE-2020-15168",
|
||||
})
|
||||
if len(cves) != 2 {
|
||||
t.Errorf("expected 2 unique CVE IDs, got %d: %v", len(cves), cves)
|
||||
}
|
||||
if cves[0] != "CVE-2021-23017" || cves[1] != "CVE-2020-15168" {
|
||||
t.Errorf("unexpected order: %v", cves)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutor_UnsupportedTemplateNoop(t *testing.T) {
|
||||
tpl := &Template{
|
||||
ID: "dns-tpl",
|
||||
DNS: []string{"placeholder"},
|
||||
}
|
||||
e := NewExecutor(nil, 5*time.Second)
|
||||
matches := e.Run(context.Background(), tpl, "https://example.com")
|
||||
if len(matches) != 0 {
|
||||
t.Errorf("unsupported template should return no matches, got %d", len(matches))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
// Package nucleitpl parses and executes a subset of the Nuclei YAML
|
||||
// template format. The goal is to run community HTTP templates unchanged
|
||||
// so God's Eye gets access to the ~8000-template ecosystem without
|
||||
// reimplementing detections one-by-one.
|
||||
//
|
||||
// Supported subset (covers roughly 70% of HTTP templates in the public
|
||||
// nuclei-templates repo at time of writing):
|
||||
//
|
||||
// - Top-level: id, info { name, severity, description, tags, author }
|
||||
// - Protocol: requests: (aliased as http: in newer templates)
|
||||
// - Per-request: method, path (with {{BaseURL}}/{{Hostname}} substitution),
|
||||
// headers, body, redirects (bool), matchers-condition (and|or)
|
||||
// - Matchers: type=word (word|part|condition),
|
||||
// type=regex (regex|part),
|
||||
// type=status (status),
|
||||
// type=size (size)
|
||||
// - Severity mapping: info/low/medium/high/critical
|
||||
//
|
||||
// Out of scope (templates using these are skipped with a reason logged):
|
||||
//
|
||||
// - Protocols other than http: dns, ssl, network, file, code, javascript,
|
||||
// workflow, headless, flow
|
||||
// - Pre-conditions, payloads, extractors, dynamic variables,
|
||||
// stop-at-first-match, cluster, self-contained
|
||||
// - Interactsh (OOB) — requires a callback server we don't ship yet
|
||||
// - Fuzzing templates
|
||||
//
|
||||
// A skipped template logs via the returned diagnostic; the executor never
|
||||
// panics on an unsupported template.
|
||||
package nucleitpl
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// Template is the parsed form of a Nuclei YAML file.
|
||||
type Template struct {
|
||||
ID string `yaml:"id"`
|
||||
Info Info `yaml:"info"`
|
||||
Requests []HTTPRequest `yaml:"requests,omitempty"`
|
||||
HTTP []HTTPRequest `yaml:"http,omitempty"` // newer alias for requests
|
||||
// Unsupported protocols — presence triggers skip with reason.
|
||||
DNS interface{} `yaml:"dns,omitempty"`
|
||||
SSL interface{} `yaml:"ssl,omitempty"`
|
||||
Network interface{} `yaml:"network,omitempty"`
|
||||
File interface{} `yaml:"file,omitempty"`
|
||||
Code interface{} `yaml:"code,omitempty"`
|
||||
Headless interface{} `yaml:"headless,omitempty"`
|
||||
Workflow interface{} `yaml:"workflows,omitempty"`
|
||||
// SourcePath is populated by Load so diagnostics can reference the file.
|
||||
SourcePath string `yaml:"-"`
|
||||
}
|
||||
|
||||
// Info is the template metadata block.
|
||||
type Info struct {
|
||||
Name string `yaml:"name"`
|
||||
Author string `yaml:"author,omitempty"`
|
||||
Severity string `yaml:"severity"`
|
||||
Description string `yaml:"description,omitempty"`
|
||||
Reference []string `yaml:"reference,omitempty"`
|
||||
Tags string `yaml:"tags,omitempty"`
|
||||
}
|
||||
|
||||
// HTTPRequest is one HTTP interaction in a template.
|
||||
type HTTPRequest struct {
|
||||
Method string `yaml:"method,omitempty"` // default GET
|
||||
Path []string `yaml:"path"`
|
||||
Headers map[string]string `yaml:"headers,omitempty"`
|
||||
Body string `yaml:"body,omitempty"`
|
||||
Redirects bool `yaml:"redirects,omitempty"`
|
||||
MaxRedirects int `yaml:"max-redirects,omitempty"`
|
||||
MatchersCondition string `yaml:"matchers-condition,omitempty"` // "and" | "or" (default "or")
|
||||
Matchers []Matcher `yaml:"matchers"`
|
||||
// Unsupported fields that, if present with values, trigger a skip.
|
||||
Payloads interface{} `yaml:"payloads,omitempty"`
|
||||
Extractors interface{} `yaml:"extractors,omitempty"`
|
||||
Fuzzing interface{} `yaml:"fuzzing,omitempty"`
|
||||
Unsafe bool `yaml:"unsafe,omitempty"`
|
||||
Attack string `yaml:"attack,omitempty"`
|
||||
Raw []string `yaml:"raw,omitempty"`
|
||||
Pipeline bool `yaml:"pipeline,omitempty"`
|
||||
Threads int `yaml:"threads,omitempty"`
|
||||
StopAtFirst bool `yaml:"stop-at-first-match,omitempty"`
|
||||
}
|
||||
|
||||
// Matcher is a single match rule within a request.
|
||||
type Matcher struct {
|
||||
Type string `yaml:"type"` // word | regex | status | size | dsl | binary
|
||||
Part string `yaml:"part,omitempty"` // header | body | response (default body)
|
||||
Condition string `yaml:"condition,omitempty"` // and | or (default or)
|
||||
Negative bool `yaml:"negative,omitempty"`
|
||||
Words []string `yaml:"words,omitempty"`
|
||||
Regex []string `yaml:"regex,omitempty"`
|
||||
Status []int `yaml:"status,omitempty"`
|
||||
Size []int `yaml:"size,omitempty"`
|
||||
// Unsupported — presence marks the matcher unusable.
|
||||
DSL []string `yaml:"dsl,omitempty"`
|
||||
Binary []string `yaml:"binary,omitempty"`
|
||||
}
|
||||
|
||||
// Load parses a single YAML file into a Template. Malformed YAML or empty
|
||||
// files return (nil, err); structurally valid YAML that references unused
|
||||
// protocols still Load successfully — IsSupported/IsSupported reason tell
|
||||
// the caller whether to execute it.
|
||||
func Load(path string) (*Template, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var t Template
|
||||
if err := yaml.Unmarshal(data, &t); err != nil {
|
||||
return nil, fmt.Errorf("parse %s: %w", path, err)
|
||||
}
|
||||
if t.ID == "" {
|
||||
return nil, fmt.Errorf("parse %s: missing id field", path)
|
||||
}
|
||||
t.SourcePath = path
|
||||
// Normalize requests vs http alias.
|
||||
if len(t.Requests) == 0 && len(t.HTTP) > 0 {
|
||||
t.Requests = t.HTTP
|
||||
}
|
||||
return &t, nil
|
||||
}
|
||||
|
||||
// LoadDir walks dir recursively, loads every .yaml / .yml file, and
|
||||
// returns the slice of successfully-parsed templates. Parse errors are
|
||||
// collected into the returned diagnostics slice but do not stop the walk.
|
||||
func LoadDir(dir string) ([]*Template, []string, error) {
|
||||
var tpls []*Template
|
||||
var diags []string
|
||||
|
||||
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil // skip unreadable files silently
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
ext := strings.ToLower(filepath.Ext(path))
|
||||
if ext != ".yaml" && ext != ".yml" {
|
||||
return nil
|
||||
}
|
||||
t, err := Load(path)
|
||||
if err != nil {
|
||||
diags = append(diags, fmt.Sprintf("parse %s: %v", path, err))
|
||||
return nil
|
||||
}
|
||||
tpls = append(tpls, t)
|
||||
return nil
|
||||
})
|
||||
return tpls, diags, err
|
||||
}
|
||||
|
||||
// TargetsCurrentHost reports whether every request path in the template
|
||||
// is scoped to the scanned host — i.e. uses {{BaseURL}}, {{Hostname}},
|
||||
// {{RootURL}}, or a leading "/". Templates with absolute URLs to
|
||||
// third-party services (common in OSINT / user-presence checks) would
|
||||
// otherwise fire against unrelated hosts with unresolved placeholders
|
||||
// like {{user}} — and their matchers often succeed on whatever generic
|
||||
// response the third party returns, producing high-volume false
|
||||
// positives against a single-target scan.
|
||||
//
|
||||
// Returns false + reason when any request path is off-host.
|
||||
func (t *Template) TargetsCurrentHost() (bool, string) {
|
||||
for i, r := range t.Requests {
|
||||
for j, p := range r.Path {
|
||||
ok := false
|
||||
switch {
|
||||
case strings.HasPrefix(p, "{{BaseURL}}"),
|
||||
strings.HasPrefix(p, "{{Hostname}}"),
|
||||
strings.HasPrefix(p, "{{RootURL}}"),
|
||||
strings.HasPrefix(p, "/"):
|
||||
ok = true
|
||||
}
|
||||
if !ok {
|
||||
// Also allow the special case where the path is exactly
|
||||
// a template variable (no literal text).
|
||||
if p == "{{BaseURL}}" || p == "{{Hostname}}" || p == "{{RootURL}}" {
|
||||
ok = true
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
return false, fmt.Sprintf("request[%d].path[%d] %q does not target the scanned host", i, j, truncateStr(p, 60))
|
||||
}
|
||||
}
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
func truncateStr(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n] + "…"
|
||||
}
|
||||
|
||||
// IsSupported returns (true, "") when the template uses only features
|
||||
// understood by the executor. Templates that would need unsupported
|
||||
// protocols, payloads, extractors, or fuzzing return (false, reason).
|
||||
// Templates that target third-party hosts (OSINT-style user lookups)
|
||||
// also return false to prevent spurious matches during targeted scans.
|
||||
func (t *Template) IsSupported() (bool, string) {
|
||||
if t == nil {
|
||||
return false, "nil template"
|
||||
}
|
||||
if t.DNS != nil {
|
||||
return false, "dns protocol (unsupported)"
|
||||
}
|
||||
if t.SSL != nil {
|
||||
return false, "ssl protocol (unsupported)"
|
||||
}
|
||||
if t.Network != nil {
|
||||
return false, "network protocol (unsupported)"
|
||||
}
|
||||
if t.File != nil {
|
||||
return false, "file protocol (unsupported)"
|
||||
}
|
||||
if t.Code != nil {
|
||||
return false, "code protocol (unsupported)"
|
||||
}
|
||||
if t.Headless != nil {
|
||||
return false, "headless protocol (unsupported)"
|
||||
}
|
||||
if t.Workflow != nil {
|
||||
return false, "workflow (unsupported)"
|
||||
}
|
||||
if len(t.Requests) == 0 {
|
||||
return false, "no http requests"
|
||||
}
|
||||
for i, r := range t.Requests {
|
||||
if r.Payloads != nil {
|
||||
return false, fmt.Sprintf("request[%d] uses payloads (unsupported)", i)
|
||||
}
|
||||
if r.Extractors != nil {
|
||||
// Tolerate extractors on the first pass; we ignore them.
|
||||
// Templates with only extractors still run; their findings are
|
||||
// just matcher-based.
|
||||
}
|
||||
if r.Fuzzing != nil {
|
||||
return false, fmt.Sprintf("request[%d] uses fuzzing (unsupported)", i)
|
||||
}
|
||||
if r.Unsafe {
|
||||
return false, fmt.Sprintf("request[%d] is unsafe (raw TCP)", i)
|
||||
}
|
||||
if len(r.Raw) > 0 {
|
||||
return false, fmt.Sprintf("request[%d] uses raw (unsupported)", i)
|
||||
}
|
||||
if len(r.Path) == 0 {
|
||||
return false, fmt.Sprintf("request[%d] has no path", i)
|
||||
}
|
||||
if len(r.Matchers) == 0 {
|
||||
return false, fmt.Sprintf("request[%d] has no matchers", i)
|
||||
}
|
||||
for j, m := range r.Matchers {
|
||||
switch m.Type {
|
||||
case "word", "regex", "status", "size":
|
||||
// supported
|
||||
case "dsl", "binary":
|
||||
return false, fmt.Sprintf("request[%d].matcher[%d] type=%s (unsupported)", i, j, m.Type)
|
||||
default:
|
||||
return false, fmt.Sprintf("request[%d].matcher[%d] type=%s (unknown)", i, j, m.Type)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Scope check: skip templates that probe third-party hosts.
|
||||
if ok, reason := t.TargetsCurrentHost(); !ok {
|
||||
return false, reason
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// Severity returns the OWASP-style severity, defaulting to "info" when
|
||||
// the template omits it.
|
||||
func (t *Template) Severity() string {
|
||||
s := strings.ToLower(strings.TrimSpace(t.Info.Severity))
|
||||
switch s {
|
||||
case "critical", "high", "medium", "low", "info":
|
||||
return s
|
||||
default:
|
||||
return "info"
|
||||
}
|
||||
}
|
||||
|
||||
// Tags returns the comma-separated tags as a string slice.
|
||||
func (t *Template) Tags() []string {
|
||||
if t.Info.Tags == "" {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, p := range strings.Split(t.Info.Tags, ",") {
|
||||
p = strings.TrimSpace(p)
|
||||
if p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
package nucleitpl
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const sampleSupported = `
|
||||
id: test-basic-word-match
|
||||
info:
|
||||
name: Test Basic Word Match
|
||||
author: vyntral
|
||||
severity: high
|
||||
description: Fires when response body contains 'phpinfo'
|
||||
tags: exposure,php
|
||||
reference:
|
||||
- https://example.com/advisory/CVE-2021-12345
|
||||
requests:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}/phpinfo.php"
|
||||
matchers:
|
||||
- type: word
|
||||
part: body
|
||||
words:
|
||||
- "PHP Version"
|
||||
- type: status
|
||||
status:
|
||||
- 200
|
||||
matchers-condition: and
|
||||
`
|
||||
|
||||
const sampleUnsupportedDNS = `
|
||||
id: test-dns
|
||||
info:
|
||||
name: Test DNS
|
||||
severity: medium
|
||||
dns:
|
||||
- name: "{{FQDN}}"
|
||||
type: TXT
|
||||
matchers:
|
||||
- type: word
|
||||
words: ["v=spf"]
|
||||
`
|
||||
|
||||
const sampleUnsupportedPayloads = `
|
||||
id: test-payloads
|
||||
info:
|
||||
name: Test Payloads
|
||||
severity: low
|
||||
requests:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}/{{word}}"
|
||||
payloads:
|
||||
word:
|
||||
- admin
|
||||
- backup
|
||||
matchers:
|
||||
- type: status
|
||||
status: [200]
|
||||
`
|
||||
|
||||
const sampleBadYAML = `
|
||||
id: [unclosed
|
||||
info:
|
||||
name:
|
||||
`
|
||||
|
||||
func writeTmp(t *testing.T, name, content string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestLoad_Supported(t *testing.T) {
|
||||
path := writeTmp(t, "ok.yaml", sampleSupported)
|
||||
tpl, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tpl.ID != "test-basic-word-match" {
|
||||
t.Errorf("ID = %q", tpl.ID)
|
||||
}
|
||||
if tpl.Info.Severity != "high" {
|
||||
t.Errorf("Severity = %q", tpl.Info.Severity)
|
||||
}
|
||||
if len(tpl.Requests) != 1 {
|
||||
t.Fatalf("Requests len = %d", len(tpl.Requests))
|
||||
}
|
||||
r := tpl.Requests[0]
|
||||
if r.Path[0] != "{{BaseURL}}/phpinfo.php" {
|
||||
t.Errorf("Path[0] = %q", r.Path[0])
|
||||
}
|
||||
if len(r.Matchers) != 2 {
|
||||
t.Errorf("Matchers len = %d", len(r.Matchers))
|
||||
}
|
||||
if r.MatchersCondition != "and" {
|
||||
t.Errorf("MatchersCondition = %q", r.MatchersCondition)
|
||||
}
|
||||
if ok, reason := tpl.IsSupported(); !ok {
|
||||
t.Errorf("should be supported; reason=%q", reason)
|
||||
}
|
||||
if tags := tpl.Tags(); len(tags) != 2 || tags[0] != "exposure" {
|
||||
t.Errorf("Tags = %v", tags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_DNSUnsupported(t *testing.T) {
|
||||
path := writeTmp(t, "dns.yaml", sampleUnsupportedDNS)
|
||||
tpl, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ok, reason := tpl.IsSupported()
|
||||
if ok {
|
||||
t.Error("dns template should be unsupported")
|
||||
}
|
||||
if reason == "" {
|
||||
t.Error("expected non-empty reason")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_PayloadsUnsupported(t *testing.T) {
|
||||
path := writeTmp(t, "payloads.yaml", sampleUnsupportedPayloads)
|
||||
tpl, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ok, reason := tpl.IsSupported()
|
||||
if ok {
|
||||
t.Error("payloads template should be unsupported")
|
||||
}
|
||||
if reason == "" {
|
||||
t.Error("expected non-empty reason")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_BadYAML(t *testing.T) {
|
||||
path := writeTmp(t, "bad.yaml", sampleBadYAML)
|
||||
if _, err := Load(path); err == nil {
|
||||
t.Error("expected parse error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_MissingID(t *testing.T) {
|
||||
path := writeTmp(t, "noid.yaml", "info:\n severity: low\n")
|
||||
if _, err := Load(path); err == nil {
|
||||
t.Error("expected missing id error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadDir(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
_ = os.WriteFile(filepath.Join(dir, "a.yaml"), []byte(sampleSupported), 0o644)
|
||||
_ = os.WriteFile(filepath.Join(dir, "b.yaml"), []byte(sampleUnsupportedDNS), 0o644)
|
||||
_ = os.WriteFile(filepath.Join(dir, "c.yml"), []byte(sampleSupported), 0o644)
|
||||
_ = os.WriteFile(filepath.Join(dir, "d.bad"), []byte("???"), 0o644)
|
||||
_ = os.WriteFile(filepath.Join(dir, "e.yaml"), []byte(sampleBadYAML), 0o644)
|
||||
|
||||
sub := filepath.Join(dir, "nested")
|
||||
_ = os.MkdirAll(sub, 0o755)
|
||||
_ = os.WriteFile(filepath.Join(sub, "f.yaml"), []byte(sampleSupported), 0o644)
|
||||
|
||||
tpls, diags, err := LoadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 3 supported (a, c, nested/f), 1 dns (b), 1 parse error (e). .bad ignored.
|
||||
if got := len(tpls); got != 4 {
|
||||
t.Errorf("loaded = %d, want 4 (3 supported + 1 dns)", got)
|
||||
}
|
||||
if len(diags) != 1 {
|
||||
t.Errorf("diags = %d, want 1", len(diags))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeverity_Default(t *testing.T) {
|
||||
tpl := &Template{Info: Info{Severity: "UNKNOWN"}}
|
||||
if sev := tpl.Severity(); sev != "info" {
|
||||
t.Errorf("got %q, want info", sev)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeverity_Normalized(t *testing.T) {
|
||||
for input, want := range map[string]string{
|
||||
"critical": "critical",
|
||||
"HIGH": "high",
|
||||
" Medium ": "medium",
|
||||
"LOW": "low",
|
||||
"info": "info",
|
||||
"": "info",
|
||||
} {
|
||||
tpl := &Template{Info: Info{Severity: input}}
|
||||
if got := tpl.Severity(); got != want {
|
||||
t.Errorf("Severity(%q) = %q, want %q", input, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPAlias(t *testing.T) {
|
||||
content := `
|
||||
id: http-alias
|
||||
info:
|
||||
severity: low
|
||||
http:
|
||||
- method: GET
|
||||
path: ["{{BaseURL}}/"]
|
||||
matchers:
|
||||
- type: status
|
||||
status: [200]
|
||||
`
|
||||
path := writeTmp(t, "http.yaml", content)
|
||||
tpl, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(tpl.Requests) != 1 {
|
||||
t.Errorf("expected http: to be aliased to Requests, got %d", len(tpl.Requests))
|
||||
}
|
||||
if ok, _ := tpl.IsSupported(); !ok {
|
||||
t.Error("http alias template should be supported")
|
||||
}
|
||||
}
|
||||
@@ -50,11 +50,10 @@ func PrintBanner() {
|
||||
fmt.Println(BoldWhite(" ╚██████╔╝╚██████╔╝██████╔╝") + BoldGreen("███████║") + BoldWhite(" ███████╗ ██║ ███████╗"))
|
||||
fmt.Println(BoldWhite(" ╚═════╝ ╚═════╝ ╚═════╝ ") + BoldGreen("╚══════╝") + BoldWhite(" ╚══════╝ ╚═╝ ╚══════╝"))
|
||||
fmt.Println()
|
||||
fmt.Printf(" %s %s\n", BoldGreen("⚡"), Dim("AI-powered attack surface discovery & security analysis"))
|
||||
fmt.Printf(" %s %s %s %s %s %s\n",
|
||||
Dim("Version:"), BoldGreen("0.1"),
|
||||
Dim("By:"), White("github.com/Vyntral"),
|
||||
Dim("For:"), Yellow("github.com/Orizon-eu"))
|
||||
fmt.Printf(" %s %s\n", BoldGreen("⚡"), Dim("AI-powered attack surface discovery & offensive security analysis"))
|
||||
fmt.Printf(" %s %s %s %s\n",
|
||||
Dim("Version:"), BoldGreen("2.0.0-rc1"),
|
||||
Dim("By:"), White("github.com/Vyntral"))
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
// Package pipeline coordinates v2 module execution. It builds a Module list
|
||||
// from the registry, applies the ConfigView filter, then runs every selected
|
||||
// module concurrently under a shared event bus and store.
|
||||
//
|
||||
// Unlike the legacy scanner.Run, this coordinator does NO domain-specific
|
||||
// work of its own. Every phase (passive, brute, resolve, probe, security,
|
||||
// AI, reporting) is a Module. Ordering emerges from events, with explicit
|
||||
// phase barriers for phases that must complete before downstream begins.
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/config"
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
// Pipeline is the v2 scan coordinator.
|
||||
type Pipeline struct {
|
||||
cfg *config.Config
|
||||
view *config.View
|
||||
bus *eventbus.Bus
|
||||
store store.Store
|
||||
modReg *module.Registry
|
||||
|
||||
// ownBus / ownStore indicate resources created by this Pipeline that
|
||||
// must be closed on Shutdown. Injected resources are left to the caller.
|
||||
ownBus bool
|
||||
ownStore bool
|
||||
}
|
||||
|
||||
// Options are optional overrides for New. Empty fields mean "use defaults".
|
||||
type Options struct {
|
||||
Bus *eventbus.Bus // injected bus; defaults to a new one
|
||||
Store store.Store // injected store; defaults to NewMemoryStore
|
||||
Registry *module.Registry // registry to draw modules from; defaults to module.Default()
|
||||
Buffer int // bus buffer size when creating default bus
|
||||
}
|
||||
|
||||
// New creates a Pipeline from cfg and opts. The pipeline is ready to Run.
|
||||
// A non-nil Config is required.
|
||||
func New(cfg *config.Config, opts Options) (*Pipeline, error) {
|
||||
if cfg == nil {
|
||||
return nil, errors.New("pipeline.New: nil config")
|
||||
}
|
||||
|
||||
p := &Pipeline{
|
||||
cfg: cfg,
|
||||
view: config.NewView(cfg),
|
||||
modReg: opts.Registry,
|
||||
}
|
||||
if p.modReg == nil {
|
||||
p.modReg = module.Default()
|
||||
}
|
||||
|
||||
if opts.Bus != nil {
|
||||
p.bus = opts.Bus
|
||||
} else {
|
||||
buf := opts.Buffer
|
||||
if buf <= 0 {
|
||||
buf = 4096
|
||||
}
|
||||
p.bus = eventbus.New(buf)
|
||||
p.ownBus = true
|
||||
}
|
||||
|
||||
if opts.Store != nil {
|
||||
p.store = opts.Store
|
||||
} else {
|
||||
p.store = store.NewMemoryStore()
|
||||
p.ownStore = true
|
||||
}
|
||||
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// Bus returns the underlying event bus. Useful for attaching external
|
||||
// subscribers (TUI, metrics, log sinks) before calling Run.
|
||||
func (p *Pipeline) Bus() *eventbus.Bus { return p.bus }
|
||||
|
||||
// Store returns the underlying store. Useful for post-scan querying or
|
||||
// report generation outside of modules.
|
||||
func (p *Pipeline) Store() store.Store { return p.store }
|
||||
|
||||
// Run executes the selected modules. Returns when every module has exited
|
||||
// OR ctx is canceled. The returned error aggregates any module errors via
|
||||
// errors.Join.
|
||||
//
|
||||
// Execution semantics:
|
||||
// - ScanStarted is published first.
|
||||
// - Modules are grouped by Phase; each Phase is a barrier: phase N starts
|
||||
// only after every module in phase N-1 has returned.
|
||||
// - Within a phase, every module runs concurrently on its own goroutine.
|
||||
// - When all phases complete, ScanCompleted is published with stats, then
|
||||
// the bus is drained (if owned) and Shutdown is called.
|
||||
func (p *Pipeline) Run(ctx context.Context) error {
|
||||
selected := p.modReg.Select(p.view)
|
||||
if len(selected) == 0 {
|
||||
return errors.New("pipeline.Run: no modules selected — check config and module registrations")
|
||||
}
|
||||
|
||||
// Group modules by phase.
|
||||
byPhase := make(map[module.Phase][]module.Module)
|
||||
for _, m := range selected {
|
||||
byPhase[m.Phase()] = append(byPhase[m.Phase()], m)
|
||||
}
|
||||
|
||||
// Sort modules within each phase for deterministic start order.
|
||||
for _, ms := range byPhase {
|
||||
sort.SliceStable(ms, func(i, j int) bool { return ms[i].Name() < ms[j].Name() })
|
||||
}
|
||||
|
||||
started := time.Now()
|
||||
p.publishScanStarted()
|
||||
|
||||
var moduleErrs []error
|
||||
var errsMu sync.Mutex
|
||||
|
||||
// Iterate phases in canonical order.
|
||||
for _, phase := range phaseOrder {
|
||||
modules := byPhase[phase]
|
||||
if len(modules) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
phaseStart := time.Now()
|
||||
p.publishPhaseStarted(phase)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for _, m := range modules {
|
||||
m := m
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
p.publishModuleError(m.Name(), fmt.Errorf("panic: %v", r), true)
|
||||
errsMu.Lock()
|
||||
moduleErrs = append(moduleErrs, fmt.Errorf("%s panicked: %v", m.Name(), r))
|
||||
errsMu.Unlock()
|
||||
}
|
||||
}()
|
||||
|
||||
mctx := module.Context{
|
||||
Ctx: ctx,
|
||||
Bus: p.bus,
|
||||
Store: p.store,
|
||||
Config: p.view,
|
||||
Target: p.cfg.Domain,
|
||||
Profile: p.cfg.Profile,
|
||||
}
|
||||
if err := m.Run(mctx); err != nil && !errors.Is(err, context.Canceled) {
|
||||
p.publishModuleError(m.Name(), err, false)
|
||||
errsMu.Lock()
|
||||
moduleErrs = append(moduleErrs, fmt.Errorf("%s: %w", m.Name(), err))
|
||||
errsMu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait for this phase OR for ctx cancellation.
|
||||
done := make(chan struct{})
|
||||
go func() { wg.Wait(); close(done) }()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
// normal completion
|
||||
case <-ctx.Done():
|
||||
// wait (bounded) for goroutines to observe the cancellation
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
p.publishPhaseCompleted(phase, time.Since(phaseStart))
|
||||
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
p.publishScanCompleted(time.Since(started))
|
||||
|
||||
if p.ownBus {
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = p.bus.Close(shutdownCtx)
|
||||
}
|
||||
|
||||
if len(moduleErrs) > 0 {
|
||||
return errors.Join(moduleErrs...)
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// Shutdown explicitly closes owned resources. Normally Run calls Shutdown
|
||||
// automatically; use this when you want to reuse the pipeline or manage
|
||||
// lifecycle externally.
|
||||
func (p *Pipeline) Shutdown(ctx context.Context) error {
|
||||
var errs []error
|
||||
if p.ownBus {
|
||||
if err := p.bus.Close(ctx); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
if p.ownStore {
|
||||
if err := p.store.Close(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// phaseOrder is the canonical sequence of pipeline phases. Modules may also
|
||||
// declare phases not in this list — those are executed at the end in arbitrary
|
||||
// order (but all still before ScanCompleted).
|
||||
var phaseOrder = []module.Phase{
|
||||
module.PhaseSetup,
|
||||
module.PhaseDiscovery,
|
||||
module.PhaseResolution,
|
||||
module.PhaseEnrichment,
|
||||
module.PhaseAnalysis,
|
||||
module.PhaseReporting,
|
||||
}
|
||||
|
||||
// --- event publishing helpers ---
|
||||
|
||||
func (p *Pipeline) publishScanStarted() {
|
||||
p.bus.Publish(context.Background(), eventbus.ScanStarted{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: "pipeline", Target: p.cfg.Domain},
|
||||
Target: p.cfg.Domain,
|
||||
Profile: p.cfg.Profile,
|
||||
})
|
||||
}
|
||||
|
||||
func (p *Pipeline) publishScanCompleted(d time.Duration) {
|
||||
stats := map[string]int64{
|
||||
"hosts": int64(p.store.Count(context.Background())),
|
||||
"published": int64(p.bus.Stats().Published),
|
||||
"delivered": int64(p.bus.Stats().Delivered),
|
||||
"dropped": int64(p.bus.Stats().Dropped),
|
||||
}
|
||||
p.bus.Publish(context.Background(), eventbus.ScanCompleted{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: "pipeline", Target: p.cfg.Domain},
|
||||
Target: p.cfg.Domain,
|
||||
Duration: d,
|
||||
Stats: stats,
|
||||
})
|
||||
}
|
||||
|
||||
func (p *Pipeline) publishPhaseStarted(phase module.Phase) {
|
||||
p.bus.Publish(context.Background(), eventbus.PhaseStarted{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: "pipeline", Target: p.cfg.Domain},
|
||||
Phase: string(phase),
|
||||
})
|
||||
}
|
||||
|
||||
func (p *Pipeline) publishPhaseCompleted(phase module.Phase, d time.Duration) {
|
||||
p.bus.Publish(context.Background(), eventbus.PhaseCompleted{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: "pipeline", Target: p.cfg.Domain},
|
||||
Phase: string(phase),
|
||||
Duration: d,
|
||||
})
|
||||
}
|
||||
|
||||
func (p *Pipeline) publishModuleError(name string, err error, fatal bool) {
|
||||
p.bus.Publish(context.Background(), eventbus.ModuleError{
|
||||
EventMeta: eventbus.EventMeta{At: time.Now(), Source: name, Target: p.cfg.Domain},
|
||||
Module: name,
|
||||
Err: err.Error(),
|
||||
Fatal: fatal,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/config"
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/module"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
// --- test doubles --------------------------------------------------------
|
||||
|
||||
type spyModule struct {
|
||||
name string
|
||||
phase module.Phase
|
||||
run func(mctx module.Context) error
|
||||
calls atomic.Int32
|
||||
enabled bool
|
||||
}
|
||||
|
||||
func (s *spyModule) Name() string { return s.name }
|
||||
func (s *spyModule) Phase() module.Phase { return s.phase }
|
||||
func (s *spyModule) Consumes() []eventbus.EventType { return nil }
|
||||
func (s *spyModule) Produces() []eventbus.EventType { return nil }
|
||||
func (s *spyModule) DefaultEnabled() bool { return s.enabled }
|
||||
func (s *spyModule) Run(mctx module.Context) error {
|
||||
s.calls.Add(1)
|
||||
if s.run != nil {
|
||||
return s.run(mctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func mkModule(name string, phase module.Phase, enabled bool) *spyModule {
|
||||
return &spyModule{name: name, phase: phase, enabled: enabled}
|
||||
}
|
||||
|
||||
func TestPipeline_RunsAllEnabledModules(t *testing.T) {
|
||||
r := module.NewRegistry()
|
||||
a := mkModule("a", module.PhaseDiscovery, true)
|
||||
b := mkModule("b", module.PhaseEnrichment, true)
|
||||
c := mkModule("c", module.PhaseReporting, true)
|
||||
off := mkModule("off", module.PhaseDiscovery, false)
|
||||
r.Register(a)
|
||||
r.Register(b)
|
||||
r.Register(c)
|
||||
r.Register(off)
|
||||
|
||||
cfg := &config.Config{Domain: "example.com"}
|
||||
p, err := New(cfg, Options{Registry: r})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := p.Run(context.Background()); err != nil {
|
||||
t.Fatalf("Run error: %v", err)
|
||||
}
|
||||
|
||||
if a.calls.Load() != 1 {
|
||||
t.Errorf("a not called: %d", a.calls.Load())
|
||||
}
|
||||
if b.calls.Load() != 1 {
|
||||
t.Errorf("b not called")
|
||||
}
|
||||
if c.calls.Load() != 1 {
|
||||
t.Errorf("c not called")
|
||||
}
|
||||
if off.calls.Load() != 0 {
|
||||
t.Errorf("disabled module was called: %d", off.calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipeline_PhaseBarrier(t *testing.T) {
|
||||
// Phase B must see A's events before B's module runs.
|
||||
r := module.NewRegistry()
|
||||
|
||||
var aDone atomic.Bool
|
||||
a := mkModule("producer", module.PhaseDiscovery, true)
|
||||
a.run = func(mctx module.Context) error {
|
||||
mctx.Bus.Publish(mctx.Ctx, eventbus.NewSubdomainDiscovered("test", "x.example.com", "p"))
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
aDone.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
var sawBefore atomic.Int32
|
||||
b := mkModule("consumer", module.PhaseEnrichment, true)
|
||||
b.run = func(mctx module.Context) error {
|
||||
if !aDone.Load() {
|
||||
sawBefore.Add(1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
r.Register(a)
|
||||
r.Register(b)
|
||||
|
||||
p, err := New(&config.Config{Domain: "example.com"}, Options{Registry: r})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := p.Run(context.Background()); err != nil {
|
||||
t.Fatalf("Run error: %v", err)
|
||||
}
|
||||
if sawBefore.Load() != 0 {
|
||||
t.Errorf("phase barrier broken: consumer ran while producer was still running (%d times)", sawBefore.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipeline_CollectsErrors(t *testing.T) {
|
||||
r := module.NewRegistry()
|
||||
good := mkModule("good", module.PhaseDiscovery, true)
|
||||
failA := mkModule("fail-a", module.PhaseDiscovery, true)
|
||||
failA.run = func(_ module.Context) error { return errors.New("boom-a") }
|
||||
failB := mkModule("fail-b", module.PhaseAnalysis, true)
|
||||
failB.run = func(_ module.Context) error { return errors.New("boom-b") }
|
||||
r.Register(good)
|
||||
r.Register(failA)
|
||||
r.Register(failB)
|
||||
|
||||
p, _ := New(&config.Config{Domain: "example.com"}, Options{Registry: r})
|
||||
err := p.Run(context.Background())
|
||||
if err == nil {
|
||||
t.Fatal("expected aggregated error")
|
||||
}
|
||||
if !contains(err.Error(), "boom-a") || !contains(err.Error(), "boom-b") {
|
||||
t.Errorf("aggregated error missing parts: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipeline_PanicIsContained(t *testing.T) {
|
||||
r := module.NewRegistry()
|
||||
panicker := mkModule("panicker", module.PhaseDiscovery, true)
|
||||
panicker.run = func(_ module.Context) error { panic("oops") }
|
||||
r.Register(panicker)
|
||||
r.Register(mkModule("normal", module.PhaseReporting, true))
|
||||
|
||||
p, _ := New(&config.Config{Domain: "example.com"}, Options{Registry: r})
|
||||
err := p.Run(context.Background())
|
||||
if err == nil {
|
||||
t.Fatal("expected error from panic")
|
||||
}
|
||||
if !contains(err.Error(), "panicked") {
|
||||
t.Errorf("error doesn't mention panic: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipeline_RespectsCtxCancellation(t *testing.T) {
|
||||
r := module.NewRegistry()
|
||||
|
||||
slow := mkModule("slow", module.PhaseDiscovery, true)
|
||||
var slowRan atomic.Bool
|
||||
slow.run = func(mctx module.Context) error {
|
||||
slowRan.Store(true)
|
||||
<-mctx.Ctx.Done()
|
||||
return mctx.Ctx.Err()
|
||||
}
|
||||
never := mkModule("never", module.PhaseAnalysis, true)
|
||||
var neverRan atomic.Bool
|
||||
never.run = func(_ module.Context) error {
|
||||
neverRan.Store(true)
|
||||
return nil
|
||||
}
|
||||
r.Register(slow)
|
||||
r.Register(never)
|
||||
|
||||
p, _ := New(&config.Config{Domain: "example.com"}, Options{Registry: r})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go func() {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
cancel()
|
||||
}()
|
||||
|
||||
_ = p.Run(ctx)
|
||||
if !slowRan.Load() {
|
||||
t.Error("slow should have run")
|
||||
}
|
||||
// never is in phase after slow, and phase B starts only after A finishes.
|
||||
// Since slow exits when ctx is canceled, pipeline breaks out before
|
||||
// scheduling phase B. never must NOT run.
|
||||
if neverRan.Load() {
|
||||
t.Error("never should NOT have run after cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipeline_PublishesScanEvents(t *testing.T) {
|
||||
r := module.NewRegistry()
|
||||
r.Register(mkModule("tiny", module.PhaseDiscovery, true))
|
||||
|
||||
p, _ := New(&config.Config{Domain: "example.com"}, Options{Registry: r, Bus: eventbus.New(128)})
|
||||
|
||||
started := make(chan struct{}, 1)
|
||||
completed := make(chan struct{}, 1)
|
||||
|
||||
p.Bus().Subscribe(eventbus.EventScanStarted, func(_ context.Context, _ eventbus.Event) {
|
||||
select {
|
||||
case started <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
})
|
||||
p.Bus().Subscribe(eventbus.EventScanCompleted, func(_ context.Context, _ eventbus.Event) {
|
||||
select {
|
||||
case completed <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
})
|
||||
|
||||
_ = p.Run(context.Background())
|
||||
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("ScanStarted not fired")
|
||||
}
|
||||
select {
|
||||
case <-completed:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("ScanCompleted not fired")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipeline_ModulesShareStore(t *testing.T) {
|
||||
r := module.NewRegistry()
|
||||
|
||||
writer := mkModule("writer", module.PhaseDiscovery, true)
|
||||
writer.run = func(mctx module.Context) error {
|
||||
return mctx.Store.Upsert(mctx.Ctx, "a.example.com", func(h *store.Host) {
|
||||
h.IPs = []string{"1.2.3.4"}
|
||||
})
|
||||
}
|
||||
|
||||
var readerSaw int
|
||||
var readerMu sync.Mutex
|
||||
reader := mkModule("reader", module.PhaseReporting, true)
|
||||
reader.run = func(mctx module.Context) error {
|
||||
readerMu.Lock()
|
||||
defer readerMu.Unlock()
|
||||
readerSaw = mctx.Store.Count(mctx.Ctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
r.Register(writer)
|
||||
r.Register(reader)
|
||||
|
||||
p, _ := New(&config.Config{Domain: "example.com"}, Options{Registry: r})
|
||||
if err := p.Run(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
readerMu.Lock()
|
||||
defer readerMu.Unlock()
|
||||
if readerSaw != 1 {
|
||||
t.Errorf("reader saw %d hosts, want 1", readerSaw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipeline_RejectsNilConfig(t *testing.T) {
|
||||
_, err := New(nil, Options{})
|
||||
if err == nil {
|
||||
t.Error("expected error for nil config")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipeline_EmptyRegistry_Errors(t *testing.T) {
|
||||
r := module.NewRegistry() // empty
|
||||
p, _ := New(&config.Config{Domain: "example.com"}, Options{Registry: r})
|
||||
if err := p.Run(context.Background()); err == nil {
|
||||
t.Error("expected error when no modules selected")
|
||||
}
|
||||
}
|
||||
|
||||
func contains(s, sub string) bool {
|
||||
for i := 0; i+len(sub) <= len(s); i++ {
|
||||
if s[i:i+len(sub)] == sub {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
// Package proxyconf centralises outbound-proxy configuration for the
|
||||
// HTTP and (where possible) DNS clients used across God's Eye modules.
|
||||
//
|
||||
// Why this lives in its own package: every source/probe/module needs to
|
||||
// honour the same proxy setting, and duplicating URL parsing + dialer
|
||||
// wiring across `internal/http`, `internal/sources`, and individual
|
||||
// modules would be a fountain of bugs. This package is the single
|
||||
// source of truth.
|
||||
//
|
||||
// Supported schemes:
|
||||
//
|
||||
// "" → direct (no proxy)
|
||||
// http://host:port → HTTP CONNECT proxy (e.g. Burp, ZAP, mitmproxy)
|
||||
// https://host:port → HTTPS CONNECT proxy
|
||||
// socks5://host:port → SOCKS5 (DNS resolved locally by god-eye)
|
||||
// socks5h://host:port → SOCKS5 (DNS resolved by the proxy — Tor convention)
|
||||
//
|
||||
// Basic auth (http://user:pass@host) is honoured for every scheme.
|
||||
//
|
||||
// DNS-over-SOCKS caveat: Go's net package uses the OS resolver by default,
|
||||
// which does NOT route through SOCKS. `socks5h://` only applies to HTTP
|
||||
// requests — the brute-force DNS resolver (`internal/dns`) continues to
|
||||
// hit its configured resolvers directly. Users who need full Tor
|
||||
// isolation for DNS should run god-eye inside a torsocks-wrapped shell
|
||||
// or a netns with all traffic captured.
|
||||
package proxyconf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/proxy"
|
||||
)
|
||||
|
||||
// DialFunc is the signature used by http.Transport.DialContext.
|
||||
type DialFunc func(ctx context.Context, network, addr string) (net.Conn, error)
|
||||
|
||||
// ProxyFunc is the signature used by http.Transport.Proxy.
|
||||
type ProxyFunc func(*http.Request) (*url.URL, error)
|
||||
|
||||
// Validate returns a descriptive error if proxyURL is non-empty and
|
||||
// doesn't parse to a supported scheme. Call this early (e.g. during
|
||||
// validator.ValidateXxx) so bad flags fail before module startup.
|
||||
func Validate(proxyURL string) error {
|
||||
proxyURL = strings.TrimSpace(proxyURL)
|
||||
if proxyURL == "" {
|
||||
return nil
|
||||
}
|
||||
u, err := url.Parse(proxyURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("proxy URL malformed: %w", err)
|
||||
}
|
||||
if u.Host == "" {
|
||||
return errors.New("proxy URL missing host:port")
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https", "socks5", "socks5h":
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("unsupported proxy scheme %q (use http/https/socks5/socks5h)", u.Scheme)
|
||||
}
|
||||
}
|
||||
|
||||
// BuildDialer returns a DialFunc that routes TCP through the configured
|
||||
// proxy. For HTTP(S) CONNECT proxies (handled at the transport layer via
|
||||
// Proxy field), this returns a direct dialer — the transport layer does
|
||||
// the CONNECT dance itself.
|
||||
//
|
||||
// For empty proxyURL, returns the direct-dialer from net.Dialer.
|
||||
func BuildDialer(proxyURL string, base *net.Dialer) (DialFunc, error) {
|
||||
if base == nil {
|
||||
base = &net.Dialer{}
|
||||
}
|
||||
if strings.TrimSpace(proxyURL) == "" {
|
||||
return base.DialContext, nil
|
||||
}
|
||||
u, err := url.Parse(proxyURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https":
|
||||
// CONNECT proxy — direct TCP, Transport.Proxy handles the handshake.
|
||||
return base.DialContext, nil
|
||||
case "socks5", "socks5h":
|
||||
var auth *proxy.Auth
|
||||
if u.User != nil {
|
||||
pass, _ := u.User.Password()
|
||||
auth = &proxy.Auth{User: u.User.Username(), Password: pass}
|
||||
}
|
||||
// proxy.Direct is the fallthrough dialer — we pass our base so
|
||||
// timeouts/keepalive settings are preserved.
|
||||
dialer, err := proxy.SOCKS5("tcp", u.Host, auth, &directAdapter{base: base})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create SOCKS5 dialer: %w", err)
|
||||
}
|
||||
if ctxDialer, ok := dialer.(proxy.ContextDialer); ok {
|
||||
return ctxDialer.DialContext, nil
|
||||
}
|
||||
// Older x/net versions: wrap non-context Dial with ctx-aware shim.
|
||||
return func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
type result struct {
|
||||
conn net.Conn
|
||||
err error
|
||||
}
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
c, e := dialer.Dial(network, addr)
|
||||
ch <- result{c, e}
|
||||
}()
|
||||
select {
|
||||
case r := <-ch:
|
||||
return r.conn, r.err
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported proxy scheme: %s", u.Scheme)
|
||||
}
|
||||
}
|
||||
|
||||
// BuildProxyFunc returns the http.Transport.Proxy callback for HTTP(S)
|
||||
// CONNECT proxies. Returns nil for SOCKS5 (handled by the dialer) and
|
||||
// for empty proxyURL.
|
||||
func BuildProxyFunc(proxyURL string) (ProxyFunc, error) {
|
||||
if strings.TrimSpace(proxyURL) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
u, err := url.Parse(proxyURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https":
|
||||
return http.ProxyURL(u), nil
|
||||
case "socks5", "socks5h":
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported proxy scheme: %s", u.Scheme)
|
||||
}
|
||||
|
||||
// Humanize returns a redacted, user-facing description of the proxy.
|
||||
// Strips credentials so logs don't leak tokens.
|
||||
func Humanize(proxyURL string) string {
|
||||
proxyURL = strings.TrimSpace(proxyURL)
|
||||
if proxyURL == "" {
|
||||
return "direct (no proxy)"
|
||||
}
|
||||
u, err := url.Parse(proxyURL)
|
||||
if err != nil {
|
||||
return "invalid"
|
||||
}
|
||||
auth := ""
|
||||
if u.User != nil {
|
||||
auth = "(auth)@"
|
||||
}
|
||||
return fmt.Sprintf("%s://%s%s", u.Scheme, auth, u.Host)
|
||||
}
|
||||
|
||||
// directAdapter adapts a *net.Dialer to the proxy.Dialer interface so
|
||||
// our configured timeouts/keepalive flow through to the socks hop.
|
||||
type directAdapter struct {
|
||||
base *net.Dialer
|
||||
}
|
||||
|
||||
func (d *directAdapter) Dial(network, addr string) (net.Conn, error) {
|
||||
return d.base.Dial(network, addr)
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package proxyconf
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestValidate(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
wantErr bool
|
||||
}{
|
||||
{"", false},
|
||||
{"http://127.0.0.1:8080", false},
|
||||
{"https://proxy.corp:3128", false},
|
||||
{"socks5://127.0.0.1:9050", false},
|
||||
{"socks5h://127.0.0.1:9050", false},
|
||||
{"socks5h://user:pass@127.0.0.1:9050", false},
|
||||
{"ftp://x:21", true},
|
||||
{"socks4://x:1080", true},
|
||||
{"not a url", true},
|
||||
{"://nohost", true},
|
||||
{"http://", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
err := Validate(c.in)
|
||||
if (err != nil) != c.wantErr {
|
||||
t.Errorf("Validate(%q) err=%v wantErr=%v", c.in, err, c.wantErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDialer_EmptyReturnsDirect(t *testing.T) {
|
||||
d, err := BuildDialer("", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d == nil {
|
||||
t.Fatal("nil dialer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDialer_SOCKS5Accepted(t *testing.T) {
|
||||
d, err := BuildDialer("socks5://127.0.0.1:9050", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("SOCKS5 should construct: %v", err)
|
||||
}
|
||||
if d == nil {
|
||||
t.Fatal("nil dialer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDialer_SOCKS5WithAuth(t *testing.T) {
|
||||
d, err := BuildDialer("socks5h://user:pass@127.0.0.1:9050", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("auth SOCKS5 should construct: %v", err)
|
||||
}
|
||||
if d == nil {
|
||||
t.Fatal("nil dialer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDialer_HTTPProxyPassthrough(t *testing.T) {
|
||||
// HTTP proxy uses Transport.Proxy; dialer should be direct-equivalent.
|
||||
d, err := BuildDialer("http://127.0.0.1:8080", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d == nil {
|
||||
t.Fatal("nil dialer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDialer_UnsupportedScheme(t *testing.T) {
|
||||
_, err := BuildDialer("ftp://127.0.0.1", nil)
|
||||
if err == nil {
|
||||
t.Error("expected error for unsupported scheme")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildProxyFunc_HTTPProxy(t *testing.T) {
|
||||
fn, err := BuildProxyFunc("http://127.0.0.1:8080")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fn == nil {
|
||||
t.Fatal("http:// should yield non-nil ProxyFunc")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildProxyFunc_SOCKSReturnsNil(t *testing.T) {
|
||||
fn, err := BuildProxyFunc("socks5://127.0.0.1:9050")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fn != nil {
|
||||
t.Error("SOCKS5 should return nil ProxyFunc (handled by dialer)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildProxyFunc_EmptyReturnsNil(t *testing.T) {
|
||||
fn, err := BuildProxyFunc("")
|
||||
if err != nil || fn != nil {
|
||||
t.Errorf("empty → (nil, nil), got (%v, %v)", fn, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHumanize(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"": "direct (no proxy)",
|
||||
"http://proxy.corp:3128": "http://proxy.corp:3128",
|
||||
"socks5://127.0.0.1:9050": "socks5://127.0.0.1:9050",
|
||||
"socks5h://user:secret@10.0.0.1:443": "socks5h://(auth)@10.0.0.1:443",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := Humanize(in); got != want {
|
||||
t.Errorf("Humanize(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHumanize_LeaksNoCredentials(t *testing.T) {
|
||||
const secret = "supersecret"
|
||||
h := Humanize("socks5://user:" + secret + "@127.0.0.1:9050")
|
||||
if contains(h, secret) {
|
||||
t.Errorf("Humanize leaked credentials: %s", h)
|
||||
}
|
||||
}
|
||||
|
||||
func contains(s, sub string) bool {
|
||||
for i := 0; i+len(sub) <= len(s); i++ {
|
||||
if s[i:i+len(sub)] == sub {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"god-eye/internal/config"
|
||||
)
|
||||
|
||||
func TestLoadWordlist(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "wordlist.txt")
|
||||
|
||||
content := `# comment line
|
||||
api
|
||||
admin
|
||||
|
||||
# another comment
|
||||
dev
|
||||
staging
|
||||
test
|
||||
`
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := LoadWordlist(path)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"api", "admin", "dev", "staging", "test"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadWordlist_NonExistent(t *testing.T) {
|
||||
_, err := LoadWordlist("/tmp/this-does-not-exist-xyz-abc.txt")
|
||||
if err == nil {
|
||||
t.Error("expected error for non-existent file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadWordlist_Empty(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "empty.txt")
|
||||
os.WriteFile(path, []byte(""), 0o644)
|
||||
|
||||
got, err := LoadWordlist(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Errorf("expected empty result, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadWordlist_CommentsOnly(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "comments.txt")
|
||||
os.WriteFile(path, []byte("# only comments\n# and more\n"), 0o644)
|
||||
|
||||
got, _ := LoadWordlist(path)
|
||||
if len(got) != 0 {
|
||||
t.Errorf("expected empty result for comments-only file, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseResolvers(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "empty uses defaults",
|
||||
in: "",
|
||||
want: config.DefaultResolvers,
|
||||
},
|
||||
{
|
||||
name: "single with port",
|
||||
in: "8.8.8.8:53",
|
||||
want: []string{"8.8.8.8:53"},
|
||||
},
|
||||
{
|
||||
name: "single without port adds :53",
|
||||
in: "8.8.8.8",
|
||||
want: []string{"8.8.8.8:53"},
|
||||
},
|
||||
{
|
||||
name: "multiple with mixed ports",
|
||||
in: "8.8.8.8,1.1.1.1:5353,9.9.9.9",
|
||||
want: []string{"8.8.8.8:53", "1.1.1.1:5353", "9.9.9.9:53"},
|
||||
},
|
||||
{
|
||||
name: "whitespace trimmed",
|
||||
in: " 8.8.8.8 , 1.1.1.1 ",
|
||||
want: []string{"8.8.8.8:53", "1.1.1.1:53"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ParseResolvers(tt.in)
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("ParseResolvers(%q) = %v, want %v", tt.in, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePorts(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want []int
|
||||
}{
|
||||
{"empty uses defaults", "", []int{80, 443, 8080, 8443}},
|
||||
{"single valid", "80", []int{80}},
|
||||
{"multiple valid", "80,443,3000", []int{80, 443, 3000}},
|
||||
{"whitespace", " 80 , 443 ", []int{80, 443}},
|
||||
{"invalid silently dropped", "80,abc,443", []int{80, 443}},
|
||||
{"out of range dropped", "80,99999,443", []int{80, 443}},
|
||||
{"negative dropped", "80,-1,443", []int{80, 443}},
|
||||
{"zero dropped", "0,80,443", []int{80, 443}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ParsePorts(tt.in)
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("ParsePorts(%q) = %v, want %v", tt.in, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountActive(t *testing.T) {
|
||||
results := map[string]*config.SubdomainResult{
|
||||
"a.example.com": {StatusCode: 200},
|
||||
"b.example.com": {StatusCode: 301},
|
||||
"c.example.com": {StatusCode: 404},
|
||||
"d.example.com": {StatusCode: 500},
|
||||
"e.example.com": {StatusCode: 0}, // not probed
|
||||
}
|
||||
got := countActive(results)
|
||||
if got != 2 {
|
||||
t.Errorf("countActive = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountVulns(t *testing.T) {
|
||||
results := map[string]*config.SubdomainResult{
|
||||
"a.example.com": {OpenRedirect: true},
|
||||
"b.example.com": {CORSMisconfig: "wildcard with credentials"},
|
||||
"c.example.com": {DangerousMethods: []string{"PUT", "DELETE"}},
|
||||
"d.example.com": {GitExposed: true},
|
||||
"e.example.com": {BackupFiles: []string{"backup.sql"}},
|
||||
"f.example.com": {StatusCode: 200}, // clean
|
||||
}
|
||||
got := countVulns(results)
|
||||
if got != 5 {
|
||||
t.Errorf("countVulns = %d, want 5", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountSubdomainsWithAI(t *testing.T) {
|
||||
results := map[string]*config.SubdomainResult{
|
||||
"a.example.com": {AIFindings: []string{"finding1"}},
|
||||
"b.example.com": {AIFindings: []string{"f1", "f2"}},
|
||||
"c.example.com": {}, // no AI findings
|
||||
}
|
||||
got := countSubdomainsWithAI(results)
|
||||
if got != 2 {
|
||||
t.Errorf("countSubdomainsWithAI = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildAISummary(t *testing.T) {
|
||||
results := map[string]*config.SubdomainResult{
|
||||
"a.example.com": {
|
||||
AIFindings: []string{"Hardcoded API key", "Weak crypto"},
|
||||
AISeverity: "critical",
|
||||
CVEFindings: []string{"CVE-2021-12345"},
|
||||
},
|
||||
"b.example.com": {
|
||||
AIFindings: []string{"Missing CSP"},
|
||||
AISeverity: "medium",
|
||||
},
|
||||
"c.example.com": {
|
||||
AIFindings: []string{"ignored"},
|
||||
AISeverity: "info",
|
||||
},
|
||||
}
|
||||
got := buildAISummary(results)
|
||||
if got == "" {
|
||||
t.Fatal("summary is empty")
|
||||
}
|
||||
// Must mention severities
|
||||
mustContain := []string{"critical", "high", "medium", "CRITICAL", "MEDIUM", "Hardcoded API key", "CVE-2021-12345"}
|
||||
for _, s := range mustContain {
|
||||
if !strings.Contains(got, s) {
|
||||
t.Errorf("summary missing expected token %q in:\n%s", s, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortedIntsInvariant(t *testing.T) {
|
||||
// Sanity: whenever we sort ints we expect ascending order (tests ScanPorts sorting guarantee).
|
||||
in := []int{443, 80, 8080, 22}
|
||||
sort.Ints(in)
|
||||
if !sort.IntsAreSorted(in) {
|
||||
t.Error("sort.IntsAreSorted returned false after sort.Ints")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/diff"
|
||||
)
|
||||
|
||||
// WebhookAlerter POSTs the diff report JSON to an arbitrary URL. Works
|
||||
// with generic webhook consumers; Slack/Discord get dedicated adapters
|
||||
// later in F5.3 when bespoke formatting matters.
|
||||
type WebhookAlerter struct {
|
||||
URL string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// NewWebhookAlerter returns a WebhookAlerter with sane defaults.
|
||||
func NewWebhookAlerter(url string) *WebhookAlerter {
|
||||
return &WebhookAlerter{URL: url, Timeout: 10 * time.Second}
|
||||
}
|
||||
|
||||
func (a *WebhookAlerter) Name() string { return "webhook" }
|
||||
|
||||
func (a *WebhookAlerter) Notify(ctx context.Context, r *diff.Report) error {
|
||||
body, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client := &http.Client{Timeout: a.Timeout}
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", a.URL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", "god-eye-v2")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
return fmt.Errorf("webhook returned %d", resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StdoutAlerter prints meaningful changes to stdout. Useful for smoke
|
||||
// testing and for users who pipe god-eye output into grep/jq.
|
||||
type StdoutAlerter struct{}
|
||||
|
||||
func (StdoutAlerter) Name() string { return "stdout" }
|
||||
|
||||
func (StdoutAlerter) Notify(_ context.Context, r *diff.Report) error {
|
||||
for _, c := range r.Changes {
|
||||
fmt.Printf("[DIFF %s] %s %s → %s (%s)\n", r.Target, c.Kind, c.Before, c.After, c.Host)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
// Package scheduler runs a scan at fixed intervals for asm-continuous
|
||||
// workflows. Each scan run feeds the diff engine; meaningful changes fan
|
||||
// out to registered Alerters.
|
||||
//
|
||||
// Minimal implementation for Fase 5 skeleton: interval ticker + in-memory
|
||||
// snapshot ring. Persistence (SQLite/BoltDB) and sophisticated scheduling
|
||||
// (cron syntax, jitter) are follow-ups.
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/diff"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
// ScanRun executes a single scan and returns the snapshot hosts.
|
||||
type ScanRun func(ctx context.Context) (hosts []*store.Host, err error)
|
||||
|
||||
// Alerter receives diff reports with meaningful changes.
|
||||
type Alerter interface {
|
||||
Notify(ctx context.Context, report *diff.Report) error
|
||||
Name() string
|
||||
}
|
||||
|
||||
// Scheduler runs ScanRun on an interval.
|
||||
type Scheduler struct {
|
||||
Target string
|
||||
Interval time.Duration
|
||||
Run ScanRun
|
||||
Alerters []Alerter
|
||||
|
||||
mu sync.Mutex
|
||||
lastSnap []*store.Host
|
||||
lastAt time.Time
|
||||
}
|
||||
|
||||
// New constructs a scheduler. Every field is required except Alerters,
|
||||
// which defaults to nil (no notifications).
|
||||
func New(target string, interval time.Duration, run ScanRun) *Scheduler {
|
||||
return &Scheduler{Target: target, Interval: interval, Run: run}
|
||||
}
|
||||
|
||||
// AddAlerter registers an Alerter that receives meaningful diff reports.
|
||||
func (s *Scheduler) AddAlerter(a Alerter) { s.Alerters = append(s.Alerters, a) }
|
||||
|
||||
// Start runs indefinitely until ctx is canceled. The first scan runs
|
||||
// immediately, subsequent scans run on s.Interval cadence.
|
||||
func (s *Scheduler) Start(ctx context.Context) error {
|
||||
if s.Run == nil {
|
||||
return errors.New("scheduler: nil Run")
|
||||
}
|
||||
if s.Interval <= 0 {
|
||||
return errors.New("scheduler: Interval must be > 0")
|
||||
}
|
||||
|
||||
// First scan now (so continuous mode produces something immediately).
|
||||
s.runOnce(ctx)
|
||||
|
||||
t := time.NewTicker(s.Interval)
|
||||
defer t.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-t.C:
|
||||
s.runOnce(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scheduler) runOnce(ctx context.Context) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
hosts, err := s.Run(ctx)
|
||||
if err != nil {
|
||||
// Scan failure is non-fatal for the scheduler itself; the next
|
||||
// tick will try again.
|
||||
return
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
prev := s.lastSnap
|
||||
prevAt := s.lastAt
|
||||
s.lastSnap = hosts
|
||||
s.lastAt = time.Now()
|
||||
s.mu.Unlock()
|
||||
|
||||
// No diff possible on the first run.
|
||||
if prev == nil {
|
||||
return
|
||||
}
|
||||
|
||||
report := diff.Compute(s.Target, prev, hosts, prevAt, time.Now())
|
||||
if !report.HasMeaningful() {
|
||||
return
|
||||
}
|
||||
for _, a := range s.Alerters {
|
||||
_ = a.Notify(ctx, report)
|
||||
}
|
||||
}
|
||||
|
||||
// LastSnapshot returns the most recent scan snapshot + timestamp. Returns
|
||||
// (nil, zero) before the first scan.
|
||||
func (s *Scheduler) LastSnapshot() ([]*store.Host, time.Time) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.lastSnap, s.lastAt
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/diff"
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
type spyAlerter struct{ called atomic.Int32 }
|
||||
|
||||
func (s *spyAlerter) Name() string { return "spy" }
|
||||
func (s *spyAlerter) Notify(_ context.Context, _ *diff.Report) error {
|
||||
s.called.Add(1)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestScheduler_RunsAndDiffsBetweenScans(t *testing.T) {
|
||||
var callCount atomic.Int32
|
||||
scan := ScanRun(func(_ context.Context) ([]*store.Host, error) {
|
||||
n := callCount.Add(1)
|
||||
if n == 1 {
|
||||
return []*store.Host{{Subdomain: "a.example.com"}}, nil
|
||||
}
|
||||
// Second scan adds a new host — meaningful diff.
|
||||
return []*store.Host{
|
||||
{Subdomain: "a.example.com"},
|
||||
{Subdomain: "b.example.com"},
|
||||
}, nil
|
||||
})
|
||||
|
||||
s := New("example.com", 100*time.Millisecond, scan)
|
||||
alerter := &spyAlerter{}
|
||||
s.AddAlerter(alerter)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
|
||||
defer cancel()
|
||||
_ = s.Start(ctx)
|
||||
|
||||
if callCount.Load() < 2 {
|
||||
t.Errorf("scan should have run at least twice, got %d", callCount.Load())
|
||||
}
|
||||
if alerter.called.Load() == 0 {
|
||||
t.Error("alerter should have been called on the second run")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduler_NoAlertOnIdenticalScans(t *testing.T) {
|
||||
scan := ScanRun(func(_ context.Context) ([]*store.Host, error) {
|
||||
return []*store.Host{{Subdomain: "a.example.com"}}, nil
|
||||
})
|
||||
s := New("example.com", 50*time.Millisecond, scan)
|
||||
alerter := &spyAlerter{}
|
||||
s.AddAlerter(alerter)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
|
||||
defer cancel()
|
||||
_ = s.Start(ctx)
|
||||
|
||||
if alerter.called.Load() != 0 {
|
||||
t.Errorf("alerter should not have been called on unchanged scans, got %d", alerter.called.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduler_RejectsBadParams(t *testing.T) {
|
||||
s := &Scheduler{Target: "x", Interval: 0}
|
||||
if err := s.Start(context.Background()); err == nil {
|
||||
t.Error("expected error for zero interval")
|
||||
}
|
||||
|
||||
s2 := &Scheduler{Target: "x", Interval: time.Second, Run: nil}
|
||||
if err := s2.Start(context.Background()); err == nil {
|
||||
t.Error("expected error for nil Run")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
// Additional passive sources added in v2.0 to close the gap with
|
||||
// subfinder / BBOT. Every source here is:
|
||||
// - Free and key-less (no API key required)
|
||||
// - Defensive (fail-open — returns an empty slice on any error)
|
||||
// - Bounded by the shared HTTP clients
|
||||
//
|
||||
// If a source goes offline upstream, the corresponding fetcher keeps
|
||||
// returning empty — the scan still succeeds.
|
||||
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// FetchOmnisint queries the free Omnisint Sonar mirror. It may be offline
|
||||
// on any given day — fail-open.
|
||||
func FetchOmnisint(domain string) ([]string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
u := fmt.Sprintf("https://sonar.omnisint.io/subdomains/%s", url.PathEscape(domain))
|
||||
req, _ := http.NewRequestWithContext(ctx, "GET", u, nil)
|
||||
req.Header.Set("User-Agent", "god-eye-v2")
|
||||
|
||||
resp, err := StandardClient.Do(req)
|
||||
if err != nil {
|
||||
return []string{}, nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return []string{}, nil
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024))
|
||||
if err != nil {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
var list []string
|
||||
if err := json.Unmarshal(body, &list); err != nil {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
seen := make(map[string]bool)
|
||||
var out []string
|
||||
for _, s := range list {
|
||||
s = strings.ToLower(strings.TrimSpace(s))
|
||||
if s != "" && strings.HasSuffix(s, domain) && !seen[s] {
|
||||
seen[s] = true
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// FetchHudsonRock queries the free Cavalier InfoStealer intelligence API.
|
||||
// Surfaces domain assets referenced in leaked stealer logs; useful for
|
||||
// discovering shadow internal hostnames.
|
||||
func FetchHudsonRock(domain string) ([]string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
u := fmt.Sprintf("https://cavalier.hudsonrock.com/api/json/v2/osint-tools/search-by-domain?domain=%s", url.QueryEscape(domain))
|
||||
req, _ := http.NewRequestWithContext(ctx, "GET", u, nil)
|
||||
req.Header.Set("User-Agent", "god-eye-v2")
|
||||
|
||||
resp, err := StandardClient.Do(req)
|
||||
if err != nil {
|
||||
return []string{}, nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return []string{}, nil
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024))
|
||||
if err != nil {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
// HudsonRock returns free-form JSON; we just mine every subdomain-like
|
||||
// token from the response body via the shared regex.
|
||||
return ExtractSubdomains(string(body), domain), nil
|
||||
}
|
||||
|
||||
// FetchWebArchiveCDX queries the Internet Archive CDX server — a richer
|
||||
// variant of the existing Wayback source. Pulls URLs with fewer limits
|
||||
// and extracts hostnames that match the target domain.
|
||||
func FetchWebArchiveCDX(domain string) ([]string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
u := fmt.Sprintf("https://web.archive.org/cdx/search/cdx?url=*.%s/*&output=json&collapse=urlkey&limit=5000&fl=original", url.QueryEscape(domain))
|
||||
req, _ := http.NewRequestWithContext(ctx, "GET", u, nil)
|
||||
req.Header.Set("User-Agent", "god-eye-v2")
|
||||
|
||||
resp, err := SlowClient.Do(req)
|
||||
if err != nil {
|
||||
return []string{}, nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return []string{}, nil
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 16*1024*1024))
|
||||
if err != nil {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
// Response shape: [["original"], ["url1"], ["url2"], ...] — first row
|
||||
// is the header, subsequent rows are single-element arrays with the URL.
|
||||
var rows [][]string
|
||||
if err := json.Unmarshal(body, &rows); err != nil {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
seen := make(map[string]bool)
|
||||
var out []string
|
||||
for i, row := range rows {
|
||||
if i == 0 { // skip header
|
||||
continue
|
||||
}
|
||||
if len(row) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, host := range ExtractSubdomains(row[0], domain) {
|
||||
if !seen[host] {
|
||||
seen[host] = true
|
||||
out = append(out, host)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// FetchDigitorus queries the free Digitorus CT log mirror — an alternative
|
||||
// to crt.sh that sometimes returns fresher data.
|
||||
func FetchDigitorus(domain string) ([]string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
|
||||
u := fmt.Sprintf("https://certificatedetails.com/api/find/%s", url.QueryEscape(domain))
|
||||
req, _ := http.NewRequestWithContext(ctx, "GET", u, nil)
|
||||
req.Header.Set("User-Agent", "god-eye-v2")
|
||||
|
||||
resp, err := StandardClient.Do(req)
|
||||
if err != nil {
|
||||
return []string{}, nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return []string{}, nil
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 4*1024*1024))
|
||||
if err != nil {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
// Free-form JSON; mine hostnames.
|
||||
return ExtractSubdomains(string(body), domain), nil
|
||||
}
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/proxyconf"
|
||||
)
|
||||
|
||||
// Shared HTTP clients - singleton pattern
|
||||
@@ -50,6 +52,65 @@ func init() {
|
||||
initRegex()
|
||||
}
|
||||
|
||||
// SetProxy configures outbound proxy for every shared HTTP client used
|
||||
// by passive sources. Must be called BEFORE any Fetch* source function
|
||||
// runs (init runs on package import, so main.go calls this after flag
|
||||
// parsing but before pipeline start, which triggers a re-init via
|
||||
// ReinitClients).
|
||||
func SetProxy(u string) error {
|
||||
if err := proxyconf.Validate(u); err != nil {
|
||||
return err
|
||||
}
|
||||
proxyMu.Lock()
|
||||
proxyURL = u
|
||||
proxyMu.Unlock()
|
||||
// Rebuild transports to pick up the new proxy.
|
||||
reinitClients()
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
proxyURL string
|
||||
proxyMu sync.RWMutex
|
||||
)
|
||||
|
||||
// reinitClients rebuilds the shared transport and clients. Safe to call
|
||||
// multiple times; in practice only called from SetProxy after startup.
|
||||
func reinitClients() {
|
||||
proxyMu.RLock()
|
||||
cfgProxy := proxyURL
|
||||
proxyMu.RUnlock()
|
||||
|
||||
baseDialer := &net.Dialer{
|
||||
Timeout: 10 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}
|
||||
dialCtx, err := proxyconf.BuildDialer(cfgProxy, baseDialer)
|
||||
if err != nil {
|
||||
dialCtx = baseDialer.DialContext
|
||||
}
|
||||
proxyFunc, _ := proxyconf.BuildProxyFunc(cfgProxy)
|
||||
|
||||
sharedTransport = &http.Transport{
|
||||
DialContext: dialCtx,
|
||||
Proxy: proxyFunc,
|
||||
MaxIdleConns: 100,
|
||||
MaxIdleConnsPerHost: 10,
|
||||
MaxConnsPerHost: 20,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
TLSClientConfig: &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
},
|
||||
ForceAttemptHTTP2: true,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
}
|
||||
|
||||
FastClient = &http.Client{Transport: sharedTransport, Timeout: 10 * time.Second}
|
||||
StandardClient = &http.Client{Transport: sharedTransport, Timeout: 15 * time.Second}
|
||||
SlowClient = &http.Client{Transport: sharedTransport, Timeout: 120 * time.Second}
|
||||
}
|
||||
|
||||
func initClients() {
|
||||
clientOnce.Do(func() {
|
||||
// Shared transport with connection pooling
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestExtractSubdomains(t *testing.T) {
|
||||
target := "example.com"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
text string
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "empty text",
|
||||
text: "",
|
||||
want: nil,
|
||||
},
|
||||
{
|
||||
name: "no matches",
|
||||
text: "some text with no domains",
|
||||
want: nil,
|
||||
},
|
||||
{
|
||||
name: "apex only",
|
||||
text: "found example.com here",
|
||||
want: []string{"example.com"},
|
||||
},
|
||||
{
|
||||
name: "single subdomain",
|
||||
text: "api.example.com was found",
|
||||
want: []string{"api.example.com"},
|
||||
},
|
||||
{
|
||||
name: "multiple subdomains",
|
||||
text: "api.example.com and admin.example.com and dev.example.com",
|
||||
want: []string{"admin.example.com", "api.example.com", "dev.example.com"},
|
||||
},
|
||||
{
|
||||
name: "deduplication",
|
||||
text: "api.example.com api.example.com api.example.com",
|
||||
want: []string{"api.example.com"},
|
||||
},
|
||||
{
|
||||
name: "uppercase normalized",
|
||||
text: "API.EXAMPLE.COM and Api.Example.com",
|
||||
want: []string{"api.example.com"},
|
||||
},
|
||||
{
|
||||
name: "wildcard prefix stripped",
|
||||
text: "*.example.com is a wildcard",
|
||||
want: []string{"example.com"},
|
||||
},
|
||||
{
|
||||
name: "different domain filtered",
|
||||
text: "api.example.com and other.different.org and sub.example.com",
|
||||
want: []string{"api.example.com", "sub.example.com"},
|
||||
},
|
||||
{
|
||||
name: "partial match not allowed",
|
||||
text: "evilexample.com should not match",
|
||||
want: nil,
|
||||
},
|
||||
{
|
||||
name: "json-wrapped",
|
||||
text: `{"name":"api.example.com","type":"A"}`,
|
||||
want: []string{"api.example.com"},
|
||||
},
|
||||
{
|
||||
name: "mixed with urls",
|
||||
text: `Visit https://api.example.com and https://docs.example.com/path`,
|
||||
want: []string{"api.example.com", "docs.example.com"},
|
||||
},
|
||||
{
|
||||
// Regex is greedy: only the longest leftmost match is returned,
|
||||
// not every suffix. This is the v1 baseline behavior.
|
||||
name: "deep subdomain longest match only",
|
||||
text: "a.b.c.example.com",
|
||||
want: []string{"a.b.c.example.com"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ExtractSubdomains(tt.text, target)
|
||||
sort.Strings(got)
|
||||
sort.Strings(tt.want)
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("ExtractSubdomains(%q)\n got: %v\n want: %v", tt.text, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetClientForTimeout(t *testing.T) {
|
||||
tests := []struct {
|
||||
timeout time.Duration
|
||||
want string // identify by Timeout field
|
||||
}{
|
||||
{5 * time.Second, "fast"},
|
||||
{10 * time.Second, "fast"},
|
||||
{15 * time.Second, "standard"},
|
||||
{30 * time.Second, "standard"},
|
||||
{60 * time.Second, "slow"},
|
||||
{120 * time.Second, "slow"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
c := GetClientForTimeout(tt.timeout)
|
||||
if c == nil {
|
||||
t.Fatalf("GetClientForTimeout(%v) returned nil", tt.timeout)
|
||||
}
|
||||
var gotClient string
|
||||
switch c {
|
||||
case FastClient:
|
||||
gotClient = "fast"
|
||||
case StandardClient:
|
||||
gotClient = "standard"
|
||||
case SlowClient:
|
||||
gotClient = "slow"
|
||||
default:
|
||||
gotClient = "unknown"
|
||||
}
|
||||
if gotClient != tt.want {
|
||||
t.Errorf("GetClientForTimeout(%v) = %s, want %s", tt.timeout, gotClient, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientsInitialized(t *testing.T) {
|
||||
if FastClient == nil {
|
||||
t.Error("FastClient is nil")
|
||||
}
|
||||
if StandardClient == nil {
|
||||
t.Error("StandardClient is nil")
|
||||
}
|
||||
if SlowClient == nil {
|
||||
t.Error("SlowClient is nil")
|
||||
}
|
||||
if FastClient.Timeout != 10*time.Second {
|
||||
t.Errorf("FastClient.Timeout = %v, want 10s", FastClient.Timeout)
|
||||
}
|
||||
if StandardClient.Timeout != 15*time.Second {
|
||||
t.Errorf("StandardClient.Timeout = %v, want 15s", StandardClient.Timeout)
|
||||
}
|
||||
if SlowClient.Timeout != 120*time.Second {
|
||||
t.Errorf("SlowClient.Timeout = %v, want 120s", SlowClient.Timeout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegexCompiled(t *testing.T) {
|
||||
if SubdomainRegex == nil {
|
||||
t.Error("SubdomainRegex not compiled")
|
||||
}
|
||||
if EmailDomainRegex == nil {
|
||||
t.Error("EmailDomainRegex not compiled")
|
||||
}
|
||||
if URLDomainRegex == nil {
|
||||
t.Error("URLDomainRegex not compiled")
|
||||
}
|
||||
if JSONSubdomainRegex == nil {
|
||||
t.Error("JSONSubdomainRegex not compiled")
|
||||
}
|
||||
if WildcardPrefixRegex == nil {
|
||||
t.Error("WildcardPrefixRegex not compiled")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MemoryStore is the default in-memory Store implementation. Thread-safe,
|
||||
// suitable for single-process scans. Persistent backends (BoltDB for ASM /
|
||||
// resume workflows) land in Fase 5; they will implement the same Store
|
||||
// interface so callers need no changes.
|
||||
type MemoryStore struct {
|
||||
mu sync.RWMutex
|
||||
hosts map[string]*Host
|
||||
// perHostLocks serializes Upsert mutations per-host without blocking
|
||||
// independent hosts. It's populated lazily and never cleared — the number
|
||||
// of subdomains per scan is bounded (thousands, not millions).
|
||||
perHostLocks map[string]*sync.Mutex
|
||||
locksMu sync.Mutex
|
||||
}
|
||||
|
||||
// NewMemoryStore creates an empty MemoryStore.
|
||||
func NewMemoryStore() *MemoryStore {
|
||||
return &MemoryStore{
|
||||
hosts: make(map[string]*Host),
|
||||
perHostLocks: make(map[string]*sync.Mutex),
|
||||
}
|
||||
}
|
||||
|
||||
// lockFor returns the mutex that protects mutations to subdomain, creating
|
||||
// it lazily if needed.
|
||||
func (s *MemoryStore) lockFor(subdomain string) *sync.Mutex {
|
||||
s.locksMu.Lock()
|
||||
defer s.locksMu.Unlock()
|
||||
l, ok := s.perHostLocks[subdomain]
|
||||
if !ok {
|
||||
l = &sync.Mutex{}
|
||||
s.perHostLocks[subdomain] = l
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
// Upsert creates or updates the record for subdomain, invoking mutate under
|
||||
// a per-host lock. Concurrent callers mutating different subdomains proceed
|
||||
// in parallel; concurrent mutations of the same subdomain are serialized.
|
||||
func (s *MemoryStore) Upsert(ctx context.Context, subdomain string, mutate func(*Host)) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if subdomain == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
hostLock := s.lockFor(subdomain)
|
||||
hostLock.Lock()
|
||||
defer hostLock.Unlock()
|
||||
|
||||
s.mu.Lock()
|
||||
h, existed := s.hosts[subdomain]
|
||||
if !existed {
|
||||
h = &Host{
|
||||
Subdomain: subdomain,
|
||||
FirstSeen: time.Now(),
|
||||
}
|
||||
s.hosts[subdomain] = h
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
if mutate != nil {
|
||||
mutate(h)
|
||||
}
|
||||
h.LastUpdated = time.Now()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get returns a deep-enough copy of the record so the caller cannot
|
||||
// accidentally mutate store state. Slice fields are copied; nested struct
|
||||
// pointers (TLSFingerprint, Takeover) are shallow-copied — callers MUST treat
|
||||
// the result as read-only.
|
||||
func (s *MemoryStore) Get(ctx context.Context, subdomain string) (*Host, bool) {
|
||||
s.mu.RLock()
|
||||
h, ok := s.hosts[subdomain]
|
||||
s.mu.RUnlock()
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
hostLock := s.lockFor(subdomain)
|
||||
hostLock.Lock()
|
||||
defer hostLock.Unlock()
|
||||
return cloneHost(h), true
|
||||
}
|
||||
|
||||
// All returns every host, sorted by subdomain. Each returned Host is a copy;
|
||||
// mutations to the slice or its elements do not affect the store.
|
||||
func (s *MemoryStore) All(ctx context.Context) []*Host {
|
||||
s.mu.RLock()
|
||||
names := make([]string, 0, len(s.hosts))
|
||||
for name := range s.hosts {
|
||||
names = append(names, name)
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
sort.Strings(names)
|
||||
|
||||
out := make([]*Host, 0, len(names))
|
||||
for _, n := range names {
|
||||
if h, ok := s.Get(ctx, n); ok {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Count returns the number of hosts in the store.
|
||||
func (s *MemoryStore) Count(ctx context.Context) int {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return len(s.hosts)
|
||||
}
|
||||
|
||||
// Close is a no-op for MemoryStore; implemented to satisfy Store interface.
|
||||
func (s *MemoryStore) Close() error { return nil }
|
||||
|
||||
// cloneHost returns a deep-enough copy that slice/map fields are detached.
|
||||
func cloneHost(h *Host) *Host {
|
||||
if h == nil {
|
||||
return nil
|
||||
}
|
||||
c := *h
|
||||
c.IPs = cloneStrings(h.IPs)
|
||||
c.Technologies = cloneStrings(h.Technologies)
|
||||
c.TLSAltNames = cloneStrings(h.TLSAltNames)
|
||||
c.DiscoveredVia = cloneStrings(h.DiscoveredVia)
|
||||
c.Ports = cloneInts(h.Ports)
|
||||
c.Headers = cloneStringMap(h.Headers)
|
||||
|
||||
if h.TLSFingerprint != nil {
|
||||
fp := *h.TLSFingerprint
|
||||
fp.InternalHosts = cloneStrings(h.TLSFingerprint.InternalHosts)
|
||||
c.TLSFingerprint = &fp
|
||||
}
|
||||
if h.Takeover != nil {
|
||||
t := *h.Takeover
|
||||
c.Takeover = &t
|
||||
}
|
||||
|
||||
c.Vulnerabilities = cloneVulns(h.Vulnerabilities)
|
||||
c.Secrets = cloneSecrets(h.Secrets)
|
||||
c.CVEs = cloneCVEs(h.CVEs)
|
||||
c.AIFindings = cloneAIFindings(h.AIFindings)
|
||||
return &c
|
||||
}
|
||||
|
||||
func cloneStrings(in []string) []string {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, len(in))
|
||||
copy(out, in)
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneInts(in []int) []int {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]int, len(in))
|
||||
copy(out, in)
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneStringMap(in map[string]string) map[string]string {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(in))
|
||||
for k, v := range in {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneVulns(in []Vulnerability) []Vulnerability {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]Vulnerability, len(in))
|
||||
for i, v := range in {
|
||||
v.CVEs = cloneStrings(v.CVEs)
|
||||
out[i] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneSecrets(in []Secret) []Secret {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]Secret, len(in))
|
||||
copy(out, in)
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneCVEs(in []CVE) []CVE {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]CVE, len(in))
|
||||
copy(out, in)
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneAIFindings(in []AIFinding) []AIFinding {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]AIFinding, len(in))
|
||||
for i, f := range in {
|
||||
f.CVEs = cloneStrings(f.CVEs)
|
||||
out[i] = f
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// AppendUnique helpers — exported for modules that want to append slice
|
||||
// fields without introducing duplicates. Keeps mutation semantics in one place.
|
||||
|
||||
// AddDiscoveryMethod appends method to h.DiscoveredVia if not already present.
|
||||
func AddDiscoveryMethod(h *Host, method string) {
|
||||
for _, m := range h.DiscoveredVia {
|
||||
if m == method {
|
||||
return
|
||||
}
|
||||
}
|
||||
h.DiscoveredVia = append(h.DiscoveredVia, method)
|
||||
}
|
||||
|
||||
// AddIPs appends new IPs (dedup, in-place).
|
||||
func AddIPs(h *Host, ips []string) {
|
||||
seen := make(map[string]bool, len(h.IPs))
|
||||
for _, ip := range h.IPs {
|
||||
seen[ip] = true
|
||||
}
|
||||
for _, ip := range ips {
|
||||
if ip == "" || seen[ip] {
|
||||
continue
|
||||
}
|
||||
seen[ip] = true
|
||||
h.IPs = append(h.IPs, ip)
|
||||
}
|
||||
}
|
||||
|
||||
// AddTechnologies appends new technologies (dedup, in-place).
|
||||
func AddTechnologies(h *Host, tech []string) {
|
||||
seen := make(map[string]bool, len(h.Technologies))
|
||||
for _, t := range h.Technologies {
|
||||
seen[t] = true
|
||||
}
|
||||
for _, t := range tech {
|
||||
if t == "" || seen[t] {
|
||||
continue
|
||||
}
|
||||
seen[t] = true
|
||||
h.Technologies = append(h.Technologies, t)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestUpsert_CreatesHost(t *testing.T) {
|
||||
s := NewMemoryStore()
|
||||
ctx := context.Background()
|
||||
|
||||
err := s.Upsert(ctx, "api.example.com", func(h *Host) {
|
||||
h.IPs = []string{"1.2.3.4"}
|
||||
h.StatusCode = 200
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
h, ok := s.Get(ctx, "api.example.com")
|
||||
if !ok {
|
||||
t.Fatal("Get returned !ok after Upsert")
|
||||
}
|
||||
if h.Subdomain != "api.example.com" {
|
||||
t.Errorf("Subdomain = %q", h.Subdomain)
|
||||
}
|
||||
if !reflect.DeepEqual(h.IPs, []string{"1.2.3.4"}) {
|
||||
t.Errorf("IPs = %v", h.IPs)
|
||||
}
|
||||
if h.StatusCode != 200 {
|
||||
t.Errorf("StatusCode = %d", h.StatusCode)
|
||||
}
|
||||
if h.FirstSeen.IsZero() {
|
||||
t.Error("FirstSeen not populated")
|
||||
}
|
||||
if h.LastUpdated.IsZero() {
|
||||
t.Error("LastUpdated not populated")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsert_UpdatesExistingHost(t *testing.T) {
|
||||
s := NewMemoryStore()
|
||||
ctx := context.Background()
|
||||
|
||||
s.Upsert(ctx, "api.example.com", func(h *Host) { h.StatusCode = 200 })
|
||||
firstSeen, _ := s.Get(ctx, "api.example.com")
|
||||
time.Sleep(5 * time.Millisecond) // ensure LastUpdated differs
|
||||
|
||||
s.Upsert(ctx, "api.example.com", func(h *Host) { h.Title = "API" })
|
||||
|
||||
h, _ := s.Get(ctx, "api.example.com")
|
||||
if h.StatusCode != 200 {
|
||||
t.Errorf("StatusCode lost: %d", h.StatusCode)
|
||||
}
|
||||
if h.Title != "API" {
|
||||
t.Errorf("Title not set: %q", h.Title)
|
||||
}
|
||||
if !h.FirstSeen.Equal(firstSeen.FirstSeen) {
|
||||
t.Error("FirstSeen changed on update")
|
||||
}
|
||||
if !h.LastUpdated.After(firstSeen.LastUpdated) {
|
||||
t.Error("LastUpdated did not advance")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsert_EmptySubdomainNoop(t *testing.T) {
|
||||
s := NewMemoryStore()
|
||||
ctx := context.Background()
|
||||
if err := s.Upsert(ctx, "", func(h *Host) {}); err != nil {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
if s.Count(ctx) != 0 {
|
||||
t.Error("empty subdomain should be a noop")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsert_CanceledContext(t *testing.T) {
|
||||
s := NewMemoryStore()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if err := s.Upsert(ctx, "a.example.com", func(h *Host) {}); err == nil {
|
||||
t.Error("expected error for canceled context")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGet_Missing(t *testing.T) {
|
||||
s := NewMemoryStore()
|
||||
_, ok := s.Get(context.Background(), "none.example.com")
|
||||
if ok {
|
||||
t.Error("expected !ok for missing host")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGet_ReturnsCopy(t *testing.T) {
|
||||
s := NewMemoryStore()
|
||||
ctx := context.Background()
|
||||
s.Upsert(ctx, "a.example.com", func(h *Host) {
|
||||
h.IPs = []string{"1.2.3.4"}
|
||||
h.Technologies = []string{"nginx"}
|
||||
h.Headers = map[string]string{"X-Test": "yes"}
|
||||
h.TLSFingerprint = &TLSFingerprint{Vendor: "Fortinet", InternalHosts: []string{"internal.local"}}
|
||||
})
|
||||
|
||||
a, _ := s.Get(ctx, "a.example.com")
|
||||
// mutate returned host aggressively
|
||||
a.IPs[0] = "MUTATED"
|
||||
a.Technologies = append(a.Technologies, "INJECTED")
|
||||
a.Headers["X-Test"] = "MUTATED"
|
||||
a.TLSFingerprint.Vendor = "MUTATED"
|
||||
a.TLSFingerprint.InternalHosts[0] = "MUTATED"
|
||||
|
||||
b, _ := s.Get(ctx, "a.example.com")
|
||||
if b.IPs[0] != "1.2.3.4" {
|
||||
t.Errorf("IPs corrupted: %v", b.IPs)
|
||||
}
|
||||
if len(b.Technologies) != 1 {
|
||||
t.Errorf("Technologies corrupted: %v", b.Technologies)
|
||||
}
|
||||
if b.Headers["X-Test"] != "yes" {
|
||||
t.Errorf("Headers corrupted: %v", b.Headers)
|
||||
}
|
||||
if b.TLSFingerprint.Vendor != "Fortinet" {
|
||||
t.Errorf("TLSFingerprint.Vendor corrupted: %q", b.TLSFingerprint.Vendor)
|
||||
}
|
||||
if b.TLSFingerprint.InternalHosts[0] != "internal.local" {
|
||||
t.Errorf("InternalHosts corrupted: %v", b.TLSFingerprint.InternalHosts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAll_Sorted(t *testing.T) {
|
||||
s := NewMemoryStore()
|
||||
ctx := context.Background()
|
||||
for _, name := range []string{"zeta.example.com", "alpha.example.com", "mid.example.com"} {
|
||||
s.Upsert(ctx, name, func(h *Host) {})
|
||||
}
|
||||
all := s.All(ctx)
|
||||
got := make([]string, len(all))
|
||||
for i, h := range all {
|
||||
got[i] = h.Subdomain
|
||||
}
|
||||
want := []string{"alpha.example.com", "mid.example.com", "zeta.example.com"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("All order = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCount(t *testing.T) {
|
||||
s := NewMemoryStore()
|
||||
ctx := context.Background()
|
||||
if s.Count(ctx) != 0 {
|
||||
t.Error("initial Count != 0")
|
||||
}
|
||||
s.Upsert(ctx, "a.example.com", func(h *Host) {})
|
||||
s.Upsert(ctx, "b.example.com", func(h *Host) {})
|
||||
s.Upsert(ctx, "a.example.com", func(h *Host) {}) // update, not new
|
||||
if got := s.Count(ctx); got != 2 {
|
||||
t.Errorf("Count = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentUpserts_SameHost(t *testing.T) {
|
||||
// All writers target the same host; only one value wins per field but
|
||||
// no race should fire.
|
||||
s := NewMemoryStore()
|
||||
ctx := context.Background()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
const writers = 50
|
||||
var counter atomic.Int32
|
||||
for i := 0; i < writers; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
s.Upsert(ctx, "hot.example.com", func(h *Host) {
|
||||
h.Technologies = append(h.Technologies, fmt.Sprintf("t%d", i))
|
||||
counter.Add(1)
|
||||
})
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if counter.Load() != writers {
|
||||
t.Errorf("not all mutators ran: %d/%d", counter.Load(), writers)
|
||||
}
|
||||
h, _ := s.Get(ctx, "hot.example.com")
|
||||
if len(h.Technologies) != writers {
|
||||
t.Errorf("expected %d technologies, got %d", writers, len(h.Technologies))
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentUpserts_DifferentHosts(t *testing.T) {
|
||||
s := NewMemoryStore()
|
||||
ctx := context.Background()
|
||||
var wg sync.WaitGroup
|
||||
const hosts = 200
|
||||
for i := 0; i < hosts; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
s.Upsert(ctx, fmt.Sprintf("h%d.example.com", i), func(h *Host) {
|
||||
h.IPs = []string{"1.2.3.4"}
|
||||
})
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
if got := s.Count(ctx); got != hosts {
|
||||
t.Errorf("expected %d hosts, got %d", hosts, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClose_Idempotent(t *testing.T) {
|
||||
s := NewMemoryStore()
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- Helper tests ----------
|
||||
|
||||
func TestAddDiscoveryMethod(t *testing.T) {
|
||||
h := &Host{}
|
||||
AddDiscoveryMethod(h, "passive:crt.sh")
|
||||
AddDiscoveryMethod(h, "brute")
|
||||
AddDiscoveryMethod(h, "passive:crt.sh") // duplicate
|
||||
if !reflect.DeepEqual(h.DiscoveredVia, []string{"passive:crt.sh", "brute"}) {
|
||||
t.Errorf("DiscoveredVia = %v", h.DiscoveredVia)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddIPs_Dedup(t *testing.T) {
|
||||
h := &Host{IPs: []string{"1.1.1.1"}}
|
||||
AddIPs(h, []string{"1.1.1.1", "2.2.2.2", "", "3.3.3.3", "2.2.2.2"})
|
||||
sort.Strings(h.IPs)
|
||||
want := []string{"1.1.1.1", "2.2.2.2", "3.3.3.3"}
|
||||
if !reflect.DeepEqual(h.IPs, want) {
|
||||
t.Errorf("IPs = %v, want %v", h.IPs, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddTechnologies_Dedup(t *testing.T) {
|
||||
h := &Host{Technologies: []string{"nginx"}}
|
||||
AddTechnologies(h, []string{"nginx", "Go", "", "React", "Go"})
|
||||
sort.Strings(h.Technologies)
|
||||
want := []string{"Go", "React", "nginx"}
|
||||
if !reflect.DeepEqual(h.Technologies, want) {
|
||||
t.Errorf("Technologies = %v, want %v", h.Technologies, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneHost_Nil(t *testing.T) {
|
||||
if got := cloneHost(nil); got != nil {
|
||||
t.Errorf("cloneHost(nil) = %v, want nil", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
// Package store defines the Store interface used by pipeline modules to record
|
||||
// per-host findings. Full implementations (in-memory + BoltDB-backed) live in
|
||||
// this same package — this file only declares the interface so other packages
|
||||
// can depend on it without pulling in storage backends.
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Host is the aggregate per-subdomain record. Fields are populated
|
||||
// incrementally as modules publish events.
|
||||
//
|
||||
// Field names intentionally mirror the legacy config.SubdomainResult shape so
|
||||
// migrating JSON output in F0.6 is mechanical. Over time this struct will
|
||||
// diverge (more fields, richer types) as v2 features land.
|
||||
type Host struct {
|
||||
Subdomain string
|
||||
IPs []string
|
||||
CNAME string
|
||||
PTR string
|
||||
|
||||
// Resolution metadata
|
||||
ASN string
|
||||
Org string
|
||||
Country string
|
||||
City string
|
||||
|
||||
// HTTP probe
|
||||
URL string
|
||||
StatusCode int
|
||||
ContentLength int64
|
||||
Title string
|
||||
Server string
|
||||
Technologies []string
|
||||
Headers map[string]string
|
||||
ResponseMs int64
|
||||
|
||||
// TLS
|
||||
TLSVersion string
|
||||
TLSIssuer string
|
||||
TLSExpiry time.Time
|
||||
TLSSelfSigned bool
|
||||
TLSAltNames []string
|
||||
TLSFingerprint *TLSFingerprint
|
||||
|
||||
// Classification
|
||||
CloudProvider string
|
||||
WAF string
|
||||
Ports []int
|
||||
|
||||
// Analysis
|
||||
Vulnerabilities []Vulnerability
|
||||
Secrets []Secret
|
||||
CVEs []CVE
|
||||
AIFindings []AIFinding
|
||||
Takeover *Takeover
|
||||
|
||||
// Discovery metadata
|
||||
DiscoveredVia []string // e.g. ["passive:crt.sh", "brute"]
|
||||
FirstSeen time.Time
|
||||
LastUpdated time.Time
|
||||
}
|
||||
|
||||
// TLSFingerprint identifies a security appliance (firewall, VPN, load balancer)
|
||||
// from its TLS certificate.
|
||||
type TLSFingerprint struct {
|
||||
Vendor string
|
||||
Product string
|
||||
Version string
|
||||
ApplianceKind string
|
||||
InternalHosts []string
|
||||
}
|
||||
|
||||
// Vulnerability is a single finding recorded on a host.
|
||||
type Vulnerability struct {
|
||||
ID string
|
||||
Title string
|
||||
Description string
|
||||
Severity string
|
||||
URL string
|
||||
Evidence string
|
||||
Remediation string
|
||||
CVEs []string
|
||||
OWASP string
|
||||
CVSS float64
|
||||
FoundAt time.Time
|
||||
}
|
||||
|
||||
// Secret is a credential/token discovered on a host.
|
||||
type Secret struct {
|
||||
Kind string
|
||||
Match string
|
||||
Value string
|
||||
Location string
|
||||
Validated bool
|
||||
Severity string
|
||||
Description string
|
||||
FoundAt time.Time
|
||||
}
|
||||
|
||||
// CVE is a CVE match correlated to a detected technology.
|
||||
type CVE struct {
|
||||
ID string
|
||||
Technology string
|
||||
Version string
|
||||
Severity string
|
||||
CVSS float64
|
||||
Description string
|
||||
URL string
|
||||
InKEV bool
|
||||
FoundAt time.Time
|
||||
}
|
||||
|
||||
// AIFinding is an AI/agent-produced insight.
|
||||
type AIFinding struct {
|
||||
Agent string
|
||||
Model string
|
||||
Severity string
|
||||
Title string
|
||||
Description string
|
||||
Evidence string
|
||||
CVEs []string
|
||||
OWASP string
|
||||
Confidence float64
|
||||
FoundAt time.Time
|
||||
}
|
||||
|
||||
// Takeover is a confirmed or candidate subdomain takeover.
|
||||
type Takeover struct {
|
||||
Service string
|
||||
CNAME string
|
||||
Evidence string
|
||||
PoC string
|
||||
Confirmed bool
|
||||
FoundAt time.Time
|
||||
}
|
||||
|
||||
// Store is the aggregate interface modules use to record findings. Methods
|
||||
// must be safe for concurrent use by many goroutines.
|
||||
type Store interface {
|
||||
// Upsert merges patch into the record for subdomain. Only non-zero fields
|
||||
// in patch overwrite existing data; slice/map fields are appended/merged.
|
||||
// The mutator is invoked under a per-host lock so concurrent callers see
|
||||
// consistent state.
|
||||
Upsert(ctx context.Context, subdomain string, mutate func(*Host)) error
|
||||
|
||||
// Get returns a snapshot copy of the record for subdomain.
|
||||
Get(ctx context.Context, subdomain string) (*Host, bool)
|
||||
|
||||
// All returns a snapshot slice of every host. The slice is sorted by
|
||||
// subdomain for deterministic output.
|
||||
All(ctx context.Context) []*Host
|
||||
|
||||
// Count returns the number of hosts in the store.
|
||||
Count(ctx context.Context) int
|
||||
|
||||
// Close releases resources (e.g. BoltDB handle). Idempotent.
|
||||
Close() error
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
// Package tui provides terminal-only live views of scan activity. No web
|
||||
// UI by design. Fase 4 will expand this into a bubbletea-powered
|
||||
// interactive TUI with panels; the current LivePrinter is the minimal
|
||||
// terminal-only viewer that emits colorized event lines in real time.
|
||||
package tui
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/eventbus"
|
||||
"god-eye/internal/output"
|
||||
)
|
||||
|
||||
// LivePrinter subscribes to every event on a bus and prints a one-line
|
||||
// summary to stdout as they arrive. Safe to attach alongside the regular
|
||||
// report module — this is purely an observability layer.
|
||||
type LivePrinter struct {
|
||||
bus *eventbus.Bus
|
||||
sub *eventbus.Subscription
|
||||
verbosity int // 0 = quiet (vulns only), 1 = normal (discovery+vulns), 2 = noisy
|
||||
started time.Time
|
||||
|
||||
evCount atomic.Uint64
|
||||
}
|
||||
|
||||
// NewLivePrinter attaches to bus and begins printing.
|
||||
//
|
||||
// verbosity levels:
|
||||
//
|
||||
// 0 — only vulnerabilities, takeovers, secrets, CVEs
|
||||
// 1 — above + subdomain discovery + HTTP probe summaries
|
||||
// 2 — everything, including module errors and phase markers
|
||||
func NewLivePrinter(bus *eventbus.Bus, verbosity int) *LivePrinter {
|
||||
p := &LivePrinter{bus: bus, verbosity: verbosity, started: time.Now()}
|
||||
p.sub = bus.SubscribeAll(p.handle)
|
||||
return p
|
||||
}
|
||||
|
||||
// Close unsubscribes from the bus and prints a summary footer.
|
||||
func (p *LivePrinter) Close() {
|
||||
if p.sub != nil {
|
||||
p.sub.Unsubscribe()
|
||||
}
|
||||
dur := time.Since(p.started).Round(time.Millisecond)
|
||||
fmt.Printf("%s scan elapsed %s, %d events seen\n",
|
||||
output.Dim("·"), output.BoldGreen(dur.String()), p.evCount.Load())
|
||||
}
|
||||
|
||||
func (p *LivePrinter) handle(_ context.Context, e eventbus.Event) {
|
||||
p.evCount.Add(1)
|
||||
switch ev := e.(type) {
|
||||
case eventbus.SubdomainDiscovered:
|
||||
if p.verbosity >= 1 {
|
||||
fmt.Printf("%s %s %s\n", output.Dim("↳"), output.Cyan(ev.Method), ev.Subdomain)
|
||||
}
|
||||
case eventbus.DNSResolved:
|
||||
if p.verbosity >= 2 {
|
||||
fmt.Printf("%s %s %s\n", output.Dim("⏚"), ev.Subdomain, output.Dim(joinIPs(ev.IPs)))
|
||||
}
|
||||
case eventbus.HTTPProbed:
|
||||
if p.verbosity >= 1 {
|
||||
color := statusColor(ev.StatusCode)
|
||||
fmt.Printf("%s %s %s %s\n", color, ev.URL, output.Dim(fmt.Sprintf("[%d]", ev.StatusCode)), output.Dim(ev.Title))
|
||||
}
|
||||
case eventbus.VulnerabilityFound:
|
||||
fmt.Printf("%s %s %s %s\n", sevBadge(ev.Severity), output.BoldWhite(ev.Title), output.Dim(ev.URL), output.Dim(ev.ID))
|
||||
case eventbus.SecretFound:
|
||||
fmt.Printf("%s %s %s %s\n", sevBadge(ev.Severity), output.BoldWhite("SECRET:"+ev.Kind), ev.Location, output.Dim(ev.Match))
|
||||
case eventbus.TakeoverCandidate:
|
||||
fmt.Printf("%s %s %s service=%s\n", sevBadge(eventbus.SeverityHigh), output.BoldYellow("TAKEOVER?"), ev.Subdomain, ev.Service)
|
||||
case eventbus.TakeoverConfirmed:
|
||||
fmt.Printf("%s %s %s service=%s\n", sevBadge(eventbus.SeverityCritical), output.BgRed(" TAKEOVER "), ev.Subdomain, ev.Service)
|
||||
case eventbus.CVEMatch:
|
||||
fmt.Printf("%s %s %s@%s → %s\n", sevBadge(ev.Severity), output.BoldWhite("CVE"), ev.Technology, ev.Version, ev.CVE)
|
||||
case eventbus.AIFinding:
|
||||
fmt.Printf("%s %s %s %s\n", sevBadge(ev.Severity), output.BoldMagenta("AI:"+ev.Agent), output.Dim(ev.Subject), ev.Title)
|
||||
case eventbus.ModuleError:
|
||||
if p.verbosity >= 2 {
|
||||
fmt.Printf("%s %s %s\n", output.Red("⚠"), output.Dim(ev.Module), ev.Err)
|
||||
}
|
||||
case eventbus.PhaseStarted:
|
||||
if p.verbosity >= 1 {
|
||||
fmt.Printf("%s %s\n", output.Dim("▶"), output.BoldCyan("phase "+ev.Phase))
|
||||
}
|
||||
case eventbus.PhaseCompleted:
|
||||
if p.verbosity >= 1 {
|
||||
fmt.Printf("%s %s %s\n", output.Dim("▣"), output.Dim("phase "+ev.Phase), output.Dim(ev.Duration.Round(time.Millisecond).String()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sevBadge(s eventbus.Severity) string {
|
||||
switch s {
|
||||
case eventbus.SeverityCritical:
|
||||
return output.BgRed(" CRIT ")
|
||||
case eventbus.SeverityHigh:
|
||||
return output.Red("[HIGH]")
|
||||
case eventbus.SeverityMedium:
|
||||
return output.Yellow("[MED]")
|
||||
case eventbus.SeverityLow:
|
||||
return output.Blue("[LOW]")
|
||||
default:
|
||||
return output.Dim("[INFO]")
|
||||
}
|
||||
}
|
||||
|
||||
func statusColor(code int) string {
|
||||
switch {
|
||||
case code >= 200 && code < 300:
|
||||
return output.Green("●")
|
||||
case code >= 300 && code < 400:
|
||||
return output.Yellow("◐")
|
||||
case code >= 400 && code < 500:
|
||||
return output.Red("○")
|
||||
case code >= 500:
|
||||
return output.BoldRed("✕")
|
||||
default:
|
||||
return output.Dim("·")
|
||||
}
|
||||
}
|
||||
|
||||
func joinIPs(ips []string) string {
|
||||
out := "["
|
||||
for i, ip := range ips {
|
||||
if i > 0 {
|
||||
out += ","
|
||||
}
|
||||
out += ip
|
||||
}
|
||||
return out + "]"
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
package validator
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidateDomain(t *testing.T) {
|
||||
v := DefaultDomainValidator()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
wantErr bool
|
||||
}{
|
||||
{"simple domain", "example.com", false},
|
||||
{"subdomain", "api.example.com", false},
|
||||
{"deep subdomain", "a.b.c.example.com", false},
|
||||
{"hyphen in middle", "my-site.example.com", false},
|
||||
{"co.uk tld", "example.co.uk", false},
|
||||
{"uppercase", "EXAMPLE.COM", false},
|
||||
|
||||
{"empty", "", true},
|
||||
{"whitespace only", " ", true},
|
||||
{"with scheme http", "http://example.com", true},
|
||||
{"with scheme https", "https://example.com", true},
|
||||
{"path traversal", "example.com/../etc", true},
|
||||
{"shell metachar ;", "example.com;whoami", true},
|
||||
{"shell metachar |", "example.com|whoami", true},
|
||||
{"shell metachar &", "example.com&ls", true},
|
||||
{"backtick", "example.com`id`", true},
|
||||
{"dollar", "example.com$USER", true},
|
||||
{"newline", "example.com\nmalicious", true},
|
||||
{"null byte", "example.com\x00.evil", true},
|
||||
{"leading hyphen label", "-example.com", true},
|
||||
{"trailing hyphen label", "example-.com", true},
|
||||
{"double dot", "example..com", true},
|
||||
{"label too long", strings.Repeat("a", 64) + ".com", true},
|
||||
{"domain too long", strings.Repeat("a.", 130) + "com", true},
|
||||
{"numeric tld", "example.123", true},
|
||||
{"single label", "localhost", true},
|
||||
{"angle brackets", "<script>.com", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := v.ValidateDomain(tt.input)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("ValidateDomain(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeDomain(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{"example.com", "example.com"},
|
||||
{" example.com ", "example.com"},
|
||||
{"EXAMPLE.COM", "example.com"},
|
||||
{"http://example.com", "example.com"},
|
||||
{"https://example.com", "example.com"},
|
||||
{"https://example.com/", "example.com"},
|
||||
{"HTTPS://Example.com/", "example.com"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
got := SanitizeDomain(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("SanitizeDomain(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateIP(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
wantErr bool
|
||||
}{
|
||||
{"ipv4", "8.8.8.8", false},
|
||||
{"ipv4 with whitespace", " 1.1.1.1 ", false},
|
||||
{"ipv6", "::1", false},
|
||||
{"ipv6 full", "2001:db8::1", false},
|
||||
{"empty", "", true},
|
||||
{"invalid format", "not-an-ip", true},
|
||||
{"out of range", "999.999.999.999", true},
|
||||
{"with port (invalid for ParseIP)", "8.8.8.8:53", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := ValidateIP(tt.input)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("ValidateIP(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatePort(t *testing.T) {
|
||||
tests := []struct {
|
||||
port int
|
||||
wantErr bool
|
||||
}{
|
||||
{1, false},
|
||||
{80, false},
|
||||
{443, false},
|
||||
{65535, false},
|
||||
{0, true},
|
||||
{-1, true},
|
||||
{65536, true},
|
||||
{100000, true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
err := ValidatePort(tt.port)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("ValidatePort(%d) error = %v, wantErr %v", tt.port, err, tt.wantErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateWordlistPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
wantErr bool
|
||||
}{
|
||||
{"empty allowed", "", false},
|
||||
{"relative path", "wordlists/subdomains.txt", false},
|
||||
{"absolute path", "/tmp/wordlist.txt", false},
|
||||
{"path traversal", "../../../etc/passwd", true},
|
||||
{"path traversal mid", "safe/../../../etc/passwd", true},
|
||||
{"null byte", "wordlist.txt\x00.evil", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := ValidateWordlistPath(tt.input)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("ValidateWordlistPath(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateOutputPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
wantErr bool
|
||||
}{
|
||||
{"empty allowed", "", false},
|
||||
{"relative output", "results.json", false},
|
||||
{"tmp output", "/tmp/results.json", false},
|
||||
{"path traversal", "../../../etc/passwd", true},
|
||||
{"null byte", "output.txt\x00", true},
|
||||
{"system path /etc/", "/etc/evil", true},
|
||||
{"system path /var/", "/var/www/shell.php", true},
|
||||
{"system path /usr/", "/usr/local/bin/backdoor", true},
|
||||
{"system path /root/", "/root/.ssh/id_rsa", true},
|
||||
{"system path /proc/", "/proc/self/mem", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := ValidateOutputPath(tt.input)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("ValidateOutputPath(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateResolvers(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
wantErr bool
|
||||
}{
|
||||
{"empty allowed", "", false},
|
||||
{"single resolver", "8.8.8.8", false},
|
||||
{"multiple resolvers", "8.8.8.8,1.1.1.1", false},
|
||||
{"multiple with spaces", "8.8.8.8, 1.1.1.1 , 9.9.9.9", false},
|
||||
{"invalid entry", "8.8.8.8,not-an-ip", true},
|
||||
{"all invalid", "foo,bar", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := ValidateResolvers(tt.input)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("ValidateResolvers(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConcurrency(t *testing.T) {
|
||||
tests := []struct {
|
||||
n int
|
||||
wantErr bool
|
||||
}{
|
||||
{1, false},
|
||||
{1000, false},
|
||||
{10000, false},
|
||||
{0, true},
|
||||
{-1, true},
|
||||
{10001, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
err := ValidateConcurrency(tt.n)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("ValidateConcurrency(%d) err=%v wantErr=%v", tt.n, err, tt.wantErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateTimeout(t *testing.T) {
|
||||
tests := []struct {
|
||||
n int
|
||||
wantErr bool
|
||||
}{
|
||||
{1, false},
|
||||
{5, false},
|
||||
{300, false},
|
||||
{0, true},
|
||||
{-1, true},
|
||||
{301, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
err := ValidateTimeout(tt.n)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("ValidateTimeout(%d) err=%v wantErr=%v", tt.n, err, tt.wantErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationError_Error(t *testing.T) {
|
||||
e := &ValidationError{Field: "domain", Message: "bad"}
|
||||
if got := e.Error(); got != "domain: bad" {
|
||||
t.Errorf("got %q, want %q", got, "domain: bad")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
// Package wizard drives the interactive CLI setup flow. When god-eye is
|
||||
// launched from a terminal with no target, the wizard asks a handful of
|
||||
// questions (AI tier, model download consent, target, scan profile,
|
||||
// live view, output) and returns a Choice the caller applies to Config.
|
||||
//
|
||||
// The wizard is terminal-only by design and uses the output package for
|
||||
// consistent colors with the rest of the tool. It never modifies state
|
||||
// itself — that's the caller's job.
|
||||
package wizard
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/mattn/go-isatty"
|
||||
|
||||
"god-eye/internal/output"
|
||||
"god-eye/internal/validator"
|
||||
)
|
||||
|
||||
// ErrCancelled is returned when the user aborts the wizard (Ctrl-C or
|
||||
// by answering "no" at the confirmation step).
|
||||
var ErrCancelled = errors.New("wizard: cancelled")
|
||||
|
||||
// IsInteractive reports whether stdin/stdout are attached to a terminal.
|
||||
// Callers use this to decide whether to prompt or fall back to
|
||||
// non-interactive defaults.
|
||||
func IsInteractive() bool {
|
||||
return isatty.IsTerminal(os.Stdin.Fd()) && isatty.IsTerminal(os.Stdout.Fd())
|
||||
}
|
||||
|
||||
// prompter is a thin wrapper around a bufio.Reader + writer with helpers
|
||||
// that respect defaults and re-prompt on invalid input.
|
||||
type prompter struct {
|
||||
r *bufio.Reader
|
||||
w io.Writer
|
||||
max int // max re-prompts before giving up (default 3)
|
||||
}
|
||||
|
||||
func newPrompter(in io.Reader, out io.Writer) *prompter {
|
||||
return &prompter{r: bufio.NewReader(in), w: out, max: 3}
|
||||
}
|
||||
|
||||
// printf writes to the prompter's writer. Allows swapping out os.Stdout
|
||||
// in tests for a bytes.Buffer.
|
||||
func (p *prompter) printf(format string, args ...interface{}) {
|
||||
fmt.Fprintf(p.w, format, args...)
|
||||
}
|
||||
|
||||
// readLine reads one line, trimming trailing whitespace. Returns EOF as
|
||||
// a cancellation.
|
||||
func (p *prompter) readLine() (string, error) {
|
||||
line, err := p.r.ReadString('\n')
|
||||
if err != nil && line == "" {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(line), nil
|
||||
}
|
||||
|
||||
// choose presents a numbered menu and returns the selected index. Empty
|
||||
// input accepts def (1-based). Re-prompts on invalid input up to max.
|
||||
func (p *prompter) choose(question string, options []string, def int) (int, error) {
|
||||
p.printf("\n%s %s\n", output.BoldCyan("?"), output.BoldWhite(question))
|
||||
for i, opt := range options {
|
||||
marker := " "
|
||||
if i+1 == def {
|
||||
marker = output.Green("▸")
|
||||
}
|
||||
p.printf(" %s %s %s\n", marker, output.Yellow(fmt.Sprintf("%d)", i+1)), opt)
|
||||
}
|
||||
prompt := fmt.Sprintf(" %s ", output.Dim(fmt.Sprintf("Choice [%d]:", def)))
|
||||
for attempt := 0; attempt < p.max; attempt++ {
|
||||
p.printf("%s", prompt)
|
||||
raw, err := p.readLine()
|
||||
if err != nil {
|
||||
return 0, ErrCancelled
|
||||
}
|
||||
if raw == "" {
|
||||
return def, nil
|
||||
}
|
||||
n, err := strconv.Atoi(raw)
|
||||
if err == nil && n >= 1 && n <= len(options) {
|
||||
return n, nil
|
||||
}
|
||||
p.printf(" %s not a valid option\n", output.Red("✘"))
|
||||
}
|
||||
return 0, fmt.Errorf("too many invalid attempts")
|
||||
}
|
||||
|
||||
// askText reads free-form text. When required is false an empty answer
|
||||
// (after max attempts) returns "".
|
||||
func (p *prompter) askText(question, def string, required bool, validate func(string) error) (string, error) {
|
||||
defHint := ""
|
||||
if def != "" {
|
||||
defHint = fmt.Sprintf(" [%s]", def)
|
||||
} else if !required {
|
||||
defHint = " (empty to skip)"
|
||||
}
|
||||
p.printf("\n%s %s%s\n", output.BoldCyan("?"), output.BoldWhite(question), output.Dim(defHint))
|
||||
|
||||
for attempt := 0; attempt < p.max; attempt++ {
|
||||
p.printf(" %s ", output.Dim(">"))
|
||||
raw, err := p.readLine()
|
||||
if err != nil {
|
||||
return "", ErrCancelled
|
||||
}
|
||||
if raw == "" {
|
||||
if def != "" {
|
||||
return def, nil
|
||||
}
|
||||
if !required {
|
||||
return "", nil
|
||||
}
|
||||
p.printf(" %s required\n", output.Red("✘"))
|
||||
continue
|
||||
}
|
||||
if validate != nil {
|
||||
if err := validate(raw); err != nil {
|
||||
p.printf(" %s %v\n", output.Red("✘"), err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
return "", fmt.Errorf("too many invalid attempts")
|
||||
}
|
||||
|
||||
// yesNo prompts for a boolean with a default. Accepts y/yes/n/no/empty
|
||||
// (case-insensitive).
|
||||
func (p *prompter) yesNo(question string, def bool) (bool, error) {
|
||||
hint := "y/N"
|
||||
if def {
|
||||
hint = "Y/n"
|
||||
}
|
||||
p.printf("\n%s %s %s\n", output.BoldCyan("?"), output.BoldWhite(question), output.Dim("["+hint+"]"))
|
||||
for attempt := 0; attempt < p.max; attempt++ {
|
||||
p.printf(" %s ", output.Dim(">"))
|
||||
raw, err := p.readLine()
|
||||
if err != nil {
|
||||
return false, ErrCancelled
|
||||
}
|
||||
if raw == "" {
|
||||
return def, nil
|
||||
}
|
||||
switch strings.ToLower(raw) {
|
||||
case "y", "yes", "si", "sì", "s":
|
||||
return true, nil
|
||||
case "n", "no":
|
||||
return false, nil
|
||||
}
|
||||
p.printf(" %s answer y or n\n", output.Red("✘"))
|
||||
}
|
||||
return false, fmt.Errorf("too many invalid attempts")
|
||||
}
|
||||
|
||||
// askDomain is a specialization of askText that validates via the
|
||||
// validator package and strips URL-style prefixes.
|
||||
func (p *prompter) askDomain() (string, error) {
|
||||
return p.askText("Target domain", "", true, func(s string) error {
|
||||
s = validator.SanitizeDomain(s)
|
||||
v := validator.DefaultDomainValidator()
|
||||
return v.ValidateDomain(s)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package wizard
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// newPrompterWithInput builds a prompter whose stdin comes from the given
|
||||
// string. Each line is terminated with \n.
|
||||
func newPrompterWithInput(input string) (*prompter, *bytes.Buffer) {
|
||||
buf := &bytes.Buffer{}
|
||||
p := newPrompter(strings.NewReader(input), buf)
|
||||
return p, buf
|
||||
}
|
||||
|
||||
func TestChoose_DefaultAcceptsEmpty(t *testing.T) {
|
||||
p, _ := newPrompterWithInput("\n")
|
||||
n, err := p.choose("pick one", []string{"a", "b", "c"}, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Errorf("default = %d, want 2", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChoose_ValidNumber(t *testing.T) {
|
||||
p, _ := newPrompterWithInput("3\n")
|
||||
n, _ := p.choose("pick one", []string{"a", "b", "c"}, 1)
|
||||
if n != 3 {
|
||||
t.Errorf("= %d, want 3", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChoose_OutOfRangeReprompts(t *testing.T) {
|
||||
p, _ := newPrompterWithInput("99\nfoo\n2\n")
|
||||
n, err := p.choose("pick one", []string{"a", "b", "c"}, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Errorf("= %d, want 2", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChoose_MaxAttemptsExhausted(t *testing.T) {
|
||||
p, _ := newPrompterWithInput("bad\nbad\nbad\n")
|
||||
p.max = 3
|
||||
if _, err := p.choose("pick one", []string{"a", "b"}, 1); err == nil {
|
||||
t.Error("expected error after max attempts")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChoose_EOFCancels(t *testing.T) {
|
||||
p, _ := newPrompterWithInput("")
|
||||
_, err := p.choose("pick", []string{"a"}, 1)
|
||||
if !errors.Is(err, ErrCancelled) {
|
||||
t.Errorf("want ErrCancelled, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestYesNo_Default(t *testing.T) {
|
||||
p, _ := newPrompterWithInput("\n")
|
||||
got, _ := p.yesNo("ok?", true)
|
||||
if !got {
|
||||
t.Error("empty input should return default true")
|
||||
}
|
||||
|
||||
p2, _ := newPrompterWithInput("\n")
|
||||
got2, _ := p2.yesNo("ok?", false)
|
||||
if got2 {
|
||||
t.Error("empty input should return default false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestYesNo_ParsesYesNo(t *testing.T) {
|
||||
cases := map[string]bool{
|
||||
"y\n": true,
|
||||
"Y\n": true,
|
||||
"yes\n": true,
|
||||
"YES\n": true,
|
||||
"s\n": true, // italian si
|
||||
"si\n": true,
|
||||
"n\n": false,
|
||||
"no\n": false,
|
||||
"NO\n": false,
|
||||
}
|
||||
for input, want := range cases {
|
||||
p, _ := newPrompterWithInput(input)
|
||||
got, err := p.yesNo("?", false)
|
||||
if err != nil {
|
||||
t.Errorf("input %q: %v", input, err)
|
||||
continue
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("input %q: got %v want %v", input, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAskText_DefaultUsed(t *testing.T) {
|
||||
p, _ := newPrompterWithInput("\n")
|
||||
got, _ := p.askText("thing?", "sensible-default", false, nil)
|
||||
if got != "sensible-default" {
|
||||
t.Errorf("= %q, want sensible-default", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAskText_OptionalEmpty(t *testing.T) {
|
||||
p, _ := newPrompterWithInput("\n")
|
||||
got, err := p.askText("optional?", "", false, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != "" {
|
||||
t.Errorf("= %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAskText_RequiredEmptyReprompts(t *testing.T) {
|
||||
p, _ := newPrompterWithInput("\n\n\n") // 3 empty attempts
|
||||
_, err := p.askText("required", "", true, nil)
|
||||
if err == nil {
|
||||
t.Error("expected error after 3 empty attempts")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAskText_Validation(t *testing.T) {
|
||||
p, _ := newPrompterWithInput("bad\ngood\n")
|
||||
got, err := p.askText("domain", "", true, func(s string) error {
|
||||
if s == "bad" {
|
||||
return errors.New("no")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != "good" {
|
||||
t.Errorf("= %q, want good", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAskDomain_InvalidReprompts(t *testing.T) {
|
||||
p, _ := newPrompterWithInput("not_a_domain\nexample.com\n")
|
||||
got, err := p.askDomain()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != "example.com" {
|
||||
t.Errorf("= %q, want example.com", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAskDomain_AcceptsSchemePrefixThroughSanitization(t *testing.T) {
|
||||
// askDomain sanitizes the input before validation — https://example.com
|
||||
// becomes example.com and passes on the first try. The caller is
|
||||
// responsible for calling SanitizeDomain again on the returned value.
|
||||
p, _ := newPrompterWithInput("https://example.com\n")
|
||||
got, err := p.askDomain()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != "https://example.com" {
|
||||
t.Errorf("= %q, want raw input preserved", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
package wizard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"god-eye/internal/ai"
|
||||
"god-eye/internal/config"
|
||||
"god-eye/internal/output"
|
||||
"god-eye/internal/validator"
|
||||
)
|
||||
|
||||
// Choice is everything the wizard decided. Caller applies it to the
|
||||
// scan Config, then runs the pipeline.
|
||||
type Choice struct {
|
||||
Target string
|
||||
AIProfile string // "lean", "balanced", "heavy", or "" (no AI)
|
||||
AIAutoPull bool
|
||||
AIVerbose bool
|
||||
ScanProfile string // "quick" / "bugbounty" / "pentest" / "asm-continuous" / "stealth-max"
|
||||
MonitorInterval string // e.g. "24h" when asm-continuous chosen, empty otherwise
|
||||
Live bool
|
||||
LiveVerbosity int
|
||||
Output string
|
||||
Format string
|
||||
Pipeline bool // always true — wizard runs the v2 pipeline
|
||||
}
|
||||
|
||||
// Options tunes wizard behavior, mainly for tests.
|
||||
type Options struct {
|
||||
In io.Reader // stdin
|
||||
Out io.Writer // stdout
|
||||
OllamaURL string // defaults to http://localhost:11434
|
||||
}
|
||||
|
||||
// Run executes the interactive flow and returns the user's choices.
|
||||
// Returns ErrCancelled if the user aborts at any stage.
|
||||
func Run(ctx context.Context, opts Options) (*Choice, error) {
|
||||
if opts.OllamaURL == "" {
|
||||
opts.OllamaURL = "http://localhost:11434"
|
||||
}
|
||||
|
||||
p := newPrompter(opts.In, opts.Out)
|
||||
choice := &Choice{Pipeline: true, LiveVerbosity: 1, Format: "txt", AIAutoPull: true}
|
||||
|
||||
printBanner(opts.Out)
|
||||
|
||||
// 1) AI tier selection.
|
||||
aiProfile, err := selectAIProfile(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
choice.AIProfile = aiProfile
|
||||
|
||||
// 2) If AI chosen, check Ollama + models, ask to pull missing.
|
||||
if aiProfile != "" {
|
||||
if err := handleAIModels(ctx, p, opts.OllamaURL, aiProfile, choice); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// 3) Target domain.
|
||||
domain, err := p.askDomain()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
choice.Target = validator.SanitizeDomain(domain)
|
||||
|
||||
// 4) Scan profile.
|
||||
scanProfile, interval, err := selectScanProfile(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
choice.ScanProfile = scanProfile
|
||||
choice.MonitorInterval = interval
|
||||
|
||||
// 5) Live terminal view.
|
||||
live, err := p.yesNo("Enable live colorized event view?", true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
choice.Live = live
|
||||
if live {
|
||||
v, err := selectLiveVerbosity(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
choice.LiveVerbosity = v
|
||||
}
|
||||
|
||||
// 6) AI verbose (only when AI selected).
|
||||
if aiProfile != "" {
|
||||
aiVerb, err := p.yesNo("Log every AI query to stderr (verbose)?", false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
choice.AIVerbose = aiVerb
|
||||
}
|
||||
|
||||
// 7) Output file (optional).
|
||||
outFile, err := p.askText("Save report to file", "", false, func(path string) error {
|
||||
return validator.ValidateOutputPath(path)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
choice.Output = outFile
|
||||
if outFile != "" {
|
||||
f, err := selectOutputFormat(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
choice.Format = f
|
||||
}
|
||||
|
||||
// 8) Final summary + confirm.
|
||||
printSummary(opts.Out, choice)
|
||||
confirm, err := p.yesNo("Start scan?", true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !confirm {
|
||||
return nil, ErrCancelled
|
||||
}
|
||||
return choice, nil
|
||||
}
|
||||
|
||||
// --- step implementations ------------------------------------------------
|
||||
|
||||
func selectAIProfile(p *prompter) (string, error) {
|
||||
opts := []string{
|
||||
output.BoldWhite("Lean") + output.Dim(" — 16GB RAM · qwen3:1.7b + qwen2.5-coder:14b (default)"),
|
||||
output.BoldWhite("Balanced") + output.Dim(" — 32GB RAM · qwen3:4b + qwen3-coder:30b (MoE, 256K ctx)"),
|
||||
output.BoldWhite("Heavy") + output.Dim(" — 64GB RAM · qwen3:8b + qwen3-coder:30b (max quality)"),
|
||||
output.BoldWhite("No AI") + output.Dim(" — Pure recon without LLM analysis"),
|
||||
}
|
||||
n, err := p.choose("Select AI tier", opts, 1)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
switch n {
|
||||
case 1:
|
||||
return "lean", nil
|
||||
case 2:
|
||||
return "balanced", nil
|
||||
case 3:
|
||||
return "heavy", nil
|
||||
default:
|
||||
return "", nil
|
||||
}
|
||||
}
|
||||
|
||||
func handleAIModels(ctx context.Context, p *prompter, ollamaURL, aiProfile string, choice *Choice) error {
|
||||
profile, _ := config.AIProfileByName(aiProfile)
|
||||
needed := []string{profile.FastModel, profile.DeepModel}
|
||||
|
||||
p.printf("\n%s Checking Ollama at %s…\n", output.BoldCyan("⚙"), output.Dim(ollamaURL))
|
||||
|
||||
ensurer := ai.NewModelEnsurer(ollamaURL)
|
||||
if err := ensurer.Reachable(ctx); err != nil {
|
||||
p.printf(" %s %v\n", output.Yellow("⚠"), err)
|
||||
p.printf(" %s Start %s in another terminal, then retry.\n",
|
||||
output.Dim("→"), output.BoldWhite("ollama serve"))
|
||||
skip, yesErr := p.yesNo("Continue without AI for this run?", true)
|
||||
if yesErr != nil {
|
||||
return yesErr
|
||||
}
|
||||
if skip {
|
||||
choice.AIProfile = ""
|
||||
return nil
|
||||
}
|
||||
return ErrCancelled
|
||||
}
|
||||
|
||||
installed, err := ensurer.Installed(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("query ollama: %w", err)
|
||||
}
|
||||
|
||||
var missing []string
|
||||
for _, m := range needed {
|
||||
if !modelInstalled(installed, m) {
|
||||
missing = append(missing, m)
|
||||
}
|
||||
}
|
||||
if len(missing) == 0 {
|
||||
p.printf(" %s All required models already present: %s\n",
|
||||
output.Green("✓"), output.Dim(strings.Join(needed, ", ")))
|
||||
return nil
|
||||
}
|
||||
|
||||
p.printf(" %s Missing models: %s\n", output.Yellow("↓"),
|
||||
output.BoldYellow(strings.Join(missing, ", ")))
|
||||
pull, err := p.yesNo("Download missing models now?", true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
choice.AIAutoPull = pull
|
||||
if !pull {
|
||||
p.printf(" %s Skipping auto-pull — AI modules will no-op if models are still missing at scan time.\n",
|
||||
output.Dim("·"))
|
||||
return nil
|
||||
}
|
||||
|
||||
ensurer.Verbose = true
|
||||
ensurer.Writer = p.w
|
||||
if err := ensurer.EnsureAll(ctx, missing); err != nil {
|
||||
return fmt.Errorf("pull: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func selectScanProfile(p *prompter) (string, string, error) {
|
||||
opts := []string{
|
||||
output.BoldWhite("Quick") + output.Dim(" — passive enum + HTTP probe, no brute"),
|
||||
output.BoldWhite("Bug bounty") + output.Dim(" — full recon, AI + all features, stealth off (default)"),
|
||||
output.BoldWhite("Pentest") + output.Dim(" — full recon + light stealth, AI on"),
|
||||
output.BoldWhite("ASM continuous") + output.Dim(" — recurring scans with diff + alerts"),
|
||||
output.BoldWhite("Stealth max") + output.Dim(" — paranoid evasion, slow, passive-first"),
|
||||
}
|
||||
n, err := p.choose("Select scan profile", opts, 2)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
switch n {
|
||||
case 1:
|
||||
return "quick", "", nil
|
||||
case 2:
|
||||
return "bugbounty", "", nil
|
||||
case 3:
|
||||
return "pentest", "", nil
|
||||
case 4:
|
||||
// Ask for interval when ASM continuous chosen.
|
||||
interval, err := p.askText("Re-scan interval (Go duration: 30m, 6h, 24h)", "24h", true, func(s string) error {
|
||||
if s == "" {
|
||||
return fmt.Errorf("interval required")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return "asm-continuous", interval, nil
|
||||
case 5:
|
||||
return "stealth-max", "", nil
|
||||
}
|
||||
return "bugbounty", "", nil
|
||||
}
|
||||
|
||||
func selectLiveVerbosity(p *prompter) (int, error) {
|
||||
opts := []string{
|
||||
output.BoldWhite("Findings only") + output.Dim(" — vulns, secrets, takeovers (quiet)"),
|
||||
output.BoldWhite("Normal") + output.Dim(" — findings + discovery + HTTP (default)"),
|
||||
output.BoldWhite("Noisy") + output.Dim(" — everything including phase markers + module errors"),
|
||||
}
|
||||
n, err := p.choose("Live view verbosity", opts, 2)
|
||||
if err != nil {
|
||||
return 1, err
|
||||
}
|
||||
return n - 1, nil
|
||||
}
|
||||
|
||||
func selectOutputFormat(p *prompter) (string, error) {
|
||||
opts := []string{"txt", "json", "csv"}
|
||||
n, err := p.choose("Report format", opts, 2)
|
||||
if err != nil {
|
||||
return "txt", err
|
||||
}
|
||||
return opts[n-1], nil
|
||||
}
|
||||
|
||||
// modelInstalled is a local copy of ai.alreadyInstalled to avoid
|
||||
// exporting the helper from the ai package.
|
||||
func modelInstalled(installed map[string]bool, model string) bool {
|
||||
if installed[model] || installed[model+":latest"] {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(model, ":") {
|
||||
base := strings.SplitN(model, ":", 2)[0]
|
||||
if installed[base] || installed[base+":latest"] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// --- presentation --------------------------------------------------------
|
||||
|
||||
func printBanner(w io.Writer) {
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, output.BoldCyan("═══════════════════════════════════════════════════════════"))
|
||||
fmt.Fprintln(w, " "+output.BoldGreen("God's Eye v2")+output.Dim(" — interactive setup"))
|
||||
fmt.Fprintln(w, " "+output.Dim("Ctrl-C to abort at any time."))
|
||||
fmt.Fprintln(w, output.BoldCyan("═══════════════════════════════════════════════════════════"))
|
||||
}
|
||||
|
||||
func printSummary(w io.Writer, c *Choice) {
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, output.BoldCyan("─── Scan summary ───"))
|
||||
// Pad the key before applying ANSI dim codes — if we padded after,
|
||||
// the ANSI escape sequences would count toward %-Ns width and the
|
||||
// output would look ragged.
|
||||
kv := func(k, v string) {
|
||||
padded := fmt.Sprintf("%-16s", k)
|
||||
fmt.Fprintf(w, " %s %s\n", output.Dim(padded), output.BoldWhite(v))
|
||||
}
|
||||
kv("Target", c.Target)
|
||||
kv("Scan profile", c.ScanProfile)
|
||||
if c.AIProfile != "" {
|
||||
kv("AI tier", c.AIProfile)
|
||||
kv("AI auto-pull", boolStr(c.AIAutoPull))
|
||||
kv("AI verbose", boolStr(c.AIVerbose))
|
||||
} else {
|
||||
kv("AI tier", "(disabled)")
|
||||
}
|
||||
kv("Live view", fmt.Sprintf("%s (v=%d)", boolStr(c.Live), c.LiveVerbosity))
|
||||
if c.MonitorInterval != "" {
|
||||
kv("Monitor every", c.MonitorInterval)
|
||||
}
|
||||
if c.Output != "" {
|
||||
kv("Output", fmt.Sprintf("%s (format=%s)", c.Output, c.Format))
|
||||
}
|
||||
}
|
||||
|
||||
func boolStr(b bool) string {
|
||||
if b {
|
||||
return "yes"
|
||||
}
|
||||
return "no"
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
// parity is a side-by-side comparison tool used to verify that the v2
|
||||
// pipeline produces equivalent results to the v1 monolithic scanner for a
|
||||
// given target domain. It runs both paths, captures JSON output from each,
|
||||
// and diffs the subdomain lists + high-signal fields.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// go run ./tools/parity -d example.com
|
||||
// go run ./tools/parity -d example.com --no-brute
|
||||
//
|
||||
// Exit codes:
|
||||
//
|
||||
// 0 — outputs match or differ only in fields we expect to differ
|
||||
// 1 — meaningful divergence (subdomain set differs, status codes differ)
|
||||
// 2 — runtime error (network, binary missing)
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type scanResult struct {
|
||||
Mode string `json:"mode"` // "v1" or "v2"
|
||||
Target string `json:"target"`
|
||||
Subdomains map[string]hostInfo `json:"subdomains"`
|
||||
ErrorOutput string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type hostInfo struct {
|
||||
Subdomain string `json:"subdomain"`
|
||||
IPs []string `json:"ips,omitempty"`
|
||||
CNAME string `json:"cname,omitempty"`
|
||||
StatusCode int `json:"status_code,omitempty"`
|
||||
Tech []string `json:"technologies,omitempty"`
|
||||
CloudProvider string `json:"cloud_provider,omitempty"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
target := flag.String("d", "", "target domain")
|
||||
noBrute := flag.Bool("no-brute", false, "skip DNS brute-force (faster, less coverage)")
|
||||
binary := flag.String("bin", "./god-eye", "path to god-eye binary")
|
||||
flag.Parse()
|
||||
|
||||
if *target == "" {
|
||||
fmt.Fprintln(os.Stderr, "usage: parity -d <domain> [--no-brute] [-bin ./god-eye]")
|
||||
os.Exit(2)
|
||||
}
|
||||
if _, err := os.Stat(*binary); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "binary not found at %s: %v\n", *binary, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
fmt.Printf(">>> Running v1 scanner on %s…\n", *target)
|
||||
v1, err := runScan(*binary, *target, *noBrute, false)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "v1 scan failed: %v\n", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
fmt.Printf(">>> Running v2 pipeline on %s…\n", *target)
|
||||
v2, err := runScan(*binary, *target, *noBrute, true)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "v2 scan failed: %v\n", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
diff := compare(v1, v2)
|
||||
printDiff(diff)
|
||||
|
||||
if diff.Meaningful() {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runScan(binary, target string, noBrute, useV2 bool) (*scanResult, error) {
|
||||
args := []string{"-d", target, "--json", "--silent"}
|
||||
if noBrute {
|
||||
args = append(args, "--no-brute")
|
||||
}
|
||||
if useV2 {
|
||||
args = append(args, "--pipeline")
|
||||
}
|
||||
|
||||
cmd := exec.Command(binary, args...)
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("exec %v: %w (stderr: %s)", args, err, strings.TrimSpace(string(cmd.Stderr.(*os.File).Name())))
|
||||
}
|
||||
|
||||
r := &scanResult{Target: target}
|
||||
r.Subdomains = map[string]hostInfo{}
|
||||
if useV2 {
|
||||
r.Mode = "v2"
|
||||
} else {
|
||||
r.Mode = "v1"
|
||||
}
|
||||
|
||||
// Both v1 and v2 emit JSON with a "subdomains" field whose shape differs
|
||||
// slightly — v1 is an array, v2 is a map. Try both.
|
||||
var shared struct {
|
||||
Subdomains []hostInfo `json:"subdomains"`
|
||||
}
|
||||
if err := json.Unmarshal(out, &shared); err == nil && len(shared.Subdomains) > 0 {
|
||||
for _, h := range shared.Subdomains {
|
||||
r.Subdomains[h.Subdomain] = h
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
var v2obj struct {
|
||||
Subdomains map[string]hostInfo `json:"subdomains"`
|
||||
}
|
||||
if err := json.Unmarshal(out, &v2obj); err == nil {
|
||||
r.Subdomains = v2obj.Subdomains
|
||||
return r, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unable to parse JSON output (%d bytes): %s", len(out), strings.TrimSpace(string(out[:min(len(out), 200)])))
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
type diffReport struct {
|
||||
Target string
|
||||
OnlyInV1 []string
|
||||
OnlyInV2 []string
|
||||
Shared []string
|
||||
StatusDiff map[string][2]int // subdomain → [v1 status, v2 status]
|
||||
}
|
||||
|
||||
func (d *diffReport) Meaningful() bool {
|
||||
// Ignore small diffs in discovery (sources can time out differently).
|
||||
if len(d.OnlyInV1) > 2 || len(d.OnlyInV2) > 2 {
|
||||
return true
|
||||
}
|
||||
if len(d.StatusDiff) > 0 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func compare(v1, v2 *scanResult) *diffReport {
|
||||
d := &diffReport{Target: v1.Target, StatusDiff: map[string][2]int{}}
|
||||
|
||||
v1set := keySet(v1.Subdomains)
|
||||
v2set := keySet(v2.Subdomains)
|
||||
|
||||
for k := range v1set {
|
||||
if _, ok := v2set[k]; !ok {
|
||||
d.OnlyInV1 = append(d.OnlyInV1, k)
|
||||
} else {
|
||||
d.Shared = append(d.Shared, k)
|
||||
}
|
||||
}
|
||||
for k := range v2set {
|
||||
if _, ok := v1set[k]; !ok {
|
||||
d.OnlyInV2 = append(d.OnlyInV2, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(d.OnlyInV1)
|
||||
sort.Strings(d.OnlyInV2)
|
||||
sort.Strings(d.Shared)
|
||||
|
||||
for _, k := range d.Shared {
|
||||
a := v1.Subdomains[k].StatusCode
|
||||
b := v2.Subdomains[k].StatusCode
|
||||
if a != b && a != 0 && b != 0 {
|
||||
d.StatusDiff[k] = [2]int{a, b}
|
||||
}
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func keySet(m map[string]hostInfo) map[string]struct{} {
|
||||
out := make(map[string]struct{}, len(m))
|
||||
for k := range m {
|
||||
out[k] = struct{}{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func printDiff(d *diffReport) {
|
||||
fmt.Println()
|
||||
fmt.Printf("=== Parity report for %s ===\n", d.Target)
|
||||
fmt.Printf(" Shared subdomains: %d\n", len(d.Shared))
|
||||
fmt.Printf(" Only in v1 : %d\n", len(d.OnlyInV1))
|
||||
fmt.Printf(" Only in v2 : %d\n", len(d.OnlyInV2))
|
||||
fmt.Printf(" Status code diff: %d\n", len(d.StatusDiff))
|
||||
|
||||
if len(d.OnlyInV1) > 0 {
|
||||
fmt.Println(" -- v1 only --")
|
||||
for _, s := range d.OnlyInV1 {
|
||||
fmt.Println(" ", s)
|
||||
}
|
||||
}
|
||||
if len(d.OnlyInV2) > 0 {
|
||||
fmt.Println(" -- v2 only --")
|
||||
for _, s := range d.OnlyInV2 {
|
||||
fmt.Println(" ", s)
|
||||
}
|
||||
}
|
||||
if len(d.StatusDiff) > 0 {
|
||||
fmt.Println(" -- status differences --")
|
||||
for k, pair := range d.StatusDiff {
|
||||
fmt.Printf(" %s: v1=%d v2=%d\n", k, pair[0], pair[1])
|
||||
}
|
||||
}
|
||||
if d.Meaningful() {
|
||||
fmt.Println()
|
||||
fmt.Println("RESULT: meaningful divergence detected.")
|
||||
} else {
|
||||
fmt.Println()
|
||||
fmt.Println("RESULT: v1 and v2 agree (differences within acceptable noise).")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user