mirror of
https://github.com/Vyntral/god-eye.git
synced 2026-05-29 18:49:27 +02:00
feat: v2.0 full rewrite — event-driven pipeline, AI + Nuclei + proxy
Complete architectural overhaul. Replaces the v0.1 monolithic scanner with an event-driven pipeline of auto-registered modules. Foundation (internal/): - eventbus: typed pub/sub, 20 event types, race-safe, drop counter - module: registry with phase-based selection - store: thread-safe host store with per-host locks + deep-copy reads - pipeline: coordinator with phase barriers + panic recovery - config: 5 scan profiles + 3 AI tiers + YAML loader + auto-discovery Modules (26 auto-registered across 6 phases): - Discovery: passive (26 sources), bruteforce, recursive, AXFR, GitHub dorks, CT streaming, permutation, reverse DNS, vhost, ASN, supply chain (npm + PyPI) - Enrichment: HTTP probe + tech fingerprint + TLS appliance ID, ports - Analysis: security checks, takeover (110+ sigs), cloud, JavaScript, GraphQL, JWT, headers (OWASP), HTTP smuggling, AI cascade, Nuclei - Reporting: TXT/JSON/CSV writer + AI scan brief AI layer (internal/ai/ + internal/modules/ai/): - Three profiles: lean (16 GB), balanced (32 GB MoE), heavy (64 GB) - Six event-driven handlers: CVE, JS file, HTTP response, secret filter, multi-agent vuln enrichment, anomaly + executive report - Content-hash cache dedups Ollama calls across hosts - Auto-pull of missing models via /api/pull with streaming progress - End-of-scan AI SCAN BRIEF in terminal with top chains + next actions Nuclei compat layer (internal/nucleitpl/): - Executes ~13k community templates (HTTP subset) - Auto-download of nuclei-templates ZIP to ~/.god-eye/nuclei-templates - Scope filter rejects off-host templates (eliminates OSINT FPs) Operations: - Interactive wizard (internal/wizard/) — zero-flag launch - LivePrinter (internal/tui/) — colorized event stream - Diff engine + scheduler (internal/diff, internal/scheduler) for continuous ASM monitoring with webhook alerts - Proxy support (internal/proxyconf/): http / https / socks5 / socks5h + basic auth Fixes #1 — native SOCKS5 / Tor compatibility via --proxy flag. 185 unit tests across 15 packages, all race-detector clean.
This commit is contained in:
Vyntral
@@ -0,0 +1,270 @@
|
||||
// Package diff computes deltas between two scans of the same target. It
|
||||
// powers Fase 5's asm-continuous mode: run the scanner on a schedule, diff
|
||||
// against the last snapshot, alert on meaningful changes.
|
||||
//
|
||||
// Diff categories:
|
||||
//
|
||||
// new_host — subdomain not seen before
|
||||
// removed_host — subdomain vanished from discovery
|
||||
// new_ip — host gained an IP
|
||||
// removed_ip — host lost an IP
|
||||
// status_change — HTTP status code changed (200→401, 200→gone)
|
||||
// tech_change — technology stack changed (upgrade or new framework)
|
||||
// new_vuln — new vulnerability finding
|
||||
// cleared_vuln — previously-reported vuln no longer detected
|
||||
// cert_change — TLS certificate issuer/expiry changed
|
||||
// new_takeover — new takeover candidate
|
||||
//
|
||||
// A Report is consumable both by humans (pretty-print) and by alerters
|
||||
// (Slack/webhook payload shape defined later in F5.3).
|
||||
package diff
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"god-eye/internal/store"
|
||||
)
|
||||
|
||||
// Change is one delta.
|
||||
type Change struct {
|
||||
Kind string `json:"kind"`
|
||||
Host string `json:"host"`
|
||||
Before string `json:"before,omitempty"`
|
||||
After string `json:"after,omitempty"`
|
||||
Severity string `json:"severity,omitempty"`
|
||||
Detected time.Time `json:"detected_at"`
|
||||
}
|
||||
|
||||
// Report is the full delta between two scans.
|
||||
type Report struct {
|
||||
Target string `json:"target"`
|
||||
OldAt time.Time `json:"old_scan_at"`
|
||||
NewAt time.Time `json:"new_scan_at"`
|
||||
Changes []Change `json:"changes"`
|
||||
}
|
||||
|
||||
// HasMeaningful returns true when the report contains any change that
|
||||
// warrants alerting. "new_host" and any "new_vuln" always qualify.
|
||||
func (r *Report) HasMeaningful() bool {
|
||||
for _, c := range r.Changes {
|
||||
switch c.Kind {
|
||||
case "new_host", "new_vuln", "new_takeover", "removed_host":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Compute compares old vs new snapshots and returns the delta. Both
|
||||
// slices are assumed to come from store.All() (sorted by subdomain).
|
||||
func Compute(target string, oldHosts, newHosts []*store.Host, oldAt, newAt time.Time) *Report {
|
||||
r := &Report{Target: target, OldAt: oldAt, NewAt: newAt}
|
||||
|
||||
oldByName := indexHosts(oldHosts)
|
||||
newByName := indexHosts(newHosts)
|
||||
|
||||
// Walk the union of hostnames.
|
||||
names := union(oldByName, newByName)
|
||||
sort.Strings(names)
|
||||
|
||||
for _, name := range names {
|
||||
o, oOK := oldByName[name]
|
||||
n, nOK := newByName[name]
|
||||
switch {
|
||||
case !oOK && nOK:
|
||||
r.Changes = append(r.Changes, Change{Kind: "new_host", Host: name, Detected: newAt})
|
||||
for _, v := range n.Vulnerabilities {
|
||||
r.Changes = append(r.Changes, Change{
|
||||
Kind: "new_vuln",
|
||||
Host: name,
|
||||
After: v.Title,
|
||||
Severity: v.Severity,
|
||||
Detected: newAt,
|
||||
})
|
||||
}
|
||||
if n.Takeover != nil {
|
||||
r.Changes = append(r.Changes, Change{
|
||||
Kind: "new_takeover",
|
||||
Host: name,
|
||||
After: n.Takeover.Service,
|
||||
Severity: "high",
|
||||
Detected: newAt,
|
||||
})
|
||||
}
|
||||
case oOK && !nOK:
|
||||
r.Changes = append(r.Changes, Change{Kind: "removed_host", Host: name, Detected: newAt})
|
||||
case oOK && nOK:
|
||||
r.Changes = append(r.Changes, diffHost(o, n, newAt)...)
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func diffHost(o, n *store.Host, at time.Time) []Change {
|
||||
var out []Change
|
||||
|
||||
if o.StatusCode != n.StatusCode {
|
||||
out = append(out, Change{
|
||||
Kind: "status_change",
|
||||
Host: n.Subdomain,
|
||||
Before: itoa(o.StatusCode),
|
||||
After: itoa(n.StatusCode),
|
||||
Detected: at,
|
||||
})
|
||||
}
|
||||
|
||||
// IP deltas
|
||||
oldIPs := toSet(o.IPs)
|
||||
newIPs := toSet(n.IPs)
|
||||
for ip := range newIPs {
|
||||
if _, present := oldIPs[ip]; !present {
|
||||
out = append(out, Change{Kind: "new_ip", Host: n.Subdomain, After: ip, Detected: at})
|
||||
}
|
||||
}
|
||||
for ip := range oldIPs {
|
||||
if _, present := newIPs[ip]; !present {
|
||||
out = append(out, Change{Kind: "removed_ip", Host: n.Subdomain, Before: ip, Detected: at})
|
||||
}
|
||||
}
|
||||
|
||||
// Tech change (set inequality)
|
||||
if !stringSetsEqual(o.Technologies, n.Technologies) {
|
||||
out = append(out, Change{
|
||||
Kind: "tech_change",
|
||||
Host: n.Subdomain,
|
||||
Before: joinSorted(o.Technologies),
|
||||
After: joinSorted(n.Technologies),
|
||||
Detected: at,
|
||||
})
|
||||
}
|
||||
|
||||
// Vuln delta (by ID)
|
||||
oldVulns := indexVulns(o.Vulnerabilities)
|
||||
newVulns := indexVulns(n.Vulnerabilities)
|
||||
for id, v := range newVulns {
|
||||
if _, present := oldVulns[id]; !present {
|
||||
out = append(out, Change{
|
||||
Kind: "new_vuln", Host: n.Subdomain, After: v.Title,
|
||||
Severity: v.Severity, Detected: at,
|
||||
})
|
||||
}
|
||||
}
|
||||
for id, v := range oldVulns {
|
||||
if _, present := newVulns[id]; !present {
|
||||
out = append(out, Change{
|
||||
Kind: "cleared_vuln", Host: n.Subdomain, Before: v.Title,
|
||||
Severity: v.Severity, Detected: at,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Certificate change
|
||||
if o.TLSIssuer != n.TLSIssuer && n.TLSIssuer != "" {
|
||||
out = append(out, Change{
|
||||
Kind: "cert_change",
|
||||
Host: n.Subdomain,
|
||||
Before: o.TLSIssuer,
|
||||
After: n.TLSIssuer,
|
||||
Detected: at,
|
||||
})
|
||||
}
|
||||
|
||||
// Takeover appeared
|
||||
if o.Takeover == nil && n.Takeover != nil {
|
||||
out = append(out, Change{
|
||||
Kind: "new_takeover", Host: n.Subdomain,
|
||||
After: n.Takeover.Service, Severity: "high", Detected: at,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- helpers -------------------------------------------------------------
|
||||
|
||||
func indexHosts(hs []*store.Host) map[string]*store.Host {
|
||||
out := make(map[string]*store.Host, len(hs))
|
||||
for _, h := range hs {
|
||||
out[h.Subdomain] = h
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func indexVulns(vs []store.Vulnerability) map[string]store.Vulnerability {
|
||||
out := make(map[string]store.Vulnerability, len(vs))
|
||||
for _, v := range vs {
|
||||
out[v.ID] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func union(a, b map[string]*store.Host) []string {
|
||||
out := make(map[string]struct{}, len(a)+len(b))
|
||||
for k := range a {
|
||||
out[k] = struct{}{}
|
||||
}
|
||||
for k := range b {
|
||||
out[k] = struct{}{}
|
||||
}
|
||||
names := make([]string, 0, len(out))
|
||||
for n := range out {
|
||||
names = append(names, n)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func toSet(ss []string) map[string]struct{} {
|
||||
out := make(map[string]struct{}, len(ss))
|
||||
for _, s := range ss {
|
||||
out[s] = struct{}{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func stringSetsEqual(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
sa := toSet(a)
|
||||
for _, s := range b {
|
||||
if _, ok := sa[s]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func joinSorted(s []string) string {
|
||||
cpy := append([]string(nil), s...)
|
||||
sort.Strings(cpy)
|
||||
out := ""
|
||||
for i, v := range cpy {
|
||||
if i > 0 {
|
||||
out += ","
|
||||
}
|
||||
out += v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func itoa(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
var buf [20]byte
|
||||
i := len(buf)
|
||||
neg := n < 0
|
||||
if neg {
|
||||
n = -n
|
||||
}
|
||||
for n > 0 {
|
||||
i--
|
||||
buf[i] = byte('0' + n%10)
|
||||
n /= 10
|
||||
}
|
||||
if neg {
|
||||
i--
|
||||
buf[i] = '-'
|
||||
}
|
||||
return string(buf[i:])
|
||||
}
|
||||
Reference in New Issue
Block a user