v0.1.1: Major AI improvements, new security modules, and documentation fixes

## AI & CVE Improvements
- Fix AI report to display actual subdomain names instead of generic placeholders
- Add 10-year CVE filter to reduce false positives from outdated vulnerabilities
- Integrate CISA KEV (Known Exploited Vulnerabilities) database support
- Improve AI analysis prompt for more accurate security findings

## New Security Modules
- Add wildcard DNS detection with multi-phase validation (DNS + HTTP)
- Add TLS certificate analyzer for certificate chain inspection
- Add comprehensive rate limiting module for API requests
- Add retry mechanism with exponential backoff
- Add stealth mode for reduced detection during scans
- Add progress tracking module for better UX

## Code Refactoring
- Extract scanner output logic to dedicated module
- Add base source interface for consistent passive source implementation
- Reduce admin panel paths to common generic patterns only
- Improve HTTP client with connection pooling
- Add JSON output formatter

## Documentation Updates
- Correct passive source count to 20 (was incorrectly stated as 34)
- Fix AI model names: deepseek-r1:1.5b (fast) + qwen2.5-coder:7b (deep)
- Update all markdown files for consistency
- Relocate demo GIFs to assets/ directory
- Add benchmark disclaimer for test variability

## Files Changed
- 4 documentation files updated (README, AI_SETUP, BENCHMARK, EXAMPLES)
- 11 new source files added
- 12 existing files modified
This commit is contained in:
Vyntral
2025-11-21 12:00:58 +01:00
parent 2be93a0dbb
commit 45295bb262
28 changed files with 4650 additions and 882 deletions
+42
View File
@@ -3,6 +3,7 @@ package http
import (
"crypto/tls"
"net/http"
"sync"
"time"
)
@@ -25,3 +26,44 @@ func GetSharedClient(timeout int) *http.Client {
},
}
}
// UserAgentManager handles User-Agent rotation
type UserAgentManager struct {
agents []string
index int
mu sync.Mutex
}
var defaultUAManager = &UserAgentManager{
agents: []string{
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0",
},
}
// GetUserAgent returns the next User-Agent in rotation
func GetUserAgent() string {
defaultUAManager.mu.Lock()
defer defaultUAManager.mu.Unlock()
ua := defaultUAManager.agents[defaultUAManager.index]
defaultUAManager.index = (defaultUAManager.index + 1) % len(defaultUAManager.agents)
return ua
}
// NewRequestWithUA creates an HTTP request with a rotated User-Agent
func NewRequestWithUA(method, url string) (*http.Request, error) {
req, err := http.NewRequest(method, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", GetUserAgent())
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
req.Header.Set("Connection", "keep-alive")
return req, nil
}
+43 -19
View File
@@ -15,17 +15,8 @@ import (
func ProbeHTTP(subdomain string, timeout int) *config.SubdomainResult {
result := &config.SubdomainResult{}
transport := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{
Timeout: time.Duration(timeout) * time.Second,
Transport: transport,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
}
// Use shared transport for connection pooling
client := GetSharedClient(timeout)
urls := []string{
fmt.Sprintf("https://%s", subdomain),
@@ -78,6 +69,19 @@ func ProbeHTTP(subdomain string, timeout int) *config.SubdomainResult {
case tls.VersionTLS10:
result.TLSVersion = "TLS 1.0"
}
// Check for self-signed certificate
result.TLSSelfSigned = IsSelfSigned(cert)
// Analyze certificate for appliance fingerprinting
// This is especially useful for self-signed certs (firewalls, VPNs, etc.)
if fp := AnalyzeTLSCertificate(cert); fp != nil {
result.TLSFingerprint = fp
// Add vendor/product to tech stack if detected
if fp.Vendor != "" && fp.Product != "" {
result.Tech = append(result.Tech, fp.Vendor+" "+fp.Product)
}
}
}
// Interesting headers
@@ -117,27 +121,47 @@ func ProbeHTTP(subdomain string, timeout int) *config.SubdomainResult {
// Detect technologies
bodyStr := string(body)
if strings.Contains(bodyStr, "wp-content") || strings.Contains(bodyStr, "wordpress") {
bodyStrLower := strings.ToLower(bodyStr)
// WordPress - specific patterns
if strings.Contains(bodyStr, "wp-content") || strings.Contains(bodyStr, "wp-includes") {
result.Tech = append(result.Tech, "WordPress")
}
if strings.Contains(bodyStr, "_next") || strings.Contains(bodyStr, "Next.js") {
// Next.js - specific patterns (check before React since Next uses React)
if strings.Contains(bodyStr, "/_next/") || strings.Contains(bodyStr, "__NEXT_DATA__") {
result.Tech = append(result.Tech, "Next.js")
}
if strings.Contains(bodyStr, "react") || strings.Contains(bodyStr, "React") {
} else if strings.Contains(bodyStr, "react-root") || strings.Contains(bodyStr, "data-reactroot") ||
strings.Contains(bodyStr, "__REACT_DEVTOOLS_GLOBAL_HOOK__") {
// React - only if not Next.js
result.Tech = append(result.Tech, "React")
}
if strings.Contains(bodyStr, "laravel") || strings.Contains(bodyStr, "Laravel") {
// Laravel - specific patterns
if strings.Contains(bodyStr, "laravel_session") || strings.Contains(bodyStr, "XSRF-TOKEN") {
result.Tech = append(result.Tech, "Laravel")
}
if strings.Contains(bodyStr, "django") || strings.Contains(bodyStr, "Django") {
// Django - specific patterns
if strings.Contains(bodyStr, "csrfmiddlewaretoken") || strings.Contains(bodyStrLower, "django") {
result.Tech = append(result.Tech, "Django")
}
if strings.Contains(bodyStr, "angular") || strings.Contains(bodyStr, "ng-") {
// Angular - more specific patterns (ng-app, ng-controller are Angular 1.x specific)
if strings.Contains(bodyStr, "ng-app") || strings.Contains(bodyStr, "ng-controller") ||
strings.Contains(bodyStr, "ng-version") || strings.Contains(bodyStrLower, "angular.js") ||
strings.Contains(bodyStrLower, "angular.min.js") || strings.Contains(bodyStr, "@angular/core") {
result.Tech = append(result.Tech, "Angular")
}
if strings.Contains(bodyStr, "vue") || strings.Contains(bodyStr, "Vue.js") {
// Vue.js - specific patterns
if strings.Contains(bodyStr, "data-v-") || strings.Contains(bodyStr, "__VUE__") ||
strings.Contains(bodyStr, "vue.js") || strings.Contains(bodyStr, "vue.min.js") {
result.Tech = append(result.Tech, "Vue.js")
}
// Svelte
if strings.Contains(bodyStr, "svelte") && strings.Contains(bodyStr, "__svelte") {
result.Tech = append(result.Tech, "Svelte")
}
// Nuxt.js (Vue-based)
if strings.Contains(bodyStr, "__NUXT__") || strings.Contains(bodyStr, "_nuxt/") {
result.Tech = append(result.Tech, "Nuxt.js")
}
}
break
+487
View File
@@ -0,0 +1,487 @@
package http
import (
"crypto/x509"
"regexp"
"strings"
"god-eye/internal/config"
)
// AppliancePattern defines a pattern to match vendor/product from certificate fields
type AppliancePattern struct {
Vendor string
Product string
ApplianceType string // firewall, vpn, loadbalancer, proxy, waf, appliance
// Match patterns (any match triggers detection)
SubjectCNPatterns []string
SubjectOrgPatterns []string
SubjectOUPatterns []string
IssuerCNPatterns []string
IssuerOrgPatterns []string
// Version extraction regex (optional)
VersionRegex string
}
// appliancePatterns contains known signatures for security appliances
var appliancePatterns = []AppliancePattern{
// Fortinet FortiGate
{
Vendor: "Fortinet",
Product: "FortiGate",
ApplianceType: "firewall",
SubjectCNPatterns: []string{
"FortiGate", "FGT", "fortinet", "FGVM",
},
SubjectOrgPatterns: []string{"Fortinet"},
IssuerCNPatterns: []string{"FortiGate", "Fortinet"},
IssuerOrgPatterns: []string{"Fortinet"},
VersionRegex: `(?i)(?:FortiGate|FGT|FGVM)[_-]?(\d+[A-Z]?)`,
},
// Palo Alto Networks
{
Vendor: "Palo Alto Networks",
Product: "PAN-OS",
ApplianceType: "firewall",
SubjectCNPatterns: []string{
"PA-", "Palo Alto", "PAN-OS", "paloaltonetworks",
},
SubjectOrgPatterns: []string{"Palo Alto Networks", "paloaltonetworks"},
IssuerOrgPatterns: []string{"Palo Alto Networks"},
VersionRegex: `(?i)PA-(\d+)`,
},
// Cisco ASA
{
Vendor: "Cisco",
Product: "ASA",
ApplianceType: "firewall",
SubjectCNPatterns: []string{
"ASA", "Cisco ASA", "adaptive security",
},
SubjectOrgPatterns: []string{"Cisco"},
IssuerOrgPatterns: []string{"Cisco"},
VersionRegex: `(?i)ASA[_-]?(\d+)`,
},
// Cisco Firepower
{
Vendor: "Cisco",
Product: "Firepower",
ApplianceType: "firewall",
SubjectCNPatterns: []string{
"Firepower", "FTD", "FMC",
},
SubjectOrgPatterns: []string{"Cisco"},
},
// SonicWall
{
Vendor: "SonicWall",
Product: "SonicWall",
ApplianceType: "firewall",
SubjectCNPatterns: []string{
"SonicWall", "sonicwall", "SonicOS", "NSA", "TZ",
},
SubjectOrgPatterns: []string{"SonicWall", "SonicWALL"},
IssuerOrgPatterns: []string{"SonicWall", "SonicWALL"},
VersionRegex: `(?i)(?:NSA|TZ)[\s-]?(\d+)`,
},
// Check Point
{
Vendor: "Check Point",
Product: "Gaia",
ApplianceType: "firewall",
SubjectCNPatterns: []string{
"Check Point", "checkpoint", "Gaia", "SmartCenter",
},
SubjectOrgPatterns: []string{"Check Point"},
IssuerOrgPatterns: []string{"Check Point"},
},
// F5 BIG-IP
{
Vendor: "F5",
Product: "BIG-IP",
ApplianceType: "loadbalancer",
SubjectCNPatterns: []string{
"BIG-IP", "BIGIP", "F5 Networks", "f5.com",
},
SubjectOrgPatterns: []string{"F5 Networks", "F5, Inc"},
IssuerOrgPatterns: []string{"F5 Networks", "F5, Inc"},
VersionRegex: `(?i)BIG-IP\s+(\d+\.\d+)`,
},
// Citrix NetScaler / ADC
{
Vendor: "Citrix",
Product: "NetScaler",
ApplianceType: "loadbalancer",
SubjectCNPatterns: []string{
"NetScaler", "Citrix ADC", "ns.citrix", "citrix.com",
},
SubjectOrgPatterns: []string{"Citrix"},
IssuerOrgPatterns: []string{"Citrix"},
},
// Juniper
{
Vendor: "Juniper",
Product: "Junos",
ApplianceType: "firewall",
SubjectCNPatterns: []string{
"Juniper", "JunOS", "SRX", "juniper.net",
},
SubjectOrgPatterns: []string{"Juniper Networks"},
IssuerOrgPatterns: []string{"Juniper Networks"},
VersionRegex: `(?i)SRX[_-]?(\d+)`,
},
// Barracuda
{
Vendor: "Barracuda",
Product: "Barracuda",
ApplianceType: "waf",
SubjectCNPatterns: []string{
"Barracuda", "barracuda", "cudatel",
},
SubjectOrgPatterns: []string{"Barracuda Networks"},
IssuerOrgPatterns: []string{"Barracuda Networks"},
},
// pfSense
{
Vendor: "Netgate",
Product: "pfSense",
ApplianceType: "firewall",
SubjectCNPatterns: []string{
"pfSense", "pfsense", "Netgate",
},
SubjectOrgPatterns: []string{"pfSense", "Netgate"},
},
// OPNsense
{
Vendor: "Deciso",
Product: "OPNsense",
ApplianceType: "firewall",
SubjectCNPatterns: []string{
"OPNsense", "opnsense",
},
},
// WatchGuard
{
Vendor: "WatchGuard",
Product: "Firebox",
ApplianceType: "firewall",
SubjectCNPatterns: []string{
"WatchGuard", "Firebox", "watchguard",
},
SubjectOrgPatterns: []string{"WatchGuard"},
},
// Sophos
{
Vendor: "Sophos",
Product: "XG Firewall",
ApplianceType: "firewall",
SubjectCNPatterns: []string{
"Sophos", "sophos", "XG Firewall", "Cyberoam",
},
SubjectOrgPatterns: []string{"Sophos"},
},
// Ubiquiti
{
Vendor: "Ubiquiti",
Product: "UniFi",
ApplianceType: "appliance",
SubjectCNPatterns: []string{
"Ubiquiti", "UniFi", "UBNT", "ubnt.com",
},
SubjectOrgPatterns: []string{"Ubiquiti"},
},
// MikroTik
{
Vendor: "MikroTik",
Product: "RouterOS",
ApplianceType: "router",
SubjectCNPatterns: []string{
"MikroTik", "mikrotik", "RouterOS",
},
SubjectOrgPatterns: []string{"MikroTik"},
},
// OpenVPN
{
Vendor: "OpenVPN",
Product: "OpenVPN AS",
ApplianceType: "vpn",
SubjectCNPatterns: []string{
"OpenVPN", "openvpn",
},
SubjectOrgPatterns: []string{"OpenVPN"},
},
// Pulse Secure / Ivanti
{
Vendor: "Pulse Secure",
Product: "Pulse Connect Secure",
ApplianceType: "vpn",
SubjectCNPatterns: []string{
"Pulse Secure", "pulse", "Ivanti",
},
SubjectOrgPatterns: []string{"Pulse Secure", "Ivanti"},
},
// GlobalProtect (Palo Alto VPN)
{
Vendor: "Palo Alto Networks",
Product: "GlobalProtect",
ApplianceType: "vpn",
SubjectCNPatterns: []string{
"GlobalProtect", "globalprotect",
},
},
// Cisco AnyConnect
{
Vendor: "Cisco",
Product: "AnyConnect",
ApplianceType: "vpn",
SubjectCNPatterns: []string{
"AnyConnect", "anyconnect",
},
SubjectOrgPatterns: []string{"Cisco"},
},
// VMware NSX / vSphere
{
Vendor: "VMware",
Product: "NSX",
ApplianceType: "appliance",
SubjectCNPatterns: []string{
"NSX", "vSphere", "VMware", "vcenter",
},
SubjectOrgPatterns: []string{"VMware"},
},
// Imperva / Incapsula
{
Vendor: "Imperva",
Product: "WAF",
ApplianceType: "waf",
SubjectCNPatterns: []string{
"Imperva", "Incapsula",
},
SubjectOrgPatterns: []string{"Imperva", "Incapsula"},
},
// HAProxy
{
Vendor: "HAProxy",
Product: "HAProxy",
ApplianceType: "loadbalancer",
SubjectCNPatterns: []string{
"HAProxy", "haproxy",
},
},
// NGINX Plus
{
Vendor: "NGINX",
Product: "NGINX Plus",
ApplianceType: "loadbalancer",
SubjectCNPatterns: []string{
"NGINX", "nginx.com",
},
SubjectOrgPatterns: []string{"NGINX", "F5 NGINX"},
},
// Kemp LoadMaster
{
Vendor: "Kemp",
Product: "LoadMaster",
ApplianceType: "loadbalancer",
SubjectCNPatterns: []string{
"Kemp", "LoadMaster",
},
SubjectOrgPatterns: []string{"Kemp Technologies"},
},
// Zyxel
{
Vendor: "Zyxel",
Product: "USG",
ApplianceType: "firewall",
SubjectCNPatterns: []string{
"Zyxel", "zyxel", "USG",
},
SubjectOrgPatterns: []string{"Zyxel"},
},
// DrayTek
{
Vendor: "DrayTek",
Product: "Vigor",
ApplianceType: "router",
SubjectCNPatterns: []string{
"DrayTek", "Vigor", "draytek",
},
SubjectOrgPatterns: []string{"DrayTek"},
},
}
// AnalyzeTLSCertificate analyzes a TLS certificate for appliance fingerprinting
func AnalyzeTLSCertificate(cert *x509.Certificate) *config.TLSFingerprint {
if cert == nil {
return nil
}
fp := &config.TLSFingerprint{
SubjectCN: cert.Subject.CommonName,
SerialNumber: cert.SerialNumber.String(),
}
// Extract organization info
if len(cert.Subject.Organization) > 0 {
fp.SubjectOrg = strings.Join(cert.Subject.Organization, ", ")
}
if len(cert.Subject.OrganizationalUnit) > 0 {
fp.SubjectOU = strings.Join(cert.Subject.OrganizationalUnit, ", ")
}
if len(cert.Issuer.CommonName) > 0 {
fp.IssuerCN = cert.Issuer.CommonName
}
if len(cert.Issuer.Organization) > 0 {
fp.IssuerOrg = strings.Join(cert.Issuer.Organization, ", ")
}
// Extract internal hostnames from DNS names
for _, name := range cert.DNSNames {
if isInternalHostname(name) {
fp.InternalHosts = append(fp.InternalHosts, name)
}
}
// Try to match against known appliance patterns
matchAppliance(fp, cert)
// Only return if we found something interesting
if fp.Vendor != "" || len(fp.InternalHosts) > 0 || fp.SubjectOrg != "" {
return fp
}
return nil
}
// matchAppliance tries to identify the appliance vendor/product
func matchAppliance(fp *config.TLSFingerprint, cert *x509.Certificate) {
subjectCN := strings.ToLower(cert.Subject.CommonName)
subjectOrg := strings.ToLower(strings.Join(cert.Subject.Organization, " "))
subjectOU := strings.ToLower(strings.Join(cert.Subject.OrganizationalUnit, " "))
issuerCN := strings.ToLower(cert.Issuer.CommonName)
issuerOrg := strings.ToLower(strings.Join(cert.Issuer.Organization, " "))
for _, pattern := range appliancePatterns {
matched := false
// Check Subject CN
for _, p := range pattern.SubjectCNPatterns {
if strings.Contains(subjectCN, strings.ToLower(p)) {
matched = true
break
}
}
// Check Subject Organization
if !matched {
for _, p := range pattern.SubjectOrgPatterns {
if strings.Contains(subjectOrg, strings.ToLower(p)) {
matched = true
break
}
}
}
// Check Subject OU
if !matched {
for _, p := range pattern.SubjectOUPatterns {
if strings.Contains(subjectOU, strings.ToLower(p)) {
matched = true
break
}
}
}
// Check Issuer CN
if !matched {
for _, p := range pattern.IssuerCNPatterns {
if strings.Contains(issuerCN, strings.ToLower(p)) {
matched = true
break
}
}
}
// Check Issuer Organization
if !matched {
for _, p := range pattern.IssuerOrgPatterns {
if strings.Contains(issuerOrg, strings.ToLower(p)) {
matched = true
break
}
}
}
if matched {
fp.Vendor = pattern.Vendor
fp.Product = pattern.Product
fp.ApplianceType = pattern.ApplianceType
// Try to extract version
if pattern.VersionRegex != "" {
re := regexp.MustCompile(pattern.VersionRegex)
// Check all relevant fields
for _, field := range []string{cert.Subject.CommonName, cert.Issuer.CommonName} {
if matches := re.FindStringSubmatch(field); len(matches) > 1 {
fp.Version = matches[1]
break
}
}
}
return
}
}
}
// isInternalHostname checks if a hostname looks like an internal name
func isInternalHostname(name string) bool {
name = strings.ToLower(name)
// Common internal TLDs
internalTLDs := []string{
".local", ".internal", ".lan", ".corp", ".home",
".intranet", ".private", ".localdomain",
}
for _, tld := range internalTLDs {
if strings.HasSuffix(name, tld) {
return true
}
}
// Internal hostname patterns
internalPatterns := []string{
"localhost", "fw-", "firewall", "vpn-", "gw-", "gateway",
"proxy-", "lb-", "router", "switch", "core-", "dc-",
"srv-", "server-", "host-", "node-", "mgmt", "management",
"admin-", "internal-", "private-", "corp-", "office-",
}
for _, pattern := range internalPatterns {
if strings.Contains(name, pattern) {
return true
}
}
// IP-like patterns in hostname
ipPattern := regexp.MustCompile(`\d{1,3}[.-]\d{1,3}[.-]\d{1,3}[.-]\d{1,3}`)
if ipPattern.MatchString(name) {
return true
}
return false
}
// IsSelfSigned checks if a certificate is self-signed
func IsSelfSigned(cert *x509.Certificate) bool {
if cert == nil {
return false
}
// Check if issuer equals subject
if cert.Issuer.String() == cert.Subject.String() {
return true
}
// Check if it's self-signed by verifying against its own public key
err := cert.CheckSignatureFrom(cert)
return err == nil
}