mirror of
https://github.com/Ed1s0nZ/CyberStrikeAI.git
synced 2026-07-19 10:28:09 +02:00
844 lines
32 KiB
Go
844 lines
32 KiB
Go
package database
|
|
|
|
import (
|
|
"database/sql"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net"
|
|
"net/url"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"github.com/google/uuid"
|
|
"golang.org/x/net/idna"
|
|
)
|
|
|
|
// Asset is a persistent, deduplicated target discovered manually or by recon providers.
|
|
type Asset struct {
|
|
ID string `json:"id"`
|
|
ProjectID string `json:"project_id,omitempty"`
|
|
ProjectName string `json:"project_name,omitempty"`
|
|
Host string `json:"host"`
|
|
IP string `json:"ip"`
|
|
Port int `json:"port"`
|
|
Domain string `json:"domain"`
|
|
Protocol string `json:"protocol"`
|
|
Title string `json:"title"`
|
|
Server string `json:"server"`
|
|
Country string `json:"country"`
|
|
Province string `json:"province"`
|
|
City string `json:"city"`
|
|
Source string `json:"source"`
|
|
SourceQuery string `json:"source_query"`
|
|
Status string `json:"status"`
|
|
Tags []string `json:"tags"`
|
|
FirstSeenAt time.Time `json:"first_seen_at"`
|
|
LastSeenAt time.Time `json:"last_seen_at"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
UpdatedAt time.Time `json:"updated_at"`
|
|
LastScanAt *time.Time `json:"last_scan_at,omitempty"`
|
|
LastScanConversationID string `json:"last_scan_conversation_id,omitempty"`
|
|
LastScanQueueID string `json:"last_scan_queue_id,omitempty"`
|
|
LastScanTaskID string `json:"last_scan_task_id,omitempty"`
|
|
VulnerabilityCount int `json:"vulnerability_count"`
|
|
RiskLevel string `json:"risk_level"`
|
|
OwnerUserID string `json:"-"`
|
|
}
|
|
|
|
type AssetListFilter struct {
|
|
Search string
|
|
Status string
|
|
Protocol string
|
|
ProjectID string
|
|
Source string
|
|
Tag string
|
|
Host string
|
|
IP string
|
|
Domain string
|
|
Port *int
|
|
ScanState string
|
|
LastScanBefore *time.Time
|
|
LastScanAfter *time.Time
|
|
LastSeenBefore *time.Time
|
|
LastSeenAfter *time.Time
|
|
SortBy string
|
|
SortOrder string
|
|
}
|
|
|
|
type AssetImportResult struct {
|
|
Created int `json:"created"`
|
|
Updated int `json:"updated"`
|
|
Skipped int `json:"skipped"`
|
|
}
|
|
|
|
func normalizeAsset(a *Asset) {
|
|
a.Host = strings.TrimSpace(a.Host)
|
|
a.IP = strings.ToLower(strings.TrimSpace(a.IP))
|
|
a.Domain = strings.ToLower(strings.TrimSpace(a.Domain))
|
|
a.Protocol = strings.ToLower(strings.TrimSpace(a.Protocol))
|
|
a.Title = strings.TrimSpace(a.Title)
|
|
a.Server = strings.TrimSpace(a.Server)
|
|
a.Country = strings.TrimSpace(a.Country)
|
|
a.Province = strings.TrimSpace(a.Province)
|
|
a.City = strings.TrimSpace(a.City)
|
|
a.Source = strings.TrimSpace(a.Source)
|
|
a.SourceQuery = strings.TrimSpace(a.SourceQuery)
|
|
a.ProjectID = strings.TrimSpace(a.ProjectID)
|
|
a.Status = strings.ToLower(strings.TrimSpace(a.Status))
|
|
if a.Status == "" {
|
|
a.Status = "active"
|
|
}
|
|
if a.Source == "" {
|
|
a.Source = "manual"
|
|
}
|
|
seen := map[string]bool{}
|
|
tags := make([]string, 0, len(a.Tags))
|
|
for _, tag := range a.Tags {
|
|
tag = strings.TrimSpace(tag)
|
|
if tag != "" && !seen[tag] {
|
|
seen[tag] = true
|
|
tags = append(tags, tag)
|
|
}
|
|
}
|
|
a.Tags = tags
|
|
// URL 型 Host 是常见输入。缺失的结构化字段在服务端同样补齐,确保
|
|
// API、MCP 与 Web 端产生一致的去重键,而不依赖某个客户端正确解析。
|
|
if strings.Contains(a.Host, "://") {
|
|
if parsed, err := url.Parse(a.Host); err == nil && parsed.Hostname() != "" && parsed.User == nil {
|
|
hostname := strings.Trim(strings.ToLower(parsed.Hostname()), "[]")
|
|
if net.ParseIP(hostname) != nil && a.IP == "" {
|
|
a.IP = hostname
|
|
} else if a.Domain == "" {
|
|
if ascii, err := idna.Lookup.ToASCII(hostname); err == nil {
|
|
a.Domain = strings.ToLower(ascii)
|
|
}
|
|
}
|
|
if a.Protocol == "" {
|
|
a.Protocol = strings.ToLower(parsed.Scheme)
|
|
}
|
|
if a.Port == 0 {
|
|
if parsed.Port() != "" {
|
|
a.Port, _ = strconv.Atoi(parsed.Port())
|
|
} else if a.Protocol == "https" {
|
|
a.Port = 443
|
|
} else if a.Protocol == "http" {
|
|
a.Port = 80
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Recon providers occasionally return placeholders, multiple values, or
|
|
// provider-specific identifiers in structured fields. They are optional
|
|
// enrichment; a valid Host must not make the entire batch fail because one
|
|
// of those fields is dirty.
|
|
if strings.EqualFold(a.Source, "fofa") {
|
|
if a.IP != "" && net.ParseIP(strings.Trim(a.IP, "[]")) == nil {
|
|
a.IP = ""
|
|
}
|
|
if a.Domain != "" {
|
|
ascii, err := idna.Lookup.ToASCII(strings.TrimSuffix(a.Domain, "."))
|
|
if err != nil || !validAssetDomain(ascii) {
|
|
a.Domain = ""
|
|
} else {
|
|
a.Domain = strings.ToLower(ascii)
|
|
}
|
|
}
|
|
if a.Protocol != "" && !assetProtocolPattern.MatchString(a.Protocol) {
|
|
a.Protocol = ""
|
|
}
|
|
}
|
|
}
|
|
|
|
var assetProtocolPattern = regexp.MustCompile(`^[a-z][a-z0-9+.-]{0,31}$`)
|
|
|
|
// AssetValidationError distinguishes user-correctable asset data from storage failures.
|
|
type AssetValidationError struct{ Message string }
|
|
|
|
func (e *AssetValidationError) Error() string { return e.Message }
|
|
|
|
func assetValidationErrorf(format string, args ...interface{}) error {
|
|
return &AssetValidationError{Message: fmt.Sprintf(format, args...)}
|
|
}
|
|
|
|
func validateAsset(a *Asset) error {
|
|
if a == nil {
|
|
return assetValidationErrorf("资产不能为空")
|
|
}
|
|
if a.Host == "" && a.IP == "" && a.Domain == "" {
|
|
return assetValidationErrorf("资产目标不能为空")
|
|
}
|
|
if a.Port < 0 || a.Port > 65535 {
|
|
return assetValidationErrorf("端口必须在 0-65535 之间")
|
|
}
|
|
if a.IP != "" && net.ParseIP(strings.Trim(a.IP, "[]")) == nil {
|
|
return assetValidationErrorf("IP 地址格式无效")
|
|
}
|
|
if a.Domain != "" {
|
|
ascii, err := idna.Lookup.ToASCII(strings.TrimSuffix(a.Domain, "."))
|
|
if err != nil || !validAssetDomain(ascii) {
|
|
return assetValidationErrorf("域名格式无效")
|
|
}
|
|
a.Domain = strings.ToLower(ascii)
|
|
}
|
|
if a.Protocol != "" && !assetProtocolPattern.MatchString(a.Protocol) {
|
|
return assetValidationErrorf("协议格式无效")
|
|
}
|
|
if a.Status != "active" && a.Status != "inactive" {
|
|
return assetValidationErrorf("资产状态必须为 active 或 inactive")
|
|
}
|
|
for name, value := range map[string]string{
|
|
"Host": a.Host, "域名": a.Domain, "协议": a.Protocol, "页面标题": a.Title,
|
|
"服务指纹": a.Server, "国家/地区": a.Country, "省份/州": a.Province, "城市": a.City,
|
|
} {
|
|
limit := 255
|
|
if name == "Host" || name == "页面标题" {
|
|
limit = 500
|
|
}
|
|
if utf8.RuneCountInString(value) > limit {
|
|
return assetValidationErrorf("%s不能超过 %d 个字符", name, limit)
|
|
}
|
|
}
|
|
if len(a.Tags) > 30 {
|
|
return assetValidationErrorf("标签不能超过 30 个")
|
|
}
|
|
for _, tag := range a.Tags {
|
|
if utf8.RuneCountInString(tag) > 64 {
|
|
return assetValidationErrorf("单个标签不能超过 64 个字符")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validAssetDomain(domain string) bool {
|
|
domain = strings.TrimSuffix(strings.ToLower(strings.TrimSpace(domain)), ".")
|
|
if domain == "" || len(domain) > 253 || net.ParseIP(domain) != nil {
|
|
return false
|
|
}
|
|
for _, label := range strings.Split(domain, ".") {
|
|
if len(label) == 0 || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' {
|
|
return false
|
|
}
|
|
for _, r := range label {
|
|
if (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '-' {
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func assetDedupKey(a *Asset) string {
|
|
target := a.Domain
|
|
if target == "" {
|
|
target = a.IP
|
|
}
|
|
if target == "" {
|
|
target = strings.ToLower(a.Host)
|
|
}
|
|
return strings.Join([]string{target, strconv.Itoa(a.Port), a.Protocol}, "|")
|
|
}
|
|
|
|
func appendAssetAccess(query string, args []interface{}, access RBACListAccess, alias string) (string, []interface{}) {
|
|
if strings.TrimSpace(access.UserID) == "" || access.Scope == RBACScopeAll {
|
|
return query, args
|
|
}
|
|
prefix := ""
|
|
if alias != "" {
|
|
prefix = alias + "."
|
|
}
|
|
query += ` AND (` + prefix + `owner_user_id = ? OR EXISTS (
|
|
SELECT 1 FROM rbac_resource_assignments ra
|
|
WHERE ra.user_id = ? AND ra.resource_type = 'asset' AND ra.resource_id = ` + prefix + `id
|
|
) OR (` + prefix + `project_id IS NOT NULL AND ` + prefix + `project_id <> '' AND (
|
|
EXISTS (SELECT 1 FROM projects ap WHERE ap.id=` + prefix + `project_id AND ap.owner_user_id=?)
|
|
OR EXISTS (SELECT 1 FROM rbac_resource_assignments pra WHERE pra.user_id=? AND pra.resource_type='project' AND pra.resource_id=` + prefix + `project_id)
|
|
)))`
|
|
return query, append(args, access.UserID, access.UserID, access.UserID, access.UserID)
|
|
}
|
|
|
|
func (db *DB) UpsertAssets(assets []*Asset, ownerUserID string, allowGlobal ...bool) (AssetImportResult, error) {
|
|
result := AssetImportResult{}
|
|
tx, err := db.Begin()
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
defer tx.Rollback()
|
|
now := time.Now()
|
|
for _, asset := range assets {
|
|
if asset == nil {
|
|
result.Skipped++
|
|
continue
|
|
}
|
|
normalizeAsset(asset)
|
|
if err := validateAsset(asset); err != nil {
|
|
return result, fmt.Errorf("第 %d 个资产无效: %w", result.Created+result.Updated+result.Skipped+1, err)
|
|
}
|
|
key := assetDedupKey(asset)
|
|
if key == "|0|" {
|
|
result.Skipped++
|
|
continue
|
|
}
|
|
var existingID string
|
|
var existingOwner sql.NullString
|
|
err := tx.QueryRow(`SELECT id,owner_user_id FROM assets WHERE dedup_key = ?`, key).Scan(&existingID, &existingOwner)
|
|
tagsJSON, _ := json.Marshal(asset.Tags)
|
|
if err == sql.ErrNoRows {
|
|
asset.ID = uuid.NewString()
|
|
asset.FirstSeenAt, asset.LastSeenAt, asset.CreatedAt, asset.UpdatedAt = now, now, now, now
|
|
_, err = tx.Exec(`INSERT INTO assets (
|
|
id,dedup_key,project_id,host,ip,port,domain,protocol,title,server,country,province,city,source,source_query,status,tags_json,
|
|
first_seen_at,last_seen_at,created_at,updated_at,owner_user_id
|
|
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
|
asset.ID, key, nullIfEmpty(asset.ProjectID), asset.Host, asset.IP, asset.Port, asset.Domain, asset.Protocol, asset.Title, asset.Server,
|
|
asset.Country, asset.Province, asset.City, asset.Source, asset.SourceQuery, asset.Status, string(tagsJSON),
|
|
now, now, now, now, nullIfEmpty(ownerUserID))
|
|
if err != nil {
|
|
return result, fmt.Errorf("创建资产失败: %w", err)
|
|
}
|
|
if ownerUserID != "" {
|
|
if _, err := tx.Exec(`INSERT OR IGNORE INTO rbac_resource_assignments (id,user_id,resource_type,resource_id,created_at) SELECT ?,id,?,?,? FROM rbac_users WHERE id=?`, uuid.NewString(), "asset", asset.ID, now, ownerUserID); err != nil {
|
|
return result, fmt.Errorf("授权新资产失败: %w", err)
|
|
}
|
|
}
|
|
result.Created++
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return result, fmt.Errorf("检查资产去重键失败: %w", err)
|
|
}
|
|
asset.ID = existingID
|
|
global := len(allowGlobal) > 0 && allowGlobal[0]
|
|
if !global && existingOwner.Valid && strings.TrimSpace(existingOwner.String) != "" && strings.TrimSpace(existingOwner.String) != strings.TrimSpace(ownerUserID) {
|
|
result.Skipped++
|
|
continue
|
|
}
|
|
_, err = tx.Exec(`UPDATE assets SET
|
|
host=CASE WHEN ?<>'' THEN ? ELSE host END, ip=CASE WHEN ?<>'' THEN ? ELSE ip END,
|
|
domain=CASE WHEN ?<>'' THEN ? ELSE domain END, protocol=CASE WHEN ?<>'' THEN ? ELSE protocol END,
|
|
title=CASE WHEN ?<>'' THEN ? ELSE title END, server=CASE WHEN ?<>'' THEN ? ELSE server END,
|
|
country=CASE WHEN ?<>'' THEN ? ELSE country END, province=CASE WHEN ?<>'' THEN ? ELSE province END,
|
|
city=CASE WHEN ?<>'' THEN ? ELSE city END, source=CASE WHEN ?<>'' THEN ? ELSE source END,
|
|
source_query=CASE WHEN ?<>'' THEN ? ELSE source_query END, project_id=CASE WHEN ?<>'' THEN ? ELSE project_id END,
|
|
tags_json=CASE WHEN ?<>'[]' THEN ? ELSE tags_json END,
|
|
last_seen_at=?, updated_at=? WHERE id=?`,
|
|
asset.Host, asset.Host, asset.IP, asset.IP, asset.Domain, asset.Domain, asset.Protocol, asset.Protocol,
|
|
asset.Title, asset.Title, asset.Server, asset.Server, asset.Country, asset.Country, asset.Province, asset.Province,
|
|
asset.City, asset.City, asset.Source, asset.Source, asset.SourceQuery, asset.SourceQuery, asset.ProjectID, nullIfEmpty(asset.ProjectID), string(tagsJSON), string(tagsJSON),
|
|
now, now, existingID)
|
|
if err != nil {
|
|
return result, fmt.Errorf("更新资产失败: %w", err)
|
|
}
|
|
if ownerUserID != "" && (!existingOwner.Valid || strings.TrimSpace(existingOwner.String) == ownerUserID) {
|
|
if _, err := tx.Exec(`INSERT OR IGNORE INTO rbac_resource_assignments (id,user_id,resource_type,resource_id,created_at) SELECT ?,id,?,?,? FROM rbac_users WHERE id=?`, uuid.NewString(), "asset", existingID, now, ownerUserID); err != nil {
|
|
return result, fmt.Errorf("授权资产失败: %w", err)
|
|
}
|
|
}
|
|
result.Updated++
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return result, err
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func assetWhere(filter AssetListFilter, access RBACListAccess) (string, []interface{}) {
|
|
query := " WHERE 1=1"
|
|
args := []interface{}{}
|
|
if q := strings.TrimSpace(filter.Search); q != "" {
|
|
pattern := "%" + escapeAssetLike(strings.ToLower(q)) + "%"
|
|
query += ` AND (LOWER(host) LIKE ? ESCAPE '\' OR LOWER(ip) LIKE ? ESCAPE '\' OR LOWER(domain) LIKE ? ESCAPE '\'
|
|
OR LOWER(title) LIKE ? ESCAPE '\' OR LOWER(server) LIKE ? ESCAPE '\' OR LOWER(tags_json) LIKE ? ESCAPE '\')`
|
|
for i := 0; i < 6; i++ {
|
|
args = append(args, pattern)
|
|
}
|
|
}
|
|
if filter.Status != "" {
|
|
query += " AND status = ?"
|
|
args = append(args, filter.Status)
|
|
}
|
|
if filter.Protocol != "" {
|
|
query += " AND protocol = ?"
|
|
args = append(args, filter.Protocol)
|
|
}
|
|
if filter.ProjectID != "" {
|
|
query += " AND assets.project_id = ?"
|
|
args = append(args, filter.ProjectID)
|
|
}
|
|
if filter.Source != "" {
|
|
query += " AND LOWER(assets.source) = LOWER(?)"
|
|
args = append(args, strings.TrimSpace(filter.Source))
|
|
}
|
|
if tag := strings.TrimSpace(filter.Tag); tag != "" {
|
|
pattern := "%\"" + escapeAssetLike(strings.ToLower(tag)) + "\"%"
|
|
query += ` AND LOWER(assets.tags_json) LIKE ? ESCAPE '\'`
|
|
args = append(args, pattern)
|
|
}
|
|
if filter.Host != "" {
|
|
query += " AND LOWER(assets.host) = LOWER(?)"
|
|
args = append(args, strings.TrimSpace(filter.Host))
|
|
}
|
|
if filter.IP != "" {
|
|
query += " AND LOWER(assets.ip) = LOWER(?)"
|
|
args = append(args, strings.TrimSpace(filter.IP))
|
|
}
|
|
if filter.Domain != "" {
|
|
query += " AND LOWER(assets.domain) = LOWER(?)"
|
|
args = append(args, strings.TrimSpace(filter.Domain))
|
|
}
|
|
if filter.Port != nil {
|
|
query += " AND assets.port = ?"
|
|
args = append(args, *filter.Port)
|
|
}
|
|
switch strings.ToLower(strings.TrimSpace(filter.ScanState)) {
|
|
case "never":
|
|
query += " AND " + assetEffectiveLastScanExpr + " IS NULL"
|
|
case "scanned":
|
|
query += " AND " + assetEffectiveLastScanExpr + " IS NOT NULL"
|
|
}
|
|
if filter.LastScanBefore != nil {
|
|
query += " AND " + assetEffectiveLastScanExpr + " < ?"
|
|
args = append(args, *filter.LastScanBefore)
|
|
}
|
|
if filter.LastScanAfter != nil {
|
|
query += " AND " + assetEffectiveLastScanExpr + " > ?"
|
|
args = append(args, *filter.LastScanAfter)
|
|
}
|
|
if filter.LastSeenBefore != nil {
|
|
query += " AND assets.last_seen_at < ?"
|
|
args = append(args, *filter.LastSeenBefore)
|
|
}
|
|
if filter.LastSeenAfter != nil {
|
|
query += " AND assets.last_seen_at > ?"
|
|
args = append(args, *filter.LastSeenAfter)
|
|
}
|
|
return appendAssetAccess(query, args, access, "assets")
|
|
}
|
|
|
|
func escapeAssetLike(value string) string {
|
|
value = strings.ReplaceAll(value, `\`, `\\`)
|
|
value = strings.ReplaceAll(value, `%`, `\%`)
|
|
return strings.ReplaceAll(value, `_`, `\_`)
|
|
}
|
|
|
|
func scanAsset(scanner interface{ Scan(...interface{}) error }) (*Asset, error) {
|
|
var a Asset
|
|
var tags string
|
|
var lastScanAt interface{}
|
|
err := scanner.Scan(&a.ID, &a.ProjectID, &a.ProjectName, &a.Host, &a.IP, &a.Port, &a.Domain, &a.Protocol, &a.Title, &a.Server, &a.Country,
|
|
&a.Province, &a.City, &a.Source, &a.SourceQuery, &a.Status, &tags, &a.FirstSeenAt, &a.LastSeenAt, &a.CreatedAt, &a.UpdatedAt,
|
|
&lastScanAt, &a.LastScanConversationID, &a.LastScanQueueID, &a.LastScanTaskID, &a.VulnerabilityCount, &a.RiskLevel)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if parsed, ok := parseAssetScanTime(lastScanAt); ok {
|
|
a.LastScanAt = &parsed
|
|
}
|
|
_ = json.Unmarshal([]byte(tags), &a.Tags)
|
|
return &a, nil
|
|
}
|
|
|
|
func parseAssetScanTime(value interface{}) (time.Time, bool) {
|
|
if value == nil {
|
|
return time.Time{}, false
|
|
}
|
|
if parsed, ok := value.(time.Time); ok {
|
|
return parsed, true
|
|
}
|
|
var raw string
|
|
switch typed := value.(type) {
|
|
case string:
|
|
raw = typed
|
|
case []byte:
|
|
raw = string(typed)
|
|
default:
|
|
raw = fmt.Sprint(typed)
|
|
}
|
|
for _, layout := range []string{
|
|
time.RFC3339Nano,
|
|
"2006-01-02 15:04:05.999999999-07:00",
|
|
"2006-01-02 15:04:05.999999999Z07:00",
|
|
"2006-01-02 15:04:05-07:00",
|
|
"2006-01-02 15:04:05",
|
|
} {
|
|
if parsed, err := time.Parse(layout, strings.TrimSpace(raw)); err == nil {
|
|
return parsed, true
|
|
}
|
|
}
|
|
return time.Time{}, false
|
|
}
|
|
|
|
const assetEffectiveLastScanExpr = `COALESCE(
|
|
(SELECT bt.completed_at FROM batch_tasks bt WHERE bt.id=assets.last_scan_task_id AND bt.completed_at IS NOT NULL LIMIT 1),
|
|
(SELECT MAX(m.updated_at) FROM messages m WHERE m.conversation_id=assets.last_scan_conversation_id AND m.role='assistant'),
|
|
assets.last_scan_at
|
|
)`
|
|
|
|
const assetSelectColumns = `assets.id,COALESCE(assets.project_id,''),COALESCE(p.name,''),assets.host,assets.ip,assets.port,assets.domain,assets.protocol,assets.title,assets.server,assets.country,
|
|
assets.province,assets.city,assets.source,assets.source_query,assets.status,assets.tags_json,assets.first_seen_at,assets.last_seen_at,assets.created_at,assets.updated_at,
|
|
` + assetEffectiveLastScanExpr + `,COALESCE(assets.last_scan_conversation_id,''),COALESCE(assets.last_scan_queue_id,''),COALESCE(assets.last_scan_task_id,''),
|
|
(SELECT COUNT(DISTINCT v.id) FROM vulnerabilities v WHERE
|
|
(COALESCE(assets.last_scan_conversation_id,'')<>'' AND v.conversation_id=assets.last_scan_conversation_id)
|
|
OR (COALESCE(assets.last_scan_task_id,'')<>'' AND EXISTS (SELECT 1 FROM batch_tasks bt WHERE bt.id=assets.last_scan_task_id AND bt.conversation_id=v.conversation_id))),
|
|
CASE WHEN assets.last_scan_at IS NULL THEN 'unassessed' ELSE CASE COALESCE((
|
|
SELECT MAX(CASE LOWER(COALESCE(v.severity,'')) WHEN 'critical' THEN 5 WHEN 'high' THEN 4 WHEN 'medium' THEN 3 WHEN 'low' THEN 2 WHEN 'info' THEN 1 ELSE 0 END)
|
|
FROM vulnerabilities v WHERE
|
|
LOWER(COALESCE(v.status,'open')) NOT IN ('fixed','false_positive','ignored') AND (
|
|
(COALESCE(assets.last_scan_conversation_id,'')<>'' AND v.conversation_id=assets.last_scan_conversation_id)
|
|
OR (COALESCE(assets.last_scan_task_id,'')<>'' AND EXISTS (SELECT 1 FROM batch_tasks bt WHERE bt.id=assets.last_scan_task_id AND bt.conversation_id=v.conversation_id))
|
|
)
|
|
),0) WHEN 5 THEN 'critical' WHEN 4 THEN 'high' WHEN 3 THEN 'medium' WHEN 2 THEN 'low' WHEN 1 THEN 'info' ELSE 'normal' END END`
|
|
|
|
// MarkAssetScanned links an asset to the conversation or batch subtask created from it.
|
|
// The link lets the asset list show the latest scan time and vulnerabilities produced by that scan.
|
|
func (db *DB) MarkAssetScanned(id, conversationID, queueID, taskID string, access RBACListAccess) error {
|
|
where, args := appendAssetAccess(" WHERE id = ?", []interface{}{strings.TrimSpace(id)}, access, "assets")
|
|
res, err := db.Exec(`UPDATE assets SET last_scan_at=?,last_scan_conversation_id=?,last_scan_queue_id=?,last_scan_task_id=?,updated_at=?`+where,
|
|
append([]interface{}{time.Now(), strings.TrimSpace(conversationID), strings.TrimSpace(queueID), strings.TrimSpace(taskID), time.Now()}, args...)...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n, _ := res.RowsAffected()
|
|
if n == 0 {
|
|
return sql.ErrNoRows
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// CompleteAssetScan records completion from inside an Agent conversation. If
|
|
// the asset was launched as a batch task, keep its task/queue link only when
|
|
// that task belongs to the current conversation; a later ad-hoc chat scan must
|
|
// not retain stale task associations.
|
|
func (db *DB) CompleteAssetScan(id, conversationID string, access RBACListAccess) error {
|
|
id = strings.TrimSpace(id)
|
|
conversationID = strings.TrimSpace(conversationID)
|
|
if conversationID == "" {
|
|
return fmt.Errorf("扫描对话不能为空")
|
|
}
|
|
where, args := appendAssetAccess(" WHERE id = ?", []interface{}{id}, access, "assets")
|
|
now := time.Now()
|
|
res, err := db.Exec(`UPDATE assets SET
|
|
last_scan_at=?,last_scan_conversation_id=?,
|
|
last_scan_queue_id=CASE WHEN EXISTS (SELECT 1 FROM batch_tasks bt WHERE bt.id=assets.last_scan_task_id AND bt.conversation_id=?) THEN last_scan_queue_id ELSE '' END,
|
|
last_scan_task_id=CASE WHEN EXISTS (SELECT 1 FROM batch_tasks bt WHERE bt.id=assets.last_scan_task_id AND bt.conversation_id=?) THEN last_scan_task_id ELSE '' END,
|
|
updated_at=?`+where,
|
|
append([]interface{}{now, conversationID, conversationID, conversationID, now}, args...)...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n, _ := res.RowsAffected()
|
|
if n == 0 {
|
|
return sql.ErrNoRows
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (db *DB) BatchTaskBelongsToQueue(taskID, queueID string) bool {
|
|
var count int
|
|
err := db.QueryRow(`SELECT COUNT(*) FROM batch_tasks WHERE id=? AND queue_id=?`, strings.TrimSpace(taskID), strings.TrimSpace(queueID)).Scan(&count)
|
|
return err == nil && count > 0
|
|
}
|
|
|
|
func (db *DB) ListAssets(limit, offset int, filter AssetListFilter, access RBACListAccess) ([]*Asset, int, error) {
|
|
if limit < 1 {
|
|
limit = 20
|
|
}
|
|
if limit > 100 {
|
|
limit = 100
|
|
}
|
|
if offset < 0 {
|
|
offset = 0
|
|
}
|
|
where, args := assetWhere(filter, access)
|
|
var total int
|
|
if err := db.QueryRow("SELECT COUNT(*) FROM assets"+where, args...).Scan(&total); err != nil {
|
|
return nil, 0, err
|
|
}
|
|
orderBy := assetOrderBy(filter.SortBy, filter.SortOrder)
|
|
rows, err := db.Query("SELECT "+assetSelectColumns+" FROM assets LEFT JOIN projects p ON p.id=assets.project_id"+where+" ORDER BY "+orderBy+" LIMIT ? OFFSET ?", append(args, limit, offset)...)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
defer rows.Close()
|
|
items := []*Asset{}
|
|
for rows.Next() {
|
|
a, err := scanAsset(rows)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
items = append(items, a)
|
|
}
|
|
return items, total, rows.Err()
|
|
}
|
|
|
|
func assetOrderBy(sortBy, sortOrder string) string {
|
|
direction := "DESC"
|
|
if strings.EqualFold(strings.TrimSpace(sortOrder), "asc") {
|
|
direction = "ASC"
|
|
}
|
|
var expression string
|
|
switch strings.ToLower(strings.TrimSpace(sortBy)) {
|
|
case "last_scan_at":
|
|
expression = assetEffectiveLastScanExpr
|
|
// For oldest-first queries, assets that have never been scanned are the
|
|
// most overdue and intentionally appear first. NULLs stay last for DESC.
|
|
if direction == "ASC" {
|
|
return "CASE WHEN " + expression + " IS NULL THEN 0 ELSE 1 END ASC, " + expression + " ASC, assets.id ASC"
|
|
}
|
|
return "CASE WHEN " + expression + " IS NULL THEN 1 ELSE 0 END ASC, " + expression + " DESC, assets.id ASC"
|
|
case "first_seen_at":
|
|
expression = "assets.first_seen_at"
|
|
case "created_at":
|
|
expression = "assets.created_at"
|
|
case "updated_at":
|
|
expression = "assets.updated_at"
|
|
case "host":
|
|
expression = "LOWER(assets.host)"
|
|
case "port":
|
|
expression = "assets.port"
|
|
default:
|
|
expression = "assets.last_seen_at"
|
|
}
|
|
return expression + " " + direction + ", assets.id ASC"
|
|
}
|
|
|
|
func (db *DB) GetAsset(id string, access RBACListAccess) (*Asset, error) {
|
|
query, args := appendAssetAccess("SELECT "+assetSelectColumns+" FROM assets LEFT JOIN projects p ON p.id=assets.project_id WHERE assets.id = ?", []interface{}{id}, access, "assets")
|
|
return scanAsset(db.QueryRow(query, args...))
|
|
}
|
|
|
|
func (db *DB) UpdateAsset(id string, a *Asset, access RBACListAccess) error {
|
|
normalizeAsset(a)
|
|
if err := validateAsset(a); err != nil {
|
|
return err
|
|
}
|
|
key := assetDedupKey(a)
|
|
if key == "|0|" {
|
|
return fmt.Errorf("资产目标不能为空")
|
|
}
|
|
tags, _ := json.Marshal(a.Tags)
|
|
where, args := appendAssetAccess(" WHERE id = ?", []interface{}{id}, access, "assets")
|
|
res, err := db.Exec(`UPDATE assets SET dedup_key=?,project_id=?,host=?,ip=?,port=?,domain=?,protocol=?,title=?,server=?,country=?,province=?,city=?,source=?,source_query=?,status=?,tags_json=?,updated_at=?`+where,
|
|
append([]interface{}{key, nullIfEmpty(a.ProjectID), a.Host, a.IP, a.Port, a.Domain, a.Protocol, a.Title, a.Server, a.Country, a.Province, a.City, a.Source, a.SourceQuery, a.Status, string(tags), time.Now()}, args...)...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n, _ := res.RowsAffected()
|
|
if n == 0 {
|
|
return sql.ErrNoRows
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// UpdateAssetsProject atomically replaces the project binding for every asset.
|
|
// It refuses the whole update when any requested asset is missing or outside
|
|
// the caller's access scope, so a bulk action can never partially succeed.
|
|
func (db *DB) UpdateAssetsProject(ids []string, projectID string, access RBACListAccess) (int, error) {
|
|
unique := make([]string, 0, len(ids))
|
|
seen := make(map[string]struct{}, len(ids))
|
|
for _, id := range ids {
|
|
id = strings.TrimSpace(id)
|
|
if id == "" {
|
|
continue
|
|
}
|
|
if _, exists := seen[id]; exists {
|
|
continue
|
|
}
|
|
seen[id] = struct{}{}
|
|
unique = append(unique, id)
|
|
}
|
|
if len(unique) == 0 {
|
|
return 0, fmt.Errorf("资产列表不能为空")
|
|
}
|
|
|
|
tx, err := db.Begin()
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer tx.Rollback()
|
|
|
|
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(unique)), ",")
|
|
idArgs := make([]interface{}, len(unique))
|
|
for i, id := range unique {
|
|
idArgs[i] = id
|
|
}
|
|
countQuery, countArgs := appendAssetAccess("SELECT COUNT(*) FROM assets WHERE id IN ("+placeholders+")", idArgs, access, "assets")
|
|
var accessible int
|
|
if err := tx.QueryRow(countQuery, countArgs...).Scan(&accessible); err != nil {
|
|
return 0, err
|
|
}
|
|
if accessible != len(unique) {
|
|
return 0, fmt.Errorf("部分资产不存在或无权更新")
|
|
}
|
|
|
|
updateArgs := []interface{}{nullIfEmpty(strings.TrimSpace(projectID)), time.Now()}
|
|
updateArgs = append(updateArgs, idArgs...)
|
|
updateQuery, updateArgs := appendAssetAccess("UPDATE assets SET project_id=?,updated_at=? WHERE id IN ("+placeholders+")", updateArgs, access, "assets")
|
|
result, err := tx.Exec(updateQuery, updateArgs...)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
updated, err := result.RowsAffected()
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if int(updated) != len(unique) {
|
|
return 0, fmt.Errorf("批量更新资产失败")
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return 0, err
|
|
}
|
|
return int(updated), nil
|
|
}
|
|
|
|
func (db *DB) DeleteAsset(id string, access RBACListAccess) error {
|
|
where, args := appendAssetAccess(" WHERE id = ?", []interface{}{id}, access, "assets")
|
|
res, err := db.Exec("DELETE FROM assets"+where, args...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n, _ := res.RowsAffected()
|
|
if n == 0 {
|
|
return sql.ErrNoRows
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (db *DB) GetAssetStats(access RBACListAccess, requestedDays ...int) (map[string]interface{}, error) {
|
|
days := 30
|
|
if len(requestedDays) > 0 && (requestedDays[0] == 7 || requestedDays[0] == 30 || requestedDays[0] == 90) {
|
|
days = requestedDays[0]
|
|
}
|
|
where, args := appendAssetAccess(" WHERE 1=1", nil, access, "assets")
|
|
stats := map[string]interface{}{}
|
|
row := db.QueryRow(`SELECT COUNT(*),COUNT(DISTINCT NULLIF(ip,'')),COUNT(DISTINCT NULLIF(domain,'')),
|
|
COUNT(DISTINCT CASE WHEN port>0 THEN CAST(port AS TEXT) END),
|
|
COALESCE(SUM(CASE WHEN datetime(last_seen_at)>=datetime('now','-7 days') THEN 1 ELSE 0 END),0) FROM assets`+where, args...)
|
|
var total, ips, domains, ports, recent int
|
|
if err := row.Scan(&total, &ips, &domains, &ports, &recent); err != nil {
|
|
return nil, err
|
|
}
|
|
stats["total"], stats["ips"], stats["domains"], stats["ports"], stats["recent"] = total, ips, domains, ports, recent
|
|
rows, err := db.Query(`SELECT CASE WHEN protocol='' THEN 'unknown' ELSE protocol END,COUNT(*) FROM assets`+where+` GROUP BY protocol ORDER BY COUNT(*) DESC LIMIT 8`, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
dist := []map[string]interface{}{}
|
|
for rows.Next() {
|
|
var name string
|
|
var count int
|
|
if err := rows.Scan(&name, &count); err != nil {
|
|
return nil, err
|
|
}
|
|
dist = append(dist, map[string]interface{}{"name": name, "count": count})
|
|
}
|
|
stats["protocols"] = dist
|
|
stats["period_days"] = days
|
|
|
|
coverage := map[string]interface{}{}
|
|
coverageRow := db.QueryRow(`SELECT
|
|
COALESCE(SUM(CASE WHEN last_scan_at IS NOT NULL THEN 1 ELSE 0 END),0),
|
|
COALESCE(SUM(CASE WHEN datetime(last_scan_at)>=datetime('now','-7 days') THEN 1 ELSE 0 END),0),
|
|
COALESCE(SUM(CASE WHEN datetime(last_scan_at)>=datetime('now','-30 days') THEN 1 ELSE 0 END),0),
|
|
COALESCE(SUM(CASE WHEN last_scan_at IS NULL THEN 1 ELSE 0 END),0),
|
|
COALESCE(SUM(CASE WHEN last_scan_at IS NOT NULL AND datetime(last_scan_at)<datetime('now','-30 days') THEN 1 ELSE 0 END),0),
|
|
COALESCE(SUM(CASE WHEN status='active' THEN 1 ELSE 0 END),0)
|
|
FROM assets`+where, args...)
|
|
var scanned, scanned7, scanned30, neverScanned, stale, active int
|
|
if err := coverageRow.Scan(&scanned, &scanned7, &scanned30, &neverScanned, &stale, &active); err != nil {
|
|
return nil, err
|
|
}
|
|
coverage["scanned"], coverage["scanned_7d"], coverage["scanned_30d"] = scanned, scanned7, scanned30
|
|
coverage["never_scanned"], coverage["stale"], coverage["active"] = neverScanned, stale, active
|
|
if total > 0 {
|
|
coverage["rate"] = int(float64(scanned) / float64(total) * 100)
|
|
coverage["recent_rate"] = int(float64(scanned30) / float64(total) * 100)
|
|
} else {
|
|
coverage["rate"], coverage["recent_rate"] = 0, 0
|
|
}
|
|
stats["coverage"] = coverage
|
|
|
|
assetDaily := map[string]map[string]int{}
|
|
trendWhere, trendArgs := appendAssetAccess(" WHERE datetime(first_seen_at)>=datetime('now',?)", []interface{}{fmt.Sprintf("-%d days", days-1)}, access, "assets")
|
|
trendRows, err := db.Query(`SELECT date(first_seen_at), COUNT(*)
|
|
FROM assets`+trendWhere+` GROUP BY date(first_seen_at) ORDER BY date(first_seen_at)`, trendArgs...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for trendRows.Next() {
|
|
var day string
|
|
var added int
|
|
if err := trendRows.Scan(&day, &added); err != nil {
|
|
trendRows.Close()
|
|
return nil, err
|
|
}
|
|
assetDaily[day] = map[string]int{"added": added, "inactive": 0}
|
|
}
|
|
if err := trendRows.Close(); err != nil {
|
|
return nil, err
|
|
}
|
|
inactiveWhere, inactiveArgs := appendAssetAccess(" WHERE status='inactive' AND datetime(updated_at)>=datetime('now',?)", []interface{}{fmt.Sprintf("-%d days", days-1)}, access, "assets")
|
|
inactiveRows, err := db.Query(`SELECT date(updated_at), COUNT(*) FROM assets`+inactiveWhere+` GROUP BY date(updated_at) ORDER BY date(updated_at)`, inactiveArgs...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for inactiveRows.Next() {
|
|
var day string
|
|
var inactive int
|
|
if err := inactiveRows.Scan(&day, &inactive); err != nil {
|
|
inactiveRows.Close()
|
|
return nil, err
|
|
}
|
|
if _, ok := assetDaily[day]; !ok {
|
|
assetDaily[day] = map[string]int{"added": 0, "inactive": 0}
|
|
}
|
|
assetDaily[day]["inactive"] = inactive
|
|
}
|
|
if err := inactiveRows.Close(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
riskDaily := map[string]map[string]int{}
|
|
riskWhere, riskArgs := appendVulnerabilityAccessFilter(" WHERE datetime(created_at)>=datetime('now',?)", []interface{}{fmt.Sprintf("-%d days", days-1)}, access)
|
|
riskRows, err := db.Query(`SELECT date(created_at), COUNT(*),
|
|
COALESCE(SUM(CASE WHEN LOWER(severity) IN ('critical','high') THEN 1 ELSE 0 END),0)
|
|
FROM vulnerabilities`+riskWhere+` GROUP BY date(created_at) ORDER BY date(created_at)`, riskArgs...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for riskRows.Next() {
|
|
var day string
|
|
var discovered, highRisk int
|
|
if err := riskRows.Scan(&day, &discovered, &highRisk); err != nil {
|
|
riskRows.Close()
|
|
return nil, err
|
|
}
|
|
riskDaily[day] = map[string]int{"discovered": discovered, "high_risk": highRisk}
|
|
}
|
|
if err := riskRows.Close(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
assetTrend := make([]map[string]interface{}, 0, days)
|
|
riskTrend := make([]map[string]interface{}, 0, days)
|
|
start := time.Now().UTC().Truncate(24*time.Hour).AddDate(0, 0, -(days - 1))
|
|
for i := 0; i < days; i++ {
|
|
day := start.AddDate(0, 0, i).Format("2006-01-02")
|
|
assetPoint := map[string]interface{}{"date": day, "added": 0, "inactive": 0}
|
|
if values, ok := assetDaily[day]; ok {
|
|
assetPoint["added"], assetPoint["inactive"] = values["added"], values["inactive"]
|
|
}
|
|
assetTrend = append(assetTrend, assetPoint)
|
|
riskPoint := map[string]interface{}{"date": day, "discovered": 0, "high_risk": 0}
|
|
if values, ok := riskDaily[day]; ok {
|
|
riskPoint["discovered"], riskPoint["high_risk"] = values["discovered"], values["high_risk"]
|
|
}
|
|
riskTrend = append(riskTrend, riskPoint)
|
|
}
|
|
stats["asset_trend"], stats["risk_trend"] = assetTrend, riskTrend
|
|
return stats, rows.Err()
|
|
}
|