mirror of
https://github.com/Ed1s0nZ/CyberStrikeAI.git
synced 2026-10-17 05:01:30 +02:00
fix: use supported FOFA region return field and preserve asset location
This commit is contained in:
Ed1s0nZ committed
1 parent
a7e483cbd5
commit
0c0d4a53e8
9 files changed
+159
-8
No files matched your search
@@ -29,3 +29,9 @@ REQ-003 / AC-003：返回的 region 可导入资产 province，兼容已有 prov
|
||||
新增独立 Go 字段规范化辅助文件；handler 加调用点，修改默认值；前端配置、HTML 和工具说明定点替换；资产映射加 region。
|
||||
验证：Go 模拟服务检查真实 outbound fields 并返回按序行，覆盖默认、旧别名、重复和结果映射；Node 验证 FOFA 预设以及对象/数组资产导入，保留旧数据及 Quake 行为。执行现有 handler、前端相关测试和语法、diff 检查。
|
||||
回退：撤销本分支代码提交，无数据迁移。真实 API 查询会消耗外部账户额度，本次采用合同模拟验证，不使用真实凭据。
|
||||
|
||||
## F4 实施记录
|
||||
|
||||
已修正后端、JS、HTML 默认和 FOFA 字段预设；独立 normalizeFofaFields 在 FOFA 分支转换 province、清理重复字段并保持请求/响应顺序。空白 fields 同空值使用默认值。旧表单可由后端兼容，无需清空浏览器存储；保留旧 FOFA 默认字段识别，避免其他提供商恢复表单时误用 FOFA 字段。
|
||||
资产导入读取 region，并兼容 province、location.province_cn；工具字段说明同步更新为 region/asn/org。
|
||||
已通过 FOFA 和其他提供商定向 Go handler 测试、16 项 Node 测试、JS 语法及 diff 检查。新增测试最初因 Node VM 提取函数的结束标记拼写错误失败，修正测试加载边界后通过；未掩盖运行错误。完整 handler 包测试另行记录于 F5。
|
||||
@@ -144,7 +144,7 @@ func defaultFieldsForProvider(provider string) string {
|
||||
case "shodan":
|
||||
return "ip_str,port,hostnames,domains,org,isp,location.country_name,location.city,product,transport"
|
||||
default:
|
||||
return "host,ip,port,domain,title,protocol,country,province,city,server"
|
||||
return "host,ip,port,domain,title,protocol,country,region,city,server"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -602,7 +602,7 @@ func (h *FofaHandler) Search(c *gin.Context) {
|
||||
if req.Size > 10000 {
|
||||
req.Size = 10000
|
||||
}
|
||||
if req.Fields == "" {
|
||||
if strings.TrimSpace(req.Fields) == "" {
|
||||
req.Fields = defaultFieldsForProvider(provider)
|
||||
}
|
||||
|
||||
@@ -611,6 +611,8 @@ func (h *FofaHandler) Search(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
req.Fields = normalizeFofaFields(req.Fields)
|
||||
|
||||
apiKey := h.resolveAPIKey(provider)
|
||||
if apiKey == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package handler
|
||||
|
||||
import "strings"
|
||||
|
||||
// normalizeFofaFields accepts the legacy province return-field alias from saved
|
||||
// forms and API clients. FOFA exports this value as region. Query syntax and
|
||||
// other search providers do not pass through this conversion.
|
||||
func normalizeFofaFields(raw string) string {
|
||||
fields := splitAndCleanCSV(raw)
|
||||
seen := make(map[string]bool, len(fields))
|
||||
normalized := make([]string, 0, len(fields))
|
||||
for _, field := range fields {
|
||||
if field == "province" {
|
||||
field = "region"
|
||||
}
|
||||
if !seen[field] {
|
||||
normalized = append(normalized, field)
|
||||
seen[field] = true
|
||||
}
|
||||
}
|
||||
return strings.Join(normalized, ",")
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"cyberstrike-ai/internal/config"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func TestFofaSearchReturnFields(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
t.Setenv("FOFA_API_KEY", "")
|
||||
// A contract-checking upstream: unsupported fields are rejected rather than
|
||||
// silently accepted by a mock, and values are returned in requested order.
|
||||
values := map[string]string{
|
||||
"host": "https://example.com", "ip": "192.0.2.1", "port": "443",
|
||||
"domain": "example.com", "title": "Example", "protocol": "https",
|
||||
"country": "CN", "region": "Beijing", "city": "Beijing", "server": "nginx",
|
||||
}
|
||||
defaults := "host,ip,port,domain,title,protocol,country,region,city,server"
|
||||
for _, tc := range []struct{ name, fields, want string }{
|
||||
{"default", "", defaults},
|
||||
{"blank default", " ", defaults},
|
||||
{"legacy form", strings.ReplaceAll(defaults, "region", "province"), defaults},
|
||||
{"legacy alias and duplicate", " host, province, region, city, province,host ", "host,region,city"},
|
||||
{"canonical region", "region,host", "region,host"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var received string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
received = r.URL.Query().Get("fields")
|
||||
row := []string{}
|
||||
for _, field := range strings.Split(received, ",") {
|
||||
value, ok := values[field]
|
||||
if !ok {
|
||||
http.Error(w, "unsupported field: "+field, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
row = append(row, value)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{"error": false, "size": 1, "page": 1, "results": [][]string{row}})
|
||||
}))
|
||||
defer server.Close()
|
||||
h := NewFofaHandler(&config.Config{FOFA: config.FofaConfig{APIKey: "test-key", BaseURL: server.URL}}, zap.NewNop())
|
||||
body, _ := json.Marshal(fofaSearchRequest{Query: `domain="example.com"`, Fields: tc.fields})
|
||||
recorder := httptest.NewRecorder()
|
||||
ctx, _ := gin.CreateTestContext(recorder)
|
||||
ctx.Request = httptest.NewRequest(http.MethodPost, "/api/fofa/search", strings.NewReader(string(body)))
|
||||
ctx.Request.Header.Set("Content-Type", "application/json")
|
||||
h.Search(ctx)
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("Search status = %d: %s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if received != tc.want {
|
||||
t.Fatalf("upstream fields = %q, want %q", received, tc.want)
|
||||
}
|
||||
var result fofaSearchResponse
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(result.Fields, strings.Split(tc.want, ",")) || len(result.Results) != 1 {
|
||||
t.Fatalf("incorrect result shape: %+v", result)
|
||||
}
|
||||
for _, field := range result.Fields {
|
||||
if result.Results[0][field] != values[field] {
|
||||
t.Errorf("result[%s] = %v, want %s", field, result.Results[0][field], values[field])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -310,7 +310,7 @@ parameters:
|
||||
|
||||
需要返回的字段，多个字段用逗号分隔。
|
||||
默认字段：ip,port,domain
|
||||
可用字段：host,title,ip,domain,port,protocol,country,province,city,server,icp,header,banner,body,as_number,as_organization
|
||||
可用字段：host,title,ip,domain,port,protocol,country,region,city,server,icp,header,banner,body,asn,org
|
||||
required: false
|
||||
position: 5
|
||||
format: "positional"
|
||||
|
||||
@@ -1660,7 +1660,7 @@ function fofaResultToAsset(row, fields) {
|
||||
return {
|
||||
host: firstValue(['host', 'hostname']), ip, port: Number.isFinite(port) ? port : 0, domain,
|
||||
protocol: /^[a-z][a-z0-9+.-]{0,31}$/.test(rawProtocol) ? rawProtocol : '', title: firstValue(['title', 'service.http.title']), server: firstValue(['server', 'product']),
|
||||
country: firstValue(['country', 'location.country_cn', 'location.country_name']), province: firstValue(['province', 'location.province_cn']), city: firstValue(['city', 'location.city_cn', 'location.city']),
|
||||
country: firstValue(['country', 'location.country_cn', 'location.country_name']), province: firstValue(['region', 'province', 'location.province_cn']), city: firstValue(['city', 'location.city_cn', 'location.city']),
|
||||
source: (window.infoCollectState?.currentPayload?.provider || 'fofa'), status: 'active'
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const vm = require('node:vm');
|
||||
const path = require('node:path');
|
||||
const info = fs.readFileSync(path.join(__dirname, 'info-collect.js'), 'utf8');
|
||||
const assets = fs.readFileSync(path.join(__dirname, 'assets.js'), 'utf8');
|
||||
const c = vm.createContext({ window: {}, URL });
|
||||
vm.runInContext(info.slice(0, info.indexOf('const infoCollectState')), c);
|
||||
const providers = vm.runInContext('INFO_COLLECT_PROVIDERS', c);
|
||||
|
||||
test('FOFA default, presets, and HTML use supported return fields', () => {
|
||||
const supported = new Set('ip,port,protocol,country,country_name,region,city,longitude,latitude,asn,org,host,domain,os,server,icp,title,jarm,header,banner,cert,base_protocol,link,body'.split(','));
|
||||
const html = fs.readFileSync(path.join(__dirname, '../../templates/index.html'), 'utf8');
|
||||
const initial = html.match(/id="fofa-fields" value="([^"]+)"/)[1];
|
||||
assert.equal(initial, providers.fofa.fields);
|
||||
for (const fields of [initial, ...providers.fofa.fieldPresets.map(p => p[1])]) {
|
||||
for (const field of fields.split(',')) assert.ok(supported.has(field), 'unsupported return field: ' + field);
|
||||
}
|
||||
assert.ok(providers.quake.fields.includes('location.province_cn'));
|
||||
});
|
||||
|
||||
// Execute the real conversion and its validation helpers, including array rows.
|
||||
vm.runInContext(assets.slice(assets.indexOf('function assetEditorIsIPv4('), assets.indexOf('function parseAssetEditorTarget(')), c);
|
||||
vm.runInContext(assets.slice(assets.indexOf('function fofaResultToAsset('), assets.indexOf('async function importFofaAssetsByIndexes(')), c);
|
||||
for (const [field, provider] of [['region', 'fofa'], ['province', 'fofa'], ['location.province_cn', 'quake']]) {
|
||||
for (const array of [false, true]) {
|
||||
test(`${provider} ${field} ${array ? 'array' : 'object'} row keeps province on import`, () => {
|
||||
c.window.infoCollectState = { currentPayload: { provider } };
|
||||
const fields = ['host', 'ip', 'port', field, 'city'];
|
||||
const values = ['https://example.com', '192.0.2.1', 443, 'Beijing', 'Beijing'];
|
||||
const row = array ? values : Object.fromEntries(fields.map((f, i) => [f, values[i]]));
|
||||
const asset = c.fofaResultToAsset(row, fields);
|
||||
assert.equal(asset.province, 'Beijing');
|
||||
assert.equal(asset.ip, '192.0.2.1');
|
||||
assert.equal(asset.port, 443);
|
||||
assert.equal(asset.city, 'Beijing');
|
||||
assert.equal(asset.source, provider);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,7 @@ const INFO_COLLECT_PROVIDERS = {
|
||||
label: '完整模式',
|
||||
hint: '向 FOFA 传 full=true，返回更完整/更实时的数据，可能消耗更多额度。'
|
||||
},
|
||||
fields: 'host,ip,port,domain,title,protocol,country,province,city,server',
|
||||
fields: 'host,ip,port,domain,title,protocol,country,region,city,server',
|
||||
presets: [
|
||||
['Apache + 中国', 'app="Apache" && country="CN"'],
|
||||
['登录页 + 中国', 'title="登录" && country="CN"'],
|
||||
@@ -28,8 +28,8 @@ const INFO_COLLECT_PROVIDERS = {
|
||||
],
|
||||
fieldPresets: [
|
||||
['最小字段', 'host,ip,port,domain'],
|
||||
['Web 常用', 'host,title,ip,port,domain,protocol,server,icp,country,province,city'],
|
||||
['情报增强', 'host,ip,port,domain,title,protocol,country,province,city,server,as_number,as_organization,icp,header,banner']
|
||||
['Web 常用', 'host,title,ip,port,domain,protocol,server,icp,country,region,city'],
|
||||
['情报增强', 'host,ip,port,domain,title,protocol,country,region,city,server,asn,org,icp,header,banner']
|
||||
],
|
||||
syntaxGuide: {
|
||||
summary: 'FOFA 使用 field="value" 精确匹配，支持 &&、||、! 和括号组合；字符串建议用双引号包裹。',
|
||||
@@ -268,6 +268,7 @@ function initInfoCollectPage() {
|
||||
const savedFields = String(saved.fields || '').trim();
|
||||
shouldResetProviderFields = provider !== 'fofa' && (
|
||||
savedFields === INFO_COLLECT_PROVIDERS.fofa.fields ||
|
||||
savedFields === 'host,ip,port,domain,title,protocol,country,province,city,server' ||
|
||||
savedFields === 'host,ip,port,domain'
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2275,7 +2275,7 @@
|
||||
<span>04</span>
|
||||
<label for="fofa-fields" data-i18n="infoCollectPage.returnFields">返回字段名（逗号分隔）</label>
|
||||
</div>
|
||||
<input type="text" id="fofa-fields" value="host,ip,port,domain,title,protocol,country,province,city,server" />
|
||||
<input type="text" id="fofa-fields" value="host,ip,port,domain,title,protocol,country,region,city,server" />
|
||||
<div class="info-collect-presets" id="info-collect-fields-presets" aria-label="字段模板"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Reference in new issue
Block a user