From 7493e70686eb89a93d6cf45e6965d4c67aa569f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=AC=E6=98=8E?= <83812544+Ed1s0nZ@users.noreply.github.com> Date: Fri, 6 Mar 2026 22:39:30 +0800 Subject: [PATCH] Add files via upload --- internal/app/app.go | 4 +- internal/config/config.go | 20 + internal/handler/config.go | 10 + internal/handler/knowledge.go | 42 +- internal/knowledge/embedder.go | 180 ++- internal/knowledge/indexer.go | 348 +++-- web/static/js/knowledge.js | 42 + web/static/js/knowledge.js.bak | 2216 ++++++++++++++++++++++++++++++++ web/static/js/settings.js | 46 + web/templates/index.html | 39 +- web/templates/index.html.bak | 2180 +++++++++++++++++++++++++++++++ 11 files changed, 4991 insertions(+), 136 deletions(-) create mode 100644 web/static/js/knowledge.js.bak create mode 100644 web/templates/index.html.bak diff --git a/internal/app/app.go b/internal/app/app.go index 1be512ea..11edb3c1 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -198,7 +198,7 @@ func New(cfg *config.Config, log *logger.Logger) (*App, error) { knowledgeRetriever = knowledge.NewRetriever(knowledgeDB, embedder, retrievalConfig, log.Logger) // 创建索引器 - knowledgeIndexer = knowledge.NewIndexer(knowledgeDB, embedder, log.Logger) + knowledgeIndexer = knowledge.NewIndexer(knowledgeDB, embedder, log.Logger, &cfg.Knowledge.Indexing) // 注册知识检索工具到MCP服务器 knowledge.RegisterKnowledgeTool(mcpServer, knowledgeRetriever, knowledgeManager, log.Logger) @@ -1102,7 +1102,7 @@ func initializeKnowledge( knowledgeRetriever := knowledge.NewRetriever(knowledgeDB, embedder, retrievalConfig, logger) // 创建索引器 - knowledgeIndexer := knowledge.NewIndexer(knowledgeDB, embedder, logger) + knowledgeIndexer := knowledge.NewIndexer(knowledgeDB, embedder, logger, &cfg.Knowledge.Indexing) // 注册知识检索工具到MCP服务器 knowledge.RegisterKnowledgeTool(mcpServer, knowledgeRetriever, knowledgeManager, logger) diff --git a/internal/config/config.go b/internal/config/config.go index 47db1bd6..bc024f85 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -595,6 +595,26 @@ type KnowledgeConfig struct { BasePath string `yaml:"base_path" json:"base_path"` // 知识库路径 Embedding EmbeddingConfig `yaml:"embedding" json:"embedding"` Retrieval RetrievalConfig `yaml:"retrieval" json:"retrieval"` + Indexing IndexingConfig `yaml:"indexing,omitempty" json:"indexing,omitempty"` // 索引构建配置 +} + +// IndexingConfig 索引构建配置(用于控制知识库索引构建时的行为) +type IndexingConfig struct { + // 分块配置 + ChunkSize int `yaml:"chunk_size,omitempty" json:"chunk_size,omitempty"` // 每个块的最大 token 数(估算),默认 512 + ChunkOverlap int `yaml:"chunk_overlap,omitempty" json:"chunk_overlap,omitempty"` // 块之间的重叠 token 数,默认 50 + MaxChunksPerItem int `yaml:"max_chunks_per_item,omitempty" json:"max_chunks_per_item,omitempty"` // 单个知识项的最大块数量,0 表示不限制 + + // 速率限制配置(用于避免 API 速率限制) + RateLimitDelayMs int `yaml:"rate_limit_delay_ms,omitempty" json:"rate_limit_delay_ms,omitempty"` // 请求间隔时间(毫秒),0 表示不使用固定延迟 + MaxRPM int `yaml:"max_rpm,omitempty" json:"max_rpm,omitempty"` // 每分钟最大请求数,0 表示不限制 + + // 重试配置(用于处理临时错误) + MaxRetries int `yaml:"max_retries,omitempty" json:"max_retries,omitempty"` // 最大重试次数,默认 3 + RetryDelayMs int `yaml:"retry_delay_ms,omitempty" json:"retry_delay_ms,omitempty"` // 重试间隔(毫秒),默认 1000 + + // 批处理配置(用于批量嵌入,当前未使用,保留扩展) + BatchSize int `yaml:"batch_size,omitempty" json:"batch_size,omitempty"` // 批量处理大小,0 表示逐个处理 } // EmbeddingConfig 嵌入配置 diff --git a/internal/handler/config.go b/internal/handler/config.go index 98d56f5f..80f2c9e2 100644 --- a/internal/handler/config.go +++ b/internal/handler/config.go @@ -1062,6 +1062,16 @@ func updateKnowledgeConfig(doc *yaml.Node, cfg config.KnowledgeConfig) { setIntInMap(retrievalNode, "top_k", cfg.Retrieval.TopK) setFloatInMap(retrievalNode, "similarity_threshold", cfg.Retrieval.SimilarityThreshold) setFloatInMap(retrievalNode, "hybrid_weight", cfg.Retrieval.HybridWeight) + + // 更新索引配置 + indexingNode := ensureMap(knowledgeNode, "indexing") + setIntInMap(indexingNode, "chunk_size", cfg.Indexing.ChunkSize) + setIntInMap(indexingNode, "chunk_overlap", cfg.Indexing.ChunkOverlap) + setIntInMap(indexingNode, "max_chunks_per_item", cfg.Indexing.MaxChunksPerItem) + setIntInMap(indexingNode, "max_rpm", cfg.Indexing.MaxRPM) + setIntInMap(indexingNode, "rate_limit_delay_ms", cfg.Indexing.RateLimitDelayMs) + setIntInMap(indexingNode, "max_retries", cfg.Indexing.MaxRetries) + setIntInMap(indexingNode, "retry_delay_ms", cfg.Indexing.RetryDelayMs) } func updateRobotsConfig(doc *yaml.Node, cfg config.RobotsConfig) { diff --git a/internal/handler/knowledge.go b/internal/handler/knowledge.go index 6578de72..c92f46e7 100644 --- a/internal/handler/knowledge.go +++ b/internal/handler/knowledge.go @@ -75,7 +75,7 @@ func (h *KnowledgeHandler) GetItems(c *gin.Context) { groupedByCategory[cat] = append(groupedByCategory[cat], item) } - // 转换为CategoryWithItems格式 + // 转换为 CategoryWithItems 格式 categoriesWithItems := make([]*knowledge.CategoryWithItems, 0, len(groupedByCategory)) for cat, catItems := range groupedByCategory { categoriesWithItems = append(categoriesWithItems, &knowledge.CategoryWithItems{ @@ -107,7 +107,7 @@ func (h *KnowledgeHandler) GetItems(c *gin.Context) { categoryPageMode := c.Query("categoryPage") != "false" // 默认使用分类分页 // 分页参数 - limit := 50 // 默认每页50条(分类分页时为分类数,项分页时为项数) + limit := 50 // 默认每页 50 条(分类分页时为分类数,项分页时为项数) offset := 0 if limitStr := c.Query("limit"); limitStr != "" { if parsed, err := parseInt(limitStr); err == nil && parsed > 0 && parsed <= 500 { @@ -120,7 +120,7 @@ func (h *KnowledgeHandler) GetItems(c *gin.Context) { } } - // 如果指定了category参数,且使用分类分页模式,则只返回该分类 + // 如果指定了 category 参数,且使用分类分页模式,则只返回该分类 if category != "" && categoryPageMode { // 单分类模式:返回该分类的所有知识项(不分页) items, total, err := h.manager.GetItemsSummary(category, 0, 0) @@ -150,9 +150,9 @@ func (h *KnowledgeHandler) GetItems(c *gin.Context) { if categoryPageMode { // 按分类分页模式(默认) - // limit表示每页分类数,推荐5-10个分类 + // limit 表示每页分类数,推荐 5-10 个分类 if limit <= 0 || limit > 100 { - limit = 10 // 默认每页10个分类 + limit = 10 // 默认每页 10 个分类 } categoriesWithItems, totalCategories, err := h.manager.GetCategoriesWithItems(limit, offset) @@ -172,7 +172,7 @@ func (h *KnowledgeHandler) GetItems(c *gin.Context) { } // 按项分页模式(向后兼容) - // 是否包含完整内容(默认false,只返回摘要) + // 是否包含完整内容(默认 false,只返回摘要) includeContent := c.Query("includeContent") == "true" if includeContent { @@ -358,7 +358,7 @@ func (h *KnowledgeHandler) ScanKnowledgeBase(c *gin.Context) { ) } - // 如果连续失败2次,立即停止增量索引 + // 如果连续失败 2 次,立即停止增量索引 if consecutiveFailures >= 2 { h.logger.Error("连续索引失败次数过多,立即停止增量索引", zap.Int("consecutiveFailures", consecutiveFailures), @@ -397,7 +397,7 @@ func (h *KnowledgeHandler) ScanKnowledgeBase(c *gin.Context) { func (h *KnowledgeHandler) GetRetrievalLogs(c *gin.Context) { conversationID := c.Query("conversationId") messageID := c.Query("messageId") - limit := 50 // 默认50条 + limit := 50 // 默认 50 条 if limitStr := c.Query("limit"); limitStr != "" { if parsed, err := parseInt(limitStr); err == nil && parsed > 0 { @@ -441,18 +441,40 @@ func (h *KnowledgeHandler) GetIndexStatus(c *gin.Context) { if h.indexer != nil { lastError, lastErrorTime := h.indexer.GetLastError() if lastError != "" { - // 如果错误是最近发生的(5分钟内),则返回错误信息 + // 如果错误是最近发生的(5 分钟内),则返回错误信息 if time.Since(lastErrorTime) < 5*time.Minute { status["last_error"] = lastError status["last_error_time"] = lastErrorTime.Format(time.RFC3339) } } + + // 获取重建索引状态 + isRebuilding, totalItems, current, failed, lastItemID, lastChunks, startTime := h.indexer.GetRebuildStatus() + if isRebuilding { + status["is_rebuilding"] = true + status["rebuild_total"] = totalItems + status["rebuild_current"] = current + status["rebuild_failed"] = failed + status["rebuild_start_time"] = startTime.Format(time.RFC3339) + if lastItemID != "" { + status["rebuild_last_item_id"] = lastItemID + } + if lastChunks > 0 { + status["rebuild_last_chunks"] = lastChunks + } + // 重建中时,is_complete 为 false + status["is_complete"] = false + // 计算重建进度百分比 + if totalItems > 0 { + status["progress_percent"] = float64(current) / float64(totalItems) * 100 + } + } } c.JSON(http.StatusOK, status) } -// Search 搜索知识库(用于API调用,Agent内部使用Retriever) +// Search 搜索知识库(用于 API 调用,Agent 内部使用 Retriever) func (h *KnowledgeHandler) Search(c *gin.Context) { var req knowledge.SearchRequest if err := c.ShouldBindJSON(&req); err != nil { diff --git a/internal/knowledge/embedder.go b/internal/knowledge/embedder.go index 2f27ab9d..ff62ea30 100644 --- a/internal/knowledge/embedder.go +++ b/internal/knowledge/embedder.go @@ -6,39 +6,75 @@ import ( "fmt" "net/http" "strings" + "sync" "time" "cyberstrike-ai/internal/config" "cyberstrike-ai/internal/openai" "go.uber.org/zap" + "golang.org/x/time/rate" ) // Embedder 文本嵌入器 type Embedder struct { - openAIClient *openai.Client - config *config.KnowledgeConfig - openAIConfig *config.OpenAIConfig // 用于获取API Key - logger *zap.Logger + openAIClient *openai.Client + config *config.KnowledgeConfig + openAIConfig *config.OpenAIConfig // 用于获取 API Key + logger *zap.Logger + rateLimiter *rate.Limiter // 速率限制器 + rateLimitDelay time.Duration // 请求间隔时间 + maxRetries int // 最大重试次数 + retryDelay time.Duration // 重试间隔 + mu sync.Mutex // 保护 rateLimiter } // NewEmbedder 创建新的嵌入器 func NewEmbedder(cfg *config.KnowledgeConfig, openAIConfig *config.OpenAIConfig, openAIClient *openai.Client, logger *zap.Logger) *Embedder { + // 初始化速率限制器 + var rateLimiter *rate.Limiter + var rateLimitDelay time.Duration + + // 如果配置了 MaxRPM,根据 RPM 计算速率限制 + if cfg.Indexing.MaxRPM > 0 { + rpm := cfg.Indexing.MaxRPM + rateLimiter = rate.NewLimiter(rate.Every(time.Minute/time.Duration(rpm)), rpm) + logger.Info("知识库索引速率限制已启用", zap.Int("maxRPM", rpm)) + } else if cfg.Indexing.RateLimitDelayMs > 0 { + // 如果没有配置 MaxRPM 但配置了固定延迟,使用固定延迟模式 + rateLimitDelay = time.Duration(cfg.Indexing.RateLimitDelayMs) * time.Millisecond + logger.Info("知识库索引固定延迟已启用", zap.Duration("delay", rateLimitDelay)) + } + + // 重试配置 + maxRetries := 3 + retryDelay := 1000 * time.Millisecond + if cfg.Indexing.MaxRetries > 0 { + maxRetries = cfg.Indexing.MaxRetries + } + if cfg.Indexing.RetryDelayMs > 0 { + retryDelay = time.Duration(cfg.Indexing.RetryDelayMs) * time.Millisecond + } + return &Embedder{ - openAIClient: openAIClient, - config: cfg, - openAIConfig: openAIConfig, - logger: logger, + openAIClient: openAIClient, + config: cfg, + openAIConfig: openAIConfig, + logger: logger, + rateLimiter: rateLimiter, + rateLimitDelay: rateLimitDelay, + maxRetries: maxRetries, + retryDelay: retryDelay, } } -// EmbeddingRequest OpenAI嵌入请求 +// EmbeddingRequest OpenAI 嵌入请求 type EmbeddingRequest struct { Model string `json:"model"` Input []string `json:"input"` } -// EmbeddingResponse OpenAI嵌入响应 +// EmbeddingResponse OpenAI 嵌入响应 type EmbeddingResponse struct { Data []EmbeddingData `json:"data"` Error *EmbeddingError `json:"error,omitempty"` @@ -56,12 +92,69 @@ type EmbeddingError struct { Type string `json:"type"` } -// EmbedText 对文本进行嵌入 -func (e *Embedder) EmbedText(ctx context.Context, text string) ([]float32, error) { - if e.openAIClient == nil { - return nil, fmt.Errorf("OpenAI客户端未初始化") +// waitRateLimiter 等待速率限制器 +func (e *Embedder) waitRateLimiter() { + e.mu.Lock() + defer e.mu.Unlock() + + if e.rateLimiter != nil { + // 等待令牌 + ctx := context.Background() + if err := e.rateLimiter.Wait(ctx); err != nil { + e.logger.Warn("速率限制器等待失败", zap.Error(err)) + } } + if e.rateLimitDelay > 0 { + time.Sleep(e.rateLimitDelay) + } +} + +// EmbedText 对文本进行嵌入(带重试和速率限制) +func (e *Embedder) EmbedText(ctx context.Context, text string) ([]float32, error) { + if e.openAIClient == nil { + return nil, fmt.Errorf("OpenAI 客户端未初始化") + } + + var lastErr error + for attempt := 0; attempt < e.maxRetries; attempt++ { + // 速率限制 + if attempt > 0 { + // 重试时等待更长时间 + waitTime := e.retryDelay * time.Duration(attempt) + e.logger.Debug("重试前等待", zap.Int("attempt", attempt+1), zap.Duration("waitTime", waitTime)) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(waitTime): + } + } else { + e.waitRateLimiter() + } + + result, err := e.doEmbedText(ctx, text) + if err == nil { + return result, nil + } + + lastErr = err + + // 检查是否是可重试的错误(429 速率限制、5xx 服务器错误、网络错误) + if !e.isRetryableError(err) { + return nil, err + } + + e.logger.Debug("嵌入请求失败,准备重试", + zap.Int("attempt", attempt+1), + zap.Int("maxRetries", e.maxRetries), + zap.Error(err)) + } + + return nil, fmt.Errorf("达到最大重试次数 (%d): %v", e.maxRetries, lastErr) +} + +// doEmbedText 执行实际的嵌入请求(内部方法) +func (e *Embedder) doEmbedText(ctx context.Context, text string) ([]float32, error) { // 使用配置的嵌入模型 model := e.config.Embedding.Model if model == "" { @@ -73,7 +166,7 @@ func (e *Embedder) EmbedText(ctx context.Context, text string) ([]float32, error Input: []string{text}, } - // 清理baseURL:去除前后空格和尾部斜杠 + // 清理 baseURL:去除前后空格和尾部斜杠 baseURL := strings.TrimSpace(e.config.Embedding.BaseURL) baseURL = strings.TrimSuffix(baseURL, "/") if baseURL == "" { @@ -83,24 +176,24 @@ func (e *Embedder) EmbedText(ctx context.Context, text string) ([]float32, error // 构建请求 body, err := json.Marshal(req) if err != nil { - return nil, fmt.Errorf("序列化请求失败: %w", err) + return nil, fmt.Errorf("序列化请求失败:%w", err) } requestURL := baseURL + "/embeddings" httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, requestURL, strings.NewReader(string(body))) if err != nil { - return nil, fmt.Errorf("创建请求失败: %w", err) + return nil, fmt.Errorf("创建请求失败:%w", err) } httpReq.Header.Set("Content-Type", "application/json") - - // 使用配置的API Key,如果没有则使用OpenAI配置的 + + // 使用配置的 API Key,如果没有则使用 OpenAI 配置的 apiKey := strings.TrimSpace(e.config.Embedding.APIKey) if apiKey == "" && e.openAIConfig != nil { apiKey = e.openAIConfig.APIKey } if apiKey == "" { - return nil, fmt.Errorf("API Key未配置") + return nil, fmt.Errorf("API Key 未配置") } httpReq.Header.Set("Authorization", "Bearer "+apiKey) @@ -110,7 +203,7 @@ func (e *Embedder) EmbedText(ctx context.Context, text string) ([]float32, error } resp, err := httpClient.Do(httpReq) if err != nil { - return nil, fmt.Errorf("发送请求失败: %w", err) + return nil, fmt.Errorf("发送请求失败:%w", err) } defer resp.Body.Close() @@ -132,7 +225,7 @@ func (e *Embedder) EmbedText(ctx context.Context, text string) ([]float32, error if len(requestBodyPreview) > 200 { requestBodyPreview = requestBodyPreview[:200] + "..." } - e.logger.Debug("嵌入API请求", + e.logger.Debug("嵌入 API 请求", zap.String("url", httpReq.URL.String()), zap.String("model", model), zap.String("requestBody", requestBodyPreview), @@ -148,12 +241,12 @@ func (e *Embedder) EmbedText(ctx context.Context, text string) ([]float32, error if len(bodyPreview) > 500 { bodyPreview = bodyPreview[:500] + "..." } - return nil, fmt.Errorf("解析响应失败 (URL: %s, 状态码: %d, 响应长度: %d字节): %w\n请求体: %s\n响应内容预览: %s", + return nil, fmt.Errorf("解析响应失败 (URL: %s, 状态码:%d, 响应长度:%d字节): %w\n请求体:%s\n响应内容预览:%s", requestURL, resp.StatusCode, len(bodyBytes), err, requestBodyPreview, bodyPreview) } if embeddingResp.Error != nil { - return nil, fmt.Errorf("OpenAI API错误 (状态码: %d): 类型=%s, 消息=%s", + return nil, fmt.Errorf("OpenAI API 错误 (状态码:%d): 类型=%s, 消息=%s", resp.StatusCode, embeddingResp.Error.Type, embeddingResp.Error.Message) } @@ -162,7 +255,7 @@ func (e *Embedder) EmbedText(ctx context.Context, text string) ([]float32, error if len(bodyPreview) > 500 { bodyPreview = bodyPreview[:500] + "..." } - return nil, fmt.Errorf("HTTP请求失败 (URL: %s, 状态码: %d): 响应内容=%s", requestURL, resp.StatusCode, bodyPreview) + return nil, fmt.Errorf("HTTP 请求失败 (URL: %s, 状态码:%d): 响应内容=%s", requestURL, resp.StatusCode, bodyPreview) } if len(embeddingResp.Data) == 0 { @@ -170,11 +263,11 @@ func (e *Embedder) EmbedText(ctx context.Context, text string) ([]float32, error if len(bodyPreview) > 500 { bodyPreview = bodyPreview[:500] + "..." } - return nil, fmt.Errorf("未收到嵌入数据 (状态码: %d, 响应长度: %d字节)\n响应内容: %s", + return nil, fmt.Errorf("未收到嵌入数据 (状态码:%d, 响应长度:%d字节)\n响应内容:%s", resp.StatusCode, len(bodyBytes), bodyPreview) } - // 转换为float32 + // 转换为 float32 embedding := make([]float32, len(embeddingResp.Data[0].Embedding)) for i, v := range embeddingResp.Data[0].Embedding { embedding[i] = float32(v) @@ -183,23 +276,48 @@ func (e *Embedder) EmbedText(ctx context.Context, text string) ([]float32, error return embedding, nil } +// isRetryableError 判断是否是可重试的错误 +func (e *Embedder) isRetryableError(err error) bool { + if err == nil { + return false + } + + errStr := err.Error() + + // 429 速率限制错误 + if strings.Contains(errStr, "429") || strings.Contains(errStr, "rate limit") { + return true + } + + // 5xx 服务器错误 + if strings.Contains(errStr, "500") || strings.Contains(errStr, "502") || + strings.Contains(errStr, "503") || strings.Contains(errStr, "504") { + return true + } + + // 网络错误 + if strings.Contains(errStr, "timeout") || strings.Contains(errStr, "connection") || + strings.Contains(errStr, "network") || strings.Contains(errStr, "EOF") { + return true + } + + return false +} + // EmbedTexts 批量嵌入文本 func (e *Embedder) EmbedTexts(ctx context.Context, texts []string) ([][]float32, error) { if len(texts) == 0 { return nil, nil } - // OpenAI API支持批量,但为了简单起见,我们逐个处理 - // 实际可以使用批量API以提高效率 embeddings := make([][]float32, len(texts)) for i, text := range texts { embedding, err := e.EmbedText(ctx, text) if err != nil { - return nil, fmt.Errorf("嵌入文本[%d]失败: %w", i, err) + return nil, fmt.Errorf("嵌入文本 [%d] 失败:%w", i, err) } embeddings[i] = embedding } return embeddings, nil } - diff --git a/internal/knowledge/indexer.go b/internal/knowledge/indexer.go index d5a49afc..7999c8d0 100644 --- a/internal/knowledge/indexer.go +++ b/internal/knowledge/indexer.go @@ -10,56 +10,104 @@ import ( "sync" "time" + "cyberstrike-ai/internal/config" + "github.com/google/uuid" "go.uber.org/zap" ) // Indexer 索引器,负责将知识项分块并向量化 type Indexer struct { - db *sql.DB - embedder *Embedder - logger *zap.Logger - chunkSize int // 每个块的最大token数(估算) - overlap int // 块之间的重叠token数 - + db *sql.DB + embedder *Embedder + logger *zap.Logger + chunkSize int // 每个块的最大 token 数(估算) + overlap int // 块之间的重叠 token 数 + maxChunks int // 单个知识项的最大块数量(0 表示不限制) + // 错误跟踪 - mu sync.RWMutex - lastError string // 最近一次错误信息 + mu sync.RWMutex + lastError string // 最近一次错误信息 lastErrorTime time.Time // 最近一次错误时间 - errorCount int // 连续错误计数 + errorCount int // 连续错误计数 + + // 重建索引状态跟踪 + rebuildMu sync.RWMutex + isRebuilding bool // 是否正在重建索引 + rebuildTotalItems int // 重建总项数 + rebuildCurrent int // 当前已处理项数 + rebuildFailed int // 重建失败项数 + rebuildStartTime time.Time // 重建开始时间 + rebuildLastItemID string // 最近处理的项 ID + rebuildLastChunks int // 最近处理的项的分块数 } // NewIndexer 创建新的索引器 -func NewIndexer(db *sql.DB, embedder *Embedder, logger *zap.Logger) *Indexer { +func NewIndexer(db *sql.DB, embedder *Embedder, logger *zap.Logger, indexingCfg *config.IndexingConfig) *Indexer { + chunkSize := 512 + overlap := 50 + maxChunks := 0 + if indexingCfg != nil { + if indexingCfg.ChunkSize > 0 { + chunkSize = indexingCfg.ChunkSize + } + if indexingCfg.ChunkOverlap >= 0 { + overlap = indexingCfg.ChunkOverlap + } + if indexingCfg.MaxChunksPerItem > 0 { + maxChunks = indexingCfg.MaxChunksPerItem + } + } return &Indexer{ db: db, embedder: embedder, logger: logger, - chunkSize: 512, // 默认512 tokens - overlap: 50, // 默认50 tokens重叠 + chunkSize: chunkSize, + overlap: overlap, + maxChunks: maxChunks, } } -// ChunkText 将文本分块(支持重叠) +// ChunkText 将文本分块(支持重叠,保留标题上下文) func (idx *Indexer) ChunkText(text string) []string { - // 按Markdown标题分割 - chunks := idx.splitByMarkdownHeaders(text) + // 按 Markdown 标题分割,获取带标题的块 + sections := idx.splitByMarkdownHeadersWithContent(text) - // 如果块太大,进一步分割 + // 处理每个块 result := make([]string, 0) - for _, chunk := range chunks { - if idx.estimateTokens(chunk) <= idx.chunkSize { - result = append(result, chunk) + for _, section := range sections { + // 构建完整的标题路径(如 "SQL 注入 > 检测方法 > 工具扫描") + headerPath := strings.Join(section.HeaderPath, " > ") + + // 如果块太大,进一步分割 + if idx.estimateTokens(section.Content) <= idx.chunkSize { + // 块大小合适,直接添加标题前缀 + if headerPath != "" { + result = append(result, fmt.Sprintf("[%s] %s", headerPath, section.Content)) + } else { + result = append(result, section.Content) + } } else { - // 按段落分割 - subChunks := idx.splitByParagraphs(chunk) - for _, subChunk := range subChunks { - if idx.estimateTokens(subChunk) <= idx.chunkSize { - result = append(result, subChunk) + // 块太大,按段落分割 + paragraphs := idx.splitByParagraphs(section.Content) + for _, para := range paragraphs { + if idx.estimateTokens(para) <= idx.chunkSize { + // 段落大小合适,添加标题前缀 + if headerPath != "" { + result = append(result, fmt.Sprintf("[%s] %s", headerPath, para)) + } else { + result = append(result, para) + } } else { - // 按句子分割(支持重叠) - chunksWithOverlap := idx.splitBySentencesWithOverlap(subChunk) - result = append(result, chunksWithOverlap...) + // 段落仍太大,按句子分割(带重叠和标题前缀) + sentenceChunks := idx.splitBySentencesWithOverlap(para) + for _, chunk := range sentenceChunks { + if headerPath != "" { + result = append(result, fmt.Sprintf("[%s] %s", headerPath, chunk)) + } else { + result = append(result, chunk) + } + } } } } @@ -68,43 +116,104 @@ func (idx *Indexer) ChunkText(text string) []string { return result } -// splitByMarkdownHeaders 按Markdown标题分割 -func (idx *Indexer) splitByMarkdownHeaders(text string) []string { - // 匹配Markdown标题 (# ## ### 等) +// Section 表示一个带标题路径的文本块 +type Section struct { + HeaderPath []string // 标题路径(如 ["# SQL 注入", "## 检测方法"]) + Content string // 块内容 +} + +// splitByMarkdownHeadersWithContent 按 Markdown 标题分割,返回带标题路径的块 +func (idx *Indexer) splitByMarkdownHeadersWithContent(text string) []Section { + // 匹配 Markdown 标题 (# ## ### 等) headerRegex := regexp.MustCompile(`(?m)^#{1,6}\s+.+$`) // 找到所有标题位置 matches := headerRegex.FindAllStringIndex(text, -1) if len(matches) == 0 { - return []string{text} + // 没有标题,返回整个文本 + return []Section{{HeaderPath: []string{}, Content: text}} } - chunks := make([]string, 0) + sections := make([]Section, 0) + currentHeaderPath := []string{} lastPos := 0 - for _, match := range matches { + for i, match := range matches { start := match[0] - if start > lastPos { - chunks = append(chunks, strings.TrimSpace(text[lastPos:start])) + end := match[1] + + // 提取当前标题 + headerLine := strings.TrimSpace(text[start:end]) + + // 计算标题层级(# 的数量) + level := 0 + for _, ch := range headerLine { + if ch == '#' { + level++ + } else { + break + } } - lastPos = start + + // 更新标题路径 + // 移除比当前层级深或等于的子标题 + newPath := make([]string, 0) + for _, h := range currentHeaderPath { + hLevel := 0 + for _, ch := range h { + if ch == '#' { + hLevel++ + } else { + break + } + } + if hLevel < level { + newPath = append(newPath, h) + } + } + newPath = append(newPath, headerLine) + currentHeaderPath = newPath + + // 提取上一个标题到当前标题之间的内容 + if start > lastPos { + content := strings.TrimSpace(text[lastPos:start]) + // 使用上一层的标题路径 + prevPath := make([]string, len(currentHeaderPath)) + copy(prevPath, currentHeaderPath) + if i > 0 { + // 去掉当前标题,使用父级路径 + if len(prevPath) > 0 { + prevPath = prevPath[:len(prevPath)-1] + } + } + sections = append(sections, Section{ + HeaderPath: prevPath, + Content: content, + }) + } + + lastPos = end } - // 添加最后一部分 + // 添加最后一部分(最后一个标题之后的内容) if lastPos < len(text) { - chunks = append(chunks, strings.TrimSpace(text[lastPos:])) + content := strings.TrimSpace(text[lastPos:]) + sections = append(sections, Section{ + HeaderPath: currentHeaderPath, + Content: content, + }) } // 过滤空块 - result := make([]string, 0) - for _, chunk := range chunks { - if strings.TrimSpace(chunk) != "" { - result = append(result, chunk) + result := make([]Section, 0) + for _, section := range sections { + if strings.TrimSpace(section.Content) != "" { + result = append(result, section) } } if len(result) == 0 { - return []string{text} + return []Section{{HeaderPath: []string{}, Content: text}} } return result @@ -124,8 +233,12 @@ func (idx *Indexer) splitByParagraphs(text string) []string { // splitBySentences 按句子分割(用于内部,不包含重叠逻辑) func (idx *Indexer) splitBySentences(text string) []string { - // 简单的句子分割(按句号、问号、感叹号) - sentenceRegex := regexp.MustCompile(`[.!?]+\s+`) + // 简单的句子分割(按句号、问号、感叹号,支持中英文) + // . ! ? = 英文标点 + // \u3002 = 。(中文句号) + // \uFF01 = !(中文叹号) + // \uFF1F = ?(中文问号) + sentenceRegex := regexp.MustCompile(`[.!?\x{3002}\x{FF01}\x{FF1F}]+`) sentences := sentenceRegex.Split(text, -1) result := make([]string, 0) for _, s := range sentences { @@ -221,13 +334,13 @@ func (idx *Indexer) splitBySentencesSimple(text string) []string { return result } -// extractLastTokens 从文本末尾提取指定token数量的内容 +// extractLastTokens 从文本末尾提取指定 token 数量的内容 func (idx *Indexer) extractLastTokens(text string, tokenCount int) string { if tokenCount <= 0 || text == "" { return "" } - // 估算字符数(1 token ≈ 4字符) + // 估算字符数(1 token ≈ 4 字符) charCount := tokenCount * 4 runes := []rune(text) @@ -236,12 +349,11 @@ func (idx *Indexer) extractLastTokens(text string, tokenCount int) string { } // 从末尾提取指定数量的字符 - // 尝试在句子边界处截断,避免截断句子中间 startPos := len(runes) - charCount extracted := string(runes[startPos:]) - // 尝试找到第一个句子边界(句号、问号、感叹号后的空格) - sentenceBoundary := regexp.MustCompile(`[.!?]+\s+`) + // 尝试找到第一个句子边界(支持中英文标点) + sentenceBoundary := regexp.MustCompile(`[.!?\x{3002}\x{FF01}\x{FF1F}]+`) matches := sentenceBoundary.FindStringIndex(extracted) if len(matches) > 0 && matches[0] > 0 { // 在句子边界处截断,保留完整句子 @@ -251,41 +363,51 @@ func (idx *Indexer) extractLastTokens(text string, tokenCount int) string { return strings.TrimSpace(extracted) } -// estimateTokens 估算token数(简单估算:1 token ≈ 4字符) +// estimateTokens 估算 token 数(简单估算:1 token ≈ 4 字符) func (idx *Indexer) estimateTokens(text string) int { return len([]rune(text)) / 4 } // IndexItem 索引知识项(分块并向量化) func (idx *Indexer) IndexItem(ctx context.Context, itemID string) error { - // 获取知识项(包含category和title,用于向量化) + // 获取知识项(包含 category 和 title,用于向量化) var content, category, title string err := idx.db.QueryRow("SELECT content, category, title FROM knowledge_base_items WHERE id = ?", itemID).Scan(&content, &category, &title) if err != nil { - return fmt.Errorf("获取知识项失败: %w", err) + return fmt.Errorf("获取知识项失败:%w", err) } // 删除旧的向量(在 RebuildIndex 中已经统一清空,这里保留是为了单独调用 IndexItem 时的兼容性) _, err = idx.db.Exec("DELETE FROM knowledge_embeddings WHERE item_id = ?", itemID) if err != nil { - return fmt.Errorf("删除旧向量失败: %w", err) + return fmt.Errorf("删除旧向量失败:%w", err) } // 分块 chunks := idx.ChunkText(content) + + // 应用最大块数限制 + if idx.maxChunks > 0 && len(chunks) > idx.maxChunks { + idx.logger.Info("知识项块数量超过限制,已截断", + zap.String("itemId", itemID), + zap.Int("originalChunks", len(chunks)), + zap.Int("maxChunks", idx.maxChunks)) + chunks = chunks[:idx.maxChunks] + } + idx.logger.Info("知识项分块完成", zap.String("itemId", itemID), zap.Int("chunks", len(chunks))) // 跟踪该知识项的错误 itemErrorCount := 0 var firstError error firstErrorChunkIndex := -1 - - // 向量化每个块(包含category和title信息,以便向量检索时能匹配到风险类型) + + // 向量化每个块(包含 category 和 title 信息,以便向量检索时能匹配到风险类型) for i, chunk := range chunks { - // 将category和title信息包含到向量化的文本中 - // 格式:"[风险类型: {category}] [标题: {title}]\n{chunk内容}" - // 这样向量嵌入就会包含风险类型信息,即使SQL过滤失败,向量相似度也能帮助匹配 - textForEmbedding := fmt.Sprintf("[风险类型: %s] [标题: %s]\n%s", category, title, chunk) + // 将 category 和 title 信息包含到向量化的文本中 + // 格式:"[风险类型:{category}] [标题:{title}]\n{chunk 内容}" + // 这样向量嵌入就会包含风险类型信息,即使 SQL 过滤失败,向量相似度也能帮助匹配 + textForEmbedding := fmt.Sprintf("[风险类型:%s] [标题:%s]\n%s", category, title, chunk) embedding, err := idx.embedder.EmbedText(ctx, textForEmbedding) if err != nil { @@ -305,17 +427,17 @@ func (idx *Indexer) IndexItem(ctx context.Context, itemID string) error { zap.String("chunkPreview", chunkPreview), zap.Error(err), ) - + // 更新全局错误跟踪 - errorMsg := fmt.Sprintf("向量化失败 (知识项: %s): %v", itemID, err) + errorMsg := fmt.Sprintf("向量化失败 (知识项:%s): %v", itemID, err) idx.mu.Lock() idx.lastError = errorMsg idx.lastErrorTime = time.Now() idx.mu.Unlock() } - - // 如果连续失败2个块,立即停止处理该知识项(降低阈值,更快停止) - // 这样可以避免继续浪费API调用,同时也能更快地检测到配置问题 + + // 如果连续失败 2 个块,立即停止处理该知识项(降低阈值,更快停止) + // 这样可以避免继续浪费 API 调用,同时也能更快地检测到配置问题 if itemErrorCount >= 2 { idx.logger.Error("知识项连续向量化失败,停止处理", zap.String("itemId", itemID), @@ -344,6 +466,13 @@ func (idx *Indexer) IndexItem(ctx context.Context, itemID string) error { } idx.logger.Info("知识项索引完成", zap.String("itemId", itemID), zap.Int("chunks", len(chunks))) + + // 更新重建状态中的最近处理信息 + idx.rebuildMu.Lock() + idx.rebuildLastItemID = itemID + idx.rebuildLastChunks = len(chunks) + idx.rebuildMu.Unlock() + return nil } @@ -352,23 +481,38 @@ func (idx *Indexer) HasIndex() (bool, error) { var count int err := idx.db.QueryRow("SELECT COUNT(*) FROM knowledge_embeddings").Scan(&count) if err != nil { - return false, fmt.Errorf("检查索引失败: %w", err) + return false, fmt.Errorf("检查索引失败:%w", err) } return count > 0, nil } // RebuildIndex 重建所有索引 func (idx *Indexer) RebuildIndex(ctx context.Context) error { + // 设置重建状态 + idx.rebuildMu.Lock() + idx.isRebuilding = true + idx.rebuildTotalItems = 0 + idx.rebuildCurrent = 0 + idx.rebuildFailed = 0 + idx.rebuildStartTime = time.Now() + idx.rebuildLastItemID = "" + idx.rebuildLastChunks = 0 + idx.rebuildMu.Unlock() + // 重置错误跟踪 idx.mu.Lock() idx.lastError = "" idx.lastErrorTime = time.Time{} idx.errorCount = 0 idx.mu.Unlock() - + rows, err := idx.db.Query("SELECT id FROM knowledge_base_items") if err != nil { - return fmt.Errorf("查询知识项失败: %w", err) + // 重置重建状态 + idx.rebuildMu.Lock() + idx.isRebuilding = false + idx.rebuildMu.Unlock() + return fmt.Errorf("查询知识项失败:%w", err) } defer rows.Close() @@ -376,34 +520,36 @@ func (idx *Indexer) RebuildIndex(ctx context.Context) error { for rows.Next() { var id string if err := rows.Scan(&id); err != nil { - return fmt.Errorf("扫描知识项ID失败: %w", err) + // 重置重建状态 + idx.rebuildMu.Lock() + idx.isRebuilding = false + idx.rebuildMu.Unlock() + return fmt.Errorf("扫描知识项 ID 失败:%w", err) } itemIDs = append(itemIDs, id) } + idx.rebuildMu.Lock() + idx.rebuildTotalItems = len(itemIDs) + idx.rebuildMu.Unlock() + idx.logger.Info("开始重建索引", zap.Int("totalItems", len(itemIDs))) - // 在开始重建前,先清空所有旧的向量,确保进度从0开始 - // 这样 GetIndexStatus 可以准确反映重建进度 - _, err = idx.db.Exec("DELETE FROM knowledge_embeddings") - if err != nil { - idx.logger.Warn("清空旧索引失败", zap.Error(err)) - // 继续执行,即使清空失败也尝试重建 - } else { - idx.logger.Info("已清空旧索引,开始重建") - } + // 注意:不再清空所有旧索引,而是按增量方式更新 + // 每个知识项在 IndexItem 中会先删除自己的旧向量,然后插入新向量 + // 这样配置更新后只重新索引变化的知识项,保留其他知识项的索引 failedCount := 0 consecutiveFailures := 0 - maxConsecutiveFailures := 2 // 连续失败2次后立即停止(降低阈值,更快停止) + maxConsecutiveFailures := 2 // 连续失败 2 次后立即停止(降低阈值,更快停止) firstFailureItemID := "" var firstFailureError error - + for i, itemID := range itemIDs { if err := idx.IndexItem(ctx, itemID); err != nil { failedCount++ consecutiveFailures++ - + // 只在第一个失败时记录详细日志 if consecutiveFailures == 1 { firstFailureItemID = itemID @@ -414,15 +560,15 @@ func (idx *Indexer) RebuildIndex(ctx context.Context) error { zap.Error(err), ) } - + // 如果连续失败过多,可能是配置问题,立即停止索引 if consecutiveFailures >= maxConsecutiveFailures { - errorMsg := fmt.Sprintf("连续 %d 个知识项索引失败,可能存在配置问题(如嵌入模型配置错误、API密钥无效、余额不足等)。第一个失败项: %s, 错误: %v", consecutiveFailures, firstFailureItemID, firstFailureError) + errorMsg := fmt.Sprintf("连续 %d 个知识项索引失败,可能存在配置问题(如嵌入模型配置错误、API 密钥无效、余额不足等)。第一个失败项:%s, 错误:%v", consecutiveFailures, firstFailureItemID, firstFailureError) idx.mu.Lock() idx.lastError = errorMsg idx.lastErrorTime = time.Now() idx.mu.Unlock() - + idx.logger.Error("连续索引失败次数过多,立即停止索引", zap.Int("consecutiveFailures", consecutiveFailures), zap.Int("totalItems", len(itemIDs)), @@ -430,17 +576,17 @@ func (idx *Indexer) RebuildIndex(ctx context.Context) error { zap.String("firstFailureItemId", firstFailureItemID), zap.Error(firstFailureError), ) - return fmt.Errorf("连续索引失败次数过多: %v", firstFailureError) + return fmt.Errorf("连续索引失败次数过多:%v", firstFailureError) } - - // 如果失败的知识项过多,记录警告但继续处理(降低阈值到30%) + + // 如果失败的知识项过多,记录警告但继续处理(降低阈值到 30%) if failedCount > len(itemIDs)*3/10 && failedCount == len(itemIDs)*3/10+1 { - errorMsg := fmt.Sprintf("索引失败的知识项过多 (%d/%d),可能存在配置问题。第一个失败项: %s, 错误: %v", failedCount, len(itemIDs), firstFailureItemID, firstFailureError) + errorMsg := fmt.Sprintf("索引失败的知识项过多 (%d/%d),可能存在配置问题。第一个失败项:%s, 错误:%v", failedCount, len(itemIDs), firstFailureItemID, firstFailureError) idx.mu.Lock() idx.lastError = errorMsg idx.lastErrorTime = time.Now() idx.mu.Unlock() - + idx.logger.Error("索引失败的知识项过多,可能存在配置问题", zap.Int("failedCount", failedCount), zap.Int("totalItems", len(itemIDs)), @@ -450,20 +596,31 @@ func (idx *Indexer) RebuildIndex(ctx context.Context) error { } continue } - + // 成功时重置连续失败计数和第一个失败信息 if consecutiveFailures > 0 { consecutiveFailures = 0 firstFailureItemID = "" firstFailureError = nil } - - // 减少进度日志频率(每10个或每10%记录一次) + + // 更新重建进度 + idx.rebuildMu.Lock() + idx.rebuildCurrent = i + 1 + idx.rebuildFailed = failedCount + idx.rebuildMu.Unlock() + + // 减少进度日志频率(每 10 个或每 10% 记录一次) if (i+1)%10 == 0 || (len(itemIDs) > 0 && (i+1)*100/len(itemIDs)%10 == 0 && (i+1)*100/len(itemIDs) > 0) { idx.logger.Info("索引进度", zap.Int("current", i+1), zap.Int("total", len(itemIDs)), zap.Int("failed", failedCount)) } } + // 重置重建状态 + idx.rebuildMu.Lock() + idx.isRebuilding = false + idx.rebuildMu.Unlock() + idx.logger.Info("索引重建完成", zap.Int("totalItems", len(itemIDs)), zap.Int("failedCount", failedCount)) return nil } @@ -474,3 +631,10 @@ func (idx *Indexer) GetLastError() (string, time.Time) { defer idx.mu.RUnlock() return idx.lastError, idx.lastErrorTime } + +// GetRebuildStatus 获取重建索引状态 +func (idx *Indexer) GetRebuildStatus() (isRebuilding bool, totalItems int, current int, failed int, lastItemID string, lastChunks int, startTime time.Time) { + idx.rebuildMu.RLock() + defer idx.rebuildMu.RUnlock() + return idx.isRebuilding, idx.rebuildTotalItems, idx.rebuildCurrent, idx.rebuildFailed, idx.rebuildLastItemID, idx.rebuildLastChunks, idx.rebuildStartTime +} diff --git a/web/static/js/knowledge.js b/web/static/js/knowledge.js index 00ece303..950120df 100644 --- a/web/static/js/knowledge.js +++ b/web/static/js/knowledge.js @@ -459,6 +459,9 @@ async function updateIndexProgress() { const isComplete = status.is_complete || false; const lastError = status.last_error || ''; + // 检查是否正在重建索引(优先使用重建状态) + const isRebuilding = status.is_rebuilding || false; + if (totalItems === 0) { // 没有知识项,隐藏进度条 progressContainer.style.display = 'none'; @@ -524,6 +527,45 @@ async function updateIndexProgress() { return; } + + // 优先处理重建状态 + if (isRebuilding) { + const rebuildTotal = status.rebuild_total || totalItems; + const rebuildCurrent = status.rebuild_current || 0; + const rebuildFailed = status.rebuild_failed || 0; + const rebuildLastItemID = status.rebuild_last_item_id || ''; + const rebuildLastChunks = status.rebuild_last_chunks || 0; + const rebuildStartTime = status.rebuild_start_time || ''; + + // 计算进度百分比(使用重建进度) + let rebuildProgress = progressPercent; + if (rebuildTotal > 0) { + rebuildProgress = (rebuildCurrent / rebuildTotal) * 100; + } + + progressContainer.innerHTML = ` +
+
+ 🔨 + 正在重建索引:${rebuildCurrent}/${rebuildTotal} (${rebuildProgress.toFixed(1)}%) - 失败:${rebuildFailed} +
+
+
+
+
+ ${rebuildLastItemID ? `正在处理:${escapeHtml(rebuildLastItemID.substring(0, 36))}... (${rebuildLastChunks} chunks)` : '正在处理...'} + ${rebuildStartTime ? `
开始时间:${new Date(rebuildStartTime).toLocaleString()}` : ''} +
+
+ `; + + // 重建中时继续轮询 + if (!indexProgressInterval) { + indexProgressInterval = setInterval(updateIndexProgress, 2000); + } + return; + } + if (isComplete) { progressContainer.innerHTML = `
diff --git a/web/static/js/knowledge.js.bak b/web/static/js/knowledge.js.bak new file mode 100644 index 00000000..00ece303 --- /dev/null +++ b/web/static/js/knowledge.js.bak @@ -0,0 +1,2216 @@ +// 知识库管理相关功能 +let knowledgeCategories = []; +let knowledgeItems = []; +let currentEditingItemId = null; +let isSavingKnowledgeItem = false; // 防止重复提交 +let retrievalLogsData = []; // 存储检索日志数据,用于详情查看 +let knowledgePagination = { + currentPage: 1, + pageSize: 10, // 每页分类数(改为按分类分页) + total: 0, + currentCategory: '' +}; +let knowledgeSearchTimeout = null; // 搜索防抖定时器 + +// 加载知识分类 +async function loadKnowledgeCategories() { + try { + // 添加时间戳参数避免缓存 + const timestamp = Date.now(); + const response = await apiFetch(`/api/knowledge/categories?_t=${timestamp}`, { + method: 'GET', + headers: { + 'Cache-Control': 'no-cache, no-store, must-revalidate', + 'Pragma': 'no-cache', + 'Expires': '0' + } + }); + if (!response.ok) { + throw new Error('获取分类失败'); + } + const data = await response.json(); + + // 检查知识库功能是否启用 + if (data.enabled === false) { + // 功能未启用,显示友好提示 + const container = document.getElementById('knowledge-items-list'); + if (container) { + container.innerHTML = ` +
+
📚
+

知识库功能未启用

+

${data.message || '请前往系统设置启用知识检索功能'}

+ +
+ `; + } + return []; + } + + knowledgeCategories = data.categories || []; + + // 更新分类筛选下拉框 + const filterDropdown = document.getElementById('knowledge-category-filter-dropdown'); + if (filterDropdown) { + filterDropdown.innerHTML = '
全部
'; + knowledgeCategories.forEach(category => { + const option = document.createElement('div'); + option.className = 'custom-select-option'; + option.setAttribute('data-value', category); + option.textContent = category; + option.onclick = function() { + selectKnowledgeCategory(category); + }; + filterDropdown.appendChild(option); + }); + } + + return knowledgeCategories; + } catch (error) { + console.error('加载分类失败:', error); + // 只在非功能未启用的情况下显示错误 + if (!error.message.includes('知识库功能未启用')) { + showNotification('加载分类失败: ' + error.message, 'error'); + } + return []; + } +} + +// 加载知识项列表(支持按分类分页,默认不加载完整内容) +async function loadKnowledgeItems(category = '', page = 1, pageSize = 10) { + try { + // 更新分页状态 + knowledgePagination.currentCategory = category; + knowledgePagination.currentPage = page; + knowledgePagination.pageSize = pageSize; + + // 构建URL(按分类分页模式,不包含完整内容) + const timestamp = Date.now(); + const offset = (page - 1) * pageSize; + let url = `/api/knowledge/items?categoryPage=true&limit=${pageSize}&offset=${offset}&_t=${timestamp}`; + if (category) { + url += `&category=${encodeURIComponent(category)}`; + } + + const response = await apiFetch(url, { + method: 'GET', + headers: { + 'Cache-Control': 'no-cache, no-store, must-revalidate', + 'Pragma': 'no-cache', + 'Expires': '0' + } + }); + + if (!response.ok) { + throw new Error('获取知识项失败'); + } + const data = await response.json(); + + // 检查知识库功能是否启用 + if (data.enabled === false) { + // 功能未启用,显示友好提示(如果还没有显示的话) + const container = document.getElementById('knowledge-items-list'); + if (container && !container.querySelector('.empty-state')) { + container.innerHTML = ` +
+
📚
+

知识库功能未启用

+

${data.message || '请前往系统设置启用知识检索功能'}

+ +
+ `; + } + knowledgeItems = []; + knowledgePagination.total = 0; + renderKnowledgePagination(); + return []; + } + + // 处理按分类分页的响应数据 + const categoriesWithItems = data.categories || []; + knowledgePagination.total = data.total || 0; // 总分类数 + + renderKnowledgeItemsByCategories(categoriesWithItems); + + // 如果选择了单个分类,不显示分页(因为只显示一个分类) + if (category) { + const paginationContainer = document.getElementById('knowledge-pagination'); + if (paginationContainer) { + paginationContainer.innerHTML = ''; + } + } else { + renderKnowledgePagination(); + } + return categoriesWithItems; + } catch (error) { + console.error('加载知识项失败:', error); + // 只在非功能未启用的情况下显示错误 + if (!error.message.includes('知识库功能未启用')) { + showNotification('加载知识项失败: ' + error.message, 'error'); + } + return []; + } +} + +// 渲染知识项列表(按分类分页的数据结构) +function renderKnowledgeItemsByCategories(categoriesWithItems) { + const container = document.getElementById('knowledge-items-list'); + if (!container) return; + + if (categoriesWithItems.length === 0) { + container.innerHTML = '
暂无知识项
'; + return; + } + + // 计算总项数和分类数 + const totalItems = categoriesWithItems.reduce((sum, cat) => sum + (cat.items?.length || 0), 0); + const categoryCount = categoriesWithItems.length; + + // 更新统计信息 + updateKnowledgeStats(categoriesWithItems, categoryCount); + + // 渲染分类及知识项 + let html = '
'; + + categoriesWithItems.forEach(categoryData => { + const category = categoryData.category || '未分类'; + const categoryItems = categoryData.items || []; + const categoryCount = categoryData.itemCount || categoryItems.length; + + html += ` +
+
+
+

${escapeHtml(category)}

+ ${categoryCount} 项 +
+
+
+ ${categoryItems.map(item => renderKnowledgeItemCard(item)).join('')} +
+
+ `; + }); + + html += '
'; + container.innerHTML = html; +} + +// 渲染知识项列表(向后兼容,用于按项分页的旧代码) +function renderKnowledgeItems(items) { + const container = document.getElementById('knowledge-items-list'); + if (!container) return; + + if (items.length === 0) { + container.innerHTML = '
暂无知识项
'; + return; + } + + // 按分类分组 + const groupedByCategory = {}; + items.forEach(item => { + const category = item.category || '未分类'; + if (!groupedByCategory[category]) { + groupedByCategory[category] = []; + } + groupedByCategory[category].push(item); + }); + + // 更新统计信息 + updateKnowledgeStats(items, Object.keys(groupedByCategory).length); + + // 渲染分组后的内容 + const categories = Object.keys(groupedByCategory).sort(); + let html = '
'; + + categories.forEach(category => { + const categoryItems = groupedByCategory[category]; + const categoryCount = categoryItems.length; + + html += ` +
+
+
+

${escapeHtml(category)}

+ ${categoryCount} 项 +
+
+
+ ${categoryItems.map(item => renderKnowledgeItemCard(item)).join('')} +
+
+ `; + }); + + html += '
'; + container.innerHTML = html; +} + +// 渲染分页控件(按分类分页) +function renderKnowledgePagination() { + const container = document.getElementById('knowledge-pagination'); + if (!container) return; + + const { currentPage, pageSize, total } = knowledgePagination; + const totalPages = Math.ceil(total / pageSize); // total是总分类数 + + if (totalPages <= 1) { + container.innerHTML = ''; + return; + } + + let html = '
'; + + // 上一页按钮 + html += ``; + + // 页码显示(显示分类数) + html += `第 ${currentPage} 页,共 ${totalPages} 页(共 ${total} 个分类)`; + + // 下一页按钮 + html += ``; + + html += '
'; + container.innerHTML = html; +} + +// 加载指定页码的知识项 +function loadKnowledgePage(page) { + const { currentCategory, pageSize, total } = knowledgePagination; + const totalPages = Math.ceil(total / pageSize); + + if (page < 1 || page > totalPages) { + return; + } + + loadKnowledgeItems(currentCategory, page, pageSize); +} + +// 渲染单个知识项卡片 +function renderKnowledgeItemCard(item) { + // 提取内容预览(如果item没有content字段,说明是摘要,不显示预览) + let previewText = ''; + if (item.content) { + // 去除markdown格式,取前150字符 + let preview = item.content; + // 移除markdown标题标记 + preview = preview.replace(/^#+\s+/gm, ''); + // 移除代码块 + preview = preview.replace(/```[\s\S]*?```/g, ''); + // 移除行内代码 + preview = preview.replace(/`[^`]+`/g, ''); + // 移除链接 + preview = preview.replace(/\[([^\]]+)\]\([^\)]+\)/g, '$1'); + // 清理多余空白 + preview = preview.replace(/\n+/g, ' ').replace(/\s+/g, ' ').trim(); + + previewText = preview.length > 150 ? preview.substring(0, 150) + '...' : preview; + } + + // 提取文件路径显示 + const filePath = item.filePath || ''; + const relativePath = filePath.split(/[/\\]/).slice(-2).join('/'); // 显示最后两级路径 + + // 格式化时间 + const createdTime = formatTime(item.createdAt); + const updatedTime = formatTime(item.updatedAt); + + // 优先显示更新时间,如果没有更新时间则显示创建时间 + const displayTime = updatedTime || createdTime; + const timeLabel = updatedTime ? '更新时间' : '创建时间'; + + // 判断是否为最近更新(7天内) + let isRecent = false; + if (item.updatedAt && updatedTime) { + const updateDate = new Date(item.updatedAt); + if (!isNaN(updateDate.getTime())) { + isRecent = (Date.now() - updateDate.getTime()) < 7 * 24 * 60 * 60 * 1000; + } + } + + return ` +
+
+
+

${escapeHtml(item.title)}

+
+ + +
+
+ ${relativePath ? `
📁 ${escapeHtml(relativePath)}
` : ''} +
+ ${previewText ? ` +
+

${escapeHtml(previewText)}

+
+ ` : ''} + +
+ `; +} + +// 更新统计信息(支持按分类分页的数据结构) +function updateKnowledgeStats(data, categoryCount) { + const statsContainer = document.getElementById('knowledge-stats'); + if (!statsContainer) return; + + // 计算当前页的知识项数 + let currentPageItemCount = 0; + if (Array.isArray(data) && data.length > 0) { + // 判断是categoriesWithItems还是items数组 + if (data[0].category !== undefined && data[0].items !== undefined) { + // 是按分类分页的数据结构 + currentPageItemCount = data.reduce((sum, cat) => sum + (cat.items?.length || 0), 0); + } else { + // 是按项分页的数据结构(向后兼容) + currentPageItemCount = data.length; + } + } + + // 总分类数(来自分页信息,只有在未定义时才使用当前页分类数作为后备值) + const totalCategories = (knowledgePagination.total != null) ? knowledgePagination.total : categoryCount; + + statsContainer.innerHTML = ` +
+ 总分类数 + ${totalCategories} +
+
+ 当前页分类 + ${categoryCount} 个 +
+
+ 当前页知识项 + ${currentPageItemCount} 项 +
+ `; + + // 更新索引进度 + updateIndexProgress(); +} + +// 更新索引进度 +let indexProgressInterval = null; + +async function updateIndexProgress() { + try { + const response = await apiFetch('/api/knowledge/index-status', { + method: 'GET', + headers: { + 'Cache-Control': 'no-cache, no-store, must-revalidate', + 'Pragma': 'no-cache', + 'Expires': '0' + } + }); + + if (!response.ok) { + return; // 静默失败,不影响主界面 + } + + const status = await response.json(); + const progressContainer = document.getElementById('knowledge-index-progress'); + if (!progressContainer) return; + + // 检查知识库功能是否启用 + if (status.enabled === false) { + // 功能未启用,隐藏进度条 + progressContainer.style.display = 'none'; + if (indexProgressInterval) { + clearInterval(indexProgressInterval); + indexProgressInterval = null; + } + return; + } + + const totalItems = status.total_items || 0; + const indexedItems = status.indexed_items || 0; + const progressPercent = status.progress_percent || 0; + const isComplete = status.is_complete || false; + const lastError = status.last_error || ''; + + if (totalItems === 0) { + // 没有知识项,隐藏进度条 + progressContainer.style.display = 'none'; + if (indexProgressInterval) { + clearInterval(indexProgressInterval); + indexProgressInterval = null; + } + return; + } + + // 显示进度条 + progressContainer.style.display = 'block'; + + // 如果有错误信息,显示错误 + if (lastError) { + progressContainer.innerHTML = ` +
+
+ + 索引构建失败 +
+
+ ${escapeHtml(lastError)} +
+
+ 可能的原因:嵌入模型配置错误、API密钥无效、余额不足等。请检查配置后重试。 +
+
+ + +
+
+ `; + // 停止轮询 + if (indexProgressInterval) { + clearInterval(indexProgressInterval); + indexProgressInterval = null; + } + // 显示错误通知 + showNotification('索引构建失败: ' + lastError.substring(0, 100), 'error'); + return; + } + + if (isComplete) { + progressContainer.innerHTML = ` +
+ + 索引构建完成 (${indexedItems}/${totalItems}) +
+ `; + // 完成后停止轮询 + if (indexProgressInterval) { + clearInterval(indexProgressInterval); + indexProgressInterval = null; + } + } else { + progressContainer.innerHTML = ` +
+
+ 🔨 + 正在构建索引: ${indexedItems}/${totalItems} (${progressPercent.toFixed(1)}%) +
+
+
+
+
索引构建完成后,语义搜索功能将可用
+
+ `; + + // 如果还没有开始轮询,开始轮询 + if (!indexProgressInterval) { + indexProgressInterval = setInterval(updateIndexProgress, 3000); // 每3秒刷新一次 + } + } + } catch (error) { + // 显示错误信息 + console.error('获取索引状态失败:', error); + const progressContainer = document.getElementById('knowledge-index-progress'); + if (progressContainer) { + progressContainer.style.display = 'block'; + progressContainer.innerHTML = ` +
+
+ ⚠️ + 无法获取索引状态 +
+
+ 无法连接到服务器获取索引状态,请检查网络连接或刷新页面。 +
+
+ `; + } + // 停止轮询 + if (indexProgressInterval) { + clearInterval(indexProgressInterval); + indexProgressInterval = null; + } + } +} + +// 停止索引进度轮询 +function stopIndexProgressPolling() { + if (indexProgressInterval) { + clearInterval(indexProgressInterval); + indexProgressInterval = null; + } + const progressContainer = document.getElementById('knowledge-index-progress'); + if (progressContainer) { + progressContainer.style.display = 'none'; + } +} + +// 选择知识分类 +function selectKnowledgeCategory(category) { + const trigger = document.getElementById('knowledge-category-filter-trigger'); + const wrapper = document.getElementById('knowledge-category-filter-wrapper'); + const dropdown = document.getElementById('knowledge-category-filter-dropdown'); + + if (trigger && wrapper && dropdown) { + const displayText = category || '全部'; + trigger.querySelector('span').textContent = displayText; + wrapper.classList.remove('open'); + + // 更新选中状态 + dropdown.querySelectorAll('.custom-select-option').forEach(opt => { + opt.classList.remove('selected'); + if (opt.getAttribute('data-value') === category) { + opt.classList.add('selected'); + } + }); + } + // 切换分类时重置到第一页(如果选择了分类,API会返回该分类的所有项) + loadKnowledgeItems(category, 1, knowledgePagination.pageSize); +} + +// 筛选知识项 +function filterKnowledgeItems() { + const wrapper = document.getElementById('knowledge-category-filter-wrapper'); + if (wrapper) { + const selectedOption = wrapper.querySelector('.custom-select-option.selected'); + const category = selectedOption ? selectedOption.getAttribute('data-value') : ''; + // 重置到第一页 + loadKnowledgeItems(category, 1, knowledgePagination.pageSize); + } +} + +// 处理搜索输入(带防抖) +function handleKnowledgeSearchInput() { + const searchInput = document.getElementById('knowledge-search'); + const searchTerm = searchInput?.value.trim() || ''; + + // 清除之前的定时器 + if (knowledgeSearchTimeout) { + clearTimeout(knowledgeSearchTimeout); + } + + // 如果搜索框为空,立即恢复列表 + if (!searchTerm) { + const wrapper = document.getElementById('knowledge-category-filter-wrapper'); + let category = ''; + if (wrapper) { + const selectedOption = wrapper.querySelector('.custom-select-option.selected'); + category = selectedOption ? selectedOption.getAttribute('data-value') : ''; + } + loadKnowledgeItems(category, 1, knowledgePagination.pageSize); + return; + } + + // 有搜索词时,延迟500ms后执行搜索(防抖) + knowledgeSearchTimeout = setTimeout(() => { + searchKnowledgeItems(); + }, 500); +} + +// 搜索知识项(后端关键字匹配,在所有数据中搜索) +async function searchKnowledgeItems() { + const searchInput = document.getElementById('knowledge-search'); + const searchTerm = searchInput?.value.trim() || ''; + + if (!searchTerm) { + // 恢复原始列表(重置到第一页) + const wrapper = document.getElementById('knowledge-category-filter-wrapper'); + let category = ''; + if (wrapper) { + const selectedOption = wrapper.querySelector('.custom-select-option.selected'); + category = selectedOption ? selectedOption.getAttribute('data-value') : ''; + } + await loadKnowledgeItems(category, 1, knowledgePagination.pageSize); + return; + } + + try { + // 获取当前选择的分类 + const wrapper = document.getElementById('knowledge-category-filter-wrapper'); + let category = ''; + if (wrapper) { + const selectedOption = wrapper.querySelector('.custom-select-option.selected'); + category = selectedOption ? selectedOption.getAttribute('data-value') : ''; + } + + // 调用后端API进行全量搜索 + const timestamp = Date.now(); + let url = `/api/knowledge/items?search=${encodeURIComponent(searchTerm)}&_t=${timestamp}`; + if (category) { + url += `&category=${encodeURIComponent(category)}`; + } + + const response = await apiFetch(url, { + method: 'GET', + headers: { + 'Cache-Control': 'no-cache, no-store, must-revalidate', + 'Pragma': 'no-cache', + 'Expires': '0' + } + }); + + if (!response.ok) { + throw new Error('搜索失败'); + } + + const data = await response.json(); + + // 检查知识库功能是否启用 + if (data.enabled === false) { + const container = document.getElementById('knowledge-items-list'); + if (container) { + container.innerHTML = ` +
+
📚
+

知识库功能未启用

+

${data.message || '请前往系统设置启用知识检索功能'}

+ +
+ `; + } + return; + } + + // 处理搜索结果 + const categoriesWithItems = data.categories || []; + + // 渲染搜索结果 + const container = document.getElementById('knowledge-items-list'); + if (!container) return; + + if (categoriesWithItems.length === 0) { + container.innerHTML = ` +
+
🔍
+

未找到匹配的知识项

+

关键词 "${escapeHtml(searchTerm)}" 在所有数据中没有匹配结果

+

请尝试其他关键词,或使用分类筛选功能

+
+ `; + } else { + // 计算总项数和分类数 + const totalItems = categoriesWithItems.reduce((sum, cat) => sum + (cat.items?.length || 0), 0); + const categoryCount = categoriesWithItems.length; + + // 更新统计信息 + updateKnowledgeStats(categoriesWithItems, categoryCount); + + // 渲染搜索结果 + renderKnowledgeItemsByCategories(categoriesWithItems); + } + + // 搜索时隐藏分页(因为搜索结果显示所有匹配结果) + const paginationContainer = document.getElementById('knowledge-pagination'); + if (paginationContainer) { + paginationContainer.innerHTML = ''; + } + + } catch (error) { + console.error('搜索知识项失败:', error); + showNotification('搜索失败: ' + error.message, 'error'); + } +} + +// 刷新知识库 +async function refreshKnowledgeBase() { + try { + showNotification('正在扫描知识库...', 'info'); + const response = await apiFetch('/api/knowledge/scan', { + method: 'POST' + }); + if (!response.ok) { + throw new Error('扫描知识库失败'); + } + const data = await response.json(); + // 根据返回的消息显示不同的提示 + if (data.items_to_index && data.items_to_index > 0) { + showNotification(`扫描完成,开始索引 ${data.items_to_index} 个新添加或更新的知识项`, 'success'); + } else { + showNotification(data.message || '扫描完成,没有需要索引的新项或更新项', 'success'); + } + // 重新加载知识项(重置到第一页) + await loadKnowledgeCategories(); + await loadKnowledgeItems(knowledgePagination.currentCategory, 1, knowledgePagination.pageSize); + + // 停止现有的轮询 + if (indexProgressInterval) { + clearInterval(indexProgressInterval); + indexProgressInterval = null; + } + + // 如果有需要索引的项,等待一小段时间后立即更新进度 + if (data.items_to_index && data.items_to_index > 0) { + await new Promise(resolve => setTimeout(resolve, 500)); + updateIndexProgress(); + // 开始轮询进度(每2秒刷新一次) + if (!indexProgressInterval) { + indexProgressInterval = setInterval(updateIndexProgress, 2000); + } + } else { + // 没有需要索引的项,也更新一次以显示当前状态 + updateIndexProgress(); + } + } catch (error) { + console.error('刷新知识库失败:', error); + showNotification('刷新知识库失败: ' + error.message, 'error'); + } +} + +// 重建索引 +async function rebuildKnowledgeIndex() { + try { + if (!confirm('确定要重建索引吗?这可能需要一些时间。')) { + return; + } + showNotification('正在重建索引...', 'info'); + + // 先停止现有的轮询 + if (indexProgressInterval) { + clearInterval(indexProgressInterval); + indexProgressInterval = null; + } + + // 立即显示"正在重建"状态,因为重建开始时会清空旧索引 + const progressContainer = document.getElementById('knowledge-index-progress'); + if (progressContainer) { + progressContainer.style.display = 'block'; + progressContainer.innerHTML = ` +
+
+ 🔨 + 正在重建索引: 准备中... +
+
+
+
+
索引构建完成后,语义搜索功能将可用
+
+ `; + } + + const response = await apiFetch('/api/knowledge/index', { + method: 'POST' + }); + if (!response.ok) { + throw new Error('重建索引失败'); + } + showNotification('索引重建已开始,将在后台进行', 'success'); + + // 等待一小段时间,确保后端已经开始处理并清空了旧索引 + await new Promise(resolve => setTimeout(resolve, 500)); + + // 立即更新一次进度 + updateIndexProgress(); + + // 开始轮询进度(每2秒刷新一次,比默认的3秒更频繁) + if (!indexProgressInterval) { + indexProgressInterval = setInterval(updateIndexProgress, 2000); + } + } catch (error) { + console.error('重建索引失败:', error); + showNotification('重建索引失败: ' + error.message, 'error'); + } +} + +// 显示添加知识项模态框 +function showAddKnowledgeItemModal() { + currentEditingItemId = null; + document.getElementById('knowledge-item-modal-title').textContent = '添加知识'; + document.getElementById('knowledge-item-category').value = ''; + document.getElementById('knowledge-item-title').value = ''; + document.getElementById('knowledge-item-content').value = ''; + document.getElementById('knowledge-item-modal').style.display = 'block'; +} + +// 编辑知识项 +async function editKnowledgeItem(id) { + try { + const response = await apiFetch(`/api/knowledge/items/${id}`); + if (!response.ok) { + throw new Error('获取知识项失败'); + } + const item = await response.json(); + + currentEditingItemId = id; + document.getElementById('knowledge-item-modal-title').textContent = '编辑知识'; + document.getElementById('knowledge-item-category').value = item.category; + document.getElementById('knowledge-item-title').value = item.title; + document.getElementById('knowledge-item-content').value = item.content; + document.getElementById('knowledge-item-modal').style.display = 'block'; + } catch (error) { + console.error('编辑知识项失败:', error); + showNotification('编辑知识项失败: ' + error.message, 'error'); + } +} + +// 保存知识项 +async function saveKnowledgeItem() { + // 防止重复提交 + if (isSavingKnowledgeItem) { + showNotification('正在保存中,请勿重复点击...', 'warning'); + return; + } + + const category = document.getElementById('knowledge-item-category').value.trim(); + const title = document.getElementById('knowledge-item-title').value.trim(); + const content = document.getElementById('knowledge-item-content').value.trim(); + + if (!category || !title || !content) { + showNotification('请填写所有必填字段', 'error'); + return; + } + + // 设置保存中标志 + isSavingKnowledgeItem = true; + + // 获取保存按钮和取消按钮 + const saveButton = document.querySelector('#knowledge-item-modal .modal-footer .btn-primary'); + const cancelButton = document.querySelector('#knowledge-item-modal .modal-footer .btn-secondary'); + const modal = document.getElementById('knowledge-item-modal'); + + const originalButtonText = saveButton ? saveButton.textContent : '保存'; + const originalButtonDisabled = saveButton ? saveButton.disabled : false; + + // 禁用所有输入字段和按钮 + const categoryInput = document.getElementById('knowledge-item-category'); + const titleInput = document.getElementById('knowledge-item-title'); + const contentInput = document.getElementById('knowledge-item-content'); + + if (categoryInput) categoryInput.disabled = true; + if (titleInput) titleInput.disabled = true; + if (contentInput) contentInput.disabled = true; + if (cancelButton) cancelButton.disabled = true; + + // 设置保存按钮加载状态 + if (saveButton) { + saveButton.disabled = true; + saveButton.style.opacity = '0.6'; + saveButton.style.cursor = 'not-allowed'; + saveButton.textContent = '保存中...'; + } + + try { + const url = currentEditingItemId + ? `/api/knowledge/items/${currentEditingItemId}` + : '/api/knowledge/items'; + const method = currentEditingItemId ? 'PUT' : 'POST'; + + const response = await apiFetch(url, { + method: method, + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + category, + title, + content + }) + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + throw new Error(errorData.error || '保存知识项失败'); + } + + const item = await response.json(); + const action = currentEditingItemId ? '更新' : '创建'; + const newItemCategory = item.category || category; // 保存新添加的知识项分类 + + // 获取当前筛选状态,以便刷新后保持 + const currentCategory = document.getElementById('knowledge-category-filter-wrapper'); + let selectedCategory = ''; + if (currentCategory) { + const selectedOption = currentCategory.querySelector('.custom-select-option.selected'); + if (selectedOption) { + selectedCategory = selectedOption.getAttribute('data-value') || ''; + } + } + + // 立即关闭模态框,给用户明确的反馈 + closeKnowledgeItemModal(); + + // 显示加载状态并刷新数据(等待完成以确保数据同步) + const itemsListContainer = document.getElementById('knowledge-items-list'); + const originalContent = itemsListContainer ? itemsListContainer.innerHTML : ''; + + if (itemsListContainer) { + itemsListContainer.innerHTML = '
刷新中...
'; + } + + try { + // 先刷新分类,再刷新知识项 + console.log('开始刷新知识库数据...'); + await loadKnowledgeCategories(); + console.log('分类刷新完成,开始刷新知识项...'); + + // 如果新添加的知识项不在当前筛选的分类中,切换到该分类显示 + let categoryToShow = selectedCategory; + if (!currentEditingItemId && selectedCategory && selectedCategory !== '' && newItemCategory !== selectedCategory) { + // 新添加的知识项,如果当前筛选的不是该分类,切换到新知识项的分类 + categoryToShow = newItemCategory; + // 更新筛选器显示(不触发加载,因为我们下面会手动加载) + const trigger = document.getElementById('knowledge-category-filter-trigger'); + const wrapper = document.getElementById('knowledge-category-filter-wrapper'); + const dropdown = document.getElementById('knowledge-category-filter-dropdown'); + if (trigger && wrapper && dropdown) { + trigger.querySelector('span').textContent = newItemCategory || '全部'; + dropdown.querySelectorAll('.custom-select-option').forEach(opt => { + opt.classList.remove('selected'); + if (opt.getAttribute('data-value') === newItemCategory) { + opt.classList.add('selected'); + } + }); + } + showNotification(`✅ ${action}成功!已切换到分类"${newItemCategory}"查看新添加的知识项。`, 'success'); + } + + // 刷新知识项列表(重置到第一页) + await loadKnowledgeItems(categoryToShow, 1, knowledgePagination.pageSize); + console.log('知识项刷新完成'); + } catch (err) { + console.error('刷新数据失败:', err); + // 如果刷新失败,恢复原内容 + if (itemsListContainer && originalContent) { + itemsListContainer.innerHTML = originalContent; + } + showNotification('⚠️ 知识项已保存,但刷新列表失败,请手动刷新页面查看', 'warning'); + } + + } catch (error) { + console.error('保存知识项失败:', error); + showNotification('❌ 保存知识项失败: ' + error.message, 'error'); + + // 如果通知系统不可用,使用alert + if (typeof window.showNotification !== 'function') { + alert('❌ 保存知识项失败: ' + error.message); + } + + // 恢复输入字段和按钮状态(错误时不关闭模态框,让用户修改后重试) + if (categoryInput) categoryInput.disabled = false; + if (titleInput) titleInput.disabled = false; + if (contentInput) contentInput.disabled = false; + if (cancelButton) cancelButton.disabled = false; + if (saveButton) { + saveButton.disabled = false; + saveButton.style.opacity = ''; + saveButton.style.cursor = ''; + saveButton.textContent = originalButtonText; + } + } finally { + // 清除保存中标志 + isSavingKnowledgeItem = false; + } +} + +// 删除知识项 +async function deleteKnowledgeItem(id) { + if (!confirm('确定要删除这个知识项吗?')) { + return; + } + + // 找到要删除的知识项卡片和删除按钮 + const itemCard = document.querySelector(`.knowledge-item-card[data-id="${id}"]`); + const deleteButton = itemCard ? itemCard.querySelector('.knowledge-item-delete-btn') : null; + const categorySection = itemCard ? itemCard.closest('.knowledge-category-section') : null; + let originalDisplay = ''; + let originalOpacity = ''; + let originalButtonOpacity = ''; + + // 设置删除按钮的加载状态 + if (deleteButton) { + originalButtonOpacity = deleteButton.style.opacity; + deleteButton.style.opacity = '0.5'; + deleteButton.style.cursor = 'not-allowed'; + deleteButton.disabled = true; + + // 添加加载动画 + const svg = deleteButton.querySelector('svg'); + if (svg) { + svg.style.animation = 'spin 1s linear infinite'; + } + } + + // 立即从UI中移除该项(乐观更新) + if (itemCard) { + originalDisplay = itemCard.style.display; + originalOpacity = itemCard.style.opacity; + itemCard.style.transition = 'opacity 0.3s ease-out, transform 0.3s ease-out'; + itemCard.style.opacity = '0'; + itemCard.style.transform = 'translateX(-20px)'; + + // 等待动画完成后移除 + setTimeout(() => { + if (itemCard.parentElement) { + itemCard.remove(); + + // 检查分类是否还有项目,如果没有则隐藏分类标题 + if (categorySection) { + const remainingItems = categorySection.querySelectorAll('.knowledge-item-card'); + if (remainingItems.length === 0) { + categorySection.style.transition = 'opacity 0.3s ease-out'; + categorySection.style.opacity = '0'; + setTimeout(() => { + if (categorySection.parentElement) { + categorySection.remove(); + } + }, 300); + } else { + // 更新分类计数 + const categoryCount = categorySection.querySelector('.knowledge-category-count'); + if (categoryCount) { + const newCount = remainingItems.length; + categoryCount.textContent = `${newCount} 项`; + } + } + } + + // 不在这里更新统计信息,等待重新加载数据后由正确的逻辑更新 + } + }, 300); + } + + try { + const response = await apiFetch(`/api/knowledge/items/${id}`, { + method: 'DELETE' + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + throw new Error(errorData.error || '删除知识项失败'); + } + + // 显示成功通知 + showNotification('✅ 删除成功!知识项已从系统中移除。', 'success'); + + // 重新加载数据以确保数据同步(保持当前页码) + await loadKnowledgeCategories(); + await loadKnowledgeItems(knowledgePagination.currentCategory, knowledgePagination.currentPage, knowledgePagination.pageSize); + + } catch (error) { + console.error('删除知识项失败:', error); + + // 如果删除失败,恢复该项显示 + if (itemCard && originalDisplay !== 'none') { + itemCard.style.display = originalDisplay || ''; + itemCard.style.opacity = originalOpacity || '1'; + itemCard.style.transform = ''; + itemCard.style.transition = ''; + + // 如果分类被移除了,需要恢复 + if (categorySection && !categorySection.parentElement) { + // 需要重新加载来恢复(保持当前分页状态) + await loadKnowledgeItems(knowledgePagination.currentCategory, knowledgePagination.currentPage, knowledgePagination.pageSize); + } + } + + // 恢复删除按钮状态 + if (deleteButton) { + deleteButton.style.opacity = originalButtonOpacity || ''; + deleteButton.style.cursor = ''; + deleteButton.disabled = false; + const svg = deleteButton.querySelector('svg'); + if (svg) { + svg.style.animation = ''; + } + } + + showNotification('❌ 删除知识项失败: ' + error.message, 'error'); + } +} + +// 临时更新统计信息(删除后) +function updateKnowledgeStatsAfterDelete() { + const statsContainer = document.getElementById('knowledge-stats'); + if (!statsContainer) return; + + const allItems = document.querySelectorAll('.knowledge-item-card'); + const allCategories = document.querySelectorAll('.knowledge-category-section'); + + const totalItems = allItems.length; + const categoryCount = allCategories.length; + + // 计算总内容大小(这里简化处理,实际应该从服务器获取) + const statsItems = statsContainer.querySelectorAll('.knowledge-stat-item'); + if (statsItems.length >= 2) { + const totalItemsSpan = statsItems[0].querySelector('.knowledge-stat-value'); + const categoryCountSpan = statsItems[1].querySelector('.knowledge-stat-value'); + + if (totalItemsSpan) { + totalItemsSpan.textContent = totalItems; + } + if (categoryCountSpan) { + categoryCountSpan.textContent = categoryCount; + } + } +} + +// 关闭知识项模态框 +function closeKnowledgeItemModal() { + const modal = document.getElementById('knowledge-item-modal'); + if (modal) { + modal.style.display = 'none'; + } + + // 重置编辑状态 + currentEditingItemId = null; + isSavingKnowledgeItem = false; + + // 恢复所有输入字段和按钮状态 + const categoryInput = document.getElementById('knowledge-item-category'); + const titleInput = document.getElementById('knowledge-item-title'); + const contentInput = document.getElementById('knowledge-item-content'); + const saveButton = document.querySelector('#knowledge-item-modal .modal-footer .btn-primary'); + const cancelButton = document.querySelector('#knowledge-item-modal .modal-footer .btn-secondary'); + + if (categoryInput) { + categoryInput.disabled = false; + categoryInput.value = ''; + } + if (titleInput) { + titleInput.disabled = false; + titleInput.value = ''; + } + if (contentInput) { + contentInput.disabled = false; + contentInput.value = ''; + } + if (saveButton) { + saveButton.disabled = false; + saveButton.style.opacity = ''; + saveButton.style.cursor = ''; + saveButton.textContent = '保存'; + } + if (cancelButton) { + cancelButton.disabled = false; + } +} + +// 加载检索日志 +async function loadRetrievalLogs(conversationId = '', messageId = '') { + try { + let url = '/api/knowledge/retrieval-logs?limit=100'; + if (conversationId) { + url += `&conversationId=${encodeURIComponent(conversationId)}`; + } + if (messageId) { + url += `&messageId=${encodeURIComponent(messageId)}`; + } + + const response = await apiFetch(url); + if (!response.ok) { + throw new Error('获取检索日志失败'); + } + const data = await response.json(); + renderRetrievalLogs(data.logs || []); + } catch (error) { + console.error('加载检索日志失败:', error); + // 即使加载失败,也显示空状态而不是一直显示"加载中..." + renderRetrievalLogs([]); + // 只在非空筛选条件下才显示错误通知(避免在没有数据时显示错误) + if (conversationId || messageId) { + showNotification('加载检索日志失败: ' + error.message, 'error'); + } + } +} + +// 渲染检索日志 +function renderRetrievalLogs(logs) { + const container = document.getElementById('retrieval-logs-list'); + if (!container) return; + + // 更新统计信息(即使为空数组也要更新) + updateRetrievalStats(logs); + + if (logs.length === 0) { + container.innerHTML = '
暂无检索记录
'; + retrievalLogsData = []; + return; + } + + // 保存日志数据供详情查看使用 + retrievalLogsData = logs; + + container.innerHTML = logs.map((log, index) => { + // 处理retrievedItems:可能是数组、字符串数组,或者特殊标记 + let itemCount = 0; + let hasResults = false; + + if (log.retrievedItems) { + if (Array.isArray(log.retrievedItems)) { + // 过滤掉特殊标记 + const realItems = log.retrievedItems.filter(id => id !== '_has_results'); + itemCount = realItems.length; + // 如果有特殊标记,表示有结果但ID未知,显示为"有结果" + if (log.retrievedItems.includes('_has_results')) { + hasResults = true; + // 如果有真实ID,使用真实数量;否则显示为"有结果"(不显示具体数量) + if (itemCount === 0) { + itemCount = -1; // -1 表示有结果但数量未知 + } + } else { + hasResults = itemCount > 0; + } + } else if (typeof log.retrievedItems === 'string') { + // 如果是字符串,尝试解析JSON + try { + const parsed = JSON.parse(log.retrievedItems); + if (Array.isArray(parsed)) { + const realItems = parsed.filter(id => id !== '_has_results'); + itemCount = realItems.length; + if (parsed.includes('_has_results')) { + hasResults = true; + if (itemCount === 0) { + itemCount = -1; + } + } else { + hasResults = itemCount > 0; + } + } + } catch (e) { + // 解析失败,忽略 + } + } + } + + const timeAgo = getTimeAgo(log.createdAt); + + return ` +
+
+
+ ${hasResults ? '🔍' : '⚠️'} +
+
+
+ ${escapeHtml(log.query || '无查询内容')} +
+
+ + 🕒 ${timeAgo} + + ${log.riskType ? `📁 ${escapeHtml(log.riskType)}` : ''} +
+
+
+ ${hasResults ? (itemCount > 0 ? `${itemCount} 项` : '有结果') : '无结果'} +
+
+
+
+ ${log.conversationId ? ` +
+ 对话ID + ${escapeHtml(log.conversationId)} +
+ ` : ''} + ${log.messageId ? ` +
+ 消息ID + ${escapeHtml(log.messageId)} +
+ ` : ''} +
+ 检索结果 + + ${hasResults ? (itemCount > 0 ? `找到 ${itemCount} 个相关知识项` : '找到相关知识项(数量未知)') : '未找到匹配的知识项'} + +
+
+ ${hasResults && log.retrievedItems && log.retrievedItems.length > 0 ? ` +
+
检索到的知识项:
+
+ ${log.retrievedItems.slice(0, 3).map((itemId, idx) => ` + ${idx + 1} + `).join('')} + ${log.retrievedItems.length > 3 ? `+${log.retrievedItems.length - 3}` : ''} +
+
+ ` : ''} +
+ + +
+
+
+ `; + }).join(''); +} + +// 更新检索统计信息 +function updateRetrievalStats(logs) { + const statsContainer = document.getElementById('retrieval-stats'); + if (!statsContainer) return; + + const totalLogs = logs.length; + // 判断是否有结果:检查retrievedItems数组,过滤掉特殊标记后长度>0,或者包含特殊标记 + const successfulLogs = logs.filter(log => { + if (!log.retrievedItems) return false; + if (Array.isArray(log.retrievedItems)) { + const realItems = log.retrievedItems.filter(id => id !== '_has_results'); + return realItems.length > 0 || log.retrievedItems.includes('_has_results'); + } + return false; + }).length; + // 计算总知识项数(只计算真实ID,不包括特殊标记) + const totalItems = logs.reduce((sum, log) => { + if (!log.retrievedItems) return sum; + if (Array.isArray(log.retrievedItems)) { + const realItems = log.retrievedItems.filter(id => id !== '_has_results'); + return sum + realItems.length; + } + return sum; + }, 0); + const successRate = totalLogs > 0 ? ((successfulLogs / totalLogs) * 100).toFixed(1) : 0; + + statsContainer.innerHTML = ` +
+ 总检索次数 + ${totalLogs} +
+
+ 成功检索 + ${successfulLogs} +
+
+ 成功率 + ${successRate}% +
+
+ 检索到知识项 + ${totalItems} +
+ `; +} + +// 获取相对时间 +function getTimeAgo(timeStr) { + if (!timeStr) return ''; + + // 处理时间字符串,支持多种格式 + let date; + if (typeof timeStr === 'string') { + // 首先尝试直接解析(支持RFC3339/ISO8601格式) + date = new Date(timeStr); + + // 如果解析失败,尝试其他格式 + if (isNaN(date.getTime())) { + // SQLite格式: "2006-01-02 15:04:05" 或带时区 + const sqliteMatch = timeStr.match(/(\d{4}-\d{2}-\d{2}[\sT]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:\d{2}|Z)?)/); + if (sqliteMatch) { + let timeStr2 = sqliteMatch[1].replace(' ', 'T'); + // 如果没有时区信息,添加Z表示UTC + if (!timeStr2.includes('Z') && !timeStr2.match(/[+-]\d{2}:\d{2}$/)) { + timeStr2 += 'Z'; + } + date = new Date(timeStr2); + } + } + + // 如果还是失败,尝试更宽松的格式 + if (isNaN(date.getTime())) { + // 尝试匹配 "YYYY-MM-DD HH:MM:SS" 格式 + const match = timeStr.match(/(\d{4})-(\d{2})-(\d{2})[\sT](\d{2}):(\d{2}):(\d{2})/); + if (match) { + date = new Date( + parseInt(match[1]), + parseInt(match[2]) - 1, + parseInt(match[3]), + parseInt(match[4]), + parseInt(match[5]), + parseInt(match[6]) + ); + } + } + } else { + date = new Date(timeStr); + } + + // 检查日期是否有效 + if (isNaN(date.getTime())) { + return formatTime(timeStr); + } + + // 检查日期是否合理(不在1970年之前,不在未来太远) + const year = date.getFullYear(); + if (year < 1970 || year > 2100) { + return formatTime(timeStr); + } + + const now = new Date(); + const diff = now - date; + + // 如果时间差为负数或过大(可能是解析错误),返回格式化时间 + if (diff < 0 || diff > 365 * 24 * 60 * 60 * 1000 * 10) { // 超过10年认为是错误 + return formatTime(timeStr); + } + + const seconds = Math.floor(diff / 1000); + const minutes = Math.floor(seconds / 60); + const hours = Math.floor(minutes / 60); + const days = Math.floor(hours / 24); + + if (days > 0) return `${days}天前`; + if (hours > 0) return `${hours}小时前`; + if (minutes > 0) return `${minutes}分钟前`; + return '刚刚'; +} + +// 截断ID显示 +function truncateId(id) { + if (!id || id.length <= 16) return id; + return id.substring(0, 8) + '...' + id.substring(id.length - 8); +} + +// 筛选检索日志 +function filterRetrievalLogs() { + const conversationId = document.getElementById('retrieval-logs-conversation-id').value.trim(); + const messageId = document.getElementById('retrieval-logs-message-id').value.trim(); + loadRetrievalLogs(conversationId, messageId); +} + +// 刷新检索日志 +function refreshRetrievalLogs() { + filterRetrievalLogs(); +} + +// 删除检索日志 +async function deleteRetrievalLog(id, index) { + if (!confirm('确定要删除这条检索记录吗?')) { + return; + } + + // 找到要删除的日志卡片和删除按钮 + const logCard = document.querySelector(`.retrieval-log-card[data-index="${index}"]`); + const deleteButton = logCard ? logCard.querySelector('.retrieval-log-delete-btn') : null; + let originalButtonOpacity = ''; + let originalButtonDisabled = false; + + // 设置删除按钮的加载状态 + if (deleteButton) { + originalButtonOpacity = deleteButton.style.opacity; + originalButtonDisabled = deleteButton.disabled; + deleteButton.style.opacity = '0.5'; + deleteButton.style.cursor = 'not-allowed'; + deleteButton.disabled = true; + + // 添加加载动画 + const svg = deleteButton.querySelector('svg'); + if (svg) { + svg.style.animation = 'spin 1s linear infinite'; + } + } + + // 立即从UI中移除该项(乐观更新) + if (logCard) { + logCard.style.transition = 'opacity 0.3s ease-out, transform 0.3s ease-out'; + logCard.style.opacity = '0'; + logCard.style.transform = 'translateX(-20px)'; + + // 等待动画完成后移除 + setTimeout(() => { + if (logCard.parentElement) { + logCard.remove(); + + // 更新统计信息(临时更新,稍后会重新加载) + updateRetrievalStatsAfterDelete(); + } + }, 300); + } + + try { + const response = await apiFetch(`/api/knowledge/retrieval-logs/${id}`, { + method: 'DELETE' + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + throw new Error(errorData.error || '删除检索日志失败'); + } + + // 显示成功通知 + showNotification('✅ 删除成功!检索记录已从系统中移除。', 'success'); + + // 从内存中移除该项 + if (retrievalLogsData && index >= 0 && index < retrievalLogsData.length) { + retrievalLogsData.splice(index, 1); + } + + // 重新加载数据以确保数据同步 + const conversationId = document.getElementById('retrieval-logs-conversation-id')?.value.trim() || ''; + const messageId = document.getElementById('retrieval-logs-message-id')?.value.trim() || ''; + await loadRetrievalLogs(conversationId, messageId); + + } catch (error) { + console.error('删除检索日志失败:', error); + + // 如果删除失败,恢复该项显示 + if (logCard) { + logCard.style.opacity = '1'; + logCard.style.transform = ''; + logCard.style.transition = ''; + } + + // 恢复删除按钮状态 + if (deleteButton) { + deleteButton.style.opacity = originalButtonOpacity || ''; + deleteButton.style.cursor = ''; + deleteButton.disabled = originalButtonDisabled; + const svg = deleteButton.querySelector('svg'); + if (svg) { + svg.style.animation = ''; + } + } + + showNotification('❌ 删除检索日志失败: ' + error.message, 'error'); + } +} + +// 临时更新统计信息(删除后) +function updateRetrievalStatsAfterDelete() { + const statsContainer = document.getElementById('retrieval-stats'); + if (!statsContainer) return; + + const allLogs = document.querySelectorAll('.retrieval-log-card'); + const totalLogs = allLogs.length; + + // 计算成功检索数 + const successfulLogs = Array.from(allLogs).filter(card => { + return card.classList.contains('has-results'); + }).length; + + // 计算总知识项数(简化处理,实际应该从服务器获取) + const totalItems = Array.from(allLogs).reduce((sum, card) => { + const badge = card.querySelector('.retrieval-log-result-badge'); + if (badge && badge.classList.contains('success')) { + const text = badge.textContent.trim(); + const match = text.match(/(\d+)\s*项/); + if (match) { + return sum + parseInt(match[1]); + } else if (text === '有结果') { + return sum + 1; // 简化处理,假设为1 + } + } + return sum; + }, 0); + + const successRate = totalLogs > 0 ? ((successfulLogs / totalLogs) * 100).toFixed(1) : 0; + + statsContainer.innerHTML = ` +
+ 总检索次数 + ${totalLogs} +
+
+ 成功检索 + ${successfulLogs} +
+
+ 成功率 + ${successRate}% +
+
+ 检索到知识项 + ${totalItems} +
+ `; +} + +// 显示检索日志详情 +async function showRetrievalLogDetails(index) { + if (!retrievalLogsData || index < 0 || index >= retrievalLogsData.length) { + showNotification('无法获取检索详情', 'error'); + return; + } + + const log = retrievalLogsData[index]; + + // 获取检索到的知识项详情 + let retrievedItemsDetails = []; + if (log.retrievedItems && Array.isArray(log.retrievedItems)) { + const realItemIds = log.retrievedItems.filter(id => id !== '_has_results'); + if (realItemIds.length > 0) { + try { + // 批量获取知识项详情 + const itemPromises = realItemIds.map(async (itemId) => { + try { + const response = await apiFetch(`/api/knowledge/items/${itemId}`); + if (response.ok) { + return await response.json(); + } + return null; + } catch (err) { + console.error(`获取知识项 ${itemId} 失败:`, err); + return null; + } + }); + + const items = await Promise.all(itemPromises); + retrievedItemsDetails = items.filter(item => item !== null); + } catch (err) { + console.error('批量获取知识项详情失败:', err); + } + } + } + + // 显示详情模态框 + showRetrievalLogDetailsModal(log, retrievedItemsDetails); +} + +// 显示检索日志详情模态框 +function showRetrievalLogDetailsModal(log, retrievedItems) { + // 创建或获取模态框 + let modal = document.getElementById('retrieval-log-details-modal'); + if (!modal) { + modal = document.createElement('div'); + modal.id = 'retrieval-log-details-modal'; + modal.className = 'modal'; + modal.innerHTML = ` + + `; + document.body.appendChild(modal); + } + + // 填充内容 + const content = document.getElementById('retrieval-log-details-content'); + const timeAgo = getTimeAgo(log.createdAt); + const fullTime = formatTime(log.createdAt); + + let itemsHtml = ''; + if (retrievedItems.length > 0) { + itemsHtml = retrievedItems.map((item, idx) => { + // 提取内容预览 + let preview = item.content || ''; + preview = preview.replace(/^#+\s+/gm, ''); + preview = preview.replace(/```[\s\S]*?```/g, ''); + preview = preview.replace(/`[^`]+`/g, ''); + preview = preview.replace(/\[([^\]]+)\]\([^\)]+\)/g, '$1'); + preview = preview.replace(/\n+/g, ' ').replace(/\s+/g, ' ').trim(); + const previewText = preview.length > 200 ? preview.substring(0, 200) + '...' : preview; + + return ` +
+
+

${idx + 1}. ${escapeHtml(item.title || '未命名')}

+ ${escapeHtml(item.category || '未分类')} +
+ ${item.filePath ? `
📁 ${escapeHtml(item.filePath)}
` : ''} +
+ ${escapeHtml(previewText || '无内容预览')} +
+
+ `; + }).join(''); + } else { + itemsHtml = '
未找到知识项详情
'; + } + + content.innerHTML = ` +
+
+

查询信息

+
+
查询内容:
+
${escapeHtml(log.query || '无查询内容')}
+
+
+ +
+

检索信息

+
+ ${log.riskType ? ` +
+
风险类型
+
${escapeHtml(log.riskType)}
+
+ ` : ''} +
+
检索时间
+
${timeAgo}
+
+
+
检索结果
+
${retrievedItems.length} 个知识项
+
+
+
+ + ${log.conversationId || log.messageId ? ` +
+

关联信息

+
+ ${log.conversationId ? ` +
+
对话ID
+ ${escapeHtml(log.conversationId)} +
+ ` : ''} + ${log.messageId ? ` +
+
消息ID
+ ${escapeHtml(log.messageId)} +
+ ` : ''} +
+
+ ` : ''} + +
+

检索到的知识项 (${retrievedItems.length})

+ ${itemsHtml} +
+
+ `; + + modal.style.display = 'block'; +} + +// 关闭检索日志详情模态框 +function closeRetrievalLogDetailsModal() { + const modal = document.getElementById('retrieval-log-details-modal'); + if (modal) { + modal.style.display = 'none'; + } +} + +// 点击模态框外部关闭 +window.addEventListener('click', function(event) { + const modal = document.getElementById('retrieval-log-details-modal'); + if (event.target === modal) { + closeRetrievalLogDetailsModal(); + } +}); + +// 页面切换时加载数据 +if (typeof switchPage === 'function') { + const originalSwitchPage = switchPage; + window.switchPage = function(page) { + originalSwitchPage(page); + + if (page === 'knowledge-management') { + loadKnowledgeCategories(); + loadKnowledgeItems(knowledgePagination.currentCategory, 1, knowledgePagination.pageSize); + updateIndexProgress(); // 更新索引进度 + } else if (page === 'knowledge-retrieval-logs') { + loadRetrievalLogs(); + // 切换到其他页面时停止轮询 + if (indexProgressInterval) { + clearInterval(indexProgressInterval); + indexProgressInterval = null; + } + } else { + // 切换到其他页面时停止轮询 + if (indexProgressInterval) { + clearInterval(indexProgressInterval); + indexProgressInterval = null; + } + } + }; +} + +// 页面卸载时清理定时器 +window.addEventListener('beforeunload', function() { + if (indexProgressInterval) { + clearInterval(indexProgressInterval); + indexProgressInterval = null; + } +}); + +// 工具函数 +function escapeHtml(text) { + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; +} + +function formatTime(timeStr) { + if (!timeStr) return ''; + + // 处理时间字符串,支持多种格式 + let date; + if (typeof timeStr === 'string') { + // 首先尝试直接解析(支持RFC3339/ISO8601格式) + date = new Date(timeStr); + + // 如果解析失败,尝试其他格式 + if (isNaN(date.getTime())) { + // SQLite格式: "2006-01-02 15:04:05" 或带时区 + const sqliteMatch = timeStr.match(/(\d{4}-\d{2}-\d{2}[\sT]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:\d{2}|Z)?)/); + if (sqliteMatch) { + let timeStr2 = sqliteMatch[1].replace(' ', 'T'); + // 如果没有时区信息,添加Z表示UTC + if (!timeStr2.includes('Z') && !timeStr2.match(/[+-]\d{2}:\d{2}$/)) { + timeStr2 += 'Z'; + } + date = new Date(timeStr2); + } + } + + // 如果还是失败,尝试更宽松的格式 + if (isNaN(date.getTime())) { + // 尝试匹配 "YYYY-MM-DD HH:MM:SS" 格式 + const match = timeStr.match(/(\d{4})-(\d{2})-(\d{2})[\sT](\d{2}):(\d{2}):(\d{2})/); + if (match) { + date = new Date( + parseInt(match[1]), + parseInt(match[2]) - 1, + parseInt(match[3]), + parseInt(match[4]), + parseInt(match[5]), + parseInt(match[6]) + ); + } + } + } else { + date = new Date(timeStr); + } + + // 如果日期无效,检查是否是零值时间 + if (isNaN(date.getTime())) { + // 检查是否是零值时间的字符串形式 + if (typeof timeStr === 'string' && (timeStr.includes('0001-01-01') || timeStr.startsWith('0001'))) { + return ''; + } + console.warn('无法解析时间:', timeStr); + return ''; + } + + // 检查日期是否合理(不在1970年之前,不在未来太远) + const year = date.getFullYear(); + if (year < 1970 || year > 2100) { + // 如果是零值时间(0001-01-01),返回空字符串,不显示 + if (year === 1) { + return ''; + } + console.warn('时间值不合理:', timeStr, '解析为:', date); + return ''; + } + + return date.toLocaleString('zh-CN', { + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + hour12: false + }); +} + +// 显示通知 +function showNotification(message, type = 'info') { + // 如果存在全局通知系统(且不是当前函数),使用它 + if (typeof window.showNotification === 'function' && window.showNotification !== showNotification) { + window.showNotification(message, type); + return; + } + + // 否则使用自定义的toast通知 + showToastNotification(message, type); +} + +// 显示Toast通知 +function showToastNotification(message, type = 'info') { + // 创建通知容器(如果不存在) + let container = document.getElementById('toast-notification-container'); + if (!container) { + container = document.createElement('div'); + container.id = 'toast-notification-container'; + container.style.cssText = ` + position: fixed; + top: 20px; + right: 20px; + z-index: 10000; + display: flex; + flex-direction: column; + gap: 12px; + pointer-events: none; + `; + document.body.appendChild(container); + } + + // 创建通知元素 + const toast = document.createElement('div'); + toast.className = `toast-notification toast-${type}`; + + // 根据类型设置颜色 + const typeStyles = { + success: { + background: '#28a745', + color: '#fff', + icon: '✅' + }, + error: { + background: '#dc3545', + color: '#fff', + icon: '❌' + }, + info: { + background: '#17a2b8', + color: '#fff', + icon: 'ℹ️' + }, + warning: { + background: '#ffc107', + color: '#000', + icon: '⚠️' + } + }; + + const style = typeStyles[type] || typeStyles.info; + + toast.style.cssText = ` + background: ${style.background}; + color: ${style.color}; + padding: 14px 20px; + border-radius: 8px; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); + min-width: 300px; + max-width: 500px; + pointer-events: auto; + animation: slideInRight 0.3s ease-out; + display: flex; + align-items: center; + gap: 12px; + font-size: 0.9375rem; + line-height: 1.5; + word-wrap: break-word; + `; + + toast.innerHTML = ` + ${style.icon} + ${escapeHtml(message)} + + `; + + container.appendChild(toast); + + // 自动移除(成功消息显示5秒,错误消息显示7秒,其他显示4秒) + const duration = type === 'success' ? 5000 : type === 'error' ? 7000 : 4000; + setTimeout(() => { + if (toast.parentElement) { + toast.style.animation = 'slideOutRight 0.3s ease-out'; + setTimeout(() => { + if (toast.parentElement) { + toast.remove(); + } + }, 300); + } + }, duration); +} + +// 添加CSS动画(如果不存在) +if (!document.getElementById('toast-notification-styles')) { + const style = document.createElement('style'); + style.id = 'toast-notification-styles'; + style.textContent = ` + @keyframes slideInRight { + from { + transform: translateX(100%); + opacity: 0; + } + to { + transform: translateX(0); + opacity: 1; + } + } + @keyframes slideOutRight { + from { + transform: translateX(0); + opacity: 1; + } + to { + transform: translateX(100%); + opacity: 0; + } + } + `; + document.head.appendChild(style); +} + +// 点击模态框外部关闭 +window.addEventListener('click', function(event) { + const modal = document.getElementById('knowledge-item-modal'); + if (event.target === modal) { + closeKnowledgeItemModal(); + } +}); + +// 切换到设置页面(用于功能未启用时的提示) +function switchToSettings() { + if (typeof switchPage === 'function') { + switchPage('settings'); + // 等待设置页面加载后,切换到知识库配置部分 + setTimeout(() => { + if (typeof switchSettingsSection === 'function') { + // 查找知识库配置部分(通常在基本设置中) + const knowledgeSection = document.querySelector('[data-section="knowledge"]'); + if (knowledgeSection) { + switchSettingsSection('knowledge'); + } else { + // 如果没有独立的知识库部分,切换到基本设置 + switchSettingsSection('basic'); + // 滚动到知识库配置区域 + setTimeout(() => { + const knowledgeEnabledCheckbox = document.getElementById('knowledge-enabled'); + if (knowledgeEnabledCheckbox) { + knowledgeEnabledCheckbox.scrollIntoView({ behavior: 'smooth', block: 'center' }); + // 高亮显示 + knowledgeEnabledCheckbox.parentElement.style.transition = 'background-color 0.3s'; + knowledgeEnabledCheckbox.parentElement.style.backgroundColor = '#e3f2fd'; + setTimeout(() => { + knowledgeEnabledCheckbox.parentElement.style.backgroundColor = ''; + }, 2000); + } + }, 300); + } + } + }, 100); + } +} + +// 自定义下拉组件交互 +document.addEventListener('DOMContentLoaded', function() { + const wrapper = document.getElementById('knowledge-category-filter-wrapper'); + const trigger = document.getElementById('knowledge-category-filter-trigger'); + + if (wrapper && trigger) { + // 点击触发器打开/关闭下拉菜单 + trigger.addEventListener('click', function(e) { + e.stopPropagation(); + wrapper.classList.toggle('open'); + }); + + // 点击外部关闭下拉菜单 + document.addEventListener('click', function(e) { + if (!wrapper.contains(e.target)) { + wrapper.classList.remove('open'); + } + }); + + // 选择选项时更新选中状态 + const dropdown = document.getElementById('knowledge-category-filter-dropdown'); + if (dropdown) { + // 默认选中"全部"选项 + const defaultOption = dropdown.querySelector('.custom-select-option[data-value=""]'); + if (defaultOption) { + defaultOption.classList.add('selected'); + } + + dropdown.addEventListener('click', function(e) { + const option = e.target.closest('.custom-select-option'); + if (option) { + // 移除之前的选中状态 + dropdown.querySelectorAll('.custom-select-option').forEach(opt => { + opt.classList.remove('selected'); + }); + // 添加选中状态 + option.classList.add('selected'); + } + }); + } + } +}); + diff --git a/web/static/js/settings.js b/web/static/js/settings.js index 35011dc5..4a150ade 100644 --- a/web/static/js/settings.js +++ b/web/static/js/settings.js @@ -172,6 +172,43 @@ async function loadConfig(loadTools = true) { // 允许0.0值,只有undefined/null时才使用默认值 retrievalWeightInput.value = (hybridWeight !== undefined && hybridWeight !== null) ? hybridWeight : 0.7; } + + // 索引配置 + const indexing = knowledge.indexing || {}; + const chunkSizeInput = document.getElementById('knowledge-indexing-chunk-size'); + if (chunkSizeInput) { + chunkSizeInput.value = indexing.chunk_size || 512; + } + + const chunkOverlapInput = document.getElementById('knowledge-indexing-chunk-overlap'); + if (chunkOverlapInput) { + chunkOverlapInput.value = indexing.chunk_overlap ?? 50; + } + + const maxChunksPerItemInput = document.getElementById('knowledge-indexing-max-chunks-per-item'); + if (maxChunksPerItemInput) { + maxChunksPerItemInput.value = indexing.max_chunks_per_item ?? 0; + } + + const maxRpmInput = document.getElementById('knowledge-indexing-max-rpm'); + if (maxRpmInput) { + maxRpmInput.value = indexing.max_rpm ?? 0; + } + + const rateLimitDelayInput = document.getElementById('knowledge-indexing-rate-limit-delay-ms'); + if (rateLimitDelayInput) { + rateLimitDelayInput.value = indexing.rate_limit_delay_ms ?? 300; + } + + const maxRetriesInput = document.getElementById('knowledge-indexing-max-retries'); + if (maxRetriesInput) { + maxRetriesInput.value = indexing.max_retries ?? 3; + } + + const retryDelayInput = document.getElementById('knowledge-indexing-retry-delay-ms'); + if (retryDelayInput) { + retryDelayInput.value = indexing.retry_delay_ms ?? 1000; + } } // 填充机器人配置 @@ -728,6 +765,15 @@ async function applySettings() { const val = parseFloat(document.getElementById('knowledge-retrieval-hybrid-weight')?.value); return isNaN(val) ? 0.7 : val; // 允许0.0值,只有NaN时才使用默认值 })() + }, + indexing: { + chunk_size: parseInt(document.getElementById("knowledge-indexing-chunk-size")?.value) || 512, + chunk_overlap: parseInt(document.getElementById("knowledge-indexing-chunk-overlap")?.value) ?? 50, + max_chunks_per_item: parseInt(document.getElementById("knowledge-indexing-max-chunks-per-item")?.value) ?? 0, + max_rpm: parseInt(document.getElementById("knowledge-indexing-max-rpm")?.value) ?? 0, + rate_limit_delay_ms: parseInt(document.getElementById("knowledge-indexing-rate-limit-delay-ms")?.value) ?? 300, + max_retries: parseInt(document.getElementById("knowledge-indexing-max-retries")?.value) ?? 3, + retry_delay_ms: parseInt(document.getElementById("knowledge-indexing-retry-delay-ms")?.value) ?? 1000 } }; diff --git a/web/templates/index.html b/web/templates/index.html index f39c4bf3..a9ad2d29 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -1203,7 +1203,44 @@ 向量检索的权重(0-1),1.0表示纯向量检索,0.0表示纯关键词检索
- +
+
索引配置
+
+
+ + + 每个块的最大 token 数(默认 512),长文本会被分割成多个块 +
+
+ + + 块之间的重叠 token 数(默认 50),保持上下文连贯性 +
+
+ + + 单个知识项的最大块数量(0 表示不限制),防止单个文件消耗过多 API 配额 +
+
+ + + 每分钟最大请求数(默认 0 表示不限制),如 OpenAI 默认 200 RPM +
+
+ + + 请求间隔毫秒数(默认 300),用于避免 API 速率限制,设为 0 不限制 +
+
+ + + 最大重试次数(默认 3),遇到速率限制或服务器错误时自动重试 +
+
+ + + 重试间隔毫秒数(默认 1000),每次重试会递增延迟 +
diff --git a/web/templates/index.html.bak b/web/templates/index.html.bak new file mode 100644 index 00000000..f39c4bf3 --- /dev/null +++ b/web/templates/index.html.bak @@ -0,0 +1,2180 @@ + + + + + + CyberStrikeAI + + + + + + + + +
+
+
+ +
+
+ +
+ + +
+
+
+
+
+ +
+ + + + +
+ +
+
+ +
+ +
+
-
运行中任务
+
-
漏洞总数
+
-
工具调用次数
+
-
工具执行成功率
+
+ +
+
+
+

漏洞严重程度分布

+
+
+ + + + + +
+
+
严重0
+
高危0
+
中危0
+
低危0
+
信息0
+
+
+
+
+

运行概览

+
+
+ +
+
+ 批量任务队列 + - +
+
+ + + - + 待执行 + + + + - + 执行中 + + + + - + 已完成 + +
+
+
+
+
+
+
+
+
+
+
+ +
+
+ 工具调用 + - +
+
+ - + 次调用 + · + - + 个工具 +
+
+
+
+ +
+
+ 知识 + - +
+
+ - + 项知识 + · + - + 个分类 +
+
+
+
+ +
+
+ Skills + - +
+
+ - + 次调用 + · + - + 个 Skill +
+
+
+
+
+
+

快捷入口

+ +
+
+
+
+

工具执行次数

+
+
暂无数据
+
+
+
+
+
+
+
+ +
+

开始你的安全之旅

+

在对话中描述目标,AI 将协助执行扫描与漏洞分析

+
+
+ +
+
+
+
+ + +
+
+ + + + + + + +
+ + +
+
+
+
+ + + +
+
+
+
+ +
+
+
+ + + +
+
+
+
+ + +
+ +
+
+
+
+

执行统计

+
+
+
加载中...
+
+
+
+
+

最新执行记录

+
+ + +
+
+ +
+
加载中...
+
+
+
+
+
+ + +
+ +
+ +
+
+

MCP 工具配置

+ +
+
+
+ + + +
+
+
+
+
+ + +
+

外部 MCP 配置

+
+
+
+
+
+
+
+
+
+ + +
+ +
+
+
+
+ 总知识项 + - +
+
+ 分类数 + - +
+
+ 总内容 + - +
+
+ +
+ + +
+
+
+
加载中...
+
+
+
+
+ + +
+ +
+
+
+
+ 总检索次数 + - +
+
+ 成功检索 + - +
+
+ 成功率 + - +
+
+ 检索到知识项 + - +
+
+
+ + + +
+
+
+
加载中...
+
+
+
+ + +
+ +
+
+
+
+ + + 查询语法参考 FOFA 文档,支持 && / || / () 等。 +
+ + + + +
+
+
+ +
+ + +
+ + 解析后会弹窗展示 FOFA 语法(可编辑),确认无误后再填入查询框并执行查询。 +
+
+
+ + +
+
+ + +
+
+ +
+
+
+ + +
+ + + +
+
+
+
+ +
+
+
+
查询结果
+
-
+
+
+
已选择 0 条
+ + + + + +
+
+
+ + + + + + + +
暂无数据
+
+
+
+
+ + +
+ +
+ +
+
+
+
总漏洞数
+
-
+
+
+
严重
+
-
+
+
+
高危
+
-
+
+
+
中危
+
-
+
+
+
低危
+
-
+
+
+
信息
+
-
+
+
+
+ + +
+
+ + + + + + +
+
+ + +
+
加载中...
+
+ + +
+
+
+ + +
+ +
+ + +
+
+ + +
+ +
+
+ +
+
+
加载中...
+
+
+
+ + +
+ +
+
+
+
+

调用统计

+
+
+
加载中...
+
+
+
+
+

Skills调用统计

+
+ +
+
+
+
加载中...
+
+
+
+
+
+ + +
+ +
+
+ +
+
+
加载中...
+
+
+
+
+ + +
+ +
+ + + + +
+ +
+
+

基本设置

+
+ + +
+

OpenAI 配置

+
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+

FOFA 配置

+
+
+ + + 留空则使用默认地址。 +
+
+ + +
+
+ + + 仅保存在服务器配置中(`config.yaml`)。 +
+
+
+ + +
+

Agent 配置

+
+
+ + +
+
+
+ + +
+

知识库配置

+
+
+ +
+
+ + + 相对于配置文件所在目录的路径 +
+ +
+
嵌入模型配置
+
+
+ + +
+
+ + + 留空则使用OpenAI配置的base_url +
+
+ + + 留空则使用OpenAI配置的api_key +
+
+ + +
+ +
+
检索配置
+
+
+ + + 检索返回的Top-K结果数量 +
+
+ + + 相似度阈值(0-1),低于此值的结果将被过滤 +
+
+ + + 向量检索的权重(0-1),1.0表示纯向量检索,0.0表示纯关键词检索 +
+
+
+ +
+ +
+
+ + +
+
+

机器人设置

+

配置企业微信、钉钉、飞书等机器人,在手机端直接与 CyberStrikeAI 对话,无需在服务器上打开网页。

+
+ + +
+

企业微信

+
+
+ +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+

钉钉

+
+
+ +
+
+ + +
+
+ + + 需开启机器人能力并配置流式接入 +
+
+
+ + +
+

飞书 (Lark)

+
+
+ +
+
+ + +
+
+ + +
+
+ + +
+
+
+ +
+

机器人命令说明

+

在对话中可发送以下命令(支持中英文):

+
    +
  • 帮助 help — 显示本帮助 | Show this help
  • +
  • 列表 list — 列出所有对话标题与 ID | List conversations
  • +
  • 切换 <ID> switch <ID> — 指定对话继续 | Switch to conversation
  • +
  • 新对话 new — 开启新对话 | Start new conversation
  • +
  • 清空 clear — 清空当前上下文 | Clear context
  • +
  • 当前 current — 显示当前对话 ID 与标题 | Show current conversation
  • +
  • 停止 stop — 中断当前任务 | Stop running task
  • +
  • 角色 roles — 列出所有可用角色 | List roles
  • +
  • 角色 <名> role <name> — 切换当前角色 | Switch role
  • +
  • 删除 <ID> delete <ID> — 删除指定对话 | Delete conversation
  • +
  • 版本 version — 显示当前版本号 | Show version
  • +
+

除以上命令外,直接输入内容将发送给 AI 进行渗透测试/安全分析。Otherwise, send any text for AI penetration testing / security analysis.

+
+ +
+ +
+
+ + +
+
+

终端

+

在服务器上执行命令,便于运维与调试。命令在服务端执行,请勿执行敏感或破坏性操作。

+
+
+
+
终端 1
+ +
+
+
+
+
+
+
+
+ + +
+
+

安全设置

+
+ +
+

修改密码

+

修改登录密码后,需要使用新密码重新登录。

+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+
+
+
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +