feat: manage task process lifetimes and preserve turn history

This commit is contained in:
Ed1s0nZ
2026-09-16 17:52:58 +08:00
parent fd1c13a43d
commit f7882be546
54 changed files with 3650 additions and 330 deletions
+23 -8
View File
@@ -221,13 +221,19 @@ func (h *AgentHandler) CancelRunningTaskForConversation(conversationID string) {
if h == nil || conversationID == "" || h.tasks == nil {
return
}
h.cancelRunningMCPToolsForConversation(conversationID)
h.tasks.AbortActiveEinoExecute(conversationID, "")
if ok, err := h.tasks.CancelTask(conversationID, ErrTaskCancelled); ok {
h.logger.Info("已取消会话运行中任务", zap.String("conversationId", conversationID))
} else if err != nil {
h.logger.Warn("取消会话运行中任务失败", zap.String("conversationId", conversationID), zap.Error(err))
ok, err := h.tasks.CancelTask(conversationID, ErrTaskCancelled)
if !ok {
h.cancelRunningMCPToolsForConversation(conversationID)
h.tasks.AbortActiveEinoExecute(conversationID, "")
}
if h.logger != nil {
if err != nil {
h.logger.Warn("取消会话运行中任务失败", zap.String("conversationId", conversationID), zap.Error(err))
} else if ok {
h.logger.Info("已取消会话运行中任务", zap.String("conversationId", conversationID))
}
}
}
// ConversationTaskRuntimeState exposes the authoritative live state and start
@@ -893,15 +899,24 @@ func (h *AgentHandler) ProcessMessageForRobot(ctx context.Context, platform stri
taskCtx, cancelWithCause := context.WithCancelCause(ctx)
defer cancelWithCause(nil)
taskStatus := "completed"
var taskRunID string
defer func() {
h.tasks.FinishTask(conversationID, taskStatus)
if taskRunID == "" {
return
}
if cleanupErr := h.tasks.FinishTaskRun(conversationID, taskRunID, taskStatus); cleanupErr != nil {
err = errors.Join(err, cleanupErr)
}
}()
if _, err := h.tasks.StartTask(conversationID, message, cancelWithCause); err != nil {
if startedTask, err := h.tasks.StartTask(conversationID, message, cancelWithCause); err != nil {
if errors.Is(err, ErrTaskAlreadyRunning) {
return "", conversationID, fmt.Errorf("当前会话已有任务正在执行中,请稍后再试")
}
return "", conversationID, fmt.Errorf("无法启动任务: %w", err)
} else {
taskRunID = startedTask.RunID
}
taskCtx = h.tasks.BindProcessScope(taskCtx, conversationID, taskRunID)
progressCallback := h.createProgressCallback(taskCtx, cancelWithCause, conversationID, assistantMessageID, nil)
robotMode := config.NormalizeAgentMode(agentMode)
+11 -8
View File
@@ -171,19 +171,14 @@ func (h *AgentHandler) executeOneBatchSubTask(queueID string, queue *BatchTaskQu
taskCtx, timeoutCancel := context.WithTimeout(baseCtx, 6*time.Hour)
registered := false
var taskRunID string
finishStatus := "completed"
defer func() {
h.batchTaskManager.SetTaskCancel(queueID, task.ID, nil)
timeoutCancel()
if registered {
if h.taskEventBus != nil {
ev := StreamEvent{Type: "done", Message: "", Data: map[string]interface{}{"conversationId": conversationID}}
if b, err := json.Marshal(ev); err == nil {
h.taskEventBus.Publish(conversationID, append(append([]byte("data: "), b...), '\n', '\n'))
}
}
h.tasks.FinishTask(conversationID, finishStatus)
h.tasks.FinishTaskRun(conversationID, taskRunID, finishStatus)
}
cancelWithCause(nil)
}()
@@ -204,7 +199,7 @@ func (h *AgentHandler) executeOneBatchSubTask(queueID string, queue *BatchTaskQu
h.taskEventBus.Publish(conversationID, line)
}
if _, err := h.tasks.StartTask(conversationID, task.Message, cancelWithCause); err != nil {
if startedTask, err := h.tasks.StartTask(conversationID, task.Message, cancelWithCause); err != nil {
h.logger.Warn("批量队列子任务注册会话运行状态失败",
zap.String("queueId", queueID),
zap.String("taskId", task.ID),
@@ -216,7 +211,11 @@ func (h *AgentHandler) executeOneBatchSubTask(queueID string, queue *BatchTaskQu
}
h.batchTaskManager.UpdateTaskStatus(queueID, task.ID, BatchTaskStatusFailed, "", failMsg)
return
} else {
taskRunID = startedTask.RunID
}
baseCtx = h.tasks.BindProcessScope(baseCtx, conversationID, taskRunID)
taskCtx = h.tasks.BindProcessScope(taskCtx, conversationID, taskRunID)
registered = true
h.batchTaskManager.SetTaskCancel(queueID, task.ID, timeoutCancel)
@@ -342,6 +341,10 @@ func (h *AgentHandler) executeOneBatchSubTask(queueID string, queue *BatchTaskQu
}
}
if cleanupErr := h.tasks.FinishTaskRun(conversationID, taskRunID, finishStatus); cleanupErr != nil {
h.batchTaskManager.UpdateTaskStatusWithConversationID(queueID, task.ID, BatchTaskStatusFailed, resText, cleanupErr.Error(), conversationID)
return
}
if !decision.Finalizable {
h.batchTaskManager.UpdateTaskStatusWithConversationID(queueID, task.ID, BatchTaskStatusFailed, resText, finalizationCheckMessage(decision), conversationID)
return
+30 -6
View File
@@ -130,9 +130,10 @@ func (h *AgentHandler) EinoSingleAgentLoopStream(c *gin.Context) {
// 仅在成功 StartTask 后再 FinishTask。若 StartTask 因 ErrTaskAlreadyRunning 失败仍 defer FinishTask
// 会误删其他连接上正在运行的同会话任务,导致「第一次拦截、第二次却放行」。
taskOwned := false
var taskRunID string
defer func() {
if taskOwned {
h.tasks.FinishTask(conversationID, taskStatus)
h.tasks.FinishTaskRun(conversationID, taskRunID, taskStatus)
}
}()
@@ -165,7 +166,7 @@ func (h *AgentHandler) EinoSingleAgentLoopStream(c *gin.Context) {
baseCtx, cancelWithCause = context.WithCancelCause(detachedAgentContext(c.Request.Context()))
taskCtx, timeoutCancel := context.WithTimeout(baseCtx, 600*time.Minute)
if _, err := h.tasks.StartTask(conversationID, req.Message, cancelWithCause); err != nil {
if startedTask, err := h.tasks.StartTask(conversationID, req.Message, cancelWithCause); err != nil {
var errorMsg string
if errors.Is(err, ErrTaskAlreadyRunning) {
errorMsg = "⚠️ 当前会话已有任务正在执行中,请等待当前任务完成或点击「停止任务」后再尝试。"
@@ -183,8 +184,13 @@ func (h *AgentHandler) EinoSingleAgentLoopStream(c *gin.Context) {
sendEvent("done", "", map[string]interface{}{"conversationId": conversationID})
timeoutCancel()
return
} else {
taskRunID = startedTask.RunID
}
baseCtx = h.tasks.BindProcessScope(baseCtx, conversationID, taskRunID)
taskCtx = h.tasks.BindProcessScope(taskCtx, conversationID, taskRunID)
taskOwned = true
sendEvent = h.taskFinishingEventSender(sendEvent, conversationID, taskRunID, func() string { return taskStatus })
var cumulativeMCPExecutionIDs []string
// 同一请求内分段续跑时,主代理 iteration 事件按偏移累计,避免 UI 出现「第3轮 → 第1轮」回跳。
@@ -454,18 +460,31 @@ func (h *AgentHandler) EinoSingleAgentLoop(c *gin.Context) {
defer cancelWithCause(nil)
taskCtx, timeoutCancel := context.WithTimeout(baseCtx, 600*time.Minute)
defer timeoutCancel()
jsonTask, startErr := h.tasks.StartTask(prep.ConversationID, req.Message, cancelWithCause)
if startErr != nil {
c.JSON(http.StatusConflict, gin.H{"error": startErr.Error()})
return
}
taskCtx = h.tasks.BindProcessScope(taskCtx, prep.ConversationID, jsonTask.RunID)
taskCtx = mcp.WithMCPConversationID(taskCtx, prep.ConversationID)
taskCtx = mcp.WithToolRunRegistry(taskCtx, h.tasks)
taskCtx = mcp.WithEinoExecuteRunRegistry(taskCtx, h.tasks)
jsonTaskStatus := "failed"
defer func() { _ = h.tasks.FinishTaskRun(prep.ConversationID, jsonTask.RunID, jsonTaskStatus) }()
respond := h.taskFinishingJSONResponder(c, prep.ConversationID, jsonTask.RunID, func() string { return jsonTaskStatus })
progressCallback := h.createProgressCallback(taskCtx, cancelWithCause, prep.ConversationID, prep.AssistantMessageID, progressCallbackRaw)
taskCtx = multiagent.WithHITLToolInterceptor(taskCtx, func(ctx context.Context, toolName, arguments string) (string, error) {
return h.interceptHITLForEinoTool(ctx, cancelWithCause, prep.ConversationID, prep.AssistantMessageID, nil, toolName, arguments)
})
if h.config == nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "服务器配置未加载"})
respond(http.StatusInternalServerError, gin.H{"error": "服务器配置未加载"})
return
}
runCfg, _, err := h.configForAIChannel(req.AIChannelID)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
respond(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
@@ -498,7 +517,7 @@ func (h *AgentHandler) EinoSingleAgentLoop(c *gin.Context) {
if shouldPersistEinoAgentTraceAfterRunError(baseCtx) {
h.persistEinoAgentTraceForResume(prep.ConversationID, result)
}
c.JSON(http.StatusInternalServerError, gin.H{"error": runErr.Error()})
respond(http.StatusInternalServerError, gin.H{"error": runErr.Error()})
return
}
mw := &h.config.MultiAgent.EinoMiddleware
@@ -525,7 +544,12 @@ func (h *AgentHandler) EinoSingleAgentLoop(c *gin.Context) {
if !decision.Finalizable {
responseText = finalizationBlockedMessage(decision)
}
c.JSON(http.StatusOK, gin.H{
jsonTaskStatus = decision.Status
if jsonTaskStatus == "" {
jsonTaskStatus = "completed"
}
respond(http.StatusOK, gin.H{
"response": responseText,
"conversationId": prep.ConversationID,
"mcpExecutionIds": result.MCPExecutionIDs,
+29 -5
View File
@@ -147,9 +147,10 @@ func (h *AgentHandler) MultiAgentLoopStream(c *gin.Context) {
taskStatus := "completed"
// 仅在成功 StartTask 后再 FinishTask;避免「任务已存在」分支 return 时误删正在运行的同会话任务。
taskOwned := false
var taskRunID string
defer func() {
if taskOwned {
h.tasks.FinishTask(conversationID, taskStatus)
h.tasks.FinishTaskRun(conversationID, taskRunID, taskStatus)
}
}()
@@ -172,7 +173,7 @@ func (h *AgentHandler) MultiAgentLoopStream(c *gin.Context) {
baseCtx, cancelWithCause = context.WithCancelCause(detachedAgentContext(c.Request.Context()))
taskCtx, timeoutCancel := context.WithTimeout(baseCtx, 600*time.Minute)
if _, err := h.tasks.StartTask(conversationID, req.Message, cancelWithCause); err != nil {
if startedTask, err := h.tasks.StartTask(conversationID, req.Message, cancelWithCause); err != nil {
var errorMsg string
if errors.Is(err, ErrTaskAlreadyRunning) {
errorMsg = "⚠️ 当前会话已有任务正在执行中,请等待当前任务完成或点击「停止任务」后再尝试。"
@@ -190,8 +191,13 @@ func (h *AgentHandler) MultiAgentLoopStream(c *gin.Context) {
sendEvent("done", "", map[string]interface{}{"conversationId": conversationID})
timeoutCancel()
return
} else {
taskRunID = startedTask.RunID
}
baseCtx = h.tasks.BindProcessScope(baseCtx, conversationID, taskRunID)
taskCtx = h.tasks.BindProcessScope(taskCtx, conversationID, taskRunID)
taskOwned = true
sendEvent = h.taskFinishingEventSender(sendEvent, conversationID, taskRunID, func() string { return taskStatus })
// 同一 HTTP 流内多段 Run(如中断并继续)合并 MCP execution id,供最终 response / 库表与工具芯片展示完整列表
var cumulativeMCPExecutionIDs []string
@@ -468,13 +474,26 @@ func (h *AgentHandler) MultiAgentLoop(c *gin.Context) {
defer cancelWithCause(nil)
taskCtx, timeoutCancel := context.WithTimeout(baseCtx, 600*time.Minute)
defer timeoutCancel()
jsonTask, startErr := h.tasks.StartTask(prep.ConversationID, req.Message, cancelWithCause)
if startErr != nil {
c.JSON(http.StatusConflict, gin.H{"error": startErr.Error()})
return
}
taskCtx = h.tasks.BindProcessScope(taskCtx, prep.ConversationID, jsonTask.RunID)
taskCtx = mcp.WithMCPConversationID(taskCtx, prep.ConversationID)
taskCtx = mcp.WithToolRunRegistry(taskCtx, h.tasks)
taskCtx = mcp.WithEinoExecuteRunRegistry(taskCtx, h.tasks)
jsonTaskStatus := "failed"
defer func() { _ = h.tasks.FinishTaskRun(prep.ConversationID, jsonTask.RunID, jsonTaskStatus) }()
respond := h.taskFinishingJSONResponder(c, prep.ConversationID, jsonTask.RunID, func() string { return jsonTaskStatus })
progressCallback := h.createProgressCallback(taskCtx, cancelWithCause, prep.ConversationID, prep.AssistantMessageID, nil)
taskCtx = multiagent.WithHITLToolInterceptor(taskCtx, func(ctx context.Context, toolName, arguments string) (string, error) {
return h.interceptHITLForEinoTool(ctx, cancelWithCause, prep.ConversationID, prep.AssistantMessageID, nil, toolName, arguments)
})
runCfg, _, err := h.configForAIChannel(req.AIChannelID)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
respond(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
@@ -522,7 +541,7 @@ func (h *AgentHandler) MultiAgentLoop(c *gin.Context) {
}
errData := multiagent.EinoClientRunErrorFields(runErr)
errData["error"] = errMsg
c.JSON(http.StatusInternalServerError, errData)
respond(http.StatusInternalServerError, errData)
return
}
mw := &h.config.MultiAgent.EinoMiddleware
@@ -552,7 +571,12 @@ func (h *AgentHandler) MultiAgentLoop(c *gin.Context) {
if !decision.Finalizable {
responseText = finalizationBlockedMessage(decision)
}
c.JSON(http.StatusOK, ChatResponse{
jsonTaskStatus = decision.Status
if jsonTaskStatus == "" {
jsonTaskStatus = "completed"
}
respond(http.StatusOK, ChatResponse{
Response: responseText,
MCPExecutionIDs: result.MCPExecutionIDs,
ConversationID: prep.ConversationID,
+62
View File
@@ -0,0 +1,62 @@
package handler
import (
"cyberstrike-ai/internal/runlease"
"errors"
"github.com/gin-gonic/gin"
"go.uber.org/zap"
"net/http"
)
// ShutdownTasks stops local work before shared MCP clients and databases close.
func (h *AgentHandler) ShutdownTasks() {
if h != nil && h.tasks != nil {
h.tasks.Shutdown()
if h.logger != nil {
for _, task := range h.tasks.GetActiveTasks() {
h.logger.Warn("服务关闭时任务仍有未完成的清理", zap.String("runId", task.RunID), zap.String("cleanupError", task.CleanupError))
}
}
}
}
// taskFinishingEventSender makes the visible done event follow local cleanup.
func (h *AgentHandler) taskFinishingEventSender(send func(string, string, interface{}), conversationID, runID string, status func() string) func(string, string, interface{}) {
return func(eventType, message string, data interface{}) {
if eventType == "done" {
if err := h.tasks.FinishTaskRun(conversationID, runID, status()); err != nil {
if h.logger != nil {
h.logger.Warn(taskCleanupMessage(err), zap.String("runId", runID), zap.Error(err))
}
send("error", taskCleanupMessage(err)+": "+err.Error(), map[string]interface{}{"errorType": taskCleanupStatus(err)})
data = map[string]interface{}{"conversationId": conversationID, "runId": runID, "status": taskCleanupStatus(err), "cleanupError": err.Error()}
}
}
send(eventType, message, data)
}
}
// taskFinishingJSONResponder applies the same ordering to successful and failed
// non-streaming requests. No response claims completion before cleanup returns.
func (h *AgentHandler) taskFinishingJSONResponder(c *gin.Context, conversationID, runID string, status func() string) func(int, interface{}) {
return func(code int, payload interface{}) {
if err := h.tasks.FinishTaskRun(conversationID, runID, status()); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error(), "status": taskCleanupStatus(err), "conversationId": conversationID})
return
}
c.JSON(code, payload)
}
}
func taskCleanupStatus(err error) string {
if errors.Is(err, runlease.ErrUnconfirmed) {
return "cleanup_unconfirmed"
}
return "cleanup_failed"
}
func taskCleanupMessage(err error) string {
if errors.Is(err, runlease.ErrUnconfirmed) {
return "本地执行已结束,远端 MCP 停止状态待确认"
}
return "任务资源清理失败,将自动重试"
}
+218 -50
View File
@@ -2,13 +2,17 @@ package handler
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"sync"
"time"
"cyberstrike-ai/internal/multiagent"
"cyberstrike-ai/internal/runlease"
"cyberstrike-ai/internal/security"
)
// ErrTaskCancelled 用户取消任务的错误
@@ -26,12 +30,20 @@ func shouldPersistEinoAgentTraceAfterRunError(baseCtx context.Context) bool {
// AgentTask 描述正在运行的Agent任务
type AgentTask struct {
ConversationID string `json:"conversationId"`
Title string `json:"title,omitempty"`
Message string `json:"message,omitempty"`
StartedAt time.Time `json:"startedAt"`
Status string `json:"status"`
CancellingAt time.Time `json:"-"` // 进入 cancelling 状态的时间,用于清理长时间卡住的任务
RunID string `json:"runId"`
CleanupError string `json:"cleanupError,omitempty"`
processes *security.ProcessScope
workers *runlease.Scope
IsolationBackend string `json:"isolationBackend,omitempty"`
finishing chan struct{}
stopping chan struct{}
finalStatus string
ConversationID string `json:"conversationId"`
Title string `json:"title,omitempty"`
Message string `json:"message,omitempty"`
StartedAt time.Time `json:"startedAt"`
Status string `json:"status"`
CancellingAt time.Time `json:"-"` // 进入 cancelling 状态的时间,用于清理长时间卡住的任务
// ActiveMCPExecutionID 当前正在执行的 MCP 工具 executionId(仅内存,供「中断并继续」= 仅掐当前工具)
ActiveMCPExecutionID string `json:"-"`
@@ -297,12 +309,15 @@ func (m *AgentTaskManager) ActiveMCPExecutionID(conversationID string) string {
// CompletedTask 已完成的任务(用于历史记录)
type CompletedTask struct {
ConversationID string `json:"conversationId"`
Title string `json:"title,omitempty"`
Message string `json:"message,omitempty"`
StartedAt time.Time `json:"startedAt"`
CompletedAt time.Time `json:"completedAt"`
Status string `json:"status"`
CleanupError string `json:"cleanupError,omitempty"`
IsolationBackend string `json:"isolationBackend,omitempty"`
RunID string `json:"runId"`
ConversationID string `json:"conversationId"`
Title string `json:"title,omitempty"`
Message string `json:"message,omitempty"`
StartedAt time.Time `json:"startedAt"`
CompletedAt time.Time `json:"completedAt"`
Status string `json:"status"`
}
// AgentTaskManager 管理正在运行的Agent任务
@@ -315,6 +330,9 @@ type AgentTaskManager struct {
eventBus *TaskEventBus // 可选:任务结束时关闭镜像 SSE 订阅
// toolCanceler 在用户整轮停止任务或会话结束时终止该会话仍在运行的 MCP 工具(非「中断并继续」)。
toolCanceler func(conversationID string)
shuttingDown bool
shutdown chan struct{}
shutdownOnce sync.Once
}
const (
@@ -330,6 +348,7 @@ const (
func NewAgentTaskManager() *AgentTaskManager {
m := &AgentTaskManager{
tasks: make(map[string]*AgentTask),
shutdown: make(chan struct{}),
completedTasks: make([]*CompletedTask, 0),
maxHistorySize: 50, // 最多保留50条历史记录
historyRetention: 24 * time.Hour, // 保留24小时
@@ -368,6 +387,7 @@ func (m *AgentTaskManager) GetTaskSnapshot(conversationID string) *AgentTask {
return nil
}
snapshot := *task
snapshot.IsolationBackend = task.processes.IsolationBackend()
return &snapshot
}
@@ -375,16 +395,26 @@ func (m *AgentTaskManager) GetTaskSnapshot(conversationID string) *AgentTask {
func (m *AgentTaskManager) runStuckCancellingCleanup() {
ticker := time.NewTicker(cleanupInterval)
defer ticker.Stop()
for range ticker.C {
m.cleanupStuckCancelling()
for {
select {
case <-m.shutdown:
return
case <-ticker.C:
m.cleanupStuckCancelling()
}
}
}
func (m *AgentTaskManager) cleanupStuckCancelling() {
m.mu.Lock()
var toFinish []string
type pendingFinish struct{ id, runID, status string }
var toFinish []pendingFinish
now := time.Now()
for id, task := range m.tasks {
if task.Status == "cleanup_failed" {
toFinish = append(toFinish, pendingFinish{id, task.RunID, task.finalStatus})
continue
}
if task.Status != "cancelling" {
continue
}
@@ -400,11 +430,11 @@ func (m *AgentTaskManager) cleanupStuckCancelling() {
continue
}
}
toFinish = append(toFinish, id)
toFinish = append(toFinish, pendingFinish{id, task.RunID, "cancelled"})
}
m.mu.Unlock()
for _, id := range toFinish {
m.FinishTask(id, "cancelled")
for _, pending := range toFinish {
_ = m.FinishTaskRun(pending.id, pending.runID, pending.status)
}
}
@@ -413,11 +443,16 @@ func (m *AgentTaskManager) StartTask(conversationID, message string, cancel cont
m.mu.Lock()
defer m.mu.Unlock()
if m.shuttingDown {
return nil, errors.New("task manager is shutting down")
}
if _, exists := m.tasks[conversationID]; exists {
return nil, ErrTaskAlreadyRunning
}
scope := security.NewProcessScope()
task := &AgentTask{
RunID: scope.ID, processes: scope, workers: runlease.New(),
ConversationID: conversationID,
Message: message,
StartedAt: time.Now(),
@@ -444,7 +479,7 @@ func (m *AgentTaskManager) CancelTask(conversationID string, cause error) (bool,
}
// 如果已经处于取消流程,视为成功(幂等),避免前端重复点击报「未找到任务」
if task.Status == "cancelling" {
if task.Status == "cancelling" || task.finishing != nil {
m.mu.Unlock()
return true, nil
}
@@ -466,6 +501,13 @@ func (m *AgentTaskManager) CancelTask(conversationID string, cause error) (bool,
interruptPush := task.agentTurnLoopInterrupt
interruptNote := task.InterruptContinueNote
runtimeCancel := task.agentRuntimeCancel
activeExecuteCancel := task.activeEinoExecuteCancel
if !errors.Is(cause, multiagent.ErrInterruptContinue) {
task.processes.Seal()
task.workers.Seal()
task.stopping = make(chan struct{})
defer close(task.stopping)
}
var toolCanceler func(string)
if errors.Is(cause, ErrTaskCancelled) {
toolCanceler = m.toolCanceler
@@ -494,6 +536,13 @@ func (m *AgentTaskManager) CancelTask(conversationID string, cause error) (bool,
if toolCanceler != nil {
toolCanceler(conversationID)
}
if !errors.Is(cause, multiagent.ErrInterruptContinue) {
task.workers.Cancel()
if activeExecuteCancel != nil {
activeExecuteCancel()
}
return true, task.processes.Close()
}
return true, nil
}
@@ -507,54 +556,172 @@ func (m *AgentTaskManager) UpdateTaskStatus(conversationID string, status string
return
}
if status != "" {
task.Status = status
if task.finishing != nil || task.Status == "cleanup_failed" {
return
}
switch status {
case "completed", "cancelled", "failed", "timeout":
task.finalStatus = status
task.Status = "cleaning"
task.processes.Seal()
task.workers.Seal()
default:
if status != "" {
task.Status = status
}
}
}
// FinishTask 完成任务并从管理器中移除
// BindProcessScope snapshots ownership once at task start. Continuations must
// derive from this context, never resolve ownership again using conversation ID.
func (m *AgentTaskManager) BindProcessScope(ctx context.Context, conversationID, runID string) context.Context {
m.mu.RLock()
defer m.mu.RUnlock()
if task := m.tasks[conversationID]; task != nil && task.RunID == runID {
return runlease.WithScope(security.WithProcessScope(ctx, task.processes), task.workers)
}
// Fail closed if a task disappeared before its execution context was bound.
scope := security.NewProcessScope()
scope.Seal()
workers := runlease.New()
workers.Seal()
return runlease.WithScope(security.WithProcessScope(ctx, scope), workers)
}
// FinishTask is retained for callers that operate on the current task. Owners
// use FinishTaskRun, so a delayed defer cannot finish a newer conversation run.
func (m *AgentTaskManager) FinishTask(conversationID string, finalStatus string) {
m.mu.RLock()
task := m.tasks[conversationID]
m.mu.RUnlock()
if task != nil {
_ = m.FinishTaskRun(conversationID, task.RunID, finalStatus)
}
}
func (m *AgentTaskManager) FinishTaskRun(conversationID, runID, finalStatus string) error {
m.mu.Lock()
task, exists := m.tasks[conversationID]
if !exists {
task := m.tasks[conversationID]
if task == nil || task.RunID != runID {
m.mu.Unlock()
return
return nil
}
if finalStatus != "" {
task.Status = finalStatus
if task.stopping != nil {
select {
case <-task.stopping:
default:
stopping := task.stopping
m.mu.Unlock()
<-stopping
return m.FinishTaskRun(conversationID, runID, finalStatus)
}
}
if task.finishing != nil {
done := task.finishing
m.mu.Unlock()
<-done
m.mu.RLock()
cleanupError := task.CleanupError
m.mu.RUnlock()
if cleanupError != "" {
return errors.New(cleanupError)
}
return nil
}
done := make(chan struct{})
task.finishing = done
task.finalStatus = finalStatus
task.Status = "cleaning"
task.processes.Seal()
task.workers.Seal()
toolCanceler := m.toolCanceler
activeEinoExecuteCancel := task.activeEinoExecuteCancel
// 保存到历史记录
completedTask := &CompletedTask{
ConversationID: task.ConversationID,
Message: task.Message,
StartedAt: task.StartedAt,
CompletedAt: time.Now(),
Status: finalStatus,
}
// 添加到历史记录
m.completedTasks = append(m.completedTasks, completedTask)
// 清理过期和过多的历史记录
m.cleanupHistory()
// 从运行任务中移除
delete(m.tasks, conversationID)
activeCancel := task.activeEinoExecuteCancel
cancel := task.cancel
bus := m.eventBus
m.mu.Unlock()
// Keep the conversation occupied throughout cleanup, including callbacks.
if cancel != nil {
cancel(nil)
}
if toolCanceler != nil {
toolCanceler(conversationID)
}
if activeEinoExecuteCancel != nil {
activeEinoExecuteCancel()
if activeCancel != nil {
activeCancel()
}
if bus != nil {
task.workers.Cancel()
processErr := task.processes.Close()
waitCtx, waitCancel := context.WithTimeout(context.Background(), 3*time.Second)
workerErr := task.workers.Wait(waitCtx)
waitCancel()
cleanupErr := errors.Join(processErr, workerErr)
if processErr != nil && errors.Is(workerErr, runlease.ErrUnconfirmed) {
// A simultaneous local failure must not be labelled as local completion.
cleanupErr = fmt.Errorf("local cleanup: %w; remote state: %v", processErr, workerErr)
}
// The local worker has returned, but remote notification cancellation is
// not an acknowledgement. Preserve an actionable history/tool status.
unconfirmed := processErr == nil && errors.Is(workerErr, runlease.ErrUnconfirmed)
if unconfirmed {
finalStatus = "cleanup_unconfirmed"
}
cleanupMessage := ""
if cleanupErr != nil {
cleanupMessage = cleanupErr.Error()
}
if (cleanupErr == nil || unconfirmed) && bus != nil {
// Subscribers must receive completion only after local processes are reaped.
payload, _ := json.Marshal(StreamEvent{Type: "done", Data: map[string]interface{}{"conversationId": conversationID, "runId": runID, "status": finalStatus, "cleanupError": cleanupMessage}})
bus.Publish(conversationID, append(append([]byte("data: "), payload...), '\n', '\n'))
bus.CloseConversation(conversationID)
}
m.mu.Lock()
defer m.mu.Unlock()
defer close(done)
if cleanupErr != nil && !unconfirmed {
task.Status = "cleanup_failed"
task.CleanupError = cleanupErr.Error()
task.finishing = nil
return cleanupErr
}
task.CleanupError = ""
if unconfirmed {
task.CleanupError = cleanupErr.Error()
}
task.Status = finalStatus
m.completedTasks = append(m.completedTasks, &CompletedTask{
RunID: task.RunID, CleanupError: task.CleanupError, IsolationBackend: task.processes.IsolationBackend(), ConversationID: task.ConversationID, Message: task.Message,
StartedAt: task.StartedAt, CompletedAt: time.Now(), Status: finalStatus,
})
m.cleanupHistory()
delete(m.tasks, conversationID)
return cleanupErr
}
// Shutdown rejects new tasks before cancelling and reaping existing task jobs.
func (m *AgentTaskManager) Shutdown() {
m.mu.Lock()
m.shuttingDown = true
m.shutdownOnce.Do(func() { close(m.shutdown) })
tasks := make([]*AgentTask, 0, len(m.tasks))
for _, task := range m.tasks {
tasks = append(tasks, task)
task.processes.Seal()
task.workers.Seal()
}
m.mu.Unlock()
var wg sync.WaitGroup
for _, task := range tasks {
wg.Add(1)
go func(task *AgentTask) {
defer wg.Done()
_, _ = m.CancelTask(task.ConversationID, ErrTaskCancelled)
_ = m.FinishTaskRun(task.ConversationID, task.RunID, "cancelled")
}(task)
}
wg.Wait()
}
// cleanupHistory 清理过期的历史记录
@@ -589,6 +756,7 @@ func (m *AgentTaskManager) GetActiveTasks() []*AgentTask {
result := make([]*AgentTask, 0, len(m.tasks))
for _, task := range m.tasks {
result = append(result, &AgentTask{
RunID: task.RunID, CleanupError: task.CleanupError, IsolationBackend: task.processes.IsolationBackend(),
ConversationID: task.ConversationID,
Message: task.Message,
StartedAt: task.StartedAt,
@@ -0,0 +1,225 @@
package handler
import (
"context"
"errors"
"os/exec"
"runtime"
"sync"
"testing"
"time"
"cyberstrike-ai/internal/mcp"
"cyberstrike-ai/internal/runlease"
"cyberstrike-ai/internal/security"
)
func TestTaskCleanupWaitsBeforeReleasingConversation(t *testing.T) {
manager := NewAgentTaskManager()
task, _ := manager.StartTask("conv", "old", func(error) {})
entered, release := make(chan struct{}), make(chan struct{})
manager.SetToolCanceler(func(string) { close(entered); <-release })
done := make(chan error, 1)
go func() { done <- manager.FinishTaskRun("conv", task.RunID, "completed") }()
<-entered
if status := manager.GetTaskSnapshot("conv").Status; status != "cleaning" {
t.Errorf("status = %s", status)
}
if _, err := manager.StartTask("conv", "new", nil); !errors.Is(err, ErrTaskAlreadyRunning) {
t.Errorf("new task admitted during cleanup: %v", err)
}
ctx := manager.BindProcessScope(context.Background(), "conv", task.RunID)
if _, err := security.StartShellSessionContext(ctx, exec.Command("unused-command")); !errors.Is(err, security.ErrProcessScopeClosed) {
t.Errorf("late process admitted: %v", err)
}
close(release)
if err := <-done; err != nil {
t.Fatal(err)
}
manager.SetToolCanceler(nil)
next, err := manager.StartTask("conv", "new", nil)
if err != nil {
t.Fatal(err)
}
defer manager.FinishTask("conv", "completed")
if next.RunID == task.RunID {
t.Fatal("run identity reused")
}
_ = manager.FinishTaskRun("conv", task.RunID, "cancelled")
if manager.GetTaskSnapshot("conv").RunID != next.RunID {
t.Fatal("old defer removed new task")
}
// A delayed worker keeps its original closed scope, even after a new run starts.
if _, err := security.StartManagedBackground(ctx, "sh", "sleep 300", ""); !errors.Is(err, security.ErrProcessScopeClosed) {
t.Fatalf("old context borrowed new task: %v", err)
}
}
func TestTaskFinishAndShutdownReapBackgroundProcesses(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("Unix shell")
}
for _, shutdown := range []bool{false, true} {
name := "finish"
if shutdown {
name = "shutdown"
}
t.Run(name, func(t *testing.T) {
manager := NewAgentTaskManager()
task, _ := manager.StartTask("conv", "job", nil)
ctx := manager.BindProcessScope(context.Background(), "conv", task.RunID)
session, err := security.StartManagedBackground(ctx, "sh", "sleep 300", "")
if err != nil {
t.Fatal(err)
}
if shutdown {
manager.Shutdown()
} else if err := manager.FinishTaskRun("conv", task.RunID, "completed"); err != nil {
t.Fatal(err)
}
done := make(chan struct{})
go func() { _ = session.Wait(); close(done) }()
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("task ended before process exited")
}
if manager.GetTaskSnapshot("conv") != nil {
t.Fatal("finished task still active")
}
if shutdown {
if _, err := manager.StartTask("new", "job", nil); err == nil {
t.Fatal("shutdown admitted a new task")
}
}
})
}
}
func TestTaskDonePublishedAfterCleanup(t *testing.T) {
manager := NewAgentTaskManager()
bus := NewTaskEventBus()
manager.SetTaskEventBus(bus)
task, _ := manager.StartTask("conv", "job", nil)
_, events := bus.Subscribe("conv")
if err := manager.FinishTaskRun("conv", task.RunID, "completed"); err != nil {
t.Fatal(err)
}
if event, ok := <-events; !ok || len(event) == 0 {
t.Fatal("subscriber closed without done event")
}
if _, ok := <-events; ok {
t.Fatal("subscriber not closed after completion")
}
}
func TestTaskCleanupFailureRetainsOwnershipAndRetries(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("Unix shell")
}
manager := NewAgentTaskManager()
task, _ := manager.StartTask("conv", "job", nil)
ctx := manager.BindProcessScope(context.Background(), "conv", task.RunID)
// Simulate an executor which has not yet reaped its direct child.
session, err := security.StartShellSessionContext(ctx, exec.Command("sh", "-c", "sleep 300"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { session.Terminate(); _ = session.Wait(); manager.Shutdown() })
if err := manager.FinishTaskRun("conv", task.RunID, "completed"); err == nil {
t.Fatal("unreaped process reported as cleaned up")
}
snapshot := manager.GetTaskSnapshot("conv")
if snapshot == nil || snapshot.Status != "cleanup_failed" || snapshot.CleanupError == "" {
t.Fatalf("missing actionable cleanup state: %+v", snapshot)
}
if len(manager.GetCompletedTasks()) != 0 {
t.Fatal("cleanup failure recorded as completed")
}
if _, err := manager.StartTask("conv", "new", nil); !errors.Is(err, ErrTaskAlreadyRunning) {
t.Fatal("cleanup failure released conversation")
}
_ = session.Wait()
manager.cleanupStuckCancelling()
if manager.GetTaskSnapshot("conv") != nil {
t.Fatal("cleanup retry did not finish reaped task")
}
}
func TestTaskFinishWaitsForCancellationCallbacks(t *testing.T) {
manager := NewAgentTaskManager()
task, _ := manager.StartTask("conv", "job", nil)
entered, release := make(chan struct{}), make(chan struct{})
var once sync.Once
manager.SetToolCanceler(func(string) { once.Do(func() { close(entered); <-release }) })
cancelled := make(chan struct{})
go func() { _, _ = manager.CancelTask("conv", ErrTaskCancelled); close(cancelled) }()
<-entered
finished := make(chan struct{})
go func() { _ = manager.FinishTaskRun("conv", task.RunID, "cancelled"); close(finished) }()
select {
case <-finished:
t.Error("task finished while old cancellation callbacks could still affect new run")
case <-time.After(30 * time.Millisecond):
}
close(release)
<-cancelled
<-finished
manager.Shutdown()
}
func TestTaskWaitsForDetachedMCPWorker(t *testing.T) {
manager := NewAgentTaskManager()
defer manager.Shutdown()
task, _ := manager.StartTask("conv", "job", nil)
ctx := manager.BindProcessScope(context.Background(), "conv", task.RunID)
service := mcp.NewExecutionService(nil, nil)
entered, cancelled, release := make(chan struct{}), make(chan struct{}), make(chan struct{})
_, err := service.Submit(ctx, mcp.ExecutionRequest{Run: func(ctx context.Context) (*mcp.ToolResult, error) {
close(entered)
<-ctx.Done()
close(cancelled)
<-release
return nil, ctx.Err()
}})
if err != nil {
t.Fatal(err)
}
<-entered
done := make(chan error, 1)
go func() { done <- manager.FinishTaskRun("conv", task.RunID, "completed") }()
<-cancelled
if manager.GetTaskSnapshot("conv") == nil {
t.Error("task released while detached worker still running")
}
close(release)
if err = <-done; err != nil {
t.Fatal(err)
}
}
func TestTaskReportsUnconfirmedRemoteCleanup(t *testing.T) {
manager := NewAgentTaskManager()
defer manager.Shutdown()
task, _ := manager.StartTask("conv", "job", nil)
ctx := manager.BindProcessScope(context.Background(), "conv", task.RunID)
service := mcp.NewExecutionService(nil, nil)
entered := make(chan struct{})
_, err := service.Submit(ctx, mcp.ExecutionRequest{Remote: true, Run: func(ctx context.Context) (*mcp.ToolResult, error) {
close(entered)
<-ctx.Done()
return nil, ctx.Err()
}})
if err != nil {
t.Fatal(err)
}
<-entered
err = manager.FinishTaskRun("conv", task.RunID, "completed")
if !errors.Is(err, runlease.ErrUnconfirmed) {
t.Fatalf("remote cancellation reported as verified: %v", err)
}
history := manager.GetCompletedTasks()
if len(history) != 1 || history[0].Status != "cleanup_unconfirmed" || history[0].CleanupError == "" {
t.Fatalf("missing retained warning: %+v", history)
}
}
+19 -7
View File
@@ -55,9 +55,10 @@ func (h *AgentHandler) runRoleWorkflowStreamIfBound(
taskStatus := "completed"
taskOwned := false
var taskRunID string
defer func() {
if taskOwned {
h.tasks.FinishTask(conversationID, taskStatus)
h.tasks.FinishTaskRun(conversationID, taskRunID, taskStatus)
}
}()
@@ -69,7 +70,7 @@ func (h *AgentHandler) runRoleWorkflowStreamIfBound(
taskCtx, timeoutCancel := context.WithTimeout(baseCtx, 600*time.Minute)
defer timeoutCancel()
if _, err := h.tasks.StartTask(conversationID, userMessage, cancelWithCause); err != nil {
if startedTask, err := h.tasks.StartTask(conversationID, userMessage, cancelWithCause); err != nil {
var errorMsg string
if errors.Is(err, ErrTaskAlreadyRunning) {
errorMsg = "⚠️ 当前会话已有任务正在执行中,请等待当前任务完成或点击「停止任务」后再尝试。"
@@ -86,8 +87,13 @@ func (h *AgentHandler) runRoleWorkflowStreamIfBound(
}
sendEvent("done", "", map[string]interface{}{"conversationId": conversationID})
return true
} else {
taskRunID = startedTask.RunID
}
baseCtx = h.tasks.BindProcessScope(baseCtx, conversationID, taskRunID)
taskCtx = h.tasks.BindProcessScope(taskCtx, conversationID, taskRunID)
taskOwned = true
sendEvent = h.taskFinishingEventSender(sendEvent, conversationID, taskRunID, func() string { return taskStatus })
progress := h.createProgressCallback(taskCtx, cancelWithCause, conversationID, assistantMessageID, sendEvent)
result, err := workflowrunner.RunRoleBoundWorkflow(taskCtx, workflowrunner.RunArgs{
@@ -202,9 +208,10 @@ func (h *AgentHandler) runRoleWorkflowJSONIfBound(c *gin.Context, req *ChatReque
taskStatus := "completed"
taskOwned := false
var taskRunID string
defer func() {
if taskOwned {
h.tasks.FinishTask(conversationID, taskStatus)
h.tasks.FinishTaskRun(conversationID, taskRunID, taskStatus)
}
}()
@@ -213,7 +220,7 @@ func (h *AgentHandler) runRoleWorkflowJSONIfBound(c *gin.Context, req *ChatReque
taskCtx, timeoutCancel := context.WithTimeout(baseCtx, 600*time.Minute)
defer timeoutCancel()
if _, err := h.tasks.StartTask(conversationID, userMessage, cancelWithCause); err != nil {
if startedTask, err := h.tasks.StartTask(conversationID, userMessage, cancelWithCause); err != nil {
if errors.Is(err, ErrTaskAlreadyRunning) {
c.JSON(http.StatusConflict, gin.H{
"error": "⚠️ 当前会话已有任务正在执行中,请等待当前任务完成或点击「停止任务」后再尝试。",
@@ -224,8 +231,13 @@ func (h *AgentHandler) runRoleWorkflowJSONIfBound(c *gin.Context, req *ChatReque
c.JSON(http.StatusInternalServerError, gin.H{"error": "❌ 无法启动任务: " + err.Error()})
}
return true
} else {
taskRunID = startedTask.RunID
}
baseCtx = h.tasks.BindProcessScope(baseCtx, conversationID, taskRunID)
taskCtx = h.tasks.BindProcessScope(taskCtx, conversationID, taskRunID)
taskOwned = true
respond := h.taskFinishingJSONResponder(c, conversationID, taskRunID, func() string { return taskStatus })
progress := h.createProgressCallback(taskCtx, cancelWithCause, conversationID, assistantMessageID, nil)
result, err := workflowrunner.RunRoleBoundWorkflow(taskCtx, workflowrunner.RunArgs{
@@ -253,7 +265,7 @@ func (h *AgentHandler) runRoleWorkflowJSONIfBound(c *gin.Context, req *ChatReque
_ = h.appendAssistantMessageNotice(assistantMessageID, cancelMsg)
_ = h.db.AddProcessDetail(assistantMessageID, conversationID, "cancelled", cancelMsg, nil)
}
c.JSON(http.StatusOK, gin.H{
respond(http.StatusOK, gin.H{
"status": "cancelled",
"message": cancelMsg,
"conversationId": conversationID,
@@ -265,7 +277,7 @@ func (h *AgentHandler) runRoleWorkflowJSONIfBound(c *gin.Context, req *ChatReque
if assistantMessageID != "" {
_, _ = h.db.Exec("UPDATE messages SET content = ?, updated_at = ? WHERE id = ?", errMsg, time.Now(), assistantMessageID)
}
c.JSON(http.StatusInternalServerError, gin.H{"error": errMsg, "conversationId": conversationID})
respond(http.StatusInternalServerError, gin.H{"error": errMsg, "conversationId": conversationID})
return true
}
decision := h.finalizeCandidateForDeliveryWithPolicy(
@@ -283,7 +295,7 @@ func (h *AgentHandler) runRoleWorkflowJSONIfBound(c *gin.Context, req *ChatReque
responseText = finalizationBlockedMessage(decision)
taskStatus = decision.Status
}
c.JSON(http.StatusOK, gin.H{
respond(http.StatusOK, gin.H{
"response": responseText,
"conversationId": prep.ConversationID,
"assistantMessageId": prep.AssistantMessageID,