mirror of
https://github.com/Ed1s0nZ/CyberStrikeAI.git
synced 2026-09-18 07:32:20 +02:00
feat: manage task process lifetimes and preserve turn history
This commit is contained in:
@@ -221,13 +221,19 @@ func (h *AgentHandler) CancelRunningTaskForConversation(conversationID string) {
|
||||
if h == nil || conversationID == "" || h.tasks == nil {
|
||||
return
|
||||
}
|
||||
h.cancelRunningMCPToolsForConversation(conversationID)
|
||||
h.tasks.AbortActiveEinoExecute(conversationID, "")
|
||||
if ok, err := h.tasks.CancelTask(conversationID, ErrTaskCancelled); ok {
|
||||
h.logger.Info("已取消会话运行中任务", zap.String("conversationId", conversationID))
|
||||
} else if err != nil {
|
||||
h.logger.Warn("取消会话运行中任务失败", zap.String("conversationId", conversationID), zap.Error(err))
|
||||
ok, err := h.tasks.CancelTask(conversationID, ErrTaskCancelled)
|
||||
if !ok {
|
||||
h.cancelRunningMCPToolsForConversation(conversationID)
|
||||
h.tasks.AbortActiveEinoExecute(conversationID, "")
|
||||
}
|
||||
if h.logger != nil {
|
||||
if err != nil {
|
||||
h.logger.Warn("取消会话运行中任务失败", zap.String("conversationId", conversationID), zap.Error(err))
|
||||
} else if ok {
|
||||
h.logger.Info("已取消会话运行中任务", zap.String("conversationId", conversationID))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// ConversationTaskRuntimeState exposes the authoritative live state and start
|
||||
@@ -893,15 +899,24 @@ func (h *AgentHandler) ProcessMessageForRobot(ctx context.Context, platform stri
|
||||
taskCtx, cancelWithCause := context.WithCancelCause(ctx)
|
||||
defer cancelWithCause(nil)
|
||||
taskStatus := "completed"
|
||||
var taskRunID string
|
||||
defer func() {
|
||||
h.tasks.FinishTask(conversationID, taskStatus)
|
||||
if taskRunID == "" {
|
||||
return
|
||||
}
|
||||
if cleanupErr := h.tasks.FinishTaskRun(conversationID, taskRunID, taskStatus); cleanupErr != nil {
|
||||
err = errors.Join(err, cleanupErr)
|
||||
}
|
||||
}()
|
||||
if _, err := h.tasks.StartTask(conversationID, message, cancelWithCause); err != nil {
|
||||
if startedTask, err := h.tasks.StartTask(conversationID, message, cancelWithCause); err != nil {
|
||||
if errors.Is(err, ErrTaskAlreadyRunning) {
|
||||
return "", conversationID, fmt.Errorf("当前会话已有任务正在执行中,请稍后再试")
|
||||
}
|
||||
return "", conversationID, fmt.Errorf("无法启动任务: %w", err)
|
||||
} else {
|
||||
taskRunID = startedTask.RunID
|
||||
}
|
||||
taskCtx = h.tasks.BindProcessScope(taskCtx, conversationID, taskRunID)
|
||||
progressCallback := h.createProgressCallback(taskCtx, cancelWithCause, conversationID, assistantMessageID, nil)
|
||||
|
||||
robotMode := config.NormalizeAgentMode(agentMode)
|
||||
|
||||
@@ -171,19 +171,14 @@ func (h *AgentHandler) executeOneBatchSubTask(queueID string, queue *BatchTaskQu
|
||||
taskCtx, timeoutCancel := context.WithTimeout(baseCtx, 6*time.Hour)
|
||||
|
||||
registered := false
|
||||
var taskRunID string
|
||||
finishStatus := "completed"
|
||||
|
||||
defer func() {
|
||||
h.batchTaskManager.SetTaskCancel(queueID, task.ID, nil)
|
||||
timeoutCancel()
|
||||
if registered {
|
||||
if h.taskEventBus != nil {
|
||||
ev := StreamEvent{Type: "done", Message: "", Data: map[string]interface{}{"conversationId": conversationID}}
|
||||
if b, err := json.Marshal(ev); err == nil {
|
||||
h.taskEventBus.Publish(conversationID, append(append([]byte("data: "), b...), '\n', '\n'))
|
||||
}
|
||||
}
|
||||
h.tasks.FinishTask(conversationID, finishStatus)
|
||||
h.tasks.FinishTaskRun(conversationID, taskRunID, finishStatus)
|
||||
}
|
||||
cancelWithCause(nil)
|
||||
}()
|
||||
@@ -204,7 +199,7 @@ func (h *AgentHandler) executeOneBatchSubTask(queueID string, queue *BatchTaskQu
|
||||
h.taskEventBus.Publish(conversationID, line)
|
||||
}
|
||||
|
||||
if _, err := h.tasks.StartTask(conversationID, task.Message, cancelWithCause); err != nil {
|
||||
if startedTask, err := h.tasks.StartTask(conversationID, task.Message, cancelWithCause); err != nil {
|
||||
h.logger.Warn("批量队列子任务注册会话运行状态失败",
|
||||
zap.String("queueId", queueID),
|
||||
zap.String("taskId", task.ID),
|
||||
@@ -216,7 +211,11 @@ func (h *AgentHandler) executeOneBatchSubTask(queueID string, queue *BatchTaskQu
|
||||
}
|
||||
h.batchTaskManager.UpdateTaskStatus(queueID, task.ID, BatchTaskStatusFailed, "", failMsg)
|
||||
return
|
||||
} else {
|
||||
taskRunID = startedTask.RunID
|
||||
}
|
||||
baseCtx = h.tasks.BindProcessScope(baseCtx, conversationID, taskRunID)
|
||||
taskCtx = h.tasks.BindProcessScope(taskCtx, conversationID, taskRunID)
|
||||
registered = true
|
||||
h.batchTaskManager.SetTaskCancel(queueID, task.ID, timeoutCancel)
|
||||
|
||||
@@ -342,6 +341,10 @@ func (h *AgentHandler) executeOneBatchSubTask(queueID string, queue *BatchTaskQu
|
||||
}
|
||||
}
|
||||
|
||||
if cleanupErr := h.tasks.FinishTaskRun(conversationID, taskRunID, finishStatus); cleanupErr != nil {
|
||||
h.batchTaskManager.UpdateTaskStatusWithConversationID(queueID, task.ID, BatchTaskStatusFailed, resText, cleanupErr.Error(), conversationID)
|
||||
return
|
||||
}
|
||||
if !decision.Finalizable {
|
||||
h.batchTaskManager.UpdateTaskStatusWithConversationID(queueID, task.ID, BatchTaskStatusFailed, resText, finalizationCheckMessage(decision), conversationID)
|
||||
return
|
||||
|
||||
@@ -130,9 +130,10 @@ func (h *AgentHandler) EinoSingleAgentLoopStream(c *gin.Context) {
|
||||
// 仅在成功 StartTask 后再 FinishTask。若 StartTask 因 ErrTaskAlreadyRunning 失败仍 defer FinishTask,
|
||||
// 会误删其他连接上正在运行的同会话任务,导致「第一次拦截、第二次却放行」。
|
||||
taskOwned := false
|
||||
var taskRunID string
|
||||
defer func() {
|
||||
if taskOwned {
|
||||
h.tasks.FinishTask(conversationID, taskStatus)
|
||||
h.tasks.FinishTaskRun(conversationID, taskRunID, taskStatus)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -165,7 +166,7 @@ func (h *AgentHandler) EinoSingleAgentLoopStream(c *gin.Context) {
|
||||
baseCtx, cancelWithCause = context.WithCancelCause(detachedAgentContext(c.Request.Context()))
|
||||
taskCtx, timeoutCancel := context.WithTimeout(baseCtx, 600*time.Minute)
|
||||
|
||||
if _, err := h.tasks.StartTask(conversationID, req.Message, cancelWithCause); err != nil {
|
||||
if startedTask, err := h.tasks.StartTask(conversationID, req.Message, cancelWithCause); err != nil {
|
||||
var errorMsg string
|
||||
if errors.Is(err, ErrTaskAlreadyRunning) {
|
||||
errorMsg = "⚠️ 当前会话已有任务正在执行中,请等待当前任务完成或点击「停止任务」后再尝试。"
|
||||
@@ -183,8 +184,13 @@ func (h *AgentHandler) EinoSingleAgentLoopStream(c *gin.Context) {
|
||||
sendEvent("done", "", map[string]interface{}{"conversationId": conversationID})
|
||||
timeoutCancel()
|
||||
return
|
||||
} else {
|
||||
taskRunID = startedTask.RunID
|
||||
}
|
||||
baseCtx = h.tasks.BindProcessScope(baseCtx, conversationID, taskRunID)
|
||||
taskCtx = h.tasks.BindProcessScope(taskCtx, conversationID, taskRunID)
|
||||
taskOwned = true
|
||||
sendEvent = h.taskFinishingEventSender(sendEvent, conversationID, taskRunID, func() string { return taskStatus })
|
||||
|
||||
var cumulativeMCPExecutionIDs []string
|
||||
// 同一请求内分段续跑时,主代理 iteration 事件按偏移累计,避免 UI 出现「第3轮 → 第1轮」回跳。
|
||||
@@ -454,18 +460,31 @@ func (h *AgentHandler) EinoSingleAgentLoop(c *gin.Context) {
|
||||
defer cancelWithCause(nil)
|
||||
taskCtx, timeoutCancel := context.WithTimeout(baseCtx, 600*time.Minute)
|
||||
defer timeoutCancel()
|
||||
jsonTask, startErr := h.tasks.StartTask(prep.ConversationID, req.Message, cancelWithCause)
|
||||
if startErr != nil {
|
||||
c.JSON(http.StatusConflict, gin.H{"error": startErr.Error()})
|
||||
return
|
||||
}
|
||||
taskCtx = h.tasks.BindProcessScope(taskCtx, prep.ConversationID, jsonTask.RunID)
|
||||
taskCtx = mcp.WithMCPConversationID(taskCtx, prep.ConversationID)
|
||||
taskCtx = mcp.WithToolRunRegistry(taskCtx, h.tasks)
|
||||
taskCtx = mcp.WithEinoExecuteRunRegistry(taskCtx, h.tasks)
|
||||
jsonTaskStatus := "failed"
|
||||
defer func() { _ = h.tasks.FinishTaskRun(prep.ConversationID, jsonTask.RunID, jsonTaskStatus) }()
|
||||
respond := h.taskFinishingJSONResponder(c, prep.ConversationID, jsonTask.RunID, func() string { return jsonTaskStatus })
|
||||
|
||||
progressCallback := h.createProgressCallback(taskCtx, cancelWithCause, prep.ConversationID, prep.AssistantMessageID, progressCallbackRaw)
|
||||
taskCtx = multiagent.WithHITLToolInterceptor(taskCtx, func(ctx context.Context, toolName, arguments string) (string, error) {
|
||||
return h.interceptHITLForEinoTool(ctx, cancelWithCause, prep.ConversationID, prep.AssistantMessageID, nil, toolName, arguments)
|
||||
})
|
||||
|
||||
if h.config == nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "服务器配置未加载"})
|
||||
respond(http.StatusInternalServerError, gin.H{"error": "服务器配置未加载"})
|
||||
return
|
||||
}
|
||||
runCfg, _, err := h.configForAIChannel(req.AIChannelID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
respond(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
@@ -498,7 +517,7 @@ func (h *AgentHandler) EinoSingleAgentLoop(c *gin.Context) {
|
||||
if shouldPersistEinoAgentTraceAfterRunError(baseCtx) {
|
||||
h.persistEinoAgentTraceForResume(prep.ConversationID, result)
|
||||
}
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": runErr.Error()})
|
||||
respond(http.StatusInternalServerError, gin.H{"error": runErr.Error()})
|
||||
return
|
||||
}
|
||||
mw := &h.config.MultiAgent.EinoMiddleware
|
||||
@@ -525,7 +544,12 @@ func (h *AgentHandler) EinoSingleAgentLoop(c *gin.Context) {
|
||||
if !decision.Finalizable {
|
||||
responseText = finalizationBlockedMessage(decision)
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
|
||||
jsonTaskStatus = decision.Status
|
||||
if jsonTaskStatus == "" {
|
||||
jsonTaskStatus = "completed"
|
||||
}
|
||||
respond(http.StatusOK, gin.H{
|
||||
"response": responseText,
|
||||
"conversationId": prep.ConversationID,
|
||||
"mcpExecutionIds": result.MCPExecutionIDs,
|
||||
|
||||
@@ -147,9 +147,10 @@ func (h *AgentHandler) MultiAgentLoopStream(c *gin.Context) {
|
||||
taskStatus := "completed"
|
||||
// 仅在成功 StartTask 后再 FinishTask;避免「任务已存在」分支 return 时误删正在运行的同会话任务。
|
||||
taskOwned := false
|
||||
var taskRunID string
|
||||
defer func() {
|
||||
if taskOwned {
|
||||
h.tasks.FinishTask(conversationID, taskStatus)
|
||||
h.tasks.FinishTaskRun(conversationID, taskRunID, taskStatus)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -172,7 +173,7 @@ func (h *AgentHandler) MultiAgentLoopStream(c *gin.Context) {
|
||||
baseCtx, cancelWithCause = context.WithCancelCause(detachedAgentContext(c.Request.Context()))
|
||||
taskCtx, timeoutCancel := context.WithTimeout(baseCtx, 600*time.Minute)
|
||||
|
||||
if _, err := h.tasks.StartTask(conversationID, req.Message, cancelWithCause); err != nil {
|
||||
if startedTask, err := h.tasks.StartTask(conversationID, req.Message, cancelWithCause); err != nil {
|
||||
var errorMsg string
|
||||
if errors.Is(err, ErrTaskAlreadyRunning) {
|
||||
errorMsg = "⚠️ 当前会话已有任务正在执行中,请等待当前任务完成或点击「停止任务」后再尝试。"
|
||||
@@ -190,8 +191,13 @@ func (h *AgentHandler) MultiAgentLoopStream(c *gin.Context) {
|
||||
sendEvent("done", "", map[string]interface{}{"conversationId": conversationID})
|
||||
timeoutCancel()
|
||||
return
|
||||
} else {
|
||||
taskRunID = startedTask.RunID
|
||||
}
|
||||
baseCtx = h.tasks.BindProcessScope(baseCtx, conversationID, taskRunID)
|
||||
taskCtx = h.tasks.BindProcessScope(taskCtx, conversationID, taskRunID)
|
||||
taskOwned = true
|
||||
sendEvent = h.taskFinishingEventSender(sendEvent, conversationID, taskRunID, func() string { return taskStatus })
|
||||
|
||||
// 同一 HTTP 流内多段 Run(如中断并继续)合并 MCP execution id,供最终 response / 库表与工具芯片展示完整列表
|
||||
var cumulativeMCPExecutionIDs []string
|
||||
@@ -468,13 +474,26 @@ func (h *AgentHandler) MultiAgentLoop(c *gin.Context) {
|
||||
defer cancelWithCause(nil)
|
||||
taskCtx, timeoutCancel := context.WithTimeout(baseCtx, 600*time.Minute)
|
||||
defer timeoutCancel()
|
||||
jsonTask, startErr := h.tasks.StartTask(prep.ConversationID, req.Message, cancelWithCause)
|
||||
if startErr != nil {
|
||||
c.JSON(http.StatusConflict, gin.H{"error": startErr.Error()})
|
||||
return
|
||||
}
|
||||
taskCtx = h.tasks.BindProcessScope(taskCtx, prep.ConversationID, jsonTask.RunID)
|
||||
taskCtx = mcp.WithMCPConversationID(taskCtx, prep.ConversationID)
|
||||
taskCtx = mcp.WithToolRunRegistry(taskCtx, h.tasks)
|
||||
taskCtx = mcp.WithEinoExecuteRunRegistry(taskCtx, h.tasks)
|
||||
jsonTaskStatus := "failed"
|
||||
defer func() { _ = h.tasks.FinishTaskRun(prep.ConversationID, jsonTask.RunID, jsonTaskStatus) }()
|
||||
respond := h.taskFinishingJSONResponder(c, prep.ConversationID, jsonTask.RunID, func() string { return jsonTaskStatus })
|
||||
|
||||
progressCallback := h.createProgressCallback(taskCtx, cancelWithCause, prep.ConversationID, prep.AssistantMessageID, nil)
|
||||
taskCtx = multiagent.WithHITLToolInterceptor(taskCtx, func(ctx context.Context, toolName, arguments string) (string, error) {
|
||||
return h.interceptHITLForEinoTool(ctx, cancelWithCause, prep.ConversationID, prep.AssistantMessageID, nil, toolName, arguments)
|
||||
})
|
||||
runCfg, _, err := h.configForAIChannel(req.AIChannelID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
respond(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
@@ -522,7 +541,7 @@ func (h *AgentHandler) MultiAgentLoop(c *gin.Context) {
|
||||
}
|
||||
errData := multiagent.EinoClientRunErrorFields(runErr)
|
||||
errData["error"] = errMsg
|
||||
c.JSON(http.StatusInternalServerError, errData)
|
||||
respond(http.StatusInternalServerError, errData)
|
||||
return
|
||||
}
|
||||
mw := &h.config.MultiAgent.EinoMiddleware
|
||||
@@ -552,7 +571,12 @@ func (h *AgentHandler) MultiAgentLoop(c *gin.Context) {
|
||||
if !decision.Finalizable {
|
||||
responseText = finalizationBlockedMessage(decision)
|
||||
}
|
||||
c.JSON(http.StatusOK, ChatResponse{
|
||||
|
||||
jsonTaskStatus = decision.Status
|
||||
if jsonTaskStatus == "" {
|
||||
jsonTaskStatus = "completed"
|
||||
}
|
||||
respond(http.StatusOK, ChatResponse{
|
||||
Response: responseText,
|
||||
MCPExecutionIDs: result.MCPExecutionIDs,
|
||||
ConversationID: prep.ConversationID,
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"cyberstrike-ai/internal/runlease"
|
||||
"errors"
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.uber.org/zap"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// ShutdownTasks stops local work before shared MCP clients and databases close.
|
||||
func (h *AgentHandler) ShutdownTasks() {
|
||||
if h != nil && h.tasks != nil {
|
||||
h.tasks.Shutdown()
|
||||
if h.logger != nil {
|
||||
for _, task := range h.tasks.GetActiveTasks() {
|
||||
h.logger.Warn("服务关闭时任务仍有未完成的清理", zap.String("runId", task.RunID), zap.String("cleanupError", task.CleanupError))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// taskFinishingEventSender makes the visible done event follow local cleanup.
|
||||
func (h *AgentHandler) taskFinishingEventSender(send func(string, string, interface{}), conversationID, runID string, status func() string) func(string, string, interface{}) {
|
||||
return func(eventType, message string, data interface{}) {
|
||||
if eventType == "done" {
|
||||
if err := h.tasks.FinishTaskRun(conversationID, runID, status()); err != nil {
|
||||
if h.logger != nil {
|
||||
h.logger.Warn(taskCleanupMessage(err), zap.String("runId", runID), zap.Error(err))
|
||||
}
|
||||
send("error", taskCleanupMessage(err)+": "+err.Error(), map[string]interface{}{"errorType": taskCleanupStatus(err)})
|
||||
data = map[string]interface{}{"conversationId": conversationID, "runId": runID, "status": taskCleanupStatus(err), "cleanupError": err.Error()}
|
||||
}
|
||||
}
|
||||
send(eventType, message, data)
|
||||
}
|
||||
}
|
||||
|
||||
// taskFinishingJSONResponder applies the same ordering to successful and failed
|
||||
// non-streaming requests. No response claims completion before cleanup returns.
|
||||
func (h *AgentHandler) taskFinishingJSONResponder(c *gin.Context, conversationID, runID string, status func() string) func(int, interface{}) {
|
||||
return func(code int, payload interface{}) {
|
||||
if err := h.tasks.FinishTaskRun(conversationID, runID, status()); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error(), "status": taskCleanupStatus(err), "conversationId": conversationID})
|
||||
return
|
||||
}
|
||||
c.JSON(code, payload)
|
||||
}
|
||||
}
|
||||
|
||||
func taskCleanupStatus(err error) string {
|
||||
if errors.Is(err, runlease.ErrUnconfirmed) {
|
||||
return "cleanup_unconfirmed"
|
||||
}
|
||||
return "cleanup_failed"
|
||||
}
|
||||
func taskCleanupMessage(err error) string {
|
||||
if errors.Is(err, runlease.ErrUnconfirmed) {
|
||||
return "本地执行已结束,远端 MCP 停止状态待确认"
|
||||
}
|
||||
return "任务资源清理失败,将自动重试"
|
||||
}
|
||||
@@ -2,13 +2,17 @@ package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"cyberstrike-ai/internal/multiagent"
|
||||
"cyberstrike-ai/internal/runlease"
|
||||
"cyberstrike-ai/internal/security"
|
||||
)
|
||||
|
||||
// ErrTaskCancelled 用户取消任务的错误
|
||||
@@ -26,12 +30,20 @@ func shouldPersistEinoAgentTraceAfterRunError(baseCtx context.Context) bool {
|
||||
|
||||
// AgentTask 描述正在运行的Agent任务
|
||||
type AgentTask struct {
|
||||
ConversationID string `json:"conversationId"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
Status string `json:"status"`
|
||||
CancellingAt time.Time `json:"-"` // 进入 cancelling 状态的时间,用于清理长时间卡住的任务
|
||||
RunID string `json:"runId"`
|
||||
CleanupError string `json:"cleanupError,omitempty"`
|
||||
processes *security.ProcessScope
|
||||
workers *runlease.Scope
|
||||
IsolationBackend string `json:"isolationBackend,omitempty"`
|
||||
finishing chan struct{}
|
||||
stopping chan struct{}
|
||||
finalStatus string
|
||||
ConversationID string `json:"conversationId"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
Status string `json:"status"`
|
||||
CancellingAt time.Time `json:"-"` // 进入 cancelling 状态的时间,用于清理长时间卡住的任务
|
||||
|
||||
// ActiveMCPExecutionID 当前正在执行的 MCP 工具 executionId(仅内存,供「中断并继续」= 仅掐当前工具)
|
||||
ActiveMCPExecutionID string `json:"-"`
|
||||
@@ -297,12 +309,15 @@ func (m *AgentTaskManager) ActiveMCPExecutionID(conversationID string) string {
|
||||
|
||||
// CompletedTask 已完成的任务(用于历史记录)
|
||||
type CompletedTask struct {
|
||||
ConversationID string `json:"conversationId"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
CompletedAt time.Time `json:"completedAt"`
|
||||
Status string `json:"status"`
|
||||
CleanupError string `json:"cleanupError,omitempty"`
|
||||
IsolationBackend string `json:"isolationBackend,omitempty"`
|
||||
RunID string `json:"runId"`
|
||||
ConversationID string `json:"conversationId"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
CompletedAt time.Time `json:"completedAt"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
// AgentTaskManager 管理正在运行的Agent任务
|
||||
@@ -315,6 +330,9 @@ type AgentTaskManager struct {
|
||||
eventBus *TaskEventBus // 可选:任务结束时关闭镜像 SSE 订阅
|
||||
// toolCanceler 在用户整轮停止任务或会话结束时终止该会话仍在运行的 MCP 工具(非「中断并继续」)。
|
||||
toolCanceler func(conversationID string)
|
||||
shuttingDown bool
|
||||
shutdown chan struct{}
|
||||
shutdownOnce sync.Once
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -330,6 +348,7 @@ const (
|
||||
func NewAgentTaskManager() *AgentTaskManager {
|
||||
m := &AgentTaskManager{
|
||||
tasks: make(map[string]*AgentTask),
|
||||
shutdown: make(chan struct{}),
|
||||
completedTasks: make([]*CompletedTask, 0),
|
||||
maxHistorySize: 50, // 最多保留50条历史记录
|
||||
historyRetention: 24 * time.Hour, // 保留24小时
|
||||
@@ -368,6 +387,7 @@ func (m *AgentTaskManager) GetTaskSnapshot(conversationID string) *AgentTask {
|
||||
return nil
|
||||
}
|
||||
snapshot := *task
|
||||
snapshot.IsolationBackend = task.processes.IsolationBackend()
|
||||
return &snapshot
|
||||
}
|
||||
|
||||
@@ -375,16 +395,26 @@ func (m *AgentTaskManager) GetTaskSnapshot(conversationID string) *AgentTask {
|
||||
func (m *AgentTaskManager) runStuckCancellingCleanup() {
|
||||
ticker := time.NewTicker(cleanupInterval)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
m.cleanupStuckCancelling()
|
||||
for {
|
||||
select {
|
||||
case <-m.shutdown:
|
||||
return
|
||||
case <-ticker.C:
|
||||
m.cleanupStuckCancelling()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *AgentTaskManager) cleanupStuckCancelling() {
|
||||
m.mu.Lock()
|
||||
var toFinish []string
|
||||
type pendingFinish struct{ id, runID, status string }
|
||||
var toFinish []pendingFinish
|
||||
now := time.Now()
|
||||
for id, task := range m.tasks {
|
||||
if task.Status == "cleanup_failed" {
|
||||
toFinish = append(toFinish, pendingFinish{id, task.RunID, task.finalStatus})
|
||||
continue
|
||||
}
|
||||
if task.Status != "cancelling" {
|
||||
continue
|
||||
}
|
||||
@@ -400,11 +430,11 @@ func (m *AgentTaskManager) cleanupStuckCancelling() {
|
||||
continue
|
||||
}
|
||||
}
|
||||
toFinish = append(toFinish, id)
|
||||
toFinish = append(toFinish, pendingFinish{id, task.RunID, "cancelled"})
|
||||
}
|
||||
m.mu.Unlock()
|
||||
for _, id := range toFinish {
|
||||
m.FinishTask(id, "cancelled")
|
||||
for _, pending := range toFinish {
|
||||
_ = m.FinishTaskRun(pending.id, pending.runID, pending.status)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -413,11 +443,16 @@ func (m *AgentTaskManager) StartTask(conversationID, message string, cancel cont
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if m.shuttingDown {
|
||||
return nil, errors.New("task manager is shutting down")
|
||||
}
|
||||
if _, exists := m.tasks[conversationID]; exists {
|
||||
return nil, ErrTaskAlreadyRunning
|
||||
}
|
||||
|
||||
scope := security.NewProcessScope()
|
||||
task := &AgentTask{
|
||||
RunID: scope.ID, processes: scope, workers: runlease.New(),
|
||||
ConversationID: conversationID,
|
||||
Message: message,
|
||||
StartedAt: time.Now(),
|
||||
@@ -444,7 +479,7 @@ func (m *AgentTaskManager) CancelTask(conversationID string, cause error) (bool,
|
||||
}
|
||||
|
||||
// 如果已经处于取消流程,视为成功(幂等),避免前端重复点击报「未找到任务」
|
||||
if task.Status == "cancelling" {
|
||||
if task.Status == "cancelling" || task.finishing != nil {
|
||||
m.mu.Unlock()
|
||||
return true, nil
|
||||
}
|
||||
@@ -466,6 +501,13 @@ func (m *AgentTaskManager) CancelTask(conversationID string, cause error) (bool,
|
||||
interruptPush := task.agentTurnLoopInterrupt
|
||||
interruptNote := task.InterruptContinueNote
|
||||
runtimeCancel := task.agentRuntimeCancel
|
||||
activeExecuteCancel := task.activeEinoExecuteCancel
|
||||
if !errors.Is(cause, multiagent.ErrInterruptContinue) {
|
||||
task.processes.Seal()
|
||||
task.workers.Seal()
|
||||
task.stopping = make(chan struct{})
|
||||
defer close(task.stopping)
|
||||
}
|
||||
var toolCanceler func(string)
|
||||
if errors.Is(cause, ErrTaskCancelled) {
|
||||
toolCanceler = m.toolCanceler
|
||||
@@ -494,6 +536,13 @@ func (m *AgentTaskManager) CancelTask(conversationID string, cause error) (bool,
|
||||
if toolCanceler != nil {
|
||||
toolCanceler(conversationID)
|
||||
}
|
||||
if !errors.Is(cause, multiagent.ErrInterruptContinue) {
|
||||
task.workers.Cancel()
|
||||
if activeExecuteCancel != nil {
|
||||
activeExecuteCancel()
|
||||
}
|
||||
return true, task.processes.Close()
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -507,54 +556,172 @@ func (m *AgentTaskManager) UpdateTaskStatus(conversationID string, status string
|
||||
return
|
||||
}
|
||||
|
||||
if status != "" {
|
||||
task.Status = status
|
||||
if task.finishing != nil || task.Status == "cleanup_failed" {
|
||||
return
|
||||
}
|
||||
switch status {
|
||||
case "completed", "cancelled", "failed", "timeout":
|
||||
task.finalStatus = status
|
||||
task.Status = "cleaning"
|
||||
task.processes.Seal()
|
||||
task.workers.Seal()
|
||||
default:
|
||||
if status != "" {
|
||||
task.Status = status
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// FinishTask 完成任务并从管理器中移除
|
||||
// BindProcessScope snapshots ownership once at task start. Continuations must
|
||||
// derive from this context, never resolve ownership again using conversation ID.
|
||||
func (m *AgentTaskManager) BindProcessScope(ctx context.Context, conversationID, runID string) context.Context {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
if task := m.tasks[conversationID]; task != nil && task.RunID == runID {
|
||||
return runlease.WithScope(security.WithProcessScope(ctx, task.processes), task.workers)
|
||||
}
|
||||
// Fail closed if a task disappeared before its execution context was bound.
|
||||
scope := security.NewProcessScope()
|
||||
scope.Seal()
|
||||
workers := runlease.New()
|
||||
workers.Seal()
|
||||
return runlease.WithScope(security.WithProcessScope(ctx, scope), workers)
|
||||
}
|
||||
|
||||
// FinishTask is retained for callers that operate on the current task. Owners
|
||||
// use FinishTaskRun, so a delayed defer cannot finish a newer conversation run.
|
||||
func (m *AgentTaskManager) FinishTask(conversationID string, finalStatus string) {
|
||||
m.mu.RLock()
|
||||
task := m.tasks[conversationID]
|
||||
m.mu.RUnlock()
|
||||
if task != nil {
|
||||
_ = m.FinishTaskRun(conversationID, task.RunID, finalStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *AgentTaskManager) FinishTaskRun(conversationID, runID, finalStatus string) error {
|
||||
m.mu.Lock()
|
||||
task, exists := m.tasks[conversationID]
|
||||
if !exists {
|
||||
task := m.tasks[conversationID]
|
||||
if task == nil || task.RunID != runID {
|
||||
m.mu.Unlock()
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
if finalStatus != "" {
|
||||
task.Status = finalStatus
|
||||
if task.stopping != nil {
|
||||
select {
|
||||
case <-task.stopping:
|
||||
default:
|
||||
stopping := task.stopping
|
||||
m.mu.Unlock()
|
||||
<-stopping
|
||||
return m.FinishTaskRun(conversationID, runID, finalStatus)
|
||||
}
|
||||
}
|
||||
if task.finishing != nil {
|
||||
done := task.finishing
|
||||
m.mu.Unlock()
|
||||
<-done
|
||||
m.mu.RLock()
|
||||
cleanupError := task.CleanupError
|
||||
m.mu.RUnlock()
|
||||
if cleanupError != "" {
|
||||
return errors.New(cleanupError)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
done := make(chan struct{})
|
||||
task.finishing = done
|
||||
task.finalStatus = finalStatus
|
||||
task.Status = "cleaning"
|
||||
task.processes.Seal()
|
||||
task.workers.Seal()
|
||||
toolCanceler := m.toolCanceler
|
||||
activeEinoExecuteCancel := task.activeEinoExecuteCancel
|
||||
|
||||
// 保存到历史记录
|
||||
completedTask := &CompletedTask{
|
||||
ConversationID: task.ConversationID,
|
||||
Message: task.Message,
|
||||
StartedAt: task.StartedAt,
|
||||
CompletedAt: time.Now(),
|
||||
Status: finalStatus,
|
||||
}
|
||||
|
||||
// 添加到历史记录
|
||||
m.completedTasks = append(m.completedTasks, completedTask)
|
||||
|
||||
// 清理过期和过多的历史记录
|
||||
m.cleanupHistory()
|
||||
|
||||
// 从运行任务中移除
|
||||
delete(m.tasks, conversationID)
|
||||
activeCancel := task.activeEinoExecuteCancel
|
||||
cancel := task.cancel
|
||||
bus := m.eventBus
|
||||
m.mu.Unlock()
|
||||
|
||||
// Keep the conversation occupied throughout cleanup, including callbacks.
|
||||
if cancel != nil {
|
||||
cancel(nil)
|
||||
}
|
||||
if toolCanceler != nil {
|
||||
toolCanceler(conversationID)
|
||||
}
|
||||
if activeEinoExecuteCancel != nil {
|
||||
activeEinoExecuteCancel()
|
||||
if activeCancel != nil {
|
||||
activeCancel()
|
||||
}
|
||||
if bus != nil {
|
||||
task.workers.Cancel()
|
||||
processErr := task.processes.Close()
|
||||
waitCtx, waitCancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
workerErr := task.workers.Wait(waitCtx)
|
||||
waitCancel()
|
||||
cleanupErr := errors.Join(processErr, workerErr)
|
||||
if processErr != nil && errors.Is(workerErr, runlease.ErrUnconfirmed) {
|
||||
// A simultaneous local failure must not be labelled as local completion.
|
||||
cleanupErr = fmt.Errorf("local cleanup: %w; remote state: %v", processErr, workerErr)
|
||||
}
|
||||
// The local worker has returned, but remote notification cancellation is
|
||||
// not an acknowledgement. Preserve an actionable history/tool status.
|
||||
unconfirmed := processErr == nil && errors.Is(workerErr, runlease.ErrUnconfirmed)
|
||||
if unconfirmed {
|
||||
finalStatus = "cleanup_unconfirmed"
|
||||
}
|
||||
cleanupMessage := ""
|
||||
if cleanupErr != nil {
|
||||
cleanupMessage = cleanupErr.Error()
|
||||
}
|
||||
if (cleanupErr == nil || unconfirmed) && bus != nil {
|
||||
// Subscribers must receive completion only after local processes are reaped.
|
||||
payload, _ := json.Marshal(StreamEvent{Type: "done", Data: map[string]interface{}{"conversationId": conversationID, "runId": runID, "status": finalStatus, "cleanupError": cleanupMessage}})
|
||||
bus.Publish(conversationID, append(append([]byte("data: "), payload...), '\n', '\n'))
|
||||
bus.CloseConversation(conversationID)
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
defer close(done)
|
||||
if cleanupErr != nil && !unconfirmed {
|
||||
task.Status = "cleanup_failed"
|
||||
task.CleanupError = cleanupErr.Error()
|
||||
task.finishing = nil
|
||||
return cleanupErr
|
||||
}
|
||||
task.CleanupError = ""
|
||||
if unconfirmed {
|
||||
task.CleanupError = cleanupErr.Error()
|
||||
}
|
||||
task.Status = finalStatus
|
||||
m.completedTasks = append(m.completedTasks, &CompletedTask{
|
||||
RunID: task.RunID, CleanupError: task.CleanupError, IsolationBackend: task.processes.IsolationBackend(), ConversationID: task.ConversationID, Message: task.Message,
|
||||
StartedAt: task.StartedAt, CompletedAt: time.Now(), Status: finalStatus,
|
||||
})
|
||||
m.cleanupHistory()
|
||||
delete(m.tasks, conversationID)
|
||||
return cleanupErr
|
||||
}
|
||||
|
||||
// Shutdown rejects new tasks before cancelling and reaping existing task jobs.
|
||||
func (m *AgentTaskManager) Shutdown() {
|
||||
m.mu.Lock()
|
||||
m.shuttingDown = true
|
||||
m.shutdownOnce.Do(func() { close(m.shutdown) })
|
||||
tasks := make([]*AgentTask, 0, len(m.tasks))
|
||||
for _, task := range m.tasks {
|
||||
tasks = append(tasks, task)
|
||||
task.processes.Seal()
|
||||
task.workers.Seal()
|
||||
}
|
||||
m.mu.Unlock()
|
||||
var wg sync.WaitGroup
|
||||
for _, task := range tasks {
|
||||
wg.Add(1)
|
||||
go func(task *AgentTask) {
|
||||
defer wg.Done()
|
||||
_, _ = m.CancelTask(task.ConversationID, ErrTaskCancelled)
|
||||
_ = m.FinishTaskRun(task.ConversationID, task.RunID, "cancelled")
|
||||
}(task)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// cleanupHistory 清理过期的历史记录
|
||||
@@ -589,6 +756,7 @@ func (m *AgentTaskManager) GetActiveTasks() []*AgentTask {
|
||||
result := make([]*AgentTask, 0, len(m.tasks))
|
||||
for _, task := range m.tasks {
|
||||
result = append(result, &AgentTask{
|
||||
RunID: task.RunID, CleanupError: task.CleanupError, IsolationBackend: task.processes.IsolationBackend(),
|
||||
ConversationID: task.ConversationID,
|
||||
Message: task.Message,
|
||||
StartedAt: task.StartedAt,
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"cyberstrike-ai/internal/mcp"
|
||||
"cyberstrike-ai/internal/runlease"
|
||||
"cyberstrike-ai/internal/security"
|
||||
)
|
||||
|
||||
func TestTaskCleanupWaitsBeforeReleasingConversation(t *testing.T) {
|
||||
manager := NewAgentTaskManager()
|
||||
task, _ := manager.StartTask("conv", "old", func(error) {})
|
||||
entered, release := make(chan struct{}), make(chan struct{})
|
||||
manager.SetToolCanceler(func(string) { close(entered); <-release })
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- manager.FinishTaskRun("conv", task.RunID, "completed") }()
|
||||
<-entered
|
||||
if status := manager.GetTaskSnapshot("conv").Status; status != "cleaning" {
|
||||
t.Errorf("status = %s", status)
|
||||
}
|
||||
if _, err := manager.StartTask("conv", "new", nil); !errors.Is(err, ErrTaskAlreadyRunning) {
|
||||
t.Errorf("new task admitted during cleanup: %v", err)
|
||||
}
|
||||
ctx := manager.BindProcessScope(context.Background(), "conv", task.RunID)
|
||||
if _, err := security.StartShellSessionContext(ctx, exec.Command("unused-command")); !errors.Is(err, security.ErrProcessScopeClosed) {
|
||||
t.Errorf("late process admitted: %v", err)
|
||||
}
|
||||
close(release)
|
||||
if err := <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manager.SetToolCanceler(nil)
|
||||
next, err := manager.StartTask("conv", "new", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer manager.FinishTask("conv", "completed")
|
||||
if next.RunID == task.RunID {
|
||||
t.Fatal("run identity reused")
|
||||
}
|
||||
_ = manager.FinishTaskRun("conv", task.RunID, "cancelled")
|
||||
if manager.GetTaskSnapshot("conv").RunID != next.RunID {
|
||||
t.Fatal("old defer removed new task")
|
||||
}
|
||||
// A delayed worker keeps its original closed scope, even after a new run starts.
|
||||
if _, err := security.StartManagedBackground(ctx, "sh", "sleep 300", ""); !errors.Is(err, security.ErrProcessScopeClosed) {
|
||||
t.Fatalf("old context borrowed new task: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskFinishAndShutdownReapBackgroundProcesses(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Unix shell")
|
||||
}
|
||||
for _, shutdown := range []bool{false, true} {
|
||||
name := "finish"
|
||||
if shutdown {
|
||||
name = "shutdown"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
manager := NewAgentTaskManager()
|
||||
task, _ := manager.StartTask("conv", "job", nil)
|
||||
ctx := manager.BindProcessScope(context.Background(), "conv", task.RunID)
|
||||
session, err := security.StartManagedBackground(ctx, "sh", "sleep 300", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if shutdown {
|
||||
manager.Shutdown()
|
||||
} else if err := manager.FinishTaskRun("conv", task.RunID, "completed"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() { _ = session.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("task ended before process exited")
|
||||
}
|
||||
if manager.GetTaskSnapshot("conv") != nil {
|
||||
t.Fatal("finished task still active")
|
||||
}
|
||||
if shutdown {
|
||||
if _, err := manager.StartTask("new", "job", nil); err == nil {
|
||||
t.Fatal("shutdown admitted a new task")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskDonePublishedAfterCleanup(t *testing.T) {
|
||||
manager := NewAgentTaskManager()
|
||||
bus := NewTaskEventBus()
|
||||
manager.SetTaskEventBus(bus)
|
||||
task, _ := manager.StartTask("conv", "job", nil)
|
||||
_, events := bus.Subscribe("conv")
|
||||
if err := manager.FinishTaskRun("conv", task.RunID, "completed"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if event, ok := <-events; !ok || len(event) == 0 {
|
||||
t.Fatal("subscriber closed without done event")
|
||||
}
|
||||
if _, ok := <-events; ok {
|
||||
t.Fatal("subscriber not closed after completion")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskCleanupFailureRetainsOwnershipAndRetries(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Unix shell")
|
||||
}
|
||||
manager := NewAgentTaskManager()
|
||||
task, _ := manager.StartTask("conv", "job", nil)
|
||||
ctx := manager.BindProcessScope(context.Background(), "conv", task.RunID)
|
||||
// Simulate an executor which has not yet reaped its direct child.
|
||||
session, err := security.StartShellSessionContext(ctx, exec.Command("sh", "-c", "sleep 300"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { session.Terminate(); _ = session.Wait(); manager.Shutdown() })
|
||||
if err := manager.FinishTaskRun("conv", task.RunID, "completed"); err == nil {
|
||||
t.Fatal("unreaped process reported as cleaned up")
|
||||
}
|
||||
snapshot := manager.GetTaskSnapshot("conv")
|
||||
if snapshot == nil || snapshot.Status != "cleanup_failed" || snapshot.CleanupError == "" {
|
||||
t.Fatalf("missing actionable cleanup state: %+v", snapshot)
|
||||
}
|
||||
if len(manager.GetCompletedTasks()) != 0 {
|
||||
t.Fatal("cleanup failure recorded as completed")
|
||||
}
|
||||
if _, err := manager.StartTask("conv", "new", nil); !errors.Is(err, ErrTaskAlreadyRunning) {
|
||||
t.Fatal("cleanup failure released conversation")
|
||||
}
|
||||
_ = session.Wait()
|
||||
manager.cleanupStuckCancelling()
|
||||
if manager.GetTaskSnapshot("conv") != nil {
|
||||
t.Fatal("cleanup retry did not finish reaped task")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskFinishWaitsForCancellationCallbacks(t *testing.T) {
|
||||
manager := NewAgentTaskManager()
|
||||
task, _ := manager.StartTask("conv", "job", nil)
|
||||
entered, release := make(chan struct{}), make(chan struct{})
|
||||
var once sync.Once
|
||||
manager.SetToolCanceler(func(string) { once.Do(func() { close(entered); <-release }) })
|
||||
cancelled := make(chan struct{})
|
||||
go func() { _, _ = manager.CancelTask("conv", ErrTaskCancelled); close(cancelled) }()
|
||||
<-entered
|
||||
finished := make(chan struct{})
|
||||
go func() { _ = manager.FinishTaskRun("conv", task.RunID, "cancelled"); close(finished) }()
|
||||
select {
|
||||
case <-finished:
|
||||
t.Error("task finished while old cancellation callbacks could still affect new run")
|
||||
case <-time.After(30 * time.Millisecond):
|
||||
}
|
||||
close(release)
|
||||
<-cancelled
|
||||
<-finished
|
||||
manager.Shutdown()
|
||||
}
|
||||
|
||||
func TestTaskWaitsForDetachedMCPWorker(t *testing.T) {
|
||||
manager := NewAgentTaskManager()
|
||||
defer manager.Shutdown()
|
||||
task, _ := manager.StartTask("conv", "job", nil)
|
||||
ctx := manager.BindProcessScope(context.Background(), "conv", task.RunID)
|
||||
service := mcp.NewExecutionService(nil, nil)
|
||||
entered, cancelled, release := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||
_, err := service.Submit(ctx, mcp.ExecutionRequest{Run: func(ctx context.Context) (*mcp.ToolResult, error) {
|
||||
close(entered)
|
||||
<-ctx.Done()
|
||||
close(cancelled)
|
||||
<-release
|
||||
return nil, ctx.Err()
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
<-entered
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- manager.FinishTaskRun("conv", task.RunID, "completed") }()
|
||||
<-cancelled
|
||||
if manager.GetTaskSnapshot("conv") == nil {
|
||||
t.Error("task released while detached worker still running")
|
||||
}
|
||||
close(release)
|
||||
if err = <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskReportsUnconfirmedRemoteCleanup(t *testing.T) {
|
||||
manager := NewAgentTaskManager()
|
||||
defer manager.Shutdown()
|
||||
task, _ := manager.StartTask("conv", "job", nil)
|
||||
ctx := manager.BindProcessScope(context.Background(), "conv", task.RunID)
|
||||
service := mcp.NewExecutionService(nil, nil)
|
||||
entered := make(chan struct{})
|
||||
_, err := service.Submit(ctx, mcp.ExecutionRequest{Remote: true, Run: func(ctx context.Context) (*mcp.ToolResult, error) {
|
||||
close(entered)
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
<-entered
|
||||
err = manager.FinishTaskRun("conv", task.RunID, "completed")
|
||||
if !errors.Is(err, runlease.ErrUnconfirmed) {
|
||||
t.Fatalf("remote cancellation reported as verified: %v", err)
|
||||
}
|
||||
history := manager.GetCompletedTasks()
|
||||
if len(history) != 1 || history[0].Status != "cleanup_unconfirmed" || history[0].CleanupError == "" {
|
||||
t.Fatalf("missing retained warning: %+v", history)
|
||||
}
|
||||
}
|
||||
@@ -55,9 +55,10 @@ func (h *AgentHandler) runRoleWorkflowStreamIfBound(
|
||||
|
||||
taskStatus := "completed"
|
||||
taskOwned := false
|
||||
var taskRunID string
|
||||
defer func() {
|
||||
if taskOwned {
|
||||
h.tasks.FinishTask(conversationID, taskStatus)
|
||||
h.tasks.FinishTaskRun(conversationID, taskRunID, taskStatus)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -69,7 +70,7 @@ func (h *AgentHandler) runRoleWorkflowStreamIfBound(
|
||||
taskCtx, timeoutCancel := context.WithTimeout(baseCtx, 600*time.Minute)
|
||||
defer timeoutCancel()
|
||||
|
||||
if _, err := h.tasks.StartTask(conversationID, userMessage, cancelWithCause); err != nil {
|
||||
if startedTask, err := h.tasks.StartTask(conversationID, userMessage, cancelWithCause); err != nil {
|
||||
var errorMsg string
|
||||
if errors.Is(err, ErrTaskAlreadyRunning) {
|
||||
errorMsg = "⚠️ 当前会话已有任务正在执行中,请等待当前任务完成或点击「停止任务」后再尝试。"
|
||||
@@ -86,8 +87,13 @@ func (h *AgentHandler) runRoleWorkflowStreamIfBound(
|
||||
}
|
||||
sendEvent("done", "", map[string]interface{}{"conversationId": conversationID})
|
||||
return true
|
||||
} else {
|
||||
taskRunID = startedTask.RunID
|
||||
}
|
||||
baseCtx = h.tasks.BindProcessScope(baseCtx, conversationID, taskRunID)
|
||||
taskCtx = h.tasks.BindProcessScope(taskCtx, conversationID, taskRunID)
|
||||
taskOwned = true
|
||||
sendEvent = h.taskFinishingEventSender(sendEvent, conversationID, taskRunID, func() string { return taskStatus })
|
||||
|
||||
progress := h.createProgressCallback(taskCtx, cancelWithCause, conversationID, assistantMessageID, sendEvent)
|
||||
result, err := workflowrunner.RunRoleBoundWorkflow(taskCtx, workflowrunner.RunArgs{
|
||||
@@ -202,9 +208,10 @@ func (h *AgentHandler) runRoleWorkflowJSONIfBound(c *gin.Context, req *ChatReque
|
||||
|
||||
taskStatus := "completed"
|
||||
taskOwned := false
|
||||
var taskRunID string
|
||||
defer func() {
|
||||
if taskOwned {
|
||||
h.tasks.FinishTask(conversationID, taskStatus)
|
||||
h.tasks.FinishTaskRun(conversationID, taskRunID, taskStatus)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -213,7 +220,7 @@ func (h *AgentHandler) runRoleWorkflowJSONIfBound(c *gin.Context, req *ChatReque
|
||||
taskCtx, timeoutCancel := context.WithTimeout(baseCtx, 600*time.Minute)
|
||||
defer timeoutCancel()
|
||||
|
||||
if _, err := h.tasks.StartTask(conversationID, userMessage, cancelWithCause); err != nil {
|
||||
if startedTask, err := h.tasks.StartTask(conversationID, userMessage, cancelWithCause); err != nil {
|
||||
if errors.Is(err, ErrTaskAlreadyRunning) {
|
||||
c.JSON(http.StatusConflict, gin.H{
|
||||
"error": "⚠️ 当前会话已有任务正在执行中,请等待当前任务完成或点击「停止任务」后再尝试。",
|
||||
@@ -224,8 +231,13 @@ func (h *AgentHandler) runRoleWorkflowJSONIfBound(c *gin.Context, req *ChatReque
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "❌ 无法启动任务: " + err.Error()})
|
||||
}
|
||||
return true
|
||||
} else {
|
||||
taskRunID = startedTask.RunID
|
||||
}
|
||||
baseCtx = h.tasks.BindProcessScope(baseCtx, conversationID, taskRunID)
|
||||
taskCtx = h.tasks.BindProcessScope(taskCtx, conversationID, taskRunID)
|
||||
taskOwned = true
|
||||
respond := h.taskFinishingJSONResponder(c, conversationID, taskRunID, func() string { return taskStatus })
|
||||
|
||||
progress := h.createProgressCallback(taskCtx, cancelWithCause, conversationID, assistantMessageID, nil)
|
||||
result, err := workflowrunner.RunRoleBoundWorkflow(taskCtx, workflowrunner.RunArgs{
|
||||
@@ -253,7 +265,7 @@ func (h *AgentHandler) runRoleWorkflowJSONIfBound(c *gin.Context, req *ChatReque
|
||||
_ = h.appendAssistantMessageNotice(assistantMessageID, cancelMsg)
|
||||
_ = h.db.AddProcessDetail(assistantMessageID, conversationID, "cancelled", cancelMsg, nil)
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
respond(http.StatusOK, gin.H{
|
||||
"status": "cancelled",
|
||||
"message": cancelMsg,
|
||||
"conversationId": conversationID,
|
||||
@@ -265,7 +277,7 @@ func (h *AgentHandler) runRoleWorkflowJSONIfBound(c *gin.Context, req *ChatReque
|
||||
if assistantMessageID != "" {
|
||||
_, _ = h.db.Exec("UPDATE messages SET content = ?, updated_at = ? WHERE id = ?", errMsg, time.Now(), assistantMessageID)
|
||||
}
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": errMsg, "conversationId": conversationID})
|
||||
respond(http.StatusInternalServerError, gin.H{"error": errMsg, "conversationId": conversationID})
|
||||
return true
|
||||
}
|
||||
decision := h.finalizeCandidateForDeliveryWithPolicy(
|
||||
@@ -283,7 +295,7 @@ func (h *AgentHandler) runRoleWorkflowJSONIfBound(c *gin.Context, req *ChatReque
|
||||
responseText = finalizationBlockedMessage(decision)
|
||||
taskStatus = decision.Status
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
respond(http.StatusOK, gin.H{
|
||||
"response": responseText,
|
||||
"conversationId": prep.ConversationID,
|
||||
"assistantMessageId": prep.AssistantMessageID,
|
||||
|
||||
Reference in New Issue
Block a user