mirror of
https://github.com/Ed1s0nZ/CyberStrikeAI.git
synced 2026-10-18 21:47:17 +02:00
fix: distinguish task deadlines from child operation timeouts
This commit is contained in:
1 parent
93e8684571
commit
5ff781bdb7
7 files changed
+134
-4
No files matched your search
@@ -33,3 +33,9 @@ Replace the four task-level OR checks with this predicate. Child timeouts follow
|
||||
4. Commit and push each phase; merge only after checks pass and remote main is rechecked. No release or deployment requested.
|
||||
|
||||
Rollback: revert the fix commit; no data migration. Customer-specific fault attribution remains a separate evidence gap.
|
||||
|
||||
## F4 — Implementation
|
||||
|
||||
Added `taskDeadlineExceeded`, based on task context Err rather than operation error or cancellation cause. Single-agent, multi-agent, role workflow and batch task paths now use it. Existing terminal error details, trace persistence and retry policy are unchanged.
|
||||
|
||||
Regression proof: before the fix, `TestBatchTaskDeadlineClassification` failed for both child DeadlineExceeded and wrapped HTTP timeout (got timeout, wanted failed). After the fix, targeted tests pass, including real expiry, unrelated returned error after expiry, custom deadline causes and user cancellation.
|
||||
@@ -385,7 +385,7 @@ func (h *AgentHandler) handleBatchSubTaskRunError(
|
||||
strings.Contains(strings.ToLower(errStr), "context canceled") ||
|
||||
strings.Contains(strings.ToLower(errStr), "context cancelled") ||
|
||||
(partialResp != "" && (strings.Contains(partialResp, "任务已被取消") || strings.Contains(partialResp, "任务执行中断")))
|
||||
isTimeout := errors.Is(runErr, context.DeadlineExceeded) || errors.Is(context.Cause(taskCtx), context.DeadlineExceeded)
|
||||
isTimeout := taskDeadlineExceeded(taskCtx)
|
||||
|
||||
if isTimeout {
|
||||
*finishStatus = "timeout"
|
||||
|
||||
@@ -356,7 +356,7 @@ func (h *AgentHandler) EinoSingleAgentLoopStream(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
if errors.Is(runErr, context.DeadlineExceeded) || errors.Is(context.Cause(taskCtx), context.DeadlineExceeded) {
|
||||
if taskDeadlineExceeded(taskCtx) {
|
||||
taskStatus = "timeout"
|
||||
h.tasks.UpdateTaskStatus(conversationID, taskStatus)
|
||||
timeoutMsg := "任务执行超时,已自动终止。"
|
||||
|
||||
@@ -370,7 +370,7 @@ func (h *AgentHandler) MultiAgentLoopStream(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
if errors.Is(runErr, context.DeadlineExceeded) || errors.Is(context.Cause(taskCtx), context.DeadlineExceeded) {
|
||||
if taskDeadlineExceeded(taskCtx) {
|
||||
taskStatus = "timeout"
|
||||
h.tasks.UpdateTaskStatus(conversationID, taskStatus)
|
||||
timeoutMsg := "任务执行超时,已自动终止。"
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
)
|
||||
|
||||
// taskDeadlineExceeded distinguishes the task's deadline from a child operation's
|
||||
// timeout. A run error (including a cancellation cause) may wrap DeadlineExceeded
|
||||
// while the task itself still has time remaining; it must not exhaust the task budget.
|
||||
func taskDeadlineExceeded(taskCtx context.Context) bool {
|
||||
return taskCtx != nil && errors.Is(taskCtx.Err(), context.DeadlineExceeded)
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"cyberstrike-ai/internal/database"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func TestBatchTaskDeadlineClassification(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
expired bool
|
||||
runErr error
|
||||
want string
|
||||
}{
|
||||
{"child deadline", false, context.DeadlineExceeded, "failed"},
|
||||
{"wrapped HTTP timeout", false, &url.Error{Op: "Post", URL: "https://model.invalid", Err: fmt.Errorf("request timeout: %w", context.DeadlineExceeded)}, "failed"},
|
||||
{"task deadline", true, context.DeadlineExceeded, "timeout"},
|
||||
{"task deadline with unrelated error", true, errors.New("stream ended"), "timeout"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
logger := zap.NewNop()
|
||||
db, err := database.NewDB(filepath.Join(t.TempDir(), "deadline.db"), logger)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
conv, err := db.CreateConversation("deadline regression", database.ConversationCreateMeta{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
msg, err := db.AddMessage(conv.ID, "assistant", "", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manager := NewBatchTaskManager(logger)
|
||||
queue, err := manager.CreateBatchQueue("deadline", "", "eino_single", "manual", "", "", nil, 1, []string{"test"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.Now().Add(10 * time.Hour)
|
||||
if tc.expired {
|
||||
deadline = time.Now().Add(-time.Second)
|
||||
}
|
||||
taskCtx, cancel := context.WithDeadline(context.Background(), deadline)
|
||||
defer cancel()
|
||||
h := &AgentHandler{db: db, logger: logger, batchTaskManager: manager}
|
||||
status := "running"
|
||||
h.handleBatchSubTaskRunError(queue.ID, queue.Tasks[0], conv.ID, msg.ID, context.Background(), taskCtx, nil, tc.runErr, &status)
|
||||
if status != tc.want {
|
||||
t.Fatalf("finish status = %q, want %q", status, tc.want)
|
||||
}
|
||||
messages, err := db.GetMessages(conv.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(messages) != 1 || !strings.Contains(messages[0].Content, tc.runErr.Error()) {
|
||||
t.Fatalf("original diagnostic not preserved: %+v", messages)
|
||||
}
|
||||
updated, ok := manager.GetBatchQueue(queue.ID)
|
||||
if !ok || updated.Tasks[0].Status != BatchTaskStatusFailed {
|
||||
t.Fatalf("batch failure not recorded: %+v", updated)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskDeadlineExceededContextOwnership(t *testing.T) {
|
||||
taskCtx, stop := context.WithTimeout(context.Background(), 10*time.Hour)
|
||||
defer stop()
|
||||
child, stopChild := context.WithDeadline(taskCtx, time.Now().Add(-time.Second))
|
||||
defer stopChild()
|
||||
if !errors.Is(child.Err(), context.DeadlineExceeded) {
|
||||
t.Fatal("child deadline did not expire")
|
||||
}
|
||||
if taskDeadlineExceeded(taskCtx) {
|
||||
t.Fatal("child expiry exhausted parent task")
|
||||
}
|
||||
if !taskDeadlineExceeded(child) {
|
||||
t.Fatal("expired task deadline not recognized")
|
||||
}
|
||||
|
||||
cancelled, cancel := context.WithCancelCause(context.Background())
|
||||
cancel(fmt.Errorf("child failed: %w", context.DeadlineExceeded))
|
||||
if taskDeadlineExceeded(cancelled) {
|
||||
t.Fatal("cancellation cause misclassified as task deadline")
|
||||
}
|
||||
|
||||
userCtx, userCancel := context.WithCancelCause(context.Background())
|
||||
userCancel(ErrTaskCancelled)
|
||||
if taskDeadlineExceeded(userCtx) {
|
||||
t.Fatal("user cancellation misclassified as task deadline")
|
||||
}
|
||||
|
||||
customDeadline, stopCustom := context.WithDeadlineCause(context.Background(), time.Now().Add(-time.Second), errors.New("budget exhausted"))
|
||||
defer stopCustom()
|
||||
if !taskDeadlineExceeded(customDeadline) {
|
||||
t.Fatal("custom deadline cause hid real task expiry")
|
||||
}
|
||||
if taskDeadlineExceeded(nil) || taskDeadlineExceeded(context.Background()) {
|
||||
t.Fatal("context without deadline classified as expired")
|
||||
}
|
||||
}
|
||||
@@ -151,7 +151,7 @@ func (h *AgentHandler) runRoleWorkflowStreamIfBound(
|
||||
sendEvent("done", "", map[string]interface{}{"conversationId": conversationID})
|
||||
return true
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(context.Cause(taskCtx), context.DeadlineExceeded) {
|
||||
if taskDeadlineExceeded(taskCtx) {
|
||||
taskStatus = "timeout"
|
||||
h.tasks.UpdateTaskStatus(conversationID, taskStatus)
|
||||
timeoutMsg := "任务执行超时,已自动终止。"
|
||||
|
||||
Reference in new issue
Block a user