feat: manage task process lifetimes and preserve turn history

This commit is contained in:
Ed1s0nZ
2026-09-16 17:52:58 +08:00
parent fd1c13a43d
commit f7882be546
54 changed files with 3650 additions and 330 deletions
@@ -93,6 +93,7 @@ func (m *modelFacingTraceMiddleware) BeforeModelRewriteState(
) (context.Context, *adk.ChatModelAgentState, error) {
if m.holder != nil && state != nil {
m.holder.storeFromState(state)
captureEinoTurnHistory(ctx, state.Messages)
}
return ctx, state, nil
}
@@ -119,6 +120,37 @@ func (m *agenticModelFacingTraceMiddleware) BeforeModelRewriteState(
) (context.Context, *adk.TypedChatModelAgentState[*schema.AgenticMessage], error) {
if m.holder != nil && state != nil {
m.holder.storeFromAgenticState(state)
captureEinoTurnHistory(ctx, AgenticMessagesToEino(state.Messages))
}
return ctx, state, nil
}
// Capture completed output separately from the model-input trace: changing
// Snapshot's meaning would affect last_react_input persistence and retries.
func (m *modelFacingTraceMiddleware) AfterModelRewriteState(ctx context.Context, state *adk.ChatModelAgentState, _ *adk.ModelContext) (context.Context, *adk.ChatModelAgentState, error) {
if state != nil {
captureEinoTurnHistory(ctx, state.Messages)
}
return ctx, state, nil
}
func (m *agenticModelFacingTraceMiddleware) AfterModelRewriteState(ctx context.Context, state *adk.TypedChatModelAgentState[*schema.AgenticMessage], _ *adk.TypedModelContext[*schema.AgenticMessage]) (context.Context, *adk.TypedChatModelAgentState[*schema.AgenticMessage], error) {
if state != nil {
captureEinoTurnHistory(ctx, AgenticMessagesToEino(state.Messages))
}
return ctx, state, nil
}
func (m *modelFacingTraceMiddleware) BeforeAgent(ctx context.Context, runCtx *adk.ChatModelAgentContext) (context.Context, *adk.ChatModelAgentContext, error) {
if runCtx != nil {
ctx = context.WithValue(ctx, einoTurnInstructionKey{}, runCtx.Instruction)
}
return ctx, runCtx, nil
}
func (m *agenticModelFacingTraceMiddleware) BeforeAgent(ctx context.Context, runCtx *adk.ChatModelAgentContext) (context.Context, *adk.ChatModelAgentContext, error) {
if runCtx != nil {
ctx = context.WithValue(ctx, einoTurnInstructionKey{}, runCtx.Instruction)
}
return ctx, runCtx, nil
}
+196
View File
@@ -0,0 +1,196 @@
package multiagent
import (
"context"
"strings"
"sync"
"github.com/cloudwego/eino/adk"
"github.com/cloudwego/eino/schema"
)
type einoTurnHistoryKey struct{}
type einoTurnInstructionKey struct{}
// Owned by one TurnLoop, never shared between conversations. Model state is
// authoritative after compaction; events are a fallback for agents without the
// trace middleware and supply tool results completed after the last snapshot.
type einoTurnHistory struct {
mu sync.Mutex
messages []*schema.Message
modelState bool
pending map[string]bool
events []*schema.Message
}
func (h *einoTurnHistory) begin(messages []*schema.Message) {
h.mu.Lock()
defer h.mu.Unlock()
h.messages = cloneSchemaMessages(messages)
h.modelState = false
h.events = nil
}
func captureEinoTurnHistory(ctx context.Context, messages []*schema.Message) {
h, _ := ctx.Value(einoTurnHistoryKey{}).(*einoTurnHistory)
if h == nil || len(messages) == 0 {
return
}
h.mu.Lock()
defer h.mu.Unlock()
// Remove only the instruction known to be regenerated on Run. Other system
// content may contain durable context and must not be indiscriminately dropped.
instruction, _ := ctx.Value(einoTurnInstructionKey{}).(string)
h.messages = nil
for _, msg := range cloneSchemaMessages(messages) {
if msg.Role == schema.System && instruction != "" {
if msg.Content == instruction {
continue
}
msg.Content = strings.TrimPrefix(msg.Content, instruction+"\n\n")
}
h.messages = append(h.messages, msg)
}
h.modelState = true
h.pending = make(map[string]bool)
for _, msg := range h.messages {
for _, call := range msg.ToolCalls {
h.pending[call.ID] = true
}
if msg.Role == schema.Tool {
delete(h.pending, msg.ToolCallID)
}
}
// Snapshots already contain completed results. Release raw event payloads as
// compaction advances instead of retaining another full transcript for long-running turns.
for i, msg := range h.events {
if msg != nil && (msg.Role != schema.Tool || !h.pending[msg.ToolCallID]) {
h.events[i] = nil
}
}
}
func (h *einoTurnHistory) nextInput() []*schema.Message {
h.mu.Lock()
defer h.mu.Unlock()
messages := cloneSchemaMessages(h.messages)
if !h.modelState {
messages = append(messages, cloneSchemaMessages(h.events)...)
} else {
// Never resurrect events discarded by summarization. Only pending calls in
// the authoritative state may acquire results from the event stream.
results := make(map[string]*schema.Message)
for _, msg := range h.events {
if msg != nil && msg.Role == schema.Tool {
results[msg.ToolCallID] = msg
}
}
var merged []*schema.Message
for i := 0; i < len(messages); i++ {
msg := messages[i]
merged = append(merged, msg)
if msg.Role != schema.Assistant || len(msg.ToolCalls) == 0 {
continue
}
present := make(map[string]bool)
for i+1 < len(messages) && messages[i+1].Role == schema.Tool {
i++
merged = append(merged, messages[i])
present[messages[i].ToolCallID] = true
}
for _, call := range msg.ToolCalls {
if !present[call.ID] && results[call.ID] != nil {
merged = append(merged, cloneSchemaMessages([]*schema.Message{results[call.ID]})...)
}
}
}
messages = merged
}
// Cancellation may leave a partial parallel tool batch. Explicit unknown
// results keep the protocol valid without claiming an unfinished call succeeded.
_, state, _ := newToolPairReconcilerMiddleware(nil, "turn_loop_continue").BeforeModelRewriteState(
context.Background(), &adk.ChatModelAgentState{Messages: messages}, nil)
return state.Messages
}
type einoTurnEventHandler func(context.Context, *adk.TurnContext[EinoTurnLoopItem, *schema.Message], *adk.AsyncIterator[*adk.AgentEvent]) error
func (h *einoTurnHistory) wrapEvents(handler einoTurnEventHandler) einoTurnEventHandler {
return func(ctx context.Context, tc *adk.TurnContext[EinoTurnLoopItem, *schema.Message], events *adk.AsyncIterator[*adk.AgentEvent]) error {
iter, gen := adk.NewAsyncIteratorPair[*adk.AgentEvent]()
done := make(chan struct{})
go func() {
defer close(done)
defer gen.Close()
var streams sync.WaitGroup
defer streams.Wait()
for {
ev, ok := events.Next()
if !ok {
return
}
if ev != nil && ev.Output != nil && ev.Output.MessageOutput != nil {
mv := ev.Output.MessageOutput
h.mu.Lock()
index := len(h.events)
h.events = append(h.events, nil)
h.mu.Unlock()
save := func(msg *schema.Message) {
if msg == nil {
return
}
h.mu.Lock()
if !h.modelState || (msg.Role == schema.Tool && h.pending[msg.ToolCallID]) {
h.events[index] = cloneSchemaMessages([]*schema.Message{msg})[0]
}
h.mu.Unlock()
}
if mv.IsStreaming && mv.MessageStream != nil {
copies := mv.MessageStream.Copy(2)
// Copy the event as well: the framework can retain its original event.
eventCopy, outputCopy, variantCopy := *ev, *ev.Output, *mv
variantCopy.MessageStream = copies[0]
outputCopy.MessageOutput = &variantCopy
eventCopy.Output = &outputCopy
ev = &eventCopy
streams.Add(1)
go func() {
defer streams.Done()
defer copies[1].Close()
msg, err := (&adk.MessageVariant{IsStreaming: true, MessageStream: copies[1]}).GetMessage()
if err == nil {
save(msg)
} // incomplete streams are not completed history
}()
} else {
save(mv.Message)
}
}
gen.Send(ev)
}
}()
var err error
if handler != nil {
err = handler(ctx, tc, iter)
}
// Drain even if the UI bridge returned early on voluntary cancellation.
// The next GenInput must not race asynchronous event/stream consumers.
for {
ev, ok := iter.Next()
if !ok {
break
}
if ev != nil && ev.Output != nil && ev.Output.MessageOutput != nil {
mv := ev.Output.MessageOutput
if mv.IsStreaming && mv.MessageStream != nil {
mv.MessageStream.Close()
}
}
if err == nil && ev != nil && ev.Err != nil && !isEinoVoluntaryCancelErr(ev.Err) {
err = ev.Err
}
}
<-done
return err
}
}
@@ -0,0 +1,254 @@
package multiagent
import (
"context"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/cloudwego/eino/adk"
"github.com/cloudwego/eino/components/model"
"github.com/cloudwego/eino/components/tool"
"github.com/cloudwego/eino/compose"
"github.com/cloudwego/eino/schema"
)
type historyTool struct{ calls atomic.Int32 }
func (h *historyTool) Info(context.Context) (*schema.ToolInfo, error) {
return &schema.ToolInfo{Name: "history_tool", Desc: "Record a completed test operation", ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{})}, nil
}
func (h *historyTool) InvokableRun(context.Context, string, ...tool.Option) (string, error) {
h.calls.Add(1)
return "completed-tool-evidence", nil
}
type historyModel struct {
mu sync.Mutex
inputs [][]*schema.Message
started chan int
releases [2]chan struct{}
}
func (m *historyModel) WithTools([]*schema.ToolInfo) (model.ToolCallingChatModel, error) {
return m, nil
}
func (m *historyModel) Generate(ctx context.Context, input []*schema.Message, _ ...model.Option) (*schema.Message, error) {
m.mu.Lock()
m.inputs = append(m.inputs, cloneSchemaMessages(input))
n := len(m.inputs)
m.mu.Unlock()
m.started <- n
if n == 1 {
return schema.AssistantMessage("work started", []schema.ToolCall{{ID: "completed-call", Type: "function", Function: schema.FunctionCall{Name: "history_tool", Arguments: "{}"}}}), nil
}
if n == 2 || n == 3 {
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-m.releases[n-2]:
}
}
return schema.AssistantMessage("completed-response", nil), nil
}
func (m *historyModel) Stream(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
msg, err := m.Generate(ctx, input, opts...)
if err != nil {
return nil, err
}
return schema.StreamReaderFromArray([]*schema.Message{msg}), nil
}
type historyCompactor struct {
adk.BaseChatModelAgentMiddleware
}
func (*historyCompactor) BeforeModelRewriteState(ctx context.Context, state *adk.ChatModelAgentState, _ *adk.ModelContext) (context.Context, *adk.ChatModelAgentState, error) {
hasResult := false
for _, m := range state.Messages {
hasResult = hasResult || m.Role == schema.Tool
}
if !hasResult {
return ctx, state, nil
}
out := *state
out.Messages = nil
for _, m := range state.Messages {
if m.Content == "old-verbose-history" {
summary := schema.UserMessage("compressed-progress-summary")
summary.Extra = map[string]any{"_eino_adk_summarization_content_type": "summary"}
out.Messages = append(out.Messages, summary)
} else {
out.Messages = append(out.Messages, m)
}
}
return ctx, &out, nil
}
func TestEinoTurnHistoryRetainsCompletedWorkAcrossInterrupts(t *testing.T) {
for _, safe := range []bool{false, true} {
for _, stream := range []bool{false, true} {
name := "timeout"
if safe {
name = "safe"
}
if stream {
name += "/stream"
}
t.Run(name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
m := &historyModel{started: make(chan int, 8), releases: [2]chan struct{}{make(chan struct{}), make(chan struct{})}}
operation := &historyTool{}
agent, err := adk.NewChatModelAgent(ctx, &adk.ChatModelAgentConfig{
Name: "history-agent", Instruction: "stable-agent-instruction", Model: m,
ToolsConfig: adk.ToolsConfig{ToolsNodeConfig: compose.ToolsNodeConfig{Tools: []tool.BaseTool{operation}}},
Handlers: []adk.ChatModelAgentMiddleware{&historyCompactor{}, newSystemMessageNormalizerMiddleware(nil, "test"), newModelFacingTraceMiddleware(newModelFacingTraceHolder())},
})
if err != nil {
t.Fatal(err)
}
timeout := 20 * time.Millisecond
if safe {
timeout = time.Second
}
runtime := NewEinoTurnLoopRuntime(EinoTurnLoopRuntimeConfig{Agent: agent, EnableStreaming: stream, InterruptTimeout: timeout, InitialMessages: []*schema.Message{schema.UserMessage("original-task"), schema.SystemMessage("durable-system-context"), schema.AssistantMessage("old-verbose-history", nil)}})
runtime.Run(ctx)
waitCall := func(want int) {
t.Helper()
select {
case n := <-m.started:
if n != want {
t.Fatalf("call %d, want %d", n, want)
}
case <-ctx.Done():
t.Fatal("model call timed out")
}
}
waitCall(1)
waitCall(2)
if !runtime.PushInterruptContinue("first-supplement") {
t.Fatal("push rejected")
}
if safe {
close(m.releases[0])
}
waitCall(3)
if !runtime.PushInterruptContinue("second-supplement") {
t.Fatal("push rejected")
}
if safe {
close(m.releases[1])
}
waitCall(4)
runtime.StopWhenIdle()
if state := runtime.Wait(); state.ExitReason != nil {
t.Fatal(state.ExitReason)
}
if operation.calls.Load() != 1 {
t.Fatalf("tool executed %d times", operation.calls.Load())
}
m.mu.Lock()
defer m.mu.Unlock()
for _, i := range []int{2, 3} {
input := m.inputs[i]
for _, marker := range []string{"original-task", "compressed-progress-summary", "completed-tool-evidence", "first-supplement", "durable-system-context", "stable-agent-instruction"} {
count := 0
for _, msg := range input {
count += strings.Count(msg.Content, marker)
}
if count != 1 {
t.Errorf("call %d: %q occurs %d times", i+1, marker, count)
}
}
for _, msg := range input {
if strings.Contains(msg.Content, "old-verbose-history") {
t.Error("compacted history resurrected")
}
}
if input[len(input)-1].Role != schema.User {
t.Error("supplement must be last user message")
}
if safe {
count := 0
for _, msg := range input {
if msg.Content == "completed-response" {
count++
}
}
if count != i-1 {
t.Errorf("completed responses=%d, want %d", count, i-1)
}
}
}
if !strings.Contains(m.inputs[3][len(m.inputs[3])-1].Content, "second-supplement") {
t.Error("second supplement lost")
}
})
}
}
}
func TestEinoTurnHistoryPendingToolBatch(t *testing.T) {
h := &einoTurnHistory{}
ctx := context.WithValue(context.Background(), einoTurnHistoryKey{}, h)
calls := []schema.ToolCall{{ID: "done", Function: schema.FunctionCall{Name: "tool"}}, {ID: "pending", Function: schema.FunctionCall{Name: "tool"}}}
captureEinoTurnHistory(ctx, []*schema.Message{schema.UserMessage("summary"), schema.AssistantMessage("", calls)})
h.events = []*schema.Message{schema.AssistantMessage("discarded-old-output", nil), schema.ToolMessage("actual-result", "done")}
got := h.nextInput()
if len(got) != 4 || got[2].Content != "actual-result" || got[3].Content != patchedMissingToolResult {
t.Fatalf("bad reconciled messages: %#v", got)
}
if got[2].ToolCallID != "done" || got[3].ToolCallID != "pending" {
t.Fatal("tool IDs lost")
}
}
func TestEinoTurnHistoryAgenticSnapshotAndIsolation(t *testing.T) {
first, second := &einoTurnHistory{}, &einoTurnHistory{}
first.begin([]*schema.Message{schema.UserMessage("first-task")})
second.begin([]*schema.Message{schema.UserMessage("second-task")})
ctx := context.WithValue(context.Background(), einoTurnHistoryKey{}, first)
mw := newAgenticModelFacingTraceMiddleware(newModelFacingTraceHolder())
ctx, _, err := mw.BeforeAgent(ctx, &adk.ChatModelAgentContext{Instruction: "agent-instruction"})
if err != nil {
t.Fatal(err)
}
state := &adk.TypedChatModelAgentState[*schema.AgenticMessage]{Messages: EinoMessagesToAgentic([]*schema.Message{
schema.SystemMessage("agent-instruction\n\nsystem-summary"), schema.UserMessage("compacted-first-task"),
})}
if _, _, err = mw.BeforeModelRewriteState(ctx, state, nil); err != nil {
t.Fatal(err)
}
state.Messages = append(state.Messages, EinoMessagesToAgentic([]*schema.Message{schema.AssistantMessage("finished-step", nil)})[0])
if _, _, err = mw.AfterModelRewriteState(ctx, state, nil); err != nil {
t.Fatal(err)
}
got := first.nextInput()
if len(got) != 3 || got[0].Content != "system-summary" || got[2].Content != "finished-step" {
t.Fatalf("agentic state lost: %#v", got)
}
other := second.nextInput()
if len(other) != 1 || other[0].Content != "second-task" {
t.Fatalf("conversation leaked: %#v", other)
}
}
func TestEinoTurnHistoryFallbackKeepsStreamedOutput(t *testing.T) {
h := &einoTurnHistory{}
h.begin([]*schema.Message{schema.UserMessage("initial-task")})
events, gen := adk.NewAsyncIteratorPair[*adk.AgentEvent]()
gen.Send(&adk.AgentEvent{Output: &adk.AgentOutput{MessageOutput: &adk.MessageVariant{
IsStreaming: true, MessageStream: schema.StreamReaderFromArray([]*schema.Message{schema.AssistantMessage("completed-", nil), schema.AssistantMessage("stream", nil)}),
}}})
gen.Close()
if err := h.wrapEvents(nil)(context.Background(), nil, events); err != nil {
t.Fatal(err)
}
got := h.nextInput()
if len(got) != 2 || got[1].Content != "completed-stream" {
t.Fatalf("stream history lost: %#v", got)
}
}
@@ -102,6 +102,9 @@ func TestRunEinoADKAgentLoopUsesTurnLoopInterruptPush(t *testing.T) {
t.Fatalf("model calls = %d, want at least 2", len(inputs))
}
last := inputs[len(inputs)-1]
if len(last) < 2 || last[0].Content != "initial task" {
t.Fatalf("initial task lost: %#v", last)
}
if len(last) == 0 || last[len(last)-1].Role != schema.User || last[len(last)-1].Content == "initial task" {
t.Fatalf("last model input = %#v, want interrupt supplement turn", last)
}
+10 -5
View File
@@ -49,6 +49,7 @@ func NewEinoTurnLoopRuntime(cfg EinoTurnLoopRuntimeConfig) *EinoTurnLoopRuntime
}
enableStreaming := cfg.EnableStreaming
prepareAgent := cfg.PrepareAgent
history := &einoTurnHistory{}
if prepareAgent == nil {
prepareAgent = func(context.Context, *adk.TurnLoop[EinoTurnLoopItem, *schema.Message], []EinoTurnLoopItem) (adk.Agent, error) {
return cfg.Agent, nil
@@ -58,9 +59,11 @@ func NewEinoTurnLoopRuntime(cfg EinoTurnLoopRuntimeConfig) *EinoTurnLoopRuntime
Store: cfg.Store,
CheckpointID: cfg.CheckpointID,
GenInput: func(ctx context.Context, _ *adk.TurnLoop[EinoTurnLoopItem, *schema.Message], items []EinoTurnLoopItem) (*adk.GenInputResult[EinoTurnLoopItem, *schema.Message], error) {
msgs := mergeEinoTurnLoopMessages(items)
msgs := append(history.nextInput(), mergeEinoTurnLoopMessages(items)...)
history = &einoTurnHistory{}
history.begin(msgs)
return &adk.GenInputResult[EinoTurnLoopItem, *schema.Message]{
RunCtx: ctx,
RunCtx: context.WithValue(ctx, einoTurnHistoryKey{}, history),
Input: &adk.AgentInput{
Messages: msgs,
EnableStreaming: enableStreaming,
@@ -74,13 +77,15 @@ func NewEinoTurnLoopRuntime(cfg EinoTurnLoopRuntimeConfig) *EinoTurnLoopRuntime
consumed = append(consumed, newItems...)
remaining := append([]EinoTurnLoopItem(nil), unhandledItems...)
return &adk.GenResumeResult[EinoTurnLoopItem, *schema.Message]{
RunCtx: ctx,
RunCtx: context.WithValue(ctx, einoTurnHistoryKey{}, history),
Consumed: consumed,
Remaining: remaining,
}, nil
},
PrepareAgent: prepareAgent,
OnAgentEvents: cfg.OnAgentEvents,
PrepareAgent: prepareAgent,
OnAgentEvents: func(ctx context.Context, tc *adk.TurnContext[EinoTurnLoopItem, *schema.Message], events *adk.AsyncIterator[*adk.AgentEvent]) error {
return history.wrapEvents(cfg.OnAgentEvents)(ctx, tc, events)
},
})
if len(cfg.InitialMessages) > 0 {
loop.Push(EinoTurnLoopItem{Kind: "initial", Messages: cloneSchemaMessages(cfg.InitialMessages)})
@@ -183,6 +183,9 @@ func TestEinoTurnLoopRuntimePushInterruptStartsNextTurn(t *testing.T) {
t.Fatalf("first input = %q, want initial task", got)
}
lastInput := inputs[len(inputs)-1]
if len(lastInput) < 2 || lastInput[0].Content != "initial task" {
t.Fatalf("initial history lost after preempt: %#v", lastInput)
}
if len(lastInput) == 0 || !strings.Contains(lastInput[len(lastInput)-1].Content, "focus on ssh") {
t.Fatalf("last input = %#v, want interrupt note", lastInput)
}