mirror of
https://github.com/Ed1s0nZ/CyberStrikeAI.git
synced 2026-08-15 15:40:38 +02:00
Add files via upload
This commit is contained in:
@@ -0,0 +1,435 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"cyberstrike-ai/internal/config"
|
||||
"cyberstrike-ai/internal/mcp/builtin"
|
||||
|
||||
localbk "github.com/cloudwego/eino-ext/adk/backend/local"
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"github.com/cloudwego/eino/adk/middlewares/dynamictool/toolsearch"
|
||||
"github.com/cloudwego/eino/adk/middlewares/patchtoolcalls"
|
||||
"github.com/cloudwego/eino/adk/middlewares/plantask"
|
||||
"github.com/cloudwego/eino/adk/middlewares/reduction"
|
||||
"github.com/cloudwego/eino/components/tool"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// einoMWPlacement controls which optional middleware runs on orchestrator vs sub-agents.
|
||||
type einoMWPlacement int
|
||||
|
||||
const (
|
||||
einoMWMain einoMWPlacement = iota // Deep / Supervisor main chat agent
|
||||
einoMWSub // Specialist ChatModelAgent
|
||||
)
|
||||
|
||||
func sanitizeEinoPathSegment(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return "default"
|
||||
}
|
||||
s = strings.ReplaceAll(s, string(filepath.Separator), "-")
|
||||
s = strings.ReplaceAll(s, "/", "-")
|
||||
s = strings.ReplaceAll(s, "\\", "-")
|
||||
s = strings.ReplaceAll(s, "..", "__")
|
||||
if len(s) > 180 {
|
||||
s = s[:180]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func splitToolsForToolSearch(all []tool.BaseTool, alwaysVisible int) (static []tool.BaseTool, dynamic []tool.BaseTool, ok bool) {
|
||||
if alwaysVisible <= 0 || len(all) <= alwaysVisible+1 {
|
||||
return all, nil, false
|
||||
}
|
||||
return append([]tool.BaseTool(nil), all[:alwaysVisible]...), append([]tool.BaseTool(nil), all[alwaysVisible:]...), true
|
||||
}
|
||||
|
||||
func splitToolsForToolSearchByNames(all []tool.BaseTool, names []string, fallbackAlwaysVisible int) (static []tool.BaseTool, dynamic []tool.BaseTool, ok bool) {
|
||||
nameSet := expandAlwaysVisibleNameSet(names)
|
||||
if len(nameSet) == 0 {
|
||||
return splitToolsForToolSearch(all, fallbackAlwaysVisible)
|
||||
}
|
||||
static = make([]tool.BaseTool, 0, len(all))
|
||||
dynamic = make([]tool.BaseTool, 0, len(all))
|
||||
for _, t := range all {
|
||||
if t == nil {
|
||||
continue
|
||||
}
|
||||
info, err := t.Info(context.Background())
|
||||
name := ""
|
||||
if err == nil && info != nil {
|
||||
name = info.Name
|
||||
}
|
||||
if toolMatchesAlwaysVisible(name, nameSet) {
|
||||
static = append(static, t)
|
||||
continue
|
||||
}
|
||||
dynamic = append(dynamic, t)
|
||||
}
|
||||
if len(static) == 0 || len(dynamic) == 0 {
|
||||
// fallback: preserve previous behavior when whitelist misses all or includes all.
|
||||
return splitToolsForToolSearch(all, fallbackAlwaysVisible)
|
||||
}
|
||||
return static, dynamic, true
|
||||
}
|
||||
|
||||
func mergeAlwaysVisibleToolNames(configured []string) []string {
|
||||
merged := make([]string, 0, len(configured)+32)
|
||||
seen := make(map[string]struct{}, len(configured)+32)
|
||||
add := func(name string) {
|
||||
n := strings.TrimSpace(strings.ToLower(name))
|
||||
if n == "" {
|
||||
return
|
||||
}
|
||||
if _, ok := seen[n]; ok {
|
||||
return
|
||||
}
|
||||
seen[n] = struct{}{}
|
||||
merged = append(merged, n)
|
||||
}
|
||||
for _, n := range configured {
|
||||
add(n)
|
||||
}
|
||||
// Always include hardcoded backend builtin MCP tools from constants.
|
||||
for _, n := range builtin.GetAllBuiltinTools() {
|
||||
add(n)
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
func reductionCacheRootDir(configuredBase, projectID, conversationID string) string {
|
||||
base := strings.TrimSpace(configuredBase)
|
||||
if base == "" {
|
||||
base = filepath.Join("tmp", "reduction")
|
||||
}
|
||||
if pid := strings.TrimSpace(projectID); pid != "" {
|
||||
return filepath.Join(base, "projects", sanitizeEinoPathSegment(pid))
|
||||
}
|
||||
conv := strings.TrimSpace(conversationID)
|
||||
if conv == "" {
|
||||
conv = "default"
|
||||
}
|
||||
return filepath.Join(base, "conversations", sanitizeEinoPathSegment(conv))
|
||||
}
|
||||
|
||||
func buildReductionMiddleware(ctx context.Context, mw config.MultiAgentEinoMiddlewareConfig, projectID, convID string, loc *localbk.Local, logger *zap.Logger) (adk.ChatModelAgentMiddleware, error) {
|
||||
if loc == nil {
|
||||
return nil, fmt.Errorf("reduction: local backend nil")
|
||||
}
|
||||
root := reductionCacheRootDir(mw.ReductionRootDir, projectID, convID)
|
||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("reduction root: %w", err)
|
||||
}
|
||||
excl := append([]string(nil), mw.ReductionClearExclude...)
|
||||
defaultExcl := []string{
|
||||
"task", "transfer_to_agent", "exit", "write_todos", "skill", "tool_search",
|
||||
"TaskCreate", "TaskGet", "TaskUpdate", "TaskList",
|
||||
}
|
||||
excl = append(excl, defaultExcl...)
|
||||
redMW, err := reduction.New(ctx, &reduction.Config{
|
||||
Backend: loc,
|
||||
RootDir: root,
|
||||
ReadFileToolName: "read_file",
|
||||
ClearExcludeTools: excl,
|
||||
MaxLengthForTrunc: mw.ReductionMaxLengthForTruncEffective(),
|
||||
MaxTokensForClear: int64(mw.ReductionMaxTokensForClearEffective()),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if logger != nil {
|
||||
logger.Info("eino middleware: reduction enabled", zap.String("root", root))
|
||||
}
|
||||
return redMW, nil
|
||||
}
|
||||
|
||||
func buildAgenticReductionMiddleware(
|
||||
ctx context.Context,
|
||||
mw config.MultiAgentEinoMiddlewareConfig,
|
||||
projectID, convID string,
|
||||
loc *localbk.Local,
|
||||
logger *zap.Logger,
|
||||
) (adk.TypedChatModelAgentMiddleware[*schema.AgenticMessage], error) {
|
||||
if loc == nil {
|
||||
return nil, fmt.Errorf("agentic reduction: local backend nil")
|
||||
}
|
||||
root := reductionCacheRootDir(mw.ReductionRootDir, projectID, convID)
|
||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("agentic reduction root: %w", err)
|
||||
}
|
||||
excl := append([]string(nil), mw.ReductionClearExclude...)
|
||||
defaultExcl := []string{
|
||||
"task", "transfer_to_agent", "exit", "write_todos", "skill", "tool_search",
|
||||
"TaskCreate", "TaskGet", "TaskUpdate", "TaskList",
|
||||
}
|
||||
excl = append(excl, defaultExcl...)
|
||||
redMW, err := reduction.NewTyped[*schema.AgenticMessage](ctx, &reduction.TypedConfig[*schema.AgenticMessage]{
|
||||
Backend: loc,
|
||||
RootDir: root,
|
||||
ReadFileToolName: "read_file",
|
||||
ClearExcludeTools: excl,
|
||||
MaxLengthForTrunc: mw.ReductionMaxLengthForTruncEffective(),
|
||||
MaxTokensForClear: int64(mw.ReductionMaxTokensForClearEffective()),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if logger != nil {
|
||||
logger.Info("eino middleware: agentic reduction enabled", zap.String("root", root))
|
||||
}
|
||||
return redMW, nil
|
||||
}
|
||||
|
||||
// prependEinoMiddlewares returns handlers to prepend (outermost first) and optionally replaces tools when tool_search is used.
|
||||
// toolSearchActive is true when the toolsearch middleware was mounted (dynamic tools split off); callers should pass this to
|
||||
// injectToolNamesOnlyInstruction — tool_search is not part of the pre-middleware tools list, so name-scanning alone cannot detect it.
|
||||
func prependEinoMiddlewares(
|
||||
ctx context.Context,
|
||||
mw *config.MultiAgentEinoMiddlewareConfig,
|
||||
place einoMWPlacement,
|
||||
tools []tool.BaseTool,
|
||||
einoLoc *localbk.Local,
|
||||
skillsRoot string,
|
||||
conversationID string,
|
||||
projectID string,
|
||||
logger *zap.Logger,
|
||||
) (outTools []tool.BaseTool, extraHandlers []adk.ChatModelAgentMiddleware, toolSearchActive bool, err error) {
|
||||
if mw == nil {
|
||||
return tools, nil, false, nil
|
||||
}
|
||||
outTools = tools
|
||||
|
||||
if mw.PatchToolCallsEffective() {
|
||||
patchMW, perr := patchtoolcalls.New(ctx, &patchtoolcalls.Config{})
|
||||
if perr != nil {
|
||||
return nil, nil, false, fmt.Errorf("patchtoolcalls: %w", perr)
|
||||
}
|
||||
extraHandlers = append(extraHandlers, patchMW)
|
||||
}
|
||||
|
||||
if mw.ReductionEnable && einoLoc != nil {
|
||||
if place == einoMWSub && !mw.ReductionSubAgents {
|
||||
// skip
|
||||
} else {
|
||||
redMW, rerr := buildReductionMiddleware(ctx, *mw, projectID, conversationID, einoLoc, logger)
|
||||
if rerr != nil {
|
||||
return nil, nil, false, rerr
|
||||
}
|
||||
extraHandlers = append(extraHandlers, redMW)
|
||||
}
|
||||
}
|
||||
|
||||
minTools := mw.ToolSearchMinTools
|
||||
if minTools <= 0 {
|
||||
minTools = 20
|
||||
}
|
||||
alwaysVis := mw.ToolSearchAlwaysVisible
|
||||
if alwaysVis <= 0 {
|
||||
alwaysVis = 12
|
||||
}
|
||||
if mw.ToolSearchEnable && len(tools) >= minTools {
|
||||
static, dynamic, split := splitToolsForToolSearchByNames(tools, mergeAlwaysVisibleToolNames(mw.ToolSearchAlwaysVisibleTools), alwaysVis)
|
||||
if split && len(dynamic) > 0 {
|
||||
ts, terr := toolsearch.New(ctx, &toolsearch.Config{DynamicTools: dynamic})
|
||||
if terr != nil {
|
||||
return nil, nil, false, fmt.Errorf("toolsearch: %w", terr)
|
||||
}
|
||||
extraHandlers = append(extraHandlers, ts)
|
||||
outTools = static
|
||||
toolSearchActive = true
|
||||
if logger != nil {
|
||||
logger.Info("eino middleware: tool_search enabled",
|
||||
zap.Int("static_tools", len(static)),
|
||||
zap.Int("dynamic_tools", len(dynamic)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if place == einoMWMain && mw.PlantaskEnable {
|
||||
if einoLoc == nil || strings.TrimSpace(skillsRoot) == "" {
|
||||
if logger != nil {
|
||||
logger.Warn("eino middleware: plantask_enable ignored (need eino_skills + skills_dir)")
|
||||
}
|
||||
} else {
|
||||
rel := strings.TrimSpace(mw.PlantaskRelDir)
|
||||
if rel == "" {
|
||||
rel = ".eino/plantask"
|
||||
}
|
||||
baseDir := filepath.Join(skillsRoot, rel, sanitizeEinoPathSegment(conversationID))
|
||||
if mk := os.MkdirAll(baseDir, 0o755); mk != nil {
|
||||
return nil, nil, toolSearchActive, fmt.Errorf("plantask mkdir: %w", mk)
|
||||
}
|
||||
ptBE := newLocalPlantaskBackend(einoLoc)
|
||||
pt, perr := plantask.New(ctx, &plantask.Config{Backend: ptBE, BaseDir: baseDir})
|
||||
if perr != nil {
|
||||
return nil, nil, toolSearchActive, fmt.Errorf("plantask: %w", perr)
|
||||
}
|
||||
extraHandlers = append(extraHandlers, pt)
|
||||
if logger != nil {
|
||||
logger.Info("eino middleware: plantask enabled", zap.String("baseDir", baseDir))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return outTools, extraHandlers, toolSearchActive, nil
|
||||
}
|
||||
|
||||
func prependEinoAgenticMiddlewares(
|
||||
ctx context.Context,
|
||||
mw *config.MultiAgentEinoMiddlewareConfig,
|
||||
place einoMWPlacement,
|
||||
tools []tool.BaseTool,
|
||||
einoLoc *localbk.Local,
|
||||
skillsRoot string,
|
||||
conversationID string,
|
||||
projectID string,
|
||||
logger *zap.Logger,
|
||||
) (outTools []tool.BaseTool, extraHandlers []adk.TypedChatModelAgentMiddleware[*schema.AgenticMessage], toolSearchActive bool, err error) {
|
||||
if mw == nil {
|
||||
return tools, nil, false, nil
|
||||
}
|
||||
outTools = tools
|
||||
|
||||
if mw.PatchToolCallsEffective() {
|
||||
patchMW, perr := patchtoolcalls.NewTyped[*schema.AgenticMessage](ctx, &patchtoolcalls.Config{})
|
||||
if perr != nil {
|
||||
return nil, nil, false, fmt.Errorf("agentic patchtoolcalls: %w", perr)
|
||||
}
|
||||
extraHandlers = append(extraHandlers, patchMW)
|
||||
}
|
||||
|
||||
if mw.ReductionEnable && einoLoc != nil {
|
||||
if place == einoMWSub && !mw.ReductionSubAgents {
|
||||
// skip
|
||||
} else {
|
||||
redMW, rerr := buildAgenticReductionMiddleware(ctx, *mw, projectID, conversationID, einoLoc, logger)
|
||||
if rerr != nil {
|
||||
return nil, nil, false, rerr
|
||||
}
|
||||
extraHandlers = append(extraHandlers, redMW)
|
||||
}
|
||||
}
|
||||
|
||||
minTools := mw.ToolSearchMinTools
|
||||
if minTools <= 0 {
|
||||
minTools = 20
|
||||
}
|
||||
alwaysVis := mw.ToolSearchAlwaysVisible
|
||||
if alwaysVis <= 0 {
|
||||
alwaysVis = 12
|
||||
}
|
||||
if mw.ToolSearchEnable && len(tools) >= minTools {
|
||||
static, dynamic, split := splitToolsForToolSearchByNames(tools, mergeAlwaysVisibleToolNames(mw.ToolSearchAlwaysVisibleTools), alwaysVis)
|
||||
if split && len(dynamic) > 0 {
|
||||
ts, terr := toolsearch.NewTyped[*schema.AgenticMessage](ctx, &toolsearch.Config{DynamicTools: dynamic})
|
||||
if terr != nil {
|
||||
return nil, nil, false, fmt.Errorf("agentic toolsearch: %w", terr)
|
||||
}
|
||||
extraHandlers = append(extraHandlers, ts)
|
||||
outTools = static
|
||||
toolSearchActive = true
|
||||
if logger != nil {
|
||||
logger.Info("eino middleware: agentic tool_search enabled",
|
||||
zap.Int("static_tools", len(static)),
|
||||
zap.Int("dynamic_tools", len(dynamic)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if place == einoMWMain && mw.PlantaskEnable {
|
||||
if einoLoc == nil || strings.TrimSpace(skillsRoot) == "" {
|
||||
if logger != nil {
|
||||
logger.Warn("eino middleware: agentic plantask_enable ignored (need eino_skills + skills_dir)")
|
||||
}
|
||||
} else {
|
||||
rel := strings.TrimSpace(mw.PlantaskRelDir)
|
||||
if rel == "" {
|
||||
rel = ".eino/plantask"
|
||||
}
|
||||
baseDir := filepath.Join(skillsRoot, rel, sanitizeEinoPathSegment(conversationID))
|
||||
if mk := os.MkdirAll(baseDir, 0o755); mk != nil {
|
||||
return nil, nil, toolSearchActive, fmt.Errorf("agentic plantask mkdir: %w", mk)
|
||||
}
|
||||
ptBE := newLocalPlantaskBackend(einoLoc)
|
||||
pt, perr := plantask.NewTyped[*schema.AgenticMessage](ctx, &plantask.Config{Backend: ptBE, BaseDir: baseDir})
|
||||
if perr != nil {
|
||||
return nil, nil, toolSearchActive, fmt.Errorf("agentic plantask: %w", perr)
|
||||
}
|
||||
extraHandlers = append(extraHandlers, pt)
|
||||
if logger != nil {
|
||||
logger.Info("eino middleware: agentic plantask enabled", zap.String("baseDir", baseDir))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return outTools, extraHandlers, toolSearchActive, nil
|
||||
}
|
||||
|
||||
func deepExtrasFromConfig(ma *config.MultiAgentConfig) (outputKey string, taskDesc func(context.Context, []adk.Agent) (string, error)) {
|
||||
if ma == nil {
|
||||
return "", nil
|
||||
}
|
||||
mw := ma.EinoMiddleware
|
||||
if k := strings.TrimSpace(mw.DeepOutputKey); k != "" {
|
||||
outputKey = k
|
||||
}
|
||||
prefix := strings.TrimSpace(mw.TaskToolDescriptionPrefix)
|
||||
if prefix != "" {
|
||||
taskDesc = func(ctx context.Context, agents []adk.Agent) (string, error) {
|
||||
_ = ctx
|
||||
var names []string
|
||||
for _, a := range agents {
|
||||
if a == nil {
|
||||
continue
|
||||
}
|
||||
n := strings.TrimSpace(a.Name(ctx))
|
||||
if n != "" {
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return prefix, nil
|
||||
}
|
||||
return prefix + "\n可用子代理(按名称 transfer / task 调用):" + strings.Join(names, "、"), nil
|
||||
}
|
||||
}
|
||||
return outputKey, taskDesc
|
||||
}
|
||||
|
||||
func deepAgenticExtrasFromConfig(ma *config.MultiAgentConfig) (outputKey string, taskDesc func(context.Context, []adk.TypedAgent[*schema.AgenticMessage]) (string, error)) {
|
||||
if ma == nil {
|
||||
return "", nil
|
||||
}
|
||||
mw := ma.EinoMiddleware
|
||||
if k := strings.TrimSpace(mw.DeepOutputKey); k != "" {
|
||||
outputKey = k
|
||||
}
|
||||
prefix := strings.TrimSpace(mw.TaskToolDescriptionPrefix)
|
||||
if prefix != "" {
|
||||
taskDesc = func(ctx context.Context, agents []adk.TypedAgent[*schema.AgenticMessage]) (string, error) {
|
||||
_ = ctx
|
||||
var names []string
|
||||
for _, a := range agents {
|
||||
if a == nil {
|
||||
continue
|
||||
}
|
||||
n := strings.TrimSpace(a.Name(ctx))
|
||||
if n != "" {
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return prefix, nil
|
||||
}
|
||||
return prefix + "\n可用子代理(按名称 transfer / task 调用):" + strings.Join(names, "、"), nil
|
||||
}
|
||||
}
|
||||
return outputKey, taskDesc
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"cyberstrike-ai/internal/config"
|
||||
|
||||
localbk "github.com/cloudwego/eino-ext/adk/backend/local"
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"github.com/cloudwego/eino/components/tool"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
func TestReductionCacheRootDir(t *testing.T) {
|
||||
got := reductionCacheRootDir("", "proj-1", "conv-1")
|
||||
want := filepath.Join("tmp", "reduction", "projects", "proj-1")
|
||||
if got != want {
|
||||
t.Fatalf("project scope: got %q want %q", got, want)
|
||||
}
|
||||
got = reductionCacheRootDir("", "", "conv-abc")
|
||||
want = filepath.Join("tmp", "reduction", "conversations", "conv-abc")
|
||||
if got != want {
|
||||
t.Fatalf("conversation scope: got %q want %q", got, want)
|
||||
}
|
||||
custom := reductionCacheRootDir("/data/cache", "p1", "c1")
|
||||
if !strings.HasSuffix(custom, filepath.Join("projects", "p1")) {
|
||||
t.Fatalf("custom base should still scope by project, got %q", custom)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildAgenticReductionMiddlewareClearsOldAgenticToolResult(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
loc, err := localbk.NewBackend(ctx, &localbk.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("NewBackend: %v", err)
|
||||
}
|
||||
root := t.TempDir()
|
||||
mw, err := buildAgenticReductionMiddleware(ctx, config.MultiAgentEinoMiddlewareConfig{
|
||||
ReductionRootDir: root,
|
||||
ReductionMaxTokensForClear: 1,
|
||||
}, "", "conv-1", loc, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("buildAgenticReductionMiddleware: %v", err)
|
||||
}
|
||||
oldText := strings.Repeat("old-tool-output-", 20)
|
||||
newText := strings.Repeat("new-tool-output-", 20)
|
||||
state := &adk.TypedChatModelAgentState[*schema.AgenticMessage]{
|
||||
Messages: []*schema.AgenticMessage{
|
||||
agenticAssistantToolCall("old-call", "execute", `{"command":"old"}`),
|
||||
agenticToolResult("old-call", "execute", oldText),
|
||||
agenticAssistantToolCall("new-call", "execute", `{"command":"new"}`),
|
||||
agenticToolResult("new-call", "execute", newText),
|
||||
},
|
||||
}
|
||||
_, out, err := mw.BeforeModelRewriteState(ctx, state, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("BeforeModelRewriteState: %v", err)
|
||||
}
|
||||
oldGot := out.Messages[1].ContentBlocks[0].FunctionToolResult.Content[0].Text.Text
|
||||
newGot := out.Messages[3].ContentBlocks[0].FunctionToolResult.Content[0].Text.Text
|
||||
if oldGot == oldText {
|
||||
t.Fatal("agentic reduction did not clear old oversized tool result")
|
||||
}
|
||||
if !strings.Contains(oldGot, "read_file") {
|
||||
t.Fatalf("cleared content should mention read_file, got %q", oldGot)
|
||||
}
|
||||
if newGot != newText {
|
||||
t.Fatalf("latest tool result should be retained, got %q", newGot)
|
||||
}
|
||||
}
|
||||
|
||||
func agenticAssistantToolCall(callID, name, arguments string) *schema.AgenticMessage {
|
||||
return &schema.AgenticMessage{
|
||||
Role: schema.AgenticRoleTypeAssistant,
|
||||
ContentBlocks: []*schema.ContentBlock{schema.NewContentBlock(&schema.FunctionToolCall{
|
||||
CallID: callID,
|
||||
Name: name,
|
||||
Arguments: arguments,
|
||||
})},
|
||||
}
|
||||
}
|
||||
|
||||
func agenticToolResult(callID, name, text string) *schema.AgenticMessage {
|
||||
return &schema.AgenticMessage{
|
||||
Role: schema.AgenticRoleTypeUser,
|
||||
ContentBlocks: []*schema.ContentBlock{schema.NewContentBlock(&schema.FunctionToolResult{
|
||||
CallID: callID,
|
||||
Name: name,
|
||||
Content: []*schema.FunctionToolResultContentBlock{{
|
||||
Type: schema.FunctionToolResultContentBlockTypeText,
|
||||
Text: &schema.UserInputText{Text: text},
|
||||
}},
|
||||
})},
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildAgenticReductionMiddlewareHandlesSingleAgenticToolResult(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
loc, err := localbk.NewBackend(ctx, &localbk.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("NewBackend: %v", err)
|
||||
}
|
||||
mw, err := buildAgenticReductionMiddleware(ctx, config.MultiAgentEinoMiddlewareConfig{
|
||||
ReductionRootDir: t.TempDir(),
|
||||
ReductionMaxTokensForClear: 1,
|
||||
}, "", "conv-1", loc, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("buildAgenticReductionMiddleware: %v", err)
|
||||
}
|
||||
state := &adk.TypedChatModelAgentState[*schema.AgenticMessage]{
|
||||
Messages: []*schema.AgenticMessage{
|
||||
{
|
||||
Role: schema.AgenticRoleTypeUser,
|
||||
ContentBlocks: []*schema.ContentBlock{schema.NewContentBlock(&schema.FunctionToolResult{
|
||||
CallID: "call-1",
|
||||
Name: "execute",
|
||||
Content: []*schema.FunctionToolResultContentBlock{{
|
||||
Type: schema.FunctionToolResultContentBlockTypeText,
|
||||
Text: &schema.UserInputText{Text: strings.Repeat("tool-output-", 20)},
|
||||
}},
|
||||
})},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, out, err := mw.BeforeModelRewriteState(ctx, state, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("BeforeModelRewriteState: %v", err)
|
||||
}
|
||||
got := out.Messages[0].ContentBlocks[0].FunctionToolResult.Content[0].Text.Text
|
||||
if got != strings.Repeat("tool-output-", 20) {
|
||||
t.Fatalf("single retained tool result should not be cleared, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrependEinoAgenticMiddlewaresRespectsReductionPlacement(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
loc, err := localbk.NewBackend(ctx, &localbk.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("NewBackend: %v", err)
|
||||
}
|
||||
patchToolCalls := false
|
||||
mw := &config.MultiAgentEinoMiddlewareConfig{
|
||||
ReductionEnable: true,
|
||||
ReductionRootDir: t.TempDir(),
|
||||
ReductionMaxTokensForClear: 100,
|
||||
PatchToolCalls: &patchToolCalls,
|
||||
}
|
||||
_, mainHandlers, _, err := prependEinoAgenticMiddlewares(ctx, mw, einoMWMain, nil, loc, "", "conv-1", "", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("prepend main: %v", err)
|
||||
}
|
||||
if len(mainHandlers) != 1 {
|
||||
t.Fatalf("main handlers = %d, want reduction", len(mainHandlers))
|
||||
}
|
||||
_, subHandlers, _, err := prependEinoAgenticMiddlewares(ctx, mw, einoMWSub, nil, loc, "", "conv-1", "", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("prepend sub: %v", err)
|
||||
}
|
||||
if len(subHandlers) != 0 {
|
||||
t.Fatalf("sub handlers = %d, want skipped when reduction_sub_agents=false", len(subHandlers))
|
||||
}
|
||||
mw.ReductionSubAgents = true
|
||||
_, subHandlers, _, err = prependEinoAgenticMiddlewares(ctx, mw, einoMWSub, nil, loc, "", "conv-1", "", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("prepend sub enabled: %v", err)
|
||||
}
|
||||
if len(subHandlers) != 1 {
|
||||
t.Fatalf("sub handlers = %d, want reduction when reduction_sub_agents=true", len(subHandlers))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrependEinoAgenticMiddlewaresMountsToolSearchAndPatchToolCalls(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mw := &config.MultiAgentEinoMiddlewareConfig{
|
||||
ToolSearchEnable: true,
|
||||
ToolSearchMinTools: 20,
|
||||
ToolSearchAlwaysVisible: 5,
|
||||
}
|
||||
outTools, handlers, toolSearchActive, err := prependEinoAgenticMiddlewares(ctx, mw, einoMWMain, stubTools(25), nil, "", "conv-test", "", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("prependEinoAgenticMiddlewares: %v", err)
|
||||
}
|
||||
if !toolSearchActive {
|
||||
t.Fatal("agentic tool_search should be active")
|
||||
}
|
||||
if len(outTools) != 5 {
|
||||
t.Fatalf("mounted tools = %d, want static visible tools only", len(outTools))
|
||||
}
|
||||
if len(handlers) != 2 {
|
||||
t.Fatalf("handlers = %d, want patchtoolcalls + toolsearch", len(handlers))
|
||||
}
|
||||
}
|
||||
|
||||
type stubTool struct{ name string }
|
||||
|
||||
func (s stubTool) Info(_ context.Context) (*schema.ToolInfo, error) {
|
||||
return &schema.ToolInfo{Name: s.name}, nil
|
||||
}
|
||||
|
||||
func TestSplitToolsForToolSearch(t *testing.T) {
|
||||
mk := func(n int) []tool.BaseTool {
|
||||
out := make([]tool.BaseTool, n)
|
||||
for i := 0; i < n; i++ {
|
||||
out[i] = stubTool{name: fmt.Sprintf("t%d", i)}
|
||||
}
|
||||
return out
|
||||
}
|
||||
static, dynamic, ok := splitToolsForToolSearch(mk(4), 3)
|
||||
if ok || len(static) != 4 || dynamic != nil {
|
||||
t.Fatalf("expected no split when len<=alwaysVisible+1, got ok=%v static=%d dynamic=%v", ok, len(static), dynamic)
|
||||
}
|
||||
static, dynamic, ok = splitToolsForToolSearch(mk(20), 5)
|
||||
if !ok || len(static) != 5 || len(dynamic) != 15 {
|
||||
t.Fatalf("expected split 5+15, got ok=%v static=%d dynamic=%d", ok, len(static), len(dynamic))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"cyberstrike-ai/internal/agent"
|
||||
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
func TestBuildEinoRunResultNeverPersistsRawAccumulationWithoutModelFacingTrace(t *testing.T) {
|
||||
raw := []schema.Message{*schema.ToolMessage(strings.Repeat("raw-tool-output", 1000), "call-1")}
|
||||
rawMsgs := make([]*schema.Message, len(raw))
|
||||
for i := range raw {
|
||||
rawMsgs[i] = &raw[i]
|
||||
}
|
||||
result := buildEinoRunResultFromAccumulated("deep", rawMsgs, nil, "", "", "empty", nil, true)
|
||||
if result.LastAgentTraceInput != "" {
|
||||
t.Fatalf("pre-model raw accumulation must not be persisted: %d bytes", len(result.LastAgentTraceInput))
|
||||
}
|
||||
|
||||
modelFacing := []*schema.Message{schema.UserMessage("bounded-model-view")}
|
||||
result = buildEinoRunResultFromAccumulated("deep", rawMsgs, modelFacing, "ok", "", "empty", nil, false)
|
||||
if !strings.Contains(result.LastAgentTraceInput, "bounded-model-view") {
|
||||
t.Fatalf("model-facing trace missing: %s", result.LastAgentTraceInput)
|
||||
}
|
||||
if strings.Contains(result.LastAgentTraceInput, "raw-tool-output") {
|
||||
t.Fatal("raw accumulation leaked into persisted model-facing trace")
|
||||
}
|
||||
if !agent.IsModelFacingTraceJSON(result.LastAgentTraceInput) {
|
||||
t.Fatal("persisted model-facing trace is missing its version marker")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
// modelFacingTraceHolder 保存「即将送入 ChatModel」的消息快照(已走 summarization / reduction / orphan 修剪等),
|
||||
// 用于 last_react_input 落库,使续跑与「上下文压缩后」的模型视角一致,而非仅依赖事件流 append 的 runAccumulatedMsgs。
|
||||
type modelFacingTraceHolder struct {
|
||||
mu sync.Mutex
|
||||
// msgs 为深拷贝后的切片,避免框架后续原地修改污染快照
|
||||
msgs []adk.Message
|
||||
}
|
||||
|
||||
func newModelFacingTraceHolder() *modelFacingTraceHolder {
|
||||
return &modelFacingTraceHolder{}
|
||||
}
|
||||
|
||||
// Snapshot 返回当前快照的再一次深拷贝(供序列化落库,避免与 holder 互斥长期持锁)。
|
||||
func (h *modelFacingTraceHolder) Snapshot() []adk.Message {
|
||||
if h == nil {
|
||||
return nil
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return cloneADKMessagesForTrace(h.msgs)
|
||||
}
|
||||
|
||||
func (h *modelFacingTraceHolder) storeFromState(state *adk.ChatModelAgentState) {
|
||||
if h == nil || state == nil || len(state.Messages) == 0 {
|
||||
return
|
||||
}
|
||||
cloned := cloneADKMessagesForTrace(state.Messages)
|
||||
if len(cloned) == 0 {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.msgs = cloned
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
func (h *modelFacingTraceHolder) storeFromAgenticState(state *adk.TypedChatModelAgentState[*schema.AgenticMessage]) {
|
||||
if h == nil || state == nil || len(state.Messages) == 0 {
|
||||
return
|
||||
}
|
||||
cloned := cloneADKMessagesForTrace(AgenticMessagesToEino(state.Messages))
|
||||
if len(cloned) == 0 {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.msgs = cloned
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
func cloneADKMessagesForTrace(msgs []adk.Message) []adk.Message {
|
||||
if len(msgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
b, err := json.Marshal(msgs)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []adk.Message
|
||||
if err := json.Unmarshal(b, &out); err != nil {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// modelFacingTraceMiddleware 必须在 Handlers 链中处于 **BeforeModel 最后**(telemetry 之后),
|
||||
// 此时 state.Messages 即为本次 LLM 调用的最终入参。
|
||||
type modelFacingTraceMiddleware struct {
|
||||
adk.BaseChatModelAgentMiddleware
|
||||
holder *modelFacingTraceHolder
|
||||
}
|
||||
|
||||
func newModelFacingTraceMiddleware(holder *modelFacingTraceHolder) adk.ChatModelAgentMiddleware {
|
||||
if holder == nil {
|
||||
return nil
|
||||
}
|
||||
return &modelFacingTraceMiddleware{holder: holder}
|
||||
}
|
||||
|
||||
func (m *modelFacingTraceMiddleware) BeforeModelRewriteState(
|
||||
ctx context.Context,
|
||||
state *adk.ChatModelAgentState,
|
||||
mc *adk.ModelContext,
|
||||
) (context.Context, *adk.ChatModelAgentState, error) {
|
||||
if m.holder != nil && state != nil {
|
||||
m.holder.storeFromState(state)
|
||||
}
|
||||
return ctx, state, nil
|
||||
}
|
||||
|
||||
type agenticModelFacingTraceMiddleware struct {
|
||||
*adk.TypedBaseChatModelAgentMiddleware[*schema.AgenticMessage]
|
||||
holder *modelFacingTraceHolder
|
||||
}
|
||||
|
||||
func newAgenticModelFacingTraceMiddleware(holder *modelFacingTraceHolder) adk.TypedChatModelAgentMiddleware[*schema.AgenticMessage] {
|
||||
if holder == nil {
|
||||
return nil
|
||||
}
|
||||
return &agenticModelFacingTraceMiddleware{
|
||||
TypedBaseChatModelAgentMiddleware: &adk.TypedBaseChatModelAgentMiddleware[*schema.AgenticMessage]{},
|
||||
holder: holder,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *agenticModelFacingTraceMiddleware) BeforeModelRewriteState(
|
||||
ctx context.Context,
|
||||
state *adk.TypedChatModelAgentState[*schema.AgenticMessage],
|
||||
mc *adk.TypedModelContext[*schema.AgenticMessage],
|
||||
) (context.Context, *adk.TypedChatModelAgentState[*schema.AgenticMessage], error) {
|
||||
if m.holder != nil && state != nil {
|
||||
m.holder.storeFromAgenticState(state)
|
||||
}
|
||||
return ctx, state, nil
|
||||
}
|
||||
@@ -0,0 +1,500 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"cyberstrike-ai/internal/config"
|
||||
"cyberstrike-ai/internal/openai"
|
||||
"cyberstrike-ai/internal/reasoning"
|
||||
|
||||
agenticopenai "github.com/cloudwego/eino-ext/components/model/agenticopenai"
|
||||
einoopenai "github.com/cloudwego/eino-ext/components/model/openai"
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"github.com/cloudwego/eino/components/model"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type einoModelMode string
|
||||
|
||||
const (
|
||||
einoModelModeNormal einoModelMode = "normal"
|
||||
einoModelModePlanner einoModelMode = "planner"
|
||||
)
|
||||
|
||||
type einoModelFactory func(ctx context.Context, oa config.OpenAIConfig, mode einoModelMode) (model.ToolCallingChatModel, error)
|
||||
type einoAgenticModelConfigFactory func(ctx context.Context, oa config.OpenAIConfig, mode einoModelMode) (model.AgenticModel, error)
|
||||
|
||||
func newEinoBaseHTTPClient() *http.Client {
|
||||
return &http.Client{
|
||||
Timeout: 30 * time.Minute,
|
||||
Transport: &http.Transport{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: 300 * time.Second,
|
||||
KeepAlive: 300 * time.Second,
|
||||
}).DialContext,
|
||||
MaxIdleConns: 100,
|
||||
MaxIdleConnsPerHost: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 30 * time.Second,
|
||||
ResponseHeaderTimeout: 60 * time.Minute,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newEinoOpenAIChatModelFactory(
|
||||
baseHTTPClient *http.Client,
|
||||
reasoningClient *reasoning.ClientIntent,
|
||||
logger *zap.Logger,
|
||||
) einoModelFactory {
|
||||
if baseHTTPClient == nil {
|
||||
baseHTTPClient = newEinoBaseHTTPClient()
|
||||
}
|
||||
return func(ctx context.Context, oa config.OpenAIConfig, mode einoModelMode) (model.ToolCallingChatModel, error) {
|
||||
httpClient := openai.NewEinoHTTPClient(&oa, baseHTTPClient)
|
||||
openai.AttachSummarizationDiagTransport(httpClient, logger)
|
||||
maxCompletionTokens := oa.MaxCompletionTokensEffective()
|
||||
modelCfg := &einoopenai.ChatModelConfig{
|
||||
APIKey: oa.APIKey,
|
||||
BaseURL: strings.TrimSuffix(oa.BaseURL, "/"),
|
||||
Model: oa.Model,
|
||||
HTTPClient: httpClient,
|
||||
MaxCompletionTokens: &maxCompletionTokens,
|
||||
}
|
||||
if mode == einoModelModePlanner {
|
||||
reasoning.ApplyPlanExecutePlannerModelConfig(modelCfg, &oa)
|
||||
} else {
|
||||
reasoning.ApplyToEinoChatModelConfig(modelCfg, &oa, reasoningClient)
|
||||
}
|
||||
baseModel, err := einoopenai.NewChatModel(ctx, modelCfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newStreamToolCallIndexRepairModel(baseModel), nil
|
||||
}
|
||||
}
|
||||
|
||||
func newEinoOpenAIAgenticChatModelFactory(
|
||||
baseHTTPClient *http.Client,
|
||||
reasoningClient *reasoning.ClientIntent,
|
||||
logger *zap.Logger,
|
||||
) einoAgenticModelConfigFactory {
|
||||
if baseHTTPClient == nil {
|
||||
baseHTTPClient = newEinoBaseHTTPClient()
|
||||
}
|
||||
return func(ctx context.Context, oa config.OpenAIConfig, mode einoModelMode) (model.AgenticModel, error) {
|
||||
if !supportsEinoAgenticOpenAIBackend(oa) {
|
||||
return nil, fmt.Errorf("eino agentic model: provider %q is not enabled for agenticopenai backend", strings.TrimSpace(oa.Provider))
|
||||
}
|
||||
httpClient := openai.NewEinoHTTPClient(&oa, baseHTTPClient)
|
||||
openai.AttachSummarizationDiagTransport(httpClient, logger)
|
||||
maxCompletionTokens := oa.MaxCompletionTokensEffective()
|
||||
modelCfg := &agenticopenai.ChatConfig{
|
||||
APIKey: oa.APIKey,
|
||||
BaseURL: strings.TrimSuffix(oa.BaseURL, "/"),
|
||||
Model: oa.Model,
|
||||
HTTPClient: httpClient,
|
||||
MaxCompletionTokens: &maxCompletionTokens,
|
||||
ExtraFields: reasoning.AgenticOpenAIExtraFields(&oa, reasoningClient),
|
||||
}
|
||||
if mode == einoModelModePlanner {
|
||||
modelCfg.ExtraFields = reasoning.AgenticOpenAIPlannerExtraFields(&oa)
|
||||
}
|
||||
return agenticopenai.NewChatModel(ctx, modelCfg)
|
||||
}
|
||||
}
|
||||
|
||||
func supportsEinoAgenticOpenAIBackend(oa config.OpenAIConfig) bool {
|
||||
provider := strings.ToLower(strings.TrimSpace(oa.Provider))
|
||||
return provider == "" || provider == "openai" || provider == "openai_compatible"
|
||||
}
|
||||
|
||||
func agenticModelGateFactory(factory einoAgenticModelConfigFactory, oa config.OpenAIConfig, mode einoModelMode) einoAgenticModelFactory {
|
||||
if factory == nil {
|
||||
return nil
|
||||
}
|
||||
return func(ctx context.Context) (model.AgenticModel, error) {
|
||||
return factory(ctx, oa, mode)
|
||||
}
|
||||
}
|
||||
|
||||
func newEinoModelRetryConfig(
|
||||
mw *config.MultiAgentEinoMiddlewareConfig,
|
||||
logger *zap.Logger,
|
||||
scope string,
|
||||
) *adk.ModelRetryConfig {
|
||||
maxRetries := RunRetryMaxAttemptsFromConfig(mw)
|
||||
maxBackoff := einoRunRetryMaxBackoffFromConfig(mw)
|
||||
return &adk.ModelRetryConfig{
|
||||
MaxRetries: maxRetries,
|
||||
BackoffFunc: func(_ context.Context, attempt int) time.Duration {
|
||||
return einoTransientRetryBackoff(attempt-1, maxBackoff)
|
||||
},
|
||||
ShouldRetry: func(ctx context.Context, retryCtx *adk.RetryContext) *adk.RetryDecision {
|
||||
if retryCtx == nil || ctx.Err() != nil {
|
||||
return &adk.RetryDecision{}
|
||||
}
|
||||
if retryCtx.Err != nil {
|
||||
if !isEinoTransientRunError(retryCtx.Err) {
|
||||
return &adk.RetryDecision{}
|
||||
}
|
||||
if logger != nil {
|
||||
kind, summary := einoTransientRunErrorUserDetail(retryCtx.Err)
|
||||
logger.Warn("eino native model retry",
|
||||
zap.String("scope", scope),
|
||||
zap.Int("attempt", retryCtx.RetryAttempt),
|
||||
zap.Int("maxRetries", maxRetries),
|
||||
zap.String("errorKind", kind),
|
||||
zap.String("errorSummary", summary),
|
||||
)
|
||||
}
|
||||
return &adk.RetryDecision{Retry: true, RejectReason: "transient_model_error"}
|
||||
}
|
||||
if isRetryableEmptyModelOutput(retryCtx.OutputMessage) {
|
||||
if logger != nil {
|
||||
logger.Warn("eino native model retry: empty model output",
|
||||
zap.String("scope", scope),
|
||||
zap.Int("attempt", retryCtx.RetryAttempt),
|
||||
zap.Int("maxRetries", maxRetries),
|
||||
)
|
||||
}
|
||||
return &adk.RetryDecision{Retry: true, RejectReason: "empty_model_output"}
|
||||
}
|
||||
return &adk.RetryDecision{}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newEinoAgenticModelRetryConfig(
|
||||
mw *config.MultiAgentEinoMiddlewareConfig,
|
||||
logger *zap.Logger,
|
||||
scope string,
|
||||
) *adk.TypedModelRetryConfig[*schema.AgenticMessage] {
|
||||
maxRetries := RunRetryMaxAttemptsFromConfig(mw)
|
||||
maxBackoff := einoRunRetryMaxBackoffFromConfig(mw)
|
||||
return &adk.TypedModelRetryConfig[*schema.AgenticMessage]{
|
||||
MaxRetries: maxRetries,
|
||||
BackoffFunc: func(_ context.Context, attempt int) time.Duration {
|
||||
return einoTransientRetryBackoff(attempt-1, maxBackoff)
|
||||
},
|
||||
ShouldRetry: func(ctx context.Context, retryCtx *adk.TypedRetryContext[*schema.AgenticMessage]) *adk.TypedRetryDecision[*schema.AgenticMessage] {
|
||||
if retryCtx == nil || ctx.Err() != nil {
|
||||
return &adk.TypedRetryDecision[*schema.AgenticMessage]{}
|
||||
}
|
||||
if retryCtx.Err != nil {
|
||||
if !isEinoTransientRunError(retryCtx.Err) {
|
||||
return &adk.TypedRetryDecision[*schema.AgenticMessage]{}
|
||||
}
|
||||
if logger != nil {
|
||||
kind, summary := einoTransientRunErrorUserDetail(retryCtx.Err)
|
||||
logger.Warn("eino native agentic model retry",
|
||||
zap.String("scope", scope),
|
||||
zap.Int("attempt", retryCtx.RetryAttempt),
|
||||
zap.Int("maxRetries", maxRetries),
|
||||
zap.String("errorKind", kind),
|
||||
zap.String("errorSummary", summary),
|
||||
)
|
||||
}
|
||||
return &adk.TypedRetryDecision[*schema.AgenticMessage]{Retry: true, RejectReason: "transient_model_error"}
|
||||
}
|
||||
if isRetryableEmptyAgenticModelOutput(retryCtx.OutputMessage) {
|
||||
if logger != nil {
|
||||
logger.Warn("eino native agentic model retry: empty model output",
|
||||
zap.String("scope", scope),
|
||||
zap.Int("attempt", retryCtx.RetryAttempt),
|
||||
zap.Int("maxRetries", maxRetries),
|
||||
)
|
||||
}
|
||||
return &adk.TypedRetryDecision[*schema.AgenticMessage]{Retry: true, RejectReason: "empty_model_output"}
|
||||
}
|
||||
return &adk.TypedRetryDecision[*schema.AgenticMessage]{}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newEinoModelFailoverConfig(
|
||||
ctx context.Context,
|
||||
appCfg *config.Config,
|
||||
mw *config.MultiAgentEinoMiddlewareConfig,
|
||||
mode einoModelMode,
|
||||
factory einoModelFactory,
|
||||
logger *zap.Logger,
|
||||
scope string,
|
||||
progress func(eventType, message string, data interface{}),
|
||||
orchestration string,
|
||||
conversationID string,
|
||||
) (*adk.ModelFailoverConfig[*schema.Message], error) {
|
||||
channels := resolveEinoFailoverChannels(appCfg, mw)
|
||||
if len(channels) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if factory == nil {
|
||||
return nil, fmt.Errorf("eino model failover: 模型工厂为空")
|
||||
}
|
||||
|
||||
maxRetries := len(channels)
|
||||
if mw != nil && mw.ModelFailoverMaxRetries > 0 && mw.ModelFailoverMaxRetries < maxRetries {
|
||||
maxRetries = mw.ModelFailoverMaxRetries
|
||||
}
|
||||
channels = channels[:maxRetries]
|
||||
|
||||
cache := make(map[string]model.BaseModel[*schema.Message], len(channels))
|
||||
var mu sync.Mutex
|
||||
return &adk.ModelFailoverConfig[*schema.Message]{
|
||||
MaxRetries: uint(maxRetries),
|
||||
ShouldFailover: func(ctx context.Context, _ *schema.Message, err error) bool {
|
||||
if ctx.Err() != nil || err == nil {
|
||||
return false
|
||||
}
|
||||
err = unwrapEinoRetryExhausted(err)
|
||||
return isEinoTransientRunError(err)
|
||||
},
|
||||
GetFailoverModel: func(ctx context.Context, failoverCtx *adk.FailoverContext[*schema.Message]) (model.BaseModel[*schema.Message], []*schema.Message, error) {
|
||||
if failoverCtx == nil || failoverCtx.FailoverAttempt == 0 {
|
||||
return nil, nil, fmt.Errorf("eino model failover: invalid failover attempt")
|
||||
}
|
||||
idx := int(failoverCtx.FailoverAttempt) - 1
|
||||
if idx < 0 || idx >= len(channels) {
|
||||
return nil, nil, fmt.Errorf("eino model failover: no channel for attempt %d", failoverCtx.FailoverAttempt)
|
||||
}
|
||||
ch := channels[idx]
|
||||
mu.Lock()
|
||||
cached := cache[ch.id]
|
||||
mu.Unlock()
|
||||
if cached != nil {
|
||||
emitEinoModelFailoverEvent(progress, conversationID, orchestration, scope, ch.id, ch.cfg.Model, failoverCtx.FailoverAttempt)
|
||||
if logger != nil {
|
||||
logger.Warn("eino native model failover",
|
||||
zap.String("scope", scope),
|
||||
zap.String("channel", ch.id),
|
||||
zap.String("model", ch.cfg.Model),
|
||||
zap.Uint("attempt", failoverCtx.FailoverAttempt),
|
||||
)
|
||||
}
|
||||
return cached, nil, nil
|
||||
}
|
||||
m, err := factory(ctx, ch.cfg, mode)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("eino model failover channel %q: %w", ch.id, err)
|
||||
}
|
||||
mu.Lock()
|
||||
cache[ch.id] = m
|
||||
mu.Unlock()
|
||||
emitEinoModelFailoverEvent(progress, conversationID, orchestration, scope, ch.id, ch.cfg.Model, failoverCtx.FailoverAttempt)
|
||||
if logger != nil {
|
||||
logger.Warn("eino native model failover",
|
||||
zap.String("scope", scope),
|
||||
zap.String("channel", ch.id),
|
||||
zap.String("model", ch.cfg.Model),
|
||||
zap.Uint("attempt", failoverCtx.FailoverAttempt),
|
||||
)
|
||||
}
|
||||
return m, nil, nil
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newEinoAgenticModelFailoverConfig(
|
||||
ctx context.Context,
|
||||
appCfg *config.Config,
|
||||
mw *config.MultiAgentEinoMiddlewareConfig,
|
||||
mode einoModelMode,
|
||||
factory einoAgenticModelConfigFactory,
|
||||
logger *zap.Logger,
|
||||
scope string,
|
||||
progress func(eventType, message string, data interface{}),
|
||||
orchestration string,
|
||||
conversationID string,
|
||||
) (*adk.ModelFailoverConfig[*schema.AgenticMessage], error) {
|
||||
channels := resolveEinoFailoverChannels(appCfg, mw)
|
||||
if len(channels) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if factory == nil {
|
||||
return nil, fmt.Errorf("eino agentic model failover: 模型工厂为空")
|
||||
}
|
||||
|
||||
maxRetries := len(channels)
|
||||
if mw != nil && mw.ModelFailoverMaxRetries > 0 && mw.ModelFailoverMaxRetries < maxRetries {
|
||||
maxRetries = mw.ModelFailoverMaxRetries
|
||||
}
|
||||
channels = channels[:maxRetries]
|
||||
|
||||
cache := make(map[string]model.BaseModel[*schema.AgenticMessage], len(channels))
|
||||
var mu sync.Mutex
|
||||
return &adk.ModelFailoverConfig[*schema.AgenticMessage]{
|
||||
MaxRetries: uint(maxRetries),
|
||||
ShouldFailover: func(ctx context.Context, _ *schema.AgenticMessage, err error) bool {
|
||||
if ctx.Err() != nil || err == nil {
|
||||
return false
|
||||
}
|
||||
err = unwrapEinoRetryExhausted(err)
|
||||
return isEinoTransientRunError(err)
|
||||
},
|
||||
GetFailoverModel: func(ctx context.Context, failoverCtx *adk.FailoverContext[*schema.AgenticMessage]) (model.BaseModel[*schema.AgenticMessage], []*schema.AgenticMessage, error) {
|
||||
if failoverCtx == nil || failoverCtx.FailoverAttempt == 0 {
|
||||
return nil, nil, fmt.Errorf("eino agentic model failover: invalid failover attempt")
|
||||
}
|
||||
idx := int(failoverCtx.FailoverAttempt) - 1
|
||||
if idx < 0 || idx >= len(channels) {
|
||||
return nil, nil, fmt.Errorf("eino agentic model failover: no channel for attempt %d", failoverCtx.FailoverAttempt)
|
||||
}
|
||||
ch := channels[idx]
|
||||
mu.Lock()
|
||||
cached := cache[ch.id]
|
||||
mu.Unlock()
|
||||
if cached != nil {
|
||||
emitEinoModelFailoverEvent(progress, conversationID, orchestration, scope, ch.id, ch.cfg.Model, failoverCtx.FailoverAttempt)
|
||||
if logger != nil {
|
||||
logger.Warn("eino native agentic model failover",
|
||||
zap.String("scope", scope),
|
||||
zap.String("channel", ch.id),
|
||||
zap.String("model", ch.cfg.Model),
|
||||
zap.Uint("attempt", failoverCtx.FailoverAttempt),
|
||||
)
|
||||
}
|
||||
return cached, nil, nil
|
||||
}
|
||||
m, err := factory(ctx, ch.cfg, mode)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("eino agentic model failover channel %q: %w", ch.id, err)
|
||||
}
|
||||
mu.Lock()
|
||||
cache[ch.id] = m
|
||||
mu.Unlock()
|
||||
emitEinoModelFailoverEvent(progress, conversationID, orchestration, scope, ch.id, ch.cfg.Model, failoverCtx.FailoverAttempt)
|
||||
if logger != nil {
|
||||
logger.Warn("eino native agentic model failover",
|
||||
zap.String("scope", scope),
|
||||
zap.String("channel", ch.id),
|
||||
zap.String("model", ch.cfg.Model),
|
||||
zap.Uint("attempt", failoverCtx.FailoverAttempt),
|
||||
)
|
||||
}
|
||||
return m, nil, nil
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type resolvedEinoFailoverChannel struct {
|
||||
id string
|
||||
cfg config.OpenAIConfig
|
||||
}
|
||||
|
||||
func resolveEinoFailoverChannels(appCfg *config.Config, mw *config.MultiAgentEinoMiddlewareConfig) []resolvedEinoFailoverChannel {
|
||||
if appCfg == nil || mw == nil || len(mw.ModelFailoverChannels) == 0 {
|
||||
return nil
|
||||
}
|
||||
primary := appCfg.OpenAI
|
||||
seen := map[string]struct{}{}
|
||||
out := make([]resolvedEinoFailoverChannel, 0, len(mw.ModelFailoverChannels))
|
||||
for _, raw := range mw.ModelFailoverChannels {
|
||||
id := config.NormalizeAIChannelID(raw)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
oa, resolvedID, ok := appCfg.AI.ResolveChannel(id)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if sameOpenAIModelEndpoint(primary, oa) {
|
||||
continue
|
||||
}
|
||||
seen[resolvedID] = struct{}{}
|
||||
out = append(out, resolvedEinoFailoverChannel{id: resolvedID, cfg: oa})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sameOpenAIModelEndpoint(a, b config.OpenAIConfig) bool {
|
||||
return strings.EqualFold(strings.TrimSpace(a.Provider), strings.TrimSpace(b.Provider)) &&
|
||||
strings.TrimRight(strings.TrimSpace(a.BaseURL), "/") == strings.TrimRight(strings.TrimSpace(b.BaseURL), "/") &&
|
||||
strings.TrimSpace(a.APIKey) == strings.TrimSpace(b.APIKey) &&
|
||||
strings.TrimSpace(a.Model) == strings.TrimSpace(b.Model)
|
||||
}
|
||||
|
||||
func isRetryableEmptyModelOutput(msg *schema.Message) bool {
|
||||
if msg == nil {
|
||||
return true
|
||||
}
|
||||
return strings.TrimSpace(msg.Content) == "" &&
|
||||
strings.TrimSpace(msg.ReasoningContent) == "" &&
|
||||
len(msg.ToolCalls) == 0 &&
|
||||
len(msg.MultiContent) == 0 &&
|
||||
len(msg.UserInputMultiContent) == 0 &&
|
||||
len(msg.AssistantGenMultiContent) == 0
|
||||
}
|
||||
|
||||
func isRetryableEmptyAgenticModelOutput(msg *schema.AgenticMessage) bool {
|
||||
if msg == nil {
|
||||
return true
|
||||
}
|
||||
for _, block := range msg.ContentBlocks {
|
||||
if block == nil {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case block.Reasoning != nil:
|
||||
if strings.TrimSpace(block.Reasoning.Text) != "" {
|
||||
return false
|
||||
}
|
||||
case block.UserInputText != nil:
|
||||
if strings.TrimSpace(block.UserInputText.Text) != "" {
|
||||
return false
|
||||
}
|
||||
case block.AssistantGenText != nil:
|
||||
if strings.TrimSpace(block.AssistantGenText.Text) != "" {
|
||||
return false
|
||||
}
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func unwrapEinoRetryExhausted(err error) error {
|
||||
var retryErr *adk.RetryExhaustedError
|
||||
if errors.As(err, &retryErr) && retryErr.LastErr != nil {
|
||||
return retryErr.LastErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func isEinoNativeWillRetry(err error) (*adk.WillRetryError, bool) {
|
||||
var willRetry *adk.WillRetryError
|
||||
if errors.As(err, &willRetry) {
|
||||
return willRetry, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func emitEinoModelFailoverEvent(
|
||||
progress func(eventType, message string, data interface{}),
|
||||
conversationID, orchestration, scope, channelID, modelName string,
|
||||
attempt uint,
|
||||
) {
|
||||
if progress == nil {
|
||||
return
|
||||
}
|
||||
msg := fmt.Sprintf("主模型重试耗尽,正在切换备用模型 %s。", modelName)
|
||||
progress("eino_model_failover", msg, map[string]interface{}{
|
||||
"conversationId": conversationID,
|
||||
"source": "eino",
|
||||
"orchestration": orchestration,
|
||||
"scope": scope,
|
||||
"channel": channelID,
|
||||
"model": modelName,
|
||||
"attempt": attempt,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,376 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"cyberstrike-ai/internal/config"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"github.com/cloudwego/eino/components/model"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
func TestNewEinoModelRetryConfigUsesNativeFieldsFirst(t *testing.T) {
|
||||
t.Parallel()
|
||||
mw := &config.MultiAgentEinoMiddlewareConfig{
|
||||
ModelRetryMaxRetries: 2,
|
||||
ModelRetryMaxBackoffSec: 7,
|
||||
RunRetryMaxAttempts: 9,
|
||||
RunRetryMaxBackoffSec: 11,
|
||||
}
|
||||
cfg := newEinoModelRetryConfig(mw, nil, "test")
|
||||
if cfg.MaxRetries != 2 {
|
||||
t.Fatalf("MaxRetries = %d, want 2", cfg.MaxRetries)
|
||||
}
|
||||
backoff := cfg.BackoffFunc(context.Background(), 1)
|
||||
if backoff < 500*time.Millisecond || backoff > 2*time.Second {
|
||||
t.Fatalf("attempt 1 backoff = %v, want first equal-jitter window", backoff)
|
||||
}
|
||||
if got := einoRunRetryMaxBackoffFromConfig(mw); got != 7*time.Second {
|
||||
t.Fatalf("backoff from config = %v, want 7s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoModelRetryPolicyRetriesTransientAndEmptyOutput(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := newEinoModelRetryConfig(&config.MultiAgentEinoMiddlewareConfig{ModelRetryMaxRetries: 1}, nil, "test")
|
||||
if got := cfg.ShouldRetry(context.Background(), &adk.RetryContext{Err: errors.New("HTTP 429 Too Many Requests")}); got == nil || !got.Retry {
|
||||
t.Fatal("transient model error should retry")
|
||||
}
|
||||
if got := cfg.ShouldRetry(context.Background(), &adk.RetryContext{OutputMessage: schema.AssistantMessage("", nil)}); got == nil || !got.Retry {
|
||||
t.Fatal("empty assistant output should retry")
|
||||
}
|
||||
if got := cfg.ShouldRetry(context.Background(), &adk.RetryContext{OutputMessage: schema.AssistantMessage("", []schema.ToolCall{{ID: "call_1"}})}); got == nil || got.Retry {
|
||||
t.Fatal("assistant tool call output should not be treated as empty")
|
||||
}
|
||||
if got := cfg.ShouldRetry(context.Background(), &adk.RetryContext{Err: errors.New("invalid api key")}); got == nil || got.Retry {
|
||||
t.Fatal("permanent auth error should not retry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoAgenticModelRetryPolicyRetriesTransientAndEmptyOutput(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := newEinoAgenticModelRetryConfig(&config.MultiAgentEinoMiddlewareConfig{ModelRetryMaxRetries: 1}, nil, "agentic")
|
||||
if got := cfg.ShouldRetry(context.Background(), &adk.TypedRetryContext[*schema.AgenticMessage]{Err: errors.New("HTTP 429 Too Many Requests")}); got == nil || !got.Retry {
|
||||
t.Fatal("transient agentic model error should retry")
|
||||
}
|
||||
if got := cfg.ShouldRetry(context.Background(), &adk.TypedRetryContext[*schema.AgenticMessage]{
|
||||
OutputMessage: &schema.AgenticMessage{Role: schema.AgenticRoleTypeAssistant},
|
||||
}); got == nil || !got.Retry {
|
||||
t.Fatal("empty agentic assistant output should retry")
|
||||
}
|
||||
if got := cfg.ShouldRetry(context.Background(), &adk.TypedRetryContext[*schema.AgenticMessage]{
|
||||
OutputMessage: &schema.AgenticMessage{
|
||||
Role: schema.AgenticRoleTypeAssistant,
|
||||
ContentBlocks: []*schema.ContentBlock{schema.NewContentBlock(&schema.AssistantGenText{Text: "ok"})},
|
||||
},
|
||||
}); got == nil || got.Retry {
|
||||
t.Fatal("agentic assistant text should not be treated as empty")
|
||||
}
|
||||
if got := cfg.ShouldRetry(context.Background(), &adk.TypedRetryContext[*schema.AgenticMessage]{
|
||||
OutputMessage: &schema.AgenticMessage{
|
||||
Role: schema.AgenticRoleTypeAssistant,
|
||||
ContentBlocks: []*schema.ContentBlock{schema.NewContentBlock(&schema.FunctionToolCall{
|
||||
CallID: "call_1", Name: "search", Arguments: `{"q":"x"}`,
|
||||
})},
|
||||
},
|
||||
}); got == nil || got.Retry {
|
||||
t.Fatal("agentic tool call output should not be treated as empty")
|
||||
}
|
||||
if got := cfg.ShouldRetry(context.Background(), &adk.TypedRetryContext[*schema.AgenticMessage]{Err: errors.New("invalid api key")}); got == nil || got.Retry {
|
||||
t.Fatal("permanent auth error should not retry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveEinoFailoverChannelsSkipsPrimaryDuplicateAndUnknown(t *testing.T) {
|
||||
t.Parallel()
|
||||
appCfg := &config.Config{
|
||||
OpenAI: config.OpenAIConfig{Provider: "openai", APIKey: "k1", BaseURL: "https://api.example/v1", Model: "primary"},
|
||||
AI: config.AIConfig{Channels: map[string]config.AIChannelConfig{
|
||||
"same": {Provider: "openai", APIKey: "k1", BaseURL: "https://api.example/v1", Model: "primary"},
|
||||
"fb1": {Provider: "openai", APIKey: "k2", BaseURL: "https://api.example/v1", Model: "fallback-1"},
|
||||
"fb2": {Provider: "claude", APIKey: "k3", BaseURL: "https://api.anthropic.com/v1", Model: "claude-sonnet"},
|
||||
}},
|
||||
}
|
||||
got := resolveEinoFailoverChannels(appCfg, &config.MultiAgentEinoMiddlewareConfig{
|
||||
ModelFailoverChannels: []string{"same", "missing", "fb1", "fb1", "fb2"},
|
||||
ModelFailoverMaxRetries: 1,
|
||||
})
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("resolved channels len = %d, want 2 before max cap is applied by config builder", len(got))
|
||||
}
|
||||
if got[0].id != "fb1" || got[1].id != "fb2" {
|
||||
t.Fatalf("resolved channel order = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewEinoModelFailoverConfigBuildsDistinctFallbackModel(t *testing.T) {
|
||||
t.Parallel()
|
||||
appCfg := &config.Config{
|
||||
OpenAI: config.OpenAIConfig{APIKey: "k1", BaseURL: "https://api.example/v1", Model: "primary"},
|
||||
AI: config.AIConfig{Channels: map[string]config.AIChannelConfig{
|
||||
"fb1": {APIKey: "k2", BaseURL: "https://api.example/v1", Model: "fallback-1"},
|
||||
"fb2": {APIKey: "k3", BaseURL: "https://api.example/v1", Model: "fallback-2"},
|
||||
}},
|
||||
}
|
||||
var built []string
|
||||
cfg, err := newEinoModelFailoverConfig(
|
||||
context.Background(),
|
||||
appCfg,
|
||||
&config.MultiAgentEinoMiddlewareConfig{
|
||||
ModelFailoverChannels: []string{"fb1", "fb2"},
|
||||
ModelFailoverMaxRetries: 1,
|
||||
},
|
||||
einoModelModeNormal,
|
||||
func(_ context.Context, oa config.OpenAIConfig, _ einoModelMode) (model.ToolCallingChatModel, error) {
|
||||
built = append(built, oa.Model)
|
||||
return &streamToolCallIndexFakeModel{}, nil
|
||||
},
|
||||
nil,
|
||||
"test",
|
||||
nil,
|
||||
"deep",
|
||||
"conv-1",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newEinoModelFailoverConfig: %v", err)
|
||||
}
|
||||
if cfg == nil || cfg.MaxRetries != 1 {
|
||||
t.Fatalf("failover cfg = %#v, want max retries 1", cfg)
|
||||
}
|
||||
m, msgs, err := cfg.GetFailoverModel(context.Background(), &adk.FailoverContext[*schema.Message]{FailoverAttempt: 1})
|
||||
if err != nil || m == nil || msgs != nil {
|
||||
t.Fatalf("GetFailoverModel = (%v, %v, %v)", m, msgs, err)
|
||||
}
|
||||
if len(built) != 1 || built[0] != "fallback-1" {
|
||||
t.Fatalf("built models = %v, want [fallback-1]", built)
|
||||
}
|
||||
if !cfg.ShouldFailover(context.Background(), nil, &adk.RetryExhaustedError{LastErr: errors.New("upstream returned 503"), TotalRetries: 4}) {
|
||||
t.Fatal("retry-exhausted transient error should fail over")
|
||||
}
|
||||
if cfg.ShouldFailover(context.Background(), nil, &adk.RetryExhaustedError{LastErr: errors.New("invalid api key"), TotalRetries: 4}) {
|
||||
t.Fatal("retry-exhausted permanent error should not fail over")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewEinoModelFailoverConfigEmitsProgressEvent(t *testing.T) {
|
||||
t.Parallel()
|
||||
appCfg := &config.Config{
|
||||
OpenAI: config.OpenAIConfig{APIKey: "k1", BaseURL: "https://api.example/v1", Model: "primary"},
|
||||
AI: config.AIConfig{Channels: map[string]config.AIChannelConfig{
|
||||
"fb1": {APIKey: "k2", BaseURL: "https://api.example/v1", Model: "fallback-1"},
|
||||
}},
|
||||
}
|
||||
var events []struct {
|
||||
eventType string
|
||||
message string
|
||||
data interface{}
|
||||
}
|
||||
cfg, err := newEinoModelFailoverConfig(
|
||||
context.Background(),
|
||||
appCfg,
|
||||
&config.MultiAgentEinoMiddlewareConfig{ModelFailoverChannels: []string{"fb1"}},
|
||||
einoModelModeNormal,
|
||||
func(_ context.Context, _ config.OpenAIConfig, _ einoModelMode) (model.ToolCallingChatModel, error) {
|
||||
return &streamToolCallIndexFakeModel{}, nil
|
||||
},
|
||||
nil,
|
||||
"test",
|
||||
func(eventType, message string, data interface{}) {
|
||||
events = append(events, struct {
|
||||
eventType string
|
||||
message string
|
||||
data interface{}
|
||||
}{eventType: eventType, message: message, data: data})
|
||||
},
|
||||
"deep",
|
||||
"conv-1",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newEinoModelFailoverConfig: %v", err)
|
||||
}
|
||||
if _, _, err := cfg.GetFailoverModel(context.Background(), &adk.FailoverContext[*schema.Message]{FailoverAttempt: 1}); err != nil {
|
||||
t.Fatalf("GetFailoverModel: %v", err)
|
||||
}
|
||||
if len(events) != 1 || events[0].eventType != "eino_model_failover" {
|
||||
t.Fatalf("events = %#v, want one eino_model_failover", events)
|
||||
}
|
||||
payload, ok := events[0].data.(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("event payload type = %T", events[0].data)
|
||||
}
|
||||
if payload["conversationId"] != "conv-1" || payload["orchestration"] != "deep" || payload["channel"] != "fb1" || payload["model"] != "fallback-1" {
|
||||
t.Fatalf("payload = %#v", payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewEinoAgenticModelFailoverConfigBuildsDistinctFallbackModel(t *testing.T) {
|
||||
t.Parallel()
|
||||
appCfg := &config.Config{
|
||||
OpenAI: config.OpenAIConfig{Provider: "openai", APIKey: "k1", BaseURL: "https://api.example/v1", Model: "primary"},
|
||||
AI: config.AIConfig{Channels: map[string]config.AIChannelConfig{
|
||||
"fb1": {Provider: "openai", APIKey: "k2", BaseURL: "https://api.example/v1", Model: "fallback-1"},
|
||||
"fb2": {Provider: "openai", APIKey: "k3", BaseURL: "https://api.example/v1", Model: "fallback-2"},
|
||||
}},
|
||||
}
|
||||
var built []string
|
||||
cfg, err := newEinoAgenticModelFailoverConfig(
|
||||
context.Background(),
|
||||
appCfg,
|
||||
&config.MultiAgentEinoMiddlewareConfig{
|
||||
ModelFailoverChannels: []string{"fb1", "fb2"},
|
||||
ModelFailoverMaxRetries: 1,
|
||||
},
|
||||
einoModelModeNormal,
|
||||
func(_ context.Context, oa config.OpenAIConfig, _ einoModelMode) (model.AgenticModel, error) {
|
||||
built = append(built, oa.Model)
|
||||
return &fakeAgenticGateModel{}, nil
|
||||
},
|
||||
nil,
|
||||
"agentic",
|
||||
nil,
|
||||
"eino_single_agentic",
|
||||
"conv-1",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newEinoAgenticModelFailoverConfig: %v", err)
|
||||
}
|
||||
if cfg == nil || cfg.MaxRetries != 1 {
|
||||
t.Fatalf("agentic failover cfg = %#v, want max retries 1", cfg)
|
||||
}
|
||||
m, msgs, err := cfg.GetFailoverModel(context.Background(), &adk.FailoverContext[*schema.AgenticMessage]{FailoverAttempt: 1})
|
||||
if err != nil || m == nil || msgs != nil {
|
||||
t.Fatalf("GetFailoverModel = (%v, %v, %v)", m, msgs, err)
|
||||
}
|
||||
if len(built) != 1 || built[0] != "fallback-1" {
|
||||
t.Fatalf("built models = %v, want [fallback-1]", built)
|
||||
}
|
||||
if !cfg.ShouldFailover(context.Background(), nil, &adk.RetryExhaustedError{LastErr: errors.New("upstream returned 503"), TotalRetries: 4}) {
|
||||
t.Fatal("retry-exhausted transient agentic error should fail over")
|
||||
}
|
||||
if cfg.ShouldFailover(context.Background(), nil, &adk.RetryExhaustedError{LastErr: errors.New("invalid api key"), TotalRetries: 4}) {
|
||||
t.Fatal("retry-exhausted permanent agentic error should not fail over")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewEinoAgenticModelFailoverConfigEmitsProgressEvent(t *testing.T) {
|
||||
t.Parallel()
|
||||
appCfg := &config.Config{
|
||||
OpenAI: config.OpenAIConfig{Provider: "openai", APIKey: "k1", BaseURL: "https://api.example/v1", Model: "primary"},
|
||||
AI: config.AIConfig{Channels: map[string]config.AIChannelConfig{
|
||||
"fb1": {Provider: "openai", APIKey: "k2", BaseURL: "https://api.example/v1", Model: "fallback-1"},
|
||||
}},
|
||||
}
|
||||
var events []struct {
|
||||
eventType string
|
||||
message string
|
||||
data interface{}
|
||||
}
|
||||
cfg, err := newEinoAgenticModelFailoverConfig(
|
||||
context.Background(),
|
||||
appCfg,
|
||||
&config.MultiAgentEinoMiddlewareConfig{ModelFailoverChannels: []string{"fb1"}},
|
||||
einoModelModeNormal,
|
||||
func(_ context.Context, _ config.OpenAIConfig, _ einoModelMode) (model.AgenticModel, error) {
|
||||
return &fakeAgenticGateModel{}, nil
|
||||
},
|
||||
nil,
|
||||
"agentic",
|
||||
func(eventType, message string, data interface{}) {
|
||||
events = append(events, struct {
|
||||
eventType string
|
||||
message string
|
||||
data interface{}
|
||||
}{eventType: eventType, message: message, data: data})
|
||||
},
|
||||
"eino_single_agentic",
|
||||
"conv-1",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newEinoAgenticModelFailoverConfig: %v", err)
|
||||
}
|
||||
if _, _, err := cfg.GetFailoverModel(context.Background(), &adk.FailoverContext[*schema.AgenticMessage]{FailoverAttempt: 1}); err != nil {
|
||||
t.Fatalf("GetFailoverModel: %v", err)
|
||||
}
|
||||
if len(events) != 1 || events[0].eventType != "eino_model_failover" {
|
||||
t.Fatalf("events = %#v, want one eino_model_failover", events)
|
||||
}
|
||||
payload, ok := events[0].data.(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("event payload type = %T", events[0].data)
|
||||
}
|
||||
if payload["conversationId"] != "conv-1" || payload["orchestration"] != "eino_single_agentic" || payload["channel"] != "fb1" || payload["model"] != "fallback-1" {
|
||||
t.Fatalf("payload = %#v", payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewEinoOpenAIAgenticChatModelFactoryBuildsBackend(t *testing.T) {
|
||||
t.Parallel()
|
||||
factory := newEinoOpenAIAgenticChatModelFactory(newEinoBaseHTTPClient(), nil, nil)
|
||||
m, err := factory(context.Background(), config.OpenAIConfig{
|
||||
Provider: "openai",
|
||||
APIKey: "test-key",
|
||||
BaseURL: "https://api.example/v1",
|
||||
Model: "gpt-4o-mini",
|
||||
Reasoning: config.OpenAIReasoningConfig{
|
||||
Profile: "openai_compat",
|
||||
Mode: "on",
|
||||
Effort: "high",
|
||||
},
|
||||
}, einoModelModeNormal)
|
||||
if err != nil {
|
||||
t.Fatalf("agentic factory: %v", err)
|
||||
}
|
||||
if m == nil {
|
||||
t.Fatal("agentic factory returned nil model")
|
||||
}
|
||||
gate := evaluateEinoAgenticModelGate(agenticModelGateFactory(factory, config.OpenAIConfig{
|
||||
Provider: "openai",
|
||||
APIKey: "test-key",
|
||||
BaseURL: "https://api.example/v1",
|
||||
Model: "gpt-4o-mini",
|
||||
}, einoModelModeNormal), einoAgenticRuntimeSupportV0914())
|
||||
if !gate.Ready {
|
||||
t.Fatalf("gate = %#v, want ready with buildable agentic backend", gate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewEinoOpenAIAgenticChatModelFactoryRejectsUnsupportedProvider(t *testing.T) {
|
||||
t.Parallel()
|
||||
factory := newEinoOpenAIAgenticChatModelFactory(newEinoBaseHTTPClient(), nil, nil)
|
||||
if _, err := factory(context.Background(), config.OpenAIConfig{
|
||||
Provider: "claude",
|
||||
APIKey: "test-key",
|
||||
BaseURL: "https://api.anthropic.com/v1",
|
||||
Model: "claude-sonnet-4",
|
||||
}, einoModelModeNormal); err == nil {
|
||||
t.Fatal("expected unsupported provider error")
|
||||
}
|
||||
gate := evaluateEinoAgenticModelGate(agenticModelGateFactory(factory, config.OpenAIConfig{
|
||||
Provider: "claude",
|
||||
APIKey: "test-key",
|
||||
BaseURL: "https://api.anthropic.com/v1",
|
||||
Model: "claude-sonnet-4",
|
||||
}, einoModelModeNormal), einoAgenticRuntimeSupportV0914())
|
||||
if gate.Ready || !containsString(gate.Missing, "model.AgenticModel backend") {
|
||||
t.Fatalf("gate = %#v, want backend missing for unsupported provider", gate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoNativeRetryErrorsDoNotTriggerRunLevelTransientRetry(t *testing.T) {
|
||||
t.Parallel()
|
||||
err := &adk.WillRetryError{ErrStr: "HTTP 429 Too Many Requests", RetryAttempt: 1}
|
||||
if isEinoTransientRunError(err) {
|
||||
t.Fatal("WillRetryError should be observed, not treated as a run-level transient failure")
|
||||
}
|
||||
exhausted := &adk.RetryExhaustedError{LastErr: errors.New("HTTP 429 Too Many Requests"), TotalRetries: 4}
|
||||
if isEinoTransientRunError(exhausted) {
|
||||
t.Fatal("RetryExhaustedError should not trigger a second run-level retry layer")
|
||||
}
|
||||
if got := unwrapEinoRetryExhausted(exhausted); got == exhausted {
|
||||
t.Fatal("unwrapEinoRetryExhausted should return the underlying model error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
)
|
||||
|
||||
func applyBeforeModelRewriteHandlers(
|
||||
ctx context.Context,
|
||||
msgs []adk.Message,
|
||||
handlers []adk.ChatModelAgentMiddleware,
|
||||
) ([]adk.Message, error) {
|
||||
if len(msgs) == 0 || len(handlers) == 0 {
|
||||
return msgs, nil
|
||||
}
|
||||
state := &adk.ChatModelAgentState{Messages: msgs}
|
||||
modelCtx := &adk.ModelContext{}
|
||||
curCtx := ctx
|
||||
for _, h := range handlers {
|
||||
if h == nil {
|
||||
continue
|
||||
}
|
||||
nextCtx, nextState, err := h.BeforeModelRewriteState(curCtx, state, modelCtx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("before model rewrite: %w", err)
|
||||
}
|
||||
if nextCtx != nil {
|
||||
curCtx = nextCtx
|
||||
}
|
||||
if nextState != nil {
|
||||
state = nextState
|
||||
}
|
||||
}
|
||||
return state.Messages, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
)
|
||||
|
||||
const (
|
||||
einoNativeCancelImmediateWait = 1200 * time.Millisecond
|
||||
einoNativeCancelSafePointWait = 3500 * time.Millisecond
|
||||
einoNativeCancelSafePointTTL = 3 * time.Second
|
||||
)
|
||||
|
||||
type agentRuntimeCancelRegistrarKey struct{}
|
||||
type agentTurnLoopInterruptRegistrarKey struct{}
|
||||
|
||||
// AgentRuntimeCancelRegistrar binds the currently active Eino ADK cancel hook
|
||||
// into the host task manager. The hook returns true when Eino accepted and
|
||||
// handled the cancel request, so the host can avoid canceling the parent context.
|
||||
type AgentRuntimeCancelRegistrar func(cancel func(error) bool) (unregister func())
|
||||
|
||||
// WithAgentRuntimeCancelRegistrar lets the HTTP/task layer trigger Eino's native
|
||||
// Agent Cancel before falling back to the existing context cancellation path.
|
||||
func WithAgentRuntimeCancelRegistrar(ctx context.Context, registrar AgentRuntimeCancelRegistrar) context.Context {
|
||||
if ctx == nil || registrar == nil {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, agentRuntimeCancelRegistrarKey{}, registrar)
|
||||
}
|
||||
|
||||
func agentRuntimeCancelRegistrarFromContext(ctx context.Context) AgentRuntimeCancelRegistrar {
|
||||
if ctx == nil {
|
||||
return nil
|
||||
}
|
||||
if v, ok := ctx.Value(agentRuntimeCancelRegistrarKey{}).(AgentRuntimeCancelRegistrar); ok {
|
||||
return v
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AgentTurnLoopInterruptRegistrar binds a conversation-level TurnLoop interrupt
|
||||
// pusher into the host task manager. The pusher receives the user supplied note
|
||||
// and returns true when the note was accepted by the loop.
|
||||
type AgentTurnLoopInterruptRegistrar func(push func(note string) bool) (unregister func())
|
||||
|
||||
// WithAgentTurnLoopInterruptRegistrar lets the HTTP/task layer enqueue a user
|
||||
// supplement into an active Eino TurnLoop before falling back to cancellation.
|
||||
func WithAgentTurnLoopInterruptRegistrar(ctx context.Context, registrar AgentTurnLoopInterruptRegistrar) context.Context {
|
||||
if ctx == nil || registrar == nil {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, agentTurnLoopInterruptRegistrarKey{}, registrar)
|
||||
}
|
||||
|
||||
func agentTurnLoopInterruptRegistrarFromContext(ctx context.Context) AgentTurnLoopInterruptRegistrar {
|
||||
if ctx == nil {
|
||||
return nil
|
||||
}
|
||||
if v, ok := ctx.Value(agentTurnLoopInterruptRegistrarKey{}).(AgentTurnLoopInterruptRegistrar); ok {
|
||||
return v
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func requestEinoNativeAgentCancel(cancelFn adk.AgentCancelFunc, cause error) (waitErr error, submitted bool, handled bool) {
|
||||
if cancelFn == nil {
|
||||
return nil, false, false
|
||||
}
|
||||
opts, waitFor := einoNativeCancelOptions(cause)
|
||||
handle, submitted := cancelFn(opts...)
|
||||
if !submitted || handle == nil {
|
||||
return nil, submitted, false
|
||||
}
|
||||
waitCh := make(chan error, 1)
|
||||
go func() {
|
||||
waitCh <- handle.Wait()
|
||||
}()
|
||||
select {
|
||||
case err := <-waitCh:
|
||||
handled := err == nil || errors.Is(err, adk.ErrCancelTimeout) || errors.Is(err, adk.ErrExecutionEnded)
|
||||
return err, submitted, handled
|
||||
case <-time.After(waitFor):
|
||||
return context.DeadlineExceeded, submitted, false
|
||||
}
|
||||
}
|
||||
|
||||
func einoNativeCancelOptions(cause error) ([]adk.AgentCancelOption, time.Duration) {
|
||||
if errors.Is(cause, ErrInterruptContinue) {
|
||||
return []adk.AgentCancelOption{
|
||||
adk.WithAgentCancelMode(adk.CancelAfterChatModel | adk.CancelAfterToolCalls),
|
||||
adk.WithAgentCancelTimeout(einoNativeCancelSafePointTTL),
|
||||
adk.WithRecursive(),
|
||||
}, einoNativeCancelSafePointWait
|
||||
}
|
||||
return []adk.AgentCancelOption{
|
||||
adk.WithAgentCancelMode(adk.CancelImmediate),
|
||||
adk.WithRecursive(),
|
||||
}, einoNativeCancelImmediateWait
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEinoNativeCancelOptionsByCause(t *testing.T) {
|
||||
fullStopOpts, fullStopWait := einoNativeCancelOptions(context.Canceled)
|
||||
if len(fullStopOpts) != 2 {
|
||||
t.Fatalf("full stop options: got %d want 2", len(fullStopOpts))
|
||||
}
|
||||
if fullStopWait != einoNativeCancelImmediateWait {
|
||||
t.Fatalf("full stop wait: got %v want %v", fullStopWait, einoNativeCancelImmediateWait)
|
||||
}
|
||||
|
||||
interruptOpts, interruptWait := einoNativeCancelOptions(ErrInterruptContinue)
|
||||
if len(interruptOpts) != 3 {
|
||||
t.Fatalf("interrupt options: got %d want 3", len(interruptOpts))
|
||||
}
|
||||
if interruptWait != einoNativeCancelSafePointWait {
|
||||
t.Fatalf("interrupt wait: got %v want %v", interruptWait, einoNativeCancelSafePointWait)
|
||||
}
|
||||
if interruptWait <= einoNativeCancelSafePointTTL {
|
||||
t.Fatalf("interrupt wait must allow the Eino safe-point timeout to elapse: wait=%v ttl=%v", interruptWait, einoNativeCancelSafePointTTL)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func emitEinoNativeModelRetryProgress(
|
||||
conversationID, orchMode string,
|
||||
willRetry *adk.WillRetryError,
|
||||
progress func(eventType, message string, data interface{}),
|
||||
logger *zap.Logger,
|
||||
runErr error,
|
||||
) bool {
|
||||
if willRetry == nil {
|
||||
return false
|
||||
}
|
||||
if progress != nil {
|
||||
reason := ""
|
||||
if willRetry.RejectReason() != nil {
|
||||
reason = fmt.Sprint(willRetry.RejectReason())
|
||||
}
|
||||
progress("eino_model_retry", "模型调用遇到临时问题,Eino 正在原生重试…", map[string]interface{}{
|
||||
"conversationId": conversationID,
|
||||
"source": "eino",
|
||||
"orchestration": orchMode,
|
||||
"attempt": willRetry.RetryAttempt,
|
||||
"reason": reason,
|
||||
"error": willRetry.Error(),
|
||||
})
|
||||
}
|
||||
if logger != nil {
|
||||
logger.Warn("eino native model retry event",
|
||||
zap.String("orchestration", orchMode),
|
||||
zap.Int("attempt", willRetry.RetryAttempt),
|
||||
zap.Error(runErr))
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zaptest/observer"
|
||||
)
|
||||
|
||||
func TestEmitEinoNativeModelRetryProgress(t *testing.T) {
|
||||
willRetry := &adk.WillRetryError{
|
||||
ErrStr: "HTTP 429 Too Many Requests",
|
||||
RetryAttempt: 2,
|
||||
}
|
||||
var gotType, gotMessage string
|
||||
var gotData map[string]interface{}
|
||||
called := emitEinoNativeModelRetryProgress("conv-1", "deep_agent", willRetry, func(eventType, message string, data interface{}) {
|
||||
gotType = eventType
|
||||
gotMessage = message
|
||||
var ok bool
|
||||
gotData, ok = data.(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("progress data type = %T, want map[string]interface{}", data)
|
||||
}
|
||||
}, nil, willRetry)
|
||||
if !called {
|
||||
t.Fatal("called = false, want true")
|
||||
}
|
||||
if gotType != "eino_model_retry" {
|
||||
t.Fatalf("event type = %q, want eino_model_retry", gotType)
|
||||
}
|
||||
if gotMessage != "模型调用遇到临时问题,Eino 正在原生重试…" {
|
||||
t.Fatalf("message = %q", gotMessage)
|
||||
}
|
||||
assertNativeRetryMapValue(t, gotData, "conversationId", "conv-1")
|
||||
assertNativeRetryMapValue(t, gotData, "source", "eino")
|
||||
assertNativeRetryMapValue(t, gotData, "orchestration", "deep_agent")
|
||||
assertNativeRetryMapValue(t, gotData, "attempt", 2)
|
||||
assertNativeRetryMapValue(t, gotData, "reason", "")
|
||||
assertNativeRetryMapValue(t, gotData, "error", "HTTP 429 Too Many Requests")
|
||||
}
|
||||
|
||||
func TestEmitEinoNativeModelRetryProgressNilSafe(t *testing.T) {
|
||||
calledProgress := false
|
||||
called := emitEinoNativeModelRetryProgress("conv-1", "deep_agent", nil, func(string, string, interface{}) {
|
||||
calledProgress = true
|
||||
}, nil, nil)
|
||||
if called {
|
||||
t.Fatal("called = true, want false")
|
||||
}
|
||||
if calledProgress {
|
||||
t.Fatal("progress called for nil willRetry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmitEinoNativeModelRetryProgressLogsEvent(t *testing.T) {
|
||||
core, logs := observer.New(zap.WarnLevel)
|
||||
logger := zap.New(core)
|
||||
willRetry := &adk.WillRetryError{
|
||||
ErrStr: "HTTP 500",
|
||||
RetryAttempt: 3,
|
||||
}
|
||||
|
||||
emitEinoNativeModelRetryProgress("conv-1", "single_agent", willRetry, nil, logger, willRetry)
|
||||
|
||||
entry := logs.FilterMessage("eino native model retry event").TakeAll()
|
||||
if len(entry) != 1 {
|
||||
t.Fatalf("log count = %d, want 1", len(entry))
|
||||
}
|
||||
fields := entry[0].ContextMap()
|
||||
if fields["orchestration"] != "single_agent" {
|
||||
t.Fatalf("orchestration field = %v", fields["orchestration"])
|
||||
}
|
||||
if fields["attempt"] != int64(3) {
|
||||
t.Fatalf("attempt field = %v", fields["attempt"])
|
||||
}
|
||||
}
|
||||
|
||||
func assertNativeRetryMapValue(t *testing.T, data map[string]interface{}, key string, want interface{}) {
|
||||
t.Helper()
|
||||
if got := data[key]; got != want {
|
||||
t.Fatalf("%s = %v, want %v", key, got, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,408 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"cyberstrike-ai/internal/agent"
|
||||
"cyberstrike-ai/internal/config"
|
||||
"cyberstrike-ai/internal/database"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"github.com/cloudwego/eino/adk/prebuilt/planexecute"
|
||||
"github.com/cloudwego/eino/components/model"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// PlanExecuteRootArgs 构建 Eino adk/prebuilt/planexecute 根 Agent 所需参数。
|
||||
type PlanExecuteRootArgs struct {
|
||||
MainToolCallingModel model.ToolCallingChatModel
|
||||
AgenticExecModel model.AgenticModel
|
||||
OrchInstruction string
|
||||
ToolsCfg adk.ToolsConfig
|
||||
ExecMaxIter int
|
||||
LoopMaxIter int
|
||||
// AppCfg / Logger 非空时为 Executor 挂载与 Deep/Supervisor 一致的 Eino summarization 中间件。
|
||||
AppCfg *config.Config
|
||||
MwCfg *config.MultiAgentEinoMiddlewareConfig
|
||||
// ConversationID is used for transcript/isolation paths in middleware.
|
||||
ConversationID string
|
||||
DB *database.DB
|
||||
ProjectID string
|
||||
Logger *zap.Logger
|
||||
// ModelName is used for model input token estimation logs.
|
||||
ModelName string
|
||||
// AgenticExecPreMiddlewares 是由 prependEinoAgenticMiddlewares 构建的前置中间件。
|
||||
AgenticExecPreMiddlewares []adk.TypedChatModelAgentMiddleware[*schema.AgenticMessage]
|
||||
AgenticSkillMiddleware adk.TypedChatModelAgentMiddleware[*schema.AgenticMessage]
|
||||
AgenticFilesystemMiddleware adk.TypedChatModelAgentMiddleware[*schema.AgenticMessage]
|
||||
// PlannerReplannerRewriteHandlers applies BeforeModelRewriteState pipeline for planner/replanner input.
|
||||
PlannerReplannerRewriteHandlers []adk.ChatModelAgentMiddleware
|
||||
// ModelFacingTrace 可选:由 Executor Handlers 链末尾写入,供 last_react 与 summarization 后上下文对齐。
|
||||
ModelFacingTrace *modelFacingTraceHolder
|
||||
AgenticModelRetryConfig *adk.TypedModelRetryConfig[*schema.AgenticMessage]
|
||||
AgenticModelFailoverConfig *adk.ModelFailoverConfig[*schema.AgenticMessage]
|
||||
}
|
||||
|
||||
// NewPlanExecuteRoot 返回 plan → execute → replan 预置编排根节点(与 Deep / Supervisor 并列)。
|
||||
func NewPlanExecuteRoot(ctx context.Context, a *PlanExecuteRootArgs) (adk.ResumableAgent, error) {
|
||||
if a == nil {
|
||||
return nil, fmt.Errorf("plan_execute: args 为空")
|
||||
}
|
||||
if a.MainToolCallingModel == nil || a.AgenticExecModel == nil {
|
||||
return nil, fmt.Errorf("plan_execute: 模型为空")
|
||||
}
|
||||
tcm, ok := interface{}(a.MainToolCallingModel).(model.ToolCallingChatModel)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("plan_execute: 主模型需实现 ToolCallingChatModel")
|
||||
}
|
||||
plannerCfg := &planexecute.PlannerConfig{
|
||||
ToolCallingChatModel: tcm,
|
||||
NewPlan: newLenientPlan,
|
||||
}
|
||||
if fn := planExecutePlannerGenInput(a.OrchInstruction, a.AppCfg, a.MwCfg, a.Logger, a.ModelName, a.ConversationID, a.PlannerReplannerRewriteHandlers); fn != nil {
|
||||
plannerCfg.GenInputFn = fn
|
||||
}
|
||||
planner, err := planexecute.NewPlanner(ctx, plannerCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("plan_execute planner: %w", err)
|
||||
}
|
||||
replanner, err := planexecute.NewReplanner(ctx, &planexecute.ReplannerConfig{
|
||||
ChatModel: tcm,
|
||||
GenInputFn: planExecuteReplannerGenInput(a.OrchInstruction, a.AppCfg, a.MwCfg, a.Logger, a.ModelName, a.ConversationID, a.PlannerReplannerRewriteHandlers),
|
||||
NewPlan: newLenientPlan,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("plan_execute replanner: %w", err)
|
||||
}
|
||||
|
||||
var executor adk.Agent
|
||||
agenticExecHandlers, herr := buildPlanExecuteAgenticExecutorHandlers(ctx, a)
|
||||
if herr != nil {
|
||||
return nil, herr
|
||||
}
|
||||
executor, err = newPlanExecuteAgenticExecutor(ctx, &planexecute.ExecutorConfig{
|
||||
ToolsConfig: a.ToolsCfg,
|
||||
MaxIterations: a.ExecMaxIter,
|
||||
GenInputFn: planExecuteExecutorGenInput(a.OrchInstruction, a.AppCfg, a.MwCfg, a.Logger, a.ModelName, a.ConversationID),
|
||||
}, a.AgenticExecModel, agenticExecHandlers, a.AgenticModelRetryConfig, a.AgenticModelFailoverConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("plan_execute executor: %w", err)
|
||||
}
|
||||
loopMax := a.LoopMaxIter
|
||||
if loopMax <= 0 {
|
||||
loopMax = 10
|
||||
}
|
||||
return planexecute.New(ctx, &planexecute.Config{
|
||||
Planner: planner,
|
||||
Executor: executor,
|
||||
Replanner: replanner,
|
||||
MaxIterations: loopMax,
|
||||
})
|
||||
}
|
||||
|
||||
func buildPlanExecuteAgenticExecutorHandlers(ctx context.Context, a *PlanExecuteRootArgs) ([]adk.TypedChatModelAgentMiddleware[*schema.AgenticMessage], error) {
|
||||
if a == nil {
|
||||
return nil, fmt.Errorf("plan_execute: args 为空")
|
||||
}
|
||||
var execHandlers []adk.TypedChatModelAgentMiddleware[*schema.AgenticMessage]
|
||||
if len(a.AgenticExecPreMiddlewares) > 0 {
|
||||
execHandlers = append(execHandlers, a.AgenticExecPreMiddlewares...)
|
||||
}
|
||||
if a.AgenticFilesystemMiddleware != nil {
|
||||
execHandlers = append(execHandlers, a.AgenticFilesystemMiddleware)
|
||||
}
|
||||
if a.AgenticSkillMiddleware != nil {
|
||||
execHandlers = append(execHandlers, a.AgenticSkillMiddleware)
|
||||
}
|
||||
if a.AppCfg != nil {
|
||||
sumMw, sumErr := newEinoAgenticSummarizationMiddleware(ctx, a.AgenticExecModel, a.AppCfg, a.MwCfg, a.ConversationID, a.DB, a.ProjectID, a.Logger)
|
||||
if sumErr != nil {
|
||||
return nil, fmt.Errorf("plan_execute agentic executor summarization: %w", sumErr)
|
||||
}
|
||||
execHandlers = appendEinoAgenticChatModelTailMiddlewares(execHandlers, einoChatModelTailConfig{
|
||||
logger: a.Logger,
|
||||
phase: "plan_execute_executor",
|
||||
agenticSummarization: sumMw,
|
||||
modelName: a.ModelName,
|
||||
maxTotalTokens: a.AppCfg.OpenAI.MaxTotalTokens,
|
||||
toolMaxBytes: toolMaxBytesFromMW(a.MwCfg),
|
||||
conversationID: a.ConversationID,
|
||||
trace: a.ModelFacingTrace,
|
||||
middlewareConfig: a.MwCfg,
|
||||
})
|
||||
}
|
||||
return execHandlers, nil
|
||||
}
|
||||
|
||||
// planExecutePlannerGenInput 将 orchestrator instruction 作为 SystemMessage 注入 planner 输入。
|
||||
// 返回 nil 时 Eino 使用内置默认 planner prompt。
|
||||
func planExecutePlannerGenInput(
|
||||
orchInstruction string,
|
||||
appCfg *config.Config,
|
||||
mwCfg *config.MultiAgentEinoMiddlewareConfig,
|
||||
logger *zap.Logger,
|
||||
modelName string,
|
||||
conversationID string,
|
||||
rewriteHandlers []adk.ChatModelAgentMiddleware,
|
||||
) planexecute.GenPlannerModelInputFn {
|
||||
oi := strings.TrimSpace(orchInstruction)
|
||||
if oi == "" && appCfg == nil {
|
||||
return nil
|
||||
}
|
||||
return func(ctx context.Context, userInput []adk.Message) ([]adk.Message, error) {
|
||||
userInput = capPlanExecuteUserInputMessages(userInput, appCfg, mwCfg)
|
||||
msgs := make([]adk.Message, 0, len(userInput))
|
||||
msgs = append(msgs, userInput...)
|
||||
if rewritten, rerr := applyBeforeModelRewriteHandlers(ctx, msgs, rewriteHandlers); rerr == nil && len(rewritten) > 0 {
|
||||
msgs = rewritten
|
||||
}
|
||||
msgs = normalizeSingleLeadingSystemMessage(msgs, oi)
|
||||
logPlanExecuteModelInputEstimate(logger, modelName, conversationID, "plan_execute_planner", msgs)
|
||||
return msgs, nil
|
||||
}
|
||||
}
|
||||
|
||||
func planExecuteExecutorGenInput(
|
||||
orchInstruction string,
|
||||
appCfg *config.Config,
|
||||
mwCfg *config.MultiAgentEinoMiddlewareConfig,
|
||||
logger *zap.Logger,
|
||||
modelName string,
|
||||
conversationID string,
|
||||
) planexecute.GenModelInputFn {
|
||||
oi := strings.TrimSpace(orchInstruction)
|
||||
return func(ctx context.Context, in *planexecute.ExecutionContext) ([]adk.Message, error) {
|
||||
planContent, err := in.Plan.MarshalJSON()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
userMsgs, err := planexecute.ExecutorPrompt.Format(ctx, map[string]any{
|
||||
"input": planExecuteFormatInput(capPlanExecuteUserInputMessages(in.UserInput, appCfg, mwCfg)),
|
||||
"plan": string(planContent),
|
||||
"executed_steps": planExecuteFormatExecutedSteps(in.ExecutedSteps, appCfg, mwCfg),
|
||||
"step": in.Plan.FirstStep(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
userMsgs = normalizeSingleLeadingSystemMessage(userMsgs, oi)
|
||||
logPlanExecuteModelInputEstimate(logger, modelName, conversationID, "plan_execute_executor_gen_input", userMsgs)
|
||||
return userMsgs, nil
|
||||
}
|
||||
}
|
||||
|
||||
func planExecuteFormatInput(input []adk.Message) string {
|
||||
var sb strings.Builder
|
||||
for _, msg := range input {
|
||||
sb.WriteString(msg.Content)
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func planExecuteFormatExecutedSteps(results []planexecute.ExecutedStep, appCfg *config.Config, mwCfg *config.MultiAgentEinoMiddlewareConfig) string {
|
||||
capped := capPlanExecuteExecutedStepsWithConfig(results, mwCfg)
|
||||
return renderPlanExecuteStepsByBudget(capped, appCfg, mwCfg)
|
||||
}
|
||||
|
||||
// planExecuteReplannerGenInput 与 Eino 默认 Replanner 输入一致,但 executed_steps 经 cap 后再写入 prompt,
|
||||
// 且在 orchInstruction 非空时 prepend SystemMessage 使 replanner 也能接收全局指令。
|
||||
func planExecuteReplannerGenInput(
|
||||
orchInstruction string,
|
||||
appCfg *config.Config,
|
||||
mwCfg *config.MultiAgentEinoMiddlewareConfig,
|
||||
logger *zap.Logger,
|
||||
modelName string,
|
||||
conversationID string,
|
||||
rewriteHandlers []adk.ChatModelAgentMiddleware,
|
||||
) planexecute.GenModelInputFn {
|
||||
oi := strings.TrimSpace(orchInstruction)
|
||||
return func(ctx context.Context, in *planexecute.ExecutionContext) ([]adk.Message, error) {
|
||||
planContent, err := in.Plan.MarshalJSON()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
msgs, err := planexecute.ReplannerPrompt.Format(ctx, map[string]any{
|
||||
"plan": string(planContent),
|
||||
"input": planExecuteFormatInput(capPlanExecuteUserInputMessages(in.UserInput, appCfg, mwCfg)),
|
||||
"executed_steps": planExecuteFormatExecutedSteps(in.ExecutedSteps, appCfg, mwCfg),
|
||||
"plan_tool": planexecute.PlanToolInfo.Name,
|
||||
"respond_tool": planexecute.RespondToolInfo.Name,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if rewritten, rerr := applyBeforeModelRewriteHandlers(ctx, msgs, rewriteHandlers); rerr == nil && len(rewritten) > 0 {
|
||||
msgs = rewritten
|
||||
}
|
||||
msgs = normalizeSingleLeadingSystemMessage(msgs, oi)
|
||||
logPlanExecuteModelInputEstimate(logger, modelName, conversationID, "plan_execute_replanner", msgs)
|
||||
return msgs, nil
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeSingleLeadingSystemMessage enforces a provider-friendly message shape:
|
||||
// exactly one system message at index 0 (when any system context exists).
|
||||
// For strict OpenAI-compatible backends (e.g. qwen/vllm templates), this avoids
|
||||
// "System message must be at the beginning" caused by multiple/disordered system messages.
|
||||
func normalizeSingleLeadingSystemMessage(msgs []adk.Message, extraSystem string) []adk.Message {
|
||||
extraSystem = strings.TrimSpace(extraSystem)
|
||||
if len(msgs) == 0 {
|
||||
if extraSystem == "" {
|
||||
return msgs
|
||||
}
|
||||
return []adk.Message{schema.SystemMessage(extraSystem)}
|
||||
}
|
||||
|
||||
systemParts := make([]string, 0, 2)
|
||||
if extraSystem != "" {
|
||||
systemParts = append(systemParts, extraSystem)
|
||||
}
|
||||
nonSystem := make([]adk.Message, 0, len(msgs))
|
||||
for _, msg := range msgs {
|
||||
if msg == nil {
|
||||
continue
|
||||
}
|
||||
if msg.Role == schema.System {
|
||||
if s := strings.TrimSpace(msg.Content); s != "" {
|
||||
systemParts = append(systemParts, s)
|
||||
}
|
||||
continue
|
||||
}
|
||||
nonSystem = append(nonSystem, msg)
|
||||
}
|
||||
if len(systemParts) == 0 {
|
||||
return nonSystem
|
||||
}
|
||||
out := make([]adk.Message, 0, len(nonSystem)+1)
|
||||
out = append(out, schema.SystemMessage(strings.Join(systemParts, "\n\n")))
|
||||
out = append(out, nonSystem...)
|
||||
return out
|
||||
}
|
||||
|
||||
func capPlanExecuteUserInputMessages(input []adk.Message, appCfg *config.Config, mwCfg *config.MultiAgentEinoMiddlewareConfig) []adk.Message {
|
||||
if len(input) == 0 {
|
||||
return input
|
||||
}
|
||||
maxTotal := 120000
|
||||
modelName := "gpt-4o"
|
||||
if appCfg != nil {
|
||||
if appCfg.OpenAI.MaxTotalTokens > 0 {
|
||||
maxTotal = appCfg.OpenAI.MaxTotalTokens
|
||||
}
|
||||
if m := strings.TrimSpace(appCfg.OpenAI.Model); m != "" {
|
||||
modelName = m
|
||||
}
|
||||
}
|
||||
// Reserve most tokens for planner/replanner prompt and tool schema.
|
||||
ratio := 0.35
|
||||
if mwCfg != nil {
|
||||
ratio = mwCfg.PlanExecuteUserInputBudgetRatioEffective()
|
||||
}
|
||||
budget := int(float64(maxTotal) * ratio)
|
||||
if budget < 4096 {
|
||||
budget = 4096
|
||||
}
|
||||
tc := agent.NewTikTokenCounter()
|
||||
out := make([]adk.Message, 0, len(input))
|
||||
used := 0
|
||||
for i := len(input) - 1; i >= 0; i-- {
|
||||
msg := input[i]
|
||||
if msg == nil {
|
||||
continue
|
||||
}
|
||||
n, err := tc.Count(modelName, string(msg.Role)+"\n"+msg.Content)
|
||||
if err != nil {
|
||||
n = (len(msg.Content) + 3) / 4
|
||||
}
|
||||
if n <= 0 {
|
||||
n = 1
|
||||
}
|
||||
if used+n > budget {
|
||||
break
|
||||
}
|
||||
used += n
|
||||
out = append(out, msg)
|
||||
}
|
||||
for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 {
|
||||
out[i], out[j] = out[j], out[i]
|
||||
}
|
||||
if len(out) == 0 {
|
||||
// Keep the latest user message at least.
|
||||
return []adk.Message{input[len(input)-1]}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func renderPlanExecuteStepsByBudget(steps []planexecute.ExecutedStep, appCfg *config.Config, mwCfg *config.MultiAgentEinoMiddlewareConfig) string {
|
||||
if len(steps) == 0 {
|
||||
return ""
|
||||
}
|
||||
maxTotal := 120000
|
||||
modelName := "gpt-4o"
|
||||
if appCfg != nil {
|
||||
if appCfg.OpenAI.MaxTotalTokens > 0 {
|
||||
maxTotal = appCfg.OpenAI.MaxTotalTokens
|
||||
}
|
||||
if m := strings.TrimSpace(appCfg.OpenAI.Model); m != "" {
|
||||
modelName = m
|
||||
}
|
||||
}
|
||||
ratio := 0.2
|
||||
if mwCfg != nil {
|
||||
ratio = mwCfg.PlanExecuteExecutedStepsBudgetRatioEffective()
|
||||
}
|
||||
budget := int(float64(maxTotal) * ratio)
|
||||
if budget < 3072 {
|
||||
budget = 3072
|
||||
}
|
||||
tc := agent.NewTikTokenCounter()
|
||||
var kept []string
|
||||
used := 0
|
||||
skipped := 0
|
||||
for i := len(steps) - 1; i >= 0; i-- {
|
||||
block := fmt.Sprintf("Step: %s\nResult: %s\n\n", steps[i].Step, steps[i].Result)
|
||||
n, err := tc.Count(modelName, block)
|
||||
if err != nil {
|
||||
n = (len(block) + 3) / 4
|
||||
}
|
||||
if n <= 0 {
|
||||
n = 1
|
||||
}
|
||||
if used+n > budget {
|
||||
skipped = i + 1
|
||||
break
|
||||
}
|
||||
used += n
|
||||
kept = append(kept, block)
|
||||
}
|
||||
var sb strings.Builder
|
||||
if skipped > 0 {
|
||||
sb.WriteString(fmt.Sprintf("Earlier executed steps omitted due to context budget: %d steps.\n\n", skipped))
|
||||
}
|
||||
for i := len(kept) - 1; i >= 0; i-- {
|
||||
sb.WriteString(kept[i])
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// planExecuteStreamsMainAssistant 将规划/执行/重规划各阶段助手流式输出映射到主对话区。
|
||||
func planExecuteStreamsMainAssistant(agent string) bool {
|
||||
if agent == "" {
|
||||
return true
|
||||
}
|
||||
switch agent {
|
||||
case "planner", "executor", "replanner", "execute_replan", "plan_execute_replan":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func planExecuteEinoRoleTag(agent string) string {
|
||||
_ = agent
|
||||
return "orchestrator"
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
func TestNormalizeSingleLeadingSystemMessage_MergesMultipleSystems(t *testing.T) {
|
||||
in := []adk.Message{
|
||||
schema.SystemMessage("sys-1"),
|
||||
schema.UserMessage("u1"),
|
||||
schema.SystemMessage("sys-2"),
|
||||
schema.AssistantMessage("a1", nil),
|
||||
}
|
||||
out := normalizeSingleLeadingSystemMessage(in, "orch")
|
||||
if len(out) != 3 {
|
||||
t.Fatalf("unexpected output length: got %d want 3", len(out))
|
||||
}
|
||||
if out[0].Role != schema.System {
|
||||
t.Fatalf("first message role must be system, got %s", out[0].Role)
|
||||
}
|
||||
if got := out[0].Content; got != "orch\n\nsys-1\n\nsys-2" {
|
||||
t.Fatalf("unexpected merged system content: %q", got)
|
||||
}
|
||||
if out[1].Role != schema.User || out[2].Role != schema.Assistant {
|
||||
t.Fatalf("non-system message order changed unexpectedly")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSingleLeadingSystemMessage_NoSystemKeepsFlow(t *testing.T) {
|
||||
in := []adk.Message{
|
||||
schema.UserMessage("u1"),
|
||||
schema.AssistantMessage("a1", nil),
|
||||
}
|
||||
out := normalizeSingleLeadingSystemMessage(in, "")
|
||||
if len(out) != 2 {
|
||||
t.Fatalf("unexpected output length: got %d want 2", len(out))
|
||||
}
|
||||
if out[0].Role != schema.User || out[1].Role != schema.Assistant {
|
||||
t.Fatalf("message order changed unexpectedly")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type einoPendingToolCalls struct {
|
||||
conversationID string
|
||||
progress func(eventType, message string, data interface{})
|
||||
|
||||
mu sync.Mutex
|
||||
byID map[string]toolCallPendingInfo
|
||||
queueByAgent map[string][]string
|
||||
}
|
||||
|
||||
func newEinoPendingToolCalls(conversationID string, progress func(eventType, message string, data interface{})) *einoPendingToolCalls {
|
||||
return &einoPendingToolCalls{
|
||||
conversationID: conversationID,
|
||||
progress: progress,
|
||||
byID: make(map[string]toolCallPendingInfo),
|
||||
queueByAgent: make(map[string][]string),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *einoPendingToolCalls) Mark(tc toolCallPendingInfo) {
|
||||
if p == nil || strings.TrimSpace(tc.ToolCallID) == "" {
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.byID[tc.ToolCallID] = tc
|
||||
p.queueByAgent[tc.EinoAgent] = append(p.queueByAgent[tc.EinoAgent], tc.ToolCallID)
|
||||
}
|
||||
|
||||
func (p *einoPendingToolCalls) PopNextForAgent(agentName string) (toolCallPendingInfo, bool) {
|
||||
if p == nil {
|
||||
return toolCallPendingInfo{}, false
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
q := p.queueByAgent[agentName]
|
||||
for len(q) > 0 {
|
||||
id := q[0]
|
||||
q = q[1:]
|
||||
p.queueByAgent[agentName] = q
|
||||
if tc, ok := p.byID[id]; ok {
|
||||
delete(p.byID, id)
|
||||
return tc, true
|
||||
}
|
||||
}
|
||||
return toolCallPendingInfo{}, false
|
||||
}
|
||||
|
||||
func (p *einoPendingToolCalls) RemoveByID(toolCallID string) {
|
||||
if p == nil || strings.TrimSpace(toolCallID) == "" {
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
delete(p.byID, toolCallID)
|
||||
}
|
||||
|
||||
func (p *einoPendingToolCalls) PopAny() (toolCallPendingInfo, bool) {
|
||||
if p == nil {
|
||||
return toolCallPendingInfo{}, false
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
for id, tc := range p.byID {
|
||||
delete(p.byID, id)
|
||||
return tc, true
|
||||
}
|
||||
return toolCallPendingInfo{}, false
|
||||
}
|
||||
|
||||
func (p *einoPendingToolCalls) Count() int {
|
||||
if p == nil {
|
||||
return 0
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return len(p.byID)
|
||||
}
|
||||
|
||||
func (p *einoPendingToolCalls) FlushAsFailed(err error) {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
pendingSnapshot := make([]toolCallPendingInfo, 0, len(p.byID))
|
||||
for _, tc := range p.byID {
|
||||
pendingSnapshot = append(pendingSnapshot, tc)
|
||||
}
|
||||
p.byID = make(map[string]toolCallPendingInfo)
|
||||
p.queueByAgent = make(map[string][]string)
|
||||
p.mu.Unlock()
|
||||
|
||||
if p.progress == nil {
|
||||
return
|
||||
}
|
||||
msg := ""
|
||||
if err != nil {
|
||||
msg = err.Error()
|
||||
}
|
||||
for _, tc := range pendingSnapshot {
|
||||
toolName := tc.ToolName
|
||||
if strings.TrimSpace(toolName) == "" {
|
||||
toolName = "unknown"
|
||||
}
|
||||
p.progress("tool_result", fmt.Sprintf("工具结果 (%s)", toolName), map[string]interface{}{
|
||||
"toolName": toolName,
|
||||
"success": false,
|
||||
"isError": true,
|
||||
"result": msg,
|
||||
"resultPreview": msg,
|
||||
"toolCallId": tc.ToolCallID,
|
||||
"conversationId": p.conversationID,
|
||||
"einoAgent": tc.EinoAgent,
|
||||
"einoRole": tc.EinoRole,
|
||||
"source": "eino",
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEinoPendingToolCallsPopNextForAgentSkipsRemovedIDs(t *testing.T) {
|
||||
p := newEinoPendingToolCalls("conv", nil)
|
||||
p.Mark(toolCallPendingInfo{ToolCallID: "call-1", ToolName: "first", EinoAgent: "agent"})
|
||||
p.Mark(toolCallPendingInfo{ToolCallID: "call-2", ToolName: "second", EinoAgent: "agent"})
|
||||
p.RemoveByID("call-1")
|
||||
|
||||
got, ok := p.PopNextForAgent("agent")
|
||||
if !ok {
|
||||
t.Fatal("expected pending tool call")
|
||||
}
|
||||
if got.ToolCallID != "call-2" {
|
||||
t.Fatalf("toolCallID = %q, want call-2", got.ToolCallID)
|
||||
}
|
||||
if p.Count() != 0 {
|
||||
t.Fatalf("pending count = %d, want 0", p.Count())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoPendingToolCallsPopAny(t *testing.T) {
|
||||
p := newEinoPendingToolCalls("conv", nil)
|
||||
p.Mark(toolCallPendingInfo{ToolCallID: "call-1", ToolName: "tool"})
|
||||
|
||||
got, ok := p.PopAny()
|
||||
if !ok || got.ToolCallID != "call-1" {
|
||||
t.Fatalf("PopAny = %#v ok=%v", got, ok)
|
||||
}
|
||||
if _, ok := p.PopAny(); ok {
|
||||
t.Fatal("PopAny should be empty after first pop")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoPendingToolCallsFlushAsFailedEmitsAndClears(t *testing.T) {
|
||||
var events []struct {
|
||||
eventType string
|
||||
message string
|
||||
data map[string]interface{}
|
||||
}
|
||||
p := newEinoPendingToolCalls("conv-1", func(eventType, message string, data interface{}) {
|
||||
m, _ := data.(map[string]interface{})
|
||||
events = append(events, struct {
|
||||
eventType string
|
||||
message string
|
||||
data map[string]interface{}
|
||||
}{eventType: eventType, message: message, data: m})
|
||||
})
|
||||
p.Mark(toolCallPendingInfo{
|
||||
ToolCallID: "call-err",
|
||||
ToolName: "",
|
||||
EinoAgent: "agent",
|
||||
EinoRole: "sub",
|
||||
})
|
||||
|
||||
p.FlushAsFailed(errors.New("boom"))
|
||||
|
||||
if p.Count() != 0 {
|
||||
t.Fatalf("pending count = %d, want 0", p.Count())
|
||||
}
|
||||
if len(events) != 1 {
|
||||
t.Fatalf("events = %#v, want one", events)
|
||||
}
|
||||
ev := events[0]
|
||||
if ev.eventType != "tool_result" || ev.message != "工具结果 (unknown)" {
|
||||
t.Fatalf("event = %#v", ev)
|
||||
}
|
||||
if ev.data["toolCallId"] != "call-err" ||
|
||||
ev.data["conversationId"] != "conv-1" ||
|
||||
ev.data["einoAgent"] != "agent" ||
|
||||
ev.data["einoRole"] != "sub" ||
|
||||
ev.data["isError"] != true ||
|
||||
ev.data["success"] != false ||
|
||||
ev.data["result"] != "boom" {
|
||||
t.Fatalf("payload = %#v", ev.data)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"cyberstrike-ai/internal/openai"
|
||||
)
|
||||
|
||||
type einoReasoningStreamEmitter struct {
|
||||
progress func(eventType, message string, data interface{})
|
||||
conversation string
|
||||
orchMode string
|
||||
agentName string
|
||||
einoRole string
|
||||
nextStreamID func() string
|
||||
|
||||
streamID string
|
||||
rawBuf string
|
||||
displayPrev string
|
||||
}
|
||||
|
||||
func newEinoReasoningStreamEmitter(
|
||||
conversationID, orchMode, agentName, einoRole string,
|
||||
progress func(eventType, message string, data interface{}),
|
||||
nextStreamID func() string,
|
||||
) *einoReasoningStreamEmitter {
|
||||
return &einoReasoningStreamEmitter{
|
||||
progress: progress,
|
||||
conversation: conversationID,
|
||||
orchMode: orchMode,
|
||||
agentName: agentName,
|
||||
einoRole: einoRole,
|
||||
nextStreamID: nextStreamID,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *einoReasoningStreamEmitter) EmitDelta(reasoningContent string) bool {
|
||||
if e == nil || strings.TrimSpace(reasoningContent) == "" {
|
||||
return false
|
||||
}
|
||||
var rawDelta string
|
||||
e.rawBuf, rawDelta = normalizeStreamingDelta(e.rawBuf, reasoningContent)
|
||||
if rawDelta == "" || e.progress == nil {
|
||||
return false
|
||||
}
|
||||
fullDisplay := openai.DisplayReasoningContent(e.rawBuf)
|
||||
displayDelta := fullDisplay
|
||||
if strings.HasPrefix(fullDisplay, e.displayPrev) {
|
||||
displayDelta = fullDisplay[len(e.displayPrev):]
|
||||
}
|
||||
e.displayPrev = fullDisplay
|
||||
if displayDelta == "" {
|
||||
return false
|
||||
}
|
||||
if e.streamID == "" {
|
||||
if e.nextStreamID != nil {
|
||||
e.streamID = e.nextStreamID()
|
||||
}
|
||||
if e.streamID == "" {
|
||||
e.streamID = "eino-reasoning"
|
||||
}
|
||||
e.progress("reasoning_chain_stream_start", " ", map[string]interface{}{
|
||||
"streamId": e.streamID,
|
||||
"source": "eino",
|
||||
"einoAgent": e.agentName,
|
||||
"einoRole": e.einoRole,
|
||||
"orchestration": e.orchMode,
|
||||
})
|
||||
}
|
||||
e.progress("reasoning_chain_stream_delta", displayDelta, openai.WithSSEAccumulated(map[string]interface{}{
|
||||
"streamId": e.streamID,
|
||||
}, fullDisplay))
|
||||
return true
|
||||
}
|
||||
|
||||
func (e *einoReasoningStreamEmitter) Finish() string {
|
||||
if e == nil {
|
||||
return ""
|
||||
}
|
||||
display := openai.DisplayReasoningContent(strings.TrimSpace(e.rawBuf))
|
||||
if display == "" || e.streamID == "" || e.progress == nil {
|
||||
return display
|
||||
}
|
||||
e.progress("reasoning_chain_stream_end", display, map[string]interface{}{
|
||||
"streamId": e.streamID,
|
||||
"conversationId": e.conversation,
|
||||
"source": "eino",
|
||||
"einoAgent": e.agentName,
|
||||
"einoRole": e.einoRole,
|
||||
"orchestration": e.orchMode,
|
||||
})
|
||||
return display
|
||||
}
|
||||
|
||||
func (e *einoReasoningStreamEmitter) EmitComplete(reasoningContent string) bool {
|
||||
if e == nil || e.progress == nil {
|
||||
return false
|
||||
}
|
||||
display := openai.DisplayReasoningContent(strings.TrimSpace(reasoningContent))
|
||||
if display == "" {
|
||||
return false
|
||||
}
|
||||
e.progress("reasoning_chain", display, map[string]interface{}{
|
||||
"conversationId": e.conversation,
|
||||
"source": "eino",
|
||||
"einoAgent": e.agentName,
|
||||
"einoRole": e.einoRole,
|
||||
"orchestration": e.orchMode,
|
||||
})
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package multiagent
|
||||
|
||||
import "context"
|
||||
|
||||
type einoRunCancellationHandler struct {
|
||||
ctx context.Context
|
||||
conversationID string
|
||||
progress func(eventType, message string, data interface{})
|
||||
pending *einoPendingToolCalls
|
||||
takePartial einoPartialResultFunc
|
||||
}
|
||||
|
||||
type einoRunCancellationHandlerConfig struct {
|
||||
Context context.Context
|
||||
ConversationID string
|
||||
Progress func(eventType, message string, data interface{})
|
||||
Pending *einoPendingToolCalls
|
||||
TakePartial einoPartialResultFunc
|
||||
}
|
||||
|
||||
func newEinoRunCancellationHandler(cfg einoRunCancellationHandlerConfig) *einoRunCancellationHandler {
|
||||
return &einoRunCancellationHandler{
|
||||
ctx: cfg.Context,
|
||||
conversationID: cfg.ConversationID,
|
||||
progress: cfg.Progress,
|
||||
pending: cfg.Pending,
|
||||
takePartial: cfg.TakePartial,
|
||||
}
|
||||
}
|
||||
|
||||
func (h *einoRunCancellationHandler) Handle(runErr error) (*RunResult, error) {
|
||||
if h == nil {
|
||||
return nil, runErr
|
||||
}
|
||||
if h.pending != nil {
|
||||
h.pending.FlushAsFailed(runErr)
|
||||
}
|
||||
if h.progress != nil {
|
||||
if isInterruptContinue(h.ctx) {
|
||||
h.progress("progress", "已暂停当前输出,正在合并用户补充并继续…", map[string]interface{}{
|
||||
"conversationId": h.conversationID,
|
||||
"source": "eino",
|
||||
"kind": "interrupt_continue",
|
||||
})
|
||||
} else if runErr != nil {
|
||||
h.progress("error", runErr.Error(), map[string]interface{}{
|
||||
"conversationId": h.conversationID,
|
||||
"source": "eino",
|
||||
})
|
||||
}
|
||||
}
|
||||
if h.takePartial == nil {
|
||||
return nil, runErr
|
||||
}
|
||||
return h.takePartial(runErr)
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEinoRunCancellationHandlerFlushesPendingAndEmitsError(t *testing.T) {
|
||||
runErr := errors.New("context canceled")
|
||||
var events []struct {
|
||||
eventType string
|
||||
data map[string]interface{}
|
||||
}
|
||||
progress := func(eventType, _ string, data interface{}) {
|
||||
m, _ := data.(map[string]interface{})
|
||||
events = append(events, struct {
|
||||
eventType string
|
||||
data map[string]interface{}
|
||||
}{eventType: eventType, data: m})
|
||||
}
|
||||
pending := newEinoPendingToolCalls("conv-1", progress)
|
||||
pending.Mark(toolCallPendingInfo{ToolCallID: "call-1", ToolName: "execute", EinoAgent: "lead", EinoRole: "orchestrator"})
|
||||
want := &RunResult{Response: "partial"}
|
||||
|
||||
result, err := newEinoRunCancellationHandler(einoRunCancellationHandlerConfig{
|
||||
Context: context.Background(),
|
||||
ConversationID: "conv-1",
|
||||
Progress: progress,
|
||||
Pending: pending,
|
||||
TakePartial: func(got error) (*RunResult, error) {
|
||||
if !errors.Is(got, runErr) {
|
||||
t.Fatalf("partial err = %v", got)
|
||||
}
|
||||
return want, got
|
||||
},
|
||||
}).Handle(runErr)
|
||||
|
||||
if result != want || !errors.Is(err, runErr) {
|
||||
t.Fatalf("result=%#v err=%v", result, err)
|
||||
}
|
||||
if pending.Count() != 0 {
|
||||
t.Fatalf("pending count = %d, want 0", pending.Count())
|
||||
}
|
||||
var sawError, sawFailedTool bool
|
||||
for _, ev := range events {
|
||||
if ev.eventType == "error" {
|
||||
sawError = ev.data["conversationId"] == "conv-1" && ev.data["source"] == "eino"
|
||||
}
|
||||
if ev.eventType == "tool_result" {
|
||||
sawFailedTool = ev.data["toolCallId"] == "call-1" && ev.data["isError"] == true
|
||||
}
|
||||
}
|
||||
if !sawError || !sawFailedTool {
|
||||
t.Fatalf("events = %#v", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunCancellationHandlerInterruptContinueProgress(t *testing.T) {
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
cancel(ErrInterruptContinue)
|
||||
runErr := context.Canceled
|
||||
var eventType string
|
||||
var data map[string]interface{}
|
||||
|
||||
_, err := newEinoRunCancellationHandler(einoRunCancellationHandlerConfig{
|
||||
Context: ctx,
|
||||
ConversationID: "conv-1",
|
||||
Progress: func(et, _ string, raw interface{}) {
|
||||
eventType = et
|
||||
data, _ = raw.(map[string]interface{})
|
||||
},
|
||||
TakePartial: func(got error) (*RunResult, error) {
|
||||
return nil, got
|
||||
},
|
||||
}).Handle(runErr)
|
||||
|
||||
if !errors.Is(err, runErr) {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
if eventType != "progress" || data["kind"] != "interrupt_continue" || data["conversationId"] != "conv-1" {
|
||||
t.Fatalf("eventType=%q data=%#v", eventType, data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunCancellationHandlerNilSafe(t *testing.T) {
|
||||
runErr := errors.New("boom")
|
||||
var h *einoRunCancellationHandler
|
||||
result, err := h.Handle(runErr)
|
||||
if result != nil || !errors.Is(err, runErr) {
|
||||
t.Fatalf("result=%#v err=%v", result, err)
|
||||
}
|
||||
result, err = newEinoRunCancellationHandler(einoRunCancellationHandlerConfig{}).Handle(runErr)
|
||||
if result != nil || !errors.Is(err, runErr) {
|
||||
t.Fatalf("result=%#v err=%v", result, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type einoRunCompletionHandler struct {
|
||||
conversationID string
|
||||
orchMode string
|
||||
progress func(eventType, message string, data interface{})
|
||||
logger *zap.Logger
|
||||
|
||||
pending *einoPendingToolCalls
|
||||
cpStore *fileCheckPointStore
|
||||
checkPointID string
|
||||
}
|
||||
|
||||
type einoRunCompletionHandlerConfig struct {
|
||||
ConversationID string
|
||||
OrchMode string
|
||||
Progress func(eventType, message string, data interface{})
|
||||
Logger *zap.Logger
|
||||
Pending *einoPendingToolCalls
|
||||
Checkpoint *fileCheckPointStore
|
||||
CheckpointID string
|
||||
}
|
||||
|
||||
func newEinoRunCompletionHandler(cfg einoRunCompletionHandlerConfig) *einoRunCompletionHandler {
|
||||
return &einoRunCompletionHandler{
|
||||
conversationID: cfg.ConversationID,
|
||||
orchMode: cfg.OrchMode,
|
||||
progress: cfg.Progress,
|
||||
logger: cfg.Logger,
|
||||
pending: cfg.Pending,
|
||||
cpStore: cfg.Checkpoint,
|
||||
checkPointID: cfg.CheckpointID,
|
||||
}
|
||||
}
|
||||
|
||||
func (h *einoRunCompletionHandler) Complete() {
|
||||
if h == nil {
|
||||
return
|
||||
}
|
||||
h.flushOrphanedPending()
|
||||
h.cleanupCheckpoint()
|
||||
}
|
||||
|
||||
func (h *einoRunCompletionHandler) flushOrphanedPending() {
|
||||
if h.pending == nil {
|
||||
return
|
||||
}
|
||||
orphanCount := h.pending.Count()
|
||||
if orphanCount <= 0 {
|
||||
return
|
||||
}
|
||||
h.pending.FlushAsFailed(errors.New("pending tool call missing result before run completion"))
|
||||
if h.progress != nil {
|
||||
h.progress("eino_pending_orphaned", "pending tool calls were force-closed at run end", map[string]interface{}{
|
||||
"conversationId": h.conversationID,
|
||||
"source": "eino",
|
||||
"orchestration": h.orchMode,
|
||||
"pendingCount": orphanCount,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (h *einoRunCompletionHandler) cleanupCheckpoint() {
|
||||
if h.cpStore == nil || h.checkPointID == "" {
|
||||
return
|
||||
}
|
||||
p, err := h.cpStore.path(h.checkPointID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if rmErr := os.Remove(p); rmErr != nil && !os.IsNotExist(rmErr) && h.logger != nil {
|
||||
h.logger.Warn("eino checkpoint cleanup failed", zap.String("path", p), zap.Error(rmErr))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEinoRunCompletionHandlerFlushesOrphansAndCleansCheckpoint(t *testing.T) {
|
||||
var events []struct {
|
||||
eventType string
|
||||
data map[string]interface{}
|
||||
}
|
||||
progress := func(eventType, _ string, data interface{}) {
|
||||
m, _ := data.(map[string]interface{})
|
||||
events = append(events, struct {
|
||||
eventType string
|
||||
data map[string]interface{}
|
||||
}{eventType: eventType, data: m})
|
||||
}
|
||||
pending := newEinoPendingToolCalls("conv-1", progress)
|
||||
pending.Mark(toolCallPendingInfo{ToolCallID: "call-1", ToolName: "execute", EinoAgent: "lead", EinoRole: "orchestrator"})
|
||||
store, err := newFileCheckPointStore(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Set(context.Background(), "cp-1", []byte("checkpoint")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cpPath, err := store.path("cp-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
newEinoRunCompletionHandler(einoRunCompletionHandlerConfig{
|
||||
ConversationID: "conv-1",
|
||||
OrchMode: "deep",
|
||||
Progress: progress,
|
||||
Pending: pending,
|
||||
Checkpoint: store,
|
||||
CheckpointID: "cp-1",
|
||||
}).Complete()
|
||||
|
||||
if pending.Count() != 0 {
|
||||
t.Fatalf("pending count = %d, want 0", pending.Count())
|
||||
}
|
||||
if _, err := os.Stat(cpPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("checkpoint should be removed, stat err=%v", err)
|
||||
}
|
||||
var orphanEvent map[string]interface{}
|
||||
var failedToolResult map[string]interface{}
|
||||
for _, ev := range events {
|
||||
switch ev.eventType {
|
||||
case "eino_pending_orphaned":
|
||||
orphanEvent = ev.data
|
||||
case "tool_result":
|
||||
failedToolResult = ev.data
|
||||
}
|
||||
}
|
||||
if orphanEvent == nil || orphanEvent["conversationId"] != "conv-1" || orphanEvent["orchestration"] != "deep" || orphanEvent["pendingCount"] != 1 {
|
||||
t.Fatalf("orphan event = %#v", orphanEvent)
|
||||
}
|
||||
if failedToolResult == nil || failedToolResult["toolCallId"] != "call-1" || failedToolResult["isError"] != true {
|
||||
t.Fatalf("failed tool result = %#v", failedToolResult)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunCompletionHandlerNoopWithoutState(t *testing.T) {
|
||||
newEinoRunCompletionHandler(einoRunCompletionHandlerConfig{}).Complete()
|
||||
var h *einoRunCompletionHandler
|
||||
h.Complete()
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
)
|
||||
|
||||
type einoRunErrorHandler struct {
|
||||
conversationID string
|
||||
orchMode string
|
||||
progress func(eventType, message string, data interface{})
|
||||
pending *einoPendingToolCalls
|
||||
nativeCancelFallback func() error
|
||||
}
|
||||
|
||||
type einoRunErrorHandlerConfig struct {
|
||||
ConversationID string
|
||||
OrchMode string
|
||||
Progress func(eventType, message string, data interface{})
|
||||
Pending *einoPendingToolCalls
|
||||
NativeCancelFallback func() error
|
||||
}
|
||||
|
||||
func newEinoRunErrorHandler(cfg einoRunErrorHandlerConfig) *einoRunErrorHandler {
|
||||
return &einoRunErrorHandler{
|
||||
conversationID: cfg.ConversationID,
|
||||
orchMode: cfg.OrchMode,
|
||||
progress: cfg.Progress,
|
||||
pending: cfg.Pending,
|
||||
nativeCancelFallback: cfg.NativeCancelFallback,
|
||||
}
|
||||
}
|
||||
|
||||
func (h *einoRunErrorHandler) Handle(runErr error) error {
|
||||
if h == nil || runErr == nil {
|
||||
return runErr
|
||||
}
|
||||
var cancelErr *adk.CancelError
|
||||
if errors.As(runErr, &cancelErr) {
|
||||
h.flushPending(runErr)
|
||||
if h.nativeCancelFallback != nil {
|
||||
return h.nativeCancelFallback()
|
||||
}
|
||||
return context.Canceled
|
||||
}
|
||||
if errors.Is(runErr, context.DeadlineExceeded) {
|
||||
h.flushPending(runErr)
|
||||
h.emitError(runErr, "timeout")
|
||||
return runErr
|
||||
}
|
||||
if errors.Is(runErr, context.Canceled) {
|
||||
h.flushPending(runErr)
|
||||
h.emitError(runErr, "")
|
||||
return runErr
|
||||
}
|
||||
if isEinoIterationLimitError(runErr) {
|
||||
h.flushPending(runErr)
|
||||
if h.progress != nil {
|
||||
h.progress("iteration_limit_reached", runErr.Error(), map[string]interface{}{
|
||||
"conversationId": h.conversationID,
|
||||
"source": "eino",
|
||||
"orchestration": h.orchMode,
|
||||
})
|
||||
}
|
||||
h.emitError(runErr, "iteration_limit")
|
||||
return runErr
|
||||
}
|
||||
h.flushPending(runErr)
|
||||
h.emitError(runErr, "")
|
||||
return runErr
|
||||
}
|
||||
|
||||
func (h *einoRunErrorHandler) flushPending(err error) {
|
||||
if h != nil && h.pending != nil {
|
||||
h.pending.FlushAsFailed(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *einoRunErrorHandler) emitError(err error, kind string) {
|
||||
if h == nil || h.progress == nil || err == nil {
|
||||
return
|
||||
}
|
||||
data := map[string]interface{}{
|
||||
"conversationId": h.conversationID,
|
||||
"source": "eino",
|
||||
}
|
||||
if kind != "" {
|
||||
data["errorKind"] = kind
|
||||
}
|
||||
h.progress("error", err.Error(), data)
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
)
|
||||
|
||||
func TestEinoRunErrorHandlerCancelUsesNativeFallback(t *testing.T) {
|
||||
pending := newEinoPendingToolCalls("conv-1", nil)
|
||||
pending.Mark(toolCallPendingInfo{ToolCallID: "call-1", ToolName: "execute"})
|
||||
want := errors.New("native cancel")
|
||||
|
||||
got := newEinoRunErrorHandler(einoRunErrorHandlerConfig{
|
||||
ConversationID: "conv-1",
|
||||
Pending: pending,
|
||||
NativeCancelFallback: func() error {
|
||||
return want
|
||||
},
|
||||
}).Handle(&adk.CancelError{Info: &adk.AgentCancelInfo{}})
|
||||
|
||||
if !errors.Is(got, want) {
|
||||
t.Fatalf("err = %v, want native fallback", got)
|
||||
}
|
||||
if pending.Count() != 0 {
|
||||
t.Fatalf("pending count = %d, want 0", pending.Count())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunErrorHandlerTimeoutAndGeneralErrorProgress(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
err error
|
||||
errorKind interface{}
|
||||
}{
|
||||
{name: "timeout", err: context.DeadlineExceeded, errorKind: "timeout"},
|
||||
{name: "general", err: errors.New("boom"), errorKind: nil},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var data map[string]interface{}
|
||||
got := newEinoRunErrorHandler(einoRunErrorHandlerConfig{
|
||||
ConversationID: "conv-1",
|
||||
Progress: func(eventType, _ string, raw interface{}) {
|
||||
if eventType == "error" {
|
||||
data, _ = raw.(map[string]interface{})
|
||||
}
|
||||
},
|
||||
}).Handle(tc.err)
|
||||
if !errors.Is(got, tc.err) {
|
||||
t.Fatalf("err = %v", got)
|
||||
}
|
||||
if data["conversationId"] != "conv-1" || data["source"] != "eino" {
|
||||
t.Fatalf("data = %#v", data)
|
||||
}
|
||||
if gotKind := data["errorKind"]; gotKind != tc.errorKind {
|
||||
t.Fatalf("errorKind = %#v, want %#v", gotKind, tc.errorKind)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunErrorHandlerIterationLimitProgress(t *testing.T) {
|
||||
var events []string
|
||||
var errorKind interface{}
|
||||
err := errors.New("maximum iteration reached")
|
||||
|
||||
got := newEinoRunErrorHandler(einoRunErrorHandlerConfig{
|
||||
ConversationID: "conv-1",
|
||||
OrchMode: "deep",
|
||||
Progress: func(eventType, _ string, raw interface{}) {
|
||||
events = append(events, eventType)
|
||||
if eventType == "error" {
|
||||
data, _ := raw.(map[string]interface{})
|
||||
errorKind = data["errorKind"]
|
||||
}
|
||||
},
|
||||
}).Handle(err)
|
||||
|
||||
if !errors.Is(got, err) {
|
||||
t.Fatalf("err = %v", got)
|
||||
}
|
||||
if len(events) != 2 || events[0] != "iteration_limit_reached" || events[1] != "error" {
|
||||
t.Fatalf("events = %#v", events)
|
||||
}
|
||||
if errorKind != "iteration_limit" {
|
||||
t.Fatalf("errorKind = %#v", errorKind)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunErrorHandlerNilSafe(t *testing.T) {
|
||||
var h *einoRunErrorHandler
|
||||
if h.Handle(nil) != nil {
|
||||
t.Fatal("nil handler nil err should return nil")
|
||||
}
|
||||
err := errors.New("boom")
|
||||
if got := h.Handle(err); !errors.Is(got, err) {
|
||||
t.Fatalf("nil handler err = %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
|
||||
"cyberstrike-ai/internal/agent"
|
||||
"cyberstrike-ai/internal/config"
|
||||
"cyberstrike-ai/internal/einomcp"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type einoRunEventDrainConfig struct {
|
||||
Context context.Context
|
||||
ConversationID string
|
||||
OrchMode string
|
||||
OrchestratorName string
|
||||
Progress func(eventType, message string, data interface{})
|
||||
Logger *zap.Logger
|
||||
BaseMessages []adk.Message
|
||||
SnapshotMCPIDs func() []string
|
||||
StreamsMainAssistant func(agent string) bool
|
||||
EinoRoleTag func(agent string) string
|
||||
MiddlewareConfig *config.MultiAgentEinoMiddlewareConfig
|
||||
|
||||
FilesystemMonitorAgent *agent.Agent
|
||||
FilesystemMonitorRecord einomcp.ExecutionRecorder
|
||||
MCPExecutionBinder *MCPExecutionBinder
|
||||
}
|
||||
|
||||
type einoRunEventDrain struct {
|
||||
cfg einoRunEventDrainConfig
|
||||
|
||||
runMessages *einoRunMessageAccumulator
|
||||
assistantOutput *einoAssistantOutputAccumulator
|
||||
runProgress *einoRunProgressTracker
|
||||
pendingToolCalls *einoPendingToolCalls
|
||||
stdoutSuppressor *einoExecuteStdoutSuppressor
|
||||
toolResultEmitter *einoToolResultProgressEmitter
|
||||
usage *einoRunUsageAccumulator
|
||||
|
||||
reasoningStreamSeq int64
|
||||
subReplyStreamSeq int64
|
||||
mainResponseStreamSeq int64
|
||||
|
||||
toolResultHandler *einoToolResultEventHandler
|
||||
assistantStreamHandler *einoAssistantStreamEventHandler
|
||||
materializedMessageHandler *einoMaterializedMessageEventHandler
|
||||
}
|
||||
|
||||
func newEinoRunEventDrain(cfg einoRunEventDrainConfig) *einoRunEventDrain {
|
||||
if cfg.Context == nil {
|
||||
cfg.Context = context.Background()
|
||||
}
|
||||
if cfg.SnapshotMCPIDs == nil {
|
||||
cfg.SnapshotMCPIDs = func() []string { return nil }
|
||||
}
|
||||
if cfg.StreamsMainAssistant == nil {
|
||||
cfg.StreamsMainAssistant = func(agentName string) bool {
|
||||
return agentName == "" || agentName == cfg.OrchestratorName
|
||||
}
|
||||
}
|
||||
if cfg.EinoRoleTag == nil {
|
||||
cfg.EinoRoleTag = func(agentName string) string {
|
||||
if cfg.StreamsMainAssistant(agentName) {
|
||||
return "orchestrator"
|
||||
}
|
||||
return "sub"
|
||||
}
|
||||
}
|
||||
|
||||
runMessages := newEinoRunMessageAccumulator(cfg.BaseMessages)
|
||||
assistantOutput := newEinoAssistantOutputAccumulator(cfg.OrchMode)
|
||||
runProgress := newEinoRunProgressTracker(
|
||||
cfg.OrchMode,
|
||||
cfg.OrchestratorName,
|
||||
cfg.ConversationID,
|
||||
cfg.Progress,
|
||||
cfg.StreamsMainAssistant,
|
||||
cfg.EinoRoleTag,
|
||||
)
|
||||
pendingToolCalls := newEinoPendingToolCalls(cfg.ConversationID, cfg.Progress)
|
||||
stdoutSuppressor := newEinoExecuteStdoutSuppressor()
|
||||
usage := newEinoRunUsageAccumulator()
|
||||
toolResultEmitter := newEinoToolResultProgressEmitter(einoToolResultProgressEmitterConfig{
|
||||
ConversationID: cfg.ConversationID,
|
||||
OrchestratorName: cfg.OrchestratorName,
|
||||
Progress: cfg.Progress,
|
||||
EinoRoleTag: cfg.EinoRoleTag,
|
||||
Pending: pendingToolCalls,
|
||||
ExecuteStdoutDup: stdoutSuppressor,
|
||||
RunMessages: runMessages,
|
||||
FilesystemMonitorAgent: cfg.FilesystemMonitorAgent,
|
||||
FilesystemMonitorRecord: cfg.FilesystemMonitorRecord,
|
||||
MCPExecutionBinder: cfg.MCPExecutionBinder,
|
||||
})
|
||||
|
||||
return &einoRunEventDrain{
|
||||
cfg: cfg,
|
||||
runMessages: runMessages,
|
||||
assistantOutput: assistantOutput,
|
||||
runProgress: runProgress,
|
||||
pendingToolCalls: pendingToolCalls,
|
||||
stdoutSuppressor: stdoutSuppressor,
|
||||
toolResultEmitter: toolResultEmitter,
|
||||
usage: usage,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *einoRunEventDrain) BindHandlers(confirmRecovery func()) {
|
||||
if d == nil {
|
||||
return
|
||||
}
|
||||
d.toolResultHandler = newEinoToolResultEventHandler(einoToolResultEventHandlerConfig{
|
||||
Context: d.cfg.Context,
|
||||
Logger: d.cfg.Logger,
|
||||
RunMessages: d.runMessages,
|
||||
Emitter: d.toolResultEmitter,
|
||||
ConfirmRecovery: confirmRecovery,
|
||||
})
|
||||
streamToolCallCompletion := newEinoStreamToolCallCompletionHandler(einoStreamToolCallCompletionHandlerConfig{
|
||||
ConversationID: d.cfg.ConversationID,
|
||||
OrchMode: d.cfg.OrchMode,
|
||||
Progress: d.cfg.Progress,
|
||||
RunProgress: d.runProgress,
|
||||
RunMessages: d.runMessages,
|
||||
MarkPending: d.markPendingWithMonitor,
|
||||
})
|
||||
d.assistantStreamHandler = newEinoAssistantStreamEventHandler(einoAssistantStreamEventHandlerConfig{
|
||||
Context: d.cfg.Context,
|
||||
ConversationID: d.cfg.ConversationID,
|
||||
OrchMode: d.cfg.OrchMode,
|
||||
Progress: d.cfg.Progress,
|
||||
Logger: d.cfg.Logger,
|
||||
SnapshotMCPIDs: d.cfg.SnapshotMCPIDs,
|
||||
StreamsMainAssistant: d.cfg.StreamsMainAssistant,
|
||||
EinoRoleTag: d.cfg.EinoRoleTag,
|
||||
RunProgress: d.runProgress,
|
||||
StdoutSuppressor: d.stdoutSuppressor,
|
||||
AssistantOutput: d.assistantOutput,
|
||||
RunMessages: d.runMessages,
|
||||
Usage: d.usage,
|
||||
ToolCallCompletion: streamToolCallCompletion,
|
||||
NextMainStreamID: d.nextMainStreamID,
|
||||
NextReasoningStreamID: d.nextReasoningStreamID,
|
||||
NextSubAgentReplyStreamID: d.nextSubAgentReplyStreamID,
|
||||
})
|
||||
d.materializedMessageHandler = newEinoMaterializedMessageEventHandler(einoMaterializedMessageEventHandlerConfig{
|
||||
ConversationID: d.cfg.ConversationID,
|
||||
OrchMode: d.cfg.OrchMode,
|
||||
Progress: d.cfg.Progress,
|
||||
SnapshotMCPIDs: d.cfg.SnapshotMCPIDs,
|
||||
StreamsMainAssistant: d.cfg.StreamsMainAssistant,
|
||||
EinoRoleTag: d.cfg.EinoRoleTag,
|
||||
RunProgress: d.runProgress,
|
||||
StdoutSuppressor: d.stdoutSuppressor,
|
||||
AssistantOutput: d.assistantOutput,
|
||||
RunMessages: d.runMessages,
|
||||
Usage: d.usage,
|
||||
ToolResultHandler: d.toolResultHandler,
|
||||
MarkPending: d.markPendingWithMonitor,
|
||||
NextMainStreamID: d.nextMainStreamID,
|
||||
})
|
||||
}
|
||||
|
||||
func (d *einoRunEventDrain) RunMessages() *einoRunMessageAccumulator {
|
||||
if d == nil {
|
||||
return nil
|
||||
}
|
||||
return d.runMessages
|
||||
}
|
||||
|
||||
func (d *einoRunEventDrain) AssistantOutput() *einoAssistantOutputAccumulator {
|
||||
if d == nil {
|
||||
return nil
|
||||
}
|
||||
return d.assistantOutput
|
||||
}
|
||||
|
||||
func (d *einoRunEventDrain) PendingToolCalls() *einoPendingToolCalls {
|
||||
if d == nil {
|
||||
return nil
|
||||
}
|
||||
return d.pendingToolCalls
|
||||
}
|
||||
|
||||
func (d *einoRunEventDrain) Usage() *einoRunUsageAccumulator {
|
||||
if d == nil {
|
||||
return nil
|
||||
}
|
||||
return d.usage
|
||||
}
|
||||
|
||||
func (d *einoRunEventDrain) ObserveAgent(agentName string) {
|
||||
if d == nil || d.runProgress == nil {
|
||||
return
|
||||
}
|
||||
d.runProgress.ObserveAgent(agentName)
|
||||
}
|
||||
|
||||
func (d *einoRunEventDrain) HandleToolResultStreaming(mv *adk.MessageVariant, agentName string) bool {
|
||||
return d != nil && d.toolResultHandler != nil && d.toolResultHandler.HandleStreaming(mv, agentName)
|
||||
}
|
||||
|
||||
func (d *einoRunEventDrain) HandleAssistantStream(mv *adk.MessageVariant, agentName string) (bool, error) {
|
||||
if d == nil || d.assistantStreamHandler == nil {
|
||||
return false, nil
|
||||
}
|
||||
return d.assistantStreamHandler.Handle(mv, agentName)
|
||||
}
|
||||
|
||||
func (d *einoRunEventDrain) HandleMaterialized(mv *adk.MessageVariant, msg adk.Message, agentName string) bool {
|
||||
return d != nil && d.materializedMessageHandler != nil && d.materializedMessageHandler.Handle(mv, msg, agentName)
|
||||
}
|
||||
|
||||
func (d *einoRunEventDrain) markPendingWithMonitor(tc toolCallPendingInfo) {
|
||||
if d == nil || d.pendingToolCalls == nil {
|
||||
return
|
||||
}
|
||||
d.pendingToolCalls.Mark(tc)
|
||||
beginEinoADKFilesystemToolMonitor(
|
||||
d.cfg.Context,
|
||||
d.cfg.FilesystemMonitorAgent,
|
||||
d.cfg.FilesystemMonitorRecord,
|
||||
d.cfg.MCPExecutionBinder,
|
||||
tc.ToolCallID,
|
||||
tc.ToolName,
|
||||
)
|
||||
}
|
||||
|
||||
func (d *einoRunEventDrain) nextMainStreamID() string {
|
||||
return fmt.Sprintf("eino-main-%s-%d", d.cfg.ConversationID, atomic.AddInt64(&d.mainResponseStreamSeq, 1))
|
||||
}
|
||||
|
||||
func (d *einoRunEventDrain) nextReasoningStreamID() string {
|
||||
return fmt.Sprintf("eino-reasoning-%s-%d", d.cfg.ConversationID, atomic.AddInt64(&d.reasoningStreamSeq, 1))
|
||||
}
|
||||
|
||||
func (d *einoRunEventDrain) nextSubAgentReplyStreamID() string {
|
||||
return fmt.Sprintf("eino-sub-reply-%s-%d", d.cfg.ConversationID, atomic.AddInt64(&d.subReplyStreamSeq, 1))
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
func TestEinoRunEventDrainDefaultsAndStreamIDs(t *testing.T) {
|
||||
drain := newEinoRunEventDrain(einoRunEventDrainConfig{
|
||||
ConversationID: "conv-1",
|
||||
OrchestratorName: "lead",
|
||||
BaseMessages: []adk.Message{schema.UserMessage("base")},
|
||||
})
|
||||
|
||||
if drain.RunMessages().BaseCount() != 1 {
|
||||
t.Fatalf("base count = %d, want 1", drain.RunMessages().BaseCount())
|
||||
}
|
||||
if !drain.cfg.StreamsMainAssistant("lead") || drain.cfg.StreamsMainAssistant("worker") {
|
||||
t.Fatal("default main-assistant predicate should match only orchestrator")
|
||||
}
|
||||
if got := drain.cfg.EinoRoleTag("lead"); got != "orchestrator" {
|
||||
t.Fatalf("lead role = %q, want orchestrator", got)
|
||||
}
|
||||
if got := drain.cfg.EinoRoleTag("worker"); got != "sub" {
|
||||
t.Fatalf("worker role = %q, want sub", got)
|
||||
}
|
||||
if got := drain.nextMainStreamID(); got != "eino-main-conv-1-1" {
|
||||
t.Fatalf("first main stream id = %q", got)
|
||||
}
|
||||
if got := drain.nextMainStreamID(); got != "eino-main-conv-1-2" {
|
||||
t.Fatalf("second main stream id = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunEventDrainBindsHandlersAndRecordsEvents(t *testing.T) {
|
||||
var events []string
|
||||
recovered := false
|
||||
drain := newEinoRunEventDrain(einoRunEventDrainConfig{
|
||||
ConversationID: "conv-1",
|
||||
OrchMode: "deep",
|
||||
OrchestratorName: "lead",
|
||||
Progress: func(eventType, _ string, _ interface{}) {
|
||||
events = append(events, eventType)
|
||||
},
|
||||
BaseMessages: []adk.Message{schema.UserMessage("base")},
|
||||
})
|
||||
drain.BindHandlers(func() { recovered = true })
|
||||
|
||||
drain.ObserveAgent("lead")
|
||||
if !drain.HandleMaterialized(&adk.MessageVariant{Role: schema.Assistant}, schema.AssistantMessage("done", nil), "lead") {
|
||||
t.Fatal("materialized assistant should be handled")
|
||||
}
|
||||
if got := drain.AssistantOutput().LastAssistant(); got != "done" {
|
||||
t.Fatalf("last assistant = %q, want done", got)
|
||||
}
|
||||
|
||||
stream := schema.StreamReaderFromArray([]*schema.Message{
|
||||
{Role: schema.Tool, Content: "ok", ToolCallID: "call-1"},
|
||||
})
|
||||
if !drain.HandleToolResultStreaming(&adk.MessageVariant{
|
||||
IsStreaming: true,
|
||||
Role: schema.Tool,
|
||||
ToolName: "execute",
|
||||
MessageStream: stream,
|
||||
}, "lead") {
|
||||
t.Fatal("streaming tool result should be handled")
|
||||
}
|
||||
if !recovered {
|
||||
t.Fatal("tool stream completion should confirm recovery")
|
||||
}
|
||||
if len(drain.RunMessages().Messages()) != 3 {
|
||||
t.Fatalf("run messages = %#v, want base + assistant + tool", drain.RunMessages().Messages())
|
||||
}
|
||||
if !containsString(events, "iteration") || !containsString(events, "response_start") || !containsString(events, "tool_result") {
|
||||
t.Fatalf("events = %#v, want iteration, response and tool_result", events)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
type einoRunMessageAccumulator struct {
|
||||
baseCount int
|
||||
msgs []adk.Message
|
||||
}
|
||||
|
||||
func newEinoRunMessageAccumulator(base []adk.Message) *einoRunMessageAccumulator {
|
||||
msgs := append([]adk.Message(nil), base...)
|
||||
return &einoRunMessageAccumulator{
|
||||
baseCount: len(msgs),
|
||||
msgs: msgs,
|
||||
}
|
||||
}
|
||||
|
||||
func (a *einoRunMessageAccumulator) Append(msg adk.Message) bool {
|
||||
if a == nil || msg == nil {
|
||||
return false
|
||||
}
|
||||
a.msgs = append(a.msgs, msg)
|
||||
return true
|
||||
}
|
||||
|
||||
func (a *einoRunMessageAccumulator) AppendToolMessage(content, toolCallID string, opts ...schema.ToolMessageOption) bool {
|
||||
if strings.TrimSpace(toolCallID) == "" {
|
||||
return false
|
||||
}
|
||||
return a.Append(schema.ToolMessage(content, toolCallID, opts...))
|
||||
}
|
||||
|
||||
func (a *einoRunMessageAccumulator) AppendAssistantText(content string) bool {
|
||||
content = strings.TrimSpace(content)
|
||||
if content == "" {
|
||||
return false
|
||||
}
|
||||
return a.Append(schema.AssistantMessage(content, nil))
|
||||
}
|
||||
|
||||
func (a *einoRunMessageAccumulator) AppendAssistantToolCalls(toolCalls []schema.ToolCall) bool {
|
||||
if len(toolCalls) == 0 {
|
||||
return false
|
||||
}
|
||||
return a.Append(schema.AssistantMessage("", toolCalls))
|
||||
}
|
||||
|
||||
func (a *einoRunMessageAccumulator) Messages() []adk.Message {
|
||||
if a == nil {
|
||||
return nil
|
||||
}
|
||||
return a.msgs
|
||||
}
|
||||
|
||||
func (a *einoRunMessageAccumulator) BaseCount() int {
|
||||
if a == nil {
|
||||
return 0
|
||||
}
|
||||
return a.baseCount
|
||||
}
|
||||
|
||||
func (a *einoRunMessageAccumulator) HasNewMessages() bool {
|
||||
return a != nil && len(a.msgs) > a.baseCount
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
func TestEinoRunMessageAccumulatorTracksBaseAndAppends(t *testing.T) {
|
||||
acc := newEinoRunMessageAccumulator([]adk.Message{schema.UserMessage("hi")})
|
||||
|
||||
if acc.BaseCount() != 1 {
|
||||
t.Fatalf("base count = %d, want 1", acc.BaseCount())
|
||||
}
|
||||
if acc.HasNewMessages() {
|
||||
t.Fatal("fresh accumulator should not have new messages")
|
||||
}
|
||||
|
||||
if !acc.AppendAssistantText(" hello ") {
|
||||
t.Fatal("assistant text should append")
|
||||
}
|
||||
if !acc.HasNewMessages() {
|
||||
t.Fatal("expected new messages after append")
|
||||
}
|
||||
msgs := acc.Messages()
|
||||
if len(msgs) != 2 || msgs[1].Role != schema.Assistant || msgs[1].Content != "hello" {
|
||||
t.Fatalf("messages = %#v", msgs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunMessageAccumulatorToolMessage(t *testing.T) {
|
||||
acc := newEinoRunMessageAccumulator(nil)
|
||||
if acc.AppendToolMessage("ignored", "") {
|
||||
t.Fatal("blank tool call id should not append")
|
||||
}
|
||||
if !acc.AppendToolMessage("result", "call-1", schema.WithToolName("execute")) {
|
||||
t.Fatal("tool message should append")
|
||||
}
|
||||
msgs := acc.Messages()
|
||||
if len(msgs) != 1 || msgs[0].Role != schema.Tool || msgs[0].Content != "result" || msgs[0].ToolCallID != "call-1" || msgs[0].ToolName != "execute" {
|
||||
t.Fatalf("tool message = %#v", msgs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunMessageAccumulatorAssistantToolCalls(t *testing.T) {
|
||||
acc := newEinoRunMessageAccumulator(nil)
|
||||
if acc.AppendAssistantToolCalls(nil) {
|
||||
t.Fatal("empty tool calls should not append")
|
||||
}
|
||||
if !acc.AppendAssistantToolCalls([]schema.ToolCall{{
|
||||
ID: "call-1",
|
||||
Function: schema.FunctionCall{
|
||||
Name: "execute",
|
||||
Arguments: `{}`,
|
||||
},
|
||||
}}) {
|
||||
t.Fatal("assistant tool calls should append")
|
||||
}
|
||||
msgs := acc.Messages()
|
||||
if len(msgs) != 1 || msgs[0].Role != schema.Assistant || len(msgs[0].ToolCalls) != 1 {
|
||||
t.Fatalf("assistant tool call message = %#v", msgs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
type einoRunProgressTracker struct {
|
||||
orchMode string
|
||||
orchestratorName string
|
||||
conversationID string
|
||||
progress func(eventType, message string, data interface{})
|
||||
|
||||
streamsMainAssistant func(agent string) bool
|
||||
einoRoleTag func(agent string) string
|
||||
|
||||
mainRound int
|
||||
lastAgent string
|
||||
toolEmitSeen map[string]struct{}
|
||||
subAgentToolStep map[string]int
|
||||
mainAgentToolStep map[string]int
|
||||
}
|
||||
|
||||
func newEinoRunProgressTracker(
|
||||
orchMode, orchestratorName, conversationID string,
|
||||
progress func(eventType, message string, data interface{}),
|
||||
streamsMainAssistant func(agent string) bool,
|
||||
einoRoleTag func(agent string) string,
|
||||
) *einoRunProgressTracker {
|
||||
if streamsMainAssistant == nil {
|
||||
streamsMainAssistant = func(agent string) bool {
|
||||
return agent == "" || agent == orchestratorName
|
||||
}
|
||||
}
|
||||
if einoRoleTag == nil {
|
||||
einoRoleTag = func(agent string) string {
|
||||
if streamsMainAssistant(agent) {
|
||||
return "orchestrator"
|
||||
}
|
||||
return "sub"
|
||||
}
|
||||
}
|
||||
return &einoRunProgressTracker{
|
||||
orchMode: orchMode,
|
||||
orchestratorName: orchestratorName,
|
||||
conversationID: conversationID,
|
||||
progress: progress,
|
||||
streamsMainAssistant: streamsMainAssistant,
|
||||
einoRoleTag: einoRoleTag,
|
||||
toolEmitSeen: make(map[string]struct{}),
|
||||
subAgentToolStep: make(map[string]int),
|
||||
mainAgentToolStep: make(map[string]int),
|
||||
}
|
||||
}
|
||||
|
||||
func (t *einoRunProgressTracker) ObserveAgent(agentName string) {
|
||||
if t == nil || strings.TrimSpace(agentName) == "" || t.progress == nil {
|
||||
return
|
||||
}
|
||||
iterEinoAgent := t.orchestratorName
|
||||
if t.orchMode == "plan_execute" {
|
||||
if a := strings.TrimSpace(agentName); a != "" {
|
||||
iterEinoAgent = a
|
||||
}
|
||||
}
|
||||
if t.streamsMainAssistant(agentName) {
|
||||
mainIterKey := einoMainIterationKey(iterEinoAgent, t.orchestratorName)
|
||||
if t.mainRound == 0 {
|
||||
t.mainRound = 1
|
||||
t.mainAgentToolStep[mainIterKey] = 1
|
||||
t.emitMainIteration(iterEinoAgent, t.mainRound)
|
||||
} else if t.lastAgent != "" {
|
||||
needBump := false
|
||||
if !t.streamsMainAssistant(t.lastAgent) {
|
||||
needBump = true
|
||||
} else if t.lastAgent != agentName {
|
||||
needBump = true
|
||||
}
|
||||
if needBump {
|
||||
t.mainRound++
|
||||
t.mainAgentToolStep[mainIterKey] = t.mainRound
|
||||
t.emitMainIteration(iterEinoAgent, t.mainRound)
|
||||
}
|
||||
}
|
||||
}
|
||||
if t.lastAgent != agentName {
|
||||
t.progress("progress", fmt.Sprintf("[Eino] %s", agentName), map[string]interface{}{
|
||||
"conversationId": t.conversationID,
|
||||
"einoAgent": agentName,
|
||||
"einoRole": t.einoRoleTag(agentName),
|
||||
"orchestration": t.orchMode,
|
||||
})
|
||||
}
|
||||
t.lastAgent = agentName
|
||||
}
|
||||
|
||||
func (t *einoRunProgressTracker) MainIteration(agentName string) int {
|
||||
if t == nil {
|
||||
return 0
|
||||
}
|
||||
key := einoMainIterationKey(agentName, t.orchestratorName)
|
||||
if n := t.mainAgentToolStep[key]; n > 0 {
|
||||
return n
|
||||
}
|
||||
return t.mainRound
|
||||
}
|
||||
|
||||
func (t *einoRunProgressTracker) EmitToolCalls(msg *schema.Message, agentName string, markPending func(toolCallPendingInfo)) {
|
||||
if t == nil {
|
||||
return
|
||||
}
|
||||
before := t.MainIteration(agentName)
|
||||
tryEmitToolCallsOnce(
|
||||
msg,
|
||||
agentName,
|
||||
t.orchestratorName,
|
||||
t.conversationID,
|
||||
t.orchMode,
|
||||
t.progress,
|
||||
t.toolEmitSeen,
|
||||
t.subAgentToolStep,
|
||||
t.mainAgentToolStep,
|
||||
markPending,
|
||||
)
|
||||
if t.streamsMainAssistant(agentName) {
|
||||
if after := t.MainIteration(agentName); after > before {
|
||||
t.mainRound = after
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *einoRunProgressTracker) emitMainIteration(agentName string, iteration int) {
|
||||
t.progress("iteration", "", map[string]interface{}{
|
||||
"iteration": iteration,
|
||||
"einoScope": "main",
|
||||
"einoRole": "orchestrator",
|
||||
"einoAgent": agentName,
|
||||
"orchestration": t.orchMode,
|
||||
"conversationId": t.conversationID,
|
||||
"source": "eino",
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
func TestEinoRunProgressTrackerMainToolCallAdvancesResponseIteration(t *testing.T) {
|
||||
var events []string
|
||||
var iterations []int
|
||||
progress := func(eventType, _ string, raw interface{}) {
|
||||
events = append(events, eventType)
|
||||
data, _ := raw.(map[string]interface{})
|
||||
if eventType == "iteration" {
|
||||
if n, ok := data["iteration"].(int); ok {
|
||||
iterations = append(iterations, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
tracker := newEinoRunProgressTracker(
|
||||
"eino_single", "main", "conv-1", progress,
|
||||
func(agent string) bool { return agent == "" || agent == "main" },
|
||||
nil,
|
||||
)
|
||||
|
||||
tracker.ObserveAgent("main")
|
||||
if got := tracker.MainIteration("main"); got != 1 {
|
||||
t.Fatalf("initial main iteration = %d, want 1", got)
|
||||
}
|
||||
tracker.EmitToolCalls(&schema.Message{ToolCalls: []schema.ToolCall{{
|
||||
ID: "call-1",
|
||||
Type: "function",
|
||||
Function: schema.FunctionCall{
|
||||
Name: "execute",
|
||||
Arguments: `{"command":"pwd"}`,
|
||||
},
|
||||
}}}, "main", nil)
|
||||
if got := tracker.MainIteration("main"); got != 2 {
|
||||
t.Fatalf("post-tool main iteration = %d, want 2", got)
|
||||
}
|
||||
if len(iterations) != 2 || iterations[0] != 1 || iterations[1] != 2 {
|
||||
t.Fatalf("iteration events = %#v, want [1 2]; events=%#v", iterations, events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunProgressTrackerMainAgentSwitchAdvancesIteration(t *testing.T) {
|
||||
var iterations []int
|
||||
progress := func(eventType, _ string, raw interface{}) {
|
||||
if eventType != "iteration" {
|
||||
return
|
||||
}
|
||||
data, _ := raw.(map[string]interface{})
|
||||
if n, ok := data["iteration"].(int); ok {
|
||||
iterations = append(iterations, n)
|
||||
}
|
||||
}
|
||||
tracker := newEinoRunProgressTracker(
|
||||
"supervisor", "lead", "conv-1", progress,
|
||||
func(agent string) bool { return agent == "" || agent == "lead" },
|
||||
nil,
|
||||
)
|
||||
|
||||
tracker.ObserveAgent("lead")
|
||||
tracker.ObserveAgent("sub")
|
||||
tracker.ObserveAgent("lead")
|
||||
|
||||
if got := tracker.MainIteration("lead"); got != 2 {
|
||||
t.Fatalf("main iteration after sub->main = %d, want 2", got)
|
||||
}
|
||||
if len(iterations) != 2 || iterations[0] != 1 || iterations[1] != 2 {
|
||||
t.Fatalf("iteration events = %#v, want [1 2]", iterations)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunProgressTrackerDedupesToolCalls(t *testing.T) {
|
||||
var toolCalls int
|
||||
progress := func(eventType, _ string, _ interface{}) {
|
||||
if eventType == "tool_call" {
|
||||
toolCalls++
|
||||
}
|
||||
}
|
||||
tracker := newEinoRunProgressTracker("deep", "lead", "conv-1", progress, nil, nil)
|
||||
msg := &schema.Message{ToolCalls: []schema.ToolCall{{
|
||||
ID: "call-1",
|
||||
Type: "function",
|
||||
Function: schema.FunctionCall{
|
||||
Name: "search",
|
||||
Arguments: `{"q":"x"}`,
|
||||
},
|
||||
}}}
|
||||
|
||||
tracker.EmitToolCalls(msg, "lead", nil)
|
||||
tracker.EmitToolCalls(msg, "lead", nil)
|
||||
|
||||
if toolCalls != 1 {
|
||||
t.Fatalf("tool call events = %d, want 1", toolCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunProgressTrackerHidesModelOutputRecoveryToolCalls(t *testing.T) {
|
||||
var eventTypes []string
|
||||
var marked []toolCallPendingInfo
|
||||
progress := func(eventType, _ string, _ interface{}) {
|
||||
eventTypes = append(eventTypes, eventType)
|
||||
}
|
||||
tracker := newEinoRunProgressTracker("deep", "lead", "conv-1", progress, nil, nil)
|
||||
msg := &schema.Message{ToolCalls: []schema.ToolCall{{
|
||||
ID: "call-recovery",
|
||||
Type: "function",
|
||||
Function: schema.FunctionCall{
|
||||
Name: "task",
|
||||
Arguments: `{"_cyberstrike_model_output_recovery":{"reason":"invalid_tool_arguments_json","repair_attempt":1}}`,
|
||||
},
|
||||
}}}
|
||||
|
||||
tracker.EmitToolCalls(msg, "lead", func(info toolCallPendingInfo) {
|
||||
marked = append(marked, info)
|
||||
})
|
||||
|
||||
if containsString(eventTypes, "tool_calls_detected") || containsString(eventTypes, "tool_call") {
|
||||
t.Fatalf("event types = %#v, want no visible recovery tool call events", eventTypes)
|
||||
}
|
||||
if len(marked) != 0 {
|
||||
t.Fatalf("marked pending = %#v, want none", marked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunProgressTrackerHidesAnonymousToolCallFragments(t *testing.T) {
|
||||
var eventTypes []string
|
||||
var marked []toolCallPendingInfo
|
||||
progress := func(eventType, _ string, _ interface{}) {
|
||||
eventTypes = append(eventTypes, eventType)
|
||||
}
|
||||
tracker := newEinoRunProgressTracker("eino_single", "lead", "conv-1", progress, nil, nil)
|
||||
idx := 0
|
||||
msg := &schema.Message{ToolCalls: []schema.ToolCall{{
|
||||
Type: "function",
|
||||
Index: &idx,
|
||||
Function: schema.FunctionCall{
|
||||
Arguments: `"`,
|
||||
},
|
||||
}}}
|
||||
|
||||
tracker.EmitToolCalls(msg, "lead", func(info toolCallPendingInfo) {
|
||||
marked = append(marked, info)
|
||||
})
|
||||
|
||||
if containsString(eventTypes, "tool_calls_detected") || containsString(eventTypes, "tool_call") {
|
||||
t.Fatalf("event types = %#v, want no visible anonymous fragment tool call events", eventTypes)
|
||||
}
|
||||
if len(marked) != 0 {
|
||||
t.Fatalf("marked pending = %#v, want none", marked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunProgressTrackerKeepsNamedInvalidToolCallsVisible(t *testing.T) {
|
||||
var toolCalls int
|
||||
var marked []toolCallPendingInfo
|
||||
progress := func(eventType, _ string, _ interface{}) {
|
||||
if eventType == "tool_call" {
|
||||
toolCalls++
|
||||
}
|
||||
}
|
||||
tracker := newEinoRunProgressTracker("eino_single", "lead", "conv-1", progress, nil, nil)
|
||||
msg := &schema.Message{ToolCalls: []schema.ToolCall{{
|
||||
ID: "call-bad-args",
|
||||
Type: "function",
|
||||
Function: schema.FunctionCall{
|
||||
Name: "exec",
|
||||
Arguments: `command`,
|
||||
},
|
||||
}}}
|
||||
|
||||
tracker.EmitToolCalls(msg, "lead", func(info toolCallPendingInfo) {
|
||||
marked = append(marked, info)
|
||||
})
|
||||
|
||||
if toolCalls != 1 {
|
||||
t.Fatalf("tool call events = %d, want 1", toolCalls)
|
||||
}
|
||||
if len(marked) != 1 || marked[0].ToolName != "exec" {
|
||||
t.Fatalf("marked pending = %#v, want one exec call", marked)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type einoRunRecoveryHandlerConfig struct {
|
||||
ConversationID string
|
||||
OrchMode string
|
||||
Args *einoADKRunLoopArgs
|
||||
BaseMsgs []adk.Message
|
||||
Progress func(eventType, message string, data interface{})
|
||||
Logger *zap.Logger
|
||||
RunError *einoRunErrorHandler
|
||||
ContextOverflow *einoContextOverflowRetryHandler
|
||||
Transient *einoTransientRunRetryHandler
|
||||
}
|
||||
|
||||
type einoRunRecoveryResult struct {
|
||||
Handled bool
|
||||
Restarted bool
|
||||
RestartMsgs []adk.Message
|
||||
Fatal error
|
||||
}
|
||||
|
||||
type einoRunRecoveryHandler struct {
|
||||
cfg einoRunRecoveryHandlerConfig
|
||||
}
|
||||
|
||||
func newEinoRunRecoveryHandler(cfg einoRunRecoveryHandlerConfig) *einoRunRecoveryHandler {
|
||||
if cfg.Args == nil {
|
||||
cfg.Args = &einoADKRunLoopArgs{}
|
||||
}
|
||||
return &einoRunRecoveryHandler{cfg: cfg}
|
||||
}
|
||||
|
||||
func (h *einoRunRecoveryHandler) Handle(runErr error, accumulated []adk.Message, baseCount int) einoRunRecoveryResult {
|
||||
if h == nil || runErr == nil {
|
||||
return einoRunRecoveryResult{}
|
||||
}
|
||||
if willRetry, ok := isEinoNativeWillRetry(runErr); ok {
|
||||
emitEinoNativeModelRetryProgress(h.cfg.ConversationID, h.cfg.OrchMode, willRetry, h.cfg.Progress, h.cfg.Logger, runErr)
|
||||
return einoRunRecoveryResult{Handled: true}
|
||||
}
|
||||
if h.cfg.ContextOverflow != nil {
|
||||
if overflowRetry := h.cfg.ContextOverflow.Prepare(runErr, accumulated, baseCount); overflowRetry.Handled {
|
||||
return einoRunRecoveryResult{Handled: true, Restarted: true, RestartMsgs: overflowRetry.RestartMsgs}
|
||||
}
|
||||
}
|
||||
if h.cfg.Transient != nil {
|
||||
if runRetry := h.cfg.Transient.Prepare(runErr, accumulated, baseCount); runRetry.Handled {
|
||||
if runRetry.Fatal != nil {
|
||||
return einoRunRecoveryResult{Handled: true, Fatal: runRetry.Fatal}
|
||||
}
|
||||
if !runRetry.Restarted {
|
||||
return einoRunRecoveryResult{Handled: true}
|
||||
}
|
||||
return einoRunRecoveryResult{Handled: true, Restarted: true, RestartMsgs: runRetry.RestartMsgs}
|
||||
}
|
||||
}
|
||||
return einoRunRecoveryResult{Handled: true, Fatal: h.handleFatal(runErr)}
|
||||
}
|
||||
|
||||
func (h *einoRunRecoveryHandler) handleFatal(runErr error) error {
|
||||
if h != nil && h.cfg.RunError != nil {
|
||||
return h.cfg.RunError.Handle(runErr)
|
||||
}
|
||||
return runErr
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
func TestEinoRunRecoveryHandlerRoutesContextOverflowBeforeTransient(t *testing.T) {
|
||||
baseMsgs := []adk.Message{schema.UserMessage("base")}
|
||||
overflow := newEinoContextOverflowRetryHandler(einoContextOverflowRetryConfig{
|
||||
Context: context.Background(),
|
||||
Args: &einoADKRunLoopArgs{},
|
||||
BaseMsgs: baseMsgs,
|
||||
})
|
||||
transient := newEinoTransientRunRetryHandler(einoTransientRunRetryHandlerConfig{
|
||||
Args: &einoADKRunLoopArgs{},
|
||||
BaseMsgs: baseMsgs,
|
||||
Policy: einoTransientRunRetryPolicy{maxAttempts: 1, maxBackoff: time.Nanosecond},
|
||||
})
|
||||
handler := newEinoRunRecoveryHandler(einoRunRecoveryHandlerConfig{
|
||||
ContextOverflow: overflow,
|
||||
Transient: transient,
|
||||
BaseMsgs: baseMsgs,
|
||||
})
|
||||
|
||||
result := handler.Handle(errors.New("context length exceeded: upstream returned 503"), nil, 0)
|
||||
if !result.Handled || !result.Restarted || result.Fatal != nil {
|
||||
t.Fatalf("result = %+v, want context overflow restart", result)
|
||||
}
|
||||
second := handler.Handle(errors.New("upstream returned 503"), nil, 0)
|
||||
if !second.Handled || !second.Restarted || second.Fatal != nil {
|
||||
t.Fatalf("second result = %+v, want transient restart", second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunRecoveryHandlerRoutesFatalFallback(t *testing.T) {
|
||||
handler := newEinoRunRecoveryHandler(einoRunRecoveryHandlerConfig{
|
||||
RunError: newEinoRunErrorHandler(einoRunErrorHandlerConfig{}),
|
||||
})
|
||||
result := handler.Handle(errors.New("invalid api key"), nil, 0)
|
||||
if !result.Handled || result.Restarted || result.Fatal == nil {
|
||||
t.Fatalf("result = %+v, want fatal fallback", result)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"cyberstrike-ai/internal/agent"
|
||||
"cyberstrike-ai/internal/einomcp"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
type einoRunResultBuilderConfig struct {
|
||||
OrchMode string
|
||||
EmptyHint string
|
||||
RunMessages *einoRunMessageAccumulator
|
||||
AssistantOutput *einoAssistantOutputAccumulator
|
||||
SnapshotMCPIDs func() []string
|
||||
ModelFacingTrace func() []adk.Message
|
||||
}
|
||||
|
||||
type einoRunResultBuilder struct {
|
||||
cfg einoRunResultBuilderConfig
|
||||
}
|
||||
|
||||
func newEinoRunResultBuilder(cfg einoRunResultBuilderConfig) *einoRunResultBuilder {
|
||||
return &einoRunResultBuilder{cfg: cfg}
|
||||
}
|
||||
|
||||
func (b *einoRunResultBuilder) BuildPartial(runErr error) (*RunResult, error) {
|
||||
if b == nil || b.cfg.RunMessages == nil || !b.cfg.RunMessages.HasNewMessages() {
|
||||
return nil, runErr
|
||||
}
|
||||
return b.build(true), runErr
|
||||
}
|
||||
|
||||
func (b *einoRunResultBuilder) BuildFinal() *RunResult {
|
||||
if b == nil {
|
||||
return &RunResult{}
|
||||
}
|
||||
return b.build(false)
|
||||
}
|
||||
|
||||
func (b *einoRunResultBuilder) build(partial bool) *RunResult {
|
||||
var runMsgs []adk.Message
|
||||
if b.cfg.RunMessages != nil {
|
||||
runMsgs = b.cfg.RunMessages.Messages()
|
||||
}
|
||||
var lastAssistant string
|
||||
var lastPlanExecuteExecutor string
|
||||
if b.cfg.AssistantOutput != nil {
|
||||
lastAssistant = b.cfg.AssistantOutput.LastAssistant()
|
||||
lastPlanExecuteExecutor = b.cfg.AssistantOutput.LastPlanExecuteExecutor()
|
||||
}
|
||||
var modelFacing []adk.Message
|
||||
if b.cfg.ModelFacingTrace != nil {
|
||||
modelFacing = b.cfg.ModelFacingTrace()
|
||||
}
|
||||
var ids []string
|
||||
if b.cfg.SnapshotMCPIDs != nil {
|
||||
ids = b.cfg.SnapshotMCPIDs()
|
||||
}
|
||||
return buildEinoRunResultFromAccumulated(
|
||||
b.cfg.OrchMode,
|
||||
runMsgs,
|
||||
modelFacing,
|
||||
lastAssistant,
|
||||
lastPlanExecuteExecutor,
|
||||
b.cfg.EmptyHint,
|
||||
ids,
|
||||
partial,
|
||||
)
|
||||
}
|
||||
|
||||
func einoPartialRunLastOutputHint() string {
|
||||
return "[执行未正常结束(用户停止、超时或异常)。续跑时请基于上文已产生的工具与结果继续,勿重复已完成步骤。]\n" +
|
||||
"[Run ended abnormally; continue from the trace above without repeating completed steps.]"
|
||||
}
|
||||
|
||||
func buildEinoRunResultFromAccumulated(
|
||||
orchMode string,
|
||||
runAccumulatedMsgs []adk.Message,
|
||||
persistMsgs []adk.Message,
|
||||
lastAssistant string,
|
||||
lastPlanExecuteExecutor string,
|
||||
emptyHint string,
|
||||
mcpIDs []string,
|
||||
partial bool,
|
||||
) *RunResult {
|
||||
traceForJSON := persistMsgs
|
||||
traceJSON := ""
|
||||
if len(traceForJSON) > 0 {
|
||||
traceForJSON = markModelFacingTraceForPersistence(traceForJSON)
|
||||
if histJSON, err := json.Marshal(traceForJSON); err == nil {
|
||||
traceJSON = string(histJSON)
|
||||
}
|
||||
}
|
||||
cleaned := strings.TrimSpace(lastAssistant)
|
||||
if orchMode == "plan_execute" {
|
||||
if e := strings.TrimSpace(lastPlanExecuteExecutor); e != "" {
|
||||
cleaned = e
|
||||
} else {
|
||||
cleaned = UnwrapPlanExecuteUserText(cleaned)
|
||||
}
|
||||
}
|
||||
if cleaned == "" {
|
||||
if fb := strings.TrimSpace(einoExtractFallbackAssistantFromMsgs(runAccumulatedMsgs)); fb != "" {
|
||||
cleaned = fb
|
||||
}
|
||||
}
|
||||
cleaned = dedupeRepeatedParagraphs(cleaned, 80)
|
||||
cleaned = dedupeParagraphsByLineFingerprint(cleaned, 100)
|
||||
const maxResponseRunes = 100000
|
||||
if rs := []rune(cleaned); len(rs) > maxResponseRunes {
|
||||
cleaned = string(rs[:maxResponseRunes]) + "\n\n... (response truncated / 响应已截断)"
|
||||
}
|
||||
lastOut := cleaned
|
||||
resp := cleaned
|
||||
if partial && cleaned == "" {
|
||||
lastOut = einoPartialRunLastOutputHint()
|
||||
resp = emptyHint
|
||||
}
|
||||
out := &RunResult{
|
||||
Response: resp,
|
||||
MCPExecutionIDs: mcpIDs,
|
||||
LastAgentTraceInput: traceJSON,
|
||||
LastAgentTraceOutput: lastOut,
|
||||
}
|
||||
if !partial && out.Response == "" {
|
||||
out.Response = emptyHint
|
||||
out.LastAgentTraceOutput = out.Response
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func markModelFacingTraceForPersistence(msgs []adk.Message) []adk.Message {
|
||||
out := cloneADKMessagesForTrace(msgs)
|
||||
if len(out) == 0 || out[0] == nil {
|
||||
return out
|
||||
}
|
||||
if out[0].Extra == nil {
|
||||
out[0].Extra = make(map[string]any, 1)
|
||||
}
|
||||
out[0].Extra[agent.ModelFacingTraceVersionKey] = 1
|
||||
return out
|
||||
}
|
||||
|
||||
// einoExtractFallbackAssistantFromMsgs 在「主通道未产出助手正文」时,从 Eino ADK 轨迹中回填用户可见回复。
|
||||
// 典型场景:监督者仅调用 exit(final_result 落在 Tool 消息中),或工具结果已写入历史但 lastAssistant 未更新。
|
||||
//
|
||||
// 优先级:最后一次 exit 工具输出 → 最后一条含 exit 的助手 tool_calls 参数中的 final_result。
|
||||
func einoExtractFallbackAssistantFromMsgs(msgs []adk.Message) string {
|
||||
for i := len(msgs) - 1; i >= 0; i-- {
|
||||
m := msgs[i]
|
||||
if m == nil || m.Role != schema.Tool {
|
||||
continue
|
||||
}
|
||||
if !strings.EqualFold(strings.TrimSpace(m.ToolName), adk.ToolInfoExit.Name) {
|
||||
continue
|
||||
}
|
||||
content := strings.TrimSpace(m.Content)
|
||||
if content == "" || strings.HasPrefix(content, einomcp.ToolErrorPrefix) {
|
||||
continue
|
||||
}
|
||||
return content
|
||||
}
|
||||
for i := len(msgs) - 1; i >= 0; i-- {
|
||||
m := msgs[i]
|
||||
if m == nil || m.Role != schema.Assistant {
|
||||
continue
|
||||
}
|
||||
if s := einoExtractExitFinalFromAssistantToolCalls(m); s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func einoExtractExitFinalFromAssistantToolCalls(msg *schema.Message) string {
|
||||
if msg == nil || len(msg.ToolCalls) == 0 {
|
||||
return ""
|
||||
}
|
||||
for i := len(msg.ToolCalls) - 1; i >= 0; i-- {
|
||||
tc := msg.ToolCalls[i]
|
||||
if !strings.EqualFold(strings.TrimSpace(tc.Function.Name), adk.ToolInfoExit.Name) {
|
||||
continue
|
||||
}
|
||||
if s := einoParseExitFinalResultArguments(tc.Function.Arguments); s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func einoParseExitFinalResultArguments(arguments string) string {
|
||||
arguments = strings.TrimSpace(arguments)
|
||||
if arguments == "" {
|
||||
return ""
|
||||
}
|
||||
var wrap struct {
|
||||
FinalResult json.RawMessage `json:"final_result"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(arguments), &wrap); err != nil || len(wrap.FinalResult) == 0 {
|
||||
return ""
|
||||
}
|
||||
var s string
|
||||
if err := json.Unmarshal(wrap.FinalResult, &s); err == nil {
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
var anyVal interface{}
|
||||
if err := json.Unmarshal(wrap.FinalResult, &anyVal); err != nil {
|
||||
return ""
|
||||
}
|
||||
b, err := json.Marshal(anyVal)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
func TestEinoRunResultBuilderPartialWithoutNewMessagesReturnsOriginalError(t *testing.T) {
|
||||
runMessages := newEinoRunMessageAccumulator([]adk.Message{schema.UserMessage("base")})
|
||||
wantErr := errors.New("stream failed")
|
||||
|
||||
got, err := newEinoRunResultBuilder(einoRunResultBuilderConfig{
|
||||
RunMessages: runMessages,
|
||||
EmptyHint: "empty",
|
||||
}).BuildPartial(wantErr)
|
||||
|
||||
if got != nil {
|
||||
t.Fatalf("partial result = %#v, want nil", got)
|
||||
}
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("err = %v, want %v", err, wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunResultBuilderFinalUsesSnapshots(t *testing.T) {
|
||||
runMessages := newEinoRunMessageAccumulator([]adk.Message{schema.UserMessage("base")})
|
||||
runMessages.Append(schema.AssistantMessage("assistant done", nil))
|
||||
assistantOutput := newEinoAssistantOutputAccumulator("deep")
|
||||
assistantOutput.RecordMainAssistant("orchestrator", "assistant done")
|
||||
|
||||
got := newEinoRunResultBuilder(einoRunResultBuilderConfig{
|
||||
OrchMode: "deep",
|
||||
EmptyHint: "empty",
|
||||
RunMessages: runMessages,
|
||||
AssistantOutput: assistantOutput,
|
||||
SnapshotMCPIDs: func() []string {
|
||||
return []string{"exec-1"}
|
||||
},
|
||||
ModelFacingTrace: func() []adk.Message {
|
||||
return []adk.Message{schema.UserMessage("model-facing")}
|
||||
},
|
||||
}).BuildFinal()
|
||||
|
||||
if got.Response != "assistant done" {
|
||||
t.Fatalf("response = %q, want assistant done", got.Response)
|
||||
}
|
||||
if len(got.MCPExecutionIDs) != 1 || got.MCPExecutionIDs[0] != "exec-1" {
|
||||
t.Fatalf("mcp ids = %#v", got.MCPExecutionIDs)
|
||||
}
|
||||
if got.LastAgentTraceInput == "" {
|
||||
t.Fatal("model-facing trace should be persisted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunResultBuilderPlanExecutePrefersExecutorOutput(t *testing.T) {
|
||||
runMessages := newEinoRunMessageAccumulator(nil)
|
||||
runMessages.Append(schema.AssistantMessage(`{"response":"planner text"}`, nil))
|
||||
assistantOutput := newEinoAssistantOutputAccumulator("plan_execute")
|
||||
assistantOutput.RecordMainAssistant("planner", `{"response":"planner text"}`)
|
||||
assistantOutput.RecordMainAssistant("executor", `{"response":"executor text"}`)
|
||||
|
||||
got := newEinoRunResultBuilder(einoRunResultBuilderConfig{
|
||||
OrchMode: "plan_execute",
|
||||
EmptyHint: "empty",
|
||||
RunMessages: runMessages,
|
||||
AssistantOutput: assistantOutput,
|
||||
}).BuildFinal()
|
||||
|
||||
if got.Response != "executor text" {
|
||||
t.Fatalf("response = %q, want executor text", got.Response)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,372 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type einoRunRuntimeSessionConfig struct {
|
||||
Context context.Context
|
||||
Args *einoADKRunLoopArgs
|
||||
Drain *einoRunEventDrain
|
||||
BaseMessages []adk.Message
|
||||
EmptyHint string
|
||||
SnapshotMCPIDs func() []string
|
||||
EinoRoleTag func(agent string) string
|
||||
}
|
||||
|
||||
type einoRunRuntimeErrorResult struct {
|
||||
Restarted bool
|
||||
Result *RunResult
|
||||
Err error
|
||||
}
|
||||
|
||||
type einoRunRuntimeSession struct {
|
||||
ctx context.Context
|
||||
args *einoADKRunLoopArgs
|
||||
orchMode string
|
||||
conversationID string
|
||||
progress func(eventType, message string, data interface{})
|
||||
logger *zap.Logger
|
||||
baseMsgs []adk.Message
|
||||
msgs []adk.Message
|
||||
drain *einoRunEventDrain
|
||||
runMessages *einoRunMessageAccumulator
|
||||
usage *einoRunUsageAccumulator
|
||||
|
||||
iter *adk.AsyncIterator[*adk.AgentEvent]
|
||||
startFreshIter einoAgentEventIteratorStarter
|
||||
|
||||
unregisterAgentCancel func()
|
||||
unregisterTurnLoopInterrupt func()
|
||||
nativeCancelCause atomic.Value
|
||||
|
||||
transientRetry *einoTransientRunRetryHandler
|
||||
runRecoveryHandler *einoRunRecoveryHandler
|
||||
resultBuilder *einoRunResultBuilder
|
||||
streamErrorHandler *einoStreamErrorHandler
|
||||
completionHandler *einoRunCompletionHandler
|
||||
cancellationHandler *einoRunCancellationHandler
|
||||
}
|
||||
|
||||
func newEinoRunRuntimeSession(cfg einoRunRuntimeSessionConfig) *einoRunRuntimeSession {
|
||||
if cfg.Context == nil {
|
||||
cfg.Context = context.Background()
|
||||
}
|
||||
if cfg.Args == nil {
|
||||
cfg.Args = &einoADKRunLoopArgs{}
|
||||
}
|
||||
if cfg.SnapshotMCPIDs == nil {
|
||||
cfg.SnapshotMCPIDs = func() []string { return nil }
|
||||
}
|
||||
s := &einoRunRuntimeSession{
|
||||
ctx: cfg.Context,
|
||||
args: cfg.Args,
|
||||
orchMode: cfg.Args.OrchMode,
|
||||
conversationID: cfg.Args.ConversationID,
|
||||
progress: cfg.Args.Progress,
|
||||
logger: cfg.Args.Logger,
|
||||
baseMsgs: cfg.BaseMessages,
|
||||
msgs: append([]adk.Message(nil), cfg.BaseMessages...),
|
||||
drain: cfg.Drain,
|
||||
}
|
||||
if s.drain != nil {
|
||||
s.runMessages = s.drain.RunMessages()
|
||||
s.usage = s.drain.Usage()
|
||||
}
|
||||
if s.runMessages == nil {
|
||||
s.runMessages = newEinoRunMessageAccumulator(s.msgs)
|
||||
}
|
||||
s.initIteratorRuntime()
|
||||
s.initRecoveryRuntime()
|
||||
s.initResultRuntime(cfg.EmptyHint, cfg.SnapshotMCPIDs, cfg.EinoRoleTag)
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *einoRunRuntimeSession) Iterator() *adk.AsyncIterator[*adk.AgentEvent] {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
return s.iter
|
||||
}
|
||||
|
||||
func (s *einoRunRuntimeSession) Close() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
callAndClearUnregister(&s.unregisterAgentCancel)
|
||||
callAndClearUnregister(&s.unregisterTurnLoopInterrupt)
|
||||
}
|
||||
|
||||
func (s *einoRunRuntimeSession) HandleIteratorContextError(err error) (*RunResult, error) {
|
||||
if s == nil || s.cancellationHandler == nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.cancellationHandler.Handle(err)
|
||||
}
|
||||
|
||||
func (s *einoRunRuntimeSession) HandleIteratorEnd() (completed bool, result *RunResult, err error) {
|
||||
if s == nil {
|
||||
return true, nil, nil
|
||||
}
|
||||
if ctxErr := s.ctx.Err(); ctxErr != nil {
|
||||
result, err = s.HandleIteratorContextError(ctxErr)
|
||||
return false, result, err
|
||||
}
|
||||
if s.completionHandler != nil {
|
||||
s.completionHandler.Complete()
|
||||
}
|
||||
return true, nil, nil
|
||||
}
|
||||
|
||||
func (s *einoRunRuntimeSession) HandleRunError(runErr error) einoRunRuntimeErrorResult {
|
||||
if s == nil || runErr == nil {
|
||||
return einoRunRuntimeErrorResult{}
|
||||
}
|
||||
restarted, fatal := s.maybeRestart(runErr)
|
||||
if fatal != nil {
|
||||
result, err := s.takePartial(fatal)
|
||||
return einoRunRuntimeErrorResult{Result: result, Err: err}
|
||||
}
|
||||
return einoRunRuntimeErrorResult{Restarted: restarted}
|
||||
}
|
||||
|
||||
func (s *einoRunRuntimeSession) HandleStreamError(streamErr error, agentName string) einoRunRuntimeErrorResult {
|
||||
if s == nil || s.streamErrorHandler == nil || streamErr == nil {
|
||||
return einoRunRuntimeErrorResult{}
|
||||
}
|
||||
handled := s.streamErrorHandler.Handle(streamErr, agentName)
|
||||
return einoRunRuntimeErrorResult{
|
||||
Restarted: handled.Restarted,
|
||||
Result: handled.Result,
|
||||
Err: handled.Err,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *einoRunRuntimeSession) ConfirmRecovery() {
|
||||
if s != nil && s.transientRetry != nil {
|
||||
s.transientRetry.ConfirmRecovery()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *einoRunRuntimeSession) BuildFinalResult() *RunResult {
|
||||
if s == nil || s.resultBuilder == nil {
|
||||
return &RunResult{}
|
||||
}
|
||||
s.emitUsageSummary("final")
|
||||
return s.resultBuilder.BuildFinal()
|
||||
}
|
||||
|
||||
func (s *einoRunRuntimeSession) takePartial(err error) (*RunResult, error) {
|
||||
if s == nil || s.resultBuilder == nil {
|
||||
return nil, err
|
||||
}
|
||||
s.emitUsageSummary("partial")
|
||||
return s.resultBuilder.BuildPartial(err)
|
||||
}
|
||||
|
||||
func (s *einoRunRuntimeSession) maybeRestart(runErr error) (restarted bool, fatal error) {
|
||||
if s == nil || s.runRecoveryHandler == nil {
|
||||
return false, runErr
|
||||
}
|
||||
recovery := s.runRecoveryHandler.Handle(runErr, s.runMessages.Messages(), s.runMessages.BaseCount())
|
||||
if recovery.Fatal != nil {
|
||||
return false, recovery.Fatal
|
||||
}
|
||||
if !recovery.Restarted {
|
||||
return false, nil
|
||||
}
|
||||
s.msgs = recovery.RestartMsgs
|
||||
s.iter = s.startFreshIter(s.msgs)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *einoRunRuntimeSession) initIteratorRuntime() {
|
||||
if s == nil || s.args == nil {
|
||||
return
|
||||
}
|
||||
runnerCfg := adk.RunnerConfig{
|
||||
Agent: s.args.DA,
|
||||
// 启用 ADK 流式事件:plan_execute 也需要输出 reasoning/response 流,
|
||||
// 与 deep/supervisor/eino_single 的前端体验保持一致。
|
||||
EnableStreaming: true,
|
||||
}
|
||||
var cpStore *fileCheckPointStore
|
||||
var checkPointID string
|
||||
if checkpoint := newEinoCheckpointRuntime(s.args.CheckpointDir, s.conversationID, s.orchMode, s.logger); checkpoint != nil {
|
||||
cpStore = checkpoint.Store
|
||||
checkPointID = checkpoint.CheckPointID
|
||||
runnerCfg.CheckPointStore = checkpoint.Store
|
||||
}
|
||||
runner := adk.NewRunner(s.ctx, runnerCfg)
|
||||
runtimeCancelRegistrar := agentRuntimeCancelRegistrarFromContext(s.ctx)
|
||||
turnLoopInterruptRegistrar := agentTurnLoopInterruptRegistrarFromContext(s.ctx)
|
||||
runnerStarter := newEinoRunnerIteratorStarter(einoRunnerIteratorStarterConfig{
|
||||
Context: s.ctx,
|
||||
ConversationID: s.conversationID,
|
||||
OrchMode: s.orchMode,
|
||||
Logger: s.logger,
|
||||
Runner: runner,
|
||||
CheckPointID: checkPointID,
|
||||
NativeCancelCause: &s.nativeCancelCause,
|
||||
UnregisterAgentCancel: &s.unregisterAgentCancel,
|
||||
RuntimeCancelRegistrar: runtimeCancelRegistrar,
|
||||
})
|
||||
turnLoopStarter := newEinoTurnLoopIteratorStarter(einoTurnLoopIteratorStarterConfig{
|
||||
Context: s.ctx,
|
||||
Agent: s.args.DA,
|
||||
ConversationID: s.conversationID,
|
||||
OrchMode: s.orchMode,
|
||||
Progress: s.progress,
|
||||
Logger: s.logger,
|
||||
Store: cpStore,
|
||||
CheckPointID: checkPointID,
|
||||
InterruptTimeout: s.args.TurnLoopInterruptTimeout,
|
||||
NativeCancelCause: &s.nativeCancelCause,
|
||||
UnregisterAgentCancel: &s.unregisterAgentCancel,
|
||||
UnregisterTurnLoopInterrupt: &s.unregisterTurnLoopInterrupt,
|
||||
RuntimeCancelRegistrar: runtimeCancelRegistrar,
|
||||
TurnLoopInterruptRegistrar: turnLoopInterruptRegistrar,
|
||||
})
|
||||
useTurnLoop := turnLoopInterruptRegistrar != nil
|
||||
s.startFreshIter = runnerStarter.Start
|
||||
if useTurnLoop {
|
||||
s.startFreshIter = turnLoopStarter.Start
|
||||
}
|
||||
if !useTurnLoop && cpStore != nil && checkPointID != "" {
|
||||
s.iter = newEinoCheckpointResumeHandler(einoCheckpointResumeHandlerConfig{
|
||||
Context: s.ctx,
|
||||
ConversationID: s.conversationID,
|
||||
OrchMode: s.orchMode,
|
||||
Progress: s.progress,
|
||||
Logger: s.logger,
|
||||
Store: cpStore,
|
||||
CheckPointID: checkPointID,
|
||||
Resume: runnerStarter.Resume,
|
||||
}).TryResume()
|
||||
}
|
||||
s.iter = newEinoInitialIteratorStartHandler(einoInitialIteratorStartHandlerConfig{
|
||||
ConversationID: s.conversationID,
|
||||
OrchMode: s.orchMode,
|
||||
Progress: s.progress,
|
||||
UseTurnLoop: useTurnLoop,
|
||||
StartRunner: runnerStarter.Start,
|
||||
StartTurnLoop: turnLoopStarter.Start,
|
||||
}).StartIfNeeded(s.iter, s.msgs)
|
||||
|
||||
pending := s.pending()
|
||||
s.completionHandler = newEinoRunCompletionHandler(einoRunCompletionHandlerConfig{
|
||||
ConversationID: s.conversationID,
|
||||
OrchMode: s.orchMode,
|
||||
Progress: s.progress,
|
||||
Logger: s.logger,
|
||||
Pending: pending,
|
||||
Checkpoint: cpStore,
|
||||
CheckpointID: checkPointID,
|
||||
})
|
||||
s.cancellationHandler = newEinoRunCancellationHandler(einoRunCancellationHandlerConfig{
|
||||
Context: s.ctx,
|
||||
ConversationID: s.conversationID,
|
||||
Progress: s.progress,
|
||||
Pending: pending,
|
||||
TakePartial: s.takePartial,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *einoRunRuntimeSession) initRecoveryRuntime() {
|
||||
if s == nil || s.args == nil {
|
||||
return
|
||||
}
|
||||
pending := s.pending()
|
||||
contextOverflowRetry := newEinoContextOverflowRetryHandler(einoContextOverflowRetryConfig{
|
||||
Context: s.ctx,
|
||||
ConversationID: s.conversationID,
|
||||
OrchMode: s.orchMode,
|
||||
Args: s.args,
|
||||
BaseMsgs: s.baseMsgs,
|
||||
Progress: s.progress,
|
||||
Logger: s.logger,
|
||||
})
|
||||
s.transientRetry = newEinoTransientRunRetryHandler(einoTransientRunRetryHandlerConfig{
|
||||
Context: s.ctx,
|
||||
ConversationID: s.conversationID,
|
||||
OrchMode: s.orchMode,
|
||||
Args: s.args,
|
||||
BaseMsgs: s.baseMsgs,
|
||||
Progress: s.progress,
|
||||
Logger: s.logger,
|
||||
Pending: pending,
|
||||
})
|
||||
runErrorHandler := newEinoRunErrorHandler(einoRunErrorHandlerConfig{
|
||||
ConversationID: s.conversationID,
|
||||
OrchMode: s.orchMode,
|
||||
Progress: s.progress,
|
||||
Pending: pending,
|
||||
NativeCancelFallback: s.nativeCancelCauseOrCanceled,
|
||||
})
|
||||
s.runRecoveryHandler = newEinoRunRecoveryHandler(einoRunRecoveryHandlerConfig{
|
||||
ConversationID: s.conversationID,
|
||||
OrchMode: s.orchMode,
|
||||
Args: s.args,
|
||||
BaseMsgs: s.baseMsgs,
|
||||
Progress: s.progress,
|
||||
Logger: s.logger,
|
||||
RunError: runErrorHandler,
|
||||
ContextOverflow: contextOverflowRetry,
|
||||
Transient: s.transientRetry,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *einoRunRuntimeSession) initResultRuntime(emptyHint string, snapshotMCPIDs func() []string, einoRoleTag func(agent string) string) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
var assistantOutput *einoAssistantOutputAccumulator
|
||||
if s.drain != nil {
|
||||
assistantOutput = s.drain.AssistantOutput()
|
||||
}
|
||||
s.resultBuilder = newEinoRunResultBuilder(einoRunResultBuilderConfig{
|
||||
OrchMode: s.orchMode,
|
||||
EmptyHint: emptyHint,
|
||||
RunMessages: s.runMessages,
|
||||
AssistantOutput: assistantOutput,
|
||||
SnapshotMCPIDs: snapshotMCPIDs,
|
||||
ModelFacingTrace: func() []adk.Message { return modelFacingTraceSnapshot(s.args) },
|
||||
})
|
||||
s.streamErrorHandler = newEinoStreamErrorHandler(
|
||||
s.ctx,
|
||||
s.conversationID,
|
||||
s.progress,
|
||||
einoRoleTag,
|
||||
s.maybeRestart,
|
||||
s.takePartial,
|
||||
)
|
||||
}
|
||||
|
||||
func (s *einoRunRuntimeSession) pending() *einoPendingToolCalls {
|
||||
if s == nil || s.drain == nil {
|
||||
return nil
|
||||
}
|
||||
return s.drain.PendingToolCalls()
|
||||
}
|
||||
|
||||
func (s *einoRunRuntimeSession) nativeCancelCauseOrCanceled() error {
|
||||
if s != nil {
|
||||
if v := s.nativeCancelCause.Load(); v != nil {
|
||||
if err, ok := v.(error); ok && err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return context.Canceled
|
||||
}
|
||||
|
||||
func (s *einoRunRuntimeSession) emitUsageSummary(reason string) bool {
|
||||
if s == nil || s.usage == nil {
|
||||
return false
|
||||
}
|
||||
return s.usage.EmitOnce(s.conversationID, s.orchMode, reason, s.progress, s.logger)
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
type fakeRuntimeSessionAgent struct {
|
||||
runMessages []adk.Message
|
||||
runOpts int
|
||||
}
|
||||
|
||||
func (a *fakeRuntimeSessionAgent) Name(context.Context) string {
|
||||
return "lead"
|
||||
}
|
||||
|
||||
func (a *fakeRuntimeSessionAgent) Description(context.Context) string {
|
||||
return "fake runtime session agent"
|
||||
}
|
||||
|
||||
func (a *fakeRuntimeSessionAgent) Run(_ context.Context, input *adk.AgentInput, opts ...adk.AgentRunOption) *adk.AsyncIterator[*adk.AgentEvent] {
|
||||
if input != nil {
|
||||
a.runMessages = input.Messages
|
||||
}
|
||||
a.runOpts = len(opts)
|
||||
iter, gen := adk.NewAsyncIteratorPair[*adk.AgentEvent]()
|
||||
gen.Close()
|
||||
return iter
|
||||
}
|
||||
|
||||
func TestEinoRunRuntimeSessionStartsRunner(t *testing.T) {
|
||||
agent := &fakeRuntimeSessionAgent{}
|
||||
drain := newEinoRunEventDrain(einoRunEventDrainConfig{
|
||||
ConversationID: "conv-1",
|
||||
OrchMode: "deep",
|
||||
OrchestratorName: "lead",
|
||||
BaseMessages: []adk.Message{schema.UserMessage("base")},
|
||||
})
|
||||
session := newEinoRunRuntimeSession(einoRunRuntimeSessionConfig{
|
||||
Context: context.Background(),
|
||||
Args: &einoADKRunLoopArgs{
|
||||
ConversationID: "conv-1",
|
||||
OrchMode: "deep",
|
||||
OrchestratorName: "lead",
|
||||
DA: agent,
|
||||
},
|
||||
Drain: drain,
|
||||
BaseMessages: []adk.Message{schema.UserMessage("base")},
|
||||
EmptyHint: "empty",
|
||||
})
|
||||
defer session.Close()
|
||||
|
||||
if session.Iterator() == nil {
|
||||
t.Fatal("session should start an iterator")
|
||||
}
|
||||
if len(agent.runMessages) != 1 || agent.runMessages[0].Content != "base" {
|
||||
t.Fatalf("run messages = %#v", agent.runMessages)
|
||||
}
|
||||
if agent.runOpts != 1 {
|
||||
t.Fatalf("run opts = %d, want native cancel option", agent.runOpts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunRuntimeSessionCompletionFlushesPending(t *testing.T) {
|
||||
agent := &fakeRuntimeSessionAgent{}
|
||||
var events []string
|
||||
drain := newEinoRunEventDrain(einoRunEventDrainConfig{
|
||||
ConversationID: "conv-1",
|
||||
OrchMode: "deep",
|
||||
OrchestratorName: "lead",
|
||||
Progress: func(eventType, _ string, _ interface{}) {
|
||||
events = append(events, eventType)
|
||||
},
|
||||
BaseMessages: []adk.Message{schema.UserMessage("base")},
|
||||
})
|
||||
session := newEinoRunRuntimeSession(einoRunRuntimeSessionConfig{
|
||||
Context: context.Background(),
|
||||
Args: &einoADKRunLoopArgs{
|
||||
ConversationID: "conv-1",
|
||||
OrchMode: "deep",
|
||||
OrchestratorName: "lead",
|
||||
Progress: func(eventType, _ string, _ interface{}) {
|
||||
events = append(events, eventType)
|
||||
},
|
||||
DA: agent,
|
||||
},
|
||||
Drain: drain,
|
||||
BaseMessages: []adk.Message{schema.UserMessage("base")},
|
||||
EmptyHint: "empty",
|
||||
})
|
||||
defer session.Close()
|
||||
|
||||
drain.PendingToolCalls().Mark(toolCallPendingInfo{
|
||||
ToolCallID: "call-1",
|
||||
ToolName: "execute",
|
||||
EinoAgent: "lead",
|
||||
EinoRole: "orchestrator",
|
||||
})
|
||||
completed, result, err := session.HandleIteratorEnd()
|
||||
|
||||
if !completed || result != nil || err != nil {
|
||||
t.Fatalf("completed=%v result=%#v err=%v", completed, result, err)
|
||||
}
|
||||
if !containsString(events, "tool_result") || !containsString(events, "eino_pending_orphaned") {
|
||||
t.Fatalf("events = %#v, want orphan pending flush", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunRuntimeSessionCancellationReturnsPartialError(t *testing.T) {
|
||||
agent := &fakeRuntimeSessionAgent{}
|
||||
var events []string
|
||||
drain := newEinoRunEventDrain(einoRunEventDrainConfig{
|
||||
ConversationID: "conv-1",
|
||||
OrchMode: "deep",
|
||||
OrchestratorName: "lead",
|
||||
Progress: func(eventType, _ string, _ interface{}) {
|
||||
events = append(events, eventType)
|
||||
},
|
||||
BaseMessages: []adk.Message{schema.UserMessage("base")},
|
||||
})
|
||||
session := newEinoRunRuntimeSession(einoRunRuntimeSessionConfig{
|
||||
Context: context.Background(),
|
||||
Args: &einoADKRunLoopArgs{
|
||||
ConversationID: "conv-1",
|
||||
OrchMode: "deep",
|
||||
OrchestratorName: "lead",
|
||||
Progress: func(eventType, _ string, _ interface{}) {
|
||||
events = append(events, eventType)
|
||||
},
|
||||
DA: agent,
|
||||
},
|
||||
Drain: drain,
|
||||
BaseMessages: []adk.Message{schema.UserMessage("base")},
|
||||
EmptyHint: "empty",
|
||||
})
|
||||
defer session.Close()
|
||||
|
||||
stopErr := errors.New("stop")
|
||||
result, err := session.HandleIteratorContextError(stopErr)
|
||||
|
||||
if result != nil {
|
||||
t.Fatalf("result = %#v, want nil without new messages", result)
|
||||
}
|
||||
if !errors.Is(err, stopErr) {
|
||||
t.Fatalf("err = %v, want %v", err, stopErr)
|
||||
}
|
||||
if !containsString(events, "error") {
|
||||
t.Fatalf("events = %#v, want cancellation error event", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunRuntimeSessionBuildFinalEmitsUsageSummary(t *testing.T) {
|
||||
agent := &fakeRuntimeSessionAgent{}
|
||||
var usageEvent map[string]interface{}
|
||||
progress := func(eventType, _ string, data interface{}) {
|
||||
if eventType != "eino_usage_summary" {
|
||||
return
|
||||
}
|
||||
usageEvent, _ = data.(map[string]interface{})
|
||||
}
|
||||
drain := newEinoRunEventDrain(einoRunEventDrainConfig{
|
||||
ConversationID: "conv-1",
|
||||
OrchMode: "deep",
|
||||
OrchestratorName: "lead",
|
||||
Progress: progress,
|
||||
BaseMessages: []adk.Message{schema.UserMessage("base")},
|
||||
})
|
||||
session := newEinoRunRuntimeSession(einoRunRuntimeSessionConfig{
|
||||
Context: context.Background(),
|
||||
Args: &einoADKRunLoopArgs{
|
||||
ConversationID: "conv-1",
|
||||
OrchMode: "deep",
|
||||
OrchestratorName: "lead",
|
||||
Progress: progress,
|
||||
DA: agent,
|
||||
},
|
||||
Drain: drain,
|
||||
BaseMessages: []adk.Message{schema.UserMessage("base")},
|
||||
EmptyHint: "empty",
|
||||
})
|
||||
defer session.Close()
|
||||
|
||||
drain.Usage().AddUsage(&schema.TokenUsage{PromptTokens: 3, CompletionTokens: 4, TotalTokens: 7})
|
||||
_ = session.BuildFinalResult()
|
||||
|
||||
if usageEvent == nil {
|
||||
t.Fatal("usage summary event was not emitted")
|
||||
}
|
||||
if usageEvent["conversationId"] != "conv-1" || usageEvent["orchestration"] != "deep" || usageEvent["reason"] != "final" || usageEvent["totalTokens"] != 7 {
|
||||
t.Fatalf("usage event = %#v", usageEvent)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func newEinoRunID() string {
|
||||
return uuid.New().String()
|
||||
}
|
||||
|
||||
func withEinoRunIDProgress(
|
||||
runID string,
|
||||
progress func(eventType, message string, data interface{}),
|
||||
) func(eventType, message string, data interface{}) {
|
||||
runID = strings.TrimSpace(runID)
|
||||
if progress == nil || runID == "" {
|
||||
return progress
|
||||
}
|
||||
return func(eventType, message string, data interface{}) {
|
||||
progress(eventType, message, addEinoRunIDToProgressData(runID, data))
|
||||
}
|
||||
}
|
||||
|
||||
func addEinoRunIDToProgressData(runID string, data interface{}) interface{} {
|
||||
runID = strings.TrimSpace(runID)
|
||||
if runID == "" {
|
||||
return data
|
||||
}
|
||||
switch v := data.(type) {
|
||||
case map[string]interface{}:
|
||||
if existing, ok := v["runId"]; !ok || strings.TrimSpace(fmt.Sprint(existing)) == "" {
|
||||
v["runId"] = runID
|
||||
}
|
||||
return v
|
||||
default:
|
||||
return data
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package multiagent
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestWithEinoRunIDProgressAddsRunIDToMapData(t *testing.T) {
|
||||
var gotType, gotMessage string
|
||||
var gotData interface{}
|
||||
progress := withEinoRunIDProgress("run-1", func(eventType, message string, data interface{}) {
|
||||
gotType = eventType
|
||||
gotMessage = message
|
||||
gotData = data
|
||||
})
|
||||
|
||||
progress("progress", "hello", map[string]interface{}{"source": "eino"})
|
||||
|
||||
if gotType != "progress" || gotMessage != "hello" {
|
||||
t.Fatalf("event = (%q, %q)", gotType, gotMessage)
|
||||
}
|
||||
m, ok := gotData.(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("data type = %T", gotData)
|
||||
}
|
||||
if m["runId"] != "run-1" || m["source"] != "eino" {
|
||||
t.Fatalf("data = %#v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithEinoRunIDProgressPreservesExistingRunID(t *testing.T) {
|
||||
var got map[string]interface{}
|
||||
progress := withEinoRunIDProgress("outer-run", func(_, _ string, data interface{}) {
|
||||
got, _ = data.(map[string]interface{})
|
||||
})
|
||||
|
||||
progress("progress", "", map[string]interface{}{"runId": "inner-run"})
|
||||
|
||||
if got["runId"] != "inner-run" {
|
||||
t.Fatalf("runId = %q, want inner-run", got["runId"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type einoRunUsageSummary struct {
|
||||
ModelCalls int
|
||||
PromptTokens int
|
||||
CompletionTokens int
|
||||
TotalTokens int
|
||||
CachedTokens int
|
||||
ReasoningTokens int
|
||||
}
|
||||
|
||||
type einoRunUsageAccumulator struct {
|
||||
mu sync.Mutex
|
||||
summary einoRunUsageSummary
|
||||
emitted bool
|
||||
}
|
||||
|
||||
func newEinoRunUsageAccumulator() *einoRunUsageAccumulator {
|
||||
return &einoRunUsageAccumulator{}
|
||||
}
|
||||
|
||||
func (a *einoRunUsageAccumulator) AddMessage(msg *schema.Message) bool {
|
||||
if msg == nil || msg.ResponseMeta == nil || msg.ResponseMeta.Usage == nil {
|
||||
return false
|
||||
}
|
||||
return a.AddUsage(msg.ResponseMeta.Usage)
|
||||
}
|
||||
|
||||
func (a *einoRunUsageAccumulator) AddUsage(usage *schema.TokenUsage) bool {
|
||||
if a == nil || usage == nil || tokenUsageEmpty(usage) {
|
||||
return false
|
||||
}
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.summary.ModelCalls++
|
||||
a.summary.PromptTokens += usage.PromptTokens
|
||||
a.summary.CompletionTokens += usage.CompletionTokens
|
||||
a.summary.TotalTokens += usage.TotalTokens
|
||||
a.summary.CachedTokens += usage.PromptTokenDetails.CachedTokens
|
||||
a.summary.ReasoningTokens += usage.CompletionTokensDetails.ReasoningTokens
|
||||
return true
|
||||
}
|
||||
|
||||
func (a *einoRunUsageAccumulator) Summary() einoRunUsageSummary {
|
||||
if a == nil {
|
||||
return einoRunUsageSummary{}
|
||||
}
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.summary
|
||||
}
|
||||
|
||||
func (a *einoRunUsageAccumulator) EmitOnce(
|
||||
conversationID string,
|
||||
orchestration string,
|
||||
reason string,
|
||||
progress func(eventType, message string, data interface{}),
|
||||
logger *zap.Logger,
|
||||
) bool {
|
||||
if a == nil {
|
||||
return false
|
||||
}
|
||||
a.mu.Lock()
|
||||
if a.emitted || a.summary.ModelCalls == 0 {
|
||||
a.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
a.emitted = true
|
||||
s := a.summary
|
||||
a.mu.Unlock()
|
||||
|
||||
data := map[string]interface{}{
|
||||
"conversationId": conversationID,
|
||||
"source": "eino",
|
||||
"orchestration": orchestration,
|
||||
"reason": reason,
|
||||
"modelCalls": s.ModelCalls,
|
||||
"promptTokens": s.PromptTokens,
|
||||
"completionTokens": s.CompletionTokens,
|
||||
"totalTokens": s.TotalTokens,
|
||||
"cachedTokens": s.CachedTokens,
|
||||
"reasoningTokens": s.ReasoningTokens,
|
||||
}
|
||||
if progress != nil {
|
||||
progress("eino_usage_summary", "Eino token usage summary", data)
|
||||
}
|
||||
if logger != nil {
|
||||
logger.Info("eino token usage summary",
|
||||
zap.String("conversationId", conversationID),
|
||||
zap.String("orchestration", orchestration),
|
||||
zap.String("reason", reason),
|
||||
zap.Int("modelCalls", s.ModelCalls),
|
||||
zap.Int("promptTokens", s.PromptTokens),
|
||||
zap.Int("completionTokens", s.CompletionTokens),
|
||||
zap.Int("totalTokens", s.TotalTokens),
|
||||
zap.Int("cachedTokens", s.CachedTokens),
|
||||
zap.Int("reasoningTokens", s.ReasoningTokens),
|
||||
)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func maxEinoTokenUsage(dst *schema.TokenUsage, src *schema.TokenUsage) *schema.TokenUsage {
|
||||
if src == nil {
|
||||
return dst
|
||||
}
|
||||
if dst == nil {
|
||||
return cloneEinoTokenUsage(src)
|
||||
}
|
||||
if src.PromptTokens > dst.PromptTokens {
|
||||
dst.PromptTokens = src.PromptTokens
|
||||
}
|
||||
if src.CompletionTokens > dst.CompletionTokens {
|
||||
dst.CompletionTokens = src.CompletionTokens
|
||||
}
|
||||
if src.TotalTokens > dst.TotalTokens {
|
||||
dst.TotalTokens = src.TotalTokens
|
||||
}
|
||||
if src.PromptTokenDetails.CachedTokens > dst.PromptTokenDetails.CachedTokens {
|
||||
dst.PromptTokenDetails.CachedTokens = src.PromptTokenDetails.CachedTokens
|
||||
}
|
||||
if src.CompletionTokensDetails.ReasoningTokens > dst.CompletionTokensDetails.ReasoningTokens {
|
||||
dst.CompletionTokensDetails.ReasoningTokens = src.CompletionTokensDetails.ReasoningTokens
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
func cloneEinoTokenUsage(src *schema.TokenUsage) *schema.TokenUsage {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
out := *src
|
||||
return &out
|
||||
}
|
||||
|
||||
func tokenUsageEmpty(u *schema.TokenUsage) bool {
|
||||
return u == nil ||
|
||||
(u.PromptTokens == 0 &&
|
||||
u.CompletionTokens == 0 &&
|
||||
u.TotalTokens == 0 &&
|
||||
u.PromptTokenDetails.CachedTokens == 0 &&
|
||||
u.CompletionTokensDetails.ReasoningTokens == 0)
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
func TestEinoRunUsageAccumulatorSumsModelCalls(t *testing.T) {
|
||||
acc := newEinoRunUsageAccumulator()
|
||||
acc.AddUsage(&schema.TokenUsage{
|
||||
PromptTokens: 10,
|
||||
CompletionTokens: 4,
|
||||
TotalTokens: 14,
|
||||
PromptTokenDetails: schema.PromptTokenDetails{
|
||||
CachedTokens: 3,
|
||||
},
|
||||
CompletionTokensDetails: schema.CompletionTokensDetails{
|
||||
ReasoningTokens: 2,
|
||||
},
|
||||
})
|
||||
msg := schema.AssistantMessage("ok", nil)
|
||||
msg.ResponseMeta = &schema.ResponseMeta{Usage: &schema.TokenUsage{
|
||||
PromptTokens: 7,
|
||||
CompletionTokens: 5,
|
||||
TotalTokens: 12,
|
||||
CompletionTokensDetails: schema.CompletionTokensDetails{
|
||||
ReasoningTokens: 1,
|
||||
},
|
||||
}}
|
||||
acc.AddMessage(msg)
|
||||
|
||||
got := acc.Summary()
|
||||
if got.ModelCalls != 2 || got.PromptTokens != 17 || got.CompletionTokens != 9 || got.TotalTokens != 26 || got.CachedTokens != 3 || got.ReasoningTokens != 3 {
|
||||
t.Fatalf("summary = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunUsageAccumulatorEmitOnce(t *testing.T) {
|
||||
acc := newEinoRunUsageAccumulator()
|
||||
acc.AddUsage(&schema.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3})
|
||||
var events []map[string]interface{}
|
||||
progress := func(eventType, _ string, data interface{}) {
|
||||
if eventType != "eino_usage_summary" {
|
||||
return
|
||||
}
|
||||
if m, ok := data.(map[string]interface{}); ok {
|
||||
events = append(events, m)
|
||||
}
|
||||
}
|
||||
|
||||
if !acc.EmitOnce("conv-1", "deep", "final", progress, nil) {
|
||||
t.Fatal("first emit should return true")
|
||||
}
|
||||
if acc.EmitOnce("conv-1", "deep", "partial", progress, nil) {
|
||||
t.Fatal("second emit should return false")
|
||||
}
|
||||
if len(events) != 1 {
|
||||
t.Fatalf("events = %#v, want one usage summary", events)
|
||||
}
|
||||
if events[0]["conversationId"] != "conv-1" || events[0]["orchestration"] != "deep" || events[0]["reason"] != "final" || events[0]["totalTokens"] != 3 {
|
||||
t.Fatalf("event = %#v", events[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxEinoTokenUsageUsesLargestStreamChunkValues(t *testing.T) {
|
||||
var got *schema.TokenUsage
|
||||
got = maxEinoTokenUsage(got, &schema.TokenUsage{PromptTokens: 10, CompletionTokens: 2, TotalTokens: 12})
|
||||
got = maxEinoTokenUsage(got, &schema.TokenUsage{
|
||||
PromptTokens: 9,
|
||||
CompletionTokens: 5,
|
||||
TotalTokens: 14,
|
||||
CompletionTokensDetails: schema.CompletionTokensDetails{
|
||||
ReasoningTokens: 3,
|
||||
},
|
||||
})
|
||||
|
||||
if got.PromptTokens != 10 || got.CompletionTokens != 5 || got.TotalTokens != 14 || got.CompletionTokensDetails.ReasoningTokens != 3 {
|
||||
t.Fatalf("usage = %#v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type einoRunnerControl interface {
|
||||
Run(context.Context, []adk.Message, ...adk.AgentRunOption) *adk.AsyncIterator[*adk.AgentEvent]
|
||||
Resume(context.Context, string, ...adk.AgentRunOption) (*adk.AsyncIterator[*adk.AgentEvent], error)
|
||||
}
|
||||
|
||||
type einoRunnerIteratorStarterConfig struct {
|
||||
Context context.Context
|
||||
ConversationID string
|
||||
OrchMode string
|
||||
Logger *zap.Logger
|
||||
Runner einoRunnerControl
|
||||
CheckPointID string
|
||||
NativeCancelCause *atomic.Value
|
||||
UnregisterAgentCancel *func()
|
||||
RuntimeCancelRegistrar AgentRuntimeCancelRegistrar
|
||||
}
|
||||
|
||||
type einoRunnerIteratorStarter struct {
|
||||
cfg einoRunnerIteratorStarterConfig
|
||||
}
|
||||
|
||||
func newEinoRunnerIteratorStarter(cfg einoRunnerIteratorStarterConfig) *einoRunnerIteratorStarter {
|
||||
return &einoRunnerIteratorStarter{cfg: cfg}
|
||||
}
|
||||
|
||||
func (s *einoRunnerIteratorStarter) Start(runMsgs []adk.Message) *adk.AsyncIterator[*adk.AgentEvent] {
|
||||
if s == nil || s.cfg.Runner == nil {
|
||||
return nil
|
||||
}
|
||||
opts := s.newRunOptions()
|
||||
if s.cfg.CheckPointID != "" {
|
||||
opts = append(opts, adk.WithCheckPointID(s.cfg.CheckPointID))
|
||||
}
|
||||
return s.cfg.Runner.Run(s.cfg.Context, runMsgs, opts...)
|
||||
}
|
||||
|
||||
func (s *einoRunnerIteratorStarter) Resume(checkPointID string) (*adk.AsyncIterator[*adk.AgentEvent], error) {
|
||||
if s == nil || s.cfg.Runner == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return s.cfg.Runner.Resume(s.cfg.Context, checkPointID, s.newRunOptions()...)
|
||||
}
|
||||
|
||||
func (s *einoRunnerIteratorStarter) newRunOptions() []adk.AgentRunOption {
|
||||
cancelOpt, cancelFn := adk.WithCancel()
|
||||
callAndClearUnregister(s.cfg.UnregisterAgentCancel)
|
||||
if s.cfg.RuntimeCancelRegistrar != nil && s.cfg.UnregisterAgentCancel != nil {
|
||||
*s.cfg.UnregisterAgentCancel = s.cfg.RuntimeCancelRegistrar(func(cause error) bool {
|
||||
s.storeNativeCancelCause(cause)
|
||||
waitErr, submitted, handled := requestEinoNativeAgentCancel(cancelFn, cause)
|
||||
s.logNativeCancelRequest(cause, waitErr, submitted, handled)
|
||||
return handled
|
||||
})
|
||||
}
|
||||
return []adk.AgentRunOption{cancelOpt}
|
||||
}
|
||||
|
||||
func (s *einoRunnerIteratorStarter) storeNativeCancelCause(cause error) {
|
||||
if s == nil || s.cfg.NativeCancelCause == nil || cause == nil {
|
||||
return
|
||||
}
|
||||
s.cfg.NativeCancelCause.Store(cause)
|
||||
}
|
||||
|
||||
func (s *einoRunnerIteratorStarter) logNativeCancelRequest(cause error, waitErr error, submitted bool, handled bool) {
|
||||
if s == nil || s.cfg.Logger == nil {
|
||||
return
|
||||
}
|
||||
fields := []zap.Field{
|
||||
zap.String("conversation_id", s.cfg.ConversationID),
|
||||
zap.String("orchestration", s.cfg.OrchMode),
|
||||
zap.Bool("submitted", submitted),
|
||||
zap.Bool("handled", handled),
|
||||
}
|
||||
if cause != nil {
|
||||
fields = append(fields, zap.Error(cause))
|
||||
}
|
||||
if waitErr != nil {
|
||||
fields = append(fields, zap.NamedError("cancel_wait_error", waitErr))
|
||||
s.cfg.Logger.Debug("eino native cancel requested", fields...)
|
||||
} else {
|
||||
s.cfg.Logger.Info("eino native cancel requested", fields...)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
)
|
||||
|
||||
type fakeRunnerControl struct {
|
||||
runMessages []adk.Message
|
||||
runOpts int
|
||||
resumeID string
|
||||
resumeOpts int
|
||||
resumeErr error
|
||||
}
|
||||
|
||||
func (f *fakeRunnerControl) Run(_ context.Context, messages []adk.Message, opts ...adk.AgentRunOption) *adk.AsyncIterator[*adk.AgentEvent] {
|
||||
f.runMessages = messages
|
||||
f.runOpts = len(opts)
|
||||
iter, gen := adk.NewAsyncIteratorPair[*adk.AgentEvent]()
|
||||
gen.Close()
|
||||
return iter
|
||||
}
|
||||
|
||||
func (f *fakeRunnerControl) Resume(_ context.Context, checkPointID string, opts ...adk.AgentRunOption) (*adk.AsyncIterator[*adk.AgentEvent], error) {
|
||||
f.resumeID = checkPointID
|
||||
f.resumeOpts = len(opts)
|
||||
iter, gen := adk.NewAsyncIteratorPair[*adk.AgentEvent]()
|
||||
gen.Close()
|
||||
return iter, f.resumeErr
|
||||
}
|
||||
|
||||
func TestEinoRunnerIteratorStarterStartAddsCancelAndCheckpoint(t *testing.T) {
|
||||
runner := &fakeRunnerControl{}
|
||||
var cancelPush func(error) bool
|
||||
var nativeCancelCause atomic.Value
|
||||
oldUnregistered := false
|
||||
newUnregistered := false
|
||||
unregister := func() { oldUnregistered = true }
|
||||
|
||||
iter := newEinoRunnerIteratorStarter(einoRunnerIteratorStarterConfig{
|
||||
Context: context.Background(),
|
||||
Runner: runner,
|
||||
CheckPointID: "cp-1",
|
||||
NativeCancelCause: &nativeCancelCause,
|
||||
UnregisterAgentCancel: &unregister,
|
||||
RuntimeCancelRegistrar: func(push func(error) bool) func() {
|
||||
cancelPush = push
|
||||
return func() { newUnregistered = true }
|
||||
},
|
||||
}).Start([]adk.Message{})
|
||||
|
||||
if iter == nil {
|
||||
t.Fatal("iterator should be created")
|
||||
}
|
||||
if runner.runOpts != 2 {
|
||||
t.Fatalf("run opts = %d, want cancel + checkpoint", runner.runOpts)
|
||||
}
|
||||
if !oldUnregistered {
|
||||
t.Fatal("old unregister should be called before binding a new cancel hook")
|
||||
}
|
||||
if cancelPush == nil {
|
||||
t.Fatal("cancel hook should be registered")
|
||||
}
|
||||
stopErr := errors.New("stop")
|
||||
if cancelPush(stopErr) {
|
||||
t.Fatal("unbound fake runner cancel should not report handled")
|
||||
}
|
||||
if got, _ := nativeCancelCause.Load().(error); !errors.Is(got, stopErr) {
|
||||
t.Fatalf("native cancel cause = %v, want %v", got, stopErr)
|
||||
}
|
||||
unregister()
|
||||
if !newUnregistered {
|
||||
t.Fatal("new unregister should replace old unregister")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunnerIteratorStarterResumeUsesCancelOnly(t *testing.T) {
|
||||
runner := &fakeRunnerControl{}
|
||||
|
||||
iter, err := newEinoRunnerIteratorStarter(einoRunnerIteratorStarterConfig{
|
||||
Context: context.Background(),
|
||||
Runner: runner,
|
||||
CheckPointID: "fresh-run-checkpoint",
|
||||
}).Resume("resume-cp")
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("resume err = %v", err)
|
||||
}
|
||||
if iter == nil {
|
||||
t.Fatal("iterator should be created")
|
||||
}
|
||||
if runner.resumeID != "resume-cp" {
|
||||
t.Fatalf("resume id = %q, want resume-cp", runner.resumeID)
|
||||
}
|
||||
if runner.resumeOpts != 1 {
|
||||
t.Fatalf("resume opts = %d, want cancel only", runner.resumeOpts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEinoRunnerIteratorStarterResumePropagatesError(t *testing.T) {
|
||||
resumeErr := errors.New("resume failed")
|
||||
runner := &fakeRunnerControl{resumeErr: resumeErr}
|
||||
|
||||
_, err := newEinoRunnerIteratorStarter(einoRunnerIteratorStarterConfig{
|
||||
Context: context.Background(),
|
||||
Runner: runner,
|
||||
}).Resume("resume-cp")
|
||||
|
||||
if !errors.Is(err, resumeErr) {
|
||||
t.Fatalf("resume err = %v, want %v", err, resumeErr)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"cyberstrike-ai/internal/agent"
|
||||
"cyberstrike-ai/internal/config"
|
||||
"cyberstrike-ai/internal/database"
|
||||
"cyberstrike-ai/internal/einomcp"
|
||||
"cyberstrike-ai/internal/project"
|
||||
"cyberstrike-ai/internal/reasoning"
|
||||
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"github.com/cloudwego/eino/compose"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// einoSingleAgentName 与 ChatModelAgent.Name 一致,供流式事件映射主对话区。
|
||||
const einoSingleAgentName = "cyberstrike-eino-single"
|
||||
|
||||
// RunEinoSingleChatModelAgent 使用 Eino TypedChatModelAgent[*schema.AgenticMessage] + adk.NewRunner.Run(官方 Quick Start 的 Query 同属 Runner API;此处用历史 + 用户消息切片等价于多轮 Query)。
|
||||
// 与 RunDeepAgent 共享 runEinoADKAgentLoop 的 SSE 映射与 MCP 桥。
|
||||
func RunEinoSingleChatModelAgent(
|
||||
ctx context.Context,
|
||||
appCfg *config.Config,
|
||||
ma *config.MultiAgentConfig,
|
||||
ag *agent.Agent,
|
||||
db *database.DB,
|
||||
logger *zap.Logger,
|
||||
conversationID string,
|
||||
projectID string,
|
||||
userMessage string,
|
||||
history []agent.ChatMessage,
|
||||
roleTools []string,
|
||||
progress func(eventType, message string, data interface{}),
|
||||
reasoningClient *reasoning.ClientIntent,
|
||||
systemPromptExtra string,
|
||||
) (*RunResult, error) {
|
||||
if appCfg == nil || ag == nil {
|
||||
return nil, fmt.Errorf("eino single: 配置或 Agent 为空")
|
||||
}
|
||||
if ma == nil {
|
||||
return nil, fmt.Errorf("eino single: multi_agent 配置为空")
|
||||
}
|
||||
runtimeUserMessage := prepareLatestUserMessageForModel(userMessage, appCfg, &ma.EinoMiddleware, conversationID, logger)
|
||||
|
||||
einoLoc, einoSkillMW, einoFSTools, skillsRoot, einoErr := prepareEinoAgenticSkills(ctx, appCfg.SkillsDir, ma, logger)
|
||||
if einoErr != nil {
|
||||
return nil, einoErr
|
||||
}
|
||||
|
||||
holder := &einomcp.ConversationHolder{}
|
||||
holder.Set(conversationID)
|
||||
|
||||
var mcpIDsMu sync.Mutex
|
||||
var mcpIDs []string
|
||||
mcpExecBinder := NewMCPExecutionBinder()
|
||||
recorder := func(id, toolCallID string) {
|
||||
if id == "" {
|
||||
return
|
||||
}
|
||||
mcpExecBinder.Bind(toolCallID, id)
|
||||
mcpIDsMu.Lock()
|
||||
mcpIDs = append(mcpIDs, id)
|
||||
mcpIDsMu.Unlock()
|
||||
}
|
||||
|
||||
snapshotMCPIDs := func() []string {
|
||||
mcpIDsMu.Lock()
|
||||
defer mcpIDsMu.Unlock()
|
||||
out := make([]string, len(mcpIDs))
|
||||
copy(out, mcpIDs)
|
||||
return out
|
||||
}
|
||||
|
||||
toolInvokeNotify := einomcp.NewToolInvokeNotifyHolder()
|
||||
einoExecBegin, einoExecAppendPartial, einoExecRegisterCancel, einoExecUnregisterCancel, einoExecFinish := newEinoExecuteMonitorCallbacks(ctx, ag, recorder)
|
||||
mainDefs := ag.ToolsForRole(roleTools)
|
||||
mainTools, err := einomcp.ToolsFromDefinitions(ag, holder, mainDefs, recorder, nil, toolInvokeNotify, einoSingleAgentName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
mainToolsForCfg, mainOrchestratorPre, singleToolSearchActive, err := prependEinoAgenticMiddlewares(ctx, &ma.EinoMiddleware, einoMWMain, mainTools, einoLoc, skillsRoot, conversationID, projectID, logger)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("eino single eino 中间件: %w", err)
|
||||
}
|
||||
|
||||
baseHTTPClient := newEinoBaseHTTPClient()
|
||||
agenticModelFactory := newEinoOpenAIAgenticChatModelFactory(baseHTTPClient, reasoningClient, logger)
|
||||
mainModel, err := agenticModelFactory(ctx, appCfg.OpenAI, einoModelModeNormal)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("eino single agentic 模型: %w", err)
|
||||
}
|
||||
modelRetryCfg := newEinoAgenticModelRetryConfig(&ma.EinoMiddleware, logger, "eino_single")
|
||||
modelFailoverCfg, err := newEinoAgenticModelFailoverConfig(ctx, appCfg, &ma.EinoMiddleware, einoModelModeNormal, agenticModelFactory, logger, "eino_single", progress, "eino_single", conversationID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logEinoAgenticModelGate(
|
||||
logger,
|
||||
"eino_single",
|
||||
"eino_single",
|
||||
evaluateEinoAgenticModelGate(agenticModelGateFactory(agenticModelFactory, appCfg.OpenAI, einoModelModeNormal), einoAgenticRuntimeSupportV0914()),
|
||||
)
|
||||
|
||||
mainSumMw, err := newEinoAgenticSummarizationMiddleware(ctx, mainModel, appCfg, &ma.EinoMiddleware, conversationID, db, projectID, logger)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("eino single agentic summarization: %w", err)
|
||||
}
|
||||
|
||||
modelFacingTrace := newModelFacingTraceHolder()
|
||||
|
||||
handlers := make([]adk.TypedChatModelAgentMiddleware[*schema.AgenticMessage], 0, 8)
|
||||
if len(mainOrchestratorPre) > 0 {
|
||||
handlers = append(handlers, mainOrchestratorPre...)
|
||||
}
|
||||
if einoSkillMW != nil {
|
||||
if einoFSTools && einoLoc != nil {
|
||||
fsMw, fsErr := subAgentAgenticFilesystemMiddleware(ctx, einoLoc, toolInvokeNotify, einoSingleAgentName, einoExecBegin, einoExecAppendPartial, einoExecRegisterCancel, einoExecUnregisterCancel, einoExecFinish, agentToolTimeoutMinutes(appCfg), agentToolWaitTimeoutSeconds(appCfg), agentShellNoOutputTimeoutSeconds(appCfg), nil)
|
||||
if fsErr != nil {
|
||||
return nil, fmt.Errorf("eino single filesystem 中间件: %w", fsErr)
|
||||
}
|
||||
handlers = append(handlers, fsMw)
|
||||
}
|
||||
handlers = append(handlers, einoSkillMW)
|
||||
}
|
||||
handlers = appendEinoAgenticChatModelTailMiddlewares(handlers, einoChatModelTailConfig{
|
||||
logger: logger,
|
||||
phase: "eino_single",
|
||||
agenticSummarization: mainSumMw,
|
||||
modelName: appCfg.OpenAI.Model,
|
||||
maxTotalTokens: appCfg.OpenAI.MaxTotalTokens,
|
||||
toolMaxBytes: toolMaxBytesFromMW(&ma.EinoMiddleware),
|
||||
conversationID: conversationID,
|
||||
trace: modelFacingTrace,
|
||||
middlewareConfig: &ma.EinoMiddleware,
|
||||
})
|
||||
|
||||
maxIter := agentMaxIterations(appCfg)
|
||||
|
||||
mainToolsCfg := adk.ToolsConfig{
|
||||
ToolsNodeConfig: compose.ToolsNodeConfig{
|
||||
Tools: mainToolsForCfg,
|
||||
UnknownToolsHandler: einomcp.UnknownToolReminderHandler(),
|
||||
ToolCallMiddlewares: []compose.ToolMiddleware{
|
||||
modelOutputExecutionGuardMiddleware(),
|
||||
localToolRBACMiddleware(),
|
||||
hitlToolCallMiddleware(),
|
||||
softRecoveryToolMiddleware(),
|
||||
},
|
||||
},
|
||||
EmitInternalEvents: true,
|
||||
}
|
||||
ins := project.AppendSystemPromptBlock(ag.EinoSingleAgentSystemInstruction(), systemPromptExtra)
|
||||
ins = project.AppendVisionImageAnalysisIfReady(ins, appCfg.Vision.Ready())
|
||||
ins = injectToolNamesOnlyInstruction(ctx, ins, mainTools, singleToolSearchActive)
|
||||
if logger != nil {
|
||||
names := collectToolNames(ctx, mainTools)
|
||||
mountedNames := collectToolNames(ctx, mainToolsForCfg)
|
||||
logger.Info("eino tool-name injection",
|
||||
zap.String("scope", "eino_single"),
|
||||
zap.Int("tool_names", len(names)),
|
||||
zap.Int("mounted_tool_names", len(mountedNames)),
|
||||
zap.Bool("tool_search_middleware", singleToolSearchActive),
|
||||
)
|
||||
}
|
||||
|
||||
chatCfg := einoAgenticChatModelAgentConfig{
|
||||
Name: einoSingleAgentName,
|
||||
Description: "Eino ADK ChatModelAgent with MCP tools for authorized security testing.",
|
||||
Instruction: ins,
|
||||
GenModelInput: literalAgenticInstructionGenModelInput,
|
||||
Model: mainModel,
|
||||
ToolsConfig: mainToolsCfg,
|
||||
MaxIterations: maxIter,
|
||||
Handlers: handlers,
|
||||
ModelRetryConfig: modelRetryCfg,
|
||||
ModelFailoverConfig: modelFailoverCfg,
|
||||
}
|
||||
outKey, _ := deepExtrasFromConfig(ma)
|
||||
if outKey != "" {
|
||||
chatCfg.OutputKey = outKey
|
||||
}
|
||||
|
||||
chatAgent, err := newEinoAgenticChatModelAgentAdapter(ctx, chatCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("eino single Agentic ChatModelAgent: %w", err)
|
||||
}
|
||||
|
||||
baseMsgs := historyToMessages(history, appCfg, &ma.EinoMiddleware)
|
||||
baseMsgs = appendUserMessageIfNeeded(baseMsgs, runtimeUserMessage)
|
||||
|
||||
streamsMainAssistant := func(agent string) bool {
|
||||
return agent == "" || agent == einoSingleAgentName
|
||||
}
|
||||
einoRoleTag := func(agent string) string {
|
||||
_ = agent
|
||||
return "orchestrator"
|
||||
}
|
||||
|
||||
return runEinoADKAgentLoop(ctx, &einoADKRunLoopArgs{
|
||||
OrchMode: "eino_single",
|
||||
OrchestratorName: einoSingleAgentName,
|
||||
ConversationID: conversationID,
|
||||
Progress: progress,
|
||||
Logger: logger,
|
||||
SnapshotMCPIDs: snapshotMCPIDs,
|
||||
StreamsMainAssistant: streamsMainAssistant,
|
||||
EinoRoleTag: einoRoleTag,
|
||||
CheckpointDir: ma.EinoMiddleware.CheckpointDir,
|
||||
RunRetryMaxAttempts: RunRetryMaxAttemptsFromConfig(&ma.EinoMiddleware),
|
||||
RunRetryMaxBackoffSec: int(einoRunRetryMaxBackoffFromConfig(&ma.EinoMiddleware).Seconds()),
|
||||
McpIDsMu: &mcpIDsMu,
|
||||
McpIDs: &mcpIDs,
|
||||
FilesystemMonitorAgent: ag,
|
||||
FilesystemMonitorRecord: recorder,
|
||||
MCPExecutionBinder: mcpExecBinder,
|
||||
ToolInvokeNotify: toolInvokeNotify,
|
||||
DA: chatAgent,
|
||||
ModelFacingTrace: modelFacingTrace,
|
||||
EinoCallbacks: &ma.EinoCallbacks,
|
||||
MaxTotalTokens: appCfg.OpenAI.MaxTotalTokens,
|
||||
ToolMaxBytes: toolMaxBytesFromMW(&ma.EinoMiddleware),
|
||||
ModelName: appCfg.OpenAI.Model,
|
||||
MiddlewareConfig: &ma.EinoMiddleware,
|
||||
EmptyResponseMessage: "(Eino ADK single-agent session completed but no assistant text was captured. Check process details or logs.) " +
|
||||
"(Eino ADK 单代理会话已完成,但未捕获到助手文本输出。请查看过程详情或日志。)",
|
||||
}, baseMsgs)
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"cyberstrike-ai/internal/config"
|
||||
"cyberstrike-ai/internal/einomcp"
|
||||
"cyberstrike-ai/internal/security"
|
||||
|
||||
localbk "github.com/cloudwego/eino-ext/adk/backend/local"
|
||||
"github.com/cloudwego/eino/adk"
|
||||
"github.com/cloudwego/eino/adk/middlewares/filesystem"
|
||||
"github.com/cloudwego/eino/adk/middlewares/skill"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func prepareEinoAgenticSkills(
|
||||
ctx context.Context,
|
||||
skillsDir string,
|
||||
ma *config.MultiAgentConfig,
|
||||
logger *zap.Logger,
|
||||
) (loc *localbk.Local, skillMW adk.TypedChatModelAgentMiddleware[*schema.AgenticMessage], fsTools bool, skillsRoot string, err error) {
|
||||
if ma == nil {
|
||||
return nil, nil, false, "", nil
|
||||
}
|
||||
needLocalBackend := ma.EinoMiddleware.ReductionEnable
|
||||
newLocalBackend := func() (*localbk.Local, error) {
|
||||
backend, backendErr := localbk.NewBackend(ctx, &localbk.Config{})
|
||||
if backendErr != nil {
|
||||
return nil, fmt.Errorf("eino local backend: %w", backendErr)
|
||||
}
|
||||
return backend, nil
|
||||
}
|
||||
if ma.EinoSkills.Disable {
|
||||
if !needLocalBackend {
|
||||
return nil, nil, false, "", nil
|
||||
}
|
||||
loc, err = newLocalBackend()
|
||||
return loc, nil, false, "", err
|
||||
}
|
||||
root := strings.TrimSpace(skillsDir)
|
||||
if root == "" {
|
||||
if logger != nil {
|
||||
logger.Warn("eino agentic skills: skills_dir empty, skip")
|
||||
}
|
||||
if !needLocalBackend {
|
||||
return nil, nil, false, "", nil
|
||||
}
|
||||
loc, err = newLocalBackend()
|
||||
return loc, nil, false, "", err
|
||||
}
|
||||
abs, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, nil, false, "", fmt.Errorf("skills_dir abs: %w", err)
|
||||
}
|
||||
if st, err := os.Stat(abs); err != nil || !st.IsDir() {
|
||||
if logger != nil {
|
||||
logger.Warn("eino agentic skills: directory missing, skip", zap.String("dir", abs), zap.Error(err))
|
||||
}
|
||||
if !needLocalBackend {
|
||||
return nil, nil, false, "", nil
|
||||
}
|
||||
loc, err = newLocalBackend()
|
||||
return loc, nil, false, "", err
|
||||
}
|
||||
|
||||
loc, err = newLocalBackend()
|
||||
if err != nil {
|
||||
return nil, nil, false, "", err
|
||||
}
|
||||
|
||||
skillBE, err := skill.NewBackendFromFilesystem(ctx, &skill.BackendFromFilesystemConfig{
|
||||
Backend: loc,
|
||||
BaseDir: abs,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, false, "", fmt.Errorf("eino agentic skill filesystem backend: %w", err)
|
||||
}
|
||||
|
||||
sc := &skill.TypedConfig[*schema.AgenticMessage]{Backend: skillBE}
|
||||
if name := strings.TrimSpace(ma.EinoSkills.SkillToolName); name != "" {
|
||||
sc.SkillToolName = &name
|
||||
}
|
||||
skillMW, err = skill.NewTyped[*schema.AgenticMessage](ctx, sc)
|
||||
if err != nil {
|
||||
return nil, nil, false, "", fmt.Errorf("eino agentic skill middleware: %w", err)
|
||||
}
|
||||
|
||||
fsTools = ma.EinoSkills.EinoSkillFilesystemToolsEffective()
|
||||
return loc, skillMW, fsTools, abs, nil
|
||||
}
|
||||
|
||||
func subAgentAgenticFilesystemMiddleware(
|
||||
ctx context.Context,
|
||||
loc *localbk.Local,
|
||||
invokeNotify *einomcp.ToolInvokeNotifyHolder,
|
||||
einoAgentName string,
|
||||
beginMonitor func(toolCallID, command string) string,
|
||||
appendPartialMonitor func(executionID, toolCallID, chunk string),
|
||||
registerCancelMonitor func(executionID string, cancel context.CancelFunc),
|
||||
unregisterCancelMonitor func(executionID string),
|
||||
finishMonitor func(executionID, toolCallID, command, stdout string, success bool, invokeErr error),
|
||||
toolTimeoutMinutes int,
|
||||
toolWaitTimeoutSeconds int,
|
||||
shellNoOutputTimeoutSec int,
|
||||
outputChunk func(toolName, toolCallID, chunk string),
|
||||
) (adk.TypedChatModelAgentMiddleware[*schema.AgenticMessage], error) {
|
||||
if loc == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return filesystem.NewTyped[*schema.AgenticMessage](ctx, &filesystem.MiddlewareConfig{
|
||||
Backend: loc,
|
||||
StreamingShell: &einoStreamingShellWrap{
|
||||
inner: security.NewEinoStreamingShell(),
|
||||
invokeNotify: invokeNotify,
|
||||
einoAgentName: strings.TrimSpace(einoAgentName),
|
||||
outputChunk: outputChunk,
|
||||
beginMonitor: beginMonitor,
|
||||
appendPartialMonitor: appendPartialMonitor,
|
||||
registerCancelMonitor: registerCancelMonitor,
|
||||
unregisterCancelMonitor: unregisterCancelMonitor,
|
||||
finishMonitor: finishMonitor,
|
||||
toolTimeoutMinutes: toolTimeoutMinutes,
|
||||
toolWaitTimeoutSeconds: toolWaitTimeoutSeconds,
|
||||
shellNoOutputTimeoutSec: shellNoOutputTimeoutSec,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// agentToolTimeoutMinutes 返回 agent.tool_timeout_minutes(与 executeToolViaMCP 一致);cfg 为 nil 时 0。
|
||||
func agentToolTimeoutMinutes(cfg *config.Config) int {
|
||||
if cfg == nil {
|
||||
return 0
|
||||
}
|
||||
return cfg.Agent.ToolTimeoutMinutes
|
||||
}
|
||||
|
||||
func agentToolWaitTimeoutSeconds(cfg *config.Config) int {
|
||||
if cfg == nil {
|
||||
return 0
|
||||
}
|
||||
return cfg.Agent.ToolWaitTimeoutSeconds
|
||||
}
|
||||
|
||||
// agentShellNoOutputTimeoutSeconds:0=默认 300s(5 分钟);-1=关闭;>0=自定义秒数。
|
||||
func agentShellNoOutputTimeoutSeconds(cfg *config.Config) int {
|
||||
if cfg == nil {
|
||||
return 300
|
||||
}
|
||||
v := cfg.Agent.ShellNoOutputTimeoutSeconds
|
||||
if v < 0 {
|
||||
return 0
|
||||
}
|
||||
if v == 0 {
|
||||
return 300
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"cyberstrike-ai/internal/config"
|
||||
)
|
||||
|
||||
func TestPrepareEinoAgenticSkillsStillCreatesReductionBackendWhenSkillsDisabled(t *testing.T) {
|
||||
ma := &config.MultiAgentConfig{
|
||||
EinoSkills: config.MultiAgentEinoSkillsConfig{Disable: true},
|
||||
EinoMiddleware: config.MultiAgentEinoMiddlewareConfig{
|
||||
ReductionEnable: true,
|
||||
},
|
||||
}
|
||||
loc, skillMW, fsTools, skillsRoot, err := prepareEinoAgenticSkills(context.Background(), "", ma, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if loc == nil {
|
||||
t.Fatal("agentic reduction backend must exist even when Skills are disabled")
|
||||
}
|
||||
if skillMW != nil || fsTools || skillsRoot != "" {
|
||||
t.Fatalf("Agentic Skills unexpectedly enabled: mw=%v fs=%v root=%q", skillMW, fsTools, skillsRoot)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package multiagent
|
||||
|
||||
import "github.com/cloudwego/eino/schema"
|
||||
|
||||
type einoStreamToolCallCompletionHandlerConfig struct {
|
||||
ConversationID string
|
||||
OrchMode string
|
||||
Progress func(eventType, message string, data interface{})
|
||||
RunProgress *einoRunProgressTracker
|
||||
RunMessages *einoRunMessageAccumulator
|
||||
MarkPending func(toolCallPendingInfo)
|
||||
}
|
||||
|
||||
type einoStreamToolCallCompletionHandler struct {
|
||||
conversationID string
|
||||
orchMode string
|
||||
progress func(eventType, message string, data interface{})
|
||||
runProgress *einoRunProgressTracker
|
||||
runMessages *einoRunMessageAccumulator
|
||||
markPending func(toolCallPendingInfo)
|
||||
}
|
||||
|
||||
func newEinoStreamToolCallCompletionHandler(cfg einoStreamToolCallCompletionHandlerConfig) *einoStreamToolCallCompletionHandler {
|
||||
return &einoStreamToolCallCompletionHandler{
|
||||
conversationID: cfg.ConversationID,
|
||||
orchMode: cfg.OrchMode,
|
||||
progress: cfg.Progress,
|
||||
runProgress: cfg.RunProgress,
|
||||
runMessages: cfg.RunMessages,
|
||||
markPending: cfg.MarkPending,
|
||||
}
|
||||
}
|
||||
|
||||
func (h *einoStreamToolCallCompletionHandler) Complete(fragments []schema.ToolCall, agentName string) *schema.Message {
|
||||
if h == nil {
|
||||
return nil
|
||||
}
|
||||
var lastToolChunk *schema.Message
|
||||
if merged := mergeStreamingToolCallFragments(fragments); len(merged) > 0 {
|
||||
lastToolChunk = mergeMessageToolCalls(&schema.Message{ToolCalls: merged})
|
||||
}
|
||||
if h.runProgress != nil {
|
||||
h.runProgress.EmitToolCalls(lastToolChunk, agentName, h.markPending)
|
||||
}
|
||||
if lastToolChunk != nil && len(lastToolChunk.ToolCalls) > 0 && h.runMessages != nil {
|
||||
h.runMessages.AppendAssistantToolCalls(lastToolChunk.ToolCalls)
|
||||
}
|
||||
return lastToolChunk
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package multiagent
|
||||
|
||||
import (
|
||||
copenai "cyberstrike-ai/internal/openai"
|
||||
)
|
||||
|
||||
// stripReasoningFromSummarizationPayload removes thinking / reasoning fields from a
|
||||
// chat-completions JSON body. Applied only to summarization Generate calls via
|
||||
// model.ModelOptions on the shared ChatModel — main-agent requests are unchanged.
|
||||
func stripReasoningFromSummarizationPayload(rawBody []byte) ([]byte, error) {
|
||||
return copenai.StripReasoningFromChatCompletionBody(rawBody)
|
||||
}
|
||||
Reference in New Issue
Block a user