mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-16 09:55:29 +02:00
evals: SDK runner default Opus -> Sonnet (D1a)
agent-sdk-runner defaulted to Opus 4.7 while session-runner (the claude -p path) defaulted to Sonnet — an inconsistency between the two runners, not a decision anyone made. Unpinned tests were implicitly asserting the expensive model. The 30+ tests that genuinely need Opus already pin it via opts.model. Re-baselined by the periodic run in this branch's final verification; regressors get explicit Opus pins. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
d464412010
commit
07f7d30e80
@@ -299,7 +299,10 @@ export async function runAgentSdkTest(
|
|||||||
const sem = getApiSemaphore();
|
const sem = getApiSemaphore();
|
||||||
const maxRetries = opts.maxRetries ?? 3;
|
const maxRetries = opts.maxRetries ?? 3;
|
||||||
const queryImpl: QueryProvider = opts.queryProvider ?? query;
|
const queryImpl: QueryProvider = opts.queryProvider ?? query;
|
||||||
const model = opts.model ?? 'claude-opus-4-7';
|
// Default matches session-runner's Sonnet (D1a, 2026-08): the old Opus
|
||||||
|
// default was an inconsistency between the two runners, not a choice —
|
||||||
|
// tests that need Opus pin it via opts.model (30+ already do).
|
||||||
|
const model = opts.model ?? 'claude-sonnet-4-6';
|
||||||
|
|
||||||
// NOTE on env: the SDK child gets the COMPLETE hermetic env (allowlist
|
// NOTE on env: the SDK child gets the COMPLETE hermetic env (allowlist
|
||||||
// scrub + ANTHROPIC_API_KEY + hermetic CLAUDE_CONFIG_DIR/GSTACK_HOME), with
|
// scrub + ANTHROPIC_API_KEY + hermetic CLAUDE_CONFIG_DIR/GSTACK_HOME), with
|
||||||
|
|||||||
@@ -191,7 +191,7 @@ This is a capture test, not an interactive session. Skip any system-audit / envi
|
|||||||
timeout: 240_000,
|
timeout: 240_000,
|
||||||
testName: opts.testName,
|
testName: opts.testName,
|
||||||
runId: opts.runId,
|
runId: opts.runId,
|
||||||
model: opts.model ?? 'claude-opus-4-7',
|
model: opts.model ?? 'claude-sonnet-4-6', // D1a: align with session-runner default; Opus is opt-in
|
||||||
});
|
});
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -253,7 +253,7 @@ Rules for this run:
|
|||||||
timeout: opts.timeout ?? 300_000,
|
timeout: opts.timeout ?? 300_000,
|
||||||
testName: opts.testName,
|
testName: opts.testName,
|
||||||
runId: opts.runId,
|
runId: opts.runId,
|
||||||
model: opts.model ?? 'claude-opus-4-7',
|
model: opts.model ?? 'claude-sonnet-4-6', // D1a: align with session-runner default; Opus is opt-in
|
||||||
});
|
});
|
||||||
|
|
||||||
const readSections = new Set<string>();
|
const readSections = new Set<string>();
|
||||||
@@ -334,7 +334,7 @@ Write the verbatim text of that AskUserQuestion (the full decision brief: title,
|
|||||||
timeout: 240_000,
|
timeout: 240_000,
|
||||||
testName: opts.testName,
|
testName: opts.testName,
|
||||||
runId: opts.runId,
|
runId: opts.runId,
|
||||||
model: opts.model ?? 'claude-opus-4-7',
|
model: opts.model ?? 'claude-sonnet-4-6', // D1a: align with session-runner default; Opus is opt-in
|
||||||
});
|
});
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|||||||
Reference in New Issue
Block a user