perf: add model pinning infrastructure + rate-limit telemetry to E2E runner

Default E2E model changed from Opus to Sonnet (5x faster, 5x cheaper).
Session runner now accepts `model` option with EVALS_MODEL env var override.
Added timing telemetry (first_response_ms, max_inter_turn_ms) and wall_clock_ms
to eval-store for diagnosing rate-limit impact. Added EVALS_FAST test filtering.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-03-21 13:12:17 -07:00
co-authored by Claude Opus 4.6
parent d68a70d351
commit d442aadf4a
3 changed files with 52 additions and 2 deletions
+21
View File
@@ -30,6 +30,13 @@ export const evalsEnabled = !!process.env.EVALS;
// Set EVALS_ALL=1 to force all tests. Set EVALS_BASE to override base branch.
export let selectedTests: string[] | null = null; // null = run all
// EVALS_FAST: skip the 8 slowest tests (all Opus quality tests) for quick feedback
const FAST_EXCLUDED_TESTS = [
'plan-ceo-review-selective', 'plan-ceo-review', 'retro', 'retro-base-branch',
'design-consultation-core', 'design-consultation-existing',
'qa-fix-loop', 'design-review-fix',
];
if (evalsEnabled && !process.env.EVALS_ALL) {
const baseBranch = process.env.EVALS_BASE
|| detectBaseBranch(ROOT)
@@ -48,6 +55,17 @@ if (evalsEnabled && !process.env.EVALS_ALL) {
// If changedFiles is empty (e.g., on main branch), selectedTests stays null → run all
}
// Apply EVALS_FAST filter after diff-based selection
if (evalsEnabled && process.env.EVALS_FAST) {
if (selectedTests === null) {
// Run all minus excluded
selectedTests = Object.keys(E2E_TOUCHFILES).filter(t => !FAST_EXCLUDED_TESTS.includes(t));
} else {
selectedTests = selectedTests.filter(t => !FAST_EXCLUDED_TESTS.includes(t));
}
process.stderr.write(`EVALS_FAST: excluded ${FAST_EXCLUDED_TESTS.length} slow tests, running ${selectedTests.length}\n\n`);
}
export const describeE2E = evalsEnabled ? describe : describe.skip;
/** Wrap a describe block to skip entirely if none of its tests are selected. */
@@ -164,6 +182,9 @@ export function recordE2E(
exit_reason: result.exitReason,
timeout_at_turn: result.exitReason === 'timeout' ? result.costEstimate.turnsUsed : undefined,
last_tool_call: lastTool,
model: result.model,
first_response_ms: result.firstResponseMs,
max_inter_turn_ms: result.maxInterTurnMs,
...extra,
});
}