test(evals): retire every paid automatic retry

Paid evals never retry (approved 2026-09-29): delete SHORT_CASE_RETRY_FILES
and retriesWithinCaseCap, drop the retry fields from the registered wall rows
(walls now cover one run plus reserve), make retriesForFiles return 0, pass
--retry 0 explicitly, and drop --retry 1 from the package.json paid scripts.
Add the eval:pass-rates alias. Tests that pinned the old retry allowance are
updated as a policy change; review-finalization-budget now proves late-result
recording under the production zero-retry arguments.
This commit is contained in:
garrytan committed 2026-09-29 19:09:51 +00:00
1 parent a427f05c58
commit b1f5bc0032
12 files changed
+131 -209

No files matched your search

+9 -11
View File
@@ -1,4 +1,4 @@
/** The real review registrations must finish capture cleanup before Bun retries. */
/** The real review registrations record late results and clean up before finalization, under the production zero-retry arguments. */
import { expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
@@ -12,7 +12,7 @@ const CASES = [
['review-design-lite', 400, 35],
] as const;
for (const [id, workMs, maxTurns] of CASES) {
test.each(['recover', 'both-timeout'])(`${id} records late results before retry or finalization: %s`, scenario => {
test.each(['success', 'timeout'])(`${id} records late results before finalization: %s`, scenario => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'review-finalization-'));
const script = path.join(dir, 'registration.test.ts');
const facts = path.join(dir, 'events.jsonl');
@@ -41,7 +41,7 @@ mock.module(path.join(root, 'test/helpers/session-runner.ts'), () => ({
runSkillTest: async opts => {
const id = ++attempts;
event({ kind: 'start', id, timeout: opts.timeout, maxTurns: opts.maxTurns, cwd: opts.workingDirectory });
const timeout = id === 1 || ${JSON.stringify(scenario)} === 'both-timeout';
const timeout = ${JSON.stringify(scenario)} === 'timeout';
// Use the caller's actual work budget; only the provider and budget
// constants are scaled. The actual registered Bun outer deadline stays.
await new Promise(resolve => setTimeout(resolve, timeout ? opts.timeout + 50 : 80));
@@ -62,27 +62,25 @@ await import(path.join(root, ${JSON.stringify(PAID_FILE)}));
`);
try {
const retries = retriesForFiles([PAID_FILE]);
expect(retries).toBe(1);
expect(retries).toBe(0);
const child = Bun.spawnSync([process.execPath, ...buildPaidShardArgs([script], resolvePaidShardTimeoutMs([PAID_FILE]), 2, retries)], {
cwd: ROOT, timeout: 15_000, stdout: 'pipe', stderr: 'pipe',
env: { ...process.env, EVALS: '', EVALS_ALL: '', TMPDIR: dir, TMP: dir, TEMP: dir },
});
const output = child.stdout.toString() + child.stderr.toString();
expect(child.exitCode, output).toBe(scenario === 'recover' ? 0 : 1);
expect(child.exitCode, output).toBe(scenario === 'success' ? 0 : 1);
expect(output).not.toContain('Unhandled error between tests');
const events = fs.readFileSync(facts, 'utf8').trim().split('\n').map(line => JSON.parse(line));
const starts = events.filter(event => event.kind === 'start');
const ready = events.filter(event => event.kind === 'ready');
const records = events.filter(event => event.kind === 'record');
expect(starts.map(event => event.id)).toEqual([1, 2]);
expect(starts.map(event => event.id)).toEqual([1]);
expect(starts.map(({ timeout, maxTurns }) => ({ timeout, maxTurns })))
.toEqual([{ timeout: workMs, maxTurns }, { timeout: workMs, maxTurns }]);
expect(ready).toEqual([{ kind: 'ready', id: 1, fixtureExists: true }, { kind: 'ready', id: 2, fixtureExists: true }]);
.toEqual([{ timeout: workMs, maxTurns }]);
expect(ready).toEqual([{ kind: 'ready', id: 1, fixtureExists: true }]);
expect(records.map(event => [event.id, event.exitReason]))
.toEqual([[1, 'timeout'], [2, scenario === 'recover' ? 'success' : 'timeout']]);
.toEqual([[1, scenario === 'success' ? 'success' : 'timeout']]);
expect(events.findIndex(event => event.kind === 'record' && event.id === 1))
.toBeLessThan(events.findIndex(event => event.kind === 'start' && event.id === 2));
expect(events.findIndex(event => event.kind === 'record' && event.id === 2))
.toBeLessThan(events.findIndex(event => event.kind === 'finalized'));
expect(events.filter(event => event.kind === 'finalized')).toHaveLength(1);
expect(events.find(event => event.kind === 'registration')).toEqual({ kind: 'registration', name: id, outerMs: workMs + 50 + 5_000 });