test(evals): retire every paid automatic retry

Paid evals never retry (approved 2026-09-29): delete SHORT_CASE_RETRY_FILES
and retriesWithinCaseCap, drop the retry fields from the registered wall rows
(walls now cover one run plus reserve), make retriesForFiles return 0, pass
--retry 0 explicitly, and drop --retry 1 from the package.json paid scripts.
Add the eval:pass-rates alias. Tests that pinned the old retry allowance are
updated as a policy change; review-finalization-budget now proves late-result
recording under the production zero-retry arguments.
This commit is contained in:
garrytan committed 2026-09-29 19:09:51 +00:00
1 parent a427f05c58
commit b1f5bc0032
12 files changed
+131 -209

No files matched your search

+2 -2
View File
@@ -152,9 +152,9 @@ function protocol(fault?: Fault, billing?: Array<number | undefined>, controls:
return { provider, directory: () => directory, calls: () => calls, sessions };
}
test('one bounded native case preserves the existing whole-file retry allowance', () => {
test('one bounded native case fits the whole-file wall and never retries', () => {
const retries = retriesForFiles(['test/skill-e2e-ship-skip.test.ts']);
expect(retries).toBe(1);
expect(retries).toBe(0);
expect(CAPTURE_MS * (retries + 1) + 120000).toBeLessThanOrEqual(DEFAULT_SHARD_TIMEOUT_MS);
});