test(evals): retire every paid automatic retry

Paid evals never retry (approved 2026-09-29): delete SHORT_CASE_RETRY_FILES
and retriesWithinCaseCap, drop the retry fields from the registered wall rows
(walls now cover one run plus reserve), make retriesForFiles return 0, pass
--retry 0 explicitly, and drop --retry 1 from the package.json paid scripts.
Add the eval:pass-rates alias. Tests that pinned the old retry allowance are
updated as a policy change; review-finalization-budget now proves late-result
recording under the production zero-retry arguments.
This commit is contained in:
garrytan committed 2026-09-29 19:09:51 +00:00
1 parent a427f05c58
commit b1f5bc0032
12 files changed
+131 -209

No files matched your search

+2 -2
View File
@@ -292,12 +292,12 @@ test('F9 changed-input selection produces three cases with exact patterns and co
expect(prProfileTestNamePattern(files[1], selected.selection)).toBe('(?:^|\\s)(?:investigate-owned-abort|investigate-owned-ending-error)$');
});
test('both F9 files fit the existing wall with every Bun retry and reserve', () => {
test('both F9 files fit the existing wall with their one run and reserve', () => {
for (const file of files) {
const source = fs.readFileSync(path.join(import.meta.dir, '..', file), 'utf8');
const count = PR_PROFILE_FILES[file].length;
expect([...source.matchAll(/\}, CAPTURE_MS\);/g)]).toHaveLength(count);
expect(retriesForFiles([file])).toBe(1);
expect(retriesForFiles([file])).toBe(0);
const budget = resolvePaidShardBudget([file]);
expect(budget).toEqual({ timeoutMs: DEFAULT_SHARD_TIMEOUT_MS, source: 'default', policyId: null });
expect(count * CAPTURE_MS * (retriesForFiles([file]) + 1) + 120000).toBeLessThanOrEqual(budget.timeoutMs);