feat(evals): ~12-minute blocking paid lanes and a non-blocking marathon lane

- Planner budget mode (--slice-budget S --jobs J): recorded per-tier wall
  times pack into as many ~9-minute executors as the work needs; the plan
  records per-slice estimates and the CI job timeout (supervised worst case
  + 20 min). evals.yml and evals-periodic.yml derive matrix size and
  timeout-minutes from it; max-parallel covers every slice at once.
- Case shards: plan/design/review-army/shared-libs(-paths) run one registered
  case per process (<file>#<case id>, exact name pattern, exactly one case).
- Retry rule: a timed-out attempt is a verdict. Only files whose every case
  budget is CAPTURE tier or shorter keep one retry; walls shrink to match.
- Marathon tier: positive selection, excluded from gate/periodic planners,
  run by the new evals-marathon.yml (weekly + dispatch, fresh, own report).
- PR-lane E2E reuse of verified first-attempt passes on identical inputs;
  the report rejects reuse outside the fast PR profile.
- Duration seed from census run 36385945043, per tier and per case shard.
This commit is contained in:
garrytan committed 2026-09-29 16:19:52 +00:00
1 parent acb7bc02b1
commit 0023d011a3
18 files changed
+1773 -450

No files matched your search

+8 -5
View File
@@ -24,9 +24,10 @@ import {
DEFAULT_JOBS,
DEFAULT_SHARD_TIMEOUT_MS,
resolvePaidShardBudget,
expandCaseShards,
type PaidTier,
} from '../scripts/test-paid-shards';
import { FINDING_RETRY_BUDGETS } from './helpers/eval-budgets';
import { FILE_RETRY_BUDGETS } from './helpers/eval-budgets';
const ROOT = path.resolve(import.meta.dir, '..');
// 5% margin over the theoretical bound: detach setup, lock wait, aggregation.
@@ -57,7 +58,7 @@ describe('eval:bg detach timeouts cover the sharded runner worst case', () => {
['periodic', 'eval:bg:periodic'],
] as Array<[PaidTier, string]>) {
test(`${script} covers ordinary ${tier} waves plus registered excess x ${MARGIN}`, () => {
const files = selectPaidTestFiles(collectPaidTestFiles(), tier).selected;
const files = expandCaseShards(selectPaidTestFiles(collectPaidTestFiles(), tier).selected, tier);
expect(files.length).toBeGreaterThan(0);
const floor = Math.ceil(worstCaseSeconds(files) * MARGIN);
const configured = detachTimeoutSeconds(script);
@@ -78,7 +79,7 @@ describe('eval:bg detach timeouts cover the sharded runner worst case', () => {
const pkg = JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf8'));
const jobs = Number(pkg.scripts['test:pr'].match(/EVALS_JOBS=\$\{EVALS_JOBS:-(\d+)\}/)?.[1]);
expect(jobs).toBe(2);
const files = selectPaidTestFiles(collectPaidTestFiles(), 'gate').selected;
const files = expandCaseShards(selectPaidTestFiles(collectPaidTestFiles(), 'gate').selected, 'gate');
const floor = Math.ceil(worstCaseSeconds(files, jobs) * MARGIN);
expect(detachTimeoutSeconds('eval:bg:pr')).toBeGreaterThanOrEqual(floor);
});
@@ -86,7 +87,7 @@ describe('eval:bg detach timeouts cover the sharded runner worst case', () => {
test('eval:bg:release covers both complete tiers and their existing margins', () => {
const files = collectPaidTestFiles();
const floor = (['gate', 'periodic'] as const).reduce((sum, tier) =>
sum + Math.ceil(worstCaseSeconds(selectPaidTestFiles(files, tier).selected) * MARGIN), 0);
sum + Math.ceil(worstCaseSeconds(expandCaseShards(selectPaidTestFiles(files, tier).selected, tier)) * MARGIN), 0);
expect(detachTimeoutSeconds('eval:bg:release')).toBeGreaterThanOrEqual(floor);
});
});
@@ -94,7 +95,9 @@ describe('eval:bg detach timeouts cover the sharded runner worst case', () => {
// One long job and one ordinary job can run side by side; the long job still
// needs its whole wall, regardless of the number of ordinary workers.
test('a heterogeneous pair rejects the old uniform-wall floor', () => {
const pair = [FINDING_RETRY_BUDGETS[0]!.file, 'test/skill-e2e-other.test.ts'];
// The longest registered wall (single-attempt finding files now fit the ordinary wall).
const longest = [...FILE_RETRY_BUDGETS].sort((a, b) => b.shardMs - a.shardMs)[0]!;
const pair = [longest.file, 'test/skill-e2e-other.test.ts'];
const actualLongest = Math.max(...pair.map(file => resolvePaidShardBudget([file]).timeoutMs)) / 1000;
expect(worstCaseSeconds(pair, 2)).toBe(actualLongest);
expect(worstCaseSeconds(pair, 2)).toBeGreaterThan(DEFAULT_SHARD_TIMEOUT_MS / 1000);