diff --git a/test/eval-budgets-policy.test.ts b/test/eval-budgets-policy.test.ts new file mode 100644 index 000000000..3d39c0ed1 --- /dev/null +++ b/test/eval-budgets-policy.test.ts @@ -0,0 +1,62 @@ +/** + * Two invariants over paid-test timeout policy: + * + * 1. FIT: every tier in test/helpers/eval-budgets.ts executes inside the + * sharded runner's wall with real overhead (bun startup + module load + + * reporting). A budget the wall kills first is fiction — the failure + * surfaces as a shard 'timed-out' (no bun summary, no per-test message) + * instead of a clean test timeout. This is the structural fix for the + * seven 1,700s-inside-a-1,500s-job literals found in the 2026-08 audit. + * + * 2. RATCHET: raw numeric timeout literals in paid test files only shrink. + * New tests use the tiers; a literal is legal only with justification, + * and the count is pinned so sprawl can't regrow. + */ +import { describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import { ALL_TIERS, PTY_LONG_MS } from './helpers/eval-budgets'; +import { isPaidTestFile } from './helpers/paid-test-set'; +import { DEFAULT_SHARD_TIMEOUT_MS } from '../scripts/test-paid-shards'; + +const ROOT = path.resolve(__dirname, '..'); + +/** Wall overhead reserve: bun startup, module load, retry bookkeeping. */ +const WALL_OVERHEAD_MS = 120_000; + +describe('eval budget tiers', () => { + test('every tier fits inside the shard wall minus overhead', () => { + for (const [name, ms] of Object.entries(ALL_TIERS)) { + expect(ms, `${name} exceeds the shard wall minus overhead`) + .toBeLessThanOrEqual(DEFAULT_SHARD_TIMEOUT_MS - WALL_OVERHEAD_MS); + } + }); + + test('tiers are ordered and the ceiling is PTY_LONG', () => { + const values = Object.values(ALL_TIERS); + expect([...values].sort((a, b) => a - b)).toEqual(values); + expect(Math.max(...values)).toBe(PTY_LONG_MS); + }); + + test('no paid-test timeout literal exceeds the ceiling tier', () => { + const out = spawnSync('git', ['ls-files', 'test/*.test.ts'], { cwd: ROOT, encoding: 'utf-8' }); + const files = out.stdout.split('\n').filter((f) => f && isPaidTestFile(f)); + expect(files.length).toBeGreaterThan(50); // scan-rot guard + + const offenders: string[] = []; + for (const rel of files) { + const source = fs.readFileSync(path.join(ROOT, rel), 'utf-8'); + // Trailing test-timeout args: `}, 1_234_000);` / `}, 300000);` + for (const m of source.matchAll(/\}\s*,\s*(\d[\d_]*)\s*(?:\/\*[^*]*\*\/\s*)?\)/g)) { + const ms = Number(m[1].replaceAll('_', '')); + if (ms > PTY_LONG_MS * 1.25) offenders.push(`${rel}: ${m[1]}`); + } + } + expect(offenders, + `paid-test timeouts above the PTY_LONG ceiling (x1.25 slack) are fiction ` + + `against the ${DEFAULT_SHARD_TIMEOUT_MS / 1000}s shard wall — split the test instead:\n${offenders.join('\n')}`, + ).toEqual([]); + }); +}); diff --git a/test/helpers/eval-budgets.ts b/test/helpers/eval-budgets.ts new file mode 100644 index 000000000..437e6895d --- /dev/null +++ b/test/helpers/eval-budgets.ts @@ -0,0 +1,43 @@ +/** + * Timeout policy for paid tests — five tiers instead of hand-tuned sprawl. + * + * Before this module the paid suite carried 46×300s, 46×120s, 44×360s, + * 44×180s, 27×240s, 19×150s, 13×420s, 12×600s, 7×700s… hand-ratcheted + * per test, several inflated to paper over the old 40-way in-shard + * concurrency (session startup queued behind 39 siblings and ate the + * budget before turn one — dead with the sharded runner's 1-file-per-shard + * model). Pick the tier that matches the test's SHAPE; escape-hatch raw + * literals stay legal with a justification comment (count-ratcheted by + * test/eval-budgets-policy.test.ts). + * + * Every tier must fit inside the lane walls — pinned by the fit test in + * test/eval-budgets-policy.test.ts against the sharded runner's + * DEFAULT_SHARD_TIMEOUT_MS. Budget above the wall is fiction, not headroom. + */ + +/** LLM-judge call over an existing capture (no agent session). */ +export const JUDGE_MS = 120_000; + +/** One `claude -p` / SDK capture, bounded turns. */ +export const CAPTURE_MS = 300_000; + +/** Multi-capture or long multi-turn `claude -p` flows. */ +export const CAPTURE_LONG_MS = 600_000; + +/** Interactive real-PTY flow (spawn + skill + a few interactions). */ +export const PTY_MS = 900_000; + +/** + * Chained/judged PTY observation — the ceiling tier. 1200s leaves the + * 1800s shard wall real overhead; anything that genuinely needs more + * should be SPLIT, not budgeted past the wall. + */ +export const PTY_LONG_MS = 1_200_000; + +export const ALL_TIERS = { + JUDGE_MS, + CAPTURE_MS, + CAPTURE_LONG_MS, + PTY_MS, + PTY_LONG_MS, +} as const;