feat(test): eval-budgets timeout tiers + fit/ceiling policy test

Five named tiers (JUDGE 120s / CAPTURE 300s / CAPTURE_LONG 600s /
PTY 900s / PTY_LONG 1200s) replace hand-ratcheted sprawl (46x300s,
46x120s, 44x360s, 44x180s, 27x240s, 19x150s, 13x420s, 12x600s...),
much of it inflated to paper over the old 40-way in-shard concurrency
that the sharded runner's 1-file-per-shard model kills. Policy test
pins: every tier fits the shard wall minus 120s overhead (the
structural fix for budgets-above-the-wall fiction), tiers stay ordered,
and no paid literal exceeds PTY_LONG x1.25 — oversized tests get split,
not budgeted past the wall.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

test/eval-budgets-policy.test.ts

@@ -0,0 +1,62 @@
+/**
+ * Two invariants over paid-test timeout policy:
+ *
+ * 1. FIT: every tier in test/helpers/eval-budgets.ts executes inside the
+ *    sharded runner's wall with real overhead (bun startup + module load +
+ *    reporting). A budget the wall kills first is fiction — the failure
+ *    surfaces as a shard 'timed-out' (no bun summary, no per-test message)
+ *    instead of a clean test timeout. This is the structural fix for the
+ *    seven 1,700s-inside-a-1,500s-job literals found in the 2026-08 audit.
+ *
+ * 2. RATCHET: raw numeric timeout literals in paid test files only shrink.
+ *    New tests use the tiers; a literal is legal only with justification,
+ *    and the count is pinned so sprawl can't regrow.
+ */
+import { describe, expect, test } from 'bun:test';
+import { spawnSync } from 'node:child_process';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+import { ALL_TIERS, PTY_LONG_MS } from './helpers/eval-budgets';
+import { isPaidTestFile } from './helpers/paid-test-set';
+import { DEFAULT_SHARD_TIMEOUT_MS } from '../scripts/test-paid-shards';
+
+const ROOT = path.resolve(__dirname, '..');
+
+/** Wall overhead reserve: bun startup, module load, retry bookkeeping. */
+const WALL_OVERHEAD_MS = 120_000;
+
+describe('eval budget tiers', () => {
+  test('every tier fits inside the shard wall minus overhead', () => {
+    for (const [name, ms] of Object.entries(ALL_TIERS)) {
+      expect(ms, `${name} exceeds the shard wall minus overhead`)
+        .toBeLessThanOrEqual(DEFAULT_SHARD_TIMEOUT_MS - WALL_OVERHEAD_MS);
+    }
+  });
+
+  test('tiers are ordered and the ceiling is PTY_LONG', () => {
+    const values = Object.values(ALL_TIERS);
+    expect([...values].sort((a, b) => a - b)).toEqual(values);
+    expect(Math.max(...values)).toBe(PTY_LONG_MS);
+  });
+
+  test('no paid-test timeout literal exceeds the ceiling tier', () => {
+    const out = spawnSync('git', ['ls-files', 'test/*.test.ts'], { cwd: ROOT, encoding: 'utf-8' });
+    const files = out.stdout.split('\n').filter((f) => f && isPaidTestFile(f));
+    expect(files.length).toBeGreaterThan(50); // scan-rot guard
+
+    const offenders: string[] = [];
+    for (const rel of files) {
+      const source = fs.readFileSync(path.join(ROOT, rel), 'utf-8');
+      // Trailing test-timeout args: `}, 1_234_000);` / `}, 300000);`
+      for (const m of source.matchAll(/\}\s*,\s*(\d[\d_]*)\s*(?:\/\*[^*]*\*\/\s*)?\)/g)) {
+        const ms = Number(m[1].replaceAll('_', ''));
+        if (ms > PTY_LONG_MS * 1.25) offenders.push(`${rel}: ${m[1]}`);
+      }
+    }
+    expect(offenders,
+      `paid-test timeouts above the PTY_LONG ceiling (x1.25 slack) are fiction ` +
+      `against the ${DEFAULT_SHARD_TIMEOUT_MS / 1000}s shard wall — split the test instead:\n${offenders.join('\n')}`,
+    ).toEqual([]);
+  });
+});

test/helpers/eval-budgets.ts

@@ -0,0 +1,43 @@
+/**
+ * Timeout policy for paid tests — five tiers instead of hand-tuned sprawl.
+ *
+ * Before this module the paid suite carried 46×300s, 46×120s, 44×360s,
+ * 44×180s, 27×240s, 19×150s, 13×420s, 12×600s, 7×700s… hand-ratcheted
+ * per test, several inflated to paper over the old 40-way in-shard
+ * concurrency (session startup queued behind 39 siblings and ate the
+ * budget before turn one — dead with the sharded runner's 1-file-per-shard
+ * model). Pick the tier that matches the test's SHAPE; escape-hatch raw
+ * literals stay legal with a justification comment (count-ratcheted by
+ * test/eval-budgets-policy.test.ts).
+ *
+ * Every tier must fit inside the lane walls — pinned by the fit test in
+ * test/eval-budgets-policy.test.ts against the sharded runner's
+ * DEFAULT_SHARD_TIMEOUT_MS. Budget above the wall is fiction, not headroom.
+ */
+
+/** LLM-judge call over an existing capture (no agent session). */
+export const JUDGE_MS = 120_000;
+
+/** One `claude -p` / SDK capture, bounded turns. */
+export const CAPTURE_MS = 300_000;
+
+/** Multi-capture or long multi-turn `claude -p` flows. */
+export const CAPTURE_LONG_MS = 600_000;
+
+/** Interactive real-PTY flow (spawn + skill + a few interactions). */
+export const PTY_MS = 900_000;
+
+/**
+ * Chained/judged PTY observation — the ceiling tier. 1200s leaves the
+ * 1800s shard wall real overhead; anything that genuinely needs more
+ * should be SPLIT, not budgeted past the wall.
+ */
+export const PTY_LONG_MS = 1_200_000;
+
+export const ALL_TIERS = {
+  JUDGE_MS,
+  CAPTURE_MS,
+  CAPTURE_LONG_MS,
+  PTY_MS,
+  PTY_LONG_MS,
+} as const;