/** * Timeout policy for paid tests — five tiers instead of hand-tuned sprawl. * * Before this module the paid suite carried 46×300s, 46×120s, 44×360s, * 44×180s, 27×240s, 19×150s, 13×420s, 12×600s, 7×700s… hand-ratcheted * per test, several inflated to paper over the old 40-way in-shard * concurrency (session startup queued behind 39 siblings and ate the * budget before turn one — dead with the sharded runner's 1-file-per-shard * model). Pick the tier that matches the test's SHAPE; escape-hatch raw * literals stay legal with a justification comment (count-ratcheted by * test/eval-budgets-policy.test.ts). * * Every tier must fit inside the lane walls — pinned by the fit test in * test/eval-budgets-policy.test.ts against the sharded runner's * DEFAULT_SHARD_TIMEOUT_MS. Budget above the wall is fiction, not headroom. */ /** LLM-judge call over an existing capture (no agent session). */ export const JUDGE_MS = 120_000; /** One bounded capture: SDK execution or the first displayed native question. */ export const CAPTURE_MS = 300_000; /** Multi-capture or long multi-turn `claude -p` flows. */ export const CAPTURE_LONG_MS = 600_000; /** Interactive real-PTY flow (spawn + skill + a few interactions). */ export const PTY_MS = 900_000; /** * Chained/judged PTY observation — the ceiling tier. 1200s leaves the * 1800s shard wall real overhead; anything that genuinely needs more * should be split or use an explicitly registered workflow exception with * corresponding runner and CI walls; never inflate an ordinary tier. */ export const PTY_LONG_MS = 1_200_000; export const ALL_TIERS = { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS, PTY_MS, PTY_LONG_MS, } as const; /** Supervision reserve added to every registered whole-file wall. */ export const SHARD_RESERVE_MS = 2 * 60_000; /** * Retry policy (approved 2026-09-29, eval reliability wave): paid evals never * retry. Each case's kind (E2E_KINDS) fixes its trials before the run: `rule` * one trial, `behavior` a panel of EVAL_POLICY.panel independent trials, and * `judge` one case that samples its judge panel internally. A failed verdict * is final for that run; a manual re-run adds trials under a new run attempt * and never replaces the original verdict. Rows below keep only wall * supervision; per-case budgets never change with this rule. */ /** Whole-file supervision for one run of every case. * These fixtures allow 25 minutes per case. * Reserve the sequential upper bound even when Bun runs sibling cases together. */ export const FINDING_RETRY_BUDGETS = [ { file: 'test/skill-e2e-plan-ceo-split-overflow.test.ts', cases: 1 }, { file: 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts', cases: 1 }, ].map(({ file, cases }) => ({ file, cases, id: `${file.slice('test/skill-e2e-'.length, -'.test.ts'.length)}-existing-retry-v1`, testMs: 1_500_000, caseMs: 1_500_000, shardReserveMs: SHARD_RESERVE_MS, shardMs: cases * 1_500_000 + SHARD_RESERVE_MS, })); /** Three existing captures in one 16-minute case. */ export const AUQ_CONSISTENCY_RETRY_BUDGET = { file: 'test/skill-e2e-auq-consistency.test.ts', id: 'auq-consistency-existing-retry-v1', cases: 1, testMs: 3 * CAPTURE_MS + 60_000, caseMs: 3 * CAPTURE_MS + 60_000, shardReserveMs: SHARD_RESERVE_MS, shardMs: 3 * CAPTURE_MS + 60_000 + SHARD_RESERVE_MS, } as const; /** These fixtures have a fixed case count in every supported tier. */ export const STRICT_RETRY_CASE_BUDGETS = [...FINDING_RETRY_BUDGETS, AUQ_CONSISTENCY_RETRY_BUDGET]; /** Whole-file walls cover all existing cases, even if Bun runs them * sequentially. Mixed-tier files reserve their larger complete tier, never a * currently selected subset. caseMs is the longest single case budget, the * wall of one isolated case shard. These rows add no case-count or model-work * policy. The 10-second terms preserve the existing Codex/recording * finalization grace. */ export const FILE_RETRY_BUDGETS = [ ...STRICT_RETRY_CASE_BUDGETS, ...[ { file: 'test/skill-e2e-qa-callers.test.ts', attemptMs: 5 * (CAPTURE_MS + 15_000), caseMs: CAPTURE_MS + 15_000 }, { file: 'test/skill-e2e-shared-libs-paths.test.ts', attemptMs: 3 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS }, { file: 'test/skill-e2e-ship-docsync.test.ts', attemptMs: 4 * CAPTURE_LONG_MS + 8 * CAPTURE_MS, caseMs: CAPTURE_LONG_MS }, // Seventeen workflow judges include their 10s recording grace; the other // seven judges retain 120s. Supervise all 24. { file: 'test/skill-llm-eval.test.ts', attemptMs: 17 * (JUDGE_MS + 10_000) + 7 * JUDGE_MS, caseMs: JUDGE_MS + 10_000 }, { file: 'test/skill-e2e-auq-matrix.test.ts', attemptMs: 6 * CAPTURE_MS, caseMs: CAPTURE_MS }, { file: 'test/skill-e2e-plan-format.test.ts', attemptMs: 4 * (CAPTURE_MS + 10_000), caseMs: CAPTURE_MS + 10_000 }, { file: 'test/skill-e2e-auto-decide-preserved.test.ts', attemptMs: PTY_MS, caseMs: PTY_MS }, { file: 'test/skill-e2e-plan-ceo-finding-floor.test.ts', attemptMs: PTY_MS, caseMs: PTY_MS }, { file: 'test/skill-e2e-plan-eng-finding-floor.test.ts', attemptMs: PTY_MS, caseMs: PTY_MS }, { file: 'test/skill-e2e-plan-design-finding-floor.test.ts', attemptMs: PTY_MS, caseMs: PTY_MS }, { file: 'test/skill-e2e-plan-devex-finding-floor.test.ts', attemptMs: PTY_MS, caseMs: PTY_MS }, { file: 'test/skill-e2e-plan-mode-no-op.test.ts', attemptMs: 5 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS }, { file: 'test/skill-e2e-plan-ceo-mode-routing.test.ts', attemptMs: 2 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS }, { file: 'test/skill-e2e-plan-eng-plan-mode.test.ts', attemptMs: 2 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS }, { file: 'test/skill-e2e-plan-prosons.test.ts', attemptMs: 4 * (CAPTURE_MS + 10_000), caseMs: CAPTURE_MS + 10_000 }, // Gate: six 300s cases + one 610s case; periodic: two 900s + three 600s. { file: 'test/skill-e2e-plan.test.ts', attemptMs: Math.max(6 * CAPTURE_MS + CAPTURE_LONG_MS + 10_000, 2 * PTY_MS + 3 * CAPTURE_LONG_MS), caseMs: PTY_MS }, ].map(({ file, attemptMs, caseMs }) => ({ file, attemptMs, caseMs, id: `${file.slice('test/'.length, -'.test.ts'.length)}-existing-retry-v1`, shardReserveMs: SHARD_RESERVE_MS, shardMs: attemptMs + SHARD_RESERVE_MS, })), ]; /** No paid test may exceed the ordinary tiers; arbitrary per-file escapes fail. */ export function assertPaidTestBudget(file: string, ms: number): void { if (!Number.isSafeInteger(ms) || ms <= 0 || ms > PTY_LONG_MS * 1.25) { throw new Error(`Unregistered paid test budget: ${file}: ${ms}`); } }