gstack/test/helpers/eval-budgets.ts

/**
 * Timeout policy for paid tests — five tiers instead of hand-tuned sprawl.
 *
 * Before this module the paid suite carried 46×300s, 46×120s, 44×360s,
 * 44×180s, 27×240s, 19×150s, 13×420s, 12×600s, 7×700s… hand-ratcheted
 * per test, several inflated to paper over the old 40-way in-shard
 * concurrency (session startup queued behind 39 siblings and ate the
 * budget before turn one — dead with the sharded runner's 1-file-per-shard
 * model). Pick the tier that matches the test's SHAPE; escape-hatch raw
 * literals stay legal with a justification comment (count-ratcheted by
 * test/eval-budgets-policy.test.ts).
 *
 * Every tier must fit inside the lane walls — pinned by the fit test in
 * test/eval-budgets-policy.test.ts against the sharded runner's
 * DEFAULT_SHARD_TIMEOUT_MS. Budget above the wall is fiction, not headroom.
 */

/** LLM-judge call over an existing capture (no agent session). */
export const JUDGE_MS = 120_000;

/** One bounded capture: SDK execution or the first displayed native question. */
export const CAPTURE_MS = 300_000;

/** Multi-capture or long multi-turn `claude -p` flows. */
export const CAPTURE_LONG_MS = 600_000;

/** Interactive real-PTY flow (spawn + skill + a few interactions). */
export const PTY_MS = 900_000;

/**
 * Chained/judged PTY observation — the ceiling tier. 1200s leaves the
 * 1800s shard wall real overhead; anything that genuinely needs more
 * should be split or use an explicitly registered workflow exception with
 * corresponding runner and CI walls; never inflate an ordinary tier.
 */
export const PTY_LONG_MS = 1_200_000;

export const ALL_TIERS = {
  JUDGE_MS,
  CAPTURE_MS,
  CAPTURE_LONG_MS,
  PTY_MS,
  PTY_LONG_MS,
} as const;

/**
 * Explicit exception for one uninterrupted four-phase workflow. These are
 * specified allowances, not measured latency or a conservative confidence bound.
 * The historical 900-second failure remains a failure. Ordinary tiers do not grow.
 */
export const AUTOPLAN_CHAIN_BUDGET = {
  id: 'autoplan-four-native-phases-v1',
  file: 'test/skill-e2e-autoplan-chain.test.ts',
  workMs: 4 * PTY_LONG_MS,
  sessionMs: 84 * 60_000,
  testMs: 85 * 60_000,
  shardMs: 172 * 60_000,
  retries: 1,
  shardReserveMs: 2 * 60_000,
  ciJobMs: 200 * 60_000,
  ciReserveMs: 28 * 60_000,
  reason: 'One command must complete CEO, Design, DX and Eng, including native reviews and amendment handoffs.',
} as const;

/** Whole-file supervision must cover each existing attempt and its retry.
 * These six fixtures already allow 25 minutes per case; the old 30-minute
 * wall could kill a second attempt after five minutes. No case budget grows.
 * Reserve the sequential upper bound even when Bun runs sibling cases together.
 */
export const FINDING_RETRY_BUDGETS = [
  { file: 'test/skill-e2e-plan-ceo-finding-count.test.ts', cases: 2 },
  { file: 'test/skill-e2e-plan-ceo-split-overflow.test.ts', cases: 1 },
  { file: 'test/skill-e2e-plan-design-finding-count.test.ts', cases: 1 },
  { file: 'test/skill-e2e-plan-devex-finding-count.test.ts', cases: 1 },
  { file: 'test/skill-e2e-plan-eng-finding-count.test.ts', cases: 1 },
  { file: 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts', cases: 1 },
].map(({ file, cases }) => ({
  file, cases,
  id: `${file.slice('test/skill-e2e-'.length, -'.test.ts'.length)}-existing-retry-v1`,
  testMs: 1_500_000,
  retries: 1,
  shardReserveMs: AUTOPLAN_CHAIN_BUDGET.shardReserveMs,
  shardMs: cases * 1_500_000 * 2 + AUTOPLAN_CHAIN_BUDGET.shardReserveMs,
}));

/** Three existing captures and one configured retry; only supervision grows. */
export const AUQ_CONSISTENCY_RETRY_BUDGET = {
  file: 'test/skill-e2e-auq-consistency.test.ts',
  id: 'auq-consistency-existing-retry-v1',
  cases: 1,
  testMs: 3 * CAPTURE_MS + 60_000,
  retries: 1,
  shardReserveMs: AUTOPLAN_CHAIN_BUDGET.shardReserveMs,
  shardMs: (3 * CAPTURE_MS + 60_000) * 2 + AUTOPLAN_CHAIN_BUDGET.shardReserveMs,
} as const;

/** These fixtures have a fixed case count in every supported tier. */
export const STRICT_RETRY_CASE_BUDGETS = [...FINDING_RETRY_BUDGETS, AUQ_CONSISTENCY_RETRY_BUDGET];

/** Whole-file walls cover all existing cases and retries, even if Bun runs them
 * sequentially. Mixed-tier files reserve their larger complete tier, never a
 * currently selected subset. These rows add no case-count or model-work policy.
 * The 10-second terms preserve the existing Codex/recording finalization grace.
 */
export const FILE_RETRY_BUDGETS = [
  ...STRICT_RETRY_CASE_BUDGETS,
  ...[
    // Fourteen workflow judges include their 10s recording grace; the other
    // eleven judges retain 120s. Supervise all 25 and the existing one retry.
    { file: 'test/skill-llm-eval.test.ts', attemptMs: 14 * (JUDGE_MS + 10_000) + 11 * JUDGE_MS, retries: 1 },
    { file: 'test/codex-e2e-plan-format.test.ts', attemptMs: 4 * (CAPTURE_LONG_MS + 10_000), retries: 1 },
    { file: 'test/skill-e2e-auq-matrix.test.ts', attemptMs: 6 * CAPTURE_MS, retries: 1 },
    { file: 'test/skill-e2e-plan-format.test.ts', attemptMs: 4 * (CAPTURE_MS + 10_000), retries: 1 },
    { file: 'test/skill-e2e-auto-decide-preserved.test.ts', attemptMs: PTY_MS, retries: 1 },
    { file: 'test/skill-e2e-plan-ceo-finding-floor.test.ts', attemptMs: PTY_MS, retries: 1 },
    { file: 'test/skill-e2e-plan-eng-finding-floor.test.ts', attemptMs: PTY_MS, retries: 1 },
    { file: 'test/skill-e2e-plan-design-finding-floor.test.ts', attemptMs: PTY_MS, retries: 1 },
    { file: 'test/skill-e2e-plan-devex-finding-floor.test.ts', attemptMs: PTY_MS, retries: 1 },
    { file: 'test/skill-e2e-plan-mode-no-op.test.ts', attemptMs: 5 * CAPTURE_LONG_MS, retries: 2 },
    { file: 'test/skill-e2e-plan-ceo-mode-routing.test.ts', attemptMs: 2 * CAPTURE_LONG_MS, retries: 1 },
    { file: 'test/skill-e2e-plan-eng-plan-mode.test.ts', attemptMs: 2 * CAPTURE_LONG_MS, retries: 1 },
    { file: 'test/skill-e2e-plan-prosons.test.ts', attemptMs: 4 * (CAPTURE_MS + 10_000), retries: 1 },
    // Gate: six 300s cases + one 610s case; periodic: two 900s + three 600s.
    { file: 'test/skill-e2e-plan.test.ts', attemptMs: Math.max(6 * CAPTURE_MS + CAPTURE_LONG_MS + 10_000, 2 * PTY_MS + 3 * CAPTURE_LONG_MS), retries: 1 },
  ].map(({ file, attemptMs, retries }) => ({
    file, attemptMs, retries,
    id: `${file.slice('test/'.length, -'.test.ts'.length)}-existing-retry-v1`,
    shardReserveMs: AUTOPLAN_CHAIN_BUDGET.shardReserveMs,
    shardMs: attemptMs * (retries + 1) + AUTOPLAN_CHAIN_BUDGET.shardReserveMs,
  })),
];

/** The only registered over-tier test budget; arbitrary per-file escapes fail. */
export function assertPaidTestBudget(file: string, ms: number): void {
  if (!Number.isSafeInteger(ms) || ms <= 0 ||
      (ms > PTY_LONG_MS * 1.25 &&
       (file !== AUTOPLAN_CHAIN_BUDGET.file || ms !== AUTOPLAN_CHAIN_BUDGET.testMs))) {
    throw new Error(`Unregistered paid test budget: ${file}: ${ms}`);
  }
}