feat(evals): sharded paid tier runner

scripts/test-paid-shards.ts runs the gate/periodic tier one Bun process
per test file, with an EXTERNAL wall-clock timeout that SIGKILLs the
shard's detached process group and an aggregate that distinguishes
passed / failed / timed-out / never-started — partial execution can no
longer read as a pass. Bun's native --shard/--isolate covers none of
this: no process-group kill (hung claude/codex PTY grandchildren
survive in-process isolation), no never-started taxonomy, no per-shard
env. Each shard child gets GSTACK_EVAL_DIR=<evalDir>/shards/<slug>/
(slug = test filename sans extension, stable across runs) so shard
baselines compare against their own prior runs.

Output classification lives in scripts/test-strict-output.ts (strict
exit-code derivation, incremental fail-line classifier, child signal
forwarding) so the runner and any future strict bun-test wrapper share
one implementation. Enumeration derives from the shared paid-test-set
module; tier exclusion fires only on an explicit whole-file
EVALS_TIER === '<other>' guard.

package.json gains test:gate:sharded / test:periodic:sharded, and
eval:bg:gate / eval:bg:periodic now run the sharded scripts with detach
timeouts sized to the worst case (gate: 49 shards x 30min / 4 jobs ~
6.2h -> 25200s; periodic: 59 -> 28800s).

test/paid-shards.test.ts pins enumeration, tier classification, and the
kill-and-continue property with a real busy-loop shard.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
(cherry picked from commit 5e76bd5931836257f896cedfe4e93912cb759c70)

test/paid-shards.test.ts

@@ -0,0 +1,117 @@
+/**
+ * Pins the paid-tier sharded runner (scripts/test-paid-shards.ts).
+ *
+ * Two properties matter, and both are why `test:gate` has never finished a run:
+ *   1. Enumeration + sharding — every file `test:gate`'s globs expand to gets
+ *      its own process, and tier exclusion only ever fires on explicit evidence.
+ *   2. A spinning shard is killed externally and the run CONTINUES. The fake
+ *      command here is a real busy loop, so an in-process timer could not save
+ *      it — exactly the failure mode `sample` caught on the wedged run.
+ */
+
+import { describe, test, expect } from 'bun:test';
+import {
+  PAID_TEST_GLOBS,
+  classifyPaidTestFile,
+  collectPaidTestFiles,
+  isPaidTestFile,
+  planPaidShards,
+  runPaidShards,
+  summarize,
+  type ShardOutcome,
+} from '../scripts/test-paid-shards';
+
+describe('paid test enumeration', () => {
+  test('matches the globs package.json test:gate expands', () => {
+    expect(isPaidTestFile('test/skill-e2e-qa-workflow.test.ts')).toBe(true);
+    expect(isPaidTestFile('test/skill-llm-eval.test.ts')).toBe(true);
+    expect(isPaidTestFile('test/codex-e2e.test.ts')).toBe(true);
+    expect(isPaidTestFile('test/skill-e2e-triage-audit.test.ts')).toBe(true);
+    // Outside the globs: no dash, extra suffix, or a free test.
+    expect(isPaidTestFile('test/skill-e2e.test.ts')).toBe(false);
+    expect(isPaidTestFile('test/codex-e2e-recommendation-substance.test.ts')).toBe(false);
+    expect(isPaidTestFile('test/paid-shards.test.ts')).toBe(false);
+  });
+
+  test('discovers files and gives each one its own shard', () => {
+    const files = collectPaidTestFiles();
+    expect(files.length).toBeGreaterThan(0);
+    expect(files.every(isPaidTestFile)).toBe(true);
+    expect(PAID_TEST_GLOBS.length).toBe(5);
+
+    const shards = planPaidShards(files);
+    expect(shards.flat().sort()).toEqual([...files].sort());
+    expect(shards.every((shard) => shard.length === 1)).toBe(true);
+  });
+});
+
+describe('tier classification', () => {
+  test('excludes only on an explicit other-tier guard', () => {
+    const gateGuard = "const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';";
+    const periodicGuard = "const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';";
+
+    expect(classifyPaidTestFile(gateGuard, 'gate').included).toBe(true);
+    expect(classifyPaidTestFile(periodicGuard, 'gate').included).toBe(false);
+    expect(classifyPaidTestFile(gateGuard, 'periodic').included).toBe(false);
+    expect(classifyPaidTestFile(periodicGuard, 'periodic').included).toBe(true);
+  });
+
+  test('keeps files whose tier is decided per-test at runtime', () => {
+    // Naming an E2E_TIERS key is not evidence — 'retro' appears in the
+    // LLM-judge file, which test:gate does run.
+    const noGuard = "runSkillTest('retro', async () => {});";
+    expect(classifyPaidTestFile(noGuard, 'gate').included).toBe(true);
+    expect(classifyPaidTestFile(noGuard, 'periodic').included).toBe(true);
+    expect(classifyPaidTestFile('', 'gate').included).toBe(true);
+  });
+});
+
+describe('shard execution', () => {
+  const BUSY_LOOP = 'const end = Date.now() + 600000; while (Date.now() < end) {}';
+
+  const commandFor = (files: string[]) => {
+    if (files[0] === 'spin') return { command: process.execPath, args: ['-e', BUSY_LOOP] };
+    if (files[0] === 'fail') return { command: process.execPath, args: ['-e', 'process.exit(3)'] };
+    return { command: process.execPath, args: ['-e', 'console.log("ok")'] };
+  };
+
+  test('a spinning shard times out, is killed, and the run continues', async () => {
+    const lines: string[] = [];
+    const summary = await runPaidShards([['spin'], ['fail'], ['pass']], {
+      timeoutMs: 1_200,
+      jobs: 1,
+      commandFor,
+      log: (line) => lines.push(line),
+    });
+
+    const byName = (name: string) => summary.outcomes.find((o) => o.files[0] === name) as ShardOutcome;
+    expect(byName('spin').status).toBe('timed-out');
+    expect(byName('fail').status).toBe('failed');
+    expect(byName('pass').status).toBe('passed');
+
+    // The run never aborted: every shard reports, none is 'never-started'.
+    expect(summary).toMatchObject({
+      total: 3, executed: 3, passed: 1, failed: 1, timedOut: 1, neverStarted: 0,
+    });
+
+    // The spinner was killed at the deadline, not left to burn a core.
+    expect(byName('spin').elapsedMs).toBeLessThan(30_000);
+    expect(byName('spin').groupPid).toBeGreaterThan(0);
+    if (process.platform !== 'win32') {
+      expect(() => process.kill(byName('spin').groupPid as number, 0)).toThrow();
+    }
+
+    // Heartbeat: a START and a terminal line per shard, with elapsed seconds.
+    expect(lines.filter((l) => l.includes(' START ')).length).toBe(3);
+    expect(lines.some((l) => /TIMED-OUT in \d+s/.test(l))).toBe(true);
+    expect(lines.some((l) => /PASSED in \d+s/.test(l))).toBe(true);
+  }, 30_000);
+
+  test('summarize reports shards that never ran', () => {
+    const summary = summarize([
+      { shard: 1, files: ['a'], status: 'passed', exitCode: 0, elapsedMs: 1, groupPid: 1 },
+      { shard: 2, files: ['b'], status: 'never-started', exitCode: null, elapsedMs: 0, groupPid: null },
+    ]);
+    expect(summary).toMatchObject({ total: 2, executed: 1, passed: 1, neverStarted: 1 });
+  });
+});