refactor(evals): paid shards spool to disk + shared runShardChild lifecycle

- runPaidShard no longer buffers whole 30-min stream-json streams in
  RAM (x concurrent jobs): every byte tees to a per-shard log file
  (slug-named, path printed at START for mid-run inspection and on the
  FAILED terminal line); failures print a 64KiB tail read back from
  disk; passing shards stay quiet (the file is the record) — the free
  runner's proven contract. Classification unchanged: the strict
  classifier still sees every byte first.
- the ~35 duplicated spawn/group-kill/wall-timer/finally-reap lines
  move into runShardChild in test-strict-output.ts (detached-per-
  platform spawn, signal forwarding, SIGKILL group kill at the wall,
  drain-before-verdict); designed so the free runner can migrate later
- expectedFiles drift fixed toward ENFORCEMENT: the injected-command
  exemption is gone — a fake command exiting 0 without bun's terminal
  summary now reads FAILED (pinned: silent-pass → failed)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-29 05:31:02 +00:00
co-authored by Claude Fable 5
parent 6ef8aaba65
commit 05bca51961
4 changed files with 352 additions and 47 deletions
+51 -1
View File
@@ -11,6 +11,7 @@
import { describe, test, expect } from 'bun:test';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
const ROOT = path.resolve(import.meta.dir, '..');
@@ -115,10 +116,17 @@ describe('tier classification', () => {
describe('shard execution', () => {
const BUSY_LOOP = 'const end = Date.now() + 600000; while (Date.now() < end) {}';
// PIN UPDATE (deliberate): the strict expectedFiles check is now enforced
// for injected fake commands too (drift fix toward the free runner's
// behavior), so a fake PASSING command must print a synthetic bun terminal
// summary — a summary-less exit 0 is the truncation class and reads FAILED.
const PASS_WITH_SUMMARY = 'console.log("ok"); console.log("Ran 1 tests across 1 files. [1ms]")';
const commandFor = (files: string[]) => {
if (files[0] === 'spin') return { command: process.execPath, args: ['-e', BUSY_LOOP] };
if (files[0] === 'fail') return { command: process.execPath, args: ['-e', 'process.exit(3)'] };
return { command: process.execPath, args: ['-e', 'console.log("ok")'] };
if (files[0] === 'silent-pass') return { command: process.execPath, args: ['-e', 'console.log("ok")'] };
return { command: process.execPath, args: ['-e', PASS_WITH_SUMMARY] };
};
test('a spinning shard times out, is killed, and the run continues', async () => {
@@ -153,6 +161,48 @@ describe('shard execution', () => {
expect(lines.some((l) => /PASSED in \d+s/.test(l))).toBe(true);
}, 30_000);
test('exit 0 WITHOUT the terminal summary is FAILED — enforced for injected commands too', async () => {
// The invisible-non-execution backstop: previously the paid runner
// exempted injected commandFor from the expectedFiles check, so a fake
// that exited 0 without bun's terminal summary recorded 'passed'. Now it
// matches the free runner: enforcement always on.
const summary = await runPaidShards([['silent-pass']], {
timeoutMs: 30_000, jobs: 1, commandFor, log: () => {},
});
expect(summary.outcomes[0].status).toBe('failed');
}, 30_000);
test('shard output spools to a per-shard log file; failures name the path', async () => {
const logDir = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-shard-logs-'));
const lines: string[] = [];
try {
const summary = await runPaidShards([['fail'], ['pass']], {
timeoutMs: 30_000, jobs: 2, commandFor, logDir, log: (line) => lines.push(line),
});
const byName = (name: string) => summary.outcomes.find((o) => o.files[0] === name) as ShardOutcome;
expect(byName('fail').status).toBe('failed');
expect(byName('pass').status).toBe('passed');
// One log per shard, named by slug, and it holds the child's full stream
// (nothing buffered in RAM: the file IS the record).
const logs = fs.readdirSync(logDir).sort();
expect(logs.length).toBe(2);
expect(logs.some((f) => f.includes('fail'))).toBe(true);
const passLog = logs.find((f) => f.includes('pass')) as string;
expect(fs.readFileSync(path.join(logDir, passLog), 'utf8')).toContain('Ran 1 tests across 1 files.');
// Every shard announces its log path up front; the FAILED terminal line
// repeats it, the PASSED one stays clean.
expect(lines.filter((l) => l.includes('full log:') && !l.includes('FAILED')).length).toBe(2);
const failLine = lines.find((l) => l.includes('FAILED')) as string;
expect(failLine).toContain(logDir);
const passLine = lines.find((l) => l.includes('PASSED')) as string;
expect(passLine).not.toContain(logDir);
} finally {
fs.rmSync(logDir, { recursive: true, force: true });
}
}, 30_000);
test('summarize reports shards that never ran', () => {
const summary = summarize([
{ shard: 1, files: ['a'], status: 'passed', exitCode: 0, elapsedMs: 1, groupPid: 1 },