test: free runner — strict output, parallel execution, stable shard indices

Three coupled changes to scripts/test-free-shards.ts: 1. STRICT OUTPUT: runFreeShard streams through the paid runner's BunTestOutputClassifier — exit 0 without bun's 'Ran N tests across M files' summary, with (fail) lines, or with a wrong file count is a FAILURE (anti-truncation backstop at the runner layer), plus an external wall-clock timeout that SIGKILLs the process group (timed-out distinct from failed; exit 124 vs 1). Also fixes a latent shard-bleed: file selectors now use exactTestFileSelectors (relative paths were substring filters that matched sibling roots). 2. PARALLEL: full-suite mode is one 'bun test --parallel' invocation (Bun 1.3.13). Measured semantics recorded in the header: per-file worker isolation, standard summary, and mid-suite process.exit surfaces as a crashed-worker FAIL with exit 1 — strictly safer than serial, where the same exit truncates silently. No static weight lists; --shards M --shard i keeps deterministic hash partitioning for CI matrices (native --shard rejected: round-robin renumbers when files land). Spawned shards get throwaway GSTACK_HOME/TMPDIR so parallel shards can't contend on real state. Per-shard epilogue prints files/seconds/status every run. 3. Stable indices: assignFilesToShards no longer drops empty shards, so a shard's index depends only on the file hash and requested count — an empty CI matrix slot is a fast no-op success, not a renumbering. package.json 'test' now delegates to the runner (TEST_ROOTS becomes the single source of truth for roots; slop:diff tail preserved; the runner inherits the 30s per-test timeout the old glob passed inline). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-22 14:07:14 +02:00 · 2026-08-15 08:22:44 -07:00
parent 587217e2ec
commit 41160a14ed
4 changed files with 501 additions and 58 deletions
@@ -9,7 +9,10 @@ import {
  curateWindowsSafe,
  stableHash,
  assignFilesToShards,
+  buildShardArgs,
  normalizeRelativePath,
+  runFreeShard,
+  FREE_TEST_TIMEOUT_MS,
 } from '../scripts/test-free-shards';

 const ROOT = path.resolve(import.meta.dir, '..');
@@ -106,12 +109,33 @@ describe('test-free-shards: sharding', () => {
    expect(stableHash('foo.test.ts')).not.toBe(stableHash('bar.test.ts'));
  });

-  test('assignFilesToShards distributes files into N non-empty shards', () => {
+  test('assignFilesToShards partitions every file across exactly shardCount shards', () => {
    const files = ['a.test.ts', 'b.test.ts', 'c.test.ts', 'd.test.ts', 'e.test.ts'];
    const shards = assignFilesToShards(files, 3);
-    const flattened = shards.flat();
-    expect(flattened.sort()).toEqual([...files].sort());
-    expect(shards.every((s) => s.length > 0)).toBe(true);
+    expect(shards.length).toBe(3);
+    expect(shards.flat().sort()).toEqual([...files].sort());
+  });
+
+  test('empty shards are preserved so indices stay stable for a CI matrix', () => {
+    // 2 files can never occupy 10 shards — the rest MUST be present and empty,
+    // not filtered out (filtering renumbered every later shard by occupancy).
+    const files = ['a.test.ts', 'b.test.ts'];
+    const shards = assignFilesToShards(files, 10);
+    expect(shards.length).toBe(10);
+    expect(shards.flat().sort()).toEqual([...files].sort());
+    expect(shards.some((s) => s.length === 0)).toBe(true);
+  });
+
+  test("a file's shard index depends only on its own path — other files never renumber it", () => {
+    const target = 'test/target.test.ts';
+    const expected = stableHash(target) % 7;
+    const alone = assignFilesToShards([target], 7);
+    const crowded = assignFilesToShards(
+      [target, 'test/a.test.ts', 'test/b.test.ts', 'test/c.test.ts', 'test/d.test.ts', 'browse/test/e.test.ts'],
+      7,
+    );
+    expect(alone.findIndex((s) => s.includes(target))).toBe(expected);
+    expect(crowded.findIndex((s) => s.includes(target))).toBe(expected);
  });

  test('assignFilesToShards rejects invalid shard counts', () => {
@@ -126,3 +150,136 @@ describe('test-free-shards: sharding', () => {
    expect(a).toEqual(b);
  });
 });
+
+describe('test-free-shards: shard args', () => {
+  test('resolves exact absolute selectors (no substring shard bleed) and pins the per-test timeout', () => {
+    const args = buildShardArgs(['test/foo.test.ts'], { rootDir: ROOT });
+    expect(args[0]).toBe('test');
+    expect(args[1]).toBe(path.resolve(ROOT, 'test/foo.test.ts'));
+    expect(args).toContain(`--timeout=${FREE_TEST_TIMEOUT_MS}`);
+    expect(args).toContain('--max-concurrency=1');
+    expect(args).not.toContain('--parallel');
+  });
+
+  test('parallel mode swaps serial max-concurrency for --parallel', () => {
+    const args = buildShardArgs(['test/foo.test.ts'], { rootDir: ROOT, parallel: true });
+    expect(args).toContain('--parallel');
+    expect(args).not.toContain('--max-concurrency=1');
+  });
+
+  test('per-test timeout matches the 30s the package.json test script used before the repoint', () => {
+    expect(FREE_TEST_TIMEOUT_MS).toBe(30_000);
+  });
+});
+
+describe('test-free-shards: strict shard execution', () => {
+  // Fake command seam, same pattern as test/paid-shards.test.ts: each "file"
+  // label selects a child command. Unlike the paid runner, runFreeShard
+  // enforces the terminal-summary file count on injected commands too, so
+  // fake PASSING commands must print a synthetic bun summary line.
+  const SUMMARY_1 = 'Ran 3 tests across 1 files. [12.00ms]';
+  const BUSY_LOOP = 'const end = Date.now() + 600000; while (Date.now() < end) {}';
+  const FAIL_LINE = '(fa' + 'il) planted failure [0.10ms]'; // split so this source file never contains a raw bun fail line
+
+  const commandFor = (files: string[]) => {
+    const mode = files[0];
+    if (mode === 'spin') return { command: process.execPath, args: ['-e', BUSY_LOOP] };
+    if (mode === 'no-summary') return { command: process.execPath, args: ['-e', 'console.log("ok")'] };
+    if (mode === 'fail-exit') {
+      return { command: process.execPath, args: ['-e', `console.log(${JSON.stringify(SUMMARY_1)}); process.exit(3)`] };
+    }
+    if (mode === 'fail-line-exit-zero') {
+      return { command: process.execPath, args: ['-e', `console.log(${JSON.stringify(FAIL_LINE)}); console.log(${JSON.stringify(SUMMARY_1)})`] };
+    }
+    if (mode === 'wrong-file-count') {
+      return { command: process.execPath, args: ['-e', 'console.log("Ran 3 tests across 4 files. [12.00ms]")'] };
+    }
+    return { command: process.execPath, args: ['-e', `console.log(${JSON.stringify(SUMMARY_1)})`] };
+  };
+
+  test('exit 0 WITHOUT bun\'s terminal summary is a FAILURE (anti-truncation backstop)', async () => {
+    const outcome = await runFreeShard(['no-summary'], 1, 1, { commandFor, quiet: true, log: () => {} });
+    expect(outcome.status).toBe('failed');
+    expect(outcome.exitCode).toBe(0);
+  });
+
+  test('exit 0 WITH the terminal summary passes, and the per-shard epilogue line is printed', async () => {
+    const lines: string[] = [];
+    const outcome = await runFreeShard(['pass'], 1, 1, { commandFor, quiet: true, log: (l) => lines.push(l) });
+    expect(outcome.status).toBe('passed');
+    expect(lines.some((l) => /^\[test:free\] shard 1\/1: 1 files, \d+s, pass$/.test(l))).toBe(true);
+  });
+
+  test('a non-zero exit stays a failure even when the summary is present', async () => {
+    const outcome = await runFreeShard(['fail-exit'], 1, 1, { commandFor, quiet: true, log: () => {} });
+    expect(outcome.status).toBe('failed');
+    expect(outcome.exitCode).toBe(3);
+  });
+
+  test('a printed (fail) result line is a failure even on exit 0 (bun exit-code bug class)', async () => {
+    const outcome = await runFreeShard(['fail-line-exit-zero'], 1, 1, { commandFor, quiet: true, log: () => {} });
+    expect(outcome.status).toBe('failed');
+    expect(outcome.exitCode).toBe(0);
+  });
+
+  test('a summary reporting the wrong file count is a failure (partial execution)', async () => {
+    const outcome = await runFreeShard(['wrong-file-count'], 1, 1, { commandFor, quiet: true, log: () => {} });
+    expect(outcome.status).toBe('failed');
+  });
+
+  test('a spinning shard is killed at the wall-clock deadline and reported timed-out, distinct from failed', async () => {
+    const lines: string[] = [];
+    const outcome = await runFreeShard(['spin'], 1, 1, {
+      commandFor, quiet: true, wallTimeoutMs: 1_200, log: (l) => lines.push(l),
+    });
+    expect(outcome.status).toBe('timed-out');
+    expect(outcome.status).not.toBe('failed');
+    // Killed at the deadline, not left to burn the full 600s busy loop.
+    expect(outcome.elapsedMs).toBeLessThan(30_000);
+    expect(outcome.groupPid).toBeGreaterThan(0);
+    if (process.platform !== 'win32') {
+      expect(() => process.kill(outcome.groupPid as number, 0)).toThrow();
+    }
+    expect(lines.some((l) => /^\[test:free\] shard 1\/1: 1 files, \d+s, timed-out$/.test(l))).toBe(true);
+  }, 30_000);
+
+  test('an empty shard is a fast no-op success and never spawns (stable CI-matrix indices)', async () => {
+    const lines: string[] = [];
+    const outcome = await runFreeShard([], 7, 20, {
+      commandFor: () => { throw new Error('an empty shard must not spawn a child'); },
+      log: (l) => lines.push(l),
+    });
+    expect(outcome.status).toBe('passed');
+    expect(lines.some((l) => /^\[test:free\] shard 7\/20: 0 files, 0s, pass$/.test(l))).toBe(true);
+  });
+
+  test('each spawned shard gets its own throwaway GSTACK_HOME and TMPDIR, removed after the run', async () => {
+    const captureDir = fs.mkdtempSync(path.join(os.tmpdir(), 'free-shard-env-'));
+    const dump = path.join(captureDir, 'env.json');
+    try {
+      const script =
+        `const fs = require("fs");`
+        + `fs.writeFileSync(${JSON.stringify(dump)}, JSON.stringify({`
+        + `  home: process.env.GSTACK_HOME, tmp: process.env.TMPDIR,`
+        + `  homeExists: fs.existsSync(process.env.GSTACK_HOME || ""),`
+        + `  tmpExists: fs.existsSync(process.env.TMPDIR || "") }));`
+        + `console.log(${JSON.stringify(SUMMARY_1)});`;
+      const outcome = await runFreeShard(['env-dump'], 1, 1, {
+        commandFor: () => ({ command: process.execPath, args: ['-e', script] }),
+        quiet: true,
+        log: () => {},
+      });
+      expect(outcome.status).toBe('passed');
+      const seen = JSON.parse(fs.readFileSync(dump, 'utf8'));
+      expect(seen.home).toContain('gstack-free-shard-');
+      expect(seen.homeExists).toBe(true);
+      expect(seen.tmpExists).toBe(true);
+      expect(seen.home).not.toBe(process.env.GSTACK_HOME ?? '');
+      expect(seen.tmp).not.toBe(process.env.TMPDIR ?? '');
+      // The throwaway state dir is cleaned up once the shard finishes.
+      expect(fs.existsSync(seen.home)).toBe(false);
+    } finally {
+      fs.rmSync(captureDir, { recursive: true, force: true });
+    }
+  });
+});