add gstack 2 parity and lifecycle gates

scripts/test-free-shards.ts

@@ -3,10 +3,12 @@
  * test-free-shards — enumerate, shard, and curate the free test suite.
  *
  * Three jobs:
- *   1. Enumeration. Walk `browse/test/`, `test/`, `make-pdf/test/` and return
+ *   1. Enumeration. Walk all five free-test roots and return
  *      every `*.test.{ts,tsx,js,jsx,mjs,cjs}` that isn't a paid-eval test.
- *   2. Sharding. Stable-hash assign each test to one of N shards. Used by CI
- *      to parallelize the free suite when needed.
+ *   2. Sharding. Build deterministic, size-bounded shards and isolate tests
+ *      whose module setup mutates process.env or whose cleanup schedules
+ *      process.exit(0). Used by CI to parallelize the free suite without
+ *      letting one file leak state into, or truncate, unrelated work.
  *   3. Curation (Windows-safe filter). Scan each test's content for POSIX-only
  *      patterns (`/bin/bash`, `sh -c`, raw `/tmp/`, `chmod`, `xargs`). Files
  *      that match are excluded from the Windows-safe subset — they would fail
@@ -26,10 +28,15 @@
 
 import * as fs from 'fs';
 import * as path from 'path';
-import { spawnSync } from 'child_process';
 
 const ROOT = path.resolve(import.meta.dir, '..');
-const TEST_ROOTS = ['browse/test', 'test', 'make-pdf/test'] as const;
+export const FREE_TEST_ROOTS = [
+  'browse/test',
+  'test',
+  'make-pdf/test',
+  'design/test',
+  'ios-qa/daemon/test',
+] as const;
 const TEST_FILE_REGEX = /\.test\.(?:[cm]?[jt]s|tsx|jsx)$/;
 
 // Tests that require API spend, external services, or e2e harnesses.
@@ -109,8 +116,14 @@ const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }> = [
 ];
 
 export const DEFAULT_SHARD_COUNT = 20;
+export const DEFAULT_MAX_FILES_PER_SHARD = 20;
 export const FREE_TEST_TIMEOUT_MS = 10_000;
 
+const SCHEDULED_EXIT_ZERO = /\b(?:setTimeout|setInterval|setImmediate|queueMicrotask)\s*\(\s*(?:(?:async\s*)?(?:\([^)]*\)|[$\w]+)\s*=>|function(?:\s+[$\w]+)?\s*\([^)]*\)\s*\{)[\s\S]{0,256}?\bprocess\.exit\s*\(\s*0\s*\)/;
+// Deliberately require column zero. That identifies conventional module-scope
+// setup while avoiding process.env changes indented inside hooks and tests.
+const TOP_LEVEL_PROCESS_ENV_MUTATION = /^(?:process\.env\.[A-Za-z_][A-Za-z0-9_]*[ \t]*=(?!=)|delete[ \t]+process\.env\.[A-Za-z_][A-Za-z0-9_]*(?:[ \t]*;)?[ \t]*(?:\/\/.*)?$)/m;
+
 export function normalizeRelativePath(filePath: string): string {
   return filePath.replace(/\\/g, '/');
 }
@@ -155,7 +168,7 @@ function walkTestFiles(dirPath: string): string[] {
 
 export function collectFreeTestFiles(rootDir = ROOT): string[] {
   const discovered = new Set<string>();
-  for (const testRoot of TEST_ROOTS) {
+  for (const testRoot of FREE_TEST_ROOTS) {
     const absoluteRoot = path.join(rootDir, testRoot);
     if (!fs.existsSync(absoluteRoot)) continue;
     for (const fullPath of walkTestFiles(absoluteRoot)) {
@@ -219,6 +232,69 @@ export function assignFilesToShards(files: string[], shardCount: number): string
     .filter(filesInShard => filesInShard.length > 0);
 }
 
+export function containsScheduledProcessExitZero(source: string): boolean {
+  return SCHEDULED_EXIT_ZERO.test(source);
+}
+
+export function hasScheduledProcessExitZero(absolutePath: string): boolean {
+  try {
+    return containsScheduledProcessExitZero(fs.readFileSync(absolutePath, 'utf8'));
+  } catch {
+    return false;
+  }
+}
+
+export function containsTopLevelProcessEnvMutation(source: string): boolean {
+  return TOP_LEVEL_PROCESS_ENV_MUTATION.test(source);
+}
+
+export function hasTopLevelProcessEnvMutation(absolutePath: string): boolean {
+  try {
+    return containsTopLevelProcessEnvMutation(fs.readFileSync(absolutePath, 'utf8'));
+  } catch {
+    return false;
+  }
+}
+
+export interface BoundedShardOptions {
+  rootDir?: string;
+  maxFilesPerShard?: number;
+}
+
+/**
+ * Produce deterministically bounded shards. Tests with module-scope process.env
+ * mutations are isolated so state cannot leak across files. Tests that schedule
+ * process.exit(0) are isolated so cleanup cannot terminate unrelated files.
+ */
+export function planBoundedFreeTestShards(
+  files: string[],
+  options: BoundedShardOptions = {},
+): string[][] {
+  const rootDir = options.rootDir ?? ROOT;
+  const maxFilesPerShard = options.maxFilesPerShard ?? DEFAULT_MAX_FILES_PER_SHARD;
+  if (!Number.isInteger(maxFilesPerShard) || maxFilesPerShard <= 0) {
+    throw new Error(`Maximum files per shard must be a positive integer. Received: ${maxFilesPerShard}`);
+  }
+
+  const normal: string[] = [];
+  const isolated: string[] = [];
+  for (const file of [...new Set(files)].sort()) {
+    const absolutePath = path.join(rootDir, file);
+    if (
+      hasScheduledProcessExitZero(absolutePath)
+      || hasTopLevelProcessEnvMutation(absolutePath)
+    ) isolated.push(file);
+    else normal.push(file);
+  }
+
+  const shards: string[][] = [];
+  for (let index = 0; index < normal.length; index += maxFilesPerShard) {
+    shards.push(normal.slice(index, index + maxFilesPerShard));
+  }
+  for (const file of isolated) shards.push([file]);
+  return shards;
+}
+
 export function buildShardArgs(files: string[]): string[] {
   return ['test', ...files, '--max-concurrency=1', `--timeout=${FREE_TEST_TIMEOUT_MS}`];
 }
@@ -271,21 +347,18 @@ function formatShardSummary(shards: string[][]): string[] {
   });
 }
 
-function runShard(files: string[], shardNumber: number, totalShards: number): number {
+async function runShard(files: string[], shardNumber: number, totalShards: number): Promise<number> {
   const header = `[test:free] shard ${shardNumber}/${totalShards} (${files.length} files)`;
   console.log(header);
-  const result = spawnSync(process.execPath, buildShardArgs(files), {
-    cwd: ROOT,
-    stdio: 'inherit',
-    env: process.env,
-  });
-  if (result.status !== 0) {
-    console.error(`${header} failed with exit code ${result.status ?? 1}`);
+  const { runStrictTestShard } = await import('./test-free-strict');
+  const exitCode = await runStrictTestShard(files);
+  if (exitCode !== 0) {
+    console.error(`${header} failed with exit code ${exitCode}`);
   }
-  return result.status ?? 1;
+  return exitCode;
 }
 
-function main(): number {
+async function main(): Promise<number> {
   const options = parseCliOptions(process.argv.slice(2));
   const allFiles = collectFreeTestFiles();
   if (allFiles.length === 0) {
@@ -312,7 +385,11 @@ function main(): number {
     return 0;
   }
 
-  const shards = assignFilesToShards(files, options.shardCount);
+  if (!Number.isInteger(options.shardCount) || options.shardCount <= 0) {
+    throw new Error(`--shards must be a positive integer. Received: ${options.shardCount}`);
+  }
+  const maxFilesPerShard = Math.max(1, Math.ceil(files.length / options.shardCount));
+  const shards = planBoundedFreeTestShards(files, { maxFilesPerShard });
   if (options.dryRun) {
     console.log(`\nWould run ${files.length} files across ${shards.length} shards.`);
     for (const line of formatShardSummary(shards)) console.log(line);
@@ -323,11 +400,11 @@ function main(): number {
     if (!Number.isInteger(options.shardIndex) || options.shardIndex < 1 || options.shardIndex > shards.length) {
       throw new Error(`--shard must be between 1 and ${shards.length}. Received: ${options.shardIndex}`);
     }
-    return runShard(shards[options.shardIndex - 1], options.shardIndex, shards.length);
+    return await runShard(shards[options.shardIndex - 1], options.shardIndex, shards.length);
   }
 
   for (let index = 0; index < shards.length; index += 1) {
-    const exitCode = runShard(shards[index], index + 1, shards.length);
+    const exitCode = await runShard(shards[index], index + 1, shards.length);
     if (exitCode !== 0) return exitCode;
   }
 
@@ -335,5 +412,5 @@ function main(): number {
 }
 
 if (import.meta.main) {
-  process.exitCode = main();
+  process.exitCode = await main();
 }

scripts/test-free-strict.ts

@@ -0,0 +1,347 @@
+#!/usr/bin/env bun
+/**
+ * Run the default free suite while working around a Bun test runner bug where
+ * failures can be printed even though the child exits successfully.
+ *
+ * The shared free-test enumerator supplies the canonical roots and exclusions.
+ * Output is forwarded byte-for-byte as it arrives; only complete Bun result
+ * lines and terminal summaries are classified.
+ */
+
+import { spawn, type ChildProcess } from 'node:child_process';
+import { StringDecoder } from 'node:string_decoder';
+import * as path from 'node:path';
+import {
+  buildShardArgs,
+  collectFreeTestFiles,
+  planBoundedFreeTestShards,
+} from './test-free-shards';
+
+const ROOT = path.resolve(import.meta.dir, '..');
+const ANSI_ESCAPE = /\u001B\[[0-?]*[ -/]*[@-~]/g;
+const BUN_FAIL_RESULT = /^\(fail\) .+ \[(?:\d+(?:\.\d+)?)(?:ns|us|\u00b5s|ms|s)\]$/;
+const BUN_BETWEEN_TESTS_ERROR = '# Unhandled error between tests';
+const BUN_TERMINAL_SUMMARY = /^Ran \d+ tests? across (\d+) files?\. \[(?:\d+(?:\.\d+)?)(?:ns|us|\u00b5s|ms|s)\]$/;
+
+export type BunTestOutputFinding = 'failed-test' | 'unhandled-between-tests';
+
+export interface BunTestOutputSummary {
+  failedTests: number;
+  unhandledBetweenTests: number;
+  terminalFileCounts: number[];
+}
+
+export type ForwardedTerminationSignal = 'SIGINT' | 'SIGTERM';
+
+export interface TerminationSignalSource {
+  on(event: string, listener: () => void): unknown;
+  off(event: string, listener: () => void): unknown;
+}
+
+export interface TerminationTimerApi {
+  schedule(callback: () => void, delayMs: number): unknown;
+  cancel(handle: unknown): void;
+}
+
+export interface ChildSignalForwarding {
+  readonly receivedSignal: ForwardedTerminationSignal | null;
+  dispose(): void;
+}
+
+const DEFAULT_TERMINATION_TIMER: TerminationTimerApi = {
+  schedule: (callback, delayMs) => setTimeout(callback, delayMs),
+  cancel: (handle) => clearTimeout(handle as ReturnType<typeof setTimeout>),
+};
+
+function killWithoutThrowing(
+  child: Pick<ChildProcess, 'kill'>,
+  signal: NodeJS.Signals,
+): void {
+  try {
+    child.kill(signal);
+  } catch {
+    // The child may have exited between close detection and signal delivery.
+  }
+}
+
+/**
+ * Bind one active child to the parent's termination lifecycle. SIGINT and
+ * SIGTERM get a grace period so Bun can clean up; a repeated signal, timeout,
+ * or synchronous parent exit uses SIGKILL so the child cannot be orphaned.
+ */
+export function installChildSignalForwarding(
+  child: Pick<ChildProcess, 'kill'>,
+  source: TerminationSignalSource = process,
+  timer: TerminationTimerApi = DEFAULT_TERMINATION_TIMER,
+  graceMs = 5_000,
+): ChildSignalForwarding {
+  let receivedSignal: ForwardedTerminationSignal | null = null;
+  let forceTimer: unknown = null;
+  let disposed = false;
+
+  const forward = (signal: ForwardedTerminationSignal): void => {
+    if (disposed) return;
+    if (receivedSignal !== null) {
+      killWithoutThrowing(child, 'SIGKILL');
+      return;
+    }
+    receivedSignal = signal;
+    killWithoutThrowing(child, signal);
+    forceTimer = timer.schedule(() => {
+      forceTimer = null;
+      killWithoutThrowing(child, 'SIGKILL');
+    }, graceMs);
+  };
+  const onSigint = () => forward('SIGINT');
+  const onSigterm = () => forward('SIGTERM');
+  const onExit = () => killWithoutThrowing(child, 'SIGKILL');
+
+  source.on('SIGINT', onSigint);
+  source.on('SIGTERM', onSigterm);
+  source.on('exit', onExit);
+
+  return {
+    get receivedSignal() {
+      return receivedSignal;
+    },
+    dispose() {
+      if (disposed) return;
+      disposed = true;
+      source.off('SIGINT', onSigint);
+      source.off('SIGTERM', onSigterm);
+      source.off('exit', onExit);
+      if (forceTimer !== null) timer.cancel(forceTimer);
+      forceTimer = null;
+    },
+  };
+}
+
+export function terminationSignalExitCode(signal: ForwardedTerminationSignal): number {
+  return signal === 'SIGINT' ? 130 : 143;
+}
+
+export function classifyBunTestOutputLine(rawLine: string): BunTestOutputFinding | null {
+  const line = rawLine.replace(ANSI_ESCAPE, '').replace(/\r$/, '');
+  if (BUN_FAIL_RESULT.test(line)) return 'failed-test';
+  if (line === BUN_BETWEEN_TESTS_ERROR) return 'unhandled-between-tests';
+  return null;
+}
+
+export function parseBunTerminalSummaryLine(rawLine: string): number | null {
+  const line = rawLine.replace(ANSI_ESCAPE, '').replace(/\r$/, '');
+  const match = BUN_TERMINAL_SUMMARY.exec(line);
+  return match ? Number.parseInt(match[1], 10) : null;
+}
+
+/** Incrementally classifies output without assuming process chunks align to lines. */
+export class BunTestOutputClassifier {
+  private readonly decoder = new StringDecoder('utf8');
+  private pending = '';
+  private failedTests = 0;
+  private unhandledBetweenTests = 0;
+  private terminalFileCounts: number[] = [];
+
+  write(chunk: Uint8Array | string): void {
+    this.pending += typeof chunk === 'string'
+      ? chunk
+      : this.decoder.write(Buffer.from(chunk));
+    this.consumeCompleteLines();
+  }
+
+  end(): BunTestOutputSummary {
+    this.pending += this.decoder.end();
+    if (this.pending.length > 0) this.classify(this.pending);
+    this.pending = '';
+    return this.summary();
+  }
+
+  summary(): BunTestOutputSummary {
+    return {
+      failedTests: this.failedTests,
+      unhandledBetweenTests: this.unhandledBetweenTests,
+      terminalFileCounts: [...this.terminalFileCounts],
+    };
+  }
+
+  private consumeCompleteLines(): void {
+    let newline = this.pending.indexOf('\n');
+    while (newline !== -1) {
+      this.classify(this.pending.slice(0, newline));
+      this.pending = this.pending.slice(newline + 1);
+      newline = this.pending.indexOf('\n');
+    }
+  }
+
+  private classify(line: string): void {
+    const finding = classifyBunTestOutputLine(line);
+    if (finding === 'failed-test') this.failedTests += 1;
+    if (finding === 'unhandled-between-tests') this.unhandledBetweenTests += 1;
+    const terminalFileCount = parseBunTerminalSummaryLine(line);
+    if (terminalFileCount !== null) this.terminalFileCounts.push(terminalFileCount);
+  }
+}
+
+export function strictTestExitCode(
+  childExitCode: number,
+  summary: BunTestOutputSummary,
+  expectedFiles?: number,
+): number {
+  if (childExitCode !== 0) return childExitCode;
+  if (summary.failedTests > 0 || summary.unhandledBetweenTests > 0) return 1;
+  if (expectedFiles !== undefined && !summary.terminalFileCounts.includes(expectedFiles)) return 1;
+  return 0;
+}
+
+/** The default safety boundary is one Bun process per test file. */
+export function planDefaultFreeTestShards(files: string[], rootDir = ROOT): string[][] {
+  return planBoundedFreeTestShards(files, { rootDir, maxFilesPerShard: 1 });
+}
+
+/**
+ * Bun treats positional test paths as substring filters. Resolve every
+ * canonical relative path before spawning so `test/foo.test.ts` cannot also
+ * select `browse/test/foo.test.ts`.
+ */
+export function exactTestFileSelectors(files: string[], rootDir = ROOT): string[] {
+  return files.map((file) => path.isAbsolute(file) ? path.normalize(file) : path.resolve(rootDir, file));
+}
+
+function forwardAndClassify(
+  stream: NodeJS.ReadableStream,
+  destination: NodeJS.WriteStream,
+  classifier: BunTestOutputClassifier,
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    stream.on('data', (chunk: Buffer | string) => {
+      classifier.write(chunk);
+      destination.write(chunk);
+    });
+    stream.on('end', resolve);
+    stream.on('error', reject);
+  });
+}
+
+function waitForClose(child: ChildProcess): Promise<number> {
+  return new Promise((resolve, reject) => {
+    child.once('error', reject);
+    child.once('close', (code, signal) => {
+      if (typeof code === 'number') {
+        resolve(code);
+        return;
+      }
+      console.error(`[test:strict] Bun test terminated by signal ${signal ?? 'unknown'}`);
+      resolve(1);
+    });
+  });
+}
+
+async function runBestEffortSlopDiff(): Promise<ForwardedTerminationSignal | null> {
+  let forwarding: ChildSignalForwarding | null = null;
+  try {
+    const child = spawn(process.execPath, ['run', 'slop:diff'], {
+      cwd: ROOT,
+      env: process.env,
+      stdio: ['inherit', 'inherit', 'ignore'],
+      windowsHide: true,
+    });
+    forwarding = installChildSignalForwarding(child);
+    await waitForClose(child);
+    return forwarding.receivedSignal;
+  } catch {
+    // This command was best-effort in the previous package.json entry too.
+    return forwarding?.receivedSignal ?? null;
+  } finally {
+    forwarding?.dispose();
+  }
+}
+
+export async function runDefaultFreeTests(): Promise<number> {
+  const files = collectFreeTestFiles(ROOT);
+  if (files.length === 0) throw new Error('No free test files were discovered.');
+  const shards = planDefaultFreeTestShards(files, ROOT);
+  console.log(`[test:strict] ${files.length} files across ${shards.length} singleton shards`);
+
+  for (let index = 0; index < shards.length; index += 1) {
+    const shard = shards[index];
+    console.log(`[test:strict] shard ${index + 1}/${shards.length} (${shard.length} files)`);
+    const exitCode = await runStrictTestShard(shard);
+    if (exitCode !== 0) return exitCode;
+  }
+
+  const slopSignal = await runBestEffortSlopDiff();
+  return slopSignal === null ? 0 : terminationSignalExitCode(slopSignal);
+}
+
+export async function runStrictTestShard(files: string[]): Promise<number> {
+  if (files.length === 0) throw new Error('Cannot run an empty free-test shard.');
+  const child = spawn(process.execPath, buildShardArgs(exactTestFileSelectors(files)), {
+    cwd: ROOT,
+    env: process.env,
+    stdio: ['inherit', 'pipe', 'pipe'],
+    windowsHide: true,
+  });
+  const forwarding = installChildSignalForwarding(child);
+
+  if (!child.stdout || !child.stderr) {
+    killWithoutThrowing(child, 'SIGKILL');
+    forwarding.dispose();
+    throw new Error('Bun test output pipes were not created');
+  }
+
+  const stdoutClassifier = new BunTestOutputClassifier();
+  const stderrClassifier = new BunTestOutputClassifier();
+  const stdoutDone = forwardAndClassify(child.stdout, process.stdout, stdoutClassifier);
+  const stderrDone = forwardAndClassify(child.stderr, process.stderr, stderrClassifier);
+  let childExitCode: number;
+  try {
+    childExitCode = await waitForClose(child);
+    await Promise.all([stdoutDone, stderrDone]);
+  } finally {
+    forwarding.dispose();
+  }
+
+  if (forwarding.receivedSignal !== null) {
+    return terminationSignalExitCode(forwarding.receivedSignal);
+  }
+
+  const stdoutSummary = stdoutClassifier.end();
+  const stderrSummary = stderrClassifier.end();
+  const summary: BunTestOutputSummary = {
+    failedTests: stdoutSummary.failedTests + stderrSummary.failedTests,
+    unhandledBetweenTests:
+      stdoutSummary.unhandledBetweenTests + stderrSummary.unhandledBetweenTests,
+    terminalFileCounts: [
+      ...stdoutSummary.terminalFileCounts,
+      ...stderrSummary.terminalFileCounts,
+    ],
+  };
+  const exitCode = strictTestExitCode(childExitCode, summary, files.length);
+
+  if (childExitCode === 0 && exitCode !== 0) {
+    if (summary.failedTests > 0 || summary.unhandledBetweenTests > 0) {
+      console.error(
+        `[test:strict] Bun exited 0 despite ${summary.failedTests} failed test result(s) `
+        + `and ${summary.unhandledBetweenTests} unhandled between-tests error(s).`,
+      );
+    }
+    if (!summary.terminalFileCounts.includes(files.length)) {
+      const reported = summary.terminalFileCounts.length > 0
+        ? summary.terminalFileCounts.join(', ')
+        : 'none';
+      console.error(
+        `[test:strict] Bun exited 0 without a terminal summary for all ${files.length} `
+        + `expected file(s); reported file counts: ${reported}.`,
+      );
+    }
+  }
+  return exitCode;
+}
+
+if (import.meta.main) {
+  try {
+    process.exitCode = await runDefaultFreeTests();
+  } catch (error) {
+    console.error(`[test:strict] ${error instanceof Error ? error.message : String(error)}`);
+    process.exitCode = 1;
+  }
+}