Merge remote-tracking branch 'origin/main' into garrytan/binding-polarity-wave

scripts/capture-baseline.ts

@@ -14,6 +14,7 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import { captureBaseline } from '../test/helpers/capture-parity-baseline';
+import { PARITY_INVARIANTS } from '../test/helpers/parity-harness';
 
 const ROOT = path.resolve(import.meta.dir, '..');
 
@@ -33,7 +34,13 @@ const defaultOut = path.join(
 );
 const outPath = outOverride ? path.resolve(outOverride) : defaultOut;
 
-const baseline = captureBaseline({ repoRoot: ROOT, tag });
+const baseline = captureBaseline({
+  repoRoot: ROOT,
+  tag,
+  // Carved skills record UNION bytes (skeleton + sections/*.md) so the
+  // baseline measures the same thing parity-harness checks against.
+  sectionedSkills: PARITY_INVARIANTS.filter(i => i.sectioned).map(i => i.skill),
+});
 
 fs.mkdirSync(path.dirname(outPath), { recursive: true });
 fs.writeFileSync(outPath, JSON.stringify(baseline, null, 2) + '\n');

scripts/eval-select.ts

@@ -38,8 +38,11 @@ if (changedFiles.length === 0) {
   process.exit(0);
 }
 
-const e2eSelection = selectTests(changedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES);
-const llmSelection = selectTests(changedFiles, LLM_JUDGE_TOUCHFILES, GLOBAL_TOUCHFILES);
+// baseRef/cwd scope the map-diff path (used when touchfiles-data.ts changed)
+// to the same base this script diffed against — including a --base override.
+const selectOpts = { baseRef: baseBranch, cwd: ROOT };
+const e2eSelection = selectTests(changedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES, selectOpts);
+const llmSelection = selectTests(changedFiles, LLM_JUDGE_TOUCHFILES, GLOBAL_TOUCHFILES, selectOpts);
 
 if (jsonMode) {
   console.log(JSON.stringify({
@@ -49,6 +52,7 @@ if (jsonMode) {
       selected: e2eSelection.selected,
       skipped: e2eSelection.skipped,
       reason: e2eSelection.reason,
+      removed_tests: e2eSelection.removedTests ?? [],
       count: `${e2eSelection.selected.length}/${Object.keys(E2E_TOUCHFILES).length}`,
     },
     llm_judge: {
@@ -63,7 +67,10 @@ if (jsonMode) {
   console.log(`Changed files: ${changedFiles.length}`);
   console.log();
 
-  console.log(`E2E (${e2eSelection.reason}): ${e2eSelection.selected.length}/${Object.keys(E2E_TOUCHFILES).length} tests`);
+  console.log(`E2E: selected ${e2eSelection.selected.length} of ${Object.keys(E2E_TOUCHFILES).length}, reason: ${e2eSelection.reason}`);
+  if (e2eSelection.removedTests && e2eSelection.removedTests.length > 0) {
+    console.log(`  Removed from maps (reported, not selected): ${e2eSelection.removedTests.join(', ')}`);
+  }
   if (e2eSelection.selected.length > 0 && e2eSelection.selected.length < Object.keys(E2E_TOUCHFILES).length) {
     console.log(`  Selected: ${e2eSelection.selected.join(', ')}`);
     console.log(`  Skipped:  ${e2eSelection.skipped.join(', ')}`);
@@ -74,7 +81,7 @@ if (jsonMode) {
   }
   console.log();
 
-  console.log(`LLM-judge (${llmSelection.reason}): ${llmSelection.selected.length}/${Object.keys(LLM_JUDGE_TOUCHFILES).length} tests`);
+  console.log(`LLM-judge: selected ${llmSelection.selected.length} of ${Object.keys(LLM_JUDGE_TOUCHFILES).length}, reason: ${llmSelection.reason}`);
   if (llmSelection.selected.length > 0 && llmSelection.selected.length < Object.keys(LLM_JUDGE_TOUCHFILES).length) {
     console.log(`  Selected: ${llmSelection.selected.join(', ')}`);
     console.log(`  Skipped:  ${llmSelection.skipped.join(', ')}`);

scripts/test-paid-shards.ts

@@ -23,9 +23,18 @@
  *   3. No per-shard env / eval dir. Each shard needs its own GSTACK_EVAL_DIR
  *      so eval baselines are per-test-file instead of last-flush-wins.
  *
- * Worst-case wall clock (all shards hit the 30min timeout, 4 parallel jobs):
- * gate tier is 49 shards × 30min / 4 jobs ≈ 6.2h; periodic is 59 shards ≈ 7.4h.
- * The eval:bg:* detach timeouts (25200s / 28800s) are sized against these.
+ * Worst-case wall clock = ceil(shards / jobs) × shard timeout. Shard counts
+ * drift as test files land, so treat any number written here as stale.
+ * Do NOT hand-derive the eval:bg:* detach timeouts from a snapshot of
+ * these counts — test/eval-detach-timeout-floor.test.ts recomputes the bound
+ * from the live shard census every run and fails CI if package.json's numbers
+ * dip below it (undersized detach timeouts recreate never-started truncation).
+ *
+ * Env contract: EVALS_JOBS = how many shard PROCESSES run at once (this
+ * runner). EVALS_CONCURRENCY = bun's --max-concurrency WITHIN a shard (and the
+ * legacy single-process scripts). They were previously conflated: exporting
+ * the legacy value 15 gave you 15 concurrent Bun processes each spawning
+ * claude — the 429 storm.
  *
  * Enumeration matches package.json's `test:gate` globs (via the shared
  * test/helpers/paid-test-set.ts) and honors EVALS_TIER against the E2E_TIERS
@@ -41,7 +50,7 @@
  *   bun run scripts/test-paid-shards.ts --timeout 600 --jobs 2
  */
 
-import { spawn, type ChildProcess } from 'node:child_process';
+import { spawn } from 'node:child_process';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { normalizeRelativePath } from './test-free-shards';
@@ -50,10 +59,21 @@ import {
   exactTestFileSelectors,
   forwardAndClassify,
   installChildSignalForwarding,
+  isTerminationRequested,
+  killProcessGroup,
   strictTestExitCode,
 } from './test-strict-output';
 import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set';
 import { getProjectEvalDir } from '../test/helpers/eval-store';
+import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight';
+import {
+  detectBaseBranch,
+  getChangedFiles,
+  selectTests,
+  E2E_TOUCHFILES,
+  E2E_TIERS,
+  GLOBAL_TOUCHFILES,
+} from '../test/helpers/touchfiles';
 
 export { PAID_TEST_GLOBS, isPaidTestFile };
 
@@ -65,6 +85,10 @@ export const DEFAULT_TIER: PaidTier = 'gate';
 export const DEFAULT_SHARD_TIMEOUT_MS = 30 * 60_000;
 export const DEFAULT_MAX_FILES_PER_SHARD = 1;
 export const DEFAULT_JOBS = 4;
+// Within one shard's bun process. 4 jobs × 4 ≈ the legacy single-process
+// default of 15, keeping total in-flight `claude` sessions inside known-safe
+// API rate headroom.
+export const DEFAULT_WITHIN_SHARD_CONCURRENCY = 4;
 
 export function collectPaidTestFiles(rootDir = ROOT): string[] {
   const testDir = path.join(rootDir, 'test');
@@ -128,6 +152,159 @@ export function selectPaidTestFiles(files: string[], tier: PaidTier, rootDir = R
   return { selected, excluded };
 }
 
+// --- Parent-side diff selection (shard skipping) ---
+
+/**
+ * The test names the parent mapper recognizes: every E2E map key. LLM-judge
+ * keys are deliberately excluded — skill-llm-eval.test.ts is not a
+ * skill-e2e-* file, so it is always kept (child self-skip authoritative).
+ */
+export const PARENT_MAPPER_TEST_NAMES: string[] = [
+  ...new Set([...Object.keys(E2E_TOUCHFILES), ...Object.keys(E2E_TIERS)]),
+];
+
+/**
+ * Which of `names` appear in `source` as a quoted string ('x', "x", or `x`).
+ * Same class of detection test/e2e-tier-alignment.test.ts uses: exact
+ * quote-delimited match, raw source (comments count — a false hit can only
+ * KEEP a shard, and the registration union below covers constructed names).
+ */
+export function knownTestNamesInSource(source: string, names: Iterable<string>): string[] {
+  const hits: string[] = [];
+  for (const name of names) {
+    if (
+      source.includes(`'${name}'`)
+      || source.includes(`"${name}"`)
+      || source.includes(`\`${name}\``)
+    ) hits.push(name);
+  }
+  return hits;
+}
+
+export interface PaidDiffSelection {
+  /** null = run everything (EVALS_ALL, or no changes vs base). */
+  selectedNames: Set<string> | null;
+  reason: string;
+  totalTests: number;
+}
+
+/**
+ * Compute diff selection in the PARENT, mirroring the module-scope selection
+ * block in test/helpers/e2e-helpers.ts exactly: EVALS_ALL → run all;
+ * base = EVALS_BASE || detectBaseBranch || 'main'; empty changed-file union →
+ * run all. (e2e-helpers additionally gates on EVALS=1, which this runner sets
+ * for every child unconditionally, so the parent mirror omits it.)
+ *
+ * getChangedFiles THROWS on git errors (fail-closed) — the children would hit
+ * the same throw at module load, so the parent surfaces it before any shard
+ * spawns.
+ */
+export function computePaidDiffSelection(
+  env: NodeJS.ProcessEnv = process.env,
+  rootDir = ROOT,
+): PaidDiffSelection {
+  const totalTests = Object.keys(E2E_TOUCHFILES).length;
+  if (env.EVALS_ALL) {
+    return { selectedNames: null, reason: 'run-all (EVALS_ALL=1)', totalTests };
+  }
+  const baseBranch = env.EVALS_BASE || detectBaseBranch(rootDir) || 'main';
+  const changedFiles = getChangedFiles(baseBranch, rootDir);
+  if (changedFiles.length === 0) {
+    return { selectedNames: null, reason: `run-all (no changes vs ${baseBranch})`, totalTests };
+  }
+  const selection = selectTests(changedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES, {
+    baseRef: baseBranch, cwd: rootDir,
+  });
+  return { selectedNames: new Set(selection.selected), reason: selection.reason, totalTests };
+}
+
+export interface ShardSkipDecision {
+  file: string;
+  kept: boolean;
+  reason: string;
+}
+
+export interface DiffSkipOptions {
+  rootDir?: string;
+  /** Injectable for tests. Throwing reads fail OPEN (shard kept). */
+  readSource?: (file: string) => string;
+  /** Injectable name census (default: PARENT_MAPPER_TEST_NAMES). */
+  allNames?: string[];
+  /** Injectable registration map (default: E2E_TOUCHFILES). */
+  e2eTouchfiles?: Record<string, string[]>;
+}
+
+/**
+ * Decide whether a paid test file can be skipped under the current diff
+ * selection. A file's MAPPED names are the union of:
+ *   - E2E map keys quoted in its source, and
+ *   - E2E map keys whose dep list registers the file (the tier-alignment
+ *     mapping) — this covers files whose testNames are constructed rather
+ *     than literal.
+ *
+ * FAIL-OPEN by construction: run-all selection, non-skill-e2e paid files
+ * (llm-judge / codex-e2e / gemini-e2e / routing, keyed off other maps),
+ * unreadable sources, and files with zero mapped names all KEEP their shard —
+ * the child's self-skip stays authoritative. A parent bug may only run
+ * extra work, never drop it.
+ */
+export function diffSkipDecisionForFile(
+  file: string,
+  selectedNames: Set<string> | null,
+  options: DiffSkipOptions = {},
+): ShardSkipDecision {
+  if (selectedNames === null) return { file, kept: true, reason: 'run-all selection' };
+  const rel = normalizeRelativePath(file);
+  if (!/^test\/skill-e2e-.*\.test\.ts$/.test(rel)) {
+    return { file, kept: true, reason: 'non-skill-e2e paid file — child self-skip authoritative' };
+  }
+  let source: string;
+  try {
+    const read = options.readSource
+      ?? ((f: string) => fs.readFileSync(path.join(options.rootDir ?? ROOT, f), 'utf8'));
+    source = read(file);
+  } catch {
+    return { file, kept: true, reason: 'source unreadable — fail-open' };
+  }
+  const allNames = options.allNames ?? PARENT_MAPPER_TEST_NAMES;
+  const touchfiles = options.e2eTouchfiles ?? E2E_TOUCHFILES;
+  const quoted = knownTestNamesInSource(source, allNames);
+  const registered = Object.keys(touchfiles).filter((k) => touchfiles[k].includes(rel));
+  const mapped = [...new Set([...quoted, ...registered])];
+  if (mapped.length === 0) {
+    return { file, kept: true, reason: 'no mappable test names — fail-open, child self-skip authoritative' };
+  }
+  const selectedHere = mapped.filter((n) => selectedNames.has(n));
+  if (selectedHere.length > 0) {
+    const shown = selectedHere.slice(0, 3).join(', ') + (selectedHere.length > 3 ? ', …' : '');
+    return { file, kept: true, reason: `selected: ${shown}` };
+  }
+  return { file, kept: false, reason: `none of its ${mapped.length} mapped test(s) selected` };
+}
+
+/**
+ * Partition planned shards into runnable vs skipped-by-diff. A shard is
+ * skipped only when EVERY file in it is skippable.
+ */
+export function partitionShardsByDiffSelection(
+  shards: string[][],
+  selectedNames: Set<string> | null,
+  options: DiffSkipOptions = {},
+): { runnable: string[][]; skipped: Array<{ files: string[]; reason: string }> } {
+  if (selectedNames === null) return { runnable: shards, skipped: [] };
+  const runnable: string[][] = [];
+  const skipped: Array<{ files: string[]; reason: string }> = [];
+  for (const shard of shards) {
+    const decisions = shard.map((file) => diffSkipDecisionForFile(file, selectedNames, options));
+    if (decisions.every((d) => !d.kept)) {
+      skipped.push({ files: shard, reason: [...new Set(decisions.map((d) => d.reason))].join('; ') });
+    } else {
+      runnable.push(shard);
+    }
+  }
+  return { runnable, skipped };
+}
+
 export function planPaidShards(
   files: string[],
   options: { maxFilesPerShard?: number } = {},
@@ -139,8 +316,15 @@ export function planPaidShards(
   return shards;
 }
 
-export function buildPaidShardArgs(files: string[], timeoutMs: number): string[] {
-  return ['test', ...files, '--retry', '2', `--timeout=${timeoutMs}`];
+export function buildPaidShardArgs(
+  files: string[],
+  timeoutMs: number,
+  maxConcurrency: number = DEFAULT_WITHIN_SHARD_CONCURRENCY,
+): string[] {
+  // Explicit --concurrent/--max-concurrency: the legacy path always set one;
+  // omitting it here made within-shard parallelism differ silently between
+  // the two runners (observed: 1.6x sumdur/wall sharded vs 8x legacy).
+  return ['test', ...files, '--retry', '1', '--concurrent', `--max-concurrency=${maxConcurrency}`, `--timeout=${timeoutMs}`];
 }
 
 /**
@@ -154,7 +338,7 @@ export function shardSlug(files: string[]): string {
     .replace(/[^a-zA-Z0-9._+-]/g, '-');
 }
 
-export type ShardStatus = 'passed' | 'failed' | 'timed-out' | 'never-started';
+export type ShardStatus = 'passed' | 'failed' | 'timed-out' | 'never-started' | 'skipped-by-diff';
 
 export interface ShardOutcome {
   shard: number;
@@ -173,6 +357,8 @@ export interface ShardCommand {
 export interface RunShardsOptions {
   timeoutMs?: number;
   jobs?: number;
+  /** bun --max-concurrency inside each shard (EVALS_CONCURRENCY). */
+  withinShardConcurrency?: number;
   rootDir?: string;
   env?: NodeJS.ProcessEnv;
   /** When set, each shard child gets GSTACK_EVAL_DIR=<evalDirBase>/shards/<slug>/. */
@@ -182,35 +368,6 @@ export interface RunShardsOptions {
   log?: (line: string) => void;
 }
 
-/**
- * SIGKILL the shard's whole process group. Orphaned grandchildren (browsers,
- * claude sessions) are how a stalled run once burned a core for 15.7 hours.
- */
-function killProcessGroup(child: ChildProcess, signal: NodeJS.Signals): void {
-  if (process.platform === 'win32' || typeof child.pid !== 'number') {
-    child.kill(signal);
-    return;
-  }
-  try {
-    process.kill(-child.pid, signal);
-  } catch (err) {
-    const code = (err as NodeJS.ErrnoException).code;
-    if (code === 'ESRCH') return; // group already gone
-    if (code !== 'EPERM') throw err;
-    // Observed on macOS after a SIGKILLed group is reaped: signalling the
-    // now-empty group id returns EPERM, not ESRCH. Throwing here loses the
-    // shard's real outcome (a timeout gets recorded as a failure) and, from
-    // the timeout timer, leaves the shard promise unsettled — a hang, which
-    // is the exact failure class this runner exists to kill. Fall back to the
-    // direct pid so a genuinely-live child is still signalled.
-    try {
-      child.kill(signal);
-    } catch {
-      // Best-effort reap: nothing actionable is left if this fails too.
-    }
-  }
-}
-
 export async function runPaidShard(
   files: string[],
   shardNumber: number,
@@ -228,7 +385,11 @@ export async function runPaidShard(
     ? options.commandFor(files)
     : {
       command: process.execPath,
-      args: buildPaidShardArgs(exactTestFileSelectors(files, rootDir), timeoutMs),
+      args: buildPaidShardArgs(
+        exactTestFileSelectors(files, rootDir),
+        timeoutMs,
+        options.withinShardConcurrency ?? DEFAULT_WITHIN_SHARD_CONCURRENCY,
+      ),
     };
 
   const env = { ...(options.env ?? process.env) };
@@ -270,8 +431,8 @@ export async function runPaidShard(
   let exitCode: number | null = null;
   try {
     const streams: Array<Promise<void>> = [];
-    if (child.stdout) streams.push(forwardAndClassify(child.stdout, sink(process.stdout), classifier));
-    if (child.stderr) streams.push(forwardAndClassify(child.stderr, sink(process.stderr), classifier));
+    if (child.stdout) streams.push(forwardAndClassify(child.stdout, sink(process.stdout), classifier, 'stdout'));
+    if (child.stderr) streams.push(forwardAndClassify(child.stderr, sink(process.stderr), classifier, 'stderr'));
     exitCode = await new Promise<number | null>((resolve, reject) => {
       child.once('error', reject);
       child.once('close', (code) => resolve(code));
@@ -311,6 +472,8 @@ export interface RunSummary {
   failed: number;
   timedOut: number;
   neverStarted: number;
+  /** Shards the parent skipped via diff selection — successes, never conflated with never-started. */
+  skippedByDiff: number;
   outcomes: ShardOutcome[];
 }
 
@@ -318,15 +481,25 @@ export function summarize(outcomes: ShardOutcome[]): RunSummary {
   const count = (status: ShardStatus) => outcomes.filter((o) => o.status === status).length;
   return {
     total: outcomes.length,
-    executed: outcomes.length - count('never-started'),
+    executed: outcomes.length - count('never-started') - count('skipped-by-diff'),
     passed: count('passed'),
     failed: count('failed'),
     timedOut: count('timed-out'),
     neverStarted: count('never-started'),
+    skippedByDiff: count('skipped-by-diff'),
     outcomes,
   };
 }
 
+/**
+ * Exit code for a finished run: skipped-by-diff shards are successes (the
+ * parent proved none of their tests were selected); everything else must
+ * have passed.
+ */
+export function summaryExitCode(summary: RunSummary): number {
+  return summary.passed + summary.skippedByDiff === summary.total ? 0 : 1;
+}
+
 /** Run every shard in its own process. A timeout or failure never aborts the run. */
 export async function runPaidShards(
   shards: string[][],
@@ -345,6 +518,10 @@ export async function runPaidShards(
   let next = 0;
   const worker = async (): Promise<void> => {
     while (true) {
+      // Cancellation (SIGINT/SIGTERM) must stop the RUN: the signal
+      // forwarders kill in-flight children, and this guard stops the pool
+      // from launching replacement shards that would keep burning API spend.
+      if (isTerminationRequested()) return;
       const index = next;
       next += 1;
       if (index >= shards.length) return;
@@ -373,11 +550,12 @@ export function formatSummary(summary: RunSummary): string[] {
     '',
     `[test:paid] ${summary.executed}/${summary.total} shards executed — `
     + `${summary.passed} passed, ${summary.failed} failed, `
-    + `${summary.timedOut} timed out, ${summary.neverStarted} never started`,
+    + `${summary.timedOut} timed out, ${summary.neverStarted} never started, `
+    + `${summary.skippedByDiff} skipped by diff`,
   ];
   for (const outcome of summary.outcomes) {
     lines.push(
-      `  ${outcome.status.padEnd(13)} ${String(Math.round(outcome.elapsedMs / 1000)).padStart(5)}s  `
+      `  ${outcome.status.padEnd(15)} ${String(Math.round(outcome.elapsedMs / 1000)).padStart(5)}s  `
       + outcome.files.join(' '),
     );
   }
@@ -389,6 +567,7 @@ type CliOptions = {
   listOnly: boolean;
   timeoutMs: number;
   jobs: number;
+  withinShardConcurrency: number;
   maxFilesPerShard: number;
 };
 
@@ -417,7 +596,14 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process
     timeoutMs: env.EVALS_SHARD_TIMEOUT_MS
       ? parsePositiveInt(env.EVALS_SHARD_TIMEOUT_MS, 'EVALS_SHARD_TIMEOUT_MS')
       : DEFAULT_SHARD_TIMEOUT_MS,
-    jobs: env.EVALS_CONCURRENCY ? parsePositiveInt(env.EVALS_CONCURRENCY, 'EVALS_CONCURRENCY') : DEFAULT_JOBS,
+    // EVALS_JOBS = shard process count. EVALS_CONCURRENCY deliberately does
+    // NOT set jobs anymore — it's bun's within-shard --max-concurrency (its
+    // legacy meaning). Conflating them turned "EVALS_CONCURRENCY=15" into 15
+    // parallel Bun processes each spawning claude.
+    jobs: env.EVALS_JOBS ? parsePositiveInt(env.EVALS_JOBS, 'EVALS_JOBS') : DEFAULT_JOBS,
+    withinShardConcurrency: env.EVALS_CONCURRENCY
+      ? parsePositiveInt(env.EVALS_CONCURRENCY, 'EVALS_CONCURRENCY')
+      : DEFAULT_WITHIN_SHARD_CONCURRENCY,
     maxFilesPerShard: DEFAULT_MAX_FILES_PER_SHARD,
   };
 
@@ -445,14 +631,30 @@ async function main(): Promise<number> {
 
   const { selected, excluded } = selectPaidTestFiles(discovered, options.tier);
   const shards = planPaidShards(selected, { maxFilesPerShard: options.maxFilesPerShard });
+
+  // Parent-side diff selection (D9): skip whole shards whose mapped tests are
+  // all unselected. Fail-open everywhere — the child's self-skip stays
+  // authoritative for anything the mapper can't attribute.
+  const diffSelection = computePaidDiffSelection(process.env);
+  const { runnable, skipped } = partitionShardsByDiffSelection(shards, diffSelection.selectedNames);
+  const selectedCount = diffSelection.selectedNames
+    ? diffSelection.selectedNames.size
+    : diffSelection.totalTests;
+  console.log(
+    `[test:paid] selection: selected ${selectedCount} of ${diffSelection.totalTests} tests -> `
+    + `running ${runnable.length} of ${shards.length} shards, reason: ${diffSelection.reason}`,
+  );
   console.log(
     `[test:paid] tier=${options.tier}: ${selected.length}/${discovered.length} files, `
     + `${shards.length} shards, jobs=${options.jobs}, timeout=${Math.round(options.timeoutMs / 1000)}s`,
   );
 
   if (options.listOnly) {
+    const skipReasons = new Map(skipped.map((s) => [s.files.join(' '), s.reason]));
     for (let index = 0; index < shards.length; index += 1) {
-      console.log(`  shard ${index + 1}/${shards.length}: ${shards[index].join(' ')}`);
+      const key = shards[index].join(' ');
+      const note = skipReasons.has(key) ? `  [would skip: ${skipReasons.get(key)}]` : '';
+      console.log(`  shard ${index + 1}/${shards.length}: ${key}${note}`);
     }
     if (excluded.length > 0) {
       console.log(`\nExcluded (${excluded.length}):`);
@@ -461,16 +663,33 @@ async function main(): Promise<number> {
     return 0;
   }
 
-  const summary = await runPaidShards(shards, {
+  // One preflight ping in the parent; children skip theirs via the env flag.
+  // Before this, every shard's e2e-helpers module load re-pinged the API —
+  // ~30 paid claude -p calls (30s timeout each) per full run for one bit of
+  // information. A dead API now fails here, before any shard spawns.
+  // Nothing runnable → nothing to ping.
+  if (runnable.length > 0) preflightAnthropicApi(process.env);
+
+  const runSummary = await runPaidShards(runnable, {
     // Tier reaches the children only via EVALS_TIER below; the runtime
     // E2E_TIERS filter inside each child is the real selection mechanism.
     timeoutMs: options.timeoutMs,
     jobs: options.jobs,
-    env: { ...process.env, EVALS: '1', EVALS_TIER: options.tier },
+    withinShardConcurrency: options.withinShardConcurrency,
+    env: { ...process.env, EVALS: '1', EVALS_TIER: options.tier, EVALS_PREFLIGHT_OK: '1' },
     evalDirBase: process.env.GSTACK_EVAL_DIR || getProjectEvalDir(),
   });
+  const skippedOutcomes: ShardOutcome[] = skipped.map((s, index) => ({
+    shard: runnable.length + index + 1,
+    files: s.files,
+    status: 'skipped-by-diff',
+    exitCode: null,
+    elapsedMs: 0,
+    groupPid: null,
+  }));
+  const summary = summarize([...runSummary.outcomes, ...skippedOutcomes]);
   for (const line of formatSummary(summary)) console.log(line);
-  return summary.passed === summary.total ? 0 : 1;
+  return summaryExitCode(summary);
 }
 
 if (import.meta.main) {

scripts/test-strict-output.ts

@@ -51,25 +51,69 @@ const DEFAULT_TERMINATION_TIMER: TerminationTimerApi = {
   cancel: (handle) => clearTimeout(handle as ReturnType<typeof setTimeout>),
 };
 
+/**
+ * Per-source termination bookkeeping, shared across every forwarder bound to
+ * the same source. Installing ANY signal listener suppresses Node's default
+ * terminate-on-SIGINT/SIGTERM, so without this the parent runner survived
+ * cancellation: it killed the current child, then kept LAUNCHING new shards
+ * (observed: paid runs continuing to burn API spend after Ctrl-C). The first
+ * signal now also schedules the parent's own exit after the children's
+ * SIGKILL grace, and runners consult isTerminationRequested() before
+ * launching more work.
+ */
+interface SourceTerminationState {
+  requested: boolean;
+  exitScheduled: boolean;
+}
+const SOURCE_TERMINATION_STATE = new WeakMap<TerminationSignalSource, SourceTerminationState>();
+function terminationStateFor(source: TerminationSignalSource): SourceTerminationState {
+  let state = SOURCE_TERMINATION_STATE.get(source);
+  if (!state) {
+    state = { requested: false, exitScheduled: false };
+    SOURCE_TERMINATION_STATE.set(source, state);
+  }
+  return state;
+}
+export function isTerminationRequested(source: TerminationSignalSource = process): boolean {
+  return SOURCE_TERMINATION_STATE.get(source)?.requested ?? false;
+}
+const signalExitCode = (signal: ForwardedTerminationSignal): number =>
+  128 + (signal === 'SIGINT' ? 2 : 15);
+
 /**
  * Bind one active child to the parent's termination lifecycle. SIGINT and
  * SIGTERM get a grace period so Bun can clean up; a repeated signal, timeout,
  * or synchronous parent exit uses SIGKILL so the child cannot be orphaned.
+ * The parent itself exits shortly after the grace window (or immediately on
+ * a repeated signal) — cancellation must terminate the RUN, not just the
+ * currently-running children.
  */
 export function installChildSignalForwarding(
   child: Pick<ChildProcess, 'kill'>,
   source: TerminationSignalSource = process,
   timer: TerminationTimerApi = DEFAULT_TERMINATION_TIMER,
   graceMs = 5_000,
+  exitImpl: (code: number) => void = (code) => process.exit(code),
 ): ChildSignalForwarding {
   let receivedSignal: ForwardedTerminationSignal | null = null;
   let forceTimer: unknown = null;
   let disposed = false;
 
+  const scheduleParentExit = (signal: ForwardedTerminationSignal, delayMs: number): void => {
+    const state = terminationStateFor(source);
+    state.requested = true;
+    if (state.exitScheduled) return;
+    state.exitScheduled = true;
+    // Never cancelled by dispose(): once cancellation is requested, the run
+    // is going down even if this particular shard finishes cleanly first.
+    timer.schedule(() => exitImpl(signalExitCode(signal)), delayMs);
+  };
+
   const forward = (signal: ForwardedTerminationSignal): void => {
     if (disposed) return;
     if (receivedSignal !== null) {
       child.kill('SIGKILL');
+      scheduleParentExit(signal, 0);
       return;
     }
     receivedSignal = signal;
@@ -78,6 +122,8 @@ export function installChildSignalForwarding(
       forceTimer = null;
       child.kill('SIGKILL');
     }, graceMs);
+    // Exit AFTER the children's SIGKILL grace so the group kills land first.
+    scheduleParentExit(signal, graceMs + 1_000);
   };
   const onSigint = () => forward('SIGINT');
   const onSigterm = () => forward('SIGTERM');
@@ -103,38 +149,91 @@ export function installChildSignalForwarding(
   };
 }
 
+/**
+ * SIGKILL the shard's whole process group. Orphaned grandchildren (browsers,
+ * claude sessions) are how a stalled run once burned a core for 15.7 hours.
+ */
+export function killProcessGroup(child: ChildProcess, signal: NodeJS.Signals): void {
+  if (process.platform === 'win32' || typeof child.pid !== 'number') {
+    child.kill(signal);
+    return;
+  }
+  try {
+    process.kill(-child.pid, signal);
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException).code;
+    if (code === 'ESRCH') return; // group already gone
+    if (code !== 'EPERM') throw err;
+    // Observed on macOS after a SIGKILLed group is reaped: signalling the
+    // now-empty group id returns EPERM, not ESRCH. Throwing here loses the
+    // shard's real outcome (a timeout gets recorded as a failure) and, from
+    // the timeout timer, leaves the shard promise unsettled — a hang, which
+    // is the exact failure class this runner exists to kill. Fall back to the
+    // direct pid so a genuinely-live child is still signalled.
+    try {
+      child.kill(signal);
+    } catch {
+      // Best-effort reap: nothing actionable is left if this fails too.
+    }
+  }
+}
+
+/**
+ * Strip ANSI escapes and a trailing CR from one output line. Every line
+ * matcher (here and in the free runner's console filter / failure
+ * attribution) MUST match against this form — a prior grep for `(fail)`
+ * lines missed real failures because color codes sat inside the line.
+ */
+export function stripAnsiLine(rawLine: string): string {
+  return rawLine.replace(ANSI_ESCAPE, '').replace(/\r$/, '');
+}
+
 export function classifyBunTestOutputLine(rawLine: string): BunTestOutputFinding | null {
-  const line = rawLine.replace(ANSI_ESCAPE, '').replace(/\r$/, '');
+  const line = stripAnsiLine(rawLine);
   if (BUN_FAIL_RESULT.test(line)) return 'failed-test';
   if (line === BUN_BETWEEN_TESTS_ERROR) return 'unhandled-between-tests';
   return null;
 }
 
 export function parseBunTerminalSummaryLine(rawLine: string): number | null {
-  const line = rawLine.replace(ANSI_ESCAPE, '').replace(/\r$/, '');
+  const line = stripAnsiLine(rawLine);
   const match = BUN_TERMINAL_SUMMARY.exec(line);
   return match ? Number.parseInt(match[1], 10) : null;
 }
 
-/** Incrementally classifies output without assuming process chunks align to lines. */
+/**
+ * Incrementally classifies output without assuming process chunks align to
+ * lines. Buffers are PER ORIGIN: stdout and stderr are independent pipes, so
+ * a chunk from one can arrive between two halves of a line from the other.
+ * A single shared buffer would glue those fragments into garbled lines — a
+ * sheared `(fail)` line goes uncounted and a sheared terminal summary reads
+ * as truncation. Counters are shared; only line assembly is per-stream.
+ */
+export type ClassifierOrigin = 'stdout' | 'stderr';
+
 export class BunTestOutputClassifier {
-  private readonly decoder = new StringDecoder('utf8');
-  private pending = '';
+  private readonly decoders: Record<ClassifierOrigin, StringDecoder> = {
+    stdout: new StringDecoder('utf8'),
+    stderr: new StringDecoder('utf8'),
+  };
+  private pending: Record<ClassifierOrigin, string> = { stdout: '', stderr: '' };
   private failedTests = 0;
   private unhandledBetweenTests = 0;
   private terminalFileCounts: number[] = [];
 
-  write(chunk: Uint8Array | string): void {
-    this.pending += typeof chunk === 'string'
+  write(chunk: Uint8Array | string, origin: ClassifierOrigin = 'stdout'): void {
+    this.pending[origin] += typeof chunk === 'string'
       ? chunk
-      : this.decoder.write(Buffer.from(chunk));
-    this.consumeCompleteLines();
+      : this.decoders[origin].write(Buffer.from(chunk));
+    this.consumeCompleteLines(origin);
   }
 
   end(): BunTestOutputSummary {
-    this.pending += this.decoder.end();
-    if (this.pending.length > 0) this.classify(this.pending);
-    this.pending = '';
+    for (const origin of ['stdout', 'stderr'] as const) {
+      this.pending[origin] += this.decoders[origin].end();
+      if (this.pending[origin].length > 0) this.classify(this.pending[origin]);
+      this.pending[origin] = '';
+    }
     return this.summary();
   }
 
@@ -146,12 +245,12 @@ export class BunTestOutputClassifier {
     };
   }
 
-  private consumeCompleteLines(): void {
-    let newline = this.pending.indexOf('\n');
+  private consumeCompleteLines(origin: ClassifierOrigin): void {
+    let newline = this.pending[origin].indexOf('\n');
     while (newline !== -1) {
-      this.classify(this.pending.slice(0, newline));
-      this.pending = this.pending.slice(newline + 1);
-      newline = this.pending.indexOf('\n');
+      this.classify(this.pending[origin].slice(0, newline));
+      this.pending[origin] = this.pending[origin].slice(newline + 1);
+      newline = this.pending[origin].indexOf('\n');
     }
   }
 
@@ -188,10 +287,11 @@ export function forwardAndClassify(
   stream: NodeJS.ReadableStream,
   destination: NodeJS.WriteStream,
   classifier: BunTestOutputClassifier,
+  origin: ClassifierOrigin = 'stdout',
 ): Promise<void> {
   return new Promise((resolve, reject) => {
     stream.on('data', (chunk: Buffer | string) => {
-      classifier.write(chunk);
+      classifier.write(chunk, origin);
       destination.write(chunk);
     });
     stream.on('end', resolve);