test-free-shards: --ci-run --jobs k sub-shards inside one CI job (k=1 default), per-file JUnit counts (GSTA-203 PR 2)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

scripts/lib/shard-jobs.ts

@@ -0,0 +1,381 @@
+/**
+ * Sub-shards inside one CI job: `--ci-run <plan> --shard i --result r --jobs k`
+ * (GSTA-203 PR 2, principle "buy cores, not shards").
+ *
+ * A CI shard's files split into k disjoint, identity-bound sub-shards that run
+ * side by side through the free runner's own process-shard path (one bun child
+ * each, serial inside, own sandbox, group kill, log), never Bun's worker
+ * `--parallel` (decision V3/D6 in scripts/test-free-shards.ts). The rules:
+ *
+ *   - k = 1 (the default) is today's single child, unchanged.
+ *   - Files in TREE_MUTATING or SLOW_UNDER_LOAD run after the parallel
+ *     sub-shards settle, serially, in one trailing sub-shard.
+ *   - The parallel phase uses the concurrent-phase home guard; every child
+ *     uses the shared guard inside it, exactly as full-suite mode does.
+ *   - Every sub-shard gets the CI shard's own wall deadline, so k never moves
+ *     a deadline, and every launched child settles before the parent goes on.
+ *   - The parent merges every sub-shard result before the single retry pass,
+ *     so the five-file retry cap stays per CI shard; a missing, duplicate or
+ *     incomplete sub-shard result fails the shard.
+ *   - Sub-shard runs never write the duration seed (only `--record-durations`
+ *     does, from one-file children).
+ *
+ * Per-file test and expect() counts come from bun's JUnit reporter, which
+ * attributes both per file inside a multi-file child on the pinned Bun 1.4.2
+ * (the console stream is byte-identical with or without it). Counts are
+ * evidence for k > 1 comparisons; they never decide a verdict.
+ */
+import * as os from 'node:os';
+import type { FreeCiPlan, FreeCiResult, FreeShardOutcome } from '../test-free-shards';
+import { guardFreeHome, sharedFreeHome, type FreeHomeGuard, type FreeHomeGuardFactory } from './free-home-guard';
+import { normalizeRelativePath } from './shard-engine';
+
+/**
+ * Files that pass alone but time out as a sub-shard under load. Each entry
+ * names the CI run id where it timed out (FREE_SUBSHARD_TIMEOUT prints it).
+ * They run in the trailing serial sub-shard under `--jobs`; at k = 1 nothing
+ * changes. Keys are pinned against the live file census by
+ * test/shard-jobs.test.ts. Never raise a timeout instead of adding an entry.
+ */
+export const SLOW_UNDER_LOAD: Record<string, { reason: string; runId: string }> = {};
+
+export const FREE_JOBS_INVALID = 'FREE_JOBS_INVALID';
+export const FREE_SUBSHARD_TIMEOUT = 'FREE_SUBSHARD_TIMEOUT';
+export const FREE_SUBSHARD_REJECTED = 'FREE_SUBSHARD_REJECTED';
+const DOCS = 'docs/TESTING_INTERNALS.md#sub-shards-inside-one-ci-job';
+const JOBS_FIX = `--jobs applies only to --ci-run and defaults to 1; GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run. See ${DOCS}`;
+
+/** `--jobs <k>` at parse time: digits only, at least 1. */
+export function parseJobsFlag(raw: string | undefined): number {
+  if (raw === undefined || !/^\d+$/.test(raw) || Number(raw) < 1) {
+    throw new Error(`${FREE_JOBS_INVALID}: --jobs needs a positive whole number; got ${JSON.stringify(raw ?? '')}. ${JOBS_FIX}`);
+  }
+  return Number(raw);
+}
+
+export function rejectJobsWithoutCiRun(): never {
+  throw new Error(`${FREE_JOBS_INVALID}: --jobs needs --ci-run. ${JOBS_FIX}`);
+}
+
+/** `--jobs <k>` against the planned shard: k may not exceed its file count (an empty shard allows 1). */
+export function validateJobsForShard(jobs: number, fileCount: number, shard: number): void {
+  const max = Math.max(1, fileCount);
+  if (jobs > max) {
+    throw new Error(`${FREE_JOBS_INVALID}: --jobs ${jobs} is more than CI shard ${shard}'s ${fileCount} file(s); use 1 to ${max}. ${JOBS_FIX}`);
+  }
+}
+
+export interface FileTestCounts { tests: number; pass: number; fail: number; skip: number; todo: number; expects: number }
+export interface ShardTestCounts {
+  source: 'bun-junit';
+  /** Every reported file was planned and the per-file tests add up to bun's own `Ran N tests` count. */
+  complete: boolean;
+  problem?: string;
+  files: Record<string, FileTestCounts>;
+  totals: FileTestCounts;
+}
+
+const zeroCounts = (): FileTestCounts => ({ tests: 0, pass: 0, fail: 0, skip: 0, todo: 0, expects: 0 });
+const addCounts = (into: FileTestCounts, from: FileTestCounts) => {
+  for (const key of Object.keys(into) as Array<keyof FileTestCounts>) into[key] += from[key];
+};
+const TAG = /<(\/?)(testsuites|testsuite|testcase|failure|error|skipped)\b((?:[^>"']|"[^"]*"|'[^']*')*?)(\/?)>/g;
+const ATTR = /([\w:-]+)="([^"]*)"/g;
+const ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" };
+const decode = (text: string) => text.replace(/&(#x[0-9a-f]+|#\d+|\w+);/gi, (whole, entity: string) => entity[0] === '#'
+  ? String.fromCodePoint(entity[1].toLowerCase() === 'x' ? parseInt(entity.slice(2), 16) : Number(entity.slice(1)))
+  : ENTITIES[entity] ?? whole);
+const attributes = (text: string): Record<string, string> =>
+  Object.fromEntries([...text.matchAll(ATTR)].map(([, key, value]) => [key, decode(value)]));
+
+/**
+ * Per-file counts from bun's JUnit XML. A testcase is a fail when it holds a
+ * failure or error, a todo when skipped with message TODO, a skip when
+ * skipped otherwise, else a pass; expects sum each testcase's assertions.
+ * Printed paths are matched back to planned files (bun prints them relative
+ * to the child's real cwd). Files that never ran a test are absent.
+ */
+export function parseJunitCounts(xml: string | null, planned: readonly string[], testsRan: number | null): ShardTestCounts {
+  const files: Record<string, FileTestCounts> = {};
+  const totals = zeroCounts();
+  const result = (complete: boolean, problem?: string): ShardTestCounts =>
+    ({ source: 'bun-junit', complete, ...(problem ? { problem } : {}), files, totals });
+  if (xml === null) return result(false, 'bun wrote no JUnit report');
+  const plannedSet = new Set(planned);
+  const unknown = new Set<string>();
+  const canonical = (printed: string) => {
+    const stripped = normalizeRelativePath(printed).replace(/^(?:\.{1,2}\/)+/, '');
+    if (plannedSet.has(stripped)) return stripped;
+    return planned.find(file => stripped.endsWith(`/${file}`) || file.endsWith(`/${stripped}`)) ?? stripped;
+  };
+  let current: { file: string; state: 'pass' | 'fail' | 'skip' | 'todo'; expects: number } | null = null;
+  const finish = () => {
+    if (!current) return;
+    const entry = files[current.file] ??= zeroCounts();
+    entry.tests += 1;
+    entry[current.state] += 1;
+    entry.expects += current.expects;
+    if (!plannedSet.has(current.file)) unknown.add(current.file);
+    current = null;
+  };
+  for (const [, closing, name, body, selfClosing] of xml.matchAll(TAG)) {
+    if (name === 'testcase') {
+      if (closing) { finish(); continue; }
+      const attrs = attributes(body);
+      current = { file: canonical(attrs.file ?? ''), state: 'pass', expects: Number(attrs.assertions ?? 0) || 0 };
+      if (selfClosing) finish();
+    } else if (current && !closing && (name === 'failure' || name === 'error')) {
+      current.state = 'fail';
+    } else if (current && !closing && name === 'skipped' && current.state === 'pass') {
+      current.state = attributes(body).message === 'TODO' ? 'todo' : 'skip';
+    }
+  }
+  finish();
+  for (const entry of Object.values(files)) addCounts(totals, entry);
+  if (unknown.size) return result(false, `JUnit named unplanned file(s): ${[...unknown].sort().join(', ')}`);
+  if (testsRan === null) return result(false, "bun's summary has no test count");
+  if (totals.tests !== testsRan) return result(false, `JUnit counted ${totals.tests} tests, bun's summary ${testsRan}`);
+  return result(true);
+}
+
+/** One shard's counts from its sub-shards' counts; complete only when every part is and no file repeats. */
+export function mergeTestCounts(parts: Array<ShardTestCounts | undefined>): ShardTestCounts | undefined {
+  if (parts.every(part => !part)) return undefined;
+  const files: Record<string, FileTestCounts> = {};
+  const totals = zeroCounts();
+  const problems: string[] = [];
+  parts.forEach((part, index) => {
+    if (!part) { problems.push(`sub-shard ${index + 1} has no counts`); return; }
+    if (!part.complete) problems.push(`sub-shard ${index + 1}: ${part.problem ?? 'incomplete'}`);
+    for (const [file, counts] of Object.entries(part.files)) {
+      if (files[file]) problems.push(`${file} counted twice`);
+      files[file] = { ...counts };
+    }
+    addCounts(totals, part.totals);
+  });
+  return { source: 'bun-junit', complete: problems.length === 0, ...(problems.length ? { problem: problems.join('; ') } : {}), files, totals };
+}
+
+export interface SubShardAssignment {
+  /** `<plan id prefix>:<shard>.<index>`: binds a result to one plan, shard and file set. */
+  id: string;
+  index: number;
+  phase: 'parallel' | 'serial';
+  files: string[];
+}
+
+/**
+ * Split one CI shard into disjoint sub-shards: up to k parallel ones packed by
+ * `pack`, then one serial sub-shard for the files `isSerial` names. Throws
+ * unless the assignments cover the shard's files exactly once.
+ */
+export function assignSubShards(
+  planId: string, shard: { shard: number; files: string[] }, jobs: number,
+  isSerial: (file: string) => boolean, pack: (files: string[], count: number) => string[][],
+): SubShardAssignment[] {
+  const serial = shard.files.filter(isSerial);
+  const parallel = shard.files.filter(file => !isSerial(file));
+  const groups = parallel.length ? pack(parallel, Math.min(jobs, parallel.length)).filter(files => files.length) : [];
+  const assignments: SubShardAssignment[] = [
+    ...groups.map(files => ({ phase: 'parallel' as const, files })),
+    ...(serial.length ? [{ phase: 'serial' as const, files: serial }] : []),
+  ].map((entry, index) => ({ id: `${planId.slice(0, 12)}:${shard.shard}.${index + 1}`, index: index + 1, ...entry }));
+  const assigned = assignments.flatMap(entry => entry.files);
+  if (new Set(assigned).size !== assigned.length || assigned.length !== shard.files.length
+    || !shard.files.every(file => assigned.includes(file))) {
+    throw new Error(`${FREE_SUBSHARD_REJECTED}: sub-shard assignment for CI shard ${shard.shard} does not cover its files exactly once`);
+  }
+  return assignments;
+}
+
+export type SubShardSettlement = { assignment: SubShardAssignment } & ({ outcome: FreeShardOutcome } | { error: string });
+
+/** Why one sub-shard's result cannot stand for its assignment; empty when it can. */
+export function subShardProblems(settled: SubShardSettlement): string[] {
+  const { assignment } = settled;
+  if (!('outcome' in settled)) return [`${assignment.id} has no result (${settled.error})`];
+  const { outcome } = settled;
+  const problems: string[] = [];
+  if (JSON.stringify(outcome.files) !== JSON.stringify(assignment.files)) problems.push(`${assignment.id} reported another file set`);
+  const summary = outcome.summary;
+  if (outcome.status === 'passed' && (!summary || summary.filesRan !== assignment.files.length || summary.sawTerminalSummary !== true
+    || outcome.exitCode !== 0 || outcome.failingFiles.length || outcome.unattributedFailures)) {
+    problems.push(`${assignment.id} passed without a complete execution summary`);
+  }
+  return problems;
+}
+
+export interface SubShardRecord {
+  id: string;
+  phase: SubShardAssignment['phase'];
+  files: string[];
+  status: FreeShardOutcome['status'] | 'missing';
+  exitCode: number | null;
+  elapsedMs: number;
+  summary?: FreeShardOutcome['summary'];
+  failingFiles: string[];
+  unattributedFailures: number;
+}
+
+/**
+ * One CI shard outcome from every sub-shard: the planned file order, summed
+ * summaries, the union of failing files, and the worst status. Every rejected
+ * sub-shard adds unattributed evidence, which also vetoes the retry pass.
+ */
+export function mergeSubShardOutcomes(
+  shard: { shard: number; files: string[] }, settled: SubShardSettlement[], elapsedMs: number, extraProblems: string[] = [],
+): { outcome: FreeShardOutcome; records: SubShardRecord[]; problems: string[] } {
+  const problems = [...extraProblems];
+  const assigned = settled.flatMap(entry => entry.assignment.files);
+  if (new Set(assigned).size !== assigned.length) problems.push('a file is assigned to more than one sub-shard');
+  if (assigned.length !== shard.files.length || !shard.files.every(file => assigned.includes(file))) problems.push('sub-shards do not cover the CI shard');
+  for (const entry of settled) problems.push(...subShardProblems(entry));
+  const outcomes = settled.flatMap(entry => ('outcome' in entry ? [entry.outcome] : []));
+  const missing = settled.length - outcomes.length;
+  const status: FreeShardOutcome['status'] = outcomes.some(outcome => outcome.status === 'timed-out') ? 'timed-out'
+    : problems.length || outcomes.some(outcome => outcome.status !== 'passed') ? 'failed' : 'passed';
+  const testsRan = outcomes.map(outcome => outcome.summary?.testsRan ?? null);
+  const failures = outcomes.flatMap(outcome => outcome.failures ?? []);
+  const inFlight = outcomes.flatMap(outcome => outcome.inFlight ?? []);
+  const counts = mergeTestCounts(settled.map(entry => ('outcome' in entry ? entry.outcome.counts : undefined)));
+  const outcome: FreeShardOutcome = {
+    shard: shard.shard,
+    files: shard.files,
+    status,
+    exitCode: status === 'passed' ? 0 : outcomes.find(entry => entry.exitCode)?.exitCode ?? 1,
+    elapsedMs,
+    groupPid: null,
+    failingFiles: status === 'passed' ? [] : [...new Set(outcomes.flatMap(entry => entry.failingFiles))],
+    unattributedFailures: status === 'passed' ? 0 : outcomes.reduce((sum, entry) => sum + entry.unattributedFailures, 0) + problems.length,
+    ...(failures.length ? { failures } : {}),
+    summary: {
+      testsRan: missing || testsRan.some(value => value === null) ? null : testsRan.reduce<number>((sum, value) => sum + value!, 0),
+      filesRan: outcomes.reduce((sum, entry) => sum + (entry.summary?.filesRan ?? 0), 0),
+      sawTerminalSummary: !missing && outcomes.every(entry => entry.files.length === 0 || entry.summary?.sawTerminalSummary === true),
+    },
+    ...(counts ? { counts } : {}),
+    ...(inFlight.length ? { inFlight } : {}),
+  };
+  const records = settled.map((entry): SubShardRecord => {
+    const { id, phase, files } = entry.assignment;
+    if (!('outcome' in entry)) return { id, phase, files, status: 'missing', exitCode: null, elapsedMs: 0, failingFiles: [], unattributedFailures: 1 };
+    const { status: subStatus, exitCode, elapsedMs: subElapsed, summary, failingFiles, unattributedFailures } = entry.outcome;
+    return { id, phase, files, status: subStatus, exitCode, elapsedMs: subElapsed, summary, failingFiles, unattributedFailures };
+  });
+  return { outcome, records, problems };
+}
+
+/** `--ci-verify` re-checks a k > 1 result's sub-shard records; empty means they stand. */
+export function subShardRecordProblems(result: FreeCiResult): string[] {
+  if (result.jobs === undefined && result.subShards === undefined) return [];
+  const records = result.subShards ?? [];
+  if (!Number.isInteger(result.jobs) || result.jobs! < 2 || !records.length) return ['sub-shard result without a valid --jobs and records'];
+  const files = records.flatMap(record => record.files);
+  const problems: string[] = [];
+  if (new Set(files).size !== files.length || files.length !== result.outcome.files.length
+    || !result.outcome.files.every(file => files.includes(file))) problems.push('sub-shard records do not cover the shard exactly once');
+  if (result.outcome.status === 'passed' && records.some(record => record.status !== 'passed' || record.summary?.filesRan !== record.files.length
+    || (record.files.length > 0 && record.summary?.sawTerminalSummary !== true))) problems.push('a passing shard holds a failed or incomplete sub-shard');
+  return problems;
+}
+
+export interface CiRunContext { planPath: string; runId: string; runner: string }
+
+/** Where this run is: the CI run id (with attempt) and runner, else `local` and the host. */
+export function ciRunContext(planPath: string, env: NodeJS.ProcessEnv = process.env): CiRunContext {
+  const runId = env.GITHUB_RUN_ID ? `${env.GITHUB_RUN_ID}${env.GITHUB_RUN_ATTEMPT ? `/attempt ${env.GITHUB_RUN_ATTEMPT}` : ''}` : 'local';
+  const cpus = os.availableParallelism?.() ?? os.cpus().length;
+  return { planPath, runId, runner: `${env.RUNNER_NAME || os.hostname()} (${process.platform}, ${cpus} CPUs)` };
+}
+
+const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
+
+/** The FREE_SUBSHARD_TIMEOUT text: the file, k, runner, a serial reproduce command and the fix. */
+export function subShardTimeoutText(
+  shard: number, jobs: number, record: SubShardRecord, inFlight: string[], wallTimeoutMs: number, context: CiRunContext,
+): string[] {
+  const suspects = inFlight.filter(file => record.files.includes(file));
+  const files = suspects.length ? suspects : record.files;
+  return [
+    `[test:free] ${FREE_SUBSHARD_TIMEOUT}: CI shard ${shard} sub-shard ${record.id} (${record.files.length} files) hit its ${Math.round(wallTimeoutMs / 1000)}s wall at --jobs ${jobs} on ${context.runner}, run ${context.runId}.`,
+    `[test:free]   in flight: ${suspects.length ? suspects.join(', ') : 'unknown (no open file chunk; see the sub-shard log)'}`,
+    `[test:free]   reproduce serially: printf '%s\\n' ${files.map(quote).join(' ')} > files.txt && GSTACK_FREE_JOBS=1 bun run scripts/test-free-shards.ts --files-from files.txt`,
+    `[test:free]   or the whole CI shard at k=1: bun run scripts/test-free-shards.ts --ci-run ${quote(context.planPath)} --shard ${shard} --result shard-${shard}.json --jobs 1`,
+    `[test:free]   if it passes serially, add it to SLOW_UNDER_LOAD in scripts/lib/shard-jobs.ts with run id ${context.runId}; never raise the timeout. See ${DOCS}`,
+  ];
+}
+
+export interface RunShardOptions { wallTimeoutMs: number; verbose: boolean; testCounts: true; homeGuard?: FreeHomeGuardFactory }
+
+export interface CiShardInput {
+  plan: FreeCiPlan;
+  shard: FreeCiPlan['shards'][number];
+  revision: string;
+  /** Sub-shard count; 1 is today's single child. */
+  jobs: number;
+  wallTimeoutMs: number;
+  verbose: boolean;
+  context: CiRunContext;
+}
+
+export interface CiShardDeps {
+  runShard(files: string[], shardNumber: number, totalShards: number, options: RunShardOptions): Promise<FreeShardOutcome>;
+  /** The free runner's single flaky-retry pass over every collected outcome. */
+  retry(outcomes: FreeShardOutcome[]): Promise<{ exitCode: number; retry: FreeShardOutcome | null }>;
+  pack(files: string[], count: number): string[][];
+  isSerial(file: string): boolean;
+  terminationRequested(): boolean;
+  phaseGuard?(files: string[], shards: number): FreeHomeGuard;
+  log?(line: string): void;
+}
+
+/** Run one planned CI shard at k sub-shards and build its result JSON. */
+export async function runCiShard(input: CiShardInput, deps: CiShardDeps): Promise<{ result: FreeCiResult; exitCode: number }> {
+  const { plan, shard, revision, jobs, wallTimeoutMs, verbose } = input;
+  const log = deps.log ?? ((line: string) => console.log(line));
+  if (jobs === 1) {
+    const outcome = await deps.runShard(shard.files, shard.shard, plan.shards.length, { wallTimeoutMs, verbose, testCounts: true });
+    const retried = await deps.retry([outcome]);
+    return { result: { planId: plan.id, revision, outcome, retry: retried.retry }, exitCode: retried.exitCode };
+  }
+  const startedAt = Date.now();
+  const assignments = assignSubShards(plan.id, shard, jobs, deps.isSerial, deps.pack);
+  const total = assignments.length;
+  log(`[test:free] CI shard ${shard.shard}/${plan.shards.length} at --jobs ${jobs}: ${assignments
+    .map(entry => `sub-shard ${entry.id} = runner shard ${entry.index}/${total} (${entry.phase}, ${entry.files.length} files)`).join('; ')}`);
+  const run = (entry: SubShardAssignment, homeGuard?: FreeHomeGuardFactory): Promise<SubShardSettlement> =>
+    deps.runShard(entry.files, entry.index, total, { wallTimeoutMs, verbose, testCounts: true, ...(homeGuard ? { homeGuard } : {}) })
+      .then(outcome => ({ assignment: entry, outcome }), (error: unknown) => ({ assignment: entry, error: error instanceof Error ? error.message : String(error) }));
+  const parallel = assignments.filter(entry => entry.phase === 'parallel');
+  const serial = assignments.filter(entry => entry.phase === 'serial');
+  const phaseProblems: string[] = [];
+  const settled: SubShardSettlement[] = [];
+  if (parallel.length) {
+    const guard = deps.phaseGuard ?? ((files: string[], shards: number) => guardFreeHome(files, process.env, { kind: 'concurrent', shards }));
+    const phaseHome = guard(parallel.flatMap(entry => entry.files), parallel.length);
+    settled.push(...await Promise.all(parallel.map(entry => run(entry, sharedFreeHome))));
+    const homeChange = phaseHome.verify();
+    if (homeChange) phaseProblems.push(homeChange);
+  }
+  for (const entry of serial) {
+    settled.push(deps.terminationRequested() ? { assignment: entry, error: 'cancelled before the serial phase' } : await run(entry));
+  }
+  const merged = mergeSubShardOutcomes(shard, settled, Date.now() - startedAt, phaseProblems);
+  for (const problem of merged.problems) console.error(`[test:free] ${FREE_SUBSHARD_REJECTED}: CI shard ${shard.shard}: ${problem}`);
+  for (const record of merged.records.filter(entry => entry.status === 'timed-out')) {
+    for (const line of subShardTimeoutText(shard.shard, jobs, record, merged.outcome.inFlight ?? [], wallTimeoutMs, input.context)) console.error(line);
+  }
+  log(`[test:free] CI shard ${shard.shard}: ${merged.outcome.status} across ${total} sub-shards in ${Math.round(merged.outcome.elapsedMs / 1000)}s`);
+  const retried = await deps.retry([merged.outcome]);
+  return {
+    result: { planId: plan.id, revision, outcome: merged.outcome, retry: retried.retry, jobs, subShards: merged.records },
+    exitCode: retried.exitCode,
+  };
+}
+
+/** The duration seed takes only one-file children's walls, never a sub-shard's or a shard's. */
+export function assertSoloSeedSample(file: string, outcome: Pick<FreeShardOutcome, 'files'>): void {
+  if (outcome.files.length !== 1 || outcome.files[0] !== file) {
+    throw new Error(`Duration seed sample for ${file} came from a ${outcome.files.length}-file child; seeds are recorded only from one-file serial children.`);
+  }
+}

scripts/test-free-shards.ts

@@ -77,6 +77,11 @@
  *   bun run scripts/test-free-shards.ts --attribute-home           # name real-home writers: each file alone, private HOME
  *   bun run scripts/test-free-shards.ts --ci-plan plan.json --shards 20
  *   bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json
+ *   bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json --jobs 4
+ *       # k sub-shards side by side inside one CI job (default 1 = one child, today's behavior;
+ *       # scripts/lib/shard-jobs.ts). --jobs applies only to --ci-run: a positive whole number up
+ *       # to the shard's file count, else FREE_JOBS_INVALID. GSTACK_FREE_JOBS sets the local
+ *       # full-suite process count and never changes --ci-run; --jobs is its only knob.
  *   bun run scripts/test-free-shards.ts --ci-verify plan.json --results results/
  */
 
@@ -91,6 +96,10 @@ import { resolveStateRoot } from '../lib/state-root';
 import { attributeFreeHomeWriters, guardFreeHome, sharedFreeHome, type FreeHomeGuardFactory } from './lib/free-home-guard';
 import { appendStepSummary, ciHealthSummary, SEED_REFRESH_COMMAND, unseededWarning, windowsCurationLine } from './lib/free-ci-health';
 import { curateWindowsSafe, type CurationResult } from './lib/windows-curation';
+import {
+  assertSoloSeedSample, ciRunContext, parseJobsFlag, parseJunitCounts, rejectJobsWithoutCiRun, runCiShard, SLOW_UNDER_LOAD, subShardRecordProblems,
+  validateJobsForShard, type ShardTestCounts, type SubShardRecord,
+} from './lib/shard-jobs';
 export { curateWindowsSafe, detectWindowsFragility, KNOWN_WINDOWS_INCOMPATIBLE, type CurationResult } from './lib/windows-curation';
 import {
   BunTestOutputClassifier,
@@ -447,6 +456,9 @@ export interface FreeCiResult {
   revision: string;
   outcome: FreeShardOutcome;
   retry: FreeShardOutcome | null;
+  /** Present only at --jobs k > 1: k and one record per sub-shard. */
+  jobs?: number;
+  subShards?: SubShardRecord[];
 }
 
 /** Preserve the full-suite retry cap across independently running CI jobs. */
@@ -472,6 +484,8 @@ export function verifyFreeCiResults(plan: FreeCiPlan, results: FreeCiResult[]):
       || JSON.stringify(outcome.files) !== JSON.stringify(shard.files)) throw new Error('CI result identity, shard or file coverage mismatch');
     seen.add(outcome.shard);
     if (!hasCompleteCiSummary(outcome)) throw new Error('Missing or incomplete CI execution summary');
+    const subShardProblems = subShardRecordProblems(result);
+    if (subShardProblems.length) throw new Error(`CI shard ${outcome.shard} sub-shards rejected: ${subShardProblems.join('; ')}`);
     if (outcome.status === 'passed') {
       if (outcome.exitCode !== 0 || outcome.failingFiles.length || outcome.unattributedFailures || result.retry) throw new Error('Inconsistent passing CI result');
     } else {
@@ -542,6 +556,8 @@ type CliOptions = {
   result: string | null;
   results: string | null;
   filesFrom: string | null;
+  /** Set only by --jobs (with --ci-run); absent means k = 1. */
+  jobs?: number;
 };
 
 export function parseCliOptions(argv: string[]): CliOptions {
@@ -556,6 +572,7 @@ export function parseCliOptions(argv: string[]): CliOptions {
   let wallTimeoutExplicit = false;
   let quick = false;
   let attributeHome = false;
+  let jobs: number | undefined;
   const paths: Record<'ciPlan' | 'ciRun' | 'ciVerify' | 'result' | 'results' | 'filesFrom', string | null> = {
     ciPlan: null, ciRun: null, ciVerify: null, result: null, results: null, filesFrom: null,
   };
@@ -579,6 +596,7 @@ export function parseCliOptions(argv: string[]): CliOptions {
     '--result': pathFlag('--result', 'result'),
     '--results': pathFlag('--results', 'results'),
     '--files-from': pathFlag('--files-from', 'filesFrom'),
+    '--jobs': (next) => { jobs = parseJobsFlag(next()); },
     '--shards': (next) => {
       const value = next();
       if (!value) throw new Error('Missing value for --shards');
@@ -602,8 +620,9 @@ export function parseCliOptions(argv: string[]): CliOptions {
   if (ciModes > 1 || (ciModes && (quick || listOnly || dryRun || recordDurations || attributeHome))) throw new Error('CI modes cannot be combined with other selection modes');
   if (paths.ciRun && (shardIndex === null || !paths.result)) throw new Error('--ci-run requires --shard and --result');
   if (paths.ciVerify && !paths.results) throw new Error('--ci-verify requires --results');
+  if (jobs !== undefined && !paths.ciRun) rejectJobsWithoutCiRun();
   if (quick && (recordDurations || windowsOnly || shardIndex !== null)) throw new Error('--quick cannot change recording, Windows or shard selection');
-  return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths };
+  return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths, ...(jobs === undefined ? {} : { jobs }) };
 }
 
 const describeLines = (lines: string[]) => `${lines.slice(0, 5).join(', ')}${lines.length > 5 ? ` (+${lines.length - 5} more)` : ''}`;
@@ -1075,6 +1094,10 @@ export interface FreeShardOutcome {
   unattributedFailures: number;
   /** Named failures with their excerpts, so a flaky pass records what failed the first time. */
   failures?: FreeRunFailure[];
+  /** Per-file test and expect() counts from bun's JUnit report (--ci-run only). */
+  counts?: ShardTestCounts;
+  /** Wedge suspects at a wall-timeout kill (FreeRunReport.inFlight). */
+  inFlight?: string[];
 }
 
 export interface ShardCommand {
@@ -1087,8 +1110,10 @@ export interface RunFreeShardOptions {
   wallTimeoutMs?: number;
   rootDir?: string;
   env?: NodeJS.ProcessEnv;
-  /** Override the spawned command. Tests inject fake pass/fail/slow commands. */
-  commandFor?: (files: string[]) => ShardCommand;
+  /** Override the spawned command. Tests inject fake pass/fail/slow commands; `junitPath` is set with testCounts. */
+  commandFor?: (files: string[], junitPath?: string) => ShardCommand;
+  /** Ask bun for a JUnit report in the shard sandbox and record per-file counts on the outcome (--ci-run). */
+  testCounts?: boolean;
   /** Suppress ALL child output from the console (tests). The classifier and the log file still see every byte. */
   quiet?: boolean;
   /** Forward the full child stream to the console (legacy firehose). Default: the quiet filtered console. */
@@ -1633,12 +1658,14 @@ export async function runFreeShard(
   const shardLog = openShardLog(logPath, label, 0o600);
   log(`[test:free] full log: ${logPath}`);
 
-  const { command, args } = options.commandFor
-    ? options.commandFor(files)
-    : { command: process.execPath, args: buildShardArgs(files, { rootDir }) };
-
   // realpath: the browser tracker refuses a state dir whose path resolves elsewhere.
   const { stateDir, env } = createShardSandbox('gstack-free-shard-', options.env ?? process.env, { realpath: true });
+  // The JUnit reporter leaves the console stream byte-identical (Bun 1.4.2); it only adds per-file counts.
+  const junitPath = options.testCounts ? path.join(stateDir, 'junit.xml') : undefined;
+  const { command, args } = options.commandFor
+    ? options.commandFor(files, junitPath)
+    : { command: process.execPath, args: [...buildShardArgs(files, { rootDir }), ...(junitPath ? ['--reporter=junit', `--reporter-outfile=${junitPath}`] : [])] };
+  let junitXml = null as string | null;
   // CLI renders otherwise share the repo's .gstack/browse.json, where concurrent
   // shards and prior daemons replace each other's state; override inherited state.
   env.BROWSE_STATE_FILE = path.join(stateDir, '.gstack', 'browse.json');
@@ -1703,6 +1730,9 @@ export async function runFreeShard(
       shardLog.write(diagnostic + '\n');
     }
     await new Promise<void>((resolve) => shardLog.stream.end(() => resolve()));
+    if (junitPath) {
+      try { junitXml = fs.readFileSync(junitPath, 'utf8'); } catch { /* counts record the missing report */ }
+    }
     try {
       if (!cleanupError) fs.rmSync(stateDir, { recursive: true, force: true });
     } catch {
@@ -1745,6 +1775,8 @@ export async function runFreeShard(
     shard: shardNumber, files, status, exitCode, elapsedMs: Date.now() - startedAt, groupPid, failingFiles, unattributedFailures,
     ...(status === 'passed' ? {} : { failures: report.failures }),
     summary: { testsRan: report.testsRan, filesRan: report.filesRan, sawTerminalSummary: report.sawTerminalSummary },
+    ...(junitPath ? { counts: parseJunitCounts(junitXml, files, report.testsRan) } : {}),
+    ...(status === 'timed-out' && report.inFlight.length ? { inFlight: report.inFlight } : {}),
   };
   log(shardEpilogue(outcome, totalShards));
   for (const line of buildRunEpilogue(status, report, outcome.elapsedMs, logPath)) log(line);
@@ -1799,6 +1831,7 @@ async function recordFreeTestDurations(files: string[], jobs: number, wallTimeou
     const outcome = await runFreeShard([file], index + 1, files.length, {
       wallTimeoutMs, quiet: true, homeGuard: sharedFreeHome,
     });
+    assertSoloSeedSample(file, outcome);
     durations[normalizeRelativePath(file)] = outcome.elapsedMs;
     if (outcome.status !== 'passed') failed.push(file);
   });
@@ -1944,14 +1977,22 @@ async function main(): Promise<number> {
     }
     const shard = plan.shards[options.shardIndex! - 1];
     if (!shard || shard.shard !== options.shardIndex) throw new Error('CI shard index is outside the plan');
-    const outcome = await runFreeShard(shard.files, shard.shard, plan.shards.length, {
+    const jobs = options.jobs ?? 1;
+    validateJobsForShard(jobs, shard.files.length, shard.shard);
+    const durations = loadFreeTestDurations() ?? {};
+    const { result, exitCode } = await runCiShard({
+      plan, shard, revision, jobs, verbose: options.verbose, context: ciRunContext(options.ciRun!),
       wallTimeoutMs: options.wallTimeoutExplicit ? options.wallTimeoutMs
         : wallTimeoutForPackedShard(shard.predictedMs, options.wallTimeoutMs, shard.files.length),
-      verbose: options.verbose,
+    }, {
+      runShard: (files, shardNumber, totalShards, runOptions) => runFreeShard(files, shardNumber, totalShards, runOptions),
+      retry: outcomes => retryFailedFreeFiles(outcomes, plan.shards.length, options),
+      pack: (files, count) => packShardsByDuration(files, count, durations).shards,
+      isSerial: file => file in TREE_MUTATING || file in SLOW_UNDER_LOAD,
+      terminationRequested: isTerminationRequested,
     });
-    const retried = await retryFailedFreeFiles([outcome], plan.shards.length, options);
-    writeJson(options.result!, { planId: plan.id, revision, outcome, retry: retried.retry } satisfies FreeCiResult);
-    return retried.exitCode;
+    writeJson(options.result!, result);
+    return exitCode;
   }
 
   let files = allFiles;

scripts/test-pr-profile.ts

@@ -237,6 +237,7 @@ export const PAID_WORKFLOW_FILES = [
 export const FREE_ONLY_PR_FILES = [
   'scripts/test-free-shards.ts',
   'scripts/lib/free-home-guard.ts', // Imported only by the free shard runner.
+  'scripts/lib/shard-jobs.ts', // Imported only by the free shard runner (--ci-run --jobs sub-shards).
   'test/helpers/auq-parallel-worker.ts',
   'test/helpers/stored-zip.ts', // Read only by the free test-health report tests.
   'scripts/free-test-durations.json',