gstack/scripts/lib/shard-jobs.ts

/**
 * Sub-shards inside one CI job: `--ci-run <plan> --shard i --result r --jobs k`
 * (GSTA-203 PR 2, principle "buy cores, not shards").
 *
 * A CI shard's files split into k disjoint, identity-bound sub-shards that run
 * side by side through the free runner's own process-shard path (one bun child
 * each, serial inside, own sandbox, group kill, log), never Bun's worker
 * `--parallel` (decision V3/D6 in scripts/test-free-shards.ts). The rules:
 *
 *   - k = 1 (the default) is today's single child, unchanged.
 *   - Files in TREE_MUTATING or SLOW_UNDER_LOAD run after the parallel
 *     sub-shards settle, serially, in one trailing sub-shard.
 *   - The parallel phase uses the concurrent-phase home guard; every child
 *     uses the shared guard inside it, exactly as full-suite mode does.
 *   - The whole CI shard keeps its k = 1 wall: the parallel sub-shards get it,
 *     and the serial sub-shard gets what is left of it, so k never extends a
 *     deadline; every launched child settles before the parent goes on.
 *   - The parent merges every sub-shard result before the single retry pass,
 *     so the five-file retry cap stays per CI shard; a missing, duplicate or
 *     incomplete sub-shard result fails the shard.
 *   - Sub-shard runs never write the duration seed (only `--record-durations`
 *     does, from one-file children).
 *
 * Per-file test and expect() counts come from bun's JUnit reporter, which
 * attributes both per file inside a multi-file child on the pinned Bun 1.4.2
 * (the console stream is byte-identical with or without it). Counts are
 * evidence for k > 1 comparisons; they never decide a verdict.
 */
import * as fs from 'node:fs';
import * as os from 'node:os';
import type { FreeCiPlan, FreeCiResult, FreeShardOutcome } from '../test-free-shards';
import { guardFreeHome, sharedFreeHome, type FreeHomeGuard, type FreeHomeGuardFactory } from './free-home-guard';
import { normalizeRelativePath } from './shard-engine';

/**
 * Files that pass alone but time out as a sub-shard under load. Each entry
 * names the CI run id where it timed out (FREE_SUBSHARD_TIMEOUT prints it).
 * They run in the trailing serial sub-shard under `--jobs`; at k = 1 nothing
 * changes. Keys are pinned against the live file census by
 * test/shard-jobs.test.ts. Never raise a timeout instead of adding an entry.
 */
export const SLOW_UNDER_LOAD: Record<string, { reason: string; runId: string }> = {};

export const FREE_JOBS_INVALID = 'FREE_JOBS_INVALID';
export const FREE_SUBSHARD_TIMEOUT = 'FREE_SUBSHARD_TIMEOUT';
export const FREE_SUBSHARD_REJECTED = 'FREE_SUBSHARD_REJECTED';
const DOCS = 'docs/TESTING_INTERNALS.md#sub-shards-inside-one-ci-job';
const JOBS_FIX = `--jobs applies only to --ci-run and defaults to the plan's per-shard k (1 when the plan has none); --shard-jobs applies only to --ci-plan; GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run. See ${DOCS}`;

/** `--jobs <k>` or `--shard-jobs <k>` at parse time: digits only, at least 1. */
export function parseJobsFlag(raw: string | undefined, flag = '--jobs'): number {
  if (raw === undefined || !/^\d+$/.test(raw) || Number(raw) < 1) {
    throw new Error(`${FREE_JOBS_INVALID}: ${flag} needs a positive whole number; got ${JSON.stringify(raw ?? '')}. ${JOBS_FIX}`);
  }
  return Number(raw);
}

export function rejectShardJobsWithoutCiPlan(): never {
  throw new Error(`${FREE_JOBS_INVALID}: --shard-jobs needs --ci-plan. ${JOBS_FIX}`);
}

/**
 * One planned shard's k under `--ci-plan --shard-jobs k`: at most one parallel
 * sub-shard per file that may run in parallel, so a shard of only serial files
 * (the exclusive TREE_MUTATING shard) keeps k = 1. Undefined means k = 1, and
 * the plan then carries no `jobs` key, so its id matches a plan without k.
 */
export function plannedShardJobs(files: readonly string[], shardJobs: number, isSerial: (file: string) => boolean): number | undefined {
  const jobs = Math.min(shardJobs, files.filter(file => !isSerial(file)).length);
  return jobs > 1 ? jobs : undefined;
}

/** Why a plan's per-shard k cannot stand; null when absent or valid. */
export function plannedShardJobsProblem(shard: { shard: number; files: readonly string[]; jobs?: unknown }, isSerial: (file: string) => boolean): string | null {
  if (shard.jobs === undefined) return null;
  const parallel = shard.files.filter(file => !isSerial(file)).length;
  return Number.isInteger(shard.jobs) && (shard.jobs as number) >= 2 && (shard.jobs as number) <= parallel ? null
    : `CI shard ${shard.shard} plans --jobs ${JSON.stringify(shard.jobs)} for ${parallel} parallel file(s)`;
}

export function rejectJobsWithoutCiRun(): never {
  throw new Error(`${FREE_JOBS_INVALID}: --jobs needs --ci-run. ${JOBS_FIX}`);
}

/** `--jobs <k>` against the planned shard: k may not exceed its file count (an empty shard allows 1). */
export function validateJobsForShard(jobs: number, fileCount: number, shard: number): void {
  const max = Math.max(1, fileCount);
  if (jobs > max) {
    throw new Error(`${FREE_JOBS_INVALID}: --jobs ${jobs} is more than CI shard ${shard}'s ${fileCount} file(s); use 1 to ${max}. ${JOBS_FIX}`);
  }
}

export interface FileTestCounts { tests: number; pass: number; fail: number; skip: number; todo: number; expects: number }
export interface ShardTestCounts {
  source: 'bun-junit';
  /** Every reported file was planned and the per-file tests add up to bun's own `Ran N tests` count. */
  complete: boolean;
  problem?: string;
  files: Record<string, FileTestCounts>;
  totals: FileTestCounts;
}

const zeroCounts = (): FileTestCounts => ({ tests: 0, pass: 0, fail: 0, skip: 0, todo: 0, expects: 0 });
const addCounts = (into: FileTestCounts, from: FileTestCounts) => {
  for (const key of Object.keys(into) as Array<keyof FileTestCounts>) into[key] += from[key];
};
const TAG = /<(\/?)(testsuites|testsuite|testcase|failure|error|skipped)\b((?:[^>"']|"[^"]*"|'[^']*')*?)(\/?)>/g;
const ATTR = /([\w:-]+)="([^"]*)"/g;
const ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" };
const decode = (text: string) => text.replace(/&(#x[0-9a-f]+|#\d+|\w+);/gi, (whole, entity: string) => entity[0] === '#'
  ? String.fromCodePoint(entity[1].toLowerCase() === 'x' ? parseInt(entity.slice(2), 16) : Number(entity.slice(1)))
  : ENTITIES[entity] ?? whole);
const attributes = (text: string): Record<string, string> =>
  Object.fromEntries([...text.matchAll(ATTR)].map(([, key, value]) => [key, decode(value)]));

/**
 * Per-file counts from bun's JUnit XML. A testcase is a fail when it holds a
 * failure or error, a todo when skipped with message TODO, a skip when
 * skipped otherwise, else a pass; expects sum each testcase's assertions.
 * Printed paths are matched back to planned files. Bun prints them relative
 * to the child's real cwd, so a plain relative path must match exactly; an
 * absolute or ../-prefixed path matches only when what precedes the planned
 * file is the child's root. Files that never ran a test are absent.
 */
export function parseJunitCounts(xml: string | null, planned: readonly string[], testsRan: number | null, root = process.cwd()): ShardTestCounts {
  const files: Record<string, FileTestCounts> = {};
  const totals = zeroCounts();
  const result = (complete: boolean, problem?: string): ShardTestCounts =>
    ({ source: 'bun-junit', complete, ...(problem ? { problem } : {}), files, totals });
  if (xml === null) return result(false, 'bun wrote no JUnit report');
  const plannedSet = new Set(planned);
  const unknown = new Set<string>();
  const roots = new Set([root, (() => { try { return fs.realpathSync(root); } catch { return root; } })()].map(normalizeRelativePath));
  const isRoot = (prefix: string) => [...roots].some(dir => dir === prefix || dir.endsWith(`/${prefix}`));
  const canonical = (printed: string) => {
    const normalized = normalizeRelativePath(printed);
    const stripped = normalized.replace(/^(?:\.{1,2}\/)+/, '');
    if (plannedSet.has(stripped)) return stripped;
    if (stripped === normalized && !/^(?:\/|[A-Za-z]:\/)/.test(stripped)) return stripped;
    const matches = planned.filter(file => stripped.endsWith(`/${file}`) && isRoot(stripped.slice(0, -file.length - 1)));
    return matches.length === 1 ? matches[0] : stripped;
  };
  let current: { file: string; state: 'pass' | 'fail' | 'skip' | 'todo'; expects: number } | null = null;
  const finish = () => {
    if (!current) return;
    const entry = files[current.file] ??= zeroCounts();
    entry.tests += 1;
    entry[current.state] += 1;
    entry.expects += current.expects;
    if (!plannedSet.has(current.file)) unknown.add(current.file);
    current = null;
  };
  for (const [, closing, name, body, selfClosing] of xml.matchAll(TAG)) {
    if (name === 'testcase') {
      if (closing) { finish(); continue; }
      const attrs = attributes(body);
      current = { file: canonical(attrs.file ?? ''), state: 'pass', expects: Number(attrs.assertions ?? 0) || 0 };
      if (selfClosing) finish();
    } else if (current && !closing && (name === 'failure' || name === 'error')) {
      current.state = 'fail';
    } else if (current && !closing && name === 'skipped' && current.state === 'pass') {
      current.state = attributes(body).message === 'TODO' ? 'todo' : 'skip';
    }
  }
  finish();
  for (const entry of Object.values(files)) addCounts(totals, entry);
  if (unknown.size) return result(false, `JUnit named unplanned file(s): ${[...unknown].sort().join(', ')}`);
  if (testsRan === null) return result(false, "bun's summary has no test count");
  if (totals.tests !== testsRan) return result(false, `JUnit counted ${totals.tests} tests, bun's summary ${testsRan}`);
  return result(true);
}

const JUNIT_MAX_BYTES = 64 * 1024 * 1024;

/** The child-written report, only when it is a regular file of bounded size (never a link, FIFO or device). */
export function readJunitReport(file: string): string | null {
  let fd: number;
  try { fd = fs.openSync(file, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0) | (fs.constants.O_NONBLOCK ?? 0)); } catch { return null; }
  try {
    const stat = fs.fstatSync(fd);
    return stat.isFile() && stat.size <= JUNIT_MAX_BYTES ? fs.readFileSync(fd, 'utf8') : null;
  } catch { return null; } finally { fs.closeSync(fd); }
}

/** One shard's counts from its sub-shards' counts; complete only when every part is and no file repeats. */
export function mergeTestCounts(parts: Array<ShardTestCounts | undefined>): ShardTestCounts | undefined {
  if (parts.every(part => !part)) return undefined;
  const files: Record<string, FileTestCounts> = {};
  const totals = zeroCounts();
  const problems: string[] = [];
  parts.forEach((part, index) => {
    if (!part) { problems.push(`sub-shard ${index + 1} has no counts`); return; }
    if (!part.complete) problems.push(`sub-shard ${index + 1}: ${part.problem ?? 'incomplete'}`);
    for (const [file, counts] of Object.entries(part.files)) {
      if (files[file]) problems.push(`${file} counted twice`);
      files[file] = { ...counts };
    }
    addCounts(totals, part.totals);
  });
  return { source: 'bun-junit', complete: problems.length === 0, ...(problems.length ? { problem: problems.join('; ') } : {}), files, totals };
}

export interface SubShardAssignment {
  /** `<plan id prefix>:<shard>.<index>`: binds a result to one plan, shard and file set. */
  id: string;
  index: number;
  phase: 'parallel' | 'serial';
  files: string[];
}

/**
 * Split one CI shard into disjoint sub-shards: up to k parallel ones packed by
 * `pack`, then one serial sub-shard for the files `isSerial` names. Throws
 * unless the assignments cover the shard's files exactly once.
 */
export function assignSubShards(
  planId: string, shard: { shard: number; files: string[] }, jobs: number,
  isSerial: (file: string) => boolean, pack: (files: string[], count: number) => string[][],
): SubShardAssignment[] {
  const serial = shard.files.filter(isSerial);
  const parallel = shard.files.filter(file => !isSerial(file));
  const groups = parallel.length ? pack(parallel, Math.min(jobs, parallel.length)).filter(files => files.length) : [];
  const assignments: SubShardAssignment[] = [
    ...groups.map(files => ({ phase: 'parallel' as const, files })),
    ...(serial.length ? [{ phase: 'serial' as const, files: serial }] : []),
  ].map((entry, index) => ({ id: `${planId.slice(0, 12)}:${shard.shard}.${index + 1}`, index: index + 1, ...entry }));
  const assigned = assignments.flatMap(entry => entry.files);
  if (new Set(assigned).size !== assigned.length || assigned.length !== shard.files.length
    || !shard.files.every(file => assigned.includes(file))) {
    throw new Error(`${FREE_SUBSHARD_REJECTED}: sub-shard assignment for CI shard ${shard.shard} does not cover its files exactly once`);
  }
  return assignments;
}

export type SubShardSettlement = { assignment: SubShardAssignment } & ({ outcome: FreeShardOutcome } | { error: string });

/** Why one sub-shard's result cannot stand for its assignment; empty when it can. */
export function subShardProblems(settled: SubShardSettlement): string[] {
  const { assignment } = settled;
  if (!('outcome' in settled)) return [`${assignment.id} has no result (${settled.error})`];
  const { outcome } = settled;
  const problems: string[] = [];
  if (JSON.stringify(outcome.files) !== JSON.stringify(assignment.files)) problems.push(`${assignment.id} reported another file set`);
  const summary = outcome.summary;
  if (!summary || (summary.sawTerminalSummary && summary.filesRan !== assignment.files.length)) {
    problems.push(`${assignment.id} reported ${summary?.filesRan ?? 'no'} executed file(s) for ${assignment.files.length} assigned`);
  } else if (outcome.status === 'passed' && (summary.filesRan !== assignment.files.length || summary.sawTerminalSummary !== true
    || outcome.exitCode !== 0 || outcome.failingFiles.length || outcome.unattributedFailures)) {
    problems.push(`${assignment.id} passed without a complete execution summary`);
  }
  return problems;
}

export interface SubShardRecord {
  id: string;
  phase: SubShardAssignment['phase'];
  files: string[];
  status: FreeShardOutcome['status'] | 'missing';
  exitCode: number | null;
  elapsedMs: number;
  summary?: FreeShardOutcome['summary'];
  failingFiles: string[];
  unattributedFailures: number;
}

/**
 * One CI shard outcome from every sub-shard: the planned file order, summed
 * summaries, the union of failing files, and the worst status. Every rejected
 * sub-shard adds unattributed evidence, which also vetoes the retry pass.
 */
export function mergeSubShardOutcomes(
  shard: { shard: number; files: string[] }, settled: SubShardSettlement[], elapsedMs: number, extraProblems: string[] = [],
): { outcome: FreeShardOutcome; records: SubShardRecord[]; problems: string[] } {
  const problems = [...extraProblems];
  const assigned = settled.flatMap(entry => entry.assignment.files);
  if (new Set(assigned).size !== assigned.length) problems.push('a file is assigned to more than one sub-shard');
  if (assigned.length !== shard.files.length || !shard.files.every(file => assigned.includes(file))) problems.push('sub-shards do not cover the CI shard');
  for (const entry of settled) problems.push(...subShardProblems(entry));
  const outcomes = settled.flatMap(entry => ('outcome' in entry ? [entry.outcome] : []));
  const missing = settled.length - outcomes.length;
  const status: FreeShardOutcome['status'] = outcomes.some(outcome => outcome.status === 'timed-out') ? 'timed-out'
    : problems.length || outcomes.some(outcome => outcome.status !== 'passed') ? 'failed' : 'passed';
  const testsRan = outcomes.map(outcome => outcome.summary?.testsRan ?? null);
  const failures = outcomes.flatMap(outcome => outcome.failures ?? []);
  const inFlight = outcomes.flatMap(outcome => outcome.inFlight ?? []);
  const counts = mergeTestCounts(settled.map(entry => ('outcome' in entry ? entry.outcome.counts : undefined)));
  const outcome: FreeShardOutcome = {
    shard: shard.shard,
    files: shard.files,
    status,
    exitCode: status === 'passed' ? 0 : outcomes.find(entry => entry.exitCode)?.exitCode ?? 1,
    elapsedMs,
    groupPid: null,
    failingFiles: status === 'passed' ? [] : [...new Set(outcomes.flatMap(entry => entry.failingFiles))],
    unattributedFailures: status === 'passed' ? 0 : outcomes.reduce((sum, entry) => sum + entry.unattributedFailures, 0) + problems.length,
    ...(failures.length ? { failures } : {}),
    summary: {
      testsRan: missing || testsRan.some(value => value === null) ? null : testsRan.reduce<number>((sum, value) => sum + value!, 0),
      filesRan: outcomes.reduce((sum, entry) => sum + (entry.summary?.filesRan ?? 0), 0),
      sawTerminalSummary: !missing && outcomes.every(entry => entry.files.length === 0 || entry.summary?.sawTerminalSummary === true),
    },
    ...(counts ? { counts } : {}),
    ...(inFlight.length ? { inFlight } : {}),
  };
  const records = settled.map((entry): SubShardRecord => {
    const { id, phase, files } = entry.assignment;
    if (!('outcome' in entry)) return { id, phase, files, status: 'missing', exitCode: null, elapsedMs: 0, failingFiles: [], unattributedFailures: 1 };
    const { status: subStatus, exitCode, elapsedMs: subElapsed, summary, failingFiles, unattributedFailures } = entry.outcome;
    return { id, phase, files, status: subStatus, exitCode, elapsedMs: subElapsed, summary, failingFiles, unattributedFailures };
  });
  return { outcome, records, problems };
}

/** `--ci-verify` re-checks a k > 1 result's sub-shard records; empty means they stand. */
export function subShardRecordProblems(result: FreeCiResult): string[] {
  if (result.jobs === undefined && result.subShards === undefined) return [];
  const records = result.subShards ?? [];
  if (!Number.isInteger(result.jobs) || result.jobs! < 2 || !records.length) return ['sub-shard result without a valid --jobs and records'];
  const files = records.flatMap(record => record.files);
  const problems: string[] = [];
  if (new Set(files).size !== files.length || files.length !== result.outcome.files.length
    || !result.outcome.files.every(file => files.includes(file))) problems.push('sub-shard records do not cover the shard exactly once');
  const prefix = `${result.planId.slice(0, 12)}:${result.outcome.shard}.`;
  if (records.some((record, index) => record.id !== `${prefix}${index + 1}`)
    || records.filter(record => record.phase === 'parallel').length > result.jobs!
    || records.some((record, index) => record.phase === 'serial' ? index !== records.length - 1 : record.phase !== 'parallel')) {
    problems.push('sub-shard ids or phases do not match the plan and --jobs');
  }
  if (records.some(record => record.status === 'missing' || record.summary?.filesRan !== record.files.length
    || (record.files.length > 0 && record.summary?.sawTerminalSummary !== true))) problems.push('a sub-shard did not run every assigned file');
  if (result.outcome.status === 'passed' && records.some(record => record.status !== 'passed')) problems.push('a passing shard holds a failed sub-shard');
  return problems;
}

export interface CiRunContext { planPath: string; runId: string; runner: string }

/** Where this run is: the CI run id (with attempt) and runner, else `local` and the host. */
export function ciRunContext(planPath: string, env: NodeJS.ProcessEnv = process.env): CiRunContext {
  const runId = env.GITHUB_RUN_ID ? `${env.GITHUB_RUN_ID}${env.GITHUB_RUN_ATTEMPT ? `/attempt ${env.GITHUB_RUN_ATTEMPT}` : ''}` : 'local';
  const cpus = os.availableParallelism?.() ?? os.cpus().length;
  return { planPath, runId, runner: `${env.RUNNER_NAME || os.hostname()} (${process.platform}, ${cpus} CPUs)` };
}

const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;

/** The FREE_SUBSHARD_TIMEOUT text: the file, k, runner, a serial reproduce command and the fix. */
export function subShardTimeoutText(
  shard: number, jobs: number, record: SubShardRecord, inFlight: string[], wallTimeoutMs: number, context: CiRunContext, shardWallMs = wallTimeoutMs,
): string[] {
  const suspects = inFlight.filter(file => record.files.includes(file));
  const files = suspects.length ? suspects : record.files;
  const seconds = (ms: number) => Math.round(ms / 1000);
  const fix = record.phase !== 'serial'
    ? `if it passes serially, add it to SLOW_UNDER_LOAD in scripts/lib/shard-jobs.ts with run id ${context.runId}; never raise the timeout`
    : wallTimeoutMs < shardWallMs
      ? `the parallel sub-shards left this serial sub-shard ${seconds(wallTimeoutMs)}s of the ${seconds(shardWallMs)}s shard wall; if the k=1 command passes, the parallel phase used up the budget and this shard needs a lower --jobs; never raise the timeout`
      : 'this file already runs alone after the parallel sub-shards; fix or split it, never raise the timeout';
  return [
    `[test:free] ${FREE_SUBSHARD_TIMEOUT}: CI shard ${shard} sub-shard ${record.id} (${record.files.length} files) hit its ${seconds(wallTimeoutMs)}s wall at --jobs ${jobs} on ${context.runner}, run ${context.runId}.`,
    `[test:free]   in flight: ${suspects.length ? suspects.join(', ') : 'unknown (no open file chunk; see the sub-shard log)'}`,
    `[test:free]   reproduce serially: printf '%s\\n' ${files.map(quote).join(' ')} > files.txt && GSTACK_FREE_JOBS=1 bun run scripts/test-free-shards.ts --files-from files.txt`,
    `[test:free]   or the whole CI shard at k=1: bun run scripts/test-free-shards.ts --ci-run ${quote(context.planPath)} --shard ${shard} --result shard-${shard}.json --jobs 1`,
    `[test:free]   ${fix}. See ${DOCS}`,
  ];
}

export interface RunShardOptions { wallTimeoutMs: number; verbose: boolean; testCounts: true; homeGuard?: FreeHomeGuardFactory }

export interface CiShardInput {
  plan: FreeCiPlan;
  shard: FreeCiPlan['shards'][number];
  revision: string;
  /** Sub-shard count; 1 is today's single child. */
  jobs: number;
  wallTimeoutMs: number;
  verbose: boolean;
  context: CiRunContext;
}

export interface CiShardDeps {
  runShard(files: string[], shardNumber: number, totalShards: number, options: RunShardOptions): Promise<FreeShardOutcome>;
  /** The free runner's single flaky-retry pass over every collected outcome. */
  retry(outcomes: FreeShardOutcome[]): Promise<{ exitCode: number; retry: FreeShardOutcome | null }>;
  pack(files: string[], count: number): string[][];
  isSerial(file: string): boolean;
  terminationRequested(): boolean;
  phaseGuard?(files: string[], shards: number): FreeHomeGuard;
  log?(line: string): void;
}

/** Run one planned CI shard at k sub-shards and build its result JSON. */
export async function runCiShard(input: CiShardInput, deps: CiShardDeps): Promise<{ result: FreeCiResult; exitCode: number }> {
  const { plan, shard, revision, jobs, wallTimeoutMs, verbose } = input;
  const log = deps.log ?? ((line: string) => console.log(line));
  if (jobs === 1) {
    const outcome = await deps.runShard(shard.files, shard.shard, plan.shards.length, { wallTimeoutMs, verbose, testCounts: true });
    const retried = await deps.retry([outcome]);
    return { result: { planId: plan.id, revision, outcome, retry: retried.retry }, exitCode: retried.exitCode };
  }
  const startedAt = Date.now();
  const assignments = assignSubShards(plan.id, shard, jobs, deps.isSerial, deps.pack);
  const total = assignments.length;
  log(`[test:free] CI shard ${shard.shard}/${plan.shards.length} at --jobs ${jobs}: ${assignments
    .map(entry => `sub-shard ${entry.id} = runner shard ${entry.index}/${total} (${entry.phase}, ${entry.files.length} files)`).join('; ')}`);
  const walls = new Map<string, number>();
  const run = (entry: SubShardAssignment, homeGuard?: FreeHomeGuardFactory, wall = wallTimeoutMs): Promise<SubShardSettlement> => {
    walls.set(entry.id, wall);
    return deps.runShard(entry.files, entry.index, total, { wallTimeoutMs: wall, verbose, testCounts: true, ...(homeGuard ? { homeGuard } : {}) })
      .then(outcome => ({ assignment: entry, outcome }), (error: unknown) => ({ assignment: entry, error: error instanceof Error ? error.message : String(error) }));
  };
  const parallel = assignments.filter(entry => entry.phase === 'parallel');
  const serial = assignments.filter(entry => entry.phase === 'serial');
  const phaseProblems: string[] = [];
  const settled: SubShardSettlement[] = [];
  if (parallel.length) {
    const guard = deps.phaseGuard ?? ((files: string[], shards: number) => guardFreeHome(files, process.env, { kind: 'concurrent', shards }));
    const phaseHome = guard(parallel.flatMap(entry => entry.files), parallel.length);
    settled.push(...await Promise.all(parallel.map(entry => run(entry, sharedFreeHome))));
    const homeChange = phaseHome.verify();
    if (homeChange) phaseProblems.push(homeChange);
  }
  for (const entry of serial) {
    const left = startedAt + wallTimeoutMs - Date.now();
    settled.push(deps.terminationRequested() ? { assignment: entry, error: 'cancelled before the serial phase' }
      : left <= 0 ? { assignment: entry, error: 'no wall left after the parallel phase' } : await run(entry, undefined, left));
  }
  const merged = mergeSubShardOutcomes(shard, settled, Date.now() - startedAt, phaseProblems);
  for (const problem of merged.problems) console.error(`[test:free] ${FREE_SUBSHARD_REJECTED}: CI shard ${shard.shard}: ${problem}`);
  for (const record of merged.records.filter(entry => entry.status === 'timed-out')) {
    for (const line of subShardTimeoutText(shard.shard, jobs, record, merged.outcome.inFlight ?? [], walls.get(record.id) ?? wallTimeoutMs, input.context, wallTimeoutMs)) console.error(line);
  }
  log(`[test:free] CI shard ${shard.shard}: ${merged.outcome.status} across ${total} sub-shards in ${Math.round(merged.outcome.elapsedMs / 1000)}s`);
  const retried = await deps.retry([merged.outcome]);
  return {
    result: { planId: plan.id, revision, outcome: merged.outcome, retry: retried.retry, jobs, subShards: merged.records },
    exitCode: retried.exitCode,
  };
}

/** The duration seed takes only one-file children's walls, never a sub-shard's or a shard's. */
export function assertSoloSeedSample(file: string, outcome: Pick<FreeShardOutcome, 'files' | 'summary'>): void {
  if (outcome.files.length !== 1 || outcome.files[0] !== file || (outcome.summary?.filesRan ?? 0) > 1) {
    throw new Error(`Duration seed sample for ${file} came from a ${outcome.files.length}-file child; seeds are recorded only from one-file serial children.`);
  }
}