From 4e96f928131f1b3a2806e8f5168cc27bdadf5548 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 31 Oct 2026 18:09:46 +0000 Subject: [PATCH] test-free-shards: --ci-run --jobs k sub-shards inside one CI job (k=1 default), per-file JUnit counts (GSTA-203 PR 2) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- changelog.d/gsta203-pr2-subshards.md | 1 + docs/TESTING_INTERNALS.md | 46 +++ scripts/lib/shard-jobs.ts | 381 ++++++++++++++++++++++++ scripts/test-free-shards.ts | 65 ++++- scripts/test-pr-profile.ts | 1 + test/fixtures/module-size-ratchet.json | 1 + test/shard-jobs.test.ts | 389 +++++++++++++++++++++++++ test/test-free-shards.test.ts | 2 +- test/ubicloud-pregate-remote.test.ts | 2 +- 9 files changed, 874 insertions(+), 14 deletions(-) create mode 100644 changelog.d/gsta203-pr2-subshards.md create mode 100644 scripts/lib/shard-jobs.ts create mode 100644 test/shard-jobs.test.ts diff --git a/changelog.d/gsta203-pr2-subshards.md b/changelog.d/gsta203-pr2-subshards.md new file mode 100644 index 000000000..bc620b736 --- /dev/null +++ b/changelog.d/gsta203-pr2-subshards.md @@ -0,0 +1 @@ +Free test shards can now run their files as several sub-shards side by side inside one CI job. `bun run scripts/test-free-shards.ts --ci-run --shard --result --jobs ` splits the shard into k parts that each run in their own sandboxed `bun test` process. The default stays at 1, so CI runs exactly as before until a workflow opts in. Bad `--jobs` values stop with `FREE_JOBS_INVALID`, a sub-shard that runs out of time prints `FREE_SUBSHARD_TIMEOUT` with a serial reproduce command, and every CI shard result now records per-file test and `expect()` counts. See docs/TESTING_INTERNALS.md#sub-shards-inside-one-ci-job. diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 278790990..22e7e677d 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1555,6 +1555,52 @@ refresh to a maintainer. besides the bot's one refresh commit, later runs leave it alone. Only runs on `main` push the branch; a dispatch from another ref reports only. +### Sub-shards inside one CI job + +`--ci-run --shard i --result --jobs k` runs one planned CI shard as +k sub-shards side by side on the same machine (`scripts/lib/shard-jobs.ts`). Each +sub-shard is an ordinary free shard child: one `bun test` process, serial +inside, with its own sandbox, process-group kill and log. Bun's worker +`--parallel` mode is still not used (decision V3/D6 in +`scripts/test-free-shards.ts`). The default is `k = 1`, which runs today's one +child, so no workflow changes behavior until it passes `--jobs`. + +- `--jobs` applies only to `--ci-run`. It takes a positive whole number up to the + shard's file count (an empty shard takes 1); anything else stops with + `FREE_JOBS_INVALID`. `GSTACK_FREE_JOBS` sets the local full-suite process count + and never changes `--ci-run`. The exclusive host-state shard has one file, so + a workflow passes it `--jobs 1` or no `--jobs`. +- Files are split into disjoint sub-shards packed by the duration seed. Each + sub-shard has an id (`:.`) bound to its file list. + Files in `TREE_MUTATING` or `SLOW_UNDER_LOAD` run after the parallel + sub-shards settle, serially, in one trailing sub-shard. +- The parallel phase uses the concurrent-phase real-home guard, the same one + full-suite mode uses; the serial sub-shard guards itself and names its files. +- Every sub-shard gets the CI shard's own wall deadline, so k never moves a + deadline, and the parent waits for every child to settle. +- The parent merges every sub-shard result into the shard's one `outcome` + before the single flaky-retry pass, so the five-file retry cap stays per CI + shard. A missing, duplicate or incomplete sub-shard result fails the shard + with `FREE_SUBSHARD_REJECTED` and vetoes the retry. `--ci-verify` re-checks + the `jobs` and `subShards` records a `k > 1` result carries. +- Sub-shard runs never write `scripts/free-test-durations.json`; only + `--record-durations` does, and only from one-file children. + +`FREE_SUBSHARD_TIMEOUT` means a sub-shard hit its wall at `k > 1`. The message +names the file in flight, k, the runner and the CI run id, and prints a serial +reproduce command. If the file passes serially, add it to `SLOW_UNDER_LOAD` in +`scripts/lib/shard-jobs.ts` with that run id and a reason. Never raise the +timeout. `SLOW_UNDER_LOAD` is empty today. + +Every `--ci-run` result, at any k, records per-file test and `expect()` counts +in `outcome.counts` (`tests`, `pass`, `fail`, `skip`, `todo`, `expects` per +file, plus shard totals). They come from bun's JUnit reporter, which the runner +writes into the shard sandbox; on the pinned Bun 1.4.2 it attributes both counts +per file inside a multi-file child and leaves the console stream unchanged. +`complete: false` (with a `problem`) means the report was missing or did not add +up to bun's own test count. Counts are evidence for comparing k values; they +never decide a verdict. + ### Windows free lane `windows-free-tests.yml` plans on Linux (`--windows-only --ci-plan --shards 6`), diff --git a/scripts/lib/shard-jobs.ts b/scripts/lib/shard-jobs.ts new file mode 100644 index 000000000..a7df08c24 --- /dev/null +++ b/scripts/lib/shard-jobs.ts @@ -0,0 +1,381 @@ +/** + * Sub-shards inside one CI job: `--ci-run --shard i --result r --jobs k` + * (GSTA-203 PR 2, principle "buy cores, not shards"). + * + * A CI shard's files split into k disjoint, identity-bound sub-shards that run + * side by side through the free runner's own process-shard path (one bun child + * each, serial inside, own sandbox, group kill, log), never Bun's worker + * `--parallel` (decision V3/D6 in scripts/test-free-shards.ts). The rules: + * + * - k = 1 (the default) is today's single child, unchanged. + * - Files in TREE_MUTATING or SLOW_UNDER_LOAD run after the parallel + * sub-shards settle, serially, in one trailing sub-shard. + * - The parallel phase uses the concurrent-phase home guard; every child + * uses the shared guard inside it, exactly as full-suite mode does. + * - Every sub-shard gets the CI shard's own wall deadline, so k never moves + * a deadline, and every launched child settles before the parent goes on. + * - The parent merges every sub-shard result before the single retry pass, + * so the five-file retry cap stays per CI shard; a missing, duplicate or + * incomplete sub-shard result fails the shard. + * - Sub-shard runs never write the duration seed (only `--record-durations` + * does, from one-file children). + * + * Per-file test and expect() counts come from bun's JUnit reporter, which + * attributes both per file inside a multi-file child on the pinned Bun 1.4.2 + * (the console stream is byte-identical with or without it). Counts are + * evidence for k > 1 comparisons; they never decide a verdict. + */ +import * as os from 'node:os'; +import type { FreeCiPlan, FreeCiResult, FreeShardOutcome } from '../test-free-shards'; +import { guardFreeHome, sharedFreeHome, type FreeHomeGuard, type FreeHomeGuardFactory } from './free-home-guard'; +import { normalizeRelativePath } from './shard-engine'; + +/** + * Files that pass alone but time out as a sub-shard under load. Each entry + * names the CI run id where it timed out (FREE_SUBSHARD_TIMEOUT prints it). + * They run in the trailing serial sub-shard under `--jobs`; at k = 1 nothing + * changes. Keys are pinned against the live file census by + * test/shard-jobs.test.ts. Never raise a timeout instead of adding an entry. + */ +export const SLOW_UNDER_LOAD: Record = {}; + +export const FREE_JOBS_INVALID = 'FREE_JOBS_INVALID'; +export const FREE_SUBSHARD_TIMEOUT = 'FREE_SUBSHARD_TIMEOUT'; +export const FREE_SUBSHARD_REJECTED = 'FREE_SUBSHARD_REJECTED'; +const DOCS = 'docs/TESTING_INTERNALS.md#sub-shards-inside-one-ci-job'; +const JOBS_FIX = `--jobs applies only to --ci-run and defaults to 1; GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run. See ${DOCS}`; + +/** `--jobs ` at parse time: digits only, at least 1. */ +export function parseJobsFlag(raw: string | undefined): number { + if (raw === undefined || !/^\d+$/.test(raw) || Number(raw) < 1) { + throw new Error(`${FREE_JOBS_INVALID}: --jobs needs a positive whole number; got ${JSON.stringify(raw ?? '')}. ${JOBS_FIX}`); + } + return Number(raw); +} + +export function rejectJobsWithoutCiRun(): never { + throw new Error(`${FREE_JOBS_INVALID}: --jobs needs --ci-run. ${JOBS_FIX}`); +} + +/** `--jobs ` against the planned shard: k may not exceed its file count (an empty shard allows 1). */ +export function validateJobsForShard(jobs: number, fileCount: number, shard: number): void { + const max = Math.max(1, fileCount); + if (jobs > max) { + throw new Error(`${FREE_JOBS_INVALID}: --jobs ${jobs} is more than CI shard ${shard}'s ${fileCount} file(s); use 1 to ${max}. ${JOBS_FIX}`); + } +} + +export interface FileTestCounts { tests: number; pass: number; fail: number; skip: number; todo: number; expects: number } +export interface ShardTestCounts { + source: 'bun-junit'; + /** Every reported file was planned and the per-file tests add up to bun's own `Ran N tests` count. */ + complete: boolean; + problem?: string; + files: Record; + totals: FileTestCounts; +} + +const zeroCounts = (): FileTestCounts => ({ tests: 0, pass: 0, fail: 0, skip: 0, todo: 0, expects: 0 }); +const addCounts = (into: FileTestCounts, from: FileTestCounts) => { + for (const key of Object.keys(into) as Array) into[key] += from[key]; +}; +const TAG = /<(\/?)(testsuites|testsuite|testcase|failure|error|skipped)\b((?:[^>"']|"[^"]*"|'[^']*')*?)(\/?)>/g; +const ATTR = /([\w:-]+)="([^"]*)"/g; +const ENTITIES: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }; +const decode = (text: string) => text.replace(/&(#x[0-9a-f]+|#\d+|\w+);/gi, (whole, entity: string) => entity[0] === '#' + ? String.fromCodePoint(entity[1].toLowerCase() === 'x' ? parseInt(entity.slice(2), 16) : Number(entity.slice(1))) + : ENTITIES[entity] ?? whole); +const attributes = (text: string): Record => + Object.fromEntries([...text.matchAll(ATTR)].map(([, key, value]) => [key, decode(value)])); + +/** + * Per-file counts from bun's JUnit XML. A testcase is a fail when it holds a + * failure or error, a todo when skipped with message TODO, a skip when + * skipped otherwise, else a pass; expects sum each testcase's assertions. + * Printed paths are matched back to planned files (bun prints them relative + * to the child's real cwd). Files that never ran a test are absent. + */ +export function parseJunitCounts(xml: string | null, planned: readonly string[], testsRan: number | null): ShardTestCounts { + const files: Record = {}; + const totals = zeroCounts(); + const result = (complete: boolean, problem?: string): ShardTestCounts => + ({ source: 'bun-junit', complete, ...(problem ? { problem } : {}), files, totals }); + if (xml === null) return result(false, 'bun wrote no JUnit report'); + const plannedSet = new Set(planned); + const unknown = new Set(); + const canonical = (printed: string) => { + const stripped = normalizeRelativePath(printed).replace(/^(?:\.{1,2}\/)+/, ''); + if (plannedSet.has(stripped)) return stripped; + return planned.find(file => stripped.endsWith(`/${file}`) || file.endsWith(`/${stripped}`)) ?? stripped; + }; + let current: { file: string; state: 'pass' | 'fail' | 'skip' | 'todo'; expects: number } | null = null; + const finish = () => { + if (!current) return; + const entry = files[current.file] ??= zeroCounts(); + entry.tests += 1; + entry[current.state] += 1; + entry.expects += current.expects; + if (!plannedSet.has(current.file)) unknown.add(current.file); + current = null; + }; + for (const [, closing, name, body, selfClosing] of xml.matchAll(TAG)) { + if (name === 'testcase') { + if (closing) { finish(); continue; } + const attrs = attributes(body); + current = { file: canonical(attrs.file ?? ''), state: 'pass', expects: Number(attrs.assertions ?? 0) || 0 }; + if (selfClosing) finish(); + } else if (current && !closing && (name === 'failure' || name === 'error')) { + current.state = 'fail'; + } else if (current && !closing && name === 'skipped' && current.state === 'pass') { + current.state = attributes(body).message === 'TODO' ? 'todo' : 'skip'; + } + } + finish(); + for (const entry of Object.values(files)) addCounts(totals, entry); + if (unknown.size) return result(false, `JUnit named unplanned file(s): ${[...unknown].sort().join(', ')}`); + if (testsRan === null) return result(false, "bun's summary has no test count"); + if (totals.tests !== testsRan) return result(false, `JUnit counted ${totals.tests} tests, bun's summary ${testsRan}`); + return result(true); +} + +/** One shard's counts from its sub-shards' counts; complete only when every part is and no file repeats. */ +export function mergeTestCounts(parts: Array): ShardTestCounts | undefined { + if (parts.every(part => !part)) return undefined; + const files: Record = {}; + const totals = zeroCounts(); + const problems: string[] = []; + parts.forEach((part, index) => { + if (!part) { problems.push(`sub-shard ${index + 1} has no counts`); return; } + if (!part.complete) problems.push(`sub-shard ${index + 1}: ${part.problem ?? 'incomplete'}`); + for (const [file, counts] of Object.entries(part.files)) { + if (files[file]) problems.push(`${file} counted twice`); + files[file] = { ...counts }; + } + addCounts(totals, part.totals); + }); + return { source: 'bun-junit', complete: problems.length === 0, ...(problems.length ? { problem: problems.join('; ') } : {}), files, totals }; +} + +export interface SubShardAssignment { + /** `:.`: binds a result to one plan, shard and file set. */ + id: string; + index: number; + phase: 'parallel' | 'serial'; + files: string[]; +} + +/** + * Split one CI shard into disjoint sub-shards: up to k parallel ones packed by + * `pack`, then one serial sub-shard for the files `isSerial` names. Throws + * unless the assignments cover the shard's files exactly once. + */ +export function assignSubShards( + planId: string, shard: { shard: number; files: string[] }, jobs: number, + isSerial: (file: string) => boolean, pack: (files: string[], count: number) => string[][], +): SubShardAssignment[] { + const serial = shard.files.filter(isSerial); + const parallel = shard.files.filter(file => !isSerial(file)); + const groups = parallel.length ? pack(parallel, Math.min(jobs, parallel.length)).filter(files => files.length) : []; + const assignments: SubShardAssignment[] = [ + ...groups.map(files => ({ phase: 'parallel' as const, files })), + ...(serial.length ? [{ phase: 'serial' as const, files: serial }] : []), + ].map((entry, index) => ({ id: `${planId.slice(0, 12)}:${shard.shard}.${index + 1}`, index: index + 1, ...entry })); + const assigned = assignments.flatMap(entry => entry.files); + if (new Set(assigned).size !== assigned.length || assigned.length !== shard.files.length + || !shard.files.every(file => assigned.includes(file))) { + throw new Error(`${FREE_SUBSHARD_REJECTED}: sub-shard assignment for CI shard ${shard.shard} does not cover its files exactly once`); + } + return assignments; +} + +export type SubShardSettlement = { assignment: SubShardAssignment } & ({ outcome: FreeShardOutcome } | { error: string }); + +/** Why one sub-shard's result cannot stand for its assignment; empty when it can. */ +export function subShardProblems(settled: SubShardSettlement): string[] { + const { assignment } = settled; + if (!('outcome' in settled)) return [`${assignment.id} has no result (${settled.error})`]; + const { outcome } = settled; + const problems: string[] = []; + if (JSON.stringify(outcome.files) !== JSON.stringify(assignment.files)) problems.push(`${assignment.id} reported another file set`); + const summary = outcome.summary; + if (outcome.status === 'passed' && (!summary || summary.filesRan !== assignment.files.length || summary.sawTerminalSummary !== true + || outcome.exitCode !== 0 || outcome.failingFiles.length || outcome.unattributedFailures)) { + problems.push(`${assignment.id} passed without a complete execution summary`); + } + return problems; +} + +export interface SubShardRecord { + id: string; + phase: SubShardAssignment['phase']; + files: string[]; + status: FreeShardOutcome['status'] | 'missing'; + exitCode: number | null; + elapsedMs: number; + summary?: FreeShardOutcome['summary']; + failingFiles: string[]; + unattributedFailures: number; +} + +/** + * One CI shard outcome from every sub-shard: the planned file order, summed + * summaries, the union of failing files, and the worst status. Every rejected + * sub-shard adds unattributed evidence, which also vetoes the retry pass. + */ +export function mergeSubShardOutcomes( + shard: { shard: number; files: string[] }, settled: SubShardSettlement[], elapsedMs: number, extraProblems: string[] = [], +): { outcome: FreeShardOutcome; records: SubShardRecord[]; problems: string[] } { + const problems = [...extraProblems]; + const assigned = settled.flatMap(entry => entry.assignment.files); + if (new Set(assigned).size !== assigned.length) problems.push('a file is assigned to more than one sub-shard'); + if (assigned.length !== shard.files.length || !shard.files.every(file => assigned.includes(file))) problems.push('sub-shards do not cover the CI shard'); + for (const entry of settled) problems.push(...subShardProblems(entry)); + const outcomes = settled.flatMap(entry => ('outcome' in entry ? [entry.outcome] : [])); + const missing = settled.length - outcomes.length; + const status: FreeShardOutcome['status'] = outcomes.some(outcome => outcome.status === 'timed-out') ? 'timed-out' + : problems.length || outcomes.some(outcome => outcome.status !== 'passed') ? 'failed' : 'passed'; + const testsRan = outcomes.map(outcome => outcome.summary?.testsRan ?? null); + const failures = outcomes.flatMap(outcome => outcome.failures ?? []); + const inFlight = outcomes.flatMap(outcome => outcome.inFlight ?? []); + const counts = mergeTestCounts(settled.map(entry => ('outcome' in entry ? entry.outcome.counts : undefined))); + const outcome: FreeShardOutcome = { + shard: shard.shard, + files: shard.files, + status, + exitCode: status === 'passed' ? 0 : outcomes.find(entry => entry.exitCode)?.exitCode ?? 1, + elapsedMs, + groupPid: null, + failingFiles: status === 'passed' ? [] : [...new Set(outcomes.flatMap(entry => entry.failingFiles))], + unattributedFailures: status === 'passed' ? 0 : outcomes.reduce((sum, entry) => sum + entry.unattributedFailures, 0) + problems.length, + ...(failures.length ? { failures } : {}), + summary: { + testsRan: missing || testsRan.some(value => value === null) ? null : testsRan.reduce((sum, value) => sum + value!, 0), + filesRan: outcomes.reduce((sum, entry) => sum + (entry.summary?.filesRan ?? 0), 0), + sawTerminalSummary: !missing && outcomes.every(entry => entry.files.length === 0 || entry.summary?.sawTerminalSummary === true), + }, + ...(counts ? { counts } : {}), + ...(inFlight.length ? { inFlight } : {}), + }; + const records = settled.map((entry): SubShardRecord => { + const { id, phase, files } = entry.assignment; + if (!('outcome' in entry)) return { id, phase, files, status: 'missing', exitCode: null, elapsedMs: 0, failingFiles: [], unattributedFailures: 1 }; + const { status: subStatus, exitCode, elapsedMs: subElapsed, summary, failingFiles, unattributedFailures } = entry.outcome; + return { id, phase, files, status: subStatus, exitCode, elapsedMs: subElapsed, summary, failingFiles, unattributedFailures }; + }); + return { outcome, records, problems }; +} + +/** `--ci-verify` re-checks a k > 1 result's sub-shard records; empty means they stand. */ +export function subShardRecordProblems(result: FreeCiResult): string[] { + if (result.jobs === undefined && result.subShards === undefined) return []; + const records = result.subShards ?? []; + if (!Number.isInteger(result.jobs) || result.jobs! < 2 || !records.length) return ['sub-shard result without a valid --jobs and records']; + const files = records.flatMap(record => record.files); + const problems: string[] = []; + if (new Set(files).size !== files.length || files.length !== result.outcome.files.length + || !result.outcome.files.every(file => files.includes(file))) problems.push('sub-shard records do not cover the shard exactly once'); + if (result.outcome.status === 'passed' && records.some(record => record.status !== 'passed' || record.summary?.filesRan !== record.files.length + || (record.files.length > 0 && record.summary?.sawTerminalSummary !== true))) problems.push('a passing shard holds a failed or incomplete sub-shard'); + return problems; +} + +export interface CiRunContext { planPath: string; runId: string; runner: string } + +/** Where this run is: the CI run id (with attempt) and runner, else `local` and the host. */ +export function ciRunContext(planPath: string, env: NodeJS.ProcessEnv = process.env): CiRunContext { + const runId = env.GITHUB_RUN_ID ? `${env.GITHUB_RUN_ID}${env.GITHUB_RUN_ATTEMPT ? `/attempt ${env.GITHUB_RUN_ATTEMPT}` : ''}` : 'local'; + const cpus = os.availableParallelism?.() ?? os.cpus().length; + return { planPath, runId, runner: `${env.RUNNER_NAME || os.hostname()} (${process.platform}, ${cpus} CPUs)` }; +} + +const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; + +/** The FREE_SUBSHARD_TIMEOUT text: the file, k, runner, a serial reproduce command and the fix. */ +export function subShardTimeoutText( + shard: number, jobs: number, record: SubShardRecord, inFlight: string[], wallTimeoutMs: number, context: CiRunContext, +): string[] { + const suspects = inFlight.filter(file => record.files.includes(file)); + const files = suspects.length ? suspects : record.files; + return [ + `[test:free] ${FREE_SUBSHARD_TIMEOUT}: CI shard ${shard} sub-shard ${record.id} (${record.files.length} files) hit its ${Math.round(wallTimeoutMs / 1000)}s wall at --jobs ${jobs} on ${context.runner}, run ${context.runId}.`, + `[test:free] in flight: ${suspects.length ? suspects.join(', ') : 'unknown (no open file chunk; see the sub-shard log)'}`, + `[test:free] reproduce serially: printf '%s\\n' ${files.map(quote).join(' ')} > files.txt && GSTACK_FREE_JOBS=1 bun run scripts/test-free-shards.ts --files-from files.txt`, + `[test:free] or the whole CI shard at k=1: bun run scripts/test-free-shards.ts --ci-run ${quote(context.planPath)} --shard ${shard} --result shard-${shard}.json --jobs 1`, + `[test:free] if it passes serially, add it to SLOW_UNDER_LOAD in scripts/lib/shard-jobs.ts with run id ${context.runId}; never raise the timeout. See ${DOCS}`, + ]; +} + +export interface RunShardOptions { wallTimeoutMs: number; verbose: boolean; testCounts: true; homeGuard?: FreeHomeGuardFactory } + +export interface CiShardInput { + plan: FreeCiPlan; + shard: FreeCiPlan['shards'][number]; + revision: string; + /** Sub-shard count; 1 is today's single child. */ + jobs: number; + wallTimeoutMs: number; + verbose: boolean; + context: CiRunContext; +} + +export interface CiShardDeps { + runShard(files: string[], shardNumber: number, totalShards: number, options: RunShardOptions): Promise; + /** The free runner's single flaky-retry pass over every collected outcome. */ + retry(outcomes: FreeShardOutcome[]): Promise<{ exitCode: number; retry: FreeShardOutcome | null }>; + pack(files: string[], count: number): string[][]; + isSerial(file: string): boolean; + terminationRequested(): boolean; + phaseGuard?(files: string[], shards: number): FreeHomeGuard; + log?(line: string): void; +} + +/** Run one planned CI shard at k sub-shards and build its result JSON. */ +export async function runCiShard(input: CiShardInput, deps: CiShardDeps): Promise<{ result: FreeCiResult; exitCode: number }> { + const { plan, shard, revision, jobs, wallTimeoutMs, verbose } = input; + const log = deps.log ?? ((line: string) => console.log(line)); + if (jobs === 1) { + const outcome = await deps.runShard(shard.files, shard.shard, plan.shards.length, { wallTimeoutMs, verbose, testCounts: true }); + const retried = await deps.retry([outcome]); + return { result: { planId: plan.id, revision, outcome, retry: retried.retry }, exitCode: retried.exitCode }; + } + const startedAt = Date.now(); + const assignments = assignSubShards(plan.id, shard, jobs, deps.isSerial, deps.pack); + const total = assignments.length; + log(`[test:free] CI shard ${shard.shard}/${plan.shards.length} at --jobs ${jobs}: ${assignments + .map(entry => `sub-shard ${entry.id} = runner shard ${entry.index}/${total} (${entry.phase}, ${entry.files.length} files)`).join('; ')}`); + const run = (entry: SubShardAssignment, homeGuard?: FreeHomeGuardFactory): Promise => + deps.runShard(entry.files, entry.index, total, { wallTimeoutMs, verbose, testCounts: true, ...(homeGuard ? { homeGuard } : {}) }) + .then(outcome => ({ assignment: entry, outcome }), (error: unknown) => ({ assignment: entry, error: error instanceof Error ? error.message : String(error) })); + const parallel = assignments.filter(entry => entry.phase === 'parallel'); + const serial = assignments.filter(entry => entry.phase === 'serial'); + const phaseProblems: string[] = []; + const settled: SubShardSettlement[] = []; + if (parallel.length) { + const guard = deps.phaseGuard ?? ((files: string[], shards: number) => guardFreeHome(files, process.env, { kind: 'concurrent', shards })); + const phaseHome = guard(parallel.flatMap(entry => entry.files), parallel.length); + settled.push(...await Promise.all(parallel.map(entry => run(entry, sharedFreeHome)))); + const homeChange = phaseHome.verify(); + if (homeChange) phaseProblems.push(homeChange); + } + for (const entry of serial) { + settled.push(deps.terminationRequested() ? { assignment: entry, error: 'cancelled before the serial phase' } : await run(entry)); + } + const merged = mergeSubShardOutcomes(shard, settled, Date.now() - startedAt, phaseProblems); + for (const problem of merged.problems) console.error(`[test:free] ${FREE_SUBSHARD_REJECTED}: CI shard ${shard.shard}: ${problem}`); + for (const record of merged.records.filter(entry => entry.status === 'timed-out')) { + for (const line of subShardTimeoutText(shard.shard, jobs, record, merged.outcome.inFlight ?? [], wallTimeoutMs, input.context)) console.error(line); + } + log(`[test:free] CI shard ${shard.shard}: ${merged.outcome.status} across ${total} sub-shards in ${Math.round(merged.outcome.elapsedMs / 1000)}s`); + const retried = await deps.retry([merged.outcome]); + return { + result: { planId: plan.id, revision, outcome: merged.outcome, retry: retried.retry, jobs, subShards: merged.records }, + exitCode: retried.exitCode, + }; +} + +/** The duration seed takes only one-file children's walls, never a sub-shard's or a shard's. */ +export function assertSoloSeedSample(file: string, outcome: Pick): void { + if (outcome.files.length !== 1 || outcome.files[0] !== file) { + throw new Error(`Duration seed sample for ${file} came from a ${outcome.files.length}-file child; seeds are recorded only from one-file serial children.`); + } +} diff --git a/scripts/test-free-shards.ts b/scripts/test-free-shards.ts index ba82c3a48..5f94189ef 100755 --- a/scripts/test-free-shards.ts +++ b/scripts/test-free-shards.ts @@ -77,6 +77,11 @@ * bun run scripts/test-free-shards.ts --attribute-home # name real-home writers: each file alone, private HOME * bun run scripts/test-free-shards.ts --ci-plan plan.json --shards 20 * bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json + * bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json --jobs 4 + * # k sub-shards side by side inside one CI job (default 1 = one child, today's behavior; + * # scripts/lib/shard-jobs.ts). --jobs applies only to --ci-run: a positive whole number up + * # to the shard's file count, else FREE_JOBS_INVALID. GSTACK_FREE_JOBS sets the local + * # full-suite process count and never changes --ci-run; --jobs is its only knob. * bun run scripts/test-free-shards.ts --ci-verify plan.json --results results/ */ @@ -91,6 +96,10 @@ import { resolveStateRoot } from '../lib/state-root'; import { attributeFreeHomeWriters, guardFreeHome, sharedFreeHome, type FreeHomeGuardFactory } from './lib/free-home-guard'; import { appendStepSummary, ciHealthSummary, SEED_REFRESH_COMMAND, unseededWarning, windowsCurationLine } from './lib/free-ci-health'; import { curateWindowsSafe, type CurationResult } from './lib/windows-curation'; +import { + assertSoloSeedSample, ciRunContext, parseJobsFlag, parseJunitCounts, rejectJobsWithoutCiRun, runCiShard, SLOW_UNDER_LOAD, subShardRecordProblems, + validateJobsForShard, type ShardTestCounts, type SubShardRecord, +} from './lib/shard-jobs'; export { curateWindowsSafe, detectWindowsFragility, KNOWN_WINDOWS_INCOMPATIBLE, type CurationResult } from './lib/windows-curation'; import { BunTestOutputClassifier, @@ -447,6 +456,9 @@ export interface FreeCiResult { revision: string; outcome: FreeShardOutcome; retry: FreeShardOutcome | null; + /** Present only at --jobs k > 1: k and one record per sub-shard. */ + jobs?: number; + subShards?: SubShardRecord[]; } /** Preserve the full-suite retry cap across independently running CI jobs. */ @@ -472,6 +484,8 @@ export function verifyFreeCiResults(plan: FreeCiPlan, results: FreeCiResult[]): || JSON.stringify(outcome.files) !== JSON.stringify(shard.files)) throw new Error('CI result identity, shard or file coverage mismatch'); seen.add(outcome.shard); if (!hasCompleteCiSummary(outcome)) throw new Error('Missing or incomplete CI execution summary'); + const subShardProblems = subShardRecordProblems(result); + if (subShardProblems.length) throw new Error(`CI shard ${outcome.shard} sub-shards rejected: ${subShardProblems.join('; ')}`); if (outcome.status === 'passed') { if (outcome.exitCode !== 0 || outcome.failingFiles.length || outcome.unattributedFailures || result.retry) throw new Error('Inconsistent passing CI result'); } else { @@ -542,6 +556,8 @@ type CliOptions = { result: string | null; results: string | null; filesFrom: string | null; + /** Set only by --jobs (with --ci-run); absent means k = 1. */ + jobs?: number; }; export function parseCliOptions(argv: string[]): CliOptions { @@ -556,6 +572,7 @@ export function parseCliOptions(argv: string[]): CliOptions { let wallTimeoutExplicit = false; let quick = false; let attributeHome = false; + let jobs: number | undefined; const paths: Record<'ciPlan' | 'ciRun' | 'ciVerify' | 'result' | 'results' | 'filesFrom', string | null> = { ciPlan: null, ciRun: null, ciVerify: null, result: null, results: null, filesFrom: null, }; @@ -579,6 +596,7 @@ export function parseCliOptions(argv: string[]): CliOptions { '--result': pathFlag('--result', 'result'), '--results': pathFlag('--results', 'results'), '--files-from': pathFlag('--files-from', 'filesFrom'), + '--jobs': (next) => { jobs = parseJobsFlag(next()); }, '--shards': (next) => { const value = next(); if (!value) throw new Error('Missing value for --shards'); @@ -602,8 +620,9 @@ export function parseCliOptions(argv: string[]): CliOptions { if (ciModes > 1 || (ciModes && (quick || listOnly || dryRun || recordDurations || attributeHome))) throw new Error('CI modes cannot be combined with other selection modes'); if (paths.ciRun && (shardIndex === null || !paths.result)) throw new Error('--ci-run requires --shard and --result'); if (paths.ciVerify && !paths.results) throw new Error('--ci-verify requires --results'); + if (jobs !== undefined && !paths.ciRun) rejectJobsWithoutCiRun(); if (quick && (recordDurations || windowsOnly || shardIndex !== null)) throw new Error('--quick cannot change recording, Windows or shard selection'); - return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths }; + return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths, ...(jobs === undefined ? {} : { jobs }) }; } const describeLines = (lines: string[]) => `${lines.slice(0, 5).join(', ')}${lines.length > 5 ? ` (+${lines.length - 5} more)` : ''}`; @@ -1075,6 +1094,10 @@ export interface FreeShardOutcome { unattributedFailures: number; /** Named failures with their excerpts, so a flaky pass records what failed the first time. */ failures?: FreeRunFailure[]; + /** Per-file test and expect() counts from bun's JUnit report (--ci-run only). */ + counts?: ShardTestCounts; + /** Wedge suspects at a wall-timeout kill (FreeRunReport.inFlight). */ + inFlight?: string[]; } export interface ShardCommand { @@ -1087,8 +1110,10 @@ export interface RunFreeShardOptions { wallTimeoutMs?: number; rootDir?: string; env?: NodeJS.ProcessEnv; - /** Override the spawned command. Tests inject fake pass/fail/slow commands. */ - commandFor?: (files: string[]) => ShardCommand; + /** Override the spawned command. Tests inject fake pass/fail/slow commands; `junitPath` is set with testCounts. */ + commandFor?: (files: string[], junitPath?: string) => ShardCommand; + /** Ask bun for a JUnit report in the shard sandbox and record per-file counts on the outcome (--ci-run). */ + testCounts?: boolean; /** Suppress ALL child output from the console (tests). The classifier and the log file still see every byte. */ quiet?: boolean; /** Forward the full child stream to the console (legacy firehose). Default: the quiet filtered console. */ @@ -1633,12 +1658,14 @@ export async function runFreeShard( const shardLog = openShardLog(logPath, label, 0o600); log(`[test:free] full log: ${logPath}`); - const { command, args } = options.commandFor - ? options.commandFor(files) - : { command: process.execPath, args: buildShardArgs(files, { rootDir }) }; - // realpath: the browser tracker refuses a state dir whose path resolves elsewhere. const { stateDir, env } = createShardSandbox('gstack-free-shard-', options.env ?? process.env, { realpath: true }); + // The JUnit reporter leaves the console stream byte-identical (Bun 1.4.2); it only adds per-file counts. + const junitPath = options.testCounts ? path.join(stateDir, 'junit.xml') : undefined; + const { command, args } = options.commandFor + ? options.commandFor(files, junitPath) + : { command: process.execPath, args: [...buildShardArgs(files, { rootDir }), ...(junitPath ? ['--reporter=junit', `--reporter-outfile=${junitPath}`] : [])] }; + let junitXml = null as string | null; // CLI renders otherwise share the repo's .gstack/browse.json, where concurrent // shards and prior daemons replace each other's state; override inherited state. env.BROWSE_STATE_FILE = path.join(stateDir, '.gstack', 'browse.json'); @@ -1703,6 +1730,9 @@ export async function runFreeShard( shardLog.write(diagnostic + '\n'); } await new Promise((resolve) => shardLog.stream.end(() => resolve())); + if (junitPath) { + try { junitXml = fs.readFileSync(junitPath, 'utf8'); } catch { /* counts record the missing report */ } + } try { if (!cleanupError) fs.rmSync(stateDir, { recursive: true, force: true }); } catch { @@ -1745,6 +1775,8 @@ export async function runFreeShard( shard: shardNumber, files, status, exitCode, elapsedMs: Date.now() - startedAt, groupPid, failingFiles, unattributedFailures, ...(status === 'passed' ? {} : { failures: report.failures }), summary: { testsRan: report.testsRan, filesRan: report.filesRan, sawTerminalSummary: report.sawTerminalSummary }, + ...(junitPath ? { counts: parseJunitCounts(junitXml, files, report.testsRan) } : {}), + ...(status === 'timed-out' && report.inFlight.length ? { inFlight: report.inFlight } : {}), }; log(shardEpilogue(outcome, totalShards)); for (const line of buildRunEpilogue(status, report, outcome.elapsedMs, logPath)) log(line); @@ -1799,6 +1831,7 @@ async function recordFreeTestDurations(files: string[], jobs: number, wallTimeou const outcome = await runFreeShard([file], index + 1, files.length, { wallTimeoutMs, quiet: true, homeGuard: sharedFreeHome, }); + assertSoloSeedSample(file, outcome); durations[normalizeRelativePath(file)] = outcome.elapsedMs; if (outcome.status !== 'passed') failed.push(file); }); @@ -1944,14 +1977,22 @@ async function main(): Promise { } const shard = plan.shards[options.shardIndex! - 1]; if (!shard || shard.shard !== options.shardIndex) throw new Error('CI shard index is outside the plan'); - const outcome = await runFreeShard(shard.files, shard.shard, plan.shards.length, { + const jobs = options.jobs ?? 1; + validateJobsForShard(jobs, shard.files.length, shard.shard); + const durations = loadFreeTestDurations() ?? {}; + const { result, exitCode } = await runCiShard({ + plan, shard, revision, jobs, verbose: options.verbose, context: ciRunContext(options.ciRun!), wallTimeoutMs: options.wallTimeoutExplicit ? options.wallTimeoutMs : wallTimeoutForPackedShard(shard.predictedMs, options.wallTimeoutMs, shard.files.length), - verbose: options.verbose, + }, { + runShard: (files, shardNumber, totalShards, runOptions) => runFreeShard(files, shardNumber, totalShards, runOptions), + retry: outcomes => retryFailedFreeFiles(outcomes, plan.shards.length, options), + pack: (files, count) => packShardsByDuration(files, count, durations).shards, + isSerial: file => file in TREE_MUTATING || file in SLOW_UNDER_LOAD, + terminationRequested: isTerminationRequested, }); - const retried = await retryFailedFreeFiles([outcome], plan.shards.length, options); - writeJson(options.result!, { planId: plan.id, revision, outcome, retry: retried.retry } satisfies FreeCiResult); - return retried.exitCode; + writeJson(options.result!, result); + return exitCode; } let files = allFiles; diff --git a/scripts/test-pr-profile.ts b/scripts/test-pr-profile.ts index 911e393d0..287319f96 100644 --- a/scripts/test-pr-profile.ts +++ b/scripts/test-pr-profile.ts @@ -237,6 +237,7 @@ export const PAID_WORKFLOW_FILES = [ export const FREE_ONLY_PR_FILES = [ 'scripts/test-free-shards.ts', 'scripts/lib/free-home-guard.ts', // Imported only by the free shard runner. + 'scripts/lib/shard-jobs.ts', // Imported only by the free shard runner (--ci-run --jobs sub-shards). 'test/helpers/auq-parallel-worker.ts', 'test/helpers/stored-zip.ts', // Read only by the free test-health report tests. 'scripts/free-test-durations.json', diff --git a/test/fixtures/module-size-ratchet.json b/test/fixtures/module-size-ratchet.json index d56450722..367669d2c 100644 --- a/test/fixtures/module-size-ratchet.json +++ b/test/fixtures/module-size-ratchet.json @@ -57,6 +57,7 @@ { "file": "scripts/eval-bg.ts", "movedFrom": "(new: eval:bg dispatch/local backend, W5d)" }, { "file": "test/helpers/module-size.ts", "movedFrom": "(new: ratchet (c) counter)" }, { "file": "scripts/lib/free-home-guard.ts", "movedFrom": "(new: free-lane home-write tripwire, kept out of scripts/test-free-shards.ts)" }, + { "file": "scripts/lib/shard-jobs.ts", "movedFrom": "(new: --ci-run --jobs sub-shards and per-file JUnit counts, kept out of scripts/test-free-shards.ts and scripts/lib/shard-engine.ts, GSTA-203)" }, { "file": "browse/src/routes/activity.ts", "movedFrom": "browse/src/server.ts" }, { "file": "browse/src/routes/commands.ts", "movedFrom": "browse/src/server.ts" }, { "file": "browse/src/routes/core.ts", "movedFrom": "browse/src/server.ts" }, diff --git a/test/shard-jobs.test.ts b/test/shard-jobs.test.ts new file mode 100644 index 000000000..9296a3c4c --- /dev/null +++ b/test/shard-jobs.test.ts @@ -0,0 +1,389 @@ +/** + * Sub-shards inside one CI job (`--ci-run … --jobs k`, scripts/lib/shard-jobs.ts): + * the k = 1 golden path, identity-bound assignment, rejection of missing, + * duplicate and incomplete sub-shard results, one retry pass per CI shard, + * serial TREE_MUTATING and SLOW_UNDER_LOAD files, the timeout text, and per-file + * test and expect() counts from real bun children at k = 1 and k > 1. + */ +import { describe, expect, spyOn, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { + collectFreeTestFiles, createFreeCiPlan, eligibleFreeRetryFiles, parseCliOptions, runFreeShard, TREE_MUTATING, verifyFreeCiResults, + type FreeCiResult, type FreeShardOutcome, +} from '../scripts/test-free-shards'; +import { + assertSoloSeedSample, assignSubShards, ciRunContext, FREE_JOBS_INVALID, mergeSubShardOutcomes, parseJobsFlag, parseJunitCounts, + runCiShard, SLOW_UNDER_LOAD, subShardTimeoutText, validateJobsForShard, type CiShardDeps, type RunShardOptions, +} from '../scripts/lib/shard-jobs'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const EXCLUSIVE = Object.keys(TREE_MUTATING)[0]; +const files = Array.from({ length: 7 }, (_, i) => `test/sample-${i}.test.ts`); +const durations = Object.fromEntries(files.map((file, i) => [file, (i + 1) * 1_000])); +const context = { planPath: 'plan.json', runId: '123/attempt 2', runner: 'runner-7 (linux, 8 CPUs)' }; + +const passed = (shardFiles: string[], shard = 1): FreeShardOutcome => ({ + shard, files: shardFiles, status: 'passed', exitCode: 0, elapsedMs: 5, groupPid: null, failingFiles: [], unattributedFailures: 0, + summary: { testsRan: shardFiles.length * 2, filesRan: shardFiles.length, sawTerminalSummary: true }, +}); +const failed = (shardFiles: string[], failing: string[], shard = 1): FreeShardOutcome => ({ + ...passed(shardFiles, shard), status: 'failed', exitCode: 1, failingFiles: failing, +}); + +function harness(shardFiles: string[], respond: (files: string[]) => FreeShardOutcome | Promise = passed) { + const calls: Array<{ files: string[]; shardNumber: number; totalShards: number; options: RunShardOptions; startedAt: number; endedAt?: number }> = []; + const retries: FreeShardOutcome[][] = []; + const guards: Array<{ files: string[]; shards: number }> = []; + let homeChange: string | null = null; + let terminate = false; + const deps: CiShardDeps = { + async runShard(runFiles, shardNumber, totalShards, options) { + const call: (typeof calls)[number] = { files: runFiles, shardNumber, totalShards, options, startedAt: performance.now() }; + calls.push(call); + await new Promise(resolve => setTimeout(resolve, 20)); + try { return await respond(runFiles); } finally { call.endedAt = performance.now(); } + }, + async retry(outcomes) { + retries.push(outcomes); + return { exitCode: outcomes.every(outcome => outcome.status === 'passed') ? 0 : 1, retry: null }; + }, + pack: (packFiles, count) => Array.from({ length: count }, (_, i) => packFiles.filter((_, j) => j % count === i)), + isSerial: file => file in TREE_MUTATING || file in SLOW_UNDER_LOAD, + terminationRequested: () => terminate, + phaseGuard: (guarded, shards) => { guards.push({ files: guarded, shards }); return { verify: () => homeChange }; }, + log: () => {}, + }; + const plan = { version: 1 as const, revision: 'revision-a', id: '0123456789abcdef', shards: [{ shard: 1, files: shardFiles, predictedMs: 0 }] }; + const shard = plan.shards[0]; + const run = (jobs: number) => runCiShard({ plan, shard, revision: 'revision-a', jobs, wallTimeoutMs: 360_000, verbose: false, context }, deps); + return { calls, retries, guards, deps, plan, shard, run, setHomeChange: (value: string) => { homeChange = value; }, cancel: () => { terminate = true; } }; +} + +describe('--jobs validation (FREE_JOBS_INVALID)', () => { + test('accepts positive whole numbers and rejects everything else with the code and the precedence note', () => { + expect(parseJobsFlag('1')).toBe(1); + expect(parseJobsFlag('4')).toBe(4); + for (const raw of ['0', '-1', '2.5', 'abc', '', ' 2', undefined]) { + expect(() => parseJobsFlag(raw)).toThrow(FREE_JOBS_INVALID); + expect(() => parseJobsFlag(raw)).toThrow(/GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run/); + } + }); + + test('--jobs belongs to --ci-run only, and an absent --jobs leaves the parsed options unchanged', () => { + const base = ['--ci-run', 'plan.json', '--shard', '1', '--result', 'r.json']; + expect(parseCliOptions([...base, '--jobs', '4']).jobs).toBe(4); + expect('jobs' in parseCliOptions(base)).toBe(false); + expect(() => parseCliOptions([...base, '--jobs', '0'])).toThrow(FREE_JOBS_INVALID); + expect(() => parseCliOptions([...base, '--jobs'])).toThrow(FREE_JOBS_INVALID); + expect(() => parseCliOptions(['--jobs', '2'])).toThrow(`${FREE_JOBS_INVALID}: --jobs needs --ci-run`); + expect(() => parseCliOptions(['--ci-plan', 'p.json', '--jobs', '2'])).toThrow(FREE_JOBS_INVALID); + }); + + test('k may not exceed the shard file count; an empty shard still takes 1', () => { + expect(() => validateJobsForShard(4, 4, 3)).not.toThrow(); + expect(() => validateJobsForShard(1, 0, 21)).not.toThrow(); + expect(() => validateJobsForShard(5, 4, 3)).toThrow(`${FREE_JOBS_INVALID}: --jobs 5 is more than CI shard 3's 4 file(s); use 1 to 4.`); + expect(() => validateJobsForShard(2, 0, 21)).toThrow(FREE_JOBS_INVALID); + }); +}); + +describe('k = 1 golden: PR 2 changes nothing in CI', () => { + test('one child over the whole planned shard with the same wall, one retry pass, and the same result keys', async () => { + const h = harness(files); + const { result, exitCode } = await h.run(1); + expect(h.calls.map(({ files: f, shardNumber, totalShards, options }) => ({ f, shardNumber, totalShards, options }))).toEqual([ + { f: h.shard.files, shardNumber: 1, totalShards: 1, options: { wallTimeoutMs: 360_000, verbose: false, testCounts: true } }, + ]); + expect(h.guards).toEqual([]); + expect(h.retries).toEqual([[passed(h.shard.files)]]); + expect(h.calls[0].options.wallTimeoutMs).toBe(360_000); + expect(Object.keys(result)).toEqual(['planId', 'revision', 'outcome', 'retry']); + expect(result).toEqual({ planId: h.plan.id, revision: 'revision-a', outcome: passed(h.shard.files), retry: null }); + expect(exitCode).toBe(0); + }); + + test('a k = 1 failure reaches the retry pass as the child reported it', async () => { + const h = harness(files, runFiles => failed(runFiles, [runFiles[0]])); + const { result, exitCode } = await h.run(1); + expect(h.retries).toEqual([[failed(h.shard.files, [h.shard.files[0]])]]); + expect(result.outcome).toBe(h.retries[0][0]); + expect(exitCode).toBe(1); + }); +}); + +describe('sub-shard assignment', () => { + const pack = (packFiles: string[], count: number) => Array.from({ length: count }, (_, i) => packFiles.filter((_, j) => j % count === i)); + + test('disjoint, identity-bound, covering the shard once, with exclusive files in one trailing serial sub-shard', () => { + const shardFiles = [...files, EXCLUSIVE]; + const assignments = assignSubShards('0123456789abcdef', { shard: 4, files: shardFiles }, 3, file => file in TREE_MUTATING, pack); + expect(assignments.map(entry => [entry.id, entry.phase, entry.files.length])).toEqual([ + ['0123456789ab:4.1', 'parallel', 3], ['0123456789ab:4.2', 'parallel', 2], ['0123456789ab:4.3', 'parallel', 2], ['0123456789ab:4.4', 'serial', 1], + ]); + expect(assignments.at(-1)!.files).toEqual([EXCLUSIVE]); + expect(assignments.flatMap(entry => entry.files).sort()).toEqual([...shardFiles].sort()); + }); + + test('k is clamped to the parallel file count and a shard of only exclusive files stays one serial child', () => { + expect(assignSubShards('p', { shard: 1, files: files.slice(0, 2) }, 4, () => false, pack).length).toBe(2); + expect(assignSubShards('p', { shard: 1, files: [EXCLUSIVE] }, 1, file => file in TREE_MUTATING, pack) + .map(entry => entry.phase)).toEqual(['serial']); + }); + + test('a packer that drops or repeats a file is rejected before any child starts', () => { + expect(() => assignSubShards('p', { shard: 2, files }, 2, () => false, packFiles => [packFiles, [packFiles[0]]])).toThrow(/FREE_SUBSHARD_REJECTED/); + expect(() => assignSubShards('p', { shard: 2, files }, 2, () => false, packFiles => [packFiles.slice(1)])).toThrow(/exactly once/); + }); +}); + +describe('k > 1: merge, rejection and one retry pass per CI shard', () => { + test('parallel sub-shards share the concurrent home guard, then exclusive files run alone after they settle', async () => { + const h = harness([...files, EXCLUSIVE]); + const { result, exitCode } = await h.run(3); + expect(exitCode).toBe(0); + const parallel = h.calls.filter(call => !call.files.includes(EXCLUSIVE)); + const serial = h.calls.find(call => call.files.includes(EXCLUSIVE))!; + expect(parallel).toHaveLength(3); + expect(parallel.every(call => typeof call.options.homeGuard === 'function')).toBe(true); + expect(serial.options.homeGuard).toBeUndefined(); + expect(serial.files).toEqual([EXCLUSIVE]); + expect(serial.startedAt).toBeGreaterThanOrEqual(Math.max(...parallel.map(call => call.endedAt!))); + expect(h.guards).toEqual([{ files: parallel.flatMap(call => call.files), shards: 3 }]); + expect(h.calls.every(call => call.options.wallTimeoutMs === 360_000 && call.totalShards === 4)).toBe(true); + expect(result.jobs).toBe(3); + expect(result.outcome).toMatchObject({ shard: 1, files: h.shard.files, status: 'passed', exitCode: 0, failingFiles: [], unattributedFailures: 0, + summary: { testsRan: 16, filesRan: 8, sawTerminalSummary: true } }); + expect(result.subShards!.map(record => record.status)).toEqual(['passed', 'passed', 'passed', 'passed']); + expect(h.retries).toHaveLength(1); + expect(() => verifyFreeCiResults(h.plan, [result])).not.toThrow(); + }); + + test('failures from every sub-shard are collected first, so k never multiplies the five-file retry cap', async () => { + const h = harness(files, runFiles => failed(runFiles, runFiles.slice(0, 2))); + const { result, exitCode } = await h.run(3); + expect(h.retries).toHaveLength(1); + expect(h.retries[0]).toHaveLength(1); + expect(result.outcome.failingFiles).toHaveLength(6); + expect(eligibleFreeRetryFiles(h.retries[0])).toBeNull(); + expect(exitCode).toBe(1); + const spread = harness(files, runFiles => failed(runFiles, runFiles.slice(0, 1))); + await spread.run(3); + expect(eligibleFreeRetryFiles(spread.retries[0])!.sort()).toEqual(spread.calls.map(call => call.files[0]).sort()); + }); + + test('a missing, duplicate or incomplete sub-shard result fails the CI shard and vetoes the retry', async () => { + const errors = spyOn(console, 'error').mockImplementation(() => {}); + try { + const cases: Array<(runFiles: string[]) => FreeShardOutcome | Promise> = [ + runFiles => runFiles.includes(files[0]) ? Promise.reject(new Error('spawn failed')) : passed(runFiles), + runFiles => passed(runFiles.includes(files[0]) ? [...runFiles, files[1]] : runFiles), + runFiles => runFiles.includes(files[0]) ? { ...passed(runFiles), summary: { testsRan: 1, filesRan: runFiles.length - 1, sawTerminalSummary: true } } : passed(runFiles), + runFiles => runFiles.includes(files[0]) ? { ...passed(runFiles), summary: undefined } : passed(runFiles), + ]; + for (const respond of cases) { + errors.mockClear(); + const h = harness(files, respond); + const { result, exitCode } = await h.run(3); + expect(h.calls).toHaveLength(3); + expect(result.outcome.status).toBe('failed'); + expect(result.outcome.unattributedFailures).toBeGreaterThan(0); + expect(eligibleFreeRetryFiles([result.outcome])).toBeNull(); + expect(exitCode).toBe(1); + expect(errors.mock.calls.map(call => String(call[0])).some(line => line.startsWith('[test:free] FREE_SUBSHARD_REJECTED: CI shard 1: 0123456789ab:1.'))).toBe(true); + } + } finally { errors.mockRestore(); } + }); + + test('a real-home change in the concurrent phase and a cancelled serial phase both fail the shard', async () => { + const errors = spyOn(console, 'error').mockImplementation(() => {}); + try { + const home = harness(files); + home.setHomeChange('real home changed while 3 shards ran concurrently'); + expect((await home.run(3)).result.outcome.status).toBe('failed'); + const cancelled = harness([...files, EXCLUSIVE]); + cancelled.cancel(); + const { result } = await cancelled.run(2); + expect(cancelled.calls.some(call => call.files.includes(EXCLUSIVE))).toBe(false); + expect(result.subShards!.at(-1)).toMatchObject({ status: 'missing', files: [EXCLUSIVE] }); + expect(result.outcome.status).toBe('failed'); + } finally { errors.mockRestore(); } + }); + + test('a sub-shard timeout prints FREE_SUBSHARD_TIMEOUT with the file, k, runner, run id and a serial reproduce', async () => { + const errors = spyOn(console, 'error').mockImplementation(() => {}); + try { + const h = harness(files, runFiles => runFiles.includes(files[3]) + ? { ...failed(runFiles, []), status: 'timed-out', exitCode: null, inFlight: [files[3]] } : passed(runFiles)); + const { result, exitCode } = await h.run(3); + expect(result.outcome.status).toBe('timed-out'); + expect(exitCode).toBe(1); + const text = errors.mock.calls.map(call => String(call[0])).join('\n'); + expect(text).toContain('FREE_SUBSHARD_TIMEOUT: CI shard 1 sub-shard 0123456789ab:1.1 (3 files)'); + expect(text).toContain('hit its 360s wall at --jobs 3 on runner-7 (linux, 8 CPUs), run 123/attempt 2.'); + expect(text).toContain(`in flight: ${files[3]}`); + expect(text).toContain(`printf '%s\\n' '${files[3]}' > files.txt && GSTACK_FREE_JOBS=1 bun run scripts/test-free-shards.ts --files-from files.txt`); + expect(text).toContain("--ci-run 'plan.json' --shard 1 --result shard-1.json --jobs 1"); + expect(text).toContain('add it to SLOW_UNDER_LOAD in scripts/lib/shard-jobs.ts with run id 123/attempt 2; never raise the timeout.'); + } finally { errors.mockRestore(); } + }); + + test('--ci-verify rejects tampered sub-shard records', () => { + const plan = createFreeCiPlan(files, 1, durations, 'revision-a'); + const shard = plan.shards[0]; + const assignments = assignSubShards(plan.id, shard, 2, () => false, packFiles => [packFiles.slice(0, 3), packFiles.slice(3)]); + const merged = mergeSubShardOutcomes(shard, assignments.map(assignment => ({ assignment, outcome: passed(assignment.files) })), 10); + const result: FreeCiResult = { planId: plan.id, revision: 'revision-a', outcome: merged.outcome, retry: null, jobs: 2, subShards: merged.records }; + expect(() => verifyFreeCiResults(plan, [result])).not.toThrow(); + const overlap = structuredClone(result); + overlap.subShards![1].files = [...overlap.subShards![1].files, overlap.subShards![0].files[0]]; + expect(() => verifyFreeCiResults(plan, [overlap])).toThrow(/sub-shards rejected/); + const hidden = structuredClone(result); + hidden.subShards![0].status = 'failed'; + expect(() => verifyFreeCiResults(plan, [hidden])).toThrow(/failed or incomplete sub-shard/); + expect(() => verifyFreeCiResults(plan, [{ ...result, jobs: 1 }])).toThrow(/valid --jobs/); + }); +}); + +describe('per-file test and expect() counts from bun JUnit', () => { + const xml = (file: string) => ` + + + + + + + + x + +`; + + test('classifies every testcase and sums assertions per file; matches cwd-relative paths back to planned files', () => { + const counts = parseJunitCounts(xml('../../test/a&b.test.ts'), ['test/a&b.test.ts'], 4); + expect(counts).toEqual({ source: 'bun-junit', complete: true, files: { 'test/a&b.test.ts': { tests: 4, pass: 1, fail: 1, skip: 1, todo: 1, expects: 9 } }, + totals: { tests: 4, pass: 1, fail: 1, skip: 1, todo: 1, expects: 9 } }); + }); + + test('a missing report, an unplanned file or a count that disagrees with bun summary is incomplete, never silently complete', () => { + expect(parseJunitCounts(null, ['test/a.test.ts'], 4)).toMatchObject({ complete: false, problem: 'bun wrote no JUnit report' }); + expect(parseJunitCounts(xml('test/z.test.ts'), ['test/a.test.ts'], 4)).toMatchObject({ complete: false, problem: expect.stringContaining('unplanned') }); + expect(parseJunitCounts(xml('test/a.test.ts'), ['test/a.test.ts'], 5)).toMatchObject({ complete: false, problem: "JUnit counted 4 tests, bun's summary 5" }); + expect(parseJunitCounts(xml('test/a.test.ts'), ['test/a.test.ts'], null).complete).toBe(false); + }); + + test('real bun children: counts are identical per file at k = 1 and k = 2, add up to bun summary, and leave the stream unchanged', async () => { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-shard-jobs-'))); + try { + const fixtures: Record = { + 't/a.test.ts': "import { test, expect } from 'bun:test';\ntest('a1', () => { expect(1).toBe(1); expect(2).toBe(2); });\ntest('a2', () => { expect(3).toBe(3); });\ntest.skip('a3', () => {});\n", + 't/b.test.ts': "import { test, expect, describe } from 'bun:test';\ndescribe('g', () => { test('b1', () => { for (let i = 0; i < 5; i++) expect(i).toBe(i); }); });\ntest.todo('b2');\n", + 't/c.test.ts': "import { test, expect } from 'bun:test';\ntest('c1', () => { expect([1]).toEqual([1]); });\n", + 't/d.test.ts': "import { test, expect } from 'bun:test';\ntest.each([1, 2, 3])('d%p', n => { expect(n).toBeGreaterThan(0); expect(n).toBeLessThan(4); });\n", + }; + for (const [file, body] of Object.entries(fixtures)) { + fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + fs.writeFileSync(path.join(root, file), body); + } + const planned = Object.keys(fixtures); + const plan = createFreeCiPlan(planned, 1, {}, 'rev'); + const pids: number[] = []; + const deps = (logs: string[]): CiShardDeps => ({ + runShard: async (runFiles, n, total, options) => { + const logFilePath = path.join(root, `stream-${logs.length}-${runFiles.length}-${n}-${total}.log`); + logs.push(logFilePath); + const outcome = await runFreeShard(runFiles, n, total, { ...options, rootDir: root, quiet: true, log: () => {}, logFilePath }); + if (outcome.groupPid) pids.push(outcome.groupPid); + return outcome; + }, + retry: async outcomes => ({ exitCode: outcomes[0].status === 'passed' ? 0 : 1, retry: null }), + pack: (packFiles, count) => Array.from({ length: count }, (_, i) => packFiles.filter((_, j) => j % count === i)), + isSerial: () => false, + terminationRequested: () => false, + log: () => {}, + }); + const input = { plan, shard: plan.shards[0], revision: 'rev', wallTimeoutMs: 60_000, verbose: false, context }; + const serialLogs: string[] = []; + const one = await runCiShard({ ...input, jobs: 1 }, deps(serialLogs)); + const two = await runCiShard({ ...input, jobs: 2 }, deps([])); + expect([one.exitCode, two.exitCode]).toEqual([0, 0]); + const expected = { + 't/a.test.ts': { tests: 3, pass: 2, fail: 0, skip: 1, todo: 0, expects: 3 }, + 't/b.test.ts': { tests: 2, pass: 1, fail: 0, skip: 0, todo: 1, expects: 5 }, + 't/c.test.ts': { tests: 1, pass: 1, fail: 0, skip: 0, todo: 0, expects: 1 }, + 't/d.test.ts': { tests: 3, pass: 3, fail: 0, skip: 0, todo: 0, expects: 6 }, + }; + expect(one.result.outcome.counts).toMatchObject({ complete: true, files: expected, totals: { tests: 9, expects: 15 } }); + expect(two.result.outcome.counts).toEqual(one.result.outcome.counts); + expect(two.result.subShards).toHaveLength(2); + expect(one.result.outcome.summary).toEqual(two.result.outcome.summary); + const bare = await runFreeShard(planned, 1, 1, { rootDir: root, quiet: true, log: () => {}, logFilePath: path.join(root, 'bare.log') }); + expect(bare.counts).toBeUndefined(); + const stream = (file: string) => fs.readFileSync(file, 'utf8').replace(/ \[[\d.]+m?s\]/g, '').split('\n').filter(line => !line.startsWith('[test:free]')); + expect(stream(serialLogs[0])).toEqual(stream(path.join(root, 'bare.log'))); + for (const pid of pids) expect(() => process.kill(pid, 0)).toThrow(); + } finally { fs.rmSync(root, { recursive: true, force: true }); } + }, 60_000); + + test('a hung sub-shard is killed at its wall while its sibling finishes, and no child outlives the parent', async () => { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-shard-jobs-'))); + const errors = spyOn(console, 'error').mockImplementation(() => {}); + try { + const planned = ['t/ok.test.ts', 't/hang.test.ts']; + for (const file of planned) { + fs.mkdirSync(path.join(root, 't'), { recursive: true }); + fs.writeFileSync(path.join(root, file), "import { test, expect } from 'bun:test';\ntest('ok', () => { expect(1).toBe(1); });\n"); + } + const plan = createFreeCiPlan(planned, 1, {}, 'rev'); + const outcomes: FreeShardOutcome[] = []; + const startedAt = performance.now(); + const { result } = await runCiShard({ plan, shard: plan.shards[0], revision: 'rev', jobs: 2, wallTimeoutMs: 2_000, verbose: false, context }, { + runShard: async (runFiles, n, total, options) => { + const outcome = await runFreeShard(runFiles, n, total, { ...options, rootDir: root, quiet: true, log: () => {}, + ...(runFiles.includes('t/hang.test.ts') ? { commandFor: () => ({ command: process.execPath, args: ['-e', 'setInterval(() => {}, 1000)'] }) } : {}) }); + outcomes.push(outcome); + return outcome; + }, + retry: async () => ({ exitCode: 1, retry: null }), + pack: packFiles => packFiles.map(file => [file]), + isSerial: () => false, + terminationRequested: () => false, + log: () => {}, + }); + expect(performance.now() - startedAt).toBeLessThan(20_000); + expect(result.outcome.status).toBe('timed-out'); + expect(result.subShards!.map(record => [record.files[0], record.status]).sort()).toEqual([['t/hang.test.ts', 'timed-out'], ['t/ok.test.ts', 'passed']]); + for (const outcome of outcomes) expect(() => process.kill(outcome.groupPid!, 0)).toThrow(); + expect(errors.mock.calls.some(call => String(call[0]).includes('FREE_SUBSHARD_TIMEOUT'))).toBe(true); + } finally { errors.mockRestore(); fs.rmSync(root, { recursive: true, force: true }); } + }, 60_000); +}); + +describe('serial lists and the duration seed', () => { + test('SLOW_UNDER_LOAD entries name live free files, a reason and a CI run id, and never overlap TREE_MUTATING', () => { + const census = new Set(collectFreeTestFiles()); + for (const [file, entry] of Object.entries(SLOW_UNDER_LOAD)) { + expect(census.has(file), `SLOW_UNDER_LOAD names ${file}, which is not a free test file`).toBe(true); + expect(file in TREE_MUTATING).toBe(false); + expect(entry.reason.trim()).not.toBe(''); + expect(entry.runId).toMatch(/^\d+(\/attempt \d+)?$/); + } + }); + + test('seeds take only one-file children; sub-shard runs have no path to the seed writer', () => { + expect(() => assertSoloSeedSample('test/a.test.ts', { files: ['test/a.test.ts'] })).not.toThrow(); + expect(() => assertSoloSeedSample('test/a.test.ts', { files: ['test/a.test.ts', 'test/b.test.ts'] })).toThrow(/one-file serial children/); + expect(fs.readFileSync(path.join(ROOT, 'scripts/lib/shard-jobs.ts'), 'utf8')).not.toContain('writeDurationSeed'); + const runner = fs.readFileSync(path.join(ROOT, 'scripts/test-free-shards.ts'), 'utf8'); + const recorder = runner.slice(runner.indexOf('async function recordFreeTestDurations('), runner.indexOf('async function retryFailedFreeFiles(')); + expect(runner.split('writeDurationSeed(').length - 1).toBe(recorder.split('writeDurationSeed(').length - 1); + expect(recorder).toContain('assertSoloSeedSample(file, outcome)'); + }); + + test('the run context names the CI run and attempt, else local', () => { + expect(ciRunContext('p.json', { GITHUB_RUN_ID: '42', GITHUB_RUN_ATTEMPT: '3', RUNNER_NAME: 'ubi-1' })).toMatchObject({ runId: '42/attempt 3', runner: expect.stringMatching(/^ubi-1 \(/) }); + expect(ciRunContext('p.json', {}).runId).toBe('local'); + expect(subShardTimeoutText(2, 4, { id: 'x:2.1', phase: 'parallel', files: ['test/a.test.ts'], status: 'timed-out', exitCode: null, elapsedMs: 1, failingFiles: [], unattributedFailures: 0 }, + [], 10_000, context)[1]).toContain('in flight: unknown'); + }); +}); diff --git a/test/test-free-shards.test.ts b/test/test-free-shards.test.ts index 620fbaf51..3d84740c3 100644 --- a/test/test-free-shards.test.ts +++ b/test/test-free-shards.test.ts @@ -196,7 +196,7 @@ describe('test-free-shards: exclusive host-state phase', () => { let child: ReturnType | undefined; let watchdog: ReturnType | undefined; try { - for (const file of ['scripts/test-free-shards.ts', 'scripts/test-strict-output.ts', 'scripts/lib/shard-engine.ts', 'scripts/lib/free-home-guard.ts', 'scripts/lib/free-ci-health.ts', 'scripts/lib/windows-curation.ts', 'lib/state-root.ts', + for (const file of ['scripts/test-free-shards.ts', 'scripts/test-strict-output.ts', 'scripts/lib/shard-engine.ts', 'scripts/lib/free-home-guard.ts', 'scripts/lib/free-ci-health.ts', 'scripts/lib/windows-curation.ts', 'scripts/lib/shard-jobs.ts', 'lib/state-root.ts', 'test/helpers/paid-test-set.ts', 'test/helpers/touchfiles.ts', 'test/helpers/touchfiles-data.ts', 'test/helpers/test-selection.ts']) { const target = path.join(directory, file); fs.mkdirSync(path.dirname(target), { recursive: true }); diff --git a/test/ubicloud-pregate-remote.test.ts b/test/ubicloud-pregate-remote.test.ts index 2263b617a..79afd1296 100644 --- a/test/ubicloud-pregate-remote.test.ts +++ b/test/ubicloud-pregate-remote.test.ts @@ -20,7 +20,7 @@ afterAll(() => fs.rmSync(SCRATCH, { recursive: true, force: true })); const COPIED = ['scripts/ubicloud/pregate-remote.sh', 'bin/gstack-wtree', 'bin/gstack-path-spelling.sh', 'scripts/test-free-shards.ts', 'scripts/test-strict-output.ts', 'scripts/lib/shard-engine.ts', 'scripts/lib/free-home-guard.ts', - 'scripts/lib/free-ci-health.ts', 'scripts/lib/windows-curation.ts', 'lib/state-root.ts', 'test/helpers/paid-test-set.ts', + 'scripts/lib/free-ci-health.ts', 'scripts/lib/windows-curation.ts', 'scripts/lib/shard-jobs.ts', 'lib/state-root.ts', 'test/helpers/paid-test-set.ts', 'test/helpers/touchfiles.ts', 'test/helpers/touchfiles-data.ts', 'test/helpers/test-selection.ts']; const TESTS: Record = { 'test/alpha.test.ts': "import { expect, test } from 'bun:test';\ntest('alpha', () => { expect(1).toBe(1); });\n",