mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-22 08:01:32 +01:00
Merge commit '69c14b17d57d5eef537dacbd533ee88865a0a00a' into capy/gsta240-flaky
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> # Conflicts: # test/helpers/restamp-fixture.ts
This commit is contained in:
commit
bc873cbb11
352 files changed
+14646
-1746
No files matched your search
@@ -259,7 +259,38 @@ export function readFlakeLedgerDir(dir: string): FlakeLedgerEntry[] {
|
||||
export const FLAKE_POLICY = { maxRate: 0.05, minRuns: 20 } as const;
|
||||
|
||||
export interface FlakyFile { file: string; flakyRuns: number; entries: number; rate: number; enforced: boolean; failing: boolean }
|
||||
export interface FlakeWindow { runs: number; flakyRuns: number; since: string; files: FlakyFile[]; enforceable: boolean }
|
||||
/** `skipped`: reused main pushes left out of the window (reuseChecker); `note`: runs that could not be checked. */
|
||||
export interface FlakeWindow { runs: number; flakyRuns: number; since: string; files: FlakyFile[]; enforceable: boolean; skipped?: number; note?: string }
|
||||
|
||||
/** The most job listings one reuseChecker spends. */
|
||||
export const REUSE_LOOKUPS_MAX = 40;
|
||||
|
||||
/**
|
||||
* Finds runs that reused their merged pull request's verified run (#3709): such
|
||||
* a run uploads no artifact and its `reuse` job succeeds, because every suite
|
||||
* job was skipped. It holds no flake evidence, so flake counts leave it out of
|
||||
* their runs and their denominator (#3725). A run with any artifact costs no
|
||||
* call. A run with none costs one job listing, at most REUSE_LOOKUPS_MAX per
|
||||
* checker; past the cap, or when the listing fails, the run counts as it did
|
||||
* before #3725 and note() says so.
|
||||
*/
|
||||
export function reuseChecker(client: GhClient, repo: string) {
|
||||
let lookups = 0;
|
||||
let unchecked = 0;
|
||||
let reason = '';
|
||||
return {
|
||||
reused(run: Pick<CiRun, 'id' | 'attempt'>, artifactNames: string[]): boolean {
|
||||
if (artifactNames.length) return false;
|
||||
if (lookups >= REUSE_LOOKUPS_MAX) { unchecked++; reason ||= `cap of ${REUSE_LOOKUPS_MAX} job listings reached`; return false; }
|
||||
lookups++;
|
||||
try {
|
||||
return listRunJobs(client, repo, run.id, run.attempt).some(job => job.name === 'reuse' && job.conclusion === 'success');
|
||||
} catch (error) { unchecked++; reason ||= unavailableReason(error); return false; }
|
||||
},
|
||||
note: (): string | undefined => (unchecked
|
||||
? `${unchecked} run(s) with no artifacts not checked for main-push reuse (${reason}); counted as before #3725` : undefined),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Rank files by the share of runs in which they needed the retry pass. A file
|
||||
@@ -286,9 +317,10 @@ export function rankFlakyFiles(runEntries: Map<number, FlakeLedgerEntry[]>, runC
|
||||
/**
|
||||
* Read the flake ledgers of main's completed free-tests runs: the runs created
|
||||
* in the last `sinceDays`, widened to the newest `minRuns` runs (within
|
||||
* `maxLookbackDays`) so the 5% rule has a window of at least 20 runs. A run
|
||||
* with no `flake-ledger-*` artifact had no flaky passes (the upload is skipped
|
||||
* when the ledger file does not exist).
|
||||
* `maxLookbackDays`) so the 5% rule has a window of at least 20 runs. Reused
|
||||
* main pushes (reuseChecker) are left out and older runs take their place. A
|
||||
* run with no `flake-ledger-*` artifact had no flaky passes (the upload is
|
||||
* skipped when the ledger file does not exist).
|
||||
*/
|
||||
export function readCiFlakeLedgers(opts: { client: GhClient; repo: string; sinceDays: number; now: number; cacheDir: string;
|
||||
minRuns?: number; maxLookbackDays?: number; workflow?: string }): FlakeWindow {
|
||||
@@ -297,16 +329,25 @@ export function readCiFlakeLedgers(opts: { client: GhClient; repo: string; since
|
||||
const all = listWorkflowRuns(opts.client, opts.repo, opts.workflow ?? 'free-tests.yml',
|
||||
{ since: iso(Math.max(opts.sinceDays, opts.maxLookbackDays ?? 60)), branch: 'main' })
|
||||
.filter(run => run.conclusion === 'success' || run.conclusion === 'failure');
|
||||
const recent = all.filter(run => run.createdAt >= iso(opts.sinceDays));
|
||||
const window = recent.length >= minRuns ? recent : all.slice(0, minRuns);
|
||||
const since = iso(opts.sinceDays);
|
||||
const recent = all.filter(run => run.createdAt >= since).length;
|
||||
const fetcher = historyFetcher(opts.client);
|
||||
const reuse = reuseChecker(opts.client, opts.repo);
|
||||
const window: CiRun[] = [];
|
||||
let skipped = 0;
|
||||
for (const [index, run] of all.entries()) {
|
||||
if (index >= recent && window.length >= minRuns) break;
|
||||
if (reuse.reused(run, fetcher.listArtifacts(opts.repo, run.id).map(artifact => artifact.name))) skipped++;
|
||||
else window.push(run);
|
||||
}
|
||||
const runEntries = new Map<number, FlakeLedgerEntry[]>();
|
||||
for (const run of window) {
|
||||
const dirs = downloadRunArtifacts({ repo: opts.repo, run, cacheDir: opts.cacheDir, fetcher, match: name => name.startsWith('flake-ledger-') });
|
||||
const entries = dirs.flatMap(readFlakeLedgerDir);
|
||||
if (entries.length) runEntries.set(run.id, entries);
|
||||
}
|
||||
return rankFlakyFiles(runEntries, window.length, window[window.length - 1]?.createdAt ?? iso(opts.sinceDays));
|
||||
const note = reuse.note();
|
||||
return { ...rankFlakyFiles(runEntries, window.length, window[window.length - 1]?.createdAt ?? since), skipped, ...(note ? { note } : {}) };
|
||||
}
|
||||
|
||||
/** Markdown table of one run's merged flake ledgers, for a job summary. */
|
||||
|
||||
@@ -0,0 +1,422 @@
|
||||
/**
|
||||
* Sub-shards inside one CI job: `--ci-run <plan> --shard i --result r --jobs k`
|
||||
* (GSTA-203 PR 2, principle "buy cores, not shards").
|
||||
*
|
||||
* A CI shard's files split into k disjoint, identity-bound sub-shards that run
|
||||
* side by side through the free runner's own process-shard path (one bun child
|
||||
* each, serial inside, own sandbox, group kill, log), never Bun's worker
|
||||
* `--parallel` (decision V3/D6 in scripts/test-free-shards.ts). The rules:
|
||||
*
|
||||
* - k = 1 (the default) is today's single child, unchanged.
|
||||
* - Files in TREE_MUTATING or SLOW_UNDER_LOAD run after the parallel
|
||||
* sub-shards settle, serially, in one trailing sub-shard.
|
||||
* - The parallel phase uses the concurrent-phase home guard; every child
|
||||
* uses the shared guard inside it, exactly as full-suite mode does.
|
||||
* - The whole CI shard keeps its k = 1 wall: the parallel sub-shards get it,
|
||||
* and the serial sub-shard gets what is left of it, so k never extends a
|
||||
* deadline; every launched child settles before the parent goes on.
|
||||
* - The parent merges every sub-shard result before the single retry pass,
|
||||
* so the five-file retry cap stays per CI shard; a missing, duplicate or
|
||||
* incomplete sub-shard result fails the shard.
|
||||
* - Sub-shard runs never write the duration seed (only `--record-durations`
|
||||
* does, from one-file children).
|
||||
*
|
||||
* Per-file test and expect() counts come from bun's JUnit reporter, which
|
||||
* attributes both per file inside a multi-file child on the pinned Bun 1.4.2
|
||||
* (the console stream is byte-identical with or without it). Counts are
|
||||
* evidence for k > 1 comparisons; they never decide a verdict.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import type { FreeCiPlan, FreeCiResult, FreeShardOutcome } from '../test-free-shards';
|
||||
import { guardFreeHome, sharedFreeHome, type FreeHomeGuard, type FreeHomeGuardFactory } from './free-home-guard';
|
||||
import { normalizeRelativePath } from './shard-engine';
|
||||
|
||||
/**
|
||||
* Files that pass alone but time out as a sub-shard under load. Each entry
|
||||
* names the CI run id where it timed out (FREE_SUBSHARD_TIMEOUT prints it).
|
||||
* They run in the trailing serial sub-shard under `--jobs`; at k = 1 nothing
|
||||
* changes. Keys are pinned against the live file census by
|
||||
* test/shard-jobs.test.ts. Never raise a timeout instead of adding an entry.
|
||||
*/
|
||||
export const SLOW_UNDER_LOAD: Record<string, { reason: string; runId: string }> = {};
|
||||
|
||||
export const FREE_JOBS_INVALID = 'FREE_JOBS_INVALID';
|
||||
export const FREE_SUBSHARD_TIMEOUT = 'FREE_SUBSHARD_TIMEOUT';
|
||||
export const FREE_SUBSHARD_REJECTED = 'FREE_SUBSHARD_REJECTED';
|
||||
const DOCS = 'docs/TESTING_INTERNALS.md#sub-shards-inside-one-ci-job';
|
||||
const JOBS_FIX = `--jobs applies only to --ci-run and defaults to 1; GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run. See ${DOCS}`;
|
||||
|
||||
/** `--jobs <k>` at parse time: digits only, at least 1. */
|
||||
export function parseJobsFlag(raw: string | undefined): number {
|
||||
if (raw === undefined || !/^\d+$/.test(raw) || Number(raw) < 1) {
|
||||
throw new Error(`${FREE_JOBS_INVALID}: --jobs needs a positive whole number; got ${JSON.stringify(raw ?? '')}. ${JOBS_FIX}`);
|
||||
}
|
||||
return Number(raw);
|
||||
}
|
||||
|
||||
export function rejectJobsWithoutCiRun(): never {
|
||||
throw new Error(`${FREE_JOBS_INVALID}: --jobs needs --ci-run. ${JOBS_FIX}`);
|
||||
}
|
||||
|
||||
/** `--jobs <k>` against the planned shard: k may not exceed its file count (an empty shard allows 1). */
|
||||
export function validateJobsForShard(jobs: number, fileCount: number, shard: number): void {
|
||||
const max = Math.max(1, fileCount);
|
||||
if (jobs > max) {
|
||||
throw new Error(`${FREE_JOBS_INVALID}: --jobs ${jobs} is more than CI shard ${shard}'s ${fileCount} file(s); use 1 to ${max}. ${JOBS_FIX}`);
|
||||
}
|
||||
}
|
||||
|
||||
export interface FileTestCounts { tests: number; pass: number; fail: number; skip: number; todo: number; expects: number }
|
||||
export interface ShardTestCounts {
|
||||
source: 'bun-junit';
|
||||
/** Every reported file was planned and the per-file tests add up to bun's own `Ran N tests` count. */
|
||||
complete: boolean;
|
||||
problem?: string;
|
||||
files: Record<string, FileTestCounts>;
|
||||
totals: FileTestCounts;
|
||||
}
|
||||
|
||||
const zeroCounts = (): FileTestCounts => ({ tests: 0, pass: 0, fail: 0, skip: 0, todo: 0, expects: 0 });
|
||||
const addCounts = (into: FileTestCounts, from: FileTestCounts) => {
|
||||
for (const key of Object.keys(into) as Array<keyof FileTestCounts>) into[key] += from[key];
|
||||
};
|
||||
const TAG = /<(\/?)(testsuites|testsuite|testcase|failure|error|skipped)\b((?:[^>"']|"[^"]*"|'[^']*')*?)(\/?)>/g;
|
||||
const ATTR = /([\w:-]+)="([^"]*)"/g;
|
||||
const ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" };
|
||||
const decode = (text: string) => text.replace(/&(#x[0-9a-f]+|#\d+|\w+);/gi, (whole, entity: string) => entity[0] === '#'
|
||||
? String.fromCodePoint(entity[1].toLowerCase() === 'x' ? parseInt(entity.slice(2), 16) : Number(entity.slice(1)))
|
||||
: ENTITIES[entity] ?? whole);
|
||||
const attributes = (text: string): Record<string, string> =>
|
||||
Object.fromEntries([...text.matchAll(ATTR)].map(([, key, value]) => [key, decode(value)]));
|
||||
|
||||
/**
|
||||
* Per-file counts from bun's JUnit XML. A testcase is a fail when it holds a
|
||||
* failure or error, a todo when skipped with message TODO, a skip when
|
||||
* skipped otherwise, else a pass; expects sum each testcase's assertions.
|
||||
* Printed paths are matched back to planned files. Bun prints them relative
|
||||
* to the child's real cwd, so a plain relative path must match exactly; an
|
||||
* absolute or ../-prefixed path matches only when what precedes the planned
|
||||
* file is the child's root. Files that never ran a test are absent.
|
||||
*/
|
||||
export function parseJunitCounts(xml: string | null, planned: readonly string[], testsRan: number | null, root = process.cwd()): ShardTestCounts {
|
||||
const files: Record<string, FileTestCounts> = {};
|
||||
const totals = zeroCounts();
|
||||
const result = (complete: boolean, problem?: string): ShardTestCounts =>
|
||||
({ source: 'bun-junit', complete, ...(problem ? { problem } : {}), files, totals });
|
||||
if (xml === null) return result(false, 'bun wrote no JUnit report');
|
||||
const plannedSet = new Set(planned);
|
||||
const unknown = new Set<string>();
|
||||
const roots = new Set([root, (() => { try { return fs.realpathSync(root); } catch { return root; } })()].map(normalizeRelativePath));
|
||||
const isRoot = (prefix: string) => [...roots].some(dir => dir === prefix || dir.endsWith(`/${prefix}`));
|
||||
const canonical = (printed: string) => {
|
||||
const normalized = normalizeRelativePath(printed);
|
||||
const stripped = normalized.replace(/^(?:\.{1,2}\/)+/, '');
|
||||
if (plannedSet.has(stripped)) return stripped;
|
||||
if (stripped === normalized && !/^(?:\/|[A-Za-z]:\/)/.test(stripped)) return stripped;
|
||||
const matches = planned.filter(file => stripped.endsWith(`/${file}`) && isRoot(stripped.slice(0, -file.length - 1)));
|
||||
return matches.length === 1 ? matches[0] : stripped;
|
||||
};
|
||||
let current: { file: string; state: 'pass' | 'fail' | 'skip' | 'todo'; expects: number } | null = null;
|
||||
const finish = () => {
|
||||
if (!current) return;
|
||||
const entry = files[current.file] ??= zeroCounts();
|
||||
entry.tests += 1;
|
||||
entry[current.state] += 1;
|
||||
entry.expects += current.expects;
|
||||
if (!plannedSet.has(current.file)) unknown.add(current.file);
|
||||
current = null;
|
||||
};
|
||||
for (const [, closing, name, body, selfClosing] of xml.matchAll(TAG)) {
|
||||
if (name === 'testcase') {
|
||||
if (closing) { finish(); continue; }
|
||||
const attrs = attributes(body);
|
||||
current = { file: canonical(attrs.file ?? ''), state: 'pass', expects: Number(attrs.assertions ?? 0) || 0 };
|
||||
if (selfClosing) finish();
|
||||
} else if (current && !closing && (name === 'failure' || name === 'error')) {
|
||||
current.state = 'fail';
|
||||
} else if (current && !closing && name === 'skipped' && current.state === 'pass') {
|
||||
current.state = attributes(body).message === 'TODO' ? 'todo' : 'skip';
|
||||
}
|
||||
}
|
||||
finish();
|
||||
for (const entry of Object.values(files)) addCounts(totals, entry);
|
||||
if (unknown.size) return result(false, `JUnit named unplanned file(s): ${[...unknown].sort().join(', ')}`);
|
||||
if (testsRan === null) return result(false, "bun's summary has no test count");
|
||||
if (totals.tests !== testsRan) return result(false, `JUnit counted ${totals.tests} tests, bun's summary ${testsRan}`);
|
||||
return result(true);
|
||||
}
|
||||
|
||||
const JUNIT_MAX_BYTES = 64 * 1024 * 1024;
|
||||
|
||||
/** The child-written report, only when it is a regular file of bounded size (never a link, FIFO or device). */
|
||||
export function readJunitReport(file: string): string | null {
|
||||
let fd: number;
|
||||
try { fd = fs.openSync(file, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0) | (fs.constants.O_NONBLOCK ?? 0)); } catch { return null; }
|
||||
try {
|
||||
const stat = fs.fstatSync(fd);
|
||||
return stat.isFile() && stat.size <= JUNIT_MAX_BYTES ? fs.readFileSync(fd, 'utf8') : null;
|
||||
} catch { return null; } finally { fs.closeSync(fd); }
|
||||
}
|
||||
|
||||
/** One shard's counts from its sub-shards' counts; complete only when every part is and no file repeats. */
|
||||
export function mergeTestCounts(parts: Array<ShardTestCounts | undefined>): ShardTestCounts | undefined {
|
||||
if (parts.every(part => !part)) return undefined;
|
||||
const files: Record<string, FileTestCounts> = {};
|
||||
const totals = zeroCounts();
|
||||
const problems: string[] = [];
|
||||
parts.forEach((part, index) => {
|
||||
if (!part) { problems.push(`sub-shard ${index + 1} has no counts`); return; }
|
||||
if (!part.complete) problems.push(`sub-shard ${index + 1}: ${part.problem ?? 'incomplete'}`);
|
||||
for (const [file, counts] of Object.entries(part.files)) {
|
||||
if (files[file]) problems.push(`${file} counted twice`);
|
||||
files[file] = { ...counts };
|
||||
}
|
||||
addCounts(totals, part.totals);
|
||||
});
|
||||
return { source: 'bun-junit', complete: problems.length === 0, ...(problems.length ? { problem: problems.join('; ') } : {}), files, totals };
|
||||
}
|
||||
|
||||
export interface SubShardAssignment {
|
||||
/** `<plan id prefix>:<shard>.<index>`: binds a result to one plan, shard and file set. */
|
||||
id: string;
|
||||
index: number;
|
||||
phase: 'parallel' | 'serial';
|
||||
files: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Split one CI shard into disjoint sub-shards: up to k parallel ones packed by
|
||||
* `pack`, then one serial sub-shard for the files `isSerial` names. Throws
|
||||
* unless the assignments cover the shard's files exactly once.
|
||||
*/
|
||||
export function assignSubShards(
|
||||
planId: string, shard: { shard: number; files: string[] }, jobs: number,
|
||||
isSerial: (file: string) => boolean, pack: (files: string[], count: number) => string[][],
|
||||
): SubShardAssignment[] {
|
||||
const serial = shard.files.filter(isSerial);
|
||||
const parallel = shard.files.filter(file => !isSerial(file));
|
||||
const groups = parallel.length ? pack(parallel, Math.min(jobs, parallel.length)).filter(files => files.length) : [];
|
||||
const assignments: SubShardAssignment[] = [
|
||||
...groups.map(files => ({ phase: 'parallel' as const, files })),
|
||||
...(serial.length ? [{ phase: 'serial' as const, files: serial }] : []),
|
||||
].map((entry, index) => ({ id: `${planId.slice(0, 12)}:${shard.shard}.${index + 1}`, index: index + 1, ...entry }));
|
||||
const assigned = assignments.flatMap(entry => entry.files);
|
||||
if (new Set(assigned).size !== assigned.length || assigned.length !== shard.files.length
|
||||
|| !shard.files.every(file => assigned.includes(file))) {
|
||||
throw new Error(`${FREE_SUBSHARD_REJECTED}: sub-shard assignment for CI shard ${shard.shard} does not cover its files exactly once`);
|
||||
}
|
||||
return assignments;
|
||||
}
|
||||
|
||||
export type SubShardSettlement = { assignment: SubShardAssignment } & ({ outcome: FreeShardOutcome } | { error: string });
|
||||
|
||||
/** Why one sub-shard's result cannot stand for its assignment; empty when it can. */
|
||||
export function subShardProblems(settled: SubShardSettlement): string[] {
|
||||
const { assignment } = settled;
|
||||
if (!('outcome' in settled)) return [`${assignment.id} has no result (${settled.error})`];
|
||||
const { outcome } = settled;
|
||||
const problems: string[] = [];
|
||||
if (JSON.stringify(outcome.files) !== JSON.stringify(assignment.files)) problems.push(`${assignment.id} reported another file set`);
|
||||
const summary = outcome.summary;
|
||||
if (!summary || (summary.sawTerminalSummary && summary.filesRan !== assignment.files.length)) {
|
||||
problems.push(`${assignment.id} reported ${summary?.filesRan ?? 'no'} executed file(s) for ${assignment.files.length} assigned`);
|
||||
} else if (outcome.status === 'passed' && (summary.filesRan !== assignment.files.length || summary.sawTerminalSummary !== true
|
||||
|| outcome.exitCode !== 0 || outcome.failingFiles.length || outcome.unattributedFailures)) {
|
||||
problems.push(`${assignment.id} passed without a complete execution summary`);
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
export interface SubShardRecord {
|
||||
id: string;
|
||||
phase: SubShardAssignment['phase'];
|
||||
files: string[];
|
||||
status: FreeShardOutcome['status'] | 'missing';
|
||||
exitCode: number | null;
|
||||
elapsedMs: number;
|
||||
summary?: FreeShardOutcome['summary'];
|
||||
failingFiles: string[];
|
||||
unattributedFailures: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* One CI shard outcome from every sub-shard: the planned file order, summed
|
||||
* summaries, the union of failing files, and the worst status. Every rejected
|
||||
* sub-shard adds unattributed evidence, which also vetoes the retry pass.
|
||||
*/
|
||||
export function mergeSubShardOutcomes(
|
||||
shard: { shard: number; files: string[] }, settled: SubShardSettlement[], elapsedMs: number, extraProblems: string[] = [],
|
||||
): { outcome: FreeShardOutcome; records: SubShardRecord[]; problems: string[] } {
|
||||
const problems = [...extraProblems];
|
||||
const assigned = settled.flatMap(entry => entry.assignment.files);
|
||||
if (new Set(assigned).size !== assigned.length) problems.push('a file is assigned to more than one sub-shard');
|
||||
if (assigned.length !== shard.files.length || !shard.files.every(file => assigned.includes(file))) problems.push('sub-shards do not cover the CI shard');
|
||||
for (const entry of settled) problems.push(...subShardProblems(entry));
|
||||
const outcomes = settled.flatMap(entry => ('outcome' in entry ? [entry.outcome] : []));
|
||||
const missing = settled.length - outcomes.length;
|
||||
const status: FreeShardOutcome['status'] = outcomes.some(outcome => outcome.status === 'timed-out') ? 'timed-out'
|
||||
: problems.length || outcomes.some(outcome => outcome.status !== 'passed') ? 'failed' : 'passed';
|
||||
const testsRan = outcomes.map(outcome => outcome.summary?.testsRan ?? null);
|
||||
const failures = outcomes.flatMap(outcome => outcome.failures ?? []);
|
||||
const inFlight = outcomes.flatMap(outcome => outcome.inFlight ?? []);
|
||||
const counts = mergeTestCounts(settled.map(entry => ('outcome' in entry ? entry.outcome.counts : undefined)));
|
||||
const outcome: FreeShardOutcome = {
|
||||
shard: shard.shard,
|
||||
files: shard.files,
|
||||
status,
|
||||
exitCode: status === 'passed' ? 0 : outcomes.find(entry => entry.exitCode)?.exitCode ?? 1,
|
||||
elapsedMs,
|
||||
groupPid: null,
|
||||
failingFiles: status === 'passed' ? [] : [...new Set(outcomes.flatMap(entry => entry.failingFiles))],
|
||||
unattributedFailures: status === 'passed' ? 0 : outcomes.reduce((sum, entry) => sum + entry.unattributedFailures, 0) + problems.length,
|
||||
...(failures.length ? { failures } : {}),
|
||||
summary: {
|
||||
testsRan: missing || testsRan.some(value => value === null) ? null : testsRan.reduce<number>((sum, value) => sum + value!, 0),
|
||||
filesRan: outcomes.reduce((sum, entry) => sum + (entry.summary?.filesRan ?? 0), 0),
|
||||
sawTerminalSummary: !missing && outcomes.every(entry => entry.files.length === 0 || entry.summary?.sawTerminalSummary === true),
|
||||
},
|
||||
...(counts ? { counts } : {}),
|
||||
...(inFlight.length ? { inFlight } : {}),
|
||||
};
|
||||
const records = settled.map((entry): SubShardRecord => {
|
||||
const { id, phase, files } = entry.assignment;
|
||||
if (!('outcome' in entry)) return { id, phase, files, status: 'missing', exitCode: null, elapsedMs: 0, failingFiles: [], unattributedFailures: 1 };
|
||||
const { status: subStatus, exitCode, elapsedMs: subElapsed, summary, failingFiles, unattributedFailures } = entry.outcome;
|
||||
return { id, phase, files, status: subStatus, exitCode, elapsedMs: subElapsed, summary, failingFiles, unattributedFailures };
|
||||
});
|
||||
return { outcome, records, problems };
|
||||
}
|
||||
|
||||
/** `--ci-verify` re-checks a k > 1 result's sub-shard records; empty means they stand. */
|
||||
export function subShardRecordProblems(result: FreeCiResult): string[] {
|
||||
if (result.jobs === undefined && result.subShards === undefined) return [];
|
||||
const records = result.subShards ?? [];
|
||||
if (!Number.isInteger(result.jobs) || result.jobs! < 2 || !records.length) return ['sub-shard result without a valid --jobs and records'];
|
||||
const files = records.flatMap(record => record.files);
|
||||
const problems: string[] = [];
|
||||
if (new Set(files).size !== files.length || files.length !== result.outcome.files.length
|
||||
|| !result.outcome.files.every(file => files.includes(file))) problems.push('sub-shard records do not cover the shard exactly once');
|
||||
const prefix = `${result.planId.slice(0, 12)}:${result.outcome.shard}.`;
|
||||
if (records.some((record, index) => record.id !== `${prefix}${index + 1}`)
|
||||
|| records.filter(record => record.phase === 'parallel').length > result.jobs!
|
||||
|| records.some((record, index) => record.phase === 'serial' ? index !== records.length - 1 : record.phase !== 'parallel')) {
|
||||
problems.push('sub-shard ids or phases do not match the plan and --jobs');
|
||||
}
|
||||
if (records.some(record => record.status === 'missing' || record.summary?.filesRan !== record.files.length
|
||||
|| (record.files.length > 0 && record.summary?.sawTerminalSummary !== true))) problems.push('a sub-shard did not run every assigned file');
|
||||
if (result.outcome.status === 'passed' && records.some(record => record.status !== 'passed')) problems.push('a passing shard holds a failed sub-shard');
|
||||
return problems;
|
||||
}
|
||||
|
||||
export interface CiRunContext { planPath: string; runId: string; runner: string }
|
||||
|
||||
/** Where this run is: the CI run id (with attempt) and runner, else `local` and the host. */
|
||||
export function ciRunContext(planPath: string, env: NodeJS.ProcessEnv = process.env): CiRunContext {
|
||||
const runId = env.GITHUB_RUN_ID ? `${env.GITHUB_RUN_ID}${env.GITHUB_RUN_ATTEMPT ? `/attempt ${env.GITHUB_RUN_ATTEMPT}` : ''}` : 'local';
|
||||
const cpus = os.availableParallelism?.() ?? os.cpus().length;
|
||||
return { planPath, runId, runner: `${env.RUNNER_NAME || os.hostname()} (${process.platform}, ${cpus} CPUs)` };
|
||||
}
|
||||
|
||||
const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
|
||||
|
||||
/** The FREE_SUBSHARD_TIMEOUT text: the file, k, runner, a serial reproduce command and the fix. */
|
||||
export function subShardTimeoutText(
|
||||
shard: number, jobs: number, record: SubShardRecord, inFlight: string[], wallTimeoutMs: number, context: CiRunContext, shardWallMs = wallTimeoutMs,
|
||||
): string[] {
|
||||
const suspects = inFlight.filter(file => record.files.includes(file));
|
||||
const files = suspects.length ? suspects : record.files;
|
||||
const seconds = (ms: number) => Math.round(ms / 1000);
|
||||
const fix = record.phase !== 'serial'
|
||||
? `if it passes serially, add it to SLOW_UNDER_LOAD in scripts/lib/shard-jobs.ts with run id ${context.runId}; never raise the timeout`
|
||||
: wallTimeoutMs < shardWallMs
|
||||
? `the parallel sub-shards left this serial sub-shard ${seconds(wallTimeoutMs)}s of the ${seconds(shardWallMs)}s shard wall; if the k=1 command passes, the parallel phase used up the budget and this shard needs a lower --jobs; never raise the timeout`
|
||||
: 'this file already runs alone after the parallel sub-shards; fix or split it, never raise the timeout';
|
||||
return [
|
||||
`[test:free] ${FREE_SUBSHARD_TIMEOUT}: CI shard ${shard} sub-shard ${record.id} (${record.files.length} files) hit its ${seconds(wallTimeoutMs)}s wall at --jobs ${jobs} on ${context.runner}, run ${context.runId}.`,
|
||||
`[test:free] in flight: ${suspects.length ? suspects.join(', ') : 'unknown (no open file chunk; see the sub-shard log)'}`,
|
||||
`[test:free] reproduce serially: printf '%s\\n' ${files.map(quote).join(' ')} > files.txt && GSTACK_FREE_JOBS=1 bun run scripts/test-free-shards.ts --files-from files.txt`,
|
||||
`[test:free] or the whole CI shard at k=1: bun run scripts/test-free-shards.ts --ci-run ${quote(context.planPath)} --shard ${shard} --result shard-${shard}.json --jobs 1`,
|
||||
`[test:free] ${fix}. See ${DOCS}`,
|
||||
];
|
||||
}
|
||||
|
||||
export interface RunShardOptions { wallTimeoutMs: number; verbose: boolean; testCounts: true; homeGuard?: FreeHomeGuardFactory }
|
||||
|
||||
export interface CiShardInput {
|
||||
plan: FreeCiPlan;
|
||||
shard: FreeCiPlan['shards'][number];
|
||||
revision: string;
|
||||
/** Sub-shard count; 1 is today's single child. */
|
||||
jobs: number;
|
||||
wallTimeoutMs: number;
|
||||
verbose: boolean;
|
||||
context: CiRunContext;
|
||||
}
|
||||
|
||||
export interface CiShardDeps {
|
||||
runShard(files: string[], shardNumber: number, totalShards: number, options: RunShardOptions): Promise<FreeShardOutcome>;
|
||||
/** The free runner's single flaky-retry pass over every collected outcome. */
|
||||
retry(outcomes: FreeShardOutcome[]): Promise<{ exitCode: number; retry: FreeShardOutcome | null }>;
|
||||
pack(files: string[], count: number): string[][];
|
||||
isSerial(file: string): boolean;
|
||||
terminationRequested(): boolean;
|
||||
phaseGuard?(files: string[], shards: number): FreeHomeGuard;
|
||||
log?(line: string): void;
|
||||
}
|
||||
|
||||
/** Run one planned CI shard at k sub-shards and build its result JSON. */
|
||||
export async function runCiShard(input: CiShardInput, deps: CiShardDeps): Promise<{ result: FreeCiResult; exitCode: number }> {
|
||||
const { plan, shard, revision, jobs, wallTimeoutMs, verbose } = input;
|
||||
const log = deps.log ?? ((line: string) => console.log(line));
|
||||
if (jobs === 1) {
|
||||
const outcome = await deps.runShard(shard.files, shard.shard, plan.shards.length, { wallTimeoutMs, verbose, testCounts: true });
|
||||
const retried = await deps.retry([outcome]);
|
||||
return { result: { planId: plan.id, revision, outcome, retry: retried.retry }, exitCode: retried.exitCode };
|
||||
}
|
||||
const startedAt = Date.now();
|
||||
const assignments = assignSubShards(plan.id, shard, jobs, deps.isSerial, deps.pack);
|
||||
const total = assignments.length;
|
||||
log(`[test:free] CI shard ${shard.shard}/${plan.shards.length} at --jobs ${jobs}: ${assignments
|
||||
.map(entry => `sub-shard ${entry.id} = runner shard ${entry.index}/${total} (${entry.phase}, ${entry.files.length} files)`).join('; ')}`);
|
||||
const walls = new Map<string, number>();
|
||||
const run = (entry: SubShardAssignment, homeGuard?: FreeHomeGuardFactory, wall = wallTimeoutMs): Promise<SubShardSettlement> => {
|
||||
walls.set(entry.id, wall);
|
||||
return deps.runShard(entry.files, entry.index, total, { wallTimeoutMs: wall, verbose, testCounts: true, ...(homeGuard ? { homeGuard } : {}) })
|
||||
.then(outcome => ({ assignment: entry, outcome }), (error: unknown) => ({ assignment: entry, error: error instanceof Error ? error.message : String(error) }));
|
||||
};
|
||||
const parallel = assignments.filter(entry => entry.phase === 'parallel');
|
||||
const serial = assignments.filter(entry => entry.phase === 'serial');
|
||||
const phaseProblems: string[] = [];
|
||||
const settled: SubShardSettlement[] = [];
|
||||
if (parallel.length) {
|
||||
const guard = deps.phaseGuard ?? ((files: string[], shards: number) => guardFreeHome(files, process.env, { kind: 'concurrent', shards }));
|
||||
const phaseHome = guard(parallel.flatMap(entry => entry.files), parallel.length);
|
||||
settled.push(...await Promise.all(parallel.map(entry => run(entry, sharedFreeHome))));
|
||||
const homeChange = phaseHome.verify();
|
||||
if (homeChange) phaseProblems.push(homeChange);
|
||||
}
|
||||
for (const entry of serial) {
|
||||
const left = startedAt + wallTimeoutMs - Date.now();
|
||||
settled.push(deps.terminationRequested() ? { assignment: entry, error: 'cancelled before the serial phase' }
|
||||
: left <= 0 ? { assignment: entry, error: 'no wall left after the parallel phase' } : await run(entry, undefined, left));
|
||||
}
|
||||
const merged = mergeSubShardOutcomes(shard, settled, Date.now() - startedAt, phaseProblems);
|
||||
for (const problem of merged.problems) console.error(`[test:free] ${FREE_SUBSHARD_REJECTED}: CI shard ${shard.shard}: ${problem}`);
|
||||
for (const record of merged.records.filter(entry => entry.status === 'timed-out')) {
|
||||
for (const line of subShardTimeoutText(shard.shard, jobs, record, merged.outcome.inFlight ?? [], walls.get(record.id) ?? wallTimeoutMs, input.context, wallTimeoutMs)) console.error(line);
|
||||
}
|
||||
log(`[test:free] CI shard ${shard.shard}: ${merged.outcome.status} across ${total} sub-shards in ${Math.round(merged.outcome.elapsedMs / 1000)}s`);
|
||||
const retried = await deps.retry([merged.outcome]);
|
||||
return {
|
||||
result: { planId: plan.id, revision, outcome: merged.outcome, retry: retried.retry, jobs, subShards: merged.records },
|
||||
exitCode: retried.exitCode,
|
||||
};
|
||||
}
|
||||
|
||||
/** The duration seed takes only one-file children's walls, never a sub-shard's or a shard's. */
|
||||
export function assertSoloSeedSample(file: string, outcome: Pick<FreeShardOutcome, 'files' | 'summary'>): void {
|
||||
if (outcome.files.length !== 1 || outcome.files[0] !== file || (outcome.summary?.filesRan ?? 0) > 1) {
|
||||
throw new Error(`Duration seed sample for ${file} came from a ${outcome.files.length}-file child; seeds are recorded only from one-file serial children.`);
|
||||
}
|
||||
}
|
||||
@@ -315,6 +315,10 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }>
|
||||
file: 'test/ship-queue-train-land.test.ts',
|
||||
reason: 'drives train land against test/helpers/ship-queue-train-fixture.ts plus an extensionless node gh wrapper on a colon-joined PATH; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead, as for test/ship-queue-train.test.ts and test/ship-queue-land.test.ts (windows-free-shard 6, run 40824965988)',
|
||||
},
|
||||
{
|
||||
file: 'test/ship-queue-land-lock.test.ts',
|
||||
reason: 'drives land and train land against test/helpers/ship-queue-train-fixture.ts plus an extensionless node gh wrapper on a colon-joined PATH; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead, as for test/ship-queue-train-land.test.ts and test/ship-queue-land.test.ts (windows-free-shard 6, run 40824965988)',
|
||||
},
|
||||
{
|
||||
file: 'test/ship-queue-train.test.ts',
|
||||
reason: 'drives bin/gstack-ship-queue against the stateful #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead of the shim (windows-free-shard run 39152702056)',
|
||||
@@ -402,10 +406,14 @@ export const KNOWN_WINDOWS_SAFE: Array<{ file: string; reason: string }> = [
|
||||
{ file: 'test/cso-windows-docker.test.ts', reason: '/cso docker.exe discovery under known-folder roots (real path, no reparse points) and the native-transport-unsupported outcome; pure win32 path logic' },
|
||||
{ file: 'test/copilot-windows-bash.test.ts', reason: 'the Copilot glossary line that runs bash blocks in Git for Windows Bash; reads host config only' },
|
||||
{ file: 'test/ship-hook-windows-paths.test.ts', reason: 'runs bin/ helpers through explicit bash and Bun argv with forward-slash paths; never executes a shebang; the path-spelling simulation is skipIf win32' },
|
||||
{ file: 'test/review-scope-windows-launch.test.ts', reason: '#3541: runs gstack-review-scope through lib/script-invocation.ts, which must reach the bash gstack-review-read through Git Bash (red at main: windows-free-shard 3, run 41228091417); os.tmpdir state only' },
|
||||
{ file: 'test/jsonl-merge-encoding.test.ts', reason: '#3659: runs gstack-jsonl-merge through lib/script-invocation.ts (Git Bash on win32); the CRLF half of the bug only reproduces with a Windows Python' },
|
||||
{ file: 'test/gstack-wtree-windows-paths.test.ts', reason: '#3101: runs bin/gstack-wtree through explicit bash in an os.tmpdir linked worktree with forward-slash paths; never executes a shebang; the drive-letter simulation is skipIf win32' },
|
||||
{ file: 'test/setup-hook-syntax-gate.test.ts', reason: '#3088/#3100: runs bin/gstack-hook-check and the setup hook sections through explicit bash with forward-slash paths; never executes a shebang. Its Bun drive-path spelling is the Windows bug it guards' },
|
||||
{ file: 'test/bun-config-shell-bins.test.ts', reason: '#3415: runs the record bins through lib/script-invocation.ts (Git Bash on Windows) with MSYS path conversion off, the case it guards; its bun and cygpath stand-ins and simulated-OSTYPE cases are skipIf win32' },
|
||||
{ file: 'test/ship-receipt-session-kind.test.ts', reason: '#3541: runs gstack-ship-receipt write through lib/script-invocation.ts, which must reach the bash gstack-session-kind through Git Bash; os.tmpdir git fixture only' },
|
||||
{ file: 'test/path-spelling.test.ts', reason: 'sources bin/gstack-path-spelling.sh through explicit bash and checks lib/path-spelling.ts; on Windows it exercises the real Git Bash spellings' },
|
||||
{ file: 'test/timeline-stop-hook-stdin.test.ts', reason: '#3628: the hook runs through Bun argv with a stdin pipe held open; the return-within-budget timing must meet real Windows process startup' },
|
||||
{ file: 'test/state-root-parity.test.ts', reason: 'runs the bash twin and lib/state-root.ts over an env table with PATH empty; no shebang execution, raw-string comparison is platform-neutral' },
|
||||
{ file: 'test/generator-eexist.test.ts',
|
||||
reason: 'E4: runs the generators through Bun argv with the Windows EEXIST emulation preload; no shebang execution' },
|
||||
|
||||
Reference in new issue
Block a user