v1.93.0.0 stamp: merge main 1a7f2578b7 into #3858 (land)

This commit is contained in:
garrytan committed 2026-11-02 14:39:46 +00:00
commit 2b8bbff7a7
57 files changed
+1409 -206

No files matched your search

+6 -3
View File
@@ -5,8 +5,10 @@
* the real home before and after each free shard, and names the shard's files
* when one of those entries changed. It watches only these surfaces; it is not
* a universal write-containment check. Live agent-session logs that a
* concurrently running host keeps writing are excluded, and directory mtimes
* are ignored, so only file, link and directory-set changes count.
* concurrently running host keeps writing are excluded, as is the one lane log
* `gstack-evidence run` tees this run's own output into (its exact path, from
* GSTACK_EVIDENCE_LOG). Directory mtimes are ignored, so only file, link and
* directory-set changes count.
*
* A shard that runs alone owns its window, so its guard names its files.
* Concurrent shards share one HOME and cannot tell whose write a change was, so
@@ -104,6 +106,7 @@ export type FreeHomeGuardFactory = (files: string[], env: NodeJS.ProcessEnv, sta
/** Take the baseline now; verify() returns null or the failure to report. */
export function guardFreeHome(files: string[], env: NodeJS.ProcessEnv = process.env, scope: FreeHomeScope = { kind: 'shard' }): FreeHomeGuard {
const home = freeHomeDir(env);
const ownLog = env.GSTACK_EVIDENCE_LOG && path.isAbsolute(env.GSTACK_EVIDENCE_LOG) ? path.relative(home, env.GSTACK_EVIDENCE_LOG) : null;
const unreadable = (error: unknown) => `real home surfaces could not be snapshotted (${(error as NodeJS.ErrnoException).code ?? 'error'}); shard files: ${files.join(', ')}`;
let before: FreeHomeSnapshot | undefined;
let baselineError: string | undefined;
@@ -112,7 +115,7 @@ export function guardFreeHome(files: string[], env: NodeJS.ProcessEnv = process.
verify() {
if (!before) return baselineError!;
try {
const changed = diffFreeHome(before, snapshotFreeHome(home));
const changed = diffFreeHome(before, snapshotFreeHome(home)).filter(entry => entry !== ownLog);
return changed.length ? formatFreeHomeChange(changed, files, scope) : null;
} catch (error) { return unreadable(error); }
},
+27 -4
View File
@@ -45,16 +45,39 @@ export const FREE_JOBS_INVALID = 'FREE_JOBS_INVALID';
export const FREE_SUBSHARD_TIMEOUT = 'FREE_SUBSHARD_TIMEOUT';
export const FREE_SUBSHARD_REJECTED = 'FREE_SUBSHARD_REJECTED';
const DOCS = 'docs/TESTING_INTERNALS.md#sub-shards-inside-one-ci-job';
const JOBS_FIX = `--jobs applies only to --ci-run and defaults to 1; GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run. See ${DOCS}`;
const JOBS_FIX = `--jobs applies only to --ci-run and defaults to the plan's per-shard k (1 when the plan has none); --shard-jobs applies only to --ci-plan; GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run. See ${DOCS}`;
/** `--jobs <k>` at parse time: digits only, at least 1. */
export function parseJobsFlag(raw: string | undefined): number {
/** `--jobs <k>` or `--shard-jobs <k>` at parse time: digits only, at least 1. */
export function parseJobsFlag(raw: string | undefined, flag = '--jobs'): number {
if (raw === undefined || !/^\d+$/.test(raw) || Number(raw) < 1) {
throw new Error(`${FREE_JOBS_INVALID}: --jobs needs a positive whole number; got ${JSON.stringify(raw ?? '')}. ${JOBS_FIX}`);
throw new Error(`${FREE_JOBS_INVALID}: ${flag} needs a positive whole number; got ${JSON.stringify(raw ?? '')}. ${JOBS_FIX}`);
}
return Number(raw);
}
export function rejectShardJobsWithoutCiPlan(): never {
throw new Error(`${FREE_JOBS_INVALID}: --shard-jobs needs --ci-plan. ${JOBS_FIX}`);
}
/**
* One planned shard's k under `--ci-plan --shard-jobs k`: at most one parallel
* sub-shard per file that may run in parallel, so a shard of only serial files
* (the exclusive TREE_MUTATING shard) keeps k = 1. Undefined means k = 1, and
* the plan then carries no `jobs` key, so its id matches a plan without k.
*/
export function plannedShardJobs(files: readonly string[], shardJobs: number, isSerial: (file: string) => boolean): number | undefined {
const jobs = Math.min(shardJobs, files.filter(file => !isSerial(file)).length);
return jobs > 1 ? jobs : undefined;
}
/** Why a plan's per-shard k cannot stand; null when absent or valid. */
export function plannedShardJobsProblem(shard: { shard: number; files: readonly string[]; jobs?: unknown }, isSerial: (file: string) => boolean): string | null {
if (shard.jobs === undefined) return null;
const parallel = shard.files.filter(file => !isSerial(file)).length;
return Number.isInteger(shard.jobs) && (shard.jobs as number) >= 2 && (shard.jobs as number) <= parallel ? null
: `CI shard ${shard.shard} plans --jobs ${JSON.stringify(shard.jobs)} for ${parallel} parallel file(s)`;
}
export function rejectJobsWithoutCiRun(): never {
throw new Error(`${FREE_JOBS_INVALID}: --jobs needs --ci-run. ${JOBS_FIX}`);
}
+37 -15
View File
@@ -77,11 +77,15 @@
* bun run scripts/test-free-shards.ts --attribute-home # name real-home writers: each file alone, private HOME
* bun run scripts/test-free-shards.ts --ci-plan plan.json --shards 20
* bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json
* bun run scripts/test-free-shards.ts --ci-plan plan.json --shards 20 --shard-jobs 2
* # the plan stores a per-shard k: min(k, the shard's files that may run in parallel),
* # kept only when above 1, so the exclusive TREE_MUTATING shard stays one child.
* bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json --jobs 4
* # k sub-shards side by side inside one CI job (default 1 = one child, today's behavior;
* # scripts/lib/shard-jobs.ts). --jobs applies only to --ci-run: a positive whole number up
* # to the shard's file count, else FREE_JOBS_INVALID. GSTACK_FREE_JOBS sets the local
* # full-suite process count and never changes --ci-run; --jobs is its only knob.
* # k sub-shards side by side inside one CI job (default: the plan's per-shard k, else 1 =
* # one child, today's behavior; scripts/lib/shard-jobs.ts). --jobs applies only to
* # --ci-run and overrides the plan: a positive whole number up to the shard's file count,
* # else FREE_JOBS_INVALID. --ci-verify requires every result to have run at the plan's k.
* # GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run.
* bun run scripts/test-free-shards.ts --ci-verify plan.json --results results/
*/
@@ -97,8 +101,8 @@ import { attributeFreeHomeWriters, guardFreeHome, sharedFreeHome, type FreeHomeG
import { appendStepSummary, ciHealthSummary, SEED_REFRESH_COMMAND, unseededWarning, windowsCurationLine } from './lib/free-ci-health';
import { curateWindowsSafe, type CurationResult } from './lib/windows-curation';
import {
assertSoloSeedSample, ciRunContext, parseJobsFlag, parseJunitCounts, readJunitReport, rejectJobsWithoutCiRun, runCiShard, SLOW_UNDER_LOAD, subShardRecordProblems,
validateJobsForShard, type ShardTestCounts, type SubShardRecord,
assertSoloSeedSample, ciRunContext, parseJobsFlag, parseJunitCounts, plannedShardJobs, plannedShardJobsProblem, readJunitReport, rejectJobsWithoutCiRun,
rejectShardJobsWithoutCiPlan, runCiShard, SLOW_UNDER_LOAD, subShardRecordProblems, validateJobsForShard, type ShardTestCounts, type SubShardRecord,
} from './lib/shard-jobs';
export { curateWindowsSafe, detectWindowsFragility, KNOWN_WINDOWS_INCOMPATIBLE, type CurationResult } from './lib/windows-curation';
import {
@@ -424,19 +428,27 @@ export interface FreeCiPlan {
version: 1;
revision: string;
id: string;
shards: Array<{ shard: number; files: string[]; predictedMs: number }>;
/** `jobs` is the shard's k from --shard-jobs, present only when above 1. */
shards: Array<{ shard: number; files: string[]; predictedMs: number; jobs?: number }>;
}
/** Files that run in the trailing serial sub-shard under k > 1, never beside another child. */
const runsSerialUnderJobs = (file: string) => file in TREE_MUTATING || file in SLOW_UNDER_LOAD;
const planDigest = (plan: Omit<FreeCiPlan, 'id'>): string =>
createHash('sha256').update(JSON.stringify(plan)).digest('hex');
/** One immutable plan is shared by isolated CI machines; never repack per job. */
export function createFreeCiPlan(files: string[], count: number, durations: Record<string, number>, revision: string): FreeCiPlan {
export function createFreeCiPlan(files: string[], count: number, durations: Record<string, number>, revision: string, shardJobs = 1): FreeCiPlan {
const readers = files.filter(file => !(file in TREE_MUTATING));
const exclusive = files.filter(file => file in TREE_MUTATING).sort();
const packed = packShardsByDuration(readers, count, durations);
const shards = packed.shards.map((files, index) => ({ shard: index + 1, files, predictedMs: packed.predictedMs[index] }));
if (exclusive.length) shards.push({ shard: shards.length + 1, files: exclusive, predictedMs: exclusive.reduce((ms, file) => ms + (durations[file] ?? 0), 0) });
const planned = packed.shards.map((files, index) => ({ files, predictedMs: packed.predictedMs[index] }));
if (exclusive.length) planned.push({ files: exclusive, predictedMs: exclusive.reduce((ms, file) => ms + (durations[file] ?? 0), 0) });
const shards = planned.map(({ files, predictedMs }, index) => {
const jobs = plannedShardJobs(files, shardJobs, runsSerialUnderJobs);
return { shard: index + 1, files, predictedMs, ...(jobs ? { jobs } : {}) };
});
const body = { version: 1 as const, revision, shards };
return { ...body, id: planDigest(body) };
}
@@ -447,6 +459,8 @@ export function validateFreeCiPlan(plan: FreeCiPlan, files: string[], revision:
|| id !== planDigest({ version, revision: plan.revision, shards })) throw new Error('CI plan identity or revision mismatch');
if (shards.some((shard, index) => shard.shard !== index + 1 || !Array.isArray(shard.files)
|| !Number.isFinite(shard.predictedMs) || shard.predictedMs < 0)) throw new Error('Invalid CI shard plan');
const jobsProblems = shards.map(shard => plannedShardJobsProblem(shard, runsSerialUnderJobs)).filter(Boolean);
if (jobsProblems.length) throw new Error(`Invalid CI shard plan: ${jobsProblems.join('; ')}`);
const planned = shards.flatMap(shard => shard.files).sort();
if (new Set(planned).size !== planned.length || JSON.stringify(planned) !== JSON.stringify([...files].sort())) throw new Error('CI plan must cover every free file exactly once');
}
@@ -486,6 +500,7 @@ export function verifyFreeCiResults(plan: FreeCiPlan, results: FreeCiResult[]):
if (!hasCompleteCiSummary(outcome)) throw new Error('Missing or incomplete CI execution summary');
const subShardProblems = subShardRecordProblems(result);
if (subShardProblems.length) throw new Error(`CI shard ${outcome.shard} sub-shards rejected: ${subShardProblems.join('; ')}`);
if ((result.jobs ?? 1) !== (shard.jobs ?? 1)) throw new Error(`CI shard ${outcome.shard} ran at --jobs ${result.jobs ?? 1}; the plan says ${shard.jobs ?? 1}`);
if (outcome.status === 'passed') {
if (outcome.exitCode !== 0 || outcome.failingFiles.length || outcome.unattributedFailures || result.retry) throw new Error('Inconsistent passing CI result');
} else {
@@ -556,8 +571,10 @@ type CliOptions = {
result: string | null;
results: string | null;
filesFrom: string | null;
/** Set only by --jobs (with --ci-run); absent means k = 1. */
/** Set only by --jobs (with --ci-run); absent means the plan's per-shard k. */
jobs?: number;
/** Set only by --shard-jobs (with --ci-plan); absent means k = 1 in every shard. */
shardJobs?: number;
};
export function parseCliOptions(argv: string[]): CliOptions {
@@ -573,6 +590,7 @@ export function parseCliOptions(argv: string[]): CliOptions {
let quick = false;
let attributeHome = false;
let jobs: number | undefined;
let shardJobs: number | undefined;
const paths: Record<'ciPlan' | 'ciRun' | 'ciVerify' | 'result' | 'results' | 'filesFrom', string | null> = {
ciPlan: null, ciRun: null, ciVerify: null, result: null, results: null, filesFrom: null,
};
@@ -597,6 +615,7 @@ export function parseCliOptions(argv: string[]): CliOptions {
'--results': pathFlag('--results', 'results'),
'--files-from': pathFlag('--files-from', 'filesFrom'),
'--jobs': (next) => { jobs = parseJobsFlag(next()); },
'--shard-jobs': (next) => { shardJobs = parseJobsFlag(next(), '--shard-jobs'); },
'--shards': (next) => {
const value = next();
if (!value) throw new Error('Missing value for --shards');
@@ -621,8 +640,9 @@ export function parseCliOptions(argv: string[]): CliOptions {
if (paths.ciRun && (shardIndex === null || !paths.result)) throw new Error('--ci-run requires --shard and --result');
if (paths.ciVerify && !paths.results) throw new Error('--ci-verify requires --results');
if (jobs !== undefined && !paths.ciRun) rejectJobsWithoutCiRun();
if (shardJobs !== undefined && !paths.ciPlan) rejectShardJobsWithoutCiPlan();
if (quick && (recordDurations || windowsOnly || shardIndex !== null)) throw new Error('--quick cannot change recording, Windows or shard selection');
return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths, ...(jobs === undefined ? {} : { jobs }) };
return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths, ...(jobs === undefined ? {} : { jobs }), ...(shardJobs === undefined ? {} : { shardJobs }) };
}
const describeLines = (lines: string[]) => `${lines.slice(0, 5).join(', ')}${lines.length > 5 ? ` (+${lines.length - 5} more)` : ''}`;
@@ -1350,6 +1370,8 @@ function trackShardBrowser(stateDir: string, env: NodeJS.ProcessEnv) {
check(operation);
if (!Number.isSafeInteger(pid) || (pid as number) <= 1) throw new BrowserCleanupError('invalid browser child identity');
const identity = identities.get(pid as number);
if (!identity && identities.get(daemon as number)?.root) throw new BrowserCleanupError(
`browser child ownership is unconfirmed: recorded pid ${pid} (${await inspect(operation, pid as number) ? 'alive' : 'exited'}) was never seen in daemon ${daemon}'s process tree`);
if (!identity || identity.daemon !== daemon || identity.root) throw new BrowserCleanupError('browser child ownership is unconfirmed');
if (await live(operation, identity) && (typeof start !== 'string' || !start
|| await nativeStart(operation, identity) !== start.replace(/\s+/g, ' '))) throw new BrowserCleanupError('browser child identity was replaced');
@@ -1954,7 +1976,7 @@ async function main(): Promise<number> {
appendStepSummary(line);
}
warnUnseededFreeFiles(ciFiles, durations);
const plan = createFreeCiPlan(ciFiles, options.shardCount, durations, revision);
const plan = createFreeCiPlan(ciFiles, options.shardCount, durations, revision, options.shardJobs);
validateFreeCiPlan(plan, ciFiles, revision);
writeJson(options.ciPlan, plan);
console.log(JSON.stringify({ shard: plan.shards.map(shard => shard.shard) }));
@@ -1975,7 +1997,7 @@ async function main(): Promise<number> {
}
const shard = plan.shards[options.shardIndex! - 1];
if (!shard || shard.shard !== options.shardIndex) throw new Error('CI shard index is outside the plan');
const jobs = options.jobs ?? 1;
const jobs = options.jobs ?? shard.jobs ?? 1;
validateJobsForShard(jobs, shard.files.length, shard.shard);
const durations = loadFreeTestDurations() ?? {};
const { result, exitCode } = await runCiShard({
@@ -1986,7 +2008,7 @@ async function main(): Promise<number> {
runShard: (files, shardNumber, totalShards, runOptions) => runFreeShard(files, shardNumber, totalShards, runOptions),
retry: outcomes => retryFailedFreeFiles(outcomes, plan.shards.length, options),
pack: (files, count) => packShardsByDuration(files, count, durations).shards,
isSerial: file => file in TREE_MUTATING || file in SLOW_UNDER_LOAD,
isSerial: runsSerialUnderJobs,
terminationRequested: isTerminationRequested,
});
writeJson(options.result!, result);