mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-16 13:21:30 +01:00
v1.93.0.0 stamp: merge main 1a7f2578b7 into #3858 (land)
This commit is contained in:
commit
2b8bbff7a7
57 files changed
+1409
-206
No files matched your search
@@ -5,8 +5,10 @@
|
||||
* the real home before and after each free shard, and names the shard's files
|
||||
* when one of those entries changed. It watches only these surfaces; it is not
|
||||
* a universal write-containment check. Live agent-session logs that a
|
||||
* concurrently running host keeps writing are excluded, and directory mtimes
|
||||
* are ignored, so only file, link and directory-set changes count.
|
||||
* concurrently running host keeps writing are excluded, as is the one lane log
|
||||
* `gstack-evidence run` tees this run's own output into (its exact path, from
|
||||
* GSTACK_EVIDENCE_LOG). Directory mtimes are ignored, so only file, link and
|
||||
* directory-set changes count.
|
||||
*
|
||||
* A shard that runs alone owns its window, so its guard names its files.
|
||||
* Concurrent shards share one HOME and cannot tell whose write a change was, so
|
||||
@@ -104,6 +106,7 @@ export type FreeHomeGuardFactory = (files: string[], env: NodeJS.ProcessEnv, sta
|
||||
/** Take the baseline now; verify() returns null or the failure to report. */
|
||||
export function guardFreeHome(files: string[], env: NodeJS.ProcessEnv = process.env, scope: FreeHomeScope = { kind: 'shard' }): FreeHomeGuard {
|
||||
const home = freeHomeDir(env);
|
||||
const ownLog = env.GSTACK_EVIDENCE_LOG && path.isAbsolute(env.GSTACK_EVIDENCE_LOG) ? path.relative(home, env.GSTACK_EVIDENCE_LOG) : null;
|
||||
const unreadable = (error: unknown) => `real home surfaces could not be snapshotted (${(error as NodeJS.ErrnoException).code ?? 'error'}); shard files: ${files.join(', ')}`;
|
||||
let before: FreeHomeSnapshot | undefined;
|
||||
let baselineError: string | undefined;
|
||||
@@ -112,7 +115,7 @@ export function guardFreeHome(files: string[], env: NodeJS.ProcessEnv = process.
|
||||
verify() {
|
||||
if (!before) return baselineError!;
|
||||
try {
|
||||
const changed = diffFreeHome(before, snapshotFreeHome(home));
|
||||
const changed = diffFreeHome(before, snapshotFreeHome(home)).filter(entry => entry !== ownLog);
|
||||
return changed.length ? formatFreeHomeChange(changed, files, scope) : null;
|
||||
} catch (error) { return unreadable(error); }
|
||||
},
|
||||
|
||||
@@ -45,16 +45,39 @@ export const FREE_JOBS_INVALID = 'FREE_JOBS_INVALID';
|
||||
export const FREE_SUBSHARD_TIMEOUT = 'FREE_SUBSHARD_TIMEOUT';
|
||||
export const FREE_SUBSHARD_REJECTED = 'FREE_SUBSHARD_REJECTED';
|
||||
const DOCS = 'docs/TESTING_INTERNALS.md#sub-shards-inside-one-ci-job';
|
||||
const JOBS_FIX = `--jobs applies only to --ci-run and defaults to 1; GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run. See ${DOCS}`;
|
||||
const JOBS_FIX = `--jobs applies only to --ci-run and defaults to the plan's per-shard k (1 when the plan has none); --shard-jobs applies only to --ci-plan; GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run. See ${DOCS}`;
|
||||
|
||||
/** `--jobs <k>` at parse time: digits only, at least 1. */
|
||||
export function parseJobsFlag(raw: string | undefined): number {
|
||||
/** `--jobs <k>` or `--shard-jobs <k>` at parse time: digits only, at least 1. */
|
||||
export function parseJobsFlag(raw: string | undefined, flag = '--jobs'): number {
|
||||
if (raw === undefined || !/^\d+$/.test(raw) || Number(raw) < 1) {
|
||||
throw new Error(`${FREE_JOBS_INVALID}: --jobs needs a positive whole number; got ${JSON.stringify(raw ?? '')}. ${JOBS_FIX}`);
|
||||
throw new Error(`${FREE_JOBS_INVALID}: ${flag} needs a positive whole number; got ${JSON.stringify(raw ?? '')}. ${JOBS_FIX}`);
|
||||
}
|
||||
return Number(raw);
|
||||
}
|
||||
|
||||
export function rejectShardJobsWithoutCiPlan(): never {
|
||||
throw new Error(`${FREE_JOBS_INVALID}: --shard-jobs needs --ci-plan. ${JOBS_FIX}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* One planned shard's k under `--ci-plan --shard-jobs k`: at most one parallel
|
||||
* sub-shard per file that may run in parallel, so a shard of only serial files
|
||||
* (the exclusive TREE_MUTATING shard) keeps k = 1. Undefined means k = 1, and
|
||||
* the plan then carries no `jobs` key, so its id matches a plan without k.
|
||||
*/
|
||||
export function plannedShardJobs(files: readonly string[], shardJobs: number, isSerial: (file: string) => boolean): number | undefined {
|
||||
const jobs = Math.min(shardJobs, files.filter(file => !isSerial(file)).length);
|
||||
return jobs > 1 ? jobs : undefined;
|
||||
}
|
||||
|
||||
/** Why a plan's per-shard k cannot stand; null when absent or valid. */
|
||||
export function plannedShardJobsProblem(shard: { shard: number; files: readonly string[]; jobs?: unknown }, isSerial: (file: string) => boolean): string | null {
|
||||
if (shard.jobs === undefined) return null;
|
||||
const parallel = shard.files.filter(file => !isSerial(file)).length;
|
||||
return Number.isInteger(shard.jobs) && (shard.jobs as number) >= 2 && (shard.jobs as number) <= parallel ? null
|
||||
: `CI shard ${shard.shard} plans --jobs ${JSON.stringify(shard.jobs)} for ${parallel} parallel file(s)`;
|
||||
}
|
||||
|
||||
export function rejectJobsWithoutCiRun(): never {
|
||||
throw new Error(`${FREE_JOBS_INVALID}: --jobs needs --ci-run. ${JOBS_FIX}`);
|
||||
}
|
||||
|
||||
+37
-15
@@ -77,11 +77,15 @@
|
||||
* bun run scripts/test-free-shards.ts --attribute-home # name real-home writers: each file alone, private HOME
|
||||
* bun run scripts/test-free-shards.ts --ci-plan plan.json --shards 20
|
||||
* bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json
|
||||
* bun run scripts/test-free-shards.ts --ci-plan plan.json --shards 20 --shard-jobs 2
|
||||
* # the plan stores a per-shard k: min(k, the shard's files that may run in parallel),
|
||||
* # kept only when above 1, so the exclusive TREE_MUTATING shard stays one child.
|
||||
* bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json --jobs 4
|
||||
* # k sub-shards side by side inside one CI job (default 1 = one child, today's behavior;
|
||||
* # scripts/lib/shard-jobs.ts). --jobs applies only to --ci-run: a positive whole number up
|
||||
* # to the shard's file count, else FREE_JOBS_INVALID. GSTACK_FREE_JOBS sets the local
|
||||
* # full-suite process count and never changes --ci-run; --jobs is its only knob.
|
||||
* # k sub-shards side by side inside one CI job (default: the plan's per-shard k, else 1 =
|
||||
* # one child, today's behavior; scripts/lib/shard-jobs.ts). --jobs applies only to
|
||||
* # --ci-run and overrides the plan: a positive whole number up to the shard's file count,
|
||||
* # else FREE_JOBS_INVALID. --ci-verify requires every result to have run at the plan's k.
|
||||
* # GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run.
|
||||
* bun run scripts/test-free-shards.ts --ci-verify plan.json --results results/
|
||||
*/
|
||||
|
||||
@@ -97,8 +101,8 @@ import { attributeFreeHomeWriters, guardFreeHome, sharedFreeHome, type FreeHomeG
|
||||
import { appendStepSummary, ciHealthSummary, SEED_REFRESH_COMMAND, unseededWarning, windowsCurationLine } from './lib/free-ci-health';
|
||||
import { curateWindowsSafe, type CurationResult } from './lib/windows-curation';
|
||||
import {
|
||||
assertSoloSeedSample, ciRunContext, parseJobsFlag, parseJunitCounts, readJunitReport, rejectJobsWithoutCiRun, runCiShard, SLOW_UNDER_LOAD, subShardRecordProblems,
|
||||
validateJobsForShard, type ShardTestCounts, type SubShardRecord,
|
||||
assertSoloSeedSample, ciRunContext, parseJobsFlag, parseJunitCounts, plannedShardJobs, plannedShardJobsProblem, readJunitReport, rejectJobsWithoutCiRun,
|
||||
rejectShardJobsWithoutCiPlan, runCiShard, SLOW_UNDER_LOAD, subShardRecordProblems, validateJobsForShard, type ShardTestCounts, type SubShardRecord,
|
||||
} from './lib/shard-jobs';
|
||||
export { curateWindowsSafe, detectWindowsFragility, KNOWN_WINDOWS_INCOMPATIBLE, type CurationResult } from './lib/windows-curation';
|
||||
import {
|
||||
@@ -424,19 +428,27 @@ export interface FreeCiPlan {
|
||||
version: 1;
|
||||
revision: string;
|
||||
id: string;
|
||||
shards: Array<{ shard: number; files: string[]; predictedMs: number }>;
|
||||
/** `jobs` is the shard's k from --shard-jobs, present only when above 1. */
|
||||
shards: Array<{ shard: number; files: string[]; predictedMs: number; jobs?: number }>;
|
||||
}
|
||||
|
||||
/** Files that run in the trailing serial sub-shard under k > 1, never beside another child. */
|
||||
const runsSerialUnderJobs = (file: string) => file in TREE_MUTATING || file in SLOW_UNDER_LOAD;
|
||||
|
||||
const planDigest = (plan: Omit<FreeCiPlan, 'id'>): string =>
|
||||
createHash('sha256').update(JSON.stringify(plan)).digest('hex');
|
||||
|
||||
/** One immutable plan is shared by isolated CI machines; never repack per job. */
|
||||
export function createFreeCiPlan(files: string[], count: number, durations: Record<string, number>, revision: string): FreeCiPlan {
|
||||
export function createFreeCiPlan(files: string[], count: number, durations: Record<string, number>, revision: string, shardJobs = 1): FreeCiPlan {
|
||||
const readers = files.filter(file => !(file in TREE_MUTATING));
|
||||
const exclusive = files.filter(file => file in TREE_MUTATING).sort();
|
||||
const packed = packShardsByDuration(readers, count, durations);
|
||||
const shards = packed.shards.map((files, index) => ({ shard: index + 1, files, predictedMs: packed.predictedMs[index] }));
|
||||
if (exclusive.length) shards.push({ shard: shards.length + 1, files: exclusive, predictedMs: exclusive.reduce((ms, file) => ms + (durations[file] ?? 0), 0) });
|
||||
const planned = packed.shards.map((files, index) => ({ files, predictedMs: packed.predictedMs[index] }));
|
||||
if (exclusive.length) planned.push({ files: exclusive, predictedMs: exclusive.reduce((ms, file) => ms + (durations[file] ?? 0), 0) });
|
||||
const shards = planned.map(({ files, predictedMs }, index) => {
|
||||
const jobs = plannedShardJobs(files, shardJobs, runsSerialUnderJobs);
|
||||
return { shard: index + 1, files, predictedMs, ...(jobs ? { jobs } : {}) };
|
||||
});
|
||||
const body = { version: 1 as const, revision, shards };
|
||||
return { ...body, id: planDigest(body) };
|
||||
}
|
||||
@@ -447,6 +459,8 @@ export function validateFreeCiPlan(plan: FreeCiPlan, files: string[], revision:
|
||||
|| id !== planDigest({ version, revision: plan.revision, shards })) throw new Error('CI plan identity or revision mismatch');
|
||||
if (shards.some((shard, index) => shard.shard !== index + 1 || !Array.isArray(shard.files)
|
||||
|| !Number.isFinite(shard.predictedMs) || shard.predictedMs < 0)) throw new Error('Invalid CI shard plan');
|
||||
const jobsProblems = shards.map(shard => plannedShardJobsProblem(shard, runsSerialUnderJobs)).filter(Boolean);
|
||||
if (jobsProblems.length) throw new Error(`Invalid CI shard plan: ${jobsProblems.join('; ')}`);
|
||||
const planned = shards.flatMap(shard => shard.files).sort();
|
||||
if (new Set(planned).size !== planned.length || JSON.stringify(planned) !== JSON.stringify([...files].sort())) throw new Error('CI plan must cover every free file exactly once');
|
||||
}
|
||||
@@ -486,6 +500,7 @@ export function verifyFreeCiResults(plan: FreeCiPlan, results: FreeCiResult[]):
|
||||
if (!hasCompleteCiSummary(outcome)) throw new Error('Missing or incomplete CI execution summary');
|
||||
const subShardProblems = subShardRecordProblems(result);
|
||||
if (subShardProblems.length) throw new Error(`CI shard ${outcome.shard} sub-shards rejected: ${subShardProblems.join('; ')}`);
|
||||
if ((result.jobs ?? 1) !== (shard.jobs ?? 1)) throw new Error(`CI shard ${outcome.shard} ran at --jobs ${result.jobs ?? 1}; the plan says ${shard.jobs ?? 1}`);
|
||||
if (outcome.status === 'passed') {
|
||||
if (outcome.exitCode !== 0 || outcome.failingFiles.length || outcome.unattributedFailures || result.retry) throw new Error('Inconsistent passing CI result');
|
||||
} else {
|
||||
@@ -556,8 +571,10 @@ type CliOptions = {
|
||||
result: string | null;
|
||||
results: string | null;
|
||||
filesFrom: string | null;
|
||||
/** Set only by --jobs (with --ci-run); absent means k = 1. */
|
||||
/** Set only by --jobs (with --ci-run); absent means the plan's per-shard k. */
|
||||
jobs?: number;
|
||||
/** Set only by --shard-jobs (with --ci-plan); absent means k = 1 in every shard. */
|
||||
shardJobs?: number;
|
||||
};
|
||||
|
||||
export function parseCliOptions(argv: string[]): CliOptions {
|
||||
@@ -573,6 +590,7 @@ export function parseCliOptions(argv: string[]): CliOptions {
|
||||
let quick = false;
|
||||
let attributeHome = false;
|
||||
let jobs: number | undefined;
|
||||
let shardJobs: number | undefined;
|
||||
const paths: Record<'ciPlan' | 'ciRun' | 'ciVerify' | 'result' | 'results' | 'filesFrom', string | null> = {
|
||||
ciPlan: null, ciRun: null, ciVerify: null, result: null, results: null, filesFrom: null,
|
||||
};
|
||||
@@ -597,6 +615,7 @@ export function parseCliOptions(argv: string[]): CliOptions {
|
||||
'--results': pathFlag('--results', 'results'),
|
||||
'--files-from': pathFlag('--files-from', 'filesFrom'),
|
||||
'--jobs': (next) => { jobs = parseJobsFlag(next()); },
|
||||
'--shard-jobs': (next) => { shardJobs = parseJobsFlag(next(), '--shard-jobs'); },
|
||||
'--shards': (next) => {
|
||||
const value = next();
|
||||
if (!value) throw new Error('Missing value for --shards');
|
||||
@@ -621,8 +640,9 @@ export function parseCliOptions(argv: string[]): CliOptions {
|
||||
if (paths.ciRun && (shardIndex === null || !paths.result)) throw new Error('--ci-run requires --shard and --result');
|
||||
if (paths.ciVerify && !paths.results) throw new Error('--ci-verify requires --results');
|
||||
if (jobs !== undefined && !paths.ciRun) rejectJobsWithoutCiRun();
|
||||
if (shardJobs !== undefined && !paths.ciPlan) rejectShardJobsWithoutCiPlan();
|
||||
if (quick && (recordDurations || windowsOnly || shardIndex !== null)) throw new Error('--quick cannot change recording, Windows or shard selection');
|
||||
return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths, ...(jobs === undefined ? {} : { jobs }) };
|
||||
return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths, ...(jobs === undefined ? {} : { jobs }), ...(shardJobs === undefined ? {} : { shardJobs }) };
|
||||
}
|
||||
|
||||
const describeLines = (lines: string[]) => `${lines.slice(0, 5).join(', ')}${lines.length > 5 ? ` (+${lines.length - 5} more)` : ''}`;
|
||||
@@ -1350,6 +1370,8 @@ function trackShardBrowser(stateDir: string, env: NodeJS.ProcessEnv) {
|
||||
check(operation);
|
||||
if (!Number.isSafeInteger(pid) || (pid as number) <= 1) throw new BrowserCleanupError('invalid browser child identity');
|
||||
const identity = identities.get(pid as number);
|
||||
if (!identity && identities.get(daemon as number)?.root) throw new BrowserCleanupError(
|
||||
`browser child ownership is unconfirmed: recorded pid ${pid} (${await inspect(operation, pid as number) ? 'alive' : 'exited'}) was never seen in daemon ${daemon}'s process tree`);
|
||||
if (!identity || identity.daemon !== daemon || identity.root) throw new BrowserCleanupError('browser child ownership is unconfirmed');
|
||||
if (await live(operation, identity) && (typeof start !== 'string' || !start
|
||||
|| await nativeStart(operation, identity) !== start.replace(/\s+/g, ' '))) throw new BrowserCleanupError('browser child identity was replaced');
|
||||
@@ -1954,7 +1976,7 @@ async function main(): Promise<number> {
|
||||
appendStepSummary(line);
|
||||
}
|
||||
warnUnseededFreeFiles(ciFiles, durations);
|
||||
const plan = createFreeCiPlan(ciFiles, options.shardCount, durations, revision);
|
||||
const plan = createFreeCiPlan(ciFiles, options.shardCount, durations, revision, options.shardJobs);
|
||||
validateFreeCiPlan(plan, ciFiles, revision);
|
||||
writeJson(options.ciPlan, plan);
|
||||
console.log(JSON.stringify({ shard: plan.shards.map(shard => shard.shard) }));
|
||||
@@ -1975,7 +1997,7 @@ async function main(): Promise<number> {
|
||||
}
|
||||
const shard = plan.shards[options.shardIndex! - 1];
|
||||
if (!shard || shard.shard !== options.shardIndex) throw new Error('CI shard index is outside the plan');
|
||||
const jobs = options.jobs ?? 1;
|
||||
const jobs = options.jobs ?? shard.jobs ?? 1;
|
||||
validateJobsForShard(jobs, shard.files.length, shard.shard);
|
||||
const durations = loadFreeTestDurations() ?? {};
|
||||
const { result, exitCode } = await runCiShard({
|
||||
@@ -1986,7 +2008,7 @@ async function main(): Promise<number> {
|
||||
runShard: (files, shardNumber, totalShards, runOptions) => runFreeShard(files, shardNumber, totalShards, runOptions),
|
||||
retry: outcomes => retryFailedFreeFiles(outcomes, plan.shards.length, options),
|
||||
pack: (files, count) => packShardsByDuration(files, count, durations).shards,
|
||||
isSerial: file => file in TREE_MUTATING || file in SLOW_UNDER_LOAD,
|
||||
isSerial: runsSerialUnderJobs,
|
||||
terminationRequested: isTerminationRequested,
|
||||
});
|
||||
writeJson(options.result!, result);
|
||||
|
||||
Reference in new issue
Block a user