diff --git a/scripts/test-free-shards.ts b/scripts/test-free-shards.ts index 140eac361..e84fc2034 100755 --- a/scripts/test-free-shards.ts +++ b/scripts/test-free-shards.ts @@ -331,10 +331,33 @@ export function wallTimeoutForShard(fileCount: number, baseMs = DEFAULT_WALL_TIM * OS, cap at MAX_FULL_SUITE_JOBS — beyond ~6 concurrent bun processes the * playwright-heavy shards contend on browser launches instead of finishing * sooner (measured on an M-series dev box). + * + * GSTACK_FREE_JOBS overrides the computed count (the free runner's analogue + * of the paid runner's EVALS_JOBS). Exists for syscall-supervised sandboxes: + * on Vercel sandboxes, PID 1 (sandbox-init) installs a seccomp filter whose + * user-space supervisor saturates under ~6 concurrent bun+playwright shards + * and starts returning EACCES from plain file syscalls (measured: 200/200 + * `git init` probes in fresh mktemp dirs fail with + * "Cannot access work tree: Permission denied" while the suite runs, 0/200 + * when idle — access(dir, X_OK) = EACCES under strace). Fewer shards keep + * the supervisor inside its budget. Not clamped by MAX_FULL_SUITE_JOBS so a + * beefy box can also raise it deliberately. */ export const MAX_FULL_SUITE_JOBS = 6; export const RESERVED_CPUS = 2; +export function fullSuiteJobs(): number { + const raw = process.env.GSTACK_FREE_JOBS; + if (raw !== undefined && raw !== '') { + const value = Number.parseInt(raw, 10); + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`GSTACK_FREE_JOBS must be a positive integer, got: ${raw}`); + } + return value; + } + return Math.max(1, Math.min(MAX_FULL_SUITE_JOBS, os.cpus().length - RESERVED_CPUS)); +} + /** * Files that crash or wedge Bun's --parallel WORKERS but run fine in a plain * serial process. Full-suite mode now uses shard PROCESSES (no workers), so @@ -915,6 +938,13 @@ export interface FreeShardOutcome { exitCode: number | null; elapsedMs: number; groupPid: number | null; + /** + * Repo-relative files with attributed test failures or crashes, deduped. + * Feeds the opt-in flaky retry pass (GSTACK_FREE_RETRY_FLAKY) — empty on + * pass, and empty when every failure was unattributed (retry would be + * meaningless without knowing what to re-run). + */ + failingFiles: string[]; } export interface ShardCommand { @@ -1125,11 +1155,16 @@ export async function runFreeShard( console.error(`${label} failed with exit code ${exitCode ?? 'signal'}`); } + const report = reporter.report(); + const failingFiles = status === 'passed' ? [] : [...new Set([ + ...report.failures.map((f) => f.file).filter((f): f is string => !!f), + ...report.crashedFiles, + ])]; const outcome: FreeShardOutcome = { - shard: shardNumber, files, status, exitCode, elapsedMs: Date.now() - startedAt, groupPid, + shard: shardNumber, files, status, exitCode, elapsedMs: Date.now() - startedAt, groupPid, failingFiles, }; log(shardEpilogue(outcome, totalShards)); - for (const line of buildRunEpilogue(status, reporter.report(), outcome.elapsedMs, logPath)) log(line); + for (const line of buildRunEpilogue(status, report, outcome.elapsedMs, logPath)) log(line); return outcome; } @@ -1212,7 +1247,7 @@ async function main(): Promise { // proven spawn semantics, per-shard group-kill, per-shard logs, and a // wedge only ever costs its own shard. WORKER_HOSTILE files are moot in // process shards (no workers) and fold back into normal assignment. - const jobs = Math.max(1, Math.min(MAX_FULL_SUITE_JOBS, os.cpus().length - RESERVED_CPUS)); + const jobs = fullSuiteJobs(); // Phase split: tree-mutating tests run AFTER the parallel shards, in one // serial shard, so no concurrent shard ever reads a half-regenerated tree. const mutators = files.filter((f) => f in TREE_MUTATING); @@ -1250,6 +1285,42 @@ async function main(): Promise { console.error('[test:free] restore with: bun run gen:skill-docs (or git checkout -- )'); } } + outcomes.push(mutatorOutcome); + } + + // Opt-in flaky retry (GSTACK_FREE_RETRY_FLAKY=1): when every failure is an + // attributed test failure (no timeouts, no unattributed carnage), re-run + // just the failing files ONCE in a fresh serial shard. A clean retry + // downgrades the run to a loud flaky-pass; a repeat failure stays a + // failure. Default OFF: dev boxes should see flakes, not absorb them. + // Exists for syscall-supervised sandboxes (see fullSuiteJobs) where a run + // lands 0-1 spurious browser-timing failures under an otherwise-green + // suite. Capped so a genuinely broken tree never masquerades as flaky. + const RETRY_CAP = 5; + if ( + worst !== 0 + && process.env.GSTACK_FREE_RETRY_FLAKY === '1' + && !isTerminationRequested() + && outcomes.every((o) => o.status !== 'timed-out') + ) { + const flakyFiles = [...new Set(outcomes.flatMap((o) => o.failingFiles))]; + const allAttributed = outcomes.every((o) => o.status === 'passed' || o.failingFiles.length > 0); + if (allAttributed && flakyFiles.length > 0 && flakyFiles.length <= RETRY_CAP) { + console.log(`[test:free] flaky-retry: re-running ${flakyFiles.length} failing file(s) once, serially: ${flakyFiles.join(', ')}`); + const retryOutcome = await runFreeShard(flakyFiles, totalShards + 1, totalShards + 1, { + wallTimeoutMs: shardTimeout(flakyFiles.length), + verbose: options.verbose, + }); + if (retryOutcome.status === 'passed') { + console.log(`[test:free] FLAKY-PASS — ${flakyFiles.length} file(s) failed once and passed on serial retry: ${flakyFiles.join(', ')}`); + console.log('[test:free] treat repeat offenders as real flakes worth fixing, not noise.'); + worst = 0; + } else { + console.error('[test:free] flaky-retry FAILED — the failures reproduce serially; not flaky.'); + } + } else if (worst !== 0) { + console.log(`[test:free] flaky-retry skipped: ${allAttributed ? `${flakyFiles.length} failing file(s) exceeds cap ${RETRY_CAP}` : 'failures not fully attributed'}.`); + } } return worst; }