CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-09 22:48:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(test-runner): GSTACK_FREE_JOBS + opt-in flaky-retry pass for syscall-supervised sandboxes

Browse Source 
GSTACK_FREE_JOBS overrides the computed shard count (the free runner's
analogue of the paid runner's EVALS_JOBS). On Vercel sandboxes, PID 1
installs a seccomp filter whose supervisor spuriously fails access(2) for
busy processes — measured: 200/200 git-init probes fail 'Cannot access work
tree: Permission denied' while the suite runs at 6 shards, 0/200 idle;
statx succeeds while access fails on the same path in the same process.
One serial mega-shard maximizes per-process pressure and fails too; 2
shards is the measured sweet spot.

GSTACK_FREE_RETRY_FLAKY=1 (default OFF — dev boxes should see flakes)
re-runs attributed failures once, serially, capped at 5 files; a clean
retry downgrades to a loud FLAKY-PASS naming the offenders, a repeat
failure stays red, timeouts and unattributed failures never retry.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-28 19:58:47 +00:00
co-authored by Claude Fable 5
parent cd53c0663c
commit 626ebc1ba9
1 changed files with 74 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/test-free-shards.ts
+74 -3
View File
@@ -331,10 +331,33 @@ export function wallTimeoutForShard(fileCount: number, baseMs = DEFAULT_WALL_TIM
* OS, cap at MAX_FULL_SUITE_JOBS — beyond ~6 concurrent bun processes the
* playwright-heavy shards contend on browser launches instead of finishing
* sooner (measured on an M-series dev box).
*
* GSTACK_FREE_JOBS overrides the computed count (the free runner's analogue
* of the paid runner's EVALS_JOBS). Exists for syscall-supervised sandboxes:
* on Vercel sandboxes, PID 1 (sandbox-init) installs a seccomp filter whose
* user-space supervisor saturates under ~6 concurrent bun+playwright shards
* and starts returning EACCES from plain file syscalls (measured: 200/200
* `git init` probes in fresh mktemp dirs fail with
* "Cannot access work tree: Permission denied" while the suite runs, 0/200
* when idle — access(dir, X_OK) = EACCES under strace). Fewer shards keep
* the supervisor inside its budget. Not clamped by MAX_FULL_SUITE_JOBS so a
* beefy box can also raise it deliberately.
*/
export const MAX_FULL_SUITE_JOBS = 6;
export const RESERVED_CPUS = 2;
export function fullSuiteJobs(): number {
const raw = process.env.GSTACK_FREE_JOBS;
if (raw !== undefined && raw !== '') {
const value = Number.parseInt(raw, 10);
if (!Number.isInteger(value) || value <= 0) {
throw new Error(`GSTACK_FREE_JOBS must be a positive integer, got: ${raw}`);
}
return value;
}
return Math.max(1, Math.min(MAX_FULL_SUITE_JOBS, os.cpus().length - RESERVED_CPUS));
}
/**
* Files that crash or wedge Bun's --parallel WORKERS but run fine in a plain
* serial process. Full-suite mode now uses shard PROCESSES (no workers), so
@@ -915,6 +938,13 @@ export interface FreeShardOutcome {
exitCode: number | null;
elapsedMs: number;
groupPid: number | null;
/**
* Repo-relative files with attributed test failures or crashes, deduped.
* Feeds the opt-in flaky retry pass (GSTACK_FREE_RETRY_FLAKY) — empty on
* pass, and empty when every failure was unattributed (retry would be
* meaningless without knowing what to re-run).
*/
failingFiles: string[];
}
export interface ShardCommand {
@@ -1125,11 +1155,16 @@ export async function runFreeShard(
console.error(`${label} failed with exit code ${exitCode ?? 'signal'}`);
}
const report = reporter.report();
const failingFiles = status === 'passed' ? [] : [...new Set([
...report.failures.map((f) => f.file).filter((f): f is string => !!f),
...report.crashedFiles,
])];
const outcome: FreeShardOutcome = {
shard: shardNumber, files, status, exitCode, elapsedMs: Date.now() - startedAt, groupPid,
shard: shardNumber, files, status, exitCode, elapsedMs: Date.now() - startedAt, groupPid, failingFiles,
};
log(shardEpilogue(outcome, totalShards));
for (const line of buildRunEpilogue(status, reporter.report(), outcome.elapsedMs, logPath)) log(line);
for (const line of buildRunEpilogue(status, report, outcome.elapsedMs, logPath)) log(line);
return outcome;
}
@@ -1212,7 +1247,7 @@ async function main(): Promise<number> {
// proven spawn semantics, per-shard group-kill, per-shard logs, and a
// wedge only ever costs its own shard. WORKER_HOSTILE files are moot in
// process shards (no workers) and fold back into normal assignment.
const jobs = Math.max(1, Math.min(MAX_FULL_SUITE_JOBS, os.cpus().length - RESERVED_CPUS));
const jobs = fullSuiteJobs();
// Phase split: tree-mutating tests run AFTER the parallel shards, in one
// serial shard, so no concurrent shard ever reads a half-regenerated tree.
const mutators = files.filter((f) => f in TREE_MUTATING);
@@ -1250,6 +1285,42 @@ async function main(): Promise<number> {
console.error('[test:free] restore with: bun run gen:skill-docs (or git checkout -- <paths>)');
}
}
outcomes.push(mutatorOutcome);
}
// Opt-in flaky retry (GSTACK_FREE_RETRY_FLAKY=1): when every failure is an
// attributed test failure (no timeouts, no unattributed carnage), re-run
// just the failing files ONCE in a fresh serial shard. A clean retry
// downgrades the run to a loud flaky-pass; a repeat failure stays a
// failure. Default OFF: dev boxes should see flakes, not absorb them.
// Exists for syscall-supervised sandboxes (see fullSuiteJobs) where a run
// lands 0-1 spurious browser-timing failures under an otherwise-green
// suite. Capped so a genuinely broken tree never masquerades as flaky.
const RETRY_CAP = 5;
if (
worst !== 0
&& process.env.GSTACK_FREE_RETRY_FLAKY === '1'
&& !isTerminationRequested()
&& outcomes.every((o) => o.status !== 'timed-out')
) {
const flakyFiles = [...new Set(outcomes.flatMap((o) => o.failingFiles))];
const allAttributed = outcomes.every((o) => o.status === 'passed' || o.failingFiles.length > 0);
if (allAttributed && flakyFiles.length > 0 && flakyFiles.length <= RETRY_CAP) {
console.log(`[test:free] flaky-retry: re-running ${flakyFiles.length} failing file(s) once, serially: ${flakyFiles.join(', ')}`);
const retryOutcome = await runFreeShard(flakyFiles, totalShards + 1, totalShards + 1, {
wallTimeoutMs: shardTimeout(flakyFiles.length),
verbose: options.verbose,
});
if (retryOutcome.status === 'passed') {
console.log(`[test:free] FLAKY-PASS — ${flakyFiles.length} file(s) failed once and passed on serial retry: ${flakyFiles.join(', ')}`);
console.log('[test:free] treat repeat offenders as real flakes worth fixing, not noise.');
worst = 0;
} else {
console.error('[test:free] flaky-retry FAILED — the failures reproduce serially; not flaky.');
}
} else if (worst !== 0) {
console.log(`[test:free] flaky-retry skipped: ${allAttributed ? `${flakyFiles.length} failing file(s) exceeds cap ${RETRY_CAP}` : 'failures not fully attributed'}.`);
}
}
return worst;
}