From c45b2c2d17a9c9f7da2ec26fbf3c6483484ea40e Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 29 Aug 2026 16:27:20 +0000 Subject: [PATCH] =?UTF-8?q?fix(test-runner):=20duration-packed=20walls=20k?= =?UTF-8?q?eep=20the=20per-file=20floor=20=E2=80=94=20predictions=20don't?= =?UTF-8?q?=20transfer=20across=20machines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The committed duration seed is recorded on fast CI; a syscall-supervised sandbox replays the same files 2-4x slower. Observed post-merge: a 253-file shard predicted ~242s was wall-killed at its predicted-x3 725s wall while genuinely progressing (the old count heuristic guaranteed 1265s). Packed walls may be looser than the count floor, never tighter. Co-Authored-By: Claude Fable 5 --- scripts/test-free-shards.ts | 12 +++++++++--- test/test-free-shards.test.ts | 11 +++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/test-free-shards.ts b/scripts/test-free-shards.ts index 8381c152b..7fd8e6814 100755 --- a/scripts/test-free-shards.ts +++ b/scripts/test-free-shards.ts @@ -339,8 +339,14 @@ export function wallTimeoutForShard(fileCount: number, baseMs = DEFAULT_WALL_TIM * slow Playwright files), so packed shards get max(base, predicted x 3) — * generous against seed drift, still bounded. */ -export function wallTimeoutForPackedShard(predictedMs: number, baseMs = DEFAULT_WALL_TIMEOUT_MS): number { - return Math.max(baseMs, Math.ceil(predictedMs * 3)); +export function wallTimeoutForPackedShard(predictedMs: number, baseMs = DEFAULT_WALL_TIMEOUT_MS, fileCount = 0): number { + // Predictions transfer badly across machines: the committed duration seed + // is recorded on fast CI, and a syscall-supervised sandbox replays those + // files 2-4x slower (observed: a 253-file shard predicted ~242s wall-killed + // at its 725s predicted-x3 wall while genuinely still progressing). The + // packed wall may therefore be LOOSER than the count heuristic, never + // tighter — it keeps the per-file floor the runner has always guaranteed. + return Math.max(baseMs, Math.ceil(predictedMs * 3), fileCount * PER_FILE_WALL_MS); } /** * Full-suite parallelism: leave RESERVED_CPUS cores for the parent runner + @@ -1444,7 +1450,7 @@ async function main(): Promise { // cost BY DESIGN, so the 5s/file heuristic would undersize a shard // holding few expensive files. wallTimeoutMs: packed && !options.wallTimeoutExplicit - ? wallTimeoutForPackedShard(packed.predictedMs[index], options.wallTimeoutMs) + ? wallTimeoutForPackedShard(packed.predictedMs[index], options.wallTimeoutMs, shardFiles.length) : shardTimeout(shardFiles.length), verbose: options.verbose, })), diff --git a/test/test-free-shards.test.ts b/test/test-free-shards.test.ts index 08887364a..0e7daa157 100644 --- a/test/test-free-shards.test.ts +++ b/test/test-free-shards.test.ts @@ -654,4 +654,15 @@ describe('test-free-shards: duration-aware packing (full-suite LPT)', () => { // Tiny prediction: base still floors it. expect(wallTimeoutForPackedShard(1_000)).toBe(WALL_BASE_MS); }); + + test('packed walls never undercut the per-file floor (predictions do not transfer across machines)', () => { + // The duration seed is recorded on fast CI; a syscall-supervised sandbox + // replays the same files 2-4x slower. A 253-file shard predicted at ~242s + // got wall-killed at predicted×3 = 725s while genuinely progressing — + // the count-based floor (253 × 5s = 1265s) the runner always guaranteed + // must survive duration packing. Looser is allowed, tighter is not. + expect(wallTimeoutForPackedShard(242_000, WALL_BASE_MS, 253)).toBe(253 * 5_000); + // When the prediction is the larger bound, it still wins. + expect(wallTimeoutForPackedShard(600_000, WALL_BASE_MS, 10)).toBe(1_800_000); + }); });