From be30f0eb5269c5f9ed19bcc1950c3fb3644c19dd Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 31 Aug 2026 05:02:08 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20paid-runner=20defaults=204x4=20->=208x2?= =?UTF-8?q?=20=E2=80=94=20halve=20the=20local=20gate=20worst=20case?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 39 of 75 skill-e2e files hold exactly ONE test, so within-shard concurrency was dead weight for most shards: 4 jobs x 4 concurrency yielded only ~4-6 real in-flight sessions and a 13-wave local gate worst case (~6.5h). 8 jobs x 2 gives ~10-13 in-flight — under the documented-safe ~15 — and ~7 waves (~3.3h worst case). CI lanes keep their explicit EVALS_JOBS env (2 per slice; 4 for gate-census); this changes local defaults. Rollback trigger: sustained 429 storms in the WS1 telemetry across 2 PR cycles. test/eval-detach-timeout-floor.test.ts recomputed green (the raise LOWERS the worst-case floor). Co-Authored-By: Claude Fable 5 --- scripts/test-paid-shards.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index cf488116b..8a2874e33 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -85,11 +85,16 @@ export type PaidTier = 'gate' | 'periodic'; export const DEFAULT_TIER: PaidTier = 'gate'; export const DEFAULT_SHARD_TIMEOUT_MS = 30 * 60_000; export const DEFAULT_MAX_FILES_PER_SHARD = 1; -export const DEFAULT_JOBS = 4; -// Within one shard's bun process. 4 jobs × 4 ≈ the legacy single-process -// default of 15, keeping total in-flight `claude` sessions inside known-safe -// API rate headroom. -export const DEFAULT_WITHIN_SHARD_CONCURRENCY = 4; +// 8 jobs × 2 within-shard ≈ 10-13 real in-flight sessions (39 of 75 +// skill-e2e files hold exactly ONE test, so within-shard concurrency is +// dead weight for most shards) — under the documented-safe ~15 the legacy +// 40-way runner established. The old 4×4 yielded only ~4-6 in-flight and a +// 13-wave local gate worst case (~6.5h); 8×2 halves it. Watch the WS1 +// flake telemetry for sustained 429 storms across 2 PR cycles — that is +// the rollback trigger. Prerequisite (landed): per-shard TMPDIR/ +// CHROMIUM_PROFILE isolation in runPaidShard. +export const DEFAULT_JOBS = 8; +export const DEFAULT_WITHIN_SHARD_CONCURRENCY = 2; export function collectPaidTestFiles(rootDir = ROOT): string[] { const testDir = path.join(rootDir, 'test');