From 5fce3cdc02140332e0638fd9c9e0e729cbf28311 Mon Sep 17 00:00:00 2001 From: garrytan Date: Mon, 5 Oct 2026 02:56:12 +0000 Subject: [PATCH] fix: --case no longer deadlocks importing the running CLI runCaseDiagnosis loaded runPaidShards with a dynamic import of scripts/test-paid-shards.ts; when that file is the entry it is still in its top-level await, so the import never resolved and `--case` hung after its first line (on main since #3031, found by the per-case measurement runs). The CLI now passes its runner in; library callers keep the lazy import. --- scripts/harness-version.json | 4 ++-- scripts/lib/paid-report.ts | 4 +++- scripts/test-paid-shards.ts | 2 +- test/paid-shards.test.ts | 13 +++++++++++++ 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/scripts/harness-version.json b/scripts/harness-version.json index c7b005f3a..248acc8e6 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -12,10 +12,10 @@ "files": { "scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"}, "scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"}, - "scripts/lib/paid-report.ts": {"blob":"866a9a2bb41e9152989fca23b64e138ba61bbfee","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"}, + "scripts/lib/paid-report.ts": {"blob":"d46d7dd0e6ba878dd52d8598f90f8d3f29265b14","since":1,"nonBehavioral":"--case CLI passes its own runner to runCaseDiagnosis (a self-import during top-level await deadlocked); CI slices never take this path"}, "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"}, "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"}, - "scripts/test-paid-shards.ts": {"blob":"f335b16112f0de585320d1c7de03ce22b56f0e65","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"}, + "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":1,"nonBehavioral":"--case CLI passes its own runner to runCaseDiagnosis (a self-import during top-level await deadlocked); CI slices never take this path"}, "scripts/test-pr-profile.ts": {"blob":"d7860897397de8fd605623570d1e7ebbbd5ec743","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":1}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"}, diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index 866a9a2bb..d46d7dd0e 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -70,6 +70,8 @@ export function caseFile(id: string, rootDir = ROOT, discovered = collectPaidTes export async function runCaseDiagnosis(id: string, options: { trials?: number; jobs?: number; withinShardConcurrency?: number; timeoutMs?: number; rootDir?: string; env?: NodeJS.ProcessEnv; evalDirBase?: string; commandFor?: RunShardsOptions['commandFor']; log?: (line: string) => void; file?: string; + /** The CLI passes its own runner: importing the entry module during its top-level await never resolves. */ + runShards?: typeof import('../test-paid-shards').runPaidShards; } = {}): Promise { const rootDir = options.rootDir ?? ROOT; const log = options.log ?? ((line: string) => console.log(line)); @@ -82,7 +84,7 @@ export async function runCaseDiagnosis(id: string, options: { const tier = E2E_TIERS[id] as PaidTier; log(`[test:paid] --case ${id}: ${n} trial(s) of ${file} (kind ${plan.kind}, PASS at ${plan.panel.k}/${n}${plan.quarantined ? ', quarantined' : ''}), tier=${tier}`); // The runner is the CLI module; load it lazily so this library never imports it statically (no cycle). - const { runPaidShards } = await import('../test-paid-shards'); + const runPaidShards = options.runShards ?? (await import('../test-paid-shards')).runPaidShards; const shardOptions = { withinShardConcurrency: options.withinShardConcurrency, timeoutMs: options.timeoutMs, rootDir, log, commandFor: options.commandFor, env: { ...(options.env ?? process.env), EVALS: '1', EVALS_TIER: tier, EVALS_PREFLIGHT_OK: '1', EVALS_ALL: '1', ...paidSelectionEnv('full', { e2e: [id], judges: [] }, `--case ${id}`) } }; diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index f335b1611..8ddee5f11 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -891,7 +891,7 @@ async function main(): Promise { } if (options.caseId) { preflightAnthropicApi(process.env); - const verdict = await runCaseDiagnosis(options.caseId, { trials: options.trials ?? undefined, jobs: options.jobs, + const verdict = await runCaseDiagnosis(options.caseId, { trials: options.trials ?? undefined, jobs: options.jobs, runShards: runPaidShards, withinShardConcurrency: options.withinShardConcurrency, timeoutMs: timeoutOverride, evalDirBase: process.env.GSTACK_EVAL_DIR || getProjectEvalDir() }); return verdict.status === 'PASS' ? 0 : 1; diff --git a/test/paid-shards.test.ts b/test/paid-shards.test.ts index 600a3ae51..ead8c64db 100644 --- a/test/paid-shards.test.ts +++ b/test/paid-shards.test.ts @@ -822,6 +822,19 @@ console.log("Ran 1 tests across 1 files. [1ms]"); process.exit(${fail ? 1 : 0}); expect(unknown.stdout).not.toContain('shard'); }); + test('the --case CLI starts its trial shards (it hands runCaseDiagnosis its own runner)', () => { + // A dynamic import of scripts/test-paid-shards.ts while it is still in its top-level await never resolves, + // so the CLI hung after its first line. The trial is killed at its 1 s wall and cannot reach a model. + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'case-cli-')); + try { + const run = spawnSync(process.execPath, [path.join(ROOT, 'scripts/test-paid-shards.ts'), '--tier', 'gate', '--case', 'review-sql-injection', '--trials', '1', '--timeout', '1'], { + encoding: 'utf8', timeout: 60_000, cwd: ROOT, + env: { PATH: `${path.dirname(process.execPath)}:/usr/bin:/bin`, HOME: home, TMPDIR: home, EVALS_PREFLIGHT_OK: '1', GSTACK_EVAL_DIR: path.join(home, 'evals'), EVALS_RUN_ID: 'case-cli-start' } }); + expect(run.error).toBeUndefined(); + expect(run.stdout).toContain('[test:paid] shard 1/1 START test/skill-e2e-review.test.ts#review-sql-injection~t1'); + } finally { fs.rmSync(home, { recursive: true, force: true }); } + }, 70_000); + test('B4: a whole-file case runs one file shard per trial and judges each by shard status', async () => { const evalDirBase = fs.mkdtempSync(path.join(os.tmpdir(), 'case-file-diagnosis-')); const lines: string[] = [];