fix: --case no longer deadlocks importing the running CLI

runCaseDiagnosis loaded runPaidShards with a dynamic import of
scripts/test-paid-shards.ts; when that file is the entry it is still in its
top-level await, so the import never resolved and `--case` hung after its
first line (on main since #3031, found by the per-case measurement runs). The
CLI now passes its runner in; library callers keep the lazy import.
This commit is contained in:
garrytan committed 2026-10-05 02:56:12 +00:00
1 parent c95210d4c5
commit 5fce3cdc02
4 files changed
+19 -4

No files matched your search

+2 -2
View File
@@ -12,10 +12,10 @@
"files": {
"scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
"scripts/lib/paid-report.ts": {"blob":"866a9a2bb41e9152989fca23b64e138ba61bbfee","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"scripts/lib/paid-report.ts": {"blob":"d46d7dd0e6ba878dd52d8598f90f8d3f29265b14","since":1,"nonBehavioral":"--case CLI passes its own runner to runCaseDiagnosis (a self-import during top-level await deadlocked); CI slices never take this path"},
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
"scripts/test-paid-shards.ts": {"blob":"f335b16112f0de585320d1c7de03ce22b56f0e65","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":1,"nonBehavioral":"--case CLI passes its own runner to runCaseDiagnosis (a self-import during top-level await deadlocked); CI slices never take this path"},
"scripts/test-pr-profile.ts": {"blob":"d7860897397de8fd605623570d1e7ebbbd5ec743","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":1},
"test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
+3 -1
View File
@@ -70,6 +70,8 @@ export function caseFile(id: string, rootDir = ROOT, discovered = collectPaidTes
export async function runCaseDiagnosis(id: string, options: {
trials?: number; jobs?: number; withinShardConcurrency?: number; timeoutMs?: number; rootDir?: string; env?: NodeJS.ProcessEnv;
evalDirBase?: string; commandFor?: RunShardsOptions['commandFor']; log?: (line: string) => void; file?: string;
/** The CLI passes its own runner: importing the entry module during its top-level await never resolves. */
runShards?: typeof import('../test-paid-shards').runPaidShards;
} = {}): Promise<PanelVerdict> {
const rootDir = options.rootDir ?? ROOT;
const log = options.log ?? ((line: string) => console.log(line));
@@ -82,7 +84,7 @@ export async function runCaseDiagnosis(id: string, options: {
const tier = E2E_TIERS[id] as PaidTier;
log(`[test:paid] --case ${id}: ${n} trial(s) of ${file} (kind ${plan.kind}, PASS at ${plan.panel.k}/${n}${plan.quarantined ? ', quarantined' : ''}), tier=${tier}`);
// The runner is the CLI module; load it lazily so this library never imports it statically (no cycle).
const { runPaidShards } = await import('../test-paid-shards');
const runPaidShards = options.runShards ?? (await import('../test-paid-shards')).runPaidShards;
const shardOptions = { withinShardConcurrency: options.withinShardConcurrency, timeoutMs: options.timeoutMs, rootDir, log, commandFor: options.commandFor,
env: { ...(options.env ?? process.env), EVALS: '1', EVALS_TIER: tier, EVALS_PREFLIGHT_OK: '1', EVALS_ALL: '1',
...paidSelectionEnv('full', { e2e: [id], judges: [] }, `--case ${id}`) } };
+1 -1
View File
@@ -891,7 +891,7 @@ async function main(): Promise<number> {
}
if (options.caseId) {
preflightAnthropicApi(process.env);
const verdict = await runCaseDiagnosis(options.caseId, { trials: options.trials ?? undefined, jobs: options.jobs,
const verdict = await runCaseDiagnosis(options.caseId, { trials: options.trials ?? undefined, jobs: options.jobs, runShards: runPaidShards,
withinShardConcurrency: options.withinShardConcurrency, timeoutMs: timeoutOverride,
evalDirBase: process.env.GSTACK_EVAL_DIR || getProjectEvalDir() });
return verdict.status === 'PASS' ? 0 : 1;
+13
View File
@@ -822,6 +822,19 @@ console.log("Ran 1 tests across 1 files. [1ms]"); process.exit(${fail ? 1 : 0});
expect(unknown.stdout).not.toContain('shard');
});
test('the --case CLI starts its trial shards (it hands runCaseDiagnosis its own runner)', () => {
// A dynamic import of scripts/test-paid-shards.ts while it is still in its top-level await never resolves,
// so the CLI hung after its first line. The trial is killed at its 1 s wall and cannot reach a model.
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'case-cli-'));
try {
const run = spawnSync(process.execPath, [path.join(ROOT, 'scripts/test-paid-shards.ts'), '--tier', 'gate', '--case', 'review-sql-injection', '--trials', '1', '--timeout', '1'], {
encoding: 'utf8', timeout: 60_000, cwd: ROOT,
env: { PATH: `${path.dirname(process.execPath)}:/usr/bin:/bin`, HOME: home, TMPDIR: home, EVALS_PREFLIGHT_OK: '1', GSTACK_EVAL_DIR: path.join(home, 'evals'), EVALS_RUN_ID: 'case-cli-start' } });
expect(run.error).toBeUndefined();
expect(run.stdout).toContain('[test:paid] shard 1/1 START test/skill-e2e-review.test.ts#review-sql-injection~t1');
} finally { fs.rmSync(home, { recursive: true, force: true }); }
}, 70_000);
test('B4: a whole-file case runs one file shard per trial and judges each by shard status', async () => {
const evalDirBase = fs.mkdtempSync(path.join(os.tmpdir(), 'case-file-diagnosis-'));
const lines: string[] = [];