diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index 9ddc8e9f8..dc3a89516 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -64,6 +64,7 @@ import { } from './test-strict-output'; import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set'; import { getProjectEvalDir } from '../test/helpers/eval-store'; +import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight'; export { PAID_TEST_GLOBS, isPaidTestFile }; @@ -463,13 +464,19 @@ async function main(): Promise { return 0; } + // One preflight ping in the parent; children skip theirs via the env flag. + // Before this, every shard's e2e-helpers module load re-pinged the API — + // ~30 paid claude -p calls (30s timeout each) per full run for one bit of + // information. A dead API now fails here, before any shard spawns. + preflightAnthropicApi(process.env); + const summary = await runPaidShards(shards, { // Tier reaches the children only via EVALS_TIER below; the runtime // E2E_TIERS filter inside each child is the real selection mechanism. timeoutMs: options.timeoutMs, jobs: options.jobs, withinShardConcurrency: options.withinShardConcurrency, - env: { ...process.env, EVALS: '1', EVALS_TIER: options.tier }, + env: { ...process.env, EVALS: '1', EVALS_TIER: options.tier, EVALS_PREFLIGHT_OK: '1' }, evalDirBase: process.env.GSTACK_EVAL_DIR || getProjectEvalDir(), }); for (const line of formatSummary(summary)) console.log(line); diff --git a/test/anthropic-preflight.test.ts b/test/anthropic-preflight.test.ts new file mode 100644 index 000000000..a207bee96 --- /dev/null +++ b/test/anthropic-preflight.test.ts @@ -0,0 +1,53 @@ +/** + * Regression pins for the preflight-dedup seam (test/helpers/anthropic-preflight.ts). + * + * The preflight runs at MODULE LOAD in every paid test file that imports + * e2e-helpers, so a broken skip-check either brings back ~30 paid pings per + * sharded run or — worse — skips the fail-fast everywhere. Both directions + * are pinned here with an injected spawn; no real claude call is made. + */ + +import { describe, test, expect } from 'bun:test'; +import { preflightAnthropicApi } from './helpers/anthropic-preflight'; + +type SpawnCall = { command: string; args: string[] }; + +function fakeSpawn(stdout: string, calls: SpawnCall[]) { + return ((command: string, args: string[]) => { + calls.push({ command, args }); + return { stdout: Buffer.from(stdout), stderr: Buffer.from(''), status: 0 } as ReturnType< + typeof import('child_process').spawnSync + >; + }) as typeof import('child_process').spawnSync; +} + +describe('preflightAnthropicApi', () => { + test('EVALS_PREFLIGHT_OK=1 skips the ping entirely (sharded-child path)', () => { + const calls: SpawnCall[] = []; + const result = preflightAnthropicApi({ EVALS_PREFLIGHT_OK: '1' }, fakeSpawn('never read', calls)); + expect(result).toBe('skipped'); + expect(calls.length).toBe(0); + }); + + test('without the flag, pings exactly once and passes on healthy output', () => { + const calls: SpawnCall[] = []; + const result = preflightAnthropicApi({}, fakeSpawn('{"type":"result"}', calls)); + expect(result).toBe('ok'); + expect(calls.length).toBe(1); + expect(calls[0].args.join(' ')).toContain('claude -p'); + }); + + test('unreachable API throws (fail-fast before any shard spawns)', () => { + const calls: SpawnCall[] = []; + expect(() => + preflightAnthropicApi({}, fakeSpawn('error: ConnectionRefused connecting to api', calls)), + ).toThrow(/Anthropic API unreachable/); + }); + + test('a truthy-but-not-"1" flag still pings (no accidental widening)', () => { + const calls: SpawnCall[] = []; + const result = preflightAnthropicApi({ EVALS_PREFLIGHT_OK: 'true' }, fakeSpawn('ok', calls)); + expect(result).toBe('ok'); + expect(calls.length).toBe(1); + }); +}); diff --git a/test/helpers/anthropic-preflight.ts b/test/helpers/anthropic-preflight.ts new file mode 100644 index 000000000..718e6d8f5 --- /dev/null +++ b/test/helpers/anthropic-preflight.ts @@ -0,0 +1,35 @@ +/** + * Anthropic-API preflight ping, shared by e2e-helpers (module load in every + * paid test file) and the sharded paid runner (once, in the parent). + * + * The ping is a real `claude -p` call with a 30s timeout. Before the parent + * dedup, every one of the ~30 paid test files that import e2e-helpers fired + * it at module load — 30 paid pings per full sharded run for one bit of + * information. The sharded runner now pings ONCE and sets + * EVALS_PREFLIGHT_OK=1 in each shard's env; the module-load path honors the + * flag and skips. + * + * Lives in its own module (not e2e-helpers) so the runner can import it + * without dragging in bun:test. + */ + +import { spawnSync } from 'child_process'; + +export type PreflightResult = 'skipped' | 'ok'; + +export function preflightAnthropicApi( + env: NodeJS.ProcessEnv = process.env, + spawn: typeof spawnSync = spawnSync, +): PreflightResult { + if (env.EVALS_PREFLIGHT_OK === '1') return 'skipped'; + const check = spawn( + 'sh', + ['-c', 'echo "ping" | claude -p --max-turns 1 --output-format stream-json --verbose --dangerously-skip-permissions'], + { stdio: 'pipe', timeout: 30_000 }, + ); + const output = check.stdout?.toString() || ''; + if (output.includes('ConnectionRefused') || output.includes('Unable to connect')) { + throw new Error('Anthropic API unreachable — aborting E2E suite. Fix connectivity and retry.'); + } + return 'ok'; +} diff --git a/test/helpers/e2e-helpers.ts b/test/helpers/e2e-helpers.ts index 32510f13a..0f472c946 100644 --- a/test/helpers/e2e-helpers.ts +++ b/test/helpers/e2e-helpers.ts @@ -15,6 +15,7 @@ import { selectTests, detectBaseBranch, getChangedFiles, E2E_TOUCHFILES, E2E_TIE import { WorktreeManager } from '../../lib/worktree'; import type { HarvestResult } from '../../lib/worktree'; import { spawnSync } from 'child_process'; +import { preflightAnthropicApi } from './anthropic-preflight'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; @@ -260,15 +261,12 @@ if (evalsEnabled) { } } -// Fail fast if Anthropic API is unreachable — don't burn through tests getting ConnectionRefused +// Fail fast if Anthropic API is unreachable — don't burn through tests getting +// ConnectionRefused. The sharded paid runner pings once in the parent and sets +// EVALS_PREFLIGHT_OK=1 for its children, so per-file module loads skip this +// (was: ~30 paid pings per full sharded run, one per importing file). if (evalsEnabled) { - const check = spawnSync('sh', ['-c', 'echo "ping" | claude -p --max-turns 1 --output-format stream-json --verbose --dangerously-skip-permissions'], { - stdio: 'pipe', timeout: 30_000, - }); - const output = check.stdout?.toString() || ''; - if (output.includes('ConnectionRefused') || output.includes('Unable to connect')) { - throw new Error('Anthropic API unreachable — aborting E2E suite. Fix connectivity and retry.'); - } + preflightAnthropicApi(); } /** Skip an individual test if not selected (for multi-test describe blocks). */