evals: preflight ping once in the sharded parent, not per shard

The Anthropic fail-fast ping ran at module load in every paid test file
importing e2e-helpers — ~30 paid claude -p calls (30s timeout each) per
full sharded run for one bit of information. The parent now pings once
before spawning shards and sets EVALS_PREFLIGHT_OK=1; the module-load
path honors the flag. Extracted to test/helpers/anthropic-preflight.ts
(injectable spawn seam) with regression pins in both directions: the
flag must skip, its absence must ping exactly once, dead API must throw.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

scripts/test-paid-shards.ts

@@ -64,6 +64,7 @@ import {
 } from './test-strict-output';
 import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set';
 import { getProjectEvalDir } from '../test/helpers/eval-store';
+import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight';
 
 export { PAID_TEST_GLOBS, isPaidTestFile };
 
@@ -463,13 +464,19 @@ async function main(): Promise<number> {
     return 0;
   }
 
+  // One preflight ping in the parent; children skip theirs via the env flag.
+  // Before this, every shard's e2e-helpers module load re-pinged the API —
+  // ~30 paid claude -p calls (30s timeout each) per full run for one bit of
+  // information. A dead API now fails here, before any shard spawns.
+  preflightAnthropicApi(process.env);
+
   const summary = await runPaidShards(shards, {
     // Tier reaches the children only via EVALS_TIER below; the runtime
     // E2E_TIERS filter inside each child is the real selection mechanism.
     timeoutMs: options.timeoutMs,
     jobs: options.jobs,
     withinShardConcurrency: options.withinShardConcurrency,
-    env: { ...process.env, EVALS: '1', EVALS_TIER: options.tier },
+    env: { ...process.env, EVALS: '1', EVALS_TIER: options.tier, EVALS_PREFLIGHT_OK: '1' },
     evalDirBase: process.env.GSTACK_EVAL_DIR || getProjectEvalDir(),
   });
   for (const line of formatSummary(summary)) console.log(line);

test/anthropic-preflight.test.ts

@@ -0,0 +1,53 @@
+/**
+ * Regression pins for the preflight-dedup seam (test/helpers/anthropic-preflight.ts).
+ *
+ * The preflight runs at MODULE LOAD in every paid test file that imports
+ * e2e-helpers, so a broken skip-check either brings back ~30 paid pings per
+ * sharded run or — worse — skips the fail-fast everywhere. Both directions
+ * are pinned here with an injected spawn; no real claude call is made.
+ */
+
+import { describe, test, expect } from 'bun:test';
+import { preflightAnthropicApi } from './helpers/anthropic-preflight';
+
+type SpawnCall = { command: string; args: string[] };
+
+function fakeSpawn(stdout: string, calls: SpawnCall[]) {
+  return ((command: string, args: string[]) => {
+    calls.push({ command, args });
+    return { stdout: Buffer.from(stdout), stderr: Buffer.from(''), status: 0 } as ReturnType<
+      typeof import('child_process').spawnSync
+    >;
+  }) as typeof import('child_process').spawnSync;
+}
+
+describe('preflightAnthropicApi', () => {
+  test('EVALS_PREFLIGHT_OK=1 skips the ping entirely (sharded-child path)', () => {
+    const calls: SpawnCall[] = [];
+    const result = preflightAnthropicApi({ EVALS_PREFLIGHT_OK: '1' }, fakeSpawn('never read', calls));
+    expect(result).toBe('skipped');
+    expect(calls.length).toBe(0);
+  });
+
+  test('without the flag, pings exactly once and passes on healthy output', () => {
+    const calls: SpawnCall[] = [];
+    const result = preflightAnthropicApi({}, fakeSpawn('{"type":"result"}', calls));
+    expect(result).toBe('ok');
+    expect(calls.length).toBe(1);
+    expect(calls[0].args.join(' ')).toContain('claude -p');
+  });
+
+  test('unreachable API throws (fail-fast before any shard spawns)', () => {
+    const calls: SpawnCall[] = [];
+    expect(() =>
+      preflightAnthropicApi({}, fakeSpawn('error: ConnectionRefused connecting to api', calls)),
+    ).toThrow(/Anthropic API unreachable/);
+  });
+
+  test('a truthy-but-not-"1" flag still pings (no accidental widening)', () => {
+    const calls: SpawnCall[] = [];
+    const result = preflightAnthropicApi({ EVALS_PREFLIGHT_OK: 'true' }, fakeSpawn('ok', calls));
+    expect(result).toBe('ok');
+    expect(calls.length).toBe(1);
+  });
+});

test/helpers/anthropic-preflight.ts

@@ -0,0 +1,35 @@
+/**
+ * Anthropic-API preflight ping, shared by e2e-helpers (module load in every
+ * paid test file) and the sharded paid runner (once, in the parent).
+ *
+ * The ping is a real `claude -p` call with a 30s timeout. Before the parent
+ * dedup, every one of the ~30 paid test files that import e2e-helpers fired
+ * it at module load — 30 paid pings per full sharded run for one bit of
+ * information. The sharded runner now pings ONCE and sets
+ * EVALS_PREFLIGHT_OK=1 in each shard's env; the module-load path honors the
+ * flag and skips.
+ *
+ * Lives in its own module (not e2e-helpers) so the runner can import it
+ * without dragging in bun:test.
+ */
+
+import { spawnSync } from 'child_process';
+
+export type PreflightResult = 'skipped' | 'ok';
+
+export function preflightAnthropicApi(
+  env: NodeJS.ProcessEnv = process.env,
+  spawn: typeof spawnSync = spawnSync,
+): PreflightResult {
+  if (env.EVALS_PREFLIGHT_OK === '1') return 'skipped';
+  const check = spawn(
+    'sh',
+    ['-c', 'echo "ping" | claude -p --max-turns 1 --output-format stream-json --verbose --dangerously-skip-permissions'],
+    { stdio: 'pipe', timeout: 30_000 },
+  );
+  const output = check.stdout?.toString() || '';
+  if (output.includes('ConnectionRefused') || output.includes('Unable to connect')) {
+    throw new Error('Anthropic API unreachable — aborting E2E suite. Fix connectivity and retry.');
+  }
+  return 'ok';
+}

test/helpers/e2e-helpers.ts

@@ -15,6 +15,7 @@ import { selectTests, detectBaseBranch, getChangedFiles, E2E_TOUCHFILES, E2E_TIE
 import { WorktreeManager } from '../../lib/worktree';
 import type { HarvestResult } from '../../lib/worktree';
 import { spawnSync } from 'child_process';
+import { preflightAnthropicApi } from './anthropic-preflight';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as os from 'os';
@@ -260,15 +261,12 @@ if (evalsEnabled) {
   }
 }
 
-// Fail fast if Anthropic API is unreachable — don't burn through tests getting ConnectionRefused
+// Fail fast if Anthropic API is unreachable — don't burn through tests getting
+// ConnectionRefused. The sharded paid runner pings once in the parent and sets
+// EVALS_PREFLIGHT_OK=1 for its children, so per-file module loads skip this
+// (was: ~30 paid pings per full sharded run, one per importing file).
 if (evalsEnabled) {
-  const check = spawnSync('sh', ['-c', 'echo "ping" | claude -p --max-turns 1 --output-format stream-json --verbose --dangerously-skip-permissions'], {
+  preflightAnthropicApi();
-    stdio: 'pipe', timeout: 30_000,
-  });
-  const output = check.stdout?.toString() || '';
-  if (output.includes('ConnectionRefused') || output.includes('Unable to connect')) {
-    throw new Error('Anthropic API unreachable — aborting E2E suite. Fix connectivity and retry.');
-  }
 }
 
 /** Skip an individual test if not selected (for multi-test describe blocks). */