CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-08-22 14:07:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

evals: preflight ping once in the sharded parent, not per shard

Browse Source 
The Anthropic fail-fast ping ran at module load in every paid test file
importing e2e-helpers — ~30 paid claude -p calls (30s timeout each) per
full sharded run for one bit of information. The parent now pings once
before spawning shards and sets EVALS_PREFLIGHT_OK=1; the module-load
path honors the flag. Extracted to test/helpers/anthropic-preflight.ts
(injectable spawn seam) with regression pins in both directions: the
flag must skip, its absence must ping exactly once, dead API must throw.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-15 08:13:30 -07:00
co-authored by Claude Fable 5
parent f9e3fc8ab4
commit 6d109d3232
4 changed files with 102 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/helpers/anthropic-preflight.ts
+35
View File
@@ -0,0 +1,35 @@
/**
* Anthropic-API preflight ping, shared by e2e-helpers (module load in every
* paid test file) and the sharded paid runner (once, in the parent).
*
* The ping is a real `claude -p` call with a 30s timeout. Before the parent
* dedup, every one of the ~30 paid test files that import e2e-helpers fired
* it at module load — 30 paid pings per full sharded run for one bit of
* information. The sharded runner now pings ONCE and sets
* EVALS_PREFLIGHT_OK=1 in each shard's env; the module-load path honors the
* flag and skips.
*
* Lives in its own module (not e2e-helpers) so the runner can import it
* without dragging in bun:test.
*/
import { spawnSync } from 'child_process';
export type PreflightResult = 'skipped' | 'ok';
export function preflightAnthropicApi(
env: NodeJS.ProcessEnv = process.env,
spawn: typeof spawnSync = spawnSync,
): PreflightResult {
if (env.EVALS_PREFLIGHT_OK === '1') return 'skipped';
const check = spawn(
'sh',
['-c', 'echo "ping" | claude -p --max-turns 1 --output-format stream-json --verbose --dangerously-skip-permissions'],
{ stdio: 'pipe', timeout: 30_000 },
);
const output = check.stdout?.toString() || '';
if (output.includes('ConnectionRefused') || output.includes('Unable to connect')) {
throw new Error('Anthropic API unreachable — aborting E2E suite. Fix connectivity and retry.');
}
return 'ok';
}
test/helpers/e2e-helpers.ts
+6 -8
View File
@@ -15,6 +15,7 @@ import { selectTests, detectBaseBranch, getChangedFiles, E2E_TOUCHFILES, E2E_TIE
import { WorktreeManager } from '../../lib/worktree';
import type { HarvestResult } from '../../lib/worktree';
import { spawnSync } from 'child_process';
import { preflightAnthropicApi } from './anthropic-preflight';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
@@ -260,15 +261,12 @@ if (evalsEnabled) {
}
}
// Fail fast if Anthropic API is unreachable — don't burn through tests getting ConnectionRefused
// Fail fast if Anthropic API is unreachable — don't burn through tests getting
// ConnectionRefused. The sharded paid runner pings once in the parent and sets
// EVALS_PREFLIGHT_OK=1 for its children, so per-file module loads skip this
// (was: ~30 paid pings per full sharded run, one per importing file).
if (evalsEnabled) {
const check = spawnSync('sh', ['-c', 'echo "ping" | claude -p --max-turns 1 --output-format stream-json --verbose --dangerously-skip-permissions'], {
stdio: 'pipe', timeout: 30_000,
});
const output = check.stdout?.toString() || '';
if (output.includes('ConnectionRefused') || output.includes('Unable to connect')) {
throw new Error('Anthropic API unreachable — aborting E2E suite. Fix connectivity and retry.');
}
preflightAnthropicApi();
}
/** Skip an individual test if not selected (for multi-test describe blocks). */