mirror of
https://github.com/garrytan/gstack.git
synced 2026-08-22 14:07:14 +02:00
evals: preflight ping once in the sharded parent, not per shard
The Anthropic fail-fast ping ran at module load in every paid test file importing e2e-helpers — ~30 paid claude -p calls (30s timeout each) per full sharded run for one bit of information. The parent now pings once before spawning shards and sets EVALS_PREFLIGHT_OK=1; the module-load path honors the flag. Extracted to test/helpers/anthropic-preflight.ts (injectable spawn seam) with regression pins in both directions: the flag must skip, its absence must ping exactly once, dead API must throw. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
f9e3fc8ab4
commit
6d109d3232
@@ -0,0 +1,53 @@
|
||||
/**
|
||||
* Regression pins for the preflight-dedup seam (test/helpers/anthropic-preflight.ts).
|
||||
*
|
||||
* The preflight runs at MODULE LOAD in every paid test file that imports
|
||||
* e2e-helpers, so a broken skip-check either brings back ~30 paid pings per
|
||||
* sharded run or — worse — skips the fail-fast everywhere. Both directions
|
||||
* are pinned here with an injected spawn; no real claude call is made.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { preflightAnthropicApi } from './helpers/anthropic-preflight';
|
||||
|
||||
type SpawnCall = { command: string; args: string[] };
|
||||
|
||||
function fakeSpawn(stdout: string, calls: SpawnCall[]) {
|
||||
return ((command: string, args: string[]) => {
|
||||
calls.push({ command, args });
|
||||
return { stdout: Buffer.from(stdout), stderr: Buffer.from(''), status: 0 } as ReturnType<
|
||||
typeof import('child_process').spawnSync
|
||||
>;
|
||||
}) as typeof import('child_process').spawnSync;
|
||||
}
|
||||
|
||||
describe('preflightAnthropicApi', () => {
|
||||
test('EVALS_PREFLIGHT_OK=1 skips the ping entirely (sharded-child path)', () => {
|
||||
const calls: SpawnCall[] = [];
|
||||
const result = preflightAnthropicApi({ EVALS_PREFLIGHT_OK: '1' }, fakeSpawn('never read', calls));
|
||||
expect(result).toBe('skipped');
|
||||
expect(calls.length).toBe(0);
|
||||
});
|
||||
|
||||
test('without the flag, pings exactly once and passes on healthy output', () => {
|
||||
const calls: SpawnCall[] = [];
|
||||
const result = preflightAnthropicApi({}, fakeSpawn('{"type":"result"}', calls));
|
||||
expect(result).toBe('ok');
|
||||
expect(calls.length).toBe(1);
|
||||
expect(calls[0].args.join(' ')).toContain('claude -p');
|
||||
});
|
||||
|
||||
test('unreachable API throws (fail-fast before any shard spawns)', () => {
|
||||
const calls: SpawnCall[] = [];
|
||||
expect(() =>
|
||||
preflightAnthropicApi({}, fakeSpawn('error: ConnectionRefused connecting to api', calls)),
|
||||
).toThrow(/Anthropic API unreachable/);
|
||||
});
|
||||
|
||||
test('a truthy-but-not-"1" flag still pings (no accidental widening)', () => {
|
||||
const calls: SpawnCall[] = [];
|
||||
const result = preflightAnthropicApi({ EVALS_PREFLIGHT_OK: 'true' }, fakeSpawn('ok', calls));
|
||||
expect(result).toBe('ok');
|
||||
expect(calls.length).toBe(1);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user