mirror of
https://github.com/garrytan/gstack.git
synced 2026-08-21 21:47:32 +02:00
Garry TanandClaude Fable 5
6d109d3232
The Anthropic fail-fast ping ran at module load in every paid test file importing e2e-helpers — ~30 paid claude -p calls (30s timeout each) per full sharded run for one bit of information. The parent now pings once before spawning shards and sets EVALS_PREFLIGHT_OK=1; the module-load path honors the flag. Extracted to test/helpers/anthropic-preflight.ts (injectable spawn seam) with regression pins in both directions: the flag must skip, its absence must ping exactly once, dead API must throw. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
54 lines
2.1 KiB
TypeScript
54 lines
2.1 KiB
TypeScript
/**
|
|
* Regression pins for the preflight-dedup seam (test/helpers/anthropic-preflight.ts).
|
|
*
|
|
* The preflight runs at MODULE LOAD in every paid test file that imports
|
|
* e2e-helpers, so a broken skip-check either brings back ~30 paid pings per
|
|
* sharded run or — worse — skips the fail-fast everywhere. Both directions
|
|
* are pinned here with an injected spawn; no real claude call is made.
|
|
*/
|
|
|
|
import { describe, test, expect } from 'bun:test';
|
|
import { preflightAnthropicApi } from './helpers/anthropic-preflight';
|
|
|
|
type SpawnCall = { command: string; args: string[] };
|
|
|
|
function fakeSpawn(stdout: string, calls: SpawnCall[]) {
|
|
return ((command: string, args: string[]) => {
|
|
calls.push({ command, args });
|
|
return { stdout: Buffer.from(stdout), stderr: Buffer.from(''), status: 0 } as ReturnType<
|
|
typeof import('child_process').spawnSync
|
|
>;
|
|
}) as typeof import('child_process').spawnSync;
|
|
}
|
|
|
|
describe('preflightAnthropicApi', () => {
|
|
test('EVALS_PREFLIGHT_OK=1 skips the ping entirely (sharded-child path)', () => {
|
|
const calls: SpawnCall[] = [];
|
|
const result = preflightAnthropicApi({ EVALS_PREFLIGHT_OK: '1' }, fakeSpawn('never read', calls));
|
|
expect(result).toBe('skipped');
|
|
expect(calls.length).toBe(0);
|
|
});
|
|
|
|
test('without the flag, pings exactly once and passes on healthy output', () => {
|
|
const calls: SpawnCall[] = [];
|
|
const result = preflightAnthropicApi({}, fakeSpawn('{"type":"result"}', calls));
|
|
expect(result).toBe('ok');
|
|
expect(calls.length).toBe(1);
|
|
expect(calls[0].args.join(' ')).toContain('claude -p');
|
|
});
|
|
|
|
test('unreachable API throws (fail-fast before any shard spawns)', () => {
|
|
const calls: SpawnCall[] = [];
|
|
expect(() =>
|
|
preflightAnthropicApi({}, fakeSpawn('error: ConnectionRefused connecting to api', calls)),
|
|
).toThrow(/Anthropic API unreachable/);
|
|
});
|
|
|
|
test('a truthy-but-not-"1" flag still pings (no accidental widening)', () => {
|
|
const calls: SpawnCall[] = [];
|
|
const result = preflightAnthropicApi({ EVALS_PREFLIGHT_OK: 'true' }, fakeSpawn('ok', calls));
|
|
expect(result).toBe('ok');
|
|
expect(calls.length).toBe(1);
|
|
});
|
|
});
|