fix: fail fast on API connectivity — pre-check before E2E suite

Spawn a quick claude -p ping before running 13 tests. If the Anthropic API
is unreachable (ConnectionRefused), throw immediately instead of burning
through the entire suite with silent false passes.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-03-14 12:37:44 -05:00
parent 5aae3ce117
commit 9f5aa32e67
+12
View File
@@ -5,6 +5,7 @@ import { outcomeJudge } from './helpers/llm-judge';
import { EvalCollector } from './helpers/eval-store';
import type { EvalTestEntry } from './helpers/eval-store';
import { startTestServer } from '../browse/test/test-server';
import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
@@ -116,6 +117,17 @@ function dumpOutcomeDiagnostic(dir: string, label: string, report: string, judge
} catch { /* non-fatal */ }
}
// Fail fast if Anthropic API is unreachable — don't burn through 13 tests getting ConnectionRefused
if (evalsEnabled) {
const check = spawnSync('sh', ['-c', 'echo "ping" | claude -p --max-turns 1 --output-format stream-json --verbose --dangerously-skip-permissions'], {
stdio: 'pipe', timeout: 30_000,
});
const output = check.stdout?.toString() || '';
if (output.includes('ConnectionRefused') || output.includes('Unable to connect')) {
throw new Error('Anthropic API unreachable — aborting E2E suite. Fix connectivity and retry.');
}
}
describeE2E('Skill E2E tests', () => {
beforeAll(() => {
testServer = startTestServer();