test(evals): record a pre-turn API or CLI failure as infra

recordE2E sets failure_class 'infra' on a failed session whose runner
reports error_api, timeout_startup, error_output_stream or a non-zero CLI
exit with zero turns and no assistant event. A model refusal, a timeout
after model work, max turns, or an explicit caller pass/class keeps its
ordinary classification.

test/helpers/e2e-helpers.ts

@@ -229,6 +229,18 @@ export function createEvalCollector(suite: string): EvalCollector | null {
 }
 
 /** DRY helper to record an E2E test result into the eval collector. */
+/** Exit reasons for an API or transport failure (session-runner.ts). */
+const INFRA_EXIT_REASONS = new Set(['error_api', 'timeout_startup', 'error_output_stream']);
+
+/** API/transport error or CLI crash before the first model turn: INFRA, never a
+ *  verdict on the product. Any assistant event or counted turn means the model
+ *  ran, so its refusal, timeout or wrong answer stays an ordinary failure. */
+export function isPreTurnInfraFailure(result: Pick<SkillTestResult, 'exitReason' | 'transcript' | 'costEstimate'>): boolean {
+  return result.costEstimate.turnsUsed === 0
+    && (INFRA_EXIT_REASONS.has(result.exitReason) || /^exit_code_\d+$/.test(result.exitReason))
+    && !result.transcript.some(event => event?.type === 'assistant');
+}
+
 export function recordE2E(
   evalCollector: EvalCollector | null,
   name: string,
@@ -241,9 +253,11 @@ export function recordE2E(
     ? `${result.toolCalls[result.toolCalls.length - 1].tool}(${JSON.stringify(result.toolCalls[result.toolCalls.length - 1].input).slice(0, 60)})`
     : undefined;
 
+  const passed = extra?.passed ?? (result.exitReason === 'success' && result.browseErrors.length === 0);
   evalCollector?.addTest({
     name, suite, tier: 'e2e',
-    passed: result.exitReason === 'success' && result.browseErrors.length === 0,
+    passed,
+    ...(!passed && isPreTurnInfraFailure(result) ? { failure_class: 'infra' as const } : {}),
     duration_ms: result.duration,
     cost_usd: result.costEstimate.estimatedCost,
     transcript: result.transcript,

test/plan-review-report-recording.test.ts

@@ -4,7 +4,7 @@ import * as fs from 'node:fs';
 import * as os from 'node:os';
 import * as path from 'node:path';
 import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
-import { recordE2E } from './helpers/e2e-helpers';
+import { isPreTurnInfraFailure, recordE2E } from './helpers/e2e-helpers';
 import { EvalCollector, isFinalizedEvalResultFile, listEvalJsonFiles, type EvalTestEntry } from './helpers/eval-store';
 import { OFFICE_HOURS_BUN_GRACE_MS, runRecordedOfficeHoursAttempt } from './helpers/office-hours-attempt';
 import { isPaidTestFile } from './helpers/paid-test-set';
@@ -257,3 +257,37 @@ test('report deadline aborts, records once, cleans up and ignores late completio
 test('report recording controls stay outside the paid test filename patterns', () => {
   expect(isPaidTestFile('test/plan-review-report-recording.test.ts')).toBe(false);
 });
+
+// A pre-turn API/transport failure is INFRA; once the model has run, a failure
+// keeps its ordinary class.
+function runnerResult(exitReason: string, turnsUsed = 0, transcript: any[] = [{ type: 'system', subtype: 'init' }]): any {
+  return { exitReason, transcript, toolCalls: [], browseErrors: [], duration: 1, output: '',
+    costEstimate: { inputChars: 1, outputChars: 0, estimatedTokens: 0, estimatedCost: 0, turnsUsed } };
+}
+function recordedClass(result: any, extra?: Partial<EvalTestEntry>) {
+  const collector = new EvalCollector('e2e');
+  recordE2E(collector, 'infra-probe', 'Infra probe', result, extra);
+  return (collector as any).tests.at(-1)?.failure_class;
+}
+
+test.each(['error_api', 'timeout_startup', 'error_output_stream', 'exit_code_1'])(
+  'a %s before the first model turn is recorded as infra', exitReason => {
+    expect(isPreTurnInfraFailure(runnerResult(exitReason))).toBe(true);
+    expect(recordedClass(runnerResult(exitReason))).toBe('infra');
+  });
+
+test.each([
+  ['a timeout after model work', runnerResult('timeout')],
+  ['max turns', runnerResult('error_max_turns', 24)],
+  ['an API error after a turn', runnerResult('error_api', 3)],
+  ['an API error after an assistant message', runnerResult('error_api', 0, [{ type: 'assistant', message: { content: [{ type: 'text', text: 'I cannot help with that.' }] } }])],
+  ['a successful run', runnerResult('success', 2)],
+] as const)('%s is not infra', (_name, result) => {
+  expect(isPreTurnInfraFailure(result)).toBe(false);
+  expect(recordedClass(result)).toBeUndefined();
+});
+
+test('an explicit pass or class from the caller wins', () => {
+  expect(recordedClass(runnerResult('error_api'), { passed: true })).toBeUndefined();
+  expect(recordedClass(runnerResult('error_api'), { failure_class: 'assertion' })).toBe('assertion');
+});