diff --git a/test/helpers/e2e-helpers.ts b/test/helpers/e2e-helpers.ts index 63ba664cb..4b9e1a747 100644 --- a/test/helpers/e2e-helpers.ts +++ b/test/helpers/e2e-helpers.ts @@ -229,6 +229,18 @@ export function createEvalCollector(suite: string): EvalCollector | null { } /** DRY helper to record an E2E test result into the eval collector. */ +/** Exit reasons for an API or transport failure (session-runner.ts). */ +const INFRA_EXIT_REASONS = new Set(['error_api', 'timeout_startup', 'error_output_stream']); + +/** API/transport error or CLI crash before the first model turn: INFRA, never a + * verdict on the product. Any assistant event or counted turn means the model + * ran, so its refusal, timeout or wrong answer stays an ordinary failure. */ +export function isPreTurnInfraFailure(result: Pick): boolean { + return result.costEstimate.turnsUsed === 0 + && (INFRA_EXIT_REASONS.has(result.exitReason) || /^exit_code_\d+$/.test(result.exitReason)) + && !result.transcript.some(event => event?.type === 'assistant'); +} + export function recordE2E( evalCollector: EvalCollector | null, name: string, @@ -241,9 +253,11 @@ export function recordE2E( ? `${result.toolCalls[result.toolCalls.length - 1].tool}(${JSON.stringify(result.toolCalls[result.toolCalls.length - 1].input).slice(0, 60)})` : undefined; + const passed = extra?.passed ?? (result.exitReason === 'success' && result.browseErrors.length === 0); evalCollector?.addTest({ name, suite, tier: 'e2e', - passed: result.exitReason === 'success' && result.browseErrors.length === 0, + passed, + ...(!passed && isPreTurnInfraFailure(result) ? { failure_class: 'infra' as const } : {}), duration_ms: result.duration, cost_usd: result.costEstimate.estimatedCost, transcript: result.transcript, diff --git a/test/plan-review-report-recording.test.ts b/test/plan-review-report-recording.test.ts index d6e6c5f8f..28354d1d6 100644 --- a/test/plan-review-report-recording.test.ts +++ b/test/plan-review-report-recording.test.ts @@ -4,7 +4,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { CAPTURE_LONG_MS } from './helpers/eval-budgets'; -import { recordE2E } from './helpers/e2e-helpers'; +import { isPreTurnInfraFailure, recordE2E } from './helpers/e2e-helpers'; import { EvalCollector, isFinalizedEvalResultFile, listEvalJsonFiles, type EvalTestEntry } from './helpers/eval-store'; import { OFFICE_HOURS_BUN_GRACE_MS, runRecordedOfficeHoursAttempt } from './helpers/office-hours-attempt'; import { isPaidTestFile } from './helpers/paid-test-set'; @@ -257,3 +257,37 @@ test('report deadline aborts, records once, cleans up and ignores late completio test('report recording controls stay outside the paid test filename patterns', () => { expect(isPaidTestFile('test/plan-review-report-recording.test.ts')).toBe(false); }); + +// A pre-turn API/transport failure is INFRA; once the model has run, a failure +// keeps its ordinary class. +function runnerResult(exitReason: string, turnsUsed = 0, transcript: any[] = [{ type: 'system', subtype: 'init' }]): any { + return { exitReason, transcript, toolCalls: [], browseErrors: [], duration: 1, output: '', + costEstimate: { inputChars: 1, outputChars: 0, estimatedTokens: 0, estimatedCost: 0, turnsUsed } }; +} +function recordedClass(result: any, extra?: Partial) { + const collector = new EvalCollector('e2e'); + recordE2E(collector, 'infra-probe', 'Infra probe', result, extra); + return (collector as any).tests.at(-1)?.failure_class; +} + +test.each(['error_api', 'timeout_startup', 'error_output_stream', 'exit_code_1'])( + 'a %s before the first model turn is recorded as infra', exitReason => { + expect(isPreTurnInfraFailure(runnerResult(exitReason))).toBe(true); + expect(recordedClass(runnerResult(exitReason))).toBe('infra'); + }); + +test.each([ + ['a timeout after model work', runnerResult('timeout')], + ['max turns', runnerResult('error_max_turns', 24)], + ['an API error after a turn', runnerResult('error_api', 3)], + ['an API error after an assistant message', runnerResult('error_api', 0, [{ type: 'assistant', message: { content: [{ type: 'text', text: 'I cannot help with that.' }] } }])], + ['a successful run', runnerResult('success', 2)], +] as const)('%s is not infra', (_name, result) => { + expect(isPreTurnInfraFailure(result)).toBe(false); + expect(recordedClass(result)).toBeUndefined(); +}); + +test('an explicit pass or class from the caller wins', () => { + expect(recordedClass(runnerResult('error_api'), { passed: true })).toBeUndefined(); + expect(recordedClass(runnerResult('error_api'), { failure_class: 'assertion' })).toBe('assertion'); +});