From 694e970c65ada9c678da317044fb640ad3e3af65 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 4 Oct 2026 18:14:21 +0000 Subject: [PATCH] 0.2: failureCauseOf takes the stall window instead of importing it; the PTY ledger never throws eval-store.ts imported STALL_WINDOW_MS from eval-budgets.ts, which free tests replace with mock.module stubs (review-finalization-budget, office-hours, qa-fix-loop, qa-only), so loading eval-store failed there. failureCauseOf is now dependency-free: the record builders pass STALL_WINDOW_MS. A PTY session whose handle cannot show its text (test doubles) records no panel instead of throwing from runPtySession's cleanup. The 37186854666 replay also runs through classifyTrialShard. --- scripts/lib/paid-report.ts | 4 ++-- scripts/test-paid-shards.ts | 4 ++-- test/helpers/eval-store.ts | 9 +++++---- test/helpers/pty/session.ts | 7 ++++++- test/trial-cause.test.ts | 9 +++++++-- 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index eaf41c7ce..a640d77a5 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -28,7 +28,7 @@ import { } from './shard-engine'; import { PAID_TEST_GLOBS, isPaidTestFile } from '../../test/helpers/paid-test-set'; import { CASE_CI_EXCLUDE, CASE_QUARANTINE, EVAL_POLICY, PERIODIC_CI_EXCLUDE } from '../../test/helpers/periodic-exclude-data'; -import { FILE_RETRY_BUDGETS, STRICT_RETRY_CASE_BUDGETS } from '../../test/helpers/eval-budgets'; +import { FILE_RETRY_BUDGETS, STALL_WINDOW_MS, STRICT_RETRY_CASE_BUDGETS } from '../../test/helpers/eval-budgets'; import { getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE, @@ -305,7 +305,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact const error = sanitizeTrialError(failedRecord?.error ?? failed?.message); const sessions = caseSessions(ledger, id, byCase.size === 1); const diagnosis = failed && failureClass ? trialFailureFields({ failure_class: failureClass, exit_reason: failedRecord?.exit_reason, - error: failedRecord?.error ?? failed.message, sessions, record: failedRecord }) : undefined; + error: failedRecord?.error ?? failed.message, sessions, record: failedRecord }, STALL_WINDOW_MS) : undefined; if (failed) census.failedShards.add(key); if (caseOutcome === 'skipped') { bump(reasons, RUNTIME_SKIP_REASON); diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index 24882ab4b..8da9070cf 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -78,7 +78,7 @@ import { } from './lib/shard-engine'; import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set'; import { CASE_CI_EXCLUDE, CASE_QUARANTINE, EVAL_POLICY, PERIODIC_CI_EXCLUDE } from '../test/helpers/periodic-exclude-data'; -import { FILE_RETRY_BUDGETS, STRICT_RETRY_CASE_BUDGETS } from '../test/helpers/eval-budgets'; +import { FILE_RETRY_BUDGETS, STALL_WINDOW_MS, STRICT_RETRY_CASE_BUDGETS } from '../test/helpers/eval-budgets'; import { getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE, @@ -769,7 +769,7 @@ export function classifyTrialShard( ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}), ...(Number.isInteger(failedRecord?.timeout_at_turn) ? { timeout_at_turn: failedRecord.timeout_at_turn } : {}), ...(sanitizeTrialError(raw) ? { error: sanitizeTrialError(raw) } : {}), - ...trialFailureFields({ failure_class: cls, exit_reason: failedRecord?.exit_reason, error: raw, sessions: evidence.sessions, record: failedRecord }) }; + ...trialFailureFields({ failure_class: cls, exit_reason: failedRecord?.exit_reason, error: raw, sessions: evidence.sessions, record: failedRecord }, STALL_WINDOW_MS) }; }; if (outcome.runnerError !== undefined) return { ...base, harness: `runner error: ${sanitizeTrialError(outcome.runnerError) ?? 'unknown'}` }; if (outcome.status === 'never-started') return { ...base, harness: 'never started' }; diff --git a/test/helpers/eval-store.ts b/test/helpers/eval-store.ts index cacdc731d..bc135ddcf 100644 --- a/test/helpers/eval-store.ts +++ b/test/helpers/eval-store.ts @@ -14,7 +14,6 @@ import * as os from 'os'; import { spawnSync } from 'child_process'; import { isManualReviewEntry } from './cookie-workflow-manual-review'; import type { ManualJudgeReview } from './cookie-workflow-manual-review'; -import { STALL_WINDOW_MS } from './eval-budgets'; // v2: EvalTestEntry.harvest gains optional {insertions, deletions, net} and // may be explicitly null (arm-benchmark harvest-failure taxonomy). Readers @@ -256,8 +255,10 @@ const oneLine = (text: string | undefined, max: number): string | undefined => { * A provider stall needs a session that streamed partial messages, did not * complete, and shows STALL_WINDOW_MS of silence with a request in flight and * nothing outstanding; sessions without partial messages are never stalls. + * The window is STALL_WINDOW_MS (eval-budgets.ts), passed in so this stays + * dependency-free (tests that mock eval-budgets still load eval-store). */ -export function failureCauseOf(facts: FailureCauseFacts, stallWindowMs = STALL_WINDOW_MS): { cause: TrialFailureCause; evidence?: string } { +export function failureCauseOf(facts: FailureCauseFacts, stallWindowMs: number): { cause: TrialFailureCause; evidence?: string } { const sessions = facts.sessions ?? []; const ended = (end: string) => sessions.find((s) => s.end === end); const out = (cause: TrialFailureCause, evidence?: string) => { @@ -361,10 +362,10 @@ export function trialCostKnown(records: ReadonlyArray<{ cost_known?: unknown }>, } /** Cause, cause evidence and detail of one failed trial, for both record builders. */ -export function trialFailureFields(input: FailureCauseFacts & { record?: { judge_scores?: unknown; judge_reasoning?: unknown } }): { +export function trialFailureFields(input: FailureCauseFacts & { record?: { judge_scores?: unknown; judge_reasoning?: unknown } }, stallWindowMs: number): { failure_cause: TrialFailureCause; failure_cause_evidence?: string; failure_detail?: TrialFailureDetail; } { - const { cause, evidence } = failureCauseOf(input); + const { cause, evidence } = failureCauseOf(input, stallWindowMs); const detail = failureDetailOf(input.error, input.record); return { failure_cause: cause, ...(evidence ? { failure_cause_evidence: evidence } : {}), ...(detail ? { failure_detail: detail } : {}) }; } diff --git a/test/helpers/pty/session.ts b/test/helpers/pty/session.ts index 11bd57473..58533cea9 100644 --- a/test/helpers/pty/session.ts +++ b/test/helpers/pty/session.ts @@ -33,6 +33,11 @@ export const realPtyDriver: PtyDriver = { sleep: ms => Bun.sleep(ms), }; +/** The last idle turn's error panel, if the session can show its text; diagnostics never throw. */ +export function safePanelEnd(session: Pick): ReturnType { + try { return idlePanelEnd(session.visibleText()); } catch { return null; } +} + /** One poll step: finish with a result, keep polling, or end the loop as a timeout. */ export type PtyStep = { done: R } | 'continue' | 'break'; @@ -92,7 +97,7 @@ export async function runPtySession(plan: PtySessionPlan): Promise { } finally { appendSessionLedger({ key: sessionKey('pty', undefined), runner: 'pty', started_at: startedAt, budget_ms: plan.budgetMs ?? plan.launch.timeoutMs ?? 240_000, elapsed_ms: plan.driver.monotonic() - started, - end, ...idlePanelEnd(session.visibleText()), billed: false }); + end, ...safePanelEnd(session), billed: false }); try { try { plan.beforeClose?.(session, failure); } catch (error) { if (!failure) { failure = { error }; throw error; } } diff --git a/test/trial-cause.test.ts b/test/trial-cause.test.ts index 8b19ddab2..1dbd9226a 100644 --- a/test/trial-cause.test.ts +++ b/test/trial-cause.test.ts @@ -15,6 +15,7 @@ import { runPlanSkillObservation } from './helpers/claude-pty-runner'; import { createFakePtyDriver } from './helpers/pty/fake-session'; import { runAgentSdkTest, __resetSemaphoreForTests, type QueryProvider } from './helpers/agent-sdk-runner'; import { runRecordedCodexEval } from './helpers/codex-eval'; +import { classifyTrialShard } from '../scripts/test-paid-shards'; const FIXTURES = path.join(import.meta.dir, 'fixtures', 'trial-cause'); const thinking = JSON.parse(fs.readFileSync(path.join(FIXTURES, 'thinking-captures-37198445662.json'), 'utf8')); @@ -49,7 +50,7 @@ function observe(events: Timed[], endAt: number, hooks: { permissionAt?: number } const timedOutSession = (liveness: SessionCauseFacts['liveness']): SessionCauseFacts => ({ key: 'claude-p:x#1', end: 'session_timeout', elapsed_ms: 600_000, budget_ms: 600_000, liveness }); -const causeOf = (facts: Partial) => failureCauseOf({ failure_class: 'assertion', ...facts }); +const causeOf = (facts: Partial) => failureCauseOf({ failure_class: 'assertion', ...facts }, STALL_WINDOW_MS); describe('failureCauseOf precedence', () => { const stalled = { partial: true, max_request_silence_ms: STALL_WINDOW_MS + 10_000, silence_started_ms: 40_000, silence_after: 'thinking_delta' }; @@ -167,6 +168,10 @@ describe('replays of stored census evidence', () => { expect(liveness.partial).toBe(false); expect(liveness.max_request_silence_ms).toBe(0); expect(causeOf({ failure_class: r.failure_class, exit_reason: r.exit_reason, error: r.error }).cause).toBe('session_timeout'); + // Through the record builder the trial shard uses: failure_class stays timeout, the cause is the session timeout. + expect(classifyTrialShard({ status: 'failed', executedTests: 1, skippedTests: 0, elapsedMs: r.duration_ms }, r.case, 1, + { kind: 'rule', panel: { n: 1, k: 1 }, quarantined: false }, { records: [{ passed: false, exit_reason: r.exit_reason, error: r.error }], contract: null })) + .toMatchObject({ outcome: 'failed', failure_class: 'timeout', failure_cause: 'session_timeout', error: 'Error: Claude Code process aborted by user' }); expect(causeOf({ failure_class: r.failure_class, exit_reason: r.exit_reason, error: r.error, sessions: [{ key: 'agent-sdk:x#1', end: 'session_timeout', elapsed_ms: r.duration_ms, budget_ms: 300_000, liveness }] }).cause).toBe('session_timeout'); }); @@ -232,7 +237,7 @@ describe('one fixture per runner writes the ledger the classifier reads', () => { type: 'stream_event', event: { type: 'content_block_delta', delta: { type: 'thinking_delta' } } }]); expect(streaming.rows[0]!.liveness).toMatchObject({ partial: true, silence_after: 'thinking_delta' }); expect(failureCauseOf({ failure_class: 'timeout', sessions: streaming.rows }, 300).cause).toBe('provider_stall'); - expect(failureCauseOf({ failure_class: 'timeout', sessions: streaming.rows }).cause).toBe('session_timeout'); + expect(failureCauseOf({ failure_class: 'timeout', sessions: streaming.rows }, STALL_WINDOW_MS).cause).toBe('session_timeout'); }); test('Codex: an expired budget is a session timeout', async () => {