0.2: failureCauseOf takes the stall window instead of importing it; the PTY ledger never throws

eval-store.ts imported STALL_WINDOW_MS from eval-budgets.ts, which free tests
replace with mock.module stubs (review-finalization-budget, office-hours,
qa-fix-loop, qa-only), so loading eval-store failed there. failureCauseOf is
now dependency-free: the record builders pass STALL_WINDOW_MS. A PTY session
whose handle cannot show its text (test doubles) records no panel instead of
throwing from runPtySession's cleanup. The 37186854666 replay also runs
through classifyTrialShard.
This commit is contained in:
garrytan committed 2026-10-04 18:14:21 +00:00
1 parent 30d2904c66
commit 694e970c65
5 files changed
+22 -11

No files matched your search

+2 -2
View File
@@ -28,7 +28,7 @@ import {
} from './shard-engine';
import { PAID_TEST_GLOBS, isPaidTestFile } from '../../test/helpers/paid-test-set';
import { CASE_CI_EXCLUDE, CASE_QUARANTINE, EVAL_POLICY, PERIODIC_CI_EXCLUDE } from '../../test/helpers/periodic-exclude-data';
import { FILE_RETRY_BUDGETS, STRICT_RETRY_CASE_BUDGETS } from '../../test/helpers/eval-budgets';
import { FILE_RETRY_BUDGETS, STALL_WINDOW_MS, STRICT_RETRY_CASE_BUDGETS } from '../../test/helpers/eval-budgets';
import {
getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict,
sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
@@ -305,7 +305,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
const error = sanitizeTrialError(failedRecord?.error ?? failed?.message);
const sessions = caseSessions(ledger, id, byCase.size === 1);
const diagnosis = failed && failureClass ? trialFailureFields({ failure_class: failureClass, exit_reason: failedRecord?.exit_reason,
error: failedRecord?.error ?? failed.message, sessions, record: failedRecord }) : undefined;
error: failedRecord?.error ?? failed.message, sessions, record: failedRecord }, STALL_WINDOW_MS) : undefined;
if (failed) census.failedShards.add(key);
if (caseOutcome === 'skipped') {
bump(reasons, RUNTIME_SKIP_REASON);
+2 -2
View File
@@ -78,7 +78,7 @@ import {
} from './lib/shard-engine';
import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set';
import { CASE_CI_EXCLUDE, CASE_QUARANTINE, EVAL_POLICY, PERIODIC_CI_EXCLUDE } from '../test/helpers/periodic-exclude-data';
import { FILE_RETRY_BUDGETS, STRICT_RETRY_CASE_BUDGETS } from '../test/helpers/eval-budgets';
import { FILE_RETRY_BUDGETS, STALL_WINDOW_MS, STRICT_RETRY_CASE_BUDGETS } from '../test/helpers/eval-budgets';
import {
getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict,
sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
@@ -769,7 +769,7 @@ export function classifyTrialShard(
...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
...(Number.isInteger(failedRecord?.timeout_at_turn) ? { timeout_at_turn: failedRecord.timeout_at_turn } : {}),
...(sanitizeTrialError(raw) ? { error: sanitizeTrialError(raw) } : {}),
...trialFailureFields({ failure_class: cls, exit_reason: failedRecord?.exit_reason, error: raw, sessions: evidence.sessions, record: failedRecord }) };
...trialFailureFields({ failure_class: cls, exit_reason: failedRecord?.exit_reason, error: raw, sessions: evidence.sessions, record: failedRecord }, STALL_WINDOW_MS) };
};
if (outcome.runnerError !== undefined) return { ...base, harness: `runner error: ${sanitizeTrialError(outcome.runnerError) ?? 'unknown'}` };
if (outcome.status === 'never-started') return { ...base, harness: 'never started' };
+5 -4
View File
@@ -14,7 +14,6 @@ import * as os from 'os';
import { spawnSync } from 'child_process';
import { isManualReviewEntry } from './cookie-workflow-manual-review';
import type { ManualJudgeReview } from './cookie-workflow-manual-review';
import { STALL_WINDOW_MS } from './eval-budgets';
// v2: EvalTestEntry.harvest gains optional {insertions, deletions, net} and
// may be explicitly null (arm-benchmark harvest-failure taxonomy). Readers
@@ -256,8 +255,10 @@ const oneLine = (text: string | undefined, max: number): string | undefined => {
* A provider stall needs a session that streamed partial messages, did not
* complete, and shows STALL_WINDOW_MS of silence with a request in flight and
* nothing outstanding; sessions without partial messages are never stalls.
* The window is STALL_WINDOW_MS (eval-budgets.ts), passed in so this stays
* dependency-free (tests that mock eval-budgets still load eval-store).
*/
export function failureCauseOf(facts: FailureCauseFacts, stallWindowMs = STALL_WINDOW_MS): { cause: TrialFailureCause; evidence?: string } {
export function failureCauseOf(facts: FailureCauseFacts, stallWindowMs: number): { cause: TrialFailureCause; evidence?: string } {
const sessions = facts.sessions ?? [];
const ended = (end: string) => sessions.find((s) => s.end === end);
const out = (cause: TrialFailureCause, evidence?: string) => {
@@ -361,10 +362,10 @@ export function trialCostKnown(records: ReadonlyArray<{ cost_known?: unknown }>,
}
/** Cause, cause evidence and detail of one failed trial, for both record builders. */
export function trialFailureFields(input: FailureCauseFacts & { record?: { judge_scores?: unknown; judge_reasoning?: unknown } }): {
export function trialFailureFields(input: FailureCauseFacts & { record?: { judge_scores?: unknown; judge_reasoning?: unknown } }, stallWindowMs: number): {
failure_cause: TrialFailureCause; failure_cause_evidence?: string; failure_detail?: TrialFailureDetail;
} {
const { cause, evidence } = failureCauseOf(input);
const { cause, evidence } = failureCauseOf(input, stallWindowMs);
const detail = failureDetailOf(input.error, input.record);
return { failure_cause: cause, ...(evidence ? { failure_cause_evidence: evidence } : {}), ...(detail ? { failure_detail: detail } : {}) };
}
+6 -1
View File
@@ -33,6 +33,11 @@ export const realPtyDriver: PtyDriver = {
sleep: ms => Bun.sleep(ms),
};
/** The last idle turn's error panel, if the session can show its text; diagnostics never throw. */
export function safePanelEnd(session: Pick<ClaudePtySession, 'visibleText'>): ReturnType<typeof idlePanelEnd> {
try { return idlePanelEnd(session.visibleText()); } catch { return null; }
}
/** One poll step: finish with a result, keep polling, or end the loop as a timeout. */
export type PtyStep<R> = { done: R } | 'continue' | 'break';
@@ -92,7 +97,7 @@ export async function runPtySession<R>(plan: PtySessionPlan<R>): Promise<R> {
} finally {
appendSessionLedger({ key: sessionKey('pty', undefined), runner: 'pty', started_at: startedAt,
budget_ms: plan.budgetMs ?? plan.launch.timeoutMs ?? 240_000, elapsed_ms: plan.driver.monotonic() - started,
end, ...idlePanelEnd(session.visibleText()), billed: false });
end, ...safePanelEnd(session), billed: false });
try {
try { plan.beforeClose?.(session, failure); }
catch (error) { if (!failure) { failure = { error }; throw error; } }
+7 -2
View File
@@ -15,6 +15,7 @@ import { runPlanSkillObservation } from './helpers/claude-pty-runner';
import { createFakePtyDriver } from './helpers/pty/fake-session';
import { runAgentSdkTest, __resetSemaphoreForTests, type QueryProvider } from './helpers/agent-sdk-runner';
import { runRecordedCodexEval } from './helpers/codex-eval';
import { classifyTrialShard } from '../scripts/test-paid-shards';
const FIXTURES = path.join(import.meta.dir, 'fixtures', 'trial-cause');
const thinking = JSON.parse(fs.readFileSync(path.join(FIXTURES, 'thinking-captures-37198445662.json'), 'utf8'));
@@ -49,7 +50,7 @@ function observe(events: Timed[], endAt: number, hooks: { permissionAt?: number
}
const timedOutSession = (liveness: SessionCauseFacts['liveness']): SessionCauseFacts => ({ key: 'claude-p:x#1', end: 'session_timeout', elapsed_ms: 600_000, budget_ms: 600_000, liveness });
const causeOf = (facts: Partial<FailureCauseFacts>) => failureCauseOf({ failure_class: 'assertion', ...facts });
const causeOf = (facts: Partial<FailureCauseFacts>) => failureCauseOf({ failure_class: 'assertion', ...facts }, STALL_WINDOW_MS);
describe('failureCauseOf precedence', () => {
const stalled = { partial: true, max_request_silence_ms: STALL_WINDOW_MS + 10_000, silence_started_ms: 40_000, silence_after: 'thinking_delta' };
@@ -167,6 +168,10 @@ describe('replays of stored census evidence', () => {
expect(liveness.partial).toBe(false);
expect(liveness.max_request_silence_ms).toBe(0);
expect(causeOf({ failure_class: r.failure_class, exit_reason: r.exit_reason, error: r.error }).cause).toBe('session_timeout');
// Through the record builder the trial shard uses: failure_class stays timeout, the cause is the session timeout.
expect(classifyTrialShard({ status: 'failed', executedTests: 1, skippedTests: 0, elapsedMs: r.duration_ms }, r.case, 1,
{ kind: 'rule', panel: { n: 1, k: 1 }, quarantined: false }, { records: [{ passed: false, exit_reason: r.exit_reason, error: r.error }], contract: null }))
.toMatchObject({ outcome: 'failed', failure_class: 'timeout', failure_cause: 'session_timeout', error: 'Error: Claude Code process aborted by user' });
expect(causeOf({ failure_class: r.failure_class, exit_reason: r.exit_reason, error: r.error,
sessions: [{ key: 'agent-sdk:x#1', end: 'session_timeout', elapsed_ms: r.duration_ms, budget_ms: 300_000, liveness }] }).cause).toBe('session_timeout');
});
@@ -232,7 +237,7 @@ describe('one fixture per runner writes the ledger the classifier reads', () =>
{ type: 'stream_event', event: { type: 'content_block_delta', delta: { type: 'thinking_delta' } } }]);
expect(streaming.rows[0]!.liveness).toMatchObject({ partial: true, silence_after: 'thinking_delta' });
expect(failureCauseOf({ failure_class: 'timeout', sessions: streaming.rows }, 300).cause).toBe('provider_stall');
expect(failureCauseOf({ failure_class: 'timeout', sessions: streaming.rows }).cause).toBe('session_timeout');
expect(failureCauseOf({ failure_class: 'timeout', sessions: streaming.rows }, STALL_WINDOW_MS).cause).toBe('session_timeout');
});
test('Codex: an expired budget is a session timeout', async () => {