mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
0.2: failureCauseOf takes the stall window instead of importing it; the PTY ledger never throws
eval-store.ts imported STALL_WINDOW_MS from eval-budgets.ts, which free tests replace with mock.module stubs (review-finalization-budget, office-hours, qa-fix-loop, qa-only), so loading eval-store failed there. failureCauseOf is now dependency-free: the record builders pass STALL_WINDOW_MS. A PTY session whose handle cannot show its text (test doubles) records no panel instead of throwing from runPtySession's cleanup. The 37186854666 replay also runs through classifyTrialShard.
This commit is contained in:
1 parent
30d2904c66
commit
694e970c65
5 files changed
+22
-11
No files matched your search
@@ -28,7 +28,7 @@ import {
|
||||
} from './shard-engine';
|
||||
import { PAID_TEST_GLOBS, isPaidTestFile } from '../../test/helpers/paid-test-set';
|
||||
import { CASE_CI_EXCLUDE, CASE_QUARANTINE, EVAL_POLICY, PERIODIC_CI_EXCLUDE } from '../../test/helpers/periodic-exclude-data';
|
||||
import { FILE_RETRY_BUDGETS, STRICT_RETRY_CASE_BUDGETS } from '../../test/helpers/eval-budgets';
|
||||
import { FILE_RETRY_BUDGETS, STALL_WINDOW_MS, STRICT_RETRY_CASE_BUDGETS } from '../../test/helpers/eval-budgets';
|
||||
import {
|
||||
getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict,
|
||||
sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
|
||||
@@ -305,7 +305,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
|
||||
const error = sanitizeTrialError(failedRecord?.error ?? failed?.message);
|
||||
const sessions = caseSessions(ledger, id, byCase.size === 1);
|
||||
const diagnosis = failed && failureClass ? trialFailureFields({ failure_class: failureClass, exit_reason: failedRecord?.exit_reason,
|
||||
error: failedRecord?.error ?? failed.message, sessions, record: failedRecord }) : undefined;
|
||||
error: failedRecord?.error ?? failed.message, sessions, record: failedRecord }, STALL_WINDOW_MS) : undefined;
|
||||
if (failed) census.failedShards.add(key);
|
||||
if (caseOutcome === 'skipped') {
|
||||
bump(reasons, RUNTIME_SKIP_REASON);
|
||||
|
||||
@@ -78,7 +78,7 @@ import {
|
||||
} from './lib/shard-engine';
|
||||
import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set';
|
||||
import { CASE_CI_EXCLUDE, CASE_QUARANTINE, EVAL_POLICY, PERIODIC_CI_EXCLUDE } from '../test/helpers/periodic-exclude-data';
|
||||
import { FILE_RETRY_BUDGETS, STRICT_RETRY_CASE_BUDGETS } from '../test/helpers/eval-budgets';
|
||||
import { FILE_RETRY_BUDGETS, STALL_WINDOW_MS, STRICT_RETRY_CASE_BUDGETS } from '../test/helpers/eval-budgets';
|
||||
import {
|
||||
getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict,
|
||||
sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
|
||||
@@ -769,7 +769,7 @@ export function classifyTrialShard(
|
||||
...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
|
||||
...(Number.isInteger(failedRecord?.timeout_at_turn) ? { timeout_at_turn: failedRecord.timeout_at_turn } : {}),
|
||||
...(sanitizeTrialError(raw) ? { error: sanitizeTrialError(raw) } : {}),
|
||||
...trialFailureFields({ failure_class: cls, exit_reason: failedRecord?.exit_reason, error: raw, sessions: evidence.sessions, record: failedRecord }) };
|
||||
...trialFailureFields({ failure_class: cls, exit_reason: failedRecord?.exit_reason, error: raw, sessions: evidence.sessions, record: failedRecord }, STALL_WINDOW_MS) };
|
||||
};
|
||||
if (outcome.runnerError !== undefined) return { ...base, harness: `runner error: ${sanitizeTrialError(outcome.runnerError) ?? 'unknown'}` };
|
||||
if (outcome.status === 'never-started') return { ...base, harness: 'never started' };
|
||||
|
||||
@@ -14,7 +14,6 @@ import * as os from 'os';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { isManualReviewEntry } from './cookie-workflow-manual-review';
|
||||
import type { ManualJudgeReview } from './cookie-workflow-manual-review';
|
||||
import { STALL_WINDOW_MS } from './eval-budgets';
|
||||
|
||||
// v2: EvalTestEntry.harvest gains optional {insertions, deletions, net} and
|
||||
// may be explicitly null (arm-benchmark harvest-failure taxonomy). Readers
|
||||
@@ -256,8 +255,10 @@ const oneLine = (text: string | undefined, max: number): string | undefined => {
|
||||
* A provider stall needs a session that streamed partial messages, did not
|
||||
* complete, and shows STALL_WINDOW_MS of silence with a request in flight and
|
||||
* nothing outstanding; sessions without partial messages are never stalls.
|
||||
* The window is STALL_WINDOW_MS (eval-budgets.ts), passed in so this stays
|
||||
* dependency-free (tests that mock eval-budgets still load eval-store).
|
||||
*/
|
||||
export function failureCauseOf(facts: FailureCauseFacts, stallWindowMs = STALL_WINDOW_MS): { cause: TrialFailureCause; evidence?: string } {
|
||||
export function failureCauseOf(facts: FailureCauseFacts, stallWindowMs: number): { cause: TrialFailureCause; evidence?: string } {
|
||||
const sessions = facts.sessions ?? [];
|
||||
const ended = (end: string) => sessions.find((s) => s.end === end);
|
||||
const out = (cause: TrialFailureCause, evidence?: string) => {
|
||||
@@ -361,10 +362,10 @@ export function trialCostKnown(records: ReadonlyArray<{ cost_known?: unknown }>,
|
||||
}
|
||||
|
||||
/** Cause, cause evidence and detail of one failed trial, for both record builders. */
|
||||
export function trialFailureFields(input: FailureCauseFacts & { record?: { judge_scores?: unknown; judge_reasoning?: unknown } }): {
|
||||
export function trialFailureFields(input: FailureCauseFacts & { record?: { judge_scores?: unknown; judge_reasoning?: unknown } }, stallWindowMs: number): {
|
||||
failure_cause: TrialFailureCause; failure_cause_evidence?: string; failure_detail?: TrialFailureDetail;
|
||||
} {
|
||||
const { cause, evidence } = failureCauseOf(input);
|
||||
const { cause, evidence } = failureCauseOf(input, stallWindowMs);
|
||||
const detail = failureDetailOf(input.error, input.record);
|
||||
return { failure_cause: cause, ...(evidence ? { failure_cause_evidence: evidence } : {}), ...(detail ? { failure_detail: detail } : {}) };
|
||||
}
|
||||
|
||||
@@ -33,6 +33,11 @@ export const realPtyDriver: PtyDriver = {
|
||||
sleep: ms => Bun.sleep(ms),
|
||||
};
|
||||
|
||||
/** The last idle turn's error panel, if the session can show its text; diagnostics never throw. */
|
||||
export function safePanelEnd(session: Pick<ClaudePtySession, 'visibleText'>): ReturnType<typeof idlePanelEnd> {
|
||||
try { return idlePanelEnd(session.visibleText()); } catch { return null; }
|
||||
}
|
||||
|
||||
/** One poll step: finish with a result, keep polling, or end the loop as a timeout. */
|
||||
export type PtyStep<R> = { done: R } | 'continue' | 'break';
|
||||
|
||||
@@ -92,7 +97,7 @@ export async function runPtySession<R>(plan: PtySessionPlan<R>): Promise<R> {
|
||||
} finally {
|
||||
appendSessionLedger({ key: sessionKey('pty', undefined), runner: 'pty', started_at: startedAt,
|
||||
budget_ms: plan.budgetMs ?? plan.launch.timeoutMs ?? 240_000, elapsed_ms: plan.driver.monotonic() - started,
|
||||
end, ...idlePanelEnd(session.visibleText()), billed: false });
|
||||
end, ...safePanelEnd(session), billed: false });
|
||||
try {
|
||||
try { plan.beforeClose?.(session, failure); }
|
||||
catch (error) { if (!failure) { failure = { error }; throw error; } }
|
||||
|
||||
@@ -15,6 +15,7 @@ import { runPlanSkillObservation } from './helpers/claude-pty-runner';
|
||||
import { createFakePtyDriver } from './helpers/pty/fake-session';
|
||||
import { runAgentSdkTest, __resetSemaphoreForTests, type QueryProvider } from './helpers/agent-sdk-runner';
|
||||
import { runRecordedCodexEval } from './helpers/codex-eval';
|
||||
import { classifyTrialShard } from '../scripts/test-paid-shards';
|
||||
|
||||
const FIXTURES = path.join(import.meta.dir, 'fixtures', 'trial-cause');
|
||||
const thinking = JSON.parse(fs.readFileSync(path.join(FIXTURES, 'thinking-captures-37198445662.json'), 'utf8'));
|
||||
@@ -49,7 +50,7 @@ function observe(events: Timed[], endAt: number, hooks: { permissionAt?: number
|
||||
}
|
||||
|
||||
const timedOutSession = (liveness: SessionCauseFacts['liveness']): SessionCauseFacts => ({ key: 'claude-p:x#1', end: 'session_timeout', elapsed_ms: 600_000, budget_ms: 600_000, liveness });
|
||||
const causeOf = (facts: Partial<FailureCauseFacts>) => failureCauseOf({ failure_class: 'assertion', ...facts });
|
||||
const causeOf = (facts: Partial<FailureCauseFacts>) => failureCauseOf({ failure_class: 'assertion', ...facts }, STALL_WINDOW_MS);
|
||||
|
||||
describe('failureCauseOf precedence', () => {
|
||||
const stalled = { partial: true, max_request_silence_ms: STALL_WINDOW_MS + 10_000, silence_started_ms: 40_000, silence_after: 'thinking_delta' };
|
||||
@@ -167,6 +168,10 @@ describe('replays of stored census evidence', () => {
|
||||
expect(liveness.partial).toBe(false);
|
||||
expect(liveness.max_request_silence_ms).toBe(0);
|
||||
expect(causeOf({ failure_class: r.failure_class, exit_reason: r.exit_reason, error: r.error }).cause).toBe('session_timeout');
|
||||
// Through the record builder the trial shard uses: failure_class stays timeout, the cause is the session timeout.
|
||||
expect(classifyTrialShard({ status: 'failed', executedTests: 1, skippedTests: 0, elapsedMs: r.duration_ms }, r.case, 1,
|
||||
{ kind: 'rule', panel: { n: 1, k: 1 }, quarantined: false }, { records: [{ passed: false, exit_reason: r.exit_reason, error: r.error }], contract: null }))
|
||||
.toMatchObject({ outcome: 'failed', failure_class: 'timeout', failure_cause: 'session_timeout', error: 'Error: Claude Code process aborted by user' });
|
||||
expect(causeOf({ failure_class: r.failure_class, exit_reason: r.exit_reason, error: r.error,
|
||||
sessions: [{ key: 'agent-sdk:x#1', end: 'session_timeout', elapsed_ms: r.duration_ms, budget_ms: 300_000, liveness }] }).cause).toBe('session_timeout');
|
||||
});
|
||||
@@ -232,7 +237,7 @@ describe('one fixture per runner writes the ledger the classifier reads', () =>
|
||||
{ type: 'stream_event', event: { type: 'content_block_delta', delta: { type: 'thinking_delta' } } }]);
|
||||
expect(streaming.rows[0]!.liveness).toMatchObject({ partial: true, silence_after: 'thinking_delta' });
|
||||
expect(failureCauseOf({ failure_class: 'timeout', sessions: streaming.rows }, 300).cause).toBe('provider_stall');
|
||||
expect(failureCauseOf({ failure_class: 'timeout', sessions: streaming.rows }).cause).toBe('session_timeout');
|
||||
expect(failureCauseOf({ failure_class: 'timeout', sessions: streaming.rows }, STALL_WINDOW_MS).cause).toBe('session_timeout');
|
||||
});
|
||||
|
||||
test('Codex: an expired budget is a session timeout', async () => {
|
||||
|
||||
Reference in new issue
Block a user