mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
0.2: failure_cause beside failure_class, from runner facts and the session ledger
One pure failureCauseOf() beside failureClassOf() sets failure_cause (contract > pre_turn_infra > api_error > refusal > provider_stall > session_timeout > observer_timeout > assertion > unknown; no machine-set 'detector') with one bounded evidence line, and failureDetailOf() keeps a Bun matcher's Expected/Received or a judge's failing dimensions in failure_detail. Both record builders (classifyTrialShard, junitCensus) call them and attach the trial's ledger sessions. failure_class, error and TRIAL_ERROR_MAX are computed exactly as before; the new fields are optional, bounded and validated only when present, and a frozen copy of the v1 reader accepts every new record. A provider stall needs partial messages and STALL_WINDOW_MS (120 s, eval-budgets.ts beside HEADROOM_WARN/FAIL) of in-request silence with nothing outstanding; it stays a failure under EVAL_POLICY v1.
This commit is contained in:
garrytan committed
1 parent
470d068087
commit
80fe5de7bd
11 files changed
+635
-12
No files matched your search
@@ -33,7 +33,9 @@ import {
|
||||
getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict,
|
||||
sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
|
||||
type EvalCaseKind, type PanelShape, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord,
|
||||
trialFailureFields, trialSessions,
|
||||
} from '../../test/helpers/eval-store';
|
||||
import { readSessionLedger, type SessionLedgerRow } from '../../test/helpers/session-ledger';
|
||||
import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
|
||||
import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review';
|
||||
import { preflightAnthropicApi } from '../../test/helpers/anthropic-preflight';
|
||||
@@ -225,6 +227,11 @@ export interface JUnitCensus {
|
||||
|
||||
const bump = (counts: Map<string, number>, key: string) => counts.set(key, (counts.get(key) ?? 0) + 1);
|
||||
|
||||
/** Ledger rows of one JUnit case: its own id or test name, else the shard's only case. */
|
||||
function caseSessions(rows: SessionLedgerRow[], id: string, onlyCase: boolean): SessionLedgerRow[] {
|
||||
return rows.filter(row => row.case === id || (!row.case && (onlyCase || row.test_name === id || row.test_name === CASE_TEST_NAMES[id])));
|
||||
}
|
||||
|
||||
export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map<string, any[]>,
|
||||
base: Record<string, unknown>, cliVersion?: string): JUnitCensus {
|
||||
const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [] };
|
||||
@@ -245,6 +252,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
|
||||
else census.unattributed.push(`${key} :: ${tc.name || '(no name)'} (${tc.outcome})`);
|
||||
}
|
||||
const records = recordsByShard.get(key) ?? [];
|
||||
const ledger = readSessionLedger(path.join(root, 'shards', shardSlug([key])));
|
||||
for (const [id, cases] of byCase) {
|
||||
const kind = (E2E_KINDS[id] ?? 'rule') as EvalCaseKind;
|
||||
const failed = cases.find(tc => tc.outcome === 'failed');
|
||||
@@ -255,6 +263,9 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
|
||||
: failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion';
|
||||
const error = sanitizeTrialError(failedRecord?.error ?? failed?.message);
|
||||
const rerun = Object.hasOwn(E2E_TIERS, id) ? rerunCommand(manifest.tier, id, 1) : `EVALS=1 EVALS_TIER=${manifest.tier} bun test ${shardFile(key)}`;
|
||||
const sessions = caseSessions(ledger, id, byCase.size === 1);
|
||||
const diagnosis = failed && failureClass ? trialFailureFields({ failure_class: failureClass, exit_reason: failedRecord?.exit_reason,
|
||||
error: failedRecord?.error ?? failed.message, sessions, record: failedRecord }) : undefined;
|
||||
if (failed) census.failedShards.add(key);
|
||||
if (caseOutcome === 'skipped') {
|
||||
bump(reasons, RUNTIME_SKIP_REASON);
|
||||
@@ -264,7 +275,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
|
||||
...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}] rerun: ${rerun}` } : {}) });
|
||||
census.history.push({ ...base, case: id, file: shardFile(key), kind, trial: 1, panel: { n: 1, k: 1 }, outcome: caseOutcome,
|
||||
...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
|
||||
...(error && failed ? { error } : {}), duration_ms: cases.reduce((sum, tc) => sum + tc.timeMs, 0),
|
||||
...(error && failed ? { error } : {}), ...diagnosis, ...trialSessions(sessions), duration_ms: cases.reduce((sum, tc) => sum + tc.timeMs, 0),
|
||||
cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100,
|
||||
...(typeof mine[0]?.model === 'string' ? { model: mine[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}),
|
||||
quarantined: false, execution: outcome.reused ? 'reused' : 'executed', source: 'junit' } as TrialOutcomeRecord);
|
||||
@@ -576,6 +587,9 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
history.push({ ...common(attempt), case: t.case, file: shardFile(outcome.files[0]!), kind: t.kind, trial: t.trial, panel: t.panel,
|
||||
outcome: t.outcome, ...(t.outcome === 'failed' ? { failure_class: t.failure_class ?? 'assertion' } : {}),
|
||||
...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}),
|
||||
...(t.outcome === 'failed' && t.failure_cause ? { failure_cause: t.failure_cause } : {}),
|
||||
...(t.outcome === 'failed' && t.failure_cause_evidence ? { failure_cause_evidence: t.failure_cause_evidence } : {}),
|
||||
...(t.outcome === 'failed' && t.failure_detail ? { failure_detail: t.failure_detail } : {}), ...(t.sessions ? { sessions: t.sessions } : {}),
|
||||
duration_ms: t.duration_ms, cost_usd: t.cost_usd, ...(t.model ? { model: t.model } : {}),
|
||||
...(outcome.reused ? { input_identity: outcome.reused.inputKey } : {}),
|
||||
quarantined: t.quarantined, execution: outcome.reused ? 'reused' : 'executed', source: 'shard' } as TrialOutcomeRecord);
|
||||
|
||||
+22
-11
@@ -83,7 +83,9 @@ import {
|
||||
getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict,
|
||||
sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
|
||||
type EvalCaseKind, type PanelShape, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord,
|
||||
trialFailureFields, trialSessions, type TrialFailureCause, type TrialFailureDetail, type TrialSessionSummary,
|
||||
} from '../test/helpers/eval-store';
|
||||
import { readSessionLedger, type SessionLedgerRow } from '../test/helpers/session-ledger';
|
||||
import { E2E_KINDS } from '../test/helpers/touchfiles-data';
|
||||
import { manualReviewProblem } from '../test/helpers/cookie-workflow-manual-review';
|
||||
import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight';
|
||||
@@ -720,10 +722,16 @@ export interface ShardTrialRecord {
|
||||
cost_usd: number;
|
||||
duration_ms: number;
|
||||
model?: string;
|
||||
failure_cause?: TrialFailureCause;
|
||||
failure_cause_evidence?: string;
|
||||
failure_detail?: TrialFailureDetail;
|
||||
sessions?: TrialSessionSummary[];
|
||||
}
|
||||
|
||||
/** Records and contract evidence an isolated shard left in its eval dir. */
|
||||
export function readTrialEvidence(evalDir: string | undefined): { records: any[]; contract: string | null } {
|
||||
type TrialEvidence = { records: any[]; contract: string | null; sessions?: SessionLedgerRow[] };
|
||||
|
||||
/** Records, contract evidence and session-ledger rows an isolated shard left in its eval dir. */
|
||||
export function readTrialEvidence(evalDir: string | undefined): TrialEvidence {
|
||||
if (!evalDir || !fs.existsSync(evalDir)) return { records: [], contract: null };
|
||||
const names = fs.readdirSync(evalDir);
|
||||
const parse = (name: string) => { try { return JSON.parse(fs.readFileSync(path.join(evalDir, name), 'utf8')); } catch { return null; } };
|
||||
@@ -736,28 +744,31 @@ export function readTrialEvidence(evalDir: string | undefined): { records: any[]
|
||||
const line = fs.readFileSync(path.join(evalDir, CONTRACT_VIOLATIONS_FILE), 'utf8').split('\n').find(l => l.trim());
|
||||
if (line) contract = String(JSON.parse(line).message ?? 'contract violation');
|
||||
} catch { /* no sidecar */ }
|
||||
return { records, contract };
|
||||
return { records, contract, sessions: readSessionLedger(evalDir) };
|
||||
}
|
||||
|
||||
/** Classify one isolated trial shard. Contract evidence always fails the trial. */
|
||||
export function classifyTrialShard(
|
||||
outcome: Pick<ShardOutcome, 'status' | 'executedTests' | 'skippedTests' | 'elapsedMs' | 'runnerError'>,
|
||||
caseId: string, trial: number, plan: CaseTrialPlan,
|
||||
evidence: { records: any[]; contract: string | null },
|
||||
evidence: TrialEvidence,
|
||||
): ShardTrialRecord {
|
||||
const failedRecord = evidence.records.find(record => record.passed === false) ?? evidence.records[0];
|
||||
const base: ShardTrialRecord = {
|
||||
case: caseId, trial, kind: plan.kind, panel: plan.panel, quarantined: plan.quarantined, outcome: null,
|
||||
cost_usd: Math.round(evidence.records.reduce((sum, record) => sum + (Number(record.cost_usd) || 0), 0) * 100) / 100,
|
||||
duration_ms: outcome.elapsedMs,
|
||||
...(typeof failedRecord?.model === 'string' ? { model: failedRecord.model } : {}),
|
||||
...(typeof failedRecord?.model === 'string' ? { model: failedRecord.model } : {}), ...trialSessions(evidence.sessions ?? []),
|
||||
};
|
||||
const failed = (failureClass: TrialFailureClass, error?: string): ShardTrialRecord => {
|
||||
const cls = evidence.contract !== null ? 'contract' : failureClass;
|
||||
const raw = evidence.contract ?? failedRecord?.error ?? error;
|
||||
return { ...base, outcome: 'failed', failure_class: cls,
|
||||
...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
|
||||
...(Number.isInteger(failedRecord?.timeout_at_turn) ? { timeout_at_turn: failedRecord.timeout_at_turn } : {}),
|
||||
...(sanitizeTrialError(raw) ? { error: sanitizeTrialError(raw) } : {}),
|
||||
...trialFailureFields({ failure_class: cls, exit_reason: failedRecord?.exit_reason, error: raw, sessions: evidence.sessions, record: failedRecord }) };
|
||||
};
|
||||
const failed = (failureClass: TrialFailureClass, error?: string): ShardTrialRecord => ({
|
||||
...base, outcome: 'failed', failure_class: evidence.contract !== null ? 'contract' : failureClass,
|
||||
...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
|
||||
...(Number.isInteger(failedRecord?.timeout_at_turn) ? { timeout_at_turn: failedRecord.timeout_at_turn } : {}),
|
||||
...(sanitizeTrialError(evidence.contract ?? failedRecord?.error ?? error) ? { error: sanitizeTrialError(evidence.contract ?? failedRecord?.error ?? error) } : {}),
|
||||
});
|
||||
if (outcome.runnerError !== undefined) return { ...base, harness: `runner error: ${sanitizeTrialError(outcome.runnerError) ?? 'unknown'}` };
|
||||
if (outcome.status === 'never-started') return { ...base, harness: 'never started' };
|
||||
if (outcome.status === 'passed-empty') return { ...base, harness: 'hollow: executed no case' };
|
||||
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"source":"stored census records; events reduced to type, subtype, ids, stop reasons and the first line of result text","cases":[{"run":"37195203538","case":"plan-ceo-review-format-approach","trial":1,"runner":"claude-p","exit_reason":"exit_code_1","error":"expect(received).toContain(expected)\n\nExpected to contain: \"exit_code_1\"\nReceived: [ \"success\", \"error_max_turns\" ]\n","failure_class":"assertion","expected":"refusal","events":[[0,{"type":"system","subtype":"init"}],[1,{"type":"assistant","message":{"id":"mt0","content":[]}}],[2,{"type":"assistant","message":{"id":"mt0","content":[{"type":"tool_use","id":"t1","name":"Bash"}]}}],[3,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t1"}]}}],[4,{"type":"system","subtype":"informational"}],[5,{"type":"user","message":{"content":[]}}],[6,{"type":"system","subtype":"thinking_tokens"}],[7,{"type":"system","subtype":"thinking_tokens"}],[8,{"type":"assistant","message":{"id":"mt2","content":[]}}],[9,{"type":"assistant","message":{"id":"mt2","content":[]}}],[10,{"type":"assistant","message":{"id":"mt2","content":[{"type":"tool_use","id":"t3","name":"Bash"}]}}],[11,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t3"}]}}],[12,{"type":"system","subtype":"model_refusal_no_fallback","api_refusal_category":"reasoning_extraction"}],[13,{"type":"assistant","message":{"id":"mt4","model":"<synthetic>","stop_reason":"refusal","stop_details":{"category":"reasoning_extraction"},"content":[{"type":"text","text":"API Error: Fable 5.1's safeguards flagged this message (https://www.anthropic.com/legal/aup). This sometimes happens with safe, normal conversations. Claude Code can't respond to this message with Fable 5.1."}]}}],[14,{"type":"result","subtype":"success","is_error":true,"stop_reason":"refusal","result":"API Error: Fable 5.1's safeguards flagged this message (https://www.anthropic.com/legal/aup). This sometimes happens with safe, normal conversations. Claude Code can't respond to this message with Fable 5.1."}]]},{"run":"37195203538","case":"plan-ceo-review-format-approach","trial":2,"runner":"claude-p","exit_reason":"exit_code_1","error":"expect(received).toContain(expected)\n\nExpected to contain: \"exit_code_1\"\nReceived: [ \"success\", \"error_max_turns\" ]\n","failure_class":"assertion","expected":"refusal","events":[[0,{"type":"system","subtype":"init"}],[1,{"type":"assistant","message":{"id":"mt0","content":[]}}],[2,{"type":"assistant","message":{"id":"mt0","content":[{"type":"tool_use","id":"t1","name":"Bash"}]}}],[3,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t1"}]}}],[4,{"type":"assistant","message":{"id":"mt2","content":[{"type":"tool_use","id":"t3","name":"Bash"}]}}],[5,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t3"}]}}],[6,{"type":"assistant","message":{"id":"mt4","content":[{"type":"tool_use","id":"t5","name":"Bash"}]}}],[7,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t5"}]}}],[8,{"type":"system","subtype":"thinking_tokens"}],[9,{"type":"system","subtype":"thinking_tokens"}],[10,{"type":"assistant","message":{"id":"mt6","content":[]}}],[11,{"type":"assistant","message":{"id":"mt6","content":[{"type":"tool_use","id":"t7","name":"Bash"}]}}],[12,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t7"}]}}],[13,{"type":"system","subtype":"thinking_tokens"}],[14,{"type":"system","subtype":"thinking_tokens"}],[15,{"type":"system","subtype":"thinking_tokens"}],[16,{"type":"system","subtype":"thinking_tokens"}],[17,{"type":"system","subtype":"thinking_tokens"}],[18,{"type":"system","subtype":"thinking_tokens"}],[19,{"type":"assistant","message":{"id":"mt8","content":[]}}],[20,{"type":"system","subtype":"informational"}],[21,{"type":"user","message":{"content":[]}}],[22,{"type":"system","subtype":"thinking_tokens"}],[23,{"type":"system","subtype":"thinking_tokens"}],[24,{"type":"assistant","message":{"id":"mt9","content":[]}}],[25,{"type":"assistant","message":{"id":"mt9","content":[{"type":"tool_use","id":"t10","name":"Write"}]}}],[26,{"type":"system","subtype":"model_refusal_no_fallback","api_refusal_category":"reasoning_extraction"}],[27,{"type":"assistant","message":{"id":"mt11","model":"<synthetic>","stop_reason":"refusal","stop_details":{"category":"reasoning_extraction"},"content":[{"type":"text","text":"API Error: Fable 5.1's safeguards flagged this message (https://www.anthropic.com/legal/aup). This sometimes happens with safe, normal conversations. Claude Code can't respond to this message with Fable 5.1."}]}}],[28,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t10"}]}}],[29,{"type":"assistant","message":{"id":"mt12","content":[]}}],[30,{"type":"assistant","message":{"id":"mt12","content":[{"type":"tool_use","id":"t13","name":"Write"}]}}],[31,{"type":"system","subtype":"model_refusal_no_fallback","api_refusal_category":"reasoning_extraction"}],[32,{"type":"assLine truncated
|
||||
@@ -0,0 +1 @@
|
||||
{"source":"census 37198445662 (Claude Code 2.1.284), publicStreamDiagnostics sessions; arrival times from elapsedMs, non-stream rows stamped with the preceding stream row; same-kind deltas thinned to one per second (every gap of 1 s or more kept); ids renamed; no message text","sessions":[{"case":"/review army consensus","artifact":"paid-slice-11-a1","events":["0 y init","0 y status","1704 s message_start","1704 s content_block_start","2361 s content_block_delta:thinking_delta","2361 s content_block_delta:signature_delta","2361 a","2362 s content_block_stop","2362 s content_block_start","2362 s content_block_delta:text_delta","2455 a","2602 s content_block_stop","2602 s content_block_start","2602 s content_block_delta:input_json_delta","3622 s content_block_delta:input_json_delta","4647 s content_block_delta:input_json_delta","4791 a t0:Bash","4802 s content_block_stop","4809 s message_delta","4809 s message_stop","4809 u t0","4809 y status","7496 s message_start","7497 s content_block_start","7497 s content_block_delta:text_delta","7504 a","7505 s content_block_stop","7505 s content_block_start","7505 s content_block_delta:input_json_delta","8566 s content_block_delta:input_json_delta","9625 s content_block_delta:input_json_delta","10627 s content_block_delta:input_json_delta","11629 s content_block_delta:input_json_delta","12629 s content_block_delta:input_json_delta","13630 s content_block_delta:input_json_delta","14631 s content_block_delta:input_json_delta","15693 s content_block_delta:input_json_delta","16070 a t1:Agent","16073 s content_block_stop","16073 y task_started","16073 u","16081 s content_block_start","16085 s content_block_delta:input_json_delta","17132 s content_block_delta:input_json_delta","18133 s content_block_delta:input_json_delta","19135 s content_block_delta:input_json_delta","20196 s content_block_delta:input_json_delta","21196 s content_block_delta:input_json_delta","21384 y task_progress","21384 a* t2:Bash","21447 s content_block_delta:input_json_delta","21509 u t2","21572 s content_block_delta:input_json_delta","22438 a t3:Agent","22440 s content_block_stop","22440 y task_started","22440 u","22448 s message_delta","22448 s message_stop","22448 y task_progress","22448 a* t4:Read","22448 u t4","22448 y task_progress","22448 a* t5:Bash","22448 u t5","22448 y task_progress","22448 a* t6:Bash","22448 u t6","22448 y task_progress","22448 a* t7:Bash","22448 u t7","22448 y task_updated","22448 y task_notification","22448 u t3","22448 y task_updated","22448 y task_notification","22448 u t1","22448 y status","56707 s message_start","57220 s content_block_start","57220 y thinking_tokens","57512 s content_block_delta:thinking_delta","57512 y thinking_tokens","58833 s content_block_delta:thinking_delta","58833 y thinking_tokens","59900 s content_block_delta:thinking_delta","59900 y thinking_tokens","61276 s content_block_delta:thinking_delta","61276 y thinking_tokens","62658 s content_block_delta:thinking_delta","62658 y thinking_tokens","63843 s content_block_delta:thinking_delta","63843 y thinking_tokens","65031 s content_block_delta:thinking_delta","65031 y thinking_tokens","66220 s content_block_delta:thinking_delta","66220 y thinking_tokens","67721 s content_block_delta:thinking_delta","67721 y thinking_tokens","68783 s content_block_delta:thinking_delta","68783 y thinking_tokens","69849 s content_block_delta:thinking_delta","69849 y thinking_tokens","70909 s content_block_delta:thinking_delta","73753 s content_block_delta:thinking_delta","73753 s content_block_delta:signature_delta","73753 a","73755 s content_block_stop","73755 s content_block_start","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73756 s content_block_delta:thinking_delta","73756 y thinking_tokens","73756 s content_block_delta:thinking_delta","73756 y thinking_tokens","73756 s content_block_delta:thinking_delta","73756 y thinking_tokens","73756 s content_block_delta:thinking_delta","73756 y thinking_tokens","73756 s content_block_delta:signature_delta","73756 a","73756 s content_block_stop","73756 s content_block_start","73756 s content_block_delta:input_json_delta","74781 s content_block_delta:input_json_delta","75781 s content_block_delta:input_json_delta","76781 s content_block_delta:input_json_delta","77782 s content_block_delta:input_json_delta","78843 s content_block_delta:input_json_delta","79845 s contLine truncated
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
/**
|
||||
* FROZEN: trialRecordProblems exactly as EVAL_POLICY v1 readers ship it
|
||||
* (origin/main 2db0b3adc, test/helpers/eval-store.ts). Never edit: it stands in
|
||||
* for older checkouts reading trial records written by this one, so new
|
||||
* optional fields must stay invisible to it (test/helpers/eval-store.test.ts).
|
||||
*/
|
||||
const TRIAL_OUTCOME_SCHEMA = 'gstack-trial-outcome/v1';
|
||||
const TRIAL_ERROR_MAX = 300;
|
||||
const EVAL_KINDS: readonly string[] = ['rule', 'behavior', 'judge'];
|
||||
const FAILURE_CLASSES: readonly string[] = ['assertion', 'contract', 'timeout', 'infra'];
|
||||
|
||||
export function trialRecordProblems(r: any): string[] {
|
||||
const problems: string[] = [];
|
||||
if (!r || typeof r !== 'object' || Array.isArray(r)) return ['not an object'];
|
||||
if (r.schema !== TRIAL_OUTCOME_SCHEMA) problems.push(`schema ${String(r.schema)}`);
|
||||
for (const key of ['case', 'file', 'tier'] as const) if (typeof r[key] !== 'string' || r[key].length === 0) problems.push(`${key} missing`);
|
||||
if (!EVAL_KINDS.includes(r.kind)) problems.push(`kind ${String(r.kind)}`);
|
||||
const n = r.panel?.n, k = r.panel?.k;
|
||||
if (!Number.isInteger(n) || !Number.isInteger(k) || n < 1 || k < 1 || k > n) problems.push('panel invalid');
|
||||
if (!Number.isInteger(r.trial) || r.trial < 1 || (Number.isInteger(n) && r.trial > n)) problems.push('trial invalid');
|
||||
if (!Number.isInteger(r.attempt) || r.attempt < 1) problems.push('attempt invalid');
|
||||
if (!['passed', 'failed', 'skipped'].includes(r.outcome)) problems.push(`outcome ${String(r.outcome)}`);
|
||||
if (r.outcome === 'failed' && !FAILURE_CLASSES.includes(r.failure_class)) problems.push('failed without failure_class');
|
||||
if (r.outcome !== 'failed' && r.failure_class !== undefined) problems.push('failure_class on a non-failed trial');
|
||||
if (typeof r.duration_ms !== 'number' || !Number.isFinite(r.duration_ms) || r.duration_ms < 0) problems.push('duration_ms invalid');
|
||||
if (typeof r.cost_usd !== 'number' || !Number.isFinite(r.cost_usd) || r.cost_usd < 0) problems.push('cost_usd invalid');
|
||||
if (!Number.isInteger(r.policy_version) || r.policy_version < 0) problems.push('policy_version invalid');
|
||||
if (typeof r.quarantined !== 'boolean') problems.push('quarantined invalid');
|
||||
if (r.execution !== 'executed' && r.execution !== 'reused') problems.push('execution invalid');
|
||||
if (!['shard', 'junit', 'backfill'].includes(r.source)) problems.push('source invalid');
|
||||
if (r.error !== undefined && (typeof r.error !== 'string' || r.error.length > TRIAL_ERROR_MAX)) problems.push('error invalid');
|
||||
if (r.series_identity !== undefined && (typeof r.series_identity !== 'string' || !/^[\w.-]{1,64}$/.test(r.series_identity))) problems.push('series_identity invalid');
|
||||
return problems;
|
||||
}
|
||||
@@ -130,3 +130,20 @@ export function assertPaidTestBudget(file: string, ms: number): void {
|
||||
throw new Error(`Unregistered paid test budget: ${file}: ${ms}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Diagnostic thresholds, outside EVAL_POLICY: v1 verdicts never read them.
|
||||
* A session above HEADROOM_WARN of the timeout it armed is flagged by
|
||||
* `eval:pass-rates --headroom`; above HEADROOM_FAIL the weekly `--gate` alarms.
|
||||
* Cut work to get under them; budgets are never raised.
|
||||
*/
|
||||
export const HEADROOM_WARN = 0.75;
|
||||
export const HEADROOM_FAIL = 0.85;
|
||||
|
||||
/**
|
||||
* In-request stream silence that labels a provider stall (failure_cause
|
||||
* provider_stall, still a failure under EVAL_POLICY v1). Pre-registered from
|
||||
* census 37198445662 on Claude Code 2.1.284: the longest gap inside one model
|
||||
* message across 50 messages was 9.6 s, thinking deltas included.
|
||||
*/
|
||||
export const STALL_WINDOW_MS = 120_000;
|
||||
@@ -25,10 +25,12 @@ import {
|
||||
parseTrialOutcomes,
|
||||
sanitizeTrialError,
|
||||
trialContextFromEnv,
|
||||
TRIAL_ERROR_MAX,
|
||||
} from './eval-store';
|
||||
import type { EvalResult, EvalTestEntry, ComparisonResult, PanelTrial, TrialOutcomeRecord } from './eval-store';
|
||||
import { EVAL_POLICY } from './periodic-exclude-data';
|
||||
import { manualReviewFixture } from './manual-judge-review-fixture';
|
||||
import { trialRecordProblems as trialRecordProblemsV1 } from '../fixtures/trial-record-v1-reader';
|
||||
|
||||
let tmpDir: string;
|
||||
|
||||
@@ -1151,6 +1153,35 @@ describe('trial-outcomes JSONL', () => {
|
||||
expect(parseTrialOutcomes(text, { maxBytes: 10 }).errors[0]).toContain('exceed');
|
||||
});
|
||||
|
||||
test('new optional fields: valid records pass the current reader and the frozen v1 reader', () => {
|
||||
const failed = record({ trial: 2, outcome: 'failed', failure_class: 'assertion', exit_reason: 'success',
|
||||
error: sanitizeTrialError('expect(received).toBe(expected)\n\nExpected: "b"\nReceived: "a"'),
|
||||
failure_cause: 'assertion', failure_cause_evidence: 'session completed; check failed',
|
||||
failure_detail: { expected: '"b"', received: '"a"' }, cost_known: false,
|
||||
sessions: [{ key: 'claude-p:case-x#1', runner: 'claude-p', elapsed_ms: 1200, budget_ms: 300_000, end: 'completed' }] });
|
||||
const judge = record({ trial: 3, outcome: 'failed', failure_class: 'assertion', failure_cause: 'assertion',
|
||||
failure_detail: { judge: [{ dimension: 'clarity', mean: 3.67, threshold: 4, samples: 3, rationale: 'Clarity suffers from density.' }] } });
|
||||
const passed = record({ sessions: [{ key: 'pty:case-x#1', runner: 'pty', elapsed_ms: 10, end: 'completed' }] });
|
||||
for (const r of [failed, judge, passed]) {
|
||||
expect(trialRecordProblemsV1(JSON.parse(JSON.stringify(r)))).toEqual([]);
|
||||
expect(r.error === undefined || r.error.length <= TRIAL_ERROR_MAX).toBe(true);
|
||||
}
|
||||
expect(parseTrialOutcomes(formatTrialOutcomes([failed, judge, passed]))).toEqual({ records: [failed, judge, passed], errors: [] });
|
||||
});
|
||||
|
||||
test('new optional fields are bounded and checked only when present', () => {
|
||||
const failed = (extra: Partial<TrialOutcomeRecord> | Record<string, unknown>) => record({ outcome: 'failed', failure_class: 'assertion', ...extra } as Partial<TrialOutcomeRecord>);
|
||||
const problems = (r: unknown) => parseTrialOutcomes(JSON.stringify(r)).errors;
|
||||
expect(problems(record({ failure_cause: 'assertion' }))[0]).toContain('failure_cause on a non-failed trial');
|
||||
expect(problems(failed({ failure_cause: 'detector' }))[0]).toContain("unknown failure_cause \"detector\" (written by a newer gstack; update this checkout to read it)");
|
||||
expect(problems(failed({ failure_cause_evidence: 'x'.repeat(301) }))[0]).toContain('failure_cause_evidence invalid');
|
||||
expect(problems(failed({ failure_detail: { expected: 'x'.repeat(201), received: '' } }))[0]).toContain('failure_detail invalid');
|
||||
expect(problems(failed({ failure_detail: { judge: [] } }))[0]).toContain('failure_detail invalid');
|
||||
expect(problems(record({ sessions: [{ key: 'k', runner: 'pty', elapsed_ms: -1, end: 'completed' }] }))[0]).toContain('sessions invalid');
|
||||
expect(problems(record({ cost_known: 'no' } as any))[0]).toContain('cost_known invalid');
|
||||
expect(problems(failed({}))).toEqual([]);
|
||||
});
|
||||
|
||||
test('sanitizeTrialError keeps one capped line without mentions', () => {
|
||||
expect(sanitizeTrialError('\n expected @garrytan to `see`\nsecond')).toBe("expected @\u200bgarrytan to 'see'");
|
||||
expect(sanitizeTrialError('x'.repeat(1000))!.length).toBe(300);
|
||||
|
||||
@@ -14,6 +14,7 @@ import * as os from 'os';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { isManualReviewEntry } from './cookie-workflow-manual-review';
|
||||
import type { ManualJudgeReview } from './cookie-workflow-manual-review';
|
||||
import { STALL_WINDOW_MS } from './eval-budgets';
|
||||
|
||||
// v2: EvalTestEntry.harvest gains optional {insertions, deletions, net} and
|
||||
// may be explicitly null (arm-benchmark harvest-failure taxonomy). Readers
|
||||
@@ -210,6 +211,151 @@ export function failureClassOf(entry: Pick<EvalTestEntry, 'failure_class' | 'exi
|
||||
return entry.exit_reason === 'timeout' ? 'timeout' : 'assertion';
|
||||
}
|
||||
|
||||
/** Machine-recorded cause of a failed trial, beside its policy class. failure_class
|
||||
* (what verdicts read) is never changed by it; whether an assertion was a
|
||||
* detector, harness or product fault is a human diagnosis, never a cause. */
|
||||
export const FAILURE_CAUSES = ['contract', 'pre_turn_infra', 'api_error', 'refusal', 'provider_stall',
|
||||
'session_timeout', 'observer_timeout', 'assertion', 'unknown'] as const;
|
||||
export type TrialFailureCause = typeof FAILURE_CAUSES[number];
|
||||
export const FAILURE_CAUSE_EVIDENCE_MAX = 300;
|
||||
export const FAILURE_DETAIL_PART_MAX = 200;
|
||||
|
||||
/** Session facts failureCauseOf reads (a structural subset of a session-ledger row). */
|
||||
export interface SessionCauseFacts {
|
||||
key?: string;
|
||||
end: string;
|
||||
evidence?: string;
|
||||
elapsed_ms?: number;
|
||||
budget_ms?: number;
|
||||
liveness?: { partial: boolean; max_request_silence_ms: number; silence_started_ms?: number; silence_after?: string };
|
||||
}
|
||||
|
||||
export interface FailureCauseFacts {
|
||||
failure_class: TrialFailureClass;
|
||||
exit_reason?: string;
|
||||
/** Raw failure text (the record's error or the JUnit message). */
|
||||
error?: string;
|
||||
sessions?: readonly SessionCauseFacts[];
|
||||
}
|
||||
|
||||
const oneLine = (text: string | undefined, max: number): string | undefined => {
|
||||
const first = text?.split('\n').map((l) => l.trim()).find((l) => l.length > 0);
|
||||
if (!first) return undefined;
|
||||
// eslint-disable-next-line no-control-regex
|
||||
const clean = first.replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/@(?=[A-Za-z0-9_-])/g, '@\u200b');
|
||||
return clean.length > max ? `${clean.slice(0, max - 1)}…` : clean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Pure and dependency-free: the cause of one failed trial from its recorded
|
||||
* facts. Precedence contract > pre_turn_infra > api_error > refusal >
|
||||
* provider_stall > session_timeout > observer_timeout > assertion > unknown;
|
||||
* the highest cause with evidence wins and its evidence line is returned.
|
||||
* A provider stall needs a session that streamed partial messages, did not
|
||||
* complete, and shows STALL_WINDOW_MS of silence with a request in flight and
|
||||
* nothing outstanding; sessions without partial messages are never stalls.
|
||||
*/
|
||||
export function failureCauseOf(facts: FailureCauseFacts, stallWindowMs = STALL_WINDOW_MS): { cause: TrialFailureCause; evidence?: string } {
|
||||
const sessions = facts.sessions ?? [];
|
||||
const ended = (end: string) => sessions.find((s) => s.end === end);
|
||||
const out = (cause: TrialFailureCause, evidence?: string) => {
|
||||
const line = oneLine(evidence, FAILURE_CAUSE_EVIDENCE_MAX);
|
||||
return line ? { cause, evidence: line } : { cause };
|
||||
};
|
||||
if (facts.failure_class === 'contract') return out('contract', facts.error);
|
||||
if (facts.failure_class === 'infra') return out('pre_turn_infra', facts.error ?? facts.exit_reason);
|
||||
const api = ended('api_error');
|
||||
if (api) return out('api_error', api.evidence);
|
||||
if (facts.exit_reason === 'error_api') return out('api_error', facts.error);
|
||||
const refusal = ended('refusal');
|
||||
if (refusal) return out('refusal', refusal.evidence);
|
||||
const stall = sessions.find((s) => s.end !== 'completed' && s.liveness?.partial === true && s.liveness.max_request_silence_ms >= stallWindowMs);
|
||||
if (stall?.liveness) {
|
||||
const l = stall.liveness;
|
||||
return out('provider_stall', `no stream event for ${Math.round(l.max_request_silence_ms / 1000)}s`
|
||||
+ `${l.silence_started_ms !== undefined ? ` from ${Math.round(l.silence_started_ms / 1000)}s` : ''}`
|
||||
+ `${l.silence_after ? ` (last: ${l.silence_after})` : ''}; model request in flight, no tool outstanding`);
|
||||
}
|
||||
const clock = (s: SessionCauseFacts) => `${s.key ?? 'session'} ran ${Math.round((s.elapsed_ms ?? 0) / 1000)}s`
|
||||
+ `${s.budget_ms ? ` of its ${Math.round(s.budget_ms / 1000)}s budget` : ''}`;
|
||||
const timedOut = ended('session_timeout');
|
||||
if (timedOut) return out('session_timeout', timedOut.evidence ?? clock(timedOut));
|
||||
if (facts.exit_reason === 'timeout') return out('session_timeout', 'the runner\'s armed session timeout fired');
|
||||
const observerText = /\boutcome=timeout\b/.test(facts.error ?? '') ? facts.error : undefined;
|
||||
const observer = ended('observer_timeout');
|
||||
if (observer) return out('observer_timeout', observer.evidence ?? observerText ?? clock(observer));
|
||||
if (observerText) return out('observer_timeout', observerText);
|
||||
if (facts.failure_class === 'assertion') return out('assertion', facts.exit_reason === 'success' ? 'session completed; check failed' : undefined);
|
||||
return out('unknown', facts.failure_class === 'timeout' ? 'case budget expired with no session evidence' : undefined);
|
||||
}
|
||||
|
||||
/** Expected/Received values of a failed Bun matcher, or a judge's failing dimensions. */
|
||||
export type TrialFailureDetail =
|
||||
| { expected: string; received: string }
|
||||
| { judge: Array<{ dimension: string; mean: number; threshold: number; samples: number; rationale?: string }> };
|
||||
|
||||
const detailPart = (text: string): string => {
|
||||
// eslint-disable-next-line no-control-regex
|
||||
const flat = text.replace(/[\u0000-\u001f\u007f]+/g, ' ').replace(/\s+/g, ' ').trim().replace(/@(?=[A-Za-z0-9_-])/g, '@\u200b');
|
||||
return flat.length > FAILURE_DETAIL_PART_MAX ? `${flat.slice(0, FAILURE_DETAIL_PART_MAX - 1)}…` : flat;
|
||||
};
|
||||
|
||||
/**
|
||||
* Detail the stored error line cannot carry (it keeps only the matcher header).
|
||||
* A judge record with a `>= N` threshold names each dimension below it, its
|
||||
* mean, its sample count and the first sample sentence naming that dimension;
|
||||
* otherwise a Bun matcher message yields its Expected/Received values.
|
||||
*/
|
||||
export function failureDetailOf(text: string | undefined,
|
||||
record?: { judge_scores?: unknown; judge_reasoning?: unknown }): TrialFailureDetail | undefined {
|
||||
if (!text) return undefined;
|
||||
const threshold = /^Expected:\s*>=?\s*(-?[\d.]+)\s*$/m.exec(text);
|
||||
const scores = record?.judge_scores;
|
||||
if (threshold && scores && typeof scores === 'object' && !Array.isArray(scores)) {
|
||||
const limit = Number(threshold[1]);
|
||||
const reasoning = typeof record?.judge_reasoning === 'string' ? record.judge_reasoning : '';
|
||||
const samples = reasoning.split(/\[sample \d+\]\s*/).map((s) => s.trim()).filter(Boolean);
|
||||
const judge = Object.entries(scores as Record<string, unknown>)
|
||||
.filter((entry): entry is [string, number] => typeof entry[1] === 'number' && Number.isFinite(entry[1]) && entry[1] < limit)
|
||||
.slice(0, 8).map(([dimension, mean]) => {
|
||||
const word = dimension.replace(/_/g, ' ').toLowerCase();
|
||||
const sentence = samples.flatMap((s) => s.split(/(?<=[.!?])\s+/)).find((s) => s.toLowerCase().includes(word));
|
||||
return { dimension: detailPart(dimension), mean: Math.round(mean * 100) / 100, threshold: limit,
|
||||
samples: Math.max(1, samples.length), ...(sentence ? { rationale: detailPart(sentence.slice(0, 150)) } : {}) };
|
||||
});
|
||||
if (judge.length) return { judge };
|
||||
}
|
||||
const lines = text.split('\n');
|
||||
const expected = lines.map((l) => /^Expected(?:[^:]*)?:\s?(.*)$/.exec(l)).find(Boolean);
|
||||
const received = lines.map((l) => /^Received(?:[^:]*)?:\s?(.*)$/.exec(l)).find(Boolean);
|
||||
if (expected && received && !/^\s*[-+]\s*\d+\s*$/.test(expected[1]!)) {
|
||||
return { expected: detailPart(expected[1]!), received: detailPart(received[1]!) };
|
||||
}
|
||||
const minus = lines.filter((l) => /^-\s/.test(l) && !/^- Expected\s/.test(l)).map((l) => l.slice(1).trim());
|
||||
const plus = lines.filter((l) => /^\+\s/.test(l) && !/^\+ Received\s/.test(l)).map((l) => l.slice(1).trim());
|
||||
if (minus.length || plus.length) return { expected: detailPart(minus.join(' ')), received: detailPart(plus.join(' ')) };
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export interface TrialSessionSummary { key: string; runner: string; elapsed_ms: number; budget_ms?: number; end: string }
|
||||
|
||||
/** The compact per-session view a trial record carries (ledger evidence and liveness stay in the artifact). */
|
||||
export function trialSessions(rows: ReadonlyArray<SessionCauseFacts & { runner?: string }>): { sessions?: TrialSessionSummary[] } {
|
||||
const sessions = rows.filter((r) => typeof r.key === 'string' && Number.isFinite(r.elapsed_ms)).slice(0, 32).map((r) => ({
|
||||
key: r.key!.slice(0, 160), runner: String(r.runner ?? 'unknown').slice(0, 20), elapsed_ms: r.elapsed_ms!,
|
||||
...(Number.isFinite(r.budget_ms) ? { budget_ms: r.budget_ms } : {}), end: r.end.slice(0, 20) }));
|
||||
return sessions.length ? { sessions } : {};
|
||||
}
|
||||
|
||||
/** Cause, cause evidence and detail of one failed trial, for both record builders. */
|
||||
export function trialFailureFields(input: FailureCauseFacts & { record?: { judge_scores?: unknown; judge_reasoning?: unknown } }): {
|
||||
failure_cause: TrialFailureCause; failure_cause_evidence?: string; failure_detail?: TrialFailureDetail;
|
||||
} {
|
||||
const { cause, evidence } = failureCauseOf(input);
|
||||
const detail = failureDetailOf(input.error, input.record);
|
||||
return { failure_cause: cause, ...(evidence ? { failure_cause_evidence: evidence } : {}), ...(detail ? { failure_detail: detail } : {}) };
|
||||
}
|
||||
|
||||
export class ContractViolation extends Error {
|
||||
constructor(message: string) {
|
||||
super(`CONTRACT: ${message}`);
|
||||
@@ -404,6 +550,14 @@ export interface TrialOutcomeRecord {
|
||||
recorded_at?: string;
|
||||
/** History series key: a hash of the case's own touchfiles (GLOBAL_TOUCHFILES excluded), stamped by the report job. */
|
||||
series_identity?: string;
|
||||
/** Failed trials only: what the runners observed (failureCauseOf); verdicts never read it. */
|
||||
failure_cause?: TrialFailureCause;
|
||||
failure_cause_evidence?: string;
|
||||
failure_detail?: TrialFailureDetail;
|
||||
/** Ledger sessions of this trial; absent means unknown, never scored. */
|
||||
sessions?: TrialSessionSummary[];
|
||||
/** False when the harness captured no billing, so cost_usd 0 means unknown. Absent means known. */
|
||||
cost_known?: boolean;
|
||||
}
|
||||
|
||||
/** First line of free text, stripped of @-mentions and control characters, capped. */
|
||||
@@ -437,6 +591,34 @@ function trialRecordProblems(r: any): string[] {
|
||||
if (!['shard', 'junit', 'backfill'].includes(r.source)) problems.push('source invalid');
|
||||
if (r.error !== undefined && (typeof r.error !== 'string' || r.error.length > TRIAL_ERROR_MAX)) problems.push('error invalid');
|
||||
if (r.series_identity !== undefined && (typeof r.series_identity !== 'string' || !/^[\w.-]{1,64}$/.test(r.series_identity))) problems.push('series_identity invalid');
|
||||
problems.push(...optionalFieldProblems(r));
|
||||
return problems;
|
||||
}
|
||||
|
||||
const boundedString = (v: unknown, max: number) => typeof v === 'string' && v.length <= max;
|
||||
const finiteNonNegative = (v: unknown) => typeof v === 'number' && Number.isFinite(v) && v >= 0;
|
||||
|
||||
/** The optional diagnostic fields, checked only when present. */
|
||||
function optionalFieldProblems(r: any): string[] {
|
||||
const problems: string[] = [];
|
||||
if (r.failure_cause !== undefined) {
|
||||
if (r.outcome !== 'failed') problems.push('failure_cause on a non-failed trial');
|
||||
else if (!FAILURE_CAUSES.includes(r.failure_cause)) problems.push(`unknown failure_cause ${JSON.stringify(String(r.failure_cause).slice(0, 40))} (written by a newer gstack; update this checkout to read it)`);
|
||||
}
|
||||
if (r.failure_cause_evidence !== undefined && !boundedString(r.failure_cause_evidence, FAILURE_CAUSE_EVIDENCE_MAX)) problems.push('failure_cause_evidence invalid');
|
||||
const d = r.failure_detail;
|
||||
if (d !== undefined) {
|
||||
const assertion = d && typeof d === 'object' && !Array.isArray(d) && Object.keys(d).length === 2
|
||||
&& boundedString(d.expected, FAILURE_DETAIL_PART_MAX) && boundedString(d.received, FAILURE_DETAIL_PART_MAX);
|
||||
const judge = d && typeof d === 'object' && Object.keys(d).length === 1 && Array.isArray(d.judge) && d.judge.length >= 1 && d.judge.length <= 8
|
||||
&& d.judge.every((j: any) => j && boundedString(j.dimension, FAILURE_DETAIL_PART_MAX) && Number.isFinite(j.mean) && Number.isFinite(j.threshold)
|
||||
&& Number.isInteger(j.samples) && j.samples >= 1 && j.samples <= 20 && (j.rationale === undefined || boundedString(j.rationale, FAILURE_DETAIL_PART_MAX)));
|
||||
if (!assertion && !judge) problems.push('failure_detail invalid');
|
||||
}
|
||||
if (r.sessions !== undefined && !(Array.isArray(r.sessions) && r.sessions.length <= 32 && r.sessions.every((x: any) => x
|
||||
&& boundedString(x.key, 160) && boundedString(x.runner, 20) && finiteNonNegative(x.elapsed_ms) && boundedString(x.end, 20)
|
||||
&& (x.budget_ms === undefined || finiteNonNegative(x.budget_ms))))) problems.push('sessions invalid');
|
||||
if (r.cost_known !== undefined && typeof r.cost_known !== 'boolean') problems.push('cost_known invalid');
|
||||
return problems;
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { DESELECTION_REASONS, RUNTIME_SKIP_REASON, attributeJUnitCase, formatPanelLine, runPaidReport, type JUnitCase } from '../scripts/test-paid-shards';
|
||||
import { panelVerdict } from './helpers/eval-store';
|
||||
import { trialRecordProblems as trialRecordProblemsV1 } from './fixtures/trial-record-v1-reader';
|
||||
|
||||
const FIXTURES = path.join(import.meta.dir, 'fixtures', 'paid-report-census');
|
||||
|
||||
@@ -92,6 +93,37 @@ describe('census attribution from the 2026-10-03 artifacts', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('record builders attach ledger sessions and failure causes (plan 0.1/0.2)', () => {
|
||||
const noOp = (dir: string) => path.join(dir, 'paid-slice-1-a1', 'shards', 'skill-e2e-plan-mode-no-op');
|
||||
const failNoOp = (dir: string) => {
|
||||
const file = path.join(noOp(dir), 'junit.xml');
|
||||
fs.writeFileSync(file, fs.readFileSync(file, 'utf8')
|
||||
.replace('failures="0"', 'failures="1"')
|
||||
.replace(/(<testcase name="plan-ceo-review reaches a terminal outcome outside plan mode"[^>]*?) \/>/,
|
||||
'$1><failure type="AssertionError" message="plan-ceo-review: outcome=timeout (no terminal outcome within 180000ms)" /></testcase>'));
|
||||
fs.writeFileSync(path.join(noOp(dir), 'session-ledger.jsonl'), [
|
||||
{ key: 'pty:session#1', runner: 'pty', started_at: 't', budget_ms: 180_000, elapsed_ms: 16_800, end: 'completed' },
|
||||
{ key: 'pty:session#2', runner: 'pty', started_at: 't', budget_ms: 180_000, elapsed_ms: 180_050, end: 'observer_timeout' },
|
||||
].map(row => JSON.stringify(row)).join('\n') + '\nnot json\n');
|
||||
};
|
||||
|
||||
test('a JUnit-only PTY case gets failure_cause and sessions from its ledger; failure_class is unchanged', () => {
|
||||
const r = report('gate', failNoOp);
|
||||
const noOpRecord = r.history.find(record => record.case === 'plan-mode-no-op');
|
||||
expect(noOpRecord).toMatchObject({ outcome: 'failed', failure_class: 'assertion', failure_cause: 'observer_timeout',
|
||||
failure_cause_evidence: 'plan-ceo-review: outcome=timeout (no terminal outcome within 180000ms)',
|
||||
sessions: [{ key: 'pty:session#1', runner: 'pty', elapsed_ms: 16_800, budget_ms: 180_000, end: 'completed' },
|
||||
{ key: 'pty:session#2', runner: 'pty', elapsed_ms: 180_050, budget_ms: 180_000, end: 'observer_timeout' }] });
|
||||
for (const record of r.history) expect(trialRecordProblemsV1(record), record.case).toEqual([]);
|
||||
const plain = report('gate', dir => { failNoOp(dir); fs.rmSync(path.join(noOp(dir), 'session-ledger.jsonl')); });
|
||||
const without = plain.history.find(record => record.case === 'plan-mode-no-op');
|
||||
expect(without.failure_class).toBe(noOpRecord.failure_class);
|
||||
expect(without.sessions).toBeUndefined();
|
||||
const collector = (dir: string) => JSON.parse(fs.readFileSync(path.join(dir, 'collector-outcomes.json'), 'utf8'));
|
||||
expect(collector(plain.dir).verdict).toEqual(collector(r.dir).verdict);
|
||||
});
|
||||
});
|
||||
|
||||
describe('attributeJUnitCase', () => {
|
||||
const tc = (name: string, outcome: JUnitCase['outcome'], line: number | null = 1): JUnitCase =>
|
||||
({ name, classname: 's', outcome, timeMs: 0, ...(line === null ? {} : { line }) });
|
||||
|
||||
@@ -686,6 +686,33 @@ describe('isolated trial shards: record, classification and slice exit', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('classification adds cause, detail and ledger sessions; failure_class and outcome never change', () => {
|
||||
const sessions = [{ key: 'claude-p:review-sql-injection#1', runner: 'claude-p' as const, started_at: 't', budget_ms: 300_000, elapsed_ms: 300_100,
|
||||
end: 'session_timeout' as const, evidence: 'armed 300000ms session timeout fired', liveness: { partial: true, events: 9, turns: 2, max_request_silence_ms: 150_000 } }];
|
||||
const inputs: Array<[Partial<ShardOutcome>, { records: any[]; contract: string | null }]> = [
|
||||
[{}, none], [{}, { records: [], contract: 'handoff missing' }],
|
||||
[{ status: 'failed' }, { records: [{ passed: false, exit_reason: 'timeout', timeout_at_turn: 9, error: 'x' }], contract: null }],
|
||||
[{ status: 'failed' }, { records: [{ passed: false, exit_reason: 'success', error: 'expect(received).toBe(expected)\n\nExpected: 3\nReceived: 2\n' }], contract: null }],
|
||||
[{ status: 'timed-out', executedTests: null, skippedTests: null }, none],
|
||||
[{ status: 'failed', executedTests: null, skippedTests: null }, none],
|
||||
];
|
||||
for (const [over, evidence] of inputs) {
|
||||
const before = classifyTrialShard({ ...base, ...over }, 'review-sql-injection', 1, plan, evidence);
|
||||
const after = classifyTrialShard({ ...base, ...over }, 'review-sql-injection', 1, plan, { ...evidence, sessions });
|
||||
expect([after.outcome, after.failure_class, after.error], JSON.stringify(over)).toEqual([before.outcome, before.failure_class, before.error]);
|
||||
expect(after.sessions).toEqual([{ key: 'claude-p:review-sql-injection#1', runner: 'claude-p', elapsed_ms: 300_100, budget_ms: 300_000, end: 'session_timeout' }]);
|
||||
expect(after.failure_cause === undefined).toBe(after.outcome !== 'failed');
|
||||
}
|
||||
const c = (over: Partial<ShardOutcome>, evidence: { records: any[]; contract: string | null; sessions?: any[] }) =>
|
||||
classifyTrialShard({ ...base, ...over }, 'review-sql-injection', 1, plan, evidence);
|
||||
expect(c({ status: 'failed' }, { records: [{ passed: false, exit_reason: 'success', error: 'expect(received).toBe(expected)\n\nExpected: 3\nReceived: 2\n' }], contract: null }))
|
||||
.toMatchObject({ failure_class: 'assertion', error: 'expect(received).toBe(expected)', failure_cause: 'assertion',
|
||||
failure_cause_evidence: 'session completed; check failed', failure_detail: { expected: '3', received: '2' } });
|
||||
expect(c({ status: 'failed' }, { records: [{ passed: false, exit_reason: 'timeout', error: 'Error: Claude Code process aborted by user' }], contract: null, sessions }))
|
||||
.toMatchObject({ failure_class: 'timeout', failure_cause: 'provider_stall' });
|
||||
expect(c({}, { records: [], contract: 'handoff missing', sessions })).toMatchObject({ failure_class: 'contract', failure_cause: 'contract', failure_cause_evidence: 'handoff missing' });
|
||||
});
|
||||
|
||||
test('slice exit: rule shards stay strict; failed trials never red the runner, missing records do', () => {
|
||||
const trial = (outcome: 'passed' | 'failed' | null) => ({ status: outcome === 'failed' ? 'failed' as const : 'passed' as const,
|
||||
trial: { case: 'c', trial: 1, ...plan, outcome, cost_usd: 0, duration_ms: 1, ...(outcome === null ? { harness: 'never started' } : {}) } });
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
/**
|
||||
* failure_cause (plan 0.2): the pure failureCauseOf() precedence table, the
|
||||
* session observer's liveness and structured refusal/API-error facts, and
|
||||
* replays of stored census evidence. failure_class is never an output here:
|
||||
* verdicts keep reading it unchanged.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { FAILURE_CAUSES, failureCauseOf, failureDetailOf, type FailureCauseFacts, type SessionCauseFacts } from './helpers/eval-store';
|
||||
import { SessionObserver, readSessionLedger, type SessionLedgerRow } from './helpers/session-ledger';
|
||||
import { STALL_WINDOW_MS } from './helpers/eval-budgets';
|
||||
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
|
||||
import { createFakePtyDriver } from './helpers/pty/fake-session';
|
||||
import { runAgentSdkTest, __resetSemaphoreForTests, type QueryProvider } from './helpers/agent-sdk-runner';
|
||||
import { runRecordedCodexEval } from './helpers/codex-eval';
|
||||
|
||||
const FIXTURES = path.join(import.meta.dir, 'fixtures', 'trial-cause');
|
||||
const thinking = JSON.parse(fs.readFileSync(path.join(FIXTURES, 'thinking-captures-37198445662.json'), 'utf8'));
|
||||
const replays = JSON.parse(fs.readFileSync(path.join(FIXTURES, 'replays.json'), 'utf8'));
|
||||
|
||||
type Timed = [number, any];
|
||||
/** Decode one compact capture line: `<ms> s <kind>[:<delta>]`, `a[*] <id>:<tool>,...`, `u <id>,...`, `y <subtype>`, `r`. */
|
||||
function decode(line: string): Timed {
|
||||
const [ms, code, arg = ''] = line.split(' ');
|
||||
const at = Number(ms);
|
||||
if (code === 's') {
|
||||
const [type, delta] = arg.split(':');
|
||||
return [at, { type: 'stream_event', event: { type, ...(delta ? { delta: { type: delta } } : {}) } }];
|
||||
}
|
||||
if (code === 'a' || code === 'a*') {
|
||||
const content = arg ? arg.split(',').map(t => { const [id, name] = t.split(':'); return { type: 'tool_use', id, name }; }) : [];
|
||||
return [at, { type: 'assistant', ...(code === 'a*' ? { parent_tool_use_id: 'parent' } : {}), message: { content } }];
|
||||
}
|
||||
if (code === 'u') return [at, { type: 'user', message: { content: arg ? arg.split(',').map(id => ({ type: 'tool_result', tool_use_id: id })) : [] } }];
|
||||
if (code === 'y') return [at, { type: 'system', subtype: arg }];
|
||||
return [at, { type: 'result', subtype: 'success' }];
|
||||
}
|
||||
|
||||
function observe(events: Timed[], endAt: number, hooks: { permissionAt?: number } = {}) {
|
||||
const observer = new SessionObserver(0);
|
||||
for (const [at, event] of events) {
|
||||
if (hooks.permissionAt !== undefined && at > hooks.permissionAt) { observer.permission(1); hooks.permissionAt = undefined; }
|
||||
observer.observe(event, at);
|
||||
}
|
||||
if (hooks.permissionAt !== undefined) observer.permission(1);
|
||||
return { observer, liveness: observer.summary(endAt) };
|
||||
}
|
||||
|
||||
const timedOutSession = (liveness: SessionCauseFacts['liveness']): SessionCauseFacts => ({ key: 'claude-p:x#1', end: 'session_timeout', elapsed_ms: 600_000, budget_ms: 600_000, liveness });
|
||||
const causeOf = (facts: Partial<FailureCauseFacts>) => failureCauseOf({ failure_class: 'assertion', ...facts });
|
||||
|
||||
describe('failureCauseOf precedence', () => {
|
||||
const stalled = { partial: true, max_request_silence_ms: STALL_WINDOW_MS + 10_000, silence_started_ms: 40_000, silence_after: 'thinking_delta' };
|
||||
const rows: Array<[string, Partial<FailureCauseFacts>]> = [
|
||||
['contract', { failure_class: 'contract', error: 'CONTRACT: no write', sessions: [{ end: 'api_error', evidence: 'API Error: 500' }] }],
|
||||
['pre_turn_infra', { failure_class: 'infra', exit_reason: 'timeout_startup', sessions: [{ end: 'refusal' }] }],
|
||||
['api_error', { sessions: [{ end: 'refusal', evidence: 'refusal at turn 2' }, { end: 'api_error', evidence: 'API Error: Connection lost mid-response.' }] }],
|
||||
['refusal', { sessions: [timedOutSession(stalled), { end: 'refusal', evidence: 'refusal at turn 3 (model_refusal_no_fallback) — reasoning_extraction' }] }],
|
||||
['provider_stall', { failure_class: 'timeout', exit_reason: 'timeout', sessions: [timedOutSession(stalled)] }],
|
||||
['session_timeout', { failure_class: 'timeout', exit_reason: 'timeout', sessions: [timedOutSession({ partial: false, max_request_silence_ms: 0 }), { end: 'observer_timeout' }] }],
|
||||
['observer_timeout', { sessions: [{ end: 'observer_timeout' }] }],
|
||||
['assertion', { exit_reason: 'success', error: 'expect(received).toBe(expected)' }],
|
||||
['unknown', { failure_class: 'timeout', error: 'shard wall reached' }],
|
||||
];
|
||||
|
||||
test('one row per cause, in precedence order, and nothing else', () => {
|
||||
expect(rows.map(([cause]) => cause)).toEqual([...FAILURE_CAUSES]);
|
||||
expect(FAILURE_CAUSES).not.toContain('detector');
|
||||
});
|
||||
|
||||
test.each(rows)('%s', (cause, facts) => {
|
||||
expect(causeOf(facts).cause).toBe(cause as any);
|
||||
});
|
||||
|
||||
test('evidence is one bounded line naming what was observed', () => {
|
||||
expect(causeOf(rows[4]![1]).evidence).toBe('no stream event for 130s from 40s (last: thinking_delta); model request in flight, no tool outstanding');
|
||||
expect(causeOf({ exit_reason: 'success' }).evidence).toBe('session completed; check failed');
|
||||
const long = causeOf({ failure_class: 'contract', error: `CONTRACT: ${'x'.repeat(1000)}\nsecond line @team` });
|
||||
expect(long.evidence!.length).toBeLessThanOrEqual(300);
|
||||
expect(long.evidence).not.toContain('second line');
|
||||
});
|
||||
|
||||
test('a stall needs partial messages and a session that did not complete', () => {
|
||||
const quiet = { partial: false, max_request_silence_ms: 0 };
|
||||
expect(causeOf({ failure_class: 'timeout', sessions: [timedOutSession(quiet)] }).cause).toBe('session_timeout');
|
||||
const recovered = { key: 'k', end: 'completed', liveness: { partial: true, max_request_silence_ms: STALL_WINDOW_MS * 2 } };
|
||||
expect(causeOf({ sessions: [recovered] }).cause).toBe('assertion');
|
||||
});
|
||||
});
|
||||
|
||||
describe('liveness on real thinking captures (Claude Code 2.1.284, census 37198445662)', () => {
|
||||
test('the seven partial-message sessions never reach the stall window', () => {
|
||||
expect(thinking.sessions).toHaveLength(7);
|
||||
for (const session of thinking.sessions) {
|
||||
const events: Timed[] = session.events.map(decode);
|
||||
const { liveness } = observe(events, events.at(-1)![0]);
|
||||
expect(liveness.partial).toBe(true);
|
||||
expect(liveness.max_request_silence_ms, session.case).toBeLessThan(STALL_WINDOW_MS);
|
||||
expect(causeOf({ failure_class: 'timeout', sessions: [timedOutSession(liveness)] }).cause).toBe('session_timeout');
|
||||
}
|
||||
});
|
||||
|
||||
const base: Timed[] = thinking.sessions.find((s: any) => s.case === '/review army consensus').events.map(decode);
|
||||
const SILENCE = STALL_WINDOW_MS + 10_000;
|
||||
const cutAfter = (pick: (event: any, i: number) => boolean) => {
|
||||
const at = base.findIndex(([, e], i) => i > 20 && pick(e, i));
|
||||
expect(at).toBeGreaterThan(0);
|
||||
return base.slice(0, at + 1);
|
||||
};
|
||||
|
||||
test('synthetic positive: a 130 s silence inside a model message is a provider stall', () => {
|
||||
const events = cutAfter(e => e.type === 'stream_event' && e.event.delta?.type === 'thinking_delta');
|
||||
const { liveness } = observe(events, events.at(-1)![0] + SILENCE);
|
||||
expect(liveness.max_request_silence_ms).toBe(SILENCE);
|
||||
expect(liveness.silence_after).toBe('thinking_delta');
|
||||
expect(causeOf({ failure_class: 'timeout', exit_reason: 'timeout', sessions: [timedOutSession(liveness)] }).cause).toBe('provider_stall');
|
||||
});
|
||||
|
||||
test('controls: hung tool, hook, permission prompt, subagent and long thinking are not stalls', () => {
|
||||
const toolOpen = cutAfter(e => e.type === 'assistant' && e.message.content.length > 0);
|
||||
const hook: Timed[] = [...cutAfter(e => e.type === 'user'), [0, { type: 'system', subtype: 'hook_started' }]];
|
||||
hook[hook.length - 1]![0] = hook[hook.length - 2]![0];
|
||||
const subagent: Timed[] = [[0, { type: 'stream_event', event: { type: 'message_start' } }],
|
||||
[1000, { type: 'assistant', message: { content: [{ type: 'tool_use', id: 'agent', name: 'Agent' }] } }],
|
||||
[1500, { type: 'stream_event', event: { type: 'message_stop' } }],
|
||||
[2000, { type: 'assistant', parent_tool_use_id: 'agent', message: { content: [{ type: 'tool_use', id: 'child', name: 'Bash' }] } }]];
|
||||
const thinkingLong: Timed[] = [[0, { type: 'stream_event', event: { type: 'message_start' } }],
|
||||
...Array.from({ length: 40 }, (_, i): Timed => [(i + 1) * 9_000, { type: 'stream_event', event: { type: 'content_block_delta', delta: { type: 'thinking_delta' } } }])];
|
||||
const permission = cutAfter(e => e.type === 'user');
|
||||
for (const [name, events, hooks] of [['hung tool', toolOpen, {}], ['hook', hook, {}], ['subagent', subagent, {}],
|
||||
['long thinking', thinkingLong, {}], ['permission', permission, { permissionAt: permission.at(-1)![0] }]] as const) {
|
||||
// Long thinking runs 360 s of deltas 9 s apart and ends at the next one; the others go silent for 130 s.
|
||||
const { liveness } = observe(events as Timed[], (events as Timed[]).at(-1)![0] + (name === 'long thinking' ? 9_000 : SILENCE), { ...hooks });
|
||||
expect(liveness.max_request_silence_ms, name).toBeLessThan(STALL_WINDOW_MS);
|
||||
expect(causeOf({ failure_class: 'timeout', sessions: [timedOutSession(liveness)] }).cause, name).toBe('session_timeout');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('replays of stored census evidence', () => {
|
||||
const byCase = (run: string, c: string, trial = 1) => replays.cases.find((r: any) => r.run === run && r.case === c && r.trial === trial);
|
||||
|
||||
test.each([1, 2])('37195203538 plan-ceo-review-format-approach t%d is a refusal from the structured stream', trial => {
|
||||
const r = byCase('37195203538', 'plan-ceo-review-format-approach', trial);
|
||||
const { observer } = observe(r.events, r.events.at(-1)[0]);
|
||||
const verdict = observer.verdict()!;
|
||||
expect(verdict.end).toBe('refusal');
|
||||
expect(verdict.evidence).toMatch(/^refusal at turn \d+ \(model_refusal_no_fallback\) — reasoning_extraction$/);
|
||||
const cause = causeOf({ exit_reason: r.exit_reason, error: r.error, sessions: [{ key: 'k', end: verdict.end, evidence: verdict.evidence }] });
|
||||
expect(cause.cause).toBe('refusal');
|
||||
});
|
||||
|
||||
test('a tool result quoting refusal text is not a refusal', () => {
|
||||
const observer = new SessionObserver(0);
|
||||
observer.observe({ type: 'assistant', message: { content: [{ type: 'tool_use', id: 't', name: 'Read' }] } }, 1);
|
||||
observer.observe({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 't',
|
||||
content: "API Error: Fable 5.1's safeguards flagged this message. Claude Code can't respond to this message." }] } }, 2);
|
||||
observer.observe({ type: 'result', subtype: 'success', is_error: false, result: 'Quoted the refusal text from docs.' }, 3);
|
||||
expect(observer.verdict()).toBeNull();
|
||||
});
|
||||
|
||||
test('37186854666 shared-libs-review-revalidation ran without partial messages, so it is a session timeout', () => {
|
||||
const r = byCase('37186854666', 'shared-libs-review-revalidation');
|
||||
const { liveness } = observe(r.events, r.duration_ms);
|
||||
expect(liveness.partial).toBe(false);
|
||||
expect(liveness.max_request_silence_ms).toBe(0);
|
||||
expect(causeOf({ failure_class: r.failure_class, exit_reason: r.exit_reason, error: r.error }).cause).toBe('session_timeout');
|
||||
expect(causeOf({ failure_class: r.failure_class, exit_reason: r.exit_reason, error: r.error,
|
||||
sessions: [{ key: 'agent-sdk:x#1', end: 'session_timeout', elapsed_ms: r.duration_ms, budget_ms: 300_000, liveness }] }).cause).toBe('session_timeout');
|
||||
});
|
||||
|
||||
test.each([['37174266054', 'plan-eng-review-artifact'], ['37176837432', 'auto-decide-preserved']])('%s %s is an observer timeout', (run, c) => {
|
||||
const r = byCase(run, c);
|
||||
expect(r.failure_class).toBe('assertion');
|
||||
expect(causeOf({ failure_class: r.failure_class, exit_reason: r.exit_reason, error: r.error }).cause).toBe('observer_timeout');
|
||||
});
|
||||
});
|
||||
|
||||
async function withEvalDir<T>(run: (dir: string) => Promise<T>): Promise<{ value: T; rows: SessionLedgerRow[] }> {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trial-cause-ledger-'));
|
||||
const saved = { dir: process.env.GSTACK_EVAL_DIR, run: process.env.EVALS_RUN_ID, id: process.env.GSTACK_EVAL_CASE_ID };
|
||||
process.env.GSTACK_EVAL_DIR = dir;
|
||||
delete process.env.EVALS_RUN_ID; delete process.env.GSTACK_EVAL_CASE_ID;
|
||||
try {
|
||||
const value = await run(dir);
|
||||
return { value, rows: readSessionLedger(dir) };
|
||||
} finally {
|
||||
for (const [key, name] of [['dir', 'GSTACK_EVAL_DIR'], ['run', 'EVALS_RUN_ID'], ['id', 'GSTACK_EVAL_CASE_ID']] as const) {
|
||||
if (saved[key] === undefined) delete process.env[name]; else process.env[name] = saved[key];
|
||||
}
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
describe('one fixture per runner writes the ledger the classifier reads', () => {
|
||||
test('PTY: the captured API Error idle turn records api_error with its panel line', async () => {
|
||||
const captures = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures', 'pty-idle-turn-end.json'), 'utf8'));
|
||||
const configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'trial-cause-pty-'));
|
||||
try {
|
||||
const { value: obs, rows } = await withEvalDir(async () => {
|
||||
const fake = createFakePtyDriver({ frames: [{ screen: 'Claude Code\n> ' }, { onInput: '/plan-eng-review\r', screen: captures.apiErrorIdle.visible }], configDir });
|
||||
return runPlanSkillObservation({ skillName: 'plan-eng-review', inPlanMode: false, model: 'fake-model', driver: fake.driver,
|
||||
extraArgs: ['--disallowedTools', 'AskUserQuestion'], requireProseEvidence: true, timeoutMs: 300_000 });
|
||||
});
|
||||
expect(obs.outcome).toBe('timeout');
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]).toMatchObject({ runner: 'pty', budget_ms: 300_000, end: 'api_error', evidence: '●API Error: Connection lost mid-response. The response above may be incomplete.' });
|
||||
expect(causeOf({ error: obs.summary, sessions: rows }).cause).toBe('api_error');
|
||||
} finally { fs.rmSync(configDir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('Agent SDK: an abort at the armed budget is a session timeout; partial-message silence is a stall', async () => {
|
||||
__resetSemaphoreForTests(4);
|
||||
const hangingQuery = (events: any[]): QueryProvider => ((args: any) => {
|
||||
const signal: AbortSignal = args.options.abortController.signal;
|
||||
const iterator = (async function* () {
|
||||
for (const event of events) yield event;
|
||||
await new Promise<void>((_, reject) => signal.addEventListener('abort', () => reject(new Error('Claude Code process aborted by user')), { once: true }));
|
||||
})();
|
||||
setTimeout(() => args.options.abortController.abort(), 400);
|
||||
return Object.assign(iterator, { close: () => {} });
|
||||
}) as unknown as QueryProvider;
|
||||
const run = (events: any[]) => withEvalDir(() => runAgentSdkTest({ systemPrompt: '', userPrompt: 'x', workingDirectory: os.tmpdir(), maxRetries: 0,
|
||||
testName: 'sdk-fixture', sessionBudgetMs: 400, streamLiveness: true, queryProvider: hangingQuery(events) }).catch(error => error));
|
||||
const quiet = await run([{ type: 'system', subtype: 'init' }]);
|
||||
expect(quiet.rows).toEqual([expect.objectContaining({ key: 'agent-sdk:sdk-fixture#1', runner: 'agent-sdk', budget_ms: 400, end: 'session_timeout',
|
||||
liveness: expect.objectContaining({ partial: false }) })]);
|
||||
expect(causeOf({ failure_class: 'timeout', exit_reason: 'timeout', sessions: quiet.rows }).cause).toBe('session_timeout');
|
||||
const streaming = await run([{ type: 'system', subtype: 'init' }, { type: 'stream_event', event: { type: 'message_start' } },
|
||||
{ type: 'stream_event', event: { type: 'content_block_delta', delta: { type: 'thinking_delta' } } }]);
|
||||
expect(streaming.rows[0]!.liveness).toMatchObject({ partial: true, silence_after: 'thinking_delta' });
|
||||
expect(failureCauseOf({ failure_class: 'timeout', sessions: streaming.rows }, 300).cause).toBe('provider_stall');
|
||||
expect(failureCauseOf({ failure_class: 'timeout', sessions: streaming.rows }).cause).toBe('session_timeout');
|
||||
});
|
||||
|
||||
test('Codex: an expired budget is a session timeout', async () => {
|
||||
const { rows } = await withEvalDir(() => runRecordedCodexEval({ name: 'codex-fixture', suite: 's', budgetMs: 50, drainGraceMs: 10,
|
||||
run: signal => new Promise((_, reject) => signal.addEventListener('abort', () => reject(signal.reason), { once: true })),
|
||||
validate: () => {}, record: () => {} }).catch(() => undefined));
|
||||
expect(rows).toEqual([expect.objectContaining({ key: 'codex:codex-fixture#1', runner: 'codex', budget_ms: 50, end: 'session_timeout' })]);
|
||||
expect(causeOf({ failure_class: 'assertion', exit_reason: 'timeout', sessions: rows }).cause).toBe('session_timeout');
|
||||
});
|
||||
});
|
||||
|
||||
describe('failureDetailOf (Bun 1.4.0 matcher shapes)', () => {
|
||||
test.each([
|
||||
['expect(received).toBe(expected)\n\nExpected: "b"\nReceived: "a"\n', { expected: '"b"', received: '"a"' }],
|
||||
['expect(received).toContain(expected)\n\nExpected to contain: "xyz"\nReceived: "hello world"\n', { expected: '"xyz"', received: '"hello world"' }],
|
||||
['expect(received).not.toMatch(expected)\n\nExpected substring or pattern: not /taste/\nReceived: "taste call"\n', { expected: 'not /taste/', received: '"taste call"' }],
|
||||
['expect(received).toEqual(expected)\n\n {\n- "a": 2,\n+ "a": 1,\n }\n\n- Expected - 1\n+ Received + 1\n', { expected: '"a": 2,', received: '"a": 1,' }],
|
||||
])('%#', (message, detail) => {
|
||||
expect(failureDetailOf(message)).toEqual(detail);
|
||||
});
|
||||
|
||||
test('the live message shape on this Bun matches the fixtures', () => {
|
||||
let message = '';
|
||||
try { expect('a').toBe('b'); } catch (error) { message = (error as Error).message; }
|
||||
expect(failureDetailOf(message)).toEqual({ expected: '"b"', received: '"a"' });
|
||||
});
|
||||
|
||||
test('a judge red names each failing dimension, its mean, sample count and a rationale', () => {
|
||||
const detail = failureDetailOf('expect(received).toBeGreaterThanOrEqual(expected)\n\nExpected: >= 4\nReceived: 3.6666666666666665\n', {
|
||||
judge_scores: { clarity: 3.6666666666666665, completeness: 4.333333333333333, actionability: 3.6666666666666665 },
|
||||
judge_reasoning: '[sample 1] Thorough. Clarity suffers from extreme density. Two actionability concerns: the budget is infeasible.\n[sample 2] Fine.\n[sample 3] Ok.',
|
||||
});
|
||||
expect(detail).toEqual({ judge: [
|
||||
{ dimension: 'clarity', mean: 3.67, threshold: 4, samples: 3, rationale: 'Clarity suffers from extreme density.' },
|
||||
{ dimension: 'actionability', mean: 3.67, threshold: 4, samples: 3, rationale: 'Two actionability concerns: the budget is infeasible.' },
|
||||
] });
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user